772 files changed, 360205 insertions, 345948 deletions

diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index ea1be844ef4..ef20cbb64fa 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,9 +1,12 @@
-if ENABLE_EXPERIMENTAL
-  EXPERIMENTAL = experimental
+if BUILD_GNFS
+  GNFS_DIR = nfs
 endif
 
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
-          playground meta $(EXPERIMENTAL)
+DIST_SUBDIRS = cluster storage protocol performance debug features \
+          mount nfs mgmt system playground meta
+
+SUBDIRS = cluster storage protocol performance debug features \
+          mount ${GNFS_DIR} mgmt system playground meta
 
 EXTRA_DIST = xlator.sym
 
diff --git a/xlators/cluster/Makefile.am b/xlators/cluster/Makefile.am
index 903fbb39f12..8e067d5ab58 100644
--- a/xlators/cluster/Makefile.am
+++ b/xlators/cluster/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = stripe afr dht ec
+SUBDIRS = afr dht ec
 
 CLEANFILES = 
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index 5612733d3ed..610819b28fc 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,4 +1,4 @@
-xlator_LTLIBRARIES = afr.la pump.la
+xlator_LTLIBRARIES = afr.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
@@ -14,18 +14,15 @@ afr_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 afr_la_SOURCES = $(afr_common_source) $(AFR_SELFHEAL_SOURCES) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-pump_la_SOURCES =  $(afr_common_source) $(AFR_SELFHEAL_SOURCES) pump.c
-pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
 noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
 	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-mem-types.h \
-	afr-common.c afr-self-heald.h pump.h \
+	afr-common.c afr-self-heald.h \
 	$(top_builddir)/xlators/lib/src/libxlator.h afr-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index e59f160db0c..032ab5c8001 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -15,24 +15,20 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-#include "inode.h"
-
-#include "fd.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/hashfn.h>
+#include <glusterfs/list.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/events.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "afr-inode-read.h"
 #include "afr-inode-write.h"
@@ -43,87 +39,799 @@
 #include "afr-self-heald.h"
 #include "afr-messages.h"
 
-call_frame_t *
-afr_copy_frame (call_frame_t *base)
+int32_t
+afr_quorum_errno(afr_private_t *priv)
+{
+    return ENOTCONN;
+}
+
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid)
 {
-	afr_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	int op_errno = 0;
+    if (!__is_root_gfid(pargfid)) {
+        return _gf_false;
+    }
+
+    if (strcmp(name, GF_REPLICATE_TRASH_DIR) == 0) {
+        /*For backward compatibility /.landfill is private*/
+        return _gf_true;
+    }
 
-	frame = copy_frame (base);
-	if (!frame)
-		return NULL;
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local) {
-		AFR_STACK_DESTROY (frame);
-		return NULL;
-	}
+    if (pid == GF_CLIENT_PID_GSYNCD) {
+        /*geo-rep needs to create/sync private directory on slave because
+         * it appears in changelog*/
+        return _gf_false;
+    }
 
-	return frame;
+    if (pid == GF_CLIENT_PID_GLFS_HEAL || pid == GF_CLIENT_PID_SELF_HEALD) {
+        if (strcmp(name, priv->anon_inode_name) == 0) {
+            /* anonymous-inode dir is private*/
+            return _gf_true;
+        }
+    } else {
+        if (strncmp(name, AFR_ANON_DIR_PREFIX, strlen(AFR_ANON_DIR_PREFIX)) ==
+            0) {
+            /* anonymous-inode dir prefix is private for geo-rep to work*/
+            return _gf_true;
+        }
+    }
+
+    return _gf_false;
 }
 
-/* Check if an entry or inode could be undergoing a transaction. */
-gf_boolean_t
-afr_is_possibly_under_txn (afr_transaction_type type, afr_local_t *local,
-                           xlator_t *this)
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies)
 {
-        int i = 0;
-        int tmp = 0;
-	afr_private_t *priv = NULL;
-        GF_UNUSED char *key = NULL;
+    int i = 0;
 
-        priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            replies[i] = 1;
+        } else {
+            replies[i] = 0;
+        }
+    }
+}
 
-        if (type == AFR_ENTRY_TRANSACTION)
-                key = GLUSTERFS_PARENT_ENTRYLK;
-        else if (type == AFR_DATA_TRANSACTION)
-                /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
-                 * pl_inodelk_xattr_fill supports separate keys for different
-                 * domains.*/
-                key = GLUSTERFS_INODELK_COUNT;
+int
+afr_fav_child_reset_sink_xattrs(void *opaque);
 
+int
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *frame, void *opaque);
+
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this);
+
+int
+afr_dom_lock_acquire_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
+
+    local->cont.lk.dom_lock_op_ret[i] = op_ret;
+    local->cont.lk.dom_lock_op_errno[i] = op_errno;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to acquire %s on %s",
+               uuid_utoa(local->fd->inode->gfid), AFR_LK_HEAL_DOM,
+               priv->children[i]->name);
+    } else {
+        local->cont.lk.dom_locked_nodes[i] = 1;
+    }
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
+
+int
+afr_dom_lock_acquire(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+
+    priv = frame->this->private;
+    local = frame->local;
+    local->cont.lk.dom_locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+    if (!local->cont.lk.dom_locked_nodes) {
+        return -ENOMEM;
+    }
+    local->cont.lk.dom_lock_op_ret = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_ret),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_ret) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    local->cont.lk.dom_lock_op_errno = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_errno),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_errno) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    flock.l_type = F_WRLCK;
+
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLK, &flock, NULL);
+
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL))
+        goto blocking_lock;
+
+    /*If any of the bricks returned EAGAIN, we still need blocking locks.*/
+    if (AFR_COUNT(local->cont.lk.dom_locked_nodes, priv->child_count) !=
+        priv->child_count) {
         for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].xdata)
-			continue;
-		if (dict_get_int32 (local->replies[i].xdata, key, &tmp) == 0)
-			if (tmp)
-				return _gf_true;
-	}
+            if (local->cont.lk.dom_lock_op_ret[i] == -1 &&
+                local->cont.lk.dom_lock_op_errno[i] == EAGAIN)
+                goto blocking_lock;
+        }
+    }
 
-        return _gf_false;
+    return 0;
+
+blocking_lock:
+    afr_dom_lock_release(frame);
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLKW, &flock, NULL);
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL)) {
+        afr_dom_lock_release(frame);
+        return -afr_quorum_errno(priv);
+    }
+
+    return 0;
 }
 
 int
-__afr_inode_ctx_get (xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+afr_dom_lock_release_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        uint64_t                ctx_int = 0;
-        int                     ret     = -1;
-        afr_inode_ctx_t        *tmp_ctx = NULL;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
 
-        ret = __inode_ctx_get (inode, this, &ctx_int);
-        if (ret) {
-                tmp_ctx = GF_CALLOC (1, sizeof (afr_inode_ctx_t),
-                                     gf_afr_mt_inode_ctx_t);
-                if (!tmp_ctx)
-                        goto out;
-
-                ctx_int = (long) tmp_ctx;
-                ret = __inode_ctx_set (inode, this, &ctx_int);
-                if (ret) {
-                        GF_FREE (tmp_ctx);
-                        goto out;
-                }
-                tmp_ctx->spb_choice = -1;
-                tmp_ctx->read_subvol = 0;
-        } else {
-                tmp_ctx = (afr_inode_ctx_t *) ctx_int;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to release %s on %s", local->loc.path,
+               AFR_LK_HEAL_DOM, priv->children[i]->name);
+    }
+    local->cont.lk.dom_locked_nodes[i] = 0;
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
+
+void
+afr_dom_lock_release(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+    priv = frame->this->private;
+    locked_on = local->cont.lk.dom_locked_nodes;
+    if (AFR_COUNT(locked_on, priv->child_count) == 0)
+        return;
+    flock.l_type = F_UNLCK;
+
+    AFR_ONLIST(locked_on, frame, afr_dom_lock_release_cbk, finodelk,
+               AFR_LK_HEAL_DOM, local->fd, F_SETLK, &flock, NULL);
+
+    return;
+}
+
+static void
+afr_lk_heal_info_cleanup(afr_lk_heal_info_t *info)
+{
+    if (!info)
+        return;
+    if (info->xdata_req)
+        dict_unref(info->xdata_req);
+    if (info->fd)
+        fd_unref(info->fd);
+    GF_FREE(info->locked_nodes);
+    GF_FREE(info->child_up_event_gen);
+    GF_FREE(info->child_down_event_gen);
+    GF_FREE(info);
+}
+
+static int
+afr_add_lock_to_saved_locks(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -ENOMEM;
+
+    info = GF_CALLOC(sizeof(*info), 1, gf_afr_mt_lk_heal_info_t);
+    if (!info) {
+        goto cleanup;
+    }
+    INIT_LIST_HEAD(&info->pos);
+    info->fd = fd_ref(local->fd);
+    info->cmd = local->cont.lk.cmd;
+    info->pid = frame->root->pid;
+    info->flock = local->cont.lk.user_flock;
+    info->xdata_req = dict_copy_with_ref(local->xdata_req, NULL);
+    if (!info->xdata_req) {
+        goto cleanup;
+    }
+    info->lk_owner = frame->root->lk_owner;
+    info->locked_nodes = GF_MALLOC(
+        sizeof(*info->locked_nodes) * priv->child_count, gf_afr_mt_char);
+    if (!info->locked_nodes) {
+        goto cleanup;
+    }
+    memcpy(info->locked_nodes, local->cont.lk.locked_nodes,
+           sizeof(*info->locked_nodes) * priv->child_count);
+    info->child_up_event_gen = GF_CALLOC(sizeof(*info->child_up_event_gen),
+                                         priv->child_count, gf_afr_mt_int32_t);
+    if (!info->child_up_event_gen) {
+        goto cleanup;
+    }
+    info->child_down_event_gen = GF_CALLOC(sizeof(*info->child_down_event_gen),
+                                           priv->child_count,
+                                           gf_afr_mt_int32_t);
+    if (!info->child_down_event_gen) {
+        goto cleanup;
+    }
+
+    LOCK(&local->fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(local->fd, this);
+        if (fd_ctx)
+            fd_ctx->lk_heal_info = info;
+    }
+    UNLOCK(&local->fd->lock);
+    if (!fd_ctx) {
+        goto cleanup;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_add_tail(&info->pos, &priv->saved_locks);
+    }
+    UNLOCK(&priv->lock);
+
+    return 0;
+cleanup:
+    gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+           "%s: Failed to add lock to healq",
+           uuid_utoa(local->fd->inode->gfid));
+    if (info) {
+        afr_lk_heal_info_cleanup(info);
+        if (fd_ctx) {
+            LOCK(&local->fd->lock);
+            {
+                fd_ctx->lk_heal_info = NULL;
+            }
+            UNLOCK(&local->fd->lock);
         }
+    }
+    return ret;
+}
 
-        *ctx = tmp_ctx;
-        ret = 0;
+static int
+afr_remove_lock_from_saved_locks(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = local->cont.lk.user_flock;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -EINVAL;
+
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx || !fd_ctx->lk_heal_info) {
+        goto out;
+    }
+
+    info = fd_ctx->lk_heal_info;
+    if ((info->flock.l_start != flock.l_start) ||
+        (info->flock.l_whence != flock.l_whence) ||
+        (info->flock.l_len != flock.l_len)) {
+        /*TODO: Compare lkowners too.*/
+        goto out;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_del(&fd_ctx->lk_heal_info->pos);
+    }
+    UNLOCK(&priv->lock);
+
+    afr_lk_heal_info_cleanup(info);
+    fd_ctx->lk_heal_info = NULL;
+    ret = 0;
 out:
-        return ret;
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to remove lock from healq",
+               uuid_utoa(local->fd->inode->gfid));
+    return ret;
 }
+
+int
+afr_lock_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed to heal lock on child %d for %s", i,
+               uuid_utoa(local->fd->inode->gfid));
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+int
+afr_getlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed getlk for %s", uuid_utoa(local->fd->inode->gfid));
+    } else {
+        local->cont.lk.getlk_rsp[i] = *lock;
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+static gf_boolean_t
+afr_does_lk_owner_match(call_frame_t *frame, afr_private_t *priv,
+                        afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    afr_local_t *local = frame->local;
+    struct gf_flock flock = {
+        0,
+    };
+    gf_boolean_t ret = _gf_true;
+    char *wind_on = alloca0(priv->child_count);
+    unsigned char *success_replies = alloca0(priv->child_count);
+    local->cont.lk.getlk_rsp = GF_CALLOC(sizeof(*local->cont.lk.getlk_rsp),
+                                         priv->child_count, gf_afr_mt_gf_lock);
+
+    flock = info->flock;
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            wind_on[i] = 1;
+    }
+
+    AFR_ONLIST(wind_on, frame, afr_getlk_cbk, lk, info->fd, F_GETLK, &flock,
+               info->xdata_req);
+
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) == 0) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret != 0)
+            continue;
+        if (local->cont.lk.getlk_rsp[i].l_type == F_UNLCK)
+            continue;
+        /*TODO: Do we really need to compare lkowner if F_UNLCK is true?*/
+        if (!is_same_lkowner(&local->cont.lk.getlk_rsp[i].l_owner,
+                             &info->lk_owner)) {
+            ret = _gf_false;
+            break;
+        }
+    }
+out:
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->cont.lk.getlk_rsp);
+    local->cont.lk.getlk_rsp = NULL;
+    return ret;
+}
+
+static void
+afr_mark_fd_bad(fd_t *fd, xlator_t *this)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    if (!fd)
+        return;
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+        if (fd_ctx) {
+            fd_ctx->is_fd_bad = _gf_true;
+            fd_ctx->lk_heal_info = NULL;
+        }
+    }
+    UNLOCK(&fd->lock);
+}
+
+static void
+afr_add_lock_to_lkhealq(afr_private_t *priv, afr_lk_heal_info_t *info)
+{
+    LOCK(&priv->lock);
+    {
+        list_del(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+    UNLOCK(&priv->lock);
+}
+
+static void
+afr_lock_heal_do(call_frame_t *frame, afr_private_t *priv,
+                 afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    int op_errno = 0;
+    int32_t *current_event_gen = NULL;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    char *wind_on = alloca0(priv->child_count);
+    gf_boolean_t retry = _gf_true;
+
+    frame->root->pid = info->pid;
+    lk_owner_copy(&frame->root->lk_owner, &info->lk_owner);
+
+    op_errno = -afr_dom_lock_acquire(frame);
+    if ((op_errno != 0)) {
+        goto release;
+    }
+
+    if (!afr_does_lk_owner_match(frame, priv, info)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_LK_HEAL_DOM,
+               "Ignoring lock heal for %s since lk-onwers mismatch. "
+               "Lock possibly pre-empted by another client.",
+               uuid_utoa(info->fd->inode->gfid));
+        goto release;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            continue;
+        wind_on[i] = 1;
+    }
+
+    current_event_gen = alloca(priv->child_count);
+    memcpy(current_event_gen, info->child_up_event_gen,
+           priv->child_count * sizeof *current_event_gen);
+    AFR_ONLIST(wind_on, frame, afr_lock_heal_cbk, lk, info->fd, info->cmd,
+               &info->flock, info->xdata_req);
+
+    LOCK(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if ((!local->replies[i].valid) || (local->replies[i].op_ret != 0)) {
+                continue;
+            }
+
+            if ((current_event_gen[i] == info->child_up_event_gen[i]) &&
+                (current_event_gen[i] > info->child_down_event_gen[i])) {
+                info->locked_nodes[i] = 1;
+                retry = _gf_false;
+                list_del_init(&info->pos);
+                list_add_tail(&info->pos, &priv->saved_locks);
+            } else {
+                /*We received subsequent child up/down events while heal was in
+                 * progress; don't mark child as healed. Attempt again on the
+                 * new child up*/
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_LK_HEAL_DOM,
+                       "Event gen mismatch: skipped healing lock on child %d "
+                       "for %s.",
+                       i, uuid_utoa(info->fd->inode->gfid));
+            }
+        }
+    }
+    UNLOCK(&priv->lock);
+
+release:
+    afr_dom_lock_release(frame);
+    if (retry)
+        afr_add_lock_to_lkhealq(priv, info);
+    return;
+}
+
+static int
+afr_lock_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+
+static int
+afr_lock_heal(void *opaque)
+{
+    call_frame_t *frame = (call_frame_t *)opaque;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+    struct list_head healq = {
+        0,
+    };
+    int ret = 0;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        return ENOMEM;
+    }
+
+    INIT_LIST_HEAD(&healq);
+    LOCK(&priv->lock);
+    {
+        list_splice_init(&priv->lk_healq, &healq);
+    }
+    UNLOCK(&priv->lock);
+
+    list_for_each_entry_safe(info, tmp, &healq, pos)
+    {
+        GF_ASSERT((AFR_COUNT(info->locked_nodes, priv->child_count) <
+                   priv->child_count));
+        ((afr_local_t *)(iter_frame->local))->fd = fd_ref(info->fd);
+        afr_lock_heal_do(iter_frame, priv, info);
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = ENOTCONN;
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOTCONN,
+                   AFR_MSG_LK_HEAL_DOM,
+                   "Aborting processing of lk_healq."
+                   "Healing will be reattempted on next child up for locks "
+                   "that are still in quorum.");
+            LOCK(&priv->lock);
+            {
+                list_add_tail(&healq, &priv->lk_healq);
+            }
+            UNLOCK(&priv->lock);
+            break;
+        }
+    }
+
+    AFR_STACK_DESTROY(iter_frame);
+    return ret;
+}
+
+static int
+__afr_lock_heal_synctask(xlator_t *this, afr_private_t *priv, int child)
+{
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+
+    if (priv->shd.iamshd)
+        return 0;
+
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_up_event_gen[child] = priv->event_generation;
+        list_del_init(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return -1;
+
+    ret = synctask_new(this->ctx->env, afr_lock_heal, afr_lock_heal_done, frame,
+                       frame);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_LK_HEAL_DOM,
+               "Failed to launch lock heal synctask");
+
+    return ret;
+}
+
+static int
+__afr_mark_pending_lk_heal(xlator_t *this, afr_private_t *priv, int child)
+{
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+
+    if (priv->shd.iamshd)
+        return 0;
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_down_event_gen[child] = priv->event_generation;
+        if (info->locked_nodes[child] == 1)
+            info->locked_nodes[child] = 0;
+        if (!afr_has_quorum(info->locked_nodes, this, NULL)) {
+            /* Since the lock was lost on quorum no. of nodes, we should
+             * not attempt to heal it anymore. Some other client could have
+             * acquired the lock, modified data and released it and this
+             * client wouldn't know about it if we heal it.*/
+            afr_mark_fd_bad(info->fd, this);
+            list_del(&info->pos);
+            afr_lk_heal_info_cleanup(info);
+            /* We're not winding an unlock on the node where the lock is still
+             * present because when fencing logic switches over to the new
+             * client (since we marked the fd bad), it should preempt any
+             * existing lock. */
+        }
+    }
+    return 0;
+}
+
+gf_boolean_t
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno)
+{
+    if (priv->consistent_io && local->call_count != priv->child_count) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are not up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        return _gf_false;
+    }
+    return _gf_true;
+}
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata)
+{
+    int ret = 0;
+    uint32_t lk_mode = GF_LK_ADVISORY;
+
+    ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_mode);
+    if (!ret && lk_mode == GF_LK_MANDATORY)
+        return _gf_true;
+
+    return _gf_false;
+}
+
+call_frame_t *
+afr_copy_frame(call_frame_t *base)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int op_errno = 0;
+
+    frame = copy_frame(base);
+    if (!frame)
+        return NULL;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        AFR_STACK_DESTROY(frame);
+        return NULL;
+    }
+
+    return frame;
+}
+
+/* Check if an entry or inode could be undergoing a transaction. */
+gf_boolean_t
+afr_is_possibly_under_txn(afr_transaction_type type, afr_local_t *local,
+                          xlator_t *this)
+{
+    int i = 0;
+    int tmp = 0;
+    afr_private_t *priv = NULL;
+    GF_UNUSED char *key = NULL;
+    int keylen = 0;
+
+    priv = this->private;
+
+    if (type == AFR_ENTRY_TRANSACTION) {
+        key = GLUSTERFS_PARENT_ENTRYLK;
+        keylen = SLEN(GLUSTERFS_PARENT_ENTRYLK);
+    } else if (type == AFR_DATA_TRANSACTION) {
+        /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
+         * pl_inodelk_xattr_fill supports separate keys for different
+         * domains.*/
+        key = GLUSTERFS_INODELK_COUNT;
+        keylen = SLEN(GLUSTERFS_INODELK_COUNT);
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].xdata)
+            continue;
+        if (dict_get_int32n(local->replies[i].xdata, key, keylen, &tmp) == 0)
+            if (tmp)
+                return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static void
+afr_inode_ctx_destroy(afr_inode_ctx_t *ctx)
+{
+    int i = 0;
+
+    if (!ctx)
+        return;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        GF_FREE(ctx->pre_op_done[i]);
+    }
+
+    GF_FREE(ctx);
+}
+
+int
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+{
+    uint64_t ctx_int = 0;
+    int ret = -1;
+    int i = -1;
+    int num_locks = -1;
+    afr_inode_ctx_t *ictx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_private_t *priv = this->private;
+
+    ret = __inode_ctx_get(inode, this, &ctx_int);
+    if (ret == 0) {
+        *ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+        return 0;
+    }
+
+    ictx = GF_CALLOC(1, sizeof(afr_inode_ctx_t), gf_afr_mt_inode_ctx_t);
+    if (!ictx)
+        goto out;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        ictx->pre_op_done[i] = GF_CALLOC(sizeof *ictx->pre_op_done[i],
+                                         priv->child_count, gf_afr_mt_int32_t);
+        if (!ictx->pre_op_done[i]) {
+            ret = -ENOMEM;
+            goto out;
+        }
+    }
+
+    num_locks = sizeof(ictx->lock) / sizeof(afr_lock_t);
+    for (i = 0; i < num_locks; i++) {
+        lock = &ictx->lock[i];
+        INIT_LIST_HEAD(&lock->post_op);
+        INIT_LIST_HEAD(&lock->frozen);
+        INIT_LIST_HEAD(&lock->waiting);
+        INIT_LIST_HEAD(&lock->owners);
+    }
+
+    ctx_int = (uint64_t)(uintptr_t)ictx;
+    ret = __inode_ctx_set(inode, this, &ctx_int);
+    if (ret) {
+        goto out;
+    }
+
+    ictx->spb_choice = -1;
+    ictx->read_subvol = 0;
+    ictx->write_subvol = 0;
+    ictx->lock_count = 0;
+    ret = 0;
+    *ctx = ictx;
+out:
+    if (ret) {
+        afr_inode_ctx_destroy(ictx);
+    }
+    return ret;
+}
+
 /*
  * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
  *
@@ -155,1672 +863,2258 @@ out:
  */
 
 int
-__afr_inode_read_subvol_get_small (inode_t *inode, xlator_t *this,
-				   unsigned char *data, unsigned char *metadata,
-				   int *event_p)
-{
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint32_t event = 0;
-	uint64_t val = 0;
-	int i = 0;
-        afr_inode_ctx_t *ctx = NULL;
+__afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local,
+                              inode_t *inode)
+{
+    int i = 0;
+    int txn_type = 0;
+    int count = 0;
+    int index = -1;
+    uint16_t datamap_old = 0;
+    uint16_t metadatamap_old = 0;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint16_t tmp_map = 0;
+    uint16_t mask = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    txn_type = local->transaction.type;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        val = local->inode_ctx->write_subvol;
+    else
+        val = local->inode_ctx->read_subvol;
+
+    metadatamap_old = metadatamap = (val & 0x000000000000ffff);
+    datamap_old = datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        tmp_map = datamap;
+    else if (txn_type == AFR_METADATA_TRANSACTION)
+        tmp_map = metadatamap;
+
+    count = gf_bits_count(tmp_map);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i])
+            continue;
+
+        mask = 1 << i;
+        if (txn_type == AFR_METADATA_TRANSACTION)
+            metadatamap &= ~mask;
+        else if (txn_type == AFR_DATA_TRANSACTION)
+            datamap &= ~mask;
+    }
+
+    switch (txn_type) {
+        case AFR_METADATA_TRANSACTION:
+            if ((metadatamap_old != 0) && (metadatamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                metadatamap |= (1 << index);
+            }
+            if (metadatamap_old != metadatamap) {
+                __afr_inode_need_refresh_set(inode, this);
+            }
+            break;
+
+        case AFR_DATA_TRANSACTION:
+            if ((datamap_old != 0) && (datamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                datamap |= (1 << index);
+            }
+            if (datamap_old != datamap)
+                __afr_inode_need_refresh_set(inode, this);
+            break;
+
+        default:
+            break;
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        local->inode_ctx->write_subvol = val;
+    local->inode_ctx->read_subvol = val;
+
+    return 0;
+}
 
-	priv = this->private;
+gf_boolean_t
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int op_errno = 0;
+    int i_errno = 0;
+    gf_boolean_t matching_errors = _gf_true;
+    int i = 0;
 
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-	if (ret < 0)
-		return ret;
+    priv = this->private;
+    local = frame->local;
 
-        val = ctx->read_subvol;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret != -1) {
+            /* Operation succeeded on at least one subvol,
+               so it is not a failed-everywhere situation.
+            */
+            matching_errors = _gf_false;
+            break;
+        }
+        i_errno = local->replies[i].op_errno;
 
-	metadatamap = (val & 0x000000000000ffff);
-	datamap =     (val & 0x00000000ffff0000) >> 16;
-	event =       (val & 0xffffffff00000000) >> 32;
+        if (i_errno == ENOTCONN) {
+            /* ENOTCONN is not a symmetric error. We do not
+               know if the operation was performed on the
+               backend or not.
+            */
+            matching_errors = _gf_false;
+            break;
+        }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (metadata)
-			metadata[i] = (metadatamap >> i) & 1;
-		if (data)
-			data[i] = (datamap >> i) & 1;
-	}
+        if (!op_errno) {
+            op_errno = i_errno;
+        } else if (op_errno != i_errno) {
+            /* Mismatching op_errno's */
+            matching_errors = _gf_false;
+            break;
+        }
+    }
 
-	if (event_p)
-		*event_p = event;
-	return ret;
+    return matching_errors;
 }
 
-
 int
-__afr_inode_read_subvol_set_small (inode_t *inode, xlator_t *this,
-				   unsigned char *data, unsigned char *metadata,
-				   int event)
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame, inode_t *inode)
 {
-	afr_private_t *priv = NULL;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint64_t val = 0;
-	int i = 0;
-        int ret = -1;
-        afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-	priv = this->private;
+    priv = this->private;
+    local = frame->local;
 
-        ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                goto out;
+    /* If this transaction saw no failures, then exit. */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) == 0)
+        return 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (data[i])
-			datamap |= (1 << i);
-		if (metadata[i])
-			metadatamap |= (1 << i);
-	}
+    if (afr_is_symmetric_error(frame, this))
+        return 0;
 
-	val = ((uint64_t) metadatamap) |
-		(((uint64_t) datamap) << 16) |
-		(((uint64_t) event) << 32);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_set_in_flight_sb_status(this, local, inode);
+    }
+    UNLOCK(&inode->lock);
 
-        ctx->read_subvol = val;
+    return ret;
+}
 
-        ret = 0;
-out:
+int
+__afr_inode_read_subvol_get_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int *event_p)
+{
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    int i = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    priv = this->private;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
         return ret;
+
+    val = ctx->read_subvol;
+
+    metadatamap = (val & 0x000000000000ffff);
+    datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (metadata)
+            metadata[i] = (metadatamap >> i) & 1;
+        if (data)
+            data[i] = (datamap >> i) & 1;
+    }
+
+    if (event_p)
+        *event_p = event;
+    return ret;
 }
 
 int
-__afr_inode_read_subvol_reset_small (inode_t *inode, xlator_t *this)
+__afr_inode_read_subvol_set_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int event)
 {
-	int ret = -1;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint32_t event = 0;
-	uint64_t val = 0;
-        afr_inode_ctx_t *ctx = NULL;
+    afr_private_t *priv = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int i = 0;
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
 
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                return ret;
+    priv = this->private;
 
-        val = ctx->read_subvol;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-	metadatamap = (val & 0x000000000000ffff) >> 0;
-	datamap =     (val & 0x00000000ffff0000) >> 16;
-	event = 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (data[i])
+            datamap |= (1 << i);
+        if (metadata[i])
+            metadatamap |= (1 << i);
+    }
 
-	val = ((uint64_t) metadatamap) |
-		(((uint64_t) datamap) << 16) |
-		(((uint64_t) event) << 32);
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
 
-        ctx->read_subvol = val;
+    ctx->read_subvol = val;
 
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
 
-
 int
-__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			     unsigned char *data, unsigned char *metadata,
-			     int *event_p)
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int *event_p)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-	priv = this->private;
+    priv = this->private;
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_get_small (inode, this, data,
-							 metadata, event_p);
-	else
-		/* TBD: allocate structure with array and read from it */
-		ret = -1;
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_get_small(inode, this, data, metadata,
+                                                event_p);
+    else
+        /* TBD: allocate structure with array and read from it */
+        ret = -1;
 
-	return ret;
+    return ret;
 }
 
 int
-__afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-			            int *spb_choice)
+__afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                   int *spb_choice)
 {
-        afr_inode_ctx_t *ctx = NULL;
-        int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret < 0)
-                return ret;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-        *spb_choice = ctx->spb_choice;
-        return 0;
+    *spb_choice = ctx->spb_choice;
+    return 0;
 }
 
 int
-__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
-			     unsigned char *metadata, int event)
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int event)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-	priv = this->private;
+    priv = this->private;
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_set_small (inode, this, data,
-							 metadata, event);
-	else
-		ret = -1;
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_set_small(inode, this, data, metadata,
+                                                event);
+    else
+        ret = -1;
 
-	return ret;
+    return ret;
 }
 
 int
-__afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                    int spb_choice)
+__afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                   int spb_choice)
 {
-        afr_inode_ctx_t *ctx = NULL;
-	int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                goto out;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-        ctx->spb_choice = spb_choice;
+    ctx->spb_choice = spb_choice;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-__afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int *event_p)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-
-	priv = this->private;
+    int ret = -1;
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_reset_small (inode, this);
-	else
-		ret = -1;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-	return ret;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_get(inode, this, data, metadata, event_p);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
 }
 
-
 int
-afr_inode_read_subvol_get (inode_t *inode, xlator_t *this, unsigned char *data,
-			   unsigned char *metadata, int *event_p)
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type)
 {
-	int ret = -1;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    unsigned char *data = alloca0(priv->child_count);
+    unsigned char *metadata = alloca0(priv->child_count);
+    int data_count = 0;
+    int metadata_count = 0;
+    int event_generation = 0;
+    int ret = 0;
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        return -EIO;
+
+    data_count = AFR_COUNT(data, priv->child_count);
+    metadata_count = AFR_COUNT(metadata, priv->child_count);
+
+    if (inode->ia_type == IA_IFDIR) {
+        /* For directories, allow even if it is in data split-brain. */
+        if (type == AFR_METADATA_TRANSACTION || local->op == GF_FOP_STAT ||
+            local->op == GF_FOP_FSTAT) {
+            if (!metadata_count)
+                return -EIO;
+        }
+    } else {
+        /* For files, abort in case of data/metadata split-brain. */
+        if (!data_count || !metadata_count) {
+            return -EIO;
+        }
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    if (type == AFR_METADATA_TRANSACTION && readable)
+        memcpy(readable, metadata, priv->child_count * sizeof *metadata);
+    if (type == AFR_DATA_TRANSACTION && readable) {
+        if (!data_count)
+            memcpy(readable, local->child_up,
+                   priv->child_count * sizeof *readable);
+        else
+            memcpy(readable, data, priv->child_count * sizeof *data);
+    }
+    if (event_p)
+        *event_p = event_generation;
+    return 0;
+}
 
-        LOCK(&inode->lock);
-	{
-		ret = __afr_inode_read_subvol_get (inode, this, data,
-						   metadata, event_p);
-	}
-	UNLOCK(&inode->lock);
+static int
+afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                 int *spb_choice)
+{
+    int ret = -1;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_get(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
+/*
+ * frame is used to get the favourite policy. Since
+ * afr_inode_split_brain_choice_get was called with afr_open, it is possible to
+ * have a frame with out local->replies. So in that case, frame is passed as
+ * null, hence this function will handle the frame NULL case.
+ */
 int
-afr_inode_get_readable (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                       unsigned char *readable, int *event_p, int type)
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol)
 {
+    int ret = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        afr_private_t *priv = this->private;
-        afr_local_t *local = frame->local;
-        unsigned char *data = alloca0 (priv->child_count);
-        unsigned char *metadata = alloca0 (priv->child_count);
-        int data_count = 0;
-        int metadata_count = 0;
-        int event_generation = 0;
-        int ret = 0;
+    GF_VALIDATE_OR_GOTO("afr", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, spb_subvol, out);
 
-        ret = afr_inode_read_subvol_get (inode, this, data, metadata,
-                                         &event_generation);
-        if (ret == -1)
-                return -EIO;
+    priv = this->private;
 
-        data_count = AFR_COUNT (data, priv->child_count);
-        metadata_count = AFR_COUNT (metadata, priv->child_count);
+    ret = afr_inode_split_brain_choice_get(inode, this, spb_subvol);
+    if (*spb_subvol < 0 && priv->fav_child_policy && frame && frame->local) {
+        local = frame->local;
+        *spb_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (*spb_subvol >= 0) {
+            ret = 0;
+        }
+    }
 
-        if (inode->ia_type == IA_IFDIR) {
-                /* For directories, allow even if it is in data split-brain. */
-                if (type == AFR_METADATA_TRANSACTION ||
-                    local->op == GF_FOP_STAT || local->op == GF_FOP_FSTAT) {
-                        if (!metadata_count)
-                                return -EIO;
-                }
-        } else {
-                /* For files, abort in case of data/metadata split-brain. */
-                if (!data_count || !metadata_count)
-                        return -EIO;
-        }
-
-        if (type == AFR_METADATA_TRANSACTION && readable)
-                memcpy (readable, metadata, priv->child_count * sizeof *metadata);
-        if (type == AFR_DATA_TRANSACTION && readable) {
-                if (!data_count)
-                        memcpy (readable, local->child_up,
-                                priv->child_count * sizeof *readable);
-                else
-                        memcpy (readable, data, priv->child_count * sizeof *data);
-        }
-        if (event_p)
-                *event_p = event_generation;
-        return 0;
+out:
+    return ret;
 }
-
 int
-afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-                                  int *spb_choice)
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int event)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-	LOCK(&inode->lock);
-	{
-		ret = __afr_inode_split_brain_choice_get (inode, this,
-                                                          spb_choice);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_set(inode, this, data, metadata, event);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
-			   unsigned char *metadata, int event)
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this, int spb_choice)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-	{
-		ret = __afr_inode_read_subvol_set (inode, this, data, metadata,
-						   event);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_set(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                  int spb_choice)
+/* The caller of this should perform afr_inode_refresh, if this function
+ * returns _gf_true
+ */
+gf_boolean_t
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2)
 {
-	int ret = -1;
+    gf_boolean_t need_refresh = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-	LOCK(&inode->lock);
-	{
-		ret = __afr_inode_split_brain_choice_set (inode, this,
-                                                          spb_choice);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret)
+            goto unlock;
+
+        need_refresh = ctx->need_refresh;
+        /* Hoping that the caller will do inode_refresh followed by
+         * this, hence setting the need_refresh to false */
+        ctx->need_refresh = _gf_false;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+
+    if (event_gen1 != event_gen2)
+        need_refresh = _gf_true;
 out:
-	return ret;
+    return need_refresh;
 }
 
+int
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret == 0) {
+        ctx->need_refresh = _gf_true;
+    }
+
+    return ret;
+}
 
 int
-afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_read_subvol_reset (inode, this);
-        }
-        UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_need_refresh_set(inode, this);
+    }
+    UNLOCK(&inode->lock);
 out:
-	return ret;
+    return ret;
 }
 
 int
-afr_spb_choice_timeout_cancel (xlator_t *this, inode_t *inode)
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode)
 {
-        afr_inode_ctx_t *ctx    = NULL;
-        int              ret    = -1;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        if (!inode)
-                return ret;
+    if (!inode)
+        return ret;
 
-        LOCK(&inode->lock);
-        {
-                __afr_inode_ctx_get (this, inode, &ctx);
-                if (!ctx) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                                "Failed to cancel split-brain choice timer.");
-                        goto out;
-                }
-                ctx->spb_choice = -1;
-                if (ctx->timer) {
-                        gf_timer_call_cancel (this->ctx, ctx->timer);
-                        ctx->timer = NULL;
-                }
-                ret = 0;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret < 0 || !ctx) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to cancel split-brain choice timer.");
+            goto out;
+        }
+        ctx->spb_choice = -1;
+        if (ctx->timer) {
+            gf_timer_call_cancel(this->ctx, ctx->timer);
+            ctx->timer = NULL;
         }
+        ret = 0;
+    }
+    UNLOCK(&inode->lock);
 out:
-        UNLOCK(&inode->lock);
-        return ret;
+    return ret;
 }
 
 void
-afr_set_split_brain_choice_cbk (void *data)
+afr_set_split_brain_choice_cbk(void *data)
 {
-        inode_t      *inode     = data;
-        xlator_t     *this      = THIS;
+    inode_t *inode = data;
+    xlator_t *this = THIS;
 
-        afr_spb_choice_timeout_cancel (this, inode);
-        inode_unref (inode);
-        return;
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_invalidate(inode);
+    inode_unref(inode);
+    return;
 }
 
-
 int
-afr_set_split_brain_choice (int ret, call_frame_t *frame, void *opaque)
-{
-        int     op_errno         = ENOMEM;
-        afr_private_t *priv      = NULL;
-        afr_inode_ctx_t *ctx     = NULL;
-        inode_t *inode           = NULL;
-        loc_t   *loc             = NULL;
-        xlator_t *this           = NULL;
-        afr_spbc_timeout_t *data = opaque;
-        struct timespec delta    = {0, };
-
-        if (ret)
-                goto out;
-
-        frame = data->frame;
-        loc = data->loc;
-        this = frame->this;
-        priv = this->private;
-
-        delta.tv_sec = priv->spb_choice_timeout;
-        delta.tv_nsec = 0;
-
-        inode = loc->inode;
-        if (!inode)
-                goto out;
-
-        if (!(data->d_spb || data->m_spb)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, "Cannot set "
-                        "replica.split-brain-choice on %s. File is"
-                        " not in data/metadata split-brain.",
-                        uuid_utoa (loc->gfid));
-                ret = -1;
-                op_errno = EINVAL;
-                goto out;
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque)
+{
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    inode_t *inode = NULL;
+    loc_t *loc = NULL;
+    xlator_t *this = NULL;
+    afr_spbc_timeout_t *data = opaque;
+    struct timespec delta = {
+        0,
+    };
+    gf_boolean_t timer_set = _gf_false;
+    gf_boolean_t timer_cancelled = _gf_false;
+    gf_boolean_t timer_reset = _gf_false;
+    int old_spb_choice = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+    priv = this->private;
+
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    delta.tv_sec = priv->spb_choice_timeout;
+    delta.tv_nsec = 0;
+
+    if (!loc->inode) {
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (!(data->d_spb || data->m_spb)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+               "Cannot set "
+               "replica.split-brain-choice on %s. File is"
+               " not in data/metadata split-brain.",
+               uuid_utoa(loc->gfid));
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /*
+     * we're ref'ing the inode before LOCK like it is done elsewhere in the
+     * code. If we ref after LOCK, coverity complains of possible deadlocks.
+     */
+    inode = inode_ref(loc->inode);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to get inode_ctx for %s", loc->name);
+            goto post_unlock;
         }
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_ctx_get (this, inode, &ctx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                                "Failed to get inode_ctx for %s", loc->name);
-                        goto unlock;
-                }
-
-                ctx->spb_choice = data->spb_child_index;
+        old_spb_choice = ctx->spb_choice;
+        ctx->spb_choice = data->spb_child_index;
 
-                /* Possible changes in spb-choice :
-                 *         -1 to valid    : ref and inject timer
-                 *
-                 *         valid to valid : cancel timer and inject new one
-                 *
-                 *         valid to -1    : cancel timer and unref
-                 *
-                 *         -1    to -1    : do not do anything
-                 */
+        /* Possible changes in spb-choice :
+         *         valid to -1    : cancel timer and unref
+         *         valid to valid : cancel timer and inject new one
+         *         -1    to -1    : unref and do not do anything
+         *         -1 to valid    : inject timer
+         */
 
-                /* ctx->timer is NULL iff previous value of
-                 * ctx->spb_choice is -1
-                 */
-                if (ctx->timer) {
-                        if (ctx->spb_choice == -1) {
-                                gf_timer_call_cancel (this->ctx, ctx->timer);
-                                ctx->timer = NULL;
-                                inode_unref (inode);
-                                goto unlock;
-                        }
-                        goto reset_timer;
-                } else {
-                        if (ctx->spb_choice == -1)
-                                goto unlock;
+        /* ctx->timer is NULL iff previous value of
+         * ctx->spb_choice is -1
+         */
+        if (ctx->timer) {
+            if (ctx->spb_choice == -1) {
+                if (!gf_timer_call_cancel(this->ctx, ctx->timer)) {
+                    ctx->timer = NULL;
+                    timer_cancelled = _gf_true;
                 }
-
-                inode = inode_ref (loc->inode);
-                goto set_timer;
-
-reset_timer:
-                gf_timer_call_cancel (this->ctx, ctx->timer);
-                ctx->timer = NULL;
-
-set_timer:
-                ctx->timer = gf_timer_call_after (this->ctx, delta,
-                                                  afr_set_split_brain_choice_cbk,
-                                                  inode);
+                /* If timer cancel failed here it means that the
+                 *  previous cbk will be executed which will set
+                 *  spb_choice to -1. So we can consider the
+                 *  'valid to -1' case to be a success
+                 *  (i.e. ret = 0) and goto unlock.
+                 */
+                goto unlock;
+            }
+            goto reset_timer;
+        } else {
+            if (ctx->spb_choice == -1)
+                goto unlock;
+            goto set_timer;
         }
+
+    reset_timer:
+        ret = gf_timer_call_cancel(this->ctx, ctx->timer);
+        if (ret != 0) {
+            /* We need to bail out now instead of launching a new
+             * timer. Otherwise the cbk of the previous timer event
+             * will cancel the new ctx->timer.
+             */
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = EAGAIN;
+            goto unlock;
+        }
+        ctx->timer = NULL;
+        timer_reset = _gf_true;
+
+    set_timer:
+        ctx->timer = gf_timer_call_after(this->ctx, delta,
+                                         afr_set_split_brain_choice_cbk, inode);
+        if (!ctx->timer) {
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = ENOMEM;
+        }
+        if (!timer_reset && ctx->timer)
+            timer_set = _gf_true;
+        if (timer_reset && !ctx->timer)
+            timer_cancelled = _gf_true;
+    }
 unlock:
-        UNLOCK(&inode->lock);
-        inode_invalidate (inode);
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!timer_set)
+        inode_unref(inode);
+    if (timer_cancelled)
+        inode_unref(inode);
+    /*
+     * We need to invalidate the inode to prevent the kernel from serving
+     * reads from an older cached value despite a change in spb_choice to
+     * a new value.
+     */
+    inode_invalidate(inode);
 out:
-        if (data)
-                GF_FREE (data);
-        AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        return 0;
+    GF_FREE(data);
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
 }
 
 int
-afr_accused_fill (xlator_t *this, dict_t *xdata, unsigned char *accused,
-		  afr_transaction_type type)
+afr_accused_fill(xlator_t *this, dict_t *xdata, unsigned char *accused,
+                 afr_transaction_type type)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int idx = afr_index_for_transaction_type (type);
-	void *pending_raw = NULL;
-	int pending[3];
-	int ret = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int idx = afr_index_for_transaction_type(type);
+    void *pending_raw = NULL;
+    int pending[3];
+    int ret = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		ret = dict_get_ptr (xdata, priv->pending_key[i],
-				    &pending_raw);
-		if (ret) /* no pending flags */
-			continue;
-		memcpy (pending, pending_raw, sizeof(pending));
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_get_ptr(xdata, priv->pending_key[i], &pending_raw);
+        if (ret) /* no pending flags */
+            continue;
+        memcpy(pending, pending_raw, sizeof(pending));
 
-		if (ntoh32 (pending[idx]))
-			accused[i] = 1;
-	}
+        if (ntoh32(pending[idx]))
+            accused[i] = 1;
+    }
 
-	return 0;
+    return 0;
 }
 
 int
-afr_accuse_smallfiles (xlator_t *this, struct afr_reply *replies,
-		       unsigned char *data_accused)
+afr_accuse_smallfiles(xlator_t *this, struct afr_reply *replies,
+                      unsigned char *data_accused)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	uint64_t maxsize = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t maxsize = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid && replies[i].xdata &&
-                    dict_get (replies[i].xdata, GLUSTERFS_BAD_INODE))
-                        continue;
-		if (data_accused[i])
-			continue;
-		if (replies[i].poststat.ia_size > maxsize)
-			maxsize = replies[i].poststat.ia_size;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].xdata &&
+            dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))
+            continue;
+        if (data_accused[i])
+            continue;
+        if (replies[i].poststat.ia_size > maxsize)
+            maxsize = replies[i].poststat.ia_size;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (data_accused[i])
-			continue;
-                if (AFR_IS_ARBITER_BRICK(priv, i))
-                        continue;
-		if (replies[i].poststat.ia_size < maxsize)
-			data_accused[i] = 1;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i])
+            continue;
+        if (AFR_IS_ARBITER_BRICK(priv, i))
+            continue;
+        if (replies[i].poststat.ia_size < maxsize)
+            data_accused[i] = 1;
+    }
 
-	return 0;
+    return 0;
 }
 
 int
-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                       gf_boolean_t *start_heal)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	struct afr_reply *replies = NULL;
-	int event_generation = 0;
-	int i = 0;
-	unsigned char *data_accused = NULL;
-	unsigned char *metadata_accused = NULL;
-	unsigned char *data_readable = NULL;
-	unsigned char *metadata_readable = NULL;
-	int ret = 0;
-
-	local = frame->local;
-	priv = this->private;
-	replies = local->replies;
-	event_generation = local->event_generation;
-
-	data_accused = alloca0 (priv->child_count);
-	data_readable = alloca0 (priv->child_count);
-	metadata_accused = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		data_readable[i] = 1;
-		metadata_readable[i] = 1;
-	}
-        if (AFR_IS_ARBITER_BRICK (priv, ARBITER_BRICK_INDEX)) {
-                data_readable[ARBITER_BRICK_INDEX] =  0;
-                metadata_readable[ARBITER_BRICK_INDEX] = 0;
-        }
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid) {
-			data_readable[i] = 0;
-			metadata_readable[i] = 0;
-			continue;
-		}
-
-		if (replies[i].op_ret == -1) {
-			data_readable[i] = 0;
-			metadata_readable[i] = 0;
-			continue;
-		}
-
-                if (replies[i].xdata &&
-                    dict_get (replies[i].xdata, GLUSTERFS_BAD_INODE)) {
-			data_readable[i] = 0;
-			metadata_readable[i] = 0;
-			continue;
-                }
-
-		afr_accused_fill (this, replies[i].xdata, data_accused,
-				  (replies[i].poststat.ia_type == IA_IFDIR) ?
-				   AFR_ENTRY_TRANSACTION : AFR_DATA_TRANSACTION);
-
-		afr_accused_fill (this, replies[i].xdata,
-				  metadata_accused, AFR_METADATA_TRANSACTION);
-
-	}
-
-	if ((inode->ia_type != IA_IFDIR) &&
-            /* We want to accuse small files only when we know for sure that
-             * there is no IO happening. Otherwise, the ia_sizes obtained in
-             * post-refresh replies may  mismatch due to a race between inode-
-             * refresh and ongoing writes, causing spurious heal launches*/
-            !afr_is_possibly_under_txn (AFR_DATA_TRANSACTION, local, this))
-		afr_accuse_smallfiles (this, replies, data_accused);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (data_accused[i]) {
-			data_readable[i] = 0;
-			ret = 1;
-		}
-		if (metadata_accused[i]) {
-			metadata_readable[i] = 0;
-			ret = 1;
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (start_heal && priv->child_up[i] &&
-                    (!data_readable[i] || !metadata_readable[i])) {
-                        *start_heal = _gf_true;
-                        break;
-                }
-        }
-	afr_inode_read_subvol_set (inode, this, data_readable,
-				   metadata_readable, event_generation);
-	return ret;
+afr_readables_fill(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   unsigned char *data_accused, unsigned char *metadata_accused,
+                   unsigned char *data_readable,
+                   unsigned char *metadata_readable, struct afr_reply *replies)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+    int ret = 0;
+    ia_type_t ia_type = IA_INVAL;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        data_readable[i] = 1;
+        metadata_readable[i] = 1;
+    }
+    if (AFR_IS_ARBITER_BRICK(priv, ARBITER_BRICK_INDEX)) {
+        data_readable[ARBITER_BRICK_INDEX] = 0;
+        metadata_readable[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies) { /* Lookup */
+            if (!replies[i].valid || replies[i].op_ret == -1 ||
+                (replies[i].xdata &&
+                 dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))) {
+                data_readable[i] = 0;
+                metadata_readable[i] = 0;
+                continue;
+            }
+
+            xdata = replies[i].xdata;
+            ia_type = replies[i].poststat.ia_type;
+        } else { /* pre-op xattrop */
+            xdata = local->transaction.changelog_xdata[i];
+            ia_type = inode->ia_type;
+        }
+
+        if (!xdata)
+            continue; /* mkdir_cbk sends NULL xdata_rsp. */
+        afr_accused_fill(this, xdata, data_accused,
+                         (ia_type == IA_IFDIR) ? AFR_ENTRY_TRANSACTION
+                                               : AFR_DATA_TRANSACTION);
+
+        afr_accused_fill(this, xdata, metadata_accused,
+                         AFR_METADATA_TRANSACTION);
+    }
+
+    if (replies && ia_type != IA_INVAL && ia_type != IA_IFDIR &&
+        /* We want to accuse small files only when we know for
+         * sure that there is no IO happening. Otherwise, the
+         * ia_sizes obtained in post-refresh replies may
+         * mismatch due to a race between inode-refresh and
+         * ongoing writes, causing spurious heal launches*/
+        !afr_is_possibly_under_txn(AFR_DATA_TRANSACTION, local, this)) {
+        afr_accuse_smallfiles(this, replies, data_accused);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i]) {
+            data_readable[i] = 0;
+            ret = 1;
+        }
+        if (metadata_accused[i]) {
+            metadata_readable[i] = 0;
+            ret = 1;
+        }
+    }
+    return ret;
 }
 
-
+int
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int event_generation = 0;
+    int i = 0;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+    event_generation = local->event_generation;
+
+    data_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    ret = afr_readables_fill(frame, this, inode, data_accused, metadata_accused,
+                             data_readable, metadata_readable, replies);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (start_heal && priv->child_up[i] &&
+            (data_accused[i] || metadata_accused[i])) {
+            *start_heal = _gf_true;
+            break;
+        }
+    }
+    afr_inode_read_subvol_set(inode, this, data_readable, metadata_readable,
+                              event_generation);
+    return ret;
+}
 
 int
-afr_refresh_selfheal_done (int ret, call_frame_t *heal, void *opaque)
+afr_refresh_selfheal_done(int ret, call_frame_t *heal, void *opaque)
 {
-	if (heal)
-		STACK_DESTROY (heal->root);
-	return 0;
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return 0;
 }
 
 int
-afr_inode_refresh_err (call_frame_t *frame, xlator_t *this)
+afr_inode_refresh_err(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int err = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int err = 0;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].valid && !local->replies[i].op_ret) {
-			err = 0;
-			goto ret;
-		}
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && !local->replies[i].op_ret) {
+            err = 0;
+            goto ret;
+        }
+    }
 
-	err = afr_final_errno (local, priv);
+    err = afr_final_errno(local, priv);
 ret:
-	return -err;
+    return err;
 }
 
 gf_boolean_t
-afr_selfheal_enabled (xlator_t *this)
+afr_selfheal_enabled(const xlator_t *this)
 {
-	afr_private_t *priv = NULL;
-	gf_boolean_t data = _gf_false;
-        int ret = 0;
+    const afr_private_t *priv = this->private;
+
+    return priv->data_self_heal || priv->metadata_self_heal ||
+           priv->entry_self_heal;
+}
 
-	priv = this->private;
+int
+afr_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int event_generation = 0;
+    int read_subvol = -1;
+    int ret = 0;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err)
+        goto refresh_done;
+
+    if (local->op == GF_FOP_LOOKUP)
+        goto refresh_done;
+
+    ret = afr_inode_get_readable(frame, inode, this, local->readable,
+                                 &event_generation, local->transaction.type);
+
+    if (ret == -EIO) {
+        /* No readable subvolume even after refresh ==> splitbrain.*/
+        if (!priv->fav_child_policy) {
+            err = EIO;
+            goto refresh_done;
+        }
+        read_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (read_subvol == -1) {
+            err = EIO;
+            goto refresh_done;
+        }
+
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame) {
+            err = EIO;
+            goto refresh_done;
+        }
+        heal_local = heal_frame->local;
+        heal_local->xdata_req = dict_new();
+        if (!heal_local->xdata_req) {
+            err = EIO;
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
+        }
+        heal_local->heal_frame = frame;
+        ret = synctask_new(this->ctx->env, afr_fav_child_reset_sink_xattrs,
+                           afr_fav_child_reset_sink_xattrs_cbk, heal_frame,
+                           heal_frame);
+        return 0;
+    }
 
-	ret = gf_string2boolean (priv->data_self_heal, &data);
-        GF_ASSERT (!ret);
+refresh_done:
+    afr_local_replies_wipe(local, this->private);
+    local->refreshfn(frame, this, err);
 
-	return data || priv->metadata_self_heal || priv->entry_self_heal;
+    return 0;
 }
 
 int
-afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)
-{
-	call_frame_t *heal_frame = NULL;
-	afr_local_t *local = NULL;
-        gf_boolean_t start_heal = _gf_false;
-        afr_local_t *heal_local = NULL;
-        int op_errno = ENOMEM;
-	int ret = 0;
-	int err = 0;
-
-	local = frame->local;
-
-	ret = afr_replies_interpret (frame, this, local->refreshinode,
-                                     &start_heal);
-
-	err = afr_inode_refresh_err (frame, this);
-
-        afr_local_replies_wipe (local, this->private);
-
-	if (ret && afr_selfheal_enabled (this) && start_heal) {
-                heal_frame = copy_frame (frame);
-                if (!heal_frame)
-                        goto refresh_done;
-                heal_frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
-                heal_local = AFR_FRAME_INIT (heal_frame, op_errno);
-                if (!heal_local) {
-                        AFR_STACK_DESTROY (heal_frame);
-                        goto refresh_done;
-                }
-                heal_local->refreshinode = inode_ref (local->refreshinode);
-                heal_local->heal_frame = heal_frame;
-                afr_throttled_selfheal (heal_frame, this);
-        }
+afr_inode_refresh_done(call_frame_t *frame, xlator_t *this, int error)
+{
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t start_heal = _gf_false;
+    afr_local_t *heal_local = NULL;
+    unsigned char *success_replies = NULL;
+    int ret = 0;
+
+    if (error != 0) {
+        goto refresh_done;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->thin_arbiter_count && local->is_read_txn &&
+        AFR_COUNT(success_replies, priv->child_count) != priv->child_count) {
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        if (success_replies[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (success_replies[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
+        }
+        error = EINVAL;
+        goto refresh_done;
+    }
+
+    if (!afr_has_quorum(success_replies, this, frame)) {
+        error = afr_final_errno(frame->local, this->private);
+        if (!error)
+            error = afr_quorum_errno(priv);
+        goto refresh_done;
+    }
+
+    ret = afr_replies_interpret(frame, this, local->refreshinode, &start_heal);
+
+    if (ret && afr_selfheal_enabled(this) && start_heal) {
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame)
+            goto refresh_done;
+        heal_local = heal_frame->local;
+        heal_local->refreshinode = inode_ref(local->refreshinode);
+        heal_local->heal_frame = heal_frame;
+        if (!afr_throttled_selfheal(heal_frame, this)) {
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
+        }
+    }
 
 refresh_done:
-        local->refreshfn (frame, this, err);
+    afr_txn_refresh_done(frame, this, error);
 
-	return 0;
+    return 0;
 }
 
 void
-afr_inode_refresh_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, struct iatt *buf,
-                              dict_t *xdata, struct iatt *par)
-{
-        afr_local_t *local = NULL;
-        int call_child = (long) cookie;
-        int8_t need_heal = 1;
-        int call_count = 0;
-        GF_UNUSED int ret = 0;
+afr_inode_refresh_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *buf,
+                             dict_t *xdata, struct iatt *par)
+{
+    afr_local_t *local = NULL;
+    int call_child = (long)cookie;
+    int8_t need_heal = 1;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    local->replies[call_child].valid = 1;
+    local->replies[call_child].op_ret = op_ret;
+    local->replies[call_child].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[call_child].poststat = *buf;
+        if (par)
+            local->replies[call_child].postparent = *par;
+        if (xdata)
+            local->replies[call_child].xdata = dict_ref(xdata);
+    }
 
-        local = frame->local;
-        local->replies[call_child].valid = 1;
-        local->replies[call_child].op_ret = op_ret;
-        local->replies[call_child].op_errno = op_errno;
-	if (op_ret != -1) {
-		local->replies[call_child].poststat = *buf;
-		if (par)
-                        local->replies[call_child].postparent = *par;
-                if (xdata)
-		        local->replies[call_child].xdata = dict_ref (xdata);
-	}
-        if (xdata) {
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[call_child].need_heal = need_heal;
-        } else {
-                local->replies[call_child].need_heal = need_heal;
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        if (ret) {
+            gf_msg_debug(this->name, -ret, "Unable to get link count");
         }
+    }
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                afr_set_need_heal (this, local);
-		afr_inode_refresh_done (frame, this);
+    local->replies[call_child].need_heal = need_heal;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        ret = afr_inode_refresh_err(frame, this);
+        if (ret) {
+            gf_msg_debug(this->name, ret, "afr_inode_refresh_err failed");
         }
-
+        afr_inode_refresh_done(frame, this, ret);
+    }
 }
 
 int
-afr_inode_refresh_subvol_with_lookup_cbk (call_frame_t *frame, void *cookie,
-                                          xlator_t *this, int op_ret,
-                                          int op_errno, inode_t *inode,
-                                          struct iatt *buf, dict_t *xdata,
-                                          struct iatt *par)
+afr_inode_refresh_subvol_with_lookup_cbk(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int op_ret,
+                                         int op_errno, inode_t *inode,
+                                         struct iatt *buf, dict_t *xdata,
+                                         struct iatt *par)
 {
-        afr_inode_refresh_subvol_cbk (frame, cookie, this, op_ret, op_errno,
-                                      buf, xdata, par);
-	return 0;
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, par);
+    return 0;
 }
 
-
 int
-afr_inode_refresh_subvol_with_lookup (call_frame_t *frame, xlator_t *this,
-                                      int i, inode_t *inode, uuid_t gfid,
-                                      dict_t *xdata)
+afr_inode_refresh_subvol_with_lookup(call_frame_t *frame, xlator_t *this, int i,
+                                     inode_t *inode, uuid_t gfid, dict_t *xdata)
 {
-	loc_t loc = {0, };
-	afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	loc.inode = inode;
-        if (gf_uuid_is_null (inode->gfid) && gfid) {
-                /* To handle setattr/setxattr on yet to be linked inode from
-                 * dht */
-                gf_uuid_copy (loc.gfid, gfid);
-        } else {
-                gf_uuid_copy (loc.gfid, inode->gfid);
-        }
+    loc.inode = inode;
+    if (gf_uuid_is_null(inode->gfid) && gfid) {
+        /* To handle setattr/setxattr on yet to be linked inode from
+         * dht */
+        gf_uuid_copy(loc.gfid, gfid);
+    } else {
+        gf_uuid_copy(loc.gfid, inode->gfid);
+    }
 
-	STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_with_lookup_cbk,
-			   (void *) (long) i, priv->children[i],
-			   priv->children[i]->fops->lookup, &loc, xdata);
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_lookup_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->lookup, &loc, xdata);
+    return 0;
 }
 
 int
-afr_inode_refresh_subvol_with_fstat_cbk (call_frame_t *frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *buf, dict_t *xdata)
+afr_inode_refresh_subvol_with_fstat_cbk(call_frame_t *frame, void *cookie,
+                                        xlator_t *this, int32_t op_ret,
+                                        int32_t op_errno, struct iatt *buf,
+                                        dict_t *xdata)
 {
-        afr_inode_refresh_subvol_cbk (frame, cookie, this, op_ret, op_errno,
-                                      buf, xdata, NULL);
-        return 0;
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, NULL);
+    return 0;
 }
 
 int
-afr_inode_refresh_subvol_with_fstat (call_frame_t *frame, xlator_t *this, int i,
-			             dict_t *xdata)
+afr_inode_refresh_subvol_with_fstat(call_frame_t *frame, xlator_t *this, int i,
+                                    dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_with_fstat_cbk,
-                           (void *) (long) i, priv->children[i],
-                           priv->children[i]->fops->fstat, local->fd, xdata);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_fstat_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->fstat, local->fd, xdata);
+    return 0;
 }
 
 int
-afr_inode_refresh_do (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-	int i = 0;
-        int ret = 0;
-	dict_t *xdata = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-        unsigned char *wind_subvols = NULL;
-
-	priv = this->private;
-	local = frame->local;
-        wind_subvols = alloca0 (priv->child_count);
-
-        afr_local_replies_wipe (local, priv);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        afr_inode_refresh_done (frame, this);
-                        return 0;
-                }
-        }
+afr_inode_refresh_do(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *wind_subvols = NULL;
 
-	xdata = dict_new ();
-	if (!xdata) {
-		afr_inode_refresh_done (frame, this);
-		return 0;
-	}
+    priv = this->private;
+    local = frame->local;
+    wind_subvols = alloca0(priv->child_count);
 
-	if (afr_xattr_req_prepare (this, xdata) != 0) {
-		dict_unref (xdata);
-		afr_inode_refresh_done (frame, this);
-		return 0;
-	}
+    afr_local_replies_wipe(local, priv);
 
-        ret = dict_set_str (xdata, "link-count", GF_XATTROP_INDEX_COUNT);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set link-count in dict ");
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            afr_inode_refresh_done(frame, this, EINVAL);
+            return 0;
         }
+    }
 
-        ret = dict_set_str (xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set inodelk-dom-count in dict ");
+    xdata = dict_new();
+    if (!xdata) {
+        afr_inode_refresh_done(frame, this, ENOMEM);
+        return 0;
+    }
 
-        }
+    ret = afr_xattr_req_prepare(this, xdata);
+    if (ret != 0) {
+        dict_unref(xdata);
+        afr_inode_refresh_done(frame, this, -ret);
+        return 0;
+    }
 
-        if (local->fd) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] &&
-                            fd_ctx->opened_on[i] == AFR_FD_OPENED)
-                                wind_subvols[i] = 1;
-                }
-        } else {
-                memcpy (wind_subvols, local->child_up,
-                        sizeof (*local->child_up) * priv->child_count);
-        }
+    ret = dict_set_sizen_str_sizen(xdata, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
 
-	local->call_count = AFR_COUNT (wind_subvols, priv->child_count);
+    ret = dict_set_str_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "Unable to set inodelk-dom-count in dict ");
+    }
 
-	call_count = local->call_count;
-        if (!call_count) {
-                dict_unref (xdata);
-                afr_inode_refresh_done (frame, this);
-                return 0;
+    if (local->fd) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->child_up[i] && fd_ctx->opened_on[i] == AFR_FD_OPENED)
+                wind_subvols[i] = 1;
         }
-	for (i = 0; i < priv->child_count; i++) {
-		if (!wind_subvols[i])
-			continue;
+    } else {
+        memcpy(wind_subvols, local->child_up,
+               sizeof(*local->child_up) * priv->child_count);
+    }
 
-                if (local->fd)
-                        afr_inode_refresh_subvol_with_fstat (frame, this, i,
-                                                             xdata);
-                else
-                        afr_inode_refresh_subvol_with_lookup (frame, this, i,
-                                                    local->refreshinode,
-                                                    local->refreshgfid, xdata);
+    local->call_count = AFR_COUNT(wind_subvols, priv->child_count);
 
-		if (!--call_count)
-			break;
-	}
+    call_count = local->call_count;
+    if (!call_count) {
+        dict_unref(xdata);
+        if (local->fd && AFR_COUNT(local->child_up, priv->child_count))
+            afr_inode_refresh_done(frame, this, EBADFD);
+        else
+            afr_inode_refresh_done(frame, this, ENOTCONN);
+        return 0;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_subvols[i])
+            continue;
 
-	dict_unref (xdata);
+        if (local->fd)
+            afr_inode_refresh_subvol_with_fstat(frame, this, i, xdata);
+        else
+            afr_inode_refresh_subvol_with_lookup(
+                frame, this, i, local->refreshinode, local->refreshgfid, xdata);
 
-	return 0;
-}
+        if (!--call_count)
+            break;
+    }
 
+    dict_unref(xdata);
+
+    return 0;
+}
 
 int
-afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                   uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	local->refreshfn = refreshfn;
+    local->refreshfn = refreshfn;
 
-	if (local->refreshinode) {
-		inode_unref (local->refreshinode);
-		local->refreshinode = NULL;
-	}
+    if (local->refreshinode) {
+        inode_unref(local->refreshinode);
+        local->refreshinode = NULL;
+    }
 
-	local->refreshinode = inode_ref (inode);
+    local->refreshinode = inode_ref(inode);
 
-        if (gfid)
-                gf_uuid_copy (local->refreshgfid, gfid);
-        else
-                gf_uuid_clear (local->refreshgfid);
+    if (gfid)
+        gf_uuid_copy(local->refreshgfid, gfid);
+    else
+        gf_uuid_clear(local->refreshgfid);
 
-	afr_inode_refresh_do (frame, this);
+    afr_inode_refresh_do(frame, this);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req)
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req)
 {
-        int             i           = 0;
-        afr_private_t   *priv       = NULL;
-        int             ret         = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-        priv   = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
-                                       AFR_NUM_CHANGE_LOGS * sizeof(int));
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Unable to set dict value for %s",
-                                priv->pending_key[i]);
-                /* 3 = data+metadata+entry */
-        }
-        ret = dict_set_uint64 (xattr_req, AFR_DIRTY,
-			       AFR_NUM_CHANGE_LOGS * sizeof(int));
-        if (ret) {
-                gf_msg_debug (this->name, -ret, "failed to set dirty "
-                              "query flag");
-        }
-
-        ret = dict_set_int32 (xattr_req, "list-xattr", 1);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set list-xattr in dict ");
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_uint64(xattr_req, priv->pending_key[i],
+                              AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Unable to set dict value for %s", priv->pending_key[i]);
+        /* 3 = data+metadata+entry */
+    }
+    ret = dict_set_uint64(xattr_req, AFR_DIRTY,
+                          AFR_NUM_CHANGE_LOGS * sizeof(int));
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "failed to set dirty "
+                     "query flag");
+    }
+
+    ret = dict_set_int32_sizen(xattr_req, "list-xattr", 1);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set list-xattr in dict ");
+    }
+
+    return ret;
+}
 
-	return ret;
+int
+afr_lookup_xattr_req_prepare(afr_local_t *local, xlator_t *this,
+                             dict_t *xattr_req, loc_t *loc)
+{
+    int ret = -ENOMEM;
+
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    if (!local->xattr_req)
+        goto out;
+
+    if (xattr_req && (xattr_req != local->xattr_req))
+        dict_copy(xattr_req, local->xattr_req);
+
+    ret = afr_xattr_req_prepare(this, local->xattr_req);
+
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_INODELK_COUNT);
+    }
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_ENTRYLK_COUNT);
+    }
+
+    ret = dict_set_uint32(local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_PARENT_ENTRYLK);
+    }
+
+    ret = dict_set_sizen_str_sizen(local->xattr_req, "link-count",
+                                   GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
-                              dict_t *xattr_req, loc_t *loc)
+afr_least_pending_reads_child(afr_private_t *priv, unsigned char *readable)
 {
-        int     ret = -ENOMEM;
-
-        if (!local->xattr_req)
-                local->xattr_req = dict_new ();
+    int i = 0;
+    int child = -1;
+    int64_t read_iter = -1;
+    int64_t pending_read = -1;
 
-        if (!local->xattr_req)
-                goto out;
-
-        if (xattr_req && (xattr_req != local->xattr_req))
-                dict_copy (xattr_req, local->xattr_req);
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i])
+            continue;
+        read_iter = GF_ATOMIC_GET(priv->pending_reads[i]);
+        if (child == -1 || read_iter < pending_read) {
+            pending_read = read_iter;
+            child = i;
+        }
+    }
 
-        ret = afr_xattr_req_prepare (this, local->xattr_req);
+    return child;
+}
 
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_INODELK_COUNT);
-        }
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
-        }
+static int32_t
+afr_least_latency_child(afr_private_t *priv, unsigned char *readable)
+{
+    int32_t i = 0;
+    int child = -1;
 
-        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
 
-        ret = dict_set_str (xattr_req, "link-count", GF_XATTROP_INDEX_COUNT);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set link-count in dict ");
+        if (child == -1 ||
+            priv->child_latency[i] < priv->child_latency[child]) {
+            child = i;
         }
-
-        ret = 0;
-out:
-        return ret;
+    }
+    return child;
 }
 
-
-int
-afr_hash_child (afr_read_subvol_args_t *args, int32_t child_count, int hashmode)
+static int32_t
+afr_least_latency_times_pending_reads_child(afr_private_t *priv,
+                                            unsigned char *readable)
 {
-        uuid_t gfid_copy = {0,};
-        pid_t pid;
+    int32_t i = 0;
+    int child = -1;
+    int64_t pending_read = 0;
+    int64_t latency = -1;
+    int64_t least_latency = -1;
 
-        if (!hashmode) {
-                return -1;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
+
+        pending_read = GF_ATOMIC_GET(priv->pending_reads[i]);
+        latency = (pending_read + 1) * priv->child_latency[i];
 
-        gf_uuid_copy (gfid_copy, args->gfid);
+        if (child == -1 || latency < least_latency) {
+            least_latency = latency;
+            child = i;
+        }
+    }
+    return child;
+}
 
-        if ((hashmode > 1) && (args->ia_type != IA_IFDIR)) {
+int
+afr_hash_child(afr_read_subvol_args_t *args, afr_private_t *priv,
+               unsigned char *readable)
+{
+    uuid_t gfid_copy = {
+        0,
+    };
+    pid_t pid;
+    int child = -1;
+
+    switch (priv->hash_mode) {
+        case AFR_READ_POLICY_FIRST_UP:
+            break;
+        case AFR_READ_POLICY_GFID_HASH:
+            gf_uuid_copy(gfid_copy, args->gfid);
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_GFID_PID_HASH:
+            if (args->ia_type != IA_IFDIR) {
                 /*
                  * Why getpid?  Because it's one of the cheapest calls
-                 * available - faster than gethostname etc. - and returns a
-                 * constant-length value that's sure to be shorter than a UUID.
-                 * It's still very unlikely to be the same across clients, so
-                 * it still provides good mixing.  We're not trying for
-                 * perfection here.  All we need is a low probability that
-                 * multiple clients won't converge on the same subvolume.
+                 * available - faster than gethostname etc. - and
+                 * returns a constant-length value that's sure to be
+                 * shorter than a UUID. It's still very unlikely to be
+                 * the same across clients, so it still provides good
+                 * mixing.  We're not trying for perfection here. All we
+                 * need is a low probability that multiple clients
+                 * won't converge on the same subvolume.
                  */
+                gf_uuid_copy(gfid_copy, args->gfid);
                 pid = getpid();
-                memcpy (gfid_copy, &pid, sizeof(pid));
-        }
-
-        return SuperFastHash((char *)gfid_copy,
-                             sizeof(gfid_copy)) % child_count;
+                *(pid_t *)gfid_copy ^= pid;
+            }
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_LESS_LOAD:
+            child = afr_least_pending_reads_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LEAST_LATENCY:
+            child = afr_least_latency_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LOAD_LATENCY_HYBRID:
+            child = afr_least_latency_times_pending_reads_child(priv, readable);
+            break;
+    }
+
+    return child;
 }
 
-
 int
-afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
-				  unsigned char *readable,
-                                  afr_read_subvol_args_t *args)
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args)
 {
-	int             i           = 0;
-	int             read_subvol = -1;
-	afr_private_t  *priv        = NULL;
-        afr_read_subvol_args_t local_args = {0,};
+    int i = 0;
+    int read_subvol = -1;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t local_args = {
+        0,
+    };
 
-	priv = this->private;
+    priv = this->private;
 
-	/* first preference - explicitly specified or local subvolume */
-	if (priv->read_child >= 0 && readable[priv->read_child])
-                return priv->read_child;
+    /* first preference - explicitly specified or local subvolume */
+    if (priv->read_child >= 0 && readable[priv->read_child])
+        return priv->read_child;
 
-        if (inode_is_linked (inode)) {
-                gf_uuid_copy (local_args.gfid, inode->gfid);
-                local_args.ia_type = inode->ia_type;
-        } else if (args) {
-                local_args = *args;
-        }
+    if (inode_is_linked(inode)) {
+        gf_uuid_copy(local_args.gfid, inode->gfid);
+        local_args.ia_type = inode->ia_type;
+    } else if (args) {
+        local_args = *args;
+    }
 
-	/* second preference - use hashed mode */
-	read_subvol = afr_hash_child (&local_args, priv->child_count,
-                                      priv->hash_mode);
-	if (read_subvol >= 0 && readable[read_subvol])
-                return read_subvol;
+    /* second preference - use hashed mode */
+    read_subvol = afr_hash_child(&local_args, priv, readable);
+    if (read_subvol >= 0 && readable[read_subvol])
+        return read_subvol;
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (readable[i])
-                return i;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (readable[i])
+            return i;
+    }
 
-        /* no readable subvolumes, either split brain or all subvols down */
+    /* no readable subvolumes, either split brain or all subvols down */
 
-        return -1;
+    return -1;
 }
 
-
 int
-afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
-				unsigned char *readable, int *event_p,
-				int type)
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (type == AFR_METADATA_TRANSACTION)
-		ret = afr_inode_read_subvol_get (inode, this, 0, readable,
-						 event_p);
-	else
-		ret = afr_inode_read_subvol_get (inode, this, readable, 0,
-						 event_p);
-	return ret;
+    if (type == AFR_METADATA_TRANSACTION)
+        ret = afr_inode_read_subvol_get(inode, this, 0, readable, event_p);
+    else
+        ret = afr_inode_read_subvol_get(inode, this, readable, 0, event_p);
+    return ret;
 }
 
+void
+afr_readables_intersect_get(inode_t *inode, xlator_t *this, int *event,
+                            unsigned char *intersection)
+{
+    afr_private_t *priv = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    unsigned char *intersect = NULL;
 
-int
-afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
-                     unsigned char *readables,
-		     int *event_p, afr_transaction_type type,
-                     afr_read_subvol_args_t *args)
-{
-	afr_private_t *priv = NULL;
-	unsigned char *data_readable = NULL;
-	unsigned char *metadata_readable = NULL;
-	unsigned char *readable = NULL;
-	unsigned char *intersection = NULL;
-	int subvol = -1;
-	int event = 0;
-
-	priv = this->private;
-
-	readable = alloca0 (priv->child_count);
-	data_readable = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-	intersection = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_type_get (inode, this, readable, &event, type);
-
-	afr_inode_read_subvol_get (inode, this, data_readable, metadata_readable,
-				   &event);
-
-	AFR_INTERSECT (intersection, data_readable, metadata_readable,
-		       priv->child_count);
-
-	if (AFR_COUNT (intersection, priv->child_count) > 0)
-		subvol = afr_read_subvol_select_by_policy (inode, this,
-							   intersection, args);
-	else
-		subvol = afr_read_subvol_select_by_policy (inode, this,
-							   readable, args);
-	if (subvol_p)
-		*subvol_p = subvol;
-	if (event_p)
-		*event_p = event;
-        if (readables)
-                memcpy (readables, readable,
-                        sizeof (*readables) * priv->child_count);
-	return subvol;
-}
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    intersect = alloca0(priv->child_count);
 
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              event);
 
-void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
+    AFR_INTERSECT(intersect, data_readable, metadata_readable,
+                  priv->child_count);
+    if (intersection)
+        memcpy(intersection, intersect,
+               sizeof(*intersection) * priv->child_count);
+}
+
+int
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args)
 {
-        afr_private_t *priv    = NULL;
-        int           i        = 0;
+    afr_private_t *priv = NULL;
+    unsigned char *readable = NULL;
+    unsigned char *intersection = NULL;
+    int subvol = -1;
+    int event = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        afr_matrix_cleanup (local->pending, priv->child_count);
+    readable = alloca0(priv->child_count);
+    intersection = alloca0(priv->child_count);
 
-        GF_FREE (local->internal_lock.locked_nodes);
+    afr_inode_read_subvol_type_get(inode, this, readable, &event, type);
 
-        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
-                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
-        }
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-        GF_FREE (local->internal_lock.lower_locked_nodes);
+    if (AFR_COUNT(intersection, priv->child_count) > 0)
+        subvol = afr_read_subvol_select_by_policy(inode, this, intersection,
+                                                  args);
+    else
+        subvol = afr_read_subvol_select_by_policy(inode, this, readable, args);
+    if (subvol_p)
+        *subvol_p = subvol;
+    if (event_p)
+        *event_p = event;
+    if (readables)
+        memcpy(readables, readable, sizeof(*readables) * priv->child_count);
+    return subvol;
+}
 
-        afr_entry_lockee_cleanup (&local->internal_lock);
+void
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-        GF_FREE (local->transaction.pre_op);
+    priv = this->private;
 
-        GF_FREE (local->transaction.pre_op_sources);
-        if (local->transaction.pre_op_xdata) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->transaction.pre_op_xdata[i])
-                                continue;
-                        dict_unref (local->transaction.pre_op_xdata[i]);
-                }
-                GF_FREE (local->transaction.pre_op_xdata);
-        }
+    afr_matrix_cleanup(local->pending, priv->child_count);
 
-        GF_FREE (local->transaction.eager_lock);
-        GF_FREE (local->transaction.failed_subvols);
+    GF_FREE(local->internal_lock.lower_locked_nodes);
 
-        GF_FREE (local->transaction.basename);
-        GF_FREE (local->transaction.new_basename);
+    afr_lockees_cleanup(&local->internal_lock);
 
-        loc_wipe (&local->transaction.parent_loc);
-        loc_wipe (&local->transaction.new_parent_loc);
+    GF_FREE(local->transaction.pre_op);
 
-}
+    GF_FREE(local->transaction.pre_op_sources);
+    if (local->transaction.changelog_xdata) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->transaction.changelog_xdata[i])
+                continue;
+            dict_unref(local->transaction.changelog_xdata[i]);
+        }
+        GF_FREE(local->transaction.changelog_xdata);
+    }
 
+    GF_FREE(local->transaction.failed_subvols);
+
+    GF_FREE(local->transaction.basename);
+    GF_FREE(local->transaction.new_basename);
+
+    loc_wipe(&local->transaction.parent_loc);
+    loc_wipe(&local->transaction.new_parent_loc);
+}
 
 void
-afr_replies_wipe (struct afr_reply *replies, int count)
+afr_reply_wipe(struct afr_reply *reply)
 {
-        int i = 0;
+    if (reply->xdata) {
+        dict_unref(reply->xdata);
+        reply->xdata = NULL;
+    }
 
-        for (i = 0; i < count; i++) {
-                if (replies[i].xdata) {
-                        dict_unref (replies[i].xdata);
-                        replies[i].xdata = NULL;
-                }
+    if (reply->xattr) {
+        dict_unref(reply->xattr);
+        reply->xattr = NULL;
+    }
+}
 
-                if (replies[i].xattr) {
-                        dict_unref (replies[i].xattr);
-                        replies[i].xattr = NULL;
-                }
-        }
+void
+afr_replies_wipe(struct afr_reply *replies, int count)
+{
+    int i = 0;
+
+    for (i = 0; i < count; i++) {
+        afr_reply_wipe(&replies[i]);
+    }
 }
 
 void
-afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv)
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv)
 {
+    if (!local->replies)
+        return;
 
-	if (!local->replies)
-		return;
+    afr_replies_wipe(local->replies, priv->child_count);
 
-        afr_replies_wipe (local->replies, priv->child_count);
+    memset(local->replies, 0, sizeof(*local->replies) * priv->child_count);
+}
 
-	memset (local->replies, 0, sizeof(*local->replies) * priv->child_count);
+static gf_boolean_t
+afr_fop_lock_is_unlock(call_frame_t *frame)
+{
+    afr_local_t *local = frame->local;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            if ((F_UNLCK == local->cont.inodelk.in_flock.l_type) &&
+                (local->cont.inodelk.in_cmd == F_SETLKW ||
+                 local->cont.inodelk.in_cmd == F_SETLK))
+                return _gf_true;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            if (ENTRYLK_UNLOCK == local->cont.entrylk.in_cmd)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
 }
 
-void
-afr_remove_eager_lock_stub (afr_local_t *local)
-{
-        LOCK (&local->fd->lock);
-        {
-                list_del_init (&local->transaction.eager_locked);
-        }
-        UNLOCK (&local->fd->lock);
+static gf_boolean_t
+afr_lk_is_unlock(int32_t cmd, struct gf_flock *flock)
+{
+    switch (cmd) {
+        case F_RESLK_UNLCK:
+            return _gf_true;
+            break;
+
+#if F_SETLKW != F_SETLKW64
+        case F_SETLKW64:
+#endif
+        case F_SETLKW:
+
+#if F_SETLK != F_SETLK64
+        case F_SETLK64:
+#endif
+        case F_SETLK:
+            if (F_UNLCK == flock->l_type)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
 }
 
 void
-afr_local_cleanup (afr_local_t *local, xlator_t *this)
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno)
 {
-        afr_private_t * priv = NULL;
-
-        if (!local)
-                return;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-	syncbarrier_destroy (&local->barrier);
+    if (!frame || !frame->this || !frame->local || !frame->this->private)
+        return;
 
-        if (local->transaction.eager_lock_on &&
-            !list_empty (&local->transaction.eager_locked))
-                afr_remove_eager_lock_stub (local);
+    if (*op_ret < 0)
+        return;
 
-        afr_local_transaction_cleanup (local, this);
+    /* Failing inodelk/entrylk/lk here is not a good idea because we
+     * need to cleanup the locks on the other bricks if we choose to fail
+     * the fop here. The brick may go down just after unwind happens as well
+     * so anyways the fop will fail when the next fop is sent so leaving
+     * it like this for now.*/
+    local = frame->local;
+    switch (local->op) {
+        case GF_FOP_LOOKUP:
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+        case GF_FOP_LK:
+            return;
+        default:
+            break;
+    }
+
+    priv = frame->this->private;
+    if (!priv->consistent_io)
+        return;
 
-        priv = this->private;
+    if (local->event_generation &&
+        (local->event_generation != priv->event_generation))
+        goto inconsistent;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
+    return;
+inconsistent:
+    *op_ret = -1;
+    *op_errno = ENOTCONN;
+}
 
-        if (local->fd)
-                fd_unref (local->fd);
+void
+afr_local_cleanup(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+    if (!local)
+        return;
 
-        if (local->xattr_rsp)
-                dict_unref (local->xattr_rsp);
+    syncbarrier_destroy(&local->barrier);
 
-        if (local->dict)
-                dict_unref (local->dict);
+    afr_local_transaction_cleanup(local, this);
 
-        afr_local_replies_wipe (local, priv);
-	GF_FREE(local->replies);
+    priv = this->private;
 
-        GF_FREE (local->child_up);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
 
-        GF_FREE (local->read_attempted);
+    if (local->fd)
+        fd_unref(local->fd);
 
-        GF_FREE (local->readable);
-        GF_FREE (local->readable2);
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
 
-	if (local->inode)
-		inode_unref (local->inode);
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
 
-	if (local->parent)
-		inode_unref (local->parent);
+    if (local->dict)
+        dict_unref(local->dict);
 
-	if (local->parent2)
-		inode_unref (local->parent2);
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->replies);
 
-	if (local->refreshinode)
-		inode_unref (local->refreshinode);
+    GF_FREE(local->child_up);
 
-        { /* getxattr */
-                GF_FREE (local->cont.getxattr.name);
-        }
+    GF_FREE(local->read_attempted);
+
+    GF_FREE(local->readable);
+    GF_FREE(local->readable2);
+
+    if (local->inode)
+        inode_unref(local->inode);
 
-        { /* lk */
-                GF_FREE (local->cont.lk.locked_nodes);
-        }
+    if (local->parent)
+        inode_unref(local->parent);
 
-        { /* create */
-                if (local->cont.create.fd)
-                        fd_unref (local->cont.create.fd);
-                if (local->cont.create.params)
-                        dict_unref (local->cont.create.params);
-        }
+    if (local->parent2)
+        inode_unref(local->parent2);
 
-        { /* mknod */
-                if (local->cont.mknod.params)
-                        dict_unref (local->cont.mknod.params);
-        }
+    if (local->refreshinode)
+        inode_unref(local->refreshinode);
 
-        { /* mkdir */
-                if (local->cont.mkdir.params)
-                        dict_unref (local->cont.mkdir.params);
-        }
+    { /* getxattr */
+        GF_FREE(local->cont.getxattr.name);
+    }
 
-        { /* symlink */
-                if (local->cont.symlink.params)
-                        dict_unref (local->cont.symlink.params);
-        }
+    { /* lk */
+        GF_FREE(local->cont.lk.locked_nodes);
+        GF_FREE(local->cont.lk.dom_locked_nodes);
+        GF_FREE(local->cont.lk.dom_lock_op_ret);
+        GF_FREE(local->cont.lk.dom_lock_op_errno);
+    }
 
-        { /* writev */
-                GF_FREE (local->cont.writev.vector);
-                if (local->cont.writev.iobref)
-                        iobref_unref (local->cont.writev.iobref);
-        }
+    { /* create */
+        if (local->cont.create.fd)
+            fd_unref(local->cont.create.fd);
+        if (local->cont.create.params)
+            dict_unref(local->cont.create.params);
+    }
 
-        { /* setxattr */
-                if (local->cont.setxattr.dict)
-                        dict_unref (local->cont.setxattr.dict);
-        }
+    { /* mknod */
+        if (local->cont.mknod.params)
+            dict_unref(local->cont.mknod.params);
+    }
 
-        { /* fsetxattr */
-                if (local->cont.fsetxattr.dict)
-                        dict_unref (local->cont.fsetxattr.dict);
-        }
+    { /* mkdir */
+        if (local->cont.mkdir.params)
+            dict_unref(local->cont.mkdir.params);
+    }
 
-        { /* removexattr */
-                GF_FREE (local->cont.removexattr.name);
-        }
-        { /* xattrop */
-                if (local->cont.xattrop.xattr)
-                        dict_unref (local->cont.xattrop.xattr);
-        }
-        { /* symlink */
-                GF_FREE (local->cont.symlink.linkpath);
-        }
+    { /* symlink */
+        if (local->cont.symlink.params)
+            dict_unref(local->cont.symlink.params);
+    }
 
-        { /* opendir */
-                GF_FREE (local->cont.opendir.checksum);
-        }
+    { /* writev */
+        GF_FREE(local->cont.writev.vector);
+        if (local->cont.writev.iobref)
+            iobref_unref(local->cont.writev.iobref);
+    }
 
-        { /* readdirp */
-                if (local->cont.readdir.dict)
-                        dict_unref (local->cont.readdir.dict);
-        }
+    { /* setxattr */
+        if (local->cont.setxattr.dict)
+            dict_unref(local->cont.setxattr.dict);
+    }
 
-        { /* inodelk */
-                GF_FREE (local->cont.inodelk.volume);
-        }
+    { /* fsetxattr */
+        if (local->cont.fsetxattr.dict)
+            dict_unref(local->cont.fsetxattr.dict);
+    }
 
-        if (local->xdata_req)
-                dict_unref (local->xdata_req);
+    { /* removexattr */
+        GF_FREE(local->cont.removexattr.name);
+    }
+    { /* xattrop */
+        if (local->cont.xattrop.xattr)
+            dict_unref(local->cont.xattrop.xattr);
+    }
+    { /* symlink */
+        GF_FREE(local->cont.symlink.linkpath);
+    }
 
-        if (local->xdata_rsp)
-                dict_unref (local->xdata_rsp);
+    { /* opendir */
+        GF_FREE(local->cont.opendir.checksum);
+    }
+
+    { /* open */
+        if (local->cont.open.fd)
+            fd_unref(local->cont.open.fd);
+    }
+
+    { /* readdirp */
+        if (local->cont.readdir.dict)
+            dict_unref(local->cont.readdir.dict);
+    }
+
+    { /* inodelk */
+        GF_FREE(local->cont.inodelk.volume);
+        if (local->cont.inodelk.xdata)
+            dict_unref(local->cont.inodelk.xdata);
+    }
+
+    { /* entrylk */
+        GF_FREE(local->cont.entrylk.volume);
+        GF_FREE(local->cont.entrylk.basename);
+        if (local->cont.entrylk.xdata)
+            dict_unref(local->cont.entrylk.xdata);
+    }
+
+    if (local->xdata_req)
+        dict_unref(local->xdata_req);
+
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
 }
 
-
 int
-afr_frame_return (call_frame_t *frame)
+afr_frame_return(call_frame_t *frame)
 {
-        afr_local_t *local = NULL;
-        int          call_count = 0;
+    afr_local_t *local = NULL;
+    int call_count = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+    LOCK(&frame->lock);
+    {
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        return call_count;
+    return call_count;
 }
 
-static char *afr_ignore_xattrs[] = {
-        GLUSTERFS_OPEN_FD_COUNT,
-        GLUSTERFS_PARENT_ENTRYLK,
-        GLUSTERFS_ENTRYLK_COUNT,
-        GLUSTERFS_INODELK_COUNT,
-        GF_SELINUX_XATTR_KEY,
-        QUOTA_SIZE_KEY,
-        NULL
-};
+static char *afr_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
 
 gf_boolean_t
-afr_is_xattr_ignorable (char *key)
+afr_is_xattr_ignorable(char *key)
 {
-        int i = 0;
+    int i = 0;
 
-        if (!strncmp (key, AFR_XATTR_PREFIX, strlen(AFR_XATTR_PREFIX)))
-                return _gf_true;
-        for (i = 0; afr_ignore_xattrs[i]; i++) {
-                if (!strcmp (key, afr_ignore_xattrs[i]))
-                       return _gf_true;
-        }
-        return _gf_false;
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)))
+        return _gf_true;
+    for (i = 0; afr_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, afr_ignore_xattrs[i]))
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_xattr_match (dict_t *this, char *key1, data_t *value1, void *data)
+afr_xattr_match_needed(dict_t *this, char *key1, data_t *value1, void *data)
 {
-        if (!afr_is_xattr_ignorable (key1))
-                return _gf_true;
-
+    /* Ignore all non-disk (i.e. virtual) xattrs right away. */
+    if (!gf_is_valid_xattr_namespace(key1))
         return _gf_false;
+
+    /* Ignore on-disk xattrs that AFR doesn't need to heal. */
+    if (!afr_is_xattr_ignorable(key1))
+        return _gf_true;
+
+    return _gf_false;
 }
 
 gf_boolean_t
-afr_xattrs_are_equal (dict_t *dict1, dict_t *dict2)
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2)
 {
-        return are_dicts_equal (dict1, dict2, afr_xattr_match, NULL);
+    return are_dicts_equal(dict1, dict2, afr_xattr_match_needed, NULL);
 }
 
 static int
-afr_get_parent_read_subvol (xlator_t *this, inode_t *parent,
-                            struct afr_reply *replies, unsigned char *readable)
+afr_get_parent_read_subvol(xlator_t *this, inode_t *parent,
+                           struct afr_reply *replies, unsigned char *readable)
 {
-        int             i                    = 0;
-        int             par_read_subvol      = -1;
-        int             par_read_subvol_iter = -1;
-        afr_private_t  *priv                 = NULL;
-
-        priv = this->private;
-
-        if (parent)
-                par_read_subvol = afr_data_subvol_get (parent, this, NULL, NULL,
-                                                       NULL, NULL);
+    int i = 0;
+    int par_read_subvol = -1;
+    int par_read_subvol_iter = -1;
+    afr_private_t *priv = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    priv = this->private;
 
-                if (replies[i].op_ret < 0)
-                        continue;
+    if (parent)
+        par_read_subvol = afr_data_subvol_get(parent, this, NULL, NULL, NULL,
+                                              NULL);
 
-                if (par_read_subvol_iter == -1) {
-                        par_read_subvol_iter = i;
-                        continue;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if ((par_read_subvol_iter != par_read_subvol) && readable[i])
-                        par_read_subvol_iter = i;
+        if (replies[i].op_ret < 0)
+            continue;
 
-                if (i == par_read_subvol)
-                        par_read_subvol_iter = i;
+        if (par_read_subvol_iter == -1) {
+            par_read_subvol_iter = i;
+            continue;
         }
-        /* At the end of the for-loop, the only reason why @par_read_subvol_iter
-         * could be -1 is when this LOOKUP has failed on all sub-volumes.
-         * So it is okay to send an arbitrary subvolume (0 in this case)
-         * as parent read subvol.
-         */
-        if (par_read_subvol_iter == -1)
-                par_read_subvol_iter = 0;
 
-        return par_read_subvol_iter;
+        if ((par_read_subvol_iter != par_read_subvol) && readable[i])
+            par_read_subvol_iter = i;
 
+        if (i == par_read_subvol)
+            par_read_subvol_iter = i;
+    }
+    /* At the end of the for-loop, the only reason why @par_read_subvol_iter
+     * could be -1 is when this LOOKUP has failed on all sub-volumes.
+     * So it is okay to send an arbitrary subvolume (0 in this case)
+     * as parent read subvol.
+     */
+    if (par_read_subvol_iter == -1)
+        par_read_subvol_iter = 0;
+
+    return par_read_subvol_iter;
 }
 
 int
-afr_read_subvol_decide (inode_t *inode, xlator_t *this,
-                        afr_read_subvol_args_t *args)
+afr_read_subvol_decide(inode_t *inode, xlator_t *this,
+                       afr_read_subvol_args_t *args, unsigned char *readable)
 {
-        int data_subvol  = -1;
-        int mdata_subvol = -1;
+    int event = 0;
+    afr_private_t *priv = NULL;
+    unsigned char *intersection = NULL;
+
+    priv = this->private;
+    intersection = alloca0(priv->child_count);
 
-        data_subvol = afr_data_subvol_get (inode, this, NULL, NULL, NULL, args);
-        mdata_subvol = afr_metadata_subvol_get (inode, this,
-                                                NULL, NULL, NULL, args);
-        if (data_subvol == -1 || mdata_subvol == -1)
-                return -1;
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-        return data_subvol;
+    if (AFR_COUNT(intersection, priv->child_count) <= 0) {
+        /* TODO: If we have one brick with valid data_readable and
+         * another with metadata_readable, try to send an iatt with
+         * valid bits from both.*/
+        return -1;
+    }
+
+    memcpy(readable, intersection, sizeof(*readable) * priv->child_count);
+
+    return afr_read_subvol_select_by_policy(inode, this, intersection, args);
 }
 
 static inline int
-afr_first_up_child (call_frame_t *frame, xlator_t *this)
+afr_first_up_child(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-        int                  i     = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++)
-                if (local->replies[i].valid &&
-                    local->replies[i].op_ret == 0)
-                        return i;
-        return 0;
+    for (i = 0; i < priv->child_count; i++)
+        if (local->replies[i].valid && local->replies[i].op_ret == 0)
+            return i;
+    return -1;
 }
 
 static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-	int                 i = -1;
-	int                 op_errno = 0;
-	int                 read_subvol = 0;
-        int                 par_read_subvol = 0;
-	unsigned char      *readable = NULL;
-	int                 event = 0;
-	struct afr_reply   *replies = NULL;
-	uuid_t              read_gfid = {0, };
-	gf_boolean_t        locked_entry = _gf_false;
-	gf_boolean_t        can_interpret = _gf_true;
-        inode_t            *parent = NULL;
-        int                 spb_choice = -1;
-        ia_type_t           ia_type = IA_INVAL;
-        afr_read_subvol_args_t args = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-	replies = local->replies;
-        parent = local->loc.parent;
-
-	locked_entry = afr_is_possibly_under_txn (AFR_ENTRY_TRANSACTION, local,
-                                                  this);
-
-	readable = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_get (parent, this, readable, NULL, &event);
-
-        afr_inode_split_brain_choice_get (local->inode, this,
-                                                &spb_choice);
-	/* First, check if we have a gfid-change from somewhere,
-	   If so, propagate that so that a fresh lookup can be
-	   issued
-	*/
-        if (local->cont.lookup.needs_fresh_lookup) {
-                local->op_ret = -1;
-                local->op_errno = ESTALE;
-                goto unwind;
-        }
-
-	op_errno = afr_final_errno (frame->local, this->private);
-	local->op_errno = op_errno;
-
-	read_subvol = -1;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (locked_entry && replies[i].op_ret == -1 &&
-		    replies[i].op_errno == ENOENT) {
-			/* Second, check entry is still
-			   "underway" in creation */
-			local->op_ret = -1;
-			local->op_errno = ENOENT;
-			goto unwind;
-		}
-
-		if (replies[i].op_ret == -1)
-			continue;
-
-		if (read_subvol == -1 || !readable[read_subvol]) {
-			read_subvol = i;
-			gf_uuid_copy (read_gfid, replies[i].poststat.ia_gfid);
-                        ia_type = replies[i].poststat.ia_type;
-			local->op_ret = 0;
-		}
-	}
-
-	if (read_subvol == -1)
-		goto unwind;
-	/* We now have a read_subvol, which is readable[] (if there
-	   were any). Next we look for GFID mismatches. We don't
-	   consider a GFID mismatch as an error if read_subvol is
-	   readable[] but the mismatching GFID subvol is not.
-	*/
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid || replies[i].op_ret == -1) {
-			if (priv->child_up[i])
-				can_interpret = _gf_false;
-			continue;
-		}
-
-		if (!gf_uuid_compare (replies[i].poststat.ia_gfid, read_gfid))
-                        continue;
-
-		can_interpret = _gf_false;
-
-		if (locked_entry)
-			continue;
-
-		/* Now GFIDs mismatch. It's OK as long as this subvol
-		   is not readable[] but read_subvol is */
-		if (readable[read_subvol] && !readable[i])
-			continue;
-
-		/* LOG ERROR */
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto unwind;
-	}
-
-	/* Forth, for the finalized GFID, pick the best subvolume
-	   to return stats from.
-	*/
-	if (can_interpret) {
-		/* It is safe to call afr_replies_interpret() because we have
-		   a response from all the UP subvolumes and all of them resolved
-		   to the same GFID
-		*/
-                gf_uuid_copy (args.gfid, read_gfid);
-                args.ia_type = ia_type;
-		if (afr_replies_interpret (frame, this, local->inode, NULL)) {
-                        read_subvol = afr_read_subvol_decide (local->inode,
-                                                              this, &args);
-			afr_inode_read_subvol_reset (local->inode, this);
-			goto cant_interpret;
-		} else {
-                        read_subvol = afr_data_subvol_get (local->inode, this,
-                                                       NULL, NULL, NULL, &args);
-		}
-	} else {
-	cant_interpret:
-                if (read_subvol == -1) {
-                        if (spb_choice >= 0)
-                                read_subvol = spb_choice;
-                        else
-                                read_subvol = afr_first_up_child (frame, this);
-                }
-		dict_del (replies[read_subvol].xdata, GF_CONTENT_KEY);
-	}
+afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this,
+                           unsigned char *success_replies,
+                           unsigned char *data_readable, int *read_subvol)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = -1;
+    int child_count = -1;
 
-	afr_handle_quota_size (frame, this);
+    if (*read_subvol != -1)
+        return;
 
-unwind:
-        afr_set_need_heal (this, local);
-        if (read_subvol == -1) {
-                if (spb_choice >= 0)
-                        read_subvol = spb_choice;
-                else
-                        read_subvol = afr_first_up_child (frame, this);
+    priv = this->private;
+    local = frame->local;
+    child_count = priv->child_count;
+
+    afr_split_brain_read_subvol_get(local->inode, this, frame, &spb_subvol);
+    if ((spb_subvol >= 0) &&
+        (AFR_COUNT(success_replies, child_count) == child_count)) {
+        *read_subvol = spb_subvol;
+    } else if (!priv->quorum_count ||
+               frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) {
+        *read_subvol = afr_first_up_child(frame, this);
+    } else if (priv->quorum_count &&
+               afr_has_quorum(data_readable, this, NULL)) {
+        /* read_subvol is guaranteed to be valid if we hit this path. */
+        *read_subvol = afr_first_up_child(frame, this);
+    } else {
+        /* If quorum is enabled and we do not have a
+           readable yet, it means all good copies are down.
+        */
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "no read "
+               "subvols for %s",
+               local->loc.path);
+    }
+    if (*read_subvol >= 0)
+        dict_del_sizen(local->replies[*read_subvol].xdata, GF_CONTENT_KEY);
+}
+
+static void
+afr_lookup_done(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = -1;
+    int op_errno = 0;
+    int read_subvol = 0;
+    int par_read_subvol = 0;
+    int ret = -1;
+    unsigned char *readable = NULL;
+    unsigned char *success_replies = NULL;
+    int event = 0;
+    struct afr_reply *replies = NULL;
+    uuid_t read_gfid = {
+        0,
+    };
+    gf_boolean_t locked_entry = _gf_false;
+    gf_boolean_t in_flight_create = _gf_false;
+    gf_boolean_t can_interpret = _gf_true;
+    inode_t *parent = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+    char *gfid_heal_msg = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+    parent = local->loc.parent;
+
+    locked_entry = afr_is_possibly_under_txn(AFR_ENTRY_TRANSACTION, local,
+                                             this);
+
+    readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(parent, this, readable, NULL, &event);
+    par_read_subvol = afr_get_parent_read_subvol(this, parent, replies,
+                                                 readable);
+
+    /* First, check if we have a gfid-change from somewhere,
+       If so, propagate that so that a fresh lookup can be
+       issued
+    */
+    if (local->cont.lookup.needs_fresh_lookup) {
+        local->op_ret = -1;
+        local->op_errno = ESTALE;
+        goto error;
+    }
+
+    op_errno = afr_final_errno(frame->local, this->private);
+    local->op_errno = op_errno;
 
+    read_subvol = -1;
+    afr_fill_success_replies(local, priv, success_replies);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == -1) {
+            if (locked_entry && replies[i].op_errno == ENOENT) {
+                in_flight_create = _gf_true;
+            }
+            continue;
+        }
+
+        if (read_subvol == -1 || !readable[read_subvol]) {
+            read_subvol = i;
+            gf_uuid_copy(read_gfid, replies[i].poststat.ia_gfid);
+            ia_type = replies[i].poststat.ia_type;
+            local->op_ret = 0;
         }
-        par_read_subvol = afr_get_parent_read_subvol (this, parent, replies,
-                                                      readable);
-        if (AFR_IS_ARBITER_BRICK (priv, read_subvol) && local->op_ret == 0) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOTCONN;
+    }
+
+    if (in_flight_create && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = ENOENT;
+        goto error;
+    }
+
+    if (read_subvol == -1)
+        goto error;
+    /* We now have a read_subvol, which is readable[] (if there
+       were any). Next we look for GFID mismatches. We don't
+       consider a GFID mismatch as an error if read_subvol is
+       readable[] but the mismatching GFID subvol is not.
+    */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1) {
+            continue;
+        }
+
+        if (!gf_uuid_compare(replies[i].poststat.ia_gfid, read_gfid))
+            continue;
+
+        can_interpret = _gf_false;
+
+        if (locked_entry)
+            continue;
+
+        /* Now GFIDs mismatch. It's OK as long as this subvol
+           is not readable[] but read_subvol is */
+        if (readable[read_subvol] && !readable[i])
+            continue;
+
+        /* If we were called from glfsheal and there is still a gfid
+         * mismatch, succeed the lookup and let glfsheal print the
+         * response via gfid-heal-msg.*/
+        if (!dict_get_str_sizen(local->xattr_req, "gfid-heal-msg",
+                                &gfid_heal_msg))
+            goto cant_interpret;
+
+        /* LOG ERROR */
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        goto error;
+    }
+
+    /* Forth, for the finalized GFID, pick the best subvolume
+       to return stats from.
+    */
+    read_subvol = -1;
+    memset(readable, 0, sizeof(*readable) * priv->child_count);
+    if (can_interpret) {
+        if (!afr_has_quorum(success_replies, this, NULL))
+            goto cant_interpret;
+        /* It is safe to call afr_replies_interpret() because we have
+           a response from all the UP subvolumes and all of them resolved
+           to the same GFID
+        */
+        gf_uuid_copy(args.gfid, read_gfid);
+        args.ia_type = ia_type;
+        ret = afr_replies_interpret(frame, this, local->inode, NULL);
+        read_subvol = afr_read_subvol_decide(local->inode, this, &args,
+                                             readable);
+        if (read_subvol == -1)
+            goto cant_interpret;
+        if (ret) {
+            afr_inode_need_refresh_set(local->inode, this);
+            dict_del_sizen(local->replies[read_subvol].xdata, GF_CONTENT_KEY);
+        }
+    } else {
+    cant_interpret:
+        afr_attempt_readsubvol_set(frame, this, success_replies, readable,
+                                   &read_subvol);
+        if (read_subvol == -1) {
+            goto error;
         }
+    }
 
-	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-			  local->inode, &local->replies[read_subvol].poststat,
-			  local->replies[read_subvol].xdata,
-			  &local->replies[par_read_subvol].postparent);
+    afr_handle_quota_size(frame, this);
+
+    afr_set_need_heal(this, local);
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+        goto error;
+    }
+
+    ret = dict_get_str_sizen(local->xattr_req, "gfid-heal-msg", &gfid_heal_msg);
+    if (!ret) {
+        ret = dict_set_str_sizen(local->replies[read_subvol].xdata,
+                                 "gfid-heal-msg", gfid_heal_msg);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting gfid-heal-msg dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        }
+    }
+
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[par_read_subvol].postparent);
+    return;
+
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
 /*
@@ -1828,651 +3122,845 @@ unwind:
  * others in that they must be given higher priority while
  * returning to the user.
  *
- * The hierarchy is ENODATA > ENOENT > ESTALE > others
+ * The hierarchy is ENODATA > ENOENT > ESTALE > ENOSPC others
  */
 
 int
-afr_higher_errno (int32_t old_errno, int32_t new_errno)
+afr_higher_errno(int32_t old_errno, int32_t new_errno)
 {
-	if (old_errno == ENODATA || new_errno == ENODATA)
-		return ENODATA;
-        if (old_errno == ENOENT || new_errno == ENOENT)
-                return ENOENT;
-	if (old_errno == ESTALE || new_errno == ESTALE)
-		return ESTALE;
+    if (old_errno == ENODATA || new_errno == ENODATA)
+        return ENODATA;
+    if (old_errno == ENOENT || new_errno == ENOENT)
+        return ENOENT;
+    if (old_errno == ESTALE || new_errno == ESTALE)
+        return ESTALE;
+    if (old_errno == ENOSPC || new_errno == ENOSPC)
+        return ENOSPC;
 
-	return new_errno;
+    return new_errno;
 }
 
-
 int
-afr_final_errno (afr_local_t *local, afr_private_t *priv)
+afr_final_errno(afr_local_t *local, afr_private_t *priv)
 {
-	int i = 0;
-	int op_errno = 0;
-	int tmp_errno = 0;
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret >= 0)
-			continue;
-		tmp_errno = local->replies[i].op_errno;
-		op_errno = afr_higher_errno (op_errno, tmp_errno);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret >= 0)
+            continue;
+        tmp_errno = local->replies[i].op_errno;
+        op_errno = afr_higher_errno(op_errno, tmp_errno);
+    }
 
-	return op_errno;
+    return op_errno;
 }
 
 static int32_t
-afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno, dict_t *dict,
-			 dict_t *xdata)
-{
-        int              ret            = 0;
-        char            *pathinfo       = NULL;
-        gf_boolean_t     is_local        = _gf_false;
-        afr_private_t   *priv           = NULL;
-        int32_t          child_index    = -1;
-
-        if (op_ret != 0) {
-                goto out;
-        }
-
-	priv = this->private;
-	child_index = (int32_t)(long)cookie;
-
-        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
-        if (ret != 0) {
-                goto out;
-        }
-
-        ret = glusterfs_is_local_pathinfo (pathinfo, &is_local);
-        if (ret) {
-                goto out;
-        }
-
-        /*
-         * Note that one local subvolume will override another here.  The only
-         * way to avoid that would be to retain extra information about whether
-         * the previous read_child is local, and it's just not worth it.  Even
-         * the slowest local subvolume is far preferable to a remote one.
-         */
-        if (is_local) {
-                priv->local[child_index] = 1;
-                /* Don't set arbiter as read child. */
-                if (AFR_IS_ARBITER_BRICK(priv, child_index))
-                        goto out;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        AFR_MSG_LOCAL_CHILD, "selecting local read_child %s",
-                        priv->children[child_index]->name);
-
-                priv->read_child = child_index;
-        }
+afr_local_discovery_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
+{
+    int ret = 0;
+    char *pathinfo = NULL;
+    gf_boolean_t is_local = _gf_false;
+    afr_private_t *priv = NULL;
+    int32_t child_index = -1;
+
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    priv = this->private;
+    child_index = (int32_t)(long)cookie;
+
+    ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+    if (ret != 0) {
+        goto out;
+    }
+
+    ret = glusterfs_is_local_pathinfo(pathinfo, &is_local);
+    if (ret) {
+        goto out;
+    }
+
+    /*
+     * Note that one local subvolume will override another here.  The only
+     * way to avoid that would be to retain extra information about whether
+     * the previous read_child is local, and it's just not worth it.  Even
+     * the slowest local subvolume is far preferable to a remote one.
+     */
+    if (is_local) {
+        priv->local[child_index] = 1;
+        /* Don't set arbiter as read child. */
+        if (AFR_IS_ARBITER_BRICK(priv, child_index))
+            goto out;
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_LOCAL_CHILD,
+               "selecting local read_child %s",
+               priv->children[child_index]->name);
+
+        priv->read_child = child_index;
+    }
 out:
-        STACK_DESTROY(frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static void
-afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
+afr_attempt_local_discovery(xlator_t *this, int32_t child_index)
 {
-        call_frame_t    *newframe = NULL;
-        loc_t            tmploc = {0,};
-        afr_private_t   *priv = this->private;
+    call_frame_t *newframe = NULL;
+    loc_t tmploc = {
+        0,
+    };
+    afr_private_t *priv = this->private;
 
-        newframe = create_frame(this,this->ctx->pool);
-        if (!newframe) {
-                return;
-        }
+    newframe = create_frame(this, this->ctx->pool);
+    if (!newframe) {
+        return;
+    }
 
-        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
-        STACK_WIND_COOKIE (newframe, afr_local_discovery_cbk,
-                           (void *)(long)child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->getxattr,
-                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+    tmploc.gfid[sizeof(tmploc.gfid) - 1] = 1;
+    STACK_WIND_COOKIE(newframe, afr_local_discovery_cbk,
+                      (void *)(long)child_index, priv->children[child_index],
+                      priv->children[child_index]->fops->getxattr, &tmploc,
+                      GF_XATTR_PATHINFO_KEY, NULL);
 }
 
 int
-afr_lookup_sh_metadata_wrap (void *opaque)
-{
-        call_frame_t *frame       = opaque;
-        afr_local_t  *local       = NULL;
-        xlator_t     *this        = NULL;
-        inode_t      *inode       = NULL;
-        afr_private_t *priv       = NULL;
-        struct afr_reply *replies = NULL;
-        int i= 0, first = -1;
-        int ret = -1;
-        dict_t *dict = NULL;
+afr_lookup_sh_metadata_wrap(void *opaque)
+{
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    replies = local->replies;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        first = i;
+        break;
+    }
+    if (first == -1)
+        goto out;
+
+    if (afr_selfheal_metadata_by_stbuf(this, &replies[first].poststat))
+        goto out;
+
+    afr_local_replies_wipe(local, this->private);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    if (local->xattr_req) {
+        dict_copy(local->xattr_req, dict);
+    }
+
+    ret = dict_set_sizen_str_sizen(dict, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    if (loc_is_nameless(&local->loc)) {
+        ret = afr_selfheal_unlocked_discover_on(frame, local->inode,
+                                                local->loc.gfid, local->replies,
+                                                local->child_up, dict);
+    } else {
+        inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                                local->loc.name, local->replies,
+                                                local->child_up, dict);
+    }
+    if (inode)
+        inode_unref(inode);
+out:
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
 
-        local = frame->local;
-        this  = frame->this;
-        priv  = this->private;
-        replies = local->replies;
-
-        for (i =0; i < priv->child_count; i++) {
-                if(!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                first = i;
-                break;
-        }
-        if (first == -1)
-                goto out;
+    if (dict)
+        dict_unref(dict);
 
-        if (afr_selfheal_metadata_by_stbuf (this, &replies[first].poststat))
-                goto out;
+    return 0;
+}
 
-        afr_local_replies_wipe (local, this->private);
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type)
+{
+    int idx = -1;
+    afr_private_t *priv = NULL;
+    void *pending_raw = NULL;
+    int *pending_int = NULL;
+    int i = 0;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-        ret = dict_set_str (dict, "link-count", GF_XATTROP_INDEX_COUNT);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set link-count in dict ");
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
+
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw) == 0) {
+        if (pending_raw) {
+            pending_int = pending_raw;
+
+            if (ntoh32(pending_int[idx]))
+                return _gf_true;
         }
+    }
 
-        inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
-                                                 local->loc.name, local->replies,
-                                                 local->child_up, dict);
-        if (inode)
-                inode_unref (inode);
-out:
-        afr_lookup_done (frame, this);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+        if (!pending_raw)
+            continue;
+        pending_int = pending_raw;
 
-        if (dict)
-                dict_unref (dict);
+        if (ntoh32(pending_int[idx]))
+            return _gf_true;
+    }
 
-        return 0;
+    return _gf_false;
 }
 
 static gf_boolean_t
 afr_can_start_metadata_self_heal(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        struct afr_reply *replies = NULL;
-        int i = 0, first = -1;
-        gf_boolean_t start = _gf_false;
-        struct iatt stbuf = {0, };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t start = _gf_false;
+    struct iatt stbuf = {
+        0,
+    };
 
-        local = frame->local;
-        replies = local->replies;
-        priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
 
-        if (!priv->metadata_self_heal)
-                return _gf_false;
+    if (!priv->metadata_self_heal)
+        return _gf_false;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if(!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (first == -1) {
-                        first = i;
-                        stbuf = replies[i].poststat;
-                        continue;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (first == -1) {
+            first = i;
+            stbuf = replies[i].poststat;
+            continue;
+        }
 
-                if (gf_uuid_compare (stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
-                        start = _gf_false;
-                        break;
-                }
-                if (!IA_EQUAL (stbuf, replies[i].poststat, type)) {
-                        start = _gf_false;
-                        break;
-                }
+        if (afr_is_pending_set(this, replies[i].xdata,
+                               AFR_METADATA_TRANSACTION)) {
+            /* Let shd do the heal so that lookup is not blocked
+             * on getting metadata lock/doing the heal */
+            start = _gf_false;
+            break;
+        }
 
-                /*Check if iattrs need heal*/
-                if ((!IA_EQUAL (stbuf, replies[i].poststat, uid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, gid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, prot))) {
-                        start = _gf_true;
-                        continue;
-                }
+        if (gf_uuid_compare(stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
+            start = _gf_false;
+            break;
+        }
+        if (!IA_EQUAL(stbuf, replies[i].poststat, type)) {
+            start = _gf_false;
+            break;
+        }
 
-                /*Check if xattrs need heal*/
-                if (!afr_xattrs_are_equal (replies[first].xdata,
-                                           replies[i].xdata))
-                        start = _gf_true;
+        /*Check if iattrs need heal*/
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot))) {
+            start = _gf_true;
+            continue;
         }
 
-        return start;
+        /*Check if xattrs need heal*/
+        if (!afr_xattrs_are_equal(replies[first].xdata, replies[i].xdata))
+            start = _gf_true;
+    }
+
+    return start;
 }
 
 int
-afr_lookup_metadata_heal_check (call_frame_t *frame, xlator_t *this)
+afr_lookup_metadata_heal_check(call_frame_t *frame, xlator_t *this)
 
 {
-        call_frame_t *heal = NULL;
-        int ret            = 0;
+    call_frame_t *heal = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
 
-        if (!afr_can_start_metadata_self_heal (frame, this))
-                goto out;
+    local = frame->local;
+    if (!afr_can_start_metadata_self_heal(frame, this))
+        goto out;
 
-        heal = copy_frame (frame);
-        if (heal)
-                heal->root->pid = GF_CLIENT_PID_SELF_HEALD;
-        ret = synctask_new (this->ctx->env, afr_lookup_sh_metadata_wrap,
-                            afr_refresh_selfheal_done, heal, frame);
-        if(ret)
-                goto out;
-        return ret;
+    heal = afr_frame_create(this, &ret);
+    if (!heal) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = synctask_new(this->ctx->env, afr_lookup_sh_metadata_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret)
+        goto out;
+    return ret;
 out:
-        afr_lookup_done (frame, this);
-        return ret;
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return ret;
 }
 
 int
-afr_lookup_selfheal_wrap (void *opaque)
+afr_lookup_selfheal_wrap(void *opaque)
 {
-        int ret = 0;
-	call_frame_t *frame = opaque;
-	afr_local_t *local = NULL;
-	xlator_t *this = NULL;
-	inode_t *inode = NULL;
+    int ret = 0;
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    uuid_t pargfid = {
+        0,
+    };
 
-	local = frame->local;
-	this = frame->this;
+    local = frame->local;
+    this = frame->this;
+    loc_pargfid(&local->loc, pargfid);
 
-	ret = afr_selfheal_name (frame->this, local->loc.pargfid,
-                                 local->loc.name, &local->cont.lookup.gfid_req);
-        if (ret == -EIO)
-                goto unwind;
+    ret = afr_selfheal_name(frame->this, pargfid, local->loc.name,
+                            &local->cont.lookup.gfid_req, local->xattr_req);
+    if (ret == -EIO)
+        goto unwind;
 
-        afr_local_replies_wipe (local, this->private);
+    afr_local_replies_wipe(local, this->private);
 
-	inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
-						 local->loc.name, local->replies,
-						 local->child_up, NULL);
-	if (inode)
-		inode_unref (inode);
+    inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                            local->loc.name, local->replies,
+                                            local->child_up, local->xattr_req);
+    if (inode)
+        inode_unref(inode);
 
-        afr_lookup_metadata_heal_check(frame, this);
-        return 0;
+    afr_lookup_metadata_heal_check(frame, this);
+    return 0;
 
 unwind:
-	AFR_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	call_frame_t *heal = NULL;
-	int i = 0, first = -1;
-	gf_boolean_t need_heal = _gf_false;
-	struct afr_reply *replies = NULL;
-	int ret = 0;
-
-	local = frame->local;
-	replies = local->replies;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        need_heal = _gf_true;
-
-		if (first == -1) {
-			first = i;
-			continue;
-		}
-
-		if (replies[i].op_ret != replies[first].op_ret) {
-			need_heal = _gf_true;
-			break;
-		}
-
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first].poststat.ia_gfid)) {
-			need_heal = _gf_true;
-			break;
-		}
-	}
-
-	if (need_heal) {
-		heal = copy_frame (frame);
-		if (heal)
-			heal->root->pid = GF_CLIENT_PID_SELF_HEALD;
-		ret = synctask_new (this->ctx->env, afr_lookup_selfheal_wrap,
-				    afr_refresh_selfheal_done, heal, frame);
-		if (ret)
-			goto metadata_heal;
-                return ret;
-	}
-metadata_heal:
-        ret = afr_lookup_metadata_heal_check (frame, this);
-
-	return ret;
-}
+afr_lookup_entry_heal(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    call_frame_t *heal = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t name_state_mismatch = _gf_false;
+    struct afr_reply *replies = NULL;
+    int ret = 0;
+    unsigned char *par_readables = NULL;
+    unsigned char *success = NULL;
+    int32_t op_errno = 0;
+    uuid_t gfid = {0};
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+    par_readables = alloca0(priv->child_count);
+    success = alloca0(priv->child_count);
+
+    ret = afr_inode_read_subvol_get(local->loc.parent, this, par_readables,
+                                    NULL, NULL);
+    if (ret < 0 || AFR_COUNT(par_readables, priv->child_count) == 0) {
+        /* In this case set par_readables to all 1 so that name_heal
+         * need checks at the end of this function will flag missing
+         * entry when name state mismatches*/
+        memset(par_readables, 1, priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == 0) {
+            if (gf_uuid_is_null(gfid)) {
+                gf_uuid_copy(gfid, replies[i].poststat.ia_gfid);
+            }
+            success[i] = 1;
+        } else {
+            if ((replies[i].op_errno != ENOTCONN) &&
+                (replies[i].op_errno != ENOENT) &&
+                (replies[i].op_errno != ESTALE)) {
+                op_errno = replies[i].op_errno;
+            }
+        }
 
+        /*gfid is missing, needs heal*/
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            goto name_heal;
+        }
 
-int
-afr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-		dict_t *xdata, struct iatt *postparent)
-{
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        GF_UNUSED int   ret             = 0;
-	int8_t need_heal                = 1;
-
-	child_index = (long) cookie;
-
-	local = frame->local;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-        /*
-         * On revalidate lookup if the gfid-changed, afr should unwind the fop
-         * with ESTALE so that a fresh lookup will be sent by the top xlator.
-         * So remember it.
-         */
-        if (xdata && dict_get (xdata, "gfid-changed"))
-                local->cont.lookup.needs_fresh_lookup = _gf_true;
+        if (first == -1) {
+            first = i;
+            continue;
+        }
 
-        if (xdata) {
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[child_index].need_heal = need_heal;
-        } else {
-                local->replies[child_index].need_heal = need_heal;
+        if (replies[i].op_ret != replies[first].op_ret) {
+            name_state_mismatch = _gf_true;
         }
-	if (op_ret != -1) {
-		local->replies[child_index].poststat = *buf;
-		local->replies[child_index].postparent = *postparent;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-	}
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                afr_set_need_heal (this, local);
-		afr_lookup_entry_heal (frame, this);
+        if (replies[i].op_ret == 0) {
+            /* Rename after this lookup may succeed if we don't do
+             * a name-heal and the destination may not have pending xattrs
+             * to indicate which name is good and which is bad so always do
+             * this heal*/
+            if (gf_uuid_compare(replies[i].poststat.ia_gfid, gfid)) {
+                goto name_heal;
+            }
         }
+    }
+
+    if (name_state_mismatch) {
+        if (!priv->quorum_count)
+            goto name_heal;
+        if (!afr_has_quorum(success, this, NULL))
+            goto name_heal;
+        if (op_errno)
+            goto name_heal;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!replies[i].valid)
+                continue;
+            if (par_readables[i] && replies[i].op_ret < 0 &&
+                replies[i].op_errno != ENOTCONN) {
+                goto name_heal;
+            }
+        }
+    }
+
+    goto metadata_heal;
+
+name_heal:
+    heal = afr_frame_create(this, NULL);
+    if (!heal)
+        goto metadata_heal;
+
+    ret = synctask_new(this->ctx->env, afr_lookup_selfheal_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret) {
+        AFR_STACK_DESTROY(heal);
+        goto metadata_heal;
+    }
+    return ret;
 
-	return 0;
+metadata_heal:
+    ret = afr_lookup_metadata_heal_check(frame, this);
+
+    return ret;
 }
 
+int
+afr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+               struct iatt *postparent)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    /*
+     * On revalidate lookup if the gfid-changed, afr should unwind the fop
+     * with ESTALE so that a fresh lookup will be sent by the top xlator.
+     * So remember it.
+     */
+    if (xdata && dict_get_sizen(xdata, "gfid-changed"))
+        local->cont.lookup.needs_fresh_lookup = _gf_true;
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_entry_heal(frame, this);
+    }
 
+    return 0;
+}
 
 static void
-afr_discover_done (call_frame_t *frame, xlator_t *this)
+afr_discover_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-	int                 i = -1;
-	int                 op_errno = 0;
-	int                 spb_choice = -1;
-	int                 read_subvol = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    int ret = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *success_replies = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
+    data_readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
 
-        afr_inode_split_brain_choice_get (local->inode, this,
-                                          &spb_choice);
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) > 0)
+        local->op_ret = 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret == 0)
-			local->op_ret = 0;
-	}
+    if (local->op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(frame->local, this->private);
+        goto error;
+    }
 
-	op_errno = afr_final_errno (frame->local, this->private);
+    if (!afr_has_quorum(success_replies, this, frame))
+        goto unwind;
 
-        if (local->op_ret < 0) {
-		local->op_errno = op_errno;
-		local->op_ret = -1;
-                goto unwind;
-	}
+    ret = afr_replies_interpret(frame, this, local->inode, NULL);
+    if (ret) {
+        afr_inode_need_refresh_set(local->inode, this);
+    }
 
-	afr_replies_interpret (frame, this, local->inode, NULL);
+    read_subvol = afr_read_subvol_decide(local->inode, this, NULL,
+                                         data_readable);
 
-	read_subvol = afr_read_subvol_decide (local->inode, this, NULL);
-	if (read_subvol == -1) {
-	        gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_READ_SUBVOL_ERROR, "no read subvols for %s",
-			local->loc.path);
+unwind:
+    afr_attempt_readsubvol_set(frame, this, success_replies, data_readable,
+                               &read_subvol);
+    if (read_subvol == -1)
+        goto error;
 
-                if (spb_choice >= 0) {
-                        read_subvol = spb_choice;
-                } else {
-                        read_subvol = afr_first_up_child (frame, this);
-                }
-	}
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+    }
 
-unwind:
-	if (read_subvol == -1) {
-                if (spb_choice >= 0)
-                        read_subvol = spb_choice;
-                else
-                        read_subvol = afr_first_up_child (frame, this);
-        }
-        if (AFR_IS_ARBITER_BRICK (priv, read_subvol) && local->op_ret == 0) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOTCONN;
-        }
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[read_subvol].postparent);
+    return;
 
-	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-			  local->inode, &local->replies[read_subvol].poststat,
-			  local->replies[read_subvol].xdata,
-			  &local->replies[read_subvol].postparent);
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
+static int
+afr_ta_id_file_check(void *opaque)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *dict = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    this = opaque;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], &loc, &stbuf,
+                        0, 0, 0);
+    if (ret == 0) {
+        goto out;
+    } else if (ret == -ENOENT) {
+        fd = fd_create(loc.inode, getpid());
+        if (!fd)
+            goto out;
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        gf_uuid_generate(gfid);
+        ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
+        ret = syncop_create(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                            O_RDWR, 0664, fd, &stbuf, dict, NULL);
+    }
 
-int
-afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-		  dict_t *xdata, struct iatt *postparent)
-{
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        GF_UNUSED int ret               = 0;
-	int8_t need_heal                = 1;
-
-	child_index = (long) cookie;
-
-	local = frame->local;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-	if (op_ret != -1) {
-		local->replies[child_index].poststat = *buf;
-		local->replies[child_index].postparent = *postparent;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-	}
-
-        if (local->do_discovery && (op_ret == 0))
-                afr_attempt_local_discovery (this, child_index);
-
-        if (xdata) {
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[child_index].need_heal = need_heal;
-        } else {
-                local->replies[child_index].need_heal = need_heal;
-        }
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-               afr_set_need_heal (this, local);
-               afr_discover_done (frame, this);
-        }
+out:
+    if (ret == 0) {
+        gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to lookup/create thin-arbiter id file.");
+    }
+    if (dict)
+        dict_unref(dict);
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
 
-	return 0;
+    return 0;
 }
 
+static int
+afr_ta_id_file_check_cbk(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    return 0;
+}
 
-int
-afr_discover_do (call_frame_t *frame, xlator_t *this, int err)
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this)
 {
-	int ret = 0;
-	int i = 0;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
+    int ret = 0;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    priv = this->private;
+    if (!priv->thin_arbiter_count)
+        goto unwind;
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto unwind;
 
-	if (err) {
-		local->op_errno = -err;
-		ret = -1;
-		goto out;
-	}
+    ret = synctask_new(this->ctx->env, afr_ta_id_file_check,
+                       afr_ta_id_file_check_cbk, NULL, this);
+    if (ret)
+        goto unwind;
+unwind:
+    afr_discover_unwind(frame, this);
+}
 
-	call_count = local->call_count = AFR_COUNT (local->child_up,
-						    priv->child_count);
+int
+afr_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+                 struct iatt *postparent)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    if (local->do_discovery && (op_ret == 0))
+        afr_attempt_local_discovery(this, child_index);
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_metadata_heal_check(frame, this);
+    }
 
-        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
-					    &local->loc);
-        if (ret) {
-                local->op_errno = -ret;
-		ret = -1;
-                goto out;
-        }
+    return 0;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_discover_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+int
+afr_discover_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_discover_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
 }
 
-
 int
-afr_discover (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+afr_discover(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-	int op_errno = ENOMEM;
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int event = 0;
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int event = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        if (!local->call_count) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-	if (__is_root_gfid (loc->inode->gfid)) {
-		if (!this->itable)
-			this->itable = loc->inode->table;
-		if (!priv->root_inode)
-			priv->root_inode = inode_ref (loc->inode);
+    if (__is_root_gfid(loc->inode->gfid)) {
+        if (!priv->root_inode)
+            priv->root_inode = inode_ref(loc->inode);
 
-		if (priv->choose_local && !priv->did_discovery) {
-			/* Logic to detect which subvolumes of AFR are
-			   local, in order to prefer them for reads
-			*/
-			local->do_discovery = _gf_true;
-                        priv->did_discovery = _gf_true;
-                }
-	}
+        if (priv->choose_local && !priv->did_discovery) {
+            /* Logic to detect which subvolumes of AFR are
+               local, in order to prefer them for reads
+            */
+            local->do_discovery = _gf_true;
+            priv->did_discovery = _gf_true;
+        }
+    }
 
-        local->op = GF_FOP_LOOKUP;
+    local->op = GF_FOP_LOOKUP;
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-	local->inode = inode_ref (loc->inode);
+    local->inode = inode_ref(loc->inode);
 
-	if (xattr_req)
-		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
-		   allocate one for us */
-		local->xattr_req = dict_ref (xattr_req);
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+    }
 
-	if (gf_uuid_is_null (loc->inode->gfid)) {
-		afr_discover_do (frame, this, 0);
-		return 0;
-	}
+    if (gf_uuid_is_null(loc->inode->gfid)) {
+        afr_discover_do(frame, this, 0);
+        return 0;
+    }
 
-	afr_read_subvol_get (loc->inode, this, NULL, NULL, &event,
-			     AFR_DATA_TRANSACTION, NULL);
+    afr_read_subvol_get(loc->inode, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-	if (event != local->event_generation)
-		afr_inode_refresh (frame, this, loc->inode, NULL,
-                                   afr_discover_do);
-	else
-		afr_discover_do (frame, this, 0);
+    afr_discover_do(frame, this, 0);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-afr_lookup_do (call_frame_t *frame, xlator_t *this, int err)
-{
-	int ret = 0;
-	int i = 0;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (err < 0) {
-		local->op_errno = -err;
-		ret = -1;
-		goto out;
-	}
-
-	call_count = local->call_count = AFR_COUNT (local->child_up,
-						    priv->child_count);
-
-        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
-					    &local->loc);
-        if (ret) {
-                local->op_errno = -ret;
-		ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+afr_lookup_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err < 0) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_lookup_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
         }
-	return 0;
+    }
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
 }
 
 /*
@@ -2512,1400 +4000,1721 @@ out:
  */
 
 int
-afr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
-{
-        afr_local_t   *local = NULL;
-        int32_t        op_errno = 0;
-	int            event = 0;
-        void          *gfid_req = NULL;
-        int            ret = 0;
-
-	if (!loc->parent && gf_uuid_is_null (loc->pargfid)) {
-                if (xattr_req)
-                        dict_del (xattr_req, "gfid-req");
-		afr_discover (frame, this, loc, xattr_req);
-		return 0;
-	}
-
-	if (__is_root_gfid (loc->parent->gfid)) {
-		if (!strcmp (loc->name, GF_REPLICATE_TRASH_DIR)) {
-			op_errno = EPERM;
-			goto out;
-		}
-	}
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        if (!local->call_count) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        local->op = GF_FOP_LOOKUP;
-
-        loc_copy (&local->loc, loc);
-
-	local->inode = inode_ref (loc->inode);
-
-	if (xattr_req) {
-		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
-		   allocate one for us */
-		local->xattr_req = dict_copy_with_ref (xattr_req, NULL);
-		if (!local->xattr_req) {
-		        op_errno = ENOMEM;
-		        goto out;
-                }
-                ret = dict_get_ptr (local->xattr_req, "gfid-req", &gfid_req);
-                if (ret == 0) {
-                        gf_uuid_copy (local->cont.lookup.gfid_req, gfid_req);
-                        dict_del (local->xattr_req, "gfid-req");
-                }
-        }
+afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int event = 0;
+    int ret = 0;
 
-	afr_read_subvol_get (loc->parent, this, NULL, NULL, &event,
-			     AFR_DATA_TRANSACTION, NULL);
+    if (loc_is_nameless(loc)) {
+        if (xattr_req)
+            dict_del_sizen(xattr_req, "gfid-req");
+        afr_discover(frame, this, loc, xattr_req);
+        return 0;
+    }
 
-	if (event != local->event_generation)
-		afr_inode_refresh (frame, this, loc->parent, NULL,
-                                   afr_lookup_do);
-	else
-		afr_lookup_do (frame, this, 0);
+    if (afr_is_private_directory(this->private, loc->parent->gfid, loc->name,
+                                 frame->root->pid)) {
+        op_errno = EPERM;
+        goto out;
+    }
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        return 0;
-}
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
+    local->op = GF_FOP_LOOKUP;
 
-/* {{{ open */
+    loc_copy(&local->loc, loc);
 
-afr_fd_ctx_t *
-__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
-{
-        uint64_t       ctx = 0;
-        int            ret = 0;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+    local->inode = inode_ref(loc->inode);
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = dict_get_gfuuid(local->xattr_req, "gfid-req",
+                              &local->cont.lookup.gfid_req);
+        if (ret == 0) {
+            dict_del_sizen(local->xattr_req, "gfid-req");
+        }
+    }
 
-        if (ret < 0) {
-                ret = __afr_fd_ctx_set (this, fd);
-                if (ret < 0)
-                        goto out;
+    afr_read_subvol_get(loc->parent, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-                ret = __fd_ctx_get (fd, this, &ctx);
-                if (ret < 0)
-                        goto out;
-        }
+    afr_lookup_do(frame, this, 0);
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    return 0;
 out:
-        return fd_ctx;
-}
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
+    return 0;
+}
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+void
+_afr_cleanup_fd_ctx(xlator_t *this, afr_fd_ctx_t *fd_ctx)
 {
-        afr_fd_ctx_t  *fd_ctx = NULL;
+    afr_private_t *priv = this->private;
 
-        LOCK(&fd->lock);
+    if (fd_ctx->lk_heal_info) {
+        LOCK(&priv->lock);
         {
-                fd_ctx = __afr_fd_ctx_get (fd, this);
+            list_del(&fd_ctx->lk_heal_info->pos);
         }
-        UNLOCK(&fd->lock);
-
-        return fd_ctx;
+        afr_lk_heal_info_cleanup(fd_ctx->lk_heal_info);
+        fd_ctx->lk_heal_info = NULL;
+    }
+    GF_FREE(fd_ctx->opened_on);
+    GF_FREE(fd_ctx);
+    return;
 }
 
-
 int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd)
 {
-        afr_private_t * priv   = NULL;
-        int             ret    = -1;
-        uint64_t        ctx    = 0;
-        afr_fd_ctx_t *  fd_ctx = NULL;
-	int             i = 0;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = 0;
 
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
+    ret = fd_ctx_get(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        priv = this->private;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+    if (fd_ctx) {
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    }
 
-        if (ret == 0)
-                goto out;
-
-        fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
-                            gf_afr_mt_afr_fd_ctx_t);
-        if (!fd_ctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-	for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
-		fd_ctx->pre_op_done[i] = GF_CALLOC (sizeof (*fd_ctx->pre_op_done[i]),
-						    priv->child_count,
-						    gf_afr_mt_int32_t);
-		if (!fd_ctx->pre_op_done[i]) {
-			ret = -ENOMEM;
-			goto out;
-		}
-	}
-
-        fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
-                                       priv->child_count,
-                                       gf_afr_mt_int32_t);
-        if (!fd_ctx->opened_on) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (fd_is_anonymous (fd))
-			fd_ctx->opened_on[i] = AFR_FD_OPENED;
-		else
-			fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
-	}
-
-        fd_ctx->lock_piggyback = GF_CALLOC (sizeof (*fd_ctx->lock_piggyback),
-                                            priv->child_count,
-                                            gf_afr_mt_char);
-        if (!fd_ctx->lock_piggyback) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        fd_ctx->lock_acquired = GF_CALLOC (sizeof (*fd_ctx->lock_acquired),
-                                           priv->child_count,
-                                           gf_afr_mt_char);
-        if (!fd_ctx->lock_acquired) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-	fd_ctx->readdir_subvol = -1;
-
-	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
-
-        INIT_LIST_HEAD (&fd_ctx->eager_locked);
-
-        ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
-        if (ret)
-                gf_msg_debug (this->name, 0,
-                              "failed to set fd ctx (%p)", fd);
 out:
-        return ret;
+    return 0;
 }
 
-
 int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+afr_release(xlator_t *this, fd_t *fd)
 {
-        int ret = -1;
-
-        LOCK (&fd->lock);
-        {
-                ret = __afr_fd_ctx_set (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    afr_cleanup_fd_ctx(this, fd);
 
-        return ret;
+    return 0;
 }
 
-/* {{{ flush */
-
-int
-afr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-			local->op_ret = op_ret;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		} else {
-			local->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		AFR_STACK_UNWIND (flush, frame, local->op_ret,
-				  local->op_errno, local->xdata_rsp);
-
-        return 0;
-}
-
-static int
-afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+afr_fd_ctx_t *
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this)
 {
-        int           i      = 0;
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int call_count       = -1;
+    uint64_t ctx = 0;
+    int ret = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        priv = this->private;
-        local = frame->local;
-        call_count = local->call_count;
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->flush,
-                                           local->fd, xdata);
-                        if (!--call_count)
-                                break;
+    if (ret < 0) {
+        ret = __afr_fd_ctx_set(this, fd);
+        if (ret < 0)
+            goto out;
 
-                }
-        }
+        ret = __fd_ctx_get(fd, this, &ctx);
+        if (ret < 0)
+            goto out;
+    }
 
-        return 0;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
+out:
+    return fd_ctx;
 }
 
-int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+afr_fd_ctx_t *
+afr_fd_ctx_get(fd_t *fd, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        call_stub_t   *stub = NULL;
-        int            op_errno   = ENOMEM;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	if (!local->call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-	local->fd = fd_ref(fd);
-
-        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
-        if (!stub)
-                goto out;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        afr_delayed_changelog_wake_resume (this, fd, stub);
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+    }
+    UNLOCK(&fd->lock);
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
+    return fd_ctx;
 }
 
-/* }}} */
-
-
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd)
 {
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
-	int             i = 0;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int i = 0;
 
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
+    VALIDATE_OR_GOTO(this->private, out);
+    VALIDATE_OR_GOTO(fd, out);
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    priv = this->private;
 
-        if (fd_ctx) {
-                //no need to take any locks
-                if (!list_empty (&fd_ctx->eager_locked))
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_INVALID_DATA, "%s: Stale "
-                                "Eager-lock stubs found",
-                                uuid_utoa (fd->inode->gfid));
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-		for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++)
-			GF_FREE (fd_ctx->pre_op_done[i]);
+    if (ret == 0)
+        goto out;
 
-                GF_FREE (fd_ctx->opened_on);
-
-                GF_FREE (fd_ctx->lock_piggyback);
+    fd_ctx = GF_CALLOC(1, sizeof(afr_fd_ctx_t), gf_afr_mt_afr_fd_ctx_t);
+    if (!fd_ctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-                GF_FREE (fd_ctx->lock_acquired);
+    fd_ctx->opened_on = GF_CALLOC(sizeof(*fd_ctx->opened_on), priv->child_count,
+                                  gf_afr_mt_int32_t);
+    if (!fd_ctx->opened_on) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-		pthread_mutex_destroy (&fd_ctx->delay_lock);
+    for (i = 0; i < priv->child_count; i++) {
+        if (fd_is_anonymous(fd))
+            fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        else
+            fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+    }
 
-                GF_FREE (fd_ctx);
-        }
+    fd_ctx->readdir_subvol = -1;
+    fd_ctx->lk_heal_info = NULL;
 
+    ret = __fd_ctx_set(fd, this, (uint64_t)(long)fd_ctx);
+    if (ret)
+        gf_msg_debug(this->name, 0, "failed to set fd ctx (%p)", fd);
 out:
-        return 0;
+    if (ret && fd_ctx)
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    return ret;
 }
 
+/* {{{ flush */
 
 int
-afr_release (xlator_t *this, fd_t *fd)
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
 {
-        afr_cleanup_fd_ctx (this, fd);
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        return 0;
-}
+    local = frame->local;
 
+    LOCK(&frame->lock);
+    {
+        if (op_ret != -1) {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
+        }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-/* {{{ fsync */
+    if (call_count == 0)
+        AFR_STACK_UNWIND(flush, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-int
-afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
-{
-        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
-        return 0;
+    return 0;
 }
 
-int
-afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int call_count = -1;
-        int child_index = (long) cookie;
-	int read_subvol = 0;
-	call_stub_t *stub = NULL;
+static int
+afr_flush_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_flush_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->flush,
+                              local->fd, xdata);
+            if (!--call_count)
+                break;
+        }
+    }
 
-        local = frame->local;
+    return 0;
+}
 
-	read_subvol = afr_data_subvol_get (local->inode, this, NULL, NULL,
-                                           NULL, NULL);
+afr_local_t *
+afr_wakeup_same_fd_delayed_op(xlator_t *this, afr_lock_t *lock, fd_t *fd)
+{
+    afr_local_t *local = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (local->op_ret == -1) {
-				local->op_ret = 0;
-
-                                local->cont.inode_wfop.prebuf  = *prebuf;
-                                local->cont.inode_wfop.postbuf = *postbuf;
-
-				if (xdata)
-					local->xdata_rsp = dict_ref (xdata);
-                        }
-
-                        if (child_index == read_subvol) {
-                                local->cont.inode_wfop.prebuf  = *prebuf;
-                                local->cont.inode_wfop.postbuf = *postbuf;
-				if (xdata) {
-					if (local->xdata_rsp)
-						dict_unref (local->xdata_rsp);
-					local->xdata_rsp = dict_ref (xdata);
-				}
-                        }
-                } else {
-			local->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		/* Make a stub out of the frame, and register it
-		   with the waking up post-op. When the call-stub resumes,
-		   we are guaranteed that there was no post-op pending
-		   (i.e changelogs were unset in the server). This is an
-		   essential "guarantee", that fsync() returns only after
-		   completely finishing EVERYTHING, including the delayed
-		   post-op. This guarantee is expected by FUSE graph switching
-		   for example.
-		*/
-		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
-                                           local->op_ret, local->op_errno,
-                                           &local->cont.inode_wfop.prebuf,
-                                           &local->cont.inode_wfop.postbuf,
-                                           local->xdata_rsp);
-		if (!stub) {
-			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
-			return 0;
-		}
-
-		/* If no new unstable writes happened between the
-		   time we cleared the unstable write witness flag in afr_fsync
-		   and now, calling afr_delayed_changelog_wake_up() should
-		   wake up and skip over the fsync phase and go straight to
-		   afr_changelog_post_op_now()
-		*/
-		afr_delayed_changelog_wake_resume (this, local->fd, stub);
+    if (lock->delay_timer) {
+        local = list_entry(lock->post_op.next, afr_local_t,
+                           transaction.owner_list);
+        if (fd == local->fd) {
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                local = NULL;
+            } else {
+                lock->delay_timer = NULL;
+            }
+        } else {
+            local = NULL;
         }
+    }
 
-        return 0;
+    return local;
 }
 
+void
+afr_delayed_changelog_wake_resume(xlator_t *this, inode_t *inode,
+                                  call_stub_t *stub)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_local_t *metadata_local = NULL;
+    afr_local_t *data_local = NULL;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
+        lock = &ctx->lock[AFR_DATA_TRANSACTION];
+        data_local = afr_wakeup_same_fd_delayed_op(this, lock, stub->args.fd);
+        lock = &ctx->lock[AFR_METADATA_TRANSACTION];
+        metadata_local = afr_wakeup_same_fd_delayed_op(this, lock,
+                                                       stub->args.fd);
+    }
+    UNLOCK(&inode->lock);
+
+    if (data_local) {
+        data_local->transaction.resume_stub = stub;
+    } else if (metadata_local) {
+        metadata_local->transaction.resume_stub = stub;
+    } else {
+        call_resume(stub);
+    }
+    if (data_local) {
+        afr_delayed_changelog_wake_up_cbk(data_local);
+    }
+    if (metadata_local) {
+        afr_delayed_changelog_wake_up_cbk(metadata_local);
+    }
+}
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-	   dict_t *xdata)
+afr_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = ENOMEM;
 
-	priv = this->private;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local->op = GF_FOP_FLUSH;
+    if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+        goto out;
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    local->fd = fd_ref(fd);
 
-        local->fd = fd_ref (fd);
+    stub = fop_flush_stub(frame, afr_flush_wrapper, fd, xdata);
+    if (!stub)
+        goto out;
 
-	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
-		/* don't care. we only wanted to CLEAR the bit */
-	}
+    afr_delayed_changelog_wake_resume(this, fd->inode, stub);
 
-	local->inode = inode_ref (fd->inode);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsync_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsync,
-                                           fd, datasync, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
+    AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-/* }}} */
-
-/* {{{ fsync */
-
 int
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		} else {
-			local->op_errno = op_errno;
-		}
+    LOCK(&frame->lock);
+    {
+        if (op_ret == 0) {
+            local->op_ret = 0;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-				  local->op_errno, local->xdata_rsp);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-	      dict_t *xdata)
-{
-	afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
-
-	priv = this->private;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fsyncdir_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fsyncdir,
-                                    fd, datasync, xdata);
-                        if (!--call_count)
-                                break;
-                }
+afr_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int32_t call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FSYNCDIR;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    call_count = local->call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND(frame, afr_fsyncdir_cbk, priv->children[i],
+                       priv->children[i]->fops->fsyncdir, fd, datasync, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
 
-int32_t
-afr_unlock_partial_inodelk_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xdata)
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this);
 
+static gf_boolean_t
+afr_is_conflicting_lock_present(int32_t op_ret, int32_t op_errno)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int child_index = (long)cookie;
-        uuid_t  gfid = {0};
+    if (op_ret == -1 && op_errno == EAGAIN)
+        return _gf_true;
+    return _gf_false;
+}
 
-        local = frame->local;
-        priv = this->private;
+static void
+afr_fop_lock_unwind(call_frame_t *frame, glusterfs_fop_t op, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata)
+{
+    switch (op) {
+        case GF_FOP_INODELK:
+            AFR_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FINODELK:
+            AFR_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            AFR_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            AFR_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-        if (op_ret < 0 && op_errno != ENOTCONN) {
-                loc_gfid (&local->loc, gfid);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_INODE_UNLOCK_FAIL,
-                        "%s: Failed to unlock %s "
-                        "with lk_owner: %s (%s)", uuid_utoa (gfid),
-                        priv->children[child_index]->name,
-                        lkowner_utoa (&frame->root->lk_owner),
-                        strerror (op_errno));
-        }
+static void
+afr_fop_lock_wind(call_frame_t *frame, xlator_t *this, int child_index,
+                  int32_t (*lock_cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *))
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = child_index;
+
+    switch (local->op) {
+        case GF_FOP_INODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->inodelk,
+                (const char *)local->cont.inodelk.volume, &local->loc,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_FINODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->finodelk,
+                (const char *)local->cont.inodelk.volume, local->fd,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->entrylk, local->cont.entrylk.volume,
+                &local->loc, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->fentrylk, local->cont.entrylk.volume,
+                local->fd, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno, local->xdata_rsp);
-        }
+void
+afr_fop_lock_proceed(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        return 0;
+    local = frame->local;
+    priv = frame->this->private;
+
+    if (local->fop_lock_state != AFR_FOP_LOCK_PARALLEL) {
+        afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                            local->xdata_rsp);
+        return;
+    }
+    /* At least one child is up */
+    /*
+     * Non-blocking locks also need to be serialized.  Otherwise there is
+     * a chance that both the mounts which issued same non-blocking inodelk
+     * may endup not acquiring the lock on any-brick.
+     * Ex: Mount1 and Mount2
+     * request for full length lock on file f1.  Mount1 afr may acquire the
+     * partial lock on brick-1 and may not acquire the lock on brick-2
+     * because Mount2 already got the lock on brick-2, vice versa.  Since
+     * both the mounts only got partial locks, afr treats them as failure in
+     * gaining the locks and unwinds with EAGAIN errno.
+     */
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop_lock_state = AFR_FOP_LOCK_SERIAL;
+    afr_local_replies_wipe(local, priv);
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
+    local->xdata_rsp = NULL;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.cmd = local->cont.inodelk.in_cmd;
+            local->cont.inodelk.flock = local->cont.inodelk.in_flock;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = local->cont.entrylk.in_cmd;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+            break;
+        default:
+            break;
+    }
+    afr_serialized_lock_wind(frame, frame->this);
 }
 
-int32_t
-afr_unlock_inodelks_and_unwind (call_frame_t *frame, xlator_t *this,
-                                int call_count)
+static int32_t
+afr_unlock_partial_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    uuid_t gfid = {0};
 
-        local = frame->local;
-        priv = this->private;
-        local->call_count = call_count;
-        local->cont.inodelk.flock.l_type = F_UNLCK;
+    local = frame->local;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
+    if (op_ret < 0 && op_errno != ENOTCONN) {
+        if (local->fd)
+            gf_uuid_copy(gfid, local->fd->inode->gfid);
+        else
+            loc_gfid(&local->loc, gfid);
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "%s: Failed to unlock %s on %s "
+               "with lk_owner: %s",
+               uuid_utoa(gfid), gf_fop_list[local->op],
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
 
-                if (local->replies[i].op_ret == -1)
-                        continue;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_proceed(frame);
 
-                STACK_WIND_COOKIE (frame, afr_unlock_partial_inodelk_cbk,
-                                   (void*) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->inodelk,
-                                   local->cont.inodelk.volume,
-                                   &local->loc, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock, 0);
+    return 0;
+}
 
-                if (!--call_count)
-                        break;
-        }
+static int32_t
+afr_unlock_locks_and_proceed(call_frame_t *frame, xlator_t *this,
+                             int call_count)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    if (call_count == 0) {
+        afr_fop_lock_proceed(frame);
+        goto out;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    local->call_count = call_count;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.flock.l_type = F_UNLCK;
+            local->cont.inodelk.cmd = F_SETLK;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = ENTRYLK_UNLOCK;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            break;
+        default:
+            break;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret == -1)
+            continue;
+
+        afr_fop_lock_wind(frame, this, i, afr_unlock_partial_lock_cbk);
+
+        if (!--call_count)
+            break;
+    }
 
-        return 0;
+out:
+    return 0;
 }
 
 int32_t
-afr_inodelk_done (call_frame_t *frame, xlator_t *this)
+afr_fop_lock_done(call_frame_t *frame, xlator_t *this)
 {
-        int i = 0;
-        int lock_count = 0;
+    int i = 0;
+    int lock_count = 0;
+    unsigned char *success = NULL;
 
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
+    success = alloca0(priv->child_count);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-                if (local->replies[i].op_ret == 0)
-                        lock_count++;
+        if (local->replies[i].op_ret == 0) {
+            lock_count++;
+            success[i] = 1;
+        }
 
-                if (local->op_ret == -1 && local->op_errno == EAGAIN)
-                        continue;
+        if (local->op_ret == -1 && local->op_errno == EAGAIN)
+            continue;
 
-                if ((local->replies[i].op_ret == -1) &&
-                    (local->replies[i].op_errno == EAGAIN)) {
-                        local->op_ret = -1;
-                        local->op_errno = EAGAIN;
-                        continue;
-                }
+        if ((local->replies[i].op_ret == -1) &&
+            (local->replies[i].op_errno == EAGAIN)) {
+            local->op_ret = -1;
+            local->op_errno = EAGAIN;
+            continue;
+        }
 
-                if (local->replies[i].op_ret == 0)
-                        local->op_ret = 0;
+        if (local->replies[i].op_ret == 0)
+            local->op_ret = 0;
 
-                local->op_errno = local->replies[i].op_errno;
-        }
+        local->op_errno = local->replies[i].op_errno;
+    }
 
-        if (lock_count && local->cont.inodelk.flock.l_type != F_UNLCK &&
-            (local->op_ret == -1 && local->op_errno == EAGAIN)) {
-                afr_unlock_inodelks_and_unwind (frame, this,
-                                                lock_count);
-        } else {
-                AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno, local->xdata_rsp);
-        }
+    if (afr_fop_lock_is_unlock(frame))
+        goto unwind;
 
-        return 0;
+    if (afr_is_conflicting_lock_present(local->op_ret, local->op_errno)) {
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else if (priv->quorum_count && !afr_has_quorum(success, this, NULL)) {
+        local->fop_lock_state = AFR_FOP_LOCK_QUORUM_FAILED;
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        if (local->op_errno == 0)
+            local->op_errno = afr_quorum_errno(priv);
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else {
+        goto unwind;
+    }
+
+    return 0;
+unwind:
+    afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                        local->xdata_rsp);
+    return 0;
 }
 
-int
-afr_common_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+afr_common_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int child_index = (long)cookie;
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->replies[child_index].valid = 1;
-        local->replies[child_index].op_ret = op_ret;
-        local->replies[child_index].op_errno = op_errno;
-        if (op_ret == 0 && xdata) {
-                local->replies[child_index].xdata = dict_ref (xdata);
-                LOCK (&frame->lock);
-                {
-                        if (!local->xdata_rsp)
-                                local->xdata_rsp = dict_ref (xdata);
-                }
-                UNLOCK (&frame->lock);
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret == 0 && xdata) {
+        local->replies[child_index].xdata = dict_ref(xdata);
+        LOCK(&frame->lock);
+        {
+            if (!local->xdata_rsp)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        return 0;
+        UNLOCK(&frame->lock);
+    }
+    return 0;
 }
 
 static int32_t
-afr_parallel_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_serialized_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        int     call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int next_child = 0;
 
-        afr_common_inodelk_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    local = frame->local;
+    priv = this->private;
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                afr_inodelk_done (frame, this);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        return 0;
+    for (next_child = child_index + 1; next_child < priv->child_count;
+         next_child++) {
+        if (local->child_up[next_child])
+            break;
+    }
+
+    if (afr_is_conflicting_lock_present(op_ret, op_errno) ||
+        (next_child == priv->child_count)) {
+        afr_fop_lock_done(frame, this);
+    } else {
+        afr_fop_lock_wind(frame, this, next_child, afr_serialized_lock_cbk);
+    }
+
+    return 0;
 }
 
-static gf_boolean_t
-afr_is_conflicting_lock_present (int32_t op_ret, int32_t op_errno)
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this)
 {
-        if (op_ret == -1 && op_errno == EAGAIN)
-                return _gf_true;
-        return _gf_false;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            afr_fop_lock_wind(frame, this, i, afr_serialized_lock_cbk);
+            break;
+        }
+    }
+    return 0;
 }
 
 static int32_t
-afr_serialized_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_parallel_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int child_index = (long)cookie;
-        int next_child  = 0;
+    int call_count = 0;
 
-        local = frame->local;
-        priv = this->private;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        afr_common_inodelk_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_done(frame, this);
 
-        for (next_child = child_index + 1; next_child < priv->child_count;
-             next_child++) {
-                if (local->child_up[next_child])
-                        break;
-        }
-
-        if (afr_is_conflicting_lock_present (op_ret, op_errno) ||
-            (next_child == priv->child_count)) {
-                afr_inodelk_done (frame, this);
-        } else {
-                STACK_WIND_COOKIE (frame, afr_serialized_inodelk_cbk,
-                                   (void *) (long) next_child,
-                                   priv->children[next_child],
-                                   priv->children[next_child]->fops->inodelk,
-                                   (const char *)local->cont.inodelk.volume,
-                                   &local->loc, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock,
-                                   local->xdata_req);
-        }
-
-        return 0;
+    return 0;
 }
 
 static int
-afr_parallel_inodelk_wind (call_frame_t *frame, xlator_t *this)
+afr_parallel_lock_wind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int         call_count = 0;
-        int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int i = 0;
 
-        priv = this->private;
-        local = frame->local;
-        call_count = local->call_count;
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-                STACK_WIND_COOKIE (frame, afr_parallel_inodelk_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->inodelk,
-                                   (const char *)local->cont.inodelk.volume,
-                                   &local->loc, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock,
-                                   local->xdata_req);
-                if (!--call_count)
-                        break;
-        }
-        return 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+        afr_fop_lock_wind(frame, this, i, afr_parallel_lock_cbk);
+        if (!--call_count)
+            break;
+    }
+    return 0;
 }
 
 static int
-afr_serialized_inodelk_wind (call_frame_t *frame, xlator_t *this)
+afr_fop_handle_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
+    afr_local_t *local = frame->local;
+    int op_errno = 0;
 
-        priv = this->private;
-        local = frame->local;
+    if (!afr_fop_lock_is_unlock(frame)) {
+        if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+            goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_serialized_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                       (const char *)local->cont.inodelk.volume,
-                                           &local->loc, local->cont.inodelk.cmd,
-                                           &local->cont.inodelk.flock,
-                                           local->xdata_req);
-                        break;
-                }
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.cmd = F_SETLK;
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.cmd = ENTRYLK_LOCK_NB;
+                break;
+            default:
+                break;
         }
-        return 0;
-}
-
-int32_t
-afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd,
-             struct gf_flock *flock, dict_t *xdata)
-{
-        afr_local_t *local  = NULL;
-        int32_t op_errno = ENOMEM;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+    }
 
-        loc_copy (&local->loc, loc);
-        local->cont.inodelk.volume = gf_strdup (volume);
-        if (!local->cont.inodelk.volume) {
-                op_errno = ENOMEM;
-                goto out;
+    if (local->xdata_req) {
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+                break;
+            default:
+                break;
         }
+    }
 
-        local->cont.inodelk.cmd = cmd;
-        local->cont.inodelk.flock = *flock;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
-
-        /* At least one child is up */
-        /*
-         * Non-blocking locks also need to be serialized.  Otherwise there is
-         * a chance that both the mounts which issued same non-blocking inodelk
-         * may endup not acquiring the lock on any-brick.
-         * Ex: Mount1 and Mount2
-         * request for full length lock on file f1.  Mount1 afr may acquire the
-         * partial lock on brick-1 and may not acquire the lock on brick-2
-         * because Mount2 already got the lock on brick-2, vice versa.  Since
-         * both the mounts only got partial locks, afr treats them as failure in
-         * gaining the locks and unwinds with EAGAIN errno.
-         */
-        if (flock->l_type == F_UNLCK) {
-                afr_parallel_inodelk_wind (frame, this);
-        } else {
-                afr_serialized_inodelk_wind (frame, this);
-        }
+    local->fop_lock_state = AFR_FOP_LOCK_PARALLEL;
+    afr_parallel_lock_wind(frame, this);
+out:
+    return -op_errno;
+}
 
-	return 0;
+static int32_t
+afr_handle_inodelk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd, int32_t cmd,
+                   struct gf_flock *flock, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (flock->l_type != F_UNLCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+
+    local->cont.inodelk.volume = gf_strdup(volume);
+    if (!local->cont.inodelk.volume) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->cont.inodelk.in_cmd = cmd;
+    local->cont.inodelk.cmd = cmd;
+    local->cont.inodelk.in_flock = *flock;
+    local->cont.inodelk.flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+    return 0;
 out:
-	AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-
+afr_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno, xdata);
-
-        return 0;
+    afr_handle_inodelk(frame, this, GF_FOP_INODELK, volume, loc, NULL, cmd,
+                       flock, xdata);
+    return 0;
 }
 
-
 int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-	      int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+afr_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_finodelk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    afr_handle_inodelk(frame, this, GF_FOP_FINODELK, volume, NULL, fd, cmd,
+                       flock, xdata);
+    return 0;
+}
 
-	return 0;
+static int
+afr_handle_entrylk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd,
+                   const char *basename, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (cmd != ENTRYLK_UNLOCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+    local->cont.entrylk.cmd = cmd;
+    local->cont.entrylk.in_cmd = cmd;
+    local->cont.entrylk.type = type;
+    local->cont.entrylk.volume = gf_strdup(volume);
+    local->cont.entrylk.basename = gf_strdup(basename);
+    if (!local->cont.entrylk.volume || !local->cont.entrylk.basename) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+
+    return 0;
 out:
-	AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
-
-        return 0;
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
+    return 0;
 }
 
-
-int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+afr_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno, xdata);
-
-        return 0;
+    afr_handle_entrylk(frame, this, GF_FOP_ENTRYLK, volume, loc, NULL, basename,
+                       cmd, type, xdata);
+    return 0;
 }
 
+int
+afr_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+    afr_handle_entrylk(frame, this, GF_FOP_FENTRYLK, volume, NULL, fd, basename,
+                       cmd, type, xdata);
+    return 0;
+}
 
 int
-afr_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-	     loc_t *loc, const char *basename, entrylk_cmd cmd,
-	     entrylk_type type, dict_t *xdata)
+afr_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    struct statvfs *buf = NULL;
 
-        priv = this->private;
+    local = frame->local;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    LOCK(&frame->lock);
+    {
+        if (op_ret != 0) {
+            local->op_errno = op_errno;
+            goto unlock;
+        }
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+        local->op_ret = op_ret;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_entrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type, xdata);
-
-                        if (!--call_count)
-                                break;
+        buf = &local->cont.statfs.buf;
+        if (local->cont.statfs.buf_set) {
+            if (statvfs->f_bavail < buf->f_bavail) {
+                *buf = *statvfs;
+                if (xdata) {
+                    if (local->xdata_rsp)
+                        dict_unref(local->xdata_rsp);
+                    local->xdata_rsp = dict_ref(xdata);
                 }
+            }
+        } else {
+            *buf = *statvfs;
+            local->cont.statfs.buf_set = 1;
+            if (xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
+    }
+unlock:
+    call_count = --local->call_count;
+    UNLOCK(&frame->lock);
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->cont.statfs.buf, local->xdata_rsp);
 
-        return 0;
+    return 0;
 }
 
+int
+afr_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_STATFS;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
+        local->call_count--;
+    call_count = local->call_count;
+    if (!call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            if (AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+            STACK_WIND(frame, afr_statfs_cbk, priv->children[i],
+                       priv->children[i]->fops->statfs, loc, xdata);
+            if (!--call_count)
+                break;
+        }
+    }
 
+    return 0;
+out:
+    AFR_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-int
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    return 0;
+}
 
+int32_t
+afr_lk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    int call_count = -1;
+    int child_index = (long)cookie;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+    }
 
-        call_count = afr_frame_return (frame);
+    return 0;
+}
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno, xdata);
+int32_t
+afr_lk_unlock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        return 0;
-}
+    local = frame->local;
+    priv = this->private;
 
+    call_count = afr_locked_nodes_count(local->cont.lk.locked_nodes,
+                                        priv->child_count);
 
-int
-afr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type,
-	      dict_t *xdata)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+        return 0;
+    }
 
-        priv = this->private;
+    local->call_count = call_count;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local->cont.lk.user_flock.l_type = F_UNLCK;
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lk.locked_nodes[i]) {
+            STACK_WIND_COOKIE(frame, afr_lk_unlock_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->lk,
+                              local->fd, F_SETLK, &local->cont.lk.user_flock,
+                              NULL);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fentrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
-
-        return 0;
+    return 0;
 }
 
-
-int
-afr_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-		int op_errno, struct statvfs *statvfs, dict_t *xdata)
+int32_t
+afr_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = 0;
-	struct statvfs *buf = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                if (op_ret != 0) {
-                        local->op_errno = op_errno;
-			goto unlock;
-		}
-
-		local->op_ret = op_ret;
-
-		buf = &local->cont.statfs.buf;
-		if (local->cont.statfs.buf_set) {
-			if (statvfs->f_bavail < buf->f_bavail) {
-				*buf = *statvfs;
-				if (xdata) {
-					if (local->xdata_rsp)
-						dict_unref (local->xdata_rsp);
-					local->xdata_rsp = dict_ref (xdata);
-				}
-			}
-		} else {
-			*buf = *statvfs;
-			local->cont.statfs.buf_set = 1;
-			if (xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		}
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    local = frame->local;
+    priv = this->private;
 
-        call_count = afr_frame_return (frame);
+    child_index = (long)cookie;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf, local->xdata_rsp);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
+        afr_lk_unlock(frame, this);
         return 0;
-}
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+
+    child_index++;
+
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lk, local->fd,
+                          local->cont.lk.cmd, &local->cont.lk.user_flock,
+                          local->xdata_req);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+
+        afr_lk_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
 
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
+
+    return 0;
+}
 
 int
-afr_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+afr_lk_transaction_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        afr_local_t   *  local       = NULL;
-	afr_private_t   *priv        = NULL;
-        int              i           = 0;
-        int              call_count = 0;
-        int32_t          op_errno    = ENOMEM;
+    return 0;
+}
 
-	priv = this->private;
+int
+afr_lk_txn_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                    dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = -1;
+
+    local = frame->local;
+    child_index = (long)cookie;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+int
+afr_lk_txn_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int child_index = (long)cookie;
+
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+    return 0;
+}
+int
+afr_lk_transaction(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *wind_on = NULL;
+    int op_errno = 0;
+    int i = 0;
+    int ret = 0;
+
+    frame = (call_frame_t *)opaque;
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+
+    if (priv->arbiter_count || priv->child_count != 3) {
+        op_errno = ENOTSUP;
+        gf_msg(frame->this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Lock healing supported only for replica 3 volumes.",
+               uuid_utoa(local->fd->inode->gfid));
+        goto err;
+    }
+
+    op_errno = -afr_dom_lock_acquire(frame);  // Released during
+                                              // AFR_STACK_UNWIND
+    if (op_errno != 0) {
+        goto err;
+    }
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.dom_locked_nodes, this, NULL)) {
+        op_errno = afr_final_errno(local, priv);
+        goto err;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && local->cont.lk.dom_locked_nodes[i])
+            wind_on[i] = 1;
+    }
+    AFR_ONLIST(wind_on, frame, afr_lk_txn_wind_cbk, lk, local->fd,
+               local->cont.lk.cmd, &local->cont.lk.user_flock,
+               local->xdata_req);
+
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        goto unlock;
+    } else {
+        if (local->cont.lk.user_flock.l_type == F_UNLCK)
+            ret = afr_remove_lock_from_saved_locks(local, this);
+        else
+            ret = afr_add_lock_to_saved_locks(frame, this);
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            goto unlock;
+        }
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-        if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
-                local->call_count--;
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    return 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        if (AFR_IS_ARBITER_BRICK(priv, i))
-                                continue;
-                        STACK_WIND (frame, afr_statfs_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->statfs,
-                                    loc, xdata);
-                        if (!--call_count)
-                                break;
-                }
+unlock:
+    local->cont.lk.user_flock.l_type = F_UNLCK;
+    AFR_ONLIST(local->cont.lk.locked_nodes, frame, afr_lk_txn_unlock_cbk, lk,
+               local->fd, F_SETLK, &local->cont.lk.user_flock, NULL);
+err:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return -1;
+}
+
+int
+afr_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    int i = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_LK;
+    if (!afr_lk_is_unlock(cmd, flock)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+            goto out;
+    }
+
+    local->cont.lk.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+
+    if (!local->cont.lk.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->fd = fd_ref(fd);
+    local->cont.lk.cmd = cmd;
+    local->cont.lk.user_flock = *flock;
+    local->cont.lk.ret_flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    if (afr_is_lock_mode_mandatory(xdata)) {
+        ret = synctask_new(this->ctx->env, afr_lk_transaction,
+                           afr_lk_transaction_cbk, frame, frame);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
         }
+        return 0;
+    }
 
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)0, priv->children[i],
+                      priv->children[i]->fops->lk, fd, cmd, flock,
+                      local->xdata_req);
+
+    return 0;
 out:
-	AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                   dict_t *xdata)
+afr_lease_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                     dict_t *xdata)
 {
-        afr_local_t * local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        local = frame->local;
-        call_count = afr_frame_return (frame);
+    local = frame->local;
+    call_count = afr_frame_return(frame);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock, xdata);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno, lease,
+                         xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_lk_unlock (call_frame_t *frame, xlator_t *this)
+afr_lease_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   * local = NULL;
-        afr_private_t * priv  = NULL;
-        int i = 0;
-        int call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        call_count = afr_locked_nodes_count (local->cont.lk.locked_nodes,
-                                             priv->child_count);
+    call_count = afr_locked_nodes_count(local->cont.lease.locked_nodes,
+                                        priv->child_count);
 
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock, NULL);
-                return 0;
-        }
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+        return 0;
+    }
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        local->cont.lk.user_flock.l_type = F_UNLCK;
+    local->cont.lease.user_lease.cmd = GF_UNLK_LEASE;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->cont.lk.locked_nodes[i]) {
-                        STACK_WIND (frame, afr_lk_unlock_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->lk,
-                                    local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lease.locked_nodes[i]) {
+            STACK_WIND(frame, afr_lease_unlock_cbk, priv->children[i],
+                       priv->children[i]->fops->lease, &local->loc,
+                       &local->cont.lease.user_lease, NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+afr_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int child_index = -1;
-/*        int            ret  = 0; */
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
+    local = frame->local;
+    priv = this->private;
 
-        local = frame->local;
-        priv  = this->private;
-
-        child_index = (long) cookie;
-
-        if (!child_went_down (op_ret, op_errno) && (op_ret == -1)) {
-                local->op_ret   = -1;
-                local->op_errno = op_errno;
+    child_index = (long)cookie;
 
-                afr_lk_unlock (frame, this);
-                return 0;
-        }
-
-        if (op_ret == 0) {
-                local->op_ret        = 0;
-                local->op_errno      = 0;
-                local->cont.lk.locked_nodes[child_index] = 1;
-                local->cont.lk.ret_flock = *lock;
-        }
-
-        child_index++;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-        if (child_index < priv->child_count) {
-                STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->lk,
-                                   local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock, xdata);
-        } else if (local->op_ret == -1) {
-                /* all nodes have gone down */
+        afr_lease_unlock(frame, this);
+        return 0;
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lease.locked_nodes[child_index] = 1;
+        local->cont.lease.ret_lease = *lease;
+    }
+
+    child_index++;
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lease, &local->loc,
+                          &local->cont.lease.user_lease, xdata);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lease.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-                AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock, NULL);
-        } else {
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock, NULL);
-        }
+        afr_lease_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+afr_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
 
-        priv = this->private;
+    priv = this->private;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
-                                                 sizeof (*local->cont.lk.locked_nodes),
-                                                 gf_afr_mt_char);
+    local->op = GF_FOP_LEASE;
+    local->cont.lease.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lease.locked_nodes),
+        gf_afr_mt_char);
 
-        if (!local->cont.lk.locked_nodes) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (!local->cont.lease.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        local->fd            = fd_ref (fd);
-        local->cont.lk.cmd   = cmd;
-        local->cont.lk.user_flock = *flock;
-        local->cont.lk.ret_flock = *flock;
+    loc_copy(&local->loc, loc);
+    local->cont.lease.user_lease = *lease;
+    local->cont.lease.ret_lease = *lease;
 
-        STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
-                           priv->children[i],
-                           priv->children[i]->fops->lk,
-                           fd, cmd, flock, xdata);
+    STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)0, priv->children[0],
+                      priv->children[0]->fops->lease, loc, lease, xdata);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t        ctx_int = 0;
-        afr_inode_ctx_t *ctx    = NULL;
+afr_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = 0;
+    gf_boolean_t failed = _gf_false;
+    gf_boolean_t succeeded = _gf_false;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    call_count = afr_frame_return(frame);
+    if (call_count)
+        goto out;
+    /* If any of the subvolumes failed with other than ENOTCONN
+     * return error else return success unless all the subvolumes
+     * failed.
+     * TODO: In case of failure, we need to unregister the xattrs
+     * from the other subvolumes where it succeeded (once upcall
+     * fixes the Bz-1371622)*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0 &&
+            local->replies[i].op_errno != ENOTCONN) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+            if (local->xdata_rsp)
+                dict_unref(local->xdata_rsp);
+            local->xdata_rsp = NULL;
+            if (local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            failed = _gf_true;
+            break;
+        }
+        if (local->replies[i].op_ret == 0) {
+            succeeded = _gf_true;
+            local->op_ret = 0;
+            local->op_errno = 0;
+            if (!local->xdata_rsp && local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
+    }
+
+    if (!succeeded && !failed) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+    }
 
-        afr_spb_choice_timeout_cancel (this, inode);
-        inode_ctx_del (inode, this, &ctx_int);
-        if (!ctx_int)
-                return 0;
+    AFR_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
 
-        ctx = (afr_inode_ctx_t *)ctx_int;
-        GF_FREE (ctx);
-        return 0;
+out:
+    return 0;
 }
 
 int
-afr_priv_dump (xlator_t *this)
+afr_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char  key[GF_DUMP_MAX_BUF_LEN];
-        int   i = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_cnt = -1;
 
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        GF_ASSERT (this);
-        priv = this->private;
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        GF_ASSERT (priv);
-        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-        gf_proc_dump_add_section(key_prefix);
-        gf_proc_dump_write("child_count", "%u", priv->child_count);
+    VALIDATE_OR_GOTO(this->private, err);
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto err;
+
+    call_cnt = local->call_count;
+
+    if (xdata) {
         for (i = 0; i < priv->child_count; i++) {
-                sprintf (key, "child_up[%d]", i);
-                gf_proc_dump_write(key, "%d", priv->child_up[i]);
-                sprintf (key, "pending_key[%d]", i);
-                gf_proc_dump_write(key, "%s", priv->pending_key[i]);
-        }
-        gf_proc_dump_write("data_self_heal", "%s", priv->data_self_heal);
-        gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
-        gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
-        gf_proc_dump_write("data_change_log", "%d", priv->data_change_log);
-        gf_proc_dump_write("metadata_change_log", "%d", priv->metadata_change_log);
-        gf_proc_dump_write("entry-change_log", "%d", priv->entry_change_log);
-        gf_proc_dump_write("read_child", "%d", priv->read_child);
-        gf_proc_dump_write("favorite_child", "%d", priv->favorite_child);
-        gf_proc_dump_write("wait_count", "%u", priv->wait_count);
-        gf_proc_dump_write("quorum-reads", "%d", priv->quorum_reads);
-        gf_proc_dump_write("heal-wait-queue-length", "%d",
-                           priv->heal_wait_qlen);
-        gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
-        gf_proc_dump_write("background-self-heal-count", "%d",
-                           priv->background_self_heal_count);
-        gf_proc_dump_write("healers", "%d", priv->healers);
+            if (dict_set_int8(xdata, priv->pending_key[i], 0) < 0)
+                goto err;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+
+        STACK_WIND_COOKIE(frame, afr_ipc_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->ipc, op,
+                          xdata);
+        if (!--call_cnt)
+            break;
+    }
+    return 0;
+
+err:
+    if (op_errno == -1)
+        op_errno = errno;
+    AFR_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+
+    return 0;
+
+wind_default:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
 
+int
+afr_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ctx_int = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
         return 0;
+
+    ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+    afr_inode_ctx_destroy(ctx);
+    return 0;
 }
 
+int
+afr_priv_dump(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("child_count", "%u", priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        sprintf(key, "child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->child_up[i]);
+        sprintf(key, "pending_key[%d]", i);
+        gf_proc_dump_write(key, "%s", priv->pending_key[i]);
+        sprintf(key, "pending_reads[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64,
+                           GF_ATOMIC_GET(priv->pending_reads[i]));
+        sprintf(key, "child_latency[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64, priv->child_latency[i]);
+        sprintf(key, "halo_child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->halo_child_up[i]);
+    }
+    gf_proc_dump_write("data_self_heal", "%d", priv->data_self_heal);
+    gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
+    gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
+    gf_proc_dump_write("read_child", "%d", priv->read_child);
+    gf_proc_dump_write("wait_count", "%u", priv->wait_count);
+    gf_proc_dump_write("heal-wait-queue-length", "%d", priv->heal_wait_qlen);
+    gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
+    gf_proc_dump_write("background-self-heal-count", "%d",
+                       priv->background_self_heal_count);
+    gf_proc_dump_write("healers", "%d", priv->healers);
+    gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode);
+    gf_proc_dump_write("use-anonymous-inode", "%d", priv->use_anon_inode);
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        gf_proc_dump_write("quorum-type", "auto");
+    } else if (priv->quorum_count == 0) {
+        gf_proc_dump_write("quorum-type", "none");
+    } else {
+        gf_proc_dump_write("quorum-type", "fixed");
+        gf_proc_dump_write("quorum-count", "%d", priv->quorum_count);
+    }
+    gf_proc_dump_write("up", "%u", afr_has_quorum(priv->child_up, this, NULL));
+    if (priv->thin_arbiter_count) {
+        gf_proc_dump_write("ta_child_up", "%d", priv->ta_child_up);
+        gf_proc_dump_write("ta_bad_child_index", "%d",
+                           priv->ta_bad_child_index);
+        gf_proc_dump_write("ta_notify_dom_lock_offset", "%" PRId64,
+                           priv->ta_notify_dom_lock_offset);
+    }
+
+    return 0;
+}
 
 /**
  * find_child_index - find the child's index in the array of subvolumes
@@ -3914,1339 +5723,2156 @@ afr_priv_dump (xlator_t *this)
  */
 
 static int
-find_child_index (xlator_t *this, xlator_t *child)
+afr_find_child_index(xlator_t *this, xlator_t *child)
 {
-        afr_private_t *priv = NULL;
-        int i = -1;
+    afr_private_t *priv = NULL;
+    int child_count = -1;
+    int i = -1;
 
-        priv = this->private;
+    priv = this->private;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if ((xlator_t *) child == priv->children[i])
-                        break;
-        }
+    for (i = 0; i < child_count; i++) {
+        if ((xlator_t *)child == priv->children[i])
+            break;
+    }
 
-        return i;
+    return i;
 }
 
-static int
-__afr_get_up_children_count (afr_private_t *priv)
+int
+__afr_get_up_children_count(afr_private_t *priv)
 {
-        int             up_children         = 0;
-        int             i = 0;
+    int up_children = 0;
+    int i = 0;
 
-        for (i = 0; i < priv->child_count; i++)
-                if (priv->child_up[i] == 1)
-                        up_children++;
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 1)
+            up_children++;
 
-        return up_children;
+    return up_children;
 }
 
-glusterfs_event_t
-__afr_transform_event_from_state (afr_private_t *priv)
-{
-        int i = 0;
-        int up_children = 0;
-
-        if (AFR_COUNT (priv->last_event, priv->child_count) ==
-                       priv->child_count)
-                /* have_heard_from_all. Let afr_notify() do the propagation. */
-                return GF_EVENT_MAXVAL;
-
-        up_children = __afr_get_up_children_count (priv);
-        if (up_children) {
-                /* We received at least one child up and there are pending
-                 * notifications from some children. Treat these children as
-                 * having sent a GF_EVENT_CHILD_DOWN. i.e. set the event as
-                 * GF_EVENT_CHILD_MODIFIED, as done in afr_notify() */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (priv->last_event[i])
-                                continue;
-                        priv->last_event[i] = GF_EVENT_CHILD_MODIFIED;
-                        priv->child_up[i] = 0;
-                }
-                return GF_EVENT_CHILD_UP;
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (priv->last_event[i])
-                                continue;
-                        priv->last_event[i] = GF_EVENT_SOME_CHILD_DOWN;
-                        priv->child_up[i] = 0;
-                }
-                return GF_EVENT_CHILD_DOWN;
+static int
+__get_heard_from_all_status(xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    int i;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->last_event[i]) {
+            return 0;
         }
+    }
+    if (priv->thin_arbiter_count && !priv->ta_child_up) {
+        return 0;
+    }
+    return 1;
+}
+
+glusterfs_event_t
+__afr_transform_event_from_state(xlator_t *this)
+{
+    int i = 0;
+    int up_children = 0;
+    afr_private_t *priv = this->private;
 
+    if (__get_heard_from_all_status(this))
+        /* have_heard_from_all. Let afr_notify() do the propagation. */
         return GF_EVENT_MAXVAL;
+
+    up_children = __afr_get_up_children_count(priv);
+    /* Treat the children with pending notification, as having sent a
+     * GF_EVENT_CHILD_DOWN. i.e. set the event as GF_EVENT_SOME_DESCENDENT_DOWN,
+     * as done in afr_notify() */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->last_event[i])
+            continue;
+        priv->last_event[i] = GF_EVENT_SOME_DESCENDENT_DOWN;
+        priv->child_up[i] = 0;
+    }
+
+    if (up_children)
+        /* We received at least one child up */
+        return GF_EVENT_CHILD_UP;
+    else
+        return GF_EVENT_CHILD_DOWN;
+
+    return GF_EVENT_MAXVAL;
 }
 
 static void
-afr_notify_cbk (void *data)
-{
-        xlator_t *this = data;
-        afr_private_t *priv = this->private;
-        glusterfs_event_t event = GF_EVENT_MAXVAL;
-        gf_boolean_t propagate = _gf_false;
-
-        LOCK (&priv->lock);
-        {
-                if (!priv->timer) {
-                        /*
-                         * Either child_up/child_down is already sent to parent.
-                         * This is a spurious wake up.
-                         */
-                        goto unlock;
-                }
-                priv->timer = NULL;
-                event = __afr_transform_event_from_state (priv);
-                if (event != GF_EVENT_MAXVAL)
-                        propagate = _gf_true;
-        }
+afr_notify_cbk(void *data)
+{
+    xlator_t *this = data;
+    afr_private_t *priv = this->private;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent.
+             * This is a spurious wake up.
+             */
+            goto unlock;
+        }
+        priv->timer = NULL;
+        event = __afr_transform_event_from_state(this);
+        if (event != GF_EVENT_MAXVAL)
+            propagate = _gf_true;
+    }
 unlock:
-        UNLOCK (&priv->lock);
-        if (propagate)
-                default_notify (this, event, NULL);
+    UNLOCK(&priv->lock);
+    if (propagate)
+        default_notify(this, event, NULL);
 }
 
 static void
-__afr_launch_notify_timer (xlator_t *this, afr_private_t *priv)
+__afr_launch_notify_timer(xlator_t *this, afr_private_t *priv)
+{
+    struct timespec delay = {
+        0,
+    };
+
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    priv->timer = gf_timer_call_after(this->ctx, delay, afr_notify_cbk, this);
+    if (priv->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer for delayed initialization");
+    }
+}
+
+static int
+find_best_down_child(xlator_t *this)
 {
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t best_child = -1;
+    int64_t best_latency = INT64_MAX;
 
-        struct timespec delay = {0, };
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Initiating child-down timer");
-        delay.tv_sec = 10;
-        delay.tv_nsec = 0;
-        priv->timer = gf_timer_call_after (this->ctx, delay,
-                                           afr_notify_cbk, this);
-        if (priv->timer == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
-                        "Cannot create timer for delayed initialization");
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] < best_latency) {
+            best_child = i;
+            best_latency = priv->child_latency[i];
         }
+    }
+    if (best_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found best down child (%d) @ %" PRId64 " ms latency",
+                     best_child, best_latency);
+    }
+    return best_child;
 }
 
 int
-__get_heard_from_all_status (xlator_t *this)
+find_worst_up_child(xlator_t *this)
 {
-        afr_private_t *priv          = this->private;
-        int           heard_from_all = 1;
-        int           i              = 0;
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t worst_child = -1;
+    int64_t worst_latency = INT64_MIN;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!priv->last_event[i]) {
-                        heard_from_all = 0;
-                        break;
-                }
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] > worst_latency) {
+            worst_child = i;
+            worst_latency = priv->child_latency[i];
         }
-        return heard_from_all;
+    }
+    if (worst_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found worst up child (%d) @ %" PRId64 " ms latency",
+                     worst_child, worst_latency);
+    }
+    return worst_child;
 }
 
-int32_t
-afr_notify (xlator_t *this, int32_t event,
-            void *data, void *data2)
-{
-        afr_private_t   *priv               = NULL;
-        int             i                   = -1;
-        int             up_children         = 0;
-        int             down_children       = 0;
-        int             propagate           = 0;
-        int             had_heard_from_all  = 0;
-        int             have_heard_from_all = 0;
-        int             idx                 = -1;
-        int             ret                 = -1;
-        int             call_psh            = 0;
-        dict_t          *input              = NULL;
-        dict_t          *output             = NULL;
-        gf_boolean_t    had_quorum          = _gf_false;
-        gf_boolean_t    has_quorum          = _gf_false;
-
-        priv = this->private;
-
-        if (!priv)
-                return 0;
-
-        /*
-         * We need to reset this in case children come up in "staggered"
-         * fashion, so that we discover a late-arriving local subvolume.  Note
-         * that we could end up issuing N lookups to the first subvolume, and
-         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
-         */
-        priv->did_discovery = _gf_false;
+void
+__afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx,
+                        int64_t halo_max_latency_msec, int32_t *event,
+                        int64_t child_latency_msec)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
 
+    priv = this->private;
 
-        /* parent xlators dont need to know about every child_up, child_down
-         * because of afr ha. If all subvolumes go down, child_down has
-         * to be triggered. In that state when 1 subvolume comes up child_up
-         * needs to be triggered. dht optimizes revalidate lookup by sending
-         * it only to one of its subvolumes. When child up/down happens
-         * for afr's subvolumes dht should be notified by child_modified. The
-         * subsequent revalidate lookup happens on all the dht's subvolumes
-         * which triggers afr self-heals if any.
-         */
-        idx = find_child_index (this, data);
-        if (idx < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
-                        "Received child_up from invalid subvolume");
-                goto out;
-        }
+    priv->child_latency[idx] = child_latency_msec;
+    gf_msg_debug(child_xlator->name, 0, "Client ping @ %" PRId64 " ms",
+                 child_latency_msec);
+    if (priv->shd.iamshd)
+        return;
 
-        had_quorum = priv->quorum_count && afr_has_quorum (priv->child_up,
-                                                           this);
-        if (event == GF_EVENT_TRANSLATOR_OP) {
-                LOCK (&priv->lock);
-                {
-                        had_heard_from_all = __get_heard_from_all_status (this);
-                }
-                UNLOCK (&priv->lock);
-
-                if (!had_heard_from_all) {
-                        ret = -1;
-                } else {
-                        input = data;
-                        output = data2;
-                        ret = afr_xl_op (this, input, output);
-                }
-                goto out;
-        }
+    up_children = __afr_get_up_children_count(priv);
 
-        LOCK (&priv->lock);
-        {
-                had_heard_from_all = __get_heard_from_all_status (this);
-                switch (event) {
-                case GF_EVENT_PARENT_UP:
-                        __afr_launch_notify_timer (this, priv);
-                        propagate = 1;
-                        break;
-                case GF_EVENT_CHILD_UP:
-                        /*
-                         * This only really counts if the child was never up
-                         * (value = -1) or had been down (value = 0).  See
-                         * comment at GF_EVENT_CHILD_DOWN for a more detailed
-                         * explanation.
-                         */
-                        if (priv->child_up[idx] != 1) {
-                                priv->event_generation++;
-                        }
-                        priv->child_up[idx] = 1;
-
-                        call_psh = 1;
-                        up_children = __afr_get_up_children_count (priv);
-                        if (up_children == 1) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        AFR_MSG_SUBVOL_UP,
-                                        "Subvolume '%s' came back up; "
-                                     "going online.", ((xlator_t *)data)->name);
-                        } else {
-                                event = GF_EVENT_CHILD_MODIFIED;
-                        }
-
-                        priv->last_event[idx] = event;
-
-                        break;
-
-                case GF_EVENT_CHILD_DOWN:
-                        if (priv->child_up[idx] == 1) {
-                                priv->event_generation++;
-                        }
-                        priv->child_up[idx] = 0;
-
-                        for (i = 0; i < priv->child_count; i++)
-                                if (priv->child_up[i] == 0)
-                                        down_children++;
-                        if (down_children == priv->child_count) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        AFR_MSG_ALL_SUBVOLS_DOWN,
-                                       "All subvolumes are down. Going offline "
-                                    "until atleast one of them comes back up.");
-                        } else {
-                                event = GF_EVENT_SOME_CHILD_DOWN;
-                        }
-
-                        priv->last_event[idx] = event;
-
-                        break;
-
-                case GF_EVENT_CHILD_CONNECTING:
-                        priv->last_event[idx] = event;
-
-                        break;
-
-                case GF_EVENT_SOME_CHILD_DOWN:
-                        priv->last_event[idx] = event;
-                        break;
-
-                default:
-                        propagate = 1;
-                        break;
-                }
-                have_heard_from_all = __get_heard_from_all_status (this);
-                if (!had_heard_from_all && have_heard_from_all) {
-                        if (priv->timer) {
-                                gf_timer_call_cancel (this->ctx, priv->timer);
-                                priv->timer = NULL;
-                        }
-                        /* This is the first event which completes aggregation
-                           of events from all subvolumes. If at least one subvol
-                           had come up, propagate CHILD_UP, but only this time
-                        */
-                        event = GF_EVENT_CHILD_DOWN;
-                        up_children = __afr_get_up_children_count (priv);
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
-                                        event = GF_EVENT_CHILD_UP;
-                                        break;
-                                }
-
-                                if (priv->last_event[i] ==
-                                                GF_EVENT_CHILD_CONNECTING) {
-                                        event = GF_EVENT_CHILD_CONNECTING;
-                               /* continue to check other events for CHILD_UP */
-                                }
-                        }
-                }
+    if (child_latency_msec > halo_max_latency_msec &&
+        priv->child_up[idx] == 1 && up_children > priv->halo_min_replicas) {
+        if ((up_children - 1) < priv->halo_min_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Overriding halo threshold, "
+                   "min replicas: %d",
+                   priv->halo_min_replicas);
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "exceeds halo threshold (%" PRId64
+                   "), "
+                   "marking child down.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_DOWN;
+            }
+        }
+    } else if (child_latency_msec < halo_max_latency_msec &&
+               priv->child_up[idx] == 0) {
+        if (up_children < priv->halo_max_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "below halo threshold (%" PRId64
+                   "), "
+                   "marking child up.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_UP;
+            }
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Not marking child %d up, "
+                   "max replicas (%d) reached.",
+                   idx, priv->halo_max_replicas);
         }
-        UNLOCK (&priv->lock);
+    }
+}
 
-        if (priv->quorum_count) {
-                has_quorum = afr_has_quorum (priv->child_up, this);
-                if (!had_quorum && has_quorum)
-                        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
-                                "Client-quorum is met");
-                if (had_quorum && !has_quorum)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_QUORUM_FAIL,
-                                "Client-quorum is not met");
-        }
+static int64_t
+afr_get_halo_latency(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int64_t halo_max_latency_msec = 0;
 
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all)
-                propagate = 1;
+    priv = this->private;
 
-        ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
-
-        if ((!had_heard_from_all) || call_psh) {
-                /* Launch self-heal on all local subvolumes if:
-                 * a) We have_heard_from_all for the first time
-                 * b) Already heard from everyone, but we now got a child-up
-                 *    event.
-                 */
-                if (have_heard_from_all && priv->shd.iamshd) {
-                        for (i = 0; i < priv->child_count; i++)
-                                if (priv->child_up[i])
-                                        afr_selfheal_childup (this, i);
-                }
-        }
+    if (priv->shd.iamshd) {
+        halo_max_latency_msec = priv->shd.halo_max_latency_msec;
+    } else if (priv->nfsd.iamnfsd) {
+        halo_max_latency_msec = priv->nfsd.halo_max_latency_msec;
+    } else {
+        halo_max_latency_msec = priv->halo_max_latency_msec;
+    }
+    gf_msg_debug(this->name, 0, "Using halo latency %" PRId64,
+                 halo_max_latency_msec);
+    return halo_max_latency_msec;
+}
+
+void
+__afr_handle_child_up_event(xlator_t *this, xlator_t *child_xlator,
+                            const int idx, int64_t child_latency_msec,
+                            int32_t *event, int32_t *call_psh,
+                            int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
+    int worst_up_child = -1;
+    int64_t halo_max_latency_msec = afr_get_halo_latency(this);
+
+    priv = this->private;
+
+    /*
+     * This only really counts if the child was never up
+     * (value = -1) or had been down (value = 0).  See
+     * comment at GF_EVENT_CHILD_DOWN for a more detailed
+     * explanation.
+     */
+    if (priv->child_up[idx] != 1) {
+        priv->event_generation++;
+    }
+    priv->child_up[idx] = 1;
+
+    *call_psh = 1;
+    *up_child = idx;
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * If this is an _actual_ CHILD_UP event, we
+     * want to set the child_latency to MAX to indicate
+     * the child needs ping data to be available before doing child-up
+     */
+    if (!priv->halo_enabled)
+        goto out;
+
+    if (child_latency_msec < 0) {
+        /*set to INT64_MAX-1 so that it is found for best_down_child*/
+        priv->halo_child_up[idx] = 1;
+        if (priv->child_latency[idx] < 0) {
+            priv->child_latency[idx] = AFR_HALO_MAX_LATENCY;
+        }
+    }
+
+    /*
+     * Handle the edge case where we exceed
+     * halo_min_replicas and we've got a child which is
+     * marked up as it was helping to satisfy the
+     * halo_min_replicas even though it's latency exceeds
+     * halo_max_latency_msec.
+     */
+    if (up_children > priv->halo_min_replicas) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child >= 0 &&
+            priv->child_latency[worst_up_child] > halo_max_latency_msec) {
+            gf_msg_debug(this->name, 0,
+                         "Marking child %d down, "
+                         "doesn't meet halo threshold (%" PRId64
+                         "), and > "
+                         "halo_min_replicas (%d)",
+                         worst_up_child, halo_max_latency_msec,
+                         priv->halo_min_replicas);
+            priv->child_up[worst_up_child] = 0;
+            up_children--;
+        }
+    }
+
+    if (up_children > priv->halo_max_replicas && !priv->shd.iamshd) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child < 0) {
+            worst_up_child = idx;
+        }
+        priv->child_up[worst_up_child] = 0;
+        up_children--;
+        gf_msg_debug(this->name, 0,
+                     "Marking child %d down, "
+                     "up_children (%d) > halo_max_replicas (%d)",
+                     worst_up_child, up_children, priv->halo_max_replicas);
+    }
 out:
-        return ret;
+    if (up_children == 1) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOL_UP,
+               "Subvolume '%s' came back up; "
+               "going online.",
+               child_xlator->name);
+        gf_event(EVENT_AFR_SUBVOL_UP, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_UP;
+    }
+
+    priv->last_event[idx] = *event;
 }
 
+void
+__afr_handle_child_down_event(xlator_t *this, xlator_t *child_xlator, int idx,
+                              int64_t child_latency_msec, int32_t *event,
+                              int32_t *call_psh, int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int up_children = 0;
+    int down_children = 0;
+    int best_down_child = -1;
+
+    priv = this->private;
+
+    /*
+     * If a brick is down when we start, we'll get a
+     * CHILD_DOWN to indicate its initial state.  There
+     * was never a CHILD_UP in this case, so if we
+     * increment "down_count" the difference between than
+     * and "up_count" will no longer be the number of
+     * children that are currently up.  This has serious
+     * implications e.g. for quorum enforcement, so we
+     * don't increment these values unless the event
+     * represents an actual state transition between "up"
+     * (value = 1) and anything else.
+     */
+    if (priv->child_up[idx] == 1) {
+        priv->event_generation++;
+    }
+
+    /*
+     * If this is an _actual_ CHILD_DOWN event, we
+     * want to set the child_latency to < 0 to indicate
+     * the child is really disconnected.
+     */
+    if (child_latency_msec < 0) {
+        priv->child_latency[idx] = child_latency_msec;
+        priv->halo_child_up[idx] = 0;
+    }
+    priv->child_up[idx] = 0;
+
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * Handle the edge case where we need to find the
+     * next best child (to mark up) as marking this child
+     * down would cause us to fall below halo_min_replicas.
+     * We will also force the SHD to heal this child _now_
+     * as we want it to be up to date if we are going to
+     * begin using it synchronously.
+     */
+    if (priv->halo_enabled && up_children < priv->halo_min_replicas) {
+        best_down_child = find_best_down_child(this);
+        if (best_down_child >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "Swapping out child %d for "
+                         "child %d to satisfy halo_min_replicas (%d).",
+                         idx, best_down_child, priv->halo_min_replicas);
+            priv->child_up[best_down_child] = 1;
+            *call_psh = 1;
+            *up_child = best_down_child;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 0)
+            down_children++;
+    if (down_children == priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are down. Going "
+               "offline until at least one of them "
+               "comes back up.");
+        gf_event(EVENT_AFR_SUBVOLS_DOWN, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_DOWN;
+    }
+    priv->last_event[idx] = *event;
+}
 
-int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+void
+afr_ta_lock_release_synctask(xlator_t *this)
 {
-        local->op_ret = -1;
-        local->op_errno = EUCLEAN;
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
 
-	syncbarrier_init (&local->barrier);
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        return;
+    }
 
-        local->child_up = GF_CALLOC (priv->child_count,
-                                     sizeof (*local->child_up),
-                                     gf_afr_mt_char);
-        if (!local->child_up) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
+    ret = synctask_new(this->ctx->env, afr_release_notify_lock_for_ta,
+                       afr_ta_lock_release_done, ta_frame, this);
+    if (ret) {
+        STACK_DESTROY(ta_frame->root);
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to release "
+               "AFR_TA_DOM_NOTIFY lock.");
+    }
+}
 
-        memcpy (local->child_up, priv->child_up,
-                sizeof (*local->child_up) * priv->child_count);
-        local->call_count = AFR_COUNT (local->child_up, priv->child_count);
-        if (local->call_count == 0) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0,
-                        AFR_MSG_ALL_SUBVOLS_DOWN, "no subvolumes up");
-                if (op_errno)
-                        *op_errno = ENOTCONN;
-                goto out;
-        }
-	local->event_generation = priv->event_generation;
-
-	local->read_attempted = GF_CALLOC (priv->child_count, sizeof (char),
-					   gf_afr_mt_char);
-	if (!local->read_attempted) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
-
-	local->readable = GF_CALLOC (priv->child_count, sizeof (char),
-				     gf_afr_mt_char);
-	if (!local->readable) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
-
-        local->readable2 = GF_CALLOC (priv->child_count, sizeof (char),
-                                      gf_afr_mt_char);
-        if (!local->readable2) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
+static void
+afr_handle_inodelk_contention(xlator_t *this, struct gf_upcall *upcall)
+{
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    afr_private_t *priv = this->private;
 
-	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
-				   gf_afr_mt_reply_t);
-	if (!local->replies) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
+    lc = upcall->data;
 
-        local->need_full_crawl = _gf_false;
+    if (strcmp(lc->domain, AFR_TA_DOM_NOTIFY) != 0)
+        return;
 
-        INIT_LIST_HEAD (&local->healer);
-	return 0;
-out:
-        return -1;
+    if (priv->shd.iamshd) {
+        /* shd should ignore AFR_TA_DOM_NOTIFY release requests. */
+        return;
+    }
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Ignore multiple release requests from shds.*/
+            UNLOCK(&priv->lock);
+            return;
+        }
+        priv->release_ta_notify_dom_lock = _gf_true;
+        inmem_count = priv->ta_in_mem_txn_count;
+        onwire_count = priv->ta_on_wire_txn_count;
+    }
+    UNLOCK(&priv->lock);
+    if (inmem_count || onwire_count)
+        /* lock release will happen in txn code path after
+         * in-memory or on-wire txns are over.*/
+        return;
+
+    afr_ta_lock_release_synctask(this);
 }
 
-int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
-                        transaction_lk_type_t lk_type)
-{
-        int             ret = -ENOMEM;
+static void
+afr_handle_upcall_event(xlator_t *this, struct gf_upcall *upcall)
+{
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    afr_private_t *priv = this->private;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    int i = 0;
+
+    switch (upcall->event_type) {
+        case GF_UPCALL_INODELK_CONTENTION:
+            afr_handle_inodelk_contention(this, upcall);
+            break;
+        case GF_UPCALL_CACHE_INVALIDATION:
+            up_ci = (struct gf_upcall_cache_invalidation *)upcall->data;
+
+            /* Since md-cache will be aggressively filtering
+             * lookups, the stale read issue will be more
+             * pronounced. Hence when a pending xattr is set notify
+             * all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time */
+            if (!up_ci->dict)
+                break;
+            for (i = 0; i < priv->child_count; i++) {
+                if (!dict_get(up_ci->dict, priv->pending_key[i]))
+                    continue;
+                up_ci->flags |= UP_INVAL_ATTR;
+                itable = ((xlator_t *)this->graph->top)->itable;
+                /*Internal processes may not have itable for
+                 *top xlator*/
+                if (itable)
+                    inode = inode_find(itable, upcall->gfid);
+                if (inode)
+                    afr_inode_need_refresh_set(inode, this);
+                break;
+            }
+            break;
+        default:
+            break;
+    }
+}
 
-        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
-                                      child_count, gf_afr_mt_char);
-        if (NULL == lk->locked_nodes)
-                goto out;
+int32_t
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *child_xlator = NULL;
+    int i = -1;
+    int propagate = 0;
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    int idx = -1;
+    int ret = -1;
+    int call_psh = 0;
+    int up_child = -1;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t had_quorum = _gf_false;
+    gf_boolean_t has_quorum = _gf_false;
+    int64_t halo_max_latency_msec = 0;
+    int64_t child_latency_msec = -1;
+
+    child_xlator = (xlator_t *)data;
+
+    priv = this->private;
+
+    if (!priv)
+        return 0;
 
-        lk->lower_locked_nodes = GF_CALLOC (sizeof (*lk->lower_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->lower_locked_nodes)
-                goto out;
+    /*
+     * We need to reset this in case children come up in "staggered"
+     * fashion, so that we discover a late-arriving local subvolume.  Note
+     * that we could end up issuing N lookups to the first subvolume, and
+     * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+     */
+    priv->did_discovery = _gf_false;
+
+    /* parent xlators don't need to know about every child_up, child_down
+     * because of afr ha. If all subvolumes go down, child_down has
+     * to be triggered. In that state when 1 subvolume comes up child_up
+     * needs to be triggered. dht optimizes revalidate lookup by sending
+     * it only to one of its subvolumes. When child up/down happens
+     * for afr's subvolumes dht should be notified by child_modified. The
+     * subsequent revalidate lookup happens on all the dht's subvolumes
+     * which triggers afr self-heals if any.
+     */
+    idx = afr_find_child_index(this, child_xlator);
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
+               "Received child_up from invalid subvolume");
+        goto out;
+    }
+
+    had_quorum = priv->quorum_count &&
+                 afr_has_quorum(priv->child_up, this, NULL);
+    if (event == GF_EVENT_CHILD_PING) {
+        child_latency_msec = (int64_t)(uintptr_t)data2;
+        if (priv->halo_enabled) {
+            halo_max_latency_msec = afr_get_halo_latency(this);
+
+            /* Calculates the child latency and sets event
+             */
+            LOCK(&priv->lock);
+            {
+                __afr_handle_ping_event(this, child_xlator, idx,
+                                        halo_max_latency_msec, &event,
+                                        child_latency_msec);
+            }
+            UNLOCK(&priv->lock);
+        } else {
+            LOCK(&priv->lock);
+            {
+                priv->child_latency[idx] = child_latency_msec;
+            }
+            UNLOCK(&priv->lock);
+        }
+    }
 
-        lk->lock_op_ret   = -1;
-        lk->lock_op_errno = EUCLEAN;
-        lk->transaction_lk_type = lk_type;
+    if (event == GF_EVENT_CHILD_PING) {
+        /* This is the only xlator that handles PING, no reason to
+         * propagate.
+         */
+        goto out;
+    }
 
-        ret = 0;
-out:
-        return ret;
-}
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        LOCK(&priv->lock);
+        {
+            had_heard_from_all = __get_heard_from_all_status(this);
+        }
+        UNLOCK(&priv->lock);
 
-void
-afr_matrix_cleanup (int32_t **matrix, unsigned int m)
-{
-        int             i         = 0;
+        if (!had_heard_from_all) {
+            ret = -1;
+        } else {
+            input = data;
+            output = data2;
+            ret = afr_xl_op(this, input, output);
+        }
+        goto out;
+    }
+
+    if (event == GF_EVENT_UPCALL) {
+        afr_handle_upcall_event(this, data);
+    }
+
+    LOCK(&priv->lock);
+    {
+        had_heard_from_all = __get_heard_from_all_status(this);
+        switch (event) {
+            case GF_EVENT_PARENT_UP:
+                __afr_launch_notify_timer(this, priv);
+                propagate = 1;
+                break;
+            case GF_EVENT_CHILD_UP:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 1;
+                    priv->ta_event_gen++;
+                    break;
+                }
+                __afr_handle_child_up_event(this, child_xlator, idx,
+                                            child_latency_msec, &event,
+                                            &call_psh, &up_child);
+                __afr_lock_heal_synctask(this, priv, idx);
+                break;
 
-        if (!matrix)
-                goto out;
-        for (i = 0; i < m; i++) {
-                GF_FREE (matrix[i]);
-        }
+            case GF_EVENT_CHILD_DOWN:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 0;
+                    priv->ta_event_gen++;
+                    afr_ta_locked_priv_invalidate(priv);
+                    break;
+                }
+                __afr_handle_child_down_event(this, child_xlator, idx,
+                                              child_latency_msec, &event,
+                                              &call_psh, &up_child);
+                __afr_mark_pending_lk_heal(this, priv, idx);
+                break;
 
-        GF_FREE (matrix);
-out:
-        return;
-}
+            case GF_EVENT_CHILD_CONNECTING:
+                priv->last_event[idx] = event;
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n)
-{
-        int32_t         **matrix = NULL;
-        int             i       = 0;
+                break;
 
-        matrix = GF_CALLOC (sizeof (*matrix), m, gf_afr_mt_int32_t);
-        if (!matrix)
-                goto out;
+            case GF_EVENT_SOME_DESCENDENT_DOWN:
+                priv->last_event[idx] = event;
+                break;
+            default:
+                propagate = 1;
+                break;
+        }
+        have_heard_from_all = __get_heard_from_all_status(this);
+        if (!had_heard_from_all && have_heard_from_all) {
+            if (priv->timer) {
+                gf_timer_call_cancel(this->ctx, priv->timer);
+                priv->timer = NULL;
+            }
+            /* This is the first event which completes aggregation
+               of events from all subvolumes. If at least one subvol
+               had come up, propagate CHILD_UP, but only this time
+            */
+            event = GF_EVENT_CHILD_DOWN;
+            for (i = 0; i < priv->child_count; i++) {
+                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
+                    event = GF_EVENT_CHILD_UP;
+                    break;
+                }
 
-        for (i = 0; i < m; i++) {
-                matrix[i] = GF_CALLOC (sizeof (*matrix[i]), n,
-                                       gf_afr_mt_int32_t);
-                if (!matrix[i])
-                        goto out;
+                if (priv->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                    event = GF_EVENT_CHILD_CONNECTING;
+                    /* continue to check other events for CHILD_UP */
+                }
+            }
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->quorum_count) {
+        has_quorum = afr_has_quorum(priv->child_up, this, NULL);
+        if (!had_quorum && has_quorum) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
+                   "Client-quorum is met");
+            gf_event(EVENT_AFR_QUORUM_MET, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+        if (had_quorum && !has_quorum) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+                   "Client-quorum is not met");
+            gf_event(EVENT_AFR_QUORUM_FAIL, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all)
+        propagate = 1;
+
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
+
+    if ((!had_heard_from_all) || call_psh) {
+        /* Launch self-heal on all local subvolumes if:
+         * a) We have_heard_from_all for the first time
+         * b) Already heard from everyone, but we now got a child-up
+         *    event.
+         */
+        if (have_heard_from_all) {
+            afr_selfheal_childup(this, priv);
         }
-        return matrix;
+    }
 out:
-        afr_matrix_cleanup (matrix, m);
-        return NULL;
+    return ret;
 }
 
 int
-afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
-{
-        int             ret = -ENOMEM;
-
-        lk->domain = dom;
-        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
-                                      child_count, gf_afr_mt_char);
-        if (NULL == lk->locked_nodes)
-                goto out;
-        ret = 0;
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+{
+    int __ret = -1;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+
+    __ret = syncbarrier_init(&local->barrier);
+    if (__ret) {
+        if (op_errno)
+            *op_errno = __ret;
+        goto out;
+    }
+
+    local->child_up = GF_MALLOC(priv->child_count * sizeof(*local->child_up),
+                                gf_afr_mt_char);
+    if (!local->child_up) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    memcpy(local->child_up, priv->child_up,
+           sizeof(*local->child_up) * priv->child_count);
+    local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (local->call_count == 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "no subvolumes up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        goto out;
+    }
+
+    local->event_generation = priv->event_generation;
+
+    local->read_attempted = GF_CALLOC(priv->child_count, sizeof(char),
+                                      gf_afr_mt_char);
+    if (!local->read_attempted) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable = GF_CALLOC(priv->child_count, sizeof(char),
+                                gf_afr_mt_char);
+    if (!local->readable) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable2 = GF_CALLOC(priv->child_count, sizeof(char),
+                                 gf_afr_mt_char);
+    if (!local->readable2) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->read_subvol = -1;
+
+    local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+                               gf_afr_mt_reply_t);
+    if (!local->replies) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->need_full_crawl = _gf_false;
+    if (priv->thin_arbiter_count) {
+        local->ta_child_up = priv->ta_child_up;
+        local->ta_failed_subvol = AFR_CHILD_UNKNOWN;
+        local->read_txn_query_child = AFR_CHILD_UNKNOWN;
+        local->ta_event_gen = priv->ta_event_gen;
+        local->fop_state = TA_SUCCESS;
+    }
+    local->is_new_entry = _gf_false;
+
+    INIT_LIST_HEAD(&local->healer);
+    return 0;
 out:
-        return ret;
+    return -1;
 }
 
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this)
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count)
 {
-        int            child_up_count = 0;
-        int            ret = -ENOMEM;
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        ret = afr_internal_lock_init (&local->internal_lock, priv->child_count,
-                                      AFR_TRANSACTION_LK);
-        if (ret < 0)
-                goto out;
-
-        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
-            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
-                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
-                                        this->name, priv->child_count);
-                if (ret < 0)
-                        goto out;
-        }
+    int ret = -ENOMEM;
 
-        ret = -ENOMEM;
-        child_up_count = AFR_COUNT (local->child_up, priv->child_count);
-        if (priv->optimistic_change_log && child_up_count == priv->child_count)
-                local->optimistic_change_log = 1;
-
-	local->pre_op_compat = priv->pre_op_compat;
+    lk->lower_locked_nodes = GF_CALLOC(sizeof(*lk->lower_locked_nodes),
+                                       child_count, gf_afr_mt_char);
+    if (NULL == lk->lower_locked_nodes)
+        goto out;
 
-        local->transaction.eager_lock =
-                GF_CALLOC (sizeof (*local->transaction.eager_lock),
-                           priv->child_count,
-                           gf_afr_mt_int32_t);
+    lk->lock_op_ret = -1;
+    lk->lock_op_errno = EUCLEAN;
 
-        if (!local->transaction.eager_lock)
-                goto out;
-
-        local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-        if (!local->transaction.pre_op)
-                goto out;
+    ret = 0;
+out:
+    return ret;
+}
 
-        if (priv->arbiter_count == 1) {
-                local->transaction.pre_op_xdata =
-                        GF_CALLOC (sizeof (*local->transaction.pre_op_xdata),
-                                   priv->child_count, gf_afr_mt_dict_t);
-                if (!local->transaction.pre_op_xdata)
-                        goto out;
+void
+afr_matrix_cleanup(int32_t **matrix, unsigned int m)
+{
+    int i = 0;
 
-                local->transaction.pre_op_sources =
-                        GF_CALLOC (sizeof (*local->transaction.pre_op_sources),
-                                   priv->child_count, gf_afr_mt_char);
-                if (!local->transaction.pre_op_sources)
-                        goto out;
-        }
+    if (!matrix)
+        goto out;
+    for (i = 0; i < m; i++) {
+        GF_FREE(matrix[i]);
+    }
 
-        local->transaction.failed_subvols = GF_CALLOC (sizeof (*local->transaction.failed_subvols),
-						       priv->child_count,
-						       gf_afr_mt_char);
-        if (!local->transaction.failed_subvols)
-                goto out;
+    GF_FREE(matrix);
+out:
+    return;
+}
 
-        local->pending = afr_matrix_create (priv->child_count,
-                                            AFR_NUM_CHANGE_LOGS);
-        if (!local->pending)
-                goto out;
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n)
+{
+    int32_t **matrix = NULL;
+    int i = 0;
 
-	INIT_LIST_HEAD (&local->transaction.eager_locked);
+    matrix = GF_CALLOC(sizeof(*matrix), m, gf_afr_mt_int32_t);
+    if (!matrix)
+        goto out;
 
-        ret = 0;
+    for (i = 0; i < m; i++) {
+        matrix[i] = GF_CALLOC(sizeof(*matrix[i]), n, gf_afr_mt_int32_t);
+        if (!matrix[i])
+            goto out;
+    }
+    return matrix;
 out:
-        return ret;
+    afr_matrix_cleanup(matrix, m);
+    return NULL;
 }
 
+int
+afr_transaction_local_init(afr_local_t *local, xlator_t *this)
+{
+    int ret = -ENOMEM;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&local->transaction.wait_list);
+    INIT_LIST_HEAD(&local->transaction.owner_list);
+    INIT_LIST_HEAD(&local->ta_waitq);
+    INIT_LIST_HEAD(&local->ta_onwireq);
+    ret = afr_internal_lock_init(&local->internal_lock, priv->child_count);
+    if (ret < 0)
+        goto out;
+
+    ret = -ENOMEM;
+    local->pre_op_compat = priv->pre_op_compat;
+
+    local->transaction.pre_op = GF_CALLOC(sizeof(*local->transaction.pre_op),
+                                          priv->child_count, gf_afr_mt_char);
+    if (!local->transaction.pre_op)
+        goto out;
+
+    local->transaction.changelog_xdata = GF_CALLOC(
+        sizeof(*local->transaction.changelog_xdata), priv->child_count,
+        gf_afr_mt_dict_t);
+    if (!local->transaction.changelog_xdata)
+        goto out;
+
+    if (priv->arbiter_count == 1) {
+        local->transaction.pre_op_sources = GF_CALLOC(
+            sizeof(*local->transaction.pre_op_sources), priv->child_count,
+            gf_afr_mt_char);
+        if (!local->transaction.pre_op_sources)
+            goto out;
+    }
+
+    local->transaction.failed_subvols = GF_CALLOC(
+        sizeof(*local->transaction.failed_subvols), priv->child_count,
+        gf_afr_mt_char);
+    if (!local->transaction.failed_subvols)
+        goto out;
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
 
 void
-afr_set_low_priority (call_frame_t *frame)
+afr_set_low_priority(call_frame_t *frame)
 {
-        frame->root->pid = LOW_PRIO_PROC_PID;
+    frame->root->pid = LOW_PRIO_PROC_PID;
 }
 
+void
+afr_priv_destroy(afr_private_t *priv)
+{
+    int i = 0;
+    int child_count = -1;
+
+    if (!priv)
+        goto out;
+
+    GF_FREE(priv->sh_domain);
+    GF_FREE(priv->last_event);
+
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
+    if (priv->pending_key) {
+        for (i = 0; i < child_count; i++)
+            GF_FREE(priv->pending_key[i]);
+    }
+
+    GF_FREE(priv->pending_reads);
+    GF_FREE(priv->local);
+    GF_FREE(priv->pending_key);
+    GF_FREE(priv->children);
+    GF_FREE(priv->anon_inode);
+    GF_FREE(priv->child_up);
+    GF_FREE(priv->halo_child_up);
+    GF_FREE(priv->child_latency);
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
+out:
+    return;
+}
 
-gf_boolean_t
-afr_have_quorum (char *logname, afr_private_t *priv)
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat)
 {
-        unsigned int        quorum = 0;
-        unsigned int        up_children = 0;
+    int i = 0;
+    int **changelog = NULL;
+    int idx = -1;
+    int m_idx = 0;
+    int d_idx = 0;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO(logname,priv,out);
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
 
-        up_children = __afr_get_up_children_count (priv);
-        quorum = priv->quorum_count;
-        if (quorum != AFR_QUORUM_AUTO)
-                return up_children >= quorum;
+    idx = afr_index_from_ia_type(iat);
 
-        quorum = priv->child_count / 2 + 1;
-        if (up_children >= quorum)
-                return _gf_true;
+    changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!changelog)
+        goto out;
 
-        /*
-         * Special case for even numbers of nodes: if we have exactly half
-         * and that includes the first ("senior-most") node, then that counts
-         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
-         * while preserving the critical property that there can only be one
-         * such group.
-         */
-        if ((priv->child_count % 2) == 0) {
-                quorum = priv->child_count / 2;
-                if (up_children >= quorum) {
-                        if (priv->child_up[0]) {
-                                return _gf_true;
-                        }
-                }
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            continue;
 
+        changelog[i][m_idx] = hton32(1);
+        if (idx != -1)
+            changelog[i][idx] = hton32(1);
+        /* If the newentry marking is on a newly created directory,
+         * then mark it with the full-heal indicator.
+         */
+        if ((IA_ISDIR(iat)) && (priv->esh_granular))
+            changelog[i][d_idx] = hton32(1);
+    }
+    ret = afr_set_pending_dict(priv, xattr, changelog);
+    if (ret < 0) {
+        afr_matrix_cleanup(changelog, priv->child_count);
+        return NULL;
+    }
 out:
-        return _gf_false;
+    return changelog;
 }
 
-void
-afr_priv_destroy (afr_private_t *priv)
+static dict_t *
+afr_set_heal_info(char *status)
 {
-        int            i           = 0;
-
-        if (!priv)
-                goto out;
-        GF_FREE (priv->last_event);
-        if (priv->pending_key) {
-                for (i = 0; i < priv->child_count; i++)
-                        GF_FREE (priv->pending_key[i]);
-        }
-        GF_FREE (priv->pending_key);
-        GF_FREE (priv->children);
-        GF_FREE (priv->child_up);
-        LOCK_DESTROY (&priv->lock);
+    dict_t *dict = NULL;
+    int ret = -1;
 
-        GF_FREE (priv);
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_sizen(dict, "heal-info", status);
+    if (ret)
+        gf_msg("", GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
 out:
-        return;
+    /* Any error other than EINVAL, dict_set_dynstr frees status */
+    if (ret == -ENOMEM || ret == -EINVAL) {
+        GF_FREE(status);
+    }
+
+    if (ret && dict) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+    return dict;
 }
 
-void
-afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+static gf_boolean_t
+afr_is_dirty_count_non_unary_for_txn(xlator_t *this, struct afr_reply *replies,
+                                     afr_transaction_type type)
 {
-        afr_local_t     *local = NULL;
-        afr_fd_ctx_t    *fd_ctx   = NULL;
+    afr_private_t *priv = this->private;
+    int *dirty = alloca0(priv->child_count * sizeof(int));
+    int i = 0;
 
-        local = frame->local;
+    afr_selfheal_extract_xattr(this, replies, type, dirty, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dirty[i] > 1)
+            return _gf_true;
+    }
 
-        if (!local->fd)
-		return;
+    return _gf_false;
+}
 
-	fd_ctx = afr_fd_ctx_get (local->fd, this);
-	if (!fd_ctx)
-		return;
+static gf_boolean_t
+afr_is_dirty_count_non_unary(xlator_t *this, struct afr_reply *replies,
+                             ia_type_t ia_type)
+{
+    gf_boolean_t data_chk = _gf_false;
+    gf_boolean_t mdata_chk = _gf_false;
+    gf_boolean_t entry_chk = _gf_false;
+
+    switch (ia_type) {
+        case IA_IFDIR:
+            mdata_chk = _gf_true;
+            entry_chk = _gf_true;
+            break;
+        case IA_IFREG:
+            mdata_chk = _gf_true;
+            data_chk = _gf_true;
+            break;
+        default:
+            /*IA_IFBLK, IA_IFCHR, IA_IFLNK, IA_IFIFO, IA_IFSOCK*/
+            mdata_chk = _gf_true;
+            break;
+    }
+
+    if (data_chk && afr_is_dirty_count_non_unary_for_txn(
+                        this, replies, AFR_DATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (mdata_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_METADATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (entry_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_ENTRY_TRANSACTION)) {
+        return _gf_true;
+    }
 
-	fd_ctx->open_fd_count = local->open_fd_count;
+    return _gf_false;
 }
 
-int**
-afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending,
-                            dict_t *xattr, ia_type_t iat)
-{
-       int i = 0;
-       int **changelog = NULL;
-       int idx = -1;
-       int m_idx = 0;
-       int d_idx = 0;
-       int ret = 0;
+static int
+afr_update_heal_status(xlator_t *this, struct afr_reply *replies,
+                       ia_type_t ia_type, gf_boolean_t *esh, gf_boolean_t *dsh,
+                       gf_boolean_t *msh, unsigned char pending)
+{
+    int ret = -1;
+    GF_UNUSED int ret1 = 0;
+    int i = 0;
+    int io_domain_lk_count = 0;
+    int shd_domain_lk_count = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if ((replies[i].valid != 1) || (replies[i].op_ret != 0))
+            continue;
+        if (!io_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key1, &io_domain_lk_count);
+        }
+        if (!shd_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key2, &shd_domain_lk_count);
+        }
+    }
+
+    if (!pending) {
+        if ((afr_is_dirty_count_non_unary(this, replies, ia_type)) ||
+            (!io_domain_lk_count)) {
+            /* Needs heal. */
+            ret = 0;
+        } else {
+            /* No heal needed. */
+            *dsh = *esh = *msh = 0;
+        }
+    } else {
+        if (shd_domain_lk_count) {
+            ret = -EAGAIN; /*For 'possibly-healing'. */
+        } else {
+            ret = 0; /*needs heal. Just set a non -ve value so that it is
+                       assumed as the source index.*/
+        }
+    }
+    return ret;
+}
 
-       m_idx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-       d_idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+/*return EIO, EAGAIN or pending*/
+int
+afr_lockless_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                     inode_t **inode, gf_boolean_t *entry_selfheal,
+                     gf_boolean_t *data_selfheal,
+                     gf_boolean_t *metadata_selfheal, unsigned char *pending)
+{
+    int ret = -1;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    gf_boolean_t dsh = _gf_false;
+    gf_boolean_t msh = _gf_false;
+    gf_boolean_t esh = _gf_false;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *valid_on = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+    sources = alloca0(sizeof(*sources) * priv->child_count);
+    sinks = alloca0(sizeof(*sinks) * priv->child_count);
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    valid_on = alloca0(sizeof(*valid_on) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, inode, &dsh, &msh,
+                                        &esh, replies);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            valid_on[i] = 1;
+        }
+    }
+    if (msh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_METADATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (dsh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_DATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (esh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_ENTRY_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
 
-       idx = afr_index_from_ia_type (iat);
+    ret = afr_update_heal_status(this, replies, (*inode)->ia_type, &esh, &dsh,
+                                 &msh, *pending);
+out:
+    *data_selfheal = dsh;
+    *entry_selfheal = esh;
+    *metadata_selfheal = msh;
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return ret;
+}
 
-       changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
-       if (!changelog)
+int
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    unsigned char pending = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    inode_t *inode = NULL;
+    char *substr = NULL;
+    char *status = NULL;
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+
+    /*Use frame with lk-owner set*/
+    heal_frame = afr_frame_create(frame->this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+
+    ret = afr_lockless_inspect(heal_frame, this, loc->gfid, &inode,
+                               &entry_selfheal, &data_selfheal,
+                               &metadata_selfheal, &pending);
+
+    if (ret == -ENOMEM) {
+        ret = -1;
+        goto out;
+    }
+
+    if (pending & PFLAG_PENDING) {
+        gf_asprintf(&substr, "-pending");
+        if (!substr)
+            goto out;
+    }
+
+    if (ret == -EIO) {
+        ret = gf_asprintf(&status, "split-brain%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret == -EAGAIN) {
+        ret = gf_asprintf(&status, "possibly-healing%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret >= 0) {
+        /* value of ret = source index
+         * so ret >= 0 and at least one of the 3 booleans set to
+         * true means a source is identified; heal is required.
+         */
+        if (!data_selfheal && !entry_selfheal && !metadata_selfheal) {
+            status = gf_strdup("no-heal");
+            if (!status) {
+                ret = -1;
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        } else {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        }
+    } else if (ret < 0) {
+        /* Apart from above checked -ve ret values, there are
+         * other possible ret values like ENOTCONN
+         * (returned when number of valid replies received are
+         * less than 2)
+         * in which case heal is required when one of the
+         * selfheal booleans is set.
+         */
+        if (data_selfheal || entry_selfheal || metadata_selfheal) {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
                 goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        }
+    }
 
-       for (i = 0; i < priv->child_count; i++) {
-               if (!pending[i])
-                       continue;
+    ret = 0;
+    op_errno = 0;
 
-               changelog[i][m_idx] = hton32(1);
-               if (idx != -1)
-                       changelog[i][idx] = hton32(1);
-                /* If the newentry marking is on a newly created directory,
-                 * then mark it with the full-heal indicator.
-                 */
-                if ((IA_ISDIR (iat)) && (priv->esh_granular))
-                        changelog[i][d_idx] = hton32(1);
-       }
-       ret = afr_set_pending_dict (priv, xattr, changelog);
-       if (ret < 0) {
-               afr_matrix_cleanup (changelog, priv->child_count);
-               return NULL;
-       }
 out:
-       return changelog;
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    GF_FREE(substr);
+    return ret;
 }
 
-gf_boolean_t
-afr_decide_heal_info (afr_private_t *priv, unsigned char *sources, int source)
-{
-        int sources_count = 0;
+int
+_afr_is_split_brain(call_frame_t *frame, xlator_t *this,
+                    struct afr_reply *replies, afr_transaction_type type,
+                    gf_boolean_t *spb)
+{
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    witness = alloca0(priv->child_count * sizeof(*witness));
+
+    ret = afr_selfheal_find_direction(frame, this, replies, type,
+                                      priv->child_up, sources, sinks, witness,
+                                      NULL);
+    if (ret)
+        return ret;
 
-        if (source < 0)
-                goto out;
+    sources_count = AFR_COUNT(sources, priv->child_count);
+    if (!sources_count)
+        *spb = _gf_true;
 
-        sources_count = AFR_COUNT (sources, priv->child_count);
-        if (sources_count == priv->child_count)
-                return _gf_false;
-out:
-        return _gf_true;
+    return ret;
 }
 
 int
-afr_selfheal_locked_metadata_inspect (call_frame_t *frame, xlator_t *this,
-                                      inode_t *inode, gf_boolean_t *msh,
-                                      gf_boolean_t *pending)
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
 {
-        int ret = -1;
-        unsigned char *locked_on = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        struct afr_reply *locked_replies = NULL;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
 
-        afr_private_t *priv = this->private;
+    priv = this->private;
 
-        locked_on = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
+    replies = alloca0(sizeof(*replies) * priv->child_count);
 
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
 
-        ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-                                    LLONG_MAX - 1, 0, locked_on);
-        {
-                if (ret == 0) {
-                        /* Not a single lock */
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
-                ret = __afr_selfheal_metadata_prepare (frame, this, inode,
-                                                       locked_on, sources,
-                                                       sinks, healed_sinks,
-                                                       locked_replies,
-                                                       pending);
-                *msh = afr_decide_heal_info (priv, sources, ret);
-        }
-        afr_selfheal_uninodelk (frame, this, inode, this->name,
-                                LLONG_MAX - 1, 0, locked_on);
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = -EAGAIN;
+        goto out;
+    }
+
+    ret = _afr_is_split_brain(frame, this, replies, AFR_DATA_TRANSACTION,
+                              d_spb);
+    if (ret)
+        goto out;
+
+    ret = _afr_is_split_brain(frame, this, replies, AFR_METADATA_TRANSACTION,
+                              m_spb);
 out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
+    if (replies) {
+        afr_replies_wipe(replies, priv->child_count);
+        replies = NULL;
+    }
+    return ret;
 }
 
 int
-afr_selfheal_locked_data_inspect (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, gf_boolean_t *dsh,
-                                  gf_boolean_t *pflag)
-{
-        int ret = -1;
-        unsigned char *locked_on = NULL;
-        unsigned char *data_lock = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        afr_private_t   *priv = NULL;
-        fd_t          *fd = NULL;
-        struct afr_reply *locked_replies = NULL;
-        gf_boolean_t granular_locks = _gf_false;
-
-        priv = this->private;
-	if (strcmp ("granular", priv->locking_scheme) == 0)
-	        granular_locks = _gf_true;
-        locked_on = alloca0 (priv->child_count);
-        data_lock = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-
-        /* Heal-info does an open() on the file being examined so that the
-         * current eager-lock holding client, if present, at some point sees
-         * open-fd count being > 1 and releases the eager-lock so that heal-info
-         * doesn't remain blocked forever until IO completes.
-         */
-        ret = afr_selfheal_data_open (this, inode, &fd);
-        if (ret < 0) {
-                gf_msg_debug (this->name, -ret, "%s: Failed to open",
-                              uuid_utoa (inode->gfid));
-                goto out;
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    GF_FREE(opaque);
+    return 0;
+}
+
+int
+afr_get_split_brain_status(void *opaque)
+{
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    int ret = -1;
+    int op_errno = 0;
+    int i = 0;
+    char *choices = NULL;
+    char *status = NULL;
+    dict_t *dict = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    afr_spb_status_t *data = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    this = frame->this;
+    loc = data->loc;
+    priv = this->private;
+    children = priv->children;
+
+    inode = afr_inode_find(this, loc->gfid);
+    if (!inode)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Calculation for string length :
+     * (child_count X length of child-name) + SLEN("    Choices :")
+     * child-name consists of :
+     * a) 251 = max characters for volname according to GD_VOLUME_NAME_MAX
+     * b) strlen("-client-00,") assuming 16 replicas
+     */
+    choices = alloca0(priv->child_count * (256 + SLEN("-client-00,")) +
+                      SLEN("    Choices:"));
+
+    ret = afr_is_split_brain(frame, this, inode, loc->gfid, &d_spb, &m_spb);
+    if (ret) {
+        op_errno = -ret;
+        if (ret == -EAGAIN) {
+            ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                           SBRAIN_HEAL_NO_GO_MSG);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set GF_AFR_SBRAIN_STATUS in dict");
+            }
+        }
+        ret = -1;
+        goto out;
+    }
+
+    if (d_spb || m_spb) {
+        sprintf(choices, "    Choices:");
+        for (i = 0; i < priv->child_count; i++) {
+            strcat(choices, children[i]->name);
+            strcat(choices, ",");
         }
+        choices[strlen(choices) - 1] = '\0';
 
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+        ret = gf_asprintf(&status,
+                          "data-split-brain:%s    "
+                          "metadata-split-brain:%s%s",
+                          (d_spb) ? "yes" : "no", (m_spb) ? "yes" : "no",
+                          choices);
 
-        if (!granular_locks) {
-                ret = afr_selfheal_tryinodelk (frame, this, inode,
-                                              priv->sh_domain, 0, 0, locked_on);
+        if (-1 == ret) {
+            op_errno = ENOMEM;
+            goto out;
         }
-        {
-                if (!granular_locks && (ret == 0)) {
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
-                ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-                                            0, 0, data_lock);
-                {
-                        if (ret == 0) {
-                                ret = -afr_final_errno (frame->local, priv);
-                                if (ret == 0)
-                                        ret = -ENOTCONN;
-                                /* all invalid responses */
-                                goto unlock;
-                        }
-                        ret = __afr_selfheal_data_prepare (frame, this, inode,
-                                                           data_lock, sources,
-                                                           sinks, healed_sinks,
-                                                           locked_replies,
-                                                           pflag);
-                        *dsh = afr_decide_heal_info (priv, sources, ret);
-                }
-                afr_selfheal_uninodelk (frame, this, inode, this->name, 0, 0,
-                                        data_lock);
+        ret = dict_set_dynstr_sizen(dict, GF_AFR_SBRAIN_STATUS, status);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
-unlock:
-        if (!granular_locks)
-                afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0,
-                                        0, locked_on);
+    } else {
+        ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                       SFILE_NOT_UNDER_DATA);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = 0;
 out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        if (fd)
-                fd_unref (fd);
-        return ret;
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    return ret;
 }
 
-int
-afr_selfheal_locked_entry_inspect (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode,
-                                   gf_boolean_t *esh, gf_boolean_t *pflag)
-{
-        int ret = -1;
-        int source = -1;
-        afr_private_t   *priv = NULL;
-        unsigned char *locked_on = NULL;
-        unsigned char *data_lock = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        struct afr_reply *locked_replies = NULL;
-        gf_boolean_t granular_locks = _gf_false;
-
-        priv = this->private;
-	if (strcmp ("granular", priv->locking_scheme) == 0)
-	        granular_locks = _gf_true;
-        locked_on = alloca0 (priv->child_count);
-        data_lock = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-        if (!granular_locks) {
-                ret = afr_selfheal_tryentrylk (frame, this, inode,
-                                              priv->sh_domain, NULL, locked_on);
+int32_t
+afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    int op_errno = 0;
+    dict_t *dict = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+
+    local = frame->local;
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    heal_frame = afr_frame_create(this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+    /*Initiate heal with heal_frame with lk-owner set so that inodelk/entrylk
+     * work correctly*/
+    ret = afr_selfheal_do(heal_frame, this, loc->gfid);
+
+    if (ret == 1 || ret == 2) {
+        ret = dict_set_sizen_str_sizen(dict, "sh-fail-msg",
+                                       SFILE_NOT_IN_SPLIT_BRAIN);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Failed to set sh-fail-msg in dict");
+        ret = 0;
+        goto out;
+    } else {
+        if (local->xdata_rsp) {
+            /* 'sh-fail-msg' has been set in the dict during self-heal.*/
+            dict_copy(local->xdata_rsp, dict);
+            ret = 0;
+        } else if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
         }
-        {
-                if (!granular_locks && ret == 0) {
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
+    }
 
-                ret = afr_selfheal_entrylk (frame, this, inode, this->name,
-                                            NULL, data_lock);
-                {
-                        if (ret == 0) {
-                                ret = -afr_final_errno (frame->local, priv);
-                                if (ret == 0)
-                                        ret = -ENOTCONN;
-                                /* all invalid responses */
-                                goto unlock;
-                        }
-                        ret = __afr_selfheal_entry_prepare (frame, this, inode,
-                                                            data_lock, sources,
-                                                            sinks, healed_sinks,
-                                                            locked_replies,
-                                                            &source, pflag);
-                        if ((ret == 0) && source < 0)
-                                ret = -EIO;
-                        *esh = afr_decide_heal_info (priv, sources, ret);
-                }
-                afr_selfheal_unentrylk (frame, this, inode, this->name, NULL,
-                                        data_lock, NULL);
-        }
-unlock:
-        if (!granular_locks)
-                afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain,
-                                        NULL, locked_on, NULL);
 out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    if (local->op == GF_FOP_GETXATTR)
+        AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    else if (local->op == GF_FOP_SETXATTR)
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int
-afr_selfheal_locked_inspect (call_frame_t *frame, xlator_t *this, uuid_t gfid,
-                             inode_t **inode,
-                             gf_boolean_t *entry_selfheal,
-                             gf_boolean_t *data_selfheal,
-                             gf_boolean_t *metadata_selfheal,
-                             gf_boolean_t *pending)
-
+afr_get_child_index_from_name(xlator_t *this, char *name)
 {
-        int ret             = -1;
-        gf_boolean_t    dsh = _gf_false;
-        gf_boolean_t    msh = _gf_false;
-        gf_boolean_t    esh = _gf_false;
+    afr_private_t *priv = this->private;
+    int index = -1;
 
-        ret = afr_selfheal_unlocked_inspect (frame, this, gfid, inode,
-                                             &dsh, &msh, &esh);
-        if (ret)
-                goto out;
+    for (index = 0; index < priv->child_count; index++) {
+        if (!strcmp(priv->children[index]->name, name))
+            goto out;
+    }
+    index = -1;
+out:
+    return index;
+}
 
-        /* For every heal type hold locks and check if it indeed needs heal */
+void
+afr_priv_need_heal_set(afr_private_t *priv, gf_boolean_t need_heal)
+{
+    LOCK(&priv->lock);
+    {
+        priv->need_heal = need_heal;
+    }
+    UNLOCK(&priv->lock);
+}
 
-        if (msh) {
-                ret = afr_selfheal_locked_metadata_inspect (frame, this,
-                                                            *inode, &msh,
-                                                            pending);
-                if (ret == -EIO)
-                        goto out;
-        }
+void
+afr_set_need_heal(xlator_t *this, afr_local_t *local)
+{
+    int i = 0;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_false;
 
-        if (dsh) {
-                ret = afr_selfheal_locked_data_inspect (frame, this, *inode,
-                                                        &dsh, pending);
-                if (ret == -EIO || (ret == -EAGAIN))
-                        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].need_heal) {
+            need_heal = _gf_true;
+            break;
         }
+    }
+    afr_priv_need_heal_set(priv, need_heal);
+    return;
+}
 
-        if (esh) {
-                ret = afr_selfheal_locked_entry_inspect (frame, this, *inode,
-                                                         &esh, pending);
-        }
+gf_boolean_t
+afr_get_need_heal(xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_true;
 
-out:
-        *data_selfheal = dsh;
-        *entry_selfheal = esh;
-        *metadata_selfheal = msh;
-        return ret;
+    LOCK(&priv->lock);
+    {
+        need_heal = priv->need_heal;
+    }
+    UNLOCK(&priv->lock);
+    return need_heal;
 }
 
-dict_t*
-afr_set_heal_info (char *status)
+int
+afr_get_msg_id(char *op_type)
 {
-        dict_t *dict = NULL;
-        int    ret   = -1;
+    if (!strcmp(op_type, GF_AFR_REPLACE_BRICK))
+        return AFR_MSG_REPLACE_BRICK_STATUS;
+    else if (!strcmp(op_type, GF_AFR_ADD_BRICK))
+        return AFR_MSG_ADD_BRICK_STATUS;
+    return -1;
+}
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -ENOMEM;
-                goto out;
-        }
+int
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *heal_frame,
+                                    void *opaque)
+{
+    call_frame_t *txn_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    xlator_t *this = NULL;
 
-        ret = dict_set_str (dict, "heal-info", status);
-        if (ret)
-                gf_msg ("", GF_LOG_WARNING, -ret,
-                        AFR_MSG_DICT_SET_FAILED,
-                        "Failed to set heal-info key to "
-                        "%s", status);
-out:
-        return dict;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    local = txn_frame->local;
+    this = txn_frame->this;
+
+    /* Refresh the inode agan and proceed with the transaction.*/
+    afr_inode_refresh(txn_frame, this, local->inode, NULL, local->refreshfn);
+
+    AFR_STACK_DESTROY(heal_frame);
+
+    return 0;
 }
 
 int
-afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_boolean_t    data_selfheal     = _gf_false;
-        gf_boolean_t    metadata_selfheal = _gf_false;
-        gf_boolean_t    entry_selfheal    = _gf_false;
-        gf_boolean_t    pending           = _gf_false;
-        dict_t         *dict              = NULL;
-        int             ret               = -1;
-        int             op_errno          = 0;
-        int             size              = 0;
-        inode_t        *inode             = NULL;
-        char           *substr            = NULL;
-        char           *status            = NULL;
-
-        ret = afr_selfheal_locked_inspect (frame, this, loc->gfid, &inode,
-                                           &entry_selfheal,
-                                           &data_selfheal, &metadata_selfheal,
-                                           &pending);
-
-        if (ret == -ENOMEM) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
+afr_fav_child_reset_sink_xattrs(void *opaque)
+{
+    call_frame_t *heal_frame = NULL;
+    call_frame_t *txn_frame = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *txn_local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int ret = 0;
+
+    heal_frame = (call_frame_t *)opaque;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    txn_local = txn_frame->local;
+    this = txn_frame->this;
+    inode = txn_local->inode;
+    priv = this->private;
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_DATA_TRANSACTION, &d_spb);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_METADATA_TRANSACTION, &m_spb);
+
+    /* Take appropriate locks and reset sink xattrs. */
+    if (d_spb) {
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name, 0, 0,
+                                   locked_on);
+        {
+            if (ret < priv->child_count)
+                goto data_unlock;
+            ret = __afr_selfheal_data_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
+        }
+    data_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name, 0, 0,
+                               locked_on);
+    }
+
+    if (m_spb) {
+        memset(locked_on, 0, sizeof(*locked_on) * priv->child_count);
+        memset(undid_pending, 0, sizeof(*undid_pending) * priv->child_count);
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name,
+                                   LLONG_MAX - 1, 0, locked_on);
+        {
+            if (ret < priv->child_count)
+                goto mdata_unlock;
+            ret = __afr_selfheal_metadata_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
         }
+    mdata_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name,
+                               LLONG_MAX - 1, 0, locked_on);
+    }
 
-        if (pending) {
-                size = strlen ("-pending") + 1;
-                gf_asprintf (&substr, "-pending");
-                if (!substr)
-                        goto out;
-        }
+    return ret;
+}
 
-        if (ret == -EIO) {
-                size += strlen ("split-brain") + 1;
-                ret = gf_asprintf (&status, "split-brain%s",
-                                   substr? substr : "");
-                if (ret < 0)
-                        goto out;
-                dict = afr_set_heal_info (status);
-        } else if (ret == -EAGAIN) {
-                size += strlen ("possibly-healing") + 1;
-                ret = gf_asprintf (&status, "possibly-healing%s",
-                                   substr? substr : "");
-                if (ret < 0)
-                        goto out;
-                dict = afr_set_heal_info (status);
-        } else if (ret >= 0) {
-                /* value of ret = source index
-                 * so ret >= 0 and at least one of the 3 booleans set to
-                 * true means a source is identified; heal is required.
-                 */
-                if (!data_selfheal && !entry_selfheal &&
-                    !metadata_selfheal) {
-                        dict = afr_set_heal_info ("no-heal");
-                } else {
-                        size += strlen ("heal") + 1;
-                        ret = gf_asprintf (&status, "heal%s",
-                                           substr? substr : "");
-                        if (ret < 0)
-                                goto out;
-                        dict = afr_set_heal_info (status);
-                }
-        } else if (ret < 0) {
-                /* Apart from above checked -ve ret values, there are
-                 * other possible ret values like ENOTCONN
-                 * (returned when number of valid replies received are
-                 * less than 2)
-                 * in which case heal is required when one of the
-                 * selfheal booleans is set.
-                 */
-                if (data_selfheal || entry_selfheal ||
-                    metadata_selfheal) {
-                        size += strlen ("heal") + 1;
-                        ret = gf_asprintf (&status, "heal%s",
-                                           substr? substr : "");
-                        if (ret < 0)
-                                goto out;
-                        dict = afr_set_heal_info (status);
-                }
-        }
-        ret = 0;
+/*
+ * Concatenates the xattrs in local->replies separated by a delimiter.
+ */
+int
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *xattr = NULL;
+    int i = 0;
+    int len = 0;
+    int keylen = 0;
+    size_t str_len = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+
+    keylen = strlen(local->cont.getxattr.name);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret) {
+            str_len = strlen(default_str);
+            buf = strncat(buf, default_str, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        } else {
+            ret = dict_get_strn(local->replies[i].xattr,
+                                local->cont.getxattr.name, keylen, &xattr);
+            if (ret) {
+                gf_msg("TEST", GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                       "Failed to get the node_uuid of brick "
+                       "%d",
+                       i);
+                goto out;
+            }
+            str_len = strlen(xattr);
+            buf = strncat(buf, xattr, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        }
+    }
+    buf[--len] = '\0'; /*remove the last delimiter*/
+    if (serz_len)
+        *serz_len = ++len;
+    ret = 0;
 
 out:
-        AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        if (dict)
-               dict_unref (dict);
-        if (inode)
-                inode_unref (inode);
-        GF_FREE (substr);
-        return ret;
+    return ret;
 }
 
-int
-_afr_is_split_brain (call_frame_t *frame, xlator_t *this,
-                         struct afr_reply *replies,
-                         afr_transaction_type type,
-                         gf_boolean_t *spb)
-{
-        afr_private_t    *priv              = NULL;
-        uint64_t         *witness           = NULL;
-        unsigned char    *sources           = NULL;
-        unsigned char    *sinks             = NULL;
-        int               sources_count     = 0;
-        int               ret               = 0;
-
-        priv = this->private;
-
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        witness = alloca0(priv->child_count * sizeof (*witness));
-
-        ret = afr_selfheal_find_direction (frame, this, replies,
-					   type, priv->child_up, sources,
-                                           sinks, witness, NULL);
-        if (ret)
-                return ret;
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    uint64_t write_subvol = 0;
 
-        sources_count = AFR_COUNT (sources, priv->child_count);
-        if (!sources_count)
-                *spb = _gf_true;
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    write_subvol = local->inode_ctx->write_subvol;
+    UNLOCK(&local->inode->lock);
 
-        return ret;
+    return write_subvol;
 }
 
 int
-afr_is_split_brain (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                    uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
-{
-        int    ret                          = -1;
-        afr_private_t    *priv              = NULL;
-        struct afr_reply *replies           = NULL;
-
-        priv = this->private;
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int event = 0;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    data_accused = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    event = local->event_generation;
+
+    afr_readables_fill(frame, this, local->inode, data_accused,
+                       metadata_accused, data_readable, metadata_readable,
+                       NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_readable[i])
+            datamap |= (1 << i);
+        if (metadata_readable[i])
+            metadatamap |= (1 << i);
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    LOCK(&local->inode->lock);
+    {
+        if (local->inode_ctx->write_subvol == 0 &&
+            local->transaction.type == AFR_DATA_TRANSACTION) {
+            local->inode_ctx->write_subvol = val;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
+}
 
-        replies = alloca0 (sizeof (*replies) * priv->child_count);
+int
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
 
-        ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
-        if (ret)
-                goto out;
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    {
+        GF_ASSERT(local->inode_ctx->lock_count > 0);
+        local->inode_ctx->lock_count--;
 
-        ret = _afr_is_split_brain (frame, this, replies,
-                                    AFR_DATA_TRANSACTION, d_spb);
-        if (ret)
-                goto out;
+        if (!local->inode_ctx->lock_count)
+            local->inode_ctx->write_subvol = 0;
+    }
+    UNLOCK(&local->inode->lock);
 
-        ret = _afr_is_split_brain (frame, this, replies,
-                                    AFR_METADATA_TRANSACTION, m_spb);
-out:
-        if (replies) {
-                afr_replies_wipe (replies, priv->child_count);
-                replies = NULL;
-        }
-        return ret;
+    return 0;
 }
 
 int
-afr_get_split_brain_status_cbk (int ret, call_frame_t *frame, void *opaque)
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode)
 {
-        GF_FREE (opaque);
-        return 0;
-}
+    int ret = 0;
 
-int
-afr_get_split_brain_status (void *opaque)
-{
-        gf_boolean_t      d_spb             = _gf_false;
-        gf_boolean_t      m_spb             = _gf_false;
-        int               ret               = -1;
-        int               op_errno          = 0;
-        int               i                 = 0;
-        char             *choices           = NULL;
-        char             *status            = NULL;
-        dict_t           *dict              = NULL;
-        inode_t          *inode             = NULL;
-        afr_private_t    *priv              = NULL;
-        xlator_t         **children         = NULL;
-        call_frame_t     *frame             = NULL;
-        xlator_t         *this              = NULL;
-        loc_t            *loc               = NULL;
-        afr_spb_status_t *data              = NULL;
-
-        data     = opaque;
-        frame    = data->frame;
-        this     = frame->this;
-        loc      = data->loc;
-        priv     = this->private;
-        children = priv->children;
-
-        inode = afr_inode_find (this, loc->gfid);
-        if (!inode)
-                goto out;
+    local->inode = inode_ref(inode);
+    LOCK(&local->inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, local->inode, &local->inode_ctx);
+    }
+    UNLOCK(&local->inode->lock);
+    if (ret < 0) {
+        gf_msg_callingfn(
+            this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_INODE_CTX_GET_FAILED,
+            "Error getting inode ctx %s", uuid_utoa(local->inode->gfid));
+    }
+    return ret;
+}
 
-        /* Calculation for string length :
-        * (child_count X length of child-name) + strlen ("    Choices :")
-        * child-name consists of :
-        * a) 256 = max characters for volname according to GD_VOLUME_NAME_MAX
-        * b) strlen ("-client-00,") assuming 16 replicas
-        */
-        choices = alloca0 (priv->child_count * (256 + strlen ("-client-00,")) +
-                           strlen ("    Choices:"));
+gf_boolean_t
+afr_ta_is_fop_called_from_synctask(xlator_t *this)
+{
+    struct synctask *task = NULL;
+    gf_lkowner_t tmp_owner = {
+        0,
+    };
 
-        ret = afr_is_split_brain (frame, this, inode, loc->gfid, &d_spb,
-                                  &m_spb);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+    task = synctask_get();
+    if (!task)
+        return _gf_false;
 
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    set_lk_owner_from_ptr(&tmp_owner, (void *)this);
 
-        if (d_spb || m_spb) {
-                sprintf (choices, "    Choices:");
-                for (i = 0; i < priv->child_count; i++) {
-                        strcat (choices, children[i]->name);
-                        strcat (choices, ",");
-                }
-                choices[strlen (choices) - 1] = '\0';
+    if (!is_same_lkowner(&tmp_owner, &task->frame->root->lk_owner))
+        return _gf_false;
 
-                ret = gf_asprintf (&status, "data-split-brain:%s    "
-                                    "metadata-split-brain:%s%s",
-                                    (d_spb) ? "yes" : "no",
-                                    (m_spb) ? "yes" : "no", choices);
+    return _gf_true;
+}
 
-                if (-1 == ret) {
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, GF_AFR_SBRAIN_STATUS, status);
-                if (ret) {
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
+int
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    afr_private_t *priv = this->private;
+    gf_boolean_t locked = _gf_false;
+    struct gf_flock flock1 = {
+        0,
+    };
+    struct gf_flock flock2 = {
+        0,
+    };
+    int32_t cmd = 0;
+
+    /* Clients must take AFR_TA_DOM_NOTIFY lock only when the previous lock
+     * has been released in afr_notify due to upcall notification from shd.
+     */
+    GF_ASSERT(priv->ta_notify_dom_lock_offset == 0);
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock1.l_type = F_WRLCK;
+
+    while (!locked) {
+        if (priv->shd.iamshd) {
+            cmd = F_SETLKW;
+            flock1.l_start = 0;
+            flock1.l_len = 0;
         } else {
-                ret = dict_set_str (dict, GF_AFR_SBRAIN_STATUS,
-                                    "The file is not under data or"
-                                    " metadata split-brain");
-                if (ret) {
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-        }
+            cmd = F_SETLK;
+            gf_uuid_generate(gfid);
+            flock1.l_start = gfid_to_ino(gfid);
+            if (flock1.l_start < 0)
+                flock1.l_start = -flock1.l_start;
+            flock1.l_len = 1;
+        }
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, cmd, &flock1, NULL, NULL);
+        if (!ret) {
+            locked = _gf_true;
+            priv->ta_notify_dom_lock_offset = flock1.l_start;
+        } else if (ret == -EAGAIN) {
+            continue;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+                   "Failed to get "
+                   "AFR_TA_DOM_NOTIFY lock on %s.",
+                   loc->name);
+            goto out;
+        }
+    }
+
+    flock2.l_type = F_WRLCK;
+    flock2.l_start = 0;
+    flock2.l_len = 0;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLKW, &flock2, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to get AFR_TA_DOM_MODIFY lock on %s.", loc->name);
+        flock1.l_type = F_UNLCK;
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock1, NULL,
+                             NULL);
+    }
+out:
+    return ret;
+}
 
-        ret = 0;
+int
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock.l_type = F_UNLCK;
+    flock.l_start = 0;
+    flock.l_len = 0;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_MODIFY lock.");
+        goto out;
+    }
+
+    if (!priv->shd.iamshd)
+        /* Mounts (clients) will not release the AFR_TA_DOM_NOTIFY lock
+         * in post-op as they use it as a notification mechanism. When
+         * shd sends a lock request on TA during heal, the clients will
+         * receive a lock-contention upcall notification upon which they
+         * will release the AFR_TA_DOM_NOTIFY lock after completing the
+         * in flight I/O.*/
+        goto out;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
 out:
-        AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        if (dict)
-               dict_unref (dict);
-        if (inode)
-                inode_unref (inode);
-        return ret;
+    return ret;
 }
 
-int32_t
-afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
+call_frame_t *
+afr_ta_frame_create(xlator_t *this)
 {
-        int             ret               = 0;
-        int             op_errno          = 0;
-        dict_t         *dict              = NULL;
-        afr_local_t    *local             = NULL;
+    call_frame_t *frame = NULL;
+    void *lk_owner = NULL;
 
-        local = frame->local;
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return NULL;
+    lk_owner = (void *)this;
+    afr_set_lk_owner(frame, this, lk_owner);
+    return frame;
+}
 
-        ret = afr_selfheal_do (frame, this, loc->gfid);
+gf_boolean_t
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local)
+{
+    int data_count = 0;
 
-        if (ret == 1 || ret == 2) {
-                ret = dict_set_str (dict, "sh-fail-msg",
-                                    "File not in split-brain");
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Failed to set sh-fail-msg in dict");
-                ret = 0;
-                goto out;
-        } else {
-                if (local->xdata_rsp) {
-                        /* 'sh-fail-msg' has been set in the dict during self-heal.*/
-                        dict_copy (local->xdata_rsp, dict);
-                        ret = 0;
-                } else if (ret < 0) {
-                        op_errno = -ret;
-                        ret = -1;
-                }
-        }
+    data_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (data_count == 2) {
+        return _gf_true;
+    } else if (data_count == 1 && local->ta_child_up) {
+        return _gf_true;
+    }
 
-out:
-        if (local->op == GF_FOP_GETXATTR)
-                AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        else if (local->op == GF_FOP_SETXATTR)
-                AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        if (dict)
-                dict_unref(dict);
-        return ret;
+    return _gf_false;
 }
 
-int
-afr_get_child_index_from_name (xlator_t *this, char *name)
+static gf_boolean_t
+afr_is_add_replica_mount_lookup_on_root(call_frame_t *frame)
 {
-        afr_private_t *priv  = this->private;
-        int            index = -1;
+    afr_local_t *local = NULL;
 
-        for (index = 0; index < priv->child_count; index++) {
-                if (!strcmp (priv->children[index]->name, name))
-                        goto out;
-        }
-        index = -1;
-out:
-        return index;
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT)
+        return _gf_false;
+
+    local = frame->local;
+
+    if (local->op != GF_FOP_LOOKUP)
+        /* TODO:If the replica count is being increased on a plain distribute
+         * volume that was never mounted, we need to allow setxattr on '/' with
+         * GF_CLIENT_PID_NO_ROOT_SQUASH to accomodate for DHT layout setting */
+        return _gf_false;
+
+    if (local->inode == NULL)
+        return _gf_false;
+
+    if (!__is_root_gfid(local->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
-void
-afr_priv_need_heal_set (afr_private_t *priv, gf_boolean_t need_heal)
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame, const unsigned int up_children_count)
 {
-        LOCK (&priv->lock);
-        {
-                priv->need_heal = need_heal;
-        }
-        UNLOCK (&priv->lock);
+    if (frame && (up_children_count > 0) &&
+        afr_is_add_replica_mount_lookup_on_root(frame))
+        return _gf_true;
+
+    return _gf_false;
 }
 
 void
-afr_set_need_heal (xlator_t *this, afr_local_t *local)
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this)
 {
-        int             i         = 0;
-        afr_private_t  *priv      = this->private;
-        gf_boolean_t    need_heal = _gf_false;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    unsigned char *success_replies = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->replies[i].valid && local->replies[i].need_heal) {
-                        need_heal = _gf_true;
-                        break;
-                }
-        }
-        afr_priv_need_heal_set (priv, need_heal);
-        return;
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->quorum_count && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_errno = afr_final_errno(local, priv);
+        if (!local->op_errno)
+            local->op_errno = afr_quorum_errno(priv);
+        local->op_ret = -1;
+    }
 }
 
 gf_boolean_t
-afr_get_need_heal (xlator_t *this)
-{
-        afr_private_t  *priv      = this->private;
-        gf_boolean_t    need_heal = _gf_true;
-
-        LOCK (&priv->lock);
-        {
-                need_heal = priv->need_heal;
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
+{
+    int *pending = NULL;
+    int ret = 0;
+    int i = 0;
+
+    ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
+    if (ret == 0) {
+        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+            /* Not doing a ntoh32(pending) as we just want to check
+             * if it is non-zero or not. */
+            if (pending[i]) {
+                return _gf_true;
+            }
         }
-        UNLOCK (&priv->lock);
-        return need_heal;
-}
+    }
 
-int
-afr_get_msg_id (char *op_type)
-{
-
-        if (!strcmp (op_type, GF_AFR_REPLACE_BRICK))
-                return AFR_MSG_REPLACE_BRICK_STATUS;
-        else if (!strcmp (op_type, GF_AFR_ADD_BRICK))
-                return AFR_MSG_ADD_BRICK_STATUS;
-        return -1;
+    return _gf_false;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 2260e5dac26..f8bf8340dab 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -8,345 +8,339 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <string.h>
 
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "checksum.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
-
 int32_t
-afr_opendir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 fd_t *fd, dict_t *xdata)
+afr_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t   *local             = NULL;
-        int            call_count        = -1;
-        int32_t        child_index       = 0;
-	afr_fd_ctx_t  *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                } else {
-                        local->op_ret = op_ret;
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                }
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int32_t child_index = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+    child_index = (long)cookie;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        UNLOCK (&frame->lock);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        call_count = afr_frame_return (frame);
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        AFR_STACK_UNWIND(opendir, frame, local->op_ret, local->op_errno,
+                         local->fd, NULL);
+    }
 
-        if (call_count == 0)
-		AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-				  local->op_errno, local->fd, NULL);
-        return 0;
+    return 0;
 }
 
-
 int
-afr_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        afr_private_t * priv        = NULL;
-        afr_local_t   * local       = NULL;
-        int             i           = 0;
-        int             call_count  = -1;
-        int32_t         op_errno    = ENOMEM;
-	afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = -1;
+    int32_t op_errno = ENOMEM;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		goto out;
+    local->op = GF_FOP_OPENDIR;
 
-        loc_copy (&local->loc, loc);
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
 
-        local->fd    = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
 
-        call_count = local->call_count;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_opendir_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           loc, fd, NULL);
+    loc_copy(&local->loc, loc);
 
-                        if (!--call_count)
-                                break;
-                }
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
+
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_opendir_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, loc, fd, NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
-        return 0;
+    AFR_STACK_UNWIND(opendir, frame, -1, op_errno, fd, NULL);
+    return 0;
 }
 
 static int
-afr_validate_read_subvol (inode_t *inode, xlator_t *this, int par_read_subvol)
+afr_validate_read_subvol(inode_t *inode, xlator_t *this, int par_read_subvol)
 {
-	int             gen               = 0;
-        int             entry_read_subvol = 0;
-	unsigned char  *data_readable     = NULL;
-	unsigned char  *metadata_readable = NULL;
-        afr_private_t  *priv              = NULL;
-
-        priv = this->private;
-	data_readable = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_get (inode, this, data_readable,
-				   metadata_readable, &gen);
-
-        if (gen != priv->event_generation ||
-                !data_readable[par_read_subvol] ||
-                !metadata_readable[par_read_subvol])
-                return -1;
-
-        /* Once the control reaches the following statement, it means that the
-         * parent's read subvol is perfectly readable. So calling
-         * either afr_data_subvol_get() or afr_metadata_subvol_get() would
-         * yield the same result. Hence, choosing afr_data_subvol_get() below.
-         */
-
-        if (!priv->consistent_metadata)
-                return 0;
-
-        /* For an inode fetched through readdirp which is yet to be linked,
-         * inode ctx would not be initialised (yet). So this function returns
-         * -1 above due to gen being 0, which is why it is OK to pass NULL for
-         *  read_subvol_args here.
-         */
-        entry_read_subvol = afr_data_subvol_get (inode, this, NULL, NULL,
-                                                 NULL, NULL);
-        if (entry_read_subvol != par_read_subvol)
-                return -1;
-
+    int gen = 0;
+    int entry_read_subvol = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              &gen);
+
+    if (gen != priv->event_generation || !data_readable[par_read_subvol] ||
+        !metadata_readable[par_read_subvol])
+        return -1;
+
+    /* Once the control reaches the following statement, it means that the
+     * parent's read subvol is perfectly readable. So calling
+     * either afr_data_subvol_get() or afr_metadata_subvol_get() would
+     * yield the same result. Hence, choosing afr_data_subvol_get() below.
+     */
+
+    if (!priv->consistent_metadata)
         return 0;
 
+    /* For an inode fetched through readdirp which is yet to be linked,
+     * inode ctx would not be initialised (yet). So this function returns
+     * -1 above due to gen being 0, which is why it is OK to pass NULL for
+     *  read_subvol_args here.
+     */
+    entry_read_subvol = afr_data_subvol_get(inode, this, NULL, NULL, NULL,
+                                            NULL);
+    if (entry_read_subvol != par_read_subvol)
+        return -1;
+
+    return 0;
 }
 
 static void
-afr_readdir_transform_entries (gf_dirent_t *subvol_entries, int subvol,
-			       gf_dirent_t *entries, fd_t *fd)
+afr_readdir_transform_entries(call_frame_t *frame, gf_dirent_t *subvol_entries,
+                              int subvol, gf_dirent_t *entries, fd_t *fd)
 {
-        int            ret   = -1;
-        gf_dirent_t   *entry = NULL;
-        gf_dirent_t   *tmp   = NULL;
-        xlator_t      *this  = NULL;
-        afr_private_t *priv  = NULL;
-        gf_boolean_t  need_heal = _gf_false;
-        gf_boolean_t  validate_subvol = _gf_false;
-
-        this = THIS;
-        priv = this->private;
-
-        need_heal = afr_get_need_heal (this);
-        validate_subvol = need_heal | priv->consistent_metadata;
-
-        list_for_each_entry_safe (entry, tmp, &subvol_entries->list, list) {
-                if (__is_root_gfid (fd->inode->gfid) &&
-                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-			continue;
-                }
-
-		list_del_init (&entry->list);
-		list_add_tail (&entry->list, &entries->list);
-
-                if (!validate_subvol)
-                        continue;
-
-		if (entry->inode) {
-                        ret = afr_validate_read_subvol (entry->inode, this,
-                                                        subvol);
-                        if (ret == -1) {
-				inode_unref (entry->inode);
-				entry->inode = NULL;
-                                continue;
-			}
-		}
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t validate_subvol = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+
+    need_heal = afr_get_need_heal(this);
+    validate_subvol = need_heal | priv->consistent_metadata;
+
+    list_for_each_entry_safe(entry, tmp, &subvol_entries->list, list)
+    {
+        if (afr_is_private_directory(priv, fd->inode->gfid, entry->d_name,
+                                     frame->root->pid)) {
+            continue;
         }
-}
 
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &entries->list);
+
+        if (!validate_subvol)
+            continue;
+
+        if (entry->inode) {
+            ret = afr_validate_read_subvol(entry->inode, this, subvol);
+            if (ret == -1) {
+                inode_unref(entry->inode);
+                entry->inode = NULL;
+                continue;
+            }
+        }
+    }
+}
 
 int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
-		 dict_t *xdata)
+afr_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+                dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	gf_dirent_t  entries;
+    afr_local_t *local = NULL;
+    gf_dirent_t entries;
 
-	INIT_LIST_HEAD (&entries.list);
+    INIT_LIST_HEAD(&entries.list);
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0 && !local->cont.readdir.offset) {
-		/* failover only if this was first readdir, detected
-		   by offset == 0 */
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0 && !local->cont.readdir.offset) {
+        /* failover only if this was first readdir, detected
+           by offset == 0 */
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (op_ret >= 0)
-		afr_readdir_transform_entries (subvol_entries, (long) cookie,
-					       &entries, local->fd);
+    if (op_ret >= 0)
+        afr_readdir_transform_entries(frame, subvol_entries, (long)cookie,
+                                      &entries, local->fd);
 
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, xdata);
+    AFR_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, xdata);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_readdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readdir, frame, local->op_ret,
-				  local->op_errno, 0, 0);
-		return 0;
-	}
-
-	fd_ctx->readdir_subvol = subvol;
-
-        if (local->op == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) subvol,
-                                   priv->children[subvol],
-                                   priv->children[subvol]->fops->readdir,
-				   local->fd, local->cont.readdir.size,
-                                   local->cont.readdir.offset,
-				   local->xdata_req);
-        else
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) subvol,
-                                   priv->children[subvol],
-                                   priv->children[subvol]->fops->readdirp,
-				   local->fd, local->cont.readdir.size,
-				   local->cont.readdir.offset,
-				   local->xdata_req);
-	return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx) {
+        local->op_errno = EINVAL;
+        local->op_ret = -1;
+    }
+
+    if (subvol == -1 || !fd_ctx) {
+        AFR_STACK_UNWIND(readdir, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    fd_ctx->readdir_subvol = subvol;
+
+    if (local->op == GF_FOP_READDIR)
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdir, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    else
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdirp, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-		off_t offset, int whichop, dict_t *dict)
+afr_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, int whichop, dict_t *dict)
 {
-        afr_local_t   *local     = NULL;
-        int32_t       op_errno   = 0;
-	int           subvol = -1;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx) {
-	        op_errno = EINVAL;
-		goto out;
-        }
-
-	local->op = whichop;
-        local->fd = fd_ref (fd);
-        local->cont.readdir.size = size;
-	local->cont.readdir.offset = offset;
-        local->xdata_req = (dict)? dict_ref (dict) : NULL;
-
-	subvol = fd_ctx->readdir_subvol;
-
-	if (offset == 0 || subvol == -1) {
-		/* First readdir has option of failing over and selecting
-		   an appropriate read subvolume */
-		afr_read_txn (frame, this, fd->inode, afr_readdir_wind,
-			      AFR_DATA_TRANSACTION);
-	} else {
-		/* But continued readdirs MUST stick to the same subvolume
-		   without an option to failover */
-		afr_readdir_wind (frame, this, subvol);
-	}
-
-        return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int subvol = -1;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    local->op = whichop;
+    local->fd = fd_ref(fd);
+    local->cont.readdir.size = size;
+    local->cont.readdir.offset = offset;
+    local->xdata_req = (dict) ? dict_ref(dict) : NULL;
+
+    subvol = fd_ctx->readdir_subvol;
+
+    if (offset == 0 || subvol == -1) {
+        /* First readdir has option of failing over and selecting
+           an appropriate read subvolume */
+        afr_read_txn(frame, this, fd->inode, afr_readdir_wind,
+                     AFR_DATA_TRANSACTION);
+    } else {
+        /* But continued readdirs MUST stick to the same subvolume
+           without an option to failover */
+        afr_readdir_wind(frame, this, subvol);
+    }
+
+    return 0;
 out:
-        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-	afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *dict)
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd)
+afr_releasedir(xlator_t *this, fd_t *fd)
 {
-        afr_cleanup_fd_ctx (this, fd);
+    afr_cleanup_fd_ctx(this, fd);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 09456d15949..773e925ec6c 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -11,26 +11,23 @@
 #ifndef __DIR_READ_H__
 #define __DIR_READ_H__
 
-
 int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd, dict_t *xdata);
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
 
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd);
+afr_releasedir(xlator_t *this, fd_t *fd);
 
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
-
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata);
 
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, size_t size, off_t offset, dict_t *dict);
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict);
 
 int32_t
-afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags, dict_t *xdata);
-
+afr_checksum(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             dict_t *xdata);
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index f3de5352d7e..b7cceb79158 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -8,532 +8,493 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
 void
-afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this);
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this);
 
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-        int     ret = -1;
-        char    *child_path = NULL;
+    int ret = -1;
+    char *child_path = NULL;
+
+    if (!child->parent) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
+
+    child_path = gf_strdup(child->path);
+    if (!child_path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->path = gf_strdup(dirname(child_path));
+    if (!parent->path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->inode = inode_ref(child->parent);
+    gf_uuid_copy(parent->gfid, child->pargfid);
+
+    ret = 0;
+out:
+    GF_FREE(child_path);
 
-        if (!child->parent) {
-                if (op_errno)
-                        *op_errno = EINVAL;
-                goto out;
-        }
+    return ret;
+}
 
-        child_path = gf_strdup (child->path);
-        if (!child_path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
+static void
+__afr_dir_write_finalize(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int inode_read_subvol = -1;
+    int parent_read_subvol = -1;
+    int parent2_read_subvol = -1;
+    int i = 0;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret == -1)
+            continue;
+        gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+        args.ia_type = local->replies[i].poststat.ia_type;
+        break;
+    }
+
+    if (local->inode) {
+        if (local->op != GF_FOP_RENAME && local->op != GF_FOP_LINK)
+            afr_replies_interpret(frame, this, local->inode, NULL);
+
+        inode_read_subvol = afr_data_subvol_get(local->inode, this, NULL, NULL,
+                                                NULL, &args);
+    }
+
+    if (local->parent)
+        parent_read_subvol = afr_data_subvol_get(local->parent, this, NULL,
+                                                 local->readable, NULL, NULL);
+
+    if (local->parent2)
+        parent2_read_subvol = afr_data_subvol_get(local->parent2, this, NULL,
+                                                  local->readable2, NULL, NULL);
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                         local->parent2, local->readable2);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0) {
+            if (local->inode)
+                afr_inode_need_refresh_set(local->inode, this);
+            if (local->parent)
+                afr_inode_need_refresh_set(local->parent, this);
+            if (local->parent2)
+                afr_inode_need_refresh_set(local->parent2, this);
+            continue;
         }
 
-        parent->path = gf_strdup (dirname (child_path));
-        if (!parent->path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
+        if (local->op_ret == -1) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+            if (local->xdata_rsp) {
+                dict_unref(local->xdata_rsp);
+                local->xdata_rsp = NULL;
+            }
+
+            if (local->replies[i].xdata)
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            continue;
         }
 
-        parent->inode = inode_ref (child->parent);
-	gf_uuid_copy (parent->gfid, child->pargfid);
-
-        ret = 0;
-out:
-        GF_FREE (child_path);
-
-        return ret;
-}
-
+        if (i == inode_read_subvol) {
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
 
-static void
-__afr_dir_write_finalize (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int inode_read_subvol = -1;
-	int parent_read_subvol = -1;
-	int parent2_read_subvol = -1;
-	int i = 0;
-        afr_read_subvol_args_t args = {0,};
-
-	local = frame->local;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-	        if (!local->replies[i].valid)
-	                continue;
-	        if (local->replies[i].op_ret == -1)
-	                continue;
-                gf_uuid_copy (args.gfid, local->replies[i].poststat.ia_gfid);
-                args.ia_type = local->replies[i].poststat.ia_type;
-                break;
+        if (i == parent_read_subvol) {
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
         }
 
-	if (local->inode) {
-		afr_replies_interpret (frame, this, local->inode, NULL);
-		inode_read_subvol = afr_data_subvol_get (local->inode, this,
-                                                       NULL, NULL, NULL, &args);
-	}
-
-	if (local->parent)
-		parent_read_subvol = afr_data_subvol_get (local->parent, this,
-                                             NULL, local->readable, NULL, NULL);
-
-	if (local->parent2)
-		parent2_read_subvol = afr_data_subvol_get (local->parent2, this,
-                                            NULL, local->readable2, NULL, NULL);
-
-	local->op_ret = -1;
-	local->op_errno = afr_final_errno (local, priv);
-        afr_pick_error_xdata (local, priv, local->parent, local->readable,
-                              local->parent2, local->readable2);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret < 0) {
-			if (local->inode)
-				afr_inode_read_subvol_reset (local->inode,
-							     this);
-			if (local->parent)
-				afr_inode_read_subvol_reset (local->parent,
-							     this);
-			if (local->parent2)
-				afr_inode_read_subvol_reset (local->parent2,
-							     this);
-			continue;
-		}
-
-		if (local->op_ret == -1) {
-			local->op_ret = local->replies[i].op_ret;
-			local->op_errno = local->replies[i].op_errno;
-
-			local->cont.dir_fop.buf =
-				local->replies[i].poststat;
-			local->cont.dir_fop.preparent =
-				local->replies[i].preparent;
-			local->cont.dir_fop.postparent =
-				local->replies[i].postparent;
-			local->cont.dir_fop.prenewparent =
-				local->replies[i].preparent2;
-			local->cont.dir_fop.postnewparent =
-				local->replies[i].postparent2;
-                        if (local->xdata_rsp) {
-                                dict_unref (local->xdata_rsp);
-                                local->xdata_rsp = NULL;
-                        }
-
-			if (local->replies[i].xdata)
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			continue;
-		}
-
-		if (i == inode_read_subvol) {
-			local->cont.dir_fop.buf =
-				local->replies[i].poststat;
-			if (local->replies[i].xdata) {
-				if (local->xdata_rsp)
-					dict_unref (local->xdata_rsp);
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			}
-		}
-
-		if (i == parent_read_subvol) {
-			local->cont.dir_fop.preparent =
-				local->replies[i].preparent;
-			local->cont.dir_fop.postparent =
-				local->replies[i].postparent;
-		}
-
-		if (i == parent2_read_subvol) {
-			local->cont.dir_fop.prenewparent =
-				local->replies[i].preparent2;
-			local->cont.dir_fop.postnewparent =
-				local->replies[i].postparent2;
-		}
-	}
-
-        afr_txn_arbitrate_fop_cbk (frame, this);
+        if (i == parent2_read_subvol) {
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+        }
+    }
 }
 
-
 static void
-__afr_dir_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
-		      int op_ret, int op_errno, struct iatt *poststat,
-		      struct iatt *preparent, struct iatt *postparent,
-		      struct iatt *preparent2, struct iatt *postparent2,
-		      dict_t *xdata)
+__afr_dir_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int op_ret, int op_errno, struct iatt *poststat,
+                     struct iatt *preparent, struct iatt *postparent,
+                     struct iatt *preparent2, struct iatt *postparent2,
+                     dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-        if (xdata)
-                local->replies[child_index].xdata = dict_ref (xdata);
-
-
-	if (op_ret >= 0) {
-		if (poststat)
-			local->replies[child_index].poststat = *poststat;
-		if (preparent)
-			local->replies[child_index].preparent = *preparent;
-		if (postparent)
-			local->replies[child_index].postparent = *postparent;
-		if (preparent2)
-			local->replies[child_index].preparent2 = *preparent2;
-		if (postparent2)
-			local->replies[child_index].postparent2 = *postparent2;
-		if (fd_ctx)
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-	} else {
-		if (op_errno != ENOTEMPTY)
-			afr_transaction_fop_failed (frame, this, child_index);
-		if (fd_ctx)
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-	}
-
-        return;
+    afr_local_t *local = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (poststat)
+            local->replies[child_index].poststat = *poststat;
+        if (preparent)
+            local->replies[child_index].preparent = *preparent;
+        if (postparent)
+            local->replies[child_index].postparent = *postparent;
+        if (preparent2)
+            local->replies[child_index].preparent2 = *preparent2;
+        if (postparent2)
+            local->replies[child_index].postparent2 = *postparent2;
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+    } else {
+        if (op_errno != ENOTEMPTY)
+            afr_transaction_fop_failed(frame, this, child_index);
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+    }
+
+    return;
 }
 
-
 static int
-__afr_dir_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int op_ret, int op_errno, struct iatt *buf,
-		     struct iatt *preparent, struct iatt *postparent,
-		     struct iatt *preparent2, struct iatt *postparent2,
-                     dict_t *xdata)
+__afr_dir_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    struct iatt *preparent2, struct iatt *postparent2,
+                    dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-        afr_private_t *priv = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		__afr_dir_write_fill (frame, this, child_index, op_ret,
-				      op_errno, buf, preparent, postparent,
-				      preparent2, postparent2, xdata);
-	}
-	UNLOCK (&frame->lock);
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		__afr_dir_write_finalize (frame, this);
-
-		if (afr_txn_nothing_failed (frame, this)) {
-                        /*if it did pre-op, it will do post-op changing ctime*/
-                        if (priv->consistent_metadata &&
-                            afr_needs_changelog_update (local))
-                                afr_zero_fill_stat (local);
-                        local->transaction.unwind (frame, this);
-                }
-
-		afr_mark_entry_pending_changelog (frame, this);
-
-                local->transaction.resume (frame, this);
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        __afr_dir_write_fill(frame, this, child_index, op_ret, op_errno, buf,
+                             preparent, postparent, preparent2, postparent2,
+                             xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        __afr_dir_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
 
-        return 0;
-}
+        afr_mark_entry_pending_changelog(frame, this);
 
+        afr_transaction_resume(frame, this);
+    }
+
+    return 0;
+}
 
 int
-afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
-				  xlator_t *this, int op_ret, int op_errno,
-                                  dict_t *xattr, dict_t *xdata)
+afr_mark_new_entry_changelog_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int op_ret, int op_errno,
+                                 dict_t *xattr, dict_t *xdata)
 {
-        int call_count = 0;
+    int call_count = 0;
 
-        call_count = afr_frame_return (frame);
+    call_count = afr_frame_return(frame);
 
-        if (call_count == 0)
-                AFR_STACK_DESTROY (frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
 void
-afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t  *new_frame  = NULL;
-        afr_local_t   *local      = NULL;
-        afr_local_t   *new_local  = NULL;
-        afr_private_t *priv       = NULL;
-        dict_t        *xattr      = NULL;
-        int32_t       **changelog = NULL;
-        int           i           = 0;
-        int           op_errno    = ENOMEM;
-	unsigned char *pending    = NULL;
-	int           call_count   = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-	new_local = AFR_FRAME_INIT (new_frame, op_errno);
-	if (!new_local)
-		goto out;
-
-        xattr = dict_new ();
-	if (!xattr)
-		goto out;
-
-	pending = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->transaction.pre_op[i] &&
-		    !local->transaction.failed_subvols[i]) {
-			call_count ++;
-			continue;
-		}
-		pending[i] = 1;
-	}
-
-        changelog = afr_mark_pending_changelog (priv, pending, xattr,
-                                            local->cont.dir_fop.buf.ia_type);
-        if (!changelog)
-                goto out;
-
-        new_local->pending = changelog;
-        gf_uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
-        new_local->loc.inode = inode_ref (local->inode);
-
-        new_local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-		if (pending[i])
-                        continue;
-
-                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
-                                   xattr, NULL);
-		if (!--call_count)
-			break;
+    call_frame_t *new_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *new_local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int32_t **changelog = NULL;
+    int i = 0;
+    int op_errno = ENOMEM;
+    unsigned char *pending = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_local = AFR_FRAME_INIT(new_frame, op_errno);
+    if (!new_local)
+        goto out;
+
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            !local->transaction.failed_subvols[i]) {
+            call_count++;
+            continue;
         }
+        pending[i] = 1;
+    }
+
+    changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                           local->cont.dir_fop.buf.ia_type);
+    if (!changelog)
+        goto out;
+
+    new_local->pending = changelog;
+    gf_uuid_copy(new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+    new_local->loc.inode = inode_ref(local->inode);
+
+    new_local->call_count = call_count;
 
-        new_frame = NULL;
+    for (i = 0; i < priv->child_count; i++) {
+        if (pending[i])
+            continue;
+
+        STACK_WIND_COOKIE(new_frame, afr_mark_new_entry_changelog_cbk,
+                          (void *)(long)i, priv->children[i],
+                          priv->children[i]->fops->xattrop, &new_local->loc,
+                          GF_XATTROP_ADD_ARRAY, xattr, NULL);
+        if (!--call_count)
+            break;
+    }
+
+    new_frame = NULL;
 out:
-        if (new_frame)
-                AFR_STACK_DESTROY (new_frame);
-	if (xattr)
-		dict_unref (xattr);
-        return;
+    if (new_frame)
+        AFR_STACK_DESTROY(new_frame);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
-
 void
-afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-	int pre_op_count = 0;
-	int failed_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_count = 0;
+    int failed_count = 0;
+    unsigned char *success_replies = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (local->op_ret < 0)
-		return;
+    if (local->op_ret < 0)
+        return;
 
-	if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD &&
-            local->op != GF_FOP_MKDIR)
-		return;
+    if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD &&
+        local->op != GF_FOP_MKDIR)
+        return;
 
-	pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
-	failed_count = AFR_COUNT (local->transaction.failed_subvols,
-				  priv->child_count);
+    pre_op_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                             priv->child_count);
 
-	if (pre_op_count == priv->child_count && !failed_count)
-		return;
+    /* FOP succeeded on all bricks. */
+    if (pre_op_count == priv->child_count && !failed_count)
+        return;
 
-        afr_mark_new_entry_changelog (frame, this);
+    /* FOP did not suceed on quorum no. of bricks. */
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+    if (!afr_has_quorum(success_replies, this, NULL))
+        return;
 
+    if (priv->thin_arbiter_count) {
+        /*Mark new entry using ta file*/
+        local->is_new_entry = _gf_true;
         return;
-}
+    }
 
+    afr_mark_new_entry_changelog(frame, this);
+
+    return;
+}
 
 /* {{{ create */
 
 int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+afr_create_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
+    local = frame->local;
 
-        if (!main_frame)
-		return 0;
+    main_frame = afr_transaction_detach_fop_frame(frame);
 
-	AFR_STACK_UNWIND (create, main_frame, local->op_ret, local->op_errno,
-			  local->cont.create.fd, local->inode,
-			  &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(create, main_frame, local->op_ret, local->op_errno,
+                     local->cont.create.fd, local->inode,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
+afr_create_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_create_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_create_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_create_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->create,
-			   &local->loc, local->cont.create.flags,
-			   local->cont.create.mode, local->umask,
-			   local->cont.create.fd, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_create_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->create, &local->loc,
+                      local->cont.create.flags, local->cont.create.mode,
+                      local->umask, local->cont.create.fd, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-	    mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-
-	local->fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!local->fd_ctx)
-		goto out;
-
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->op                = GF_FOP_CREATE;
-        local->cont.create.flags = flags;
-        local->fd_ctx->flags     = flags;
-        local->cont.create.mode  = mode;
-        local->cont.create.fd    = fd_ref (fd);
-        local->umask  = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->transaction.wind   = afr_create_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_create_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-            op_errno = -ret;
-            goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+
+    local->fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!local->fd_ctx)
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_CREATE;
+    local->cont.create.flags = flags;
+    local->fd_ctx->flags = flags;
+    local->cont.create.mode = mode;
+    local->cont.create.fd = fd_ref(fd);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_create_wind;
+    local->transaction.unwind = afr_create_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -541,524 +502,436 @@ out:
 /* {{{ mknod */
 
 int
-afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
+afr_mknod_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (mknod, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mknod, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_mknod_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mknod_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_mknod_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->mknod,
-			   &local->loc, local->cont.mknod.mode,
-			   local->cont.mknod.dev, local->umask,
-			   local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_mknod_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mknod, &local->loc,
+                      local->cont.mknod.mode, local->cont.mknod.dev,
+                      local->umask, local->xdata_req);
+    return 0;
 }
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t dev, mode_t umask, dict_t *xdata)
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->op               = GF_FOP_MKNOD;
-        local->cont.mknod.mode  = mode;
-        local->cont.mknod.dev   = dev;
-        local->umask = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->transaction.wind   = afr_mknod_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_mknod_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_MKNOD;
+    local->cont.mknod.mode = mode;
+    local->cont.mknod.dev = dev;
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_mknod_wind;
+    local->transaction.unwind = afr_mknod_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ mkdir */
 
-
 int
-afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (mkdir, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mkdir, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_mkdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_mkdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->mkdir, &local->loc,
-			   local->cont.mkdir.mode, local->umask,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_mkdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mkdir, &local->loc,
+                      local->cont.mkdir.mode, local->umask, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-	   mode_t umask, dict_t *xdata)
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.mkdir.mode  = mode;
-        local->umask = umask;
-
-        if (!xdata || !dict_get (xdata, "gfid-req")) {
-                op_errno = EPERM;
-                gf_msg_callingfn (this->name, GF_LOG_WARNING, op_errno,
-                                  AFR_MSG_GFID_NULL, "mkdir: %s is received "
-                                  "without gfid-req %p", loc->path, xdata);
-	        goto out;
-        }
-
-        local->xdata_req = dict_copy_with_ref (xdata, NULL);
-        if (!local->xdata_req) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        local->op = GF_FOP_MKDIR;
-        local->transaction.wind   = afr_mkdir_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_mkdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.mkdir.mode = mode;
+    local->umask = umask;
+
+    if (!xdata || !dict_get_sizen(xdata, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         AFR_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, xdata);
+        goto out;
+    }
+
+    local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->op = GF_FOP_MKDIR;
+    local->transaction.wind = afr_mkdir_wind;
+    local->transaction.unwind = afr_mkdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ link */
 
-
 int
-afr_link_unwind (call_frame_t *frame, xlator_t *this)
+afr_link_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (link, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(link, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+afr_link_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_link_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->link,
-			   &local->loc, &local->newloc, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_link_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->link, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-	  dict_t *xdata)
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-
-	local->inode = inode_ref (oldloc->inode);
-	local->parent = inode_ref (newloc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_LINK;
-
-        local->transaction.wind   = afr_link_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_link_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (newloc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
 
-	return 0;
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_LINK;
+
+    local->transaction.wind = afr_link_wind;
+    local->transaction.unwind = afr_link_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ symlink */
 
-
 int
-afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_symlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (symlink, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(symlink, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, inode_t *inode,
-                      struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+afr_symlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_symlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->symlink,
-			   local->cont.symlink.linkpath, &local->loc,
-			   local->umask, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_symlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->symlink,
+                      local->cont.symlink.linkpath, &local->loc, local->umask,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-	     loc_t *loc, mode_t umask, dict_t *xdata)
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.symlink.linkpath = gf_strdup (linkpath);
-        local->umask = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_SYMLINK;
-        local->transaction.wind   = afr_symlink_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_symlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.symlink.linkpath = gf_strdup(linkpath);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_SYMLINK;
+    local->transaction.wind = afr_symlink_wind;
+    local->transaction.unwind = afr_symlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1066,161 +939,118 @@ out:
 /* {{{ rename */
 
 int
-afr_rename_unwind (call_frame_t *frame, xlator_t *this)
+afr_rename_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (rename, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent,
-			  &local->cont.dir_fop.prenewparent,
-			  &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(rename, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent,
+                     &local->cont.dir_fop.prenewparent,
+                     &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent,
-                     dict_t *xdata)
+afr_rename_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preoldparent, postoldparent, prenewparent,
-				    postnewparent, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preoldparent, postoldparent, prenewparent,
+                               postnewparent, xdata);
 }
 
-
 int
-afr_rename_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_rename_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->rename,
-			   &local->loc, &local->newloc, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rename_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rename, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-	    dict_t *xdata)
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-        int                     nlockee                 = 0;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-
-	local->inode = inode_ref (oldloc->inode);
-	local->parent = inode_ref (oldloc->parent);
-	local->parent2 = inode_ref (newloc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_RENAME;
-        local->transaction.wind   = afr_rename_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_rename_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-        ret = afr_build_parent_loc (&local->transaction.new_parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (oldloc->path);
-        local->transaction.new_basename = AFR_BASENAME (newloc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = nlockee = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.new_parent_loc,
-                                     local->transaction.new_basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
-                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                             &local->newloc,
-                                             NULL,
-                                             priv->child_count);
-                if (ret)
-                        goto out;
-
-                nlockee++;
-        }
-        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
-               afr_entry_lockee_cmp);
-        int_lock->lockee_count = nlockee;
-
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
+
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(oldloc->parent);
+    local->parent2 = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RENAME;
+    local->transaction.wind = afr_rename_wind;
+    local->transaction.unwind = afr_rename_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, oldloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+    ret = afr_build_parent_loc(&local->transaction.new_parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(oldloc->path);
+    local->transaction.new_basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this,
+                          AFR_ENTRY_RENAME_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1228,263 +1058,205 @@ out:
 /* {{{ unlink */
 
 int
-afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_unlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (unlink, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(unlink, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+afr_unlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_unlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_unlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->unlink,
-			   &local->loc, local->xflag, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_unlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->unlink, &local->loc,
+                      local->xflag, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-	    dict_t *xdata)
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_UNLINK;
-        local->transaction.wind   = afr_unlink_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_unlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->xflag = xflag;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_UNLINK;
+    local->transaction.wind = afr_unlink_wind;
+    local->transaction.unwind = afr_unlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ rmdir */
 
-
-
 int
-afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_rmdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_rmdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->rmdir,
-			   &local->loc, local->cont.rmdir.flags, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rmdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rmdir, &local->loc,
+                      local->cont.rmdir.flags, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-	   dict_t *xdata)
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-        int                     nlockee                 = 0;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.rmdir.flags = flags;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_RMDIR;
-        local->transaction.wind   = afr_rmdir_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_rmdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = nlockee = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->loc,
-                                     NULL,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
-               afr_entry_lockee_cmp);
-        int_lock->lockee_count = nlockee;
-
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.rmdir.flags = flags;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RMDIR;
+    local->transaction.wind = afr_rmdir_wind;
+    local->transaction.unwind = afr_rmdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 02f0a3682d9..1d88c3b9b26 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -12,36 +12,35 @@
 #define __DIR_WRITE_H__
 
 int32_t
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata);
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
 
 int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int xflag, dict_t *xdata);
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
 int32_t
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags, dict_t *xdata);
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
 int32_t
-afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
 
 int32_t
-afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
 
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 1690cb684dd..c5521704de2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
 #include <fnmatch.h>
@@ -16,21 +15,17 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "byte-order.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "quota-common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/quota-common-utils.h>
 
 #include "afr-transaction.h"
 #include "afr-messages.h"
@@ -45,146 +40,146 @@
  * */
 
 int
-afr_handle_quota_size (call_frame_t *frame, xlator_t *this)
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this)
 {
-        unsigned char *readable   = NULL;
-        afr_local_t *local        = NULL;
-        afr_private_t *priv       = NULL;
-        struct afr_reply *replies = NULL;
-        int i                     = 0;
-        int ret                   = 0;
-        quota_meta_t size         = {0, };
-        quota_meta_t max_size     = {0, };
-        int readable_cnt          = 0;
-        int read_subvol           = -1;
-
-        local = frame->local;
-        priv = this->private;
-        replies = local->replies;
-
-        readable = alloca0 (priv->child_count);
-
-        afr_inode_read_subvol_get (local->inode, this, readable, 0, 0);
-
-        readable_cnt = AFR_COUNT (readable, priv->child_count);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (readable_cnt && !readable[i])
-                        continue;
-                if (!replies[i].xdata)
-                        continue;
-                ret = quota_dict_get_meta (replies[i].xdata, QUOTA_SIZE_KEY,
-                                           &size);
-                if (ret == -1)
-                        continue;
-                if (read_subvol == -1)
-                        read_subvol = i;
-                if (size.size > max_size.size ||
-                    (size.file_count + size.dir_count) >
-                    (max_size.file_count + max_size.dir_count))
-                        read_subvol = i;
-
-                if (size.size > max_size.size)
-                        max_size.size = size.size;
-                if (size.file_count > max_size.file_count)
-                        max_size.file_count = size.file_count;
-                if (size.dir_count > max_size.dir_count)
-                        max_size.dir_count = size.dir_count;
-        }
-
-        if (max_size.size == 0 && max_size.file_count == 0 &&
-            max_size.dir_count == 0)
-                return read_subvol;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (readable_cnt && !readable[i])
-                        continue;
-                if (!replies[i].xdata)
-                        continue;
-                quota_dict_set_meta (replies[i].xdata, QUOTA_SIZE_KEY,
-                                     &max_size, IA_IFDIR);
-        }
-
+    unsigned char *readable = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0;
+    int ret = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
+    int readable_cnt = 0;
+    int read_subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+
+    readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(local->inode, this, readable, 0, 0);
+
+    readable_cnt = AFR_COUNT(readable, priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        ret = quota_dict_get_meta(replies[i].xdata, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &size);
+        if (ret == -1)
+            continue;
+        if (read_subvol == -1)
+            read_subvol = i;
+        if (size.size > max_size.size ||
+            (size.file_count + size.dir_count) >
+                (max_size.file_count + max_size.dir_count))
+            read_subvol = i;
+
+        if (size.size > max_size.size)
+            max_size.size = size.size;
+        if (size.file_count > max_size.file_count)
+            max_size.file_count = size.file_count;
+        if (size.dir_count > max_size.dir_count)
+            max_size.dir_count = size.dir_count;
+    }
+
+    if (max_size.size == 0 && max_size.file_count == 0 &&
+        max_size.dir_count == 0)
         return read_subvol;
-}
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        quota_dict_set_meta(replies[i].xdata, QUOTA_SIZE_KEY, &max_size,
+                            IA_IFDIR);
+    }
+
+    return read_subvol;
+}
 
 /* {{{ access */
 
 int
-afr_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, dict_t *xdata)
+afr_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+    AFR_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_access_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_access_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (access, frame, local->op_ret,
-				  local->op_errno, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_access_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->access,
-                           &local->loc, local->cont.access.mask,
-			   local->xdata_req);
-	return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(access, frame, local->op_ret, local->op_errno, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_access_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->access, &local->loc,
+                      local->cont.access.mask, local->xdata_req);
+    return 0;
 }
 
 int
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	    int mask, dict_t *xdata)
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
+           dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_ACCESS;
-	loc_copy (&local->loc, loc);
-	local->cont.access.mask = mask;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_ACCESS;
+    loc_copy(&local->loc, loc);
+    local->cont.access.mask = mask;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_access_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_access_wind,
+                 AFR_METADATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -192,152 +187,140 @@ out:
 /* {{{ stat */
 
 int
-afr_stat_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              struct iatt *buf, dict_t *xdata)
+afr_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+    AFR_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_stat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_stat_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-				  0, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->stat,
-                           &local->loc, local->xdata_req);
-	return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_stat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->stat, &local->loc, local->xdata_req);
+    return 0;
 }
 
 int
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_STAT;
-	loc_copy (&local->loc, loc);
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_STAT;
+    loc_copy(&local->loc, loc);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_stat_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_stat_wind, AFR_DATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ fstat */
 
 int
-afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf,
-               dict_t *xdata)
+afr_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+    AFR_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fstat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fstat_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (fstat, frame, local->op_ret, local->op_errno,
-				  0, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->fstat,
-                           local->fd, local->xdata_req);
-	return 0;
-}
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fstat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_fstat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->fstat, local->fd, local->xdata_req);
+    return 0;
+}
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, dict_t *xdata)
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_FSTAT;
-	local->fd = fd_ref (fd);
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_FSTAT;
+    local->fd = fd_ref(fd);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_fstat_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_fstat_wind, AFR_DATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -345,1446 +328,1493 @@ out:
 /* {{{ readlink */
 
 int
-afr_readlink_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  const char *buf, struct iatt *sbuf, dict_t *xdata)
+afr_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *buf,
+                 struct iatt *sbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno,
-			  buf, sbuf, xdata);
-	return 0;
+    AFR_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
 }
 
 int
-afr_readlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readlink, frame, local->op_ret,
-				  local->op_errno, 0, 0, 0);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_readlink_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->readlink,
-			   &local->loc, local->cont.readlink.size,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readlink, frame, local->op_ret, local->op_errno, 0, 0,
+                         0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_readlink_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->readlink, &local->loc,
+                      local->cont.readlink.size, local->xdata_req);
+    return 0;
+}
 
 int
-afr_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size, dict_t *xdata)
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        afr_local_t   * local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_READLINK;
-        loc_copy (&local->loc, loc);
-        local->cont.readlink.size = size;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_READLINK;
+    loc_copy(&local->loc, loc);
+    local->cont.readlink.size = size;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_readlink_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_readlink_wind,
+                 AFR_DATA_TRANSACTION);
 
-        return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
+    AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
 
-	return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ getxattr */
 
 struct _xattr_key {
-        char *key;
-        struct list_head list;
+    char *key;
+    struct list_head list;
 };
 
-
 int
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
+__gather_xattr_keys(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
-
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
+    struct list_head *list = data;
+    struct _xattr_key *xkey = NULL;
 
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return -1;
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        xkey = GF_MALLOC(sizeof(*xkey), gf_afr_mt_xattr_key);
+        if (!xkey)
+            return -1;
 
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
+        xkey->key = key;
+        INIT_LIST_HEAD(&xkey->list);
 
-                list_add_tail (&xkey->list, list);
-        }
-        return 0;
+        list_add_tail(&xkey->list, list);
+    }
+    return 0;
 }
 
-
 void
-afr_filter_xattrs (dict_t *dict)
+afr_filter_xattrs(dict_t *dict)
 {
-        struct list_head   keys = {0,};
-        struct _xattr_key *key  = NULL;
-        struct _xattr_key *tmp  = NULL;
+    struct list_head keys = {
+        0,
+    };
+    struct _xattr_key *key = NULL;
+    struct _xattr_key *tmp = NULL;
 
-        INIT_LIST_HEAD (&keys);
+    INIT_LIST_HEAD(&keys);
 
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
+    dict_foreach(dict, __gather_xattr_keys, (void *)&keys);
 
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
+    list_for_each_entry_safe(key, tmp, &keys, list)
+    {
+        dict_del(dict, key->key);
 
-                list_del_init (&key->list);
+        list_del_init(&key->list);
 
-                GF_FREE (key);
-        }
+        GF_FREE(key);
+    }
 }
 
-static
-gf_boolean_t
-afr_getxattr_ignorable_errnos (int32_t op_errno)
+static gf_boolean_t
+afr_getxattr_ignorable_errnos(int32_t op_errno)
 {
-        if (op_errno == ENODATA || op_errno == ENOTSUP || op_errno == ERANGE ||
-            op_errno == ENAMETOOLONG)
-                return _gf_true;
+    if (op_errno == ENODATA || op_errno == ENOTSUP || op_errno == ERANGE ||
+        op_errno == ENAMETOOLONG)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 int
-afr_getxattr_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *dict, dict_t *xdata)
+afr_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0 && !afr_getxattr_ignorable_errnos(op_errno)) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0 && !afr_getxattr_ignorable_errnos(op_errno)) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (dict)
-		afr_filter_xattrs (dict);
+    if (dict)
+        afr_filter_xattrs(dict);
 
-	AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_getxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_getxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-				  local->op_errno, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_getxattr_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->getxattr,
-			   &local->loc, local->cont.getxattr.name,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
 
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_getxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->getxattr, &local->loc,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
 int32_t
-afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
-                     dict_t *dict, dict_t *xdata)
+afr_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *dict, dict_t *xdata)
+afr_fgetxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
 {
-        afr_local_t     *local                  = NULL;
-        afr_private_t   *priv                   = NULL;
-        xlator_t        **children              = NULL;
-        dict_t          *xattr                  = NULL;
-        char            *tmp_report             = NULL;
-        char            lk_summary[1024]        = {0,};
-        int             serz_len                = 0;
-        int32_t         callcnt                 = 0;
-        long int        cky                     = 0;
-        int             ret                     = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1)
-                        local->replies[cky].op_errno = op_errno;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (local->dict) {
-                        ret = dict_get_str (dict, local->cont.getxattr.name,
-                                            &tmp_report);
-                        if (ret)
-                                goto unlock;
-                        ret = dict_set_dynstr (local->dict,
-                                               children[cky]->name,
-                                               gf_strdup (tmp_report));
-                        if (ret)
-                                goto unlock;
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                xattr = dict_new ();
-                if (!xattr) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       lk_summary,
-                                                       &serz_len, '\n');
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                if (serz_len == -1)
-                        snprintf (lk_summary, sizeof (lk_summary),
-                                  "No locks cleared.");
-                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
-                                       gf_strdup (lk_summary));
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                ENOMEM, AFR_MSG_DICT_SET_FAILED,
-                                "Error setting dictionary");
-                        goto unwind;
-                }
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
 
-        unwind:
-                // Updating child_errno with more recent 'events'
-                op_errno = afr_final_errno (local, priv);
+        op_errno = afr_final_errno(local, priv);
 
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
-                                  xdata);
-                if (xattr)
-                        dict_unref (xattr);
-        }
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                        dict_t *dict, dict_t *xdata)
+afr_getxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        afr_local_t     *local                  = NULL;
-        afr_private_t   *priv                   = NULL;
-        xlator_t        **children              = NULL;
-        dict_t          *xattr                  = NULL;
-        char            *tmp_report             = NULL;
-        char            lk_summary[1024]        = {0,};
-        int             serz_len                = 0;
-        int32_t         callcnt                 = 0;
-        long int        cky                     = 0;
-        int             ret                     = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1)
-                        local->replies[cky].op_errno = op_errno;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (local->dict) {
-                        ret = dict_get_str (dict, local->cont.getxattr.name,
-                                            &tmp_report);
-                        if (ret)
-                                goto unlock;
-                        ret = dict_set_dynstr (local->dict,
-                                               children[cky]->name,
-                                               gf_strdup (tmp_report));
-                        if (ret)
-                                goto unlock;
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                xattr = dict_new ();
-                if (!xattr) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       lk_summary,
-                                                       &serz_len, '\n');
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                if (serz_len == -1)
-                        snprintf (lk_summary, sizeof (lk_summary),
-                                  "No locks cleared.");
-                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
-                                       gf_strdup (lk_summary));
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                ENOMEM, AFR_MSG_DICT_SET_FAILED,
-                                "Error setting dictionary");
-                        goto unwind;
-                }
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
 
-        unwind:
-                // Updating child_errno with more recent 'events'
-                op_errno = afr_final_errno (local, priv);
+        op_errno = afr_final_errno(local, priv);
 
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-                if (xattr)
-                        dict_unref (xattr);
-        }
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        return ret;
+    return ret;
 }
 
 /**
  * node-uuid cbk uses next child querying mechanism
  */
 int32_t
-afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_getxattr_node_uuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_private_t  *priv            = NULL;
-        afr_local_t    *local           = NULL;
-        xlator_t      **children        = NULL;
-        int             unwind          = 1;
-        int             curr_call_child = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int unwind = 1;
+    int curr_call_child = 0;
 
-        priv = this->private;
-        children = priv->children;
+    priv = this->private;
+    children = priv->children;
 
-        local = frame->local;
+    local = frame->local;
+
+    if (op_ret == -1) { /** query the _next_ child */
+
+        /**
+         * _current_ becomes _next_
+         * If done with all children and yet no success; give up !
+         */
+        curr_call_child = (int)((long)cookie);
+        if (++curr_call_child == priv->child_count)
+            goto unwind;
+
+        gf_msg_debug(this->name, op_errno,
+                     "op_ret (-1): Re-querying afr-child (%d/%d)",
+                     curr_call_child, priv->child_count);
+
+        unwind = 0;
+        STACK_WIND_COOKIE(
+            frame, afr_getxattr_node_uuid_cbk, (void *)(long)curr_call_child,
+            children[curr_call_child],
+            children[curr_call_child]->fops->getxattr, &local->loc,
+            local->cont.getxattr.name, local->xdata_req);
+    }
+
+unwind:
+    if (unwind)
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
+}
+
+/**
+ * list-node-uuids cbk returns the list of node_uuids for the subvolume.
+ */
+int32_t
+afr_getxattr_list_node_uuids_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr_serz = NULL;
+    long cky = 0;
+    int32_t tlen = 0;
+
+    local = frame->local;
+    priv = this->private;
+    cky = (long)cookie;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        local->replies[cky].valid = 1;
+        local->replies[cky].op_ret = op_ret;
+        local->replies[cky].op_errno = op_errno;
+
+        if (op_ret < 0)
+            goto unlock;
+
+        local->op_ret = 0;
+
+        if (!local->xdata_rsp && xdata)
+            local->xdata_rsp = dict_ref(xdata);
+        local->replies[cky].xattr = dict_ref(dict);
+    }
+
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (op_ret == -1) { /** query the _next_ child */
-
-                /**
-                 * _current_ becomes _next_
-                 * If done with all childs and yet no success; give up !
-                 */
-                curr_call_child = (int) ((long)cookie);
-                if (++curr_call_child == priv->child_count)
-                        goto unwind;
-
-                gf_msg_debug (this->name, op_errno,
-                              "op_ret (-1): Re-querying afr-child (%d/%d)",
-                              curr_call_child, priv->child_count);
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) curr_call_child,
-                                   children[curr_call_child],
-                                   children[curr_call_child]->fops->getxattr,
-                                   &local->loc,
-                                   local->cont.getxattr.name,
-                                   NULL);
+    if (!callcnt) {
+        if (local->op_ret != 0) {
+            /* All bricks gave an error. */
+            local->op_errno = afr_final_errno(local, priv);
+            goto unwind;
         }
 
- unwind:
-        if (unwind)
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  NULL);
+        /*Since we store the UUID0_STR as node uuid for down bricks and
+         *for non zero op_ret, assigning length to  priv->child_count
+         *number of uuids*/
+        local->cont.getxattr.xattr_len = (SLEN(UUID0_STR) + 2) *
+                                         priv->child_count;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (!local->dict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        return 0;
+        xattr_serz = GF_CALLOC(local->cont.getxattr.xattr_len, sizeof(char),
+                               gf_common_mt_char);
+
+        if (!xattr_serz) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
+
+        ret = afr_serialize_xattrs_with_delimiter(frame, this, xattr_serz,
+                                                  UUID0_STR, &tlen, ' ');
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, GF_XATTR_LIST_NODE_UUIDS_KEY,
+                                    xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set node_uuid key in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        } else {
+            local->op_ret = local->cont.getxattr.xattr_len - 1;
+            local->op_errno = 0;
+        }
+
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, local->xdata_rsp);
+    }
+
+    return ret;
 }
 
 int32_t
-afr_getxattr_quota_size_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *dict, dict_t *xdata)
+afr_getxattr_quota_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *dict,
+                            dict_t *xdata)
 {
-        int         idx         = (long) cookie;
-        int         call_count  = 0;
-        afr_local_t *local      = frame->local;
-        int         read_subvol = -1;
-
-        local->replies[idx].valid = 1;
-        local->replies[idx].op_ret = op_ret;
-        local->replies[idx].op_errno = op_errno;
-        if (dict)
-                local->replies[idx].xdata = dict_ref (dict);
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                local->inode = inode_ref (local->loc.inode);
-                read_subvol = afr_handle_quota_size (frame, this);
-                if (read_subvol != -1) {
-                        op_ret = local->replies[read_subvol].op_ret;
-                        op_errno = local->replies[read_subvol].op_errno;
-                        dict = local->replies[read_subvol].xdata;
-                }
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
+    int idx = (long)cookie;
+    int call_count = 0;
+    afr_local_t *local = frame->local;
+    int read_subvol = -1;
+
+    local->replies[idx].valid = 1;
+    local->replies[idx].op_ret = op_ret;
+    local->replies[idx].op_errno = op_errno;
+    if (dict)
+        local->replies[idx].xdata = dict_ref(dict);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        local->inode = inode_ref(local->loc.inode);
+        read_subvol = afr_handle_quota_size(frame, this);
+        if (read_subvol != -1) {
+            op_ret = local->replies[read_subvol].op_ret;
+            op_errno = local->replies[read_subvol].op_errno;
+            dict = local->replies[read_subvol].xdata;
         }
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict, dict_t *xdata)
+afr_getxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
 {
-        int          call_cnt     = 0, len = 0;
-        char        *lockinfo_buf = NULL;
-        dict_t      *lockinfo     = NULL, *newdict = NULL;
-        afr_local_t *local        = NULL;
-
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-                call_cnt = --local->call_count;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-                if ((op_ret < 0) || (!dict && !xdata)) {
-                        goto unlock;
-                }
+        call_cnt = --local->call_count;
 
-                if (xdata) {
-                        if (!local->xdata_rsp) {
-                                local->xdata_rsp = dict_new ();
-                                if (!local->xdata_rsp) {
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-                        }
-                }
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
 
-                if (!dict) {
-                        goto unlock;
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
 
-                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                               (void **)&lockinfo_buf, &len);
+        if (!dict) {
+            goto unlock;
+        }
 
-                if (!lockinfo_buf) {
-                        goto unlock;
-                }
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-                if (!local->dict) {
-                        local->dict = dict_new ();
-                        if (!local->dict) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (lockinfo_buf != NULL) {
-                lockinfo = dict_new ();
-                if (lockinfo == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                } else {
-                        op_ret = dict_unserialize (lockinfo_buf, len,
-                                                   &lockinfo);
-
-                        if (lockinfo && local->dict) {
-                                dict_copy (lockinfo, local->dict);
-                        }
-                }
+        if (!lockinfo_buf) {
+            goto unlock;
         }
 
-        if (xdata && local->xdata_rsp) {
-                dict_copy (xdata, local->xdata_rsp);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (!call_cnt) {
-                newdict = dict_new ();
-                if (!newdict) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                len = dict_serialized_length (local->dict);
-                if (len <= 0) {
-                        goto unwind;
-                }
-
-                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (!lockinfo_buf) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-                op_ret = dict_serialize (local->dict, lockinfo_buf);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                }
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
-                                          (void *)lockinfo_buf, len);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                        goto unwind;
-                }
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret,
-                                  op_errno, newdict,
-                                  local->xdata_rsp);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
 
-        dict_unref (lockinfo);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_fgetxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        int          call_cnt     = 0, len = 0;
-        char        *lockinfo_buf = NULL;
-        dict_t      *lockinfo     = NULL, *newdict = NULL;
-        afr_local_t *local        = NULL;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-                call_cnt = --local->call_count;
+        call_cnt = --local->call_count;
 
-                if ((op_ret < 0) || (!dict && !xdata)) {
-                        goto unlock;
-                }
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
 
-                if (xdata) {
-                        if (!local->xdata_rsp) {
-                                local->xdata_rsp = dict_new ();
-                                if (!local->xdata_rsp) {
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-                        }
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
 
-                if (!dict) {
-                        goto unlock;
-                }
+        if (!dict) {
+            goto unlock;
+        }
 
-                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                               (void **)&lockinfo_buf, &len);
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-                if (!lockinfo_buf) {
-                        goto unlock;
-                }
-
-                if (!local->dict) {
-                        local->dict = dict_new ();
-                        if (!local->dict) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (lockinfo_buf != NULL) {
-                lockinfo = dict_new ();
-                if (lockinfo == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                } else {
-                        op_ret = dict_unserialize (lockinfo_buf, len,
-                                                   &lockinfo);
-
-                        if (lockinfo && local->dict) {
-                                dict_copy (lockinfo, local->dict);
-                        }
-                }
+        if (!lockinfo_buf) {
+            goto unlock;
         }
 
-        if (xdata && local->xdata_rsp) {
-                dict_copy (xdata, local->xdata_rsp);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (!call_cnt) {
-                newdict = dict_new ();
-                if (!newdict) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                len = dict_serialized_length (local->dict);
-                if (len <= 0) {
-                        goto unwind;
-                }
-
-                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (!lockinfo_buf) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-                op_ret = dict_serialize (local->dict, lockinfo_buf);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                }
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
-                                          (void *)lockinfo_buf, len);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                        goto unwind;
-                }
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
-                                  op_errno, newdict,
-                                  local->xdata_rsp);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
 
-        dict_unref (lockinfo);
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_fgetxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-        int          ret            = 0;
-        char        *xattr          = NULL;
-        char        *xattr_serz     = NULL;
-        char        xattr_cky[1024] = {0,};
-        dict_t      *nxattr         = NULL;
-        long         cky            = 0;
-        int32_t      padding        = 0;
-        int32_t      tlen           = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "possible NULL deref");
-                goto out;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    int keylen = 0;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
 
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret < 0) {
-                        local->op_errno = op_errno;
-                } else {
-                        local->op_ret = op_ret;
-                        if (!local->xdata_rsp && xdata)
-                                local->xdata_rsp = dict_ref (xdata);
-                }
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-                if (!dict || (op_ret < 0))
-                        goto unlock;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-
-                if (local->dict) {
-                        ret = dict_get_str (dict,
-                                            local->cont.getxattr.name,
-                                            &xattr);
-                        if (ret)
-                                goto unlock;
-
-                        xattr = gf_strdup (xattr);
-
-                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
-                                        local->cont.getxattr.name, cky);
-                        ret = dict_set_dynstr (local->dict,
-                                               xattr_cky, xattr);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        -ret, AFR_MSG_DICT_SET_FAILED,
-                                        "Cannot set xattr cookie key");
-                                goto unlock;
-                        }
-
-                        local->cont.getxattr.xattr_len
-                                += strlen (xattr) + 1;
-                }
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->cont.getxattr.xattr_len)
-                        goto unwind;
-
-                nxattr = dict_new ();
-                if (!nxattr)
-                        goto unwind;
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding += strlen (this->name)
-                        + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.xattr_len += (padding + 2);
-
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
-
-                if (!xattr_serz)
-                        goto unwind;
-
-                /* the xlator info */
-                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
-                                this->name);
-
-                /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       xattr_serz
-                                                       + strlen (xattr_serz),
-                                                       &tlen, ' ');
-                if (ret) {
-                        goto unwind;
-                }
 
-                /* closing part */
-                *(xattr_serz + padding + tlen) = ')';
-                *(xattr_serz + padding + tlen + 1) = '\0';
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
-                                       xattr_serz);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Cannot set pathinfo key in dict");
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
-                                  local->op_errno, nxattr, local->xdata_rsp);
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
 
-                if (nxattr)
-                        dict_unref (nxattr);
-        }
+        if (nxattr)
+            dict_unref(nxattr);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict, dict_t *xdata)
+afr_getxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-        int          ret            = 0;
-        char        *xattr          = NULL;
-        char        *xattr_serz     = NULL;
-        char        xattr_cky[1024] = {0,};
-        dict_t      *nxattr         = NULL;
-        long         cky            = 0;
-        int32_t      padding        = 0;
-        int32_t      tlen           = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "possible NULL deref");
-                goto out;
-        }
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int keylen = 0;
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
 
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-
-                        if (op_ret < 0) {
-                                local->op_errno = op_errno;
-                        } else {
-                                local->op_ret = op_ret;
-                                if (!local->xdata_rsp && xdata)
-                                        local->xdata_rsp = dict_ref (xdata);
-                        }
-
-                        if (!dict || (op_ret < 0))
-                                goto unlock;
-
-                        if (!local->dict)
-                                local->dict = dict_new ();
-
-                        if (local->dict) {
-                                ret = dict_get_str (dict,
-                                                    local->cont.getxattr.name,
-                                                    &xattr);
-                                if (ret)
-                                        goto unlock;
-
-                                xattr = gf_strdup (xattr);
-
-                                (void)snprintf (xattr_cky, 1024, "%s-%ld",
-                                                local->cont.getxattr.name, cky);
-                                ret = dict_set_dynstr (local->dict,
-                                                       xattr_cky, xattr);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR,
-                                                -ret,
-                                                AFR_MSG_DICT_SET_FAILED,
-                                                "Cannot set xattr "
-                                                "cookie key");
-                                        goto unlock;
-                                }
-
-                                local->cont.getxattr.xattr_len += strlen (xattr) + 1;
-                        }
-                }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->cont.getxattr.xattr_len)
-                        goto unwind;
-
-                nxattr = dict_new ();
-                if (!nxattr)
-                        goto unwind;
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding += strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.xattr_len += (padding + 2);
-
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
-
-                if (!xattr_serz)
-                        goto unwind;
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        }
 
-                /* the xlator info */
-                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
-                                this->name);
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-                /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       xattr_serz + strlen (xattr_serz),
-                                                       &tlen, ' ');
-                if (ret) {
-                        goto unwind;
-                }
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
+        }
 
-                /* closing part */
-                *(xattr_serz + padding + tlen) = ')';
-                *(xattr_serz + padding + tlen + 1) = '\0';
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
-                                       xattr_serz);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Cannot set pathinfo key in dict");
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, nxattr, local->xdata_rsp);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
 
-                if (nxattr)
-                        dict_unref (nxattr);
-        }
+        if (nxattr)
+            dict_unref(nxattr);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+afr_aggregate_stime_xattr(dict_t *this, char *key, data_t *value, void *data)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
-                ret = gf_get_max_stime (THIS, data, key, value);
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+        ret = gf_get_max_stime(THIS, data, key, value);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *dict, dict_t *xdata)
+afr_common_getxattr_stime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *dict,
+                              dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
 
-        if (!frame || !frame->local || !this) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "possible NULL deref");
-                goto out;
-        }
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
 
-                if (!dict || (op_ret < 0)) {
-                        local->op_errno = op_errno;
-                        goto cleanup;
-                }
-
-                if (!local->dict)
-                        local->dict = dict_copy_with_ref (dict, NULL);
-                else
-                        dict_foreach (dict, afr_aggregate_stime_xattr,
-                                      local->dict);
-                local->op_ret = 0;
+        if (!dict || (op_ret < 0)) {
+            local->op_errno = op_errno;
+            goto cleanup;
         }
 
+        if (!local->dict)
+            local->dict = dict_copy_with_ref(dict, NULL);
+        else
+            dict_foreach(dict, afr_aggregate_stime_xattr, local->dict);
+        local->op_ret = 0;
+    }
+
 cleanup:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (!callcnt) {
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, local->dict, xdata);
-        }
+    if (!callcnt) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, xdata);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
-                      gf_boolean_t is_fgetxattr)
+afr_is_special_xattr(const char *name, fop_getxattr_cbk_t *cbk,
+                     gf_boolean_t is_fgetxattr)
 {
-        gf_boolean_t    is_spl = _gf_true;
-
-        GF_ASSERT (cbk);
-        if (!cbk || !name) {
-                is_spl = _gf_false;
-                goto out;
+    gf_boolean_t is_spl = _gf_true;
+
+    GF_ASSERT(cbk);
+    if (!cbk || !name) {
+        is_spl = _gf_false;
+        goto out;
+    }
+
+    if (!strcmp(name, GF_XATTR_PATHINFO_KEY) ||
+        !strcmp(name, GF_XATTR_USER_PATHINFO_KEY)) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_pathinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_pathinfo_cbk;
         }
-
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY) ||
-	    !strcmp (name, GF_XATTR_USER_PATHINFO_KEY)) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_pathinfo_cbk;
-                } else {
-                        *cbk = afr_getxattr_pathinfo_cbk;
-                }
-        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
-                             strlen (GF_XATTR_CLRLK_CMD))) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_clrlk_cbk;
-                } else {
-                        *cbk = afr_getxattr_clrlk_cbk;
-                }
-        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
-                             strlen (GF_XATTR_LOCKINFO_KEY))) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_lockinfo_cbk;
-                } else {
-                        *cbk = afr_getxattr_lockinfo_cbk;
-                }
-        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
-                *cbk = afr_common_getxattr_stime_cbk;
-        } else if (strcmp (name, QUOTA_SIZE_KEY) == 0) {
-                *cbk = afr_getxattr_quota_size_cbk;
+    } else if (!strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_clrlk_cbk;
         } else {
-                is_spl = _gf_false;
+            *cbk = afr_getxattr_clrlk_cbk;
         }
+    } else if (!strncmp(name, GF_XATTR_LOCKINFO_KEY,
+                        SLEN(GF_XATTR_LOCKINFO_KEY))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_lockinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_lockinfo_cbk;
+        }
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+        *cbk = afr_common_getxattr_stime_cbk;
+    } else if (strcmp(name, QUOTA_SIZE_KEY) == 0) {
+        *cbk = afr_getxattr_quota_size_cbk;
+    } else if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        *cbk = afr_getxattr_list_node_uuids_cbk;
+    } else {
+        is_spl = _gf_false;
+    }
 
 out:
-        return is_spl;
+    return is_spl;
 }
 
 static void
-afr_getxattr_all_subvols (xlator_t *this, call_frame_t *frame,
-			  const char *name, loc_t *loc,
-			  fop_getxattr_cbk_t cbk)
+afr_getxattr_all_subvols(xlator_t *this, call_frame_t *frame, const char *name,
+                         loc_t *loc, fop_getxattr_cbk_t cbk)
 {
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        int             i               = 0;
-        int             call_count      = 0;
-
-        priv     = this->private;
-
-        local = frame->local;
-        //local->call_count set in afr_local_init
-        call_count = local->call_count;
-
-        //If up-children count is 0, afr_local_init would have failed already
-        //and the call would have unwound so not handling it here.
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, cbk,
-                                           (void *) (long) i, priv->children[i],
-                                           priv->children[i]->fops->getxattr,
-                                           loc, name, NULL);
-                        if (!--call_count)
-                                break;
-                }
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        GF_FREE(local->cont.getxattr.name);
+        local->cont.getxattr.name = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+    }
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->getxattr, loc,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
-        return;
+    }
+    return;
 }
 
 int
-afr_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                          xlator_t **subvols)
+afr_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
 {
-        xlator_t *this = frame->this;
-        afr_private_t *priv = this->private;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
 
-        memcpy (subvols, priv->children, sizeof (*subvols) * priv->child_count);
+    memcpy(subvols, priv->children, sizeof(*subvols) * priv->child_count);
 
-        if (type == MARKER_XTIME_TYPE) {
-                /*Don't error out on ENOENT/ENOTCONN */
-                gauge[MCNT_NOTFOUND] = 0;
-                gauge[MCNT_ENOTCONN] = 0;
-        }
-        return priv->child_count;
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
+    return priv->child_count;
 }
 
 static int
-afr_handle_heal_xattrs (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        const char *heal_op)
+afr_handle_heal_xattrs(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       const char *heal_op)
 {
-        int                     ret     = -1;
-        afr_spb_status_t       *data    = NULL;
+    int ret = -1;
+    afr_spb_status_t *data = NULL;
 
-        if (!strcmp (heal_op, GF_HEAL_INFO)) {
-                afr_get_heal_info (frame, this, loc);
-                ret = 0;
-                goto out;
-        }
+    if (!strcmp(heal_op, GF_HEAL_INFO)) {
+        afr_get_heal_info(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
 
-        if (!strcmp (heal_op, GF_AFR_HEAL_SBRAIN)) {
-                afr_heal_splitbrain_file (frame, this, loc);
-                ret = 0;
-                goto out;
+    if (!strcmp(heal_op, GF_AFR_HEAL_SBRAIN)) {
+        afr_heal_splitbrain_file(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
+
+    if (!strcmp(heal_op, GF_AFR_SBRAIN_STATUS)) {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spb_status_t);
+        if (!data) {
+            ret = 1;
+            goto out;
         }
-
-        if (!strcmp (heal_op, GF_AFR_SBRAIN_STATUS)) {
-                data = GF_CALLOC (1, sizeof (*data), gf_afr_mt_spb_status_t);
-                if (!data) {
-                        ret = 1;
-                        goto out;
-                }
-                data->frame = frame;
-                data->loc = loc;
-                ret = synctask_new (this->ctx->env,
-                                    afr_get_split_brain_status,
-                                    afr_get_split_brain_status_cbk,
-                                    NULL, data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN_STATUS,
-                                "Failed to create"
-                                " synctask. Unable to fetch split-brain status"
-                                " for %s.", loc->name);
-                        ret = 1;
-                        goto out;
-                }
-                goto out;
+        data->frame = frame;
+        data->loc = loc;
+        ret = synctask_new(this->ctx->env, afr_get_split_brain_status,
+                           afr_get_split_brain_status_cbk, NULL, data);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                   "Failed to create"
+                   " synctask. Unable to fetch split-brain status"
+                   " for %s.",
+                   loc->name);
+            ret = 1;
+            goto out;
         }
+        goto out;
+    }
 
 out:
-        if (ret == 1) {
-                AFR_STACK_UNWIND (getxattr, frame, -1, ENOMEM, NULL, NULL);
-                if (data)
-                        GF_FREE (data);
-                ret = 0;
-        }
-        return ret;
+    if (ret == 1) {
+        AFR_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *name, dict_t *xdata)
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
 {
-        afr_private_t           *priv         = NULL;
-        xlator_t                **children    = NULL;
-        afr_local_t             *local        = NULL;
-        int                     i             = 0;
-        int32_t                 op_errno      = 0;
-        int                     ret           = -1;
-        fop_getxattr_cbk_t      cbk           = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int i = 0;
+    int32_t op_errno = 0;
+    int ret = -1;
+    fop_getxattr_cbk_t cbk = NULL;
 
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    priv = this->private;
 
-        priv     = this->private;
+    children = priv->children;
 
-        children = priv->children;
+    loc_copy(&local->loc, loc);
 
-        loc_copy (&local->loc, loc);
+    local->op = GF_FOP_GETXATTR;
 
-	local->op = GF_FOP_GETXATTR;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    if (!name)
+        goto no_name;
 
-        if (!name)
-                goto no_name;
+    local->cont.getxattr.name = gf_strdup(name);
 
-        local->cont.getxattr.name = gf_strdup (name);
+    if (!local->cont.getxattr.name) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-	if (!local->cont.getxattr.name) {
-		op_errno = ENOMEM;
-		goto out;
-	}
+    if (!strncmp(name, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        op_errno = ENODATA;
+        goto out;
+    }
 
-        if (!strncmp (name, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-                op_errno = ENODATA;
-                goto out;
-        }
-
-        if (cluster_handle_marker_getxattr (frame, loc, name, priv->vol_uuid,
-                                            afr_getxattr_unwind,
-                                            afr_marker_populate_args) == 0)
-                return 0;
+    if (cluster_handle_marker_getxattr(frame, loc, name, priv->vol_uuid,
+                                       afr_getxattr_unwind,
+                                       afr_marker_populate_args) == 0)
+        return 0;
 
-        ret = afr_handle_heal_xattrs (frame, this, &local->loc, name);
-        if (ret == 0)
-                return 0;
+    ret = afr_handle_heal_xattrs(frame, this, &local->loc, name);
+    if (ret == 0)
+        return 0;
 
-        /*
-         * Special xattrs which need responses from all subvols
-         */
-        if (afr_is_special_xattr (name, &cbk, 0)) {
-                afr_getxattr_all_subvols (this, frame, name, loc, cbk);
-                return 0;
-        }
+    /*
+     * Heal daemons don't have IO threads ... and as a result they
+     * send this getxattr down and eventually crash :(
+     */
+    op_errno = -1;
+    GF_CHECK_XATTR_KEY_AND_GOTO(name, IO_THREADS_QUEUE_SIZE_KEY, op_errno, out);
+
+    /*
+     * Special xattrs which need responses from all subvols
+     */
+    if (afr_is_special_xattr(name, &cbk, 0)) {
+        afr_getxattr_all_subvols(this, frame, name, loc, cbk);
+        return 0;
+    }
 
-        if (XATTR_IS_NODE_UUID (name)) {
-                i = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) i,
-                                   children[i],
-                                   children[i]->fops->getxattr,
-                                   loc, name, xdata);
-                return 0;
-        }
+    if (XATTR_IS_NODE_UUID(name)) {
+        i = 0;
+        STACK_WIND_COOKIE(frame, afr_getxattr_node_uuid_cbk, (void *)(long)i,
+                          children[i], children[i]->fops->getxattr, loc, name,
+                          xdata);
+        return 0;
+    }
 
 no_name:
 
-	afr_read_txn (frame, this, local->loc.inode, afr_getxattr_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, local->loc.inode, afr_getxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    if (ret < 0)
+        AFR_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 /* {{{ fgetxattr */
 
-
 int32_t
-afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
-                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                   dict_t *dict, dict_t *xdata)
+afr_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (dict)
-		afr_filter_xattrs (dict);
+    if (dict)
+        afr_filter_xattrs(dict);
 
-	AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
-afr_fgetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fgetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
-				  local->op_errno, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fgetxattr,
-			   local->fd, local->cont.getxattr.name,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
 
-static void
-afr_fgetxattr_all_subvols (xlator_t *this, call_frame_t *frame,
-			   fop_fgetxattr_cbk_t cbk)
-{
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        int             i               = 0;
-        int             call_count      = 0;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
 
-        priv     = this->private;
+    STACK_WIND_COOKIE(frame, afr_fgetxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fgetxattr, local->fd,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
-        local = frame->local;
-        //local->call_count set in afr_local_init
-        call_count = local->call_count;
-
-        //If up-children count is 0, afr_local_init would have failed already
-        //and the call would have unwound so not handling it here.
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fgetxattr,
-                                           local->fd, local->cont.getxattr.name,
-					   NULL);
-                        if (!--call_count)
-                                break;
-                }
+static void
+afr_fgetxattr_all_subvols(xlator_t *this, call_frame_t *frame,
+                          fop_fgetxattr_cbk_t cbk)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->fgetxattr, local->fd,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
+    }
 
-        return;
+    return;
 }
 
-
 int
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata)
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
 {
-        afr_local_t          *local      = NULL;
-        int32_t               op_errno   = 0;
-        fop_fgetxattr_cbk_t   cbk        = NULL;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	local->op = GF_FOP_FGETXATTR;
-        local->fd = fd_ref (fd);
-        if (name) {
-                local->cont.getxattr.name = gf_strdup (name);
-		if (!local->cont.getxattr.name) {
-			op_errno = ENOMEM;
-			goto out;
-		}
-	}
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
-
-        /* pathinfo gets handled only in getxattr(), but we need to handle
-         * lockinfo.
-         * If we are doing fgetxattr with lockinfo as the key then we
-         * collect information from all children.
-         */
-        if (afr_is_special_xattr (name, &cbk, 1)) {
-                afr_fgetxattr_all_subvols (this, frame, cbk);
-                return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    fop_fgetxattr_cbk_t cbk = NULL;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FGETXATTR;
+    local->fd = fd_ref(fd);
+    if (name) {
+        local->cont.getxattr.name = gf_strdup(name);
+        if (!local->cont.getxattr.name) {
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    /* pathinfo gets handled only in getxattr(), but we need to handle
+     * lockinfo.
+     * If we are doing fgetxattr with lockinfo as the key then we
+     * collect information from all children.
+     */
+    if (afr_is_special_xattr(name, &cbk, 1)) {
+        afr_fgetxattr_all_subvols(this, frame, cbk);
+        return 0;
+    }
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_fgetxattr_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_fgetxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ readv */
 
 int
-afr_readv_cbk (call_frame_t *frame, void *cookie,
-               xlator_t *this, int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref, dict_t *xdata)
+afr_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *buf, struct iobref *iobref, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-			  vector, count, buf, iobref, xdata);
-	return 0;
+    AFR_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf, iobref,
+                     xdata);
+    return 0;
 }
 
-
 int
-afr_readv_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readv_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readv, frame, local->op_ret, local->op_errno,
-				  0, 0, 0, 0, 0);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_readv_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->readv,
-			   local->fd, local->cont.readv.size,
-			   local->cont.readv.offset, local->cont.readv.flags,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, 0, 0, 0,
+                         0, 0);
+        return 0;
+    }
 
+    STACK_WIND_COOKIE(
+        frame, afr_readv_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->readv, local->fd, local->cont.readv.size,
+        local->cont.readv.offset, local->cont.readv.flags, local->xdata_req);
+    return 0;
+}
 
 int
-afr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	   off_t offset, uint32_t flags, dict_t *xdata)
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        afr_local_t   * local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_READ;
-        local->fd = fd_ref (fd);
-        local->cont.readv.size = size;
-        local->cont.readv.offset = offset;
-        local->cont.readv.flags = flags;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_READ;
+    local->fd = fd_ref(fd);
+    local->cont.readv.size = size;
+    local->cont.readv.offset = offset;
+    local->cont.readv.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_readv_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_readv_wind, AFR_DATA_TRANSACTION);
 
-        return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+    AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
 
-	return 0;
+    return 0;
 }
 
 /* }}} */
@@ -1792,77 +1822,73 @@ out:
 /* {{{ seek */
 
 int
-afr_seek_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+afr_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, off_t offset, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-
-                afr_read_txn_continue (frame, this, (long) cookie);
-                return 0;
-        }
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-        AFR_STACK_UNWIND (seek, frame, op_ret, op_errno, offset, xdata);
+        afr_read_txn_continue(frame, this, (long)cookie);
         return 0;
-}
+    }
 
+    AFR_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
 
 int
-afr_seek_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_seek_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        if (subvol == -1) {
-                AFR_STACK_UNWIND (seek, frame, local->op_ret, local->op_errno,
-                                  0, NULL);
-                return 0;
-        }
+    local = frame->local;
+    priv = this->private;
 
-        STACK_WIND_COOKIE (frame, afr_seek_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->seek,
-                           local->fd, local->cont.seek.offset,
-                           local->cont.seek.what, local->xdata_req);
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(seek, frame, local->op_ret, local->op_errno, 0, NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_COOKIE(
+        frame, afr_seek_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->seek, local->fd, local->cont.seek.offset,
+        local->cont.seek.what, local->xdata_req);
+    return 0;
+}
 
 int
-afr_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-          gf_seek_what_t what, dict_t *xdata)
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata)
 {
-        afr_local_t    *local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->op = GF_FOP_SEEK;
-        local->fd = fd_ref (fd);
-        local->cont.seek.offset = offset;
-        local->cont.seek.what = what;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_SEEK;
+    local->fd = fd_ref(fd);
+    local->cont.seek.offset = offset;
+    local->cont.seek.what = what;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        afr_read_txn (frame, this, fd->inode, afr_seek_wind,
-                      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_seek_wind, AFR_DATA_TRANSACTION);
 
-        return 0;
+    return 0;
 out:
-        AFR_STACK_UNWIND (seek, frame, -1, op_errno, 0, NULL);
+    AFR_STACK_UNWIND(seek, frame, -1, op_errno, 0, NULL);
 
-        return 0;
+    return 0;
 }
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index d128134ef2a..8c982bc7e6f 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -12,34 +12,34 @@
 #define __INODE_READ_H__
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask, dict_t *xdata);
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc, dict_t *xdata);
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, dict_t *xdata);
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size, dict_t *xdata);
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
 
 int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name, dict_t *xdata);
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
 
 int32_t
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata);
-
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
 
 int
-afr_handle_quota_size (call_frame_t *frame, xlator_t *this);
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata);
+int
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this);
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 76526bcf177..1d6e4f3570a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -8,777 +8,779 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 #include "protocol-common.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
 #include "afr-messages.h"
 
 static void
-__afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int read_subvol = 0;
-	int i = 0;
-        afr_read_subvol_args_t args = {0,};
-        struct iatt  *stbuf = NULL;
-        int    ret = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-        /*This code needs to stay till DHT sends fops on linked
-         * inodes*/
-        if (local->inode && !inode_is_linked (local->inode)) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->replies[i].valid)
-                                continue;
-                        if (local->replies[i].op_ret == -1)
-                                continue;
-                        if (!gf_uuid_is_null
-                                        (local->replies[i].poststat.ia_gfid)) {
-                                gf_uuid_copy (args.gfid,
-                                            local->replies[i].poststat.ia_gfid);
-                                args.ia_type =
-                                        local->replies[i].poststat.ia_type;
-                                break;
-                        } else {
-                                ret = dict_get_bin (local->replies[i].xdata,
-                                                    DHT_IATT_IN_XDATA_KEY,
-                                                    (void **) &stbuf);
-                                if (ret)
-                                        continue;
-                                gf_uuid_copy (args.gfid, stbuf->ia_gfid);
-                                args.ia_type = stbuf->ia_type;
-                                break;
-                        }
-                }
+__afr_inode_write_finalize(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int ret = 0;
+    int read_subvol = 0;
+    struct iatt *stbuf = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, local->inode, out);
+
+    /*This code needs to stay till DHT sends fops on linked
+     * inodes*/
+    if (!inode_is_linked(local->inode)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
+            if (local->replies[i].op_ret == -1)
+                continue;
+            if (!gf_uuid_is_null(local->replies[i].poststat.ia_gfid)) {
+                gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+                args.ia_type = local->replies[i].poststat.ia_type;
+                break;
+            } else {
+                ret = dict_get_bin(local->replies[i].xdata,
+                                   DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+                if (ret)
+                    continue;
+                gf_uuid_copy(args.gfid, stbuf->ia_gfid);
+                args.ia_type = stbuf->ia_type;
+                break;
+            }
+        }
+    }
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        read_subvol = afr_metadata_subvol_get(local->inode, this, NULL,
+                                              local->readable, NULL, &args);
+    } else {
+        read_subvol = afr_data_subvol_get(local->inode, this, NULL,
+                                          local->readable, NULL, &args);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->inode, local->readable, NULL,
+                         NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0)
+            continue;
+
+        /* Order of checks in the compound conditional
+           below is important.
+
+           - Highest precedence: largest op_ret
+           - Next precedence: if all op_rets are equal, read subvol
+           - Least precedence: any succeeded subvol
+        */
+        if ((local->op_ret < local->replies[i].op_ret) ||
+            ((local->op_ret == local->replies[i].op_ret) &&
+             (i == read_subvol))) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.inode_wfop.prebuf = local->replies[i].prestat;
+            local->cont.inode_wfop.postbuf = local->replies[i].poststat;
+
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            if (local->replies[i].xattr) {
+                if (local->xattr_rsp)
+                    dict_unref(local->xattr_rsp);
+                local->xattr_rsp = dict_ref(local->replies[i].xattr);
+            }
         }
+    }
 
-	if (local->inode) {
-		if (local->transaction.type == AFR_METADATA_TRANSACTION)
-                        read_subvol = afr_metadata_subvol_get (local->inode,
-                                      this, NULL, local->readable, NULL, &args);
-		else
-			read_subvol = afr_data_subvol_get (local->inode, this,
-                                            NULL, local->readable, NULL, &args);
-	}
-
-	local->op_ret = -1;
-	local->op_errno = afr_final_errno (local, priv);
-        afr_pick_error_xdata (local, priv, local->inode, local->readable, NULL,
-                              NULL);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret < 0) {
-			afr_inode_read_subvol_reset (local->inode, this);
-			continue;
-		}
-
-		/* Order of checks in the compound conditional
-		   below is important.
-
-		   - Highest precedence: largest op_ret
-		   - Next precendence: if all op_rets are equal, read subvol
-		   - Least precedence: any succeeded subvol
-		*/
-		if ((local->op_ret < local->replies[i].op_ret) ||
-		    ((local->op_ret == local->replies[i].op_ret) &&
-		     (i == read_subvol))) {
-
-			local->op_ret = local->replies[i].op_ret;
-			local->op_errno = local->replies[i].op_errno;
-
-			local->cont.inode_wfop.prebuf =
-				local->replies[i].prestat;
-			local->cont.inode_wfop.postbuf =
-				local->replies[i].poststat;
-
-			if (local->replies[i].xdata) {
-				if (local->xdata_rsp)
-					dict_unref (local->xdata_rsp);
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			}
-			if (local->replies[i].xattr) {
-				if (local->xattr_rsp)
-					dict_unref (local->xattr_rsp);
-				local->xattr_rsp =
-					dict_ref (local->replies[i].xattr);
-			}
-		}
-	}
-
-        afr_txn_arbitrate_fop_cbk (frame, this);
+    afr_set_in_flight_sb_status(this, frame, local->inode);
+out:
+    return;
 }
 
-
 static void
-__afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
-			int op_ret, int op_errno,
-			struct iatt *prebuf, struct iatt *postbuf,
-			dict_t *xattr, dict_t *xdata)
+__afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	local->replies[child_index].valid = 1;
+    local->replies[child_index].valid = 1;
 
-        if (AFR_IS_ARBITER_BRICK(priv, child_index) && op_ret == 1)
-                op_ret = iov_length (local->cont.writev.vector,
-                                     local->cont.writev.count);
+    if (AFR_IS_ARBITER_BRICK(priv, child_index) && op_ret == 1)
+        op_ret = iov_length(local->cont.writev.vector,
+                            local->cont.writev.count);
 
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-        if (xdata)
-                local->replies[child_index].xdata = dict_ref (xdata);
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
 
-	if (op_ret >= 0) {
-		if (prebuf)
-			local->replies[child_index].prestat = *prebuf;
-		if (postbuf)
-			local->replies[child_index].poststat = *postbuf;
-		if (xattr)
-			local->replies[child_index].xattr = dict_ref (xattr);
-	} else {
-		afr_transaction_fop_failed (frame, this, child_index);
-	}
+    if (op_ret >= 0) {
+        if (prebuf)
+            local->replies[child_index].prestat = *prebuf;
+        if (postbuf)
+            local->replies[child_index].poststat = *postbuf;
+        if (xattr)
+            local->replies[child_index].xattr = dict_ref(xattr);
+    } else {
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-        return;
+    return;
 }
 
-
 static int
-__afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xattr,
-					xdata);
+__afr_inode_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, xattr, xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        __afr_inode_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		__afr_inode_write_finalize (frame, this);
 
-		if (afr_txn_nothing_failed (frame, this)) {
-                        /*if it did pre-op, it will do post-op changing ctime*/
-                        if (priv->consistent_metadata &&
-                            afr_needs_changelog_update (local))
-                                afr_zero_fill_stat (local);
-                        local->transaction.unwind (frame, this);
-                }
+        afr_transaction_resume(frame, this);
+    }
 
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    return 0;
 }
 
 /* {{{ writev */
 
 void
-afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame)
 {
-        afr_local_t *src_local = NULL;
-        afr_local_t *dst_local = NULL;
+    afr_local_t *src_local = NULL;
+    afr_local_t *dst_local = NULL;
 
-        src_local = src_frame->local;
-        dst_local = dst_frame->local;
+    src_local = src_frame->local;
+    dst_local = dst_frame->local;
 
-        dst_local->op_ret = src_local->op_ret;
-        dst_local->op_errno = src_local->op_errno;
-        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
-        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
-	if (src_local->xdata_rsp)
-		dst_local->xdata_rsp = dict_ref (src_local->xdata_rsp);
+    dst_local->op_ret = src_local->op_ret;
+    dst_local->op_errno = src_local->op_errno;
+    dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+    dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+    if (src_local->xdata_rsp)
+        dst_local->xdata_rsp = dict_ref(src_local->xdata_rsp);
 }
 
 void
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+afr_writev_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
 
-        local = frame->local;
+    local = frame->local;
 
-       if (priv->consistent_metadata)
-               afr_zero_fill_stat (local);
+    if (priv->consistent_metadata)
+        afr_zero_fill_stat(local);
 
-        AFR_STACK_UNWIND (writev, frame,
-                          local->op_ret, local->op_errno,
-                          &local->cont.inode_wfop.prebuf,
-                          &local->cont.inode_wfop.postbuf,
-                          local->xdata_rsp);
+    AFR_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
 }
 
-
 int
-afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
+afr_transaction_writev_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *fop_frame = NULL;
+    call_frame_t *fop_frame = NULL;
 
-        fop_frame = afr_transaction_detach_fop_frame (frame);
+    fop_frame = afr_transaction_detach_fop_frame(frame);
 
-        if (fop_frame) {
-                afr_writev_copy_outvars (frame, fop_frame);
-                afr_writev_unwind (fop_frame, this);
-        }
-        return 0;
+    if (fop_frame) {
+        afr_writev_copy_outvars(frame, fop_frame);
+        afr_writev_unwind(fop_frame, this);
+    }
+    return 0;
 }
 
 static void
-afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int           i      = 0;
-
-        local = frame->local;
-        priv = this->private;
-        /*
-         * We already have the best case result of the writev calls staged
-         * as the return value. Any writev that returns some value less
-         * than the best case is now out of sync, so mark the fop as
-         * failed. Note that fops that have returned with errors have
-         * already been marked as failed.
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if ((!local->replies[i].valid) ||
-                    (local->replies[i].op_ret == -1))
-                        continue;
-
-                if (local->replies[i].op_ret < local->op_ret)
-                        afr_transaction_fop_failed (frame, this, i);
-        }
+afr_writev_handle_short_writes(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    /*
+     * We already have the best case result of the writev calls staged
+     * as the return value. Any writev that returns some value less
+     * than the best case is now out of sync, so mark the fop as
+     * failed. Note that fops that have returned with errors have
+     * already been marked as failed.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!local->replies[i].valid) || (local->replies[i].op_ret == -1))
+            continue;
+
+        if (local->replies[i].op_ret < local->op_ret)
+            afr_transaction_fop_failed(frame, this, i);
+    }
 }
 
-int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+void
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t    *fop_frame = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int ret = 0;
-        uint32_t open_fd_count = 0;
-        uint32_t write_is_append = 0;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, NULL, xdata);
-		if (op_ret == -1 || !xdata)
-			goto unlock;
-
-		write_is_append = 0;
-		ret = dict_get_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
-				       &write_is_append);
-		if (ret || !write_is_append)
-			local->append_write = _gf_false;
-
-		ret = dict_get_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
-				       &open_fd_count);
-		if (ret == -1)
-			goto unlock;
-		if ((open_fd_count > local->open_fd_count)) {
-			local->open_fd_count = open_fd_count;
-			local->update_open_fd_count = _gf_true;
-		}
+    int ret = 0;
+    afr_local_t *local = frame->local;
+    uint32_t open_fd_count = 0;
+    uint32_t write_is_append = 0;
+    int32_t num_inodelks = 0;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, NULL, xdata);
+        if (op_ret == -1 || !xdata)
+            goto unlock;
+
+        write_is_append = 0;
+        ret = dict_get_uint32(xdata, GLUSTERFS_WRITE_IS_APPEND,
+                              &write_is_append);
+        if (ret || !write_is_append)
+            local->append_write = _gf_false;
+
+        ret = dict_get_uint32(xdata, GLUSTERFS_ACTIVE_FD_COUNT, &open_fd_count);
+        if (ret < 0)
+            goto unlock;
+        if (open_fd_count > local->open_fd_count) {
+            local->open_fd_count = open_fd_count;
+            local->update_open_fd_count = _gf_true;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		if (!local->stable_write && !local->append_write)
-			/* An appended write removes the necessity to
-			   fsync() the file. This is because self-heal
-			   has the logic to check for larger file when
-			   the xattrs are not reliably pointing at
-			   a stale file.
-			*/
-			afr_fd_report_unstable_write (this, local->fd);
-
-		__afr_inode_write_finalize (frame, this);
-
-                afr_writev_handle_short_writes (frame, this);
-
-                if (local->update_open_fd_count)
-                        afr_handle_open_fd_count (frame, this);
-
-                if (!afr_txn_nothing_failed (frame, this)) {
-                        //Don't unwind until post-op is complete
-                        local->transaction.resume (frame, this);
-                } else {
-                /*
-                 * Generally inode-write fops do transaction.unwind then
-                 * transaction.resume, but writev needs to make sure that
-                 * delayed post-op frame is placed in fdctx before unwind
-                 * happens. This prevents the race of flush doing the
-                 * changelog wakeup first in fuse thread and then this
-                 * writev placing its delayed post-op frame in fdctx.
-                 * This helps flush make sure all the delayed post-ops are
-                 * completed.
-                 */
-
-                        fop_frame = afr_transaction_detach_fop_frame (frame);
-                        afr_writev_copy_outvars (frame, fop_frame);
-                        local->transaction.resume (frame, this);
-                        afr_writev_unwind (fop_frame, this);
-                }
+
+        ret = dict_get_int32_sizen(xdata, GLUSTERFS_INODELK_COUNT,
+                                   &num_inodelks);
+        if (ret < 0)
+            goto unlock;
+        if (num_inodelks > local->num_inodelks) {
+            local->num_inodelks = num_inodelks;
+            local->update_num_inodelks = _gf_true;
         }
-        return 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 }
 
-static int
-afr_arbiter_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
+void
+afr_process_post_writev(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = frame->local;
-        afr_private_t *priv = this->private;
-        static char byte = 0xFF;
-        static struct iovec vector = {&byte, 1};
-        int32_t count = 1;
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
 
-        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->writev,
-			   local->fd, &vector, count, local->cont.writev.offset,
-			   local->cont.writev.flags, local->cont.writev.iobref,
-			   local->xdata_req);
+    local = frame->local;
 
-        return 0;
+    if (!local->stable_write && !local->append_write)
+        /* An appended write removes the necessity to
+           fsync() the file. This is because self-heal
+           has the logic to check for larger file when
+           the xattrs are not reliably pointing at
+           a stale file.
+        */
+        afr_fd_report_unstable_write(this, local);
+
+    __afr_inode_write_finalize(frame, this);
+
+    afr_writev_handle_short_writes(frame, this);
+
+    if (local->update_open_fd_count)
+        local->inode_ctx->open_fd_count = local->open_fd_count;
+    if (local->update_num_inodelks &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        lock->num_inodelks = local->num_inodelks;
+    }
 }
 
 int
-afr_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_writev_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    call_frame_t *fop_frame = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
 
-        local = frame->local;
-        priv = this->private;
+    afr_inode_write_fill(frame, this, child_index, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
-        if (AFR_IS_ARBITER_BRICK(priv, subvol)) {
-                afr_arbiter_writev_wind (frame, this, subvol);
-                return 0;
-        }
+    call_count = afr_frame_return(frame);
 
-	STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->writev,
-			   local->fd, local->cont.writev.vector,
-			   local->cont.writev.count, local->cont.writev.offset,
-			   local->cont.writev.flags, local->cont.writev.iobref,
-			   local->xdata_req);
-        return 0;
+    if (call_count == 0) {
+        afr_process_post_writev(frame, this);
+
+        if (!afr_txn_nothing_failed(frame, this)) {
+            // Don't unwind until post-op is complete
+            afr_transaction_resume(frame, this);
+        } else {
+            /*
+             * Generally inode-write fops do transaction.unwind then
+             * transaction.resume, but writev needs to make sure that
+             * delayed post-op frame is placed in fdctx before unwind
+             * happens. This prevents the race of flush doing the
+             * changelog wakeup first in fuse thread and then this
+             * writev placing its delayed post-op frame in fdctx.
+             * This helps flush make sure all the delayed post-ops are
+             * completed.
+             */
+
+            fop_frame = afr_transaction_detach_fop_frame(frame);
+            afr_writev_copy_outvars(frame, fop_frame);
+            afr_transaction_resume(frame, this);
+            afr_writev_unwind(fop_frame, this);
+        }
+    }
+    return 0;
 }
 
+static int
+afr_arbiter_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    static char byte = 0xFF;
+    static struct iovec vector = {&byte, 1};
+    int32_t count = 1;
+
+    STACK_WIND_COOKIE(
+        frame, afr_writev_wind_cbk, (void *)(long)subvol,
+        priv->children[subvol], priv->children[subvol]->fops->writev, local->fd,
+        &vector, count, local->cont.writev.offset, local->cont.writev.flags,
+        local->cont.writev.iobref, local->xdata_req);
+
+    return 0;
+}
 
 int
-afr_do_writev (call_frame_t *frame, xlator_t *this)
+afr_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        call_frame_t    *transaction_frame = NULL;
-        afr_local_t     *local             = NULL;
-        int             ret   = -1;
-        int             op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    local = frame->local;
+    priv = this->private;
+
+    if (AFR_IS_ARBITER_BRICK(priv, subvol)) {
+        afr_arbiter_writev_wind(frame, this, subvol);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_writev_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->writev, local->fd,
+                      local->cont.writev.vector, local->cont.writev.count,
+                      local->cont.writev.offset, local->cont.writev.flags,
+                      local->cont.writev.iobref, local->xdata_req);
+    return 0;
+}
 
-        local = frame->local;
-        transaction_frame->local = local;
-	frame->local = NULL;
+int
+afr_do_writev(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	if (!AFR_FRAME_INIT (frame, op_errno))
-		goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local->op = GF_FOP_WRITE;
+    local = frame->local;
+    transaction_frame->local = local;
+    frame->local = NULL;
 
-        local->transaction.wind   = afr_writev_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_transaction_writev_unwind;
+    if (!AFR_FRAME_INIT(frame, op_errno))
+        goto out;
 
-        local->transaction.main_frame = frame;
+    local->op = GF_FOP_WRITE;
 
-        if (local->fd->flags & O_APPEND) {
-               /*
-                * Backend vfs ignores the 'offset' for append mode fd so
-                * locking just the region provided for the writev does not
-                * give consistency guarantee. The actual write may happen at a
-                * completely different range than the one provided by the
-                * offset, len in the fop. So lock the entire file.
-                */
-                local->transaction.start   = 0;
-                local->transaction.len     = 0;
-        } else {
-                local->transaction.start   = local->cont.writev.offset;
-                local->transaction.len     = iov_length (local->cont.writev.vector,
-                                                         local->cont.writev.count);
-        }
+    local->transaction.wind = afr_writev_wind;
+    local->transaction.unwind = afr_transaction_writev_unwind;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-            op_errno = -ret;
-            goto out;
-        }
+    local->transaction.main_frame = frame;
 
-	return 0;
+    if (local->fd->flags & O_APPEND) {
+        /*
+         * Backend vfs ignores the 'offset' for append mode fd so
+         * locking just the region provided for the writev does not
+         * give consistency guarantee. The actual write may happen at a
+         * completely different range than the one provided by the
+         * offset, len in the fop. So lock the entire file.
+         */
+        local->transaction.start = 0;
+        local->transaction.len = 0;
+    } else {
+        local->transaction.start = local->cont.writev.offset;
+        local->transaction.len = iov_length(local->cont.writev.vector,
+                                            local->cont.writev.count);
+    }
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata)
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    int op_errno = ENOMEM;
+    int ret = -1;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.writev.vector = iov_dup (vector, count);
-	if (!local->cont.writev.vector)
-		goto out;
-        local->cont.writev.count      = count;
-        local->cont.writev.offset     = offset;
-        local->cont.writev.flags      = flags;
-        local->cont.writev.iobref     = iobref_ref (iobref);
+    local->cont.writev.vector = iov_dup(vector, count);
+    if (!local->cont.writev.vector)
+        goto out;
+    local->cont.writev.count = count;
+    local->cont.writev.offset = offset;
+    local->cont.writev.flags = flags;
+    local->cont.writev.iobref = iobref_ref(iobref);
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_OPEN_FD_COUNT, 4)) {
-		op_errno = ENOMEM;
-		goto out;
-	}
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_ACTIVE_FD_COUNT, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
-		op_errno = ENOMEM;
-		goto out;
-	}
+    if (dict_set_str_sizen(local->xdata_req, GLUSTERFS_INODELK_DOM_COUNT,
+                           this->name)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-	/* Set append_write to be true speculatively. If on any
-	   server it turns not be true, we unset it in the
-	   callback.
-	*/
-	local->append_write = _gf_true;
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-	/* detect here, but set it in writev_wind_cbk *after* the unstable
-	   write is performed
-	*/
-	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
+    /* Set append_write to be true speculatively. If on any
+       server it turns not be true, we unset it in the
+       callback.
+    */
+    local->append_write = _gf_true;
 
-        afr_fix_open (fd, this);
+    /* detect here, but set it in writev_wind_cbk *after* the unstable
+       write is performed
+    */
+    local->stable_write = !!((fd->flags | flags) & (O_SYNC | O_DSYNC));
 
-        afr_do_writev (frame, this);
+    afr_fix_open(fd, this);
 
-	return 0;
+    afr_do_writev(frame, this);
+
+    return 0;
 out:
-	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ truncate */
 
 int
-afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_truncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (truncate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(truncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+afr_truncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
-		local->stable_write = _gf_false;
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_truncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_truncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->truncate,
-			   &local->loc, local->cont.truncate.offset,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_truncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->truncate, &local->loc,
+                      local->cont.truncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset, dict_t *xdata)
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-	int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.truncate.offset  = offset;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.truncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-	local->transaction.wind   = afr_truncate_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_truncate_unwind;
+    local->transaction.wind = afr_truncate_wind;
+    local->transaction.unwind = afr_truncate_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->op = GF_FOP_TRUNCATE;
+    local->op = GF_FOP_TRUNCATE;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = offset;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = offset;
+    local->transaction.len = 0;
 
-	/* Set it true speculatively, will get reset in afr_truncate_wind_cbk
-	   if truncate was not a NOP */
-	local->stable_write = _gf_true;
+    /* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ ftruncate */
 
-
 int
-afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(ftruncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_ftruncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
-		local->stable_write = _gf_false;
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_ftruncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->ftruncate,
-			   local->fd, local->cont.ftruncate.offset,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_ftruncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->ftruncate, local->fd,
+                      local->cont.ftruncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       dict_t *xdata)
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	call_frame_t *transaction_frame = NULL;
-	int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local = AFR_FRAME_INIT (transaction_frame, op_errno);
-        if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.ftruncate.offset  = offset;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.ftruncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->op = GF_FOP_FTRUNCATE;
+    local->op = GF_FOP_FTRUNCATE;
 
-	local->transaction.wind   = afr_ftruncate_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_ftruncate_unwind;
+    local->transaction.wind = afr_ftruncate_wind;
+    local->transaction.unwind = afr_ftruncate_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.ftruncate.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.ftruncate.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	/* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
-	   if truncate was not a NOP */
-	local->stable_write = _gf_true;
+    /* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -786,1706 +788,1778 @@ out:
 /* {{{ setattr */
 
 int
-afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_setattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (setattr, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(setattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno,
-                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_setattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
-
 int
-afr_setattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_setattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->setattr,
-			   &local->loc, &local->cont.setattr.in_buf,
-			   local->cont.setattr.valid, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_setattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setattr, &local->loc,
+                      &local->cont.setattr.in_buf, local->cont.setattr.valid,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
-	     int32_t valid, dict_t *xdata)
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.setattr.in_buf = *buf;
-        local->cont.setattr.valid  = valid;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.setattr.in_buf = *buf;
+    local->cont.setattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_setattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_setattr_unwind;
+    local->transaction.wind = afr_setattr_wind;
+    local->transaction.unwind = afr_setattr_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_SETATTR;
+    local->op = GF_FOP_SETATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* {{{ fsetattr */
 
 int
-afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fsetattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_fsetattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                      struct iatt *postop, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
-
 int
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fsetattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fsetattr,
-			   local->fd, &local->cont.fsetattr.in_buf,
-			   local->cont.fsetattr.valid, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fsetattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetattr, local->fd,
+                      &local->cont.fsetattr.in_buf, local->cont.fsetattr.valid,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fsetattr.in_buf = *buf;
-        local->cont.fsetattr.valid  = valid;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.fsetattr.in_buf = *buf;
+    local->cont.fsetattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fsetattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fsetattr_unwind;
+    local->transaction.wind = afr_fsetattr_wind;
+    local->transaction.unwind = afr_fsetattr_unwind;
 
-        local->fd                 = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FSETATTR;
+    local->op = GF_FOP_FSETATTR;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* {{{ setxattr */
 
-
 int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (setxattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(setxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_setxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_setxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->setxattr,
-			   &local->loc, local->cont.setxattr.dict,
-			   local->cont.setxattr.flags, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_setxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setxattr, &local->loc,
+                      local->cont.setxattr.dict, local->cont.setxattr.flags,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_emptyb_set_pending_changelog_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int op_ret, int op_errno,
-                                  dict_t *xattr, dict_t *xdata)
+afr_emptyb_set_pending_changelog_cbk(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int op_ret, int op_errno,
+                                     dict_t *xattr, dict_t *xdata)
 
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i, ret = 0;
-        char *op_type = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        i = (long) cookie;
-
-        local->replies[i].valid = 1;
-        local->replies[i].op_ret = op_ret;
-        local->replies[i].op_errno = op_errno;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i, ret = 0;
+    char *op_type = NULL;
 
-        ret = dict_get_str (local->xdata_req, "replicate-brick-op", &op_type);
-        if (ret)
-                goto out;
+    local = frame->local;
+    priv = this->private;
+    i = (long)cookie;
 
-        gf_msg (this->name, op_ret ? GF_LOG_ERROR : GF_LOG_INFO,
-                op_ret ? op_errno : 0,
-                afr_get_msg_id (op_type),
-                "Set of pending xattr %s on"
-                " %s.", op_ret ? "failed" : "succeeded",
-                priv->children[i]->name);
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-out:
-        syncbarrier_wake (&local->barrier);
-        return 0;
-}
+    ret = dict_get_str_sizen(local->xdata_req, "replicate-brick-op", &op_type);
+    if (ret)
+        goto out;
 
-int
-afr_emptyb_set_pending_changelog (call_frame_t *frame, xlator_t *this,
-                                unsigned char *locked_nodes)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int ret = 0, i = 0;
+    gf_smsg(this->name, op_ret ? GF_LOG_ERROR : GF_LOG_INFO,
+            op_ret ? op_errno : 0, AFR_MSG_SET_PEND_XATTR, "name=%s",
+            priv->children[i]->name, "op_ret=%s",
+            op_ret ? "failed" : "succeeded", NULL);
 
-        local = frame->local;
-        priv = this->private;
-
-        AFR_ONLIST (locked_nodes, frame, afr_emptyb_set_pending_changelog_cbk,
-                    xattrop, &local->loc, GF_XATTROP_ADD_ARRAY,
-                    local->xattr_req, NULL);
-
-        /* It is sufficient if xattrop was successful on one child */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
-
-                if (local->replies[i].op_ret == 0) {
-                        ret = 0;
-                        goto out;
-                } else {
-                        ret = afr_higher_errno (ret,
-                                                local->replies[i].op_errno);
-                }
-        }
 out:
-        return -ret;
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
 int
-_afr_handle_empty_brick_type (xlator_t *this, call_frame_t *frame,
-                            loc_t *loc, int empty_index,
-                            afr_transaction_type type,
-                            char *op_type)
+afr_emptyb_set_pending_changelog(call_frame_t *frame, xlator_t *this,
+                                 unsigned char *locked_nodes)
 {
-        int              count            = 0;
-        int              ret              = -ENOMEM;
-        int              idx              = -1;
-        int              d_idx            = -1;
-        unsigned char   *locked_nodes     = NULL;
-        afr_local_t     *local            = NULL;
-        afr_private_t   *priv             = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = 0, i = 0;
 
-        priv = this->private;
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        locked_nodes = alloca0 (priv->child_count);
+    AFR_ONLIST(locked_nodes, frame, afr_emptyb_set_pending_changelog_cbk,
+               xattrop, &local->loc, GF_XATTROP_ADD_ARRAY, local->xattr_req,
+               NULL);
 
-        idx = afr_index_for_transaction_type (type);
-        d_idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+    /* It is sufficient if xattrop was successful on one child */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        local->pending = afr_matrix_create (priv->child_count,
-                                            AFR_NUM_CHANGE_LOGS);
-        if (!local->pending)
-                goto out;
-
-        local->pending[empty_index][idx] = hton32 (1);
-
-        if ((priv->esh_granular) && (type == AFR_ENTRY_TRANSACTION))
-                        local->pending[empty_index][d_idx] = hton32 (1);
-
-        local->xdata_req = dict_new ();
-        if (!local->xdata_req)
-                goto out;
-
-        ret = dict_set_str (local->xdata_req, "replicate-brick-op", op_type);
-        if (ret)
-                goto out;
-
-        local->xattr_req = dict_new ();
-        if (!local->xattr_req)
-                goto out;
-
-        ret = afr_set_pending_dict (priv, local->xattr_req, local->pending);
-        if (ret < 0)
-                goto out;
-
-        if (AFR_ENTRY_TRANSACTION == type) {
-                count = afr_selfheal_entrylk (frame, this, loc->inode,
-                                              this->name, NULL, locked_nodes);
+        if (local->replies[i].op_ret == 0) {
+            ret = 0;
+            goto out;
         } else {
-                count = afr_selfheal_inodelk (frame, this, loc->inode,
-                                              this->name, LLONG_MAX - 1, 0,
-                                              locked_nodes);
-        }
-
-        if (!count) {
-                gf_msg (this->name, GF_LOG_ERROR, EAGAIN,
-                        AFR_MSG_REPLACE_BRICK_STATUS, "Couldn't acquire lock on"
-                        " any child.");
-                ret = -EAGAIN;
-                goto unlock;
+            ret = afr_higher_errno(ret, local->replies[i].op_errno);
         }
+    }
+out:
+    return -ret;
+}
 
-        ret = afr_emptyb_set_pending_changelog (frame, this, locked_nodes);
-        if (ret)
-                goto unlock;
-        ret = 0;
+static int
+_afr_handle_empty_brick_type(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                             int empty_index, afr_transaction_type type,
+                             char *op_type, const int op_type_len)
+{
+    int count = 0;
+    int ret = -ENOMEM;
+    int idx = -1;
+    int d_idx = -1;
+    unsigned char *locked_nodes = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    locked_nodes = alloca0(priv->child_count);
+
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    local->pending[empty_index][idx] = hton32(1);
+
+    if ((priv->esh_granular) && (type == AFR_ENTRY_TRANSACTION))
+        local->pending[empty_index][d_idx] = hton32(1);
+
+    local->xdata_req = dict_new();
+    if (!local->xdata_req)
+        goto out;
+
+    ret = dict_set_nstrn(local->xdata_req, "replicate-brick-op",
+                         SLEN("replicate-brick-op"), op_type, op_type_len);
+    if (ret)
+        goto out;
+
+    local->xattr_req = dict_new();
+    if (!local->xattr_req)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, local->xattr_req, local->pending);
+    if (ret < 0)
+        goto out;
+
+    if (AFR_ENTRY_TRANSACTION == type) {
+        count = afr_selfheal_entrylk(frame, this, loc->inode, this->name, NULL,
+                                     locked_nodes);
+    } else {
+        count = afr_selfheal_inodelk(frame, this, loc->inode, this->name,
+                                     LLONG_MAX - 1, 0, locked_nodes);
+    }
+
+    if (!count) {
+        gf_smsg(this->name, GF_LOG_ERROR, EAGAIN, AFR_MSG_REPLACE_BRICK_STATUS,
+                NULL);
+        ret = -EAGAIN;
+        goto unlock;
+    }
+
+    ret = afr_emptyb_set_pending_changelog(frame, this, locked_nodes);
+    if (ret)
+        goto unlock;
+    ret = 0;
 unlock:
-        if (AFR_ENTRY_TRANSACTION == type) {
-                afr_selfheal_unentrylk (frame, this, loc->inode, this->name,
-                                        NULL, locked_nodes, NULL);
-        } else {
-                afr_selfheal_uninodelk (frame, this, loc->inode, this->name,
-                                        LLONG_MAX - 1, 0, locked_nodes);
-        }
+    if (AFR_ENTRY_TRANSACTION == type) {
+        afr_selfheal_unentrylk(frame, this, loc->inode, this->name, NULL,
+                               locked_nodes, NULL);
+    } else {
+        afr_selfheal_uninodelk(frame, this, loc->inode, this->name,
+                               LLONG_MAX - 1, 0, locked_nodes);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 void
-afr_brick_args_cleanup (void *opaque)
-{
-        afr_empty_brick_args_t *data = NULL;
-
-        data = opaque;
-        loc_wipe (&data->loc);
-        GF_FREE (data);
-}
-
-int
-_afr_handle_empty_brick_cbk (int ret, call_frame_t *frame, void *opaque)
-{
-        afr_brick_args_cleanup (opaque);
-        return 0;
-}
-
-int
-_afr_handle_empty_brick (void *opaque)
-{
-
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int              empty_index       = -1;
-        int              ret            = -1;
-        int              op_errno       = ENOMEM;
-        call_frame_t    *frame          = NULL;
-        xlator_t        *this           = NULL;
-        char            *op_type        = NULL;
-        afr_empty_brick_args_t *data  = NULL;
-
-        data = opaque;
-        frame = data->frame;
-        empty_index = data->empty_index;
-        op_type = data->op_type;
-        this = frame->this;
-        priv = this->private;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
-
-        loc_copy (&local->loc, &data->loc);
-
-        gf_msg_debug (this->name, 0, "New brick is : %s",
-                      priv->children[empty_index]->name);
-
-        ret = _afr_handle_empty_brick_type (this, frame, &local->loc, empty_index,
-                                          AFR_METADATA_TRANSACTION, op_type);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        dict_unref (local->xdata_req);
-        dict_unref (local->xattr_req);
-        afr_matrix_cleanup (local->pending, priv->child_count);
-        local->pending = NULL;
-        local->xattr_req = NULL;
-        local->xdata_req = NULL;
-
-        ret = _afr_handle_empty_brick_type (this, frame, &local->loc, empty_index,
-                                          AFR_ENTRY_TRANSACTION, op_type);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+afr_brick_args_cleanup(void *opaque)
+{
+    afr_empty_brick_args_t *data = NULL;
+
+    data = opaque;
+    loc_wipe(&data->loc);
+    GF_FREE(data);
+}
+
+int
+_afr_handle_empty_brick_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    afr_brick_args_cleanup(opaque);
+    return 0;
+}
+
+int
+_afr_handle_empty_brick(void *opaque)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int empty_index = -1;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *op_type = NULL;
+    int op_type_len = 0;
+    afr_empty_brick_args_t *data = NULL;
+    call_frame_t *op_frame = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    empty_index = data->empty_index;
+    if (!data->op_type)
+        goto out;
+
+    op_frame = copy_frame(frame);
+    if (!op_frame) {
+        ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_type = data->op_type;
+    op_type_len = strlen(op_type);
+    this = op_frame->this;
+    priv = this->private;
+
+    afr_set_lk_owner(op_frame, this, op_frame->root);
+    local = AFR_FRAME_INIT(op_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, &data->loc);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, AFR_MSG_NEW_BRICK, "name=%s",
+            priv->children[empty_index]->name, NULL);
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_METADATA_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dict_unref(local->xdata_req);
+    dict_unref(local->xattr_req);
+    afr_matrix_cleanup(local->pending, priv->child_count);
+    local->pending = NULL;
+    local->xattr_req = NULL;
+    local->xdata_req = NULL;
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_ENTRY_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        return 0;
+    if (op_frame) {
+        AFR_STACK_DESTROY(op_frame);
+    }
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
+}
+
+int
+afr_split_brain_resolve_do(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           char *data)
+{
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    local->xdata_req = dict_new();
+
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(local->xdata_req, "heal-op",
+                               GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str_sizen(local->xdata_req, "child-name", data);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    /* set spb choice to -1 whether heal succeeds or not:
+     * If heal succeeds : spb-choice should be set to -1 as
+     *                    it is no longer valid; file is not
+     *                    in split-brain anymore.
+     * If heal doesn't succeed:
+     *                    spb-choice should be set to -1
+     *                    otherwise reads will be served
+     *                    from spb-choice which is misleading.
+     */
+    ret = afr_inode_split_brain_choice_set(loc->inode, this, -1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+                NULL);
+    afr_heal_splitbrain_file(frame, this, loc);
+    ret = 0;
+out:
+    if (ret < 0)
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-afr_split_brain_resolve_do (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            char *data)
+afr_get_split_brain_child_index(xlator_t *this, void *value, size_t len)
 {
-        afr_local_t    *local             = NULL;
-        int     ret                       = -1;
-        int     op_errno                  = EINVAL;
+    int spb_child_index = -1;
+    char *spb_child_str = NULL;
 
-        local = frame->local;
-        local->xdata_req = dict_new ();
+    spb_child_str = alloca0(len + 1);
+    memcpy(spb_child_str, value, len);
 
-        if (!local->xdata_req) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (!strcmp(spb_child_str, "none"))
+        return -2;
 
-        ret = dict_set_int32 (local->xdata_req, "heal-op",
-                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_str (local->xdata_req, "child-name", data);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-        /* set spb choice to -1 whether heal succeeds or not:
-         * If heal succeeds : spb-choice should be set to -1 as
-         *                    it is no longer valid; file is not
-         *                    in split-brain anymore.
-         * If heal doesn't succeed:
-         *                    spb-choice should be set to -1
-         *                    otherwise reads will be served
-         *                    from spb-choice which is misleading.
-         */
-        ret = afr_inode_split_brain_choice_set (loc->inode, this, -1);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, "Failed to set"
-                        "split-brain choice to -1");
-        afr_heal_splitbrain_file (frame, this, loc);
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-        return 0;
+    spb_child_index = afr_get_child_index_from_name(this, spb_child_str);
+    if (spb_child_index < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                "subvol=%s", spb_child_str, NULL);
+    }
+    return spb_child_index;
 }
 
 int
-afr_get_split_brain_child_index (xlator_t *this, void *value, size_t len)
+afr_can_set_split_brain_choice(void *opaque)
 {
-        int             spb_child_index   = -1;
-        char           *spb_child_str     = NULL;
+    afr_spbc_timeout_t *data = opaque;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    int ret = -1;
 
-        spb_child_str =  alloca0 (len + 1);
-        memcpy (spb_child_str, value, len);
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
 
-        if (!strcmp (spb_child_str, "none"))
-                return -2;
+    ret = afr_is_split_brain(frame, this, loc->inode, loc->gfid, &data->d_spb,
+                             &data->m_spb);
 
-        spb_child_index = afr_get_child_index_from_name (this,
-                                                         spb_child_str);
-        if (spb_child_index < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_SUBVOL, "Invalid subvol: %s",
-                        spb_child_str);
-        }
-        return spb_child_index;
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, "gfid=%s",
+                uuid_utoa(loc->gfid), NULL);
+    return ret;
 }
 
 int
-afr_can_set_split_brain_choice (void *opaque)
+afr_handle_split_brain_commands(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                                dict_t *dict)
 {
-        afr_spbc_timeout_t        *data         = opaque;
-        call_frame_t              *frame        = NULL;
-        xlator_t                  *this         = NULL;
-        loc_t                     *loc          = NULL;
-        int                        ret          = -1;
+    void *choice_value = NULL;
+    void *resolve_value = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_spbc_timeout_t *data = NULL;
+    int len = 0;
+    int spb_child_index = -1;
+    int ret = -1;
+    int op_errno = EINVAL;
 
-        frame = data->frame;
-        loc = data->loc;
-        this = frame->this;
+    priv = this->private;
 
-        ret = afr_is_split_brain (frame, this, loc->inode, loc->gfid,
-                                  &data->d_spb, &data->m_spb);
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_CHOICE, &choice_value, &len);
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_RESOLVE, &resolve_value,
+                               &len);
+    if (!choice_value && !resolve_value) {
+        ret = -1;
+        goto out;
+    }
 
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                        "Failed to determine if %s"
-                        " is in split-brain. "
-                        "Aborting split-brain-choice set.",
-                        uuid_utoa (loc->gfid));
-        return ret;
-}
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        ret = 1;
+        goto out;
+    }
 
-int
-afr_handle_split_brain_commands (xlator_t *this, call_frame_t *frame,
-                                loc_t *loc, dict_t *dict)
-{
-        void           *value             = NULL;
-        afr_private_t  *priv              = NULL;
-        afr_local_t    *local             = NULL;
-        afr_spbc_timeout_t *data          = NULL;
-        int             len               = 0;
-        int             spb_child_index   = -1;
-        int             ret               = -1;
-        int             op_errno          = EINVAL;
-
-        priv = this->private;
+    local->op = GF_FOP_SETXATTR;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local) {
+    if (choice_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, choice_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            /* Case where value was "none" */
+            if (spb_child_index == -2)
+                spb_child_index = -1;
+            else {
                 ret = 1;
+                op_errno = EINVAL;
                 goto out;
+            }
         }
 
-        local->op = GF_FOP_SETXATTR;
-
-        ret =  dict_get_ptr_and_len (dict, GF_AFR_SBRAIN_CHOICE, &value,
-                                     &len);
-        if (value) {
-                spb_child_index = afr_get_split_brain_child_index (this, value,
-                                                                   len);
-                if (spb_child_index < 0) {
-                        /* Case where value was "none" */
-                        if (spb_child_index == -2)
-                                spb_child_index = -1;
-                        else {
-                                ret = 1;
-                                op_errno = EINVAL;
-                                goto out;
-                        }
-                }
-
-                data = GF_CALLOC (1, sizeof (*data), gf_afr_mt_spbc_timeout_t);
-                if (!data) {
-                        ret = 1;
-                        goto out;
-                }
-                data->spb_child_index = spb_child_index;
-                data->frame = frame;
-                data->loc = loc;
-                ret = synctask_new (this->ctx->env,
-                                    afr_can_set_split_brain_choice,
-                                    afr_set_split_brain_choice, NULL, data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                                "Failed to create"
-                                " synctask. Aborting split-brain choice set"
-                                " for %s", loc->name);
-                        ret = 1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-                ret = 0;
-                goto out;
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spbc_timeout_t);
+        if (!data) {
+            ret = 1;
+            goto out;
         }
+        data->spb_child_index = spb_child_index;
+        data->frame = frame;
+        loc_copy(&local->loc, loc);
+        data->loc = &local->loc;
+        ret = synctask_new(this->ctx->env, afr_can_set_split_brain_choice,
+                           afr_set_split_brain_choice, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    "name=%s", loc->name, NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_ptr_and_len (dict, GF_AFR_SBRAIN_RESOLVE, &value, &len);
-        if (value) {
-                spb_child_index = afr_get_split_brain_child_index (this, value,
-                                                                   len);
-                if (spb_child_index < 0) {
-                        ret = 1;
-                        goto out;
-                }
-
-                afr_split_brain_resolve_do (frame, this, loc,
-                                            priv->children[spb_child_index]->name);
-                ret = 0;
+    if (resolve_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, resolve_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            ret = 1;
+            goto out;
         }
+
+        afr_split_brain_resolve_do(frame, this, loc,
+                                   priv->children[spb_child_index]->name);
+        ret = 0;
+    }
 out:
-        /* key was correct but value was invalid when ret == 1 */
-        if (ret == 1) {
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-                if (data)
-                        GF_FREE (data);
-                ret = 0;
-        }
-        return ret;
+    /* key was correct but value was invalid when ret == 1 */
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
 int
-afr_handle_spb_choice_timeout (xlator_t *this, call_frame_t *frame,
-                               dict_t *dict)
+afr_handle_spb_choice_timeout(xlator_t *this, call_frame_t *frame, dict_t *dict)
 {
-        int             ret               = -1;
-        int             op_errno          = 0;
-        uint64_t        timeout           = 0;
-        afr_private_t  *priv              = NULL;
+    int ret = -1;
+    int op_errno = 0;
+    uint64_t timeout = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        ret = dict_get_uint64 (dict, GF_AFR_SPB_CHOICE_TIMEOUT, &timeout);
-        if (!ret) {
-                priv->spb_choice_timeout = timeout * 60;
-                AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        }
+    ret = dict_get_uint64(dict, GF_AFR_SPB_CHOICE_TIMEOUT, &timeout);
+    if (!ret) {
+        priv->spb_choice_timeout = timeout * 60;
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-afr_handle_empty_brick (xlator_t *this, call_frame_t *frame, loc_t *loc,
-                        dict_t *dict)
+afr_handle_empty_brick(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                       dict_t *dict)
 {
-        int             ret               = -1;
-        int             ab_ret            = -1;
-        int             empty_index        = -1;
-        int             op_errno          = EPERM;
-        char           *empty_brick         = NULL;
-        char           *op_type           = NULL;
-        afr_empty_brick_args_t *data        = NULL;
+    int ret = -1;
+    int ab_ret = -1;
+    int empty_index = -1;
+    int op_errno = EPERM;
+    char *empty_brick = NULL;
+    char *op_type = NULL;
+    afr_empty_brick_args_t *data = NULL;
 
-        ret =  dict_get_str (dict, GF_AFR_REPLACE_BRICK, &empty_brick);
-        if (!ret)
-                op_type = GF_AFR_REPLACE_BRICK;
+    ret = dict_get_str_sizen(dict, GF_AFR_REPLACE_BRICK, &empty_brick);
+    if (!ret)
+        op_type = GF_AFR_REPLACE_BRICK;
 
-        ab_ret = dict_get_str (dict, GF_AFR_ADD_BRICK, &empty_brick);
-        if (!ab_ret)
-                op_type = GF_AFR_ADD_BRICK;
+    ab_ret = dict_get_str_sizen(dict, GF_AFR_ADD_BRICK, &empty_brick);
+    if (!ab_ret)
+        op_type = GF_AFR_ADD_BRICK;
 
-        if (ret && ab_ret)
-                goto out;
+    if (ret && ab_ret)
+        goto out;
 
-        if (frame->root->pid != GF_CLIENT_PID_SELF_HEALD) {
-                gf_msg (this->name, GF_LOG_ERROR, EPERM,
-                        afr_get_msg_id (op_type),
-                        "'%s' is an internal extended attribute.",
-                        op_type);
-                ret = 1;
-                goto out;
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT) {
+        gf_smsg(this->name, GF_LOG_ERROR, EPERM, AFR_MSG_INTERNAL_ATTR,
+                "op_type=%s", op_type, NULL);
+        ret = 1;
+        goto out;
+    }
+    empty_index = afr_get_child_index_from_name(this, empty_brick);
+
+    if (empty_index < 0) {
+        /* Didn't belong to this replica pair
+         * Just do a no-op
+         */
+        AFR_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return 0;
+    } else {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_empty_brick_t);
+        if (!data) {
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
         }
-        empty_index = afr_get_child_index_from_name (this, empty_brick);
-
-        if (empty_index < 0) {
-                 /* Didn't belong to this replica pair
-                  * Just do a no-op
-                  */
-                AFR_STACK_UNWIND (setxattr, frame, 0, 0, NULL);
-                return 0;
-        } else {
-                data = GF_CALLOC (1, sizeof (*data),
-                                  gf_afr_mt_empty_brick_t);
-                if (!data) {
-                        ret = 1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-                data->frame = frame;
-                loc_copy (&data->loc, loc);
-                data->empty_index = empty_index;
-                data->op_type = op_type;
-                ret = synctask_new (this->ctx->env,
-                                    _afr_handle_empty_brick,
-                                    _afr_handle_empty_brick_cbk,
-                                    NULL, data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                afr_get_msg_id (op_type),
-                                "Failed to create synctask.");
-                        ret = 1;
-                        op_errno = ENOMEM;
-                        afr_brick_args_cleanup (data);
-                        goto out;
-                }
+        data->frame = frame;
+        loc_copy(&data->loc, loc);
+        data->empty_index = empty_index;
+        data->op_type = op_type;
+        ret = synctask_new(this->ctx->env, _afr_handle_empty_brick,
+                           _afr_handle_empty_brick_cbk, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            afr_brick_args_cleanup(data);
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 out:
-        if (ret == 1) {
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-                ret = 0;
-        }
-        return ret;
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        ret = 0;
+    }
+    return ret;
 }
 
 static int
-afr_handle_special_xattr (xlator_t *this, call_frame_t *frame, loc_t *loc,
-                          dict_t *dict)
+afr_handle_special_xattr(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                         dict_t *dict)
 {
-        int     ret     = -1;
+    int ret = -1;
 
-        ret = afr_handle_split_brain_commands (this, frame, loc, dict);
-        if (ret == 0)
-                goto out;
+    ret = afr_handle_split_brain_commands(this, frame, loc, dict);
+    if (ret == 0)
+        goto out;
 
-        ret = afr_handle_spb_choice_timeout (this, frame, dict);
-        if (ret == 0)
-                goto out;
+    ret = afr_handle_spb_choice_timeout(this, frame, dict);
+    if (ret == 0)
+        goto out;
 
-        /* Applicable for replace-brick and add-brick commands */
-        ret = afr_handle_empty_brick (this, frame, loc, dict);
+    /* Applicable for replace-brick and add-brick commands */
+    ret = afr_handle_empty_brick(this, frame, loc, dict);
 out:
-        return ret;
+    return ret;
 }
 
 int
-afr_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-	      int32_t flags, dict_t *xdata)
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        afr_local_t    *local             = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret               = -1;
-        int             op_errno          = EINVAL;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        ret = afr_handle_special_xattr (this, frame, loc, dict);
-        if (ret == 0)
-                return 0;
+    ret = afr_handle_special_xattr(this, frame, loc, dict);
+    if (ret == 0)
+        return 0;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.setxattr.dict  = dict_ref (dict);
-        local->cont.setxattr.flags = flags;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.setxattr.dict = dict_ref(dict);
+    local->cont.setxattr.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_setxattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_setxattr_unwind;
+    local->transaction.wind = afr_setxattr_wind;
+    local->transaction.unwind = afr_setxattr_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-	local->op = GF_FOP_SETXATTR;
+    local->op = GF_FOP_SETXATTR;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* {{{ fsetxattr */
 
-
 int
-afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t    *local         = NULL;
-        call_frame_t   *main_frame    = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fsetxattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fsetxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_fsetxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fsetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t        *local       = NULL;
-        afr_private_t      *priv        = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fsetxattr,
-			   local->fd, local->cont.fsetxattr.dict,
-			   local->cont.fsetxattr.flags, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fsetxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetxattr, local->fd,
+                      local->cont.fsetxattr.dict, local->cont.fsetxattr.flags,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
 {
-        afr_local_t      *local             = NULL;
-        call_frame_t     *transaction_frame = NULL;
-        int               ret               = -1;
-        int               op_errno          = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fsetxattr.dict  = dict_ref (dict);
-        local->cont.fsetxattr.flags = flags;
+    local->cont.fsetxattr.dict = dict_ref(dict);
+    local->cont.fsetxattr.flags = flags;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fsetxattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fsetxattr_unwind;
+    local->transaction.wind = afr_fsetxattr_wind;
+    local->transaction.unwind = afr_fsetxattr_unwind;
 
-        local->fd                 = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FSETXATTR;
+    local->op = GF_FOP_FSETXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start  = LLONG_MAX - 1;
-        local->transaction.len    = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 /* }}} */
 
-
 /* {{{ removexattr */
 
-
 int
-afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_removexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (removexattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(removexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_removexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_removexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->removexattr,
-			   &local->loc, local->cont.removexattr.name,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_removexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->removexattr, &local->loc,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        afr_local_t     *local             = NULL;
-        call_frame_t    *transaction_frame = NULL;
-        int              ret               = -1;
-        int              op_errno          = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.removexattr.name = gf_strdup (name);
+    local->cont.removexattr.name = gf_strdup(name);
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_removexattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_removexattr_unwind;
+    local->transaction.wind = afr_removexattr_wind;
+    local->transaction.unwind = afr_removexattr_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_REMOVEXATTR;
+    local->op = GF_FOP_REMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 /* ffremovexattr */
 int
-afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fremovexattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fremovexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_fremovexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fremovexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fremovexattr,
-			   local->fd, local->cont.removexattr.name,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fremovexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fremovexattr, local->fd,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  const char *name, dict_t *xdata)
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.removexattr.name = gf_strdup (name);
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.removexattr.name = gf_strdup(name);
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fremovexattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fremovexattr_unwind;
+    local->transaction.wind = afr_fremovexattr_wind;
+    local->transaction.unwind = afr_fremovexattr_unwind;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FREMOVEXATTR;
+    local->op = GF_FOP_FREMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fallocate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_fallocate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_fallocate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fallocate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fallocate,
-			   local->fd, local->cont.fallocate.mode,
-			   local->cont.fallocate.offset,
-			   local->cont.fallocate.len, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fallocate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fallocate, local->fd,
+                      local->cont.fallocate.mode, local->cont.fallocate.offset,
+                      local->cont.fallocate.len, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-               off_t offset, size_t len, dict_t *xdata)
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
 {
-        call_frame_t *transaction_frame = NULL;
-        afr_local_t *local = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fallocate.mode = mode;
-        local->cont.fallocate.offset  = offset;
-        local->cont.fallocate.len = len;
+    local->cont.fallocate.mode = mode;
+    local->cont.fallocate.offset = offset;
+    local->cont.fallocate.len = len;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_FALLOCATE;
+    local->op = GF_FOP_FALLOCATE;
 
-        local->transaction.wind   = afr_fallocate_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fallocate_unwind;
+    local->transaction.wind = afr_fallocate_wind;
+    local->transaction.unwind = afr_fallocate_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.fallocate.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.fallocate.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ discard */
 
 int
-afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+afr_discard_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+afr_discard_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_discard_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_discard_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_discard_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->discard,
-			   local->fd, local->cont.discard.offset,
-			   local->cont.discard.len, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_discard_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->discard, local->fd,
+                      local->cont.discard.offset, local->cont.discard.len,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             size_t len, dict_t *xdata)
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.discard.offset  = offset;
-        local->cont.discard.len = len;
+    local->cont.discard.offset = offset;
+    local->cont.discard.len = len;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_DISCARD;
+    local->op = GF_FOP_DISCARD;
 
-        local->transaction.wind   = afr_discard_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_discard_unwind;
+    local->transaction.wind = afr_discard_wind;
+    local->transaction.unwind = afr_discard_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.discard.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.discard.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* {{{ zerofill */
 
 int
-afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
+afr_zerofill_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                       struct iatt *postbuf, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_zerofill_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_zerofill_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->zerofill,
-			   local->fd, local->cont.zerofill.offset,
-			   local->cont.zerofill.len, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_zerofill_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->zerofill, local->fd,
+                      local->cont.zerofill.offset, local->cont.zerofill.len,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              size_t len, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.zerofill.offset  = offset;
-        local->cont.zerofill.len = len;
+    local->cont.zerofill.offset = offset;
+    local->cont.zerofill.len = len;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_ZEROFILL;
+    local->op = GF_FOP_ZEROFILL;
 
-        local->transaction.wind   = afr_zerofill_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_zerofill_unwind;
+    local->transaction.wind = afr_zerofill_wind;
+    local->transaction.unwind = afr_zerofill_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.discard.offset;
-        local->transaction.len     = len;
+    local->transaction.start = local->cont.zerofill.offset;
+    local->transaction.len = len;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 int32_t
-afr_xattrop_wind_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int32_t op_ret, int32_t op_errno,
-                      dict_t *xattr, dict_t *xdata)
+afr_xattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xattr, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
 }
 
 int
-afr_xattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_xattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_xattrop_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->xattrop,
-			   &local->loc, local->cont.xattrop.optype,
-			   local->cont.xattrop.xattr, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_xattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->xattrop, &local->loc,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_xattrop_unwind (call_frame_t *frame, xlator_t *this)
+afr_xattrop_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (xattrop, main_frame, local->op_ret, local->op_errno,
-			  local->xattr_rsp, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(xattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
 }
 
 int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.xattrop.xattr = dict_ref (xattr);
-        local->cont.xattrop.optype = optype;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        local->transaction.wind   = afr_xattrop_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_xattrop_unwind;
+    local->transaction.wind = afr_xattrop_wind;
+    local->transaction.unwind = afr_xattrop_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_XATTROP;
+    local->op = GF_FOP_XATTROP;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int32_t
-afr_fxattrop_wind_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr, dict_t *xdata)
+afr_fxattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xattr, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
 }
 
 int
-afr_fxattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fxattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fxattrop_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fxattrop,
-			   local->fd, local->cont.xattrop.optype,
-			   local->cont.xattrop.xattr, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fxattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fxattrop, local->fd,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_fxattrop_unwind (call_frame_t *frame, xlator_t *this)
+afr_fxattrop_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fxattrop, main_frame, local->op_ret, local->op_errno,
-			  local->xattr_rsp, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(fxattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
 }
 
 int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.xattrop.xattr = dict_ref (xattr);
-        local->cont.xattrop.optype = optype;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        local->transaction.wind   = afr_fxattrop_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fxattrop_unwind;
+    local->transaction.wind = afr_fxattrop_wind;
+    local->transaction.unwind = afr_fxattrop_unwind;
 
-	local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FXATTROP;
+    local->op = GF_FOP_FXATTROP;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this,
-                               AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+int
+afr_fsync_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
+
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(fsync, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+
+    return 0;
+}
+
+int
+afr_fsync_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
+
+int
+afr_fsync_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fsync_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsync, local->fd,
+                      local->cont.fsync.datasync, local->xdata_req);
+    return 0;
+}
+
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int32_t op_errno = ENOMEM;
+    int8_t last_fsync = 0;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    if (xdata) {
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+        if (dict_get_int8(xdata, "last-fsync", &last_fsync) == 0) {
+            if (last_fsync) {
+                local->transaction.disable_delayed_post_op = _gf_true;
+            }
+        }
+    } else {
+        local->xdata_req = dict_new();
+    }
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_FSYNC;
+    local->cont.fsync.datasync = datasync;
+
+    if (afr_fd_has_witnessed_unstable_write(this, fd->inode)) {
+        /* don't care. we only wanted to CLEAR the bit */
+    }
+
+    local->transaction.wind = afr_fsync_wind;
+    local->transaction.unwind = afr_fsync_unwind;
+
+    local->transaction.main_frame = frame;
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index e174cc2d610..a787069b7a1 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -12,79 +12,83 @@
 #define __INODE_WRITE_H__
 
 int32_t
-afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *xdata);
+afr_chmod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dict_t *xdata);
 
 int32_t
-afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
+afr_chown(call_frame_t *frame, xlator_t *this, loc_t *loc, uid_t uid, gid_t gid,
+          dict_t *xdata);
 
 int
-afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
+afr_fchown(call_frame_t *frame, xlator_t *this, fd_t *fd, uid_t uid, gid_t gid,
+           dict_t *xdata);
 
 int32_t
-afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode, dict_t *xdata);
+afr_fchmod(call_frame_t *frame, xlator_t *this, fd_t *fd, mode_t mode,
+           dict_t *xdata);
 
 int32_t
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	    struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata);
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
 
 int32_t
-afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset, dict_t *xdata);
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
 
 int32_t
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset, dict_t *xdata);
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
 
 int32_t
-afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
+afr_utimens(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct timespec tv[2], dict_t *xdata);
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata);
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata);
 
 int32_t
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
 
 int32_t
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
 
 int32_t
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name, dict_t *xdata);
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
 
 int32_t
-afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name, dict_t *xdata);
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
 
 int
-afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             size_t len, dict_t *xdata);
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
 
 int
-afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-               off_t offset, size_t len, dict_t *xdata);
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
 
 int
 afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              off_t len, dict_t *xdata);
 
 int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
 
 int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index c2a5f526c08..bc8eabe0f43 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -8,9 +8,9 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
@@ -18,1747 +18,774 @@
 
 #include <signal.h>
 
-
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
-#define LOCKED_LOWER    0x2        /* for lower path */
-
-#define AFR_TRACE_INODELK_IN(frame, this, params ...)           \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->inodelk_trace)                      \
-                        break;                                  \
-                afr_trace_inodelk_in (frame, this, params);     \
-        } while (0);
-
-#define AFR_TRACE_INODELK_OUT(frame, this, params ...)          \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->inodelk_trace)                      \
-                        break;                                  \
-                afr_trace_inodelk_out (frame, this, params);    \
-        } while (0);
-
-#define AFR_TRACE_ENTRYLK_IN(frame, this, params ...)           \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->entrylk_trace)                      \
-                        break;                                  \
-                afr_trace_entrylk_in (frame, this, params);     \
-        } while (0);
-
-#define AFR_TRACE_ENTRYLK_OUT(frame, this, params ...)          \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->entrylk_trace)                      \
-                        break;                                  \
-                afr_trace_entrylk_out (frame, this, params);    \
-        } while (0);
-
-int
-afr_entry_lockee_cmp (const void *l1, const void *l2)
-{
-        const afr_entry_lockee_t       *r1 = l1;
-        const afr_entry_lockee_t       *r2 = l2;
-        int                            ret = 0;
-        uuid_t                         gfid1 = {0};
-        uuid_t                         gfid2 = {0};
-
-        loc_gfid ((loc_t*)&r1->loc, gfid1);
-        loc_gfid ((loc_t*)&r2->loc, gfid2);
-        ret = gf_uuid_compare (gfid1, gfid2);
-        /*Entrylks with NULL basename are the 'smallest'*/
-        if (ret == 0) {
-                if (!r1->basename)
-                        return -1;
-                if (!r2->basename)
-                        return 1;
-                ret = strcmp (r1->basename, r2->basename);
-        }
-
-        if (ret <= 0)
-                return -1;
-        else
-                return 1;
-}
-
-int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
-
-static int
-afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
-
-static uint64_t afr_lock_number = 1;
-
-static uint64_t
-get_afr_lock_number ()
-{
-        return (++afr_lock_number);
-}
-
-int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_number = get_afr_lock_number ();
-
-        return 0;
-}
+#define LOCKED_NO 0x0    /* no lock held */
+#define LOCKED_YES 0x1   /* for DATA, METADATA, ENTRY and higher_path */
+#define LOCKED_LOWER 0x2 /* for lower path */
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
-{
-        gf_msg_trace (this->name, 0,
-                      "Setting lk-owner=%llu",
-                      (unsigned long long) (unsigned long)lk_owner);
-
-        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
-}
-
-static int
-is_afr_lock_selfheal (afr_local_t *local)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        int                  ret      = -1;
-
-        int_lock = &local->internal_lock;
-
-        switch (int_lock->selfheal_lk_type) {
-        case AFR_DATA_SELF_HEAL_LK:
-        case AFR_METADATA_SELF_HEAL_LK:
-                ret = 1;
-                break;
-        case AFR_ENTRY_SELF_HEAL_LK:
-                ret = 0;
-                break;
-        }
-
-        return ret;
-
-}
-
-int32_t
-internal_lock_count (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t call_count = 0;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i])
-                        ++call_count;
-        }
-
-        return call_count;
-}
-
-static void
-afr_print_inodelk (char *str, int size, int cmd,
-                   struct gf_flock *flock, gf_lkowner_t *owner)
+afr_lockee_cleanup(afr_lockee_t *lockee)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
-
-        switch (cmd) {
-#if F_GETLK != F_GETLK64
-        case F_GETLK64:
-#endif
-        case F_GETLK:
-                cmd_str = "GETLK";
-                break;
-
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-        case F_SETLK:
-                cmd_str = "SETLK";
-                break;
-
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-        case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
-
-        default:
-                cmd_str = "<null>";
-                break;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                type_str = "READ";
-                break;
-        case F_WRLCK:
-                type_str = "WRITE";
-                break;
-        case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
-        default:
-                type_str = "UNKNOWN";
-                break;
-        }
+    if (lockee->fd) {
+        fd_unref(lockee->fd);
+        lockee->fd = NULL;
+    } else {
+        loc_wipe(&lockee->loc);
+    }
 
-        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                  cmd_str, type_str, (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid,
-                  lkowner_utoa (owner));
+    GF_FREE(lockee->basename);
+    lockee->basename = NULL;
+    GF_FREE(lockee->locked_nodes);
+    lockee->locked_nodes = NULL;
 
-}
-
-static void
-afr_print_lockee (char *str, int size, loc_t *loc, fd_t *fd,
-                  int child_index)
-{
-        snprintf (str, size, "path=%s, fd=%p, child=%d",
-                  loc->path ? loc->path : "<nul>",
-                  fd ? fd : NULL,
-                  child_index);
+    return;
 }
 
 void
-afr_print_entrylk (char *str, int size, const char *basename,
-                   gf_lkowner_t *owner)
-{
-        snprintf (str, size, "Basename=%s, lk-owner=%s",
-                  basename ? basename : "<nul>",
-                  lkowner_utoa (owner));
-}
-
-static void
-afr_print_verdict (int op_ret, int op_errno, char *str)
+afr_lockees_cleanup(afr_internal_lock_t *int_lock)
 {
-        if (op_ret < 0) {
-                if (op_errno == EAGAIN)
-                        strcpy (str, "EAGAIN");
-                else
-                        strcpy (str, "FAILED");
-        }
-        else
-                strcpy (str, "GRANTED");
-}
+    int i = 0;
 
-static void
-afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
-                        char *lock_call_type_str,
-                        afr_internal_lock_t *int_lock)
-{
-        switch (lock_call_type) {
-        case AFR_INODELK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_SELFHEAL");
-                break;
-        case AFR_INODELK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_SELFHEAL");
-                break;
-        default:
-                strcpy (lock_call_type_str, "UNKNOWN");
-                break;
-        }
+    for (i = 0; i < int_lock->lockee_count; i++) {
+        afr_lockee_cleanup(&int_lock->lockee[i]);
+    }
 
+    return;
 }
-
-static void
-afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
-                       afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                       int op_ret, int op_errno, int32_t child_index)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_LOCK_INFO,
-                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
-                (unsigned long long) int_lock->lock_number);
-
-}
-
-static void
-afr_trace_inodelk_in (call_frame_t *frame, xlator_t *this,
-                      afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                      int32_t cmd, int32_t child_index)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_inodelk (lock, 256, cmd, flock, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_LOCK_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
-
+int
+afr_entry_lockee_cmp(const void *l1, const void *l2)
+{
+    const afr_lockee_t *r1 = l1;
+    const afr_lockee_t *r2 = l2;
+    int ret = 0;
+    uuid_t gfid1 = {0};
+    uuid_t gfid2 = {0};
+
+    loc_gfid((loc_t *)&r1->loc, gfid1);
+    loc_gfid((loc_t *)&r2->loc, gfid2);
+    ret = gf_uuid_compare(gfid1, gfid2);
+    /*Entrylks with NULL basename are the 'smallest'*/
+    if (ret == 0) {
+        if (!r1->basename)
+            return -1;
+        if (!r2->basename)
+            return 1;
+        ret = strcmp(r1->basename, r2->basename);
+    }
+
+    if (ret <= 0)
+        return -1;
+    else
+        return 1;
 }
 
-static void
-afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
-                      afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t cookie)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        int                 child_index = 0;
-        int                 lockee_no = 0;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->entrylk_trace) {
-                return;
-        }
-        lockee_no = cookie / priv->child_count;
-        child_index = cookie % priv->child_count;
-
-        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
-                          child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_LOCK_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number,
-                cookie);
-}
+int
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int child_index);
 
-static void
-afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
-                       afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, const char *basename,
-                       int op_ret, int op_errno, int32_t cookie)
+void
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        int                 lockee_no = 0;
-        int                 child_index = 0;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->entrylk_trace) {
-                return;
-        }
-        lockee_no = cookie / priv->child_count;
-        child_index = cookie % priv->child_count;
-
-        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
-                          child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_LOCK_INFO,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number,
-                cookie);
+    gf_msg_trace(this->name, 0, "Setting lk-owner=%llu",
+                 (unsigned long long)(unsigned long)lk_owner);
 
+    set_lk_owner_from_ptr(&frame->root->lk_owner, lk_owner);
 }
 
-static int
-transaction_lk_op (afr_local_t *local)
+int32_t
+internal_lock_count(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        int ret = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t call_count = 0;
+    int i = 0;
 
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
 
-        if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK) {
-                gf_msg_debug (THIS->name, 0,
-                              "lk op is for a transaction");
-                ret = 1;
-        }
-        else if (int_lock->transaction_lk_type == AFR_SELFHEAL_LK) {
-                gf_msg_debug (THIS->name, 0,
-                              "lk op is for a self heal");
-
-                ret = 0;
-        }
-
-        if (ret == -1)
-                gf_msg_debug (THIS->name, 0,
-                              "lk op is not set");
-
-        return ret;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i])
+            ++call_count;
+    }
 
+    return call_count;
 }
 
-static int
-is_afr_lock_transaction (afr_local_t *local)
+int
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count)
 {
-        int ret = 0;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                ret = 1;
-                break;
+    GF_ASSERT(int_lock->lockee_count < AFR_LOCKEE_COUNT_MAX);
+    loc_copy(&lockee->loc, loc);
+    lockee->basename = (basename) ? gf_strdup(basename) : NULL;
+    if (basename && !lockee->basename)
+        goto out;
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                ret = 0;
-                break;
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        }
+    if (!lockee->locked_nodes)
+        goto out;
 
-        return ret;
+    ret = 0;
+    int_lock->lockee_count++;
+out:
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 int
-afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
-                       loc_t *loc, char *basename, int child_count)
+afr_add_inode_lockee(afr_local_t *local, int child_count)
 {
-        int     ret     = -1;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        loc_copy (&lockee->loc, loc);
-        lockee->basename        = (basename)? gf_strdup (basename): NULL;
-        if (basename && !lockee->basename)
-                goto out;
+    if (local->fd) {
+        lockee->fd = fd_ref(local->fd);
+    } else {
+        loc_copy(&lockee->loc, &local->loc);
+    }
 
-        lockee->locked_count    = 0;
-        lockee->locked_nodes    = GF_CALLOC (child_count,
-                                             sizeof (*lockee->locked_nodes),
-                                             gf_afr_mt_afr_node_character);
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        if (!lockee->locked_nodes)
-                goto out;
+    if (!lockee->locked_nodes)
+        goto out;
 
-        ret = 0;
+    ret = 0;
+    int_lock->lockee_count++;
 out:
-        return ret;
-
-}
-
-void
-afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
-{
-        int     i   = 0;
-
-        for (i = 0; i < int_lock->lockee_count; i++) {
-                loc_wipe (&int_lock->lockee[i].loc);
-                if (int_lock->lockee[i].basename)
-                        GF_FREE (int_lock->lockee[i].basename);
-                if (int_lock->lockee[i].locked_nodes)
-                       GF_FREE (int_lock->lockee[i].locked_nodes);
-        }
-
-        return;
-}
-
-static int
-initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-
-        int i = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->entrylk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
-
-        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
-                if (!int_lock->lockee[i].locked_nodes)
-                        break;
-                int_lock->lockee[i].locked_count = 0;
-                memset (int_lock->lockee[i].locked_nodes, 0,
-                        sizeof (*int_lock->lockee[i].locked_nodes) *
-                        priv->child_count);
-        }
-
-        return 0;
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 static int
-initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
+initialize_internal_lock_variables(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    int i = 0;
 
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        inodelk->lock_count    = 0;
-        int_lock->lk_attempted_count = 0;
-        int_lock->lock_op_ret   = -1;
-        int_lock->lock_op_errno = 0;
+    int_lock->lock_count = 0;
+    int_lock->lock_op_ret = -1;
+    int_lock->lock_op_errno = 0;
+    int_lock->lk_attempted_count = 0;
 
-        memset (inodelk->locked_nodes, 0,
-                sizeof (*inodelk->locked_nodes) * priv->child_count);
-        memset (int_lock->locked_nodes, 0,
-                sizeof (*int_lock->locked_nodes) * priv->child_count);
+    for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+        if (!int_lock->lockee[i].locked_nodes)
+            break;
+        int_lock->lockee[i].locked_count = 0;
+        memset(int_lock->lockee[i].locked_nodes, 0,
+               sizeof(*int_lock->lockee[i].locked_nodes) * priv->child_count);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
+afr_lockee_locked_nodes_count(afr_internal_lock_t *int_lock)
 {
-        int call_count  = 0;
-        int i           = 0;
+    int call_count = 0;
+    int i = 0;
 
-        for (i = 0; i < int_lock->lockee_count; i++)
-                call_count += int_lock->lockee[i].locked_count;
+    for (i = 0; i < int_lock->lockee_count; i++)
+        call_count += int_lock->lockee[i].locked_count;
 
-        return call_count;
+    return call_count;
 }
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count)
 
 {
-        int i = 0;
-        int call_count = 0;
+    int i = 0;
+    int call_count = 0;
 
-        for (i = 0; i < child_count; i++) {
-                if (locked_nodes[i] & LOCKED_YES)
-                        call_count++;
-        }
+    for (i = 0; i < child_count; i++) {
+        if (locked_nodes[i] & LOCKED_YES)
+            call_count++;
+    }
 
-        return call_count;
+    return call_count;
 }
 
-/* FIXME: What if UNLOCK fails */
-static int32_t
-afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static void
+afr_log_locks_failure(call_frame_t *frame, char *where, char *what,
+                      int op_errno)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int call_count = 0;
+    xlator_t *this = frame->this;
+    gf_lkowner_t *lk_owner = &frame->root->lk_owner;
+    afr_local_t *local = frame->local;
+    const char *fop = NULL;
+    char *gfid = NULL;
+    const char *name = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    fop = gf_fop_list[local->op];
 
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_msg_trace (this->name, 0,
-                              "All internal locks unlocked");
-
-                int_lock->lock_cbk (frame, this);
-        }
-
-        return 0;
+    switch (local->transaction.type) {
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+            switch (local->op) {
+                case GF_FOP_LINK:
+                    gfid = uuid_utoa(local->newloc.pargfid);
+                    name = local->newloc.name;
+                    break;
+                default:
+                    gfid = uuid_utoa(local->loc.pargfid);
+                    name = local->loc.name;
+                    break;
+            }
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do entry %s with lk-owner:%s on %s "
+                   "while attempting %s on {pgfid:%s, name:%s}.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid, name);
+            break;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            gfid = uuid_utoa(local->inode->gfid);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do inode %s with lk-owner:%s on %s "
+                   "while attempting %s on gfid:%s.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid);
+            break;
+    }
 }
 
 static int32_t
-afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t         *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk = NULL;
-        int32_t             child_index = (long)cookie;
-        afr_private_t       *priv = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
-
-        priv = this->private;
-
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_INODE_UNLOCK_FAIL,
-                        "path=%s gfid=%s: unlock failed on subvolume %s "
-                        "with lock owner %s", local->loc.path,
-                        loc_gfid_utoa (&(local->loc)),
-                        priv->children[child_index]->name,
-                        lkowner_utoa (&frame->root->lk_owner));
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int lockee_num = 0;
+    int call_count = 0;
+    int child_index = 0;
+    int ret = 0;
 
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    lockee_num = (int)((long)cookie) / priv->child_count;
+    child_index = (int)((long)cookie) % priv->child_count;
 
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-        inodelk->locked_nodes[child_index] &= LOCKED_NO;
-        if (local->transaction.eager_lock)
-                local->transaction.eager_lock[child_index] = 0;
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        afr_log_locks_failure(frame, priv->children[child_index]->name,
+                              "unlock", op_errno);
+    }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    int_lock->lockee[lockee_num].locked_nodes[child_index] &= LOCKED_NO;
+    if (local->transaction.type == AFR_DATA_TRANSACTION && op_ret != 1)
+        ret = afr_write_subvol_reset(frame, this);
 
-        return 0;
-
-}
-
-static int
-afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        struct gf_flock flock = {0,};
-        struct gf_flock full_flock = {0,};
-        struct gf_flock *flock_use = NULL;
-        int call_count = 0;
-        int i = 0;
-        int piggyback = 0;
-        afr_fd_ctx_t        *fd_ctx      = NULL;
+    LOCK(&frame->lock);
+    {
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
 
+    if (call_count == 0) {
+        int_lock->lock_cbk(frame, this);
+    }
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-
-        flock.l_start = inodelk->flock.l_start;
-        flock.l_len   = inodelk->flock.l_len;
-        flock.l_type  = F_UNLCK;
-
-        full_flock.l_type = F_UNLCK;
-        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
-                                             priv->child_count);
-
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count) {
-                gf_msg_trace (this->name, 0,
-                              "No internal locks unlocked");
-
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        if (local->fd)
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
-                        continue;
-
-                if (local->fd) {
-                        flock_use = &flock;
-                        if (!local->transaction.eager_lock[i]) {
-                                goto wind;
-                        }
-
-                        piggyback = 0;
-
-                        LOCK (&local->fd->lock);
-                        {
-                                if (fd_ctx->lock_piggyback[i]) {
-                                        fd_ctx->lock_piggyback[i]--;
-                                        piggyback = 1;
-                                } else {
-                                        fd_ctx->lock_acquired[i]--;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback) {
-                                afr_unlock_inodelk_cbk (frame, (void *) (long) i,
-                                                        this, 1, 0, NULL);
-                                if (!--call_count)
-                                        break;
-                                continue;
-                        }
-
-                        flock_use = &full_flock;
-                wind:
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, flock_use, F_SETLK,
-                                              i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLK, flock_use, NULL);
-
-                        if (!--call_count)
-                                break;
-
-                } else {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, &flock, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLK, &flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-out:
-        return 0;
+    return ret;
 }
 
-static int32_t
-afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t         *local = NULL;
-        afr_private_t       *priv  = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int32_t             child_index = 0;
-        int                 lockee_no   = 0;
-
-        priv = this->private;
-        lockee_no = (int)((long) cookie) / priv->child_count;
-        child_index = (int) ((long) cookie) % priv->child_count;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_UNLOCK_OP,
-                               int_lock->lockee[lockee_no].basename, op_ret,
-                               op_errno, (int) ((long)cookie));
-
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_ENTRY_UNLOCK_FAIL,
-                        "%s: unlock failed on %s", local->loc.path,
-                        priv->children[child_index]->name);
-        }
-
-        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
+void
+afr_internal_lock_wind(call_frame_t *frame,
+                       int32_t (*cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *),
+                       void *cookie, int child, int lockee_num,
+                       gf_boolean_t blocking, gf_boolean_t unlock)
+{
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    entrylk_cmd cmd = ENTRYLK_LOCK_NB;
+    int32_t cmd1 = F_SETLK;
+    struct gf_flock flock = {
+        0,
+    };
+
+    switch (local->transaction.type) {
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            if (unlock) {
+                cmd = ENTRYLK_UNLOCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd = ENTRYLK_LOCK;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->fentrylk,
+                                  int_lock->domain,
+                                  int_lock->lockee[lockee_num].fd,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            } else {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->entrylk,
+                                  int_lock->domain,
+                                  &int_lock->lockee[lockee_num].loc,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            }
+            break;
 
-        return 0;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            flock = int_lock->lockee[lockee_num].flock;
+            if (unlock) {
+                flock.l_type = F_UNLCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd1 = F_SETLKW;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->finodelk, int_lock->domain,
+                    int_lock->lockee[lockee_num].fd, cmd1, &flock, NULL);
+            } else {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->inodelk, int_lock->domain,
+                    &int_lock->lockee[lockee_num].loc, cmd1, &flock, NULL);
+            }
+            break;
+    }
 }
 
 static int
-afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t     *int_lock       = NULL;
-        afr_local_t             *local          = NULL;
-        afr_private_t           *priv           = NULL;
-        int                     call_count      = 0;
-        int                     index           = 0;
-        int                     lockee_no       = 0;
-        int                     copies          = 0;
-        int                     i               = -1;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-        copies   = priv->child_count;
-
-        call_count = afr_lockee_locked_nodes_count (int_lock);
-
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_msg_trace (this->name, 0,
-                              "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
-                lockee_no = i / copies;
-                index     = i % copies;
-                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP,
-                                              int_lock->lockee[lockee_no].basename,
-                                              i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[index],
-                                           priv->children[index]->fops->entrylk,
-                                           int_lock->domain,
-                                           &int_lock->lockee[lockee_no].loc,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+afr_unlock_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    int i = -1;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    call_count = afr_lockee_locked_nodes_count(int_lock);
+
+    int_lock->lk_call_count = call_count;
+
+    if (!call_count) {
+        gf_msg_trace(this->name, 0, "No internal locks unlocked");
+        int_lock->lock_cbk(frame, this);
+        goto out;
+    }
+
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        lockee_num = i / priv->child_count;
+        child_index = i % priv->child_count;
+        if (int_lock->lockee[lockee_num].locked_nodes[child_index] &
+            LOCKED_YES) {
+            afr_internal_lock_wind(frame, afr_unlock_common_cbk,
+                                   (void *)(long)i, child_index, lockee_num,
+                                   _gf_false, _gf_true);
+            if (!--call_count)
+                break;
         }
+    }
 
 out:
-        return 0;
-
+    return 0;
 }
 
 static int32_t
-afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t     *int_lock       = NULL;
-        afr_local_t             *local          = NULL;
-        afr_private_t           *priv           = NULL;
-        int                     cky             = (long) cookie;
-        int                     child_index     = 0;
-        int                     lockee_no       = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        child_index = ((int)cky) % priv->child_count;
-        lockee_no   = ((int)cky) / priv->child_count;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-                                gf_msg (this->name, GF_LOG_ERROR, ENOSYS,
-                                        AFR_MSG_LOCK_XLATOR_NOT_LOADED,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-                                local->op_ret = op_ret;
-                                int_lock->lock_op_ret = op_ret;
-                        }
-
-                        local->op_errno              = op_errno;
-                        int_lock->lock_op_errno      = op_errno;
+afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int cky = (long)cookie;
+    int child_index = 0;
+    int lockee_num = 0;
+
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    child_index = ((int)cky) % priv->child_count;
+    lockee_num = ((int)cky) / priv->child_count;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support locking. "
+                       "please load features/locks xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+            }
+
+            local->op_errno = op_errno;
+            int_lock->lock_op_errno = op_errno;
+        }
+
+        int_lock->lk_attempted_count++;
+    }
+    UNLOCK(&frame->lock);
+
+    if ((op_ret == -1) && (op_errno == ENOSYS)) {
+        afr_unlock_now(frame, this);
+    } else {
+        if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+            if (local->transaction.type == AFR_DATA_TRANSACTION) {
+                LOCK(&local->inode->lock);
+                {
+                    local->inode_ctx->lock_count++;
                 }
-
-		int_lock->lk_attempted_count++;
-        }
-        UNLOCK (&frame->lock);
-
-        if ((op_ret == -1) &&
-            (op_errno == ENOSYS)) {
-                afr_unlock (frame, this);
-        } else {
-                if (op_ret == 0) {
-                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
-                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
-                                int_lock->lockee[lockee_no].locked_count++;
-                                int_lock->entrylk_lock_count++;
-                        } else {
-                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                                int_lock->lock_count++;
-                        }
-                }
-                afr_lock_blocking (frame, this, cky + 1);
-        }
-
-        return 0;
-}
-
-static int32_t
-afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
-
-}
-
-static int32_t
-afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long)cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
-}
-
-static int
-afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
-                        sizeof (*inodelk->locked_nodes) * priv->child_count);
-                inodelk->lock_count = int_lock->lock_count;
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                /*entrylk_count is being used in both non-blocking and blocking
-                 * modes */
-                break;
+                UNLOCK(&local->inode->lock);
+            }
         }
+        afr_lock_blocking(frame, this, cky + 1);
+    }
 
-        return 0;
-
+    return 0;
 }
 
 static gf_boolean_t
-afr_is_entrylk (afr_internal_lock_t *int_lock,
-                afr_transaction_type trans_type)
+_is_lock_wind_needed(afr_local_t *local, int child_index)
 {
-        gf_boolean_t is_entrylk = _gf_false;
-
-        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
-            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
-
-                is_entrylk = _gf_true;
-
-        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
-                 (trans_type == AFR_ENTRY_TRANSACTION ||
-                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
+    if (!local->child_up[child_index])
+        return _gf_false;
 
-                is_entrylk = _gf_true;
-
-        } else {
-                is_entrylk = _gf_false;
-        }
-
-        return is_entrylk;
+    return _gf_true;
 }
 
 static gf_boolean_t
-_is_lock_wind_needed (afr_local_t *local, int child_index)
-{
-        if (!local->child_up[child_index])
-                return _gf_false;
-
-        return _gf_true;
-}
-
-static void
-afr_log_entry_locks_failure(xlator_t *this, afr_local_t *local,
-                            afr_internal_lock_t *int_lock)
-{
-        const char *fop = NULL;
-        char *pargfid = NULL;
-        const char *name = NULL;
-
-        fop = gf_fop_list[local->op];
-
-        switch (local->op) {
-        case GF_FOP_LINK:
-                pargfid = uuid_utoa(local->newloc.pargfid);
-                name = local->newloc.name;
-                break;
-        default:
-                pargfid = uuid_utoa(local->loc.pargfid);
-                name = local->loc.name;
-                break;
-        }
-
-        gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_BLOCKING_LKS_FAILED,
-                "Unable to obtain sufficient blocking entry locks on at least "
-                "one child while attempting %s on {pgfid:%s, name:%s}.", fop,
-                pargfid, name);
-}
-
-static gf_boolean_t
-is_blocking_locks_count_sufficient (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t  *local = NULL;
-        afr_private_t *priv = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        gf_boolean_t is_entrylk = _gf_false;
-        int child = 0;
-        int nlockee = 0;
-        int lockee_count = 0;
-        gf_boolean_t ret = _gf_true;
-
-        local = frame->local;
-        priv = this->private;
-        int_lock = &local->internal_lock;
-        lockee_count = int_lock->lockee_count;
-        is_entrylk = afr_is_entrylk (int_lock, local->transaction.type);
-
-        if (!is_entrylk) {
-                if (int_lock->lock_count == 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_BLOCKING_LKS_FAILED, "Unable to obtain "
-                                "blocking inode lock on even one child for "
-                                "gfid:%s.", uuid_utoa (local->inode->gfid));
-                        return _gf_false;
-                } else {
-                        /*inodelk succeded on atleast one child. */
-                        return _gf_true;
-                }
-
-        } else {
-                if (int_lock->entrylk_lock_count == 0) {
-                        afr_log_entry_locks_failure (this, local, int_lock);
-                        return _gf_false;
-                }
-                /* For FOPS that take multiple sets of locks (mkdir, rename),
-                 * there must be atleast one brick on which the locks from
-                 * all lock sets were successful. */
-                for (child = 0; child < priv->child_count; child++) {
-                        ret = _gf_true;
-                        for (nlockee = 0; nlockee < lockee_count; nlockee++) {
-                                if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES))
-                                        ret = _gf_false;
-                        }
-                        if (ret)
-                                return ret;
-                }
-                if (!ret)
-                        afr_log_entry_locks_failure (this, local, int_lock);
-        }
-
-        return ret;
-
+is_blocking_locks_count_sufficient(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int child = 0;
+    int nlockee = 0;
+    int lockee_count = 0;
+    gf_boolean_t ret = _gf_true;
+
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
+    lockee_count = int_lock->lockee_count;
+
+    if (int_lock->lock_count == 0) {
+        afr_log_locks_failure(frame, "any subvolume", "lock",
+                              int_lock->lock_op_errno);
+        return _gf_false;
+    }
+    /* For FOPS that take multiple sets of locks (mkdir, rename),
+     * there must be at least one brick on which the locks from
+     * all lock sets were successful. */
+    for (child = 0; child < priv->child_count; child++) {
+        ret = _gf_true;
+        for (nlockee = 0; nlockee < lockee_count; nlockee++) {
+            if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES))
+                ret = _gf_false;
+        }
+        if (ret)
+            return ret;
+    }
+    if (!ret)
+        afr_log_locks_failure(frame, "all", "lock", int_lock->lock_op_errno);
+
+    return ret;
 }
 
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int cookie)
 {
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_inodelk_t       *inodelk     = NULL;
-        afr_local_t         *local       = NULL;
-        afr_private_t       *priv        = NULL;
-        struct gf_flock flock = {0,};
-        uint64_t ctx = 0;
-        int ret = 0;
-        int child_index = 0;
-        int lockee_no   = 0;
-        gf_boolean_t is_entrylk = _gf_false;
-
-        local         = frame->local;
-        int_lock      = &local->internal_lock;
-        priv          = this->private;
-        child_index   = cookie % priv->child_count;
-        lockee_no     = cookie / priv->child_count;
-        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
-
-
-        if (!is_entrylk) {
-                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-                flock.l_start = inodelk->flock.l_start;
-                flock.l_len   = inodelk->flock.l_len;
-                flock.l_type  = inodelk->flock.l_type;
-        }
-
-        if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+    int child_index = 0;
+    int lockee_num = 0;
 
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_FD_CTX_GET_FAILED,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-
-                        afr_copy_locked_nodes (frame, this);
-
-                        afr_unlock (frame, this);
-
-                        return 0;
-                }
-        }
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    child_index = cookie % priv->child_count;
+    lockee_num = cookie / priv->child_count;
 
-        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
-                if (!is_blocking_locks_count_sufficient (frame, this)) {
+    if (local->fd) {
+        ret = fd_ctx_get(local->fd, this, &ctx);
 
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
 
-                        afr_copy_locked_nodes (frame, this);
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                        afr_unlock(frame, this);
+            afr_unlock_now(frame, this);
 
-                        return 0;
-                }
+            return 0;
         }
+    }
 
-        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
-                /* we're done locking */
-
-                gf_msg_debug (this->name, 0,
-                              "we're done locking");
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        if (!is_blocking_locks_count_sufficient(frame, this)) {
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                afr_copy_locked_nodes (frame, this);
+            afr_unlock_now(frame, this);
 
-                int_lock->lock_op_ret = 0;
-                int_lock->lock_cbk (frame, this);
-                return 0;
+            return 0;
         }
+    }
 
-        if (!_is_lock_wind_needed (local, child_index)) {
-                afr_lock_blocking (frame, this, cookie + 1);
-                return 0;
-        }
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        /* we're done locking */
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
+        gf_msg_debug(this->name, 0, "we're done locking");
 
-                if (local->fd) {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLKW, &flock, NULL);
-
-                } else {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLKW, &flock, NULL);
-                }
-
-                break;
+        int_lock->lock_op_ret = 0;
+        int_lock->lock_cbk(frame, this);
+        return 0;
+    }
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                /*Accounting for child_index increments on 'down'
-                 *and 'fd-less' children */
-
-                if (local->fd) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP,
-                                              int_lock->lockee[lockee_no].basename,
-                                              cookie);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) cookie,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->fentrylk,
-                                           int_lock->domain, local->fd,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                } else {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) cookie,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->entrylk,
-                                           int_lock->domain,
-                                           &int_lock->lockee[lockee_no].loc,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                }
+    if (!_is_lock_wind_needed(local, child_index)) {
+        afr_lock_blocking(frame, this, cookie + 1);
+        return 0;
+    }
 
-                break;
-        }
+    afr_internal_lock_wind(frame, afr_lock_cbk, (void *)(long)cookie,
+                           child_index, lockee_num, _gf_true, _gf_false);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_blocking_lock (call_frame_t *frame, xlator_t *this)
+afr_blocking_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        int                  up_count = 0;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int up_count = 0;
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                initialize_inodelk_variables (frame, this);
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                up_count = AFR_COUNT (local->child_up, priv->child_count);
-                int_lock->lk_call_count = int_lock->lk_expected_count
-                                        = (int_lock->lockee_count *
-                                           up_count);
-                initialize_entrylk_variables (frame, this);
-                break;
-        }
+    up_count = AFR_COUNT(local->child_up, priv->child_count);
+    int_lock->lk_call_count = int_lock->lk_expected_count =
+        (int_lock->lockee_count * up_count);
+    initialize_internal_lock_variables(frame, this);
 
-        afr_lock_blocking (frame, this, 0);
+    afr_lock_blocking(frame, this, 0);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_nb_internal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int call_count          = 0;
-        int child_index         = (long) cookie;
-        int copies              = 0;
-        int index               = 0;
-        int lockee_no           = 0;
-        afr_private_t       *priv = NULL;
-
-        priv = this->private;
-
-        copies = priv->child_count;
-        index = child_index % copies;
-        lockee_no = child_index / copies;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP,
-                               int_lock->lockee[lockee_no].basename, op_ret,
-                               op_errno, (long) cookie);
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret < 0 ) {
-			if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-			        gf_msg (this->name, GF_LOG_ERROR,
-                                        ENOSYS, AFR_MSG_LOCK_XLATOR_NOT_LOADED,
-                                        "subvolume does not support "
-                                        "locking. please load features/locks"
-                                        " xlator on server");
-				local->op_ret         = op_ret;
-				int_lock->lock_op_ret = op_ret;
-
-				int_lock->lock_op_errno      = op_errno;
-				local->op_errno              = op_errno;
-			}
-		} else if (op_ret == 0) {
-			int_lock->lockee[lockee_no].locked_nodes[index] |= \
-				LOCKED_YES;
-			int_lock->lockee[lockee_no].locked_count++;
-			int_lock->entrylk_lock_count++;
-		}
-
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_msg_trace (this->name, 0,
-                              "Last locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (int_lock->entrylk_lock_count ==
-                                int_lock->lk_expected_count) {
-                        gf_msg_trace (this->name, 0,
-                                      "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_msg_trace (this->name, 0,
-                                      "%d servers locked. Trying again "
-                                      "with blocking calls",
-                                      int_lock->lock_count);
-
-                        afr_unlock(frame, this);
-                }
-        }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    afr_private_t *priv = NULL;
 
-        return 0;
-}
+    priv = this->private;
 
-int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock   = NULL;
-        afr_local_t         *local      = NULL;
-        afr_private_t       *priv       = NULL;
-        afr_fd_ctx_t        *fd_ctx     = NULL;
-        int                 copies      = 0;
-        int                 index       = 0;
-        int                 lockee_no   = 0;
-        int32_t             call_count  = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        copies = priv->child_count;
-        initialize_entrylk_variables (frame, this);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_FD_CTX_GET_FAILED,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-			afr_unlock (frame, this);
-                        return -1;
-                }
-
-                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                if (!call_count) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_INFO_COMMON,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
-
-                /* Send non-blocking entrylk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
-                        index = i%copies;
-                        lockee_no = i/copies;
-                        if (local->child_up[index]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP,
-                                                      int_lock->lockee[lockee_no].basename,
-                                                      i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[index],
-                                                   priv->children[index]->fops->fentrylk,
-                                                   this->name, local->fd,
-                                                   int_lock->lockee[lockee_no].basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-                                if (!--call_count)
-                                        break;
-                        }
-                }
-        } else {
-                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
-                        index = i%copies;
-                        lockee_no = i/copies;
-                        if (local->child_up[index]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP,
-                                                      int_lock->lockee[lockee_no].basename,
-                                                      i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[index],
-                                                   priv->children[index]->fops->entrylk,
-                                                   this->name, &int_lock->lockee[lockee_no].loc,
-                                                   int_lock->lockee[lockee_no].basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-
-                                if (!--call_count)
-                                        break;
-                        }
-                }
-        }
-out:
-        return 0;
-}
-
-int32_t
-afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        int call_count  = 0;
-        int child_index = (long) cookie;
-        afr_fd_ctx_t        *fd_ctx = NULL;
-
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+    child_index = ((long)cookie) % priv->child_count;
+    lockee_num = ((long)cookie) / priv->child_count;
 
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-	if (local->fd)
-		fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        LOCK (&frame->lock);
+    if (op_ret == 0 && local->transaction.type == AFR_DATA_TRANSACTION) {
+        LOCK(&local->inode->lock);
         {
-		if (op_ret < 0) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-		                gf_msg (this->name, GF_LOG_ERROR, ENOSYS,
-                                        AFR_MSG_LOCK_XLATOR_NOT_LOADED,
-					"subvolume does not support "
-                                        "locking. please load features/locks"
-                                        " xlator on server");
-				local->op_ret                = op_ret;
-				int_lock->lock_op_ret        = op_ret;
-				int_lock->lock_op_errno      = op_errno;
-				local->op_errno              = op_errno;
-			}
-			if (local->transaction.eager_lock)
-				local->transaction.eager_lock[child_index] = 0;
-		} else {
-			inodelk->locked_nodes[child_index] |= LOCKED_YES;
-			inodelk->lock_count++;
-
-			if (local->transaction.eager_lock &&
-			    local->transaction.eager_lock[child_index] &&
-			    local->fd) {
-				/* piggybacked */
-				if (op_ret == 1) {
-					/* piggybacked */
-				} else if (op_ret == 0) {
-					/* lock acquired from server */
-                                        fd_ctx->lock_acquired[child_index]++;
-				}
-			}
-		}
-
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_msg_trace (this->name, 0,
-                              "Last inode locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (inodelk->lock_count == int_lock->lk_expected_count) {
-                        gf_msg_trace (this->name, 0,
-                                      "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_msg_trace (this->name, 0,
-                                      "%d servers locked. "
-                                      "Trying again with blocking calls",
-                                      int_lock->lock_count);
-
-                        afr_unlock(frame, this);
-                }
+            local->inode_ctx->lock_count++;
         }
+        UNLOCK(&local->inode->lock);
+    }
 
-        return 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support "
+                       "locking. please load features/locks"
+                       " xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+
+                int_lock->lock_op_errno = op_errno;
+                local->op_errno = op_errno;
+            }
+        } else if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+        }
+
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        gf_msg_trace(this->name, 0, "Last locking reply received");
+        /* all locks successful. Proceed to call FOP */
+        if (int_lock->lock_count == int_lock->lk_expected_count) {
+            gf_msg_trace(this->name, 0, "All servers locked. Calling the cbk");
+            int_lock->lock_op_ret = 0;
+            int_lock->lock_cbk(frame, this);
+        }
+        /* Not all locks were successful. Unlock and try locking
+           again, this time with serially blocking locks */
+        else {
+            gf_msg_trace(this->name, 0,
+                         "%d servers locked. Trying again "
+                         "with blocking calls",
+                         int_lock->lock_count);
+
+            afr_unlock_now(frame, this);
+        }
+    }
+
+    return 0;
 }
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        int32_t             call_count = 0;
-        int                 i          = 0;
-        int                 ret        = 0;
-        struct              gf_flock flock = {0,};
-        struct              gf_flock full_flock = {0,};
-        struct              gf_flock *flock_use = NULL;
-        int                 piggyback = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-
-        flock.l_start = inodelk->flock.l_start;
-        flock.l_len   = inodelk->flock.l_len;
-        flock.l_type  = inodelk->flock.l_type;
-
-        full_flock.l_type = inodelk->flock.l_type;
-
-        initialize_inodelk_variables (frame, this);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_FD_CTX_GET_FAILED,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-			afr_unlock (frame, this);
-                        ret = -1;
-                        goto out;
-                }
-
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                if (!call_count) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_ALL_SUBVOLS_DOWN,
-                                "All bricks are down, aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
-
-                /* Send non-blocking inodelk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i])
-                                continue;
-
-                        flock_use = &flock;
-                        if (!local->transaction.eager_lock_on) {
-                                goto wind;
-                        }
-
-                        piggyback = 0;
-                        local->transaction.eager_lock[i] = 1;
-
-			afr_set_delayed_post_op (frame, this);
-
-                        LOCK (&local->fd->lock);
-                        {
-                                if (fd_ctx->lock_acquired[i]) {
-                                        fd_ctx->lock_piggyback[i]++;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback) {
-                                /* (op_ret == 1) => indicate piggybacked lock */
-                                afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
-                                                             this, 1, 0, NULL);
-                                if (!--call_count)
-                                        break;
-                                continue;
-                        }
-                        flock_use = &full_flock;
-                wind:
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_NB_TRANSACTION,
-                                              AFR_LOCK_OP, flock_use, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLK, flock_use, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        } else {
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i])
-                                continue;
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_NB_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLK, &flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int child = 0;
+    int lockee_num = 0;
+    int32_t call_count = 0;
+    int i = 0;
+    int ret = 0;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    initialize_internal_lock_variables(frame, this);
+
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
+
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
+            local->op_errno = EINVAL;
+            int_lock->lock_op_errno = EINVAL;
+
+            afr_unlock_now(frame, this);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    call_count = int_lock->lockee_count * internal_lock_count(frame, this);
+    int_lock->lk_call_count = call_count;
+    int_lock->lk_expected_count = call_count;
+
+    if (!call_count) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON,
+               "fd not open on any subvolumes. aborting.");
+        afr_unlock_now(frame, this);
+        goto out;
+    }
+
+    /* Send non-blocking lock calls only on up children
+       and where the fd has been opened */
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        child = i % priv->child_count;
+        lockee_num = i / priv->child_count;
+        if (local->child_up[child]) {
+            afr_internal_lock_wind(frame, afr_nb_internal_lock_cbk,
+                                   (void *)(long)i, child, lockee_num,
+                                   _gf_false, _gf_false);
+            if (!--call_count)
+                break;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (transaction_lk_op (local)) {
-                if (is_afr_lock_transaction (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        afr_unlock_entrylk (frame, this);
-
-        } else {
-                if (is_afr_lock_selfheal (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        afr_unlock_entrylk (frame, this);
-        }
-
+afr_unlock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->transaction.eager_lock_on)
+        goto out;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        if (list_empty(&lock->owners) && list_empty(&lock->post_op)) {
+            local->transaction.do_eager_unlock = _gf_true;
+            /*TODO: Need to get metadata use on_disk and inherit/uninherit
+             *GF_ASSERT (!local->inode_ctx->on_disk[local->transaction.type]);
+             *GF_ASSERT (!local->inode_ctx->inherited[local->transaction.type]);
+             */
+            GF_ASSERT(lock->release);
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    if (!local->transaction.do_eager_unlock) {
+        local->internal_lock.lock_cbk(frame, this);
         return 0;
-}
-
-int
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
-                          unsigned int child_count)
-{
-        afr_local_t         *dst_local   = NULL;
-        afr_local_t         *src_local   = NULL;
-        afr_internal_lock_t *dst_lock    = NULL;
-        afr_internal_lock_t *src_lock    = NULL;
-        afr_inodelk_t       *dst_inodelk = NULL;
-        afr_inodelk_t       *src_inodelk = NULL;
-        int                 ret = -1;
-
-        src_local = src->local;
-        src_lock  = &src_local->internal_lock;
-        src_inodelk = afr_get_inodelk (src_lock, dom);
-        dst_local = dst->local;
-        dst_lock  = &dst_local->internal_lock;
-        dst_inodelk = afr_get_inodelk (dst_lock, dom);
-        if (!dst_inodelk || !src_inodelk)
-                goto out;
-        if (src_inodelk->locked_nodes) {
-                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
-                        sizeof (*dst_inodelk->locked_nodes) * child_count);
-                memset (src_inodelk->locked_nodes, 0,
-                        sizeof (*src_inodelk->locked_nodes) * child_count);
-        }
+    }
 
-        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
-        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
-        dst_inodelk->lock_count = src_inodelk->lock_count;
-        src_inodelk->lock_count = 0;
-        ret = 0;
 out:
-        return ret;
+    afr_unlock_now(frame, this);
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index 7f7962013d7..816065fb57a 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -8,45 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_MEM_TYPES_H__
 #define __AFR_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_afr_mem_types_ {
-        gf_afr_mt_iovec  = gf_common_mt_end + 1,
-        gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_private_t,
-        gf_afr_mt_int32_t,
-        gf_afr_mt_char,
-        gf_afr_mt_xattr_key,
-        gf_afr_mt_dict_t,
-        gf_afr_mt_xlator_t,
-        gf_afr_mt_iatt,
-        gf_afr_mt_int,
-        gf_afr_mt_afr_node_character,
-        gf_afr_mt_sh_diff_loop_state,
-        gf_afr_mt_uint8_t,
-        gf_afr_mt_loc_t,
-        gf_afr_mt_entry_name,
-        gf_afr_mt_pump_priv,
-        gf_afr_mt_locked_fd,
-        gf_afr_mt_inode_ctx_t,
-        gf_afr_fd_paused_call_t,
-        gf_afr_mt_crawl_data_t,
-        gf_afr_mt_brick_pos_t,
-        gf_afr_mt_shd_bool_t,
-        gf_afr_mt_shd_timer_t,
-        gf_afr_mt_shd_event_t,
-        gf_afr_mt_time_t,
-        gf_afr_mt_pos_data_t,
-	gf_afr_mt_reply_t,
-	gf_afr_mt_subvol_healer_t,
-	gf_afr_mt_spbc_timeout_t,
-        gf_afr_mt_spb_status_t,
-        gf_afr_mt_empty_brick_t,
-        gf_afr_mt_end
+    gf_afr_mt_afr_fd_ctx_t = gf_common_mt_end + 1,
+    gf_afr_mt_afr_private_t,
+    gf_afr_mt_int32_t,
+    gf_afr_mt_char,
+    gf_afr_mt_xattr_key,
+    gf_afr_mt_dict_t,
+    gf_afr_mt_xlator_t,
+    gf_afr_mt_afr_node_character,
+    gf_afr_mt_inode_ctx_t,
+    gf_afr_mt_shd_event_t,
+    gf_afr_mt_reply_t,
+    gf_afr_mt_subvol_healer_t,
+    gf_afr_mt_spbc_timeout_t,
+    gf_afr_mt_spb_status_t,
+    gf_afr_mt_empty_brick_t,
+    gf_afr_mt_child_latency_t,
+    gf_afr_mt_atomic_t,
+    gf_afr_mt_lk_heal_info_t,
+    gf_afr_mt_gf_lock,
+    gf_afr_mt_end
 };
 #endif
-
diff --git a/xlators/cluster/afr/src/afr-messages.h b/xlators/cluster/afr/src/afr-messages.h
index c7af18d0f25..e73fd997765 100644
--- a/xlators/cluster/afr/src/afr-messages.h
+++ b/xlators/cluster/afr/src/afr-messages.h
@@ -11,363 +11,157 @@
 #ifndef _AFR_MESSAGES_H_
 #define _AFR_MESSAGES_H_
 
-#include "glfs-message-id.h"
-
-/*! \file afr-messages.h
- *  \brief AFR log-message IDs and their descriptions.
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- * anywhere. If reused then then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
 
-#define GLFS_COMP_BASE_AFR      GLFS_MSGID_COMP_AFR
-#define GLFS_NUM_MESSAGES       42
-#define GLFS_MSGID_END          (GLFS_COMP_BASE_AFR + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x GLFS_COMP_BASE_AFR, "Invalid: Start of messages"
-
-/*!
- * @messageid 108001
- * @diagnosis Client quorum is not met due to which file modification
- * operations are disallowed.
- * @recommendedaction Some brick processes are down/ not visible from the
- * client. Ensure that the bricks are up/ network traffic is not blocked.
- */
-#define AFR_MSG_QUORUM_FAIL             (GLFS_COMP_BASE_AFR + 1)
-
-
-/*!
- * @messageid 108002
- * @diagnosis The bricks that were down are now up and quorum is restored.
- * @recommendedaction Possibly check why the bricks went down to begin with.
- */
-#define AFR_MSG_QUORUM_MET              (GLFS_COMP_BASE_AFR + 2)
-
-
-/*!
- * @messageid 108003
- * @diagnosis Client quorum-type was set to auto due to which the quorum-count
- * option is no longer valid.
- * @recommendedaction None.
- */
-#define AFR_MSG_QUORUM_OVERRIDE         (GLFS_COMP_BASE_AFR + 3)
-
-
-/*!
- * @messageid 108004
- * @diagnosis Replication sub volume witnessed a connection notification
- * from a brick which does not belong to its replica set.
- * @recommendedaction None. This is a safety check in code.
- */
-#define AFR_MSG_INVALID_CHILD_UP        (GLFS_COMP_BASE_AFR + 4)
-
-
-/*!
- * @messageid 108005
- * @diagnosis A replica set that was inaccessible because all its bricks were
- * down is now accessible because at least one of its bricks came back up.
- * @recommendedaction Possibly check why all the bricks of that replica set
- * went down to begin with.
- */
-#define AFR_MSG_SUBVOL_UP               (GLFS_COMP_BASE_AFR + 5)
-
-
-/*!
- * @messageid 108006
- * @diagnosis All bricks of a replica set are down. Data residing in that
- * replica cannot be accessed until one of the bricks come back up.
- * @recommendedaction Ensure that the bricks are up.
- */
-#define AFR_MSG_ALL_SUBVOLS_DOWN        (GLFS_COMP_BASE_AFR + 6)
-
-
-/*!
- * @messageid 108007
- * @diagnosis Entry unlocks failed on a brick.
- * @recommendedaction Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
-*/
-#define AFR_MSG_ENTRY_UNLOCK_FAIL       (GLFS_COMP_BASE_AFR + 7)
-
-
-/*!
- * @messageid 108008
- * @diagnosis There is an inconsistency in the file's data/metadata/gfid
- * amongst the bricks of a replica set.
- * @recommendedaction Resolve the split brain by clearing the AFR changelog
- * attributes from the appropriate brick and trigger self-heal.
- */
-#define AFR_MSG_SPLIT_BRAIN             (GLFS_COMP_BASE_AFR + 8)
-
-
-/*!
- * @messageid 108009
- * @diagnosis open/opendir failed on a brick.
- * @recommendedaction Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
- */
-#define AFR_MSG_OPEN_FAIL               (GLFS_COMP_BASE_AFR + 9)
-
-
-/*!
- * @messageid 108010
- * @diagnosis Inode unlocks failed on a brick.
- * @recommendedaction Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
-*/
-#define AFR_MSG_INODE_UNLOCK_FAIL       (GLFS_COMP_BASE_AFR + 10)
-
-/*!
- * @messageid 108011
- * @diagnosis Setting of pending xattrs succeeded/failed during replace-brick
- * operation.
- * @recommendedaction In case of failure, error number in the log should give
- * the reason why it failed. Also observe brick logs for more information.
-*/
-#define AFR_MSG_REPLACE_BRICK_STATUS     (GLFS_COMP_BASE_AFR + 11)
-
-/*!
- * @messageid 108012
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_GFID_NULL                       (GLFS_COMP_BASE_AFR + 12)
-
-/*!
- * @messageid 108013
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_FD_CREATE_FAILED                (GLFS_COMP_BASE_AFR + 13)
-
-/*!
- * @messageid 108014
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_DICT_SET_FAILED                 (GLFS_COMP_BASE_AFR + 14)
-
-/*!
- * @messageid 108015
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_EXPUNGING_FILE_OR_DIR           (GLFS_COMP_BASE_AFR + 15)
-
-/*!
- * @messageid 108016
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_MIGRATION_IN_PROGRESS           (GLFS_COMP_BASE_AFR + 16)
-
-/*!
- * @messageid 108017
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_CHILD_MISCONFIGURED             (GLFS_COMP_BASE_AFR + 17)
-
-/*!
- * @messageid 108018
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_VOL_MISCONFIGURED               (GLFS_COMP_BASE_AFR + 18)
-
-/*!
- * @messageid 108019
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_BLOCKING_LKS_FAILED             (GLFS_COMP_BASE_AFR + 19)
-
-/*!
- * @messageid 108020
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_INVALID_FD                      (GLFS_COMP_BASE_AFR + 20)
-
-/*!
- * @messageid 108021
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_LOCK_INFO                       (GLFS_COMP_BASE_AFR + 21)
-
-/*!
- * @messageid 108022
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_LOCK_XLATOR_NOT_LOADED          (GLFS_COMP_BASE_AFR + 22)
-
-/*!
- * @messageid 108023
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_FD_CTX_GET_FAILED               (GLFS_COMP_BASE_AFR + 23)
-
-/*!
- * @messageid 108024
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_INVALID_SUBVOL                    (GLFS_COMP_BASE_AFR + 24)
-
-/*!
- * @messageid 108025
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_PUMP_XLATOR_ERROR               (GLFS_COMP_BASE_AFR + 25)
-
-/*!
- * @messageid 108026
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_SELF_HEAL_INFO                  (GLFS_COMP_BASE_AFR + 26)
-
-/*!
- * @messageid 108027
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_READ_SUBVOL_ERROR               (GLFS_COMP_BASE_AFR + 27)
-
-/*!
- * @messageid 108028
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_DICT_GET_FAILED                 (GLFS_COMP_BASE_AFR + 28)
-
-
-/*!
- * @messageid 108029
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_INFO_COMMON                     (GLFS_COMP_BASE_AFR + 29)
-
-/*!
- * @messageid 108030
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR         (GLFS_COMP_BASE_AFR + 30)
-
-/*!
- * @messageid 108031
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_LOCAL_CHILD         (GLFS_COMP_BASE_AFR + 31)
-
-/*!
- * @messageid 108032
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_INVALID_DATA         (GLFS_COMP_BASE_AFR + 32)
-
-/*!
- * @messageid 108033
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_INVALID_ARG         (GLFS_COMP_BASE_AFR + 33)
-
-/*!
- * @messageid 108034
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_INDEX_DIR_GET_FAILED         (GLFS_COMP_BASE_AFR + 34)
-
-/*!
- * @messageid 108035
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_FSYNC_FAILED         (GLFS_COMP_BASE_AFR + 35)
-
-/*!
- * @messageid 108036
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_FAVORITE_CHILD         (GLFS_COMP_BASE_AFR + 36)
-/*!
- * @messageid 108037
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_SELF_HEAL_FAILED                (GLFS_COMP_BASE_AFR + 37)
-
-/*!
- * @messageid 108038
- * @diagnosis
- * @recommendedaction
-*/
-#define AFR_MSG_SPLIT_BRAIN_STATUS      (GLFS_COMP_BASE_AFR + 38)
-
-/*!
- * @messageid 108039
- * @diagnosis Setting of pending xattrs succeeded/failed during add-brick
- * operation.
- * @recommendedaction In case of failure, error number in the log should give
- * the reason why it failed. Also observe brick logs for more information.
-*/
-#define AFR_MSG_ADD_BRICK_STATUS        (GLFS_COMP_BASE_AFR + 39)
-
-
-/*!
- * @messageid 108040
- * @diagnosis AFR was unable to be loaded because the pending-changelog xattrs
- * were not found in the volfile.
- * @recommendedaction Please ensure cluster op-version is atleast 30707 and the
- * volfiles are regenerated.
-*/
-#define AFR_MSG_NO_CHANGELOG  (GLFS_COMP_BASE_AFR + 40)
-
-/*!
- * @messageid 108041
- * @diagnosis Unable to create timer thread for delayed initialization.
- * @recommendedaction Possibly check process's log file for messages from
- * timer infra.
-*/
-#define AFR_MSG_TIMER_CREATE_FAIL               (GLFS_COMP_BASE_AFR + 41)
-
-/*!
- * @messageid 108042
- * @diagnosis Log messages relating to automated resolution of split-brain files
- * based on favorite child policies.
- * @recommendedaction
-*/
-#define AFR_MSG_SBRAIN_FAV_CHILD_POLICY  (GLFS_COMP_BASE_AFR + 42)
-
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+GLFS_MSGID(
+    AFR, AFR_MSG_QUORUM_FAIL, AFR_MSG_QUORUM_MET, AFR_MSG_QUORUM_OVERRIDE,
+    AFR_MSG_INVALID_CHILD_UP, AFR_MSG_SUBVOL_UP, AFR_MSG_SUBVOLS_DOWN,
+    AFR_MSG_ENTRY_UNLOCK_FAIL, AFR_MSG_SPLIT_BRAIN, AFR_MSG_OPEN_FAIL,
+    AFR_MSG_UNLOCK_FAIL, AFR_MSG_REPLACE_BRICK_STATUS, AFR_MSG_GFID_NULL,
+    AFR_MSG_FD_CREATE_FAILED, AFR_MSG_DICT_SET_FAILED,
+    AFR_MSG_EXPUNGING_FILE_OR_DIR, AFR_MSG_MIGRATION_IN_PROGRESS,
+    AFR_MSG_CHILD_MISCONFIGURED, AFR_MSG_VOL_MISCONFIGURED,
+    AFR_MSG_INTERNAL_LKS_FAILED, AFR_MSG_INVALID_FD, AFR_MSG_LOCK_INFO,
+    AFR_MSG_LOCK_XLATOR_NOT_LOADED, AFR_MSG_FD_CTX_GET_FAILED,
+    AFR_MSG_INVALID_SUBVOL, AFR_MSG_PUMP_XLATOR_ERROR, AFR_MSG_SELF_HEAL_INFO,
+    AFR_MSG_READ_SUBVOL_ERROR, AFR_MSG_DICT_GET_FAILED, AFR_MSG_INFO_COMMON,
+    AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, AFR_MSG_LOCAL_CHILD, AFR_MSG_INVALID_DATA,
+    AFR_MSG_INVALID_ARG, AFR_MSG_INDEX_DIR_GET_FAILED, AFR_MSG_FSYNC_FAILED,
+    AFR_MSG_FAVORITE_CHILD, AFR_MSG_SELF_HEAL_FAILED,
+    AFR_MSG_SPLIT_BRAIN_STATUS, AFR_MSG_ADD_BRICK_STATUS, AFR_MSG_NO_CHANGELOG,
+    AFR_MSG_TIMER_CREATE_FAIL, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+    AFR_MSG_INODE_CTX_GET_FAILED, AFR_MSG_THIN_ARB,
+    AFR_MSG_THIN_ARB_XATTROP_FAILED, AFR_MSG_THIN_ARB_LOC_POP_FAILED,
+    AFR_MSG_GET_PEND_VAL, AFR_MSG_THIN_ARB_SKIP_SHD, AFR_MSG_UNKNOWN_SET,
+    AFR_MSG_NO_XL_ID, AFR_MSG_SELF_HEAL_INFO_START,
+    AFR_MSG_SELF_HEAL_INFO_FINISH, AFR_MSG_INCRE_COUNT,
+    AFR_MSG_ADD_TO_OUTPUT_FAILED, AFR_MSG_SET_TIME_FAILED,
+    AFR_MSG_GFID_MISMATCH_DETECTED, AFR_MSG_GFID_HEAL_MSG,
+    AFR_MSG_THIN_ARB_LOOKUP_FAILED, AFR_MSG_DICT_CREATE_FAILED,
+    AFR_MSG_NO_MAJORITY_TO_RESOLVE, AFR_MSG_TYPE_MISMATCH,
+    AFR_MSG_SIZE_POLICY_NOT_APPLICABLE, AFR_MSG_NO_CHILD_SELECTED,
+    AFR_MSG_INVALID_CHILD, AFR_MSG_RESOLVE_CONFLICTING_DATA,
+    SERROR_GETTING_SRC_BRICK, SNO_DIFF_IN_MTIME, SNO_BIGGER_FILE,
+    SALL_BRICKS_UP_TO_RESOLVE, AFR_MSG_UNLOCK_FAILED, AFR_MSG_POST_OP_FAILED,
+    AFR_MSG_TA_FRAME_CREATE_FAILED, AFR_MSG_SET_KEY_XATTROP_FAILED,
+    AFR_MSG_BLOCKING_ENTRYLKS_FAILED, AFR_MSG_FOP_FAILED,
+    AFR_MSG_CLEAN_UP_FAILED, AFR_MSG_UNABLE_TO_FETCH, AFR_MSG_XATTR_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_REPLICA, AFR_MSG_INODE_CTX_FAILED,
+    AFR_MSG_LOOKUP_FAILED, AFR_MSG_ALL_SUBVOLS_DOWN,
+    AFR_MSG_RELEASE_LOCK_FAILED, AFR_MSG_CLEAR_TIME_SPLIT_BRAIN,
+    AFR_MSG_READ_FAILED, AFR_MSG_LAUNCH_FAILED, AFR_MSG_READ_SUBVOL_NOT_UP,
+    AFR_MSG_LK_HEAL_DOM, AFR_MSG_NEW_BRICK, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, AFR_MSG_HEALER_SPAWN_FAILED,
+    AFR_MSG_ADD_CRAWL_EVENT_FAILED, AFR_MSG_NULL_DEREF, AFR_MSG_SET_PEND_XATTR,
+    AFR_MSG_INTERNAL_ATTR);
+
+#define AFR_MSG_DICT_GET_FAILED_STR "Dict get failed"
+#define AFR_MSG_DICT_SET_FAILED_STR "Dict set failed"
+#define AFR_MSG_HEALER_SPAWN_FAILED_STR "Healer spawn failed"
+#define AFR_MSG_ADD_CRAWL_EVENT_FAILED_STR "Adding crawl event failed"
+#define AFR_MSG_INVALID_ARG_STR "Invalid argument"
+#define AFR_MSG_INDEX_DIR_GET_FAILED_STR "unable to get index-dir on "
+#define AFR_MSG_THIN_ARB_LOOKUP_FAILED_STR "Failed lookup on file"
+#define AFR_MSG_DICT_CREATE_FAILED_STR "Failed to create dict."
+#define AFR_MSG_THIN_ARB_XATTROP_FAILED_STR "Xattrop failed."
+#define AFR_MSG_THIN_ARB_LOC_POP_FAILED_STR                                    \
+    "Failed to populate loc for thin-arbiter"
+#define AFR_MSG_GET_PEND_VAL_STR "Error getting value of pending"
+#define AFR_MSG_THIN_ARB_SKIP_SHD_STR "I am not the god shd. skipping."
+#define AFR_MSG_UNKNOWN_SET_STR "Unknown set"
+#define AFR_MSG_NO_XL_ID_STR "xl does not have id"
+#define AFR_MSG_SELF_HEAL_INFO_START_STR "starting full sweep on"
+#define AFR_MSG_SELF_HEAL_INFO_FINISH_STR "finished full sweep on"
+#define AFR_MSG_INCRE_COUNT_STR "Could not increment the counter."
+#define AFR_MSG_ADD_TO_OUTPUT_FAILED_STR "Could not add to output"
+#define AFR_MSG_SET_TIME_FAILED_STR "Could not set time"
+#define AFR_MSG_GFID_HEAL_MSG_STR "Error setting gfid-heal-msg dict"
+#define AFR_MSG_NO_MAJORITY_TO_RESOLVE_STR                                     \
+    "No majority to resolve gfid split brain"
+#define AFR_MSG_GFID_MISMATCH_DETECTED_STR "Gfid mismatch dectected"
+#define AFR_MSG_SELF_HEAL_INFO_STR "performing selfheal"
+#define AFR_MSG_TYPE_MISMATCH_STR "TYPE mismatch"
+#define AFR_MSG_SIZE_POLICY_NOT_APPLICABLE_STR                                 \
+    "Size policy is not applicable to directories."
+#define AFR_MSG_NO_CHILD_SELECTED_STR                                          \
+    "No child selected by favorite-child policy"
+#define AFR_MSG_INVALID_CHILD_STR "Invalid child"
+#define AFR_MSG_RESOLVE_CONFLICTING_DATA_STR                                   \
+    "selected as authentic to resolve conflicting data"
+#define SERROR_GETTING_SRC_BRICK_STR "Error getting the source brick"
+#define SNO_DIFF_IN_MTIME_STR "No difference in mtime"
+#define SNO_BIGGER_FILE_STR "No bigger file"
+#define SALL_BRICKS_UP_TO_RESOLVE_STR                                          \
+    "All the bricks should be up to resolve the gfid split brain"
+#define AFR_MSG_UNLOCK_FAILED_STR "Failed to unlock"
+#define AFR_MSG_POST_OP_FAILED_STR "Post-op on thin-arbiter failed"
+#define AFR_MSG_TA_FRAME_CREATE_FAILED_STR "Failed to create ta_frame"
+#define AFR_MSG_SET_KEY_XATTROP_FAILED_STR "Could not set key during xattrop"
+#define AFR_MSG_BLOCKING_ENTRYLKS_FAILED_STR "Blocking entrylks failed"
+#define AFR_MSG_FSYNC_FAILED_STR "fsync failed"
+#define AFR_MSG_QUORUM_FAIL_STR "quorum is not met"
+#define AFR_MSG_FOP_FAILED_STR "Failing Fop"
+#define AFR_MSG_INVALID_SUBVOL_STR "not a subvolume"
+#define AFR_MSG_VOL_MISCONFIGURED_STR "Volume is dangling"
+#define AFR_MSG_CHILD_MISCONFIGURED_STR                                        \
+    "replicate translator needs more than one subvolume defined"
+#define AFR_MSG_CLEAN_UP_FAILED_STR "Failed to clean up healer threads"
+#define AFR_MSG_QUORUM_OVERRIDE_STR "overriding quorum-count"
+#define AFR_MSG_UNABLE_TO_FETCH_STR                                            \
+    "Unable to fetch afr-pending-xattr option from volfile. Falling back to "  \
+    "using client translator names"
+#define AFR_MSG_NULL_DEREF_STR "possible NULL deref"
+#define AFR_MSG_XATTR_SET_FAILED_STR "Cannot set xattr cookie key"
+#define AFR_MSG_SPLIT_BRAIN_STATUS_STR "Failed to create synctask"
+#define AFR_MSG_SUBVOLS_DOWN_STR "All subvolumes are not up"
+#define AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR_STR                                   \
+    "Failed to cancel split-brain choice"
+#define AFR_MSG_SPLIT_BRAIN_REPLICA_STR                                        \
+    "Cannot set replica. File is not in data/metadata split-brain"
+#define AFR_MSG_INODE_CTX_FAILED_STR "Failed to get inode_ctx"
+#define AFR_MSG_READ_SUBVOL_ERROR_STR "no read subvols"
+#define AFR_MSG_LOCAL_CHILD_STR "selecting local read-child"
+#define AFR_MSG_LOOKUP_FAILED_STR "Failed to lookup/create thin-arbiter id file"
+#define AFR_MSG_TIMER_CREATE_FAIL_STR                                          \
+    "Cannot create timer for delayed initialization"
+#define AFR_MSG_SUBVOL_UP_STR "Subvolume came back up; going online"
+#define AFR_MSG_ALL_SUBVOLS_DOWN_STR                                           \
+    "All subvolumes are down. Going offline until atleast one of them is up"
+#define AFR_MSG_RELEASE_LOCK_FAILED_STR "Failed to release lock"
+#define AFR_MSG_INVALID_CHILD_UP_STR "Received child_up from invalid subvolume"
+#define AFR_MSG_QUORUM_MET_STR "Client-quorum is met"
+#define AFR_MSG_EXPUNGING_FILE_OR_DIR_STR "expunging file or dir"
+#define AFR_MSG_SELF_HEAL_FAILED_STR "Invalid"
+#define AFR_MSG_SPLIT_BRAIN_STR "Skipping conservative mergeon the file"
+#define AFR_MSG_CLEAR_TIME_SPLIT_BRAIN_STR "clear time split brain"
+#define AFR_MSG_READ_FAILED_STR "Failing read since good brick is down"
+#define AFR_MSG_LAUNCH_FAILED_STR "Failed to launch synctask"
+#define AFR_MSG_READ_SUBVOL_NOT_UP_STR                                         \
+    "read subvolume in this generation is not up"
+#define AFR_MSG_INTERNAL_LKS_FAILED_STR                                        \
+    "Unable to work with lk-owner while attempting fop"
+#define AFR_MSG_LOCK_XLATOR_NOT_LOADED_STR                                     \
+    "subvolume does not support locking. please load features/locks xlator "   \
+    "on server."
+#define AFR_MSG_FD_CTX_GET_FAILED_STR "unable to get fd ctx"
+#define AFR_MSG_INFO_COMMON_STR "fd not open on any subvolumes, aborting."
+#define AFR_MSG_REPLACE_BRICK_STATUS_STR "Couldn't acquire lock on any child."
+#define AFR_MSG_NEW_BRICK_STR "New brick"
+#define AFR_MSG_SPLIT_BRAIN_SET_FAILED_STR                                     \
+    "Failed to set split-brain choice to -1"
+#define AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED_STR                               \
+    "Failed to determine split-brain. Aborting split-brain-choice set"
+#define AFR_MSG_OPEN_FAIL_STR "Failed to open subvolume"
+#define AFR_MSG_SET_PEND_XATTR_STR "Set of pending xattr"
+#define AFR_MSG_INTERNAL_ATTR_STR "is an internal extended attribute"
 #endif /* !_AFR_MESSAGES_H_ */
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index 059d3f9bd71..64856042b65 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -8,322 +8,346 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-#include "fd.h"
-
-#include "afr-inode-read.h"
-#include "afr-inode-write.h"
-#include "afr-dir-read.h"
-#include "afr-dir-write.h"
-#include "afr-transaction.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
+#include "afr-transaction.h"
 
 gf_boolean_t
-afr_is_fd_fixable (fd_t *fd)
+afr_is_fd_fixable(fd_t *fd)
 {
-        if (!fd || !fd->inode)
-                return _gf_false;
-        else if (fd_is_anonymous (fd))
-                return _gf_false;
-        else if (gf_uuid_is_null (fd->inode->gfid))
-                return _gf_false;
-
-        return _gf_true;
+    if (!fd || !fd->inode)
+        return _gf_false;
+    else if (fd_is_anonymous(fd))
+        return _gf_false;
+    else if (gf_uuid_is_null(fd->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
-
 int
-afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_open_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t * local = frame->local;
+    afr_local_t *local = frame->local;
 
-        AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                          local->fd, xdata);
-        return 0;
+    AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                     local->cont.open.fd, xdata);
+    return 0;
 }
 
-
 int
-afr_open_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              fd_t *fd, dict_t *xdata)
+afr_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *  local       = NULL;
-        int            call_count  = -1;
-        int            child_index = (long) cookie;
-	afr_fd_ctx_t  *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                } else {
-                        local->op_ret = op_ret;
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                }
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if ((fd_ctx->flags & O_TRUNC) && (local->op_ret >= 0)) {
-                        STACK_WIND (frame, afr_open_ftruncate_cbk,
-                                    this, this->fops->ftruncate,
-                                    fd, 0, NULL);
-                } else {
-                        AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd,
-					  local->xdata_rsp);
-                }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        if (local->op_ret == -1) {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, NULL,
+                             NULL);
+        } else if (fd_ctx->flags & O_TRUNC) {
+            STACK_WIND(frame, afr_open_ftruncate_cbk, this,
+                       this->fops->ftruncate, fd, 0, NULL);
+        } else {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                             local->cont.open.fd, local->xdata_rsp);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+afr_open_continue(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_private_t * priv       = NULL;
-        afr_local_t *   local      = NULL;
-        int             i          = 0;
-        int32_t         call_count = 0;
-        int32_t         op_errno   = 0;
-	afr_fd_ctx_t   *fd_ctx = NULL;
-
-        //We can't let truncation to happen outside transaction.
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx) {
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-        local->fd = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
-	fd_ctx->flags = flags;
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err) {
+        AFR_STACK_UNWIND(open, frame, -1, err, NULL, NULL);
+    } else {
+        local->call_count = AFR_COUNT(local->child_up, priv->child_count);
         call_count = local->call_count;
 
-        local->cont.open.flags = flags;
-
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           loc, (flags & ~O_TRUNC), fd, xdata);
-                        if (!--call_count)
-                                break;
-                }
+            if (local->child_up[i]) {
+                STACK_WIND_COOKIE(frame, afr_open_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->open, &local->loc,
+                                  (local->cont.open.flags & ~O_TRUNC),
+                                  local->cont.open.fd, local->xdata_req);
+                if (!--call_count)
+                    break;
+            }
         }
+    }
+    return 0;
+}
 
-	return 0;
+int
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = 0;
+    int event_generation = 0;
+    int ret = 0;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    // We can't let truncation to happen outside transaction.
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_OPEN;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    loc_copy(&local->loc, loc);
+    local->fd_ctx = fd_ctx;
+    fd_ctx->flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->cont.open.flags = flags;
+    local->cont.open.fd = fd_ref(fd);
+
+    ret = afr_inode_get_readable(frame, local->inode, this, NULL,
+                                 &event_generation, AFR_DATA_TRANSACTION);
+    if ((ret < 0) &&
+        (afr_split_brain_read_subvol_get(local->inode, this, NULL,
+                                         &spb_subvol) == 0) &&
+        spb_subvol < 0) {
+        afr_inode_refresh(frame, this, local->inode, local->inode->gfid,
+                          afr_open_continue);
+    } else {
+        afr_open_continue(frame, this, 0);
+    }
+
+    return 0;
 out:
-	AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, NULL);
+    AFR_STACK_UNWIND(open, frame, -1, op_errno, fd, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, fd_t *fd,
-                         dict_t *xdata)
+afr_openfd_fix_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata)
 {
-        afr_local_t   *local      = NULL;
-        afr_private_t *priv       = NULL;
-        afr_fd_ctx_t  *fd_ctx     = NULL;
-        int           call_count  = 0;
-        int           child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int call_count = 0;
+    int child_index = (long)cookie;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (op_ret >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "fd for %s opened "
+                     "successfully on subvolume %s",
+                     local->loc.path, priv->children[child_index]->name);
+    } else {
+        gf_smsg(this->name, fop_log_level(GF_FOP_OPEN, op_errno), op_errno,
+                AFR_MSG_OPEN_FAIL, "path=%s", local->loc.path, "subvolume=%s",
+                priv->children[child_index]->name, NULL);
+    }
+
+    fd_ctx = local->fd_ctx;
+
+    LOCK(&local->fd->lock);
+    {
         if (op_ret >= 0) {
-                gf_msg_debug (this->name, 0, "fd for %s opened "
-                              "successfully on subvolume %s", local->loc.path,
-                              priv->children[child_index]->name);
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
         } else {
-                gf_msg (this->name, fop_log_level (GF_FOP_OPEN, op_errno),
-                        op_errno, AFR_MSG_OPEN_FAIL, "Failed to open %s on "
-                        "subvolume %s", local->loc.path,
-                        priv->children[child_index]->name);
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
         }
+    }
+    UNLOCK(&local->fd->lock);
 
-        fd_ctx = local->fd_ctx;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-        LOCK (&local->fd->lock);
-        {
-                if (op_ret >= 0) {
-                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-                } else {
-                        fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                }
-        }
-        UNLOCK (&local->fd->lock);
+    return 0;
+}
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                AFR_STACK_DESTROY (frame);
+static int
+afr_fd_ctx_need_open(fd_t *fd, xlator_t *this, unsigned char *need_open)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int count = 0;
+
+    priv = this->private;
 
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
         return 0;
-}
 
+    LOCK(&fd->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+                priv->child_up[i]) {
+                fd_ctx->opened_on[i] = AFR_FD_OPENING;
+                need_open[i] = 1;
+                count++;
+            } else {
+                need_open[i] = 0;
+            }
+        }
+    }
+    UNLOCK(&fd->lock);
 
-static int
-afr_fd_ctx_need_open (fd_t *fd, xlator_t *this, unsigned char *need_open)
-{
-	afr_fd_ctx_t *fd_ctx = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int count = 0;
-
-	priv = this->private;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return 0;
-
-	LOCK (&fd->lock);
-	{
-		for (i = 0; i < priv->child_count; i++) {
-			if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
-			    priv->child_up[i]) {
-				fd_ctx->opened_on[i] = AFR_FD_OPENING;
-				need_open[i] = 1;
-				count++;
-			} else {
-				need_open[i] = 0;
-			}
-		}
-	}
-	UNLOCK (&fd->lock);
-
-	return count;
+    return count;
 }
 
-
 void
-afr_fix_open (fd_t *fd, xlator_t *this)
+afr_fix_open(fd_t *fd, xlator_t *this)
 {
-        afr_private_t *priv    = NULL;
-        int           i        = 0;
-        call_frame_t  *frame   = NULL;
-        afr_local_t   *local   = NULL;
-        int           ret      = -1;
-        int32_t       op_errno = 0;
-        afr_fd_ctx_t  *fd_ctx  = NULL;
-	unsigned char *need_open = NULL;
-	int call_count = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *need_open = NULL;
+    int call_count = 0;
 
-        priv  = this->private;
+    priv = this->private;
 
-        if (!afr_is_fd_fixable (fd))
-                goto out;
+    if (!afr_is_fd_fixable(fd))
+        goto out;
 
-        fd_ctx = afr_fd_ctx_get (fd, this);
-        if (!fd_ctx)
-                goto out;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-	need_open = alloca0 (priv->child_count);
+    need_open = alloca0(priv->child_count);
 
-	call_count = afr_fd_ctx_need_open (fd, this, need_open);
-	if (!call_count)
-		goto out;
+    call_count = afr_fd_ctx_need_open(fd, this, need_open);
+    if (!call_count)
+        goto out;
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->loc.inode = inode_ref (fd->inode);
-        ret = loc_path (&local->loc, NULL);
-        if (ret < 0)
-                goto out;
+    local->loc.inode = inode_ref(fd->inode);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
 
-        local->fd = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        gf_msg_debug (this->name, 0, "need open count: %d",
-                      call_count);
+    gf_msg_debug(this->name, 0, "need open count: %d", call_count);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!need_open[i])
-                        continue;
-
-                if (IA_IFDIR == fd->inode->ia_type) {
-                        gf_msg_debug (this->name, 0,
-                                      "opening fd for dir %s on subvolume %s",
-                                      local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           &local->loc, local->fd,
-                                           NULL);
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                      "opening fd for file %s on subvolume %s",
-                                      local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
-                                           (void *)(long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           &local->loc,
-                                           fd_ctx->flags & (~O_TRUNC),
-                                           local->fd, NULL);
-                }
-
-		if (!--call_count)
-			break;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!need_open[i])
+            continue;
+
+        if (IA_IFDIR == fd->inode->ia_type) {
+            gf_msg_debug(this->name, 0, "opening fd for dir %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, &local->loc,
+                              local->fd, NULL);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "opening fd for file %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->open,
+                              &local->loc, fd_ctx->flags & (~O_TRUNC),
+                              local->fd, NULL);
         }
 
-	return;
+        if (!--call_count)
+            break;
+    }
+
+    return;
 out:
-        if (frame)
-                AFR_STACK_DESTROY (frame);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 }
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
index 32ad6a46d17..6fc2c75145c 100644
--- a/xlators/cluster/afr/src/afr-read-txn.c
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -12,125 +12,328 @@
 #include "afr-transaction.h"
 #include "afr-messages.h"
 
-int
-afr_read_txn_next_subvol (call_frame_t *frame, xlator_t *this)
+void
+afr_pending_read_increment(afr_private_t *priv, int child_index)
+{
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_INC(priv->pending_reads[child_index]);
+}
+
+void
+afr_pending_read_decrement(afr_private_t *priv, int child_index)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int subvol = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->readable[i]) {
-			/* don't even bother trying here.
-			   just mark as attempted and move on. */
-			local->read_attempted[i] = 1;
-			continue;
-		}
-
-		if (!local->read_attempted[i]) {
-			subvol = i;
-			break;
-		}
-	}
-
-	/* If no more subvols were available for reading, we leave
-	   @subvol as -1, which is an indication we have run out of
-	   readable subvols. */
-	if (subvol != -1)
-		local->read_attempted[subvol] = 1;
-	local->readfn (frame, this, subvol);
-
-	return 0;
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_DEC(priv->pending_reads[child_index]);
 }
 
-#define AFR_READ_TXN_SET_ERROR_AND_GOTO(ret, errnum, index, label) \
-        do {                                                      \
-                local->op_ret = ret;                              \
-                local->op_errno = errnum;                          \
-                read_subvol = index;                              \
-                gf_msg (this->name, GF_LOG_ERROR, EIO, AFR_MSG_SPLIT_BRAIN,\
-                        "Failing %s on gfid %s: split-brain observed.",\
-                        gf_fop_list[local->op], uuid_utoa (inode->gfid));\
-                goto label;                                       \
-        } while (0)
+void
+afr_read_txn_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    afr_pending_read_decrement(priv, local->read_subvol);
+    local->read_subvol = subvol;
+    afr_pending_read_increment(priv, subvol);
+    local->readfn(frame, this, subvol);
+}
 
 int
-afr_read_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+afr_read_txn_next_subvol(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	int read_subvol = 0;
-	int event_generation = 0;
-	inode_t *inode = NULL;
-	int ret = -1;
-        int spb_choice = -1;
-
-	local = frame->local;
-	inode = local->inode;
-
-        if (err) {
-                local->op_errno = -err;
-                local->op_ret = -1;
-                read_subvol = -1;
-                goto readfn;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->readable[i]) {
+            /* don't even bother trying here.
+               just mark as attempted and move on. */
+            local->read_attempted[i] = 1;
+            continue;
         }
 
-	ret = afr_inode_get_readable (frame, inode, this, local->readable,
-			              &event_generation,
-				      local->transaction.type);
+        if (!local->read_attempted[i]) {
+            subvol = i;
+            break;
+        }
+    }
 
-	if (ret == -1 || !event_generation)
-		/* Even after refresh, we don't have a good
-		   read subvolume. Time to bail */
-                AFR_READ_TXN_SET_ERROR_AND_GOTO (-1, EIO, -1, readfn);
+    /* If no more subvols were available for reading, we leave
+       @subvol as -1, which is an indication we have run out of
+       readable subvols. */
+    if (subvol != -1)
+        local->read_attempted[subvol] = 1;
+    afr_read_txn_wind(frame, this, subvol);
 
-	read_subvol = afr_read_subvol_select_by_policy (inode, this,
-							local->readable, NULL);
-	if (read_subvol == -1)
-                AFR_READ_TXN_SET_ERROR_AND_GOTO (-1, EIO, -1, readfn);
+    return 0;
+}
 
-	if (local->read_attempted[read_subvol]) {
-		afr_read_txn_next_subvol (frame, this);
-		return 0;
-	}
+static int
+afr_ta_read_txn_done(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
 
-	local->read_attempted[read_subvol] = 1;
-readfn:
-        if (read_subvol == -1) {
-                ret = afr_inode_split_brain_choice_get (inode, this,
-                                                        &spb_choice);
-                if ((ret == 0) && spb_choice >= 0)
-                        read_subvol = spb_choice;
-        }
-	local->readfn (frame, this, read_subvol);
+static int
+afr_ta_read_txn(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    int read_subvol = -1;
+    int query_child = AFR_CHILD_UNKNOWN;
+    int possible_bad_child = AFR_CHILD_UNKNOWN;
+    int ret = 0;
+    int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int **pending = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    frame = (call_frame_t *)opaque;
+    this = frame->this;
+    local = frame->local;
+    priv = this->private;
+    query_child = local->read_txn_query_child;
+
+    if (query_child == AFR_CHILD_ZERO) {
+        possible_bad_child = AFR_CHILD_ONE;
+    } else if (query_child == AFR_CHILD_ONE) {
+        possible_bad_child = AFR_CHILD_ZERO;
+    } else {
+        /*read_txn_query_child is AFR_CHILD_UNKNOWN*/
+        goto out;
+    }
+
+    /* Ask the query_child to see if it blames the possibly bad one. */
+    xdata_req = dict_new();
+    if (!xdata_req)
+        goto out;
+
+    pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!pending)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, xdata_req, pending);
+    if (ret < 0)
+        goto out;
+
+    if (local->fd) {
+        ret = syncop_fxattrop(priv->children[query_child], local->fd,
+                              GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                              NULL);
+    } else {
+        ret = syncop_xattrop(priv->children[query_child], &local->loc,
+                             GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                             NULL);
+    }
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed xattrop for gfid %s on %s",
+               uuid_utoa(local->inode->gfid),
+               priv->children[query_child]->name);
+        op_errno = -ret;
+        goto out;
+    }
+
+    if (afr_ta_dict_contains_pending_xattr(xdata_rsp, priv,
+                                           possible_bad_child)) {
+        read_subvol = query_child;
+        goto out;
+    }
+    dict_unref(xdata_rsp);
+    xdata_rsp = NULL;
+
+    /* It doesn't. So query thin-arbiter to see if it blames any data brick. */
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+    flock.l_type = F_WRLCK; /*start and length are already zero. */
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLKW, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to get AFR_TA_DOM_MODIFY lock on %s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto out;
+    }
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                         NULL);
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed xattrop on %s.", uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto unlock;
+    }
+
+    if (!afr_ta_dict_contains_pending_xattr(xdata_rsp, priv, query_child)) {
+        read_subvol = query_child;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Failing read for gfid %s since good brick %s is down",
+               uuid_utoa(local->inode->gfid),
+               priv->children[possible_bad_child]->name);
+        op_errno = EIO;
+    }
+
+unlock:
+    flock.l_type = F_UNLCK;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to unlock AFR_TA_DOM_MODIFY lock on "
+               "%s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+    }
+out:
+    if (xdata_req)
+        dict_unref(xdata_req);
+    if (xdata_rsp)
+        dict_unref(xdata_rsp);
+    if (pending)
+        afr_matrix_cleanup(pending, priv->child_count);
+    loc_wipe(&loc);
+
+    if (read_subvol == -1) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+    return ret;
+}
 
-	return 0;
+void
+afr_ta_read_txn_synctask(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto out;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_read_txn, afr_ta_read_txn_done,
+                       ta_frame, frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch "
+               "afr_ta_read_txn synctask for gfid %s.",
+               uuid_utoa(local->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        STACK_DESTROY(ta_frame->root);
+        goto out;
+    }
+    return;
+out:
+    afr_read_txn_wind(frame, this, -1);
 }
 
+int
+afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    inode_t *inode = NULL;
+    int ret = -1;
+    int spb_subvol = -1;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err) {
+        if (!priv->thin_arbiter_count)
+            goto readfn;
+        if (err != EINVAL)
+            goto readfn;
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+    if (read_subvol == -1) {
+        err = EIO;
+        goto readfn;
+    }
+
+    if (local->read_attempted[read_subvol]) {
+        afr_read_txn_next_subvol(frame, this);
+        return 0;
+    }
+
+    local->read_attempted[read_subvol] = 1;
+readfn:
+    if (read_subvol == -1) {
+        ret = afr_split_brain_read_subvol_get(inode, this, frame, &spb_subvol);
+        if ((ret == 0) && spb_subvol >= 0)
+            read_subvol = spb_subvol;
+    }
+
+    if (read_subvol == -1) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+
+    return 0;
+}
 
 int
-afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (!local->refreshed) {
-		local->refreshed = _gf_true;
-		afr_inode_refresh (frame, this, local->inode, NULL,
-				   afr_read_txn_refresh_done);
-	} else {
-		afr_read_txn_next_subvol (frame, this);
-	}
+    if (!local->refreshed) {
+        local->refreshed = _gf_true;
+        afr_inode_refresh(frame, this, local->inode, NULL,
+                          afr_read_txn_refresh_done);
+    } else {
+        afr_read_txn_next_subvol(frame, this);
+    }
 
-	return 0;
+    return 0;
 }
 
-
 /* afr_read_txn_wipe:
 
    clean internal variables in @local in order to make
@@ -139,27 +342,26 @@ afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
 */
 
 void
-afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
+afr_read_txn_wipe(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	local->readfn = NULL;
+    local->readfn = NULL;
 
-	if (local->inode)
-		inode_unref (local->inode);
+    if (local->inode)
+        inode_unref(local->inode);
 
-	for (i = 0; i < priv->child_count; i++) {
-		local->read_attempted[i] = 0;
-		local->readable[i] = 0;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        local->read_attempted[i] = 0;
+        local->readable[i] = 0;
+    }
 }
 
-
 /*
   afr_read_txn:
 
@@ -188,87 +390,105 @@ afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
 */
 
 int
-afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
-	      afr_read_txn_wind_t readfn, afr_transaction_type type)
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        unsigned char *data = NULL;
-        unsigned char *metadata = NULL;
-	int read_subvol = -1;
-	int event_generation = 0;
-	int ret = -1;
-
-	priv = this->private;
-	local = frame->local;
-        data = alloca0 (priv->child_count);
-        metadata = alloca0 (priv->child_count);
-
-	afr_read_txn_wipe (frame, this);
-
-	local->readfn = readfn;
-	local->inode = inode_ref (inode);
-
-        if (priv->quorum_reads &&
-            priv->quorum_count && !afr_has_quorum (priv->child_up, this)) {
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-                read_subvol = -1;
-                goto read;
-        }
-
-	local->transaction.type = type;
-        if (local->op == GF_FOP_FSTAT || local->op == GF_FOP_STAT) {
-                ret = afr_inode_read_subvol_get (inode, this, data, metadata,
-                                                 &event_generation);
-                AFR_INTERSECT (local->readable, data, metadata,
-                               priv->child_count);
-        } else {
-                ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
-                                                      &event_generation, type);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data = NULL;
+    unsigned char *metadata = NULL;
+    int read_subvol = -1;
+    int event_generation = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+    data = alloca0(priv->child_count);
+    metadata = alloca0(priv->child_count);
+
+    afr_read_txn_wipe(frame, this);
+
+    local->readfn = readfn;
+    local->inode = inode_ref(inode);
+    local->is_read_txn = _gf_true;
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &local->op_errno)) {
+        local->op_ret = -1;
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        local->op_ret = -1;
+        local->op_errno = -afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count &&
+        AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        if (local->child_up[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (local->child_up[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
         }
-	if (ret == -1)
-		/* very first transaction on this inode */
-		goto refresh;
-
-        gf_msg_debug (this->name, 0, "%s: generation now vs cached: %d, "
-                      "%d", uuid_utoa (inode->gfid), local->event_generation,
-                      event_generation);
-	if (local->event_generation != event_generation)
-		/* servers have disconnected / reconnected, and possibly
-		   rebooted, very likely changing the state of freshness
-		   of copies */
-		goto refresh;
-
-	read_subvol = afr_read_subvol_select_by_policy (inode, this,
-							local->readable, NULL);
-
-	if (read_subvol < 0 || read_subvol > priv->child_count) {
-	        gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN,
-                       "Unreadable subvolume %d found with event generation "
-                       "%d for gfid %s. (Possible split-brain)",
-                        read_subvol, event_generation, uuid_utoa(inode->gfid));
-		goto refresh;
-	}
-
-	if (!local->child_up[read_subvol]) {
-		/* should never happen, just in case */
-	        gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_READ_SUBVOL_ERROR, "subvolume %d is the "
-			"read subvolume in this generation, but is not up",
-			read_subvol);
-		goto refresh;
-	}
-
-	local->read_attempted[read_subvol] = 1;
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        /* very first transaction on this inode */
+        goto refresh;
+    AFR_INTERSECT(local->readable, data, metadata, priv->child_count);
+
+    gf_msg_debug(this->name, 0,
+                 "%s: generation now vs cached: %d, "
+                 "%d",
+                 uuid_utoa(inode->gfid), local->event_generation,
+                 event_generation);
+    if (afr_is_inode_refresh_reqd(inode, this, local->event_generation,
+                                  event_generation))
+        /* servers have disconnected / reconnected, and possibly
+           rebooted, very likely changing the state of freshness
+           of copies */
+        goto refresh;
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+
+    if (read_subvol < 0 || read_subvol > priv->child_count) {
+        gf_msg_debug(this->name, 0,
+                     "Unreadable subvolume %d found "
+                     "with event generation %d for gfid %s.",
+                     read_subvol, event_generation, uuid_utoa(inode->gfid));
+        goto refresh;
+    }
+
+    if (!local->child_up[read_subvol]) {
+        /* should never happen, just in case */
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "subvolume %d is the "
+               "read subvolume in this generation, but is not up",
+               read_subvol);
+        goto refresh;
+    }
+
+    local->read_attempted[read_subvol] = 1;
 
 read:
-	local->readfn (frame, this, read_subvol);
+    afr_read_txn_wind(frame, this, read_subvol);
 
-	return 0;
+    return 0;
 
 refresh:
-	afr_inode_refresh (frame, this, inode, NULL, afr_read_txn_refresh_done);
+    afr_inode_refresh(frame, this, inode, NULL, afr_read_txn_refresh_done);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index a4c0e89e434..a580a1584cc 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -8,350 +8,808 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
 #include "afr-messages.h"
+#include <glusterfs/events.h>
 
 void
-afr_heal_synctask (xlator_t *this, afr_local_t *local);
+afr_heal_synctask(xlator_t *this, afr_local_t *local);
 
 int
-afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx)
 {
-	afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_on = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    dict_t *xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    int i = 0;
+
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+    if (source >= 0 && replies[source].valid && replies[source].op_ret == 0)
+        ia_type = replies[source].poststat.ia_type;
+
+    if (ia_type != IA_INVAL)
+        goto heal;
+
+    /* If ia_type is still invalid, it means either
+     * (a)'source' was -1, i.e. parent dir pending xattrs are in split-brain
+     * (or) (b) The parent dir pending xattrs are all zeroes (i.e. all bricks
+     * are sources) and the 'source' we selected earlier might be the one where
+     * the file is not actually present.
+     *
+     * In both cases, let us pick a brick with a successful reply and use its
+     * ia_type.
+     * */
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            /* case (a) above. */
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        } else {
+            /* case (b) above. */
+            if (i == source)
+                continue;
+            if (sources[i] && replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        }
+    }
+
+heal:
+    /* gfid heal on those subvolumes that do not have gfid associated
+     * with the inode and update those replies.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(gfid) &&
+            !gf_uuid_is_null(replies[i].poststat.ia_gfid) &&
+            replies[i].poststat.ia_type == ia_type)
+            gfid = replies[i].poststat.ia_gfid;
 
-	local = frame->local;
+        if (!gf_uuid_is_null(replies[i].poststat.ia_gfid) ||
+            replies[i].poststat.ia_type != ia_type)
+            continue;
 
-	syncbarrier_wake (&local->barrier);
+        wind_on[i] = 1;
+    }
+
+    if (AFR_COUNT(wind_on, priv->child_count) == 0)
+        return 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    local = frame->local;
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    AFR_ONLIST(wind_on, frame, afr_selfheal_discover_cbk, lookup, &loc, xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_on[i])
+            continue;
+        afr_reply_wipe(&replies[i]);
+        afr_reply_copy(&replies[i], &local->replies[i]);
+    }
+    if (gfid_idx && (*gfid_idx == -1)) {
+        /*Pick a brick where the gifd heal was successful.*/
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                !gf_uuid_is_null(replies[i].poststat.ia_gfid)) {
+                *gfid_idx = i;
+                break;
+            }
+        }
+    }
+out:
+    if (gfid_idx && (*gfid_idx == -1) && (ret == 0) && local) {
+        ret = -afr_final_errno(local, priv);
+    }
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (xdata)
+        dict_unref(xdata);
+
+    return ret;
+}
+
+int
+afr_gfid_sbrain_source_from_src_brick(xlator_t *this, struct afr_reply *replies,
+                                      char *src_brick)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (strcmp(priv->children[i]->name, src_brick) == 0)
+            return i;
+    }
+    return -1;
+}
+
+int
+afr_selfheal_gfid_mismatch_by_majority(struct afr_reply *replies,
+                                       int child_count)
+{
+    int j = 0;
+    int i = 0;
+    int votes;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+
+        votes = 1;
+        for (j = i + 1; j < child_count; j++) {
+            if ((!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                  replies[j].poststat.ia_gfid)))
+                votes++;
+            if (votes > child_count / 2)
+                return i;
+        }
+    }
+
+    return -1;
+}
 
-	return 0;
+int
+afr_gfid_sbrain_source_from_bigger_file(struct afr_reply *replies,
+                                        int child_count)
+{
+    int i = 0;
+    int src = -1;
+    uint64_t size = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (size < replies[i].poststat.ia_size) {
+            src = i;
+            size = replies[i].poststat.ia_size;
+        } else if (replies[i].poststat.ia_size == size) {
+            src = -1;
+        }
+    }
+    return src;
 }
 
+int
+afr_gfid_sbrain_source_from_latest_mtime(struct afr_reply *replies,
+                                         int child_count)
+{
+    int i = 0;
+    int src = -1;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            src = i;
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        } else if ((mtime == replies[i].poststat.ia_mtime) &&
+                   (mtime_nsec == replies[i].poststat.ia_mtime_nsec)) {
+            src = -1;
+        }
+    }
+    return src;
+}
 
 int
-afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      int subvol, dict_t *xattr, dict_t *xdata)
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	loc_t loc = {0, };
+    afr_private_t *priv = NULL;
+    char g1[64] = {
+        0,
+    };
+    char g2[64] = {
+        0,
+    };
+    int up_count = 0;
+    int heal_op = -1;
+    int ret = -1;
+    char *src_brick = NULL;
+
+    *src = -1;
+    priv = this->private;
+    up_count = AFR_COUNT(locked_on, priv->child_count);
+    if (up_count != priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "All the bricks should be up to resolve the gfid split "
+               "barin");
+        if (xdata) {
+            ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                           SALL_BRICKS_UP_TO_RESOLVE);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Error setting"
+                       " gfid-heal-msg dict");
+        }
+        goto out;
+    }
 
-	priv = this->private;
-	local = frame->local;
+    if (xdata) {
+        ret = dict_get_int32_sizen(xdata, "heal-op", &heal_op);
+        if (ret)
+            goto fav_child;
+    } else {
+        goto fav_child;
+    }
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            *src = afr_gfid_sbrain_source_from_bigger_file(replies,
+                                                           priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_BIGGER_FILE);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_BIGGER_FILE);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
 
-	STACK_WIND (frame, afr_selfheal_post_op_cbk, priv->children[subvol],
-		    priv->children[subvol]->fops->xattrop, &loc,
-		    GF_XATTROP_ADD_ARRAY, xattr, xdata);
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            *src = afr_gfid_sbrain_source_from_latest_mtime(replies,
+                                                            priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_DIFF_IN_MTIME);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_DIFF_IN_MTIME);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               "setting gfid-heal-msg dict");
+                }
+            }
+            break;
 
-	syncbarrier_wait (&local->barrier, 1);
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str_sizen(xdata, "child-name", &src_brick);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Error getting the source "
+                       "brick");
+                break;
+            }
+            *src = afr_gfid_sbrain_source_from_src_brick(this, replies,
+                                                         src_brick);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SERROR_GETTING_SRC_BRICK);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SERROR_GETTING_SRC_BRICK);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
 
-        loc_wipe (&loc);
+        default:
+            break;
+    }
+    goto out;
+
+fav_child:
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            *src = afr_sh_fav_by_size(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            *src = afr_sh_fav_by_mtime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            *src = afr_sh_fav_by_ctime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            if (priv->child_count != 2)
+                *src = afr_selfheal_gfid_mismatch_by_majority(
+                    replies, priv->child_count);
+            else
+                *src = -1;
+
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "No majority to resolve "
+                       "gfid split brain");
+            }
+            break;
+        default:
+            break;
+    }
 
-	return 0;
+out:
+    if (*src == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "Gfid mismatch detected for <gfid:%s>/%s>, %s on %s and"
+               " %s on %s.",
+               uuid_utoa(pargfid), bname,
+               uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1),
+               priv->children[child_idx]->name,
+               uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2),
+               priv->children[src_idx]->name);
+        gf_event(EVENT_AFR_SPLIT_BRAIN,
+                 "client-pid=%d;"
+                 "subvol=%s;type=gfid;file="
+                 "<gfid:%s>/%s>;count=2;child-%d=%s;gfid-%d=%s;"
+                 "child-%d=%s;gfid-%d=%s",
+                 this->ctx->cmd_args.client_pid, this->name, uuid_utoa(pargfid),
+                 bname, child_idx, priv->children[child_idx]->name, child_idx,
+                 uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1), src_idx,
+                 priv->children[src_idx]->name, src_idx,
+                 uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2));
+        return -1;
+    }
+    return 0;
 }
 
 int
-afr_check_stale_error (struct afr_reply *replies, afr_private_t *priv)
+afr_selfheal_post_op_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int i = 0;
-        int op_errno = 0;
-        int tmp_errno = 0;
-        int stale_count = 0;
+    afr_local_t *local = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                tmp_errno = replies[i].op_errno;
-                if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
-                        op_errno = afr_higher_errno (op_errno, tmp_errno);
-                        stale_count++;
-                }
+    local = frame->local;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
+
+int
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    local->op_ret = 0;
+
+    STACK_WIND(frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+               priv->children[subvol]->fops->xattrop, &loc,
+               GF_XATTROP_ADD_ARRAY, xattr, xdata);
+
+    syncbarrier_wait(&local->barrier, 1);
+    if (local->op_ret < 0)
+        ret = -local->op_errno;
+
+    loc_wipe(&loc);
+    local->op_ret = 0;
+
+    return ret;
+}
+
+int
+afr_check_stale_error(struct afr_reply *replies, afr_private_t *priv)
+{
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
+    int stale_count = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        tmp_errno = replies[i].op_errno;
+        if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
+            op_errno = afr_higher_errno(op_errno, tmp_errno);
+            stale_count++;
         }
-        if (stale_count != priv->child_count)
-                return -ENOTCONN;
-        else
-                return -op_errno;
+    }
+    if (stale_count != priv->child_count)
+        return -ENOTCONN;
+    else
+        return -op_errno;
+}
+
+int
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
+{
+    int i = (long)cookie;
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (pre)
+        local->replies[i].prestat = *pre;
+    if (post)
+        local->replies[i].poststat = *post;
+    if (xdata)
+        local->replies[i].xdata = dict_ref(xdata);
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
+int
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies)
+{
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
+               &replies[source].poststat,
+               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
+               NULL);
+
+    loc_wipe(&loc);
+
+    return 0;
+}
 
 dict_t *
-afr_selfheal_output_xattr (xlator_t *this, gf_boolean_t is_full_crawl,
-                           afr_transaction_type type, int *output_dirty,
-                           int **output_matrix, int subvol,
-                           int **full_heal_mtx_out)
-{
-	int                j     = 0;
-	int                idx   = 0;
-	int                d_idx = 0;
-	int                ret   = 0;
-	int               *raw   = 0;
-	dict_t            *xattr = NULL;
-	afr_private_t     *priv  = NULL;
-
-	priv = this->private;
-	idx = afr_index_for_transaction_type (type);
-        d_idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-
-	xattr = dict_new ();
-	if (!xattr)
-		return NULL;
-
-	/* clear dirty */
-	raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
-	if (!raw)
-		goto err;
-
-	raw[idx] = hton32 (output_dirty[subvol]);
-	ret = dict_set_bin (xattr, AFR_DIRTY, raw,
-			    sizeof(int) * AFR_NUM_CHANGE_LOGS);
-	if (ret) {
-                GF_FREE (raw);
-		goto err;
-        }
+afr_selfheal_output_xattr(xlator_t *this, gf_boolean_t is_full_crawl,
+                          afr_transaction_type type, int *output_dirty,
+                          int **output_matrix, int subvol,
+                          int **full_heal_mtx_out)
+{
+    int j = 0;
+    int idx = 0;
+    int d_idx = 0;
+    int ret = 0;
+    int *raw = 0;
+    dict_t *xattr = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    xattr = dict_new();
+    if (!xattr)
+        return NULL;
 
-	/* clear/set pending */
-	for (j = 0; j < priv->child_count; j++) {
-		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS,
-				 gf_afr_mt_int32_t);
-		if (!raw)
-			goto err;
-
-		raw[idx] = hton32 (output_matrix[subvol][j]);
-                if (is_full_crawl)
-                        raw[d_idx] = hton32 (full_heal_mtx_out[subvol][j]);
-
-		ret = dict_set_bin (xattr, priv->pending_key[j],
-				    raw, sizeof(int) * AFR_NUM_CHANGE_LOGS);
-		if (ret) {
-                        GF_FREE (raw);
-			goto err;
-                }
-	}
+    /* clear dirty */
+    raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+    if (!raw)
+        goto err;
+
+    raw[idx] = hton32(output_dirty[subvol]);
+    ret = dict_set_bin(xattr, AFR_DIRTY, raw,
+                       sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        GF_FREE(raw);
+        goto err;
+    }
+
+    /* clear/set pending */
+    for (j = 0; j < priv->child_count; j++) {
+        raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+        if (!raw)
+            goto err;
+
+        raw[idx] = hton32(output_matrix[subvol][j]);
+        if (is_full_crawl)
+            raw[d_idx] = hton32(full_heal_mtx_out[subvol][j]);
+
+        ret = dict_set_bin(xattr, priv->pending_key[j], raw,
+                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            GF_FREE(raw);
+            goto err;
+        }
+    }
 
-	return xattr;
+    return xattr;
 err:
-	if (xattr)
-		dict_unref (xattr);
-	return NULL;
+    if (xattr)
+        dict_unref(xattr);
+    return NULL;
 }
 
-
 int
-afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			   unsigned char *sources, unsigned char *sinks,
-			   unsigned char *healed_sinks, afr_transaction_type type,
-			   struct afr_reply *replies, unsigned char *locked_on)
-{
-	afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-	int i = 0;
-	int j = 0;
-	unsigned char *pending = NULL;
-	int *input_dirty = NULL;
-	int **input_matrix = NULL;
-	int **full_heal_mtx_in = NULL;
-	int **full_heal_mtx_out = NULL;
-	int *output_dirty = NULL;
-	int **output_matrix = NULL;
-	dict_t *xattr = NULL;
-	dict_t *xdata = NULL;
-
-	priv = this->private;
-        local = frame->local;
-
-	pending = alloca0 (priv->child_count);
-
-	input_dirty = alloca0 (priv->child_count * sizeof (int));
-	input_matrix = ALLOC_MATRIX (priv->child_count, int);
-	full_heal_mtx_in = ALLOC_MATRIX (priv->child_count, int);
-	full_heal_mtx_out = ALLOC_MATRIX (priv->child_count, int);
-	output_dirty = alloca0 (priv->child_count * sizeof (int));
-	output_matrix = ALLOC_MATRIX (priv->child_count, int);
-
-        xdata = dict_new ();
-        if (!xdata)
-                return -1;
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int j = 0;
+    unsigned char *pending = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int **full_heal_mtx_in = NULL;
+    int **full_heal_mtx_out = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    pending = alloca0(priv->child_count);
+
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_in = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_out = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
+
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
+
+    if (local->need_full_crawl)
+        afr_selfheal_extract_xattr(this, replies, AFR_DATA_TRANSACTION, NULL,
+                                   full_heal_mtx_in);
+
+    for (i = 0; i < priv->child_count; i++)
+        if (sinks[i] && !healed_sinks[i])
+            pending[i] = 1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (pending[j]) {
+                output_matrix[i][j] = 1;
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = 1;
+            } else if (locked_on[j]) {
+                output_matrix[i][j] = -input_matrix[i][j];
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
+            }
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            output_dirty[i] = -input_dirty[i];
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            /* perform post-op only on subvols we had locked
+               and inspected on.
+            */
+            continue;
+        if (undid_pending[i])
+            /* We already unset the pending xattrs in
+             * _afr_fav_child_reset_sink_xattrs(). */
+            continue;
+
+        xattr = afr_selfheal_output_xattr(this, local->need_full_crawl, type,
+                                          output_dirty, output_matrix, i,
+                                          full_heal_mtx_out);
+        if (!xattr) {
+            continue;
+        }
 
-	afr_selfheal_extract_xattr (this, replies, type, input_dirty,
-				    input_matrix);
-
-        if (local->need_full_crawl)
-                afr_selfheal_extract_xattr (this, replies, AFR_DATA_TRANSACTION,
-                                            NULL, full_heal_mtx_in);
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sinks[i] && !healed_sinks[i])
-			pending[i] = 1;
-
-	for (i = 0; i < priv->child_count; i++) {
-		for (j = 0; j < priv->child_count; j++) {
-			if (pending[j]) {
-				output_matrix[i][j] = 1;
-                                if (type == AFR_ENTRY_TRANSACTION)
-                                        full_heal_mtx_out[i][j] = 1;
-			} else {
-				output_matrix[i][j] = -input_matrix[i][j];
-                                if (type == AFR_ENTRY_TRANSACTION)
-                                        full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
-                        }
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!pending[i])
-			output_dirty[i] = -input_dirty[i];
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!locked_on[i])
-			/* perform post-op only on subvols we had locked
-			   and inspected on.
-			*/
-			continue;
-
-		xattr = afr_selfheal_output_xattr (this, local->need_full_crawl,
-                                                   type, output_dirty,
-                                                   output_matrix, i,
-                                                   full_heal_mtx_out);
-		if (!xattr) {
-			continue;
-		}
-
-                if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
-                        if (xdata &&
-                            dict_set_int8 (xdata, GF_XATTROP_PURGE_INDEX, 1))
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        AFR_MSG_DICT_SET_FAILED, "Failed to set"
-                                        " dict value for %s",
-                                        GF_XATTROP_PURGE_INDEX);
-                }
+        if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
+            if (xdata && dict_set_int8(xdata, GF_XATTROP_PURGE_INDEX, 1))
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set"
+                       " dict value for %s",
+                       GF_XATTROP_PURGE_INDEX);
+        }
 
-		afr_selfheal_post_op (frame, this, inode, i, xattr, xdata);
-		dict_unref (xattr);
-	}
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
+        dict_unref(xattr);
+    }
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-	return 0;
+    return 0;
 }
 
-
 void
-afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count)
-{
-	int i = 0;
-	dict_t *xdata = NULL;
-
-	if (dst == src)
-		return;
-
-	for (i = 0; i < count; i++) {
-		dst[i].valid = src[i].valid;
-		dst[i].op_ret = src[i].op_ret;
-		dst[i].op_errno = src[i].op_errno;
-		dst[i].prestat = src[i].prestat;
-		dst[i].poststat = src[i].poststat;
-		dst[i].preparent = src[i].preparent;
-		dst[i].postparent = src[i].postparent;
-		dst[i].preparent2 = src[i].preparent2;
-		dst[i].postparent2 = src[i].postparent2;
-		if (src[i].xdata)
-			xdata = dict_ref (src[i].xdata);
-		else
-			xdata = NULL;
-		if (dst[i].xdata)
-			dict_unref (dst[i].xdata);
-		dst[i].xdata = xdata;
-		memcpy (dst[i].checksum, src[i].checksum,
-			MD5_DIGEST_LENGTH);
-	}
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src)
+{
+    dict_t *xdata = NULL;
+
+    dst->valid = src->valid;
+    dst->op_ret = src->op_ret;
+    dst->op_errno = src->op_errno;
+    dst->prestat = src->prestat;
+    dst->poststat = src->poststat;
+    dst->preparent = src->preparent;
+    dst->postparent = src->postparent;
+    dst->preparent2 = src->preparent2;
+    dst->postparent2 = src->postparent2;
+    if (src->xdata)
+        xdata = dict_ref(src->xdata);
+    else
+        xdata = NULL;
+    if (dst->xdata)
+        dict_unref(dst->xdata);
+    dst->xdata = xdata;
+    if (xdata && dict_get_str_boolean(xdata, "fips-mode-rchecksum",
+                                      _gf_false) == _gf_true) {
+        memcpy(dst->checksum, src->checksum, SHA256_DIGEST_LENGTH);
+    } else {
+        memcpy(dst->checksum, src->checksum, MD5_DIGEST_LENGTH);
+    }
+    dst->fips_mode_rchecksum = src->fips_mode_rchecksum;
 }
 
+void
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count)
+{
+    int i = 0;
+
+    if (dst == src)
+        return;
+
+    for (i = 0; i < count; i++) {
+        afr_reply_copy(&dst[i], &src[i]);
+    }
+}
 
 int
-afr_selfheal_fill_dirty (xlator_t *this, int *dirty, int subvol,
-			 int idx, dict_t *xdata)
+afr_selfheal_fill_dirty(xlator_t *this, int *dirty, int subvol, int idx,
+                        dict_t *xdata)
 {
-	void *pending_raw = NULL;
-	int pending[3] = {0, };
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
 
-        if (!dirty)
-                return 0;
+    if (!dirty)
+        return 0;
 
-	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw))
-		return -1;
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw))
+        return -1;
 
-	if (!pending_raw)
-		return -1;
+    if (!pending_raw)
+        return -1;
 
-	memcpy (pending, pending_raw, sizeof(pending));
+    memcpy(pending, pending_raw, sizeof(pending));
 
-	dirty[subvol] = ntoh32 (pending[idx]);
+    dirty[subvol] = ntoh32(pending[idx]);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol,
-			  int idx, dict_t *xdata)
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata)
 {
-	int i = 0;
-	void *pending_raw = NULL;
-	int pending[3] = {0, };
-	afr_private_t *priv = NULL;
+    int i = 0;
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-        if (!matrix)
-                return 0;
+    if (!matrix)
+        return 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
-			continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
 
-		if (!pending_raw)
-			continue;
+        if (!pending_raw)
+            continue;
 
-		memcpy (pending, pending_raw, sizeof(pending));
+        memcpy(pending, pending_raw, sizeof(pending));
 
-		matrix[subvol][i] = ntoh32 (pending[idx]);
-	}
+        matrix[subvol][i] = ntoh32(pending[idx]);
+    }
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
-			    afr_transaction_type type, int *dirty, int **matrix)
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	dict_t *xdata = NULL;
-	int idx = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    dict_t *xdata = NULL;
+    int idx = -1;
+
+    idx = afr_index_for_transaction_type(type);
 
-	idx = afr_index_for_transaction_type (type);
+    priv = this->private;
 
-	priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].xdata)
-			continue;
+        if (!replies[i].xdata)
+            continue;
 
-		xdata = replies[i].xdata;
+        xdata = replies[i].xdata;
 
-		afr_selfheal_fill_dirty (this, dirty, i, idx, xdata);
-		afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
-	}
+        afr_selfheal_fill_dirty(this, dirty, i, idx, xdata);
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
 
-	return 0;
+    return 0;
 }
 
 /*
@@ -361,449 +819,566 @@ afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
  * This can happen if data was directly modified in the backend or for snapshots
  */
 void
-afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies)
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint64_t size = 0;
-
-        /* Find source with biggest file size */
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (!replies[i].valid || replies[i].op_ret != 0) {
-                        sources[i] = 0;
-                        continue;
-                }
-                if (size <= replies[i].poststat.ia_size) {
-                        size = replies[i].poststat.ia_size;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t size = 0;
+
+    /* Find source with biggest file size */
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
         }
-
-        /* Mark sources with less size as not source */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (size > replies[i].poststat.ia_size)
-                        sources[i] = 0;
+        if (size <= replies[i].poststat.ia_size) {
+            size = replies[i].poststat.ia_size;
         }
+    }
+
+    /* Mark sources with less size as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (size > replies[i].poststat.ia_size)
+            sources[i] = 0;
+    }
 }
 
 void
-afr_mark_latest_mtime_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies)
+afr_mark_latest_mtime_file_as_source(xlator_t *this, unsigned char *sources,
+                                     struct afr_reply *replies)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint32_t mtime = 0;
-        uint32_t mtime_nsec = 0;
-
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (!replies[i].valid || replies[i].op_ret != 0) {
-                        sources[i] = 0;
-                        continue;
-                }
-                if ((mtime < replies[i].poststat.ia_mtime) ||
-                    ((mtime == replies[i].poststat.ia_mtime) &&
-                     (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
-                        mtime = replies[i].poststat.ia_mtime;
-                        mtime_nsec = replies[i].poststat.ia_mtime_nsec;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
         }
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if ((mtime > replies[i].poststat.ia_mtime) ||
-                    ((mtime == replies[i].poststat.ia_mtime) &&
-                     (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
-                        sources[i] = 0;
-                }
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if ((mtime > replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
+            sources[i] = 0;
         }
+    }
 }
 
 void
-afr_mark_active_sinks (xlator_t *this, unsigned char *sources,
-                       unsigned char *locked_on, unsigned char *sinks)
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        memset (sinks, 0, sizeof (*sinks) * priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i] && locked_on[i])
-                        sinks[i] = 1;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] && locked_on[i])
+            sinks[i] = 1;
+        else
+            sinks[i] = 0;
+    }
 }
 
 gf_boolean_t
-afr_dict_contains_heal_op (call_frame_t *frame)
+afr_dict_contains_heal_op(call_frame_t *frame)
 {
-        afr_local_t   *local     = NULL;
-        dict_t        *xdata_req = NULL;
-        int            ret       = 0;
-        int            heal_op   = -1;
+    afr_local_t *local = NULL;
+    dict_t *xdata_req = NULL;
+    int ret = 0;
+    int heal_op = -1;
+
+    local = frame->local;
+    xdata_req = local->xdata_req;
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        return _gf_false;
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp)
+            return _gf_true;
+    }
+    ret = dict_set_sizen_str_sizen(local->xdata_rsp, "sh-fail-msg",
+                                   SFILE_NOT_IN_SPLIT_BRAIN);
+
+    return _gf_true;
+}
 
-        local = frame->local;
-        xdata_req = local->xdata_req;
-        ret = dict_get_int32 (xdata_req, "heal-op", &heal_op);
-        if (ret)
-                return _gf_false;
-        if (local->xdata_rsp == NULL) {
-                local->xdata_rsp = dict_new();
-                if (!local->xdata_rsp)
-                        return _gf_true;
-        }
-        ret = dict_set_str (local->xdata_rsp, "sh-fail-msg",
-                            "File not in split-brain");
+gf_boolean_t
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count)
+{
+    int i = 0;
+
+    for (i = 0; i < child_count; i++)
+        if (replies[i].valid != 1 || replies[i].op_ret != 0)
+            return _gf_false;
 
-        return _gf_true;
+    return _gf_true;
 }
 
 int
-afr_mark_split_brain_source_sinks_by_heal_op (call_frame_t *frame,
-                                   xlator_t *this, unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type, int heal_op)
-{
-        afr_local_t   *local     = NULL;
-        afr_private_t *priv      = NULL;
-        dict_t        *xdata_req = NULL;
-        dict_t        *xdata_rsp = NULL;
-        int            ret       = 0;
-        int            i         = 0;
-        char          *name      = NULL;
-        int            source     = -1;
-
-        local = frame->local;
-        priv = this->private;
-        xdata_req = local->xdata_req;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (locked_on[i])
-                        if (sources[i] || !sinks[i] || !healed_sinks[i]) {
-                                ret = -1;
-                                goto out;
-                        }
-        }
-        if (local->xdata_rsp == NULL) {
-                local->xdata_rsp = dict_new();
-                if (!local->xdata_rsp) {
-                        ret = -1;
-                        goto out;
-                }
+afr_mark_split_brain_source_sinks_by_heal_op(
+    call_frame_t *frame, xlator_t *this, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type, int heal_op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int ret = 0;
+    int i = 0;
+    char *name = NULL;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_on[i])
+            if (sources[i] || !sinks[i] || !healed_sinks[i]) {
+                ret = -1;
+                goto out;
+            }
+    }
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp) {
+            ret = -1;
+            goto out;
         }
-        xdata_rsp = local->xdata_rsp;
-
-        for (i = 0 ; i < priv->child_count; i++)
-                if (locked_on[i])
-                        sources[i] = 1;
-        switch (heal_op) {
+    }
+    xdata_rsp = local->xdata_rsp;
+
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                       SBRAIN_HEAL_NO_GO_MSG);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++)
+        if (locked_on[i])
+            sources[i] = 1;
+    switch (heal_op) {
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-                if (type == AFR_METADATA_TRANSACTION) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Use source-brick option to"
-                                            " heal metadata split-brain");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                afr_mark_largest_file_as_source (this, sources, replies);
-                if (AFR_COUNT (sources, priv->child_count) != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "No bigger file");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                break;
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_largest_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_BIGGER_FILE);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
-                if (type == AFR_METADATA_TRANSACTION) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Use source-brick option to"
-                                            " heal metadata split-brain");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                afr_mark_latest_mtime_file_as_source (this, sources, replies);
-                if (AFR_COUNT (sources, priv->child_count) != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "No difference in mtime");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                break;
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_latest_mtime_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_DIFF_IN_MTIME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_get_str (xdata_req, "child-name", &name);
-                if (ret)
-                        goto out;
-                source = afr_get_child_index_from_name (this, name);
-                if (source < 0) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Invalid brick name");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                if (locked_on[source] != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Brick is not up");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                sources[source] = 1;
-                break;
-        default:
-                ret = -1;
+            ret = dict_get_str_sizen(xdata_req, "child-name", &name);
+            if (ret)
                 goto out;
+            source = afr_get_child_index_from_name(this, name);
+            if (source < 0) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SINVALID_BRICK_NAME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            if (locked_on[source] != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SBRICK_IS_NOT_UP);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            memset(sources, 0, sizeof(*sources) * priv->child_count);
+            sources[source] = 1;
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
         }
-        for (i = 0 ; i < priv->child_count; i++) {
-                if (sources[i]) {
-                        source = i;
-                        break;
-                }
-        }
-        sinks[source] = 0;
-        healed_sinks[source] = 0;
-        ret = source;
+    }
+    sinks[source] = 0;
+    healed_sinks[source] = 0;
+    ret = source;
 out:
-        if (ret < 0)
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-        return ret;
-
+    if (ret < 0)
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+    return ret;
 }
 
 int
-afr_sh_fav_by_majority (xlator_t *this, struct afr_reply *replies,
-                        inode_t *inode)
+afr_sh_fav_by_majority(xlator_t *this, struct afr_reply *replies,
+                       inode_t *inode)
 {
-        afr_private_t   *priv;
-        int             vote_count = -1;
-        int             fav_child = -1;
-        int             i = 0;
-        int             k = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "mtime_sec = %d, size = %lu for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_mtime,
-                                replies[i].poststat.ia_size,
-                                uuid_utoa (inode->gfid));
-                                vote_count = 0;
-                        for (k = 0; k < priv->child_count; k++) {
-                                if ((replies[k].poststat.ia_mtime ==
-                                     replies[i].poststat.ia_mtime) &&
-                                    (replies[k].poststat.ia_size ==
-                                     replies[i].poststat.ia_size)
-                                   ) {
-                                        vote_count++;
-                                }
-                        }
-                        if (vote_count > priv->child_count/2) {
-                                fav_child = i;
-                                break;
-                        }
+    afr_private_t *priv;
+    int vote_count = -1;
+    int fav_child = -1;
+    int i = 0;
+    int k = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime_sec = %" PRId64 ", size = %" PRIu64
+                         " for gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_size, uuid_utoa(inode->gfid));
+            vote_count = 0;
+            for (k = 0; k < priv->child_count; k++) {
+                if ((replies[k].poststat.ia_mtime ==
+                     replies[i].poststat.ia_mtime) &&
+                    (replies[k].poststat.ia_size ==
+                     replies[i].poststat.ia_size)) {
+                    vote_count++;
                 }
+            }
+            if (vote_count > priv->child_count / 2) {
+                fav_child = i;
+                break;
+            }
         }
-        return fav_child;
+    }
+    return fav_child;
 }
 
 /*
  * afr_sh_fav_by_mtime: Choose favorite child by mtime.
  */
 int
-afr_sh_fav_by_mtime (xlator_t *this, struct afr_reply *replies, inode_t *inode)
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
 {
-        afr_private_t *priv;
-        int fav_child = -1;
-        int i = 0;
-        uint32_t cmp_mtime = 0;
-        uint32_t cmp_mtime_nsec = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "mtime = %d, mtime_nsec = %d for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_mtime,
-                                replies[i].poststat.ia_mtime_nsec,
-                                uuid_utoa (inode->gfid));
-                        if (replies[i].poststat.ia_mtime > cmp_mtime) {
-                                cmp_mtime = replies[i].poststat.ia_mtime;
-                                cmp_mtime_nsec =
-                                        replies[i].poststat.ia_mtime_nsec;
-                                fav_child = i;
-                        } else if ((replies[i].poststat.ia_mtime == cmp_mtime)
-                                    && (replies[i].poststat.ia_mtime_nsec >
-                                    cmp_mtime_nsec)) {
-                                cmp_mtime = replies[i].poststat.ia_mtime;
-                                cmp_mtime_nsec =
-                                        replies[i].poststat.ia_mtime_nsec;
-                                fav_child = i;
-                        }
-                }
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_mtime = 0;
+    uint32_t cmp_mtime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime = %" PRId64
+                         ", mtime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_mtime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_mtime > cmp_mtime) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_mtime == cmp_mtime) &&
+                       (replies[i].poststat.ia_mtime_nsec > cmp_mtime_nsec)) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            }
         }
-        return fav_child;
+    }
+    return fav_child;
 }
 
 /*
  * afr_sh_fav_by_ctime: Choose favorite child by ctime.
  */
 int
-afr_sh_fav_by_ctime (xlator_t *this, struct afr_reply *replies, inode_t *inode)
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
 {
-        afr_private_t *priv;
-        int fav_child = -1;
-        int i = 0;
-        uint32_t cmp_ctime = 0;
-        uint32_t cmp_ctime_nsec = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "ctime = %d, ctime_nsec = %d for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_ctime,
-                                replies[i].poststat.ia_ctime_nsec,
-                                uuid_utoa (inode->gfid));
-                        if (replies[i].poststat.ia_ctime > cmp_ctime) {
-                                cmp_ctime = replies[i].poststat.ia_ctime;
-                                cmp_ctime_nsec =
-                                        replies[i].poststat.ia_ctime_nsec;
-                                fav_child = i;
-                        } else if ((replies[i].poststat.ia_ctime == cmp_ctime)
-                                    && (replies[i].poststat.ia_ctime_nsec >
-                                    cmp_ctime_nsec)) {
-                                cmp_ctime = replies[i].poststat.ia_ctime;
-                                cmp_ctime_nsec =
-                                        replies[i].poststat.ia_ctime_nsec;
-                                fav_child = i;
-                        }
-                }
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_ctime = 0;
+    uint32_t cmp_ctime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s ctime = %" PRId64
+                         ", ctime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_ctime,
+                         replies[i].poststat.ia_ctime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_ctime > cmp_ctime) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_ctime == cmp_ctime) &&
+                       (replies[i].poststat.ia_ctime_nsec > cmp_ctime_nsec)) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            }
         }
-        return fav_child;
+    }
+    return fav_child;
 }
 
 /*
- * afr_sh_fav_by_size: Choose favorite child by size.
+ * afr_sh_fav_by_size: Choose favorite child by size
+ * when not all files are of zero size.
  */
 int
-afr_sh_fav_by_size (xlator_t *this, struct afr_reply *replies, inode_t *inode)
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint64_t cmp_sz = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid) {
+            continue;
+        }
+        gf_msg_debug(this->name, 0,
+                     "Child:%s file size = %" PRIu64 " for gfid %s",
+                     priv->children[i]->name, replies[i].poststat.ia_size,
+                     uuid_utoa(inode->gfid));
+        if (replies[i].poststat.ia_type == IA_IFDIR) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+                   "Cannot perform selfheal on %s. "
+                   "Size policy is not applicable to directories.",
+                   uuid_utoa(inode->gfid));
+            break;
+        }
+        if (replies[i].poststat.ia_size > cmp_sz) {
+            cmp_sz = replies[i].poststat.ia_size;
+            fav_child = i;
+        } else if (replies[i].poststat.ia_size == cmp_sz) {
+            fav_child = -1;
+        }
+    }
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "No bigger file");
+    }
+    return fav_child;
+}
+
+int
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str)
 {
-        afr_private_t *priv;
-        int fav_child = -1;
-        int i = 0;
-        uint64_t cmp_sz = 0;
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+
+    priv = this->private;
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        return -1;
+    }
+
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            fav_child = afr_sh_fav_by_size(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "SIZE";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            fav_child = afr_sh_fav_by_ctime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "CTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            fav_child = afr_sh_fav_by_mtime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            fav_child = afr_sh_fav_by_majority(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MAJORITY";
+            }
+            break;
+        case AFR_FAV_CHILD_NONE:
+        default:
+            break;
+    }
 
-        priv = this->private;
+    return fav_child;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "file size = %lu for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_size,
-                                uuid_utoa (inode->gfid));
-                        if (replies[i].poststat.ia_size > cmp_sz) {
-                                cmp_sz = replies[i].poststat.ia_size;
-                                fav_child = i;
-                        }
-                }
-        }
-        return fav_child;
+int
+afr_mark_split_brain_source_sinks_by_policy(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+    char mtime_str[256];
+    char ctime_str[256];
+    char *policy_str = NULL;
+    struct tm *tm_ptr;
+    time_t time;
+
+    priv = this->private;
+
+    fav_child = afr_sh_get_fav_by_policy(this, replies, inode, &policy_str);
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "No child selected by favorite-child policy.");
+    } else if (fav_child > priv->child_count - 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Invalid child (%d) "
+               "selected by policy %s.",
+               fav_child, policy_str);
+    } else if (fav_child >= 0) {
+        time = replies[fav_child].poststat.ia_mtime;
+        tm_ptr = localtime(&time);
+        strftime(mtime_str, sizeof(mtime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+        time = replies[fav_child].poststat.ia_ctime;
+        tm_ptr = localtime(&time);
+        strftime(ctime_str, sizeof(ctime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Source %s selected as authentic to resolve conflicting data "
+               "in file (gfid:%s) by %s (%" PRIu64
+               " bytes @ %s mtime, %s "
+               "ctime).",
+               priv->children[fav_child]->name, uuid_utoa(inode->gfid),
+               policy_str, replies[fav_child].poststat.ia_size, mtime_str,
+               ctime_str);
+
+        sources[fav_child] = 1;
+        sinks[fav_child] = 0;
+        healed_sinks[fav_child] = 0;
+    }
+    return fav_child;
 }
 
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies)
+{
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!replies[i].valid) || (replies[i].op_ret != 0) ||
+            (replies[i].poststat.ia_size != 0))
+            return _gf_false;
+    }
+
+    return _gf_true;
+}
 
 int
-afr_mark_split_brain_source_sinks_by_policy (call_frame_t *frame,
-                                             xlator_t *this,
-                                             inode_t *inode,
-                                             unsigned char *sources,
-                                             unsigned char *sinks,
-                                             unsigned char *healed_sinks,
-                                             unsigned char *locked_on,
-                                             struct afr_reply *replies,
-                                             afr_transaction_type type)
-{
-        afr_private_t *priv = NULL;
-        int fav_child = -1;
-        char mtime_str[256];
-        char ctime_str[256];
-        char *policy_str = NULL;
-        struct tm *tm_ptr;
-        time_t time;
-
-        priv = this->private;
-        if (priv->fav_child_policy == AFR_FAV_CHILD_BY_MAJORITY)  {
-                fav_child = afr_sh_fav_by_majority (this, replies, inode);
-                if (fav_child >= 0)
-                        policy_str = "MAJORITY";
-        } else if (priv->fav_child_policy == AFR_FAV_CHILD_BY_MTIME)  {
-                fav_child = afr_sh_fav_by_mtime (this, replies, inode);
-                if (fav_child >= 0)
-                        policy_str = "MTIME";
-        } else if (priv->fav_child_policy == AFR_FAV_CHILD_BY_CTIME)  {
-                fav_child = afr_sh_fav_by_ctime (this, replies, inode);
-                if (fav_child >= 0)
-                        policy_str = "CTIME";
-        } else if (priv->fav_child_policy == AFR_FAV_CHILD_BY_SIZE)  {
-                fav_child = afr_sh_fav_by_size (this, replies, inode);
-                if (fav_child >= 0)
-                        policy_str = "SIZE";
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+    afr_private_t *priv = this->private;
+    struct iatt stbuf = {
+        0,
+    };
+
+    if ((AFR_COUNT(locked_on, priv->child_count) < priv->child_count) ||
+        (afr_success_count(replies, priv->child_count) < priv->child_count))
+        return -1;
+
+    if (type == AFR_DATA_TRANSACTION) {
+        if (!afr_is_file_empty_on_all_children(priv, replies))
+            return -1;
+        goto mark;
+    }
+
+    /*For AFR_METADATA_TRANSACTION, metadata must be same on all bricks.*/
+    stbuf = replies[0].poststat;
+    for (i = 1; i < priv->child_count; i++) {
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, type)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot)))
+            return -1;
+    }
+    for (i = 1; i < priv->child_count; i++) {
+        if (!afr_xattrs_are_equal(replies[0].xdata, replies[i].xdata))
+            return -1;
+    }
+
+mark:
+    /* data/metadata is same on all bricks. Pick one of them as source. Rest
+     * are sinks.*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            source = i;
+            sources[i] = 1;
+            sinks[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
         }
+        sources[i] = 0;
+        sinks[i] = 1;
+        healed_sinks[i] = 1;
+    }
 
-        if (fav_child > priv->child_count - 1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_SBRAIN_FAV_CHILD_POLICY, "Invalid child (%d) "
-                        "selected by policy %s.", fav_child, policy_str);
-        } else if (fav_child >= 0) {
-                time = replies[fav_child].poststat.ia_mtime;
-                tm_ptr = localtime (&time);
-                strftime (mtime_str, sizeof (mtime_str), "%Y-%m-%d %H:%M:%S",
-                          tm_ptr);
-                time = replies[fav_child].poststat.ia_ctime;
-                tm_ptr = localtime (&time);
-                strftime (ctime_str, sizeof (ctime_str), "%Y-%m-%d %H:%M:%S",
-                          tm_ptr);
-
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_SBRAIN_FAV_CHILD_POLICY, "Source %s "
-                        "selected as authentic to resolve conflicting "
-                         "data in file (gfid:%s) by %s (%lu bytes @ %s mtime, "
-                         "%s ctime).",
-                         priv->children[fav_child]->name,
-                         uuid_utoa (inode->gfid),
-                         policy_str,
-                         replies[fav_child].poststat.ia_size,
-                         mtime_str,
-                         ctime_str);
-
-                sources[fav_child] = 1;
-                sinks[fav_child] = 0;
-                healed_sinks[fav_child] = 0;
-        }
-        return fav_child;
+    return source;
 }
 
 /* Return a source depending on the type of heal_op, and set sources[source],
@@ -814,65 +1389,156 @@ afr_mark_split_brain_source_sinks_by_policy (call_frame_t *frame,
  * sinks[node] are 1. This should be the case if the file is in split-brain.
  */
 int
-afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode,
-                                   unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        dict_t *xdata_req = NULL;
-        int heal_op = -1;
-        int ret = -1;
-
-        local = frame->local;
-        priv = this->private;
-        xdata_req = local->xdata_req;
-
-        ret = dict_get_int32 (xdata_req, "heal-op", &heal_op);
-        if (ret)
-                goto autoheal;
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    int heal_op = -1;
+    int ret = -1;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    source = afr_mark_source_sinks_if_file_empty(
+        this, sources, sinks, healed_sinks, locked_on, replies, type);
+    if (source >= 0)
+        return source;
 
-        ret = afr_mark_split_brain_source_sinks_by_heal_op (frame, this,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                            type, heal_op);
-        return ret;
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        goto autoheal;
+
+    source = afr_mark_split_brain_source_sinks_by_heal_op(
+        frame, this, sources, sinks, healed_sinks, locked_on, replies, type,
+        heal_op);
+    return source;
 
 autoheal:
-        /* Automatically heal if fav_child_policy is set. */
-        if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
-                ret = afr_mark_split_brain_source_sinks_by_policy (frame, this,
-                                                                   inode,
-                                                                   sources,
-                                                                   sinks,
-                                                                   healed_sinks,
-                                                                   locked_on,
-                                                                   replies,
-                                                                   type);
+    /* Automatically heal if fav_child_policy is set. */
+    if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
+        source = afr_mark_split_brain_source_sinks_by_policy(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, type);
+        if (source != -1) {
+            ret = dict_set_int32_sizen(xdata_req, "fav-child-policy", 1);
+            if (ret)
+                return -1;
         }
+    }
 
-        return ret;
+    return source;
+}
+
+int
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (!dict_get_sizen(local->xdata_req, "fav-child-policy"))
+        return 0;
+
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
+
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
+        output_dirty[i] = -input_dirty[i];
+        output_matrix[i][source] = -input_matrix[i][source];
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i] || !locked_on[i])
+            continue;
+        xattr = afr_selfheal_output_xattr(this, _gf_false, type, output_dirty,
+                                          output_matrix, i, NULL);
+
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
+
+        undid_pending[i] = 1;
+        dict_unref(xattr);
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
 
 gf_boolean_t
-afr_does_witness_exist (xlator_t *this, uint64_t *witness)
+afr_does_witness_exist(xlator_t *this, uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (witness[i])
-                        return _gf_true;
+    for (i = 0; i < priv->child_count; i++) {
+        if (witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
+}
+
+unsigned int
+afr_get_quorum_count(afr_private_t *priv)
+{
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        return priv->child_count / 2 + 1;
+    } else {
+        return priv->quorum_count;
+    }
+}
+
+void
+afr_selfheal_post_op_failure_accounting(afr_private_t *priv, char *accused,
+                                        unsigned char *sources,
+                                        unsigned char *locked_on)
+{
+    int i = 0;
+    unsigned int quorum_count = 0;
+
+    if (AFR_COUNT(sources, priv->child_count) != 0)
+        return;
+
+    quorum_count = afr_get_quorum_count(priv);
+    for (i = 0; i < priv->child_count; i++) {
+        if ((accused[i] < quorum_count) && locked_on[i]) {
+            sources[i] = 1;
         }
-        return _gf_false;
+    }
+    return;
 }
 
 /*
@@ -895,691 +1561,711 @@ afr_does_witness_exist (xlator_t *this, uint64_t *witness)
  */
 
 int
-afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
-                             struct afr_reply *replies,
-                             afr_transaction_type type,
-                             unsigned char *locked_on, unsigned char *sources,
-                             unsigned char *sinks, uint64_t *witness,
-                             gf_boolean_t *pflag)
-{
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int j = 0;
-        int *dirty = NULL; /* Denotes if dirty xattr is set */
-        int **matrix = NULL;/* Changelog matrix */
-        char *accused = NULL;/* Accused others without any self-accusal */
-        char *pending = NULL;/* Have pending operations on others */
-        char *self_accused = NULL; /* Accused itself */
-
-	priv = this->private;
-
-	dirty = alloca0 (priv->child_count * sizeof (int));
-	accused = alloca0 (priv->child_count);
-        pending = alloca0 (priv->child_count);
-        self_accused = alloca0 (priv->child_count);
-	matrix = ALLOC_MATRIX(priv->child_count, int);
-        memset (witness, 0, sizeof (*witness) * priv->child_count);
-
-	/* First construct the pending matrix for further analysis */
-	afr_selfheal_extract_xattr (this, replies, type, dirty, matrix);
-
-        if (pflag) {
-                for (i = 0; i < priv->child_count; i++) {
-                        for (j = 0; j < priv->child_count; j++)
-                                if (matrix[i][j])
-                                        *pflag = _gf_true;
-                        if (*pflag)
-                                break;
-                }
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *pflag)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int j = 0;
+    int *dirty = NULL;         /* Denotes if dirty xattr is set */
+    int **matrix = NULL;       /* Changelog matrix */
+    char *accused = NULL;      /* Accused others without any self-accusal */
+    char *pending = NULL;      /* Have pending operations on others */
+    char *self_accused = NULL; /* Accused itself */
+
+    priv = this->private;
+
+    dirty = alloca0(priv->child_count * sizeof(int));
+    accused = alloca0(priv->child_count);
+    pending = alloca0(priv->child_count);
+    self_accused = alloca0(priv->child_count);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+    memset(witness, 0, sizeof(*witness) * priv->child_count);
+
+    /* First construct the pending matrix for further analysis */
+    afr_selfheal_extract_xattr(this, replies, type, dirty, matrix);
+
+    if (pflag) {
+        for (i = 0; i < priv->child_count; i++) {
+            for (j = 0; j < priv->child_count; j++)
+                if (matrix[i][j])
+                    *pflag |= PFLAG_PENDING;
+            if (*pflag)
+                break;
         }
-
-        if (afr_success_count (replies,
-                               priv->child_count) < AFR_SH_MIN_PARTICIPANTS) {
-                /* Treat this just like locks not being acquired */
-                return -ENOTCONN;
+    }
+
+    if (afr_success_count(replies, priv->child_count) < priv->child_count) {
+        /* Treat this just like locks not being acquired */
+        return -ENOTCONN;
+    }
+
+    /* short list all self-accused */
+    for (i = 0; i < priv->child_count; i++) {
+        if (matrix[i][i])
+            self_accused[i] = 1;
+    }
+
+    /* Next short list all accused to exclude them from being sources */
+    /* Self-accused can't accuse others as they are FOOLs */
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j]) {
+                if (!self_accused[i])
+                    accused[j] += 1;
+                if (i != j)
+                    pending[i] += 1;
+            }
         }
+    }
 
-        /* short list all self-accused */
-        for (i = 0; i < priv->child_count; i++) {
-                if (matrix[i][i])
-                        self_accused[i] = 1;
+    /* Short list all non-accused as sources */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!accused[i] && locked_on[i])
+            sources[i] = 1;
+        else
+            sources[i] = 0;
+    }
+
+    /* Everyone accused by non-self-accused sources are sinks */
+    memset(sinks, 0, priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j])
+                sinks[j] = 1;
         }
-
-	/* Next short list all accused to exclude them from being sources */
-        /* Self-accused can't accuse others as they are FOOLs */
-	for (i = 0; i < priv->child_count; i++) {
-		for (j = 0; j < priv->child_count; j++) {
-                        if (matrix[i][j]) {
-                                 if (!self_accused[i])
-                                         accused[j] = 1;
-
-                                 if (i != j)
-                                         pending[i] = 1;
-                         }
-		}
-	}
-
-	/* Short list all non-accused as sources */
-	memset (sources, 0, priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		if (!accused[i] && locked_on[i])
-			sources[i] = 1;
-	}
-
-        /* Everyone accused by non-self-accused sources are sinks */
-        memset (sinks, 0, priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (self_accused[i])
-                        continue;
-                for (j = 0; j < priv->child_count; j++) {
-                        if (matrix[i][j])
-                                sinks[j] = 1;
-                }
+    }
+
+    /* For breaking ties provide with number of fops they witnessed */
+
+    /*
+     * count the pending fops witnessed from itself to others when it is
+     * self-accused
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (i == j)
+                continue;
+            witness[i] += matrix[i][j];
         }
+    }
 
-        /* For breaking ties provide with number of fops they witnessed */
+    if (type == AFR_DATA_TRANSACTION || type == AFR_METADATA_TRANSACTION)
+        afr_selfheal_post_op_failure_accounting(priv, accused, sources,
+                                                locked_on);
 
-        /*
-         * count the pending fops witnessed from itself to others when it is
-         * self-accused
-         */
+    /* If no sources, all locked nodes are sinks - split brain */
+    if (AFR_COUNT(sources, priv->child_count) == 0) {
         for (i = 0; i < priv->child_count; i++) {
-                if (!self_accused[i])
-                        continue;
-                for (j = 0; j < priv->child_count; j++) {
-                        if (i == j)
-                                continue;
-                        witness[i] += matrix[i][j];
-                }
+            if (locked_on[i])
+                sinks[i] = 1;
         }
-
-         /* If no sources, all locked nodes are sinks - split brain */
-         if (AFR_COUNT (sources, priv->child_count) == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (locked_on[i])
-                                sinks[i] = 1;
-                }
+        if (pflag)
+            *pflag |= PFLAG_SBRAIN;
+    }
+
+    /* One more class of witness similar to dirty in v2 is where no pending
+     * exists but we have self-accusing markers. This can happen in afr-v1
+     * if the brick crashes just after doing xattrop on self but
+     * before xattrop on the other xattrs on the brick in pre-op. */
+    if (AFR_COUNT(pending, priv->child_count) == 0) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (self_accused[i])
+                witness[i] += matrix[i][i];
         }
-
-        /* One more class of witness similar to dirty in v2 is where no pending
-         * exists but we have self-accusing markers. This can happen in afr-v1
-         * if the brick crashes just after doing xattrop on self but
-         * before xattrop on the other xattrs on the brick in pre-op. */
-        if (AFR_COUNT (pending, priv->child_count) == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (self_accused[i])
-                                witness[i] += matrix[i][i];
-                }
-        } else {
-                /* In afr-v1 if a file is self-accused and has pending
-                 * operations on others then it is similar to 'dirty' in afr-v2.
-                 * Consider such cases as witness.
-                 */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (self_accused[i] && pending[i])
-                                witness[i] += matrix[i][i];
-                }
+    } else {
+        /* In afr-v1 if a file is self-accused and has pending
+         * operations on others then it is similar to 'dirty' in afr-v2.
+         * Consider such cases as witness.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+            if (self_accused[i] && pending[i])
+                witness[i] += matrix[i][i];
         }
+    }
 
+    /* count the number of dirty fops witnessed */
+    for (i = 0; i < priv->child_count; i++)
+        witness[i] += dirty[i];
 
-        /* count the number of dirty fops witnessed */
-        for (i = 0; i < priv->child_count; i++)
-                witness[i] += dirty[i];
-
-	return 0;
+    return 0;
 }
 
 void
-afr_log_selfheal (uuid_t gfid, xlator_t *this, int ret, char *type,
-                  int source, unsigned char *sources,
-                  unsigned char *healed_sinks)
-{
-        char *status = NULL;
-        char *sinks_str = NULL;
-        char *p = NULL;
-        char *sources_str = NULL;
-        char *q = NULL;
-        afr_private_t *priv = NULL;
-        gf_loglevel_t loglevel = GF_LOG_NONE;
-        int i = 0;
-
-        priv = this->private;
-        sinks_str = alloca0 (priv->child_count * 8);
-        p = sinks_str;
-        sources_str = alloca0 (priv->child_count * 8);
-        q = sources_str;
-        for (i = 0; i < priv->child_count; i++) {
-                if (healed_sinks[i])
-                        p += sprintf (p, "%d ", i);
-                if (sources[i]) {
-                        if (source == i) {
-                                q += sprintf (q, "[%d] ", i);
-                        } else {
-                                q += sprintf (q, "%d ", i);
-                        }
-                }
-        }
-
-        if (ret < 0) {
-                status = "Failed";
-                loglevel = GF_LOG_DEBUG;
-        } else {
-                status = "Completed";
-                loglevel = GF_LOG_INFO;
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks)
+{
+    char *status = NULL;
+    char *sinks_str = NULL;
+    char *p = NULL;
+    char *sources_str = NULL;
+    char *q = NULL;
+    afr_private_t *priv = NULL;
+    gf_loglevel_t loglevel = GF_LOG_NONE;
+    int i = 0;
+
+    priv = this->private;
+    sinks_str = alloca0(priv->child_count * 8);
+    p = sinks_str;
+    sources_str = alloca0(priv->child_count * 8);
+    q = sources_str;
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i])
+            p += sprintf(p, "%d ", i);
+        if (sources[i]) {
+            if (source == i) {
+                q += sprintf(q, "[%d] ", i);
+            } else {
+                q += sprintf(q, "%d ", i);
+            }
         }
-
-        gf_msg (this->name, loglevel, 0,
-                AFR_MSG_SELF_HEAL_INFO, "%s %s selfheal on %s. "
-                "sources=%s sinks=%s", status, type, uuid_utoa (gfid),
-                sources_str, sinks_str);
+    }
+
+    if (ret < 0) {
+        status = "Failed";
+        loglevel = GF_LOG_DEBUG;
+    } else {
+        status = "Completed";
+        loglevel = GF_LOG_INFO;
+    }
+
+    gf_msg(this->name, loglevel, 0, AFR_MSG_SELF_HEAL_INFO,
+           "%s %s selfheal on %s. "
+           "sources=%s sinks=%s",
+           status, type, uuid_utoa(gfid), sources_str, sinks_str);
 }
 
 int
-afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int op_ret, int op_errno, inode_t *inode,
-			   struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
-{
-	afr_local_t *local = NULL;
-	int i = -1;
-        GF_UNUSED int ret = -1;
-	int8_t need_heal = 1;
-
-	local = frame->local;
-	i = (long) cookie;
-
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
-	if (buf)
-		local->replies[i].poststat = *buf;
-	if (parbuf)
-		local->replies[i].postparent = *parbuf;
-	if (xdata) {
-		local->replies[i].xdata = dict_ref (xdata);
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[i].need_heal = need_heal;
-        } else {
-                local->replies[i].need_heal = need_heal;
-        }
-
-	syncbarrier_wake (&local->barrier);
-
-	return 0;
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
+{
+    afr_local_t *local = NULL;
+    int i = -1;
+    GF_UNUSED int ret = -1;
+    int8_t need_heal = 1;
+
+    local = frame->local;
+    i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (buf)
+        local->replies[i].poststat = *buf;
+    if (parbuf)
+        local->replies[i].postparent = *parbuf;
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+    }
+
+    local->replies[i].need_heal = need_heal;
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
-
 inode_t *
-afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
-				 const char *name, struct afr_reply *replies,
-				 unsigned char *lookup_on, dict_t *xattr)
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr)
 {
-	loc_t loc = {0, };
-	dict_t *xattr_req = NULL;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+
+    local = frame->local;
+    priv = frame->this->private;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return NULL;
+
+    if (xattr)
+        dict_copy(xattr, xattr_req);
+
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
-	local = frame->local;
-	priv = frame->this->private;
+    inode = inode_new(parent->table);
+    if (!inode) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
-	xattr_req = dict_new ();
-	if (!xattr_req)
-		return NULL;
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
 
-        if (xattr)
-                dict_copy (xattr, xattr_req);
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
-		dict_destroy (xattr_req);
-		return NULL;
-	}
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-	inode = inode_new (parent->table);
-	if (!inode) {
-		dict_destroy (xattr_req);
-		return NULL;
-	}
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
 
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, parent->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
+    return inode;
+}
 
-	AFR_ONLIST (lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
-		    xattr_req);
+static int
+afr_set_multi_dom_lock_count_request(xlator_t *this, dict_t *dict)
+{
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+
+    ret = dict_set_uint32(dict, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS, 1);
+    if (ret)
+        return ret;
 
-	afr_replies_copy (replies, local->replies, priv->child_count);
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    ret = dict_set_uint32(dict, key1, 1);
+    if (ret)
+        return ret;
 
-	loc_wipe (&loc);
-	dict_unref (xattr_req);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+    ret = dict_set_uint32(dict, key2, 1);
+    if (ret)
+        return ret;
 
-	return inode;
+    return 0;
 }
 
 int
-afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
-				   uuid_t gfid, struct afr_reply *replies,
-				   unsigned char *discover_on)
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict)
 {
-	loc_t loc = {0, };
-	dict_t *xattr_req = NULL;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = frame->this->private;
 
-	local = frame->local;
-	priv = frame->this->private;
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return -ENOMEM;
+    if (dict)
+        dict_copy(dict, xattr_req);
 
-	xattr_req = dict_new ();
-	if (!xattr_req)
-		return -ENOMEM;
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return -ENOMEM;
+    }
 
-	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
-		dict_destroy (xattr_req);
-		return -ENOMEM;
-	}
+    if (afr_set_multi_dom_lock_count_request(frame->this, xattr_req)) {
+        dict_unref(xattr_req);
+        return -1;
+    }
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, gfid);
 
-	AFR_ONLIST (discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
-		    xattr_req);
+    AFR_ONLIST(discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-	afr_replies_copy (replies, local->replies, priv->child_count);
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-	loc_wipe (&loc);
-	dict_unref (xattr_req);
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
 
-	return 0;
+    return 0;
 }
 
 int
-afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
-				uuid_t gfid, struct afr_reply *replies)
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies)
 {
-	afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    local = frame->local;
 
-	priv = frame->this->private;
+    if (local->xattr_req)
+        dict = local->xattr_req;
 
-	return afr_selfheal_unlocked_discover_on (frame, inode, gfid, replies,
-						  priv->child_up);
+    return afr_selfheal_unlocked_discover_on(frame, inode, gfid, replies,
+                                             local->child_up, dict);
 }
 
 unsigned int
-afr_success_count (struct afr_reply *replies, unsigned int count)
+afr_success_count(struct afr_reply *replies, unsigned int count)
 {
-        int     i = 0;
-        unsigned int success = 0;
+    int i = 0;
+    unsigned int success = 0;
 
-        for (i = 0; i < count; i++)
-                if (replies[i].valid && replies[i].op_ret == 0)
-                        success++;
-        return success;
+    for (i = 0; i < count; i++)
+        if (replies[i].valid && replies[i].op_ret == 0)
+            success++;
+    return success;
 }
 
 int
-afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, dict_t *xdata)
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	int i = 0;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-	local = frame->local;
-	i = (long) cookie;
+    local = frame->local;
+    i = (long)cookie;
 
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-	syncbarrier_wake (&local->barrier);
+    syncbarrier_wake(&local->barrier);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_locked_fill (call_frame_t *frame, xlator_t *this,
-                 unsigned char *locked_on)
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int count = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].valid && local->replies[i].op_ret == 0) {
-			locked_on[i] = 1;
-			count++;
-		} else {
-			locked_on[i] = 0;
-		}
-	}
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            locked_on[i] = 1;
+            count++;
+        } else {
+            locked_on[i] = 0;
+        }
+    }
 
-	return count;
+    return count;
 }
 
-
 int
-afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, off_t off, size_t size,
-			 unsigned char *locked_on)
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		    &loc, F_SETLK, &flock, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, off_t off, size_t size,
-		      unsigned char *locked_on)
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
-	afr_local_t *local = NULL;
-	int i = 0;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
-
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		    &loc, F_SETLK, &flock, NULL);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].op_ret == -1 &&
-		    local->replies[i].op_errno == EAGAIN) {
-			afr_locked_fill (frame, this, locked_on);
-			afr_selfheal_uninodelk (frame, this, inode, dom, off,
-						size, locked_on);
-
-			AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
-				 &loc, F_SETLKW, &flock, NULL);
-			break;
-		}
-	}
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_uninodelk(frame, this, inode, dom, off, size,
+                                   locked_on);
+
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                    &flock, NULL);
+            break;
+        }
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 static void
-afr_get_lock_and_eagain_counts (afr_private_t *priv, struct afr_reply *replies,
-                                int *lock_count, int *eagain_count)
-{
-	int i = 0;
-
-	for (i = 0; i < priv->child_count; i++) {
-	        if (!replies[i].valid)
-	                continue;
-	        if (replies[i].op_ret == 0) {
-	                (*lock_count)++;
-                } else if (replies[i].op_ret == -1 &&
-		         replies[i].op_errno == EAGAIN) {
-		        (*eagain_count)++;
-                }
-	}
+afr_get_lock_and_eagain_counts(afr_private_t *priv, struct afr_reply *replies,
+                               int *lock_count, int *eagain_count)
+{
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == 0) {
+            (*lock_count)++;
+        } else if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            (*eagain_count)++;
+        }
+    }
 }
 
 /*Do blocking locks if number of locks acquired is majority and there were some
  * EAGAINs. Useful for odd-way replication*/
 int
-afr_selfheal_tie_breaker_inodelk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, off_t off,
-                                  size_t size, unsigned char *locked_on)
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int lock_count = 0;
-	int eagain_count = 0;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-	priv = this->private;
-	local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		   &loc, F_SETLK, &flock, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-        afr_get_lock_and_eagain_counts (priv, local->replies, &lock_count,
-                                        &eagain_count);
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-	if (lock_count > priv->child_count/2 && eagain_count) {
-                afr_locked_fill (frame, this, locked_on);
-                afr_selfheal_uninodelk (frame, this, inode, dom, off,
-                                        size, locked_on);
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_uninodelk(frame, this, inode, dom, off, size, locked_on);
 
-                AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
-                         &loc, F_SETLKW, &flock, NULL);
-        }
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                &flock, NULL);
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 int
-afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, off_t off, size_t size,
-			const unsigned char *locked_on)
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
-
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_UNLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_UNLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, inodelk,
-		    dom, &loc, F_SETLK, &flock, NULL);
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, inodelk, dom, &loc,
+               F_SETLK, &flock, NULL);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, const char *name, unsigned char *locked_on)
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
+    loc_t loc = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom,
-		   &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, const char *name, unsigned char *locked_on)
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	afr_local_t *local = NULL;
-	int i = 0;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
-		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].op_ret == -1 &&
-		    local->replies[i].op_errno == EAGAIN) {
-			afr_locked_fill (frame, this, locked_on);
-			afr_selfheal_unentrylk (frame, this, inode, dom, name,
-						locked_on, NULL);
-
-			AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
-				 &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-			break;
-		}
-	}
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on,
+                                   NULL);
+
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                    ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+            break;
+        }
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 int
-afr_selfheal_tie_breaker_entrylk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, const char *name,
-                                  unsigned char *locked_on)
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int lock_count = 0;
-	int eagain_count = 0;
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-	priv = this->private;
-	local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
-		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-        afr_get_lock_and_eagain_counts (priv, local->replies, &lock_count,
-                                        &eagain_count);
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-	if (lock_count > priv->child_count/2 && eagain_count) {
-                afr_locked_fill (frame, this, locked_on);
-                afr_selfheal_unentrylk (frame, this, inode, dom, name,
-                                        locked_on, NULL);
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on, NULL);
 
-                AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
-                         &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-	}
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, const char *name, unsigned char *locked_on,
-                        dict_t *xdata)
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata)
 {
-	loc_t loc = {0,};
+    loc_t loc = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, entrylk,
-		    dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+               name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return 0;
+    return 0;
 }
 
-
 gf_boolean_t
-afr_is_pending_set (xlator_t *this, dict_t *xdata, int type)
+afr_is_data_set(xlator_t *this, dict_t *xdata)
 {
-	int idx = -1;
-	afr_private_t *priv = NULL;
-	void *pending_raw = NULL;
-	int *pending_int = NULL;
-	int i = 0;
-
-	priv = this->private;
-	idx = afr_index_for_transaction_type (type);
-
-	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw) == 0) {
-		if (pending_raw) {
-			pending_int = pending_raw;
-
-			if (ntoh32 (pending_int[idx]))
-				return _gf_true;
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (dict_get_ptr (xdata, priv->pending_key[i],
-				  &pending_raw))
-			continue;
-		if (!pending_raw)
-			continue;
-		pending_int = pending_raw;
-
-		if (ntoh32 (pending_int[idx]))
-			return _gf_true;
-	}
-
-	return _gf_false;
+    return afr_is_pending_set(this, xdata, AFR_DATA_TRANSACTION);
 }
 
-
 gf_boolean_t
-afr_is_data_set (xlator_t *this, dict_t *xdata)
+afr_is_metadata_set(xlator_t *this, dict_t *xdata)
 {
-	return afr_is_pending_set (this, xdata, AFR_DATA_TRANSACTION);
+    return afr_is_pending_set(this, xdata, AFR_METADATA_TRANSACTION);
 }
 
 gf_boolean_t
-afr_is_metadata_set (xlator_t *this, dict_t *xdata)
+afr_is_entry_set(xlator_t *this, dict_t *xdata)
 {
-	return afr_is_pending_set (this, xdata, AFR_METADATA_TRANSACTION);
-}
-
-gf_boolean_t
-afr_is_entry_set (xlator_t *this, dict_t *xdata)
-{
-	return afr_is_pending_set (this, xdata, AFR_ENTRY_TRANSACTION);
+    return afr_is_pending_set(this, xdata, AFR_ENTRY_TRANSACTION);
 }
 
 /*
@@ -1592,291 +2278,310 @@ afr_is_entry_set (xlator_t *this, dict_t *xdata)
  */
 
 int
-afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-			       uuid_t gfid, inode_t **link_inode,
-			       gf_boolean_t *data_selfheal,
-			       gf_boolean_t *metadata_selfheal,
-			       gf_boolean_t *entry_selfheal)
-{
-	afr_private_t *priv = NULL;
-        inode_t *inode = NULL;
-	int i = 0;
-	int valid_cnt = 0;
-	struct iatt first = {0, };
-	struct afr_reply *replies = NULL;
-	int ret = -1;
-
-	priv = this->private;
-
-        inode = afr_inode_find (this, gfid);
-        if (!inode)
-                goto out;
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies_dst)
+{
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int i = 0;
+    int valid_cnt = 0;
+    struct iatt first = {
+        0,
+    };
+    int first_idx = 0;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1)
+            continue;
+
+        /* The data segment of the changelog can be non-zero to indicate
+         * the directory needs a full heal. So the check below ensures
+         * it's not a directory before setting the data_selfheal boolean.
+         */
+        if (data_selfheal && !IA_ISDIR(replies[i].poststat.ia_type) &&
+            afr_is_data_set(this, replies[i].xdata))
+            *data_selfheal = _gf_true;
 
-	replies = alloca0 (sizeof (*replies) * priv->child_count);
+        if (metadata_selfheal && afr_is_metadata_set(this, replies[i].xdata))
+            *metadata_selfheal = _gf_true;
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
-	if (ret)
-                goto out;
+        if (entry_selfheal && afr_is_entry_set(this, replies[i].xdata))
+            *entry_selfheal = _gf_true;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-		if (replies[i].op_ret == -1)
-			continue;
-
-                /* The data segment of the changelog can be non-zero to indicate
-                 * the directory needs a full heal. So the check below ensures
-                 * it's not a directory before setting the data_selfheal boolean.
-                 */
-		if (data_selfheal && !IA_ISDIR (replies[i].poststat.ia_type) &&
-                    afr_is_data_set (this, replies[i].xdata))
-			*data_selfheal = _gf_true;
-
-		if (metadata_selfheal &&
-                    afr_is_metadata_set (this, replies[i].xdata))
-			*metadata_selfheal = _gf_true;
-
-		if (entry_selfheal && afr_is_entry_set (this, replies[i].xdata))
-			*entry_selfheal = _gf_true;
-
-		valid_cnt++;
-		if (valid_cnt == 1) {
-			first = replies[i].poststat;
-			continue;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, type)) {
-		        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN,
-				"TYPE mismatch %d vs %d on %s for gfid:%s",
-				(int) first.ia_type,
-				(int) replies[i].poststat.ia_type,
-				priv->children[i]->name,
-				uuid_utoa (replies[i].poststat.ia_gfid));
-                        ret = -EIO;
-                        goto out;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, uid)) {
-		        gf_msg_debug (this->name, 0,
-				      "UID mismatch "
-                                      "%d vs %d on %s for gfid:%s",
-				      (int) first.ia_uid,
-				      (int) replies[i].poststat.ia_uid,
-				      priv->children[i]->name,
-				      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, gid)) {
-		        gf_msg_debug (this->name, 0,
-				      "GID mismatch "
-                                      "%d vs %d on %s for gfid:%s",
-				      (int) first.ia_uid,
-				      (int) replies[i].poststat.ia_uid,
-				      priv->children[i]->name,
-				      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, prot)) {
-		        gf_msg_debug (this->name, 0,
-			              "MODE mismatch "
-                                      "%d vs %d on %s for gfid:%s",
-				      (int) st_mode_from_ia (first.ia_prot, 0),
-				      (int) st_mode_from_ia
-                                      (replies[i].poststat.ia_prot, 0),
-				      priv->children[i]->name,
-				      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (IA_ISREG(first.ia_type) &&
-		    !IA_EQUAL (first, replies[i].poststat, size)) {
-		        gf_msg_debug (this->name, 0,
-                                      "SIZE mismatch "
-                                      "%lld vs %lld on %s for gfid:%s",
-                                      (long long) first.ia_size,
-                                      (long long) replies[i].poststat.ia_size,
-                                      priv->children[i]->name,
-                                      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (data_selfheal)
-                                *data_selfheal = _gf_true;
-		}
-	}
-
-	if (valid_cnt > 0 && link_inode) {
-		*link_inode = inode_link (inode, NULL, NULL, &first);
-                if (!*link_inode) {
-                        ret = -EINVAL;
-                        goto out;
-                }
-        } else if (valid_cnt < 2) {
-                ret = afr_check_stale_error (replies, priv);
-                goto out;
+        valid_cnt++;
+        if (valid_cnt == 1) {
+            first = replies[i].poststat;
+            first_idx = i;
+            continue;
         }
 
-        ret = 0;
-out:
-        if (inode)
-                inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
+        if (!IA_EQUAL(first, replies[i].poststat, type)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "TYPE mismatch %d vs %d on %s for gfid:%s",
+                   (int)first.ia_type, (int)replies[i].poststat.ia_type,
+                   priv->children[i]->name,
+                   uuid_utoa(replies[i].poststat.ia_gfid));
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=file;gfid=%s;"
+                     "ia_type-%d=%s;ia_type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(replies[i].poststat.ia_gfid), first_idx,
+                     gf_inode_type_to_str(first.ia_type), i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type));
+            ret = -EIO;
+            goto out;
+        }
 
-	return ret;
-}
+        if (!IA_EQUAL(first, replies[i].poststat, uid)) {
+            gf_msg_debug(this->name, 0,
+                         "UID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, gid)) {
+            gf_msg_debug(this->name, 0,
+                         "GID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
 
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "MODE mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)st_mode_from_ia(first.ia_prot, 0),
+                         (int)st_mode_from_ia(replies[i].poststat.ia_prot, 0),
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (IA_ISREG(first.ia_type) &&
+            !IA_EQUAL(first, replies[i].poststat, size)) {
+            gf_msg_debug(this->name, 0,
+                         "SIZE mismatch "
+                         "%lld vs %lld on %s for gfid:%s",
+                         (long long)first.ia_size,
+                         (long long)replies[i].poststat.ia_size,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (data_selfheal)
+                *data_selfheal = _gf_true;
+        }
+    }
+
+    if (valid_cnt > 0 && link_inode) {
+        *link_inode = inode_link(inode, NULL, NULL, &first);
+        if (!*link_inode) {
+            ret = -EINVAL;
+            goto out;
+        }
+    } else if (valid_cnt < 2) {
+        ret = afr_check_stale_error(replies, priv);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (replies && replies_dst)
+        afr_replies_copy(replies_dst, replies, priv->child_count);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+
+    return ret;
+}
 
 inode_t *
-afr_inode_find (xlator_t *this, uuid_t gfid)
+afr_inode_find(xlator_t *this, uuid_t gfid)
 {
-	inode_table_t *table = NULL;
-	inode_t *inode = NULL;
+    inode_table_t *table = NULL;
+    inode_t *inode = NULL;
 
-	table = this->itable;
-	if (!table)
-		return NULL;
+    table = this->itable;
+    if (!table)
+        return NULL;
 
-	inode = inode_find (table, gfid);
-	if (inode)
-		return inode;
+    inode = inode_find(table, gfid);
+    if (inode)
+        return inode;
 
-	inode = inode_new (table);
-	if (!inode)
-		return NULL;
+    inode = inode_new(table);
+    if (!inode)
+        return NULL;
 
-	gf_uuid_copy (inode->gfid, gfid);
+    gf_uuid_copy(inode->gfid, gfid);
 
-	return inode;
+    return inode;
 }
 
-
 call_frame_t *
-afr_frame_create (xlator_t *this)
+afr_frame_create(xlator_t *this, int32_t *op_errno)
 {
-	call_frame_t *frame    = NULL;
-	afr_local_t  *local    = NULL;
-	int           op_errno = 0;
-	pid_t         pid      = GF_CLIENT_PID_SELF_HEALD;
-
-	frame = create_frame (this, this->ctx->pool);
-	if (!frame)
-		return NULL;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    pid_t pid = GF_CLIENT_PID_SELF_HEALD;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        return NULL;
+    }
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local) {
-		STACK_DESTROY (frame->root);
-		return NULL;
-	}
+    local = AFR_FRAME_INIT(frame, (*op_errno));
+    if (!local) {
+        STACK_DESTROY(frame->root);
+        return NULL;
+    }
 
-	syncopctx_setfspid (&pid);
+    syncopctx_setfspid(&pid);
 
-	frame->root->pid = pid;
+    frame->root->pid = pid;
 
-	afr_set_lk_owner (frame, this, frame->root);
+    afr_set_lk_owner(frame, this, frame->root);
 
-	return frame;
+    return frame;
 }
 
 int
-afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			    int source, struct afr_reply *replies,
-			    unsigned char *sources, unsigned char *newentry)
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry)
 {
-	int ret = 0;
-	int i = 0;
-	afr_private_t *priv = NULL;
-	dict_t *xattr = NULL;
-	int **changelog = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int **changelog = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	gf_uuid_copy (inode->gfid, replies[source].poststat.ia_gfid);
+    gf_uuid_copy(inode->gfid, replies[source].poststat.ia_gfid);
 
-	xattr = dict_new();
-	if (!xattr)
-		return -ENOMEM;
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
 
-        changelog = afr_mark_pending_changelog (priv, newentry, xattr,
-                                            replies[source].poststat.ia_type);
+    changelog = afr_mark_pending_changelog(priv, newentry, xattr,
+                                           replies[source].poststat.ia_type);
 
-        if (!changelog)
-                goto out;
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i])
-			continue;
-		afr_selfheal_post_op (frame, this, inode, i, xattr, NULL);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret |= afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+    }
 out:
-        if (changelog)
-                afr_matrix_cleanup (changelog, priv->child_count);
-        if (xattr)
-                dict_unref (xattr);
-	return ret;
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
 int
-afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid)
-{
-	int           ret               = -1;
-        int           entry_ret         = 1;
-        int           metadata_ret      = 1;
-        int           data_ret          = 1;
-        int           or_ret            = 0;
-        inode_t      *inode             = NULL;
-	gf_boolean_t  data_selfheal     = _gf_false;
-	gf_boolean_t  metadata_selfheal = _gf_false;
-	gf_boolean_t  entry_selfheal    = _gf_false;
-        afr_private_t *priv            = NULL;
-        gf_boolean_t dataheal_enabled   = _gf_false;
-
-        priv = this->private;
-        gf_string2boolean (priv->data_self_heal, &dataheal_enabled);
-
-	ret = afr_selfheal_unlocked_inspect (frame, this, gfid, &inode,
-					     &data_selfheal,
-					     &metadata_selfheal,
-					     &entry_selfheal);
-	if (ret)
-		goto out;
-
-        if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
-                ret = 2;
-                goto out;
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid)
+{
+    int ret = -1;
+    int entry_ret = 1;
+    int metadata_ret = 1;
+    int data_ret = 1;
+    int or_ret = 0;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, &inode,
+                                        &data_selfheal, &metadata_selfheal,
+                                        &entry_selfheal, NULL);
+    if (ret)
+        goto out;
+
+    if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
+        ret = 2;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFREG) {
+        ret = afr_selfheal_data_open(this, inode, &fd);
+        if (!fd) {
+            ret = -EIO;
+            goto out;
         }
+    }
 
-	if (data_selfheal && dataheal_enabled)
-                data_ret = afr_selfheal_data (frame, this, inode);
+    if (data_selfheal && priv->data_self_heal)
+        data_ret = afr_selfheal_data(frame, this, fd);
 
-	if (metadata_selfheal && priv->metadata_self_heal)
-                metadata_ret = afr_selfheal_metadata (frame, this, inode);
+    if (metadata_selfheal && priv->metadata_self_heal)
+        metadata_ret = afr_selfheal_metadata(frame, this, inode);
 
-	if (entry_selfheal && priv->entry_self_heal)
-                entry_ret = afr_selfheal_entry (frame, this, inode);
+    if (entry_selfheal && priv->entry_self_heal)
+        entry_ret = afr_selfheal_entry(frame, this, inode);
 
-        or_ret = (data_ret | metadata_ret | entry_ret);
+    or_ret = (data_ret | metadata_ret | entry_ret);
 
-        if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
-                ret = -EIO;
-        else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
-                ret = 1;
-        else if (or_ret < 0)
-                ret = or_ret;
-        else
-                ret = 0;
+    if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
+        ret = -EIO;
+    else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
+        ret = 1;
+    else if (or_ret < 0)
+        ret = or_ret;
+    else
+        ret = 0;
 
 out:
-        if (inode)
-                inode_unref (inode);
-        return ret;
+    if (inode)
+        inode_unref(inode);
+    if (fd)
+        fd_unref(fd);
+    return ret;
 }
 /*
  * This is the entry point for healing a given GFID. The return values for this
@@ -1888,155 +2593,342 @@ out:
  */
 
 int
-afr_selfheal (xlator_t *this, uuid_t gfid)
+afr_selfheal(xlator_t *this, uuid_t gfid)
 {
-        int           ret   = -1;
-	call_frame_t *frame = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        return ret;
 
-	frame = afr_frame_create (this);
-	if (!frame)
-		return ret;
+    local = frame->local;
+    local->xdata_req = dict_new();
 
-        ret = afr_selfheal_do (frame, this, gfid);
+    ret = afr_selfheal_do(frame, this, gfid);
 
-	if (frame)
-		AFR_STACK_DESTROY (frame);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 
-	return ret;
+    return ret;
 }
 
-afr_local_t*
-__afr_dequeue_heals (afr_private_t *priv)
+afr_local_t *
+__afr_dequeue_heals(afr_private_t *priv)
 {
-        afr_local_t *local = NULL;
-
-        if (list_empty (&priv->heal_waiting))
-                goto none;
-        if ((priv->background_self_heal_count > 0) &&
-            (priv->healers >= priv->background_self_heal_count))
-                goto none;
-
-        local = list_entry (priv->heal_waiting.next, afr_local_t, healer);
-        priv->heal_waiters--;
-        GF_ASSERT (priv->heal_waiters >= 0);
-        list_del_init(&local->healer);
-        list_add(&local->healer, &priv->healing);
-        priv->healers++;
-        return local;
+    afr_local_t *local = NULL;
+
+    if (list_empty(&priv->heal_waiting))
+        goto none;
+    if ((priv->background_self_heal_count > 0) &&
+        (priv->healers >= priv->background_self_heal_count))
+        goto none;
+
+    local = list_entry(priv->heal_waiting.next, afr_local_t, healer);
+    priv->heal_waiters--;
+    GF_ASSERT(priv->heal_waiters >= 0);
+    list_del_init(&local->healer);
+    list_add(&local->healer, &priv->healing);
+    priv->healers++;
+    return local;
 none:
-        gf_msg_debug (THIS->name, 0, "Nothing dequeued. "
-                      "Num healers: %d, Num Waiters: %d",
-                      priv->healers, priv->heal_waiters);
-        return NULL;
+    gf_msg_debug(THIS->name, 0,
+                 "Nothing dequeued. "
+                 "Num healers: %d, Num Waiters: %d",
+                 priv->healers, priv->heal_waiters);
+    return NULL;
 }
 
 int
-afr_refresh_selfheal_wrap (void *opaque)
+afr_refresh_selfheal_wrap(void *opaque)
 {
-        call_frame_t *heal_frame = opaque;
-        afr_local_t *local = heal_frame->local;
-        int ret = 0;
+    call_frame_t *heal_frame = opaque;
+    afr_local_t *local = heal_frame->local;
+    int ret = 0;
 
-        ret = afr_selfheal (heal_frame->this, local->refreshinode->gfid);
-        return ret;
+    ret = afr_selfheal(heal_frame->this, local->refreshinode->gfid);
+    return ret;
 }
 
 int
-afr_refresh_heal_done (int ret, call_frame_t *frame, void *opaque)
-{
-        call_frame_t *heal_frame = opaque;
-        xlator_t *this = heal_frame->this;
-        afr_private_t *priv = this->private;
-        afr_local_t *local = heal_frame->local;
-
-        LOCK (&priv->lock);
-        {
-                list_del_init(&local->healer);
-                priv->healers--;
-                GF_ASSERT (priv->healers >= 0);
-                local = __afr_dequeue_heals (priv);
-        }
-        UNLOCK (&priv->lock);
+afr_refresh_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    call_frame_t *heal_frame = opaque;
+    xlator_t *this = heal_frame->this;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = heal_frame->local;
 
-        if (heal_frame)
-                AFR_STACK_DESTROY (heal_frame);
+    LOCK(&priv->lock);
+    {
+        list_del_init(&local->healer);
+        priv->healers--;
+        GF_ASSERT(priv->healers >= 0);
+        local = __afr_dequeue_heals(priv);
+    }
+    UNLOCK(&priv->lock);
 
-        if (local)
-                afr_heal_synctask (this, local);
-        return 0;
+    AFR_STACK_DESTROY(heal_frame);
 
+    if (local)
+        afr_heal_synctask(this, local);
+    return 0;
 }
 
 void
-afr_heal_synctask (xlator_t *this, afr_local_t *local)
+afr_heal_synctask(xlator_t *this, afr_local_t *local)
+{
+    int ret = 0;
+    call_frame_t *heal_frame = NULL;
+
+    heal_frame = local->heal_frame;
+    ret = synctask_new(this->ctx->env, afr_refresh_selfheal_wrap,
+                       afr_refresh_heal_done, heal_frame, heal_frame);
+    if (ret < 0)
+        /* Heal not launched. Will be queued when the next inode
+         * refresh happens and shd hasn't healed it yet. */
+        afr_refresh_heal_done(ret, heal_frame, heal_frame);
+}
+
+gf_boolean_t
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this)
 {
-        int ret = 0;
-        call_frame_t *heal_frame = NULL;
+    gf_boolean_t can_heal = _gf_true;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    LOCK(&priv->lock);
+    {
+        if ((priv->background_self_heal_count > 0) &&
+            (priv->heal_wait_qlen + priv->background_self_heal_count) >
+                (priv->heal_waiters + priv->healers)) {
+            list_add_tail(&local->healer, &priv->heal_waiting);
+            priv->heal_waiters++;
+            local = __afr_dequeue_heals(priv);
+        } else {
+            can_heal = _gf_false;
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (can_heal) {
+        if (local)
+            afr_heal_synctask(this, local);
+        else
+            gf_msg_debug(this->name, 0,
+                         "Max number of heals are "
+                         "pending, background self-heal rejected.");
+    }
 
-        heal_frame = local->heal_frame;
-        ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,
-                            afr_refresh_heal_done, heal_frame, heal_frame);
-        if (ret < 0)
-                /* Heal not launched. Will be queued when the next inode
-                 * refresh happens and shd hasn't healed it yet. */
-                afr_refresh_heal_done (ret, heal_frame, heal_frame);
+    return can_heal;
 }
 
-void
-afr_throttled_selfheal (call_frame_t *frame, xlator_t *this)
-{
-        gf_boolean_t can_heal = _gf_true;
-        afr_private_t *priv = this->private;
-        afr_local_t *local = frame->local;
-
-        LOCK (&priv->lock);
-        {
-                if ((priv->background_self_heal_count > 0) &&
-                    (priv->heal_wait_qlen + priv->background_self_heal_count) >
-                    (priv->heal_waiters + priv->healers)) {
-                        list_add_tail(&local->healer, &priv->heal_waiting);
-                        priv->heal_waiters++;
-                        local = __afr_dequeue_heals (priv);
-                } else {
-                        can_heal = _gf_false;
-                }
+int
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+
+    /* Give preference to local child to save on bandwidth */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->local[i] && sources[i]) {
+            if ((type == AFR_DATA_TRANSACTION) && AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+
+            source = i;
+            goto out;
         }
-        UNLOCK (&priv->lock);
-
-        if (can_heal) {
-                if (local)
-                        afr_heal_synctask (this, local);
-                else
-                        gf_msg_debug (this->name, 0, "Max number of heals are "
-                                      "pending, background self-heal rejected.");
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            goto out;
         }
+    }
+out:
+    return source;
+}
+
+static int
+afr_anon_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->replies[i].poststat = *buf;
+        local->replies[i].preparent = *preparent;
+        local->replies[i].postparent = *postparent;
+    }
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
 int
-afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
-                             afr_transaction_type type)
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode)
 {
-        int source = -1;
-        int i      = 0;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    unsigned char *mkdir_on = alloca0(priv->child_count);
+    unsigned char *lookup_on = alloca0(priv->child_count);
+    loc_t loc = {0};
+    int32_t op_errno = 0;
+    int32_t child_op_errno = 0;
+    struct iatt iatt = {0};
+    dict_t *xdata = NULL;
+    uuid_t anon_inode_gfid = {0};
+    int mkdir_count = 0;
+    int i = 0;
+
+    /*Try to mkdir everywhere and return success if the dir exists on 'child'
+     */
+
+    if (!priv->use_anon_inode) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &op_errno);
+    if (op_errno) {
+        goto out;
+    }
+    local = frame->local;
+    if (!local->child_up[child]) {
+        /*Other bricks may need mkdir so don't error out yet*/
+        child_op_errno = ENOTCONN;
+    }
+    gf_uuid_parse(priv->anon_gfid_str, anon_inode_gfid);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+
+        if (priv->anon_inode[i]) {
+            mkdir_on[i] = 0;
+        } else {
+            mkdir_on[i] = 1;
+            mkdir_count++;
+        }
+    }
 
-        /* Give preference to local child to save on bandwidth */
-        for (i = 0; i < priv->child_count; i++) {
-                if (priv->local[i] && sources[i]) {
-                        if ((type == AFR_DATA_TRANSACTION) &&
-                            AFR_IS_ARBITER_BRICK (priv, i))
-                                continue;
+    if (mkdir_count == 0) {
+        *linked_inode = inode_find(this->itable, anon_inode_gfid);
+        if (*linked_inode) {
+            op_errno = 0;
+            goto out;
+        }
+    }
+
+    loc.parent = inode_ref(this->itable->root);
+    loc.name = priv->anon_inode_name;
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_errno = -dict_set_gfuuid(xdata, "gfid-req", anon_inode_gfid, _gf_true);
+    if (op_errno) {
+        goto out;
+    }
+
+    if (mkdir_count == 0) {
+        memcpy(lookup_on, local->child_up, priv->child_count);
+        goto lookup;
+    }
+
+    AFR_ONLIST(mkdir_on, frame, afr_anon_inode_mkdir_cbk, mkdir, &loc, 0755, 0,
+               xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!mkdir_on[i]) {
+            continue;
+        }
 
-                        source = i;
-                        goto out;
-                }
+        if (local->replies[i].op_ret == 0) {
+            priv->anon_inode[i] = 1;
+            iatt = local->replies[i].poststat;
+        } else if (local->replies[i].op_ret < 0 &&
+                   local->replies[i].op_errno == EEXIST) {
+            lookup_on[i] = 1;
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+
+    if (AFR_COUNT(lookup_on, priv->child_count) == 0) {
+        goto link;
+    }
+
+lookup:
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xdata);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!lookup_on[i]) {
+            continue;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i]) {
-                        source = i;
-                        goto out;
-                }
+        if (local->replies[i].op_ret == 0) {
+            if (gf_uuid_compare(anon_inode_gfid,
+                                local->replies[i].poststat.ia_gfid) == 0) {
+                priv->anon_inode[i] = 1;
+                iatt = local->replies[i].poststat;
+            } else {
+                if (i == child)
+                    child_op_errno = EINVAL;
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_DATA,
+                       "%s has gfid: %s", priv->anon_inode_name,
+                       uuid_utoa(local->replies[i].poststat.ia_gfid));
+            }
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+link:
+    if (!gf_uuid_is_null(iatt.ia_gfid)) {
+        *linked_inode = inode_link(loc.inode, loc.parent, loc.name, &iatt);
+        if (*linked_inode) {
+            op_errno = 0;
+            inode_lookup(*linked_inode);
+        } else {
+            op_errno = ENOMEM;
         }
+        goto out;
+    }
+
 out:
-        return source;
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    /*child_op_errno takes precedence*/
+    if (child_op_errno == 0) {
+        child_op_errno = op_errno;
+    }
+
+    if (child_op_errno && *linked_inode) {
+        inode_unref(*linked_inode);
+        *linked_inode = NULL;
+    }
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return -child_op_errno;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 2a33e53764c..37bcc2b3f9e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -8,614 +8,592 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
 #include "afr-messages.h"
-
-enum {
-	AFR_SELFHEAL_DATA_FULL = 0,
-	AFR_SELFHEAL_DATA_DIFF,
-};
-
+#include <glusterfs/events.h>
 
 #define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
 static int
-__checksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, uint32_t weak, uint8_t *strong,
-		dict_t *xdata)
+__checksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, uint32_t weak, uint8_t *strong, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-        struct afr_reply *replies = NULL;
-	int i = (long) cookie;
-
-	local = frame->local;
-        replies = local->replies;
-
-	replies[i].valid = 1;
-	replies[i].op_ret = op_ret;
-	replies[i].op_errno = op_errno;
-        if (xdata)
-                replies[i].buf_has_zeroes = dict_get_str_boolean (xdata,
-                                                   "buf-has-zeroes", _gf_false);
-	if (strong)
-		memcpy (local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
-
-	syncbarrier_wake (&local->barrier);
-	return 0;
-}
-
-
-static int
-attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	  int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
-	  dict_t *xdata)
-{
-	int i = (long) cookie;
-	afr_local_t *local = NULL;
-
-	local = frame->local;
-
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
-	if (pre)
-		local->replies[i].prestat = *pre;
-	if (post)
-		local->replies[i].poststat = *post;
-	if (xdata)
-		local->replies[i].xdata = dict_ref (xdata);
-
-	syncbarrier_wake (&local->barrier);
+    afr_local_t *local = NULL;
+    struct afr_reply *replies = NULL;
+    int i = (long)cookie;
+
+    local = frame->local;
+    replies = local->replies;
+
+    replies[i].valid = 1;
+    replies[i].op_ret = op_ret;
+    replies[i].op_errno = op_errno;
+    if (xdata) {
+        replies[i].buf_has_zeroes = dict_get_str_boolean(
+            xdata, "buf-has-zeroes", _gf_false);
+        replies[i].fips_mode_rchecksum = dict_get_str_boolean(
+            xdata, "fips-mode-rchecksum", _gf_false);
+    }
+    if (strong) {
+        if (replies[i].fips_mode_rchecksum) {
+            memcpy(local->replies[i].checksum, strong, SHA256_DIGEST_LENGTH);
+        } else {
+            memcpy(local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
+        }
+    }
 
-	return 0;
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
-
 static gf_boolean_t
-__afr_can_skip_data_block_heal (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                                int source, unsigned char *healed_sinks,
-				off_t offset, size_t size,
-                                struct iatt *poststat)
+__afr_can_skip_data_block_heal(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size, struct iatt *poststat)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	unsigned char *wind_subvols = NULL;
-        gf_boolean_t checksum_match = _gf_true;
-        dict_t *xdata = NULL;
-	int i = 0;
-
-	priv = this->private;
-	local = frame->local;
-        xdata = dict_new();
-        if (xdata)
-                i = dict_set_int32 (xdata, "check-zero-filled", 1);
-	wind_subvols = alloca0 (priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || healed_sinks[i])
-			wind_subvols[i] = 1;
-	}
-
-	AFR_ONLIST (wind_subvols, frame, __checksum_cbk, rchecksum, fd,
-		    offset, size, xdata);
-        if (xdata)
-                dict_unref (xdata);
-
-	if (!local->replies[source].valid || local->replies[source].op_ret != 0)
-		return _gf_false;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source)
-			continue;
-                if (local->replies[i].valid) {
-                        if (memcmp (local->replies[source].checksum,
-                                    local->replies[i].checksum,
-                                    MD5_DIGEST_LENGTH)) {
-                                checksum_match = _gf_false;
-                                break;
-                        }
-                }
-	}
-
-        if (checksum_match) {
-                if (HAS_HOLES (poststat))
-                        return _gf_true;
-
-                /* For non-sparse files, we might be better off writing the
-                 * zeroes to sinks to avoid mismatch of disk-usage in bricks. */
-                if (local->replies[source].buf_has_zeroes)
-                        return _gf_false;
-                else
-                        return _gf_true;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_subvols = NULL;
+    gf_boolean_t checksum_match = _gf_true;
+    struct afr_reply *replies = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+    if (dict_set_int32_sizen(xdata, "check-zero-filled", 1)) {
+        dict_unref(xdata);
+        goto out;
+    }
+
+    wind_subvols = alloca0(priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || healed_sinks[i])
+            wind_subvols[i] = 1;
+    }
+
+    AFR_ONLIST(wind_subvols, frame, __checksum_cbk, rchecksum, fd, offset, size,
+               xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    if (!replies[source].valid || replies[source].op_ret != 0)
+        return _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+        if (replies[i].valid) {
+            if (memcmp(replies[source].checksum, replies[i].checksum,
+                       replies[source].fips_mode_rchecksum
+                           ? SHA256_DIGEST_LENGTH
+                           : MD5_DIGEST_LENGTH)) {
+                checksum_match = _gf_false;
+                break;
+            }
         }
+    }
 
-        return _gf_false;
-}
+    if (checksum_match) {
+        if (HAS_HOLES(poststat))
+            return _gf_true;
 
+        /* For non-sparse files, we might be better off writing the
+         * zeroes to sinks to avoid mismatch of disk-usage in bricks. */
+        if (local->replies[source].buf_has_zeroes)
+            return _gf_false;
+        else
+            return _gf_true;
+    }
+out:
+    return _gf_false;
+}
 
 static gf_boolean_t
-__afr_is_sink_zero_filled (xlator_t *this, fd_t *fd, size_t size,
-                           off_t offset, int sink)
+__afr_is_sink_zero_filled(xlator_t *this, fd_t *fd, size_t size, off_t offset,
+                          int sink)
 {
-        afr_private_t *priv   = NULL;
-        struct iobref *iobref = NULL;
-        struct iovec  *iovec  = NULL;
-        int            count  = 0;
-        int            ret    = 0;
-        gf_boolean_t   zero_filled   = _gf_false;
-
-        priv = this->private;
-        ret = syncop_readv (priv->children[sink], fd, size, offset, 0, &iovec,
-                            &count, &iobref, NULL, NULL);
-        if (ret < 0)
-                goto out;
-        ret = iov_0filled (iovec, count);
-        if (!ret)
-                zero_filled = _gf_true;
+    afr_private_t *priv = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec *iovec = NULL;
+    int count = 0;
+    int ret = 0;
+    gf_boolean_t zero_filled = _gf_false;
+
+    priv = this->private;
+    ret = syncop_readv(priv->children[sink], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+    ret = iov_0filled(iovec, count);
+    if (!ret)
+        zero_filled = _gf_true;
 out:
-        if (iovec)
-                GF_FREE (iovec);
-        if (iobref)
-                iobref_unref (iobref);
-        return zero_filled;
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+    return zero_filled;
 }
 
 static int
-__afr_selfheal_data_read_write (call_frame_t *frame, xlator_t *this, fd_t *fd,
-				int source, unsigned char *healed_sinks,
-				off_t offset, size_t size,
-				struct afr_reply *replies, int type)
+__afr_selfheal_data_read_write(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size,
+                               struct afr_reply *replies, int type)
 {
-	struct iovec *iovec = NULL;
-	int count = 0;
-	struct iobref *iobref = NULL;
-	int ret = 0;
-	int i = 0;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
-
-	ret = syncop_readv (priv->children[source], fd, size, offset, 0,
-			    &iovec, &count, &iobref, NULL, NULL);
-	if (ret <= 0)
-		return ret;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!healed_sinks[i])
-			continue;
-
-		/*
-		 * TODO: Use fiemap() and discard() to heal holes
-		 * in the future.
-		 *
-		 * For now,
-		 *
-		 * - if the source had any holes at all,
-		 * AND
-		 * - if we are writing past the original file size
-		 *   of the sink
-		 * AND
-		 * - is NOT the last block of the source file. if
-		 *   the block contains EOF, it has to be written
-		 *   in order to set the file size even if the
-		 *   last block is 0-filled.
-		 * AND
-		 * - if the read buffer is filled with only 0's
-		 *
-		 * then, skip writing to this source. We don't depend
-		 * on the write to happen to update the size as we
-		 * have performed an ftruncate() upfront anyways.
-		 */
-#define is_last_block(o,b,s) ((s >= o) && (s <= (o + b)))
-		if (HAS_HOLES ((&replies[source].poststat)) &&
-		    offset >= replies[i].poststat.ia_size &&
-		    !is_last_block (offset, size,
-				    replies[source].poststat.ia_size) &&
-		    (iov_0filled (iovec, count) == 0))
-			continue;
-
-                /* Avoid filling up sparse regions of the sink with 0-filled
-                 * writes.*/
-                if (type == AFR_SELFHEAL_DATA_FULL &&
-                    HAS_HOLES ((&replies[source].poststat)) &&
-                    ((offset + size) <= replies[i].poststat.ia_size) &&
-                    (iov_0filled (iovec, count) == 0) &&
-                    __afr_is_sink_zero_filled (this, fd, size, offset, i)) {
-                                continue;
-                }
-
-		ret = syncop_writev (priv->children[i], fd, iovec, count,
-				     offset, iobref, 0, NULL, NULL);
-		if (ret != iov_length (iovec, count)) {
-			/* write() failed on this sink. unset the corresponding
-			   member in sinks[] (which is healed_sinks[] in the
-			   caller) so that this server does NOT get considered
-			   as successfully healed.
-			*/
-			healed_sinks[i] = 0;
-		}
-	}
-        if (iovec)
-                GF_FREE (iovec);
-	if (iobref)
-		iobref_unref (iobref);
-
-	return ret;
+    struct iovec *iovec = NULL;
+    int count = 0;
+    struct iobref *iobref = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = syncop_readv(priv->children[source], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret <= 0)
+        return ret;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+
+            /*
+             * TODO: Use fiemap() and discard() to heal holes
+             * in the future.
+             *
+             * For now,
+             *
+             * - if the source had any holes at all,
+             * AND
+             * - if we are writing past the original file size
+             *   of the sink
+             * AND
+             * - is NOT the last block of the source file. if
+             *   the block contains EOF, it has to be written
+             *   in order to set the file size even if the
+             *   last block is 0-filled.
+             * AND
+             * - if the read buffer is filled with only 0's
+             *
+             * then, skip writing to this source. We don't depend
+             * on the write to happen to update the size as we
+             * have performed an ftruncate() upfront anyways.
+             */
+#define is_last_block(o, b, s) ((s >= o) && (s <= (o + b)))
+        if (HAS_HOLES((&replies[source].poststat)) &&
+            offset >= replies[i].poststat.ia_size &&
+            !is_last_block(offset, size, replies[source].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0))
+            continue;
+
+        /* Avoid filling up sparse regions of the sink with 0-filled
+         * writes.*/
+        if (type == AFR_SELFHEAL_DATA_FULL &&
+            HAS_HOLES((&replies[source].poststat)) &&
+            ((offset + size) <= replies[i].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0) &&
+            __afr_is_sink_zero_filled(this, fd, size, offset, i)) {
+            continue;
+        }
+
+        ret = syncop_writev(priv->children[i], fd, iovec, count, offset, iobref,
+                            0, NULL, NULL, NULL, NULL);
+        if (ret != iov_length(iovec, count)) {
+            /* write() failed on this sink. unset the corresponding
+               member in sinks[] (which is healed_sinks[] in the
+               caller) so that this server does NOT get considered
+               as successfully healed.
+            */
+            healed_sinks[i] = 0;
+        }
+    }
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+
+    return ret;
 }
 
-static int
-afr_selfheal_data_block (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			 int source, unsigned char *healed_sinks, off_t offset,
-			 size_t size, int type, struct afr_reply *replies)
+static gf_boolean_t
+afr_source_sinks_locked(xlator_t *this, unsigned char *locked_on, int source,
+                        unsigned char *healed_sinks)
 {
-	int ret = -1;
-	int sink_count = 0;
-	afr_private_t *priv = NULL;
-	unsigned char *data_lock = NULL;
-
-	priv = this->private;
-	sink_count = AFR_COUNT (healed_sinks, priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
-				    offset, size, data_lock);
-	{
-		if (ret < sink_count) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		if (type == AFR_SELFHEAL_DATA_DIFF &&
-		    __afr_can_skip_data_block_heal (frame, this, fd, source,
-					            healed_sinks, offset, size,
-                                                   &replies[source].poststat)) {
-			ret = 0;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data_read_write (frame, this, fd, source,
-						      healed_sinks, offset, size,
-						      replies, type);
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
-				offset, size, data_lock);
-	return ret;
-}
+    afr_private_t *priv = this->private;
+    int i = 0;
+
+    if (!locked_on[source])
+        return _gf_false;
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i] && locked_on[i])
+            return _gf_true;
+    }
 
+    return _gf_false;
+}
 
 static int
-afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			 unsigned char *healed_sinks)
+afr_selfheal_data_block(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        int source, unsigned char *healed_sinks, off_t offset,
+                        size_t size, int type, struct afr_reply *replies)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-        if (!priv->ensure_durability)
-                return 0;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    unsigned char *data_lock = NULL;
+
+    priv = this->private;
+    data_lock = alloca0(priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, offset, size,
+                               data_lock);
+    {
+        if (!afr_source_sinks_locked(this, data_lock, source, healed_sinks)) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, fsync, fd, 0, NULL);
+        if (type == AFR_SELFHEAL_DATA_DIFF &&
+            __afr_can_skip_data_block_heal(frame, this, fd, source,
+                                           healed_sinks, offset, size,
+                                           &replies[source].poststat)) {
+            ret = 0;
+            goto unlock;
+        }
 
-	for (i = 0; i < priv->child_count; i++)
-		if (healed_sinks[i] && local->replies[i].op_ret != 0)
-			/* fsync() failed. Do NOT consider this server
-			   as successfully healed. Mark it so.
-			*/
-			healed_sinks[i] = 0;
-	return 0;
+        ret = __afr_selfheal_data_read_write(
+            frame, this, fd, source, healed_sinks, offset, size, replies, type);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, offset, size,
+                           data_lock);
+    return ret;
 }
 
-
 static int
-afr_selfheal_data_restore_time (call_frame_t *frame, xlator_t *this,
-				inode_t *inode, int source,
-				unsigned char *healed_sinks,
-				struct afr_reply *replies)
+afr_selfheal_data_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        unsigned char *healed_sinks)
 {
-	loc_t loc = {0, };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    local = frame->local;
+    priv = this->private;
 
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, setattr, &loc,
-		    &replies[source].poststat,
-		    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
+    if (!priv->ensure_durability)
+        return 0;
 
-	loc_wipe (&loc);
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, fsync, fd, 0, NULL);
 
-	return 0;
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret != 0)
+            /* fsync() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+    return 0;
 }
 
 static int
-afr_data_self_heal_type_get (afr_private_t *priv, unsigned char *healed_sinks,
-                             int source, struct afr_reply *replies)
+afr_data_self_heal_type_get(afr_private_t *priv, unsigned char *healed_sinks,
+                            int source, struct afr_reply *replies)
 {
-        int type = AFR_SELFHEAL_DATA_FULL;
-        int i = 0;
-
-        if (priv->data_self_heal_algorithm == NULL) {
-                type = AFR_SELFHEAL_DATA_FULL;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!healed_sinks[i] && i != source)
-                                continue;
-                        if (replies[i].poststat.ia_size) {
-                                type = AFR_SELFHEAL_DATA_DIFF;
-                                break;
-                        }
-                }
-        } else if (strcmp (priv->data_self_heal_algorithm, "full") == 0) {
-                type = AFR_SELFHEAL_DATA_FULL;
-        } else if (strcmp (priv->data_self_heal_algorithm, "diff") == 0) {
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int i = 0;
+
+    if (priv->data_self_heal_algorithm == AFR_SELFHEAL_DATA_DYNAMIC) {
+        type = AFR_SELFHEAL_DATA_FULL;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!healed_sinks[i] && i != source)
+                continue;
+            if (replies[i].poststat.ia_size) {
                 type = AFR_SELFHEAL_DATA_DIFF;
+                break;
+            }
         }
-        return type;
+    } else {
+        type = priv->data_self_heal_algorithm;
+    }
+    return type;
 }
 
 static int
-afr_selfheal_data_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      int source, unsigned char *healed_sinks,
-		      struct afr_reply *replies)
+afr_selfheal_data_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                     unsigned char *healed_sinks, struct afr_reply *replies)
 {
-	afr_private_t *priv = NULL;
-	off_t off = 0;
-	size_t block = 128 * 1024;
-	int type = AFR_SELFHEAL_DATA_FULL;
-	int ret = -1;
-	call_frame_t *iter_frame = NULL;
-        unsigned char arbiter_sink_status = 0;
-
-	priv = this->private;
-        if (priv->arbiter_count) {
-                arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
-                healed_sinks[ARBITER_BRICK_INDEX] = 0;
+    afr_private_t *priv = NULL;
+    off_t off = 0;
+    size_t block = 0;
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int ret = -1;
+    call_frame_t *iter_frame = NULL;
+    unsigned char arbiter_sink_status = 0;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing data selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    priv = this->private;
+    if (priv->arbiter_count) {
+        arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
+        healed_sinks[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    block = 128 * 1024 * priv->data_self_heal_window_size;
+
+    type = afr_data_self_heal_type_get(priv, healed_sinks, source, replies);
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            ret = -ENOTCONN;
+            goto out;
         }
 
-        type = afr_data_self_heal_type_get (priv, healed_sinks, source,
-                                            replies);
+        ret = afr_selfheal_data_block(iter_frame, this, fd, source,
+                                      healed_sinks, off, block, type, replies);
+        if (ret < 0)
+            goto out;
 
-	iter_frame = afr_copy_frame (frame);
-	if (!iter_frame) {
-                ret = -ENOMEM;
-                goto out;
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = -ENOTCONN;
+            goto out;
         }
+    }
 
-	for (off = 0; off < replies[source].poststat.ia_size; off += block) {
-                if (AFR_COUNT (healed_sinks, priv->child_count) == 0) {
-                        ret = -ENOTCONN;
-                        goto out;
-                }
-
-		ret = afr_selfheal_data_block (iter_frame, this, fd, source,
-					       healed_sinks, off, block, type,
-					       replies);
-		if (ret < 0)
-			goto out;
-
-		AFR_STACK_RESET (iter_frame);
-		if (iter_frame->local == NULL) {
-		        ret = -ENOTCONN;
-		        goto out;
-                }
-	}
-
-	ret = afr_selfheal_data_fsync (frame, this, fd, healed_sinks);
+    ret = afr_selfheal_data_fsync(frame, this, fd, healed_sinks);
 
 out:
-        if (arbiter_sink_status)
-                healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
+    if (arbiter_sink_status)
+        healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
 
-	if (iter_frame)
-		AFR_STACK_DESTROY (iter_frame);
-	return ret;
+    if (iter_frame)
+        AFR_STACK_DESTROY(iter_frame);
+    return ret;
 }
 
-
 static int
-__afr_selfheal_truncate_sinks (call_frame_t *frame, xlator_t *this,
-			       fd_t *fd, unsigned char *healed_sinks,
-			       uint64_t size)
+__afr_selfheal_truncate_sinks(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              unsigned char *healed_sinks, uint64_t size)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        unsigned char arbiter_sink_status = 0;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-        if (priv->arbiter_count) {
-                arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
-                healed_sinks[ARBITER_BRICK_INDEX] = 0;
-        }
-
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, ftruncate, fd, size, NULL);
-
-	for (i = 0; i < priv->child_count; i++)
-		if (healed_sinks[i] && local->replies[i].op_ret == -1)
-			/* truncate() failed. Do NOT consider this server
-			   as successfully healed. Mark it so.
-			*/
-			healed_sinks[i] = 0;
-
-        if (arbiter_sink_status)
-                healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
-	return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    /* This will send truncate on the arbiter brick as well if it is marked as
+     * sink. If changelog is enabled on the volume it captures truncate as a
+     * data transactions on the arbiter brick. This will help geo-rep to
+     * properly sync the data from master to slave if arbiter is the ACTIVE
+     * brick during syncing and which had got some entries healed for data as
+     * part of self heal.
+     */
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, ftruncate, fd, size,
+               NULL);
+
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret == -1)
+            /* truncate() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+
+    return 0;
 }
 
 gf_boolean_t
-afr_has_source_witnesses (xlator_t *this, unsigned char *sources,
-                          uint64_t *witness)
+afr_has_source_witnesses(xlator_t *this, unsigned char *sources,
+                         uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] && witness[i])
-                        return _gf_true;
-        }
-        return _gf_false;
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_does_size_mismatch (xlator_t *this, unsigned char *sources,
-                        struct afr_reply *replies)
+afr_does_size_mismatch(xlator_t *this, unsigned char *sources,
+                       struct afr_reply *replies)
 {
-        int     i = 0;
-        afr_private_t *priv = NULL;
-        struct iatt *min = NULL;
-        struct iatt *max = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt *min = NULL;
+    struct iatt *max = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (replies[i].op_ret < 0)
-                        continue;
+        if (replies[i].op_ret < 0)
+            continue;
 
-                if (!sources[i])
-                        continue;
+        if (!sources[i])
+            continue;
 
-                if (AFR_IS_ARBITER_BRICK (priv, i) &&
-                    (replies[i].poststat.ia_size == 0))
-                        continue;
+        if (AFR_IS_ARBITER_BRICK(priv, i) && (replies[i].poststat.ia_size == 0))
+            continue;
 
-                if (!min)
-                        min = &replies[i].poststat;
+        if (!min)
+            min = &replies[i].poststat;
 
-                if (!max)
-                        max = &replies[i].poststat;
+        if (!max)
+            max = &replies[i].poststat;
 
-                if (min->ia_size > replies[i].poststat.ia_size)
-                        min = &replies[i].poststat;
+        if (min->ia_size > replies[i].poststat.ia_size)
+            min = &replies[i].poststat;
 
-                if (max->ia_size < replies[i].poststat.ia_size)
-                        max = &replies[i].poststat;
-        }
+        if (max->ia_size < replies[i].poststat.ia_size)
+            max = &replies[i].poststat;
+    }
 
-        if (min && max) {
-                if (min->ia_size != max->ia_size)
-                        return _gf_true;
-        }
+    if (min && max) {
+        if (min->ia_size != max->ia_size)
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static void
-afr_mark_biggest_witness_as_source (xlator_t *this, unsigned char *sources,
-                                    uint64_t *witness)
+afr_mark_biggest_witness_as_source(xlator_t *this, unsigned char *sources,
+                                   uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint64_t biggest_witness = 0;
-
-        priv = this->private;
-        /* Find source with biggest witness count */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (biggest_witness < witness[i])
-                        biggest_witness = witness[i];
-        }
-
-        /* Mark files with less witness count as not source */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (witness[i] < biggest_witness)
-                        sources[i] = 0;
-        }
-
-        return;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t biggest_witness = 0;
+
+    priv = this->private;
+    /* Find source with biggest witness count */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (biggest_witness < witness[i])
+            biggest_witness = witness[i];
+    }
+
+    /* Mark files with less witness count as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (witness[i] < biggest_witness)
+            sources[i] = 0;
+    }
+
+    return;
 }
 
 /* This is a tie breaker function. Only one source be assigned here */
 static void
-afr_mark_newest_file_as_source (xlator_t *this, unsigned char *sources,
-                                struct afr_reply *replies)
+afr_mark_newest_file_as_source(xlator_t *this, unsigned char *sources,
+                               struct afr_reply *replies)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        int source = -1;
-        uint32_t max_ctime = 0;
-
-        priv = this->private;
-        /* Find source with latest ctime */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-
-                if (max_ctime <= replies[i].poststat.ia_ctime) {
-                        source = i;
-                        max_ctime = replies[i].poststat.ia_ctime;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    uint32_t max_ctime = 0;
+
+    priv = this->private;
+    /* Find source with latest ctime */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+
+        if (max_ctime <= replies[i].poststat.ia_ctime) {
+            source = i;
+            max_ctime = replies[i].poststat.ia_ctime;
         }
+    }
 
-        /* Only mark one of the files as source to break ties */
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-        sources[source] = 1;
+    /* Only mark one of the files as source to break ties */
+    memset(sources, 0, sizeof(*sources) * priv->child_count);
+    sources[source] = 1;
 }
 
 static int
-__afr_selfheal_data_finalize_source (call_frame_t *frame, xlator_t *this,
-                                     inode_t *inode,
-                                     unsigned char *sources,
-                                     unsigned char *sinks,
-				     unsigned char *healed_sinks,
-				     unsigned char *locked_on,
-				     struct afr_reply *replies,
-                                     uint64_t *witness)
+__afr_selfheal_data_finalize_source(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    unsigned char *undid_pending, struct afr_reply *replies, uint64_t *witness)
 {
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count) {
-		/* split brain */
-                source = afr_mark_split_brain_source_sinks (frame, this, inode,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                          AFR_DATA_TRANSACTION);
-                if (source < 0)
-                        return -EIO;
-                return source;
-	}
-
-        /* No split brain at this point. If we were called from
-         * afr_heal_splitbrain_file(), abort.*/
-        if (afr_dict_contains_heal_op(frame))
-                return -EIO;
-
-        /* If there are no witnesses/size-mismatches on sources we are done*/
-        if (!afr_does_size_mismatch (this, sources, replies) &&
-            !afr_has_source_witnesses (this, sources, witness))
-                goto out;
-
-        afr_mark_largest_file_as_source (this, sources, replies);
-        afr_mark_biggest_witness_as_source (this, sources, witness);
-        afr_mark_newest_file_as_source (this, sources, replies);
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        /* split brain */
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_DATA_TRANSACTION);
+        if (source < 0) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=data;"
+                     "file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
+        }
+
+        _afr_fav_child_reset_sink_xattrs(
+            frame, this, inode, source, healed_sinks, undid_pending,
+            AFR_DATA_TRANSACTION, locked_on, replies);
+        goto out;
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    /* If there are no witnesses/size-mismatches on sources we are done*/
+    if (!afr_does_size_mismatch(this, sources, replies) &&
+        !afr_has_source_witnesses(this, sources, witness))
+        goto out;
+
+    afr_mark_largest_file_as_source(this, sources, replies);
+    afr_mark_biggest_witness_as_source(this, sources, witness);
+    afr_mark_newest_file_as_source(this, sources, replies);
+    if (priv->arbiter_count)
+        /* Choose non-arbiter brick as source for empty files. */
+        afr_mark_source_sinks_if_file_empty(this, sources, sinks, healed_sinks,
+                                            locked_on, replies,
+                                            AFR_DATA_TRANSACTION);
 
 out:
-        afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-        source = afr_choose_source_by_policy (priv, sources,
-                                              AFR_DATA_TRANSACTION);
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    source = afr_choose_source_by_policy(priv, sources, AFR_DATA_TRANSACTION);
 
-	return source;
+    return source;
 }
 
 /*
@@ -628,248 +606,286 @@ out:
  * for self-healing, or -1 if no healing is necessary/split brain.
  */
 int
-__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
-                             inode_t *inode, unsigned char *locked_on,
-                             unsigned char *sources, unsigned char *sinks,
-                             unsigned char *healed_sinks,
-			     struct afr_reply *replies, gf_boolean_t *pflag)
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        uint64_t *witness = NULL;
-
-	priv = this->private;
-
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-
-        if (ret)
-                return ret;
-
-        witness = alloca0(priv->child_count * sizeof (*witness));
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_DATA_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           pflag);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_data_finalize_source (frame, this, inode,
-                                                      sources, sinks,
-                                                      healed_sinks,
-                                                      locked_on, replies,
-                                                      witness);
-	if (source < 0)
-		return -EIO;
-
-	return source;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+
+    if (ret)
+        return ret;
+
+    witness = alloca0(priv->child_count * sizeof(*witness));
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_DATA_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_data_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, locked_on,
+        undid_pending, replies, witness);
+    if (source < 0)
+        return -EIO;
+
+    return source;
 }
 
-
 static int
-__afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     unsigned char *locked_on)
+__afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    unsigned char *locked_on)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *data_lock = NULL;
-	unsigned char *healed_sinks = NULL;
-	struct afr_reply *locked_replies = NULL;
-	int source = -1;
-        gf_boolean_t did_sh = _gf_true;
-        gf_boolean_t is_arbiter_the_only_sink = _gf_false;
-
-	priv = this->private;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name, 0, 0,
-				    data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "self-heal as only %d number "
-                                      "of subvolumes "
-                                      "could be locked",
-                                      uuid_utoa (fd->inode->gfid),
-                                      ret);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data_prepare (frame, this, fd->inode,
-                                                   data_lock, sources, sinks,
-                                                   healed_sinks,
-						   locked_replies,
-                                                   NULL);
-		if (ret < 0)
-			goto unlock;
-
-                if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-		source = ret;
-
-                if (AFR_IS_ARBITER_BRICK(priv, source)) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-                if (priv->arbiter_count &&
-                    AFR_COUNT (healed_sinks, priv->child_count) == 1 &&
-                    healed_sinks[ARBITER_BRICK_INDEX]) {
-                        is_arbiter_the_only_sink = _gf_true;
-                        goto restore_time;
-                }
-
-		ret = __afr_selfheal_truncate_sinks (frame, this, fd, healed_sinks,
-						     locked_replies[source].poststat.ia_size);
-		if (ret < 0)
-			goto unlock;
-
-		ret = 0;
-
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
-				data_lock);
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int source = -1;
+    gf_boolean_t did_sh = _gf_true;
+    gf_boolean_t is_arbiter_the_only_sink = _gf_false;
+    gf_boolean_t empty_file = _gf_false;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number "
+                         "of subvolumes "
+                         "could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_data_prepare(frame, this, fd->inode, data_lock,
+                                          sources, sinks, healed_sinks,
+                                          undid_pending, locked_replies, NULL);
         if (ret < 0)
-		goto out;
+            goto unlock;
 
-        if (!did_sh)
-                goto out;
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        source = ret;
+
+        if (AFR_IS_ARBITER_BRICK(priv, source)) {
+            empty_file = afr_is_file_empty_on_all_children(priv,
+                                                           locked_replies);
+            if (empty_file)
+                goto restore_time;
+
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_truncate_sinks(
+            frame, this, fd, healed_sinks,
+            locked_replies[source].poststat.ia_size);
+        if (ret < 0)
+            goto unlock;
+
+        if (priv->arbiter_count &&
+            AFR_COUNT(healed_sinks, priv->child_count) == 1 &&
+            healed_sinks[ARBITER_BRICK_INDEX]) {
+            is_arbiter_the_only_sink = _gf_true;
+            goto restore_time;
+        }
+        ret = 0;
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
+    if (ret < 0)
+        goto out;
+
+    if (!did_sh)
+        goto out;
 
-	ret = afr_selfheal_data_do (frame, this, fd, source, healed_sinks,
-				    locked_replies);
-	if (ret)
-                goto out;
+    ret = afr_selfheal_data_do(frame, this, fd, source, healed_sinks,
+                               locked_replies);
+    if (ret)
+        goto out;
 restore_time:
-	afr_selfheal_data_restore_time (frame, this, fd->inode, source,
-					healed_sinks, locked_replies);
-
-        if (!is_arbiter_the_only_sink) {
-                ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
-                                            0, 0, data_lock);
-                if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        ret = -ENOTCONN;
-                        did_sh = _gf_false;
-                        goto skip_undo_pending;
-                }
+    afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                              locked_replies);
+
+    if (!is_arbiter_the_only_sink && !empty_file) {
+        ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                                   data_lock);
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            did_sh = _gf_false;
+            goto skip_undo_pending;
         }
-        ret = afr_selfheal_undo_pending (frame, this, fd->inode,
-                                         sources, sinks, healed_sinks,
-                                         AFR_DATA_TRANSACTION,
-                                         locked_replies, data_lock);
+    }
+    ret = afr_selfheal_undo_pending(
+        frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+        AFR_DATA_TRANSACTION, locked_replies, data_lock);
 skip_undo_pending:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
-				data_lock);
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
 out:
 
-        if (did_sh)
-                afr_log_selfheal (fd->inode->gfid, this, ret, "data", source,
-                                  sources, healed_sinks);
-        else
-                ret = 1;
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "data", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
 
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
 
-	return ret;
+    return ret;
 }
 
+int
+afr_selfheal_data_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int i = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
 
 int
-afr_selfheal_data_open (xlator_t *this, inode_t *inode, fd_t **fd)
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd)
 {
-	int         ret    = 0;
-        fd_t       *fd_tmp = NULL;
-	loc_t       loc    = {0,};
-
-	fd_tmp = fd_create (inode, 0);
-	if (!fd_tmp)
-		return -ENOMEM;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	ret = syncop_open (this, &loc, O_RDWR|O_LARGEFILE, fd_tmp, NULL, NULL);
-	if (ret < 0) {
-		fd_unref (fd_tmp);
-                goto out;
-	} else {
-		fd_bind (fd_tmp);
-	}
-
-        *fd = fd_tmp;
+    int ret = 0;
+    fd_t *fd_tmp = NULL;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    fd_tmp = fd_create(inode, 0);
+    if (!fd_tmp)
+        return -ENOMEM;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        fd_unref(fd_tmp);
+        goto out;
+    }
+    local = frame->local;
+
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_data_open_cbk, open, &loc,
+               O_RDWR | O_LARGEFILE, fd_tmp, NULL);
+
+    ret = -ENOTCONN;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret < 0) {
+            ret = -local->replies[i].op_errno;
+            continue;
+        }
+
+        ret = 0;
+        break;
+    }
+
+    if (ret < 0) {
+        fd_unref(fd_tmp);
+        goto out;
+    } else {
+        fd_bind(fd_tmp);
+    }
+
+    *fd = fd_tmp;
 out:
-        loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode)
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *locked_on = NULL;
-	int ret = 0;
-	fd_t *fd = NULL;
-
-	priv = this->private;
-
-	ret = afr_selfheal_data_open (this, inode, &fd);
-	if (!fd) {
-                gf_msg_debug (this->name, -ret, "%s: Failed to open",
-                              uuid_utoa (inode->gfid));
-                return -EIO;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    int ret = 0;
+    inode_t *inode = fd->inode;
+
+    priv = this->private;
+
+    locked_on = alloca0(priv->child_count);
+
+    ret = afr_selfheal_tie_breaker_inodelk(frame, this, inode, priv->sh_domain,
+                                           0, 0, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number of "
+                         "subvolumes could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-	locked_on = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_tie_breaker_inodelk (frame, this, inode,
-	                                        priv->sh_domain, 0, 0,
-				                locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "self-heal as only %d number of "
-                                      "subvolumes could be locked",
-                                      uuid_utoa (fd->inode->gfid),
-                                ret);
-			/* Either less than two subvols available, or another
-			   selfheal (from another server) is in progress. Skip
-			   for now in any case there isn't anything to do.
-			*/
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data (frame, this, fd, locked_on);
-	}
+        ret = __afr_selfheal_data(frame, this, fd, locked_on);
+    }
 unlock:
-	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0,
-	                        locked_on);
-
-	if (fd)
-		fd_unref (fd);
+    afr_selfheal_uninodelk(frame, this, inode, priv->sh_domain, 0, 0,
+                           locked_on);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index c9a27bf8e20..64893f441e3 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -8,1088 +8,1269 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
 #include "afr-messages.h"
-#include "syncop-utils.h"
-
-/* Max file name length is 255 this filename is of length 256. No file with
- * this name can ever come, entry-lock with this name is going to prevent
- * self-heals from older versions while the granular entry-self-heal is going
- * on in newer version.*/
-#define LONG_FILENAME "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+#include <glusterfs/syncop-utils.h>
+#include <glusterfs/events.h>
 
-static int
-afr_selfheal_entry_delete (xlator_t *this, inode_t *dir, const char *name,
-                           inode_t *inode, int child, struct afr_reply *replies)
+int
+afr_selfheal_entry_anon_inode(xlator_t *this, inode_t *dir, const char *name,
+                              inode_t *inode, int child,
+                              struct afr_reply *replies,
+                              gf_boolean_t *anon_inode)
 {
-	afr_private_t *priv = NULL;
-	xlator_t *subvol = NULL;
-	int ret = 0;
-	loc_t loc = {0, };
-	char g[64];
-
-	priv = this->private;
-
-	subvol = priv->children[child];
-
-	loc.parent = inode_ref (dir);
-	gf_uuid_copy (loc.pargfid, dir->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
-
-	if (replies[child].valid && replies[child].op_ret == 0) {
-		switch (replies[child].poststat.ia_type) {
-		case IA_IFDIR:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-				"expunging dir %s/%s (%s) on %s",
-				uuid_utoa (dir->gfid), name,
-				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
-				subvol->name);
-			ret = syncop_rmdir (subvol, &loc, 1, NULL, NULL);
-			break;
-		default:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-				"expunging file %s/%s (%s) on %s",
-				uuid_utoa (dir->gfid), name,
-				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
-				subvol->name);
-			ret = syncop_unlink (subvol, &loc, NULL, NULL);
-			break;
-		}
-	}
-
-	loc_wipe (&loc);
-
-	return ret;
-}
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    char g[64] = {0};
+    unsigned char *lookup_success = NULL;
+    call_frame_t *frame = NULL;
+    loc_t loc2 = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+    lookup_success = alloca0(priv->child_count);
+    uuid_utoa_r(replies[child].poststat.ia_gfid, g);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (replies[child].poststat.ia_type == IA_IFDIR) {
+        /* This directory may have sub-directory hierarchy which may need to
+         * be preserved for subsequent heals. So unconditionally move the
+         * directory to anonymous-inode directory*/
+        *anon_inode = _gf_true;
+        goto anon_inode;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    gf_uuid_copy(loc.gfid, replies[child].poststat.ia_gfid);
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            lookup_success[i] = 1;
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            ret = -local->replies[i].op_errno;
+        }
+    }
+
+    if (priv->quorum_count) {
+        if (afr_has_quorum(lookup_success, this, NULL)) {
+            *anon_inode = _gf_true;
+        }
+    } else if (AFR_COUNT(lookup_success, priv->child_count) > 1) {
+        *anon_inode = _gf_true;
+    } else if (ret) {
+        goto out;
+    }
+
+anon_inode:
+    if (!*anon_inode) {
+        ret = 0;
+        goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+
+    ret = afr_anon_inode_create(this, child, &loc2.parent);
+    if (ret < 0)
+        goto out;
+
+    loc2.name = g;
+    ret = syncop_rename(subvol, &loc, &loc2, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s failed",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s successful",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    }
+
+out:
+    loc_wipe(&loc);
+    loc_wipe(&loc2);
+    if (frame) {
+        AFR_STACK_DESTROY(frame);
+    }
 
+    return ret;
+}
 
 int
-afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
-                             const char *name, inode_t *inode,
-                             struct afr_reply *replies,
-                             unsigned char *newentry)
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies)
 {
-	int ret = 0;
-	loc_t loc = {0,};
-	loc_t srcloc = {0,};
-	afr_private_t *priv = NULL;
-	dict_t *xdata = NULL;
-	struct iatt *iatt = NULL;
-	char *linkname = NULL;
-	mode_t mode = 0;
-	struct iatt newent = {0,};
-	priv = this->private;
-
-	xdata = dict_new();
-	if (!xdata)
-		return -ENOMEM;
-
-	loc.parent = inode_ref (dir);
-	gf_uuid_copy (loc.pargfid, dir->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
-
-	ret = afr_selfheal_entry_delete (this, dir, name, inode, dst, replies);
-	if (ret)
-		goto out;
-
-	ret = dict_set_static_bin (xdata, "gfid-req",
-				   replies[source].poststat.ia_gfid, 16);
-	if (ret)
-		goto out;
-
-	iatt = &replies[source].poststat;
-
-	srcloc.inode = inode_ref (inode);
-	gf_uuid_copy (srcloc.gfid, iatt->ia_gfid);
-
-	mode = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
-
-	switch (iatt->ia_type) {
-	case IA_IFDIR:
-		ret = syncop_mkdir (priv->children[dst], &loc, mode, 0,
-                                    xdata, NULL);
-                if (ret == 0)
-                        newentry[dst] = 1;
-		break;
-	case IA_IFLNK:
-		ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
-		if (ret == 0) {
-			ret = syncop_link (priv->children[dst], &srcloc, &loc,
-					   &newent, NULL, NULL);
-		} else {
-			ret = syncop_readlink (priv->children[source], &srcloc,
-					       &linkname, 4096, NULL, NULL);
-			if (ret <= 0)
-				goto out;
-			ret = syncop_symlink (priv->children[dst], &loc,
-                                              linkname, NULL, xdata, NULL);
-                        if (ret == 0)
-                                newentry[dst] = 1;
-                }
-		break;
-	default:
-		ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
-		if (ret)
-			goto out;
-		ret = syncop_mknod (priv->children[dst], &loc, mode,
-				    iatt->ia_rdev, &newent, xdata, NULL);
-		if (ret == 0 && newent.ia_nlink == 1) {
-			/* New entry created. Mark @dst pending on all sources */
-                        newentry[dst] = 1;
-		}
-		break;
-	}
+    char g[64] = {0};
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t anon_inode = _gf_false;
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    if ((!replies[child].valid) || (replies[child].op_ret < 0)) {
+        /*Nothing to do*/
+        ret = 0;
+        goto out;
+    }
+
+    if (priv->use_anon_inode) {
+        ret = afr_selfheal_entry_anon_inode(this, dir, name, inode, child,
+                                            replies, &anon_inode);
+        if (ret < 0 || anon_inode)
+            goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    loc.name = name;
+    switch (replies[child].poststat.ia_type) {
+        case IA_IFDIR:
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging dir %s/%s (%s) on %s", uuid_utoa(dir->gfid), name,
+                   uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging file %s/%s (%s) on %s", uuid_utoa(dir->gfid),
+                   name, uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_unlink(subvol, &loc, NULL, NULL);
+            break;
+    }
 
 out:
-	if (xdata)
-		dict_unref (xdata);
-	GF_FREE (linkname);
-	loc_wipe (&loc);
-	loc_wipe (&srcloc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
+int
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies)
+{
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    loc_t srcloc = {
+        0,
+    };
+    loc_t anonloc = {
+        0,
+    };
+    xlator_t *this = frame->this;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    struct iatt *iatt = NULL;
+    char *linkname = NULL;
+    mode_t mode = 0;
+    struct iatt newent = {
+        0,
+    };
+    unsigned char *newentry = NULL;
+    char iatt_uuid_str[64] = {0};
+    char dir_uuid_str[64] = {0};
+
+    priv = this->private;
+    iatt = &replies[source].poststat;
+    uuid_utoa_r(iatt->ia_gfid, iatt_uuid_str);
+    if (iatt->ia_type == IA_INVAL || gf_uuid_is_null(iatt->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SELF_HEAL_FAILED,
+               "Invalid ia_type (%d) or gfid(%s). source brick=%d, "
+               "pargfid=%s, name=%s",
+               iatt->ia_type, iatt_uuid_str, source,
+               uuid_utoa_r(dir->gfid, dir_uuid_str), name);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        return -ENOMEM;
+    newentry = alloca0(priv->child_count);
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    ret = afr_selfheal_entry_delete(this, dir, name, inode, dst, replies);
+    if (ret)
+        goto out;
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", replies[source].poststat.ia_gfid,
+                          true);
+    if (ret)
+        goto out;
+
+    srcloc.inode = inode_ref(inode);
+    gf_uuid_copy(srcloc.gfid, iatt->ia_gfid);
+    ret = syncop_lookup(priv->children[dst], &srcloc, 0, 0, 0, 0);
+    if (ret == -ENOENT || ret == -ESTALE) {
+        newentry[dst] = 1;
+        ret = afr_selfheal_newentry_mark(frame, this, inode, source, replies,
+                                         sources, newentry);
+        if (ret)
+            goto out;
+    } else if (ret == 0 && iatt->ia_type == IA_IFDIR && priv->use_anon_inode) {
+        // Try rename from hidden directory
+        ret = afr_anon_inode_create(this, dst, &anonloc.parent);
+        if (ret < 0)
+            goto out;
+        anonloc.inode = inode_ref(inode);
+        anonloc.name = iatt_uuid_str;
+        ret = syncop_rename(priv->children[dst], &anonloc, &loc, NULL, NULL);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = -1; /*This sets 'mismatch' to true*/
+        goto out;
+    }
+
+    mode = st_mode_from_ia(iatt->ia_prot, iatt->ia_type);
+
+    switch (iatt->ia_type) {
+        case IA_IFDIR:
+            ret = syncop_mkdir(priv->children[dst], &loc, mode, 0, xdata, NULL);
+            break;
+        case IA_IFLNK:
+            if (!newentry[dst]) {
+                ret = syncop_link(priv->children[dst], &srcloc, &loc, &newent,
+                                  NULL, NULL);
+            } else {
+                ret = syncop_readlink(priv->children[source], &srcloc,
+                                      &linkname, 4096, NULL, NULL);
+                if (ret <= 0)
+                    goto out;
+                ret = syncop_symlink(priv->children[dst], &loc, linkname, NULL,
+                                     xdata, NULL);
+            }
+            break;
+        default:
+            ret = dict_set_int32_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+            if (ret)
+                goto out;
+            ret = syncop_mknod(
+                priv->children[dst], &loc, mode,
+                makedev(ia_major(iatt->ia_rdev), ia_minor(iatt->ia_rdev)),
+                &newent, xdata, NULL);
+            break;
+    }
+
+out:
+    if (xdata)
+        dict_unref(xdata);
+    GF_FREE(linkname);
+    loc_wipe(&loc);
+    loc_wipe(&srcloc);
+    loc_wipe(&anonloc);
+    return ret;
+}
 
 static int
-__afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			    char *name, inode_t *inode, int source,
-			    unsigned char *sources, unsigned char *healed_sinks,
-			    unsigned char *locked_on, struct afr_reply *replies)
+__afr_selfheal_heal_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                           char *name, inode_t *inode, int source,
+                           unsigned char *sources, unsigned char *healed_sinks,
+                           unsigned char *locked_on, struct afr_reply *replies)
 {
-	int ret = 0;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	unsigned char *newentry = NULL;
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	newentry = alloca0 (priv->child_count);
+    if (!replies[source].valid)
+        return -EIO;
 
-	if (!replies[source].valid)
-		return -EIO;
+    /* Skip healing this entry if the last lookup on it failed for reasons
+     * other than ENOENT.
+     */
+    if ((replies[source].op_ret < 0) && (replies[source].op_errno != ENOENT))
+        return -replies[source].op_errno;
 
-        /* Skip healing this entry if the last lookup on it failed for reasons
-         * other than ENOENT.
-         */
-        if ((replies[source].op_ret < 0) &&
-            (replies[source].op_errno != ENOENT))
-                return -replies[source].op_errno;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!healed_sinks[i])
-			continue;
-		if (replies[source].op_ret == -1 &&
-		    replies[source].op_errno == ENOENT) {
-			ret = afr_selfheal_entry_delete (this, fd->inode, name,
-                                                         inode, i, replies);
-                } else {
-			if (!gf_uuid_compare (replies[i].poststat.ia_gfid,
-					   replies[source].poststat.ia_gfid))
-				continue;
-
-			ret = afr_selfheal_recreate_entry (this, i, source,
-							   fd->inode, name, inode,
-							   replies, newentry);
-                }
-		if (ret < 0)
-			break;
-	}
-
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
-					    sources, newentry);
-	return ret;
+    if (replies[source].op_ret == 0) {
+        ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                       source, sources,
+                                       &replies[source].poststat.ia_gfid, NULL);
+        if (ret)
+            return ret;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+        if (replies[source].op_ret == -1 &&
+            replies[source].op_errno == ENOENT) {
+            ret = afr_selfheal_entry_delete(this, fd->inode, name, inode, i,
+                                            replies);
+        } else {
+            if (!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                 replies[source].poststat.ia_gfid))
+                continue;
+
+            ret = afr_selfheal_recreate_entry(frame, i, source, sources,
+                                              fd->inode, name, inode, replies);
+        }
+        if (ret < 0)
+            break;
+    }
+
+    return ret;
 }
 
 static int
-afr_selfheal_detect_gfid_and_type_mismatch (xlator_t *this,
-                                            struct afr_reply *replies,
-                                            uuid_t pargfid, char *bname,
-                                            int src_idx)
+afr_selfheal_detect_gfid_and_type_mismatch(xlator_t *this,
+                                           struct afr_reply *replies,
+                                           inode_t *inode, uuid_t pargfid,
+                                           char *bname, int src_idx,
+                                           unsigned char *locked_on, int *src)
 {
-        int             i      = 0;
-        char            g1[64] = {0,};
-        char            g2[64] = {0,};
-        afr_private_t  *priv   = NULL;
+    int i = 0;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    void *gfid = NULL;
+    ia_type_t ia_type = IA_INVAL;
 
-        priv = this->private;
+    priv = this->private;
+    gfid = &replies[src_idx].poststat.ia_gfid;
+    ia_type = replies[src_idx].poststat.ia_type;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == src_idx)
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
-
-                if (gf_uuid_compare (replies[src_idx].poststat.ia_gfid,
-                                  replies[i].poststat.ia_gfid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "Gfid mismatch "
-                                "detected for <%s/%s>, %s on %s and %s on %s. "
-                                "Skipping conservative merge on the file.",
-                                uuid_utoa (pargfid), bname,
-                                uuid_utoa_r (replies[i].poststat.ia_gfid, g1),
-                                priv->children[i]->name,
-                                uuid_utoa_r (replies[src_idx].poststat.ia_gfid,
-                                g2), priv->children[src_idx]->name);
-                        return -1;
-                }
-
-                if ((replies[src_idx].poststat.ia_type) !=
-                    (replies[i].poststat.ia_type)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "Type mismatch "
-                                "detected for <%s/%s>, %d on %s and %d on %s. "
-                                "Skipping conservative merge on the file.",
-                                uuid_utoa (pargfid), bname,
-                                replies[i].poststat.ia_type,
-                                priv->children[i]->name,
-                                replies[src_idx].poststat.ia_type,
-                                priv->children[src_idx]->name);
-                        return -1;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == src_idx)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
+
+        if (ia_type == IA_INVAL || gf_uuid_is_null(gfid)) {
+            src_idx = i;
+            ia_type = replies[src_idx].poststat.ia_type;
+            gfid = &replies[src_idx].poststat.ia_gfid;
+            continue;
         }
 
-        return 0;
+        if (gf_uuid_compare(gfid, replies[i].poststat.ia_gfid) &&
+            (ia_type == replies[i].poststat.ia_type)) {
+            ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                              bname, src_idx, i, locked_on, src,
+                                              NULL);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Skipping conservative merge on the "
+                       "file.");
+            return ret;
+        }
+
+        if (ia_type != replies[i].poststat.ia_type) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "Type mismatch detected "
+                   "for <gfid:%s>/%s>, %s on %s and %s on %s. "
+                   "Skipping conservative merge on the file.",
+                   uuid_utoa(pargfid), bname,
+                   gf_inode_type_to_str(replies[i].poststat.ia_type),
+                   priv->children[i]->name,
+                   gf_inode_type_to_str(replies[src_idx].poststat.ia_type),
+                   priv->children[src_idx]->name);
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=file;"
+                     "file=<gfid:%s>/%s>;count=2;child-%d=%s;type-"
+                     "%d=%s;child-%d=%s;type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(pargfid), bname, i, priv->children[i]->name, i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type), src_idx,
+                     priv->children[src_idx]->name, src_idx,
+                     gf_inode_type_to_str(replies[src_idx].poststat.ia_type));
+            return -1;
+        }
+    }
+
+    return 0;
 }
 
 static int
-__afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			     char *name, inode_t *inode, unsigned char *sources,
-			     unsigned char *healed_sinks, unsigned char *locked_on,
-			     struct afr_reply *replies)
+__afr_selfheal_merge_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, unsigned char *sources,
+                            unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-        int             ret       = 0;
-        int             i         = 0;
-        int             source    = -1;
-        unsigned char  *newentry  = NULL;
-        afr_private_t  *priv      = NULL;
-
-	priv = this->private;
-
-	newentry = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (replies[i].valid && replies[i].op_ret == 0) {
-			source = i;
-			break;
-		}
-	}
-
-	if (source == -1) {
-		/* entry got deleted in the mean time? */
-		return 0;
-	}
-
-        /* Set all the sources as 1, otheriwse newentry_mark won't be set */
-	for (i = 0; i < priv->child_count; i++) {
-		if (replies[i].valid && replies[i].op_ret == 0) {
-			sources[i] = 1;
-		}
-	}
-
-        /* In case of a gfid or type mismatch on the entry, return -1.*/
-        ret = afr_selfheal_detect_gfid_and_type_mismatch (this, replies,
-                                                          fd->inode->gfid,
-                                                          name, source);
+    int ret = 0;
+    int i = 0;
+    int source = -1;
+    int src = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            source = i;
+            break;
+        }
+    }
 
-        if (ret < 0)
-                return ret;
+    if (source == -1) {
+        /* entry got deleted in the mean time? */
+        return 0;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || !healed_sinks[i])
-			continue;
+    /* Set all the sources as 1, otheriwse newentry_mark won't be set */
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            sources[i] = 1;
+        }
+    }
 
-		if (replies[i].op_errno != ENOENT)
-			continue;
+    ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                   source, sources,
+                                   &replies[source].poststat.ia_gfid, NULL);
+    if (ret)
+        return ret;
 
-		ret = afr_selfheal_recreate_entry (this, i, source, fd->inode,
-                                                   name, inode, replies,
-                                                   newentry);
-	}
+    /* In case of type mismatch / unable to resolve gfid mismatch on the
+     * entry, return -1.*/
+    ret = afr_selfheal_detect_gfid_and_type_mismatch(
+        this, replies, inode, fd->inode->gfid, name, source, locked_on, &src);
 
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
-					    sources, newentry);
-	return ret;
-}
+    if (ret < 0)
+        return ret;
+    if (src != -1) {
+        source = src;
+        for (i = 0; i < priv->child_count; i++) {
+            if (i != src && replies[i].valid &&
+                gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                replies[i].poststat.ia_gfid)) {
+                sources[i] = 0;
+            }
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
+
+        if (src != -1) {
+            if (!gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                 replies[i].poststat.ia_gfid))
+                continue;
+        } else if (replies[i].op_errno != ENOENT) {
+            continue;
+        }
 
+        ret |= afr_selfheal_recreate_entry(frame, i, source, sources, fd->inode,
+                                           name, inode, replies);
+    }
+
+    return ret;
+}
 
 static int
-__afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                             char *name, inode_t *inode, int source,
-                             unsigned char *sources, unsigned char *healed_sinks,
-                             unsigned char *locked_on,
-                             struct afr_reply *replies)
+__afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, int source,
+                            unsigned char *sources, unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-	int ret = -1;
-
-	if (source < 0)
-		ret = __afr_selfheal_merge_dirent (frame, this, fd, name, inode,
-						   sources, healed_sinks,
-						   locked_on, replies);
-	else
-		ret = __afr_selfheal_heal_dirent (frame, this, fd, name, inode,
-						  source, sources, healed_sinks,
-						  locked_on, replies);
-	return ret;
+    int ret = -1;
+
+    if (source < 0)
+        ret = __afr_selfheal_merge_dirent(frame, this, fd, name, inode, sources,
+                                          healed_sinks, locked_on, replies);
+    else
+        ret = __afr_selfheal_heal_dirent(frame, this, fd, name, inode, source,
+                                         sources, healed_sinks, locked_on,
+                                         replies);
+    return ret;
 }
 
 static gf_boolean_t
-is_full_heal_marker_present (xlator_t *this, dict_t *xdata, int idx)
+is_full_heal_marker_present(xlator_t *this, dict_t *xdata, int idx)
 {
-        int                  i           = 0;
-        int                  pending[3]  = {0,};
-        void                *pending_raw = NULL;
-        afr_private_t       *priv        = NULL;
+    int i = 0;
+    int pending[3] = {
+        0,
+    };
+    void *pending_raw = NULL;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!xdata)
-                return _gf_false;
+    if (!xdata)
+        return _gf_false;
 
-        /* Iterate over each of the priv->pending_keys[] elements and then
-         * see if any of them have data segment non-zero. If they do, return
-         * true. Else return false.
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
-                        continue;
+    /* Iterate over each of the priv->pending_keys[] elements and then
+     * see if any of them have data segment non-zero. If they do, return
+     * true. Else return false.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
 
-                if (!pending_raw)
-                        continue;
+        if (!pending_raw)
+            continue;
 
-                memcpy (pending, pending_raw, sizeof (pending));
-                if (ntoh32 (pending[idx]))
-                        return _gf_true;
-        }
+        memcpy(pending, pending_raw, sizeof(pending));
+        if (ntoh32(pending[idx]))
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_need_full_heal (xlator_t *this, struct afr_reply *replies, int source,
-                    unsigned char *healed_sinks, afr_transaction_type type)
+afr_need_full_heal(xlator_t *this, struct afr_reply *replies, int source,
+                   unsigned char *healed_sinks, afr_transaction_type type)
 {
-        int                i     = 0;
-        int                idx   = 0;
-        afr_private_t     *priv  = NULL;
+    int i = 0;
+    int idx = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->esh_granular)
-                return _gf_true;
+    if (!priv->esh_granular)
+        return _gf_true;
 
-        if (type != AFR_ENTRY_TRANSACTION)
-                return _gf_true;
+    if (type != AFR_ENTRY_TRANSACTION)
+        return _gf_true;
 
-        priv = this->private;
-        idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+    priv = this->private;
+    idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
 
-        /* If there is a clear source, check whether the full-heal-indicator
-         * is present in its xdata. Otherwise, we need to examine all the
-         * participating bricks and then figure if *even* one of them has a
-         * full-heal-indicator.
-         */
+    /* If there is a clear source, check whether the full-heal-indicator
+     * is present in its xdata. Otherwise, we need to examine all the
+     * participating bricks and then figure if *even* one of them has a
+     * full-heal-indicator.
+     */
 
-        if (source != -1) {
-                if (is_full_heal_marker_present (this, replies[source].xdata,
-                                                 idx))
-                        return _gf_true;
-        }
+    if (source != -1) {
+        if (is_full_heal_marker_present(this, replies[source].xdata, idx))
+            return _gf_true;
+    }
 
-        /* else ..*/
+    /* else ..*/
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!healed_sinks[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
 
-                if (is_full_heal_marker_present (this, replies[i].xdata, idx))
-                        return _gf_true;
-        }
+        if (is_full_heal_marker_present(this, replies[i].xdata, idx))
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static int
-__afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
-				      unsigned char *healed_sinks,
-                                      unsigned char *locked_on,
-                                      struct afr_reply *replies,
-                                      uint64_t *witness)
+__afr_selfheal_entry_finalize_source(xlator_t *this, unsigned char *sources,
+                                     unsigned char *healed_sinks,
+                                     unsigned char *locked_on,
+                                     struct afr_reply *replies,
+                                     uint64_t *witness)
 {
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    int i = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	sources_count = AFR_COUNT (sources, priv->child_count);
+    sources_count = AFR_COUNT(sources, priv->child_count);
 
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count || afr_does_witness_exist (this, witness)) {
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
 
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-		return -1;
-	}
+    source = afr_choose_source_by_policy(priv, sources, AFR_ENTRY_TRANSACTION);
 
-        source = afr_choose_source_by_policy (priv, sources,
-                                              AFR_ENTRY_TRANSACTION);
-	return source;
+    /*If the selected source does not blame any other brick, then mark
+     * everything as sink to trigger conservative merge.
+     */
+    if (source != -1 && !AFR_COUNT(healed_sinks, priv->child_count)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (locked_on[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
+        }
+        return -1;
+    }
+
+    return source;
 }
 
 int
-__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
-                              inode_t *inode, unsigned char *locked_on,
-                              unsigned char *sources, unsigned char *sinks,
-                              unsigned char *healed_sinks,
-			      struct afr_reply *replies, int *source_p,
-                              gf_boolean_t *pflag)
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
+                             inode_t *inode, unsigned char *locked_on,
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        uint64_t *witness = NULL;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-        if (ret)
-                return ret;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_ENTRY_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           pflag);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_entry_finalize_source (this, sources,
-                                                       healed_sinks,
-						       locked_on, replies,
-                                                       witness);
-
-	if (source < 0) {
-		/* If source is < 0 (typically split-brain), we perform a
-		   conservative merge of entries rather than erroring out */
-	}
-	*source_p = source;
-
-	return ret;
-}
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
+        return ret;
 
-static int
-afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this,
-                           fd_t *fd, char *name, inode_t *parent_idx_inode,
-                           xlator_t *subvol, gf_boolean_t full_crawl)
-{
-        int                ret          = 0;
-        int                source       = -1;
-        unsigned char     *locked_on    = NULL;
-        unsigned char     *sources      = NULL;
-        unsigned char     *sinks        = NULL;
-        unsigned char     *healed_sinks = NULL;
-        inode_t           *inode        = NULL;
-        struct afr_reply  *replies      = NULL;
-        struct afr_reply  *par_replies  = NULL;
-        afr_private_t     *priv         = NULL;
-
-	priv = this->private;
-
-        sources = alloca0 (priv->child_count);
-        sinks   = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-	locked_on = alloca0 (priv->child_count);
-
-	replies = alloca0 (priv->child_count * sizeof(*replies));
-	par_replies = alloca0 (priv->child_count * sizeof(*par_replies));
-
-        ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-                                    locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "entry self-heal as only %d sub-volumes "
-                                      " could be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid),
-                                      ret, this->name);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-                ret = __afr_selfheal_entry_prepare (frame, this, fd->inode,
-                                                    locked_on,
-                                                    sources, sinks,
-                                                    healed_sinks, par_replies,
-                                                    &source, NULL);
-                if (ret < 0)
-                        goto unlock;
-
-		inode = afr_selfheal_unlocked_lookup_on (frame, fd->inode, name,
-							 replies, locked_on,
-                                                         NULL);
-		if (!inode) {
-			ret = -ENOMEM;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_entry_dirent (frame, this, fd, name, inode,
-						   source, sources, healed_sinks,
-						   locked_on, replies);
-
-                if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
-                        ret = afr_shd_index_purge (subvol, parent_idx_inode,
-                                                   name);
-                        /* Why is ret force-set to 0? We do not care about
-                         * index purge failing for full heal as it is quite
-                         * possible during replace-brick that not all files
-                         * and directories have their name indices present in
-                         * entry-changes/.
-                         */
-                        ret = 0;
-                }
-	}
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
 
-unlock:
-        afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-                                locked_on, NULL);
-	if (inode)
-		inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-        if (par_replies)
-                afr_replies_wipe (par_replies, priv->child_count);
-
-	return ret;
-}
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
 
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
 
-static inode_t *
-afr_shd_entry_changes_index_inode (xlator_t *this, xlator_t *subvol,
-                                   uuid_t pargfid)
+    source = __afr_selfheal_entry_finalize_source(this, sources, healed_sinks,
+                                                  locked_on, replies, witness);
+
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
+
+    return ret;
+}
+
+static int
+afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                          char *name, inode_t *parent_idx_inode,
+                          xlator_t *subvol, gf_boolean_t full_crawl)
 {
-        int             ret         = -1;
-        void           *index_gfid  = NULL;
-        loc_t           rootloc     = {0,};
-        loc_t           loc         = {0,};
-        dict_t         *xattr       = NULL;
-        inode_t        *inode       = NULL;
-        struct iatt     iatt        = {0,};
-
-        rootloc.inode = inode_ref (this->itable->root);
-        gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
-
-        ret = syncop_getxattr (subvol, &rootloc, &xattr,
-                               GF_XATTROP_ENTRY_CHANGES_GFID, NULL, NULL);
-        if (ret || !xattr) {
-                errno = -ret;
-                goto out;
+    int ret = 0;
+    int source = -1;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    inode_t *inode = NULL;
+    struct afr_reply *replies = NULL;
+    struct afr_reply *par_replies = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+
+    if (afr_is_private_directory(priv, fd->inode->gfid, name,
+                                 GF_CLIENT_PID_SELF_HEALD)) {
+        return 0;
+    }
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    locked_on = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+    par_replies = alloca0(priv->child_count * sizeof(*par_replies));
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes "
+                         " could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-        ret = dict_get_ptr (xattr, GF_XATTROP_ENTRY_CHANGES_GFID, &index_gfid);
-        if (ret) {
-                errno = EINVAL;
-                goto out;
-        }
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, locked_on,
+                                           sources, sinks, healed_sinks,
+                                           par_replies, &source, NULL);
+        if (ret < 0)
+            goto unlock;
 
-        loc.inode = inode_new (this->itable);
-        if (!loc.inode) {
-                errno = ENOMEM;
-                goto out;
+        inode = afr_selfheal_unlocked_lookup_on(frame, fd->inode, name, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
         }
 
-        gf_uuid_copy (loc.pargfid, index_gfid);
-        loc.name = gf_strdup (uuid_utoa (pargfid));
-
-        ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
-        if (ret < 0) {
-                errno = -ret;
-                goto out;
+        ret = __afr_selfheal_entry_dirent(frame, this, fd, name, inode, source,
+                                          sources, healed_sinks, locked_on,
+                                          replies);
+
+        if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
+            ret = afr_shd_entry_purge(subvol, parent_idx_inode, name,
+                                      inode->ia_type);
+            /* Why is ret force-set to 0? We do not care about
+             * index purge failing for full heal as it is quite
+             * possible during replace-brick that not all files
+             * and directories have their name indices present in
+             * entry-changes/.
+             */
+            ret = 0;
         }
+    }
+
+unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (par_replies)
+        afr_replies_wipe(par_replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
+}
 
-        inode = inode_link (loc.inode, NULL, NULL, &iatt);
+static inode_t *
+afr_shd_entry_changes_index_inode(xlator_t *this, xlator_t *subvol,
+                                  uuid_t pargfid)
+{
+    int ret = -1;
+    void *index_gfid = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
+
+    ret = syncop_getxattr(subvol, &rootloc, &xattr,
+                          GF_XATTROP_ENTRY_CHANGES_GFID, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, GF_XATTROP_ENTRY_CHANGES_GFID, &index_gfid);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(loc.pargfid, index_gfid);
+    loc.name = gf_strdup(uuid_utoa(pargfid));
+
+    ret = syncop_lookup(subvol, &loc, &iatt, NULL, NULL, NULL);
+    if (ret < 0) {
+        errno = -ret;
+        goto out;
+    }
+
+    inode = inode_link(loc.inode, NULL, NULL, &iatt);
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-        loc_wipe (&rootloc);
-        GF_FREE ((char *)loc.name);
-        loc_wipe (&loc);
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
+    GF_FREE((char *)loc.name);
+    loc_wipe(&loc);
 
-        return inode;
+    return inode;
 }
 
 static int
-afr_selfheal_entry_do_subvol (call_frame_t *frame, xlator_t *this,
-                              fd_t *fd, int child)
+afr_selfheal_entry_do_subvol(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                             int child)
 {
-	int ret = 0;
-	gf_dirent_t entries;
-	gf_dirent_t *entry = NULL;
-	off_t offset = 0;
-	call_frame_t *iter_frame = NULL;
-	xlator_t *subvol = NULL;
-	afr_private_t *priv = NULL;
-        gf_boolean_t mismatch = _gf_false;
-        afr_local_t *iter_local = NULL;
-        afr_local_t *local = NULL;
-        loc_t loc = {0,};
-
-	priv = this->private;
-	subvol = priv->children[child];
-
-	INIT_LIST_HEAD (&entries.list);
-
-        local = frame->local;
-
-	iter_frame = afr_copy_frame (frame);
-	if (!iter_frame)
-		return -ENOMEM;
-
-        loc.inode = afr_shd_entry_changes_index_inode (this, subvol,
-                                                       fd->inode->gfid);
-
-	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries,
-                                      NULL, NULL))) {
-		if (ret > 0)
-			ret = 0;
-		list_for_each_entry (entry, &entries.list, list) {
-			offset = entry->d_off;
-
-			if (!strcmp (entry->d_name, ".") ||
-			    !strcmp (entry->d_name, ".."))
-				continue;
-
-			if (__is_root_gfid (fd->inode->gfid) &&
-			    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR))
-				continue;
-
-			ret = afr_selfheal_entry_dirent (iter_frame, this, fd,
-                                                         entry->d_name,
-                                                         loc.inode, subvol,
-                                                        local->need_full_crawl);
-			AFR_STACK_RESET (iter_frame);
-			if (iter_frame->local == NULL) {
-                                ret = -ENOTCONN;
-                                break;
-                        }
-
-                        if (ret == -1) {
-                                /* gfid or type mismatch. */
-                                mismatch = _gf_true;
-                                ret = 0;
-                        }
-			if (ret)
-				break;
-		}
-
-		gf_dirent_free (&entries);
-		if (ret)
-			break;
-	}
-
-        loc_wipe (&loc);
-
-	AFR_STACK_DESTROY (iter_frame);
-        if (mismatch == _gf_true)
-                /* undo pending will be skipped */
-                ret = -1;
-	return ret;
-}
+    int ret = 0;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    off_t offset = 0;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    INIT_LIST_HEAD(&entries.list);
+
+    local = frame->local;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame)
+        return -ENOMEM;
+
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+
+    while ((ret = syncop_readdir(subvol, fd, 131072, offset, &entries, NULL,
+                                 NULL))) {
+        if (ret > 0)
+            ret = 0;
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
 
-static int
-afr_selfheal_entry_granular_dirent (xlator_t *subvol, gf_dirent_t *entry,
-                                    loc_t *parent, void *data)
-{
-        int                      ret  = 0;
-        loc_t                    loc  = {0,};
-        struct iatt              iatt = {0,};
-        afr_granular_esh_args_t *args = data;
-
-        /* Look up the actual inode associated with entry. If the lookup returns
-         * ESTALE or ENOENT, then it means we have a stale index. Remove it.
-         * This is analogous to the check in afr_shd_index_heal() except that
-         * here it is achieved through LOOKUP and in afr_shd_index_heal() through
-         * a GETXATTR.
-         */
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
 
-        loc.inode = inode_new (args->xl->itable);
-        loc.parent = inode_ref (args->heal_fd->inode);
-        gf_uuid_copy (loc.pargfid, loc.parent->gfid);
-        loc.name = entry->d_name;
+            ret = afr_selfheal_entry_dirent(iter_frame, this, fd, entry->d_name,
+                                            loc.inode, subvol,
+                                            local->need_full_crawl);
+            AFR_STACK_RESET(iter_frame);
+            if (iter_frame->local == NULL) {
+                ret = -ENOTCONN;
+                break;
+            }
 
-        ret = syncop_lookup (args->xl, &loc, &iatt, NULL, NULL, NULL);
-        if ((ret == -ENOENT) || (ret == -ESTALE)) {
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name);
+            if (ret == -1) {
+                /* gfid or type mismatch. */
+                mismatch = _gf_true;
                 ret = 0;
-                goto out;
+            }
+            if (ret)
+                break;
         }
-        /* TBD: afr_shd_zero_xattrop? */
 
-        ret = afr_selfheal_entry_dirent (args->frame, args->xl, args->heal_fd,
-                                         entry->d_name, parent->inode, subvol,
-                                         _gf_false);
-        AFR_STACK_RESET (args->frame);
-        if (args->frame->local == NULL)
-                ret = -ENOTCONN;
+        gf_dirent_free(&entries);
+        if (ret)
+            break;
+    }
 
-        if (ret == -1)
-                args->mismatch = _gf_true;
+    loc_wipe(&loc);
 
-out:
-        loc_wipe (&loc);
-        return 0;
+    AFR_STACK_DESTROY(iter_frame);
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
 }
 
 static int
-afr_selfheal_entry_granular (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                             int subvol_idx, gf_boolean_t is_src)
+afr_selfheal_entry_granular_dirent(xlator_t *subvol, gf_dirent_t *entry,
+                                   loc_t *parent, void *data)
 {
-        int                         ret    = 0;
-        loc_t                       loc    = {0,};
-        xlator_t                   *subvol = NULL;
-        afr_private_t              *priv   = NULL;
-        afr_granular_esh_args_t     args   = {0,};
-
-        priv = this->private;
-        subvol = priv->children[subvol_idx];
-
-        args.frame = afr_copy_frame (frame);
-        args.xl = this;
-        /* args.heal_fd represents the fd associated with the original directory
-         * on which entry heal is being attempted.
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    afr_granular_esh_args_t *args = data;
+
+    /* Look up the actual inode associated with entry. If the lookup returns
+     * ESTALE or ENOENT, then it means we have a stale index. Remove it.
+     * This is analogous to the check in afr_shd_index_heal() except that
+     * here it is achieved through LOOKUP and in afr_shd_index_heal() through
+     * a GETXATTR.
+     */
+
+    loc.inode = inode_new(args->xl->itable);
+    loc.parent = inode_ref(args->heal_fd->inode);
+    gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+    loc.name = entry->d_name;
+
+    ret = syncop_lookup(args->xl, &loc, &iatt, NULL, NULL, NULL);
+    if ((ret == -ENOENT) || (ret == -ESTALE)) {
+        /* The name indices under the pgfid index dir are guaranteed
+         * to be regular files. Hence the hardcoding.
          */
-        args.heal_fd = fd;
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, IA_IFREG);
+        ret = 0;
+        goto out;
+    }
+    /* TBD: afr_shd_zero_xattrop? */
+
+    ret = afr_selfheal_entry_dirent(args->frame, args->xl, args->heal_fd,
+                                    entry->d_name, parent->inode, subvol,
+                                    _gf_false);
+    AFR_STACK_RESET(args->frame);
+    if (args->frame->local == NULL)
+        ret = -ENOTCONN;
+
+    if (ret == -1)
+        args->mismatch = _gf_true;
 
-        /* @subvol here represents the subvolume of AFR where
-         * indices/entry-changes/<pargfid> will be processed
-         */
-        loc.inode = afr_shd_entry_changes_index_inode (this, subvol,
-                                                       fd->inode->gfid);
-        if (!loc.inode) {
-                /* If granular heal failed on the sink (as it might sometimes
-                 * because it is the src that would mostly contain the granular
-                 * changelogs and the sink's entry-changes would be empty),
-                 * do not treat heal as failure.
-                 */
-                if (is_src)
-                        return -errno;
-                else
-                        return 0;
-        }
+out:
+    loc_wipe(&loc);
+    return 0;
+}
 
-        ret = syncop_dir_scan (subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
-                               &args, afr_selfheal_entry_granular_dirent);
+static int
+afr_selfheal_entry_granular(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            int subvol_idx, gf_boolean_t is_src)
+{
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    afr_granular_esh_args_t args = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[subvol_idx];
+
+    args.frame = afr_copy_frame(frame);
+    if (!args.frame)
+        goto out;
+    args.xl = this;
+    /* args.heal_fd represents the fd associated with the original directory
+     * on which entry heal is being attempted.
+     */
+    args.heal_fd = fd;
+
+    /* @subvol here represents the subvolume of AFR where
+     * indices/entry-changes/<pargfid> will be processed
+     */
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+    if (!loc.inode) {
+        /* If granular heal failed on the sink (as it might sometimes
+         * because it is the src that would mostly contain the granular
+         * changelogs and the sink's entry-changes would be empty),
+         * do not treat heal as failure.
+         */
+        if (is_src)
+            ret = -errno;
+        else
+            ret = 0;
+        goto out;
+    }
 
-        loc_wipe (&loc);
+    ret = syncop_dir_scan(subvol, &loc, GF_CLIENT_PID_SELF_HEALD, &args,
+                          afr_selfheal_entry_granular_dirent);
 
-        if (args.mismatch == _gf_true)
-                ret = -1;
+    loc_wipe(&loc);
 
-        return ret;
+    if (args.mismatch == _gf_true)
+        ret = -1;
+out:
+    if (args.frame)
+        AFR_STACK_DESTROY(args.frame);
+    return ret;
 }
 
 static int
-afr_selfheal_entry_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       int source, unsigned char *sources,
-		       unsigned char *healed_sinks)
+afr_selfheal_entry_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                      unsigned char *sources, unsigned char *healed_sinks)
 {
-	int i                   = 0;
-	int ret                 = 0;
-        gf_boolean_t   mismatch = _gf_false;
-        afr_local_t   *local    = NULL;
-	afr_private_t *priv     = NULL;
-
-	priv = this->private;
-        local = frame->local;
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                AFR_MSG_SELF_HEAL_INFO, "performing entry selfheal on %s",
-		uuid_utoa (fd->inode->gfid));
-
-	for (i = 0; i < priv->child_count; i++) {
-                /* Expunge */
-		if (!healed_sinks[i])
-			continue;
-
-                if (!local->need_full_crawl)
-                /* Why call afr_selfheal_entry_granular() on a "healed sink",
-                 * given that it is the source that contains the granular
-                 * indices?
-                 * If the index for this directory is non-existent or empty on
-                 * this subvol (=> clear sink), then it will return early
-                 * without failure status.
-                 * If the index is non-empty and it is yet a 'healed sink', then
-                 * it is due to a split-brain in which case we anyway need to
-                 * crawl the indices/entry-changes/pargfid directory.
-                 */
-                        ret = afr_selfheal_entry_granular (frame, this, fd, i,
-                                                           _gf_false);
-                else
-                        ret = afr_selfheal_entry_do_subvol (frame, this, fd, i);
-
-                if (ret == -1) {
-                        /* gfid or type mismatch. */
-                        mismatch = _gf_true;
-                        ret = 0;
-                }
-                if (ret)
-                        break;
-	}
-
-        if (!ret && source != -1) {
-                /* Impunge */
-                if (local->need_full_crawl)
-                        ret = afr_selfheal_entry_do_subvol (frame, this, fd,
-                                                            source);
-                else
-                        ret = afr_selfheal_entry_granular (frame, this, fd,
-                                                           source, _gf_true);
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing entry selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    for (i = 0; i < priv->child_count; i++) {
+        /* Expunge */
+        if (!healed_sinks[i])
+            continue;
+
+        if (!local->need_full_crawl)
+            /* Why call afr_selfheal_entry_granular() on a "healed sink",
+             * given that it is the source that contains the granular
+             * indices?
+             * If the index for this directory is non-existent or empty on
+             * this subvol (=> clear sink), then it will return early
+             * without failure status.
+             * If the index is non-empty and it is yet a 'healed sink', then
+             * it is due to a split-brain in which case we anyway need to
+             * crawl the indices/entry-changes/pargfid directory.
+             */
+            ret = afr_selfheal_entry_granular(frame, this, fd, i, _gf_false);
+        else
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, i);
+
+        if (ret == -1) {
+            /* gfid or type mismatch. */
+            mismatch = _gf_true;
+            ret = 0;
         }
+        if (ret)
+            break;
+    }
 
-        if (mismatch == _gf_true)
-                /* undo pending will be skipped */
-                ret = -1;
-	return ret;
+    if (!ret && source != -1) {
+        /* Impunge */
+        if (local->need_full_crawl)
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, source);
+        else
+            ret = afr_selfheal_entry_granular(frame, this, fd, source,
+                                              _gf_true);
+    }
+
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
 }
 
 static int
-__afr_selfheal_entry (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      unsigned char *locked_on)
+__afr_selfheal_entry(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     unsigned char *locked_on)
 {
-	int                     ret                   = -1;
-	int                     source                = -1;
-	unsigned char          *sources               = NULL;
-	unsigned char          *sinks                 = NULL;
-	unsigned char          *data_lock             = NULL;
-        unsigned char          *postop_lock           = NULL;
-	unsigned char          *healed_sinks          = NULL;
-	struct afr_reply       *locked_replies        = NULL;
-        afr_local_t            *local                 = NULL;
-	afr_private_t          *priv                  = NULL;
-        gf_boolean_t            did_sh                = _gf_true;
-
-	priv = this->private;
-        local = frame->local;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-        postop_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-				    data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "entry self-heal as only %d sub-volumes could "
-                                      "be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid), ret,
-                                      this->name);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_entry_prepare (frame, this, fd->inode,
-                                                    data_lock, sources, sinks,
-                                                    healed_sinks,
-						    locked_replies, &source,
-                                                    NULL);
-                if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-                local->need_full_crawl = afr_need_full_heal (this,
-                                                             locked_replies,
-                                                             source,
-                                                             healed_sinks,
-                                                         AFR_ENTRY_TRANSACTION);
-	}
-unlock:
-	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-				data_lock, NULL);
-	if (ret < 0)
-		goto out;
+    int ret = -1;
+    int source = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *postop_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t did_sh = _gf_true;
+
+    priv = this->private;
+    local = frame->local;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    postop_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-        if (!did_sh)
-                goto out;
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, data_lock,
+                                           sources, sinks, healed_sinks,
+                                           locked_replies, &source, NULL);
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
 
-	ret = afr_selfheal_entry_do (frame, this, fd, source, sources,
-                                     healed_sinks);
-	if (ret)
-		goto out;
-
-        /* Take entrylks in xlator domain before doing post-op (undo-pending) in
-         * entry self-heal. This is to prevent a parallel name self-heal on
-         * an entry under @fd->inode from reading pending xattrs while it is
-         * being modified by SHD after entry sh below, given that
-         * name self-heal takes locks ONLY in xlator domain and is free to read
-         * pending changelog in the absence of the following locking.
-         */
-        ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-                                    postop_lock);
-        {
-                if (AFR_CMP (data_lock, postop_lock, priv->child_count) != 0) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "post-op after entry self-heal as %d "
-                                      "sub-volumes, as opposed to %d, "
-                                      "could be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid),
-                                      ret, AFR_COUNT (data_lock,
-                                      priv->child_count), this->name);
-                        ret = -ENOTCONN;
-                        goto postop_unlock;
-                }
-
-                ret = afr_selfheal_undo_pending (frame, this, fd->inode,
-                                                 sources, sinks, healed_sinks,
-                                                 AFR_ENTRY_TRANSACTION,
-                                                 locked_replies, postop_lock);
+        local->need_full_crawl = afr_need_full_heal(
+            this, locked_replies, source, healed_sinks, AFR_ENTRY_TRANSACTION);
+    }
+unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, data_lock,
+                           NULL);
+    if (ret < 0)
+        goto out;
+
+    if (!did_sh)
+        goto out;
+
+    ret = afr_selfheal_entry_do(frame, this, fd, source, sources, healed_sinks);
+    if (ret)
+        goto out;
+
+    /* Take entrylks in xlator domain before doing post-op (undo-pending) in
+     * entry self-heal. This is to prevent a parallel name self-heal on
+     * an entry under @fd->inode from reading pending xattrs while it is
+     * being modified by SHD after entry sh below, given that
+     * name self-heal takes locks ONLY in xlator domain and is free to read
+     * pending changelog in the absence of the following locking.
+     */
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               postop_lock);
+    {
+        if (AFR_CMP(data_lock, postop_lock, priv->child_count) != 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "post-op after entry self-heal as %d "
+                         "sub-volumes, as opposed to %d, "
+                         "could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret,
+                         AFR_COUNT(data_lock, priv->child_count), this->name);
+            ret = -ENOTCONN;
+            goto postop_unlock;
         }
+
+        afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                                  locked_replies);
+        ret = afr_selfheal_undo_pending(
+            frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_ENTRY_TRANSACTION, locked_replies, postop_lock);
+    }
 postop_unlock:
-        afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-                                postop_lock, NULL);
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL,
+                           postop_lock, NULL);
 out:
-        if (did_sh)
-                afr_log_selfheal (fd->inode->gfid, this, ret, "entry", source,
-                                  sources, healed_sinks);
-        else
-                ret = 1;
-
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-	return ret;
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "entry", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
 }
 
-
 static fd_t *
-afr_selfheal_data_opendir (xlator_t *this, inode_t *inode)
+afr_selfheal_data_opendir(xlator_t *this, inode_t *inode)
 {
-	loc_t loc = {0,};
-	int ret = 0;
-	fd_t *fd = NULL;
-
-	fd = fd_create (inode, 0);
-	if (!fd)
-		return NULL;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	ret = syncop_opendir (this, &loc, fd, NULL, NULL);
-	if (ret) {
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-	}
-
-	loc_wipe (&loc);
-        return fd;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    fd_t *fd = NULL;
+
+    fd = fd_create(inode, 0);
+    if (!fd)
+        return NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_opendir(this, &loc, fd, NULL, NULL);
+    if (ret) {
+        fd_unref(fd);
+        fd = NULL;
+    } else {
+        fd_bind(fd);
+    }
+
+    loc_wipe(&loc);
+    return fd;
 }
 
-
 int
-afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode)
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *locked_on = NULL;
-        unsigned char *long_name_locked = NULL;
-	fd_t *fd = NULL;
-	int ret = 0;
-	gf_boolean_t granular_locks = _gf_false;
-
-	priv = this->private;
-	if (strcmp ("granular", priv->locking_scheme) == 0)
-	        granular_locks = _gf_true;
-
-	fd = afr_selfheal_data_opendir (this, inode);
-	if (!fd)
-		return -EIO;
-
-	locked_on = alloca0 (priv->child_count);
-	long_name_locked = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_tie_breaker_entrylk (frame, this, inode,
-	                                        priv->sh_domain, NULL,
-                                                locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "entry self-heal as only %d sub-volumes could "
-                                      "be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid), ret,
-                                      priv->sh_domain);
-			/* Either less than two subvols available, or another
-			   selfheal (from another server) is in progress. Skip
-			   for now in any case there isn't anything to do.
-			*/
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-                if (!granular_locks) {
-                        ret = afr_selfheal_tryentrylk (frame, this, inode,
-                                                      this->name, LONG_FILENAME,
-                                                      long_name_locked);
-                }
-                {
-                        if (!granular_locks && ret < 1) {
-                                gf_msg_debug (this->name, 0, "%s: Skipping"
-                                              " entry self-heal as only %d "
-                                              "sub-volumes could be "
-                                              "locked in special-filename "
-                                              "domain",
-                                              uuid_utoa (fd->inode->gfid),
-                                              ret);
-                                ret = -ENOTCONN;
-                                goto unlock;
-                        }
-                        ret = __afr_selfheal_entry (frame, this, fd, locked_on);
-                }
-                if (!granular_locks)
-                        afr_selfheal_unentrylk (frame, this, inode, this->name,
-                                               LONG_FILENAME, long_name_locked,
-                                               NULL);
-	}
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    priv = this->private;
+
+    fd = afr_selfheal_data_opendir(this, inode);
+    if (!fd)
+        return -EIO;
+
+    locked_on = alloca0(priv->child_count);
+
+    ret = afr_selfheal_tie_breaker_entrylk(frame, this, inode, priv->sh_domain,
+                                           NULL, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, priv->sh_domain);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry(frame, this, fd, locked_on);
+    }
 unlock:
-	afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL,
-                                locked_on, NULL);
+    afr_selfheal_unentrylk(frame, this, inode, priv->sh_domain, NULL, locked_on,
+                           NULL);
 
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 130a3daa203..03f43bad16e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -8,106 +8,108 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
+#include <glusterfs/events.h>
 
-#define AFR_HEAL_ATTR (GF_SET_ATTR_UID|GF_SET_ATTR_GID|GF_SET_ATTR_MODE)
+#define AFR_HEAL_ATTR (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE)
 
 static gf_boolean_t
-_afr_ignorable_key_match (dict_t *d, char *k, data_t *val, void *mdata)
+_afr_ignorable_key_match(dict_t *d, char *k, data_t *val, void *mdata)
 {
-        return afr_is_xattr_ignorable (k);
+    return afr_is_xattr_ignorable(k);
 }
 
 void
-afr_delete_ignorable_xattrs (dict_t *xattr)
+afr_delete_ignorable_xattrs(dict_t *xattr)
 {
-        dict_foreach_match (xattr, _afr_ignorable_key_match, NULL,
-                            dict_remove_foreach_fn, NULL);
+    dict_foreach_match(xattr, _afr_ignorable_key_match, NULL,
+                       dict_remove_foreach_fn, NULL);
 }
 
 int
-__afr_selfheal_metadata_do (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                            int source, unsigned char *healed_sinks,
-                            struct afr_reply *locked_replies)
+__afr_selfheal_metadata_do(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, unsigned char *healed_sinks,
+                           struct afr_reply *locked_replies)
 {
-	int ret = -1;
-	loc_t loc = {0,};
-	dict_t *xattr = NULL;
-	dict_t *old_xattr = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-
-	priv = this->private;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                AFR_MSG_SELF_HEAL_INFO, "performing metadata selfheal on %s",
-		uuid_utoa (inode->gfid));
-
-	ret = syncop_getxattr (priv->children[source], &loc, &xattr, NULL,
-                               NULL, NULL);
-	if (ret < 0) {
-		ret = -EIO;
-                goto out;
-	}
-
-	afr_delete_ignorable_xattrs (xattr);
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (old_xattr) {
-                        dict_unref (old_xattr);
-                        old_xattr = NULL;
-                }
-
-		if (!healed_sinks[i])
-			continue;
-
-		ret = syncop_setattr (priv->children[i], &loc,
-				      &locked_replies[source].poststat,
-				      AFR_HEAL_ATTR, NULL, NULL, NULL, NULL);
-		if (ret)
-			healed_sinks[i] = 0;
-
-		ret = syncop_getxattr (priv->children[i], &loc, &old_xattr, 0,
-                                       NULL, NULL);
-		if (old_xattr) {
-			afr_delete_ignorable_xattrs (old_xattr);
-			ret = syncop_removexattr (priv->children[i], &loc, "",
-						  old_xattr, NULL);
-		}
-
-		ret = syncop_setxattr (priv->children[i], &loc, xattr, 0, NULL,
-                                       NULL);
-		if (ret)
-			healed_sinks[i] = 0;
-	}
-        ret = 0;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    dict_t *old_xattr = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing metadata selfheal on %s", uuid_utoa(inode->gfid));
+
+    ret = syncop_getxattr(priv->children[source], &loc, &xattr, NULL, NULL,
+                          NULL);
+    if (ret < 0) {
+        ret = -EIO;
+        goto out;
+    }
+
+    afr_delete_ignorable_xattrs(xattr);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (old_xattr) {
+            dict_unref(old_xattr);
+            old_xattr = NULL;
+        }
+
+        if (!healed_sinks[i])
+            continue;
+
+        ret = syncop_setattr(priv->children[i], &loc,
+                             &locked_replies[source].poststat, AFR_HEAL_ATTR,
+                             NULL, NULL, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+
+        ret = syncop_getxattr(priv->children[i], &loc, &old_xattr, 0, NULL,
+                              NULL);
+        if (old_xattr) {
+            afr_delete_ignorable_xattrs(old_xattr);
+            ret = syncop_removexattr(priv->children[i], &loc, "", old_xattr,
+                                     NULL);
+            if (ret)
+                healed_sinks[i] = 0;
+        }
+
+        ret = syncop_setxattr(priv->children[i], &loc, xattr, 0, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+    }
+    ret = 0;
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
-        if (old_xattr)
-                dict_unref (old_xattr);
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
+    if (old_xattr)
+        dict_unref(old_xattr);
 
-	return ret;
+    return ret;
 }
 
 static uint64_t
 mtime_ns(struct iatt *ia)
 {
-        uint64_t ret;
+    uint64_t ret;
 
-        ret = (((uint64_t)(ia->ia_mtime)) * 1000000000)
-            + (uint64_t)(ia->ia_mtime_nsec);
+    ret = (((uint64_t)(ia->ia_mtime)) * 1000000000) +
+          (uint64_t)(ia->ia_mtime_nsec);
 
-        return ret;
+    return ret;
 }
 
 /*
@@ -120,357 +122,425 @@ mtime_ns(struct iatt *ia)
  * the source with the most recent modification date.
  */
 static int
-afr_dirtime_splitbrain_source (call_frame_t *frame, xlator_t *this,
-                               struct afr_reply *replies,
-                               unsigned char *locked_on)
+afr_dirtime_splitbrain_source(call_frame_t *frame, xlator_t *this,
+                              struct afr_reply *replies,
+                              unsigned char *locked_on)
 {
-        afr_private_t *priv  = NULL;
-        int            source = -1;
-        struct iatt    source_ia;
-        struct iatt    child_ia;
-        uint64_t       mtime = 0;
-        int            i;
-        int            ret   = -1;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!locked_on[i])
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
-
-                if (mtime_ns(&replies[i].poststat) <= mtime)
-                        continue;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    struct iatt source_ia;
+    struct iatt child_ia;
+    uint64_t mtime = 0;
+    int i;
+    int ret = -1;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (mtime_ns(&replies[i].poststat) <= mtime)
+            continue;
+
+        mtime = mtime_ns(&replies[i].poststat);
+        source = i;
+    }
+
+    if (source == -1)
+        goto out;
+
+    source_ia = replies[source].poststat;
+    if (source_ia.ia_type != IA_IFDIR)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        child_ia = replies[i].poststat;
+
+        if (!IA_EQUAL(source_ia, child_ia, gfid) ||
+            !IA_EQUAL(source_ia, child_ia, type) ||
+            !IA_EQUAL(source_ia, child_ia, prot) ||
+            !IA_EQUAL(source_ia, child_ia, uid) ||
+            !IA_EQUAL(source_ia, child_ia, gid) ||
+            !afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata))
+            goto out;
+    }
+
+    /*
+     * Metadata split brain is just about [amc]time
+     * We return our source.
+     */
+    ret = source;
+out:
+    return ret;
+}
 
-                mtime = mtime_ns(&replies[i].poststat);
-                source = i;
+static int
+__afr_selfheal_metadata_mark_pending_xattrs(call_frame_t *frame, xlator_t *this,
+                                            inode_t *inode,
+                                            struct afr_reply *replies,
+                                            unsigned char *sources)
+{
+    int ret = 0;
+    int i = 0;
+    int m_idx = 0;
+    afr_private_t *priv = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    raw[m_idx] = 1;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            ret = -1;
+            goto out;
         }
-
-        if (source == -1)
-                goto out;
-
-        source_ia = replies[source].poststat;
-        if (source_ia.ia_type != IA_IFDIR)
-                goto out;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source)
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
-
-                child_ia = replies[i].poststat;
-
-                if (!IA_EQUAL(source_ia, child_ia, gfid) ||
-                    !IA_EQUAL(source_ia, child_ia, type) ||
-                    !IA_EQUAL(source_ia, child_ia, prot) ||
-                    !IA_EQUAL(source_ia, child_ia, uid) ||
-                    !IA_EQUAL(source_ia, child_ia, gid) ||
-                    !afr_xattrs_are_equal (replies[source].xdata,
-                                           replies[i].xdata))
-                        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret = afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, AFR_MSG_SELF_HEAL_INFO,
+                   "Failed to set pending metadata xattr on child %d for %s", i,
+                   uuid_utoa(inode->gfid));
+            goto out;
         }
+    }
+
+    afr_replies_wipe(replies, priv->child_count);
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
 
-        /*
-         * Metadata split brain is just about [amc]time
-         * We return our source.
-         */
-        ret = source;
 out:
-        return ret;
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
-
 /*
  * Look for mismatching uid/gid or mode or user xattrs even if
  * AFR xattrs don't say so, and pick one arbitrarily as winner. */
 
 static int
-__afr_selfheal_metadata_finalize_source (call_frame_t *frame, xlator_t *this,
-                                         inode_t *inode,
-                                         unsigned char *sources,
-                                         unsigned char *sinks,
-					 unsigned char *healed_sinks,
-					 unsigned char *locked_on,
-					 struct afr_reply *replies)
+__afr_selfheal_metadata_finalize_source(call_frame_t *frame, xlator_t *this,
+                                        inode_t *inode, unsigned char *sources,
+                                        unsigned char *sinks,
+                                        unsigned char *healed_sinks,
+                                        unsigned char *undid_pending,
+                                        unsigned char *locked_on,
+                                        struct afr_reply *replies)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	struct iatt srcstat = {0, };
-	int source = -1;
-	int sources_count = 0;
-
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-	    || !sources_count) {
-
-                source = afr_mark_split_brain_source_sinks (frame, this, inode,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                      AFR_METADATA_TRANSACTION);
-                if (source >= 0)
-                        return source;
-
-		/* If this is a directory mtime/ctime only split brain
-		   use the most recent */
-		source = afr_dirtime_splitbrain_source (frame, this,
-							replies, locked_on);
-		if (source != -1) {
-		        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_SPLIT_BRAIN, "clear time "
-				"split brain on %s",
-				 uuid_utoa (replies[source].poststat.ia_gfid));
-			sources[source] = 1;
-			healed_sinks[source] = 0;
-			return source;
-		}
-
-		if (!priv->metadata_splitbrain_forced_heal) {
-			return -EIO;
-		}
-
-		/* Metadata split brain, select one subvol
-		   arbitrarily */
-		for (i = 0; i < priv->child_count; i++) {
-			if (locked_on[i] && healed_sinks[i]) {
-				sources[i] = 1;
-				healed_sinks[i] = 0;
-				break;
-			}
-		}
-	}
-
-        /* No split brain at this point. If we were called from
-         * afr_heal_splitbrain_file(), abort.*/
-        if (afr_dict_contains_heal_op(frame))
-                return -EIO;
-
-        source = afr_choose_source_by_policy (priv, sources,
-                                              AFR_METADATA_TRANSACTION);
-        srcstat = replies[source].poststat;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i] || i == source)
-			continue;
-		if (!IA_EQUAL (srcstat, replies[i].poststat, type) ||
-		    !IA_EQUAL (srcstat, replies[i].poststat, uid) ||
-		    !IA_EQUAL (srcstat, replies[i].poststat, gid) ||
-		    !IA_EQUAL (srcstat, replies[i].poststat, prot)) {
-                        gf_msg_debug (this->name, 0, "%s: iatt mismatch "
-                                      "for source(%d) vs (%d)",
-                                      uuid_utoa
-                                      (replies[source].poststat.ia_gfid),
-                                      source, i);
-			sources[i] = 0;
-			healed_sinks[i] = 1;
-		}
-	}
-
-        for (i =0; i < priv->child_count; i++) {
-		if (!sources[i] || i == source)
-			continue;
-                if (!afr_xattrs_are_equal (replies[source].xdata,
-                                           replies[i].xdata)) {
-                        gf_msg_debug (this->name, 0, "%s: xattr mismatch "
-                                      "for source(%d) vs (%d)",
-                                      uuid_utoa
-                                      (replies[source].poststat.ia_gfid),
-                                      source, i);
-                        sources[i] = 0;
-                        healed_sinks[i] = 1;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt srcstat = {
+        0,
+    };
+    int source = -1;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_METADATA_TRANSACTION);
+        if (source >= 0) {
+            _afr_fav_child_reset_sink_xattrs(
+                frame, this, inode, source, healed_sinks, undid_pending,
+                AFR_METADATA_TRANSACTION, locked_on, replies);
+            goto out;
         }
 
-	return source;
-}
+        /* If this is a directory mtime/ctime only split brain
+           use the most recent */
+        source = afr_dirtime_splitbrain_source(frame, this, replies, locked_on);
+        if (source != -1) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SPLIT_BRAIN,
+                   "clear time "
+                   "split brain on %s",
+                   uuid_utoa(replies[source].poststat.ia_gfid));
+            sources[source] = 1;
+            healed_sinks[source] = 0;
+            goto out;
+        }
 
+        if (!priv->metadata_splitbrain_forced_heal) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=metadata;file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
+        }
+
+        /* Metadata split brain, select one subvol
+           arbitrarily */
+        for (i = 0; i < priv->child_count; i++) {
+            if (locked_on[i] && healed_sinks[i]) {
+                sources[i] = 1;
+                healed_sinks[i] = 0;
+                break;
+            }
+        }
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    source = afr_choose_source_by_policy(priv, sources,
+                                         AFR_METADATA_TRANSACTION);
+    srcstat = replies[source].poststat;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!IA_EQUAL(srcstat, replies[i].poststat, type) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, uid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, gid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: iatt mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: xattr mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
+        }
+    }
+    if ((sources_count == priv->child_count) && (source > -1) &&
+        (AFR_COUNT(healed_sinks, priv->child_count) != 0)) {
+        ret = __afr_selfheal_metadata_mark_pending_xattrs(frame, this, inode,
+                                                          replies, sources);
+        if (ret < 0)
+            return ret;
+    }
+out:
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    return source;
+}
 
 int
-__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this, inode_t *inode,
-				 unsigned char *locked_on, unsigned char *sources,
-				 unsigned char *sinks, unsigned char *healed_sinks,
-				 struct afr_reply *replies, gf_boolean_t *pflag)
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-	int i = 0;
-        uint64_t *witness = NULL;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    uint64_t *witness = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-        if (ret)
-                return ret;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-        ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_METADATA_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           pflag);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-        /* If any source has witness, pick first
-         * witness source and make everybody else sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] && witness[i]) {
-                        source = i;
-                        break;
-                }
-        }
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
+        return ret;
 
-        if (source != -1) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (i != source && sources[i]) {
-                                sources[i] = 0;
-                                healed_sinks[i] = 1;
-                        }
-                }
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_METADATA_TRANSACTION, locked_on,
+                                      sources, sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    /* If any source has witness, pick first
+     * witness source and make everybody else sinks */
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i]) {
+            source = i;
+            break;
         }
+    }
 
-	source = __afr_selfheal_metadata_finalize_source (frame, this, inode,
-                                                          sources, sinks,
-                                                          healed_sinks,
-                                                          locked_on, replies);
+    if (source != -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (i != source && sources[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
+        }
+    }
 
-	if (source < 0)
-		return -EIO;
+    source = __afr_selfheal_metadata_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+        locked_on, replies);
 
-	return source;
-}
+    if (source < 0)
+        return -EIO;
 
-int
-afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode)
-{
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *data_lock = NULL;
-	unsigned char *healed_sinks = NULL;
-	struct afr_reply *locked_replies = NULL;
-        gf_boolean_t did_sh = _gf_true;
-	int source = -1;
-
-	priv = this->private;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-				    LLONG_MAX - 1, 0, data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_metadata_prepare (frame, this, inode,
-                                                       data_lock, sources,
-                                                       sinks, healed_sinks,
-						       locked_replies, NULL);
-		if (ret < 0)
-			goto unlock;
-
-		source = ret;
-
-                if (AFR_COUNT (healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-                ret = __afr_selfheal_metadata_do (frame, this, inode, source,
-                                                  healed_sinks, locked_replies);
-                if (ret)
-                        goto unlock;
-
-                ret = afr_selfheal_undo_pending (frame, this, inode, sources,
-                                                 sinks, healed_sinks,
-                                                 AFR_METADATA_TRANSACTION,
-                                                 locked_replies, data_lock);
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, inode, this->name,
-				LLONG_MAX -1, 0, data_lock);
-
-        if (did_sh)
-                afr_log_selfheal (inode->gfid, this, ret, "metadata", source,
-                                  sources, healed_sinks);
-        else
-                ret = 1;
-
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-	return ret;
+    return source;
 }
 
 int
-afr_selfheal_metadata_by_stbuf (xlator_t *this, struct iatt *stbuf)
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        inode_t      *inode      = NULL;
-        inode_t      *link_inode = NULL;
-        call_frame_t *frame      = NULL;
-        int          ret         = 0;
-
-        if (gf_uuid_is_null (stbuf->ia_gfid)) {
-                ret = -EINVAL;
-                goto out;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    gf_boolean_t did_sh = _gf_true;
+    int source = -1;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-        inode = inode_new (this->itable);
-        if (!inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+        ret = __afr_selfheal_metadata_prepare(
+            frame, this, inode, data_lock, sources, sinks, healed_sinks,
+            undid_pending, locked_replies, NULL);
+        if (ret < 0)
+            goto unlock;
 
-        link_inode = inode_link (inode, NULL, NULL, stbuf);
-        if (!link_inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+        source = ret;
 
-        frame = afr_frame_create (this);
-        if (!frame) {
-                ret = -ENOMEM;
-                goto out;
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
         }
 
-        ret = afr_selfheal_metadata (frame, this, link_inode);
+        ret = __afr_selfheal_metadata_do(frame, this, inode, source,
+                                         healed_sinks, locked_replies);
+        if (ret)
+            goto unlock;
+
+        afr_selfheal_restore_time(frame, this, inode, source, healed_sinks,
+                                  locked_replies);
+
+        ret = afr_selfheal_undo_pending(
+            frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_METADATA_TRANSACTION, locked_replies, data_lock);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                           data_lock);
+
+    if (did_sh)
+        afr_log_selfheal(inode->gfid, this, ret, "metadata", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
+}
+
+int
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf)
+{
+    inode_t *inode = NULL;
+    inode_t *link_inode = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+
+    if (gf_uuid_is_null(stbuf->ia_gfid)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    inode = inode_new(this->itable);
+    if (!inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    link_inode = inode_link(inode, NULL, NULL, stbuf);
+    if (!link_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = afr_selfheal_metadata(frame, this, link_inode);
 out:
-        if (inode)
-                inode_unref (inode);
-        if (link_inode)
-                inode_unref (link_inode);
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-        return ret;
+    if (inode)
+        inode_unref(inode);
+    if (link_inode)
+        inode_unref(link_inode);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
index 3445ecccf9c..834aac86d48 100644
--- a/xlators/cluster/afr/src/afr-self-heal-name.c
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -8,712 +8,609 @@
   cases as published by the Free Software Foundation.
 */
 
-
+#include <glusterfs/events.h>
 #include "afr.h"
 #include "afr-self-heal.h"
 #include "afr-messages.h"
 
 int
-__afr_selfheal_assign_gfid (xlator_t *this, inode_t *parent, uuid_t pargfid,
-                            const char *bname, inode_t *inode,
-                            struct afr_reply *replies, void *gfid,
-                            unsigned char *locked_on,
-                            gf_boolean_t is_gfid_absent)
+__afr_selfheal_assign_gfid(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                           const char *bname, inode_t *inode,
+                           struct afr_reply *replies, void *gfid,
+                           unsigned char *locked_on, int source,
+                           unsigned char *sources, gf_boolean_t is_gfid_absent,
+                           int *gfid_idx)
 {
-	int             ret          = 0;
-        int             up_count     = 0;
-        int             locked_count = 0;
-	afr_private_t  *priv         = NULL;
-	dict_t         *xdata        = NULL;
-	loc_t           loc          = {0, };
-        call_frame_t   *new_frame    = NULL;
-        afr_local_t    *new_local    = NULL;
-
-	priv = this->private;
-
-        new_frame = afr_frame_create (this);
-        if (!new_frame) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        new_local = new_frame->local;
-
-	gf_uuid_copy (parent->gfid, pargfid);
+    int ret = 0;
+    int up_count = 0;
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
 
-	xdata = dict_new ();
-	if (!xdata) {
-                ret = -ENOMEM;
-		goto out;
-	}
+    priv = this->private;
 
-	ret = dict_set_static_bin (xdata, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -ENOMEM;
-		goto out;
-	}
+    gf_uuid_copy(parent->gfid, pargfid);
 
-	loc.parent = inode_ref (parent);
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.pargfid, pargfid);
-	loc.name = bname;
+    if (is_gfid_absent) {
+        /* Ensure all children of AFR are up before performing gfid heal, to
+         * guard against the possibility of gfid split brain. */
 
-        if (is_gfid_absent) {
-                /* Ensure all children of AFR are up before performing gfid heal, to
-                 * guard against the possibility of gfid split brain. */
-
-                up_count = AFR_COUNT (priv->child_up, priv->child_count);
-                if (up_count != priv->child_count) {
-                        ret = -EIO;
-                        goto out;
-                }
-
-                locked_count = AFR_COUNT (locked_on, priv->child_count);
-                if (locked_count != priv->child_count) {
-                        ret = -EIO;
-                        goto out;
-                }
+        up_count = AFR_COUNT(priv->child_up, priv->child_count);
+        if (up_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
         }
 
-        /* Clear out old replies here and wind lookup on all locked
-         * subvolumes to achieve two things:
-         *   a. gfid heal on those subvolumes that do not have gfid associated
-         *      with the inode, and
-         *   b. refresh replies, which can be consumed by
-         *      __afr_selfheal_name_impunge().
-         */
-
-        AFR_ONLIST (locked_on, new_frame, afr_selfheal_discover_cbk, lookup,
-                    &loc, xdata);
-
-        afr_replies_wipe (replies, priv->child_count);
+        locked_count = AFR_COUNT(locked_on, priv->child_count);
+        if (locked_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
+        }
+    }
 
-        afr_replies_copy (replies, new_local->replies, priv->child_count);
+    ret = afr_lookup_and_heal_gfid(this, parent, bname, inode, replies, source,
+                                   sources, gfid, gfid_idx);
 
 out:
-	loc_wipe (&loc);
-        if (xdata)
-                dict_unref (xdata);
-        if (new_frame)
-                AFR_STACK_DESTROY (new_frame);
-
-	return ret;
+    return ret;
 }
 
 int
-__afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this,
-                             inode_t *parent, uuid_t pargfid,
-                             const char *bname, inode_t *inode,
-                             struct afr_reply *replies, int gfid_idx)
+__afr_selfheal_name_impunge(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid, const char *bname,
+                            inode_t *inode, struct afr_reply *replies,
+                            int gfid_idx)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-        int ret = 0;
-        unsigned char *newentry = NULL;
-        unsigned char *sources = NULL;
-
-        priv = this->private;
-
-	newentry = alloca0 (priv->child_count);
-	sources = alloca0 (priv->child_count);
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    unsigned char *sources = NULL;
 
-	gf_uuid_copy (parent->gfid, pargfid);
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
+    sources = alloca0(priv->child_count);
 
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[gfid_idx].poststat.ia_gfid) == 0) {
-                        sources[i] = 1;
-			continue;
-                }
+    gf_uuid_copy(parent->gfid, pargfid);
 
-		ret |= afr_selfheal_recreate_entry (this, i, gfid_idx, parent,
-                                                    bname, inode, replies,
-                                                    newentry);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, gfid_idx, replies,
-					    sources, newentry);
-	return ret;
-}
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[gfid_idx].poststat.ia_gfid) == 0) {
+            sources[i] = 1;
+            continue;
+        }
+    }
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
 
-int
-__afr_selfheal_name_expunge (xlator_t *this, inode_t *parent, uuid_t pargfid,
-                             const char *bname, inode_t *inode,
-			     struct afr_reply *replies)
-{
-	loc_t loc = {0, };
-	int i = 0;
-	afr_private_t *priv = NULL;
-	char g[64];
-	int ret = 0;
-
-	priv = this->private;
-
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, pargfid);
-	loc.name = bname;
-	loc.inode = inode_ref (inode);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (replies[i].op_ret)
-			continue;
-
-		switch (replies[i].poststat.ia_type) {
-		case IA_IFDIR:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-			        "expunging dir %s/%s (%s) on %s",
-			        uuid_utoa (pargfid), bname,
-			        uuid_utoa_r (replies[i].poststat.ia_gfid, g),
-			        priv->children[i]->name);
-
-			ret |= syncop_rmdir (priv->children[i], &loc, 1, NULL,
-                                             NULL);
-			break;
-		default:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-		                "expunging file %s/%s (%s) on %s",
-			        uuid_utoa (pargfid), bname,
-			        uuid_utoa_r (replies[i].poststat.ia_gfid, g),
-			        priv->children[i]->name);
-
-			ret |= syncop_unlink (priv->children[i], &loc, NULL,
-                                              NULL);
-			break;
-		}
-	}
-
-	loc_wipe (&loc);
-
-	return ret;
+        ret |= afr_selfheal_recreate_entry(frame, i, gfid_idx, sources, parent,
+                                           bname, inode, replies);
+    }
 
+    return ret;
 }
 
-/* This function is to be called after ensuring that there is no gfid mismatch
- * for the inode across multiple sources
- */
-static int
-afr_selfheal_gfid_idx_get (xlator_t *this, struct afr_reply *replies,
-                           unsigned char *sources)
+int
+__afr_selfheal_name_expunge(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                            const char *bname, inode_t *inode,
+                            struct afr_reply *replies)
 {
-        int             i        =  0;
-        int             gfid_idx = -1;
-        afr_private_t  *priv     =  NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (!sources[i])
-                        continue;
+        if (replies[i].op_ret)
+            continue;
 
-                if (gf_uuid_is_null (replies[i].poststat.ia_gfid))
-                        continue;
+        ret |= afr_selfheal_entry_delete(this, parent, bname, inode, i,
+                                         replies);
+    }
 
-                gfid_idx = i;
-                break;
-        }
-        return gfid_idx;
+    return ret;
 }
 
 static gf_boolean_t
-afr_selfheal_name_need_heal_check (xlator_t *this, struct afr_reply *replies)
+afr_selfheal_name_need_heal_check(xlator_t *this, struct afr_reply *replies)
 {
-        int             i           = 0;
-	int             first_idx   = -1;
-        gf_boolean_t    need_heal   = _gf_false;
-        afr_private_t  *priv        = NULL;
-
-        priv = this->private;
+    int i = 0;
+    int first_idx = -1;
+    gf_boolean_t need_heal = _gf_false;
+    afr_private_t *priv = NULL;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
+    priv = this->private;
 
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        need_heal = _gf_true;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-		if (first_idx == -1) {
-			first_idx = i;
-			continue;
-		}
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA))
+            need_heal = _gf_true;
 
-		if (replies[i].op_ret != replies[first_idx].op_ret)
-			need_heal = _gf_true;
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
 
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first_idx].poststat.ia_gfid))
-			need_heal = _gf_true;
+        if (replies[i].op_ret != replies[first_idx].op_ret)
+            need_heal = _gf_true;
 
-                if ((replies[i].op_ret == 0) &&
-                    (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
-                        need_heal = _gf_true;
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid))
+            need_heal = _gf_true;
 
-	}
+        if ((replies[i].op_ret == 0) &&
+            (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
+            need_heal = _gf_true;
+    }
 
-        return need_heal;
+    return need_heal;
 }
 
 static int
-afr_selfheal_name_type_mismatch_check (xlator_t *this, struct afr_reply *replies,
-                                       int source, unsigned char *sources,
-                                       uuid_t pargfid, const char *bname)
+afr_selfheal_name_type_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      uuid_t pargfid, const char *bname)
 {
-        int             i           = 0;
-        int             type_idx    = -1;
-        ia_type_t       inode_type  = IA_INVAL;
-        afr_private_t  *priv        = NULL;
-
-        priv = this->private;
+    int i = 0;
+    int type_idx = -1;
+    ia_type_t inode_type = IA_INVAL;
+    ia_type_t inode_type1 = IA_INVAL;
+    afr_private_t *priv = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    priv = this->private;
 
-                if (replies[i].poststat.ia_type == IA_INVAL)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-                if (inode_type == IA_INVAL) {
-                        inode_type = replies[i].poststat.ia_type;
-                        type_idx = i;
-                        continue;
-                }
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
 
-                if (sources[i] || source == -1) {
-                        if ((sources[type_idx] || source == -1) &&
-                            (inode_type != replies[i].poststat.ia_type)) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        AFR_MSG_SPLIT_BRAIN,
-                                        "Type mismatch for <gfid:%s>/%s: "
-                                        "%d on %s and %d on %s",
-                                        uuid_utoa(pargfid), bname,
-                                        replies[i].poststat.ia_type,
-                                        priv->children[i]->name,
-                                        replies[type_idx].poststat.ia_type,
-                                        priv->children[type_idx]->name);
-
-                                    return -EIO;
-                        }
-                        inode_type = replies[i].poststat.ia_type;
-                        type_idx = i;
-                }
+        if (inode_type == IA_INVAL) {
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+            continue;
         }
-        return 0;
+        inode_type1 = replies[i].poststat.ia_type;
+        if (sources[i] || source == -1) {
+            if ((sources[type_idx] || source == -1) &&
+                (inode_type != inode_type1)) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Type mismatch for <gfid:%s>/%s: "
+                       "%s on %s and %s on %s",
+                       uuid_utoa(pargfid), bname,
+                       gf_inode_type_to_str(inode_type1),
+                       priv->children[i]->name,
+                       gf_inode_type_to_str(inode_type),
+                       priv->children[type_idx]->name);
+                gf_event(EVENT_AFR_SPLIT_BRAIN,
+                         "client-pid=%d;"
+                         "subvol=%s;type=file;"
+                         "file=<gfid:%s>/%s;count=2;"
+                         "child-%d=%s;type-%d=%s;child-%d=%s;"
+                         "type-%d=%s",
+                         this->ctx->cmd_args.client_pid, this->name,
+                         uuid_utoa(pargfid), bname, i, priv->children[i]->name,
+                         i, gf_inode_type_to_str(inode_type1), type_idx,
+                         priv->children[type_idx]->name, type_idx,
+                         gf_inode_type_to_str(inode_type));
+                return -EIO;
+            }
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+        }
+    }
+    return 0;
 }
 
 static int
-afr_selfheal_name_gfid_mismatch_check (xlator_t *this, struct afr_reply *replies,
-                                       int source, unsigned char *sources,
-                                       int *gfid_idx, uuid_t pargfid,
-                                       const char *bname)
+afr_selfheal_name_gfid_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      int *gfid_idx, uuid_t pargfid,
+                                      const char *bname, inode_t *inode,
+                                      unsigned char *locked_on, dict_t *xdata)
 {
-        int             i             = 0;
-	int             gfid_idx_iter = -1;
-        void           *gfid          = NULL;
-        afr_private_t  *priv          = NULL;
-	char g1[64], g2[64];
-
-        priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (gf_uuid_is_null (replies[i].poststat.ia_gfid))
-			continue;
-
-		if (!gfid) {
-			gfid = &replies[i].poststat.ia_gfid;
-			gfid_idx_iter = i;
-			continue;
-		}
-
-		if (sources[i] || source == -1) {
-			if ((sources[gfid_idx_iter] || source == -1) &&
-			    gf_uuid_compare (gfid, replies[i].poststat.ia_gfid)) {
-			        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        AFR_MSG_SPLIT_BRAIN,
-					"GFID mismatch for <gfid:%s>/%s "
-					"%s on %s and %s on %s",
-					uuid_utoa (pargfid), bname,
-					uuid_utoa_r (replies[i].poststat.ia_gfid, g1),
-					priv->children[i]->name,
-					uuid_utoa_r (replies[gfid_idx_iter].poststat.ia_gfid, g2),
-					priv->children[gfid_idx_iter]->name);
-
-				return -EIO;
-			}
-
-                        gfid = &replies[i].poststat.ia_gfid;
-			gfid_idx_iter = i;
-		}
-	}
-
-        *gfid_idx = gfid_idx_iter;
-        return 0;
+    int i = 0;
+    int gfid_idx_iter = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    void *gfid1 = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (!gfid) {
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+            continue;
+        }
+
+        gfid1 = &replies[i].poststat.ia_gfid;
+        if (sources[i] || source == -1) {
+            if ((sources[gfid_idx_iter] || source == -1) &&
+                gf_uuid_compare(gfid, gfid1)) {
+                ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                                  bname, gfid_idx_iter, i,
+                                                  locked_on, gfid_idx, xdata);
+                if (!ret && *gfid_idx >= 0) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   "GFID split-brain resolved");
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error setting gfid-"
+                               "heal-msg dict");
+                }
+                return ret;
+            }
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+        }
+    }
+
+    *gfid_idx = gfid_idx_iter;
+    return 0;
 }
 
 static gf_boolean_t
-afr_selfheal_name_source_empty_check (xlator_t *this, struct afr_reply *replies,
-                                      unsigned char *sources, int source)
+afr_selfheal_name_source_empty_check(xlator_t *this, struct afr_reply *replies,
+                                     unsigned char *sources, int source)
 {
-	int             i               = 0;
-	afr_private_t  *priv            = NULL;
-	gf_boolean_t    source_is_empty = _gf_true;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
 
-	priv = this->private;
+    priv = this->private;
 
-        if (source == -1) {
-                source_is_empty = _gf_false;
-                goto out;
-        }
+    if (source == -1) {
+        source_is_empty = _gf_false;
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
 
-                if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
-                        continue;
+        if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
+            continue;
 
-                source_is_empty = _gf_false;
-                break;
-	}
+        source_is_empty = _gf_false;
+        break;
+    }
 out:
-        return source_is_empty;
+    return source_is_empty;
 }
 
 int
-__afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
-                        uuid_t pargfid, const char *bname, inode_t *inode,
-                        unsigned char *sources, unsigned char *sinks,
-			unsigned char *healed_sinks, int source,
-			unsigned char *locked_on, struct afr_reply *replies,
-                        void *gfid_req)
+__afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                       uuid_t pargfid, const char *bname, inode_t *inode,
+                       unsigned char *sources, unsigned char *sinks,
+                       unsigned char *healed_sinks, int source,
+                       unsigned char *locked_on, struct afr_reply *replies,
+                       void *gfid_req, dict_t *xdata)
 {
-	int             gfid_idx        = -1;
-        int             ret             = -1;
-	void           *gfid            = NULL;
-	gf_boolean_t    source_is_empty = _gf_true;
-	gf_boolean_t    need_heal       = _gf_false;
-        gf_boolean_t    is_gfid_absent  = _gf_false;
-
-        need_heal = afr_selfheal_name_need_heal_check (this, replies);
-	if (!need_heal)
-		return 0;
-
-        source_is_empty = afr_selfheal_name_source_empty_check (this, replies,
-                                                                sources,
-                                                                source);
-	if (source_is_empty) {
-		ret = __afr_selfheal_name_expunge (this, parent, pargfid,
-						    bname, inode, replies);
-                if (ret == -EIO)
-                        ret = -1;
-                return ret;
-        }
-
-        ret = afr_selfheal_name_type_mismatch_check (this, replies, source,
-                                                     sources, pargfid, bname);
-        if (ret)
-                return ret;
+    int gfid_idx = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t is_gfid_absent = _gf_false;
+
+    need_heal = afr_selfheal_name_need_heal_check(this, replies);
+    if (!need_heal)
+        return 0;
 
-        ret = afr_selfheal_name_gfid_mismatch_check (this, replies, source,
-                                                     sources, &gfid_idx,
-                                                     pargfid, bname);
-        if (ret)
-                return ret;
+    source_is_empty = afr_selfheal_name_source_empty_check(this, replies,
+                                                           sources, source);
+    if (source_is_empty) {
+        ret = __afr_selfheal_name_expunge(this, parent, pargfid, bname, inode,
+                                          replies);
+        if (ret == -EIO)
+            ret = -1;
+        return ret;
+    }
 
-	if (gfid_idx == -1) {
-                if (!gfid_req || gf_uuid_is_null (gfid_req))
-                        return -1;
-                gfid = gfid_req;
-        } else {
-                gfid = &replies[gfid_idx].poststat.ia_gfid;
-        }
+    ret = afr_selfheal_name_type_mismatch_check(this, replies, source, sources,
+                                                pargfid, bname);
+    if (ret)
+        return ret;
 
-        is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
-	ret = __afr_selfheal_assign_gfid (this, parent, pargfid, bname, inode,
-                                          replies, gfid, locked_on,
-                                          is_gfid_absent);
-        if (ret)
-                return ret;
+    ret = afr_selfheal_name_gfid_mismatch_check(this, replies, source, sources,
+                                                &gfid_idx, pargfid, bname,
+                                                inode, locked_on, xdata);
+    if (ret)
+        return ret;
 
-        if (gfid_idx == -1) {
-                gfid_idx = afr_selfheal_gfid_idx_get (this, replies, sources);
-                if (gfid_idx == -1)
-                        return -1;
-        }
+    if (gfid_idx == -1) {
+        if (!gfid_req || gf_uuid_is_null(gfid_req))
+            return -1;
+        gfid = gfid_req;
+    } else {
+        gfid = &replies[gfid_idx].poststat.ia_gfid;
+        if (source == -1)
+            /* Either entry split-brain or dirty xattrs are present on parent.*/
+            source = gfid_idx;
+    }
+
+    is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
+    ret = __afr_selfheal_assign_gfid(this, parent, pargfid, bname, inode,
+                                     replies, gfid, locked_on, source, sources,
+                                     is_gfid_absent, &gfid_idx);
+    if (ret || (gfid_idx < 0))
+        return ret;
 
-	ret = __afr_selfheal_name_impunge (frame, this, parent, pargfid,
-                                           bname, inode,
-                                           replies, gfid_idx);
-        if (ret == -EIO)
-                ret = -1;
+    ret = __afr_selfheal_name_impunge(frame, this, parent, pargfid, bname,
+                                      inode, replies, gfid_idx);
+    if (ret == -EIO)
+        ret = -1;
 
-        return ret;
+    return ret;
 }
 
-
 int
-__afr_selfheal_name_finalize_source (xlator_t *this, unsigned char *sources,
-				     unsigned char *healed_sinks,
-				     unsigned char *locked_on,
-				     struct afr_reply *replies,
-                                     uint64_t *witness)
+__afr_selfheal_name_finalize_source(xlator_t *this, unsigned char *sources,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on, uint64_t *witness)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
-
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count || afr_does_witness_exist (this, witness)) {
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-		return -1;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i]) {
-			source = i;
-			break;
-		}
-	}
-
-	return source;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+
+    return source;
 }
 
 int
-__afr_selfheal_name_prepare (call_frame_t *frame, xlator_t *this, inode_t *parent,
-			     uuid_t pargfid, unsigned char *locked_on,
-			     unsigned char *sources, unsigned char *sinks,
-			     unsigned char *healed_sinks, int *source_p)
+__afr_selfheal_name_prepare(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            int *source_p)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        struct afr_reply *replies = NULL;
-        uint64_t *witness = NULL;
-
-	priv = this->private;
-
-        replies = alloca0 (priv->child_count * sizeof(*replies));
-
-	ret = afr_selfheal_unlocked_discover (frame, parent, pargfid, replies);
-	if (ret)
-		goto out;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_ENTRY_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           NULL);
-	if (ret)
-		goto out;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_name_finalize_source (this, sources,
-                                                      healed_sinks,
-						      locked_on, replies,
-                                                      witness);
-	if (source < 0) {
-		/* If source is < 0 (typically split-brain), we perform a
-		   conservative merge of entries rather than erroring out */
-	}
-	*source_p = source;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_unlocked_discover(frame, parent, pargfid, replies);
+    if (ret)
+        goto out;
+
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, NULL);
+    if (ret)
+        goto out;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_name_finalize_source(this, sources, healed_sinks,
+                                                 locked_on, witness);
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
 
 out:
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
-		      uuid_t pargfid, const char *bname, void *gfid_req)
+afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                     uuid_t pargfid, const char *bname, void *gfid_req,
+                     dict_t *xdata)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *healed_sinks = NULL;
-	unsigned char *locked_on = NULL;
-	int source = -1;
-        struct afr_reply *replies = NULL;
-	int ret = -1;
-	inode_t *inode = NULL;
-        dict_t *xattr = NULL;
-
-        xattr = dict_new ();
-        if (!xattr)
-                return -ENOMEM;
-
-        ret = dict_set_int32 (xattr, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                dict_destroy (xattr);
-                return -1;
+    afr_private_t *priv = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *locked_on = NULL;
+    int source = -1;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    dict_t *xattr = NULL;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    priv = this->private;
+
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_entrylk(frame, this, parent, this->name, bname,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_name_prepare(frame, this, parent, pargfid,
+                                          locked_on, sources, sinks,
+                                          healed_sinks, &source);
+        if (ret)
+            goto unlock;
+
+        inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
         }
 
-	priv = this->private;
-
-	locked_on = alloca0 (priv->child_count);
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-
-	replies = alloca0 (priv->child_count * sizeof(*replies));
-
-	ret = afr_selfheal_entrylk (frame, this, parent, this->name, bname,
-				    locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_name_prepare (frame, this, parent, pargfid,
-						   locked_on, sources, sinks,
-						   healed_sinks, &source);
-		if (ret)
-			goto unlock;
-
-                inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
-						         replies, locked_on,
-                                                         xattr);
-		if (!inode) {
-			ret = -ENOMEM;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_name_do (frame, this, parent, pargfid,
-                                              bname, inode, sources, sinks,
-                                              healed_sinks, source, locked_on,
-                                              replies, gfid_req);
-	}
+        ret = __afr_selfheal_name_do(frame, this, parent, pargfid, bname, inode,
+                                     sources, sinks, healed_sinks, source,
+                                     locked_on, replies, gfid_req, xdata);
+    }
 unlock:
-	afr_selfheal_unentrylk (frame, this, parent, this->name, bname,
-				locked_on, NULL);
-	if (inode)
-		inode_unref (inode);
+    afr_selfheal_unentrylk(frame, this, parent, this->name, bname, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
 
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-        if (xattr)
-                dict_unref (xattr);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_selfheal_name_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-				    inode_t *parent, uuid_t pargfid,
-				    const char *bname, gf_boolean_t *need_heal)
+afr_selfheal_name_unlocked_inspect(call_frame_t *frame, xlator_t *this,
+                                   inode_t *parent, uuid_t pargfid,
+                                   const char *bname, gf_boolean_t *need_heal)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	struct afr_reply *replies = NULL;
-	inode_t *inode = NULL;
-	int first_idx = -1;
-
-	priv = this->private;
-
-	replies = alloca0 (sizeof (*replies) * priv->child_count);
-
-	inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
-						 replies, priv->child_up, NULL);
-	if (!inode)
-		return -ENOMEM;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        *need_heal = _gf_true;
-
-		if (first_idx == -1) {
-			first_idx = i;
-			continue;
-		}
-
-		if (replies[i].op_ret != replies[first_idx].op_ret)
-			*need_heal = _gf_true;
-
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first_idx].poststat.ia_gfid))
-			*need_heal = _gf_true;
-	}
-
-	if (inode)
-		inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-	return 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    struct afr_reply *replies = NULL;
+    inode_t *inode = NULL;
+    int first_idx = -1;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                            local->child_up, NULL);
+    if (!inode)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
+
+        if (replies[i].op_ret != replies[first_idx].op_ret) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid)) {
+            *need_heal = _gf_true;
+            break;
+        }
+    }
+
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return 0;
 }
 
 int
-afr_selfheal_name (xlator_t *this, uuid_t pargfid, const char *bname,
-                   void *gfid_req)
+afr_selfheal_name(xlator_t *this, uuid_t pargfid, const char *bname,
+                  void *gfid_req, dict_t *xdata)
 {
-	inode_t *parent = NULL;
-	call_frame_t *frame = NULL;
-	int ret = -1;
-	gf_boolean_t need_heal = _gf_false;
-
-	parent = afr_inode_find (this, pargfid);
-	if (!parent)
-		goto out;
-
-	frame = afr_frame_create (this);
-	if (!frame)
-		goto out;
-
-	ret = afr_selfheal_name_unlocked_inspect (frame, this, parent, pargfid,
-						  bname, &need_heal);
-	if (ret)
-		goto out;
-
-	if (need_heal) {
-		ret = afr_selfheal_name_do (frame, this, parent, pargfid, bname,
-                                            gfid_req);
-                if (ret)
-                        goto out;
-        }
+    inode_t *parent = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    gf_boolean_t need_heal = _gf_false;
+
+    parent = afr_inode_find(this, pargfid);
+    if (!parent)
+        goto out;
+
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+
+    ret = afr_selfheal_name_unlocked_inspect(frame, this, parent, pargfid,
+                                             bname, &need_heal);
+    if (ret)
+        goto out;
+
+    if (need_heal) {
+        ret = afr_selfheal_name_do(frame, this, parent, pargfid, bname,
+                                   gfid_req, xdata);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-	if (parent)
-		inode_unref (parent);
-	if (frame)
-		AFR_STACK_DESTROY (frame);
+    if (parent)
+        inode_unref(parent);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index ec5337e60b2..48e6dbcfb18 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -8,278 +8,370 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _AFR_SELFHEAL_H
 #define _AFR_SELFHEAL_H
 
-#define AFR_SH_MIN_PARTICIPANTS 2
-
 /* Perform fop on all UP subvolumes and wait for all callbacks to return */
 
-#define AFR_ONALL(frame, rfn, fop, args ...) do {			\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0, __count = 0;					\
-									\
-	afr_local_replies_wipe (__local, __priv);				\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!__priv->child_up[__i]) continue;			\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-		__count++;						\
-	}								\
-	syncbarrier_wait (&__local->barrier, __count);			\
-	} while (0)
-
+#define AFR_ONALL(frame, rfn, fop, args...)                                    \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0, __count = 0;                                              \
+        unsigned char *__child_up = alloca(__priv->child_count);               \
+                                                                               \
+        memcpy(__child_up, __priv->child_up,                                   \
+               sizeof(*__child_up) * __priv->child_count);                     \
+        __count = AFR_COUNT(__child_up, __priv->child_count);                  \
+                                                                               \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__child_up[__i])                                              \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
 
 /* Perform fop on all subvolumes represented by list[] array and wait
    for all callbacks to return */
 
-#define AFR_ONLIST(list, frame, rfn, fop, args ...) do {		\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0, __count = 0;					\
-									\
-	afr_local_replies_wipe (__local, __priv);				\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!list[__i]) continue;				\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-		__count++;						\
-	}								\
-	syncbarrier_wait (&__local->barrier, __count);			\
-	} while (0)
-
-
-#define AFR_SEQ(frame, rfn, fop, args ...) do {				\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0;							\
-									\
-	afr_local_replies_wipe (__local, __priv);				\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!__priv->child_up[__i]) continue;			\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-		syncbarrier_wait (&__local->barrier, 1);		\
-	}								\
-	} while (0)
-
-
-#define ALLOC_MATRIX(n, type) ({type **__ptr = NULL; \
-	int __i; \
-	__ptr = alloca0 (n * sizeof(type *)); \
-	for (__i = 0; __i < n; __i++) __ptr[__i] = alloca0 (n * sizeof(type)); \
-	__ptr;})
-
-
-#define IA_EQUAL(f,s,field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
-
-
-int
-afr_selfheal (xlator_t *this, uuid_t gfid);
+#define AFR_ONLIST(list, frame, rfn, fop, args...)                             \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+        int __count = 0;                                                       \
+        unsigned char *__list = alloca(__priv->child_count);                   \
+                                                                               \
+        memcpy(__list, list, sizeof(*__list) * __priv->child_count);           \
+        __count = AFR_COUNT(__list, __priv->child_count);                      \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__list[__i])                                                  \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
+
+#define AFR_SEQ(frame, rfn, fop, args...)                                      \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+                                                                               \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__priv->child_up[__i])                                        \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+            syncbarrier_wait(&__local->barrier, 1);                            \
+        }                                                                      \
+    } while (0)
+
+#define ALLOC_MATRIX(n, type)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        type **__ptr = alloca(n * sizeof(type *));                             \
+                                                                               \
+        for (__i = 0; __i < n; __i++)                                          \
+            __ptr[__i] = alloca0(n * sizeof(type));                            \
+        __ptr;                                                                 \
+    })
+
+#define IA_EQUAL(f, s, field)                                                  \
+    (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0)
+
+#define SBRAIN_HEAL_NO_GO_MSG                                                  \
+    "Failed to obtain replies from all bricks of "                             \
+    "the replica (are they up?). Cannot resolve split-brain."
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SNO_BIGGER_FILE "No bigger file"
+#define SNO_DIFF_IN_MTIME "No difference in mtime"
+#define SUSE_SOURCE_BRICK_TO_HEAL                                              \
+    "Use source-brick option to heal metadata"                                 \
+    " split-brain"
+#define SINVALID_BRICK_NAME "Invalid brick name"
+#define SBRICK_IS_NOT_UP "Brick is not up"
+#define SBRICK_NOT_CONNECTED "Brick is not connected"
+#define SLESS_THAN2_BRICKS_in_REP "< 2 bricks in replica are up"
+#define SBRICK_IS_REMOTE "Brick is remote"
+#define SSTARTED_SELF_HEAL "Started self-heal"
+#define SOP_NOT_SUPPORTED "Operation Not Supported"
+#define SFILE_NOT_UNDER_DATA                                                   \
+    "The file is not under data or metadata "                                  \
+    "split-brain"
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SALL_BRICKS_UP_TO_RESOLVE                                              \
+    "All the bricks should be up to resolve the"                               \
+    " gfid split brain"
+#define SERROR_GETTING_SRC_BRICK "Error getting the source brick"
+int
+afr_selfheal(xlator_t *this, uuid_t gfid);
 
-void
-afr_throttled_selfheal (call_frame_t *frame, xlator_t *this);
+gf_boolean_t
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this);
 
 int
-afr_selfheal_name (xlator_t *this, uuid_t gfid, const char *name,
-                   void *gfid_req);
+afr_selfheal_name(xlator_t *this, uuid_t gfid, const char *name, void *gfid_req,
+                  dict_t *xdata);
 
 int
-afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd);
 
 int
-afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode);
 
+int
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx);
 
 int
-afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, off_t off, size_t size,
-		      unsigned char *locked_on);
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on);
 
 int
-afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, off_t off, size_t size,
-			 unsigned char *locked_on);
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on);
 
 int
-afr_selfheal_tie_breaker_inodelk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, off_t off,
-                                  size_t size, unsigned char *locked_on);
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on);
 
 int
-afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, off_t off, size_t size,
-			const unsigned char *locked_on);
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on);
 
 int
-afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, const char *name, unsigned char *locked_on);
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, const char *name, unsigned char *locked_on);
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_selfheal_tie_breaker_entrylk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, const char *name,
-                                  unsigned char *locked_on);
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on);
 
 int
-afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, const char *name, unsigned char *locked_on,
-                        dict_t *xdata);
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata);
 
 int
-afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
-				uuid_t gfid, struct afr_reply *replies);
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies);
 
+int
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict);
 inode_t *
-afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
-				 const char *name, struct afr_reply *replies,
-				 unsigned char *lookup_on, dict_t *xattr);
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr);
 
 int
-afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
-                             struct afr_reply *replies,
-                             afr_transaction_type type,
-                             unsigned char *locked_on, unsigned char *sources,
-                             unsigned char *sinks, uint64_t *witness,
-                             gf_boolean_t *flag);
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *flag);
+int
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata);
+
 int
-afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol, int idx,
-                          dict_t *xdata);
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix);
 
 int
-afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
-			    afr_transaction_type type, int *dirty, int **matrix);
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata);
 
 int
-afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			   unsigned char *sources, unsigned char *sinks,
-			   unsigned char *healed_sinks, afr_transaction_type type,
-			   struct afr_reply *replies, unsigned char *locked_on);
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies);
+int
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on);
 
 int
-afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
-                             const char *name, inode_t *inode,
-                             struct afr_reply *replies,
-                             unsigned char *newentry);
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies);
 
 int
-afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      int subvol, dict_t *xattr, dict_t *xdata);
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata);
 
 call_frame_t *
-afr_frame_create (xlator_t *this);
+afr_frame_create(xlator_t *this, int32_t *op_errno);
 
 inode_t *
-afr_inode_find (xlator_t *this, uuid_t gfid);
+afr_inode_find(xlator_t *this, uuid_t gfid);
 
 int
-afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int op_ret, int op_errno, inode_t *inode,
-			   struct iatt *buf, dict_t *xdata,
-                           struct iatt *parbuf);
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf);
+void
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src);
 
 void
-afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count);
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count);
 
 int
-afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                            int source, struct afr_reply *replies,
-                            unsigned char *sources, unsigned char *newentry);
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry);
 
 unsigned int
-afr_success_count (struct afr_reply *replies, unsigned int count);
+afr_success_count(struct afr_reply *replies, unsigned int count);
 
 void
-afr_log_selfheal (uuid_t gfid, xlator_t *this, int ret, char *type,
-                  int source, unsigned char *sources,
-                  unsigned char *healed_sinks);
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks);
 
 void
-afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies);
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies);
 void
-afr_mark_active_sinks (xlator_t *this, unsigned char *sources,
-                       unsigned char *locked_on, unsigned char *sinks);
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks);
 
 gf_boolean_t
-afr_dict_contains_heal_op (call_frame_t *frame);
+afr_dict_contains_heal_op(call_frame_t *frame);
+
+gf_boolean_t
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count);
+int
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type);
+
+int
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str);
 
 int
-afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode,
-                                   unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type);
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies);
 
 int
-afr_get_child_index_from_name (xlator_t *this, char *name);
+afr_get_child_index_from_name(xlator_t *this, char *name);
 
 gf_boolean_t
-afr_does_witness_exist (xlator_t *this, uint64_t *witness);
+afr_does_witness_exist(xlator_t *this, uint64_t *witness);
+
+int
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *flag);
 
 int
-__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *flag);
+int
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
                              inode_t *inode, unsigned char *locked_on,
-                             unsigned char *sources,
-                             unsigned char *sinks, unsigned char *healed_sinks,
-                             struct afr_reply *replies,
-                             gf_boolean_t *flag);
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *flag);
 
 int
-__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this,
-                                 inode_t *inode, unsigned char *locked_on,
-                                 unsigned char *sources,
-                                 unsigned char *sinks,
-                                 unsigned char *healed_sinks,
-                                 struct afr_reply *replies,
-                                 gf_boolean_t *flag);
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies);
+
 int
-__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
-                              inode_t *inode, unsigned char *locked_on,
-                              unsigned char *sources,
-                              unsigned char *sinks,
-                              unsigned char *healed_sinks,
-                              struct afr_reply *replies, int *source_p,
-                              gf_boolean_t *flag);
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid);
 
 int
-afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-                               uuid_t gfid, inode_t **link_inode,
-                               gf_boolean_t *data_selfheal,
-                               gf_boolean_t *metadata_selfheal,
-                               gf_boolean_t *entry_selfheal);
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
 
 int
-afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid);
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on);
+int
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type);
+
+int
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf);
 
 int
-afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, dict_t *xdata);
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode);
+int
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
+int
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
 
 int
-afr_locked_fill (call_frame_t *frame, xlator_t *this,
-                 unsigned char *locked_on);
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata);
 int
-afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
-                             afr_transaction_type type);
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type);
+
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies);
 
 int
-afr_selfheal_metadata_by_stbuf (xlator_t *this, struct iatt *stbuf);
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies);
+int
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode);
 #endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 545ffa0a1d7..109fd4b7421 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -8,1180 +8,1709 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
 #include "afr-self-heald.h"
 #include "protocol-common.h"
-#include "syncop-utils.h"
+#include <glusterfs/syncop-utils.h>
 #include "afr-messages.h"
-
-#define SHD_INODE_LRU_LIMIT          2048
-#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
-#define AFR_STATISTICS_HISTORY_SIZE    50
-
-
-#define ASSERT_LOCAL(this, healer)				\
-	if (!afr_shd_is_subvol_local(this, healer->subvol)) {	\
-		healer->local = _gf_false;			\
-		if (safe_break (healer)) {			\
-			break;					\
-		} else {					\
-			continue;				\
-		}						\
-	} else {						\
-		healer->local = _gf_true;			\
-	}
-
-
-#define NTH_INDEX_HEALER(this, n) &((((afr_private_t *)this->private))->shd.index_healers[n])
-#define NTH_FULL_HEALER(this, n) &((((afr_private_t *)this->private))->shd.full_healers[n])
+#include <glusterfs/byte-order.h>
+
+#define AFR_EH_SPLIT_BRAIN_LIMIT 1024
+#define AFR_STATISTICS_HISTORY_SIZE 50
+
+#define ASSERT_LOCAL(this, healer)                                             \
+    if (!afr_shd_is_subvol_local(this, healer->subvol)) {                      \
+        healer->local = _gf_false;                                             \
+        if (safe_break(healer)) {                                              \
+            break;                                                             \
+        } else {                                                               \
+            continue;                                                          \
+        }                                                                      \
+    } else {                                                                   \
+        healer->local = _gf_true;                                              \
+    }
+
+#define NTH_INDEX_HEALER(this, n)                                              \
+    &((((afr_private_t *)this->private))->shd.index_healers[n])
+#define NTH_FULL_HEALER(this, n)                                               \
+    &((((afr_private_t *)this->private))->shd.full_healers[n])
 
 char *
-afr_subvol_name (xlator_t *this, int subvol)
+afr_subvol_name(xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
-	if (subvol < 0 || subvol > priv->child_count)
-		return NULL;
+    priv = this->private;
+    if (subvol < 0 || subvol > priv->child_count)
+        return NULL;
 
-	return priv->children[subvol]->name;
+    return priv->children[subvol]->name;
 }
 
-
 void
-afr_destroy_crawl_event_data (void *data)
+afr_destroy_crawl_event_data(void *data)
 {
-        return;
+    return;
 }
 
-
 void
-afr_destroy_shd_event_data (void *data)
+afr_destroy_shd_event_data(void *data)
 {
-	shd_event_t *shd_event = data;
-
-	if (!shd_event)
-		return;
-	GF_FREE (shd_event->path);
+    shd_event_t *shd_event = data;
 
+    if (!shd_event)
         return;
-}
+    GF_FREE(shd_event->path);
 
+    return;
+}
 
 gf_boolean_t
-afr_shd_is_subvol_local (xlator_t *this, int subvol)
+afr_shd_is_subvol_local(xlator_t *this, int subvol)
 {
-	afr_private_t *priv    = NULL;
-	gf_boolean_t  is_local = _gf_false;
-        loc_t         loc      = {0, };
-
-        loc.inode = this->itable->root;
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-	priv = this->private;
-        syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
-        return is_local;
+    afr_private_t *priv = NULL;
+    gf_boolean_t is_local = _gf_false;
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = this->itable->root;
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    priv = this->private;
+    syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
+    return is_local;
 }
 
-
 int
-__afr_shd_healer_wait (struct subvol_healer *healer)
+__afr_shd_healer_wait(struct subvol_healer *healer)
 {
-	afr_private_t *priv = NULL;
-	struct timespec wait_till = {0, };
-	int ret = 0;
+    afr_private_t *priv = NULL;
+    struct timespec wait_till = {
+        0,
+    };
+    int ret = 0;
 
-	priv = healer->this->private;
+    priv = healer->this->private;
 
 disabled_loop:
-	wait_till.tv_sec = time (NULL) + priv->shd.timeout;
+    wait_till.tv_sec = gf_time() + priv->shd.timeout;
 
-	while (!healer->rerun) {
-		ret = pthread_cond_timedwait (&healer->cond,
-					      &healer->mutex,
-					      &wait_till);
-		if (ret == ETIMEDOUT)
-			break;
-	}
+    while (!healer->rerun) {
+        ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
+        if (ret == ETIMEDOUT)
+            break;
+    }
 
-	ret = healer->rerun;
-	healer->rerun = 0;
+    ret = healer->rerun;
+    healer->rerun = 0;
 
-	if (!priv->shd.enabled)
-		goto disabled_loop;
+    if (!priv->shd.enabled)
+        goto disabled_loop;
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_shd_healer_wait (struct subvol_healer *healer)
+afr_shd_healer_wait(struct subvol_healer *healer)
 {
-	int ret = 0;
+    int ret = 0;
 
-	pthread_mutex_lock (&healer->mutex);
-	{
-		ret = __afr_shd_healer_wait (healer);
-	}
-	pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_lock(&healer->mutex);
+    {
+        ret = __afr_shd_healer_wait(healer);
+    }
+    pthread_mutex_unlock(&healer->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 gf_boolean_t
-safe_break (struct subvol_healer *healer)
+safe_break(struct subvol_healer *healer)
 {
-	gf_boolean_t ret = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-	pthread_mutex_lock (&healer->mutex);
-	{
-		if (healer->rerun)
-			goto unlock;
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->rerun)
+            goto unlock;
 
-		healer->running = _gf_false;
-		ret = _gf_true;
-	}
+        healer->running = _gf_false;
+        ret = _gf_true;
+    }
 unlock:
-	pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_unlock(&healer->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
+afr_shd_inode_find(xlator_t *this, xlator_t *subvol, uuid_t gfid)
 {
-	inode_t *inode = NULL;
-	int ret = 0;
-	loc_t loc = {0, };
-	struct iatt iatt = {0, };
-
-	inode = inode_find (this->itable, gfid);
-	if (inode)
-		goto out;
-
-	loc.inode = inode_new (this->itable);
-	if (!loc.inode)
-		goto out;
-	gf_uuid_copy (loc.gfid, gfid);
-
-	ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
-	if (ret < 0)
-		goto out;
-
-	inode = inode_link (loc.inode, NULL, NULL, &iatt);
+    int ret = 0;
+    uint64_t val = IA_INVAL;
+    dict_t *xdata = NULL;
+    dict_t *rsp_dict = NULL;
+    inode_t *inode = NULL;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+
+    ret = dict_set_int8(xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
+    if (ret)
+        goto out;
+
+    ret = syncop_inode_find(this, subvol, gfid, &inode, xdata, &rsp_dict);
+    if (ret < 0)
+        goto out;
+
+    if (rsp_dict) {
+        ret = dict_get_uint64(rsp_dict, GF_INDEX_IA_TYPE_GET_RSP, &val);
+        if (ret)
+            goto out;
+    }
+    ret = inode_ctx_set2(inode, subvol, 0, &val);
 out:
-	loc_wipe (&loc);
-	return inode;
+    if (ret && inode) {
+        inode_unref(inode);
+        inode = NULL;
+    }
+    if (xdata)
+        dict_unref(xdata);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return inode;
 }
 
-inode_t*
-afr_shd_index_inode (xlator_t *this, xlator_t *subvol, char *vgfid)
+inode_t *
+afr_shd_index_inode(xlator_t *this, xlator_t *subvol, char *vgfid)
 {
-	loc_t rootloc = {0, };
-	inode_t *inode = NULL;
-	int ret = 0;
-	dict_t *xattr = NULL;
-	void *index_gfid = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    void *index_gfid = NULL;
 
-	rootloc.inode = inode_ref (this->itable->root);
-	gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-	ret = syncop_getxattr (subvol, &rootloc, &xattr,
-			       vgfid, NULL, NULL);
-	if (ret || !xattr) {
-		errno = -ret;
-		goto out;
-	}
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, vgfid, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
 
-	ret = dict_get_ptr (xattr, vgfid, &index_gfid);
-	if (ret)
-		goto out;
+    ret = dict_get_ptr(xattr, vgfid, &index_gfid);
+    if (ret)
+        goto out;
 
-        gf_msg_debug (this->name, 0, "%s dir gfid for %s: %s",
-	              vgfid, subvol->name, uuid_utoa (index_gfid));
+    gf_msg_debug(this->name, 0, "%s dir gfid for %s: %s", vgfid, subvol->name,
+                 uuid_utoa(index_gfid));
 
-	inode = afr_shd_inode_find (this, subvol, index_gfid);
+    inode = afr_shd_inode_find(this, subvol, index_gfid);
 
 out:
-	loc_wipe (&rootloc);
+    loc_wipe(&rootloc);
 
-	if (xattr)
-		dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-	return inode;
+    return inode;
 }
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type)
 {
-	loc_t loc = {0, };
-	int ret = 0;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
 
-	loc.parent = inode_ref (inode);
-	loc.name = name;
+    loc.parent = inode_ref(inode);
+    loc.name = name;
 
-	ret = syncop_unlink (subvol, &loc, NULL, NULL);
+    if (IA_ISDIR(type))
+        ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+    else
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
 
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 void
-afr_shd_zero_xattrop (xlator_t *this, uuid_t gfid)
+afr_shd_zero_xattrop(xlator_t *this, uuid_t gfid)
 {
-
-        call_frame_t *frame = NULL;
-        inode_t *inode = NULL;
-        afr_private_t *priv = NULL;
-        dict_t  *xattr = NULL;
-        int ret = 0;
-        int i = 0;
-        int raw[AFR_NUM_CHANGE_LOGS] = {0};
-
-        priv = this->private;
-        frame = afr_frame_create (this);
-        if (!frame)
-                goto out;
-        inode = afr_inode_find (this, gfid);
-        if (!inode)
-                goto out;
-        xattr = dict_new();
-        if (!xattr)
-                goto out;
-        ret = dict_set_static_bin (xattr, AFR_DIRTY, raw,
-                                   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    call_frame_t *frame = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int ret = 0;
+    int i = 0;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+
+    priv = this->private;
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, raw,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
         if (ret)
-                goto out;
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_static_bin (xattr, priv->pending_key[i], raw,
-                                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
-        * has valid repies for this gfid seems a bit of an overkill.*/
-        for (i = 0; i < priv->child_count; i++)
-                afr_selfheal_post_op (frame, this, inode, i, xattr, NULL);
+    /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
+     * has valid repies for this gfid seems a bit of an overkill.*/
+    for (i = 0; i < priv->child_count; i++)
+        afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
 
 out:
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-        if (inode)
-                inode_unref (inode);
-        if (xattr)
-                dict_unref (xattr);
-        return;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (inode)
+        inode_unref(inode);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
 int
-afr_shd_selfheal_name (struct subvol_healer *healer, int child, uuid_t parent,
-		       const char *bname)
+afr_shd_selfheal_name(struct subvol_healer *healer, int child, uuid_t parent,
+                      const char *bname)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = afr_selfheal_name (THIS, parent, bname, NULL);
+    ret = afr_selfheal_name(THIS, parent, bname, NULL, NULL);
 
-	return ret;
+    return ret;
 }
 
 int
-afr_shd_selfheal (struct subvol_healer *healer, int child, uuid_t gfid)
+afr_shd_selfheal(struct subvol_healer *healer, int child, uuid_t gfid)
 {
-	int ret = 0;
-	eh_t *eh = NULL;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	shd_event_t *shd_event = NULL;
-	char *path = NULL;
-	xlator_t *subvol = NULL;
-	xlator_t *this = NULL;
-	crawl_event_t *crawl_event = NULL;
-
-	this = healer->this;
-	priv = this->private;
-	shd = &priv->shd;
-	crawl_event = &healer->crawl_event;
-
-	subvol = priv->children[child];
-
-        //If this fails with ENOENT/ESTALE index is stale
-        ret = syncop_gfid_to_path (this->itable, subvol, gfid, &path);
-        if (ret < 0)
-                return ret;
-
-	ret = afr_selfheal (this, gfid);
-
-        LOCK (&priv->lock);
-        {
-                if (ret == -EIO) {
-                        eh = shd->split_brain;
-                        crawl_event->split_brain_count++;
-                } else if (ret < 0) {
-                        crawl_event->heal_failed_count++;
-                } else if (ret == 0) {
-                        crawl_event->healed_count++;
-                }
+    int ret = 0;
+    eh_t *eh = NULL;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    this = healer->this;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = &healer->crawl_event;
+
+    subvol = priv->children[child];
+
+    // If this fails with ENOENT/ESTALE index is stale
+    ret = syncop_gfid_to_path(this->itable, subvol, gfid, &path);
+    if (ret < 0)
+        return ret;
+
+    ret = afr_selfheal(this, gfid);
+
+    LOCK(&priv->lock);
+    {
+        if (ret == -EIO) {
+            eh = shd->split_brain;
+            crawl_event->split_brain_count++;
+        } else if (ret < 0) {
+            crawl_event->heal_failed_count++;
+        } else if (ret == 0) {
+            crawl_event->healed_count++;
         }
-        UNLOCK (&priv->lock);
+    }
+    UNLOCK(&priv->lock);
 
-	if (eh) {
-		shd_event = GF_CALLOC (1, sizeof(*shd_event),
-				       gf_afr_mt_shd_event_t);
-		if (!shd_event)
-                        goto out;
+    if (eh) {
+        shd_event = GF_CALLOC(1, sizeof(*shd_event), gf_afr_mt_shd_event_t);
+        if (!shd_event)
+            goto out;
 
-		shd_event->child = child;
-		shd_event->path = path;
+        shd_event->child = child;
+        shd_event->path = path;
 
-		if (eh_save_history (eh, shd_event) < 0)
-                        goto out;
+        if (eh_save_history(eh, shd_event) < 0)
+            goto out;
 
-                shd_event = NULL;
-                path = NULL;
-	}
+        shd_event = NULL;
+        path = NULL;
+    }
 out:
-        GF_FREE (shd_event);
-        GF_FREE (path);
-	return ret;
+    GF_FREE(shd_event);
+    GF_FREE(path);
+    return ret;
 }
 
-
 void
-afr_shd_sweep_prepare (struct subvol_healer *healer)
+afr_shd_sweep_prepare(struct subvol_healer *healer)
 {
-	crawl_event_t *event = NULL;
+    crawl_event_t *event = NULL;
 
-	event = &healer->crawl_event;
+    event = &healer->crawl_event;
 
-	event->healed_count = 0;
-	event->split_brain_count = 0;
-	event->heal_failed_count = 0;
+    event->healed_count = 0;
+    event->split_brain_count = 0;
+    event->heal_failed_count = 0;
 
-	time (&event->start_time);
-	event->end_time = 0;
+    event->start_time = gf_time();
+    event->end_time = 0;
+    _mask_cancellation();
 }
 
-
 void
-afr_shd_sweep_done (struct subvol_healer *healer)
+afr_shd_sweep_done(struct subvol_healer *healer)
 {
-	crawl_event_t *event = NULL;
-	crawl_event_t *history = NULL;
-	afr_self_heald_t *shd = NULL;
+    crawl_event_t *event = NULL;
+    crawl_event_t *history = NULL;
+    afr_self_heald_t *shd = NULL;
 
-	event = &healer->crawl_event;
-	shd = &(((afr_private_t *)healer->this->private)->shd);
+    event = &healer->crawl_event;
+    shd = &(((afr_private_t *)healer->this->private)->shd);
 
-	time (&event->end_time);
-	history = memdup (event, sizeof (*event));
-	event->start_time = 0;
+    event->end_time = gf_time();
+    history = gf_memdup(event, sizeof(*event));
+    event->start_time = 0;
 
-	if (!history)
-		return;
+    if (!history)
+        return;
 
-	if (eh_save_history (shd->statistics[healer->subvol], history) < 0)
-		GF_FREE (history);
+    if (eh_save_history(shd->statistics[healer->subvol], history) < 0)
+        GF_FREE(history);
+    _unmask_cancellation();
 }
 
 int
-afr_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                    void *data)
+afr_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                   void *data)
 {
-        struct subvol_healer *healer = data;
-        afr_private_t        *priv   = NULL;
-        uuid_t               gfid    = {0};
-        int                  ret     = 0;
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = NULL;
+    uuid_t gfid = {0};
+    int ret = 0;
+    uint64_t val = IA_INVAL;
 
-        priv = healer->this->private;
-        if (!priv->shd.enabled)
-                return -EBUSY;
+    priv = healer->this->private;
+    if (!priv->shd.enabled)
+        return -EBUSY;
 
-        gf_msg_debug (healer->this->name, 0, "got entry: %s",
-                      entry->d_name);
+    gf_msg_debug(healer->this->name, 0, "got entry: %s from %s", entry->d_name,
+                 priv->children[healer->subvol]->name);
 
-        ret = gf_uuid_parse (entry->d_name, gfid);
-        if (ret)
-                return 0;
+    ret = gf_uuid_parse(entry->d_name, gfid);
+    if (ret)
+        return 0;
 
-        ret = afr_shd_selfheal (healer, healer->subvol, gfid);
+    inode_ctx_get2(parent->inode, subvol, NULL, &val);
 
-        if (ret == -ENOENT || ret == -ESTALE)
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name);
-        if (ret == 2)
-                /* If bricks crashed in pre-op after creating indices/xattrop
-                 * link but before setting afr changelogs, we end up with stale
-                 * xattrop links but zero changelogs. Remove such entries by
-                 * sending a post-op with zero changelogs.
-                 */
-                afr_shd_zero_xattrop (healer->this, gfid);
+    ret = afr_shd_selfheal(healer, healer->subvol, gfid);
 
-        return 0;
+    if (ret == -ENOENT || ret == -ESTALE)
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, val);
+
+    if (ret == 2)
+        /* If bricks crashed in pre-op after creating indices/xattrop
+         * link but before setting afr changelogs, we end up with stale
+         * xattrop links but zero changelogs. Remove such entries by
+         * sending a post-op with zero changelogs.
+         */
+        afr_shd_zero_xattrop(healer->this, gfid);
+
+    return 0;
 }
 
 int
-afr_shd_index_sweep (struct subvol_healer *healer, char *vgfid)
+afr_shd_index_sweep(struct subvol_healer *healer, char *vgfid)
 {
-	loc_t         loc     = {0};
-	afr_private_t *priv   = NULL;
-	int           ret     = 0;
-	xlator_t      *subvol = NULL;
-	dict_t        *xdata  = NULL;
-        call_frame_t  *frame  = NULL;
-
-	priv = healer->this->private;
-	subvol = priv->children[healer->subvol];
-
-        frame = afr_frame_create (healer->this);
-        if (!frame) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    loc_t loc = {0};
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+
+    priv = healer->this->private;
+    subvol = priv->children[healer->subvol];
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    loc.inode = afr_shd_index_inode(healer->this, subvol, vgfid);
+    if (!loc.inode) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_INDEX_DIR_GET_FAILED, "unable to get index-dir on %s",
+               subvol->name);
+        ret = -errno;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int32_sizen(xdata, "get-gfid-type", 1)) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
+                             healer, afr_shd_index_heal, xdata,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
+
+    if (ret == 0)
+        ret = healer->crawl_event.healed_count;
 
-	loc.inode = afr_shd_index_inode (healer->this, subvol, vgfid);
-	if (!loc.inode) {
-	        gf_msg (healer->this->name, GF_LOG_WARNING,
-                        0, AFR_MSG_INDEX_DIR_GET_FAILED,
-		        "unable to get index-dir on %s", subvol->name);
-		ret = -errno;
-	        goto out;
-	}
-
-        xdata = dict_new ();
-        if (!xdata || dict_set_int32 (xdata, "get-gfid-type", 1)) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = syncop_mt_dir_scan (frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
-                                  healer, afr_shd_index_heal, xdata,
-                                 priv->shd.max_threads, priv->shd.wait_qlength);
+out:
+    loc_wipe(&loc);
 
-        if (ret == 0)
-                ret = healer->crawl_event.healed_count;
+    if (xdata)
+        dict_unref(xdata);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
+}
 
+int
+afr_shd_index_sweep_all(struct subvol_healer *healer)
+{
+    int ret = 0;
+    int count = 0;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_INDEX_GFID);
+    if (ret < 0)
+        goto out;
+    count = ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_DIRTY_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_ENTRY_CHANGES_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
 out:
-        loc_wipe (&loc);
-
-        if (xdata)
-                dict_unref (xdata);
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-	return ret;
+    if (ret < 0)
+        return ret;
+    else
+        return count;
 }
 
 int
-afr_shd_index_sweep_all (struct subvol_healer *healer)
+afr_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                  void *data)
 {
-        int            ret    = 0;
-        int            count  = 0;
+    struct subvol_healer *healer = data;
+    xlator_t *this = healer->this;
+    afr_private_t *priv = NULL;
 
-        ret = afr_shd_index_sweep (healer, GF_XATTROP_INDEX_GFID);
-        if (ret < 0)
-                goto out;
-        count = ret;
+    priv = this->private;
 
-        ret = afr_shd_index_sweep (healer, GF_XATTROP_DIRTY_GFID);
-        if (ret < 0)
-                goto out;
-        count += ret;
+    if (this->cleanup_starting) {
+        return -ENOTCONN;
+    }
 
-out:
-        if (ret < 0)
-                return ret;
-        else
-                return count;
+    if (!priv->shd.enabled)
+        return -EBUSY;
+
+    afr_shd_selfheal_name(healer, healer->subvol, parent->inode->gfid,
+                          entry->d_name);
+
+    afr_shd_selfheal(healer, healer->subvol, entry->d_stat.ia_gfid);
+
+    return 0;
 }
 
 int
-afr_shd_full_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+afr_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
 {
-        struct subvol_healer *healer = data;
-        xlator_t             *this   = healer->this;
-        afr_private_t        *priv   = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {0};
 
-        priv = this->private;
-        if (!priv->shd.enabled)
-                return -EBUSY;
+    priv = healer->this->private;
+    loc.inode = inode;
+    return syncop_ftw(priv->children[healer->subvol], &loc,
+                      GF_CLIENT_PID_SELF_HEALD, healer, afr_shd_full_heal);
+}
 
-        afr_shd_selfheal_name (healer, healer->subvol,
-                               parent->inode->gfid, entry->d_name);
+int
+afr_shd_fill_ta_loc(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+
+    priv = this->private;
+    loc->parent = inode_ref(this->itable->root);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    loc->inode = inode_new(loc->parent->table);
+    GF_CHECK_ALLOC(loc->inode, ret, out);
+
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto assign_gfid;
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], loc, &stbuf,
+                        0, 0, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed lookup on file %s.", loc->name);
+        goto out;
+    }
+
+    gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+
+assign_gfid:
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    ret = 0;
 
-        afr_shd_selfheal (healer, healer->subvol, entry->d_stat.ia_gfid);
+out:
+    if (ret)
+        loc_wipe(loc);
 
-        return 0;
+    return ret;
 }
 
 int
-afr_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
+_afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata)
 {
-        afr_private_t *priv = NULL;
-        loc_t          loc  = {0};
-
-        priv = healer->this->private;
-        loc.inode = inode;
-        return syncop_ftw (priv->children[healer->subvol], &loc,
-                           GF_CLIENT_PID_SELF_HEALD, healer,
-                           afr_shd_full_heal);
-}
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_GET_FAILED,
+               "Failed to create dict.");
+        goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], &raw,
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret)
+            goto out;
+    }
 
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, xdata, NULL);
+    if (ret || !(*xdata)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed on %s.", loc->name);
+    }
 
-void *
-afr_shd_index_healer (void *data)
-{
-	struct subvol_healer *healer = NULL;
-	xlator_t *this = NULL;
-	int ret = 0;
-	afr_private_t *priv = NULL;
-
-	healer = data;
-	THIS = this = healer->this;
-	priv = this->private;
-
-	for (;;) {
-		afr_shd_healer_wait (healer);
-
-		ASSERT_LOCAL(this, healer);
-		priv->local[healer->subvol] = healer->local;
-
-		do {
-		        gf_msg_debug (this->name, 0,
-		                      "starting index sweep on subvol %s",
-			              afr_subvol_name (this, healer->subvol));
-
-			afr_shd_sweep_prepare (healer);
-
-                        ret = afr_shd_index_sweep_all (healer);
-
-			afr_shd_sweep_done (healer);
-			/*
-			  As long as at least one gfid was
-			  healed, keep retrying. We may have
-			  just healed a directory and thereby
-			  created entries for other gfids which
-			  could not be healed thus far.
-			*/
-
-		        gf_msg_debug (this->name, 0,
-			              "finished index sweep on subvol %s",
-			              afr_subvol_name (this, healer->subvol));
-			/*
-			  Give a pause before retrying to avoid a busy loop
-			  in case the only entry in index is because of
-			  an ongoing I/O.
-			*/
-			sleep (1);
-		} while (ret > 0);
-	}
-
-	return NULL;
-}
+out:
+    if (xattr)
+        dict_unref(xattr);
 
+    return ret;
+}
 
-void *
-afr_shd_full_healer (void *data)
+void
+afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, struct subvol_healer *healer,
+                      dict_t **xdata)
 {
-	struct subvol_healer *healer = NULL;
-	xlator_t *this = NULL;
-	int run = 0;
+    int ret = 0;
+
+    loc_wipe(loc);
+    if (afr_shd_fill_ta_loc(this, loc)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, loc);
+    if (ret)
+        goto out;
+
+    ret = _afr_shd_ta_get_xattrs(this, loc, xdata);
+    if (ret) {
+        if (*xdata) {
+            dict_unref(*xdata);
+            *xdata = NULL;
+        }
+    }
 
-	healer = data;
-	THIS = this = healer->this;
+    afr_ta_post_op_unlock(this, loc);
 
-	for (;;) {
-		pthread_mutex_lock (&healer->mutex);
-		{
-			run = __afr_shd_healer_wait (healer);
-			if (!run)
-				healer->running = _gf_false;
-		}
-		pthread_mutex_unlock (&healer->mutex);
+out:
+    if (ret)
+        healer->rerun = 1;
+}
 
-		if (!run)
-			break;
+int
+afr_shd_ta_unset_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata, int healer)
+{
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    gf_boolean_t need_xattrop = _gf_false;
+    void *pending_raw = NULL;
+    int *raw = NULL;
+    int pending[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+    int val = 0;
+    int ret = -1;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        raw = GF_CALLOC(AFR_NUM_CHANGE_LOGS, sizeof(int), gf_afr_mt_int32_t);
+        if (!raw) {
+            goto out;
+        }
 
-		ASSERT_LOCAL(this, healer);
+        ret = dict_get_ptr(*xdata, priv->pending_key[i], &pending_raw);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                   "Error getting value "
+                   "of pending key %s",
+                   priv->pending_key[i]);
+            GF_FREE(raw);
+            goto out;
+        }
 
-	        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
-		        "starting full sweep on subvol %s",
-		        afr_subvol_name (this, healer->subvol));
+        memcpy(pending, pending_raw, sizeof(pending));
+        for (j = 0; j < AFR_NUM_CHANGE_LOGS; j++) {
+            val = ntoh32(pending[j]);
+            if (val) {
+                if (i == healer) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_THIN_ARB,
+                           "I am "
+                           "not the good shd. Skipping. "
+                           "SHD = %d.",
+                           healer);
+                    ret = 0;
+                    GF_FREE(raw);
+                    goto out;
+                }
+                need_xattrop = _gf_true;
+                raw[j] = hton32(-val);
+            }
+        }
 
-		afr_shd_sweep_prepare (healer);
+        ret = dict_set_bin(xattr, priv->pending_key[i], raw,
+                           AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret) {
+            GF_FREE(raw);
+            goto out;
+        }
 
-		afr_shd_full_sweep (healer, this->itable->root);
+        if (need_xattrop)
+            break;
+    }
 
-		afr_shd_sweep_done (healer);
+    if (!need_xattrop) {
+        ret = 0;
+        goto out;
+    }
 
-	        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
-		        "finished full sweep on subvol %s",
-		        afr_subvol_name (this, healer->subvol));
-	}
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed.");
 
-	return NULL;
-}
+out:
+    if (xattr)
+        dict_unref(xattr);
 
+    return ret;
+}
 
-int
-afr_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
+void
+afr_shd_ta_check_and_unset_xattrs(xlator_t *this, loc_t *loc,
+                                  struct subvol_healer *healer,
+                                  dict_t *pre_crawl_xdata)
 {
-	int ret = 0;
-
-	ret = pthread_mutex_init (&healer->mutex, NULL);
-	if (ret)
-		goto out;
+    int ret_lock = 0;
+    int ret = 0;
+    dict_t *post_crawl_xdata = NULL;
 
-	ret = pthread_cond_init (&healer->cond, NULL);
-	if (ret)
-		goto out;
+    ret_lock = afr_ta_post_op_lock(this, loc);
+    if (ret_lock)
+        goto unref;
 
-	healer->this = this;
-	healer->running = _gf_false;
-	healer->rerun = _gf_false;
-	healer->local = _gf_false;
-out:
-	return ret;
-}
+    ret = _afr_shd_ta_get_xattrs(this, loc, &post_crawl_xdata);
+    if (ret)
+        goto unref;
 
+    if (!are_dicts_equal(pre_crawl_xdata, post_crawl_xdata, NULL, NULL)) {
+        ret = -1;
+        goto unref;
+    }
 
-int
-afr_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
-		      void *(threadfn)(void *))
-{
-	int ret = 0;
-
-	pthread_mutex_lock (&healer->mutex);
-	{
-		if (healer->running) {
-			pthread_cond_signal (&healer->cond);
-		} else {
-			ret = gf_thread_create (&healer->thread, NULL,
-						threadfn, healer);
-			if (ret)
-				goto unlock;
-			healer->running = 1;
-		}
-
-		healer->rerun = 1;
-	}
-unlock:
-	pthread_mutex_unlock (&healer->mutex);
+    ret = afr_shd_ta_unset_xattrs(this, loc, &post_crawl_xdata, healer->subvol);
 
-	return ret;
-}
+unref:
+    if (post_crawl_xdata) {
+        dict_unref(post_crawl_xdata);
+        post_crawl_xdata = NULL;
+    }
 
+    if (ret || ret_lock)
+        healer->rerun = 1;
 
-int
-afr_shd_full_healer_spawn (xlator_t *this, int subvol)
-{
-	return afr_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
-				     afr_shd_full_healer);
+    if (!ret_lock)
+        afr_ta_post_op_unlock(this, loc);
 }
 
-
-int
-afr_shd_index_healer_spawn (xlator_t *this, int subvol)
+gf_boolean_t
+afr_bricks_available_for_heal(afr_private_t *priv)
 {
-	return afr_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
-				     afr_shd_index_healer);
-}
+    int up_children = 0;
 
+    up_children = __afr_get_up_children_count(priv);
+    if (up_children < 2) {
+        return _gf_false;
+    }
+    return _gf_true;
+}
 
-int
-afr_shd_dict_add_crawl_event (xlator_t *this, dict_t *output,
-			      crawl_event_t *crawl_event)
+static gf_boolean_t
+afr_shd_ta_needs_heal(xlator_t *this, struct subvol_healer *healer)
 {
-        int             ret = 0;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
-        uint64_t        healed_count = 0;
-        uint64_t        split_brain_count = 0;
-        uint64_t        heal_failed_count = 0;
-        char            *start_time_str = 0;
-        char            *end_time_str = NULL;
-        char            *crawl_type = NULL;
-        int             progress = -1;
-	int             child = -1;
-
-	child = crawl_event->child;
-        healed_count = crawl_event->healed_count;
-        split_brain_count = crawl_event->split_brain_count;
-        heal_failed_count = crawl_event->heal_failed_count;
-        crawl_type = crawl_event->crawl_type;
-
-	if (!crawl_event->start_time)
-		goto out;
-
-        start_time_str = gf_strdup (ctime (&crawl_event->start_time));
-
-	if (crawl_event->end_time)
-		end_time_str = gf_strdup (ctime (&crawl_event->end_time));
-
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_GET_FAILED, "xl does not have id");
-                goto out;
+    dict_t *xdata = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+    gf_boolean_t need_heal = _gf_false;
+
+    priv = this->private;
+
+    ret = afr_shd_fill_ta_loc(this, &loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        healer->rerun = 1;
+        goto out;
+    }
+
+    if (_afr_shd_ta_get_xattrs(this, &loc, &xdata)) {
+        healer->rerun = 1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (afr_ta_dict_contains_pending_xattr(xdata, priv, i)) {
+            need_heal = _gf_true;
+            break;
         }
+    }
 
-        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
 
+    return need_heal;
+}
 
-        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64(output, key, healed_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_healed_count to outout");
+static int
+afr_shd_anon_inode_cleaner(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                           void *data)
+{
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = healer->this->private;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    loc_t loc = {0};
+    int count = 0;
+    int i = 0;
+    int op_errno = 0;
+    struct iatt *iatt = NULL;
+    gf_boolean_t multiple_links = _gf_false;
+    unsigned char *gfid_present = alloca0(priv->child_count);
+    unsigned char *entry_present = alloca0(priv->child_count);
+    char *type = "file";
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    if (AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        gf_msg_debug(healer->this->name, 0,
+                     "Not all bricks are up. Skipping "
+                     "cleanup of %s on %s",
+                     entry->d_name, subvol->name);
+        ret = 0;
+        goto out;
+    }
+
+    loc.inode = inode_new(parent->inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = gf_uuid_parse(entry->d_name, loc.gfid);
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            count++;
+            gfid_present[i] = 1;
+            iatt = &local->replies[i].poststat;
+            if (iatt->ia_type == IA_IFDIR) {
+                type = "dir";
+            }
+
+            if (i == healer->subvol) {
+                if (local->replies[i].poststat.ia_nlink > 1) {
+                    multiple_links = _gf_true;
+                }
+            }
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            /*We don't have complete view. Skip the entry*/
+            gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                         "Skipping cleanup of %s on %s", entry->d_name,
+                         subvol->name);
+            ret = 0;
+            goto out;
+        }
+    }
+
+    /*Inode is deleted from subvol*/
+    if (count == 1 || (iatt->ia_type != IA_IFDIR && multiple_links)) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR, "expunging %s %s/%s on %s", type,
+               priv->anon_inode_name, entry->d_name, subvol->name);
+        ret = afr_shd_entry_purge(subvol, parent->inode, entry->d_name,
+                                  iatt->ia_type);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = 0;
+    } else if (count > 1) {
+        loc_wipe(&loc);
+        loc.parent = inode_ref(parent->inode);
+        loc.name = entry->d_name;
+        loc.inode = inode_new(parent->inode->table);
+        if (!loc.inode) {
+            ret = -ENOMEM;
+            goto out;
+        }
+        AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup,
+                   &loc, NULL);
+        count = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->replies[i].op_ret == 0) {
+                count++;
+                entry_present[i] = 1;
+                iatt = &local->replies[i].poststat;
+            } else if (local->replies[i].op_errno != ENOENT &&
+                       local->replies[i].op_errno != ESTALE) {
+                /*We don't have complete view. Skip the entry*/
+                gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                             "Skipping cleanup of %s on %s", entry->d_name,
+                             subvol->name);
+                ret = 0;
                 goto out;
-	}
-
-        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64 (output, key, split_brain_count);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_split_brain_count to outout");
+            }
+        }
+        for (i = 0; i < priv->child_count; i++) {
+            if (gfid_present[i] && !entry_present[i]) {
+                /*Entry is not anonymous on at least one subvol*/
+                gf_msg_debug(healer->this->name, 0,
+                             "Valid entry present on %s "
+                             "Skipping cleanup of %s on %s",
+                             priv->children[i]->name, entry->d_name,
+                             subvol->name);
+                ret = 0;
                 goto out;
+            }
         }
 
-        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_str (output, key, crawl_type);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-	                "Could not add statistics_crawl_type to output");
-                goto out;
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "expunging %s %s/%s on all subvols", type, priv->anon_inode_name,
+               entry->d_name);
+        ret = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            op_errno = -afr_shd_entry_purge(priv->children[i], loc.parent,
+                                            entry->d_name, iatt->ia_type);
+            if (op_errno != ENOENT && op_errno != ESTALE) {
+                ret |= -op_errno;
+            }
         }
+    }
 
-        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64 (output, key, heal_failed_count);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-	                "Could not add statistics_healed_failed_count to outout");
-                goto out;
+out:
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return ret;
+}
+
+static void
+afr_cleanup_anon_inode_dir(struct subvol_healer *healer)
+{
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_private_t *priv = healer->this->private;
+    loc_t loc = {0};
+
+    ret = afr_anon_inode_create(healer->this, healer->subvol, &loc.inode);
+    if (ret)
+        goto out;
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, priv->children[healer->subvol], &loc,
+                             GF_CLIENT_PID_SELF_HEALD, healer,
+                             afr_shd_anon_inode_cleaner, NULL,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
+out:
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return;
+}
+
+void *
+afr_shd_index_healer(void *data)
+{
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    dict_t *pre_crawl_xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    healer = data;
+    THIS = this = healer->this;
+    priv = this->private;
+
+    for (;;) {
+        afr_shd_healer_wait(healer);
+
+        if (!afr_bricks_available_for_heal(priv))
+            continue;
+
+        ASSERT_LOCAL(this, healer);
+        priv->local[healer->subvol] = healer->local;
+
+        if (priv->thin_arbiter_count) {
+            if (afr_shd_ta_needs_heal(this, healer))
+                afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
         }
 
-        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_dynstr (output, key, start_time_str);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_crawl_start_time to outout");
-                goto out;
-        } else {
-		start_time_str = NULL;
-	}
-
-	if (!end_time_str)
-                progress = 1;
-        else
-                progress = 0;
-
-        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        if (!end_time_str)
-                end_time_str = gf_strdup ("Could not determine the end time");
-        ret = dict_set_dynstr (output, key, end_time_str);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_crawl_end_time to outout");
-                goto out;
-        } else {
-		end_time_str = NULL;
-	}
+        do {
+            gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+
+            afr_shd_sweep_prepare(healer);
+
+            ret = afr_shd_index_sweep_all(healer);
+
+            afr_shd_sweep_done(healer);
+            /*
+              As long as at least one gfid was
+              healed, keep retrying. We may have
+              just healed a directory and thereby
+              created entries for other gfids which
+              could not be healed thus far.
+            */
+
+            gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+            /*
+              Give a pause before retrying to avoid a busy loop
+              in case the only entry in index is because of
+              an ongoing I/O.
+            */
+            sleep(1);
+        } while (ret > 0);
+
+        if (ret == 0) {
+            afr_cleanup_anon_inode_dir(healer);
+        }
 
-        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
-                  xl_id, child, count);
+        if (ret == 0 && pre_crawl_xdata &&
+            !healer->crawl_event.heal_failed_count) {
+            afr_shd_ta_check_and_unset_xattrs(this, &loc, healer,
+                                              pre_crawl_xdata);
+        }
 
-        ret = dict_set_int32 (output, key, progress);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_inprogress to outout");
-                goto out;
+        if (pre_crawl_xdata) {
+            dict_unref(pre_crawl_xdata);
+            pre_crawl_xdata = NULL;
         }
+    }
 
-	snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
-	ret = dict_set_uint64 (output, key, count + 1);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not increment the counter.");
-                goto out;
-	}
-out:
-	GF_FREE (start_time_str);
-	GF_FREE (end_time_str);
-        return ret;
+    return NULL;
 }
 
-
-int
-afr_shd_dict_add_path (xlator_t *this, dict_t *output, int child, char *path,
-		       struct timeval *tv)
+void *
+afr_shd_full_healer(void *data)
 {
-        int             ret = -1;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int run = 0;
 
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_GET_FAILED, "xl does not have id");
-                goto out;
+    healer = data;
+    THIS = this = healer->this;
+
+    for (;;) {
+        pthread_mutex_lock(&healer->mutex);
+        {
+            run = __afr_shd_healer_wait(healer);
+            if (!run)
+                healer->running = _gf_false;
         }
+        pthread_mutex_unlock(&healer->mutex);
 
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
+        if (!run)
+            break;
 
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
-	ret = dict_set_dynstr (output, key, path);
+        ASSERT_LOCAL(this, healer);
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_SET_FAILED, "%s: Could not add to output",
-                        path);
-                goto out;
-        }
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "starting full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
 
-	if (tv) {
-		snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
-			  child, count);
-		ret = dict_set_uint32 (output, key, tv->tv_sec);
-		if (ret) {
-		        gf_msg (this->name, GF_LOG_ERROR,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "%s: Could not set time",
-			        path);
-			goto out;
-		}
-	}
-
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-
-        ret = dict_set_uint64 (output, key, count + 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "Could not increment count");
-                goto out;
-        }
+        afr_shd_sweep_prepare(healer);
 
-        ret = 0;
-out:
-        return ret;
+        afr_shd_full_sweep(healer, this->itable->root);
+
+        afr_shd_sweep_done(healer);
+
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "finished full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
+    }
+
+    return NULL;
 }
 
 int
-afr_add_shd_event (circular_buffer_t *cb, void *data)
+afr_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
 {
-	dict_t *output = NULL;
-	xlator_t *this = THIS;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	shd_event_t *shd_event = NULL;
-	char *path = NULL;
-
-	output = data;
-	priv = this->private;
-	shd = &priv->shd;
-	shd_event = cb->data;
-
-	if (!shd->index_healers[shd_event->child].local)
-		return 0;
-
-	path = gf_strdup (shd_event->path);
-	if (!path)
-		return -ENOMEM;
-
-	afr_shd_dict_add_path (this, output, shd_event->child, path,
-			       &cb->tv);
-	return 0;
+    int ret = 0;
+
+    ret = pthread_mutex_init(&healer->mutex, NULL);
+    if (ret)
+        goto out;
+
+    ret = pthread_cond_init(&healer->cond, NULL);
+    if (ret)
+        goto out;
+
+    healer->this = this;
+    healer->running = _gf_false;
+    healer->rerun = _gf_false;
+    healer->local = _gf_false;
+out:
+    return ret;
 }
 
 int
-afr_add_crawl_event (circular_buffer_t *cb, void *data)
+afr_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
+                     void *(threadfn)(void *))
 {
-	dict_t *output = NULL;
-	xlator_t *this = THIS;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	crawl_event_t *crawl_event = NULL;
+    int ret = 0;
 
-	output = data;
-	priv = this->private;
-	shd = &priv->shd;
-	crawl_event = cb->data;
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->running) {
+            pthread_cond_signal(&healer->cond);
+        } else {
+            ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
+                                   "shdheal");
+            if (ret)
+                goto unlock;
+            healer->running = 1;
+        }
 
-	if (!shd->index_healers[crawl_event->child].local)
-		return 0;
+        healer->rerun = 1;
+    }
+unlock:
+    pthread_mutex_unlock(&healer->mutex);
 
-	afr_shd_dict_add_crawl_event (this, output, crawl_event);
+    return ret;
+}
 
-	return 0;
+int
+afr_shd_full_healer_spawn(xlator_t *this, int subvol)
+{
+    return afr_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
+                                afr_shd_full_healer);
 }
 
+int
+afr_shd_index_healer_spawn(xlator_t *this, int subvol)
+{
+    return afr_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
+                                afr_shd_index_healer);
+}
 
 int
-afr_selfheal_daemon_init (xlator_t *this)
+afr_shd_dict_add_crawl_event(xlator_t *this, dict_t *output,
+                             crawl_event_t *crawl_event)
 {
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	int ret = -1;
-	int i = 0;
-
-	priv = this->private;
-	shd = &priv->shd;
-
-	this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
-	if (!this->itable)
-		goto out;
-
-	shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
-					priv->child_count,
-					gf_afr_mt_subvol_healer_t);
-	if (!shd->index_healers)
-		goto out;
-
-	for (i = 0; i < priv->child_count; i++) {
-		shd->index_healers[i].subvol = i;
-		ret = afr_shd_healer_init (this, &shd->index_healers[i]);
-		if (ret)
-			goto out;
-	}
-
-	shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
-				       priv->child_count,
-				       gf_afr_mt_subvol_healer_t);
-	if (!shd->full_healers)
-		goto out;
-	for (i = 0; i < priv->child_count; i++) {
-		shd->full_healers[i].subvol = i;
-		ret = afr_shd_healer_init (this, &shd->full_healers[i]);
-		if (ret)
-			goto out;
-	}
-
-	shd->split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
-				   afr_destroy_shd_event_data);
-	if (!shd->split_brain)
-		goto out;
-
-        shd->statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
-				     gf_common_mt_eh_t);
-        if (!shd->statistics)
-                goto out;
+    int ret = 0;
+    uint64_t count = 0;
+    char key[128] = {0};
+    int keylen = 0;
+    char suffix[64] = {0};
+    int xl_id = 0;
+    uint64_t healed_count = 0;
+    uint64_t split_brain_count = 0;
+    uint64_t heal_failed_count = 0;
+    char *start_time_str = 0;
+    char *end_time_str = NULL;
+    char *crawl_type = NULL;
+    int progress = -1;
+    int child = -1;
+
+    child = crawl_event->child;
+    healed_count = crawl_event->healed_count;
+    split_brain_count = crawl_event->split_brain_count;
+    heal_failed_count = crawl_event->heal_failed_count;
+    crawl_type = crawl_event->crawl_type;
+
+    if (!crawl_event->start_time)
+        goto out;
+
+    start_time_str = gf_strdup(ctime(&crawl_event->start_time));
+
+    if (crawl_event->end_time)
+        end_time_str = gf_strdup(ctime(&crawl_event->end_time));
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_get_uint64(output, key, &count);
+
+    snprintf(suffix, sizeof(suffix), "%d-%d-%" PRIu64, xl_id, child, count);
+    snprintf(key, sizeof(key), "statistics_healed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, healed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_count to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_sb_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, split_brain_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_split_brain_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_crawl_type-%s", suffix);
+    ret = dict_set_strn(output, key, keylen, crawl_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_type to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_heal_failed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, heal_failed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_failed_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_strt_time-%s", suffix);
+    ret = dict_set_dynstrn(output, key, keylen, start_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_start_time to output");
+        goto out;
+    } else {
+        start_time_str = NULL;
+    }
+
+    if (!end_time_str)
+        progress = 1;
+    else
+        progress = 0;
+
+    keylen = snprintf(key, sizeof(key), "statistics_end_time-%s", suffix);
+    if (!end_time_str)
+        end_time_str = gf_strdup("Could not determine the end time");
+    ret = dict_set_dynstrn(output, key, keylen, end_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_end_time to output");
+        goto out;
+    } else {
+        end_time_str = NULL;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_inprogress-%s", suffix);
+
+    ret = dict_set_int32n(output, key, keylen, progress);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_inprogress to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment the counter.");
+        goto out;
+    }
+out:
+    GF_FREE(start_time_str);
+    GF_FREE(end_time_str);
+    return ret;
+}
 
-        for (i = 0; i < priv->child_count ; i++) {
-                shd->statistics[i] = eh_new (AFR_STATISTICS_HISTORY_SIZE,
-					     _gf_false,
-					     afr_destroy_crawl_event_data);
-                if (!shd->statistics[i])
-                        goto out;
-		shd->full_healers[i].crawl_event.child = i;
-		shd->full_healers[i].crawl_event.crawl_type = "FULL";
-		shd->index_healers[i].crawl_event.child = i;
-		shd->index_healers[i].crawl_event.crawl_type = "INDEX";
+int
+afr_shd_dict_add_path(xlator_t *this, dict_t *output, int child, char *path,
+                      struct timeval *tv)
+{
+    int ret = -1;
+    uint64_t count = 0;
+    char key[64] = {0};
+    int keylen = 0;
+    char xl_id_child_str[32] = {0};
+    int xl_id = 0;
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(xl_id_child_str, sizeof(xl_id_child_str), "%d-%d", xl_id, child);
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
+    ret = dict_get_uint64(output, key, &count);
+
+    keylen = snprintf(key, sizeof(key), "%s-%" PRIu64, xl_id_child_str, count);
+    ret = dict_set_dynstrn(output, key, keylen, path);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Could not add to output", path);
+        goto out;
+    }
+
+    if (tv) {
+        snprintf(key, sizeof(key), "%s-%" PRIu64 "-time", xl_id_child_str,
+                 count);
+        ret = dict_set_uint32(output, key, tv->tv_sec);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "%s: Could not set time", path);
+            goto out;
         }
+    }
+
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
 
-	ret = 0;
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment count");
+        goto out;
+    }
+
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
 
+int
+afr_add_shd_event(circular_buffer_t *cb, void *data)
+{
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    shd_event = cb->data;
+
+    if (!shd->index_healers[shd_event->child].local)
+        return 0;
+
+    path = gf_strdup(shd_event->path);
+    if (!path)
+        return -ENOMEM;
+
+    afr_shd_dict_add_path(this, output, shd_event->child, path, &cb->tv);
+    return 0;
+}
 
 int
-afr_selfheal_childup (xlator_t *this, int subvol)
+afr_add_crawl_event(circular_buffer_t *cb, void *data)
 {
-	afr_shd_index_healer_spawn (this, subvol);
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = cb->data;
+
+    if (!shd->index_healers[crawl_event->child].local)
+        return 0;
+
+    afr_shd_dict_add_crawl_event(this, output, crawl_event);
 
-	return 0;
+    return 0;
+}
+
+int
+afr_selfheal_daemon_init(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers),
+                                   priv->child_count,
+                                   gf_afr_mt_subvol_healer_t);
+    if (!shd->index_healers)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->index_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->index_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), priv->child_count,
+                                  gf_afr_mt_subvol_healer_t);
+    if (!shd->full_healers)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        shd->full_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->full_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    shd->split_brain = eh_new(AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                              afr_destroy_shd_event_data);
+    if (!shd->split_brain)
+        goto out;
+
+    shd->statistics = GF_CALLOC(sizeof(eh_t *), priv->child_count,
+                                gf_common_mt_eh_t);
+    if (!shd->statistics)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->statistics[i] = eh_new(AFR_STATISTICS_HISTORY_SIZE, _gf_false,
+                                    afr_destroy_crawl_event_data);
+        if (!shd->statistics[i])
+            goto out;
+        shd->full_healers[i].crawl_event.child = i;
+        shd->full_healers[i].crawl_event.crawl_type = "FULL";
+        shd->index_healers[i].crawl_event.child = i;
+        shd->index_healers[i].crawl_event.crawl_type = "INDEX";
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv)
+{
+    int subvol = 0;
+
+    if (!priv->shd.iamshd)
+        return;
+    for (subvol = 0; subvol < priv->child_count; subvol++)
+        if (priv->child_up[subvol])
+            afr_shd_index_healer_spawn(this, subvol);
+
+    return;
+}
 
 int
-afr_shd_get_index_count (xlator_t *this, int i, uint64_t *count)
+afr_shd_get_index_count(xlator_t *this, int i, uint64_t *count)
 {
-	afr_private_t *priv = NULL;
-	xlator_t *subvol = NULL;
-	loc_t rootloc = {0, };
-	dict_t *xattr = NULL;
-	int ret = -1;
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int ret = -1;
 
-	priv = this->private;
-	subvol = priv->children[i];
+    priv = this->private;
+    subvol = priv->children[i];
 
-	rootloc.inode = inode_ref (this->itable->root);
-	gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-	ret = syncop_getxattr (subvol, &rootloc, &xattr,
-			       GF_XATTROP_INDEX_COUNT, NULL, NULL);
-	if (ret < 0)
-		goto out;
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_COUNT,
+                          NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-	ret = dict_get_uint64 (xattr, GF_XATTROP_INDEX_COUNT, count);
-	if (ret)
-		goto out;
+    ret = dict_get_uint64(xattr, GF_XATTROP_INDEX_COUNT, count);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-	loc_wipe (&rootloc);
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 {
-        gf_xl_afr_op_t   op = GF_SHD_OP_INVALID;
-        int              ret = 0;
-        int              xl_id = 0;
-	afr_private_t   *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	struct subvol_healer *healer = NULL;
-	int i = 0;
-	char key[64];
-	int op_ret = 0;
-	uint64_t cnt = 0;
-
-	priv = this->private;
-	shd = &priv->shd;
-
-        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
-        switch (op) {
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    int xl_id = 0;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    struct subvol_healer *healer = NULL;
+    int i = 0;
+    char key[64];
+    int keylen = 0;
+    int this_name_len = 0;
+    int op_ret = 0;
+    uint64_t cnt = 0;
+
+#define AFR_SET_DICT_AND_LOG(name, output, key, keylen, dict_str,              \
+                             dict_str_len)                                     \
+    {                                                                          \
+        int ret;                                                               \
+                                                                               \
+        ret = dict_set_nstrn(output, key, keylen, dict_str, dict_str_len);     \
+        if (ret) {                                                             \
+            gf_smsg(name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,         \
+                    "key=%s", key, "value=%s", dict_str, NULL);                \
+        }                                                                      \
+    }
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    ret = dict_get_int32_sizen(input, "xl-op", (int32_t *)&op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=xl-op", NULL);
+        goto out;
+    }
+    this_name_len = strlen(this->name);
+    ret = dict_get_int32n(input, this->name, this_name_len, &xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(output, this->name, this_name_len, xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    switch (op) {
         case GF_SHD_OP_HEAL_INDEX:
-		op_ret = 0;
-
-		for (i = 0; i < priv->child_count; i++) {
-			healer = &shd->index_healers[i];
-			snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-			if (!priv->child_up[i]) {
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-                                op_ret = -1;
-			} else if (AFR_COUNT (priv->child_up,
-					      priv->child_count) < 2) {
-				ret = dict_set_str (output, key,
-						    "< 2 bricks in replica are up");
-                                op_ret = -1;
-			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
-				ret = dict_set_str (output, key,
-						    "Brick is remote");
-			} else {
-				ret = dict_set_str (output, key,
-						    "Started self-heal");
-				afr_shd_index_healer_spawn (this, i);
-			}
-		}
-                break;
+            op_ret = 0;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->index_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                    op_ret = -1;
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                    op_ret = -1;
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
+
+                    ret = afr_shd_index_healer_spawn(this, i);
+
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                }
+            }
+            break;
         case GF_SHD_OP_HEAL_FULL:
-		op_ret = -1;
-
-		for (i = 0; i < priv->child_count; i++) {
-			healer = &shd->full_healers[i];
-			snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-			if (!priv->child_up[i]) {
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-			} else if (AFR_COUNT (priv->child_up,
-					      priv->child_count) < 2) {
-				ret = dict_set_str (output, key,
-						    "< 2 bricks in replica are up");
-			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
-				ret = dict_set_str (output, key,
-						    "Brick is remote");
-			} else {
-				ret = dict_set_str (output, key,
-						    "Started self-heal");
-				afr_shd_full_healer_spawn (this, i);
-				op_ret = 0;
-			}
-		}
-                break;
-        case GF_SHD_OP_INDEX_SUMMARY:
-                /* this case has been handled in glfs-heal.c */
-                break;
-        case GF_SHD_OP_HEALED_FILES:
-        case GF_SHD_OP_HEAL_FAILED_FILES:
-                for (i = 0; i < priv->child_count; i++) {
-                        snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-                        ret = dict_set_str (output, key, "Operation Not "
-                                            "Supported");
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->full_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
+
+                    ret = afr_shd_full_healer_spawn(this, i);
+
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                    op_ret = 0;
                 }
-                break;
+            }
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+            /* this case has been handled in glfs-heal.c */
+            break;
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-		eh_dump (shd->split_brain, output, afr_add_shd_event);
-                break;
+            eh_dump(shd->split_brain, output, afr_add_shd_event);
+            break;
         case GF_SHD_OP_STATISTICS:
-		for (i = 0; i < priv->child_count; i++) {
-			eh_dump (shd->statistics[i], output,
-				 afr_add_crawl_event);
-			afr_shd_dict_add_crawl_event (this, output,
-						      &shd->index_healers[i].crawl_event);
-			afr_shd_dict_add_crawl_event (this, output,
-						      &shd->full_healers[i].crawl_event);
-		}
-                break;
+            for (i = 0; i < priv->child_count; i++) {
+                eh_dump(shd->statistics[i], output, afr_add_crawl_event);
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->index_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->full_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+            }
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-		op_ret = -1;
-
-		for (i = 0; i < priv->child_count; i++) {
-			if (!priv->child_up[i]) {
-				snprintf (key, sizeof (key), "%d-%d-status",
-                                          xl_id, i);
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-			} else {
-				snprintf (key, sizeof (key), "%d-%d-hardlinks",
-                                          xl_id, i);
-				ret = afr_shd_get_index_count (this, i, &cnt);
-				if (ret == 0) {
-					ret = dict_set_uint64 (output, key, cnt);
-				}
-				op_ret = 0;
-			}
-		}
-
-//                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
-//                                                     STATISTICS_TO_BE_HEALED,
-//                                                     output);
-                break;
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                if (!priv->child_up[i]) {
+                    keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id,
+                                      i);
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else {
+                    snprintf(key, sizeof(key), "%d-%d-hardlinks", xl_id, i);
+                    ret = afr_shd_get_index_count(this, i, &cnt);
+                    if (ret == 0) {
+                        ret = dict_set_uint64(output, key, cnt);
+                    }
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_DICT_SET_FAILED, NULL);
+                    }
+                    op_ret = 0;
+                }
+            }
+
+            break;
 
         default:
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "Unknown set op %d", op);
-                break;
-        }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "op=%d",
+                    op, NULL);
+            break;
+    }
 out:
-        dict_del (output, this->name);
-        return op_ret;
+    dict_deln(output, this->name, this_name_len);
+    return op_ret;
+
+#undef AFR_SET_DICT_AND_LOG
 }
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index f591515669c..18db728ea7b 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -8,72 +8,68 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _AFR_SELF_HEALD_H
 #define _AFR_SELF_HEALD_H
 
 #include <pthread.h>
 
-
 typedef struct {
-	int child;
-	char *path;
+    char *path;
+    int child;
 } shd_event_t;
 
 typedef struct {
-	int      child;
-	uint64_t healed_count;
-        uint64_t split_brain_count;
-        uint64_t heal_failed_count;
-
-	/* If start_time is 0, it means crawler is not in progress
-	   and stats are not valid */
-	time_t   start_time;
-	/* If start_time is NOT 0 and end_time is 0, it means
-	   cralwer is in progress */
-        time_t   end_time;
-        char     *crawl_type;
+    uint64_t healed_count;
+    uint64_t split_brain_count;
+    uint64_t heal_failed_count;
+
+    /* If start_time is 0, it means crawler is not in progress
+       and stats are not valid */
+    time_t start_time;
+    /* If start_time is NOT 0 and end_time is 0, it means
+       cralwer is in progress */
+    time_t end_time;
+    char *crawl_type;
+    int child;
 } crawl_event_t;
 
 struct subvol_healer {
-	xlator_t        *this;
-	int              subvol;
-	gf_boolean_t     local;
-	gf_boolean_t     running;
-	gf_boolean_t     rerun;
-	crawl_event_t    crawl_event;
-	pthread_mutex_t  mutex;
-	pthread_cond_t   cond;
-	pthread_t        thread;
+    xlator_t *this;
+    crawl_event_t crawl_event;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t thread;
+    int subvol;
+    gf_boolean_t local;
+    gf_boolean_t running;
+    gf_boolean_t rerun;
 };
 
 typedef struct {
-	gf_boolean_t            iamshd;
-	gf_boolean_t            enabled;
-	int                     timeout;
-	struct subvol_healer   *index_healers;
-	struct subvol_healer   *full_healers;
-
-        eh_t                    *split_brain;
-        eh_t                    **statistics;
-        uint32_t                max_threads;
-        uint32_t                wait_qlength;
+    struct subvol_healer *index_healers;
+    struct subvol_healer *full_healers;
+
+    eh_t *split_brain;
+    eh_t **statistics;
+    int timeout;
+    uint32_t max_threads;
+    uint32_t wait_qlength;
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamshd;
+    gf_boolean_t enabled;
 } afr_self_heald_t;
 
-
-int
-afr_selfheal_childup (xlator_t *this, int subvol);
-
 int
-afr_selfheal_daemon_init (xlator_t *this);
+afr_selfheal_daemon_init(xlator_t *this);
 
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output);
 
 int
-afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid,
-                      char **path_p);
+afr_shd_gfid_to_path(xlator_t *this, xlator_t *subvol, uuid_t gfid,
+                     char **path_p);
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name);
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type);
 #endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 41f36f4b2d8..a51f79b1f43 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -8,10 +8,10 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
-#include "timer.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/timer.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
@@ -21,2240 +21,2907 @@
 #include <signal.h>
 
 typedef enum {
-        AFR_TRANSACTION_PRE_OP,
-        AFR_TRANSACTION_POST_OP,
+    AFR_TRANSACTION_PRE_OP,
+    AFR_TRANSACTION_POST_OP,
 } afr_xattrop_type_t;
 
-gf_boolean_t
-afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this);
+static void
+afr_lock_resume_shared(struct list_head *list);
 
-gf_boolean_t
-afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this);
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this);
 
-int
-afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
-		  afr_changelog_resume_t changelog_resume,
-                  afr_xattrop_type_t op);
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno);
 
 void
-afr_zero_fill_stat (afr_local_t *local)
-{
-        if (!local)
-                return;
-        if (local->transaction.type == AFR_DATA_TRANSACTION ||
-            local->transaction.type == AFR_METADATA_TRANSACTION) {
-                gf_zero_fill_stat (&local->cont.inode_wfop.prebuf);
-                gf_zero_fill_stat (&local->cont.inode_wfop.postbuf);
-        } else if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
-                   local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                gf_zero_fill_stat (&local->cont.dir_fop.buf);
-                gf_zero_fill_stat (&local->cont.dir_fop.preparent);
-                gf_zero_fill_stat (&local->cont.dir_fop.postparent);
-                if (local->transaction.type == AFR_ENTRY_TRANSACTION)
-                        return;
-                gf_zero_fill_stat (&local->cont.dir_fop.prenewparent);
-                gf_zero_fill_stat (&local->cont.dir_fop.postnewparent);
-        }
-}
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared);
 
-/* In case of errors afr needs to choose which xdata from lower xlators it needs
- * to unwind with. The way it is done is by checking if there are
- * any good subvols which failed. Give preference to errnos other than
- * ENOTCONN even if the child is source */
 void
-afr_pick_error_xdata (afr_local_t *local, afr_private_t *priv,
-                      inode_t *inode1, unsigned char *readable1,
-                      inode_t *inode2, unsigned char *readable2)
-{
-        int     s = -1;/*selection*/
-        int     i = 0;
-        unsigned char *readable = NULL;
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this);
 
-        if (local->xdata_rsp) {
-                dict_unref (local->xdata_rsp);
-                local->xdata_rsp = NULL;
-        }
+int
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this);
 
-        readable = alloca0 (priv->child_count * sizeof (*readable));
-        if (inode2 && readable2) {/*rename fop*/
-                AFR_INTERSECT (readable, readable1, readable2,
-                               priv->child_count);
-        } else {
-                memcpy (readable, readable1,
-                        sizeof (*readable) * priv->child_count);
-        }
+gf_boolean_t
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
+gf_boolean_t
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this);
 
-                if (local->replies[i].op_ret >= 0)
-                        continue;
+int
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count);
+int
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume,
+                 afr_xattrop_type_t op);
 
-                if (local->replies[i].op_errno == ENOTCONN)
-                        continue;
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this);
 
-                /*Order is important in the following condition*/
-                if ((s < 0) || (!readable[s] && readable[i]))
-                        s = i;
-        }
+static int
+afr_ta_post_op_do(void *opaque);
 
-        if (s != -1 && local->replies[s].xdata) {
-                local->xdata_rsp = dict_ref (local->replies[s].xdata);
-        } else if (s == -1) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->replies[i].valid)
-                                continue;
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local);
 
-                        if (local->replies[i].op_ret >= 0)
-                                continue;
+static int
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this);
 
-                        if (!local->replies[i].xdata)
-                                continue;
-                        local->xdata_rsp = dict_ref (local->replies[i].xdata);
-                        break;
-                }
-        }
-}
+static void
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno);
 
-gf_boolean_t
-afr_needs_changelog_update (afr_local_t *local)
+void
+afr_ta_locked_priv_invalidate(afr_private_t *priv)
 {
-        if (local->transaction.type == AFR_DATA_TRANSACTION)
-                return _gf_true;
-        if (!local->optimistic_change_log)
-                return _gf_true;
-        return _gf_false;
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    priv->ta_notify_dom_lock_offset = 0;
 }
 
-static int32_t
-afr_quorum_errno (afr_private_t *priv)
+static void
+afr_ta_process_waitq(xlator_t *this)
 {
-        if (priv->quorum_reads)
-                return ENOTCONN;
-        return EROFS;
+    afr_local_t *entry = NULL;
+    afr_private_t *priv = this->private;
+    struct list_head waitq = {
+        0,
+    };
+
+    INIT_LIST_HEAD(&waitq);
+    LOCK(&priv->lock);
+    list_splice_init(&priv->ta_waitq, &waitq);
+    UNLOCK(&priv->lock);
+    list_for_each_entry(entry, &waitq, ta_waitq)
+    {
+        afr_ta_decide_post_op_state(entry->transaction.frame, this);
+    }
 }
 
 int
-__afr_txn_write_fop (call_frame_t *frame, xlator_t *this)
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        unsigned char *failed_subvols = NULL;
-        int i = 0;
+    afr_ta_process_waitq(ta_frame->this);
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
 
-        local = frame->local;
-        priv = this->private;
+int
+afr_release_notify_lock_for_ta(void *opaque)
+{
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1;
+
+    this = (xlator_t *)opaque;
+    priv = this->private;
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+    flock.l_type = F_UNLCK;
+    flock.l_start = priv->ta_notify_dom_lock_offset;
+    flock.l_len = 1;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
+
+    LOCK(&priv->lock);
+    {
+        afr_ta_locked_priv_invalidate(priv);
+    }
+    UNLOCK(&priv->lock);
+out:
+    loc_wipe(&loc);
+    return ret;
+}
 
-        failed_subvols = local->transaction.failed_subvols;
+void
+afr_zero_fill_stat(afr_local_t *local)
+{
+    if (!local)
+        return;
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.inode_wfop.prebuf);
+        gf_zero_fill_stat(&local->cont.inode_wfop.postbuf);
+    } else if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+               local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.dir_fop.buf);
+        gf_zero_fill_stat(&local->cont.dir_fop.preparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postparent);
+        if (local->transaction.type == AFR_ENTRY_TRANSACTION)
+            return;
+        gf_zero_fill_stat(&local->cont.dir_fop.prenewparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postnewparent);
+    }
+}
 
-        call_count = priv->child_count - AFR_COUNT (failed_subvols,
-                                                    priv->child_count);
+/* In case of errors afr needs to choose which xdata from lower xlators it needs
+ * to unwind with. The way it is done is by checking if there are
+ * any good subvols which failed. Give preference to errnos other than
+ * ENOTCONN even if the child is source */
+void
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2)
+{
+    int s = -1; /*selection*/
+    int i = 0;
+    unsigned char *readable = NULL;
+
+    if (local->xdata_rsp) {
+        dict_unref(local->xdata_rsp);
+        local->xdata_rsp = NULL;
+    }
+
+    readable = alloca0(priv->child_count * sizeof(*readable));
+    if (inode2 && readable2) { /*rename fop*/
+        AFR_INTERSECT(readable, readable1, readable2, priv->child_count);
+    } else {
+        memcpy(readable, readable1, sizeof(*readable) * priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret >= 0)
+            continue;
+
+        if (local->replies[i].op_errno == ENOTCONN)
+            continue;
+
+        /*Order is important in the following condition*/
+        if ((s < 0) || (!readable[s] && readable[i]))
+            s = i;
+    }
+
+    if (s != -1 && local->replies[s].xdata) {
+        local->xdata_rsp = dict_ref(local->replies[s].xdata);
+    } else if (s == -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
+            if (local->replies[i].op_ret >= 0)
+                continue;
+
+            if (!local->replies[i].xdata)
+                continue;
+            local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            break;
         }
+    }
+}
+
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local)
+{
+    if (local->transaction.type == AFR_DATA_TRANSACTION)
+        return _gf_true;
+    if (!local->optimistic_change_log)
+        return _gf_true;
+    return _gf_false;
+}
 
-        local->call_count = call_count;
+gf_boolean_t
+afr_changelog_has_quorum(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    unsigned char *success_children = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i] && !failed_subvols[i]) {
-			local->transaction.wind (frame, this, i);
+    priv = this->private;
+    success_children = alloca0(priv->child_count);
 
-                        if (!--call_count)
-                                break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i]) {
+            success_children[i] = 1;
         }
+    }
 
-        return 0;
-}
+    if (afr_has_quorum(success_children, this, NULL)) {
+        return _gf_true;
+    }
 
+    return _gf_false;
+}
 
-int
-__afr_txn_write_done (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_write_subvol_valid(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        gf_boolean_t unwind = _gf_false;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t write_subvol = 0;
+    unsigned char *writable = NULL;
+    uint16_t datamap = 0;
 
-        priv  = this->private;
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
+    writable = alloca0(priv->child_count);
 
-        if (priv->consistent_metadata) {
-                LOCK (&frame->lock);
-                {
-                        unwind = (local->transaction.main_frame != NULL);
-                }
-                UNLOCK (&frame->lock);
-                if (unwind)/*It definitely did post-op*/
-                        afr_zero_fill_stat (local);
-        }
-        local->transaction.unwind (frame, this);
+    write_subvol = afr_write_subvol_get(frame, this);
+    datamap = (write_subvol & 0x00000000ffff0000) >> 16;
+    for (i = 0; i < priv->child_count; i++) {
+        if (datamap & (1 << i))
+            writable[i] = 1;
 
-        AFR_STACK_DESTROY (frame);
+        if (writable[i] && !local->transaction.failed_subvols[i])
+            return _gf_true;
+    }
 
-        return 0;
+    return _gf_false;
 }
 
+int
+afr_transaction_fop(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    unsigned char *failed_subvols = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    failed_subvols = local->transaction.failed_subvols;
+    call_count = priv->child_count -
+                 AFR_COUNT(failed_subvols, priv->child_count);
+    /* Fail if pre-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this) || !call_count) {
+        local->op_ret = -1;
+        /* local->op_errno is already captured in changelog cbk. */
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
 
-call_frame_t*
-afr_transaction_detach_fop_frame (call_frame_t *frame)
-{
-        afr_local_t *   local = NULL;
-        call_frame_t   *fop_frame = NULL;
+    /* Fail if at least one writeable brick isn't up.*/
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !afr_is_write_subvol_valid(frame, this)) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
 
-        local = frame->local;
+    local->call_count = call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] && !failed_subvols[i]) {
+            local->transaction.wind(frame, this, i);
 
-        LOCK (&frame->lock);
-        {
-                fop_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
+            if (!--call_count)
+                break;
         }
-        UNLOCK (&frame->lock);
+    }
 
-        return fop_frame;
+    return 0;
 }
 
-
-static void
-afr_save_lk_owner (call_frame_t *frame)
+int
+afr_transaction_done(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t unwind = _gf_false;
+    afr_lock_t *lock = NULL;
+    afr_local_t *lock_local = NULL;
 
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        local->saved_lk_owner = frame->root->lk_owner;
-}
+    if (priv->consistent_metadata) {
+        LOCK(&frame->lock);
+        {
+            unwind = (local->transaction.main_frame != NULL);
+        }
+        UNLOCK(&frame->lock);
+        if (unwind) /*It definitely did post-op*/
+            afr_zero_fill_stat(local);
+    }
+
+    if (local->transaction.do_eager_unlock) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_false;
+            lock->release = _gf_false;
+            list_splice_init(&lock->frozen, &lock->waiting);
+            if (list_empty(&lock->waiting))
+                goto unlock;
+            lock_local = list_entry(lock->waiting.next, afr_local_t,
+                                    transaction.wait_list);
+            list_del_init(&lock_local->transaction.wait_list);
+            list_add(&lock_local->transaction.owner_list, &lock->owners);
+        }
+    unlock:
+        UNLOCK(&local->inode->lock);
+    }
+    if (lock_local) {
+        afr_lock(lock_local->transaction.frame,
+                 lock_local->transaction.frame->this);
+    }
+    local->transaction.unwind(frame, this);
+
+    GF_ASSERT(list_empty(&local->transaction.owner_list));
+    GF_ASSERT(list_empty(&local->transaction.wait_list));
+    AFR_STACK_DESTROY(frame);
 
+    return 0;
+}
 
 static void
-afr_restore_lk_owner (call_frame_t *frame)
+afr_lock_fail_shared(afr_local_t *local, struct list_head *list)
 {
-        afr_local_t * local = NULL;
-
-        local = frame->local;
+    afr_local_t *each = NULL;
 
-        frame->root->lk_owner = local->saved_lk_owner;
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        each->op_ret = -1;
+        each->op_errno = local->op_errno;
+        afr_transaction_done(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
 }
 
-void
-__mark_all_success (call_frame_t *frame, xlator_t *this)
+static void
+afr_handle_lock_acquire_failure(afr_local_t *local)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int i;
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    if (!local->transaction.eager_lock_on)
+        goto out;
 
-	for (i = 0; i < priv->child_count; i++) {
-		local->transaction.failed_subvols[i] = 0;
-	}
-}
+    lock = &local->inode_ctx->lock[local->transaction.type];
 
-void
-afr_compute_pre_op_sources (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        afr_transaction_type type = -1;
-        dict_t *xdata = NULL;
-        int **matrix = NULL;
-        int idx = -1;
-        int i = 0;
-        int j = 0;
-
-        priv = this->private;
-        local = frame->local;
-        type = local->transaction.type;
-        idx = afr_index_for_transaction_type (type);
-        matrix = ALLOC_MATRIX (priv->child_count, int);
+    INIT_LIST_HEAD(&shared);
+    LOCK(&local->inode->lock);
+    {
+        lock->release = _gf_true;
+        list_splice_init(&lock->waiting, &shared);
+    }
+    UNLOCK(&local->inode->lock);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op_xdata[i])
-                        continue;
-                xdata = local->transaction.pre_op_xdata[i];
-                afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
-        }
+    afr_lock_fail_shared(local, &shared);
+    local->transaction.do_eager_unlock = _gf_true;
+out:
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    afr_unlock(local->transaction.frame, local->transaction.frame->this);
+}
 
-        memset (local->transaction.pre_op_sources, 1, priv->child_count);
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *fop_frame = NULL;
 
-        /*If lock or pre-op failed on a brick, it is not a source. */
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.failed_subvols[i])
-                        local->transaction.pre_op_sources[i] = 0;
-        }
+    local = frame->local;
 
-        /* If brick is blamed by others, it is not a source. */
-        for (i = 0; i < priv->child_count; i++)
-                for (j = 0; j < priv->child_count; j++)
-                        if (matrix[i][j] != 0)
-                                local->transaction.pre_op_sources[j] = 0;
+    afr_handle_inconsistent_fop(frame, &local->op_ret, &local->op_errno);
+    LOCK(&frame->lock);
+    {
+        fop_frame = local->transaction.main_frame;
+        local->transaction.main_frame = NULL;
+    }
+    UNLOCK(&frame->lock);
 
-        /*We don't need the xattrs any more. */
-        for (i = 0; i < priv->child_count; i++)
-                if (local->transaction.pre_op_xdata[i]) {
-                        dict_unref (local->transaction.pre_op_xdata[i]);
-                        local->transaction.pre_op_xdata[i] = NULL;
-                }
+    return fop_frame;
 }
 
-void
-afr_txn_arbitrate_fop_cbk (call_frame_t *frame, xlator_t *this)
+static void
+afr_save_lk_owner(call_frame_t *frame)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        gf_boolean_t fop_failed = _gf_false;
-        unsigned char *pre_op_sources = NULL;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
-        pre_op_sources = local->transaction.pre_op_sources;
+    afr_local_t *local = NULL;
 
-        if (priv->arbiter_count != 1 || local->op_ret < 0)
-                return;
+    local = frame->local;
 
-        /* If the fop failed on the brick, it is not a source. */
-        for (i = 0; i < priv->child_count; i++)
-                if (local->transaction.failed_subvols[i])
-                        pre_op_sources[i] = 0;
+    local->saved_lk_owner = frame->root->lk_owner;
+}
 
-        switch (AFR_COUNT (pre_op_sources, priv->child_count)) {
-        case 1:
-                if (pre_op_sources[ARBITER_BRICK_INDEX])
-                        fop_failed = _gf_true;
-                break;
-        case 0:
-                fop_failed = _gf_true;
-                break;
-        }
+static void
+afr_restore_lk_owner(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
 
-        if (fop_failed) {
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-        }
+    local = frame->local;
 
-        return;
+    frame->root->lk_owner = local->saved_lk_owner;
 }
 
 void
-afr_txn_arbitrate_fop (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int pre_op_sources_count = 0;
-
-        priv = this->private;
-        local = frame->local;
-
-        afr_compute_pre_op_sources (frame, this);
-        pre_op_sources_count = AFR_COUNT (local->transaction.pre_op_sources,
-                                          priv->child_count);
-
-        /* If arbiter is the only source, do not proceed. */
-        if (pre_op_sources_count < 2 &&
-            local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
-                local->internal_lock.lock_cbk = local->transaction.done;
-                local->op_ret = -1;
-                local->op_errno =  ENOTCONN;
-                afr_restore_lk_owner (frame);
-                afr_unlock (frame, this);
-        } else {
-                local->transaction.fop (frame, this);
-        }
-
-        return;
-}
-
-int
-afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
+__mark_all_success(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv = NULL;
-        fd_t            *fd   = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i;
 
-        local = frame->local;
-        priv = this->private;
-        fd    = local->fd;
+    local = frame->local;
+    priv = this->private;
 
-        /*  Perform fops with the lk-owner from top xlator.
-         *  Eg: lk-owner of posix-lk and flush should be same,
-         *  flush cant clear the  posix-lks without that lk-owner.
-         */
-        afr_save_lk_owner (frame);
-        frame->root->lk_owner =
-                local->transaction.main_frame->root->lk_owner;
-
-	if (local->pre_op_compat)
-		/* old mode, pre-op was done as afr_changelog_do()
-		   just now, before OP */
-		afr_changelog_pre_op_update (frame, this);
-
-        /* The wake up needs to happen independent of
-           what type of fop arrives here. If it was
-           a write, then it has already inherited the
-           lock and changelog. If it was not a write,
-           then the presumption of the optimization (of
-           optimizing for successive write operations)
-           fails.
-        */
-        if (fd)
-                afr_delayed_changelog_wake_up (this, fd);
-        if (priv->arbiter_count == 1) {
-                afr_txn_arbitrate_fop (frame, this);
-        } else {
-                local->transaction.fop (frame, this);
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        local->transaction.failed_subvols[i] = 0;
+    }
+}
 
-	return 0;
+void
+afr_compute_pre_op_sources(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_transaction_type type = -1;
+    dict_t *xdata = NULL;
+    int **matrix = NULL;
+    int idx = -1;
+    int i = 0;
+    int j = 0;
+
+    priv = this->private;
+    local = frame->local;
+    type = local->transaction.type;
+    idx = afr_index_for_transaction_type(type);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.changelog_xdata[i])
+            continue;
+        xdata = local->transaction.changelog_xdata[i];
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
+
+    memset(local->transaction.pre_op_sources, 1, priv->child_count);
+
+    /*If lock or pre-op failed on a brick, it is not a source. */
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->transaction.pre_op_sources[i] = 0;
+    }
+
+    /* If brick is blamed by others, it is not a source. */
+    for (i = 0; i < priv->child_count; i++)
+        for (j = 0; j < priv->child_count; j++)
+            if (matrix[i][j] != 0)
+                local->transaction.pre_op_sources[j] = 0;
 }
 
-static int
-__changelog_enabled (afr_private_t *priv, afr_transaction_type type)
+void
+afr_txn_arbitrate_fop(call_frame_t *frame, xlator_t *this)
 {
-        int ret = 0;
-
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                if (priv->data_change_log)
-                        ret = 1;
-
-                break;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_sources_count = 0;
+    int i = 0;
 
-        case AFR_METADATA_TRANSACTION:
-                if (priv->metadata_change_log)
-                        ret = 1;
+    priv = this->private;
+    local = frame->local;
 
-                break;
+    afr_compute_pre_op_sources(frame, this);
+    pre_op_sources_count = AFR_COUNT(local->transaction.pre_op_sources,
+                                     priv->child_count);
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                if (priv->entry_change_log)
-                        ret = 1;
+    /* If arbiter is the only source, do not proceed. */
+    if (pre_op_sources_count < 2 &&
+        local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        for (i = 0; i < priv->child_count; i++)
+            local->transaction.failed_subvols[i] = 1;
+    }
 
-                break;
-        }
+    afr_transaction_fop(frame, this);
 
-        return ret;
+    return;
 }
 
+int
+afr_transaction_perform_fop(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int ret = 0;
+    int failure_count = 0;
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    INIT_LIST_HEAD(&shared);
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !local->transaction.inherited) {
+        ret = afr_write_subvol_set(frame, this);
+        if (ret) {
+            /*act as if operation failed on all subvols*/
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            for (i = 0; i < priv->child_count; i++)
+                local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    if (local->pre_op_compat)
+        /* old mode, pre-op was done as afr_changelog_do()
+           just now, before OP */
+        afr_changelog_pre_op_update(frame, this);
+
+    if (!local->transaction.eager_lock_on || local->transaction.inherited)
+        goto fop;
+    failure_count = AFR_COUNT(local->transaction.failed_subvols,
+                              priv->child_count);
+    if (failure_count == priv->child_count) {
+        afr_handle_lock_acquire_failure(local);
+        return 0;
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_true;
+            __afr_transaction_wake_shared(local, &shared);
+        }
+        UNLOCK(&local->inode->lock);
+    }
 
-static int
-__fop_changelog_needed (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        int op_ret = 0;
-        afr_transaction_type type = -1;
-
-        priv  = this->private;
-        local = frame->local;
-        type  = local->transaction.type;
-
-        if (__changelog_enabled (priv, type)) {
-                switch (local->op) {
-
-                case GF_FOP_WRITE:
-                case GF_FOP_FTRUNCATE:
-                        op_ret = 1;
-                        break;
-
-                case GF_FOP_FLUSH:
-                        op_ret = 0;
-                        break;
+fop:
+    /*  Perform fops with the lk-owner from top xlator.
+     *  Eg: lk-owner of posix-lk and flush should be same,
+     *  flush cant clear the  posix-lks without that lk-owner.
+     */
+    afr_save_lk_owner(frame);
+    frame->root->lk_owner = local->transaction.main_frame->root->lk_owner;
 
-                default:
-                        op_ret = 1;
-                }
-        }
+    if (priv->arbiter_count == 1) {
+        afr_txn_arbitrate_fop(frame, this);
+    } else {
+        afr_transaction_fop(frame, this);
+    }
 
-        return op_ret;
+    afr_lock_resume_shared(&shared);
+    return 0;
 }
 
-
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int **pending)
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int **pending)
 {
-        int i = 0;
-        int ret = 0;
-
-        for (i = 0; i < priv->child_count; i++) {
+    int i = 0;
+    int ret = 0;
 
-                ret = dict_set_static_bin (xattr, priv->pending_key[i],
-					   pending[i],
-					   AFR_NUM_CHANGE_LOGS * sizeof (int));
-                /* 3 = data+metadata+entry */
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], pending[i],
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        /* 3 = data+metadata+entry */
 
-                if (ret)
-                        break;
-        }
+        if (ret)
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
-{
-        int ret = 0;
-
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                ret = priv->child_count;
+static void
+afr_ta_dom_lock_check_and_release(afr_ta_fop_state_t fop_state, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    gf_boolean_t release = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        /*Once we get notify lock release upcall notification,
+         if any of the fop state counters are non-zero, we will
+         not release the lock.
+         */
+        onwire_count = priv->ta_on_wire_txn_count;
+        inmem_count = priv->ta_in_mem_txn_count;
+        switch (fop_state) {
+            case TA_GET_INFO_FROM_TA_FILE:
+                onwire_count = --priv->ta_on_wire_txn_count;
                 break;
-
-        case AFR_METADATA_TRANSACTION:
-                ret = priv->child_count;
+            case TA_INFO_IN_MEMORY_SUCCESS:
+            case TA_INFO_IN_MEMORY_FAILED:
+                inmem_count = --priv->ta_in_mem_txn_count;
                 break;
-
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                ret = priv->child_count;
+            case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+                GF_ASSERT(0);
+                break;
+            case TA_SUCCESS:
                 break;
         }
+        release = priv->release_ta_notify_dom_lock;
+    }
+    UNLOCK(&priv->lock);
+
+    if (inmem_count != 0 || release == _gf_false || onwire_count != 0)
+        return;
 
-        return ret;
+    afr_ta_lock_release_synctask(this);
 }
 
-/* {{{ pending */
+static void
+afr_ta_process_onwireq(afr_ta_fop_state_t fop_state, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *entry = NULL;
+    int bad_child = AFR_CHILD_UNKNOWN;
+
+    struct list_head onwireq = {
+        0,
+    };
+    INIT_LIST_HEAD(&onwireq);
+
+    LOCK(&priv->lock);
+    {
+        bad_child = priv->ta_bad_child_index;
+        if (bad_child == AFR_CHILD_UNKNOWN) {
+            /*The previous on-wire ta_post_op was a failure. Just dequeue
+             *one element to wind on-wire again. */
+            entry = list_entry(priv->ta_onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+        } else {
+            /* Prepare to process all fops based on bad_child_index. */
+            list_splice_init(&priv->ta_onwireq, &onwireq);
+        }
+    }
+    UNLOCK(&priv->lock);
 
+    if (entry) {
+        afr_ta_post_op_synctask(this, entry);
+        return;
+    } else {
+        while (!list_empty(&onwireq)) {
+            entry = list_entry(onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+            if (entry->ta_failed_subvol == bad_child) {
+                afr_post_op_handle_success(entry->transaction.frame, this);
+            } else {
+                afr_post_op_handle_failure(entry->transaction.frame, this, EIO);
+            }
+        }
+    }
+}
 
 int
-afr_changelog_post_op_done (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
 
-	if (local->transaction.resume_stub) {
-		call_resume (local->transaction.resume_stub);
-		local->transaction.resume_stub = NULL;
-	}
+    if (priv->thin_arbiter_count) {
+        /*fop should not come here with TA_WAIT_FOR_NOTIFY_LOCK_REL state */
+        afr_ta_dom_lock_check_and_release(local->fop_state, this);
+    }
 
-	if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-		local->transaction.done (frame, this);
-	} else {
-		int_lock->lock_cbk = local->transaction.done;
-		afr_unlock (frame, this);
-	}
+    /* Fail the FOP if post-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this)) {
+        local->op_ret = -1;
+        /*local->op_errno is already captured in changelog cbk*/
+    }
 
-	return 0;
-}
+    if (local->transaction.resume_stub) {
+        call_resume(local->transaction.resume_stub);
+        local->transaction.resume_stub = NULL;
+    }
 
+    int_lock->lock_cbk = afr_transaction_done;
+    afr_unlock(frame, this);
 
-afr_inodelk_t*
-afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
-{
-        afr_inodelk_t *inodelk = NULL;
-        int           i = 0;
-
-        for (i = 0; int_lock->inodelk[i].domain; i++) {
-                inodelk = &int_lock->inodelk[i];
-                if (strcmp (dom, inodelk->domain) == 0)
-                        return inodelk;
-        }
-        return NULL;
+    return 0;
 }
 
-unsigned char*
-afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
+static void
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno)
 {
-        unsigned char *locked_nodes = NULL;
-        afr_inodelk_t *inodelk = NULL;
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-                locked_nodes = inodelk->locked_nodes;
-        break;
+    afr_local_t *local = frame->local;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                /*Because same set of subvols participate in all lockee
-                 * entities*/
-                locked_nodes = int_lock->lockee[0].locked_nodes;
-        break;
-        }
-        return locked_nodes;
+    gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_THIN_ARB,
+           "Failing %s for gfid %s. Fop state is:%d", gf_fop_list[local->op],
+           uuid_utoa(local->inode->gfid), local->fop_state);
+
+    afr_changelog_post_op_done(frame, this);
 }
 
+unsigned char *
+afr_locked_nodes_get(afr_transaction_type type, afr_internal_lock_t *int_lock)
+{
+    /*Because same set of subvols participate in all lockee
+     * entities*/
+    return int_lock->lockee[0].locked_nodes;
+}
 
 int
-afr_changelog_call_count (afr_transaction_type type,
-			  unsigned char *pre_op_subvols,
-			  unsigned int child_count)
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count)
 {
-        int call_count = 0;
+    int i = 0;
+    int call_count = 0;
 
-	call_count = AFR_COUNT(pre_op_subvols, child_count);
+    for (i = 0; i < child_count; i++) {
+        if (pre_op_subvols[i] && !failed_subvols[i]) {
+            call_count++;
+        }
+    }
 
-        if (type == AFR_ENTRY_RENAME_TRANSACTION)
-                call_count *= 2;
+    if (type == AFR_ENTRY_RENAME_TRANSACTION)
+        call_count *= 2;
 
-        return call_count;
+    return call_count;
 }
 
-
 gf_boolean_t
-afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i] &&
-                    local->transaction.failed_subvols[i])
-                        return _gf_false;
-        }
+    if (priv->thin_arbiter_count) {
+        /* We need to perform post-op even if 1 data brick was down
+         * before the txn started.*/
+        if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count))
+            return _gf_false;
+    }
 
-        return _gf_true;
-}
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            local->transaction.failed_subvols[i])
+            return _gf_false;
+    }
 
+    return _gf_true;
+}
 
 void
-afr_handle_symmetric_errors (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int op_errno = 0;
-	int i_errno = 0;
-	gf_boolean_t matching_errors = _gf_true;
-	int i = 0;
-
-	priv = this->private;
-	local = frame->local;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret != -1) {
-			/* Operation succeeded on at least on subvol,
-			   so it is not a failed-everywhere situation.
-			*/
-			matching_errors = _gf_false;
-			break;
-		}
-		i_errno = local->replies[i].op_errno;
-
-		if (i_errno == ENOTCONN || i_errno == EDQUOT ||
-                    i_errno == ENOSPC) {
-			/* ENOTCONN is not a symmetric error. We do not
-			   know if the operation was performed on the
-			   backend or not.
-			*  Before reaching EDQUOT and ENOSPC, each brick would
-			*  have written some amount of data, hence this is not
-			*  symmetric error.
-			*/
-			matching_errors = _gf_false;
-			break;
-		}
-
-		if (!op_errno) {
-			op_errno = i_errno;
-		} else if (op_errno != i_errno) {
-			/* Mismatching op_errno's */
-			matching_errors = _gf_false;
-			break;
-		}
-	}
-
-	if (matching_errors)
-		__mark_all_success (frame, this);
+afr_handle_symmetric_errors(call_frame_t *frame, xlator_t *this)
+{
+    if (afr_is_symmetric_error(frame, this))
+        __mark_all_success(frame, this);
 }
 
 gf_boolean_t
-afr_has_quorum (unsigned char *subvols, xlator_t *this)
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame)
 {
-        unsigned int  quorum_count = 0;
-        afr_private_t *priv  = NULL;
-        unsigned int  up_children_count = 0;
+    unsigned int quorum_count = 0;
+    afr_private_t *priv = NULL;
+    unsigned int up_children_count = 0;
 
-        priv = this->private;
-        up_children_count = AFR_COUNT (subvols, priv->child_count);
+    priv = this->private;
+    up_children_count = AFR_COUNT(subvols, priv->child_count);
 
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
+    if (afr_lookup_has_quorum(frame, up_children_count))
+        return _gf_true;
+
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
         /*
-         * Special case for even numbers of nodes in auto-quorum:
-         * if we have exactly half children up
-         * and that includes the first ("senior-most") node, then that counts
-         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
-         * while preserving the critical property that there can only be one
-         * such group.
+         * Special case for auto-quorum with an even number of nodes.
+         *
+         * A replica set with even count N can only handle the same
+         * number of failures as odd N-1 before losing "vanilla"
+         * quorum, and the probability of more simultaneous failures is
+         * actually higher.  For example, with a 1% chance of failure
+         * we'd have a 0.03% chance of two simultaneous failures with
+         * N=3 but a 0.06% chance with N=4.  However, the special case
+         * is necessary for N=2 because there's no real quorum in that
+         * case (i.e. can't normally survive *any* failures).  In that
+         * case, we treat the first node as a tie-breaker, allowing
+         * quorum to be retained in some cases while still honoring the
+         * all-important constraint that there can not simultaneously
+         * be two partitioned sets of nodes each believing they have
+         * quorum.  Of two equally sized sets, the one without that
+         * first node will lose.
+         *
+         * It turns out that the special case is beneficial for higher
+         * values of N as well.  Continuing the example above, the
+         * probability of losing quorum with N=4 and this type of
+         * quorum is (very) slightly lower than with N=3 and vanilla
+         * quorum.  The difference becomes even more pronounced with
+         * higher N.  Therefore, even though such replica counts are
+         * unlikely to be seen in practice, we might as well use the
+         * "special" quorum then as well.
          */
-                if ((priv->child_count % 2 == 0) &&
-                    (up_children_count == (priv->child_count/2)))
-                        return subvols[0];
+        if ((up_children_count * 2) == priv->child_count) {
+            return subvols[0];
         }
+    }
 
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
-                quorum_count = priv->child_count/2 + 1;
-        } else {
-                quorum_count = priv->quorum_count;
-        }
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        quorum_count = priv->child_count / 2 + 1;
+    } else {
+        quorum_count = priv->quorum_count;
+    }
 
-        if (up_children_count >= quorum_count)
-                return _gf_true;
+    if (up_children_count >= quorum_count)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_has_fop_quorum (call_frame_t *frame)
+afr_has_fop_quorum(call_frame_t *frame)
 {
-        xlator_t        *this = frame->this;
-        afr_local_t     *local = frame->local;
-        unsigned char *locked_nodes = NULL;
+    xlator_t *this = frame->this;
+    afr_local_t *local = frame->local;
+    unsigned char *locked_nodes = NULL;
 
-        locked_nodes = afr_locked_nodes_get (local->transaction.type,
-                                             &local->internal_lock);
-        return afr_has_quorum (locked_nodes, this);
+    locked_nodes = afr_locked_nodes_get(local->transaction.type,
+                                        &local->internal_lock);
+    return afr_has_quorum(locked_nodes, this, NULL);
 }
 
 static gf_boolean_t
-afr_has_fop_cbk_quorum (call_frame_t *frame)
+afr_has_fop_cbk_quorum(call_frame_t *frame)
 {
-        afr_local_t   *local   = frame->local;
-        xlator_t      *this    = frame->this;
-        afr_private_t *priv    = this->private;
-        unsigned char *success = alloca0(priv->child_count);
-        int           i        = 0;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    unsigned char *success = alloca0(priv->child_count);
+    int i = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i])
-                        if (!local->transaction.failed_subvols[i])
-                                success[i] = 1;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            if (!local->transaction.failed_subvols[i])
+                success[i] = 1;
+    }
 
-        return afr_has_quorum (success, this);
+    return afr_has_quorum(success, this, NULL);
 }
 
-void
-afr_handle_quorum (call_frame_t *frame)
+gf_boolean_t
+afr_need_dirty_marking(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int           i      = 0;
-        const char    *file  = NULL;
-        uuid_t        gfid   = {0};
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    gf_boolean_t need_dirty = _gf_false;
 
-        local = frame->local;
-        priv  = frame->this->private;
+    local = frame->local;
 
-        if (priv->quorum_count == 0)
-                return;
+    if (!priv->quorum_count || !local->optimistic_change_log)
+        return _gf_false;
 
-        /* If the fop already failed return right away to preserve errno */
-        if (local->op_ret == -1)
-                return;
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        return _gf_false;
 
-        /*
-         * Network split may happen just after the fops are unwound, so check
-         * if the fop succeeded in a way it still follows quorum. If it doesn't,
-         * mark the fop as failure, mark the changelogs so it reflects that
-         * failure.
-         *
-         * Scenario:
-         * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
-         * single file. Network split happened in a way that node1 can't see
-         * node2, node3. Node2, node3 both of them can't see node1. Now at the
-         * time of sending write all the bricks are up. Just after write fop is
-         * wound on node1, network split happens. Node1 thinks write fop failed
-         * on node2, node3 so marks pending changelog for those 2 extended
-         * attributes on node1. Node2, node3 thinks writes failed on node1 so
-         * they mark pending changelog for node1. When the network is stable
-         * again the file already is in split-brain. These checks prevent
-         * marking pending changelog on other subvolumes if the fop doesn't
-         * succeed in a way it is still following quorum. So with this fix what
-         * is happening is, node1 will have all pending changelog(FOOL) because
-         * the write succeeded only on node1 but failed on node2, node3 so
-         * instead of marking pending changelogs on node2, node3 it just treats
-         * the fop as failure and goes into DIRTY state. Where as node2, node3
-         * say they are sources and have pending changelog to node1 so there is
-         * no split-brain with the fix. The problem is eliminated completely.
-         */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) ==
+        priv->child_count)
+        return _gf_false;
 
-        if (afr_has_fop_cbk_quorum (frame))
-                return;
+    if (!afr_has_fop_cbk_quorum(frame))
+        need_dirty = _gf_true;
 
-        if (local->fd) {
-                gf_uuid_copy (gfid, local->fd->inode->gfid);
-                file = uuid_utoa (gfid);
-        } else {
-                loc_path (&local->loc, local->loc.name);
-                file = local->loc.path;
-        }
+    return need_dirty;
+}
+
+void
+afr_handle_quorum(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    const char *file = NULL;
+    uuid_t gfid = {0};
 
-        gf_msg (frame->this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
-                "%s: Failing %s as quorum is not met",
-                file, gf_fop_list[local->op]);
+    local = frame->local;
+    priv = frame->this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i])
-                        afr_transaction_fop_failed (frame, frame->this, i);
-        }
+    if (priv->quorum_count == 0)
+        return;
 
-        local->op_ret = -1;
-        local->op_errno = afr_final_errno (local, priv);
-        if (local->op_errno == 0)
-                local->op_errno = afr_quorum_errno (priv);
-        switch (local->transaction.type) {
+    /* If the fop already failed return right away to preserve errno */
+    if (local->op_ret == -1)
+        return;
+
+    /*
+     * Network split may happen just after the fops are unwound, so check
+     * if the fop succeeded in a way it still follows quorum. If it doesn't,
+     * mark the fop as failure, mark the changelogs so it reflects that
+     * failure.
+     *
+     * Scenario:
+     * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
+     * single file. Network split happened in a way that node1 can't see
+     * node2, node3. Node2, node3 both of them can't see node1. Now at the
+     * time of sending write all the bricks are up. Just after write fop is
+     * wound on node1, network split happens. Node1 thinks write fop failed
+     * on node2, node3 so marks pending changelog for those 2 extended
+     * attributes on node1. Node2, node3 thinks writes failed on node1 so
+     * they mark pending changelog for node1. When the network is stable
+     * again the file already is in split-brain. These checks prevent
+     * marking pending changelog on other subvolumes if the fop doesn't
+     * succeed in a way it is still following quorum. So with this fix what
+     * is happening is, node1 will have all pending changelog(FOOL) because
+     * the write succeeded only on node1 but failed on node2, node3 so
+     * instead of marking pending changelogs on node2, node3 it just treats
+     * the fop as failure and goes into DIRTY state. Where as node2, node3
+     * say they are sources and have pending changelog to node1 so there is
+     * no split-brain with the fix. The problem is eliminated completely.
+     */
+
+    if (afr_has_fop_cbk_quorum(frame))
+        return;
+
+    if (afr_need_dirty_marking(frame, this))
+        goto set_response;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            afr_transaction_fop_failed(frame, frame->this, i);
+    }
+
+set_response:
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    if (local->op_errno == 0)
+        local->op_errno = afr_quorum_errno(priv);
+
+    if (local->fd) {
+        gf_uuid_copy(gfid, local->fd->inode->gfid);
+        file = uuid_utoa(gfid);
+    } else {
+        loc_path(&local->loc, local->loc.name);
+        file = local->loc.path;
+    }
+
+    gf_msg(frame->this->name, GF_LOG_WARNING, local->op_errno,
+           AFR_MSG_QUORUM_FAIL, "%s: Failing %s as quorum is not met", file,
+           gf_fop_list[local->op]);
+
+    switch (local->transaction.type) {
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                afr_pick_error_xdata (local, priv, local->parent,
-                                      local->readable, local->parent2,
-                                      local->readable2);
-                break;
+            afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                                 local->parent2, local->readable2);
+            break;
         default:
-                afr_pick_error_xdata (local, priv, local->inode,
-                                      local->readable, NULL, NULL);
-                break;
-        }
+            afr_pick_error_xdata(local, priv, local->inode, local->readable,
+                                 NULL, NULL);
+            break;
+    }
 }
 
 int
-afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = this->private;
-        int i          = 0;
-	int ret = 0;
-	int idx = 0;
-        afr_local_t *  local = NULL;
-        dict_t        *xattr = NULL;
-        int            nothing_failed = 1;
-	gf_boolean_t   need_undirty = _gf_false;
-
-        afr_handle_quorum (frame);
-        local = frame->local;
-	idx = afr_index_for_transaction_type (local->transaction.type);
-
-        nothing_failed = afr_txn_nothing_failed (frame, this);
-
-	if (afr_changelog_pre_op_uninherit (frame, this))
-		need_undirty = _gf_false;
-	else
-		need_undirty = _gf_true;
-
-        if (local->op_ret < 0 && !nothing_failed) {
-                afr_changelog_post_op_done (frame, this);
-                goto out;
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop)
+{
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    loc->parent = inode_ref(priv->root_inode);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    if (is_gfid_based_fop && gf_uuid_is_null(priv->ta_gfid)) {
+        /* Except afr_ta_id_file_check() which is path based, all other gluster
+         * FOPS need gfid.*/
+        return -EINVAL;
+    }
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    loc->inode = inode_new(loc->parent->table);
+    if (!loc->inode) {
+        loc_wipe(loc);
+        return -ENOMEM;
+    }
+    return 0;
+}
+
+static int
+afr_ta_post_op_done(int ret, call_frame_t *frame, void *opaque)
+{
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *txn_frame = NULL;
+    afr_ta_fop_state_t fop_state;
+
+    local = (afr_local_t *)opaque;
+    fop_state = local->fop_state;
+    txn_frame = local->transaction.frame;
+    this = frame->this;
+
+    if (ret == 0) {
+        /*Mark pending xattrs on the up data brick.*/
+        afr_post_op_handle_success(txn_frame, this);
+    } else {
+        afr_post_op_handle_failure(txn_frame, this, -ret);
+    }
+
+    STACK_DESTROY(frame->root);
+    afr_ta_process_onwireq(fop_state, this);
+
+    return 0;
+}
+
+int **
+afr_set_changelog_xattr(afr_private_t *priv, unsigned char *pending,
+                        dict_t *xattr, afr_local_t *local)
+{
+    int **changelog = NULL;
+    int idx = 0;
+    int ret = 0;
+    int i;
+
+    if (local->is_new_entry == _gf_true) {
+        changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                               local->cont.dir_fop.buf.ia_type);
+    } else {
+        idx = afr_index_for_transaction_type(local->transaction.type);
+        changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog) {
+            goto out;
         }
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.failed_subvols[i])
+                changelog[i][idx] = hton32(1);
+        }
+        ret = afr_set_pending_dict(priv, xattr, changelog);
+        if (ret < 0) {
+            afr_matrix_cleanup(changelog, priv->child_count);
+            return NULL;
+        }
+    }
 
-	if (nothing_failed && !need_undirty) {
-		afr_changelog_post_op_done (frame, this);
-                goto out;
-	}
-
-	xattr = dict_new ();
-	if (!xattr) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->transaction.failed_subvols[i])
-			local->pending[i][idx] = hton32(1);
-	}
-
-	ret = afr_set_pending_dict (priv, xattr, local->pending);
-	if (ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-        if (need_undirty)
-		local->dirty[idx] = hton32(-1);
-	else
-		local->dirty[idx] = hton32(0);
-
-	ret = dict_set_static_bin (xattr, AFR_DIRTY, local->dirty,
-				   sizeof(int) * AFR_NUM_CHANGE_LOGS);
-	if (ret) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-	afr_changelog_do (frame, this, xattr, afr_changelog_post_op_done,
-                          AFR_TRANSACTION_POST_OP);
 out:
-	if (xattr)
-                dict_unref (xattr);
-
-        return 0;
+    return changelog;
 }
 
+static void
+afr_ta_locked_xattrop_validate(afr_private_t *priv, afr_local_t *local,
+                               gf_boolean_t *valid)
+{
+    if (priv->ta_event_gen > local->ta_event_gen) {
+        /* We can't trust the ta's response anymore.*/
+        afr_ta_locked_priv_invalidate(priv);
+        *valid = _gf_false;
+        return;
+    }
+    return;
+}
 
-gf_boolean_t
-afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	fd_t *fd = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	afr_fd_ctx_t *fd_ctx = NULL;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-	fd = local->fd;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-	if (type != AFR_DATA_TRANSACTION)
-		return !local->transaction.dirtied;
-
-	if (!fd)
-		return !local->transaction.dirtied;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return _gf_false;
-
-	if (local->transaction.no_uninherit)
-		return _gf_false;
-
-	/* This function must be idempotent. So check if we
-	   were called before and return the same answer again.
-
-	   It is important to keep this function idempotent for
-	   the call in afr_changelog_post_op_safe() to not have
-	   side effects on the call from afr_changelog_post_op_now()
-	*/
-	if (local->transaction.uninherit_done)
-		return local->transaction.uninherit_value;
-
-	LOCK(&fd->lock);
-	{
-		for (i = 0; i < priv->child_count; i++) {
-			if (local->transaction.pre_op[i] !=
-			    fd_ctx->pre_op_done[type][i]) {
-				ret = !local->transaction.dirtied;
-				goto unlock;
-			}
-		}
-
-		if (fd_ctx->inherited[type]) {
-			ret = _gf_true;
-			fd_ctx->inherited[type]--;
-		} else if (fd_ctx->on_disk[type]) {
-			ret = _gf_false;
-			fd_ctx->on_disk[type]--;
-		} else {
-			/* ASSERT */
-			ret = _gf_false;
-		}
-
-		if (!fd_ctx->inherited[type] && !fd_ctx->on_disk[type]) {
-			for (i = 0; i < priv->child_count; i++)
-				fd_ctx->pre_op_done[type][i] = 0;
-		}
-	}
-unlock:
-	UNLOCK(&fd->lock);
+static int
+afr_ta_post_op_do(void *opaque)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    dict_t *xattr = NULL;
+    unsigned char *pending = NULL;
+    int **changelog = NULL;
+    int failed_subvol = -1;
+    int success_subvol = -1;
+    loc_t loc = {
+        0,
+    };
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t valid = _gf_true;
+
+    local = (afr_local_t *)opaque;
+    this = local->transaction.frame->this;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            pending[i] = 1;
+            failed_subvol = i;
+        } else {
+            success_subvol = i;
+        }
+    }
+
+    changelog = afr_set_changelog_xattr(priv, pending, xattr, local);
+
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, &loc);
+    if (ret)
+        goto out;
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s failed for gfid %s.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+    }
+    LOCK(&priv->lock);
+    {
+        if (ret == 0) {
+            priv->ta_bad_child_index = failed_subvol;
+        } else if (ret == -EINVAL) {
+            priv->ta_bad_child_index = success_subvol;
+            ret = -EIO; /* TA failed the fop. Return EIO to application. */
+        }
 
-	local->transaction.uninherit_done = _gf_true;
-	local->transaction.uninherit_value = ret;
+        afr_ta_locked_xattrop_validate(priv, local, &valid);
+    }
+    UNLOCK(&priv->lock);
+    if (valid == _gf_false) {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s for gfid %s invalidated due "
+               "to event-gen mismatch.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+        ret = -EIO;
+    }
+
+    afr_ta_post_op_unlock(this, &loc);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
 
+    loc_wipe(&loc);
 
-gf_boolean_t
-afr_changelog_pre_op_inherit (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	fd_t *fd = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	afr_fd_ctx_t *fd_ctx = NULL;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-	fd = local->fd;
-
-	if (local->transaction.type != AFR_DATA_TRANSACTION)
-		return _gf_false;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-
-	if (!fd)
-		return _gf_false;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return _gf_false;
-
-	LOCK(&fd->lock);
-	{
-		if (!fd_ctx->on_disk[type]) {
-			/* nothing to inherit yet */
-			ret = _gf_false;
-			goto unlock;
-		}
-
-		for (i = 0; i < priv->child_count; i++) {
-			if (local->transaction.pre_op[i] !=
-			    fd_ctx->pre_op_done[type][i]) {
-				/* either inherit exactly, or don't */
-				ret = _gf_false;
-				goto unlock;
-			}
-		}
-
-		fd_ctx->inherited[type]++;
-
-		ret = _gf_true;
-
-		local->transaction.inherited = _gf_true;
-	}
-unlock:
-	UNLOCK(&fd->lock);
+    return ret;
+}
 
-	return ret;
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local)
+{
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
+
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto err;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_post_op_do, afr_ta_post_op_done,
+                       ta_frame, local);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch post-op on thin arbiter for gfid %s",
+               uuid_utoa(local->inode->gfid));
+        STACK_DESTROY(ta_frame->root);
+        goto err;
+    }
+
+    return ret;
+err:
+    afr_changelog_post_op_fail(local->transaction.frame, this, ENOMEM);
+    return ret;
 }
 
+static void
+afr_ta_set_fop_state(afr_private_t *priv, afr_local_t *local,
+                     int *on_wire_count)
+{
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Put the fop in waitq until notify dom lock is released.*/
+            local->fop_state = TA_WAIT_FOR_NOTIFY_LOCK_REL;
+            list_add_tail(&local->ta_waitq, &priv->ta_waitq);
+        } else if (priv->ta_bad_child_index == AFR_CHILD_UNKNOWN) {
+            /* Post-op on thin-arbiter to decide success/failure. */
+            local->fop_state = TA_GET_INFO_FROM_TA_FILE;
+            *on_wire_count = ++priv->ta_on_wire_txn_count;
+            if (*on_wire_count > 1) {
+                /*Avoid sending multiple on-wire post-ops on TA*/
+                list_add_tail(&local->ta_onwireq, &priv->ta_onwireq);
+            }
+        } else if (local->ta_failed_subvol == priv->ta_bad_child_index) {
+            /* Post-op on TA not needed as the fop failed on the in-memory bad
+             * brick. Just mark pending xattrs on the good data brick.*/
+            local->fop_state = TA_INFO_IN_MEMORY_SUCCESS;
+            priv->ta_in_mem_txn_count++;
+        } else {
+            /* Post-op on TA not needed as the fop succeeded only on the
+             * in-memory bad data brick and not the good one. Fail the fop.*/
+            local->fop_state = TA_INFO_IN_MEMORY_FAILED;
+            priv->ta_in_mem_txn_count++;
+        }
+    }
+    UNLOCK(&priv->lock);
+}
 
-gf_boolean_t
-afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	fd_t *fd = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-	fd = local->fd;
-
-	if (!fd)
-		return _gf_false;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return _gf_false;
-
-	if (local->transaction.inherited)
-		/* was already inherited in afr_changelog_pre_op */
-		return _gf_false;
-
-	if (!local->transaction.dirtied)
-		return _gf_false;
-
-        if (!afr_txn_nothing_failed (frame, this))
-		return _gf_false;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-
-	ret = _gf_false;
-
-	LOCK(&fd->lock);
-	{
-		if (!fd_ctx->on_disk[type]) {
-			for (i = 0; i < priv->child_count; i++)
-				fd_ctx->pre_op_done[type][i] =
-					local->transaction.pre_op[i];
-		} else {
-			for (i = 0; i < priv->child_count; i++)
-				if (fd_ctx->pre_op_done[type][i] !=
-				    local->transaction.pre_op[i]) {
-					local->transaction.no_uninherit = 1;
-					goto unlock;
-				}
-		}
-		fd_ctx->on_disk[type]++;
-
-		ret = _gf_true;
-	}
-unlock:
-	UNLOCK(&fd->lock);
+static void
+afr_ta_fill_failed_subvol(afr_private_t *priv, afr_local_t *local)
+{
+    int i = 0;
 
-	return ret;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            local->ta_failed_subvol = i;
+            break;
+        }
+    }
 }
 
-
-int
-afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int child_index = -1;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
-        priv = this->private;
-        child_index = (long) cookie;
+    local = frame->local;
+    if (local->is_new_entry == _gf_true) {
+        afr_mark_new_entry_changelog(frame, this);
+    }
+    afr_changelog_post_op_do(frame, this);
 
-	if (op_ret == -1) {
-                local->op_errno = op_errno;
-		afr_transaction_fop_failed (frame, this, child_index);
-        }
+    return;
+}
 
-        if (priv->arbiter_count == 1 && !op_ret) {
-                if (xattr)
-                        local->transaction.pre_op_xdata[child_index] =
-                                                               dict_ref (xattr);
-        }
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno)
+{
+    afr_changelog_post_op_fail(frame, this, op_errno);
 
-	call_count = afr_frame_return (frame);
+    return;
+}
 
-        if (call_count == 0)
-		local->transaction.changelog_resume (frame, this);
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int on_wire_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    afr_ta_set_fop_state(priv, local, &on_wire_count);
+
+    switch (local->fop_state) {
+        case TA_GET_INFO_FROM_TA_FILE:
+            if (on_wire_count == 1)
+                afr_ta_post_op_synctask(this, local);
+            /*else, fop is queued in ta_onwireq.*/
+            break;
+        case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+            /*Post releasing the notify lock, we will act on this queue*/
+            break;
+        case TA_INFO_IN_MEMORY_SUCCESS:
+            afr_post_op_handle_success(frame, this);
+            break;
+        case TA_INFO_IN_MEMORY_FAILED:
+            afr_post_op_handle_failure(frame, this, EIO);
+            break;
+        default:
+            break;
+    }
+    return;
+}
 
-        return 0;
+static void
+afr_handle_failure_using_thin_arbiter(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    afr_ta_fill_failed_subvol(priv, local);
+    gf_msg_debug(this->name, 0,
+                 "Fop failed on data brick (%s) for gfid=%s. "
+                 "ta info needed to decide fop result.",
+                 priv->children[local->ta_failed_subvol]->name,
+                 uuid_utoa(local->inode->gfid));
+    afr_ta_decide_post_op_state(frame, this);
 }
 
 void
-afr_changelog_populate_xdata (call_frame_t *frame, afr_xattrop_type_t op,
-                              dict_t **xdata, dict_t **newloc_xdata)
-{
-        dict_t      *xdata1 = NULL;
-        dict_t      *xdata2 = NULL;
-        afr_local_t *local  = NULL;
-        afr_private_t *priv = NULL;
-        int         ret     = 0;
-        const char  *name   = NULL;
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    dict_t *xattr = NULL;
+    int i = 0;
+    int ret = 0;
+    int idx = 0;
+    int nothing_failed = 1;
+    gf_boolean_t need_undirty = _gf_false;
+
+    afr_handle_quorum(frame, this);
+    local = frame->local;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    xattr = dict_new();
+    if (!xattr) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    nothing_failed = afr_txn_nothing_failed(frame, this);
+
+    if (afr_changelog_pre_op_uninherit(frame, this))
+        need_undirty = _gf_false;
+    else
+        need_undirty = _gf_true;
+
+    if (local->op_ret < 0 && !nothing_failed) {
+        if (afr_need_dirty_marking(frame, this)) {
+            local->dirty[idx] = hton32(1);
+            goto set_dirty;
+        }
 
-        local = frame->local;
-        priv = THIS->private;
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (nothing_failed && !need_undirty) {
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (local->transaction.in_flight_sb) {
+        afr_changelog_post_op_fail(frame, this,
+                                   local->transaction.in_flight_sb_errno);
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->pending[i][idx] = hton32(1);
+    }
+
+    ret = afr_set_pending_dict(priv, xattr, local->pending);
+    if (ret < 0) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    if (need_undirty)
+        local->dirty[idx] = hton32(-1);
+    else
+        local->dirty[idx] = hton32(0);
+
+set_dirty:
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, local->dirty,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    afr_changelog_do(frame, this, xattr, afr_changelog_post_op_done,
+                     AFR_TRANSACTION_POST_OP);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-        /*Populate xdata for POST_OP only.*/
-        if (op == AFR_TRANSACTION_PRE_OP)
-                goto out;
-        if (local->transaction.type == AFR_DATA_TRANSACTION ||
-            local->transaction.type == AFR_METADATA_TRANSACTION)
-                goto out;
+    return;
+}
 
-        if (!priv->esh_granular)
-                goto out;
+static int
+afr_changelog_post_op_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int failed_count = 0;
 
-        xdata1 = dict_new();
-        if (!xdata1)
-                goto out;
-        name = local->loc.name;
-        if (local->op == GF_FOP_LINK)
-                name = local->newloc.name;
-        ret = dict_set_str (xdata1, GF_XATTROP_ENTRY_IN_KEY, (char *)name);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
-                        "%s/%s: Could not set xattrop-entry key during post-op",
-                        uuid_utoa (local->loc.pargfid), local->loc.name);
-        if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                xdata2 = dict_new();
-                if (!xdata2)
-                        goto out;
-                ret = dict_set_str (xdata2, GF_XATTROP_ENTRY_IN_KEY,
-                                    (char *)local->newloc.name);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_DICT_SET_FAILED,
-                                "%s/%s: Could not set xattrop-entry key during"
-                                " post-op", uuid_utoa (local->newloc.pargfid),
-                                local->newloc.name);
+    priv = this->private;
+    local = frame->local;
+
+    if (priv->thin_arbiter_count) {
+        failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                                 priv->child_count);
+        if (failed_count == 1) {
+            afr_handle_failure_using_thin_arbiter(frame, this);
+            return 0;
+        } else {
+            /* Txn either succeeded or failed on both data bricks. Let
+             * post_op_do handle it as the case might be. */
         }
+    }
 
-        *xdata = xdata1;
-        *newloc_xdata = xdata2;
-        xdata1 = xdata2 = NULL;
-out:
-        if (xdata1)
-                dict_unref (xdata1);
-        if (xdata2)
-                dict_unref (xdata2);
-        return;
+    afr_changelog_post_op_do(frame, this);
+    return 0;
 }
 
-int
-afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
-		  afr_changelog_resume_t changelog_resume,
-                  afr_xattrop_type_t op)
+gf_boolean_t
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        dict_t *xdata = NULL;
-        dict_t *newloc_xdata = NULL;
-	int i = 0;
-	int call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
+    ctx = local->inode_ctx;
 
-        call_count = afr_changelog_call_count (local->transaction.type,
-					       local->transaction.pre_op,
-					       priv->child_count);
+    type = afr_index_for_transaction_type(local->transaction.type);
+    if (type != AFR_DATA_TRANSACTION)
+        return !local->transaction.dirtied;
 
-	if (call_count == 0) {
-		changelog_resume (frame, this);
-		return 0;
-	}
+    if (local->transaction.no_uninherit)
+        return _gf_false;
 
-        afr_changelog_populate_xdata (frame, op, &xdata, &newloc_xdata);
-	local->call_count = call_count;
+    /* This function must be idempotent. So check if we
+       were called before and return the same answer again.
 
-	local->transaction.changelog_resume = changelog_resume;
+       It is important to keep this function idempotent for
+       the call in afr_changelog_post_op_safe() to not have
+       side effects on the call from afr_changelog_post_op_now()
+    */
+    if (local->transaction.uninherit_done)
+        return local->transaction.uninherit_value;
 
+    LOCK(&local->inode->lock);
+    {
         for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
-
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                case AFR_METADATA_TRANSACTION:
-                        if (!local->fd) {
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->xattrop,
-						   &local->loc,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-                        } else {
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->fxattrop,
-						   local->fd,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-                        }
-			break;
-                case AFR_ENTRY_RENAME_TRANSACTION:
-
-			STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->xattrop,
-					   &local->transaction.new_parent_loc,
-					   GF_XATTROP_ADD_ARRAY, xattr,
-					   newloc_xdata);
-                        call_count--;
+            if (local->transaction.pre_op[i] != ctx->pre_op_done[type][i]) {
+                ret = !local->transaction.dirtied;
+                goto unlock;
+            }
+        }
 
-                /* fall through */
+        if (ctx->inherited[type]) {
+            ret = _gf_true;
+            ctx->inherited[type]--;
+        } else if (ctx->on_disk[type]) {
+            ret = _gf_false;
+            ctx->on_disk[type]--;
+        } else {
+            /* ASSERT */
+            ret = _gf_false;
+        }
 
-                case AFR_ENTRY_TRANSACTION:
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->fxattrop,
-						   local->fd,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-                        else
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->xattrop,
-						   &local->transaction.parent_loc,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-			break;
-		}
-
-                if (!--call_count)
-                        break;
+        if (!ctx->inherited[type] && !ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                ctx->pre_op_done[type][i] = 0;
         }
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
+
+    local->transaction.uninherit_done = _gf_true;
+    local->transaction.uninherit_value = ret;
 
-        if (xdata)
-                dict_unref (xdata);
-        if (newloc_xdata)
-                dict_unref (newloc_xdata);
-	return 0;
+    return ret;
 }
 
+gf_boolean_t
+afr_changelog_pre_op_inherit(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-int
-afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = this->private;
-        int i = 0;
-        int ret = 0;
-        int call_count = 0;
-	int op_errno = 0;
-        afr_local_t *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        unsigned char       *locked_nodes = NULL;
-	int idx = -1;
-	gf_boolean_t pre_nop = _gf_true;
-	dict_t *xdata_req = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-	idx = afr_index_for_transaction_type (local->transaction.type);
-
-        locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (locked_nodes[i]) {
-			local->transaction.pre_op[i] = 1;
-			call_count++;
-		} else {
-                        local->transaction.failed_subvols[i] = 1;
-                }
-	}
+    local = frame->local;
+    priv = this->private;
 
-        /* This condition should not be met with present code, as
-         * transaction.done will be called if locks are not acquired on even a
-         * single node.
-         */
-        if (call_count == 0) {
-		op_errno = ENOTCONN;
-		goto err;
-	}
+    if (local->transaction.type != AFR_DATA_TRANSACTION)
+        return _gf_false;
 
-        /* Check if the fop can be performed on at least
-         * quorum number of nodes.
-         */
-        if (priv->quorum_count && !afr_has_fop_quorum (frame)) {
-                op_errno = afr_quorum_errno (priv);
-                goto err;
+    type = afr_index_for_transaction_type(local->transaction.type);
+
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            /* nothing to inherit yet */
+            ret = _gf_false;
+            goto unlock;
         }
 
-	xdata_req = dict_new();
-	if (!xdata_req) {
-		op_errno = ENOMEM;
-		goto err;
-	}
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.pre_op[i] !=
+                local->inode_ctx->pre_op_done[type][i]) {
+                /* either inherit exactly, or don't */
+                ret = _gf_false;
+                goto unlock;
+            }
+        }
 
-	if (afr_changelog_pre_op_inherit (frame, this))
-		goto next;
+        local->inode_ctx->inherited[type]++;
 
-        if (call_count < priv->child_count)
-                pre_nop = _gf_false;
+        ret = _gf_true;
 
-        /* Set an all-zero pending changelog so that in the cbk, we can get the
-         * current on-disk values. In a replica 3 volume with arbiter enabled,
-         * these values are needed to arrive at a go/ no-go of the fop phase to
-         * avoid ending up in split-brain.*/
+        local->transaction.inherited = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-        ret = afr_set_pending_dict (priv, xdata_req, local->pending);
-	if (ret < 0) {
-		op_errno = ENOMEM;
-		goto err;
-	}
+    return ret;
+}
 
-	if (afr_needs_changelog_update (local)) {
+gf_boolean_t
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-		local->dirty[idx] = hton32(1);
+    local = frame->local;
+    priv = this->private;
 
-		ret = dict_set_static_bin (xdata_req, AFR_DIRTY, local->dirty,
-					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
-		if (ret) {
-			op_errno = ENOMEM;
-			goto err;
-		}
+    if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+        local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION)
+        return _gf_false;
 
-		pre_nop = _gf_false;
-		local->transaction.dirtied = 1;
-	}
+    if (local->transaction.inherited)
+        /* was already inherited in afr_changelog_pre_op */
+        return _gf_false;
 
-	if (pre_nop)
-		goto next;
+    if (!local->transaction.dirtied)
+        return _gf_false;
 
-	if (!local->pre_op_compat) {
-		dict_copy (xdata_req, local->xdata_req);
-		goto next;
-	}
+    if (!afr_txn_nothing_failed(frame, this))
+        return _gf_false;
 
-	afr_changelog_do (frame, this, xdata_req, afr_transaction_perform_fop,
-                          AFR_TRANSACTION_PRE_OP);
+    type = afr_index_for_transaction_type(local->transaction.type);
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+    ret = _gf_false;
 
-	return 0;
-next:
-	afr_transaction_perform_fop (frame, this);
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                local->inode_ctx->pre_op_done[type][i] =
+                    (!local->transaction.failed_subvols[i]);
+        } else {
+            for (i = 0; i < priv->child_count; i++)
+                if (local->inode_ctx->pre_op_done[type][i] !=
+                    (!local->transaction.failed_subvols[i])) {
+                    local->transaction.no_uninherit = 1;
+                    goto unlock;
+                }
+        }
+        local->inode_ctx->on_disk[type]++;
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+        ret = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-        return 0;
-err:
-	local->internal_lock.lock_cbk = local->transaction.done;
-	local->op_ret = -1;
-	local->op_errno = op_errno;
+    return ret;
+}
 
-	afr_unlock (frame, this);
+int
+afr_changelog_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+    local = frame->local;
+    child_index = (long)cookie;
 
-	return 0;
-}
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
+    if (xattr)
+        local->transaction.changelog_xdata[child_index] = dict_ref(xattr);
 
-int
-afr_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    call_count = afr_frame_return(frame);
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    if (call_count == 0) {
+        local->transaction.changelog_resume(frame, this);
+    }
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO,
-                        0, AFR_MSG_BLOCKING_LKS_FAILED,
-                        "Blocking inodelks failed.");
-                local->transaction.done (frame, this);
-        } else {
+    return 0;
+}
+
+void
+afr_changelog_populate_xdata(call_frame_t *frame, afr_xattrop_type_t op,
+                             dict_t **xdata, dict_t **newloc_xdata)
+{
+    int i = 0;
+    int ret = 0;
+    char *key = NULL;
+    int keylen = 0;
+    const char *name = NULL;
+    dict_t *xdata1 = NULL;
+    dict_t *xdata2 = NULL;
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_entry_key_set = _gf_true;
+
+    local = frame->local;
+    this = THIS;
+    priv = this->private;
+
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        goto out;
+
+    if (!priv->esh_granular)
+        goto out;
+
+    xdata1 = dict_new();
+    if (!xdata1)
+        goto out;
+
+    name = local->loc.name;
+    if (local->op == GF_FOP_LINK)
+        name = local->newloc.name;
+
+    switch (op) {
+        case AFR_TRANSACTION_PRE_OP:
+            key = GF_XATTROP_ENTRY_IN_KEY;
+            break;
+        case AFR_TRANSACTION_POST_OP:
+            if (afr_txn_nothing_failed(frame, this)) {
+                key = GF_XATTROP_ENTRY_OUT_KEY;
+                for (i = 0; i < priv->child_count; i++) {
+                    if (!local->transaction.failed_subvols[i])
+                        continue;
+                    need_entry_key_set = _gf_false;
+                    break;
+                }
+                /* If the transaction itself did not fail and there
+                 * are no failed subvolumes, check whether the fop
+                 * failed due to a symmetric error. If it did, do
+                 * not set the ENTRY_OUT xattr which would end up
+                 * deleting a name index which was created possibly by
+                 * an earlier entry txn that may have failed on some
+                 * of the sub-volumes.
+                 */
+                if (local->op_ret)
+                    need_entry_key_set = _gf_false;
+            } else {
+                key = GF_XATTROP_ENTRY_IN_KEY;
+            }
+            break;
+    }
+
+    if (need_entry_key_set) {
+        keylen = strlen(key);
+        ret = dict_set_strn(xdata1, key, keylen, (char *)name);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "%s/%s: Could not set %s key during xattrop",
+                   uuid_utoa(local->loc.pargfid), local->loc.name, key);
+        if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+            xdata2 = dict_new();
+            if (!xdata2)
+                goto out;
 
-                gf_msg_debug (this->name, 0,
-                              "Blocking inodelks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
+            ret = dict_set_strn(xdata2, key, keylen,
+                                (char *)local->newloc.name);
+            if (ret)
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "%s/%s: Could not set %s key during "
+                       "xattrop",
+                       uuid_utoa(local->newloc.pargfid), local->newloc.name,
+                       key);
         }
+    }
 
-        return 0;
+    *xdata = xdata1;
+    *newloc_xdata = xdata2;
+    xdata1 = xdata2 = NULL;
+out:
+    if (xdata1)
+        dict_unref(xdata1);
+    return;
 }
 
-
 int
-afr_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+afr_changelog_prepare(xlator_t *this, call_frame_t *frame, int *call_count,
+                      afr_changelog_resume_t changelog_resume,
+                      afr_xattrop_type_t op, dict_t **xdata,
+                      dict_t **newloc_xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
 
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Non blocking inodelks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
+    *call_count = afr_changelog_call_count(
+        local->transaction.type, local->transaction.pre_op,
+        local->transaction.failed_subvols, priv->child_count);
 
-                gf_msg_debug (this->name, 0,
-                              "Non blocking inodelks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
+    if (*call_count == 0) {
+        changelog_resume(frame, this);
+        return -1;
+    }
+
+    afr_changelog_populate_xdata(frame, op, xdata, newloc_xdata);
+    local->call_count = *call_count;
+
+    local->transaction.changelog_resume = changelog_resume;
+    return 0;
+}
+
+int
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume, afr_xattrop_type_t op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    dict_t *newloc_xdata = NULL;
+    int i = 0;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.changelog_xdata[i]) {
+            dict_unref(local->transaction.changelog_xdata[i]);
+            local->transaction.changelog_xdata[i] = NULL;
         }
+    }
 
+    ret = afr_changelog_prepare(this, frame, &call_count, changelog_resume, op,
+                                &xdata, &newloc_xdata);
+
+    if (ret)
         return 0;
-}
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i] ||
+            local->transaction.failed_subvols[i])
+            continue;
 
-int
-afr_post_blocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+        switch (local->transaction.type) {
+            case AFR_DATA_TRANSACTION:
+            case AFR_METADATA_TRANSACTION:
+                if (!local->fd) {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->xattrop,
+                        &local->loc, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                } else {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                }
+                break;
+            case AFR_ENTRY_RENAME_TRANSACTION:
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+                STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->xattrop,
+                                  &local->transaction.new_parent_loc,
+                                  GF_XATTROP_ADD_ARRAY, xattr, newloc_xdata);
+                call_count--;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        AFR_MSG_BLOCKING_LKS_FAILED,
-                        "Blocking entrylks failed.");
-                local->transaction.done (frame, this);
-        } else {
+                /* fall through */
 
-                gf_msg_debug (this->name, 0,
-                             "Blocking entrylks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
+            case AFR_ENTRY_TRANSACTION:
+                if (local->fd)
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                else
+                    STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                      priv->children[i],
+                                      priv->children[i]->fops->xattrop,
+                                      &local->transaction.parent_loc,
+                                      GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                break;
         }
 
-        return 0;
-}
+        if (!--call_count)
+            break;
+    }
 
+    if (xdata)
+        dict_unref(xdata);
+    if (newloc_xdata)
+        dict_unref(newloc_xdata);
+    return 0;
+}
 
-int
-afr_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
+static void
+afr_init_optimistic_changelog_for_txn(xlator_t *this, afr_local_t *local)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Non blocking entrylks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_post_blocking_entrylk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
 
-                gf_msg_debug (this->name, 0,
-                              "Non blocking entrylks done. Proceeding to FOP");
+    priv = this->private;
 
-                afr_internal_lock_finish (frame, this);
-        }
+    locked_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    if (priv->optimistic_change_log && locked_count == priv->child_count)
+        local->optimistic_change_log = 1;
 
-        return 0;
+    return;
 }
 
-
 int
-afr_post_blocking_rename_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+afr_changelog_pre_op(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    int i = 0;
+    int ret = 0;
+    int call_count = 0;
+    int op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    unsigned char *locked_nodes = NULL;
+    int idx = -1;
+    gf_boolean_t pre_nop = _gf_true;
+    dict_t *xdata_req = NULL;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    locked_nodes = afr_locked_nodes_get(local->transaction.type, int_lock);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_nodes[i]) {
+            local->transaction.pre_op[i] = 1;
+            call_count++;
+        } else {
+            local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    afr_init_optimistic_changelog_for_txn(this, local);
+
+    if (afr_changelog_pre_op_inherit(frame, this))
+        goto next;
+
+    /* This condition should not be met with present code, as
+     * transaction.done will be called if locks are not acquired on even a
+     * single node.
+     */
+    if (call_count == 0) {
+        op_errno = ENOTCONN;
+        goto err;
+    }
+
+    /* Check if the fop can be performed on at least
+     * quorum number of nodes.
+     */
+    if (priv->quorum_count && !afr_has_fop_quorum(frame)) {
+        op_errno = int_lock->lock_op_errno;
+        if (op_errno == 0)
+            op_errno = afr_quorum_errno(priv);
+        goto err;
+    }
+
+    xdata_req = dict_new();
+    if (!xdata_req) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (call_count < priv->child_count)
+        pre_nop = _gf_false;
+
+    /* Set an all-zero pending changelog so that in the cbk, we can get the
+     * current on-disk values. In a replica 3 volume with arbiter enabled,
+     * these values are needed to arrive at a go/ no-go of the fop phase to
+     * avoid ending up in split-brain.*/
+
+    ret = afr_set_pending_dict(priv, xdata_req, local->pending);
+    if (ret < 0) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (afr_needs_changelog_update(local)) {
+        local->dirty[idx] = hton32(1);
+
+        ret = dict_set_static_bin(xdata_req, AFR_DIRTY, local->dirty,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto err;
+        }
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+        pre_nop = _gf_false;
+        local->transaction.dirtied = 1;
+    }
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        AFR_MSG_BLOCKING_LKS_FAILED,
-                        "Blocking entrylks failed.");
+    if (pre_nop)
+        goto next;
 
-                local->transaction.done (frame, this);
-        } else {
+    if (!local->pre_op_compat) {
+        dict_copy(xdata_req, local->xdata_req);
+        goto next;
+    }
 
-                gf_msg_debug (this->name, 0,
-                              "Blocking entrylks done. Proceeding to FOP");
+    afr_changelog_do(frame, this, xdata_req, afr_transaction_perform_fop,
+                     AFR_TRANSACTION_PRE_OP);
 
-                afr_internal_lock_finish (frame, this);
-        }
-        return 0;
-}
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-int
-afr_post_lower_unlock_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    return 0;
+next:
+    afr_transaction_perform_fop(frame, this);
+
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    return 0;
+err:
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-        GF_ASSERT (!int_lock->higher_locked);
+    afr_handle_lock_acquire_failure(local);
 
-        int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-        afr_blocking_lock (frame, this);
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_set_transaction_flock (xlator_t *this, afr_local_t *local)
+afr_post_nonblocking_lock_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_private_t       *priv     = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        int_lock = &local->internal_lock;
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-        priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        if (priv->arbiter_count) {
-                /*Lock entire file to avoid network split brains.*/
-                inodelk->flock.l_len   = 0;
-                inodelk->flock.l_start = 0;
-        } else {
-                inodelk->flock.l_len   = local->transaction.len;
-                inodelk->flock.l_start = local->transaction.start;
-        }
-        inodelk->flock.l_type  = F_WRLCK;
+    /* Initiate blocking locks if non-blocking has failed */
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks failed. Proceeding to blocking");
+        int_lock->lock_cbk = afr_internal_lock_finish;
+        afr_blocking_lock(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks done. Proceeding to FOP");
 
-        return 0;
+        afr_internal_lock_finish(frame, this);
+    }
+
+    return 0;
 }
 
 int
-afr_lock_rec (call_frame_t *frame, xlator_t *this)
+afr_post_blocking_rename_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
-        int_lock->domain = this->name;
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INTERNAL_LKS_FAILED,
+               "Blocking entrylks failed.");
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                afr_set_transaction_flock (this, local);
+        afr_transaction_done(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Blocking entrylks done. Proceeding to FOP");
 
-                int_lock->lock_cbk = afr_post_nonblocking_inodelk_cbk;
-
-                afr_nonblocking_inodelk (frame, this);
-                break;
+        afr_internal_lock_finish(frame, this);
+    }
+    return 0;
+}
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
+int
+afr_post_lower_unlock_cbk(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
-                afr_nonblocking_entrylk (frame, this);
-                break;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        case AFR_ENTRY_TRANSACTION:
-                int_lock->lk_basename = local->transaction.basename;
-                if (local->transaction.parent_loc.path)
-                        int_lock->lk_loc = &local->transaction.parent_loc;
-                else
-                        GF_ASSERT (local->fd);
+    GF_ASSERT(!int_lock->higher_locked);
 
-                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
-                afr_nonblocking_entrylk (frame, this);
-                break;
-        }
+    int_lock->lock_cbk = afr_post_blocking_rename_cbk;
+    afr_blocking_lock(frame, this);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_lock (call_frame_t *frame, xlator_t *this)
+afr_set_transaction_flock(xlator_t *this, afr_local_t *local,
+                          afr_lockee_t *lockee)
 {
-        afr_set_lock_number (frame, this);
+    afr_private_t *priv = NULL;
+    struct gf_flock *flock = NULL;
 
-        return afr_lock_rec (frame, this);
-}
+    priv = this->private;
+    flock = &lockee->flock;
 
+    if ((priv->arbiter_count || local->transaction.eager_lock_on ||
+         priv->full_lock) &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        /*Lock entire file to avoid network split brains.*/
+        flock->l_len = 0;
+        flock->l_start = 0;
+    } else {
+        flock->l_len = local->transaction.len;
+        flock->l_start = local->transaction.start;
+    }
+    flock->l_type = F_WRLCK;
 
-/* }}} */
+    return 0;
+}
 
 int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
+afr_lock(call_frame_t *frame, xlator_t *this)
 {
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_pre_op (frame, this);
-        } else {
-                afr_transaction_perform_fop (frame, this);
-        }
-
-        return 0;
-}
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-void
-afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t    *local = NULL;
-        afr_private_t  *priv = NULL;
+    int_lock->lock_cbk = afr_post_nonblocking_lock_cbk;
+    int_lock->domain = this->name;
 
-        /* call this function from any of the related optimizations
-           which benefit from delaying post op are enabled, namely:
+    switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            for (i = 0; i < int_lock->lockee_count; i++) {
+                afr_set_transaction_flock(this, local, &int_lock->lockee[i]);
+            }
 
-           - changelog piggybacking
-           - eager locking
-        */
+            break;
 
-        priv = this->private;
-        if (!priv)
-                return;
+        case AFR_ENTRY_TRANSACTION:
+            int_lock->lk_basename = local->transaction.basename;
+            if (local->transaction.parent_loc.path)
+                int_lock->lk_loc = &local->transaction.parent_loc;
+            else
+                GF_ASSERT(local->fd);
+            break;
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            break;
+    }
+    afr_lock_nonblocking(frame, this);
 
-        if (!priv->post_op_delay_secs)
-                return;
+    return 0;
+}
 
-        local = frame->local;
-        if (!local)
-                return;
+static gf_boolean_t
+afr_locals_overlap(afr_local_t *local1, afr_local_t *local2)
+{
+    uint64_t start1 = local1->transaction.start;
+    uint64_t start2 = local2->transaction.start;
+    uint64_t end1 = 0;
+    uint64_t end2 = 0;
 
-        if (!local->transaction.eager_lock_on)
-                return;
+    if (local1->transaction.len)
+        end1 = start1 + local1->transaction.len - 1;
+    else
+        end1 = ULLONG_MAX;
 
-        if (!local->fd)
-                return;
+    if (local2->transaction.len)
+        end2 = start2 + local2->transaction.len - 1;
+    else
+        end2 = ULLONG_MAX;
 
-        if (local->op == GF_FOP_WRITE)
-                local->delayed_post_op = _gf_true;
+    return ((end1 >= start2) && (end2 >= start1));
 }
 
 gf_boolean_t
-afr_are_multiple_fds_opened (fd_t *fd, xlator_t *this)
-{
-        afr_fd_ctx_t *fd_ctx = NULL;
+afr_has_lock_conflict(afr_local_t *local, gf_boolean_t waitlist_check)
+{
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = NULL;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    /*
+     * Once full file lock is acquired in eager-lock phase, overlapping
+     * writes do not compete for inode-locks, instead are transferred to the
+     * next writes. Because of this overlapping writes are not ordered.
+     * This can cause inconsistencies in replication.
+     * Example:
+     * Two overlapping writes w1, w2 are sent in parallel on same fd
+     * in two threads t1, t2.
+     * Both threads can execute afr_writev_wind in the following manner.
+     * t1 winds w1 on brick-0
+     * t2 winds w2 on brick-0
+     * t2 winds w2 on brick-1
+     * t1 winds w1 on brick-1
+     *
+     * This check makes sure the locks are not transferred for
+     * overlapping writes.
+     */
+    list_for_each_entry(each, &lock->owners, transaction.owner_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
+        }
+    }
 
-        if (!fd) {
-                /* If false is returned, it may keep on taking eager-lock
-                 * which may lead to starvation, so return true to avoid that.
-                 */
-                gf_msg_callingfn (this->name, GF_LOG_ERROR, EBADF,
-                                  AFR_MSG_INVALID_ARG, "Invalid fd");
-                return _gf_true;
+    if (!waitlist_check)
+        return _gf_false;
+    list_for_each_entry(each, &lock->waiting, transaction.wait_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
         }
-        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
-         * is taken mount2 opened the same file, it won't be able to
-         * perform any data operations until mount1 releases eager-lock.
-         * To avoid such scenario do not enable eager-lock for this transaction
-         * if open-fd-count is > 1
-         */
+    }
+    return _gf_false;
+}
 
-        fd_ctx = afr_fd_ctx_get (fd, this);
-        if (!fd_ctx)
-                return _gf_true;
+/* }}} */
+static void
+afr_copy_inodelk_vars(afr_internal_lock_t *dst, afr_internal_lock_t *src,
+                      xlator_t *this, int lockee_num)
+{
+    afr_private_t *priv = this->private;
+    afr_lockee_t *sl = &src->lockee[lockee_num];
+    afr_lockee_t *dl = &dst->lockee[lockee_num];
 
-        if (fd_ctx->open_fd_count > 1)
-                return _gf_true;
+    dst->domain = src->domain;
+    dl->flock.l_len = sl->flock.l_len;
+    dl->flock.l_start = sl->flock.l_start;
+    dl->flock.l_type = sl->flock.l_type;
+    dl->locked_count = sl->locked_count;
+    memcpy(dl->locked_nodes, sl->locked_nodes,
+           priv->child_count * sizeof(*dl->locked_nodes));
+}
 
-        return _gf_false;
+void
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared)
+{
+    gf_boolean_t conflict = _gf_false;
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = &local->inode_ctx->lock[local->transaction.type];
+
+    while (!conflict) {
+        if (list_empty(&lock->waiting))
+            return;
+        each = list_entry(lock->waiting.next, afr_local_t,
+                          transaction.wait_list);
+        if (afr_has_lock_conflict(each, _gf_false)) {
+            conflict = _gf_true;
+        }
+        if (conflict && !list_empty(&lock->owners))
+            return;
+        afr_copy_inodelk_vars(&each->internal_lock, &local->internal_lock,
+                              each->transaction.frame->this, 0);
+        list_move_tail(&each->transaction.wait_list, shared);
+        list_add_tail(&each->transaction.owner_list, &lock->owners);
+    }
 }
 
+static void
+afr_lock_resume_shared(struct list_head *list)
+{
+    afr_local_t *each = NULL;
 
-gf_boolean_t
-is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        afr_changelog_pre_op(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
+}
+
+int
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t      *local = NULL;
-        gf_boolean_t      res = _gf_false;
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    local->internal_lock.lock_cbk = NULL;
+    if (!local->transaction.eager_lock_on) {
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_transaction_done(frame, this);
+            return 0;
+        }
+        afr_changelog_pre_op(frame, this);
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_handle_lock_acquire_failure(local);
+        } else {
+            lock->event_generation = local->event_generation;
+            afr_changelog_pre_op(frame, this);
+        }
+    }
 
-        if (!local->delayed_post_op)
-                goto out;
+    return 0;
+}
 
-        //Mark pending changelog ASAP
-        if (!afr_txn_nothing_failed (frame, this))
-                goto out;
+gf_boolean_t
+afr_are_conflicting_ops_waiting(afr_local_t *local, xlator_t *this)
+{
+    afr_lock_t *lock = NULL;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+
+    /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+     * is taken mount2 opened the same file, it won't be able to
+     * perform any {meta,}data operations until mount1 releases eager-lock.
+     * To avoid such scenario do not enable eager-lock for this transaction
+     * if open-fd-count is > 1 for metadata transactions and if num-inodelks > 1
+     * for data transactions
+     */
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        if (local->inode_ctx->open_fd_count > 1) {
+            return _gf_true;
+        }
+    } else if (local->transaction.type == AFR_DATA_TRANSACTION) {
+        if (lock->num_inodelks > 1) {
+            return _gf_true;
+        }
+    }
 
-        if (local->fd && afr_are_multiple_fds_opened (local->fd, this))
-                goto out;
+    return _gf_false;
+}
 
-        res = _gf_true;
+gf_boolean_t
+afr_is_delayed_changelog_post_op_needed(call_frame_t *frame, xlator_t *this,
+                                        int delay)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t res = _gf_false;
+
+    local = frame->local;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+
+    if (!afr_txn_nothing_failed(frame, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (afr_are_conflicting_ops_waiting(local, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (!list_empty(&lock->owners))
+        goto out;
+    else
+        GF_ASSERT(list_empty(&lock->waiting));
+
+    if (lock->release) {
+        goto out;
+    }
+
+    if (!delay) {
+        goto out;
+    }
+
+    if (local->transaction.disable_delayed_post_op) {
+        goto out;
+    }
+
+    if ((local->op != GF_FOP_WRITE) && (local->op != GF_FOP_FXATTROP) &&
+        (local->op != GF_FOP_FSYNC)) {
+        /*Only allow writes/fsyncs but shard does [f]xattrops on writes, so
+         * they are fine too*/
+        goto out;
+    }
+
+    res = _gf_true;
 out:
-        return res;
+    return res;
 }
 
-
 void
-afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
-                               call_stub_t *stub);
-
-void
-afr_delayed_changelog_wake_up_cbk (void *data)
-{
-        fd_t           *fd = NULL;
-
-        fd = data;
-
-        afr_delayed_changelog_wake_up (THIS, fd);
+afr_delayed_changelog_wake_up_cbk(void *data)
+{
+    afr_lock_t *lock = NULL;
+    afr_local_t *local = data;
+    afr_local_t *timer_local = NULL;
+    struct list_head shared;
+
+    INIT_LIST_HEAD(&shared);
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                 transaction.owner_list);
+        if (list_empty(&lock->owners) && (local == timer_local)) {
+            GF_ASSERT(list_empty(&lock->waiting));
+            /*Last owner*/
+            lock->release = _gf_true;
+            lock->delay_timer = NULL;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    afr_changelog_post_op_now(local->transaction.frame,
+                              local->transaction.frame->this);
 }
 
-
 /* SET operation */
 int
-afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local)
 {
-        afr_fd_ctx_t *fdctx = NULL;
-
-        fdctx = afr_fd_ctx_get (fd, this);
+    LOCK(&local->inode->lock);
+    {
+        local->inode_ctx->witnessed_unstable_write = _gf_true;
+    }
+    UNLOCK(&local->inode->lock);
 
-        LOCK(&fd->lock);
-        {
-                fdctx->witnessed_unstable_write = _gf_true;
-        }
-        UNLOCK(&fd->lock);
-
-        return 0;
+    return 0;
 }
 
 /* TEST and CLEAR operation */
 gf_boolean_t
-afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode)
 {
-        afr_fd_ctx_t *fdctx = NULL;
-        gf_boolean_t witness = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    gf_boolean_t witness = _gf_false;
 
-        fdctx = afr_fd_ctx_get (fd, this);
-        if (!fdctx)
-                return _gf_true;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
 
-        LOCK(&fd->lock);
-        {
-                if (fdctx->witnessed_unstable_write) {
-                        witness = _gf_true;
-                        fdctx->witnessed_unstable_write = _gf_false;
-                }
+        if (ctx->witnessed_unstable_write) {
+            witness = _gf_true;
+            ctx->witnessed_unstable_write = _gf_false;
         }
-        UNLOCK (&fd->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        return witness;
+    return witness;
 }
 
-
 int
-afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *pre,
-                         struct iatt *post, dict_t *xdata)
+afr_changelog_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *pre,
+                        struct iatt *post, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        if (op_ret != 0) {
-                /* Failure of fsync() is as good as failure of previous
-                   write(). So treat it like one.
-		*/
-                gf_msg (this->name, GF_LOG_WARNING,
-                        op_errno, AFR_MSG_FSYNC_FAILED,
-                        "fsync(%s) failed on subvolume %s. Transaction was %s",
-                        uuid_utoa (local->fd->inode->gfid),
-                        priv->children[child_index]->name,
-                        gf_fop_list[local->op]);
+    if (op_ret != 0) {
+        /* Failure of fsync() is as good as failure of previous
+           write(). So treat it like one.
+        */
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, AFR_MSG_FSYNC_FAILED,
+               "fsync(%s) failed on subvolume %s. Transaction was %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name, gf_fop_list[local->op]);
 
-                afr_transaction_fop_failed (frame, this, child_index);
-	}
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-	call_count = afr_frame_return (frame);
+    call_count = afr_frame_return(frame);
 
-	if (call_count == 0)
-		afr_changelog_post_op_now (frame, this);
+    if (call_count == 0)
+        afr_changelog_post_op_now(frame, this);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+afr_changelog_fsync(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int i = 0;
-        int call_count = 0;
-        afr_private_t *priv = NULL;
-        dict_t *xdata = NULL;
-        GF_UNUSED int ret = -1;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    GF_UNUSED int ret = -1;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+    call_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
 
-        if (!call_count) {
-                /* will go straight to unlock */
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
+    if (!call_count) {
+        /* will go straight to unlock */
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        xdata = dict_new();
-        if (xdata)
-                ret = dict_set_int32 (xdata, "batch-fsync", 1);
+    xdata = dict_new();
+    if (xdata) {
+        ret = dict_set_int32_sizen(xdata, "batch-fsync", 1);
+        ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i])
+            continue;
 
-                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
-				   (void *) (long) i, priv->children[i],
-				   priv->children[i]->fops->fsync, local->fd,
-				   1, xdata);
-                if (!--call_count)
-                        break;
-        }
+        STACK_WIND_COOKIE(frame, afr_changelog_fsync_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->fsync,
+                          local->fd, 1, xdata);
+        if (!--call_count)
+            break;
+    }
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t    *local = NULL;
-        afr_private_t  *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
-
-        if (afr_changelog_pre_op_uninherit (frame, this) &&
-	    afr_txn_nothing_failed (frame, this)) {
-                /* just detected that this post-op is about to
-                   be optimized away as a new write() has
-                   already piggybacked on this frame's changelog.
-                   */
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
+    if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-        /* Calling afr_changelog_post_op_now() now will result in
-           issuing ->[f]xattrop().
-
-           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
-           responsible operation that what it might appear on the surface.
-
-           The changelog of a file (in the xattr of the file on the server)
-           stores information (pending count) about the state of the file
-           on the OTHER server. This changelog is blindly trusted, and must
-           therefore be updated in such a way it remains trustworthy. This
-           implies that decrementing the pending count (essentially "clearing
-           the dirty flag") must be done STRICTLY after we are sure that the
-           operation on the other server has reached stable storage.
-
-           While the backend filesystem on that server will eventually flush
-           it to stable storage, we (being in userspace) have no mechanism
-           to get notified when the write became "stable".
-
-           This means we need take matter into our own hands and issue an
-           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
-           and get an acknowledgement for it. And we need to wait for the
-           fsync() acknowledgement before initiating the hard POST-OP.
-
-           However if the FD itself was opened in O_SYNC or O_DSYNC then
-           we are already guaranteed that the writes were made stable as
-           part of the FOP itself. The same holds true for NFS stable
-           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
-           flag set in the writev() @flags param. For all other write types,
-           mark a flag in the fdctx whenever an unstable write is witnessed.
+    if (afr_changelog_pre_op_uninherit(frame, this) &&
+        afr_txn_nothing_failed(frame, this)) {
+        /* just detected that this post-op is about to
+           be optimized away as a new write() has
+           already piggybacked on this frame's changelog.
            */
-
-        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
-
-        /* Check whether users want durability and perform fsync/post-op
-         * accordingly.
-         */
-        if (priv->ensure_durability) {
-                /* Time to fsync() */
-                afr_changelog_fsync (frame, this);
-        } else {
-                afr_changelog_post_op_now (frame, this);
-        }
-
+        afr_changelog_post_op_now(frame, this);
         return 0;
-}
-
-
-void
-afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
-                               call_stub_t *stub)
-{
-	afr_fd_ctx_t      *fd_ctx = NULL;
-	call_frame_t      *prev_frame = NULL;
-	struct timespec    delta = {0, };
-	afr_private_t     *priv = NULL;
-	afr_local_t       *local = NULL;
-
-	priv = this->private;
+    }
+
+    /* Calling afr_changelog_post_op_now() now will result in
+       issuing ->[f]xattrop().
+
+       Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+       responsible operation that what it might appear on the surface.
+
+       The changelog of a file (in the xattr of the file on the server)
+       stores information (pending count) about the state of the file
+       on the OTHER server. This changelog is blindly trusted, and must
+       therefore be updated in such a way it remains trustworthy. This
+       implies that decrementing the pending count (essentially "clearing
+       the dirty flag") must be done STRICTLY after we are sure that the
+       operation on the other server has reached stable storage.
+
+       While the backend filesystem on that server will eventually flush
+       it to stable storage, we (being in userspace) have no mechanism
+       to get notified when the write became "stable".
+
+       This means we need take matter into our own hands and issue an
+       fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+       and get an acknowledgement for it. And we need to wait for the
+       fsync() acknowledgement before initiating the hard POST-OP.
+
+       However if the FD itself was opened in O_SYNC or O_DSYNC then
+       we are already guaranteed that the writes were made stable as
+       part of the FOP itself. The same holds true for NFS stable
+       writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+       flag set in the writev() @flags param. For all other write types,
+       mark a flag in the fdctx whenever an unstable write is witnessed.
+       */
+
+    if (!afr_fd_has_witnessed_unstable_write(this, local->inode)) {
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-                goto out;
+    /* Check whether users want durability and perform fsync/post-op
+     * accordingly.
+     */
+    if (priv->ensure_durability) {
+        /* Time to fsync() */
+        afr_changelog_fsync(frame, this);
+    } else {
+        afr_changelog_post_op_now(frame, this);
+    }
 
-	delta.tv_sec = priv->post_op_delay_secs;
-	delta.tv_nsec = 0;
-
-	pthread_mutex_lock (&fd_ctx->delay_lock);
-	{
-		prev_frame = fd_ctx->delay_frame;
-		fd_ctx->delay_frame = NULL;
-		if (fd_ctx->delay_timer)
-			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
-		fd_ctx->delay_timer = NULL;
-		if (!frame)
-			goto unlock;
-		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
-							   afr_delayed_changelog_wake_up_cbk,
-							   fd);
-		fd_ctx->delay_frame = frame;
-	}
-unlock:
-	pthread_mutex_unlock (&fd_ctx->delay_lock);
-
-out:
-	if (prev_frame) {
-		local = prev_frame->local;
-		local->transaction.resume_stub = stub;
-		afr_changelog_post_op_now (prev_frame, this);
-	} else if (stub) {
-		call_resume (stub);
-	}
+    return 0;
 }
 
-
 void
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
-
-        if (is_afr_delayed_changelog_post_op_needed (frame, this))
-                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
-        else
-                afr_changelog_post_op_safe (frame, this);
-}
-
-
-
-/* Wake up the sleeping/delayed post-op, and also register
-   a stub to have it resumed after this transaction
-   completely finishes.
+afr_changelog_post_op(call_frame_t *frame, xlator_t *this)
+{
+    struct timespec delta = {
+        0,
+    };
+    afr_private_t *priv = NULL;
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t post_op = _gf_true;
+    struct list_head shared;
+
+    priv = this->private;
+    delta.tv_sec = priv->post_op_delay_secs;
+    delta.tv_nsec = 0;
+
+    INIT_LIST_HEAD(&shared);
+    if (!local->transaction.eager_lock_on)
+        goto out;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        list_add(&local->transaction.owner_list, &lock->post_op);
+        __afr_transaction_wake_shared(local, &shared);
+
+        if (!afr_is_delayed_changelog_post_op_needed(frame, this,
+                                                     delta.tv_sec)) {
+            if (list_empty(&lock->owners))
+                lock->release = _gf_true;
+            goto unlock;
+        }
 
-   The @stub gets saved in @local and gets resumed in
-   afr_local_cleanup()
-   */
-void
-afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
-{
-        afr_delayed_changelog_post_op (this, NULL, fd, stub);
-}
+        GF_ASSERT(lock->delay_timer == NULL);
+        lock->delay_timer = gf_timer_call_after(
+            this->ctx, delta, afr_delayed_changelog_wake_up_cbk, local);
+        if (!lock->delay_timer) {
+            lock->release = _gf_true;
+        } else {
+            post_op = _gf_false;
+        }
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
+    if (!list_empty(&shared)) {
+        afr_lock_resume_shared(&shared);
+    }
 
-void
-afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
-{
-        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+out:
+    if (post_op) {
+        if (!local->transaction.eager_lock_on || lock->release) {
+            afr_changelog_post_op_safe(frame, this);
+        } else {
+            afr_changelog_post_op_now(frame, this);
+        }
+    }
 }
 
-
 int
-afr_transaction_resume (call_frame_t *frame, xlator_t *this)
+afr_transaction_resume(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
+    local = frame->local;
 
-        if (local->transaction.eager_lock_on) {
-                /* We don't need to retain "local" in the
-                   fd list anymore, writes to all subvols
-                   are finished by now */
-                afr_remove_eager_lock_stub (local);
-        }
+    afr_restore_lk_owner(frame);
 
-        afr_restore_lk_owner (frame);
+    afr_handle_symmetric_errors(frame, this);
 
-        afr_handle_symmetric_errors (frame, this);
+    if (!local->pre_op_compat)
+        /* new mode, pre-op was done along
+           with OP */
+        afr_changelog_pre_op_update(frame, this);
 
-	if (!local->pre_op_compat)
-		/* new mode, pre-op was done along
-		   with OP */
-		afr_changelog_pre_op_update (frame, this);
+    afr_changelog_post_op(frame, this);
 
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_post_op (frame, this);
-        } else {
-		afr_changelog_post_op_done (frame, this);
-        }
-
-        return 0;
+    return 0;
 }
 
-
 /**
  * afr_transaction_fop_failed - inform that an fop failed
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
-                            int child_index)
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this, int child_index)
 {
-        afr_local_t *   local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	local->transaction.failed_subvols[child_index] = 1;
+    local->transaction.failed_subvols[child_index] = 1;
 }
 
-
-
-        static gf_boolean_t
-afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+static gf_boolean_t
+__need_previous_lock_unlocked(afr_local_t *local)
 {
-        uint64_t start1 = local1->transaction.start;
-        uint64_t start2 = local2->transaction.start;
-        uint64_t end1 = 0;
-        uint64_t end2 = 0;
-
-        if (local1->transaction.len)
-                end1 = start1 + local1->transaction.len - 1;
-        else
-                end1 = ULLONG_MAX;
+    afr_lock_t *lock = NULL;
 
-        if (local2->transaction.len)
-                end2 = start2 + local2->transaction.len - 1;
-        else
-                end2 = ULLONG_MAX;
-
-        return ((end1 >= start2) && (end2 >= start1));
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (!lock->acquired)
+        return _gf_false;
+    if (lock->acquired && lock->event_generation != local->event_generation)
+        return _gf_true;
+    return _gf_false;
 }
 
 void
-afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        afr_local_t   *each = NULL;
-
-        priv = this->private;
-
-        if (!local->fd)
-                return;
-
-        if (local->transaction.type != AFR_DATA_TRANSACTION)
-                return;
-
-        if (!priv->eager_lock)
-                return;
-
-        fdctx = afr_fd_ctx_get (local->fd, this);
-        if (!fdctx)
-                return;
-
-        if (afr_are_multiple_fds_opened (local->fd, this))
-                return;
-        /*
-         * Once full file lock is acquired in eager-lock phase, overlapping
-         * writes do not compete for inode-locks, instead are transferred to the
-         * next writes. Because of this overlapping writes are not ordered.
-         * This can cause inconsistencies in replication.
-         * Example:
-         * Two overlapping writes w1, w2 are sent in parallel on same fd
-         * in two threads t1, t2.
-         * Both threads can execute afr_writev_wind in the following manner.
-         * t1 winds w1 on brick-0
-         * t2 winds w2 on brick-0
-         * t2 winds w2 on brick-1
-         * t1 winds w1 on brick-1
-         *
-         * This check makes sure the locks are not transferred for
-         * overlapping writes.
-         */
-        LOCK (&local->fd->lock);
-        {
-                list_for_each_entry (each, &fdctx->eager_locked,
-                                     transaction.eager_locked) {
-                        if (afr_locals_overlap (each, local)) {
-                                local->transaction.eager_lock_on = _gf_false;
-                                goto unlock;
-                        }
-                }
-
-                local->transaction.eager_lock_on = _gf_true;
-                list_add_tail (&local->transaction.eager_locked,
-                               &fdctx->eager_locked);
+__afr_eager_lock_handle(afr_local_t *local, gf_boolean_t *take_lock,
+                        gf_boolean_t *do_pre_op, afr_local_t **timer_local)
+{
+    afr_lock_t *lock = NULL;
+    afr_local_t *owner_local = NULL;
+    xlator_t *this = local->transaction.frame->this;
+
+    local->transaction.eager_lock_on = _gf_true;
+    afr_set_lk_owner(local->transaction.frame, this, local->inode);
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (__need_previous_lock_unlocked(local)) {
+        if (!list_empty(&lock->owners)) {
+            lock->release = _gf_true;
+        } else if (lock->delay_timer) {
+            lock->release = _gf_true;
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                /* It will be put in frozen list
+                 * in the code flow below*/
+            } else {
+                *timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                          transaction.owner_list);
+                lock->delay_timer = NULL;
+            }
         }
-unlock:
-        UNLOCK (&local->fd->lock);
+    }
+
+    if (lock->release) {
+        list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        *take_lock = _gf_false;
+        goto out;
+    }
+
+    if (lock->delay_timer) {
+        *take_lock = _gf_false;
+        if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+            list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        } else {
+            *timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                      transaction.owner_list);
+            afr_copy_inodelk_vars(&local->internal_lock,
+                                  &(*timer_local)->internal_lock, this, 0);
+            lock->delay_timer = NULL;
+            *do_pre_op = _gf_true;
+            list_add_tail(&local->transaction.owner_list, &lock->owners);
+        }
+        goto out;
+    }
+
+    if (!list_empty(&lock->owners)) {
+        if (!lock->acquired || afr_has_lock_conflict(local, _gf_true)) {
+            list_add_tail(&local->transaction.wait_list, &lock->waiting);
+            *take_lock = _gf_false;
+            goto out;
+        }
+        owner_local = list_entry(lock->owners.next, afr_local_t,
+                                 transaction.owner_list);
+        afr_copy_inodelk_vars(&local->internal_lock,
+                              &owner_local->internal_lock, this, 0);
+        *take_lock = _gf_false;
+        *do_pre_op = _gf_true;
+    }
+
+    if (lock->acquired)
+        GF_ASSERT(!(*take_lock));
+    list_add_tail(&local->transaction.owner_list, &lock->owners);
+out:
+    return;
 }
 
 void
-afr_transaction_start (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = frame->local;
-        afr_private_t *priv  = this->private;
-        fd_t          *fd    = NULL;
-
-        afr_transaction_eager_lock_init (local, this);
-
-        if (local->fd && local->transaction.eager_lock_on)
-                afr_set_lk_owner (frame, this, local->fd);
-        else
-                afr_set_lk_owner (frame, this, frame->root);
-
-        if (!local->transaction.eager_lock_on && local->loc.inode) {
-                fd = fd_lookup (local->loc.inode, frame->root->pid);
-                if (fd == NULL)
-                        fd = fd_lookup_anonymous (local->loc.inode,
-                                                  GF_ANON_FD_FLAGS);
-
-                if (fd) {
-                        afr_delayed_changelog_wake_up (this, fd);
-                        fd_unref (fd);
-                }
-        }
-
-        if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                afr_internal_lock_finish (frame, this);
-        } else {
-                afr_lock (frame, this);
-        }
+afr_transaction_start(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    gf_boolean_t take_lock = _gf_true;
+    gf_boolean_t do_pre_op = _gf_false;
+    afr_local_t *timer_local = NULL;
+
+    priv = this->private;
+
+    if (local->transaction.type != AFR_DATA_TRANSACTION &&
+        local->transaction.type != AFR_METADATA_TRANSACTION)
+        goto lock_phase;
+
+    if (!priv->eager_lock)
+        goto lock_phase;
+
+    LOCK(&local->inode->lock);
+    {
+        __afr_eager_lock_handle(local, &take_lock, &do_pre_op, &timer_local);
+    }
+    UNLOCK(&local->inode->lock);
+lock_phase:
+    if (!local->transaction.eager_lock_on) {
+        afr_set_lk_owner(local->transaction.frame, this,
+                         local->transaction.frame->root);
+    }
+
+    if (take_lock) {
+        afr_lock(local->transaction.frame, this);
+    } else if (do_pre_op) {
+        afr_changelog_pre_op(local->transaction.frame, this);
+    }
+    /*Always call delayed_changelog_wake_up_cbk after calling pre-op above
+     * so that any inheriting can happen*/
+    if (timer_local)
+        afr_delayed_changelog_wake_up_cbk(timer_local);
 }
 
 int
-afr_write_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+afr_write_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_local_t   *local           = frame->local;
-        int           ret              = 0;
+    afr_local_t *local = frame->local;
 
-        if (err) {
-                local->op_errno = -err;
-                local->op_ret = -1;
-                goto fail;
-        }
-	ret = afr_inode_get_readable (frame, local->inode, this,
-                                      local->readable, NULL,
-				      local->transaction.type);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret, AFR_MSG_SPLIT_BRAIN,
-                        "Failing %s on gfid %s: split-brain observed.",
-                        gf_fop_list[local->op], uuid_utoa (local->inode->gfid));
-                local->op_ret = -1;
-                local->op_errno = -ret;
-                goto fail;
-        }
-        afr_transaction_start (frame, this);
-        return 0;
+    if (err) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+        goto fail;
+    }
+
+    afr_transaction_start(local, this);
+    return 0;
 fail:
-        local->transaction.unwind (frame, this);
-        AFR_STACK_DESTROY (frame);
-        return 0;
+    local->transaction.unwind(frame, this);
+    AFR_STACK_DESTROY(frame);
+    return 0;
 }
 
 int
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+afr_transaction_lockee_init(call_frame_t *frame)
 {
-        afr_local_t   *local           = NULL;
-        afr_private_t *priv            = NULL;
-        int           ret              = -1;
-        int           event_generation = 0;
-
-        local = frame->local;
-        priv  = this->private;
+    afr_local_t *local = frame->local;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_private_t *priv = frame->this->private;
+    int ret = 0;
 
-        local->transaction.resume = afr_transaction_resume;
-        local->transaction.type   = type;
+    switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            ret = afr_add_inode_lockee(local, priv->child_count);
+            break;
 
-        ret = afr_transaction_local_init (local, this);
-        if (ret < 0)
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            ret = afr_add_entry_lockee(local, &local->transaction.parent_loc,
+                                       local->transaction.basename,
+                                       priv->child_count);
+            if (ret) {
                 goto out;
+            }
+            if (local->op == GF_FOP_RENAME) {
+                ret = afr_add_entry_lockee(
+                    local, &local->transaction.new_parent_loc,
+                    local->transaction.new_basename, priv->child_count);
+                if (ret) {
+                    goto out;
+                }
 
-        if (type == AFR_ENTRY_TRANSACTION ||
-            type == AFR_ENTRY_RENAME_TRANSACTION) {
-                afr_transaction_start (frame, this);
-                ret = 0;
-                goto out;
-        }
+                if (local->newloc.inode &&
+                    IA_ISDIR(local->newloc.inode->ia_type)) {
+                    ret = afr_add_entry_lockee(local, &local->newloc, NULL,
+                                               priv->child_count);
+                    if (ret) {
+                        goto out;
+                    }
+                }
+            } else if (local->op == GF_FOP_RMDIR) {
+                ret = afr_add_entry_lockee(local, &local->loc, NULL,
+                                           priv->child_count);
+                if (ret) {
+                    goto out;
+                }
+            }
+
+            if (int_lock->lockee_count > 1) {
+                qsort(int_lock->lockee, int_lock->lockee_count,
+                      sizeof(*int_lock->lockee), afr_entry_lockee_cmp);
+            }
+            break;
+    }
+out:
+    return ret;
+}
 
-        ret = afr_inode_get_readable (frame, local->inode, this,
-                                      local->readable, &event_generation, type);
-        if (ret < 0 || event_generation != priv->event_generation) {
-                afr_inode_refresh (frame, this, local->inode, local->loc.gfid,
-                                   afr_write_txn_refresh_done);
-        } else {
-                afr_transaction_start (frame, this);
-        }
+int
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    int event_generation = 0;
+
+    local = frame->local;
+    priv = this->private;
+    local->transaction.frame = frame;
+
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &ret)) {
+        ret = -ret; /*op_errno to ret conversion*/
+        goto out;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    ret = afr_transaction_local_init(local, this);
+    if (ret < 0)
+        goto out;
+
+    ret = afr_transaction_lockee_init(frame);
+    if (ret)
+        goto out;
+
+    if (type != AFR_METADATA_TRANSACTION) {
+        goto txn_start;
+    }
+
+    ret = afr_inode_get_readable(frame, local->inode, this, local->readable,
+                                 &event_generation, type);
+    if (ret < 0 ||
+        afr_is_inode_refresh_reqd(local->inode, this, priv->event_generation,
+                                  event_generation)) {
+        afr_inode_refresh(frame, this, local->inode, local->loc.gfid,
+                          afr_write_txn_refresh_done);
         ret = 0;
+        goto out;
+    }
+
+txn_start:
+    ret = 0;
+    afr_transaction_start(local, this);
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index ca8fcfefa89..beefa26f4a6 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -14,49 +14,62 @@
 #include "afr.h"
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
-			    int child_index);
-void
-afr_txn_arbitrate_fop_cbk (call_frame_t *frame, xlator_t *this);
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this,
+                           int child_index);
+
+int32_t
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
 int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int32_t **pending);
 
-afr_inodelk_t*
-afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+void
+afr_delayed_changelog_wake_up(xlator_t *this, fd_t *fd);
 
-int32_t
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
+void
+__mark_all_success(call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this);
 
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type);
 
-void
-afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+int
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol);
 
 void
-afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+afr_pending_read_increment(afr_private_t *priv, int child_index);
 
 void
-__mark_all_success (call_frame_t *frame, xlator_t *this);
+afr_pending_read_decrement(afr_private_t *priv, int child_index);
 
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame);
 gf_boolean_t
-afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
-
-int afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		  afr_read_txn_wind_t readfn, afr_transaction_type type);
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame);
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local);
+void
+afr_zero_fill_stat(afr_local_t *local);
 
-int afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol);
+void
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2);
+int
+afr_transaction_resume(call_frame_t *frame, xlator_t *this);
 
-int __afr_txn_write_fop (call_frame_t *frame, xlator_t *this);
-int __afr_txn_write_done (call_frame_t *frame, xlator_t *this);
-call_frame_t *afr_transaction_detach_fop_frame (call_frame_t *frame);
-gf_boolean_t afr_has_quorum (unsigned char *subvols, xlator_t *this);
-gf_boolean_t afr_needs_changelog_update (afr_local_t *local);
-void afr_zero_fill_stat (afr_local_t *local);
+int
+afr_lock(call_frame_t *frame, xlator_t *this);
 
 void
-afr_pick_error_xdata (afr_local_t *local, afr_private_t *priv,
-                      inode_t *inode1, unsigned char *readable1,
-                      inode_t *inode2, unsigned char *readable2);
+afr_delayed_changelog_wake_up_cbk(void *data);
+
+int
+afr_release_notify_lock_for_ta(void *opaque);
+
+int
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque);
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index da62564e93a..df7366f0a65 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -21,970 +21,1324 @@
 struct volume_options options[];
 
 static char *afr_favorite_child_policies[AFR_FAV_CHILD_POLICY_MAX + 1] = {
-        [AFR_FAV_CHILD_NONE] = "none",
-        [AFR_FAV_CHILD_BY_SIZE] = "size",
-        [AFR_FAV_CHILD_BY_CTIME] = "ctime",
-        [AFR_FAV_CHILD_BY_MTIME] = "mtime",
-        [AFR_FAV_CHILD_BY_MAJORITY] = "majority",
-        [AFR_FAV_CHILD_POLICY_MAX] = NULL,
+    [AFR_FAV_CHILD_NONE] = "none",
+    [AFR_FAV_CHILD_BY_SIZE] = "size",
+    [AFR_FAV_CHILD_BY_CTIME] = "ctime",
+    [AFR_FAV_CHILD_BY_MTIME] = "mtime",
+    [AFR_FAV_CHILD_BY_MAJORITY] = "majority",
+    [AFR_FAV_CHILD_POLICY_MAX] = NULL,
 };
 
 int32_t
-notify (xlator_t *this, int32_t event,
-        void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int ret = -1;
-        va_list         ap;
-        void *data2 = NULL;
+    int ret = -1;
+    va_list ap;
+    void *data2 = NULL;
 
-        va_start (ap, data);
-        data2 = va_arg (ap, dict_t*);
-        va_end (ap);
-        ret = afr_notify (this, event, data, data2);
+    va_start(ap, data);
+    data2 = va_arg(ap, dict_t *);
+    va_end(ap);
+    ret = afr_notify(this, event, data, data2);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_afr_mt_end + 1);
 
+    if (ret != 0) {
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int
-xlator_subvolume_index (xlator_t *this, xlator_t *subvol)
+xlator_subvolume_index(xlator_t *this, xlator_t *subvol)
 {
-        int index = -1;
-        int i = 0;
-        xlator_list_t *list = NULL;
-
-        list = this->children;
-
-        while (list) {
-                if (subvol == list->xlator ||
-                    strcmp (subvol->name, list->xlator->name) == 0) {
-                        index = i;
-                        break;
-                }
-                list = list->next;
-                i++;
+    int index = -1;
+    int i = 0;
+    xlator_list_t *list = NULL;
+
+    list = this->children;
+
+    while (list) {
+        if (subvol == list->xlator ||
+            strcmp(subvol->name, list->xlator->name) == 0) {
+            index = i;
+            break;
         }
+        list = list->next;
+        i++;
+    }
 
-        return index;
+    return index;
 }
 
 static void
-fix_quorum_options (xlator_t *this, afr_private_t *priv, char *qtype,
-                    dict_t *options)
+fix_quorum_options(xlator_t *this, afr_private_t *priv, char *qtype,
+                   dict_t *options)
 {
-        if (dict_get (options, "quorum-type") == NULL) {
-                /* If user doesn't configure anything enable auto-quorum if the
-                 * replica has odd number of subvolumes */
-                if (priv->child_count % 2)
-                        qtype = "auto";
-        }
-
-        if (priv->quorum_count && strcmp (qtype, "fixed")) {
-                gf_msg (this->name,GF_LOG_WARNING, 0, AFR_MSG_QUORUM_OVERRIDE,
-                       "quorum-type %s overriding quorum-count %u",
-                       qtype, priv->quorum_count);
-        }
-
-        if (!strcmp (qtype, "none")) {
-                priv->quorum_count = 0;
-        } else if (!strcmp (qtype, "auto")) {
-                priv->quorum_count = AFR_QUORUM_AUTO;
-        }
+    if (dict_get_sizen(options, "quorum-type") == NULL) {
+        /* If user doesn't configure anything enable auto-quorum if the
+         * replica has more than two subvolumes */
+        if (priv->child_count > 2)
+            qtype = "auto";
+    }
+
+    if (priv->quorum_count && strcmp(qtype, "fixed")) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_OVERRIDE,
+               "quorum-type %s overriding quorum-count %u", qtype,
+               priv->quorum_count);
+    }
+
+    if (!strcmp(qtype, "none")) {
+        priv->quorum_count = 0;
+    } else if (!strcmp(qtype, "auto")) {
+        priv->quorum_count = AFR_QUORUM_AUTO;
+    }
 }
 
 int
-afr_set_favorite_child_policy (afr_private_t *priv, char *policy)
+afr_set_favorite_child_policy(afr_private_t *priv, char *policy)
 {
-        int index = -1;
+    int index = -1;
 
-        index = gf_get_index_by_elem (afr_favorite_child_policies, policy);
-        if (index  < 0 || index >= AFR_FAV_CHILD_POLICY_MAX)
-                return -1;
+    index = gf_get_index_by_elem(afr_favorite_child_policies, policy);
+    if (index < 0 || index >= AFR_FAV_CHILD_POLICY_MAX)
+        return -1;
 
-        priv->fav_child_policy = index;
+    priv->fav_child_policy = index;
 
-        return 0;
+    return 0;
 }
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        afr_private_t *priv        = NULL;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        int            ret         = -1;
-        int            index       = -1;
-        char          *qtype       = NULL;
-        char          *fav_child_policy = NULL;
-
-        priv = this->private;
-
-	GF_OPTION_RECONF ("afr-dirty-xattr",
-			  priv->afr_dirty, options, str,
-			  out);
-
-	GF_OPTION_RECONF ("metadata-splitbrain-forced-heal",
-			  priv->metadata_splitbrain_forced_heal, options, bool,
-			  out);
-
-        GF_OPTION_RECONF ("background-self-heal-count",
-                          priv->background_self_heal_count, options, uint32,
-                          out);
-
-        GF_OPTION_RECONF ("heal-wait-queue-length",
-                          priv->heal_wait_qlen, options, uint32, out);
-
 
-        GF_OPTION_RECONF ("metadata-self-heal",
-                          priv->metadata_self_heal, options, bool, out);
-
-        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, str,
-                          out);
-
-        GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("data-self-heal-window-size",
-                          priv->data_self_heal_window_size, options,
-                          uint32, out);
-
-        GF_OPTION_RECONF ("data-change-log", priv->data_change_log, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("metadata-change-log",
-                          priv->metadata_change_log, options, bool, out);
+static void
+set_data_self_heal_algorithm(afr_private_t *priv, char *algo)
+{
+    if (!algo) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    } else if (strcmp(algo, "full") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_FULL;
+    } else if (strcmp(algo, "diff") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DIFF;
+    } else {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    }
+}
 
-        GF_OPTION_RECONF ("entry-change-log", priv->entry_change_log, options,
-                          bool, out);
+void
+afr_handle_anon_inode_options(afr_private_t *priv, dict_t *options)
+{
+    char *volfile_id_str = NULL;
+    uuid_t anon_inode_gfid = {0};
+
+    /*If volume id is not present don't enable anything*/
+    if (dict_get_str(options, "volume-id", &volfile_id_str))
+        return;
+    GF_ASSERT(strlen(AFR_ANON_DIR_PREFIX) + strlen(volfile_id_str) <= NAME_MAX);
+    /*anon_inode_name is not supposed to change once assigned*/
+    if (!priv->anon_inode_name[0]) {
+        snprintf(priv->anon_inode_name, sizeof(priv->anon_inode_name), "%s-%s",
+                 AFR_ANON_DIR_PREFIX, volfile_id_str);
+        gf_uuid_parse(volfile_id_str, anon_inode_gfid);
+        /*Flip a bit to make sure volfile-id and anon-gfid are not same*/
+        anon_inode_gfid[0] ^= 1;
+        uuid_utoa_r(anon_inode_gfid, priv->anon_gfid_str);
+    }
+}
 
-        GF_OPTION_RECONF ("data-self-heal-algorithm",
-                          priv->data_self_heal_algorithm, options, str, out);
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    int timeout_old = 0;
+    int ret = -1;
+    int index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *data_self_heal = NULL;
+    char *data_self_heal_algorithm = NULL;
+    char *locking_scheme = NULL;
+    gf_boolean_t consistent_io = _gf_false;
+    gf_boolean_t choose_local_old = _gf_false;
+    gf_boolean_t enabled_old = _gf_false;
 
-        GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
+    priv = this->private;
 
-        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
-                          options, uint32, out);
+    GF_OPTION_RECONF("metadata-splitbrain-forced-heal",
+                     priv->metadata_splitbrain_forced_heal, options, bool, out);
 
-        if (read_subvol) {
-                index = xlator_subvolume_index (this, read_subvol);
-                if (index == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-                priv->read_child = index;
-        }
+    GF_OPTION_RECONF("background-self-heal-count",
+                     priv->background_self_heal_count, options, uint32, out);
 
-        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
-
-        if (read_subvol_index >-1) {
-                index=read_subvol_index;
-                if (index >= priv->child_count) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL,
-                                "%d not a subvolume-index", index);
-                        goto out;
-                }
-                priv->read_child = index;
-        }
+    GF_OPTION_RECONF("heal-wait-queue-length", priv->heal_wait_qlen, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("pre-op-compat", priv->pre_op_compat, options, bool,
-                          out);
-        GF_OPTION_RECONF ("locking-scheme", priv->locking_scheme, options, str,
-                          out);
-        GF_OPTION_RECONF ("granular-entry-heal", priv->esh_granular, options,
-                          bool, out);
+    GF_OPTION_RECONF("metadata-self-heal", priv->metadata_self_heal, options,
+                     bool, out);
 
-        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
-        GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
-        GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
-                          uint32, out);
-        fix_quorum_options (this, priv, qtype, options);
-        if (priv->quorum_count && !afr_has_quorum (priv->child_up, this))
-                gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
-                        "Client-quorum is not met");
+    GF_OPTION_RECONF("data-self-heal", data_self_heal, options, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
+    GF_OPTION_RECONF("entry-self-heal", priv->entry_self_heal, options, bool,
+                     out);
 
-	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
-			  uint32, out);
+    GF_OPTION_RECONF("data-self-heal-window-size",
+                     priv->data_self_heal_window_size, options, uint32, out);
 
-        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
-                          options, size_uint64, out);
-        /* Reset this so we re-discover in case the topology changed.  */
-        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
-                          bool, out);
+    GF_OPTION_RECONF("data-self-heal-algorithm", data_self_heal_algorithm,
+                     options, str, out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-	GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options,
-			  bool, out);
+    GF_OPTION_RECONF("halo-enabled", priv->halo_enabled, options, bool, out);
 
-	GF_OPTION_RECONF ("iam-self-heal-daemon", priv->shd.iamshd, options,
-			  bool, out);
+    GF_OPTION_RECONF("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                     options, uint32, out);
 
-        GF_OPTION_RECONF ("heal-timeout", priv->shd.timeout, options,
-                          int32, out);
+    GF_OPTION_RECONF("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                     options, uint32, out);
 
-        GF_OPTION_RECONF ("quorum-reads", priv->quorum_reads, options,
-                          bool, out);
-        GF_OPTION_RECONF ("consistent-metadata", priv->consistent_metadata,
-                          options, bool, out);
+    GF_OPTION_RECONF("halo-max-latency", priv->halo_max_latency_msec, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("shd-max-threads", priv->shd.max_threads,
-                          options, uint32, out);
+    GF_OPTION_RECONF("halo-max-replicas", priv->halo_max_replicas, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("shd-wait-qlength", priv->shd.wait_qlength,
-                          options, uint32, out);
+    GF_OPTION_RECONF("halo-min-replicas", priv->halo_min_replicas, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("favorite-child-policy", fav_child_policy, options,
-                          str, out);
-        if (afr_set_favorite_child_policy (priv, fav_child_policy) == -1)
-                goto out;
+    GF_OPTION_RECONF("read-subvolume", read_subvol, options, xlator, out);
 
-        priv->did_discovery = _gf_false;
+    choose_local_old = priv->choose_local;
+    GF_OPTION_RECONF("choose-local", priv->choose_local, options, bool, out);
 
-        ret = 0;
+    if (choose_local_old != priv->choose_local) {
+        priv->read_child = -1;
+        if (choose_local_old == _gf_false)
+            priv->did_discovery = _gf_false;
+    }
+
+    GF_OPTION_RECONF("read-hash-mode", priv->hash_mode, options, uint32, out);
+
+    if (read_subvol) {
+        index = xlator_subvolume_index(this, read_subvol);
+        if (index == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
+        }
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("read-subvolume-index", read_subvol_index, options, int32,
+                     out);
+
+    if (read_subvol_index > -1) {
+        index = read_subvol_index;
+        if (index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", index);
+            goto out;
+        }
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("pre-op-compat", priv->pre_op_compat, options, bool, out);
+    GF_OPTION_RECONF("locking-scheme", locking_scheme, options, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_RECONF("full-lock", priv->full_lock, options, bool, out);
+    GF_OPTION_RECONF("granular-entry-heal", priv->esh_granular, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("eager-lock", priv->eager_lock, options, bool, out);
+    GF_OPTION_RECONF("optimistic-change-log", priv->optimistic_change_log,
+                     options, bool, out);
+    GF_OPTION_RECONF("quorum-type", qtype, options, str, out);
+    GF_OPTION_RECONF("quorum-count", priv->quorum_count, options, uint32, out);
+    fix_quorum_options(this, priv, qtype, options);
+    if (priv->quorum_count && !afr_has_quorum(priv->child_up, this, NULL))
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+               "Client-quorum is not met");
+
+    GF_OPTION_RECONF("post-op-delay-secs", priv->post_op_delay_secs, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, options,
+                     size_uint64, out);
+    /* Reset this so we re-discover in case the topology changed.  */
+    GF_OPTION_RECONF("ensure-durability", priv->ensure_durability, options,
+                     bool, out);
+
+    enabled_old = priv->shd.enabled;
+    GF_OPTION_RECONF("self-heal-daemon", priv->shd.enabled, options, bool, out);
+
+    GF_OPTION_RECONF("iam-self-heal-daemon", priv->shd.iamshd, options, bool,
+                     out);
+
+    timeout_old = priv->shd.timeout;
+    GF_OPTION_RECONF("heal-timeout", priv->shd.timeout, options, int32, out);
+
+    GF_OPTION_RECONF("consistent-metadata", priv->consistent_metadata, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("shd-max-threads", priv->shd.max_threads, options, uint32,
+                     out);
+
+    GF_OPTION_RECONF("shd-wait-qlength", priv->shd.wait_qlength, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF("favorite-child-policy", fav_child_policy, options, str,
+                     out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
+
+    priv->did_discovery = _gf_false;
+
+    GF_OPTION_RECONF("consistent-io", consistent_io, options, bool, out);
+    if (priv->quorum_count != 0)
+        consistent_io = _gf_false;
+    priv->consistent_io = consistent_io;
+
+    afr_handle_anon_inode_options(priv, options);
+
+    GF_OPTION_RECONF("use-anonymous-inode", priv->use_anon_inode, options, bool,
+                     out);
+    if (priv->shd.enabled) {
+        if ((priv->shd.enabled != enabled_old) ||
+            (timeout_old != priv->shd.timeout))
+            afr_selfheal_childup(this, priv);
+    }
+
+    ret = 0;
 out:
-        return ret;
-
+    return ret;
 }
 
-
-static const char *favorite_child_warning_str = "You have specified subvolume '%s' "
-        "as the 'favorite child'. This means that if a discrepancy in the content "
-        "or attributes (ownership, permission, etc.) of a file is detected among "
-        "the subvolumes, the file on '%s' will be considered the definitive "
-        "version and its contents will OVERWRITE the contents of the file on other "
-        "subvolumes. All versions of the file except that on '%s' "
-        "WILL BE LOST.";
-
-
 static int
-afr_pending_xattrs_init (afr_private_t *priv, xlator_t *this)
+afr_pending_xattrs_init(afr_private_t *priv, xlator_t *this)
 {
-        int ret = -1;
-        int i = 0;
-        char *ptr = NULL;
-        char *ptr1 = NULL;
-        char *xattrs_list = NULL;
-        xlator_list_t *trav = NULL;
-
-        trav = this->children;
-
-        GF_OPTION_INIT ("afr-pending-xattr", xattrs_list, str, out);
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
-                                       priv->child_count, gf_afr_mt_char);
-        if (!priv->pending_key) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        if (!xattrs_list) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_NO_CHANGELOG,
-                        "Unable to fetch afr-pending-xattr option from volfile."
-                        " Falling back to using client translator names. ");
-
-                while (i < priv->child_count) {
-                        ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                           AFR_XATTR_PREFIX,
-                                           trav->xlator->name);
-                        if (ret == -1) {
-                                ret = -ENOMEM;
-                                goto out;
-                        }
-                        trav = trav->next;
-                        i++;
-                }
-                ret = 0;
-                goto out;
-        }
+    int ret = -1;
+    int i = 0;
+    char *ptr = NULL;
+    char *ptr1 = NULL;
+    char *xattrs_list = NULL;
+    xlator_list_t *trav = NULL;
+    int child_count = -1;
+
+    trav = this->children;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        /* priv->pending_key[THIN_ARBITER_BRICK_INDEX] is used as the
+         * name of the thin arbiter file for persistence across add/
+         * removal of DHT subvols.*/
+        child_count++;
+    }
+
+    GF_OPTION_INIT("afr-pending-xattr", xattrs_list, str, out);
+    priv->pending_key = GF_CALLOC(sizeof(*priv->pending_key), child_count,
+                                  gf_afr_mt_char);
+    if (!priv->pending_key) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    if (!xattrs_list) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_NO_CHANGELOG,
+               "Unable to fetch afr-pending-xattr option from volfile."
+               " Falling back to using client translator names. ");
 
-        ptr = ptr1 = gf_strdup (xattrs_list);
-        if (!ptr) {
+        while (i < child_count) {
+            ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                              trav->xlator->name);
+            if (ret == -1) {
                 ret = -ENOMEM;
                 goto out;
-        }
-        for (i = 0, ptr = strtok (ptr, ","); ptr; ptr = strtok (NULL, ",")) {
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                   AFR_XATTR_PREFIX, ptr);
-                if (ret == -1) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                i++;
+            }
+            trav = trav->next;
+            i++;
         }
         ret = 0;
+        goto out;
+    }
+
+    ptr = ptr1 = gf_strdup(xattrs_list);
+    if (!ptr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0, ptr = strtok(ptr, ","); ptr; ptr = strtok(NULL, ",")) {
+        ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                          ptr);
+        if (ret == -1) {
+            ret = -ENOMEM;
+            goto out;
+        }
+        i++;
+    }
+    ret = 0;
 
 out:
-        GF_FREE (ptr1);
-        return ret;
+    GF_FREE(ptr1);
+    return ret;
+}
 
+void
+afr_ta_init(afr_private_t *priv)
+{
+    priv->thin_arbiter_count = 1;
+    priv->child_count--;
+    priv->ta_child_up = 0;
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->ta_notify_dom_lock_offset = 0;
+    priv->ta_in_mem_txn_count = 0;
+    priv->ta_on_wire_txn_count = 0;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    INIT_LIST_HEAD(&priv->ta_waitq);
+    INIT_LIST_HEAD(&priv->ta_onwireq);
+    gf_uuid_clear(priv->ta_gfid);
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        afr_private_t *priv        = NULL;
-        int            child_count = 0;
-        xlator_list_t *trav        = NULL;
-        int            i           = 0;
-        int            ret         = -1;
-        GF_UNUSED int  op_errno    = 0;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        xlator_t      *fav_child   = NULL;
-        char          *qtype       = NULL;
-        char          *fav_child_policy = NULL;
-
-        if (!this->children) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_CHILD_MISCONFIGURED,
-                        "replicate translator needs more than one "
-                        "subvolume defined.");
-                return -1;
+    afr_private_t *priv = NULL;
+    int child_count = 0;
+    xlator_list_t *trav = NULL;
+    int i = 0;
+    int ret = -1;
+    GF_UNUSED int op_errno = 0;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *thin_arbiter = NULL;
+    char *data_self_heal = NULL;
+    char *locking_scheme = NULL;
+    char *data_self_heal_algorithm = NULL;
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_CHILD_MISCONFIGURED,
+               "replicate translator needs more than one "
+               "subvolume defined.");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_VOL_MISCONFIGURED,
+               "Volume is dangling.");
+    }
+
+    this->private = GF_CALLOC(1, sizeof(afr_private_t),
+                              gf_afr_mt_afr_private_t);
+    if (!this->private)
+        goto out;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&priv->saved_locks);
+    INIT_LIST_HEAD(&priv->lk_healq);
+    LOCK_INIT(&priv->lock);
+
+    child_count = xlator_subvolume_count(this);
+
+    priv->child_count = child_count;
+
+    priv->read_child = -1;
+
+    GF_OPTION_INIT("arbiter-count", priv->arbiter_count, uint32, out);
+    GF_OPTION_INIT("thin-arbiter", thin_arbiter, str, out);
+    if (thin_arbiter && strlen(thin_arbiter) > 0) {
+        afr_ta_init(priv);
+    }
+    INIT_LIST_HEAD(&priv->healing);
+    INIT_LIST_HEAD(&priv->heal_waiting);
+
+    priv->spb_choice_timeout = AFR_DEFAULT_SPB_CHOICE_TIMEOUT;
+
+    GF_OPTION_INIT("afr-dirty-xattr", priv->afr_dirty, str, out);
+
+    GF_OPTION_INIT("metadata-splitbrain-forced-heal",
+                   priv->metadata_splitbrain_forced_heal, bool, out);
+
+    GF_OPTION_INIT("read-subvolume", read_subvol, xlator, out);
+    if (read_subvol) {
+        priv->read_child = xlator_subvolume_index(this, read_subvol);
+        if (priv->read_child == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
         }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_VOL_MISCONFIGURED, "Volume is dangling.");
+    }
+    GF_OPTION_INIT("read-subvolume-index", read_subvol_index, int32, out);
+    if (read_subvol_index > -1) {
+        if (read_subvol_index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", read_subvol_index);
+            goto out;
         }
+        priv->read_child = read_subvol_index;
+    }
+    GF_OPTION_INIT("choose-local", priv->choose_local, bool, out);
 
-	this->private = GF_CALLOC (1, sizeof (afr_private_t),
-                                   gf_afr_mt_afr_private_t);
-        if (!this->private)
-                goto out;
-
-        priv = this->private;
-        LOCK_INIT (&priv->lock);
-
-        child_count = xlator_subvolume_count (this);
-
-        priv->child_count = child_count;
-
-        priv->read_child = -1;
-
-        GF_OPTION_INIT ("arbiter-count", priv->arbiter_count, uint32, out);
-        INIT_LIST_HEAD (&priv->healing);
-        INIT_LIST_HEAD (&priv->heal_waiting);
+    priv->pending_reads = GF_CALLOC(sizeof(*priv->pending_reads),
+                                    priv->child_count, gf_afr_mt_atomic_t);
 
-        priv->spb_choice_timeout = AFR_DEFAULT_SPB_CHOICE_TIMEOUT;
+    GF_OPTION_INIT("read-hash-mode", priv->hash_mode, uint32, out);
 
-	GF_OPTION_INIT ("afr-dirty-xattr", priv->afr_dirty, str, out);
+    priv->favorite_child = -1;
 
-	GF_OPTION_INIT ("metadata-splitbrain-forced-heal",
-			priv->metadata_splitbrain_forced_heal, bool, out);
+    GF_OPTION_INIT("favorite-child-policy", fav_child_policy, str, out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
 
-        GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
-        if (read_subvol) {
-                priv->read_child = xlator_subvolume_index (this, read_subvol);
-                if (priv->read_child == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-        }
-        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
-        if (read_subvol_index > -1) {
-                if (read_subvol_index >= priv->child_count) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL,
-                                "%d not a subvolume-index", read_subvol_index);
-                        goto out;
-                }
-                priv->read_child = read_subvol_index;
-        }
-        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
-
-        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
-
-        priv->favorite_child = -1;
-        GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
-        if (fav_child) {
-                priv->favorite_child = xlator_subvolume_index (this, fav_child);
-                if (priv->favorite_child == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL, "%s not a subvolume, "
-                                "cannot set it as favorite child",
-                                fav_child->name);
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_FAVORITE_CHILD,
-                        favorite_child_warning_str, fav_child->name,
-                        fav_child->name, fav_child->name);
-        }
+    GF_OPTION_INIT("shd-max-threads", priv->shd.max_threads, uint32, out);
 
-        GF_OPTION_INIT ("favorite-child-policy", fav_child_policy, str, out);
-        if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
-                goto out;
+    GF_OPTION_INIT("shd-wait-qlength", priv->shd.wait_qlength, uint32, out);
 
-        GF_OPTION_INIT ("shd-max-threads", priv->shd.max_threads,
-                         uint32, out);
+    GF_OPTION_INIT("background-self-heal-count",
+                   priv->background_self_heal_count, uint32, out);
 
-        GF_OPTION_INIT ("shd-wait-qlength", priv->shd.wait_qlength,
-                         uint32, out);
+    GF_OPTION_INIT("heal-wait-queue-length", priv->heal_wait_qlen, uint32, out);
 
-        GF_OPTION_INIT ("background-self-heal-count",
-                        priv->background_self_heal_count, uint32, out);
+    GF_OPTION_INIT("data-self-heal", data_self_heal, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
-        GF_OPTION_INIT ("heal-wait-queue-length",
-                        priv->heal_wait_qlen, uint32, out);
+    GF_OPTION_INIT("data-self-heal-algorithm", data_self_heal_algorithm, str,
+                   out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);
+    GF_OPTION_INIT("data-self-heal-window-size",
+                   priv->data_self_heal_window_size, uint32, out);
 
-        GF_OPTION_INIT ("data-self-heal-algorithm",
-                        priv->data_self_heal_algorithm, str, out);
+    GF_OPTION_INIT("metadata-self-heal", priv->metadata_self_heal, bool, out);
 
-        GF_OPTION_INIT ("data-self-heal-window-size",
-                        priv->data_self_heal_window_size, uint32, out);
+    GF_OPTION_INIT("entry-self-heal", priv->entry_self_heal, bool, out);
 
-        GF_OPTION_INIT ("metadata-self-heal", priv->metadata_self_heal, bool,
-                        out);
+    GF_OPTION_INIT("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
+    GF_OPTION_INIT("halo-max-latency", priv->halo_max_latency_msec, uint32,
+                   out);
+    GF_OPTION_INIT("halo-max-replicas", priv->halo_max_replicas, uint32, out);
+    GF_OPTION_INIT("halo-min-replicas", priv->halo_min_replicas, uint32, out);
 
-        GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
+    GF_OPTION_INIT("halo-enabled", priv->halo_enabled, bool, out);
 
-        GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
-                        out);
+    GF_OPTION_INIT("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("entry-change-log", priv->entry_change_log, bool, out);
+    GF_OPTION_INIT("iam-nfs-daemon", priv->nfsd.iamnfsd, bool, out);
 
-        GF_OPTION_INIT ("optimistic-change-log", priv->optimistic_change_log,
-                        bool, out);
+    GF_OPTION_INIT("optimistic-change-log", priv->optimistic_change_log, bool,
+                   out);
 
-        GF_OPTION_INIT ("inodelk-trace", priv->inodelk_trace, bool, out);
+    GF_OPTION_INIT("pre-op-compat", priv->pre_op_compat, bool, out);
+    GF_OPTION_INIT("locking-scheme", locking_scheme, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_INIT("full-lock", priv->full_lock, bool, out);
+    GF_OPTION_INIT("granular-entry-heal", priv->esh_granular, bool, out);
 
-        GF_OPTION_INIT ("entrylk-trace", priv->entrylk_trace, bool, out);
+    GF_OPTION_INIT("eager-lock", priv->eager_lock, bool, out);
+    GF_OPTION_INIT("quorum-type", qtype, str, out);
+    GF_OPTION_INIT("quorum-count", priv->quorum_count, uint32, out);
+    GF_OPTION_INIT(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
+                   out);
+    fix_quorum_options(this, priv, qtype, this->options);
 
-        GF_OPTION_INIT ("pre-op-compat", priv->pre_op_compat, bool, out);
-        GF_OPTION_INIT ("locking-scheme", priv->locking_scheme, str, out);
-        GF_OPTION_INIT ("granular-entry-heal", priv->esh_granular, bool, out);
+    GF_OPTION_INIT("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+    GF_OPTION_INIT("ensure-durability", priv->ensure_durability, bool, out);
 
-        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
-        GF_OPTION_INIT ("quorum-type", qtype, str, out);
-        GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
-        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
-                        out);
-        fix_quorum_options (this, priv, qtype, this->options);
+    GF_OPTION_INIT("self-heal-daemon", priv->shd.enabled, bool, out);
 
-	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
-        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
-                        out);
+    GF_OPTION_INIT("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+    GF_OPTION_INIT("heal-timeout", priv->shd.timeout, int32, out);
 
-	GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
+    GF_OPTION_INIT("consistent-metadata", priv->consistent_metadata, bool, out);
+    GF_OPTION_INIT("consistent-io", priv->consistent_io, bool, out);
+    afr_handle_anon_inode_options(priv, this->options);
 
-	GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
-        GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
+    GF_OPTION_INIT("use-anonymous-inode", priv->use_anon_inode, bool, out);
+    if (priv->quorum_count != 0)
+        priv->consistent_io = _gf_false;
 
-        GF_OPTION_INIT ("quorum-reads", priv->quorum_reads, bool, out);
-        GF_OPTION_INIT ("consistent-metadata", priv->consistent_metadata, bool,
-                        out);
+    priv->wait_count = 1;
 
-        priv->wait_count = 1;
+    priv->local = GF_CALLOC(sizeof(unsigned char), child_count, gf_afr_mt_char);
+    if (!priv->local) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        priv->local = GF_CALLOC (sizeof (unsigned char), child_count,
+    priv->anon_inode = GF_CALLOC(sizeof(unsigned char), child_count,
                                  gf_afr_mt_char);
-        if (!priv->local) {
-                ret = -ENOMEM;
-                goto out;
-        }
 
-        priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
-                                    gf_afr_mt_char);
-        if (!priv->child_up) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    priv->child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                               gf_afr_mt_char);
 
-        for (i = 0; i < child_count; i++)
-                priv->child_up[i] = -1; /* start with unknown state.
-                                           this initialization needed
-                                           for afr_notify() to work
-                                           reliably
-                                        */
+    priv->child_latency = GF_MALLOC(sizeof(*priv->child_latency) * child_count,
+                                    gf_afr_mt_child_latency_t);
+    priv->halo_child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                                    gf_afr_mt_char);
 
-        priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
-                                    gf_afr_mt_xlator_t);
-        if (!priv->children) {
-                ret = -ENOMEM;
-                goto out;
+    if (!priv->child_up || !priv->child_latency || !priv->halo_child_up ||
+        !priv->anon_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    /*Initialize to -ve ping timeout so that they are not considered
+     * in child-up events until ping-event comes*/
+    for (i = 0; i < child_count; i++)
+        priv->child_latency[i] = -1;
+
+    priv->children = GF_CALLOC(sizeof(xlator_t *), child_count,
+                               gf_afr_mt_xlator_t);
+    if (!priv->children) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_pending_xattrs_init(priv, this);
+    if (ret)
+        goto out;
+
+    trav = this->children;
+    i = 0;
+    while (i < child_count) {
+        priv->children[i] = trav->xlator;
+        trav = trav->next;
+        i++;
+    }
+
+    ret = gf_asprintf(&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT, this->name);
+    if (-1 == ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    priv->last_event = GF_CALLOC(child_count, sizeof(*priv->last_event),
+                                 gf_afr_mt_int32_t);
+    if (!priv->last_event) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    this->itable = inode_table_new(SHD_INODE_LRU_LIMIT, this);
+    if (!this->itable) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (priv->shd.iamshd) {
+        ret = afr_selfheal_daemon_init(this);
+        if (ret) {
+            ret = -ENOMEM;
+            goto out;
         }
+    }
 
-        ret = afr_pending_xattrs_init (priv, this);
-        if (ret)
-                goto out;
+    /* keep more local here as we may need them for self-heal etc */
+    this->local_pool = mem_pool_new(afr_local_t, 512);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
 
-        trav = this->children;
-        i = 0;
-        while (i < child_count) {
-                priv->children[i] = trav->xlator;
-                trav = trav->next;
-                i++;
-        }
+    priv->root_inode = NULL;
 
-        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
-                           this->name);
-        if (-1 == ret) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    ret = 0;
+out:
+    return ret;
+}
+void
+afr_destroy_healer_object(xlator_t *this, struct subvol_healer *healer)
+{
+    int ret = -1;
 
-        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
-                                      gf_afr_mt_int32_t);
-        if (!priv->last_event) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (!healer)
+        return;
 
-	ret = afr_selfheal_daemon_init (this);
-	if (ret) {
-		ret = -ENOMEM;
-		goto out;
-	}
+    if (healer->running) {
+        /*
+         * If there are any resources to cleanup, We need
+         * to do that gracefully using pthread_cleanup_push
+         */
+        ret = gf_thread_cleanup_xint(healer->thread);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SELF_HEAL_FAILED,
+                   "Failed to clean up healer threads.");
+        healer->thread = 0;
+    }
+    pthread_cond_destroy(&healer->cond);
+    pthread_mutex_destroy(&healer->mutex);
+}
 
-        /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 512);
-        if (!this->local_pool) {
-                ret = -1;
-                goto out;
-        }
+void
+afr_selfheal_daemon_fini(xlator_t *this)
+{
+    struct subvol_healer *healer = NULL;
+    afr_self_heald_t *shd = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+    if (!priv)
+        return;
+
+    shd = &priv->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < priv->child_count; i++) {
+        healer = &shd->index_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        healer = &shd->full_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        if (shd->statistics[i])
+            eh_destroy(shd->statistics[i]);
+    }
+    GF_FREE(shd->index_healers);
+    GF_FREE(shd->full_healers);
+    GF_FREE(shd->statistics);
+    if (shd->split_brain)
+        eh_destroy(shd->split_brain);
+}
+void
+fini(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
 
-        priv->root_inode = NULL;
+    priv = this->private;
 
-        ret = 0;
-out:
-        return ret;
-}
+    afr_selfheal_daemon_fini(this);
+    GF_ASSERT(list_empty(&priv->saved_locks));
 
+    LOCK(&priv->lock);
+    if (priv->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
+    UNLOCK(&priv->lock);
 
-int
-fini (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
+    if (this->local_pool != NULL) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 
-        priv = this->private;
-        LOCK (&priv->lock);
-        if (priv->timer != NULL) {
-                gf_timer_call_cancel(this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
-        UNLOCK (&priv->lock);
-        this->private = NULL;
-        afr_priv_destroy (priv);
-        //if (this->itable);//I dont see any destroy func
+    this->private = NULL;
+    afr_priv_destroy(priv);
+    if (this->itable) {
+        inode_table_destroy(this->itable);
+        this->itable = NULL;
+    }
 
-        return 0;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .lookup      = afr_lookup,
-        .open        = afr_open,
-        .lk          = afr_lk,
-        .flush       = afr_flush,
-        .statfs      = afr_statfs,
-        .fsync       = afr_fsync,
-        .fsyncdir    = afr_fsyncdir,
-        .xattrop     = afr_xattrop,
-        .fxattrop    = afr_fxattrop,
-        .inodelk     = afr_inodelk,
-        .finodelk    = afr_finodelk,
-        .entrylk     = afr_entrylk,
-        .fentrylk    = afr_fentrylk,
-
-        /* inode read */
-        .access      = afr_access,
-        .stat        = afr_stat,
-        .fstat       = afr_fstat,
-        .readlink    = afr_readlink,
-        .getxattr    = afr_getxattr,
-        .fgetxattr   = afr_fgetxattr,
-        .readv       = afr_readv,
-
-        /* inode write */
-        .writev      = afr_writev,
-        .truncate    = afr_truncate,
-        .ftruncate   = afr_ftruncate,
-        .setxattr    = afr_setxattr,
-        .fsetxattr   = afr_fsetxattr,
-        .setattr     = afr_setattr,
-        .fsetattr    = afr_fsetattr,
-        .removexattr = afr_removexattr,
-        .fremovexattr = afr_fremovexattr,
-        .fallocate   = afr_fallocate,
-        .discard     = afr_discard,
-        .zerofill    = afr_zerofill,
-
-        /* dir read */
-        .opendir     = afr_opendir,
-        .readdir     = afr_readdir,
-        .readdirp    = afr_readdirp,
-
-        /* dir write */
-        .create      = afr_create,
-        .mknod       = afr_mknod,
-        .mkdir       = afr_mkdir,
-        .unlink      = afr_unlink,
-        .rmdir       = afr_rmdir,
-        .link        = afr_link,
-        .symlink     = afr_symlink,
-        .rename      = afr_rename,
+    .lookup = afr_lookup,
+    .lk = afr_lk,
+    .flush = afr_flush,
+    .statfs = afr_statfs,
+    .fsyncdir = afr_fsyncdir,
+    .inodelk = afr_inodelk,
+    .finodelk = afr_finodelk,
+    .entrylk = afr_entrylk,
+    .fentrylk = afr_fentrylk,
+    .ipc = afr_ipc,
+    .lease = afr_lease,
+
+    /* inode read */
+    .access = afr_access,
+    .stat = afr_stat,
+    .fstat = afr_fstat,
+    .readlink = afr_readlink,
+    .getxattr = afr_getxattr,
+    .fgetxattr = afr_fgetxattr,
+    .readv = afr_readv,
+    .seek = afr_seek,
+
+    /* inode write */
+    .writev = afr_writev,
+    .truncate = afr_truncate,
+    .ftruncate = afr_ftruncate,
+    .setxattr = afr_setxattr,
+    .fsetxattr = afr_fsetxattr,
+    .setattr = afr_setattr,
+    .fsetattr = afr_fsetattr,
+    .removexattr = afr_removexattr,
+    .fremovexattr = afr_fremovexattr,
+    .fallocate = afr_fallocate,
+    .discard = afr_discard,
+    .zerofill = afr_zerofill,
+    .xattrop = afr_xattrop,
+    .fxattrop = afr_fxattrop,
+    .fsync = afr_fsync,
+
+    /*inode open*/
+    .opendir = afr_opendir,
+    .open = afr_open,
+
+    /* dir read */
+    .readdir = afr_readdir,
+    .readdirp = afr_readdirp,
+
+    /* dir write */
+    .create = afr_create,
+    .mknod = afr_mknod,
+    .mkdir = afr_mkdir,
+    .unlink = afr_unlink,
+    .rmdir = afr_rmdir,
+    .link = afr_link,
+    .symlink = afr_symlink,
+    .rename = afr_rename,
 };
 
-
 struct xlator_dumpops dumpops = {
-        .priv       = afr_priv_dump,
+    .priv = afr_priv_dump,
 };
 
-
 struct xlator_cbks cbks = {
-        .release     = afr_release,
-        .releasedir  = afr_releasedir,
-        .forget      = afr_forget,
+    .release = afr_release,
+    .releasedir = afr_releasedir,
+    .forget = afr_forget,
 };
 
-
 struct volume_options options[] = {
-        { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR,
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. Afr will prefer the one specified using "
-                         "this option if it is not stale. Option value must be "
-                         "one of the xlator names of the children. "
-                         "Ex: <volname>-client-0 till "
-                         "<volname>-client-<number-of-bricks - 1>"
-        },
-        { .key  = {"read-subvolume-index" },
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "-1",
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. AFR will prefer the one specified using "
-                         "this option if it is not stale. allowed options"
-                         " include -1 till replica-count - 1"
-        },
-        { .key = {"read-hash-mode" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 2,
-          .default_value = "1",
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. AFR will prefer the one computed using "
-                         "the method specified using this option"
-                         "0 = first up server, "
-                         "1 = hash by GFID of file (all clients use "
-                                                    "same subvolume), "
-                         "2 = hash by GFID of file and client PID",
-        },
-        { .key  = {"choose-local" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "true",
-          .description = "Choose a local subvolume (i.e. Brick) to read from"
-	                 " if read-subvolume is not explicitly set.",
-        },
-        { .key  = {"favorite-child"},
-          .type = GF_OPTION_TYPE_XLATOR,
-          .description = "If a split-brain happens choose subvol/brick set by "
-                         "this option as source."
-        },
-        { .key  = {"background-self-heal-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = 256,
-          .default_value = "8",
-          .validate = GF_OPT_VALIDATE_MIN,
-          .description = "This specifies the number of per client self-heal "
-                         "jobs that can perform parallel heals in the "
-                         "background."
-        },
-        { .key  = {"heal-wait-queue-length"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/
-          .default_value = "128",
-          .validate = GF_OPT_VALIDATE_MIN,
-          .description = "This specifies the number of heals that can be queued"
-                         " for the parallel background self heal jobs."
-        },
-        { .key  = {"data-self-heal"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"1", "on", "yes", "true", "enable",
-                    "0", "off", "no", "false", "disable",
-                    "open"},
-          .default_value = "on",
-          .description   = "Using this option we can enable/disable data "
-                           "self-heal on the file. \"open\" means data "
-                           "self-heal action will only be triggered by file "
-                           "open operations."
-        },
-        { .key  = {"data-self-heal-algorithm"},
-          .type = GF_OPTION_TYPE_STR,
-          .description   = "Select between \"full\", \"diff\". The "
-                           "\"full\" algorithm copies the entire file from "
-                           "source to sink. The \"diff\" algorithm copies to "
-                           "sink only those blocks whose checksums don't match "
-                           "with those of source. If no option is configured "
-                           "the option is chosen dynamically as follows: "
-                           "If the file does not exist on one of the sinks "
-                           "or empty file exists or if the source file size is "
-                           "about the same as page size the entire file will "
-                           "be read and written i.e \"full\" algo, "
-                           "otherwise \"diff\" algo is chosen.",
-          .value = { "diff", "full"}
-        },
-        { .key  = {"data-self-heal-window-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 1024,
-          .default_value = "1",
-          .description = "Maximum number blocks per file for which self-heal "
-                         "process would be applied simultaneously."
-        },
-        { .key  = {"metadata-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Using this option we can enable/disable metadata "
-                         "i.e. Permissions, ownerships, xattrs self-heal on "
-                         "the file/directory."
-        },
-        { .key  = {"entry-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Using this option we can enable/disable entry "
-                         "self-heal on the directory."
-        },
-        { .key  = {"data-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Data fops like write/truncate will not perform "
-                         "pre/post fop changelog operations in afr transaction "
-                         "if this option is disabled"
-        },
-        { .key  = {"metadata-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Metadata fops like setattr/setxattr will not perform "
-                         "pre/post fop changelog operations in afr transaction "
-                         "if this option is disabled"
-        },
-        { .key  = {"entry-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Entry fops like create/unlink will not perform "
-                         "pre/post fop changelog operations in afr transaction "
-                         "if this option is disabled"
-        },
-        { .key  = {"optimistic-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Entry/Metadata fops will not perform "
-                         "pre fop changelog operations in afr transaction "
-                         "if this option is enabled."
-        },
-        { .key = {"inodelk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Enabling this option logs inode lock/unlocks"
-        },
-        { .key = {"entrylk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Enabling this option logs entry lock/unlocks"
-        },
-	{ .key = {"pre-op-compat"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "on",
-	  .description = "Use separate pre-op xattrop() FOP rather than "
-	                 "overloading xdata of the OP"
-	},
-        { .key = {"eager-lock"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Enable/Disable eager lock for replica volume. "
-                         "Lock phase of a transaction has two sub-phases. "
-                         "First is an attempt to acquire locks in parallel by "
-                         "broadcasting non-blocking lock requests. If lock "
-                         "acquisition fails on any server, then the held locks "
-                         "are unlocked and we revert to a blocking locks mode "
-                         "sequentially on one server after another.  If this "
-                         "option is enabled the initial broadcasting lock "
-                         "request attempts to acquire a full lock on the entire file. "
-                         "If this fails, we revert back to the sequential "
-                         "\"regional\" blocking locks as before. In the case "
-                         "where such an \"eager\" lock is granted in the "
-                         "non-blocking phase, it gives rise to an opportunity "
-                         "for optimization. i.e, if the next write transaction "
-                         "on the same FD arrives before the unlock phase of "
-                         "the first transaction, it \"takes over\" the full "
-                         "file lock. Similarly if yet another data transaction "
-                         "arrives before the unlock phase of the \"optimized\" "
-                         "transaction, that in turn \"takes over\" the lock as "
-                         "well. The actual unlock now happens at the end of "
-                         "the last \"optimized\" transaction."
-
-        },
-        { .key = {"self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option applies to only self-heal-daemon. "
-                         "Index directory crawl and automatic healing of files "
-                         "will not be performed if this option is turned off."
-        },
-        { .key = {"iam-self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option differentiates if the replicate "
-                         "translator is running as part of self-heal-daemon "
-                         "or not."
-        },
-        { .key = {"quorum-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed"},
-          .default_value = "none",
-          .description = "If value is \"fixed\" only allow writes if "
-                         "quorum-count bricks are present.  If value is "
-                         "\"auto\" only allow writes if more than half of "
-                         "bricks, or exactly half including the first, are "
-                         "present.",
-        },
-        { .key = {"quorum-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min = 1,
-          .max = INT_MAX,
-          .default_value = 0,
-          .description = "If quorum-type is \"fixed\" only allow writes if "
-                         "this many bricks or present.  Other quorum types "
-                         "will OVERWRITE this value.",
-        },
-        { .key = {"quorum-reads"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "If quorum-reads is \"true\" only allow reads if "
-                         "quorum is met when quorum is enabled.",
-        },
-        { .key  = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Local glusterd uuid string, used in starting "
-                         "self-heal-daemon so that it can crawl only on "
-                         "local index directories.",
-        },
-        { .key  = {"post-op-delay-secs"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = INT_MAX,
-          .default_value = "1",
-          .description = "Time interval induced artificially before "
-	                 "post-operation phase of the transaction to "
-                         "enhance overlap of adjacent write operations.",
-        },
-        { .key = {AFR_SH_READDIR_SIZE_KEY},
-          .type = GF_OPTION_TYPE_SIZET,
-          .description = "readdirp size for performing entry self-heal",
-          .min = 1024,
-          .max = 131072,
-          .default_value = "1KB",
-        },
-        { .key = {"ensure-durability"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "Afr performs fsyncs for transactions if this "
-                         "option is on to make sure the changelogs/data is "
-                         "written to the disk",
-          .default_value = "on",
-        },
-	{ .key = {"afr-dirty-xattr"},
-	  .type = GF_OPTION_TYPE_STR,
-	  .default_value = AFR_DIRTY_DEFAULT,
-	},
-	{ .key = {"afr-pending-xattr"},
-	  .type = GF_OPTION_TYPE_STR,
-          .description = "Comma separated list of xattrs that are used to  "
-                         "capture information on pending heals."
-	},
-	{ .key = {"metadata-splitbrain-forced-heal"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "off",
-	},
-        { .key  = {"heal-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 60,
-          .max  = INT_MAX,
-          .default_value = "600",
-          .description = "time interval for checking the need to self-heal "
-                         "in self-heal-daemon"
-        },
-        { .key = {"consistent-metadata"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "If this option is enabled, readdirp will force "
-                         "lookups on those entries read whose read child is "
-                         "not the same as that of the parent. This will "
-                         "guarantee that all read operations on a file serve "
-                         "attributes from the same subvol as long as it holds "
-                         " a good copy of the file/dir.",
-        },
-        { .key = {"arbiter-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "subset of child_count. Has to be 0 or 1."
-        },
-        { .key   = {"shd-max-threads"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 64,
-          .default_value = "1",
-           .description = "Maximum number of threads SHD can use per local "
-                          "brick.  This can substantially lower heal times, "
-                          "but can also crush your bricks if you don't have "
-                          "the storage hardware to support this."
-        },
-        { .key   = {"shd-wait-qlength"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 655536,
-          .default_value = "1024",
-           .description = "This option can be used to control number of heals"
-                          " that can wait in SHD per subvolume",
-        },
-        { .key = {"locking-scheme"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "full", "granular"},
-          .default_value = "full",
-          .description = "If this option is set to granular, self-heal will "
-                         "stop being compatible with afr-v1, which helps afr "
-                         "be more granular while self-healing",
-        },
-        { .key = {"granular-entry-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "If this option is enabled, self-heal will resort to "
-                         "granular way of recording changelogs and doing entry "
-                         "self-heal.",
-        },
-        { .key   = {"favorite-child-policy"},
-          .type  = GF_OPTION_TYPE_STR,
-          .value = {"none", "size", "ctime", "mtime", "majority"},
-          .default_value = "none",
-          .description = "This option can be used to automatically resolve "
-                         "split-brains using various policies without user "
-                         "intervention. \"size\" picks the file with the "
-                         "biggest size as the source. \"ctime\" and \"mtime\" "
-                         "pick the file with the latest ctime and mtime "
-                         "respectively as the source. \"majority\" picks a file"
-                         " with identical mtime and size in more than half the "
-                         "number of bricks in the replica.",
-        },
-        { .key  = {NULL} },
+    {.key = {"read-subvolume"},
+     .type = GF_OPTION_TYPE_XLATOR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. Afr will prefer the one specified using "
+                    "this option if it is not stale. Option value must be "
+                    "one of the xlator names of the children. "
+                    "Ex: <volname>-client-0 till "
+                    "<volname>-client-<number-of-bricks - 1>"},
+    {.key = {"read-subvolume-index"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "-1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. AFR will prefer the one specified using "
+                    "this option if it is not stale. allowed options"
+                    " include -1 till replica-count - 1"},
+    {.key = {"read-hash-mode"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 5,
+     .default_value = "1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "inode-read fops happen only on one of the bricks in "
+         "replicate. AFR will prefer the one computed using "
+         "the method specified using this option.\n"
+         "0 = first readable child of AFR, starting from 1st child.\n"
+         "1 = hash by GFID of file (all clients use "
+         "same subvolume).\n"
+         "2 = hash by GFID of file and client PID.\n"
+         "3 = brick having the least outstanding read requests.\n"
+         "4 = brick having the least network ping latency.\n"
+         "5 = Hybrid mode between 3 and 4, ie least value among "
+         "network-latency multiplied by outstanding-read-requests."},
+    {
+        .key = {"choose-local"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Choose a local subvolume (i.e. Brick) to read from"
+                       " if read-subvolume is not explicitly set.",
+    },
+    {.key = {"background-self-heal-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 256,
+     .default_value = "8",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of per client self-heal "
+                    "jobs that can perform parallel heals in the "
+                    "background."},
+    {.key = {"halo-shd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for shd halo replication in msec."},
+    {.key = {"halo-enabled"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "False",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Enable Halo (geo) replication mode."},
+    {.key = {"halo-nfsd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for nfsd halo replication in msec."},
+    {.key = {"halo-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = AFR_HALO_MAX_LATENCY,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for halo replication in msec."},
+    {.key = {"halo-max-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The maximum number of halo replicas; replicas"
+                    " beyond this value will be written asynchronously"
+                    "via the SHD."},
+    {.key = {"halo-min-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "2",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The minimmum number of halo replicas, before adding "
+                    "out of region replicas."},
+    {.key = {"heal-wait-queue-length"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/
+     .default_value = "128",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {GD_OP_VERSION_3_7_10},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of heals that can be queued"
+                    " for the parallel background self heal jobs."},
+    {.key = {"data-self-heal"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"1", "on", "yes", "true", "enable", "0", "off", "no", "false",
+               "disable", "open"},
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Using this option we can enable/disable data "
+                    "self-heal on the file. \"open\" means data "
+                    "self-heal action will only be triggered by file "
+                    "open operations."},
+    {.key = {"data-self-heal-algorithm"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Select between \"full\", \"diff\". The "
+                    "\"full\" algorithm copies the entire file from "
+                    "source to sink. The \"diff\" algorithm copies to "
+                    "sink only those blocks whose checksums don't match "
+                    "with those of source. If no option is configured "
+                    "the option is chosen dynamically as follows: "
+                    "If the file does not exist on one of the sinks "
+                    "or empty file exists or if the source file size is "
+                    "about the same as page size the entire file will "
+                    "be read and written i.e \"full\" algo, "
+                    "otherwise \"diff\" algo is chosen.",
+     .value = {"diff", "full"}},
+    {.key = {"data-self-heal-window-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = "1",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number blocks per file for which self-heal "
+                    "process would be applied simultaneously."},
+    {.key = {"metadata-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable metadata "
+                    "i.e. Permissions, ownerships, xattrs self-heal on "
+                    "the file/directory."},
+    {.key = {"entry-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable entry "
+                    "self-heal on the directory."},
+    {.key = {"data-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"metadata-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"entry-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"optimistic-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Entry/Metadata fops will not perform "
+                    "pre fop changelog operations in afr transaction "
+                    "if this option is enabled."},
+    {.key = {"inodelk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs inode lock/unlocks"},
+    {.key = {"entrylk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs entry lock/unlocks"},
+    {.key = {"pre-op-compat"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Use separate pre-op xattrop() FOP rather than "
+                    "overloading xdata of the OP"},
+    {.key = {"eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "Enable/Disable eager lock for replica volume. "
+         "Lock phase of a transaction has two sub-phases. "
+         "First is an attempt to acquire locks in parallel by "
+         "broadcasting non-blocking lock requests. If lock "
+         "acquisition fails on any server, then the held locks "
+         "are unlocked and we revert to a blocking locks mode "
+         "sequentially on one server after another.  If this "
+         "option is enabled the initial broadcasting lock "
+         "request attempts to acquire a full lock on the entire file. "
+         "If this fails, we revert back to the sequential "
+         "\"regional\" blocking locks as before. In the case "
+         "where such an \"eager\" lock is granted in the "
+         "non-blocking phase, it gives rise to an opportunity "
+         "for optimization. i.e, if the next write transaction "
+         "on the same FD arrives before the unlock phase of "
+         "the first transaction, it \"takes over\" the full "
+         "file lock. Similarly if yet another data transaction "
+         "arrives before the unlock phase of the \"optimized\" "
+         "transaction, that in turn \"takes over\" the lock as "
+         "well. The actual unlock now happens at the end of "
+         "the last \"optimized\" transaction."
+
+    },
+    {.key = {"self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     /*.validate_fn   = validate_replica_heal_enable_disable*/
+     .description = "This option applies to only self-heal-daemon. "
+                    "Index directory crawl and automatic healing of files "
+                    "will not be performed if this option is turned off."},
+    {.key = {"iam-self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of self-heal-daemon "
+                    "or not."},
+    {.key = {"iam-nfs-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of an NFS daemon "
+                    "or not."},
+    {
+        .key = {"quorum-type"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "auto", "fixed"},
+        .default_value = "none",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-type*/
+        .description = "If value is \"fixed\" only allow writes if "
+                       "quorum-count bricks are present.  If value is "
+                       "\"auto\" only allow writes if more than half of "
+                       "bricks, or exactly half including the first, are "
+                       "present.",
+    },
+    {
+        .key = {"quorum-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = INT_MAX,
+        .default_value = 0,
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-count*/
+        /*.validate_fn = validate_quorum_count*/
+        .description = "If quorum-type is \"fixed\" only allow writes if "
+                       "this many bricks are present.  Other quorum types "
+                       "will OVERWRITE this value.",
+    },
+    {
+        .key = {"quorum-reads"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option has been removed. Reads are not allowed "
+                       "if quorum is not met.",
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Local glusterd uuid string, used in starting "
+                       "self-heal-daemon so that it can crawl only on "
+                       "local index directories.",
+    },
+    {
+        .key = {"post-op-delay-secs"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max = INT_MAX,
+        .default_value = "1",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Time interval induced artificially before "
+                       "post-operation phase of the transaction to "
+                       "enhance overlap of adjacent write operations.",
+    },
+    {
+        .key = {AFR_SH_READDIR_SIZE_KEY},
+        .type = GF_OPTION_TYPE_SIZET,
+        .description = "readdirp size for performing entry self-heal",
+        .min = 1024,
+        .max = 131072,
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .default_value = "1KB",
+    },
+    {
+        .key = {"ensure-durability"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {3},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Afr performs fsyncs for transactions if this "
+                       "option is on to make sure the changelogs/data is "
+                       "written to the disk",
+        .default_value = "on",
+    },
+    {
+        .key = {"afr-dirty-xattr"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = AFR_DIRTY_DEFAULT,
+    },
+    {.key = {"afr-pending-xattr"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are used to  "
+                    "capture information on pending heals."},
+    {
+        .key = {"metadata-splitbrain-forced-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"heal-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 5,
+     .max = INT_MAX,
+     .default_value = "600",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "time interval for checking the need to self-heal "
+                    "in self-heal-daemon"},
+    {
+        .key = {"consistent-metadata"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, readdirp will force "
+                       "lookups on those entries read whose read child is "
+                       "not the same as that of the parent. This will "
+                       "guarantee that all read operations on a file serve "
+                       "attributes from the same subvol as long as it holds "
+                       " a good copy of the file/dir.",
+    },
+    {.key = {"arbiter-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .description = "subset of child_count. Has to be 0 or 1."},
+    {
+        .key = {"thin-arbiter"},
+        .type = GF_OPTION_TYPE_STR,
+        .op_version = {GD_OP_VERSION_4_1_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .description = "contains host:path of thin abriter brick",
+    },
+    {.key = {"shd-max-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 64,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_7_12},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number of parallel heals SHD can do per "
+                    "local brick. This can substantially lower heal times"
+                    ", but can also crush your bricks if you don't have "
+                    "the storage hardware to support this."},
+    {
+        .key = {"shd-wait-qlength"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = 655536,
+        .default_value = "1024",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to control number of heals"
+                       " that can wait in SHD per subvolume",
+    },
+    {
+        .key = {"locking-scheme"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"full", "granular"},
+        .default_value = "full",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is set to granular, self-heal will "
+                       "stop being compatible with afr-v1, which helps afr "
+                       "be more granular while self-healing",
+    },
+    {.key = {"full-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "yes",
+     .op_version = {GD_OP_VERSION_3_13_2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "If this option is disabled, then the IOs will take "
+                    "range locks same as versions till 3.13.1."},
+    {
+        .key = {"granular-entry-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_8_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, self-heal will resort to "
+                       "granular way of recording changelogs and doing entry "
+                       "self-heal.",
+    },
+    {
+        .key = {"favorite-child-policy"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "size", "ctime", "mtime", "majority"},
+        .default_value = "none",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to automatically resolve "
+                       "split-brains using various policies without user "
+                       "intervention. \"size\" picks the file with the "
+                       "biggest size as the source. \"ctime\" and \"mtime\" "
+                       "pick the file with the latest ctime and mtime "
+                       "respectively as the source. \"majority\" picks a file"
+                       " with identical mtime and size in more than half the "
+                       "number of bricks in the replica.",
+    },
+    {
+        .key = {"consistent-io"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .description = "If this option is enabled, i/o will fail even if "
+                       "one of the bricks is down in the replicas",
+    },
+    {.key = {"use-compound-fops"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_3_8_4},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"use-anonymous-inode"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_8_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "Setting this option heals directory renames efficiently"},
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "replicate",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 31d761f638d..d62f9a9caf2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -8,187 +8,292 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_H__
 #define __AFR_H__
 
-#include "call-stub.h"
-#include "compat-errno.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/compat-errno.h>
 #include "afr-mem-types.h"
 
 #include "libxlator.h"
-#include "timer.h"
-#include "syncop.h"
+#include <glusterfs/timer.h>
+#include <glusterfs/syncop.h>
 
 #include "afr-self-heald.h"
 #include "afr-messages.h"
 
-#define AFR_XATTR_PREFIX "trusted.afr"
+#define SHD_INODE_LRU_LIMIT 1
 #define AFR_PATHINFO_HEADER "REPLICATE:"
 #define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
 #define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
 #define AFR_DIRTY_DEFAULT AFR_XATTR_PREFIX ".dirty"
-#define AFR_DIRTY (((afr_private_t *) (THIS->private))->afr_dirty)
+#define AFR_DIRTY (((afr_private_t *)(THIS->private))->afr_dirty)
 
-#define AFR_LOCKEE_COUNT_MAX    3
-#define AFR_DOM_COUNT_MAX    3
-#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
+#define AFR_LOCKEE_COUNT_MAX 3
+#define AFR_DOM_COUNT_MAX 3
+#define AFR_NUM_CHANGE_LOGS 3              /*data + metadata + entry*/
 #define AFR_DEFAULT_SPB_CHOICE_TIMEOUT 300 /*in seconds*/
 
 #define ARBITER_BRICK_INDEX 2
+#define THIN_ARBITER_BRICK_INDEX 2
+#define AFR_TA_DOM_NOTIFY "afr.ta.dom-notify"
+#define AFR_TA_DOM_MODIFY "afr.ta.dom-modify"
+
+#define AFR_LK_HEAL_DOM "afr.lock-heal.domain"
+
+#define AFR_HALO_MAX_LATENCY 99999
+#define AFR_ANON_DIR_PREFIX ".glusterfs-anonymous-inode"
+
+#define PFLAG_PENDING (1 << 0)
+#define PFLAG_SBRAIN (1 << 1)
+
+typedef int (*afr_lock_cbk_t)(call_frame_t *frame, xlator_t *this);
+
+typedef int (*afr_read_txn_wind_t)(call_frame_t *frame, xlator_t *this,
+                                   int subvol);
+
+typedef int (*afr_inode_refresh_cbk_t)(call_frame_t *frame, xlator_t *this,
+                                       int err);
+
+typedef int (*afr_changelog_resume_t)(call_frame_t *frame, xlator_t *this);
+
+#define AFR_COUNT(array, max)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        int __res = 0;                                                         \
+        for (__i = 0; __i < max; __i++)                                        \
+            if (array[__i])                                                    \
+                __res++;                                                       \
+        __res;                                                                 \
+    })
+#define AFR_INTERSECT(dst, src1, src2, max)                                    \
+    ({                                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < max; __i++)                                        \
+            dst[__i] = src1[__i] && src2[__i];                                 \
+    })
+#define AFR_CMP(a1, a2, len)                                                   \
+    ({                                                                         \
+        int __cmp = 0;                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < len; __i++)                                        \
+            if (a1[__i] != a2[__i]) {                                          \
+                __cmp = 1;                                                     \
+                break;                                                         \
+            }                                                                  \
+        __cmp;                                                                 \
+    })
+#define AFR_IS_ARBITER_BRICK(priv, index)                                      \
+    ((priv->arbiter_count == 1) && (index == ARBITER_BRICK_INDEX))
+
+#define AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(ret, errnum)                       \
+    do {                                                                       \
+        local->op_ret = ret;                                                   \
+        local->op_errno = errnum;                                              \
+        if (local->op_errno == EIO)                                            \
+            gf_msg(this->name, GF_LOG_ERROR, local->op_errno,                  \
+                   AFR_MSG_SPLIT_BRAIN,                                        \
+                   "Failing %s on gfid %s: "                                   \
+                   "split-brain observed.",                                    \
+                   gf_fop_list[local->op], uuid_utoa(local->inode->gfid));     \
+    } while (0)
+
+#define AFR_ERROR_OUT_IF_FDCTX_INVALID(__fd, __this, __error, __label)         \
+    do {                                                                       \
+        afr_fd_ctx_t *__fd_ctx = NULL;                                         \
+        __fd_ctx = afr_fd_ctx_get(__fd, __this);                               \
+        if (__fd_ctx && __fd_ctx->is_fd_bad) {                                 \
+            __error = EBADF;                                                   \
+            goto __label;                                                      \
+        }                                                                      \
+    } while (0)
 
-typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
-
-typedef int (*afr_read_txn_wind_t) (call_frame_t *frame, xlator_t *this, int subvol);
-
-typedef int (*afr_inode_refresh_cbk_t) (call_frame_t *frame, xlator_t *this, int err);
-
-typedef int (*afr_changelog_resume_t) (call_frame_t *frame, xlator_t *this);
-
-#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr;})
-#define AFR_COUNT(array,max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res;})
-#define AFR_INTERSECT(dst,src1,src2,max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i];})
-#define AFR_CMP(a1,a2,len) ({int __cmp = 0; int __i; for (__i = 0; __i < len; __i++) if (a1[__i] != a2[__i]) { __cmp = 1; break;} __cmp;})
-#define AFR_IS_ARBITER_BRICK(priv, index) ((priv->arbiter_count == 1) && (index == ARBITER_BRICK_INDEX))
+typedef enum {
+    AFR_READ_POLICY_FIRST_UP,
+    AFR_READ_POLICY_GFID_HASH,
+    AFR_READ_POLICY_GFID_PID_HASH,
+    AFR_READ_POLICY_LESS_LOAD,
+    AFR_READ_POLICY_LEAST_LATENCY,
+    AFR_READ_POLICY_LOAD_LATENCY_HYBRID,
+} afr_read_hash_mode_t;
 
 typedef enum {
-        AFR_FAV_CHILD_NONE,
-        AFR_FAV_CHILD_BY_SIZE,
-        AFR_FAV_CHILD_BY_CTIME,
-        AFR_FAV_CHILD_BY_MTIME,
-        AFR_FAV_CHILD_BY_MAJORITY,
-        AFR_FAV_CHILD_POLICY_MAX,
+    AFR_FAV_CHILD_NONE,
+    AFR_FAV_CHILD_BY_SIZE,
+    AFR_FAV_CHILD_BY_CTIME,
+    AFR_FAV_CHILD_BY_MTIME,
+    AFR_FAV_CHILD_BY_MAJORITY,
+    AFR_FAV_CHILD_POLICY_MAX,
 } afr_favorite_child_policy;
 
-typedef struct _afr_private {
-        gf_lock_t lock;               /* to guard access to child_count, etc */
-        unsigned int child_count;     /* total number of children   */
-        unsigned int arbiter_count;   /*subset of child_count.
-                                        Has to be 0 or 1.*/
-
-        xlator_t **children;
-
-        inode_t *root_inode;
-
-        unsigned char *child_up;
-        unsigned char *local;
-
-        char **pending_key;
-
-        char         *data_self_heal;              /* on/off/open */
-        char *       data_self_heal_algorithm;    /* name of algorithm */
-        unsigned int data_self_heal_window_size;  /* max number of pipelined
-                                                     read/writes */
-
-        struct list_head heal_waiting; /*queue for files that need heal*/
-        uint32_t  heal_wait_qlen; /*configurable queue length for heal_waiting*/
-        int32_t  heal_waiters; /* No. of elements currently in wait queue.*/
-
-        struct list_head healing;/* queue for files that are undergoing
-                                    background heal*/
-        uint32_t  background_self_heal_count;/*configurable queue length for
-                                               healing queue*/
-        int32_t  healers;/* No. of elements currently undergoing background
-                          heal*/
-
-        gf_boolean_t metadata_self_heal;   /* on/off */
-        gf_boolean_t entry_self_heal;      /* on/off */
-
-        gf_boolean_t data_change_log;       /* on/off */
-        gf_boolean_t metadata_change_log;   /* on/off */
-        gf_boolean_t entry_change_log;      /* on/off */
-
-	gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
-        int read_child;               /* read-subvolume */
-        unsigned int hash_mode;       /* for when read_child is not set */
-        int favorite_child;  /* subvolume to be preferred in resolving
-                                         split-brain cases */
-
-        afr_favorite_child_policy fav_child_policy;/*Policy to use for automatic
-                                                 resolution of split-brains.*/
-
-        gf_boolean_t inodelk_trace;
-        gf_boolean_t entrylk_trace;
-
-        unsigned int wait_count;      /* # of servers to wait for success */
-
-        gf_timer_t *timer;      /* launched when parent up is received */
-
-        gf_boolean_t      optimistic_change_log;
-        gf_boolean_t      eager_lock;
-        gf_boolean_t      pre_op_compat;      /* on/off */
-	uint32_t          post_op_delay_secs;
-        unsigned int      quorum_count;
-        gf_boolean_t      quorum_reads;
-
-        char                   vol_uuid[UUID_SIZE + 1];
-        int32_t                *last_event;
-
-	/* @event_generation: Keeps count of number of events received which can
-	   potentially impact consistency decisions. The events are CHILD_UP
-	   and CHILD_DOWN, when we have to recalculate the freshness/staleness
-	   of copies to detect if changes had happened while the other server
-	   was down. CHILD_DOWN and CHILD_UP can also be received on network
-	   disconnect/reconnects and not necessarily server going down/up.
-	   Recalculating freshness/staleness on network events is equally
-	   important as we might have had a network split brain.
-	*/
-	uint32_t               event_generation;
-
-        gf_boolean_t           choose_local;
-        gf_boolean_t           did_discovery;
-        uint64_t               sh_readdir_size;
-        gf_boolean_t           ensure_durability;
-        char                   *sh_domain;
-	char                   *afr_dirty;
-
-	afr_self_heald_t       shd;
-
-        gf_boolean_t           consistent_metadata;
-        uint64_t               spb_choice_timeout;
-        gf_boolean_t           need_heal;
-
-	/* pump dependencies */
-	void                   *pump_private;
-	gf_boolean_t           use_afr_in_pump;
-	char                   *locking_scheme;
-        gf_boolean_t            esh_granular;
-} afr_private_t;
-
-
 typedef enum {
-        AFR_DATA_TRANSACTION,          /* truncate, write, ... */
-        AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
-        AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
-        AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
-} afr_transaction_type;
+    AFR_SELFHEAL_DATA_FULL = 0,
+    AFR_SELFHEAL_DATA_DIFF,
+    AFR_SELFHEAL_DATA_DYNAMIC,
+} afr_data_self_heal_type_t;
 
 typedef enum {
-        AFR_TRANSACTION_LK,
-        AFR_SELFHEAL_LK,
-} transaction_lk_type_t;
+    AFR_CHILD_UNKNOWN = -1,
+    AFR_CHILD_ZERO,
+    AFR_CHILD_ONE,
+    AFR_CHILD_THIN_ARBITER,
+} afr_child_index;
 
 typedef enum {
-        AFR_LOCK_OP,
-        AFR_UNLOCK_OP,
-} afr_lock_op_type_t;
+    TA_WAIT_FOR_NOTIFY_LOCK_REL, /*FOP came after notify domain lock upcall
+                                   notification and waiting for its release.*/
+    TA_GET_INFO_FROM_TA_FILE,    /*FOP needs post-op on ta file to get
+                                  *info about which brick is bad.*/
+    TA_INFO_IN_MEMORY_SUCCESS,   /*Bad brick info is in memory and fop failed
+                                  *on BAD brick - Success*/
+    TA_INFO_IN_MEMORY_FAILED,    /*Bad brick info is in memory and fop failed
+                                  *on GOOD brick - Failed*/
+    TA_SUCCESS,                  /*FOP succeeded on both data bricks.*/
+} afr_ta_fop_state_t;
+
+struct afr_nfsd {
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamnfsd;
+};
 
-typedef enum {
-        AFR_DATA_SELF_HEAL_LK,
-        AFR_METADATA_SELF_HEAL_LK,
-        AFR_ENTRY_SELF_HEAL_LK,
-}selfheal_lk_type_t;
+typedef struct _afr_lk_heal_info {
+    fd_t *fd;
+    int32_t cmd;
+    struct gf_flock flock;
+    dict_t *xdata_req;
+    unsigned char *locked_nodes;
+    struct list_head pos;
+    gf_lkowner_t lk_owner;
+    pid_t pid;
+    int32_t *child_up_event_gen;
+    int32_t *child_down_event_gen;
+} afr_lk_heal_info_t;
+
+typedef struct _afr_private {
+    gf_lock_t lock;             /* to guard access to child_count, etc */
+    unsigned int child_count;   /* total number of children   */
+    unsigned int arbiter_count; /*subset of child_count.
+                                  Has to be 0 or 1.*/
+
+    xlator_t **children;
+
+    inode_t *root_inode;
+
+    int favorite_child; /* subvolume to be preferred in resolving
+                                    split-brain cases */
+    /* For thin-arbiter. */
+    uuid_t ta_gfid;
+    unsigned int thin_arbiter_count; /* 0 or 1 at the moment.*/
+    int ta_bad_child_index;
+    int ta_event_gen;
+    unsigned int ta_in_mem_txn_count;
+    unsigned int ta_on_wire_txn_count;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+
+    unsigned char *anon_inode;
+    unsigned char *child_up;
+    unsigned char *halo_child_up;
+    int64_t *child_latency;
+    unsigned char *local;
+
+    char **pending_key;
+
+    afr_data_self_heal_type_t data_self_heal_algorithm;
+    unsigned int data_self_heal_window_size; /* max number of pipelined
+                                                read/writes */
+
+    struct list_head heal_waiting; /*queue for files that need heal*/
+    uint32_t heal_wait_qlen; /*configurable queue length for heal_waiting*/
+    int32_t heal_waiters;    /* No. of elements currently in wait queue.*/
+
+    struct list_head healing;            /* queue for files that are undergoing
+                                            background heal*/
+    uint32_t background_self_heal_count; /*configurable queue length for
+                                           healing queue*/
+    int32_t healers; /* No. of elements currently undergoing background
+                      heal*/
+
+    gf_boolean_t release_ta_notify_dom_lock;
+
+    gf_boolean_t metadata_self_heal; /* on/off */
+    gf_boolean_t entry_self_heal;    /* on/off */
+
+    gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
+    int read_child;                               /* read-subvolume */
+    gf_atomic_t *pending_reads; /*No. of pending read cbks per child.*/
+
+    gf_timer_t *timer; /* launched when parent up is received */
+
+    unsigned int wait_count; /* # of servers to wait for success */
+
+    unsigned char ta_child_up;
+    gf_boolean_t optimistic_change_log;
+    gf_boolean_t eager_lock;
+    gf_boolean_t pre_op_compat; /* on/off */
+    uint32_t post_op_delay_secs;
+    unsigned int quorum_count;
+
+    off_t ta_notify_dom_lock_offset;
+    afr_favorite_child_policy fav_child_policy; /*Policy to use for automatic
+                                              resolution of split-brains.*/
+    afr_read_hash_mode_t hash_mode; /* for when read_child is not set */
+
+    int32_t *last_event;
+
+    /* @event_generation: Keeps count of number of events received which can
+       potentially impact consistency decisions. The events are CHILD_UP
+       and CHILD_DOWN, when we have to recalculate the freshness/staleness
+       of copies to detect if changes had happened while the other server
+       was down. CHILD_DOWN and CHILD_UP can also be received on network
+       disconnect/reconnects and not necessarily server going down/up.
+       Recalculating freshness/staleness on network events is equally
+       important as we might have had a network split brain.
+    */
+    uint32_t event_generation;
+    char vol_uuid[UUID_SIZE + 1];
+
+    gf_boolean_t choose_local;
+    gf_boolean_t did_discovery;
+    gf_boolean_t ensure_durability;
+    gf_boolean_t halo_enabled;
+    gf_boolean_t consistent_metadata;
+    gf_boolean_t need_heal;
+    gf_boolean_t granular_locks;
+    uint64_t sh_readdir_size;
+    char *sh_domain;
+    char *afr_dirty;
+
+    uint64_t spb_choice_timeout;
+
+    afr_self_heald_t shd;
+    struct afr_nfsd nfsd;
+
+    uint32_t halo_max_latency_msec;
+    uint32_t halo_max_replicas;
+    uint32_t halo_min_replicas;
+
+    gf_boolean_t full_lock;
+    gf_boolean_t esh_granular;
+    gf_boolean_t consistent_io;
+    gf_boolean_t data_self_heal; /* on/off */
+    gf_boolean_t use_anon_inode;
+
+    /*For lock healing.*/
+    struct list_head saved_locks;
+    struct list_head lk_healq;
+
+    /*For anon-inode handling */
+    char anon_inode_name[NAME_MAX + 1];
+    char anon_gfid_str[UUID_SIZE + 1];
+} afr_private_t;
 
 typedef enum {
-        AFR_INODELK_TRANSACTION,
-        AFR_INODELK_NB_TRANSACTION,
-        AFR_ENTRYLK_TRANSACTION,
-        AFR_ENTRYLK_NB_TRANSACTION,
-        AFR_INODELK_SELFHEAL,
-        AFR_INODELK_NB_SELFHEAL,
-        AFR_ENTRYLK_SELFHEAL,
-        AFR_ENTRYLK_NB_SELFHEAL,
-} afr_lock_call_type_t;
+    AFR_DATA_TRANSACTION,         /* truncate, write, ... */
+    AFR_METADATA_TRANSACTION,     /* chmod, chown, ... */
+    AFR_ENTRY_TRANSACTION,        /* create, rmdir, ... */
+    AFR_ENTRY_RENAME_TRANSACTION, /* rename */
+} afr_transaction_type;
 
 /*
   xattr format: trusted.afr.volume = [x y z]
@@ -198,882 +303,940 @@ typedef enum {
 */
 
 static inline int
-afr_index_for_transaction_type (afr_transaction_type type)
+afr_index_for_transaction_type(afr_transaction_type type)
 {
-        switch (type) {
-
+    switch (type) {
         case AFR_DATA_TRANSACTION:
-                return 0;
+            return 0;
 
         case AFR_METADATA_TRANSACTION:
-                return 1;
+            return 1;
 
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                return 2;
-        }
+            return 2;
+    }
 
-        return -1;  /* make gcc happy */
+    return -1; /* make gcc happy */
 }
 
 static inline int
-afr_index_from_ia_type (ia_type_t type)
+afr_index_from_ia_type(ia_type_t type)
 {
-        switch (type) {
+    switch (type) {
         case IA_IFDIR:
-                return afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+            return afr_index_for_transaction_type(AFR_ENTRY_TRANSACTION);
         case IA_IFREG:
-                return afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        default: return -1;
-        }
+            return afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+        default:
+            return -1;
+    }
 }
 
 typedef struct {
-        loc_t                   loc;
-        char                    *basename;
-        unsigned char           *locked_nodes;
-        int                     locked_count;
+    struct gf_flock flock;
+    loc_t loc;
+    fd_t *fd;
+    char *basename;
+    unsigned char *locked_nodes;
+    int locked_count;
 
-} afr_entry_lockee_t;
+} afr_lockee_t;
 
 int
-afr_entry_lockee_cmp (const void *l1, const void *l2);
+afr_entry_lockee_cmp(const void *l1, const void *l2);
 
 typedef struct {
-        char    *domain; /* Domain on which inodelk is taken */
-        struct gf_flock flock;
-        unsigned char *locked_nodes;
-        int32_t lock_count;
-} afr_inodelk_t;
+    loc_t *lk_loc;
 
-typedef struct {
-        loc_t *lk_loc;
-
-        int                     lockee_count;
-        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+    afr_lockee_t lockee[AFR_LOCKEE_COUNT_MAX];
 
-        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
-        const char *lk_basename;
-        const char *lower_basename;
-        const char *higher_basename;
-        char lower_locked;
-        char higher_locked;
+    const char *lk_basename;
+    const char *lower_basename;
+    const char *higher_basename;
 
-        unsigned char *locked_nodes;
-        unsigned char *lower_locked_nodes;
+    unsigned char *lower_locked_nodes;
 
-        selfheal_lk_type_t selfheal_lk_type;
-        transaction_lk_type_t transaction_lk_type;
+    afr_lock_cbk_t lock_cbk;
 
-        int32_t lock_count;
-        int32_t entrylk_lock_count;
+    int lockee_count;
 
-        uint64_t lock_number;
-        int32_t lk_call_count;
-        int32_t lk_expected_count;
-        int32_t lk_attempted_count;
+    int32_t lk_call_count;
+    int32_t lk_expected_count;
+    int32_t lk_attempted_count;
 
-        int32_t lock_op_ret;
-        int32_t lock_op_errno;
-        afr_lock_cbk_t lock_cbk;
-        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+    int32_t lock_op_ret;
+    int32_t lock_op_errno;
+    char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+    int32_t lock_count;
+    char lower_locked;
+    char higher_locked;
 } afr_internal_lock_t;
 
 struct afr_reply {
-	int	valid;
-	int32_t	op_ret;
-	int32_t	op_errno;
-	dict_t *xattr;/*For xattrop*/
-	dict_t *xdata;
-	struct iatt poststat;
-	struct iatt postparent;
-	struct iatt prestat;
-	struct iatt preparent;
-	struct iatt preparent2;
-	struct iatt postparent2;
-        /* For rchecksum */
-	uint8_t checksum[MD5_DIGEST_LENGTH];
-        gf_boolean_t buf_has_zeroes;
-        /* For lookup */
-        int8_t need_heal;
+    int valid;
+    int32_t op_ret;
+    dict_t *xattr; /*For xattrop*/
+    dict_t *xdata;
+    struct iatt poststat;
+    struct iatt postparent;
+    struct iatt prestat;
+    struct iatt preparent;
+    struct iatt preparent2;
+    struct iatt postparent2;
+    int32_t op_errno;
+    /* For rchecksum */
+    uint8_t checksum[SHA256_DIGEST_LENGTH];
+    gf_boolean_t buf_has_zeroes;
+    gf_boolean_t fips_mode_rchecksum;
+    /* For lookup */
+    int8_t need_heal;
 };
 
 typedef enum {
-        AFR_FD_NOT_OPENED,
-        AFR_FD_OPENED,
-        AFR_FD_OPENING
+    AFR_FD_NOT_OPENED,
+    AFR_FD_OPENED,
+    AFR_FD_OPENING
 } afr_fd_open_status_t;
 
 typedef struct {
-        unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
-	int inherited[AFR_NUM_CHANGE_LOGS];
-	int on_disk[AFR_NUM_CHANGE_LOGS];
-        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
-
-        unsigned int *lock_piggyback;
-        unsigned int *lock_acquired;
-
-        int flags;
-
-	/* used for delayed-post-op optimization */
-	pthread_mutex_t    delay_lock;
-	gf_timer_t        *delay_timer;
-	call_frame_t      *delay_frame;
-
-	/* set if any write on this fd was a non stable write
-	   (i.e, without O_SYNC or O_DSYNC)
-	*/
-	gf_boolean_t      witnessed_unstable_write;
-
-	/* @open_fd_count:
-	   Number of open FDs queried from the server, as queried through
-	   xdata in FOPs. Currently, used to decide if eager-locking must be
-	   temporarily disabled.
-	*/
-        uint32_t        open_fd_count;
-
-
-	/* list of frames currently in progress */
-	struct list_head  eager_locked;
-
-	/* the subvolume on which the latest sequence of readdirs (starting
-	   at offset 0) has begun. Till the next readdir request with 0 offset
-	   arrives, we continue to read off this subvol.
-	*/
-	int readdir_subvol;
+    afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+    int flags;
+
+    /* the subvolume on which the latest sequence of readdirs (starting
+       at offset 0) has begun. Till the next readdir request with 0 offset
+       arrives, we continue to read off this subvol.
+    */
+    int readdir_subvol;
+    /* lock-healing related members. */
+    gf_boolean_t is_fd_bad;
+    afr_lk_heal_info_t *lk_heal_info;
+
 } afr_fd_ctx_t;
 
+typedef enum {
+    AFR_FOP_LOCK_PARALLEL,
+    AFR_FOP_LOCK_SERIAL,
+    AFR_FOP_LOCK_QUORUM_FAILED,
+} afr_fop_lock_state_t;
+
+typedef struct _afr_inode_lock_t {
+    /* @num_inodelks:
+       Number of inodelks queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    int32_t num_inodelks;
+    unsigned int event_generation;
+    gf_timer_t *delay_timer;
+    struct list_head owners;  /*Transactions that are performing fop*/
+    struct list_head post_op; /*Transactions that are done with the fop
+                               *So can not conflict with the fops*/
+    struct list_head waiting; /*Transaction that are waiting for
+                               *conflicting transactions to complete*/
+    struct list_head frozen;  /*Transactions that need to go as part of
+                               * next batch of eager-lock*/
+    gf_boolean_t release;
+    gf_boolean_t acquired;
+} afr_lock_t;
+
+typedef struct _afr_inode_ctx {
+    uint64_t read_subvol;
+    uint64_t write_subvol;
+    int lock_count;
+    int spb_choice;
+    gf_timer_t *timer;
+    unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
+    int inherited[AFR_NUM_CHANGE_LOGS];
+    int on_disk[AFR_NUM_CHANGE_LOGS];
+    /*Only 2 types of transactions support eager-locks now. DATA/METADATA*/
+    afr_lock_t lock[2];
+
+    /* @open_fd_count:
+       Number of open FDs queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    uint32_t open_fd_count;
+    gf_boolean_t need_refresh;
+
+    /* set if any write on this fd was a non stable write
+       (i.e, without O_SYNC or O_DSYNC)
+    */
+    gf_boolean_t witnessed_unstable_write;
+} afr_inode_ctx_t;
 
 typedef struct _afr_local {
-	glusterfs_fop_t  op;
-        unsigned int call_count;
+    glusterfs_fop_t op;
+    unsigned int call_count;
 
-	/* @event_generation: copy of priv->event_generation taken at the
-	   time of starting the transaction. The copy is made so that we
-	   have a stable value through the various phases of the transaction.
-	*/
-	unsigned int event_generation;
+    /* @event_generation: copy of priv->event_generation taken at the
+       time of starting the transaction. The copy is made so that we
+       have a stable value through the various phases of the transaction.
+    */
+    unsigned int event_generation;
 
-        uint32_t     open_fd_count;
-        gf_boolean_t update_open_fd_count;
+    uint32_t open_fd_count;
+    int32_t num_inodelks;
 
-	gf_lkowner_t  saved_lk_owner;
+    int32_t op_ret;
+    int32_t op_errno;
 
-        int32_t op_ret;
-        int32_t op_errno;
+    int dirty[AFR_NUM_CHANGE_LOGS];
 
-        int32_t **pending;
+    int32_t **pending;
 
-	int dirty[AFR_NUM_CHANGE_LOGS];
+    loc_t loc;
+    loc_t newloc;
 
-        loc_t loc;
-        loc_t newloc;
+    fd_t *fd;
+    afr_fd_ctx_t *fd_ctx;
 
-        fd_t *fd;
-	afr_fd_ctx_t *fd_ctx;
+    /* @child_up: copy of priv->child_up taken at the time of transaction
+       start. The copy is taken so that we have a stable child_up array
+       through the phases of the transaction as priv->child_up[i] can keep
+       changing through time.
+    */
+    unsigned char *child_up;
 
-	/* @child_up: copy of priv->child_up taken at the time of transaction
-	   start. The copy is taken so that we have a stable child_up array
-	   through the phases of the transaction as priv->child_up[i] can keep
-	   changing through time.
-	*/
-        unsigned char *child_up;
+    /* @read_attempted:
+       array of flags representing subvolumes where read operations of
+       the read transaction have already been attempted. The array is
+       first pre-filled with down subvolumes, and as reads are performed
+       on other subvolumes, those are set as well. This way if the read
+       operation fails we do not retry on that subvolume again.
+    */
+    unsigned char *read_attempted;
 
-	/* @read_attempted:
-	   array of flags representing subvolumes where read operations of
-	   the read transaction have already been attempted. The array is
-	   first pre-filled with down subvolumes, and as reads are performed
-	   on other subvolumes, those are set as well. This way if the read
-	   operation fails we do not retry on that subvolume again.
-	*/
-	unsigned char *read_attempted;
+    /* @readfn:
 
-	/* @readfn:
+       pointer to function which will perform the read operation on a given
+       subvolume. Used in read transactions.
+    */
 
-	   pointer to function which will perform the read operation on a given
-	   subvolume. Used in read transactions.
-	*/
+    afr_read_txn_wind_t readfn;
 
-	afr_read_txn_wind_t readfn;
+    /* @inode:
 
-	/* @refreshed:
+       the inode on which the read txn is performed on. ref'ed and copied
+       from either fd->inode or loc.inode
+    */
 
-	   the inode was "refreshed" (i.e, pending xattrs from all subvols
-	   freshly inspected and inode ctx updated accordingly) as part of
-	   this transaction already.
-	*/
-	gf_boolean_t refreshed;
+    inode_t *inode;
 
-	/* @inode:
+    /* @parent[2]:
 
-	   the inode on which the read txn is performed on. ref'ed and copied
-	   from either fd->inode or loc.inode
-	*/
+       parent inode[s] on which directory transactions are performed.
+    */
 
-	inode_t *inode;
+    inode_t *parent;
+    inode_t *parent2;
 
-	/* @parent[2]:
+    /* @readable:
 
-	   parent inode[s] on which directory transactions are performed.
-	*/
+       array of flags representing servers from which a read can be
+       performed. This is the output of afr_inode_refresh()
+    */
+    unsigned char *readable;
+    unsigned char *readable2; /*For rename transaction*/
 
-	inode_t *parent;
-	inode_t *parent2;
+    afr_inode_refresh_cbk_t refreshfn;
 
-	/* @readable:
+    /* @refreshinode:
 
-	   array of flags representing servers from which a read can be
-	   performed. This is the output of afr_inode_refresh()
-	*/
-	unsigned char *readable;
-	unsigned char *readable2; /*For rename transaction*/
+       Inode currently getting refreshed.
+    */
+    inode_t *refreshinode;
 
-	afr_inode_refresh_cbk_t refreshfn;
+    dict_t *xattr_req;
 
-	/* @refreshinode:
+    dict_t *dict;
 
-	   Inode currently getting refreshed.
-	*/
-	inode_t *refreshinode;
+    int read_subvol; /* Current read subvolume */
 
-        /*To handle setattr/setxattr on yet to be linked inode from dht*/
-        uuid_t  refreshgfid;
+    int optimistic_change_log;
 
-	/*
-	  @pre_op_compat:
+    afr_internal_lock_t internal_lock;
 
-	  compatibility mode of pre-op. send a separate pre-op and
-	  op operations as part of transaction, rather than combining
-	*/
+    /*To handle setattr/setxattr on yet to be linked inode from dht*/
+    uuid_t refreshgfid;
 
-	gf_boolean_t pre_op_compat;
+    /* @refreshed:
 
-        dict_t  *xattr_req;
+       the inode was "refreshed" (i.e, pending xattrs from all subvols
+       freshly inspected and inode ctx updated accordingly) as part of
+       this transaction already.
+    */
+    gf_boolean_t refreshed;
 
-        afr_internal_lock_t internal_lock;
+    gf_boolean_t update_num_inodelks;
+    gf_boolean_t update_open_fd_count;
 
-        dict_t  *dict;
+    /*
+      @pre_op_compat:
 
-        int      optimistic_change_log;
-	gf_boolean_t      delayed_post_op;
+      compatibility mode of pre-op. send a separate pre-op and
+      op operations as part of transaction, rather than combining
+    */
 
-	/* Is the current writev() going to perform a stable write?
-	   i.e, is fd->flags or @flags writev param have O_SYNC or
-	   O_DSYNC?
-	*/
-	gf_boolean_t      stable_write;
+    gf_boolean_t pre_op_compat;
 
-	/* This write appended to the file. Nnot necessarily O_APPEND,
-	   just means the offset of write was at the end of file.
-	*/
-	gf_boolean_t      append_write;
+    /* Is the current writev() going to perform a stable write?
+       i.e, is fd->flags or @flags writev param have O_SYNC or
+       O_DSYNC?
+    */
+    gf_boolean_t stable_write;
 
-        /*
-          This struct contains the arguments for the "continuation"
-          (scheme-like) of fops
-        */
+    /* This write appended to the file. Nnot necessarily O_APPEND,
+       just means the offset of write was at the end of file.
+    */
+    gf_boolean_t append_write;
+
+    /*
+      This struct contains the arguments for the "continuation"
+      (scheme-like) of fops
+    */
 
+    struct {
         struct {
-                struct {
-                        gf_boolean_t needs_fresh_lookup;
-                        uuid_t gfid_req;
-                } lookup;
-
-                struct {
-                        unsigned char buf_set;
-                        struct statvfs buf;
-                } statfs;
-
-                struct {
-                        int32_t flags;
-                } open;
-
-                struct {
-                        int32_t cmd;
-                        struct gf_flock user_flock;
-                        struct gf_flock ret_flock;
-                        unsigned char *locked_nodes;
-                } lk;
-
-                /* inode read */
-
-                struct {
-                        int32_t mask;
-                        int last_index;  /* index of the child we tried previously */
-                } access;
-
-                struct {
-                        int last_index;
-                } stat;
-
-                struct {
-                        int last_index;
-                } fstat;
-
-                struct {
-                        size_t size;
-                        int last_index;
-                } readlink;
-
-                struct {
-                        char *name;
-                        int last_index;
-                        long xattr_len;
-                } getxattr;
-
-                struct {
-                        size_t size;
-                        off_t offset;
-                        int last_index;
-                        uint32_t flags;
-                } readv;
-
-                /* dir read */
-
-                struct {
-                        int success_count;
-                        int32_t op_ret;
-                        int32_t op_errno;
-
-                        uint32_t *checksum;
-                } opendir;
-
-                struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        size_t size;
-                        off_t offset;
-                        dict_t *dict;
-                        gf_boolean_t failed;
-                        int last_index;
-                } readdir;
-                /* inode write */
-
-                struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } inode_wfop; //common structure for all inode-write-fops
-
-                struct {
-                        int32_t op_ret;
-
-                        struct iovec *vector;
-                        struct iobref *iobref;
-                        int32_t count;
-                        off_t offset;
-                        uint32_t flags;
-                } writev;
-
-                struct {
-                        off_t offset;
-                } truncate;
-
-                struct {
-                        off_t offset;
-                } ftruncate;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                } setattr;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                } fsetattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } setxattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } fsetxattr;
-
-                struct {
-                        char *name;
-                } removexattr;
-
-                struct {
-                        dict_t *xattr;
-                        gf_xattrop_flags_t optype;
-                } xattrop;
-
-                /* dir write */
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt prenewparent;
-                        struct iatt postnewparent;
-                } dir_fop; //common structure for all dir fops
-
-                struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
-                } create;
-
-                struct {
-                        dev_t dev;
-                        mode_t mode;
-                        dict_t *params;
-                } mknod;
-
-                struct {
-                        int32_t mode;
-                        dict_t *params;
-                } mkdir;
-
-                struct {
-                        int flags;
-                } rmdir;
-
-                struct {
-                        dict_t *params;
-                        char *linkpath;
-                } symlink;
-
-		struct {
-			int32_t mode;
-			off_t offset;
-			size_t len;
-		} fallocate;
-
-		struct {
-			off_t offset;
-			size_t len;
-		} discard;
-
-                struct {
-                        off_t offset;
-                        off_t len;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } zerofill;
-
-                struct {
-                        char *volume;
-                        int32_t cmd;
-                        struct gf_flock flock;
-                } inodelk;
-
-                struct {
-                        off_t offset;
-                        gf_seek_what_t what;
-                } seek;
-
-        } cont;
+            struct statvfs buf;
+            unsigned char buf_set;
+        } statfs;
 
         struct {
-                off_t start, len;
+            fd_t *fd;
+            int32_t flags;
+        } open;
 
-                gf_boolean_t    eager_lock_on;
-                int *eager_lock;
+        struct {
+            struct gf_flock user_flock;
+            struct gf_flock ret_flock;
+            unsigned char *locked_nodes;
+            int32_t cmd;
+            /*For lock healing only.*/
+            unsigned char *dom_locked_nodes;
+            int32_t *dom_lock_op_ret;
+            int32_t *dom_lock_op_errno;
+            struct gf_flock *getlk_rsp;
+        } lk;
+
+        /* inode read */
 
-                char *basename;
-                char *new_basename;
+        struct {
+            int32_t mask;
+            int last_index; /* index of the child we tried previously */
+        } access;
 
-                loc_t parent_loc;
-                loc_t new_parent_loc;
+        struct {
+            int last_index;
+        } stat;
 
-                afr_transaction_type type;
+        struct {
+            int last_index;
+        } fstat;
 
-		/* stub to resume on destruction
-		   of the transaction frame */
-		call_stub_t      *resume_stub;
+        struct {
+            size_t size;
+            int last_index;
+        } readlink;
 
-		struct list_head  eager_locked;
+        struct {
+            char *name;
+            long xattr_len;
+            int last_index;
+        } getxattr;
 
-                unsigned char   *pre_op;
+        struct {
+            size_t size;
+            off_t offset;
+            int last_index;
+            uint32_t flags;
+        } readv;
 
-                /* For arbiter configuration only. */
-                dict_t **pre_op_xdata;
-                unsigned char *pre_op_sources;
+        /* dir read */
 
-		/* @failed_subvols: subvolumes on which a pre-op or a
-                    FOP failed. */
-                unsigned char   *failed_subvols;
+        struct {
+            uint32_t *checksum;
+            int success_count;
+            int32_t op_ret;
+            int32_t op_errno;
+        } opendir;
 
-		/* @dirtied: flag which indicates whether we set dirty flag
-		   in the OP. Typically true when we are performing operation
-		   on more than one subvol and optimistic changelog is disabled
+        struct {
+            int32_t op_ret;
+            int32_t op_errno;
+            size_t size;
+            off_t offset;
+            dict_t *dict;
+            int last_index;
+            gf_boolean_t failed;
+        } readdir;
+        /* inode write */
 
-		   A 'true' value set in @dirtied flag means an 'undirtying'
-		   has to be done in POST-OP phase.
-		*/
-		gf_boolean_t  dirtied;
+        struct {
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } inode_wfop;  // common structure for all inode-write-fops
 
-		/* @inherited: flag which indicates that the dirty flags
-		   of the previous transaction were inherited
-		*/
-		gf_boolean_t  inherited;
+        struct {
+            struct iovec *vector;
+            struct iobref *iobref;
+            off_t offset;
+            int32_t op_ret;
+            int32_t count;
+            uint32_t flags;
+        } writev;
 
-		/*
-		  @no_uninherit: flag which indicates that a pre_op_uninherit()
-		  must _not_ be attempted (and returned as failure) always. This
-		  flag is set when a hard pre-op is performed, but not accounted
-		  for it in fd_ctx->on_disk[]. Such transactions are "isolated"
-		  from the pre-op piggybacking entirely and therefore uninherit
-		  must not be attempted.
-		*/
-		gf_boolean_t no_uninherit;
+        struct {
+            off_t offset;
+        } truncate;
 
-		/* @uninherit_done:
-		   @uninherit_value:
+        struct {
+            off_t offset;
+        } ftruncate;
 
-		   The above pair variables make pre_op_uninherit() idempotent.
-		   Both are FALSE initially. The first call to pre_op_uninherit
-		   sets @uninherit_done to TRUE and the return value to
-		   @uninherit_value. Further calls will check for @uninherit_done
-		   to be TRUE and if so will simply return @uninherit_value.
-		*/
-		gf_boolean_t uninherit_done;
-		gf_boolean_t uninherit_value;
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } setattr;
 
-		/* @changelog_resume: function to be called after changlogging
-		   (either pre-op or post-op) is done
-		*/
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } fsetattr;
 
-		afr_changelog_resume_t changelog_resume;
+        struct {
+            dict_t *dict;
+            int32_t flags;
+        } setxattr;
 
-                call_frame_t *main_frame;
+        struct {
+            dict_t *dict;
+            int32_t flags;
+        } fsetxattr;
 
-                int (*wind) (call_frame_t *frame, xlator_t *this, int subvol);
+        struct {
+            char *name;
+        } removexattr;
 
-                int (*fop) (call_frame_t *frame, xlator_t *this);
+        struct {
+            dict_t *xattr;
+            gf_xattrop_flags_t optype;
+        } xattrop;
 
-                int (*done) (call_frame_t *frame, xlator_t *this);
+        /* dir write */
 
-                int (*resume) (call_frame_t *frame, xlator_t *this);
+        struct {
+            inode_t *inode;
+            struct iatt buf;
+            struct iatt preparent;
+            struct iatt postparent;
+            struct iatt prenewparent;
+            struct iatt postnewparent;
+        } dir_fop;  // common structure for all dir fops
 
-                int (*unwind) (call_frame_t *frame, xlator_t *this);
+        struct {
+            fd_t *fd;
+            dict_t *params;
+            int32_t flags;
+            mode_t mode;
+        } create;
 
-                /* post-op hook */
-        } transaction;
+        struct {
+            dict_t *params;
+            dev_t dev;
+            mode_t mode;
+        } mknod;
 
-	syncbarrier_t barrier;
+        struct {
+            dict_t *params;
+            int32_t mode;
+        } mkdir;
 
-        /* extra data for fops */
-        dict_t         *xdata_req;
-        dict_t         *xdata_rsp;
+        struct {
+            dict_t *params;
+            char *linkpath;
+        } symlink;
 
-        dict_t         *xattr_rsp; /*for [f]xattrop*/
+        struct {
+            off_t offset;
+            size_t len;
+            int32_t mode;
+        } fallocate;
 
-        mode_t          umask;
-        int             xflag;
-        gf_boolean_t    do_discovery;
-	struct afr_reply *replies;
+        struct {
+            off_t offset;
+            size_t len;
+        } discard;
 
-        /* For  client side background heals. */
-        struct list_head healer;
-        call_frame_t *heal_frame;
+        struct {
+            off_t offset;
+            off_t len;
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } zerofill;
 
-        gf_boolean_t need_full_crawl;
-} afr_local_t;
+        struct {
+            char *volume;
+            int32_t cmd;
+            int32_t in_cmd;
+            struct gf_flock in_flock;
+            struct gf_flock flock;
+            void *xdata;
+        } inodelk;
 
+        struct {
+            char *volume;
+            char *basename;
+            void *xdata;
+            entrylk_cmd in_cmd;
+            entrylk_cmd cmd;
+            entrylk_type type;
+        } entrylk;
 
-typedef struct _afr_inode_ctx {
-        uint64_t        read_subvol;
-        int             spb_choice;
-        gf_timer_t      *timer;
-} afr_inode_ctx_t;
+        struct {
+            off_t offset;
+            gf_seek_what_t what;
+        } seek;
+
+        struct {
+            struct gf_lease user_lease;
+            struct gf_lease ret_lease;
+            unsigned char *locked_nodes;
+        } lease;
+
+        struct {
+            int flags;
+        } rmdir;
+
+        struct {
+            int32_t datasync;
+        } fsync;
+
+        struct {
+            uuid_t gfid_req;
+            gf_boolean_t needs_fresh_lookup;
+        } lookup;
+
+    } cont;
+
+    struct {
+        char *basename;
+        char *new_basename;
+
+        loc_t parent_loc;
+        loc_t new_parent_loc;
+
+        /* stub to resume on destruction
+           of the transaction frame */
+        call_stub_t *resume_stub;
+
+        struct list_head owner_list;
+        struct list_head wait_list;
+
+        unsigned char *pre_op;
+
+        /* Changelog xattr dict for [f]xattrop*/
+        dict_t **changelog_xdata;
+        unsigned char *pre_op_sources;
+
+        /* @failed_subvols: subvolumes on which a pre-op or a
+            FOP failed. */
+        unsigned char *failed_subvols;
+
+        call_frame_t *main_frame; /*Fop frame*/
+        call_frame_t *frame;      /*Transaction frame*/
+
+        int (*wind)(call_frame_t *frame, xlator_t *this, int subvol);
+
+        int (*unwind)(call_frame_t *frame, xlator_t *this);
+
+        off_t start, len;
+
+        afr_transaction_type type;
+
+        int32_t in_flight_sb_errno; /* This is where the cause of the
+                                       failure on the last good copy of
+                                       the file is stored.
+                                       */
+
+        /* @changelog_resume: function to be called after changlogging
+           (either pre-op or post-op) is done
+        */
+        afr_changelog_resume_t changelog_resume;
+
+        gf_boolean_t eager_lock_on;
+        gf_boolean_t do_eager_unlock;
+
+        /* @dirtied: flag which indicates whether we set dirty flag
+           in the OP. Typically true when we are performing operation
+           on more than one subvol and optimistic changelog is disabled
+
+           A 'true' value set in @dirtied flag means an 'undirtying'
+           has to be done in POST-OP phase.
+        */
+        gf_boolean_t dirtied;
+
+        /* @inherited: flag which indicates that the dirty flags
+           of the previous transaction were inherited
+        */
+        gf_boolean_t inherited;
+
+        /*
+          @no_uninherit: flag which indicates that a pre_op_uninherit()
+          must _not_ be attempted (and returned as failure) always. This
+          flag is set when a hard pre-op is performed, but not accounted
+          for it in fd_ctx->on_disk[]. Such transactions are "isolated"
+          from the pre-op piggybacking entirely and therefore uninherit
+          must not be attempted.
+        */
+        gf_boolean_t no_uninherit;
+
+        gf_boolean_t in_flight_sb; /* Indicator for occurrence of
+                                      split-brain while in the middle of
+                                      a txn. */
+
+        /* @uninherit_done:
+           @uninherit_value:
+
+           The above pair variables make pre_op_uninherit() idempotent.
+           Both are FALSE initially. The first call to pre_op_uninherit
+           sets @uninherit_done to TRUE and the return value to
+           @uninherit_value. Further calls will check for @uninherit_done
+           to be TRUE and if so will simply return @uninherit_value.
+        */
+        gf_boolean_t uninherit_done;
+        gf_boolean_t uninherit_value;
+
+        gf_boolean_t disable_delayed_post_op;
+    } transaction;
+
+    syncbarrier_t barrier;
+
+    /* extra data for fops */
+    dict_t *xdata_req;
+    dict_t *xdata_rsp;
+
+    dict_t *xattr_rsp; /*for [f]xattrop*/
+
+    mode_t umask;
+    int xflag;
+    struct afr_reply *replies;
+
+    /* For  client side background heals. */
+    struct list_head healer;
+    call_frame_t *heal_frame;
+
+    afr_inode_ctx_t *inode_ctx;
+
+    /*For thin-arbiter transactions.*/
+    int ta_failed_subvol;
+    int ta_event_gen;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+    afr_ta_fop_state_t fop_state;
+    afr_fop_lock_state_t fop_lock_state;
+    gf_lkowner_t saved_lk_owner;
+    unsigned char read_txn_query_child;
+    unsigned char ta_child_up;
+    gf_boolean_t do_discovery;
+    gf_boolean_t need_full_crawl;
+    gf_boolean_t is_read_txn;
+    gf_boolean_t is_new_entry;
+} afr_local_t;
 
 typedef struct afr_spbc_timeout {
-        call_frame_t *frame;
-        gf_boolean_t d_spb;
-        gf_boolean_t m_spb;
-        loc_t        *loc;
-        int          spb_child_index;
+    call_frame_t *frame;
+    loc_t *loc;
+    int spb_child_index;
+    gf_boolean_t d_spb;
+    gf_boolean_t m_spb;
 } afr_spbc_timeout_t;
 
 typedef struct afr_spb_status {
-        call_frame_t *frame;
-        loc_t        *loc;
+    call_frame_t *frame;
+    loc_t *loc;
 } afr_spb_status_t;
 
 typedef struct afr_empty_brick_args {
-        call_frame_t *frame;
-        loc_t loc;
-        int empty_index;
-        char *op_type;
+    call_frame_t *frame;
+    char *op_type;
+    loc_t loc;
+    int empty_index;
 } afr_empty_brick_args_t;
 
 typedef struct afr_read_subvol_args {
-        ia_type_t ia_type;
-        uuid_t gfid;
+    ia_type_t ia_type;
+    uuid_t gfid;
 } afr_read_subvol_args_t;
 
 typedef struct afr_granular_esh_args {
-        fd_t *heal_fd;
-        xlator_t *xl;
-        call_frame_t *frame;
-        gf_boolean_t mismatch; /* flag to represent occurrence of type/gfid
-                                  mismatch */
+    fd_t *heal_fd;
+    xlator_t *xl;
+    call_frame_t *frame;
+    gf_boolean_t mismatch; /* flag to represent occurrence of type/gfid
+                              mismatch */
 } afr_granular_esh_args_t;
 
-/* did a call fail due to a child failing? */
-#define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&            \
-                                           ((op_errno == ENOTCONN) ||   \
-                                            (op_errno == EBADFD)))
-
 int
-afr_inode_get_readable (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                        unsigned char *readable, int *event_p, int type);
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type);
 int
-afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			   unsigned char *data_subvols,
-			   unsigned char *metadata_subvols,
-			   int *event_generation);
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int *event_generation);
 int
-__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			     unsigned char *data_subvols,
-			     unsigned char *metadata_subvols,
-			     int *event_generation);
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvols,
+                            int *event_generation);
 
 int
-__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
-			     unsigned char *data_subvols,
-			     unsigned char *metadata_subvol,
-			     int event_generation);
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvol,
+                            int event_generation);
 int
-afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
-			   unsigned char *data_subvols,
-			   unsigned char *metadata_subvols,
-			   int event_generation);
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int event_generation);
 
 int
-afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this);
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
 int
-afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
-				  unsigned char *readable,
-                                  afr_read_subvol_args_t *args);
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
 int
-afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
-				unsigned char *readable, int *event_p,
-				int type);
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args);
+
+int
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type);
 int
-afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
-                     unsigned char *readables,
-		     int *event_p, afr_transaction_type type,
-                     afr_read_subvol_args_t *args);
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args);
 
-#define afr_data_subvol_get(i, t, s, r, e, a) \
-	afr_read_subvol_get(i, t, s, r, e, AFR_DATA_TRANSACTION, a)
+#define afr_data_subvol_get(i, t, s, r, e, a)                                  \
+    afr_read_subvol_get(i, t, s, r, e, AFR_DATA_TRANSACTION, a)
 
-#define afr_metadata_subvol_get(i, t, s, r, e, a) \
-	afr_read_subvol_get(i, t, s, r, e, AFR_METADATA_TRANSACTION, a)
+#define afr_metadata_subvol_get(i, t, s, r, e, a)                              \
+    afr_read_subvol_get(i, t, s, r, e, AFR_METADATA_TRANSACTION, a)
 
 int
-afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                   uuid_t gfid, afr_inode_refresh_cbk_t cbk);
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t cbk);
 
 int32_t
-afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2);
 
 int
-xattr_is_equal (dict_t *this, char *key1, data_t *value1, void *data);
+xattr_is_equal(dict_t *this, char *key1, data_t *value1, void *data);
 
 int
-afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
-                       loc_t *loc, char *basename, int child_count);
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count);
+
+int
+afr_add_inode_lockee(afr_local_t *local, int child_count);
 
 void
-afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
+afr_lockees_cleanup(afr_internal_lock_t *int_lock);
 
 int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
+afr_attempt_lock_recovery(xlator_t *this, int32_t child_index);
 
 int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes);
+afr_mark_locked_nodes(xlator_t *this, fd_t *fd, unsigned char *locked_nodes);
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner);
 
 int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this);
+afr_set_lock_number(call_frame_t *frame, xlator_t *this);
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this);
-
-int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this);
+afr_unlock(call_frame_t *frame, xlator_t *this);
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this);
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this);
 
 int
-afr_blocking_lock (call_frame_t *frame, xlator_t *this);
+afr_blocking_lock(call_frame_t *frame, xlator_t *this);
 
 int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this);
 
 int
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
-                          unsigned int child_count);
-
-int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd);
-
-int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd);
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd);
 
 afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno);
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count);
 
 int
-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                       gf_boolean_t *start_heal);
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal);
 
 void
-afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv);
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv);
 
 void
-afr_local_cleanup (afr_local_t *local, xlator_t *this);
+afr_local_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_frame_return (call_frame_t *frame);
+afr_frame_return(call_frame_t *frame);
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata);
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
 
 void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
-
-#define AFR_STACK_UNWIND(fop, frame, params ...)                \
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                if (frame) {                                    \
-                        __local = frame->local;                 \
-                        __this = frame->this;                   \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0)
-
-#define AFR_STACK_DESTROY(frame)                                \
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                __local = frame->local;                         \
-                __this = frame->this;                           \
-                frame->local = NULL;                            \
-                STACK_DESTROY (frame->root);                    \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0);
-
-#define AFR_FRAME_INIT(frame, op_errno)				       \
-	({frame->local = mem_get0 (THIS->local_pool);		       \
-	if (afr_local_init (frame->local, THIS->private, &op_errno)) { \
-		afr_local_cleanup (frame->local, THIS);		       \
-		mem_put (frame->local);				       \
-		frame->local = NULL; };				       \
-	frame->local;})
-
-#define AFR_STACK_RESET(frame)                                         \
-        do {                                                           \
-                afr_local_t *__local = NULL;                           \
-                xlator_t    *__this = NULL;                            \
-                __local = frame->local;                                \
-                __this = frame->this;                                  \
-                frame->local = NULL;                                   \
-                int __opr;                                             \
-                STACK_RESET (frame->root);                             \
-                if (__local) {                                         \
-                        afr_local_cleanup (__local, __this);           \
-                        mem_put (__local);                             \
-                }                                                      \
-                AFR_FRAME_INIT (frame, __opr);                         \
-        } while (0)
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd);
+
+#define AFR_STACK_UNWIND(fop, frame, op_ret, op_errno, params...)              \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+                                                                               \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __this = frame->this;                                              \
+            afr_handle_inconsistent_fop(frame, &__op_ret, &__op_errno);        \
+            if (__local && __local->is_read_txn)                               \
+                afr_pending_read_decrement(__this->private,                    \
+                                           __local->read_subvol);              \
+            if (__local && __local->xdata_req &&                               \
+                afr_is_lock_mode_mandatory(__local->xdata_req))                \
+                afr_dom_lock_release(frame);                                   \
+            frame->local = NULL;                                               \
+        }                                                                      \
+                                                                               \
+        STACK_UNWIND_STRICT(fop, frame, __op_ret, __op_errno, params);         \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
+
+#define AFR_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0);
+
+#define AFR_FRAME_INIT(frame, op_errno)                                        \
+    ({                                                                         \
+        frame->local = mem_get0(THIS->local_pool);                             \
+        if (afr_local_init(frame->local, frame->this->private, &op_errno)) {   \
+            afr_local_cleanup(frame->local, frame->this);                      \
+            mem_put(frame->local);                                             \
+            frame->local = NULL;                                               \
+        };                                                                     \
+        frame->local;                                                          \
+    })
+
+#define AFR_STACK_RESET(frame)                                                 \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        int __opr;                                                             \
+        STACK_RESET(frame->root);                                              \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+        AFR_FRAME_INIT(frame, __opr);                                          \
+    } while (0)
 
 /* allocate and return a string that is the basename of argument */
 static inline char *
-AFR_BASENAME (const char *str)
+AFR_BASENAME(const char *str)
 {
-        char *__tmp_str = NULL;
-        char *__basename_str = NULL;
-        __tmp_str = gf_strdup (str);
-        __basename_str = gf_strdup (basename (__tmp_str));
-        GF_FREE (__tmp_str);
-        return __basename_str;
+    char *__tmp_str = NULL;
+    char *__basename_str = NULL;
+    __tmp_str = gf_strdup(str);
+    __basename_str = gf_strdup(basename(__tmp_str));
+    GF_FREE(__tmp_str);
+    return __basename_str;
 }
 
 call_frame_t *
-afr_copy_frame (call_frame_t *base);
+afr_copy_frame(call_frame_t *base);
 
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this);
+afr_transaction_local_init(afr_local_t *local, xlator_t *this);
 
 int32_t
-afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
-                     loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
+afr_marker_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, afr_local_t *local, afr_private_t *priv);
 
 int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
 int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
-                        transaction_lk_type_t lk_type);
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count);
 
 int
-afr_higher_errno (int32_t old_errno, int32_t new_errno);
+afr_higher_errno(int32_t old_errno, int32_t new_errno);
 
 int
-afr_final_errno (afr_local_t *local, afr_private_t *priv);
+afr_final_errno(afr_local_t *local, afr_private_t *priv);
 
 int
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req);
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req);
 
 void
-afr_fix_open (fd_t *fd, xlator_t *this);
+afr_fix_open(fd_t *fd, xlator_t *this);
 
 afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 void
-afr_set_low_priority (call_frame_t *frame);
+afr_set_low_priority(call_frame_t *frame);
 int
-afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags);
+afr_child_fd_ctx_set(xlator_t *this, fd_t *fd, int32_t child, int flags);
 
 void
-afr_matrix_cleanup (int32_t **pending, unsigned int m);
+afr_matrix_cleanup(int32_t **pending, unsigned int m);
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n);
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n);
 
-int**
-afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending,
-                            dict_t *xattr, ia_type_t iat);
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat);
 
 void
-afr_filter_xattrs (dict_t *xattr);
+afr_filter_xattrs(dict_t *xattr);
 
 /*
  * Special value indicating we should use the "auto" quorum method instead of
@@ -1082,71 +1245,179 @@ afr_filter_xattrs (dict_t *xattr);
 #define AFR_QUORUM_AUTO INT_MAX
 
 int
-afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode);
+
+void
+afr_reply_wipe(struct afr_reply *reply);
+
+void
+afr_replies_wipe(struct afr_reply *replies, int count);
+
+gf_boolean_t
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2);
+
+gf_boolean_t
+afr_is_xattr_ignorable(char *key);
+
+int
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
+
+int
+afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc);
+
+int
+afr_get_split_brain_status(void *opaque);
+
+int
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque);
+
+int
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                 int spb_choice);
+int
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol);
+int
+afr_get_child_index_from_name(xlator_t *this, char *name);
+
+int
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb);
+int
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode);
+
+int
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque);
 
 gf_boolean_t
-afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
+afr_get_need_heal(xlator_t *this);
 
 void
-afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
+afr_set_need_heal(xlator_t *this, afr_local_t *local);
+
+int
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd);
+
+int
+afr_get_msg_id(char *op_type);
 
 int
-afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame,
+                            inode_t *inode);
 
+int32_t
+afr_quorum_errno(afr_private_t *priv);
+
+gf_boolean_t
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno);
 void
-afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno);
 
 void
-afr_remove_eager_lock_stub (afr_local_t *local);
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
+void
+afr_process_post_writev(call_frame_t *frame, xlator_t *this);
 
 void
-afr_replies_wipe (struct afr_reply *replies, int count);
+afr_writev_unwind(call_frame_t *frame, xlator_t *this);
 
-gf_boolean_t
-afr_xattrs_are_equal (dict_t *dict1, dict_t *dict2);
+void
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame);
+
+void
+afr_update_uninodelk(afr_local_t *local, afr_internal_lock_t *int_lock,
+                     int32_t child_index);
+afr_fd_ctx_t *
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 gf_boolean_t
-afr_is_xattr_ignorable (char *key);
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2);
 
 int
-afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter);
+gf_boolean_t
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this);
 
 int
-afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc);
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx);
 
-int
-afr_get_split_brain_status (void *opaque);
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this);
 
 int
-afr_get_split_brain_status_cbk (int ret, call_frame_t *frame, void *opaque);
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this);
 
 int
-afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                  int spb_choice);
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this);
+
 int
-afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-                                  int *spb_choice);
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode);
+
 int
-afr_get_child_index_from_name (xlator_t *this, char *name);
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop);
 
 int
-afr_is_split_brain (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                    uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb);
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc);
+
 int
-afr_spb_choice_timeout_cancel (xlator_t *this, inode_t *inode);
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc);
+
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type);
 
 int
-afr_set_split_brain_choice (int ret, call_frame_t *frame, void *opaque);
+__afr_get_up_children_count(afr_private_t *priv);
+
+call_frame_t *
+afr_ta_frame_create(xlator_t *this);
 
 gf_boolean_t
-afr_get_need_heal (xlator_t *this);
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local);
 
 void
-afr_set_need_heal (xlator_t *this, afr_local_t *local);
+afr_ta_lock_release_synctask(xlator_t *this);
 
-int
-afr_selfheal_data_open (xlator_t *this, inode_t *inode, fd_t **fd);
+void
+afr_ta_locked_priv_invalidate(afr_private_t *priv);
 
-int
-afr_get_msg_id (char *op_type);
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame,
+                      const unsigned int up_children_count);
+
+void
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this);
+
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv,
+                                   int child);
+
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv);
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata);
+
+void
+afr_dom_lock_release(call_frame_t *frame);
+
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies);
+
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid);
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
deleted file mode 100644
index ef299ec5855..00000000000
--- a/xlators/cluster/afr/src/pump.c
+++ /dev/null
@@ -1,2470 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <unistd.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <fnmatch.h>
-
-#include "afr-common.c"
-#include "defaults.h"
-#include "glusterfs.h"
-#include "pump.h"
-#include "afr-messages.h"
-
-
-static int
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
-{
-        int ret       = 0;
-        uuid_t *pgfid = NULL;
-
-        GF_ASSERT (gfid);
-
-        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!pgfid) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (*pgfid, gfid);
-
-        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_SET_FAILED, "gfid set failed");
-
-out:
-        if (ret && pgfid)
-                GF_FREE (pgfid);
-	return ret;
-}
-
-static int
-afr_set_root_gfid (dict_t *dict)
-{
-        uuid_t gfid;
-        int ret = 0;
-
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
-
-        ret = afr_set_dict_gfid (dict, gfid);
-
-        return ret;
-}
-
-static int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
-{
-        int   ret = -1;
-        uuid_t pargfid = {0};
-
-        if (!child)
-                goto out;
-
-        if (!gf_uuid_is_null (parent->inode->gfid))
-                gf_uuid_copy (pargfid, parent->inode->gfid);
-        else if (!gf_uuid_is_null (parent->gfid))
-                gf_uuid_copy (pargfid, parent->gfid);
-
-        if (gf_uuid_is_null (pargfid))
-                goto out;
-
-        if (strcmp (parent->path, "/") == 0)
-                ret = gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
-                                   name);
-
-        if (-1 == ret) {
-        }
-
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
-
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
-        gf_uuid_copy (child->pargfid, pargfid);
-
-        if (!child->inode) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        if ((ret == -1) && child)
-                loc_wipe (child);
-
-        return ret;
-}
-
-static void
-afr_build_root_loc (xlator_t *this, loc_t *loc)
-{
-        afr_private_t   *priv = NULL;
-
-        priv = this->private;
-        loc->path = gf_strdup ("/");
-        loc->name = "";
-        loc->inode = inode_ref (priv->root_inode);
-        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-}
-
-static void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
-{
-        GF_ASSERT (loc);
-        GF_ASSERT (buf);
-
-        gf_uuid_copy (loc->gfid, buf->ia_gfid);
-        if (postparent)
-                gf_uuid_copy (loc->pargfid, postparent->ia_gfid);
-}
-
-static uint64_t pump_pid = 0;
-static void
-pump_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent)
-{
-        afr_update_loc_gfids (loc, iatt, parent);
-        gf_uuid_copy (loc->inode->gfid, iatt->ia_gfid);
-}
-
-static int
-pump_mark_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        pump_priv->pump_start_pending = 1;
-
-        return 0;
-}
-
-static int
-is_pump_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        return (pump_priv->pump_start_pending);
-}
-
-static int
-pump_remove_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        pump_priv->pump_start_pending = 0;
-
-        return 0;
-}
-
-static pump_state_t
-pump_get_state ()
-{
-        xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        pump_state_t ret;
-
-        this = THIS;
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->pump_state_lock);
-        {
-                ret = pump_priv->pump_state;
-        }
-        UNLOCK (&pump_priv->pump_state_lock);
-
-        return ret;
-}
-
-int
-pump_change_state (xlator_t *this, pump_state_t state)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        pump_state_t state_old;
-        pump_state_t state_new;
-
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (pump_priv);
-
-        LOCK (&pump_priv->pump_state_lock);
-        {
-                state_old = pump_priv->pump_state;
-                state_new = state;
-
-                pump_priv->pump_state = state;
-
-        }
-        UNLOCK (&pump_priv->pump_state_lock);
-
-        gf_msg_debug (this->name, 0,
-                      "Pump changing state from %d to %d",
-                      state_old, state_new);
-
-        return  0;
-}
-
-static int
-pump_set_resume_path (xlator_t *this, const char *path)
-{
-        int ret = 0;
-
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (pump_priv);
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                strncpy (pump_priv->resume_path, path, strlen (path) + 1);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        return ret;
-}
-
-static int
-pump_save_path (xlator_t *this, const char *path)
-{
-        afr_private_t *priv = NULL;
-        pump_state_t state;
-        dict_t *dict = NULL;
-        loc_t  loc = {0};
-        int dict_ret = 0;
-        int ret = -1;
-
-        state = pump_get_state ();
-        if (state == PUMP_STATE_RESUME)
-                return 0;
-
-        priv = this->private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        dict = dict_new ();
-        dict_ret = dict_set_str (dict, PUMP_PATH, (char *)path);
-        if (dict_ret)
-                gf_msg (this->name, GF_LOG_WARNING,
-                        -dict_ret, AFR_MSG_DICT_SET_FAILED,
-                        "%s: failed to set the key %s", path, PUMP_PATH);
-
-        ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0, NULL,
-                               NULL);
-
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, -ret, AFR_MSG_INFO_COMMON,
-                        "setxattr failed - could not save path=%s", path);
-        } else {
-                gf_msg_debug (this->name, 0,
-                              "setxattr succeeded - saved path=%s", path);
-        }
-
-        dict_unref (dict);
-
-        loc_wipe (&loc);
-        return 0;
-}
-
-static int
-pump_check_and_update_status (xlator_t *this)
-{
-        pump_state_t state;
-        int ret = -1;
-
-        state = pump_get_state ();
-
-        switch (state) {
-
-        case PUMP_STATE_RESUME:
-        case PUMP_STATE_RUNNING:
-        {
-                ret = 0;
-                break;
-        }
-        case PUMP_STATE_PAUSE:
-        {
-                ret = -1;
-                break;
-        }
-        case PUMP_STATE_ABORT:
-        {
-                pump_save_path (this, "/");
-                ret = -1;
-                break;
-        }
-        default:
-        {
-                gf_msg_debug (this->name, 0,
-                              "Unknown pump state");
-                ret = -1;
-                break;
-        }
-
-        }
-
-        return ret;
-}
-
-static const char *
-pump_get_resume_path (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        const char *resume_path = NULL;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        resume_path = pump_priv->resume_path;
-
-        return resume_path;
-}
-
-static int
-pump_update_resume_state (xlator_t *this, const char *path)
-{
-        pump_state_t state;
-        const char *resume_path = NULL;
-
-        state = pump_get_state ();
-
-        if (state == PUMP_STATE_RESUME) {
-                resume_path = pump_get_resume_path (this);
-                if (strcmp (resume_path, "/") == 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "Reached the resume path (/). Proceeding to change state"
-                                      " to running");
-
-                        pump_change_state (this, PUMP_STATE_RUNNING);
-                } else if (strcmp (resume_path, path) == 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "Reached the resume path. Proceeding to change state"
-                                      " to running");
-
-                        pump_change_state (this, PUMP_STATE_RUNNING);
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                      "Not yet hit the resume path:res-path=%s,path=%s",
-                                      resume_path, path);
-                }
-        }
-
-        return 0;
-}
-
-static gf_boolean_t
-is_pump_traversal_allowed (xlator_t *this, const char *path)
-{
-        pump_state_t state;
-        const char *resume_path = NULL;
-        gf_boolean_t ret = _gf_true;
-
-        state = pump_get_state ();
-
-        if (state == PUMP_STATE_RESUME) {
-                resume_path = pump_get_resume_path (this);
-                if (strstr (resume_path, path)) {
-                        gf_msg_debug (this->name, 0,
-                                      "On the right path to resumption path");
-                        ret = _gf_true;
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                      "Not the right path to resuming=> ignoring traverse");
-                        ret = _gf_false;
-                }
-        }
-
-        return ret;
-}
-
-static int
-pump_save_file_stats (xlator_t *this, const char *path)
-{
-        afr_private_t  *priv        = NULL;
-        pump_private_t *pump_priv   = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                pump_priv->number_files_pumped++;
-
-                strncpy (pump_priv->current_file, path,
-                         PATH_MAX);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        return 0;
-}
-
-static int
-gf_pump_traverse_directory (loc_t *loc)
-{
-        xlator_t        *this              = NULL;
-        fd_t            *fd                = NULL;
-        off_t           offset             = 0;
-        loc_t           entry_loc          = {0};
-        gf_dirent_t     *entry             = NULL;
-        gf_dirent_t     *tmp               = NULL;
-        gf_dirent_t     entries;
-	struct iatt     iatt               = {0};
-        struct iatt     parent             = {0};
-	dict_t          *xattr_rsp         = NULL;
-        int             ret                = 0;
-        gf_boolean_t    is_directory_empty = _gf_true;
-        gf_boolean_t    free_entries       = _gf_false;
-
-        INIT_LIST_HEAD (&entries.list);
-        this = THIS;
-
-        GF_ASSERT (loc->inode);
-
-	fd = fd_create (loc->inode, pump_pid);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_FD_CREATE_FAILED,
-                        "Failed to create fd for %s", loc->path);
-                goto out;
-        }
-
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "opendir failed on %s", loc->path);
-                goto out;
-        }
-
-        gf_msg_trace (this->name, 0,
-                      "pump opendir on %s returned=%d",
-                      loc->path, ret);
-
-        while (syncop_readdirp (this, fd, 131072, offset, &entries, NULL,
-                                NULL)) {
-                free_entries = _gf_true;
-
-                if (list_empty (&entries.list)) {
-                        gf_msg_trace (this->name, 0,
-                                      "no more entries in directory");
-                        goto out;
-                }
-
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        gf_msg_debug (this->name, 0,
-                                      "found readdir entry=%s", entry->d_name);
-
-                        offset = entry->d_off;
-                        if (gf_uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        AFR_MSG_GFID_NULL, "%s/%s: No "
-                                        "gfid present skipping",
-                                        loc->path, entry->d_name);
-                                continue;
-                        }
-                        loc_wipe (&entry_loc);
-                        ret = afr_build_child_loc (this, &entry_loc, loc,
-                                                   entry->d_name);
-                        if (ret)
-                                goto out;
-
-			if ((strcmp (entry->d_name, ".") == 0) ||
-			    (strcmp (entry->d_name, "..") == 0))
-				continue;
-
-			is_directory_empty = _gf_false;
-	                gf_msg_debug (this->name, 0,
-	                              "lookup %s => %"PRId64,
-                                      entry_loc.path,
-                                      iatt.ia_ino);
-
-			ret = syncop_lookup (this, &entry_loc, &iatt, &parent,
-					     NULL, &xattr_rsp);
-
-			if (ret) {
-			        gf_msg (this->name, GF_LOG_ERROR,
-                                        -ret, AFR_MSG_INFO_COMMON,
-			                "%s: lookup failed", entry_loc.path);
-				continue;
-			}
-
-			ret = afr_selfheal_name (this, loc->gfid, entry->d_name,
-                                                 NULL);
-			if (ret) {
-			        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        AFR_MSG_SELF_HEAL_FAILED,
-			                "%s: name self-heal failed (%s/%s)",
-			                entry_loc.path, uuid_utoa (loc->gfid),
-				        entry->d_name);
-				continue;
-			}
-
-			ret = afr_selfheal (this, iatt.ia_gfid);
-			if (ret < 0) {
-			        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        AFR_MSG_SELF_HEAL_FAILED,
-				        "%s: self-heal failed (%s)",
-				        entry_loc.path,
-                                        uuid_utoa (iatt.ia_gfid));
-				continue;
-			}
-
-			pump_fill_loc_info (&entry_loc, &iatt, &parent);
-
-			pump_update_resume_state (this, entry_loc.path);
-
-			pump_save_path (this, entry_loc.path);
-			pump_save_file_stats (this, entry_loc.path);
-
-			ret = pump_check_and_update_status (this);
-			if (ret < 0) {
-			        gf_msg_debug (this->name, 0,
-			                      "Pump beginning to exit out");
-				goto out;
-			}
-
-			if (IA_ISDIR (iatt.ia_type)) {
-				if (is_pump_traversal_allowed (this, entry_loc.path)) {
-				        gf_msg_trace (this->name, 0,
-				                      "entering dir=%s",
-                                                      entry->d_name);
-					gf_pump_traverse_directory (&entry_loc);
-				}
-                        }
-                }
-
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                gf_msg_trace (this->name, 0, "offset incremented to %d",
-                              (int32_t) offset);
-
-        }
-
-        ret = syncop_close (fd);
-        if (ret < 0)
-                gf_msg_debug (this->name, 0, "closing the fd failed");
-
-        if (is_directory_empty && (strcmp (loc->path, "/") == 0)) {
-               pump_change_state (this, PUMP_STATE_RUNNING);
-                gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON,
-                       "Empty source brick. Nothing to be done.");
-        }
-
-out:
-        if (entry_loc.path)
-                loc_wipe (&entry_loc);
-        if (free_entries)
-                gf_dirent_free (&entries);
-        return 0;
-}
-
-static int
-pump_update_resume_path (xlator_t *this)
-{
-        const char *resume_path = NULL;
-
-        resume_path = pump_get_resume_path (this);
-
-        if (resume_path) {
-                gf_msg_debug (this->name, 0,
-                              "Found a path to resume from: %s",
-                              resume_path);
-
-        }else {
-                gf_msg_debug (this->name, 0,
-                              "Did not find a path=> setting to '/'");
-                pump_set_resume_path (this, "/");
-        }
-
-        pump_change_state (this, PUMP_STATE_RESUME);
-
-        return 0;
-}
-
-static int32_t
-pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_private_t  *priv      = NULL;
-        loc_t           loc       = {0};
-        int             i         = 0;
-        int             ret       = 0;
-        int             source    = 0;
-        int             sink      = 1;
-
-        priv      = this->private;
-
-        afr_build_root_loc (this, &loc);
-
-        ret = syncop_removexattr (priv->children[source], &loc,
-				  PUMP_PATH, 0, NULL);
-
-        ret = syncop_removexattr (priv->children[sink], &loc,
-                                  PUMP_SINK_COMPLETE, 0, NULL);
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = syncop_removexattr (priv->children[i], &loc,
-                                          PUMP_SOURCE_COMPLETE, 0, NULL);
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "removexattr "
-                                     "failed with %s", strerror (-ret));
-                }
-        }
-
-        loc_wipe (&loc);
-        return pump_command_reply (frame, this);
-}
-
-static int
-pump_complete_migration (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-        dict_t *dict = NULL;
-        pump_state_t state;
-        loc_t  loc = {0};
-        int dict_ret = 0;
-        int ret = -1;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        dict = dict_new ();
-
-        state = pump_get_state ();
-        if (state == PUMP_STATE_RUNNING) {
-                gf_msg_debug (this->name, 0,
-                              "Pump finished pumping");
-
-                pump_priv->pump_finished = _gf_true;
-
-                dict_ret = dict_set_str (dict, PUMP_SOURCE_COMPLETE, "jargon");
-                if (dict_ret)
-                        gf_msg (this->name, GF_LOG_WARNING, -dict_ret,
-                                AFR_MSG_DICT_SET_FAILED,
-                                "%s: failed to set the key %s",
-                                loc.path, PUMP_SOURCE_COMPLETE);
-
-                ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0,
-                                       NULL, NULL);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "setxattr failed - while "
-                                      "notifying source complete");
-                }
-                dict_ret = dict_set_str (dict, PUMP_SINK_COMPLETE, "jargon");
-                if (dict_ret)
-                        gf_msg (this->name, GF_LOG_WARNING, -dict_ret,
-                                AFR_MSG_DICT_SET_FAILED,
-                                "%s: failed to set the key %s",
-                                loc.path, PUMP_SINK_COMPLETE);
-
-                ret = syncop_setxattr (PUMP_SINK_CHILD (this), &loc, dict, 0,
-                                       NULL, NULL);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "setxattr failed - while "
-                                      "notifying sink complete");
-                }
-
-                pump_save_path (this, "/");
-
-        } else if (state == PUMP_STATE_ABORT) {
-                gf_msg_debug (this->name, 0, "Starting cleanup "
-                              "of pump internal xattrs");
-                call_resume (pump_priv->cleaner);
-        }
-
-        loc_wipe (&loc);
-        return 0;
-}
-
-static int
-pump_lookup_sink (loc_t *loc)
-{
-        xlator_t *this = NULL;
-	struct iatt iatt, parent;
-	dict_t *xattr_rsp;
-        dict_t *xattr_req = NULL;
-        int ret = 0;
-
-        this = THIS;
-
-        xattr_req = dict_new ();
-
-        ret = afr_set_root_gfid (xattr_req);
-        if (ret)
-                goto out;
-
-        ret = syncop_lookup (PUMP_SINK_CHILD (this), loc, &iatt, &parent,
-                             xattr_req, &xattr_rsp);
-
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "Lookup on sink child failed");
-                ret = -1;
-                goto out;
-        }
-
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        return ret;
-}
-
-static int
-pump_task (void *data)
-{
-	xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-
-
-        loc_t loc = {0};
-	struct iatt iatt, parent;
-	dict_t *xattr_rsp = NULL;
-        dict_t *xattr_req = NULL;
-
-        int ret = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                gf_msg_debug (this->name, ENOMEM,
-                              "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        afr_set_root_gfid (xattr_req);
-        ret = syncop_lookup (this, &loc, &iatt, &parent,
-                             xattr_req, &xattr_rsp);
-
-        gf_msg_trace (this->name, 0,
-                      "lookup: path=%s gfid=%s",
-                      loc.path, uuid_utoa (loc.inode->gfid));
-
-        ret = pump_check_and_update_status (this);
-        if (ret < 0) {
-                goto out;
-        }
-
-        pump_update_resume_path (this);
-
-        afr_set_root_gfid (xattr_req);
-        ret = pump_lookup_sink (&loc);
-        if (ret) {
-                pump_update_resume_path (this);
-                goto out;
-        }
-
-        gf_pump_traverse_directory (&loc);
-
-        pump_complete_migration (this);
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        loc_wipe (&loc);
-	return 0;
-}
-
-
-static int
-pump_task_completion (int ret, call_frame_t *sync_frame, void *data)
-{
-        xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-
-        this = THIS;
-
-        priv = this->private;
-
-        inode_unref (priv->root_inode);
-        STACK_DESTROY (sync_frame->root);
-
-        gf_msg_debug (this->name, 0,
-                      "Pump xlator exiting");
-	return 0;
-}
-
-int
-pump_start (call_frame_t *pump_frame, xlator_t *this)
-{
-	afr_private_t *priv = NULL;
-	pump_private_t *pump_priv = NULL;
-
-	int ret = -1;
-
-	priv = this->private;
-        pump_priv = priv->pump_private;
-
-        afr_set_lk_owner (pump_frame, this, pump_frame->root);
-	pump_pid = (uint64_t) (unsigned long)pump_frame->root;
-
-	ret = synctask_new (pump_priv->env, pump_task,
-                            pump_task_completion,
-                            pump_frame, NULL);
-        if (ret == -1) {
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "setting pump as started lk_owner: %s %"PRIu64,
-                      lkowner_utoa (&pump_frame->root->lk_owner), pump_pid);
-
-        priv->use_afr_in_pump = 1;
-out:
-	return ret;
-}
-
-static int
-pump_start_synctask (xlator_t *this)
-{
-        call_frame_t *frame = NULL;
-        int ret = 0;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        pump_change_state (this, PUMP_STATE_RUNNING);
-
-        ret = pump_start (frame, this);
-
-out:
-        return ret;
-}
-
-int32_t
-pump_cmd_start_setxattr_cbk (call_frame_t *frame,
-                             void *cookie,
-                             xlator_t *this,
-                             int32_t op_ret,
-                             int32_t op_errno, dict_t *xdata)
-
-{
-        call_frame_t *prev = NULL;
-        afr_local_t *local = NULL;
-        int ret = 0;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_INFO_COMMON,
-                        "Could not initiate destination "
-                        "brick connect");
-                ret = op_ret;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "Successfully initiated destination "
-                      "brick connect");
-
-        pump_mark_start_pending (this);
-
-        /* send the PARENT_UP as pump is ready now */
-        prev = cookie;
-        if (prev && prev->this)
-                prev->this->notify (prev->this, GF_EVENT_PARENT_UP, this);
-
-out:
-        local->op_ret = ret;
-        pump_command_reply (frame, this);
-
-        return 0;
-}
-
-static int
-pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local     = NULL;
-        afr_private_t *priv      = NULL;
-        dict_t        *dict      = NULL;
-        data_t        *data      = NULL;
-        char          *clnt_cmd  = NULL;
-        loc_t loc = {0};
-
-        int ret = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        data = data_ref (dict_get (local->dict, RB_PUMP_CMD_START));
-        if (!data) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        AFR_MSG_DICT_GET_FAILED,
-                        "Could not get destination brick value");
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        clnt_cmd = GF_CALLOC (1, data->len+1, gf_common_mt_char);
-        if (!clnt_cmd) {
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (clnt_cmd, data->data, data->len);
-        clnt_cmd[data->len] = '\0';
-        gf_msg_debug (this->name, 0, "Got destination brick %s\n",
-                      clnt_cmd);
-
-        ret = dict_set_dynstr (dict, CLIENT_CMD_CONNECT, clnt_cmd);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_SET_FAILED,
-                        "Could not inititiate destination brick "
-                        "connect");
-                goto out;
-        }
-
-	STACK_WIND (frame,
-		    pump_cmd_start_setxattr_cbk,
-		    PUMP_SINK_CHILD(this),
-		    PUMP_SINK_CHILD(this)->fops->setxattr,
-		    &loc,
-		    dict,
-		    0, NULL);
-
-        ret = 0;
-
-out:
-        if (dict)
-                dict_unref (dict);
-
-        if (data)
-                data_unref (data);
-
-        if (ret && clnt_cmd)
-                GF_FREE (clnt_cmd);
-
-        loc_wipe (&loc);
-        return ret;
-}
-
-static int
-is_pump_aborted (xlator_t *this)
-{
-        pump_state_t state;
-
-        state = pump_get_state ();
-
-        return ((state == PUMP_STATE_ABORT));
-}
-
-int32_t
-pump_cmd_start_getxattr_cbk (call_frame_t *frame,
-                             void *cookie,
-                             xlator_t *this,
-                             int32_t op_ret,
-                             int32_t op_errno,
-                             dict_t *dict, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        char *path = NULL;
-
-        pump_state_t state;
-        int ret = 0;
-        int need_unwind = 0;
-        int dict_ret = -1;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "getxattr failed - changing pump "
-                              "state to RUNNING with '/'");
-                path = "/";
-                ret = op_ret;
-        } else {
-                gf_msg_trace (this->name, 0,
-                              "getxattr succeeded");
-
-                dict_ret =  dict_get_str (dict, PUMP_PATH, &path);
-                if (dict_ret < 0)
-                        path = "/";
-        }
-
-        state = pump_get_state ();
-        if ((state == PUMP_STATE_RUNNING) ||
-            (state == PUMP_STATE_RESUME)) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        0, AFR_MSG_PUMP_XLATOR_ERROR,
-                        "Pump is already started");
-                ret = -1;
-                goto out;
-        }
-
-        pump_set_resume_path (this, path);
-
-        if (is_pump_aborted (this))
-                /* We're re-starting pump afresh */
-                ret = pump_initiate_sink_connect (frame, this);
-        else {
-                /* We're re-starting pump from a previous
-                   pause */
-                gf_msg_debug (this->name, 0,
-                              "about to start synctask");
-                ret = pump_start_synctask (this);
-                need_unwind = 1;
-        }
-
-out:
-        if ((ret < 0) || (need_unwind == 1)) {
-                local->op_ret = ret;
-                pump_command_reply (frame, this);
-        }
-	return 0;
-}
-
-int
-pump_execute_status (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        uint64_t number_files = 0;
-
-        char filename[PATH_MAX];
-        char summary[PATH_MAX+256];
-        char *dict_str = NULL;
-
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        dict_t *dict = NULL;
-        int ret = -1;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                number_files  = pump_priv->number_files_pumped;
-                strncpy (filename, pump_priv->current_file, PATH_MAX);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        dict_str     = GF_CALLOC (1, PATH_MAX + 256, gf_afr_mt_char);
-        if (!dict_str) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        if (pump_priv->pump_finished) {
-                snprintf (summary, PATH_MAX+256,
-                          "no_of_files=%"PRIu64, number_files);
-        } else {
-                snprintf (summary, PATH_MAX+256,
-                          "no_of_files=%"PRIu64":current_file=%s",
-                          number_files, filename);
-        }
-        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
-                  (pump_priv->pump_finished)?1:0, summary);
-
-        dict = dict_new ();
-
-        ret = dict_set_dynstr (dict, RB_PUMP_CMD_STATUS, dict_str);
-        if (ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "dict_set_dynstr returned negative value");
-        } else {
-                dict_str = NULL;
-        }
-
-        op_ret = 0;
-
-out:
-
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
-
-        if (dict)
-                dict_unref (dict);
-
-        GF_FREE (dict_str);
-
-        return 0;
-}
-
-int
-pump_execute_pause (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        pump_change_state (this, PUMP_STATE_PAUSE);
-
-        local->op_ret = 0;
-        pump_command_reply (frame, this);
-
-        return 0;
-}
-
-int
-pump_execute_start (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t   *local = NULL;
-
-        int ret = 0;
-        loc_t loc = {0};
-
-        priv = this->private;
-        local = frame->local;
-
-        if (!priv->root_inode) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        0, AFR_MSG_PUMP_XLATOR_ERROR,
-                        "Pump xlator cannot be started without an initial "
-                        "lookup");
-                ret = -1;
-                goto out;
-        }
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-	STACK_WIND (frame,
-		    pump_cmd_start_getxattr_cbk,
-		    PUMP_SOURCE_CHILD(this),
-		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
-		    &loc,
-		    PUMP_PATH, NULL);
-
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                local->op_ret = ret;
-                pump_command_reply (frame, this);
-        }
-
-        loc_wipe (&loc);
-	return 0;
-}
-
-static int
-pump_cleanup_helper (void *data) {
-        call_frame_t *frame = data;
-
-        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
-
-        return 0;
-}
-
-static int
-pump_cleanup_done (int ret, call_frame_t *sync_frame, void *data)
-{
-        STACK_DESTROY (sync_frame->root);
-
-        return 0;
-}
-
-int
-pump_execute_commit (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t  *priv       = NULL;
-        pump_private_t *pump_priv  = NULL;
-        afr_local_t    *local      = NULL;
-        call_frame_t   *sync_frame = NULL;
-        int             ret        = 0;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-        local     = frame->local;
-
-        local->op_ret = 0;
-        if (pump_priv->pump_finished) {
-                pump_change_state (this, PUMP_STATE_COMMIT);
-                sync_frame = create_frame (this, this->ctx->pool);
-                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
-                                    pump_cleanup_done, sync_frame, frame);
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "Couldn't create "
-                                      "synctask for cleaning up xattrs.");
-                }
-
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, EINPROGRESS,
-                        AFR_MSG_MIGRATION_IN_PROGRESS,
-                        "Commit can't proceed. Migration in progress");
-                local->op_ret = -1;
-                local->op_errno = EINPROGRESS;
-                pump_command_reply (frame, this);
-        }
-
-        return 0;
-}
-int
-pump_execute_abort (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t  *priv       = NULL;
-        pump_private_t *pump_priv  = NULL;
-        afr_local_t    *local      = NULL;
-        call_frame_t   *sync_frame = NULL;
-        int             ret        = 0;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-        local     = frame->local;
-
-        pump_change_state (this, PUMP_STATE_ABORT);
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                pump_priv->number_files_pumped = 0;
-                pump_priv->current_file[0] = '\0';
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        local->op_ret = 0;
-        if (pump_priv->pump_finished) {
-                sync_frame = create_frame (this, this->ctx->pool);
-                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
-                                    pump_cleanup_done, sync_frame, frame);
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "Couldn't create "
-                                      "synctask for cleaning up xattrs.");
-                }
-
-        } else {
-                pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
-                                                            pump_xattr_cleaner,
-                                                            0, 0, NULL);
-        }
-
-        return 0;
-}
-
-gf_boolean_t
-pump_command_status (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_STATUS, &cmd);
-        if (dict_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Not a pump status command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "Hit a pump command - status");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_pause (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_PAUSE, &cmd);
-        if (dict_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Not a pump pause command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "Hit a pump command - pause");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_commit (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_COMMIT, &cmd);
-        if (dict_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Not a pump commit command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "Hit a pump command - commit");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_abort (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_ABORT, &cmd);
-        if (dict_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Not a pump abort command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "Hit a pump command - abort");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_start (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_START, &cmd);
-        if (dict_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Not a pump start command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "Hit a pump command - start");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-int
-pump_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	       const char *name, dict_t *xdata)
-{
-	afr_private_t *priv = NULL;
-	int op_errno = 0;
-	int ret = 0;
-
-	priv = this->private;
-
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_getxattr_cbk,
-                            FIRST_CHILD (this),
-                            (FIRST_CHILD (this))->fops->getxattr,
-                            loc, name, xdata);
-                return 0;
-        }
-
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
-
-                        op_errno = ENODATA;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!strcmp (name, RB_PUMP_CMD_STATUS)) {
-                        gf_msg_debug (this->name, 0,
-                                      "Hit pump command - status");
-                        pump_execute_status (frame, this);
-                        goto out;
-                }
-        }
-
-	afr_getxattr (frame, this, loc, name, xdata);
-
-out:
-	if (ret < 0)
-		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-int
-pump_command_reply (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (local->op_ret < 0)
-                gf_msg (this->name, GF_LOG_INFO,
-                        0, AFR_MSG_INFO_COMMON,
-                        "Command failed");
-        else
-                gf_msg (this->name, GF_LOG_INFO,
-                        0, AFR_MSG_INFO_COMMON,
-                        "Command succeeded");
-
-        AFR_STACK_UNWIND (setxattr,
-                          frame,
-                          local->op_ret,
-                          local->op_errno, NULL);
-
-        return 0;
-}
-
-int
-pump_parse_command (call_frame_t *frame, xlator_t *this, dict_t *dict,
-		    int *op_errno_p)
-{
-	afr_local_t *local = NULL;
-        int ret = -1;
-	int op_errno = 0;
-
-        if (pump_command_start (this, dict)) {
-                local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_start (frame, this);
-
-        } else if (pump_command_pause (this, dict)) {
-		local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_pause (frame, this);
-
-        } else if (pump_command_abort (this, dict)) {
-		local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_abort (frame, this);
-
-        } else if (pump_command_commit (this, dict)) {
-		local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_commit (frame, this);
-        }
-out:
-	if (op_errno_p)
-		*op_errno_p = op_errno;
-        return ret;
-}
-
-int
-pump_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-	       int32_t flags, dict_t *xdata)
-{
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	int op_errno = 0;
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict, op_errno, out);
-
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_setxattr_cbk,
-                            FIRST_CHILD (this),
-                            (FIRST_CHILD (this))->fops->setxattr,
-                            loc, dict, flags, xdata);
-                return 0;
-        }
-
-        ret = pump_parse_command (frame, this, dict, &op_errno);
-        if (ret >= 0)
-                goto out;
-
-	afr_setxattr (frame, this, loc, dict, flags, xdata);
-
-	ret = 0;
-out:
-	if (ret < 0) {
-		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-	}
-
-	return 0;
-}
-
-/* Defaults */
-static int32_t
-pump_lookup (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc,
-             dict_t *xattr_req)
-{
-	afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_lookup_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup,
-                            loc,
-                            xattr_req);
-                return 0;
-        }
-
-        afr_lookup (frame, this, loc, xattr_req);
-        return 0;
-}
-
-
-static int32_t
-pump_truncate (call_frame_t *frame,
-               xlator_t *this,
-               loc_t *loc,
-               off_t offset, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_truncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            loc,
-                            offset, xdata);
-                return 0;
-        }
-
-        afr_truncate (frame, this, loc, offset, xdata);
-        return 0;
-}
-
-
-static int32_t
-pump_ftruncate (call_frame_t *frame,
-                xlator_t *this,
-                fd_t *fd,
-                off_t offset, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_ftruncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate,
-                            fd,
-                            offset, xdata);
-                return 0;
-        }
-
-        afr_ftruncate (frame, this, fd, offset, xdata);
-        return 0;
-}
-
-
-
-
-int
-pump_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_mknod_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, umask, xdata);
-                return 0;
-        }
-        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
-        return 0;
-
-}
-
-
-
-int
-pump_mkdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_mkdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, umask, xdata);
-                return 0;
-        }
-        afr_mkdir (frame, this, loc, mode, umask, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_unlink (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc, int xflag, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_unlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            loc, xflag, xdata);
-                return 0;
-        }
-        afr_unlink (frame, this, loc, xflag, xdata);
-        return 0;
-
-}
-
-
-static int
-pump_rmdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-
-	priv = this->private;
-
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_rmdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags, xdata);
-                return 0;
-        }
-
-        afr_rmdir (frame, this, loc, flags, xdata);
-        return 0;
-
-}
-
-
-
-int
-pump_symlink (call_frame_t *frame, xlator_t *this,
-              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_symlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, umask, xdata);
-                return 0;
-        }
-        afr_symlink (frame, this, linkpath, loc, umask, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_rename (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *oldloc,
-             loc_t *newloc, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_rename_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-        afr_rename (frame, this, oldloc, newloc, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_link (call_frame_t *frame,
-           xlator_t *this,
-           loc_t *oldloc,
-           loc_t *newloc, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_link_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-        afr_link (frame, this, oldloc, newloc, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_create_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->create,
-                            loc, flags, mode, umask, fd, xdata);
-                return 0;
-        }
-        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_open (call_frame_t *frame,
-           xlator_t *this,
-           loc_t *loc,
-           int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_open_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open,
-                            loc, flags, fd, xdata);
-                return 0;
-        }
-        afr_open (frame, this, loc, flags, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_writev (call_frame_t *frame,
-             xlator_t *this,
-             fd_t *fd,
-             struct iovec *vector,
-             int32_t count,
-             off_t off, uint32_t flags,
-             struct iobref *iobref, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_writev_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->writev,
-                            fd,
-                            vector,
-                            count,
-                            off, flags,
-                            iobref, xdata);
-                return 0;
-        }
-
-        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
-        return 0;
-}
-
-
-static int32_t
-pump_flush (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_flush_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush,
-                            fd, xdata);
-                return 0;
-        }
-        afr_flush (frame, this, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsync (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd,
-            int32_t flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsync_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsync,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
-        afr_fsync (frame, this, fd, flags, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_opendir (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_opendir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->opendir,
-                            loc, fd, xdata);
-                return 0;
-        }
-        afr_opendir (frame, this, loc, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsyncdir (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               int32_t flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsyncdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsyncdir,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
-        afr_fsyncdir (frame, this, fd, flags, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_xattrop (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              gf_xattrop_flags_t flags,
-              dict_t *dict, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_xattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->xattrop,
-                            loc,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
-        afr_xattrop (frame, this, loc, flags, dict, xdata);
-        return 0;
-
-}
-
-static int32_t
-pump_fxattrop (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               gf_xattrop_flags_t flags,
-               dict_t *dict, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fxattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fxattrop,
-                            fd,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
-        afr_fxattrop (frame, this, fd, flags, dict, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_removexattr (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t *loc,
-                  const char *name, dict_t *xdata)
-{
-        afr_private_t *priv     = NULL;
-        int            op_errno = -1;
-
-        VALIDATE_OR_GOTO (this, out);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.pump*",
-                                 name, op_errno, out);
-
-        op_errno = 0;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_removexattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->removexattr,
-                            loc,
-                            name, xdata);
-                return 0;
-        }
-        afr_removexattr (frame, this, loc, name, xdata);
-
- out:
-        if (op_errno)
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
-
-}
-
-
-
-static int32_t
-pump_readdir (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              size_t size,
-              off_t off, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_readdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off, xdata);
-                return 0;
-        }
-        afr_readdir (frame, this, fd, size, off, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t off, dict_t *dict)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_readdirp_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, off, dict);
-                return 0;
-        }
-        afr_readdirp (frame, this, fd, size, off, dict);
-        return 0;
-
-}
-
-
-
-static int32_t
-pump_releasedir (xlator_t *this,
-                 fd_t *fd)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_releasedir (this, fd);
-	return 0;
-
-}
-
-static int32_t
-pump_release (xlator_t *this,
-              fd_t *fd)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_release (this, fd);
-	return 0;
-
-}
-
-static int32_t
-pump_forget (xlator_t *this, inode_t *inode)
-{
-        afr_private_t  *priv  = NULL;
-
-        priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_forget (this, inode);
-
-        return 0;
-}
-
-static int32_t
-pump_setattr (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              struct iatt *stbuf,
-              int32_t valid, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_setattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, xdata);
-                return 0;
-        }
-        afr_setattr (frame, this, loc, stbuf, valid, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsetattr (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               struct iatt *stbuf,
-               int32_t valid, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsetattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
-                return 0;
-        }
-        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
-        return 0;
-
-}
-
-
-/* End of defaults */
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
-
-        if (ret != 0) {
-                return ret;
-        }
-
-        return ret;
-}
-
-static int
-is_xlator_pump_sink (xlator_t *child)
-{
-        return (child == PUMP_SINK_CHILD(THIS));
-}
-
-static int
-is_xlator_pump_source (xlator_t *child)
-{
-        return (child == PUMP_SOURCE_CHILD(THIS));
-}
-
-int32_t
-notify (xlator_t *this, int32_t event,
-	void *data, ...)
-{
-        int ret = -1;
-        xlator_t *child_xl = NULL;
-
-        child_xl = (xlator_t *) data;
-
-        ret = afr_notify (this, event, data, NULL);
-
-	switch (event) {
-	case GF_EVENT_CHILD_DOWN:
-                if (is_xlator_pump_source (child_xl))
-                        pump_change_state (this, PUMP_STATE_ABORT);
-                break;
-
-        case GF_EVENT_CHILD_UP:
-                if (is_xlator_pump_sink (child_xl))
-                        if (is_pump_start_pending (this)) {
-                                gf_msg_debug (this->name, 0,
-                                              "about to start synctask");
-                                ret = pump_start_synctask (this);
-                                if (ret < 0)
-                                        gf_msg_debug (this->name, 0,
-                                                      "Could not start pump "
-                                                      "synctask");
-                                else
-                                        pump_remove_start_pending (this);
-                        }
-        }
-
-        return ret;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv   = NULL;
-	int             child_count = 0;
-	xlator_list_t * trav        = NULL;
-	int             i           = 0;
-	int             ret         = -1;
-	GF_UNUSED int   op_errno    = 0;
-
-        int source_child = 0;
-
-	if (!this->children) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        0, AFR_MSG_CHILD_MISCONFIGURED,
-		        "pump translator needs a source and sink"
-                        "subvolumes defined.");
-		return -1;
-	}
-
-	if (!this->parents) {
-	        gf_msg (this->name, GF_LOG_WARNING, 0,
-		        AFR_MSG_VOL_MISCONFIGURED, "Volume is dangling.");
-	}
-
-	priv = GF_CALLOC (1, sizeof (afr_private_t), gf_afr_mt_afr_private_t);
-        if (!priv)
-                goto out;
-
-        LOCK_INIT (&priv->lock);
-
-        child_count = xlator_subvolume_count (this);
-        if (child_count != 2) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        0, AFR_MSG_CHILD_MISCONFIGURED,
-                        "There should be exactly 2 children - one source "
-                        "and one sink");
-                return -1;
-        }
-	priv->child_count = child_count;
-
-        priv->read_child = source_child;
-        priv->favorite_child = source_child;
-        priv->background_self_heal_count = 0;
-
-	priv->data_self_heal     = "on";
-	priv->metadata_self_heal = 1;
-	priv->entry_self_heal    = 1;
-
-        priv->data_self_heal_window_size = 16;
-
-	priv->data_change_log     = 1;
-	priv->metadata_change_log = 1;
-	priv->entry_change_log    = 1;
-        priv->use_afr_in_pump = 1;
-        priv->sh_readdir_size = 65536;
-
-	/* Locking options */
-
-        /* Lock server count infact does not matter. Locks are held
-           on all subvolumes, in this case being the source
-           and the sink.
-        */
-
-	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
-                                 gf_afr_mt_char);
-	if (!priv->child_up) {
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-	priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
-                                 gf_afr_mt_xlator_t);
-	if (!priv->children) {
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
-                                       child_count,
-                                       gf_afr_mt_char);
-        if (!priv->pending_key) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-	trav = this->children;
-	i = 0;
-	while (i < child_count) {
-		priv->children[i] = trav->xlator;
-
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                   AFR_XATTR_PREFIX,
-                                   trav->xlator->name);
-                if (-1 == ret) {
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-		trav = trav->next;
-		i++;
-	}
-
-        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
-        if (-1 == ret) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        priv->root_inode = NULL;
-
-        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
-                                      gf_afr_mt_int32_t);
-        if (!priv->last_event) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-	pump_priv = GF_CALLOC (1, sizeof (*pump_priv),
-                            gf_afr_mt_pump_priv);
-	if (!pump_priv) {
-                op_errno = ENOMEM;
-		goto out;
-	}
-
-        LOCK_INIT (&pump_priv->resume_path_lock);
-        LOCK_INIT (&pump_priv->pump_state_lock);
-
-        pump_priv->resume_path = GF_CALLOC (1, PATH_MAX,
-                                            gf_afr_mt_char);
-        if (!pump_priv->resume_path) {
-                ret = -1;
-                goto out;
-        }
-
-	pump_priv->env = this->ctx->env;
-        if (!pump_priv->env) {
-                ret = -1;
-                goto out;
-        }
-
-        /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                goto out;
-        }
-
-	priv->pump_private = pump_priv;
-        pump_priv = NULL;
-
-        this->private = priv;
-        priv = NULL;
-
-        pump_change_state (this, PUMP_STATE_ABORT);
-
-	ret = 0;
-out:
-
-        if (pump_priv) {
-                GF_FREE (pump_priv->resume_path);
-                LOCK_DESTROY (&pump_priv->resume_path_lock);
-                LOCK_DESTROY (&pump_priv->pump_state_lock);
-                GF_FREE (pump_priv);
-        }
-
-        if (priv) {
-                GF_FREE (priv->child_up);
-                GF_FREE (priv->children);
-                GF_FREE (priv->pending_key);
-                GF_FREE (priv->last_event);
-                LOCK_DESTROY (&priv->lock);
-                GF_FREE (priv);
-        }
-
-	return ret;
-}
-
-int
-fini (xlator_t *this)
-{
-        afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        this->private = NULL;
-        if (!priv)
-                goto out;
-
-        pump_priv = priv->pump_private;
-        if (!pump_priv)
-                goto afr_priv;
-
-        GF_FREE (pump_priv->resume_path);
-        LOCK_DESTROY (&pump_priv->resume_path_lock);
-        LOCK_DESTROY (&pump_priv->pump_state_lock);
-        GF_FREE (pump_priv);
-afr_priv:
-        afr_priv_destroy (priv);
-out:
-	return 0;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = pump_lookup,
-	.open        = pump_open,
-	.flush       = pump_flush,
-	.fsync       = pump_fsync,
-	.fsyncdir    = pump_fsyncdir,
-	.xattrop     = pump_xattrop,
-	.fxattrop    = pump_fxattrop,
-        .getxattr    = pump_getxattr,
-
-	/* inode write */
-	.writev      = pump_writev,
-	.truncate    = pump_truncate,
-	.ftruncate   = pump_ftruncate,
-	.setxattr    = pump_setxattr,
-        .setattr     = pump_setattr,
-	.fsetattr    = pump_fsetattr,
-	.removexattr = pump_removexattr,
-
-	/* dir read */
-	.opendir     = pump_opendir,
-	.readdir     = pump_readdir,
-	.readdirp    = pump_readdirp,
-
-	/* dir write */
-	.create      = pump_create,
-	.mknod       = pump_mknod,
-	.mkdir       = pump_mkdir,
-	.unlink      = pump_unlink,
-	.rmdir       = pump_rmdir,
-	.link        = pump_link,
-	.symlink     = pump_symlink,
-	.rename      = pump_rename,
-};
-
-struct xlator_dumpops dumpops = {
-        .priv       = afr_priv_dump,
-};
-
-
-struct xlator_cbks cbks = {
-	.release     = pump_release,
-	.releasedir  = pump_releasedir,
-        .forget      = pump_forget,
-};
-
-struct volume_options options[] = {
-	{ .key  = {NULL} },
-};
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
deleted file mode 100644
index 7d5acd02bf6..00000000000
--- a/xlators/cluster/afr/src/pump.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __PUMP_H__
-#define __PUMP_H__
-
-#include "syncop.h"
-
-/* FIXME: Needs to be defined in a common file */
-#define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
-#define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
-
-#define PUMP_SOURCE_COMPLETE "trusted.glusterfs.pump-source-complete"
-#define PUMP_SINK_COMPLETE "trusted.glusterfs.pump-sink-complete"
-
-#define PUMP_PATH "trusted.glusterfs.pump-path"
-
-#define PUMP_SOURCE_CHILD(xl) (xl->children->xlator)
-#define PUMP_SINK_CHILD(xl) (xl->children->next->xlator)
-
-typedef enum {
-        PUMP_STATE_RUNNING,             /* Pump is running and migrating files */
-        PUMP_STATE_RESUME,              /* Pump is resuming from a previous pause */
-        PUMP_STATE_PAUSE,               /* Pump is paused */
-        PUMP_STATE_ABORT,               /* Pump is aborted */
-        PUMP_STATE_COMMIT,              /* Pump is committed */
-} pump_state_t;
-
-typedef struct _pump_private {
-	struct syncenv *env;            /* The env pointer to the pump synctask */
-        char *resume_path;              /* path to resume from the last pause */
-        gf_lock_t resume_path_lock;     /* Synchronize resume_path changes */
-        gf_lock_t pump_state_lock;      /* Synchronize pump_state changes */
-        pump_state_t pump_state;        /* State of pump */
-        char current_file[PATH_MAX];    /* Current file being pumped */
-        uint64_t number_files_pumped;   /* Number of files pumped */
-        gf_boolean_t pump_finished;     /* Boolean to indicate pump termination */
-        char pump_start_pending;        /* Boolean to mark start pending until
-                                           CHILD_UP */
-        call_stub_t *cleaner;
-} pump_private_t;
-
-void
-build_root_loc (inode_t *inode, loc_t *loc);
-int pump_start (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_start (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_start (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_pause (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_pause (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_abort (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_abort (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_status (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_status (call_frame_t *frame, xlator_t *this);
-
-int
-pump_command_reply (call_frame_t *frame, xlator_t *this);
-
-#endif /* __PUMP_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index 29be5ce4776..56f1f2ad7c8 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -1,7 +1,4 @@
 xlator_LTLIBRARIES = dht.la nufa.la switch.la
-if BUILD_GFDB
-   xlator_LTLIBRARIES += tier.la
-endif
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
@@ -10,40 +7,34 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
 	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
 	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
-	$(top_builddir)/xlators/lib/src/libxlator.c
+	dht-lock.c $(top_builddir)/xlators/lib/src/libxlator.c
 
 dht_la_SOURCES = $(dht_common_source) dht.c
 
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
-tier_la_SOURCES = $(dht_common_source) tier.c tier-common.c
 
-dht_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/cluster/dht/src/dht.sym
+dht_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/cluster/dht/src/nufa.sym
+nufa_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/cluster/dht/src/switch.sym
+switch_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-tier_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/cluster/dht/src/tier.sym
-tier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = dht-common.h dht-mem-types.h dht-messages.h dht-helper.h tier-common.h tier.h\
-	$(top_builddir)/xlators/lib/src/libxlator.h
+noinst_HEADERS = dht-common.h dht-mem-types.h dht-messages.h \
+	dht-lock.h $(top_builddir)/xlators/lib/src/libxlator.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/libglusterfs/src/gfdb \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
 	-I$(top_srcdir)/xlators/lib/src \
 	-DDATADIR=\"$(localstatedir)\" \
-	-DLIBDIR=\"$(libdir)\" \
-	-DLIBGFDB_VERSION=\"$(LIBGFDB_VERSION)\"
+	-DLIBDIR=\"$(libdir)\"
 
 CLEANFILES =
 
-EXTRA_DIST = dht.sym nufa.sym switch.sym tier.sym
-
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/distribute.so
 
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index c667266fed8..8ba0cc4c732 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -8,183 +8,439 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "libxlator.h"
 #include "dht-common.h"
-#include "defaults.h"
-#include "byte-order.h"
-#include "glusterfs-acl.h"
-#include "quota-common-utils.h"
+#include "dht-lock.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/quota-common-utils.h>
+#include <glusterfs/upcall-utils.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+#include <glusterfs/common-utils.h>
 
 #include <sys/time.h>
 #include <libgen.h>
 #include <signal.h>
 
-int run_defrag = 0;
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata);
 
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
 
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req);
 
-int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame,
-               int ret);
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this);
 
-int
-dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-                  int ret);
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata);
 
-int
-dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-               int ret);
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this);
 
+static const char *dht_dbg_vxattrs[] = {DHT_DBG_HASHED_SUBVOL_PATTERN, NULL};
+
+/* Check the xdata to make sure EBADF has been set by client xlator */
+int32_t
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno)
+{
+    if (op_ret == -1 && (op_errno == EBADF || op_errno == EBADFD) &&
+        !(local->fd_checked)) {
+        return 1;
+    }
+    return 0;
+}
 
 /* Sets the blocks and size values to fixed values. This is to be called
  * only for dirs. The caller is responsible for checking the type
  */
-int32_t dht_set_fixed_dir_stat (struct iatt *stat)
+int32_t
+dht_set_fixed_dir_stat(struct iatt *stat)
 {
-        if (stat) {
-                stat->ia_blocks = DHT_DIR_STAT_BLOCKS;
-                stat->ia_size = DHT_DIR_STAT_SIZE;
-                return 0;
-        }
-        return -1;
+    if (stat) {
+        stat->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        stat->ia_size = DHT_DIR_STAT_SIZE;
+        return 0;
+    }
+    return -1;
 }
 
+/* Return true if key exists in array
+ */
+static gf_boolean_t
+dht_match_xattr(const char *key)
+{
+    char **xattrs_to_heal = get_xattrs_to_heal();
 
-int
-dht_rmdir_unlock (call_frame_t *frame, xlator_t *this);
-
-int
-dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value)
-{
-        int              ret            = -1;
-        quota_meta_t    *meta_dst       = NULL;
-        quota_meta_t    *meta_src       = NULL;
-        int64_t         *size           = NULL;
-        int64_t          dst_dir_count  = 0;
-        int64_t          src_dir_count  = 0;
-
-        if (value == NULL) {
-                gf_msg ("dht", GF_LOG_WARNING, 0,
-                        DHT_MSG_DATA_NULL, "data value is NULL");
-                ret = -1;
-                goto out;
-        }
+    return gf_get_index_by_elem(xattrs_to_heal, (char *)key) >= 0;
+}
 
-        ret = dict_get_bin (dst, key, (void **)&meta_dst);
+static int
+dht_aggregate_quota_xattr(dict_t *dst, char *key, data_t *value)
+{
+    int ret = -1;
+    quota_meta_t *meta_dst = NULL;
+    quota_meta_t *meta_src = NULL;
+    int64_t *size = NULL;
+    int64_t dst_dir_count = 0;
+    int64_t src_dir_count = 0;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "data value is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(dst, key, (void **)&meta_dst);
+    if (ret < 0) {
+        meta_dst = GF_CALLOC(1, sizeof(quota_meta_t), gf_common_quota_meta_t);
+        if (meta_dst == NULL) {
+            gf_msg("dht", GF_LOG_WARNING, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_bin(dst, key, meta_dst, sizeof(quota_meta_t));
         if (ret < 0) {
-                meta_dst = GF_CALLOC (1, sizeof (quota_meta_t),
-                                      gf_common_quota_meta_t);
-                if (meta_dst == NULL) {
-                        gf_msg ("dht", GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_NO_MEMORY,
-                                "Memory allocation failed");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_bin (dst, key, meta_dst,
-                                    sizeof (quota_meta_t));
-                if (ret < 0) {
-                        gf_msg ("dht", GF_LOG_WARNING, EINVAL,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "dht aggregate dict set failed");
-                        GF_FREE (meta_dst);
-                        ret = -1;
-                        goto out;
-                }
+            gf_msg("dht", GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "dht aggregate dict set failed");
+            GF_FREE(meta_dst);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (value->len > sizeof (int64_t)) {
-                meta_src = data_to_bin (value);
+    if (value->len > sizeof(int64_t)) {
+        meta_src = data_to_bin(value);
 
-                meta_dst->size = hton64 (ntoh64 (meta_dst->size) +
-                                         ntoh64 (meta_src->size));
-                meta_dst->file_count = hton64 (ntoh64 (meta_dst->file_count) +
-                                               ntoh64 (meta_src->file_count));
+        meta_dst->size = hton64(ntoh64(meta_dst->size) +
+                                ntoh64(meta_src->size));
+        meta_dst->file_count = hton64(ntoh64(meta_dst->file_count) +
+                                      ntoh64(meta_src->file_count));
 
-                if (value->len > (2 * sizeof (int64_t))) {
-                        dst_dir_count = ntoh64 (meta_dst->dir_count);
-                        src_dir_count = ntoh64 (meta_src->dir_count);
+        if (value->len > (2 * sizeof(int64_t))) {
+            dst_dir_count = ntoh64(meta_dst->dir_count);
+            src_dir_count = ntoh64(meta_src->dir_count);
 
-                        if (src_dir_count > dst_dir_count)
-                                meta_dst->dir_count = meta_src->dir_count;
-                } else {
-                        meta_dst->dir_count = 0;
-                }
+            if (src_dir_count > dst_dir_count)
+                meta_dst->dir_count = meta_src->dir_count;
         } else {
-                size = data_to_bin (value);
-                meta_dst->size = hton64 (ntoh64 (meta_dst->size) +
-                                         ntoh64 (*size));
+            meta_dst->dir_count = 0;
         }
+    } else {
+        size = data_to_bin(value);
+        meta_dst->size = hton64(ntoh64(meta_dst->size) + ntoh64(*size));
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
+static int
+add_opt(char **optsp, const char *opt)
+{
+    char *newopts = NULL;
+    unsigned oldsize = 0;
+    unsigned newsize = 0;
+
+    if (*optsp == NULL)
+        newopts = gf_strdup(opt);
+    else {
+        oldsize = strlen(*optsp);
+        newsize = oldsize + 1 + strlen(opt) + 1;
+        newopts = GF_REALLOC(*optsp, newsize);
+        if (newopts)
+            sprintf(newopts + oldsize, ",%s", opt);
+    }
+    if (newopts == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+               "Error to add choices in buffer in add_opt");
+        return -1;
+    }
+    *optsp = newopts;
+    return 0;
+}
 
+/* Return Choice list from Split brain status */
+static char *
+getChoices(const char *value)
+{
+    int i = 0;
+    char *ptr = NULL;
+    char *tok = NULL;
+    char *result = NULL;
+    char *newval = NULL;
 
-int
-dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
+    ptr = strstr(value, "Choices:");
+    if (!ptr) {
+        result = ptr;
+        goto out;
+    }
+
+    newval = gf_strdup(ptr);
+    if (!newval) {
+        result = newval;
+        goto out;
+    }
+
+    tok = strtok(newval, ":");
+    if (!tok) {
+        result = tok;
+        goto out;
+    }
+
+    while (tok) {
+        i++;
+        if (i == 2)
+            break;
+        tok = strtok(NULL, ":");
+    }
+
+    result = gf_strdup(tok);
+
+out:
+    if (newval)
+        GF_FREE(newval);
+
+    return result;
+}
+
+/* This function prepare a list of choices for key
+   (replica.split-brain-status) in   case of metadata split brain
+   only on the basis of key-value passed to this function.
+   After prepare the list of choices it update the same key in dict
+   with this value to reflect the same in
+   replica.split-brain-status attr for file.
+
+*/
+
+static int
+dht_aggregate_split_brain_xattr(dict_t *dst, char *key, data_t *value)
+{
+    int ret = 0;
+    char *oldvalue = NULL;
+    char *old_choice = NULL;
+    char *new_choice = NULL;
+    char *full_choice = NULL;
+    char *status = NULL;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "GF_AFR_SBRAIN_STATUS value is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dst, key, &oldvalue);
+    if (ret)
+        goto out;
+
+    /* skip code that is irrelevant if !oldvalue */
+    if (!oldvalue)
+        goto out;
+
+    if (strstr(oldvalue, "not")) {
+        gf_msg_debug("dht", 0, "Need to update split-brain status in dict");
+        ret = -1;
+        goto out;
+    }
+    if (strstr(oldvalue, "metadata-split-brain:yes") &&
+        (strstr(oldvalue, "data-split-brain:no"))) {
+        if (strstr(value->data, "not")) {
+            gf_msg_debug("dht", 0, "No need to update split-brain status");
+            ret = 0;
+            goto out;
+        }
+        if (strstr(value->data, "yes") &&
+            (strncmp(oldvalue, value->data, strlen(oldvalue)))) {
+            old_choice = getChoices(oldvalue);
+            if (!old_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
+                ret = -1;
+                goto out;
+            }
+
+            ret = add_opt(&full_choice, old_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices");
+                ret = -1;
+                goto out;
+            }
+
+            new_choice = getChoices(value->data);
+            if (!new_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
+                ret = -1;
+                goto out;
+            }
+
+            ret = add_opt(&full_choice, new_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices ");
+                ret = -1;
+                goto out;
+            }
+            ret = gf_asprintf(&status,
+                              "data-split-brain:%s    "
+                              "metadata-split-brain:%s   Choices:%s",
+                              "no", "yes", full_choice);
+
+            if (-1 == ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to prepare status ");
+                goto out;
+            }
+            ret = dict_set_dynstr(dst, key, status);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set full choice");
+            }
+        }
+    }
+
+out:
+    if (old_choice)
+        GF_FREE(old_choice);
+    if (new_choice)
+        GF_FREE(new_choice);
+    if (full_choice)
+        GF_FREE(full_choice);
+
+    return ret;
+}
+
+static int
+dht_aggregate(dict_t *this, char *key, data_t *value, void *data)
 {
-        dict_t          *dst            = NULL;
-        int32_t          ret            = -1;
-        data_t          *dict_data      = NULL;
+    dict_t *dst = NULL;
+    int32_t ret = -1;
+    data_t *dict_data = NULL;
 
-        dst = data;
+    dst = data;
 
-        if (strcmp (key, QUOTA_SIZE_KEY) == 0) {
-                ret = dht_aggregate_quota_xattr (dst, key, value);
-                if (ret) {
-                        gf_msg ("dht", GF_LOG_WARNING, 0,
-                                DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
-                                "Failed to aggregate quota xattr");
-                        goto out;
-                }
-        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
-                ret = gf_get_min_stime (THIS, dst, key, value);
-                if (ret < 0)
-                        goto out;
-        } else {
-                /* compare user xattrs only */
-                if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &dict_data);
-                        if (!ret && dict_data && value) {
-                                ret = is_data_equal (dict_data, value);
-                                if (!ret)
-                                        gf_msg_debug ("dht", 0,
-                                                      "xattr mismatch for %s",
-                                                      key);
-                        }
-                }
-                ret = dict_set (dst, key, value);
-                if (ret) {
-                        gf_msg ("dht", GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value: key = %s",
-                                key);
-                }
+    /* compare split brain xattr only */
+    if (strcmp(key, GF_AFR_SBRAIN_STATUS) == 0) {
+        ret = dht_aggregate_split_brain_xattr(dst, key, value);
+        if (!ret)
+            goto out;
+    } else if (strcmp(key, QUOTA_SIZE_KEY) == 0) {
+        ret = dht_aggregate_quota_xattr(dst, key, value);
+        if (ret) {
+            gf_msg("dht", GF_LOG_WARNING, 0,
+                   DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+                   "Failed to aggregate quota xattr");
         }
+        goto out;
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+        ret = gf_get_min_stime(THIS, dst, key, value);
+        goto out;
+    } else {
+        /* compare user xattrs only */
+        if (!strncmp(key, "user.", SLEN("user."))) {
+            ret = dict_lookup(dst, key, &dict_data);
+            if (!ret && dict_data && value) {
+                ret = is_data_equal(dict_data, value);
+                if (!ret)
+                    gf_msg_debug("dht", 0, "xattr mismatch for %s", key);
+            }
+        }
+    }
+
+    ret = dict_set(dst, key, value);
+    if (ret) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s", key);
+    }
 
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
+static void
+dht_aggregate_xattr(dict_t *dst, dict_t *src)
+{
+    if ((dst == NULL) || (src == NULL)) {
+        goto out;
+    }
 
-void
-dht_aggregate_xattr (dict_t *dst, dict_t *src)
+    dict_foreach(src, dht_aggregate, dst);
+out:
+    return;
+}
+
+/* Code to save hashed subvol on inode ctx as a mds subvol
+ */
+int
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t ctx_int = 0;
+    gf_boolean_t ctx_free = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
+        if (ctx_int) {
+            ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
+            ctx->mds_subvol = mds_subvol;
+        } else {
+            ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+            if (!ctx)
+                goto unlock;
+            ctx->mds_subvol = mds_subvol;
+            ctx_free = _gf_true;
+            ctx_int = (long)ctx;
+            ret = __inode_ctx_set(inode, this, &ctx_int);
+        }
+    }
+unlock:
+    UNLOCK(&inode->lock);
+    if (ret && ctx_free)
+        GF_FREE(ctx);
+    return ret;
+}
+
+/*Code to get mds subvol from inode ctx */
+
+int
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol)
 {
-        if ((dst == NULL) || (src == NULL)) {
-                goto out;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+
+    if (!mdsvol)
+        return ret;
+
+    if (__is_root_gfid(inode->gfid)) {
+        (*mdsvol) = FIRST_CHILD(this);
+        return 0;
+    }
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+
+    if (!ret && ctx) {
+        if (ctx->mds_subvol) {
+            *mdsvol = ctx->mds_subvol;
+            ret = 0;
+        } else {
+            ret = -1;
         }
+    }
 
-        dict_foreach (src, dht_aggregate, dst);
-out:
-        return;
+    return ret;
 }
 
 /* TODO:
@@ -194,1015 +450,1851 @@ out:
    - complete linkfile selfheal
 */
 
-
-int
-dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this,
-                         int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_lookup_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-        local = frame->local;
-        ret = op_ret;
+    local = frame->local;
+    conf = this->private;
+    ret = op_ret;
 
-        FRAME_SU_UNDO (frame, dht_local_t);
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        if (ret == 0) {
-                layout = local->selfheal.layout;
-                ret = dht_layout_set (this, local->inode, layout);
-        }
+    if (ret == 0) {
+        layout = local->selfheal.layout;
+        ret = dht_layout_set(this, local->inode, layout);
+    }
 
-        dht_inode_ctx_time_update (local->inode, this, &local->stbuf, 1);
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           &local->postparent, 1);
-        }
+    dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, &local->postparent,
+                                  1);
+    }
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        dht_set_fixed_dir_stat (&local->postparent);
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-        DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
-                          &local->stbuf, local->xattr, &local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, ret, local->op_errno, local->inode,
+                     &local->stbuf, local->xattr, &local->postparent);
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
-{
-        dht_local_t     *local           = NULL;
-        dht_local_t     *heal_local      = NULL;
-        call_frame_t    *main_frame      = NULL;
-        call_frame_t    *heal_frame      = NULL;
-        int              op_errno        = 0;
-        int              ret             = -1;
-        dht_layout_t    *layout          = NULL;
-        dht_conf_t      *conf            = NULL;
-        uint32_t         vol_commit_hash = 0;
-        xlator_t        *source          = NULL;
-        int              heal_path       = 0;
-        int              i               = 0;
-        loc_t            loc             = {0 };
-        int8_t           is_read_only    = 0, layout_anomalies = 0;
-
-        local = discover_frame->local;
-        layout = local->layout;
-        conf = this->private;
+static int
+dht_discover_complete(xlator_t *this, call_frame_t *discover_frame)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
+    call_frame_t *heal_frame = NULL;
+    int op_errno = 0;
+    int ret = -1;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    uint32_t vol_commit_hash = 0;
+    xlator_t *source = NULL;
+    int heal_path = 0;
+    int error_while_marking_mds = 0;
+    int i = 0;
+    loc_t loc = {0};
+    int8_t is_read_only = 0, layout_anomalies = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    local = discover_frame->local;
+    layout = local->layout;
+    conf = this->private;
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&discover_frame->lock);
+    {
+        main_frame = local->main_frame;
+        local->main_frame = NULL;
+    }
+    UNLOCK(&discover_frame->lock);
+
+    if (!main_frame)
+        return 0;
+
+    /* Code to update all extended attributed from
+       subvol to local->xattr on that internal xattr has found
+    */
+    if (conf->subvolume_cnt == 1)
+        local->need_xattr_heal = 0;
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    ret = dict_get_int8(local->xattr_req, QUOTA_READ_ONLY_KEY, &is_read_only);
+    if (ret < 0)
+        gf_msg_debug(this->name, 0, "key = %s not present in dict",
+                     QUOTA_READ_ONLY_KEY);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s exists as a file on one subvolume "
+               "and directory on another. "
+               "Please fix it manually",
+               local->loc.path);
+        op_errno = EIO;
+        goto out;
+    }
 
-        LOCK(&discover_frame->lock);
-        {
-                main_frame = local->main_frame;
-                local->main_frame = NULL;
+    if (local->cached_subvol) {
+        ret = dht_layout_preset(this, local->cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SET_FAILED,
+                   "failed to set layout for subvolume %s",
+                   local->cached_subvol ? local->cached_subvol->name : "<nil>");
+            op_errno = EINVAL;
+            goto out;
+        }
+    } else {
+        ret = dht_layout_normalize(this, &local->loc, layout);
+        if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+            /* either the layout is incorrect or the directory is
+             * not found even in one subvolume.
+             */
+            gf_msg_debug(this->name, 0,
+                         "normalizing failed on %s "
+                         "(overlaps/holes present: %s, "
+                         "ENOENT errors: %d)",
+                         local->loc.path, (ret < 0) ? "yes" : "no",
+                         (ret > 0) ? ret : 0);
+            layout_anomalies = 1;
+        } else if (local->inode) {
+            dht_layout_set(this, local->inode, layout);
+        }
+    }
+
+    if (!conf->vch_forced) {
+        ret = dict_get_uint32(local->xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
-        UNLOCK(&discover_frame->lock);
-
-        if (!main_frame)
-                return 0;
+    }
 
-        ret = dict_get_int8 (local->xattr_req, QUOTA_READ_ONLY_KEY,
-                             &is_read_only);
-        if (ret < 0)
-                gf_msg_debug (this->name, 0, "key = %s not present in dict",
-                              QUOTA_READ_ONLY_KEY);
-
-        if (local->file_count && local->dir_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_FILE_TYPE_MISMATCH,
-                        "path %s exists as a file on one subvolume "
-                        "and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path);
-                op_errno = EIO;
-                goto out;
-        }
+    if (IA_ISDIR(local->stbuf.ia_type) && !is_read_only) {
+        for (i = 0; i < layout->cnt; i++) {
+            if (!source && !layout->list[i].err)
+                source = layout->list[i].xlator;
+            if (layout->list[i].err == ENOENT ||
+                layout->list[i].err == ESTALE) {
+                heal_path = 1;
+            }
 
-        if (local->cached_subvol) {
-                ret = dht_layout_preset (this, local->cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_SET_FAILED,
-                                "failed to set layout for subvolume %s",
-                                local->cached_subvol ? local->cached_subvol->name : "<nil>");
-                        op_errno = EINVAL;
-                        goto out;
-                }
-        } else {
-                ret = dht_layout_normalize (this, &local->loc, layout);
-                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
-                        /* either the layout is incorrect or the directory is
-                         * not found even in one subvolume.
-                         */
-                        gf_msg_debug (this->name, 0,
-                                      "normalizing failed on %s "
-                                      "(overlaps/holes present: %s, "
-                                      "ENOENT errors: %d)", local->loc.path,
-                                      (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
-                        layout_anomalies = 1;
-                } else if (local->inode) {
-                        dht_layout_set (this, local->inode, layout);
-                }
+            if (source && heal_path)
+                break;
         }
+    }
 
-        if (!conf->vch_forced) {
-                ret = dict_get_uint32 (local->xattr,
-                                       conf->commithash_xattr_name,
-                                       &vol_commit_hash);
-                if (ret == 0) {
-                        conf->vol_commit_hash = vol_commit_hash;
-                }
+    if (IA_ISDIR(local->stbuf.ia_type)) {
+        /* Call function to save hashed subvol on inode ctx if
+           internal mds xattr is not present and all subvols are up
+        */
+        if (!local->op_ret && !__is_root_gfid(local->stbuf.ia_gfid))
+            (void)dht_common_mark_mdsxattr(discover_frame,
+                                           &error_while_marking_mds, 1);
+
+        if (local->need_xattr_heal && !heal_path) {
+            local->need_xattr_heal = 0;
+            ret = dht_dir_xattr_heal(this, local, &op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "xattr heal failed for "
+                       "directory  gfid is %s ",
+                       gfid_local);
+            }
         }
+    }
 
-        if (IA_ISDIR (local->stbuf.ia_type) && !is_read_only) {
-                for (i = 0; i < layout->cnt; i++) {
-                       if (!source && !layout->list[i].err)
-                                source = layout->list[i].xlator;
-                        if (layout->list[i].err == ENOENT ||
-                            layout->list[i].err == ESTALE) {
-                                heal_path = 1;
-                        }
-
-                        if (source && heal_path)
-                                break;
-                }
+    if (source && (heal_path || layout_anomalies || error_while_marking_mds)) {
+        gf_uuid_copy(loc.gfid, local->gfid);
+        if (gf_uuid_is_null(loc.gfid)) {
+            goto done;
         }
 
-        if (source && (heal_path || layout_anomalies)) {
-                gf_uuid_copy (loc.gfid, local->gfid);
-                if (gf_uuid_is_null (loc.gfid)) {
-                        goto done;
-                }
+        if (local->inode)
+            loc.inode = inode_ref(local->inode);
+        else
+            goto done;
+
+        heal_frame = create_frame(this, this->ctx->pool);
+        if (heal_frame) {
+            heal_local = dht_local_init(heal_frame, &loc, NULL, 0);
+            if (!heal_local)
+                goto cleanup;
+
+            gf_uuid_copy(heal_local->gfid, local->gfid);
+            heal_frame->cookie = source;
+            heal_local->xattr = dict_ref(local->xattr);
+            heal_local->stbuf = local->stbuf;
+            heal_local->postparent = local->postparent;
+            heal_local->inode = inode_ref(loc.inode);
+            heal_local->main_frame = main_frame;
+            FRAME_SU_DO(heal_frame, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_heal_full_path,
+                               dht_heal_full_path_done, heal_frame, heal_frame);
+            if (!ret) {
+                loc_wipe(&loc);
+                return 0;
+            }
+            /*
+             * Failed to spawn the synctask. Returning
+             * with out doing heal.
+             */
+        cleanup:
+            loc_wipe(&loc);
+            DHT_STACK_DESTROY(heal_frame);
+        }
+    }
+done:
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    if (local->xattr)
+        GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-                if (local->inode)
-                        loc.inode = inode_ref (local->inode);
-                else
-                        goto done;
-
-               heal_frame = create_frame (this, this->ctx->pool);
-               if (heal_frame) {
-                        heal_local = dht_local_init (heal_frame, &loc,
-                                                     NULL, 0);
-                        if (!heal_local)
-                                goto cleanup;
-
-                        gf_uuid_copy (heal_local->gfid, local->gfid);
-                        heal_frame->cookie = source;
-                        heal_local->xattr = dict_ref (local->xattr);
-                        heal_local->stbuf = local->stbuf;
-                        heal_local->postparent = local->postparent;
-                        heal_local->inode = inode_ref (loc.inode);
-                        heal_local->main_frame = main_frame;
-                        FRAME_SU_DO (heal_frame, dht_local_t);
-                        ret = synctask_new (this->ctx->env,
-                                            dht_heal_full_path,
-                                            dht_heal_full_path_done,
-                                            heal_frame, heal_frame);
-                        if (!ret) {
-                                loc_wipe (&loc);
-                                return 0;
-                        }
-                        /*
-                         * Failed to spawn the synctask. Returning
-                         * with out doing heal.
-                         */
-cleanup:
-                        loc_wipe (&loc);
-                        DHT_STACK_DESTROY (heal_frame);
-                }
+    DHT_STACK_UNWIND(lookup, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
 
-        }
-done:
-        dht_set_fixed_dir_stat (&local->postparent);
-        DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
 out:
-        DHT_STACK_UNWIND (lookup, main_frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
+    DHT_STACK_UNWIND(lookup, main_frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        return ret;
+    return ret;
 }
 
-int
-dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno,
-                  inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                  struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
-        call_frame_t *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-        int           is_linkfile             = 0;
-        int           attempt_unwind          = 0;
-        dht_conf_t   *conf                    = 0;
-        char         gfid_local[GF_UUID_BUF_SIZE]  = {0};
-        char         gfid_node[GF_UUID_BUF_SIZE]  = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = cookie;
+    int ret = -1;
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    int32_t mds_heal_fresh_lookup = 0;
+
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    mds_heal_fresh_lookup = local->mds_heal_fresh_lookup;
+
+    if (op_ret) {
+        gf_msg_debug(this->name, op_ret,
+                     "Failed to set %s on the MDS %s for path %s. ",
+                     conf->mds_xattr_key, prev->name, local->loc.path);
+    } else {
+        /* Save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set mds subvol on inode ctx"
+                   " %s for %s ",
+                   prev->name, local->loc.path);
+        }
+    }
+    if (!local->mds_heal_fresh_lookup && layout) {
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffffff,
+                                 layout);
+    }
+out:
+    if (mds_heal_fresh_lookup)
+        DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+static xlator_t *
+dht_inode_get_hashed_subvol(inode_t *inode, xlator_t *this, loc_t *loc)
+{
+    char *path = NULL;
+    loc_t populate_loc = {
+        0,
+    };
+    char *name = NULL;
+    xlator_t *hash_subvol = NULL;
+
+    if (!inode)
+        return hash_subvol;
+
+    if (loc && loc->parent && loc->path) {
+        if (!loc->name) {
+            name = strrchr(loc->path, '/');
+            if (name) {
+                loc->name = name + 1;
+            } else {
+                goto out;
+            }
+        }
+        hash_subvol = dht_subvol_get_hashed(this, loc);
+        goto out;
+    }
 
-        layout = local->layout;
+    if (!gf_uuid_is_null(inode->gfid)) {
+        populate_loc.inode = inode_ref(inode);
+        populate_loc.parent = inode_parent(populate_loc.inode, NULL, NULL);
+        inode_path(populate_loc.inode, NULL, &path);
 
+        if (!path)
+            goto out;
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
+        populate_loc.path = path;
+        if (!populate_loc.name && populate_loc.path) {
+            name = strrchr(populate_loc.path, '/');
+            if (name) {
+                populate_loc.name = name + 1;
 
-                gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
-                gf_uuid_unparse(local->gfid, gfid_local);
+            } else {
+                goto out;
+            }
+        }
+        hash_subvol = dht_subvol_get_hashed(this, &populate_loc);
+    }
+out:
+    if (populate_loc.inode)
+        loc_wipe(&populate_loc);
+    return hash_subvol;
+}
+
+/* Common function call by revalidate/selfheal code path to populate
+   internal xattr if it is not present, mark_during_fresh_lookup value
+   determines either function is call by revalidate_cbk(discover_complete)
+   or call by selfheal code path while fresh lookup.
+   Here we do wind a call serially in case of fresh lookup and
+   for other lookup code path we do wind a call parallel.The reason
+   to wind a call serially is at the time of fresh lookup directory is not
+   discovered and at the time of revalidate_lookup directory is
+   already discovered. So, revalidate codepath can race with setxattr
+   codepath and can get into spurious heals because of an ongoing setxattr.
+   This can slow down revalidates, if healing happens in foreground.
+   However, if healing happens in background, there is no direct performance
+   penalty.
+*/
+int
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst,
+                         int mark_during_fresh_lookup)
+{
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    dict_t *xattrs = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    int32_t zero[1] = {0};
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    dht_local_t *copy_local = NULL;
+    call_frame_t *xattr_frame = NULL;
+    gf_boolean_t vol_down = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    /* Code to update hashed subvol consider as a mds subvol
+       and wind a setxattr call on hashed subvol to update
+       internal xattr
+    */
+    if (!local->xattr || !dict_get(local->xattr, conf->mds_xattr_key)) {
+        /* It means no internal MDS xattr has been set yet
+         */
+        /* Check the status of all subvol are up while call
+           this function call by lookup code path
+        */
+        if (mark_during_fresh_lookup) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->subvolume_status[i]) {
+                    vol_down = _gf_true;
+                    break;
+                }
+            }
+            if (vol_down) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s is down. Unable to "
+                             " save mds subvol on inode for "
+                             " path %s gfid is %s ",
+                             conf->subvolumes[i]->name, local->loc.path,
+                             gfid_local);
+                goto out;
+            }
+        }
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on %s, gfid local = %s"
-                        "gfid other = %s",
-                        local->loc.path, prev->this->name,
-                        gfid_local, gfid_node);
+        /* Calculate hashed subvol based on inode and parent node
+         */
+        hashed_subvol = dht_inode_get_hashed_subvol(local->inode, this,
+                                                    &local->loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_DEBUG, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for path %s"
+                   "gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (errst)
+                (*errst) = 1;
+            ret = -1;
+            goto out;
+        }
+        xattrs = dict_new();
+        if (!xattrs) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "dict_new failed");
+            ret = -1;
+            goto out;
         }
+        /* Add internal MDS xattr on disk for hashed subvol
+         */
+        ret = dht_dict_set_array(xattrs, conf->mds_xattr_key, zero, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary"
+                   "  value:key = %s for "
+                   "path %s",
+                   conf->mds_xattr_key, local->loc.path);
+            ret = -1;
+            goto out;
+        }
+        /* Create a new frame to wind a call only while
+           this function call by revalidate_cbk code path
+           To wind a call parallel need to create a new frame
+        */
+        if (mark_during_fresh_lookup) {
+            xattr_frame = create_frame(this, this->ctx->pool);
+            if (!xattr_frame) {
+                ret = -1;
+                goto out;
+            }
+            copy_local = dht_local_init(xattr_frame, &(local->loc), NULL, 0);
+            if (!copy_local) {
+                ret = -1;
+                DHT_STACK_DESTROY(xattr_frame);
+                goto out;
+            }
+            copy_local->stbuf = local->stbuf;
+            copy_local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+            if (!copy_local->inode)
+                copy_local->inode = inode_ref(local->inode);
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            FRAME_SU_DO(xattr_frame, dht_local_t);
+            STACK_WIND_COOKIE(xattr_frame, dht_common_mark_mdsxattr_cbk,
+                              hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_common_mark_mdsxattr_cbk,
+                              (void *)hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "internal xattr %s is present on subvol"
+                     "on path %s gfid is %s ",
+                     conf->mds_xattr_key, local->loc.path, gfid_local);
+        if (!mark_during_fresh_lookup)
+            dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf,
+                                     0xffffffff, layout);
+    }
 
+out:
+    if (xattrs)
+        dict_unref(xattrs);
+    return ret;
+}
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
+/* Get the value of key from dict in the bytewise and save in array after
+   convert from network byte order to host byte order
+*/
+static int32_t
+dht_dict_get_array(dict_t *dict, char *key, int32_t value[], int32_t size,
+                   int *errst)
+{
+    void *ptr = NULL;
+    int32_t len = -1;
+    int32_t vindex = -1;
+    int32_t err = -1;
+    int ret = 0;
+
+    if (dict == NULL) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+    err = dict_get_ptr_and_len(dict, key, &ptr, &len);
+    if (err != 0) {
+        (*errst) = -1;
+        return err;
+    }
+
+    if (len != (size * sizeof(int32_t))) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+
+    for (vindex = 0; vindex < size; vindex++) {
+        value[vindex] = ntoh32(*((int32_t *)ptr + vindex));
+        if (value[vindex] < 0)
+            ret = -1;
+    }
+
+    return ret;
+}
 
-                   else mkdir/chmod/chown and fix
-                */
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, xattr);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_MERGE_FAILED,
-                                "%s: failed to merge layouts for subvol %s",
-                                local->loc.path, prev->this->name);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "lookup of %s on %s returned error",
-                                      local->loc.path, prev->this->name);
-
-                        goto unlock;
-                }
+static int
+dht_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int is_linkfile = 0;
+    int attempt_unwind = 0;
+    dht_conf_t *conf = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+    int errst = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+        gf_uuid_unparse(local->gfid, gfid_local);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s, gfid local = %s"
+               "gfid other = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
+
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
+
+           else mkdir/chmod/chown and fix
+        */
 
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                                 conf->link_xattr_name);
-                is_dir = check_is_dir (inode, stbuf, xattr);
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
 
-                if (is_dir) {
-                        local->dir_count ++;
-                } else {
-                        local->file_count ++;
-
-                        if (!is_linkfile) {
-                                /* real file */
-                                local->cached_subvol = prev->this;
-                                attempt_unwind = 1;
-                        } else {
-                                goto unlock;
-                        }
-                }
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            gf_msg_debug(this->name, op_errno,
+                         "lookup of %s on %s returned error", local->loc.path,
+                         prev->name);
 
-                local->op_ret = 0;
+            goto unlock;
+        }
 
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        is_dir = check_is_dir(inode, stbuf, xattr);
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+        if (is_dir) {
+            local->dir_count++;
+        } else {
+            local->file_count++;
+
+            if (!is_linkfile && !local->cached_subvol) {
+                /* real file */
+                /* Ok, we somehow managed to find a file on
+                 * more than one subvol. ignore this or we
+                 * will end up overwriting information while a
+                 * a thread is potentially unwinding from
+                 * dht_discover_complete
+                 */
+                local->cached_subvol = prev;
+                attempt_unwind = 1;
+            } else {
+                goto unlock;
+            }
+        }
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+        local->op_ret = 0;
+
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            /* Don't aggregate for files. See BZ#1484709 */
+            if (is_dir)
+                dht_aggregate_xattr(local->xattr, xattr);
         }
+
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
+
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            goto unlock;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "internal xattr %s is present on subvol"
+                         "on path %s gfid is %s ",
+                         conf->mds_xattr_key, local->loc.path, gfid_local);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        /* save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s vol is %s",
+                   local->loc.path, prev->name);
+        }
+
+        if ((check_mds < 0) && !errst) {
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "Value of %s is not zero on mds subvol"
+                         "so xattr needs to be healed on non mds"
+                         " path is %s and vol name is %s "
+                         " gfid is %s",
+                         conf->mds_xattr_key, local->loc.path, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+            local->mds_subvol = prev;
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 out:
-        /* Make sure, the thread executing dht_discover_complete is the one
-         * which calls STACK_DESTROY (frame). In the case of "attempt_unwind",
-         * this makes sure that the thread don't call dht_frame_return, till
-         * call to dht_discover_complete is done.
-         */
-        if (attempt_unwind) {
-                dht_discover_complete (this, frame);
-        }
+    /* Make sure, the thread executing dht_discover_complete is the one
+     * which calls STACK_DESTROY (frame). In the case of "attempt_unwind",
+     * this makes sure that the thread don't call dht_frame_return, till
+     * call to dht_discover_complete is done.
+     */
+    if (attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt) && !attempt_unwind) {
-                dht_discover_complete (this, frame);
-        }
+    if (is_last_call(this_call_cnt) && !attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_DESTROY (frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
+static int
+dht_set_file_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Used to check whether this is a linkto file.
+     */
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->link_xattr_name, loc->path);
+        goto err;
+    }
+
+    /* This is used to make sure we don't unlink linkto files
+     * which are the target of an ongoing file migration.
+     */
+    ret = dict_set_uint32(xattr_req, GLUSTERFS_OPEN_FD_COUNT, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               GLUSTERFS_OPEN_FD_COUNT, loc->path);
+        goto err;
+    }
+
+    ret = 0;
+err:
+    return ret;
+}
 
-int
-dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        int          ret;
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf = NULL;
-        int          call_cnt = 0;
-        int          op_errno = EINVAL;
-        int          i = 0;
-        call_frame_t *discover_frame = NULL;
+/* This is a gfid based nameless lookup. Without a name, the hashed subvol
+ * cannot be calculated so a lookup is sent to all subvols.
+ */
+static int
+dht_do_discover(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int op_errno = EINVAL;
+    int i = 0;
+    call_frame_t *discover_frame = NULL;
+
+    conf = this->private;
+    local = frame->local;
+
+    /* As we do not know if this is a file or directory, request
+     * both file and directory xattrs
+     */
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-        conf = this->private;
-        local = frame->local;
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        loc->path, conf->xattr_name);
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
 
-        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        loc->path, conf->link_xattr_name);
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        if (__is_root_gfid(local->loc.gfid)) {
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->commithash_xattr_name,
-                                       sizeof(uint32_t));
-        }
+    gf_uuid_copy(local->gfid, loc->gfid);
 
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    discover_frame = copy_frame(frame);
+    if (!discover_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+    discover_frame->local = local;
+    frame->local = NULL;
+    local->main_frame = frame;
 
-        if (!local->layout) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(discover_frame, dht_discover_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        gf_uuid_copy (local->gfid, loc->gfid);
+    return 0;
 
-        discover_frame = copy_frame (frame);
-        if (!discover_frame) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+err:
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        discover_frame->local = local;
-        frame->local = NULL;
-        local->main_frame = frame;
+    return 0;
+}
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (discover_frame, dht_discover_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
+/* Code to call syntask to heal custom xattr from hashed subvol
+   to non hashed subvol
+*/
+int
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno)
+{
+    dht_local_t *copy_local = NULL;
+    call_frame_t *copy = NULL;
+    int ret = -1;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    if (gf_uuid_is_null(local->gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DIR_XATTR_HEAL_FAILED,
+               "No gfid exists for path %s "
+               "so healing xattr is not possible",
+               local->loc.path);
+        *op_errno = EIO;
+        goto out;
+    }
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+    copy = create_frame(this, this->ctx->pool);
+    if (copy) {
+        copy_local = dht_local_init(copy, &(local->loc), NULL, 0);
+        if (!copy_local) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                   DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                   "Memory allocation failed "
+                   "for path %s gfid %s ",
+                   local->loc.path, gfid_local);
+            *op_errno = ENOMEM;
+            DHT_STACK_DESTROY(copy);
+        } else {
+            copy_local->stbuf = local->stbuf;
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            copy_local->mds_subvol = local->mds_subvol;
+            FRAME_SU_DO(copy, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_dir_heal_xattrs,
+                               dht_dir_heal_xattrs_done, copy, copy);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "Synctask creation failed to heal xattr "
+                       "for path %s gfid %s ",
+                       local->loc.path, gfid_local);
+                *op_errno = ENOMEM;
+                DHT_STACK_DESTROY(copy);
+            }
         }
+    }
+out:
+    return ret;
+}
 
-        return 0;
+static int
+dht_needs_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int needs_selfheal = 0;
+    int ret = 0;
 
-err:
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
+    local = frame->local;
+    layout = local->layout;
 
-        return 0;
+    if (local->need_attrheal || local->need_xattr_heal ||
+        local->need_selfheal) {
+        needs_selfheal = 1;
+    }
+
+    ret = dht_layout_normalize(this, &local->loc, layout);
+
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "fixing assignment on %s", local->loc.path);
+        needs_selfheal = 1;
+    }
+    return needs_selfheal;
 }
 
+static int
+is_permission_different(ia_prot_t *prot1, ia_prot_t *prot2)
+{
+    if ((prot1->owner.read != prot2->owner.read) ||
+        (prot1->owner.write != prot2->owner.write) ||
+        (prot1->owner.exec != prot2->owner.exec) ||
+        (prot1->group.read != prot2->group.read) ||
+        (prot1->group.write != prot2->group.write) ||
+        (prot1->group.exec != prot2->group.exec) ||
+        (prot1->other.read != prot2->other.read) ||
+        (prot1->other.write != prot2->other.write) ||
+        (prot1->other.exec != prot2->other.exec) ||
+        (prot1->suid != prot2->suid) || (prot1->sgid != prot2->sgid) ||
+        (prot1->sticky != prot2->sticky)) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
 
 int
-dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
-        call_frame_t *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-        char         gfid_local[GF_UUID_BUF_SIZE]  = {0};
-        char         gfid_node[GF_UUID_BUF_SIZE]  = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int errst = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+    gf_msg_debug(this->name, op_errno,
+                 "%s: lookup on %s returned with op_ret = %d, op_errno = %d",
+                 local->loc.path, prev->name, op_ret, op_errno);
+
+    /* The first successful lookup*/
+    if (!op_ret && gf_uuid_is_null(local->gfid)) {
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s."
+               " gfid local = %s, gfid subvol = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
+
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
+           else mkdir/chmod/chown and fix
+        */
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
 
-        local = frame->local;
-        prev  = cookie;
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
 
-        layout = local->layout;
+            /* The GFID is missing on this subvol. Force a heal. */
+            if (op_errno == ENODATA) {
+                local->need_lookup_everywhere = 1;
+            }
+            goto unlock;
+        }
 
-        if (!op_ret && gf_uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        if (!is_dir) {
+            gf_msg_debug(this->name, 0,
+                         "%s: lookup on %s returned non dir 0%o"
+                         "calling lookup_everywhere",
+                         local->loc.path, prev->name, stbuf->ia_type);
 
-        memcpy (local->loc.gfid, local->gfid, 16);
+            local->need_lookup_everywhere = 1;
+            goto unlock;
+        }
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
+        local->op_ret = 0;
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            dht_aggregate_xattr(local->xattr, xattr);
+        }
 
-                gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
-                gf_uuid_unparse(local->gfid, gfid_local);
+        if (__is_root_gfid(stbuf->ia_gfid)) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on %s."
-                        " gfid local = %s, gfid subvol = %s",
-                        local->loc.path, prev->this->name,
-                        gfid_local, gfid_node);
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
+                    local->prebuf.ia_prot = stbuf->ia_prot;
+                }
+            }
         }
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
+        if (local->stbuf.ia_type != IA_INVAL) {
+            /* This is not the first subvol to respond
+             * Compare values to see if attrs need to be healed
+             */
+            if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                (is_permission_different(&local->stbuf.ia_prot,
+                                         &stbuf->ia_prot))) {
+                local->need_attrheal = 1;
+            }
+        }
 
-                   else mkdir/chmod/chown and fix
-                */
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, xattr);
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
 
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "lookup of %s on %s returned error",
-                                      local->loc.path, prev->this->name);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
-                        goto unlock;
-                }
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: mds xattr %s is not present "
+                         "on %s(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            goto unlock;
+        }
 
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                if (!is_dir) {
+        /* Save the mds subvol info and stbuf. This is the value that will
+         * be used for healing
+         */
+        local->mds_subvol = prev;
+        local->mds_stbuf = *stbuf;
 
-                        gf_msg_debug (this->name, 0,
-                                      "lookup of %s on %s returned non"
-                                      "dir 0%o"
-                                      "calling lookup_everywhere",
-                                      local->loc.path, prev->this->name,
-                                      stbuf->ia_type);
+        /* Save mds subvol on inode ctx */
 
-                        local->need_selfheal = 1;
-                        goto unlock;
-                }
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "%s: Failed to set mds (%s)", local->loc.path, prev->name);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        if ((check_mds < 0) && !errst) {
+            /* Check if xattrs need to be healed on the directories */
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "%s: %s is not zero on %s. Xattrs need to be healed."
+                         "(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+        }
+    }
 
-                local->op_ret = 0;
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+unlock:
+    UNLOCK(&frame->lock);
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+    this_call_cnt = dht_frame_return(frame);
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+    if (is_last_call(this_call_cnt)) {
+        /* If the mds subvol is not set correctly*/
+        if (!__is_root_gfid(local->gfid) &&
+            (!dict_get(local->xattr, conf->mds_xattr_key))) {
+            local->need_selfheal = 1;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
 
-        this_call_cnt = dht_frame_return (frame);
+        /* No need to call xattr heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1) {
+            local->need_xattr_heal = 0;
+        }
+
+        if (local->need_selfheal || local->need_lookup_everywhere) {
+            /* Set the gfid-req so posix will set the GFID*/
+            if (!gf_uuid_is_null(local->gfid)) {
+                /* Ok, this should _never_ happen */
+                ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                          local->gfid, 16);
+            } else {
+                if (!gf_uuid_is_null(local->gfid_req))
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid_req, 16);
+            }
+        }
 
-        if (is_last_call (this_call_cnt)) {
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
+        if (local->need_lookup_everywhere) {
+            local->need_lookup_everywhere = 0;
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
 
-                if (local->op_ret == 0) {
-                        ret = dht_layout_normalize (this, &local->loc, layout);
+        if (local->op_ret == 0) {
+            if (dht_needs_selfheal(frame, this)) {
+                goto selfheal;
+            }
 
-                        if (ret != 0) {
-                                gf_msg_debug (this->name, 0,
-                                              "fixing assignment on %s",
-                                              local->loc.path);
-                                goto selfheal;
-                        }
+            dht_layout_set(this, local->inode, layout);
+            if (local->inode) {
+                dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+            }
 
-                        dht_layout_set (this, local->inode, layout);
-                }
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
+        }
 
-                dht_inode_ctx_time_update (local->inode, this,
-                                           &local->stbuf, 1);
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
-        }
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
 
-        return 0;
+    return 0;
 
 selfheal:
-        FRAME_SU_DO (frame, dht_local_t);
-        gf_uuid_copy (local->loc.gfid, local->gfid);
-        ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
-                                      &local->loc, layout);
+    FRAME_SU_DO(frame, dht_local_t);
+    ret = dht_selfheal_directory(frame, dht_lookup_selfheal_cbk, &local->loc,
+                                 layout);
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        dht_layout_t *layout        = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret  = -1;
-        int           is_dir = 0;
-        int           is_linkfile = 0;
-        int           follow_link = 0;
-        call_frame_t *copy          = NULL;
-        dht_local_t  *copy_local    = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0};
-        uint32_t      vol_commit_hash = 0;
-        xlator_t      *subvol = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, err);
-
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
-        if (!conf)
-                goto out;
+static int
+dht_lookup_directory(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int call_cnt = 0;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", loc, unwind);
+
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        goto unwind;
+    }
+
+    if (local->xattr != NULL) {
+        dict_unref(local->xattr);
+        local->xattr = NULL;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        /* use this gfid in order to heal any missing ones */
+        ret = dict_set_gfuuid(local->xattr_req, "gfid-req", local->gfid, true);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "%s: Failed to set dictionary value:"
+                   " key = gfid-req",
+                   local->loc.path);
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(
+            frame, dht_lookup_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
+unwind:
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+out:
+    return 0;
+}
 
-        if (!conf->vch_forced) {
-                ret = dict_get_uint32 (xattr, conf->commithash_xattr_name,
-                                       &vol_commit_hash);
-                if (ret == 0) {
-                        conf->vol_commit_hash = vol_commit_hash;
-                }
+int
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int is_linkfile = 0;
+    int follow_link = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    uint32_t vol_commit_hash = 0;
+    xlator_t *subvol = NULL;
+    int32_t check_mds = 0;
+    int errst = 0, i = 0;
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", cookie, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (!conf->vch_forced) {
+        /* Update the commithash value if available
+         */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
+    }
 
-        gf_uuid_unparse (local->loc.gfid, gfid);
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        LOCK (&frame->lock);
-        {
+    gf_msg_debug(this->name, op_errno,
+                 "%s: revalidate lookup on %s returned op_ret %d",
+                 local->loc.path, prev->name, op_ret);
 
-                gf_msg_debug (this->name, op_errno,
-                              "revalidate lookup of %s "
-                              "returned with op_ret %d",
-                              local->loc.path, op_ret);
+    LOCK(&frame->lock);
+    {
+        if (gf_uuid_is_null(local->gfid)) {
+            memcpy(local->gfid, local->loc.gfid, 16);
+        }
 
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+
+            if ((op_errno != ENOTCONN) && (op_errno != ENOENT) &&
+                (op_errno != ESTALE)) {
+                gf_msg(this->name, GF_LOG_INFO, op_errno,
+                       DHT_MSG_REVALIDATE_CBK_INFO,
+                       "Revalidate: subvolume %s for %s "
+                       "(gfid = %s) returned -1",
+                       prev->name, local->loc.path, gfid);
+            }
+            if (op_errno == ESTALE) {
+                /* propagate the ESTALE to parent.
+                 * setting local->return_estale would send
+                 * ESTALE to parent. */
+                local->return_estale = 1;
+            }
 
-                        if ((op_errno != ENOTCONN)
-                            && (op_errno != ENOENT)
-                            && (op_errno != ESTALE)) {
-                                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                                        DHT_MSG_REVALIDATE_CBK_INFO,
-                                        "Revalidate: subvolume %s for %s "
-                                        "(gfid = %s) returned -1",
-                                        prev->this->name, local->loc.path,
-                                        gfid);
-                        }
-                        if (op_errno == ESTALE) {
-                                /* propagate the ESTALE to parent.
-                                 * setting local->return_estale would send
-                                 * ESTALE to parent. */
-                                local->return_estale = 1;
+            /* if it is ENOENT, we may have to do a
+             * 'lookup_everywhere()' to make sure
+             * the file is not migrated */
+            if (op_errno == ENOENT) {
+                if (IA_ISREG(local->loc.inode->ia_type)) {
+                    gf_msg_debug(this->name, 0,
+                                 "found ENOENT for %s. "
+                                 "Setting "
+                                 "need_lookup_everywhere"
+                                 " flag to 1",
+                                 local->loc.path);
+
+                    local->need_lookup_everywhere = 1;
+                } else if (IA_ISDIR(local->loc.inode->ia_type)) {
+                    layout = local->layout;
+                    for (i = 0; i < layout->cnt; i++) {
+                        if (layout->list[i].xlator == prev) {
+                            layout->list[i].err = op_errno;
+                            break;
                         }
+                    }
 
-                        /* if it is ENOENT, we may have to do a
-                         * 'lookup_everywhere()' to make sure
-                         * the file is not migrated */
-                        if (op_errno == ENOENT) {
-                                if (IA_ISREG (local->loc.inode->ia_type)) {
-
-                                        gf_msg_debug (this->name, 0,
-                                                      "found ENOENT for %s. "
-                                                      "Setting "
-                                                      "need_lookup_everywhere"
-                                                      " flag to 1",
-                                                      local->loc.path);
-
-                                        local->need_lookup_everywhere = 1;
-                                }
-                        }
-                        goto unlock;
+                    local->need_selfheal = 1;
                 }
+            }
 
-                if ((!IA_ISINVAL(local->inode->ia_type)) &&
-                    stbuf->ia_type != local->inode->ia_type) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_FILE_TYPE_MISMATCH,
-                                "mismatching filetypes 0%o v/s 0%o for %s,"
-                                " gfid = %s",
-                                (stbuf->ia_type), (local->inode->ia_type),
-                                local->loc.path, gfid);
+            /* The GFID is missing on this subvol. Lookup everywhere to force a
+             * gfid heal
+             */
+            if ((op_errno == ENODATA) &&
+                (IA_ISDIR(local->loc.inode->ia_type))) {
+                local->need_lookup_everywhere = 1;
+            }
 
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
+            goto unlock;
+        }
 
-                        goto unlock;
+        if ((!IA_ISINVAL(local->inode->ia_type)) &&
+            stbuf->ia_type != local->inode->ia_type) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+                   "mismatching filetypes 0%o v/s 0%o for %s,"
+                   " gfid = %s",
+                   (stbuf->ia_type), (local->inode->ia_type), local->loc.path,
+                   gfid);
 
-                }
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
 
-                layout = local->layout;
+            goto unlock;
+        }
+
+        layout = local->layout;
 
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                                 conf->link_xattr_name);
-                if (is_linkfile) {
-                        follow_link = 1;
-                        goto unlock;
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        if (is_linkfile) {
+            follow_link = 1;
+            goto unlock;
+        }
+        if (is_dir) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
+
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
+
+                    if (__is_root_gfid(stbuf->ia_gfid))
+                        local->prebuf.ia_prot = stbuf->ia_prot;
                 }
-                if (is_dir) {
-                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
-                        if (ret >= 0) {
-                                if (is_greater_time(local->stbuf.ia_ctime,
-                                                    local->stbuf.ia_ctime_nsec,
-                                                    stbuf->ia_ctime,
-                                                    stbuf->ia_ctime_nsec)) {
-                                        local->prebuf.ia_gid = stbuf->ia_gid;
-                                        local->prebuf.ia_uid = stbuf->ia_uid;
-                                }
-                        }
-                        if (local->stbuf.ia_type != IA_INVAL)
-                        {
-                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
-                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
-                                        local->need_selfheal = 1;
-                                }
-                        }
-                        ret = dht_layout_dir_mismatch (this, layout,
-                                                       prev->this, &local->loc,
-                                                       xattr);
-                        if (ret != 0) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LAYOUT_MISMATCH,
-                                        "Mismatching layouts for %s, gfid = %s",
-                                        local->loc.path, gfid);
-
-                                local->layout_mismatch = 1;
-
-                                goto unlock;
-                        }
+            }
+
+            if (local->stbuf.ia_type != IA_INVAL) {
+                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                    (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                    is_permission_different(&local->stbuf.ia_prot,
+                                            &stbuf->ia_prot)) {
+                    local->need_attrheal = 1;
                 }
+            }
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+            if (!dict_get(xattr, conf->mds_xattr_key)) {
+                gf_msg_debug(this->name, 0,
+                             "%s: internal xattr %s is not present"
+                             " on subvol %s(gfid is %s)",
+                             local->loc.path, conf->mds_xattr_key, prev->name,
+                             gfid);
+            } else {
+                check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                               mds_xattr_val, 1, &errst);
+                local->mds_subvol = prev;
+                local->mds_stbuf.ia_gid = stbuf->ia_gid;
+                local->mds_stbuf.ia_uid = stbuf->ia_uid;
+                local->mds_stbuf.ia_prot = stbuf->ia_prot;
+
+                /* save mds subvol on inode ctx */
+                ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           DHT_MSG_SET_INODE_CTX_FAILED,
+                           "Failed to set MDS subvol for %s vol is %s",
+                           local->loc.path, prev->name);
+                }
+                if ((check_mds < 0) && !errst) {
+                    /* Check if xattrs need to be healed on the directory
+                     */
+                    local->mds_xattr = dict_ref(xattr);
+                    gf_msg_debug(this->name, 0,
+                                 "Value of %s is not zero on "
+                                 "hashed subvol so xattr needs to"
+                                 " be healed on non hashed"
+                                 " path is %s and vol name is %s "
+                                 " gfid is %s",
+                                 conf->mds_xattr_key, local->loc.path,
+                                 prev->name, gfid);
+                    local->need_xattr_heal = 1;
+                }
+            }
+            ret = dht_layout_dir_mismatch(this, layout, prev, &local->loc,
+                                          xattr);
+            if (ret != 0) {
+                /* In memory layout does not match on-disk layout.
+                 */
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_MISMATCH,
+                       "Mismatching layouts for %s, gfid = %s", local->loc.path,
+                       gfid);
 
-                local->op_ret = 0;
+                local->layout_mismatch = 1;
 
-                if (!local->xattr) {
-                        local->xattr = dict_ref (xattr);
-                } else if (is_dir) {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+                goto unlock;
+            }
+        }
+
+        gf_uuid_copy(local->stbuf.ia_gfid, stbuf->ia_gfid);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
+
+        local->op_ret = 0;
+
+        if (!local->xattr) {
+            local->xattr = dict_ref(xattr);
+        } else if (is_dir) {
+            dht_aggregate_xattr(local->xattr, xattr);
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (follow_link) {
-                gf_uuid_copy (local->gfid, stbuf->ia_gfid);
+    if (follow_link) {
+        /* Found a linkto file. Follow it to see if the target file exists
+         */
+        gf_uuid_copy(local->gfid, stbuf->ia_gfid);
 
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
-                if (!subvol) {
-                        op_errno = ESTALE;
-                        local->op_ret = -1;
-                } else {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+        if (!subvol) {
+            op_errno = ESTALE;
+            local->op_ret = -1;
+        } else {
+            STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                              subvol->fops->lookup, &local->loc,
+                              local->xattr_req);
+            return 0;
+        }
+    }
+
+    this_call_cnt = dht_frame_return(frame);
 
-                        STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                        subvol, subvol->fops->lookup,
-                        &local->loc, local->xattr_req);
-                        return 0;
+    if (is_last_call(this_call_cnt)) {
+        if (!IA_ISDIR(local->stbuf.ia_type) &&
+            (local->hashed_subvol != local->cached_subvol) &&
+            (local->stbuf.ia_nlink == 1) &&
+            (conf && conf->unhashed_sticky_bit)) {
+            local->stbuf.ia_prot.sticky = 1;
+        }
+        /* No need to call heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1)
+            local->need_xattr_heal = 0;
+
+        if (IA_ISDIR(local->stbuf.ia_type)) {
+            /* No mds xattr found. Trigger a heal to set it */
+            if (!__is_root_gfid(local->loc.inode->gfid) &&
+                (!dict_get(local->xattr, conf->mds_xattr_key)))
+                local->need_selfheal = 1;
+
+            if (dht_needs_selfheal(frame, this)) {
+                if (!__is_root_gfid(local->loc.inode->gfid)) {
+                    if (local->mds_subvol) {
+                        local->stbuf.ia_gid = local->mds_stbuf.ia_gid;
+                        local->stbuf.ia_uid = local->mds_stbuf.ia_uid;
+                        local->stbuf.ia_prot = local->mds_stbuf.ia_prot;
+                    }
+                } else {
+                    local->stbuf.ia_gid = local->prebuf.ia_gid;
+                    local->stbuf.ia_uid = local->prebuf.ia_uid;
+                    local->stbuf.ia_prot = local->prebuf.ia_prot;
                 }
+
+                layout = local->layout;
+                dht_selfheal_directory(frame, dht_lookup_selfheal_cbk,
+                                       &local->loc, layout);
+                return 0;
+            }
         }
 
-out:
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                if (!IA_ISDIR (local->stbuf.ia_type)
-                    && (local->hashed_subvol != local->cached_subvol)
-                    && (local->stbuf.ia_nlink == 1)
-                    && (conf && conf->unhashed_sticky_bit)) {
-                        local->stbuf.ia_prot.sticky = 1;
-                }
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        gf_uuid_copy (local->gfid, local->stbuf.ia_gfid);
-                        local->stbuf.ia_gid = local->prebuf.ia_gid;
-                        local->stbuf.ia_uid = local->prebuf.ia_uid;
-                        copy = create_frame (this, this->ctx->pool);
-                        if (copy) {
-                                copy_local = dht_local_init (copy, &local->loc,
-                                                             NULL, 0);
-                                if (!copy_local)
-                                        goto cont;
-                                copy_local->stbuf = local->stbuf;
-                                copy->local = copy_local;
-                                FRAME_SU_DO (copy, dht_local_t);
-                                ret = synctask_new (this->ctx->env,
-                                                    dht_dir_attr_heal,
-                                                    dht_dir_attr_heal_done,
-                                                    copy, copy);
-                        }
-                }
-cont:
-                if (local->layout_mismatch) {
-                        /* Found layout mismatch in the directory, need to
-                           fix this in the inode context */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        dht_lookup_directory (frame, this, &local->loc);
-                        return 0;
-                }
+        if (local->layout_mismatch) {
+            /* Found layout mismatch in the directory, need to
+               fix this in the inode context */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
+            dht_lookup_directory(frame, this, &local->loc);
+            return 0;
+        }
 
-                if (local->need_lookup_everywhere) {
-                        /* As the current layout gave ENOENT error, we would
-                           need a new layout */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-
-                        /* We know that current cached subvol is no more
-                           valid, get the new one */
-                        local->cached_subvol = NULL;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
-                if (local->return_estale) {
-                        local->op_ret = -1;
-                        local->op_errno = ESTALE;
-                }
+        if (local->need_lookup_everywhere) {
+            /* As the current layout gave ENOENT error, we would
+               need a new layout */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
+            /* We know that current cached subvol is no longer
+               valid, get the new one */
+            local->cached_subvol = NULL;
+            if (local->xattr_req) {
+                if (!gf_uuid_is_null(local->gfid)) {
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid, 16);
                 }
+            }
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
+        if (local->return_estale) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
         }
 
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
+
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+
+        /* local->stbuf is updated only from subvols which have a layout
+         * The reason is to avoid choosing attr heal source from newly
+         * added bricks. In case e.g we have only one subvol and for
+         * some reason layout is not present on it, then local->stbuf
+         * will be EINVAL. This is an indication that the subvols
+         * active in the cluster do not have layouts on disk.
+         * Unwind with ESTALE to trigger a fresh lookup */
+        if (is_dir && local->stbuf.ia_type == IA_INVAL) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
+        }
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
+
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
+
 err:
-        return ret;
+    return ret;
 }
 
+static int
+dht_lookup_linkfile_create_cbk(call_frame_t *frame, void *cooie, xlator_t *this,
+                               int32_t op_ret, int32_t op_errno, inode_t *inode,
+                               struct iatt *stbuf, struct iatt *preparent,
+                               struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-int
-dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
 
-        local = frame->local;
-        cached_subvol = local->cached_subvol;
-        conf = this->private;
+    local = frame->local;
+    cached_subvol = local->cached_subvol;
+    conf = this->private;
 
-        gf_uuid_unparse(local->loc.gfid, gfid);
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        ret = dht_layout_preset (this, local->cached_subvol, local->loc.inode);
-        if (ret < 0) {
-                gf_msg_debug (this->name, EINVAL,
-                              "Failed to set layout for subvolume %s, "
-                              "(gfid = %s)",
-                              cached_subvol ? cached_subvol->name : "<nil>",
-                              gfid);
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
-        }
+    if (local->locked)
+        dht_unlock_namespace(frame, &local->lock[0]);
 
-        local->op_ret = 0;
-        if ((local->stbuf.ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                local->stbuf.ia_prot.sticky = 1;
-        }
+    ret = dht_layout_preset(this, local->cached_subvol, local->loc.inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "Failed to set layout for subvolume %s, "
+                     "(gfid = %s)",
+                     cached_subvol ? cached_subvol->name : "<nil>", gfid);
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
 
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+    local->op_ret = 0;
+    if ((local->stbuf.ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        local->stbuf.ia_prot.sticky = 1;
+    }
 
-unwind:
-        gf_msg_debug (this->name, 0,
-                      "creation of linkto on hashed subvol:%s, "
-                      "returned with op_ret %d and op_errno %d: %s",
-                      local->hashed_subvol->name,
-                      op_ret, op_errno, uuid_utoa (local->loc.gfid));
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
-        if (local->linked == _gf_true)
-                dht_linkfile_attr_heal (frame, this);
+unwind:
+    gf_msg_debug(this->name, 0,
+                 "creation of linkto on hashed subvol:%s, "
+                 "returned with op_ret %d and op_errno %d: %s",
+                 local->hashed_subvol->name, op_ret, op_errno,
+                 uuid_utoa(local->loc.gfid));
 
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
 
-        dht_set_fixed_dir_stat (&local->postparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
+static int
+dht_lookup_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        const char      *path =  NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        local =  (dht_local_t*)frame->local;
-        path = local->loc.path;
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_UNLINK_LOOKUP_INFO, "lookup_unlink returned with "
-                "op_ret -> %d and op-errno -> %d for %s", op_ret, op_errno,
-                ((path == NULL)? "null" : path ));
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
-        }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lookup_unlink_of_false_linkto_cbk (call_frame_t *frame, void *cookie,
-                                       xlator_t *this, int op_ret, int op_errno,
-                                       struct iatt *preparent,
-                                       struct iatt *postparent, dict_t *xdata)
+static int
+dht_lookup_unlink_of_false_linkto_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      struct iatt *preparent,
+                                      struct iatt *postparent, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        const char      *path =  NULL;
-
-        local =  (dht_local_t*)frame->local;
-        path = local->loc.path;
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_UNLINK_LOOKUP_INFO, "lookup_unlink returned with "
-                "op_ret -> %d and op-errno -> %d for %s", op_ret, op_errno,
-                ((path == NULL)? "null" : path ));
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
 
-                if (op_ret == 0) {
-                        dht_lookup_everywhere_done (frame, this);
-                } else {
-                       /*When dht_lookup_everywhere is performed, one cached
-                         *and one hashed file was found and hashed file does
-                         *not point to the above mentioned cached node. So it
-                         *was considered as stale and an unlink was performed.
-                         *But unlink fails. So may be rebalance is in progress.
-                        *now ideally we have two data-files. One obtained during
-                         *lookup_everywhere and one where unlink-failed. So
-                         *at this point in time we cannot decide which one to
-                         *choose because there are chances of first cached
-                         *file is truncated after rebalance and if it is chosen
-                        *as cached node, application will fail. So return EIO.*/
-
-                        if (op_errno == EBUSY) {
-
-                                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                        DHT_MSG_UNLINK_FAILED,
-                                        "Could not unlink the linkto file as "
-                                        "either fd is open and/or linkto xattr "
-                                        "is set for %s",
-                                        ((path == NULL)? "null":path));
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-                        }
-                        DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL,
-                                          NULL, NULL);
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-                }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((op_ret == 0) || ((op_errno != EBUSY) && (op_errno != ENOTCONN))) {
+            dht_lookup_everywhere_done(frame, this);
+        } else {
+            /*When dht_lookup_everywhere is performed, one cached
+             *and one hashed file was found and hashed file does
+             *not point to the above mentioned cached node. So it
+             *was considered as stale and an unlink was performed.
+             *But unlink fails. So may be rebalance is in progress.
+             *now ideally we have two data-files. One obtained during
+             *lookup_everywhere and one where unlink-failed. So
+             *at this point in time we cannot decide which one to
+             *choose because there are chances of first cached
+             *file is truncated after rebalance and if it is chosen
+             *as cached node, application will fail. So return EIO.*/
+
+            if (op_errno == EBUSY) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_UNLINK_FAILED,
+                       "Could not unlink the linkto file as "
+                       "either fd is open and/or linkto xattr "
+                       "is set for %s",
+                       ((path == NULL) ? "null" : path));
+            }
+            DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lookup_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int op_ret, int op_errno,
-                                    struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
+static int
+dht_lookup_unlink_stale_linkto_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
 {
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        dht_local_t     *local = NULL;
-        const char      *path  = NULL;
+    /* NOTE:
+     * If stale file unlink fails either there is an open-fd or is not an
+     * dht-linkto-file then posix_unlink returns EBUSY, which is overwritten
+     *  to ENOENT
+     */
 
-        /* NOTE:
-         * If stale file unlink fails either there is an open-fd or is not an
-         * dht-linkto-file then posix_unlink returns EBUSY, which is overwritten
-         *  to ENOENT
-         */
-
-        local = frame->local;
+    local = frame->local;
 
-        if (local && local->loc.path)
-                path = local->loc.path;
+    if (local) {
+        FRAME_SU_UNDO(frame, dht_local_t);
+        if (local->loc.path)
+            path = local->loc.path;
+    }
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_UNLINK_LOOKUP_INFO,
-                "Returned with op_ret %d and "
-                "op_errno %d for %s", op_ret, op_errno,
-                ((path==NULL)?"null":path));
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "Returned with op_ret %d and "
+           "op_errno %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL, NULL,
-                          NULL);
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict) {
+static int
+dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_t *dict)
+{
+    int ret = 0;
+
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_NON_LINKTO_UNLINK, 1);
+
+    if (ret)
+        return -1;
 
-        int                      ret = 0;
-        xlator_t                *this           = NULL;
-        char                    *linktoskip_key = NULL;
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_OPEN_FD_UNLINK, 1);
 
-        this    = THIS;
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
+    if (ret)
+        return -1;
 
-        if (dht_is_tier_xlator (this))
-                linktoskip_key = TIER_SKIP_NON_LINKTO_UNLINK;
+    return 0;
+}
+
+static int32_t
+dht_linkfile_create_lookup_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int call_cnt = 0, ret = 0;
+    xlator_t *subvol = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    char gfid_str[GF_UUID_BUF_SIZE] = {0};
+
+    subvol = cookie;
+    local = frame->local;
+
+    if (subvol == local->hashed_subvol) {
+        if ((op_ret == 0) || (op_errno != ENOENT))
+            local->dont_create_linkto = _gf_true;
+    } else {
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(gfid, local->loc.gfid);
         else
-                linktoskip_key = DHT_SKIP_NON_LINKTO_UNLINK;
+            gf_uuid_copy(gfid, local->gfid);
+
+        if ((op_ret == 0) && gf_uuid_compare(gfid, buf->ia_gfid)) {
+            gf_uuid_unparse(gfid, gfid_str);
+            gf_msg_debug(this->name, 0,
+                         "gfid (%s) different on cached subvol "
+                         "(%s) and looked up inode (%s), not "
+                         "creating linkto",
+                         uuid_utoa(buf->ia_gfid), subvol->name, gfid_str);
+            local->dont_create_linkto = _gf_true;
+        } else if (op_ret == -1) {
+            local->dont_create_linkto = _gf_true;
+        }
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->dont_create_linkto)
+            goto no_linkto;
+        else {
+            gf_msg_debug(this->name, 0,
+                         "Creating linkto file on %s(hash) to "
+                         "%s on %s (gfid = %s)",
+                         local->hashed_subvol->name, local->loc.path,
+                         local->cached_subvol->name, gfid_str);
+
+            ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk,
+                                      this, local->cached_subvol,
+                                      local->hashed_subvol, &local->loc);
+
+            if (ret < 0)
+                goto no_linkto;
+        }
+    }
+
+    return 0;
+
+no_linkto:
+    gf_msg_debug(this->name, 0,
+                 "skipped linkto creation (path:%s) (gfid:%s) "
+                 "(hashed-subvol:%s) (cached-subvol:%s)",
+                 local->loc.path, gfid_str, local->hashed_subvol->name,
+                 local->cached_subvol->name);
+
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
+}
+
+static int32_t
+dht_call_lookup_linkfile_create(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int i = 0;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+    if (gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->loc.gfid, gfid);
+    else
+        gf_uuid_unparse(local->gfid, gfid);
 
-        ret = dict_set_int32 (dict, linktoskip_key, 1);
+    if (op_ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "protecting namespace failed, skipping linkto "
+               "creation (path:%s)(gfid:%s)(hashed-subvol:%s)"
+               "(cached-subvol:%s)",
+               local->loc.path, gfid, local->hashed_subvol->name,
+               local->cached_subvol->name);
+        goto err;
+    }
 
-        if (ret)
-                goto err;
+    local->locked = _gf_true;
 
-        ret =  dict_set_int32 (dict, DHT_SKIP_OPEN_FD_UNLINK, 1);
+    local->call_cnt = 2;
 
-        if (ret)
-                goto err;
+    for (i = 0; i < 2; i++) {
+        subvol = (subvol == NULL) ? local->hashed_subvol : local->cached_subvol;
 
+        STACK_WIND_COOKIE(frame, dht_linkfile_create_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, NULL);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        return -1;
-
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
 }
+
 /* Rebalance is performed from cached_node to hashed_node. Initial cached_node
  * contains a non-linkto file. After migration it is converted to linkto and
  * then unlinked. And at hashed_subvolume, first a linkto file is present,
@@ -1243,7742 +2335,9057 @@ err:
  * dht_lookup_everywhere_done takes decision based on any of the above case
  */
 
-int
-dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
-{
-        int           ret = 0;
-        dht_local_t  *local = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_layout_t *layout = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0};
-        gf_boolean_t  found_non_linkto_on_hashed = _gf_false;
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t found_non_linkto_on_hashed = _gf_false;
+
+    local = frame->local;
+    hashed_subvol = local->hashed_subvol;
+    cached_subvol = local->cached_subvol;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s (gfid = %s)exists as a file on one "
+               "subvolume and directory on another. "
+               "Please fix it manually",
+               local->loc.path, gfid);
+        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+        return 0;
+    }
+    if (local->op_ret && local->gfid_missing) {
+        if (gf_uuid_is_null(local->gfid_req)) {
+            DHT_STACK_UNWIND(lookup, frame, -1, ENODATA, NULL, NULL, NULL,
+                             NULL);
+            return 0;
+        }
+        /* A hack */
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
+
+    if (local->dir_count) {
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "STATUS: hashed_subvol %s "
+                 "cached_subvol %s",
+                 (hashed_subvol == NULL) ? "null" : hashed_subvol->name,
+                 (cached_subvol == NULL) ? "null" : cached_subvol->name);
+
+    if (!cached_subvol) {
+        if (local->skip_unlink.handle_valid_link && hashed_subvol) {
+            /*Purpose of "DHT_SKIP_NON_LINKTO_UNLINK":
+             * If this lookup is performed by rebalance and this
+             * rebalance process detected hashed file and by
+             * the time it sends the lookup request to cached node,
+             * file got migrated and now at initial hashed_node,
+             * final migrated file is present. With current logic,
+             * because this process fails to find the cached_node,
+             * it will unlink the file at initial hashed_node.
+             *
+             * So we avoid this by setting key, and checking at the
+             * posix_unlink that unlink the file only if file is a
+             * linkto file and not a migrated_file.
+             */
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                local->xattr_req);
+
+            if (ret) {
+                /* If for some reason, setting key in the dict
+                 * fails, return with ENOENT, as with respect to
+                 * this process, it detected only a stale link
+                 * file.
+                 *
+                 * Next lookup will delete it.
+                 *
+                 * Performing deletion of stale link file when
+                 * setting key in dict fails, may cause the data
+                 * loss because of the above mentioned race.
+                 */
 
-        local = frame->local;
-        hashed_subvol = local->hashed_subvol;
-        cached_subvol = local->cached_subvol;
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "No Cached was found and "
+                             "unlink on hashed was skipped"
+                             " so performing now: %s",
+                             local->loc.path);
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_stale_linkto_cbk,
+                           hashed_subvol, hashed_subvol->fops->unlink,
+                           &local->loc, 0, local->xattr_req);
+            }
 
-        gf_uuid_unparse (local->loc.gfid, gfid);
-
-        if (local->file_count && local->dir_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_FILE_TYPE_MISMATCH,
-                        "path %s (gfid = %s)exists as a file on one "
-                        "subvolume and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path, gfid);
-                DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL,
-                                  NULL);
-                return 0;
-        }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "There was no cached file and  "
+                         "unlink on hashed is not skipped %s",
+                         local->loc.path);
 
-        if (local->dir_count) {
-                dht_lookup_directory (frame, this, &local->loc);
-                return 0;
+            DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
         }
+        return 0;
+    }
 
-        gf_msg_debug (this->name, 0, "STATUS: hashed_subvol %s "
-                      "cached_subvol %s",
-                      (hashed_subvol == NULL)?"null":hashed_subvol->name,
-                      (cached_subvol == NULL)?"null":cached_subvol->name);
-
-        if (!cached_subvol) {
-
-                if (local->skip_unlink.handle_valid_link && hashed_subvol) {
-
-                        /*Purpose of "DHT_SKIP_NON_LINKTO_UNLINK":
-                         * If this lookup is performed by rebalance and this
-                         * rebalance process detected hashed file and by
-                         * the time it sends the lookup request to cached node,
-                         * file got migrated and now at initial hashed_node,
-                         * final migrated file is present. With current logic,
-                         * because this process fails to find the cached_node,
-                         * it will unlink the file at initial hashed_node.
-                         *
-                         * So we avoid this by setting key, and checking at the
-                         * posix_unlink that unlink the file only if file is a
-                         * linkto file and not a migrated_file.
-                         */
-
-
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (local->xattr_req);
-
-                        if (ret) {
-                                /* If for some reason, setting key in the dict
-                                 * fails, return with ENOENT, as with respect to
-                                 * this process, it detected only a stale link
-                                 * file.
-                                 *
-                                 * Next lookup will delete it.
-                                 *
-                                 * Performing deletion of stale link file when
-                                 * setting key in dict fails, may cause the data
-                                 * loss becase of the above mentioned race.
-                                 */
-
-
-                                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT,
-                                                  NULL, NULL, NULL, NULL);
-                        } else {
-                               local->skip_unlink.handle_valid_link = _gf_false;
-
-                                gf_msg_debug (this->name, 0,
-                                              "No Cached was found and "
-                                              "unlink on hashed was skipped"
-                                              " so performing now: %s",
-                                              local->loc.path);
-
-                               STACK_WIND (frame,
-                                            dht_lookup_unlink_stale_linkto_cbk,
-                                            hashed_subvol,
-                                            hashed_subvol->fops->unlink,
-                                            &local->loc, 0, local->xattr_req);
-                        }
+    /* At the time of dht_lookup, no file was found on hashed and that is
+     * why dht_lookup_everywhere is called, but by the time
+     * dht_lookup_everywhere
+     * reached to server, file might have already migrated. In that case we
+     * will find a migrated file at the hashed_node. In this case store the
+     * layout in context and return successfully.
+     */
 
-                } else  {
+    if (hashed_subvol || local->need_lookup_everywhere) {
+        if (local->need_lookup_everywhere) {
+            found_non_linkto_on_hashed = _gf_true;
 
-                        gf_msg_debug (this->name, 0,
-                                      "There was no cached file and  "
-                                      "unlink on hashed is not skipped %s",
-                                      local->loc.path);
+        } else if ((local->file_count == 1) &&
+                   (hashed_subvol == cached_subvol)) {
+            gf_msg_debug(this->name, 0,
+                         "found cached file on hashed subvolume "
+                         "so store in context and return for %s",
+                         local->loc.path);
 
-                        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL,
-                                          NULL, NULL);
-                }
-                return 0;
+            found_non_linkto_on_hashed = _gf_true;
         }
 
-        /* At the time of dht_lookup, no file was found on hashed and that is
-         * why dht_lookup_everywhere is called, but by the time
-         * dht_lookup_everywhere
-         * reached to server, file might have already migrated. In that case we
-         * will find a migrated file at the hashed_node. In this case store the
-         * layout in context and return successfully.
-         */
-
-        if (hashed_subvol || local->need_lookup_everywhere) {
-
-                if (local->need_lookup_everywhere) {
+        if (found_non_linkto_on_hashed)
+            goto preset_layout;
+    }
 
-                        found_non_linkto_on_hashed = _gf_true;
+    if (hashed_subvol) {
+        if (local->skip_unlink.handle_valid_link == _gf_true) {
+            if (cached_subvol == local->skip_unlink.hash_links_to) {
+                if (gf_uuid_compare(local->skip_unlink.cached_gfid,
+                                    local->skip_unlink.hashed_gfid)) {
+                    /*GFID different, return error*/
+                    DHT_STACK_UNWIND(lookup, frame, -1, ESTALE, NULL, NULL,
+                                     NULL, NULL);
 
-                } else if ((local->file_count == 1) &&
-                            (hashed_subvol == cached_subvol)) {
-
-                        gf_msg_debug (this->name, 0,
-                                      "found cached file on hashed subvolume "
-                                      "so store in context and return for %s",
-                                      local->loc.path);
-
-                        found_non_linkto_on_hashed = _gf_true;
+                    return 0;
                 }
 
-                if (found_non_linkto_on_hashed)
-                        goto preset_layout;
-
-        }
-
-
-        if (hashed_subvol) {
-                if (local->skip_unlink.handle_valid_link == _gf_true) {
-                        if (cached_subvol == local->skip_unlink.hash_links_to) {
-
-                             if (gf_uuid_compare (local->skip_unlink.cached_gfid,
-                                               local->skip_unlink.hashed_gfid)){
-
-                                        /*GFID different, return error*/
-                                        DHT_STACK_UNWIND (lookup, frame, -1,
-                                                          ESTALE, NULL, NULL,
-                                                          NULL, NULL);
-
-                                        return 0;
-                                }
-
-                                ret = dht_layout_preset (this, cached_subvol,
-                                                         local->loc.inode);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                                DHT_MSG_LAYOUT_PRESET_FAILED,
-                                                "Could not set pre-set layout "
-                                                "for subvolume %s",
-                                                cached_subvol->name);
-                                }
-
-                                local->op_ret = (ret == 0) ? ret : -1;
-                                local->op_errno = (ret == 0) ? ret : EINVAL;
-
-                                /* Presence of local->cached_subvol validates
-                                 * that lookup from cached node is successful
-                                 */
-
-                                if (!local->op_ret && local->loc.parent) {
-                                        dht_inode_ctx_time_update
-                                                (local->loc.parent, this,
-                                                 &local->postparent, 1);
-                                }
-
-                                gf_msg_debug (this->name, 0,
-                                              "Skipped unlinking linkto file "
-                                              "on the hashed subvolume. "
-                                              "Returning success as it is a "
-                                              "valid linkto file. Path:%s"
-                                              ,local->loc.path);
-
-                                goto unwind_hashed_and_cached;
-                        } else {
-
-                               local->skip_unlink.handle_valid_link = _gf_false;
-
-                               gf_msg_debug (this->name, 0,
-                                             "Linkto file found on hashed "
-                                             "subvol "
-                                             "and data file found on cached "
-                                             "subvolume. But linkto points to "
-                                             "different cached subvolume (%s) "
-                                             "path %s",
-                                            (local->skip_unlink.hash_links_to ?
-                                        local->skip_unlink.hash_links_to->name :
-                                             " <nil>"), local->loc.path);
-
-                               if (local->skip_unlink.opend_fd_count == 0) {
-
-
-                          ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                                  (local->xattr_req);
-
-
-                                        if (ret) {
-                                          DHT_STACK_UNWIND (lookup, frame, -1,
-                                                            EIO, NULL, NULL,
-                                                            NULL, NULL);
-                                        } else {
-                                                local->call_cnt = 1;
-                                                STACK_WIND (frame,
-                                          dht_lookup_unlink_of_false_linkto_cbk,
-                                                    hashed_subvol,
-                                                    hashed_subvol->fops->unlink,
-                                                    &local->loc, 0,
-                                                    local->xattr_req);
-                                        }
+                ret = dht_layout_preset(this, cached_subvol, local->loc.inode);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_INFO, 0,
+                           DHT_MSG_LAYOUT_PRESET_FAILED,
+                           "Could not set pre-set layout "
+                           "for subvolume %s",
+                           cached_subvol->name);
+                }
 
-                                        return 0;
+                local->op_ret = (ret == 0) ? ret : -1;
+                local->op_errno = (ret == 0) ? ret : EINVAL;
 
-                                }
-                        }
+                /* Presence of local->cached_subvol validates
+                 * that lookup from cached node is successful
+                 */
 
+                if (!local->op_ret && local->loc.parent) {
+                    dht_inode_ctx_time_update(local->loc.parent, this,
+                                              &local->postparent, 1);
+                }
+
+                gf_msg_debug(this->name, 0,
+                             "Skipped unlinking linkto file "
+                             "on the hashed subvolume. "
+                             "Returning success as it is a "
+                             "valid linkto file. Path:%s",
+                             local->loc.path);
+
+                goto unwind_hashed_and_cached;
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "Linkto file found on hashed "
+                             "subvol "
+                             "and data file found on cached "
+                             "subvolume. But linkto points to "
+                             "different cached subvolume (%s) "
+                             "path %s",
+                             (local->skip_unlink.hash_links_to
+                                  ? local->skip_unlink.hash_links_to->name
+                                  : " <nil>"),
+                             local->loc.path);
+
+                if (local->skip_unlink.opend_fd_count == 0) {
+                    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                        local->xattr_req);
+
+                    if (ret) {
+                        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL,
+                                         NULL, NULL);
+                    } else {
+                        local->call_cnt = 1;
+                        FRAME_SU_DO(frame, dht_local_t);
+                        STACK_WIND(frame, dht_lookup_unlink_of_false_linkto_cbk,
+                                   hashed_subvol, hashed_subvol->fops->unlink,
+                                   &local->loc, 0, local->xattr_req);
+                    }
+
+                    return 0;
                 }
+            }
         }
-
+    }
 
 preset_layout:
 
-        if (found_non_linkto_on_hashed) {
-
-                if (local->need_lookup_everywhere) {
-                        if (gf_uuid_compare (local->gfid, local->inode->gfid)) {
-                                /* GFID different, return error */
-                                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT,
-                                                  NULL, NULL, NULL, NULL);
-                                return 0;
-                        }
-                }
-
-                local->op_ret = 0;
-                local->op_errno = 0;
-                layout = dht_layout_for_subvol (this, cached_subvol);
-                if (!layout) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_SUBVOL_INFO,
-                                "%s: no pre-set layout for subvolume %s,"
-                                " gfid = %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"), gfid);
-                }
-
-                ret = dht_layout_set (this, local->inode, layout);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_SUBVOL_INFO,
-                                "%s: failed to set layout for subvol %s, "
-                                "gfid = %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"), gfid);
-                }
+    if (found_non_linkto_on_hashed) {
+        if (local->need_lookup_everywhere) {
+            if (gf_uuid_compare(local->gfid, local->inode->gfid)) {
+                /* GFID different, return error */
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
+                return 0;
+            }
+        }
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
+        layout = dht_layout_for_subvol(this, cached_subvol);
+        if (!layout) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: no pre-set layout for subvolume %s,"
+                   " gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
+        }
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
-                return 0;
+        ret = dht_layout_set(this, local->inode, layout);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: failed to set layout for subvol %s, "
+                   "gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
         }
 
-        if (!hashed_subvol) {
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
 
-                gf_msg_debug (this->name, 0,
-                              "Cannot create linkfile for %s on %s: "
-                              "hashed subvolume cannot be found, gfid = %s.",
-                              local->loc.path, cached_subvol->name, gfid);
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+        return 0;
+    }
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "Cannot create linkfile for %s on %s: "
+                     "hashed subvolume cannot be found, gfid = %s.",
+                     local->loc.path, cached_subvol->name, gfid);
 
-                ret = dht_layout_preset (frame->this, cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LAYOUT_PRESET_FAILED,
-                                "Failed to set layout for subvol %s"
-                                ", gfid = %s",
-                                cached_subvol ? cached_subvol->name :
-                                "<nil>", gfid);
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+        ret = dht_layout_preset(frame->this, cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "Failed to set layout for subvol %s"
+                   ", gfid = %s",
+                   cached_subvol ? cached_subvol->name : "<nil>", gfid);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+        }
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
-                return 0;
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
 
-        gf_msg_debug (this->name, 0,
-                      "Creating linkto file on %s(hash) to %s on %s (gfid = %s)",
-                      hashed_subvol->name, local->loc.path,
-                      cached_subvol->name, gfid);
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+        return 0;
+    }
 
-        ret = dht_linkfile_create (frame,
-                                   dht_lookup_linkfile_create_cbk, this,
-                                   cached_subvol, hashed_subvol, &local->loc);
+    if (frame->root->op != GF_FOP_RENAME) {
+        local->current = &local->lock[0];
+        ret = dht_protect_namespace(frame, &local->loc, hashed_subvol,
+                                    &local->current->ns,
+                                    dht_call_lookup_linkfile_create);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Creating linkto file on %s(hash) to %s on %s "
+                     "(gfid = %s)",
+                     hashed_subvol->name, local->loc.path, cached_subvol->name,
+                     gfid);
 
-        return ret;
+        ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk, this,
+                                  cached_subvol, hashed_subvol, &local->loc);
+    }
+
+    return ret;
 
 unwind_hashed_and_cached:
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        dht_set_fixed_dir_stat (&local->postparent);
-        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
 }
 
-int
-dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           inode_t *inode, struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        int           is_linkfile   = 0;
-        int           is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        xlator_t     *link_subvol   = NULL;
-        int           ret           = -1;
-        int32_t       fd_count      = 0;
-        dht_conf_t   *conf          = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-        dict_t       *dict_req      = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        local  = frame->local;
-        loc    = &local->loc;
-        conf   = this->private;
-
-        prev   = cookie;
-        subvol = prev->this;
-
-        gf_msg_debug (this->name, 0,
-                      "returned with op_ret %d and op_errno %d (%s) "
-                      "from subvol %s", op_ret, op_errno, loc->path,
-                      subvol->name);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno != ENOENT)
-                                local->op_errno = op_errno;
-                        goto unlock;
-                }
+static int
+dht_lookup_everywhere_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xattr,
+                          struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int is_linkfile = 0;
+    int is_dir = 0;
+    loc_t *loc = NULL;
+    xlator_t *link_subvol = NULL;
+    int ret = -1;
+    int32_t fd_count = 0;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *dict_req = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    local = frame->local;
+    loc = &local->loc;
+    conf = this->private;
+
+    prev = cookie;
+
+    gf_msg_debug(this->name, 0,
+                 "returned with op_ret %d and op_errno %d (%s) "
+                 "from subvol %s",
+                 op_ret, op_errno, loc->path, prev->name);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno != ENOENT)
+                local->op_errno = op_errno;
+            if (op_errno == ENODATA)
+                local->gfid_missing = _gf_true;
+            goto unlock;
+        }
 
-                if (gf_uuid_is_null (local->gfid))
-                        gf_uuid_copy (local->gfid, buf->ia_gfid);
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
 
-                gf_uuid_unparse(local->gfid, gfid);
+        gf_uuid_unparse(local->gfid, gfid);
 
-                if (gf_uuid_compare (local->gfid, buf->ia_gfid)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "%s: gfid differs on subvolume %s,"
-                                " gfid local = %s, gfid node = %s",
-                                loc->path, prev->this->name, gfid,
-                                uuid_utoa(buf->ia_gfid));
-                }
+        if (gf_uuid_compare(local->gfid, buf->ia_gfid)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+                   "%s: gfid differs on subvolume %s,"
+                   " gfid local = %s, gfid node = %s",
+                   loc->path, prev->name, gfid, uuid_utoa(buf->ia_gfid));
+        }
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr,
-                                                 conf->link_xattr_name);
-
-                if (is_linkfile) {
-                        link_subvol = dht_linkfile_subvol (this, inode, buf,
-                                                           xattr);
-                        gf_msg_debug (this->name, 0,
-                                      "found on %s linkfile %s (-> %s)",
-                                      subvol->name, loc->path,
-                                      link_subvol ? link_subvol->name : "''");
-                        goto unlock;
-                }
+        is_linkfile = check_is_linkfile(inode, buf, xattr,
+                                        conf->link_xattr_name);
 
-                is_dir = check_is_dir (inode, buf, xattr);
+        if (is_linkfile) {
+            link_subvol = dht_linkfile_subvol(this, inode, buf, xattr);
+            gf_msg_debug(this->name, 0, "found on %s linkfile %s (-> %s)",
+                         prev->name, loc->path,
+                         link_subvol ? link_subvol->name : "''");
+            goto unlock;
+        }
 
-                /* non linkfile GFID takes precedence but don't overwrite
-                 gfid if we have already found a cached file*/
-                if (!local->cached_subvol)
-                        gf_uuid_copy (local->gfid, buf->ia_gfid);
+        is_dir = check_is_dir(inode, buf, xattr);
 
-                if (is_dir) {
-                        local->dir_count++;
+        /* non linkfile GFID takes precedence but don't overwrite
+         gfid if we have already found a cached file*/
+        if (!local->cached_subvol)
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
 
-                        gf_msg_debug (this->name, 0,
-                                      "found on %s directory %s",
-                                      subvol->name, loc->path);
-                } else {
-                        local->file_count++;
-
-                        gf_msg_debug (this->name, 0,
-                                      "found cached file on %s for %s",
-                                      subvol->name, loc->path);
-
-                        if (!local->cached_subvol) {
-                                /* found one file */
-                                dht_iatt_merge (this, &local->stbuf, buf,
-                                                subvol);
-                                local->xattr = dict_ref (xattr);
-                                local->cached_subvol = subvol;
-
-                                gf_msg_debug (this->name, 0,
-                                              "storing cached on %s file"
-                                              " %s", subvol->name, loc->path);
-
-                                dht_iatt_merge (this, &local->postparent,
-                                                postparent, subvol);
-
-                                gf_uuid_copy (local->skip_unlink.cached_gfid,
-                                           buf->ia_gfid);
-                        } else {
-                                /* This is where we need 'rename' both entries logic */
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_FILE_ON_MULT_SUBVOL,
-                                        "multiple subvolumes (%s and %s) have "
-                                        "file %s (preferably rename the file "
-                                        "in the backend,and do a fresh lookup)",
-                                        local->cached_subvol->name,
-                                        subvol->name, local->loc.path);
-                        }
-                }
+        if (is_dir) {
+            local->dir_count++;
+
+            gf_msg_debug(this->name, 0, "found on %s directory %s", prev->name,
+                         loc->path);
+        } else {
+            local->file_count++;
+
+            gf_msg_debug(this->name, 0, "found cached file on %s for %s",
+                         prev->name, loc->path);
+
+            if (!local->cached_subvol) {
+                /* found one file */
+                dht_iatt_merge(this, &local->stbuf, buf);
+
+                local->xattr = dict_ref(xattr);
+                local->cached_subvol = prev;
+
+                gf_msg_debug(this->name, 0,
+                             "storing cached on %s file"
+                             " %s",
+                             prev->name, loc->path);
+
+                dht_iatt_merge(this, &local->postparent, postparent);
+
+                gf_uuid_copy(local->skip_unlink.cached_gfid, buf->ia_gfid);
+            } else {
+                /* This is where we need 'rename' both entries logic */
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_FILE_ON_MULT_SUBVOL,
+                       "multiple subvolumes (%s and %s) have "
+                       "file %s (preferably rename the file "
+                       "in the backend,and do a fresh lookup)",
+                       local->cached_subvol->name, prev->name, local->loc.path);
+            }
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (is_linkfile) {
-                ret = dict_get_int32 (xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
+    if (is_linkfile) {
+        ret = dict_get_int32(xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
 
-                /*  Any linkto file found on the non-hashed subvolume should
-                 *  be unlinked (performed in the "else if" block below)
-                 *
-                 *  But if a linkto file is found on hashed subvolume, it may be
-                 *  pointing to vaild cached node. So unlinking of linkto
-                 *  file on hashed subvolume is skipped and inside
-                 *  dht_lookup_everywhere_done, checks are performed. If this
-                 *  linkto file is found as stale linkto file, it is deleted
-                 *  otherwise unlink is skipped.
-                 */
-
-                if (local->hashed_subvol && local->hashed_subvol == subvol) {
+        /*  Any linkto file found on the non-hashed subvolume should
+         *  be unlinked (performed in the "else if" block below)
+         *
+         *  But if a linkto file is found on hashed subvolume, it may be
+         *  pointing to valid cached node. So unlinking of linkto
+         *  file on hashed subvolume is skipped and inside
+         *  dht_lookup_everywhere_done, checks are performed. If this
+         *  linkto file is found as stale linkto file, it is deleted
+         *  otherwise unlink is skipped.
+         */
 
-                        local->skip_unlink.handle_valid_link = _gf_true;
-                        local->skip_unlink.opend_fd_count = fd_count;
-                        local->skip_unlink.hash_links_to = link_subvol;
-                        gf_uuid_copy (local->skip_unlink.hashed_gfid,
-                                   buf->ia_gfid);
+        if (local->hashed_subvol && local->hashed_subvol == prev) {
+            local->skip_unlink.handle_valid_link = _gf_true;
+            local->skip_unlink.opend_fd_count = fd_count;
+            local->skip_unlink.hash_links_to = link_subvol;
+            gf_uuid_copy(local->skip_unlink.hashed_gfid, buf->ia_gfid);
+
+            gf_msg_debug(this->name, 0,
+                         "Found"
+                         " one linkto file on hashed subvol %s "
+                         "for %s: Skipping unlinking till "
+                         "everywhere_done",
+                         prev->name, loc->path);
+
+        } else if (!ret && (fd_count == 0)) {
+            dict_req = dict_new();
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_req);
+
+            if (ret) {
+                /* Skip unlinking for dict_failure
+                 *File is found as a linkto file on non-hashed,
+                 *subvolume. In the current implementation,
+                 *finding a linkto-file on non-hashed does not
+                 *always implies that it is stale. So deletion
+                 *of file should be done only when both fd is
+                 *closed and linkto-xattr is set. In case of
+                 *dict_set failure, avoid skipping of file.
+                 *NOTE: dht_frame_return should get called for
+                 *      this block.
+                 */
 
-                        gf_msg_debug (this->name, 0, "Found"
-                                      " one linkto file on hashed subvol %s "
-                                      "for %s: Skipping unlinking till "
-                                      "everywhere_done", subvol->name,
-                                      loc->path);
+                dict_unref(dict_req);
+
+            } else {
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                       "attempting deletion of stale linkfile "
+                       "%s on %s (hashed subvol is %s)",
+                       loc->path, prev->name,
+                       (local->hashed_subvol ? local->hashed_subvol->name
+                                             : "<null>"));
+                /* *
+                 * These stale files may be created using root
+                 * user. Hence deletion will work only with
+                 * root.
+                 */
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_cbk, prev,
+                           prev->fops->unlink, loc, 0, dict_req);
 
-                } else if (!ret && (fd_count == 0)) {
+                dict_unref(dict_req);
 
-                        dict_req = dict_new ();
+                return 0;
+            }
+        }
+    }
 
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (dict_req);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
 
-                        if (ret) {
+out:
+    return ret;
+}
 
-                                /* Skip unlinking for dict_failure
-                                 *File is found as a linkto file on non-hashed,
-                                 *subvolume. In the current implementation,
-                                 *finding a linkto-file on non-hashed does not
-                                 *always implies that it is stale. So deletion
-                                 *of file should be done only when both fd is
-                                 *closed and linkto-xattr is set. In case of
-                                 *dict_set failure, avoid skipping of file.
-                                 *NOTE: dht_frame_return should get called for
-                                 *      this block.
-                                 */
+int
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    int call_cnt = 0;
 
-                                dict_unref (dict_req);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", loc, out);
 
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_SUBVOL_INFO,
-                                        "attempting deletion of stale linkfile "
-                                        "%s on %s (hashed subvol is %s)",
-                                        loc->path, subvol->name,
-                                        (local->hashed_subvol?
-                                        local->hashed_subvol->name : "<null>"));
+    conf = this->private;
+    local = frame->local;
 
-                                STACK_WIND (frame, dht_lookup_unlink_cbk,
-                                            subvol, subvol->fops->unlink, loc,
-                                            0, dict_req);
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-                                dict_unref (dict_req);
+    if (!local->inode)
+        local->inode = inode_ref(loc->inode);
 
-                                return 0;
-                        }
-                }
-        }
+    gf_msg_debug(this->name, 0, "winding lookup call to %d subvols", call_cnt);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_lookup_everywhere_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, loc,
+                          local->xattr_req);
+    }
 
+    return 0;
 out:
-        return ret;
+    DHT_STACK_UNWIND(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+err:
+    return -1;
 }
 
-
 int
-dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        dht_conf_t     *conf = NULL;
-        dht_local_t    *local = NULL;
-        int             i = 0;
-        int             call_cnt = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", loc, out);
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    loc_t *loc = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", cookie, unwind);
+
+    prev = cookie;
+    subvol = prev;
+    conf = this->private;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) failed "
+               ",gfid = %s",
+               local->loc.path, subvol->name, gfid);
+
+        /* If cached subvol returned ENOTCONN, do not do
+        lookup_everywhere. We need to make sure linkfile does not get
+        removed, which can take away the namespace, and subvol is
+        anyways down. */
+
+        local->cached_subvol = NULL;
+        if (op_errno != ENOTCONN)
+            goto err;
+        else
+            goto unwind;
+    }
+
+    if (check_is_dir(inode, stbuf, xattr)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) reached dir,"
+               " gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "lookup of %s on %s (following linkfile) reached link,"
+               "gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on data file on %s,"
+               " gfid local = %s, gfid node = %s ",
+               local->loc.path, subvol->name, gfid, uuid_utoa(stbuf->ia_gfid));
+        goto err;
+    }
+
+    if ((stbuf->ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        stbuf->ia_prot.sticky = 1;
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+               "Failed to set layout for subvolume %s,"
+               "gfid = %s",
+               prev->name, gfid);
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
-        conf = this->private;
-        local = frame->local;
+unwind:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
 
-        call_cnt = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    return 0;
 
-        if (!local->inode)
-                local->inode = inode_ref (loc->inode);
+err:
+    dht_lookup_everywhere(frame, this, loc);
+out:
+    return 0;
+}
 
-        gf_msg_debug (this->name, 0,
-                      "winding lookup call to %d subvols", call_cnt);
+/* Code to get hashed subvol based on inode and loc
+   First it check if loc->parent and loc->path exist then it get
+   hashed subvol based on loc.
+*/
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_lookup_everywhere_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            loc, local->xattr_req);
+static gf_boolean_t
+dht_should_lookup_everywhere(xlator_t *this, dht_conf_t *conf, loc_t *loc)
+{
+    dht_layout_t *parent_layout = NULL;
+    int ret = 0;
+    gf_boolean_t lookup_everywhere = _gf_true;
+
+    /* lookup-optimize supersedes lookup-unhashed settings.
+     * If it is set, do not process search_unhashed
+     * If lookup-optimize if enabled, lookup everywhere if:
+     *  - this is the rebalance daemon.
+     *  - loc->parent is unavailable.
+     *  - parent_layout is unavailable
+     *  - parent_layout->commit_hash != conf->vol_commit_hash
+     */
+
+    if (conf->lookup_optimize) {
+        if (!conf->defrag && loc->parent) {
+            ret = dht_inode_ctx_layout_get(loc->parent, this, &parent_layout);
+            if (!ret && parent_layout &&
+                (parent_layout->commit_hash == conf->vol_commit_hash)) {
+                lookup_everywhere = _gf_false;
+            }
         }
+        goto out;
+    } else {
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            if (loc->parent) {
+                ret = dht_inode_ctx_layout_get(loc->parent, this,
+                                               &parent_layout);
+                if (ret || !parent_layout ||
+                    (!parent_layout->search_unhashed)) {
+                    lookup_everywhere = _gf_false;
+                }
+            } else {
+                lookup_everywhere = _gf_false;
+            }
 
-        return 0;
+            goto out;
+        }
+    }
 out:
-        DHT_STACK_UNWIND (lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
-err:
-        return -1;
+    return lookup_everywhere;
 }
 
-
 int
-dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
-{
-        call_frame_t *prev          = NULL;
-        dht_local_t  *local         = NULL;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret           = 0;
-        char          gfid[GF_UUID_BUF_SIZE]     = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, unwind);
-
-        prev   = cookie;
-        subvol = prev->this;
-        conf   = this->private;
-        local  = frame->local;
-        loc    = &local->loc;
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent)
+{
+    char is_linkfile = 0;
+    char is_dir = 0;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    uint32_t vol_commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_msg_debug(this->name, op_errno,
+                 "%s: fresh_lookup on %s returned with op_ret %d", loc->path,
+                 prev->name, op_ret);
+
+    if (op_ret == -1) {
+        if (ENTRY_MISSING(op_ret, op_errno)) {
+            if (1 == conf->subvolume_cnt) {
+                /* No need to lookup again */
+                goto out;
+            }
 
-        gf_uuid_unparse(loc->gfid, gfid);
+            gf_msg_debug(this->name, 0, "Entry %s missing on subvol %s",
+                         loc->path, prev->name);
 
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_LINK_FILE_LOOKUP_INFO,
-                        "Lookup of %s on %s (following linkfile) failed "
-                        ",gfid = %s", local->loc.path, subvol->name, gfid);
-
-                /* If cached subvol returned ENOTCONN, do not do
-                lookup_everywhere. We need to make sure linkfile does not get
-                removed, which can take away the namespace, and subvol is
-                anyways down. */
-
-                if (op_errno != ENOTCONN)
-                        goto err;
-                else
-                        goto unwind;
-        }
+            if (dht_should_lookup_everywhere(this, conf, loc)) {
+                local->op_errno = ENOENT;
+                dht_lookup_everywhere(frame, this, loc);
+                return 0;
+            }
 
-        if (check_is_dir (inode, stbuf, xattr)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LINK_FILE_LOOKUP_INFO,
-                        "Lookup of %s on %s (following linkfile) reached dir,"
-                        " gfid = %s", local->loc.path, subvol->name, gfid);
-                goto err;
+        } else {
+            /* posix returns ENODATA if the gfid is not set but the client and
+             * server protocol layers do not send the stbuf. We need to
+             * heal this so check if this is a directory on the other subvols.
+             */
+            if ((op_errno == ENOTCONN) || (op_errno == ENODATA)) {
+                dht_lookup_directory(frame, this, &local->loc);
+                return 0;
+            }
         }
-
-        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LINK_FILE_LOOKUP_INFO,
-                        "lookup of %s on %s (following linkfile) reached link,"
-                        "gfid = %s", local->loc.path, subvol->name, gfid);
-                goto err;
+        gf_msg_debug(this->name, op_errno, "%s: Lookup on subvolume %s failed",
+                     loc->path, prev->name);
+        goto out;
+    }
+
+    /* Lookup succeeded - op_ret = 0 */
+
+    /* This is required for handling stale linkfile deletion,
+     * or any more call which happens from this 'loc'.
+     */
+    if (gf_uuid_is_null(local->gfid)) {
+        /*This is set from the first successful response*/
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+
+    if (!conf->vch_forced) {
+        /* Update the commit hash in conf if it is found */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
+    }
 
-        if (gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on data file on %s,"
-                        " gfid local = %s, gfid node = %s ",
-                        local->loc.path, subvol->name, gfid,
-                        uuid_utoa(stbuf->ia_gfid));
-                goto err;
-        }
+    is_dir = check_is_dir(inode, stbuf, xattr);
+    if (is_dir) {
+        /* A directory is present on all subvols, send the lookup to
+         * all subvols now */
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
 
-        if ((stbuf->ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                stbuf->ia_prot.sticky = 1;
-        }
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
 
-        ret = dht_layout_preset (this, prev->this, inode);
+    if (!is_linkfile) {
+        /* non-directory and not a linkto file. This is a data file
+         * Update the layout to point to the cached subvol
+         */
+
+        ret = dht_layout_preset(this, prev, inode);
         if (ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LAYOUT_PRESET_FAILED,
-                        "Failed to set layout for subvolume %s,"
-                        "gfid = %s", prev->this->name, gfid);
-                op_ret   = -1;
-                op_errno = EINVAL;
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "%s: could not set pre-set layout for subvolume %s",
+                   loc->path, prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
         }
+        goto out;
+    }
 
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+    /* This is a linkto file. Get the value of the target subvol from the
+     * linkto xattr and lookup there to see if the file exists
+     */
+    subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "%s: No link subvol for linkto", loc->path);
+        dht_lookup_everywhere(frame, this, loc);
+        return 0;
+    }
 
-unwind:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
+    gf_msg_debug(this->name, 0, "%s: Calling lookup on linkto target %s",
+                 loc->path, subvol->name);
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                      subvol->fops->lookup, &local->loc, local->xattr_req);
+
+    return 0;
 
-err:
-        dht_lookup_everywhere (frame, this, loc);
 out:
-        return 0;
+    /*
+     * FIXME: postparent->ia_size and postparent->st_blocks do not have
+     * correct values. since, postparent corresponds to a directory these
+     * two members should have values equal to sum of corresponding values
+     * from each of the subvolume. See dht_iatt_merge for reference.
+     */
+
+    if (!op_ret && local && local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
+err:
+    return 0;
 }
 
-
-int
-dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc)
+/* For directories, check if acl xattrs have been requested (by the acl
+ * xlator), if not, request for them. These xattrs are needed for dht dir
+ * self-heal to perform proper self-healing of dirs
+ */
+static void
+dht_check_and_set_acl_xattr_req(xlator_t *this, dict_t *xattr_req)
 {
-        int           call_cnt = 0;
-        int           i = 0;
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           ret = 0;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", loc, unwind);
+    GF_ASSERT(xattr_req);
 
-        conf = this->private;
-        local = frame->local;
+    if (!dict_get(xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_ACCESS_XATTR);
+    }
+
+    if (!dict_get(xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_DEFAULT_XATTR);
+    }
+
+    return;
+}
 
-        call_cnt        = conf->subvolume_cnt;
+/* for directories, we need the following info:
+ * the layout : trusted.glusterfs.dht
+ * the mds information : trusted.glusterfs.dht.mds
+ * the acl info: See above
+ */
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Xattr to get the layout for a directory
+     */
+    ret = dict_set_uint32(xattr_req, conf->xattr_name, 4 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->xattr_name, loc->path);
+        goto err;
+    }
+
+    /*Non-fatal failure */
+    ret = dict_set_uint32(xattr_req, conf->mds_xattr_key, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    dht_check_and_set_acl_xattr_req(this, xattr_req);
+    ret = 0;
+err:
+    return ret;
+}
+
+/* If the hashed subvol is present, send the lookup to only that subvol first.
+ * If no hashed subvol, send a lookup to all subvols and proceed based on the
+ * responses.
+ */
+static int
+dht_do_fresh_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int call_cnt = 0;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Since we don't know whether this is a file or a directory,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    /* Fuse sets a random value in gfid-req. If the gfid is missing
+     * on one or more subvols, posix will set the gfid to this value,
+     * causing GFID mismatches for directories. Remove the value fuse
+     * has sent before sending the lookup.
+     */
+    ret = dict_get_gfuuid(local->xattr_req, "gfid-req", &local->gfid_req);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "%s: No gfid-req available", loc->path);
+    } else {
+        dict_del(local->xattr_req, "gfid-req");
+    }
+    /* This should have been set in dht_lookup */
+    hashed_subvol = local->hashed_subvol;
+
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "%s: no subvolume in layout for path, "
+                     "checking on all the subvols to see if "
+                     "it is a directory",
+                     loc->path);
+
+        call_cnt = conf->subvolume_cnt;
         local->call_cnt = call_cnt;
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+        /* Allocate a layout. This will be populated and saved in
+         * the dht inode_ctx on successful lookup
+         */
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
         if (!local->layout) {
-                goto unwind;
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        if (local->xattr != NULL) {
-                dict_unref (local->xattr);
-                local->xattr = NULL;
-        }
-
-        if (!gf_uuid_is_null (local->gfid)) {
-                ret = dict_set_static_bin (local->xattr_req, "gfid-req",
-                                           local->gfid, 16);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:"
-                                " key = gfid-req", local->loc.path);
-        }
+        gf_msg_debug(this->name, 0,
+                     "%s: Found null hashed subvol. Calling lookup"
+                     " on all nodes.",
+                     loc->path);
 
         for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_lookup_dir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
         }
         return 0;
-unwind:
-        DHT_STACK_UNWIND (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
-out:
-        return 0;
+    }
 
-}
+    /* if the hashed_subvol is non-null, send the lookup there first so
+     * as to see whether we have a file or a directory */
+    gf_msg_debug(this->name, 0, "%s: Calling fresh lookup on %s", loc->path,
+                 hashed_subvol->name);
 
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->lookup, loc, local->xattr_req);
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-int
-dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                struct iatt *postparent)
-{
-        char          is_linkfile   = 0;
-        char          is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        loc_t        *loc           = NULL;
-        call_frame_t *prev          = NULL;
-        int           ret           = 0;
-        dht_layout_t *parent_layout = NULL;
-        uint32_t      vol_commit_hash = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
+static int
+dht_do_revalidate(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+    int gen = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0,
+                     "path = %s. No layout found in the inode ctx.", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Generation number has changed. This layout may be stale. */
+    if (layout->gen && (layout->gen < conf->gen)) {
+        gen = layout->gen;
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+        local->cached_subvol = NULL;
+
+        gf_msg_debug(this->name, 0,
+                     "path = %s. In memory layout may be stale."
+                     "(layout->gen (%d) is less than "
+                     "conf->gen (%d)). Calling fresh lookup.",
+                     loc->path, gen, conf->gen);
+
+        dht_do_fresh_lookup(frame, this, loc);
+        return 0;
+    }
+
+    local->inode = inode_ref(loc->inode);
+
+    /* Since we don't know whether this has changed,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    if (IA_ISDIR(local->inode->ia_type)) {
+        ret = dht_inode_ctx_mdsvol_get(local->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg_debug(this->name, 0, "path = %s. No mds subvol in inode ctx",
+                         local->loc.path);
+        }
+        local->mds_subvol = mds_subvol;
+        local->call_cnt = conf->subvolume_cnt;
 
-        /* This is required for handling stale linkfile deletion,
-         * or any more call which happens from this 'loc'.
+        /* local->call_cnt will change as responses are processed. Always use a
+         * local copy to loop through the STACK_WIND calls
          */
-        if (!op_ret && gf_uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
-
-        gf_msg_debug (this->name, op_errno,
-                      "fresh_lookup returned for %s with op_ret %d",
-                      loc->path, op_ret);
-
-        if (!conf->vch_forced) {
-                ret = dict_get_uint32 (xattr, conf->commithash_xattr_name,
-                                       &vol_commit_hash);
-                if (ret == 0) {
-                        conf->vol_commit_hash = vol_commit_hash;
-                }
-        }
 
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                gf_msg_debug (this->name, 0,
-                              "Entry %s missing on subvol %s",
-                              loc->path, prev->this->name);
-
-                /* lookup-optimize supercedes lookup-unhashed settings,
-                 *   - so if it is set, do not process search_unhashed
-                 *   - except, in the case of rebalance deamon, we want to
-                 *     force the lookup_everywhere behavior */
-                if (!conf->defrag && conf->lookup_optimize && loc->parent) {
-                        ret = dht_inode_ctx_layout_get (loc->parent, this,
-                                                        &parent_layout);
-                        if (ret || !parent_layout ||
-                            (parent_layout->commit_hash !=
-                             conf->vol_commit_hash)) {
-                                gf_msg_debug (this->name, 0,
-                                              "hashes don't match (ret - %d,"
-                                              " parent_layout - %p, parent_hash - %x,"
-                                              " vol_hash - %x), do global lookup",
-                                              ret, parent_layout,
-                                              (parent_layout ?
-                                              parent_layout->commit_hash : -1),
-                                              conf->vol_commit_hash);
-                                local->op_errno = ENOENT;
-                                dht_lookup_everywhere (frame, this, loc);
-                                return 0;
-                        }
-                } else {
-                        if (conf->search_unhashed ==
-                            GF_DHT_LOOKUP_UNHASHED_ON) {
-                                local->op_errno = ENOENT;
-                                dht_lookup_everywhere (frame, this, loc);
-                                return 0;
-                        }
-
-                        if ((conf->search_unhashed ==
-                            GF_DHT_LOOKUP_UNHASHED_AUTO) &&
-                            (loc->parent)) {
-                                ret = dht_inode_ctx_layout_get (loc->parent,
-                                                                this,
-                                                                &parent_layout);
-                                if (ret || !parent_layout)
-                                        goto out;
-                                if (parent_layout->search_unhashed) {
-                                        local->op_errno = ENOENT;
-                                        dht_lookup_everywhere (frame, this,
-                                                               loc);
-                                        return 0;
-                                }
-                        }
-                }
-        }
+        call_cnt = local->call_cnt;
 
-        if (op_ret == 0) {
-                is_dir      = check_is_dir (inode, stbuf, xattr);
-                if (is_dir) {
-                        local->inode = inode_ref (inode);
-                        local->xattr = dict_ref (xattr);
-                }
-        }
-
-        if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
-                dht_lookup_directory (frame, this, &local->loc);
-                return 0;
-        }
-
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, op_errno,
-                              "Lookup of %s for subvolume"
-                              " %s failed", loc->path,
-                              prev->this->name);
-                goto out;
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, loc,
+                              local->xattr_req);
         }
+        return 0;
+    }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-
-        if (!is_linkfile) {
-                /* non-directory and not a linkfile */
+    /* If not a dir, this should be 1 */
+    local->call_cnt = layout->cnt;
+    call_cnt = local->call_cnt;
 
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LAYOUT_PRESET_FAILED,
-                                "could not set pre-set layout for subvolume %s",
-                                prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto out;
-                }
-                goto out;
-        }
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
 
-        subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
-        if (!subvol) {
+        gf_msg_debug(this->name, 0,
+                     "path = %s. Calling "
+                     "revalidate lookup on %s",
+                     loc->path, subvol->name);
 
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO, "linkfile not having link "
-                        "subvol for %s", loc->path);
+        STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-                gf_msg_debug (this->name, 0,
-                              "linkfile not having link subvolume. path=%s",
-                              loc->path);
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
-        }
+/* Depending on the input, decide if this is a:
+ * fresh-lookup: loc->name is provided but no dht inode ctx
+ * revalidation: loc->name is provided, dht inode ctx is present
+ * discover: gfid based nameless lookup.
+ */
 
-        gf_msg_debug (this->name, 0,
-                      "Calling lookup on linkto target %s for path %s",
-                      subvol->name, loc->path);
+int
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    loc_t new_loc = {
+        0,
+    };
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    if (!conf)
+        goto err;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_filter_loc_subvol_key(this, loc, &new_loc, &hashed_subvol);
+    if (ret) {
+        loc_wipe(&local->loc);
+        ret = loc_dup(&new_loc, &local->loc);
+
+        /* we no longer need 'new_loc' entries */
+        loc_wipe(&new_loc);
+
+        /* check if loc_dup() is successful */
+        if (ret == -1) {
+            op_errno = errno;
+            gf_msg_debug(this->name, errno,
+                         "copying location failed for path=%s", loc->path);
+            goto err;
+        }
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    /* Nameless lookup */
+
+    /* This is usually sent by NFS. Lookups are done based on the gfid and
+     * no name information is available. Without the name, dht cannot calculate
+     * the hash and has to send a lookup to all subvols.
+     */
+    if (gf_uuid_is_null(loc->pargfid) && !gf_uuid_is_null(loc->gfid) &&
+        !__is_root_gfid(loc->inode->gfid)) {
+        local->cached_subvol = NULL;
+        dht_do_discover(frame, this, loc);
+        return 0;
+    }
+
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-        STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                    subvol, subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+    if (!hashed_subvol)
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+    local->hashed_subvol = hashed_subvol;
 
+    if (is_revalidate(loc)) {
+        /* The entry has been looked up before and has a dht inode_ctx
+         */
+        dht_do_revalidate(frame, this, loc);
         return 0;
-
-out:
-        /*
-         * FIXME: postparent->ia_size and postparent->st_blocks do not have
-         * correct values. since, postparent corresponds to a directory these
-         * two members should have values equal to sum of corresponding values
-         * from each of the subvolume. See dht_iatt_merge for reference.
+    } else {
+        /* Entry has not been looked up before
          */
+        dht_do_fresh_lookup(frame, this, loc);
+        return 0;
+    }
 
-        if (!op_ret && local && local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
-
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
+    return 0;
 err:
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
-/* For directories, check if acl xattrs have been requested (by the acl xlator),
- * if not, request for them. These xattrs are needed for dht dir self-heal to
- * perform proper self-healing of dirs
- */
-void
-dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+static int
+dht_unlink_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        int     ret = 0;
-
-        GF_ASSERT (inode);
-        GF_ASSERT (xattr_req);
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        if (inode->ia_type != IA_IFDIR)
-                return;
+    local = frame->local;
+    prev = cookie;
 
-        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s",
-                                POSIX_ACL_ACCESS_XATTR);
+    LOCK(&frame->lock);
+    {
+        if ((op_ret == -1) &&
+            !((op_errno == ENOENT) || (op_errno == ENOTCONN))) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink link: subvolume %s returned -1", prev->name);
+            goto post_unlock;
         }
 
-        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s",
-                                POSIX_ACL_DEFAULT_XATTR);
-        }
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
 
-        return;
+    return 0;
 }
 
-int
-dht_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
-{
-        xlator_t     *subvol = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-        loc_t         new_loc = {0,};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        conf = this->private;
-        if (!conf)
-                goto err;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+static int
+dht_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *hashed_subvol = NULL;
 
-        ret = dht_filter_loc_subvol_key (this, loc, &new_loc,
-                                         &hashed_subvol);
-        if (ret) {
-                loc_wipe (&local->loc);
-                ret = loc_dup (&new_loc, &local->loc);
-
-                /* we no more need 'new_loc' entries */
-                loc_wipe (&new_loc);
-
-                /* check if loc_dup() is successful */
-                if (ret == -1) {
-                        op_errno = errno;
-                        gf_msg_debug (this->name, errno,
-                                      "copying location failed for path=%s",
-                                      loc->path);
-                        goto err;
-                }
-        }
+    local = frame->local;
+    prev = cookie;
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno != ENOENT) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+            } else {
+                local->op_ret = 0;
+            }
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink: subvolume %s returned -1", prev->name);
+            goto post_unlock;
         }
 
-        if (gf_uuid_is_null (loc->pargfid) && !gf_uuid_is_null (loc->gfid) &&
-            !__is_root_gfid (loc->inode->gfid)) {
-                local->cached_subvol = NULL;
-                dht_discover (frame, this, loc);
-                return 0;
-        }
+        local->op_ret = 0;
 
-        if (__is_root_gfid(loc->gfid)) {
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->commithash_xattr_name,
-                                       sizeof(uint32_t));
-        }
+        local->postparent = *postparent;
+        local->preparent = *preparent;
 
-        if (!hashed_subvol)
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-        local->hashed_subvol = hashed_subvol;
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!local->op_ret) {
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+        if (hashed_subvol && hashed_subvol != local->cached_subvol) {
+            /*
+             * If hashed and cached are different, then we need
+             * to unlink linkfile from hashed subvol if data
+             * file is deleted successfully
+             */
+            STACK_WIND_COOKIE(frame, dht_unlink_linkfile_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->unlink,
+                              &local->loc, local->flags, xdata);
+            return 0;
+        }
+    }
+
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
+
+    return 0;
+}
 
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "Revalidate lookup without cache."
-                                      " path=%s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+static int
+dht_common_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_trace (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
+static int
+dht_fix_layout_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        local->cached_subvol = NULL;
+    if (op_ret == 0) {
+        /* update the layout in the inode ctx */
+        local = frame->local;
+        layout = local->selfheal.layout;
 
-                        gf_msg_debug(this->name, 0,
-                                     "Called revalidate lookup for %s, "
-                                     "but layout->gen (%d) is less than "
-                                     "conf->gen (%d), calling fresh_lookup",
-                                     loc->path, layout->gen, conf->gen);
+        dht_layout_set(this, local->loc.inode, layout);
+    }
 
-                        goto do_fresh_lookup;
-                }
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-                local->inode = inode_ref (loc->inode);
+static int
+dht_err_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->xattr_name, loc->path);
-                        goto err;
-                }
-                /* need it in case file is not found on cached file
-                 * on revalidate path and we may encounter linkto files on
-                 * with dht_lookup_everywhere*/
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->link_xattr_name, loc->path);
-                        goto err;
-                }
-                if (IA_ISDIR (local->inode->ia_type)) {
-                        local->call_cnt = call_cnt = conf->subvolume_cnt;
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_revalidate_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            loc, local->xattr_req);
-                        }
-                        return 0;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                call_cnt = local->call_cnt = layout->cnt;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
+        }
 
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", GLUSTERFS_OPEN_FD_COUNT, loc->path);
-                        goto err;
-                }
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((local->fop == GF_FOP_SETXATTR) ||
+            (local->fop == GF_FOP_FSETXATTR)) {
+            DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+            /* 'local' itself may not be valid after this */
+            goto out;
+        }
+        if ((local->fop == GF_FOP_REMOVEXATTR) ||
+            (local->fop == GF_FOP_FREMOVEXATTR)) {
+            DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+        }
+    }
 
-                for (i = 0; i < call_cnt; i++) {
-                        subvol = layout->list[i].xlator;
+out:
+    return 0;
+}
 
-                        gf_msg_debug (this->name, 0, "calling "
-                                      "revalidate lookup for %s at %s",
-                                      loc->path, subvol->name);
+/* Set the value[] of key into dict after convert from
+   host byte order to network byte order
+*/
+int32_t
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size)
+{
+    int ret = -1;
+    int32_t *ptr = NULL;
+    int32_t vindex;
 
-                        STACK_WIND (frame, dht_revalidate_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    &local->loc, local->xattr_req);
+    if (value == NULL) {
+        return -EINVAL;
+    }
 
-                }
-        } else {
-        do_fresh_lookup:
-                /* TODO: remove the hard-coding */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->xattr_name, loc->path);
-                        goto err;
-                }
+    ptr = GF_MALLOC(sizeof(int32_t) * size, gf_common_mt_char);
+    if (ptr == NULL) {
+        return -ENOMEM;
+    }
+    for (vindex = 0; vindex < size; vindex++) {
+        ptr[vindex] = hton32(value[vindex]);
+    }
+    ret = dict_set_bin(dict, key, ptr, sizeof(int32_t) * size);
+    if (ret)
+        GF_FREE(ptr);
+    return ret;
+}
 
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->link_xattr_name, loc->path);
-                        goto err;
-                }
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", GLUSTERFS_OPEN_FD_COUNT, loc->path);
-                        goto err;
-                }
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+static int
+dht_common_mds_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    call_frame_t *prev = cookie;
 
-                if (!hashed_subvol) {
+    local = frame->local;
 
-                        gf_msg_debug (this->name, 0,
-                                      "no subvolume in layout for path=%s, "
-                                      "checking on all the subvols to see if "
-                                      "it is a directory", loc->path);
+    if (op_ret)
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
 
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
 
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
 
-                        gf_msg_debug (this->name, 0,
-                                      "Found null hashed subvol. Calling lookup"
-                                      " on all nodes.");
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, 0, op_errno, NULL);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
 
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_lookup_dir_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, 0, op_errno, NULL);
+    }
 
-                gf_msg_debug (this->name, 0, "Calling fresh lookup for %s on"
-                              " %s", loc->path, hashed_subvol->name);
+out:
+    return 0;
+}
 
-                STACK_WIND (frame, dht_lookup_cbk,
-                            hashed_subvol, hashed_subvol->fops->lookup,
-                            loc, local->xattr_req);
+/* Code to wind a xattrop call to add 1 on current mds internal xattr
+   value
+*/
+static int
+dht_setxattr_non_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    dict_t *xattrop = NULL;
+    int32_t addone[1] = {1};
+    call_frame_t *prev = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret && !local->op_ret) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->this->name);
+            goto post_unlock;
         }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
 
-        return 0;
+    if (is_last_call(this_call_cnt)) {
+        if (!local->op_ret) {
+            xattrop = dict_new();
+            if (!xattrop) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                       "dictionary creation failed");
+                ret = -1;
+                goto out;
+            }
+            ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, addone, 1);
+            if (ret != 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                       "dictionary set array failed ");
+                ret = -1;
+                goto out;
+            }
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_REMOVEXATTR)) {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->xattrop, &local->loc,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            } else {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->fxattrop, local->fd,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            }
+        } else {
+            if (local->fop == GF_FOP_SETXATTR) {
+                DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
-}
+            if (local->fop == GF_FOP_FSETXATTR) {
+                DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
 
-int
-dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t     *local          = NULL;
-        call_frame_t    *prev           = NULL;
+            if (local->fop == GF_FOP_REMOVEXATTR) {
+                DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if ((op_ret == -1) && !((op_errno == ENOENT) ||
-                                        (op_errno == ENOTCONN))) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1",
-                                      prev->this->name);
-                        goto unlock;
-                }
+            if (local->fop == GF_FOP_FREMOVEXATTR) {
+                DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+        }
+    }
+out:
+    if (ret) {
+        if (local->fop == GF_FOP_SETXATTR) {
+            DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
+        }
 
-                local->op_ret = 0;
+        if (local->fop == GF_FOP_FSETXATTR) {
+            DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
+        }
 
-        return 0;
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
+        }
+    }
+just_return:
+    if (xattrop)
+        dict_unref(xattrop);
+    return 0;
 }
 
-int
-dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int
+dht_setxattr_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t     *local          = NULL;
-        call_frame_t    *prev           = NULL;
-        xlator_t        *hashed_subvol  = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    call_frame_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    int i = 0;
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno != ENOENT) {
-                                local->op_ret   = -1;
-                                local->op_errno = op_errno;
-                        } else {
-                                local->op_ret = 0;
-                        }
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink: subvolume %s returned -1",
-                                       prev->this->name);
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
 
-                local->op_ret = 0;
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
 
-                local->postparent = *postparent;
-                local->preparent = *preparent;
+    local->op_ret = 0;
+    local->call_cnt = conf->subvolume_cnt - 1;
+    local->xdata = dict_ref(xdata);
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && (mds_subvol == conf->subvolumes[i]))
+            continue;
+        if (local->fop == GF_FOP_SETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->setxattr, &local->loc,
+                       local->xattr, local->flags, local->xattr_req);
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        if (!local->op_ret) {
-                hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-                if (hashed_subvol &&
-                hashed_subvol != local->cached_subvol) {
-                        /*
-                         * If hashed and cached are different, then we need
-                         * to unlink linkfile from hashed subvol if data
-                         * file is deleted successfully
-                         */
-                        STACK_WIND (frame, dht_unlink_linkfile_cbk,
-                                    hashed_subvol,
-                                    hashed_subvol->fops->unlink, &local->loc,
-                                    local->flags, xdata);
-                        return 0;
-                }
+        if (local->fop == GF_FOP_FSETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fsetxattr, local->fd,
+                       local->xattr, local->flags, local->xattr_req);
+        }
+
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->removexattr, &local->loc,
+                       local->key, local->xattr_req);
         }
 
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fremovexattr, local->fd,
+                       local->key, local->xattr_req);
+        }
+    }
 
-        return 0;
+    return 0;
+out:
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
 }
 
-int
-dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_xattrop_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->this->name);
-                        goto unlock;
-                }
+    dht_local_t *local = NULL;
+    call_frame_t *prev = NULL;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+    local = frame->local;
+    prev = cookie;
 
-        return 0;
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        local->op_ret = op_ret;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->setxattr, &local->loc, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fsetxattr, local->fd, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->removexattr, &local->loc,
+                   local->key, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fremovexattr, local->fd, local->key,
+                   local->xattr_req);
+    }
+
+    return 0;
+out:
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
 }
 
 static void
-fill_layout_info (dht_layout_t *layout, char *buf)
+fill_layout_info(dht_layout_t *layout, char *buf)
 {
-        int i = 0;
-        char tmp_buf[128] = {0,};
+    int i = 0;
+    char tmp_buf[128] = {
+        0,
+    };
 
-        for (i = 0; i < layout->cnt; i++) {
-                snprintf (tmp_buf, 128, "(%s %u %u)",
-                          layout->list[i].xlator->name,
-                          layout->list[i].start,
-                          layout->list[i].stop);
-                if (i)
-                        strcat (buf, " ");
-                strcat (buf, tmp_buf);
-        }
+    for (i = 0; i < layout->cnt; i++) {
+        snprintf(tmp_buf, sizeof(tmp_buf), "(%s %u %u)",
+                 layout->list[i].xlator->name, layout->list[i].start,
+                 layout->list[i].stop);
+        if (i)
+            strcat(buf, " ");
+        strcat(buf, tmp_buf);
+    }
 }
 
-void
-dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
-                         char *xattr_buf, int32_t alloc_len,
-                         int flag, char *layout_buf)
-{
-        if (flag && local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
-                          this->name, local->xattr_val, this->name,
-                          layout_buf);
-        else if (local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "(<"DHT_PATHINFO_HEADER"%s> %s)",
-                          this->name, local->xattr_val);
-        else if (flag)
-                snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
-                          this->name, layout_buf);
+static void
+dht_fill_pathinfo_xattr(xlator_t *this, dht_local_t *local, char *xattr_buf,
+                        int32_t alloc_len, int flag, char *layout_buf)
+{
+    if (flag) {
+        if (local->xattr_val) {
+            snprintf(xattr_buf, alloc_len,
+                     "((<" DHT_PATHINFO_HEADER "%s> %s) (%s-layout %s))",
+                     this->name, local->xattr_val, this->name, layout_buf);
+        } else {
+            snprintf(xattr_buf, alloc_len, "(%s-layout %s)", this->name,
+                     layout_buf);
+        }
+    } else if (local->xattr_val) {
+        snprintf(xattr_buf, alloc_len, "(<" DHT_PATHINFO_HEADER "%s> %s)",
+                 this->name, local->xattr_val);
+    } else {
+        xattr_buf[0] = '\0';
+    }
 }
 
-int
-dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
-                              int op_errno)
+static int
+dht_vgetxattr_alloc_and_fill(dht_local_t *local, dict_t *xattr, xlator_t *this,
+                             int op_errno)
 {
-        int      ret       = -1;
-        char    *value     = NULL;
-        int32_t  plen      = 0;
+    int ret = -1;
+    char *value = NULL;
 
-        ret = dict_get_str (xattr, local->xsel, &value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "Subvolume %s returned -1", this->name);
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto out;
-        }
+    ret = dict_get_str(xattr, local->xsel, &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "Subvolume %s returned -1", this->name);
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto out;
+    }
 
-        local->alloc_len += strlen(value);
+    local->alloc_len += strlen(value);
 
+    if (!local->xattr_val) {
+        local->alloc_len += (SLEN(DHT_PATHINFO_HEADER) + 10);
+        local->xattr_val = GF_MALLOC(local->alloc_len, gf_common_mt_char);
         if (!local->xattr_val) {
-                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
-                                              gf_common_mt_char);
-                if (!local->xattr_val) {
-                        ret = -1;
-                        goto out;
-                }
+            ret = -1;
+            goto out;
         }
+        local->xattr_val[0] = '\0';
+    }
 
-        if (local->xattr_val) {
-                plen = strlen (local->xattr_val);
-                if (plen) {
-                        /* extra byte(s) for \0 to be safe */
-                        local->alloc_len += (plen + 2);
-                        local->xattr_val = GF_REALLOC (local->xattr_val,
-                                                       local->alloc_len);
-                        if (!local->xattr_val) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
-
-                (void) strcat (local->xattr_val, value);
-                (void) strcat (local->xattr_val, " ");
-                local->op_ret = 0;
+    int plen = strlen(local->xattr_val);
+    if (plen) {
+        /* extra byte(s) for \0 to be safe */
+        local->alloc_len += (plen + 2);
+        local->xattr_val = GF_REALLOC(local->xattr_val, local->alloc_len);
+        if (!local->xattr_val) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = 0;
+    (void)strcat(local->xattr_val, value);
+    (void)strcat(local->xattr_val, " ");
+    local->op_ret = 0;
 
- out:
-        return ret;
+    ret = 0;
+
+out:
+    return ret;
 }
 
-int
-dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
-                            gf_boolean_t flag)
+static int
+dht_vgetxattr_fill_and_set(dht_local_t *local, dict_t **dict, xlator_t *this,
+                           gf_boolean_t flag)
 {
-        int   ret             = -1;
-        char *xattr_buf       = NULL;
-        char layout_buf[8192] = {0,};
-
-        if (flag)
-                fill_layout_info (local->layout, layout_buf);
+    int ret = -1;
+    char *xattr_buf = NULL;
+    char layout_buf[8192] = {
+        0,
+    };
 
-        *dict = dict_new ();
-        if (!*dict)
-                goto out;
+    if (flag)
+        fill_layout_info(local->layout, layout_buf);
 
-        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+    *dict = dict_new();
+    if (!*dict)
+        goto out;
 
-        /* we would need max this many bytes to create xattr string
-         * extra 40 bytes is just an estimated amount of additional
-         * space required as we include translator name and some
-         * spaces, brackets etc. when forming the pathinfo string.
-         *
-         * For node-uuid we just don't have all the pretty formatting,
-         * but since this is a generic routine for pathinfo & node-uuid
-         * we dont have conditional space allocation and try to be
-         * generic
-         */
-        local->alloc_len += (2 * strlen (this->name))
-                + strlen (layout_buf)
-                + 40;
-        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
-                               gf_common_mt_char);
-        if (!xattr_buf)
-                goto out;
+    local->xattr_val[strlen(local->xattr_val) - 1] = '\0';
+
+    /* we would need max this many bytes to create xattr string
+     * extra 40 bytes is just an estimated amount of additional
+     * space required as we include translator name and some
+     * spaces, brackets etc. when forming the pathinfo string.
+     *
+     * For node-uuid we just don't have all the pretty formatting,
+     * but since this is a generic routine for pathinfo & node-uuid
+     * we don't have conditional space allocation and try to be
+     * generic
+     */
+    local->alloc_len += (2 * strlen(this->name)) + strlen(layout_buf) + 40;
+    xattr_buf = GF_MALLOC(local->alloc_len, gf_common_mt_char);
+    if (!xattr_buf)
+        goto out;
 
-        if (XATTR_IS_PATHINFO (local->xsel)) {
-                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
-                                                local->alloc_len, flag,
-                                                layout_buf);
-        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
-                (void) snprintf (xattr_buf, local->alloc_len, "%s",
-                                 local->xattr_val);
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "Unknown local->xsel (%s)", local->xsel);
-                GF_FREE (xattr_buf);
-                goto out;
-        }
+    if (XATTR_IS_PATHINFO(local->xsel)) {
+        (void)dht_fill_pathinfo_xattr(this, local, xattr_buf, local->alloc_len,
+                                      flag, layout_buf);
+    } else if ((XATTR_IS_NODE_UUID(local->xsel)) ||
+               (XATTR_IS_NODE_UUID_LIST(local->xsel))) {
+        (void)snprintf(xattr_buf, local->alloc_len, "%s", local->xattr_val);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GET_XATTR_FAILED,
+               "Unknown local->xsel (%s)", local->xsel);
+        GF_FREE(xattr_buf);
+        goto out;
+    }
 
-        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
-        if (ret)
-                GF_FREE (xattr_buf);
-        GF_FREE (local->xattr_val);
+    ret = dict_set_dynstr(*dict, local->xsel, xattr_buf);
+    if (ret)
+        GF_FREE(xattr_buf);
+    GF_FREE(local->xattr_val);
 
- out:
-        return ret;
+out:
+    return ret;
 }
-int
-dht_find_local_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
-{
-        dht_local_t  *local         = NULL;
-        dht_conf_t   *conf          = NULL;
-        call_frame_t *prev          = NULL;
-        int           this_call_cnt = 0;
-        int           ret           = 0;
-        char         *uuid_str      = NULL;
-        char         *uuid_list     = NULL;
-        char         *next_uuid_str = NULL;
-        char         *saveptr       = NULL;
-        uuid_t        node_uuid     = {0,};
 
+static int
+dht_find_local_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    char *uuid_str = NULL;
+    char *uuid_list = NULL;
+    char *next_uuid_str = NULL;
+    char *saveptr = NULL;
+    uuid_t node_uuid = {
+        0,
+    };
+    char *uuid_list_copy = NULL;
+    int count = 0;
+    int i = 0;
+    int index = 0;
+    int found = 0;
+    nodeuuid_info_t *tmp_ptr = NULL;
+
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    VALIDATE_OR_GOTO(conf->defrag, out);
+
+    gf_msg_debug(this->name, 0, "subvol %s returned", prev->name);
+
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            if (op_errno == ENODATA)
+                gf_msg_debug(this->name, 0, "failed to get node-uuid");
+            else
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "failed to get node-uuid");
+            goto post_unlock;
+        }
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
+        ret = dict_get_str(xattr, local->xsel, &uuid_list);
 
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-                if (op_ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                DHT_MSG_GET_XATTR_FAILED,
-                                "getxattr err for dir");
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_GET_FAILED,
+                   "Failed to get %s", local->xsel);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+            goto unlock;
+        }
 
-                ret = dict_get_str (xattr, local->xsel, &uuid_list);
+        /* As DHT will not know details of its child xlators
+         * we need to parse this twice to get the count first
+         * and allocate memory later.
+         */
+        count = 0;
+        index = conf->local_subvols_cnt;
 
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_GET_FAILED,
-                                "Failed to get %s", local->xsel);
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                        goto unlock;
-                }
+        uuid_list_copy = gf_strdup(uuid_list);
+        if (!uuid_list_copy)
+            goto unlock;
 
-                for (uuid_str = strtok_r (uuid_list, " ", &saveptr);
-                     uuid_str;
-                     uuid_str = next_uuid_str) {
-
-                        next_uuid_str = strtok_r (NULL, " ", &saveptr);
-                        if (gf_uuid_parse (uuid_str, node_uuid)) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_UUID_PARSE_ERROR,
-                                        "Failed to parse uuid"
-                                        " failed for %s", prev->this->name);
-                                local->op_ret = -1;
-                                local->op_errno = EINVAL;
-                                goto unlock;
-                        }
+        for (uuid_str = strtok_r(uuid_list, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            if (gf_uuid_parse(uuid_str, node_uuid)) {
+                local->op_ret = -1;
+                local->op_errno = EINVAL;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UUID_PARSE_ERROR,
+                       "Failed to parse uuid for %s", prev->name);
+                goto post_unlock;
+            }
 
-                        if (gf_uuid_compare (node_uuid, conf->defrag->node_uuid)) {
-                                gf_msg_debug (this->name, 0, "subvol %s does not"
-                                              "belong to this node",
-                                              prev->this->name);
-                        } else {
-                                conf->local_subvols[(conf->local_subvols_cnt)++]
-                                        = prev->this;
-                                gf_msg_debug (this->name, 0, "subvol %s belongs to"
-                                              " this node", prev->this->name);
-                                break;
-                        }
-                }
+            count++;
+            if (gf_uuid_compare(node_uuid, conf->defrag->node_uuid)) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s does not"
+                             "belong to this node",
+                             prev->name);
+            } else {
+                /* handle multiple bricks of the same replica
+                 * on the same node */
+                if (found)
+                    continue;
+                conf->local_subvols[(conf->local_subvols_cnt)++] = prev;
+                found = 1;
+                gf_msg_debug(this->name, 0,
+                             "subvol %s belongs to"
+                             " this node",
+                             prev->name);
+            }
         }
 
-        local->op_ret = 0;
- unlock:
-        UNLOCK (&frame->lock);
-
-        if (!is_last_call (this_call_cnt))
-                goto out;
-
-        if (local->op_ret == -1) {
-                goto unwind;
+        if (!found) {
+            local->op_ret = 0;
+            goto unlock;
         }
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, xattr, xdata);
-        goto out;
-
- unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, xdata);
- out:
-        return 0;
-}
+        conf->local_nodeuuids[index].count = count;
+        conf->local_nodeuuids[index].elements = GF_CALLOC(
+            count, sizeof(nodeuuid_info_t), 1);
 
-int
-dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int          ret           = 0;
-        dht_local_t *local         = NULL;
-        int          this_call_cnt = 0;
-        dict_t      *dict          = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
+        /* The node-uuids are guaranteed to be returned in the same
+         * order as the bricks
+         * A null node-uuid is returned for a brick that is down.
+         */
 
-        local = frame->local;
+        saveptr = NULL;
+        i = 0;
 
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-                if (op_ret < 0) {
-                        if (op_errno != ENOTCONN) {
-                                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                        DHT_MSG_GET_XATTR_FAILED,
-                                        "getxattr err for dir");
-                                local->op_ret = -1;
-                                local->op_errno = op_errno;
-                        }
+        for (uuid_str = strtok_r(uuid_list_copy, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            tmp_ptr = &(conf->local_nodeuuids[index].elements[i]);
+            gf_uuid_parse(uuid_str, tmp_ptr->uuid);
 
-                        goto unlock;
-                }
-
-                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
-                                                    op_errno);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "alloc or fill failure");
+            if (!gf_uuid_compare(tmp_ptr->uuid, conf->defrag->node_uuid)) {
+                tmp_ptr->info = REBAL_NODEUUID_MINE;
+            }
+            i++;
+            tmp_ptr = NULL;
         }
- unlock:
-        UNLOCK (&frame->lock);
+    }
 
-        if (!is_last_call (this_call_cnt))
-                goto out;
+    local->op_ret = 0;
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
 
-        /* -- last call: do patch ups -- */
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, xattr, xdata);
+    goto out;
 
-        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
-        if (ret)
-                goto unwind;
+unwind:
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
-        goto cleanup;
+    GF_FREE(conf->local_nodeuuids[index].elements);
+    conf->local_nodeuuids[index].elements = NULL;
 
- unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
- cleanup:
-        if (dict)
-                dict_unref (dict);
- out:
-        return 0;
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, xdata);
+out:
+    GF_FREE(uuid_list_copy);
+    return 0;
 }
 
-int
-dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+static int
+dht_vgetxattr_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        dht_local_t  *local         = NULL;
-        int           ret           = 0;
-        dict_t       *dict          = NULL;
-        call_frame_t *prev          = NULL;
-        gf_boolean_t  flag          = _gf_true;
+    int ret = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    dict_t *dict = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
+
+    local = frame->local;
 
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
         if (op_ret < 0) {
+            if (op_errno != ENOTCONN) {
                 local->op_ret = -1;
                 local->op_errno = op_errno;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "vgetxattr: Subvolume %s returned -1",
-                         prev->this->name);
-                goto unwind;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "getxattr err for dir");
+                goto post_unlock;
+            }
+
+            goto unlock;
         }
 
-        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
-                                            op_errno);
+        ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_NO_MEMORY,
-                        "Allocation or fill failure");
-                goto unwind;
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_DICT_SET_FAILED,
+                   "alloc or fill failure");
+            goto post_unlock;
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
 
-        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+    /* -- last call: do patch ups -- */
 
-        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
-        if (ret)
-                goto unwind;
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
-        goto cleanup;
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, _gf_true);
+    if (ret)
+        goto unwind;
 
- unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
-                          NULL, NULL);
- cleanup:
-        if (dict)
-                dict_unref (dict);
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
 
-        return 0;
+unwind:
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
+out:
+    return 0;
 }
 
-int
-dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
+static int
+dht_vgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int   ret   = 0;
-        char *value = NULL;
-
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
-                if (!ret) {
-                        ret = dict_set_str (xattr, GF_XATTR_LINKINFO_KEY, value);
-                        if (!ret)
-                                gf_msg_trace (this->name, 0,
-                                              "failed to set linkinfo");
-                }
-        }
+    dht_local_t *local = NULL;
+    int ret = 0;
+    dict_t *dict = NULL;
+    xlator_t *prev = NULL;
+    gf_boolean_t flag = _gf_true;
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    local = frame->local;
+    prev = cookie;
 
-        return 0;
-}
-
-int
-dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        dht_conf_t      *conf = NULL;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "vgetxattr: Subvolume %s returned -1", prev->name);
+        goto unwind;
+    }
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Allocation or fill failure");
+        goto unwind;
+    }
 
-        conf = this->private;
-        local = frame->local;
+    flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
 
-        LOCK (&frame->lock);
-        {
-                if (!xattr || (op_ret == -1)) {
-                        local->op_ret = op_ret;
-                        goto unlock;
-                }
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, flag);
+    if (ret)
+        goto unwind;
 
-                if (dict_get (xattr, conf->xattr_name)) {
-                        dict_del (xattr, conf->xattr_name);
-                }
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
 
-                if (frame->root->pid >= 0) {
-                        GF_REMOVE_INTERNAL_XATTR
-                                ("trusted.glusterfs.quota*", xattr);
-                        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
-                }
+unwind:
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
 
-                local->op_ret = 0;
+    return 0;
+}
 
-                if (!local->xattr) {
-                        local->xattr = dict_copy_with_ref (xattr, NULL);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+static int
+dht_linkinfo_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    int ret = 0;
+    char *value = NULL;
 
+    if (op_ret != -1) {
+        ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+        if (!ret) {
+            ret = dict_set_str(xattr, GF_XATTR_LINKINFO_KEY, value);
+            if (!ret)
+                gf_msg_trace(this->name, 0, "failed to set linkinfo");
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-out:
-        if (is_last_call (this_call_cnt)) {
+    }
 
-                /* If we have a valid xattr received from any one of the
-                 * subvolume, let's return it */
-                if (local->xattr) {
-                        local->op_ret = 0;
-                }
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                  local->xattr, NULL);
-        }
-        return 0;
+    return 0;
 }
 
-int32_t
-dht_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
-{
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
-
-
-int
-dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int op_ret, int op_errno,
-                                    dict_t *xattr, dict_t *xdata)
+static int
+dht_mds_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->op_errno == ENODATA ||
-                    local->op_errno == EOPNOTSUPP) {
-                        /* Nothing to do here, we have already found
-                         * a subvol which does not have the get_real_filename
-                         * optimization. If condition is for simple logic.
-                         */
-                        goto unlock;
-                }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
 
-                if (op_ret == -1) {
-
-                        if (op_errno == ENODATA || op_errno == EOPNOTSUPP) {
-                                /* This subvol does not have the optimization.
-                                 * Better let the user know we don't support it.
-                                 * Remove previous results if any.
-                                 */
-
-                                if (local->xattr) {
-                                        dict_unref (local->xattr);
-                                        local->xattr = NULL;
-                                }
-
-                                if (local->xattr_req) {
-                                        dict_unref (local->xattr_req);
-                                        local->xattr_req = NULL;
-                                }
-
-                                local->op_ret = op_ret;
-                                local->op_errno = op_errno;
-                                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                                        DHT_MSG_UPGRADE_BRICKS, "At least "
-                                        "one of the bricks does not support "
-                                        "this operation. Please upgrade all "
-                                        "bricks.");
-                                goto unlock;
-                        }
-
-                        if (op_errno == ENOENT) {
-                                /* Do nothing, our defaults are set to this.
-                                 */
-                                goto unlock;
-                        }
-
-                        /* This is a place holder for every other error
-                         * case. I am not sure of how to interpret
-                         * ENOTCONN etc. As of now, choosing to ignore
-                         * down subvol and return a good result(if any)
-                         * from other subvol.
-                         */
-                        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                                DHT_MSG_GET_XATTR_FAILED,
-                                "Failed to get real filename.");
-                        goto unlock;
-
-                }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
+    conf = this->private;
+    local = frame->local;
 
-                /* This subvol has the required file.
-                 * There could be other subvols which have returned
-                 * success already, choosing to return the latest good
-                 * result.
-                 */
-                if (local->xattr)
-                        dict_unref (local->xattr);
-                local->xattr = dict_ref (xattr);
-
-                if (local->xattr_req) {
-                        dict_unref (local->xattr_req);
-                        local->xattr_req = NULL;
-                }
-                if (xdata)
-                        local->xattr_req = dict_ref (xdata);
-
-                local->op_ret = op_ret;
-                local->op_errno = 0;
-                gf_msg_debug (this->name, 0, "Found a matching "
-                              "file.");
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    if (!xattr || (op_ret == -1)) {
+        local->op_ret = op_ret;
+        goto out;
+    }
+    dict_del(xattr, conf->xattr_name);
+    local->op_ret = 0;
 
+    if (!local->xattr) {
+        local->xattr = dict_copy_with_ref(xattr, NULL);
+    }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, local->xattr,
-                                  local->xattr_req);
-        }
-
-        return 0;
+out:
+    DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                     xdata);
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
-                                loc_t *loc, const char *key, dict_t *xdata)
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
-        int              i = 0;
-        dht_layout_t    *layout = NULL;
-        int              cnt = 0;
-        xlator_t        *subvol = NULL;
-
-
-        local = frame->local;
-        layout = local->layout;
-
-        cnt = local->call_cnt = layout->cnt;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
 
-        local->op_ret = -1;
-        local->op_errno = ENOENT;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
-                            subvol, subvol->fops->getxattr,
-                            loc, key, xdata);
-        }
+    conf = this->private;
+    local = frame->local;
 
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto err;
         return 0;
-}
-
-int
-dht_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                          xlator_t **subvols)
-{
-        dht_local_t *local = NULL;
-        int         i = 0;
-        dht_layout_t *layout = NULL;
-
-        local  = frame->local;
-        layout = local->layout;
+    }
 
-        for (i = 0; i < layout->cnt; i++)
-                subvols[i] = layout->list[i].xlator;
+    LOCK(&frame->lock);
+    {
+        if (!xattr || (op_ret == -1)) {
+            local->op_ret = op_ret;
+            goto unlock;
+        }
 
-        return layout->cnt;
-}
+        dict_del(xattr, conf->xattr_name);
+        dict_del(xattr, conf->mds_xattr_key);
 
-int
-dht_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *key, dict_t *xdata)
-#define DHT_IS_DIR(layout)  (layout->cnt > 1)
-{
-
-        xlator_t     *subvol        = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
-        char         *node_uuid_key = NULL;
-        int           ret           = -1;
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf   = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_GETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
+        dict_del(xattr, conf->commithash_xattr_name);
 
-                goto err;
+        if (frame->root->pid >= 0) {
+            GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+            GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
         }
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LAYOUT_NULL,
-                        "Layout is NULL");
-                op_errno = ENOENT;
-                goto err;
-        }
+        local->op_ret = 0;
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+        if (!local->xattr) {
+            local->xattr = dict_copy_with_ref(xattr, NULL);
+        } else {
+            dht_aggregate_xattr(local->xattr, xattr);
         }
 
-        if (key &&
-            (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
-                      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
-            && DHT_IS_DIR(layout)) {
-                dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
-                return 0;
+        if (!local->xdata) {
+            local->xdata = dict_ref(xdata);
+        } else if ((local->inode && IA_ISDIR(local->inode->ia_type)) ||
+                   (local->fd && IA_ISDIR(local->fd->inode->ia_type))) {
+            dht_aggregate_xattr(local->xdata, xdata);
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (key && DHT_IS_DIR(layout) &&
-           (!strcmp (key, GF_REBAL_FIND_LOCAL_SUBVOL))) {
-                ret = gf_asprintf
-                           (&node_uuid_key, "%s", GF_XATTR_NODE_UUID_KEY);
-                if (ret == -1 || !node_uuid_key) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_NO_MEMORY,
-                                "Failed to copy key");
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                (void) strncpy (local->xsel, node_uuid_key, 256);
-                cnt = local->call_cnt = conf->subvolume_cnt;
-                for (i = 0; i < cnt; i++) {
-                        STACK_WIND (frame, dht_find_local_subvol_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->getxattr,
-                                    loc, node_uuid_key, xdata);
-                }
-                if (node_uuid_key)
-                        GF_FREE (node_uuid_key);
-                return 0;
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* If we have a valid xattr received from any one of the
+         * subvolume, let's return it */
+        if (local->xattr) {
+            local->op_ret = 0;
         }
 
-        /* for file use cached subvolume (obviously!): see if {}
-         * below
-         * for directory:
-         *  wind to all subvolumes and exclude subvolumes which
-         *  return ENOTCONN (in callback)
-         *
-         * NOTE: Don't trust inode here, as that may not be valid
-         *       (until inode_link() happens)
-         */
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                         local->xdata);
+    }
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
 
-        if (key && DHT_IS_DIR(layout) &&
-            (XATTR_IS_PATHINFO (key)
-             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
-                (void) strncpy (local->xsel, key, 256);
-                cnt = local->call_cnt = layout->cnt;
-                for (i = 0; i < cnt; i++) {
-                        subvol = layout->list[i].xlator;
-                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
-                                    subvol, subvol->fops->getxattr,
-                                    loc, key, xdata);
-                }
-                return 0;
-        }
+static int32_t
+dht_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
 
-        /* node-uuid or pathinfo for files */
-        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
-                    || XATTR_IS_PATHINFO (key))) {
-                cached_subvol = local->cached_subvol;
-                (void) strncpy (local->xsel, key, 256);
+static int
+dht_getxattr_get_real_filename_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   dict_t *xattr, dict_t *xdata)
+{
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
 
-                local->call_cnt = 1;
-                STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
-                            cached_subvol->fops->getxattr, loc, key, xdata);
+    local = frame->local;
 
-                return 0;
+    LOCK(&frame->lock);
+    {
+        if (local->op_errno == EOPNOTSUPP) {
+            /* Nothing to do here, we have already found
+             * a subvol which does not have the get_real_filename
+             * optimization. If condition is for simple logic.
+             */
+            goto unlock;
         }
 
-        if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
-
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+        if (op_ret == -1) {
+            if (op_errno == EOPNOTSUPP) {
+                /* This subvol does not have the optimization.
+                 * Better let the user know we don't support it.
+                 * Remove previous results if any.
+                 */
 
-                cached_subvol = dht_subvol_get_cached (this, loc->inode);
-                if (!cached_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_CACHED_SUBVOL_GET_FAILED,
-                                "Failed to get cached subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
+                if (local->xattr) {
+                    dict_unref(local->xattr);
+                    local->xattr = NULL;
                 }
 
-                if (hashed_subvol == cached_subvol) {
-                        op_errno = ENODATA;
-                        goto err;
+                if (local->xattr_req) {
+                    dict_unref(local->xattr_req);
+                    local->xattr_req = NULL;
                 }
 
-                STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
-                            hashed_subvol->fops->getxattr, loc,
-                            GF_XATTR_PATHINFO_KEY, xdata);
-                return 0;
-        }
+                local->op_ret = op_ret;
+                local->op_errno = op_errno;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                       DHT_MSG_UPGRADE_BRICKS,
+                       "At least "
+                       "one of the bricks does not support "
+                       "this operation. Please upgrade all "
+                       "bricks.");
+                goto post_unlock;
+            }
 
-        if (key && (!strcmp (QUOTA_LIMIT_KEY, key) ||
-                    !strcmp (QUOTA_LIMIT_OBJECTS_KEY, key))) {
-                /* quota hardlimit and aggregated size of a directory is stored
-                 * in inode contexts of each brick. Hence its good enough that
-                 * we send getxattr for this key to any brick.
+            if (op_errno == ENOATTR) {
+                /* Do nothing, our defaults are set to this.
                  */
-                local->call_cnt = 1;
-                subvol = dht_first_up_subvol (this);
-                STACK_WIND (frame, dht_getxattr_cbk, subvol,
-                            subvol->fops->getxattr, loc, key, xdata);
-                return 0;
-        }
-
-        if (cluster_handle_marker_getxattr (frame, loc, key, conf->vol_uuid,
-                                            dht_getxattr_unwind,
-                                            dht_marker_populate_args) == 0)
-                return 0;
+                goto unlock;
+            }
 
-        if (DHT_IS_DIR(layout)) {
-                cnt = local->call_cnt = layout->cnt;
-        } else {
-                cnt = local->call_cnt  = 1;
-        }
+            /* This is a place holder for every other error
+             * case. I am not sure of how to interpret
+             * ENOTCONN etc. As of now, choosing to ignore
+             * down subvol and return a good result(if any)
+             * from other subvol.
+             */
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   DHT_MSG_GET_XATTR_FAILED, "Failed to get real filename.");
+            goto post_unlock;
+        }
+
+        /* This subvol has the required file.
+         * There could be other subvols which have returned
+         * success already, choosing to return the latest good
+         * result.
+         */
+        if (local->xattr)
+            dict_unref(local->xattr);
+        local->xattr = dict_ref(xattr);
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->getxattr,
-                            loc, key, xdata);
+        if (local->xattr_req) {
+            dict_unref(local->xattr_req);
+            local->xattr_req = NULL;
         }
-        return 0;
+        if (xdata)
+            local->xattr_req = dict_ref(xdata);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+        local->op_ret = op_ret;
+        local->op_errno = 0;
+        UNLOCK(&frame->lock);
+        gf_msg_debug(this->name, 0, "Found a matching file.");
+        goto post_unlock;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->xattr, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 }
-#undef DHT_IS_DIR
 
-int
-dht_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *key, dict_t *xdata)
-{
-        xlator_t     *subvol        = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FGETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
+static int
+dht_getxattr_get_real_filename(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               const char *key, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
+    int cnt = 0;
+    xlator_t *subvol = NULL;
 
-                goto err;
-        }
+    local = frame->local;
+    layout = local->layout;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LAYOUT_NULL,
-                        "Layout is NULL");
-                op_errno = ENOENT;
-                goto err;
-        }
+    cnt = local->call_cnt = layout->cnt;
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-        }
+    local->op_ret = -1;
+    local->op_errno = ENOATTR;
 
-        if ((fd->inode->ia_type == IA_IFDIR)
-            && key
-            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
-                         strlen (GF_XATTR_LOCKINFO_KEY)) != 0)) {
-                cnt = local->call_cnt = layout->cnt;
-        } else {
-                cnt = local->call_cnt  = 1;
-        }
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_get_real_filename_cbk, subvol,
+                   subvol->fops->getxattr, loc, key, xdata);
+    }
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->fgetxattr,
-                            fd, key, NULL);
-        }
-        return 0;
+    return 0;
+}
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+static int
+dht_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
 
-        return 0;
+    local = frame->local;
+    layout = local->layout;
+
+    for (i = 0; i < layout->cnt; i++)
+        subvols[i] = layout->list[i].xlator;
+
+    return layout->cnt;
 }
 
-int
-dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_is_debug_xattr_key(const char **array, char *key)
 {
-        int           ret     = -1;
-        dht_local_t  *local   = NULL;
-        call_frame_t *prev    = NULL;
-        struct iatt  *stbuf   = NULL;
-        inode_t      *inode   = NULL;
-        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
+    int i = 0;
 
-        local = frame->local;
-        prev = cookie;
+    for (i = 0; array[i]; i++) {
+        if (fnmatch(array[i], key, FNM_NOESCAPE) == 0)
+            return i;
+    }
 
-        local->op_errno = op_errno;
+    return -1;
+}
 
-        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1.",
-                              prev->this->name);
-                goto out;
-        }
+/* Note we already have frame->local initialised here*/
 
-        if (local->call_cnt != 1)
-                goto out;
+static int
+dht_handle_debug_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                          const char *key)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    char *value = NULL;
+    loc_t file_loc = {0};
+    const char *name = NULL;
 
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+    local = frame->local;
 
-        if ((!op_ret) && !stbuf) {
-                goto out;
-        }
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) == -1) {
+        goto out;
+    }
 
-        local->op_ret = op_ret;
-        local->rebalance.target_op_fn = dht_setxattr2;
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+    if (strncmp(key, DHT_DBG_HASHED_SUBVOL_KEY,
+                SLEN(DHT_DBG_HASHED_SUBVOL_KEY)) == 0) {
+        name = key + strlen(DHT_DBG_HASHED_SUBVOL_KEY);
+        if (strlen(name) == 0) {
+            op_errno = EINVAL;
+            goto out;
         }
 
-        /* Phase 1 of migration */
-        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
-
-                ret = dht_inode_ctx_get_mig_info (this, inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        dht_setxattr2 (this, subvol2, frame, 0);
-                        return 0;
-                }
+        ret = dht_build_child_loc(this, &file_loc, loc, (char *)name);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
+        }
 
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        local->hashed_subvol = dht_subvol_get_hashed(this, &file_loc);
+        if (local->hashed_subvol == NULL) {
+            op_errno = ENODATA;
+            goto out;
         }
 
-out:
+        value = gf_strdup(local->hashed_subvol->name);
+        if (!value) {
+            op_errno = ENOMEM;
+            goto out;
+        }
 
-        if (local->fop == GF_FOP_SETXATTR) {
-                DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
-        } else {
-                DHT_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        ret = dict_set_dynstr(local->xattr, (char *)key, value);
+        if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
+        ret = 0;
+        goto out;
+    }
 
-        return 0;
+out:
+    loc_wipe(&file_loc);
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
 }
 
+/* Virtual Xattr which returns 1 if all subvols are up,
+   else returns 0. Geo-rep then uses this virtual xattr
+   after a fresh mount and starts the I/O.
+*/
 
+enum dht_vxattr_subvol {
+    DHT_VXATTR_SUBVOLS_UP = 1,
+    DHT_VXATTR_SUBVOLS_DOWN = 0,
+};
 
 int
-dht_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
-{
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        int           op_errno = EINVAL;
-        dht_conf_t   *conf     = NULL;
-        dht_layout_t *layout   = NULL;
-        int           ret      = -1;
-        int           call_cnt = 0;
-        int           i        = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        if (!conf->defrag)
-                GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                           op_errno, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+dht_vgetxattr_subvol_status(call_frame_t *frame, xlator_t *this,
+                            const char *key)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    int value = DHT_VXATTR_SUBVOLS_UP;
+    int i = 0;
+    dht_conf_t *conf = NULL;
 
-        local->call_cnt = call_cnt = layout->cnt;
+    conf = this->private;
+    local = frame->local;
 
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_err_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->fsetxattr,
-                                    fd, xattr, flags, NULL);
-                }
+    if (!key) {
+        op_errno = EINVAL;
+        goto out;
+    }
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            value = DHT_VXATTR_SUBVOLS_DOWN;
+            gf_msg_debug(this->name, 0, "subvol %s is down ",
+                         conf->subvolumes[i]->name);
+            break;
+        }
+    }
+    ret = dict_set_int8(local->xattr, (char *)key, value);
+    if (ret < 0) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 
-        } else {
+out:
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
+}
 
-                local->call_cnt = 1;
-                local->rebalance.xattr = dict_ref (xattr);
-                local->rebalance.flags = flags;
+int
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+             dict_t *xdata)
+#define DHT_IS_DIR(layout) (layout->cnt > 1)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    char *node_uuid_key = NULL;
+    int ret = -1;
+
+    GF_CHECK_XATTR_KEY_AND_GOTO(key, IO_THREADS_QUEUE_SIZE_KEY, op_errno, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_GETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* skip over code which is irrelevant without a valid key */
+    if (!key)
+        goto no_key;
+
+    local->key = gf_strdup(key);
+    if (!local->key) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (strncmp(key, DHT_SUBVOL_STATUS_KEY, SLEN(DHT_SUBVOL_STATUS_KEY)) == 0) {
+        dht_vgetxattr_subvol_status(frame, this, key);
+        return 0;
+    }
+
+    /* skip over code which is irrelevant if !DHT_IS_DIR(layout) */
+    if (!DHT_IS_DIR(layout))
+        goto no_dht_is_dir;
+
+    if ((strncmp(key, GF_XATTR_GET_REAL_FILENAME_KEY,
+                 SLEN(GF_XATTR_GET_REAL_FILENAME_KEY)) == 0) &&
+        DHT_IS_DIR(layout)) {
+        dht_getxattr_get_real_filename(frame, this, loc, key, xdata);
+        return 0;
+    }
+
+    if (!strcmp(key, GF_REBAL_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_LIST_NODE_UUIDS_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
+        }
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
+        }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
+
+    if (!strcmp(key, GF_REBAL_OLD_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_NODE_UUID_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
+        }
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
+        }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
+
+    /* for file use cached subvolume (obviously!): see if {}
+     * below
+     * for directory:
+     *  wind to all subvolumes and exclude subvolumes which
+     *  return ENOTCONN (in callback)
+     *
+     * NOTE: Don't trust inode here, as that may not be valid
+     *       (until inode_link() happens)
+     */
+
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_LIST_NODE_UUIDS_KEY) == 0)) {
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        cnt = local->call_cnt = layout->cnt;
+        for (i = 0; i < cnt; i++) {
+            subvol = layout->list[i].xlator;
+            STACK_WIND(frame, dht_vgetxattr_dir_cbk, subvol,
+                       subvol->fops->getxattr, loc, key, xdata);
+        }
+        return 0;
+    }
 
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata)
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                        DHT_IATT_IN_XDATA_KEY, "yes");
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                      "Failed to set dictionary key %s for fd=%p",
-                                      DHT_IATT_IN_XDATA_KEY, fd);
-                }
+no_dht_is_dir:
+    /* node-uuid or pathinfo for files */
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0)) {
+        cached_subvol = local->cached_subvol;
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_vgetxattr_cbk, cached_subvol,
+                          cached_subvol, cached_subvol->fops->getxattr, loc,
+                          key, xdata);
 
-                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
-                    subvol->fops->fsetxattr, fd, xattr, flags, xdata);
+        return 0;
+    }
 
-                if (xdata)
-                        dict_unref (xdata);
+    if (strcmp(key, GF_XATTR_LINKINFO_KEY) == 0) {
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
 
+        cached_subvol = dht_subvol_get_cached(this, loc->inode);
+        if (!cached_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+                   "Failed to get cached subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
-        return 0;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+        if (hashed_subvol == cached_subvol) {
+            op_errno = ENODATA;
+            goto err;
+        }
 
+        STACK_WIND(frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
+                   hashed_subvol->fops->getxattr, loc, GF_XATTR_PATHINFO_KEY,
+                   xdata);
         return 0;
-}
+    }
 
-static int
-dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
-{
-        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) >= 0) {
+        dht_handle_debug_getxattr(frame, this, loc, key);
+        return 0;
+    }
 
+no_key:
+    if (cluster_handle_marker_getxattr(frame, loc, key, conf->vol_uuid,
+                                       dht_getxattr_unwind,
+                                       dht_marker_populate_args) == 0)
         return 0;
-}
 
+    if (DHT_IS_DIR(layout)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Cannot determine MDS, fetching xattr %s randomly"
+                   " from a subvol for path %s ",
+                   key, loc->path);
+        } else {
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of mds_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_INFO, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS %s is down for path"
+                               " path %s so fetching xattr "
+                               "%s randomly from a subvol ",
+                               local->mds_subvol->name, loc->path, key);
+                        ret = 1;
+                    }
+                }
+            }
+        }
 
-int
-dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
-{
-        int           i     = -1;
-        int           ret   = -1;
-        char         *value = NULL;
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        call_frame_t *prev  = NULL;
-        int           this_call_cnt = 0;
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->getxattr, loc, key, xdata);
 
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
+            return 0;
+        }
+    } else {
+        cnt = local->call_cnt = 1;
+    }
 
-        if (op_ret == -1)
-                goto out;
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->getxattr, loc,
+                   key, xdata);
+    }
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
 
-        ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
-        if (ret)
-                goto out;
+    return 0;
+}
+#undef DHT_IS_DIR
 
-        if (!strcmp (value, local->key)) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvolumes[i] == prev->this)
-                                conf->decommissioned_bricks[i] = prev->this;
+int
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+              dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    xlator_t *mds_subvol = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FGETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    if (key) {
+        local->key = gf_strdup(key);
+        if (!local->key) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    gf_uuid_unparse(fd->inode->gfid, gfid);
+
+    if ((fd->inode->ia_type == IA_IFDIR) && key &&
+        (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) !=
+         0)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "cannot determine MDS, fetching xattr %s "
+                   " randomly from a subvol for gfid %s ",
+                   key, gfid);
+        } else {
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of hashed_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS subvolume %s is down"
+                               " for gfid %s so fetching xattr "
+                               " %s randomly from a subvol ",
+                               local->mds_subvol->name, gfid, key);
+                        ret = 1;
+                    }
                 }
+            }
         }
 
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fgetxattr, fd, key, NULL);
+
+            return 0;
         }
-        return 0;
 
-}
+    } else {
+        cnt = local->call_cnt = 1;
+    }
 
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr, fd,
+                   key, NULL);
+    }
+    return 0;
 
-int
-dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
-{
-        dht_local_t  *local  = NULL;
-        int          op_errno = EINVAL;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        if (!frame || !frame->local)
-                goto err;
+    return 0;
+}
 
-        local = frame->local;
+static int
+dht_setxattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, local->rebalance.xdata);
-                return 0;
-        }
+    if (!frame || !frame->local)
+        goto err;
 
-        if (subvol == NULL)
-                goto err;
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        op_errno = local->op_errno;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto err;
 
-        if (local->fop == GF_FOP_SETXATTR) {
-                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
-                            subvol->fops->setxattr, &local->loc,
-                            local->rebalance.xattr, local->rebalance.flags,
-                            NULL);
-        } else {
-                STACK_WIND (frame, dht_file_setxattr_cbk, subvol,
-                            subvol->fops->fsetxattr, local->fd,
-                            local->rebalance.xattr, local->rebalance.flags,
-                            NULL);
-        }
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, &local->loc,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, local->fd,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (setxattr, frame, (local ? local->op_ret : -1),
-                          op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(setxattr, frame, (local ? local->op_ret : -1), op_errno,
+                     NULL);
+    return 0;
 }
 
 int
-dht_nuke_dir (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-        if (!IA_ISDIR(loc->inode->ia_type)) {
-                DHT_STACK_UNWIND (setxattr, frame, -1, ENOTSUP, NULL);
-                return 0;
-        }
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
-        /* Setxattr didn't need the parent, but rmdir does. */
-        loc->parent = inode_parent (loc->inode, NULL, NULL);
-        if (!loc->parent) {
-                DHT_STACK_UNWIND (setxattr, frame, -1, ENOENT, NULL);
-                return 0;
-        }
-        gf_uuid_copy (loc->pargfid, loc->parent->gfid);
+    local = frame->local;
+    prev = cookie;
 
-        if (!loc->name && loc->path) {
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name) {
-                        ++(loc->name);
-                }
-        }
-
-        /*
-         * We do this instead of calling dht_rmdir_do directly for two reasons.
-         * The first is that we want to reuse all of the initialization that
-         * dht_rmdir does, so if it ever changes we'll just follow along.  The
-         * second (i.e. why we don't use STACK_WIND_TAIL) is so that we don't
-         * obscure the fact that we came in via this path instead of a genuine
-         * rmdir.  That makes debugging just a tiny bit easier.
-         */
-        STACK_WIND (frame, default_rmdir_cbk, this, this->fops->rmdir,
-                    loc, 1, NULL);
+    local->op_errno = op_errno;
 
+    if ((local->fop == GF_FOP_FSETXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
         return 0;
-}
+    }
 
-int
-dht_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
-{
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        dht_conf_t   *conf     = NULL;
-        dht_methods_t *methods = NULL;
-        dht_layout_t *layout   = NULL;
-        int           i        = 0;
-        int           op_errno = EINVAL;
-        int           ret      = -1;
-        data_t       *tmp      = NULL;
-        uint32_t      dir_spread = 0;
-        char          value[4096] = {0,};
-        gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
-        int           call_cnt = 0;
-        uint32_t      new_hash = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        conf   = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, err);
-
-        methods = &(conf->methods);
-
-        /* Rebalance daemon is allowed to set internal keys */
-        if (!conf->defrag)
-                GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                           op_errno, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s",
-                              loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (local->call_cnt != 1)
+        goto out;
 
-        local->call_cnt = call_cnt = layout->cnt;
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-        tmp = dict_get (xattr, GF_XATTR_FILE_MIGRATE_KEY);
-        if (tmp) {
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
-                if (IA_ISDIR (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_setxattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-                /* TODO: need to interpret the 'value' for more meaning
-                   (ie, 'target' subvolume given there, etc) */
-                memcpy (value, tmp->data, tmp->len);
-                if (strcmp (value, "force") == 0)
-                       forced_rebalance =
-                                GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-                if (conf->decommission_in_progress)
-                        forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
 
-                if (!loc->path) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_setxattr2(this, subvol2, frame, 0);
+            return 0;
+        }
 
-                if (!local->loc.name)
-                        local->loc.name = strrchr (local->loc.path, '/')+1;
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-                if (!local->loc.parent)
-                        local->loc.parent =
-                                inode_parent(local->loc.inode, NULL, NULL);
+out:
 
-                if ((!local->loc.name) || (!local->loc.parent)) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    }
 
-                methods->migration_get_dst_subvol(this, local);
+    return 0;
+}
 
-                if (!local->rebalance.target_node) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+/* Function is call by dict_foreach_fnmatch if key is match with
+   user.* and set boolean flag to true
+*/
+static int
+dht_is_user_xattr(dict_t *this, char *key, data_t *value, void *data)
+{
+    gf_boolean_t *user_xattr_found = data;
+    *user_xattr_found = _gf_true;
+    return 0;
+}
 
-                local->rebalance.from_subvol = local->cached_subvol;
+/* Common code to wind a (f)(set|remove)xattr call to set xattr on directory
+ */
+static int
+dht_dir_common_set_remove_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                fd_t *fd, dict_t *xattr, int flags,
+                                dict_t *xdata, int *op_errno)
+
+{
+    dict_t *xattrop = NULL;
+    int32_t subone[1] = {-1};
+    gf_boolean_t uxattr_key_found = _gf_false;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *travvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int i = 0;
+    int call_cnt = 0;
+    dht_local_t *local = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char **xattrs_to_heal;
+
+    conf = this->private;
+    local = frame->local;
+    call_cnt = conf->subvolume_cnt;
+    local->flags = flags;
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
+
+    if ((local->fop == GF_FOP_SETXATTR) || (local->fop == GF_FOP_FSETXATTR)) {
+        /* Check if any user xattr present in xattr
+         */
+        dict_foreach_fnmatch(xattr, "user*", dht_is_user_xattr,
+                             &uxattr_key_found);
 
-                if (local->rebalance.target_node == local->rebalance.from_subvol) {
-                        op_errno = EEXIST;
-                        goto err;
-                }
-                if (local->rebalance.target_node) {
-                        local->flags = forced_rebalance;
-
-                        /* Flag to suggest its a tiering migration
-                         * The reason for this dic key-value is that
-                         * promotions and demotions are multithreaded
-                         * so the original frame from gf_defrag_start()
-                         * is not carried. A new frame will be created when
-                         * we do syncop_setxattr(). This doesnot have the
-                         * frame->root->pid of the original frame. So we pass
-                         * this dic key-value when we do syncop_setxattr() to do
-                         * data migration and set the frame->root->pid to
-                         * GF_CLIENT_PID_TIER_DEFRAG in dht_setxattr() just before
-                         * calling dht_start_rebalance_task() */
-                        tmp = dict_get (xattr, TIERING_MIGRATION_KEY);
-                        if (tmp)
-                                frame->root->pid = GF_CLIENT_PID_TIER_DEFRAG;
-                        else
-                                frame->root->pid = GF_CLIENT_PID_DEFRAG;
-
-                        ret = dht_start_rebalance_task (this, frame);
-                        if (!ret)
-                                return 0;
-
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_REBALANCE_START_FAILED,
-                                "%s: failed to create a new rebalance synctask",
-                                loc->path);
+        /* Check if any custom key xattr present in dict xattr
+           and start index from 1 because user xattr already
+           checked in previous line
+        */
+        for (i = 1; xattrs_to_heal[i]; i++)
+            if (dict_get(xattr, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
+
+    if ((local->fop == GF_FOP_REMOVEXATTR) ||
+        (local->fop == GF_FOP_FREMOVEXATTR)) {
+        /* Check if any custom key xattr present in local->key
+         */
+        for (i = 0; xattrs_to_heal[i]; i++)
+            if (strstr(local->key, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
+
+    /* If there is no custom key xattr present or gfid is root
+       or call_cnt is 1 then wind a (f)setxattr call on all subvols
+    */
+    if (!uxattr_key_found || __is_root_gfid(local->gfid) || call_cnt == 1) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            travvol = conf->subvolumes[i];
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_FSETXATTR)) {
+                if (fd) {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fsetxattr, fd, xattr,
+                                      flags, xdata);
+                } else {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->setxattr, loc, xattr,
+                                      flags, xdata);
                 }
-                op_errno = EINVAL;
-                goto err;
-
-        }
+            }
 
-        tmp = dict_get (xattr, "decommission-brick");
-        if (tmp) {
-                /* This operation should happen only on '/' */
-                if (!__is_root_gfid (loc->inode->gfid)) {
-                        op_errno = ENOTSUP;
-                        goto err;
+            if ((local->fop == GF_FOP_REMOVEXATTR) ||
+                (local->fop == GF_FOP_FREMOVEXATTR)) {
+                if (fd) {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fremovexattr, fd,
+                                      local->key, local->xattr_req);
+                } else {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->removexattr, loc,
+                                      local->key, local->xattr_req);
                 }
+            }
+        }
 
-                memcpy (value, tmp->data, ((tmp->len < 4095) ? tmp->len : 4095));
-                local->key = gf_strdup (value);
-                local->call_cnt = conf->subvolume_cnt;
+        return 0;
+    }
 
-                for (i = 0 ; i < conf->subvolume_cnt; i++) {
-                        /* Get the pathinfo, and then compare */
-                        STACK_WIND (frame, dht_checking_pathinfo_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->getxattr,
-                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
-                }
-                return 0;
+    /* Calculate hash subvol based on inode and parent inode
+     */
+    if (fd) {
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+    } else {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+    }
+    if (ret || !mds_subvol) {
+        if (fd) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for fd %p"
+                   "gfid is %s ",
+                   fd, gfid_local);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "%s: Failed to get mds subvol. (gfid is %s)", loc->path,
+                   gfid_local);
+        }
+        (*op_errno) = ENOENT;
+        goto err;
+    }
+
+    local->mds_subvol = mds_subvol;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == mds_subvol) {
+            if (!conf->subvolume_status[i]) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_HASHED_SUBVOL_DOWN,
+                       "MDS subvol is down for path "
+                       " %s gfid is %s Unable to set xattr ",
+                       local->loc.path, gfid_local);
+                (*op_errno) = ENOTCONN;
+                goto err;
+            }
         }
+    }
+
+    if (uxattr_key_found) {
+        xattrop = dict_new();
+        if (!xattrop) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                   "dictionary creation failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            (*op_errno) = ENOMEM;
+            goto err;
+        }
+        local->xattr = dict_ref(xattr);
+        /* Subtract current MDS xattr value to -1 , value of MDS
+           xattr represents no. of times xattr modification failed
+           on non MDS subvols.
+        */
+        ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, subone, 1);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "dictionary set array failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (xattrop)
+                dict_unref(xattrop);
+            (*op_errno) = ret;
+            goto err;
+        }
+        /* Wind a xattrop call to use ref counting approach
+           update mds xattr to -1 before update xattr on
+           hashed subvol and update mds xattr to +1 after update
+           xattr on all non hashed subvol
+        */
+        if (fd) {
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fxattrop, fd,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+        } else {
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->xattrop, loc,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+        }
+        if (xattrop)
+            dict_unref(xattrop);
+    }
 
-        tmp = dict_get (xattr, GF_XATTR_FIX_LAYOUT_KEY);
-        if (tmp) {
-                ret = dict_get_uint32(xattr, "new-commit-hash", &new_hash);
-                if (ret == 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "updating commit hash for %s from %u to %u",
-                                      uuid_utoa(loc->gfid),
-                                      layout->commit_hash, new_hash);
-                        layout->commit_hash = new_hash;
-
-                        ret = dht_update_commit_hash_for_layout (frame);
-                        if (ret) {
-                                op_errno = ENOTCONN;
-                                goto err;
-                        }
-                        return ret;
-                }
-
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_FIX_LAYOUT_INFO,
-                        "fixing the layout of %s", loc->path);
+    return 0;
+err:
+    return -1;
+}
 
-                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
-                                                layout);
-                if (ret) {
-                        op_errno = ENOTCONN;
-                        goto err;
-                }
-                return ret;
-        }
+int
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+              int flags, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, xattr,
+                                              flags, xdata, &op_errno);
+        if (ret)
+            goto err;
+    } else {
+        local->call_cnt = 1;
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
 
-        tmp = dict_get (xattr, "distribute.directory-spread-count");
-        if (tmp) {
-                /* Setxattr value is packed as 'binary', not string */
-                memcpy (value, tmp->data, ((tmp->len < 4095)?tmp->len:4095));
-                ret = gf_string2uint32 (value, &dir_spread);
-                if (!ret && ((dir_spread <= conf->subvolume_cnt) &&
-                             (dir_spread > 0))) {
-                        layout->spread_cnt = dir_spread;
-
-                        ret = dht_fix_directory_layout (frame,
-                                                        dht_common_setxattr_cbk,
-                                                        layout);
-                        if (ret) {
-                                op_errno = ENOTCONN;
-                                goto err;
-                        }
-                        return ret;
-                }
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_OPERATION_NOT_SUP,
-                        "wrong 'directory-spread-count' value (%s)", value);
-                op_errno = ENOTSUP;
-                goto err;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s for fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
         }
 
-        tmp = dict_get (xattr, "glusterfs.dht.nuke");
-        if (tmp) {
-                return dht_nuke_dir (frame, this, loc, tmp);
-        }
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, fd, xattr, flags,
+                          local->xattr_req);
+    }
+    return 0;
 
-        if (IA_ISDIR (loc->inode->ia_type)) {
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
 
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_err_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setxattr,
-                                    loc, xattr, flags, xdata);
-                }
+    return 0;
+}
 
-        } else {
+static int
+dht_checking_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    int i = -1;
+    int ret = -1;
+    char *value = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret == -1)
+        goto out;
 
-                local->rebalance.xattr = dict_ref (xattr);
-                local->rebalance.flags = flags;
-                local->call_cnt = 1;
+    ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+    if (ret)
+        goto out;
 
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata)
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                              DHT_IATT_IN_XDATA_KEY, "yes");
+    if (!strcmp(value, local->key)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == prev)
+                conf->decommissioned_bricks[i] = prev;
+        }
+    }
 
-                STACK_WIND (frame, dht_file_setxattr_cbk,
-                            subvol, subvol->fops->setxattr,
-                            loc, xattr, flags, xdata);
+out:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, ENOTSUP, NULL);
+    }
+    return 0;
+}
 
-                if (xdata)
-                        dict_unref (xdata);
-        }
+static int
+dht_nuke_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
+}
 
+static int
+dht_nuke_dir(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
+{
+    if (!IA_ISDIR(loc->inode->ia_type)) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOTSUP, NULL);
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-
+    /* Setxattr didn't need the parent, but rmdir does. */
+    loc->parent = inode_parent(loc->inode, NULL, NULL);
+    if (!loc->parent) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOENT, NULL);
         return 0;
-}
+    }
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
 
+    if (!loc->name && loc->path) {
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name) {
+            ++(loc->name);
+        }
+    }
 
+    /*
+     * We do this instead of calling dht_rmdir_do directly for two reasons.
+     * The first is that we want to reuse all of the initialization that
+     * dht_rmdir does, so if it ever changes we'll just follow along.  The
+     * second (i.e. why we don't use STACK_WIND_TAIL) is so that we don't
+     * obscure the fact that we came in via this path instead of a genuine
+     * rmdir.  That makes debugging just a tiny bit easier.
+     */
+    STACK_WIND(frame, dht_nuke_dir_cbk, this, this->fops->rmdir, loc, 1, NULL);
 
+    return 0;
+}
 
 int
-dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xdata)
-{
-        int           ret     = -1;
-        dht_local_t  *local   = NULL;
-        call_frame_t *prev    = NULL;
-        struct iatt  *stbuf   = NULL;
-        inode_t      *inode   = NULL;
-        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+             int flags, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int op_errno = EINVAL;
+    int ret = -1;
+    data_t *tmp = NULL;
+    uint32_t dir_spread = 0;
+    char value[4096] = {
+        0,
+    };
+    gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
+    int call_cnt = 0;
+    uint32_t new_hash = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
+
+    methods = &(conf->methods);
+
+    /* Rebalance daemon is allowed to set internal keys */
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+    tmp = dict_get(xattr, conf->mds_xattr_key);
+    if (tmp) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        if (IA_ISDIR(loc->inode->ia_type)) {
+            op_errno = ENOTSUP;
+            goto err;
+        }
+
+        /* TODO: need to interpret the 'value' for more meaning
+           (ie, 'target' subvolume given there, etc) */
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            forced_rebalance = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
+
+        if (!loc->path) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (!local->loc.name)
+            local->loc.name = strrchr(local->loc.path, '/') + 1;
+
+        if (!local->loc.parent)
+            local->loc.parent = inode_parent(local->loc.inode, NULL, NULL);
+
+        if ((!local->loc.name) || (!local->loc.parent)) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (gf_uuid_is_null(local->loc.pargfid))
+            gf_uuid_copy(local->loc.pargfid, local->loc.parent->gfid);
+
+        methods->migration_get_dst_subvol(this, local);
+
+        if (!local->rebalance.target_node) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        local->rebalance.from_subvol = local->cached_subvol;
+
+        if (local->rebalance.target_node == local->rebalance.from_subvol) {
+            op_errno = EEXIST;
+            goto err;
+        }
+        if (local->rebalance.target_node) {
+            local->flags = forced_rebalance;
 
-        local = frame->local;
-        prev = cookie;
+            frame->root->pid = GF_CLIENT_PID_DEFRAG;
 
-        local->op_errno = op_errno;
+            ret = dht_start_rebalance_task(this, frame);
+            if (!ret)
+                return 0;
 
-        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+                   "%s: failed to create a new rebalance synctask", loc->path);
         }
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        if (local->call_cnt != 1)
-                goto out;
+    tmp = dict_get(xattr, "decommission-brick");
+    if (tmp) {
+        /* This operation should happen only on '/' */
+        if (!__is_root_gfid(loc->inode->gfid)) {
+            op_errno = ENOTSUP;
+            goto err;
+        }
 
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        local->key = gf_strdup(value);
+        local->call_cnt = conf->subvolume_cnt;
 
-        if ((!op_ret) && !stbuf) {
-                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            /* Get the pathinfo, and then compare */
+            STACK_WIND_COOKIE(frame, dht_checking_pathinfo_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              GF_XATTR_PATHINFO_KEY, NULL);
         }
+        return 0;
+    }
 
-        local->op_ret = 0;
+    tmp = dict_get(xattr, GF_XATTR_FIX_LAYOUT_KEY);
+    if (tmp) {
+        ret = dict_get_uint32(xattr, "new-commit-hash", &new_hash);
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0,
+                         "updating commit hash for %s from %u to %u",
+                         uuid_utoa(loc->gfid), layout->commit_hash, new_hash);
+            layout->commit_hash = new_hash;
+
+            ret = dht_update_commit_hash_for_layout(frame);
+            if (ret) {
+                op_errno = ENOTCONN;
+                goto err;
+            }
+            return ret;
+        }
 
-        local->rebalance.target_op_fn = dht_removexattr2;
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_FIX_LAYOUT_INFO,
+               "fixing the layout of %s", loc->path);
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dht_fix_directory_layout(frame, dht_fix_layout_setxattr_cbk,
+                                       layout);
+        if (ret) {
+            op_errno = ENOTCONN;
+            goto err;
         }
+        return ret;
+    }
+
+    tmp = dict_get(xattr, "distribute.directory-spread-count");
+    if (tmp) {
+        /* Setxattr value is packed as 'binary', not string */
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        ret = gf_string2uint32(value, &dir_spread);
+        if (!ret && ((dir_spread <= conf->subvolume_cnt) && (dir_spread > 0))) {
+            layout->spread_cnt = dir_spread;
+
+            ret = dht_fix_directory_layout(frame, dht_common_setxattr_cbk,
+                                           layout);
+            if (ret) {
+                op_errno = ENOTCONN;
+                goto err;
+            }
+            return ret;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_OPERATION_NOT_SUP,
+               "wrong 'directory-spread-count' value (%s)", value);
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, "glusterfs.dht.nuke");
+    if (tmp) {
+        return dht_nuke_dir(frame, this, loc, tmp);
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, xattr,
+                                              flags, xdata, &op_errno);
+        if (ret)
+            goto err;
+    } else {
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
+        local->call_cnt = 1;
 
-        /* Phase 1 of migration */
-        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
 
-                ret = dht_inode_ctx_get_mig_info (this, inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        dht_removexattr2 (this, subvol2, frame, 0);
-                        return 0;
-                }
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, loc, xattr, flags,
+                          local->xattr_req);
+    }
 
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    return 0;
 
-out:
-        if (local->fop == GF_FOP_REMOVEXATTR) {
-                DHT_STACK_UNWIND (removexattr, frame, op_ret, op_errno, NULL);
-        } else {
-                DHT_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
-        }
-        return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
+    return 0;
 }
 
-int
-dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-                  int ret)
+static int
+dht_removexattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        if (!frame || !frame->local || !subvol)
-                goto err;
+    if (!frame || !frame->local)
+        goto err;
 
-        local = frame->local;
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        if (we_are_not_migrating (ret)) {
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
-                                  local->op_errno, local->rebalance.xdata);
-                return 0;
-        }
+    if (subvol == NULL)
+        goto err;
 
-        if (local->fop == GF_FOP_REMOVEXATTR) {
-                STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
-                            subvol->fops->removexattr, &local->loc,
-                            local->key, NULL);
-        } else {
-                STACK_WIND (frame, dht_file_removexattr_cbk, subvol,
-                            subvol->fops->fremovexattr, local->fd,
-                            local->key, NULL);
-        }
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, &local->loc, local->key,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, local->fd, local->key,
+                          local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xdata)
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->this->name);
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
+    local = frame->local;
+    prev = cookie;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+    local->op_errno = op_errno;
 
+    if ((local->fop == GF_FOP_FREMOVEXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
         return 0;
-}
+    }
 
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-int
-dht_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *key, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
-        dht_conf_t   *conf = NULL;
-        int i;
-        int           ret = 0;
+    if (local->call_cnt != 1)
+        goto out;
 
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-        conf = this->private;
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
-        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
+    local->op_ret = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
+    local->rebalance.target_op_fn = dht_removexattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_removexattr2(this, subvol2, frame, 0);
+            return 0;
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
-
-        if (IA_ISDIR (loc->inode->ia_type)) {
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_removexattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->removexattr,
-                                    loc, key, NULL);
-                }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        } else {
-
-                local->call_cnt = 1;
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata)
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                 DHT_IATT_IN_XDATA_KEY, "yes");
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED, "Failed to "
-                                "set dictionary key %s for %s",
-                                DHT_IATT_IN_XDATA_KEY, loc->path);
-                }
+out:
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    }
+    return 0;
+}
 
-                STACK_WIND (frame, dht_file_removexattr_cbk,
-                            subvol, subvol->fops->removexattr,
-                            loc, key, xdata);
+int
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_REMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (key && (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0)) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-                if (xdata)
-                        dict_unref (xdata);
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for %s",
+                   DHT_IATT_IN_XDATA_KEY, loc->path);
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, loc, key,
+                          local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *key, dict_t *xdata)
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
-        dht_conf_t   *conf = 0;
-        int           ret = 0;
-
-        int i;
-
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
-
-        VALIDATE_OR_GOTO (frame, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FREMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for inode=%s",
-                              uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = 0;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FREMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for inode=%s",
-                              uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for fd=%p",
+                   DHT_IATT_IN_XDATA_KEY, fd);
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
-
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_removexattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->fremovexattr,
-                                    fd, key, NULL);
-                }
-
-        } else {
-
-                local->call_cnt = 1;
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata)
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                 DHT_IATT_IN_XDATA_KEY, "yes");
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED, "Failed to "
-                                "set dictionary key %s for fd=%p",
-                                DHT_IATT_IN_XDATA_KEY, fd);
-                }
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, fd, key,
+                          local->xattr_req);
+    }
 
-                STACK_WIND (frame, dht_file_removexattr_cbk,
-                            subvol, subvol->fops->fremovexattr,
-                            fd, key, xdata);
-
-                if (xdata)
-                        dict_unref (xdata);
-        }
-
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+dht_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->this->name);
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                                  local->fd, NULL);
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, local->fd,
+                         NULL);
 
-        return 0;
+    return 0;
 }
 
 /*
  * dht_normalize_stats -
  */
 static void
-dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
-                     unsigned long frsize)
+dht_normalize_stats(struct statvfs *buf, unsigned long bsize,
+                    unsigned long frsize)
 {
-        double factor = 0;
+    double factor = 0;
 
-        if (buf->f_bsize != bsize) {
-                buf->f_bsize = bsize;
-        }
+    if (buf->f_bsize != bsize) {
+        buf->f_bsize = bsize;
+    }
 
-        if (buf->f_frsize != frsize) {
-                factor = ((double) buf->f_frsize) / frsize;
-                buf->f_frsize = frsize;
-                buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-                buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-                buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
+    if (buf->f_frsize != frsize) {
+        factor = ((double)buf->f_frsize) / frsize;
+        buf->f_frsize = frsize;
+        buf->f_blocks = (fsblkcnt_t)(factor * buf->f_blocks);
+        buf->f_bfree = (fsblkcnt_t)(factor * buf->f_bfree);
+        buf->f_bavail = (fsblkcnt_t)(factor * buf->f_bavail);
+    }
+}
 
+static int
+dht_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
+{
+    gf_boolean_t event = _gf_false;
+    qdstatfs_action_t action = qdstatfs_action_OFF;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int bsize = 0;
+    int frsize = 0;
+    GF_UNUSED int ret = 0;
+    unsigned long new_usage = 0;
+    unsigned long cur_usage = 0;
+
+    local = frame->local;
+    GF_ASSERT(local);
+
+    if (xdata)
+        ret = dict_get_int8(xdata, "quota-deem-statfs", (int8_t *)&event);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
-}
+        if (!statvfs) {
+            op_errno = EINVAL;
+            local->op_ret = -1;
+            goto unlock;
+        }
+        local->op_ret = 0;
 
-int
-dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs,
-                dict_t *xdata)
-{
+        if (local->quota_deem_statfs) {
+            if (event == _gf_true) {
+                action = qdstatfs_action_COMPARE;
+            } else {
+                action = qdstatfs_action_NEGLECT;
+            }
+        } else {
+            if (event == _gf_true) {
+                action = qdstatfs_action_REPLACE;
+                local->quota_deem_statfs = _gf_true;
+            }
+        }
 
-        gf_boolean_t            event              = _gf_false;
-        qdstatfs_action_t       action             = qdstatfs_action_OFF;
-        dht_local_t *           local              = NULL;
-        int                     this_call_cnt      = 0;
-        int                     bsize              = 0;
-        int                     frsize             = 0;
-        GF_UNUSED int           ret                = 0;
-        unsigned long           new_usage          = 0;
-        unsigned long           cur_usage          = 0;
+        if (local->quota_deem_statfs) {
+            switch (action) {
+                case qdstatfs_action_NEGLECT:
+                    goto unlock;
 
-        local = frame->local;
-        GF_ASSERT (local);
+                case qdstatfs_action_REPLACE:
+                    local->statvfs = *statvfs;
+                    goto unlock;
 
-        if (xdata)
-                ret = dict_get_int8 (xdata, "quota-deem-statfs",
-                                     (int8_t *)&event);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-                if (!statvfs) {
-                        op_errno = EINVAL;
-                        local->op_ret = -1;
-                        goto unlock;
-                }
-                local->op_ret = 0;
+                case qdstatfs_action_COMPARE:
+                    new_usage = statvfs->f_blocks - statvfs->f_bfree;
+                    cur_usage = local->statvfs.f_blocks -
+                                local->statvfs.f_bfree;
 
-                switch (local->quota_deem_statfs) {
-                case _gf_true:
-                        if (event == _gf_true)
-                                action = qdstatfs_action_COMPARE;
-                        else
-                                action = qdstatfs_action_NEGLECT;
-                        break;
-
-                case _gf_false:
-                        if (event == _gf_true) {
-                                action = qdstatfs_action_REPLACE;
-                                local->quota_deem_statfs = _gf_true;
-                        }
-                        break;
+                    /* Take the max of the usage from subvols */
+                    if (new_usage >= cur_usage)
+                        local->statvfs = *statvfs;
+                    goto unlock;
 
                 default:
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_VALUE,
-                                "Encountered third "
-                                "value for boolean variable %d",
-                                local->quota_deem_statfs);
-                        break;
-                }
-
-                if (local->quota_deem_statfs) {
-                        switch (action) {
-                        case qdstatfs_action_NEGLECT:
-                                goto unlock;
+                    break;
+            }
+        }
 
-                        case qdstatfs_action_REPLACE:
-                                local->statvfs = *statvfs;
-                                goto unlock;
+        if (local->statvfs.f_bsize != 0) {
+            bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+            frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+            dht_normalize_stats(&local->statvfs, bsize, frsize);
+            dht_normalize_stats(statvfs, bsize, frsize);
+        } else {
+            local->statvfs.f_bsize = statvfs->f_bsize;
+            local->statvfs.f_frsize = statvfs->f_frsize;
+        }
+
+        local->statvfs.f_blocks += statvfs->f_blocks;
+        local->statvfs.f_bfree += statvfs->f_bfree;
+        local->statvfs.f_bavail += statvfs->f_bavail;
+        local->statvfs.f_files += statvfs->f_files;
+        local->statvfs.f_ffree += statvfs->f_ffree;
+        local->statvfs.f_favail += statvfs->f_favail;
+        local->statvfs.f_fsid = statvfs->f_fsid;
+        local->statvfs.f_flag = statvfs->f_flag;
+        local->statvfs.f_namemax = statvfs->f_namemax;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-                        case qdstatfs_action_COMPARE:
-                                new_usage = statvfs->f_blocks -
-                                             statvfs->f_bfree;
-                                cur_usage = local->statvfs.f_blocks -
-                                             local->statvfs.f_bfree;
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->statvfs, xdata);
 
-                                /* Take the max of the usage from subvols */
-                                if (new_usage >= cur_usage)
-                                        local->statvfs = *statvfs;
-                                goto unlock;
+    return 0;
+}
 
-                        default:
-                                break;
-                        }
-                }
+int
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    loc_t newloc = {
+        0,
+    };
 
-                if (local->statvfs.f_bsize != 0) {
-                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
-                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
-                        dht_normalize_stats(&local->statvfs, bsize, frsize);
-                        dht_normalize_stats(statvfs, bsize, frsize);
-                } else {
-                        local->statvfs.f_bsize    = statvfs->f_bsize;
-                        local->statvfs.f_frsize   = statvfs->f_frsize;
-                }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-                local->statvfs.f_blocks  += statvfs->f_blocks;
-                local->statvfs.f_bfree   += statvfs->f_bfree;
-                local->statvfs.f_bavail  += statvfs->f_bavail;
-                local->statvfs.f_files   += statvfs->f_files;
-                local->statvfs.f_ffree   += statvfs->f_ffree;
-                local->statvfs.f_favail  += statvfs->f_favail;
-                local->statvfs.f_fsid     = statvfs->f_fsid;
-                local->statvfs.f_flag     = statvfs->f_flag;
-                local->statvfs.f_namemax  = statvfs->f_namemax;
+    conf = this->private;
 
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_STATFS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
+    if (loc->inode && !IA_ISDIR(loc->inode->ia_type)) {
+        itable = loc->inode->table;
+        if (!itable) {
+            op_errno = EINVAL;
+            goto err;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
+        loc = &local->loc2;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs, xdata);
-
-        return 0;
-}
-
-
-int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+        inode = inode_find(itable, root_gfid);
+        if (!inode) {
+            op_errno = EINVAL;
+            goto err;
+        }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (this->private, err);
+        dht_build_root_loc(inode, &newloc);
+        loc = &newloc;
+    }
 
-        conf = this->private;
+    local->call_cnt = conf->subvolume_cnt;
 
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_statfs_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->statfs, loc, xdata);
+    }
+    return 0;
 
-        if (!loc->inode || IA_ISDIR (loc->inode->ia_type)) {
-                local->call_cnt = conf->subvolume_cnt;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        STACK_WIND (frame, dht_statfs_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs, loc,
-                                    xdata);
-                }
-                return 0;
-        }
+    return 0;
+}
 
-        subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+int
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = 0;
+    gf_boolean_t new_xdata = _gf_false;
+    xlator_t **subvolumes = NULL;
+    int call_count = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPENDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    local->first_up_subvol = dht_first_up_subvol(this);
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        new_xdata = _gf_true;
+    }
+
+    ret = dict_set_uint32(xdata, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value : key = %s",
+               conf->link_xattr_name);
+
+    /* dht_readdirp will wind to all subvols so open has to be sent to
+     * all subvols whether or not conf->local_subvols is set */
+
+    call_count = local->call_cnt = conf->subvolume_cnt;
+    subvolumes = conf->subvolumes;
+
+    /* In case of parallel-readdir, the readdir-ahead will be loaded
+     * below dht, in this case, if we want to enable or disable SKIP_DIRs
+     * it has to be done in opendir, so that prefetching logic in
+     * readdir-ahead, honors it */
+    for (i = 0; i < call_count; i++) {
+        if (conf->readdir_optimize == _gf_true) {
+            if (subvolumes[i] != local->first_up_subvol) {
+                ret = dict_set_int32(xdata, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary"
+                           " value :key = %s, ret:%d",
+                           GF_READDIR_SKIP_DIRS, ret);
+            }
         }
 
-        local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_fd_cbk, subvolumes[i], subvolumes[i],
+                          subvolumes[i]->fops->opendir, loc, fd, xdata);
+        dict_del(xdata, GF_READDIR_SKIP_DIRS);
+    }
 
-        STACK_WIND (frame, dht_statfs_cbk,
-                    subvol, subvol->fops->statfs, loc, xdata);
+    if (new_xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+/* dht_readdirp_cbk creates a new dentry and dentry->inode is not assigned.
+   This functions assigns an inode if all of the following conditions are
+   true:
 
-int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-             dict_t *xdata)
+   * DHT has only one child. In this case the entire layout is present on
+   this single child and hence we can set complete layout in inode.
+   * backend has complete layout and there are no anomalies in it and from
+   this information layout can be constructed and set in inode.
+*/
+
+static void
+dht_populate_inode_for_dentry(xlator_t *this, xlator_t *subvol,
+                              gf_dirent_t *entry, gf_dirent_t *orig_entry)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    if (gf_uuid_is_null(orig_entry->d_stat.ia_gfid)) {
+        /* this skips the '..' entry for the root of the volume */
+        return;
+    }
 
-        conf = this->private;
+    gf_uuid_copy(loc.gfid, orig_entry->d_stat.ia_gfid);
+    loc.inode = inode_ref(orig_entry->inode);
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPENDIR);
-        if (!local) {
-                op_errno = ENOMEM;
+    if (is_revalidate(&loc)) {
+        goto out;
+    }
 
-                goto err;
-        }
+    layout = dht_layout_new(this, 1);
+    if (!layout)
+        goto out;
 
-        if ((conf->defrag && conf->defrag->cmd == GF_DEFRAG_CMD_START_TIER) ||
-            (conf->defrag && conf->defrag->cmd ==
-             GF_DEFRAG_CMD_START_DETACH_TIER) ||
-            (!(conf->local_subvols_cnt) || !conf->defrag)) {
-                local->call_cnt = conf->subvolume_cnt;
+    ret = dht_layout_merge(this, layout, subvol, 0, 0, orig_entry->dict);
+    if (!ret) {
+        ret = dht_layout_normalize(this, &loc, layout);
+        if (ret == 0) {
+            dht_layout_set(this, orig_entry->inode, layout);
+            entry->inode = inode_ref(orig_entry->inode);
+            layout = NULL;
+        }
+    }
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        STACK_WIND (frame, dht_fd_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->opendir,
-                                    loc, fd, xdata);
+    if (layout)
+        dht_layout_unref(this, layout);
 
-                }
-        } else {
-                local->call_cnt = conf->local_subvols_cnt;
-                for (i = 0; i < conf->local_subvols_cnt; i++) {
-                        STACK_WIND (frame, dht_fd_cbk,
-                                    conf->local_subvols[i],
-                                    conf->local_subvols[i]->fops->opendir,
-                                    loc, fd, xdata);
-                }
-        }
+out:
+    loc_wipe(&loc);
+    return;
+}
 
-        return 0;
+/* Posix returns op_errno = ENOENT to indicate that there are no more
+ * entries
+ */
+static int
+dht_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    xlator_t *subvol = 0;
+    xlator_t *hashed_subvol = 0;
+    int ret = 0;
+    int readdir_optimize = 0;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    prev = cookie;
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(this->name, local->fd, unwind);
+
+    itable = local->fd->inode->table;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
+
+    methods = &(conf->methods);
+
+    if (op_ret <= 0) {
+        goto done;
+    }
+
+    /* Why aren't we skipping DHT entirely in case of a single subvol?
+     * Because if this was a larger volume earlier and all but one subvol
+     * was removed, there might be stale linkto files on the subvol.
+     */
+    if (conf->subvolume_cnt == 1) {
+        /* return all directory and file entries except
+         * linkto files for a single child DHT
+         */
+        skip_hashed_check = _gf_true;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-        return 0;
-}
+    layout = local->layout;
 
+    /* This will skip the entries on the subvol without a layout,
+     * hence preventing the crash but rmdir might fail with
+     * "directory not empty" errors*/
 
-int
-dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        dht_local_t             *local = NULL;
-        gf_dirent_t              entries;
-        gf_dirent_t             *orig_entry = NULL;
-        gf_dirent_t             *entry = NULL;
-        call_frame_t            *prev = NULL;
-        xlator_t                *next_subvol = NULL;
-        off_t                    next_offset = 0;
-        int                      count = 0;
-        dht_layout_t            *layout = 0;
-        dht_conf_t              *conf   = NULL;
-        dht_methods_t           *methods = NULL;
-        xlator_t                *subvol = 0;
-        xlator_t                *hashed_subvol = 0;
-        int                      ret    = 0;
-        int                      readdir_optimize = 0;
-        inode_table_t           *itable = NULL;
-        inode_t                 *inode = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
-        itable = local->fd ? local->fd->inode->table : NULL;
+    if (layout == NULL)
+        goto done;
 
-        conf  = this->private;
-        GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
+    if (conf->readdir_optimize == _gf_true)
+        readdir_optimize = 1;
 
-        methods = &(conf->methods);
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-        if (op_ret < 0)
-                goto done;
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-        layout = local->layout;
+        if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
+            /*stat failed somewhere- display this entry but the data may
+             * be inaccurate.
+             */
+            gf_msg_debug(this->name, EINVAL, "Invalid stat for %s (gfid %s)",
+                         orig_entry->d_name,
+                         uuid_utoa(orig_entry->d_stat.ia_gfid));
+        }
 
-        /* We have seen crashes in while running "rm -rf" on tier volumes
-           when the layout was NULL on the hot tier. This will skip the
-           entries on the subvol without a layout, hence preventing the crash
-           but rmdir might fail with "directory not empty" errors*/
+        if (check_is_linkfile(NULL, (&orig_entry->d_stat), orig_entry->dict,
+                              conf->link_xattr_name)) {
+            gf_msg_debug(this->name, 0, "%s: %s is a linkto file", prev->name,
+                         orig_entry->d_name);
+            continue;
+        }
 
-        if (layout == NULL)
-                goto done;
+        if (skip_hashed_check) {
+            goto list;
+        }
 
-        if (conf->readdir_optimize == _gf_true)
-                 readdir_optimize = 1;
+        if (check_is_dir(NULL, (&orig_entry->d_stat), NULL)) {
+            /*Directory entries filtering :
+             * a) If rebalance is running, pick from first_up_subvol
+             * b) (rebalance not running)hashed subvolume is NULL or
+             * down then filter in first_up_subvolume. Other wise the
+             * corresponding hashed subvolume will take care of the
+             * directory entry.
+             */
+            if (readdir_optimize) {
+                if (prev == local->first_up_subvol)
+                    goto list;
+                else
+                    continue;
+            }
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
+            hashed_subvol = methods->layout_search(this, layout,
+                                                   orig_entry->d_name);
 
-                if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
-                        /*stat failed somewhere- ignore this entry*/
-                        gf_msg_debug (this->name, EINVAL,
-                                      "Invalid stat, ignoring entry "
-                                      "%s gfid %s", orig_entry->d_name,
-                                      uuid_utoa (orig_entry->d_stat.ia_gfid));
-                        continue;
-                }
+            if (prev == hashed_subvol)
+                goto list;
+            if ((hashed_subvol && dht_subvol_status(conf, hashed_subvol)) ||
+                (prev != local->first_up_subvol))
+                continue;
 
-                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL)) {
+            goto list;
+        }
 
-                /*Directory entries filtering :
-                 * a) If rebalance is running, pick from first_up_subvol
-                 * b) (rebalance not running)hashed subvolume is NULL or
-                 * down then filter in first_up_subvolume. Other wise the
-                 * corresponding hashed subvolume will take care of the
-                 * directory entry.
-                 */
-                        if (readdir_optimize) {
-                                if (prev->this == local->first_up_subvol)
-                                        goto list;
-                                else
-                                        continue;
+    list:
+        entry = gf_dirent_for_name(orig_entry->d_name);
+        if (!entry) {
+            goto unwind;
+        }
 
-                        }
+        /* Do this if conf->search_unhashed is set to "auto" */
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            subvol = methods->layout_search(this, layout, orig_entry->d_name);
+            if (!subvol || (subvol != prev)) {
+                /* TODO: Count the number of entries which need
+                   linkfile to prove its existence in fs */
+                layout->search_unhashed++;
+            }
+        }
 
-                        hashed_subvol = methods->layout_search (this, layout,
-                                                         orig_entry->d_name);
+        entry->d_off = orig_entry->d_off;
+        entry->d_stat = orig_entry->d_stat;
+        entry->d_ino = orig_entry->d_ino;
+        entry->d_type = orig_entry->d_type;
+        entry->d_len = orig_entry->d_len;
 
-                        if (prev->this == hashed_subvol)
-                                goto list;
-                        if ((hashed_subvol
-                                && dht_subvol_status (conf, hashed_subvol))
-                                ||(prev->this != local->first_up_subvol))
-                                continue;
+        if (orig_entry->dict)
+            entry->dict = dict_ref(orig_entry->dict);
 
-                        goto list;
-                }
+        /* making sure we set the inode ctx right with layout,
+           currently possible only for non-directories, so for
+           directories don't set entry inodes */
+        if (IA_ISDIR(entry->d_stat.ia_type)) {
+            entry->d_stat.ia_blocks = DHT_DIR_STAT_BLOCKS;
+            entry->d_stat.ia_size = DHT_DIR_STAT_SIZE;
+            if (orig_entry->inode) {
+                dht_inode_ctx_time_update(orig_entry->inode, this,
+                                          &entry->d_stat, 1);
 
-                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict,
-                                       conf->link_xattr_name)) {
-                        continue;
+                if (conf->subvolume_cnt == 1) {
+                    dht_populate_inode_for_dentry(this, prev, entry,
+                                                  orig_entry);
                 }
-list:
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
+            }
+        } else {
+            if (orig_entry->dict &&
+                dict_get(orig_entry->dict, conf->link_xattr_name)) {
+                /* Strip out the S and T flags set by rebalance*/
+                DHT_STRIP_PHASE1_FLAGS(&entry->d_stat);
+            }
 
-                        goto unwind;
+            if (orig_entry->inode) {
+                ret = dht_layout_preset(this, prev, orig_entry->inode);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           DHT_MSG_LAYOUT_SET_FAILED,
+                           "failed to link the layout "
+                           "in inode for %s",
+                           orig_entry->d_name);
+
+                entry->inode = inode_ref(orig_entry->inode);
+            } else if (itable) {
+                /*
+                 * orig_entry->inode might be null if any upper
+                 * layer xlators below client set to null, to
+                 * force a lookup on the inode even if the inode
+                 * is present in the inode table. In that case
+                 * we just update the ctx to make sure we didn't
+                 * missed anything.
+                 */
+                inode = inode_find(itable, orig_entry->d_stat.ia_gfid);
+                if (inode) {
+                    ret = dht_layout_preset(this, prev, inode);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_LAYOUT_SET_FAILED,
+                               "failed to link the layout"
+                               " in inode for %s",
+                               orig_entry->d_name);
+                    inode_unref(inode);
+                    inode = NULL;
                 }
+            }
+        }
 
-                /* Do this if conf->search_unhashed is set to "auto" */
-                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
-                        subvol = methods->layout_search (this, layout,
-                                                         orig_entry->d_name);
-                        if (!subvol || (subvol != prev->this)) {
-                                /* TODO: Count the number of entries which need
-                                   linkfile to prove its existence in fs */
-                                layout->search_unhashed++;
-                        }
-                }
+        gf_msg_debug(this->name, 0, "%s: Adding entry = %s", prev->name,
+                     entry->d_name);
 
-                entry->d_off  = orig_entry->d_off;
-                entry->d_stat = orig_entry->d_stat;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                if (orig_entry->dict)
-                        entry->dict = dict_ref (orig_entry->dict);
-
-                /* making sure we set the inode ctx right with layout,
-                   currently possible only for non-directories, so for
-                   directories don't set entry inodes */
-                if (IA_ISDIR(entry->d_stat.ia_type)) {
-                        entry->d_stat.ia_blocks = DHT_DIR_STAT_BLOCKS;
-                        entry->d_stat.ia_size = DHT_DIR_STAT_SIZE;
-                        if (orig_entry->inode) {
-                                dht_inode_ctx_time_update (orig_entry->inode,
-                                                           this, &entry->d_stat,
-                                                           1);
-                        }
-                } else {
-                        if (orig_entry->inode) {
-                                ret = dht_layout_preset (this, prev->this,
-                                                         orig_entry->inode);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                DHT_MSG_LAYOUT_SET_FAILED,
-                                                "failed to link the layout "
-                                                "in inode");
-
-                                entry->inode = inode_ref (orig_entry->inode);
-                        } else if (itable) {
-                                /*
-                                 * orig_entry->inode might be null if any upper
-                                 * layer xlators below client set to null, to
-                                 * force a lookup on the inode even if the inode
-                                 * is present in the inode table. In that case
-                                 * we just update the ctx to make sure we didn't
-                                 * missed anything.
-                                 */
-                                inode = inode_find (itable,
-                                                    orig_entry->d_stat.ia_gfid);
-                                if (inode) {
-                                        ret = dht_layout_preset
-                                                            (this, prev->this,
-                                                             inode);
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                     GF_LOG_WARNING, 0,
-                                                     DHT_MSG_LAYOUT_SET_FAILED,
-                                                     "failed to link the layout"
-                                                     " in inode");
-                                        inode_unref (inode);
-                                        inode = NULL;
-                                }
-                        }
-                }
-                list_add_tail (&entry->list, &entries.list);
-                count++;
-        }
-        op_ret = count;
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if (prev->this != dht_last_up_subvol (this))
-                op_errno = 0;
+        list_add_tail(&entry->list, &entries.list);
+        count++;
+    }
 
 done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = dht_subvol_next (this, prev->this);
-                } else {
-                        next_subvol = prev->this;
-                }
 
-                if (!next_subvol) {
-                        goto unwind;
-                }
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     * Possible values:
+     * op_ret == 0 and op_errno != 0
+     *   if op_errno != ENOENT : Error.Unwind.
+     *   if op_errno == ENOENT : There are no more entries on this subvol.
+     *                           Move to the next one.
+     * op_ret > 0 and count == 0 :
+     *    The subvol returned entries to dht but all were stripped out.
+     *    For example, if they were linkto files or dirs where
+     *    hashed_subvol != prev. Try to get some entries by winding
+     *    to the next subvol. This can be dangerous if parallel readdir
+     *    is enabled as it grows the stack.
+     *
+     * op_ret > 0 and count > 0:
+     *   We found some entries. Unwind even if the buffer is not full.
+     *
+     */
+
+    op_ret = count;
+    if (count == 0) {
+        /* non-zero next_offset means that
+         * EOF is not yet hit on the current subvol
+         */
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
+        }
 
-                if (conf->readdir_optimize == _gf_true) {
-                        if (next_subvol != local->first_up_subvol) {
-                                ret = dict_set_int32 (local->xattr,
-                                                      GF_READDIR_SKIP_DIRS, 1);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_DICT_SET_FAILED,
-                                                "Failed to set dictionary value"
-                                                ":key = %s",
-                                                GF_READDIR_SKIP_DIRS );
-                        } else {
-                                 dict_del (local->xattr,
-                                           GF_READDIR_SKIP_DIRS);
-                        }
-                }
+        if (!next_subvol) {
+            goto unwind;
+        }
 
-                STACK_WIND (frame, dht_readdirp_cbk,
-                            next_subvol, next_subvol->fops->readdirp,
-                            local->fd, local->size, next_offset,
-                            local->xattr);
-                return 0;
+        if (conf->readdir_optimize == _gf_true) {
+            if (next_subvol != local->first_up_subvol) {
+                ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary value"
+                           ":key = %s",
+                           GF_READDIR_SKIP_DIRS);
+            } else {
+                dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
+            }
         }
 
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdirp, local->fd, local->size,
+                          next_offset, local->xattr);
+        return 0;
+    }
+
 unwind:
-        if (op_ret < 0)
-                op_ret = 0;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (op_ret < 0)
+        op_ret = 0;
 
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
 
-        gf_dirent_free (&entries);
+    DHT_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries, NULL);
 
-        return 0;
+    gf_dirent_free(&entries);
+    return 0;
 }
 
-
-
-int
-dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                 dict_t *xdata)
+static int
+dht_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_layout_t *layout = 0;
-        xlator_t     *subvol = 0;
-        dht_conf_t   *conf = NULL;
-        dht_methods_t *methods = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, done);
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
 
-        methods = &(conf->methods);
+    INIT_LIST_HEAD(&entries.list);
 
-        if (op_ret < 0)
-                goto done;
+    prev = cookie;
+    local = frame->local;
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, done);
 
-        layout = local->layout;
+    methods = &(conf->methods);
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
+    if (op_ret <= 0)
+        goto done;
 
-                subvol = methods->layout_search (this, layout,
-                                                 orig_entry->d_name);
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-                if (!subvol || (subvol == prev->this)) {
-                        entry = gf_dirent_for_name (orig_entry->d_name);
-                        if (!entry) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "Memory allocation failed ");
-                                goto unwind;
-                        }
+    layout = local->layout;
 
-                        entry->d_off  = orig_entry->d_off;
-                        entry->d_ino  = orig_entry->d_ino;
-                        entry->d_type = orig_entry->d_type;
-                        entry->d_len  = orig_entry->d_len;
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-                        list_add_tail (&entry->list, &entries.list);
-                        count++;
-                }
-        }
-        op_ret = count;
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if (prev->this != dht_last_up_subvol (this))
-                op_errno = 0;
+    if (conf->subvolume_cnt == 1) {
+        /*return everything*/
+        skip_hashed_check = _gf_true;
+        count = op_ret;
+        goto done;
+    }
 
-done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = dht_subvol_next (this, prev->this);
-                } else {
-                        next_subvol = prev->this;
-                }
-
-                if (!next_subvol) {
-                        goto unwind;
-                }
-
-                STACK_WIND (frame, dht_readdir_cbk,
-                            next_subvol, next_subvol->fops->readdir,
-                            local->fd, local->size, next_offset, NULL);
-                return 0;
-        }
-
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-        gf_dirent_free (&entries);
-
-        return 0;
-}
+        subvol = methods->layout_search(this, layout, orig_entry->d_name);
 
+        if (!subvol || (subvol == prev)) {
+            entry = gf_dirent_for_name(orig_entry->d_name);
+            if (!entry) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "Memory allocation failed ");
+                goto unwind;
+            }
 
-int
-dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop, dict_t *dict)
-{
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        xlator_t     *xvol = NULL;
-        int           ret = 0;
-        dht_conf_t   *conf = NULL;
+            entry->d_off = orig_entry->d_off;
+            entry->d_ino = orig_entry->d_ino;
+            entry->d_type = orig_entry->d_type;
+            entry->d_len = orig_entry->d_len;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+            gf_msg_debug(this->name, 0, "%s: Adding = entry %s", prev->name,
+                         entry->d_name);
 
-        conf = this->private;
+            list_add_tail(&entry->list, &entries.list);
+            count++;
+        }
+    }
+done:
+    op_ret = count;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (count == 0) {
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
+        }
 
-        local = dht_local_init (frame, NULL, NULL, whichop);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+        if (!next_subvol) {
+            goto unwind;
         }
 
-        local->fd = fd_ref (fd);
-        local->size = size;
-        local->xattr_req = (dict)? dict_ref (dict) : NULL;
-        local->first_up_subvol = dht_first_up_subvol (this);
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdir, local->fd, local->size,
+                          next_offset, NULL);
+        return 0;
+    }
 
-        dht_deitransform (this, yoff, &xvol);
+unwind:
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
 
-        /* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIRP) {
-                if (dict)
-                        local->xattr = dict_ref (dict);
-                else
-                        local->xattr = dict_new ();
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
 
-                if (local->xattr) {
-                        ret = dict_set_uint32 (local->xattr,
-                                               conf->link_xattr_name, 256);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value"
-                                        " : key = %s",
-                                        conf->link_xattr_name);
+    if (!skip_hashed_check) {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, NULL);
+        gf_dirent_free(&entries);
 
-                        if (conf->readdir_optimize == _gf_true) {
-                                if (xvol != local->first_up_subvol) {
-                                        ret = dict_set_int32 (local->xattr,
-                                                       GF_READDIR_SKIP_DIRS, 1);
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        DHT_MSG_DICT_SET_FAILED,
-                                                        "Failed to set "
-                                                        "dictionary value: "
-                                                        "key = %s",
-                                                        GF_READDIR_SKIP_DIRS);
-                                } else {
-                                        dict_del (local->xattr,
-                                                  GF_READDIR_SKIP_DIRS);
-                                }
-                        }
+    } else {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, orig_entries, NULL);
+    }
+    return 0;
+}
+
+static int
+dht_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t yoff, int whichop, dict_t *dict)
+{
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    xlator_t *xvol = NULL;
+    int ret = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, NULL, whichop);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->fd = fd_ref(fd);
+    local->size = size;
+    local->xattr_req = (dict) ? dict_ref(dict) : NULL;
+    local->first_up_subvol = dht_first_up_subvol(this);
+    local->op_ret = -1;
+
+    dht_deitransform(this, yoff, &xvol);
+
+    /* TODO: do proper readdir */
+    if (whichop == GF_FOP_READDIRP) {
+        if (dict)
+            local->xattr = dict_ref(dict);
+        else
+            local->xattr = dict_new();
+
+        if (local->xattr) {
+            ret = dict_set_uint32(local->xattr, conf->link_xattr_name, 256);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set dictionary value"
+                       " : key = %s",
+                       conf->link_xattr_name);
+
+            if (conf->readdir_optimize == _gf_true) {
+                if (xvol != local->first_up_subvol) {
+                    ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               DHT_MSG_DICT_SET_FAILED,
+                               "Failed to set "
+                               "dictionary value: "
+                               "key = %s",
+                               GF_READDIR_SKIP_DIRS);
+                } else {
+                    dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
                 }
+            }
 
-                STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
-                            fd, size, yoff, local->xattr);
-        } else {
-                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, yoff, local->xattr);
+            if (conf->subvolume_cnt == 1) {
+                ret = dict_set_uint32(local->xattr, conf->xattr_name, 4 * 4);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                           DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary "
+                           "value:key = %s ",
+                           conf->xattr_name);
+                }
+            }
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, xvol, xvol,
+                          xvol->fops->readdirp, fd, size, yoff, local->xattr);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, xvol, xvol,
+                          xvol->fops->readdir, fd, size, yoff, local->xattr);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff, dict_t *xdata)
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t yoff, dict_t *xdata)
 {
-        int          op = GF_FOP_READDIR;
-        dht_conf_t  *conf = NULL;
-        int          i = 0;
+    int op = GF_FOP_READDIR;
+    dht_conf_t *conf = NULL;
+    int i = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        op = GF_FOP_READDIRP;
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            op = GF_FOP_READDIRP;
+            break;
         }
+    }
 
-        if (conf->use_readdirp)
-                op = GF_FOP_READDIRP;
+    if (conf->use_readdirp)
+        op = GF_FOP_READDIRP;
 
 out:
-        dht_do_readdir (frame, this, fd, size, yoff, op, 0);
-        return 0;
+    dht_do_readdir(frame, this, fd, size, yoff, op, 0);
+    return 0;
 }
 
 int
-dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff, dict_t *dict)
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t yoff, dict_t *dict)
 {
-        dht_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
-        return 0;
+    dht_do_readdir(frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
+    return 0;
 }
 
-
-
-int
-dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
+    local = frame->local;
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1)
-                        local->op_errno = op_errno;
-
-                if (op_ret == 0)
-                        local->op_ret = 0;
-        }
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno, xdata);
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1)
+            local->op_errno = op_errno;
+        else if (op_ret == 0)
+            local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int datasync, dict_t *xdata)
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+             dict_t *xdata)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        conf = this->private;
+    conf = this->private;
 
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_FSYNCDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_FSYNCDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local->fd = fd_ref (fd);
-        local->call_cnt = conf->subvolume_cnt;
+    local->fd = fd_ref(fd);
+    local->call_cnt = conf->subvolume_cnt;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_fsyncdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->fsyncdir,
-                            fd, datasync, xdata);
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_fsyncdir_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->fsyncdir, fd, datasync, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
 
+    if (op_ret == -1)
+        goto out;
 
-        if (op_ret == -1)
-                goto out;
-
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        prev = cookie;
-
-        if (local->loc.parent) {
-
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
-
-        ret = dht_layout_preset (this, prev, inode);
-        if (ret < 0) {
-                gf_msg_debug (this->name, EINVAL,
-                              "could not set pre-set layout for subvolume %s",
-                              prev? prev->name: NULL);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-        if (local->linked == _gf_true)
-                dht_linkfile_attr_heal (frame, this);
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    prev = cookie;
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "could not set pre-set layout for subvolume %s",
+                     prev ? prev->name : NULL);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
 out:
-        /*
-         * FIXME: ia_size and st_blocks of preparent and postparent do not have
-         * correct values. since, preparent and postparent buffers correspond
-         * to a directory these two members should have values equal to sum of
-         * corresponding values from each of the subvolume.
-         * See dht_iatt_merge for reference.
-         */
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (postparent);
-        dht_set_fixed_dir_stat (preparent);
-
-        if (local && local->lock.locks) {
-                /* store op_errno for failure case*/
-                local->op_errno = op_errno;
-                local->refresh_layout_unlock (frame, this, op_ret, 1);
+    /*
+     * FIXME: ia_size and st_blocks of preparent and postparent do not have
+     * correct values. since, preparent and postparent buffers correspond
+     * to a directory these two members should have values equal to sum of
+     * corresponding values from each of the subvolume.
+     * See dht_iatt_merge for reference.
+     */
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    dht_set_fixed_dir_stat(preparent);
+
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
 
-                if (op_ret == 0) {
-                        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno,
-                                          inode, stbuf, preparent, postparent,
-                                          xdata);
-                }
-        } else {
-                DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                                  stbuf, preparent, postparent, xdata);
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                             preparent, postparent, xdata);
         }
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this,
-                               int32_t op_ret, int32_t op_errno,
-                               inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent,
-                               dict_t *xdata)
+static int
+dht_mknod_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *stbuf, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *cached_subvol  = NULL;
-        dht_conf_t      *conf           = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        conf = this->private;
-        if (!conf) {
-                local->op_errno =  EINVAL;
-                op_errno = EINVAL;
-                goto err;
-        }
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        cached_subvol = local->cached_subvol;
+    cached_subvol = local->cached_subvol;
 
-        if (local->params) {
-                 dict_del (local->params, conf->link_xattr_name);
-                 dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
-                           cached_subvol, cached_subvol->fops->mknod,
-                           &local->loc, local->mode, local->rdev, local->umask,
-                           local->params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)cached_subvol,
+                      cached_subvol, cached_subvol->fops->mknod, &local->loc,
+                      local->mode, local->rdev, local->umask, local->params);
 
-        return 0;
+    return 0;
 err:
-        if (local && local->lock.locks) {
-                local->refresh_layout_unlock (frame, this, -1, 1);
-        } else {
-                DHT_STACK_UNWIND (mknod, frame, -1,
-                                  op_errno, NULL, NULL, NULL,
-                                  NULL, NULL);
-        }
-        return 0;
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    }
+    return 0;
 }
 
-int
-dht_mknod_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this,
-                                 xlator_t *subvol, loc_t *loc, dev_t rdev,
-                                 mode_t mode, mode_t umask, dict_t *params)
+static int
+dht_mknod_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                               xlator_t *subvol, loc_t *loc, dev_t rdev,
+                               mode_t mode, mode_t umask, dict_t *params)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *avail_subvol   = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path,
-                              subvol->name);
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
-                                   subvol, subvol->fops->mknod, loc, mode,
-                                   rdev, umask, params);
-        } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
-
-                if (avail_subvol != subvol) {
-                        local->params = dict_ref (params);
-                        local->rdev = rdev;
-                        local->mode = mode;
-                        local->umask = umask;
-                        local->cached_subvol = avail_subvol;
-                        local->hashed_subvol = subvol;
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-                        gf_msg_debug (this->name, 0,
-                                      "creating %s on %s (link at %s)", loc->path,
-                                      avail_subvol->name, subvol->name);
+        if (avail_subvol != subvol) {
+            local->params = dict_ref(params);
+            local->rdev = rdev;
+            local->mode = mode;
+            local->umask = umask;
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
 
-                        dht_linkfile_create (frame,
-                                             dht_mknod_linkfile_create_cbk,
-                                             this, avail_subvol, subvol, loc);
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
 
-                        goto out;
-                }
+            dht_linkfile_create(frame, dht_mknod_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
 
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path, subvol->name);
+            goto out;
+        }
 
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                                   (void *)subvol, subvol,
-                                   subvol->fops->mknod, loc, mode,
-                                   rdev, umask, params);
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-        }
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    }
 out:
-        return 0;
+    return 0;
 }
 
-int32_t
-dht_mknod_do (call_frame_t *frame)
+static int32_t
+dht_mknod_do(call_frame_t *frame)
 {
-        dht_local_t     *local          = NULL;
-        dht_layout_t    *refreshed      = NULL;
-        xlator_t        *subvol         = NULL;
-        xlator_t        *this           = NULL;
-        dht_conf_t      *conf           = NULL;
-        dht_methods_t   *methods        = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        this = THIS;
+    this = THIS;
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_VALIDATE_OR_GOTO (this->name, conf, err);
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
 
-        methods = &(conf->methods);
+    methods = &(conf->methods);
 
-        /* We don't need parent_loc anymore */
-        loc_wipe (&local->loc);
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
 
-        loc_copy (&local->loc, &local->loc2);
+    loc_copy(&local->loc, &local->loc2);
 
-        loc_wipe (&local->loc2);
+    loc_wipe(&local->loc2);
 
-        refreshed = local->selfheal.refreshed_layout;
+    refreshed = local->selfheal.refreshed_layout;
 
-        subvol = methods->layout_search (this, refreshed, local->loc.name);
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
 
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "no subvolume in "
-                        "layout for path=%s", local->loc.path);
-                local->op_errno = ENOENT;
-                goto err;
-        }
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
 
-        dht_mknod_wind_to_avail_subvol (frame, this, subvol, &local->loc,
-                                         local->rdev, local->mode,
-                                         local->umask, local->params);
-        return 0;
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                   local->rdev, local->mode, local->umask,
+                                   local->params);
+    return 0;
 err:
-        local->refresh_layout_unlock (frame, this, -1, 1);
+    local->refresh_layout_unlock(frame, this, -1, 1);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-dht_mknod_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_mknod_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
-int32_t
-dht_mknod_finish (call_frame_t *frame, xlator_t *this, int op_ret,
-                  int invoke_cbk)
+static int32_t
+dht_mknod_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                 int invoke_cbk)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        local = frame->local;
-        lock_count = dht_lock_count (local->lock.locks, local->lock.lk_count);
-        if (lock_count == 0)
-                goto done;
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
 
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                goto done;
-        }
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
 
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL) {
-                goto done;
-        }
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
 
-        lock_local->lock.locks = local->lock.locks;
-        lock_local->lock.lk_count = local->lock.lk_count;
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
 
-        dht_unlock_inodelk (lock_frame, lock_local->lock.locks,
-                            lock_local->lock.lk_count,
-                            dht_mknod_unlock_cbk);
-        lock_frame = NULL;
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_mknod_unlock_cbk);
+    lock_frame = NULL;
 
 done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-        if (op_ret == 0)
-                return 0;
-
-        DHT_STACK_UNWIND (mknod, frame, op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL);
+    if (op_ret == 0)
         return 0;
+
+    DHT_STACK_UNWIND(mknod, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL);
+    return 0;
 }
 
-int32_t
-dht_mknod_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_mknod_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                goto err;
-        }
+    if (!local) {
+        goto err;
+    }
 
-        if (op_ret < 0) {
-                gf_msg ("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
-                        "mknod lock failed for file: %s", local->loc2.name);
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "mknod lock failed for file: %s", local->loc2.name);
 
-                local->op_errno = op_errno;
+        local->op_errno = op_errno;
 
-                goto err;
-        }
+        goto err;
+    }
 
-        local->refresh_layout_unlock = dht_mknod_finish;
+    local->refresh_layout_unlock = dht_mknod_finish;
 
-        local->refresh_layout_done = dht_mknod_do;
+    local->refresh_layout_done = dht_mknod_do;
 
-        dht_refresh_layout (frame);
+    dht_refresh_layout(frame);
 
-        return 0;
+    return 0;
 err:
-        dht_mknod_finish (frame, this, -1, 0);
-        return 0;
+    if (local)
+        dht_mknod_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(mknod, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
-int32_t
-dht_mknod_lock (call_frame_t *frame, xlator_t *subvol)
+static int32_t
+dht_mknod_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        dht_local_t     *local          = NULL;
-        int              count  = 1,    ret = -1;
-        dht_lock_t     **lk_array       = NULL;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        local = frame->local;
+    local = frame->local;
 
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-        if (lk_array == NULL)
-                goto err;
+    if (lk_array == NULL)
+        goto err;
 
-        lk_array[0] = dht_lock_new (frame->this, subvol, &local->loc, F_RDLCK,
-                                    DHT_LAYOUT_HEAL_DOMAIN);
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
 
-        if (lk_array[0] == NULL)
-                goto err;
+    if (lk_array[0] == NULL)
+        goto err;
 
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    IGNORE_ENOENT_ESTALE, dht_mknod_lock_cbk);
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_mknod_lock_cbk);
 
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                goto err;
-        }
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
 
-        return 0;
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
-int
-dht_refresh_parent_layout_resume (call_frame_t *frame, xlator_t *this, int ret,
-                                  int invoke_cbk)
+static int
+dht_refresh_parent_layout_resume(call_frame_t *frame, xlator_t *this, int ret,
+                                 int invoke_cbk)
 {
-        dht_local_t  *local        = NULL, *parent_local = NULL;
-        call_stub_t  *stub         = NULL;
-        call_frame_t *parent_frame = NULL;
+    dht_local_t *local = NULL, *parent_local = NULL;
+    call_stub_t *stub = NULL;
+    call_frame_t *parent_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        stub = local->stub;
-        local->stub = NULL;
+    stub = local->stub;
+    local->stub = NULL;
 
-        parent_frame = stub->frame;
-        parent_local = parent_frame->local;
+    parent_frame = stub->frame;
+    parent_local = parent_frame->local;
 
-        if (ret < 0) {
-                parent_local->op_ret = -1;
-                parent_local->op_errno = local->op_errno
-                        ? local->op_errno : EIO;
-        } else {
-                parent_local->op_ret = 0;
-        }
+    if (ret < 0) {
+        parent_local->op_ret = -1;
+        parent_local->op_errno = local->op_errno ? local->op_errno : EIO;
+    } else {
+        parent_local->op_ret = 0;
+    }
 
-        call_resume (stub);
+    call_resume(stub);
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_refresh_parent_layout_done (call_frame_t *frame)
+static int
+dht_refresh_parent_layout_done(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        int          ret   = 0;
+    dht_local_t *local = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                ret = -1;
-                goto resume;
-        }
+    if (local->op_ret < 0) {
+        ret = -1;
+        goto resume;
+    }
 
-        dht_layout_set (frame->this, local->loc.inode,
-                        local->selfheal.refreshed_layout);
+    dht_layout_set(frame->this, local->loc.inode,
+                   local->selfheal.refreshed_layout);
 
 resume:
-        dht_refresh_parent_layout_resume (frame, frame->this, ret, 1);
-        return 0;
+    dht_refresh_parent_layout_resume(frame, frame->this, ret, 1);
+    return 0;
 }
 
-
-int
-dht_handle_parent_layout_change (xlator_t *this, call_stub_t *stub)
-{
-        call_frame_t *refresh_frame = NULL, *frame = NULL;
-        dht_local_t  *refresh_local = NULL, *local = NULL;
-
-        frame = stub->frame;
-        local = frame->local;
-
-        refresh_frame = copy_frame (frame);
-        refresh_local = dht_local_init (refresh_frame, NULL, NULL,
-                                        stub->fop);
-
-        refresh_local->loc.inode = inode_ref (local->loc.parent);
-        gf_uuid_copy (refresh_local->loc.gfid, local->loc.parent->gfid);
-
-        refresh_local->stub = stub;
-
-        refresh_local->refresh_layout_unlock = dht_refresh_parent_layout_resume;
-        refresh_local->refresh_layout_done = dht_refresh_parent_layout_done;
-
-        dht_refresh_layout (refresh_frame);
-        return 0;
-}
-
-int32_t
-dht_unlock_parent_layout_during_entry_fop_done (call_frame_t *frame,
-                                                void *cookie,
-                                                xlator_t *this,
-                                                int32_t op_ret,
-                                                int32_t op_errno,
-                                                dict_t *xdata)
-{
-        dht_local_t *local                   = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        gf_uuid_unparse (local->lock.locks[0]->loc.inode->gfid, gfid);
-
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "unlock failed on gfid: %s, stale lock might be left "
-                        "in DHT_LAYOUT_HEAL_DOMAIN", gfid);
-        }
-
-        DHT_STACK_DESTROY (frame);
-        return 0;
-}
-
-int32_t
-dht_unlock_parent_layout_during_entry_fop (call_frame_t *frame)
+static int
+dht_handle_parent_layout_change(xlator_t *this, call_stub_t *stub)
 {
-        dht_local_t  *local                   = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame              = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
+    call_frame_t *refresh_frame = NULL, *frame = NULL;
+    dht_local_t *refresh_local = NULL, *local = NULL;
 
-        gf_uuid_unparse (local->loc.parent->gfid, pgfid);
+    frame = stub->frame;
+    local = frame->local;
 
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "copy frame failed", pgfid, local->loc.name,
-                        local->loc.path);
-                goto done;
-        }
-
-        lock_local = mem_get0 (THIS->local_pool);
-        if (lock_local == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "local creation failed", pgfid, local->loc.name,
-                        local->loc.path);
-                goto done;
-        }
+    refresh_frame = copy_frame(frame);
+    if (!refresh_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_frame");
+        return -1;
+    }
 
-        lock_frame->local = lock_local;
+    refresh_local = dht_local_init(refresh_frame, NULL, NULL, stub->fop);
+    if (!refresh_local) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_local");
+        return -1;
+    }
 
-        lock_local->lock.locks = local->lock.locks;
-        lock_local->lock.lk_count = local->lock.lk_count;
+    refresh_local->loc.inode = inode_ref(local->loc.parent);
+    gf_uuid_copy(refresh_local->loc.gfid, local->loc.parent->gfid);
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
+    refresh_local->stub = stub;
 
-        dht_unlock_inodelk (lock_frame, lock_local->lock.locks,
-                            lock_local->lock.lk_count,
-                            dht_unlock_parent_layout_during_entry_fop_done);
+    refresh_local->refresh_layout_unlock = dht_refresh_parent_layout_resume;
+    refresh_local->refresh_layout_done = dht_refresh_parent_layout_done;
 
-done:
-        return 0;
+    dht_refresh_layout(refresh_frame);
+    return 0;
 }
 
-int32_t
-dht_guard_parent_layout_during_entry_fop_cbk (call_frame_t *frame, void *cookie,
-                                              xlator_t *this, int32_t op_ret,
-                                              int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_call_mkdir_stub(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        call_stub_t *stub  = NULL;
+    dht_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        local = frame->local;
-        stub = local->stub;
-        local->stub = NULL;
+    local = frame->local;
+    stub = local->stub;
+    local->stub = NULL;
 
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-        } else {
-                local->op_ret = 0;
-        }
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    } else {
+        local->op_ret = 0;
+    }
 
-        call_resume (stub);
+    call_resume(stub);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-dht_guard_parent_layout_during_entry_fop (xlator_t *subvol, call_stub_t *stub)
+static int32_t
+dht_guard_parent_layout_and_namespace(xlator_t *subvol, call_stub_t *stub)
 {
-        dht_local_t   *local                  = NULL;
-        int            count                  = 1,    ret = -1;
-        dht_lock_t   **lk_array               = NULL;
-        loc_t         *loc                    = NULL;
-        xlator_t      *hashed_subvol          = NULL, *this = NULL;;
-        call_frame_t  *frame                  = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        loc_t          parent                 = {0, };
-        int32_t       *parent_disk_layout     = NULL;
-        dht_layout_t  *parent_layout          = NULL;
-        dht_conf_t    *conf                   = NULL;
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    xlator_t *hashed_subvol = NULL, *this = NULL;
+    ;
+    call_frame_t *frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    dht_conf_t *conf = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", stub, err);
+    GF_VALIDATE_OR_GOTO("dht", stub, err);
 
-        frame = stub->frame;
-        this = frame->this;
+    frame = stub->frame;
+    this = frame->this;
 
-        conf = this->private;
+    conf = this->private;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->stub = stub;
+    local->stub = stub;
 
-        /* TODO: recheck whether we should lock on src or dst if we do similar
-         * stale layout checks for rename.
-         */
-        loc = &stub->args.loc;
+    /* TODO: recheck whether we should lock on src or dst if we do similar
+     * stale layout checks for rename.
+     */
+    loc = &stub->args.loc;
 
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
 
+    if (local->params == NULL) {
+        local->params = dict_new();
         if (local->params == NULL) {
-                local->params = dict_new ();
-                if (local->params == NULL) {
-                        local->op_errno = ENOMEM;
-                        gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                                DHT_MSG_PARENT_LAYOUT_CHANGED,
-                                "%s (%s/%s) (path: %s): "
-                                "dict allocation failed",
-                                gf_fop_list[stub->fop],
-                                pgfid, loc->name, loc->path);
-                        goto err;
-                }
-        }
-
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                local->op_errno = EINVAL;
-
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "hashed subvolume not found", gf_fop_list[stub->fop],
-                        pgfid, loc->name, loc->path);
-                goto err;
-        }
-
-        parent_layout = dht_layout_get (this, loc->parent);
-
-        ret = dht_disk_layout_extract_for_subvol (this, parent_layout,
-                                                  hashed_subvol,
-                                                  &parent_disk_layout);
-        if (ret == -1) {
-                local->op_errno = EINVAL;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "extracting in-memory layout of parent failed. ",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
-                goto err;
-        }
-
-        memcpy ((void *)local->parent_disk_layout, (void *)parent_disk_layout,
-                sizeof (local->parent_disk_layout));
-
-        dht_layout_unref (this, parent_layout);
-        parent_layout = NULL;
-
-        ret = dict_set_str (local->params, GF_PREOP_PARENT_KEY,
-                            conf->xattr_name);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "setting %s key in params dictionary failed. ",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path,
-                        GF_PREOP_PARENT_KEY);
-                goto err;
-        }
-
-        ret = dict_set_bin (local->params, conf->xattr_name, parent_disk_layout,
-                            4 * 4);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "setting parent-layout in params dictionary failed. ",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
-                goto err;
-        }
-
-        parent_disk_layout = NULL;
-
-        parent.inode = inode_ref (loc->parent);
-        gf_uuid_copy (parent.gfid, loc->parent->gfid);
-
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
-
-        if (lk_array == NULL) {
-                local->op_errno = ENOMEM;
-
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "calloc failure",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
-
-                goto err;
-        }
-
-        lk_array[0] = dht_lock_new (frame->this, hashed_subvol, &parent,
-                                    F_RDLCK, DHT_LAYOUT_HEAL_DOMAIN);
-
-        if (lk_array[0] == NULL) {
-                local->op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "lock allocation failed",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
-
-                goto err;
-        }
-
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
-
-        ret = dht_blocking_inodelk (frame, lk_array, count, FAIL_ON_ANY_ERROR,
-                                    dht_guard_parent_layout_during_entry_fop_cbk);
-
-        if (ret < 0) {
-                local->op_errno = EIO;
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "dht_blocking_inodelk failed",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
-
-                goto err;
-        }
-
-        loc_wipe (&parent);
-
-        return 0;
+            local->op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+                   DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "%s (%s/%s) (path: %s): "
+                   "dict allocation failed",
+                   gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+            goto err;
+        }
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        local->op_errno = EINVAL;
+
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "hashed subvolume not found",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(local->params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+    local->hashed_subvol = hashed_subvol;
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, hashed_subvol, &local->current->ns,
+                                dht_call_mkdir_stub);
+    if (ret < 0)
+        goto err;
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
 
-        loc_wipe (&parent);
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
 
-        if (parent_disk_layout != NULL)
-                GF_FREE (parent_disk_layout);
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
 
-        if (parent_layout != NULL)
-                dht_layout_unref (this, parent_layout);
-
-        return -1;
+    return -1;
 }
 
 int
-dht_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
-{
-        xlator_t       *subvol     = NULL;
-        int             op_errno   = -1;
-        int             i          = 0;
-        int             ret        = 0;
-        dht_local_t    *local      = NULL;
-        dht_conf_t     *conf       = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = EIO;
-                goto err;
-        }
-
-        /* Post remove-brick, the client layout may not be in sync with
-        * disk layout because of lack of lookup. Hence,a mknod call
-        * may fall on the decommissioned brick.  Hence, if the
-        * hashed_subvol is part of decommissioned bricks  list, do a
-        * lookup on parent dir. If a fix-layout is already done by the
-        * remove-brick process, the parent directory layout will be in
-        * sync with that of the disk. If fix-layout is still ending
-        * on the parent directory, we can let the file get created on
-        * the decommissioned brick which will be eventually migrated to
-        * non-decommissioned brick based on the new layout.
-        */
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *params)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int ret = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a mknod call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
 
-        if (conf->decommission_subvols_cnt) {
-            for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->decommissioned_bricks[i] &&
-                        conf->decommissioned_bricks[i] == subvol) {
-
-                        gf_msg_debug (this->name, 0, "hashed subvol:%s is "
-                                      "part of decommission brick list for "
-                                      "file: %s", subvol->name, loc->path);
-
-                        /* dht_refresh_layout needs directory info in
-                         * local->loc. Hence, storing the parent_loc in
-                         * local->loc and storing the create context in
-                         * local->loc2. We will restore this information
-                         * in dht_creation do */
-
-                        ret = loc_copy (&local->loc2, &local->loc);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "loc_copy failed %s", loc->path);
-
-                                goto err;
-                        }
+                    goto err;
+                }
 
-                        local->params = dict_ref (params);
-                        local->rdev = rdev;
-                        local->mode = mode;
-                        local->umask = umask;
+                local->params = dict_ref(params);
+                local->rdev = rdev;
+                local->mode = mode;
+                local->umask = umask;
 
-                        loc_wipe (&local->loc);
+                loc_wipe(&local->loc);
 
-                        ret = dht_build_parent_loc (this, &local->loc, loc,
-                                                                 &op_errno);
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
 
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "parent loc build failed");
-                                goto err;
-                        }
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
+                }
 
-                        ret = dht_mknod_lock (frame, subvol);
+                ret = dht_mknod_lock(frame, subvol);
 
-                        if (ret < 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_INODE_LK_ERROR,
-                                        "locking parent failed");
-                                goto err;
-                        }
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
+                }
 
-                        goto done;
-               }
+                goto done;
             }
         }
+    }
 
-        dht_mknod_wind_to_avail_subvol (frame, this, subvol, loc, rdev, mode,
-                                        umask, params);
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, loc, rdev, mode, umask,
+                                   params);
 
 done:
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *params)
 {
-        xlator_t    *subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SYMLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SYMLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = EIO;
-                goto err;
-        }
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->symlink, linkname, loc, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->symlink, linkname, loc, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-            dict_t *xdata)
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        xlator_t    *cached_subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    local = dht_local_init(frame, loc, NULL, GF_FOP_UNLINK);
+    if (!local) {
+        op_errno = ENOMEM;
 
-                goto err;
-        }
+        goto err;
+    }
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->flags = xflag;
-        STACK_WIND (frame, dht_unlink_cbk,
-                    cached_subvol, cached_subvol->fops->unlink, loc,
-                    xflag, xdata);
+    local->flags = xflag;
+    STACK_WIND_COOKIE(frame, dht_unlink_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->unlink, loc, xflag, xdata);
 
-        return 0;
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
+static int
+dht_remove_stale_linkto_cbk(int ret, call_frame_t *sync_frame, void *data)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
-        gf_boolean_t  stbuf_merged = _gf_false;
-        xlator_t     *subvol = NULL;
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
-        local = frame->local;
+static int
+dht_remove_stale_linkto(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata_in = NULL;
+    int ret = 0;
 
-        if (op_ret == -1) {
-                /* No continuation on DHT inode missing errors, as we should
-                 * then have a good stbuf that states P2 happened. We would
-                 * get inode missing if, the file completed migrated between
-                 * the lookup and the link call */
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("dht", data, out);
 
-        /* Update parent on success, even if P1/2 checks are positve.
-         * The second call on success will further update the parent */
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
-
-        /* Update linkto attrs, if this is the first call and non-P2,
-         * if we detect P2 then we need to trust the attrs from the
-         * second call, not the first */
-        if (local->linked == _gf_true &&
-            ((local->call_cnt == 1 && !IS_DHT_MIGRATION_PHASE2 (stbuf))
-             || (local->call_cnt != 1 &&
-                 IS_DHT_MIGRATION_PHASE2 (&local->stbuf)))) {
-                dht_iatt_merge (this, &local->stbuf, stbuf, NULL);
-                stbuf_merged = _gf_true;
-                dht_linkfile_attr_heal (frame, this);
-        }
-
-        /* No further P1/2 checks if we are in the second iteration of
-         * the call */
-        if (local->call_cnt != 1) {
-                goto out;
-        } else {
-                /* Preserve the return values, in case the migration decides
-                 * to recreate the link on the same subvol that the current
-                 * hased for the link was created on. */
-                dht_iatt_merge (this, &local->preparent,
-                                preparent, NULL);
-                dht_iatt_merge (this, &local->postparent,
-                                postparent, NULL);
-                if (!stbuf_merged) {
-                        dht_iatt_merge (this, &local->stbuf,
-                                        stbuf, NULL);
-                        stbuf_merged = _gf_true;
-                }
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
 
-                local->inode = inode_ref (inode);
-        }
+    xdata_in = dict_new();
+    if (!xdata_in)
+        goto out;
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
-        local->rebalance.target_op_fn = dht_link2;
-        dht_set_local_rebalance (this, local, stbuf, preparent,
-                                 postparent, xdata);
-
-        /* Check if the rebalance phase2 is true */
-        if (IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL,
-                                                  &subvol);
-                if (!subvol) {
-                        /* Phase 2 of migration */
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        dht_link2 (this, subvol, frame, 0);
-                        return 0;
-                }
-        }
+    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(xdata_in);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Failed to set keys for stale linkto"
+               "deletion on path %s",
+               local->loc.path);
+        goto out;
+    }
+
+    ret = syncop_unlink(local->link_subvol, &local->loc, xdata_in, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Removal of linkto failed"
+               " on path %s at subvol %s",
+               local->loc.path, local->link_subvol->name);
+    }
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
+    return ret;
+}
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
-                ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL,
-                                                  &subvol);
-                if (subvol) {
-                        dht_link2 (this, subvol, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+static int
+dht_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, inode_t *inode, struct iatt *stbuf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    gf_boolean_t stbuf_merged = _gf_false;
+    xlator_t *subvol = NULL;
+    call_frame_t *cleanup_frame = NULL;
+    dht_local_t *cleanup_local = NULL;
+
+    local = frame->local;
+
+    if (op_ret == -1) {
+        /* Remove the linkto if exists */
+        if (local->linked) {
+            cleanup_frame = create_frame(this, this->ctx->pool);
+            if (cleanup_frame) {
+                cleanup_local = dht_local_init(cleanup_frame, &local->loc2,
+                                               NULL, 0);
+                if (!cleanup_local || !local->link_subvol) {
+                    DHT_STACK_DESTROY(cleanup_frame);
+                    goto out;
+                }
+                cleanup_local->link_subvol = local->link_subvol;
+                FRAME_SU_DO(cleanup_frame, dht_local_t);
+                ret = synctask_new(this->ctx->env, dht_remove_stale_linkto,
+                                   dht_remove_stale_linkto_cbk, cleanup_frame,
+                                   cleanup_frame);
+            }
         }
+        /* No continuation on DHT inode missing errors, as we should
+         * then have a good stbuf that states P2 happened. We would
+         * get inode missing if, the file completed migrated between
+         * the lookup and the link call */
+        goto out;
+    }
+
+    /* Update parent on success, even if P1/2 checks are positive.
+     * The second call on success will further update the parent */
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    /* Update linkto attrs, if this is the first call and non-P2,
+     * if we detect P2 then we need to trust the attrs from the
+     * second call, not the first */
+    if (local->linked == _gf_true &&
+        ((local->call_cnt == 1 && !IS_DHT_MIGRATION_PHASE2(stbuf)) ||
+         (local->call_cnt != 1 && IS_DHT_MIGRATION_PHASE2(&local->stbuf)))) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        stbuf_merged = _gf_true;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    /* No further P1/2 checks if we are in the second iteration of
+     * the call */
+    if (local->call_cnt != 1) {
+        goto out;
+    } else {
+        /* Preserve the return values, in case the migration decides
+         * to recreate the link on the same subvol that the current
+         * hased for the link was created on. */
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+        if (!stbuf_merged) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            stbuf_merged = _gf_true;
+        }
+
+        local->inode = inode_ref(inode);
+    }
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_link2;
+    dht_set_local_rebalance(this, local, stbuf, preparent, postparent, xdata);
+
+    /* Check if the rebalance phase2 is true */
+    if (IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
+        if (!subvol) {
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
+        if (subvol) {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
-        dht_set_fixed_dir_stat (preparent);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf,
-                          preparent, postparent, NULL);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto err;
+    local = frame->local;
+    if (!local)
+        goto err;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                dht_set_fixed_dir_stat (&local->preparent);
-                dht_set_fixed_dir_stat (&local->postparent);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
 
-                DHT_STACK_UNWIND (link, frame, local->op_ret, op_errno,
-                                  local->inode,
-                                  &local->stbuf, &local->preparent,
-                                  &local->postparent, NULL);
-                return 0;
-        }
+        DHT_STACK_UNWIND(link, frame, local->op_ret, op_errno, local->inode,
+                         &local->stbuf, &local->preparent, &local->postparent,
+                         NULL);
+        return 0;
+    }
 
-        if (subvol == NULL) {
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (subvol == NULL) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        /* Second call to create link file could result in EEXIST as the
-         * first call created the linkto in the currently
-         * migrating subvol, which could be the new hashed subvol */
-        if (local->link_subvol == subvol) {
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->preparent);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (link, frame, 0, 0, local->inode,
-                                  &local->stbuf, &local->preparent,
-                                  &local->postparent, NULL);
+    /* Second call to create link file could result in EEXIST as the
+     * first call created the linkto in the currently
+     * migrating subvol, which could be the new hashed subvol */
+    if (local->link_subvol == subvol) {
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(link, frame, 0, 0, local->inode, &local->stbuf,
+                         &local->preparent, &local->postparent, NULL);
 
-                return 0;
-        }
+        return 0;
+    }
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        STACK_WIND (frame, dht_link_cbk, subvol, subvol->fops->link,
-                    &local->loc, &local->loc2, NULL);
+    STACK_WIND(frame, dht_link_cbk, subvol, subvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
 
-        return 0;
+    return 0;
 err:
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
+static int
+dht_link_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *srcvol = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *srcvol = NULL;
 
-        if (op_ret == -1)
-                goto err;
+    if (op_ret == -1)
+        goto err;
 
-        local = frame->local;
-        srcvol = local->linkfile.srcvol;
+    local = frame->local;
+    srcvol = local->linkfile.srcvol;
 
-        STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2, xdata);
+    STACK_WIND(frame, dht_link_cbk, srcvol, srcvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (preparent);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent, NULL);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
+{
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_LINK);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+    local->call_cnt = 1;
+
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     oldloc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, newloc);
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     newloc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (hashed_subvol != cached_subvol) {
+        gf_uuid_copy(local->gfid, oldloc->inode->gfid);
+        dht_linkfile_create(frame, dht_link_linkfile_cbk, this, cached_subvol,
+                            hashed_subvol, newloc);
+    } else {
+        STACK_WIND(frame, dht_link_cbk, cached_subvol,
+                   cached_subvol->fops->link, oldloc, newloc, xdata);
+    }
+
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    return 0;
+}
 
-                goto err;
-        }
-        local->call_cnt = 1;
+int
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    gf_boolean_t parent_layout_changed = _gf_false;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    xlator_t *subvol = NULL;
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", oldloc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+    local = frame->local;
 
-        hashed_subvol = dht_subvol_get_hashed (this, newloc);
-        if (!hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              newloc->path);
-                op_errno = EIO;
-                goto err;
-        }
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? _gf_true
+                                    : _gf_false;
+
+        if (parent_layout_changed) {
+            if (local && local->lock[0].layout.parent_layout.locks) {
+                /* Returning failure as the layout could not be fixed even under
+                 * the lock */
+                goto out;
+            }
 
-        if (hashed_subvol != cached_subvol) {
-                gf_uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
-                                     cached_subvol, hashed_subvol, newloc);
-        } else {
-                STACK_WIND (frame, dht_link_cbk,
-                            cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc, xdata);
-        }
+            gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "create (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a layout refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
 
-        return 0;
+            /*
+              dht_refresh_layout needs directory info in local->loc.Hence,
+              storing the parent_loc in local->loc and storing the create
+              context in local->loc2. We will restore this information in
+              dht_creation_do.
+             */
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+            loc_wipe(&local->loc2);
 
-        return 0;
-}
+            ret = loc_copy(&local->loc2, &local->loc);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "loc_copy failed %s", local->loc.path);
 
+                goto out;
+            }
 
-int
-dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
+            loc_wipe(&local->loc);
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+            ret = dht_build_parent_loc(this, &local->loc, &local->loc2,
+                                       &op_errno);
 
-        if (op_ret == -1)
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                       "parent loc build failed");
                 goto out;
+            }
 
-        prev = cookie;
+            subvol = dht_subvol_get_hashed(this, &local->loc2);
 
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
+            ret = dht_create_lock(frame, subvol);
+            if (ret < 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                       "locking parent failed");
+                goto out;
+            }
 
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
+            return 0;
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
-        if (ret != 0) {
-                gf_msg_debug (this->name, 0,
-                              "could not set preset layout for subvol %s",
-                              prev->this->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+        goto out;
+    }
+
+    prev = cookie;
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    ret = dht_fd_ctx_set(this, fd, prev);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0,
+                     "Possible fd leak. "
+                     "Could not set fd ctx for subvol %s",
+                     prev->name);
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "could not set preset layout for subvol %s",
+                     prev->name);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        local->op_errno = op_errno;
+    local->op_errno = op_errno;
 
-        if (local->linked == _gf_true) {
-                local->stbuf = *stbuf;
-                dht_linkfile_attr_heal (frame, this);
-        }
+    if (local->linked == _gf_true) {
+        local->stbuf = *stbuf;
+        dht_linkfile_attr_heal(frame, this);
+    }
 out:
 
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (preparent);
-        dht_set_fixed_dir_stat (postparent);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
 
-        if (local && local->lock.locks) {
-                /* store op_errno for failure case*/
-                local->op_errno = op_errno;
-                local->refresh_layout_unlock (frame, this, op_ret, 1);
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
 
-                if (op_ret == 0) {
-                        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
-                                          inode, stbuf, preparent, postparent,
-                                          xdata);
-                }
-        } else {
-                DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode,
-                                  stbuf, preparent, postparent, xdata);
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                             preparent, postparent, xdata);
         }
-        return 0;
+    } else {
+        DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
+    return 0;
 }
 
-int
-dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
+static int
+dht_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *stbuf,
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
 {
-        dht_local_t     *local             = NULL;
-        xlator_t        *cached_subvol     = NULL;
-        dht_conf_t      *conf              = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
 
-        local = frame->local;
-        if (!local) {
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        conf = this->private;
-        if (!conf) {
-                local->op_errno = EINVAL;
-                op_errno = EINVAL;
-                goto err;
-        }
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        cached_subvol = local->cached_subvol;
+    cached_subvol = local->cached_subvol;
 
-        if (local->params) {
-                dict_del (local->params, conf->link_xattr_name);
-                dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
 
-        STACK_WIND (frame, dht_create_cbk,
-                    cached_subvol, cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode,
-                    local->umask, local->fd, local->params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->create, &local->loc, local->flags,
+                      local->mode, local->umask, local->fd, local->params);
 
-        return 0;
+    return 0;
 err:
-        if (local && local->lock.locks) {
-                local->refresh_layout_unlock (frame, this, -1, 1);
-        } else {
-                DHT_STACK_UNWIND (create, frame, -1,
-                                  op_errno, NULL, NULL, NULL,
-                                  NULL, NULL, NULL);
-        }
-        return 0;
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    }
+    return 0;
 }
 
-int
-dht_create_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this,
-                                 xlator_t *subvol, loc_t *loc, int32_t flags,
-                                 mode_t mode, mode_t umask, fd_t *fd,
-                                 dict_t *params)
+static int
+dht_create_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                                xlator_t *subvol, loc_t *loc, int32_t flags,
+                                mode_t mode, mode_t umask, fd_t *fd,
+                                dict_t *params)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *avail_subvol   = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path,
-                              subvol->name);
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-                STACK_WIND (frame, dht_create_cbk,
-                            subvol, subvol->fops->create,
-                            loc, flags, mode, umask, fd, params);
+        dht_set_parent_layout_in_dict(loc, this, local);
 
-        } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
-
-                if (avail_subvol != subvol) {
-                        local->params = dict_ref (params);
-                        local->flags = flags;
-                        local->mode = mode;
-                        local->umask = umask;
-                        local->cached_subvol = avail_subvol;
-                        local->hashed_subvol = subvol;
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
 
-                        gf_msg_debug (this->name, 0,
-                                      "creating %s on %s (link at %s)", loc->path,
-                                      avail_subvol->name, subvol->name);
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-                        dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
-                                             this, avail_subvol, subvol, loc);
+        if (avail_subvol != subvol) {
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
 
-                        goto out;
-                }
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
 
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path, subvol->name);
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
 
-                STACK_WIND (frame, dht_create_cbk,
-                            subvol, subvol->fops->create,
-                            loc, flags, mode, umask, fd, params);
+            goto out;
         }
+
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
+
+        dht_set_parent_layout_in_dict(loc, this, local);
+
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
+    }
 out:
-        return 0;
+    return 0;
 }
 
 int
-dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child,
-                                                 int32_t *op_errno)
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno)
 {
-        inode_table_t   *table = NULL;
-        int     ret = -1;
+    inode_table_t *table = NULL;
+    int ret = -1;
 
-        if (!parent || !child) {
-                if (op_errno)
-                        *op_errno = EINVAL;
-                goto out;
-        }
+    if (!parent || !child) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
 
-        if (child->parent) {
-                parent->inode = inode_ref (child->parent);
-                if (!parent->inode) {
-                        if (op_errno)
-                                *op_errno = EINVAL;
-                        goto out;
-                }
+    if (child->parent) {
+        parent->inode = inode_ref(child->parent);
+        if (!parent->inode) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
+        }
 
-                gf_uuid_copy (parent->gfid, child->pargfid);
+        gf_uuid_copy(parent->gfid, child->pargfid);
 
-                ret = 0;
+        ret = 0;
 
-                goto out;
-        } else {
-                if (gf_uuid_is_null (child->pargfid)) {
-                        if (op_errno)
-                                *op_errno = EINVAL;
-                        goto out;
-                }
+        goto out;
+    } else {
+        if (gf_uuid_is_null(child->pargfid)) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
+        }
 
-                table = this->itable;
+        table = this->itable;
 
-                if (!table) {
-                        if (op_errno) {
-                                *op_errno = EINVAL;
-                                goto out;
-                        }
-                }
+        if (!table) {
+            if (op_errno) {
+                *op_errno = EINVAL;
+                goto out;
+            }
+        }
 
-                parent->inode = inode_find (table, child->pargfid);
+        parent->inode = inode_find(table, child->pargfid);
 
-                if (!parent->inode) {
-                         if (op_errno) {
-                                *op_errno = EINVAL;
-                                goto out;
-                        }
-                }
+        if (!parent->inode) {
+            if (op_errno) {
+                *op_errno = EINVAL;
+                goto out;
+            }
+        }
 
-                gf_uuid_copy (parent->gfid, child->pargfid);
+        gf_uuid_copy(parent->gfid, child->pargfid);
 
-                ret = 0;
-        }
+        ret = 0;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
-int32_t
-dht_create_do (call_frame_t *frame)
+static int32_t
+dht_create_do(call_frame_t *frame)
 {
-        dht_local_t     *local          = NULL;
-        dht_layout_t    *refreshed      = NULL;
-        xlator_t        *subvol         = NULL;
-        xlator_t        *this           = NULL;
-        dht_conf_t      *conf           = NULL;
-        dht_methods_t   *methods        = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        this = THIS;
+    this = THIS;
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_VALIDATE_OR_GOTO (this->name, conf, err);
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
 
-        methods = &(conf->methods);
+    methods = &(conf->methods);
 
-        /* We don't need parent_loc anymore */
-        loc_wipe (&local->loc);
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
 
-        loc_copy (&local->loc, &local->loc2);
+    loc_copy(&local->loc, &local->loc2);
 
-        loc_wipe (&local->loc2);
+    loc_wipe(&local->loc2);
 
-        refreshed = local->selfheal.refreshed_layout;
+    refreshed = local->selfheal.refreshed_layout;
 
-        subvol = methods->layout_search (this, refreshed, local->loc.name);
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
 
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "no subvolume in "
-                        "layout for path=%s", local->loc.path);
-                local->op_errno = ENOENT;
-                goto err;
-        }
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
 
-        dht_create_wind_to_avail_subvol (frame, this, subvol, &local->loc,
-                                         local->flags, local->mode,
-                                         local->umask, local->fd, local->params);
-        return 0;
+    dht_create_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                    local->flags, local->mode, local->umask,
+                                    local->fd, local->params);
+    return 0;
 err:
-        local->refresh_layout_unlock (frame, this, -1, 1);
+    local->refresh_layout_unlock(frame, this, -1, 1);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-dht_create_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_create_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
-int32_t
-dht_create_finish (call_frame_t *frame, xlator_t *this, int op_ret,
-                   int invoke_cbk)
+static int32_t
+dht_create_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                  int invoke_cbk)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
-
-        local = frame->local;
-        lock_count = dht_lock_count (local->lock.locks, local->lock.lk_count);
-        if (lock_count == 0)
-                goto done;
-
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                goto done;
-        }
-
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL) {
-                goto done;
-        }
-
-        lock_local->lock.locks = local->lock.locks;
-        lock_local->lock.lk_count = local->lock.lk_count;
-
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
-
-        dht_unlock_inodelk (lock_frame, lock_local->lock.locks,
-                            lock_local->lock.lk_count,
-                            dht_create_unlock_cbk);
-        lock_frame = NULL;
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
+
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
+
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
+
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_create_unlock_cbk);
+    lock_frame = NULL;
 
 done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-        if (op_ret == 0)
-                return 0;
-
-        DHT_STACK_UNWIND (create, frame, op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL, NULL);
+    if (op_ret == 0)
         return 0;
+
+    DHT_STACK_UNWIND(create, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-dht_create_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_create_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                goto err;
-        }
+    if (!local) {
+        goto err;
+    }
 
-        if (op_ret < 0) {
-                gf_msg ("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
-                        "Create lock failed for file: %s", local->loc2.name);
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "Create lock failed for file: %s", local->loc2.name);
 
-                local->op_errno = op_errno;
+        local->op_errno = op_errno;
 
-                goto err;
-        }
+        goto err;
+    }
 
-        local->refresh_layout_unlock = dht_create_finish;
+    local->refresh_layout_unlock = dht_create_finish;
 
-        local->refresh_layout_done = dht_create_do;
+    local->refresh_layout_done = dht_create_do;
 
-        dht_refresh_layout (frame);
+    dht_refresh_layout(frame);
 
-        return 0;
+    return 0;
 err:
-        dht_create_finish (frame, this, -1, 0);
-        return 0;
+    if (local)
+        dht_create_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    return 0;
 }
 
 int32_t
-dht_create_lock (call_frame_t *frame, xlator_t *subvol)
+dht_create_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        dht_local_t     *local          = NULL;
-        int              count  = 1,    ret = -1;
-        dht_lock_t     **lk_array       = NULL;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        local = frame->local;
+    local = frame->local;
 
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-        if (lk_array == NULL)
-                goto err;
+    if (lk_array == NULL)
+        goto err;
 
-        lk_array[0] = dht_lock_new (frame->this, subvol, &local->loc, F_RDLCK,
-                                    DHT_LAYOUT_HEAL_DOMAIN);
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
 
-        if (lk_array[0] == NULL)
-                goto err;
+    if (lk_array[0] == NULL)
+        goto err;
 
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    IGNORE_ENOENT_ESTALE, dht_create_lock_cbk);
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_create_lock_cbk);
 
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                goto err;
-        }
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
 
-        return 0;
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
 int
-dht_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
-{
-        int             op_errno           = -1;
-        xlator_t       *subvol             = NULL;
-        dht_local_t    *local              = NULL;
-        int             i                  = 0;
-        dht_conf_t     *conf               = NULL;
-        int             ret                = 0;
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local)
+{
+    dht_conf_t *conf = this->private;
+    dht_layout_t *parent_layout = NULL;
+    int *parent_disk_layout = NULL;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    parent_layout = dht_layout_get(this, loc->parent);
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    ret = dict_set_str_sizen(local->params, GF_PREOP_PARENT_KEY,
+                             conf->xattr_name);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+err:
+    dht_layout_unref(this, parent_layout);
+    return ret;
+}
 
-        conf = this->private;
+int
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
+{
+    int op_errno = -1;
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->params = dict_ref(params);
+    local->flags = flags;
+    local->mode = mode;
+    local->umask = umask;
+
+    if (dht_filter_loc_subvol_key(this, loc, &local->loc, &subvol)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "creating %s on %s (got create on %s)", local->loc.path,
+               subvol->name, loc->path);
+
+        /* Since lookup-optimize is enabled by default, we need
+         * to create the linkto file if required.
+         * Note this does not check for decommisioned bricks
+         * and min-free-disk limits as this is a debugging tool
+         * and not expected to be used in production.
+         */
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-        dht_get_du_info (frame, this, loc);
+        if (hashed_subvol && (hashed_subvol != subvol)) {
+            /* Create the linkto file and then the data file */
+            local->cached_subvol = subvol;
+            local->hashed_subvol = hashed_subvol;
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                subvol, hashed_subvol, &local->loc);
+            goto done;
         }
+        /* We either don't have a hashed subvol or the hashed subvol is
+         * the same as the one specified. No need to create the linkto
+         * file as we expect a lookup everywhere if there are problems
+         * with the parent layout
+         */
 
-        if (dht_filter_loc_subvol_key (this, loc, &local->loc,
-                                       &subvol)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO,
-                        "creating %s on %s (got create on %s)",
-                        local->loc.path, subvol->name, loc->path);
-                STACK_WIND (frame, dht_create_cbk,
-                            subvol, subvol->fops->create,
-                            &local->loc, flags, mode, umask, fd, params);
-                goto done;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
-
-                op_errno = EIO;
-                goto err;
-        }
-
-       /* Post remove-brick, the client layout may not be in sync with
-        * disk layout because of lack of lookup. Hence,a create call
-        * may fall on the decommissioned brick.  Hence, if the
-        * hashed_subvol is part of decommissioned bricks  list, do a
-        * lookup on parent dir. If a fix-layout is already done by the
-        * remove-brick process, the parent directory layout will be in
-        * sync with that of the disk. If fix-layout is still ending
-        * on the parent directory, we can let the file get created on
-        * the decommissioned brick which will be eventually migrated to
-        * non-decommissioned brick based on the new layout.
-        */
-
-        if (conf->decommission_subvols_cnt) {
-            for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->decommissioned_bricks[i] &&
-                        conf->decommissioned_bricks[i] == subvol) {
-
-                        gf_msg_debug (this->name, 0, "hashed subvol:%s is "
-                                      "part of decommission brick list for "
-                                      "file: %s", subvol->name, loc->path);
-
-                        /* dht_refresh_layout needs directory info in
-                         * local->loc. Hence, storing the parent_loc in
-                         * local->loc and storing the create context in
-                         * local->loc2. We will restore this information
-                         * in dht_creation do */
-
-                        ret = loc_copy (&local->loc2, &local->loc);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "loc_copy failed %s", loc->path);
-
-                                goto err;
-                        }
+        dht_set_parent_layout_in_dict(loc, this, local);
+
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, &local->loc, flags, mode, umask,
+                          fd, params);
+        goto done;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in layout for path=%s", loc->path);
+
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a create call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
 
-                        local->params = dict_ref (params);
-                        local->flags = flags;
-                        local->mode = mode;
-                        local->umask = umask;
+                    goto err;
+                }
 
-                        loc_wipe (&local->loc);
+                loc_wipe(&local->loc);
 
-                        ret = dht_build_parent_loc (this, &local->loc, loc,
-                                                                 &op_errno);
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
 
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "parent loc build failed");
-                                goto err;
-                        }
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
+                }
 
-                        ret = dht_create_lock (frame, subvol);
+                ret = dht_create_lock(frame, subvol);
 
-                        if (ret < 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_INODE_LK_ERROR,
-                                        "locking parent failed");
-                                goto err;
-                        }
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
+                }
 
-                        goto done;
-               }
+                goto done;
             }
         }
+    }
 
-
-        dht_create_wind_to_avail_subvol (frame, this, subvol, loc, flags, mode,
-                                         umask, fd, params);
+    dht_create_wind_to_avail_subvol(frame, this, subvol, loc, flags, mode,
+                                    umask, fd, params);
 done:
-        return 0;
+    return 0;
 
 err:
 
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_mkdir_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
-        local = frame->local;
-        layout = local->selfheal.layout;
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
+    FRAME_SU_UNDO(frame, dht_local_t);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        if (op_ret == 0) {
-                dht_layout_set (this, local->inode, layout);
+    if (op_ret == 0) {
+        dht_layout_set(this, local->inode, layout);
 
-                dht_inode_ctx_time_update (local->inode, this,
-                                           &local->stbuf, 1);
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
+        dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
 
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
+    }
 
-        DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                          local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent, NULL);
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, local->inode,
+                     &local->stbuf, &local->preparent, &local->postparent,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        int           ret = -1;
-        gf_boolean_t subvol_filled = _gf_false;
-        gf_boolean_t dir_exists = _gf_false;
-        call_frame_t *prev = NULL;
-        dht_layout_t *layout = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
-
-        subvol_filled = dht_is_subvol_filled (this, prev->this);
-
-        LOCK (&frame->lock);
-        {
-                if (subvol_filled && (op_ret != -1)) {
-                        ret = dht_layout_merge (this, layout, prev->this,
-                                                -1, ENOSPC, NULL);
-                } else {
-                        if (op_ret == -1 && op_errno == EEXIST) {
-                                /* Very likely just a race between mkdir and
-                                   self-heal (from lookup of a concurrent mkdir
-                                   attempt).
-                                   Ignore error for now. layout setting will
-                                   anyways fail if this was a different (old)
-                                   pre-existing different directory.
-                                */
-                                op_ret = 0;
-                                dir_exists = _gf_true;
-                        }
-                        ret = dht_layout_merge (this, layout, prev->this,
-                                                op_ret, op_errno, NULL);
-                }
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_MERGE_FAILED,
-                                "%s: failed to merge layouts for subvol %s",
-                                local->loc.path, prev->this->name);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-
-                if (dir_exists)
-                        goto unlock;
-
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+static int
+dht_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = -1;
+    gf_boolean_t subvol_filled = _gf_false;
+    gf_boolean_t dir_exists = _gf_false;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+
+    subvol_filled = dht_is_subvol_filled(this, prev);
+
+    LOCK(&frame->lock);
+    {
+        if (subvol_filled && (op_ret != -1)) {
+            ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+        } else {
+            if (op_ret == -1 && op_errno == EEXIST) {
+                /* Very likely just a race between mkdir and
+                   self-heal (from lookup of a concurrent mkdir
+                   attempt).
+                   Ignore error for now. layout setting will
+                   anyways fail if this was a different (old)
+                   pre-existing different directory.
+                */
+                op_ret = 0;
+                dir_exists = _gf_true;
+            }
+            ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
         }
-unlock:
-        UNLOCK (&frame->lock);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
-                                            layout);
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
 
-        return 0;
-}
+        if (dir_exists)
+            goto unlock;
 
-int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-int
-dht_mkdir_helper (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
-{
-        dht_local_t  *local                   = NULL;
-        dht_conf_t   *conf                    = NULL;
-        int           op_errno                = -1, ret = -1;
-        xlator_t     *hashed_subvol           = NULL;
-        int32_t      *parent_disk_layout      = NULL;
-        dht_layout_t *parent_layout           = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
-
-        conf = this->private;
-        local = frame->local;
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /*Unlock entrylk and inodelk once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_new_directory(frame, dht_mkdir_selfheal_cbk, layout);
+    }
 
-        if (local->op_ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): refreshing parent layout "
-                        "failed.", pgfid, loc->name,
-                        loc->path);
+    return 0;
+}
 
-                op_errno = local->op_errno;
-                goto err;
-        }
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
 
-        local->op_ret = -1;
+static int
+dht_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                 mode_t umask, dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+    local = frame->local;
+
+    if (local->op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): refreshing parent layout "
+               "failed.",
+               pgfid, loc->name, loc->path);
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                gf_msg_debug (this->name, 0,
-                              "mkdir (%s/%s) (path: %s): hashed subvol not "
-                              "found", pgfid, loc->name, loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    local->op_ret = -1;
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "mkdir (%s/%s) (path: %s): hashed subvol not "
+                     "found",
+                     pgfid, loc->name, loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    if (memcmp(local->parent_disk_layout, parent_disk_layout,
+               sizeof(local->parent_disk_layout)) == 0) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): loop detected. "
+               "parent layout didn't change even though "
+               "previous attempt of mkdir failed because of "
+               "in-memory layout not matching with that on disk.",
+               pgfid, loc->name, loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               pgfid, loc->name, loc->path, GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(params, conf->xattr_name, parent_disk_layout, 4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "setting parent-layout in params dictionary failed. "
+               "mkdir (%s/%s) (path: %s)",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->mkdir, loc, mode, umask, params);
+
+    return 0;
 
-        local->hashed_subvol = hashed_subvol;
+err:
+    dht_unlock_namespace(frame, &local->lock[0]);
 
-        parent_layout = dht_layout_get (this, loc->parent);
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        ret = dht_disk_layout_extract_for_subvol (this, parent_layout,
-                                                  hashed_subvol,
-                                                  &parent_disk_layout);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, EIO,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "extracting in-memory layout of parent failed. ",
-                        pgfid, loc->name, loc->path);
-                goto err;
-        }
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
 
-        if (memcmp (local->parent_disk_layout, parent_disk_layout,
-                    sizeof (local->parent_disk_layout)) == 0) {
-                gf_msg (this->name, GF_LOG_WARNING, EIO,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): loop detected. "
-                        "parent layout didn't change even though "
-                        "previous attempt of mkdir failed because of "
-                        "in-memory layout not matching with that on disk.",
-                        pgfid, loc->name, loc->path);
-                op_errno = EIO;
-                goto err;
-        }
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
 
-        memcpy ((void *)local->parent_disk_layout, (void *)parent_disk_layout,
-                sizeof (local->parent_disk_layout));
+    return 0;
+}
 
-        dht_layout_unref (this, parent_layout);
-        parent_layout = NULL;
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t parent_layout_changed = _gf_false;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+    conf = this->private;
+    hashed_subvol = local->hashed_subvol;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    if (gf_uuid_is_null(local->loc.gfid) && !op_ret)
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
 
-        ret = dict_set_str (params, GF_PREOP_PARENT_KEY, conf->xattr_name);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "setting %s key in params dictionary failed. ",
-                        pgfid, loc->name, loc->path, GF_PREOP_PARENT_KEY);
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? 1
+                                    : 0;
+        if (parent_layout_changed) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "mkdir (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
+
+            stub = fop_mkdir_stub(frame, dht_mkdir_helper, &local->loc,
+                                  local->mode, local->umask, local->params);
+            if (stub == NULL) {
                 goto err;
-        }
+            }
 
-        ret = dict_set_bin (params, conf->xattr_name, parent_disk_layout,
-                            4 * 4);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "setting parent-layout in params dictionary failed. "
-                        "mkdir (%s/%s) (path: %s)", pgfid, loc->name,
-                        loc->path);
+            ret = dht_handle_parent_layout_change(this, stub);
+            if (ret) {
                 goto err;
-        }
-
-        parent_disk_layout = NULL;
-
-        STACK_WIND (frame, dht_mkdir_hashed_cbk,
-                    hashed_subvol,
-                    hashed_subvol->fops->mkdir,
-                    loc, mode, umask, params);
-
-        return 0;
+            }
 
+            stub = NULL;
+
+            return 0;
+        }
+
+        goto err;
+    }
+
+    dict_del(local->params, GF_PREOP_PARENT_KEY);
+    dict_del(local->params, conf->xattr_name);
+
+    if (dht_is_subvol_filled(this, hashed_subvol))
+        ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+    else
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
+
+    /* TODO: we may have to return from the function
+       if layout merge fails. For now, lets just log an error */
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+               "%s: failed to merge layouts for subvol %s", local->loc.path,
+               prev->name);
+
+    local->op_ret = 0;
+
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    /* Delete internal mds xattr from params dict to avoid store
+      internal mds xattr on other subvols
+    */
+    dict_del(local->params, conf->mds_xattr_key);
+
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    /* Set hashed subvol as a mds subvol on inode ctx */
+    /*if (!local->inode)
+            local->inode  = inode_ref (inode);
+    */
+    ret = dht_inode_ctx_mdsvol_set(local->inode, this, hashed_subvol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+               "Failed to set hashed subvol for %s on inode vol is %s",
+               local->loc.path, hashed_subvol->name);
+    }
+
+    if (local->call_cnt == 0) {
+        /*Unlock namespace lock once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_directory(frame, dht_mkdir_selfheal_cbk, &local->loc,
+                               layout);
+        return 0;
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == hashed_subvol)
+            continue;
+        STACK_WIND_COOKIE(frame, dht_mkdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->mkdir,
+                          &local->loc, local->mode, local->umask,
+                          local->params);
+    }
+
+    return 0;
 err:
-        dht_unlock_parent_layout_during_entry_fop (frame);
+    if (local->op_ret != 0) {
+        dht_unlock_namespace(frame, &local->lock[0]);
+    }
 
-        op_errno = local ? local->op_errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        if (parent_disk_layout != NULL)
-                GF_FREE (parent_disk_layout);
+    return 0;
+}
 
-        if (parent_layout != NULL)
-                dht_layout_unref (this, parent_layout);
+static int
+dht_mkdir_guard_parent_layout_cbk(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, mode_t mode, mode_t umask,
+                                  dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = -1;
+    int32_t zero[1] = {0};
+
+    local = frame->local;
+    conf = this->private;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "Acquiring lock on parent to guard against "
+               "layout-change failed.",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    local->op_ret = -1;
+    /* Add internal MDS xattr on disk for hashed subvol
+     */
+    ret = dht_dict_set_array(params, conf->mds_xattr_key, zero, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->mkdir,
+                      loc, mode, umask, params);
+
+    return 0;
+err:
+    DHT_STACK_UNWIND(mkdir, frame, -1, local->op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
-{
-        dht_local_t  *local                   = NULL;
-        int           ret                     = -1;
-        call_frame_t *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        dht_conf_t   *conf                    = NULL;
-        int           i                       = 0;
-        xlator_t     *hashed_subvol           = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        gf_boolean_t  parent_layout_changed   = _gf_false;
-        call_stub_t  *stub                    = NULL;
-
-        VALIDATE_OR_GOTO (this->private, err);
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = EINVAL, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    call_stub_t *stub = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+
+    if (!params || !dict_get(params, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         DHT_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, params);
+        goto err;
+    }
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0, "hashed subvol not found for %s",
+                     loc->path);
+        local->op_errno = EIO;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+    local->mode = mode;
+    local->umask = umask;
+    if (params)
+        local->params = dict_ref(params);
+
+    local->inode = inode_ref(loc->inode);
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* set the newly created directory hash to the commit hash
+     * if the configuration option is set. If configuration option
+     * is not set, the older clients may still be connecting to the
+     * volume and hence we need to preserve the 1 in disk[0] part of the
+     * layout xattr */
+    if (conf->lookup_optimize)
+        local->layout->commit_hash = conf->vol_commit_hash;
+    else
+        local->layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+
+    stub = fop_mkdir_stub(frame, dht_mkdir_guard_parent_layout_cbk, loc, mode,
+                          umask, params);
+    if (stub == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "creating stub failed.",
+               pgfid, loc->name, loc->path);
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_guard_parent_layout_and_namespace(this, stub);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s) cannot wind lock request to "
+               "guard parent layout",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    return 0;
 
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
-        conf = this->private;
-        hashed_subvol = local->hashed_subvol;
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        gf_uuid_unparse (local->loc.parent->gfid, pgfid);
+    return 0;
+}
 
-        if (gf_uuid_is_null (local->loc.gfid) && !op_ret)
-                gf_uuid_copy (local->loc.gfid, stbuf->ia_gfid);
+static int
+dht_rmdir_selfheal_cbk(call_frame_t *heal_frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
+    heal_local = heal_frame->local;
+    main_frame = heal_local->main_frame;
+    local = main_frame->local;
 
-                parent_layout_changed = (xdata && dict_get (xdata, GF_PREOP_CHECK_FAILED))
-                        ? 1 : 0;
-                if (parent_layout_changed) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_PARENT_LAYOUT_CHANGED,
-                                "mkdir (%s/%s) (path: %s): parent layout "
-                                "changed. Attempting a refresh and then a "
-                                "retry", pgfid, local->loc.name,
-                                local->loc.path);
-
-                        stub = fop_mkdir_stub (frame, dht_mkdir_helper,
-                                               &local->loc, local->mode,
-                                               local->umask, local->params);
-                        if (stub == NULL) {
-                                goto err;
-                        }
+    DHT_STACK_DESTROY(heal_frame);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-                        dht_handle_parent_layout_change (this, stub);
-                        stub = NULL;
+    DHT_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
 
-                        return 0;
+    return 0;
+}
+
+static int
+dht_rmdir_hashed_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            local->op_ret = -1;
+            if (conf->subvolume_cnt != 1) {
+                if (op_errno != ENOENT && op_errno != EACCES &&
+                    op_errno != ESTALE) {
+                    local->need_selfheal = 1;
                 }
+            }
 
-                goto err;
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed "
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
         }
 
-        dht_unlock_parent_layout_during_entry_fop (frame);
-        dict_del (local->params, GF_PREOP_PARENT_KEY);
-        dict_del (local->params, conf->xattr_name);
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (dht_is_subvol_filled (this, hashed_subvol))
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        -1, ENOSPC, NULL);
-        else
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, NULL);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->need_selfheal) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-        /* TODO: we may have to return from the function
-           if layout merge fails. For now, lets just log an error */
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "%s: failed to merge layouts for subvol %s",
-                        local->loc.path, prev->this->name);
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
 
-        local->op_ret = 0;
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
 
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
+            /* Use a different frame or else the rmdir op_ret is
+             * overwritten by that of the selfheal */
 
-        local->call_cnt = conf->subvolume_cnt - 1;
+            heal_frame = copy_frame(frame);
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, stbuf->ia_gfid);
-        if (local->call_cnt == 0) {
-                dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
-                                        &local->loc, layout);
-        }
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == hashed_subvol)
-                        continue;
-                STACK_WIND (frame, dht_mkdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->mkdir, &local->loc,
-                            local->mode, local->umask, local->params);
-        }
-        return 0;
-err:
-        if (local->op_ret != 0)
-                dht_unlock_parent_layout_during_entry_fop (frame);
+            if (heal_frame == NULL) {
+                goto err;
+            }
 
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
-        if (stub) {
-                call_stub_destroy (stub);
-        }
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
 
-        return 0;
-}
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
 
-int
-dht_mkdir_guard_parent_layout_cbk (call_frame_t *frame, xlator_t *this,
-                                   loc_t *loc, mode_t mode, mode_t umask,
-                                   dict_t *params)
-{
-        dht_local_t *local                    = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
+            dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                 &heal_local->loc, heal_local->layout);
+            return 0;
+        } else {
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
 
-        local = frame->local;
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
 
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
 
-        if (local->op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "Acquiring lock on parent to guard against "
-                        "layout-change failed.", pgfid, loc->name, loc->path);
-                goto err;
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
+    }
 
-        local->op_ret = -1;
-
-        STACK_WIND (frame, dht_mkdir_hashed_cbk,
-                    local->hashed_subvol,
-                    local->hashed_subvol->fops->mkdir,
-                    loc, mode, umask, params);
+    return 0;
 
-        return 0;
 err:
-        DHT_STACK_UNWIND (mkdir, frame, -1, local->op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL);
+    return 0;
+}
 
-        return 0;
+static int
+dht_rmdir_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
-int
-dht_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
-{
-        dht_local_t  *local                   = NULL;
-        dht_conf_t   *conf                    = NULL;
-        int           op_errno                = -1, ret = -1;
-        xlator_t     *hashed_subvol           = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        call_stub_t  *stub                    = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
-
-        conf = this->private;
-
-        if (!params || !dict_get (params, "gfid-req")) {
-                op_errno = EPERM;
-                gf_msg_callingfn (this->name, GF_LOG_WARNING, op_errno,
-                                  DHT_MSG_GFID_NULL, "mkdir: %s is received "
-                                  "without gfid-req %p", loc->path, params);
-                goto err;
-        }
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        dht_get_du_info (frame, this, loc);
+    local = frame->local;
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                gf_msg_debug (this->name, 0,
-                              "hashed subvol not found for %s",
-                              loc->path);
-                local->op_errno = EIO;
-                goto err;
-        }
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
 
+    if (lock_count == 0)
+        goto done;
 
-        local->hashed_subvol = hashed_subvol;
-        local->mode = mode;
-        local->umask = umask;
-        if (params)
-                local->params = dict_ref (params);
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL)
+        goto done;
 
-        local->inode  = inode_ref (loc->inode);
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL)
+        goto done;
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
-        if (!local->layout) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
 
-        /* set the newly created directory hash to the commit hash
-         * if the configuration option is set. If configuration option
-         * is not set, the older clients may still be connecting to the
-         * volume and hence we need to preserve the 1 in disk[0] part of the
-         * layout xattr */
-        if (conf->lookup_optimize)
-                local->layout->commit_hash = conf->vol_commit_hash;
-        else
-                local->layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_rmdir_unlock_cbk);
+    lock_frame = NULL;
 
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-        stub = fop_mkdir_stub (frame, dht_mkdir_guard_parent_layout_cbk, loc,
-                               mode, umask, params);
-        if (stub == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "creating stub failed.", pgfid, loc->name, loc->path);
-                local->op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
+}
 
-        ret = dht_guard_parent_layout_during_entry_fop (this, stub);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s) cannot wind lock request to "
-                        "guard parent layout", pgfid, loc->name, loc->path);
-                goto err;
-        }
+static int
+dht_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int done = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    int ret = -1;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
 
-        return 0;
+                if (op_errno != EACCES)
+                    local->need_selfheal = 1;
+            }
 
-err:
-        op_errno = local ? local->op_errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-        return 0;
-}
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed."
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
+        }
 
+        /* Track if rmdir succeeded on at least one subvol*/
+        local->fop_succeeded = 1;
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-int
-dht_rmdir_selfheal_cbk (call_frame_t *heal_frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        dht_local_t  *heal_local = NULL;
-        call_frame_t *main_frame = NULL;
+    this_call_cnt = dht_frame_return(frame);
 
-        heal_local = heal_frame->local;
-        main_frame = heal_local->main_frame;
-        local = main_frame->local;
+    /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+    if (local->hashed_subvol && (this_call_cnt == 1)) {
+        done = 1;
+    } else if (!local->hashed_subvol && !this_call_cnt) {
+        done = 1;
+    }
 
-        DHT_STACK_DESTROY (heal_frame);
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
+    if (done) {
+        if (local->need_selfheal && local->fop_succeeded) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-        DHT_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
 
-        return 0;
-}
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
+            heal_frame = copy_frame(frame);
+            if (heal_frame == NULL) {
+                goto err;
+            }
 
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
 
-int
-dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int op_ret, int op_errno, struct iatt *preparent,
-                             struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        dht_local_t  *heal_local = NULL;
-        call_frame_t *heal_frame = NULL;
-        dht_conf_t   *conf = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
+            ret = dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                       &heal_local->loc, heal_local->layout);
+            if (ret) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
 
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
+        } else if (this_call_cnt) {
+            /* If non-hashed subvol's have responded, proceed */
+            if (local->op_ret == 0) {
+                /* Delete the dir from the hashed subvol if:
+                 * The fop succeeded on at least one subvol
+                 *  and did not fail on any
+                 *  or
+                 *  The fop failed with ENOENT/ESTALE on
+                 *  all subvols */
+
+                STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                                  local->hashed_subvol, local->hashed_subvol,
+                                  local->hashed_subvol->fops->rmdir,
+                                  &local->loc, local->flags, NULL);
+            } else {
+                /* hashed-subvol was non-NULL and rmdir failed on
+                 * all non hashed-subvols. Unwind rmdir with
+                 * local->op_ret and local->op_errno. */
+                dht_rmdir_unlock(frame, this);
+                DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                                 &local->preparent, &local->postparent, NULL);
 
-        gf_uuid_unparse(local->loc.gfid, gfid);
+                return 0;
+            }
+        } else if (!this_call_cnt) {
+            /* All subvol's have responded, proceed */
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret   = -1;
-                        if (conf->subvolume_cnt != 1) {
-                                if (op_errno != ENOENT && op_errno != EACCES
-                                    && op_errno != ESTALE) {
-                                        local->need_selfheal = 1;
-                                }
-                        }
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
 
-                        gf_msg_debug (this->name, op_errno,
-                                      "rmdir on %s for %s failed "
-                                      "(gfid = %s)",
-                                      prev->this->name, local->loc.path,
-                                      gfid);
-                        goto unlock;
-                }
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
 
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
 
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-               if (local->need_selfheal) {
-                        dht_rmdir_unlock (frame, this);
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
+    }
 
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
+    return 0;
 
-                        gf_uuid_copy (local->gfid, local->loc.inode->gfid);
+err:
+    DHT_STACK_UNWIND(rmdir, frame, -1, local->op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
-                        /* Use a different frame or else the rmdir op_ret is
-                         * overwritten by that of the selfheal */
+static int
+dht_rmdir_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol;
 
-                        heal_frame = copy_frame (frame);
+    conf = this->private;
+    local = frame->local;
 
-                        if (heal_frame == NULL) {
-                                goto err;
-                        }
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed rmdir for %s)",
+               local->loc.path);
 
-                        heal_local = dht_local_init (heal_frame,
-                                                     &local->loc,
-                                                     NULL, 0);
-                        if (!heal_local) {
-                                DHT_STACK_DESTROY (heal_frame);
-                                goto err;
-                        }
-
-                        heal_local->inode = inode_ref (local->loc.inode);
-                        heal_local->main_frame = frame;
-                        gf_uuid_copy (heal_local->gfid, local->loc.inode->gfid);
-
-                        dht_selfheal_restore (heal_frame,
-                                              dht_rmdir_selfheal_cbk,
-                                              &heal_local->loc,
-                                              heal_local->layout);
-                        return 0;
-               } else {
-
-                        if (local->loc.parent) {
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->preparent,
-                                                           0);
-
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->postparent,
-                                                           1);
-                        }
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-                        dht_set_fixed_dir_stat (&local->preparent);
-                        dht_set_fixed_dir_stat (&local->postparent);
+    hashed_subvol = local->hashed_subvol;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (hashed_subvol && (hashed_subvol == conf->subvolumes[i]))
+            continue;
 
-                        dht_rmdir_unlock (frame, this);
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-               }
-        }
+        STACK_WIND_COOKIE(frame, dht_rmdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->rmdir,
+                          &local->loc, local->flags, NULL);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                          local->op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
 
+    return 0;
 }
 
+static int
+dht_rmdir_do(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-int
-dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-        int           done = 0;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
-        dht_local_t  *heal_local = NULL;
-        call_frame_t *heal_frame = NULL;
-        int           ret        = -1;
-
-        local = frame->local;
-        prev  = cookie;
-
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
-                                local->op_errno = op_errno;
-                                local->op_ret = -1;
-
-                                if (op_errno != EACCES)
-                                        local->need_selfheal = 1;
-                        }
-
-                        gf_uuid_unparse(local->loc.gfid, gfid);
+    VALIDATE_OR_GOTO(frame->local, err);
+    local = frame->local;
+    VALIDATE_OR_GOTO(this->private, out);
+    conf = this->private;
 
-                        gf_msg_debug (this->name, op_errno,
-                                      "rmdir on %s for %s failed."
-                                      "(gfid = %s)",
-                                      prev->this->name, local->loc.path,
-                                      gfid);
-                        goto unlock;
-                }
+    if (local->op_ret == -1)
+        goto out;
 
-                /* Track if rmdir succeeded on atleast one subvol*/
-                local->fop_succeeded = 1;
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    local->call_cnt = conf->subvolume_cnt;
 
+    /* first remove from non-hashed_subvol */
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-        this_call_cnt = dht_frame_return (frame);
+    if (!hashed_subvol) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
-        if (local->hashed_subvol && (this_call_cnt == 1)) {
-                done = 1;
-        } else if (!local->hashed_subvol && !this_call_cnt) {
-                done = 1;
-        }
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s (gfid = %s)",
+               local->loc.path, gfid);
+    } else {
+        local->hashed_subvol = hashed_subvol;
+    }
 
+    /* When DHT has only 1 child */
+    if (conf->subvolume_cnt == 1) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                          conf->subvolumes[0], conf->subvolumes[0],
+                          conf->subvolumes[0]->fops->rmdir, &local->loc,
+                          local->flags, NULL);
+        return 0;
+    }
 
-        if (done) {
-                if (local->need_selfheal && local->fop_succeeded) {
-                        dht_rmdir_unlock (frame, this);
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, &local->loc, local->hashed_subvol,
+                                &local->current->ns, dht_rmdir_lock_cbk);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = errno ? errno : EINVAL;
+        goto out;
+    }
 
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
+    return 0;
 
-                        gf_uuid_copy (local->gfid, local->loc.inode->gfid);
-                        heal_frame = copy_frame (frame);
-                        if (heal_frame == NULL) {
-                                goto err;
-                        }
-
-                        heal_local = dht_local_init (heal_frame, &local->loc,
-                                                     NULL, 0);
-                        if (!heal_local) {
-                                DHT_STACK_DESTROY (heal_frame);
-                                goto err;
-                        }
+out:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-                        heal_local->inode = inode_ref (local->loc.inode);
-                        heal_local->main_frame = frame;
-                        gf_uuid_copy (heal_local->gfid, local->loc.inode->gfid);
-                        ret = dht_selfheal_restore (heal_frame,
-                                                    dht_rmdir_selfheal_cbk,
-                                                    &heal_local->loc,
-                                                    heal_local->layout);
-                        if (ret) {
-                                DHT_STACK_DESTROY (heal_frame);
-                                goto err;
-                        }
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
+    return 0;
+err:
+    DHT_STACK_UNWIND(rmdir, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
 
-                } else if (this_call_cnt) {
-                        /* If non-hashed subvol's have responded, proceed */
-                        if (local->op_ret == 0) {
-                                /* Delete the dir from the hashed subvol if:
-                                 * The fop succeeded on at least one subvol
-                                 *  and did not fail on any
-                                 *  or
-                                 *  The fop failed with ENOENT/ESTALE on
-                                 *  all subvols */
-
-                                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
-                                            local->hashed_subvol,
-                                            local->hashed_subvol->fops->rmdir,
-                                            &local->loc, local->flags, NULL);
-                        } else {
-                         /* hashed-subvol was non-NULL and rmdir failed on
-                          * all non hashed-subvols. Unwind rmdir with
-                          * local->op_ret and local->op_errno. */
-                                dht_rmdir_unlock (frame, this);
-                                DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-
-                                return 0;
+static void
+dht_rmdir_readdirp_done(call_frame_t *readdirp_frame, xlator_t *this)
+{
+    call_frame_t *main_frame = NULL;
+    dht_local_t *main_local = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-                        }
-                } else if (!this_call_cnt) {
-                        /* All subvol's have responded, proceed */
+    local = readdirp_frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
 
-                        if (local->loc.parent) {
+    /* At least one readdirp failed.
+     * This is a bit hit or miss - if readdirp failed on more than
+     * one subvol, we don't know which error is returned.
+     */
+    if (local->op_ret == -1) {
+        main_local->op_ret = local->op_ret;
+        main_local->op_errno = local->op_errno;
+    }
 
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->preparent,
-                                                           0);
+    this_call_cnt = dht_frame_return(main_frame);
 
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->postparent,
-                                                           1);
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_do(main_frame, this);
 
-                        }
+    DHT_STACK_DESTROY(readdirp_frame);
+}
 
-                        dht_set_fixed_dir_stat (&local->preparent);
-                        dht_set_fixed_dir_stat (&local->postparent);
+/* Keep sending readdirp on the subvol until it returns no more entries
+ * It is possible that not all entries will fit in a single readdirp in
+ * which case the rmdir will keep failing with ENOTEMPTY
+ */
 
-                        dht_rmdir_unlock (frame, this);
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-                }
-        }
+static int
+dht_rmdir_readdirp_do(call_frame_t *readdirp_frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
 
-        return 0;
+    local = readdirp_frame->local;
 
-err:
-        DHT_STACK_UNWIND (rmdir, frame, -1, local->op_errno, NULL, NULL, NULL);
+    if (local->op_ret == -1) {
+        /* there is no point doing another readdirp on this
+         * subvol . */
+        dht_rmdir_readdirp_done(readdirp_frame, this);
         return 0;
+    }
 
-}
-
+    STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                      local->hashed_subvol, local->hashed_subvol,
+                      local->hashed_subvol->fops->readdirp, local->fd, 4096, 0,
+                      local->xattr);
 
-int
-dht_rmdir_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    return 0;
 }
 
-
-int
-dht_rmdir_unlock (call_frame_t *frame, xlator_t *this)
+static int
+dht_rmdir_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        lock_count = dht_lock_count (local->lock.locks, local->lock.lk_count);
+    local = frame->local;
+    prev = cookie;
+    src = prev;
 
-        if (lock_count == 0)
-                goto done;
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
 
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL)
-                goto done;
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL)
-                goto done;
+    if (op_ret == 0) {
+        gf_msg_trace(this->name, 0, "Unlinked linkfile %s on %s, gfid = %s",
+                     local->loc.path, src->name, gfid);
+    } else {
+        if (op_errno != ENOENT) {
+            readdirp_local->op_ret = -1;
+            readdirp_local->op_errno = op_errno;
+        }
+        gf_msg_debug(this->name, op_errno,
+                     "Unlink of %s on %s failed. (gfid = %s)", local->loc.path,
+                     src->name, gfid);
+    }
 
-        lock_local->lock.locks = local->lock.locks;
-        lock_local->lock.lk_count = local->lock.lk_count;
+    this_call_cnt = dht_frame_return(readdirp_frame);
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
-        dht_unlock_inodelk (lock_frame, lock_local->lock.locks,
-                            lock_local->lock.lk_count,
-                            dht_rmdir_unlock_cbk);
-        lock_frame = NULL;
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
 
-done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
-
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static int
+dht_rmdir_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    gf_msg_debug(this->name, 0, "dht_rmdir_lookup_cbk %s", local->loc.path);
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_FILE_LOOKUP_FAILED,
+               "lookup failed for %s on %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    if (!check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
 
-int
-dht_rmdir_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        int           i     = 0;
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-        VALIDATE_OR_GOTO (this->private, err);
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+               "%s on %s is not a linkfile (type=0%o, gfid = %s)",
+               local->loc.path, src->name, stbuf->ia_type, gfid);
+        goto err;
+    }
 
-        conf = this->private;
-        local = frame->local;
+    STACK_WIND_COOKIE(frame, dht_rmdir_linkfile_unlink_cbk, src, src,
+                      src->fops->unlink, &local->loc, 0, NULL);
+    return 0;
+err:
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring inodelk failed rmdir for %s)",
-                        local->loc.path);
+    this_call_cnt = dht_frame_return(readdirp_frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+    }
 
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto err;
-        }
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (local->hashed_subvol &&
-                    (local->hashed_subvol == conf->subvolumes[i]))
-                        continue;
+static int
+dht_rmdir_cached_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, inode_t *inode,
+                            struct iatt *stbuf, dict_t *xattr,
+                            struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *xattrs = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    src = local->hashed_subvol;
+
+    /* main_frame here is the readdirp_frame */
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    gf_msg_debug(this->name, 0, "returning for %s ", local->loc.path);
+
+    if (op_ret == 0) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_ERROR,
+               "%s found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    } else if (op_errno != ENOENT) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = op_errno;
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_SUBVOL_ERROR,
+               "%s not found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        goto err;
+    }
+
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
+        if (xattrs)
+            dict_unref(xattrs);
+        goto err;
+    }
+    STACK_WIND_COOKIE(frame, dht_rmdir_lookup_cbk, src, src, src->fops->lookup,
+                      &local->loc, xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
+
+    return 0;
+err:
 
-                STACK_WIND (frame, dht_rmdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->rmdir,
-                            &local->loc, local->flags, NULL);
-        }
+    this_call_cnt = dht_frame_return(readdirp_frame);
 
-        return 0;
+    /* Once all the lookups/unlinks etc have returned, proceed to wind
+     * readdirp on the subvol again until no entries are returned.
+     * This is required if there are more entries than can be returned
+     * in a single readdirp call.
+     */
 
-err:
-        /* No harm in calling an extra rmdir unlock */
-        dht_rmdir_unlock (frame, this);
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
 
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static int
+dht_rmdir_is_subvol_empty(call_frame_t *frame, xlator_t *this,
+                          gf_dirent_t *entries, xlator_t *src)
+{
+    int ret = 0;
+    int build_ret = 0;
+    gf_dirent_t *trav = NULL;
+    call_frame_t *lookup_frame = NULL;
+    dht_local_t *lookup_local = NULL;
+    dht_local_t *local = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = this->private;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    gf_boolean_t unwind = _gf_false;
+
+    local = frame->local;
+
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
+        if (check_is_linkfile(NULL, (&trav->d_stat), trav->dict,
+                              conf->link_xattr_name)) {
+            count++;
+            continue;
+        }
+
+        /* this entry is either a directory which is neither "." nor "..",
+           or a non directory which is not a linkfile. the directory is to
+           be treated as non-empty
+        */
+        return 0;
+    }
 
-int
-dht_rmdir_do (call_frame_t *frame, xlator_t *this)
-{
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf = NULL;
-        dht_lock_t   **lk_array = NULL;
-        int           i = 0, ret = -1;
-        int           count = 1;
-        xlator_t     *hashed_subvol = NULL;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
-
-        VALIDATE_OR_GOTO (this->private, err);
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        return -1;
+    }
 
-        conf = this->private;
-        local = frame->local;
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
 
-        if (local->op_ret == -1)
-                goto err;
+        if (xattrs)
+            dict_unref(xattrs);
+        return -1;
+    }
 
-        local->call_cnt = conf->subvolume_cnt;
+    local->call_cnt = count;
+    ret = 0;
 
-        /* first remove from non-hashed_subvol */
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
 
-        if (!hashed_subvol) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
+        lookup_frame = copy_frame(frame);
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "Failed to get hashed subvol for %s (gfid = %s)",
-                        local->loc.path, gfid);
-        } else {
-                local->hashed_subvol = hashed_subvol;
+        if (!lookup_frame) {
+            /* out of memory, let the rmdir fail
+               (as non-empty, unfortunately) */
+            goto err;
         }
 
-        /* When DHT has only 1 child */
-        if (conf->subvolume_cnt == 1) {
-                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
-                            conf->subvolumes[0],
-                            conf->subvolumes[0]->fops->rmdir,
-                            &local->loc, local->flags, NULL);
-                return 0;
+        lookup_local = dht_local_init(lookup_frame, NULL, NULL, GF_FOP_LOOKUP);
+        if (!lookup_local) {
+            goto err;
         }
 
-        count = conf->subvolume_cnt;
+        lookup_frame->local = lookup_local;
+        lookup_local->main_frame = frame;
+        lookup_local->hashed_subvol = src;
 
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
-        if (lk_array == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
-        }
+        build_ret = dht_build_child_loc(this, &lookup_local->loc, &local->loc,
+                                        trav->d_name);
+        if (build_ret != 0)
+            goto err;
 
-        for (i = 0; i < count; i++) {
-                lk_array[i] = dht_lock_new (frame->this,
-                                            conf->subvolumes[i],
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN);
-                if (lk_array[i] == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                        goto err;
-                }
-        }
+        gf_uuid_copy(lookup_local->loc.gfid, trav->d_stat.ia_gfid);
 
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
+        gf_uuid_unparse(lookup_local->loc.gfid, gfid);
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    IGNORE_ENOENT_ESTALE,
-                                    dht_rmdir_lock_cbk);
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                local->op_ret = -1;
-                local->op_errno = errno ? errno : EINVAL;
-                goto err;
-        }
+        gf_msg_trace(this->name, 0, "looking up %s on subvolume %s, gfid = %s",
+                     lookup_local->loc.path, src->name, gfid);
 
-        return 0;
+        subvol = dht_linkfile_subvol(this, NULL, &trav->d_stat, trav->dict);
+        if (!subvol || (subvol == src)) {
+            /* we need to delete the linkto file if it does not have a
+             * valid subvol or it points to itself.
+             */
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_INVALID_LINKFILE,
+                   "Linkfile does not have link subvolume. "
+                   "path = %s, gfid = %s",
+                   lookup_local->loc.path, gfid);
 
-err:
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
+            gf_msg_debug(this->name, 0, "looking up %s on subvol %s, gfid = %s",
+                         lookup_local->loc.path, src->name, gfid);
 
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
-
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
-        return 0;
-}
+            STACK_WIND_COOKIE(lookup_frame, dht_rmdir_lookup_cbk, src, src,
+                              src->fops->lookup, &lookup_local->loc, xattrs);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Looking up linkfile target %s on "
+                         " subvol %s, gfid = %s",
+                         lookup_local->loc.path, subvol->name, gfid);
 
+            STACK_WIND(lookup_frame, dht_rmdir_cached_lookup_cbk, subvol,
+                       subvol->fops->lookup, &lookup_local->loc, xattrs);
+        }
+        ret++;
 
-int
-dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t    *local = NULL;
-        call_frame_t   *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *main_frame = NULL;
-        dht_local_t    *main_local = NULL;
-        int             this_call_cnt = 0;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+        lookup_frame = NULL;
+        lookup_local = NULL;
+    }
 
+    if (xattrs)
+        dict_unref(xattrs);
 
-        local  = frame->local;
-        prev   = cookie;
-        src    = prev->this;
+    return ret;
+err:
+    if (xattrs)
+        dict_unref(xattrs);
 
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
+    if (lookup_frame)
+        DHT_STACK_DESTROY(lookup_frame);
 
-        gf_uuid_unparse(local->loc.gfid, gfid);
+    /* Handle the case where the wound calls have unwound before the
+     * loop processing is done
+     */
 
-        if (op_ret == 0) {
-                gf_msg_trace (this->name, 0,
-                              "Unlinked linkfile %s on %s, gfid = %s",
-                              local->loc.path, src->name, gfid);
-        } else {
-                main_local->op_ret   = -1;
-                main_local->op_errno = op_errno;
-                gf_msg_debug (this->name, op_errno,
-                              "Unlink of %s on %s failed. (gfid = %s)",
-                              local->loc.path, src->name, gfid);
-        }
+    LOCK(&frame->lock);
+    {
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
 
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+        local->call_cnt -= (count - ret);
+        if (!local->call_cnt)
+            unwind = _gf_true;
+    }
+    UNLOCK(&frame->lock);
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    if (!unwind) {
+        return ret;
+    }
+    return 0;
 }
 
+/*
+ * No more entries on this subvol. Proceed to the actual rmdir operation.
+ */
 
-int
-dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, inode_t *inode,
-                      struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-        dht_local_t    *local = NULL;
-        call_frame_t   *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *main_frame = NULL;
-        dht_local_t    *main_local = NULL;
-        int             this_call_cnt = 0;
-        dht_conf_t     *conf = this->private;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    int ret = 0;
+    char *path = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-        src   = prev->this;
-
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
-
-        if (op_ret != 0)
-                goto err;
+    local = frame->local;
+    prev = cookie;
+    src = prev;
 
-        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
-                main_local->op_ret  = -1;
-                main_local->op_errno = ENOTEMPTY;
+    if (op_ret > 2) {
+        /* dht_rmdir_is_subvol_empty() may free the frame,
+         * copy path for logging.
+         */
+        path = gf_strdup(local->loc.path);
 
-                 gf_uuid_unparse(local->loc.gfid, gfid);
+        ret = dht_rmdir_is_subvol_empty(frame, this, entries, src);
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_NOT_LINK_FILE_ERROR,
-                        "%s on %s is not a linkfile (type=0%o, gfid = %s)",
-                        local->loc.path, src->name, stbuf->ia_type, gfid);
-                goto err;
+        switch (ret) {
+            case 0: /* non linkfiles exist */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s returned %d "
+                             "entries",
+                             prev->name, local->loc.path, op_ret);
+                local->op_ret = -1;
+                local->op_errno = ENOTEMPTY;
+                break;
+            default:
+                /* @ret number of linkfiles are getting unlinked */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s found %d "
+                             "linkfiles",
+                             prev->name, path, ret);
+                break;
         }
+    }
 
-        STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
-                    src, src->fops->unlink, &local->loc, 0, NULL);
-        return 0;
-err:
-
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+    /* readdirp failed or no linkto files were found on this subvol */
+    if (!ret)
+        dht_rmdir_readdirp_done(frame, this);
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    GF_FREE(path);
+    return 0;
 }
 
+static int
+dht_rmdir_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *dict = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *readdirp_local = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    this_call_cnt = dht_frame_return(frame);
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-int
-dht_rmdir_cached_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int op_ret, int op_errno, inode_t *inode,
-                             struct iatt *stbuf, dict_t *xattr,
-                             struct iatt *parent)
-{
-        dht_local_t    *local         = NULL;
-        xlator_t       *src           = NULL;
-        call_frame_t   *main_frame    = NULL;
-        dht_local_t    *main_local    = NULL;
-        int             this_call_cnt = 0;
-        dht_conf_t     *conf          = this->private;
-        dict_t         *xattrs        = NULL;
-        int             ret           = 0;
-
-        local = frame->local;
-        src   = local->hashed_subvol;
-
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
-
-        if (op_ret == 0) {
-                main_local->op_ret  = -1;
-                main_local->op_errno = ENOTEMPTY;
-
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_SUBVOL_ERROR,
-                        "%s found on cached subvol %s",
-                        local->loc.path, src->name);
-                goto err;
-        } else if (op_errno != ENOENT) {
-                main_local->op_ret  = -1;
-                main_local->op_errno = op_errno;
-                goto err;
-        }
-
-        xattrs = dict_new ();
-        if (!xattrs) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY, "dict_new failed");
-                goto err;
-        }
-
-        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        conf->link_xattr_name);
-                if (xattrs)
-                        dict_unref (xattrs);
-                goto err;
+        gf_msg_debug(this->name, op_errno,
+                     "opendir on %s for %s failed, "
+                     "gfid = %s,",
+                     prev->name, local->loc.path, gfid);
+        if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
         }
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_rmdir_lookup_cbk,
-                    src, src->fops->lookup, &local->loc, xattrs);
-        if (xattrs)
-                dict_unref (xattrs);
-
+    if (!is_last_call(this_call_cnt))
         return 0;
-err:
 
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+    if (local->op_ret == -1)
+        goto err;
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
-}
+    fd_bind(fd);
 
+    dict = dict_new();
+    if (!dict) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
 
-int
-dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
-                           gf_dirent_t *entries, xlator_t *src)
-{
-        int                 ret          = 0;
-        int                 build_ret    = 0;
-        gf_dirent_t        *trav         = NULL;
-        call_frame_t       *lookup_frame = NULL;
-        dht_local_t        *lookup_local = NULL;
-        dht_local_t        *local        = NULL;
-        dict_t             *xattrs       = NULL;
-        dht_conf_t         *conf         = this->private;
-        xlator_t           *subvol       = NULL;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
+    ret = dict_set_uint32(dict, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: Failed to set dictionary value:key = %s", local->loc.path,
+               conf->link_xattr_name);
 
-        local = frame->local;
+    cnt = local->call_cnt = conf->subvolume_cnt;
 
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
-                                              conf->link_xattr_name)) {
-                        ret++;
-                        continue;
-                }
+    /* Create a separate frame per subvol as we might need
+     * to resend readdirp multiple times to get all the
+     * entries.
+     */
 
-                /* this entry is either a directory which is neither "." nor "..",
-                   or a non directory which is not a linkfile. the directory is to
-                   be treated as non-empty
-                */
-                return 0;
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        readdirp_frame = copy_frame(frame);
 
-        xattrs = dict_new ();
-        if (!xattrs) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY, "dict_new failed");
-                return -1;
+        if (!readdirp_frame) {
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            (void)dht_frame_return(frame);
+            continue;
         }
 
-        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        conf->link_xattr_name);
-
-                if (xattrs)
-                        dict_unref (xattrs);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-
-                lookup_frame = NULL;
-                lookup_local = NULL;
-
-                lookup_frame = copy_frame (frame);
-                if (!lookup_frame) {
-                        /* out of memory, let the rmdir fail
-                           (as non-empty, unfortunately) */
-                        goto err;
-                }
+        readdirp_local = dht_local_init(readdirp_frame, &local->loc, local->fd,
+                                        0);
 
-                lookup_local = mem_get0 (this->local_pool);
-                if (!lookup_local) {
-                        goto err;
-                }
-
-                lookup_frame->local = lookup_local;
-                lookup_local->main_frame = frame;
-                lookup_local->hashed_subvol = src;
-
-                build_ret = dht_build_child_loc (this, &lookup_local->loc,
-                                                 &local->loc, trav->d_name);
-                if (build_ret != 0)
-                        goto err;
+        if (!readdirp_local) {
+            DHT_STACK_DESTROY(readdirp_frame);
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            dht_frame_return(frame);
+            continue;
+        }
+        readdirp_local->main_frame = frame;
+        readdirp_local->op_ret = 0;
+        readdirp_local->xattr = dict_ref(dict);
+        /* overload this field to save the subvol info */
+        readdirp_local->hashed_subvol = conf->subvolumes[i];
 
-                gf_uuid_copy (lookup_local->loc.gfid, trav->d_stat.ia_gfid);
+        STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                          conf->subvolumes[i], conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->readdirp,
+                          readdirp_local->fd, 4096, 0, readdirp_local->xattr);
+    }
 
-                gf_uuid_unparse(lookup_local->loc.gfid, gfid);
+    if (dict)
+        dict_unref(dict);
 
-                gf_msg_trace (this->name, 0,
-                              "looking up %s on subvolume %s, gfid = %s",
-                              lookup_local->loc.path, src->name, gfid);
+    /* Could not wind readdirp to any subvol */
 
-                LOCK (&frame->lock);
-                {
-                        local->call_cnt++;
-                }
-                UNLOCK (&frame->lock);
-
-                subvol = dht_linkfile_subvol (this, NULL, &trav->d_stat,
-                                              trav->dict);
-                if (!subvol) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_INVALID_LINKFILE,
-                                "Linkfile does not have link subvolume. "
-                                "path = %s, gfid = %s",
-                                lookup_local->loc.path, gfid);
-                        STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
-                                    src, src->fops->lookup,
-                                    &lookup_local->loc, xattrs);
-                } else {
-                        STACK_WIND (lookup_frame, dht_rmdir_cached_lookup_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    &lookup_local->loc, xattrs);
-                }
-                ret++;
-        }
+    if (!cnt)
+        goto err;
 
-        if (xattrs)
-                dict_unref (xattrs);
+    return 0;
 
-        return ret;
 err:
-        if (xattrs)
-                dict_unref (xattrs);
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_do(frame, this);
+    }
 
-        if (lookup_frame)
-                DHT_STACK_DESTROY (lookup_frame);
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        xlator_t     *src = NULL;
-        int           ret = 0;
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_RMDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+    local->op_ret = 0;
+    local->fop_succeeded = 0;
+
+    local->flags = flags;
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (flags) {
+        return dht_rmdir_do(frame, this);
+    }
+    if (xdata) {
+        xattr_req = dict_ref(xdata);
+    } else {
+        xattr_req = dict_new();
+    }
+    if (xattr_req) {
+        ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+        /* If parallel-readdir is enabled, this is required
+         * to handle stale linkto files in the directory
+         * being deleted. If this fails, log an error but
+         * do not prevent the operation.
+         */
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+                   loc->path, conf->link_xattr_name);
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+               loc->path, conf->link_xattr_name);
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, loc, local->fd,
+                          xattr_req);
+    }
+
+    if (xattr_req) {
+        dict_unref(xattr_req);
+    }
+    return 0;
 
-        local = frame->local;
-        prev  = cookie;
-        src   = prev->this;
-
-        if (op_ret > 2) {
-                ret = dht_rmdir_is_subvol_empty (frame, this, entries, src);
-
-                switch (ret) {
-                case 0: /* non linkfiles exist */
-                        gf_msg_trace (this->name, 0,
-                                      "readdir on %s for %s returned %d "
-                                      "entries", prev->this->name,
-                                      local->loc.path, op_ret);
-                        local->op_ret = -1;
-                        local->op_errno = ENOTEMPTY;
-                        break;
-                default:
-                        /* @ret number of linkfiles are getting unlinked */
-                        gf_msg_trace (this->name, 0,
-                                      "readdir on %s for %s found %d "
-                                      "linkfiles", prev->this->name,
-                                      local->loc.path, ret);
-                        break;
-                }
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
 
-        this_call_cnt = dht_frame_return (frame);
+    return 0;
+}
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_do (frame, this);
-        }
+static int
+dht_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-        return 0;
+{
+    DHT_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
+/* TODO
+ * Sending entrylk to cached subvol can result in stale lock
+ * as described in the bug 1311002.
+ */
 int
-dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev          = NULL;
-        dict_t       *dict          = NULL;
-        int           ret           = 0;
-        dht_conf_t   *conf          = this->private;
-        int           i             = 0;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev  = cookie;
-
-
-        this_call_cnt = dht_frame_return (frame);
-        if (op_ret == -1) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
-
-                gf_msg_debug (this->name, op_errno,
-                              "opendir on %s for %s failed, "
-                              "gfid = %s,",
-                              prev->this->name, local->loc.path, gfid);
-                if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-                goto err;
-        }
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        if (!is_last_call (this_call_cnt))
-                return 0;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
 
-        if (local->op_ret == -1)
-                goto err;
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ENTRYLK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        fd_bind (fd);
-        dict = dict_new ();
-        if (!dict) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
-        }
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_uuid_unparse(loc->gfid, gfid);
 
-        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        local->loc.path, conf->link_xattr_name);
+        gf_msg_debug(this->name, 0,
+                     "no cached subvolume for path=%s, "
+                     "gfid = %s",
+                     loc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->call_cnt = conf->subvolume_cnt;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rmdir_readdirp_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->readdirp,
-                            local->fd, 4096, 0, dict);
-        }
+    local->call_cnt = 1;
 
-        if (dict)
-                dict_unref (dict);
+    STACK_WIND(frame, dht_entrylk_cbk, subvol, subvol->fops->entrylk, volume,
+               loc, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_do (frame, this);
-        }
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(entrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+static int
+dht_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           dict_t *xdata)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_RMDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->call_cnt = conf->subvolume_cnt;
-        local->op_ret   = 0;
-        local->fop_succeeded = 0;
+    DHT_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, NULL);
+    return 0;
+}
 
-        local->flags = flags;
+int
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
 
-                op_errno = ENOMEM;
-                goto err;
-        }
+    gf_uuid_unparse(fd->inode->gfid, gfid);
 
-        if (flags) {
-                return dht_rmdir_do (frame, this);
-        }
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No cached subvolume for fd=%p,"
+                     " gfid = %s",
+                     fd, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rmdir_opendir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            loc, local->fd, NULL);
-        }
+    STACK_WIND(frame, dht_fentrylk_cbk, subvol, subvol->fops->fentrylk, volume,
+               fd, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fentrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
-
+static int32_t
+dht_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-/* TODO
- * Sending entrylk to cached subvol can result in stale lock
- * as described in the bug 1311002.
- */
-int
-dht_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
+    local = frame->local;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_uuid_unparse(loc->gfid, gfid);
-
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s, "
-                              "gfid = %s", loc->path, gfid);
-                op_errno = EINVAL;
-                goto err;
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0 && op_errno != ENOTCONN) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
+        local->op_ret = 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        local->call_cnt = 1;
-
-        STACK_WIND (frame, dht_entrylk_cbk,
-                    subvol, subvol->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
-
-        return 0;
-
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno, NULL);
+    }
 
-        return 0;
+out:
+    return 0;
 }
 
-
-int
-dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
-
+int32_t
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
-        return 0;
-}
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int i = 0;
 
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-int
-dht_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
+    VALIDATE_OR_GOTO(this->private, err);
+    conf = this->private;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO(fd->inode, err);
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_IPC);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        gf_uuid_unparse(fd->inode->gfid, gfid);
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "No cached subvolume for fd=%p,"
-                              " gfid = %s", fd, gfid);
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (xdata) {
+        if (dict_set_int8(xdata, conf->xattr_name, 0) < 0)
+            goto err;
+    }
 
-        STACK_WIND (frame, dht_fentrylk_cbk,
-                    subvol, subvol->fops->fentrylk,
-                    volume, fd, basename, cmd, type, xdata);
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND(frame, dht_ipc_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->ipc, op, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+    DHT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
 
-        return 0;
-}
+    return 0;
 
+wind_default:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
 
 int
-dht_forget (xlator_t *this, inode_t *inode)
+dht_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t        ctx_int = 0;
-        dht_inode_ctx_t *ctx    = NULL;
-        dht_layout_t *layout = NULL;
+    uint64_t ctx_int = 0;
+    dht_inode_ctx_t *ctx = NULL;
+    dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &ctx_int);
+    inode_ctx_del(inode, this, &ctx_int);
 
-        if (!ctx_int)
-                return 0;
+    if (!ctx_int)
+        return 0;
 
-        ctx = (dht_inode_ctx_t *) (long) ctx_int;
+    ctx = (dht_inode_ctx_t *)(long)ctx_int;
 
-        layout = ctx->layout;
-        ctx->layout = NULL;
-        dht_layout_unref (this, layout);
-        GF_FREE (ctx);
+    layout = ctx->layout;
+    ctx->layout = NULL;
+    dht_layout_unref(this, layout);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_notify (xlator_t *this, int event, void *data, ...)
-{
-        xlator_t                *subvol = NULL;
-        int                      cnt    = -1;
-        int                      i      = -1;
-        dht_conf_t              *conf   = NULL;
-        int                      ret    = -1;
-        int                      propagate = 0;
-
-        int                      had_heard_from_all = 0;
-        int                      have_heard_from_all = 0;
-        struct timeval           time = {0,};
-        gf_defrag_info_t        *defrag = NULL;
-        dict_t                  *dict   = NULL;
-        gf_defrag_type           cmd    = 0;
-        dict_t                  *output = NULL;
-        va_list                  ap;
-        dht_methods_t           *methods = NULL;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        methods = &(conf->methods);
-
-        /* had all subvolumes reported status once till now? */
-        had_heard_from_all = 1;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i]) {
-                        had_heard_from_all = 0;
-                }
-        }
-
-        switch (event) {
+dht_notify(xlator_t *this, int event, void *data, ...)
+{
+    xlator_t *subvol = NULL;
+    int cnt = -1;
+    int i = -1;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int propagate = 0;
+
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *dict = NULL;
+    gf_defrag_type cmd = 0;
+    dict_t *output = NULL;
+    va_list ap;
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* had all subvolumes reported status once till now? */
+    had_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i]) {
+            had_heard_from_all = 0;
+        }
+    }
+
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                subvol = data;
-
-                conf->gen++;
+            subvol = data;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+            conf->gen++;
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_UP bad "
-                                      "subvolume %s",
-                                      subvol->name);
-                        break;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
+            }
 
-                gettimeofday (&time, NULL);
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 1;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = time.tv_sec;
-                }
-                UNLOCK (&conf->subvolume_lock);
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_UP bad "
+                             "subvolume %s",
+                             subvol->name);
+                break;
+            }
 
-                /* one of the node came back up, do a stat update */
-                dht_get_du_info_for_subvol (this, cnt);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 1;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = gf_time();
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                break;
+            /* one of the node came back up, do a stat update */
+            dht_get_du_info_for_subvol(this, cnt);
 
-        case GF_EVENT_CHILD_MODIFIED:
-                subvol = data;
+            break;
 
-                conf->gen++;
-                propagate = 1;
+        case GF_EVENT_SOME_DESCENDENT_UP:
+            subvol = data;
+            conf->gen++;
+            propagate = 1;
 
-                break;
+            break;
 
-        case GF_EVENT_SOME_CHILD_DOWN:
-                subvol = data;
-                propagate = 1;
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+            subvol = data;
+            propagate = 1;
 
-                break;
+            break;
 
         case GF_EVENT_CHILD_DOWN:
-                subvol = data;
-
-                if (conf->assert_no_child_down) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_CHILD_DOWN,
-                                "Received CHILD_DOWN. Exiting");
-                        if (conf->defrag) {
-                                gf_defrag_stop (conf->defrag,
-                                                GF_DEFRAG_STATUS_FAILED, NULL);
-                        } else {
-                                kill (getpid(), SIGTERM);
-                        }
-                }
-
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+            subvol = data;
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_DOWN bad "
-                                      "subvolume %s", subvol->name);
-                        break;
+            if (conf->assert_no_child_down) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_CHILD_DOWN,
+                       "Received CHILD_DOWN. Exiting");
+                if (conf->defrag) {
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_FAILED, NULL);
+                } else {
+                    kill(getpid(), SIGTERM);
                 }
+            }
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 0;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = 0;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
-                for (i = 0; i < conf->subvolume_cnt; i++)
-                        if (conf->last_event[i] != event)
-                                event = GF_EVENT_CHILD_MODIFIED;
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_DOWN bad "
+                             "subvolume %s",
+                             subvol->name);
                 break;
+            }
 
-        case GF_EVENT_CHILD_CONNECTING:
-                subvol = data;
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 0;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = 0;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+            for (i = 0; i < conf->subvolume_cnt; i++)
+                if (conf->last_event[i] != event)
+                    event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            break;
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_CONNECTING"
-                                      " bad subvolume %s",
-                                      subvol->name);
-                        break;
-                }
+        case GF_EVENT_CHILD_CONNECTING:
+            subvol = data;
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->last_event[cnt] = event;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_CONNECTING"
+                             " bad subvolume %s",
+                             subvol->name);
                 break;
-        case GF_EVENT_VOLUME_DEFRAG:
-        {
-                if (!conf->defrag) {
-                        return ret;
-                }
-                defrag = conf->defrag;
+            }
 
-                dict = data;
-                va_start (ap, data);
-                output = va_arg (ap, dict_t*);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->last_event[cnt] = event;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                ret = dict_get_int32 (dict, "rebalance-command",
-                                      (int32_t*)&cmd);
-                if (ret)
-                        return ret;
-                LOCK (&defrag->lock);
-                {
-                        if (defrag->is_exiting)
-                                goto unlock;
-                        if ((cmd == GF_DEFRAG_CMD_STATUS) ||
-                            (cmd == GF_DEFRAG_CMD_STATUS_TIER))
-                                gf_defrag_status_get (defrag, output);
-                        else if (cmd == GF_DEFRAG_CMD_START_DETACH_TIER)
-                                gf_defrag_start_detach_tier(defrag);
-                        else if (cmd == GF_DEFRAG_CMD_STOP ||
-                                 cmd == GF_DEFRAG_CMD_STOP_DETACH_TIER)
-                                gf_defrag_stop (defrag,
-                                                GF_DEFRAG_STATUS_STOPPED, output);
-                        else if (cmd == GF_DEFRAG_CMD_PAUSE_TIER)
-                                ret = gf_defrag_pause_tier (this, defrag);
-                        else if (cmd == GF_DEFRAG_CMD_RESUME_TIER)
-                                ret = gf_defrag_resume_tier (this, defrag);
-                }
-unlock:
-                UNLOCK (&defrag->lock);
+            break;
+        case GF_EVENT_VOLUME_DEFRAG: {
+            if (!conf->defrag) {
                 return ret;
-                break;
-        }
+            }
+            defrag = conf->defrag;
 
-        default:
-                propagate = 1;
-                break;
-        }
+            dict = data;
+            va_start(ap, data);
+            output = va_arg(ap, dict_t *);
 
+            ret = dict_get_int32(dict, "rebalance-command", (int32_t *)&cmd);
+            if (ret) {
+                va_end(ap);
+                return ret;
+            }
+            LOCK(&defrag->lock);
+            {
+                if (defrag->is_exiting)
+                    goto unlock;
+                if ((cmd == GF_DEFRAG_CMD_STATUS) ||
+                    (cmd == GF_DEFRAG_CMD_DETACH_STATUS))
+                    gf_defrag_status_get(conf, output);
+                else if (cmd == GF_DEFRAG_CMD_DETACH_START)
+                    defrag->cmd = GF_DEFRAG_CMD_DETACH_START;
+                else if (cmd == GF_DEFRAG_CMD_STOP ||
+                         cmd == GF_DEFRAG_CMD_DETACH_STOP)
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_STOPPED, output);
+            }
+        unlock:
+            UNLOCK(&defrag->lock);
+            va_end(ap);
+            return ret;
+            break;
+        }
+        case GF_EVENT_UPCALL:
+            up_data = (struct gf_upcall *)data;
+            if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+                break;
+            up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+            /* Since md-cache will be aggressively filtering lookups,
+             * the stale layout issue will be more pronounced. Hence
+             * when a layout xattr is changed by the rebalance process
+             * notify all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time*/
+            if (up_ci->dict && dict_get(up_ci->dict, conf->xattr_name))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            /* TODO: Instead of invalidating iatt, update the new
+             * hashed/cached subvolume in dht inode_ctx */
+            if (IS_DHT_LINKFILE_MODE(&up_ci->stat))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            propagate = 1;
+            break;
+        default:
+            propagate = 1;
+            break;
+    }
+
+    /* have all subvolumes reported status once by now? */
+    have_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i])
+            have_heard_from_all = 0;
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all) {
+        propagate = 1;
+    }
+
+    if (!had_heard_from_all && have_heard_from_all) {
+        static int run_defrag = 0;
+        /* This is the first event which completes aggregation
+           of events from all subvolumes. If at least one subvol
+           had come up, propagate CHILD_UP, but only this time
+        */
+        event = GF_EVENT_CHILD_DOWN;
 
-        /* have all subvolumes reported status once by now? */
-        have_heard_from_all = 1;
         for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i])
-                        have_heard_from_all = 0;
-        }
-
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all) {
-                propagate = 1;
+            if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
+                event = GF_EVENT_CHILD_UP;
+                break;
+            }
 
+            if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                event = GF_EVENT_CHILD_CONNECTING;
+                /* continue to check other events for CHILD_UP */
+            }
         }
 
-
-        if (!had_heard_from_all && have_heard_from_all) {
-                /* This is the first event which completes aggregation
-                   of events from all subvolumes. If at least one subvol
-                   had come up, propagate CHILD_UP, but only this time
-                */
-                event = GF_EVENT_CHILD_DOWN;
-
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
-                                event = GF_EVENT_CHILD_UP;
-                                break;
-                        }
-
-                        if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
-                                event = GF_EVENT_CHILD_CONNECTING;
-                                /* continue to check other events for CHILD_UP */
-                        }
-                }
-
-                /* Rebalance is started with assert_no_child_down. So we do
-                 * not need to handle CHILD_DOWN event here.
-                 *
-                 * If there is a graph switch, we should not restart the
-                 * rebalance daemon. Use 'run_defrag' to indicate if the
-                 * thread has already started.
-                 */
-                 if (conf->defrag && !run_defrag) {
-                        if (methods->migration_needed(this)) {
-                                run_defrag = 1;
-                                ret = gf_thread_create(&conf->defrag->th,
-                                                       NULL,
-                                                       gf_defrag_start, this);
-                                if (ret) {
-                                        GF_FREE (conf->defrag);
-                                        conf->defrag = NULL;
-                                        kill (getpid(), SIGTERM);
-                                }
-                        }
-                }
+        /* Rebalance is started with assert_no_child_down. So we do
+         * not need to handle CHILD_DOWN event here.
+         *
+         * If there is a graph switch, we should not restart the
+         * rebalance daemon. Use 'run_defrag' to indicate if the
+         * thread has already started.
+         */
+        if (conf->defrag && !run_defrag) {
+            run_defrag = 1;
+            ret = gf_thread_create(&conf->defrag->th, NULL, gf_defrag_start,
+                                   this, "dhtdg");
+            if (ret) {
+                GF_FREE(conf->defrag);
+                conf->defrag = NULL;
+                kill(getpid(), SIGTERM);
+            }
         }
+    }
 
-        ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this, dht_layout_t **layout)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        if (!ret && ctx) {
-                if (ctx->layout) {
-                        if (layout)
-                                *layout = ctx->layout;
-                        ret = 0;
-                } else {
-                        ret = -1;
-                }
+    if (!ret && ctx) {
+        if (ctx->layout) {
+            if (layout)
+                *layout = ctx->layout;
+            ret = 0;
+        } else {
+            ret = -1;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
-                                 dht_layout_t *layout)
-{
-
-        char                    string[2048] = {0};
-        char                  *output_string = NULL;
-        int                              len = 0;
-        int                              off = 0;
-        int                                i = 0;
-        gf_loglevel_t             log_level = gf_log_get_loglevel();
-        int                              ret = 0;
-        int                   max_string_len = 0;
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout)
+{
+    char string[2048] = {0};
+    char *output_string = NULL;
+    int len = 0;
+    int off = 0;
+    int i = 0;
+    gf_loglevel_t log_level = gf_log_get_loglevel();
+    int ret = 0;
+
+    if (log_level < GF_LOG_INFO)
+        return;
 
-        if (log_level < GF_LOG_INFO)
-                return;
+    if (!layout)
+        return;
 
-        if (!layout)
-                return;
+    if (!layout->cnt)
+        return;
 
-        if (!layout->cnt)
-                return;
+    if (!loc)
+        return;
 
-        if (!loc)
-                return;
+    if (!loc->path)
+        return;
 
-        if (!loc->path)
-                return;
+    ret = snprintf(string, sizeof(string), "Setting layout of %s with ",
+                   loc->path);
 
-        max_string_len = sizeof (string);
+    if (ret < 0)
+        return;
 
-        ret = snprintf (string, max_string_len, "Setting layout of %s with ",
-                        loc->path);
+    len += ret;
+
+    /* Calculation  of total length of the string required to calloc
+     * output_string. Log includes subvolume-name, start-range, end-range
+     * and err value.
+     *
+     * This log will help to debug cases where:
+     * a) Different processes set different layout of a directory.
+     * b) Error captured in lookup, which will be filled in layout->err
+     * (like ENOENT, ESTALE etc)
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(string, sizeof(string),
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
 
         if (ret < 0)
-                return;
+            return;
 
         len += ret;
+    }
 
-       /* Calculation  of total length of the string required to calloc
-        * output_string. Log includes subvolume-name, start-range, end-range and
-        * err value.
-        *
-        * This log will help to debug cases where:
-        * a) Different processes set different layout of a directory.
-        * b) Error captured in lookup, which will be filled in layout->err
-        * (like ENOENT, ESTALE etc)
-        */
-
-        for (i = 0; i < layout->cnt; i++) {
-
-                ret  = snprintf (string, max_string_len,
-                                 "[Subvol_name: %s, Err: %d , Start: "
-                                 "%"PRIu32 " , Stop: %"PRIu32 " , Hash: %"
-                                 PRIu32 " ], ",
-                                 layout->list[i].xlator->name,
-                                 layout->list[i].err, layout->list[i].start,
-                                 layout->list[i].stop,
-                                 layout->list[i].commit_hash);
+    len++;
 
-                if (ret < 0)
-                        return;
+    output_string = GF_MALLOC(len + 1, gf_common_mt_char);
 
-                len += ret;
-
-        }
+    if (!output_string)
+        return;
 
-        len++;
+    ret = snprintf(output_string, len + 1, "Setting layout of %s with ",
+                   loc->path);
 
-        output_string = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+    if (ret < 0)
+        goto err;
 
-        if (!output_string)
-                return;
+    off += ret;
 
-        ret = snprintf (output_string, len, "Setting layout of %s with ",
-                        loc->path);
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(output_string + off, len - off,
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
 
         if (ret < 0)
-                goto err;
+            goto err;
 
         off += ret;
+    }
 
+    gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_LOG_FIXED_LAYOUT, "%s",
+           output_string);
 
-        for (i = 0; i < layout->cnt; i++) {
-
-                ret  =  snprintf (output_string + off, len - off,
-                                  "[Subvol_name: %s, Err: %d , Start: "
-                                  "%"PRIu32 " , Stop: %"PRIu32 " , Hash: %"
-                                  PRIu32  " ], ",
-                                  layout->list[i].xlator->name,
-                                  layout->list[i].err, layout->list[i].start,
-                                  layout->list[i].stop,
-                                  layout->list[i].commit_hash);
+err:
+    GF_FREE(output_string);
+}
 
-                if (ret < 0)
-                        goto err;
+int32_t
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local)
+{
+    int ret = -1;
 
-                off += ret;
+    if (!local)
+        goto out;
 
-        }
+    local->rebalance.target_node = dht_subvol_get_hashed(this, &local->loc);
 
-        gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_FIXED_LAYOUT,
-                "%s", output_string);
+    if (local->rebalance.target_node)
+        ret = 0;
 
-err:
-        GF_FREE (output_string);
+out:
+    return ret;
 }
 
-int32_t dht_migration_get_dst_subvol(xlator_t *this, dht_local_t  *local)
+/*
+This function should not be called more then once during a FOP
+handling path. It is valid only for for ops on files
+*/
+int32_t
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata)
 {
-        int ret = -1;
+    if (!local)
+        return -1;
 
-        if (!local)
-                goto out;
+    if (local->rebalance.set) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REBAL_STRUCT_SET,
+               "local->rebalance already set");
+    }
 
-        local->rebalance.target_node =
-                dht_subvol_get_hashed (this, &local->loc);
+    if (stbuf)
+        memcpy(&local->rebalance.stbuf, stbuf, sizeof(struct iatt));
 
-        if (local->rebalance.target_node)
-                ret = 0;
+    if (prebuf)
+        memcpy(&local->rebalance.prebuf, prebuf, sizeof(struct iatt));
 
-out:
-        return ret;
+    if (postbuf)
+        memcpy(&local->rebalance.postbuf, postbuf, sizeof(struct iatt));
+
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    local->rebalance.set = 1;
+
+    return 0;
 }
 
-int32_t dht_migration_needed(xlator_t *this)
+int32_t
+dht_release(xlator_t *this, fd_t *fd)
 {
-        gf_defrag_info_t        *defrag = NULL;
-        dht_conf_t              *conf   = NULL;
-        int                      ret = 0;
+    return dht_fd_ctx_destroy(this, fd);
+}
 
-        conf = this->private;
+static int
+dht_pt_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", conf, out);
-        GF_VALIDATE_OR_GOTO ("dht", conf->defrag, out);
+    local = frame->local;
 
-        defrag = conf->defrag;
+    if (!op_ret) {
+        dht_layout_set(this, inode, local->layout);
+    }
 
-        if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) &&
-            (defrag->cmd != GF_DEFRAG_CMD_START_DETACH_TIER))
-                ret = 1;
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
 
-out:
-        return ret;
+    return 0;
 }
 
+int32_t
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    bool free_xdata = false;
+    int ret = 0;
+    int op_errno = 0;
+    int32_t *disk_layout_p = NULL;
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!layout)
+        goto wind;
+
+    local->layout = layout;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto wind;
+        free_xdata = true;
+    }
+
+    /*Set the xlator or the following will crash*/
+    layout->list[0].xlator = conf->subvolumes[0];
+
+    dht_selfheal_layout_new_directory(frame, loc, layout);
+
+    dht_disk_layout_extract(this, layout, 0, &disk_layout_p);
+
+    ret = dict_set_bin(xdata, conf->xattr_name, disk_layout_p, 4 * 4);
+    if (ret) {
+        gf_msg("dht", GF_LOG_DEBUG, EINVAL, DHT_MSG_DICT_SET_FAILED,
+               "dht layout dict set failed");
+    }
+wind:
+    STACK_WIND(frame, dht_pt_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    if (free_xdata)
+        dict_unref(xdata);
+    return 0;
 
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-/*
-This function should not be called more then once during a FOP
-handling path. It is valid only for for ops on files
-*/
-int32_t dht_set_local_rebalance (xlator_t *this, dht_local_t *local,
-                                 struct iatt *stbuf,
-                                 struct iatt *prebuf, struct iatt *postbuf,
-                                 dict_t *xdata)
-{
+    return 0;
+}
 
-        if (!local)
-                return -1;
+static int
+dht_pt_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
 
-        if (local->rebalance.set) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_REBAL_STRUCT_SET,
-                        "local->rebalance already set");
-        }
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
+    dict_del(xattr, conf->mds_xattr_key);
+    dict_del(xattr, conf->commithash_xattr_name);
 
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
 
-        if (stbuf)
-                memcpy (&local->rebalance.stbuf, stbuf, sizeof (struct iatt));
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
 
-        if (prebuf)
-                memcpy (&local->rebalance.prebuf, prebuf, sizeof (struct iatt));
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+    return 0;
+}
 
-        if (postbuf)
-                memcpy (&local->rebalance.postbuf, postbuf,
-                        sizeof (struct iatt));
+static int
+dht_pt_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
 
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
 
-        local->rebalance.set = 1;
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
 
-        return 0;
+    DHT_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
 }
 
-gf_boolean_t
-dht_is_tier_xlator (xlator_t *this)
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
 {
-
-        if (strcmp (this->type, "cluster/tier") == 0)
-                return _gf_true;
-        return _gf_false;
+    STACK_WIND(frame, dht_pt_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata);
+    return 0;
 }
 
-int32_t
-dht_release (xlator_t *this, fd_t *fd)
+/* The job of this function is to check if all the xlators have updated
+ * error in the layout. */
+int
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode)
 {
-        return dht_fd_ctx_destroy (this, fd);
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    layout = dht_layout_get(this, inode);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == 0) {
+            return 0;
+        }
+    }
+
+    /* Returning the first xlator error as all xlators have errors */
+    return layout->list[0].err;
 }
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 9a71c46c8e4..fe0dc3db34a 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -9,128 +9,148 @@
 */
 
 #include <regex.h>
-#include <signal.h>
 
 #include "dht-mem-types.h"
 #include "dht-messages.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 #include "libxlator.h"
-#include "syncop.h"
-#include "refcount.h"
-#include "timer.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/refcount.h>
+#include <glusterfs/timer.h>
+#include "protocol-common.h"
+#include <glusterfs/glusterfs-acl.h>
 
 #ifndef _DHT_H
 #define _DHT_H
 
-#define GF_XATTR_FIX_LAYOUT_KEY         "distribute.fix.layout"
-#define GF_XATTR_TIER_LAYOUT_FIXED_KEY  "trusted.tier.fix.layout.complete"
-#define GF_XATTR_FILE_MIGRATE_KEY       "trusted.distribute.migrate-data"
-#define GF_DHT_LOOKUP_UNHASHED_ON       1
-#define GF_DHT_LOOKUP_UNHASHED_AUTO     2
-#define DHT_PATHINFO_HEADER             "DISTRIBUTE:"
-#define DHT_FILE_MIGRATE_DOMAIN         "dht.file.migrate"
-#define DHT_LAYOUT_HEAL_DOMAIN          "dht.layout.heal"
-#define TIERING_MIGRATION_KEY           "tiering.migration"
-#define DHT_LAYOUT_HASH_INVALID         1
-
-#define DHT_DIR_STAT_BLOCKS          8
-#define DHT_DIR_STAT_SIZE            4096
-
-#include <fnmatch.h>
-
-typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
-                                       xlator_t     *this,
-                                       int32_t       op_ret, int32_t op_errno,
-                                       dict_t       *xdata);
-typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, xlator_t *dst_node,
-                                    call_frame_t    *frame, int ret);
-
-typedef int (*dht_refresh_layout_unlock) (call_frame_t *frame, xlator_t *this,
+#define GF_XATTR_FIX_LAYOUT_KEY "distribute.fix.layout"
+#define GF_XATTR_FILE_MIGRATE_KEY "trusted.distribute.migrate-data"
+#define DHT_MDS_STR "mds"
+#define GF_DHT_LOOKUP_UNHASHED_OFF 0
+#define GF_DHT_LOOKUP_UNHASHED_ON 1
+#define GF_DHT_LOOKUP_UNHASHED_AUTO 2
+#define DHT_PATHINFO_HEADER "DISTRIBUTE:"
+#define DHT_FILE_MIGRATE_DOMAIN "dht.file.migrate"
+/* Layout synchronization */
+#define DHT_LAYOUT_HEAL_DOMAIN "dht.layout.heal"
+/* Namespace synchronization */
+#define DHT_ENTRY_SYNC_DOMAIN "dht.entry.sync"
+#define DHT_LAYOUT_HASH_INVALID 1
+#define MAX_REBAL_THREADS sysconf(_SC_NPROCESSORS_ONLN)
+
+#define DHT_DIR_STAT_BLOCKS 8
+#define DHT_DIR_STAT_SIZE 4096
+
+/* Virtual xattr for subvols status */
+
+#define DHT_SUBVOL_STATUS_KEY "dht.subvol.status"
+
+/* Virtual xattrs for debugging */
+
+#define DHT_DBG_HASHED_SUBVOL_PATTERN "dht.file.hashed-subvol.*"
+#define DHT_DBG_HASHED_SUBVOL_KEY "dht.file.hashed-subvol."
+
+/* Rebalance nodeuuid flags */
+#define REBAL_NODEUUID_MINE 0x01
+
+typedef int (*dht_selfheal_dir_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+typedef int (*dht_defrag_cbk_fn_t)(xlator_t *this, xlator_t *dst_node,
+                                   call_frame_t *frame, int ret);
+
+typedef int (*dht_refresh_layout_unlock)(call_frame_t *frame, xlator_t *this,
                                          int op_ret, int invoke_cbk);
 
-typedef int (*dht_refresh_layout_done_handle) (call_frame_t *frame);
+typedef int (*dht_refresh_layout_done_handle)(call_frame_t *frame);
 
 struct dht_layout {
-        int                spread_cnt;  /* layout spread count per directory,
-                                           is controlled by 'setxattr()' with
-                                           special key */
-        int                cnt;
-        int                preset;
-        /*
-         * The last *configuration* state for which this directory was known
-         * to be in balance.  The corresponding vol_commit_hash changes
-         * whenever bricks are added or removed.  This value changes when a
-         * (full) rebalance is complete.  If they match, it's safe to assume
-         * that every file is where it should be and there's no need to do
-         * lookups for files elsewhere.  If they don't, then we have to do a
-         * global lookup to be sure.
-         */
-        uint32_t           commit_hash;
-        /*
-         * The *runtime* state of the volume, changes when connections to
-         * bricks are made or lost.
-         */
-        int                gen;
-        int                type;
-        int                ref; /* use with dht_conf_t->layout_lock */
-        gf_boolean_t       search_unhashed;
-        struct {
-                int        err;   /* 0 = normal
-                                     -1 = dir exists and no xattr
-                                     >0 = dir lookup failed with errno
-                                  */
-                uint32_t   start;
-                uint32_t   stop;
-                uint32_t   commit_hash;
-                xlator_t  *xlator;
-        } list[];
+    int spread_cnt; /* layout spread count per directory,
+                       is controlled by 'setxattr()' with
+                       special key */
+    int cnt;
+    int preset;
+    /*
+     * The last *configuration* state for which this directory was known
+     * to be in balance.  The corresponding vol_commit_hash changes
+     * whenever bricks are added or removed.  This value changes when a
+     * (full) rebalance is complete.  If they match, it's safe to assume
+     * that every file is where it should be and there's no need to do
+     * lookups for files elsewhere.  If they don't, then we have to do a
+     * global lookup to be sure.
+     */
+    uint32_t commit_hash;
+    /*
+     * The *runtime* state of the volume, changes when connections to
+     * bricks are made or lost.
+     */
+    int gen;
+    int type;
+    gf_atomic_t ref; /* use with dht_conf_t->layout_lock */
+    uint32_t search_unhashed;
+    struct {
+        int err; /* 0 = normal
+                    -1 = dir exists and no xattr
+                    >0 = dir lookup failed with errno
+                 */
+        uint32_t start;
+        uint32_t stop;
+        uint32_t commit_hash;
+        xlator_t *xlator;
+    } list[];
 };
-typedef struct dht_layout  dht_layout_t;
+typedef struct dht_layout dht_layout_t;
 
 struct dht_stat_time {
-        uint32_t        atime;
-        uint32_t        atime_nsec;
-        uint32_t        ctime;
-        uint32_t        ctime_nsec;
-        uint32_t        mtime;
-        uint32_t        mtime_nsec;
+    uint32_t atime;
+    uint32_t atime_nsec;
+    uint32_t ctime;
+    uint32_t ctime_nsec;
+    uint32_t mtime;
+    uint32_t mtime_nsec;
 };
 
 typedef struct dht_stat_time dht_stat_time_t;
 
 struct dht_inode_ctx {
-        dht_layout_t    *layout;
-        dht_stat_time_t  time;
-        xlator_t        *lock_subvol;
+    dht_layout_t *layout;
+    dht_stat_time_t time;
+    xlator_t *lock_subvol;
+    xlator_t *mds_subvol; /* This is only used for directories */
 };
 
 typedef struct dht_inode_ctx dht_inode_ctx_t;
 
-
 typedef enum {
-        DHT_HASH_TYPE_DM,
-        DHT_HASH_TYPE_DM_USER,
+    DHT_HASH_TYPE_DM,
+    DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
+typedef enum {
+    DHT_INODELK,
+    DHT_ENTRYLK,
+} dht_lock_type_t;
+
 /* rebalance related */
 struct dht_rebalance_ {
-        xlator_t            *from_subvol;
-        xlator_t            *target_node;
-        off_t                offset;
-        size_t               size;
-        int32_t              flags;
-        int                  count;
-        struct iobref       *iobref;
-        struct iovec        *vector;
-        struct iatt          stbuf;
-        struct iatt          prebuf;
-        struct iatt          postbuf;
-        dht_defrag_cbk_fn_t  target_op_fn;
-        dict_t              *xdata;
-        dict_t              *xattr;
-        int32_t              set;
-        struct gf_flock      flock;
-        int                  lock_cmd;
+    xlator_t *from_subvol;
+    xlator_t *target_node;
+    off_t offset;
+    size_t size;
+    int32_t flags;
+    int count;
+    struct iobref *iobref;
+    struct iovec *vector;
+    struct iatt stbuf;
+    struct iatt prebuf;
+    struct iatt postbuf;
+    dht_defrag_cbk_fn_t target_op_fn;
+    dict_t *xdata;
+    dict_t *xattr;
+    dict_t *dict;
+    struct gf_flock flock;
+    int32_t set;
+    int lock_cmd;
 };
 
 /**
@@ -138,1095 +158,1227 @@ struct dht_rebalance_ {
  * events
  **/
 typedef enum {
-        qdstatfs_action_OFF = 0,
-        qdstatfs_action_REPLACE,
-        qdstatfs_action_NEGLECT,
-        qdstatfs_action_COMPARE,
+    qdstatfs_action_OFF = 0,
+    qdstatfs_action_REPLACE,
+    qdstatfs_action_NEGLECT,
+    qdstatfs_action_COMPARE,
 } qdstatfs_action_t;
 
 typedef enum {
-        FAIL_ON_ANY_ERROR,
-        IGNORE_ENOENT_ESTALE
+    REACTION_INVALID,
+    FAIL_ON_ANY_ERROR,
+    IGNORE_ENOENT_ESTALE,
+    IGNORE_ENOENT_ESTALE_EIO,
 } dht_reaction_type_t;
 
 struct dht_skip_linkto_unlink {
-
-        gf_boolean_t    handle_valid_link;
-        int             opend_fd_count;
-        xlator_t        *hash_links_to;
-        uuid_t          cached_gfid;
-        uuid_t          hashed_gfid;
+    xlator_t *hash_links_to;
+    uuid_t cached_gfid;
+    uuid_t hashed_gfid;
+    int opend_fd_count;
+    gf_boolean_t handle_valid_link;
 };
 
 typedef struct {
-        xlator_t     *xl;
-        loc_t         loc;     /* contains/points to inode to lock on. */
-        short         type;    /* read/write lock.                     */
-        char         *domain;  /* Only locks within a single domain
-                                * contend with each other
-                                */
-        gf_lkowner_t  lk_owner;
-        gf_boolean_t  locked;
+    xlator_t *xl;
+    loc_t loc;      /* contains/points to inode to lock on. */
+    char *domain;   /* Only locks within a single domain
+                     * contend with each other
+                     */
+    char *basename; /* Required for entrylk */
+    gf_boolean_t locked;
+    dht_reaction_type_t do_on_failure;
+    short type; /* read/write lock.                     */
+    gf_lkowner_t lk_owner;
 } dht_lock_t;
 
-typedef
-int (*dht_selfheal_layout_t)(call_frame_t *frame, loc_t *loc,
-                             dht_layout_t *layout);
-
-typedef
-gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame, dht_layout_t **inmem,
-                                dht_layout_t **ondisk);
-
-struct dht_local {
-        int                      call_cnt;
-        loc_t                    loc;
-        loc_t                    loc2;
-        int                      op_ret;
-        int                      op_errno;
-        int                      layout_mismatch;
-        /* Use stbuf as the postbuf, when we require both
-         * pre and post attrs */
-        struct iatt              stbuf;
-        struct iatt              prebuf;
-        struct iatt              preoldparent;
-        struct iatt              postoldparent;
-        struct iatt              preparent;
-        struct iatt              postparent;
-        struct statvfs           statvfs;
-        fd_t                    *fd;
-        inode_t                 *inode;
-        dict_t                  *params;
-        dict_t                  *xattr;
-        dict_t                  *xattr_req;
-        dht_layout_t            *layout;
-        size_t                   size;
-        ino_t                    ia_ino;
-        xlator_t                *src_hashed, *src_cached;
-        xlator_t                *dst_hashed, *dst_cached;
-        xlator_t                *cached_subvol;
-        xlator_t                *hashed_subvol;
-        char                     need_selfheal;
-        int                      file_count;
-        int                      dir_count;
-        call_frame_t            *main_frame;
-        int                      fop_succeeded;
-        struct {
-                fop_mknod_cbk_t  linkfile_cbk;
-                struct iatt      stbuf;
-                loc_t            loc;
-                inode_t         *inode;
-                dict_t          *xattr;
-                xlator_t        *srcvol;
-        } linkfile;
-        struct {
-                uint32_t                hole_cnt;
-                uint32_t                overlaps_cnt;
-                uint32_t                down;
-                uint32_t                misc;
-                dht_selfheal_dir_cbk_t  dir_cbk;
-                dht_selfheal_layout_t   healer;
-                dht_need_heal_t         should_heal;
-                gf_boolean_t            force_mkdir;
-                dht_layout_t           *layout, *refreshed_layout;
-        } selfheal;
-
-        dht_refresh_layout_unlock              refresh_layout_unlock;
-        dht_refresh_layout_done_handle         refresh_layout_done;
-
-        uint32_t                 uid;
-        uint32_t                 gid;
-
-        /* needed by nufa */
-        int32_t flags;
-        mode_t  mode;
-        dev_t   rdev;
-        mode_t  umask;
-
-        /* need for file-info */
-        char   *xattr_val;
-        char   *key;
-
-        /* which xattr request? */
-        char xsel[256];
-        int32_t alloc_len;
-
-        char   *newpath;
-
-        /* gfid related */
-        uuid_t  gfid;
-
-        /* flag used to make sure we need to return estale in
-           {lookup,revalidate}_cbk */
-        char return_estale;
-        char need_lookup_everywhere;
-
-        glusterfs_fop_t      fop;
-
-        gf_boolean_t     linked;
-        xlator_t        *link_subvol;
-
-        struct dht_rebalance_ rebalance;
-        xlator_t        *first_up_subvol;
-
-        gf_boolean_t     quota_deem_statfs;
-
-        gf_boolean_t     added_link;
-        gf_boolean_t     is_linkfile;
-
-        struct dht_skip_linkto_unlink  skip_unlink;
+/* The lock structure represents inodelk. */
+typedef struct {
+    fop_inodelk_cbk_t inodelk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
 
-        struct {
-                fop_inodelk_cbk_t   inodelk_cbk;
-                dht_lock_t        **locks;
-                int                 lk_count;
-                dht_reaction_type_t reaction;
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
+} dht_ilock_wrap_t;
 
-                /* whether locking failed on _any_ of the "locks" above */
-                int                 op_ret;
-                int                 op_errno;
-        } lock;
+/* The lock structure represents entrylk. */
+typedef struct {
+    fop_entrylk_cbk_t entrylk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
+
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
+} dht_elock_wrap_t;
+
+/* The first member of dht_dir_transaction_t should be of type dht_ilock_wrap_t.
+ * Otherwise it can result in subtle memory corruption issues as in most of the
+ * places we use lock[0].layout.my_layout or lock[0].layout.parent_layout and
+ * lock[0].ns.parent_layout (like in dht_local_wipe).
+ */
+typedef union {
+    union {
+        dht_ilock_wrap_t my_layout;
+        dht_ilock_wrap_t parent_layout;
+    } layout;
+    struct dht_namespace {
+        dht_ilock_wrap_t parent_layout;
+        dht_elock_wrap_t directory_ns;
+        fop_entrylk_cbk_t ns_cbk;
+    } ns;
+} dht_dir_transaction_t;
+
+typedef int (*dht_selfheal_layout_t)(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *layout);
 
-        short           lock_type;
+typedef gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame,
+                                        dht_layout_t **inmem,
+                                        dht_layout_t **ondisk);
 
-        call_stub_t *stub;
-        int32_t      parent_disk_layout[4];
+struct dht_local {
+    loc_t loc;
+    loc_t loc2;
+    int call_cnt;
+    int op_ret;
+    int op_errno;
+    int layout_mismatch;
+    /* Use stbuf as the postbuf, when we require both
+     * pre and post attrs */
+    struct iatt stbuf;
+    struct iatt mds_stbuf;
+    struct iatt prebuf;
+    struct iatt preoldparent;
+    struct iatt postoldparent;
+    struct iatt preparent;
+    struct iatt postparent;
+    struct statvfs statvfs;
+    fd_t *fd;
+    inode_t *inode;
+    dict_t *params;
+    dict_t *xattr;
+    dict_t *mds_xattr;
+    dict_t *xdata; /* dict used to save xdata response by xattr fop */
+    dict_t *xattr_req;
+    dht_layout_t *layout;
+    size_t size;
+    ino_t ia_ino;
+    xlator_t *src_hashed, *src_cached;
+    xlator_t *dst_hashed, *dst_cached;
+    xlator_t *cached_subvol;
+    xlator_t *hashed_subvol;
+    xlator_t *mds_subvol; /* This is use for dir only */
+    int file_count;
+    int dir_count;
+    call_frame_t *main_frame;
+    int fop_succeeded;
+    struct {
+        fop_mknod_cbk_t linkfile_cbk;
+        struct iatt stbuf;
+        loc_t loc;
+        inode_t *inode;
+        dict_t *xattr;
+        xlator_t *srcvol;
+    } linkfile;
+    struct {
+        uint32_t hole_cnt;
+        uint32_t overlaps_cnt;
+        uint32_t down;
+        uint32_t misc;
+        dht_selfheal_dir_cbk_t dir_cbk;
+        dht_selfheal_layout_t healer;
+        dht_need_heal_t should_heal;
+        dht_layout_t *layout, *refreshed_layout;
+        uint32_t missing_cnt;
+        gf_boolean_t force_mkdir;
+    } selfheal;
+
+    dht_refresh_layout_unlock refresh_layout_unlock;
+    dht_refresh_layout_done_handle refresh_layout_done;
+
+    uint32_t uid;
+    uint32_t gid;
+    pid_t pid;
+
+    glusterfs_fop_t fop;
+
+    /* need for file-info */
+    char *xattr_val;
+    char *key;
+
+    /* needed by nufa */
+    int32_t flags;
+    mode_t mode;
+    dev_t rdev;
+    mode_t umask;
+
+    /* which xattr request? */
+    char xsel[256];
+    int32_t alloc_len;
+
+    /* gfid related */
+    uuid_t gfid;
+    uuid_t gfid_req;
+
+    xlator_t *link_subvol;
+
+    struct dht_rebalance_ rebalance;
+    xlator_t *first_up_subvol;
+
+    struct dht_skip_linkto_unlink skip_unlink;
+
+    dht_dir_transaction_t lock[2], *current;
+
+    /* inodelks during filerename for backward compatibility */
+    dht_lock_t **rename_inodelk_backward_compatible;
+
+    call_stub_t *stub;
+    int32_t parent_disk_layout[4];
+
+    /* rename rollback */
+    int *ret_cache;
+
+    loc_t loc2_copy;
+
+    int rename_inodelk_bc_count;
+    /* This is use only for directory operation */
+    int32_t valid;
+    int32_t mds_heal_fresh_lookup;
+    short lock_type;
+    char need_selfheal;
+    char need_xattr_heal;
+    char need_attrheal;
+    /* flag used to make sure we need to return estale in
+       {lookup,revalidate}_cbk */
+    char return_estale;
+    char need_lookup_everywhere;
+    /* fd open check */
+    gf_boolean_t fd_checked;
+    gf_boolean_t linked;
+    gf_boolean_t added_link;
+    gf_boolean_t is_linkfile;
+    gf_boolean_t quota_deem_statfs;
+    gf_boolean_t heal_layout;
+    gf_boolean_t locked;
+    gf_boolean_t dont_create_linkto;
+    gf_boolean_t gfid_missing;
 };
 typedef struct dht_local dht_local_t;
 
 /* du - disk-usage */
 struct dht_du {
-        double   avail_percent;
-	double   avail_inodes;
-        uint64_t avail_space;
-        uint32_t log;
-        uint32_t chunks;
+    double avail_percent;
+    double avail_inodes;
+    uint64_t avail_space;
+    uint32_t log;
+    uint32_t chunks;
+    uint32_t total_blocks;
+    uint32_t avail_blocks;
+    uint32_t frsize; /*fragment size*/
 };
 typedef struct dht_du dht_du_t;
 
 enum gf_defrag_type {
-        GF_DEFRAG_CMD_START = 1,
-        GF_DEFRAG_CMD_STOP = 1 + 1,
-        GF_DEFRAG_CMD_STATUS = 1 + 2,
-        GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
-        GF_DEFRAG_CMD_START_FORCE = 1 + 4,
-        GF_DEFRAG_CMD_START_TIER = 1 + 5,
-        GF_DEFRAG_CMD_STATUS_TIER = 1 + 6,
-	GF_DEFRAG_CMD_START_DETACH_TIER = 1 + 7,
-	GF_DEFRAG_CMD_STOP_DETACH_TIER = 1 + 8,
-	GF_DEFRAG_CMD_PAUSE_TIER = 1 + 9,
-	GF_DEFRAG_CMD_RESUME_TIER = 1 + 10,
+    GF_DEFRAG_CMD_NONE = 0,
+    GF_DEFRAG_CMD_START = 1,
+    GF_DEFRAG_CMD_STOP = 1 + 1,
+    GF_DEFRAG_CMD_STATUS = 1 + 2,
+    GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
+    GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+    GF_DEFRAG_CMD_DETACH_STATUS = 1 + 11,
+    GF_DEFRAG_CMD_DETACH_START = 1 + 13,
+    GF_DEFRAG_CMD_DETACH_COMMIT = 1 + 14,
+    GF_DEFRAG_CMD_DETACH_COMMIT_FORCE = 1 + 15,
+    GF_DEFRAG_CMD_DETACH_STOP = 1 + 16,
+    /* new labels are used so it will help
+     * while removing old labels by easily differentiating.
+     * A few labels are added so that the count remains same
+     * between this enum and the ones on the xdr file.
+     * different values for the same enum cause errors and
+     * confusion.
+     */
 };
 typedef enum gf_defrag_type gf_defrag_type;
 
 enum gf_defrag_status_t {
-        GF_DEFRAG_STATUS_NOT_STARTED,
-        GF_DEFRAG_STATUS_STARTED,
-        GF_DEFRAG_STATUS_STOPPED,
-        GF_DEFRAG_STATUS_COMPLETE,
-        GF_DEFRAG_STATUS_FAILED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
+    GF_DEFRAG_STATUS_NOT_STARTED,
+    GF_DEFRAG_STATUS_STARTED,
+    GF_DEFRAG_STATUS_STOPPED,
+    GF_DEFRAG_STATUS_COMPLETE,
+    GF_DEFRAG_STATUS_FAILED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
 typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
 
 struct gf_defrag_pattern_list {
-        char                       path_pattern[256];
-        uint64_t                   size;
-        gf_defrag_pattern_list_t  *next;
+    char path_pattern[256];
+    uint64_t size;
+    gf_defrag_pattern_list_t *next;
 };
 
 struct dht_container {
-        union {
-                struct list_head             list;
-                struct {
-                        struct _gf_dirent_t *next;
-                        struct _gf_dirent_t *prev;
-                };
+    union {
+        struct list_head list;
+        struct {
+            struct _gf_dirent_t *next;
+            struct _gf_dirent_t *prev;
         };
-        gf_dirent_t     *df_entry;
-        xlator_t        *this;
-        loc_t           *parent_loc;
-        dict_t          *migrate_data;
+    };
+    gf_dirent_t *df_entry;
+    xlator_t *this;
+    loc_t *parent_loc;
+    dict_t *migrate_data;
+    int local_subvol_index;
 };
 
-typedef enum tier_mode_ {
-        TIER_MODE_NONE = 0,
-        TIER_MODE_TEST,
-        TIER_MODE_WM
-} tier_mode_t;
-
-typedef enum tier_pause_state_ {
-        TIER_RUNNING = 0,
-        TIER_REQUEST_PAUSE,
-        TIER_PAUSED
-} tier_pause_state_t;
-
-/* This Structure is only used in tiering fixlayout */
-typedef struct gf_tier_fix_layout_arg {
-        xlator_t                *this;
-        dict_t                  *fix_layout;
-        pthread_t               thread_id;
-} gf_tier_fix_layout_arg_t;
-
-typedef struct gf_tier_conf {
-        int                          is_tier;
-        int                          watermark_hi;
-        int                          watermark_low;
-        int                          watermark_last;
-        fsblkcnt_t                   blocks_total;
-        fsblkcnt_t                   blocks_used;
-        int                          percent_full;
-        uint64_t                     max_migrate_bytes;
-        int                          max_migrate_files;
-        tier_mode_t                  mode;
-        int                          tier_max_promote_size;
-        int                          tier_promote_frequency;
-        int                          tier_demote_frequency;
-        uint64_t                     st_last_promoted_size;
-        uint64_t                     st_last_demoted_size;
-        tier_pause_state_t           pause_state;
-        struct synctask             *pause_synctask;
-        gf_timer_t                  *pause_timer;
-        pthread_mutex_t              pause_mutex;
-        int                          promote_in_progress;
-        int                          demote_in_progress;
-        /* This Structure is only used in tiering fixlayout */
-        gf_tier_fix_layout_arg_t     tier_fix_layout_arg;
-        /* Indicates the index of the first queryfile picked
-         * in the last cycle of promote or demote */
-        int32_t last_promote_qfile_index;
-        int32_t last_demote_qfile_index;
-} gf_tier_conf_t;
+typedef struct nodeuuid_info {
+    char info;   /* Set to 1 is this is my node's uuid*/
+    uuid_t uuid; /* Store the nodeuuid as well for debugging*/
+} nodeuuid_info_t;
+
+typedef struct subvol_nodeuuids_info {
+    nodeuuid_info_t *elements;
+    int count;
+} subvol_nodeuuids_info_t;
 
 struct gf_defrag_info_ {
-        uint64_t                     total_files;
-        uint64_t                     total_data;
-        uint64_t                     num_files_lookedup;
-        uint64_t                     total_failures;
-        uint64_t                     skipped;
-        gf_lock_t                    lock;
-        int                          cmd;
-        pthread_t                    th;
-        gf_defrag_status_t           defrag_status;
-        struct rpc_clnt             *rpc;
-        uint32_t                     connected;
-        uint32_t                     is_exiting;
-        pid_t                        pid;
-        inode_t                     *root_inode;
-        uuid_t                       node_uuid;
-        struct timeval               start_time;
-        gf_boolean_t                 stats;
-        uint32_t                     new_commit_hash;
-        gf_defrag_pattern_list_t    *defrag_pattern;
-        gf_tier_conf_t               tier_conf;
-
-        /*Data Tiering params for scanner*/
-        uint64_t                     total_files_promoted;
-        uint64_t                     total_files_demoted;
-        int                          write_freq_threshold;
-        int                          read_freq_threshold;
-
-        pthread_cond_t               parallel_migration_cond;
-        pthread_mutex_t              dfq_mutex;
-        pthread_cond_t               rebalance_crawler_alarm;
-        int32_t                      q_entry_count;
-        int32_t                      global_error;
-        struct  dht_container       *queue;
-        int32_t                      crawl_done;
-        int32_t                      abort;
-        int32_t                      wakeup_crawler;
-
-        /*Throttle params*/
-        /*stands for reconfigured thread count*/
-        int32_t                      recon_thread_count;
-        /*stands for current running thread count*/
-        int32_t                      current_thread_count;
-        pthread_cond_t               df_wakeup_thread;
-
-        /* Hard link handle requirement */
-        synclock_t                   link_lock;
-
-        /* lock migration flag */
-        gf_boolean_t                 lock_migration_enabled;
+    uint64_t total_files;
+    uint64_t total_data;
+    uint64_t num_files_lookedup;
+    uint64_t total_failures;
+    uint64_t skipped;
+    uint64_t num_dirs_processed;
+    uint64_t size_processed;
+    gf_lock_t lock;
+    pthread_t th;
+    struct rpc_clnt *rpc;
+    uint32_t connected;
+    uint32_t is_exiting;
+    pid_t pid;
+    int cmd;
+    inode_t *root_inode;
+    uuid_t node_uuid;
+    time_t start_time;
+    uint32_t new_commit_hash;
+    gf_defrag_status_t defrag_status;
+    gf_defrag_pattern_list_t *defrag_pattern;
+
+    pthread_cond_t parallel_migration_cond;
+    pthread_mutex_t dfq_mutex;
+    pthread_cond_t rebalance_crawler_alarm;
+    int32_t q_entry_count;
+    int32_t global_error;
+    struct dht_container *queue;
+    int32_t crawl_done;
+    int32_t abort;
+    int32_t wakeup_crawler;
+
+    /*Throttle params*/
+    /*stands for reconfigured thread count*/
+    int32_t recon_thread_count;
+    pthread_cond_t df_wakeup_thread;
+
+    /* backpointer to make it easier to write functions for rebalance */
+    xlator_t *this;
+
+    pthread_cond_t fc_wakeup_cond;
+    pthread_mutex_t fc_mutex;
+
+    /*stands for current running thread count*/
+    int32_t current_thread_count;
+
+    gf_boolean_t stats;
+    /* lock migration flag */
+    gf_boolean_t lock_migration_enabled;
 };
 
 typedef struct gf_defrag_info_ gf_defrag_info_t;
 
 struct dht_methods_s {
-        int32_t      (*migration_get_dst_subvol)(xlator_t *this,
-                                                 dht_local_t *local);
-        int32_t      (*migration_other)(xlator_t *this,
-                                        gf_defrag_info_t *defrag);
-        int32_t      (*migration_needed)(xlator_t *this);
-        xlator_t*    (*layout_search)(xlator_t *this,
-                                      dht_layout_t *layout,
-                                         const char *name);
+    int32_t (*migration_get_dst_subvol)(xlator_t *this, dht_local_t *local);
+    int32_t (*migration_other)(xlator_t *this, gf_defrag_info_t *defrag);
+    xlator_t *(*layout_search)(xlator_t *this, dht_layout_t *layout,
+                               const char *name);
 };
 
 typedef struct dht_methods_s dht_methods_t;
 
 struct dht_conf {
-        gf_lock_t      subvolume_lock;
-        int            subvolume_cnt;
-        xlator_t     **subvolumes;
-        char          *subvolume_status;
-        int           *last_event;
-        dht_layout_t **file_layouts;
-        dht_layout_t **dir_layouts;
-        unsigned int   search_unhashed;
-        gf_boolean_t   lookup_optimize;
-        int            gen;
-        dht_du_t      *du_stats;
-        double         min_free_disk;
-        double         min_free_inodes;
-        char           disk_unit;
-        int32_t        refresh_interval;
-        gf_boolean_t   unhashed_sticky_bit;
-        struct timeval last_stat_fetch;
-        gf_lock_t      layout_lock;
-        dict_t        *leaf_to_subvol;
-        void          *private;     /* Can be used by wrapper xlators over
-                                       dht */
-        gf_boolean_t   use_readdirp;
-        char           vol_uuid[UUID_SIZE + 1];
-        gf_boolean_t   assert_no_child_down;
-        time_t        *subvol_up_time;
-
-        /* This is the count used as the distribute layout for a directory */
-        /* Will be a global flag to control the layout spread count */
-        uint32_t       dir_spread_cnt;
-
-        /* to keep track of nodes which are decommissioned */
-        xlator_t     **decommissioned_bricks;
-        int            decommission_in_progress;
-        int            decommission_subvols_cnt;
-
-        /* defrag related */
-        gf_defrag_info_t *defrag;
-
-        /* Request to filter directory entries in readdir request */
-
-        gf_boolean_t    readdir_optimize;
-
-        /* Support regex-based name reinterpretation. */
-        regex_t         rsync_regex;
-        gf_boolean_t    rsync_regex_valid;
-        regex_t         extra_regex;
-        gf_boolean_t    extra_regex_valid;
-
-        /* Support variable xattr names. */
-        char            *xattr_name;
-        char            *link_xattr_name;
-        char            *commithash_xattr_name;
-        char            *wild_xattr_name;
-
-        /* Support size-weighted rebalancing (heterogeneous bricks). */
-        gf_boolean_t    do_weighting;
-        gf_boolean_t    randomize_by_gfid;
-        char           *dthrottle;
-
-        dht_methods_t   methods;
-
-        struct mem_pool *lock_pool;
-
-        /*local subvol storage for rebalance*/
-        xlator_t       **local_subvols;
-        int32_t          local_subvols_cnt;
-
-        /*
-         * "Commit hash" for this volume topology.  Changed whenever bricks
-         * are added or removed.
-         */
-        uint32_t        vol_commit_hash;
-        gf_boolean_t    vch_forced;
-
-        /* lock migration */
-
-        gf_boolean_t    lock_migration_enabled;
+    xlator_t **subvolumes;
+    char *subvolume_status;
+    int *last_event;
+    dht_layout_t **file_layouts;
+    dht_layout_t **dir_layouts;
+    unsigned int search_unhashed;
+    int gen;
+    dht_du_t *du_stats;
+    double min_free_disk;
+    double min_free_inodes;
+    int subvolume_cnt;
+    int32_t refresh_interval;
+    gf_lock_t subvolume_lock;
+    time_t last_stat_fetch;
+    gf_lock_t layout_lock;
+    dict_t *leaf_to_subvol;
+    void *private; /* Can be used by wrapper xlators over
+                      dht */
+    time_t *subvol_up_time;
+
+    /* to keep track of nodes which are decommissioned */
+    xlator_t **decommissioned_bricks;
+    int decommission_in_progress;
+    int decommission_subvols_cnt;
+
+    /* defrag related */
+    gf_defrag_info_t *defrag;
+
+    /* Support regex-based name reinterpretation. */
+    regex_t rsync_regex;
+    regex_t extra_regex;
+
+    /* Support variable xattr names. */
+    char *xattr_name;
+    char *mds_xattr_key;
+    char *link_xattr_name;
+    char *commithash_xattr_name;
+    char *wild_xattr_name;
+
+    dht_methods_t methods;
+
+    struct mem_pool *lock_pool;
+
+    /*local subvol storage for rebalance*/
+    xlator_t **local_subvols;
+    subvol_nodeuuids_info_t *local_nodeuuids;
+    int32_t local_subvols_cnt;
+
+    int dthrottle;
+
+    /* Hard link handle requirement for migration triggered from client*/
+    synclock_t link_lock;
+
+    /* lock migration */
+    gf_lock_t lock;
+
+    /* This is the count used as the distribute layout for a directory */
+    /* Will be a global flag to control the layout spread count */
+    uint32_t dir_spread_cnt;
+
+    /*
+     * "Commit hash" for this volume topology.  Changed whenever bricks
+     * are added or removed.
+     */
+    uint32_t vol_commit_hash;
+
+    char vol_uuid[UUID_SIZE + 1];
+
+    char disk_unit;
+
+    gf_boolean_t lock_migration_enabled;
+
+    gf_boolean_t vch_forced;
+
+    gf_boolean_t use_fallocate;
+
+    gf_boolean_t force_migration;
+
+    gf_boolean_t lookup_optimize;
+
+    gf_boolean_t unhashed_sticky_bit;
+
+    gf_boolean_t assert_no_child_down;
+
+    gf_boolean_t use_readdirp;
+
+    /* Request to filter directory entries in readdir request */
+    gf_boolean_t readdir_optimize;
+
+    gf_boolean_t rsync_regex_valid;
+
+    gf_boolean_t extra_regex_valid;
+
+    /* Support size-weighted rebalancing (heterogeneous bricks). */
+    gf_boolean_t do_weighting;
+
+    gf_boolean_t randomize_by_gfid;
 };
 typedef struct dht_conf dht_conf_t;
 
 struct dht_dfoffset_ctx {
-        xlator_t       *this;
-        off_t           offset;
-        int32_t         readdir_done;
+    xlator_t *this;
+    off_t offset;
+    int32_t readdir_done;
 };
 typedef struct dht_dfoffset_ctx dht_dfoffset_ctx_t;
 
 struct dht_disk_layout {
-        uint32_t           cnt;
-        uint32_t           type;
-        struct {
-                uint32_t   start;
-                uint32_t   stop;
-        } list[1];
+    uint32_t cnt;
+    uint32_t type;
+    struct {
+        uint32_t start;
+        uint32_t stop;
+    } list[1];
 };
 typedef struct dht_disk_layout dht_disk_layout_t;
 
 typedef enum {
-        GF_DHT_MIGRATE_DATA,
-        GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
-        GF_DHT_MIGRATE_HARDLINK,
-        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
+    GF_DHT_MIGRATE_DATA,
+    GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
+    GF_DHT_MIGRATE_HARDLINK,
+    GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
 } gf_dht_migrate_data_type_t;
 
 typedef enum {
-        GF_DHT_EQUAL_DISTRIBUTION,
-        GF_DHT_WEIGHTED_DISTRIBUTION
+    GF_DHT_EQUAL_DISTRIBUTION,
+    GF_DHT_WEIGHTED_DISTRIBUTION
 } dht_distribution_type_t;
 
 struct dir_dfmeta {
-        gf_dirent_t             *equeue;
-        dht_dfoffset_ctx_t      *offset_var;
-        struct list_head        **head;
-        struct list_head        **iterator;
-        int                     *fetch_entries;
+    gf_dirent_t *equeue;
+    dht_dfoffset_ctx_t *offset_var;
+    struct list_head **head;
+    struct list_head **iterator;
+    int *fetch_entries;
+    /* fds corresponding to local subvols only */
+    fd_t **lfd;
 };
 
 typedef struct dht_migrate_info {
-        xlator_t *src_subvol;
-        xlator_t *dst_subvol;
-        GF_REF_DECL;
+    xlator_t *src_subvol;
+    xlator_t *dst_subvol;
+    GF_REF_DECL;
 } dht_migrate_info_t;
 
-
-
 typedef struct dht_fd_ctx {
-        uint64_t opened_on_dst;
-        GF_REF_DECL;
+    uint64_t opened_on_dst;
+    GF_REF_DECL;
 } dht_fd_ctx_t;
 
-
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc)                                                     \
+    (dht_inode_ctx_layout_get((loc)->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
-#define DHT_MIGRATION_COMPLETED   2
+#define DHT_MIGRATION_COMPLETED 2
 
-#define check_is_linkfile(i,s,x,n) (IS_DHT_LINKFILE_MODE (s) && dict_get (x, n))
+#define check_is_linkfile(i, s, x, n)                                          \
+    (IS_DHT_LINKFILE_MODE(s) && dict_get(x, n))
 
-#define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((st_mode_from_ia ((buf)->ia_prot, (buf)->ia_type) &    \
-                  ~S_IFMT) == DHT_LINKFILE_MODE))
+#define IS_DHT_MIGRATION_PHASE2(buf)                                           \
+    (IA_ISREG((buf)->ia_type) &&                                               \
+     ((st_mode_from_ia((buf)->ia_prot, (buf)->ia_type) & ~S_IFMT) ==           \
+      DHT_LINKFILE_MODE))
 
-#define IS_DHT_MIGRATION_PHASE1(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((buf)->ia_prot.sticky == 1) &&                         \
-                ((buf)->ia_prot.sgid == 1))
+#define IS_DHT_MIGRATION_PHASE1(buf)                                           \
+    (IA_ISREG((buf)->ia_type) && ((buf)->ia_prot.sticky == 1) &&               \
+     ((buf)->ia_prot.sgid == 1))
 
-#define DHT_STRIP_PHASE1_FLAGS(buf)  do {                       \
-                if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {    \
-                        (buf)->ia_prot.sticky = 0;              \
-                        (buf)->ia_prot.sgid = 0;                \
-                }                                               \
-        } while (0)
+#define DHT_STRIP_PHASE1_FLAGS(buf)                                            \
+    do {                                                                       \
+        if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {                           \
+            (buf)->ia_prot.sticky = 0;                                         \
+            (buf)->ia_prot.sgid = 0;                                           \
+        }                                                                      \
+    } while (0)
 
-#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE \
-                                     || op_errno == EIO) \
-/*Bad fix. Please revert the commit after fixing the bug 1329505*/
+#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE)
 
-#define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
+#define check_is_dir(i, s, x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
 
-#define we_are_not_migrating(x)   ((x) == 1)
-
-#define DHT_STACK_UNWIND(fop, frame, params ...) do {           \
-                dht_local_t *__local = NULL;                    \
-                xlator_t    *__xl    = NULL;                    \
-                if (frame) {                                    \
-                        __xl         = frame->this;             \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                dht_local_wipe (__xl, __local);                 \
-        } while (0)
-
-#define DHT_STACK_DESTROY(frame) do {                   \
-                dht_local_t *__local = NULL;            \
-                xlator_t    *__xl    = NULL;            \
-                __xl                 = frame->this;     \
-                __local              = frame->local;    \
-                frame->local         = NULL;            \
-                STACK_DESTROY (frame->root);            \
-                dht_local_wipe (__xl, __local);         \
-        } while (0)
-
-#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
-                LOCK (&inode->lock);                                    \
-                {                                                       \
-                        if (ctx_sec == new_sec)                         \
-                                new_nsec = max (new_nsec, ctx_nsec);    \
-                        else if (ctx_sec > new_sec) {                   \
-                                new_sec = ctx_sec;                      \
-                                new_nsec = ctx_nsec;                    \
-                        }                                               \
-                        if (post) {                                     \
-                                ctx_sec = new_sec;                      \
-                                ctx_nsec = new_nsec;                    \
-                        }                                               \
-                }                                                       \
-                UNLOCK (&inode->lock);                                  \
-        } while (0)
-
-#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
-
-#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
-                int tmp = -1;                                                  \
-                if (!xattr) {                                                  \
-                        xattr = dict_new ();                                   \
-                        if (!xattr)                                            \
-                                break;                                         \
-                }                                                              \
-                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
-                if (tmp) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                   \
-                                DHT_MSG_DICT_SET_FAILED,                       \
-                                "Failed to set dictionary value: key = %s,"    \
-                                " path = %s", GLUSTERFS_INTERNAL_FOP_KEY,      \
-                                 local->loc.path);                             \
-                }                                                              \
-        } while (0)
-
-dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
-dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
-dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_layout_search (xlator_t   *this, dht_layout_t *layout,
-                             const char *name);
+#define we_are_not_migrating(x) ((x) == 1)
+
+#define DHT_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, post)            \
+    do {                                                                       \
+        if (ctx_sec == new_sec)                                                \
+            new_nsec = max(new_nsec, ctx_nsec);                                \
+        else if (ctx_sec > new_sec) {                                          \
+            new_sec = ctx_sec;                                                 \
+            new_nsec = ctx_nsec;                                               \
+        }                                                                      \
+        if (post) {                                                            \
+            ctx_sec = new_sec;                                                 \
+            ctx_nsec = new_nsec;                                               \
+        }                                                                      \
+    } while (0)
+
+#define is_greater_time(a, an, b, bn)                                          \
+    (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
+
+#define DHT_MARK_FOP_INTERNAL(xattr)                                           \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes");          \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_INTERNAL_FOP_KEY, local->loc.path);               \
+        }                                                                      \
+    } while (0)
+
+dht_layout_t *
+dht_layout_new(xlator_t *this, int cnt);
+dht_layout_t *
+dht_layout_get(xlator_t *this, inode_t *inode);
+dht_layout_t *
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name);
 int32_t
-dht_migration_get_dst_subvol(xlator_t *this, dht_local_t  *local);
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local);
 int32_t
 dht_migration_needed(xlator_t *this);
-int                                      dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
-int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
-                          uint32_t      *holes_p, uint32_t *overlaps_p,
-                          uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p, uint32_t *no_space_p);
-int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
-                             xlator_t   *subvol, loc_t *loc, dict_t *xattr);
-
-xlator_t *dht_linkfile_subvol (xlator_t *this, inode_t *inode,
-                               struct iatt *buf, dict_t *xattr);
-int dht_linkfile_unlink (call_frame_t      *frame, xlator_t *this,
-                         xlator_t          *subvol, loc_t *loc);
-
-int dht_layouts_init (xlator_t *this, dht_conf_t *conf);
-int dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                      int       op_ret, int op_errno, dict_t *xattr);
-
-int     dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                             int       pos, int32_t **disk_layout_p);
-int dht_disk_layout_merge (xlator_t   *this, dht_layout_t *layout,
-                           int         pos, void *disk_layout_raw, int disk_layout_len);
 int
-dht_disk_layout_extract_for_subvol (xlator_t *this, dht_layout_t *layout,
-                                    xlator_t *subvol, int32_t **disk_layout_p);
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p);
+int
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr);
+xlator_t *
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *buf,
+                    dict_t *xattr);
+int
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc);
 
-int dht_frame_return (call_frame_t *frame);
+int
+dht_layouts_init(xlator_t *this, dht_conf_t *conf);
+int
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr);
 
-int dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol);
+int
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p);
+int
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p);
 
-void dht_local_wipe (xlator_t *this, dht_local_t *local);
-dht_local_t *dht_local_init (call_frame_t    *frame, loc_t *loc, fd_t *fd,
-                             glusterfs_fop_t  fop);
-int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt *from,
-                    xlator_t                 *subvol);
+int
+dht_frame_return(call_frame_t *frame);
 
-xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
-xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
-xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
-xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
-int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
+int
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol);
 
-int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
+void
+dht_local_wipe(xlator_t *this, dht_local_t *local);
+dht_local_t *
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop);
+int
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from);
 
-int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *this, xlator_t *tovol,
-                         xlator_t        *fromvol, loc_t *loc);
-int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
-int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
+xlator_t *
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc);
+xlator_t *
+dht_subvol_get_cached(xlator_t *this, inode_t *inode);
+xlator_t *
+dht_subvol_next(xlator_t *this, xlator_t *prev);
+xlator_t *
+dht_subvol_next_available(xlator_t *this, xlator_t *prev);
 int
-dht_selfheal_directory (call_frame_t     *frame, dht_selfheal_dir_cbk_t cbk,
-                        loc_t            *loc, dht_layout_t *layout);
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol);
 
 int
-dht_selfheal_directory_for_nameless_lookup (call_frame_t  *frame,
-                                            dht_selfheal_dir_cbk_t cbk,
-                                            loc_t  *loc, dht_layout_t *layout);
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
 int
-dht_selfheal_new_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-                            dht_layout_t *layout);
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc);
+int
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
-dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
-                      loc_t              *loc, dht_layout_t *layout);
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                       loc_t *loc, dht_layout_t *layout);
 int
-dht_layout_sort_volname (dht_layout_t *layout);
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                           dht_layout_t *layout);
+int
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                     loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_sort_volname(dht_layout_t *layout);
 
-int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
+int
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
 
-gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
-                                          dht_local_t *layout);
-int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
+gf_boolean_t
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *layout);
+int
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx);
 
-int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int dht_layout_index_for_subvol (dht_layout_t *layout, xlator_t *subvol);
-int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
-void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
-dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
-xlator_t     *dht_first_up_subvol (xlator_t *this);
-xlator_t     *dht_last_up_subvol (xlator_t *this);
+int
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode);
+int
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout);
+;
+void
+dht_layout_unref(xlator_t *this, dht_layout_t *layout);
+dht_layout_t *
+dht_layout_ref(xlator_t *this, dht_layout_t *layout);
+int
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol);
+xlator_t *
+dht_first_up_subvol(xlator_t *this);
+xlator_t *
+dht_last_up_subvol(xlator_t *this);
 
-int dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
+int
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name);
 
-int dht_filter_loc_subvol_key (xlator_t  *this, loc_t *loc, loc_t *new_loc,
-                               xlator_t **subvol);
+int
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol);
 
-int                                     dht_rename_cleanup (call_frame_t *frame);
-int dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t           op_ret, int32_t op_errno,
-                         inode_t          *inode, struct iatt *stbuf,
-                         struct iatt      *preparent, struct iatt *postparent,
-                         dict_t *xdata);
+int
+dht_rename_cleanup(call_frame_t *frame);
+int
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
 
-int dht_update_commit_hash_for_layout (call_frame_t *frame);
-int dht_fix_directory_layout (call_frame_t *frame,
-                              dht_selfheal_dir_cbk_t  dir_cbk,
-                              dht_layout_t           *layout);
+int
+dht_update_commit_hash_for_layout(call_frame_t *frame);
+int
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout);
 
-int dht_init_subvolumes (xlator_t *this, dht_conf_t *conf);
+int
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf);
 
 /* migration/rebalance */
-int dht_start_rebalance_task (xlator_t *this, call_frame_t *frame);
+int
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame);
 
-int dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame);
-int dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame);
+int
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame);
+int
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame);
 
 int
-dht_init_local_subvolumes (xlator_t *this, dht_conf_t *conf);
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf);
 
 /* FOPS */
-int32_t dht_lookup (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    dict_t   *xattr_req);
-
-int32_t dht_stat (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc, dict_t *xdata);
-
-int32_t dht_fstat (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_truncate (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      off_t     offset, dict_t *xdata);
-
-int32_t dht_ftruncate (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       off_t     offset, dict_t *xdata);
-
-int32_t dht_access (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    int32_t   mask, dict_t *xdata);
-
-int32_t dht_readlink (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      size_t    size, dict_t *xdata);
-
-int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
-
-int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
-
-int32_t dht_unlink (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, int xflag, dict_t *xdata);
-
-int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, int flags, dict_t *xdata);
-
-int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
-                     const char *linkpath, loc_t *loc, mode_t umask,
-                     dict_t *xdata);
-
-int32_t dht_rename (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *oldloc,
-                    loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_link (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *oldloc,
-                  loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_create (call_frame_t *frame, xlator_t *this,
-                    loc_t    *loc, int32_t flags, mode_t mode,
-                    mode_t umask, fd_t     *fd, dict_t *params);
-
-int32_t dht_open (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc,
-                  int32_t   flags, fd_t *fd, dict_t *xdata);
-
-int32_t dht_readv (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   size_t    size,
-                   off_t     offset, uint32_t flags, dict_t *xdata);
-
-int32_t dht_writev (call_frame_t      *frame,
-                    xlator_t      *this,
-                    fd_t          *fd,
-                    struct iovec  *vector,
-                    int32_t        count,
-                    off_t          offset,
-                    uint32_t       flags,
-                    struct iobref *iobref, dict_t *xdata);
-
-int32_t dht_flush (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_fsync (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   int32_t   datasync, dict_t *xdata);
-
-int32_t dht_opendir (call_frame_t *frame,
-                     xlator_t *this,
-                     loc_t    *loc, fd_t *fd, dict_t *xdata);
-
-int32_t dht_fsyncdir (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      int32_t   datasync, dict_t *xdata);
-
-int32_t dht_statfs (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, dict_t *xdata);
-
-int32_t dht_setxattr (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      dict_t   *dict,
-                      int32_t   flags, dict_t *xdata);
-
-int32_t dht_getxattr (call_frame_t   *frame,
-                      xlator_t   *this,
-                      loc_t      *loc,
-                      const char *name, dict_t *xdata);
-
-int32_t dht_fsetxattr (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       dict_t   *dict,
-                       int32_t   flags, dict_t *xdata);
-
-int32_t dht_fgetxattr (call_frame_t   *frame,
-                       xlator_t   *this,
-                       fd_t       *fd,
-                       const char *name, dict_t *xdata);
-
-int32_t dht_removexattr (call_frame_t   *frame,
-                         xlator_t   *this,
-                         loc_t      *loc,
-                         const char *name, dict_t *xdata);
-int32_t dht_fremovexattr (call_frame_t   *frame,
-                          xlator_t   *this,
-                          fd_t      *fd,
-                          const char *name, dict_t *xdata);
-
-int32_t dht_lk (call_frame_t        *frame,
-                xlator_t        *this,
-                fd_t            *fd,
-                int32_t          cmd,
-                struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_lease (call_frame_t    *frame,
-                   xlator_t        *this,
-                   loc_t           *loc,
-                   struct gf_lease *lease, dict_t *xdata);
-
-int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
-                     const char      *volume, loc_t *loc, int32_t cmd,
-                     struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
-                      const char      *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
-                     const char  *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
-                      const char  *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_readdir (call_frame_t  *frame,
-                     xlator_t *this,
-                     fd_t     *fd,
-                     size_t    size, off_t off, dict_t *xdata);
-
-int32_t dht_readdirp (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      size_t    size, off_t off, dict_t *dict);
-
-int32_t dht_xattrop (call_frame_t           *frame,
-                     xlator_t           *this,
-                     loc_t              *loc,
-                     gf_xattrop_flags_t  flags,
-                     dict_t             *dict, dict_t *xdata);
-
-int32_t dht_fxattrop (call_frame_t *frame,
-                      xlator_t           *this,
-                      fd_t               *fd,
-                      gf_xattrop_flags_t  flags,
-                      dict_t             *dict, dict_t *xdata);
-
-int32_t dht_forget (xlator_t *this, inode_t *inode);
-int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
-                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
-int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
-int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
-int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    off_t offset, size_t len, dict_t *xdata);
-int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    off_t offset, off_t len, dict_t *xdata);
+int32_t
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req);
+
+int32_t
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
+
+int32_t
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
+
+int32_t
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
+
+int32_t
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
+
+int32_t
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata);
+
+int32_t
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
+
+int32_t
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
+
+int32_t
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
+
+int32_t
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata);
+
+int32_t
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
+
+int32_t
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
+
+int32_t
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params);
+
+int32_t
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
+
+int32_t
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
+
+int32_t
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
+
+int32_t
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
+
+int32_t
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata);
+
+int32_t
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
+
+int32_t
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
+
+int32_t
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
+
+int32_t
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
+
+int32_t
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
+int32_t
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
+
+int32_t
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata);
+
+int32_t
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
+
+int32_t
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata);
+
+int32_t
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata);
+
+int32_t
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *dict);
+
+int32_t
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_forget(xlator_t *this, inode_t *inode);
+int32_t
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata);
+int32_t
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata);
+int32_t
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
+int32_t
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
+int32_t
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
+int32_t
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata);
 
 int
 dht_set_subvol_range(xlator_t *this);
-int32_t dht_init (xlator_t *this);
-void    dht_fini (xlator_t *this);
-int     dht_reconfigure (xlator_t *this, dict_t *options);
-int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
+int32_t
+dht_init(xlator_t *this);
+void
+dht_fini(xlator_t *this);
+int
+dht_reconfigure(xlator_t *this, dict_t *options);
+int32_t
+dht_notify(xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
-int dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *stbuf, dict_t *xattr,
-                        struct iatt *postparent);
-int dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+int
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                         int op_ret, int op_errno, inode_t *inode,
                         struct iatt *stbuf, dict_t *xattr,
                         struct iatt *postparent);
-int dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int op_ret, int op_errno,
-                             inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                             struct iatt *postparent);
-int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent);
-int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata);
-int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno,
-                     inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata);
-
-int
-gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
-
-void
-gf_defrag_set_pause_state (gf_tier_conf_t *tier_conf, tier_pause_state_t state);
-
-tier_pause_state_t
-gf_defrag_get_pause_state (gf_tier_conf_t *tier_conf);
-
 int
-gf_defrag_pause_tier (xlator_t *this, gf_defrag_info_t *defrag);
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent);
+int
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
+int
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
 
-tier_pause_state_t
-gf_defrag_check_pause_tier (gf_tier_conf_t *defrag);
+int
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int
-gf_defrag_resume_tier (xlator_t *this, gf_defrag_info_t *defrag);
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata);
 
 int
-gf_defrag_start_detach_tier (gf_defrag_info_t *defrag);
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
+int
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict);
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
-                dict_t *output);
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output);
 
-void*
-gf_defrag_start (void *this);
+void *
+gf_defrag_start(void *this);
 
 int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
-                           struct iatt *stbuf);
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno);
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
-                 int flag);
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag, int *fop_errno);
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
-                          dht_layout_t **layout_int);
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this,
+                         dht_layout_t **layout_int);
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t* layout_int);
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int);
 int
-dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
-                           int32_t update_ctx);
-void dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat);
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t update_ctx);
+void
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat);
 
-int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
-int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
 int
-dht_dir_attr_heal (void *data);
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
 int
-dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
 int
-dht_dir_has_layout (dict_t *xattr, char *name);
-gf_boolean_t
-dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+dht_dir_attr_heal(void *data);
+int
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data);
 xlator_t *
-dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
-                                   dht_layout_t *layout);
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize);
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout);
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout);
+int
+dht_dir_has_layout(dict_t *xattr, char *name);
 int
-dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this);
 
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix);
 int32_t
-dht_priv_dump (xlator_t *this);
+dht_priv_dump(xlator_t *this);
 int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode);
+dht_inodectx_dump(xlator_t *this, inode_t *inode);
+
+gf_boolean_t
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator);
 
 int
-dht_inode_ctx_get_mig_info (xlator_t *this, inode_t *inode,
-                            xlator_t **src_subvol, xlator_t **dst_subvol);
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol);
 gf_boolean_t
-dht_mig_info_is_invalid (xlator_t *current, xlator_t *src_subvol,
-                         xlator_t *dst_subvol);
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol);
 
 int
-dht_subvol_status (dht_conf_t *conf, xlator_t *subvol);
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol);
 
 void
-dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
-                                     dht_layout_t *layout);
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout);
+
 int
-dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this);
+dht_layout_sort(dht_layout_t *layout);
 
 int
-dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict);
+dht_heal_full_path(void *data);
 
+int
+dht_heal_full_path_done(int op_ret, call_frame_t *frame, void *data);
 
-/* Acquire non-blocking inodelk on a list of xlators.
- *
- * @lk_array: array of lock requests lock on.
- *
- * @lk_count: number of locks in @lk_array
- *
- * @inodelk_cbk: will be called after inodelk replies are received
- *
- * @retval: -1 if stack_winding inodelk fails. 0 otherwise.
- *          inodelk_cbk is called with appropriate error on errors.
- *          On failure to acquire lock on all members of list, successful
- *          locks are unlocked before invoking cbk.
- */
+int
+dht_layout_missing_dirs(dht_layout_t *layout);
 
 int
-dht_nonblocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                         int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+dht_refresh_layout(call_frame_t *frame);
 
-/* same as dht_nonblocking_inodelk, but issues sequential blocking locks on
- * @lk_array directly. locks are issued on some order which remains same
- * for a list of xlators (irrespective of order of xlators within list).
- */
 int
-dht_blocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, dht_reaction_type_t reaction,
-                      fop_inodelk_cbk_t inodelk_cbk);
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno);
 
 int32_t
-dht_unlock_inodelk (call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
-                    fop_inodelk_cbk_t inodelk_cbk);
-
-dht_lock_t *
-dht_lock_new (xlator_t *this, xlator_t *xl, loc_t *loc, short type,
-              const char *domain);
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata);
 void
-dht_lock_array_free (dht_lock_t **lk_array, int count);
+dht_build_root_loc(inode_t *inode, loc_t *loc);
+
+gf_boolean_t
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst);
+
+int32_t
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd);
+
+int32_t
+dht_release(xlator_t *this, fd_t *fd);
 
 int32_t
-dht_lock_count (dht_lock_t **lk_array, int lk_count);
+dht_set_fixed_dir_stat(struct iatt *stat);
+
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local);
 
 int
-dht_layout_sort (dht_layout_t *layout);
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret);
 
 int
-dht_heal_full_path (void *data);
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *subvol);
 
 int
-dht_heal_full_path_done (int op_ret, call_frame_t *frame, void *data);
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame);
+
+/* FD fop callbacks */
 
 int
-dht_layout_missing_dirs (dht_layout_t *layout);
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata);
 
 int
-dht_refresh_layout (call_frame_t *frame);
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata);
 
-gf_boolean_t
-dht_is_tier_xlator (xlator_t *this);
+int
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
+
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata);
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata);
+
+int
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
 
 int
-dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child,
-                                                 int32_t *op_errno);
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata);
+
+int
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata);
+
+int
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata);
+
+int
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata);
+
+int
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
+
+/* All custom xattr heal functions */
+int
+dht_dir_heal_xattrs(void *data);
+
+int
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data);
 
 int32_t
-dht_set_local_rebalance (xlator_t *this, dht_local_t *local,
-                         struct iatt *stbuf,
-                         struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata);
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size);
+
+int
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data);
+
 void
-dht_build_root_loc (inode_t *inode, loc_t *loc);
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag);
 
-gf_boolean_t
-dht_fd_open_on_dst (xlator_t *this, fd_t *fd, xlator_t *dst);
+int
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno);
+
+int
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst, int flag);
+
+int
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol);
+
+int
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout);
+
+/* Abstract out the DHT-IATT-IN-DICT */
+
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
+
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata);
+
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata);
 
 int32_t
-dht_fd_ctx_destroy (xlator_t *this, fd_t *fd);
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata);
+
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata);
 
 int32_t
-dht_release (xlator_t *this, fd_t *fd);
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno);
 
+int
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
 
 int32_t
-dht_set_fixed_dir_stat (struct iatt *stat);
+dht_create_lock(call_frame_t *frame, xlator_t *subvol);
 
-xlator_t*
-dht_get_lock_subvolume (xlator_t *this, struct gf_flock *lock,
-                        dht_local_t *local);
+int
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local);
 
 int
-dht_lk_inode_unref (call_frame_t *frame, int32_t op_ret);
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode);
 
-#endif/* _DHT_H */
+int
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol);
+#endif /* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index 1eb9e63c531..c0588828fdb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -8,453 +8,480 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "dht-messages.h"
-#include "defaults.h"
 
 #include <sys/time.h>
-
+#include <glusterfs/events.h>
 
 int
-dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct statvfs *statvfs,
-                 dict_t *xdata)
+dht_du_info_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *prev          = NULL;
-	int            this_call_cnt = 0;
-	int            i = 0;
-	double         percent = 0;
-	double         percent_inodes = 0;
-	uint64_t       bytes = 0;
-        uint32_t       bpc;     /* blocks per chunk */
-        uint32_t       chunks   = 0;
-
-	conf = this->private;
-	prev = cookie;
-
-	if (op_ret == -1) {
-		gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_GET_DISK_INFO_ERROR,
-			"failed to get disk info from %s", prev->this->name);
-		goto out;
-	}
-
-	if (statvfs && statvfs->f_blocks) {
-		percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
-		bytes = (statvfs->f_bavail * statvfs->f_frsize);
-                /*
-                 * A 32-bit count of 1MB chunks allows a maximum brick size of
-                 * ~4PB.  It's possible that we could see a single local FS
-                 * bigger than that some day, but this code is likely to be
-                 * irrelevant by then.  Meanwhile, it's more important to keep
-                 * the chunk size small so the layout-calculation code that
-                 * uses this value can be tested on normal machines.
-                 */
-                bpc = (1 << 20) / statvfs->f_bsize;
-                chunks = (statvfs->f_blocks + bpc - 1) / bpc;
-	}
-
-	if (statvfs && statvfs->f_files) {
-		percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
-	} else {
-                /*
-                 * Set percent inodes to 100 for dynamically allocated inode
-                 * filesystems. The rationale is that distribute need not
-                 * worry about total inodes; rather, let the 'create()' be
-                 * scheduled on the hashed subvol regardless of the total
-                 * inodes.
-		 */
-		percent_inodes = 100;
-	}
-
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++)
-			if (prev->this == conf->subvolumes[i]) {
-				conf->du_stats[i].avail_percent = percent;
-				conf->du_stats[i].avail_space   = bytes;
-				conf->du_stats[i].avail_inodes  = percent_inodes;
-                                conf->du_stats[i].chunks        = chunks;
-				gf_msg_debug (this->name, 0,
-				              "subvolume '%s': avail_percent "
-					      "is: %.2f and avail_space "
-                                              "is: %" PRIu64" and avail_inodes"
-                                              " is: %.2f",
-					      prev->this->name,
-					      conf->du_stats[i].avail_percent,
-					      conf->du_stats[i].avail_space,
-					      conf->du_stats[i].avail_inodes);
-                                break;  /* no point in looping further */
-			}
-	}
-	UNLOCK (&conf->subvolume_lock);
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int i = 0;
+    double percent = 0;
+    double percent_inodes = 0;
+    uint64_t bytes = 0;
+    uint32_t bpc; /* blocks per chunk */
+    uint32_t chunks = 0;
+
+    conf = this->private;
+    prev = cookie;
+
+    if (op_ret == -1 || !statvfs) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               DHT_MSG_GET_DISK_INFO_ERROR, "failed to get disk info from %s",
+               prev->name);
+        goto out;
+    }
+
+    if (statvfs->f_blocks) {
+        percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
+        bytes = (statvfs->f_bavail * statvfs->f_frsize);
+        /*
+         * A 32-bit count of 1MB chunks allows a maximum brick size of
+         * ~4PB.  It's possible that we could see a single local FS
+         * bigger than that some day, but this code is likely to be
+         * irrelevant by then.  Meanwhile, it's more important to keep
+         * the chunk size small so the layout-calculation code that
+         * uses this value can be tested on normal machines.
+         */
+        bpc = (1 << 20) / statvfs->f_bsize;
+        chunks = (statvfs->f_blocks + bpc - 1) / bpc;
+    }
+
+    if (statvfs->f_files) {
+        percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
+    } else {
+        /*
+         * Set percent inodes to 100 for dynamically allocated inode
+         * filesystems. The rationale is that distribute need not
+         * worry about total inodes; rather, let the 'create()' be
+         * scheduled on the hashed subvol regardless of the total
+         * inodes.
+         */
+        percent_inodes = 100;
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++)
+            if (prev == conf->subvolumes[i]) {
+                conf->du_stats[i].avail_percent = percent;
+                conf->du_stats[i].avail_space = bytes;
+                conf->du_stats[i].avail_inodes = percent_inodes;
+                conf->du_stats[i].chunks = chunks;
+                conf->du_stats[i].total_blocks = statvfs->f_blocks;
+                conf->du_stats[i].avail_blocks = statvfs->f_bavail;
+                conf->du_stats[i].frsize = statvfs->f_frsize;
+
+                gf_msg_debug(this->name, 0,
+                             "subvolume '%s': avail_percent "
+                             "is: %.2f and avail_space "
+                             "is: %" PRIu64
+                             " and avail_inodes"
+                             " is: %.2f",
+                             prev->name, conf->du_stats[i].avail_percent,
+                             conf->du_stats[i].avail_space,
+                             conf->du_stats[i].avail_inodes);
+                break; /* no point in looping further */
+            }
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_DESTROY (frame);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
-	return 0;
+    return 0;
 }
 
 int
-dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx)
 {
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	call_pool_t   *pool         = NULL;
-	loc_t          tmp_loc      = {0,};
-
-	conf = this->private;
-	pool = this->ctx->pool;
-
-	statfs_frame = create_frame (this, pool);
-	if (!statfs_frame) {
-		goto err;
-	}
-
-	/* local->fop value is not used in this case */
-	statfs_local = dht_local_init (statfs_frame, NULL, NULL,
-				       GF_FOP_MAXVALUE);
-	if (!statfs_local) {
-		goto err;
-	}
-
-        /* make it root gfid, should be enough to get the proper info back */
-        tmp_loc.gfid[15] = 1;
-
-	statfs_local->call_cnt = 1;
-	STACK_WIND (statfs_frame, dht_du_info_cbk,
-		    conf->subvolumes[subvol_idx],
-		    conf->subvolumes[subvol_idx]->fops->statfs,
-		    &tmp_loc, NULL);
-
-	return 0;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    call_pool_t *pool = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+
+    conf = this->private;
+    pool = this->ctx->pool;
+
+    statfs_frame = create_frame(this, pool);
+    if (!statfs_frame) {
+        goto err;
+    }
+
+    /* local->fop value is not used in this case */
+    statfs_local = dht_local_init(statfs_frame, NULL, NULL, GF_FOP_MAXVALUE);
+    if (!statfs_local) {
+        goto err;
+    }
+
+    /* make it root gfid, should be enough to get the proper info back */
+    tmp_loc.gfid[15] = 1;
+
+    statfs_local->call_cnt = 1;
+    STACK_WIND_COOKIE(
+        statfs_frame, dht_du_info_cbk, conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx]->fops->statfs, &tmp_loc, NULL);
+
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
 int
-dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	int            i            = 0;
-        int            ret          = -1;
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	struct timeval tv           = {0,};
-        loc_t          tmp_loc      = {0,};
-
-	conf  = this->private;
-
-	gettimeofday (&tv, NULL);
-
-        /* make it root gfid, should be enough to get the proper
-           info back */
-        tmp_loc.gfid[15] = 1;
-
-	if (tv.tv_sec > (conf->refresh_interval
-			 + conf->last_stat_fetch.tv_sec)) {
-
-		statfs_frame = copy_frame (frame);
-		if (!statfs_frame) {
-			goto err;
-		}
-
-		/* In this case, 'local->fop' is not used */
-		statfs_local = dht_local_init (statfs_frame, loc, NULL,
-					       GF_FOP_MAXVALUE);
-		if (!statfs_local) {
-			goto err;
-		}
-
-                statfs_local->params = dict_new ();
-                if (!statfs_local->params)
-                        goto err;
-
-                ret = dict_set_int8 (statfs_local->params,
-                                     GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set "
-                                GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                        goto err;
-                }
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    time_t now;
+
+    conf = this->private;
+    now = gf_time();
+    /* make it root gfid, should be enough to get the proper
+       info back */
+    tmp_loc.gfid[15] = 1;
+
+    if (now > (conf->refresh_interval + conf->last_stat_fetch)) {
+        statfs_frame = copy_frame(frame);
+        if (!statfs_frame) {
+            goto err;
+        }
+
+        /* In this case, 'local->fop' is not used */
+        statfs_local = dht_local_init(statfs_frame, loc, NULL, GF_FOP_MAXVALUE);
+        if (!statfs_local) {
+            goto err;
+        }
+
+        statfs_local->params = dict_new();
+        if (!statfs_local->params)
+            goto err;
 
-		statfs_local->call_cnt = conf->subvolume_cnt;
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			STACK_WIND (statfs_frame, dht_du_info_cbk,
-				    conf->subvolumes[i],
-				    conf->subvolumes[i]->fops->statfs,
-				    &tmp_loc, statfs_local->params);
-		}
-
-		conf->last_stat_fetch.tv_sec = tv.tv_sec;
-	}
-	return 0;
+        ret = dict_set_int8(statfs_local->params,
+                            GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+            goto err;
+        }
+
+        statfs_local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            STACK_WIND_COOKIE(statfs_frame, dht_du_info_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->statfs, &tmp_loc,
+                              statfs_local->params);
+        }
+
+        conf->last_stat_fetch = now;
+    }
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
-
 gf_boolean_t
-dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol)
 {
-	int         i = 0;
-	dht_conf_t *conf = NULL;
-	gf_boolean_t subvol_filled_inodes = _gf_false;
-	gf_boolean_t subvol_filled_space = _gf_false;
-	gf_boolean_t is_subvol_filled = _gf_false;
-
-	conf = this->private;
-
-	/* Check for values above specified percent or free disk */
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				if (conf->disk_unit == 'p') {
-					if (conf->du_stats[i].avail_percent <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-
-				} else {
-					if (conf->du_stats[i].avail_space <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-				}
-				if (conf->du_stats[i].avail_inodes <
-				    conf->min_free_inodes) {
-					subvol_filled_inodes = _gf_true;
-					break;
-				}
-			}
-		}
-	}
-	UNLOCK (&conf->subvolume_lock);
-
-	if (subvol_filled_space && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-			gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_SUBVOL_INSUFF_SPACE,
-				"disk space on subvolume '%s' is getting "
-				"full (%.2f %%), consider adding more bricks",
-				subvol->name,
-				(100 - conf->du_stats[i].avail_percent));
-		}
-	}
-
-	if (subvol_filled_inodes && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-			gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                DHT_MSG_SUBVOL_INSUFF_INODES,
-				"inodes on subvolume '%s' are at "
-				"(%.2f %%), consider adding more bricks",
-				subvol->name,
-				(100 - conf->du_stats[i].avail_inodes));
-		}
-	}
-
-	is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
-
-	return is_subvol_filled;
-}
+    int i = 0;
+    char vol_name[256];
+    dht_conf_t *conf = NULL;
+    gf_boolean_t subvol_filled_inodes = _gf_false;
+    gf_boolean_t subvol_filled_space = _gf_false;
+    gf_boolean_t is_subvol_filled = _gf_false;
+    double usage = 0;
+
+    conf = this->private;
+
+    /* Check for values above specified percent or free disk */
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (subvol == conf->subvolumes[i]) {
+                if (conf->disk_unit == 'p') {
+                    if (conf->du_stats[i].avail_percent < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+
+                } else {
+                    if (conf->du_stats[i].avail_space < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+                }
+                if (conf->du_stats[i].avail_inodes < conf->min_free_inodes) {
+                    subvol_filled_inodes = _gf_true;
+                    break;
+                }
+            }
+        }
+    }
+    UNLOCK(&conf->subvolume_lock);
 
+    if (subvol_filled_space && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_percent;
 
-/*Get the best subvolume to create the file in*/
-xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
-                                dht_local_t *local)
-{
-	xlator_t   *avail_subvol = NULL;
-	dht_conf_t *conf = NULL;
-        dht_layout_t *layout = NULL;
-        loc_t      *loc = NULL;
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "disk space on subvolume '%s' is getting "
+                   "full (%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
 
-	conf = this->private;
-        if (!local)
-                goto out;
-        loc = &local->loc;
-        if (!local->layout) {
-                layout = dht_layout_get (this, loc->parent);
-
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "Missing layout. path=%s,"
-                                      " parent gfid = %s", loc->path,
-                                      uuid_utoa (loc->parent->gfid));
-                        goto out;
-                }
-        } else {
-                layout = dht_layout_ref (this, local->layout);
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
+
+            gf_event(EVENT_DHT_DISK_USAGE, "volume=%s;subvol=%s;usage=%.2f %%",
+                     vol_name, subvol->name, usage);
+        }
+    }
+
+    if (subvol_filled_inodes && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_inodes;
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_SUBVOL_INSUFF_INODES,
+                   "inodes on subvolume '%s' are at "
+                   "(%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
+
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
+
+            gf_event(EVENT_DHT_INODES_USAGE,
+                     "volume=%s;subvol=%s;usage=%.2f %%", vol_name,
+                     subvol->name, usage);
         }
+    }
 
-        LOCK (&conf->subvolume_lock);
-	{
-                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
-                                                                 layout);
-                if(!avail_subvol)
-                {
-                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
-                                                                        subvol,
-                                                                        layout);
-                }
+    is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
 
-	}
-	UNLOCK (&conf->subvolume_lock);
+    return is_subvol_filled;
+}
+
+/*Get the best subvolume to create the file in*/
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *local)
+{
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+
+    conf = this->private;
+    if (!local)
+        goto out;
+    loc = &local->loc;
+    if (!local->layout) {
+        layout = dht_layout_get(this, loc->parent);
+
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "Missing layout. path=%s,"
+                         " parent gfid = %s",
+                         loc->path, uuid_utoa(loc->parent->gfid));
+            goto out;
+        }
+    } else {
+        layout = dht_layout_ref(this, local->layout);
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        avail_subvol = dht_subvol_with_free_space_inodes(this, subvol, NULL,
+                                                         layout, 0);
+        if (!avail_subvol) {
+            avail_subvol = dht_subvol_maxspace_nonzeroinode(this, subvol,
+                                                            layout);
+        }
+    }
+    UNLOCK(&conf->subvolume_lock);
 out:
-	if (!avail_subvol) {
-		gf_msg_debug (this->name, 0,
-		              "No subvolume has enough free space \
+    if (!avail_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No subvolume has enough free space \
                               and/or inodes to create");
-                avail_subvol = subvol;
-	}
+        avail_subvol = subvol;
+    }
 
-        if (layout)
-                dht_layout_unref (this, layout);
-	return avail_subvol;
+    if (layout)
+        dht_layout_unref(this, layout);
+    return avail_subvol;
 }
 
-static inline
-int32_t dht_subvol_has_err (dht_conf_t *conf, xlator_t *this,
-                                         dht_layout_t *layout)
+static inline int32_t
+dht_subvol_has_err(dht_conf_t *conf, xlator_t *this, xlator_t *ignore,
+                   dht_layout_t *layout)
 {
-        int ret = -1;
-        int i   = 0;
-
-        if (!this || !layout)
-                goto out;
-
-        /* check if subvol has layout errors, before selecting it */
-        for (i = 0; i < layout->cnt; i++) {
-                if (!strcmp (layout->list[i].xlator->name, this->name) &&
-                     (layout->list[i].err != 0)) {
-                        ret = -1;
-                        goto out;
-                }
+    int ret = -1;
+    int i = 0;
+
+    if (!this || !layout)
+        goto out;
+
+    /* this check is meant for rebalance process. The source of the file
+     * should be ignored for space check */
+    if (this == ignore) {
+        goto out;
+    }
+
+    /* check if subvol has layout errors, before selecting it */
+    for (i = 0; i < layout->cnt; i++) {
+        if (!strcmp(layout->list[i].xlator->name, this->name) &&
+            (layout->list[i].err != 0)) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        /* discard decommissioned subvol */
-        if (conf->decommission_subvols_cnt) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->decommissioned_bricks[i] &&
-                            conf->decommissioned_bricks[i] == this) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
+    /* discard decommissioned subvol */
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == this) {
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*Get subvolume which has both space and inodes more than the min criteria*/
 xlator_t *
 dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout)
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize)
 {
-        int i = 0;
-        double max = 0;
-        double max_inodes = 0;
-        int    ignore_subvol = 0;
-
-        xlator_t *avail_subvol = NULL;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        for(i=0; i < conf->subvolume_cnt; i++) {
-                /* check if subvol has layout errors and also it is not a
-                 * decommissioned brick, before selecting it */
-                ignore_subvol = dht_subvol_has_err (conf, conf->subvolumes[i],
-                                                    layout);
-                if (ignore_subvol)
-                        continue;
-
-                if ((conf->disk_unit == 'p') &&
-                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
-                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
-                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
-                            (conf->du_stats[i].avail_percent > max)) {
-                                max = conf->du_stats[i].avail_percent;
-                                max_inodes = conf->du_stats[i].avail_inodes;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-                }
+    int i = 0;
+    double max = 0;
+    double max_inodes = 0;
+    int ignore_subvol = 0;
+    uint64_t total_blocks = 0;
+    uint64_t avail_blocks = 0;
+    uint64_t frsize = 0;
+    double post_availspace = 0;
+    double post_percent = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it */
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], ignore,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if ((conf->disk_unit == 'p') &&
+            (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_percent > max)) {
+                max = conf->du_stats[i].avail_percent;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+                total_blocks = conf->du_stats[i].total_blocks;
+                avail_blocks = conf->du_stats[i].avail_blocks;
+                frsize = conf->du_stats[i].frsize;
+            }
+        }
 
-                if ((conf->disk_unit != 'p') &&
-                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
-                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
-                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
-                            (conf->du_stats[i].avail_space > max)) {
-                                max = conf->du_stats[i].avail_space;
-                                max_inodes = conf->du_stats[i].avail_inodes;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-                }
+        if ((conf->disk_unit != 'p') &&
+            (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_space > max)) {
+                max = conf->du_stats[i].avail_space;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+            }
+        }
+    }
+
+    if (avail_subvol) {
+        if (conf->disk_unit == 'p') {
+            post_availspace = (avail_blocks * frsize) - filesize;
+            post_percent = (post_availspace * 100) / (total_blocks * frsize);
+            if (post_percent < conf->min_free_disk)
+                avail_subvol = NULL;
         }
+        if (conf->disk_unit != 'p') {
+            if ((max - filesize) < conf->min_free_disk)
+                avail_subvol = NULL;
+        }
+    }
 
-        return avail_subvol;
+    return avail_subvol;
 }
 
-
-/* Get subvol which has atleast one inode and maximum space */
+/* Get subvol which has at least one inode and maximum space */
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout)
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout)
 {
-        int         i = 0;
-        double      max = 0;
-        int         ignore_subvol = 0;
-
-        xlator_t   *avail_subvol = NULL;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                /* check if subvol has layout errors and also it is not a
-                 * decommissioned brick, before selecting it*/
-
-                ignore_subvol = dht_subvol_has_err (conf, conf->subvolumes[i],
-                                                    layout);
-                if (ignore_subvol)
-                        continue;
-
-                if (conf->disk_unit == 'p') {
-                        if ((conf->du_stats[i].avail_percent > max)
-                            && (conf->du_stats[i].avail_inodes > 0 )) {
-                                max = conf->du_stats[i].avail_percent;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-               } else {
-                         if ((conf->du_stats[i].avail_space > max)
-                            && (conf->du_stats[i].avail_inodes > 0)) {
-                                 max = conf->du_stats[i].avail_space;
-                                 avail_subvol = conf->subvolumes[i];
-                         }
-               }
+    int i = 0;
+    double max = 0;
+    int ignore_subvol = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it*/
+
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], NULL,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if (conf->disk_unit == 'p') {
+            if ((conf->du_stats[i].avail_percent > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_percent;
+                avail_subvol = conf->subvolumes[i];
+            }
+        } else {
+            if ((conf->du_stats[i].avail_space > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_space;
+                avail_subvol = conf->subvolumes[i];
+            }
         }
+    }
 
-        return avail_subvol;
+    return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index 66e3ede736b..acda67c312a 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -8,92 +8,103 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "hashfn.h"
-
+#include <glusterfs/hashfn.h>
 
-int
-dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
+static int
+dht_hash_compute_internal(int type, const char *name, const int len,
+                          uint32_t *hash_p)
 {
-        int      ret = 0;
-        uint32_t hash = 0;
+    int ret = 0;
+    uint32_t hash = 0;
 
-        switch (type) {
+    switch (type) {
         case DHT_HASH_TYPE_DM:
         case DHT_HASH_TYPE_DM_USER:
-                hash = gf_dm_hashfn (name, strlen (name));
-                break;
+            hash = gf_dm_hashfn(name, len);
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 
-        if (ret == 0) {
-                *hash_p = hash;
-        }
+    if (ret == 0) {
+        *hash_p = hash;
+    }
 
-        return ret;
+    return ret;
 }
 
-
-static
-gf_boolean_t
-dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+/* The function returns:
+ * 0  : in case no munge took place
+ * >0 : the length (inc. terminating NULL!) of the newly modified string,
+ *      if it was munged.
+ */
+static int
+dht_munge_name(const char *original, char *modified, size_t len, regex_t *re)
 {
-        regmatch_t      matches[2];
-        size_t          new_len;
-
-        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
-                if (matches[1].rm_so != -1) {
-                        new_len = matches[1].rm_eo - matches[1].rm_so;
-                        /* Equal would fail due to the NUL at the end. */
-                        if (new_len < len) {
-                                memcpy (modified,original+matches[1].rm_so,
-                                        new_len);
-                                modified[new_len] = '\0';
-                                return _gf_true;
-                        }
-                }
+    regmatch_t matches[2] = {
+        {0},
+    };
+    size_t new_len = 0;
+    int ret = 0;
+
+    ret = regexec(re, original, 2, matches, 0);
+
+    if (ret != REG_NOMATCH) {
+        if (matches[1].rm_so != -1) {
+            new_len = matches[1].rm_eo - matches[1].rm_so;
+            /* Equal would fail due to the NUL at the end. */
+            if (new_len < len) {
+                memcpy(modified, original + matches[1].rm_so, new_len);
+                modified[new_len] = '\0';
+                return new_len + 1; /* +1 for the terminating NULL */
+            }
         }
+    }
 
-        /* This is guaranteed safe because of how the dest was allocated. */
-        strcpy(modified,original);
-        return _gf_false;
+    /* This is guaranteed safe because of how the dest was allocated. */
+    strcpy(modified, original);
+    return 0;
 }
 
 int
-dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char            *rsync_friendly_name    = NULL;
-        dht_conf_t      *priv                   = this->private;
-        size_t           len                    = 0;
-        gf_boolean_t     munged                 = _gf_false;
+    char *rsync_friendly_name = NULL;
+    dht_conf_t *priv = NULL;
+    size_t len = 0;
+    int munged = 0;
 
+    priv = this->private;
+
+    if (name == NULL)
+        return -1;
+
+    len = strlen(name) + 1;
+    rsync_friendly_name = alloca(len);
+
+    LOCK(&priv->lock);
+    {
         if (priv->extra_regex_valid) {
-                len = strlen(name) + 1;
-                rsync_friendly_name = alloca(len);
-                munged = dht_munge_name (name, rsync_friendly_name, len,
-                                         &priv->extra_regex);
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->extra_regex);
         }
 
         if (!munged && priv->rsync_regex_valid) {
-                len = strlen(name) + 1;
-                rsync_friendly_name = alloca(len);
-                gf_msg_trace (this->name, 0, "trying regex for %s", name);
-                munged = dht_munge_name (name, rsync_friendly_name, len,
-                                         &priv->rsync_regex);
-                if (munged) {
-                        gf_msg_debug (this->name, 0,
-                                      "munged down to %s", rsync_friendly_name);
-                }
+            gf_msg_trace(this->name, 0, "trying regex for %s", name);
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->rsync_regex);
         }
-
-        if (!munged) {
-                rsync_friendly_name = (char *)name;
-        }
-
-        return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
+    }
+    UNLOCK(&priv->lock);
+    if (munged) {
+        gf_msg_debug(this->name, 0, "munged down to %s", rsync_friendly_name);
+        len = munged;
+    } else {
+        rsync_friendly_name = (char *)name;
+    }
+
+    return dht_hash_compute_internal(type, rsync_friendly_name, len - 1,
+                                     hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 0a5e1128db1..3f2fe43d5f3 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -8,1053 +8,1228 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "dht-helper.h"
-
+#include "dht-lock.h"
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
 
-void
-dht_free_fd_ctx (void *data)
+static void
+dht_free_fd_ctx(dht_fd_ctx_t *fd_ctx)
 {
-        dht_fd_ctx_t *fd_ctx = NULL;
-
-        fd_ctx = (dht_fd_ctx_t *)data;
-        GF_FREE (fd_ctx);
-
-        return;
+    GF_FREE(fd_ctx);
 }
 
-
 int32_t
-dht_fd_ctx_destroy (xlator_t *this, fd_t *fd)
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd)
 {
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        uint64_t      value   = 0;
-        int32_t       ret     = -1;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = fd_ctx_del (fd, this, &value);
-        if (ret) {
-                goto out;
-        }
+    ret = fd_ctx_del(fd, this, &value);
+    if (ret) {
+        goto out;
+    }
 
-        fd_ctx = (dht_fd_ctx_t *)value;
-        if (fd_ctx) {
-                GF_REF_PUT (fd_ctx);
-        }
+    fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+    if (fd_ctx) {
+        GF_REF_PUT(fd_ctx);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-__dht_fd_ctx_set (xlator_t *this, fd_t *fd, xlator_t *dst)
+__dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        uint64_t      value   = 0;
-        int           ret     = -1;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx), gf_dht_mt_fd_ctx_t);
+    fd_ctx = GF_CALLOC(1, sizeof(*fd_ctx), gf_dht_mt_fd_ctx_t);
 
-        if (!fd_ctx) {
-                goto out;
-        }
+    if (!fd_ctx) {
+        goto out;
+    }
 
-        fd_ctx->opened_on_dst = (uint64_t) dst;
-        GF_REF_INIT (fd_ctx, dht_free_fd_ctx);
+    fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+    GF_REF_INIT(fd_ctx, dht_free_fd_ctx);
 
-        value = (uint64_t) fd_ctx;
+    value = (uint64_t)(uintptr_t)fd_ctx;
 
-        ret = __fd_ctx_set (fd, this, value);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_FD_CTX_SET_FAILED,
-                        "Failed to set fd ctx in fd=0x%p", fd);
-                GF_REF_PUT (fd_ctx);
-        }
+    ret = __fd_ctx_set(fd, this, value);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FD_CTX_SET_FAILED,
+                "fd=0x%p", fd, NULL);
+        GF_REF_PUT(fd_ctx);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
-
 int
-dht_fd_ctx_set (xlator_t *this, fd_t *fd, xlator_t *dst)
-{
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        uint64_t      value   = 0;
-        int           ret     = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &value);
-                if (ret && value) {
-
-                        fd_ctx = (dht_fd_ctx_t *) value;
-                        if (fd_ctx->opened_on_dst == (uint64_t) dst)  {
-                                /* This could happen due to racing
-                                 * check_progress tasks*/
-                                goto unlock;
-                        } else {
-                                /* This would be a big problem*/
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_INVALID_VALUE,
-                                        "Different dst found in the fd ctx");
-
-                                /* Overwrite and hope for the best*/
-                                fd_ctx->opened_on_dst = (uint64_t)dst;
-                                goto unlock;
-                        }
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
+{
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &value);
+        if (ret && value) {
+            fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+            if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+                /* This could happen due to racing
+                 * check_progress tasks*/
+                goto unlock;
+            } else {
+                /* This would be a big problem*/
+                /* Overwrite and hope for the best*/
+                fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+                UNLOCK(&fd->lock);
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_VALUE,
+                        NULL);
 
-                }
-                ret = __dht_fd_ctx_set (this, fd, dst);
+                goto out;
+            }
         }
+        ret = __dht_fd_ctx_set(this, fd, dst);
+    }
 unlock:
-        UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
-
-static
-dht_fd_ctx_t *
-dht_fd_ctx_get (xlator_t *this, fd_t *fd)
+static dht_fd_ctx_t *
+dht_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        int           ret     = -1;
-        uint64_t      tmp_val = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &tmp_val);
-                if ((ret < 0) || (tmp_val == 0)) {
-                        UNLOCK (&fd->lock);
-                        goto out;
-                }
+    dht_fd_ctx_t *fd_ctx = NULL;
+    int ret = -1;
+    uint64_t tmp_val = 0;
 
-                fd_ctx = (dht_fd_ctx_t *)tmp_val;
-                GF_REF_GET (fd_ctx);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &tmp_val);
+        if ((ret < 0) || (tmp_val == 0)) {
+            goto unlock;
         }
-        UNLOCK (&fd->lock);
+
+        fd_ctx = (dht_fd_ctx_t *)(uintptr_t)tmp_val;
+        GF_REF_GET(fd_ctx);
+    }
+unlock:
+    UNLOCK(&fd->lock);
 
 out:
-        return fd_ctx;
+    return fd_ctx;
 }
 
 gf_boolean_t
-dht_fd_open_on_dst (xlator_t *this, fd_t *fd, xlator_t *dst)
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        dht_fd_ctx_t  *fd_ctx  = NULL;
-        gf_boolean_t   opened  = _gf_false;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    gf_boolean_t opened = _gf_false;
 
-        fd_ctx = dht_fd_ctx_get (this, fd);
+    fd_ctx = dht_fd_ctx_get(this, fd);
 
-        if (fd_ctx) {
-                if (fd_ctx->opened_on_dst == (uint64_t) dst) {
-                        opened = _gf_true;
-                }
-                GF_REF_PUT (fd_ctx);
+    if (fd_ctx) {
+        if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+            opened = _gf_true;
         }
+        GF_REF_PUT(fd_ctx);
+    }
 
-        return opened;
+    return opened;
 }
 
-
 void
-dht_free_mig_info (void *data)
+dht_free_mig_info(void *data)
 {
-        dht_migrate_info_t *miginfo = NULL;
+    dht_migrate_info_t *miginfo = NULL;
 
-        miginfo = data;
-        GF_FREE (miginfo);
+    miginfo = data;
+    GF_FREE(miginfo);
 
-        return;
+    return;
 }
 
 static int
-dht_inode_ctx_set_mig_info (xlator_t *this, inode_t *inode,
-                            xlator_t *src_subvol, xlator_t *dst_subvol)
+dht_inode_ctx_set_mig_info(xlator_t *this, inode_t *inode, xlator_t *src_subvol,
+                           xlator_t *dst_subvol)
 {
-        dht_migrate_info_t *miginfo = NULL;
-        uint64_t            value   = 0;
-        int                 ret     = -1;
+    dht_migrate_info_t *miginfo = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        miginfo = GF_CALLOC (1, sizeof (*miginfo), gf_dht_mt_miginfo_t);
-        if (miginfo == NULL)
-                goto out;
+    miginfo = GF_CALLOC(1, sizeof(*miginfo), gf_dht_mt_miginfo_t);
+    if (miginfo == NULL)
+        goto out;
 
-        miginfo->src_subvol = src_subvol;
-        miginfo->dst_subvol = dst_subvol;
-        GF_REF_INIT (miginfo, dht_free_mig_info);
+    miginfo->src_subvol = src_subvol;
+    miginfo->dst_subvol = dst_subvol;
+    GF_REF_INIT(miginfo, dht_free_mig_info);
 
-        value = (uint64_t) miginfo;
+    value = (uint64_t)(uintptr_t)miginfo;
 
-        ret = inode_ctx_set1 (inode, this, &value);
-        if (ret < 0) {
-                GF_REF_PUT (miginfo);
-        }
+    ret = inode_ctx_set1(inode, this, &value);
+    if (ret < 0) {
+        GF_REF_PUT(miginfo);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_inode_ctx_get_mig_info (xlator_t *this, inode_t *inode,
-                            xlator_t **src_subvol, xlator_t **dst_subvol)
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol)
 {
-        int                 ret         = -1;
-        uint64_t            tmp_miginfo = 0;
-        dht_migrate_info_t *miginfo     = NULL;
-
-        LOCK (&inode->lock);
-        {
-                ret =  __inode_ctx_get1 (inode, this, &tmp_miginfo);
-                if ((ret < 0) || (tmp_miginfo == 0)) {
-                        UNLOCK (&inode->lock);
-                        goto out;
-                }
+    int ret = -1;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
 
-                miginfo = (dht_migrate_info_t *)tmp_miginfo;
-                GF_REF_GET (miginfo);
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get1(inode, this, &tmp_miginfo);
+        if ((ret < 0) || (tmp_miginfo == 0)) {
+            UNLOCK(&inode->lock);
+            goto out;
         }
-        UNLOCK (&inode->lock);
 
-        if (src_subvol)
-                *src_subvol = miginfo->src_subvol;
+        miginfo = (dht_migrate_info_t *)(uintptr_t)tmp_miginfo;
+        GF_REF_GET(miginfo);
+    }
+    UNLOCK(&inode->lock);
+
+    if (src_subvol)
+        *src_subvol = miginfo->src_subvol;
 
-        if (dst_subvol)
-                *dst_subvol = miginfo->dst_subvol;
+    if (dst_subvol)
+        *dst_subvol = miginfo->dst_subvol;
 
-        GF_REF_PUT (miginfo);
+    GF_REF_PUT(miginfo);
 
 out:
-        return ret;
+    return ret;
 }
 
 gf_boolean_t
-dht_mig_info_is_invalid (xlator_t *current, xlator_t *src_subvol,
-                      xlator_t *dst_subvol)
-{
-
-/* Not set
- */
-        if (!src_subvol || !dst_subvol)
-                return _gf_true;
-
-/* Invalid scenarios:
- * The src_subvol does not match the subvol on which the current op was sent
- * so the cached subvol has changed between the last mig_info_set and now.
- * src_subvol == dst_subvol. The file was migrated without any FOP detecting
- * a P2 so the old dst is now the current subvol.
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol)
+{
+    /* Not set
+     */
+    if (!src_subvol || !dst_subvol)
+        return _gf_true;
+
+    /* Invalid scenarios:
+     * The src_subvol does not match the subvol on which the current op was sent
+     * so the cached subvol has changed between the last mig_info_set and now.
+     * src_subvol == dst_subvol. The file was migrated without any FOP detecting
+     * a P2 so the old dst is now the current subvol.
+     *
+     * There is still one scenario where the info could be outdated - if
+     * file has undergone multiple migrations and ends up on the same src_subvol
+     * on which the mig_info was first set.
+     */
+    if ((current == dst_subvol) || (current != src_subvol))
+        return _gf_true;
+
+    return _gf_false;
+}
+
+/* Used to check if fd fops have the fd opened on the cached subvol
+ * This is required when:
+ * 1. an fd is opened on FILE1 on subvol1
+ * 2. the file is migrated to subvol2
+ * 3. a lookup updates the cached subvol in the inode_ctx to subvol2
+ * 4. a write comes on the fd
+ * The write is sent to subvol2 on an fd which has been opened only on fd1
+ * Since the migration phase checks don't kick in, the fop fails with EBADF
  *
- * There is still one scenario where the info could be outdated - if
- * file has undergone multiple migrations and ends up on the same src_subvol
- * on which the mig_info was first set.
  */
-        if ((current == dst_subvol) || (current != src_subvol))
-                return _gf_true;
-
-        return _gf_false;
-}
 
 int
-dht_frame_return (call_frame_t *frame)
-{
-        dht_local_t *local = NULL;
-        int          this_call_cnt = -1;
+dht_check_and_open_fd_on_subvol_complete(int ret, call_frame_t *frame,
+                                         void *data)
+{
+    glusterfs_fop_t fop = 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    fd_t *fd = NULL;
+    int op_errno = -1;
+
+    local = frame->local;
+    this = frame->this;
+    fop = local->fop;
+    subvol = local->cached_subvol;
+    fd = local->fd;
+
+    if (ret) {
+        op_errno = local->op_errno;
+        goto handle_err;
+    }
+
+    switch (fop) {
+        case GF_FOP_WRITE:
+            STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                              subvol->fops->writev, fd, local->rebalance.vector,
+                              local->rebalance.count, local->rebalance.offset,
+                              local->rebalance.flags, local->rebalance.iobref,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FLUSH:
+            STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FSETATTR:
+            STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                              subvol->fops->fsetattr, fd,
+                              &local->rebalance.stbuf, local->rebalance.flags,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_ZEROFILL:
+            STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                              subvol->fops->zerofill, fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
+
+            break;
+
+        case GF_FOP_DISCARD:
+            STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                              subvol->fops->discard, local->fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FALLOCATE:
+            STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                              subvol->fops->fallocate, fd,
+                              local->rebalance.flags, local->rebalance.offset,
+                              local->rebalance.size, local->xattr_req);
+            break;
+
+        case GF_FOP_FTRUNCATE:
+            STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                              subvol->fops->ftruncate, fd,
+                              local->rebalance.offset, local->xattr_req);
+            break;
+
+        case GF_FOP_FSYNC:
+            STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol,
+                              subvol->fops->fsync, local->fd,
+                              local->rebalance.flags, local->xattr_req);
+            break;
+
+        case GF_FOP_READ:
+            STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv,
+                       local->fd, local->rebalance.size,
+                       local->rebalance.offset, local->rebalance.flags,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FSTAT:
+            STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                              subvol->fops->fstat, fd, local->xattr_req);
+            break;
+
+        case GF_FOP_FSETXATTR:
+            STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                              subvol->fops->fsetxattr, local->fd,
+                              local->rebalance.xattr, local->rebalance.flags,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FREMOVEXATTR:
+            STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                              subvol->fops->fremovexattr, local->fd, local->key,
+                              local->xattr_req);
+
+            break;
+
+        case GF_FOP_FXATTROP:
+            STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                       subvol->fops->fxattrop, local->fd,
+                       local->rebalance.flags, local->rebalance.xattr,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FGETXATTR:
+            STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr,
+                       local->fd, local->key, NULL);
+            break;
+
+        case GF_FOP_FINODELK:
+            STACK_WIND(frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                       local->key, local->fd, local->rebalance.lock_cmd,
+                       &local->rebalance.flock, local->xattr_req);
+            break;
+        default:
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
 
-        if (!frame)
-                return -1;
+    goto out;
 
-        local = frame->local;
+    /* Could not open the fd on the dst. Unwind */
 
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-        }
-        UNLOCK (&frame->lock);
+handle_err:
 
-        return this_call_cnt;
-}
+    switch (fop) {
+        case GF_FOP_WRITE:
+            DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
+        case GF_FOP_FLUSH:
+            DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+            break;
 
-int
-dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
-                           xlator_t **subvol)
-{
-        char          *new_name  = NULL;
-        char          *new_path  = NULL;
-        xlator_list_t *trav      = NULL;
-        char           key[1024] = {0,};
-        int            ret       = 0; /* not found */
-
-        /* Why do other tasks if first required 'char' itself is not there */
-        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
-                goto out;
+        case GF_FOP_FSETATTR:
+            DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
-        trav = this->children;
-        while (trav) {
-                snprintf (key, 1024, "*@%s:%s", this->name, trav->xlator->name);
-                if (fnmatch (key, loc->name, FNM_NOESCAPE) == 0) {
-                        new_name = GF_CALLOC(strlen (loc->name),
-                                             sizeof (char),
-                                             gf_common_mt_char);
-                        if (!new_name)
-                                goto out;
-                        if (fnmatch (key, loc->path, FNM_NOESCAPE) == 0) {
-                                new_path = GF_CALLOC(strlen (loc->path),
-                                                     sizeof (char),
-                                                     gf_common_mt_char);
-                                if (!new_path)
-                                        goto out;
-                                strncpy (new_path, loc->path, (strlen (loc->path) -
-                                                               strlen (key) + 1));
-                        }
-                        strncpy (new_name, loc->name, (strlen (loc->name) -
-                                                       strlen (key) + 1));
-
-                        if (new_loc) {
-                                new_loc->path   = ((new_path) ? new_path:
-                                                   gf_strdup (loc->path));
-                                new_loc->name   = new_name;
-                                new_loc->inode  = inode_ref (loc->inode);
-                                new_loc->parent = inode_ref (loc->parent);
-                        }
-                        *subvol         = trav->xlator;
-                        ret = 1;  /* success */
-                        goto out;
-                }
-                trav = trav->next;
-        }
-out:
-        if (!ret) {
-                /* !success */
-                GF_FREE (new_path);
-                GF_FREE (new_name);
-        }
-        return ret;
-}
+        case GF_FOP_ZEROFILL:
+            DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
-static xlator_t *
-dht_get_subvol_from_id(xlator_t *this, int client_id)
-{
-        xlator_t *xl = NULL;
-        dht_conf_t *conf = NULL;
-        char sid[6] = { 0 };
+        case GF_FOP_DISCARD:
+            DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
-        conf = this->private;
+        case GF_FOP_FALLOCATE:
+            DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
-        sprintf(sid, "%d", client_id);
-        if (dict_get_ptr(conf->leaf_to_subvol, sid, (void **) &xl))
-                xl = NULL;
+        case GF_FOP_FTRUNCATE:
+            DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
-        return xl;
-}
+        case GF_FOP_FSYNC:
+            DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
-int
-dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p)
-{
-        int         client_id = 0;
-        xlator_t   *subvol = 0;
-        dht_conf_t *conf = NULL;
+        case GF_FOP_READ:
+            DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
+            break;
 
-        if (!this->private)
-                return -1;
+        case GF_FOP_FSTAT:
+            DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
+            break;
 
-        conf = this->private;
+        case GF_FOP_FSETXATTR:
+            DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+            break;
 
-        client_id = gf_deitransform(this, y);
+        case GF_FOP_FREMOVEXATTR:
+            DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
+            break;
 
-        subvol = dht_get_subvol_from_id(this, client_id);
+        case GF_FOP_FXATTROP:
+            DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+            break;
 
-        if (!subvol)
-                subvol = conf->subvolumes[0];
+        case GF_FOP_FGETXATTR:
+            DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+            break;
 
-        if (subvol_p)
-                *subvol_p = subvol;
+        case GF_FOP_FINODELK:
+            DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
+            break;
 
-        return 0;
+        default:
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
+
+out:
+
+    return 0;
 }
 
-char *
-dht_lock_asprintf (dht_lock_t *lock)
-{
-        char *lk_buf                = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0, };
+/* Check once again if the fd has been opened on the cached subvol.
+ * If not, open and update the fd_ctx.
+ */
 
-        if (lock == NULL)
-                goto out;
+int
+dht_check_and_open_fd_on_subvol_task(void *data)
+{
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    fd_t *fd = NULL;
+    xlator_t *this = NULL;
+    xlator_t *subvol = NULL;
+
+    frame = data;
+    local = frame->local;
+    this = THIS;
+    fd = local->fd;
+    subvol = local->cached_subvol;
+
+    local->fd_checked = _gf_true;
+
+    if (fd_is_anonymous(fd) || dht_fd_open_on_dst(this, fd, subvol)) {
+        ret = 0;
+        goto out;
+    }
 
-        uuid_utoa_r (lock->loc.gfid, gfid);
+    gf_msg_debug(this->name, 0, "Opening fd (%p, flags=0%o) on file %s @ %s",
+                 fd, fd->flags, uuid_utoa(fd->inode->gfid), subvol->name);
 
-        gf_asprintf (&lk_buf, "%s:%s", lock->xl->name, gfid);
+    loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(loc.gfid, fd->inode->gfid);
 
-out:
-        return lk_buf;
-}
+    /* Open this on the dst subvol */
 
-void
-dht_log_lk_array (char *name, gf_loglevel_t log_level, dht_lock_t **lk_array,
-                  int count)
-{
-        int   i      = 0;
-        char *lk_buf = NULL;
+    SYNCTASK_SETID(0, 0);
 
-        if ((lk_array == NULL) || (count == 0))
-                goto out;
+    ret = syncop_open(subvol, &loc, (fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                      fd, NULL, NULL);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+                "fd=%p", fd, "flags=0%o", fd->flags, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), "name=%s", subvol->name, NULL);
+        /* This can happen if the cached subvol was updated in the
+         * inode_ctx and the fd was opened on the new cached suvol
+         * after this fop was wound on the old cached subvol.
+         * As we do not close the fd on the old subvol (a leak)
+         * don't treat ENOENT as an error and allow the phase1/phase2
+         * checks to handle it.
+         */
 
-        for (i = 0; i < count; i++) {
-                lk_buf = dht_lock_asprintf (lk_array[i]);
-                gf_msg (name, log_level, 0, DHT_MSG_LK_ARRAY_INFO,
-                        "%d. %s", i, lk_buf);
-                GF_FREE (lk_buf);
+        if ((-ret != ENOENT) && (-ret != ESTALE)) {
+            local->op_errno = -ret;
+            ret = -1;
+        } else {
+            ret = 0;
         }
 
+        local->op_errno = -ret;
+        ret = -1;
+
+    } else {
+        dht_fd_ctx_set(this, fd, subvol);
+    }
+
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 out:
-        return;
+    loc_wipe(&loc);
+
+    return ret;
 }
 
-void
-dht_lock_stack_destroy (call_frame_t *lock_frame)
+int
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
 
-        local = lock_frame->local;
+    /*
+            if (dht_fd_open_on_dst (this, fd, subvol))
+                    goto out;
+    */
+    local = frame->local;
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
+    ret = synctask_new(this->ctx->env, dht_check_and_open_fd_on_subvol_task,
+                       dht_check_and_open_fd_on_subvol_complete, frame, frame);
 
-        DHT_STACK_DESTROY (lock_frame);
-        return;
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SYNCTASK_CREATE_FAILED,
+                "to-check-and-open fd=%p", local->fd, NULL);
+    }
+
+    return ret;
 }
 
-void
-dht_lock_free (dht_lock_t *lock)
+int
+dht_frame_return(call_frame_t *frame)
 {
-        if (lock == NULL)
-                goto out;
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+
+    if (!frame)
+        return -1;
 
-        loc_wipe (&lock->loc);
-        GF_FREE (lock->domain);
-        mem_put (lock);
+    local = frame->local;
 
-out:
-        return;
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+    }
+    UNLOCK(&frame->lock);
+
+    return this_call_cnt;
 }
 
-void
-dht_lock_array_free (dht_lock_t **lk_array, int count)
-{
-        int            i       = 0;
-        dht_lock_t    *lock    = NULL;
+/*
+ * Use this function to specify which subvol you want the file created
+ * on - this need not be the hashed subvol.
+ * Format: <filename>@<this->name>:<subvol-name>
+ * Eg: file-1@vol1-dht:vol1-client-0
+ *     where vol1 is a pure distribute volume
+ *     will create file-1 on vol1-client-0
+ */
 
-        if (lk_array == NULL)
+int
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol)
+{
+    char *new_name = NULL;
+    char *new_path = NULL;
+    xlator_list_t *trav = NULL;
+    char key[1024] = {
+        0,
+    };
+    int ret = 0; /* not found */
+    int keylen = 0;
+    int name_len = 0;
+    int path_len = 0;
+
+    /* Why do other tasks if first required 'char' itself is not there */
+    if (!new_loc || !loc || !loc->name || !strchr(loc->name, '@')) {
+        /* Skip the GF_FREE checks here */
+        return ret;
+    }
+
+    trav = this->children;
+    while (trav) {
+        keylen = snprintf(key, sizeof(key), "*@%s:%s", this->name,
+                          trav->xlator->name);
+        /* Ignore '*' */
+        keylen = keylen - 1;
+        if (fnmatch(key, loc->name, FNM_NOESCAPE) == 0) {
+            name_len = strlen(loc->name) - keylen;
+            new_name = GF_MALLOC(name_len + 1, gf_common_mt_char);
+            if (!new_name)
                 goto out;
-
-        for (i = 0; i < count; i++) {
-                lock = lk_array[i];
-                lk_array[i] = NULL;
-                dht_lock_free (lock);
-        }
-
+            if (fnmatch(key, loc->path, FNM_NOESCAPE) == 0) {
+                path_len = strlen(loc->path) - keylen;
+                new_path = GF_MALLOC(path_len + 1, gf_common_mt_char);
+                if (!new_path)
+                    goto out;
+                snprintf(new_path, path_len + 1, "%s", loc->path);
+            }
+            snprintf(new_name, name_len + 1, "%s", loc->name);
+
+            if (new_loc) {
+                new_loc->path = ((new_path) ? new_path : gf_strdup(loc->path));
+                new_loc->name = new_name;
+                new_loc->inode = inode_ref(loc->inode);
+                new_loc->parent = inode_ref(loc->parent);
+            }
+            *subvol = trav->xlator;
+            ret = 1; /* success */
+            goto out;
+        }
+        trav = trav->next;
+    }
 out:
-        return;
+    if (!ret) {
+        /* !success */
+        GF_FREE(new_path);
+        GF_FREE(new_name);
+    }
+    return ret;
 }
 
-dht_lock_t *
-dht_lock_new (xlator_t *this, xlator_t *xl, loc_t *loc, short type,
-              const char *domain)
+static xlator_t *
+dht_get_subvol_from_id(xlator_t *this, int client_id)
 {
-        dht_conf_t *conf = NULL;
-        dht_lock_t *lock = NULL;
-
-        conf = this->private;
+    xlator_t *xl = NULL;
+    dht_conf_t *conf = NULL;
+    char *sid = NULL;
+    int32_t ret = -1;
 
-        lock = mem_get0 (conf->lock_pool);
-        if (lock == NULL)
-                goto out;
+    conf = this->private;
 
-        lock->xl = xl;
-        lock->type = type;
+    ret = gf_asprintf(&sid, "%d", client_id);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_ASPRINTF_FAILED, NULL);
+        goto out;
+    }
 
-        lock->domain = gf_strdup (domain);
-        if (lock->domain == NULL) {
-                dht_lock_free (lock);
-                lock = NULL;
-                goto out;
-        }
+    if (dict_get_ptr(conf->leaf_to_subvol, sid, (void **)&xl))
+        xl = NULL;
 
-        /* Fill only inode and gfid.
-           posix and protocol/server give preference to pargfid/basename over
-           gfid/inode for resolution if all the three parameters of loc_t are
-           present. I want to avoid the following hypothetical situation:
-
-           1. rebalance did a lookup on a dentry and got a gfid.
-           2. rebalance acquires lock on loc_t which was filled with gfid and
-              path (pargfid/bname) from step 1.
-           3. somebody deleted and recreated the same file
-           4. rename on the same path acquires lock on loc_t which now points
-              to a different inode (and hence gets the lock).
-           5. rebalance continues to migrate file (note that not all fops done
-              by rebalance during migration are inode/gfid based Eg., unlink)
-           6. rename continues.
-        */
-        lock->loc.inode = inode_ref (loc->inode);
-        loc_gfid (loc, lock->loc.gfid);
+    GF_FREE(sid);
 
 out:
-        return lock;
+    return xl;
 }
 
 int
-dht_local_lock_init (call_frame_t *frame, dht_lock_t **lk_array,
-                     int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol_p)
 {
-        int          ret   = -1;
-        dht_local_t *local = NULL;
+    int client_id = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
 
-        local = frame->local;
+    if (!this->private)
+        return -1;
 
-        if (local == NULL) {
-                local = dht_local_init (frame, NULL, NULL, 0);
-        }
+    conf = this->private;
 
-        if (local == NULL) {
-                goto out;
-        }
+    client_id = gf_deitransform(this, y);
 
-        local->lock.inodelk_cbk = inodelk_cbk;
-        local->lock.locks = lk_array;
-        local->lock.lk_count = lk_count;
+    subvol = dht_get_subvol_from_id(this, client_id);
 
-        ret = dht_lock_order_requests (local->lock.locks,
-                                       local->lock.lk_count);
-        if (ret < 0)
-                goto out;
+    if (!subvol)
+        subvol = conf->subvolumes[0];
 
-        ret = 0;
-out:
-        return ret;
+    if (subvol_p)
+        *subvol_p = subvol;
+
+    return 0;
 }
 
 void
-dht_local_wipe (xlator_t *this, dht_local_t *local)
+dht_local_wipe(xlator_t *this, dht_local_t *local)
 {
-        if (!local)
-                return;
+    int i = 0;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
+    if (!local)
+        return;
 
-        if (local->xattr)
-                dict_unref (local->xattr);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->loc2);
+    loc_wipe(&local->loc2_copy);
 
-        if (local->inode)
-                inode_unref (local->inode);
+    if (local->xattr)
+        dict_unref(local->xattr);
 
-        if (local->layout) {
-                dht_layout_unref (this, local->layout);
-                local->layout = NULL;
-        }
+    if (local->inode)
+        inode_unref(local->inode);
 
-        loc_wipe (&local->linkfile.loc);
+    if (local->layout) {
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+    }
 
-        if (local->linkfile.xattr)
-                dict_unref (local->linkfile.xattr);
+    loc_wipe(&local->linkfile.loc);
 
-        if (local->linkfile.inode)
-                inode_unref (local->linkfile.inode);
+    if (local->linkfile.xattr)
+        dict_unref(local->linkfile.xattr);
 
-        if (local->fd) {
-                fd_unref (local->fd);
-                local->fd = NULL;
-        }
+    if (local->linkfile.inode)
+        inode_unref(local->linkfile.inode);
 
-        if (local->params) {
-                dict_unref (local->params);
-                local->params = NULL;
-        }
+    if (local->fd) {
+        fd_unref(local->fd);
+        local->fd = NULL;
+    }
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+    if (local->params) {
+        dict_unref(local->params);
+        local->params = NULL;
+    }
 
-        if (local->selfheal.layout) {
-                dht_layout_unref (this, local->selfheal.layout);
-                local->selfheal.layout = NULL;
-        }
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+    if (local->mds_xattr)
+        dict_unref(local->mds_xattr);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-        if (local->selfheal.refreshed_layout) {
-                dht_layout_unref (this, local->selfheal.refreshed_layout);
-                local->selfheal.refreshed_layout = NULL;
-        }
+    if (local->selfheal.layout) {
+        dht_layout_unref(this, local->selfheal.layout);
+        local->selfheal.layout = NULL;
+    }
 
-        dht_lock_array_free (local->lock.locks, local->lock.lk_count);
-        GF_FREE (local->lock.locks);
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
 
-        GF_FREE (local->newpath);
+    for (i = 0; i < 2; i++) {
+        dht_lock_array_free(local->lock[i].ns.parent_layout.locks,
+                            local->lock[i].ns.parent_layout.lk_count);
 
-        GF_FREE (local->key);
+        GF_FREE(local->lock[i].ns.parent_layout.locks);
 
-        if (local->rebalance.xdata)
-                dict_unref (local->rebalance.xdata);
+        dht_lock_array_free(local->lock[i].ns.directory_ns.locks,
+                            local->lock[i].ns.directory_ns.lk_count);
+        GF_FREE(local->lock[i].ns.directory_ns.locks);
+    }
 
-        if (local->rebalance.xattr)
-                dict_unref (local->rebalance.xattr);
+    GF_FREE(local->key);
 
-        GF_FREE (local->rebalance.vector);
+    if (local->rebalance.xdata)
+        dict_unref(local->rebalance.xdata);
 
-        if (local->rebalance.iobref)
-                iobref_unref (local->rebalance.iobref);
+    if (local->rebalance.xattr)
+        dict_unref(local->rebalance.xattr);
 
-        if (local->stub) {
-                call_stub_destroy (local->stub);
-                local->stub = NULL;
-        }
+    if (local->rebalance.dict)
+        dict_unref(local->rebalance.dict);
 
-        mem_put (local);
-}
+    GF_FREE(local->rebalance.vector);
+
+    if (local->rebalance.iobref)
+        iobref_unref(local->rebalance.iobref);
 
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
+
+    if (local->ret_cache)
+        GF_FREE(local->ret_cache);
+
+    mem_put(local);
+}
 
 dht_local_t *
-dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
 {
-        dht_local_t *local = NULL;
-        inode_t     *inode = NULL;
-        int          ret   = 0;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    int ret = 0;
 
-        local = mem_get0 (THIS->local_pool);
-        if (!local)
-                goto out;
+    local = mem_get0(THIS->local_pool);
+    if (!local)
+        goto out;
 
-        if (loc) {
-                ret = loc_copy (&local->loc, loc);
-                if (ret)
-                        goto out;
+    if (loc) {
+        ret = loc_copy(&local->loc, loc);
+        if (ret)
+            goto out;
 
-                inode = loc->inode;
-        }
+        inode = loc->inode;
+    }
 
-        if (fd) {
-                local->fd = fd_ref (fd);
-                if (!inode)
-                        inode = fd->inode;
-        }
+    if (fd) {
+        local->fd = fd_ref(fd);
+        if (!inode)
+            inode = fd->inode;
+    }
 
-        local->op_ret   = -1;
-        local->op_errno = EUCLEAN;
-        local->fop      = fop;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop = fop;
 
-        if (inode) {
-                local->layout   = dht_layout_get (frame->this, inode);
-                local->cached_subvol = dht_subvol_get_cached (frame->this,
-                                                              inode);
-        }
+    if (inode) {
+        local->layout = dht_layout_get(frame->this, inode);
+        local->cached_subvol = dht_subvol_get_cached(frame->this, inode);
+    }
 
-        frame->local = local;
+    frame->local = local;
 
 out:
-        if (ret) {
-                if (local)
-                        mem_put (local);
-                local = NULL;
-        }
-        return local;
+    if (ret) {
+        if (local)
+            mem_put(local);
+        local = NULL;
+    }
+    return local;
 }
 
 xlator_t *
-dht_first_up_subvol (xlator_t *this)
+dht_first_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
-        time_t      time = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
+    time_t time = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvol_up_time[i]) {
-                                if (!time) {
-                                        time = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                } else if (time > conf->subvol_up_time[i]) {
-                                        time  = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                }
-                        }
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvol_up_time[i]) {
+                if (!time) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
+                } else if (time > conf->subvol_up_time[i]) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
                 }
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_last_up_subvol (xlator_t *this)
+dht_last_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = conf->subvolume_cnt-1; i >= 0; i--) {
-                        if (conf->subvolume_status[i]) {
-                                child = conf->subvolumes[i];
-                                break;
-                        }
-                }
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = conf->subvolume_cnt - 1; i >= 0; i--) {
+            if (conf->subvolume_status[i]) {
+                child = conf->subvolumes[i];
+                break;
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_subvol_get_hashed (xlator_t *this, loc_t *loc)
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
-        dht_conf_t *conf = NULL;
-        dht_methods_t *methods = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        methods = &(conf->methods);
+    methods = &(conf->methods);
 
-        if (__is_root_gfid (loc->gfid)) {
-                subvol = dht_first_up_subvol (this);
-                goto out;
-        }
+    if (__is_root_gfid(loc->gfid)) {
+        subvol = dht_first_up_subvol(this);
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, loc->parent, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->name, out);
 
-        layout = dht_layout_get (this, loc->parent);
+    layout = dht_layout_get(this, loc->parent);
 
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "Missing layout. path=%s, parent gfid =%s",
-                              loc->path, uuid_utoa (loc->parent->gfid));
-                goto out;
-        }
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "Missing layout. path=%s, parent gfid =%s",
+                     loc->path, uuid_utoa(loc->parent->gfid));
+        goto out;
+    }
 
-        subvol = methods->layout_search (this, layout, loc->name);
+    subvol = methods->layout_search(this, layout, loc->name);
 
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "No hashed subvolume for path=%s",
-                              loc->path);
-                goto out;
-        }
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "No hashed subvolume for path=%s",
+                     loc->path);
+        goto out;
+    }
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_get_cached (xlator_t *this, inode_t *inode)
+dht_subvol_get_cached(xlator_t *this, inode_t *inode)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
 
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        layout = dht_layout_get (this, inode);
+    layout = dht_layout_get(this, inode);
 
-        if (!layout) {
-                goto out;
-        }
+    if (!layout) {
+        goto out;
+    }
 
-        subvol = layout->list[0].xlator;
+    subvol = layout->list[0].xlator;
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_next (xlator_t *this, xlator_t *prev)
+dht_subvol_next(xlator_t *this, xlator_t *prev)
 {
-        dht_conf_t *conf = NULL;
-        int         i = 0;
-        xlator_t   *next = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == prev) {
-                        if ((i + 1) < conf->subvolume_cnt)
-                                next = conf->subvolumes[i + 1];
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            if ((i + 1) < conf->subvolume_cnt)
+                next = conf->subvolumes[i + 1];
+            break;
         }
+    }
 
 out:
-        return next;
+    return next;
 }
 
 /* This func wraps around, if prev is actually the last subvol.
  */
 xlator_t *
-dht_subvol_next_available (xlator_t *this, xlator_t *prev)
-{
-        dht_conf_t *conf = NULL;
-        int         i = 0;
-        xlator_t   *next = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == prev) {
-                        /* if prev is last in conf->subvolumes, then wrap
-                         * around.
-                         */
-                        if ((i + 1) < conf->subvolume_cnt) {
-                                next = conf->subvolumes[i + 1];
-                        } else {
-                                next = conf->subvolumes[0];
-                        }
-                        break;
-                }
-        }
+dht_subvol_next_available(xlator_t *this, xlator_t *prev)
+{
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            /* if prev is last in conf->subvolumes, then wrap
+             * around.
+             */
+            if ((i + 1) < conf->subvolume_cnt) {
+                next = conf->subvolumes[i + 1];
+            } else {
+                next = conf->subvolumes[0];
+            }
+            break;
+        }
+    }
 
 out:
-        return next;
+    return next;
 }
 int
-dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol)
 {
-        int i = 0;
-        int ret = -1;
-        dht_conf_t *conf = NULL;
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (subvol == conf->subvolumes[i]) {
-                        ret = i;
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (subvol == conf->subvolumes[i]) {
+            ret = i;
+            break;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
+#define set_if_greater(a, b)                                                   \
+    do {                                                                       \
+        if ((a) < (b))                                                         \
+            (a) = (b);                                                         \
+    } while (0)
 
-#define set_if_greater(a, b) do {               \
-                if ((a) < (b))                  \
-                        (a) = (b);              \
-        } while (0)
-
-
-#define set_if_greater_time(a, an, b, bn) do {                          \
-                if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))){    \
-                        (a) = (b);                                      \
-                        (an) = (bn);                                    \
-                }                                                       \
-        } while (0)                                                     \
-
+#define set_if_greater_time(a, an, b, bn)                                      \
+    do {                                                                       \
+        if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))) {                  \
+            (a) = (b);                                                         \
+            (an) = (bn);                                                       \
+        }                                                                      \
+    } while (0)
 
 int
-dht_iatt_merge (xlator_t *this, struct iatt *to,
-                struct iatt *from, xlator_t *subvol)
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from)
 {
-        if (!from || !to)
-                return 0;
+    if (!from || !to)
+        return 0;
 
-        to->ia_dev      = from->ia_dev;
+    to->ia_dev = from->ia_dev;
 
-        gf_uuid_copy (to->ia_gfid, from->ia_gfid);
+    gf_uuid_copy(to->ia_gfid, from->ia_gfid);
 
-        to->ia_ino      = from->ia_ino;
-        to->ia_prot     = from->ia_prot;
-        to->ia_type     = from->ia_type;
-        to->ia_nlink    = from->ia_nlink;
-        to->ia_rdev     = from->ia_rdev;
-        to->ia_size    += from->ia_size;
-        to->ia_blksize  = from->ia_blksize;
-        to->ia_blocks  += from->ia_blocks;
+    to->ia_ino = from->ia_ino;
+    to->ia_prot = from->ia_prot;
+    to->ia_type = from->ia_type;
+    to->ia_nlink = from->ia_nlink;
+    to->ia_rdev = from->ia_rdev;
+    to->ia_size += from->ia_size;
+    to->ia_blksize = from->ia_blksize;
+    to->ia_blocks += from->ia_blocks;
 
-        if (IA_ISDIR (from->ia_type)) {
-                to->ia_blocks = DHT_DIR_STAT_BLOCKS;
-                to->ia_size = DHT_DIR_STAT_SIZE;
-        }
-        set_if_greater (to->ia_uid, from->ia_uid);
-        set_if_greater (to->ia_gid, from->ia_gid);
+    if (IA_ISDIR(from->ia_type)) {
+        to->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        to->ia_size = DHT_DIR_STAT_SIZE;
+    }
+    set_if_greater(to->ia_uid, from->ia_uid);
+    set_if_greater(to->ia_gid, from->ia_gid);
 
-        set_if_greater_time(to->ia_atime, to->ia_atime_nsec,
-                            from->ia_atime, from->ia_atime_nsec);
-        set_if_greater_time (to->ia_mtime, to->ia_mtime_nsec,
-                             from->ia_mtime, from->ia_mtime_nsec);
-        set_if_greater_time (to->ia_ctime, to->ia_ctime_nsec,
-                             from->ia_ctime, from->ia_ctime_nsec);
+    set_if_greater_time(to->ia_atime, to->ia_atime_nsec, from->ia_atime,
+                        from->ia_atime_nsec);
+    set_if_greater_time(to->ia_mtime, to->ia_mtime_nsec, from->ia_mtime,
+                        from->ia_mtime_nsec);
+    set_if_greater_time(to->ia_ctime, to->ia_ctime_nsec, from->ia_ctime,
+                        from->ia_ctime_nsec);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name)
 {
-        if (!child) {
-                goto err;
-        }
+    if (!child) {
+        goto err;
+    }
 
-        if (strcmp (parent->path, "/") == 0)
-                gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                gf_asprintf ((char **)&child->path, "%s/%s", parent->path, name);
+    if (strcmp(parent->path, "/") == 0)
+        gf_asprintf((char **)&child->path, "/%s", name);
+    else
+        gf_asprintf((char **)&child->path, "%s/%s", parent->path, name);
 
-        if (!child->path) {
-                goto err;
-        }
+    if (!child->path) {
+        goto err;
+    }
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+    child->name = strrchr(child->path, '/');
+    if (child->name)
+        child->name++;
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
+    child->parent = inode_ref(parent->inode);
+    child->inode = inode_new(parent->inode->table);
 
-        if (!child->inode) {
-                goto err;
-        }
+    if (!child->inode) {
+        goto err;
+    }
 
-        return 0;
+    return 0;
 err:
-        loc_wipe (child);
-        return -1;
+    if (child) {
+        loc_wipe(child);
+    }
+    return -1;
 }
 
 int
-dht_init_local_subvolumes (xlator_t *this, dht_conf_t *conf)
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (!conf)
-                return -1;
+    if (!conf)
+        return -1;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        conf->local_subvols = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                        gf_dht_mt_xlator_t);
-        if (!conf->local_subvols) {
-                return -1;
-        }
+    conf->local_subvols = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                    gf_dht_mt_xlator_t);
 
-        conf->local_subvols_cnt = 0;
+    /* FIX FIX : do this dynamically*/
+    conf->local_nodeuuids = GF_CALLOC(cnt, sizeof(subvol_nodeuuids_info_t),
+                                      gf_dht_nodeuuids_t);
 
-        return 0;
+    if (!conf->local_subvols || !conf->local_nodeuuids) {
+        return -1;
+    }
+
+    conf->local_subvols_cnt = 0;
+
+    return 0;
 }
 
 int
-dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (!conf)
-                return -1;
+    if (!conf)
+        return -1;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        conf->subvolumes = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                      gf_dht_mt_xlator_t);
-        if (!conf->subvolumes) {
-                return -1;
-        }
-        conf->subvolume_cnt = cnt;
+    conf->subvolumes = GF_CALLOC(cnt, sizeof(xlator_t *), gf_dht_mt_xlator_t);
+    if (!conf->subvolumes) {
+        return -1;
+    }
+    conf->subvolume_cnt = cnt;
+    /* Doesn't make sense to do any dht layer tasks
+       if the subvol count is 1. Set it as pass_through */
+    if (cnt == 1)
+        this->pass_through = _gf_true;
 
-        conf->local_subvols_cnt = 0;
+    conf->local_subvols_cnt = 0;
 
-        dht_set_subvol_range(this);
+    dht_set_subvol_range(this);
 
-        cnt = 0;
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                conf->subvolumes[cnt++] = subvols->xlator;
+    cnt = 0;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        conf->subvolumes[cnt++] = subvols->xlator;
 
-        conf->subvolume_status = GF_CALLOC (cnt, sizeof (char),
-                                            gf_dht_mt_char);
-        if (!conf->subvolume_status) {
-                return -1;
-        }
+    conf->subvolume_status = GF_CALLOC(cnt, sizeof(char), gf_dht_mt_char);
+    if (!conf->subvolume_status) {
+        return -1;
+    }
 
-        conf->last_event = GF_CALLOC (cnt, sizeof (int),
-                                      gf_dht_mt_char);
-        if (!conf->last_event) {
-                return -1;
-        }
+    conf->last_event = GF_CALLOC(cnt, sizeof(int), gf_dht_mt_char);
+    if (!conf->last_event) {
+        return -1;
+    }
 
-        conf->subvol_up_time = GF_CALLOC (cnt, sizeof (time_t),
-                                          gf_dht_mt_subvol_time);
-        if (!conf->subvol_up_time) {
-                return -1;
-        }
+    conf->subvol_up_time = GF_CALLOC(cnt, sizeof(time_t),
+                                     gf_dht_mt_subvol_time);
+    if (!conf->subvol_up_time) {
+        return -1;
+    }
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                return -1;
-        }
+    conf->du_stats = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_du_t),
+                               gf_dht_mt_dht_du_t);
+    if (!conf->du_stats) {
+        return -1;
+    }
 
-        conf->decommissioned_bricks = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                                 gf_dht_mt_xlator_t);
-        if (!conf->decommissioned_bricks) {
-                return -1;
-        }
+    conf->decommissioned_bricks = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                            gf_dht_mt_xlator_t);
+    if (!conf->decommissioned_bricks) {
+        return -1;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 /*
  op_ret values :
   0 : Success.
@@ -1063,239 +1238,274 @@ dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
 */
 
 static int
-dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
+dht_migration_complete_check_done(int op_ret, call_frame_t *frame, void *data)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0)
+        goto out;
 
-        if (local->cached_subvol == NULL) {
-                local->op_errno = EINVAL;
-                goto out;
-        }
+    if (local->cached_subvol == NULL) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
 
-        subvol = local->cached_subvol;
+    subvol = local->cached_subvol;
 
 out:
-        local->rebalance.target_op_fn (THIS, subvol, frame, op_ret);
+    local->rebalance.target_op_fn(THIS, subvol, frame, op_ret);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_migration_complete_check_task (void *data)
-{
-        int                 ret         = -1;
-        xlator_t           *src_node    = NULL;
-        xlator_t           *dst_node    = NULL, *linkto_target = NULL;
-        dht_local_t        *local       = NULL;
-        dict_t             *dict        = NULL;
-        struct iatt         stbuf       = {0,};
-        xlator_t           *this        = NULL;
-        call_frame_t       *frame       = NULL;
-        loc_t               tmp_loc     = {0,};
-        char               *path        = NULL;
-        dht_conf_t         *conf        = NULL;
-        inode_t            *inode       = NULL;
-        fd_t               *iter_fd     = NULL;
-        uint64_t            tmp_miginfo = 0;
-        dht_migrate_info_t *miginfo     = NULL;
-        int                 open_failed = 0;
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-        conf = this->private;
-
-        src_node = local->cached_subvol;
-
-        if (!local->loc.inode && !local->fd) {
-                local->op_errno = EINVAL;
-                goto out;
-        }
-
-        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
-
-        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
-         * as root:root. If a fd is already open, access check wont be done*/
-
-        if (!local->loc.inode) {
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        conf->link_xattr_name, NULL, NULL);
-        } else {
-                SYNCTASK_SETID (0, 0);
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       conf->link_xattr_name, NULL, NULL);
-                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        }
-
-
-        /*
-         * Each DHT xlator layer has its own name for the linkto xattr.
-         * If the file mode bits indicate the the file is being migrated but
-         * this layer's linkto xattr is not set, it means that another
-         * DHT layer is migrating the file. In this case, return 1 so
-         * the mode bits can be passed on to the higher layer for appropriate
-         * action.
+dht_migration_complete_check_task(void *data)
+{
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL, *linkto_target = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    char *path = NULL;
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+    int open_failed = 0;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
+
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
+
+    if (!local->loc.inode) {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    } else {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not set, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+    if (-ret == ENODATA) {
+        /* This DHT translator is not migrating this file */
+
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    if (!ret)
+        linkto_target = dht_linkfile_subvol(this, NULL, NULL, dict);
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    ret = syncop_lookup(this, &tmp_loc, &stbuf, 0, 0, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", this->name, NULL);
+        local->op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dst_node = dht_subvol_get_cached(this, tmp_loc.inode);
+    if (linkto_target && dst_node != linkto_target) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_LINKFILE,
+                "linkto_target_name=%s", linkto_target->name, "dst_name=%s",
+                dst_node->name, NULL);
+    }
+
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "dst_name=%s", dst_node->name, NULL);
+        ret = -1;
+        local->op_errno = EIO;
+        goto out;
+    }
+
+    /* update local. A layout is set in inode-ctx in lookup already */
+
+    dht_layout_unref(this, local->layout);
+
+    local->layout = dht_layout_get(frame->this, inode);
+    local->cached_subvol = dst_node;
+
+    ret = 0;
+
+    /* once we detect the migration complete, the inode-ctx2 is no more
+       required.. delete the ctx and also, it means, open() already
+       done on all the fd of inode */
+    ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+    if (tmp_miginfo) {
+        miginfo = (void *)(uintptr_t)tmp_miginfo;
+        GF_REF_PUT(miginfo);
+        goto out;
+    }
+
+    /* perform 'open()' on all the fd's present on the inode */
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
+        }
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
          */
-        if (-ret == ENODATA) {
-                /* This DHT translator is not migrating this file */
-
-                ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
-                if (tmp_miginfo) {
-
-                        /* This can be a problem if the file was
-                         * migrated by two different layers. Raise
-                         * a warning here.
-                         */
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_HAS_MIGINFO,
-                                "%s: Found miginfo in the inode ctx",
-                                tmp_loc.path ? tmp_loc.path :
-                                uuid_utoa (tmp_loc.gfid));
-
-                        miginfo = (void *)tmp_miginfo;
-                        GF_REF_PUT (miginfo);
-                }
-                ret = 1;
-                goto out;
-        }
-
-        if (!ret)
-                linkto_target = dht_linkfile_subvol (this, NULL, NULL, dict);
-
-        if (local->loc.inode) {
-                loc_copy (&tmp_loc, &local->loc);
-        } else {
-                tmp_loc.inode = inode_ref (inode);
-                gf_uuid_copy (tmp_loc.gfid, inode->gfid);
-        }
+        fd_ref(iter_fd);
 
-        ret = syncop_lookup (this, &tmp_loc, &stbuf, 0, 0, 0);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_FILE_LOOKUP_FAILED,
-                        "%s: failed to lookup the file on %s",
-                        tmp_loc.path ? tmp_loc.path : uuid_utoa (tmp_loc.gfid),
-                        this->name);
-                local->op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+        UNLOCK(&inode->lock);
 
-        dst_node = dht_subvol_get_cached (this, tmp_loc.inode);
-        if (linkto_target && dst_node != linkto_target) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_INVALID_LINKFILE,
-                        "linkto target (%s) is "
-                        "different from cached-subvol (%s). Treating %s as "
-                        "destination subvol", linkto_target->name,
-                        dst_node->name, dst_node->name);
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
         }
+        if (skip_open)
+            goto next;
 
-        if (gf_uuid_compare (stbuf.ia_gfid, tmp_loc.inode->gfid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "%s: gfid different on the target file on %s",
-                                tmp_loc.path ? tmp_loc.path :
-                                uuid_utoa (tmp_loc.gfid), dst_node->name);
-                ret = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-
-        /* update local. A layout is set in inode-ctx in lookup already */
-
-        dht_layout_unref (this, local->layout);
-
-        local->layout   = dht_layout_get (frame->this, inode);
-        local->cached_subvol = dst_node;
-
-        ret = 0;
-
-        /* once we detect the migration complete, the inode-ctx2 is no more
-           required.. delete the ctx and also, it means, open() already
-           done on all the fd of inode */
-        ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
-        if (tmp_miginfo) {
-                miginfo = (void *)tmp_miginfo;
-                GF_REF_PUT (miginfo);
-                goto out;
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "id=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+
+            open_failed = 1;
+            local->op_errno = -ret;
+            ret = -1;
+        } else {
+            dht_fd_ctx_set(this, iter_fd, dst_node);
         }
 
-        if (list_empty (&inode->fd_list))
-                goto out;
+    next:
+        LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
 
-        /* perform open as root:root. There is window between linkfile
-         * creation(root:root) and setattr with the correct uid/gid
-         */
-        SYNCTASK_SETID(0, 0);
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 
-        /* perform 'open()' on all the fd's present on the inode */
-        tmp_loc.inode = inode;
-        inode_path (inode, NULL, &path);
-        if (path)
-                tmp_loc.path = path;
-
-        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-
-                if (fd_is_anonymous (iter_fd))
-                        continue;
-                if (dht_fd_open_on_dst (this, iter_fd, dst_node))
-                        continue;
-
-                /* flags for open are stripped down to allow following the
-                 * new location of the file, otherwise we can get EEXIST or
-                 * truncate the file again as rebalance is moving the data */
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   (iter_fd->flags &
-                                   ~(O_CREAT | O_EXCL | O_TRUNC)),
-                                   iter_fd, NULL, NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_OPEN_FD_ON_DST_FAILED, "failed"
-                                " to open the fd"
-                                " (%p, flags=0%o) on file %s @ %s",
-                                iter_fd, iter_fd->flags, path,
-                                dst_node->name);
-
-                        open_failed = 1;
-                        local->op_errno = -ret;
-                        ret = -1;
-                } else {
-                        dht_fd_ctx_set (this, iter_fd, dst_node);
-                }
-        }
+    if (open_failed) {
+        ret = -1;
+        goto unlock;
+    }
+    ret = 0;
 
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+unlock:
+    UNLOCK(&inode->lock);
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
 
-        if (open_failed) {
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
 out:
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        return ret;
+    return ret;
 }
 
 int
-dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame)
 {
-        int         ret     = -1;
-
-        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
-                            dht_migration_complete_check_done,
-                            frame, frame);
-        return ret;
+    int ret = -1;
 
+    ret = synctask_new(this->ctx->env, dht_migration_complete_check_task,
+                       dht_migration_complete_check_done, frame, frame);
+    return ret;
 }
 
 /* During 'in-progress' state, both nodes should have the file */
@@ -1306,1150 +1516,789 @@ dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
   1 : File is being migrated but not by this DHT layer.
 */
 static int
-dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data)
+dht_inprogress_check_done(int op_ret, call_frame_t *frame, void *data)
 {
-        dht_local_t *local      = NULL;
-        xlator_t    *dst_subvol = NULL, *src_subvol = NULL;
-        inode_t     *inode      = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *dst_subvol = NULL, *src_subvol = NULL;
+    inode_t *inode = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0)
+        goto out;
 
-        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    inode = local->loc.inode ? local->loc.inode : local->fd->inode;
 
-        dht_inode_ctx_get_mig_info (THIS, inode, &src_subvol, &dst_subvol);
-        if (dht_mig_info_is_invalid (local->cached_subvol,
-                                     src_subvol, dst_subvol)) {
-                dst_subvol = dht_subvol_get_cached (THIS, inode);
-                if (!dst_subvol) {
-                        local->op_errno = EINVAL;
-                        goto out;
-                }
+    dht_inode_ctx_get_mig_info(THIS, inode, &src_subvol, &dst_subvol);
+    if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol, dst_subvol)) {
+        dst_subvol = dht_subvol_get_cached(THIS, inode);
+        if (!dst_subvol) {
+            local->op_errno = EINVAL;
+            goto out;
         }
+    }
 
 out:
-        local->rebalance.target_op_fn (THIS, dst_subvol, frame, op_ret);
+    local->rebalance.target_op_fn(THIS, dst_subvol, frame, op_ret);
 
-        return 0;
+    return 0;
 }
 
 static int
-dht_rebalance_inprogress_task (void *data)
-{
-        int           ret             = -1;
-        xlator_t     *src_node        = NULL;
-        xlator_t     *dst_node        = NULL;
-        dht_local_t  *local           = NULL;
-        dict_t       *dict            = NULL;
-        call_frame_t *frame           = NULL;
-        xlator_t     *this            = NULL;
-        char         *path            = NULL;
-        struct iatt   stbuf           = {0,};
-        loc_t         tmp_loc         = {0,};
-        dht_conf_t   *conf            = NULL;
-        inode_t      *inode           = NULL;
-        fd_t         *iter_fd         = NULL;
-        int           open_failed     = 0;
-        uint64_t      tmp_miginfo     = 0;
-        dht_migrate_info_t *miginfo   = NULL;
-
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-        conf = this->private;
-
-        src_node = local->cached_subvol;
-
-        if (!local->loc.inode && !local->fd)
-                goto out;
-
-        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+dht_rebalance_inprogress_task(void *data)
+{
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *path = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    int open_failed = 0;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd)
+        goto out;
+
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
+    if (local->loc.inode) {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    } else {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not present, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+
+    if (-ret == ENODATA) {
+        /* This DHT layer is not migrating this file */
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    dst_node = dht_linkfile_subvol(this, NULL, NULL, dict);
+    if (!dst_node) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    local->rebalance.target_node = dst_node;
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    /* lookup on dst */
+    ret = syncop_lookup(dst_node, &tmp_loc, &stbuf, NULL, NULL, NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
+         */
 
-        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
-         * as root:root. If a fd is already open, access check wont be done*/
-        if (local->loc.inode) {
-                SYNCTASK_SETID (0, 0);
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       conf->link_xattr_name, NULL, NULL);
-                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        } else {
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        conf->link_xattr_name, NULL, NULL);
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
         }
 
-        /*
-         * Each DHT xlator layer has its own name for the linkto xattr.
-         * If the file mode bits indicate the the file is being migrated but
-         * this layer's linkto xattr is not present, it means that another
-         * DHT layer is migrating the file. In this case, return 1 so
-         * the mode bits can be passed on to the higher layer for appropriate
-         * action.
+        /* Yes, this is ugly but there isn't a cleaner way to do this
+         * the fd_ref is an atomic increment so not too bad. We want to
+         * reduce the number of inode locks and unlocks.
          */
 
-        if (-ret == ENODATA) {
-                /* This DHT layer is not migrating this file */
-                ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
-                if (tmp_miginfo) {
-                        /* This can be a problem if the file was
-                         * migrated by two different layers. Raise
-                         * a warning here.
-                         */
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_HAS_MIGINFO,
-                                "%s: Found miginfo in the inode ctx",
-                                tmp_loc.path ? tmp_loc.path :
-                                uuid_utoa (tmp_loc.gfid));
-                        miginfo = (void *)tmp_miginfo;
-                        GF_REF_PUT (miginfo);
-                }
-                ret = 1;
-                goto out;
-        }
+        fd_ref(iter_fd);
+        UNLOCK(&inode->lock);
 
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "%s: failed to get the 'linkto' xattr",
-                        local->loc.path);
-                ret = -1;
-                goto out;
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
         }
+        if (skip_open)
+            goto next;
 
-        dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
-        if (!dst_node) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_SUBVOL_NOT_FOUND,
-                        "%s: failed to get the 'linkto' xattr from dict",
-                        local->loc.path);
-                ret = -1;
-                goto out;
-        }
-
-        local->rebalance.target_node = dst_node;
-
-        if (local->loc.inode) {
-                loc_copy (&tmp_loc, &local->loc);
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "fd=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+            ret = -1;
+            open_failed = 1;
         } else {
-                tmp_loc.inode = inode_ref (inode);
-                gf_uuid_copy (tmp_loc.gfid, inode->gfid);
+            /* Potential fd leak if this fails here as it will be
+               reopened at the next Phase1/2 check */
+            dht_fd_ctx_set(this, iter_fd, dst_node);
         }
 
-        /* lookup on dst */
-        ret = syncop_lookup (dst_node, &tmp_loc, &stbuf, NULL,
-                             NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_FILE_LOOKUP_ON_DST_FAILED,
-                        "%s: failed to lookup the file on %s",
-                        tmp_loc.path ? tmp_loc.path : uuid_utoa (tmp_loc.gfid),
-                        dst_node->name);
-                ret = -1;
-                goto out;
-        }
-
-        if (gf_uuid_compare (stbuf.ia_gfid, tmp_loc.inode->gfid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on the target file on %s",
-                        tmp_loc.path ? tmp_loc.path : uuid_utoa (tmp_loc.gfid),
-                        dst_node->name);
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+    next:
+        LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
 
-        if (list_empty (&inode->fd_list))
-                goto done;
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 
-        /* perform open as root:root. There is window between linkfile
-         * creation(root:root) and setattr with the correct uid/gid
-         */
-        SYNCTASK_SETID (0, 0);
-
-        inode_path (inode, NULL, &path);
-        if (path)
-                tmp_loc.path = path;
-
-        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                if (fd_is_anonymous (iter_fd))
-                        continue;
-
-                if (dht_fd_open_on_dst (this, iter_fd, dst_node))
-                        continue;
-                /* flags for open are stripped down to allow following the
-                 * new location of the file, otherwise we can get EEXIST or
-                 * truncate the file again as rebalance is moving the data */
-                ret = syncop_open (dst_node, &tmp_loc,
-                                  (iter_fd->flags &
-                                   ~(O_CREAT | O_EXCL | O_TRUNC)),
-                                   iter_fd, NULL, NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_OPEN_FD_ON_DST_FAILED,
-                                "failed to send open "
-                                "the fd (%p, flags=0%o) on file %s @ %s",
-                                iter_fd, iter_fd->flags, path,
-                                dst_node->name);
-                        ret = -1;
-                        open_failed = 1;
-                } else {
-                        /* Potential fd leak if this fails here as it will be
-                           reopened at the next Phase1/2 check */
-                        dht_fd_ctx_set (this, iter_fd, dst_node);
-                }
-
-        }
-
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-
-        if (open_failed) {
-                ret = -1;
-                goto out;
-        }
-
-done:
-        ret = dht_inode_ctx_set_mig_info (this, inode, src_node, dst_node);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_SET_INODE_CTX_FAILED,
-                        "%s: failed to set inode-ctx target file at %s",
-                        local->loc.path, dst_node->name);
-                goto out;
-        }
-
-        ret = 0;
+unlock:
+    UNLOCK(&inode->lock);
+
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
+    if (open_failed) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dht_inode_ctx_set_mig_info(this, inode, src_node, dst_node);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                "path=%s", local->loc.path, "name=%s", dst_node->name, NULL);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        loc_wipe (&tmp_loc);
-        return ret;
+    if (dict) {
+        dict_unref(dict);
+    }
+
+    loc_wipe(&tmp_loc);
+    return ret;
 }
 
 int
-dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame)
 {
+    int ret = -1;
 
-        int         ret     = -1;
-
-        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
-                            dht_inprogress_check_done,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, dht_rebalance_inprogress_task,
+                       dht_inprogress_check_done, frame, frame);
+    return ret;
 }
 
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t *layout_int)
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
-        if (!ret && ctx) {
-                ctx->layout = layout_int;
-        } else {
-                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
-                if (!ctx)
-                        return ret;
-                ctx->layout = layout_int;
-        }
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+    if (!ret && ctx) {
+        ctx->layout = layout_int;
+    } else {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return ret;
+        ctx->layout = layout_int;
+    }
 
-        ret = dht_inode_ctx_set (inode, this, ctx);
+    ret = dht_inode_ctx_set(inode, this, ctx);
 
-        return ret;
+    return ret;
 }
 
-
 void
-dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat)
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        dht_stat_time_t         *time           = 0;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        if (ret)
-		return;
+    if (ret)
+        return;
 
-        time = &ctx->time;
+    time = &ctx->time;
 
-	time->mtime      = stat->ia_mtime;
-	time->mtime_nsec = stat->ia_mtime_nsec;
+    time->mtime = stat->ia_mtime;
+    time->mtime_nsec = stat->ia_mtime_nsec;
 
-	time->ctime      = stat->ia_ctime;
-	time->ctime_nsec = stat->ia_ctime_nsec;
+    time->ctime = stat->ia_ctime;
+    time->ctime_nsec = stat->ia_ctime_nsec;
 
-	time->atime      = stat->ia_atime;
-	time->atime_nsec = stat->ia_atime_nsec;
+    time->atime = stat->ia_atime;
+    time->atime_nsec = stat->ia_atime_nsec;
 
-	return;
+    return;
 }
 
-
 int
-dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
-                           int32_t post)
-{
-        dht_inode_ctx_t         *ctx            = NULL;
-        dht_stat_time_t         *time           = 0;
-        int                      ret            = -1;
-
-        GF_VALIDATE_OR_GOTO (this->name, stat, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = dht_inode_ctx_get (inode, this, &ctx);
-
-        if (ret) {
-                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
-                if (!ctx)
-                        return -1;
-        }
-
-        time = &ctx->time;
-
-        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
-                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
-        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
-                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
-        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
-                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
-
-        ret = dht_inode_ctx_set (inode, this, ctx);
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t post)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, stat, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+
+    if (ret) {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return -1;
+    }
+
+    time = &ctx->time;
+
+    LOCK(&inode->lock);
+    {
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec, stat->ia_mtime,
+                        stat->ia_mtime_nsec, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec, stat->ia_ctime,
+                        stat->ia_ctime_nsec, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec, stat->ia_atime,
+                        stat->ia_atime_nsec, post);
+    }
+    UNLOCK(&inode->lock);
+
+    ret = dht_inode_ctx_set(inode, this, ctx);
 out:
-        return 0;
+    return 0;
 }
 
 int
-dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
 {
-        int             ret     = -1;
-        uint64_t        ctx_int = 0;
+    int ret = -1;
+    uint64_t ctx_int = 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = inode_ctx_get (inode, this, &ctx_int);
+    ret = inode_ctx_get(inode, this, &ctx_int);
 
-        if (ret)
-                return ret;
-
-        if (ctx)
-                *ctx = (dht_inode_ctx_t *) ctx_int;
-out:
+    if (ret)
         return ret;
-}
-
-int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
-{
-        int             ret = -1;
-        uint64_t        ctx_int = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
-
-        ctx_int = (long)ctx;
-        ret = inode_ctx_set (inode, this, &ctx_int);
-out:
-        return ret;
-}
-
-void
-dht_set_lkowner (dht_lock_t **lk_array, int count, gf_lkowner_t *lkowner)
-{
-        int i = 0;
-
-        if (!lk_array || !lkowner)
-                goto out;
-
-        for (i = 0; i < count; i++) {
-                lk_array[i]->lk_owner = *lkowner;
-        }
 
+    if (ctx)
+        *ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
 out:
-        return;
+    return ret;
 }
 
 int
-dht_subvol_status (dht_conf_t *conf, xlator_t *subvol)
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
 {
-        int i;
+    int ret = -1;
+    uint64_t ctx_int = 0;
 
-        for (i=0 ; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == subvol) {
-                        return conf->subvolume_status[i];
-                }
-        }
-        return 0;
-}
-
-void
-dht_inodelk_done (call_frame_t *lock_frame)
-{
-        fop_inodelk_cbk_t  inodelk_cbk = NULL;
-        call_frame_t      *main_frame  = NULL;
-        dht_local_t       *local       = NULL;
-
-        local = lock_frame->local;
-        main_frame = local->main_frame;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
-
-        inodelk_cbk = local->lock.inodelk_cbk;
-        local->lock.inodelk_cbk = NULL;
-
-        inodelk_cbk (main_frame, NULL, main_frame->this, local->lock.op_ret,
-                     local->lock.op_errno, NULL);
-
-        dht_lock_stack_destroy (lock_frame);
-        return;
+    ctx_int = (long)ctx;
+    ret = inode_ctx_set(inode, this, &ctx_int);
+out:
+    return ret;
 }
 
 int
-dht_inodelk_cleanup_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
-{
-        dht_inodelk_done (frame);
-        return 0;
-}
-
-int32_t
-dht_lock_count (dht_lock_t **lk_array, int lk_count)
-{
-        int i = 0, locked = 0;
-
-        if ((lk_array == NULL) || (lk_count == 0))
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol)
+{
+    int i;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            return conf->subvolume_status[i];
+        }
+    }
+    return 0;
+}
+
+inode_t *
+dht_heal_path(xlator_t *this, char *path, inode_table_t *itable)
+{
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    char *bname = NULL;
+    char *save_ptr = NULL;
+    static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    char *tmp_path = NULL;
+
+    tmp_path = gf_strdup(path);
+    if (!tmp_path) {
+        goto out;
+    }
+
+    gf_uuid_copy(loc.pargfid, gfid);
+    loc.parent = inode_ref(itable->root);
+
+    bname = strtok_r(tmp_path, "/", &save_ptr);
+
+    /* sending a lookup on parent directory,
+     * Eg:  if  path is like /a/b/c/d/e/f/g/
+     * then we will send a lookup on a first and then b,c,d,etc
+     */
+
+    while (bname) {
+        linked_inode = NULL;
+        loc.inode = inode_grep(itable, loc.parent, bname);
+        if (loc.inode == NULL) {
+            loc.inode = inode_new(itable);
+            if (loc.inode == NULL) {
+                ret = -ENOMEM;
                 goto out;
-
-        for (i = 0; i < lk_count; i++) {
-                if (lk_array[i]->locked)
-                        locked++;
-        }
-out:
-        return locked;
-}
-
-void
-dht_inodelk_cleanup (call_frame_t *lock_frame)
-{
-        dht_lock_t  **lk_array = NULL;
-        int           lk_count = 0, lk_acquired = 0;
-        dht_local_t  *local    = NULL;
-
-        local = lock_frame->local;
-
-        lk_array = local->lock.locks;
-        lk_count = local->lock.lk_count;
-
-        lk_acquired = dht_lock_count (lk_array, lk_count);
-        if (lk_acquired != 0) {
-                dht_unlock_inodelk (lock_frame, lk_array, lk_count,
-                                    dht_inodelk_cleanup_cbk);
-        } else {
-                dht_inodelk_done (lock_frame);
-        }
-
-        return;
-}
-
-int32_t
-dht_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        dht_local_t *local                  = NULL;
-        int          lk_index               = 0, call_cnt = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        lk_index = (long) cookie;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                uuid_utoa_r (local->lock.locks[lk_index]->loc.gfid,
-                             gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "unlocking failed on %s:%s",
-                        local->lock.locks[lk_index]->xl->name,
-                        gfid);
+            }
         } else {
-                local->lock.locks[lk_index]->locked = 0;
-        }
-
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                dht_inodelk_done (frame);
-        }
-
-        return 0;
-}
-
-call_frame_t *
-dht_lock_frame (call_frame_t *parent_frame)
-{
-        call_frame_t *lock_frame = NULL;
-
-        lock_frame = copy_frame (parent_frame);
-        if (lock_frame == NULL)
+            /*
+             * Inode is already populated in the inode table.
+             * Which means we already looked up the inode and
+             * linked with a dentry. So that we will skip
+             * lookup on this entry, and proceed to next.
+             */
+            linked_inode = loc.inode;
+            bname = strtok_r(NULL, "/", &save_ptr);
+            if (!bname) {
                 goto out;
-
-        set_lk_owner_from_ptr (&lock_frame->root->lk_owner, parent_frame->root);
-
-out:
-        return lock_frame;
-}
-
-int32_t
-dht_unlock_inodelk (call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
-                    fop_inodelk_cbk_t inodelk_cbk)
-{
-        dht_local_t     *local      = NULL;
-        struct gf_flock  flock      = {0,};
-        int              ret        = -1 , i = 0;
-        call_frame_t    *lock_frame = NULL;
-        int              call_cnt   = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, done);
-
-        call_cnt = dht_lock_count (lk_array, lk_count);
-        if (call_cnt == 0) {
-                ret = 0;
-                goto done;
+            }
+            inode_unref(loc.parent);
+            loc.parent = loc.inode;
+            gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+            loc.inode = NULL;
+            continue;
         }
 
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "cannot allocate a frame, not unlocking following "
-                        "locks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-                goto done;
-        }
+        loc.name = bname;
+        ret = loc_path(&loc, bname);
 
-        ret = dht_local_lock_init (lock_frame, lk_array, lk_count, inodelk_cbk);
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "storing locks in local failed, not unlocking "
-                        "following locks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-
-                goto done;
-        }
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-        local->call_cnt = call_cnt;
-
-        flock.l_type = F_UNLCK;
-
-        for (i = 0; i < local->lock.lk_count; i++) {
-                if (!local->lock.locks[i]->locked)
-                        continue;
-
-                lock_frame->root->lk_owner = local->lock.locks[i]->lk_owner;
-                STACK_WIND_COOKIE (lock_frame, dht_unlock_inodelk_cbk,
-                                   (void *)(long)i,
-                                   local->lock.locks[i]->xl,
-                                   local->lock.locks[i]->xl->fops->inodelk,
-                                   local->lock.locks[i]->domain,
-                                   &local->lock.locks[i]->loc, F_SETLK,
-                                   &flock, NULL);
-                if (!--call_cnt)
-                        break;
+        ret = syncop_lookup(this, &loc, &iatt, NULL, NULL, NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_SELFHEAL_FAILED,
+                    "path=%s", path, "subvolume=%s", this->name, "bname=%s",
+                    bname, NULL);
+            goto out;
         }
 
-        return 0;
-
-done:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame);
-
-        /* no locks acquired, invoke inodelk_cbk */
-        if (ret == 0)
-                inodelk_cbk (frame, NULL, frame->this, 0, 0, NULL);
+        linked_inode = inode_link(loc.inode, loc.parent, bname, &iatt);
+        if (!linked_inode)
+            goto out;
 
-        return ret;
-}
-
-int32_t
-dht_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        dht_local_t *local                   = NULL;
-        int          lk_index               = 0, call_cnt = 0;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        lk_index = (long) cookie;
-
-        if (op_ret == -1) {
-                local->lock.op_ret = -1;
-                local->lock.op_errno = op_errno;
-
-                if (local && local->lock.locks[lk_index]) {
-                        uuid_utoa_r (local->lock.locks[lk_index]->loc.inode->gfid,
-                                     gfid);
-
-                        gf_msg_debug (this->name, op_errno,
-                                      "inodelk failed on gfid: %s "
-                                      "subvolume: %s", gfid,
-                                      local->lock.locks[lk_index]->xl->name);
-                }
-
-                goto out;
-        }
-
-        local->lock.locks[lk_index]->locked = _gf_true;
+        loc_wipe(&loc);
+        gf_uuid_copy(loc.pargfid, linked_inode->gfid);
+        loc.inode = NULL;
 
+        bname = strtok_r(NULL, "/", &save_ptr);
+        if (bname)
+            loc.parent = linked_inode;
+    }
 out:
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                if (local->lock.op_ret < 0) {
-                        dht_inodelk_cleanup (frame);
-                        return 0;
-                }
+    inode_ref(linked_inode);
+    loc_wipe(&loc);
+    GF_FREE(tmp_path);
 
-                dht_inodelk_done (frame);
-        }
-
-        return 0;
+    return linked_inode;
 }
 
 int
-dht_nonblocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                         int lk_count, fop_inodelk_cbk_t inodelk_cbk)
-{
-        struct gf_flock  flock      = {0,};
-        int              i          = 0, ret = 0;
-        dht_local_t     *local      = NULL;
-        call_frame_t    *lock_frame = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, out);
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL)
-                goto out;
-
-        ret = dht_local_lock_init (lock_frame, lk_array, lk_count, inodelk_cbk);
-        if (ret < 0) {
-                goto out;
-        }
-
-        dht_set_lkowner (lk_array, lk_count, &lock_frame->root->lk_owner);
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-
-        local->call_cnt = lk_count;
-
-        for (i = 0; i < lk_count; i++) {
-                flock.l_type = local->lock.locks[i]->type;
-
-                STACK_WIND_COOKIE (lock_frame, dht_nonblocking_inodelk_cbk,
-                                   (void *) (long) i,
-                                   local->lock.locks[i]->xl,
-                                   local->lock.locks[i]->xl->fops->inodelk,
-                                   local->lock.locks[i]->domain,
-                                   &local->lock.locks[i]->loc, F_SETLK,
-                                   &flock, NULL);
-        }
-
-        return 0;
-
-out:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame);
-
-        return -1;
-}
-
-int32_t
-dht_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int          lk_index = 0;
-        int          i        = 0;
-        dht_local_t *local    = NULL;
-
-        lk_index = (long) cookie;
-
-        local = frame->local;
-        if (op_ret == 0) {
-                local->lock.locks[lk_index]->locked = _gf_true;
+dht_heal_full_path(void *data)
+{
+    call_frame_t *heal_frame = data;
+    dht_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *path = NULL;
+    int ret = -1;
+    xlator_t *source = NULL;
+    xlator_t *this = NULL;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    inode_t *tmp_inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("DHT", heal_frame, out);
+
+    local = heal_frame->local;
+    this = heal_frame->this;
+    source = heal_frame->cookie;
+    heal_frame->cookie = NULL;
+    gf_uuid_copy(loc.gfid, local->gfid);
+
+    if (local->loc.inode)
+        loc.inode = inode_ref(local->loc.inode);
+    else
+        goto out;
+
+    itable = loc.inode->table;
+    ret = syncop_getxattr(source, &loc, &dict, GET_ANCESTRY_PATH_KEY, NULL,
+                          NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_HEAL_ABORT,
+                "subvol=%s", source->name, NULL);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (path) {
+        inode = dht_heal_path(this, path, itable);
+        if (inode && inode != local->inode) {
+            /*
+             * if inode returned by heal function is different
+             * from what we passed, which means a racing thread
+             * already linked a different inode for dentry.
+             * So we will update our local->inode, so that we can
+             * retrurn proper inode.
+             */
+            tmp_inode = local->inode;
+            local->inode = inode;
+            inode_unref(tmp_inode);
+            tmp_inode = NULL;
         } else {
-                switch (op_errno) {
-                case ESTALE:
-                case ENOENT:
-                        if (local->lock.reaction != IGNORE_ENOENT_ESTALE) {
-                                local->lock.op_ret = -1;
-                                local->lock.op_errno = op_errno;
-                                goto cleanup;
-                        }
-                        break;
-                default:
-                        local->lock.op_ret = -1;
-                        local->lock.op_errno = op_errno;
-                        goto cleanup;
-                }
-        }
-
-        if (lk_index == (local->lock.lk_count - 1)) {
-                for (i = 0; (i < local->lock.lk_count) &&
-                     (!local->lock.locks[i]->locked); i++)
-                        ;
-
-                if (i == local->lock.lk_count) {
-                        local->lock.op_ret = -1;
-                        local->lock.op_errno = op_errno;
-                }
-
-                dht_inodelk_done (frame);
-        } else {
-                dht_blocking_inodelk_rec (frame, ++lk_index);
-        }
-
-        return 0;
-
-cleanup:
-        dht_inodelk_cleanup (frame);
-
-        return 0;
-}
-
-void
-dht_blocking_inodelk_rec (call_frame_t *frame, int i)
-{
-        dht_local_t     *local = NULL;
-        struct gf_flock  flock = {0,};
-
-        local = frame->local;
-
-        flock.l_type = local->lock.locks[i]->type;
-
-        STACK_WIND_COOKIE (frame, dht_blocking_inodelk_cbk,
-                           (void *) (long) i,
-                           local->lock.locks[i]->xl,
-                           local->lock.locks[i]->xl->fops->inodelk,
-                           local->lock.locks[i]->domain,
-                           &local->lock.locks[i]->loc, F_SETLKW, &flock, NULL);
-
-        return;
-}
-
-int
-dht_lock_request_cmp (const void *val1, const void *val2)
-{
-        dht_lock_t *lock1 = NULL;
-        dht_lock_t *lock2 = NULL;
-        int         ret   = 0;
-
-        lock1 = *(dht_lock_t **)val1;
-        lock2 = *(dht_lock_t **)val2;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", lock1, out);
-        GF_VALIDATE_OR_GOTO ("dht-locks", lock2, out);
-
-        ret = strcmp (lock1->xl->name, lock2->xl->name);
-
-        if (ret == 0) {
-                ret = gf_uuid_compare (lock1->loc.gfid, lock2->loc.gfid);
+            inode_unref(inode);
         }
+    }
 
 out:
-        return ret;
-}
-
-int
-dht_lock_order_requests (dht_lock_t **locks, int count)
-{
-        int        ret     = -1;
-
-        if (!locks || !count)
-                goto out;
-
-        qsort (locks, count, sizeof (*locks), dht_lock_request_cmp);
-        ret = 0;
-
-out:
-        return ret;
+    loc_wipe(&loc);
+    if (dict)
+        dict_unref(dict);
+    return 0;
 }
 
 int
-dht_blocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, dht_reaction_type_t reaction,
-                      fop_inodelk_cbk_t inodelk_cbk)
-{
-        int           ret        = -1;
-        call_frame_t *lock_frame = NULL;
-        dht_local_t  *local      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, out);
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL)
-                goto out;
-
-        ret = dht_local_lock_init (lock_frame, lk_array, lk_count, inodelk_cbk);
-        if (ret < 0) {
-                goto out;
-        }
-
-        dht_set_lkowner (lk_array, lk_count, &lock_frame->root->lk_owner);
-
-        local = lock_frame->local;
-        local->lock.reaction = reaction;
-        local->main_frame = frame;
-
-        dht_blocking_inodelk_rec (lock_frame, 0);
-
-        return 0;
-out:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame);
-
-        return -1;
-}
-inode_t*
-dht_heal_path (xlator_t *this, char *path, inode_table_t *itable)
-{
-        int             ret             = -1;
-        struct iatt     iatt            = {0, };
-        inode_t        *linked_inode    = NULL;
-        loc_t           loc             = {0, };
-        char           *bname           = NULL;
-        char           *save_ptr        = NULL;
-        uuid_t          gfid            = {0, };
-        char           *tmp_path        = NULL;
-
-
-        tmp_path = gf_strdup (path);
-        if (!tmp_path) {
-                goto out;
-        }
-
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
-
-        gf_uuid_copy (loc.pargfid, gfid);
-        loc.parent = inode_ref (itable->root);
-
-        bname = strtok_r (tmp_path, "/",  &save_ptr);
-
-        /* sending a lookup on parent directory,
-         * Eg:  if  path is like /a/b/c/d/e/f/g/
-         * then we will send a lookup on a first and then b,c,d,etc
-         */
-
-        while (bname) {
-                linked_inode = NULL;
-                loc.inode = inode_grep (itable, loc.parent, bname);
-                if (loc.inode == NULL) {
-                        loc.inode = inode_new (itable);
-                        if (loc.inode == NULL) {
-                                ret = -ENOMEM;
-                                goto out;
-                        }
-                } else {
-                        /*
-                         * Inode is already populated in the inode table.
-                         * Which means we already looked up the inde and
-                         * linked with a dentry. So that we will skip
-                         * lookup on this entry, and proceed to next.
-                         */
-                        bname = strtok_r (NULL, "/",  &save_ptr);
-                        inode_unref (loc.parent);
-                        loc.parent = loc.inode;
-                        gf_uuid_copy (loc.pargfid, loc.inode->gfid);
-                        loc.inode = NULL;
-                        continue;
-                }
-
-                loc.name = bname;
-                ret = loc_path (&loc, bname);
-
-                ret = syncop_lookup (this, &loc, &iatt, NULL, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, -ret,
-                                DHT_MSG_DIR_SELFHEAL_FAILED,
-                                "Healing of path %s failed on subvolume %s for "
-                                "directory %s", path, this->name, bname);
-                        goto out;
-                }
-
-                linked_inode = inode_link (loc.inode, loc.parent, bname, &iatt);
-                if (!linked_inode)
-                        goto out;
-
-                loc_wipe (&loc);
-                gf_uuid_copy (loc.pargfid, linked_inode->gfid);
-                loc.inode = NULL;
-                loc.parent = linked_inode;
-
-                bname = strtok_r (NULL, "/",  &save_ptr);
-        }
-out:
-        inode_ref (linked_inode);
-        loc_wipe (&loc);
-        GF_FREE (tmp_path);
-
-        return linked_inode;
-}
-
-
-int
-dht_heal_full_path (void *data)
-{
-        call_frame_t            *heal_frame     = data;
-        dht_local_t             *local          = NULL;
-        loc_t                    loc            = {0, };
-        dict_t                  *dict           = NULL;
-        char                    *path           = NULL;
-        int                      ret            = -1;
-        xlator_t                *source         = NULL;
-        xlator_t                *this           = NULL;
-        inode_table_t           *itable         = NULL;
-        inode_t                 *inode          = NULL;
-        inode_t                 *tmp_inode      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("DHT", heal_frame, out);
-
-        local = heal_frame->local;
-        this = heal_frame->this;
-        source = heal_frame->cookie;
-        heal_frame->cookie = NULL;
-        gf_uuid_copy (loc.gfid, local->gfid);
-
-        if (local->loc.inode)
-                loc.inode = inode_ref (local->loc.inode);
-        else
-                goto out;
-
-        itable = loc.inode->table;
-        ret = syncop_getxattr (source, &loc, &dict,
-                       GET_ANCESTRY_PATH_KEY, NULL, NULL);
+dht_heal_full_path_done(int op_ret, call_frame_t *heal_frame, void *data)
+{
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int op_errno = 0;
+
+    local = heal_frame->local;
+    main_frame = local->main_frame;
+    local->main_frame = NULL;
+    this = heal_frame->this;
+
+    dht_set_fixed_dir_stat(&local->postparent);
+    if (local->need_xattr_heal) {
+        local->need_xattr_heal = 0;
+        ret = dht_dir_xattr_heal(this, local, &op_errno);
         if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, -ret,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Failed to get path from subvol %s. Aborting "
-                        "directory healing.", source->name);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, GET_ANCESTRY_PATH_KEY, &path);
-        if (path) {
-                inode = dht_heal_path (this, path, itable);
-                if (inode && inode != local->inode) {
-                        /*
-                         * if inode returned by heal function is different
-                         * from what we passed, which means a racing thread
-                         * already linked a different inode for dentry.
-                         * So we will update our local->inode, so that we can
-                         * retrurn proper inode.
-                         */
-                        tmp_inode = local->inode;
-                        local->inode = inode;
-                        inode_unref (tmp_inode);
-                        tmp_inode = NULL;
-                } else {
-                        inode_unref (inode);
-                }
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    NULL);
         }
+    }
 
-out:
-        loc_wipe (&loc);
-        if (dict)
-                dict_unref (dict);
-        return 0;
-}
-
-int
-dht_heal_full_path_done (int op_ret, call_frame_t *heal_frame, void *data)
-{
-
-        call_frame_t            *main_frame       = NULL;
-        dht_local_t             *local            = NULL;
-
-        local = heal_frame->local;
-        main_frame = local->main_frame;
-        local->main_frame = NULL;
-
-        dht_set_fixed_dir_stat (&local->postparent);
-
-        DHT_STACK_UNWIND (lookup, main_frame, 0, 0,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
+    DHT_STACK_UNWIND(lookup, main_frame, 0, 0, local->inode, &local->stbuf,
+                     local->xattr, &local->postparent);
 
-        DHT_STACK_DESTROY (heal_frame);
-        return 0;
+    DHT_STACK_DESTROY(heal_frame);
+    return 0;
 }
 
 /* This function must be called inside an inode lock */
 int
-__dht_lock_subvol_set (inode_t *inode, xlator_t *this,
-                       xlator_t *lock_subvol)
+__dht_lock_subvol_set(inode_t *inode, xlator_t *this, xlator_t *lock_subvol)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
-        uint64_t                 value          = 0;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t value = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = __inode_ctx_get0 (inode, this, &value);
-        if (ret || !value) {
-                return -1;
-        }
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (ret || !value) {
+        return -1;
+    }
 
-        ctx = (dht_inode_ctx_t *) value;
-        ctx->lock_subvol = lock_subvol;
+    ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+    ctx->lock_subvol = lock_subvol;
 out:
-        return ret;
+    return ret;
 }
 
-xlator_t*
-dht_get_lock_subvolume (xlator_t *this, struct gf_flock *lock,
-                        dht_local_t *local)
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local)
 {
-        xlator_t                *subvol                  = NULL;
-        inode_t                 *inode                   = NULL;
-        int32_t                  ret                     = -1;
-        uint64_t                 value                   = 0;
-        xlator_t                *cached_subvol           = NULL;
-        dht_inode_ctx_t         *ctx                     = NULL;
-        char                     gfid[GF_UUID_BUF_SIZE]  = {0};
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    int32_t ret = -1;
+    uint64_t value = 0;
+    xlator_t *cached_subvol = NULL;
+    dht_inode_ctx_t *ctx = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        GF_VALIDATE_OR_GOTO (this->name, lock, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+    GF_VALIDATE_OR_GOTO(this->name, lock, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
 
-        cached_subvol = local->cached_subvol;
+    cached_subvol = local->cached_subvol;
 
-        if (local->loc.inode || local->fd) {
-                inode = local->loc.inode ? local->loc.inode : local->fd->inode;
-        }
-
-        if (!inode)
-                goto out;
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
 
-        if (!(IA_ISDIR (inode->ia_type) || IA_ISINVAL (inode->ia_type))) {
-                /*
-                 * We may get non-linked inode for directories as part
-                 * of the selfheal code path. So checking  for IA_INVAL
-                 * type also. This will only happen for directory.
-                 */
-                subvol = local->cached_subvol;
-                goto out;
-        }
+    if (!inode)
+        goto out;
 
-        if (lock->l_type != F_UNLCK) {
-                /*
-                 * inode purging might happen on NFS between a lk
-                 * and unlk. Due to this lk and unlk might be sent
-                 * to different subvols.
-                 * So during a lock request, taking a ref on inode
-                 * to prevent inode purging. inode unref will happen
-                 * in unlock cbk code path.
-                 */
-                inode_ref (inode);
-        }
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        /*
+         * We may get non-linked inode for directories as part
+         * of the selfheal code path. So checking  for IA_INVAL
+         * type also. This will only happen for directory.
+         */
+        subvol = local->cached_subvol;
+        goto out;
+    }
 
-        LOCK (&inode->lock);
-                ret = __inode_ctx_get0 (inode, this, &value);
-                if (!ret && value) {
-                        ctx = (dht_inode_ctx_t *) value;
-                        subvol = ctx->lock_subvol;
-                }
-                if (!subvol && lock->l_type != F_UNLCK && cached_subvol) {
-                        ret = __dht_lock_subvol_set (inode, this,
-                                                     cached_subvol);
-                        if (ret) {
-                                gf_uuid_unparse(inode->gfid, gfid);
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_SET_INODE_CTX_FAILED,
-                                        "Failed to set lock_subvol in "
-                                        "inode ctx for gfid %s",
-                                        gfid);
-                                goto unlock;
-                        }
-                        subvol = cached_subvol;
-                }
-unlock:
-        UNLOCK (&inode->lock);
-        if (!subvol && inode && lock->l_type != F_UNLCK) {
-                inode_unref (inode);
-        }
+    if (lock->l_type != F_UNLCK) {
+        /*
+         * inode purging might happen on NFS between a lk
+         * and unlk. Due to this lk and unlk might be sent
+         * to different subvols.
+         * So during a lock request, taking a ref on inode
+         * to prevent inode purging. inode unref will happen
+         * in unlock cbk code path.
+         */
+        inode_ref(inode);
+    }
+
+    LOCK(&inode->lock);
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (!ret && value) {
+        ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+        subvol = ctx->lock_subvol;
+    }
+    if (!subvol && lock->l_type != F_UNLCK && cached_subvol) {
+        ret = __dht_lock_subvol_set(inode, this, cached_subvol);
+        if (ret) {
+            gf_uuid_unparse(inode->gfid, gfid);
+            UNLOCK(&inode->lock);
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                    "lock_subvol gfid=%s", gfid, NULL);
+            goto post_unlock;
+        }
+        subvol = cached_subvol;
+    }
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!subvol && inode && lock->l_type != F_UNLCK) {
+        inode_unref(inode);
+    }
 out:
-        return subvol;
+    return subvol;
 }
 
 int
-dht_lk_inode_unref (call_frame_t *frame, int32_t op_ret)
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret)
 {
-        int                     ret                     = -1;
-        dht_local_t            *local                   = NULL;
-        inode_t                *inode                   = NULL;
-        xlator_t               *this                    = NULL;
-        char                    gfid[GF_UUID_BUF_SIZE]  = {0};
+    int ret = -1;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        this = frame->this;
+    local = frame->local;
+    this = frame->this;
 
-        if (local->loc.inode || local->fd) {
-                inode = local->loc.inode ? local->loc.inode : local->fd->inode;
-        }
-        if (!inode) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LOCK_INODE_UNREF_FAILED,
-                        "Found a NULL inode. Failed to unref the inode");
-                goto out;
-        }
-
-        if (!(IA_ISDIR (inode->ia_type) || IA_ISINVAL (inode->ia_type))) {
-                ret = 0;
-                goto out;
-        }
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
+    if (!inode) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+                NULL);
+        goto out;
+    }
+
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        ret = 0;
+        goto out;
+    }
 
-        switch (local->lock_type) {
+    switch (local->lock_type) {
         case F_RDLCK:
         case F_WRLCK:
-                if (op_ret) {
-                        gf_uuid_unparse(inode->gfid, gfid);
-                        gf_msg_debug (this->name, 0,
-                                "lock request failed for gfid %s", gfid);
-                        inode_unref (inode);
-                        goto out;
-                }
-                break;
+            if (op_ret) {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_msg_debug(this->name, 0, "lock request failed for gfid %s",
+                             gfid);
+                inode_unref(inode);
+                goto out;
+            }
+            break;
 
         case F_UNLCK:
-                if (!op_ret) {
-                        inode_unref (inode);
-                } else {
-                        gf_uuid_unparse(inode->gfid, gfid);
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LOCK_INODE_UNREF_FAILED,
-                                "Unlock request failed for gfid %s."
-                                "Failed to unref the inode", gfid);
-                        goto out;
-                }
+            if (!op_ret) {
+                inode_unref(inode);
+            } else {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_smsg(this->name, GF_LOG_WARNING, 0,
+                        DHT_MSG_LOCK_INODE_UNREF_FAILED, "gfid=%s", gfid, NULL);
+                goto out;
+            }
         default:
-                break;
-        }
-        ret = 0;
+            break;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
+}
+
+/* Code to update custom extended attributes from src dict to dst dict
+ */
+void
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag)
+{
+    int ret = -1;
+    data_t *keyval = NULL;
+    int luret = -1;
+    int luflag = -1;
+    int i = 0;
+    char **xattrs_to_heal;
+
+    if (!src || !dst) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DST_NULL_SET_FAILED,
+                "path=%s", local->loc.path, NULL);
+        return;
+    }
+    /* Check if any user xattr present in src dict and set
+       it to dst dict
+    */
+    luret = dict_foreach_fnmatch(src, "user.*", dht_set_user_xattr, dst);
+    /* Check if any other custom xattr present in src dict
+       and set it to dst dict, here index start from 1 because
+       user xattr already checked in previous statement
+    */
+
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    for (i = 1; xattrs_to_heal[i]; i++) {
+        keyval = dict_get(src, xattrs_to_heal[i]);
+        if (keyval) {
+            luflag = 1;
+            ret = dict_set(dst, xattrs_to_heal[i], keyval);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                        DHT_MSG_DICT_SET_FAILED, "key=%s", xattrs_to_heal[i],
+                        "path=%s", local->loc.path, NULL);
+            keyval = NULL;
+        }
+    }
+    if (uret)
+        (*uret) = luret;
+    if (uflag)
+        (*uflag) = luflag;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 549f1b9ea7e..dbb8070b0da 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -10,1388 +10,1649 @@
 
 #include "dht-common.h"
 
-int dht_access2 (xlator_t *this, xlator_t *dst_node,
-                 call_frame_t *frame, int ret);
-int dht_readv2 (xlator_t *this, xlator_t *dst_node,
-                call_frame_t *frame, int ret);
-int dht_attr2 (xlator_t *this, xlator_t *dst_node,
-               call_frame_t *frame, int ret);
-int dht_open2 (xlator_t *this, xlator_t *dst_node,
-               call_frame_t *frame, int ret);
-int dht_flush2 (xlator_t *this, xlator_t *dst_node,
-                call_frame_t *frame, int ret);
-int dht_lk2 (xlator_t *this, xlator_t *dst_node,
-             call_frame_t *frame, int ret);
-int dht_fsync2 (xlator_t *this, xlator_t *dst_node,
-                call_frame_t *frame, int ret);
-
-
-
-int
-dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+static int
+dht_access2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_readv2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_attr2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_open2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_flush2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_lk2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_fsync2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret);
+
+static int
+dht_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        local->op_errno = op_errno;
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
-        }
-
-        if (!op_ret || (local->call_cnt != 1))
-                goto out;
-
-        /* rebalance would have happened */
-        local->rebalance.target_op_fn = dht_open2;
-        ret = dht_rebalance_complete_check (this, frame);
-        if (!ret)
-                return 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    local->op_errno = op_errno;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    /* Update ctx if the fd has been opened on the target*/
+    if (!op_ret && (local->call_cnt == 1)) {
+        dht_fd_ctx_set(this, fd, prev);
+        goto out;
+    }
+
+    if (!op_ret || (local->call_cnt != 1))
+        goto out;
+
+    /* rebalance would have happened */
+    local->rebalance.target_op_fn = dht_open2;
+    ret = dht_rebalance_complete_check(this, frame);
+    if (!ret)
+        return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
+    DHT_STACK_UNWIND(open, frame, op_ret, op_errno, local->fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_open2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
-
-        if (!frame || !frame->local)
-                goto out;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        op_errno = ENOENT;
+    if (!frame || !frame->local)
+        goto out;
 
-        if (we_are_not_migrating (ret)) {
-                /* This DHT layer is not migrating the file */
-                DHT_STACK_UNWIND (open, frame, -1, local->op_errno,
-                                  NULL, NULL);
-                return 0;
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        }
+    if (we_are_not_migrating(ret)) {
+        /* This DHT layer is not migrating the file */
+        DHT_STACK_UNWIND(open, frame, -1, local->op_errno, NULL,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    &local->loc, local->rebalance.flags, local->fd,
-                    NULL);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      &local->loc, local->rebalance.flags, local->fd,
+                      local->xattr_req);
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_open (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+         dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPEN);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->rebalance.flags = flags;
-        local->call_cnt = 1;
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPEN);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    loc, flags, fd, xdata);
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      loc, flags, fd, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        xlator_t     *subvol1 = 0;
-        xlator_t     *subvol2 = 0;
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        inode_t      *inode = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
-        }
-
-        if (local->call_cnt != 1)
-                goto out;
+    xlator_t *subvol1 = 0;
+    xlator_t *subvol2 = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    if ((local->fop == GF_FOP_FSTAT) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-        local->op_ret = op_ret;
-
-        /* Check if the rebalance phase2 is true */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-
-                local->rebalance.target_op_fn = dht_attr2;
-                dht_set_local_rebalance (this, local, NULL, NULL,
-                                         stbuf, xdata);
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
-
-                dht_inode_ctx_get_mig_info (this, inode, &subvol1, &subvol2);
-                if (dht_mig_info_is_invalid (local->cached_subvol,
-                                             subvol1, subvol2)){
-                        /* Phase 2 of migration */
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        /* it is a non-fd op or it is an fd based Fop and
-                           opened on the dst.*/
-                       if (local->fd &&
-                           !dht_fd_open_on_dst (this, local->fd, subvol2)) {
-                                ret = dht_rebalance_complete_check (this, frame);
-                                if (!ret)
-                                        return 0;
-                        } else {
-                                dht_attr2 (this, subvol2, frame, 0);
-                                return 0;
-                        }
-                }
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    local->op_errno = op_errno;
+    local->op_ret = op_ret;
+
+    /* Check if the rebalance phase2 is true */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->rebalance.target_op_fn = dht_attr2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            /* it is a non-fd op or it is an fd based Fop and
+               opened on the dst.*/
+            if (local->fd && !dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                ret = dht_rebalance_complete_check(this, frame);
+                if (!ret)
+                    return 0;
+            } else {
+                dht_attr2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(stat, frame, op_ret, op_errno, stbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_attr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (stat, frame, local->op_ret, op_errno,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
-
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, op_errno,
+                         &local->rebalance.postbuf, local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        if (local->fop == GF_FOP_FSTAT) {
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, local->fd, NULL);
-        } else {
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, &local->loc, NULL);
-        }
+    if (local->fop == GF_FOP_FSTAT) {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, local->fd, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, &local->loc, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-int
-dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+static int
+dht_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->this->name);
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+
+            goto post_unlock;
+        }
 
-                        goto unlock;
-                }
+        dht_iatt_merge(this, &local->stbuf, stbuf);
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno,
+                         &local->stbuf, xdata);
+    }
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, xdata);
-        }
-err:
-        return 0;
+    return 0;
 }
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_STAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (IA_ISREG (loc->inode->ia_type)) {
-                local->call_cnt = 1;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_STAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        local->call_cnt = 1;
 
-                subvol = local->cached_subvol;
+        subvol = local->cached_subvol;
 
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, loc, xdata);
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
 
-                return 0;
-        }
+        return 0;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->call_cnt = call_cnt = layout->cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
 
-                STACK_WIND (frame, dht_attr_cbk,
-                            subvol, subvol->fops->stat,
-                            loc, xdata);
-        }
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSTAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (IA_ISREG (fd->inode->ia_type)) {
-                local->call_cnt = 1;
-
-                subvol = local->cached_subvol;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSTAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        local->call_cnt = 1;
 
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, fd, xdata);
+        subvol = local->cached_subvol;
 
-                return 0;
-        }
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+        return 0;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->call_cnt = call_cnt = layout->cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_attr_cbk,
-                            subvol, subvol->fops->fstat,
-                            fd, xdata);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata)
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata)
 {
-        dht_local_t *local      = NULL;
-        int          ret        = 0;
-        xlator_t    *src_subvol = 0;
-        xlator_t    *dst_subvol = 0;
-
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        /* This is already second try, no need for re-check */
-        if (local->call_cnt != 1)
-                goto out;
+    dht_local_t *local = NULL;
+    int ret = 0;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* This is already second try, no need for re-check */
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if ((op_ret == -1) && !dht_inode_missing(op_errno))
-                goto out;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno))
+        goto out;
 
-        local->op_errno = op_errno;
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-
-                local->op_ret = op_ret;
-                local->rebalance.target_op_fn = dht_readv2;
-                dht_set_local_rebalance (this, local, NULL, NULL,
-                                         stbuf, xdata);
-                /* File would be migrated to other node */
-                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                                  &src_subvol,
-                                                  &dst_subvol);
-
-                if (dht_mig_info_is_invalid (local->cached_subvol,
-                                             src_subvol, dst_subvol)
-                        || !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
-
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        /* value is already set in fd_ctx, that means no need
-                           to check for whether its complete or not. */
-                        dht_readv2 (this, dst_subvol, frame, 0);
-                        return 0;
-                }
+    local->op_errno = op_errno;
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->op_ret = op_ret;
+        local->rebalance.target_op_fn = dht_readv2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        /* File would be migrated to other node */
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                         &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            /* value is already set in fd_ctx, that means no need
+               to check for whether its complete or not. */
+            dht_readv2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
-        DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref, xdata);
+    DHT_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                     iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_readv2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        int          op_errno = EINVAL;
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (readv, frame, local->op_ret, op_errno,
-                                  NULL, 0, &local->rebalance.postbuf,
-                                  NULL, local->rebalance.xdata);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(readv, frame, local->op_ret, op_errno, NULL, 0,
+                         &local->rebalance.postbuf, NULL,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
-                    local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags, NULL);
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t off,
+          uint32_t flags, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_READ);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->rebalance.offset = off;
-        local->rebalance.size   = size;
-        local->rebalance.flags  = flags;
-        local->call_cnt = 1;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_readv_cbk,
-                    subvol, subvol->fops->readv,
-                    fd, size, off, flags, xdata);
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        return 0;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.offset = off;
+    local->rebalance.size = size;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        int          ret = -1;
-        dht_local_t *local = NULL;
-        xlator_t    *subvol = NULL;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        if (!prev || !prev->this)
-                goto out;
-        if (local->call_cnt != 1)
-                goto out;
-        if ((op_ret == -1) && ((op_errno == ENOTCONN) ||
-                dht_inode_missing(op_errno)) &&
-                IA_ISDIR(local->loc.inode->ia_type)) {
-                subvol = dht_subvol_next_available (this, prev->this);
-                if (!subvol)
-                        goto out;
-
-                /* check if we are done with visiting every node */
-                if (subvol == local->cached_subvol) {
-                        goto out;
-                }
-
-                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                            &local->loc, local->rebalance.flags, NULL);
-                return 0;
-        }
-        if ((op_ret == -1) && dht_inode_missing(op_errno) &&
-                !(IA_ISDIR(local->loc.inode->ia_type))) {
-                /* File would be migrated to other node */
-                local->op_errno = op_errno;
-                local->rebalance.target_op_fn = dht_access2;
-                ret = dht_rebalance_complete_check (frame->this, frame);
-                if (!ret)
-                        return 0;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!prev)
+        goto out;
+    if (local->call_cnt != 1)
+        goto out;
+    if ((op_ret == -1) &&
+        ((op_errno == ENOTCONN) || dht_inode_missing(op_errno)) &&
+        IA_ISDIR(local->loc.inode->ia_type)) {
+        subvol = dht_subvol_next_available(this, prev);
+        if (!subvol)
+            goto out;
+
+        /* check if we are done with visiting every node */
+        if (subvol == local->cached_subvol) {
+            goto out;
         }
 
-out:
-        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+        STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                          subvol->fops->access, &local->loc,
+                          local->rebalance.flags, NULL);
         return 0;
+    }
+    if ((op_ret == -1) && dht_inode_missing(op_errno) &&
+        !(IA_ISDIR(local->loc.inode->ia_type))) {
+        /* File would be migrated to other node */
+        local->op_errno = op_errno;
+        local->rebalance.target_op_fn = dht_access2;
+        ret = dht_rebalance_complete_check(frame->this, frame);
+        if (!ret)
+            return 0;
+    }
+
+out:
+    DHT_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-int
-dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_access2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
 
-                DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-                return 0;
-        }
+        DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    &local->loc, local->rebalance.flags, NULL);
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, &local->loc, local->rebalance.flags,
+                      local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ACCESS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.flags = mask;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    loc, mask, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ACCESS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mask;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, loc, mask, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, dict_t *xdata)
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata)
 {
-        dht_local_t     *local  = NULL;
-        xlator_t        *subvol = 0;
-        int             ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = 0;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->op_errno = op_errno;
+    local->op_errno = op_errno;
 
-        if (local->call_cnt != 1)
-                goto out;
+    if (local->call_cnt != 1)
+        goto out;
 
-        local->rebalance.target_op_fn = dht_flush2;
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_flush2;
 
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        /* If context is set, then send flush() it to the destination */
-        dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol);
-        if (subvol && dht_fd_open_on_dst (this, local->fd, subvol)) {
-                dht_flush2 (this, subvol, frame, 0);
-                return 0;
-        }
+    /* If context is set, then send flush() it to the destination */
+    dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+    if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+        dht_flush2(this, subvol, frame, 0);
+        return 0;
+    }
 
-        if (op_errno == EREMOTE) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret) {
-                        return 0;
-                }
+    if (op_errno == EREMOTE) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret) {
+            return 0;
         }
+    }
 
 out:
-        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+    DHT_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_flush2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        local = frame->local;
+    local = frame->local;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, local->fd,
-                    local->rebalance.xdata);
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, local->fd,
+               local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FLUSH);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FLUSH);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, fd, xdata);
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        return 0;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+               local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
-               dict_t *xdata)
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        inode_t      *inode = NULL;
-        xlator_t     *src_subvol = 0;
-        xlator_t     *dst_subvol = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        local->op_errno = op_errno;
-        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    local->op_errno = op_errno;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        local->op_ret = op_ret;
-        inode = local->fd->inode;
-
-        local->rebalance.target_op_fn = dht_fsync2;
-        dht_set_local_rebalance (this, local, NULL, prebuf,
-                                 postbuf, xdata);
+    local->op_ret = op_ret;
+    inode = local->fd->inode;
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+    local->rebalance.target_op_fn = dht_fsync2;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                dht_inode_ctx_get_mig_info (this, inode, &src_subvol, &dst_subvol);
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-                if (dht_mig_info_is_invalid (local->cached_subvol, src_subvol,
-                                     dst_subvol) ||
-                      !dht_fd_open_on_dst (this, local->fd, dst_subvol)) {
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-                        ret = dht_rebalance_in_progress_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        dht_fsync2 (this, dst_subvol, frame, 0);
-                        return 0;
-                }
-        }
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
 
-        if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_fsync2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_fsync2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
-
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
-
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (fsync, frame, local->op_ret,
-                                  op_errno, &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fsync, frame, local->op_ret, op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    local->fd, local->rebalance.flags, NULL);
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-           dict_t *xdata)
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+          dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSYNC);
-        if (!local) {
-                op_errno = ENOMEM;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSYNC);
+    if (!local) {
+        op_errno = ENOMEM;
 
-                goto err;
-        }
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        local->call_cnt = 1;
-        local->rebalance.flags = datasync;
+    local->call_cnt = 1;
+    local->rebalance.flags = datasync;
 
-        subvol = local->cached_subvol;
+    subvol = local->cached_subvol;
 
-        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    fd, datasync, xdata);
-
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
    indicate that lock migration happened on the fd, so we can consider it as
    phase 2 of migration */
-int
-dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
+static int
+dht_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-        dht_local_t     *local  = NULL;
-        int             ret     = -1;
-        xlator_t        *subvol = NULL;
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        if (local->call_cnt != 1)
-                goto out;
+    if (local->call_cnt != 1)
+        goto out;
 
-        local->rebalance.target_op_fn = dht_lk2;
+    local->rebalance.target_op_fn = dht_lk2;
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
-
-        if (op_errno == EREMOTE) {
-                dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                            NULL, &subvol);
-                if (subvol && dht_fd_open_on_dst (this, local->fd, subvol)) {
-                        dht_lk2 (this, subvol, frame, 0);
-                        return 0;
-                } else {
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret) {
-                                return 0;
-                        }
-                }
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    if (op_errno == EREMOTE) {
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+        if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+            dht_lk2(this, subvol, frame, 0);
+            return 0;
+        } else {
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret) {
+                return 0;
+            }
         }
+    }
 
 out:
-        dht_lk_inode_unref (frame, op_ret);
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(lk, frame, op_ret, op_errno, flock, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lk2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_lk2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        local = frame->local;
+    local = frame->local;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd,
-                    local->rebalance.lock_cmd, &local->rebalance.flock,
-                    local->rebalance.xdata);
+    STACK_WIND(frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd,
+               local->rebalance.lock_cmd, &local->rebalance.flock,
+               local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL,  NULL);
-        return 0;
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-dht_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata)
 {
-        xlator_t        *lock_subvol    = NULL;
-        int             op_errno        = -1;
-        dht_local_t     *local          = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_LK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_LK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->lock_type = flock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, flock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.flock = *flock;
+    local->rebalance.lock_cmd = cmd;
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_lk_cbk, lock_subvol, lock_subvol->fops->lk, fd, cmd,
+               flock, xdata);
+
+    return 0;
 
-        local->lock_type = flock->l_type;
-        lock_subvol = dht_get_lock_subvolume (this, flock, local);
-        if (!lock_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no lock subvolume for path=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        local->rebalance.flock = *flock;
-        local->rebalance.lock_cmd = cmd;
+    return 0;
+}
 
-        local->call_cnt = 1;
+static int
+dht_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct gf_lease *lease, dict_t *xdata)
+{
+    DHT_STACK_UNWIND(lease, frame, op_ret, op_errno, lease, xdata);
 
-        STACK_WIND (frame, dht_lk_cbk, lock_subvol, lock_subvol->fops->lk, fd,
-                    cmd, flock, xdata);
+    return 0;
+}
 
-        return 0;
+int
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* TODO: for rebalance, we need to preserve the fop arguments */
+    STACK_WIND(frame, dht_lease_cbk, subvol, subvol->fops->lease, loc, lease,
+               xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lease_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct gf_lease *lease, dict_t *xdata)
+/* Symlinks are currently not migrated, so no need for any check here */
+static int
+dht_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, const char *path, struct iatt *stbuf,
+                 dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lease, frame, op_ret, op_errno, lease, xdata);
+    dht_local_t *local = NULL;
 
-        return 0;
+    local = frame->local;
+    if (op_ret == -1)
+        goto err;
+
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+err:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+
+    return 0;
 }
 
 int
-dht_lease (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, struct gf_lease *lease, dict_t *xdata)
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_READLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    STACK_WIND(frame, dht_readlink_cbk, subvol, subvol->fops->readlink, loc,
+               size, xdata);
+
+    return 0;
 
-        /* TODO: for rebalance, we need to preserve the fop arguments */
-        STACK_WIND (frame, dht_lease_cbk, subvol, subvol->fops->lease,
-                    loc, lease, xdata);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
+}
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lease, frame, -1, op_errno, NULL, NULL);
+/* Get both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available, else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ * This will return a dummy iatt with only the mode and type set
+ */
+static int
+dht_read_iatt_from_xdata(dict_t *xdata, struct iatt *stbuf)
+{
+    int ret = -1;
+    int32_t mode = 0;
 
-        return 0;
+    ret = dict_get_int32(xdata, DHT_MODE_IN_XDATA_KEY, &mode);
+
+    if (ret) {
+        ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    } else {
+        stbuf->ia_prot = ia_prot_from_st_mode(mode);
+        stbuf->ia_type = ia_type_from_st_mode(mode);
+    }
+
+    return ret;
 }
 
-/* Symlinks are currently not migrated, so no need for any check here */
 int
-dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path,
-                  struct iatt *stbuf, dict_t *xdata)
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *call_frame = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    local = frame->local;
+    call_frame = cookie;
+    prev = call_frame->this;
+
+    local->op_errno = op_errno;
+
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    ret = dht_read_iatt_from_xdata(xdata, &stbuf);
+
+    if ((!op_ret) && (ret)) {
+        /* This is a potential problem and can cause corruption
+         * with sharding.
+         * Oh well. We tried.
+         */
+        goto out;
+    }
 
-        local = frame->local;
-        if (op_ret == -1)
-                goto err;
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_common_xattrop2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
+    if (dict)
+        local->rebalance.dict = dict_ref(dict);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(&stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(&stbuf)) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_common_xattrop2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
-err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+out:
+    if (local->fop == GF_FOP_XATTROP) {
+        DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-              dict_t *xdata)
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_READLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        if (local->fop == GF_FOP_XATTROP) {
+            DHT_STACK_UNWIND(xattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
+        } else {
+            DHT_STACK_UNWIND(fxattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
         }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_readlink_cbk,
-                    subvol, subvol->fops->readlink,
-                    loc, size, xdata);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+    if (local->fop == GF_FOP_XATTROP) {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   &local->loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    } else {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, local->fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
 
-        return 0;
-}
+    return 0;
 
-/* Currently no translators on top of 'distribute' will be using
- * below fops, hence not implementing 'migration' related checks
- */
+out:
 
-int
-dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+    /* If local is unavailable we could be unwinding the wrong
+     * function here */
+
+    if (local && (local->fop == GF_FOP_XATTROP)) {
+        DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    }
+    return 0;
+}
+
+static int
+dht_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
+/* Set both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available. Else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ */
+static int
+dht_request_iatt_in_xdata(dict_t *xattr_req)
+{
+    int ret = -1;
+
+    ret = dict_set_int8(xattr_req, DHT_MODE_IN_XDATA_KEY, 1);
+    ret = dict_set_int8(xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+
+    /* At least one call succeeded */
+    return ret;
+}
 
 int
-dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_XATTROP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_XATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        STACK_WIND(frame, dht_xattrop_cbk, subvol, subvol->fops->xattrop, loc,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for gfid=%s",
-                              uuid_utoa (loc->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
 
-        local->call_cnt = 1;
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
 
-        STACK_WIND (frame,
-                    dht_xattrop_cbk,
-                    subvol, subvol->fops->xattrop,
-                    loc, flags, dict, xdata);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s file=%s",
+                         DHT_IATT_IN_XDATA_KEY, loc->path);
+        }
 
-        return 0;
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-dht_fxattrop (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FXATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        STACK_WIND(frame, dht_fxattrop_cbk, subvol, subvol->fops->fxattrop, fd,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
+
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
+
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
+
+        if (ret) {
+            gf_msg_debug(this->name, 0, "Failed to set dictionary key %s fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
         }
 
-        STACK_WIND (frame,
-                    dht_fxattrop_cbk,
-                    subvol, subvol->fops->fxattrop,
-                    fd, flags, dict, xdata);
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+/* Currently no translators on top of 'distribute' will be using
+ * below fops, hence not implementing 'migration' related checks
+ */
 
-int
-dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        dht_lk_inode_unref (frame, op_ret);
-        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        xlator_t        *lock_subvol            = NULL;
-        int              op_errno               = -1;
-        dht_local_t     *local                  = NULL;
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
+    local = dht_local_init(frame, loc, NULL, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_INODELK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local->lock_type = lock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->lock_type = lock->l_type;
-        lock_subvol = dht_get_lock_subvolume (this, lock, local);
-        if (!lock_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no lock subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        local->call_cnt = 1;
+    local->call_cnt = 1;
 
-        STACK_WIND (frame,
-                    dht_inodelk_cbk,
-                    lock_subvol, lock_subvol->fops->inodelk,
-                    volume, loc, cmd, lock, xdata);
+    STACK_WIND(frame, dht_inodelk_cbk, lock_subvol, lock_subvol->fops->inodelk,
+               volume, loc, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(inodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-
-{
-
-        dht_lk_inode_unref (frame, op_ret);
-        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-int
-dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        xlator_t     *lock_subvol       = NULL;
-        dht_local_t  *local             = NULL;
-        int           op_errno          = -1;
-
+    dht_local_t *local = NULL;
+    int ret = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_INODELK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = frame->local;
 
-        local->call_cnt = 1;
-        local->lock_type = lock->l_type;
-
-        lock_subvol = dht_get_lock_subvolume (this, lock, local);
-        if (!lock_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no lock subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
+out:
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
 
-        STACK_WIND (frame, dht_finodelk_cbk, lock_subvol,
-                    lock_subvol->fops->finodelk,
-                    volume, fd, cmd, lock, xdata);
+    return 0;
+}
 
-        return 0;
+int
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    xlator_t *lock_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = 1;
+    local->lock_type = lock->l_type;
+
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    local->rebalance.flock = *lock;
+    local->rebalance.lock_cmd = cmd;
+    local->key = gf_strdup(volume);
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND(frame, dht_finodelk_cbk, lock_subvol,
+               lock_subvol->fops->finodelk, volume, fd, cmd, lock, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 112685b659e..2f23ce90fbd 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -8,1201 +8,1397 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "dht-common.h"
 
-int dht_writev2 (xlator_t *this, xlator_t *subvol,
-                 call_frame_t *frame, int ret);
-int dht_truncate2 (xlator_t *this, xlator_t *subvol,
-                   call_frame_t *frame, int ret);
-int dht_setattr2 (xlator_t *this, xlator_t *subvol,
-                  call_frame_t *frame, int ret);
-int dht_fallocate2 (xlator_t *this, xlator_t *subvol,
-                    call_frame_t *frame, int ret);
-int dht_discard2 (xlator_t *this, xlator_t *subvol,
-                  call_frame_t *frame, int ret);
-int dht_zerofill2 (xlator_t *this, xlator_t *subvol,
-                   call_frame_t *frame, int ret);
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
 
 int
-dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        call_frame_t *prev = NULL;
-        int          ret   = -1;
-        xlator_t    *subvol1 = NULL;
-        xlator_t    *subvol2 = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL;
+    xlator_t *subvol2 = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* writev fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could be a valid bad fd error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, 0, "subvolume %s returned -1 (%s)", prev->name,
+                     strerror(op_errno));
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        /* preserve the modes of source */
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
+        }
+        goto out;
+    }
+
+    local->rebalance.target_op_fn = dht_writev2;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        if (!local->xattr_req) {
+            local->xattr_req = dict_new();
+            if (!local->xattr_req) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+                       "insufficient memory");
+                local->op_errno = ENOMEM;
                 local->op_ret = -1;
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
                 goto out;
+            }
         }
 
-        if (local->call_cnt != 1) {
-                /* preserve the modes of source */
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
+        ret = dict_set_uint32(local->xattr_req, GF_PROTECT_FROM_EXTERNAL_WRITES,
+                              1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_DICT_SET_FAILED, 0,
+                   "Failed to set key %s in dictionary",
+                   GF_PROTECT_FROM_EXTERNAL_WRITES);
+            local->op_errno = ENOMEM;
+            local->op_ret = -1;
+            goto out;
         }
 
-        local->rebalance.target_op_fn = dht_writev2;
-
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
-
-        /* We might need to pass the stbuf information to the higher DHT
-         * layer for appropriate handling.
-         */
-
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
-
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        if (dht_fd_open_on_dst (this, local->fd, subvol2)) {
-                                dht_writev2 (this, subvol2, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_writev2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    DHT_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (writev, frame, local->op_ret,
-                                  local->op_errno, &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND (frame, dht_writev_cbk,
-                    subvol, subvol->fops->writev,
-                    local->fd, local->rebalance.vector, local->rebalance.count,
-                    local->rebalance.offset, local->rebalance.flags,
-                    local->rebalance.iobref, NULL);
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, local->fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int count, off_t off, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_WRITE);
-        if (!local) {
-
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-
-        local->rebalance.vector = iov_dup (vector, count);
-        local->rebalance.offset = off;
-        local->rebalance.count = count;
-        local->rebalance.flags = flags;
-        local->rebalance.iobref = iobref_ref (iobref);
-        local->call_cnt = 1;
-
-        STACK_WIND (frame, dht_writev_cbk,
-                    subvol, subvol->fops->writev,
-                    fd, vector, count, off, flags, iobref, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_WRITE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.vector = iov_dup(vector, count);
+    local->rebalance.offset = off;
+    local->rebalance.count = count;
+    local->rebalance.flags = flags;
+    local->rebalance.iobref = iobref_ref(iobref);
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-
 int
-dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
 {
-        dht_local_t  *local      = NULL;
-        call_frame_t *prev       = NULL;
-        int           ret        = -1;
-        xlator_t     *src_subvol = NULL;
-        xlator_t     *dst_subvol = NULL;
-        inode_t      *inode      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* Needs to be checked only for ftruncate.
+     * ftruncate fails with EBADF/EINVAL if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if ((local->fop == GF_FOP_FTRUNCATE) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-                goto out;
-        }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        local->rebalance.target_op_fn = dht_truncate2;
+    local->rebalance.target_op_fn = dht_truncate2;
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        /* We might need to pass the stbuf information to the higher DHT
-         * layer for appropriate handling.
-         */
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
 
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
-
-                dht_inode_ctx_get_mig_info (this, inode, &src_subvol,
-                                            &dst_subvol);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              src_subvol, dst_subvol)) {
-                        if ((!local->fd) || ((local->fd) &&
-                            dht_fd_open_on_dst (this, local->fd, dst_subvol))) {
-                                dht_truncate2 (this, dst_subvol, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if ((!local->fd) ||
+                ((local->fd) &&
+                 dht_fd_open_on_dst(this, local->fd, dst_subvol))) {
+                dht_truncate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-
-int
-dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
-
-        local = frame->local;
-        op_errno = local->op_errno;
+    if (!frame || !frame->local)
+        goto out;
 
-        /* This dht xlator is not migrating the file  */
-        if (we_are_not_migrating (ret)) {
+    local = frame->local;
+    op_errno = local->op_errno;
 
-                DHT_STACK_UNWIND (truncate, frame, local->op_ret,
-                                  local->op_errno, &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    /* This dht xlator is not migrating the file  */
+    if (we_are_not_migrating(ret)) {
+        DHT_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        if (local->fop == GF_FOP_TRUNCATE) {
-                STACK_WIND (frame, dht_truncate_cbk, subvol,
-                            subvol->fops->truncate, &local->loc,
-                            local->rebalance.offset, NULL);
-        } else {
-                STACK_WIND (frame, dht_truncate_cbk, subvol,
-                            subvol->fops->ftruncate, local->fd,
-                            local->rebalance.offset, NULL);
-        }
+    if (local->fop == GF_FOP_TRUNCATE) {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->truncate, &local->loc,
+                          local->rebalance.offset, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->ftruncate, local->fd,
+                          local->rebalance.offset, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-              dict_t *xdata)
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_TRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for gfid=%s",
-                              uuid_utoa (loc->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_truncate_cbk,
-                    subvol, subvol->fops->truncate,
-                    loc, offset, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_TRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->truncate, loc, offset, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               dict_t *xdata)
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FTRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_truncate_cbk,
-                    subvol, subvol->fops->ftruncate,
-                    fd, offset, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FTRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, local->rebalance.offset,
+                      local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        xlator_t    *src_subvol = NULL;
-        xlator_t    *dst_subvol = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* fallocate fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->op_ret = op_ret;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-        local->rebalance.target_op_fn = dht_fallocate2;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+        goto out;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
-
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                dht_inode_ctx_get_mig_info (this, local->fd->inode, &src_subvol,
-                                            &dst_subvol);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              src_subvol, dst_subvol)) {
-                        if (dht_fd_open_on_dst (this, local->fd, dst_subvol)) {
-                                dht_fallocate2 (this, dst_subvol, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        goto out;
+    }
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_fallocate2;
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_fallocate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                     xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (fallocate, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fallocate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto out;
 
-	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
-		   local->fd, local->rebalance.flags, local->rebalance.offset,
-		   local->rebalance.size, NULL);
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, local->fd,
+                      local->rebalance.flags, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
 dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-	      off_t offset, size_t len, dict_t *xdata)
+              off_t offset, size_t len, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->rebalance.flags = mode;
-        local->rebalance.offset = offset;
-	local->rebalance.size = len;
-
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_fallocate_cbk,
-                    subvol, subvol->fops->fallocate,
-                    fd, mode, offset, len, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FALLOCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mode;
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
+
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, fd, local->rebalance.flags,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        xlator_t    *src_subvol = NULL;
-        xlator_t    *dst_subvol = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* discard fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->rebalance.target_op_fn = dht_discard2;
-        local->op_ret = op_ret;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+        goto out;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
-
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                dht_inode_ctx_get_mig_info (this, local->fd->inode, &src_subvol,
-                                            &dst_subvol);
-                if (!dht_mig_info_is_invalid(local->cached_subvol,
-                                             src_subvol, dst_subvol)) {
-                        if (dht_fd_open_on_dst (this, local->fd, dst_subvol)) {
-                                dht_discard2 (this, dst_subvol, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        goto out;
+    }
+
+    local->rebalance.target_op_fn = dht_discard2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_discard2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (discard, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(discard, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto out;
 
-	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
-		   local->fd, local->rebalance.offset, local->rebalance.size,
-		   NULL);
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, local->fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
 dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	    size_t len, dict_t *xdata)
+            size_t len, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.offset = offset;
-	local->rebalance.size = len;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, fd, GF_FOP_DISCARD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
-                    fd, offset, len, xdata);
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
 
-        return 0;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
 {
-        dht_local_t  *local   = NULL;
-        call_frame_t *prev    = NULL;
-        int           ret     = -1;
-        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* zerofill fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->rebalance.target_op_fn = dht_zerofill2;
-        local->op_ret = op_ret;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
-
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
-
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        if (dht_fd_open_on_dst (this, local->fd, subvol2)) {
-                                dht_zerofill2 (this, subvol2, frame, 0);
-                                return 0;
-                        }
-                }
-
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        goto out;
+    }
+
+    local->rebalance.target_op_fn = dht_zerofill2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_zerofill2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
 
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
+    local = frame->local;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (zerofill, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(zerofill, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
 
-                return 0;
-        }
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
-                   local->fd, local->rebalance.offset, local->rebalance.size,
-                   NULL);
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, local->fd,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
 
-        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
 dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            off_t len, dict_t *xdata)
+             off_t len, dict_t *xdata)
 {
-        xlator_t     *subvol       = NULL;
-        int           op_errno     = -1;
-        dht_local_t  *local        = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.offset = offset;
-        local->rebalance.size = len;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, fd, GF_FOP_ZEROFILL);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
-                    fd, offset, len, xdata);
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
 
-        return 0;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
-}
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
 
+    return 0;
+}
 
 /* handle cases of migration here for 'setattr()' calls */
 int
-dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        local->op_errno = op_errno;
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
-        }
+    local->op_errno = op_errno;
 
-        if (local->call_cnt != 1)
-                goto out;
+    if ((local->fop == GF_FOP_FSETATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-        local->rebalance.target_op_fn = dht_setattr2;
+    if (local->call_cnt != 1)
+        goto out;
 
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+    local->rebalance.target_op_fn = dht_setattr2;
 
-                dht_set_local_rebalance (this, local, NULL, prebuf,
-                                         postbuf, xdata);
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        /* At the end of the migration process, whatever 'attr' we
-           have on source file will be migrated to destination file
-           in one shot, hence we don't need to check for in progress
-           state here (ie, PHASE1) */
+    /* At the end of the migration process, whatever 'attr' we
+       have on source file will be migrated to destination file
+       in one shot, hence we don't need to check for in progress
+       state here (ie, PHASE1) */
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(setattr, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (setattr, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
-
-        if (local->fop == GF_FOP_SETATTR) {
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->setattr, &local->loc,
-                            &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
-        } else {
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->fsetattr, local->fd,
-                            &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
-        }
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (local->fop == GF_FOP_SETATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, &local->loc,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, local->fd,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    }
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* Keep the existing code same for all the cases other than regular file */
 int
-dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost, dict_t *xdata)
+dht_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *statpre, struct iatt *statpost,
+                dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->this->name);
-                        goto unlock;
-                }
-
-                dht_iatt_merge (this, &local->prebuf, statpre, prev->this);
-                dht_iatt_merge (this, &local->stbuf, statpost, prev->this);
-
-                local->op_ret = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-		if (local->op_ret == 0)
-			dht_inode_ctx_time_set (local->loc.inode, this,
-						&local->stbuf);
-                DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-                                  &local->prebuf, &local->stbuf, xdata);
-	}
 
-        return 0;
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
+
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0)
+            dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->prebuf, &local->stbuf, xdata);
+    }
+
+    return 0;
 }
 
-
+/* Keep the existing code same for all the cases other than regular file */
 int
-dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+dht_non_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_layout_t *layout = NULL;
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, op_errno, 0, 0, "subvolume %s returned -1",
+               prev->name);
+        goto post_unlock;
+    }
+
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
+
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, 0, 0, &local->prebuf, &local->stbuf,
+                         xdata);
+    }
+
+    return 0;
+}
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+int
+dht_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 
-        if (!layout_is_sane (layout)) {
-                gf_msg_debug (this->name, 0,
-                              "layout is not sane for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    struct iatt loc_stbuf = {
+        0,
+    };
+    int i = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    local->op_ret = 0;
+    loc_stbuf = local->stbuf;
+    dht_iatt_merge(this, &local->prebuf, statpre);
+    dht_iatt_merge(this, &local->stbuf, statpost);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol == conf->subvolumes[i])
+            continue;
+        STACK_WIND_COOKIE(frame, dht_non_mds_setattr_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->setattr, &local->loc,
+                          &loc_stbuf, local->valid, local->xattr_req);
+    }
+
+    return 0;
+out:
+    DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                     &local->prebuf, &local->stbuf, xdata);
 
-        if (IA_ISREG (loc->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
+    return 0;
+}
 
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->setattr,
-                            loc, stbuf, valid, xdata);
+int
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
 
-                return 0;
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, loc, stbuf, valid, xdata);
+
+        return 0;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(loc->inode->ia_type) && !__is_root_gfid(loc->inode->gfid) &&
+        call_cnt != 1) {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for path %s", local->loc.path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
+        local->mds_subvol = mds_subvol;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_msg(this->name, GF_LOG_WARNING, layout->list[i].err,
+                           DHT_MSG_HASHED_SUBVOL_DOWN,
+                           "MDS subvol is down for path "
+                           " %s Unable to set attr ",
+                           local->loc.path);
+                    op_errno = ENOTCONN;
+                    goto err;
+                }
+            }
+        }
+        local->valid = valid;
+        local->stbuf = *stbuf;
 
+        STACK_WIND_COOKIE(frame, dht_mds_setattr_cbk, local->mds_subvol,
+                          local->mds_subvol, local->mds_subvol->fops->setattr,
+                          loc, stbuf, valid, xdata);
+        return 0;
+    } else {
         for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_setattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->setattr,
-                            loc, stbuf, valid, xdata);
+            STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                              layout->list[i].xlator,
+                              layout->list[i].xlator->fops->setattr, loc, stbuf,
+                              valid, xdata);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-              int32_t valid, dict_t *xdata)
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_layout_t *layout = NULL;
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (!layout_is_sane (layout)) {
-                gf_msg_debug (this->name, 0,
-                              "layout is not sane for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (IA_ISREG (fd->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
-
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
+    xlator_t *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
 
-                return 0;
-        }
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, fd, &local->rebalance.stbuf,
+                          local->rebalance.flags, local->xattr_req);
+        return 0;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->call_cnt = call_cnt = layout->cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_setattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                          layout->list[i].xlator,
+                          layout->list[i].xlator->fops->fsetattr, fd, stbuf,
+                          valid, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 4352ffe5756..fda904c92c9 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -8,885 +8,801 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "byte-order.h"
-#include "dht-messages.h"
+#include <glusterfs/byte-order.h>
 #include "unittest/unittest.h"
 
+#define layout_base_size (sizeof(dht_layout_t))
 
-#define layout_base_size (sizeof (dht_layout_t))
-
-#define layout_entry_size (sizeof ((dht_layout_t *)NULL)->list[0])
+#define layout_entry_size (sizeof((dht_layout_t *)NULL)->list[0])
 
 #define layout_size(cnt) (layout_base_size + (cnt * layout_entry_size))
 
 dht_layout_t *
-dht_layout_new (xlator_t *this, int cnt)
+dht_layout_new(xlator_t *this, int cnt)
 {
-        dht_layout_t *layout = NULL;
-        dht_conf_t   *conf = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
 
-        REQUIRE(NULL != this);
-        REQUIRE(cnt >= 0);
+    REQUIRE(NULL != this);
+    REQUIRE(cnt >= 0);
 
-        conf = this->private;
+    conf = this->private;
 
-        layout = GF_CALLOC (1, layout_size (cnt),
-                            gf_dht_mt_dht_layout_t);
-        if (!layout) {
-                goto out;
-        }
+    layout = GF_CALLOC(1, layout_size(cnt), gf_dht_mt_dht_layout_t);
+    if (!layout) {
+        goto out;
+    }
 
-        layout->type = DHT_HASH_TYPE_DM;
-        layout->cnt = cnt;
+    layout->type = DHT_HASH_TYPE_DM;
+    layout->cnt = cnt;
 
-        if (conf) {
-                layout->spread_cnt = conf->dir_spread_cnt;
-                layout->gen = conf->gen;
-        }
+    if (conf) {
+        layout->spread_cnt = conf->dir_spread_cnt;
+        layout->gen = conf->gen;
+    }
 
-        layout->ref = 1;
+    GF_ATOMIC_INIT(layout->ref, 1);
 
-        ENSURE(NULL != layout);
-        ENSURE(layout->type == DHT_HASH_TYPE_DM);
-        ENSURE(layout->cnt == cnt);
-        ENSURE(layout->ref == 1);
+    ENSURE(NULL != layout);
+    ENSURE(layout->type == DHT_HASH_TYPE_DM);
+    ENSURE(layout->cnt == cnt);
+    ENSURE(GF_ATOMIC_GET(layout->ref) == 1);
 out:
-        return layout;
+    return layout;
 }
 
-
 dht_layout_t *
-dht_layout_get (xlator_t *this, inode_t *inode)
+dht_layout_get(xlator_t *this, inode_t *inode)
 {
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *layout = NULL;
-        int           ret = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                ret = dht_inode_ctx_layout_get (inode, this, &layout);
-                if ((!ret) && layout) {
-                        layout->ref++;
-                }
-        }
-        UNLOCK (&conf->layout_lock);
-
-out:
-        return layout;
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
+    if ((!ret) && layout) {
+        GF_ATOMIC_INC(layout->ref);
+    }
+    return layout;
 }
 
-
 int
-dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout)
 {
-        dht_conf_t   *conf = NULL;
-        int           oldret = -1;
-        int           ret = -1;
-        dht_layout_t *old_layout;
-
-        conf = this->private;
-        if (!conf || !layout)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
-                if (layout)
-                        layout->ref++;
-                ret = dht_inode_ctx_layout_set (inode, this, layout);
-        }
-        UNLOCK (&conf->layout_lock);
-
-        if (!oldret) {
-                dht_layout_unref (this, old_layout);
-        }
+    dht_conf_t *conf = NULL;
+    int oldret = -1;
+    int ret = -1;
+    dht_layout_t *old_layout;
+
+    conf = this->private;
+    if (!conf || !layout)
+        goto out;
+
+    LOCK(&conf->layout_lock);
+    {
+        oldret = dht_inode_ctx_layout_get(inode, this, &old_layout);
+        if (layout)
+            GF_ATOMIC_INC(layout->ref);
+        ret = dht_inode_ctx_layout_set(inode, this, layout);
+    }
+    UNLOCK(&conf->layout_lock);
+
+    if (!oldret) {
+        dht_layout_unref(this, old_layout);
+    }
+    if (ret)
+        GF_ATOMIC_DEC(layout->ref);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-dht_layout_unref (xlator_t *this, dht_layout_t *layout)
+dht_layout_unref(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-        int          ref = 0;
-
-        if (!layout || layout->preset || !this->private)
-                return;
+    int ref = 0;
 
-        conf = this->private;
+    if (!layout || layout->preset || !this->private)
+        return;
 
-        LOCK (&conf->layout_lock);
-        {
-                ref = --layout->ref;
-        }
-        UNLOCK (&conf->layout_lock);
+    ref = GF_ATOMIC_DEC(layout->ref);
 
-        if (!ref)
-                GF_FREE (layout);
+    if (!ref)
+        GF_FREE(layout);
 }
 
-
 dht_layout_t *
-dht_layout_ref (xlator_t *this, dht_layout_t *layout)
+dht_layout_ref(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-
-        if (layout->preset || !this->private)
-                return layout;
+    if (layout->preset || !this->private)
+        return layout;
 
-        conf = this->private;
-        LOCK (&conf->layout_lock);
-        {
-                layout->ref++;
-        }
-        UNLOCK (&conf->layout_lock);
+    GF_ATOMIC_INC(layout->ref);
 
-        return layout;
+    return layout;
 }
 
-
 xlator_t *
-dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name)
 {
-        uint32_t   hash = 0;
-        xlator_t  *subvol = NULL;
-        int        i = 0;
-        int        ret = 0;
-
-        ret = dht_hash_compute (this, layout->type, name, &hash);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_COMPUTE_HASH_FAILED,
-                        "hash computation failed for type=%d name=%s",
-                        layout->type, name);
-                goto out;
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].start <= hash
-                    && layout->list[i].stop >= hash) {
-                        subvol = layout->list[i].xlator;
-                        break;
-                }
-        }
-
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "no subvolume for hash (value) = %u", hash);
-        }
+    uint32_t hash = 0;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+
+    ret = dht_hash_compute(this, layout->type, name, &hash);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMPUTE_HASH_FAILED,
+                "type=%d", layout->type, "name=%s", name, NULL);
+        goto out;
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].start <= hash && layout->list[i].stop >= hash) {
+            subvol = layout->list[i].xlator;
+            break;
+        }
+    }
+
+    if (!subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                "hash-value=0x%x", hash, NULL);
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
-
 dht_layout_t *
-dht_layout_for_subvol (xlator_t *this, xlator_t *subvol)
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol)
 {
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == subvol) {
-                        layout = conf->file_layouts[i];
-                        break;
-                }
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            layout = conf->file_layouts[i];
+            break;
         }
+    }
 
 out:
-        return layout;
+    return layout;
 }
 
-
 int
-dht_layouts_init (xlator_t *this, dht_conf_t *conf)
+dht_layouts_init(xlator_t *this, dht_conf_t *conf)
 {
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           ret = -1;
-
-        if (!conf)
-                goto out;
-
-        conf->file_layouts = GF_CALLOC (conf->subvolume_cnt,
-                                        sizeof (dht_layout_t *),
-                                        gf_dht_mt_dht_layout_t);
-        if (!conf->file_layouts) {
-                goto out;
-        }
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int ret = -1;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                layout = dht_layout_new (this, 1);
+    if (!conf)
+        goto out;
 
-                if (!layout) {
-                        goto out;
-                }
+    conf->file_layouts = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_layout_t *),
+                                   gf_dht_mt_dht_layout_t);
+    if (!conf->file_layouts) {
+        goto out;
+    }
 
-                layout->preset = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        layout = dht_layout_new(this, 1);
 
-                layout->list[0].xlator = conf->subvolumes[i];
-
-                conf->file_layouts[i] = layout;
+        if (!layout) {
+            goto out;
         }
 
-        ret = 0;
+        layout->preset = 1;
+
+        layout->list[0].xlator = conf->subvolumes[i];
+
+        conf->file_layouts[i] = layout;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                         int pos, int32_t **disk_layout_p)
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p)
 {
-        int      ret = -1;
-        int32_t *disk_layout = NULL;
+    int ret = -1;
+    int32_t *disk_layout = NULL;
 
-        disk_layout = GF_CALLOC (5, sizeof (int),
-                                 gf_dht_mt_int32_t);
-        if (!disk_layout) {
-                goto out;
-        }
+    disk_layout = GF_CALLOC(5, sizeof(int), gf_dht_mt_int32_t);
+    if (!disk_layout) {
+        goto out;
+    }
 
-        disk_layout[0] = hton32 (layout->list[pos].commit_hash);
-        disk_layout[1] = hton32 (layout->type);
-        disk_layout[2] = hton32 (layout->list[pos].start);
-        disk_layout[3] = hton32 (layout->list[pos].stop);
+    disk_layout[0] = hton32(layout->list[pos].commit_hash);
+    disk_layout[1] = hton32(layout->type);
+    disk_layout[2] = hton32(layout->list[pos].start);
+    disk_layout[3] = hton32(layout->list[pos].stop);
 
-        if (disk_layout_p)
-                *disk_layout_p = disk_layout;
-        else
-                GF_FREE (disk_layout);
+    if (disk_layout_p)
+        *disk_layout_p = disk_layout;
+    else
+        GF_FREE(disk_layout);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_disk_layout_extract_for_subvol (xlator_t *this, dht_layout_t *layout,
-                                    xlator_t *subvol, int32_t **disk_layout_p)
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol)
-                        break;
-        }
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol)
+            break;
+    }
 
-        if (i == layout->cnt)
-                return -1;
+    if (i == layout->cnt)
+        return -1;
 
-        return dht_disk_layout_extract (this, layout, i, disk_layout_p);
+    return dht_disk_layout_extract(this, layout, i, disk_layout_p);
 }
 
-int
-dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-		       int pos, void *disk_layout_raw, int disk_layout_len)
+static int
+dht_disk_layout_merge(xlator_t *this, dht_layout_t *layout, int pos,
+                      void *disk_layout_raw, int disk_layout_len)
 {
-        int      type = 0;
-        int      start_off = 0;
-        int      stop_off = 0;
-        int      commit_hash = 0;
-        int      disk_layout[4];
+    int type = 0;
+    int start_off = 0;
+    int stop_off = 0;
+    int commit_hash = 0;
+    int disk_layout[4];
 
-	if (!disk_layout_raw) {
-		gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "error no layout on disk for merge");
-		return -1;
-	}
+    if (!disk_layout_raw) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                NULL);
+        return -1;
+    }
 
-	GF_ASSERT (disk_layout_len == sizeof (disk_layout));
+    GF_ASSERT(disk_layout_len == sizeof(disk_layout));
 
-        memcpy (disk_layout, disk_layout_raw, disk_layout_len);
+    memcpy(disk_layout, disk_layout_raw, disk_layout_len);
 
-        type = ntoh32 (disk_layout[1]);
-	switch (type) {
+    type = ntoh32(disk_layout[1]);
+    switch (type) {
         case DHT_HASH_TYPE_DM_USER:
-                gf_msg_debug (this->name, 0, "found user-set layout");
-                layout->type = type;
-                /* Fall through. */
-	case DHT_HASH_TYPE_DM:
-		break;
+            gf_msg_debug(this->name, 0, "found user-set layout");
+            layout->type = type;
+            /* Fall through. */
+        case DHT_HASH_TYPE_DM:
+            break;
         default:
-		gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_INVALID_DISK_LAYOUT,
-			"Invalid disk layout: "
-                        "Catastrophic error layout with unknown type found %d",
-			disk_layout[1]);
-		return -1;
-	}
-
-        commit_hash = ntoh32 (disk_layout[0]);
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
-
-        layout->list[pos].commit_hash = commit_hash;
-        layout->list[pos].start = start_off;
-        layout->list[pos].stop  = stop_off;
-
-        gf_msg_trace (this->name, 0,
-                      "merged to layout: %u - %u (type %d, hash %d) from %s",
-                      start_off, stop_off, commit_hash, type,
-                      layout->list[pos].xlator->name);
-
-        return 0;
+            gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_DISK_LAYOUT,
+                    "layout=%d", disk_layout[1], NULL);
+            return -1;
+    }
+
+    commit_hash = ntoh32(disk_layout[0]);
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
+
+    layout->list[pos].commit_hash = commit_hash;
+    layout->list[pos].start = start_off;
+    layout->list[pos].stop = stop_off;
+
+    gf_msg_trace(this->name, 0,
+                 "merged to layout: 0x%x - 0x%x (hash 0x%x, type %d) from %s",
+                 start_off, stop_off, commit_hash, type,
+                 layout->list[pos].xlator->name);
+
+    return 0;
 }
 
 int
-dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                  int op_ret, int op_errno, dict_t *xattr)
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr)
 {
-        int         i     = 0;
-        int         ret   = -1;
-        int         err   = -1;
-        void       *disk_layout_raw = NULL;
-        int         disk_layout_len = 0;
-        dht_conf_t *conf = this->private;
-
-        if (op_ret != 0) {
-                err = op_errno;
-        }
+    int i = 0;
+    int ret = -1;
+    int err = -1;
+    void *disk_layout_raw = NULL;
+    int disk_layout_len = 0;
+    dht_conf_t *conf = this->private;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == NULL) {
-                        layout->list[i].err    = err;
-                        layout->list[i].xlator = subvol;
-                        break;
-                }
-        }
+    if (op_ret != 0) {
+        err = op_errno;
+    }
 
-        if (op_ret != 0) {
-                ret = 0;
-                goto out;
-        }
+    if (!layout)
+        goto out;
 
-        if (xattr) {
-                /* during lookup and not mkdir */
-                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
-					    &disk_layout_raw, &disk_layout_len);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == NULL) {
+            layout->list[i].err = err;
+            layout->list[i].xlator = subvol;
+            break;
         }
+    }
 
-        if (ret != 0) {
-                layout->list[i].err = 0;
-                gf_msg_trace (this->name, 0,
-                              "Missing disk layout on %s. err = %d",
-                              subvol->name, err);
-                ret = 0;
-                goto out;
-        }
-
-        ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw,
-				     disk_layout_len);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "layout merge from subvolume %s failed",
-                        subvol->name);
-                goto out;
-        }
+    if (op_ret != 0) {
+        ret = 0;
+        goto out;
+    }
 
-        if (layout->commit_hash == 0) {
-                layout->commit_hash = layout->list[i].commit_hash;
-        } else if (layout->commit_hash != layout->list[i].commit_hash) {
-                layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
-        }
+    if (xattr) {
+        /* during lookup and not mkdir */
+        ret = dict_get_ptr_and_len(xattr, conf->xattr_name, &disk_layout_raw,
+                                   &disk_layout_len);
+    }
 
+    if (ret != 0) {
         layout->list[i].err = 0;
+        gf_msg_trace(this->name, 0, "Missing disk layout on %s. err = %d",
+                     subvol->name, err);
+        ret = 0;
+        goto out;
+    }
+
+    ret = dht_disk_layout_merge(this, layout, i, disk_layout_raw,
+                                disk_layout_len);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                "subvolume=%s", subvol->name, NULL);
+        goto out;
+    }
+
+    if (layout->commit_hash == 0) {
+        layout->commit_hash = layout->list[i].commit_hash;
+    } else if (layout->commit_hash != layout->list[i].commit_hash) {
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+    }
+
+    layout->list[i].err = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
+dht_layout_entry_swap(dht_layout_t *layout, int i, int j)
 {
-        uint32_t  start_swap = 0;
-        uint32_t  stop_swap = 0;
-        uint32_t  commit_hash_swap = 0;
-        xlator_t *xlator_swap = 0;
-        int       err_swap = 0;
-
-        start_swap  = layout->list[i].start;
-        stop_swap   = layout->list[i].stop;
-        xlator_swap = layout->list[i].xlator;
-        err_swap    = layout->list[i].err;
-        commit_hash_swap = layout->list[i].commit_hash;
-
-        layout->list[i].start  = layout->list[j].start;
-        layout->list[i].stop   = layout->list[j].stop;
-        layout->list[i].xlator = layout->list[j].xlator;
-        layout->list[i].err    = layout->list[j].err;
-        layout->list[i].commit_hash = layout->list[j].commit_hash;
-
-        layout->list[j].start  = start_swap;
-        layout->list[j].stop   = stop_swap;
-        layout->list[j].xlator = xlator_swap;
-        layout->list[j].err    = err_swap;
-        layout->list[j].commit_hash = commit_hash_swap;
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
+    uint32_t commit_hash_swap = 0;
+    xlator_t *xlator_swap = 0;
+    int err_swap = 0;
+
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
+    xlator_swap = layout->list[i].xlator;
+    err_swap = layout->list[i].err;
+    commit_hash_swap = layout->list[i].commit_hash;
+
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
+    layout->list[i].xlator = layout->list[j].xlator;
+    layout->list[i].err = layout->list[j].err;
+    layout->list[i].commit_hash = layout->list[j].commit_hash;
+
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
+    layout->list[j].xlator = xlator_swap;
+    layout->list[j].err = err_swap;
+    layout->list[j].commit_hash = commit_hash_swap;
 }
 
 void
-dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+dht_layout_range_swap(dht_layout_t *layout, int i, int j)
 {
-        uint32_t  start_swap = 0;
-        uint32_t  stop_swap = 0;
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
 
-        start_swap  = layout->list[i].start;
-        stop_swap   = layout->list[i].stop;
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
 
-        layout->list[i].start  = layout->list[j].start;
-        layout->list[i].stop   = layout->list[j].stop;
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
 
-        layout->list[j].start  = start_swap;
-        layout->list[j].stop   = stop_swap;
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
 }
-
-int64_t
-dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
+static int64_t
+dht_layout_entry_cmp_volname(dht_layout_t *layout, int i, int j)
 {
-        return (strcmp (layout->list[i].xlator->name,
-                        layout->list[j].xlator->name));
+    return (strcmp(layout->list[i].xlator->name, layout->list[j].xlator->name));
 }
 
 gf_boolean_t
-dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator)
 {
-        int i = 0;
-
-        for (i = 0; i < layout->cnt; i++) {
-                /* Check if xlator is already part of layout, and layout is
-                 * non-zero. */
-                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
-                        if (layout->list[i].start != layout->list[i].stop)
-                                return _gf_true;
-                        break;
-                }
-        }
-        return _gf_false;
+    int i = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        /* Check if xlator is already part of layout, and layout is
+         * non-zero. */
+        if (!strcmp(layout->list[i].xlator->name, xlator->name)) {
+            if (layout->list[i].start != layout->list[i].stop)
+                return _gf_true;
+            break;
+        }
+    }
+    return _gf_false;
 }
 
-int64_t
-dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
+static int64_t
+dht_layout_entry_cmp(dht_layout_t *layout, int i, int j)
 {
-        int64_t diff = 0;
+    int64_t diff = 0;
 
-        /* swap zero'ed out layouts to front, if needed */
-        if (!layout->list[j].start && !layout->list[j].stop) {
-                diff = (int64_t) layout->list[i].stop
-                       - (int64_t) layout->list[j].stop;
-                       goto out;
-        }
-        diff = (int64_t) layout->list[i].start
-                - (int64_t) layout->list[j].start;
+    /* swap zero'ed out layouts to front, if needed */
+    if (!layout->list[j].start && !layout->list[j].stop) {
+        diff = (int64_t)layout->list[i].stop - (int64_t)layout->list[j].stop;
+        goto out;
+    }
+    diff = (int64_t)layout->list[i].start - (int64_t)layout->list[j].start;
 
 out:
-        return diff;
+    return diff;
 }
 
-
 int
-dht_layout_sort (dht_layout_t *layout)
+dht_layout_sort(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_layout_sort_volname (dht_layout_t *layout)
+void
+dht_layout_sort_volname(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp_volname (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp_volname(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
-
-        return 0;
+    }
 }
 
-
-int
-dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-                      uint32_t *holes_p, uint32_t *overlaps_p,
-                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
-                      uint32_t *no_space_p)
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p)
 {
-        uint32_t    overlaps = 0;
-        uint32_t    missing  = 0;
-        uint32_t    down     = 0;
-        uint32_t    misc     = 0;
-        uint32_t    hole_cnt = 0;
-        uint32_t    overlap_cnt = 0;
-        int         i = 0;
-        int         ret = 0;
-        uint32_t    prev_stop = 0;
-        uint32_t    last_stop = 0;
-        char        is_virgin = 1;
-        uint32_t    no_space  = 0;
-
-        /* This function scans through the layout spread of a directory to
-           check if there are any anomalies. Prior to calling this function
-           the layout entries should be sorted in the ascending order.
-
-           If the layout entry has err != 0
-                then increment the corresponding anomaly.
-           else
-                if (start of the current layout entry > stop + 1 of previous
-                   non erroneous layout entry)
-                        then it indicates a hole in the layout
-                if (start of the current layout entry < stop + 1 of previous
-                    non erroneous layout entry)
-                         then it indicates an overlap in the layout
-        */
-        last_stop = layout->list[0].start - 1;
-        prev_stop = last_stop;
-
-        for (i = 0; i < layout->cnt; i++) {
-                switch (layout->list[i].err) {
-                case -1:
-                case ENOENT:
-                case ESTALE:
-                        missing++;
-                        continue;
-                case ENOTCONN:
-                        down++;
-                        continue;
-                case ENOSPC:
-                        no_space++;
-                        continue;
-                case 0:
-                        /* if err == 0 and start == stop, then it is a non misc++;
-                         * participating subvolume(spread-cnt). Then, do not
-                         * check for anomalies. If start != stop, then treat it
-                         * as misc err */
-                        if (layout->list[i].start == layout->list[i].stop) {
-                                continue;
-                        }
-                        break;
-                default:
-                        misc++;
-                        continue;
-                 }
-
-                is_virgin = 0;
-
-                if ((prev_stop + 1) < layout->list[i].start) {
-                        hole_cnt++;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    uint32_t hole_cnt = 0;
+    uint32_t overlap_cnt = 0;
+    int i = 0;
+    uint32_t prev_stop = 0;
+    uint32_t last_stop = 0;
+    char is_virgin = 1;
+    uint32_t no_space = 0;
+
+    /* This function scans through the layout spread of a directory to
+       check if there are any anomalies. Prior to calling this function
+       the layout entries should be sorted in the ascending order.
+
+       If the layout entry has err != 0
+            then increment the corresponding anomaly.
+       else
+            if (start of the current layout entry > stop + 1 of previous
+               non erroneous layout entry)
+                    then it indicates a hole in the layout
+            if (start of the current layout entry < stop + 1 of previous
+                non erroneous layout entry)
+                     then it indicates an overlap in the layout
+    */
+    last_stop = layout->list[0].start - 1;
+    prev_stop = last_stop;
+
+    for (i = 0; i < layout->cnt; i++) {
+        switch (layout->list[i].err) {
+            case -1:
+            case ENOENT:
+            case ESTALE:
+                missing++;
+                continue;
+            case ENOTCONN:
+                down++;
+                continue;
+            case ENOSPC:
+                no_space++;
+                continue;
+            case 0:
+                /* if err == 0 and start == stop, then it is a non misc++;
+                 * participating subvolume(spread-cnt). Then, do not
+                 * check for anomalies. If start != stop, then treat it
+                 * as misc err */
+                if (layout->list[i].start == layout->list[i].stop) {
+                    continue;
                 }
+                break;
+            default:
+                misc++;
+                continue;
+        }
 
-                if ((prev_stop + 1) > layout->list[i].start) {
-                        overlap_cnt++;
-                        overlaps += ((prev_stop + 1) - layout->list[i].start);
-                }
-                prev_stop = layout->list[i].stop;
+        is_virgin = 0;
+
+        if ((prev_stop + 1) < layout->list[i].start) {
+            hole_cnt++;
         }
 
-        if ((last_stop - prev_stop) || is_virgin)
-                hole_cnt++;
+        if ((prev_stop + 1) > layout->list[i].start) {
+            overlap_cnt++;
+            overlaps += ((prev_stop + 1) - layout->list[i].start);
+        }
+        prev_stop = layout->list[i].stop;
+    }
 
-        if (holes_p)
-                *holes_p = hole_cnt;
+    if ((last_stop - prev_stop) || is_virgin)
+        hole_cnt++;
 
-        if (overlaps_p)
-                *overlaps_p = overlap_cnt;
+    if (holes_p)
+        *holes_p = hole_cnt;
 
-        if (missing_p)
-                *missing_p = missing;
+    if (overlaps_p)
+        *overlaps_p = overlap_cnt;
 
-        if (down_p)
-                *down_p = down;
+    if (missing_p)
+        *missing_p = missing;
 
-        if (misc_p)
-                *misc_p = misc;
+    if (down_p)
+        *down_p = down;
 
-        if (no_space_p)
-                *no_space_p = no_space;
+    if (misc_p)
+        *misc_p = misc;
 
-        return ret;
+    if (no_space_p)
+        *no_space_p = no_space;
 }
 
-
 int
-dht_layout_missing_dirs (dht_layout_t *layout)
+dht_layout_missing_dirs(dht_layout_t *layout)
 {
-        int i = 0, missing = 0;
+    int i = 0, missing = 0;
 
-        if (layout == NULL)
-                goto out;
+    if (layout == NULL)
+        goto out;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if ((layout->list[i].err == ENOENT)
-                    || ((layout->list[i].err == -1)
-                        && (layout->list[i].start == 0)
-                        && (layout->list[i].stop == 0))) {
-                        missing++;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if ((layout->list[i].err == ENOENT) ||
+            ((layout->list[i].err == -1) && (layout->list[i].start == 0) &&
+             (layout->list[i].stop == 0))) {
+            missing++;
         }
+    }
 
 out:
-        return missing;
+    return missing;
 }
 
-
 int
-dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout)
 {
-        int          ret   = 0;
-        uint32_t     holes = 0;
-        uint32_t     overlaps = 0;
-        uint32_t     missing = 0;
-        uint32_t     down = 0;
-        uint32_t     misc = 0, missing_dirs = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        ret = dht_layout_sort (layout);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_SORT_FAILED,
-                        "sort failed?! how the ....");
-                goto out;
-        }
-
-        gf_uuid_unparse(loc->gfid, gfid);
-
-        ret = dht_layout_anomalies (this, loc, layout,
-                                    &holes, &overlaps,
-                                    &missing, &down, &misc, NULL);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR,
-                        "Error finding anomalies in %s, gfid = %s",
-                        loc->path, gfid);
-                goto out;
-        }
-
-        if (holes || overlaps) {
-                if (missing == layout->cnt) {
-                        gf_msg_debug (this->name, 0,
-                                      "Directory %s looked up first time"
-                                      " gfid = %s", loc->path, gfid);
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_ANOMALIES_INFO,
-                                "Found anomalies in %s (gfid = %s). "
-                                "Holes=%d overlaps=%d",
-                                loc->path, gfid, holes, overlaps );
-                }
-                ret = -1;
-        }
-
-        if (ret >= 0) {
-                missing_dirs = dht_layout_missing_dirs (layout);
-                /* TODO During DHT selfheal rewrite (almost) find a better place
-                 * to detect this - probably in dht_layout_anomalies()
-                 */
-                if (missing_dirs > 0)
-                        ret += missing_dirs;
-        }
+    int ret = 0;
+    uint32_t holes = 0;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0, missing_dirs = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    ret = dht_layout_sort(layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SORT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    dht_layout_anomalies(this, loc, layout, &holes, &overlaps, &missing, &down,
+                         &misc, NULL);
+
+    if (holes || overlaps) {
+        if (missing == layout->cnt) {
+            gf_msg_debug(this->name, 0,
+                         "Directory %s looked up first time"
+                         " gfid = %s",
+                         loc->path, gfid);
+        } else {
+            gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_ANOMALIES_INFO,
+                    "path=%s", loc->path, "gfid=%s", gfid, "holes=%d", holes,
+                    "overlaps=%d", overlaps, NULL);
+        }
+        ret = -1;
+    }
+
+    if (ret >= 0) {
+        missing_dirs = dht_layout_missing_dirs(layout);
+        /* TODO During DHT selfheal rewrite (almost) find a better place
+         * to detect this - probably in dht_layout_anomalies()
+         */
+        if (missing_dirs > 0)
+            ret += missing_dirs;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_dir_has_layout (dict_t *xattr, char *name)
+dht_dir_has_layout(dict_t *xattr, char *name)
 {
+    void *disk_layout_raw = NULL;
 
-        void     *disk_layout_raw = NULL;
-
-        return dict_get_ptr (xattr, name, &disk_layout_raw);
+    return dict_get_ptr(xattr, name, &disk_layout_raw);
 }
 
 int
-dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                         loc_t *loc, dict_t *xattr)
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr)
 {
-        int         idx = 0;
-        int         pos = -1;
-        int         ret = 0;
-        int         err = 0;
-        int         dict_ret = 0;
-        int32_t     disk_layout[4];
-        void       *disk_layout_raw = NULL;
-        uint32_t    start_off = -1;
-        uint32_t    stop_off = -1;
-        uint32_t    commit_hash = -1;
-        dht_conf_t *conf = this->private;
-        char        gfid[GF_UUID_BUF_SIZE] = {0};
-
-        if(loc && loc->inode)
-                gf_uuid_unparse(loc->inode->gfid, gfid);
-
-        for (idx = 0; idx < layout->cnt; idx++) {
-                if (layout->list[idx].xlator == subvol) {
-                        pos = idx;
-                        break;
-                }
-        }
-
-        if (pos == -1) {
-                if (loc) {
-                        gf_msg_debug (this->name, 0,
-                                      "%s - no layout info for subvolume %s",
-                                      loc ? loc->path : "path not found",
-                                      subvol->name);
-                }
-                ret = 1;
-                goto out;
-        }
-
-        err = layout->list[pos].err;
-
-        if (!xattr) {
-                if (err == 0) {
-                        if (loc) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DICT_GET_FAILED,
-                                        "%s: xattr dictionary is NULL",
-                                        loc->path);
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DICT_GET_FAILED,
-                                        "path not found: "
-                                        "xattr dictionary is NULL");
-                        }
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
-                                 &disk_layout_raw);
-
-        if (dict_ret < 0) {
-                if (err == 0 && layout->list[pos].stop) {
-                        if (loc) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DISK_LAYOUT_MISSING,
-                                    "%s: Disk layout missing, gfid = %s",
-                                    loc->path, gfid);
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DISK_LAYOUT_MISSING,
-                                        "path not found: "
-                                        "Disk layout missing, gfid = %s",
-                                        gfid);
-                        }
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        memcpy (disk_layout, disk_layout_raw, sizeof (disk_layout));
-
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
-        commit_hash = ntoh32 (disk_layout[0]);
-
-        if ((layout->list[pos].start != start_off)
-            || (layout->list[pos].stop != stop_off)
-            || (layout->list[pos].commit_hash != commit_hash)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LAYOUT_INFO,
-                        "subvol: %s; inode layout - %"PRIu32" - %"PRIu32
-                        " - %"PRIu32"; "
-                        "disk layout - %"PRIu32" - %"PRIu32" - %"PRIu32,
-                        layout->list[pos].xlator->name,
-                        layout->list[pos].start, layout->list[pos].stop,
-                        layout->list[pos].commit_hash,
-                        start_off, stop_off, commit_hash);
-                ret = 1;
-        } else {
-                ret = 0;
-        }
+    int idx = 0;
+    int pos = -1;
+    int ret = 0;
+    int err = 0;
+    int dict_ret = 0;
+    int32_t disk_layout[4];
+    void *disk_layout_raw = NULL;
+    uint32_t start_off = -1;
+    uint32_t stop_off = -1;
+    uint32_t commit_hash = -1;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    if (loc && loc->inode)
+        gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    for (idx = 0; idx < layout->cnt; idx++) {
+        if (layout->list[idx].xlator == subvol) {
+            pos = idx;
+            break;
+        }
+    }
+
+    if (pos == -1) {
+        if (loc) {
+            gf_msg_debug(this->name, 0, "%s - no layout info for subvolume %s",
+                         loc ? loc->path : "path not found", subvol->name);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    err = layout->list[pos].err;
+
+    if (!xattr) {
+        if (err == 0) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path=%s", loc->path, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path not found", NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    dict_ret = dict_get_ptr(xattr, conf->xattr_name, &disk_layout_raw);
+
+    if (dict_ret < 0) {
+        if (err == 0 && layout->list[pos].stop) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path=%s", loc->path, "gfid=%s", gfid, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path not found"
+                        "gfid=%s",
+                        gfid, NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    memcpy(disk_layout, disk_layout_raw, sizeof(disk_layout));
+
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
+    commit_hash = ntoh32(disk_layout[0]);
+
+    if ((layout->list[pos].start != start_off) ||
+        (layout->list[pos].stop != stop_off) ||
+        (layout->list[pos].commit_hash != commit_hash)) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_INFO, "subvol=%s",
+                layout->list[pos].xlator->name, "inode-layout:start=0x%x",
+                layout->list[pos].start, "inode-layout:stop=0x%x",
+                layout->list[pos].stop, "layout-commit-hash=0x%x; ",
+                layout->list[pos].commit_hash, "disk-layout:start-off=0x%x",
+                start_off, "disk-layout:top-off=0x%x", stop_off,
+                "commit-hash=0x%x", commit_hash, NULL);
+        ret = 1;
+    } else {
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode)
 {
-        dht_layout_t *layout = NULL;
-        int           ret = -1;
-        dht_conf_t   *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        layout = dht_layout_for_subvol (this, subvol);
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_NO_LAYOUT_INFO,
-                        "no pre-set layout for subvolume %s",
-                        subvol ? subvol->name : "<nil>");
-                ret = -1;
-                goto out;
-        }
+    layout = dht_layout_for_subvol(this, subvol);
+    if (!layout) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_NO_LAYOUT_INFO,
+                "subvolume=%s", subvol ? subvol->name : "<nil>", NULL);
+        ret = -1;
+        goto out;
+    }
 
-        LOCK (&conf->layout_lock);
-        {
-                dht_inode_ctx_layout_set (inode, this, layout);
-        }
-        UNLOCK (&conf->layout_lock);
+    gf_msg_debug(this->name, 0, "file = %s, subvol = %s",
+                 uuid_utoa(inode->gfid), subvol ? subvol->name : "<nil>");
 
-        ret = 0;
+    LOCK(&conf->layout_lock);
+    {
+        dht_inode_ctx_layout_set(inode, this, layout);
+    }
+
+    UNLOCK(&conf->layout_lock);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_layout_index_for_subvol (dht_layout_t *layout, xlator_t *subvol)
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol)
 {
-        int i = 0, ret = -1;
+    int i = 0, ret = -1;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol) {
-                        ret = i;
-                        break;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            ret = i;
+            break;
         }
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index deba2138672..89ec6cca56e 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -8,353 +8,321 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "compat.h"
+#include <glusterfs/compat.h>
 #include "dht-common.h"
-#include "dht-messages.h"
 
-int
-dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+static int
+dht_linkfile_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        char          is_linkfile   = 0;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        call_frame_t *prev          = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-
-        if (op_ret)
-                goto out;
-
-        gf_uuid_unparse(local->loc.gfid, gfid);
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        if (!is_linkfile)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_NOT_LINK_FILE_ERROR,
-                        "got non-linkfile %s:%s, gfid = %s",
-                        prev->this->name, local->loc.path, gfid);
+    char is_linkfile = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret)
+        goto out;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    if (!is_linkfile)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+                "name=%s", prev->name, "path=%s", local->loc.path, "gfid=%s",
+                gfid, NULL);
 out:
-        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, postparent, postparent,
-                                      xattr);
-        return 0;
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, postparent, postparent, xattr);
+    return 0;
 }
 
-#define is_equal(a, b) ((a) == (b))
-int
-dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+static int
+dht_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *subvol = NULL;
-        call_frame_t *prev = NULL;
-        dict_t       *xattrs = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-
-        local = frame->local;
-
-        if (!op_ret)
-                local->linked = _gf_true;
-
-        FRAME_SU_UNDO (frame, dht_local_t);
-
-        if (op_ret && (op_errno == EEXIST)) {
-                conf = this->private;
-                prev = cookie;
-                subvol = prev->this;
-                if (!subvol)
-                        goto out;
-                xattrs = dict_new ();
-                if (!xattrs)
-                        goto out;
-                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value. key : %s",
-                                conf->link_xattr_name);
-                        goto out;
-                }
-
-                STACK_WIND (frame, dht_linkfile_lookup_cbk, subvol,
-                            subvol->fops->lookup, &local->loc, xattrs);
-                if (xattrs)
-                        dict_unref (xattrs);
-                return 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+
+    local = frame->local;
+
+    if (!op_ret)
+        local->linked = _gf_true;
+
+    FRAME_SU_UNDO(frame, dht_local_t);
+
+    if (op_ret && (op_errno == EEXIST)) {
+        conf = this->private;
+        subvol = cookie;
+        if (!subvol)
+            goto out;
+        xattrs = dict_new();
+        if (!xattrs)
+            goto out;
+        ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "mame=%s", conf->link_xattr_name, NULL);
+            goto out;
         }
-out:
-        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, preparent, postparent,
-                                      xdata);
+
+        STACK_WIND_COOKIE(frame, dht_linkfile_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->linkfile.loc, xattrs);
         if (xattrs)
-                dict_unref (xattrs);
+            dict_unref(xattrs);
         return 0;
+    }
+out:
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, preparent, postparent, xdata);
+    if (xattrs)
+        dict_unref(xattrs);
+    return 0;
 }
 
-
 int
-dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-                     xlator_t *this,
-                     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc)
 {
-        dht_local_t *local = NULL;
-        dict_t      *dict = NULL;
-        int          need_unref = 0;
-        int          ret = 0;
-        dht_conf_t  *conf = this->private;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        local->linkfile.linkfile_cbk = linkfile_cbk;
-        local->linkfile.srcvol = tovol;
-
-        local->linked = _gf_false;
-
-        dict = local->params;
-        if (!dict) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-                need_unref = 1;
-        }
-
-
-        if (!gf_uuid_is_null (local->gfid)) {
-                gf_uuid_unparse(local->gfid, gfid);
-
-                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
-                if (ret)
-                        gf_msg ("dht-linkfile", GF_LOG_INFO, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value: "
-                                "key = gfid-req, gfid = %s ", loc->path, gfid);
-        } else {
-                gf_uuid_unparse(loc->gfid, gfid);
-        }
-
-        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    int need_unref = 0;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    local->linkfile.linkfile_cbk = linkfile_cbk;
+    local->linkfile.srcvol = tovol;
+    loc_copy(&local->linkfile.loc, loc);
+
+    local->linked = _gf_false;
+
+    dict = local->params;
+    if (!dict) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        need_unref = 1;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid);
+
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
         if (ret)
-                gf_msg ("dht-linkfile", GF_LOG_INFO, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value: key = %s,"
-                        " gfid = %s", loc->path,
-                        GLUSTERFS_INTERNAL_FOP_KEY, gfid);
-
-        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
-
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_CREATE_LINK_FAILED,
-                        "%s: failed to initialize linkfile data, gfid = %s",
-                        loc->path, gfid);
-                goto out;
-        }
-
-        local->link_subvol = fromvol;
-        /* Always create as root:root. dht_linkfile_attr_heal fixes the
-         * ownsership */
-        FRAME_SU_DO (frame, dht_local_t);
-        STACK_WIND (frame, dht_linkfile_create_cbk,
-                    fromvol, fromvol->fops->mknod, loc,
-                    S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
-
-        if (need_unref && dict)
-                dict_unref (dict);
-
-        return 0;
+            gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "gfid=%s", gfid, NULL);
+    } else {
+        gf_uuid_unparse(loc->gfid, gfid);
+    }
+
+    ret = dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret)
+        gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+
+    ret = dict_set_str(dict, conf->link_xattr_name, tovol->name);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, DHT_MSG_CREATE_LINK_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    local->link_subvol = fromvol;
+    /* Always create as root:root. dht_linkfile_attr_heal fixes the
+     * ownsership */
+    FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_linkfile_create_cbk, fromvol, fromvol,
+                      fromvol->fops->mknod, loc, S_IFREG | DHT_LINKFILE_MODE, 0,
+                      0, dict);
+
+    if (need_unref && dict)
+        dict_unref(dict);
+
+    return 0;
 out:
-        local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
-                                      loc->inode, NULL, NULL, NULL, NULL);
+    local->linkfile.linkfile_cbk(frame, frame->this, frame->this, -1, ENOMEM,
+                                 loc->inode, NULL, NULL, NULL, NULL);
 
-        if (need_unref && dict)
-                dict_unref (dict);
+    if (need_unref && dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent,
-                         dict_t *xdata)
+dht_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        call_frame_t  *prev = NULL;
-        xlator_t      *subvol = NULL;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev = cookie;
-        subvol = prev->this;
-
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        if (op_ret == -1) {
+    local = frame->local;
+    subvol = cookie;
 
-                gf_uuid_unparse(local->loc.gfid, gfid);
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_UNLINK_FAILED,
-                        "Unlinking linkfile %s (gfid = %s)on "
-                        "subvolume %s failed ",
-                        local->loc.path, gfid, subvol->name);
-        }
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_UNLINK_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, "subvolume=%s",
+                subvol->name, NULL);
+    }
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-                     xlator_t *subvol, loc_t *loc)
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc)
 {
-        call_frame_t *unlink_frame = NULL;
-        dht_local_t  *unlink_local = NULL;
+    call_frame_t *unlink_frame = NULL;
+    dht_local_t *unlink_local = NULL;
 
-        unlink_frame = copy_frame (frame);
-        if (!unlink_frame) {
-                goto err;
-        }
+    unlink_frame = copy_frame(frame);
+    if (!unlink_frame) {
+        goto err;
+    }
 
-        /* Using non-fop value here, as anyways, 'local->fop' is not used in
-           this particular case */
-        unlink_local = dht_local_init (unlink_frame, loc, NULL,
-                                       GF_FOP_MAXVALUE);
-        if (!unlink_local) {
-                goto err;
-        }
+    /* Using non-fop value here, as anyways, 'local->fop' is not used in
+       this particular case */
+    unlink_local = dht_local_init(unlink_frame, loc, NULL, GF_FOP_MAXVALUE);
+    if (!unlink_local) {
+        goto err;
+    }
 
-        STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
-                    subvol, subvol->fops->unlink,
-                    &unlink_local->loc, 0, NULL);
+    STACK_WIND_COOKIE(unlink_frame, dht_linkfile_unlink_cbk, subvol, subvol,
+                      subvol->fops->unlink, &unlink_local->loc, 0, NULL);
 
-        return 0;
+    return 0;
 err:
-        if (unlink_frame)
-                DHT_STACK_DESTROY (unlink_frame);
+    if (unlink_frame)
+        DHT_STACK_DESTROY(unlink_frame);
 
-        return -1;
+    return -1;
 }
 
-
 xlator_t *
-dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
-                     dict_t *xattr)
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *stbuf,
+                    dict_t *xattr)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *subvol = NULL;
-        void       *volname = NULL;
-        int         i = 0, ret = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    void *volname = NULL;
+    int i = 0, ret = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!xattr)
-                goto out;
+    if (!xattr)
+        goto out;
 
-        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
+    ret = dict_get_ptr(xattr, conf->link_xattr_name, &volname);
 
-        if ((-1 == ret) || !volname)
-                goto out;
+    if ((-1 == ret) || !volname)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
-                        subvol = conf->subvolumes[i];
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (strcmp(conf->subvolumes[i]->name, (char *)volname) == 0) {
+            subvol = conf->subvolumes[i];
+            break;
         }
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
-int
-dht_linkfile_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int op_ret, int op_errno, struct iatt *statpre,
-                          struct iatt *statpost, dict_t *xdata)
+static int
+dht_linkfile_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        loc_t *loc = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
 
-        local = frame->local;
-        loc = &local->loc;
+    local = frame->local;
+    loc = &local->loc;
 
-        if (op_ret)
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_SETATTR_FAILED,
-                        "Failed to set attr uid/gid on %s"
-                        " :<gfid:%s> ",
-                        (loc->path? loc->path: "NULL"),
-                        uuid_utoa(local->gfid));
+    if (op_ret)
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_SETATTR_FAILED,
+                "path=%s", (loc->path ? loc->path : "NULL"), "gfid=%s",
+                uuid_utoa(local->gfid), NULL);
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-       return 0;
+    return 0;
 }
 
 int
-dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this)
 {
-        int           ret         = -1;
-        call_frame_t  *copy       = NULL;
-        dht_local_t   *local      = NULL;
-        dht_local_t   *copy_local = NULL;
-        xlator_t      *subvol     = NULL;
-        struct iatt   stbuf       = {0,};
-        dict_t        *xattr      = NULL;
-
-        local = frame->local;
-
-        GF_VALIDATE_OR_GOTO ("dht", local, out);
-        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
-
-        if (local->stbuf.ia_type == IA_INVAL)
-                return 0;
+    int ret = -1;
+    call_frame_t *copy = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *copy_local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
+
+    if (local->stbuf.ia_type == IA_INVAL)
+        return 0;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        gf_uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
+    gf_uuid_copy(local->loc.gfid, local->stbuf.ia_gfid);
 
-        copy = copy_frame (frame);
+    copy = copy_frame(frame);
 
-        if (!copy)
-                goto out;
+    if (!copy)
+        goto out;
 
-       copy_local = dht_local_init (copy, &local->loc, NULL, 0);
+    copy_local = dht_local_init(copy, &local->loc, NULL, 0);
 
-       if (!copy_local)
-               goto out;
+    if (!copy_local)
+        goto out;
 
-        stbuf = local->stbuf;
-        subvol = local->link_subvol;
+    stbuf = local->stbuf;
+    subvol = local->link_subvol;
 
-        copy->local = copy_local;
+    copy->local = copy_local;
 
-        FRAME_SU_DO (copy, dht_local_t);
+    FRAME_SU_DO(copy, dht_local_t);
 
-        STACK_WIND (copy, dht_linkfile_setattr_cbk, subvol,
-                    subvol->fops->setattr, &copy_local->loc,
-                    &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), xattr);
-        ret = 0;
+    STACK_WIND(copy, dht_linkfile_setattr_cbk, subvol, subvol->fops->setattr,
+               &copy_local->loc, &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+               xattr);
+    ret = 0;
 out:
-        if ((ret < 0) && (copy))
-                DHT_STACK_DESTROY (copy);
+    if ((ret < 0) && (copy))
+        DHT_STACK_DESTROY(copy);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-lock.c b/xlators/cluster/dht/src/dht-lock.c
new file mode 100644
index 00000000000..638821ccee5
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-lock.c
@@ -0,0 +1,1392 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "dht-lock.h"
+
+static char *
+dht_lock_asprintf(dht_lock_t *lock)
+{
+    char *lk_buf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    if (lock == NULL)
+        goto out;
+
+    uuid_utoa_r(lock->loc.gfid, gfid);
+
+    gf_asprintf(&lk_buf, "%s:%s", lock->xl->name, gfid);
+
+out:
+    return lk_buf;
+}
+
+static void
+dht_log_lk_array(char *name, gf_loglevel_t log_level, dht_lock_t **lk_array,
+                 int count)
+{
+    int i = 0;
+    char *lk_buf = NULL;
+
+    if ((lk_array == NULL) || (count == 0))
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lk_buf = dht_lock_asprintf(lk_array[i]);
+        if (!lk_buf)
+            goto out;
+
+        gf_smsg(name, log_level, 0, DHT_MSG_LK_ARRAY_INFO, "index=%d", i,
+                "lk_buf=%s", lk_buf, NULL);
+        GF_FREE(lk_buf);
+    }
+
+out:
+    return;
+}
+
+static void
+dht_lock_stack_destroy(call_frame_t *lock_frame, dht_lock_type_t lk)
+{
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    if (lk == DHT_INODELK) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+    } else {
+        local->lock[0].ns.directory_ns.locks = NULL;
+        local->lock[0].ns.directory_ns.lk_count = 0;
+    }
+
+    DHT_STACK_DESTROY(lock_frame);
+    return;
+}
+
+static void
+dht_lock_free(dht_lock_t *lock)
+{
+    if (lock == NULL)
+        goto out;
+
+    loc_wipe(&lock->loc);
+    GF_FREE(lock->domain);
+    GF_FREE(lock->basename);
+    mem_put(lock);
+
+out:
+    return;
+}
+
+static void
+dht_set_lkowner(dht_lock_t **lk_array, int count, gf_lkowner_t *lkowner)
+{
+    int i = 0;
+
+    if (!lk_array || !lkowner)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lk_array[i]->lk_owner = *lkowner;
+    }
+
+out:
+    return;
+}
+
+static int
+dht_lock_request_cmp(const void *val1, const void *val2)
+{
+    dht_lock_t *lock1 = NULL;
+    dht_lock_t *lock2 = NULL;
+    int ret = -1;
+
+    lock1 = *(dht_lock_t **)val1;
+    lock2 = *(dht_lock_t **)val2;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", lock1, out);
+    GF_VALIDATE_OR_GOTO("dht-locks", lock2, out);
+
+    ret = strcmp(lock1->xl->name, lock2->xl->name);
+
+    if (ret == 0) {
+        ret = gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid);
+    }
+
+out:
+    return ret;
+}
+
+static int
+dht_lock_order_requests(dht_lock_t **locks, int count)
+{
+    int ret = -1;
+
+    if (!locks || !count)
+        goto out;
+
+    qsort(locks, count, sizeof(*locks), dht_lock_request_cmp);
+    ret = 0;
+
+out:
+    return ret;
+}
+
+void
+dht_lock_array_free(dht_lock_t **lk_array, int count)
+{
+    int i = 0;
+    dht_lock_t *lock = NULL;
+
+    if (lk_array == NULL)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lock = lk_array[i];
+        lk_array[i] = NULL;
+        dht_lock_free(lock);
+    }
+
+out:
+    return;
+}
+
+int32_t
+dht_lock_count(dht_lock_t **lk_array, int lk_count)
+{
+    int i = 0, locked = 0;
+
+    if ((lk_array == NULL) || (lk_count == 0))
+        goto out;
+
+    for (i = 0; i < lk_count; i++) {
+        if (lk_array[i]->locked)
+            locked++;
+    }
+out:
+    return locked;
+}
+
+static call_frame_t *
+dht_lock_frame(call_frame_t *parent_frame)
+{
+    call_frame_t *lock_frame = NULL;
+
+    lock_frame = copy_frame(parent_frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    set_lk_owner_from_ptr(&lock_frame->root->lk_owner, parent_frame->root);
+
+out:
+    return lock_frame;
+}
+
+dht_lock_t *
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure)
+{
+    dht_conf_t *conf = NULL;
+    dht_lock_t *lock = NULL;
+
+    conf = this->private;
+
+    lock = mem_get0(conf->lock_pool);
+    if (lock == NULL)
+        goto out;
+
+    lock->xl = xl;
+    lock->type = type;
+    lock->do_on_failure = do_on_failure;
+
+    lock->domain = gf_strdup(domain);
+    if (lock->domain == NULL) {
+        dht_lock_free(lock);
+        lock = NULL;
+        goto out;
+    }
+
+    if (basename) {
+        lock->basename = gf_strdup(basename);
+        if (lock->basename == NULL) {
+            dht_lock_free(lock);
+            lock = NULL;
+            goto out;
+        }
+    }
+
+    /* Fill only inode and gfid.
+       posix and protocol/server give preference to pargfid/basename over
+       gfid/inode for resolution if all the three parameters of loc_t are
+       present. I want to avoid the following hypothetical situation:
+
+       1. rebalance did a lookup on a dentry and got a gfid.
+       2. rebalance acquires lock on loc_t which was filled with gfid and
+          path (pargfid/bname) from step 1.
+       3. somebody deleted and recreated the same file
+       4. rename on the same path acquires lock on loc_t which now points
+          to a different inode (and hence gets the lock).
+       5. rebalance continues to migrate file (note that not all fops done
+          by rebalance during migration are inode/gfid based Eg., unlink)
+       6. rename continues.
+    */
+    lock->loc.inode = inode_ref(loc->inode);
+    loc_gfid(loc, lock->loc.gfid);
+
+out:
+    return lock;
+}
+
+static int
+dht_local_entrylk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_entrylk_cbk_t entrylk_cbk)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
+
+    if (local == NULL) {
+        goto out;
+    }
+
+    local->lock[0].ns.directory_ns.entrylk_cbk = entrylk_cbk;
+    local->lock[0].ns.directory_ns.locks = lk_array;
+    local->lock[0].ns.directory_ns.lk_count = lk_count;
+
+    ret = dht_lock_order_requests(local->lock[0].ns.directory_ns.locks,
+                                  local->lock[0].ns.directory_ns.lk_count);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+dht_entrylk_done(call_frame_t *lock_frame)
+{
+    fop_entrylk_cbk_t entrylk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+    main_frame = local->main_frame;
+
+    local->lock[0].ns.directory_ns.locks = NULL;
+    local->lock[0].ns.directory_ns.lk_count = 0;
+
+    entrylk_cbk = local->lock[0].ns.directory_ns.entrylk_cbk;
+    local->lock[0].ns.directory_ns.entrylk_cbk = NULL;
+
+    entrylk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].ns.directory_ns.op_ret,
+                local->lock[0].ns.directory_ns.op_errno, NULL);
+
+    dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+    return;
+}
+
+static int32_t
+dht_unlock_entrylk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].ns.directory_ns.locks[0]->loc.inode->gfid,
+                    gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "gfid=%s", gfid,
+                "DHT_LAYOUT_HEAL_DOMAIN", NULL);
+    }
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+static int32_t
+dht_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+
+    uuid_utoa_r(local->lock[0].ns.directory_ns.locks[lk_index]->loc.gfid, gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].ns.directory_ns.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = 0;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_entrylk_done(frame);
+    }
+
+    return 0;
+}
+
+static int32_t
+dht_unlock_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_entrylk_cbk_t entrylk_cbk)
+{
+    dht_local_t *local = NULL;
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, done);
+
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    for (i = 0; i < local->lock[0].ns.directory_ns.lk_count; i++) {
+        if (!local->lock[0].ns.directory_ns.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].ns.directory_ns.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_entrylk_cbk, (void *)(long)i,
+            local->lock[0].ns.directory_ns.locks[i]->xl,
+            local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+            local->lock[0].ns.directory_ns.locks[i]->domain,
+            &local->lock[0].ns.directory_ns.locks[i]->loc,
+            local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_UNLOCK,
+            ENTRYLK_WRLCK, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+
+done:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+
+    /* no locks acquired, invoke entrylk_cbk */
+    if (ret == 0)
+        entrylk_cbk(frame, NULL, frame->this, 0, 0, NULL);
+
+    return ret;
+}
+
+int32_t
+dht_unlock_entrylk_wrapper(call_frame_t *frame, dht_elock_wrap_t *entrylk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    local = frame->local;
+
+    if (!entrylk || !entrylk->locks)
+        goto out;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_frame->local = lock_local;
+
+    lock_local->lock[0].ns.directory_ns.locks = entrylk->locks;
+    lock_local->lock[0].ns.directory_ns.lk_count = entrylk->lk_count;
+    entrylk->locks = NULL;
+    entrylk->lk_count = 0;
+
+    ret = dht_unlock_entrylk(
+        lock_frame, lock_local->lock[0].ns.directory_ns.locks,
+        lock_local->lock[0].ns.directory_ns.lk_count, dht_unlock_entrylk_done);
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+out:
+    return 0;
+}
+
+static int
+dht_entrylk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_entrylk_done(frame);
+    return 0;
+}
+
+static void
+dht_entrylk_cleanup(call_frame_t *lock_frame)
+{
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    lk_array = local->lock[0].ns.directory_ns.locks;
+    lk_count = local->lock[0].ns.directory_ns.lk_count;
+
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_entrylk(lock_frame, lk_array, lk_count,
+                           dht_entrylk_cleanup_cbk);
+    } else {
+        dht_entrylk_done(lock_frame);
+    }
+
+    return;
+}
+
+static int32_t
+dht_blocking_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                if (local->lock[0]
+                        .ns.directory_ns.locks[lk_index]
+                        ->do_on_failure != IGNORE_ENOENT_ESTALE) {
+                    local->lock[0].ns.directory_ns.op_ret = -1;
+                    local->lock[0].ns.directory_ns.op_errno = op_errno;
+                    goto cleanup;
+                }
+                break;
+            default:
+                local->lock[0].ns.directory_ns.op_ret = -1;
+                local->lock[0].ns.directory_ns.op_errno = op_errno;
+                goto cleanup;
+        }
+    }
+
+    if (lk_index == (local->lock[0].ns.directory_ns.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].ns.directory_ns.lk_count) &&
+                    (!local->lock[0].ns.directory_ns.locks[i]->locked);
+             i++)
+            ;
+
+        if (i == local->lock[0].ns.directory_ns.lk_count) {
+            local->lock[0].ns.directory_ns.op_ret = -1;
+            local->lock[0].ns.directory_ns.op_errno = op_errno;
+        }
+
+        dht_entrylk_done(frame);
+    } else {
+        dht_blocking_entrylk_rec(frame, ++lk_index);
+    }
+
+    return 0;
+
+cleanup:
+    dht_entrylk_cleanup(frame);
+
+    return 0;
+}
+
+void
+dht_blocking_entrylk_rec(call_frame_t *frame, int i)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_entrylk_cbk, (void *)(long)i,
+        local->lock[0].ns.directory_ns.locks[i]->xl,
+        local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+        local->lock[0].ns.directory_ns.locks[i]->domain,
+        &local->lock[0].ns.directory_ns.locks[i]->loc,
+        local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_LOCK,
+        ENTRYLK_WRLCK, NULL);
+
+    return;
+}
+
+int
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_entrylk_cbk_t entrylk_cbk)
+{
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, out);
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    dht_blocking_entrylk_rec(lock_frame, 0);
+
+    return 0;
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+
+    return -1;
+}
+
+static int
+dht_local_inodelk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_inodelk_cbk_t inodelk_cbk)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
+
+    if (local == NULL) {
+        goto out;
+    }
+
+    local->lock[0].layout.my_layout.inodelk_cbk = inodelk_cbk;
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = lk_count;
+
+    ret = dht_lock_order_requests(local->lock[0].layout.my_layout.locks,
+                                  local->lock[0].layout.my_layout.lk_count);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+dht_inodelk_done(call_frame_t *lock_frame)
+{
+    fop_inodelk_cbk_t inodelk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+    main_frame = local->main_frame;
+
+    local->lock[0].layout.my_layout.locks = NULL;
+    local->lock[0].layout.my_layout.lk_count = 0;
+
+    inodelk_cbk = local->lock[0].layout.my_layout.inodelk_cbk;
+    local->lock[0].layout.my_layout.inodelk_cbk = NULL;
+
+    inodelk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].layout.my_layout.op_ret,
+                local->lock[0].layout.my_layout.op_errno, NULL);
+
+    dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+    return;
+}
+
+static int32_t
+dht_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        uuid_utoa_r(local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = 0;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
+}
+
+static int32_t
+dht_unlock_inodelk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].layout.my_layout.locks[0]->loc.inode->gfid,
+                    gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "DHT_LAYOUT_HEAL_DOMAIN gfid=%s",
+                gfid, NULL);
+    }
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+int32_t
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk)
+{
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, done);
+
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    flock.l_type = F_UNLCK;
+
+    for (i = 0; i < local->lock[0].layout.my_layout.lk_count; i++) {
+        if (!local->lock[0].layout.my_layout.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].layout.my_layout.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+
+done:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    /* no locks acquired, invoke inodelk_cbk */
+    if (ret == 0)
+        inodelk_cbk(frame, NULL, frame->this, 0, 0, NULL);
+
+    return ret;
+}
+
+int32_t
+dht_unlock_inodelk_wrapper(call_frame_t *frame, dht_ilock_wrap_t *inodelk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    local = frame->local;
+
+    if (!inodelk || !inodelk->locks)
+        goto out;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "gfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_frame->local = lock_local;
+
+    lock_local->lock[0].layout.my_layout.locks = inodelk->locks;
+    lock_local->lock[0].layout.my_layout.lk_count = inodelk->lk_count;
+    inodelk->locks = NULL;
+    inodelk->lk_count = 0;
+
+    ret = dht_unlock_inodelk(
+        lock_frame, lock_local->lock[0].layout.my_layout.locks,
+        lock_local->lock[0].layout.my_layout.lk_count, dht_unlock_inodelk_done);
+
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+out:
+    return 0;
+}
+
+static int
+dht_inodelk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_inodelk_done(frame);
+    return 0;
+}
+
+static void
+dht_inodelk_cleanup(call_frame_t *lock_frame)
+{
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    lk_array = local->lock[0].layout.my_layout.locks;
+    lk_count = local->lock[0].layout.my_layout.lk_count;
+
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_inodelk(lock_frame, lk_array, lk_count,
+                           dht_inodelk_cleanup_cbk);
+    } else {
+        dht_inodelk_done(lock_frame);
+    }
+
+    return;
+}
+
+static int32_t
+dht_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    lk_index = (long)cookie;
+
+    if (op_ret == -1) {
+        local->lock[0].layout.my_layout.op_ret = -1;
+        local->lock[0].layout.my_layout.op_errno = op_errno;
+
+        if (local && local->lock[0].layout.my_layout.locks[lk_index]) {
+            uuid_utoa_r(local->lock[0]
+                            .layout.my_layout.locks[lk_index]
+                            ->loc.inode->gfid,
+                        gfid);
+
+            gf_msg_debug(
+                this->name, op_errno,
+                "inodelk failed on gfid: %s "
+                "subvolume: %s",
+                gfid,
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name);
+        }
+
+        goto out;
+    }
+
+    local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+
+out:
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->lock[0].layout.my_layout.op_ret < 0) {
+            dht_inodelk_cleanup(frame);
+            return 0;
+        }
+
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
+}
+
+int
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0, ret = 0;
+    dht_local_t *local = NULL;
+    call_frame_t *lock_frame = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    local->call_cnt = lk_count;
+
+    for (i = 0; i < lk_count; i++) {
+        flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+
+        STACK_WIND_COOKIE(
+            lock_frame, dht_nonblocking_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+    }
+
+    return 0;
+
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    return -1;
+}
+
+static int32_t
+dht_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    dht_reaction_type_t reaction = 0;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if ((reaction != IGNORE_ENOENT_ESTALE) &&
+                    (reaction != IGNORE_ENOENT_ESTALE_EIO)) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
+                }
+                break;
+            case EIO:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if (reaction != IGNORE_ENOENT_ESTALE_EIO) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
+                }
+                break;
+
+            default:
+                gf_uuid_unparse(
+                    local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+                local->lock[0].layout.my_layout.op_ret = -1;
+                local->lock[0].layout.my_layout.op_errno = op_errno;
+                gf_smsg(
+                    this->name, GF_LOG_ERROR, op_errno, DHT_MSG_INODELK_FAILED,
+                    "subvol=%s",
+                    local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                    "gfid=%s", gfid, NULL);
+                goto cleanup;
+        }
+    }
+
+    if (lk_index == (local->lock[0].layout.my_layout.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].layout.my_layout.lk_count) &&
+                    (!local->lock[0].layout.my_layout.locks[i]->locked);
+             i++)
+            ;
+
+        if (i == local->lock[0].layout.my_layout.lk_count) {
+            local->lock[0].layout.my_layout.op_ret = -1;
+            local->lock[0].layout.my_layout.op_errno = op_errno;
+        }
+
+        dht_inodelk_done(frame);
+    } else {
+        dht_blocking_inodelk_rec(frame, ++lk_index);
+    }
+
+    return 0;
+
+cleanup:
+    dht_inodelk_cleanup(frame);
+
+    return 0;
+}
+
+void
+dht_blocking_inodelk_rec(call_frame_t *frame, int i)
+{
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+
+    flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_inodelk_cbk, (void *)(long)i,
+        local->lock[0].layout.my_layout.locks[i]->xl,
+        local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+        local->lock[0].layout.my_layout.locks[i]->domain,
+        &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLKW, &flock, NULL);
+
+    return;
+}
+
+int
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk)
+{
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *tmp_local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
+
+    tmp_local = frame->local;
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCK_FRAME_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    dht_blocking_inodelk_rec(lock_frame, 0);
+
+    return 0;
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    return -1;
+}
+
+void
+dht_unlock_namespace(call_frame_t *frame, dht_dir_transaction_t *lock)
+{
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lock, out);
+
+    dht_unlock_entrylk_wrapper(frame, &lock->ns.directory_ns);
+    dht_unlock_inodelk_wrapper(frame, &lock->ns.parent_layout);
+
+out:
+    return;
+}
+
+static int32_t
+dht_protect_namespace_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+    if (op_ret != 0)
+        dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
+
+    local->current->ns.ns_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    dht_lock_t **lk_array = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    dht_elock_wrap_t *entrylk = NULL;
+
+    local = frame->local;
+    entrylk = &local->current->ns.directory_ns;
+
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    loc = &entrylk->locks[0]->loc;
+    gf_uuid_unparse(loc->gfid, pgfid);
+
+    local->op_ret = 0;
+    lk_array = entrylk->locks;
+    count = entrylk->lk_count;
+
+    ret = dht_blocking_entrylk(frame, lk_array, count,
+                               dht_protect_namespace_cbk);
+
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_ENTRYLK_FAILED_AFT_INODELK, "fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "basename=%s",
+                entrylk->locks[0]->basename, NULL);
+        goto err;
+    }
+
+    return 0;
+
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
+
+    /* Unlock inodelk. No harm calling unlock twice */
+    dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
+    /* Call ns_cbk. It will take care of unwinding */
+    local->current->ns.ns_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                              NULL);
+    return 0;
+}
+
+/* Given the loc and the subvol, this routine takes the inodelk on
+ * the parent inode and entrylk on (parent, loc->name). This routine
+ * is specific as it supports only one subvol on which it takes inodelk
+ * and then entrylk serially.
+ */
+int
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk)
+{
+    dht_ilock_wrap_t *inodelk = NULL;
+    dht_elock_wrap_t *entrylk = NULL;
+    dht_lock_t **lk_array = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    loc_t parent = {
+        0,
+    };
+    int ret = -1;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = 0;
+    int count = 1;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, subvol, out);
+
+    local = frame->local;
+    this = frame->this;
+
+    inodelk = &ns->parent_layout;
+    entrylk = &ns->directory_ns;
+
+    /* Initialize entrylk_cbk and parent loc */
+    ns->ns_cbk = ns_cbk;
+
+    ret = dht_build_parent_loc(this, &parent, loc, &op_errno);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_LOC_FAILED,
+                "gfid=%s", loc->gfid, "name=%s", loc->name, "path=%s",
+                loc->path, NULL);
+        goto out;
+    }
+    gf_uuid_unparse(parent.gfid, pgfid);
+
+    /* Alloc inodelk */
+    inodelk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (inodelk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+        goto out;
+    }
+
+    inodelk->locks[0] = dht_lock_new(this, subvol, &parent, F_RDLCK,
+                                     DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                                     FAIL_ON_ANY_ERROR);
+    if (inodelk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "inodelk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+        goto err;
+    }
+    inodelk->lk_count = count;
+
+    /* Allock entrylk */
+    entrylk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (entrylk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+
+    entrylk->locks[0] = dht_lock_new(this, subvol, &parent, F_WRLCK,
+                                     DHT_ENTRY_SYNC_DOMAIN, loc->name,
+                                     FAIL_ON_ANY_ERROR);
+    if (entrylk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+    entrylk->lk_count = count;
+
+    /* Take read inodelk on parent. If it is successful, take write entrylk
+     * on name in cbk.
+     */
+    lk_array = inodelk->locks;
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_blocking_entrylk_after_inodelk);
+    if (ret < 0) {
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_BLOCK_INODELK_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+
+        goto err;
+    }
+
+    loc_wipe(&parent);
+
+    return 0;
+err:
+    if (entrylk->locks != NULL) {
+        dht_lock_array_free(entrylk->locks, count);
+        GF_FREE(entrylk->locks);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
+
+    if (inodelk->locks != NULL) {
+        dht_lock_array_free(inodelk->locks, count);
+        GF_FREE(inodelk->locks);
+        inodelk->locks = NULL;
+        inodelk->lk_count = 0;
+    }
+
+    loc_wipe(&parent);
+out:
+    return -1;
+}
diff --git a/xlators/cluster/dht/src/dht-lock.h b/xlators/cluster/dht/src/dht-lock.h
new file mode 100644
index 00000000000..6485c03fb6e
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-lock.h
@@ -0,0 +1,91 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _DHT_LOCK_H
+#define _DHT_LOCK_H
+
+#include "dht-common.h"
+
+void
+dht_lock_array_free(dht_lock_t **lk_array, int count);
+
+int32_t
+dht_lock_count(dht_lock_t **lk_array, int lk_count);
+
+dht_lock_t *
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure);
+
+int32_t
+dht_unlock_entrylk_wrapper(call_frame_t *, dht_elock_wrap_t *);
+
+void
+dht_blocking_entrylk_rec(call_frame_t *frame, int i);
+
+int
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t entrylk_cbk);
+
+int32_t
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk);
+
+int32_t
+dht_unlock_inodelk_wrapper(call_frame_t *, dht_ilock_wrap_t *);
+
+/* Acquire non-blocking inodelk on a list of xlators.
+ *
+ * @lk_array: array of lock requests lock on.
+ *
+ * @lk_count: number of locks in @lk_array
+ *
+ * @inodelk_cbk: will be called after inodelk replies are received
+ *
+ * @retval: -1 if stack_winding inodelk fails. 0 otherwise.
+ *          inodelk_cbk is called with appropriate error on errors.
+ *          On failure to acquire lock on all members of list, successful
+ *          locks are unlocked before invoking cbk.
+ */
+
+int
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+
+void
+dht_blocking_inodelk_rec(call_frame_t *frame, int i);
+
+/* same as dht_nonblocking_inodelk, but issues sequential blocking locks on
+ * @lk_array directly. locks are issued on some order which remains same
+ * for a list of xlators (irrespective of order of xlators within list).
+ */
+
+int
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk);
+
+int32_t
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata);
+
+int32_t
+dht_blocking_entrylk_after_inodelk_rename(call_frame_t *frame, void *cookie,
+                                          xlator_t *this, int32_t op_ret,
+                                          int32_t op_errno, dict_t *xdata);
+
+void
+dht_unlock_namespace(call_frame_t *, dht_dir_transaction_t *);
+
+int
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk);
+
+#endif /* _DHT_LOCK_H */
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index 5de5d1838ad..e3c4471334a 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -8,36 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __DHT_MEM_TYPES_H__
 #define __DHT_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_dht_mem_types_ {
-        gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
-        gf_dht_mt_dht_conf_t,
-        gf_dht_mt_char,
-        gf_dht_mt_int32_t,
-        gf_dht_mt_xlator_t,
-        gf_dht_mt_dht_layout_t,
-        gf_switch_mt_dht_conf_t,
-        gf_switch_mt_dht_du_t,
-        gf_switch_mt_switch_sched_array,
-        gf_switch_mt_switch_struct,
-        gf_dht_mt_subvol_time,
-        gf_dht_mt_loc_t,
-        gf_defrag_info_mt,
-        gf_dht_mt_inode_ctx_t,
-        gf_dht_mt_ctx_stat_time_t,
-        gf_dht_mt_dirent_t,
-        gf_dht_mt_container_t,
-        gf_dht_mt_octx_t,
-        gf_dht_mt_miginfo_t,
-        gf_tier_mt_bricklist_t,
-        gf_tier_mt_ipc_ctr_params_t,
-        gf_dht_mt_fd_ctx_t,
-        gf_tier_mt_qfile_array_t,
-        gf_dht_mt_end
+    gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
+    gf_dht_mt_dht_conf_t,
+    gf_dht_mt_char,
+    gf_dht_mt_int32_t,
+    gf_dht_mt_xlator_t,
+    gf_dht_mt_dht_layout_t,
+    gf_switch_mt_switch_sched_array,
+    gf_switch_mt_switch_struct,
+    gf_dht_mt_subvol_time,
+    gf_dht_mt_loc_t,
+    gf_defrag_info_mt,
+    gf_dht_mt_inode_ctx_t,
+    gf_dht_mt_dirent_t,
+    gf_dht_mt_container_t,
+    gf_dht_mt_octx_t,
+    gf_dht_mt_miginfo_t,
+    gf_dht_mt_fd_ctx_t,
+    gf_dht_ret_cache_t,
+    gf_dht_nodeuuids_t,
+    gf_dht_mt_end
 };
 #endif
diff --git a/xlators/cluster/dht/src/dht-messages.h b/xlators/cluster/dht/src/dht-messages.h
index 8c0b9103df1..601f8dad78b 100644
--- a/xlators/cluster/dht/src/dht-messages.h
+++ b/xlators/cluster/dht/src/dht-messages.h
@@ -10,1066 +10,377 @@
 #ifndef _DHT_MESSAGES_H_
 #define _DHT_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    DHT, DHT_MSG_CACHED_SUBVOL_GET_FAILED, DHT_MSG_CREATE_LINK_FAILED,
+    DHT_MSG_DICT_SET_FAILED, DHT_MSG_DIR_ATTR_HEAL_FAILED,
+    DHT_MSG_DIR_SELFHEAL_FAILED, DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+    DHT_MSG_FILE_ON_MULT_SUBVOL, DHT_MSG_FILE_TYPE_MISMATCH,
+    DHT_MSG_GFID_MISMATCH, DHT_MSG_GFID_NULL, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+    DHT_MSG_INIT_FAILED, DHT_MSG_INVALID_CONFIGURATION,
+    DHT_MSG_INVALID_DISK_LAYOUT, DHT_MSG_INVALID_OPTION,
+    DHT_MSG_LAYOUT_FIX_FAILED, DHT_MSG_LAYOUT_MERGE_FAILED,
+    DHT_MSG_LAYOUT_MISMATCH, DHT_MSG_LAYOUT_NULL, DHT_MSG_MIGRATE_DATA_COMPLETE,
+    DHT_MSG_MIGRATE_DATA_FAILED, DHT_MSG_MIGRATE_FILE_COMPLETE,
+    DHT_MSG_MIGRATE_FILE_FAILED, DHT_MSG_NO_MEMORY, DHT_MSG_OPENDIR_FAILED,
+    DHT_MSG_REBALANCE_FAILED, DHT_MSG_REBALANCE_START_FAILED,
+    DHT_MSG_REBALANCE_STATUS, DHT_MSG_REBALANCE_STOPPED, DHT_MSG_RENAME_FAILED,
+    DHT_MSG_SETATTR_FAILED, DHT_MSG_SUBVOL_INSUFF_INODES,
+    DHT_MSG_SUBVOL_INSUFF_SPACE, DHT_MSG_UNLINK_FAILED,
+    DHT_MSG_LAYOUT_SET_FAILED, DHT_MSG_LOG_FIXED_LAYOUT,
+    DHT_MSG_GET_XATTR_FAILED, DHT_MSG_FILE_LOOKUP_FAILED,
+    DHT_MSG_OPEN_FD_FAILED, DHT_MSG_SET_INODE_CTX_FAILED,
+    DHT_MSG_UNLOCKING_FAILED, DHT_MSG_DISK_LAYOUT_NULL, DHT_MSG_SUBVOL_INFO,
+    DHT_MSG_CHUNK_SIZE_INFO, DHT_MSG_LAYOUT_FORM_FAILED, DHT_MSG_SUBVOL_ERROR,
+    DHT_MSG_LAYOUT_SORT_FAILED, DHT_MSG_REGEX_INFO, DHT_MSG_FOPEN_FAILED,
+    DHT_MSG_SET_HOSTNAME_FAILED, DHT_MSG_BRICK_ERROR, DHT_MSG_SYNCOP_FAILED,
+    DHT_MSG_MIGRATE_INFO, DHT_MSG_SOCKET_ERROR, DHT_MSG_CREATE_FD_FAILED,
+    DHT_MSG_READDIR_ERROR, DHT_MSG_CHILD_LOC_BUILD_FAILED,
+    DHT_MSG_SET_SWITCH_PATTERN_ERROR, DHT_MSG_COMPUTE_HASH_FAILED,
+    DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR, DHT_MSG_ANOMALIES_INFO,
+    DHT_MSG_LAYOUT_INFO, DHT_MSG_INODE_LK_ERROR, DHT_MSG_RENAME_INFO,
+    DHT_MSG_DATA_NULL, DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+    DHT_MSG_UNLINK_LOOKUP_INFO, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+    DHT_MSG_OPERATION_NOT_SUP, DHT_MSG_NOT_LINK_FILE_ERROR, DHT_MSG_CHILD_DOWN,
+    DHT_MSG_UUID_PARSE_ERROR, DHT_MSG_GET_DISK_INFO_ERROR,
+    DHT_MSG_INVALID_VALUE, DHT_MSG_SWITCH_PATTERN_INFO,
+    DHT_MSG_SUBVOL_OP_FAILED, DHT_MSG_LAYOUT_PRESET_FAILED,
+    DHT_MSG_INVALID_LINKFILE, DHT_MSG_FIX_LAYOUT_INFO,
+    DHT_MSG_GET_HOSTNAME_FAILED, DHT_MSG_WRITE_FAILED,
+    DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED, DHT_MSG_FSYNC_FAILED,
+    DHT_MSG_SUBVOL_DECOMMISSION_INFO, DHT_MSG_BRICK_QUERY_FAILED,
+    DHT_MSG_SUBVOL_NO_LAYOUT_INFO, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+    DHT_MSG_SUBVOL_NOT_FOUND, DHT_MSG_FILE_LOOKUP_ON_DST_FAILED,
+    DHT_MSG_DISK_LAYOUT_MISSING, DHT_MSG_DICT_GET_FAILED,
+    DHT_MSG_REVALIDATE_CBK_INFO, DHT_MSG_UPGRADE_BRICKS, DHT_MSG_LK_ARRAY_INFO,
+    DHT_MSG_RENAME_NOT_LOCAL, DHT_MSG_RECONFIGURE_INFO,
+    DHT_MSG_INIT_LOCAL_SUBVOL_FAILED, DHT_MSG_SYS_CALL_GET_TIME_FAILED,
+    DHT_MSG_NO_DISK_USAGE_STATUS, DHT_MSG_SUBVOL_DOWN_ERROR,
+    DHT_MSG_REBAL_THROTTLE_INFO, DHT_MSG_COMMIT_HASH_INFO,
+    DHT_MSG_REBAL_STRUCT_SET, DHT_MSG_HAS_MIGINFO, DHT_MSG_SETTLE_HASH_FAILED,
+    DHT_MSG_DEFRAG_PROCESS_DIR_FAILED, DHT_MSG_FD_CTX_SET_FAILED,
+    DHT_MSG_STALE_LOOKUP, DHT_MSG_PARENT_LAYOUT_CHANGED,
+    DHT_MSG_LOCK_MIGRATION_FAILED, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+    DHT_MSG_ASPRINTF_FAILED, DHT_MSG_DIR_LOOKUP_FAILED, DHT_MSG_INODELK_FAILED,
+    DHT_MSG_LOCK_FRAME_FAILED, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+    DHT_MSG_ENTRYLK_ERROR, DHT_MSG_INODELK_ERROR, DHT_MSG_LOC_FAILED,
+    DHT_MSG_UNKNOWN_FOP, DHT_MSG_MIGRATE_FILE_SKIPPED,
+    DHT_MSG_DIR_XATTR_HEAL_FAILED, DHT_MSG_HASHED_SUBVOL_DOWN,
+    DHT_MSG_NON_HASHED_SUBVOL_DOWN, DHT_MSG_SYNCTASK_CREATE_FAILED,
+    DHT_MSG_DIR_HEAL_ABORT, DHT_MSG_MIGRATE_SKIP, DHT_MSG_FD_CREATE_FAILED,
+    DHT_MSG_DICT_NEW_FAILED, DHT_MSG_FAILED_TO_OPEN, DHT_MSG_CREATE_FAILED,
+    DHT_MSG_FILE_NOT_EXIST, DHT_MSG_CHOWN_FAILED, DHT_MSG_FALLOCATE_FAILED,
+    DHT_MSG_FTRUNCATE_FAILED, DHT_MSG_STATFS_FAILED, DHT_MSG_WRITE_CROSS,
+    DHT_MSG_NEW_TARGET_FOUND, DHT_MSG_INSUFF_MEMORY, DHT_MSG_SET_XATTR_FAILED,
+    DHT_MSG_SET_MODE_FAILED, DHT_MSG_FILE_EXISTS_IN_DEST,
+    DHT_MSG_SYMLINK_FAILED, DHT_MSG_LINKFILE_DEL_FAILED, DHT_MSG_MKNOD_FAILED,
+    DHT_MSG_MIGRATE_CLEANUP_FAILED, DHT_MSG_LOCK_MIGRATE,
+    DHT_MSG_PARENT_BUILD_FAILED, DHT_MSG_HASHED_SUBVOL_NOT_FOUND,
+    DHT_MSG_ACQUIRE_ENTRYLK_FAILED, DHT_MSG_CREATE_DST_FAILED,
+    DHT_MSG_MIGRATION_EXIT, DHT_MSG_CHANGED_DST, DHT_MSG_TRACE_FAILED,
+    DHT_MSG_WRITE_LOCK_FAILED, DHT_MSG_GETACTIVELK_FAILED, DHT_MSG_STAT_FAILED,
+    DHT_MSG_UNLINK_PERFORM_FAILED, DHT_MSG_CLANUP_SOURCE_FILE_FAILED,
+    DHT_MSG_UNLOCK_FILE_FAILED, DHT_MSG_REMOVE_XATTR_FAILED,
+    DHT_MSG_DATA_MIGRATE_ABORT, DHT_MSG_DEFRAG_NULL, DHT_MSG_PARENT_NULL,
+    DHT_MSG_GFID_NOT_PRESENT, DHT_MSG_CHILD_LOC_FAILED,
+    DHT_MSG_SET_LOOKUP_FAILED, DHT_MSG_DIR_REMOVED, DHT_MSG_FIX_NOT_COMP,
+    DHT_MSG_SUBVOL_DETER_FAILED, DHT_MSG_LOCAL_SUBVOL, DHT_MSG_NODE_UUID,
+    DHT_MSG_SIZE_FILE, DHT_MSG_GET_DATA_SIZE_FAILED,
+    DHT_MSG_PTHREAD_JOIN_FAILED, DHT_MSG_COUNTER_THREAD_CREATE_FAILED,
+    DHT_MSG_MIGRATION_INIT_QUEUE_FAILED, DHT_MSG_PAUSED_TIMEOUT, DHT_MSG_WOKE,
+    DHT_MSG_ABORT_REBALANCE, DHT_MSG_CREATE_TASK_REBAL_FAILED,
+    DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL, DHT_MSG_ADD_CHOICES_ERROR,
+    DHT_MSG_GET_CHOICES_ERROR, DHT_MSG_PREPARE_STATUS_ERROR,
+    DHT_MSG_SET_CHOICE_FAILED, DHT_MSG_SET_HASHED_SUBVOL_FAILED,
+    DHT_MSG_XATTR_HEAL_NOT_POSS, DHT_MSG_LINKTO_FILE_FAILED,
+    DHT_MSG_STALE_LINKFILE_DELETE, DHT_MSG_NO_SUBVOL_FOR_LINKTO,
+    DHT_MSG_SUBVOL_RETURNED, DHT_MSG_UNKNOWN_LOCAL_XSEL, DHT_MSG_GET_XATTR_ERR,
+    DHT_MSG_ALLOC_OR_FILL_FAILED, DHT_MSG_GET_REAL_NAME_FAILED,
+    DHT_MSG_COPY_UUID_FAILED, DHT_MSG_MDS_DETER_FAILED,
+    DHT_MSG_CREATE_REBAL_FAILED, DHT_MSG_LINK_LAYOUT_FAILED,
+    DHT_MSG_NO_SUBVOL_IN_LAYOUT, DHT_MSG_MEM_ALLOC_FAILED,
+    DHT_MSG_SET_IN_PARAMS_DICT_FAILED, DHT_MSG_LOC_COPY_FAILED,
+    DHT_MSG_PARENT_LOC_FAILED, DHT_MSG_CREATE_LOCK_FAILED,
+    DHT_MSG_PREV_ATTEMPT_FAILED, DHT_MSG_REFRESH_ATTEMPT,
+    DHT_MSG_ACQUIRE_LOCK_FAILED, DHT_MSG_CREATE_STUB_FAILED,
+    DHT_MSG_WIND_LOCK_REQ_FAILED, DHT_MSG_REFRESH_FAILED,
+    DHT_MSG_CACHED_SUBVOL_ERROR, DHT_MSG_NO_LINK_SUBVOL, DHT_MSG_SET_KEY_FAILED,
+    DHT_MSG_REMOVE_LINKTO_FAILED, DHT_MSG_LAYOUT_DICT_SET_FAILED,
+    DHT_MSG_XATTR_DICT_NULL, DHT_MSG_DUMMY_ALLOC_FAILED, DHT_MSG_DICT_IS_NULL,
+    DHT_MSG_LINK_INODE_FAILED, DHT_MSG_SELFHEAL_FAILED, DHT_MSG_NO_MDS_SUBVOL,
+    DHT_MSG_LIST_XATTRS_FAILED, DHT_MSG_RESET_INTER_XATTR_FAILED,
+    DHT_MSG_MDS_DOWN_UNABLE_TO_SET, DHT_MSG_WIND_UNLOCK_FAILED,
+    DHT_MSG_COMMIT_HASH_FAILED, DHT_MSG_UNLOCK_GFID_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_ENTRYLK, DHT_MSG_COPY_FRAME_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_LOCKS, DHT_MSG_ENTRYLK_FAILED_AFT_INODELK,
+    DHT_MSG_CALLOC_FAILED, DHT_MSG_LOCK_ALLOC_FAILED,
+    DHT_MSG_BLOCK_INODELK_FAILED,
+    DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+    DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+    DHT_MSG_DST_NULL_SET_FAILED);
+
+#define DHT_MSG_FD_CTX_SET_FAILED_STR "Failed to set fd ctx"
+#define DHT_MSG_INVALID_VALUE_STR "Different dst found in the fd ctx"
+#define DHT_MSG_UNKNOWN_FOP_STR "Unknown FOP on file"
+#define DHT_MSG_OPEN_FD_ON_DST_FAILED_STR "Failed to open the fd on file"
+#define DHT_MSG_SYNCTASK_CREATE_FAILED_STR "Failed to create synctask"
+#define DHT_MSG_ASPRINTF_FAILED_STR                                            \
+    "asprintf failed while fetching subvol from the id"
+#define DHT_MSG_HAS_MIGINFO_STR "Found miginfo in the inode ctx"
+#define DHT_MSG_FILE_LOOKUP_FAILED_STR "failed to lookup the file"
+#define DHT_MSG_INVALID_LINKFILE_STR                                           \
+    "linkto target is different from cached-subvol. treating as destination "  \
+    "subvol"
+#define DHT_MSG_GFID_MISMATCH_STR "gfid different on the target file"
+#define DHT_MSG_GET_XATTR_FAILED_STR "failed to get 'linkto' xattr"
+#define DHT_MSG_SET_INODE_CTX_FAILED_STR "failed to set inode-ctx target file"
+#define DHT_MSG_DIR_SELFHEAL_FAILED_STR "Healing of path failed"
+#define DHT_MSG_DIR_HEAL_ABORT_STR                                             \
+    "Failed to get path from subvol. Aborting directory healing"
+#define DHT_MSG_DIR_XATTR_HEAL_FAILED_STR "xattr heal failed for directory"
+#define DHT_MSG_LOCK_INODE_UNREF_FAILED_STR                                    \
+    "Found a NULL inode. Failed to unref the inode"
+#define DHT_MSG_DICT_SET_FAILED_STR "Failed to set dictionary value"
+#define DHT_MSG_NOT_LINK_FILE_ERROR_STR "got non-linkfile"
+#define DHT_MSG_CREATE_LINK_FAILED_STR "failed to initialize linkfile data"
+#define DHT_MSG_UNLINK_FAILED_STR "Unlinking linkfile on subvolume failed"
+#define DHT_MSG_MIGRATE_FILE_FAILED_STR "Migrate file failed"
+#define DHT_MSG_NO_MEMORY_STR "could not allocate memory for dict"
+#define DHT_MSG_SUBVOL_ERROR_STR "Failed to get linkto subvol"
+#define DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED_STR "link failed on subvol"
+#define DHT_MSG_MIGRATE_FILE_SKIPPED_STR "Migration skipped"
+#define DHT_MSG_FD_CREATE_FAILED_STR "fd create failed"
+#define DHT_MSG_DICT_NEW_FAILED_STR "dict_new failed"
+#define DHT_MSG_FAILED_TO_OPEN_STR "failed to open"
+#define DHT_MSG_CREATE_FAILED_STR "failed to create"
+#define DHT_MSG_FILE_NOT_EXIST_STR "file does not exist"
+#define DHT_MSG_CHOWN_FAILED_STR "chown failed"
+#define DHT_MSG_FALLOCATE_FAILED_STR "fallocate failed"
+#define DHT_MSG_FTRUNCATE_FAILED_STR "ftruncate failed"
+#define DHT_MSG_STATFS_FAILED_STR "failed to get statfs"
+#define DHT_MSG_WRITE_CROSS_STR                                                \
+    "write will cross min-fre-disk for file on subvol. looking for new subvol"
+#define DHT_MSG_SUBVOL_INSUFF_SPACE_STR                                        \
+    "Could not find any subvol with space accommodating the file. Cosider "    \
+    "adding bricks"
+#define DHT_MSG_NEW_TARGET_FOUND_STR "New target found for file"
+#define DHT_MSG_INSUFF_MEMORY_STR "insufficient memory"
+#define DHT_MSG_SET_XATTR_FAILED_STR "failed to set xattr"
+#define DHT_MSG_SET_MODE_FAILED_STR "failed to set mode"
+#define DHT_MSG_FILE_EXISTS_IN_DEST_STR "file exists in destination"
+#define DHT_MSG_LINKFILE_DEL_FAILED_STR "failed to delete the linkfile"
+#define DHT_MSG_SYMLINK_FAILED_STR "symlink failed"
+#define DHT_MSG_MKNOD_FAILED_STR "mknod failed"
+#define DHT_MSG_SETATTR_FAILED_STR "failed to perform setattr"
+#define DHT_MSG_MIGRATE_CLEANUP_FAILED_STR                                     \
+    "Migrate file cleanup failed: failed to fstat file"
+#define DHT_MSG_LOCK_MIGRATE_STR "locks will be migrated for file"
+#define DHT_MSG_PARENT_BUILD_FAILED_STR                                        \
+    "failed to build parent loc, which is needed to acquire entrylk to "       \
+    "synchronize with renames on this path. Skipping migration"
+#define DHT_MSG_HASHED_SUBVOL_NOT_FOUND_STR                                    \
+    "cannot find hashed subvol which is needed to synchronize with renames "   \
+    "on this path. Skipping migration"
+#define DHT_MSG_ACQUIRE_ENTRYLK_FAILED_STR "failed to acquire entrylk on subvol"
+#define DHT_MSG_CREATE_DST_FAILED_STR "create dst failed for file"
+#define DHT_MSG_MIGRATION_EXIT_STR "Exiting migration"
+#define DHT_MSG_CHANGED_DST_STR "destination changed fo file"
+#define DHT_MSG_TRACE_FAILED_STR "Trace failed"
+#define DHT_MSG_WRITE_LOCK_FAILED_STR "write lock failed"
+#define DHT_MSG_GETACTIVELK_FAILED_STR "getactivelk failed for file"
+#define DHT_MSG_STAT_FAILED_STR "failed to do a stat"
+#define DHT_MSG_UNLINK_PERFORM_FAILED_STR "failed to perform unlink"
+#define DHT_MSG_MIGRATE_FILE_COMPLETE_STR "completed migration"
+#define DHT_MSG_CLANUP_SOURCE_FILE_FAILED_STR "failed to cleanup source file"
+#define DHT_MSG_UNLOCK_FILE_FAILED_STR "failed to unlock file"
+#define DHT_MSG_REMOVE_XATTR_FAILED_STR "remove xattr failed"
+#define DHT_MSG_SOCKET_ERROR_STR "Failed to unlink listener socket"
+#define DHT_MSG_HASHED_SUBVOL_GET_FAILED_STR "Failed to get hashed subvolume"
+#define DHT_MSG_CACHED_SUBVOL_GET_FAILED_STR "Failed to get cached subvolume"
+#define DHT_MSG_MIGRATE_DATA_FAILED_STR "migrate-data failed"
+#define DHT_MSG_DEFRAG_NULL_STR "defrag is NULL"
+#define DHT_MSG_DATA_MIGRATE_ABORT_STR                                         \
+    "Readdirp failed. Aborting data migration for dict"
+#define DHT_MSG_LAYOUT_FIX_FAILED_STR "fix layout failed"
+#define DHT_MSG_PARENT_NULL_STR "parent is NULL"
+#define DHT_MSG_GFID_NOT_PRESENT_STR "gfid not present"
+#define DHT_MSG_CHILD_LOC_FAILED_STR "Child loc build failed"
+#define DHT_MSG_SET_LOOKUP_FAILED_STR "Failed to set lookup"
+#define DHT_MSG_DIR_LOOKUP_FAILED_STR "lookup failed"
+#define DHT_MSG_DIR_REMOVED_STR "Dir renamed or removed. Skipping"
+#define DHT_MSG_READDIR_ERROR_STR "readdir failed, Aborting fix-layout"
+#define DHT_MSG_SETTLE_HASH_FAILED_STR "Settle hash failed"
+#define DHT_MSG_DEFRAG_PROCESS_DIR_FAILED_STR "gf_defrag_process_dir failed"
+#define DHT_MSG_FIX_NOT_COMP_STR                                               \
+    "Unable to retrieve fixlayout xattr. Assume background fix layout not "    \
+    "complete"
+#define DHT_MSG_SUBVOL_DETER_FAILED_STR                                        \
+    "local subvolume determination failed with error"
+#define DHT_MSG_LOCAL_SUBVOL_STR "local subvol"
+#define DHT_MSG_NODE_UUID_STR "node uuid"
+#define DHT_MSG_SIZE_FILE_STR "Total size files"
+#define DHT_MSG_GET_DATA_SIZE_FAILED_STR                                       \
+    "Failed to get the total data size. Unable to estimate time to complete "  \
+    "rebalance"
+#define DHT_MSG_PTHREAD_JOIN_FAILED_STR                                        \
+    "file_counter_thread: pthread_join failed"
+#define DHT_MSG_COUNTER_THREAD_CREATE_FAILED_STR                               \
+    "Failed to create the file counter thread"
+#define DHT_MSG_MIGRATION_INIT_QUEUE_FAILED_STR                                \
+    "Failed to initialise migration queue"
+#define DHT_MSG_REBALANCE_STOPPED_STR "Received stop command on rebalance"
+#define DHT_MSG_PAUSED_TIMEOUT_STR "Request pause timer timeout"
+#define DHT_MSG_WOKE_STR "woken"
+#define DHT_MSG_ABORT_REBALANCE_STR "Aborting rebalance"
+#define DHT_MSG_REBALANCE_START_FAILED_STR                                     \
+    "Failed to start rebalance: look up on / failed"
+#define DHT_MSG_CREATE_TASK_REBAL_FAILED_STR                                   \
+    "Could not create task for rebalance"
+#define DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL_STR                                   \
+    "Rebalance estimates will not be available"
+#define DHT_MSG_REBALANCE_STATUS_STR "Rebalance status"
+#define DHT_MSG_DATA_NULL_STR "data value is NULL"
+#define DHT_MSG_ADD_CHOICES_ERROR_STR "Error to add choices in buffer"
+#define DHT_MSG_GET_CHOICES_ERROR_STR "Error to get choices"
+#define DHT_MSG_PREPARE_STATUS_ERROR_STR "Error to prepare status"
+#define DHT_MSG_SET_CHOICE_FAILED_STR "Failed to set full choice"
+#define DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED_STR                               \
+    "Failed to aggregate quota xattr"
+#define DHT_MSG_FILE_TYPE_MISMATCH_STR                                         \
+    "path exists as a file on one subvolume and directory on another. Please " \
+    "fix it manually"
+#define DHT_MSG_LAYOUT_SET_FAILED_STR "failed to set layout for subvolume"
+#define DHT_MSG_LAYOUT_MERGE_FAILED_STR "failed to merge layouts for subvolume"
+#define DHT_MSG_SET_HASHED_SUBVOL_FAILED_STR "Failed to set hashed subvolume"
+#define DHT_MSG_XATTR_HEAL_NOT_POSS_STR                                        \
+    "No gfid exists for path. so healing xattr is not possible"
+#define DHT_MSG_REVALIDATE_CBK_INFO_STR "Revalidate: subvolume returned -1"
+#define DHT_MSG_LAYOUT_MISMATCH_STR "Mismatching layouts"
+#define DHT_MSG_UNLINK_LOOKUP_INFO_STR "lookup_unlink retuened"
+#define DHT_MSG_LINKTO_FILE_FAILED_STR                                         \
+    "Could not unlink the linkto file as either fd is open and/or linkto "     \
+    "xattr is set"
+#define DHT_MSG_LAYOUT_PRESET_FAILED_STR                                       \
+    "Could not set pre-set layout for subvolume"
+#define DHT_MSG_FILE_ON_MULT_SUBVOL_STR                                        \
+    "multiple subvolumes have file (preferably rename the file in the "        \
+    "backend, and do a fresh lookup"
+#define DHT_MSG_STALE_LINKFILE_DELETE_STR                                      \
+    "attempting deletion of stale linkfile"
+#define DHT_MSG_LINK_FILE_LOOKUP_INFO_STR "Lookup on following linkfile"
+#define DHT_MSG_NO_SUBVOL_FOR_LINKTO_STR "No link subvolume for linkto"
+#define DHT_MSG_SUBVOL_RETURNED_STR "Subvolume returned -1"
+#define DHT_MSG_UNKNOWN_LOCAL_XSEL_STR "Unknown local->xsel"
+#define DHT_MSG_DICT_GET_FAILED_STR "Failed to get"
+#define DHT_MSG_UUID_PARSE_ERROR_STR "Failed to parse uuid"
+#define DHT_MSG_GET_XATTR_ERR_STR "getxattr err for dir"
+#define DHT_MSG_ALLOC_OR_FILL_FAILED_STR "alloc or fill failed"
+#define DHT_MSG_UPGRADE_BRICKS_STR                                             \
+    "At least one of the bricks does not support this operation. Please "      \
+    "upgrade all bricks"
+#define DHT_MSG_GET_REAL_NAME_FAILED_STR "Failed to get real filename"
+#define DHT_MSG_LAYOUT_NULL_STR "Layout is NULL"
+#define DHT_MSG_COPY_UUID_FAILED_STR "Failed to copy node uuid key"
+#define DHT_MSG_MDS_DETER_FAILED_STR                                           \
+    "Cannot determine MDS, fetching xattr randomly from a subvol"
+#define DHT_MSG_HASHED_SUBVOL_DOWN_STR                                         \
+    "MDS is down for path, so fetching xattr randomly from subvol"
+#define DHT_MSG_CREATE_REBAL_FAILED_STR                                        \
+    "failed to create a new rebalance synctask"
+#define DHT_MSG_FIX_LAYOUT_INFO_STR "fixing the layout"
+#define DHT_MSG_OPERATION_NOT_SUP_STR "wrong directory-spread-count value"
+#define DHT_MSG_LINK_LAYOUT_FAILED_STR "failed to link the layout in inode"
+#define DHT_MSG_NO_SUBVOL_IN_LAYOUT_STR "no subvolume in layout for path"
+#define DHT_MSG_INODE_LK_ERROR_STR "mknod lock failed for file"
+#define DHT_MSG_MEM_ALLOC_FAILED_STR "mem allocation failed"
+#define DHT_MSG_PARENT_LAYOUT_CHANGED_STR                                      \
+    "extracting in-memory layout of parent failed"
+#define DHT_MSG_SET_IN_PARAMS_DICT_FAILED_STR                                  \
+    "setting in params dictionary failed"
+#define DHT_MSG_LOC_COPY_FAILED_STR "loc_copy failed"
+#define DHT_MSG_LOC_FAILED_STR "parent loc build failed"
+#define DHT_MSG_PARENT_LOC_FAILED_STR "locking parent failed"
+#define DHT_MSG_CREATE_LOCK_FAILED_STR "Create lock failed"
+#define DHT_MSG_PREV_ATTEMPT_FAILED_STR                                        \
+    "mkdir loop detected. parent layout didn't change even though previous "   \
+    "attempt of mkdir failed because of in-memory layout not matching with "   \
+    "that on disk."
+#define DHT_MSG_REFRESH_ATTEMPT_STR                                            \
+    "mkdir parent layout changed. Attempting a refresh and then a retry"
+#define DHT_MSG_ACQUIRE_LOCK_FAILED_STR                                        \
+    "Acquiring lock on parent to guard against layout-change failed"
+#define DHT_MSG_CREATE_STUB_FAILED_STR "creating stub failed"
+#define DHT_MSG_WIND_LOCK_REQ_FAILED_STR                                       \
+    "cannot wind lock request to guard parent layout"
+#define DHT_MSG_REFRESH_FAILED_STR "refreshing parent layout failed."
+#define DHT_MSG_CACHED_SUBVOL_ERROR_STR "On cached subvol"
+#define DHT_MSG_NO_LINK_SUBVOL_STR "Linkfile does not have link subvolume"
+#define DHT_MSG_SET_KEY_FAILED_STR "failed to set key"
+#define DHT_MSG_CHILD_DOWN_STR "Received CHILD_DOWN. Exiting"
+#define DHT_MSG_LOG_FIXED_LAYOUT_STR "log layout fixed"
+#define DHT_MSG_REBAL_STRUCT_SET_STR "local->rebalance already set"
+#define DHT_MSG_REMOVE_LINKTO_FAILED_STR "Removal of linkto failed at subvol"
+#define DHT_MSG_LAYOUT_DICT_SET_FAILED_STR "dht layout dict set failed"
+#define DHT_MSG_SUBVOL_INFO_STR "creating subvolume"
+#define DHT_MSG_COMPUTE_HASH_FAILED_STR "hash computation failed"
+#define DHT_MSG_INVALID_DISK_LAYOUT_STR                                        \
+    "Invalid disk layout: Catastrophic error layout with unknown type found"
+#define DHT_MSG_LAYOUT_SORT_FAILED_STR "layout sort failed"
+#define DHT_MSG_ANOMALIES_INFO_STR "Found anomalies"
+#define DHT_MSG_XATTR_DICT_NULL_STR "xattr dictionary is NULL"
+#define DHT_MSG_DISK_LAYOUT_MISSING_STR "Disk layout missing"
+#define DHT_MSG_LAYOUT_INFO_STR "layout info"
+#define DHT_MSG_SUBVOL_NO_LAYOUT_INFO_STR "no pre-set layout for subvol"
+#define DHT_MSG_SELFHEAL_XATTR_FAILED_STR "layout setxattr failed"
+#define DHT_MSG_DIR_SELFHEAL_XATTR_FAILED_STR "Directory self heal xattr failed"
+#define DHT_MSG_DUMMY_ALLOC_FAILED_STR "failed to allocate dummy layout"
+#define DHT_MSG_DICT_IS_NULL_STR                                               \
+    "dict is NULL, need to make sure gfids are same"
+#define DHT_MSG_ENTRYLK_ERROR_STR "acquiring entrylk after inodelk failed"
+#define DHT_MSG_NO_DISK_USAGE_STATUS_STR "no du stats"
+#define DHT_MSG_LINK_INODE_FAILED_STR "linking inode failed"
+#define DHT_MSG_SELFHEAL_FAILED_STR "Directory selfheal failed"
+#define DHT_MSG_NO_MDS_SUBVOL_STR "No mds subvol"
+#define DHT_MSG_LIST_XATTRS_FAILED_STR "failed to list xattrs"
+#define DHT_MSG_RESET_INTER_XATTR_FAILED_STR "Failed to reset internal xattr"
+#define DHT_MSG_MDS_DOWN_UNABLE_TO_SET_STR                                     \
+    "mds subvol is down, unable to set xattr"
+#define DHT_MSG_DIR_ATTR_HEAL_FAILED_STR                                       \
+    "Directory attr heal failed. Failed to set uid/gid"
+#define DHT_MSG_WIND_UNLOCK_FAILED_STR                                         \
+    "Winding unlock failed: stale locks left on brick"
+#define DHT_MSG_COMMIT_HASH_FAILED_STR "Directory commit hash updaten failed"
+#define DHT_MSG_LK_ARRAY_INFO_STR "lk info"
+#define DHT_MSG_UNLOCK_GFID_FAILED_STR                                         \
+    "unlock failed on gfid: stale lock might be left"
+#define DHT_MSG_UNLOCKING_FAILED_STR "unlocking failed"
+#define DHT_MSG_UNLOCK_FOLLOW_ENTRYLK_STR "not unlocking following entrylks"
+#define DHT_MSG_COPY_FRAME_FAILED_STR "copy frame failed"
+#define DHT_MSG_UNLOCK_FOLLOW_LOCKS_STR "not unlocking following locks"
+#define DHT_MSG_INODELK_FAILED_STR "inodelk failed on subvol"
+#define DHT_MSG_LOCK_FRAME_FAILED_STR "memory allocation failed for lock_frame"
+#define DHT_MSG_LOCAL_LOCK_INIT_FAILED_STR "dht_local_lock_init failed"
+#define DHT_MSG_ENTRYLK_FAILED_AFT_INODELK_STR                                 \
+    "dht_blocking_entrylk failed after taking inodelk"
+#define DHT_MSG_BLOCK_INODELK_FAILED_STR "dht_blocking_inodelk failed"
+#define DHT_MSG_CALLOC_FAILED_STR "calloc failed"
+#define DHT_MSG_LOCK_ALLOC_FAILED_STR "lock allocation failed"
+#define DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS_STR        \
+    "cannot allocate a frame, not unlocking following entrylks"
+#define DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK_STR       \
+    "storing locks in local failed, not unlocking following entrylks"
+#define DHT_MSG_DST_NULL_SET_FAILED_STR                                        \
+    "src or dst is NULL, Failed to set dictionary value"
 
-/*! \file dht-messages.h
- *  \brief DHT log-message IDs and their descriptions
- *
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_DHT_BASE                   GLFS_MSGID_COMP_DHT
-#define GLFS_DHT_NUM_MESSAGES           116
-#define GLFS_MSGID_END          (GLFS_DHT_BASE + GLFS_DHT_NUM_MESSAGES + 1)
-
-/* Messages with message IDs */
-#define glfs_msg_start_x GLFS_DHT_BASE, "Invalid: Start of messages"
-
-
-
-
-/*!
- * @messageid 109001
- * @diagnosis   Cached subvolume could not be found for the specified
- *              path
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_CACHED_SUBVOL_GET_FAILED        (GLFS_DHT_BASE + 1)
-
-/*!
- * @messageid 109002
- * @diagnosis Linkfile creation failed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_CREATE_LINK_FAILED      (GLFS_DHT_BASE + 2)
-
-/*!
- * @messageid 109003
- * @diagnosis The value could not be set for the specified key in
- *       the dictionary
- *
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_DICT_SET_FAILED         (GLFS_DHT_BASE + 3)
-
-/*!
- * @messageid 109004
- * @diagnosis Directory attributes could not be healed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_DIR_ATTR_HEAL_FAILED    (GLFS_DHT_BASE + 4)
-
-/*!
- * @messageid 109005
- * @diagnosis Self-heal failed for the specified directory
- * @recommendedaction  Ensure that all subvolumes are online
- *              and reachable and perform a lookup operation
- *              on the directory again.
- *
- */
-
-#define DHT_MSG_DIR_SELFHEAL_FAILED     (GLFS_DHT_BASE + 5)
-
-/*!
- * @messageid 109006
- * @diagnosis The extended attributes could not be healed for
- *            the specified directory on the specified subvolume
- *
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_DIR_SELFHEAL_XATTR_FAILED       (GLFS_DHT_BASE + 6)
-
-/*!
- * @messageid 109007
- * @diagnosis   A lookup operation found the file with the same path
- *      on multiple subvolumes.
- * @recommendedaction
- *      1. Create backups of the file on other subvolumes.
- *      2. Inspect the content of the files to identify
- *                      and retain the most appropriate file.
- *
- */
-
-#define DHT_MSG_FILE_ON_MULT_SUBVOL     (GLFS_DHT_BASE + 7)
-
-/*!
- * @messageid 109008
- * @diagnosis A path resolves to a file on one subvolume and a directory
- *             on another
- * @recommendedaction
- *              1. Create a backup of the file with a different name
- *              and delete the original file.
- *              2. In the newly created back up file, remove the "trusted.gfid"
- *                      extended attribute.
- *                - Command: setfattr -x "trusted.gfid" \<path to the newly created backup file\>
- *              3. Perform a new lookup operation on both the new and old paths.
- *              4. From the mount point, inspect both the paths and retain the
- *              relevant file or directory.
- *
- */
-
-#define DHT_MSG_FILE_TYPE_MISMATCH      (GLFS_DHT_BASE + 8)
-
-/*!
- * @messageid 109009
- * @diagnosis The GFID of the file/directory is different on different subvolumes
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_GFID_MISMATCH           (GLFS_DHT_BASE + 9)
-
-/*!
- * @messageid 109010
- * @diagnosis The GFID of the specified file/directory is NULL.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_GFID_NULL               (GLFS_DHT_BASE + 10)
-
-/*
- * @messageid 109011
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_HASHED_SUBVOL_GET_FAILED   (GLFS_DHT_BASE + 11)
-
-/*!
- * @messageid 109012
- * @diagnosis The Distributed Hash Table Translator could not be initiated as the
- *            system is out of memory.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_INIT_FAILED             (GLFS_DHT_BASE + 12)
-
-/*!
- * @messageid 109013
- * @diagnosis Invalid DHT configuration in the volfile
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_INVALID_CONFIGURATION   (GLFS_DHT_BASE + 13)
-
-/*!
- * @messageid 109014
- * @diagnosis Invalid disk layout
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_INVALID_DISK_LAYOUT     (GLFS_DHT_BASE + 14)
-
-/*!
- * @messageid 109015
- * @diagnosis Invalid DHT configuration option.
- * @recommendedaction
- *              1. Reset the option with a valid value using the volume
- *                set command.
- *              2. Restart the process that logged the message in the
- *                log file.
- *
- */
-
-#define DHT_MSG_INVALID_OPTION          (GLFS_DHT_BASE + 15)
-
-/*!
- * @messageid 109016
- * @diagnosis The fix layout operation failed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_FIX_FAILED       (GLFS_DHT_BASE + 16)
-
-/*!
- * @messageid 109017
- * @diagnosis Layout merge failed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_MERGE_FAILED     (GLFS_DHT_BASE + 17)
-
-/*!
- * @messageid 109018
- * @diagnosis The layout for the specified directory does not match
-                that on the disk.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_MISMATCH         (GLFS_DHT_BASE + 18)
-
-/*!
- * @messageid 109019
- * @diagnosis No layout is present for the specified file/directory
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_NULL             (GLFS_DHT_BASE + 19)
-
-/*!
- * @messageid 109020
- * @diagnosis Informational message: Migration of data from the cached
- *      subvolume to the hashed subvolume is complete
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_MIGRATE_DATA_COMPLETE   (GLFS_DHT_BASE + 20)
-
-/*!
- * @messageid 109021
- * @diagnosis Migration of data failed during the rebalance operation
- *     \n Cause: Directories could not be read to identify the files for the
- *             migration process.
- * @recommendedaction
- *             The log message would indicate the reason for the failure and
- *             the corrective action depends on the specific error that is
- *             encountered. The error is one of the standard UNIX errors.
- *
- */
-
-#define DHT_MSG_MIGRATE_DATA_FAILED     (GLFS_DHT_BASE + 21)
-
-/*!
- * @messageid 109022
- * @diagnosis Informational message: The file was migrated successfully during
- *              the rebalance operation.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_MIGRATE_FILE_COMPLETE   (GLFS_DHT_BASE + 22)
-
-/*!
- * @messageid 109023
- * @diagnosis File migration failed during the rebalance operation
- *            \n Cause: Rebalance moves data from the cached subvolume to
- *            the hashed subvolume. Migrating a single file is a multi-step operation
- *            which involves opening, reading, and writing the data and metadata.
- *            Any failures in this multi-step operation can result in a file
- *            migration failure.
- * @recommendedaction  The log message would indicate the reason for the failure and the
- *              corrective action depends on the specific error that is encountered.
- *              The error is one of the standard UNIX errors.
- *
- */
-
-#define DHT_MSG_MIGRATE_FILE_FAILED     (GLFS_DHT_BASE + 23)
-
-/*!
- * @messageid 109024
- * @diagnosis Out of memory
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_NO_MEMORY               (GLFS_DHT_BASE + 24)
-
-/*!
- * @messageid 109025
- * @diagnosis  The opendir() call failed on the specified directory
- *              \n Cause: When a directory is renamed, the Distribute Hash
- *              table translator checks whether the destination directory
- *              is empty. This message indicates that the opendir() call
- *              on the destination directory has failed.
- * @recommendedaction The log message would indicate the reason for the
- *              failure and the corrective action depends on the specific
- *              error that is encountered. The error is one of the standard
- *              UNIX errors.
- *
- */
-
-#define DHT_MSG_OPENDIR_FAILED          (GLFS_DHT_BASE + 25)
-
-/*!
- * @messageid 109026
- * @diagnosis The rebalance operation failed.
- * @recommendedaction Check the log file for details about the failure.
- *     Possible causes:
- *     - A subvolume is down: Restart the rebalance operation after
- *             bringing up all subvolumes.
- *
- */
-
-#define DHT_MSG_REBALANCE_FAILED        (GLFS_DHT_BASE + 26)
-
-/*!
- * @messageid 109027
- * @diagnosis Failed to start the rebalance process.
- * @recommendedaction Check the log file for details about the failure.
- *
- */
-
-#define DHT_MSG_REBALANCE_START_FAILED  (GLFS_DHT_BASE + 27)
-
-/*!
- * @messageid 109028
- * @diagnosis Informational message that indicates the status of the
- *            rebalance operation and details as to how many files were
- *            migrated, skipped, failed etc
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_REBALANCE_STATUS        (GLFS_DHT_BASE + 28)
-
-/*!
- * @messageid 109029
- * @diagnosis The rebalance operation was aborted by the user.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_REBALANCE_STOPPED       (GLFS_DHT_BASE + 29)
-
-/*!
- * @messageid 109030
- * @diagnosis The file or directory could not be renamed
- * @recommendedaction   Ensure that all the subvolumes are
- *                      online and reachable and try renaming
- *                      the file or directory again.
- *
- */
-
-#define DHT_MSG_RENAME_FAILED           (GLFS_DHT_BASE + 30)
-
-/*!
- * @messageid 109031
- * @diagnosis Attributes could not be set for the specified file or
- *             directory.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_SETATTR_FAILED          (GLFS_DHT_BASE + 31)
-
-/*!
- * @messageid 109032
- * @diagnosis The specified subvolume is running out of file system inodes.
-        If all subvolumes run out of inodes, then new files cannot be created.
- * @recommendedaction  Consider adding more nodes to the cluster if all subvolumes
- *        run out of inodes
- *
- */
-
-#define DHT_MSG_SUBVOL_INSUFF_INODES    (GLFS_DHT_BASE + 32)
-
-/*!
- * @messageid 109033
- * @diagnosis The specified subvolume is running out of disk space. If all
-              subvolumes run out of space, new files cannot be created.
- * @recommendedaction  Consider adding more bricks to the cluster if all subvolumes
- *              run out of disk space.
- *
- */
-
-#define DHT_MSG_SUBVOL_INSUFF_SPACE    (GLFS_DHT_BASE + 33)
-
-/*!
- * @messageid 109034
- * @diagnosis Failed to unlink the specified file/directory
- * @recommendedaction  The log message would indicate the reason
-              for the failure and the corrective action depends on
-              the specific error that is encountered.
- */
-
-#define DHT_MSG_UNLINK_FAILED           (GLFS_DHT_BASE + 34)
-
-
-
-/*!
- * @messageid 109035
- * @diagnosis The layout information could not be set in the inode
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_SET_FAILED       (GLFS_DHT_BASE + 35)
-
-/*!
- * @messageid 109036
- * @diagnosis Informational message regarding layout range distribution
- *            for a directory across subvolumes
- * @recommendedaction None
- */
-
-#define DHT_MSG_LOG_FIXED_LAYOUT        (GLFS_DHT_BASE + 36)
-
-/*
- * @messageid 109037
- * @diagnosis Informational message regarding error in tier operation
- * @recommendedaction None
- */
-
-#define DHT_MSG_LOG_TIER_ERROR          (GLFS_DHT_BASE + 37)
-
-/*
- * @messageid 109038
- * @diagnosis Informational message regarding tier operation
- * @recommendedaction None
- */
-
-#define DHT_MSG_LOG_TIER_STATUS         (GLFS_DHT_BASE + 38)
-
-/*
- * @messageid 109039
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_GET_XATTR_FAILED        (GLFS_DHT_BASE + 39)
-
-/*
- * @messageid 109040
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_FILE_LOOKUP_FAILED      (GLFS_DHT_BASE + 40)
-
-/*
- * @messageid 109041
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_OPEN_FD_FAILED          (GLFS_DHT_BASE + 41)
-
-/*
- * @messageid 109042
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SET_INODE_CTX_FAILED    (GLFS_DHT_BASE + 42)
-
-/*
- * @messageid 109043
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_UNLOCKING_FAILED        (GLFS_DHT_BASE + 43)
-
-/*
- * @messageid 109044
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_DISK_LAYOUT_NULL        (GLFS_DHT_BASE + 44)
-
-/*
- * @messageid 109045
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SUBVOL_INFO             (GLFS_DHT_BASE + 45)
-
-/*
- * @messageid 109046
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_CHUNK_SIZE_INFO         (GLFS_DHT_BASE + 46)
-
-/*
- * @messageid 109047
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_LAYOUT_FORM_FAILED      (GLFS_DHT_BASE + 47)
-
-/*
- * @messageid 109048
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SUBVOL_ERROR            (GLFS_DHT_BASE + 48)
-
-/*
- * @messageid 109049
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_LAYOUT_SORT_FAILED      (GLFS_DHT_BASE + 49)
-
-/*
- * @messageid 109050
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_REGEX_INFO              (GLFS_DHT_BASE + 50)
-
-/*
- * @messageid 109051
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_FOPEN_FAILED            (GLFS_DHT_BASE + 51)
-
-/*
- * @messageid 109052
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SET_HOSTNAME_FAILED     (GLFS_DHT_BASE + 52)
-
-/*
- * @messageid 109053
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_BRICK_ERROR             (GLFS_DHT_BASE + 53)
-
-/*
- * @messageid 109054
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SYNCOP_FAILED           (GLFS_DHT_BASE + 54)
-
-/*
- * @messageid 109055
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_MIGRATE_INFO            (GLFS_DHT_BASE + 55)
-
-/*
- * @messageid 109056
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SOCKET_ERROR            (GLFS_DHT_BASE + 56)
-
-/*
- * @messageid 109057
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_CREATE_FD_FAILED        (GLFS_DHT_BASE + 57)
-
-/*
- * @messageid 109058
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_READDIR_ERROR           (GLFS_DHT_BASE + 58)
-
-/*
- * @messageid 109059
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_CHILD_LOC_BUILD_FAILED  (GLFS_DHT_BASE + 59)
-
-/*
- * @messageid 109060
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SET_SWITCH_PATTERN_ERROR    (GLFS_DHT_BASE + 60)
-
-/*
- * @messageid 109061
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_COMPUTE_HASH_FAILED     (GLFS_DHT_BASE + 61)
-
-/*
- * @messageid 109062
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR     (GLFS_DHT_BASE + 62)
-
-/*
- * @messageid 109063
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_ANOMALIES_INFO          (GLFS_DHT_BASE + 63)
-
-/*
- * @messageid 109064
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_LAYOUT_INFO             (GLFS_DHT_BASE + 64)
-
-/*
- * @messageid 109065
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_INODE_LK_ERROR          (GLFS_DHT_BASE + 65)
-
-/*
- * @messageid 109066
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_RENAME_INFO             (GLFS_DHT_BASE + 66)
-
-/*
- * @messageid 109067
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_DATA_NULL               (GLFS_DHT_BASE + 67)
-
-/*
- * @messageid 109068
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED   (GLFS_DHT_BASE + 68)
-
-/*
- * @messageid 109069
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_UNLINK_LOOKUP_INFO      (GLFS_DHT_BASE + 69)
-
-/*
- * @messageid 109070
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_LINK_FILE_LOOKUP_INFO   (GLFS_DHT_BASE + 70)
-
-/*
- * @messageid 109071
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_OPERATION_NOT_SUP       (GLFS_DHT_BASE + 71)
-
-/*
- * @messageid 109072
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_NOT_LINK_FILE_ERROR     (GLFS_DHT_BASE + 72)
-
-/*
- * @messageid 109073
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_CHILD_DOWN              (GLFS_DHT_BASE + 73)
-
-/*
- * @messageid 109074
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_UUID_PARSE_ERROR        (GLFS_DHT_BASE + 74)
-
-/*
- * @messageid 109075
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_GET_DISK_INFO_ERROR     (GLFS_DHT_BASE + 75)
-
-/*
- * @messageid 109076
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_INVALID_VALUE           (GLFS_DHT_BASE + 76)
-
-/*
- * @messageid 109077
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SWITCH_PATTERN_INFO     (GLFS_DHT_BASE + 77)
-
-/*
- * @messageid 109078
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SUBVOL_OP_FAILED        (GLFS_DHT_BASE + 78)
-
-/*
- * @messageid 109079
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_LAYOUT_PRESET_FAILED    (GLFS_DHT_BASE + 79)
-
-/*
- * @messageid 109080
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_INVALID_LINKFILE        (GLFS_DHT_BASE + 80)
-
-/*
- * @messageid 109081
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_FIX_LAYOUT_INFO         (GLFS_DHT_BASE + 81)
-
-/*
- * @messageid 109082
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_GET_HOSTNAME_FAILED     (GLFS_DHT_BASE + 82)
-
-/*
- * @messageid 109083
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_WRITE_FAILED            (GLFS_DHT_BASE + 83)
-
-/*
- * @messageid 109084
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED (GLFS_DHT_BASE + 84)
-
-/*
- * @messageid 109085
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_FSYNC_FAILED            (GLFS_DHT_BASE + 85)
-
-/*
- * @messageid 109086
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SUBVOL_DECOMMISSION_INFO (GLFS_DHT_BASE + 86)
-
-/*
- * @messageid 109087
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_BRICK_QUERY_FAILED      (GLFS_DHT_BASE + 87)
-
-/*
- * @messageid 109088
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SUBVOL_NO_LAYOUT_INFO   (GLFS_DHT_BASE + 88)
-
-/*
- * @messageid 109089
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_OPEN_FD_ON_DST_FAILED   (GLFS_DHT_BASE + 89)
-
-/*
- * @messageid 109090
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SUBVOL_NOT_FOUND        (GLFS_DHT_BASE + 90)
-
-/*
- * @messageid 109190
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_FILE_LOOKUP_ON_DST_FAILED   (GLFS_DHT_BASE + 91)
-
-/*
- * @messageid 109092
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_DISK_LAYOUT_MISSING     (GLFS_DHT_BASE + 92)
-
-/*
- * @messageid 109093
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_DICT_GET_FAILED         (GLFS_DHT_BASE + 93)
-
-/*
- * @messageid 109094
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_REVALIDATE_CBK_INFO     (GLFS_DHT_BASE + 94)
-
-/*
- * @messageid 109095
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_UPGRADE_BRICKS         (GLFS_DHT_BASE + 95)
-
-/*
- * @messageid 109096
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_LK_ARRAY_INFO           (GLFS_DHT_BASE + 96)
-
-/*
- * @messageid 109097
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_RENAME_NOT_LOCAL        (GLFS_DHT_BASE + 97)
-
-/*
- * @messageid 109098
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_RECONFIGURE_INFO        (GLFS_DHT_BASE + 98)
-
-/*
- * @messageid 109099
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_INIT_LOCAL_SUBVOL_FAILED        (GLFS_DHT_BASE + 99)
-
-/*
- * @messageid 109100
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SYS_CALL_GET_TIME_FAILED        (GLFS_DHT_BASE + 100)
-
-/*
- * @messageid 109101
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_NO_DISK_USAGE_STATUS    (GLFS_DHT_BASE + 101)
-
-/*
- * @messageid 109102
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SUBVOL_DOWN_ERROR       (GLFS_DHT_BASE + 102)
-
-/*
- * @messageid 109103
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_REBAL_THROTTLE_INFO       (GLFS_DHT_BASE + 103)
-
-/*
- * @messageid 109104
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_COMMIT_HASH_INFO       (GLFS_DHT_BASE + 104)
-
-/*
- * @messageid 109105
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_REBAL_STRUCT_SET       (GLFS_DHT_BASE + 105)
-
-/*
- * @messageid 109106
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_HAS_MIGINFO             (GLFS_DHT_BASE + 106)
-
-/*
- * @messageid 109107
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_LOG_IPC_TIER_ERROR      (GLFS_DHT_BASE + 107)
-
-/*
- * @messageid 109108
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_TIER_PAUSED             (GLFS_DHT_BASE + 108)
-
-/*
- * @messageid 109109
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_TIER_RESUME             (GLFS_DHT_BASE + 109)
-
-
-/* @messageid 109110
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_SETTLE_HASH_FAILED       (GLFS_DHT_BASE + 110)
-
-/*
- * @messageid 109111
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_DEFRAG_PROCESS_DIR_FAILED    (GLFS_DHT_BASE + 111)
-
-/*
- * @messageid 109112
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_FD_CTX_SET_FAILED         (GLFS_DHT_BASE + 112)
-
-/*
- * @messageid 109113
- * @diagnosis
- * @recommendedaction None
- */
-
-#define DHT_MSG_STALE_LOOKUP                    (GLFS_DHT_BASE + 113)
-
-/*
- * @messageid 109114
- * @diagnosis
- * @recommendedaction None
- */
-#define DHT_MSG_PARENT_LAYOUT_CHANGED  (GLFS_DHT_BASE + 114)
-
-/*
- * @messageid 109115
- * @diagnosis
- * @recommendedaction None
- */
-#define DHT_MSG_LOCK_MIGRATION_FAILED  (GLFS_DHT_BASE + 115)
-
-/*
- * @messageid 109116
- * @diagnosis
- * @recommendedaction None
- */
-#define DHT_MSG_LOCK_INODE_UNREF_FAILED  (GLFS_DHT_BASE + 116)
-
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* _DHT_MESSAGES_H_ */
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 4c83ed478c0..8ba8082bd86 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -8,141 +8,143 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "tier.h"
 #include "dht-common.h"
-#include "xlator.h"
-#include "syscall.h"
-#include <signal.h>
+#include <glusterfs/syscall.h>
 #include <fnmatch.h>
 #include <signal.h>
-
-
-#define GF_DISK_SECTOR_SIZE             512
-#define DHT_REBALANCE_PID               4242 /* Change it if required */
-#define DHT_REBALANCE_BLKSIZE           (128 * 1024)
-#define MAX_MIGRATOR_THREAD_COUNT       40
-#define MAX_MIGRATE_QUEUE_COUNT         500
-#define MIN_MIGRATE_QUEUE_COUNT         200
-
+#include <glusterfs/events.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+
+#define GF_DISK_SECTOR_SIZE 512
+#define DHT_REBALANCE_PID 4242        /* Change it if required */
+#define DHT_REBALANCE_BLKSIZE 1048576 /* 1 MB */
+#define MAX_MIGRATE_QUEUE_COUNT 500
+#define MIN_MIGRATE_QUEUE_COUNT 200
+#define MAX_REBAL_TYPE_SIZE 16
+#define FILE_CNT_INTERVAL 600       /* 10 mins */
+#define ESTIMATE_START_INTERVAL 600 /* 10 mins */
+#define HARDLINK_MIG_INPROGRESS -2
+#define SKIP_MIGRATION_FD_POSITIVE -3
 #ifndef MAX
-#define MAX(a, b) (((a) > (b))?(a):(b))
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
+#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt)                                       \
+    {                                                                          \
+        idx++;                                                                 \
+        idx %= sv_cnt;                                                         \
+    }
 
-#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt)  {     \
-                idx++;                          \
-                idx %= sv_cnt;                  \
-        }
+uint64_t g_totalfiles = 0;
+uint64_t g_totalsize = 0;
 
-#define GF_FREE_DIR_DFMETA(dir_dfmeta) {                        \
-                if (dir_dfmeta) {                               \
-                        GF_FREE (dir_dfmeta->head);             \
-                        GF_FREE (dir_dfmeta->equeue);           \
-                        GF_FREE (dir_dfmeta->iterator);         \
-                        GF_FREE (dir_dfmeta->offset_var);       \
-                        GF_FREE (dir_dfmeta->fetch_entries);    \
-                        GF_FREE (dir_dfmeta);                   \
-                }                                               \
-        }                                                       \
+void
+gf_defrag_free_dir_dfmeta(struct dir_dfmeta *meta, int local_subvols_cnt)
+{
+    int i = 0;
+
+    if (meta) {
+        for (i = 0; i < local_subvols_cnt; i++) {
+            if (meta->equeue)
+                gf_dirent_free(&meta->equeue[i]);
+            if (meta->lfd && meta->lfd[i])
+                fd_unref(meta->lfd[i]);
+        }
+
+        GF_FREE(meta->equeue);
+        GF_FREE(meta->head);
+        GF_FREE(meta->iterator);
+        GF_FREE(meta->offset_var);
+        GF_FREE(meta->fetch_entries);
+        GF_FREE(meta->lfd);
+        GF_FREE(meta);
+    }
+}
 
 void
-gf_defrag_free_container (struct dht_container *container)
+gf_defrag_free_container(struct dht_container *container)
 {
-        if (container) {
-                gf_dirent_entry_free (container->df_entry);
+    if (container) {
+        gf_dirent_entry_free(container->df_entry);
 
-                if (container->parent_loc) {
-                        loc_wipe (container->parent_loc);
-                }
+        if (container->parent_loc) {
+            loc_wipe(container->parent_loc);
+        }
 
-                GF_FREE (container->parent_loc);
+        GF_FREE(container->parent_loc);
 
-                GF_FREE (container);
-        }
+        GF_FREE(container);
+    }
 }
 
 void
-dht_set_global_defrag_error (gf_defrag_info_t *defrag, int ret)
+dht_set_global_defrag_error(gf_defrag_info_t *defrag, int ret)
 {
-        LOCK (&defrag->lock);
-        {
-                defrag->global_error = ret;
-        }
-        UNLOCK (&defrag->lock);
-        return;
+    LOCK(&defrag->lock);
+    {
+        defrag->global_error = ret;
+    }
+    UNLOCK(&defrag->lock);
+    return;
 }
 
 static int
-dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
-                      int32_t size, off_t offset, struct iobref *iobref)
+dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status)
 {
-        int i            = 0;
-        int ret          = -1;
-        int start_idx    = 0;
-        int tmp_offset   = 0;
-        int write_needed = 0;
-        int buf_len      = 0;
-        int size_pending = 0;
-        char *buf        = NULL;
-
-        /* loop through each vector */
-        for (i = 0; i < count; i++) {
-                buf = vec[i].iov_base;
-                buf_len = vec[i].iov_len;
-
-                for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len;
-                     start_idx += GF_DISK_SECTOR_SIZE) {
-
-                        if (mem_0filled (buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) {
-                                write_needed = 1;
-                                continue;
-                        }
+    int ret = -1;
+    char *volname = NULL;
+    char *tmpstr = NULL;
+    char *ptr = NULL;
+    char *suffix = "-dht";
+    int len = 0;
 
-                        if (write_needed) {
-                                ret = syncop_write (to, fd, (buf + tmp_offset),
-                                                    (start_idx - tmp_offset),
-                                                    (offset + tmp_offset),
-                                                    iobref, 0, NULL, NULL);
-                                /* 'path' will be logged in calling function */
-                                if (ret < 0) {
-                                        gf_log (THIS->name, GF_LOG_WARNING,
-                                                "failed to write (%s)",
-                                                strerror (-ret));
-                                        ret = -1;
-                                        goto out;
-                                }
-
-                                write_needed = 0;
-                        }
-                        tmp_offset = start_idx + GF_DISK_SECTOR_SIZE;
-                }
-
-                if ((start_idx < buf_len) || write_needed) {
-                        /* This means, last chunk is not yet written.. write it */
-                        ret = syncop_write (to, fd, (buf + tmp_offset),
-                                            (buf_len - tmp_offset),
-                                            (offset + tmp_offset), iobref, 0,
-                                            NULL, NULL);
-                        if (ret < 0) {
-                                /* 'path' will be logged in calling function */
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to write (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-                }
+    eventtypes_t event = EVENT_LAST;
 
-                size_pending = (size - buf_len);
-                if (!size_pending)
-                        break;
-        }
-
-        ret = size;
-out:
-        return ret;
+    switch (status) {
+        case GF_DEFRAG_STATUS_COMPLETE:
+            event = EVENT_VOLUME_REBALANCE_COMPLETE;
+            break;
+        case GF_DEFRAG_STATUS_FAILED:
+            event = EVENT_VOLUME_REBALANCE_FAILED;
+            break;
+        case GF_DEFRAG_STATUS_STOPPED:
+            event = EVENT_VOLUME_REBALANCE_STOP;
+            break;
+        default:
+            break;
+    }
+
+    /* DHT volume */
+    len = strlen(this->name) - strlen(suffix);
+    tmpstr = gf_strdup(this->name);
+    if (tmpstr) {
+        ptr = tmpstr + len;
+        if (!strcmp(ptr, suffix)) {
+            tmpstr[len] = '\0';
+            volname = tmpstr;
+        }
+    }
+
+    if (!volname) {
+        /* Better than nothing */
+        volname = this->name;
+    }
+
+    if (event != EVENT_LAST) {
+        gf_event(event, "volume=%s", volname);
+    }
+
+    GF_FREE(tmpstr);
+    return ret;
+}
 
+static void
+dht_strip_out_acls(dict_t *dict)
+{
+    if (dict) {
+        dict_del(dict, "trusted.SGI_ACL_FILE");
+        dict_del(dict, POSIX_ACL_ACCESS_XATTR);
+    }
 }
 
 /*
@@ -181,200 +183,316 @@ be converted to "0" in dht_migrate_file.
 */
 
 int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
-                           struct iatt *stbuf)
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno)
 {
-        int32_t                 ret             = -1;
-        xlator_t               *cached_subvol   = NULL;
-        xlator_t               *hashed_subvol   = NULL;
-        xlator_t               *linkto_subvol   = NULL;
-        data_t                 *data            = NULL;
-        struct iatt             iatt            = {0,};
-        int32_t                 op_errno        = 0;
-        dht_conf_t             *conf            = NULL;
-        gf_loglevel_t          loglevel         = 0;
-        dict_t                 *link_xattr      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("defrag", loc, out);
-        GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
-        GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
-        GF_VALIDATE_OR_GOTO ("defrag", this, out);
-        GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
-        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
-
-        conf = this->private;
-
-        if (gf_uuid_is_null (loc->pargfid)) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "loc->pargfid is NULL for %s", loc->path);
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *linkto_subvol = NULL;
+    data_t *data = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int32_t op_errno = 0;
+    dht_conf_t *conf = NULL;
+    gf_loglevel_t loglevel = 0;
+    dict_t *link_xattr = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr_rsp = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    *fop_errno = EINVAL;
+
+    GF_VALIDATE_OR_GOTO("defrag", loc, out);
+    GF_VALIDATE_OR_GOTO("defrag", loc->name, out);
+    GF_VALIDATE_OR_GOTO("defrag", this, out);
+    GF_VALIDATE_OR_GOTO("defrag", this->private, out);
+
+    conf = this->private;
+
+    if (gf_uuid_is_null(loc->pargfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->pargfid is NULL for %s",
+               loc->path);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->gfid is NULL for %s",
+               loc->path);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
 
-        if (gf_uuid_is_null (loc->gfid)) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "loc->gfid is NULL for %s", loc->path);
-                goto out;
+    link_xattr = dict_new();
+    if (!link_xattr) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    /*
+      Parallel migration can lead to migration of the hard link multiple
+      times which can lead to data loss. Hence, adding a fresh lookup to
+      decide whether migration is required or not.
+
+      Elaborating the scenario for let say 10 hardlinks [link{1..10}]:
+          Let say the first hard link "link1"  does the setxattr of the
+      new hashed subvolume info on the cached file. As there are multiple
+      threads working, we might have already all the links created on the
+      new hashed by the time we reach hardlink let say link5. Now the
+      number of links on hashed is equal to that of cached. Hence, file
+      migration will happen for link6.
+
+             Cached                                 Hashed
+      --------T link6                        rwxrwxrwx   link6
+
+      Now post above state all the link file on the cached will be zero
+      byte linkto files. Hence, if we still do migration for the following
+      files link{7..10}, we will end up migrating 0 data leading to data
+      loss.
+            Hence, a lookup can make sure whether we need to migrate the
+      file or not.
+    */
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "could not allocate memory for dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    ret = syncop_lookup(this, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        /*Ignore ENOENT and ESTALE as file might have been
+          migrated already*/
+        if (-ret == ENOENT || -ret == ESTALE) {
+            ret = -2;
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:%s lookup failed with ret = %d", loc->path,
+               ret);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    cached_subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get cached subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get hashed subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    /* Hardlink migration happens only with remove-brick. So this condition will
+     * be true only when the migration has happened. In case hardlinks are
+     * migrated for rebalance case, remove this check. Having this check here
+     * avoid redundant calls below*/
+    if (hashed_subvol == cached_subvol) {
+        ret = -2;
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_INFO,
+           "Attempting to migrate hardlink %s "
+           "with gfid %s from %s -> %s",
+           loc->name, uuid_utoa(loc->gfid), cached_subvol->name,
+           hashed_subvol->name);
+
+    data = dict_get(xattr_rsp, conf->link_xattr_name);
+    /* set linkto on cached -> hashed if not present, else link it */
+    if (!data) {
+        ret = dict_set_str(link_xattr, conf->link_xattr_name,
+                           hashed_subvol->name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Failed to set dictionary value:"
+                   " key = %s for %s",
+                   conf->link_xattr_name, loc->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        link_xattr = dict_new ();
-        if (!link_xattr) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+        ret = syncop_setxattr(cached_subvol, loc, link_xattr, 0, NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Linkto setxattr failed %s -> %s",
+                   cached_subvol->name, loc->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        /*
-          Parallel migration can lead to migration of the hard link multiple
-          times which can lead to data loss. Hence, adding a fresh lookup to
-          decide whether migration is required or not.
-
-          Elaborating the scenario for let say 10 hardlinks [link{1..10}]:
-              Let say the first hard link "link1"  does the setxattr of the
-          new hashed subvolume info on the cached file. As there are multiple
-          threads working, we might have already all the links created on the
-          new hashed by the time we reach hardlink let say link5. Now the
-          number of links on hashed is equal to that of cached. Hence, file
-          migration will happen for link6.
-
-                 Cached                                 Hashed
-          --------T link6                        rwxrwxrwx   link6
-
-          Now post above state all the link file on the cached will be zero
-          byte linkto files. Hence, if we still do migration for the following
-          files link{7..10}, we will end up migrating 0 data leading to data
-          loss.
-                Hence, a lookup can make sure whether we need to migrate the
-          file or not.
-        */
+        gf_msg_debug(this->name, 0,
+                     "hardlink target subvol created on %s "
+                     ",cached %s, file %s",
+                     hashed_subvol->name, cached_subvol->name, loc->path);
 
-        ret = syncop_lookup (this, loc, NULL, NULL,
-                                             NULL, NULL);
-        if (ret) {
-                /*Ignore ENOENT and ESTALE as file might have been
-                  migrated already*/
-                if (-ret == ENOENT || -ret == ESTALE) {
-                        ret = -2;
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:%s lookup failed with ret = %d",
-                        loc->path, ret);
-                ret = -1;
-                goto out;
+        ret = -2;
+        goto out;
+    } else {
+        linkto_subvol = dht_linkfile_subvol(this, NULL, NULL, xattr_rsp);
+        if (!linkto_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR,
+                   "Failed to get "
+                   "linkto subvol for %s",
+                   loc->name);
+        } else {
+            hashed_subvol = linkto_subvol;
         }
 
-        cached_subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!cached_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "Failed to get cached subvol"
-                        " for %s on %s", loc->name, this->name);
+        ret = syncop_link(hashed_subvol, loc, loc, &iatt, NULL, NULL);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+
+            loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_ERROR;
+            gf_msg(this->name, loglevel, op_errno,
+                   DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED,
+                   "link of %s -> %s"
+                   " failed on  subvol %s",
+                   loc->name, uuid_utoa(loc->gfid), hashed_subvol->name);
+            if (op_errno != EEXIST) {
+                *fop_errno = op_errno;
                 goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "syncop_link successful for"
+                         " hardlink %s on subvol %s, cached %s",
+                         loc->path, hashed_subvol->name, cached_subvol->name);
         }
+    }
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (!hashed_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "Failed to get hashed subvol"
-                        " for %s on %s", loc->name, this->name);
-                goto out;
-        }
+    ret = syncop_lookup(hashed_subvol, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :Failed lookup %s on %s ", loc->name,
+               hashed_subvol->name);
 
-        if (hashed_subvol == cached_subvol) {
-                ret = -2;
-                goto out;
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* There is a race where on the target subvol for the hardlink
+     * (note: hash subvol for the hardlink might differ from this), some
+     * other client(non-rebalance) would have created a linkto file for that
+     * hardlink as part of lookup. So let say there are 10 hardlinks, on the
+     * 5th hardlink it self the hardlinks might have migrated. Now for
+     * (6..10th) hardlinks the cached and target would be same as the file
+     * has already migrated. Hence this check is needed  */
+    if (cached_subvol == hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "source %s and destination %s "
+                     "for hardlink %s are same",
+                     cached_subvol->name, hashed_subvol->name, loc->path);
+        ret = -2;
+        goto out;
+    }
+
+    if (iatt.ia_nlink == stbuf.ia_nlink) {
+        ret = dht_migrate_file(this, loc, cached_subvol, hashed_subvol,
+                               GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS, fop_errno);
+        if (ret) {
+            goto out;
         }
+    }
+    ret = -2;
+out:
+    if (link_xattr)
+        dict_unref(link_xattr);
 
-        gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
-                "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
-                cached_subvol->name, hashed_subvol->name);
-        data = dict_get (xattrs, conf->link_xattr_name);
-        /* set linkto on cached -> hashed if not present, else link it */
-        if (!data) {
-                ret = dict_set_str (link_xattr, conf->link_xattr_name,
-                                    hashed_subvol->name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed :"
-                                "Failed to set dictionary value:"
-                                " key = %s for %s",
-                                conf->link_xattr_name, loc->name);
-                        goto out;
-                }
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
 
-                ret = syncop_setxattr (cached_subvol, loc, link_xattr, 0, NULL,
-                                       NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed :"
-                                "Linkto setxattr failed %s -> %s (%s)",
-                                cached_subvol->name,
-                                loc->name, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
-                ret = -2;
-                goto out;
-        } else {
-                linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
-                if (!linkto_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_SUBVOL_ERROR,
-                                "Failed to get "
-                                "linkto subvol for %s", loc->name);
-                } else {
-                        hashed_subvol = linkto_subvol;
-                }
+    if (dict)
+        dict_unref(dict);
 
-                ret = syncop_link (hashed_subvol, loc, loc, &iatt, NULL, NULL);
-                if  (ret) {
-                        op_errno = -ret;
-                        ret = -1;
+    return ret;
+}
 
-                        loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : \
-                                    GF_LOG_ERROR;
-                        gf_msg (this->name, loglevel, op_errno,
-                                DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED,
-                                "link of %s -> %s"
-                                " failed on  subvol %s", loc->name,
-                                uuid_utoa(loc->gfid),
-                                hashed_subvol->name);
-                        if (op_errno != EEXIST)
-                                goto out;
-                }
-        }
-        ret = syncop_lookup (hashed_subvol, loc, &iatt, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :Failed lookup %s on %s ",
-                        loc->name, hashed_subvol->name);
+static int
+__check_file_has_hardlink(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                          dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                          dht_conf_t *conf, int *fop_errno)
+{
+    int ret = 0;
 
-                ret = -1;
-                goto out;
+    if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
+        ret = 0;
+        return ret;
+    }
+    if (stbuf->ia_nlink > 1) {
+        /* support for decomission */
+        if (flags == GF_DHT_MIGRATE_HARDLINK) {
+            synclock_lock(&conf->link_lock);
+            ret = gf_defrag_handle_hardlink(this, loc, fop_errno);
+            synclock_unlock(&conf->link_lock);
+            /*
+            Returning zero will force the file to be remigrated.
+            Checkout gf_defrag_handle_hardlink for more information.
+            */
+            if (ret && ret != -2) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "Migrate file failed:"
+                       "%s: failed to migrate file with link",
+                       loc->path);
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migration skipped for:"
+                   "%s: file has hardlinks",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
         }
+    }
 
-        if (iatt.ia_nlink == stbuf->ia_nlink) {
-                ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,
-                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS);
-                if (ret)
-                        goto out;
-        }
-        ret = -2;
-out:
-        if (link_xattr)
-                dict_unref (link_xattr);
-        return ret;
+    return ret;
 }
 
 /*
@@ -386,787 +504,963 @@ out:
     -1 : failure
 */
 static int
-__is_file_migratable (xlator_t *this, loc_t *loc,
-                      struct iatt *stbuf, dict_t *xattrs, int flags,
-                                gf_defrag_info_t *defrag)
+__is_file_migratable(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                     dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                     dht_conf_t *conf, int *fop_errno)
 {
-        int ret = -1;
-        int lock_count = 0;
-
-        if (IA_ISDIR (stbuf->ia_type)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: migrate-file called on directory", loc->path);
-                ret = -1;
-                goto out;
-        }
-
-        if (!defrag->lock_migration_enabled) {
-                ret = dict_get_int32 (xattrs, GLUSTERFS_POSIXLK_COUNT,
-                                      &lock_count);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: Unable to get lock count for file",
-                                loc->path);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (lock_count) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed: %s: File has locks."
-                                " Skipping file migration", loc->path);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
-                ret = 0;
-                goto out;
-        }
-
-        if (stbuf->ia_nlink > 1) {
-                /* support for decomission */
-                if (flags == GF_DHT_MIGRATE_HARDLINK) {
-                        synclock_lock (&defrag->link_lock);
-                        ret = gf_defrag_handle_hardlink
-                                (this, loc, xattrs, stbuf);
-                        synclock_unlock (&defrag->link_lock);
-                        /*
-                        Returning zero will force the file to be remigrated.
-                        Checkout gf_defrag_handle_hardlink for more information.
-                        */
-                        if (ret && ret != -2) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "Migrate file failed:"
-                                        "%s: failed to migrate file with link",
-                                        loc->path);
-                        }
-                } else {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: file has hardlinks", loc->path);
-                        ret = -ENOTSUP;
-                }
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    int lock_count = 0;
+
+    if (IA_ISDIR(stbuf->ia_type)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: migrate-file called on directory",
+               loc->path);
+        *fop_errno = EISDIR;
+        ret = -1;
+        goto out;
+    }
 
+    if (!conf->lock_migration_enabled) {
+        ret = dict_get_int32(xattrs, GLUSTERFS_POSIXLK_COUNT, &lock_count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Unable to get lock count for file",
+                   loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+
+        if (lock_count) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: File has locks."
+                   " Skipping file migration",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
+            goto out;
+        }
+    }
+
+    /* Check if file has hardlink*/
+    ret = __check_file_has_hardlink(this, loc, stbuf, xattrs, flags, defrag,
+                                    conf, fop_errno);
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-__dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
-                                 fd_t **dst_fd, dict_t *xattr)
+__dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from,
+                                loc_t *loc, struct iatt *stbuf, fd_t **dst_fd,
+                                int *fop_errno, int file_has_holes)
 {
-        xlator_t    *this = NULL;
-        int          ret  = -1;
-        fd_t        *fd   = NULL;
-        struct iatt  new_stbuf = {0,};
-        struct iatt  check_stbuf= {0,};
-        dht_conf_t  *conf = NULL;
-        dict_t      *dict = NULL;
-
-        this = THIS;
-        conf = this->private;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: failed to set dictionary value: key = gfid-req",
-                        loc->path);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: failed to set dictionary value: key = %s ",
-                        loc->path, conf->link_xattr_name);
-                goto out;
-        }
-
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: fd create failed (destination) (%s)",
-                        loc->path, strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_lookup (to, loc, &new_stbuf, NULL, NULL, NULL);
-        if (!ret) {
-                /* File exits in the destination, check if gfid matches */
-                if (gf_uuid_compare (stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "file %s exists in %s with different gfid",
-                                loc->path, to->name);
-                        ret = -1;
-                        goto out;
-                }
-        }
-        if ((ret < 0) && (-ret != ENOENT)) {
-                /* File exists in destination, but not accessible */
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to lookup file (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
-
-        /* Create the destination with LINKFILE mode, and linkto xattr,
-           if the linkfile already exists, just open the file */
-        if (!ret) {
-                /*
-                 * File already present, just open the file.
-                 */
-                ret = syncop_open (to, loc, O_RDWR, fd, NULL, NULL);
-                 if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "failed to open %s on %s",
-                                loc->path, to->name);
-                        ret = -1;
-                        goto out;
-                 }
-        } else {
-                ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
-                                     &new_stbuf, dict, NULL);
-                 if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "failed to create %s on %s",
-                                loc->path, to->name);
-                        ret = -1;
-                        goto out;
-                }
-
-        }
-
-        fd_bind (fd);
+    int ret = -1;
+    int ret2 = -1;
+    fd_t *fd = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt check_stbuf = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    dict_t *dict = NULL;
+    dict_t *xdata = NULL;
+
+    conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dictionary allocation failed for"
+               "path:%s",
+               loc->path);
+        goto out;
+    }
+    ret = dict_set_gfuuid(dict, "gfid-req", stbuf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = gfid-req", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, from->name);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               conf->link_xattr_name);
+        goto out;
+    }
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (destination)", loc->path);
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: dict_new failed)", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(xdata, GF_CLEAN_WRITE_PROTECTION, 1);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               GF_CLEAN_WRITE_PROTECTION);
+        goto out;
+    }
+
+    ret = syncop_lookup(to, loc, &new_stbuf, NULL, xdata, NULL);
+    if (!ret) {
+        /* File exits in the destination, check if gfid matches */
+        if (gf_uuid_compare(stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid", loc->path,
+                   to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+    }
+    if ((ret < 0) && (-ret != ENOENT)) {
+        /* File exists in destination, but not accessible */
+        gf_msg(THIS->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to lookup file", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        /*Reason of doing lookup after create again:
-         *In the create, there is some time-gap between opening fd at the
-         *server (posix_layer) and binding it in server (incrementing fd count),
-         *so if in that time-gap, if other process sends unlink considering it
-         *as a linkto file, because inode->fd count will be 0, so file will be
-         *unlinked at the backend. And because furthur operations are performed
-         *on fd, so though migration will be done but will end with no file
-         *at  the backend.
+    /* Create the destination with LINKFILE mode, and linkto xattr,
+       if the linkfile already exists, just open the file */
+    if (!ret) {
+        /*
+         * File already present, just open the file.
          */
-
-        ret = syncop_lookup (to, loc, &check_stbuf, NULL, NULL, NULL);
-        if (!ret) {
-
-                if (gf_uuid_compare (stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "file %s exists in %s with different gfid,"
-                                "found in lookup after create",
-                                loc->path, to->name);
-                        ret = -1;
-                        goto out;
+        ret = syncop_open(to, loc, O_RDWR, fd, NULL, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to open %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    } else {
+        ret = syncop_create(to, loc, O_RDWR, DHT_LINKFILE_MODE, fd, &new_stbuf,
+                            dict, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to create %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    fd_bind(fd);
+
+    /*Reason of doing lookup after create again:
+     *In the create, there is some time-gap between opening fd at the
+     *server (posix_layer) and binding it in server (incrementing fd count),
+     *so if in that time-gap, if other process sends unlink considering it
+     *as a linkto file, because inode->fd count will be 0, so file will be
+     *unlinked at the backend. And because further operations are performed
+     *on fd, so though migration will be done but will end with no file
+     *at  the backend.
+     */
+
+    ret = syncop_lookup(to, loc, &check_stbuf, NULL, NULL, NULL);
+    if (!ret) {
+        if (gf_uuid_compare(stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid,"
+                   "found in lookup after create",
+                   loc->path, to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (-ret == ENOENT) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: file does not exist"
+               "on %s",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fsetattr(to, fd, stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                          NULL, NULL, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "chown failed for %s on %s", loc->path, to->name);
+    }
+
+    /* No need to bother about 0 byte size files */
+    if (stbuf->ia_size > 0) {
+        if (conf->use_fallocate && !file_has_holes) {
+            ret = syncop_fallocate(to, fd, 0, 0, stbuf->ia_size, NULL, NULL);
+            if (ret < 0) {
+                if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -ENOSYS) {
+                    conf->use_fallocate = _gf_false;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "fallocate failed for %s on %s", loc->path,
+                           to->name);
+
+                    *fop_errno = -ret;
+
+                    /* fallocate does not release the space
+                     * in some cases
+                     */
+                    ret2 = syncop_ftruncate(to, fd, 0, NULL, NULL, NULL, NULL);
+                    if (ret2 < 0) {
+                        gf_msg(this->name, GF_LOG_WARNING, -ret2,
+                               DHT_MSG_MIGRATE_FILE_FAILED,
+                               "ftruncate failed for "
+                               "%s on %s",
+                               loc->path, to->name);
+                    }
+                    goto out;
                 }
-
-        }
-
-        if (-ret == ENOENT) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED, "%s: file does not exists"
-                        "on %s (%s)", loc->path, to->name, strerror (-ret));
-                ret = -1;
-                goto out;
+            }
+        } else {
+            ret = syncop_ftruncate(to, fd, stbuf->ia_size, NULL, NULL, NULL,
+                                   NULL);
+            if (ret < 0) {
+                *fop_errno = -ret;
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "ftruncate failed for %s on %s", loc->path, to->name);
+            }
         }
+    }
 
-        ret = syncop_fsetxattr (to, fd, xattr, 0, NULL, NULL);
-        if (ret < 0)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to set xattr on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-
-        ret = syncop_ftruncate (to, fd, stbuf->ia_size, NULL, NULL);
-        if (ret < 0)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "ftruncate failed for %s on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-
-        ret = syncop_fsetattr (to, fd, stbuf,
-                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
-                                NULL, NULL, NULL, NULL);
-        if (ret < 0)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "chown failed for %s on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-
-        /* success */
-        ret = 0;
+    /* success */
+    ret = 0;
 
-        if (dst_fd)
-                *dst_fd = fd;
+    if (dst_fd)
+        *dst_fd = fd;
 
 out:
-        if (ret) {
-                if (fd) {
-                        fd_unref (fd);
-                }
+    if (ret) {
+        if (fd) {
+            fd_unref(fd);
         }
-        if (dict)
-                dict_unref (dict);
+    }
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+
+    return ret;
 }
 
 static int
-__dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
-                        struct iatt *stbuf, int flag)
+__dht_check_free_space(xlator_t *this, xlator_t *to, xlator_t *from, loc_t *loc,
+                       struct iatt *stbuf, int flag, dht_conf_t *conf,
+                       gf_boolean_t *target_changed, xlator_t **new_subvol,
+                       int *fop_errno)
 {
-        struct statvfs  src_statfs = {0,};
-        struct statvfs  dst_statfs = {0,};
-        int             ret        = -1;
-        xlator_t       *this       = NULL;
-        dict_t         *xdata      = NULL;
-
-        uint64_t        src_statfs_blocks = 1;
-        uint64_t        dst_statfs_blocks = 1;
-
-        this = THIS;
-
-        xdata = dict_new ();
-        if (!xdata) {
-                errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY,
-                        "failed to allocate dictionary");
-                goto out;
-        }
+    struct statvfs src_statfs = {
+        0,
+    };
+    struct statvfs dst_statfs = {
+        0,
+    };
+    int ret = -1;
+    dict_t *xdata = NULL;
+    dht_layout_t *layout = NULL;
+    uint64_t src_statfs_blocks = 1;
+    uint64_t dst_statfs_blocks = 1;
+    double dst_post_availspacepercent = 0;
+    double src_post_availspacepercent = 0;
+    uint64_t file_blocks = 0;
+    uint64_t src_total_blocks = 0;
+    uint64_t dst_total_blocks = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "failed to allocate dictionary");
+        goto out;
+    }
+
+    ret = dict_set_int8(xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_statfs(from, loc, &src_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_statfs(to, loc, &dst_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "min_free_disk - %f , block available - %" PRId64
+                 ", block size - %lu",
+                 conf->min_free_disk, dst_statfs.f_bavail, dst_statfs.f_bsize);
+
+    dst_statfs_blocks = dst_statfs.f_bavail *
+                        (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_statfs_blocks = src_statfs.f_bavail *
+                        (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    dst_total_blocks = dst_statfs.f_blocks *
+                       (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_total_blocks = src_statfs.f_blocks *
+                       (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    /* if force option is given, do not check for space @ dst.
+     * Check only if space is avail for the file */
+    if (flag != GF_DHT_MIGRATE_DATA)
+        goto check_avail_space;
+
+    /* Check:
+       During rebalance `migrate-data` - Destination subvol experiences
+       a `reduction` in 'blocks' of free space, at the same time source
+       subvol gains certain 'blocks' of free space. A valid check is
+       necessary here to avoid erroneous move to destination where
+       the space could be scantily available.
+       With heterogeneous brick support, an actual space comparison could
+       prevent any files being migrated to newly added bricks if they are
+       smaller then the free space available on the existing bricks.
+     */
+    if (!conf->use_fallocate) {
+        file_blocks = stbuf->ia_size + GF_DISK_SECTOR_SIZE - 1;
+        file_blocks /= GF_DISK_SECTOR_SIZE;
+
+        if (file_blocks >= dst_statfs_blocks) {
+            dst_statfs_blocks = 0;
+        } else {
+            dst_statfs_blocks -= file_blocks;
+        }
+    }
+
+    src_post_availspacepercent = ((src_statfs_blocks + file_blocks) * 100) /
+                                 src_total_blocks;
+
+    dst_post_availspacepercent = (dst_statfs_blocks * 100) / dst_total_blocks;
+
+    if (dst_post_availspacepercent < src_post_availspacepercent) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "data movement of file "
+               "{blocks:%" PRIu64
+               " name:(%s)} would result in "
+               "dst node (%s:%" PRIu64
+               ") having lower disk "
+               "space than the source node (%s:%" PRIu64
+               ")"
+               ".Skipping file.",
+               stbuf->ia_blocks, loc->path, to->name, dst_statfs_blocks,
+               from->name, src_statfs_blocks);
+
+        /* this is not a 'failure', but we don't want to
+           consider this as 'success' too :-/ */
+        *fop_errno = ENOSPC;
+        ret = 1;
+        goto out;
+    }
 
-        ret = dict_set_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set "
-                        GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                ret = -1;
-                goto out;
+check_avail_space:
+    if (conf->disk_unit == 'p' && dst_statfs.f_blocks) {
+        dst_post_availspacepercent = (dst_statfs_blocks * 100) /
+                                     dst_total_blocks;
+
+        gf_msg_debug(this->name, 0,
+                     "file : %s, post_availspacepercent"
+                     " : %lf f_bavail : %" PRIu64 " min-free-disk: %lf",
+                     loc->path, dst_post_availspacepercent, dst_statfs.f_bavail,
+                     conf->min_free_disk);
+
+        if (dst_post_availspacepercent < conf->min_free_disk) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "Write will cross min-free-disk for "
+                   "file - %s on subvol - %s. Looking "
+                   "for new subvol",
+                   loc->path, to->name);
+
+            goto find_new_subvol;
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
-        ret = syncop_statfs (from, loc, &src_statfs, xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to get statfs of %s on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+    if (conf->disk_unit != 'p') {
+        if ((dst_statfs_blocks * GF_DISK_SECTOR_SIZE) < conf->min_free_disk) {
+            gf_msg_debug(this->name, 0,
+                         "file : %s,  destination frsize: %lu "
+                         "f_bavail : %" PRIu64 " min-free-disk: %lf",
+                         loc->path, dst_statfs.f_frsize, dst_statfs.f_bavail,
+                         conf->min_free_disk);
 
-        ret = syncop_statfs (to, loc, &dst_statfs, xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to get statfs of %s on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "write will"
+                   " cross min-free-disk for file - %s on subvol -"
+                   " %s. looking for new subvol",
+                   loc->path, to->name);
 
-        /* if force option is given, do not check for space @ dst.
-         * Check only if space is avail for the file */
-        if (flag != GF_DHT_MIGRATE_DATA)
-                goto check_avail_space;
-
-        /* Check:
-           During rebalance `migrate-data` - Destination subvol experiences
-           a `reduction` in 'blocks' of free space, at the same time source
-           subvol gains certain 'blocks' of free space. A valid check is
-           necessary here to avoid errorneous move to destination where
-           the space could be scantily available.
-         */
-        if (stbuf) {
-                dst_statfs_blocks = ((dst_statfs.f_bavail *
-                                      dst_statfs.f_bsize) /
-                                     GF_DISK_SECTOR_SIZE);
-                src_statfs_blocks = ((src_statfs.f_bavail *
-                                      src_statfs.f_bsize) /
-                                     GF_DISK_SECTOR_SIZE);
-                if ((dst_statfs_blocks - stbuf->ia_blocks) <
-                    (src_statfs_blocks + stbuf->ia_blocks)) {
-
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "data movement attempted from node "
-                                "(%s:%"PRIu64") with higher disk space "
-                                "to a node (%s:%"PRIu64") with lesser "
-                                "disk space, file { blocks:%"PRIu64", "
-                                "name:(%s) }", from->name, src_statfs_blocks,
-                                to->name, dst_statfs_blocks,
-                                stbuf->ia_blocks, loc->path);
-
-                        /* this is not a 'failure', but we don't want to
-                           consider this as 'success' too :-/ */
-                        ret = -1;
-                        goto out;
-                }
-        }
-check_avail_space:
-        if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "data movement attempted from node (%s) to node (%s) "
-                        "which does not have required free space for (%s)",
-                        from->name, to->name, loc->path);
-                ret = -1;
-                goto out;
+            goto find_new_subvol;
+
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
+find_new_subvol:
+    layout = dht_layout_get(this, loc->parent);
+    if (!layout) {
+        gf_log(this->name, GF_LOG_ERROR, "Layout is NULL");
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    *new_subvol = dht_subvol_with_free_space_inodes(this, to, from, layout,
+                                                    stbuf->ia_size);
+    if ((!(*new_subvol)) || (*new_subvol == from)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+               "Could not find any subvol"
+               " with space accommodating the file - %s. Consider "
+               "adding bricks",
+               loc->path);
+
+        *target_changed = _gf_false;
+        *fop_errno = ENOSPC;
+        ret = -1;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "new target found - %s"
+               " for file - %s",
+               (*new_subvol)->name, loc->path);
+        *target_changed = _gf_true;
         ret = 0;
+    }
+
 out:
-        if (xdata)
-                dict_unref (xdata);
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+    return ret;
 }
 
 static int
-__dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
-                              uint64_t ia_size, int hole_exists)
+__dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag,
+                             xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
+                             uint64_t ia_size, int hole_exists, int *fop_errno)
 {
-        int            ret    = 0;
-        int            count  = 0;
-        off_t          offset = 0;
-        struct iovec  *vector = NULL;
-        struct iobref *iobref = NULL;
-        uint64_t       total  = 0;
-        size_t         read_size = 0;
-
-        /* if file size is '0', no need to enter this loop */
-        while (total < ia_size) {
-                read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ?
-                             DHT_REBALANCE_BLKSIZE : (ia_size - total));
-
-                ret = syncop_readv (from, src, read_size,
-                                    offset, 0, &vector, &count, &iobref, NULL,
-                                    NULL);
-                if (!ret || (ret < 0)) {
-                        break;
+    int ret = 0;
+    int count = 0;
+    off_t offset = 0;
+    off_t data_offset = 0;
+    off_t hole_offset = 0;
+    struct iovec *vector = NULL;
+    struct iobref *iobref = NULL;
+    uint64_t total = 0;
+    size_t read_size = 0;
+    size_t data_block_size = 0;
+    dict_t *xdata = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    /* if file size is '0', no need to enter this loop */
+    while (total < ia_size) {
+        /* This is a regular file - read it sequentially */
+        if (!hole_exists) {
+            read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : (ia_size - total));
+        } else {
+            /* This is a sparse file - read only the data segments in the file
+             */
+
+            /* If the previous data block is fully copied, find the next data
+             * segment
+             * starting at the offset of the last read and written byte,  */
+            if (data_block_size <= 0) {
+                ret = syncop_seek(from, src, offset, GF_SEEK_DATA, NULL,
+                                  &data_offset);
+                if (ret) {
+                    if (ret == -ENXIO)
+                        ret = 0; /* No more data segments */
+                    else
+                        *fop_errno = -ret; /* Error occurred */
+
+                    break;
                 }
 
-                if (hole_exists)
-                        ret = dht_write_with_holes (to, dst, vector, count,
-                                                    ret, offset, iobref);
-                else
-                        ret = syncop_writev (to, dst, vector, count,
-                                             offset, iobref, 0, NULL, NULL);
-                if (ret < 0) {
+                /* If the position of the current data segment is greater than
+                 * the position of the next hole, find the next hole in order to
+                 * calculate the length of the new data segment */
+                if (data_offset > hole_offset) {
+                    /* Starting at the offset of the last data segment, find the
+                     * next hole */
+                    ret = syncop_seek(from, src, data_offset, GF_SEEK_HOLE,
+                                      NULL, &hole_offset);
+                    if (ret) {
+                        /* If an error occurred here it's a real error because
+                         * if the seek for a data segment was successful then
+                         * necessarily another hole must exist (EOF is a hole)
+                         */
+                        *fop_errno = -ret;
                         break;
-                }
-                offset += ret;
-                total += ret;
-
-                GF_FREE (vector);
-                if (iobref)
-                        iobref_unref (iobref);
-                iobref = NULL;
-                vector = NULL;
-        }
-        if (iobref)
-                iobref_unref (iobref);
-        GF_FREE (vector);
+                    }
 
-        if (ret >= 0)
-                ret = 0;
-        else
+                    /* Calculate the total size of the current data block */
+                    data_block_size = hole_offset - data_offset;
+                }
+            } else {
+                /* There is still data in the current segment, move the
+                 * data_offset to the position of the last written byte */
+                data_offset = offset;
+            }
+
+            /* Calculate how much data needs to be read and written. If the data
+             * segment's length is bigger than DHT_REBALANCE_BLKSIZE, read and
+             * write DHT_REBALANCE_BLKSIZE data length and the rest in the
+             * next iteration(s) */
+            read_size = ((data_block_size > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : data_block_size);
+
+            /* Calculate the remaining size of the data block - maybe there's no
+             * need to seek for data in the next iteration */
+            data_block_size -= read_size;
+
+            /* Set offset to the offset of the data segment so read and write
+             * will have the correct position */
+            offset = data_offset;
+        }
+
+        ret = syncop_readv(from, src, read_size, offset, 0, &vector, &count,
+                           &iobref, NULL, NULL, NULL);
+
+        if (!ret || (ret < 0)) {
+            if (!ret) {
+                /* File was probably truncated*/
                 ret = -1;
-
-        return ret;
-}
-
-static int
-__tier_migrate_data (gf_defrag_info_t *defrag, xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
-                     uint64_t ia_size, int hole_exists)
-{
-        int            ret    = 0;
-        int            count  = 0;
-        off_t          offset = 0;
-        struct iovec  *vector = NULL;
-        struct iobref *iobref = NULL;
-        uint64_t       total  = 0;
-        size_t         read_size = 0;
-
-        /* if file size is '0', no need to enter this loop */
-        while (total < ia_size) {
-
-                read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ?
-                             DHT_REBALANCE_BLKSIZE : (ia_size - total));
-
-                ret = syncop_readv (from, src, read_size,
-                                    offset, 0, &vector, &count, &iobref, NULL,
-                                    NULL);
-                if (!ret || (ret < 0)) {
-                        break;
+                *fop_errno = ENOSPC;
+            } else {
+                *fop_errno = -ret;
+            }
+            break;
+        }
+
+        if (!conf->force_migration) {
+            if (!xdata) {
+                xdata = dict_new();
+                if (!xdata) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                           "insufficient memory");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
                 }
 
-                if (hole_exists)
-                        ret = dht_write_with_holes (to, dst, vector, count,
-                                                    ret, offset, iobref);
-                else
-                        ret = syncop_writev (to, dst, vector, count,
-                                             offset, iobref, 0, NULL, NULL);
-                if (gf_defrag_get_pause_state (&defrag->tier_conf) != TIER_RUNNING) {
-                        gf_msg ("tier", GF_LOG_INFO, 0,
-                                DHT_MSG_TIER_PAUSED,
-                                "Migrate file paused");
-                        ret = -1;
+                /* Fail this write and abort rebalance if we
+                 * detect a write from client since migration of
+                 * this file started. This is done to avoid
+                 * potential data corruption due to out of order
+                 * writes from rebalance and client to the same
+                 * region (as compared between src and dst
+                 * files). See
+                 * https://github.com/gluster/glusterfs/issues/308
+                 * for more details.
+                 */
+                ret = dict_set_int32_sizen(xdata, GF_AVOID_OVERWRITE, 1);
+                if (ret) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, ENOMEM,
+                           "failed to set dict");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
                 }
+            }
+        }
 
-                if (ret < 0) {
-                        break;
-                }
-                offset += ret;
-                total += ret;
-
-                GF_FREE (vector);
-                if (iobref)
-                        iobref_unref (iobref);
-                iobref = NULL;
-                vector = NULL;
+        ret = syncop_writev(to, dst, vector, count, offset, iobref, 0, NULL,
+                            NULL, xdata, NULL);
+        if (ret < 0) {
+            *fop_errno = -ret;
+            break;
         }
+
+        offset += ret;
+        total += ret;
+
+        GF_FREE(vector);
         if (iobref)
-                iobref_unref (iobref);
-        GF_FREE (vector);
+            iobref_unref(iobref);
+        iobref = NULL;
+        vector = NULL;
+    }
+    if (iobref)
+        iobref_unref(iobref);
+    GF_FREE(vector);
+
+    if (ret >= 0)
+        ret = 0;
+    else
+        ret = -1;
 
-        if (ret >= 0)
-                ret = 0;
-        else
-                ret = -1;
+    if (xdata) {
+        dict_unref(xdata);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 static int
-__dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
-                               struct iatt *stbuf, fd_t **src_fd,
-                               gf_boolean_t *clean_src)
+__dht_rebalance_open_src_file(xlator_t *this, xlator_t *from, xlator_t *to,
+                              loc_t *loc, struct iatt *stbuf, fd_t **src_fd,
+                              gf_boolean_t *clean_src, int *fop_errno)
 {
-        int          ret  = 0;
-        fd_t        *fd   = NULL;
-        dict_t      *dict = NULL;
-        xlator_t    *this = NULL;
-        struct iatt  iatt = {0,};
-        dht_conf_t  *conf = NULL;
-
-        this = THIS;
-        conf = this->private;
-
-        *clean_src = _gf_false;
-
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: fd create failed (source)", loc->path);
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_open (from, loc, O_RDWR, fd, NULL, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to open file %s on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
-
-        fd_bind (fd);
-
-        if (src_fd)
-                *src_fd = fd;
-
+    int ret = 0;
+    fd_t *fd = NULL;
+    dict_t *dict = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    *clean_src = _gf_false;
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (source)", loc->path);
+        *fop_errno = ENOMEM;
         ret = -1;
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set xattr in dict for %s (linkto:%s)",
-                        loc->path, to->name);
-                goto out;
-        }
-
-        /* Once the migration starts, the source should have 'linkto' key set
-           to show which is the target, so other clients can work around it */
-        ret = syncop_setxattr (from, loc, dict, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to set xattr on %s in %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+        goto out;
+    }
+
+    ret = syncop_open(from, loc, O_RDWR, fd, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to open file %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        /* Reset source mode/xattr if migration fails*/
-        *clean_src = _gf_true;
+    fd_bind(fd);
 
-        /* mode should be (+S+T) to indicate migration is in progress */
-        iatt.ia_prot = stbuf->ia_prot;
-        iatt.ia_type = stbuf->ia_type;
-        iatt.ia_prot.sticky = 1;
-        iatt.ia_prot.sgid = 1;
+    if (src_fd)
+        *src_fd = fd;
 
-        ret = syncop_setattr (from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL,
-                              NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to set mode on %s in %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+    ret = -1;
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: Could not allocate memory for dict", loc->path);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, to->name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+               to->name);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Once the migration starts, the source should have 'linkto' key set
+       to show which is the target, so other clients can work around it */
+    ret = syncop_setxattr(from, loc, dict, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set xattr on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Reset source mode/xattr if migration fails*/
+    *clean_src = _gf_true;
+
+    /* mode should be (+S+T) to indicate migration is in progress */
+    iatt.ia_prot = stbuf->ia_prot;
+    iatt.ia_type = stbuf->ia_type;
+    iatt.ia_prot.sticky = 1;
+    iatt.ia_prot.sgid = 1;
+
+    ret = syncop_setattr(from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL, NULL,
+                         NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set mode on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        /* success */
-        ret = 0;
+    /* success */
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int
-migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
-                       struct iatt *buf)
+migrate_special_files(xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
+                      struct iatt *buf, int *fop_errno)
 {
-        int          ret      = -1;
-        dict_t      *rsp_dict = NULL;
-        dict_t      *dict     = NULL;
-        char        *link     = NULL;
-        struct iatt  stbuf    = {0,};
-        dht_conf_t  *conf     = this->private;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
-        }
-
-        /* check in the destination if the file is link file */
-        ret = syncop_lookup (to, loc, &stbuf, NULL, dict, &rsp_dict);
-        if ((ret < 0) && (-ret != ENOENT)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: lookup failed (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
-
-        /* we no more require this key */
-        dict_del (dict, conf->link_xattr_name);
-
-        /* file exists in target node, only if it is 'linkfile' its valid,
-           otherwise, error out */
-        if (!ret) {
-                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
-                                        conf->link_xattr_name)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: file exists in destination", loc->path);
-                        ret = -1;
-                        goto out;
-                }
-
-                /* as file is linkfile, delete it */
-                ret = syncop_unlink (to, loc, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to delete the linkfile (%s)",
-                                loc->path, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        /* Set the gfid of the source file in dict */
-        ret = dict_set_static_bin (dict, "gfid-req", buf->ia_gfid, 16);
+    int ret = -1;
+    dict_t *rsp_dict = NULL;
+    dict_t *dict = NULL;
+    char *link = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dht_conf_t *conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set 'linkto' key in dict", loc->path);
+        goto out;
+    }
+
+    /* check in the destination if the file is link file */
+    ret = syncop_lookup(to, loc, &stbuf, NULL, dict, &rsp_dict);
+    if ((ret < 0) && (-ret != ENOENT)) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: lookup failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* we no more require this key */
+    dict_del(dict, conf->link_xattr_name);
+
+    /* file exists in target node, only if it is 'linkfile' its valid,
+       otherwise, error out */
+    if (!ret) {
+        if (!check_is_linkfile(loc->inode, &stbuf, rsp_dict,
+                               conf->link_xattr_name)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: file exists in destination", loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+
+        /* as file is linkfile, delete it */
+        ret = syncop_unlink(to, loc, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set gfid in dict for create", loc->path);
-                goto out;
-        }
-
-        /* Create the file in target */
-        if (IA_ISLNK (buf->ia_type)) {
-                /* Handle symlinks separately */
-                ret = syncop_readlink (from, loc, &link, buf->ia_size, NULL,
-                                       NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: readlink on symlink failed (%s)",
-                                loc->path, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_symlink (to, loc, link, 0, dict, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: creating symlink failed (%s)",
-                                loc->path, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
-
-                goto done;
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to delete the linkfile", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* Set the gfid of the source file in dict */
+    ret = dict_set_gfuuid(dict, "gfid-req", buf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set gfid in dict for create", loc->path);
+        goto out;
+    }
+
+    /* Create the file in target */
+    if (IA_ISLNK(buf->ia_type)) {
+        /* Handle symlinks separately */
+        ret = syncop_readlink(from, loc, &link, buf->ia_size, NULL, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: readlink on symlink failed", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
-                                                      buf->ia_type),
-                            makedev (ia_major (buf->ia_rdev),
-                                     ia_minor (buf->ia_rdev)), 0, dict, NULL);
+        ret = syncop_symlink(to, loc, link, 0, dict, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: mknod failed (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "%s: creating symlink failed",
+                   loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+
+        goto done;
+    }
+
+    ret = syncop_mknod(to, loc, st_mode_from_ia(buf->ia_prot, buf->ia_type),
+                       makedev(ia_major(buf->ia_rdev), ia_minor(buf->ia_rdev)),
+                       0, dict, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: mknod failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
 done:
-        ret = syncop_setattr (to, loc, buf,
-                              (GF_SET_ATTR_MTIME |
-                               GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                               GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
-        }
-
-        ret = syncop_unlink (from, loc, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: unlink failed (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-        }
+    ret = syncop_setattr(to, loc, buf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_UID |
+                          GF_SET_ATTR_GID | GF_SET_ATTR_MODE),
+                         NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to perform setattr on %s", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    ret = syncop_unlink(from, loc, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: unlink failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+    }
 
 out:
-        GF_FREE (link);
-        if (dict)
-                dict_unref (dict);
+    GF_FREE(link);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
 
 static int
-__dht_migration_cleanup_src_file (xlator_t *this, loc_t *loc, fd_t *fd,
-                                  xlator_t *from, ia_prot_t *src_ia_prot)
+__dht_migration_cleanup_src_file(xlator_t *this, loc_t *loc, fd_t *fd,
+                                 xlator_t *from, ia_prot_t *src_ia_prot)
 {
-        int ret                       = -1;
-        dht_conf_t     *conf          = NULL;
-        struct iatt     new_stbuf     = {0,};
-
-        if (!this || !fd || !from || !src_ia_prot) {
-                goto out;
-        }
-
-        conf = this->private;
-
-        /*Revert source mode and xattr changes*/
-        ret = syncop_fstat (from, fd, &new_stbuf, NULL, NULL);
-        if (ret < 0) {
-                /* Failed to get the stat info */
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file cleanup failed: failed to fstat "
-                        "file %s on %s ", loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
-
-
-        /* Remove the sticky bit and sgid bit set, reset it to 0*/
-        if (!src_ia_prot->sticky)
-                new_stbuf.ia_prot.sticky = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+
+    if (!this || !fd || !from || !src_ia_prot) {
+        goto out;
+    }
+
+    conf = this->private;
+
+    /*Revert source mode and xattr changes*/
+    ret = syncop_fstat(from, fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed: failed to fstat "
+               "file %s on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
 
-        if (!src_ia_prot->sgid)
-                new_stbuf.ia_prot.sgid = 0;
+    /* Remove the sticky bit and sgid bit set, reset it to 0*/
+    if (!src_ia_prot->sticky)
+        new_stbuf.ia_prot.sticky = 0;
 
-        ret = syncop_fsetattr (from, fd, &new_stbuf,
-                               (GF_SET_ATTR_GID | GF_SET_ATTR_MODE),
-                               NULL, NULL, NULL, NULL);
+    if (!src_ia_prot->sgid)
+        new_stbuf.ia_prot.sgid = 0;
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file cleanup failed:"
-                        "%s: failed to perform fsetattr on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
+    ret = syncop_fsetattr(from, fd, &new_stbuf,
+                          (GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+                          NULL, NULL);
 
-        ret = syncop_fremovexattr (from, fd, conf->link_xattr_name, 0, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to remove linkto xattr on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed:"
+               "%s: failed to perform fsetattr on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(from, fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to remove linkto xattr on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
-
 /*
   return values:
 
@@ -1175,3027 +1469,3234 @@ out:
     1 : not a failure, but we can't migrate data as of now
 */
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
-                  int flag)
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag, int *fop_errno)
 {
-        int                     ret                     = -1;
-        struct iatt             new_stbuf               = {0,};
-        struct iatt             stbuf                   = {0,};
-        struct iatt             empty_iatt              = {0,};
-        ia_prot_t               src_ia_prot             = {0,};
-        fd_t                    *src_fd                 = NULL;
-        fd_t                    *dst_fd                 = NULL;
-        dict_t                  *dict                   = NULL;
-        dict_t                  *xattr                  = NULL;
-        dict_t                  *xattr_rsp              = NULL;
-        int                     file_has_holes          = 0;
-        dht_conf_t              *conf                   = this->private;
-        int                     rcvd_enoent_from_src    = 0;
-        struct gf_flock         flock                   = {0, };
-        struct gf_flock         plock                   = {0, };
-        loc_t                   tmp_loc                 = {0, };
-        gf_boolean_t            locked                  = _gf_false;
-        gf_boolean_t            p_locked                = _gf_false;
-        int                     lk_ret                  = -1;
-        gf_defrag_info_t        *defrag                 =  NULL;
-        gf_boolean_t            clean_src               = _gf_false;
-        gf_boolean_t            clean_dst               = _gf_false;
-        int                     log_level               = GF_LOG_INFO;
-        gf_boolean_t            delete_src_linkto       = _gf_true;
-        lock_migration_info_t   locklist;
-        dict_t                  *meta_dict              = NULL;
-        gf_boolean_t            meta_locked             = _gf_false;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto out;
-
-        if (defrag->tier_conf.is_tier)
-                log_level = GF_LOG_TRACE;
+    int ret = -1;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt empty_iatt = {
+        0,
+    };
+    ia_prot_t src_ia_prot = {
+        0,
+    };
+    fd_t *src_fd = NULL;
+    fd_t *dst_fd = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xattr_rsp = NULL;
+    int file_has_holes = 0;
+    dht_conf_t *conf = this->private;
+    int rcvd_enoent_from_src = 0;
+    struct gf_flock flock = {
+        0,
+    };
+    struct gf_flock plock = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t inodelk_locked = _gf_false;
+    gf_boolean_t entrylk_locked = _gf_false;
+    gf_boolean_t p_locked = _gf_false;
+    int lk_ret = -1;
+    gf_defrag_info_t *defrag = NULL;
+    gf_boolean_t clean_src = _gf_false;
+    gf_boolean_t clean_dst = _gf_false;
+    int log_level = GF_LOG_INFO;
+    gf_boolean_t delete_src_linkto = _gf_true;
+    lock_migration_info_t locklist;
+    dict_t *meta_dict = NULL;
+    gf_boolean_t meta_locked = _gf_false;
+    gf_boolean_t target_changed = _gf_false;
+    xlator_t *new_target = NULL;
+    xlator_t *old_target = NULL;
+    xlator_t *hashed_subvol = NULL;
+    fd_t *linkto_fd = NULL;
+    dict_t *xdata = NULL;
+
+    if (from == to) {
+        gf_msg_debug(this->name, 0,
+                     "destination and source are same. file %s"
+                     " might have migrated already",
+                     loc->path);
+        ret = 0;
+        goto out;
+    }
 
-        gf_log (this->name,
-                log_level, "%s: attempting to move from %s to %s",
-                loc->path, from->name, to->name);
+    gf_log(this->name, log_level, "%s: attempting to move from %s to %s",
+           loc->path, from->name, to->name);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "Could not allocate memory for dict");
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    /* Do not migrate file in case lock migration is not enabled on the
+     * volume*/
+    if (!conf->lock_migration_enabled) {
+        ret = dict_set_int32(dict, GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t));
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
-        }
-
-
-        /* Do not migrate file in case lock migration is not enabled on the
-         * volume*/
-        if (!defrag->lock_migration_enabled) {
-                ret = dict_set_int32 (dict,
-                                 GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed: %s: failed to "
-                                "set "GLUSTERFS_POSIXLK_COUNT" key in dict",
-                                loc->path);
-                        goto out;
-                }
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "locks will be migrated"
-                        " for file: %s", loc->path);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: failed to "
+                   "set " GLUSTERFS_POSIXLK_COUNT " key in dict",
+                   loc->path);
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "locks will be migrated"
+               " for file: %s",
+               loc->path);
+    }
+
+    /* The file is locked to prevent a rename during a migration. Renames
+     * and migrations on the file at the same time can lead to data loss.
+     */
+
+    ret = dht_build_parent_loc(this, &parent_loc, loc, fop_errno);
+    if (ret < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to build parent loc, which is needed to "
+               "acquire entrylk to synchronize with renames on this "
+               "path. Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: cannot find hashed subvol which is needed to "
+               "synchronize with renames on this path. "
+               "Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    flock.l_type = F_WRLCK;
+
+    tmp_loc.inode = inode_ref(loc->inode);
+    gf_uuid_copy(tmp_loc.gfid, loc->gfid);
+    tmp_loc.path = gf_strdup(loc->path);
+
+    /* this inodelk happens with flock.owner being zero. But to synchronize
+     * hardlink migration we need to have different lkowner for each migration
+     * Filed a bug here: https://bugzilla.redhat.com/show_bug.cgi?id=1468202 to
+     * track the fix for this. Currently synclock takes care of synchronizing
+     * hardlink migration. Once this bug is fixed we can avoid taking synclock
+     */
+    ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW,
+                         &flock, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "migrate file failed: "
+               "%s: failed to lock file on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    inodelk_locked = _gf_true;
+
+    /* dht_rename has changed to use entrylk on hashed subvol for
+     * synchronization. So, rebalance too has to acquire an entrylk on
+     * hashed subvol.
+     */
+    ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, &parent_loc,
+                         loc->name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to acquire entrylk on subvol %s", loc->path,
+               hashed_subvol->name);
+        goto out;
+    }
+
+    entrylk_locked = _gf_true;
+
+    /* Phase 1 - Data migration is in progress from now on */
+    ret = syncop_lookup(from, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: lookup failed on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    /* preserve source mode, so set the same to the destination */
+    src_ia_prot = stbuf.ia_prot;
+
+    /* Check if file can be migrated */
+    ret = __is_file_migratable(this, loc, &stbuf, xattr_rsp, flag, defrag, conf,
+                               fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    /* Take care of the special files */
+    if (!IA_ISREG(stbuf.ia_type)) {
+        /* Special files */
+        ret = migrate_special_files(this, from, to, loc, &stbuf, fop_errno);
+        goto out;
+    }
+
+    /* Try to preserve 'holes' while migrating data */
+    if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
+        file_has_holes = 1;
+
+    /* create the destination, with required modes/xattr */
+    ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, &dst_fd,
+                                          fop_errno, file_has_holes);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Create dst failed"
+               " on - %s for file - %s",
+               to->name, loc->path);
+        goto out;
+    }
+
+    clean_dst = _gf_true;
+
+    ret = __dht_check_free_space(this, to, from, loc, &stbuf, flag, conf,
+                                 &target_changed, &new_target, fop_errno);
+    if (target_changed) {
+        /* Can't handle for hardlinks. Marking this as failure */
+        if (flag == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS || stbuf.ia_nlink > 1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "Exiting migration for"
+                   " file - %s. flag - %d, stbuf.ia_nlink - %d",
+                   loc->path, flag, stbuf.ia_nlink);
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
+        if (ret) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "%s: failed to perform truncate on %s (%s)", loc->path,
+                   to->name, strerror(-ret));
         }
 
-        flock.l_type = F_WRLCK;
-
-        tmp_loc.inode = inode_ref (loc->inode);
-        gf_uuid_copy (tmp_loc.gfid, loc->gfid);
-        tmp_loc.path = gf_strdup(loc->path);
+        syncop_close(dst_fd);
+        dst_fd = NULL;
 
-        ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW,
-                              &flock, NULL, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "migrate file failed: "
-                        "%s: failed to lock file on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+        old_target = to;
+        to = new_target;
 
-        locked = _gf_true;
+        clean_dst = _gf_false;
 
-        /* Phase 1 - Data migration is in progress from now on */
-        ret = syncop_lookup (from, loc, &stbuf, NULL, dict, &xattr_rsp);
+        /* if the file migration is successful to this new target, then
+         * update the xattr on the old destination to point the new
+         * destination. We need to do update this only post migration
+         * as in case of failure the linkto needs to point to the source
+         * subvol */
+        ret = __dht_rebalance_create_dst_file(
+            this, to, from, loc, &stbuf, &dst_fd, fop_errno, file_has_holes);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: lookup failed on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Create dst failed"
+                   " on - %s for file - %s",
+                   to->name, loc->path);
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                   "destination for file "
+                   "- %s is changed to - %s",
+                   loc->path, to->name);
+            clean_dst = _gf_true;
+        }
+    }
+
+    if (ret) {
+        goto out;
+    }
+
+    /* Open the source, and also update mode/xattr */
+    ret = __dht_rebalance_open_src_file(this, from, to, loc, &stbuf, &src_fd,
+                                        &clean_src, fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to open %s on %s", loc->path,
+               from->name);
+        goto out;
+    }
+
+    /* TODO: move all xattr related operations to fd based operations */
+    ret = syncop_listxattr(from, loc, &xattr, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to get xattr from %s",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    /* Copying posix acls to the linkto file messes up the permissions*/
+    dht_strip_out_acls(xattr);
+
+    /* Remove the linkto xattr as we don't want to overwrite the value
+     * set on the dst.
+     */
+    dict_del(xattr, conf->link_xattr_name);
+
+    /* We need to error out if this fails as having the wrong shard xattrs
+     * set on the dst could cause data corruption
+     */
+    ret = syncop_fsetxattr(to, dst_fd, xattr, 0, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to set xattr on %s", loc->path, to->name);
+        ret = -1;
+        goto out;
+    }
 
+    if (xattr_rsp) {
         /* we no more require this key */
-        dict_del (dict, conf->link_xattr_name);
-
-        /* preserve source mode, so set the same to the destination */
-        src_ia_prot = stbuf.ia_prot;
+        dict_del(dict, conf->link_xattr_name);
+        dict_unref(xattr_rsp);
+    }
+
+    ret = syncop_fstat(from, src_fd, &stbuf, dict, &xattr_rsp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:failed to lookup %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Check again if file has hardlink */
+    ret = __check_file_has_hardlink(this, loc, &stbuf, xattr_rsp, flag, defrag,
+                                    conf, fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    ret = __dht_rebalance_migrate_data(this, defrag, from, to, src_fd, dst_fd,
+                                       stbuf.ia_size, file_has_holes,
+                                       fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s: failed to migrate data", loc->path);
 
-        /* Check if file can be migrated */
-        ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag, defrag);
-        if (ret) {
-                if (ret == -2)
-                        ret = 0;
-                goto out;
-        }
-        /* Take care of the special files */
-        if (!IA_ISREG (stbuf.ia_type)) {
-                /* Special files */
-                ret = migrate_special_files (this, from, to, loc, &stbuf);
-                goto out;
-        }
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO: Sync the locks */
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set last-fsync flag on "
+               "%s (%s)",
+               loc->path, to->name, strerror(ENOMEM));
+    }
+
+    ret = syncop_fsync(to, dst_fd, 0, NULL, NULL, xdata, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: failed to fsync on %s (%s)",
+               loc->path, to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+
+    ret = syncop_fstat(from, src_fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to fstat file %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
+    /* Lock the entire source file to prevent clients from taking a
+       lock on it as dht_lk does not handle file migration.
 
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr, NULL, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-        }
+       This still leaves a small window where conflicting locks can
+       be granted to different clients. If client1 requests a blocking
+       lock on the src file, it will be granted after the migrating
+       process releases its lock. If client2 requests a lock on the dst
+       data file, it will also be granted, but all FOPs will be redirected
+       to the dst data file.
+    */
 
-        /* create the destination, with required modes/xattr */
-        ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
-                                               &dst_fd, xattr);
-        if (ret)
-                goto out;
-
-        clean_dst = _gf_true;
+    /* Take meta lock  */
 
-        ret = __dht_check_free_space (to, from, loc, &stbuf, flag);
+    if (conf->lock_migration_enabled) {
+        meta_dict = dict_new();
+        if (!meta_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "dict_new failed");
 
-        if (ret) {
-                goto out;
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Open the source, and also update mode/xattr */
-        ret = __dht_rebalance_open_src_file (from, to, loc, &stbuf, &src_fd,
-                                             &clean_src);
+        ret = dict_set_str(meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: failed to open %s on %s",
-                        loc->path, from->name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value: key = %s,"
+                   " path = %s",
+                   GLUSTERFS_INTERNAL_FOP_KEY, loc->path);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-
-        ret = syncop_fstat (from, src_fd, &stbuf, NULL, NULL);
+        ret = dict_set_int32(meta_dict, GF_META_LOCK_KEY, 1);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:failed to lookup %s on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
-
-        /* Try to preserve 'holes' while migrating data */
-        if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
-                file_has_holes = 1;
-
-
-        /* All I/O happens in this function */
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                ret = __tier_migrate_data (defrag, from, to, src_fd, dst_fd,
-                                                    stbuf.ia_size, file_has_holes);
-        } else {
-                ret = __dht_rebalance_migrate_data (from, to, src_fd, dst_fd,
-                                                    stbuf.ia_size, file_has_holes);
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: %s: failed to migrate data",
-                        loc->path);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr metalock failed");
 
-                ret = -1;
-                goto out;
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else {
+            meta_locked = _gf_true;
         }
+    }
 
-        /* TODO: Sync the locks */
+    if (!conf->lock_migration_enabled) {
+        plock.l_type = F_WRLCK;
+        plock.l_start = 0;
+        plock.l_len = 0;
+        plock.l_whence = SEEK_SET;
 
-        ret = syncop_fsync (to, dst_fd, 0, NULL, NULL);
+        ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to fsync on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
-        }
-
-
-        /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
-
-        ret = syncop_fstat (from, src_fd, &new_stbuf, NULL, NULL);
-        if (ret < 0) {
-                /* Failed to get the stat info */
-                gf_msg ( this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: failed to fstat file %s on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
-
-        /* Lock the entire source file to prevent clients from taking a
-           lock on it as dht_lk does not handle file migration.
-
-           This still leaves a small window where conflicting locks can
-           be granted to different clients. If client1 requests a blocking
-           lock on the src file, it will be granted after the migrating
-           process releases its lock. If client2 requests a lock on the dst
-           data file, it will also be granted, but all FOPs will be redirected
-           to the dst data file.
-        */
-
-        /* Take meta lock  */
-
-        if (defrag->lock_migration_enabled) {
-                meta_dict = dict_new ();
-                if (!meta_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace dict_new failed");
-
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_str (meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value: key = %s,"
-                                " path = %s", GLUSTERFS_INTERNAL_FOP_KEY,
-                                 loc->path);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_int32 (meta_dict, GF_META_LOCK_KEY, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace dict_set failed");
-
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_setxattr (from, loc, meta_dict, 0, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace syncop_setxattr metalock failed");
-
-                        ret = -1;
-                        goto out;
-                } else {
-                        meta_locked = _gf_true;
-                }
-        }
-
-        if (!defrag->lock_migration_enabled) {
-                plock.l_type = F_WRLCK;
-                plock.l_start = 0;
-                plock.l_len = 0;
-                plock.l_whence = SEEK_SET;
-
-                ret = syncop_lk (from, src_fd, F_SETLK, &plock, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: Failed to lock on %s",
-                                loc->path, from->name);
-                        ret = -1;
-                        goto out;
-                }
-
-                p_locked = _gf_true;
-
-        } else {
-
-                INIT_LIST_HEAD (&locklist.list);
-
-                ret = syncop_getactivelk (from, loc, &locklist, NULL, NULL);
-                if (ret == 0) {
-                        gf_log (this->name, GF_LOG_INFO, "No active locks on:%s"
-                                , loc->path);
-
-                } else if (ret > 0) {
-
-                        ret = syncop_setactivelk (to, loc, &locklist, NULL,
-                                                  NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOCK_MIGRATION_FAILED,
-                                        "write lock failed on:%s", loc->path);
-
-                                ret = -1;
-                                goto metaunlock;
-                        }
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOCK_MIGRATION_FAILED,
-                                "getactivelk failed for file: %s", loc->path);
-                }
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Failed to lock on %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
+        p_locked = _gf_true;
 
-        /* source would have both sticky bit and sgid bit set, reset it to 0,
-           and set the source permission on destination, if it was not set
-           prior to setting rebalance-modes in source  */
-        if (!src_ia_prot.sticky)
-                new_stbuf.ia_prot.sticky = 0;
+    } else {
+        INIT_LIST_HEAD(&locklist.list);
 
-        if (!src_ia_prot.sgid)
-                new_stbuf.ia_prot.sgid = 0;
+        ret = syncop_getactivelk(from, loc, &locklist, NULL, NULL);
+        if (ret == 0) {
+            gf_log(this->name, GF_LOG_INFO, "No active locks on:%s", loc->path);
 
-        /* TODO: if the source actually had sticky bit, or sgid bit set,
-           we are not handling it */
+        } else if (ret > 0) {
+            ret = syncop_setactivelk(to, loc, &locklist, NULL, NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_LOCK_MIGRATION_FAILED, "write lock failed on:%s",
+                       loc->path);
 
-        ret = syncop_fsetattr (to, dst_fd, &new_stbuf,
-                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                                GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, to->name);
+                *fop_errno = -ret;
                 ret = -1;
                 goto metaunlock;
-        }
-
-        /* Because 'futimes' is not portable */
-        ret = syncop_setattr (to, loc, &new_stbuf,
-                              (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME),
-                              NULL, NULL, NULL, NULL);
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret,
+                   DHT_MSG_LOCK_MIGRATION_FAILED,
+                   "getactivelk failed for file: %s", loc->path);
+            *fop_errno = -ret;
+        }
+    }
+
+    /* source would have both sticky bit and sgid bit set, reset it to 0,
+       and set the source permission on destination, if it was not set
+       prior to setting rebalance-modes in source  */
+    if (!src_ia_prot.sticky)
+        new_stbuf.ia_prot.sticky = 0;
+
+    if (!src_ia_prot.sgid)
+        new_stbuf.ia_prot.sgid = 0;
+
+    /* TODO: if the source actually had sticky bit, or sgid bit set,
+       we are not handling it */
+
+    ret = syncop_fsetattr(
+        to, dst_fd, &new_stbuf,
+        (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+        NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Because 'futimes' is not portable */
+    ret = syncop_setattr(to, loc, &new_stbuf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME), NULL, NULL,
+                         NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform setattr on %s ", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    if (target_changed) {
+        dict_del(dict, GLUSTERFS_POSIXLK_COUNT);
+        ret = dict_set_str(dict, conf->link_xattr_name, to->name);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, to->name);
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+                   to->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_setxattr(old_target, loc, dict, 0, NULL, NULL);
+        if (ret && -ret != ESTALE && -ret != ENOENT) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to set xattr on %s in %s", loc->path,
+                   old_target->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else if (-ret == ESTALE || -ret == ENOENT) {
+            /* The failure ESTALE indicates that the linkto
+             * file on the hashed subvol might have been deleted.
+             * In this case will create a linkto file with new target
+             * as linkto xattr value*/
+            linkto_fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+            if (!linkto_fd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno,
+                       DHT_MSG_MIGRATE_FILE_FAILED, "%s: fd create failed",
+                       loc->path);
+                *fop_errno = ENOMEM;
                 ret = -1;
-        }
-
-
-        clean_dst = _gf_false;
-
-        /* Posix acls are not set on DHT linkto files as part of the initial
-         * initial xattrs set on the dst file, so these need
-         * to be set on the dst file after the linkto attrs are removed.
-         * TODO: Optimize this.
-         */
-        if (xattr) {
-                dict_unref (xattr);
-                xattr = NULL;
-        }
-
-        ret = syncop_listxattr (from, loc, &xattr, NULL, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (-ret));
+                goto out;
+            }
+            ret = syncop_create(old_target, loc, O_RDWR, DHT_LINKFILE_MODE,
+                                linkto_fd, NULL, dict, NULL);
+            if (ret != 0 && -ret != EEXIST && -ret != ESTALE) {
+                *fop_errno = -ret;
                 ret = -1;
-        } else {
-                ret = syncop_setxattr (to, loc, xattr, 0, NULL, NULL);
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "failed to create linkto file on %s in %s", loc->path,
+                       old_target->name);
+                goto out;
+            } else if (ret == 0) {
+                ret = syncop_fsetattr(old_target, linkto_fd, &stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL,
+                                      NULL, NULL, NULL);
                 if (ret < 0) {
-                        /* Potential problem here where Posix ACLs will
-                         * not be set on the target file */
-
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: failed to set xattr on %s (%s)",
-                                loc->path, to->name, strerror (-ret));
-                        ret = -1;
+                    *fop_errno = -ret;
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "chown failed for %s on %s", loc->path,
+                           old_target->name);
                 }
-        }
-
-        /* store size of previous migrated file  */
-        if (defrag->tier_conf.is_tier) {
-                if (from != TIER_HASHED_SUBVOL) {
-                        defrag->tier_conf.st_last_promoted_size = stbuf.ia_size;
-                } else {
-                        /* Don't delete the linkto file on the hashed subvol */
-                        delete_src_linkto = _gf_false;
-                        defrag->tier_conf.st_last_demoted_size = stbuf.ia_size;
-                }
-        }
-
-        /* The src file is being unlinked after this so we don't need
-           to clean it up */
-        clean_src = _gf_false;
-
-        /* Make the source as a linkfile first before deleting it */
-        empty_iatt.ia_prot.sticky = 1;
-        ret = syncop_fsetattr (from, src_fd, &empty_iatt,
-                               GF_SET_ATTR_MODE, NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto metaunlock;
-        }
-
-       /* Free up the data blocks on the source node, as the whole
-           file is migrated */
-        ret = syncop_ftruncate (from, src_fd, 0, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform truncate on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-        }
-
-        /* remove the 'linkto' xattr from the destination */
-        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name, 0, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform removexattr on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
-        }
-
-        /* Do a stat and check the gfid before unlink */
-
-        /*
-         * Cached file changes its state from non-linkto to linkto file after
-         * migrating data. If lookup from any other mount-point is performed,
-         * converted-linkto-cached file will be treated as a stale and will be
-         * unlinked. But by this time, file is already migrated. So further
-         * failure because of ENOENT should  not be treated as error
-         */
-
-        ret = syncop_stat (from, loc, &empty_iatt, NULL, NULL);
+            }
+        }
+    }
+
+    clean_dst = _gf_false;
+
+    /* Posix acls are not set on DHT linkto files as part of the initial
+     * initial xattrs set on the dst file, so these need
+     * to be set on the dst file after the linkto attrs are removed.
+     * TODO: Optimize this.
+     */
+    if (xattr) {
+        dict_unref(xattr);
+        xattr = NULL;
+    }
+
+    /* Set only the Posix ACLs this time */
+    ret = syncop_getxattr(from, loc, &xattr, POSIX_ACL_ACCESS_XATTR, NULL,
+                          NULL);
+    if (ret < 0) {
+        if ((-ret != ENODATA) && (-ret != ENOATTR)) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to get xattr from %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+        }
+    } else {
+        ret = syncop_setxattr(to, loc, xattr, 0, NULL, NULL);
+        if (ret < 0) {
+            /* Potential problem here where Posix ACLs will
+             * not be set on the target file */
+
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to set xattr on %s",
+                   loc->path, to->name);
+            *fop_errno = -ret;
+        }
+    }
+
+    /* The src file is being unlinked after this so we don't need
+       to clean it up */
+    clean_src = _gf_false;
+
+    /* Make the source as a linkfile first before deleting it */
+    empty_iatt.ia_prot.sticky = 1;
+    ret = syncop_fsetattr(from, src_fd, &empty_iatt, GF_SET_ATTR_MODE, NULL,
+                          NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Free up the data blocks on the source node, as the whole
+        file is migrated */
+    ret = syncop_ftruncate(from, src_fd, 0, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform truncate on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* remove the 'linkto' xattr from the destination */
+    ret = syncop_fremovexattr(to, dst_fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform removexattr on %s (%s)", loc->path,
+               to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Do a stat and check the gfid before unlink */
+
+    /*
+     * Cached file changes its state from non-linkto to linkto file after
+     * migrating data. If lookup from any other mount-point is performed,
+     * converted-linkto-cached file will be treated as a stale and will be
+     * unlinked. But by this time, file is already migrated. So further
+     * failure because of ENOENT should  not be treated as error
+     */
+
+    ret = syncop_stat(from, loc, &empty_iatt, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to do a stat on %s", loc->path, from->name);
+
+        if (-ret != ENOENT) {
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
+        }
+
+        rcvd_enoent_from_src = 1;
+    }
+
+    if ((gf_uuid_compare(empty_iatt.ia_gfid, loc->gfid) == 0) &&
+        (!rcvd_enoent_from_src) && delete_src_linkto) {
+        /* take out the source from namespace */
+        ret = syncop_unlink(from, loc, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to do a stat on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-
-                if (-ret != ENOENT) {
-                        ret = -1;
-                        goto metaunlock;
-                }
-
-                rcvd_enoent_from_src = 1;
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to perform unlink on %s", loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
         }
+    }
 
+    ret = syncop_lookup(this, loc, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "%s: failed to lookup the file on subvolumes", loc->path);
+        *fop_errno = -ret;
+    }
 
-        if ((gf_uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0 ) &&
-            (!rcvd_enoent_from_src) && delete_src_linkto) {
-                /* take out the source from namespace */
-                ret = syncop_unlink (from, loc, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to perform unlink on %s (%s)",
-                                loc->path, from->name, strerror (-ret));
-                        ret = -1;
-                        goto metaunlock;
-                }
-        }
-
-        ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "%s: failed to lookup the file on subvolumes (%s)",
-                              loc->path, strerror (-ret));
-                ret = -1;
-        }
+    gf_msg(this->name, log_level, 0, DHT_MSG_MIGRATE_FILE_COMPLETE,
+           "completed migration of %s from subvolume %s to %s", loc->path,
+           from->name, to->name);
 
-        gf_msg (this->name, log_level, 0,
-                DHT_MSG_MIGRATE_FILE_COMPLETE,
-                "completed migration of %s from subvolume %s to %s",
-                loc->path, from->name, to->name);
-
-        ret = 0;
+    ret = 0;
 
 metaunlock:
 
-        if (defrag->lock_migration_enabled && meta_locked) {
-
-                dict_del (meta_dict, GF_META_LOCK_KEY);
-
-                ret = dict_set_int32 (meta_dict, GF_META_UNLOCK_KEY, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace dict_set failed");
+    if (conf->lock_migration_enabled && meta_locked) {
+        dict_del(meta_dict, GF_META_LOCK_KEY);
 
-                        ret = -1;
-                        goto out;
-                }
-
-                if (clean_dst == _gf_false)
-                        ret = dict_set_int32 (meta_dict, "status", 1);
-                else
-                        ret = dict_set_int32 (meta_dict, "status", 0);
-
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace dict_set failed");
-
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_setxattr (from, loc, meta_dict, 0, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace syncop_setxattr meta unlock failed");
-
-                        ret = -1;
-                        goto out;
-                }
-        }
+        ret = dict_set_int32(meta_dict, GF_META_UNLOCK_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
 
-out:
-        if (clean_src) {
-                /* Revert source mode and xattr changes*/
-                lk_ret = __dht_migration_cleanup_src_file (this, loc, src_fd,
-                                                        from, &src_ia_prot);
-                if (lk_ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to cleanup source file on %s",
-                                loc->path, from->name);
-                }
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* reset the destination back to 0 */
-        if (clean_dst) {
-                lk_ret = syncop_ftruncate (to, dst_fd, 0, NULL, NULL);
-                if (lk_ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -lk_ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed: "
-                                "%s: failed to reset target size back to 0",
-                                loc->path);
-                }
-        }
+        if (clean_dst == _gf_false)
+            ret = dict_set_int32(meta_dict, "status", 1);
+        else
+            ret = dict_set_int32(meta_dict, "status", 0);
 
-        if (locked) {
-                flock.l_type = F_UNLCK;
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
 
-                lk_ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN,
-                                         &tmp_loc, F_SETLK, &flock, NULL, NULL);
-                if (lk_ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to unlock file on %s (%s)",
-                                loc->path, from->name, strerror (-lk_ret));
-                }
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        if (p_locked) {
-                plock.l_type = F_UNLCK;
-                lk_ret = syncop_lk (from, src_fd, F_SETLK, &plock, NULL, NULL);
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr meta unlock failed");
 
-                if (lk_ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, -lk_ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to unlock file on %s",
-                                loc->path, from->name);
-                }
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (dict)
-                dict_unref (dict);
-
-        if (xattr)
-                dict_unref (xattr);
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
-
-        if (dst_fd)
-                syncop_close (dst_fd);
-        if (src_fd)
-                syncop_close (src_fd);
-
-        loc_wipe (&tmp_loc);
-
-        return ret;
+out:
+    if (clean_src) {
+        /* Revert source mode and xattr changes*/
+        lk_ret = __dht_migration_cleanup_src_file(this, loc, src_fd, from,
+                                                  &src_ia_prot);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to cleanup source file on %s", loc->path,
+                   from->name);
+        }
+    }
+
+    /* reset the destination back to 0 */
+    if (clean_dst) {
+        lk_ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: "
+                   "%s: failed to reset target size back to 0",
+                   loc->path);
+        }
+    }
+
+    if (inodelk_locked) {
+        flock.l_type = F_UNLCK;
+
+        lk_ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc,
+                                F_SETLK, &flock, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
+        }
+    }
+
+    if (entrylk_locked) {
+        lk_ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN,
+                                &parent_loc, loc->name, ENTRYLK_UNLOCK,
+                                ENTRYLK_UNLOCK, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock entrylk on %s", loc->path,
+                   hashed_subvol->name);
+        }
+    }
+
+    if (p_locked) {
+        plock.l_type = F_UNLCK;
+        lk_ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
+
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
+        }
+    }
+
+    lk_ret = syncop_removexattr(to, loc, GF_PROTECT_FROM_EXTERNAL_WRITES, NULL,
+                                NULL);
+    if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) {
+        gf_msg(this->name, GF_LOG_WARNING, -lk_ret, 0,
+               "%s: removexattr failed key %s", loc->path,
+               GF_PROTECT_FROM_EXTERNAL_WRITES);
+    }
+
+    if (dict)
+        dict_unref(dict);
+
+    if (xattr)
+        dict_unref(xattr);
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+
+    if (dst_fd)
+        syncop_close(dst_fd);
+
+    if (src_fd)
+        syncop_close(src_fd);
+    if (linkto_fd)
+        syncop_close(linkto_fd);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    loc_wipe(&tmp_loc);
+    loc_wipe(&parent_loc);
+
+    return ret;
 }
 
 static int
-rebalance_task (void *data)
+rebalance_task(void *data)
 {
-        int           ret   = -1;
-        dht_local_t  *local = NULL;
-        call_frame_t *frame = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int fop_errno = 0;
 
-        frame = data;
+    frame = data;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* This function is 'synchrounous', hence if it returns,
-           we are done with the task */
-        ret = dht_migrate_file (THIS, &local->loc, local->rebalance.from_subvol,
-                                local->rebalance.target_node, local->flags);
+    /* This function is 'synchrounous', hence if it returns,
+       we are done with the task */
+    ret = dht_migrate_file(THIS, &local->loc, local->rebalance.from_subvol,
+                           local->rebalance.target_node, local->flags,
+                           &fop_errno);
 
-        return ret;
+    return ret;
 }
 
 static int
-rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
+rebalance_task_completion(int op_ret, call_frame_t *sync_frame, void *data)
 {
-        int           ret        = -1;
-        uint64_t      layout_int = 0;
-        dht_layout_t *layout     = 0;
-        xlator_t     *this       = NULL;
-        dht_local_t  *local      = NULL;
-        int32_t       op_errno   = EINVAL;
-
-        this = THIS;
-        local = sync_frame->local;
-
-        if (!op_ret) {
-                /* Make sure we have valid 'layout' in inode ctx
-                   after the operation */
-                ret = inode_ctx_del (local->loc.inode, this, &layout_int);
-                if (!ret && layout_int) {
-                        layout = (dht_layout_t *)(long)layout_int;
-                        dht_layout_unref (this, layout);
-                }
-
-                ret = dht_layout_preset (this, local->rebalance.target_node,
-                                         local->loc.inode);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set inode ctx", local->loc.path);
-        }
-
-        if (op_ret == -1) {
-                /* Failure of migration process, mostly due to write process.
-                   as we can't preserve the exact errno, lets say there was
-                   no space to migrate-data
-                */
-                op_errno = ENOSPC;
-        }
-
-        if (op_ret == 1) {
-                /* migration didn't happen, but is not a failure, let the user
-                   understand that he doesn't have permission to migrate the
-                   file.
-                */
-                op_ret = -1;
-                op_errno = EPERM;
-        }
+    int32_t op_errno = EINVAL;
 
-        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
-        return 0;
+    if (op_ret == -1) {
+        /* Failure of migration process, mostly due to write process.
+           as we can't preserve the exact errno, lets say there was
+           no space to migrate-data
+        */
+        op_errno = ENOSPC;
+    } else if (op_ret == 1) {
+        /* migration didn't happen, but is not a failure, let the user
+           understand that he doesn't have permission to migrate the
+           file.
+        */
+        op_ret = -1;
+        op_errno = EPERM;
+    } else if (op_ret != 0) {
+        op_errno = -op_ret;
+        op_ret = -1;
+    }
+
+    DHT_STACK_UNWIND(setxattr, sync_frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 int
-dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame)
 {
-        int         ret     = -1;
+    int ret = -1;
 
-        ret = synctask_new (this->ctx->env, rebalance_task,
-                            rebalance_task_completion,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, rebalance_task,
+                       rebalance_task_completion, frame, frame);
+    return ret;
 }
 
 int
-gf_listener_stop (xlator_t *this)
+gf_listener_stop(xlator_t *this)
 {
-        glusterfs_ctx_t  *ctx = NULL;
-        cmd_args_t       *cmd_args = NULL;
-        int              ret = 0;
-
-        ctx = this->ctx;
-        GF_ASSERT (ctx);
-        cmd_args = &ctx->cmd_args;
-        if (cmd_args->sock_file) {
-                ret = sys_unlink (cmd_args->sock_file);
-                if (ret && (ENOENT == errno)) {
-                        ret = 0;
-                }
-        }
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        DHT_MSG_SOCKET_ERROR,
-                        "Failed to unlink listener "
-                        "socket %s", cmd_args->sock_file);
-        }
-        return ret;
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    cmd_args = &ctx->cmd_args;
+    if (cmd_args->sock_file) {
+        ret = sys_unlink(cmd_args->sock_file);
+        if (ret && (ENOENT == errno)) {
+            ret = 0;
+        }
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, DHT_MSG_SOCKET_ERROR,
+               "Failed to unlink listener "
+               "socket %s",
+               cmd_args->sock_file);
+    }
+    return ret;
 }
 
 void
-dht_build_root_inode (xlator_t *this, inode_t **inode)
+dht_build_root_inode(xlator_t *this, inode_t **inode)
 {
-        inode_table_t    *itable        = NULL;
-        uuid_t            root_gfid     = {0, };
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        itable = inode_table_new (0, this);
-        if (!itable)
-                return;
+    itable = inode_table_new(0, this);
+    if (!itable)
+        return;
 
-        root_gfid[15] = 1;
-        *inode = inode_find (itable, root_gfid);
+    *inode = inode_find(itable, root_gfid);
 }
 
 void
-dht_build_root_loc (inode_t *inode, loc_t *loc)
+dht_build_root_loc(inode_t *inode, loc_t *loc)
 {
-        loc->path = "/";
-        loc->inode = inode;
-        loc->inode->ia_type = IA_IFDIR;
-        memset (loc->gfid, 0, 16);
-        loc->gfid[15] = 1;
+    loc->path = "/";
+    loc->inode = inode;
+    loc->inode->ia_type = IA_IFDIR;
+    memset(loc->gfid, 0, 16);
+    loc->gfid[15] = 1;
 }
 
-
 /* return values: 1 -> error, bug ignore and continue
                   0 -> proceed
                  -1 -> error, handle it */
 int32_t
-gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
+gf_defrag_handle_migrate_error(int32_t op_errno, gf_defrag_info_t *defrag)
 {
-        /* if errno is not ENOSPC or ENOTCONN, we can still continue
-           with rebalance process */
-        if ((op_errno != ENOSPC) || (op_errno != ENOTCONN))
-                return 1;
-
-        if (op_errno == ENOTCONN) {
-                /* Most probably mount point went missing (mostly due
-                   to a brick down), say rebalance failure to user,
-                   let him restart it if everything is fine */
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                return -1;
-        }
-
-        if (op_errno == ENOSPC) {
-                /* rebalance process itself failed, may be
-                   remote brick went down, or write failed due to
-                   disk full etc etc.. */
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                return -1;
-        }
+    int ret = 0;
+    /* if errno is not ENOTCONN, we can still continue
+       with rebalance process */
+    if (op_errno != ENOTCONN) {
+        ret = 1;
+        goto out;
+    }
+
+    if (op_errno == ENOTCONN) {
+        /* Most probably mount point went missing (mostly due
+           to a brick down), say rebalance failure to user,
+           let him restart it if everything is fine */
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+        ret = -1;
+        goto out;
+    }
 
-        return 0;
+out:
+    return ret;
 }
 
 static gf_boolean_t
-gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+gf_defrag_pattern_match(gf_defrag_info_t *defrag, char *name, uint64_t size)
 {
-        gf_defrag_pattern_list_t *trav = NULL;
-        gf_boolean_t               match = _gf_false;
-        gf_boolean_t               ret = _gf_false;
+    gf_defrag_pattern_list_t *trav = NULL;
+    gf_boolean_t match = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+    GF_VALIDATE_OR_GOTO("dht", defrag, out);
 
-        trav = defrag->defrag_pattern;
-        while (trav) {
-                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
-                        match = _gf_true;
-                        break;
-                }
-                trav = trav->next;
+    trav = defrag->defrag_pattern;
+    while (trav) {
+        if (!fnmatch(trav->path_pattern, name, FNM_NOESCAPE)) {
+            match = _gf_true;
+            break;
         }
+        trav = trav->next;
+    }
 
-        if ((match == _gf_true) && (size >= trav->size))
-                ret = _gf_true;
+    if ((match == _gf_true) && (size >= trav->size))
+        ret = _gf_true;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-int dht_dfreaddirp_done (dht_dfoffset_ctx_t *offset_var, int cnt) {
-
-        int i;
-        int result = 1;
+int
+dht_dfreaddirp_done(dht_dfoffset_ctx_t *offset_var, int cnt)
+{
+    int i;
+    int result = 1;
 
-        for (i = 0; i < cnt; i++) {
-                if (offset_var[i].readdir_done == 0) {
-                        result = 0;
-                        break;
-                }
+    for (i = 0; i < cnt; i++) {
+        if (offset_var[i].readdir_done == 0) {
+            result = 0;
+            break;
         }
-        return result;
+    }
+    return result;
 }
 
-int static
-gf_defrag_ctx_subvols_init (dht_dfoffset_ctx_t *offset_var, xlator_t *this) {
+int static gf_defrag_ctx_subvols_init(dht_dfoffset_ctx_t *offset_var,
+                                      xlator_t *this)
+{
+    int i;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
 
-        int i;
-        dht_conf_t *conf = NULL;
+    if (!conf)
+        return -1;
 
-        conf = this->private;
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        offset_var[i].this = conf->local_subvols[i];
+        offset_var[i].offset = (off_t)0;
+        offset_var[i].readdir_done = 0;
+    }
 
-        if (!conf)
-               return -1;
+    return 0;
+}
+
+static int
+dht_get_first_non_null_index(subvol_nodeuuids_info_t *entry)
+{
+    int i = 0;
+    int index = 0;
 
-        for (i = 0; i < conf->local_subvols_cnt; i++) {
-               offset_var[i].this = conf->local_subvols[i];
-               offset_var[i].offset = (off_t) 0;
-               offset_var[i].readdir_done = 0;
+    for (i = 0; i < entry->count; i++) {
+        if (!gf_uuid_is_null(entry->elements[i].uuid)) {
+            index = i;
+            goto out;
         }
+    }
 
-        return 0;
+    if (i == entry->count) {
+        index = -1;
+    }
+out:
+    return index;
 }
 
-int
-gf_defrag_migrate_single_file (void *opaque)
+/* Return value
+ * 0 : this node does not migrate the file
+ * 1 : this node migrates the file
+ *
+ * Use the hash value of the gfid to determine which node will migrate files.
+ * Using the gfid instead of the name also ensures that the same node handles
+ * all hardlinks.
+ */
+
+gf_boolean_t
+gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid)
 {
-        xlator_t                *this = NULL;
-        dht_conf_t              *conf = NULL;
-        gf_defrag_info_t        *defrag = NULL;
-        int                     ret = 0;
-        gf_dirent_t             *entry          = NULL;
-        struct timeval           start          = {0,};
-        loc_t                    entry_loc      = {0,};
-        loc_t                   *loc            = NULL;
-        struct iatt              iatt           = {0,};
-        dict_t                  *migrate_data   = NULL;
-        int32_t                  op_errno       = 0;
-        struct timeval           end            = {0,};
-        double                   elapsed        = {0,};
-        struct dht_container    *rebal_entry    = NULL;
-        inode_t                 *inode          = NULL;
-
-        rebal_entry = (struct dht_container *)opaque;
-        if (!rebal_entry) {
-                gf_log (this->name, GF_LOG_ERROR, "rebal_entry is NULL");
-                ret = -1;
+    gf_boolean_t ret = _gf_false;
+    int i = local_subvol_index;
+    char *str = NULL;
+    uint32_t hashval = 0;
+    int32_t index = 0;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+    subvol_nodeuuids_info_t *entry = NULL;
+
+    conf = this->private;
+
+    /* Pure distribute. A subvol in this case
+        will be handled by only one node */
+
+    entry = &(conf->local_nodeuuids[i]);
+    if (entry->count == 1) {
+        return 1;
+    }
+
+    str = uuid_utoa_r(gfid, buf);
+    if (dht_hash_compute(this, 0, str, &hashval) == 0) {
+        index = (hashval % entry->count);
+        if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+            /* Index matches this node's nodeuuid.*/
+            ret = _gf_true;
+            goto out;
+        }
+
+        /* Brick down - some other node has to migrate these files*/
+        if (gf_uuid_is_null(entry->elements[index].uuid)) {
+            /* Fall back to the first non-null index */
+            index = dht_get_first_non_null_index(entry);
+
+            if (index == -1) {
+                /* None of the bricks in the subvol are up.
+                 * CHILD_DOWN will kill the process soon */
+
+                return _gf_false;
+            }
+
+            if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+                /* Index matches this node's nodeuuid.*/
+                ret = _gf_true;
                 goto out;
+            }
         }
+    }
+out:
+    return ret;
+}
 
-        this = rebal_entry->this;
+int
+gf_defrag_migrate_single_file(void *opaque)
+{
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct timeval start = {
+        0,
+    };
+    loc_t entry_loc = {
+        0,
+    };
+    loc_t *loc = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *migrate_data = NULL;
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    struct dht_container *rebal_entry = NULL;
+    inode_t *inode = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    data_t *tmp = NULL;
+    int fop_errno = 0;
+    gf_dht_migrate_data_type_t rebal_type = GF_DHT_MIGRATE_DATA;
+    char value[MAX_REBAL_TYPE_SIZE] = {
+        0,
+    };
+    struct iatt *iatt_ptr = NULL;
+    gf_boolean_t update_skippedcount = _gf_true;
+    int i = 0;
+    gf_boolean_t should_i_migrate = 0;
+
+    rebal_entry = (struct dht_container *)opaque;
+    if (!rebal_entry) {
+        gf_log("DHT", GF_LOG_ERROR, "rebal_entry is NULL");
+        ret = -1;
+        goto out;
+    }
 
-        conf = this->private;
+    this = rebal_entry->this;
 
-        defrag = conf->defrag;
+    conf = this->private;
 
-        loc = rebal_entry->parent_loc;
+    defrag = conf->defrag;
 
-        migrate_data = rebal_entry->migrate_data;
+    loc = rebal_entry->parent_loc;
 
-        entry = rebal_entry->df_entry;
+    migrate_data = rebal_entry->migrate_data;
 
-        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                ret = -1;
-                goto out;
-        }
+    entry = rebal_entry->df_entry;
+    iatt_ptr = &entry->d_stat;
 
-        if (defrag->stats == _gf_true) {
-                gettimeofday (&start, NULL);
-        }
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
 
-        if (defrag->defrag_pattern &&
-            (gf_defrag_pattern_match (defrag, entry->d_name,
-                                      entry->d_stat.ia_size) == _gf_false)) {
-                gf_log (this->name, GF_LOG_ERROR, "pattern_match failed");
-                goto out;
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&start, NULL);
+    }
+
+    if (defrag->defrag_pattern &&
+        (gf_defrag_pattern_match(defrag, entry->d_name,
+                                 entry->d_stat.ia_size) == _gf_false)) {
+        gf_log(this->name, GF_LOG_ERROR, "pattern_match failed");
+        goto out;
+    }
+
+    memset(&entry_loc, 0, sizeof(entry_loc));
+
+    ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+    if (ret) {
+        LOCK(&defrag->lock);
+        {
+            defrag->total_failures += 1;
         }
+        UNLOCK(&defrag->lock);
 
-        memset (&entry_loc, 0, sizeof (entry_loc));
+        ret = 0;
 
-        ret = dht_build_child_loc (this, &entry_loc, loc, entry->d_name);
-        if (ret) {
-                LOCK (&defrag->lock);
-                {
-                        defrag->total_failures += 1;
-                }
-                UNLOCK (&defrag->lock);
+        gf_log(this->name, GF_LOG_ERROR, "Child loc build failed");
 
-                ret = 0;
+        goto out;
+    }
 
-                gf_log (this->name, GF_LOG_ERROR, "Child loc build failed");
+    should_i_migrate = gf_defrag_should_i_migrate(
+        this, rebal_entry->local_subvol_index, entry->d_stat.ia_gfid);
 
-                goto out;
-        }
+    gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
 
-        gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+    gf_uuid_copy(entry_loc.pargfid, loc->gfid);
 
-        gf_uuid_copy (entry_loc.pargfid, loc->gfid);
+    ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
 
-        ret = syncop_lookup (this, &entry_loc, &iatt, NULL, NULL, NULL);
+    if (!should_i_migrate) {
+        /* this node isn't supposed to migrate the file. suppressing any
+         * potential error from lookup as this file is under migration by
+         * another node */
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: %s lookup failed",
-                        entry_loc.name);
-                ret = 0;
-                goto out;
+            gf_msg_debug(this->name, -ret,
+                         "Ignoring lookup failure: node isn't migrating %s",
+                         entry_loc.path);
+            ret = 0;
+        }
+        gf_msg_debug(this->name, 0, "Don't migrate %s ", entry_loc.path);
+        goto out;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s lookup failed", entry_loc.path);
+
+        /* Increase failure count only for remove-brick op, so that
+         * user is warned to check the removed-brick for any files left
+         * unmigrated
+         */
+        if (conf->decommission_subvols_cnt) {
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
         }
 
-        inode = inode_link (entry_loc.inode, entry_loc.parent, entry->d_name, &iatt);
-        inode_unref (entry_loc.inode);
-        /* use the inode returned by inode_link */
-        entry_loc.inode = inode;
+        ret = 0;
+        goto out;
+    }
 
-        ret = syncop_setxattr (this, &entry_loc, migrate_data, 0, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                /* errno is overloaded. See
-                 * rebalance_task_completion () */
-                if (op_errno == ENOSPC) {
-                        gf_msg_debug (this->name, 0, "migrate-data skipped for"
-                                      " %s due to space constraints",
-                                      entry_loc.path);
-                        LOCK (&defrag->lock);
-                        {
-                                defrag->skipped += 1;
-                        }
-                        UNLOCK (&defrag->lock);
-                } else if (op_errno == ENOTSUP) {
-                        gf_msg_debug (this->name, 0, "migrate-data skipped for"
-                                      " hardlink %s ", entry_loc.path);
-                        LOCK (&defrag->lock);
-                        {
-                                defrag->skipped += 1;
-                        }
-                        UNLOCK (&defrag->lock);
-                } else if (op_errno != EEXIST) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "migrate-data failed for %s", entry_loc.path);
+    iatt_ptr = &iatt;
+
+    hashed_subvol = dht_subvol_get_hashed(this, &entry_loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s", entry_loc.path);
+        ret = 0;
+        goto out;
+    }
 
-                        LOCK (&defrag->lock);
+    cached_subvol = dht_subvol_get_cached(this, entry_loc.inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+               "Failed to get cached subvol for %s", entry_loc.path);
+
+        ret = 0;
+        goto out;
+    }
+
+    if (hashed_subvol == cached_subvol) {
+        ret = 0;
+        goto out;
+    }
+
+    inode = inode_link(entry_loc.inode, entry_loc.parent, entry->d_name, &iatt);
+    inode_unref(entry_loc.inode);
+    /* use the inode returned by inode_link */
+    entry_loc.inode = inode;
+
+    old_THIS = THIS;
+    THIS = this;
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    /* async statfs information for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, loc);
+    THIS = old_THIS;
+
+    tmp = dict_get(migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            rebal_type = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            rebal_type = GF_DHT_MIGRATE_HARDLINK;
+    }
+
+    ret = dht_migrate_file(this, &entry_loc, cached_subvol, hashed_subvol,
+                           rebal_type, &fop_errno);
+    if (ret == 1) {
+        if (fop_errno == ENOSPC) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " %s due to space constraints",
+                         entry_loc.path);
+
+            /* For remove-brick case if the source is not one of the
+             * removed-brick, do not mark the error as failure */
+            if (conf->decommission_subvols_cnt) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (conf->decommissioned_bricks[i] == cached_subvol) {
+                        LOCK(&defrag->lock);
                         {
-                                defrag->total_failures += 1;
+                            defrag->total_failures += 1;
+                            update_skippedcount = _gf_false;
                         }
-                        UNLOCK (&defrag->lock);
+                        UNLOCK(&defrag->lock);
 
+                        break;
+                    }
                 }
+            }
 
-                ret = gf_defrag_handle_migrate_error (op_errno, defrag);
-
-                if (!ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0,
-                               DHT_MSG_MIGRATE_FILE_FAILED,
-                               "migrate-data on %s failed: %s", entry_loc.path,
-                               strerror (op_errno));
-                } else if (ret == 1) {
-                        ret = 0;
-                        goto out;
-                } else if (ret == -1) {
-                        goto out;
-                }
-        } else if (ret > 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "migrate-data failed for %s", entry_loc.path);
-                ret = 0;
-                LOCK (&defrag->lock);
+            if (update_skippedcount) {
+                LOCK(&defrag->lock);
                 {
-                        defrag->total_failures += 1;
+                    defrag->skipped += 1;
                 }
-                UNLOCK (&defrag->lock);
-        }
-
-        LOCK (&defrag->lock);
-        {
-                defrag->total_files += 1;
-                defrag->total_data += iatt.ia_size;
-        }
-        UNLOCK (&defrag->lock);
-
-        if (defrag->stats == _gf_true) {
-                gettimeofday (&end, NULL);
-                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
-                          (end.tv_usec - start.tv_usec);
-                gf_log (this->name, GF_LOG_INFO, "Migration of "
-                        "file:%s size:%"PRIu64" bytes took %.2f"
-                        "secs and ret: %d", entry_loc.name,
-                        iatt.ia_size, elapsed/1e6, ret);
-        }
-
-out:
-        loc_wipe (&entry_loc);
+                UNLOCK(&defrag->lock);
 
-        return ret;
-
-}
-
-void *
-gf_defrag_task (void *opaque)
-{
-        struct list_head        *q_head         = NULL;
-        struct dht_container    *iterator       = NULL;
-        gf_defrag_info_t        *defrag         = NULL;
-        int                      ret            = 0;
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                       "File migration skipped for %s.", entry_loc.path);
+            }
 
+        } else if (fop_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " hardlink %s ",
+                         entry_loc.path);
+            LOCK(&defrag->lock);
+            {
+                defrag->skipped += 1;
+            }
+            UNLOCK(&defrag->lock);
 
-        defrag = (gf_defrag_info_t *)opaque;
-        if (!defrag) {
-                gf_msg ("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL");
-                goto out;
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                   "File migration skipped for %s.", entry_loc.path);
         }
 
-        q_head = &(defrag->queue[0].list);
-
-       /* The following while loop will dequeue one entry from the defrag->queue
-          under lock. We will update the defrag->global_error only when there
-          is an error which is critical to stop the rebalance process. The stop
-          message will be intimated to other migrator threads by setting the
-          defrag->defrag_status to GF_DEFRAG_STATUS_FAILED.
-
-          In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is
-          maintained so that crawler does not starve the file migration
-          workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that
-          crawler does not go far ahead in filling up the queue.
-        */
-
-        while (_gf_true) {
+        ret = 0;
+        goto out;
+    } else if (ret < 0) {
+        if (fop_errno != EEXIST) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "migrate-data failed for %s",
+                   entry_loc.path);
 
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        pthread_cond_broadcast (
-                                &defrag->rebalance_crawler_alarm);
-                        pthread_cond_broadcast (
-                                &defrag->parallel_migration_cond);
-                        goto out;
-                }
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
+        }
 
-                pthread_mutex_lock (&defrag->dfq_mutex);
-                {
+        ret = gf_defrag_handle_migrate_error(fop_errno, defrag);
 
-                        /*Throttle down:
-                          If the reconfigured count is less than current thread
-                          count, then the current thread will sleep */
-
-                        /*TODO: Need to refactor the following block to work
-                         *under defrag->lock. For now access
-                         * defrag->current_thread_count and rthcount under
-                         * dfq_mutex lock */
-                        while (!defrag->crawl_done &&
-                              (defrag->recon_thread_count <
-                                        defrag->current_thread_count)) {
-                                defrag->current_thread_count--;
-                                gf_log ("DHT", GF_LOG_INFO,
-                                        "Thread sleeping. "
-                                        "defrag->current_thread_count: %d",
-                                         defrag->current_thread_count);
-
-                                pthread_cond_wait (
-                                           &defrag->df_wakeup_thread,
-                                           &defrag->dfq_mutex);
-
-                                defrag->current_thread_count++;
-
-                                gf_log ("DHT", GF_LOG_INFO,
-                                        "Thread wokeup. "
-                                        "defrag->current_thread_count: %d",
-                                         defrag->current_thread_count);
-                        }
+        if (!ret) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "migrate-data on %s failed:", entry_loc.path);
+        } else if (ret == 1) {
+            ret = 0;
+        }
+
+        goto out;
+    }
+
+    LOCK(&defrag->lock);
+    {
+        defrag->total_files += 1;
+        defrag->total_data += iatt.ia_size;
+    }
+    UNLOCK(&defrag->lock);
+
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&end, NULL);
+        elapsed = gf_tvdiff(&start, &end);
+        gf_log(this->name, GF_LOG_INFO,
+               "Migration of "
+               "file:%s size:%" PRIu64
+               " bytes took %.2f"
+               "secs and ret: %d",
+               entry_loc.name, iatt.ia_size, elapsed / 1e6, ret);
+    }
 
-                        if (defrag->q_entry_count) {
-                                iterator = list_entry (q_head->next,
-                                                typeof(*iterator), list);
-
-                                gf_msg_debug ("DHT", 0, "picking entry "
-                                              "%s", iterator->df_entry->d_name);
-
-                                list_del_init (&(iterator->list));
-
-                                defrag->q_entry_count--;
-
-                                if ((defrag->q_entry_count <
-                                        MIN_MIGRATE_QUEUE_COUNT) &&
-                                        defrag->wakeup_crawler) {
-                                        pthread_cond_broadcast (
-                                              &defrag->rebalance_crawler_alarm);
-                                }
-                                pthread_mutex_unlock (&defrag->dfq_mutex);
-                                ret = gf_defrag_migrate_single_file
-                                                        ((void *)iterator);
-
-                                /*Critical errors: ENOTCONN and ENOSPACE*/
-                                if (ret) {
-                                        dht_set_global_defrag_error
-                                                         (defrag, ret);
-
-                                        defrag->defrag_status =
-                                                       GF_DEFRAG_STATUS_FAILED;
-
-                                        pthread_cond_broadcast (
-                                             &defrag->rebalance_crawler_alarm);
-
-                                        pthread_cond_broadcast (
-                                             &defrag->parallel_migration_cond);
-
-                                        goto out;
-                                }
-
-                                gf_defrag_free_container (iterator);
-
-                                continue;
-                        } else {
-
-                        /* defrag->crawl_done flag is set means crawling
-                         file system is done and hence a list_empty when
-                         the above flag is set indicates there are no more
-                         entries to be added to the queue and rebalance is
-                         finished */
-
-                                if (!defrag->crawl_done) {
-                                        pthread_cond_wait (
-                                           &defrag->parallel_migration_cond,
-                                           &defrag->dfq_mutex);
-                                }
-
-                                if (defrag->crawl_done &&
-                                                 !defrag->q_entry_count) {
-                                        pthread_cond_broadcast (
-                                             &defrag->parallel_migration_cond);
-                                        goto unlock;
-                                } else {
-                                        pthread_mutex_unlock
-                                                 (&defrag->dfq_mutex);
-                                        continue;
-                                }
-                        }
+out:
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
 
-                }
-unlock:
-                pthread_mutex_unlock (&defrag->dfq_mutex);
-                break;
+    if (iatt_ptr) {
+        LOCK(&defrag->lock);
+        {
+            defrag->size_processed += iatt_ptr->ia_size;
         }
-out:
-        return NULL;
+        UNLOCK(&defrag->lock);
+    }
+    loc_wipe(&entry_loc);
+
+    return ret;
 }
 
-int static
-gf_defrag_get_entry (xlator_t *this, int i, struct dht_container **container,
-                     loc_t *loc, dht_conf_t *conf, gf_defrag_info_t *defrag,
-                     fd_t *fd, dict_t *migrate_data,
-                     struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req,
-                     int *should_commit_hash)
+void *
+gf_defrag_task(void *opaque)
 {
-        int                     ret             = -1;
-        char                    is_linkfile     = 0;
-        gf_dirent_t            *df_entry        = NULL;
-        loc_t                   entry_loc       = {0,};
-        dict_t                 *xattr_rsp       = NULL;
-        struct iatt             iatt            = {0,};
-        struct dht_container   *tmp_container   = NULL;
-        xlator_t               *hashed_subvol   = NULL;
-        xlator_t               *cached_subvol   = NULL;
-
+    struct list_head *q_head = NULL;
+    struct dht_container *iterator = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    pid_t pid = GF_CLIENT_PID_DEFRAG;
+
+    defrag = (gf_defrag_info_t *)opaque;
+    if (!defrag) {
+        gf_msg("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL");
+        goto out;
+    }
+
+    syncopctx_setfspid(&pid);
+
+    q_head = &(defrag->queue[0].list);
+
+    /* The following while loop will dequeue one entry from the defrag->queue
+       under lock. We will update the defrag->global_error only when there
+       is an error which is critical to stop the rebalance process. The stop
+       message will be intimated to other migrator threads by setting the
+       defrag->defrag_status to GF_DEFRAG_STATUS_FAILED.
+
+       In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is
+       maintained so that crawler does not starve the file migration
+       workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that
+       crawler does not go far ahead in filling up the queue.
+     */
+
+    while (_gf_true) {
         if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                ret = -1;
-                goto out;
-        }
-
-        if (dir_dfmeta->offset_var[i].readdir_done == 1) {
-                ret = 0;
-                goto out;
+            pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+            pthread_cond_broadcast(&defrag->parallel_migration_cond);
+            goto out;
         }
 
-        if (dir_dfmeta->fetch_entries[i] == 1) {
-                ret = syncop_readdirp (conf->local_subvols[i], fd, 131072,
-                                       dir_dfmeta->offset_var[i].offset,
-                                       &(dir_dfmeta->equeue[i]),
-                                       NULL, NULL);
-                if (ret == 0) {
-                        dir_dfmeta->offset_var[i].readdir_done = 1;
-                        ret = 0;
-                        goto out;
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle down:
+              If the reconfigured count is less than current thread
+              count, then the current thread will sleep */
+
+            /*TODO: Need to refactor the following block to work
+             *under defrag->lock. For now access
+             * defrag->current_thread_count and rthcount under
+             * dfq_mutex lock */
+            while (!defrag->crawl_done && (defrag->recon_thread_count <
+                                           defrag->current_thread_count)) {
+                defrag->current_thread_count--;
+                gf_msg_debug("DHT", 0,
+                             "Thread sleeping. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+
+                pthread_cond_wait(&defrag->df_wakeup_thread,
+                                  &defrag->dfq_mutex);
+
+                defrag->current_thread_count++;
+                gf_msg_debug("DHT", 0,
+                             "Thread wokeup. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+            }
+
+            if (defrag->q_entry_count) {
+                iterator = list_entry(q_head->next, typeof(*iterator), list);
+
+                gf_msg_debug("DHT", 0,
+                             "picking entry "
+                             "%s",
+                             iterator->df_entry->d_name);
+
+                list_del_init(&(iterator->list));
+
+                defrag->q_entry_count--;
+
+                if ((defrag->q_entry_count < MIN_MIGRATE_QUEUE_COUNT) &&
+                    defrag->wakeup_crawler) {
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
                 }
+                pthread_mutex_unlock(&defrag->dfq_mutex);
+                ret = gf_defrag_migrate_single_file((void *)iterator);
 
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_DATA_FAILED,
-                                "%s: Migrate data failed: Readdir returned"
-                                " %s. Aborting migrate-data", loc->path,
-                                strerror(-ret));
-                        ret = -1;
-                        goto out;
-                }
+                /*Critical errors: ENOTCONN and ENOSPACE*/
+                if (ret) {
+                    dht_set_global_defrag_error(defrag, ret);
 
-                if (list_empty (&(dir_dfmeta->equeue[i].list))) {
-                        dir_dfmeta->offset_var[i].readdir_done = 1;
-                        ret = 0;
-                        goto out;
-                }
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
 
-                dir_dfmeta->fetch_entries[i] = 0;
-        }
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
 
-        while (1) {
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
 
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        ret = -1;
-                        goto out;
+                    goto out;
                 }
 
-                df_entry = list_entry (dir_dfmeta->iterator[i]->next,
-                                       typeof (*df_entry), list);
-
-                if (&df_entry->list == dir_dfmeta->head[i]) {
-                        gf_dirent_free (&(dir_dfmeta->equeue[i]));
-                        INIT_LIST_HEAD (&(dir_dfmeta->equeue[i].list));
-                        dir_dfmeta->fetch_entries[i] = 1;
-                        dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
-                        ret = 0;
-                        goto out;
+                gf_defrag_free_container(iterator);
+
+                continue;
+            } else {
+                /* defrag->crawl_done flag is set means crawling
+                 file system is done and hence a list_empty when
+                 the above flag is set indicates there are no more
+                 entries to be added to the queue and rebalance is
+                 finished */
+
+                if (!defrag->crawl_done) {
+                    defrag->current_thread_count--;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread "
+                                 "sleeping while  waiting "
+                                 "for migration entries. "
+                                 "current thread  count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_cond_wait(&defrag->parallel_migration_cond,
+                                      &defrag->dfq_mutex);
                 }
 
-                dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next;
-
-                dir_dfmeta->offset_var[i].offset = df_entry->d_off;
-                if (!strcmp (df_entry->d_name, ".") ||
-                    !strcmp (df_entry->d_name, ".."))
-                        continue;
-
-                if (IA_ISDIR (df_entry->d_stat.ia_type))
-                        continue;
-
-                defrag->num_files_lookedup++;
+                if (defrag->crawl_done && !defrag->q_entry_count) {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0, "Exiting thread");
 
-                if (defrag->defrag_pattern &&
-                    (gf_defrag_pattern_match (defrag, df_entry->d_name,
-                                              df_entry->d_stat.ia_size)
-                     == _gf_false)) {
-                        continue;
-                }
-
-                loc_wipe (&entry_loc);
-                ret = dht_build_child_loc (this, &entry_loc, loc,
-                                          df_entry->d_name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Child loc"
-                                " build failed");
-                        ret = -1;
-                        goto out;
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
+                    goto unlock;
+                } else {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread woke up"
+                                 " as found migrating entries. "
+                                 "current thread count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_mutex_unlock(&defrag->dfq_mutex);
+                    continue;
                 }
+            }
+        }
+    unlock:
+        pthread_mutex_unlock(&defrag->dfq_mutex);
+        break;
+    }
+out:
+    return NULL;
+}
 
-                if (gf_uuid_is_null (df_entry->d_stat.ia_gfid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_NULL,
-                                "%s/%s gfid not present", loc->path,
-                                df_entry->d_name);
-                        continue;
-                }
+int static gf_defrag_get_entry(xlator_t *this, int i,
+                               struct dht_container **container, loc_t *loc,
+                               dht_conf_t *conf, gf_defrag_info_t *defrag,
+                               fd_t *fd, dict_t *migrate_data,
+                               struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req,
+                               int *perrno)
+{
+    int ret = 0;
+    char is_linkfile = 0;
+    gf_dirent_t *df_entry = NULL;
+    struct dht_container *tmp_container = NULL;
 
-                gf_uuid_copy (entry_loc.gfid, df_entry->d_stat.ia_gfid);
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
 
-                if (gf_uuid_is_null (loc->gfid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_NULL,
-                                "%s/%s gfid not present", loc->path,
-                                df_entry->d_name);
-                        continue;
-                }
+    if (dir_dfmeta->offset_var[i].readdir_done == 1) {
+        ret = 0;
+        goto out;
+    }
 
-                gf_uuid_copy (entry_loc.pargfid, loc->gfid);
+    if (dir_dfmeta->fetch_entries[i] == 1) {
+        if (!fd) {
+            dir_dfmeta->fetch_entries[i] = 0;
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
+        }
 
-                entry_loc.inode->ia_type = df_entry->d_stat.ia_type;
+        ret = syncop_readdirp(conf->local_subvols[i], fd, 131072,
+                              dir_dfmeta->offset_var[i].offset,
+                              &(dir_dfmeta->equeue[i]), xattr_req, NULL);
+        if (ret == 0) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
+        }
 
-                if (xattr_rsp) {
-                        dict_unref (xattr_rsp);
-                        xattr_rsp = NULL;
-                }
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_DATA_FAILED,
+                   "Readdirp failed. Aborting data migration for "
+                   "directory: %s",
+                   loc->path);
+            *perrno = -ret;
+            ret = -1;
+            goto out;
+        }
 
-                ret = syncop_lookup (conf->local_subvols[i], &entry_loc,
-                                        &iatt, NULL, xattr_req, &xattr_rsp);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:%s lookup failed",
-                                entry_loc.path);
-
-                        if (-ret != ENOENT && -ret != ESTALE) {
-
-                                defrag->total_failures++;
-
-                                if (conf->decommission_in_progress) {
-                                        ret = -1;
-                                        goto out;
-                                } else {
-                                       *should_commit_hash = 0;
-                                        continue;
-                                }
-                        }
+        if (list_empty(&(dir_dfmeta->equeue[i].list))) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
+        }
 
-                        continue;
-                }
+        dir_dfmeta->fetch_entries[i] = 0;
+    }
 
+    while (1) {
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+            ret = -1;
+            goto out;
+        }
 
-                is_linkfile = check_is_linkfile (NULL, &iatt, xattr_rsp,
-                                                conf->link_xattr_name);
+        df_entry = list_entry(dir_dfmeta->iterator[i]->next, typeof(*df_entry),
+                              list);
 
-                if (is_linkfile) {
-                        /* No need to add linkto file to the queue for
-                           migration. Only the actual data file need to
-                           be checked for migration criteria.
-                        */
-                        gf_msg_debug (this->name, 0, "Skipping linkfile"
-                                      " %s on subvol: %s", entry_loc.path,
-                                      conf->local_subvols[i]->name);
-                        continue;
-                }
+        if (&df_entry->list == dir_dfmeta->head[i]) {
+            gf_dirent_free(&(dir_dfmeta->equeue[i]));
+            INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+            dir_dfmeta->fetch_entries[i] = 1;
+            dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+            ret = 0;
+            goto out;
+        }
 
+        dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next;
 
-                ret = syncop_lookup (this, &entry_loc, NULL, NULL,
-                                     NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:%s lookup failed",
-                                entry_loc.path);
-
-                        if (-ret != ENOENT && -ret != ESTALE) {
-
-                                defrag->total_failures++;
-
-                                if (conf->decommission_in_progress) {
-                                        ret = -1;
-                                        goto out;
-                                } else {
-                                        *should_commit_hash = 0;
-                                        continue;
-                                }
-                        }
+        dir_dfmeta->offset_var[i].offset = df_entry->d_off;
+        if (!strcmp(df_entry->d_name, ".") || !strcmp(df_entry->d_name, ".."))
+            continue;
 
-                        continue;
-                }
+        if (IA_ISDIR(df_entry->d_stat.ia_type)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
-               /* if distribute is present, it will honor this key.
-                * -1, ENODATA is returned if distribute is not present
-                * or file doesn't have a link-file. If file has
-                * link-file, the path of link-file will be the value,
-                * and also that guarantees that file has to be mostly
-                * migrated */
-
-                hashed_subvol = dht_subvol_get_hashed (this, &entry_loc);
-                if (!hashed_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        continue;
-                }
+        defrag->num_files_lookedup++;
 
-                cached_subvol = dht_subvol_get_cached (this, entry_loc.inode);
-                if (!cached_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_CACHED_SUBVOL_GET_FAILED,
-                                "Failed to get cached subvol for %s",
-                                loc->path);
+        if (defrag->defrag_pattern &&
+            (gf_defrag_pattern_match(defrag, df_entry->d_name,
+                                     df_entry->d_stat.ia_size) == _gf_false)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
-                        continue;
-                }
+        is_linkfile = check_is_linkfile(NULL, &df_entry->d_stat, df_entry->dict,
+                                        conf->link_xattr_name);
 
-                if (hashed_subvol == cached_subvol) {
-                        continue;
-                }
+        if (is_linkfile) {
+            /* No need to add linkto file to the queue for
+               migration. Only the actual data file need to
+               be checked for migration criteria.
+            */
 
-               /*Build Container Structure */
+            gf_msg_debug(this->name, 0,
+                         "Skipping linkfile"
+                         " %s on subvol: %s",
+                         df_entry->d_name, conf->local_subvols[i]->name);
+            continue;
+        }
 
-                tmp_container =  GF_CALLOC (1, sizeof(struct dht_container),
-                                            gf_dht_mt_container_t);
-                if (!tmp_container) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
-                                "memory for container");
-                        ret = -1;
-                        goto out;
-                }
-                tmp_container->df_entry = gf_dirent_for_name (df_entry->d_name);
-                if (!tmp_container->df_entry) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
-                                "memory for df_entry");
-                        ret = -1;
-                        goto out;
-                }
+        /*Build Container Structure */
 
-                tmp_container->df_entry->d_stat = df_entry->d_stat;
+        tmp_container = GF_CALLOC(1, sizeof(struct dht_container),
+                                  gf_dht_mt_container_t);
+        if (!tmp_container) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for container");
+            ret = -1;
+            goto out;
+        }
+        tmp_container->df_entry = gf_dirent_for_name(df_entry->d_name);
+        if (!tmp_container->df_entry) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for df_entry");
+            ret = -1;
+            goto out;
+        }
 
-                tmp_container->df_entry->d_ino  = df_entry->d_ino;
+        tmp_container->local_subvol_index = i;
 
-                tmp_container->df_entry->d_type = df_entry->d_type;
+        tmp_container->df_entry->d_stat = df_entry->d_stat;
 
-                tmp_container->df_entry->d_len  = df_entry->d_len;
+        tmp_container->df_entry->d_ino = df_entry->d_ino;
 
-                tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc),
-                                                      gf_dht_mt_loc_t);
-                if (!tmp_container->parent_loc) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
-                                "memory for loc");
-                        ret = -1;
-                        goto out;
-                }
+        tmp_container->df_entry->d_type = df_entry->d_type;
 
+        tmp_container->df_entry->d_len = df_entry->d_len;
 
-                ret = loc_copy (tmp_container->parent_loc, loc);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "loc_copy failed");
-                        ret = -1;
-                        goto out;
-                }
+        tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc), gf_dht_mt_loc_t);
+        if (!tmp_container->parent_loc) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for loc");
+            ret = -1;
+            goto out;
+        }
 
-                tmp_container->migrate_data = migrate_data;
+        ret = loc_copy(tmp_container->parent_loc, loc);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "loc_copy failed");
+            ret = -1;
+            goto out;
+        }
 
-                tmp_container->this = this;
+        tmp_container->migrate_data = migrate_data;
 
-                if (df_entry->dict)
-                        tmp_container->df_entry->dict =
-                                        dict_ref (df_entry->dict);
+        tmp_container->this = this;
 
-               /*Build Container Structue >> END*/
+        if (df_entry->dict)
+            tmp_container->df_entry->dict = dict_ref(df_entry->dict);
 
-                ret = 0;
-                goto out;
+        /*Build Container Structure >> END*/
 
-        }
+        ret = 0;
+        goto out;
+    }
 
 out:
-        loc_wipe (&entry_loc);
-
-        if (ret == 0) {
-                *container = tmp_container;
-        } else {
-                if (tmp_container) {
-                        gf_defrag_free_container (tmp_container);
-                }
+    if (ret == 0) {
+        *container = tmp_container;
+    } else {
+        if (tmp_container) {
+            gf_defrag_free_container(tmp_container);
         }
+    }
 
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
-        return ret;
+    return ret;
 }
 
 int
-gf_defrag_process_dir (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                       dict_t *migrate_data)
+gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *migrate_data, int *perrno)
 {
-        int                      ret               = -1;
-        fd_t                    *fd                = NULL;
-        dht_conf_t              *conf              = NULL;
-        gf_dirent_t              entries;
-        dict_t                  *dict              = NULL;
-        dict_t                  *xattr_req         = NULL;
-        struct timeval           dir_start         = {0,};
-        struct timeval           end               = {0,};
-        double                   elapsed           = {0,};
-        int                      local_subvols_cnt = 0;
-        int                      i                 = 0;
-        int                      j                 = 0;
-        struct  dht_container   *container         = NULL;
-        int                      ldfq_count        = 0;
-        int                      dfc_index         = 0;
-        int                      throttle_up       = 0;
-        struct dir_dfmeta       *dir_dfmeta        = NULL;
-        int                      should_commit_hash = 1;
-
-        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
-                loc->path);
-        gettimeofday (&dir_start, NULL);
-
-        conf = this->private;
-        local_subvols_cnt = conf->local_subvols_cnt;
-
-        if (!local_subvols_cnt) {
-                ret = 0;
-                goto out;
-        }
-
-        fd = fd_create (loc->inode, defrag->pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_DATA_FAILED,
-                        "Migrate data failed: Failed to open dir %s",
-                        loc->path);
-                ret = -1;
-                goto out;
-        }
-
-        fd_bind (fd);
-        dir_dfmeta = GF_CALLOC (1, sizeof (*dir_dfmeta),
-                                                gf_common_mt_pointer);
-        if (!dir_dfmeta) {
-                gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta is NULL");
-                ret = -1;
-                goto out;
-        }
-
-
-        dir_dfmeta->head = GF_CALLOC (local_subvols_cnt,
-                                      sizeof (*(dir_dfmeta->head)),
-                                      gf_common_mt_pointer);
-        if (!dir_dfmeta->head) {
-                gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL");
-                ret = -1;
-                goto out;
-        }
-
-        dir_dfmeta->iterator = GF_CALLOC (local_subvols_cnt,
-                                          sizeof (*(dir_dfmeta->iterator)),
-                                          gf_common_mt_pointer);
-        if (!dir_dfmeta->iterator) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dir_dfmeta->iterator is NULL");
-                ret = -1;
-                goto out;
-        }
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_dirent_t entries;
+    dict_t *xattr_req = NULL;
+    struct timeval dir_start = {
+        0,
+    };
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    int local_subvols_cnt = 0;
+    int i = 0;
+    int j = 0;
+    struct dht_container *container = NULL;
+    int ldfq_count = 0;
+    int dfc_index = 0;
+    int throttle_up = 0;
+    struct dir_dfmeta *dir_dfmeta = NULL;
+    xlator_t *old_THIS = NULL;
+
+    gf_log(this->name, GF_LOG_INFO, "migrate data called on %s", loc->path);
+    gettimeofday(&dir_start, NULL);
+
+    conf = this->private;
+    local_subvols_cnt = conf->local_subvols_cnt;
+
+    if (!local_subvols_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        dir_dfmeta->equeue = GF_CALLOC (local_subvols_cnt, sizeof (entries),
-                                        gf_dht_mt_dirent_t);
-        if (!dir_dfmeta->equeue) {
-                gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL");
-                ret = -1;
-                goto out;
-        }
+    old_THIS = THIS;
+    THIS = this;
 
-        dir_dfmeta->offset_var = GF_CALLOC (local_subvols_cnt,
-                                            sizeof (dht_dfoffset_ctx_t),
-                                            gf_dht_mt_octx_t);
-        if (!dir_dfmeta->offset_var) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dir_dfmeta->offset_var is NULL");
-                ret = -1;
-                goto out;
-        }
-        ret = gf_defrag_ctx_subvols_init (dir_dfmeta->offset_var, this);
+    dir_dfmeta = GF_CALLOC(1, sizeof(*dir_dfmeta), gf_common_mt_pointer);
+    if (!dir_dfmeta) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->lfd = GF_CALLOC(local_subvols_cnt, sizeof(fd_t *),
+                                gf_common_mt_pointer);
+    if (!dir_dfmeta->lfd) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta", NULL);
+        ret = -1;
+        *perrno = ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        dir_dfmeta->lfd[i] = fd_create(loc->inode, defrag->pid);
+        if (!dir_dfmeta->lfd[i]) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_FD_CREATE_FAILED,
+                    NULL);
+            *perrno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_opendir(conf->local_subvols[i], loc, dir_dfmeta->lfd[i],
+                             NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "dht_dfoffset_ctx_t"
-                        "initialization failed");
-                ret = -1;
-                goto out;
-        }
-
-        dir_dfmeta->fetch_entries = GF_CALLOC (local_subvols_cnt,
-                                               sizeof (int), gf_common_mt_int);
-        if (!dir_dfmeta->fetch_entries) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dir_dfmeta->fetch_entries is NULL");
+            fd_unref(dir_dfmeta->lfd[i]);
+            dir_dfmeta->lfd[i] = NULL;
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FAILED_TO_OPEN,
+                    "dir: %s", loc->path, "subvol: %s",
+                    conf->local_subvols[i]->name, NULL);
+
+            if (conf->decommission_in_progress) {
+                *perrno = -ret;
                 ret = -1;
                 goto out;
+            }
+        } else {
+            fd_bind(dir_dfmeta->lfd[i]);
         }
+    }
 
-        for (i = 0; i < local_subvols_cnt ; i++) {
-                INIT_LIST_HEAD (&(dir_dfmeta->equeue[i].list));
-                dir_dfmeta->head[i]          = &(dir_dfmeta->equeue[i].list);
-                dir_dfmeta->iterator[i]      = dir_dfmeta->head[i];
-                dir_dfmeta->fetch_entries[i] = 1;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-               gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-               ret = -1;
-               goto out;
-        }
-
-        ret = dict_set_uint32 (xattr_req,
-                               conf->link_xattr_name, 256);
-        if (ret) {
-               gf_log (this->name, GF_LOG_ERROR, "failed to set dict for "
-                       "key: %s", conf->link_xattr_name);
-               ret = -1;
-               goto out;
-        }
-
-        /*
-         Job: Read entries from each local subvol and store the entries
-              in equeue array of linked list. Now pick one entry from the
-              equeue array in a round robin basis and add them to defrag Queue.
-        */
-
-        while (!dht_dfreaddirp_done(dir_dfmeta->offset_var,
-                                         local_subvols_cnt)) {
-
-                pthread_mutex_lock (&defrag->dfq_mutex);
-                {
-
-                      /*Throttle up: If reconfigured count is higher than
-                        current thread count, wake up the sleeping threads
-                        TODO: Need to refactor this. Instead of making the
-                        thread sleep and wake, we should terminate and spawn
-                        threads on-demand*/
-
-                        if (defrag->recon_thread_count >
-                                         defrag->current_thread_count) {
-                                throttle_up =
-                                        (defrag->recon_thread_count -
-                                            defrag->current_thread_count);
-                                for (j = 0; j < throttle_up; j++) {
-                                        pthread_cond_signal (
-                                             &defrag->df_wakeup_thread);
-                                }
-
-                        }
+    dir_dfmeta->head = GF_CALLOC(local_subvols_cnt, sizeof(*(dir_dfmeta->head)),
+                                 gf_common_mt_pointer);
+    if (!dir_dfmeta->head) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->iterator = GF_CALLOC(local_subvols_cnt,
+                                     sizeof(*(dir_dfmeta->iterator)),
+                                     gf_common_mt_pointer);
+    if (!dir_dfmeta->iterator) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->iterator is NULL");
+        ret = -1;
+        goto out;
+    }
 
-                        while (defrag->q_entry_count >
-                                        MAX_MIGRATE_QUEUE_COUNT) {
-                                defrag->wakeup_crawler = 1;
-                                pthread_cond_wait (
-                                        &defrag->rebalance_crawler_alarm,
-                                        &defrag->dfq_mutex);
-                        }
+    dir_dfmeta->equeue = GF_CALLOC(local_subvols_cnt, sizeof(entries),
+                                   gf_dht_mt_dirent_t);
+    if (!dir_dfmeta->equeue) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL");
+        ret = -1;
+        goto out;
+    }
 
-                       ldfq_count = defrag->q_entry_count;
+    dir_dfmeta->offset_var = GF_CALLOC(
+        local_subvols_cnt, sizeof(dht_dfoffset_ctx_t), gf_dht_mt_octx_t);
+    if (!dir_dfmeta->offset_var) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->offset_var is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_defrag_ctx_subvols_init(dir_dfmeta->offset_var, this);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "dht_dfoffset_ctx_t"
+               "initialization failed");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->fetch_entries = GF_CALLOC(local_subvols_cnt, sizeof(int),
+                                          gf_common_mt_int);
+    if (!dir_dfmeta->fetch_entries) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta->fetch_entries", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+        dir_dfmeta->head[i] = &(dir_dfmeta->equeue[i].list);
+        dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+        dir_dfmeta->fetch_entries[i] = 1;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set dict for "
+               "key: %s",
+               conf->link_xattr_name);
+        ret = -1;
+        goto out;
+    }
 
-                       if (defrag->wakeup_crawler) {
-                               defrag->wakeup_crawler = 0;
-                       }
+    /*
+     Job: Read entries from each local subvol and store the entries
+          in equeue array of linked list. Now pick one entry from the
+          equeue array in a round robin basis and add them to defrag Queue.
+    */
 
+    while (!dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle up: If reconfigured count is higher than
+              current thread count, wake up the sleeping threads
+              TODO: Need to refactor this. Instead of making the
+              thread sleep and wake, we should terminate and spawn
+              threads on-demand*/
+
+            if (defrag->recon_thread_count > defrag->current_thread_count) {
+                throttle_up = (defrag->recon_thread_count -
+                               defrag->current_thread_count);
+                for (j = 0; j < throttle_up; j++) {
+                    pthread_cond_signal(&defrag->df_wakeup_thread);
                 }
-                pthread_mutex_unlock (&defrag->dfq_mutex);
-
-                while (ldfq_count <= MAX_MIGRATE_QUEUE_COUNT &&
-                       !dht_dfreaddirp_done(dir_dfmeta->offset_var,
-                                               local_subvols_cnt)) {
-
-                        ret = gf_defrag_get_entry (this, dfc_index, &container,
-                                                   loc, conf, defrag, fd,
-                                                   migrate_data, dir_dfmeta,
-                                                   xattr_req,
-                                                   &should_commit_hash);
-                        if (ret) {
-                                gf_log ("DHT", GF_LOG_INFO, "Found critical "
-                                        "error from gf_defrag_get_entry");
-                                ret = -1;
-                                goto out;
-                        }
+            }
 
-                        /* Check if we got an entry, else we need to move the
-                           index to the next subvol */
-                        if (!container) {
-                                GF_CRAWL_INDEX_MOVE(dfc_index,
-                                                    local_subvols_cnt);
-                                continue;
-                        }
+            while (defrag->q_entry_count > MAX_MIGRATE_QUEUE_COUNT) {
+                defrag->wakeup_crawler = 1;
+                pthread_cond_wait(&defrag->rebalance_crawler_alarm,
+                                  &defrag->dfq_mutex);
+            }
 
-                        /* Q this entry in the dfq */
-                        pthread_mutex_lock (&defrag->dfq_mutex);
-                        {
-                                list_add_tail (&container->list,
-                                        &(defrag->queue[0].list));
-                                defrag->q_entry_count++;
-                                ldfq_count = defrag->q_entry_count;
-
-                                gf_msg_debug (this->name, 0, "added "
-                                              "file:%s parent:%s to the queue ",
-                                              container->df_entry->d_name,
-                                              container->parent_loc->path);
-
-                                pthread_cond_signal (
-                                        &defrag->parallel_migration_cond);
-                        }
-                        pthread_mutex_unlock (&defrag->dfq_mutex);
+            ldfq_count = defrag->q_entry_count;
 
-                        GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
-                }
+            if (defrag->wakeup_crawler) {
+                defrag->wakeup_crawler = 0;
+            }
         }
+        pthread_mutex_unlock(&defrag->dfq_mutex);
 
-        gettimeofday (&end, NULL);
-        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
-                  (end.tv_usec - dir_start.tv_usec);
-        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
-                "%.2f secs", loc->path, elapsed/1e6);
-        ret = 0;
-out:
-
-        GF_FREE_DIR_DFMETA (dir_dfmeta);
+        while (
+            ldfq_count <= MAX_MIGRATE_QUEUE_COUNT &&
+            !dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+            ret = gf_defrag_get_entry(this, dfc_index, &container, loc, conf,
+                                      defrag, dir_dfmeta->lfd[dfc_index],
+                                      migrate_data, dir_dfmeta, xattr_req,
+                                      perrno);
 
-        if (dict)
-                dict_unref(dict);
-
-        if (xattr_req)
-                dict_unref(xattr_req);
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED) {
+                goto out;
+            }
 
-        if (fd)
-                fd_unref (fd);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Found "
+                       "error from gf_defrag_get_entry");
 
-        if (ret == 0 && should_commit_hash == 0) {
-                ret = 2;
-        }
-
-        return ret;
+                ret = -1;
+                goto out;
+            }
+
+            /* Check if we got an entry, else we need to move the
+               index to the next subvol */
+            if (!container) {
+                GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
+                continue;
+            }
+
+            /* Q this entry in the dfq */
+            pthread_mutex_lock(&defrag->dfq_mutex);
+            {
+                list_add_tail(&container->list, &(defrag->queue[0].list));
+                defrag->q_entry_count++;
+                ldfq_count = defrag->q_entry_count;
+
+                gf_msg_debug(this->name, 0,
+                             "added "
+                             "file:%s parent:%s to the queue ",
+                             container->df_entry->d_name,
+                             container->parent_loc->path);
+
+                pthread_cond_signal(&defrag->parallel_migration_cond);
+            }
+            pthread_mutex_unlock(&defrag->dfq_mutex);
+
+            GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
+        }
+    }
+
+    gettimeofday(&end, NULL);
+    elapsed = gf_tvdiff(&dir_start, &end);
+    gf_log(this->name, GF_LOG_INFO,
+           "Migration operation on dir %s took "
+           "%.2f secs",
+           loc->path, elapsed / 1e6);
+    ret = 0;
+out:
+    THIS = old_THIS;
+    gf_defrag_free_dir_dfmeta(dir_dfmeta, local_subvols_cnt);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    /* It does not matter if it errored out - this number is
+     * used to calculate rebalance estimated time to complete.
+     * No locking required as dirs are processed by a single thread.
+     */
+    defrag->num_dirs_processed++;
+    return ret;
 }
+
 int
-gf_defrag_settle_hash (xlator_t *this, gf_defrag_info_t *defrag,
-                       loc_t *loc, dict_t *fix_layout)
+gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *fix_layout)
 {
-        int     ret;
-        dht_conf_t *conf = NULL;
-        /*
-         * Now we're ready to update the directory commit hash for the volume
-         * root, so that hash miscompares and broadcast lookups can stop.
-         * However, we want to skip that if fix-layout is all we did.  In
-         * that case, we want the miscompares etc. to continue until a real
-         * rebalance is complete.
+    int ret;
+    dht_conf_t *conf = NULL;
+    /*
+     * Now we're ready to update the directory commit hash for the volume
+     * root, so that hash miscompares and broadcast lookups can stop.
+     * However, we want to skip that if fix-layout is all we did.  In
+     * that case, we want the miscompares etc. to continue until a real
+     * rebalance is complete.
+     */
+    if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX ||
+        defrag->cmd == GF_DEFRAG_CMD_DETACH_START) {
+        return 0;
+    }
+
+    conf = this->private;
+    if (!conf) {
+        /*Uh oh
          */
-        if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX
-            || defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
-                return 0;
-        }
+        return -1;
+    }
 
-        conf = this->private;
-        if (!conf) {
-                /*Uh oh
-                 */
-                return -1;
-        }
+    if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) {
+        /* Commit hash updates are only done on local subvolumes and
+         * only when lookup optimization is needed (for older client
+         * support)
+         */
+        return 0;
+    }
 
-        if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) {
-                /* Commit hash updates are only done on local subvolumes and
-                 * only when lookup optmization is needed (for older client
-                 * support)
-                 */
-                return 0;
-        }
+    ret = dict_set_uint32(fix_layout, "new-commit-hash",
+                          defrag->new_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set new-commit-hash");
+        return -1;
+    }
 
-        ret = dict_set_uint32 (fix_layout, "new-commit-hash",
-                               defrag->new_commit_hash);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set new-commit-hash");
-                return -1;
-        }
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+               "fix layout on %s failed", loc->path);
 
-        ret = syncop_setxattr (this, loc, fix_layout, 0, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "fix layout on %s failed", loc->path);
-                return -1;
+        if (-ret == ENOENT || -ret == ESTALE) {
+            /* Dir most likely is deleted */
+            return 0;
         }
 
-        /* TBD: find more efficient solution than adding/deleting every time */
-        dict_del(fix_layout, "new-commit-hash");
-
-        return 0;
-}
+        return -1;
+    }
 
+    /* TBD: find more efficient solution than adding/deleting every time */
+    dict_del(fix_layout, "new-commit-hash");
 
+    return 0;
+}
 
-/* Function for doing a named lookup on file inodes during an attach tier
- * So that a hardlink lookup heal i.e gfid to parent gfid lookup heal
- * happens on pre-existing data. This is required so that the ctr database has
- * hardlinks of all the exisitng file in the volume. CTR xlator on the
- * brick/server side does db update/insert of the hardlink on a namelookup.
- * Currently the namedlookup is done synchronous to the fixlayout that is
- * triggered by attach tier. This is not performant, adding more time to
- * fixlayout. The performant approach is record the hardlinks on a compressed
- * datastore and then do the namelookup asynchronously later, giving the ctr db
- * eventual consistency
- * */
 int
-gf_fix_layout_tier_attach_lookup (xlator_t *this,
-                                 loc_t *parent_loc,
-                                 gf_dirent_t *file_dentry)
+gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                     dict_t *fix_layout, dict_t *migrate_data)
 {
-        int                      ret            = -1;
-        dict_t                  *lookup_xdata   = NULL;
-        dht_conf_t              *conf           = NULL;
-        loc_t                    file_loc       = {0,};
-        struct iatt              iatt           = {0,};
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, parent_loc, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, file_dentry, out);
+    int ret = -1;
+    loc_t entry_loc = {
+        0,
+    };
+    fd_t *fd = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_boolean_t free_entries = _gf_false;
+    off_t offset = 0;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL, *inode = NULL;
+    dht_conf_t *conf = NULL;
+    int perrno = 0;
+
+    conf = this->private;
+    if (!conf) {
+        ret = -1;
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    ret = syncop_lookup(this, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        if (strcmp(loc->path, "/") == 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
 
-        if (!parent_loc->inode) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "%s/%s parent is NULL", parent_loc->path,
-                        file_dentry->d_name);
-                goto out;
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
         }
 
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "Dir:%s renamed or removed. Skipping", loc->path);
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
 
-        conf   = this->private;
-
-        loc_wipe (&file_loc);
-
-        if (gf_uuid_is_null (file_dentry->d_stat.ia_gfid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "%s/%s gfid not present", parent_loc->path,
-                        file_dentry->d_name);
-                goto out;
+            defrag->total_failures++;
+            goto out;
         }
+    }
 
-        gf_uuid_copy (file_loc.gfid, file_dentry->d_stat.ia_gfid);
+    fd = fd_create(loc->inode, defrag->pid);
+    if (!fd) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to create fd");
+        ret = -1;
+        goto out;
+    }
 
-        if (gf_uuid_is_null (parent_loc->gfid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "%s/%s"
-                        " gfid not present", parent_loc->path,
-                        file_dentry->d_name);
-                goto out;
+    ret = syncop_opendir(this, loc, fd, NULL, NULL);
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
         }
 
-        gf_uuid_copy (file_loc.pargfid, parent_loc->gfid);
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to open dir %s, "
+               "err:%d",
+               loc->path, -ret);
 
+        ret = -1;
+        goto out;
+    }
 
-        ret = dht_build_child_loc (this, &file_loc, parent_loc,
-                                                file_dentry->d_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Child loc build failed");
-                ret = -1;
-                goto out;
-        }
+    fd_bind(fd);
+    INIT_LIST_HEAD(&entries.list);
 
-        lookup_xdata = dict_new ();
-        if (!lookup_xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed creating lookup dict for %s",
-                        file_dentry->d_name);
+    while ((ret = syncop_readdirp(this, fd, 131072, offset, &entries, NULL,
+                                  NULL)) != 0) {
+        if (ret < 0) {
+            if (-ret == ENOENT || -ret == ESTALE) {
+                if (conf->decommission_subvols_cnt) {
+                    defrag->total_failures++;
+                }
+                ret = 0;
                 goto out;
-        }
+            }
 
-        ret = dict_set_int32 (lookup_xdata, CTR_ATTACH_TIER_LOOKUP, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to set lookup flag");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_READDIR_ERROR,
+                   "readdirp failed for "
+                   "path %s. Aborting fix-layout",
+                   loc->path);
+
+            ret = -1;
+            goto out;
         }
 
-        gf_uuid_copy (file_loc.parent->gfid, parent_loc->gfid);
+        if (list_empty(&entries.list))
+            break;
 
-        /* Sending lookup to cold tier only */
-        ret = syncop_lookup (conf->subvolumes[0], &file_loc, &iatt,
-                        NULL, lookup_xdata, NULL);
-        if (ret) {
-                /* If the file does not exist on the cold tier than it must */
-                /* have been discovered on the hot tier. This is not an error. */
-                gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS,
-                        "%s lookup to cold tier on attach heal failed", file_loc.path);
+        free_entries = _gf_true;
+
+        list_for_each_entry_safe(entry, tmp, &entries.list, list)
+        {
+            if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                ret = 1;
                 goto out;
-        }
+            }
 
-        ret = 0;
+            offset = entry->d_off;
 
-out:
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
+            if (!IA_ISDIR(entry->d_stat.ia_type)) {
+                continue;
+            }
+            loc_wipe(&entry_loc);
 
-        loc_wipe (&file_loc);
+            ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Child loc"
+                       " build failed for entry: %s",
+                       entry->d_name);
 
-        if (lookup_xdata)
-                dict_unref (lookup_xdata);
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
 
-        return ret;
-}
+                    goto out;
+                } else {
+                    continue;
+                }
+            }
+
+            if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
+
+            /*In case the gfid stored in the inode by inode_link
+             * and the gfid obtained in the lookup differs, then
+             * client3_3_lookup_cbk will return ESTALE and proper
+             * error will be captured
+             */
+
+            linked_inode = inode_link(entry_loc.inode, loc->inode,
+                                      entry->d_name, &entry->d_stat);
+
+            inode = entry_loc.inode;
+            entry_loc.inode = linked_inode;
+            inode_unref(inode);
+
+            if (gf_uuid_is_null(loc->gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.pargfid, loc->gfid);
+
+            ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
+            if (ret) {
+                if (-ret == ENOENT || -ret == ESTALE) {
+                    gf_msg(this->name, GF_LOG_INFO, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED,
+                           "Dir:%s renamed or removed. "
+                           "Skipping",
+                           loc->path);
+                    ret = 0;
+                    if (conf->decommission_subvols_cnt) {
+                        defrag->total_failures++;
+                    }
+                    continue;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED, "lookup failed for:%s",
+                           entry_loc.path);
 
+                    defrag->total_failures++;
 
-int
-gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                  dict_t *fix_layout, dict_t *migrate_data)
-{
-        int                      ret            = -1;
-        loc_t                    entry_loc      = {0,};
-        fd_t                    *fd             = NULL;
-        gf_dirent_t              entries;
-        gf_dirent_t             *tmp            = NULL;
-        gf_dirent_t             *entry          = NULL;
-        gf_boolean_t             free_entries   = _gf_false;
-        off_t                    offset         = 0;
-        struct iatt              iatt           = {0,};
-        inode_t                 *linked_inode   = NULL, *inode = NULL;
-        dht_conf_t              *conf           = NULL;
-        int                      should_commit_hash = 1;
-
-        conf = this->private;
-        if (!conf) {
-                ret = -1;
-                goto out;
-        }
+                    if (conf->decommission_in_progress) {
+                        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                        ret = -1;
+                        goto out;
+                    } else {
+                        continue;
+                    }
+                }
+            }
 
+            /* A return value of 2 means, either process_dir or
+             * lookup of a dir failed. Hence, don't commit hash
+             * for the current directory*/
 
+            ret = gf_defrag_fix_layout(this, defrag, &entry_loc, fix_layout,
+                                       migrate_data);
 
-        ret = syncop_lookup (this, loc, &iatt, NULL, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
-                        loc->path);
-                ret = -1;
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED ||
+                defrag->defrag_status == GF_DEFRAG_STATUS_FAILED) {
                 goto out;
-        }
+            }
 
-        if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) &&
-            (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)) {
-                ret = gf_defrag_process_dir (this, defrag, loc, migrate_data);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                       "Fix layout failed for %s", entry_loc.path);
 
-                if (ret && ret != 2) {
-                        defrag->total_failures++;
+                defrag->total_failures++;
 
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DEFRAG_PROCESS_DIR_FAILED,
-                                "gf_defrag_process_dir failed for directory: %s"
-                                , loc->path);
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
 
-                        if (conf->decommission_in_progress) {
-                                goto out;
-                        }
-
-                        should_commit_hash = 0;
-                } else if (ret == 2) {
-                        should_commit_hash = 0;
+                    goto out;
+                } else {
+                    /* Let's not commit-hash if
+                     * gf_defrag_fix_layout failed*/
+                    continue;
                 }
-        }
+            }
+        }
+
+        gf_dirent_free(&entries);
+        free_entries = _gf_false;
+        INIT_LIST_HEAD(&entries.list);
+    }
+
+    /* A directory layout is fixed only after its subdirs are healed to
+     * any newly added bricks. If the layout is fixed before subdirs are
+     * healed, the newly added brick will get a non-null layout.
+     * Any subdirs which hash to that layout will no longer show up
+     * in a directory listing until they are healed.
+     */
+
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+
+    /* In case of a race where the directory is deleted just before
+     * layout setxattr, the errors are updated in the layout structure.
+     * We can use this information to make a decision whether the directory
+     * is deleted entirely.
+     */
+    if (ret == 0) {
+        ret = dht_dir_layout_error_check(this, loc->inode);
+        ret = -ret;
+    }
+
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed. Dir %s "
+                   "renamed or removed",
+                   loc->path);
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed for %s", loc->path);
 
-        gf_msg_trace (this->name, 0, "fix layout called on %s", loc->path);
+            defrag->total_failures++;
 
-        fd = fd_create (loc->inode, defrag->pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
+            if (conf->decommission_in_progress) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
                 ret = -1;
                 goto out;
+            }
         }
+    }
+
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        ret = gf_defrag_process_dir(this, defrag, loc, migrate_data, &perrno);
 
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
-                        loc->path);
-                ret = -1;
+            if (perrno == ENOENT || perrno == ESTALE) {
+                ret = 0;
                 goto out;
-        }
+            } else {
+                defrag->total_failures++;
 
-        fd_bind (fd);
-        INIT_LIST_HEAD (&entries.list);
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, &entries,
-                                       NULL, NULL)) != 0)
-        {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       DHT_MSG_DEFRAG_PROCESS_DIR_FAILED,
+                       "gf_defrag_process_dir failed for "
+                       "directory: %s",
+                       loc->path);
 
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s"
-                                ". Aborting fix-layout",strerror(-ret));
-                        ret = -1;
-                        goto out;
+                if (conf->decommission_in_progress) {
+                    goto out;
                 }
+            }
+        }
+    }
 
-                if (list_empty (&entries.list))
-                        break;
-
-                free_entries = _gf_true;
-
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                                ret = 1;
-                                goto out;
-                        }
-
-                        offset = entry->d_off;
-
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
-                        if (!IA_ISDIR (entry->d_stat.ia_type)) {
-
-                                /* If its a fix layout during the attach
-                                 * tier operation do lookups on files
-                                 * on cold subvolume so that there is a
-                                 * CTR DB Lookup Heal triggered on existing
-                                 * data.
-                                 * */
-                                if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                                        gf_fix_layout_tier_attach_lookup
-                                                (this, loc, entry);
-                                }
-
-                                continue;
-                        }
-                        loc_wipe (&entry_loc);
-
-                        ret = dht_build_child_loc (this, &entry_loc, loc,
-                                                  entry->d_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
-                                        " build failed for entry: %s",
-                                        entry->d_name);
-
-                                if (conf->decommission_in_progress) {
-                                        defrag->defrag_status =
-                                        GF_DEFRAG_STATUS_FAILED;
-
-                                        goto out;
-                                } else {
-                                        should_commit_hash = 0;
-
-                                        continue;
-                                }
-                        }
-
-                        if (gf_uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
-
-
-                        gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
-
-                        /*In case the gfid stored in the inode by inode_link
-                         * and the gfid obtained in the lookup differs, then
-                         * client3_3_lookup_cbk will return ESTALE and proper
-                         * error will be captured
-                         */
-
-                        linked_inode = inode_link (entry_loc.inode, loc->inode,
-                                                   entry->d_name,
-                                                   &entry->d_stat);
-
-                        inode = entry_loc.inode;
-                        entry_loc.inode = linked_inode;
-                        inode_unref (inode);
-
-                        if (gf_uuid_is_null (loc->gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+    gf_msg_trace(this->name, 0, "fix layout called on %s", loc->path);
 
-                        gf_uuid_copy (entry_loc.pargfid, loc->gfid);
+    if (gf_defrag_settle_hash(this, defrag, loc, fix_layout) != 0) {
+        defrag->total_failures++;
 
-                        ret = syncop_lookup (this, &entry_loc, &iatt, NULL,
-                                             NULL, NULL);
-                        /*Check whether it is ENOENT or ESTALE*/
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s"
-                                        " lookup failed with %d",
-                                        entry_loc.path, -ret);
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SETTLE_HASH_FAILED,
+               "Settle hash failed for %s", loc->path);
 
-                                if (!conf->decommission_in_progress &&
-                                    -ret != ENOENT && -ret != ESTALE) {
-                                        should_commit_hash = 0;
-                                }
+        ret = -1;
 
-                                continue;
-                        }
+        if (conf->decommission_in_progress) {
+            defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+            goto out;
+        }
+    }
 
-                        ret = syncop_setxattr (this, &entry_loc, fix_layout,
-                                               0, NULL, NULL);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Setxattr "
-                                        "failed for %s", entry_loc.path);
+    ret = 0;
+out:
+    if (free_entries)
+        gf_dirent_free(&entries);
 
-                                defrag->total_failures++;
+    loc_wipe(&entry_loc);
 
-                                /*Don't go for fix-layout of child subtree if"
-                                  fix-layout failed*/
-                                if (conf->decommission_in_progress) {
-                                        defrag->defrag_status =
-                                        GF_DEFRAG_STATUS_FAILED;
+    if (fd)
+        fd_unref(fd);
 
-                                        ret = -1;
+    return ret;
+}
 
-                                        goto out;
-                                } else {
-                                        continue;
-                                }
-                        }
+int
+dht_init_local_subvols_and_nodeuuids(xlator_t *this, dht_conf_t *conf,
+                                     loc_t *loc)
+{
+    dict_t *dict = NULL;
+    uuid_t *uuid_ptr = NULL;
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+
+    /* Find local subvolumes */
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_FIND_LOCAL_SUBVOL, NULL,
+                          NULL);
+    if (ret && (ret != -ENODATA)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+
+    if (!ret)
+        goto out;
+
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_OLD_FIND_LOCAL_SUBVOL,
+                          NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 
+out:
+    if (ret) {
+        return ret;
+    }
 
-                        /* A return value of 2 means, either process_dir or
-                         * lookup of a dir failed. Hence, don't commit hash
-                         * for the current directory*/
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: "
+               "%s",
+               conf->local_subvols[i]->name);
 
-                        ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
-                                                    fix_layout, migrate_data);
+        for (j = 0; j < conf->local_nodeuuids[i].count; j++) {
+            uuid_ptr = &(conf->local_nodeuuids[i].elements[j].uuid);
+            gf_msg(this->name, GF_LOG_INFO, 0, 0, "node uuid : %s",
+                   uuid_utoa(*uuid_ptr));
+        }
+    }
 
-                        if (ret && ret != 2) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LAYOUT_FIX_FAILED,
-                                        "Fix layout failed for %s",
-                                        entry_loc.path);
+    return ret;
+}
 
-                                defrag->total_failures++;
+/* Functions for the rebalance estimates feature */
 
-                                if (conf->decommission_in_progress) {
-                                        defrag->defrag_status =
-                                        GF_DEFRAG_STATUS_FAILED;
+uint64_t
+gf_defrag_subvol_file_size(xlator_t *this, loc_t *root_loc)
+{
+    int ret = -1;
+    struct statvfs buf = {
+        0,
+    };
+
+    ret = syncop_statfs(this, root_loc, &buf, NULL, NULL);
+    if (ret) {
+        /* Aargh! */
+        return 0;
+    }
+    return ((buf.f_blocks - buf.f_bfree) * buf.f_frsize);
+}
 
-                                        ret = -1;
+uint64_t
+gf_defrag_total_file_size(xlator_t *this, loc_t *root_loc)
+{
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    uint64_t size_files = 0;
+    uint64_t total_size = 0;
 
-                                        goto out;
-                                } else {
-                                        /* Let's not commit-hash if
-                                         * gf_defrag_fix_layout failed*/
-                                        continue;
-                                }
-                        }
+    conf = this->private;
+    if (!conf) {
+        return 0;
+    }
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        size_files = gf_defrag_subvol_file_size(conf->local_subvols[i],
+                                                root_loc);
+        total_size += size_files;
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: %s,"
+               "cnt = %" PRIu64,
+               conf->local_subvols[i]->name, size_files);
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, 0, "Total size files = %" PRIu64,
+           total_size);
+
+    return total_size;
+}
 
-                        if (ret != 2 &&
-                            gf_defrag_settle_hash (this, defrag, &entry_loc,
-                            fix_layout) != 0) {
-                                defrag->total_failures++;
+static void *
+dht_file_counter_thread(void *args)
+{
+    gf_defrag_info_t *defrag = NULL;
+    loc_t root_loc = {
+        0,
+    };
+    struct timespec time_to_wait = {
+        0,
+    };
+    uint64_t tmp_size = 0;
+
+    if (!args)
+        return NULL;
 
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_SETTLE_HASH_FAILED,
-                                        "Settle hash failed for %s",
-                                        entry_loc.path);
+    defrag = (gf_defrag_info_t *)args;
+    dht_build_root_loc(defrag->root_inode, &root_loc);
 
-                                ret = -1;
+    while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        timespec_now(&time_to_wait);
+        time_to_wait.tv_sec += 600;
 
-                                if (conf->decommission_in_progress) {
-                                        defrag->defrag_status =
-                                        GF_DEFRAG_STATUS_FAILED;
+        pthread_mutex_lock(&defrag->fc_mutex);
+        pthread_cond_timedwait(&defrag->fc_wakeup_cond, &defrag->fc_mutex,
+                               &time_to_wait);
 
-                                        goto out;
-                                }
-                        }
-                }
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                INIT_LIST_HEAD (&entries.list);
-        }
+        pthread_mutex_unlock(&defrag->fc_mutex);
 
-        ret = 0;
-out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED)
+            break;
 
-        loc_wipe (&entry_loc);
+        tmp_size = gf_defrag_total_file_size(defrag->this, &root_loc);
 
-        if (fd)
-                fd_unref (fd);
+        gf_log("dht", GF_LOG_INFO, "tmp data size =%" PRIu64, tmp_size);
 
-        if (ret == 0 && should_commit_hash == 0) {
-                ret = 2;
+        if (!tmp_size) {
+            gf_msg("dht", GF_LOG_ERROR, 0, 0,
+                   "Failed to get "
+                   "the total data size. Unable to estimate "
+                   "time to complete rebalance.");
+        } else {
+            g_totalsize = tmp_size;
+            gf_msg_debug("dht", 0, "total data size =%" PRIu64, g_totalsize);
         }
+    }
 
-        return ret;
-
+    return NULL;
 }
 
-
-
-/******************************************************************************
- *                      Tier background Fix layout functions
- ******************************************************************************/
-/* This is the background tier fixlayout thread */
-void *
-gf_tier_do_fix_layout (void *args)
+int
+gf_defrag_estimates_cleanup(xlator_t *this, gf_defrag_info_t *defrag,
+                            pthread_t filecnt_thread)
 {
-        gf_tier_fix_layout_arg_t *tier_fix_layout_arg   =  args;
-        int                 ret                         = -1;
-        xlator_t            *this                       = NULL;
-        dht_conf_t          *conf                       = NULL;
-        gf_defrag_info_t    *defrag                     = NULL;
-        dict_t              *dict                       = NULL;
-        loc_t               loc                         = {0,};
-        struct iatt         iatt                        = {0,};
-        struct iatt         parent                      = {0,};
-
-        GF_VALIDATE_OR_GOTO ("tier", tier_fix_layout_arg, out);
-        GF_VALIDATE_OR_GOTO ("tier", tier_fix_layout_arg->this, out);
-        this = tier_fix_layout_arg->this;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-        GF_VALIDATE_OR_GOTO (this->name, defrag->root_inode, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, tier_fix_layout_arg->fix_layout, out);
+    int ret = -1;
+
+    /* Wake up the filecounter thread.
+     * By now the defrag status will no longer be
+     * GF_DEFRAG_STATUS_STARTED so the thread will exit the loop.
+     */
+    pthread_mutex_lock(&defrag->fc_mutex);
+    {
+        pthread_cond_broadcast(&defrag->fc_wakeup_cond);
+    }
+    pthread_mutex_unlock(&defrag->fc_mutex);
+
+    ret = pthread_join(filecnt_thread, NULL);
+    if (ret) {
+        gf_msg("dht", GF_LOG_ERROR, ret, 0,
+               "file_counter_thread: pthread_join failed.");
+        ret = -1;
+    }
+    return ret;
+}
 
+int
+gf_defrag_estimates_init(xlator_t *this, loc_t *loc, pthread_t *filecnt_thread)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+
+    conf = this->private;
+    defrag = conf->defrag;
+
+    g_totalsize = gf_defrag_total_file_size(this, loc);
+    if (!g_totalsize) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Failed to get "
+               "the total data size. Unable to estimate "
+               "time to complete rebalance.");
+        goto out;
+    }
+
+    ret = gf_thread_create(filecnt_thread, NULL, dht_file_counter_thread,
+                           (void *)defrag, "dhtfcnt");
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, 0,
+               "Failed to "
+               "create the file counter thread ");
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
 
-        /* Get Root loc_t */
-        dht_build_root_loc (defrag->root_inode, &loc);
-        ret = syncop_lookup (this, &loc, &iatt, &parent, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Lookup on root failed.");
-                ret = -1;
-                goto out;
-        }
+/* Init and cleanup functions for parallel file migration*/
+int
+gf_defrag_parallel_migration_init(xlator_t *this, gf_defrag_info_t *defrag,
+                                  pthread_t **tid_array, int *thread_index)
+{
+    int ret = -1;
+    int thread_spawn_count = 0;
+    int index = 0;
+    pthread_t *tid = NULL;
 
+    if (!defrag)
+        goto out;
 
-        /* Start the crawl */
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS, "Tiering Fixlayout started");
+    /* Initialize global entry queue */
+    defrag->queue = GF_CALLOC(1, sizeof(struct dht_container),
+                              gf_dht_mt_container_t);
 
-        ret = gf_defrag_fix_layout (this, defrag, &loc,
-                                    tier_fix_layout_arg->fix_layout, NULL);
-        if (ret && ret != 2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_FAILED,
-                        "Tiering fixlayout failed.");
-                ret = -1;
-                goto out;
-        }
+    if (!defrag->queue) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to initialise migration queue");
+        ret = -1;
+        goto out;
+    }
 
-        if (ret != 2 && gf_defrag_settle_hash
-                        (this, defrag, &loc,
-                                tier_fix_layout_arg->fix_layout) != 0) {
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
-        }
+    INIT_LIST_HEAD(&(defrag->queue[0].list));
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    thread_spawn_count = MAX(MAX_REBAL_THREADS, 4);
 
-        ret = dict_set_str (dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, "yes");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        GF_XATTR_TIER_LAYOUT_FIXED_KEY);
-                ret = -1;
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "thread_spawn_count: %d", thread_spawn_count);
 
-        /* Marking the completion of tiering fix layout via a xattr on root */
-        ret = syncop_setxattr (this, &loc, dict, 0, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set tiering fix "
-                        "layout completed xattr on %s", loc.path);
-                ret = -1;
-                goto out;
+    tid = GF_CALLOC(thread_spawn_count, sizeof(pthread_t),
+                    gf_common_mt_pthread_t);
+    if (!tid) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to create migration threads");
+        ret = -1;
+        goto out;
+    }
+    defrag->current_thread_count = thread_spawn_count;
+
+    /*Spawn Threads Here*/
+    while (index < thread_spawn_count) {
+        ret = gf_thread_create(&(tid[index]), NULL, gf_defrag_task,
+                               (void *)defrag, "dhtmig%d", (index + 1) & 0x3ff);
+        if (ret != 0) {
+            gf_msg("DHT", GF_LOG_ERROR, ret, 0, "Thread[%d] creation failed. ",
+                   index);
+            ret = -1;
+            goto out;
+        } else {
+            gf_log("DHT", GF_LOG_INFO,
+                   "Thread[%d] "
+                   "creation successful",
+                   index);
         }
+        index++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret)
-                defrag->total_failures++;
-
-        if (dict)
-                dict_unref (dict);
+    *thread_index = index;
+    *tid_array = tid;
 
-        return NULL;
+    return ret;
 }
 
 int
-gf_tier_start_fix_layout (xlator_t *this,
-                         loc_t *loc,
-                         gf_defrag_info_t *defrag,
-                         dict_t *fix_layout)
+gf_defrag_parallel_migration_cleanup(gf_defrag_info_t *defrag,
+                                     pthread_t *tid_array, int thread_index)
 {
-        int ret                                       = -1;
-        dict_t  *tier_dict                            = NULL;
-        gf_tier_fix_layout_arg_t *tier_fix_layout_arg = NULL;
-
-        tier_dict = dict_new ();
-        if (!tier_dict) {
-                gf_log ("tier", GF_LOG_ERROR, "Tier fix layout failed :"
-                        "Creation of tier_dict failed");
-                ret = -1;
-                goto out;
-        }
+    int ret = -1;
+    int i = 0;
 
-        /* Check if layout is fixed already */
-        ret = syncop_getxattr (this, loc, &tier_dict,
-                                GF_XATTR_TIER_LAYOUT_FIXED_KEY,
-                                NULL, NULL);
-        if (ret != 0) {
+    if (!defrag)
+        goto out;
 
-                tier_fix_layout_arg = &defrag->tier_conf.tier_fix_layout_arg;
+    /* Wake up all migration threads */
+    pthread_mutex_lock(&defrag->dfq_mutex);
+    {
+        defrag->crawl_done = 1;
 
-                /*Fill crawl arguments */
-                tier_fix_layout_arg->this = this;
-                tier_fix_layout_arg->fix_layout = fix_layout;
+        pthread_cond_broadcast(&defrag->parallel_migration_cond);
+        pthread_cond_broadcast(&defrag->df_wakeup_thread);
+    }
+    pthread_mutex_unlock(&defrag->dfq_mutex);
 
-                /* Spawn the fix layout thread so that its done in the
-                 * background */
-                ret = pthread_create (&tier_fix_layout_arg->thread_id, NULL,
-                                gf_tier_do_fix_layout, tier_fix_layout_arg);
-                if (ret) {
-                        gf_log ("tier", GF_LOG_ERROR, "Thread creation failed. "
-                                "Background fix layout for tiering will not "
-                                "work.");
-                        defrag->total_failures++;
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        if (tier_dict)
-                dict_unref (tier_dict);
+    /*Wait for all the threads to complete their task*/
+    for (i = 0; i < thread_index; i++) {
+        pthread_join(tid_array[i], NULL);
+    }
 
-        return ret;
-}
+    GF_FREE(tid_array);
 
-void
-gf_tier_clear_fix_layout (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
-{
-        int ret         = -1;
-        dict_t *dict    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-
-        /* Check if background fixlayout is completed. This is not
-         * multi-process safe i.e there is a possibility that by the time
-         * we move to remove the xattr there it might have been cleared by some
-         * other detach process from other node. We ignore the error if such
-         * a thing happens */
-        ret = syncop_getxattr (this, loc, &dict,
-                        GF_XATTR_TIER_LAYOUT_FIXED_KEY, NULL, NULL);
-        if (ret) {
-                /* Background fixlayout not complete - nothing to clear*/
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Unable to retrieve fixlayout xattr."
-                        "Assume background fix layout not complete");
-                goto out;
-        }
+    /* Cleanup the migration queue */
+    if (defrag->queue) {
+        gf_dirent_free(defrag->queue[0].df_entry);
+        INIT_LIST_HEAD(&(defrag->queue[0].list));
+    }
 
-        ret = syncop_removexattr (this, loc, GF_XATTR_TIER_LAYOUT_FIXED_KEY,
-                                  NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Failed removing tier fix layout "
-                        "xattr from %s", loc->path);
-                goto out;
-        }
-        ret = 0;
-out:
-        if (dict)
-                dict_unref (dict);
-}
+    GF_FREE(defrag->queue);
 
-void
-gf_tier_wait_fix_lookup (gf_defrag_info_t *defrag) {
-        if (defrag->tier_conf.tier_fix_layout_arg.thread_id) {
-                pthread_join (defrag->tier_conf.tier_fix_layout_arg.thread_id,
-                        NULL);
-        }
+    ret = 0;
+out:
+    return ret;
 }
-/******************Tier background Fix layout functions END********************/
-
-
 
 int
-gf_defrag_start_crawl (void *data)
+gf_defrag_start_crawl(void *data)
 {
-        xlator_t                *this                   = NULL;
-        dht_conf_t              *conf                   = NULL;
-        gf_defrag_info_t        *defrag                 = NULL;
-        int                      ret                    = -1;
-        loc_t                    loc                    = {0,};
-        struct iatt              iatt                   = {0,};
-        struct iatt              parent                 = {0,};
-        dict_t                  *fix_layout             = NULL;
-        dict_t                  *migrate_data           = NULL;
-        dict_t                  *status                 = NULL;
-        dict_t                  *dict                   = NULL;
-        glusterfs_ctx_t         *ctx                    = NULL;
-        dht_methods_t           *methods                = NULL;
-        int                      i                      = 0;
-        int                     thread_index            = 0;
-        int                     err                     = 0;
-        int                     thread_spawn_count      = 0;
-        pthread_t tid[MAX_MIGRATOR_THREAD_COUNT];
-        gf_boolean_t            is_tier_detach          = _gf_false;
-
-        this = data;
-        if (!this)
-                goto exit;
-
-        ctx = this->ctx;
-        if (!ctx)
-                goto exit;
-
-        conf = this->private;
-        if (!conf)
-                goto exit;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto exit;
-
-        gettimeofday (&defrag->start_time, NULL);
-        dht_build_root_inode (this, &defrag->root_inode);
-        if (!defrag->root_inode)
-                goto out;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *fix_layout = NULL;
+    dict_t *migrate_data = NULL;
+    dict_t *status = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent = {
+        0,
+    };
+    int thread_index = 0;
+    pthread_t *tid = NULL;
+    pthread_t filecnt_thread;
+    gf_boolean_t fc_thread_started = _gf_false;
+
+    this = data;
+    if (!this)
+        goto exit;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto exit;
+
+    conf = this->private;
+    if (!conf)
+        goto exit;
+
+    defrag = conf->defrag;
+    if (!defrag)
+        goto exit;
+
+    defrag->start_time = gf_time();
+
+    dht_build_root_inode(this, &defrag->root_inode);
+    if (!defrag->root_inode)
+        goto out;
+
+    dht_build_root_loc(defrag->root_inode, &loc);
+
+    /* fix-layout on '/' first */
+
+    ret = syncop_lookup(this, &loc, &iatt, &parent, NULL, NULL);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance: look up on / failed");
+        ret = -1;
+        goto out;
+    }
 
-        dht_build_root_loc (defrag->root_inode, &loc);
+    old_THIS = THIS;
+    THIS = this;
 
-        /* fix-layout on '/' first */
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
 
-        ret = syncop_lookup (this, &loc, &iatt, &parent, NULL, NULL);
+    /* async statfs update for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, &loc);
+    THIS = old_THIS;
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Failed to start rebalance: look up on / failed");
-                ret = -1;
-                goto out;
-        }
+    fix_layout = dict_new();
+    if (!fix_layout) {
+        ret = -1;
+        goto out;
+    }
+
+    /*
+     * Unfortunately, we can't do special xattrs (like fix.layout) and
+     * real ones in the same call currently, and changing it seems
+     * riskier than just doing two calls.
+     */
+
+    gf_log(this->name, GF_LOG_INFO, "%s using commit hash %u", __func__,
+           conf->vol_commit_hash);
+
+    ret = dict_set_uint32(fix_layout, conf->commithash_xattr_name,
+                          conf->vol_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set %s",
+               conf->commithash_xattr_name);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set commit hash on %s. "
+               "Rebalance cannot proceed.",
+               loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    /* We now return to our regularly scheduled program. */
+
+    ret = dict_set_str(fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance:"
+               "Failed to set dictionary value: key = %s",
+               GF_XATTR_FIX_LAYOUT_KEY);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        fix_layout = dict_new ();
-        if (!fix_layout) {
-                ret = -1;
-                goto out;
-        }
+    defrag->new_commit_hash = conf->vol_commit_hash;
 
-        /*
-         * Unfortunately, we can't do special xattrs (like fix.layout) and
-         * real ones in the same call currently, and changing it seems
-         * riskier than just doing two calls.
-         */
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_FAILED,
+               "fix layout on %s failed", loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_INFO, "%s using commit hash %u",
-                __func__, conf->vol_commit_hash);
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        /* We need to migrate files */
 
-        ret = dict_set_uint32 (fix_layout, conf->commithash_xattr_name,
-                               conf->vol_commit_hash);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set %s", conf->commithash_xattr_name);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
+        migrate_data = dict_new();
+        if (!migrate_data) {
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
         }
-
-        ret = syncop_setxattr (this, &loc, fix_layout, 0, NULL, NULL);
+        ret = dict_set_str(
+            migrate_data, GF_XATTR_FILE_MIGRATE_KEY,
+            (defrag->cmd == GF_DEFRAG_CMD_START_FORCE) ? "force" : "non-force");
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "fix layout on %s failed",
-                        loc.path);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
         }
 
-        /* We now return to our regularly scheduled program. */
-
-        ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+        ret = dht_init_local_subvols_and_nodeuuids(this, conf, &loc);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Failed to start rebalance:"
-                        "Failed to set dictionary value: key = %s",
-                        GF_XATTR_FIX_LAYOUT_KEY);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
 
-        defrag->new_commit_hash = conf->vol_commit_hash;
-
-        ret = syncop_setxattr (this, &loc, fix_layout, 0, NULL, NULL);
+        /* Initialise the structures required for parallel migration */
+        ret = gf_defrag_parallel_migration_init(this, defrag, &tid,
+                                                &thread_index);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_FAILED,
-                        "fix layout on %s failed",
-                        loc.path);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
-        }
-
-        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
-                migrate_data = dict_new ();
-                if (!migrate_data) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (migrate_data, GF_XATTR_FILE_MIGRATE_KEY,
-                        (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)
-                        ?  "force" : "non-force");
-                if (ret) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-
-                /* Find local subvolumes */
-                ret = syncop_getxattr (this, &loc, &dict,
-                                       GF_REBAL_FIND_LOCAL_SUBVOL,
-                                       NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local "
-                                "subvolume determination failed with error: %d",
-                                -ret);
-                        ret = -1;
-                        goto out;
-                }
-
-                for (i = 0 ; i < conf->local_subvols_cnt; i++) {
-                        gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvols "
-                                "are %s", conf->local_subvols[i]->name);
-                }
-
-                /* Initialize global entry queue */
-                defrag->queue = GF_CALLOC (1, sizeof (struct dht_container),
-                                           gf_dht_mt_container_t);
-
-                if (!defrag->queue) {
-                        gf_log (this->name, GF_LOG_INFO, "No memory for queue");
-                        ret = -1;
-                        goto out;
-                }
-
-                INIT_LIST_HEAD (&(defrag->queue[0].list));
-
-                thread_spawn_count = MAX ((sysconf(_SC_NPROCESSORS_ONLN) - 4), 4);
-
-                gf_msg_debug (this->name, 0, "thread_spawn_count: %d",
-                              thread_spawn_count);
-
-                defrag->current_thread_count = thread_spawn_count;
-
-                /*Spawn Threads Here*/
-                while (thread_index < thread_spawn_count) {
-                        err = pthread_create(&(tid[thread_index]), NULL,
-                                     &gf_defrag_task, (void *)defrag);
-                        if (err != 0) {
-                                gf_log ("DHT", GF_LOG_ERROR,
-                                        "Thread[%d] creation failed. "
-                                        "Aborting Rebalance",
-                                         thread_index);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                gf_log ("DHT", GF_LOG_INFO, "Thread[%d] "
-                                        "creation successful", thread_index);
-                        }
-                        thread_index++;
-                }
-        }
-
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                /* Fix layout for attach tier */
-                ret = gf_tier_start_fix_layout (this, &loc, defrag, fix_layout);
-                if (ret) {
-                        goto out;
-                }
-
-                methods = &(conf->methods);
-
-                /* Calling tier_start of tier.c */
-                methods->migration_other(this, defrag);
-                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
-
-                        ret = dict_set_str (migrate_data,
-                                            GF_XATTR_FILE_MIGRATE_KEY,
-                                            "force");
-                        if (ret)
-                                goto out;
-
-                }
-        } else {
-                ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
-                                    migrate_data);
-                if (ret && ret != 2) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (ret != 2 && gf_defrag_settle_hash
-                        (this, defrag, &loc, fix_layout) != 0) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER)
-                        is_tier_detach = _gf_true;
-
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Aborting rebalance.");
+            goto out;
         }
 
-        gf_log ("DHT", GF_LOG_INFO, "crawling file-system completed");
-out:
-
-        /* We are here means crawling the entire file system is done
-           or something failed. Set defrag->crawl_done flag to intimate
-           the migrator threads to exhaust the defrag->queue and terminate*/
-
+        ret = gf_defrag_estimates_init(this, &loc, &filecnt_thread);
         if (ret) {
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+            /* Not a fatal error. Allow the rebalance to proceed*/
+            ret = 0;
+        } else {
+            fc_thread_started = _gf_true;
         }
+    }
 
-        pthread_mutex_lock (&defrag->dfq_mutex);
-        {
-                defrag->crawl_done = 1;
+    ret = gf_defrag_fix_layout(this, defrag, &loc, fix_layout, migrate_data);
+    if (ret) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-                pthread_cond_broadcast (
-                        &defrag->parallel_migration_cond);
-                pthread_cond_broadcast (
-                        &defrag->df_wakeup_thread);
-        }
-        pthread_mutex_unlock (&defrag->dfq_mutex);
+    if (gf_defrag_settle_hash(this, defrag, &loc, fix_layout) != 0) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        /*Wait for all the threads to complete their task*/
-        for (i = 0; i < thread_index; i++) {
-                pthread_join (tid[i], NULL);
-        }
+    gf_log("DHT", GF_LOG_INFO, "crawling file-system completed");
+out:
 
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                /* Wait for the tier fixlayout to
-                 * complete if its was started.*/
-                 gf_tier_wait_fix_lookup (defrag);
-        }
+    /* We are here means crawling the entire file system is done
+       or something failed. Set defrag->crawl_done flag to intimate
+       the migrator threads to exhaust the defrag->queue and terminate*/
 
-        if (is_tier_detach && ret == 0) {
-                /* If it was a detach remove the tier fix-layout
-                * xattr on root. Ignoring the failure, as nothing has to be
-                * done, logging is done in gf_tier_clear_fix_layout */
-                gf_tier_clear_fix_layout (this, &loc, defrag);
-        }
+    if (ret) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+    }
 
-        if (defrag->queue) {
-                gf_dirent_free (defrag->queue[0].df_entry);
-                INIT_LIST_HEAD (&(defrag->queue[0].list));
-        }
+    gf_defrag_parallel_migration_cleanup(defrag, tid, thread_index);
 
-        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
-            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
-                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
-        }
+    if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+        (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+    }
 
-        LOCK (&defrag->lock);
-        {
-                status = dict_new ();
-                gf_defrag_status_get (defrag, status);
-                if (ctx && ctx->notify)
-                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
-                if (status)
-                        dict_unref (status);
-                defrag->is_exiting = 1;
-        }
-        UNLOCK (&defrag->lock);
+    if (fc_thread_started) {
+        gf_defrag_estimates_cleanup(this, defrag, filecnt_thread);
+    }
 
-        GF_FREE (defrag->queue);
+    dht_send_rebalance_event(this, defrag->cmd, defrag->defrag_status);
 
-        GF_FREE (defrag);
-        conf->defrag = NULL;
+    status = dict_new();
+    LOCK(&defrag->lock);
+    {
+        gf_defrag_status_get(conf, status);
+        if (ctx && ctx->notify)
+            ctx->notify(GF_EN_DEFRAG_STATUS, status);
+        if (status)
+            dict_unref(status);
+        defrag->is_exiting = 1;
+    }
+    UNLOCK(&defrag->lock);
 
-        if (dict)
-                dict_unref (dict);
+    GF_FREE(defrag);
+    conf->defrag = NULL;
 
-        if (migrate_data)
-                dict_unref (migrate_data);
+    if (migrate_data)
+        dict_unref(migrate_data);
 
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
 exit:
-        return ret;
+    return ret;
 }
 
-
-
 static int
-gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
+gf_defrag_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        gf_listener_stop (sync_frame->this);
+    gf_listener_stop(sync_frame->this);
 
-        STACK_DESTROY (sync_frame->root);
-        kill (getpid(), SIGTERM);
-        return 0;
+    STACK_DESTROY(sync_frame->root);
+    kill(getpid(), SIGTERM);
+    return 0;
 }
 
 void *
-gf_defrag_start (void *data)
+gf_defrag_start(void *data)
 {
-        int                      ret    = -1;
-        call_frame_t            *frame  = NULL;
-        dht_conf_t              *conf   = NULL;
-        gf_defrag_info_t        *defrag = NULL;
-        xlator_t                *this  = NULL;
-        xlator_t                *old_THIS = NULL;
-
-        this = data;
-        conf = this->private;
-        if (!conf)
-                goto out;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    xlator_t *this = NULL;
+    xlator_t *old_THIS = NULL;
 
-        defrag = conf->defrag;
-        if (!defrag)
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    this = data;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        frame->root->pid = GF_CLIENT_PID_DEFRAG;
+    defrag = conf->defrag;
+    if (!defrag)
+        goto out;
 
-        defrag->pid = frame->root->pid;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+    frame->root->pid = GF_CLIENT_PID_DEFRAG;
 
-        old_THIS = THIS;
-        THIS = this;
-        ret = synctask_new (this->ctx->env, gf_defrag_start_crawl,
-                            gf_defrag_done, frame, this);
+    defrag->pid = frame->root->pid;
 
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Could not create task for rebalance");
-        THIS = old_THIS;
-out:
-        return NULL;
-}
-
-int
-gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
-{
-        int      ret    = 0;
-        uint64_t files  = 0;
-        uint64_t size   = 0;
-        uint64_t lookup = 0;
-        uint64_t failures = 0;
-        uint64_t skipped = 0;
-        uint64_t promoted = 0;
-        uint64_t demoted = 0;
-        char     *status = "";
-        double   elapsed = 0;
-        struct timeval end = {0,};
-
-
-        if (!defrag)
-                goto out;
-
-        ret = 0;
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
-                goto out;
-
-        files  = defrag->total_files;
-        size   = defrag->total_data;
-        lookup = defrag->num_files_lookedup;
-        failures = defrag->total_failures;
-        skipped = defrag->skipped;
-        promoted = defrag->total_files_promoted;
-        demoted = defrag->total_files_demoted;
-
-        gettimeofday (&end, NULL);
-
-        elapsed = end.tv_sec - defrag->start_time.tv_sec;
-
-        if (!dict)
-                goto log;
-
-        ret = dict_set_uint64 (dict, "promoted", promoted);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set promoted count");
-
-        ret = dict_set_uint64 (dict, "demoted", demoted);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set demoted count");
-
-        ret = dict_set_uint64 (dict, "files", files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set file count");
-
-        ret = dict_set_uint64 (dict, "size", size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set size of xfer");
-
-        ret = dict_set_uint64 (dict, "lookups", lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set lookedup file count");
-
-
-        ret = dict_set_int32 (dict, "status", defrag->defrag_status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set status");
-        if (elapsed) {
-                ret = dict_set_double (dict, "run-time", elapsed);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to set run-time");
-        }
-
-        ret = dict_set_uint64 (dict, "failures", failures);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set failure count");
-
-        ret = dict_set_uint64 (dict, "skipped", skipped);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set skipped file count");
-log:
-        switch (defrag->defrag_status) {
-        case GF_DEFRAG_STATUS_NOT_STARTED:
-                status = "not started";
-                break;
-        case GF_DEFRAG_STATUS_STARTED:
-                status = "in progress";
-                break;
-        case GF_DEFRAG_STATUS_STOPPED:
-                status = "stopped";
-                break;
-        case GF_DEFRAG_STATUS_COMPLETE:
-                status = "completed";
-                break;
-        case GF_DEFRAG_STATUS_FAILED:
-                status = "failed";
-                break;
-        default:
-                break;
-        }
-
-        gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
-                "Rebalance is %s. Time taken is %.2f secs",
-                status, elapsed);
-        gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
-                "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
-                "%"PRIu64, files, size, lookup, failures, skipped);
+    defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
 
+    old_THIS = THIS;
+    THIS = this;
+    ret = synctask_new(this->ctx->env, gf_defrag_start_crawl, gf_defrag_done,
+                       frame, this);
 
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Could not create task for rebalance");
+    THIS = old_THIS;
 out:
-        return 0;
+    return NULL;
 }
 
-void
-gf_defrag_set_pause_state (gf_tier_conf_t *tier_conf, tier_pause_state_t state)
+uint64_t
+gf_defrag_get_estimates_based_on_size(dht_conf_t *conf)
 {
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        tier_conf->pause_state = state;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-}
+    gf_defrag_info_t *defrag = NULL;
+    double rate_processed = 0;
+    uint64_t total_processed = 0;
+    uint64_t tmp_count = 0;
+    uint64_t time_to_complete = 0;
+    double elapsed = 0;
 
-tier_pause_state_t
-gf_defrag_get_pause_state (gf_tier_conf_t *tier_conf)
-{
-        int state;
+    defrag = conf->defrag;
 
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        state = tier_conf->pause_state;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
+    if (!g_totalsize)
+        goto out;
 
-        return state;
-}
+    elapsed = gf_time() - defrag->start_time;
 
-tier_pause_state_t
-gf_defrag_check_pause_tier (gf_tier_conf_t *tier_conf)
-{
-        int woke = 0;
-        int state  = -1;
-
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-
-        if (tier_conf->pause_state == TIER_RUNNING)
-                goto out;
-
-        if (tier_conf->pause_state == TIER_PAUSED)
-                goto out;
-
-        if (tier_conf->promote_in_progress ||
-            tier_conf->demote_in_progress)
-                goto out;
-
-        tier_conf->pause_state = TIER_PAUSED;
-
-        if (tier_conf->pause_synctask) {
-                synctask_wake (tier_conf->pause_synctask);
-                tier_conf->pause_synctask = 0;
-                woke = 1;
-        }
+    /* Don't calculate the estimates for the first 10 minutes.
+     * It is unlikely to be accurate and estimates are not required
+     * if the process finishes in less than 10 mins.
+     */
 
-        gf_msg ("tier", GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "woken %d", woke);
-out:
-        state = tier_conf->pause_state;
+    if (elapsed < ESTIMATE_START_INTERVAL) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, 0,
+               "Rebalance estimates will not be available for the "
+               "first %d seconds.",
+               ESTIMATE_START_INTERVAL);
 
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
+        goto out;
+    }
 
-        return state;
-}
-
-void
-gf_defrag_pause_tier_timeout (void *data)
-{
-        xlator_t         *this                  = NULL;
-        dht_conf_t       *conf                  = NULL;
-        gf_defrag_info_t *defrag                = NULL;
+    total_processed = defrag->size_processed;
 
-        this   = (xlator_t *) data;
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
+    /* rate at which files processed */
+    rate_processed = (total_processed) / elapsed;
 
-        conf   = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    tmp_count = g_totalsize;
 
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
+    if (rate_processed) {
+        time_to_complete = (tmp_count) / rate_processed;
 
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Request pause timer timeout");
+    } else {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, 0,
+               "Unable to calculate estimated time for rebalance");
+    }
 
-        gf_defrag_check_pause_tier (&defrag->tier_conf);
+    gf_log(THIS->name, GF_LOG_INFO,
+           "TIME: (size) total_processed=%" PRIu64 " tmp_cnt = %" PRIu64
+           ","
+           "rate_processed=%f, elapsed = %f",
+           total_processed, tmp_count, rate_processed, elapsed);
 
 out:
-        return;
+    return time_to_complete;
 }
 
 int
-gf_defrag_pause_tier (xlator_t *this, gf_defrag_info_t *defrag)
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict)
 {
-        int             ret         = 0;
-        struct timespec delta       = {0,};
-        int             delay       = 2;
-
-        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED)
-                goto out;
+    int ret = 0;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    char *status = "";
+    double elapsed = 0;
+    uint64_t time_to_complete = 0;
+    uint64_t time_left = 0;
+    gf_defrag_info_t *defrag = conf->defrag;
+
+    if (!defrag)
+        goto out;
+
+    ret = 0;
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
+        goto out;
+
+    files = defrag->total_files;
+    size = defrag->total_data;
+    lookup = defrag->num_files_lookedup;
+    failures = defrag->total_failures;
+    skipped = defrag->skipped;
+
+    elapsed = gf_time() - defrag->start_time;
+
+    /* The rebalance is still in progress */
+
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        time_to_complete = gf_defrag_get_estimates_based_on_size(conf);
+
+        if (time_to_complete && (time_to_complete > elapsed))
+            time_left = time_to_complete - elapsed;
+
+        gf_log(THIS->name, GF_LOG_INFO,
+               "TIME: Estimated total time to complete (size)= %" PRIu64
+               " seconds, seconds left = %" PRIu64 "",
+               time_to_complete, time_left);
+    }
+
+    if (!dict)
+        goto log;
+
+    ret = dict_set_uint64(dict, "files", files);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set file count");
+
+    ret = dict_set_uint64(dict, "size", size);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set size of xfer");
+
+    ret = dict_set_uint64(dict, "lookups", lookup);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set lookedup file count");
+
+    ret = dict_set_int32(dict, "status", defrag->defrag_status);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set status");
+
+    ret = dict_set_double(dict, "run-time", elapsed);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set run-time");
+
+    ret = dict_set_uint64(dict, "failures", failures);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set failure count");
+
+    ret = dict_set_uint64(dict, "skipped", skipped);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set skipped file count");
+
+    ret = dict_set_uint64(dict, "time-left", time_left);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set time-left");
 
-        /*
-         * Set flag requesting to pause tiering. Wait 'delay' seconds for
-         * tiering to actually stop as indicated by the pause state
-         * before returning success or failure.
-         */
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_REQUEST_PAUSE);
-
-        /*
-         * If migration is not underway, can pause immediately.
-         */
-        gf_defrag_check_pause_tier (&defrag->tier_conf);
-        if (gf_defrag_get_pause_state (&defrag->tier_conf) == TIER_PAUSED)
-                goto out;
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Request pause tier");
-
-        defrag->tier_conf.pause_synctask = synctask_get ();
-        delta.tv_sec  = delay;
-        delta.tv_nsec = 0;
-        defrag->tier_conf.pause_timer =
-                gf_timer_call_after (this->ctx, delta,
-                                     gf_defrag_pause_tier_timeout,
-                                     this);
-
-        synctask_yield (defrag->tier_conf.pause_synctask);
-
-        if (gf_defrag_get_pause_state (&defrag->tier_conf) == TIER_PAUSED)
-                goto out;
-
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING);
-
-        ret = -1;
+log:
+    switch (defrag->defrag_status) {
+        case GF_DEFRAG_STATUS_NOT_STARTED:
+            status = "not started";
+            break;
+        case GF_DEFRAG_STATUS_STARTED:
+            status = "in progress";
+            break;
+        case GF_DEFRAG_STATUS_STOPPED:
+            status = "stopped";
+            break;
+        case GF_DEFRAG_STATUS_COMPLETE:
+            status = "completed";
+            break;
+        case GF_DEFRAG_STATUS_FAILED:
+            status = "failed";
+            break;
+        default:
+            break;
+    }
+
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Rebalance is %s. Time taken is %.2f secs", status, elapsed);
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Files migrated: %" PRIu64 ", size: %" PRIu64 ", lookups: %" PRIu64
+           ", failures: %" PRIu64
+           ", skipped: "
+           "%" PRIu64,
+           files, size, lookup, failures, skipped);
 out:
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Pause tiering ret=%d", ret);
-
-        return ret;
-}
-
-int
-gf_defrag_resume_tier (xlator_t *this, gf_defrag_info_t *defrag)
-{
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_RESUME,
-                "Pause end. Resume tiering");
-
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING);
-
-        return 0;
+    return 0;
 }
 
 int
-gf_defrag_start_detach_tier (gf_defrag_info_t *defrag)
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output)
 {
-        defrag->cmd = GF_DEFRAG_CMD_START_DETACH_TIER;
+    /* TODO: set a variable 'stop_defrag' here, it should be checked
+       in defrag loop */
+    int ret = -1;
+    gf_defrag_info_t *defrag = conf->defrag;
 
-        return 0;
-}
-
-int
-gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
-                dict_t *output)
-{
-        /* TODO: set a variable 'stop_defrag' here, it should be checked
-           in defrag loop */
-        int     ret = -1;
-        GF_ASSERT (defrag);
+    GF_ASSERT(defrag);
 
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
-                goto out;
-        }
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+        goto out;
+    }
 
-        gf_msg ("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED,
-                "Received stop command on rebalance");
-        defrag->defrag_status = status;
+    gf_msg("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED,
+           "Received stop command on rebalance");
+    defrag->defrag_status = status;
 
-        if (output)
-                gf_defrag_status_get (defrag, output);
-        ret = 0;
+    if (output)
+        gf_defrag_status_get(conf, output);
+    ret = 0;
 out:
-        gf_msg_debug ("", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index 777c63de685..d9dbf50492f 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -11,1571 +11,1987 @@
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
  *       delete the newpath if it gets EEXISTS from link() call.
  */
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "defaults.h"
+#include "dht-lock.h"
+#include <glusterfs/defaults.h>
 
-int dht_rename_unlock (call_frame_t *frame, xlator_t *this);
+int
+dht_rename_unlock(call_frame_t *frame, xlator_t *this);
+int32_t
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int
-dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+dht_rename_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
 
+    dht_set_fixed_dir_stat(&local->preoldparent);
+    dht_set_fixed_dir_stat(&local->postoldparent);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        if (op_ret == -1) {
-                /* TODO: undo the damage */
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
+    if (IA_ISREG(local->stbuf.ia_type))
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
 
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_RENAME_FAILED,
-                        "Rename %s -> %s on %s failed, (gfid = %s)",
-                        local->loc.path, local->loc2.path,
-                        prev->this->name, gfid);
+    DHT_STACK_UNWIND(rename, frame, local->op_ret, local->op_errno,
+                     &local->stbuf, &local->preoldparent, &local->postoldparent,
+                     &local->preparent, &local->postparent, local->xattr);
+    return 0;
+}
 
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
-         */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent,
-                        prev->this);
+static void
+dht_rename_dir_unlock_src(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
 
+    local = frame->local;
+    dht_unlock_namespace(frame, &local->lock[0]);
+    return;
+}
 
-unwind:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                WIPE (&local->preoldparent);
-                WIPE (&local->postoldparent);
-                WIPE (&local->preparent);
-                WIPE (&local->postparent);
-
-                dht_rename_unlock (frame, this);
-        }
+static void
+dht_rename_dir_unlock_dst(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[1].ns.directory_ns);
+
+    /* Unlock inodelk */
+    op_ret = dht_unlock_inodelk(frame, local->lock[1].ns.parent_layout.locks,
+                                local->lock[1].ns.parent_layout.lk_count,
+                                dht_rename_unlock_cbk);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
 
-        return 0;
-}
+        dht_rename_unlock_cbk(frame, NULL, this, 0, 0, NULL);
+    }
 
+    return;
+}
 
+static int
+dht_rename_dir_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_rename_dir_unlock_src(frame, this);
+    dht_rename_dir_unlock_dst(frame, this);
+    return 0;
+}
 int
-dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                           struct iatt *preoldparent,
-                           struct iatt *postoldparent,
-                           struct iatt *prenewparent,
-                           struct iatt *postnewparent, dict_t *xdata)
+dht_rename_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
 {
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           call_cnt = 0;
-        call_frame_t *prev = NULL;
-        int           i = 0;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int subvol_cnt = -1;
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+    subvol_cnt = dht_subvol_cnt(this, prev);
+    local->ret_cache[subvol_cnt] = op_ret;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-        conf = this->private;
-        local = frame->local;
-        prev = cookie;
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "Rename %s -> %s on %s failed, (gfid = %s)", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
 
+unwind:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* We get here with local->call_cnt == 0. Which means
+         * we are the only one executing this code, there is
+         * no contention. Therefore it's safe to manipulate or
+         * deref local->call_cnt directly (without locking).
+         */
+        if (local->ret_cache[conf->subvolume_cnt] == 0) {
+            /* count errant subvols in last field of ret_cache */
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (local->ret_cache[i] != 0)
+                    ++local->ret_cache[conf->subvolume_cnt];
+            }
+            if (local->ret_cache[conf->subvolume_cnt]) {
+                /* undoing the damage:
+                 * for all subvolumes, where rename
+                 * succeeded, we perform the reverse operation
+                 */
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i] == 0)
+                        ++local->call_cnt;
+                }
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i])
+                        continue;
 
-        if (op_ret == -1) {
-                /* TODO: undo the damage */
+                    STACK_WIND(frame, dht_rename_dir_cbk, conf->subvolumes[i],
+                               conf->subvolumes[i]->fops->rename, &local->loc2,
+                               &local->loc, NULL);
+                }
 
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
+                return 0;
+            }
+        }
 
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_RENAME_FAILED,
-                        "rename %s -> %s on %s failed, (gfid = %s) ",
-                        local->loc.path, local->loc2.path,
-                        prev->this->name, gfid );
+        WIPE(&local->preoldparent);
+        WIPE(&local->postoldparent);
+        WIPE(&local->preparent);
+        WIPE(&local->postparent);
 
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
-         */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent,
-                        prev->this);
-
-        call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
-
-        if (!local->call_cnt)
-                goto unwind;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == local->dst_hashed)
-                        continue;
-                STACK_WIND (frame, dht_rename_dir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->rename,
-                            &local->loc, &local->loc2, NULL);
-                if (!--call_cnt)
-                        break;
-        }
+        dht_rename_dir_unlock(frame, this);
+    }
 
+    return 0;
+}
 
-        return 0;
+int
+dht_rename_hashed_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
+                          dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "rename %s -> %s on %s failed, (gfid = %s) ", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
+
+    call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
+
+    if (!local->call_cnt)
+        goto unwind;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == local->dst_hashed)
+            continue;
+        STACK_WIND_COOKIE(
+            frame, dht_rename_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->rename, &local->loc, &local->loc2, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
 unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_do(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret == -1)
-                goto err;
+    if (local->op_ret == -1)
+        goto err;
 
-        local->op_ret = 0;
+    local->op_ret = 0;
 
-        STACK_WIND (frame, dht_rename_hashed_dir_cbk,
-                    local->dst_hashed,
-                    local->dst_hashed->fops->rename,
-                    &local->loc, &local->loc2, NULL);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_rename_hashed_dir_cbk, local->dst_hashed,
+                      local->dst_hashed, local->dst_hashed->fops->rename,
+                      &local->loc, &local->loc2, NULL);
+    return 0;
 
 err:
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
+dht_rename_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        if (op_ret > 2) {
-                gf_msg_trace (this->name, 0,
-                              "readdir on %s for %s returned %d entries",
-                              prev->this->name, local->loc.path, op_ret);
-                local->op_ret = -1;
-                local->op_errno = ENOTEMPTY;
-        }
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
+    local = frame->local;
+    prev = cookie;
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    if (op_ret > 2) {
+        gf_msg_trace(this->name, 0, "readdir on %s for %s returned %d entries",
+                     prev->name, local->loc.path, op_ret);
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
+    }
 
-        return 0;
-}
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
+
+    return 0;
+}
 
 int
-dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+dht_rename_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev  = cookie;
-
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        if (op_ret == -1) {
+    local = frame->local;
+    prev = cookie;
 
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_OPENDIR_FAILED,
-                        "opendir on %s for %s failed,(gfid = %s) ",
-                        prev->this->name, local->loc.path, gfid);
-                goto err;
-        }
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_OPENDIR_FAILED,
+               "opendir on %s for %s failed,(gfid = %s) ", prev->name,
+               local->loc.path, gfid);
+        goto err;
+    }
 
-        fd_bind (fd);
-        STACK_WIND (frame, dht_rename_readdir_cbk,
-                    prev->this, prev->this->fops->readdir,
-                    local->fd, 4096, 0, NULL);
+    fd_bind(fd);
+    STACK_WIND_COOKIE(frame, dht_rename_readdir_cbk, prev, prev,
+                      prev->fops->readdir, local->fd, 4096, 0, NULL);
 
-        return 0;
+    return 0;
 
 err:
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rename_dir_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+dht_rename_dir_lock2_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                      = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        dht_conf_t  *conf                       = NULL;
-        int          i                          = 0;
-
-        local = frame->local;
-        conf = this->private;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring inodelk failed "
-                        "rename (%s:%s:%s %s:%s:%s)",
-                        local->loc.path, src_gfid, local->src_cached->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_cached ? local->dst_cached->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->op_ret = 0;
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_conf_t *conf = NULL;
+    int i = 0;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->op_ret = 0;
+
+    if (!local->dst_cached) {
+        dht_rename_dir_do(frame, this);
+        return 0;
+    }
 
-        if (!local->dst_cached) {
-                dht_rename_dir_do (frame, this);
-                return 0;
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rename_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, &local->loc2,
+                          local->fd, NULL);
+    }
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rename_opendir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            &local->loc2, local->fd, NULL);
-        }
-
-        return 0;
+    return 0;
 
 err:
-        /* No harm in calling an extra unlock */
-        dht_rename_unlock (frame, this);
-        return 0;
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
 int
-dht_rename_dir (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_conf_t    *conf         = NULL;
-        dht_local_t   *local        = NULL;
-        dht_lock_t   **lk_array     = NULL;
-        dht_layout_t  *dst_layout   = NULL;
-        xlator_t      *first_subvol = NULL;
-        loc_t          parent_loc   = {0, };
-        int            count        = 1;
-        int            i            = 0;
-        int            j            = 0;
-        int            ret          = 0;
-        int            op_errno     = -1;
-
-        conf = frame->this->private;
-        local = frame->local;
-
-        /* We must take a lock on all the subvols with src gfid.
-         * Along with this if dst exists we must take lock on
-         * any one subvol with dst gfid.
-         */
-        count = local->call_cnt = conf->subvolume_cnt;
-        if (local->loc2.inode) {
-                dst_layout = dht_layout_get (this, local->loc2.inode);
-                if (dst_layout)
-                        ++count;
-        } else if (gf_uuid_compare (local->loc.parent->gfid,
-                                    local->loc2.parent->gfid)) {
-                dst_layout = dht_layout_get (this, local->loc2.parent);
-                if (dst_layout)
-                        ++count;
-        }
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock2_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
+}
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_RENAME_FAILED,
-                                "Rename dir failed: subvolume down (%s)",
-                                conf->subvolumes[i]->name);
-                        op_errno = ENOTCONN;
-                        goto err;
-                }
+/*
+ * If the hashed subvolumes of both source and dst are the different,
+ * lock in dictionary order of hashed subvol->name. This is important
+ * in case the parent directory is the same for both src and dst to
+ * prevent inodelk deadlocks when racing with a fix-layout op on the parent.
+ *
+ * If the hashed subvols are the same, use the gfid/name to determine
+ * the order of taking locks to prevent entrylk deadlocks when the parent
+ * dirs are the same.
+ *
+ */
+static int
+dht_order_rename_lock(call_frame_t *frame, loc_t **loc, xlator_t **subvol)
+{
+    int ret = 0;
+    int op_ret = 0;
+    dht_local_t *local = NULL;
+    char *src = NULL;
+    char *dst = NULL;
+
+    local = frame->local;
+
+    if (local->src_hashed->name == local->dst_hashed->name) {
+        ret = 0;
+    } else {
+        ret = strcmp(local->src_hashed->name, local->dst_hashed->name);
+    }
+
+    if (ret == 0) {
+        /* hashed subvols are the same for src and dst */
+        /* Entrylks need to be ordered*/
+
+        src = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc.name) + 1);
+        if (!src) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for src");
+            op_ret = -1;
+            goto out;
         }
 
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
-        if (lk_array == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+        if (!gf_uuid_is_null(local->loc.pargfid))
+            uuid_utoa_r(local->loc.pargfid, src);
+        else if (local->loc.parent)
+            uuid_utoa_r(local->loc.parent->gfid, src);
+        else
+            src[0] = '\0';
 
-        /* Rename must take locks on src to avoid lookup selfheal from
-         * recreating src on those subvols where the rename was successful.
-         * Rename must take locks on all subvols with src because selfheal
-         * in entry creation phase may not have acquired lock on all subvols.
-        */
-        for (i = 0; i < local->call_cnt; i++) {
-                lk_array[i] = dht_lock_new (frame->this,
-                                            conf->subvolumes[i],
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN);
-                if (lk_array[i] == NULL) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-        }
+        strcat(src, local->loc.name);
 
-        /* If the dst exists, we are going to replace dst layout range with
-         * that of src. This will lead to anomalies in dst layout until the
-         * rename completes. To avoid a lookup selfheal to change dst layout
-         * during this interval we take a lock on one subvol of dst.
-         */
-        for (j = 0; dst_layout && (j < dst_layout->cnt) &&
-                        (dst_layout->list[j].err == 0); j++) {
-
-                first_subvol = dst_layout->list[j].xlator;
-                if (local->loc2.inode) {
-                        lk_array[i] = dht_lock_new (frame->this, first_subvol,
-                                                    &local->loc2, F_WRLCK,
-                                                    DHT_LAYOUT_HEAL_DOMAIN);
-                } else {
-                        ret = dht_build_parent_loc (this, &parent_loc,
-                                                    &local->loc2, &op_errno);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "parent loc build failed");
-                                goto err;
-                        }
-
-                        lk_array[i] = dht_lock_new (frame->this, first_subvol,
-                                                    &parent_loc, F_WRLCK,
-                                                    DHT_LAYOUT_HEAL_DOMAIN);
-                }
-
-                if (lk_array[i] == NULL) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                break;
+        dst = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc2.name) + 1);
+        if (!dst) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for dst");
+            op_ret = -1;
+            goto out;
         }
 
-        if (!lk_array[i])
-                --count;
+        if (!gf_uuid_is_null(local->loc2.pargfid))
+            uuid_utoa_r(local->loc2.pargfid, dst);
+        else if (local->loc2.parent)
+            uuid_utoa_r(local->loc2.parent->gfid, dst);
+        else
+            dst[0] = '\0';
 
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
+        strcat(dst, local->loc2.name);
+        ret = strcmp(src, dst);
+    }
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    IGNORE_ENOENT_ESTALE,
-                                    dht_rename_dir_lock_cbk);
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (ret <= 0) {
+        /*inodelk in dictionary order of hashed subvol names*/
+        /*entrylk in dictionary order of gfid/basename */
+        local->current = &local->lock[0];
+        *loc = &local->loc;
+        *subvol = local->src_hashed;
 
-        loc_wipe (&parent_loc);
-        return 0;
+    } else {
+        local->current = &local->lock[1];
+        *loc = &local->loc2;
+        *subvol = local->dst_hashed;
+    }
 
-err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    op_ret = 0;
 
-        loc_wipe (&parent_loc);
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
-        return 0;
+out:
+    return op_ret;
 }
 
-static int
-dht_rename_track_for_changelog (xlator_t *this, dict_t *xattr,
-                                loc_t *oldloc, loc_t *newloc)
+int
+dht_rename_dir(call_frame_t *frame, xlator_t *this)
 {
-        int ret        = -1;
-        dht_changelog_rename_info_t *info = NULL;
-        char *name     = NULL;
-        int len1       = 0;
-        int len2       = 0;
-        int size       = 0;
-
-        if (!xattr || !oldloc || !newloc || !this)
-                return ret;
-
-        len1 = strlen (oldloc->name) + 1;
-        len2 = strlen (newloc->name) + 1;
-        size = sizeof (dht_changelog_rename_info_t) + len1 + len2;
-
-        info = GF_CALLOC (size, sizeof(char), gf_common_mt_char);
-        if (!info) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to calloc memory");
-                return ret;
-        }
-
-        gf_uuid_copy (info->old_pargfid, oldloc->pargfid);
-        gf_uuid_copy (info->new_pargfid, newloc->pargfid);
-
-        info->oldname_len = len1;
-        info->newname_len = len2;
-        strncpy (info->buffer, oldloc->name, len1);
-        name = info->buffer + len1;
-        strncpy (name, newloc->name, len2);
-
-        ret = dict_set_bin (xattr, DHT_CHANGELOG_RENAME_OP_KEY,
-                            info, size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s,"
-                        " path = %s", DHT_CHANGELOG_RENAME_OP_KEY,
-                        oldloc->name);
-                GF_FREE (info);
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+    int op_errno = -1;
+
+    conf = frame->this->private;
+    local = frame->local;
+
+    local->ret_cache = GF_CALLOC(conf->subvolume_cnt + 1, sizeof(int),
+                                 gf_dht_ret_cache_t);
+
+    if (local->ret_cache == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+                   "Rename dir failed: subvolume down (%s)",
+                   conf->subvolumes[i]->name);
+            op_errno = ENOTCONN;
+            goto err;
         }
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Rename must take locks on src to avoid lookup selfheal from
+     * recreating src on those subvols where the rename was successful.
+     * The locks can't be issued parallel as two different clients might
+     * attempt same rename command and be in dead lock.
+     */
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
-        return ret;
+err:
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
-
-
-#define DHT_MARKER_DONT_ACCOUNT(xattr) do {                             \
-                int tmp = -1;                                                  \
-                if (!xattr) {                                                  \
-                        xattr = dict_new ();                                   \
-                        if (!xattr)                                            \
-                                break;                                         \
-                }                                                              \
-                tmp = dict_set_str (xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY,  \
-                                    "yes");                                    \
-                if (tmp) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                    \
-                                DHT_MSG_DICT_SET_FAILED,                       \
-                                "Failed to set dictionary value: key = %s,"    \
-                                " path = %s",GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, \
-                                local->loc.path);                             \
-                }                                                              \
-        }while (0)
-
-
-#define DHT_CHANGELOG_TRACK_AS_RENAME(xattr, oldloc, newloc) do {            \
-                int tmp = -1;                                                \
-                if (!xattr) {                                                \
-                        xattr = dict_new ();                                 \
-                        if (!xattr) {                                        \
-                                gf_msg (this->name, GF_LOG_ERROR, 0,         \
-                                        DHT_MSG_DICT_SET_FAILED,             \
-                                        "Failed to create dictionary to "    \
-                                        "track rename");                     \
-                                break;                                       \
-                        }                                                    \
-                }                                                            \
-                                                                             \
-                tmp = dht_rename_track_for_changelog (this, xattr,           \
-                                oldloc, newloc);                             \
-                                                                             \
-                if (tmp) {                                                   \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                 \
-                                DHT_MSG_DICT_SET_FAILED,                     \
-                                "Failed to set dictionary value: key = %s,"  \
-                                " path = %s", DHT_CHANGELOG_RENAME_OP_KEY,   \
-                                (oldloc)->path);                             \
-                }                                                            \
-        } while (0)
-
-int
-dht_rename_unlock_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xdata)
+static int
+dht_rename_track_for_changelog(xlator_t *this, dict_t *xattr, loc_t *oldloc,
+                               loc_t *newloc)
 {
-        dht_local_t *local = NULL;
-
-        local = frame->local;
-
-        dht_set_fixed_dir_stat (&local->preoldparent);
-        dht_set_fixed_dir_stat (&local->postoldparent);
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
+    int ret = -1;
+    dht_changelog_rename_info_t *info = NULL;
+    char *name = NULL;
+    int len1 = 0;
+    int len2 = 0;
+    int size = 0;
+
+    if (!xattr || !oldloc || !newloc || !this)
+        return ret;
 
-        if (IA_ISREG (local->stbuf.ia_type))
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+    len1 = strlen(oldloc->name) + 1;
+    len2 = strlen(newloc->name) + 1;
+    size = sizeof(dht_changelog_rename_info_t) + len1 + len2;
 
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent, local->xattr);
-        return 0;
+    info = GF_CALLOC(size, sizeof(char), gf_common_mt_char);
+    if (!info) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to calloc memory");
+        return ret;
+    }
+
+    gf_uuid_copy(info->old_pargfid, oldloc->pargfid);
+    gf_uuid_copy(info->new_pargfid, newloc->pargfid);
+
+    info->oldname_len = len1;
+    info->newname_len = len2;
+    strncpy(info->buffer, oldloc->name, len1);
+    name = info->buffer + len1;
+    strncpy(name, newloc->name, len2);
+
+    ret = dict_set_bin(xattr, DHT_CHANGELOG_RENAME_OP_KEY, info, size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s,"
+               " path = %s",
+               DHT_CHANGELOG_RENAME_OP_KEY, oldloc->name);
+        GF_FREE(info);
+    }
+
+    return ret;
 }
 
+#define DHT_MARKER_DONT_ACCOUNT(xattr)                                         \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, "yes");   \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, local->loc.path);        \
+        }                                                                      \
+    } while (0)
+
+#define DHT_CHANGELOG_TRACK_AS_RENAME(xattr, oldloc, newloc)                   \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr) {                                                      \
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,   \
+                       "Failed to create dictionary to "                       \
+                       "track rename");                                        \
+                break;                                                         \
+            }                                                                  \
+        }                                                                      \
+                                                                               \
+        tmp = dht_rename_track_for_changelog(this, xattr, oldloc, newloc);     \
+                                                                               \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   DHT_CHANGELOG_RENAME_OP_KEY, (oldloc)->path);               \
+        }                                                                      \
+    } while (0)
+
 int
-dht_rename_unlock (call_frame_t *frame, xlator_t *this)
+dht_rename_unlock(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
-        int          op_ret                     = -1;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        op_ret = dht_unlock_inodelk (frame, local->lock.locks,
-                                     local->lock.lk_count,
-                                     dht_rename_unlock_cbk);
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                if (IA_ISREG (local->stbuf.ia_type))
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_UNLOCKING_FAILED,
-                                "winding unlock inodelk failed "
-                                "rename (%s:%s:%s %s:%s:%s), "
-                                "stale locks left on bricks",
-                                local->loc.path, src_gfid,
-                                local->src_cached->name,
-                                local->loc2.path, dst_gfid,
-                                local->dst_cached ?
-                                local->dst_cached->name : NULL);
-                else
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_UNLOCKING_FAILED,
-                                "winding unlock inodelk failed "
-                                "rename (%s:%s %s:%s), "
-                                "stale locks left on bricks",
-                                local->loc.path, src_gfid,
-                                local->loc2.path, dst_gfid);
-
-                dht_rename_unlock_cbk (frame, NULL, this, 0, 0, NULL);
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_ilock_wrap_t inodelk_wrapper = {
+        0,
+    };
+
+    local = frame->local;
+    inodelk_wrapper.locks = local->rename_inodelk_backward_compatible;
+    inodelk_wrapper.lk_count = local->rename_inodelk_bc_count;
+
+    op_ret = dht_unlock_inodelk_wrapper(frame, &inodelk_wrapper);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
+    }
+
+    dht_unlock_namespace(frame, &local->lock[0]);
+    dht_unlock_namespace(frame, &local->lock[1]);
+
+    dht_rename_unlock_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                          NULL);
+    return 0;
 }
 
 int
-dht_rename_done (call_frame_t *frame, xlator_t *this)
+dht_rename_done(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->linked == _gf_true) {
-                local->linked = _gf_false;
-                dht_linkfile_attr_heal (frame, this);
-        }
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
 
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
 int
-dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+dht_rename_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           this_call_cnt = 0;
-
-        local = frame->local;
-        prev  = cookie;
-
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_INVALID_VALUE,
-                        "!local, should not happen");
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
 
-        this_call_cnt = dht_frame_return (frame);
+    local = frame->local;
+    prev = cookie;
 
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_UNLINK_FAILED,
-                        "%s: Rename: unlink on %s failed ",
-                        local->loc.path, prev->this->name);
-        }
+    FRAME_SU_UNDO(frame, dht_local_t);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_VALUE,
+               "!local, should not happen");
+        goto out;
+    }
 
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_done (frame, this);
-        }
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLINK_FAILED,
+               "%s: Rename: unlink on %s failed ", local->loc.path, prev->name);
+    }
+
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_done(frame, this);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rename_cleanup (call_frame_t *frame)
+dht_rename_cleanup(call_frame_t *frame)
 {
-        dht_local_t *local                  = NULL;
-        xlator_t    *this                   = NULL;
-        xlator_t    *src_hashed             = NULL;
-        xlator_t    *src_cached             = NULL;
-        xlator_t    *dst_hashed             = NULL;
-        xlator_t    *dst_cached             = NULL;
-        int          call_cnt               = 0;
-        dict_t      *xattr                  = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        this  = frame->this;
-
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-
-        if (src_cached == dst_cached)
-                goto nolinks;
-
-        if (local->linked && (dst_hashed != src_hashed )&&
-                (dst_hashed != src_cached)) {
-                call_cnt++;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        if (local->added_link && (src_cached != dst_hashed)) {
-                call_cnt++;
-        }
+    local = frame->local;
+    this = frame->this;
 
-        local->call_cnt = call_cnt;
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        if (!call_cnt)
-                goto nolinks;
+    if (src_cached == dst_cached)
+        goto nolinks;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        call_cnt++;
+    }
 
-        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+    if (local->added_link && (src_cached != dst_hashed)) {
+        call_cnt++;
+    }
 
-        if (local->linked && (dst_hashed != src_hashed) &&
-                        (dst_hashed != src_cached)) {
-                dict_t *xattr_new = NULL;
+    local->call_cnt = call_cnt;
 
-                gf_msg_trace (this->name, 0,
-                              "unlinking linkfile %s @ %s => %s, (gfid = %s)",
-                              local->loc.path, dst_hashed->name,
-                              src_cached->name, gfid);
+    if (!call_cnt)
+        goto nolinks;
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+    DHT_MARK_FOP_INTERNAL(xattr);
 
+    gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        dict_t *xattr_new = NULL;
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            dst_hashed, dst_hashed->fops->unlink,
-                            &local->loc, 0, xattr_new);
+        gf_msg_trace(this->name, 0,
+                     "unlinking linkfile %s @ %s => %s, (gfid = %s)",
+                     local->loc.path, dst_hashed->name, src_cached->name, gfid);
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
-        }
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        if (local->added_link && (src_cached != dst_hashed)) {
-                dict_t *xattr_new = NULL;
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                gf_msg_trace (this->name, 0,
-                              "unlinking link %s => %s (%s), (gfid = %s)",
-                              local->loc.path, local->loc2.path,
-                              src_cached->name, gfid);
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_hashed, dst_hashed,
+                          dst_hashed->fops->unlink, &local->loc, 0, xattr_new);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
+    if (local->added_link && (src_cached != dst_hashed)) {
+        dict_t *xattr_new = NULL;
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_cached, src_cached->fops->unlink,
-                            &local->loc2, 0, xattr_new);
+        gf_msg_trace(this->name, 0, "unlinking link %s => %s (%s), (gfid = %s)",
+                     local->loc.path, local->loc2.path, src_cached->name, gfid);
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
+        xattr_new = dict_copy_with_ref(xattr, NULL);
+
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
+        /* *
+         * The link to file is created using root permission.
+         * Hence deletion should happen using root. Otherwise
+         * it will fail.
+         */
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc2, 0, xattr_new);
 
-        if (xattr)
-                dict_unref (xattr);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-        return 0;
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 
 nolinks:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             inode_t *inode, struct iatt *stbuf,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
+dht_rename_unlink(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *prev = NULL;
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *rename_subvol = NULL;
+    dict_t *xattr = NULL;
 
-        prev = cookie;
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_CREATE_LINK_FAILED,
-                        "link/file %s on %s failed",
-                        local->loc.path, prev->this->name);
-        }
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        if (local->linked == _gf_true) {
-                local->linked = _gf_false;
-                dht_linkfile_attr_heal (frame, this);
-        }
-        DHT_STACK_DESTROY (frame);
+    local->call_cnt = 0;
 
-        return 0;
-}
+    /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
+     * is called. since rename has already happened on rename_subvol,
+     * unlink shouldn't be sent for oldpath (either linkfile or cached-file)
+     * on rename_subvol. */
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
 
+    /* TODO: delete files in background */
 
-int
-dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent,
-                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *src_hashed = NULL;
-        xlator_t     *src_cached = NULL;
-        xlator_t     *dst_hashed = NULL;
-        xlator_t     *dst_cached = NULL;
-        xlator_t     *rename_subvol = NULL;
-        call_frame_t *link_frame = NULL;
-        dht_local_t *link_local = NULL;
-        dict_t       *xattr     = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-
-        if (local->linked == _gf_true)
-                FRAME_SU_UNDO (frame, dht_local_t);
-
-        /* It is a critical failure iff we fail to rename the cached file
-         * if the rename of the linkto failed, it is not a critical failure,
-         * and we do not want to lose the created hard link for the new
-         * name as that could have been read by other clients.
-         *
-         * NOTE: If another client is attempting the same oldname -> newname
-         * rename, and finds both file names as existing, and are hard links
-         * to each other, then FUSE would send in an unlink for oldname. In
-         * this time duration if we treat the linkto as a critical error and
-         * unlink the newname we created, we would have effectively lost the
-         * file to rename operations.
-         *
-         * Repercussions of treating this as a non-critical error is that
-         * we could leave behind a stale linkto file and/or not create the new
-         * linkto file, the second case would be rectified by a subsequent
-         * lookup, the first case by a rebalance, like for all stale linkto
-         * files */
-
-        if (op_ret == -1) {
-                /* Critical failure: unable to rename the cached file */
-                if (prev->this == src_cached) {
-                        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                                DHT_MSG_RENAME_FAILED,
-                                "%s: Rename on %s failed, (gfid = %s) ",
-                                local->loc.path, prev->this->name,
-                                local->loc.inode ?
-                                uuid_utoa(local->loc.inode->gfid):"");
-                        local->op_ret   = op_ret;
-                        local->op_errno = op_errno;
-                        goto cleanup;
-                } else {
-                        /* Non-critical failure, unable to rename the linkto
-                         * file
-                         */
-                        gf_msg (this->name, GF_LOG_INFO, op_errno,
-                                DHT_MSG_RENAME_FAILED,
-                                "%s: Rename (linkto file) on %s failed, "
-                                "(gfid = %s) ",
-                                local->loc.path, prev->this->name,
-                                local->loc.inode ?
-                                uuid_utoa(local->loc.inode->gfid):"");
-                }
-        }
-        if (xdata) {
-                if (!local->xattr)
-                        local->xattr = dict_ref (xdata);
-                else
-                        local->xattr = dict_copy_with_ref (xdata, local->xattr);
-        }
+    if (src_cached != dst_hashed && src_cached != dst_cached)
+        local->call_cnt++;
 
-        if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
-                link_frame = copy_frame (frame);
-                if (!link_frame) {
-                        goto err;
-                }
+    if (src_hashed != rename_subvol && src_hashed != src_cached)
+        local->call_cnt++;
 
-                /* fop value sent as maxvalue because it is not used
-                   anywhere in this case */
-                link_local = dht_local_init (link_frame, &local->loc2, NULL,
-                                             GF_FOP_MAXVALUE);
-                if (!link_local) {
-                        goto err;
-                }
+    if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
+        local->call_cnt++;
 
-                if (link_local->loc.inode)
-                        inode_unref (link_local->loc.inode);
-                link_local->loc.inode = inode_ref (local->loc.inode);
-                gf_uuid_copy (link_local->gfid, local->loc.inode->gfid);
+    if (local->call_cnt == 0)
+        goto unwind;
 
-                dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
-                                     this, src_cached, dst_hashed,
-                                     &link_local->loc);
-        }
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-err:
-        /* Merge attrs only from src_cached. In case there of src_cached !=
-         * dst_hashed, this ignores linkfile attrs. */
-        if (prev->this == src_cached) {
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->preparent, prenewparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->postparent, postnewparent,
-                                prev->this);
-        }
+    if (src_cached != dst_hashed && src_cached != dst_cached) {
+        dict_t *xattr_new = NULL;
 
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
-         *       is called. since rename has already happened on rename_subvol,
-         *       unlink should not be sent for oldpath (either linkfile or cached-file)
-         *       on rename_subvol. */
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
-        else
-                rename_subvol = dst_hashed;
-
-        /* TODO: delete files in background */
+        gf_msg_trace(this->name, 0, "deleting old src datafile %s @ %s",
+                     local->loc.path, src_cached->name);
 
-        if (src_cached != dst_hashed && src_cached != dst_cached)
-                local->call_cnt++;
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        }
 
-        if (src_hashed != rename_subvol && src_hashed != src_cached)
-                local->call_cnt++;
+        DHT_CHANGELOG_TRACK_AS_RENAME(xattr_new, &local->loc, &local->loc2);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc, 0, xattr_new);
 
-        if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
-                local->call_cnt++;
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-        if (local->call_cnt == 0)
-                goto unwind;
+    if (src_hashed != rename_subvol && src_hashed != src_cached) {
+        dict_t *xattr_new = NULL;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        if (src_cached != dst_hashed && src_cached != dst_cached) {
-                dict_t *xattr_new = NULL;
+        gf_msg_trace(this->name, 0, "deleting old src linkfile %s @ %s",
+                     local->loc.path, src_hashed->name);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                gf_msg_trace (this->name, 0,
-                              "deleting old src datafile %s @ %s",
-                              local->loc.path, src_cached->name);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_hashed, src_hashed,
+                          src_hashed->fops->unlink, &local->loc, 0, xattr_new);
 
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-                DHT_CHANGELOG_TRACK_AS_RENAME(xattr_new, &local->loc,
-                                              &local->loc2);
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_cached, src_cached->fops->unlink,
-                            &local->loc, 0, xattr_new);
+    if (dst_cached && (dst_cached != dst_hashed) &&
+        (dst_cached != src_cached)) {
+        gf_msg_trace(this->name, 0, "deleting old dst datafile %s @ %s",
+                     local->loc2.path, dst_cached->name);
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
-        }
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_cached, dst_cached,
+                          dst_cached->fops->unlink, &local->loc2, 0, xattr);
+    }
+    if (xattr)
+        dict_unref(xattr);
+    return 0;
 
-        if (src_hashed != rename_subvol && src_hashed != src_cached) {
-                dict_t *xattr_new = NULL;
+unwind:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+    dht_rename_done(frame, this);
 
-                gf_msg_trace (this->name, 0,
-                              "deleting old src linkfile %s @ %s",
-                              local->loc.path, src_hashed->name);
+    return 0;
+}
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+int
+dht_rename_links_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *stbuf, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
+
+    prev = cookie;
+    local = frame->local;
+    main_frame = local->main_frame;
+
+    /* TODO: Handle this case in lookup-optimize */
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_CREATE_LINK_FAILED,
+               "link/file %s on %s failed", local->loc.path, prev->name);
+    }
+
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    dht_rename_unlink(main_frame, this);
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_hashed, src_hashed->fops->unlink,
-                            &local->loc, 0, xattr_new);
+int
+dht_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+               struct iatt *preoldparent, struct iatt *postoldparent,
+               struct iatt *prenewparent, struct iatt *postnewparent,
+               dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    call_frame_t *link_frame = NULL;
+    dht_local_t *link_local = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+
+    if (local->linked == _gf_true)
+        FRAME_SU_UNDO(frame, dht_local_t);
+
+    /* It is a critical failure iff we fail to rename the cached file
+     * if the rename of the linkto failed, it is not a critical failure,
+     * and we do not want to lose the created hard link for the new
+     * name as that could have been read by other clients.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations.
+     *
+     * Repercussions of treating this as a non-critical error is that
+     * we could leave behind a stale linkto file and/or not create the new
+     * linkto file, the second case would be rectified by a subsequent
+     * lookup, the first case by a rebalance, like for all stale linkto
+     * files */
+
+    if (op_ret == -1) {
+        /* Critical failure: unable to rename the cached file */
+        if (prev == src_cached) {
+            gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename on %s failed, (gfid = %s) ", local->loc.path,
+                   prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            goto cleanup;
+        } else {
+            /* Non-critical failure, unable to rename the linkto
+             * file
+             */
+            gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename (linkto file) on %s failed, "
+                   "(gfid = %s) ",
+                   local->loc.path, prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+        }
+    }
+    if (xdata) {
+        if (!local->xattr)
+            local->xattr = dict_ref(xdata);
+        else
+            local->xattr = dict_copy_with_ref(xdata, local->xattr);
+    }
+
+    /* Merge attrs only from src_cached. In case there of src_cached !=
+     * dst_hashed, this ignores linkfile attrs. */
+    if (prev == src_cached) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preoldparent, preoldparent);
+        dht_iatt_merge(this, &local->postoldparent, postoldparent);
+        dht_iatt_merge(this, &local->preparent, prenewparent);
+        dht_iatt_merge(this, &local->postparent, postnewparent);
+    }
+
+    /* Create the linkto file for the dst file */
+    if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
+        link_frame = copy_frame(frame);
+        if (!link_frame) {
+            goto unlink;
+        }
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
+        /* fop value sent as maxvalue because it is not used
+         * anywhere in this case */
+        link_local = dht_local_init(link_frame, &local->loc2, NULL,
+                                    GF_FOP_MAXVALUE);
+        if (!link_local) {
+            goto unlink;
         }
 
-        if (dst_cached
-            && (dst_cached != dst_hashed)
-            && (dst_cached != src_cached)) {
-                gf_msg_trace (this->name, 0,
-                              "deleting old dst datafile %s @ %s",
-                              local->loc2.path, dst_cached->name);
+        if (link_local->loc.inode)
+            inode_unref(link_local->loc.inode);
+        link_local->loc.inode = inode_ref(local->loc.inode);
+        link_local->main_frame = frame;
+        link_local->stbuf = local->stbuf;
+        gf_uuid_copy(link_local->gfid, local->loc.inode->gfid);
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            dst_cached, dst_cached->fops->unlink,
-                            &local->loc2, 0, xattr);
-        }
-        if (xattr)
-                dict_unref (xattr);
+        dht_linkfile_create(link_frame, dht_rename_links_create_cbk, this,
+                            src_cached, dst_hashed, &link_local->loc);
         return 0;
+    }
 
-unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+unlink:
 
-        dht_rename_done (frame, this);
-
-        return 0;
+    if (link_frame) {
+        DHT_STACK_DESTROY(link_frame);
+    }
+    dht_rename_unlink(frame, this);
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_do_rename (call_frame_t *frame)
+dht_do_rename(call_frame_t *frame)
 {
-        dht_local_t *local         = NULL;
-        xlator_t    *dst_hashed    = NULL;
-        xlator_t    *src_cached    = NULL;
-        xlator_t    *dst_cached    = NULL;
-        xlator_t    *this          = NULL;
-        xlator_t    *rename_subvol = NULL;
-
-        local = frame->local;
-        this  = frame->this;
-
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-        src_cached = local->src_cached;
-
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
-        else
-                rename_subvol = dst_hashed;
-
-        if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
-                DHT_MARKER_DONT_ACCOUNT(local->xattr_req);
-        }
-
-        if (rename_subvol == src_cached) {
-                DHT_CHANGELOG_TRACK_AS_RENAME(local->xattr_req, &local->loc,
-                                              &local->loc2);
-        }
-
-        gf_msg_trace (this->name, 0,
-                      "renaming %s => %s (%s)",
-                      local->loc.path, local->loc2.path, rename_subvol->name);
-
-        if (local->linked == _gf_true)
-                FRAME_SU_DO (frame, dht_local_t);
-        STACK_WIND (frame, dht_rename_cbk,
-                    rename_subvol, rename_subvol->fops->rename,
-                    &local->loc, &local->loc2, local->xattr_req);
-        return 0;
+    dht_local_t *local = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *this = NULL;
+    xlator_t *rename_subvol = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+    src_cached = local->src_cached;
+
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
+
+    if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
+        DHT_MARKER_DONT_ACCOUNT(local->xattr_req);
+    }
+
+    if (rename_subvol == src_cached) {
+        DHT_CHANGELOG_TRACK_AS_RENAME(local->xattr_req, &local->loc,
+                                      &local->loc2);
+    }
+
+    gf_msg_trace(this->name, 0, "renaming %s => %s (%s)", local->loc.path,
+                 local->loc2.path, rename_subvol->name);
+
+    if (local->linked == _gf_true)
+        FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_rename_cbk, rename_subvol, rename_subvol,
+                      rename_subvol->fops->rename, &local->loc, &local->loc2,
+                      local->xattr_req);
+    return 0;
 }
 
 int
-dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, 0,
-                              "link/file on %s failed (%s)",
-                              prev->this->name, strerror (op_errno));
-                local->op_ret   = -1;
-                local->op_errno = op_errno;
-                local->added_link = _gf_false;
-        } else
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        local->added_link = _gf_false;
+    } else
+        dht_iatt_merge(this, &local->stbuf, stbuf);
 
-        if (local->op_ret == -1)
-                goto cleanup;
+    if (local->op_ret == -1)
+        goto cleanup;
 
-        dht_do_rename (frame);
+    dht_do_rename(frame);
 
-        return 0;
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_linkto_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+dht_rename_linkto_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
-        call_frame_t    *prev = NULL;
-        xlator_t        *src_cached = NULL;
-        dict_t          *xattr = NULL;
-
-        local = frame->local;
-        DHT_MARK_FOP_INTERNAL (xattr);
-        prev = cookie;
-        src_cached = local->src_cached;
-
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, 0,
-                              "link/file on %s failed (%s)",
-                              prev->this->name, strerror (op_errno));
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    dict_t *xattr = NULL;
 
-        /* If linkto creation failed move to failure cleanup code,
-        * instead of continuing with creating the link file */
-        if (local->op_ret != 0) {
-                goto cleanup;
-        }
+    local = frame->local;
+    DHT_MARK_FOP_INTERNAL(xattr);
+    prev = cookie;
+    src_cached = local->src_cached;
 
-        gf_msg_trace (this->name, 0,
-                      "link %s => %s (%s)", local->loc.path,
-                      local->loc2.path, src_cached->name);
-        if (gf_uuid_compare (local->loc.pargfid,
-                                local->loc2.pargfid) == 0) {
-                DHT_MARKER_DONT_ACCOUNT(xattr);
-        }
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
 
-        local->added_link = _gf_true;
+    /* If linkto creation failed move to failure cleanup code,
+     * instead of continuing with creating the link file */
+    if (local->op_ret != 0) {
+        goto cleanup;
+    }
 
-        STACK_WIND (frame, dht_rename_link_cbk,
-                    src_cached, src_cached->fops->link,
-                    &local->loc, &local->loc2, xattr);
+    gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                 local->loc2.path, src_cached->name);
+    if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+        DHT_MARKER_DONT_ACCOUNT(xattr);
+    }
 
-        if (xattr)
-                dict_unref (xattr);
+    local->added_link = _gf_true;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                      src_cached->fops->link, &local->loc, &local->loc2, xattr);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
+dht_rename_unlink_links_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
+    local = frame->local;
+    prev = cookie;
 
-	local = frame->local;
-	prev = cookie;
+    if ((op_ret == -1) && (op_errno != ENOENT)) {
+        gf_msg_debug(this->name, 0, "unlink of %s on %s failed (%s)",
+                     local->loc2.path, prev->name, strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
 
-	if ((op_ret == -1) && (op_errno != ENOENT)) {
-		gf_msg_debug (this->name, 0,
-		              "unlink of %s on %s failed (%s)",
-			      local->loc2.path, prev->this->name,
-                              strerror (op_errno));
-		local->op_ret   = -1;
-		local->op_errno = op_errno;
-	}
+    if (local->op_ret == -1)
+        goto cleanup;
 
-        if (local->op_ret == -1)
-                goto cleanup;
+    dht_do_rename(frame);
 
-        dht_do_rename (frame);
-
-	return 0;
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-	return 0;
+    return 0;
 }
 
-
 int
-dht_rename_create_links (call_frame_t *frame)
+dht_rename_create_links(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *this = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        int          call_cnt = 0;
-        dict_t      *xattr = NULL;
-
-
-        local = frame->local;
-        this  = frame->this;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
 
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
+    local = frame->local;
+    this = frame->this;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        if (src_cached == dst_cached) {
-                dict_t *xattr_new = NULL;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-                if (dst_hashed == dst_cached)
-                        goto nolinks;
+    if (src_cached == dst_cached) {
+        dict_t *xattr_new = NULL;
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        if (dst_hashed == dst_cached)
+            goto nolinks;
 
-		gf_msg_trace (this->name, 0,
-		              "unlinking dst linkfile %s @ %s",
-			      local->loc2.path, dst_hashed->name);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        gf_msg_trace(this->name, 0, "unlinking dst linkfile %s @ %s",
+                     local->loc2.path, dst_hashed->name);
 
-		STACK_WIND (frame, dht_rename_unlink_links_cbk,
-			    dst_hashed, dst_hashed->fops->unlink,
-			    &local->loc2, 0, xattr_new);
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                dict_unref (xattr_new);
-                if (xattr)
-                        dict_unref (xattr);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_links_cbk, dst_hashed,
+                          dst_hashed, dst_hashed->fops->unlink, &local->loc2, 0,
+                          xattr_new);
 
-                return 0;
-        }
+        dict_unref(xattr_new);
+        if (xattr)
+            dict_unref(xattr);
 
-        if (src_cached != dst_hashed) {
-                /* needed to create the link file */
-                call_cnt++;
-                if (dst_hashed != src_hashed)
-                        /* needed to create the linkto file */
-                        call_cnt ++;
+        return 0;
+    }
+
+    if (src_cached != dst_hashed) {
+        /* needed to create the link file */
+        call_cnt++;
+        if (dst_hashed != src_hashed)
+            /* needed to create the linkto file */
+            call_cnt++;
+    }
+
+    /* We should not have any failures post the link creation, as this
+     * introduces the newname into the namespace. Clients could have cached
+     * the existence of the newname and may start taking actions based on
+     * the same. Hence create the linkto first, and then attempt the link.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations. */
+    if (dst_hashed != src_hashed && src_cached != dst_hashed) {
+        gf_msg_trace(this->name, 0, "linkfile %s @ %s => %s", local->loc.path,
+                     dst_hashed->name, src_cached->name);
+
+        memcpy(local->gfid, local->loc.inode->gfid, 16);
+        dht_linkfile_create(frame, dht_rename_linkto_cbk, this, src_cached,
+                            dst_hashed, &local->loc);
+    } else if (src_cached != dst_hashed) {
+        dict_t *xattr_new = NULL;
+
+        xattr_new = dict_copy_with_ref(xattr, NULL);
+
+        gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                     local->loc2.path, src_cached->name);
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
 
-        /* We should not have any failures post the link creation, as this
-         * introduces the newname into the namespace. Clients could have cached
-         * the existence of the newname and may start taking actions based on
-         * the same. Hence create the linkto first, and then attempt the link.
-         *
-         * NOTE: If another client is attempting the same oldname -> newname
-         * rename, and finds both file names as existing, and are hard links
-         * to each other, then FUSE would send in an unlink for oldname. In
-         * this time duration if we treat the linkto as a critical error and
-         * unlink the newname we created, we would have effectively lost the
-         * file to rename operations. */
-        if (dst_hashed != src_hashed && src_cached != dst_hashed) {
-                gf_msg_trace (this->name, 0,
-                              "linkfile %s @ %s => %s",
-                              local->loc.path, dst_hashed->name,
-                              src_cached->name);
-
-                memcpy (local->gfid, local->loc.inode->gfid, 16);
-                dht_linkfile_create (frame, dht_rename_linkto_cbk, this,
-                                     src_cached, dst_hashed, &local->loc);
-        } else if (src_cached != dst_hashed) {
-                dict_t *xattr_new = NULL;
-
-                xattr_new = dict_copy_with_ref (xattr, NULL);
-
-                gf_msg_trace (this->name, 0,
-                              "link %s => %s (%s)", local->loc.path,
-                              local->loc2.path, src_cached->name);
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
-
-                local->added_link = _gf_true;
+        local->added_link = _gf_true;
 
-                STACK_WIND (frame, dht_rename_link_cbk,
-                            src_cached, src_cached->fops->link,
-                            &local->loc, &local->loc2, xattr_new);
+        STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                          src_cached->fops->link, &local->loc, &local->loc2,
+                          xattr_new);
 
-                dict_unref (xattr_new);
-        }
+        dict_unref(xattr_new);
+    }
 
 nolinks:
-        if (!call_cnt) {
-                /* skip to next step */
-                dht_do_rename (frame);
-        }
-        if (xattr)
-                dict_unref (xattr);
-
-        return 0;
+    if (!call_cnt) {
+        /* skip to next step */
+        dht_do_rename(frame);
+    }
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 }
 
 int
-dht_rename_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                       struct iatt *postparent)
+dht_rename_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
 {
-        dht_local_t *local                      = NULL;
-        int          call_cnt                   = 0;
-        dht_conf_t  *conf                       = NULL;
-
-        local = frame->local;
-        conf = this->private;
-
-        if (op_ret < 0) {
-                /* The meaning of is_linkfile is overloaded here. For locking
-                 * to work properly both rebalance and rename should acquire
-                 * lock on datafile. The reason for sending this lookup is to
-                 * find out whether we've acquired a lock on data file.
-                 * Between the lookup before rename and this rename, the
-                 * file could be migrated by a rebalance process and now this
-                 * file this might be a linkto file. We verify that by sending
-                 * this lookup. However, if this lookup fails we cannot really
-                 * say whether we've acquired lock on a datafile or linkto file.
-                 * So, we act conservatively and _assume_
-                 * that this is a linkfile and fail the rename operation.
-                 */
-                local->is_linkfile = _gf_true;
-        } else if (xattr && check_is_linkfile (inode, stbuf, xattr,
-                                               conf->link_xattr_name)) {
-                local->is_linkfile = _gf_true;
-        }
-
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                if (local->is_linkfile) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        goto fail;
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_server[GF_UUID_BUF_SIZE] = {0};
+    int child_index = -1;
+    gf_boolean_t is_src = _gf_false;
+    loc_t *loc = NULL;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+    conf = this->private;
+
+    is_src = (child_index == 0);
+    if (is_src)
+        loc = &local->loc;
+    else
+        loc = &local->loc2;
+
+    if (op_ret >= 0) {
+        if (is_src)
+            local->src_cached = dht_subvol_get_cached(this, local->loc.inode);
+        else {
+            if (loc->inode)
+                gf_uuid_unparse(loc->inode->gfid, gfid_local);
+
+            gf_msg_debug(this->name, 0,
+                         "dst_cached before lookup: %s, "
+                         "(path:%s)(gfid:%s),",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            local->dst_cached = dht_subvol_get_cached(this,
+                                                      local->loc2_copy.inode);
+
+            gf_uuid_unparse(stbuf->ia_gfid, gfid_local);
+
+            gf_msg_debug(this->name, GF_LOG_WARNING,
+                         "dst_cached after lookup: %s, "
+                         "(path:%s)(gfid:%s)",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            if ((local->loc2.inode == NULL) ||
+                gf_uuid_compare(stbuf->ia_gfid, local->loc2.inode->gfid)) {
+                if (local->loc2.inode != NULL) {
+                    inode_unlink(local->loc2.inode, local->loc2.parent,
+                                 local->loc2.name);
+                    inode_unref(local->loc2.inode);
                 }
 
-                dht_rename_create_links (frame);
+                local->loc2.inode = inode_link(local->loc2_copy.inode,
+                                               local->loc2_copy.parent,
+                                               local->loc2_copy.name, stbuf);
+                gf_uuid_copy(local->loc2.gfid, stbuf->ia_gfid);
+            }
+        }
+    }
+
+    if (op_ret < 0) {
+        if (is_src) {
+            /* The meaning of is_linkfile is overloaded here. For locking
+             * to work properly both rebalance and rename should acquire
+             * lock on datafile. The reason for sending this lookup is to
+             * find out whether we've acquired a lock on data file.
+             * Between the lookup before rename and this rename, the
+             * file could be migrated by a rebalance process and now this
+             * file this might be a linkto file. We verify that by sending
+             * this lookup. However, if this lookup fails we cannot really
+             * say whether we've acquired lock on a datafile or linkto file.
+             * So, we act conservatively and _assume_
+             * that this is a linkfile and fail the rename operation.
+             */
+            local->is_linkfile = _gf_true;
+            local->op_errno = op_errno;
+        } else {
+            if (local->dst_cached)
+                gf_msg_debug(this->name, op_errno,
+                             "file %s (gfid:%s) was present "
+                             "(hashed-subvol=%s, "
+                             "cached-subvol=%s) before rename,"
+                             " but lookup failed",
+                             local->loc2.path,
+                             uuid_utoa(local->loc2.inode->gfid),
+                             local->dst_hashed->name, local->dst_cached->name);
+            if (dht_inode_missing(op_errno))
+                local->dst_cached = NULL;
+        }
+    } else if (is_src && xattr &&
+               check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        local->is_linkfile = _gf_true;
+        /* Found linkto file instead of data file, passdown ENOENT
+         * based on the above comment */
+        local->op_errno = ENOENT;
+    }
+
+    if (!local->is_linkfile && (op_ret >= 0) &&
+        gf_uuid_compare(loc->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(loc->gfid, gfid_local);
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_server);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "path:%s, received a different gfid, local_gfid= %s"
+               " server_gfid: %s",
+               local->loc.path, gfid_local, gfid_server);
+
+        /* Will passdown ENOENT anyway since the file we sent on
+         * rename is replaced with a different file */
+        local->op_errno = ENOENT;
+        /* Since local->is_linkfile is used here to detect failure,
+         * marking this to true */
+        local->is_linkfile = _gf_true;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->is_linkfile) {
+            local->op_ret = -1;
+            goto fail;
         }
 
-        return 0;
+        dht_rename_create_links(frame);
+    }
+
+    return 0;
 fail:
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
-int32_t
-dht_rename_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+dht_rename_file_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                      = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        dict_t      *xattr_req                  = NULL;
-        dht_conf_t  *conf                       = NULL;
-        int          i                          = 0;
-
-        local = frame->local;
-        conf = this->private;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring inodelk failed "
-                        "rename (%s:%s:%s %s:%s:%s)",
-                        local->loc.path, src_gfid, local->src_cached->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_cached ? local->dst_cached->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-
-                goto done;
-        }
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_lock_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_unlock(frame, this);
+    return 0;
+}
 
-        xattr_req = dict_new ();
-        if (xattr_req == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto done;
-        }
+int32_t
+dht_rename_file_protect_namespace(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring inodelk failed "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto err;
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_file_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
-        op_ret = dict_set_uint32 (xattr_req,
-                                  conf->link_xattr_name, 256);
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = -op_ret;
-                goto done;
-        }
+err:
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
 
-        local->call_cnt = local->lock.lk_count;
+    return 0;
+}
 
-        for (i = 0; i < local->lock.lk_count; i++) {
-                STACK_WIND (frame, dht_rename_lookup_cbk,
-                            local->lock.locks[i]->xl,
-                            local->lock.locks[i]->xl->fops->lookup,
-                            &local->lock.locks[i]->loc, xattr_req);
+int32_t
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *xattr_req = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *subvol = NULL;
+    dht_lock_t *lock = NULL;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed. "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto done;
+    }
+
+    xattr_req = dict_new();
+    if (xattr_req == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto done;
+    }
+
+    op_ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = -op_ret;
+        goto done;
+    }
+
+    /* dst_cached might've changed. This normally happens for two reasons:
+     * 1. rebalance migrated dst
+     * 2. Another parallel rename was done overwriting dst
+     *
+     * Doing a lookup on local->loc2 when dst exists, but is associated
+     * with a different gfid will result in an ESTALE error. So, do a fresh
+     * lookup with a new inode on dst-path and handle change of dst-cached
+     * in the cbk. Also, to identify dst-cached changes we do a lookup on
+     * "this" rather than the subvol.
+     */
+    loc_copy(&local->loc2_copy, &local->loc2);
+    inode_unref(local->loc2_copy.inode);
+    local->loc2_copy.inode = inode_new(local->loc.inode->table);
+
+    /* Why not use local->lock.locks[?].loc for lookup post lock phase
+     * ---------------------------------------------------------------
+     * "layout.parent_layout.locks[?].loc" does not have the name and pargfid
+     * populated.
+     * Reason: If we had populated the name and pargfid, server might
+     * resolve to a successful lookup even if there is a file with same name
+     * with a different gfid(unlink & create) as server does name based
+     * resolution on first priority. And this can result in operating on a
+     * different inode entirely.
+     *
+     * Now consider a scenario where source file was renamed by some other
+     * client to a new name just before this lock was granted. So if a
+     * lookup would be done on local->lock[0].layout.parent_layout.locks[?].loc,
+     * server will send success even if the entry was renamed (since server will
+     * do a gfid based resolution). So once a lock is granted, make sure the
+     * file exists with the name that the client requested with.
+     * */
+
+    local->call_cnt = 2;
+    for (i = 0; i < 2; i++) {
+        if (i == 0) {
+            lock = local->rename_inodelk_backward_compatible[0];
+            if (gf_uuid_compare(local->loc.gfid, lock->loc.gfid) == 0)
+                subvol = lock->xl;
+            else {
+                lock = local->rename_inodelk_backward_compatible[1];
+                subvol = lock->xl;
+            }
+        } else {
+            subvol = this;
         }
 
-        dict_unref (xattr_req);
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_rename_lookup_cbk, (void *)(long)i, subvol,
+                          subvol->fops->lookup,
+                          (i == 0) ? &local->loc : &local->loc2_copy,
+                          xattr_req);
+    }
+
+    dict_unref(xattr_req);
+    return 0;
 
 done:
-        /* Its fine to call unlock even when no locks are acquired, as we check
-         * for lock->locked before winding a unlock call.
-         */
-        dht_rename_unlock (frame, this);
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_lock (call_frame_t *frame)
+dht_rename_lock(call_frame_t *frame)
 {
-        dht_local_t  *local    = NULL;
-        int           count    = 1, ret = -1;
-        dht_lock_t  **lk_array = NULL;
-
-        local = frame->local;
-
-        if (local->dst_cached)
-                count++;
-
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
-        if (lk_array == NULL)
-                goto err;
-
-        lk_array[0] = dht_lock_new (frame->this, local->src_cached, &local->loc,
-                                    F_WRLCK, DHT_FILE_MIGRATE_DOMAIN);
-        if (lk_array[0] == NULL)
-                goto err;
-
-        if (local->dst_cached) {
-                lk_array[1] = dht_lock_new (frame->this, local->dst_cached,
-                                            &local->loc2, F_WRLCK,
-                                            DHT_FILE_MIGRATE_DOMAIN);
-                if (lk_array[1] == NULL)
-                        goto err;
-        }
-
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
-
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    FAIL_ON_ANY_ERROR, dht_rename_lock_cbk);
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
+
+    local = frame->local;
+
+    if (local->dst_cached)
+        count++;
+
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (lk_array == NULL)
+        goto err;
+
+    lk_array[0] = dht_lock_new(frame->this, local->src_cached, &local->loc,
+                               F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                               FAIL_ON_ANY_ERROR);
+    if (lk_array[0] == NULL)
+        goto err;
+
+    if (local->dst_cached) {
+        /* dst might be removed by the time inodelk reaches bricks,
+         * which can result in ESTALE errors. POSIX imposes no
+         * restriction for dst to be present for renames to be
+         * successful. So, we'll ignore ESTALE errors. As far as
+         * synchronization on dst goes, we'll achieve the same by
+         * holding entrylk on parent directory of dst in the namespace
+         * of basename(dst). Also, there might not be quorum in cluster
+         * xlators like EC/disperse on errno, in which case they return
+         * EIO. For eg., in a disperse (4 + 2), 3 might return success
+         * and three might return ESTALE. Disperse, having no Quorum
+         * unwinds inodelk with EIO. So, ignore EIO too.
+         */
+        lk_array[1] = dht_lock_new(frame->this, local->dst_cached, &local->loc2,
+                                   F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                                   IGNORE_ENOENT_ESTALE_EIO);
+        if (lk_array[1] == NULL)
+            goto err;
+    }
+
+    local->rename_inodelk_backward_compatible = lk_array;
+    local->rename_inodelk_bc_count = count;
+
+    /* retaining inodelks for the sake of backward compatibility. Please
+     * make sure to remove this inodelk once all of 3.10, 3.12 and 3.13
+     * reach EOL. Better way of getting synchronization would be to acquire
+     * entrylks on src and dst parent directories in the namespace of
+     * basenames of src and dst
+     */
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_rename_file_protect_namespace);
+    if (ret < 0) {
+        local->rename_inodelk_backward_compatible = NULL;
+        local->rename_inodelk_bc_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                int tmp_count = 0, i = 0;
+    if (lk_array != NULL) {
+        int tmp_count = 0, i = 0;
 
-                for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++);
+        for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++)
+            ;
 
-                dht_lock_array_free (lk_array, tmp_count);
-                GF_FREE (lk_array);
-        }
+        dht_lock_array_free(lk_array, tmp_count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
 int
-dht_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        xlator_t    *src_cached = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        gf_uuid_unparse(oldloc->inode->gfid, gfid);
-
-        src_hashed = dht_subvol_get_hashed (this, oldloc);
-        if (!src_hashed) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No hashed subvolume in layout for path=%s,"
-                        "(gfid = %s)", oldloc->path, gfid);
-                op_errno = EINVAL;
-                goto err;
+    xlator_t *src_cached = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    char newgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    gf_uuid_unparse(oldloc->inode->gfid, gfid);
+
+    src_hashed = dht_subvol_get_hashed(this, oldloc);
+    if (!src_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    src_cached = dht_subvol_get_cached(this, oldloc->inode);
+    if (!src_cached) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No cached subvolume for path = %s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    dst_hashed = dht_subvol_get_hashed(this, newloc);
+    if (!dst_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s", newloc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (newloc->inode)
+        dst_cached = dht_subvol_get_cached(this, newloc->inode);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_RENAME);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /* cached_subvol will be set from dht_local_init, reset it to NULL,
+       as the logic of handling rename is different  */
+    local->cached_subvol = NULL;
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->src_hashed = src_hashed;
+    local->src_cached = src_cached;
+    local->dst_hashed = dst_hashed;
+    local->dst_cached = dst_cached;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (newloc->inode)
+        gf_uuid_unparse(newloc->inode->gfid, newgfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_INFO,
+           "renaming %s (%s) (hash=%s/cache=%s) => %s (%s) "
+           "(hash=%s/cache=%s) ",
+           oldloc->path, gfid, src_hashed->name, src_cached->name, newloc->path,
+           newloc->inode ? newgfid : NULL, dst_hashed->name,
+           dst_cached ? dst_cached->name : "<nul>");
+
+    if (IA_ISDIR(oldloc->inode->ia_type)) {
+        dht_rename_dir(frame, this);
+    } else {
+        local->op_ret = 0;
+        ret = dht_rename_lock(frame);
+        if (ret < 0) {
+            op_errno = ENOMEM;
+            goto err;
         }
+    }
 
-        src_cached = dht_subvol_get_cached (this, oldloc->inode);
-        if (!src_cached) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No cached subvolume for path = %s,"
-                        "(gfid = %s)", oldloc->path, gfid);
+    return 0;
 
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        dst_hashed = dht_subvol_get_hashed (this, newloc);
-        if (!dst_hashed) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No hashed subvolume in layout for path=%s",
-                        newloc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        if (newloc->inode)
-                dst_cached = dht_subvol_get_cached (this, newloc->inode);
+    return 0;
+}
 
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_RENAME);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        /* cached_subvol will be set from dht_local_init, reset it to NULL,
-           as the logic of handling rename is different  */
-        local->cached_subvol = NULL;
-
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
+{
+    gf_boolean_t free_xdata = _gf_false;
 
-        local->src_hashed = src_hashed;
-        local->src_cached = src_cached;
-        local->dst_hashed = dst_hashed;
-        local->dst_cached = dst_cached;
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_RENAME_INFO,
-                "renaming %s (hash=%s/cache=%s) => %s (hash=%s/cache=%s)",
-                oldloc->path, src_hashed->name, src_cached->name,
-                newloc->path, dst_hashed->name,
-                dst_cached ? dst_cached->name : "<nul>");
-
-        if (IA_ISDIR (oldloc->inode->ia_type)) {
-                dht_rename_dir (frame, this);
-        } else {
-                local->op_ret = 0;
-                ret = dht_rename_lock (frame);
-                if (ret < 0)
-                        goto err;
+    /* Just a pass through */
+    if (!IA_ISDIR(oldloc->inode->ia_type)) {
+        if (!xdata) {
+            free_xdata = _gf_true;
         }
-
-        return 0;
-
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
-
-        return 0;
+        DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);
+    }
+    default_rename(frame, this, oldloc, newloc, xdata);
+    if (free_xdata && xdata) {
+        dict_unref(xdata);
+        xdata = NULL;
+    }
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index a8c0ba7bc4d..3e24065227c 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -8,2486 +8,2531 @@
   cases as published by the Free Software Foundation.
 */
 
+#include "dht-lock.h"
+
+#define DHT_SET_LAYOUT_RANGE(layout, i, srt, chunk, path)                      \
+    do {                                                                       \
+        layout->list[i].start = srt;                                           \
+        layout->list[i].stop = srt + chunk - 1;                                \
+        layout->list[i].commit_hash = layout->commit_hash;                     \
+                                                                               \
+        gf_msg_trace(this->name, 0,                                            \
+                     "gave fix: 0x%x - 0x%x, with commit-hash 0x%x"            \
+                     " on %s for %s",                                          \
+                     layout->list[i].start, layout->list[i].stop,              \
+                     layout->list[i].commit_hash,                              \
+                     layout->list[i].xlator->name, path);                      \
+    } while (0)
+
+#define DHT_RESET_LAYOUT_RANGE(layout)                                         \
+    do {                                                                       \
+        int cnt = 0;                                                           \
+        for (cnt = 0; cnt < layout->cnt; cnt++) {                              \
+            layout->list[cnt].start = 0;                                       \
+            layout->list[cnt].stop = 0;                                        \
+        }                                                                      \
+    } while (0)
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "dht-common.h"
-#include "dht-messages.h"
-#include "glusterfs-acl.h"
-
-#define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,path)    do {           \
-                layout->list[i].start = srt;                            \
-                layout->list[i].stop  = srt + chunk - 1;                \
-                layout->list[i].commit_hash = layout->commit_hash;      \
-                                                                        \
-                gf_msg_trace (this->name, 0,                            \
-                              "gave fix: %u - %u, with commit-hash %u"  \
-                              " on %s for %s",                          \
-                              layout->list[i].start,                    \
-                              layout->list[i].stop,                     \
-                              layout->list[i].commit_hash,              \
-                              layout->list[i].xlator->name, path);      \
-        } while (0)
-
-#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
-                int cnt = 0;                                            \
-                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
-                        layout->list[cnt].start = 0;                    \
-                        layout->list[cnt].stop  = 0;                    \
-                }                                                       \
-        } while (0)
-
-int
-dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
-                          gf_boolean_t newdir,
-                          dht_selfheal_layout_t healer,
-                          dht_need_heal_t should_heal);
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal);
 
 static uint32_t
-dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
+dht_overlap_calc(dht_layout_t *old, int o, dht_layout_t *new, int n)
 {
-	if (o >= old->cnt || n >= new->cnt)
-		return 0;
+    if (o >= old->cnt || n >= new->cnt)
+        return 0;
 
-	if (old->list[o].err > 0 || new->list[n].err > 0)
-		return 0;
+    if (old->list[o].err > 0 || new->list[n].err > 0)
+        return 0;
 
-        if (old->list[o].start == old->list[o].stop) {
-                return 0;
-        }
+    if (old->list[o].start == old->list[o].stop) {
+        return 0;
+    }
 
-        if (new->list[n].start == new->list[n].stop) {
-                return 0;
-        }
+    if (new->list[n].start == new->list[n].stop) {
+        return 0;
+    }
 
-	if ((old->list[o].start > new->list[n].stop) ||
-	    (old->list[o].stop < new->list[n].start))
-		return 0;
+    if ((old->list[o].start > new->list[n].stop) ||
+        (old->list[o].stop < new->list[n].start))
+        return 0;
 
-	return min (old->list[o].stop, new->list[n].stop) -
-	        max (old->list[o].start, new->list[n].start) + 1;
+    return min(old->list[o].stop, new->list[n].stop) -
+           max(old->list[o].start, new->list[n].start) + 1;
 }
 
 int
-dht_selfheal_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_selfheal_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
 int
-dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret,
-                         int invoke_cbk)
+dht_selfheal_dir_finish(call_frame_t *frame, xlator_t *this, int ret,
+                        int invoke_cbk)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
-
-        local = frame->local;
-        lock_count = dht_lock_count (local->lock.locks, local->lock.lk_count);
-        if (lock_count == 0)
-                goto done;
-
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                goto done;
-        }
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL) {
-                goto done;
-        }
+    local = frame->local;
 
-        lock_local->lock.locks = local->lock.locks;
-        lock_local->lock.lk_count = local->lock.lk_count;
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
 
-        dht_unlock_inodelk (lock_frame, lock_local->lock.locks,
-                            lock_local->lock.lk_count,
-                            dht_selfheal_unlock_cbk);
-        lock_frame = NULL;
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
 
-done:
-        if (invoke_cbk)
-                local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-                                         local->op_errno, NULL);
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
 
-        return 0;
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
+
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
+
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_selfheal_unlock_cbk);
+    lock_frame = NULL;
+
+done:
+    if (invoke_cbk)
+        local->selfheal.dir_cbk(frame, NULL, frame->this, ret, local->op_errno,
+                                NULL);
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+    return 0;
 }
 
 int
-dht_refresh_layout_done (call_frame_t *frame)
+dht_refresh_layout_done(call_frame_t *frame)
 {
-        int                    ret         = -1;
-        dht_layout_t          *refreshed   = NULL, *heal = NULL;
-        dht_local_t           *local       = NULL;
-        dht_need_heal_t        should_heal = NULL;
-        dht_selfheal_layout_t  healer      = NULL;
+    int ret = -1;
+    dht_layout_t *refreshed = NULL, *heal = NULL;
+    dht_local_t *local = NULL;
+    dht_need_heal_t should_heal = NULL;
+    dht_selfheal_layout_t healer = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        refreshed = local->selfheal.refreshed_layout;
-        heal = local->selfheal.layout;
+    refreshed = local->selfheal.refreshed_layout;
+    heal = local->selfheal.layout;
 
-        healer = local->selfheal.healer;
-        should_heal = local->selfheal.should_heal;
+    healer = local->selfheal.healer;
+    should_heal = local->selfheal.should_heal;
 
-        ret = dht_layout_sort (refreshed);
-        if (ret == -1) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_SORT_FAILED,
-                        "sorting the layout failed");
-                goto err;
-        }
+    ret = dht_layout_sort(refreshed);
+    if (ret == -1) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LAYOUT_SORT_FAILED, NULL);
+        goto err;
+    }
 
-        if (should_heal (frame, &heal, &refreshed)) {
-                healer (frame, &local->loc, heal);
-        } else {
-                local->selfheal.layout = NULL;
-                local->selfheal.refreshed_layout = NULL;
-                local->selfheal.layout = refreshed;
+    if (should_heal(frame, &heal, &refreshed)) {
+        healer(frame, &local->loc, heal);
+    } else {
+        local->selfheal.layout = NULL;
+        local->selfheal.refreshed_layout = NULL;
+        local->selfheal.layout = refreshed;
 
-                dht_layout_unref (frame->this, heal);
+        dht_layout_unref(frame->this, heal);
 
-                dht_selfheal_dir_finish (frame, frame->this, 0, 1);
-        }
+        dht_selfheal_dir_finish(frame, frame->this, 0, 1);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, frame->this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, frame->this, -1, 1);
+    return 0;
 }
 
 int
-dht_refresh_layout_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *stbuf, dict_t *xattr,
-                        struct iatt *postparent)
+dht_refresh_layout_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *stbuf, dict_t *xattr,
+                       struct iatt *postparent)
 {
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        dht_layout_t *layout        = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
 
-        local = frame->local;
-        prev  = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        layout = local->selfheal.refreshed_layout;
+    layout = local->selfheal.refreshed_layout;
 
-        LOCK (&frame->lock);
-        {
-                op_ret = dht_layout_merge (this, layout, prev->this,
-                                           op_ret, op_errno, xattr);
+    LOCK(&frame->lock);
+    {
+        op_ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
 
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "lookup of %s on %s returned error",
-                                      local->loc.path, prev->this->name);
+        if (op_ret == -1) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
+            local->op_errno = op_errno;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_FILE_LOOKUP_FAILED, "path=%s", local->loc.path,
+                    "name=%s", prev->name, "gfid=%s", gfid, NULL);
 
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
+            goto unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
+        local->op_ret = 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (is_last_call (this_call_cnt)) {
-                if (local->op_ret == 0) {
-                        local->refresh_layout_done (frame);
-                } else {
-                        goto err;
-                }
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0) {
+            local->refresh_layout_done(frame);
+        } else {
+            goto err;
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        local->refresh_layout_unlock (frame, this, -1, 1);
-        return 0;
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
 }
 
 int
-dht_refresh_layout (call_frame_t *frame)
+dht_refresh_layout(call_frame_t *frame)
 {
-        int          call_cnt = 0;
-        int          i        = 0, ret = -1;
-        dht_conf_t  *conf     = NULL;
-        dht_local_t *local    = NULL;
-        xlator_t    *this     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-
-        this = frame->this;
-        conf = this->private;
-        local = frame->local;
-
-        call_cnt = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
-        local->op_ret = -1;
-
-        if (local->selfheal.refreshed_layout) {
-                dht_layout_unref (this, local->selfheal.refreshed_layout);
-                local->selfheal.refreshed_layout = NULL;
-        }
-
-        local->selfheal.refreshed_layout = dht_layout_new (this,
-                                                           conf->subvolume_cnt);
-        if (!local->selfheal.refreshed_layout) {
-                goto out;
-        }
-
-        if (local->xattr != NULL) {
-                dict_del (local->xattr, conf->xattr_name);
-        }
-
+    int call_cnt = 0;
+    int i = 0, ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+
+    this = frame->this;
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+    local->op_ret = -1;
+
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
+
+    local->selfheal.refreshed_layout = dht_layout_new(this,
+                                                      conf->subvolume_cnt);
+    if (!local->selfheal.refreshed_layout) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if (local->xattr != NULL) {
+        dict_del(local->xattr, conf->xattr_name);
+    }
+
+    if (local->xattr_req == NULL) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        local->xattr_req = dict_new();
         if (local->xattr_req == NULL) {
-                local->xattr_req = dict_new ();
-                if (local->xattr_req == NULL) {
-                        goto out;
-                }
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                    "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+            goto out;
         }
+    }
 
-        if (dict_get (local->xattr_req, conf->xattr_name) == 0) {
-                ret = dict_set_uint32 (local->xattr_req, conf->xattr_name,
-                                       4 * 4);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:key = %s",
-                                local->loc.path, conf->xattr_name);
-        }
+    if (dict_get(local->xattr_req, conf->xattr_name) == 0) {
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", local->loc.path, "key=%s", conf->xattr_name,
+                    NULL);
+    }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_refresh_layout_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_refresh_layout_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        local->refresh_layout_unlock (frame, this, -1, 1);
-        return 0;
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
 }
 
-
 int32_t
-dht_selfheal_layout_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_selfheal_layout_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                goto err;
-        }
+    if (!local) {
+        goto err;
+    }
 
-        if (op_ret < 0) {
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (op_ret < 0) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        local->refresh_layout_unlock = dht_selfheal_dir_finish;
-        local->refresh_layout_done = dht_refresh_layout_done;
+    local->refresh_layout_unlock = dht_selfheal_dir_finish;
+    local->refresh_layout_done = dht_refresh_layout_done;
 
-        dht_refresh_layout (frame);
-        return 0;
+    dht_refresh_layout(frame);
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-
 gf_boolean_t
-dht_should_heal_layout (call_frame_t *frame, dht_layout_t **heal,
-                        dht_layout_t **ondisk)
+dht_should_heal_layout(call_frame_t *frame, dht_layout_t **heal,
+                       dht_layout_t **ondisk)
 {
-        gf_boolean_t  fixit = _gf_true;
-        dht_local_t  *local = NULL;
-        int           ret   = -1, heal_missing_dirs = 0;
-
-        local = frame->local;
-
-        if ((heal == NULL) || (*heal == NULL) || (ondisk == NULL)
-            || (*ondisk == NULL))
-                goto out;
-
-        ret = dht_layout_anomalies (frame->this, &local->loc, *ondisk,
-                                    &local->selfheal.hole_cnt,
-                                    &local->selfheal.overlaps_cnt,
-                                    NULL, &local->selfheal.down,
-                                    &local->selfheal.misc, NULL);
-
-        if (ret < 0)
-                goto out;
-
-        /* Directories might've been created as part of this self-heal. We've to
-         * sync non-layout xattrs and set range 0-0 on new directories
+    gf_boolean_t fixit = _gf_true;
+    dht_local_t *local = NULL;
+    int heal_missing_dirs = 0;
+
+    local = frame->local;
+
+    if ((heal == NULL) || (*heal == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
+
+    dht_layout_anomalies(
+        frame->this, &local->loc, *ondisk, &local->selfheal.hole_cnt,
+        &local->selfheal.overlaps_cnt, &local->selfheal.missing_cnt,
+        &local->selfheal.down, &local->selfheal.misc, NULL);
+
+    /* Directories might've been created as part of this self-heal. We've to
+     * sync non-layout xattrs and set range 0-0 on new directories
+     */
+    heal_missing_dirs = local->selfheal.force_mkdir
+                            ? local->selfheal.force_mkdir
+                            : dht_layout_missing_dirs(*heal);
+
+    if ((local->selfheal.hole_cnt == 0) &&
+        (local->selfheal.overlaps_cnt == 0) && heal_missing_dirs) {
+        dht_layout_t *tmp = NULL;
+
+        /* Just added a brick and need to set 0-0 range on this brick.
+         * But ondisk layout is well-formed. So, swap layouts "heal" and
+         * "ondisk". Now "ondisk" layout will be used for healing
+         * xattrs. If there are any non-participating subvols in
+         * "ondisk" layout, dht_selfheal_dir_xattr_persubvol will set
+         * 0-0 and non-layout xattrs. This way we won't end up in
+         * "corrupting" already set and well-formed "ondisk" layout.
          */
-        heal_missing_dirs = local->selfheal.force_mkdir
-                ? local->selfheal.force_mkdir : dht_layout_missing_dirs (*heal);
-
-        if ((local->selfheal.hole_cnt == 0)
-            && (local->selfheal.overlaps_cnt == 0) && heal_missing_dirs) {
-                dht_layout_t *tmp = NULL;
-
-                /* Just added a brick and need to set 0-0 range on this brick.
-                 * But ondisk layout is well-formed. So, swap layouts "heal" and
-                 * "ondisk". Now "ondisk" layout will be used for healing
-                 * xattrs. If there are any non-participating subvols in
-                 * "ondisk" layout, dht_selfheal_dir_xattr_persubvol will set
-                 * 0-0 and non-layout xattrs. This way we won't end up in
-                 * "corrupting" already set and well-formed "ondisk" layout.
-                 */
-                tmp = *heal;
-                *heal = *ondisk;
-                *ondisk = tmp;
-
-                /* Current selfheal code, heals non-layout xattrs only after
-                 * an add-brick. In fact non-layout xattrs are considered as
-                 * secondary citizens which are healed only if layout xattrs
-                 * need to be healed. This is wrong, since for eg., quota can be
-                 * set when layout is well-formed, but a node is down. Also,
-                 * just for healing non-layout xattrs, we don't need locking.
-                 * This issue is _NOT FIXED_ by this patch.
-                 */
-        }
+        tmp = *heal;
+        *heal = *ondisk;
+        *ondisk = tmp;
+
+        /* Current selfheal code, heals non-layout xattrs only after
+         * an add-brick. In fact non-layout xattrs are considered as
+         * secondary citizens which are healed only if layout xattrs
+         * need to be healed. This is wrong, since for eg., quota can be
+         * set when layout is well-formed, but a node is down. Also,
+         * just for healing non-layout xattrs, we don't need locking.
+         * This issue is _NOT FIXED_ by this patch.
+         */
+    }
 
-        fixit = (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt
-                 || heal_missing_dirs);
+    fixit = (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+             heal_missing_dirs);
 
 out:
-        return fixit;
+    return fixit;
 }
 
 int
-dht_layout_span (dht_layout_t *layout)
+dht_layout_span(dht_layout_t *layout)
 {
-        int i = 0, count = 0;
+    int i = 0, count = 0;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err)
-                        continue;
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err)
+            continue;
 
-                if (layout->list[i].start != layout->list[i].stop)
-                        count++;
-        }
+        if (layout->list[i].start != layout->list[i].stop)
+            count++;
+    }
 
-        return count;
+    return count;
 }
 
 int
-dht_decommissioned_bricks_in_layout (xlator_t *this, dht_layout_t *layout)
+dht_decommissioned_bricks_in_layout(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t *conf  = NULL;
-        int         count = 0, i = 0, j = 0;
+    dht_conf_t *conf = NULL;
+    int count = 0, i = 0, j = 0;
 
-        if ((this == NULL) || (layout == NULL))
-                goto out;
+    if ((this == NULL) || (layout == NULL))
+        goto out;
 
-        conf = this->private;
+    conf = this->private;
 
-        for (i = 0; i < layout->cnt; i++) {
-                for (j = 0; j < conf->subvolume_cnt; j++) {
-                        if (conf->decommissioned_bricks[j] &&
-                            conf->decommissioned_bricks[j]
-                            == layout->list[i].xlator) {
-                                count++;
-                        }
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                count++;
+            }
         }
+    }
 
 out:
-        return count;
+    return count;
 }
 
 dht_distribution_type_t
-dht_distribution_type (xlator_t *this, dht_layout_t *layout)
+dht_distribution_type(xlator_t *this, dht_layout_t *layout)
 {
-        dht_distribution_type_t type        = GF_DHT_EQUAL_DISTRIBUTION;
-        int                     i           = 0;
-        uint32_t                start_range = 0, range = 0, diff = 0;
+    dht_distribution_type_t type = GF_DHT_EQUAL_DISTRIBUTION;
+    int i = 0;
+    uint32_t start_range = 0, range = 0, diff = 0;
 
-        if ((this == NULL) || (layout == NULL) || (layout->cnt < 1)) {
-                goto out;
-        }
+    if ((this == NULL) || (layout == NULL) || (layout->cnt < 1)) {
+        goto out;
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (start_range == 0) {
-                        start_range = layout->list[i].stop
-                                - layout->list[i].start;
-                        continue;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if (start_range == 0) {
+            start_range = layout->list[i].stop - layout->list[i].start;
+            continue;
+        }
 
-                range = layout->list[i].stop - layout->list[i].start;
-                diff = (range >= start_range)
-                        ? range - start_range
-                        : start_range - range;
+        range = layout->list[i].stop - layout->list[i].start;
+        diff = (range >= start_range) ? range - start_range
+                                      : start_range - range;
 
-                if ((range != 0) && (diff > layout->cnt)) {
-                        type = GF_DHT_WEIGHTED_DISTRIBUTION;
-                        break;
-                }
+        if ((range != 0) && (diff > layout->cnt)) {
+            type = GF_DHT_WEIGHTED_DISTRIBUTION;
+            break;
         }
+    }
 
 out:
-        return type;
+    return type;
 }
 
 gf_boolean_t
-dht_should_fix_layout (call_frame_t *frame, dht_layout_t **inmem,
-                       dht_layout_t **ondisk)
+dht_should_fix_layout(call_frame_t *frame, dht_layout_t **inmem,
+                      dht_layout_t **ondisk)
 {
-        gf_boolean_t             fixit                 = _gf_true;
+    gf_boolean_t fixit = _gf_true;
 
-        dht_local_t             *local                 = NULL;
-        int                      layout_span           = 0;
-        int                      decommissioned_bricks = 0;
-        int                      ret                   = 0;
-        dht_conf_t              *conf                  = NULL;
-        dht_distribution_type_t  inmem_dist_type       = 0;
-        dht_distribution_type_t  ondisk_dist_type      = 0;
+    dht_local_t *local = NULL;
+    int layout_span = 0;
+    int decommissioned_bricks = 0;
+    dht_conf_t *conf = NULL;
+    dht_distribution_type_t inmem_dist_type = 0;
+    dht_distribution_type_t ondisk_dist_type = 0;
 
-        conf = frame->this->private;
+    conf = frame->this->private;
 
-        local = frame->local;
+    local = frame->local;
 
-        if ((inmem == NULL) || (*inmem == NULL) || (ondisk == NULL)
-            || (*ondisk == NULL))
-                goto out;
+    if ((inmem == NULL) || (*inmem == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
 
-        ret = dht_layout_anomalies (frame->this, &local->loc, *ondisk,
-                                    &local->selfheal.hole_cnt,
-                                    &local->selfheal.overlaps_cnt, NULL,
-                                    &local->selfheal.down,
-                                    &local->selfheal.misc, NULL);
-        if (ret < 0) {
-                fixit = _gf_false;
-                goto out;
-        }
+    dht_layout_anomalies(frame->this, &local->loc, *ondisk,
+                         &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt, NULL,
+                         &local->selfheal.down, &local->selfheal.misc, NULL);
 
-        if (local->selfheal.down || local->selfheal.misc) {
-                fixit = _gf_false;
-                goto out;
-        }
+    if (local->selfheal.down || local->selfheal.misc) {
+        fixit = _gf_false;
+        goto out;
+    }
 
-        if (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt)
-                goto out;
+    if (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt)
+        goto out;
 
-        /* If commit hashes are being updated, let it through */
-        if ((*inmem)->commit_hash != (*ondisk)->commit_hash)
-                goto out;
+    /* If commit hashes are being updated, let it through */
+    if ((*inmem)->commit_hash != (*ondisk)->commit_hash)
+        goto out;
 
-        layout_span = dht_layout_span (*ondisk);
+    layout_span = dht_layout_span(*ondisk);
 
-        decommissioned_bricks
-                = dht_decommissioned_bricks_in_layout (frame->this,
-                                                       *ondisk);
-        inmem_dist_type = dht_distribution_type (frame->this, *inmem);
-        ondisk_dist_type = dht_distribution_type (frame->this, *ondisk);
+    decommissioned_bricks = dht_decommissioned_bricks_in_layout(frame->this,
+                                                                *ondisk);
+    inmem_dist_type = dht_distribution_type(frame->this, *inmem);
+    ondisk_dist_type = dht_distribution_type(frame->this, *ondisk);
 
-        if ((decommissioned_bricks == 0)
-            && (layout_span == (conf->subvolume_cnt
-                                - conf->decommission_subvols_cnt))
-            && (inmem_dist_type == ondisk_dist_type))
-                fixit = _gf_false;
+    if ((decommissioned_bricks == 0) &&
+        (layout_span ==
+         (conf->subvolume_cnt - conf->decommission_subvols_cnt)) &&
+        (inmem_dist_type == ondisk_dist_type))
+        fixit = _gf_false;
 
 out:
 
-        return fixit;
+    return fixit;
 }
 
-int
-dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
-                          gf_boolean_t newdir,
-                          dht_selfheal_layout_t healer,
-                          dht_need_heal_t should_heal)
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal)
 {
-        dht_local_t   *local    = NULL;
-        int            count    = 1, ret = -1, i = 0;
-        dht_lock_t   **lk_array = NULL;
-        dht_conf_t    *conf     = NULL;
-        dht_layout_t  *tmp      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *tmp = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        conf = frame->this->private;
+    local = frame->local;
 
-        local->selfheal.healer = healer;
-        local->selfheal.should_heal = should_heal;
+    conf = frame->this->private;
 
-        tmp = local->selfheal.layout;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
-        dht_layout_unref (frame->this, tmp);
+    local->selfheal.healer = healer;
+    local->selfheal.should_heal = should_heal;
 
-        if (!newdir) {
-                count = conf->subvolume_cnt;
+    tmp = local->selfheal.layout;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
+    dht_layout_unref(frame->this, tmp);
 
-                lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                                      gf_common_mt_char);
-                if (lk_array == NULL)
-                        goto err;
+    if (!newdir) {
+        count = conf->subvolume_cnt;
 
-                for (i = 0; i < count; i++) {
-                        lk_array[i] = dht_lock_new (frame->this,
-                                                    conf->subvolumes[i],
-                                                    &local->loc, F_WRLCK,
-                                                    DHT_LAYOUT_HEAL_DOMAIN);
-                        if (lk_array[i] == NULL)
-                                goto err;
-                }
-        } else {
-                count = 1;
-                lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                                      gf_common_mt_char);
-                if (lk_array == NULL)
-                        goto err;
-
-                lk_array[0] = dht_lock_new (frame->this, local->hashed_subvol,
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN);
-                if (lk_array[0] == NULL)
-                        goto err;
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
         }
 
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
-
-        ret = dht_blocking_inodelk (frame, lk_array, count, FAIL_ON_ANY_ERROR,
-                                    dht_selfheal_layout_lock_cbk);
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
+        for (i = 0; i < count; i++) {
+            lk_array[i] = dht_lock_new(
+                frame->this, conf->subvolumes[i], &local->loc, F_WRLCK,
+                DHT_LAYOUT_HEAL_DOMAIN, NULL, FAIL_ON_ANY_ERROR);
+            if (lk_array[i] == NULL) {
+                gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+                gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM,
+                        DHT_MSG_MEM_ALLOC_FAILED, "lk_array-gfid=%s", gfid,
+                        "path=%s", local->loc.path, NULL);
                 goto err;
-        }
-
-        return 0;
+            }
+        }
+    } else {
+        count = 1;
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+
+        lk_array[0] = dht_lock_new(frame->this, local->hashed_subvol,
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[0] == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+    }
+
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_selfheal_layout_lock_cbk);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
-int
-dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_selfheal_dir_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *subvol = NULL;
-        struct iatt  *stbuf = NULL;
-        int           i = 0;
-        int           ret = 0;
-        dht_layout_t *layout = NULL;
-        int           err = 0;
-        int           this_call_cnt = 0;
-
-        local = frame->local;
-        layout = local->selfheal.layout;
-        prev = cookie;
-        subvol = prev->this;
-
-        if (op_ret == 0)
-                err = 0;
-        else
-                err = op_errno;
-
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
-        if (ret < 0) {
-                gf_msg_debug (this->name, 0, "key = %s not present in dict",
-                              DHT_IATT_IN_XDATA_KEY);
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol) {
-                        layout->list[i].err = err;
-                        break;
-                }
-        }
-
-        LOCK (&frame->lock);
-        {
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        }
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_finish (frame, this, 0, 1);
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt *stbuf = NULL;
+    int i = 0;
+    int ret = 0;
+    dht_layout_t *layout = NULL;
+    int err = 0;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    subvol = cookie;
+
+    if (op_ret == 0) {
+        err = 0;
+    } else {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "name=%s", subvol->name,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        err = op_errno;
+    }
+
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    if (ret < 0) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_msg_debug(this->name, 0,
+                     "key = %s not present in dict"
+                     ", path:%s gfid:%s",
+                     DHT_IATT_IN_XDATA_KEY, local->loc.path, gfid);
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            layout->list[i].err = err;
+            break;
+        }
+    }
+
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
+    }
+
+    return 0;
 }
 
-
+/* Code is required to set user xattr to local->xattr
+ */
 int
-dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-                                  dht_layout_t *layout, int i,
-                                  xlator_t *req_subvol)
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data)
 {
-        xlator_t          *subvol = NULL;
-        dict_t            *xattr = NULL;
-        dict_t            *xdata = NULL;
-        int                ret = 0;
-        xlator_t          *this = NULL;
-        int32_t           *disk_layout = NULL;
-        dht_local_t       *local = NULL;
-        dht_conf_t        *conf = NULL;
-        data_t            *data = NULL;
-        char              gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        if (req_subvol)
-                subvol = req_subvol;
-        else
-                subvol = layout->list[i].xlator;
-        this = frame->this;
-
-        GF_VALIDATE_OR_GOTO ("", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, layout, err);
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
+    dict_t *set_xattr = data;
+    int ret = -1;
 
-        xattr = get_new_dict ();
-        if (!xattr) {
-                goto err;
-        }
-
-        xdata = dict_new ();
-        if (!xdata)
-                goto err;
-
-        ret = dict_set_str (xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value: key = %s,"
-                        " gfid = %s", loc->path,
-                        GLUSTERFS_INTERNAL_FOP_KEY, gfid);
-                goto err;
-        }
-
-        ret = dict_set_dynstr_with_alloc (xdata, DHT_IATT_IN_XDATA_KEY, "yes");
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value: key = %s,"
-                        " gfid = %s", loc->path,
-                        DHT_IATT_IN_XDATA_KEY, gfid);
-                goto err;
-        }
-
-        gf_uuid_unparse(loc->inode->gfid, gfid);
-
-        ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory self heal xattr failed:"
-                        " %s: (subvol %s) Failed to extract disk layout,"
-                        " gfid = %s", loc->path, subvol->name, gfid);
-                goto err;
-        }
-
-        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory self heal xattr failed:"
-                        "%s: (subvol %s) Failed to set xattr dictionary,"
-                        " gfid = %s", loc->path, subvol->name, gfid);
-                GF_FREE (disk_layout);
-                goto err;
-        }
-        disk_layout = NULL;
-
-        gf_msg_trace (this->name, 0,
-                      "setting hash range %u - %u (type %d) on subvolume %s"
-                      " for %s", layout->list[i].start, layout->list[i].stop,
-                      layout->type, subvol->name, loc->path);
-
-        dict_ref (xattr);
-        if (local->xattr) {
-                data = dict_get (local->xattr, QUOTA_LIMIT_KEY);
-                if (data) {
-                        ret = dict_add (xattr, QUOTA_LIMIT_KEY, data);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "%s: Failed to set dictionary value:"
-                                        " key = %s",
-                                        loc->path, QUOTA_LIMIT_KEY);
-                        }
-                }
-                data = dict_get (local->xattr, QUOTA_LIMIT_OBJECTS_KEY);
-                if (data) {
-                        ret = dict_add (xattr, QUOTA_LIMIT_OBJECTS_KEY, data);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "%s: Failed to set dictionary value:"
-                                        " key = %s",
-                                        loc->path, QUOTA_LIMIT_OBJECTS_KEY);
-                        }
-                }
-        }
-
-        if (!gf_uuid_is_null (local->gfid))
-                gf_uuid_copy (loc->gfid, local->gfid);
-
-        STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
-                    subvol, subvol->fops->setxattr,
-                    loc, xattr, 0, xdata);
-
-        dict_unref (xattr);
-        dict_unref (xdata);
-
-        return 0;
-
-err:
-        if (xattr)
-                dict_destroy (xattr);
-
-        if (xdata)
-                dict_unref (xdata);
-
-        GF_FREE (disk_layout);
-
-        dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
-                                    -1, ENOMEM, NULL);
-        return 0;
+    ret = dict_set(set_xattr, k, v);
+    return ret;
 }
 
-int
-dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+static int
+dht_selfheal_dir_xattr_persubvol(call_frame_t *frame, loc_t *loc,
+                                 dht_layout_t *layout, int i,
+                                 xlator_t *req_subvol)
 {
-        dht_local_t *local = NULL;
-        int          i = 0;
-        int          count = 0;
-        xlator_t    *this = NULL;
-        dht_conf_t  *conf = NULL;
-        dht_layout_t *dummy = NULL;
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    int32_t *disk_layout = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    data_t *data = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    if (req_subvol)
+        subvol = req_subvol;
+    else
+        subvol = layout->list[i].xlator;
+    this = frame->this;
+
+    GF_VALIDATE_OR_GOTO("", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, layout, err);
+    GF_VALIDATE_OR_GOTO(this->name, local, err);
+    GF_VALIDATE_OR_GOTO(this->name, subvol, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto err;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        goto err;
+
+    ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, DHT_IATT_IN_XDATA_KEY, 1);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", DHT_IATT_IN_XDATA_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    ret = dht_disk_layout_extract(this, layout, i, &disk_layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+                "extract-disk-layout-failed, path=%s", loc->path, "subvol=%s",
+                subvol->name, "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_bin(xattr, conf->xattr_name, disk_layout, 4 * 4);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s", loc->path,
+                "subvol=%s", subvol->name,
+                "set-xattr-dictionary-failed"
+                "gfid=%s",
+                gfid, NULL);
+        goto err;
+    }
+    disk_layout = NULL;
+
+    gf_msg_trace(this->name, 0,
+                 "setting hash range 0x%x - 0x%x (type %d) on subvolume %s"
+                 " for %s",
+                 layout->list[i].start, layout->list[i].stop, layout->type,
+                 subvol->name, loc->path);
+
+    if (local->xattr) {
+        data = dict_get(local->xattr, QUOTA_LIMIT_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_KEY, NULL);
+            }
+        }
+        data = dict_get(local->xattr, QUOTA_LIMIT_OBJECTS_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_OBJECTS_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_OBJECTS_KEY,
+                        NULL);
+            }
+        }
+    }
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_copy(loc->gfid, local->gfid);
+
+    STACK_WIND_COOKIE(frame, dht_selfheal_dir_xattr_cbk, (void *)subvol, subvol,
+                      subvol->fops->setxattr, loc, xattr, 0, xdata);
+
+    dict_unref(xattr);
+    dict_unref(xdata);
+
+    return 0;
 
-        gf_msg_debug (this->name, 0,
-                      "%s: Writing the new range for all subvolumes",
-                      loc->path);
-
-        local->call_cnt = count = conf->subvolume_cnt;
-
-        dht_log_new_layout_for_dir_selfheal (this, loc, layout);
-
-        for (i = 0; i < layout->cnt; i++) {
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+err:
+    if (xattr)
+        dict_unref(xattr);
+    if (xdata)
+        dict_unref(xdata);
 
-                if (--count == 0)
-                        goto out;
-        }
-        /* if we are here, subvolcount > layout_count. subvols-per-directory
-         * option might be set here. We need to clear out layout from the
-         * non-participating subvolumes, else it will result in overlaps */
-        dummy = dht_layout_new (this, 1);
-        if (!dummy)
-                goto out;
-        dummy->commit_hash = layout->commit_hash;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        if (--count == 0)
-                                break;
-                }
-        }
+    GF_FREE(disk_layout);
 
-        dht_layout_unref (this, dummy);
-out:
-        return 0;
+    dht_selfheal_dir_xattr_cbk(frame, (void *)subvol, frame->this, -1, ENOMEM,
+                               NULL);
+    return 0;
 }
 
-int
-dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+static int
+dht_fix_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
-        int          missing_xattr = 0;
-        int          i = 0;
-        xlator_t    *this = NULL;
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *dummy = NULL;
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop) {
-                        /* err != -1 would mean xattr present on the directory
-                         * or the directory is non existent.
-                         * !layout->list[i].stop would mean layout absent
-                         */
-
-                        continue;
-                }
-                missing_xattr++;
-        }
-        /* Also account for subvolumes with no-layout. Used for zero'ing out
-         * the layouts and for setting quota key's if present */
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        missing_xattr++;
-                }
-        }
-        gf_msg_trace (this->name, 0,
-                      "%d subvolumes missing xattr for %s",
-                      missing_xattr, loc->path);
-
-        if (missing_xattr == 0) {
-                dht_selfheal_dir_finish (frame, this, 0, 1);
-                return 0;
-        }
-
-        local->call_cnt = missing_xattr;
-
-        dht_log_new_layout_for_dir_selfheal (this, loc, layout);
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop)
-                        continue;
-
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
-
-                if (--missing_xattr == 0)
-                        break;
-        }
-        dummy = dht_layout_new (this, 1);
-        if (!dummy)
-                goto out;
-        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        missing_xattr--;
-                }
-        }
-
-        dht_layout_unref (this, dummy);
+    dht_local_t *local = NULL;
+    int i = 0;
+    int count = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    gf_msg_debug(this->name, 0, "%s: Writing the new range for all subvolumes",
+                 loc->path);
+
+    local->call_cnt = count = conf->subvolume_cnt;
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
+
+    for (i = 0; i < layout->cnt; i++) {
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
+
+        if (--count == 0)
+            goto out;
+    }
+    /* if we are here, subvolcount > layout_count. subvols-per-directory
+     * option might be set here. We need to clear out layout from the
+     * non-participating subvolumes, else it will result in overlaps */
+    dummy = dht_layout_new(this, 1);
+    if (!dummy)
+        goto out;
+    dummy->commit_hash = layout->commit_hash;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            if (--count == 0)
+                break;
+        }
+    }
+
+    dht_layout_unref(this, dummy);
 out:
-        return 0;
+    return 0;
 }
 
-gf_boolean_t
-dht_is_subvol_part_of_layout (dht_layout_t *layout, xlator_t *xlator)
+static int
+dht_selfheal_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        int               i = 0;
-        gf_boolean_t    ret = _gf_false;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
-                        ret = _gf_true;
-                        break;
-
-                }
-        }
-
-        return ret;
+    dht_local_t *local = NULL;
+    int missing_xattr = 0;
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop) {
+            /* err != -1 would mean xattr present on the directory
+             * or the directory is non existent.
+             * !layout->list[i].stop would mean layout absent
+             */
+
+            continue;
+        }
+        missing_xattr++;
+    }
+    /* Also account for subvolumes with no-layout. Used for zero'ing out
+     * the layouts and for setting quota key's if present */
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            missing_xattr++;
+        }
+    }
+    gf_msg_trace(this->name, 0, "%d subvolumes missing xattr for %s",
+                 missing_xattr, loc->path);
+
+    if (missing_xattr == 0) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
+        return 0;
+    }
+
+    local->call_cnt = missing_xattr;
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop)
+            continue;
+
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
+
+        if (--missing_xattr == 0)
+            break;
+    }
+    dummy = dht_layout_new(this, 1);
+    if (!dummy) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DUMMY_ALLOC_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            missing_xattr--;
+        }
+    }
+
+    dht_layout_unref(this, dummy);
+out:
+    return 0;
 }
 
 int
-dht_layout_index_from_conf (dht_layout_t *layout, xlator_t *xlator)
+dht_selfheal_dir_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *statpre,
+                             struct iatt *statpost, dict_t *xdata)
 {
-        int i = -1;
-        int j = 0;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int this_call_cnt = 0, ret = -1;
 
-        for (j = 0; j < layout->cnt; j++) {
-                if (!strcmp (layout->list[j].xlator->name, xlator->name)) {
-                        i = j;
-                        break;
-                }
-        }
-
-        return i;
-}
-
-
-static int
-dht_selfheal_dir_xattr_for_nameless_lookup (call_frame_t *frame, loc_t *loc,
-                                            dht_layout_t  *layout)
-{
-        dht_local_t     *local = NULL;
-        int             missing_xattr = 0;
-        int             i = 0;
-        xlator_t        *this = NULL;
-        dht_conf_t      *conf = NULL;
-        dht_layout_t    *dummy = NULL;
-        int             j = 0;
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop) {
-                        /* err != -1 would mean xattr present on the directory
-                           or the directory is non existent.
-                           !layout->list[i].stop would mean layout absent
-                        */
-
-                        continue;
-                }
-                missing_xattr++;
-        }
-
-        /* Also account for subvolumes with no-layout. Used for zero'ing out
-           the layouts and for setting quota key's if present */
-
-        /* Send  where either the subvol is not part of layout,
-         * or it is part of the layout but error is non-zero but error
-         * is not equal to -1 or ENOENT.
-         */
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (dht_is_subvol_part_of_layout (layout, conf->subvolumes[i])
-                    == _gf_false) {
-                        missing_xattr++;
-                        continue;
-                }
-
-                j = dht_layout_index_from_conf (layout, conf->subvolumes[i]);
-
-                if ((j != -1) && (layout->list[j].err != -1) &&
-                   (layout->list[j].err != 0) &&
-                   (layout->list[j].err != ENOENT)) {
-                        missing_xattr++;
-                }
-
-        }
-
-
-        gf_msg_trace (this->name, 0,
-                      "%d subvolumes missing xattr for %s",
-                      missing_xattr, loc->path);
-
-        if (missing_xattr == 0) {
-                dht_selfheal_dir_finish (frame, this, 0, 1);
-                return 0;
-        }
-
-        local->call_cnt = missing_xattr;
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        dht_log_new_layout_for_dir_selfheal (this, loc, layout);
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop)
-                        continue;
+    this_call_cnt = dht_frame_return(frame);
 
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+    if (is_last_call(this_call_cnt)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         local->loc.path, uuid_utoa(local->gfid));
 
-                if (--missing_xattr == 0)
-                        break;
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
         }
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
 
-        dummy = dht_layout_new (this, 1);
-        if (!dummy)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
-              if (dht_is_subvol_part_of_layout (layout, conf->subvolumes[i])
-                  == _gf_false) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        missing_xattr--;
-                        continue;
-              }
-
-                j = dht_layout_index_from_conf (layout, conf->subvolumes[i]);
-
-                if ((j != -1) && (layout->list[j].err != -1) &&
-                    (layout->list[j].err != ENOENT) &&
-                    (layout->list[j].err != 0)) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        missing_xattr--;
-                }
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
+    }
 
-        dht_layout_unref (this, dummy);
-out:
-        return 0;
-
+    return 0;
 }
 
 int
-dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, struct iatt *statpre,
-                              struct iatt *statpost, dict_t *xdata)
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        int            this_call_cnt = 0, ret = -1;
-
-        local  = frame->local;
-        layout = local->selfheal.layout;
-
-        this_call_cnt = dht_frame_return (frame);
+    int missing_attr = 0;
+    int i = 0, ret = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    /* We need to heal the attrs if:
+     * 1. Any directories were missing - the newly created dirs will need
+     *    to have the correct attrs set
+     * 2. An existing dir does not have the correct permissions -they may
+     *    have been changed when a brick was down.
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == -1)
+            missing_attr++;
+    }
+
+    if ((missing_attr == 0) && (local->need_attrheal == 0)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         loc->path, uuid_utoa(loc->gfid));
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
+        }
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
 
-        if (is_last_call (this_call_cnt)) {
-                ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                                dht_selfheal_dir_xattr,
-                                                dht_should_heal_layout);
-
-                if (ret < 0) {
-                        dht_selfheal_dir_finish (frame, this, -1, 1);
-                }
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
 
         return 0;
-}
+    }
 
+    cnt = local->call_cnt = conf->subvolume_cnt;
 
-int
-dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
-                          int32_t valid, dht_layout_t *layout)
-{
-        int           missing_attr = 0;
-        int           i     = 0, ret = -1;
-        dht_local_t  *local = NULL;
-        xlator_t     *this = NULL;
-
-        local = frame->local;
-        this = frame->this;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == -1)
-                        missing_attr++;
-        }
-
-        if (missing_attr == 0) {
-                ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                                dht_selfheal_dir_xattr,
-                                                dht_should_heal_layout);
-
-                if (ret < 0) {
-                        dht_selfheal_dir_finish (frame, this, -1, 1);
-                }
+    for (i = 0; i < cnt; i++) {
+        STACK_WIND(frame, dht_selfheal_dir_setattr_cbk, layout->list[i].xlator,
+                   layout->list[i].xlator->fops->setattr, loc, stbuf, valid,
+                   NULL);
+    }
 
-                return 0;
-        }
-
-        if (!gf_uuid_is_null (local->gfid))
-                gf_uuid_copy (loc->gfid, local->gfid);
-
-        local->call_cnt = missing_attr;
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == -1) {
-                        gf_msg_trace (this->name, 0,
-                                      "%s: setattr on subvol %s, gfid = %s",
-                                      loc->path, layout->list[i].xlator->name,
-                                      uuid_utoa(loc->gfid));
-
-                        STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setattr,
-                                    loc, stbuf, valid, NULL);
-                }
-        }
-
-        return 0;
+    return 0;
 }
 
-int
-dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno,
-                            inode_t *inode, struct iatt *stbuf,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t *xdata)
+static int
+dht_selfheal_dir_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, inode_t *inode,
+                           struct iatt *stbuf, struct iatt *preparent,
+                           struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        call_frame_t  *prev = NULL;
-        xlator_t      *subvol = NULL;
-        int            i = 0, ret = -1;
-        int            this_call_cnt = 0;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local  = frame->local;
-        layout = local->selfheal.layout;
-        prev   = cookie;
-        subvol = prev->this;
-
-        if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
-                for (i = 0; i < layout->cnt; i++) {
-                        if (layout->list[i].xlator == subvol) {
-                                layout->list[i].err = -1;
-                                break;
-                        }
-                }
-        }
-
-        if (op_ret) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
-                gf_msg (this->name, ((op_errno == EEXIST) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
-                        op_errno, DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: path = %s, gfid = %s",
-                        local->loc.path, gfid );
-                goto out;
-        }
-        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
-        ret = 0;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0, ret = -1;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    prev = cookie;
+    subvol = prev;
+
+    if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
+        for (i = 0; i < layout->cnt; i++) {
+            if (layout->list[i].xlator == subvol) {
+                layout->list[i].err = -1;
+                break;
+            }
+        }
+    }
+
+    if (op_ret) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name,
+                ((op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_WARNING),
+                op_errno, DHT_MSG_DIR_SELFHEAL_FAILED, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+    ret = 0;
 
 out:
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_finish (frame, this, ret, 0);
-                dht_selfheal_dir_setattr (frame, &local->loc, &local->stbuf, 0xffffff, layout);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, ret, 0);
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffff,
+                                 layout);
+    }
 
-        return 0;
+    return 0;
 }
 
-void
-dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+static int
+dht_selfheal_dir_mkdir_lookup_done(call_frame_t *frame, xlator_t *this)
 {
-        data_t          *acl_default = NULL;
-        data_t          *acl_access = NULL;
-        xlator_t        *this = NULL;
-        int     ret = -1;
-
-        GF_ASSERT (xattr);
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
+    dht_local_t *local = NULL;
+    int i = 0;
+    dict_t *dict = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+    int cnt = 0;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        dict = dict_new();
+        if (!dict)
+            return -1;
 
-        if (!acl_default) {
-                gf_msg_debug (this->name, 0,
-                              "ACL_DEFAULT xattr not present");
-                goto cont;
-        }
-        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
         if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        POSIX_ACL_DEFAULT_XATTR);
-cont:
-        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
-        if (!acl_access) {
-                gf_msg_debug (this->name, 0,
-                              "ACL_ACCESS xattr not present");
-                goto out;
-        }
-        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        POSIX_ACL_ACCESS_XATTR);
-
-out:
-        return;
-}
-
-int
-dht_selfheal_dir_mkdir_lookup_done (call_frame_t *frame, xlator_t *this)
-{
-        dht_local_t  *local = NULL;
-        int           i     = 0;
-        int           ret   = -1;
-        dict_t       *dict = NULL;
-        dht_layout_t  *layout = NULL;
-        loc_t        *loc   = NULL;
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = frame->local;
-        layout = local->layout;
-        loc    = &local->loc;
-
-        if (!gf_uuid_is_null (local->gfid)) {
-                dict = dict_new ();
-                if (!dict)
-                        return -1;
-
-                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:"
-                                " key = gfid-req", loc->path);
-        } else if (local->params) {
-                /* Send the dictionary from higher layers directly */
-
-                dict = dict_ref (local->params);
-        }
-        /* Set acls */
-        if (local->xattr && dict)
-                dht_selfheal_dir_mkdir_setacl (local->xattr, dict);
-
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "key=gfid-req", NULL);
+    } else if (local->params) {
+        /* Send the dictionary from higher layers directly */
+
+        dict = dict_ref(local->params);
+    }
+    /* Code to update all extended attributed from local->xattr
+       to dict
+    */
+    dht_dir_set_heal_xattr(this, local, dict, local->xattr, NULL, NULL);
+
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_IS_NULL, NULL);
+        dict = dict_new();
         if (!dict)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "dict is NULL, need to make sure gfids are same");
+            return -1;
+    }
+    ret = dict_set_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "key=%s",
+                GF_INTERNAL_CTX_KEY, "path=%s", loc->path, NULL);
+        /* We can still continue. As heal can still happen
+         * unless quota limits have reached for the dir.
+         */
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == ESTALE ||
-                    layout->list[i].err == ENOENT ||
-                    local->selfheal.force_mkdir) {
-                        gf_msg_debug (this->name, 0,
-                                      "Creating directory %s on subvol %s",
-                                      loc->path, layout->list[i].xlator->name);
-
-                        STACK_WIND (frame, dht_selfheal_dir_mkdir_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->mkdir,
-                                    loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, dict);
-                }
+    cnt = layout->cnt;
+    for (i = 0; i < cnt; i++) {
+        if (layout->list[i].err == ESTALE || layout->list[i].err == ENOENT ||
+            local->selfheal.force_mkdir) {
+            gf_msg_debug(this->name, 0, "Creating directory %s on subvol %s",
+                         loc->path, layout->list[i].xlator->name);
+
+            STACK_WIND_COOKIE(
+                frame, dht_selfheal_dir_mkdir_cbk, layout->list[i].xlator,
+                layout->list[i].xlator, layout->list[i].xlator->fops->mkdir,
+                loc,
+                st_mode_from_ia(local->stbuf.ia_prot, local->stbuf.ia_type), 0,
+                dict);
         }
+    }
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-int
-dht_selfheal_dir_mkdir_lookup_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int op_ret, int op_errno,
-                                   inode_t *inode, struct iatt *stbuf,
-                                   dict_t *xattr, struct iatt *postparent)
+static int
+dht_selfheal_dir_mkdir_lookup_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  dict_t *xattr, struct iatt *postparent)
 {
-        dht_local_t  *local = NULL;
-        int           i     = 0;
-        int           this_call_cnt = 0;
-        int           missing_dirs = 0;
-        dht_layout_t  *layout = NULL;
-        loc_t         *loc    = NULL;
-        call_frame_t *prev    = NULL;
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = frame->local;
-        layout = local->layout;
-        loc = &local->loc;
-        prev  = cookie;
-
-        this_call_cnt = dht_frame_return (frame);
-
-        LOCK (&frame->lock);
-        {
-                if ((op_ret < 0) &&
-                    (op_errno == ENOENT || op_errno == ESTALE)) {
-                        local->selfheal.hole_cnt = !local->selfheal.hole_cnt ? 1
-                                                : local->selfheal.hole_cnt + 1;
-                }
-
-                if (!op_ret) {
-                        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                }
+    dht_local_t *local = NULL;
+    int i = 0;
+    int this_call_cnt = 0;
+    int missing_dirs = 0;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = 0;
+    loc_t *loc = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    int index = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+    prev = cookie;
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&frame->lock);
+    {
+        index = dht_layout_index_for_subvol(layout, prev);
+        if ((op_ret < 0) && (op_errno == ENOENT || op_errno == ESTALE)) {
+            local->selfheal.hole_cnt = !local->selfheal.hole_cnt
+                                           ? 1
+                                           : local->selfheal.hole_cnt + 1;
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = op_errno;
+            }
+        }
+
+        if (!op_ret) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            if (prev == local->mds_subvol) {
+                dict_unref(local->xattr);
+                local->xattr = dict_ref(xattr);
+            }
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = -1;
+            }
+        }
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (local->selfheal.hole_cnt == layout->cnt) {
+            gf_msg_debug(this->name, op_errno,
+                         "Lookup failed, an rmdir could have "
+                         "deleted this entry %s",
+                         loc->name);
+            local->op_errno = op_errno;
+            goto err;
+        } else {
+            for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT ||
+                    layout->list[i].err == ESTALE ||
+                    local->selfheal.force_mkdir)
+                    missing_dirs++;
+            }
+
+            if (missing_dirs == 0) {
+                dht_selfheal_dir_finish(frame, this, 0, 0);
+                dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff,
+                                         layout);
+                return 0;
+            }
 
+            local->call_cnt = missing_dirs;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
         }
-        UNLOCK (&frame->lock);
-
-        if (is_last_call (this_call_cnt)) {
-                if (local->selfheal.hole_cnt == layout->cnt) {
-                        gf_msg_debug (this->name, op_errno,
-                                      "Lookup failed, an rmdir could have "
-                                      "deleted this entry %s", loc->name);
-                        local->op_errno = op_errno;
-                        goto err;
-                } else {
-                        for (i = 0; i < layout->cnt; i++) {
-                                if (layout->list[i].err == ENOENT ||
-                                    layout->list[i].err == ESTALE ||
-                                    local->selfheal.force_mkdir)
-                                        missing_dirs++;
-                        }
-
-                        if (missing_dirs == 0) {
-                                dht_selfheal_dir_finish (frame, this, 0, 0);
-                                dht_selfheal_dir_setattr (frame, loc,
-                                                          &local->stbuf,
-                                                          0xffffffff, layout);
-                                return 0;
-                        }
-
-                        local->call_cnt = missing_dirs;
-                        dht_selfheal_dir_mkdir_lookup_done (frame, this);
-                }
-        }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-
-int
-dht_selfheal_dir_mkdir_lock_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+static int
+dht_selfheal_dir_mkdir_lock_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        int           i     = 0;
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-        local = frame->local;
-
-	    local->call_cnt = conf->subvolume_cnt;
-
-        if (op_ret < 0) {
-
-                /* We get this error when the directory entry was not created
-                 * on a newky attatched tier subvol. Hence proceed and do mkdir
-                 * on the tier subvol.
-                 */
-                if (op_errno == EINVAL) {
-                        local->call_cnt = 1;
-                        dht_selfheal_dir_mkdir_lookup_done (frame, this);
-                        return 0;
-                }
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring inodelk failed for %s",
-                        local->loc.path);
-
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        /* After getting locks, perform lookup again to ensure that the
-           directory was not deleted by a racing rmdir
-        */
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_selfheal_dir_mkdir_lookup_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, NULL);
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    int ret = -1;
+    xlator_t *mds_subvol = NULL;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+    local = frame->local;
+    mds_subvol = local->mds_subvol;
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    if (op_ret < 0) {
+        if (op_errno == EINVAL) {
+            local->call_cnt = 1;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
+            return 0;
+        }
+
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_ENTRYLK_ERROR,
+                "path=%s", local->loc.path, NULL);
+
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    /* After getting locks, perform lookup again to ensure that the
+       directory was not deleted by a racing rmdir
+    */
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    ret = dict_set_int32(local->xattr_req, "list-xattr", 1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "path=%s",
+                local->loc.path, NULL);
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && conf->subvolumes[i] == mds_subvol) {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              NULL);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-int
-dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
-                        dht_layout_t *layout, int force)
+static int
+dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout,
+                       int force)
 {
-        int           missing_dirs = 0;
-        int           i     = 0;
-        int           ret   = -1;
-        int           count = 1;
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        xlator_t     *this = NULL;
-        dht_lock_t   **lk_array = NULL;
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
-
-        local->selfheal.force_mkdir = force;
-        local->selfheal.hole_cnt = 0;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == ENOENT || force)
-                        missing_dirs++;
-        }
-
-        if (missing_dirs == 0) {
-                dht_selfheal_dir_setattr (frame, loc, &local->stbuf,
-                                          0xffffffff, layout);
-                return 0;
-        }
-
-        count = conf->subvolume_cnt;
-
-        /* Locking on all subvols in the mkdir phase of lookup selfheal is
-           is done to synchronize with rmdir/rename.
-        */
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
-        if (lk_array == NULL)
-                goto err;
-
-        for (i = 0; i < count; i++) {
-                lk_array[i] = dht_lock_new (frame->this,
-                                            conf->subvolumes[i],
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN);
-                if (lk_array[i] == NULL)
-                        goto err;
-        }
-
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
+    int missing_dirs = 0;
+    int i = 0;
+    int op_errno = 0;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    local->selfheal.force_mkdir = force;
+    local->selfheal.hole_cnt = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == ENOENT || force)
+            missing_dirs++;
+    }
+
+    if (missing_dirs == 0) {
+        /* We don't need to create any directories. Proceed to heal the
+         * attrs and xattrs
+         */
+        if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+            if (local->need_xattr_heal) {
+                local->need_xattr_heal = 0;
+                ret = dht_dir_xattr_heal(this, local, &op_errno);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s",
+                            local->loc.path, "gfid=%s", local->gfid, NULL);
+                }
+            } else {
+                if (!gf_uuid_is_null(local->gfid))
+                    gf_uuid_copy(loc->gfid, local->gfid);
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    IGNORE_ENOENT_ESTALE,
-                                    dht_selfheal_dir_mkdir_lock_cbk);
+                ret = dht_common_mark_mdsxattr(frame, NULL, 0);
+                if (!ret)
+                    return 0;
 
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                goto err;
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SET_XATTR_FAILED,
+                        "path=%s", local->loc.path, "gfid=%s", local->gfid,
+                        NULL);
+            }
         }
-
+        dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff, layout);
         return 0;
+    }
+
+    /* MDS xattr is populated only while DHT is having more than one
+     subvol.In case of graph switch while adding more dht subvols need to
+     consider hash subvol as a MDS to avoid MDS check failure at the time
+     of running fop on directory
+    */
+    if (!dict_get(local->xattr, conf->mds_xattr_key) &&
+        (conf->subvolume_cnt > 1)) {
+        if (local->hashed_subvol == NULL) {
+            local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+            if (local->hashed_subvol == NULL) {
+                local->op_errno = EINVAL;
+                gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s",
+                        loc->pargfid, "name=%s", loc->name, "path=%s",
+                        loc->path, NULL);
+                goto err;
+            }
+        }
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this,
+                                       local->hashed_subvol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s on inode vol is %s",
+                   local->loc.path,
+                   local->hashed_subvol ? local->hashed_subvol->name : "NULL");
+            goto err;
+        }
+    }
+
+    if (local->hashed_subvol == NULL) {
+        local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (local->hashed_subvol == NULL) {
+            local->op_errno = EINVAL;
+            gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                    DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid,
+                    "name=%s", loc->name, "path=%s", loc->path, NULL);
+            goto err;
+        }
+    }
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, local->hashed_subvol,
+                                &local->current->ns,
+                                dht_selfheal_dir_mkdir_lock_cbk);
+
+    if (ret < 0)
+        goto err;
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
-
-        return -1;
+    return -1;
 }
 
-int
-dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
-                                 dht_layout_t *layout)
+static int
+dht_selfheal_layout_alloc_start(xlator_t *this, loc_t *loc,
+                                dht_layout_t *layout)
 {
-        int         start                               = 0;
-        uint32_t    hashval                             = 0;
-        int         ret                                 = 0;
-        const char *str                                 = NULL;
-        dht_conf_t *conf                                = NULL;
-        char           buf[UUID_CANONICAL_FORM_LEN + 1] = {0, };
-
-        conf = this->private;
-
-        if (conf->randomize_by_gfid) {
-                str = uuid_utoa_r (loc->gfid, buf);
-        } else {
-                str = loc->path;
-        }
-
-        ret = dht_hash_compute (this, layout->type, str, &hashval);
-        if (ret == 0) {
-                start = (hashval % layout->cnt);
-        }
-
-        return start;
+    int start = 0;
+    uint32_t hashval = 0;
+    int ret = 0;
+    const char *str = NULL;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+
+    conf = this->private;
+
+    if (conf->randomize_by_gfid) {
+        str = uuid_utoa_r(loc->gfid, buf);
+    } else {
+        str = loc->path;
+    }
+
+    ret = dht_hash_compute(this, layout->type, str, &hashval);
+    if (ret == 0) {
+        start = (hashval % layout->cnt);
+    }
+
+    return start;
 }
 
 static int
-dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
+dht_get_layout_count(xlator_t *this, dht_layout_t *layout, int new_layout)
 {
-        int i = 0;
-        int j = 0;
-        int err = 0;
-        int count = 0;
-        dht_conf_t *conf = NULL;
-
-        /* Gets in use only for replace-brick, remove-brick */
-        conf = this->private;
+    int i = 0;
+    int j = 0;
+    int err = 0;
+    int count = 0;
+    dht_conf_t *conf = NULL;
+
+    /* Gets in use only for replace-brick, remove-brick */
+    conf = this->private;
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                layout->list[i].err = EINVAL;
+                break;
+            }
+        }
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        err = layout->list[i].err;
+        if (err == -1 || err == 0 || err == ENOENT) {
+            /* Take this with a pinch of salt. The behaviour seems
+             * to be slightly different when this function is
+             * invoked from mkdir codepath. For eg., err == 0 in
+             * mkdir codepath means directory created but xattr
+             * is not set yet.
+             */
+
+            /* Setting list[i].err = -1 is an indication for
+               dht_selfheal_layout_new_directory() to assign
+               a range. We set it to -1 based on any one of
+               the three criteria:
+
+               - err == -1 already, which means directory
+                 existed but layout was not set on it.
+
+               - err == 0, which means directory exists and
+                 has an old layout piece which will be
+                 overwritten now.
+
+               - err == ENOENT, which means directory does
+                 not exist (possibly racing with mkdir or
+                 finishing half done mkdir). The missing
+                 directory will be attempted to be recreated.
+            */
+            count++;
+            if (!err)
+                layout->list[i].err = -1;
+        }
+    }
+
+    /* no subvolume has enough space, but can't stop directory creation */
+    if (!count || !new_layout) {
         for (i = 0; i < layout->cnt; i++) {
-                for (j = 0; j < conf->subvolume_cnt; j++) {
-                        if (conf->decommissioned_bricks[j] &&
-                            conf->decommissioned_bricks[j] == layout->list[i].xlator) {
-                                layout->list[i].err = EINVAL;
-                                break;
-                        }
-                }
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                err = layout->list[i].err;
-                if (err == -1 || err == 0 || err == ENOENT) {
-                        /* Take this with a pinch of salt. The behaviour seems
-                         * to be slightly different when this function is
-                         * invoked from mkdir codepath. For eg., err == 0 in
-                         * mkdir codepath means directory created but xattr
-                         * is not set yet.
-                         */
-
-			/* Setting list[i].err = -1 is an indication for
-			   dht_selfheal_layout_new_directory() to assign
-			   a range. We set it to -1 based on any one of
-			   the three criteria:
-
-			   - err == -1 already, which means directory
-			     existed but layout was not set on it.
-
-			   - err == 0, which means directory exists and
-			     has an old layout piece which will be
-			     overwritten now.
-
-			   - err == ENOENT, which means directory does
-			     not exist (possibly racing with mkdir or
-			     finishing half done mkdir). The missing
-			     directory will be attempted to be recreated.
-			*/
-                        count++;
-			if (!err)
-				layout->list[i].err = -1;
-                }
-        }
-
-        /* no subvolume has enough space, but can't stop directory creation */
-        if (!count || !new_layout) {
-                for (i = 0; i < layout->cnt; i++) {
-                        err = layout->list[i].err;
-                        if (err == ENOSPC) {
-                                layout->list[i].err = -1;
-                                count++;
-                        }
-                }
-        }
-
-        /* if layout->spread_cnt is set, check if it is <= available
-         * subvolumes (down brick and decommissioned bricks are considered
-         * un-availbale). Else return count (available up bricks) */
-        count = ((layout->spread_cnt &&
-                 (layout->spread_cnt <= count)) ?
-                 layout->spread_cnt : ((count) ? count : 1));
-
-        return count;
+            err = layout->list[i].err;
+            if (err == ENOSPC) {
+                layout->list[i].err = -1;
+                count++;
+            }
+        }
+    }
+
+    /* if layout->spread_cnt is set, check if it is <= available
+     * subvolumes (down brick and decommissioned bricks are considered
+     * un-available). Else return count (available up bricks) */
+    count = ((layout->spread_cnt && (layout->spread_cnt <= count))
+                 ? layout->spread_cnt
+                 : ((count) ? count : 1));
+
+    return count;
 }
 
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
 
-void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-					dht_layout_t *new_layout);
-
-void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
-void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+void
+dht_layout_range_swap(dht_layout_t *layout, int i, int j);
 
 /*
  * It's a bit icky using local variables in a macro, but it makes the rest
  * of the code a lot clearer.
  */
-#define OV_ENTRY(x,y)      table[x*new->cnt+y]
+#define OV_ENTRY(x, y) table[x * new->cnt + y]
 
-void
-dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
-				      dht_layout_t *new, dht_layout_t *old)
+static void
+dht_selfheal_layout_maximize_overlap(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *new, dht_layout_t *old)
 {
-	int           i            = 0;
-	int           j            = 0;
-	uint32_t      curr_overlap = 0;
-	uint32_t      max_overlap  = 0;
-	int           max_overlap_idx = -1;
-	uint32_t      overlap      = 0;
-        uint32_t     *table = NULL;
-
-	dht_layout_sort_volname (old);
-	/* Now both old_layout->list[] and new_layout->list[]
-	   are match the same xlators/subvolumes. i.e,
-	   old_layout->[i] and new_layout->[i] are referring
-	   to the same subvolumes
-	*/
-
-        /* Build a table of overlaps between new[i] and old[j]. */
-        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
-        if (!table) {
-                return;
-        }
-        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
-        for (i = 0; i < new->cnt; ++i) {
-                for (j = 0; j < old->cnt; ++j) {
-                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
+    int i = 0;
+    int j = 0;
+    uint32_t curr_overlap = 0;
+    uint32_t max_overlap = 0;
+    int max_overlap_idx = -1;
+    uint32_t overlap = 0;
+    uint32_t *table = NULL;
+
+    dht_layout_sort_volname(old);
+    /* Now both old_layout->list[] and new_layout->list[]
+       are match the same xlators/subvolumes. i.e,
+       old_layout->[i] and new_layout->[i] are referring
+       to the same subvolumes
+    */
+
+    /* Build a table of overlaps between new[i] and old[j]. */
+    table = alloca(sizeof(overlap) * old->cnt * new->cnt);
+    if (!table) {
+        return;
+    }
+    memset(table, 0, sizeof(overlap) * old->cnt * new->cnt);
+    for (i = 0; i < new->cnt; ++i) {
+        for (j = 0; j < old->cnt; ++j) {
+            OV_ENTRY(i, j) = dht_overlap_calc(old, j, new, i);
+        }
+    }
+
+    for (i = 0; i < new->cnt; i++) {
+        if (new->list[i].err > 0) {
+            /* Subvol might be marked for decommission
+               with EINVAL, or some other serious error
+               marked with positive errno.
+            */
+            continue;
+        }
+
+        max_overlap = 0;
+        max_overlap_idx = i;
+        for (j = (i + 1); j < new->cnt; ++j) {
+            if (new->list[j].err > 0) {
+                /* Subvol might be marked for decommission
+                with EINVAL, or some other serious error
+                marked with positive errno.
+                */
+                continue;
+            }
+            /* Calculate the overlap now. */
+            curr_overlap = OV_ENTRY(i, i) + OV_ENTRY(j, j);
+            /* Calculate the overlap after the proposed swap. */
+            overlap = OV_ENTRY(i, j) + OV_ENTRY(j, i);
+            /* Are we better than status quo? */
+            if (overlap > curr_overlap) {
+                overlap -= curr_overlap;
+                /* Are we better than the previous choice? */
+                if (overlap > max_overlap) {
+                    max_overlap = overlap;
+                    max_overlap_idx = j;
                 }
+            }
         }
 
-	for (i = 0; i < new->cnt; i++) {
-		if (new->list[i].err > 0) {
-			/* Subvol might be marked for decommission
-			   with EINVAL, or some other serious error
-			   marked with positive errno.
-			*/
-			continue;
-                }
-
-                max_overlap = 0;
-                max_overlap_idx = i;
-                for (j = (i + 1); j < new->cnt; ++j) {
-                        if (new->list[j].err > 0) {
-			        /* Subvol might be marked for decommission
-			        with EINVAL, or some other serious error
-			        marked with positive errno.
-			        */
-			        continue;
-                        }
-                        /* Calculate the overlap now. */
-                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
-                        /* Calculate the overlap after the proposed swap. */
-                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
-                        /* Are we better than status quo? */
-                        if (overlap > curr_overlap) {
-                                overlap -= curr_overlap;
-                                /* Are we better than the previous choice? */
-                                if (overlap > max_overlap) {
-                                        max_overlap = overlap;
-                                        max_overlap_idx = j;
-                                }
-                        }
-                }
-
-		if (max_overlap_idx != i) {
- 			dht_layout_range_swap (new, i, max_overlap_idx);
-                        /* Need to swap the table values too. */
-                        for (j = 0; j < old->cnt; ++j) {
-                                overlap = OV_ENTRY(i,j);
-                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
-                                OV_ENTRY(max_overlap_idx,j) = overlap;
-                        }
-                }
-	}
+        if (max_overlap_idx != i) {
+            dht_layout_range_swap(new, i, max_overlap_idx);
+            /* Need to swap the table values too. */
+            for (j = 0; j < old->cnt; ++j) {
+                overlap = OV_ENTRY(i, j);
+                OV_ENTRY(i, j) = OV_ENTRY(max_overlap_idx, j);
+                OV_ENTRY(max_overlap_idx, j) = overlap;
+            }
+        }
+    }
 }
 
-
-dht_layout_t *
-dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
-                             dht_layout_t *layout)
+static dht_layout_t *
+dht_fix_layout_of_directory(call_frame_t *frame, loc_t *loc,
+                            dht_layout_t *layout)
 {
-        int           i            = 0;
-        xlator_t     *this         = NULL;
-        dht_layout_t *new_layout   = NULL;
-        dht_conf_t   *priv         = NULL;
-        dht_local_t  *local        = NULL;
-        uint32_t      subvol_down  = 0;
-        int           ret          = 0;
-
-        this  = frame->this;
-        priv  = this->private;
-        local = frame->local;
-
-        if (layout->type == DHT_HASH_TYPE_DM_USER) {
-                gf_msg_debug (THIS->name, 0, "leaving %s alone",
-                              loc->path);
-                goto done;
-        }
-
-        new_layout = dht_layout_new (this, priv->subvolume_cnt);
-        if (!new_layout)
-                goto done;
-
-        /* If a subvolume is down, do not re-write the layout. */
-        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
-                                    &subvol_down, NULL, NULL);
-
-        if (subvol_down || (ret == -1)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_FIX_FAILED,
-                        "Layout fix failed: %u subvolume(s) are down"
-                        ". Skipping fix layout.", subvol_down);
-                GF_FREE (new_layout);
-                return NULL;
-        }
-
-        for (i = 0; i < new_layout->cnt; i++) {
-		if (layout->list[i].err != ENOSPC)
-			new_layout->list[i].err = layout->list[i].err;
-		else
-			new_layout->list[i].err = -1;
-
-		new_layout->list[i].xlator = layout->list[i].xlator;
-        }
-
-        new_layout->commit_hash = layout->commit_hash;
-
-        if (priv->du_stats) {
-                for (i = 0; i < priv->subvolume_cnt; ++i) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_SUBVOL_INFO,
-                                "subvolume %d (%s): %u chunks", i,
-                                priv->subvolumes[i]->name,
-                                priv->du_stats[i].chunks);
-                }
-        }
-        else {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_NO_DISK_USAGE_STATUS, "no du stats ?!?");
-        }
-
-	/* First give it a layout as though it is a new directory. This
-	   ensures rotation to kick in */
-        dht_layout_sort_volname (new_layout);
-	dht_selfheal_layout_new_directory (frame, loc, new_layout);
-
-	/* Now selectively re-assign ranges only when it helps */
-	dht_selfheal_layout_maximize_overlap (frame, loc, new_layout, layout);
-
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_layout_t *new_layout = NULL;
+    dht_conf_t *priv = NULL;
+    dht_local_t *local = NULL;
+    uint32_t subvol_down = 0;
+    gf_boolean_t maximize_overlap = _gf_true;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    this = frame->this;
+    priv = this->private;
+    local = frame->local;
+
+    if (layout->type == DHT_HASH_TYPE_DM_USER) {
+        gf_msg_debug(THIS->name, 0, "leaving %s alone", loc->path);
+        goto done;
+    }
+
+    new_layout = dht_layout_new(this, priv->subvolume_cnt);
+    if (!new_layout) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "new_layout, path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto done;
+    }
+
+    /* If a subvolume is down, do not re-write the layout. */
+    dht_layout_anomalies(this, loc, layout, NULL, NULL, NULL, &subvol_down,
+                         NULL, NULL);
+
+    if (subvol_down) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                "subvol-down=%u", subvol_down, "Skipping-fix-layout", "path=%s",
+                loc->path, "gfid=%s", gfid, NULL);
+        GF_FREE(new_layout);
+        return NULL;
+    }
+
+    for (i = 0; i < new_layout->cnt; i++) {
+        if (layout->list[i].err != ENOSPC)
+            new_layout->list[i].err = layout->list[i].err;
+        else
+            new_layout->list[i].err = -1;
+
+        new_layout->list[i].xlator = layout->list[i].xlator;
+    }
+
+    new_layout->commit_hash = layout->commit_hash;
+
+    if (priv->du_stats) {
+        for (i = 0; i < priv->subvolume_cnt; ++i) {
+            gf_smsg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_SUBVOL_INFO,
+                    "index=%d", i, "name=%s", priv->subvolumes[i]->name,
+                    "chunks=%u", priv->du_stats[i].chunks, "path=%s", loc->path,
+                    NULL);
+
+            /* Maximize overlap if the bricks are all the same
+             *  size.
+             * This is probably not going to be very common on
+             * live setups but will benefit our regression tests
+             */
+            if (i && (priv->du_stats[i].chunks != priv->du_stats[0].chunks)) {
+                maximize_overlap = _gf_false;
+            }
+        }
+    } else {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_DISK_USAGE_STATUS,
+                NULL);
+    }
+
+    /* First give it a layout as though it is a new directory. This
+       ensures rotation to kick in */
+    dht_layout_sort_volname(new_layout);
+    dht_selfheal_layout_new_directory(frame, loc, new_layout);
+
+    /* Maximize overlap if weighted-rebalance is disabled */
+    if (!priv->do_weighting)
+        maximize_overlap = _gf_true;
+
+    /* Now selectively re-assign ranges only when it helps */
+    if (maximize_overlap) {
+        dht_selfheal_layout_maximize_overlap(frame, loc, new_layout, layout);
+    }
 done:
-        if (new_layout) {
-                /* Now that the new layout has all the proper layout, change the
-                   inode context */
-                dht_layout_set (this, loc->inode, new_layout);
+    if (new_layout) {
+        /* Make sure the extra 'ref' for existing layout is removed */
+        dht_layout_unref(this, local->layout);
 
-                /* Make sure the extra 'ref' for existing layout is removed */
-                dht_layout_unref (this, local->layout);
-
-                local->layout = new_layout;
-        }
+        local->layout = new_layout;
+    }
 
-        return local->layout;
+    return local->layout;
 }
 
-
 /*
  * Having to call this 2x for each entry in the layout is pretty horrible, but
  * that's what all of this layout-sorting nonsense gets us.
  */
-uint32_t
-dht_get_chunks_from_xl (xlator_t *parent, xlator_t *child)
+static uint32_t
+dht_get_chunks_from_xl(xlator_t *parent, xlator_t *child)
 {
-        dht_conf_t      *priv   = parent->private;
-        xlator_list_t   *trav;
-        uint32_t        index   = 0;
+    dht_conf_t *priv = parent->private;
+    xlator_list_t *trav;
+    uint32_t index = 0;
 
-        if (!priv->du_stats) {
-                return 0;
-        }
+    if (!priv->du_stats) {
+        return 0;
+    }
 
-        for (trav = parent->children; trav; trav = trav->next) {
-                if (trav->xlator == child) {
-                        return priv->du_stats[index].chunks;
-                }
-                ++index;
+    for (trav = parent->children; trav; trav = trav->next) {
+        if (trav->xlator == child) {
+            return priv->du_stats[index].chunks;
         }
+        ++index;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 void
-dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-                                   dht_layout_t *layout)
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *layout)
 {
-        xlator_t    *this = NULL;
-        double       chunk = 0;
-        int          i = 0;
-        uint32_t     start = 0;
-        int          bricks_to_use = 0;
-        int          err = 0;
-        int          start_subvol = 0;
-        uint32_t     curr_size;
-        uint32_t     range_size;
-        uint64_t     total_size = 0;
-        int          real_i;
-        dht_conf_t   *priv;
-        gf_boolean_t weight_by_size;
-        int          bricks_used = 0;
-
-        this = frame->this;
-        priv = this->private;
-        weight_by_size = priv->do_weighting;
-
-        bricks_to_use = dht_get_layout_count (this, layout, 1);
-        GF_ASSERT (bricks_to_use > 0);
-
-        bricks_used = 0;
-        for (i = 0; i < layout->cnt; ++i) {
-                err = layout->list[i].err;
-                if ((err != -1) && (err != ENOENT)) {
-                        continue;
-                }
-                curr_size = dht_get_chunks_from_xl (this,
-                                                    layout->list[i].xlator);
-                if (!curr_size) {
-                        weight_by_size = _gf_false;
-                        break;
-                }
-                total_size += curr_size;
-                if (++bricks_used >= bricks_to_use) {
-                        break;
-                }
-        }
-
-        if (weight_by_size && total_size) {
-                /* We know total_size is not zero. */
-                chunk = ((double) 0xffffffff) / ((double) total_size);
-                gf_msg_debug (this->name, 0,
-                              "chunk size = 0xffffffff / %lu = %f",
-                              total_size, chunk);
-        }
-        else {
-                weight_by_size = _gf_false;
-                chunk = ((unsigned long) 0xffffffff) / bricks_to_use;
+    xlator_t *this = NULL;
+    double chunk = 0;
+    int i = 0;
+    uint32_t start = 0;
+    int bricks_to_use = 0;
+    int err = 0;
+    int start_subvol = 0;
+    uint32_t curr_size;
+    uint32_t range_size;
+    uint64_t total_size = 0;
+    int real_i;
+    dht_conf_t *priv;
+    gf_boolean_t weight_by_size;
+    int bricks_used = 0;
+
+    this = frame->this;
+    priv = this->private;
+    weight_by_size = priv->do_weighting;
+
+    bricks_to_use = dht_get_layout_count(this, layout, 1);
+    GF_ASSERT(bricks_to_use > 0);
+
+    bricks_used = 0;
+    for (i = 0; i < layout->cnt; ++i) {
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
+        }
+        curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+        if (!curr_size) {
+            weight_by_size = _gf_false;
+            break;
+        }
+        total_size += curr_size;
+        if (++bricks_used >= bricks_to_use) {
+            break;
+        }
+    }
+
+    if (weight_by_size && total_size) {
+        /* We know total_size is not zero. */
+        chunk = ((double)0xffffffff) / ((double)total_size);
+        gf_msg_debug(this->name, 0,
+                     "chunk size = 0xffffffff / %" PRIu64 " = %f", total_size,
+                     chunk);
+    } else {
+        weight_by_size = _gf_false;
+        chunk = ((unsigned long)0xffffffff) / bricks_to_use;
+    }
+
+    start_subvol = dht_selfheal_layout_alloc_start(this, loc, layout);
+
+    /* clear out the range, as we are re-computing here */
+    DHT_RESET_LAYOUT_RANGE(layout);
+
+    /*
+     * OK, what's this "real_i" stuff about?  This used to be two loops -
+     * from start_subvol to layout->cnt-1, then from 0 to start_subvol-1.
+     * That way is practically an open invitation to bugs when only one
+     * of the loops is updated.  Using real_i and modulo operators to make
+     * it one loop avoids this problem.  Remember, folks: it's everyone's
+     * responsibility to help stamp out copy/paste abuse.
+     */
+    bricks_used = 0;
+    for (real_i = 0; real_i < layout->cnt; real_i++) {
+        i = (real_i + start_subvol) % layout->cnt;
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
+        }
+        if (weight_by_size) {
+            curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+            if (!curr_size) {
+                continue;
+            }
+        } else {
+            curr_size = 1;
         }
-
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
-
-        /* clear out the range, as we are re-computing here */
-        DHT_RESET_LAYOUT_RANGE (layout);
-
-        /*
-         * OK, what's this "real_i" stuff about?  This used to be two loops -
-         * from start_subvol to layout->cnt-1, then from 0 to start_subvol-1.
-         * That way is practically an open invitation to bugs when only one
-         * of the loops is updated.  Using real_i and modulo operators to make
-         * it one loop avoids this problem.  Remember, folks: it's everyone's
-         * responsibility to help stamp out copy/paste abuse.
-         */
-        bricks_used = 0;
-        for (real_i = 0; real_i < layout->cnt; real_i++) {
-                i = (real_i + start_subvol) % layout->cnt;
-                err = layout->list[i].err;
-                if ((err != -1) && (err != ENOENT)) {
-                        continue;
-                }
-                if (weight_by_size) {
-                        curr_size = dht_get_chunks_from_xl (this,
-                                layout->list[i].xlator);
-                        if (!curr_size) {
-                                continue;
-                        }
-                }
-                else {
-                        curr_size = 1;
-                }
-                range_size = chunk * curr_size;
-                gf_msg_debug (this->name, 0,
-                              "assigning range size 0x%x to %s",
-                              range_size,
-                              layout->list[i].xlator->name);
-                DHT_SET_LAYOUT_RANGE(layout, i, start, range_size,
-                                     loc->path);
-                if (++bricks_used >= bricks_to_use) {
-                        layout->list[i].stop = 0xffffffff;
-                        goto done;
-                }
-                start += range_size;
+        range_size = chunk * curr_size;
+        gf_msg_debug(this->name, 0, "assigning range size 0x%x to %s",
+                     range_size, layout->list[i].xlator->name);
+        DHT_SET_LAYOUT_RANGE(layout, i, start, range_size, loc->path);
+        if (++bricks_used >= bricks_to_use) {
+            layout->list[i].stop = 0xffffffff;
+            goto done;
         }
+        start += range_size;
+    }
 
 done:
-        return;
+    return;
 }
 
-int
-dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
-                          dht_layout_t *layout)
+static int
+dht_selfheal_dir_getafix(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
-        uint32_t     holes = 0;
-        int          ret = -1;
-        int          i = -1;
-        uint32_t     overlaps = 0;
-
-        local = frame->local;
-
-        holes = local->selfheal.hole_cnt;
-        overlaps = local->selfheal.overlaps_cnt;
-
-        if (holes || overlaps) {
-                /* If the layout has anomolies which would change the hash
-                 * ranges, then we need to reset the commit_hash for this
-                 * directory, as the layout would change and things may not
-                 * be in place as expected */
-                layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
+    dht_local_t *local = NULL;
+    uint32_t holes = 0;
+    int ret = -1;
+    int i = -1;
+    uint32_t overlaps = 0;
+
+    local = frame->local;
+
+    holes = local->selfheal.hole_cnt;
+    overlaps = local->selfheal.overlaps_cnt;
+
+    if (holes || overlaps) {
+        /* If the layout has anomalies which would change the hash
+         * ranges, then we need to reset the commit_hash for this
+         * directory, as the layout would change and things may not
+         * be in place as expected */
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+        dht_selfheal_layout_new_directory(frame, loc, layout);
+        ret = 0;
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                /* directory not present */
-                if (layout->list[i].err == ENOENT) {
-                        ret = 0;
-                        break;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        /* directory not present */
+        if (layout->list[i].err == ENOENT) {
+            ret = 0;
+            break;
         }
+    }
 
-        /* TODO: give a fix to these non-virgins */
+    /* TODO: give a fix to these non-virgins */
 
-        return ret;
+    return ret;
 }
 
 int
-dht_selfheal_new_directory (call_frame_t *frame,
-                            dht_selfheal_dir_cbk_t dir_cbk,
-                            dht_layout_t *layout)
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                           dht_layout_t *layout)
 {
-        dht_local_t *local                   = NULL;
-        int          ret                     = 0;
-        inode_t     *linked_inode            = NULL, *inode = NULL;
-        loc_t       *loc                     = NULL;
-        char         pgfid[GF_UUID_BUF_SIZE] = {0};
-        char         gfid[GF_UUID_BUF_SIZE]  = {0};
-        int32_t      op_errno                = EIO;
+    dht_local_t *local = NULL;
+    int ret = 0;
+    inode_t *linked_inode = NULL, *inode = NULL;
+    loc_t *loc = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = EIO;
 
-        local = frame->local;
+    local = frame->local;
 
-        loc = &local->loc;
+    loc = &local->loc;
 
-        gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
-        gf_uuid_unparse(loc->parent->gfid, pgfid);
+    gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
 
-        linked_inode = inode_link (loc->inode, loc->parent, loc->name,
-                                   &local->stbuf);
-        if (!linked_inode) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "linking inode failed (%s/%s) => %s",
-                        pgfid, loc->name, gfid);
-                ret = -1;
-                goto out;
-        }
+    linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                              &local->stbuf);
+    if (!linked_inode) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        inode = loc->inode;
-        loc->inode = linked_inode;
-        inode_unref (inode);
+    inode = loc->inode;
+    loc->inode = linked_inode;
+    inode_unref(inode);
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        dht_layout_sort_volname (layout);
-        dht_selfheal_layout_new_directory (frame, &local->loc, layout);
+    dht_layout_sort_volname(layout);
+    dht_selfheal_layout_new_directory(frame, &local->loc, layout);
 
-        op_errno = ENOMEM;
-        ret = dht_selfheal_layout_lock (frame, layout, _gf_true,
-                                        dht_selfheal_dir_xattr,
-                                        dht_should_heal_layout);
+    op_errno = ENOMEM;
+    ret = dht_selfheal_layout_lock(frame, layout, _gf_true,
+                                   dht_selfheal_dir_xattr,
+                                   dht_should_heal_layout);
 
 out:
-        if (ret < 0) {
-                dir_cbk (frame, NULL, frame->this, -1, op_errno, NULL);
-        }
+    if (ret < 0) {
+        dir_cbk(frame, NULL, frame->this, -1, op_errno, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-dht_fix_directory_layout (call_frame_t *frame,
-                          dht_selfheal_dir_cbk_t dir_cbk,
-                          dht_layout_t *layout)
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout)
 {
-        dht_local_t  *local      = NULL;
-        dht_layout_t *tmp_layout = NULL;
-        int           ret        = 0;
+    dht_local_t *local = NULL;
+    dht_layout_t *tmp_layout = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        /* No layout sorting required here */
-        tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
-        if (!tmp_layout) {
-                return -1;
-        }
+    /* No layout sorting required here */
+    tmp_layout = dht_fix_layout_of_directory(frame, &local->loc, layout);
+    if (!tmp_layout) {
+        return -1;
+    }
 
-        ret = dht_selfheal_layout_lock (frame, tmp_layout, _gf_false,
-                                        dht_fix_dir_xattr,
-                                        dht_should_fix_layout);
+    ret = dht_selfheal_layout_lock(frame, tmp_layout, _gf_false,
+                                   dht_fix_dir_xattr, dht_should_fix_layout);
 
-        return ret;
+    return ret;
 }
 
-
 int
-dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                        loc_t *loc, dht_layout_t *layout)
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                       loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local                   = NULL;
-        uint32_t     down                    = 0;
-        uint32_t     misc                    = 0;
-        int          ret                     = 0;
-        xlator_t    *this                    = NULL;
-        char         pgfid[GF_UUID_BUF_SIZE] = {0};
-        char         gfid[GF_UUID_BUF_SIZE]  = {0};
-        inode_t     *linked_inode            = NULL, *inode = NULL;
-
-        local = frame->local;
-        this = frame->this;
-
-        if (!__is_root_gfid (local->stbuf.ia_gfid)) {
-                gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
-                gf_uuid_unparse(loc->parent->gfid, pgfid);
-
-                linked_inode = inode_link (loc->inode, loc->parent, loc->name,
-                                           &local->stbuf);
-                if (!linked_inode) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DIR_SELFHEAL_FAILED,
-                                "linking inode failed (%s/%s) => %s",
-                                pgfid, loc->name, gfid);
-                        ret = 0;
-                        goto sorry_no_fix;
-                }
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    int ret = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    inode_t *linked_inode = NULL, *inode = NULL;
 
-                inode = loc->inode;
-                loc->inode = linked_inode;
-                inode_unref (inode);
-        }
+    local = frame->local;
+    this = frame->this;
 
-        dht_layout_anomalies (this, loc, layout,
-                              &local->selfheal.hole_cnt,
-                              &local->selfheal.overlaps_cnt,
-                              NULL, &local->selfheal.down,
-                              &local->selfheal.misc, NULL);
-
-        down     = local->selfheal.down;
-        misc     = local->selfheal.misc;
-
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (this, layout);
-
-        if (down) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: %d subvolumes down."
-                        "Not fixing. path = %s, gfid = %s",
-                        down, loc->path, gfid);
-                ret = 0;
-                goto sorry_no_fix;
-        }
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(this, layout);
 
-        if (misc) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed : %d subvolumes "
-                        "have unrecoverable errors. path = %s, gfid = %s",
-                        misc, loc->path, gfid);
+    if (local->need_attrheal) {
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            local->stbuf.ia_gid = local->prebuf.ia_gid;
+            local->stbuf.ia_uid = local->prebuf.ia_uid;
 
-                ret = 0;
-                goto sorry_no_fix;
-        }
+            local->stbuf.ia_ctime = local->prebuf.ia_ctime;
+            local->stbuf.ia_ctime_nsec = local->prebuf.ia_ctime_nsec;
+            local->stbuf.ia_prot = local->prebuf.ia_prot;
 
-        dht_layout_sort_volname (layout);
-        ret = dht_selfheal_dir_getafix (frame, loc, layout);
-
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: "
-                        "Unable to form layout for directory %s",
-                        loc->path);
-                goto sorry_no_fix;
+        } else if (!IA_ISINVAL(local->mds_stbuf.ia_type)) {
+            local->stbuf = local->mds_stbuf;
         }
+    }
 
-        dht_selfheal_dir_mkdir (frame, loc, layout, 0);
-
-        return 0;
-
-sorry_no_fix:
-        /* TODO: need to put appropriate local->op_errno */
-        dht_selfheal_dir_finish (frame, this, ret, 1);
-
-        return 0;
-}
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+        gf_uuid_unparse(loc->parent->gfid, pgfid);
 
-int
-dht_selfheal_directory_for_nameless_lookup (call_frame_t *frame,
-                                            dht_selfheal_dir_cbk_t dir_cbk,
-                                            loc_t *loc, dht_layout_t *layout)
-{
-        dht_local_t     *local  = NULL;
-        uint32_t        down    = 0;
-        uint32_t        misc    = 0;
-        int             ret     = 0;
-        xlator_t        *this   = NULL;
-
-        local = frame->local;
-        this = frame->this;
-        dht_layout_anomalies (this, loc, layout,
-                              &local->selfheal.hole_cnt,
-                              &local->selfheal.overlaps_cnt,
-                              NULL, &local->selfheal.down,
-                              &local->selfheal.misc, NULL);
-
-        down     = local->selfheal.down;
-        misc     = local->selfheal.misc;
-
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (this, layout);
-
-        if (down) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_SUBVOL_DOWN_ERROR,
-                        "%d subvolumes down -- not fixing", down);
-                ret = 0;
-                goto sorry_no_fix;
+        linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                                  &local->stbuf);
+        if (!linked_inode) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                    "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid,
+                    NULL);
+            ret = 0;
+            goto sorry_no_fix;
         }
 
-        if (misc) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_SUBVOL_ERROR,
-                        "%d subvolumes have unrecoverable errors", misc);
-                ret = 0;
-                goto sorry_no_fix;
-        }
+        inode = loc->inode;
+        loc->inode = linked_inode;
+        inode_unref(inode);
+    }
+
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    dht_layout_anomalies(this, loc, layout, &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt,
+                         &local->selfheal.missing_cnt, &local->selfheal.down,
+                         &local->selfheal.misc, NULL);
+
+    down = local->selfheal.down;
+    misc = local->selfheal.misc;
+
+    if (down) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "subvol-down=%d", down, "Not-fixing",
+                "gfid=%s", gfid, NULL);
+        ret = 0;
+        goto sorry_no_fix;
+    }
 
-        dht_layout_sort_volname (layout);
-        ret = dht_selfheal_dir_getafix (frame, loc, layout);
+    if (misc) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "misc=%d", misc, "unrecoverable-errors",
+                "gfid=%s", gfid, NULL);
 
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_FORM_FAILED,
-                        "not able to form layout for the directory");
-                goto sorry_no_fix;
-        }
+        ret = 0;
+        goto sorry_no_fix;
+    }
 
-        ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                     dht_selfheal_dir_xattr_for_nameless_lookup,
-                                        dht_should_heal_layout);
+    dht_layout_sort_volname(layout);
+    local->heal_layout = _gf_true;
 
-        if (ret < 0) {
-                goto sorry_no_fix;
-        }
+    /* Ignore return value as it can be inferred from result of
+     * dht_layout_anomalies
+     */
+    dht_selfheal_dir_getafix(frame, loc, layout);
 
-        return 0;
+    if (!(local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+          local->selfheal.missing_cnt)) {
+        local->heal_layout = _gf_false;
+    }
 
-sorry_no_fix:
-        /* TODO: need to put appropriate local->op_errno */
-        dht_selfheal_dir_finish (frame, this, ret, 1);
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 0);
+    if (ret < 0) {
+        ret = 0;
+        goto sorry_no_fix;
+    }
 
-        return 0;
+    return 0;
 
+sorry_no_fix:
+    /* TODO: need to put appropriate local->op_errno */
+    dht_selfheal_dir_finish(frame, this, ret, 1);
 
+    return 0;
 }
 
 int
-dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                      loc_t *loc, dht_layout_t *layout)
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                     loc_t *loc, dht_layout_t *layout)
 {
-        int          ret = 0;
-        dht_local_t *local    = NULL;
+    int ret = 0;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 1);
 
-        return ret;
+    return ret;
 }
 
 int
-dht_dir_attr_heal (void *data)
+dht_dir_heal_xattrs(void *data)
 {
-        call_frame_t    *frame = NULL;
-        dht_local_t     *local = NULL;
-        xlator_t        *subvol = NULL;
-        xlator_t        *this  = NULL;
-        dht_conf_t      *conf  = NULL;
-        int              call_cnt = 0;
-        int              ret   = -1;
-        int              i     = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-
-        GF_VALIDATE_OR_GOTO ("dht", data, out);
-
-        frame = data;
-        local = frame->local;
-        this = frame->this;
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", local, out);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO ("dht", conf, out);
-
-        call_cnt = conf->subvolume_cnt;
-
-        for (i = 0; i < call_cnt; i++) {
-                subvol = conf->subvolumes[i];
-                if (!subvol || (subvol == dht_first_up_subvol (this)))
-                        continue;
-                ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
-                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
-                                      NULL, NULL, NULL, NULL);
-                if (ret) {
-                        gf_uuid_unparse(local->loc.gfid, gfid);
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dict_t *user_xattr = NULL;
+    dict_t *internal_xattr = NULL;
+    dict_t *mds_xattr = NULL;
+    dict_t *xdata = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int uret = 0;
+    int uflag = 0;
+    int i = 0;
+    int xattr_hashed = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t allzero[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+    mds_subvol = local->mds_subvol;
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (!mds_subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if ((local->loc.inode && gf_uuid_is_null(local->loc.inode->gfid)) ||
+        gf_uuid_is_null(local->loc.gfid)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_NOT_PRESENT,
+                "skip-heal path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    internal_xattr = dict_new();
+    if (!internal_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+    xdata = dict_new();
+    if (!xdata) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    call_cnt = conf->subvolume_cnt;
+
+    user_xattr = dict_new();
+    if (!user_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    ret = syncop_listxattr(local->mds_subvol, &local->loc, &mds_xattr, NULL,
+                           NULL);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LIST_XATTRS_FAILED,
+                "path=%s", local->loc.path, "name=%s", local->mds_subvol->name,
+                NULL);
+    }
+
+    if (!mds_xattr)
+        goto out;
+
+    dht_dir_set_heal_xattr(this, local, user_xattr, mds_xattr, &uret, &uflag);
+
+    /* To set quota related xattr need to set GLUSTERFS_INTERNAL_FOP_KEY
+     * key value to 1
+     */
+    if (dict_get(user_xattr, QUOTA_LIMIT_KEY) ||
+        dict_get(user_xattr, QUOTA_LIMIT_OBJECTS_KEY)) {
+        ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", GLUSTERFS_INTERNAL_FOP_KEY, "path=%s",
+                    local->loc.path, NULL);
+            goto out;
+        }
+    }
+    if (uret <= 0 && !uflag)
+        goto out;
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (subvol == mds_subvol)
+            continue;
+        if (uret || uflag) {
+            /* Custom xattr heal is required - let posix handle it */
+            ret = dict_set_int8(xdata, "sync_backend_xattrs", _gf_true);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", local->loc.path, "key=%s",
+                        "sync_backend_xattrs", NULL);
+                goto out;
+            }
+
+            ret = syncop_setxattr(subvol, &local->loc, user_xattr, 0, xdata,
+                                  NULL);
+            if (ret) {
+                xattr_hashed = 1;
+                gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                        "set-user-xattr-failed path=%s", local->loc.path,
+                        "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
+            } else {
+                dict_del(xdata, "sync_backend_xattrs");
+            }
+        }
+    }
+    /* After heal all custom xattr reset internal MDS xattr to 0 */
+    if (!xattr_hashed) {
+        ret = dht_dict_set_array(internal_xattr, conf->mds_xattr_key, allzero,
+                                 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", conf->mds_xattr_key, "path=%s", local->loc.path,
+                    NULL);
+            goto out;
+        }
+        ret = syncop_setxattr(mds_subvol, &local->loc, internal_xattr, 0, NULL,
+                              NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", mds_subvol->name, "gfid=%s", gfid, NULL);
+        }
+    }
 
-                        gf_msg ("dht", GF_LOG_ERROR, -ret,
-                                DHT_MSG_DIR_ATTR_HEAL_FAILED,
-                                "Directory attr heal failed. Failed to set"
-                                " uid/gid on path %s on subvol %s, gfid = %s ",
-                                local->loc.path, subvol->name, gfid);
-                }
-        }
 out:
-        return 0;
+    if (user_xattr)
+        dict_unref(user_xattr);
+    if (mds_xattr)
+        dict_unref(mds_xattr);
+    if (internal_xattr)
+        dict_unref(internal_xattr);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
 }
 
 int
-dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        DHT_STACK_DESTROY (sync_frame);
-        return 0;
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
 }
 
-/* EXIT: dht_update_commit_hash_for_layout */
 int
-dht_update_commit_hash_for_layout_done (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xdata)
+dht_dir_attr_heal(void *data)
 {
-        dht_local_t  *local = NULL;
-
-        local = frame->local;
-
-        /* preserve oldest error */
-        if (op_ret && !local->op_ret) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO("dht", conf, out);
+
+    mds_subvol = local->mds_subvol;
+    call_cnt = conf->subvolume_cnt;
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid) && (!mds_subvol)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            DHT_MSG_MDS_DOWN_UNABLE_TO_SET, "path=%s",
+                            local->loc.path, "gfid=%s", gfid, NULL);
+                    goto out;
+                }
+            }
+        }
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (!subvol || subvol == mds_subvol)
+            continue;
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
+        } else {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->mds_stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
         }
 
-        DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                          local->op_errno, NULL);
+        if (ret) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-        return 0;
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_ATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
+        }
+    }
+out:
+    return 0;
 }
 
 int
-dht_update_commit_hash_for_layout_unlock (call_frame_t *frame, xlator_t *this)
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        dht_local_t  *local = NULL;
-        int ret = 0;
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
-        local = frame->local;
+/* EXIT: dht_update_commit_hash_for_layout */
+static int
+dht_update_commit_hash_for_layout_done(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
-        ret = dht_unlock_inodelk (frame, local->lock.locks,
-                                  local->lock.lk_count,
-                                  dht_update_commit_hash_for_layout_done);
-        if (ret < 0) {
-                /* preserve oldest error, just ... */
-                if (!local->op_ret) {
-                        local->op_errno = errno;
-                        local->op_ret = -1;
-                }
+    local = frame->local;
 
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Winding unlock failed: stale locks left on brick"
-                        " %s", local->loc.path);
+    /* preserve oldest error */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
 
-                dht_update_commit_hash_for_layout_done (frame, NULL, this,
-                                                        0, 0, NULL);
-        }
+    DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_update_commit_hash_for_layout_cbk (call_frame_t *frame, void *cookie,
-                                       xlator_t *this, int op_ret,
-                                       int op_errno, dict_t *xdata)
+static int
+dht_update_commit_hash_for_layout_unlock(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        /* store first failure, just because */
-        if (op_ret && !local->op_ret) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
+    ret = dht_unlock_inodelk(frame, local->lock[0].layout.my_layout.locks,
+                             local->lock[0].layout.my_layout.lk_count,
+                             dht_update_commit_hash_for_layout_done);
+    if (ret < 0) {
+        /* preserve oldest error, just ... */
+        if (!local->op_ret) {
+            local->op_errno = errno;
+            local->op_ret = -1;
         }
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_WIND_UNLOCK_FAILED,
+                "path=%s", local->loc.path, NULL);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_update_commit_hash_for_layout_unlock (frame, this);
-        }
+        dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_update_commit_hash_for_layout_resume (call_frame_t *frame, void *cookie,
-                                          xlator_t *this, int32_t op_ret,
-                                          int32_t op_errno, dict_t *xdata)
+static int
+dht_update_commit_hash_for_layout_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        int            count = 1, ret = -1, i = 0, j = 0;
-        dht_conf_t    *conf = NULL;
-        dht_layout_t  *layout = NULL;
-        int32_t       *disk_layout = NULL;
-        dict_t        **xattr = NULL;
-
-        local = frame->local;
-        conf = frame->this->private;
-        count = conf->local_subvols_cnt;
-        layout = local->layout;
-
-        if (op_ret < 0) {
-                goto err_done;
-        }
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-        /* We precreate the xattr list as we cannot change call count post the
-         * first wind as we may never continue from there. So we finish prep
-         * work before winding the setxattrs */
-        xattr = GF_CALLOC (count, sizeof (*xattr), gf_common_mt_char);
-        if (!xattr) {
-                local->op_errno = errno;
+    local = frame->local;
 
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory commit hash update failed:"
-                        " %s: Allocation failed", local->loc.path);
+    LOCK(&frame->lock);
+    /* store first failure, just because */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
+    UNLOCK(&frame->lock);
 
-                goto err;
-        }
+    this_call_cnt = dht_frame_return(frame);
 
-        for (i = 0; i < count; i++) {
-                /* find the layout index for the subvolume */
-                ret = dht_layout_index_for_subvol (layout,
-                                                   conf->local_subvols[i]);
-                if (ret < 0) {
-                        local->op_errno = ENOENT;
-
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory commit hash update failed:"
-                                " %s: (subvol %s) Failed to find disk layout",
-                                local->loc.path, conf->local_subvols[i]->name);
-
-                        goto err;
-                }
-                j = ret;
+    if (is_last_call(this_call_cnt)) {
+        dht_update_commit_hash_for_layout_unlock(frame, this);
+    }
 
-                /* update the commit hash for the layout */
-                layout->list[j].commit_hash = layout->commit_hash;
-
-                /* extract the current layout */
-                ret = dht_disk_layout_extract (this, layout, j, &disk_layout);
-                if (ret == -1) {
-                        local->op_errno = errno;
+    return 0;
+}
 
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory commit hash update failed:"
-                                " %s: (subvol %s) Failed to extract disk"
-                                " layout", local->loc.path,
-                                conf->local_subvols[i]->name);
+static int
+dht_update_commit_hash_for_layout_resume(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int32_t op_ret,
+                                         int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0, j = 0;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int32_t *disk_layout = NULL;
+    dict_t **xattr = NULL;
+
+    local = frame->local;
+    conf = frame->this->private;
+    count = conf->local_subvols_cnt;
+    layout = local->layout;
+
+    if (op_ret < 0) {
+        goto err_done;
+    }
+
+    /* We precreate the xattr list as we cannot change call count post the
+     * first wind as we may never continue from there. So we finish prep
+     * work before winding the setxattrs */
+    xattr = GF_CALLOC(count, sizeof(*xattr), gf_common_mt_char);
+    if (!xattr) {
+        local->op_errno = errno;
+
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_COMMIT_HASH_FAILED,
+                "allocation-failed path=%s", local->loc.path, NULL);
+
+        goto err;
+    }
+
+    for (i = 0; i < count; i++) {
+        /* find the layout index for the subvolume */
+        ret = dht_layout_index_for_subvol(layout, conf->local_subvols[i]);
+        if (ret < 0) {
+            local->op_errno = ENOENT;
 
-                        goto err;
-                }
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMMIT_HASH_FAILED,
+                    "path=%s", local->loc.path, "subvol=%s",
+                    conf->local_subvols[i]->name, "find-disk-layout-failed",
+                    NULL);
 
-                xattr[i] = get_new_dict ();
-                if (!xattr[i]) {
-                        local->op_errno = errno;
+            goto err;
+        }
+        j = ret;
 
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory commit hash update failed:"
-                                " %s: Allocation failed", local->loc.path);
+        /* update the commit hash for the layout */
+        layout->list[j].commit_hash = layout->commit_hash;
 
-                        goto err;
-                }
+        /* extract the current layout */
+        ret = dht_disk_layout_extract(this, layout, j, &disk_layout);
+        if (ret == -1) {
+            local->op_errno = errno;
 
-                ret = dict_set_bin (xattr[i], conf->xattr_name,
-                                    disk_layout, 4 * 4);
-                if (ret != 0) {
-                        local->op_errno = ENOMEM;
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", conf->local_subvols[i]->name,
+                    "extract-disk-layout-failed", NULL);
 
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory self heal xattr failed:"
-                                "%s: (subvol %s) Failed to set xattr"
-                                " dictionary,", local->loc.path,
-                                conf->local_subvols[i]->name);
+            goto err;
+        }
 
-                        GF_FREE (disk_layout);
+        xattr[i] = dict_new();
+        if (!xattr[i]) {
+            local->op_errno = errno;
 
-                        goto err;
-                }
-                disk_layout = NULL;
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s Allocation-failed",
+                    local->loc.path, NULL);
 
-                gf_msg_trace (this->name, 0,
-                              "setting commit hash %u on subvolume %s"
-                              " for %s", layout->list[j].commit_hash,
-                              conf->local_subvols[i]->name, local->loc.path);
+            goto err;
         }
 
-        /* wind the setting of the commit hash across the local subvols */
-        local->call_cnt = count;
-        local->op_ret = 0;
-        local->op_errno = 0;
-        for (i = 0; i < count; i++) {
-                dict_ref (xattr[i]);
+        ret = dict_set_bin(xattr[i], conf->xattr_name, disk_layout, 4 * 4);
+        if (ret != 0) {
+            local->op_errno = ENOMEM;
 
-                STACK_WIND (frame, dht_update_commit_hash_for_layout_cbk,
-                            conf->local_subvols[i],
-                            conf->local_subvols[i]->fops->setxattr,
-                            &local->loc, xattr[i], 0, NULL);
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s",
+                    local->loc.path, "subvol=%s", conf->local_subvols[i]->name,
+                    "set-xattr-failed", NULL);
 
-                dict_unref (xattr[i]);
+            goto err;
         }
+        disk_layout = NULL;
 
-        return 0;
+        gf_msg_trace(this->name, 0,
+                     "setting commit hash %u on subvolume %s"
+                     " for %s",
+                     layout->list[j].commit_hash, conf->local_subvols[i]->name,
+                     local->loc.path);
+    }
+
+    /* wind the setting of the commit hash across the local subvols */
+    local->call_cnt = count;
+    local->op_ret = 0;
+    local->op_errno = 0;
+    for (i = 0; i < count; i++) {
+        STACK_WIND(frame, dht_update_commit_hash_for_layout_cbk,
+                   conf->local_subvols[i],
+                   conf->local_subvols[i]->fops->setxattr, &local->loc,
+                   xattr[i], 0, NULL);
+    }
+    for (i = 0; i < count; i++)
+        dict_unref(xattr[i]);
+    GF_FREE(xattr);
+
+    return 0;
 err:
-        if (xattr) {
-                for (i = 0; i < count; i++) {
-                        if (xattr[i])
-                                dict_destroy (xattr[i]);
-                }
-
-                GF_FREE (xattr);
+    if (xattr) {
+        for (i = 0; i < count; i++) {
+            if (xattr[i])
+                dict_unref(xattr[i]);
         }
 
-        GF_FREE (disk_layout);
+        GF_FREE(xattr);
+    }
 
-        local->op_ret = -1;
+    GF_FREE(disk_layout);
 
-        dht_update_commit_hash_for_layout_unlock (frame, this);
+    local->op_ret = -1;
 
-        return 0;
+    dht_update_commit_hash_for_layout_unlock(frame, this);
+
+    return 0;
 err_done:
-        local->op_ret = -1;
+    local->op_ret = -1;
 
-        dht_update_commit_hash_for_layout_done (frame, NULL, this, 0, 0, NULL);
+    dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* ENTER: dht_update_commit_hash_for_layout (see EXIT above)
@@ -2504,54 +2549,52 @@ err_done:
  *  - Unlock and return.
  */
 int
-dht_update_commit_hash_for_layout (call_frame_t *frame)
+dht_update_commit_hash_for_layout(call_frame_t *frame)
 {
-        dht_local_t   *local = NULL;
-        int            count = 1, ret = -1, i = 0;
-        dht_lock_t   **lk_array = NULL;
-        dht_conf_t    *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
-
-        local = frame->local;
-        conf = frame->this->private;
-
-        if (!conf->defrag)
-                goto err;
-
-        count = conf->local_subvols_cnt;
-        lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                              gf_common_mt_char);
-        if (lk_array == NULL)
-                goto err;
-
-        for (i = 0; i < count; i++) {
-                lk_array[i] = dht_lock_new (frame->this,
-                                            conf->local_subvols[i],
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN);
-                if (lk_array[i] == NULL)
-                        goto err;
-        }
-
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
-
-        ret = dht_blocking_inodelk (frame, lk_array, count, FAIL_ON_ANY_ERROR,
-                                    dht_update_commit_hash_for_layout_resume);
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
+
+    local = frame->local;
+    conf = frame->this->private;
+
+    if (!conf->defrag)
+        goto err;
+
+    count = conf->local_subvols_cnt;
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+    if (lk_array == NULL)
+        goto err;
+
+    for (i = 0; i < count; i++) {
+        lk_array[i] = dht_lock_new(frame->this, conf->local_subvols[i],
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[i] == NULL)
+            goto err;
+    }
+
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_update_commit_hash_for_layout_resume);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
index 0fea1d58e58..bb72b0ffbb5 100644
--- a/xlators/cluster/dht/src/dht-shared.c
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -8,1080 +8,1097 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /* TODO: add NS locking */
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "dht-common.h"
 #include "dht-messages.h"
 
 #ifndef MAX
-#define MAX(a, b) (((a) > (b))?(a):(b))
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
-#define GF_DECIDE_DEFRAG_THROTTLE_COUNT(throttle_count, conf) {         \
-                                                                        \
-                pthread_mutex_lock (&conf->defrag->dfq_mutex);          \
-                                                                        \
-                if (!strcasecmp (conf->dthrottle, "lazy"))              \
-                        conf->defrag->recon_thread_count = 1;           \
-                                                                        \
-                throttle_count =                                        \
-                    MAX ((sysconf(_SC_NPROCESSORS_ONLN) - 4), 4);       \
-                                                                        \
-                if (!strcasecmp (conf->dthrottle, "normal"))            \
-                        conf->defrag->recon_thread_count =              \
-                                                 (throttle_count / 2);  \
-                                                                        \
-                if (!strcasecmp (conf->dthrottle, "aggressive"))        \
-                        conf->defrag->recon_thread_count =              \
-                                                 throttle_count;        \
-                                                                        \
-                pthread_mutex_unlock (&conf->defrag->dfq_mutex);        \
-        }                                                               \
-
 /* TODO:
    - use volumename in xattr instead of "dht"
    - use NS locks
    - handle all cases in self heal layout reconstruction
    - complete linkfile selfheal
 */
-struct volume_options options[];
-
-extern dht_methods_t dht_methods;
 
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+static void
+dht_layout_dump(dht_layout_t *layout, const char *prefix)
 {
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        if (!layout)
-                goto out;
-        if (!prefix)
-                goto out;
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        if (layout->type != IA_INVAL) {
-                gf_proc_dump_build_key(key, prefix, "inode type");
-                gf_proc_dump_write(key, "%d", layout->type);
-        }
-
-        if  (!IA_ISDIR (layout->type))
-                goto out;
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    if (!layout)
+        goto out;
+
+    gf_proc_dump_build_key(key, prefix, "cnt");
+    gf_proc_dump_write(key, "%d", layout->cnt);
+    gf_proc_dump_build_key(key, prefix, "preset");
+    gf_proc_dump_write(key, "%d", layout->preset);
+    gf_proc_dump_build_key(key, prefix, "gen");
+    gf_proc_dump_write(key, "%d", layout->gen);
+    if (layout->type != IA_INVAL) {
+        gf_proc_dump_build_key(key, prefix, "inode type");
+        gf_proc_dump_write(key, "%d", layout->type);
+    }
+
+    if (!IA_ISDIR(layout->type))
+        goto out;
+
+    for (i = 0; i < layout->cnt; i++) {
+        gf_proc_dump_build_key(key, prefix, "list[%d].err", i);
+        gf_proc_dump_write(key, "%d", layout->list[i].err);
+        gf_proc_dump_build_key(key, prefix, "list[%d].start", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].start);
+        gf_proc_dump_build_key(key, prefix, "list[%d].stop", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].stop);
+        if (layout->list[i].xlator) {
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.type", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->type);
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.name", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->name);
         }
+    }
 
 out:
-        return;
+    return;
 }
 
-
 int32_t
-dht_priv_dump (xlator_t *this)
+dht_priv_dump(xlator_t *this)
 {
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
-        if (!this)
-                goto out;
+    if (!this)
+        goto out;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        ret = TRY_LOCK(&conf->subvolume_lock);
-        if (ret != 0) {
-                return ret;
+    ret = TRY_LOCK(&conf->subvolume_lock);
+    if (ret != 0) {
+        return ret;
+    }
+
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+    gf_proc_dump_build_key(key_prefix, "xlator.cluster.dht", "%s.priv",
+                           this->name);
+    gf_proc_dump_write("subvol_cnt", "%d", conf->subvolume_cnt);
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        snprintf(key, sizeof(key), "subvolumes[%d]", i);
+        gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                           conf->subvolumes[i]->name);
+        if (conf->file_layouts && conf->file_layouts[i]) {
+            snprintf(key, sizeof(key), "file_layouts[%d]", i);
+            dht_layout_dump(conf->file_layouts[i], key);
         }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        if (conf->dir_layouts && conf->dir_layouts[i]) {
+            snprintf(key, sizeof(key), "dir_layouts[%d]", i);
+            dht_layout_dump(conf->dir_layouts[i], key);
+        }
+        if (conf->subvolume_status) {
+            snprintf(key, sizeof(key), "subvolume_status[%d]", i);
+            gf_proc_dump_write(key, "%d", (int)conf->subvolume_status[i]);
+        }
+    }
+
+    gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+    gf_proc_dump_write("gen", "%d", conf->gen);
+    gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+    gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+    gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+    gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+    gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+    gf_proc_dump_write("use-readdirp", "%d", conf->use_readdirp);
+
+    if (conf->du_stats && conf->subvolume_status) {
         for (i = 0; i < conf->subvolume_cnt; i++) {
-                snprintf (key, sizeof (key), "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        snprintf (key, sizeof (key), "file_layouts[%d]", i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        snprintf (key, sizeof (key), "dir_layouts[%d]", i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-
-                        snprintf (key, sizeof (key), "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                           (int)conf->subvolume_status[i]);
-                }
+            if (!conf->subvolume_status[i])
+                continue;
 
-        }
+            snprintf(key, sizeof(key), "subvolumes[%d]", i);
+            gf_proc_dump_write(key, "%s", conf->subvolumes[i]->name);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_percent", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_percent);
 
-        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
-        gf_proc_dump_write("gen", "%d", conf->gen);
-        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
-	gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
-        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
-        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
-        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
-        gf_proc_dump_write("use-readdirp", "%d", conf->use_readdirp);
-
-        if (conf->du_stats && conf->subvolume_status) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!conf->subvolume_status[i])
-                                continue;
-
-                        snprintf (key, sizeof (key), "subvolumes[%d]", i);
-                        gf_proc_dump_write (key, "%s",
-                                            conf->subvolumes[i]->name);
-
-                        snprintf (key, sizeof (key),
-                                  "du_stats[%d].avail_percent", i);
-                        gf_proc_dump_write (key, "%lf",
-                                            conf->du_stats[i].avail_percent);
-
-                        snprintf (key, sizeof (key), "du_stats[%d].avail_space",
-                                  i);
-                        gf_proc_dump_write (key, "%lu",
-                                            conf->du_stats[i].avail_space);
-
-                        snprintf (key, sizeof (key),
-                                  "du_stats[%d].avail_inodes", i);
-                        gf_proc_dump_write (key, "%lf",
-                                            conf->du_stats[i].avail_inodes);
-
-                        snprintf (key, sizeof (key), "du_stats[%d].log", i);
-                        gf_proc_dump_write (key, "%lu",
-                                            conf->du_stats[i].log);
-                }
+            snprintf(key, sizeof(key), "du_stats[%d].avail_space", i);
+            gf_proc_dump_write(key, "%" PRIu64, conf->du_stats[i].avail_space);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_inodes", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_inodes);
+
+            snprintf(key, sizeof(key), "du_stats[%d].log", i);
+            gf_proc_dump_write(key, "%" PRIu32, conf->du_stats[i].log);
         }
+    }
 
-        if (conf->last_stat_fetch.tv_sec)
-                gf_proc_dump_write("last_stat_fetch", "%s",
-                                    ctime(&conf->last_stat_fetch.tv_sec));
+    if (conf->last_stat_fetch)
+        gf_proc_dump_write("last_stat_fetch", "%s",
+                           ctime(&conf->last_stat_fetch));
 
-        UNLOCK(&conf->subvolume_lock);
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
+dht_inodectx_dump(xlator_t *this, inode_t *inode)
 {
-        int             ret = -1;
-        dht_layout_t    *layout = NULL;
+    int ret = -1;
+    dht_layout_t *layout = NULL;
 
-        if (!this)
-                goto out;
-        if (!inode)
-                goto out;
+    if (!this)
+        goto out;
+    if (!inode)
+        goto out;
 
-        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
 
-        if ((ret != 0) || !layout)
-                return ret;
+    if ((ret != 0) || !layout)
+        return ret;
 
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
-        dht_layout_dump(layout, "layout");
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+    dht_layout_dump(layout, "layout");
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-dht_fini (xlator_t *this)
+dht_fini(xlator_t *this)
 {
-        int         i = 0;
-        dht_conf_t *conf = NULL;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    conf = this->private;
+    this->private = NULL;
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
+        }
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        dict_unref(conf->leaf_to_subvol);
 
-        conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+        /* allocated in dht_init_subvolumes() */
+        GF_FREE(conf->subvolumes);
+        GF_FREE(conf->subvolume_status);
+        GF_FREE(conf->last_event);
+        GF_FREE(conf->subvol_up_time);
+        GF_FREE(conf->du_stats);
+        GF_FREE(conf->decommissioned_bricks);
 
-                dict_destroy(conf->leaf_to_subvol);
+        /* allocated in dht_init() */
+        GF_FREE(conf->mds_xattr_key);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->commithash_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
 
-                GF_FREE (conf->subvolumes);
+        /* allocated in dht_init_regex() */
+        if (conf->rsync_regex_valid)
+            regfree(&conf->rsync_regex);
+        if (conf->extra_regex_valid)
+            regfree(&conf->extra_regex);
 
-                GF_FREE (conf->subvolume_status);
+        synclock_destroy(&conf->link_lock);
 
-                if (conf->lock_pool)
-                        mem_pool_destroy (conf->lock_pool);
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
 
-                GF_FREE (conf);
-        }
+        GF_FREE(conf);
+    }
 out:
-        return;
+    return;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
 
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_dht_mt_end + 1);
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_NO_MEMORY,
-                        "Memory accounting init failed");
-                return ret;
-        }
-out:
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Memory accounting init failed");
         return ret;
+    }
+out:
+    return ret;
 }
 
-
-int
-dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
-                                 const char *bricks)
+static int
+dht_parse_decommissioned_bricks(xlator_t *this, dht_conf_t *conf,
+                                const char *bricks)
 {
-        int         i  = 0;
-        int         ret  = -1;
-        char       *tmpstr = NULL;
-        char       *dup_brick = NULL;
-        char       *node = NULL;
-
-        if (!conf || !bricks)
-                goto out;
-
-        dup_brick = gf_strdup (bricks);
-        node = strtok_r (dup_brick, ",", &tmpstr);
-        while (node) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!strcmp (conf->subvolumes[i]->name, node)) {
-                                conf->decommissioned_bricks[i] =
-                                        conf->subvolumes[i];
-                                        conf->decommission_subvols_cnt++;
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_SUBVOL_DECOMMISSION_INFO,
-                                        "decommissioning subvolume %s",
-                                        conf->subvolumes[i]->name);
-                                break;
-                        }
-                }
-                if (i == conf->subvolume_cnt) {
-                        /* Wrong node given. */
-                        goto out;
-                }
-                node = strtok_r (NULL, ",", &tmpstr);
+    int i = 0;
+    int ret = -1;
+    char *tmpstr = NULL;
+    char *dup_brick = NULL;
+    char *node = NULL;
+
+    if (!conf || !bricks)
+        goto out;
+
+    dup_brick = gf_strdup(bricks);
+    if (dup_brick == NULL) {
+        goto out;
+    }
+
+    node = strtok_r(dup_brick, ",", &tmpstr);
+    while (node) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (!strcmp(conf->subvolumes[i]->name, node)) {
+                conf->decommissioned_bricks[i] = conf->subvolumes[i];
+                conf->decommission_subvols_cnt++;
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       DHT_MSG_SUBVOL_DECOMMISSION_INFO,
+                       "decommissioning subvolume %s",
+                       conf->subvolumes[i]->name);
+                break;
+            }
         }
+        if (i == conf->subvolume_cnt) {
+            /* Wrong node given. */
+            goto out;
+        }
+        node = strtok_r(NULL, ",", &tmpstr);
+    }
 
-        ret = 0;
-        conf->decommission_in_progress = 1;
+    ret = 0;
+    conf->decommission_in_progress = 1;
 out:
-        GF_FREE (dup_brick);
+    GF_FREE(dup_brick);
 
-        return ret;
+    return ret;
 }
 
-int
-dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+static void
+dht_decommissioned_remove(xlator_t *this, dht_conf_t *conf)
 {
-        int         i  = 0;
-        int         ret  = -1;
-
-        if (!conf)
-                goto out;
+    int i = 0;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->decommissioned_bricks[i]) {
-                        conf->decommissioned_bricks[i] = NULL;
-                        conf->decommission_subvols_cnt--;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->decommissioned_bricks[i]) {
+            conf->decommissioned_bricks[i] = NULL;
+            conf->decommission_subvols_cnt--;
         }
-
-        ret = 0;
-out:
-
-        return ret;
+    }
 }
-void
-dht_init_regex (xlator_t *this, dict_t *odict, char *name,
-                regex_t *re, gf_boolean_t *re_valid)
+
+static void
+dht_init_regex(xlator_t *this, dict_t *odict, char *name, regex_t *re,
+               gf_boolean_t *re_valid, dht_conf_t *conf)
 {
-        char    *temp_str;
+    char *temp_str = NULL;
 
-        if (dict_get_str (odict, name, &temp_str) != 0) {
-                if (strcmp(name,"rsync-hash-regex")) {
-                        return;
-                }
-                temp_str = "^\\.(.+)\\.[^.]+$";
+    if (dict_get_str(odict, name, &temp_str) != 0) {
+        if (strcmp(name, "rsync-hash-regex")) {
+            return;
         }
+        temp_str = "^\\.(.+)\\.[^.]+$";
+    }
 
+    LOCK(&conf->lock);
+    {
         if (*re_valid) {
-                regfree(re);
-                *re_valid = _gf_false;
+            regfree(re);
+            *re_valid = _gf_false;
         }
 
-        if (!strcmp(temp_str,"none")) {
-                return;
+        if (!strcmp(temp_str, "none")) {
+            goto unlock;
         }
 
-        if (regcomp(re,temp_str,REG_EXTENDED) == 0) {
-                gf_msg_debug (this->name, 0,
-                              "using regex %s = %s", name, temp_str);
-                *re_valid = _gf_true;
-        }
-        else {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_REGEX_INFO,
-                        "compiling regex %s failed", temp_str);
+        if (regcomp(re, temp_str, REG_EXTENDED) == 0) {
+            gf_msg_debug(this->name, 0, "using regex %s = %s", name, temp_str);
+            *re_valid = _gf_true;
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REGEX_INFO,
+                   "compiling regex %s failed", temp_str);
         }
+    }
+unlock:
+    UNLOCK(&conf->lock);
 }
 
 int
 dht_set_subvol_range(xlator_t *this)
 {
-        int ret = -1;
-        dht_conf_t *conf = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf)
-                goto out;
+    if (!conf)
+        goto out;
 
-        conf->leaf_to_subvol = dict_new();
-        if (!conf->leaf_to_subvol)
-                goto out;
+    conf->leaf_to_subvol = dict_new();
+    if (!conf->leaf_to_subvol)
+        goto out;
 
-        ret = glusterfs_reachable_leaves(this, conf->leaf_to_subvol);
+    ret = glusterfs_reachable_leaves(this, conf->leaf_to_subvol);
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_reconfigure (xlator_t *this, dict_t *options)
+static int
+dht_configure_throttle(xlator_t *this, dht_conf_t *conf, char *temp_str)
 {
-        dht_conf_t      *conf = NULL;
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = -1;
-        int              throttle_count = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", options, out);
-
-        conf = this->private;
-        if (!conf)
-                return 0;
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_msg_debug(this->name, 0, "Reconfigure: "
-                                             "lookup-unhashed reconfigured(%s)",
-                                             temp_str);
-                                conf->search_unhashed = search_unhashed;
-                        } else {
-                                gf_msg(this->name, GF_LOG_ERROR, 0,
-                                       DHT_MSG_INVALID_OPTION,
-                                       "Invalid option: Reconfigure: "
-                                       "lookup-unhashed should be boolean,"
-                                       " not (%s), defaulting to (%d)",
-                                       temp_str, conf->search_unhashed);
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        gf_msg_debug(this->name, 0, "Reconfigure:"
-                                     " lookup-unhashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
+    int rebal_thread_count = 0;
+    int ret = 0;
+
+    pthread_mutex_lock(&conf->defrag->dfq_mutex);
+    {
+        if (!strcasecmp(temp_str, "lazy")) {
+            conf->defrag->recon_thread_count = 1;
+        } else if (!strcasecmp(temp_str, "normal")) {
+            conf->defrag->recon_thread_count = 2;
+        } else if (!strcasecmp(temp_str, "aggressive")) {
+            conf->defrag->recon_thread_count = MAX(MAX_REBAL_THREADS - 4, 4);
+        } else if ((gf_string2int(temp_str, &rebal_thread_count) == 0)) {
+            if ((rebal_thread_count > 0) &&
+                (rebal_thread_count <= MAX_REBAL_THREADS)) {
+                conf->defrag->recon_thread_count = rebal_thread_count;
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                       "rebal thread count configured to %d",
+                       rebal_thread_count);
+                goto out;
+            } else {
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "rebal-throttle should be "
+                       "within range of 0 and maximum number of"
+                       " cores available");
+                ret = -1;
+                goto out;
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option: Reconfigure: "
+                   "rebal-throttle should be {lazy|normal|aggressive}"
+                   " or a number up to the number of cores available,"
+                   " not (%s), defaulting to (%d)",
+                   temp_str, conf->dthrottle);
+            ret = -1;
         }
+    }
+    pthread_mutex_unlock(&conf->defrag->dfq_mutex);
 
-        GF_OPTION_RECONF ("lookup-optimize", conf->lookup_optimize, options,
-                          bool, out);
-
-	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
-                          percent_or_size, out);
-        /* option can be any one of percent or bytes */
-        conf->disk_unit = 0;
-        if (conf->min_free_disk < 100.0)
-                conf->disk_unit = 'p';
+out:
+    return ret;
+}
 
-	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
-                          percent, out);
+int
+dht_reconfigure(xlator_t *this, dict_t *options)
+{
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    gf_boolean_t search_unhashed;
+    int ret = -1;
 
-        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
-                          options, uint32, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", options, out);
 
-        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
-                          bool, out);
-        GF_OPTION_RECONF ("randomize-hash-range-by-gfid",
-                          conf->randomize_by_gfid,
-                          options, bool, out);
+    conf = this->private;
+    if (!conf)
+        return 0;
 
-        GF_OPTION_RECONF ("rebal-throttle", conf->dthrottle, options,
-                          str, out);
+    if (dict_get_str(options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean*/
+        if (strcasecmp(temp_str, "auto")) {
+            if (!gf_string2boolean(temp_str, &search_unhashed)) {
+                gf_msg_debug(this->name, 0,
+                             "Reconfigure: "
+                             "lookup-unhashed reconfigured(%s)",
+                             temp_str);
+                conf->search_unhashed = search_unhashed;
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "lookup-unhashed should be boolean,"
+                       " not (%s), defaulting to (%d)",
+                       temp_str, conf->search_unhashed);
+                ret = -1;
+                goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Reconfigure:"
+                         " lookup-unhashed reconfigured auto ");
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+    }
 
-        GF_OPTION_RECONF ("lock-migration", conf->lock_migration_enabled,
-                          options, bool, out);
+    GF_OPTION_RECONF("lookup-optimize", conf->lookup_optimize, options, bool,
+                     out);
 
-        if (conf->defrag) {
-                conf->defrag->lock_migration_enabled =
-                                        conf->lock_migration_enabled;
+    GF_OPTION_RECONF("min-free-disk", conf->min_free_disk, options,
+                     percent_or_size, out);
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100.0)
+        conf->disk_unit = 'p';
 
-                GF_DECIDE_DEFRAG_THROTTLE_COUNT (throttle_count, conf);
-                gf_msg ("DHT", GF_LOG_INFO, 0,
-                        DHT_MSG_REBAL_THROTTLE_INFO,
-                        "conf->dthrottle: %s, "
-                        "conf->defrag->recon_thread_count: %d",
-                         conf->dthrottle, conf->defrag->recon_thread_count);
-        }
+    GF_OPTION_RECONF("min-free-inodes", conf->min_free_inodes, options, percent,
+                     out);
 
-        if (conf->defrag) {
-                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
-                                  options, bool, out);
-        }
+    GF_OPTION_RECONF("directory-layout-spread", conf->dir_spread_cnt, options,
+                     uint32, out);
 
-        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto out;
-        } else {
-                ret = dht_decommissioned_remove (this, conf);
-                if (ret == -1)
-                        goto out;
-        }
+    GF_OPTION_RECONF("readdir-optimize", conf->readdir_optimize, options, bool,
+                     out);
+    GF_OPTION_RECONF("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                     options, bool, out);
 
-        dht_init_regex (this, options, "rsync-hash-regex",
-                        &conf->rsync_regex, &conf->rsync_regex_valid);
-        dht_init_regex (this, options, "extra-hash-regex",
-                        &conf->extra_regex, &conf->extra_regex_valid);
+    GF_OPTION_RECONF("lock-migration", conf->lock_migration_enabled, options,
+                     bool, out);
 
-        GF_OPTION_RECONF ("weighted-rebalance", conf->do_weighting, options,
-                          bool, out);
+    GF_OPTION_RECONF("force-migration", conf->force_migration, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("use-readdirp", conf->use_readdirp, options,
-                          bool, out);
-        ret = 0;
+    if (conf->defrag) {
+        if (dict_get_str(options, "rebal-throttle", &temp_str) == 0) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
+                goto out;
+        }
+    }
+
+    if (conf->defrag) {
+        conf->defrag->lock_migration_enabled = conf->lock_migration_enabled;
+    }
+
+    if (conf->defrag) {
+        GF_OPTION_RECONF("rebalance-stats", conf->defrag->stats, options, bool,
+                         out);
+    }
+
+    if (dict_get_str(options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto out;
+    } else {
+        dht_decommissioned_remove(this, conf);
+    }
+
+    dht_init_regex(this, options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    GF_OPTION_RECONF("weighted-rebalance", conf->do_weighting, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("use-readdirp", conf->use_readdirp, options, bool, out);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
+gf_defrag_pattern_list_fill(xlator_t *this, gf_defrag_info_t *defrag,
+                            char *data)
 {
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *num = NULL;
-        char                  *pattern_str = NULL;
-        char                  *pattern = NULL;
-        gf_defrag_pattern_list_t *temp_list = NULL;
-        gf_defrag_pattern_list_t *pattern_list = NULL;
-
-        if (!this || !defrag || !data)
-                goto out;
-
-        /* Get the pattern for pattern list. "pattern:<optional-size>"
-         * eg: *avi, *pdf:10MB, *:1TB
-         */
-        pattern_str = strtok_r (data, ",", &tmp_str);
-        while (pattern_str) {
-                dup_str = gf_strdup (pattern_str);
-                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
-                                        1);
-                if (!pattern_list) {
-                        goto out;
-                }
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!pattern)
-                        goto out;
-                if (!num) {
-                        if (gf_string2bytesize_uint64(pattern, &pattern_list->size)
-                             == 0) {
-                                pattern = "*";
-                        }
-                } else if (gf_string2bytesize_uint64 (num, &pattern_list->size) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_OPTION,
-                                "Invalid option. Defrag pattern:"
-                                " Invalid number format \"%s\"", num);
-                        goto out;
-                }
-                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
-
-                if (!defrag->defrag_pattern)
-                        temp_list = NULL;
-                else
-                        temp_list = defrag->defrag_pattern;
-
-                pattern_list->next = temp_list;
-
-                defrag->defrag_pattern = pattern_list;
-                pattern_list = NULL;
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                pattern_str = strtok_r (NULL, ",", &tmp_str);
+    int ret = -1;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *dup_str = NULL;
+    char *num = NULL;
+    char *pattern_str = NULL;
+    char *pattern = NULL;
+    gf_defrag_pattern_list_t *temp_list = NULL;
+    gf_defrag_pattern_list_t *pattern_list = NULL;
+
+    if (!this || !defrag || !data)
+        goto out;
+
+    /* Get the pattern for pattern list. "pattern:<optional-size>"
+     * eg: *avi, *pdf:10MB, *:1TB
+     */
+    pattern_str = strtok_r(data, ",", &tmp_str);
+    while (pattern_str) {
+        dup_str = gf_strdup(pattern_str);
+        if (!dup_str)
+            goto out;
+        pattern_list = GF_CALLOC(1, sizeof(gf_defrag_pattern_list_t), 1);
+        if (!pattern_list) {
+            goto out;
         }
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        num = strtok_r(NULL, ":", &tmp_str1);
+        if (!pattern)
+            goto out;
+        if (!num) {
+            if (gf_string2bytesize_uint64(pattern, &pattern_list->size) == 0) {
+                pattern = "*";
+            }
+        } else if (gf_string2bytesize_uint64(num, &pattern_list->size) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option. Defrag pattern:"
+                   " Invalid number format \"%s\"",
+                   num);
+            goto out;
+        }
+        memcpy(pattern_list->path_pattern, pattern, strlen(dup_str));
 
-        ret = 0;
-out:
-        if (ret)
-                GF_FREE (pattern_list);
-        GF_FREE (dup_str);
-
-        return ret;
-}
-
+        if (!defrag->defrag_pattern)
+            temp_list = NULL;
+        else
+            temp_list = defrag->defrag_pattern;
 
+        pattern_list->next = temp_list;
 
-int
-dht_init_methods (xlator_t *this)
-{
-        int ret                  = -1;
-        dht_conf_t      *conf    = NULL;
-        dht_methods_t   *methods = NULL;
+        defrag->defrag_pattern = pattern_list;
+        pattern_list = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
+        GF_FREE(dup_str);
+        dup_str = NULL;
 
-        conf = this->private;
-        methods = &(conf->methods);
+        pattern_str = strtok_r(NULL, ",", &tmp_str);
+    }
 
-        methods->migration_get_dst_subvol = dht_migration_get_dst_subvol;
-        methods->migration_needed = dht_migration_needed;
-        methods->migration_other  = NULL;
-        methods->layout_search    = dht_layout_search;
+    ret = 0;
+out:
+    if (ret)
+        GF_FREE(pattern_list);
+    GF_FREE(dup_str);
 
-        ret = 0;
-err:
-        return ret;
+    return ret;
 }
 
-int
-dht_init (xlator_t *this)
+static int
+dht_init_methods(xlator_t *this)
 {
-        dht_conf_t                      *conf           = NULL;
-        char                            *temp_str       = NULL;
-        int                              ret            = -1;
-        int                              i              = 0;
-        gf_defrag_info_t                *defrag         = NULL;
-        int                              cmd            = 0;
-        char                            *node_uuid      = NULL;
-        int                              throttle_count = 0;
-        uint32_t                         commit_hash    = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (!this->children) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_INVALID_CONFIGURATION,
-                        "Distribute needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_INVALID_CONFIGURATION,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        /* We get the commit-hash to set only for rebalance process */
-        if (dict_get_uint32 (this->options,
-                             "commit-hash", &commit_hash) == 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_COMMIT_HASH_INFO, "%s using commit hash %u",
-                        __func__, commit_hash);
-                conf->vol_commit_hash = commit_hash;
-                conf->vch_forced = _gf_true;
-        }
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
 
-        if (cmd) {
-                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
-                                    gf_defrag_info_mt);
+    conf = this->private;
+    methods = &(conf->methods);
 
-                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
+    methods->migration_get_dst_subvol = dht_migration_get_dst_subvol;
+    methods->migration_other = NULL;
+    methods->layout_search = dht_layout_search;
 
-                LOCK_INIT (&defrag->lock);
-
-                defrag->is_exiting = 0;
-
-                conf->defrag = defrag;
+    ret = 0;
+err:
+    return ret;
+}
 
-                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_CONFIGURATION,
-                                "Invalid volume configuration: "
-                                "node-uuid not specified");
-                        goto err;
-                }
+int
+dht_init(xlator_t *this)
+{
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    int ret = -1;
+    int i = 0;
+    gf_defrag_info_t *defrag = NULL;
+    int cmd = 0;
+    char *node_uuid = NULL;
+    uint32_t commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "Distribute needs more than one subvolume");
+        return -1;
+    }
 
-                if (gf_uuid_parse (node_uuid, defrag->node_uuid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_OPTION, "Invalid option:"
-                                " Cannot parse glusterd node uuid");
-                        goto err;
-                }
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "dangling volume. check volfile");
+    }
 
-                defrag->cmd = cmd;
+    conf = GF_CALLOC(1, sizeof(*conf), gf_dht_mt_dht_conf_t);
+    if (!conf) {
+        goto err;
+    }
 
-                defrag->stats = _gf_false;
+    LOCK_INIT(&conf->subvolume_lock);
+    LOCK_INIT(&conf->layout_lock);
+    LOCK_INIT(&conf->lock);
+    synclock_init(&conf->link_lock, SYNC_LOCK_DEFAULT);
 
-                defrag->queue = NULL;
+    /* We get the commit-hash to set only for rebalance process */
+    if (dict_get_uint32(this->options, "commit-hash", &commit_hash) == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_COMMIT_HASH_INFO,
+               "%s using commit hash %u", __func__, commit_hash);
+        conf->vol_commit_hash = commit_hash;
+        conf->vch_forced = _gf_true;
+    }
 
-                defrag->crawl_done = 0;
+    ret = dict_get_int32(this->options, "rebalance-cmd", &cmd);
 
-                defrag->global_error = 0;
+    if (cmd) {
+        defrag = GF_CALLOC(1, sizeof(gf_defrag_info_t), gf_defrag_info_mt);
 
-                defrag->q_entry_count = 0;
+        GF_VALIDATE_OR_GOTO(this->name, defrag, err);
 
-                defrag->wakeup_crawler = 0;
+        LOCK_INIT(&defrag->lock);
 
-                synclock_init (&defrag->link_lock, SYNC_LOCK_DEFAULT);
-                pthread_mutex_init (&defrag->dfq_mutex, 0);
-                pthread_cond_init  (&defrag->parallel_migration_cond, 0);
-                pthread_cond_init  (&defrag->rebalance_crawler_alarm, 0);
-                pthread_cond_init  (&defrag->df_wakeup_thread, 0);
+        defrag->is_exiting = 0;
 
-                defrag->global_error = 0;
+        conf->defrag = defrag;
+        defrag->this = this;
 
+        ret = dict_get_str(this->options, "node-uuid", &node_uuid);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_CONFIGURATION,
+                   "Invalid volume configuration: "
+                   "node-uuid not specified");
+            goto err;
         }
 
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto")) {
-                        ret = gf_string2boolean (temp_str,
-                                                 &conf->search_unhashed);
-                        if (ret == -1)
-                                goto err;
-                }
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        if (gf_uuid_parse(node_uuid, defrag->node_uuid)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option:"
+                   " Cannot parse glusterd node uuid");
+            goto err;
         }
 
-        GF_OPTION_INIT ("lookup-optimize", conf->lookup_optimize, bool, err);
+        defrag->cmd = cmd;
 
-        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
-                        err);
+        defrag->stats = _gf_false;
 
-        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+        defrag->queue = NULL;
 
-	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
-			err);
+        defrag->crawl_done = 0;
 
-        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
-                        err);
+        defrag->global_error = 0;
 
-        conf->dir_spread_cnt = conf->subvolume_cnt;
-        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
-                        uint32, err);
+        defrag->q_entry_count = 0;
 
-        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
-                        bool, err);
+        defrag->wakeup_crawler = 0;
 
-        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+        pthread_mutex_init(&defrag->dfq_mutex, 0);
+        pthread_cond_init(&defrag->parallel_migration_cond, 0);
+        pthread_cond_init(&defrag->rebalance_crawler_alarm, 0);
+        pthread_cond_init(&defrag->df_wakeup_thread, 0);
 
+        pthread_mutex_init(&defrag->fc_mutex, 0);
+        pthread_cond_init(&defrag->fc_wakeup_cond, 0);
 
-        GF_OPTION_INIT ("lock-migration", conf->lock_migration_enabled,
-                         bool, err);
+        defrag->global_error = 0;
+    }
 
-        if (defrag) {
-              defrag->lock_migration_enabled = conf->lock_migration_enabled;
+    conf->use_fallocate = 1;
 
-              GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
-                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
-                    == 0) {
-                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
-                            == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_INVALID_OPTION,
-                                        "Invalid option:"
-                                        " Cannot parse rebalance-filter (%s)",
-                                        temp_str);
-
-                                goto err;
-                        }
-                }
+    conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+    if (dict_get_str(this->options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean */
+        if (strcasecmp(temp_str, "auto")) {
+            gf_boolean_t search_unhashed_bool;
+            ret = gf_string2boolean(temp_str, &search_unhashed_bool);
+            if (ret == -1) {
+                goto err;
+            }
+            conf->search_unhashed = search_unhashed_bool
+                                        ? GF_DHT_LOOKUP_UNHASHED_ON
+                                        : GF_DHT_LOOKUP_UNHASHED_OFF;
+        } else {
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
         }
+    }
 
-        /* option can be any one of percent or bytes */
-        conf->disk_unit = 0;
-        if (conf->min_free_disk < 100)
-                conf->disk_unit = 'p';
+    GF_OPTION_INIT("lookup-optimize", conf->lookup_optimize, bool, err);
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+    GF_OPTION_INIT("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool, err);
 
-        if (cmd) {
-                ret = dht_init_local_subvolumes (this, conf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INIT_LOCAL_SUBVOL_FAILED,
-                                "dht_init_local_subvolumes failed");
-                        goto err;
-                }
-        }
+    GF_OPTION_INIT("use-readdirp", conf->use_readdirp, bool, err);
 
-        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto err;
-        }
+    GF_OPTION_INIT("min-free-disk", conf->min_free_disk, percent_or_size, err);
 
-        dht_init_regex (this, this->options, "rsync-hash-regex",
-                        &conf->rsync_regex, &conf->rsync_regex_valid);
-        dht_init_regex (this, this->options, "extra-hash-regex",
-                        &conf->extra_regex, &conf->extra_regex_valid);
+    GF_OPTION_INIT("min-free-inodes", conf->min_free_inodes, percent, err);
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+    conf->dir_spread_cnt = conf->subvolume_cnt;
+    GF_OPTION_INIT("directory-layout-spread", conf->dir_spread_cnt, uint32,
+                   err);
 
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
+    GF_OPTION_INIT("assert-no-child-down", conf->assert_no_child_down, bool,
+                   err);
 
-        conf->gen = 1;
+    GF_OPTION_INIT("readdir-optimize", conf->readdir_optimize, bool, err);
 
-        this->local_pool = mem_pool_new (dht_local_t, 512);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY,
-                        " DHT initialisation failed. "
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
+    GF_OPTION_INIT("lock-migration", conf->lock_migration_enabled, bool, err);
 
-        GF_OPTION_INIT ("randomize-hash-range-by-gfid",
-                        conf->randomize_by_gfid, bool, err);
+    GF_OPTION_INIT("force-migration", conf->force_migration, bool, err);
 
-        if (defrag) {
-                GF_OPTION_INIT ("rebal-throttle",
-                                 conf->dthrottle, str, err);
+    if (defrag) {
+        defrag->lock_migration_enabled = conf->lock_migration_enabled;
 
-                GF_DECIDE_DEFRAG_THROTTLE_COUNT(throttle_count, conf);
+        GF_OPTION_INIT("rebalance-stats", defrag->stats, bool, err);
+        if (dict_get_str(this->options, "rebalance-filter", &temp_str) == 0) {
+            if (gf_defrag_pattern_list_fill(this, defrag, temp_str) == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option:"
+                       " Cannot parse rebalance-filter (%s)",
+                       temp_str);
 
-                gf_msg_debug ("DHT", 0, "conf->dthrottle: %s, "
-                              "conf->defrag->recon_thread_count: %d",
-                              conf->dthrottle,
-                              conf->defrag->recon_thread_count);
+                goto err;
+            }
         }
-
-        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
-        gf_asprintf (&conf->link_xattr_name, "%s."DHT_LINKFILE_STR,
-                     conf->xattr_name);
-        gf_asprintf (&conf->commithash_xattr_name, "%s."DHT_COMMITHASH_STR,
-                     conf->xattr_name);
-        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
-        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+    }
+
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100)
+        conf->disk_unit = 'p';
+
+    ret = dht_init_subvolumes(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    if (cmd) {
+        ret = dht_init_local_subvolumes(this, conf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_INIT_LOCAL_SUBVOL_FAILED,
+                   "dht_init_local_subvolumes failed");
+            goto err;
+        }
+    }
+
+    if (dict_get_str(this->options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto err;
+    }
+
+    dht_init_regex(this, this->options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, this->options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    ret = dht_layouts_init(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    conf->gen = 1;
+
+    this->local_pool = mem_pool_new(dht_local_t, 512);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               " DHT initialisation failed. "
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    GF_OPTION_INIT("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                   bool, err);
+
+    if (defrag) {
+        GF_OPTION_INIT("rebal-throttle", temp_str, str, err);
+        if (temp_str) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
                 goto err;
         }
+    }
 
-        GF_OPTION_INIT ("weighted-rebalance", conf->do_weighting, bool, err);
+    GF_OPTION_INIT("xattr-name", conf->xattr_name, str, err);
+    gf_asprintf(&conf->mds_xattr_key, "%s." DHT_MDS_STR, conf->xattr_name);
+    gf_asprintf(&conf->link_xattr_name, "%s." DHT_LINKFILE_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->commithash_xattr_name, "%s." DHT_COMMITHASH_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->wild_xattr_name, "%s*", conf->xattr_name);
+    if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+        goto err;
+    }
 
-        conf->lock_pool = mem_pool_new (dht_lock_t, 512);
-        if (!conf->lock_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_INIT_FAILED,
-                        "failed to create lock mem_pool, failing "
-                        "initialization");
-                goto err;
-        }
+    GF_OPTION_INIT("weighted-rebalance", conf->do_weighting, bool, err);
 
-        this->private = conf;
+    conf->lock_pool = mem_pool_new(dht_lock_t, 512);
+    if (!conf->lock_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INIT_FAILED,
+               "failed to create lock mem_pool, failing "
+               "initialization");
+        goto err;
+    }
 
-        if (dht_set_subvol_range(this))
-                goto err;
+    this->private = conf;
 
-        if (dht_init_methods (this))
-                goto err;
+    if (dht_set_subvol_range(this))
+        goto err;
 
-        return 0;
+    if (dht_init_methods(this))
+        goto err;
+
+    return 0;
 
 err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
+        }
 
-                GF_FREE (conf->subvolumes);
+        GF_FREE(conf->subvolumes);
 
-                GF_FREE (conf->subvolume_status);
+        GF_FREE(conf->subvolume_status);
 
-                GF_FREE (conf->du_stats);
+        GF_FREE(conf->du_stats);
 
-                GF_FREE (conf->defrag);
+        GF_FREE(conf->defrag);
 
-                GF_FREE (conf->xattr_name);
-                GF_FREE (conf->link_xattr_name);
-                GF_FREE (conf->wild_xattr_name);
+        GF_FREE(conf->xattr_name);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
+        GF_FREE(conf->mds_xattr_key);
 
-                if (conf->lock_pool)
-                        mem_pool_destroy (conf->lock_pool);
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
 
-                GF_FREE (conf);
-        }
+        GF_FREE(conf);
+    }
 
-        return -1;
+    return -1;
 }
 
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-          .description = "This option if set to ON, does a lookup through "
-          "all the sub-volumes, in case a lookup didn't return any result "
-          "from the hash subvolume. If set to OFF, it does not do a lookup "
-          "on the remaining subvolumes."
-        },
-        { .key = {"lookup-optimize"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON enables the optimization "
-          "of -ve lookups, by not doing a lookup on non-hashed subvolumes for "
-          "files, in case the hashed subvolume does not return any result. "
-          "This option disregards the lookup-unhashed setting, when enabled."
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-          .default_value = "10%",
-          .description = "Percentage/Size of disk space, after which the "
-          "process starts balancing out the cluster, and logs will appear "
-          "in log files",
-        },
-	{ .key  = {"min-free-inodes"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "5%",
-          .description = "after system has only N% of inodes, warnings "
-          "starts to appear in log files",
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option if set to ON, forces the use of "
-          "readdirp, and hence also displays the stats of the files."
-        },
-        { .key = {"assert-no-child-down"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON, in the event of "
-          "CHILD_DOWN, will call exit."
-        },
-        { .key  = {"directory-layout-spread"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .validate = GF_OPT_VALIDATE_MIN,
-          .description = "Specifies the directory layout spread. Takes number "
-                         "of subvolumes as default value."
-        },
-        { .key  = {"decommissioned-bricks"},
-          .type = GF_OPTION_TYPE_ANY,
-          .description = "This option if set to ON, decommissions "
-          "the brick, so that no new data is allowed to be created "
-          "on that brick."
-        },
-        { .key  = {"rebalance-cmd"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"commit-hash"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-        { .key = {"rebalance-stats"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON displays and logs the "
-          " time taken for migration of each file, during the rebalance "
-          "process. If set to OFF, the rebalance logs will only display the "
-          "time spent in each directory."
-        },
-        { .key = {"readdir-optimize"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON enables the optimization "
-          "that allows DHT to requests non-first subvolumes to filter out "
-          "directory entries."
-        },
-        { .key = {"rsync-hash-regex"},
-          .type = GF_OPTION_TYPE_STR,
-          /* Setting a default here doesn't work.  See dht_init_regex. */
-          .description = "Regular expression for stripping temporary-file "
-          "suffix and prefix used by rsync, to prevent relocation when the "
-          "file is renamed."
-        },
-        { .key = {"extra-hash-regex"},
-          .type = GF_OPTION_TYPE_STR,
-          /* Setting a default here doesn't work.  See dht_init_regex. */
-          .description = "Regular expression for stripping temporary-file "
-          "suffix and prefix used by an application, to prevent relocation when "
-          "the file is renamed."
-        },
-        { .key = {"rebalance-filter"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key = {"xattr-name"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "trusted.glusterfs.dht",
-          .description = "Base for extended attributes used by this "
-          "translator instance, to avoid conflicts with others above or "
-          "below it."
-        },
-
-        { .key = {"weighted-rebalance"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "When enabled, files will be allocated to bricks "
-          "with a probability proportional to their size.  Otherwise, all "
-          "bricks will have the same probability (legacy behavior)."
-        },
-
-        /* NUFA option */
-        { .key  = {"local-volume-name"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-
-        /* tier options */
-        { .key  = {"tier-pause"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-
-        { .key  = {"tier-promote-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "120",
-          .description = "Frequency to promote files to fast tier"
-        },
-
-        { .key  = {"tier-demote-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "3600",
-          .description = "Frequency to demote files to slow tier"
-        },
-
-        { .key  = {"write-freq-threshold"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-        },
-
-        { .key  = {"read-freq-threshold"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-        },
-        { .key         = {"watermark-hi"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "90",
-        },
-        { .key         = {"watermark-low"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "75",
-        },
-        { .key         = {"tier-mode"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "test",
-        },
-        { .key         = {"tier-max-mb"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "4000",
-        },
-        { .key         = {"tier-max-promote-file-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-        },
-        { .key         = {"tier-max-files"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "10000",
-        },
-        /* switch option */
-        { .key  = {"pattern.switch.case"},
-          .type = GF_OPTION_TYPE_ANY
-        },
-
-        { .key =  {"randomize-hash-range-by-gfid"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Use gfid of directory to determine the subvolume "
-          "from which hash ranges are allocated starting with 0. "
-          "Note that we still use a directory/file's name to determine the "
-          "subvolume to which it hashes"
-        },
-
-        { .key =  {"rebal-throttle"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "normal",
-          .description = " Sets the maximum number of parallel file migrations "
-                         "allowed on a node during the rebalance operation. The"
-                         " default value is normal and allows a max of "
-                         "[($(processing units) - 4) / 2), 2]  files to be "
-                         "migrated at a time. Lazy will allow only one file to "
-                         "be migrated at a time and aggressive will allow "
-                         "max of [($(processing units) - 4) / 2), 4]"
-        },
-
-        { .key =  {"lock-migration"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = " If enabled this feature will migrate the posix locks"
-                         " associated with a file during rebalance"
-        },
-
-        { .key  = {NULL} },
+struct volume_options dht_options[] = {
+    {
+        .key = {"lookup-unhashed"},
+        .value = {"auto", "yes", "no", "enable", "disable", "1", "0", "on",
+                  "off"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "on",
+        .description =
+            "This option if set to ON, does a lookup through "
+            "all the sub-volumes, in case a lookup didn't return any result "
+            "from the hash subvolume. If set to OFF, it does not do a lookup "
+            "on the remaining subvolumes.",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"lookup-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "This option if set to ON enables the optimization "
+         "of -ve lookups, by not doing a lookup on non-hashed subvolumes for "
+         "files, in case the hashed subvolume does not return any result. "
+         "This option disregards the lookup-unhashed setting, when enabled.",
+     .op_version = {GD_OP_VERSION_3_7_2},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-disk"},
+     .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+     .default_value = "10%",
+     .description =
+         "Percentage/Size of disk space, after which the "
+         "process starts balancing out the cluster, and logs will appear "
+         "in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-inodes"},
+     .type = GF_OPTION_TYPE_PERCENT,
+     .default_value = "5%",
+     .description = "after system has only N% of inodes, warnings "
+                    "starts to appear in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"unhashed-sticky-bit"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"use-readdirp"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This option if set to ON, forces the use of "
+                    "readdirp, and hence also displays the stats of the files.",
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"assert-no-child-down"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option if set to ON, in the event of "
+                    "CHILD_DOWN, will call exit."},
+    {
+        .key = {"directory-layout-spread"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .validate = GF_OPT_VALIDATE_MIN,
+        .description = "Specifies the directory layout spread. Takes number "
+                       "of subvolumes as default value.",
+
+        .op_version = {2},
+    },
+    {
+        .key = {"decommissioned-bricks"},
+        .type = GF_OPTION_TYPE_ANY,
+        .description =
+            "This option if set to ON, decommissions "
+            "the brick, so that no new data is allowed to be created "
+            "on that brick.",
+        .level = OPT_STATUS_ADVANCED,
+    },
+    {
+        .key = {"rebalance-cmd"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"commit-hash"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+    {
+        .key = {"rebalance-stats"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "This option if set to ON displays and logs the "
+            " time taken for migration of each file, during the rebalance "
+            "process. If set to OFF, the rebalance logs will only display the "
+            "time spent in each directory.",
+        .op_version = {2},
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"readdir-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description =
+         "This option if set to ON enables the optimization "
+         "that allows DHT to requests non-first subvolumes to filter out "
+         "directory entries.",
+     .op_version = {1},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"rsync-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by rsync, to prevent relocation when the "
+         "file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"extra-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by an application, to prevent relocation when "
+         "the file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"rebalance-filter"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+
+    {
+        .key = {"xattr-name"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "trusted.glusterfs.dht",
+        .description =
+            "Base for extended attributes used by this "
+            "translator instance, to avoid conflicts with others above or "
+            "below it.",
+        .op_version = {3},
+    },
+
+    {.key = {"weighted-rebalance"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "When enabled, files will be allocated to bricks "
+         "with a probability proportional to their size.  Otherwise, all "
+         "bricks will have the same probability (legacy behavior).",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    /* NUFA option */
+    {.key = {"local-volume-name"}, .type = GF_OPTION_TYPE_XLATOR},
+
+    /* switch option */
+    {.key = {"pattern.switch.case"}, .type = GF_OPTION_TYPE_ANY},
+
+    {
+        .key = {"randomize-hash-range-by-gfid"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "Use gfid of directory to determine the subvolume "
+            "from which hash ranges are allocated starting with 0. "
+            "Note that we still use a directory/file's name to determine the "
+            "subvolume to which it hashes",
+        .op_version = {GD_OP_VERSION_3_6_0},
+    },
+
+    {.key = {"rebal-throttle"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "normal",
+     .description = " Sets the maximum number of parallel file migrations "
+                    "allowed on a node during the rebalance operation. The"
+                    " default value is normal and allows a max of "
+                    "[($(processing units) - 4) / 2), 2]  files to be "
+                    "migrated at a time. Lazy will allow only one file to "
+                    "be migrated at a time and aggressive will allow "
+                    "max of [($(processing units) - 4) / 2), 4]",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
+
+    },
+
+    {.key = {"lock-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = " If enabled this feature will migrate the posix locks"
+                    " associated with a file during rebalance",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {"force-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "If disabled, rebalance will not migrate files that "
+                    "are being written to by an application",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {NULL}},
 };
+
+#define NUM_DHT_OPTIONS (sizeof(dht_options) / sizeof(dht_options[0]))
+
+extern struct volume_options options[NUM_DHT_OPTIONS]
+    __attribute__((alias("dht_options")));
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index afdfd5c80ea..53de8292704 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -8,77 +8,116 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "statedump.h"
 #include "dht-common.h"
 
-class_methods_t class_methods = {
-        .init           = dht_init,
-        .fini           = dht_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
+struct xlator_fops dht_pt_fops = {
+    /* we need to keep mkdir to make sure we
+       have layout on new directory */
+    .mkdir = dht_pt_mkdir,
+    .getxattr = dht_pt_getxattr,
+    .fgetxattr = dht_pt_fgetxattr,
+
+    /* required to trace fop properly in changelog */
+    .rename = dht_pt_rename,
+
+    /* FIXME: commenting the '.lookup()' below made some of
+       the failing tests to pass. I would remove the below
+       line, but keeping it here as a reminder for people
+       to check for issues if they find concerns with DHT
+       pass-through logic  */
+    /*
+      .lookup = dht_lookup,
+      .readdir = dht_readdir,
+      .readdirp = dht_readdirp,
+    */
+    /* Keeping above as commented, mainly to support the
+       usecase of a gluster volume getting to 1x(anytype),
+       due to remove-brick (shrinking) exercise. In that case,
+       we would need above fops to be available, so we can
+       handle the case of dangling linkto files (if any) */
 };
 
 struct xlator_fops fops = {
-        .lookup      = dht_lookup,
-        .mknod       = dht_mknod,
-        .create      = dht_create,
+    .ipc = dht_ipc,
+    .lookup = dht_lookup,
+    .mknod = dht_mknod,
+    .create = dht_create,
 
-        .open        = dht_open,
-        .statfs      = dht_statfs,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
+    .open = dht_open,
+    .statfs = dht_statfs,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
 
-        /* Inode read operations */
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .getxattr    = dht_getxattr,
-        .fgetxattr    = dht_fgetxattr,
-        .readv       = dht_readv,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .lk          = dht_lk,
-        .lease       = dht_lease,
+    /* Inode read operations */
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .getxattr = dht_getxattr,
+    .fgetxattr = dht_fgetxattr,
+    .readv = dht_readv,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .lk = dht_lk,
+    .lease = dht_lease,
 
-        /* Inode write operations */
-        .fremovexattr = dht_fremovexattr,
-        .removexattr = dht_removexattr,
-        .setxattr    = dht_setxattr,
-        .fsetxattr   = dht_fsetxattr,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .writev      = dht_writev,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
-        .fsetattr    = dht_fsetattr,
-	.fallocate   = dht_fallocate,
-	.discard     = dht_discard,
-        .zerofill    = dht_zerofill,
+    /* Inode write operations */
+    .fremovexattr = dht_fremovexattr,
+    .removexattr = dht_removexattr,
+    .setxattr = dht_setxattr,
+    .fsetxattr = dht_fsetxattr,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .writev = dht_writev,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
+    .fsetattr = dht_fsetattr,
+    .fallocate = dht_fallocate,
+    .discard = dht_discard,
+    .zerofill = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = dht_priv_dump,
-        .inodectx = dht_inodectx_dump,
+    .priv = dht_priv_dump,
+    .inodectx = dht_inodectx_dump,
 };
 
-
 struct xlator_cbks cbks = {
-        .release    = dht_release,
-//      .releasedir = dht_releasedir,
-        .forget     = dht_forget
+    .release = dht_release,
+    //      .releasedir = dht_releasedir,
+    .forget = dht_forget,
+};
+
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+extern struct volume_options dht_options[];
+
+xlator_api_t xlator_api = {
+    .init = dht_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "distribute",
+    .pass_through_fops = &dht_pt_fops,
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/dht/src/dht.sym b/xlators/cluster/dht/src/dht.sym
deleted file mode 100644
index 780b5fc0387..00000000000
--- a/xlators/cluster/dht/src/dht.sym
+++ /dev/null
@@ -1,8 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 56e17d6e884..3648a564840 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -8,678 +8,650 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "dht-common.h"
 
 /* TODO: all 'TODO's in dht.c holds good */
 
-extern struct volume_options options[];
+extern struct volume_options dht_options[];
 
 int
-nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                       struct iatt *postparent)
+nufa_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        call_frame_t *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
-
-        if (op_ret == -1)
-                goto out;
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
-
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "could not set pre-set layout for subvol"
-                                      " %s", prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                goto out;
+    }
+
+    if (op_ret == -1)
+        goto out;
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout for subvol"
+                         " %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
-
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
-
-                local->op_ret = 0;
-                local->op_errno = 0;
-
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_lookup_dir_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->lookup,
-                                    &local->loc, local->xattr_req);
-                }
-        }
+        goto out;
+    }
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-                if (!subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "linkfile has no link subvolume. path=%s",
-                                      loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                            subvol, subvol->fops->lookup,
-                            &local->loc, local->xattr_req);
+        local->op_ret = 0;
+        local->op_errno = 0;
+
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        return 0;
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-out:
-        if (!local->hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume. path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        STACK_WIND (frame, dht_lookup_cbk,
-                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
 
+    return 0;
+
+out:
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
         return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
+
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, postparent);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
+    return 0;
 }
 
 int
-nufa_lookup (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, dict_t *xattr_req)
+nufa_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. "
+                         "path=%s",
+                         loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-
-        local->hashed_subvol = hashed_subvol;
-
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "revalidate lookup without cache. "
-                                      "path=%s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_debug (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
-
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-                 *       revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
-
-                        STACK_WIND (frame, dht_revalidate_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    loc, local->xattr_req);
-
-                        if (!--call_cnt)
-                                break;
-                }
-        } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                /* Send it to only local volume */
-                STACK_WIND (frame, nufa_local_lookup_cbk,
-                            (xlator_t *)conf->private,
-                            ((xlator_t *)conf->private)->fops->lookup,
-                            loc, local->xattr_req);
-        }
+        local->inode = inode_ref(loc->inode);
 
-        return 0;
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
-}
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+         *       revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
 
-int
-nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int op_ret, int op_errno,
-                                 inode_t *inode, struct iatt *stbuf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
 
-        local = frame->local;
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
 
-        if (op_ret == -1)
-                goto err;
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
+
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
 
-        STACK_WIND (frame, dht_create_cbk,
-                    local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->umask,
-                    local->fd, local->params);
+        /* Send it to only local volume */
+        STACK_WIND_COOKIE(
+            frame, nufa_local_lookup_cbk, ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private)->fops->lookup, loc, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-nufa_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *params)
+nufa_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int op_ret, int op_errno,
+                                inode_t *inode, struct iatt *stbuf,
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    local = frame->local;
 
-        conf  = this->private;
+    if (op_ret == -1)
+        goto err;
 
-        dht_get_du_info (frame, this, loc);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private,
-                                                        local);
-        }
+err:
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
+}
 
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+int
+nufa_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, nufa_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+nufa_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                op_ret = -1;
-                goto err;
-        }
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-        if (op_ret >= 0) {
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                            (void *)local->cached_subvol, local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
-                return 0;
-        }
+        return 0;
+    }
 err:
-        WIPE (postparent);
-        WIPE (preparent);
+    WIPE(postparent);
+    WIPE(preparent);
 
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-nufa_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
+nufa_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private,
-                                                        local);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
-                                     avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* Consider the disksize in consideration */
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (avail_subvol != subvol) {
+        /* Create linkfile first */
+
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->umask = umask;
+        local->rdev = rdev;
+        local->cached_subvol = avail_subvol;
+
+        dht_linkfile_create(frame, nufa_mknod_linkfile_cbk, this, avail_subvol,
+                            subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->mknod, loc, mode, rdev, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->mknod, loc, mode, rdev, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 gf_boolean_t
-same_first_part (char *str1, char term1, char *str2, char term2)
+same_first_part(char *str1, char term1, char *str2, char term2)
 {
-        gf_boolean_t    ended1;
-        gf_boolean_t    ended2;
-
-        for (;;) {
-                ended1 = ((*str1 == '\0') || (*str1 == term1));
-                ended2 = ((*str2 == '\0') || (*str2 == term2));
-                if (ended1 && ended2) {
-                        return _gf_true;
-                }
-                if (ended1 || ended2 || (*str1 != *str2)) {
-                        return _gf_false;
-                }
-                ++str1;
-                ++str2;
+    gf_boolean_t ended1;
+    gf_boolean_t ended2;
+
+    for (;;) {
+        ended1 = ((*str1 == '\0') || (*str1 == term1));
+        ended2 = ((*str2 == '\0') || (*str2 == term2));
+        if (ended1 && ended2) {
+            return _gf_true;
         }
+        if (ended1 || ended2 || (*str1 != *str2)) {
+            return _gf_false;
+        }
+        ++str1;
+        ++str2;
+    }
 }
 
 typedef struct nufa_args {
-        xlator_t    *this;
-        char        *volname;
-        gf_boolean_t addr_match;
+    xlator_t *this;
+    char *volname;
+    gf_boolean_t addr_match;
 } nufa_args_t;
 
 static void
-nufa_find_local_brick (xlator_t *xl, void *data)
+nufa_find_local_brick(xlator_t *xl, void *data)
 {
-        nufa_args_t     *args = data;
-        xlator_t        *this = args->this;
-        char            *local_volname = args->volname;
-        gf_boolean_t    addr_match = args->addr_match;
-        char            *brick_host = NULL;
-        dht_conf_t      *conf = this->private;
-        int             ret = -1;
-
-        /*This means a local subvol was already found. We pick the first brick
-         * that is local*/
-        if (conf->private)
-                return;
-
-        if (strcmp (xl->name, local_volname) == 0) {
-                conf->private = xl;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO,
-                        "Using specified subvol %s",
-                        local_volname);
-                return;
-        }
-
-        if (!addr_match)
-                return;
-
-        ret = dict_get_str (xl->options, "remote-host", &brick_host);
-        if ((ret == 0) &&
-            (gf_is_same_address (local_volname, brick_host) ||
-             gf_is_local_addr (brick_host))) {
-                conf->private = xl;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO, "Using the first local "
-                        "subvol %s", xl->name);
-                return;
-        }
-
+    nufa_args_t *args = data;
+    xlator_t *this = args->this;
+    char *local_volname = args->volname;
+    gf_boolean_t addr_match = args->addr_match;
+    char *brick_host = NULL;
+    dht_conf_t *conf = this->private;
+    int ret = -1;
+
+    /*This means a local subvol was already found. We pick the first brick
+     * that is local*/
+    if (conf->private)
+        return;
+
+    if (strcmp(xl->name, local_volname) == 0) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using specified subvol %s", local_volname);
+        return;
+    }
+
+    if (!addr_match)
+        return;
+
+    ret = dict_get_str(xl->options, "remote-host", &brick_host);
+    if ((ret == 0) && (gf_is_same_address(local_volname, brick_host) ||
+                       gf_is_local_addr(brick_host))) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using the first local "
+               "subvol %s",
+               xl->name);
+        return;
+    }
 }
 
 static void
-nufa_to_dht (xlator_t *this)
+nufa_to_dht(xlator_t *this)
 {
-        GF_ASSERT (this);
-        GF_ASSERT (this->fops);
+    GF_ASSERT(this);
+    GF_ASSERT(this->fops);
 
-        this->fops->lookup = dht_lookup;
-        this->fops->create = dht_create;
-        this->fops->mknod  = dht_mknod;
+    this->fops->lookup = dht_lookup;
+    this->fops->create = dht_create;
+    this->fops->mknod = dht_mknod;
 }
 
 int
-nufa_find_local_subvol (xlator_t *this,
-                        void (*fn) (xlator_t *each, void* data), void *data)
+nufa_find_local_subvol(xlator_t *this, void (*fn)(xlator_t *each, void *data),
+                       void *data)
 {
-        int             ret = -1;
-        dht_conf_t      *conf = this->private;
-        xlator_list_t   *trav = NULL;
-        xlator_t        *parent = NULL;
-        xlator_t        *candidate = NULL;
-
-        xlator_foreach_depth_first (this, fn, data);
-        if (!conf->private) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_BRICK_ERROR, "Couldn't find a local "
-                        "brick");
-                return -1;
+    int ret = -1;
+    dht_conf_t *conf = this->private;
+    xlator_list_t *trav = NULL;
+    xlator_t *parent = NULL;
+    xlator_t *candidate = NULL;
+
+    xlator_foreach_depth_first(this, fn, data);
+    if (!conf->private) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_BRICK_ERROR,
+               "Couldn't find a local "
+               "brick");
+        return -1;
+    }
+
+    candidate = conf->private;
+    trav = candidate->parents;
+    while (trav) {
+        parent = trav->xlator;
+        if (strcmp(parent->type, "cluster/nufa") == 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "Found local subvol, "
+                   "%s",
+                   candidate->name);
+            ret = 0;
+            conf->private = candidate;
+            break;
         }
 
-        candidate = conf->private;
-        trav = candidate->parents;
-        while (trav) {
-
-                parent = trav->xlator;
-                if (strcmp (parent->type, "cluster/nufa") == 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_SUBVOL_INFO, "Found local subvol, "
-                                "%s", candidate->name);
-                        ret = 0;
-                        conf->private = candidate;
-                        break;
-                }
-
-                candidate = parent;
-                trav = parent->parents;
-        }
+        candidate = parent;
+        trav = parent->parents;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nufa_init (xlator_t *this)
+nufa_init(xlator_t *this)
 {
-        data_t        *data = NULL;
-        char          *local_volname = NULL;
-        int            ret = -1;
-        char           my_hostname[256];
-        gf_boolean_t   addr_match = _gf_false;
-        nufa_args_t    args = {0, };
-
-        ret = dht_init(this);
-        if (ret) {
-                return ret;
-        }
-
-        if ((data = dict_get (this->options, "local-volume-name"))) {
-                local_volname = data->data;
-
-        } else {
-                addr_match = _gf_true;
-                local_volname = "localhost";
-                ret = gethostname (my_hostname, 256);
-                if (ret == 0)
-                        local_volname = my_hostname;
-
-                else
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                DHT_MSG_GET_HOSTNAME_FAILED,
-                                "could not find hostname");
-
-        }
-
-        args.this = this;
-        args.volname = local_volname;
-        args.addr_match = addr_match;
-        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO,
-                        "Unable to find local subvolume, switching "
-                        "to dht mode");
-                nufa_to_dht (this);
-        }
-        return 0;
+    data_t *data = NULL;
+    char *local_volname = NULL;
+    int ret = -1;
+    char my_hostname[256];
+    gf_boolean_t addr_match = _gf_false;
+    nufa_args_t args = {
+        0,
+    };
+
+    ret = dht_init(this);
+    if (ret) {
+        return ret;
+    }
+
+    if ((data = dict_get(this->options, "local-volume-name"))) {
+        local_volname = data->data;
+
+    } else {
+        addr_match = _gf_true;
+        local_volname = "localhost";
+        ret = gethostname(my_hostname, 256);
+        if (ret == 0)
+            local_volname = my_hostname;
+
+        else
+            gf_msg(this->name, GF_LOG_WARNING, errno,
+                   DHT_MSG_GET_HOSTNAME_FAILED, "could not find hostname");
+    }
+
+    args.this = this;
+    args.volname = local_volname;
+    args.addr_match = addr_match;
+    ret = nufa_find_local_subvol(this, nufa_find_local_brick, &args);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Unable to find local subvolume, switching "
+               "to dht mode");
+        nufa_to_dht(this);
+    }
+    return 0;
 }
 
 dht_methods_t dht_methods = {
-        .migration_get_dst_subvol = dht_migration_get_dst_subvol,
-        .migration_needed = dht_migration_needed,
-        .layout_search   = dht_layout_search,
-};
-
-class_methods_t class_methods = {
-        .init           = nufa_init,
-        .fini           = dht_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
+    .migration_get_dst_subvol = dht_migration_get_dst_subvol,
+    .layout_search = dht_layout_search,
 };
 
-
 struct xlator_fops fops = {
-        .lookup      = nufa_lookup,
-        .create      = nufa_create,
-        .mknod       = nufa_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+    .lookup = nufa_lookup,
+    .create = nufa_create,
+    .mknod = nufa_mknod,
+
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .setxattr = dht_setxattr,
+    .getxattr = dht_getxattr,
+    .removexattr = dht_removexattr,
+    .open = dht_open,
+    .readv = dht_readv,
+    .writev = dht_writev,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .statfs = dht_statfs,
+    .lk = dht_lk,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
 };
 
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
+struct xlator_cbks cbks = {.forget = dht_forget};
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+xlator_api_t xlator_api = {
+    .init = nufa_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "nufa",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/cluster/dht/src/nufa.sym b/xlators/cluster/dht/src/nufa.sym
deleted file mode 100644
index 780b5fc0387..00000000000
--- a/xlators/cluster/dht/src/nufa.sym
+++ /dev/null
@@ -1,8 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index f1e9a399442..207d109a025 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "dht-common.h"
 #include "dht-mem-types.h"
 
@@ -17,892 +16,876 @@
 #include <fnmatch.h>
 #include <string.h>
 
-extern struct volume_options options[];
+extern struct volume_options dht_options[];
 
 struct switch_sched_array {
-        xlator_t *xl;
-        int32_t   eligible;
-        int32_t   considered;
+    xlator_t *xl;
+    int32_t eligible;
+    int32_t considered;
 };
 
 /* Select one of this struct based on the path's pattern match */
 struct switch_struct {
-        struct switch_struct      *next;
-        struct switch_sched_array *array;
-        int32_t                    node_index; /* Index of the node in
-                                                  this pattern. */
-        int32_t                    num_child;  /* Total num of child nodes
-                                                  with this pattern. */
-        char                       path_pattern[256];
+    struct switch_struct *next;
+    struct switch_sched_array *array;
+    int32_t node_index; /* Index of the node in
+                           this pattern. */
+    int32_t num_child;  /* Total num of child nodes
+                           with this pattern. */
+    char path_pattern[256];
 };
 
 /* TODO: all 'TODO's in dht.c holds good */
 /* This function should return child node as '*:subvolumes' is inserterd */
 
 static int32_t
-gf_switch_valid_child (xlator_t *this, const char *child)
+gf_switch_valid_child(xlator_t *this, const char *child)
 {
-        xlator_list_t *children = NULL;
-        int32_t        ret = 0;
-
-        children = this->children;
-        while (children) {
-                if (!strcmp (child, children->xlator->name)) {
-                        ret = 1;
-                        break;
-                }
-                children = children->next;
+    xlator_list_t *children = NULL;
+    int32_t ret = 0;
+
+    children = this->children;
+    while (children) {
+        if (!strcmp(child, children->xlator->name)) {
+            ret = 1;
+            break;
         }
+        children = children->next;
+    }
 
-        return ret;
+    return ret;
 }
 
 static xlator_t *
-get_switch_matching_subvol (const char *path, dht_conf_t *conf,
-                            xlator_t *hashed_subvol)
+get_switch_matching_subvol(const char *path, dht_conf_t *conf,
+                           xlator_t *hashed_subvol)
 {
-        struct switch_struct *cond      = NULL;
-        struct switch_struct *trav      = NULL;
-        char                 *pathname  = NULL;
-        int                   idx       = 0;
-        xlator_t             *subvol    = NULL;
-
-        cond = conf->private;
-        subvol = hashed_subvol;
-        if (!cond)
-                goto out;
-
-        pathname = gf_strdup (path);
-        if (!pathname)
-                goto out;
-
-        trav = cond;
-        while (trav) {
-                if (fnmatch (trav->path_pattern,
-                             pathname, FNM_NOESCAPE) == 0) {
-                        for (idx = 0; idx < trav->num_child; idx++) {
-                                if (trav->array[idx].xl == hashed_subvol)
-                                        goto out;
-                        }
-                        idx = trav->node_index++;
-                        trav->node_index %= trav->num_child;
-                        subvol = trav->array[idx].xl;
-                        goto out;
-                }
-                trav = trav->next;
+    struct switch_struct *cond = NULL;
+    struct switch_struct *trav = NULL;
+    char *pathname = NULL;
+    int idx = 0;
+    xlator_t *subvol = NULL;
+
+    cond = conf->private;
+    subvol = hashed_subvol;
+    if (!cond)
+        goto out;
+
+    pathname = gf_strdup(path);
+    if (!pathname)
+        goto out;
+
+    trav = cond;
+    while (trav) {
+        if (fnmatch(trav->path_pattern, pathname, FNM_NOESCAPE) == 0) {
+            for (idx = 0; idx < trav->num_child; idx++) {
+                if (trav->array[idx].xl == hashed_subvol)
+                    goto out;
+            }
+            idx = trav->node_index++;
+            trav->node_index %= trav->num_child;
+            subvol = trav->array[idx].xl;
+            goto out;
         }
+        trav = trav->next;
+    }
 out:
-        GF_FREE (pathname);
+    GF_FREE(pathname);
 
-        return subvol;
+    return subvol;
 }
 
-
 int
-switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+switch_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        call_frame_t *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
-
-        if (op_ret == -1)
-                goto out;
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
-
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
-
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "could not set pre-set layout "
-                                      "for subvol %s",
-                                      prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                goto out;
+    }
+
+    if (op_ret == -1)
+        goto out;
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout "
+                         "for subvol %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
+        goto out;
+    }
 
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        gf_msg_debug (this->name, 0,
-                                      "memory allocation failed :(");
-                        goto err;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_lookup_dir_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->lookup,
-                                    &local->loc, local->xattr_req);
-                }
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg_debug(this->name, 0, "memory allocation failed :(");
+            goto err;
         }
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-                if (!subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "linkfile has no link subvolume.path=%s",
-                                      loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
 
-                STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                            subvol, subvol->fops->lookup,
-                            &local->loc, local->xattr_req);
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume.path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+
+    return 0;
 
 out:
-        if (!local->hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
-        }
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_lookup_cbk,
-                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, NULL);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     NULL);
+    return 0;
 }
 
 int
-switch_lookup (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *xattr_req)
+switch_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+    cached_subvol = local->cached_subvol;
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. path=%s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-        cached_subvol = local->cached_subvol;
+        local->inode = inode_ref(loc->inode);
 
-        local->hashed_subvol = hashed_subvol;
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
 
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug(this->name, 0,
-                                     "revalidate lookup without cache. path=%s",
-                                     loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_debug (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode    = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
+         * attribute, revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
 
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
-                 * attribute, revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "failed to set dict value for %s",
-                                conf->xattr_name);
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
 
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
 
-                        STACK_WIND (frame, dht_revalidate_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    loc, local->xattr_req);
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
+
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->link_xattr_name);
+
+        if (!hashed_subvol) {
+            gf_msg_debug(this->name, 0,
+                         "no subvolume in layout for path=%s, "
+                         "checking on all the subvols to see if "
+                         "it is a directory",
+                         loc->path);
+            call_cnt = conf->subvolume_cnt;
+            local->call_cnt = call_cnt;
+
+            local->layout = dht_layout_new(this, conf->subvolume_cnt);
+            if (!local->layout) {
+                op_errno = ENOMEM;
+                goto err;
+            }
+
+            for (i = 0; i < call_cnt; i++) {
+                STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk,
+                                  conf->subvolumes[i], conf->subvolumes[i],
+                                  conf->subvolumes[i]->fops->lookup,
+                                  &local->loc, local->xattr_req);
+            }
+            return 0;
+        }
 
-                        if (!--call_cnt)
-                                break;
-                }
+        /*  */
+        cached_subvol = get_switch_matching_subvol(loc->path, conf,
+                                                   hashed_subvol);
+        if (cached_subvol == hashed_subvol) {
+            STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->lookup, loc,
+                              local->xattr_req);
         } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "failed to set dict value for %s",
-                                conf->xattr_name);
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "failed to set dict value for %s",
-                                conf->link_xattr_name);
-
-                if (!hashed_subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "no subvolume in layout for path=%s, "
-                                      "checking on all the subvols to see if "
-                                      "it is a directory", loc->path);
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
-
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
-
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_lookup_dir_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
-
-                /*  */
-                cached_subvol = get_switch_matching_subvol (loc->path, conf,
-                                                            hashed_subvol);
-                if (cached_subvol == hashed_subvol) {
-                        STACK_WIND (frame, dht_lookup_cbk,
-                                    hashed_subvol,
-                                    hashed_subvol->fops->lookup,
-                                    loc, local->xattr_req);
-                } else {
-                        STACK_WIND (frame, switch_local_lookup_cbk,
-                                    cached_subvol,
-                                    cached_subvol->fops->lookup,
-                                    loc, local->xattr_req);
-                }
+            STACK_WIND_COOKIE(frame, switch_local_lookup_cbk, cached_subvol,
+                              cached_subvol, cached_subvol->fops->lookup, loc,
+                              local->xattr_req);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int op_ret, int op_errno,
-                                   inode_t *inode, struct iatt *stbuf,
-                                   struct iatt *preparent,
-                                   struct iatt *postparent, dict_t *xdata)
+switch_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  struct iatt *preparent,
+                                  struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto err;
+    if (op_ret == -1)
+        goto err;
 
-        STACK_WIND (frame, dht_create_cbk,
-                    local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->umask,
-                    local->fd, local->params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 int
-switch_create (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, mode_t mode,
-               mode_t umask, fd_t *fd, dict_t *params)
+switch_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol,
-                                                        local);
-        }
-
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = get_switch_matching_subvol(loc->path, conf, subvol);
+    if (dht_is_subvol_filled(this, avail_subvol)) {
+        avail_subvol = dht_free_disk_available_subvol(this, avail_subvol,
+                                                      local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, switch_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, inode_t *inode,
-                           struct iatt *stbuf, struct iatt *preparent,
-                           struct iatt *postparent, dict_t *xdata)
+switch_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *stbuf, struct iatt *preparent,
+                          struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                op_ret = -1;
-                goto err;
-        }
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-        if (op_ret >= 0) {
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                            (void *)local->cached_subvol, local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
-                return 0;
-        }
-err:
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
         return 0;
+    }
+err:
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *params)
+switch_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol,
-                                                        local);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* Consider the disksize in consideration */
+    avail_subvol = get_switch_matching_subvol(loc->path, conf, subvol);
+    if (dht_is_subvol_filled(this, avail_subvol)) {
+        avail_subvol = dht_free_disk_available_subvol(this, avail_subvol,
+                                                      local);
+    }
+
+    if (avail_subvol != subvol) {
+        /* Create linkfile first */
+
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->umask = umask;
+        local->rdev = rdev;
+        local->cached_subvol = avail_subvol;
+
+        dht_linkfile_create(frame, switch_mknod_linkfile_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->mknod, loc, mode, rdev, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->mknod, loc, mode, rdev, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 void
-switch_fini (xlator_t *this)
+switch_fini(xlator_t *this)
 {
-        dht_conf_t           *conf = NULL;
-        struct switch_struct *trav = NULL;
-        struct switch_struct *prev = NULL;
-
-        conf = this->private;
-
-        if (conf) {
-                trav = (struct switch_struct *)conf->private;
-                conf->private = NULL;
-                while (trav) {
-                        GF_FREE (trav->array);
-                        prev = trav;
-                        trav = trav->next;
-                        GF_FREE (prev);
-                }
+    dht_conf_t *conf = NULL;
+    struct switch_struct *trav = NULL;
+    struct switch_struct *prev = NULL;
+
+    conf = this->private;
+
+    if (conf) {
+        trav = (struct switch_struct *)conf->private;
+        conf->private = NULL;
+        while (trav) {
+            GF_FREE(trav->array);
+            prev = trav;
+            trav = trav->next;
+            GF_FREE(prev);
         }
+    }
 
-        dht_fini(this);
+    dht_fini(this);
 }
 
 int
-set_switch_pattern (xlator_t *this, dht_conf_t *conf,
-                    const char *pattern_str)
+set_switch_pattern(xlator_t *this, dht_conf_t *conf, const char *pattern_str)
 {
-        int                         flag = 0;
-        int                         idx = 0;
-        int                         index = 0;
-        int                         child_count = 0;
-        char                       *tmp = NULL;
-        char                       *tmp1 = NULL;
-        char                       *child = NULL;
-        char                       *tmp_str = NULL;
-        char                       *tmp_str1 = NULL;
-        char                       *dup_str = NULL;
-        char                       *dup_childs = NULL;
-        char                       *switch_str = NULL;
-        char                       *pattern = NULL;
-        char                       *childs = NULL;
-        char                       *option_string = NULL;
-        struct switch_struct        *switch_buf = NULL;
-        struct switch_struct        *switch_opt = NULL;
-        struct switch_struct        *trav = NULL;
-        struct switch_sched_array  *switch_buf_array = NULL;
-        xlator_list_t              *trav_xl = NULL;
-
-        trav_xl = this->children;
-        while (trav_xl) {
-                index++;
-                trav_xl = trav_xl->next;
+    int flag = 0;
+    int idx = 0;
+    int index = 0;
+    int child_count = 0;
+    char *tmp = NULL;
+    char *tmp1 = NULL;
+    char *child = NULL;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *dup_str = NULL;
+    char *dup_childs = NULL;
+    char *switch_str = NULL;
+    char *pattern = NULL;
+    char *childs = NULL;
+    char *option_string = NULL;
+    size_t pattern_length;
+    struct switch_struct *switch_buf = NULL;
+    struct switch_struct *switch_opt = NULL;
+    struct switch_struct *trav = NULL;
+    struct switch_sched_array *switch_buf_array = NULL;
+    xlator_list_t *trav_xl = NULL;
+
+    trav_xl = this->children;
+    while (trav_xl) {
+        index++;
+        trav_xl = trav_xl->next;
+    }
+    child_count = index;
+    switch_buf_array = GF_CALLOC((index + 1), sizeof(struct switch_sched_array),
+                                 gf_switch_mt_switch_sched_array);
+    if (!switch_buf_array)
+        goto err;
+
+    trav_xl = this->children;
+    index = 0;
+
+    while (trav_xl) {
+        switch_buf_array[index].xl = trav_xl->xlator;
+        switch_buf_array[index].eligible = 1;
+        trav_xl = trav_xl->next;
+        index++;
+    }
+
+    /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
+
+    /* Get the pattern for considering switch case.
+       "option block-size *avi:10MB" etc */
+    option_string = gf_strdup(pattern_str);
+    if (option_string == NULL) {
+        goto err;
+    }
+    switch_str = strtok_r(option_string, ";", &tmp_str);
+    while (switch_str) {
+        dup_str = gf_strdup(switch_str);
+        if (dup_str == NULL) {
+            goto err;
         }
-        child_count = index;
-        switch_buf_array = GF_CALLOC ((index + 1),
-                                      sizeof (struct switch_sched_array),
-                                      gf_switch_mt_switch_sched_array);
-        if (!switch_buf_array)
-                goto err;
-
-        trav_xl = this->children;
-        index = 0;
-
-        while (trav_xl) {
-                switch_buf_array[index].xl = trav_xl->xlator;
-                switch_buf_array[index].eligible = 1;
-                trav_xl = trav_xl->next;
-                index++;
+        switch_opt = GF_CALLOC(1, sizeof(struct switch_struct),
+                               gf_switch_mt_switch_struct);
+        if (!switch_opt) {
+            GF_FREE(dup_str);
+            goto err;
         }
 
-        /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
-
-        /* Get the pattern for considering switch case.
-           "option block-size *avi:10MB" etc */
-        option_string = gf_strdup (pattern_str);
-        switch_str = strtok_r (option_string, ";", &tmp_str);
-        while (switch_str) {
-                dup_str = gf_strdup (switch_str);
-                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
-                                        gf_switch_mt_switch_struct);
-                if (!switch_opt) {
-                        GF_FREE (dup_str);
-                        goto err;
-                }
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        childs = strtok_r(NULL, ":", &tmp_str1);
+        if (strncmp(pattern, "*", 2) == 0) {
+            gf_msg("switch", GF_LOG_INFO, 0, DHT_MSG_SWITCH_PATTERN_INFO,
+                   "'*' pattern will be taken by default "
+                   "for all the unconfigured child nodes,"
+                   " hence neglecting current option");
+            switch_str = strtok_r(NULL, ";", &tmp_str);
+            GF_FREE(switch_opt);
+            switch_opt = NULL;
+            GF_FREE(dup_str);
+            continue;
+        }
+        GF_FREE(dup_str);
+
+        pattern_length = strlen(pattern);
+        if (pattern_length >= (sizeof(switch_opt->path_pattern))) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_SET_SWITCH_PATTERN_ERROR, "Pattern (%s) too long",
+                   pattern);
+            goto err;
+        }
+        memcpy(switch_opt->path_pattern, pattern, pattern_length);
+        switch_opt->path_pattern[pattern_length] = '\0';
 
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                childs = strtok_r (NULL, ":", &tmp_str1);
-                if (strncmp (pattern, "*", 2) == 0) {
-                        gf_msg ("switch", GF_LOG_INFO, 0,
-                                DHT_MSG_SWITCH_PATTERN_INFO,
-                                "'*' pattern will be taken by default "
-                                "for all the unconfigured child nodes,"
-                                " hence neglecting current option");
-                        switch_str = strtok_r (NULL, ";", &tmp_str);
-                        GF_FREE (switch_opt);
-                        GF_FREE (dup_str);
-                        continue;
-                }
-                GF_FREE (dup_str);
-                memcpy (switch_opt->path_pattern, pattern, strlen (pattern));
-                if (childs) {
-                        dup_childs = gf_strdup (childs);
-                        child = strtok_r (dup_childs, ",", &tmp);
-                        while (child) {
-                                if (gf_switch_valid_child (this, child)) {
-                                        idx++;
-                                        child = strtok_r (NULL, ",", &tmp);
-                                } else {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_SUBVOL_ERROR,
-                                                "%s is not a subvolume of %s. "
-                                                "pattern can only be scheduled "
-                                                "only to a subvolume of %s",
-                                                child, this->name, this->name);
-                                        goto err;
-                                }
-                        }
-                        GF_FREE (dup_childs);
-                        child = strtok_r (childs, ",", &tmp1);
-                        switch_opt->num_child = idx;
-                        switch_opt->array = GF_CALLOC (1, (idx *
-                                                           sizeof (struct switch_sched_array)),
-                                                       gf_switch_mt_switch_sched_array);
-                        if (!switch_opt->array)
-                                goto err;
-                        idx = 0;
-                        while (child) {
-                                for (index = 0; index < child_count; index++) {
-                                        if (strcmp (switch_buf_array[index].xl->name,
-                                                    child) == 0) {
-                                                gf_msg_debug ("switch", 0,
-                                                              "'%s' pattern will be "
-                                                              "scheduled to \"%s\"",
-                                                              switch_opt->path_pattern, child);
-                                                /*
-                                                  if (switch_buf_array[index-1].considered) {
-                                                  gf_msg_debug ("switch", 0,
-                                                  "ambiguity found, exiting");
-                                                  return -1;
-                                                  }
-                                                */
-                                                switch_opt->array[idx].xl = switch_buf_array[index].xl;
-                                                switch_buf_array[index].considered = 1;
-                                                idx++;
-                                                break;
-                                        }
-                                }
-                                child = strtok_r (NULL, ",", &tmp1);
-                        }
+        if (childs) {
+            dup_childs = gf_strdup(childs);
+            if (dup_childs == NULL) {
+                goto err;
+            }
+            child = strtok_r(dup_childs, ",", &tmp);
+            while (child) {
+                if (gf_switch_valid_child(this, child)) {
+                    idx++;
+                    child = strtok_r(NULL, ",", &tmp);
                 } else {
-                        /* error */
-                        gf_msg ("switch", GF_LOG_ERROR, 0,
-                                DHT_MSG_SET_SWITCH_PATTERN_ERROR,
-                                "Check \"scheduler.switch.case\" "
-                                "option in unify volume. Exiting");
-                        goto err;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR,
+                           "%s is not a subvolume of %s. "
+                           "pattern can only be scheduled "
+                           "only to a subvolume of %s",
+                           child, this->name, this->name);
+                    GF_FREE(dup_childs);
+                    goto err;
                 }
-
-                /* Link it to the main structure */
-                if (switch_buf) {
-                        /* there are already few entries */
-                        trav = switch_buf;
-                        while (trav->next)
-                                trav = trav->next;
-                        trav->next = switch_opt;
-                } else {
-                        /* First entry */
-                        switch_buf = switch_opt;
+            }
+            GF_FREE(dup_childs);
+            child = strtok_r(childs, ",", &tmp1);
+            switch_opt->num_child = idx;
+            switch_opt->array = GF_CALLOC(
+                1, (idx * sizeof(struct switch_sched_array)),
+                gf_switch_mt_switch_sched_array);
+            if (!switch_opt->array)
+                goto err;
+            idx = 0;
+            while (child) {
+                for (index = 0; index < child_count; index++) {
+                    if (strcmp(switch_buf_array[index].xl->name, child) == 0) {
+                        gf_msg_debug("switch", 0,
+                                     "'%s' pattern will be "
+                                     "scheduled to \"%s\"",
+                                     switch_opt->path_pattern, child);
+                        /*
+                          if (switch_buf_array[index-1].considered) {
+                          gf_msg_debug ("switch", 0,
+                          "ambiguity found, exiting");
+                          return -1;
+                          }
+                        */
+                        switch_opt->array[idx].xl = switch_buf_array[index].xl;
+                        switch_buf_array[index].considered = 1;
+                        idx++;
+                        break;
+                    }
                 }
-                switch_opt = NULL;
-                switch_str = strtok_r (NULL, ";", &tmp_str);
+                child = strtok_r(NULL, ",", &tmp1);
+            }
+        } else {
+            /* error */
+            gf_msg("switch", GF_LOG_ERROR, 0, DHT_MSG_SET_SWITCH_PATTERN_ERROR,
+                   "Check \"scheduler.switch.case\" "
+                   "option in unify volume. Exiting");
+            goto err;
         }
 
-        /* Now, all the pattern based considerations done, so for all the
-         * remaining pattern, '*' to all the remaining child nodes
-         */
-        {
-                for (index=0; index < child_count; index++) {
-                        /* check for considered flag */
-                        if (switch_buf_array[index].considered)
-                                continue;
-                        flag++;
-                }
-                if (!flag) {
-                        gf_msg ("switch", GF_LOG_ERROR, 0,
-                                DHT_MSG_SET_SWITCH_PATTERN_ERROR,
-                                "No nodes left for pattern '*'. Exiting");
-                        goto err;
-                }
-                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
-                                        gf_switch_mt_switch_struct);
-                if (!switch_opt)
-                        goto err;
-
-                /* Add the '*' pattern to the array */
-                memcpy (switch_opt->path_pattern, "*", 2);
-                switch_opt->num_child = flag;
-                switch_opt->array =
-                        GF_CALLOC (1,
-                                   flag * sizeof (struct switch_sched_array),
-                                   gf_switch_mt_switch_sched_array);
-                if (!switch_opt->array)
-                        goto err;
-                flag = 0;
-                for (index=0; index < child_count; index++) {
-                        /* check for considered flag */
-                        if (switch_buf_array[index].considered)
-                                continue;
-                        gf_msg_debug ("switch", 0, "'%s'"
-                                      " pattern will be scheduled to \"%s\"",
-                                      switch_opt->path_pattern,
-                                      switch_buf_array[index].xl->name);
-
-                        switch_opt->array[flag].xl =
-                                switch_buf_array[index].xl;
-                        switch_buf_array[index].considered = 1;
-                        flag++;
-                }
-                if (switch_buf) {
-                        /* there are already few entries */
-                        trav = switch_buf;
-                        while (trav->next)
-                                trav = trav->next;
-                        trav->next = switch_opt;
-                } else {
-                        /* First entry */
-                        switch_buf = switch_opt;
-                }
-                switch_opt = NULL;
+        /* Link it to the main structure */
+        if (switch_buf) {
+            /* there are already few entries */
+            trav = switch_buf;
+            while (trav->next)
+                trav = trav->next;
+            trav->next = switch_opt;
+        } else {
+            /* First entry */
+            switch_buf = switch_opt;
+        }
+        switch_opt = NULL;
+        switch_str = strtok_r(NULL, ";", &tmp_str);
+    }
+
+    /* Now, all the pattern based considerations done, so for all the
+     * remaining pattern, '*' to all the remaining child nodes
+     */
+    {
+        for (index = 0; index < child_count; index++) {
+            /* check for considered flag */
+            if (switch_buf_array[index].considered)
+                continue;
+            flag++;
+        }
+        if (!flag) {
+            gf_msg("switch", GF_LOG_ERROR, 0, DHT_MSG_SET_SWITCH_PATTERN_ERROR,
+                   "No nodes left for pattern '*'. Exiting");
+            goto err;
+        }
+        switch_opt = GF_CALLOC(1, sizeof(struct switch_struct),
+                               gf_switch_mt_switch_struct);
+        if (!switch_opt)
+            goto err;
+
+        /* Add the '*' pattern to the array */
+        memcpy(switch_opt->path_pattern, "*", 2);
+        switch_opt->num_child = flag;
+        switch_opt->array = GF_CALLOC(1,
+                                      flag * sizeof(struct switch_sched_array),
+                                      gf_switch_mt_switch_sched_array);
+        if (!switch_opt->array)
+            goto err;
+        flag = 0;
+        for (index = 0; index < child_count; index++) {
+            /* check for considered flag */
+            if (switch_buf_array[index].considered)
+                continue;
+            gf_msg_debug("switch", 0,
+                         "'%s'"
+                         " pattern will be scheduled to \"%s\"",
+                         switch_opt->path_pattern,
+                         switch_buf_array[index].xl->name);
+
+            switch_opt->array[flag].xl = switch_buf_array[index].xl;
+            switch_buf_array[index].considered = 1;
+            flag++;
+        }
+        if (switch_buf) {
+            /* there are already few entries */
+            trav = switch_buf;
+            while (trav->next)
+                trav = trav->next;
+            trav->next = switch_opt;
+        } else {
+            /* First entry */
+            switch_buf = switch_opt;
         }
-        /* */
-        conf->private = switch_buf;
+        switch_opt = NULL;
+    }
+    /* */
+    conf->private = switch_buf;
 
-        return 0;
+    GF_FREE(option_string);
+    return 0;
 err:
-        GF_FREE (switch_buf_array);
-        GF_FREE (switch_opt);
+    GF_FREE(switch_buf_array);
+    GF_FREE(switch_opt);
+    GF_FREE(option_string);
 
-        if (switch_buf) {
-                trav = switch_buf;
-                while (trav) {
-                        GF_FREE (trav->array);
-                        switch_opt = trav;
-                        trav = trav->next;
-                        GF_FREE (switch_opt);
-                }
+    if (switch_buf) {
+        trav = switch_buf;
+        while (trav) {
+            GF_FREE(trav->array);
+            switch_opt = trav;
+            trav = trav->next;
+            GF_FREE(switch_opt);
         }
-        return -1;
+    }
+    return -1;
 }
 
-
 int32_t
-switch_init (xlator_t *this)
+switch_init(xlator_t *this)
 {
-        dht_conf_t            *conf = NULL;
-        data_t                *data = NULL;
-        int                    ret = -1;
+    dht_conf_t *conf = NULL;
+    data_t *data = NULL;
+    int ret = -1;
+
+    ret = dht_init(this);
+    if (ret) {
+        return ret;
+    }
+    conf = this->private;
 
-        ret = dht_init(this);
+    data = dict_get(this->options, "pattern.switch.case");
+    if (data) {
+        /* TODO: */
+        ret = set_switch_pattern(this, conf, data->data);
         if (ret) {
-                return ret;
-        }
-        conf = this->private;
-
-        data = dict_get (this->options, "pattern.switch.case");
-        if (data) {
-                /* TODO: */
-                ret = set_switch_pattern (this, conf, data->data);
-                if (ret) {
-                        goto err;
-                }
+            goto err;
         }
+    }
 
-        this->private = conf;
-        return 0;
+    this->private = conf;
+    return 0;
 
 err:
-        dht_fini(this);
-        return -1;
+    dht_fini(this);
+    return -1;
 }
 
-
-class_methods_t class_methods = {
-        .init           = switch_init,
-        .fini           = switch_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
-};
-
-
 struct xlator_fops fops = {
-        .lookup      = switch_lookup,
-        .create      = switch_create,
-        .mknod       = switch_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+    .lookup = switch_lookup,
+    .create = switch_create,
+    .mknod = switch_mknod,
+
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .setxattr = dht_setxattr,
+    .getxattr = dht_getxattr,
+    .removexattr = dht_removexattr,
+    .open = dht_open,
+    .readv = dht_readv,
+    .writev = dht_writev,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .statfs = dht_statfs,
+    .lk = dht_lk,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
 };
 
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
+struct xlator_cbks cbks = {.forget = dht_forget};
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+xlator_api_t xlator_api = {
+    .init = switch_init,
+    .fini = switch_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "switch",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/cluster/dht/src/switch.sym b/xlators/cluster/dht/src/switch.sym
deleted file mode 100644
index 780b5fc0387..00000000000
--- a/xlators/cluster/dht/src/switch.sym
+++ /dev/null
@@ -1,8 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/tier-common.c b/xlators/cluster/dht/src/tier-common.c
deleted file mode 100644
index 20d3f24d3bf..00000000000
--- a/xlators/cluster/dht/src/tier-common.c
+++ /dev/null
@@ -1,1084 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "libxlator.h"
-#include "dht-common.h"
-#include "defaults.h"
-#include "tier-common.h"
-#include "tier.h"
-
-int
-dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata);
-
-
-int
-tier_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        loc_t        *oldloc = NULL;
-        loc_t        *newloc = NULL;
-
-        local = frame->local;
-
-        oldloc = &local->loc;
-        newloc = &local->loc2;
-
-        if (op_ret == -1) {
-                /* No continuation on DHT inode missing errors, as we should
-                 * then have a good stbuf that states P2 happened. We would
-                 * get inode missing if, the file completed migrated between
-                 * the lookup and the link call */
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                goto out;
-        }
-
-        local->call_cnt = 2;
-
-        /* Do this on the hot tier now */
-
-        STACK_WIND (frame, tier_link_cbk, local->cached_subvol,
-                    local->cached_subvol->fops->link,
-                    oldloc, newloc, xdata);
-
-        return 0;
-
-out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf,
-                          preparent, postparent, NULL);
-
-        return 0;
-}
-
-
-int
-tier_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
-        dht_conf_t   *conf           = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->call_cnt = 1;
-
-        cached_subvol = local->cached_subvol;
-
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", oldloc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (hashed_subvol == cached_subvol) {
-                STACK_WIND (frame, dht_link_cbk,
-                            cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-
-
-        /* Create hardlinks to both the data file on the hot tier
-           and the linkto file on the cold tier */
-
-        gf_uuid_copy (local->gfid, oldloc->inode->gfid);
-
-        STACK_WIND (frame, tier_link_cbk,
-                    hashed_subvol, hashed_subvol->fops->link,
-                    oldloc, newloc, xdata);
-
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL);
-        return 0;
-}
-
-
-
-int
-tier_create_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int op_ret, int op_errno,
-                                     struct iatt *preparent,
-                                     struct iatt *postparent, dict_t *xdata)
-{
-
-        dht_local_t     *local = NULL;
-
-        local = frame->local;
-
-        if (local->params) {
-                dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
-
-        DHT_STACK_UNWIND (create, frame, -1, local->op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        call_frame_t *prev           = NULL;
-        int           ret            = -1;
-        dht_local_t  *local          = NULL;
-        xlator_t     *hashed_subvol  = NULL;
-        dht_conf_t   *conf           = NULL;
-
-        local = frame->local;
-        conf  = this->private;
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (op_ret == -1) {
-                if (local->linked == _gf_true && local->xattr_req) {
-                        local->op_errno = op_errno;
-                        local->op_ret = op_ret;
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (local->xattr_req);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value to "
-                                        "unlink of migrating file");
-                                goto out;
-                        }
-
-                        STACK_WIND (frame,
-                                    tier_create_unlink_stale_linkto_cbk,
-                                    hashed_subvol,
-                                    hashed_subvol->fops->unlink,
-                                    &local->loc, 0, local->xattr_req);
-                        return 0;
-                }
-                goto out;
-        }
-
-        prev = cookie;
-
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
-
-        ret = dht_layout_preset (this, prev->this, inode);
-        if (ret != 0) {
-                gf_msg_debug (this->name, 0,
-                              "could not set preset layout for subvol %s",
-                              prev->this->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        local->op_errno = op_errno;
-
-        if (local->linked == _gf_true) {
-                local->stbuf = *stbuf;
-                dht_linkfile_attr_heal (frame, this);
-        }
-out:
-        if (local->xattr_req) {
-                dict_del (local->xattr_req, TIER_LINKFILE_GFID);
-        }
-
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-
-        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode,
-                          stbuf, preparent, postparent, xdata);
-
-        return 0;
-}
-
-int
-tier_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
-{
-        dht_local_t     *local             = NULL;
-        xlator_t        *cached_subvol     = NULL;
-        dht_conf_t      *conf              = NULL;
-        int              ret               = -1;
-        unsigned char   *gfid              = NULL;
-
-        local = frame->local;
-        if (!local) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        conf = this->private;
-        if (!conf) {
-                local->op_errno = EINVAL;
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        cached_subvol = TIER_UNHASHED_SUBVOL;
-
-        if (local->params) {
-                dict_del (local->params, conf->link_xattr_name);
-                dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
-
-        /*
-         * We will delete the linkfile if data file creation fails.
-         * When deleting this stale linkfile, there is a possibility
-         * for a race between this linkfile deletion and a stale
-         * linkfile deletion triggered by another lookup from different
-         * client.
-         *
-         * For eg:
-         *
-         *     Client 1                        Client 2
-         *
-         * 1   linkfile created for foo
-         *
-         * 2   data file creation failed
-         *
-         * 3                                   creating a file with same name
-         *
-         * 4                                   lookup before creation deleted
-         *                                     the linkfile created by client1
-         *                                     considering as a stale linkfile.
-         *
-         * 5                                   New linkfile created for foo
-         *                                     with different gfid.
-         *
-         * 6 Trigger linkfile deletion as
-         *   data file creation failed.
-         *
-         * 7 Linkfile deleted which is
-         *   created by client2.
-         *
-         * 8                                   Data file created.
-         *
-         * With this race, we will end up having a file in a non-hashed subvol
-         * without a linkfile in hashed subvol.
-         *
-         * To avoid this, we store the gfid of linkfile created by client, So
-         * If we delete the linkfile , we validate gfid of existing file with
-         * stored value from posix layer.
-         *
-         * Storing this value in local->xattr_req as local->params was also used
-         * to create the data file. During the linkfile deletion we will use
-         * local->xattr_req dictionary.
-         */
-        if (!local->xattr_req) {
-                local->xattr_req = dict_new ();
-                if (!local->xattr_req) {
-                        local->op_errno = ENOMEM;
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-        }
-
-        gfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!gfid) {
-                local->op_errno = ENOMEM;
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        gf_uuid_copy (gfid, stbuf->ia_gfid);
-        ret = dict_set_dynptr (local->xattr_req, TIER_LINKFILE_GFID,
-                                   gfid, sizeof (uuid_t));
-        if (ret) {
-                GF_FREE (gfid);
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value"
-                        " : key = %s", TIER_LINKFILE_GFID);
-        }
-
-        STACK_WIND (frame, tier_create_cbk,
-                    cached_subvol, cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode,
-                    local->umask, local->fd, local->params);
-
-        return 0;
-err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
-        return 0;
-}
-
-gf_boolean_t
-tier_is_hot_tier_decommissioned (xlator_t *this)
-{
-        dht_conf_t              *conf        = NULL;
-        xlator_t                *hot_tier    = NULL;
-        int                      i           = 0;
-
-        conf = this->private;
-        hot_tier = conf->subvolumes[1];
-
-        if (conf->decommission_subvols_cnt) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->decommissioned_bricks[i] &&
-                                conf->decommissioned_bricks[i] == hot_tier)
-                                return _gf_true;
-                }
-        }
-
-        return _gf_false;
-}
-
-int
-tier_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
-{
-        int                     op_errno            = -1;
-        dht_local_t             *local              = NULL;
-        dht_conf_t              *conf               = NULL;
-        xlator_t                *hot_subvol         = NULL;
-        xlator_t                *cold_subvol        = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-
-        cold_subvol = TIER_HASHED_SUBVOL;
-        hot_subvol = TIER_UNHASHED_SUBVOL;
-
-        if (conf->subvolumes[0] != cold_subvol) {
-                hot_subvol = conf->subvolumes[0];
-        }
-        /*
-         * if hot tier full, write to cold.
-         * Also if hot tier is full, create in cold
-         */
-        if (dht_is_subvol_filled (this, hot_subvol) ||
-            tier_is_hot_tier_decommissioned (this)) {
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path,
-                              cold_subvol->name);
-
-                STACK_WIND (frame, tier_create_cbk,
-                            cold_subvol, cold_subvol->fops->create,
-                            loc, flags, mode, umask, fd, params);
-        } else {
-                local->params = dict_ref (params);
-                local->flags = flags;
-                local->mode = mode;
-                local->umask = umask;
-                local->cached_subvol = hot_subvol;
-                local->hashed_subvol = cold_subvol;
-
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s (link at %s)", loc->path,
-                              hot_subvol->name, cold_subvol->name);
-
-                dht_linkfile_create (frame, tier_create_linkfile_create_cbk,
-                                     this, hot_subvol, cold_subvol, loc);
-
-                goto out;
-        }
-out:
-        return 0;
-
-err:
-
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_unlink_nonhashed_linkfile_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int op_ret, int op_errno,
-                                    struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if ((op_ret == -1) && (op_errno != ENOENT)) {
-                        local->op_errno = op_errno;
-                        local->op_ret = op_ret;
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1",
-                                      prev->this->name);
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (local->op_ret == -1)
-                goto err;
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
-
-
-        return 0;
-
-err:
-        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL, NULL);
-        return 0;
-}
-
-int
-tier_unlink_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *preparent, dict_t *xdata,
-                        struct iatt *postparent)
-{
-        dht_local_t             *local             = NULL;
-        call_frame_t            *prev              = NULL;
-        dht_conf_t              *conf              = NULL;
-        xlator_t                *hot_subvol        = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
-        hot_subvol = TIER_UNHASHED_SUBVOL;
-
-        if (!op_ret) {
-                /*
-                 * linkfile present on hot tier. unlinking the linkfile
-                 */
-                STACK_WIND (frame, tier_unlink_nonhashed_linkfile_cbk,
-                            hot_subvol, hot_subvol->fops->unlink,
-                            &local->loc, local->flags, NULL);
-                return 0;
-        }
-
-        LOCK (&frame->lock);
-        {
-                if (op_errno == ENOENT) {
-                        local->op_ret   = 0;
-                        local->op_errno = op_errno;
-                } else {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
-                gf_msg_debug (this->name, op_errno,
-                              "Lookup : subvolume %s returned -1",
-                               prev->this->name);
-        }
-
-        UNLOCK (&frame->lock);
-
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
-
-        return 0;
-}
-
-int
-tier_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                /* Ignore EINVAL for tier to ignore error when the file
-                        does not exist on the other tier  */
-                if ((op_ret == -1) && !((op_errno == ENOENT) ||
-                                        (op_errno == EINVAL))) {
-                        local->op_errno = op_errno;
-                        local->op_ret   = op_ret;
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1",
-                                      prev->this->name);
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (local->op_ret == -1)
-                goto err;
-
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
-
-        return 0;
-
-err:
-        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-tier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t     *local          = NULL;
-        call_frame_t    *prev           = NULL;
-        struct iatt     *stbuf          = NULL;
-        dht_conf_t      *conf           = NULL;
-        int              ret            = -1;
-        xlator_t        *hot_tier       = NULL;
-        xlator_t        *cold_tier      = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-        conf  = this->private;
-
-        cold_tier = TIER_HASHED_SUBVOL;
-        hot_tier = TIER_UNHASHED_SUBVOL;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOENT) {
-                                local->op_ret = 0;
-                        } else {
-                                local->op_ret   = -1;
-                                local->op_errno = op_errno;
-                        }
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink: subvolume %s returned -1"
-                                      " with errno = %d",
-                                       prev->this->name, op_errno);
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
-
-                local->postparent = *postparent;
-                local->preparent = *preparent;
-
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (local->op_ret)
-                goto out;
-
-        if (cold_tier != local->cached_subvol) {
-                /*
-                 * File is present in hot tier, so there will be
-                 * a link file on cold tier, deleting the linkfile
-                 * from cold tier
-                 */
-                STACK_WIND (frame, tier_unlink_linkfile_cbk,
-                            cold_tier,
-                            cold_tier->fops->unlink, &local->loc,
-                            local->flags, xdata);
-                return 0;
-        }
-
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
-        if (!ret && stbuf && ((IS_DHT_MIGRATION_PHASE2 (stbuf)) ||
-                      IS_DHT_MIGRATION_PHASE1 (stbuf))) {
-                /*
-                 * File is migrating from cold to hot tier.
-                 * Delete the destination linkfile.
-                 */
-                STACK_WIND (frame, tier_unlink_lookup_cbk,
-                            hot_tier,
-                            hot_tier->fops->lookup,
-                            &local->loc, NULL);
-                return 0;
-
-        }
-
-out:
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
-
-        return 0;
-}
-
-int
-tier_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-            dict_t *xdata)
-{
-        xlator_t                *cached_subvol = NULL;
-        xlator_t                *hashed_subvol = NULL;
-        dht_conf_t              *conf          = NULL;
-        int                      op_errno      = -1;
-        dht_local_t             *local         = NULL;
-        int                      ret           = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-
-                goto err;
-        }
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        local->flags = xflag;
-        if (IA_ISREG (loc->inode->ia_type) &&
-            (hashed_subvol == cached_subvol)) {
-                /*
-                 * File resides in cold tier. We need to stat
-                 * the file to see if it is being promoted.
-                 * If yes we need to delete the destination
-                 * file as well.
-                 *
-                 * Currently we are doing this check only for
-                 * regular files.
-                 */
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata) {
-                        ret = dict_set_dynstr_with_alloc (xdata,
-                                DHT_IATT_IN_XDATA_KEY, "yes");
-                        if (ret) {
-                                gf_msg_debug (this->name, 0,
-                                        "Failed to set dictionary key %s",
-                                         DHT_IATT_IN_XDATA_KEY);
-                        }
-                }
-        }
-
-        /*
-         * File is on hot tier, delete the data file first, then
-         * linkfile from cold.
-         */
-        STACK_WIND (frame, tier_unlink_cbk,
-                    cached_subvol, cached_subvol->fops->unlink, loc,
-                    xflag, xdata);
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                 dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
-
-        if (op_ret < 0)
-                goto done;
-
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
-
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_NO_MEMORY,
-                                "Memory allocation failed ");
-                        goto unwind;
-                }
-
-                entry->d_off  = orig_entry->d_off;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                list_add_tail (&entry->list, &entries.list);
-                count++;
-        }
-        op_ret = count;
-
-done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset != 0) {
-                        next_subvol = prev->this;
-                } else {
-                        goto unwind;
-                }
-
-                STACK_WIND (frame, tier_readdir_cbk,
-                            next_subvol, next_subvol->fops->readdir,
-                            local->fd, local->size, next_offset, NULL);
-                return 0;
-        }
-
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
-
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-int
-tier_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_conf_t   *conf   = NULL;
-        int           ret    = 0;
-        inode_table_t           *itable = NULL;
-        inode_t                 *inode = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
-        itable = local->fd ? local->fd->inode->table : NULL;
-
-        conf  = this->private;
-        GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
-
-        if (op_ret < 0)
-                goto done;
-
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
-
-                if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
-                        /*stat failed somewhere- ignore this entry*/
-                        continue;
-                }
-
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
-
-                        goto unwind;
-                }
-
-                entry->d_off  = orig_entry->d_off;
-                entry->d_stat = orig_entry->d_stat;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                if (orig_entry->dict)
-                        entry->dict = dict_ref (orig_entry->dict);
-
-                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict,
-                                       conf->link_xattr_name)) {
-                        inode = inode_find (itable,
-                                            orig_entry->d_stat.ia_gfid);
-                        if (inode) {
-                                ret = dht_layout_preset
-                                        (this, TIER_UNHASHED_SUBVOL,
-                                         inode);
-                                if (ret)
-                                        gf_msg (this->name,
-                                                GF_LOG_WARNING, 0,
-                                                DHT_MSG_LAYOUT_SET_FAILED,
-                                                "failed to link the layout"
-                                                " in inode");
-                                inode_unref (inode);
-                                inode = NULL;
-                        }
-
-                } else if (IA_ISDIR(entry->d_stat.ia_type)) {
-                        if (orig_entry->inode) {
-                                dht_inode_ctx_time_update (orig_entry->inode,
-                                                           this, &entry->d_stat,
-                                                           1);
-                        }
-                } else {
-                        if (orig_entry->inode) {
-                                ret = dht_layout_preset (this, prev->this,
-                                                         orig_entry->inode);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                DHT_MSG_LAYOUT_SET_FAILED,
-                                                "failed to link the layout "
-                                                "in inode");
-
-                                entry->inode = inode_ref (orig_entry->inode);
-                        } else if (itable) {
-                                /*
-                                 * orig_entry->inode might be null if any upper
-                                 * layer xlators below client set to null, to
-                                 * force a lookup on the inode even if the inode
-                                 * is present in the inode table. In that case
-                                 * we just update the ctx to make sure we didn't
-                                 * missed anything.
-                                 */
-                                inode = inode_find (itable,
-                                                    orig_entry->d_stat.ia_gfid);
-                                if (inode) {
-                                        ret = dht_layout_preset
-                                                (this, TIER_HASHED_SUBVOL,
-                                                 inode);
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                     GF_LOG_WARNING, 0,
-                                                     DHT_MSG_LAYOUT_SET_FAILED,
-                                                     "failed to link the layout"
-                                                     " in inode");
-                                        inode_unref (inode);
-                                        inode = NULL;
-                                }
-                        }
-                }
-                list_add_tail (&entry->list, &entries.list);
-                count++;
-        }
-        op_ret = count;
-
-done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset != 0) {
-                        next_subvol = prev->this;
-                } else {
-                        goto unwind;
-                }
-
-                STACK_WIND (frame, tier_readdirp_cbk,
-                            next_subvol, next_subvol->fops->readdirp,
-                            local->fd, local->size, next_offset,
-                            local->xattr);
-                return 0;
-        }
-
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
-
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-int
-tier_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop, dict_t *dict)
-{
-        dht_local_t  *local         = NULL;
-        int           op_errno      = -1;
-        xlator_t     *hashed_subvol = NULL;
-        int           ret           = 0;
-        dht_conf_t   *conf          = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, NULL, NULL, whichop);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->fd = fd_ref (fd);
-        local->size = size;
-        local->xattr_req = (dict) ? dict_ref (dict) : NULL;
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-
-        /* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIRP) {
-                if (dict)
-                        local->xattr = dict_ref (dict);
-                else
-                        local->xattr = dict_new ();
-
-                if (local->xattr) {
-                        ret = dict_set_uint32 (local->xattr,
-                                               conf->link_xattr_name, 256);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value"
-                                        " : key = %s",
-                                        conf->link_xattr_name);
-
-                }
-
-                STACK_WIND (frame, tier_readdirp_cbk, hashed_subvol,
-                            hashed_subvol->fops->readdirp,
-                            fd, size, yoff, local->xattr);
-
-        } else {
-                STACK_WIND (frame, tier_readdir_cbk, hashed_subvol,
-                            hashed_subvol->fops->readdir,
-                            fd, size, yoff, local->xattr);
-        }
-
-        return 0;
-
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff, dict_t *xdata)
-{
-        int          op = GF_FOP_READDIR;
-        dht_conf_t  *conf = NULL;
-        int          i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        op = GF_FOP_READDIRP;
-                        break;
-                }
-        }
-
-        if (conf->use_readdirp)
-                op = GF_FOP_READDIRP;
-
-out:
-        tier_do_readdir (frame, this, fd, size, yoff, op, 0);
-        return 0;
-}
-
-int
-tier_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t yoff, dict_t *dict)
-{
-        tier_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
-        return 0;
-}
diff --git a/xlators/cluster/dht/src/tier-common.h b/xlators/cluster/dht/src/tier-common.h
deleted file mode 100644
index 0ef96aca032..00000000000
--- a/xlators/cluster/dht/src/tier-common.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _TIER_COMMON_H_
-#define _TIER_COMMON_H_
-/* Function definitions */
-int
-tier_create_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int op_ret, int op_errno,
-                                     struct iatt *preparent,
-                                     struct iatt *postparent, dict_t *xdata);
-
-int
-tier_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata);
-
-int
-tier_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 inode_t *inode, struct iatt *stbuf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent,
-                                 dict_t *xdata);
-
-int
-tier_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *params);
-
-int32_t
-tier_unlink (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int xflag, dict_t *xdata);
-
-int32_t
-tier_readdirp (call_frame_t *frame,
-               xlator_t *this,
-               fd_t     *fd,
-               size_t    size, off_t off, dict_t *dict);
-
-int
-tier_readdir (call_frame_t *frame,
-              xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff, dict_t *xdata);
-
-
-
-int
-tier_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata);
-#endif
-
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
deleted file mode 100644
index 356af021563..00000000000
--- a/xlators/cluster/dht/src/tier.c
+++ /dev/null
@@ -1,2518 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <dlfcn.h>
-
-#include "dht-common.h"
-#include "tier.h"
-#include "tier-common.h"
-#include "syscall.h"
-
-/*Hard coded DB info*/
-static gfdb_db_type_t dht_tier_db_type = GFDB_SQLITE3;
-/*Hard coded DB info*/
-
-/*Mutex for updating the data movement stats*/
-static pthread_mutex_t dm_stat_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-/* Stores the path location of promotion query files */
-static char *promotion_qfile;
-/* Stores the path location of demotion query files */
-static char *demotion_qfile;
-
-static void *libhandle;
-static gfdb_methods_t gfdb_methods;
-
-#define DB_QUERY_RECORD_SIZE 4096
-
-/*
- * Closes all the fds and frees the qfile_array
- * */
-static void
-qfile_array_free (tier_qfile_array_t *qfile_array)
-{
-        ssize_t i = 0;
-
-        if (qfile_array) {
-                if (qfile_array->fd_array) {
-                        for (i = 0; i < qfile_array->array_size; i++) {
-                                if (qfile_array->fd_array[i] != -1) {
-                                        sys_close (qfile_array->fd_array[i]);
-                                }
-                        }
-                }
-                GF_FREE (qfile_array->fd_array);
-        }
-        GF_FREE (qfile_array);
-}
-
-
-/* Create a new query file list with given size */
-static tier_qfile_array_t *
-qfile_array_new (ssize_t array_size)
-{
-        int ret                         = -1;
-        tier_qfile_array_t *qfile_array   = NULL;
-        ssize_t i                       = 0;
-
-        GF_VALIDATE_OR_GOTO ("tier", (array_size > 0), out);
-
-        qfile_array = GF_CALLOC (1, sizeof (tier_qfile_array_t),
-                                gf_tier_mt_qfile_array_t);
-        if (!qfile_array) {
-                gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to allocate memory for tier_qfile_array_t");
-                goto out;
-        }
-
-        qfile_array->fd_array = GF_CALLOC (array_size, sizeof (int),
-                                        gf_dht_mt_int32_t);
-        if (!qfile_array->fd_array) {
-                gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to allocate memory for "
-                        "tier_qfile_array_t->fd_array");
-                goto out;
-        }
-
-        /* Init all the fds to -1 */
-        for (i = 0; i < array_size; i++) {
-                qfile_array->fd_array[i] = -1;
-        }
-
-        qfile_array->array_size = array_size;
-        qfile_array->next_index = 0;
-
-        /* Set exhausted count to list size as the list is empty */
-        qfile_array->exhausted_count = qfile_array->array_size;
-
-        ret = 0;
-out:
-        if (ret) {
-                qfile_array_free (qfile_array);
-                qfile_array = NULL;
-        }
-        return qfile_array;
-}
-
-
-/* Checks if the query file list is empty or totally exhausted. */
-static gf_boolean_t
-is_qfile_array_empty (tier_qfile_array_t *qfile_array)
-{
-        return (qfile_array->exhausted_count == qfile_array->array_size) ?
-                _gf_true : _gf_false;
-}
-
-
-/* Shifts the next_fd pointer to the next available fd in the list */
-static void
-shift_next_index (tier_qfile_array_t *qfile_array)
-{
-        int qfile_fd = 0;
-        int spin_count = 0;
-
-        if (is_qfile_array_empty (qfile_array)) {
-                return;
-        }
-
-        do {
-                /* change next_index in a rotional manner */
-                (qfile_array->next_index == (qfile_array->array_size - 1)) ?
-                qfile_array->next_index = 0 : qfile_array->next_index++;
-
-                qfile_fd = (qfile_array->fd_array[qfile_array->next_index]);
-
-                spin_count++;
-
-        } while ((qfile_fd == -1) && (spin_count < qfile_array->array_size));
-
-}
-
-/*
- * This is a non-thread safe function to read query records
- * from a list of query files in a Round-Robin manner.
- * As in when the query files get exhuasted they are closed.
- * Returns:
- * 0   if all the query records in all the query files of the list are
- *     exhausted.
- * > 0 if a query record is successfully read. Indicates the size of the query
- *     record read.
- * < 0  if there was failure
- * */
-static int
-read_query_record_list (tier_qfile_array_t *qfile_array,
-                          gfdb_query_record_t **query_record)
-{
-        int ret = -1;
-        int qfile_fd = 0;
-
-        GF_VALIDATE_OR_GOTO ("tier", qfile_array, out);
-        GF_VALIDATE_OR_GOTO ("tier", qfile_array->fd_array, out);
-
-        do {
-                if (is_qfile_array_empty (qfile_array)) {
-                        ret = 0;
-                        break;
-                }
-
-                qfile_fd = qfile_array->fd_array[qfile_array->next_index];
-                ret = gfdb_methods.gfdb_read_query_record
-                        (qfile_fd, query_record);
-                if (ret <= 0) {
-                        /*The qfile_fd has reached EOF or
-                         * there was an error.
-                         * 1. Close the exhausted fd
-                         * 2. increment the exhausted count
-                         * 3. shift next_qfile to next qfile
-                         **/
-                         sys_close (qfile_fd);
-                         qfile_array->fd_array[qfile_array->next_index] = -1;
-                         qfile_array->exhausted_count++;
-                         /* shift next_qfile to next qfile */
-                         shift_next_index (qfile_array);
-                         continue;
-                } else {
-                        /* shift next_qfile to next qfile */
-                        shift_next_index (qfile_array);
-                        break;
-                }
-        } while (1);
-out:
-        return ret;
-}
-
-
-/* Check and update the watermark every WM_INTERVAL seconds */
-#define WM_INTERVAL            5
-
-static int
-tier_check_same_node (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
-{
-        int     ret            = -1;
-        dict_t *dict           = NULL;
-        char   *uuid_str       = NULL;
-        uuid_t  node_uuid      = {0,};
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-
-        if (syncop_getxattr (this, loc, &dict, GF_XATTR_NODE_UUID_KEY,
-                             NULL, NULL)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Unable to get NODE_UUID_KEY %s %s\n",
-                        loc->name, loc->path);
-                goto out;
-        }
-
-        if (dict_get_str (dict, GF_XATTR_NODE_UUID_KEY, &uuid_str) < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to get node-uuid for %s", loc->path);
-                goto out;
-        }
-
-        if (gf_uuid_parse (uuid_str, node_uuid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "uuid_parse failed for %s", loc->path);
-                goto out;
-        }
-
-        if (gf_uuid_compare (node_uuid, defrag->node_uuid)) {
-                gf_msg_debug (this->name, 0,
-                              "%s does not belong to this node", loc->path);
-                ret = 1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        if (dict)
-                dict_unref(dict);
-
-        return ret;
-}
-
-int
-tier_check_watermark (xlator_t *this, loc_t *root_loc)
-{
-        tier_watermark_op_t     wm = TIER_WM_NONE;
-        int                     ret = -1;
-        gf_defrag_info_t       *defrag = NULL;
-        dht_conf_t             *conf   = NULL;
-        dict_t                 *xdata  = NULL;
-        struct statvfs          statfs = {0, };
-        gf_tier_conf_t         *tier_conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto exit;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto exit;
-
-        tier_conf = &defrag->tier_conf;
-
-        if (tier_conf->mode != TIER_MODE_WM) {
-                ret = 0;
-                goto exit;
-        }
-
-        xdata = dict_new ();
-        if (!xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY,
-                        "failed to allocate dictionary");
-                ret = -1;
-                goto exit;
-        }
-
-        ret = dict_set_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set "
-                        GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                ret = -1;
-                goto exit;
-        }
-
-        /* Find how much free space is on the hot subvolume.
-         * Then see if that value */
-        /* is less than or greater than user defined watermarks.
-         * Stash results in */
-        /* the tier_conf data structure. */
-
-        ret = syncop_statfs (conf->subvolumes[1], root_loc, &statfs,
-                             xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Unable to obtain statfs.");
-                goto exit;
-        }
-
-        pthread_mutex_lock (&dm_stat_mutex);
-
-        tier_conf->blocks_total = statfs.f_blocks;
-        tier_conf->blocks_used = statfs.f_blocks - statfs.f_bfree;
-
-        tier_conf->percent_full = (100 * tier_conf->blocks_used) /
-                statfs.f_blocks;
-        pthread_mutex_unlock (&dm_stat_mutex);
-
-        if (tier_conf->percent_full < tier_conf->watermark_low) {
-                wm = TIER_WM_LOW;
-
-        } else if (tier_conf->percent_full < tier_conf->watermark_hi) {
-                wm = TIER_WM_MID;
-
-        } else {
-                wm = TIER_WM_HI;
-        }
-
-        if (wm != tier_conf->watermark_last) {
-
-                tier_conf->watermark_last = wm;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Tier watermark now %d", wm);
-        }
-
-exit:
-        if (xdata)
-                dict_unref (xdata);
-        return ret;
-}
-
-
-static gf_boolean_t
-is_hot_tier_full (gf_tier_conf_t *tier_conf)
-{
-        if (tier_conf && (tier_conf->mode == TIER_MODE_WM) &&
-           (tier_conf->watermark_last == TIER_WM_HI))
-                return _gf_true;
-
-        return _gf_false;
-}
-
-int
-tier_do_migration (xlator_t *this, int promote, loc_t *root_loc)
-{
-        gf_defrag_info_t       *defrag = NULL;
-        dht_conf_t             *conf   = NULL;
-        long                    rand = 0;
-        int                     migrate = 0;
-        gf_tier_conf_t         *tier_conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto exit;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto exit;
-
-        if (defrag->tier_conf.mode != TIER_MODE_WM) {
-                migrate = 1;
-                goto exit;
-        }
-
-        if (tier_check_watermark (this, root_loc) != 0) {
-                gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to get watermark");
-                goto exit;
-        }
-
-        tier_conf = &defrag->tier_conf;
-
-        switch (tier_conf->watermark_last) {
-        case TIER_WM_LOW:
-                migrate = promote ? 1 : 0;
-                break;
-        case TIER_WM_HI:
-                migrate = promote ? 0 : 1;
-                break;
-        case TIER_WM_MID:
-                rand = random() % 100;
-                if (promote) {
-                        migrate = (rand > tier_conf->percent_full);
-                } else {
-                        migrate = (rand <= tier_conf->percent_full);
-                }
-                break;
-        }
-
-exit:
-        return migrate;
-}
-
-int
-tier_migrate (xlator_t *this, int is_promotion, dict_t *migrate_data,
-              loc_t *loc, gf_tier_conf_t *tier_conf)
-{
-        int ret = -1;
-
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        if (is_promotion)
-                tier_conf->promote_in_progress = 1;
-        else
-                tier_conf->demote_in_progress = 1;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-
-        /* Data migration */
-        ret = syncop_setxattr (this, loc, migrate_data, 0,
-                               NULL, NULL);
-
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        if (is_promotion)
-                tier_conf->promote_in_progress = 0;
-        else
-                tier_conf->demote_in_progress = 0;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-
-        return ret;
-}
-
-static int
-tier_migrate_using_query_file (void *_args)
-{
-        int ret                                 = -1;
-        query_cbk_args_t *query_cbk_args        = (query_cbk_args_t *) _args;
-        xlator_t *this                          = NULL;
-        gf_defrag_info_t *defrag                = NULL;
-        gfdb_query_record_t *query_record       = NULL;
-        gfdb_link_info_t *link_info             = NULL;
-        struct iatt par_stbuf                   = {0,};
-        struct iatt current                     = {0,};
-        loc_t p_loc                             = {0,};
-        loc_t loc                               = {0,};
-        dict_t *migrate_data                    = NULL;
-        dict_t *xdata_request                   = NULL;
-        dict_t *xdata_response                  = NULL;
-        char *parent_path                       = NULL;
-        inode_t *linked_inode                   = NULL;
-        /*
-         * per_file_status and per_link_status
-         *  0  : success
-         * -1 : failure
-         *  1  : ignore the status and dont count for migration
-         * */
-        int per_file_status                     = 0;
-        int per_link_status                     = 0;
-        int total_status                        = 0;
-        xlator_t *src_subvol                    = NULL;
-        dht_conf_t   *conf                      = NULL;
-        uint64_t total_migrated_bytes           = 0;
-        int total_files                         = 0;
-        loc_t root_loc                          = { 0 };
-        gfdb_time_t  start_time                 = { 0 };
-        gfdb_time_t  current_time               = { 0 };
-        int total_time                          = 0;
-        int max_time                            = 0;
-
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-        GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->defrag, out);
-        GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->qfile_array, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        conf = this->private;
-
-        defrag = query_cbk_args->defrag;
-        migrate_data = dict_new ();
-        if (!migrate_data)
-                goto out;
-
-        xdata_request = dict_new ();
-        if (!xdata_request) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to create xdata_request dict");
-                goto out;
-        }
-        ret = dict_set_int32 (xdata_request,
-                              GET_ANCESTRY_PATH_KEY, 42);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to set value to dict : key %s \n",
-                        GET_ANCESTRY_PATH_KEY);
-                goto out;
-        }
-
-        dht_build_root_loc (defrag->root_inode, &root_loc);
-
-        ret = gettimeofday (&start_time, NULL);
-        if (query_cbk_args->is_promotion) {
-                max_time = defrag->tier_conf.tier_promote_frequency;
-        } else {
-                max_time = defrag->tier_conf.tier_demote_frequency;
-        }
-
-        /* Per file */
-        while ((ret = read_query_record_list (query_cbk_args->qfile_array,
-              &query_record)) != 0) {
-
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Failed to fetch query record "
-                                "from query file");
-                        goto out;
-                }
-
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Exiting tier migration as"
-                                "defrag status is not started");
-                        goto out;
-                }
-
-                ret = gettimeofday (&current_time, NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Could not get current time.");
-                        goto out;
-                }
-
-                total_time = current_time.tv_sec - start_time.tv_sec;
-                if (total_time > max_time) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS,
-                                "Max cycle time reached. Exiting migration.");
-                        goto out;
-                }
-
-                per_file_status      = 0;
-                per_link_status      = 0;
-
-                dict_del (migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
-
-                dict_del (migrate_data, "from.migrator");
-
-                if (gf_defrag_get_pause_state (&defrag->tier_conf)
-                    != TIER_RUNNING) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS,
-                                "Tiering paused. "
-                                "Exiting tier_migrate_using_query_file");
-                        break;
-                }
-
-                if (!tier_do_migration (this, query_cbk_args->is_promotion, &root_loc)) {
-                        gfdb_methods.gfdb_query_record_free (query_record);
-                        query_record = NULL;
-
-                        /* We have crossed the high watermark. Stop processing
-                         * files if this is a promotion cycle so demotion gets
-                         * a chance to start if not already running*/
-
-                        if (query_cbk_args->is_promotion &&
-                            is_hot_tier_full (&defrag->tier_conf)) {
-
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "High watermark crossed during "
-                                        "promotion. Exiting "
-                                        "tier_migrate_using_query_file");
-                                break;
-                        }
-                        continue;
-                }
-
-                if (!list_empty (&query_record->link_list)) {
-                        per_file_status =
-                                dict_set_str (migrate_data,
-                                              GF_XATTR_FILE_MIGRATE_KEY,
-                                              "force");
-                        if (per_file_status) {
-                                goto per_file_out;
-                        }
-
-                        /* Flag to suggest the xattr call is from migrator */
-                        per_file_status = dict_set_str (migrate_data,
-                                                        "from.migrator", "yes");
-                        if (per_file_status) {
-                                goto per_file_out;
-                        }
-
-                        /* Flag to suggest its a tiering migration
-                         * The reason for this dic key-value is that
-                         * promotions and demotions are multithreaded
-                         * so the original frame from gf_defrag_start()
-                         * is not carried. A new frame will be created when
-                         * we do syncop_setxattr(). This doesnot have the
-                         * frame->root->pid of the original frame. So we pass
-                         * this dic key-value when we do syncop_setxattr() to do
-                         * data migration and set the frame->root->pid to
-                         * GF_CLIENT_PID_TIER_DEFRAG in dht_setxattr() just before
-                         * calling dht_start_rebalance_task() */
-                        per_file_status = dict_set_str (migrate_data,
-                                                TIERING_MIGRATION_KEY, "yes");
-                        if (per_file_status) {
-                                goto per_file_out;
-                        }
-
-                }
-                per_link_status = 0;
-
-                /* For now we only support single link migration. And we will
-                 * ignore other hard links in the link info list of query record
-                 * TODO: Multiple hard links migration */
-                if (!list_empty (&query_record->link_list)) {
-                        link_info = list_first_entry (&query_record->link_list,
-                                                      gfdb_link_info_t, list);
-                }
-                if (link_info != NULL) {
-
-                        /* Lookup for parent and get the path of parent */
-                        gf_uuid_copy (p_loc.gfid, link_info->pargfid);
-                        p_loc.inode = inode_new (defrag->root_inode->table);
-                        if (!p_loc.inode) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Failed to create reference to inode"
-                                        " for %s", uuid_utoa (p_loc.gfid));
-
-                                per_link_status = -1;
-                                goto abort;
-                        }
-
-                        ret = syncop_lookup (this, &p_loc, &par_stbuf, NULL,
-                                             xdata_request, &xdata_response);
-                        /* When the parent gfid is a stale entry, the lookup
-                         * will fail and stop the demotion process.
-                         * The parent gfid can be stale when a huge folder is
-                         * deleted while the files within it are being migrated
-                         */
-                        if (ret == -ESTALE) {
-                                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                        DHT_MSG_STALE_LOOKUP,
-                                        "Stale entry in parent lookup for %s",
-                                        uuid_utoa (p_loc.gfid));
-                                per_link_status = 1;
-                                goto abort;
-                        } else if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Error in parent lookup for %s",
-                                        uuid_utoa (p_loc.gfid));
-                                per_link_status = -1;
-                                goto abort;
-                        }
-                        ret = dict_get_str (xdata_response,
-                                            GET_ANCESTRY_PATH_KEY,
-                                            &parent_path);
-                        if (ret || !parent_path) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Failed to get parent path for %s",
-                                        uuid_utoa (p_loc.gfid));
-                                per_link_status = -1;
-                                goto abort;
-                        }
-
-                        linked_inode = inode_link (p_loc.inode, NULL, NULL,
-                                                        &par_stbuf);
-                        inode_unref (p_loc.inode);
-                        p_loc.inode = linked_inode;
-
-
-                        /* Preparing File Inode */
-                        gf_uuid_copy (loc.gfid, query_record->gfid);
-                        loc.inode = inode_new (defrag->root_inode->table);
-                        gf_uuid_copy (loc.pargfid, link_info->pargfid);
-                        loc.parent = inode_ref (p_loc.inode);
-
-                        /* Get filename and Construct file path */
-                        loc.name = gf_strdup (link_info->file_name);
-                        if (!loc.name) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR, "Memory "
-                                        "allocation failed for %s",
-                                        uuid_utoa (query_record->gfid));
-                                per_link_status = -1;
-                                goto abort;
-                        }
-                        ret = gf_asprintf((char **)&(loc.path), "%s/%s",
-                                          parent_path, loc.name);
-                        if (ret < 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR, "Failed to "
-                                        "construct file path for %s %s\n",
-                                        parent_path, loc.name);
-                                per_link_status = -1;
-                                goto abort;
-                        }
-
-                        gf_uuid_copy (loc.parent->gfid, link_info->pargfid);
-
-                        /* lookup file inode */
-                        ret = syncop_lookup (this, &loc, &current, NULL,
-                                             NULL, NULL);
-                        /* The file may be deleted even when the parent
-                         * is available and the lookup will
-                         * return a stale entry which would stop the
-                         * migration. so if its a stale entry, then skip
-                         * the file and keep migrating.
-                         */
-                        if (ret == -ESTALE) {
-                                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                        DHT_MSG_STALE_LOOKUP,
-                                        "Stale lookup for %s",
-                                        uuid_utoa (p_loc.gfid));
-                                per_link_status = 1;
-                                goto abort;
-                        } else if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                        DHT_MSG_LOG_TIER_ERROR, "Failed to "
-                                        "lookup file %s\n", loc.name);
-                                per_link_status = -1;
-                                goto abort;
-                        }
-
-                        if (query_cbk_args->is_promotion &&
-                            defrag->tier_conf.tier_max_promote_size &&
-                            (current.ia_size > defrag->tier_conf.tier_max_promote_size)) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "File size exceeds maxsize for promotion. ");
-                                per_link_status = 1;
-                                goto abort;
-                        }
-
-                        linked_inode = inode_link (loc.inode, NULL, NULL,
-                                                        &current);
-                        inode_unref (loc.inode);
-                        loc.inode = linked_inode;
-
-
-                        /*
-                         * Do not promote/demote if file already is where it
-                         * should be. It means another brick moved the file
-                         * so is not an error. So we set per_link_status = 1
-                         * so that we ignore counting this.
-                         */
-                        src_subvol = dht_subvol_get_cached (this, loc.inode);
-
-                        if (src_subvol == NULL) {
-                                per_link_status = 1;
-                                goto abort;
-                        }
-                        if (query_cbk_args->is_promotion &&
-                             src_subvol == conf->subvolumes[1]) {
-                                per_link_status = 1;
-                                goto abort;
-                        }
-
-                        if (!query_cbk_args->is_promotion &&
-                            src_subvol == conf->subvolumes[0]) {
-                                per_link_status = 1;
-                                goto abort;
-                        }
-
-                        gf_msg_debug (this->name, 0,
-                                      "Tier %s: src_subvol %s file %s",
-                                      (query_cbk_args->is_promotion ?
-                                      "promote" : "demote"),
-                                      src_subvol->name,
-                                      loc.path);
-
-
-                        ret = tier_check_same_node (this, &loc, defrag);
-                        if (ret != 0) {
-                                if (ret < 0) {
-                                        per_link_status = -1;
-                                        goto abort;
-                                }
-                                ret = 0;
-                                /* By setting per_link_status to 1 we are
-                                 * ignoring this status and will not be counting
-                                 * this file for migration */
-                                per_link_status = 1;
-                                goto abort;
-                        }
-
-                        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-
-                        if (gf_defrag_get_pause_state (&defrag->tier_conf)
-                            != TIER_RUNNING) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "Tiering paused. "
-                                        "Exiting "
-                                        "tier_migrate_using_query_file");
-                                goto abort;
-                        }
-
-                        ret = tier_migrate (this, query_cbk_args->is_promotion,
-                                            migrate_data, &loc, &defrag->tier_conf);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                        DHT_MSG_LOG_TIER_ERROR, "Failed to "
-                                        "migrate %s ", loc.path);
-                                per_link_status = -1;
-                                goto abort;
-                        }
-
-                        if (query_cbk_args->is_promotion) {
-                                defrag->total_files_promoted++;
-                                total_migrated_bytes +=
-                                        defrag->tier_conf.st_last_promoted_size;
-                                pthread_mutex_lock (&dm_stat_mutex);
-                                defrag->tier_conf.blocks_used +=
-                                        defrag->tier_conf.st_last_promoted_size;
-                                pthread_mutex_unlock (&dm_stat_mutex);
-                        } else {
-                                defrag->total_files_demoted++;
-                                total_migrated_bytes +=
-                                        defrag->tier_conf.st_last_demoted_size;
-                                pthread_mutex_lock (&dm_stat_mutex);
-                                defrag->tier_conf.blocks_used -=
-                                        defrag->tier_conf.st_last_demoted_size;
-                                pthread_mutex_unlock (&dm_stat_mutex);
-                        }
-                        if (defrag->tier_conf.blocks_total) {
-                                pthread_mutex_lock (&dm_stat_mutex);
-                                defrag->tier_conf.percent_full =
-                                        (100 * defrag->tier_conf.blocks_used) /
-                                        defrag->tier_conf.blocks_total;
-                                pthread_mutex_unlock (&dm_stat_mutex);
-                        }
-                        total_files++;
-abort:
-                        GF_FREE ((char *) loc.name);
-                        loc.name = NULL;
-                        loc_wipe (&loc);
-                        loc_wipe (&p_loc);
-
-
-                        if (xdata_response) {
-                                dict_unref (xdata_response);
-                                xdata_response = NULL;
-                        }
-
-                        if ((total_files >= defrag->tier_conf.max_migrate_files)
-                            || (total_migrated_bytes >
-                                defrag->tier_conf.max_migrate_bytes)) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "Reached cycle migration limit."
-                                        "migrated bytes %"PRId64" files %d",
-                                        total_migrated_bytes,
-                                        total_files);
-                                goto out;
-                        }
-                }
-                per_file_status = per_link_status;
-per_file_out:
-                if (per_file_status < 0) {/* Failure */
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->total_failures++;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                } else if (per_file_status == 0) {/* Success */
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->total_files++;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                } else if (per_file_status == 1) {/* Ignore */
-                        per_file_status = 0;
-                        /* Since this attempt was ignored we
-                         * decrement the lookup count*/
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->num_files_lookedup--;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                }
-                total_status = total_status + per_file_status;
-                per_link_status = 0;
-                per_file_status = 0;
-
-                gfdb_methods.gfdb_query_record_free (query_record);
-                query_record = NULL;
-        }
-
-out:
-        if (xdata_request) {
-                dict_unref (xdata_request);
-        }
-
-        if (migrate_data)
-                dict_unref (migrate_data);
-
-
-        gfdb_methods.gfdb_query_record_free (query_record);
-        query_record = NULL;
-
-        return total_status;
-}
-
-
-/* This is the call back function per record/file from data base */
-static int
-tier_gf_query_callback (gfdb_query_record_t *gfdb_query_record,
-                        void *_args) {
-        int ret = -1;
-        query_cbk_args_t *query_cbk_args = _args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->defrag, out);
-        GF_VALIDATE_OR_GOTO ("tier", (query_cbk_args->query_fd > 0), out);
-
-        ret = gfdb_methods.gfdb_write_query_record (query_cbk_args->query_fd,
-                                                        gfdb_query_record);
-        if (ret) {
-                gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed writing query record to query file");
-                goto out;
-        }
-
-        pthread_mutex_lock (&dm_stat_mutex);
-        query_cbk_args->defrag->num_files_lookedup++;
-        pthread_mutex_unlock (&dm_stat_mutex);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-
-
-/* Create query file in tier process */
-static int
-tier_process_self_query (tier_brick_list_t *local_brick, void *args)
-{
-        int ret                                         = -1;
-        char *db_path                                   = NULL;
-        query_cbk_args_t *query_cbk_args                = NULL;
-        xlator_t *this                                  = NULL;
-        gfdb_conn_node_t *conn_node                     = NULL;
-        dict_t *params_dict                             = NULL;
-        dict_t *ctr_ipc_dict                            = NULL;
-        gfdb_brick_info_t *gfdb_brick_info              = args;
-
-        /*Init of all the essentials*/
-        GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_info , out);
-        query_cbk_args = gfdb_brick_info->_query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-
-        GF_VALIDATE_OR_GOTO (this->name,
-                             gfdb_brick_info->_query_cbk_args, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->xlator, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->brick_db_path, out);
-
-        db_path = local_brick->brick_db_path;
-
-        /*Preparing DB parameters before init_db i.e getting db connection*/
-        params_dict = dict_new ();
-        if (!params_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "DB Params cannot initialized");
-                goto out;
-        }
-        SET_DB_PARAM_TO_DICT(this->name, params_dict,
-                             (char *) gfdb_methods.get_db_path_key(),
-                             db_path, ret, out);
-
-        /*Get the db connection*/
-        conn_node = gfdb_methods.init_db ((void *)params_dict, dht_tier_db_type);
-        if (!conn_node) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "FATAL: Failed initializing db operations");
-                         goto out;
-        }
-
-        /* Query for eligible files from db */
-        query_cbk_args->query_fd = open (local_brick->qfile_path,
-                        O_WRONLY | O_CREAT | O_APPEND,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (query_cbk_args->query_fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to open query file %s",
-                        local_brick->qfile_path);
-                goto out;
-        }
-        if (!gfdb_brick_info->_gfdb_promote) {
-                if (query_cbk_args->defrag->write_freq_threshold == 0 &&
-                        query_cbk_args->defrag->read_freq_threshold == 0) {
-                                ret = gfdb_methods.find_unchanged_for_time (
-                                        conn_node,
-                                        tier_gf_query_callback,
-                                        (void *)query_cbk_args,
-                                        gfdb_brick_info->time_stamp);
-                } else {
-                                ret = gfdb_methods.find_unchanged_for_time_freq (
-                                        conn_node,
-                                        tier_gf_query_callback,
-                                        (void *)query_cbk_args,
-                                        gfdb_brick_info->time_stamp,
-                                        query_cbk_args->defrag->
-                                                        write_freq_threshold,
-                                        query_cbk_args->defrag->
-                                                        read_freq_threshold,
-                                        _gf_false);
-                }
-        } else {
-                if (query_cbk_args->defrag->write_freq_threshold == 0 &&
-                        query_cbk_args->defrag->read_freq_threshold == 0) {
-                        ret = gfdb_methods.find_recently_changed_files (
-                                conn_node,
-                                tier_gf_query_callback,
-                                (void *)query_cbk_args,
-                                gfdb_brick_info->time_stamp);
-                } else {
-                        ret = gfdb_methods.find_recently_changed_files_freq (
-                                conn_node,
-                                tier_gf_query_callback,
-                                (void *)query_cbk_args,
-                                gfdb_brick_info->time_stamp,
-                                query_cbk_args->defrag->
-                                write_freq_threshold,
-                                query_cbk_args->defrag->read_freq_threshold,
-                                _gf_false);
-                }
-        }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "FATAL: query from db failed");
-                        goto out;
-        }
-
-        /*Clear the heat on the DB entries*/
-        /*Preparing ctr_ipc_dict*/
-        ctr_ipc_dict = dict_new ();
-        if (!ctr_ipc_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "ctr_ipc_dict cannot initialized");
-                goto out;
-        }
-
-        SET_DB_PARAM_TO_DICT(this->name, ctr_ipc_dict,
-                             GFDB_IPC_CTR_KEY, GFDB_IPC_CTR_CLEAR_OPS,
-                             ret, out);
-
-        ret = syncop_ipc (local_brick->xlator, GF_IPC_TARGET_CTR, ctr_ipc_dict,
-                                                                        NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed clearing the heat "
-                        "on db %s error %d", local_brick->brick_db_path, ret);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        if (params_dict) {
-                dict_unref (params_dict);
-                params_dict = NULL;
-        }
-
-        if (ctr_ipc_dict) {
-                dict_unref (ctr_ipc_dict);
-                ctr_ipc_dict = NULL;
-        }
-
-        if (query_cbk_args && query_cbk_args->query_fd >= 0) {
-                sys_close (query_cbk_args->query_fd);
-                query_cbk_args->query_fd = -1;
-        }
-        gfdb_methods.fini_db (conn_node);
-
-        return ret;
-}
-
-
-
-
-
-/*Ask CTR to create the query file*/
-static int
-tier_process_ctr_query (tier_brick_list_t *local_brick, void *args)
-{
-        int ret                                         = -1;
-        query_cbk_args_t *query_cbk_args                = NULL;
-        xlator_t *this                                  = NULL;
-        dict_t *ctr_ipc_in_dict                         = NULL;
-        dict_t *ctr_ipc_out_dict                        = NULL;
-        gfdb_brick_info_t *gfdb_brick_info              = args;
-        gfdb_ipc_ctr_params_t *ipc_ctr_params           = NULL;
-        int count                                       = 0;
-
-        /*Init of all the essentials*/
-        GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_info , out);
-        query_cbk_args = gfdb_brick_info->_query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-
-        GF_VALIDATE_OR_GOTO (this->name,
-                             gfdb_brick_info->_query_cbk_args, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->xlator, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->brick_db_path, out);
-
-
-        /*Preparing ctr_ipc_in_dict*/
-        ctr_ipc_in_dict = dict_new ();
-        if (!ctr_ipc_in_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "ctr_ipc_in_dict cannot initialized");
-                goto out;
-        }
-
-        ipc_ctr_params = GF_CALLOC (1, sizeof (gfdb_ipc_ctr_params_t),
-                                        gf_tier_mt_ipc_ctr_params_t);
-        if (!ipc_ctr_params) {
-                goto out;
-        }
-
-        /* set all the query params*/
-        ipc_ctr_params->is_promote = gfdb_brick_info->_gfdb_promote;
-        ipc_ctr_params->write_freq_threshold = query_cbk_args->
-                                                defrag->write_freq_threshold;
-        ipc_ctr_params->read_freq_threshold = query_cbk_args->
-                                                defrag->read_freq_threshold;
-        memcpy (&ipc_ctr_params->time_stamp,
-                gfdb_brick_info->time_stamp,
-                sizeof (gfdb_time_t));
-
-        SET_DB_PARAM_TO_DICT(this->name, ctr_ipc_in_dict,
-                             GFDB_IPC_CTR_KEY, GFDB_IPC_CTR_QUERY_OPS,
-                             ret, out);
-
-
-        SET_DB_PARAM_TO_DICT(this->name, ctr_ipc_in_dict,
-                             GFDB_IPC_CTR_GET_QFILE_PATH,
-                             local_brick->qfile_path,
-                             ret, out);
-
-        ret = dict_set_bin (ctr_ipc_in_dict, GFDB_IPC_CTR_GET_QUERY_PARAMS,
-                                ipc_ctr_params, sizeof (*ipc_ctr_params));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, LG_MSG_SET_PARAM_FAILED,
-                        "Failed setting %s to params dictionary",
-                        GFDB_IPC_CTR_GET_QUERY_PARAMS);
-                GF_FREE (ipc_ctr_params);
-                goto out;
-        }
-
-        ret = syncop_ipc (local_brick->xlator, GF_IPC_TARGET_CTR,
-                                ctr_ipc_in_dict, &ctr_ipc_out_dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_IPC_TIER_ERROR, "Failed query on %s ret %d",
-                        local_brick->brick_db_path, ret);
-                goto out;
-        }
-
-        ret = dict_get_int32(ctr_ipc_out_dict, GFDB_IPC_CTR_RET_QUERY_COUNT,
-                                &count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed getting count "
-                        "of records on %s",
-                        local_brick->brick_db_path);
-                goto out;
-        }
-
-        if (count < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed query on %s",
-                        local_brick->brick_db_path);
-                ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&dm_stat_mutex);
-        query_cbk_args->defrag->num_files_lookedup = count;
-        pthread_mutex_unlock (&dm_stat_mutex);
-
-        ret = 0;
-out:
-
-        if (ctr_ipc_in_dict) {
-                dict_unref(ctr_ipc_in_dict);
-                ctr_ipc_in_dict = NULL;
-        }
-
-        if (ctr_ipc_out_dict) {
-                dict_unref(ctr_ipc_out_dict);
-                ctr_ipc_out_dict = NULL;
-        }
-
-        return ret;
-}
-
-
-
-
-/* This is the call back function for each brick from hot/cold bricklist
- * It picks up each bricks db and queries for eligible files for migration.
- * The list of eligible files are populated in appropriate query files*/
-static int
-tier_process_brick (tier_brick_list_t *local_brick, void *args) {
-        int ret = -1;
-        dict_t *ctr_ipc_in_dict = NULL;
-        dict_t *ctr_ipc_out_dict = NULL;
-        char *strval = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", local_brick, out);
-
-        GF_VALIDATE_OR_GOTO ("tier", local_brick->xlator, out);
-
-        if (dht_tier_db_type == GFDB_SQLITE3) {
-
-                /*Preparing ctr_ipc_in_dict*/
-                ctr_ipc_in_dict = dict_new ();
-                if (!ctr_ipc_in_dict) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "ctr_ipc_in_dict cannot initialized");
-                        goto out;
-                }
-
-                ret = dict_set_str (ctr_ipc_in_dict, GFDB_IPC_CTR_KEY,
-                                    GFDB_IPC_CTR_GET_DB_PARAM_OPS);
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,\
-                                LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                                "to params dictionary", GFDB_IPC_CTR_KEY);
-                        goto out;
-                }
-
-                ret = dict_set_str (ctr_ipc_in_dict,
-                                    GFDB_IPC_CTR_GET_DB_PARAM_OPS, "");
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,\
-                                LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                                "to params dictionary",
-                                GFDB_IPC_CTR_GET_DB_PARAM_OPS);
-                        goto out;
-                }
-
-                ret = dict_set_str (ctr_ipc_in_dict,
-                                    GFDB_IPC_CTR_GET_DB_KEY, "journal_mode");
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                                "to params dictionary",
-                                GFDB_IPC_CTR_GET_DB_KEY);\
-                        goto out;
-                }
-
-
-
-                ret = syncop_ipc (local_brick->xlator, GF_IPC_TARGET_CTR,
-                                ctr_ipc_in_dict, &ctr_ipc_out_dict);
-                if (ret || ctr_ipc_out_dict == NULL) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR, "Failed to get "
-                                "journal_mode of sql db %s",
-                                local_brick->brick_db_path);
-                        goto out;
-                }
-
-                ret = dict_get_str (ctr_ipc_out_dict, "journal_mode", &strval);
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                LG_MSG_GET_PARAM_FAILED, "Failed to get %s "
-                                "from params dictionary"
-                                "journal_mode", strval);
-                        goto out;
-                }
-
-                if (strval && (strncmp(strval, "wal", strlen ("wal")) == 0)) {
-                        ret = tier_process_self_query (local_brick, args);
-                        if (ret) {
-                                goto out;
-                        }
-                } else {
-                        ret = tier_process_ctr_query (local_brick, args);
-                        if (ret) {
-                                goto out;
-                        }
-                }
-                ret = 0;
-
-        } else {
-                ret = tier_process_self_query (local_brick, args);
-                if (ret) {
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        if  (ctr_ipc_in_dict)
-                dict_unref (ctr_ipc_in_dict);
-
-        if  (ctr_ipc_out_dict)
-                dict_unref (ctr_ipc_out_dict);
-
-        return ret;
-}
-
-
-
-
-static int
-tier_build_migration_qfile (migration_args_t *args,
-                            query_cbk_args_t *query_cbk_args,
-                            gf_boolean_t is_promotion)
-{
-        gfdb_time_t               current_time;
-        gfdb_brick_info_t         gfdb_brick_info;
-        gfdb_time_t               time_in_past;
-        int                       ret = -1;
-        tier_brick_list_t         *local_brick = NULL;
-        int                       i = 0;
-        time_in_past.tv_sec = args->freq_time;
-        time_in_past.tv_usec = 0;
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_msg (args->this->name, GF_LOG_ERROR, errno,
-                        DHT_MSG_SYS_CALL_GET_TIME_FAILED,
-                        "Failed to get current time");
-                goto out;
-        }
-        time_in_past.tv_sec = current_time.tv_sec - time_in_past.tv_sec;
-
-        /* The migration daemon may run a varrying numberof usec after the sleep */
-        /* call triggers. A file may be registered in CTR some number of usec X */
-        /* after the daemon started and missed in the subsequent cycle if the */
-        /* daemon starts Y usec after the period in seconds where Y>X. Normalize */
-        /* away this problem by always setting usec to 0. */
-        time_in_past.tv_usec = 0;
-
-        gfdb_brick_info.time_stamp = &time_in_past;
-        gfdb_brick_info._gfdb_promote = is_promotion;
-        gfdb_brick_info._query_cbk_args = query_cbk_args;
-
-        list_for_each_entry (local_brick, args->brick_list, list) {
-
-                /* Construct query file path for this brick
-                 * i.e
-                 * /var/run/gluster/xlator_name/
-                 * {promote/demote}-brickname-indexinbricklist
-                 * So that no two query files will have same path even
-                 * bricks have the same name
-                 * */
-                snprintf (local_brick->qfile_path, PATH_MAX , "%s-%s-%d",
-                        GET_QFILE_PATH (gfdb_brick_info._gfdb_promote),
-                        local_brick->brick_name, i);
-
-                /* Delete any old query files for this brick */
-                sys_unlink (local_brick->qfile_path);
-
-                ret = tier_process_brick (local_brick,
-                                          &gfdb_brick_info);
-                if (ret) {
-                        gf_msg (args->this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_BRICK_QUERY_FAILED,
-                                "Brick %s query failed\n",
-                                local_brick->brick_db_path);
-                }
-                i++;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-static int
-tier_migrate_files_using_qfile (migration_args_t *comp,
-                                query_cbk_args_t *query_cbk_args)
-{
-        int ret                                 = -1;
-        tier_brick_list_t *local_brick          = NULL;
-        tier_brick_list_t *temp                 = NULL;
-        char query_file_path_err[PATH_MAX]      = "";
-        struct tm tm                            = {0};
-        gfdb_time_t current_time                = {0};
-        char time_str[256]                      = {0};
-        char time_format[20]                    = "%Y-%m-%d-%H-%M-%S";
-        ssize_t qfile_array_size                = 0;
-        int count                               = 0;
-        int temp_fd                             = 0;
-        gf_tier_conf_t  *tier_conf              = NULL;
-
-        tier_conf = &(query_cbk_args->defrag->tier_conf);
-
-        /* Time format for error query files */
-        gettimeofday (&current_time, NULL);
-        gmtime_r (&current_time.tv_sec, &tm);
-        strftime (time_str, 256, time_format, &tm);
-
-        /* Build the qfile list */
-        list_for_each_entry_safe (local_brick, temp, comp->brick_list, list) {
-                qfile_array_size++;
-        }
-        query_cbk_args->qfile_array = qfile_array_new (qfile_array_size);
-        if (!query_cbk_args->qfile_array) {
-                gf_msg ("tier", GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed to create new "
-                        "qfile_array");
-                goto out;
-        }
-
-        /*Open all qfiles*/
-        count = 0;
-        query_cbk_args->qfile_array->exhausted_count = 0;
-        list_for_each_entry_safe (local_brick, temp, comp->brick_list, list) {
-                temp_fd = query_cbk_args->qfile_array->fd_array[count];
-                temp_fd = open (local_brick->qfile_path, O_RDONLY,
-                                S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-                if (temp_fd < 0) {
-                        gf_msg ("tier", GF_LOG_ERROR, errno,
-                                DHT_MSG_LOG_TIER_ERROR, "Failed to open "
-                                "%s to the query file",
-                                local_brick->qfile_path);
-                        query_cbk_args->qfile_array->exhausted_count++;
-                }
-                query_cbk_args->qfile_array->fd_array[count] = temp_fd;
-                count++;
-        }
-
-        /* Moving the query file index to the next, so that we won't the same
-         * query file every cycle as the first one */
-        query_cbk_args->qfile_array->next_index =
-                        (query_cbk_args->is_promotion) ?
-                        tier_conf->last_promote_qfile_index :
-                        tier_conf->last_demote_qfile_index;
-        shift_next_index (query_cbk_args->qfile_array);
-        if (query_cbk_args->is_promotion) {
-                tier_conf->last_promote_qfile_index =
-                        query_cbk_args->qfile_array->next_index;
-        } else {
-                tier_conf->last_demote_qfile_index =
-                        query_cbk_args->qfile_array->next_index;
-        }
-
-        /* Migrate files using query file list */
-        ret = tier_migrate_using_query_file ((void *)query_cbk_args);
-out:
-        qfile_array_free (query_cbk_args->qfile_array);
-
-        /* If there is an error rename all the query files to .err files
-         * with a timestamp for better debugging */
-        if (ret) {
-                list_for_each_entry_safe (local_brick, temp, comp->brick_list,
-                                        list) {
-                        /* rename error qfile*/
-                        snprintf (query_file_path_err, PATH_MAX, "%s-%s.err",
-                                local_brick->qfile_path, time_str);
-                        sys_rename (local_brick->qfile_path,
-                                query_file_path_err);
-                }
-        }
-
-        query_cbk_args->qfile_array = NULL;
-
-        return ret;
-}
-
-
-
-int
-tier_demote (migration_args_t *demotion_args)
-{
-        query_cbk_args_t query_cbk_args;
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("tier", demotion_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", demotion_args->this, out);
-        GF_VALIDATE_OR_GOTO (demotion_args->this->name,
-                             demotion_args->brick_list, out);
-        GF_VALIDATE_OR_GOTO (demotion_args->this->name,
-                             demotion_args->defrag, out);
-
-        THIS = demotion_args->this;
-
-        query_cbk_args.this = demotion_args->this;
-        query_cbk_args.defrag = demotion_args->defrag;
-        query_cbk_args.is_promotion = 0;
-
-        /*Build the query file using bricklist*/
-        ret = tier_build_migration_qfile (demotion_args, &query_cbk_args,
-                                          _gf_false);
-        if (ret)
-                goto out;
-
-        /* Migrate files using the query file */
-        ret = tier_migrate_files_using_qfile (demotion_args,
-                                              &query_cbk_args);
-        if (ret)
-                goto out;
-
-out:
-        demotion_args->return_value = ret;
-        return ret;
-}
-
-
-int
-tier_promote (migration_args_t *promotion_args)
-{
-        int ret = -1;
-        query_cbk_args_t query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", promotion_args->this, out);
-        GF_VALIDATE_OR_GOTO (promotion_args->this->name,
-                             promotion_args->brick_list, out);
-        GF_VALIDATE_OR_GOTO (promotion_args->this->name,
-                             promotion_args->defrag, out);
-
-        THIS = promotion_args->this;
-
-        query_cbk_args.this = promotion_args->this;
-        query_cbk_args.defrag = promotion_args->defrag;
-        query_cbk_args.is_promotion = 1;
-
-        /*Build the query file using bricklist*/
-        ret = tier_build_migration_qfile (promotion_args, &query_cbk_args,
-                                          _gf_true);
-        if (ret)
-                goto out;
-
-        /* Migrate files using the query file */
-        ret = tier_migrate_files_using_qfile (promotion_args, &query_cbk_args);
-        if (ret)
-                goto out;
-
-out:
-        promotion_args->return_value = ret;
-        return ret;
-}
-
-static int
-tier_get_bricklist (xlator_t *xl, struct list_head *local_bricklist_head)
-{
-        xlator_list_t           *child = NULL;
-        char                    *rv        = NULL;
-        char                    *rh        = NULL;
-        char                    localhost[256] = {0};
-        char                    *brickname = NULL;
-        char                    db_name[PATH_MAX] = "";
-        int                     ret = 0;
-        tier_brick_list_t       *local_brick = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", xl, out);
-        GF_VALIDATE_OR_GOTO ("tier", local_bricklist_head, out);
-
-        gethostname (localhost, sizeof (localhost));
-
-        /*
-         * This function obtains remote subvolumes and filters out only
-         * those running on the same node as the tier daemon.
-         */
-        if (strcmp(xl->type, "protocol/client") == 0) {
-                ret = dict_get_str (xl->options, "remote-host", &rh);
-                if (ret < 0)
-                        goto out;
-
-                if (gf_is_local_addr (rh)) {
-
-                       local_brick = GF_CALLOC (1, sizeof(tier_brick_list_t),
-                                                gf_tier_mt_bricklist_t);
-                        if (!local_brick) {
-                                goto out;
-                        }
-
-                        ret = dict_get_str (xl->options, "remote-subvolume",
-                                           &rv);
-                        if (ret < 0)
-                                goto out;
-
-                        brickname = strrchr(rv, '/') + 1;
-                        snprintf(db_name, sizeof(db_name), "%s.db",
-                                 brickname);
-
-                        local_brick->brick_db_path =
-                                GF_CALLOC (PATH_MAX, 1, gf_common_mt_char);
-                        if (!local_brick->brick_db_path) {
-                                gf_msg ("tier", GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "Failed to allocate memory for"
-                                        " bricklist.");
-                                goto out;
-                        }
-
-                        snprintf(local_brick->brick_db_path,
-                                PATH_MAX, "%s/%s/%s", rv,
-                                GF_HIDDEN_PATH, db_name);
-
-                        local_brick->xlator = xl;
-
-                        snprintf (local_brick->brick_name,
-                                NAME_MAX, "%s", brickname);
-
-                        list_add_tail (&(local_brick->list),
-                                        local_bricklist_head);
-
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        for (child = xl->children; child; child = child->next) {
-                ret = tier_get_bricklist (child->xlator, local_bricklist_head);
-                if (ret) {
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-
-        if (ret) {
-                if (local_brick) {
-                        GF_FREE (local_brick->brick_db_path);
-                }
-                GF_FREE (local_brick);
-        }
-
-        return ret;
-}
-
-int
-tier_get_freq_demote (gf_tier_conf_t *tier_conf)
-{
-        if ((tier_conf->mode == TIER_MODE_WM) &&
-            (tier_conf->watermark_last == TIER_WM_HI))
-                return DEFAULT_DEMOTE_DEGRADED;
-        else
-                return tier_conf->tier_demote_frequency;
-}
-
-int
-tier_get_freq_promote (gf_tier_conf_t *tier_conf)
-{
-        return tier_conf->tier_promote_frequency;
-}
-
-static int
-tier_check_demote (gfdb_time_t  current_time, int freq)
-{
-        return ((current_time.tv_sec % freq) == 0) ?
-                _gf_true : _gf_false;
-}
-
-static gf_boolean_t
-tier_check_promote (gf_tier_conf_t   *tier_conf,
-                    gfdb_time_t  current_time,
-                    int freq)
-{
-        if ((tier_conf->mode == TIER_MODE_WM) &&
-            (tier_conf->watermark_last == TIER_WM_HI))
-                return _gf_false;
-
-        else
-                return ((current_time.tv_sec % freq) == 0) ?
-                        _gf_true : _gf_false;
-}
-
-
-
-
-void
-clear_bricklist (struct list_head *brick_list)
-{
-        tier_brick_list_t  *local_brick      = NULL;
-        tier_brick_list_t  *temp             = NULL;
-
-        if (list_empty(brick_list)) {
-                return;
-        }
-
-        list_for_each_entry_safe (local_brick, temp, brick_list, list) {
-                list_del (&local_brick->list);
-                GF_FREE (local_brick->brick_db_path);
-                GF_FREE (local_brick);
-        }
-}
-
-
-static void
-set_brick_list_qpath (struct list_head *brick_list, gf_boolean_t is_cold)
-{
-
-        tier_brick_list_t *local_brick = NULL;
-        int i                          = 0;
-
-        GF_VALIDATE_OR_GOTO ("tier", brick_list, out);
-
-        list_for_each_entry (local_brick, brick_list, list) {
-
-                /* Construct query file path for this brick
-                 * i.e
-                 * /var/run/gluster/xlator_name/
-                 * {promote/demote}-brickname-indexinbricklist
-                 * So that no two query files will have same path even
-                 * bricks have the same name
-                 * */
-                snprintf (local_brick->qfile_path, PATH_MAX , "%s-%s-%d",
-                        GET_QFILE_PATH (is_cold),
-                        local_brick->brick_name, i);
-                i++;
-        }
-out:
-        return;
-}
-
-/*
- * Main tiering loop. This is called from the promotion and the
- * demotion threads spawned in tier_start().
- *
- * Every second, wake from sleep to perform tasks.
- * 1. Check trigger to migrate data.
- * 2. Check for state changes (pause, unpause, stop).
- */
-static void
-*tier_run (void *in_args)
-{
-        dht_conf_t *conf                        = NULL;
-        gfdb_time_t  current_time               = { 0 };
-        int freq                                = 0;
-        int ret                                 = 0;
-        xlator_t *any                           = NULL;
-        xlator_t *xlator                        = NULL;
-        gf_tier_conf_t *tier_conf               = NULL;
-        loc_t root_loc                          = { 0 };
-        int check_watermark                     = 0;
-        gf_defrag_info_t *defrag                = NULL;
-        xlator_t  *this                         = NULL;
-        migration_args_t *args = in_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", args, out);
-        GF_VALIDATE_OR_GOTO ("tier", args->brick_list, out);
-
-        this = args->this;
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-
-        conf   = this->private;
-        GF_VALIDATE_OR_GOTO ("tier", conf, out);
-
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO ("tier", defrag, out);
-
-        if (list_empty (args->brick_list)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Brick list for tier is empty. Exiting.");
-                goto out;
-        }
-
-        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
-        tier_conf = &defrag->tier_conf;
-
-        dht_build_root_loc (defrag->root_inode, &root_loc);
-
-        while (1) {
-
-                /*
-                 * Check if a graph switch occured. If so, stop migration
-                 * thread. It will need to be restarted manually.
-                 */
-                any = THIS->ctx->active->first;
-                xlator = xlator_search_by_name (any, this->name);
-
-                if (xlator != this) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS,
-                                "Detected graph switch. Exiting migration daemon.");
-                        goto out;
-                }
-
-                gf_defrag_check_pause_tier (tier_conf);
-
-                sleep(1);
-
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        ret = 1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "defrag->defrag_status != "
-                                "GF_DEFRAG_STATUS_STARTED");
-                        goto out;
-                }
-
-                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
-                        ret = 0;
-                        defrag->defrag_status =
-                                GF_DEFRAG_STATUS_COMPLETE;
-                        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "defrag->defrag_cmd == "
-                                "GF_DEFRAG_CMD_START_DETACH_TIER");
-                        goto out;
-                }
-
-                if (gf_defrag_get_pause_state (&defrag->tier_conf) != TIER_RUNNING)
-                        continue;
-
-
-                /* To have proper synchronization amongst all
-                 * brick holding nodes, so that promotion and demotions
-                 * start atomicly w.r.t promotion/demotion frequency
-                 * period, all nodes should have thier system time
-                 * in-sync with each other either manually set or
-                 * using a NTP server*/
-                ret = gettimeofday (&current_time, NULL);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                DHT_MSG_SYS_CALL_GET_TIME_FAILED,
-                                "Failed to get current time");
-                        goto out;
-                }
-
-                check_watermark++;
-
-                if (check_watermark >= WM_INTERVAL) {
-                        check_watermark = 0;
-                        ret = tier_check_watermark (this, &root_loc);
-                        if (ret != 0) {
-                                gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Failed to get watermark");
-                                continue;
-                        }
-                }
-
-                if (args->is_promotion) {
-
-                        freq = tier_get_freq_promote (tier_conf);
-
-                        if (tier_check_promote (tier_conf, current_time, freq)) {
-                                args->freq_time = freq;
-                                ret = tier_promote (args);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_LOG_TIER_ERROR,
-                                                "Promotion failed");
-                                }
-                        }
-
-                } else {
-
-                        freq = tier_get_freq_demote (tier_conf);
-
-                        if (tier_check_demote (current_time, freq)) {
-                                args->freq_time = freq;
-                                ret = tier_demote (args);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_LOG_TIER_ERROR,
-                                                "Demotion failed");
-                                }
-                        }
-
-                }
-
-                /* Check the statfs immediately after the processing threads
-                   return */
-                check_watermark = WM_INTERVAL;
-        }
-
-        ret = 0;
-out:
-
-        args->return_value = ret;
-
-        return NULL;
-}
-
-int
-tier_start (xlator_t *this, gf_defrag_info_t *defrag)
-{
-        pthread_t promote_thread;
-        pthread_t demote_thread;
-        int ret = -1;
-        struct list_head bricklist_hot          = { 0 };
-        struct list_head bricklist_cold         = { 0 };
-        migration_args_t promotion_args         = { 0 };
-        migration_args_t demotion_args          = { 0 };
-        dht_conf_t *conf                        = NULL;
-
-        INIT_LIST_HEAD ((&bricklist_hot));
-        INIT_LIST_HEAD ((&bricklist_cold));
-
-        conf   = this->private;
-
-        tier_get_bricklist (conf->subvolumes[1], &bricklist_hot);
-        set_brick_list_qpath (&bricklist_hot, _gf_false);
-
-        demotion_args.this = this;
-        demotion_args.brick_list = &bricklist_hot;
-        demotion_args.defrag = defrag;
-        demotion_args.is_promotion = _gf_false;
-
-        ret = pthread_create (&demote_thread,
-                              NULL, &tier_run,
-                              &demotion_args);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to start demotion thread.");
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                goto cleanup;
-        }
-
-        tier_get_bricklist (conf->subvolumes[0], &bricklist_cold);
-        set_brick_list_qpath (&bricklist_cold, _gf_true);
-
-        promotion_args.this = this;
-        promotion_args.brick_list = &bricklist_cold;
-        promotion_args.defrag = defrag;
-        promotion_args.is_promotion = _gf_true;
-
-        ret = pthread_create (&promote_thread,
-                              NULL, &tier_run,
-                              &promotion_args);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to start promotion thread.");
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                goto waitforspawned;
-        }
-
-        pthread_join (promote_thread, NULL);
-
-waitforspawned:
-        pthread_join (demote_thread, NULL);
-
-cleanup:
-        clear_bricklist (&bricklist_cold);
-        clear_bricklist (&bricklist_hot);
-
-        return ret;
-}
-
-int32_t
-tier_migration_needed (xlator_t *this)
-{
-        gf_defrag_info_t        *defrag = NULL;
-        dht_conf_t              *conf   = NULL;
-        int                      ret = 0;
-
-        conf = this->private;
-
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-        GF_VALIDATE_OR_GOTO (this->name, conf->defrag, out);
-
-        defrag = conf->defrag;
-
-        if ((defrag->cmd == GF_DEFRAG_CMD_START_TIER) ||
-            (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER))
-                ret = 1;
-out:
-        return ret;
-}
-
-int32_t
-tier_migration_get_dst (xlator_t *this, dht_local_t *local)
-{
-        dht_conf_t              *conf   = NULL;
-        int32_t                  ret = -1;
-        gf_defrag_info_t        *defrag = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        conf = this->private;
-
-        defrag = conf->defrag;
-
-        if (defrag && defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
-                local->rebalance.target_node = conf->subvolumes[0];
-
-        } else if (conf->subvolumes[0] == local->cached_subvol)
-                local->rebalance.target_node =
-                        conf->subvolumes[1];
-        else
-                local->rebalance.target_node =
-                        conf->subvolumes[0];
-
-        if (local->rebalance.target_node)
-                ret = 0;
-
-out:
-        return ret;
-}
-
-xlator_t *
-tier_search (xlator_t *this, dht_layout_t *layout, const char *name)
-{
-        xlator_t                *subvol = NULL;
-        dht_conf_t              *conf   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        conf = this->private;
-
-        subvol = TIER_HASHED_SUBVOL;
-
- out:
-        return subvol;
-}
-
-
-static int
-tier_load_externals (xlator_t *this)
-{
-        int               ret            = -1;
-        char *libpathfull = (LIBDIR "/libgfdb.so.0");
-        get_gfdb_methods_t get_gfdb_methods;
-
-        GF_VALIDATE_OR_GOTO ("this", this, out);
-
-        libhandle = dlopen (libpathfull, RTLD_NOW);
-        if (!libhandle) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Error loading libgfdb.so %s\n", dlerror());
-                ret = -1;
-                goto out;
-        }
-
-        get_gfdb_methods = dlsym (libhandle, "get_gfdb_methods");
-        if (!get_gfdb_methods) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Error loading get_gfdb_methods()");
-                ret = -1;
-                goto out;
-        }
-
-        get_gfdb_methods (&gfdb_methods);
-
-        ret = 0;
-
-out:
-        if (ret && libhandle)
-                dlclose (libhandle);
-
-        return ret;
-}
-
-static
-int tier_validate_mode (char *mode)
-{
-        int ret = -1;
-
-        if (strcmp (mode, "test") == 0) {
-                ret = TIER_MODE_TEST;
-        } else {
-                ret = TIER_MODE_WM;
-        }
-
-        return ret;
-}
-
-
-int
-tier_init_methods (xlator_t *this)
-{
-        int ret                  = -1;
-        dht_conf_t      *conf    = NULL;
-        dht_methods_t   *methods = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, err);
-
-        conf = this->private;
-
-        methods = &(conf->methods);
-
-        methods->migration_get_dst_subvol = tier_migration_get_dst;
-        methods->migration_other   = tier_start;
-        methods->migration_needed  = tier_migration_needed;
-        methods->layout_search     = tier_search;
-
-        ret = 0;
-err:
-        return ret;
-}
-
-
-
-int
-tier_init (xlator_t *this)
-{
-        int               ret            = -1;
-        int               freq           = 0;
-        int               maxsize        = 0;
-        dht_conf_t       *conf           = NULL;
-        gf_defrag_info_t *defrag         = NULL;
-        char             *voldir         = NULL;
-        char             *mode           = NULL;
-        char             *paused         = NULL;
-
-        ret = dht_init (this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "tier_init failed");
-                goto out;
-        }
-
-        conf = this->private;
-
-        ret = tier_init_methods (this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "tier_init_methods failed");
-                goto out;
-        }
-
-        if (conf->subvolume_cnt != 2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Invalid number of subvolumes %d", conf->subvolume_cnt);
-                goto out;
-        }
-
-        /* if instatiated from client side initialization is complete. */
-        if (!conf->defrag) {
-                ret = 0;
-                goto out;
-        }
-
-        /* if instatiated from server side, load db libraries */
-        ret = tier_load_externals (this);
-        if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Could not load externals. Aborting");
-                goto out;
-        }
-
-        defrag = conf->defrag;
-
-        defrag->tier_conf.last_demote_qfile_index = 0;
-        defrag->tier_conf.last_promote_qfile_index = 0;
-
-        defrag->tier_conf.is_tier = 1;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-max-promote-file-size", &maxsize);
-        if (ret) {
-                maxsize = 0;
-        }
-
-        defrag->tier_conf.tier_max_promote_size = maxsize;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-promote-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_PROMOTE_FREQ_SEC;
-        }
-
-        defrag->tier_conf.tier_promote_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-demote-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_DEMOTE_FREQ_SEC;
-        }
-
-        defrag->tier_conf.tier_demote_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "watermark-hi", &freq);
-        if (ret) {
-                freq = DEFAULT_WM_HI;
-        }
-
-        defrag->tier_conf.watermark_hi = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "watermark-low", &freq);
-        if (ret) {
-                freq = DEFAULT_WM_LOW;
-        }
-
-        defrag->tier_conf.watermark_low = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "write-freq-threshold", &freq);
-        if (ret) {
-                freq = DEFAULT_WRITE_FREQ_SEC;
-        }
-
-        defrag->write_freq_threshold = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "read-freq-threshold", &freq);
-        if (ret) {
-                freq = DEFAULT_READ_FREQ_SEC;
-        }
-
-        defrag->read_freq_threshold = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-max-mb", &freq);
-        if (ret) {
-                freq = DEFAULT_TIER_MAX_MIGRATE_MB;
-        }
-
-        defrag->tier_conf.max_migrate_bytes = (uint64_t) freq * 1024 * 1024;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-max-files", &freq);
-        if (ret) {
-                freq = DEFAULT_TIER_MAX_MIGRATE_FILES;
-        }
-
-        defrag->tier_conf.max_migrate_files = freq;
-
-        ret = dict_get_str (this->options,
-                            "tier-mode", &mode);
-        if (ret) {
-                defrag->tier_conf.mode = DEFAULT_TIER_MODE;
-        } else {
-                ret = tier_validate_mode (mode);
-                if (ret < 0) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0,
-                               DHT_MSG_LOG_TIER_ERROR,
-                               "tier_init failed - invalid mode");
-                        goto out;
-                }
-                defrag->tier_conf.mode = ret;
-        }
-
-        pthread_mutex_init (&defrag->tier_conf.pause_mutex, 0);
-
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING);
-
-        ret = dict_get_str (this->options,
-                              "tier-pause", &paused);
-
-        if (paused && strcmp (paused, "on") == 0)
-                gf_defrag_set_pause_state (&defrag->tier_conf, TIER_REQUEST_PAUSE);
-
-        ret = gf_asprintf(&voldir, "%s/%s",
-                          DEFAULT_VAR_RUN_DIRECTORY,
-                          this->name);
-        if (ret < 0)
-                goto out;
-
-        ret = mkdir_p(voldir, 0777, _gf_true);
-        if (ret == -1 && errno != EEXIST) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "tier_init failed");
-
-                GF_FREE(voldir);
-                goto out;
-        }
-
-        GF_FREE(voldir);
-
-        ret = gf_asprintf (&promotion_qfile, "%s/%s/promote",
-                           DEFAULT_VAR_RUN_DIRECTORY,
-                           this->name);
-        if (ret < 0)
-                goto out;
-
-        ret = gf_asprintf (&demotion_qfile, "%s/%s/demote",
-                           DEFAULT_VAR_RUN_DIRECTORY,
-                           this->name);
-        if (ret < 0) {
-                GF_FREE (promotion_qfile);
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_LOG_TIER_STATUS,
-               "Promote/demote frequency %d/%d "
-               "Write/Read freq thresholds %d/%d",
-               defrag->tier_conf.tier_promote_frequency,
-               defrag->tier_conf.tier_demote_frequency,
-               defrag->write_freq_threshold,
-               defrag->read_freq_threshold);
-
-        ret = 0;
-
-out:
-
-        return ret;
-}
-
-
-int
-tier_cli_pause_done (int op_ret, call_frame_t *sync_frame, void *data)
-{
-        gf_msg ("tier", GF_LOG_INFO, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Migrate file paused with op_ret %d", op_ret);
-
-        return op_ret;
-}
-
-int
-tier_cli_pause (void *data)
-{
-        gf_defrag_info_t        *defrag         = NULL;
-        xlator_t                *this           = NULL;
-        dht_conf_t              *conf           = NULL;
-        int                      ret            = -1;
-
-        this = data;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, exit);
-
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO (this->name, defrag, exit);
-
-        gf_defrag_pause_tier (this, defrag);
-
-        ret = 0;
-exit:
-        return ret;
-}
-
-
-int
-tier_reconfigure (xlator_t *this, dict_t *options)
-{
-        dht_conf_t       *conf           = NULL;
-        gf_defrag_info_t *defrag         = NULL;
-        char             *mode           = NULL;
-        int               migrate_mb     = 0;
-        gf_boolean_t      req_pause      = _gf_false;
-        int               ret            = 0;
-        call_frame_t            *frame  = NULL;
-
-        conf = this->private;
-
-        if (conf->defrag) {
-                defrag = conf->defrag;
-                GF_OPTION_RECONF ("tier-max-promote-file-size",
-                                  defrag->tier_conf.tier_max_promote_size,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("tier-promote-frequency",
-                                  defrag->tier_conf.tier_promote_frequency,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("tier-demote-frequency",
-                                  defrag->tier_conf.tier_demote_frequency,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("write-freq-threshold",
-                                  defrag->write_freq_threshold, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("read-freq-threshold",
-                                  defrag->read_freq_threshold, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("watermark-hi",
-                                  defrag->tier_conf.watermark_hi, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("watermark-low",
-                                  defrag->tier_conf.watermark_low, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("tier-mode",
-                                  mode, options,
-                                  str, out);
-                defrag->tier_conf.mode = tier_validate_mode (mode);
-
-                GF_OPTION_RECONF ("tier-max-mb",
-                                  migrate_mb, options,
-                                  int32, out);
-                defrag->tier_conf.max_migrate_bytes = (uint64_t) migrate_mb *
-                                                        1024 * 1024;
-
-                GF_OPTION_RECONF ("tier-max-files",
-                                  defrag->tier_conf.max_migrate_files, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("tier-pause",
-                                  req_pause, options,
-                                  bool, out);
-
-                if (req_pause == _gf_true) {
-
-                        frame = create_frame (this, this->ctx->pool);
-                        if (!frame)
-                                goto out;
-
-                        frame->root->pid = GF_CLIENT_PID_DEFRAG;
-
-                        ret = synctask_new (this->ctx->env, tier_cli_pause,
-                                            tier_cli_pause_done, frame, this);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "pause tier failed on reconfigure");
-                        }
-                } else {
-                        ret = gf_defrag_resume_tier (this, defrag);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "resume tier failed on reconfigure");
-                        }
-                }
-
-        }
-
-out:
-        return dht_reconfigure (this, options);
-}
-
-void
-tier_fini (xlator_t *this)
-{
-        if (libhandle)
-                dlclose (libhandle);
-
-        GF_FREE (demotion_qfile);
-        GF_FREE (promotion_qfile);
-
-        dht_fini(this);
-}
-
-class_methods_t class_methods = {
-        .init           = tier_init,
-        .fini           = tier_fini,
-        .reconfigure    = tier_reconfigure,
-        .notify         = dht_notify
-};
-
-
-struct xlator_fops fops = {
-
-        .lookup      = dht_lookup,
-        .create      = tier_create,
-        .mknod       = dht_mknod,
-
-        .open        = dht_open,
-        .statfs      = dht_statfs,
-        .opendir     = dht_opendir,
-        .readdir     = tier_readdir,
-        .readdirp    = tier_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = tier_unlink,
-        .link        = tier_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-
-        /* Inode read operations */
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .getxattr    = dht_getxattr,
-        .fgetxattr    = dht_fgetxattr,
-        .readv       = dht_readv,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .lk          = dht_lk,
-
-        /* Inode write operations */
-        .fremovexattr = dht_fremovexattr,
-        .removexattr = dht_removexattr,
-        .setxattr    = dht_setxattr,
-        .fsetxattr   = dht_fsetxattr,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .writev      = dht_writev,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
-        .fsetattr    = dht_fsetattr,
-        .fallocate   = dht_fallocate,
-        .discard     = dht_discard,
-        .zerofill    = dht_zerofill,
-};
-
-
-struct xlator_cbks cbks = {
-        .release    = dht_release,
-        .forget     = dht_forget
-};
-
diff --git a/xlators/cluster/dht/src/tier.h b/xlators/cluster/dht/src/tier.h
deleted file mode 100644
index 0807608fda2..00000000000
--- a/xlators/cluster/dht/src/tier.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _TIER_H_
-#define _TIER_H_
-
-
-/******************************************************************************/
-/* This is from dht-rebalancer.c as we dont have dht-rebalancer.h */
-#include "dht-common.h"
-#include "xlator.h"
-#include <signal.h>
-#include <fnmatch.h>
-#include <signal.h>
-
-/*
- * Size of timer wheel. We would not promote or demote less
- * frequently than this number.
- */
-#define TIMER_SECS 3600
-
-#include "gfdb_data_store.h"
-#include <ctype.h>
-#include <sys/stat.h>
-
-#define PROMOTION_QFILE "promotequeryfile"
-#define DEMOTION_QFILE "demotequeryfile"
-
-#define TIER_HASHED_SUBVOL   conf->subvolumes[0]
-#define TIER_UNHASHED_SUBVOL   conf->subvolumes[1]
-
-#define GET_QFILE_PATH(is_promotion)\
-        (is_promotion) ? promotion_qfile : demotion_qfile
-
-typedef struct tier_qfile_array {
-        int             *fd_array;
-        ssize_t         array_size;
-        ssize_t         next_index;
-        /* Indicate the number of exhuasted FDs*/
-        ssize_t         exhausted_count;
-} tier_qfile_array_t;
-
-
-typedef struct _query_cbk_args {
-        xlator_t                *this;
-        gf_defrag_info_t        *defrag;
-        /* This is write */
-        int                     query_fd;
-        int                     is_promotion;
-        /* This is for read */
-        tier_qfile_array_t       *qfile_array;
-} query_cbk_args_t;
-
-int
-gf_run_tier(xlator_t *this, gf_defrag_info_t *defrag);
-
-typedef struct gfdb_brick_info {
-        gfdb_time_t             *time_stamp;
-        gf_boolean_t            _gfdb_promote;
-        query_cbk_args_t        *_query_cbk_args;
-} gfdb_brick_info_t;
-
-typedef struct brick_list {
-        xlator_t          *xlator;
-        char              *brick_db_path;
-        char              brick_name[NAME_MAX];
-        char              qfile_path[PATH_MAX];
-        struct list_head  list;
-} tier_brick_list_t;
-
-typedef struct _dm_thread_args {
-        xlator_t                *this;
-        gf_defrag_info_t        *defrag;
-        struct list_head        *brick_list;
-        int                     freq_time;
-        int                     return_value;
-        int                     is_promotion;
-} migration_args_t;
-
-typedef enum tier_watermark_op_ {
-        TIER_WM_NONE = 0,
-        TIER_WM_LOW,
-        TIER_WM_HI,
-        TIER_WM_MID
-} tier_watermark_op_t;
-
-#define DEFAULT_PROMOTE_FREQ_SEC       120
-#define DEFAULT_DEMOTE_FREQ_SEC        120
-#define DEFAULT_DEMOTE_DEGRADED        10
-#define DEFAULT_WRITE_FREQ_SEC         0
-#define DEFAULT_READ_FREQ_SEC          0
-#define DEFAULT_WM_LOW                 75
-#define DEFAULT_WM_HI                  90
-#define DEFAULT_TIER_MODE              TIER_MODE_TEST
-#define DEFAULT_TIER_MAX_MIGRATE_MB    1000
-#define DEFAULT_TIER_MAX_MIGRATE_FILES 5000
-
-#endif
diff --git a/xlators/cluster/dht/src/tier.sym b/xlators/cluster/dht/src/tier.sym
deleted file mode 100644
index 60205d145b6..00000000000
--- a/xlators/cluster/dht/src/tier.sym
+++ /dev/null
@@ -1,9 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-tier_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_mock.c b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
index 6544f4208f5..771452963d1 100644
--- a/xlators/cluster/dht/src/unittest/dht_layout_mock.c
+++ b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
@@ -7,66 +7,67 @@
   later), or the GNU General Public License, version 2 (GPLv2), in all
   cases as published by the Free Software Foundation.
 */
-#include "glusterfs.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
 #include "dht-common.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
 int
-dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
     return 0;
 }
 
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this, dht_layout_t **layout)
 {
     return 0;
 }
 
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t *layout_int)
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int)
 {
     return 0;
 }
 
 int
-dict_get_ptr (dict_t *this, char *key, void **ptr)
+dict_get_ptr(dict_t *this, char *key, void **ptr)
 {
     return 0;
 }
 
 int
-dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len)
+dict_get_ptr_and_len(dict_t *this, char *key, void **ptr, int *len)
 {
     return 0;
 }
 
-int _gf_log (const char *domain, const char *file,
-             const char *function, int32_t line, gf_loglevel_t level,
-             const char *fmt, ...)
+int
+_gf_log(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, const char *fmt, ...)
 {
     return 0;
 }
 
-int _gf_log_callingfn (const char *domain, const char *file,
-                       const char *function, int32_t line, gf_loglevel_t level,
-                       const char *fmt, ...)
+int
+_gf_log_callingfn(const char *domain, const char *file, const char *function,
+                  int32_t line, gf_loglevel_t level, const char *fmt, ...)
 {
     return 0;
 }
 
-void gf_uuid_unparse(const uuid_t uu, char *out)
+void
+gf_uuid_unparse(const uuid_t uu, char *out)
 {
     // could call a will-return function here
     // to place the correct data in *out
 }
 
 int
-_gf_msg (const char *domain, const char *file, const char *function,
-         int32_t line, gf_loglevel_t level, int errnum, int trace,
-         uint64_t msgid, const char *fmt, ...)
+_gf_msg(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, int errnum, int trace,
+        uint64_t msgid, const char *fmt, ...)
 {
     return 0;
 }
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
index 84a89160e38..c94a1d0a2e1 100644
--- a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
+++ b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
@@ -9,8 +9,8 @@
 */
 
 #include "dht-common.h"
-#include "logging.h"
-#include "xlator.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/xlator.h>
 
 #include <inttypes.h>
 #include <stdarg.h>
@@ -34,16 +34,16 @@ helper_xlator_init(uint32_t num_types)
     xl = test_calloc(1, sizeof(xlator_t));
     assert_non_null(xl);
     xl->mem_acct->num_types = num_types;
-    xl->mem_acct = test_calloc (sizeof(struct mem_acct)
-                                + sizeof(struct mem_acct_rec) + num_types);
+    xl->mem_acct = test_calloc(sizeof(struct mem_acct) +
+                               sizeof(struct mem_acct_rec) + num_types);
     assert_non_null(xl->mem_acct);
 
     xl->ctx = test_calloc(1, sizeof(glusterfs_ctx_t));
     assert_non_null(xl->ctx);
 
     for (i = 0; i < num_types; i++) {
-            ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
-            assert_false(ret);
+        ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
+        assert_false(ret);
     }
 
     ENSURE(num_types == xl->mem_acct.num_types);
@@ -58,8 +58,8 @@ helper_xlator_destroy(xlator_t *xl)
     int i, ret;
 
     for (i = 0; i < xl->mem_acct.num_types; i++) {
-            ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
-            assert_int_equal(ret, 0);
+        ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
+        assert_int_equal(ret, 0);
     }
 
     free(xl->mem_acct.rec);
@@ -76,7 +76,7 @@ test_dht_layout_new(void **state)
 {
     xlator_t *xl;
     dht_layout_t *layout;
-    dht_conf_t   *conf;
+    dht_conf_t *conf;
     int cnt;
 
     expect_assert_failure(dht_layout_new(NULL, 0));
@@ -90,7 +90,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(layout->ref, 1);
+    assert_int_equal(GF_ATOMIC_GET(layout->ref), 1);
     assert_int_equal(layout->gen, 0);
     assert_int_equal(layout->spread_cnt, 0);
     free(layout);
@@ -107,7 +107,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(layout->ref, 1);
+    assert_int_equal(GF_ATOMIC_GET(layout->ref), 1);
     assert_int_equal(layout->gen, conf->gen);
     assert_int_equal(layout->spread_cnt, conf->dir_spread_cnt);
     free(layout);
@@ -116,7 +116,9 @@ test_dht_layout_new(void **state)
     helper_xlator_destroy(xl);
 }
 
-int main(void) {
+int
+main(void)
+{
     const struct CMUnitTest xlator_dht_layout_tests[] = {
         unit_test(test_dht_layout_new),
     };
diff --git a/xlators/cluster/ec/src/Makefile.am b/xlators/cluster/ec/src/Makefile.am
index cbdceefdbe0..406a636bbc2 100644
--- a/xlators/cluster/ec/src/Makefile.am
+++ b/xlators/cluster/ec/src/Makefile.am
@@ -12,8 +12,11 @@ ec_sources += ec-dir-write.c
 ec_sources += ec-inode-read.c
 ec_sources += ec-inode-write.c
 ec_sources += ec-combine.c
-ec_sources += ec-gf.c
 ec_sources += ec-method.c
+ec_sources += ec-galois.c
+ec_sources += ec-code.c
+ec_sources += ec-code-c.c
+ec_sources += ec-gf8.c
 ec_sources += ec-heal.c
 ec_sources += ec-heald.c
 
@@ -24,10 +27,34 @@ ec_headers += ec-data.h
 ec_headers += ec-fops.h
 ec_headers += ec-common.h
 ec_headers += ec-combine.h
-ec_headers += ec-gf.h
 ec_headers += ec-method.h
+ec_headers += ec-galois.h
+ec_headers += ec-code.h
+ec_headers += ec-code-c.h
+ec_headers += ec-gf8.h
 ec_headers += ec-heald.h
 ec_headers += ec-messages.h
+ec_headers += ec-types.h
+
+if ENABLE_EC_DYNAMIC_INTEL
+  ec_sources += ec-code-intel.c
+  ec_headers += ec-code-intel.h
+endif
+
+if ENABLE_EC_DYNAMIC_X64
+  ec_sources += ec-code-x64.c
+  ec_headers += ec-code-x64.h
+endif
+
+if ENABLE_EC_DYNAMIC_SSE
+  ec_sources += ec-code-sse.c
+  ec_headers += ec-code-sse.h
+endif
+
+if ENABLE_EC_DYNAMIC_AVX
+  ec_sources += ec-code-avx.c
+  ec_headers += ec-code-avx.h
+endif
 
 ec_ext_sources = $(top_builddir)/xlators/lib/src/libxlator.c
 
@@ -37,10 +64,13 @@ ec_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 ec_la_SOURCES = $(ec_sources) $(ec_headers) $(ec_ext_sources) $(ec_ext_headers)
 ec_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CPPFLAGS  = $(GF_CPPFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS)
 AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
 AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
 AM_CPPFLAGS += -I$(top_srcdir)/rpc/rpc-lib/src
+AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src
+AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src
+AM_CPPFLAGS += -DGLUSTERFS_LIBEXECDIR=\"$(GLUSTERFS_LIBEXECDIR)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/cluster/ec/src/ec-code-avx.c b/xlators/cluster/ec/src/ec-code-avx.c
new file mode 100644
index 00000000000..70afaa00f54
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-avx.c
@@ -0,0 +1,109 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static void
+ec_code_avx_prolog(ec_code_builder_t *builder)
+{
+    builder->loop = builder->address;
+}
+
+static void
+ec_code_avx_epilog(ec_code_builder_t *builder)
+{
+    ec_code_intel_op_add_i2r(builder, 32, REG_DX);
+    ec_code_intel_op_add_i2r(builder, 32, REG_DI);
+    ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX);
+    ec_code_intel_op_jne(builder, builder->loop);
+
+    ec_code_intel_op_ret(builder, 0);
+}
+
+static void
+ec_code_avx_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_mov_m2avx(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_mov_m2avx(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static void
+ec_code_avx_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit)
+{
+    ec_code_intel_op_mov_avx2m(builder, src, REG_DI, REG_NULL, 0,
+                               bit * builder->width);
+}
+
+static void
+ec_code_avx_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_mov_avx2avx(builder, src, dst);
+}
+
+static void
+ec_code_avx_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_xor_avx2avx(builder, src, dst);
+}
+
+static void
+ec_code_avx_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1,
+                 uint32_t src2)
+{
+    ec_code_intel_op_mov_avx2avx(builder, src1, dst);
+    ec_code_intel_op_xor_avx2avx(builder, src2, dst);
+}
+
+static void
+ec_code_avx_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_xor_m2avx(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_xor_m2avx(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static char *ec_code_avx_needed_flags[] = {"avx2", NULL};
+
+ec_code_gen_t ec_code_gen_avx = {.name = "avx",
+                                 .flags = ec_code_avx_needed_flags,
+                                 .width = 32,
+                                 .prolog = ec_code_avx_prolog,
+                                 .epilog = ec_code_avx_epilog,
+                                 .load = ec_code_avx_load,
+                                 .store = ec_code_avx_store,
+                                 .copy = ec_code_avx_copy,
+                                 .xor2 = ec_code_avx_xor2,
+                                 .xor3 = ec_code_avx_xor3,
+                                 .xorm = ec_code_avx_xorm};
diff --git a/xlators/cluster/dht/src/dht-helper.h b/xlators/cluster/ec/src/ec-code-avx.h
index e3ab9c4d93b..fdca4ad2c8f 100644
--- a/xlators/cluster/dht/src/dht-helper.h
+++ b/xlators/cluster/ec/src/ec-code-avx.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2014 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -7,13 +7,12 @@
   later), or the GNU General Public License, version 2 (GPLv2), in all
   cases as published by the Free Software Foundation.
 */
-#ifndef _DHT_HELPER_H
-#define _DHT_HELPER_H
 
-int
-dht_lock_order_requests (dht_lock_t **lk_array, int count);
+#ifndef __EC_CODE_AVX_H__
+#define __EC_CODE_AVX_H__
 
-void
-dht_blocking_inodelk_rec (call_frame_t *frame, int i);
+#include "ec-code.h"
 
-#endif
+extern ec_code_gen_t ec_code_gen_avx;
+
+#endif /* __EC_CODE_AVX_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-c.c b/xlators/cluster/ec/src/ec-code-c.c
new file mode 100644
index 00000000000..acdc665c2cf
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-c.c
@@ -0,0 +1,11679 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <inttypes.h>
+#include <string.h>
+
+#include "ec-method.h"
+#include "ec-code-c.h"
+
+#define WIDTH (EC_METHOD_WORD_SIZE / sizeof(uint64_t))
+
+static void
+gf8_muladd_00(void *out, void *in)
+{
+    memcpy(out, in, EC_METHOD_WORD_SIZE * 8);
+}
+
+static void
+gf8_muladd_01(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        out_ptr[0] ^= in_ptr[0];
+        out_ptr[WIDTH] ^= in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] ^= in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] ^= in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] ^= in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] ^= in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] ^= in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] ^= in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_02(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in7;
+        out1 = in0;
+        out7 = in6;
+        out5 = in4;
+        out6 = in5;
+        out3 = in2 ^ in7;
+        out4 = in3 ^ in7;
+        out2 = in1 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_03(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in7;
+        tmp0 = in2 ^ in7;
+        out1 = in0 ^ in1;
+        out7 = in6 ^ in7;
+        out5 = in4 ^ in5;
+        out6 = in5 ^ in6;
+        out4 = in3 ^ in4 ^ in7;
+        out2 = tmp0 ^ in1;
+        out3 = tmp0 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_04(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in6;
+        out1 = in7;
+        out7 = in5;
+        out6 = in4;
+        tmp0 = in6 ^ in7;
+        out2 = in0 ^ in6;
+        out5 = in3 ^ in7;
+        out3 = tmp0 ^ in1;
+        out4 = tmp0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_05(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in6;
+        out1 = in1 ^ in7;
+        out7 = in5 ^ in7;
+        out6 = in4 ^ in6;
+        out2 = out0 ^ in2;
+        out3 = out1 ^ in3 ^ in6;
+        out5 = out7 ^ in3;
+        out4 = out6 ^ in2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_06(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in6 ^ in7;
+        tmp0 = in1 ^ in6;
+        out1 = in0 ^ in7;
+        out7 = in5 ^ in6;
+        out6 = in4 ^ in5;
+        out4 = in2 ^ in3 ^ in6;
+        out5 = in3 ^ in4 ^ in7;
+        out3 = tmp0 ^ in2;
+        out2 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_07(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        tmp1 = in5 ^ in6;
+        tmp2 = in0 ^ in7;
+        tmp3 = tmp0 ^ in3;
+        out6 = tmp1 ^ in4;
+        out7 = tmp1 ^ in7;
+        out0 = tmp2 ^ in6;
+        out1 = tmp2 ^ in1;
+        out3 = tmp3 ^ in1;
+        out4 = tmp3 ^ in4;
+        out5 = out4 ^ out7 ^ in2;
+        out2 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_08(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in5;
+        out1 = in6;
+        out7 = in4;
+        out6 = in3 ^ in7;
+        out3 = in0 ^ in5 ^ in6;
+        out5 = in2 ^ in6 ^ in7;
+        out2 = in5 ^ in7;
+        out4 = out2 ^ in1 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_09(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in5;
+        tmp0 = in3 ^ in6;
+        out1 = in1 ^ in6;
+        out7 = in4 ^ in7;
+        out2 = in2 ^ in5 ^ in7;
+        out3 = tmp0 ^ out0;
+        out6 = tmp0 ^ in7;
+        out4 = out1 ^ out7 ^ in5;
+        out5 = out2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in5 ^ in7;
+        out1 = in0 ^ in6;
+        out7 = in4 ^ in6;
+        out2 = in1 ^ in5;
+        out6 = out0 ^ in3;
+        out3 = out0 ^ out1 ^ in2;
+        out5 = out7 ^ in2 ^ in7;
+        out4 = out2 ^ in3 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in0 ^ in6;
+        tmp2 = in4 ^ in7;
+        out0 = in0 ^ in5 ^ in7;
+        out2 = tmp0 ^ in1;
+        out1 = tmp1 ^ in1;
+        out6 = tmp1 ^ out0 ^ in3;
+        out7 = tmp2 ^ in6;
+        out4 = tmp2 ^ out6 ^ in1;
+        out3 = out6 ^ in0 ^ in2;
+        out5 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in5 ^ in6;
+        out1 = in6 ^ in7;
+        out7 = in4 ^ in5;
+        tmp0 = in1 ^ in5;
+        tmp1 = in0 ^ in7;
+        out5 = in2 ^ in3 ^ in6;
+        out6 = in3 ^ in4 ^ in7;
+        out2 = tmp1 ^ out0;
+        out4 = tmp0 ^ in2;
+        out3 = tmp0 ^ tmp1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in5;
+        tmp1 = in5 ^ in6;
+        out1 = in1 ^ in6 ^ in7;
+        out7 = tmp0 ^ in7;
+        out4 = tmp0 ^ in1 ^ in2;
+        out0 = tmp1 ^ in0;
+        tmp2 = tmp1 ^ in3;
+        out6 = tmp2 ^ out7;
+        out2 = out0 ^ in2 ^ in7;
+        out3 = out0 ^ out1 ^ in3;
+        out5 = tmp2 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in5;
+        tmp2 = in5 ^ in6;
+        out1 = in0 ^ in6 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out2 = tmp0 ^ tmp2;
+        tmp3 = tmp1 ^ in3;
+        out7 = tmp2 ^ in4;
+        out0 = tmp2 ^ in7;
+        out4 = tmp3 ^ in1 ^ in7;
+        out5 = tmp3 ^ out7;
+        out6 = out0 ^ out5 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in6 ^ in7;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp0 ^ in5;
+        out1 = tmp1 ^ in0;
+        out7 = tmp2 ^ in4;
+        out0 = tmp2 ^ in0;
+        out6 = out7 ^ in3;
+        out5 = out6 ^ in2 ^ in7;
+        tmp3 = tmp1 ^ out0 ^ in2;
+        out4 = tmp1 ^ out5;
+        out2 = tmp3 ^ in6;
+        out3 = tmp3 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_10(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4;
+        out1 = in5;
+        out7 = in3 ^ in7;
+        tmp0 = in6 ^ in7;
+        out2 = in4 ^ in6;
+        tmp1 = out2 ^ in5;
+        out6 = tmp0 ^ in2;
+        out3 = tmp0 ^ tmp1;
+        out5 = out2 ^ out3 ^ in1;
+        out4 = tmp1 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_11(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in3;
+        out0 = in0 ^ in4;
+        out1 = in1 ^ in5;
+        out6 = in2 ^ in7;
+        out4 = in0 ^ in5 ^ in6;
+        out5 = in1 ^ in6 ^ in7;
+        out2 = in2 ^ in4 ^ in6;
+        out3 = in3 ^ in4 ^ in5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_12(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4 ^ in7;
+        out1 = in0 ^ in5;
+        out3 = in2 ^ in4 ^ in5;
+        tmp0 = out0 ^ in6;
+        out2 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in3;
+        out6 = tmp0 ^ out3;
+        out5 = out2 ^ in5;
+        out7 = tmp1 ^ in4;
+        out4 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_13(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in3 ^ in6;
+        tmp0 = in0 ^ in5;
+        tmp1 = in4 ^ in7;
+        out6 = in2 ^ in5 ^ in7;
+        out4 = tmp0 ^ out7 ^ in7;
+        out1 = tmp0 ^ in1;
+        out0 = tmp1 ^ in0;
+        out5 = tmp1 ^ in1 ^ in6;
+        out3 = tmp1 ^ out6 ^ in3;
+        out2 = out5 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_14(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4 ^ in6;
+        out1 = in5 ^ in7;
+        out2 = in0 ^ in4;
+        tmp0 = out0 ^ in5;
+        out7 = out1 ^ in3;
+        tmp1 = out1 ^ in2;
+        out3 = tmp0 ^ in1;
+        out6 = tmp0 ^ tmp1;
+        out4 = tmp1 ^ out2;
+        out5 = out3 ^ in3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_15(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in3 ^ in5;
+        tmp0 = in0 ^ in4;
+        out1 = in1 ^ in5 ^ in7;
+        out5 = in1 ^ in3 ^ in6;
+        out0 = tmp0 ^ in6;
+        out2 = tmp0 ^ in2;
+        out3 = out5 ^ in4 ^ in5;
+        out6 = out2 ^ in0 ^ in7;
+        out4 = tmp0 ^ out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_16(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in4 ^ in7;
+        tmp2 = in2 ^ in3 ^ in4;
+        out1 = tmp0 ^ in7;
+        out4 = tmp0 ^ tmp2;
+        out0 = tmp1 ^ in6;
+        tmp3 = tmp1 ^ in1;
+        out6 = out0 ^ in2 ^ in5;
+        out2 = tmp3 ^ in0;
+        out3 = out6 ^ in1;
+        out7 = tmp2 ^ out6;
+        out5 = tmp3 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_17(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in3 ^ in6;
+        tmp2 = tmp0 ^ in4;
+        out4 = tmp0 ^ in0 ^ in3;
+        out7 = tmp1 ^ in5;
+        tmp3 = tmp1 ^ in1;
+        out6 = tmp2 ^ in7;
+        out5 = tmp3 ^ in4;
+        out3 = tmp3 ^ out6;
+        out0 = out3 ^ out4 ^ in1;
+        out2 = out3 ^ out7 ^ in0;
+        out1 = tmp2 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_18(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4 ^ in5;
+        out1 = in5 ^ in6;
+        tmp0 = in4 ^ in7;
+        out5 = in1 ^ in2 ^ in5;
+        out6 = in2 ^ in3 ^ in6;
+        out2 = tmp0 ^ out1;
+        out7 = tmp0 ^ in3;
+        tmp1 = tmp0 ^ in0;
+        out3 = tmp1 ^ in6;
+        out4 = tmp1 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_19(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in1 ^ in2;
+        out7 = in3 ^ in4;
+        tmp0 = in0 ^ in7;
+        out6 = in2 ^ in3;
+        out1 = in1 ^ in5 ^ in6;
+        out0 = in0 ^ in4 ^ in5;
+        out4 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in6;
+        out2 = tmp1 ^ out0 ^ in2;
+        out3 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in5;
+        tmp1 = in5 ^ in6;
+        tmp2 = tmp0 ^ in1;
+        out0 = tmp0 ^ in7;
+        out1 = tmp1 ^ in0;
+        tmp3 = tmp1 ^ in3;
+        out5 = tmp2 ^ in2;
+        out2 = tmp2 ^ in6;
+        out7 = tmp3 ^ out0;
+        out6 = tmp3 ^ in2;
+        out4 = tmp3 ^ out2 ^ in0;
+        out3 = tmp0 ^ out1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = in2 ^ in5;
+        tmp2 = in3 ^ in6;
+        out5 = tmp0 ^ in1;
+        tmp3 = tmp0 ^ in0;
+        out6 = tmp1 ^ in3;
+        out0 = tmp1 ^ tmp3 ^ in7;
+        out7 = tmp2 ^ in4;
+        tmp4 = out5 ^ in6;
+        out3 = tmp2 ^ tmp3;
+        out2 = tmp4 ^ in5;
+        out4 = tmp4 ^ out3;
+        out1 = tmp3 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in4 ^ in6;
+        tmp2 = in5 ^ in7;
+        out6 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ in5;
+        out1 = tmp2 ^ in6;
+        tmp3 = tmp2 ^ in1;
+        tmp4 = tmp2 ^ in4;
+        out2 = tmp4 ^ in0;
+        out7 = tmp4 ^ in3;
+        out5 = tmp0 ^ tmp3;
+        out3 = tmp3 ^ out2;
+        out4 = out3 ^ in2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in0 ^ in4;
+        tmp2 = in3 ^ in4;
+        tmp3 = in2 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out5 = tmp0 ^ tmp3;
+        tmp4 = tmp1 ^ in5;
+        out6 = tmp2 ^ in2;
+        out7 = tmp2 ^ in5;
+        out2 = tmp3 ^ tmp4;
+        out4 = out3 ^ out6 ^ in6;
+        out0 = tmp4 ^ in6;
+        out1 = out2 ^ out4 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in1;
+        out3 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ in5;
+        out4 = out3 ^ in3 ^ in6;
+        tmp3 = out4 ^ in7;
+        out6 = tmp3 ^ out2 ^ in4;
+        out7 = tmp1 ^ out6;
+        out0 = out7 ^ in3;
+        out1 = tmp0 ^ out0;
+        out5 = tmp3 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in6;
+        tmp1 = tmp0 ^ in5;
+        out7 = tmp1 ^ in3;
+        out0 = tmp1 ^ in0 ^ in7;
+        out6 = out7 ^ in2 ^ in6;
+        out1 = out0 ^ in1 ^ in4;
+        out4 = out0 ^ out6 ^ in1;
+        out3 = tmp0 ^ out4;
+        out2 = out4 ^ out7 ^ in7;
+        out5 = out3 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_20(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in4;
+        out0 = in3 ^ in7;
+        tmp0 = in3 ^ in4;
+        tmp1 = in6 ^ in7;
+        out2 = out0 ^ in5;
+        out4 = tmp0 ^ in5;
+        out3 = tmp0 ^ tmp1;
+        out7 = tmp1 ^ in2;
+        out6 = tmp1 ^ in1 ^ in5;
+        out5 = out2 ^ out3 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_21(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in1 ^ in4;
+        tmp0 = in4 ^ in6;
+        out4 = in3 ^ in5;
+        out7 = in2 ^ in6;
+        out0 = in0 ^ in3 ^ in7;
+        out6 = in1 ^ in5 ^ in7;
+        out3 = tmp0 ^ in7;
+        out5 = tmp0 ^ in0;
+        out2 = out4 ^ in2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_22(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3;
+        out1 = in0 ^ in4;
+        out7 = in2 ^ in7;
+        out4 = in4 ^ in5 ^ in7;
+        out5 = in0 ^ in5 ^ in6;
+        out6 = in1 ^ in6 ^ in7;
+        out3 = in2 ^ in3 ^ in4 ^ in6;
+        out2 = in1 ^ in3 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_23(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in2;
+        out0 = in0 ^ in3;
+        out4 = in5 ^ in7;
+        out5 = in0 ^ in6;
+        out6 = in1 ^ in7;
+        out3 = in2 ^ in4 ^ in6;
+        out1 = in0 ^ in1 ^ in4;
+        out2 = out4 ^ out6 ^ in2 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_24(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in4 ^ in7;
+        tmp0 = in3 ^ in4;
+        out0 = in3 ^ in6 ^ in7;
+        out3 = tmp0 ^ in1;
+        tmp1 = out0 ^ in5;
+        out6 = tmp1 ^ out3;
+        out2 = tmp1 ^ in0;
+        out7 = tmp1 ^ in2 ^ in3;
+        out5 = out2 ^ in4;
+        out4 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_25(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in4;
+        tmp0 = in2 ^ in5;
+        out1 = out3 ^ in7;
+        out7 = tmp0 ^ in6;
+        out6 = out1 ^ in5;
+        out4 = out7 ^ in3 ^ in7;
+        out2 = out4 ^ in0;
+        out0 = tmp0 ^ out2;
+        out5 = out0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_26(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3 ^ in6;
+        tmp0 = in4 ^ in7;
+        out7 = in2 ^ in5 ^ in7;
+        tmp1 = out0 ^ in0 ^ in5;
+        out1 = tmp0 ^ in0;
+        tmp2 = tmp0 ^ in6;
+        out2 = tmp1 ^ in1;
+        out5 = tmp1 ^ in7;
+        out6 = tmp2 ^ in1;
+        out4 = tmp2 ^ out7;
+        out3 = out0 ^ out6 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_27(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in2 ^ in5;
+        out0 = in0 ^ in3 ^ in6;
+        out6 = in1 ^ in4 ^ in7;
+        out4 = out7 ^ in6;
+        out2 = out0 ^ out7 ^ in1;
+        out5 = out0 ^ in7;
+        out1 = out6 ^ in0;
+        out3 = out6 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_28(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3;
+        out1 = in4 ^ in6;
+        out0 = in3 ^ in5 ^ in7;
+        tmp0 = out1 ^ in7;
+        tmp1 = out0 ^ in4;
+        out7 = tmp0 ^ in2;
+        tmp2 = tmp0 ^ in1;
+        out3 = tmp1 ^ in0;
+        out6 = tmp1 ^ tmp2;
+        out4 = tmp2 ^ in3;
+        out5 = out3 ^ in2 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_29(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in3;
+        tmp0 = in1 ^ in3;
+        tmp1 = in4 ^ in6;
+        tmp2 = in0 ^ in4 ^ in7;
+        out6 = tmp0 ^ in5;
+        out4 = tmp0 ^ in6 ^ in7;
+        out1 = tmp1 ^ in1;
+        out7 = tmp1 ^ in2;
+        out3 = tmp2 ^ in5;
+        out5 = tmp2 ^ in2;
+        out0 = out3 ^ in3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3 ^ in5;
+        tmp0 = in1 ^ in3;
+        tmp1 = in0 ^ in4;
+        out7 = in2 ^ in4 ^ in7;
+        out3 = tmp1 ^ out0 ^ in2;
+        out2 = tmp0 ^ in7;
+        out6 = tmp0 ^ in6;
+        out1 = tmp1 ^ in6;
+        out5 = tmp1 ^ out7 ^ in5;
+        out4 = out1 ^ in0 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in6;
+        out7 = in2 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in2 ^ in7;
+        out6 = in1 ^ in3;
+        out1 = out4 ^ in0 ^ in4;
+        out3 = tmp0 ^ out7;
+        out0 = tmp0 ^ in3;
+        out5 = tmp1 ^ in0;
+        out2 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in2 ^ in3 ^ in4;
+        tmp2 = tmp0 ^ in6;
+        out4 = tmp1 ^ in1;
+        out5 = tmp1 ^ in0 ^ in5;
+        tmp3 = tmp2 ^ in4;
+        out6 = tmp2 ^ out4;
+        out7 = tmp3 ^ in7;
+        out2 = tmp3 ^ out5;
+        out3 = out6 ^ in0;
+        out0 = tmp1 ^ out7;
+        out1 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out4 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in0;
+        out2 = tmp1 ^ in6;
+        out5 = tmp1 ^ in4;
+        tmp2 = out2 ^ in2;
+        tmp3 = tmp2 ^ in5;
+        out0 = tmp3 ^ in7;
+        out7 = tmp3 ^ out5;
+        out6 = out4 ^ out7 ^ in6;
+        out3 = tmp2 ^ out6;
+        out1 = out0 ^ out6 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in7;
+        out0 = in3 ^ in5 ^ in6;
+        tmp1 = tmp0 ^ in0;
+        tmp2 = tmp0 ^ in2;
+        out1 = tmp1 ^ in6;
+        out4 = tmp2 ^ in1;
+        out7 = tmp2 ^ in5;
+        out3 = out0 ^ out4 ^ in0;
+        out2 = out3 ^ out7 ^ in7;
+        out6 = tmp1 ^ out2;
+        out5 = tmp1 ^ out7 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in2 ^ in5;
+        out4 = in1 ^ in2 ^ in7;
+        out6 = in1 ^ in3 ^ in4;
+        out5 = tmp0 ^ in2;
+        tmp2 = tmp0 ^ in6;
+        out7 = tmp1 ^ in4;
+        out0 = tmp2 ^ in5;
+        out2 = tmp2 ^ out4;
+        out1 = tmp2 ^ out6 ^ in7;
+        out3 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_30(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in4 ^ in5;
+        tmp0 = in3 ^ in6;
+        tmp1 = in4 ^ in7;
+        out6 = in1 ^ in2 ^ in5;
+        out3 = tmp0 ^ in5;
+        out4 = tmp0 ^ in0;
+        out7 = tmp0 ^ in2;
+        out0 = tmp1 ^ in3;
+        out2 = tmp1 ^ out3;
+        out5 = tmp1 ^ in0 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_31(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in5 ^ in6;
+        tmp0 = in4 ^ in5;
+        tmp1 = in0 ^ in3 ^ in4;
+        tmp2 = out3 ^ in2;
+        out1 = tmp0 ^ in1;
+        out0 = tmp1 ^ in7;
+        out4 = tmp1 ^ in6;
+        out6 = tmp2 ^ in1;
+        out2 = tmp2 ^ out0 ^ in0;
+        out5 = out1 ^ in0 ^ in7;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_32(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3 ^ in4;
+        out7 = in2 ^ in3;
+        tmp0 = in5 ^ in6;
+        tmp1 = in0 ^ in7;
+        out6 = in1 ^ in2;
+        out1 = in0 ^ in4 ^ in5;
+        out2 = tmp0 ^ out0 ^ in1;
+        out3 = tmp0 ^ out7 ^ in7;
+        out4 = tmp1 ^ in6;
+        out5 = tmp1 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_33(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in0 ^ in4;
+        tmp2 = in1 ^ in5;
+        out6 = in1 ^ in2 ^ in6;
+        out7 = tmp0 ^ in7;
+        out0 = tmp1 ^ in3;
+        out1 = tmp1 ^ tmp2;
+        tmp3 = tmp2 ^ in7;
+        tmp4 = tmp2 ^ in4 ^ in6;
+        out5 = tmp3 ^ in0;
+        out3 = tmp3 ^ out6;
+        out4 = tmp4 ^ out5;
+        out2 = tmp0 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_34(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in4 ^ in5;
+        tmp2 = tmp0 ^ in1;
+        tmp3 = tmp0 ^ in6;
+        out1 = tmp1 ^ in7;
+        tmp4 = tmp1 ^ in2;
+        out5 = tmp2 ^ in0;
+        out3 = tmp2 ^ out1;
+        out0 = tmp3 ^ in7;
+        out7 = tmp3 ^ tmp4;
+        out6 = tmp4 ^ in1;
+        out2 = out3 ^ out5 ^ in3;
+        out4 = tmp4 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_35(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        tmp1 = in5 ^ in7;
+        out7 = tmp0 ^ tmp1 ^ in3;
+        out3 = tmp1 ^ in1;
+        out1 = out3 ^ in4;
+        tmp2 = out1 ^ in7;
+        out5 = tmp2 ^ in0 ^ in3;
+        out6 = tmp0 ^ tmp2;
+        out0 = out3 ^ out5 ^ in6;
+        out4 = tmp0 ^ out0;
+        out2 = out4 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_36(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in2;
+        tmp0 = in1 ^ in3;
+        out0 = in3 ^ in4 ^ in6;
+        out6 = in1 ^ in2 ^ in4;
+        out5 = tmp0 ^ in0;
+        tmp1 = out5 ^ in5;
+        out2 = tmp1 ^ in4;
+        out3 = tmp1 ^ out4;
+        out1 = tmp0 ^ out2 ^ in7;
+        out7 = out3 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_37(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in4;
+        tmp2 = tmp0 ^ in6;
+        out3 = tmp0 ^ in5;
+        out4 = tmp1 ^ in0;
+        out6 = tmp2 ^ in4;
+        out1 = out3 ^ out4 ^ in7;
+        tmp3 = out4 ^ in1 ^ in3;
+        out7 = tmp3 ^ out1;
+        out2 = tmp3 ^ in5;
+        out5 = tmp1 ^ out2;
+        out0 = tmp2 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_38(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in3;
+        tmp0 = in3 ^ in4;
+        tmp1 = in5 ^ in7;
+        tmp2 = out3 ^ in1;
+        out2 = tmp0 ^ in6;
+        out0 = tmp0 ^ tmp1;
+        out4 = tmp1 ^ tmp2;
+        out7 = out2 ^ in2;
+        out1 = out2 ^ in3 ^ in5;
+        out6 = out4 ^ in0 ^ in2;
+        out5 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_39(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0;
+        tmp0 = in1 ^ in5;
+        tmp1 = tmp0 ^ in4;
+        out1 = tmp1 ^ in6;
+        out5 = out1 ^ in0 ^ in2;
+        tmp2 = tmp0 ^ out5;
+        out2 = tmp2 ^ in0 ^ in3;
+        out7 = out2 ^ in7;
+        out6 = tmp1 ^ out7;
+        out4 = tmp2 ^ out6;
+        out0 = out4 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in0 ^ in2;
+        tmp2 = in3 ^ in4;
+        tmp3 = in1 ^ in6;
+        tmp4 = in3 ^ in7;
+        out4 = tmp0 ^ in5;
+        out5 = tmp1 ^ tmp3;
+        out3 = tmp1 ^ tmp4;
+        out0 = tmp2 ^ in5;
+        out7 = tmp2 ^ in2;
+        tmp5 = tmp3 ^ in4;
+        out2 = tmp4 ^ tmp5;
+        out1 = tmp5 ^ out4;
+        out6 = tmp0 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in6;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in3;
+        out3 = tmp1 ^ in0;
+        out6 = tmp1 ^ tmp2;
+        out2 = out6 ^ in4;
+        out7 = tmp0 ^ out2;
+        out0 = out3 ^ out7 ^ in5;
+        out5 = out0 ^ out2 ^ in7;
+        out1 = tmp2 ^ out0;
+        out4 = out1 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in2 ^ in7;
+        tmp2 = in1 ^ in6 ^ in7;
+        out2 = tmp0 ^ in4;
+        out3 = tmp0 ^ tmp2;
+        out4 = tmp1 ^ out3 ^ in5;
+        out5 = tmp2 ^ out2 ^ in2;
+        out1 = out4 ^ out5 ^ in6;
+        out0 = out1 ^ in3;
+        out7 = tmp1 ^ out0;
+        out6 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = tmp0 ^ in3;
+        out2 = tmp1 ^ in4;
+        tmp2 = out2 ^ in5;
+        out4 = tmp2 ^ in1 ^ in6;
+        out5 = out4 ^ in7;
+        out6 = out5 ^ in0;
+        out7 = out6 ^ in1;
+        out0 = tmp0 ^ out7;
+        out1 = tmp1 ^ out5;
+        out3 = tmp2 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = tmp0 ^ in4;
+        out0 = tmp1 ^ in6;
+        out7 = tmp1 ^ in2;
+        out6 = out7 ^ in1 ^ in5 ^ in7;
+        out2 = out6 ^ in0 ^ in2;
+        out4 = out0 ^ out6 ^ in0;
+        out5 = tmp0 ^ out4;
+        out3 = out5 ^ in7;
+        out1 = out3 ^ out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        out3 = tmp0 ^ in2 ^ in6;
+        tmp1 = out3 ^ in5 ^ in7;
+        out4 = tmp1 ^ in4;
+        out5 = tmp1 ^ in3;
+        out1 = out4 ^ in2;
+        out7 = out1 ^ out3 ^ in3;
+        out2 = tmp0 ^ out7 ^ in5;
+        tmp2 = out2 ^ in0;
+        out6 = tmp2 ^ in6;
+        out0 = tmp1 ^ tmp2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_40(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3 ^ in7;
+        tmp0 = in3 ^ in4;
+        tmp1 = in6 ^ in7;
+        out4 = tmp0 ^ in2;
+        out5 = tmp0 ^ in5;
+        out0 = tmp1 ^ in2;
+        out7 = tmp1 ^ in1 ^ in5;
+        out2 = out0 ^ in4;
+        out3 = out2 ^ out5 ^ in7;
+        out6 = out3 ^ out4 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_41(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in3;
+        tmp0 = in5 ^ in6;
+        tmp1 = in6 ^ in7;
+        out5 = in3 ^ in4;
+        out1 = in1 ^ in3 ^ in7;
+        out6 = in0 ^ in4 ^ in5;
+        out3 = tmp0 ^ in2;
+        out7 = tmp0 ^ in1;
+        out2 = tmp1 ^ in4;
+        out0 = tmp1 ^ in0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_42(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in6;
+        out5 = in3 ^ in5;
+        out1 = in0 ^ in3 ^ in7;
+        out7 = in1 ^ in5 ^ in7;
+        out4 = in2 ^ in4 ^ in7;
+        out6 = in0 ^ in4 ^ in6;
+        out2 = out0 ^ in1 ^ in4;
+        out3 = out5 ^ in6 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_43(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in3;
+        out7 = in1 ^ in5;
+        out4 = in2 ^ in7;
+        out6 = in0 ^ in4;
+        out0 = in0 ^ in2 ^ in6;
+        out3 = in5 ^ in6 ^ in7;
+        out2 = in1 ^ in4 ^ in6;
+        out1 = in0 ^ in1 ^ in3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_44(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3;
+        out0 = in2 ^ in7;
+        tmp0 = in4 ^ in7;
+        out7 = in1 ^ in6 ^ in7;
+        out6 = in0 ^ in5 ^ in6;
+        out4 = tmp0 ^ in3 ^ in6;
+        out3 = out0 ^ in1 ^ in3 ^ in5;
+        out2 = out0 ^ in0 ^ in4;
+        out5 = tmp0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_45(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in1 ^ in3;
+        out7 = in1 ^ in6;
+        out5 = in4 ^ in7;
+        out6 = in0 ^ in5;
+        out0 = in0 ^ in2 ^ in7;
+        out4 = in3 ^ in6 ^ in7;
+        out2 = out5 ^ in0;
+        out3 = out0 ^ out6 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_46(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2;
+        out1 = in0 ^ in3;
+        out7 = in1 ^ in7;
+        out4 = in4 ^ in6;
+        out5 = in5 ^ in7;
+        out6 = in0 ^ in6;
+        out3 = in1 ^ in3 ^ in5;
+        out2 = out4 ^ out6 ^ in1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_47(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in6;
+        out7 = in1;
+        out5 = in7;
+        out6 = in0;
+        tmp0 = in0 ^ in1;
+        out3 = in1 ^ in5;
+        out0 = in0 ^ in2;
+        out1 = tmp0 ^ in3;
+        out2 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_48(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out1 = in3 ^ in6 ^ in7;
+        out3 = tmp0 ^ in0;
+        out0 = tmp0 ^ out1 ^ in5;
+        tmp1 = out0 ^ in4;
+        out2 = tmp1 ^ in7;
+        out5 = tmp1 ^ in3;
+        out4 = out5 ^ in1;
+        out7 = tmp0 ^ out4;
+        out6 = tmp1 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_49(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in2;
+        tmp0 = in2 ^ in5;
+        out2 = in4 ^ in5 ^ in6;
+        tmp1 = tmp0 ^ out2 ^ in3;
+        out7 = out2 ^ in1;
+        out5 = tmp1 ^ in7;
+        out4 = out5 ^ out7 ^ in6;
+        out1 = tmp0 ^ out4;
+        out6 = out1 ^ out7 ^ in0;
+        out0 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        tmp1 = in3 ^ in7;
+        out0 = tmp0 ^ in5;
+        out3 = tmp1 ^ in0;
+        out5 = tmp1 ^ out0;
+        out4 = out0 ^ in1 ^ in4;
+        out1 = out3 ^ in6;
+        out2 = out4 ^ in7;
+        out6 = out1 ^ in4;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in7;
+        tmp0 = in1 ^ in5;
+        tmp1 = in2 ^ in6;
+        tmp2 = out3 ^ in3;
+        out7 = tmp0 ^ in4;
+        out4 = tmp0 ^ tmp1;
+        tmp3 = tmp1 ^ in0;
+        out6 = tmp2 ^ in4;
+        out5 = tmp2 ^ tmp3;
+        out1 = tmp2 ^ in1 ^ in6;
+        out2 = out7 ^ in6 ^ in7;
+        out0 = tmp3 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3 ^ in6;
+        tmp0 = in2 ^ in5;
+        tmp1 = out1 ^ in5 ^ in7;
+        out0 = tmp0 ^ in7;
+        tmp2 = tmp0 ^ in4;
+        out6 = tmp1 ^ in0;
+        out2 = tmp2 ^ in0;
+        out5 = tmp2 ^ in6;
+        out3 = tmp0 ^ out6 ^ in1;
+        out7 = out0 ^ out5 ^ in1;
+        out4 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in6;
+        out4 = in1 ^ in3 ^ in5;
+        tmp2 = tmp0 ^ in7;
+        out2 = tmp0 ^ in4;
+        out1 = tmp1 ^ in3;
+        out7 = tmp1 ^ in4;
+        out0 = tmp2 ^ in2;
+        out6 = tmp2 ^ in3;
+        out5 = out7 ^ in1 ^ in2;
+        out3 = tmp1 ^ out0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in5;
+        out7 = in1 ^ in4 ^ in7;
+        out1 = in0 ^ in3 ^ in6;
+        out5 = out0 ^ in6;
+        out4 = out7 ^ in5;
+        out3 = out1 ^ in1;
+        out6 = out1 ^ in7;
+        out2 = out4 ^ in0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in2 ^ in6;
+        out7 = in1 ^ in4;
+        out3 = in0 ^ in1 ^ in6;
+        out4 = in1 ^ in5 ^ in7;
+        out0 = in0 ^ in2 ^ in5;
+        out6 = in0 ^ in3 ^ in7;
+        out1 = out3 ^ in3;
+        out2 = out4 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_50(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in7;
+        tmp0 = in3 ^ in5;
+        out0 = out2 ^ in4 ^ in6;
+        out1 = tmp0 ^ in7;
+        tmp1 = tmp0 ^ in6;
+        out3 = out0 ^ in3;
+        out7 = tmp1 ^ in1;
+        tmp2 = tmp1 ^ in0;
+        out5 = out3 ^ in1 ^ in2;
+        out4 = tmp2 ^ in2;
+        out6 = tmp2 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_51(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in7;
+        out3 = in2 ^ in4 ^ in6 ^ in7;
+        out0 = out3 ^ in0;
+        out6 = out0 ^ in5;
+        out4 = out6 ^ in3 ^ in7;
+        out1 = out0 ^ out4 ^ in1;
+        out7 = out1 ^ in6;
+        out5 = out7 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_52(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in2;
+        tmp0 = in2 ^ in4;
+        tmp1 = in3 ^ in5;
+        tmp2 = in3 ^ in6;
+        tmp3 = in0 ^ in7;
+        out0 = tmp0 ^ in6;
+        out6 = tmp0 ^ tmp3;
+        out7 = tmp1 ^ in1;
+        out1 = tmp1 ^ tmp3;
+        out3 = tmp2 ^ in4;
+        out5 = tmp2 ^ in1 ^ in7;
+        out4 = tmp2 ^ out1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_53(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1;
+        out3 = in4 ^ in6;
+        out0 = out3 ^ in0 ^ in2;
+        out6 = out0 ^ in7;
+        out4 = out6 ^ in5;
+        out7 = out0 ^ out4 ^ in1 ^ in3;
+        out1 = out7 ^ in0;
+        out5 = out7 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_54(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3 ^ in5;
+        tmp0 = in1 ^ in3;
+        tmp1 = in2 ^ in4;
+        tmp2 = in0 ^ in7;
+        out5 = in1 ^ in4 ^ in6;
+        out4 = tmp2 ^ out1;
+        out7 = tmp0 ^ in6;
+        out3 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ in7;
+        tmp3 = tmp2 ^ in2;
+        out2 = tmp3 ^ in6;
+        out6 = tmp3 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_55(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in1 ^ in4;
+        tmp2 = in6 ^ in7;
+        out7 = tmp0 ^ tmp2;
+        out1 = tmp0 ^ in5;
+        out3 = tmp1 ^ in2;
+        out5 = tmp1 ^ in5 ^ in6;
+        out2 = tmp2 ^ in0;
+        out4 = out5 ^ out7 ^ in0;
+        out6 = out2 ^ in2 ^ in5;
+        out0 = out5 ^ out6 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_56(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in4;
+        tmp0 = in0 ^ in2;
+        out4 = in0 ^ in5;
+        out7 = in1 ^ in3;
+        out5 = in1 ^ in6;
+        out6 = tmp0 ^ in7;
+        out2 = tmp0 ^ out5;
+        out1 = out4 ^ in3;
+        out3 = out7 ^ in4 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_57(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in7;
+        out0 = in0 ^ in2 ^ in4;
+        out5 = in1 ^ in5 ^ in6;
+        out4 = tmp0 ^ in4;
+        out1 = tmp0 ^ in1 ^ in3;
+        out2 = tmp0 ^ out5;
+        out3 = tmp1 ^ in4;
+        out7 = tmp1 ^ in3;
+        out6 = tmp1 ^ out2 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_58(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in5;
+        tmp0 = in2 ^ in3 ^ in4;
+        out5 = tmp0 ^ in1;
+        out6 = tmp0 ^ in0 ^ in5;
+        out3 = out6 ^ in7;
+        tmp1 = out2 ^ out5;
+        out7 = tmp1 ^ in6;
+        out4 = tmp1 ^ out3 ^ in3;
+        out0 = out4 ^ out7 ^ in0;
+        out1 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_59(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in5;
+        tmp0 = in0 ^ in5 ^ in7;
+        out3 = tmp0 ^ in2 ^ in4;
+        out0 = out3 ^ in6;
+        tmp1 = out0 ^ in7;
+        out6 = tmp1 ^ in3;
+        out5 = out6 ^ in0 ^ in1 ^ in6;
+        out4 = tmp0 ^ out5;
+        out1 = tmp1 ^ out4;
+        out7 = out1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in5;
+        out5 = tmp0 ^ in3;
+        out4 = tmp0 ^ in0;
+        tmp2 = tmp1 ^ in4;
+        out2 = tmp1 ^ in1 ^ in7;
+        out7 = tmp2 ^ out5;
+        out6 = out4 ^ out7 ^ in5;
+        out0 = tmp2 ^ in6;
+        out1 = out0 ^ out6 ^ in7;
+        out3 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in0 ^ in4;
+        tmp2 = in1 ^ in5;
+        out5 = tmp0 ^ tmp2;
+        tmp3 = tmp1 ^ in6;
+        out3 = tmp1 ^ in5;
+        out2 = tmp2 ^ in7;
+        tmp4 = out3 ^ in2;
+        out7 = out2 ^ in3 ^ in4;
+        out0 = tmp4 ^ in6;
+        out6 = tmp0 ^ tmp3;
+        out4 = tmp2 ^ tmp4;
+        out1 = tmp3 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in6;
+        tmp1 = in0 ^ in2 ^ in5;
+        out1 = tmp0 ^ in5;
+        tmp2 = tmp0 ^ in1;
+        out2 = tmp1 ^ in6;
+        out6 = tmp1 ^ in3;
+        out4 = tmp2 ^ in0;
+        out7 = tmp2 ^ in4;
+        out3 = tmp1 ^ out7;
+        out0 = out3 ^ out4 ^ in7;
+        out5 = out0 ^ in1 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in0 ^ in6;
+        out2 = tmp1 ^ in5;
+        tmp2 = out2 ^ in3;
+        out6 = tmp2 ^ in2;
+        out1 = tmp0 ^ tmp2;
+        tmp3 = out1 ^ in4 ^ in5;
+        out4 = tmp3 ^ in0;
+        out7 = tmp3 ^ in7;
+        tmp4 = out4 ^ out6;
+        out5 = tmp4 ^ in7;
+        out0 = tmp0 ^ out5;
+        out3 = tmp1 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in3 ^ in5;
+        tmp2 = in1 ^ in7;
+        out7 = in1 ^ in3 ^ in4;
+        out0 = tmp0 ^ in4;
+        tmp3 = tmp1 ^ in0;
+        out5 = tmp2 ^ in2;
+        out1 = tmp3 ^ in6;
+        out6 = tmp0 ^ tmp3;
+        tmp4 = tmp2 ^ out1;
+        out3 = tmp4 ^ in4;
+        out4 = tmp1 ^ tmp4;
+        out2 = tmp0 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in5;
+        tmp1 = in0 ^ in6;
+        tmp2 = tmp0 ^ in7;
+        tmp3 = tmp1 ^ in3;
+        out2 = tmp1 ^ tmp2;
+        out5 = tmp2 ^ in2;
+        out6 = tmp3 ^ in2;
+        out3 = out2 ^ in4;
+        out4 = out3 ^ in5;
+        out1 = tmp0 ^ tmp3;
+        out7 = tmp3 ^ out4;
+        out0 = out4 ^ out5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_60(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in5;
+        tmp0 = in3 ^ in6;
+        out1 = in3 ^ in4 ^ in7;
+        out7 = out4 ^ in1;
+        tmp1 = out4 ^ in4;
+        out0 = tmp0 ^ in2;
+        out5 = tmp0 ^ in0;
+        out2 = tmp0 ^ tmp1;
+        out3 = tmp1 ^ in7;
+        out6 = out3 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_61(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        out4 = tmp0 ^ in4;
+        tmp1 = out4 ^ in3;
+        out3 = tmp1 ^ in7;
+        out2 = tmp1 ^ in2 ^ in6;
+        out1 = tmp0 ^ out3 ^ in1;
+        out0 = out2 ^ out4 ^ in0;
+        out7 = tmp1 ^ out1;
+        out6 = out0 ^ out1 ^ in2;
+        out5 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_62(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in4 ^ in5;
+        tmp0 = in0 ^ in3 ^ in4;
+        out1 = tmp0 ^ in7;
+        out5 = tmp0 ^ in6;
+        tmp1 = out1 ^ in0;
+        tmp2 = tmp1 ^ out3;
+        out4 = tmp2 ^ in2;
+        tmp3 = tmp2 ^ in1;
+        out0 = out4 ^ in5 ^ in6;
+        out7 = tmp3 ^ out0;
+        out6 = tmp0 ^ tmp3;
+        out2 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_63(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in1 ^ in7;
+        out3 = tmp0 ^ in5;
+        tmp2 = out3 ^ in6;
+        out4 = out3 ^ in2 ^ in7;
+        out5 = tmp2 ^ in0;
+        tmp3 = out5 ^ in3;
+        out0 = tmp3 ^ out4;
+        out2 = tmp1 ^ tmp2;
+        out6 = tmp1 ^ tmp3;
+        tmp4 = tmp0 ^ out2;
+        out1 = tmp4 ^ out5;
+        out7 = tmp4 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_64(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in3;
+        out1 = in3 ^ in4;
+        out7 = in1 ^ in2;
+        tmp0 = in4 ^ in5;
+        tmp1 = in0 ^ in7;
+        out4 = in5 ^ in6 ^ in7;
+        out2 = tmp0 ^ out0 ^ in0;
+        out3 = tmp0 ^ out7 ^ in6;
+        out5 = tmp1 ^ in6;
+        out6 = tmp1 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_65(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in4 ^ in5;
+        tmp2 = in6 ^ in7;
+        out7 = in1 ^ in2 ^ in7;
+        out1 = in1 ^ in3 ^ in4;
+        out0 = tmp0 ^ in2;
+        out2 = tmp0 ^ tmp1;
+        out4 = tmp1 ^ tmp2;
+        tmp3 = tmp2 ^ in0;
+        out3 = out4 ^ out7 ^ in3;
+        out5 = tmp3 ^ in5;
+        out6 = tmp3 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_66(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in3;
+        tmp2 = in0 ^ in4;
+        out7 = tmp0 ^ in6;
+        out0 = tmp1 ^ in7;
+        out1 = tmp2 ^ in3;
+        tmp3 = tmp2 ^ in6;
+        tmp4 = out1 ^ in5;
+        out5 = tmp3 ^ in7;
+        out4 = tmp3 ^ tmp4;
+        out2 = tmp0 ^ tmp4 ^ in7;
+        out6 = tmp1 ^ out2 ^ in4;
+        out3 = tmp3 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_67(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp0 ^ in7;
+        out1 = tmp1 ^ in4;
+        out0 = tmp2 ^ in2;
+        tmp3 = out1 ^ in7;
+        out2 = tmp3 ^ in5;
+        out3 = out2 ^ in0 ^ in6;
+        out7 = tmp1 ^ out0 ^ in6;
+        out5 = tmp1 ^ out3;
+        out4 = tmp2 ^ out5;
+        out6 = tmp3 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_68(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in2 ^ in3 ^ in5;
+        tmp2 = tmp0 ^ in1;
+        tmp3 = tmp0 ^ in6;
+        out0 = tmp1 ^ in6;
+        out6 = tmp2 ^ in0;
+        out7 = tmp1 ^ tmp2;
+        out1 = tmp3 ^ in7;
+        out2 = out1 ^ in2;
+        out4 = tmp2 ^ out2;
+        out3 = out4 ^ out6 ^ in3;
+        out5 = tmp3 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_69(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in6 ^ in7;
+        out2 = tmp0 ^ in3 ^ in4;
+        out1 = out2 ^ in1;
+        out3 = out2 ^ in0 ^ in2;
+        out4 = out1 ^ in2 ^ in3;
+        out6 = out1 ^ in0 ^ in7;
+        out7 = out4 ^ in5 ^ in6;
+        out5 = out4 ^ out6 ^ in5;
+        out0 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        out3 = in0 ^ in4 ^ in6;
+        tmp1 = tmp0 ^ in3;
+        out4 = tmp1 ^ in1;
+        tmp2 = tmp1 ^ in7;
+        out2 = out4 ^ in4;
+        out0 = tmp2 ^ in5;
+        out5 = tmp2 ^ out3;
+        out7 = out2 ^ in3 ^ in5;
+        out1 = tmp0 ^ out5;
+        out6 = tmp1 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in6;
+        out2 = tmp0 ^ in1 ^ in3;
+        out4 = out2 ^ in2;
+        tmp1 = out2 ^ in0;
+        out7 = out4 ^ in3 ^ in5 ^ in7;
+        out1 = tmp1 ^ in7;
+        out3 = tmp1 ^ in1;
+        out6 = tmp1 ^ in5;
+        out0 = tmp1 ^ out7 ^ in6;
+        out5 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1;
+        tmp0 = in2 ^ in3;
+        out5 = in0 ^ in2;
+        out1 = in3 ^ in4 ^ in6;
+        tmp1 = out5 ^ in1;
+        out0 = tmp0 ^ in5;
+        out6 = tmp0 ^ tmp1;
+        out3 = tmp1 ^ in4;
+        out7 = out3 ^ in0;
+        out2 = out6 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in4;
+        tmp0 = in0 ^ in2;
+        tmp1 = out4 ^ in3;
+        out7 = out4 ^ in2 ^ in7;
+        out5 = tmp0 ^ in5;
+        out3 = tmp0 ^ tmp1;
+        out1 = tmp1 ^ in6;
+        out0 = out5 ^ in3;
+        out2 = out3 ^ out7 ^ in4;
+        out6 = out1 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in0 ^ in4;
+        out4 = tmp0 ^ in7;
+        out6 = tmp0 ^ in0 ^ in5;
+        out5 = tmp1 ^ in2;
+        tmp2 = tmp1 ^ in3;
+        out3 = tmp2 ^ out4;
+        out1 = tmp2 ^ in6;
+        out2 = tmp0 ^ out5;
+        out0 = out2 ^ out3 ^ in5;
+        out7 = out1 ^ out2 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in7;
+        tmp1 = tmp0 ^ in4;
+        tmp2 = tmp0 ^ in0 ^ in2;
+        out4 = tmp1 ^ in1;
+        out0 = tmp2 ^ in5;
+        out3 = out4 ^ in0;
+        out2 = out3 ^ in7;
+        out1 = out2 ^ in6;
+        out6 = out1 ^ in4 ^ in5;
+        out7 = tmp2 ^ out1;
+        out5 = tmp1 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_70(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2;
+        tmp0 = in2 ^ in4;
+        out2 = in2 ^ in3 ^ in5;
+        tmp1 = tmp0 ^ in6;
+        tmp2 = out2 ^ in7;
+        out0 = tmp1 ^ in3;
+        out4 = tmp1 ^ in0;
+        out7 = tmp2 ^ in1;
+        out6 = out4 ^ in1;
+        out5 = out7 ^ in0 ^ in2;
+        out1 = tmp0 ^ tmp2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_71(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in5;
+        out3 = in2 ^ in3;
+        tmp0 = in0 ^ in2;
+        tmp1 = out2 ^ in1;
+        out4 = tmp0 ^ in6;
+        tmp2 = tmp0 ^ in1;
+        out7 = tmp1 ^ in2;
+        out1 = tmp1 ^ in4 ^ in7;
+        out0 = out4 ^ in3 ^ in4;
+        out6 = tmp2 ^ in4;
+        out5 = tmp2 ^ out3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_72(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in7;
+        tmp0 = in0 ^ in4;
+        tmp1 = tmp0 ^ in3 ^ in7;
+        out1 = tmp1 ^ in5;
+        out5 = out1 ^ in1;
+        tmp2 = tmp0 ^ out5;
+        out2 = tmp2 ^ in2;
+        out7 = out2 ^ in6;
+        out6 = tmp1 ^ out7;
+        out4 = tmp2 ^ out6;
+        out0 = out4 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_73(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in7;
+        out2 = out3 ^ in1 ^ in5;
+        out1 = out2 ^ in0 ^ in4;
+        out5 = out1 ^ in5;
+        out6 = out1 ^ out3 ^ in2;
+        out0 = out2 ^ out6 ^ in6;
+        out7 = out0 ^ out1 ^ in3;
+        out4 = out0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_74(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in1 ^ in2 ^ in6;
+        out4 = in0 ^ in4 ^ in7;
+        out5 = in0 ^ in1 ^ in5;
+        out0 = tmp0 ^ in2;
+        out1 = tmp0 ^ in5;
+        out3 = tmp1 ^ in7;
+        out6 = tmp1 ^ in0;
+        out2 = tmp1 ^ out5 ^ in3;
+        out7 = out3 ^ in3 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_75(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in7;
+        tmp0 = in1 ^ in3;
+        out5 = in0 ^ in1;
+        out7 = tmp0 ^ in2;
+        tmp1 = tmp0 ^ in4;
+        out6 = out5 ^ in2;
+        tmp2 = out7 ^ in6;
+        out1 = tmp1 ^ in5;
+        out0 = tmp1 ^ out6;
+        out3 = tmp2 ^ in7;
+        out2 = tmp2 ^ out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_76(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in6;
+        tmp0 = in0 ^ in5;
+        tmp1 = in3 ^ in7;
+        tmp2 = tmp0 ^ in4;
+        tmp3 = tmp1 ^ in2;
+        out5 = tmp2 ^ in1;
+        out1 = tmp2 ^ in3;
+        out0 = tmp3 ^ in4;
+        out4 = out1 ^ in5;
+        out7 = tmp3 ^ out3;
+        out2 = tmp0 ^ out7;
+        out6 = tmp1 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_77(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in3;
+        tmp0 = in1 ^ in4;
+        tmp1 = in1 ^ in6;
+        tmp2 = out4 ^ in5;
+        out5 = tmp0 ^ in0;
+        out1 = tmp0 ^ tmp2;
+        out3 = tmp1 ^ in3;
+        out2 = tmp1 ^ tmp2 ^ in7;
+        out7 = out3 ^ in2;
+        tmp3 = out7 ^ in6;
+        out6 = tmp2 ^ tmp3;
+        out0 = tmp3 ^ out5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_78(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in2 ^ in7;
+        tmp2 = in0 ^ in5 ^ in6;
+        out2 = tmp1 ^ in3;
+        out3 = tmp2 ^ in2;
+        out5 = out3 ^ in1 ^ in3;
+        out0 = tmp0 ^ out3 ^ in4;
+        out1 = tmp1 ^ out0;
+        out4 = out1 ^ out5 ^ in5;
+        out7 = tmp0 ^ out4;
+        out6 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_79(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in7;
+        tmp0 = in3 ^ in4;
+        tmp1 = in1 ^ in5;
+        tmp2 = tmp1 ^ in2;
+        out4 = tmp2 ^ in0 ^ in7;
+        tmp3 = out4 ^ in5;
+        out5 = tmp3 ^ out2 ^ in6;
+        out7 = tmp0 ^ tmp2;
+        out6 = tmp0 ^ tmp3;
+        out3 = tmp1 ^ out5;
+        out0 = out3 ^ in4;
+        out1 = tmp3 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        out2 = tmp0 ^ in3;
+        tmp1 = out2 ^ in4;
+        out4 = tmp1 ^ in0 ^ in5;
+        out5 = out4 ^ in6;
+        out6 = out5 ^ in7;
+        out7 = out6 ^ in0;
+        out0 = out7 ^ in1;
+        out1 = tmp0 ^ out6;
+        out3 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in3;
+        tmp0 = in0 ^ in5;
+        out4 = tmp0 ^ out2 ^ in2;
+        tmp1 = out4 ^ in4;
+        out6 = tmp1 ^ in7;
+        out5 = tmp1 ^ in5 ^ in6;
+        out0 = out6 ^ in1 ^ in6;
+        tmp2 = out0 ^ in2;
+        out1 = tmp2 ^ in1;
+        out3 = tmp2 ^ in4;
+        out7 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = tmp0 ^ in4;
+        out0 = tmp1 ^ in2;
+        out1 = tmp1 ^ in6;
+        out7 = out0 ^ in1 ^ in5 ^ in7;
+        out5 = out1 ^ out7 ^ in0;
+        out3 = out5 ^ in6;
+        out6 = tmp0 ^ out5;
+        out2 = out6 ^ in1;
+        out4 = out2 ^ out7 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = tmp0 ^ in3;
+        tmp2 = tmp0 ^ in6;
+        out7 = tmp1 ^ in4;
+        tmp3 = tmp2 ^ in0;
+        out5 = tmp3 ^ in7;
+        out4 = tmp3 ^ in2 ^ in5;
+        out2 = tmp1 ^ out5;
+        out6 = tmp2 ^ out2;
+        out0 = out4 ^ out7 ^ in6;
+        out1 = tmp3 ^ out0;
+        out3 = out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in0 ^ in5;
+        out1 = tmp0 ^ tmp1 ^ in6;
+        out3 = tmp1 ^ in1;
+        out4 = out1 ^ in1 ^ in7;
+        tmp2 = out4 ^ in3;
+        out5 = tmp2 ^ in2;
+        out6 = tmp0 ^ out5;
+        out7 = tmp1 ^ out4 ^ in2;
+        out2 = out6 ^ in5 ^ in7;
+        out0 = tmp2 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in7;
+        tmp1 = tmp0 ^ in3 ^ in5;
+        tmp2 = tmp1 ^ in0;
+        out0 = tmp2 ^ in4;
+        out6 = tmp2 ^ in1;
+        out3 = tmp0 ^ out6;
+        tmp3 = out3 ^ in6;
+        out1 = tmp3 ^ in4;
+        out2 = tmp3 ^ in5;
+        out4 = tmp3 ^ in7;
+        out5 = tmp1 ^ out1;
+        out7 = out0 ^ out4 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_80(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in4 ^ in5;
+        out1 = in2 ^ in6 ^ in7;
+        out5 = tmp0 ^ in4;
+        tmp2 = tmp0 ^ in1;
+        out6 = tmp1 ^ in3;
+        out7 = tmp1 ^ in0 ^ in6;
+        out4 = tmp2 ^ in7;
+        out3 = tmp2 ^ out6;
+        out2 = out3 ^ out5 ^ in6;
+        out0 = out2 ^ in3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_81(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in6;
+        tmp1 = tmp0 ^ in3;
+        out6 = tmp1 ^ in5;
+        out5 = out6 ^ in2 ^ in6;
+        out3 = out5 ^ in1;
+        out2 = tmp0 ^ out3;
+        out1 = out3 ^ out6 ^ in7;
+        out4 = tmp1 ^ out1;
+        out7 = out2 ^ out4 ^ in0;
+        out0 = out7 ^ in1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_82(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in2;
+        tmp0 = in6 ^ in7;
+        out5 = in2 ^ in3;
+        out6 = in3 ^ in4;
+        out7 = in0 ^ in4 ^ in5;
+        out0 = in1 ^ in5 ^ in6;
+        out1 = tmp0 ^ in0 ^ in2;
+        out2 = tmp0 ^ in3 ^ in5;
+        out3 = tmp0 ^ out0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_83(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in5;
+        tmp2 = in3 ^ in6;
+        out4 = in1 ^ in2 ^ in4;
+        out0 = tmp0 ^ in5 ^ in6;
+        out5 = tmp1 ^ in3;
+        tmp3 = tmp1 ^ in7;
+        out6 = tmp2 ^ in4;
+        out2 = tmp2 ^ tmp3;
+        tmp4 = tmp3 ^ out4;
+        out1 = tmp3 ^ out0;
+        out3 = tmp4 ^ in3;
+        out7 = tmp0 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_84(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2 ^ in6;
+        out6 = in3 ^ in5;
+        out0 = in1 ^ in5 ^ in7;
+        out7 = in0 ^ in4 ^ in6;
+        out4 = in1 ^ in3 ^ in6;
+        out5 = in2 ^ in4 ^ in7;
+        out2 = out6 ^ in0 ^ in1;
+        out3 = out5 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_85(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in6;
+        tmp1 = in3 ^ in6;
+        tmp2 = tmp0 ^ in4;
+        out1 = tmp0 ^ in2;
+        out6 = tmp1 ^ in5;
+        out4 = tmp2 ^ in3;
+        tmp3 = out1 ^ out6;
+        out2 = tmp3 ^ in0;
+        out3 = tmp2 ^ tmp3 ^ in7;
+        out7 = out2 ^ out3 ^ in1;
+        out5 = tmp1 ^ out3;
+        out0 = tmp2 ^ out7 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_86(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in3;
+        out7 = in0 ^ in4;
+        out0 = in1 ^ in5;
+        out5 = in2 ^ in7;
+        out3 = in4 ^ in5 ^ in6;
+        out1 = in0 ^ in2 ^ in6;
+        out4 = in1 ^ in6 ^ in7;
+        out2 = in0 ^ in3 ^ in5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_87(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in3 ^ in6;
+        tmp0 = in0 ^ in1;
+        out7 = in0 ^ in4 ^ in7;
+        out5 = in2 ^ in5 ^ in7;
+        out3 = out6 ^ in4 ^ in5;
+        out0 = tmp0 ^ in5;
+        tmp1 = tmp0 ^ in6;
+        out2 = out5 ^ in0 ^ in3;
+        out1 = tmp1 ^ in2;
+        out4 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_88(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2 ^ in7;
+        tmp0 = in5 ^ in6;
+        out0 = in1 ^ in6 ^ in7;
+        out6 = in4 ^ in5 ^ in7;
+        out3 = out0 ^ out1 ^ in0 ^ in4;
+        out7 = tmp0 ^ in0;
+        tmp1 = tmp0 ^ in3;
+        out2 = out0 ^ in3;
+        out4 = tmp1 ^ in2;
+        out5 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_89(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in7;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in6;
+        out1 = tmp1 ^ in1;
+        out7 = tmp2 ^ in5;
+        out0 = tmp2 ^ in1;
+        out2 = out1 ^ in3 ^ in6;
+        out6 = out7 ^ in0 ^ in4;
+        out5 = out6 ^ in3;
+        out3 = tmp0 ^ out2 ^ in4;
+        out4 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in6;
+        out7 = in0 ^ in5;
+        out2 = in3 ^ in6;
+        out6 = in4 ^ in7;
+        out1 = in0 ^ in2 ^ in7;
+        out3 = out0 ^ out6 ^ in0;
+        out4 = out1 ^ out7 ^ in6;
+        out5 = out2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in3 ^ in6;
+        tmp2 = in5 ^ in7;
+        tmp3 = tmp0 ^ in7;
+        out0 = tmp0 ^ in6;
+        out2 = tmp1 ^ in2;
+        out5 = tmp1 ^ tmp2;
+        out7 = tmp2 ^ in0;
+        tmp4 = tmp3 ^ in4;
+        out1 = tmp3 ^ in2;
+        out6 = tmp4 ^ out0;
+        out4 = out6 ^ in2 ^ in5;
+        out3 = tmp1 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2;
+        out0 = in1 ^ in7;
+        out7 = in0 ^ in6;
+        out5 = in4 ^ in6;
+        out6 = in5 ^ in7;
+        out2 = out0 ^ in0 ^ in3;
+        out3 = out5 ^ out7 ^ in2 ^ in7;
+        out4 = out6 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in1 ^ in2;
+        tmp0 = in6 ^ in7;
+        out0 = in0 ^ in1 ^ in7;
+        out5 = in4 ^ in5 ^ in6;
+        out6 = tmp0 ^ in5;
+        out7 = tmp0 ^ in0;
+        out4 = tmp0 ^ out5 ^ in3;
+        out2 = out0 ^ in2 ^ in3;
+        out3 = out2 ^ in1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1;
+        out4 = in5;
+        out7 = in0;
+        out5 = in6;
+        out6 = in7;
+        out3 = in0 ^ in4;
+        out1 = in0 ^ in2;
+        out2 = in0 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in1;
+        tmp0 = in0 ^ in3;
+        out4 = in4 ^ in5;
+        out7 = in0 ^ in7;
+        out5 = in5 ^ in6;
+        out6 = in6 ^ in7;
+        out1 = out0 ^ in2;
+        out2 = tmp0 ^ in2;
+        out3 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_90(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in6 ^ in7;
+        out3 = tmp0 ^ in7;
+        out1 = tmp1 ^ in5;
+        tmp2 = out1 ^ in4;
+        out6 = tmp2 ^ in3;
+        out5 = out6 ^ in1;
+        out4 = out5 ^ in0;
+        out0 = tmp0 ^ tmp2;
+        out7 = tmp0 ^ out4;
+        out2 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_91(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = tmp0 ^ in3 ^ in5;
+        out2 = tmp1 ^ in1;
+        out6 = tmp1 ^ in7;
+        tmp2 = out2 ^ in5 ^ in7;
+        out3 = tmp2 ^ in4;
+        out5 = tmp2 ^ in6;
+        out1 = tmp1 ^ out5 ^ in2;
+        tmp3 = out1 ^ in0;
+        out4 = tmp3 ^ in3;
+        out0 = tmp0 ^ tmp3;
+        out7 = tmp2 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_92(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1;
+        tmp0 = in4 ^ in5;
+        tmp1 = tmp0 ^ in1;
+        out2 = tmp0 ^ in3 ^ in7;
+        out0 = tmp1 ^ in6;
+        out7 = out2 ^ in0;
+        out4 = out0 ^ in0 ^ in2;
+        out5 = out4 ^ out7 ^ in5;
+        out6 = tmp1 ^ out5;
+        out1 = out6 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_93(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in3;
+        tmp0 = in2 ^ in7;
+        tmp1 = out3 ^ in6;
+        tmp2 = tmp0 ^ in4;
+        out5 = tmp0 ^ tmp1;
+        out6 = tmp2 ^ in3;
+        out2 = out6 ^ in5;
+        out0 = out2 ^ out5 ^ in0;
+        out7 = tmp1 ^ out0;
+        out1 = tmp2 ^ out0;
+        out4 = out1 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_94(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in6;
+        tmp0 = in1 ^ in4 ^ in5;
+        out1 = out3 ^ in5;
+        out5 = tmp0 ^ out3;
+        out0 = tmp0 ^ in7;
+        out4 = tmp0 ^ in0 ^ in3;
+        out6 = out1 ^ in3 ^ in7;
+        out2 = out4 ^ in6;
+        out7 = out0 ^ out2 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_95(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out3 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in7;
+        tmp2 = out3 ^ in0;
+        out6 = tmp1 ^ in5;
+        tmp3 = tmp2 ^ in4;
+        out7 = tmp3 ^ in2;
+        tmp4 = tmp3 ^ in5;
+        out2 = tmp4 ^ in1;
+        tmp5 = out2 ^ in6;
+        out0 = tmp1 ^ tmp5;
+        out1 = tmp5 ^ out7;
+        out4 = tmp2 ^ out1;
+        out5 = tmp4 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_96(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in6 ^ in7;
+        tmp0 = in1 ^ in5;
+        tmp1 = in5 ^ in6;
+        out6 = out3 ^ in2 ^ in3;
+        out0 = tmp0 ^ in4;
+        tmp2 = tmp1 ^ in2;
+        out4 = out0 ^ in0 ^ in7;
+        out1 = tmp2 ^ in0;
+        out5 = tmp2 ^ in1;
+        out7 = tmp0 ^ out4 ^ in3;
+        out2 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_97(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        tmp1 = in2 ^ in6;
+        out3 = in3 ^ in6 ^ in7;
+        out7 = tmp0 ^ in3;
+        tmp2 = tmp0 ^ in5;
+        out5 = tmp1 ^ in1;
+        out6 = tmp1 ^ out3;
+        out0 = tmp2 ^ in1;
+        out2 = tmp2 ^ out3 ^ in2;
+        tmp3 = out0 ^ in4;
+        out4 = tmp3 ^ in7;
+        out1 = tmp1 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_98(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in7;
+        tmp1 = in1 ^ in4 ^ in7;
+        out1 = tmp0 ^ in2;
+        out0 = tmp1 ^ in6;
+        out2 = tmp1 ^ in3;
+        out6 = out0 ^ out1 ^ in1;
+        out5 = tmp0 ^ out2;
+        out3 = tmp1 ^ out6 ^ in0;
+        out7 = out0 ^ out5 ^ in0;
+        out4 = out6 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_99(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        out5 = in1 ^ in3 ^ in4;
+        out6 = in2 ^ in4 ^ in5;
+        out4 = tmp0 ^ in2;
+        tmp1 = tmp0 ^ in6;
+        tmp2 = out5 ^ in7;
+        out7 = tmp1 ^ in5;
+        out0 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ in2;
+        out3 = out0 ^ out6 ^ in3;
+        out1 = tmp1 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in6;
+        out5 = in1 ^ in3 ^ in5;
+        tmp2 = tmp0 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ in4;
+        out7 = tmp2 ^ in3;
+        out1 = tmp2 ^ in2;
+        out6 = out0 ^ in1 ^ in2;
+        out4 = out1 ^ in4 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in1 ^ in3;
+        tmp0 = in3 ^ in5;
+        out6 = in2 ^ in4;
+        out4 = in0 ^ in2 ^ in7;
+        out7 = tmp0 ^ in0;
+        out2 = out6 ^ in3;
+        out1 = out4 ^ in1 ^ in5;
+        out3 = out7 ^ in1 ^ in6;
+        out0 = tmp0 ^ out3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2 ^ in5;
+        tmp0 = in0 ^ in3 ^ in6;
+        out3 = out1 ^ in0;
+        out6 = out1 ^ in6;
+        out7 = tmp0 ^ in7;
+        out4 = out7 ^ in4;
+        out2 = out4 ^ in1;
+        out0 = tmp0 ^ out2;
+        out5 = out0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in2 ^ in5;
+        tmp0 = in0 ^ in3;
+        out5 = in1 ^ in4 ^ in7;
+        out1 = out6 ^ in1;
+        out3 = tmp0 ^ out6;
+        out7 = tmp0 ^ in6;
+        out0 = out5 ^ in0;
+        out4 = out7 ^ in7;
+        out2 = out5 ^ out7 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in4;
+        tmp0 = in0 ^ in5;
+        out6 = in2 ^ in6;
+        out7 = in0 ^ in3 ^ in7;
+        out4 = in0 ^ in4 ^ in6;
+        out5 = in1 ^ in5 ^ in7;
+        out1 = tmp0 ^ in2;
+        out3 = tmp0 ^ in7;
+        out2 = out4 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in2;
+        out7 = in0 ^ in3;
+        tmp0 = in0 ^ in1;
+        out4 = in0 ^ in6;
+        out5 = in1 ^ in7;
+        out1 = tmp0 ^ in2 ^ in5;
+        out2 = out7 ^ in2 ^ in4 ^ in6;
+        out3 = out7 ^ in5 ^ in7;
+        out0 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in6;
+        out2 = tmp0 ^ in7;
+        tmp1 = tmp0 ^ in5;
+        out6 = out2 ^ in3 ^ in4;
+        out0 = tmp1 ^ in3;
+        tmp2 = out0 ^ in2;
+        out3 = tmp2 ^ in7;
+        tmp3 = tmp2 ^ in1;
+        out5 = tmp3 ^ in0;
+        out4 = tmp3 ^ out6;
+        out7 = out5 ^ out6 ^ in1;
+        out1 = tmp1 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp0 ^ in4;
+        out4 = tmp1 ^ in7;
+        out7 = tmp2 ^ in0;
+        out6 = tmp2 ^ out4 ^ in3;
+        out3 = out4 ^ in6;
+        out2 = out3 ^ in5;
+        out1 = out2 ^ in4;
+        out5 = out1 ^ out6 ^ in0;
+        out0 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in6;
+        tmp0 = in1 ^ in3 ^ in5;
+        out3 = tmp0 ^ in6;
+        out4 = tmp0 ^ in2 ^ in4;
+        out0 = out3 ^ in7;
+        out6 = out0 ^ in4;
+        out1 = out0 ^ out4 ^ in0;
+        out7 = out1 ^ in5;
+        out5 = out7 ^ in3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in6;
+        out3 = in1 ^ in5 ^ in6;
+        tmp0 = out2 ^ in0;
+        out4 = out2 ^ out3 ^ in3;
+        tmp1 = tmp0 ^ in4;
+        out0 = tmp0 ^ out4 ^ in7;
+        out5 = tmp1 ^ in3;
+        out7 = tmp1 ^ in5;
+        out1 = tmp1 ^ in1 ^ in7;
+        out6 = tmp1 ^ out0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in2 ^ in4;
+        tmp2 = in2 ^ in5;
+        tmp3 = in0 ^ in7;
+        out0 = tmp0 ^ in5;
+        out6 = tmp0 ^ in6 ^ in7;
+        out1 = tmp1 ^ in6;
+        out7 = tmp1 ^ tmp3;
+        out3 = tmp2 ^ in3;
+        tmp4 = tmp2 ^ out1;
+        out2 = tmp3 ^ in1;
+        out5 = tmp4 ^ out7;
+        out4 = tmp4 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in5;
+        tmp0 = in1 ^ in6;
+        tmp1 = in0 ^ in1;
+        tmp2 = in2 ^ in4;
+        out6 = in1 ^ in3 ^ in7;
+        out4 = tmp0 ^ in5;
+        out1 = tmp0 ^ tmp2;
+        out0 = tmp1 ^ in3 ^ in5;
+        out2 = tmp1 ^ in2 ^ in7;
+        out7 = tmp2 ^ in0;
+        out5 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0;
+        out3 = in3 ^ in5 ^ in7;
+        out1 = in0 ^ in2 ^ in4 ^ in6;
+        out0 = out3 ^ in1;
+        out7 = out1 ^ in7;
+        out6 = out0 ^ in6;
+        out5 = out7 ^ in5;
+        out4 = out6 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in2;
+        out3 = in5 ^ in7;
+        out7 = out2 ^ in4 ^ in6;
+        out6 = out3 ^ in1 ^ in3;
+        out1 = out7 ^ in1;
+        out5 = out7 ^ in7;
+        out0 = out6 ^ in0;
+        out4 = out6 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = in1 ^ in6;
+        tmp2 = in0 ^ in2 ^ in7;
+        out1 = tmp0 ^ in7;
+        out4 = tmp0 ^ in6;
+        out0 = tmp1 ^ in3;
+        out2 = tmp1 ^ in5;
+        out6 = tmp1 ^ in4;
+        out7 = tmp2 ^ in5;
+        out3 = tmp2 ^ out0 ^ in6;
+        out5 = out7 ^ in2 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in6;
+        out6 = in1 ^ in4;
+        out7 = in0 ^ in2 ^ in5;
+        out5 = in0 ^ in3 ^ in7;
+        out2 = out4 ^ in1 ^ in5;
+        out1 = out6 ^ in2 ^ in7;
+        out0 = out2 ^ out7 ^ in3;
+        out3 = out1 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in1 ^ in3;
+        tmp2 = in6 ^ in7;
+        out1 = tmp0 ^ in4 ^ in7;
+        out3 = tmp1 ^ in0;
+        out0 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ in5;
+        out7 = tmp0 ^ out2;
+        out6 = out1 ^ out7 ^ in1;
+        out5 = out0 ^ out6 ^ in0;
+        out4 = out5 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in1;
+        tmp0 = in1 ^ in4;
+        tmp1 = in0 ^ in7;
+        out6 = tmp0 ^ in5;
+        out1 = tmp0 ^ tmp1 ^ in2;
+        out5 = tmp1 ^ in3 ^ in4;
+        out0 = tmp0 ^ out5 ^ in6;
+        out4 = out0 ^ out3 ^ in2;
+        out2 = out4 ^ in3 ^ in5;
+        out7 = tmp1 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in3;
+        out1 = in2 ^ in4;
+        tmp0 = in0 ^ in2;
+        out4 = in4 ^ in7;
+        out5 = in0 ^ in5;
+        out6 = in1 ^ in6;
+        out7 = tmp0 ^ in7;
+        out3 = tmp0 ^ in3 ^ in6;
+        out2 = out5 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in7;
+        out5 = in0;
+        out6 = in1;
+        out7 = in0 ^ in2;
+        out0 = in0 ^ in1 ^ in3;
+        out2 = out7 ^ in1 ^ in5;
+        out1 = in1 ^ in2 ^ in4;
+        out3 = out7 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in3 ^ in4;
+        tmp0 = in0 ^ in4;
+        tmp1 = in0 ^ in7;
+        out0 = in1 ^ in3 ^ in7;
+        out1 = tmp0 ^ in2;
+        out5 = tmp0 ^ in5;
+        tmp2 = tmp1 ^ in6;
+        out2 = tmp1 ^ in5;
+        out3 = tmp2 ^ in3;
+        out7 = tmp2 ^ in2;
+        out6 = tmp2 ^ out2 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in3;
+        tmp0 = in0 ^ in7;
+        out5 = in0 ^ in4;
+        out6 = in1 ^ in5;
+        out7 = in0 ^ in2 ^ in6;
+        out0 = tmp0 ^ in1 ^ in3;
+        out3 = tmp0 ^ in6;
+        out2 = tmp0 ^ in2 ^ in5;
+        out1 = out5 ^ in1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in4;
+        tmp1 = in3 ^ in6;
+        out2 = tmp0 ^ in7;
+        tmp2 = tmp0 ^ tmp1;
+        out0 = tmp2 ^ in5;
+        out3 = tmp2 ^ in2;
+        out6 = out3 ^ in6;
+        tmp3 = out6 ^ in0 ^ in1;
+        out7 = tmp3 ^ in5;
+        out5 = tmp3 ^ out2;
+        out1 = out0 ^ out5 ^ in0;
+        out4 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in4;
+        out2 = tmp0 ^ in2 ^ in7;
+        tmp1 = out2 ^ in6;
+        out1 = tmp1 ^ in5;
+        out3 = tmp1 ^ in7;
+        out4 = tmp1 ^ in0;
+        out6 = out3 ^ in3;
+        out0 = out6 ^ in0 ^ in2 ^ in5;
+        out5 = tmp1 ^ out0 ^ in1;
+        out7 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in4;
+        tmp0 = in4 ^ in7;
+        tmp1 = in1 ^ in3 ^ in6;
+        out3 = tmp0 ^ tmp1;
+        tmp2 = tmp1 ^ in0;
+        out0 = out3 ^ in5;
+        out4 = tmp2 ^ in2;
+        tmp3 = out4 ^ in6;
+        out5 = tmp0 ^ tmp3;
+        out1 = tmp3 ^ out0;
+        tmp4 = out1 ^ in7;
+        out7 = tmp4 ^ in3;
+        out6 = tmp2 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in6;
+        out3 = tmp1 ^ in4 ^ in7;
+        tmp2 = tmp0 ^ out3;
+        out0 = tmp2 ^ in3;
+        out1 = tmp2 ^ in2;
+        out5 = out0 ^ in2 ^ in6;
+        out7 = tmp1 ^ out5;
+        out4 = out7 ^ in1 ^ in5 ^ in7;
+        out6 = tmp0 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in1;
+        out5 = out4 ^ in2;
+        tmp0 = out4 ^ in4;
+        out6 = out5 ^ in0 ^ in3;
+        out7 = tmp0 ^ out6;
+        out2 = tmp0 ^ in6 ^ in7;
+        out3 = out7 ^ in0 ^ in7;
+        out0 = out5 ^ out7 ^ in5;
+        out1 = out0 ^ out6 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in4;
+        out4 = tmp0 ^ in4;
+        out3 = tmp1 ^ in7;
+        tmp2 = out4 ^ in5;
+        out7 = out3 ^ in0 ^ in3;
+        out0 = tmp2 ^ in3;
+        out2 = tmp0 ^ out3 ^ in6;
+        out5 = tmp1 ^ tmp2;
+        out6 = out2 ^ out7 ^ in2;
+        out1 = tmp0 ^ out0 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in4;
+        tmp0 = in1 ^ in2;
+        tmp1 = in0 ^ in4;
+        tmp2 = in3 ^ in5;
+        tmp3 = out3 ^ in1 ^ in7;
+        out5 = tmp0 ^ tmp1;
+        out6 = tmp0 ^ tmp2;
+        out2 = tmp1 ^ in6;
+        out4 = tmp1 ^ tmp3;
+        out0 = tmp3 ^ in5;
+        out1 = out2 ^ in2 ^ in5;
+        out7 = tmp2 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in4;
+        tmp0 = in0 ^ in4;
+        out2 = tmp0 ^ in2 ^ in6;
+        tmp1 = out2 ^ in7;
+        out1 = out2 ^ in1 ^ in5;
+        out7 = tmp1 ^ in3;
+        out5 = out1 ^ in6;
+        out6 = tmp0 ^ out1 ^ in3;
+        out0 = tmp1 ^ out6;
+        out4 = out0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in4;
+        tmp1 = in2 ^ in5;
+        out2 = tmp0 ^ in5;
+        out4 = tmp1 ^ in0;
+        tmp2 = tmp1 ^ in7;
+        out6 = tmp2 ^ out2;
+        out7 = out4 ^ in3;
+        out1 = tmp2 ^ in4;
+        out3 = tmp0 ^ out7;
+        out0 = out3 ^ out4 ^ in6;
+        out5 = out0 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in4 ^ in5;
+        out4 = tmp0 ^ tmp1;
+        tmp2 = tmp0 ^ in3 ^ in7;
+        out3 = out4 ^ in1;
+        out7 = tmp2 ^ in5;
+        out2 = out3 ^ in0;
+        out1 = out2 ^ in7;
+        out6 = out1 ^ in5 ^ in6;
+        out0 = tmp2 ^ out6;
+        out5 = tmp1 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in7;
+        out2 = tmp0 ^ in4;
+        tmp1 = out2 ^ in2;
+        out1 = tmp1 ^ in0;
+        out6 = tmp1 ^ in1;
+        out4 = out1 ^ in3 ^ in4;
+        tmp2 = out4 ^ out6;
+        out7 = out4 ^ in6 ^ in7;
+        out5 = tmp2 ^ in6;
+        out3 = tmp0 ^ tmp2;
+        out0 = out6 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in4 ^ in5 ^ in7;
+        tmp0 = out2 ^ in1;
+        out4 = out2 ^ in0 ^ in3;
+        out1 = tmp0 ^ in0;
+        out6 = tmp0 ^ in6;
+        out3 = out1 ^ in2;
+        tmp1 = out4 ^ out6 ^ in4;
+        out0 = tmp1 ^ in7;
+        out5 = tmp1 ^ in5;
+        out7 = tmp0 ^ tmp1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in2 ^ in4;
+        out0 = in1 ^ in3 ^ in4;
+        out6 = in1 ^ in2 ^ in7;
+        out7 = tmp0 ^ in3;
+        out5 = tmp0 ^ out6 ^ in6;
+        out1 = tmp1 ^ in5;
+        tmp2 = out1 ^ out5 ^ in1;
+        out3 = tmp2 ^ in3;
+        out4 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in1 ^ in4;
+        out0 = tmp0 ^ tmp1;
+        out7 = tmp0 ^ in2 ^ in7;
+        out1 = tmp1 ^ in2 ^ in5;
+        tmp2 = out1 ^ in0;
+        out2 = tmp2 ^ in6;
+        out3 = out2 ^ in1 ^ in7;
+        out4 = out3 ^ in2;
+        out5 = tmp1 ^ out4;
+        out6 = tmp2 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3 ^ in6;
+        out4 = tmp0 ^ in5;
+        out7 = tmp0 ^ in2;
+        out3 = out4 ^ in4;
+        out1 = out3 ^ out7 ^ in0;
+        out2 = out3 ^ in3 ^ in7;
+        out0 = out2 ^ out4 ^ in1;
+        out5 = tmp0 ^ out0;
+        out6 = out1 ^ out5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        out3 = tmp0 ^ in5 ^ in6;
+        out4 = out3 ^ in3;
+        tmp1 = out3 ^ in7;
+        out2 = tmp1 ^ in2;
+        out5 = tmp1 ^ in1;
+        tmp2 = out2 ^ in5;
+        out7 = tmp2 ^ in3 ^ in4;
+        tmp3 = tmp0 ^ out5;
+        out0 = tmp3 ^ out4;
+        out1 = tmp2 ^ tmp3;
+        out6 = tmp3 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in2 ^ in5;
+        tmp0 = in1 ^ in4;
+        tmp1 = in3 ^ in6;
+        out0 = out5 ^ in1;
+        out4 = tmp0 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out1 = tmp1 ^ in2;
+        out6 = tmp1 ^ in0;
+        out7 = out4 ^ in0;
+        out2 = out4 ^ out5 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in2;
+        tmp0 = in0 ^ in1;
+        out4 = in1 ^ in7;
+        out6 = in0 ^ in3;
+        out3 = in1 ^ in4 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out7 = tmp0 ^ in4;
+        out0 = tmp1 ^ in5;
+        out1 = tmp1 ^ out6 ^ in6;
+        out2 = out6 ^ out7 ^ in5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in3 ^ in4;
+        tmp0 = in0 ^ in3 ^ in6;
+        out5 = in2 ^ in4 ^ in5;
+        tmp1 = out4 ^ in7;
+        out1 = tmp0 ^ in2;
+        out6 = tmp0 ^ in5;
+        out2 = out5 ^ in3;
+        out7 = tmp0 ^ tmp1;
+        out3 = tmp1 ^ in2 ^ in6;
+        out0 = tmp1 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in3;
+        tmp0 = in0 ^ in2;
+        tmp1 = in3 ^ in5;
+        out5 = in2 ^ in4;
+        tmp2 = tmp0 ^ out4;
+        out2 = tmp1 ^ in4;
+        out6 = tmp1 ^ in0;
+        out0 = tmp1 ^ tmp2 ^ in7;
+        out1 = tmp2 ^ in6;
+        out7 = out1 ^ out5 ^ in3;
+        out3 = tmp0 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in7;
+        out3 = tmp0 ^ in4;
+        tmp1 = tmp0 ^ in2;
+        out1 = tmp1 ^ in6;
+        out5 = tmp1 ^ in5;
+        out4 = out1 ^ out3 ^ in1;
+        out0 = out4 ^ in4 ^ in5;
+        out2 = out0 ^ out3 ^ in0;
+        out7 = out1 ^ out2 ^ in7;
+        out6 = tmp1 ^ out0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in4 ^ in7;
+        tmp0 = in3 ^ in7;
+        out4 = in1 ^ in2 ^ in6;
+        out6 = in0 ^ in3 ^ in4;
+        out5 = tmp0 ^ in2;
+        out1 = tmp0 ^ out4;
+        out0 = out4 ^ in0 ^ in5;
+        out2 = out0 ^ out5 ^ in4;
+        out7 = tmp0 ^ out2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in6;
+        tmp1 = in1 ^ in7;
+        tmp2 = tmp0 ^ in0;
+        tmp3 = tmp0 ^ tmp1;
+        tmp4 = tmp2 ^ in4;
+        out0 = tmp3 ^ in2;
+        out6 = tmp4 ^ in3;
+        out2 = out6 ^ in2;
+        out7 = tmp1 ^ tmp4;
+        out3 = tmp2 ^ out2;
+        tmp5 = out3 ^ in5;
+        out5 = tmp5 ^ in7;
+        out4 = tmp3 ^ tmp5;
+        out1 = tmp4 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in4;
+        tmp0 = in3 ^ in5;
+        tmp1 = out3 ^ in7;
+        out6 = tmp0 ^ in0 ^ in4;
+        out5 = tmp1 ^ in3;
+        out2 = out6 ^ in6;
+        out7 = out2 ^ in1 ^ in3;
+        out0 = tmp1 ^ out7;
+        out1 = tmp0 ^ out0;
+        out4 = out1 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in2;
+        out1 = in2 ^ in3;
+        tmp0 = in5 ^ in6;
+        tmp1 = in0 ^ in7;
+        out2 = out1 ^ in1 ^ in4;
+        out4 = tmp0 ^ in4;
+        out5 = tmp0 ^ in7;
+        out6 = tmp1 ^ in6;
+        out7 = tmp1 ^ in1;
+        out3 = out2 ^ in0 ^ in2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in5 ^ in6;
+        out7 = in0 ^ in1;
+        tmp0 = in1 ^ in3;
+        out5 = in6 ^ in7;
+        out6 = in0 ^ in7;
+        out0 = out7 ^ in2;
+        out3 = out7 ^ in4 ^ in5;
+        out1 = tmp0 ^ in2;
+        out2 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in7;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in6;
+        out0 = tmp1 ^ in1;
+        tmp3 = tmp1 ^ in3;
+        out6 = tmp2 ^ in5;
+        out7 = tmp2 ^ in1;
+        out2 = tmp3 ^ in4;
+        out5 = out6 ^ in0 ^ in4;
+        out4 = out5 ^ in3;
+        out1 = tmp0 ^ tmp3;
+        out3 = tmp3 ^ out5 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in7;
+        tmp1 = in5 ^ in7;
+        out7 = in0 ^ in1 ^ in6;
+        out5 = tmp0 ^ in6;
+        out2 = tmp0 ^ in3;
+        out6 = tmp1 ^ in0;
+        out4 = tmp1 ^ in3 ^ in6;
+        tmp2 = out5 ^ out7 ^ in2;
+        out1 = tmp2 ^ out2;
+        out0 = tmp2 ^ in4;
+        out3 = tmp2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = in1 ^ in6;
+        out1 = in2 ^ in3 ^ in7;
+        out5 = tmp0 ^ in6;
+        out0 = tmp1 ^ in2;
+        tmp2 = out5 ^ in0 ^ in7;
+        out3 = tmp2 ^ in4;
+        out6 = tmp0 ^ out3;
+        out7 = tmp1 ^ tmp2 ^ in3;
+        tmp3 = out1 ^ out6;
+        out4 = tmp2 ^ tmp3;
+        out2 = tmp3 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in3 ^ in6;
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in7;
+        out6 = in0 ^ in4 ^ in7;
+        out2 = tmp0 ^ out5 ^ in4;
+        out7 = tmp0 ^ in5;
+        out0 = tmp0 ^ in2 ^ in6;
+        out4 = tmp1 ^ in5;
+        out1 = tmp1 ^ in1 ^ in3;
+        out3 = out6 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = tmp0 ^ in3;
+        out4 = tmp1 ^ in4;
+        tmp2 = out4 ^ in6;
+        out3 = tmp2 ^ in0;
+        out5 = tmp2 ^ in2;
+        out2 = out3 ^ in5 ^ in7;
+        out6 = tmp1 ^ out2;
+        out7 = out2 ^ out4 ^ in1;
+        out1 = tmp2 ^ out6;
+        out0 = tmp0 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in6;
+        tmp1 = in0 ^ in1 ^ in5;
+        out4 = in2 ^ in3 ^ in5;
+        out5 = tmp0 ^ in4;
+        out7 = tmp1 ^ in6;
+        out1 = tmp1 ^ out4 ^ in7;
+        tmp2 = out5 ^ in0;
+        out2 = tmp2 ^ in7;
+        out3 = tmp2 ^ out4;
+        out6 = tmp0 ^ out2 ^ in5;
+        out0 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in1 ^ in4;
+        tmp2 = in2 ^ in5;
+        out7 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ tmp2;
+        tmp3 = tmp2 ^ in3;
+        out1 = tmp3 ^ in6;
+        tmp4 = out1 ^ in1;
+        out2 = tmp4 ^ in7;
+        out3 = out2 ^ in2;
+        out4 = tmp0 ^ out3;
+        out5 = tmp3 ^ out3;
+        out6 = tmp4 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5 ^ in6;
+        tmp1 = tmp0 ^ in1;
+        out1 = tmp1 ^ in2;
+        out2 = tmp1 ^ in7;
+        out3 = out2 ^ in3;
+        out5 = out3 ^ in2;
+        tmp2 = out3 ^ in0;
+        out4 = tmp2 ^ in4;
+        out7 = tmp0 ^ out4;
+        out6 = tmp2 ^ out1 ^ in6;
+        out0 = out2 ^ out6 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in6;
+        out2 = tmp0 ^ in2 ^ in3;
+        out1 = out2 ^ in0;
+        out3 = out2 ^ in1;
+        out4 = out1 ^ in1 ^ in2;
+        out6 = out1 ^ in6 ^ in7;
+        out7 = out4 ^ in4 ^ in5;
+        out5 = out4 ^ out6 ^ in4;
+        out0 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in5 ^ in6;
+        tmp0 = out2 ^ in2;
+        tmp1 = tmp0 ^ in1;
+        out1 = tmp1 ^ in0;
+        out3 = tmp1 ^ in3;
+        out4 = out1 ^ in2 ^ in4;
+        tmp2 = out4 ^ in5;
+        out7 = tmp2 ^ in7;
+        out0 = tmp0 ^ out7;
+        tmp3 = out0 ^ in0;
+        out5 = tmp3 ^ in6;
+        out6 = tmp2 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in5;
+        tmp0 = in1 ^ in5;
+        tmp1 = tmp0 ^ in2;
+        out4 = tmp1 ^ in0;
+        tmp2 = tmp1 ^ in6;
+        out2 = out4 ^ in3 ^ in7;
+        out0 = tmp2 ^ in4;
+        out5 = tmp2 ^ out3;
+        out1 = tmp0 ^ out5 ^ in7;
+        out6 = tmp0 ^ out2 ^ in4;
+        out7 = tmp1 ^ out6 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in5;
+        tmp0 = in0 ^ in4;
+        tmp1 = tmp0 ^ in1 ^ in5;
+        out4 = tmp1 ^ in2;
+        out0 = out4 ^ in6;
+        tmp2 = tmp0 ^ out0;
+        out5 = tmp2 ^ in3;
+        out1 = out5 ^ in7;
+        out6 = tmp1 ^ out1;
+        out7 = tmp2 ^ out6;
+        out2 = out7 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2 ^ in4 ^ in6;
+        out5 = tmp0 ^ in3;
+        out0 = tmp0 ^ in5 ^ in7;
+        out3 = out0 ^ out5 ^ in2;
+        tmp1 = out3 ^ in0;
+        out1 = tmp1 ^ in6;
+        out2 = tmp1 ^ in7;
+        out4 = tmp1 ^ in1;
+        out6 = tmp1 ^ in4;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        out3 = in2 ^ in5 ^ in7;
+        out2 = tmp0 ^ in5;
+        tmp1 = tmp0 ^ out3 ^ in1;
+        out1 = tmp1 ^ in6;
+        out4 = tmp1 ^ in4;
+        tmp2 = out1 ^ in4;
+        out6 = tmp2 ^ in1;
+        out7 = tmp2 ^ in2;
+        out0 = tmp2 ^ in3;
+        out5 = tmp2 ^ in0 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0;
+        out5 = in1;
+        tmp0 = in1 ^ in2;
+        out6 = in0 ^ in2;
+        out0 = tmp0 ^ in4;
+        tmp1 = tmp0 ^ in3;
+        out7 = tmp1 ^ out6;
+        out2 = tmp1 ^ in6;
+        out3 = out7 ^ in7;
+        out1 = tmp1 ^ in1 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in4;
+        out5 = in1 ^ in5;
+        out2 = in1 ^ in3 ^ in6;
+        out3 = in0 ^ in1 ^ in7;
+        out6 = in0 ^ in2 ^ in6;
+        out0 = out4 ^ in1 ^ in2;
+        out1 = out5 ^ in2 ^ in3;
+        out7 = out3 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in1 ^ in4;
+        tmp0 = in2 ^ in7;
+        tmp1 = in0 ^ in2 ^ in3;
+        out0 = tmp0 ^ out5;
+        out4 = tmp0 ^ tmp1;
+        out2 = tmp0 ^ in3 ^ in6;
+        out1 = tmp1 ^ in5;
+        out3 = tmp1 ^ in1;
+        out6 = out1 ^ in3;
+        out7 = out3 ^ in2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in1 ^ in5;
+        tmp2 = in3 ^ in7;
+        out3 = tmp0 ^ in2;
+        out5 = tmp1 ^ in4;
+        out6 = tmp1 ^ out3 ^ in6;
+        out2 = tmp2 ^ in6;
+        tmp3 = tmp2 ^ in4;
+        tmp4 = out3 ^ in3;
+        out4 = tmp3 ^ in0;
+        out1 = tmp4 ^ in5;
+        out0 = tmp3 ^ tmp4;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in0 ^ in3;
+        out6 = tmp0 ^ in4;
+        tmp2 = tmp0 ^ in7;
+        out3 = tmp1 ^ in6;
+        tmp3 = tmp1 ^ in1;
+        out1 = tmp1 ^ tmp2 ^ in5;
+        out4 = tmp2 ^ in6;
+        out2 = tmp3 ^ in2;
+        out7 = tmp3 ^ in5;
+        out5 = tmp2 ^ out2;
+        out0 = out2 ^ out3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in6;
+        out2 = in0 ^ in1 ^ in3;
+        out6 = out3 ^ in2 ^ in4;
+        out7 = out2 ^ in5 ^ in7;
+        out0 = out6 ^ in1;
+        out4 = out6 ^ in7;
+        out5 = out7 ^ in0;
+        out1 = out5 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3 ^ in6;
+        tmp1 = in3 ^ in4 ^ in7;
+        out4 = tmp0 ^ in0;
+        out5 = tmp1 ^ in1;
+        out3 = out4 ^ in7;
+        out2 = out3 ^ in6;
+        out1 = out2 ^ in5;
+        out6 = tmp1 ^ out1;
+        out0 = tmp0 ^ out5;
+        out7 = out0 ^ out1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in3 ^ in7;
+        tmp0 = out2 ^ in1 ^ in5;
+        out1 = tmp0 ^ in2;
+        out7 = tmp0 ^ in6;
+        out5 = tmp0 ^ in0 ^ in4;
+        tmp1 = out1 ^ out5 ^ in6;
+        out4 = tmp1 ^ in3;
+        out6 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in7;
+        out0 = tmp2 ^ in1;
+        out3 = tmp2 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in7;
+        tmp0 = in2 ^ in4;
+        out4 = out3 ^ in3 ^ in5;
+        out2 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in6;
+        out0 = out4 ^ in2;
+        out6 = out4 ^ in0;
+        out1 = tmp1 ^ in3;
+        out5 = tmp1 ^ in0;
+        out7 = out5 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in4;
+        tmp0 = in1 ^ in7;
+        out3 = tmp0 ^ in3;
+        tmp1 = out3 ^ in5;
+        out4 = tmp1 ^ in4;
+        tmp2 = tmp1 ^ in0;
+        out0 = tmp2 ^ in2;
+        out6 = tmp2 ^ in6;
+        tmp3 = out0 ^ out4 ^ in6;
+        out5 = tmp3 ^ in5;
+        out7 = tmp0 ^ tmp3;
+        out1 = tmp2 ^ out5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in2;
+        out4 = in1 ^ in5;
+        out2 = in2 ^ in4 ^ in7;
+        out5 = in0 ^ in2 ^ in6;
+        out0 = out3 ^ in3 ^ in5;
+        out7 = out3 ^ in0 ^ in4;
+        out6 = out2 ^ out7 ^ in3;
+        out1 = out5 ^ in3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in4 ^ in7;
+        tmp0 = in1 ^ in3;
+        out3 = tmp0 ^ in2;
+        tmp1 = out3 ^ in0;
+        out0 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in4;
+        out1 = tmp2 ^ in6;
+        tmp3 = tmp2 ^ in3;
+        out7 = tmp3 ^ in7;
+        out6 = out1 ^ out2 ^ in2;
+        tmp4 = tmp0 ^ out0;
+        out5 = tmp4 ^ in6;
+        out4 = tmp3 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in6;
+        tmp0 = in0 ^ in4;
+        tmp1 = tmp0 ^ in2 ^ in6;
+        out2 = tmp1 ^ in1;
+        out7 = out2 ^ in5;
+        tmp2 = tmp0 ^ out7;
+        out4 = tmp2 ^ in3;
+        out0 = out4 ^ in7;
+        out6 = tmp1 ^ out0;
+        out5 = tmp2 ^ out6;
+        out1 = out5 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in6;
+        tmp0 = in0 ^ in1;
+        tmp1 = in5 ^ in7;
+        out2 = tmp0 ^ in4 ^ in6;
+        tmp2 = tmp1 ^ out2;
+        out6 = tmp2 ^ in3;
+        out7 = tmp2 ^ in2;
+        out0 = out6 ^ in2 ^ in4;
+        out5 = out6 ^ in1 ^ in2;
+        out1 = tmp0 ^ out5 ^ in5;
+        out4 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in6 ^ in7;
+        out2 = out3 ^ in0 ^ in4;
+        out4 = out3 ^ in1 ^ in5;
+        out1 = out2 ^ in3;
+        out7 = out2 ^ out4 ^ in2;
+        out0 = out4 ^ in3 ^ in7;
+        out5 = out1 ^ in4;
+        out6 = out0 ^ out2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out3 = tmp0 ^ in6 ^ in7;
+        tmp1 = out3 ^ in0;
+        out5 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in4;
+        tmp3 = out5 ^ in7;
+        out1 = tmp2 ^ in1;
+        out0 = tmp3 ^ in1;
+        out6 = out1 ^ in2;
+        out2 = tmp0 ^ tmp2;
+        tmp4 = tmp3 ^ out6;
+        out4 = tmp4 ^ in6;
+        out7 = tmp4 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in3 ^ in6;
+        tmp0 = in4 ^ in7;
+        out1 = in2 ^ in3 ^ in4;
+        out5 = tmp0 ^ in0;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp1 ^ in5;
+        out0 = tmp1 ^ out1;
+        out2 = tmp2 ^ in2;
+        out6 = tmp2 ^ out5;
+        tmp3 = out6 ^ in6;
+        out3 = tmp3 ^ in7;
+        out7 = tmp3 ^ in2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in3 ^ in6;
+        tmp2 = tmp0 ^ in6;
+        out4 = tmp1 ^ in4;
+        out6 = tmp2 ^ in5;
+        out7 = tmp2 ^ in2 ^ in7;
+        out3 = out6 ^ in3 ^ in7;
+        out0 = tmp1 ^ out7;
+        out2 = out3 ^ out4 ^ in0;
+        out5 = tmp0 ^ out2;
+        out1 = out0 ^ out5 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in6 ^ in7;
+        out5 = in0 ^ in7;
+        out6 = in0 ^ in1;
+        out0 = in1 ^ in2 ^ in3;
+        out2 = in2 ^ in4 ^ in5;
+        out7 = out6 ^ in2;
+        out1 = out0 ^ out6 ^ in4;
+        out3 = out7 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in4 ^ in5;
+        tmp0 = in0 ^ in1;
+        out4 = in4 ^ in6 ^ in7;
+        out5 = in0 ^ in5 ^ in7;
+        out6 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out0 = tmp1 ^ in3;
+        out7 = tmp1 ^ in7;
+        out1 = out0 ^ in4;
+        out3 = out0 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in5;
+        out4 = in2 ^ in3 ^ in7;
+        out5 = in0 ^ in3 ^ in4;
+        out6 = out3 ^ in1 ^ in4;
+        out1 = out4 ^ in4;
+        out0 = out4 ^ in1 ^ in6;
+        out2 = out0 ^ out5 ^ in5;
+        out7 = out2 ^ in4 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_ED(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = in3 ^ in5;
+        out4 = tmp0 ^ in3 ^ in7;
+        out3 = tmp1 ^ in0;
+        out1 = out4 ^ in1;
+        out5 = out3 ^ in4;
+        out7 = out1 ^ out5 ^ in6;
+        out2 = tmp0 ^ out7;
+        out0 = tmp1 ^ out7;
+        out6 = out2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2;
+        tmp0 = in0 ^ in1;
+        out5 = in0 ^ in3;
+        tmp1 = tmp0 ^ in2;
+        out6 = tmp0 ^ in4;
+        tmp2 = tmp1 ^ out5;
+        out7 = tmp1 ^ in5;
+        out1 = tmp2 ^ out6 ^ in7;
+        out0 = tmp2 ^ in6;
+        tmp3 = out7 ^ in1;
+        out3 = tmp3 ^ in7;
+        out2 = tmp3 ^ in4 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in4 ^ in6;
+        out5 = tmp0 ^ in3;
+        out2 = tmp0 ^ tmp1;
+        out6 = tmp1 ^ in0 ^ in1;
+        out3 = out5 ^ in2 ^ in7;
+        out7 = out3 ^ in1 ^ in3;
+        out0 = out4 ^ out6 ^ in3;
+        out1 = tmp1 ^ out0 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in4 ^ in5;
+        out2 = tmp0 ^ in6;
+        out3 = tmp1 ^ in1;
+        tmp2 = tmp1 ^ in7;
+        out1 = out2 ^ out3 ^ in3;
+        tmp3 = tmp0 ^ tmp2;
+        out0 = tmp3 ^ in3;
+        out5 = tmp3 ^ in0;
+        out4 = out1 ^ out5 ^ in4;
+        out7 = out4 ^ in2;
+        out6 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in6;
+        tmp0 = in3 ^ in5;
+        out3 = tmp0 ^ in1 ^ in4;
+        tmp1 = out3 ^ in2;
+        out1 = tmp1 ^ in6;
+        tmp2 = tmp1 ^ in0;
+        tmp3 = out1 ^ in5;
+        out0 = tmp2 ^ in7;
+        out6 = tmp2 ^ in4;
+        out7 = tmp3 ^ in0;
+        out5 = tmp0 ^ out0;
+        out4 = tmp3 ^ out5 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in5;
+        out2 = in2 ^ in6 ^ in7;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp1 ^ in2;
+        out0 = tmp2 ^ in3;
+        out3 = tmp2 ^ in7;
+        out5 = out3 ^ in0 ^ in4;
+        tmp3 = tmp0 ^ out5;
+        out7 = tmp3 ^ in3;
+        out4 = tmp3 ^ out2;
+        out1 = out0 ^ out4 ^ in4;
+        out6 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in6 ^ in7;
+        tmp0 = in0 ^ in1;
+        out4 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out5 = tmp1 ^ in7;
+        out6 = tmp1 ^ in3;
+        out7 = out6 ^ in4;
+        out0 = out7 ^ in5;
+        out1 = out0 ^ in6;
+        out3 = out0 ^ in0 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in1 ^ in2;
+        tmp0 = out2 ^ in3;
+        out4 = tmp0 ^ in4;
+        out5 = out4 ^ in5;
+        out6 = out5 ^ in6;
+        out7 = out6 ^ in7;
+        out0 = out7 ^ in0;
+        out1 = out0 ^ in1;
+        out3 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in1;
+        tmp0 = out2 ^ in2;
+        out4 = tmp0 ^ in3;
+        out5 = out4 ^ in4;
+        out6 = out5 ^ in5;
+        out7 = out6 ^ in6;
+        out0 = out7 ^ in7;
+        out1 = out0 ^ in0;
+        out3 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in7;
+        out2 = tmp0 ^ in2;
+        out4 = out2 ^ in1 ^ in4;
+        out7 = out4 ^ in3 ^ in5;
+        out5 = out7 ^ in4 ^ in7;
+        out0 = tmp0 ^ out7 ^ in6;
+        tmp1 = out0 ^ in1;
+        out6 = out0 ^ in0 ^ in5;
+        out3 = tmp1 ^ in3;
+        out1 = tmp0 ^ tmp1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in7;
+        tmp0 = out2 ^ in1;
+        out4 = tmp0 ^ in2;
+        out5 = out4 ^ in3 ^ in7;
+        out6 = out5 ^ in4;
+        out7 = out6 ^ in5;
+        out0 = out7 ^ in6;
+        out1 = out0 ^ in7;
+        out3 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        tmp1 = in3 ^ in5;
+        tmp2 = tmp0 ^ in6;
+        out4 = tmp0 ^ tmp1;
+        out1 = tmp1 ^ in2 ^ in4;
+        out3 = tmp2 ^ in1;
+        out5 = out3 ^ in5;
+        out7 = out1 ^ out5 ^ in7;
+        out6 = tmp1 ^ out7;
+        out0 = tmp2 ^ out7;
+        out2 = out6 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = in0 ^ in6;
+        out4 = tmp0 ^ in0;
+        tmp2 = tmp1 ^ in4;
+        tmp3 = tmp1 ^ in2;
+        out5 = tmp2 ^ in1;
+        out3 = out5 ^ in3;
+        tmp4 = tmp3 ^ out3;
+        out1 = tmp4 ^ in5;
+        out0 = tmp4 ^ in0 ^ in7;
+        out6 = tmp0 ^ out0 ^ in4;
+        out7 = tmp2 ^ tmp4;
+        out2 = tmp3 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = tmp0 ^ in2;
+        tmp2 = tmp0 ^ in5;
+        tmp3 = tmp1 ^ in7;
+        out5 = tmp2 ^ in6;
+        out6 = tmp3 ^ in6;
+        out7 = tmp3 ^ in3;
+        out3 = out6 ^ in4;
+        out2 = tmp1 ^ out5;
+        out4 = out2 ^ out3 ^ in1;
+        out0 = out4 ^ out7 ^ in5;
+        out1 = tmp2 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in5 ^ in6;
+        tmp0 = in0 ^ in1;
+        out4 = in0 ^ in5 ^ in7;
+        out5 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out6 = tmp1 ^ in7;
+        out7 = tmp1 ^ in3;
+        out0 = out7 ^ in4;
+        out1 = out0 ^ in5;
+        out3 = out0 ^ in6 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in0 ^ in7;
+        out2 = tmp0 ^ tmp1 ^ in5;
+        out3 = tmp1 ^ in4;
+        tmp2 = out2 ^ in6;
+        out6 = tmp2 ^ in4;
+        out7 = tmp2 ^ in3;
+        out4 = out6 ^ in1 ^ in3;
+        tmp3 = out4 ^ in0;
+        out1 = tmp3 ^ in6;
+        out0 = tmp3 ^ in1 ^ in5;
+        out5 = tmp0 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in7;
+        out2 = tmp0 ^ tmp1;
+        out6 = out2 ^ in2 ^ in4;
+        tmp2 = out6 ^ in0;
+        out1 = tmp2 ^ in3;
+        out0 = tmp0 ^ out1 ^ in6;
+        out5 = out0 ^ in2;
+        tmp3 = out5 ^ in1;
+        out3 = tmp3 ^ in6;
+        out7 = tmp2 ^ tmp3;
+        out4 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        out2 = tmp0 ^ in5;
+        out3 = tmp0 ^ in4;
+        tmp1 = out3 ^ in6;
+        out4 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in1;
+        out6 = tmp2 ^ in7;
+        tmp3 = tmp2 ^ in0;
+        out0 = tmp3 ^ in3;
+        tmp4 = out0 ^ out4 ^ in7;
+        out5 = tmp4 ^ in6;
+        out7 = tmp4 ^ in2;
+        out1 = tmp3 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in5;
+        tmp0 = in4 ^ in7;
+        tmp1 = out2 ^ in2;
+        out4 = tmp1 ^ in6;
+        out7 = tmp1 ^ in1 ^ in3;
+        out1 = tmp0 ^ out7;
+        tmp2 = out1 ^ in5;
+        out6 = tmp2 ^ in3;
+        tmp3 = tmp2 ^ in7;
+        out0 = tmp3 ^ in6;
+        out3 = tmp3 ^ in1;
+        out5 = tmp0 ^ out0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void (*gf8_muladd[])(void *out, void *in) = {
+    gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03, gf8_muladd_04,
+    gf8_muladd_05, gf8_muladd_06, gf8_muladd_07, gf8_muladd_08, gf8_muladd_09,
+    gf8_muladd_0A, gf8_muladd_0B, gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E,
+    gf8_muladd_0F, gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13,
+    gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17, gf8_muladd_18,
+    gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B, gf8_muladd_1C, gf8_muladd_1D,
+    gf8_muladd_1E, gf8_muladd_1F, gf8_muladd_20, gf8_muladd_21, gf8_muladd_22,
+    gf8_muladd_23, gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27,
+    gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B, gf8_muladd_2C,
+    gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F, gf8_muladd_30, gf8_muladd_31,
+    gf8_muladd_32, gf8_muladd_33, gf8_muladd_34, gf8_muladd_35, gf8_muladd_36,
+    gf8_muladd_37, gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B,
+    gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F, gf8_muladd_40,
+    gf8_muladd_41, gf8_muladd_42, gf8_muladd_43, gf8_muladd_44, gf8_muladd_45,
+    gf8_muladd_46, gf8_muladd_47, gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A,
+    gf8_muladd_4B, gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F,
+    gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53, gf8_muladd_54,
+    gf8_muladd_55, gf8_muladd_56, gf8_muladd_57, gf8_muladd_58, gf8_muladd_59,
+    gf8_muladd_5A, gf8_muladd_5B, gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E,
+    gf8_muladd_5F, gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63,
+    gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67, gf8_muladd_68,
+    gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B, gf8_muladd_6C, gf8_muladd_6D,
+    gf8_muladd_6E, gf8_muladd_6F, gf8_muladd_70, gf8_muladd_71, gf8_muladd_72,
+    gf8_muladd_73, gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77,
+    gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B, gf8_muladd_7C,
+    gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F, gf8_muladd_80, gf8_muladd_81,
+    gf8_muladd_82, gf8_muladd_83, gf8_muladd_84, gf8_muladd_85, gf8_muladd_86,
+    gf8_muladd_87, gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B,
+    gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F, gf8_muladd_90,
+    gf8_muladd_91, gf8_muladd_92, gf8_muladd_93, gf8_muladd_94, gf8_muladd_95,
+    gf8_muladd_96, gf8_muladd_97, gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A,
+    gf8_muladd_9B, gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F,
+    gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3, gf8_muladd_A4,
+    gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7, gf8_muladd_A8, gf8_muladd_A9,
+    gf8_muladd_AA, gf8_muladd_AB, gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE,
+    gf8_muladd_AF, gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3,
+    gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7, gf8_muladd_B8,
+    gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB, gf8_muladd_BC, gf8_muladd_BD,
+    gf8_muladd_BE, gf8_muladd_BF, gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2,
+    gf8_muladd_C3, gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7,
+    gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB, gf8_muladd_CC,
+    gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF, gf8_muladd_D0, gf8_muladd_D1,
+    gf8_muladd_D2, gf8_muladd_D3, gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6,
+    gf8_muladd_D7, gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB,
+    gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF, gf8_muladd_E0,
+    gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3, gf8_muladd_E4, gf8_muladd_E5,
+    gf8_muladd_E6, gf8_muladd_E7, gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA,
+    gf8_muladd_EB, gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF,
+    gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3, gf8_muladd_F4,
+    gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7, gf8_muladd_F8, gf8_muladd_F9,
+    gf8_muladd_FA, gf8_muladd_FB, gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE,
+    gf8_muladd_FF};
+
+static uint64_t zero[EC_METHOD_WORD_SIZE * 8] = {
+    0,
+};
+
+void
+ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count)
+{
+    uint32_t i, last, tmp;
+
+    last = 1;
+    for (i = count; i > 0; i--) {
+        if (values[i - 1] != 0) {
+            tmp = values[i - 1];
+            values[i - 1] = ec_gf_div(gf, tmp, last);
+            last = tmp;
+        }
+    }
+}
+
+void
+ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
+                 uint32_t count)
+{
+    src += offset;
+    gf8_muladd_00(dst, src);
+    while (--count > 0) {
+        src += EC_METHOD_CHUNK_SIZE;
+        gf8_muladd[*values](dst, src);
+        values++;
+    }
+}
+
+void
+ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values,
+                      uint32_t count)
+{
+    uint32_t i, last, tmp;
+
+    i = 0;
+    while ((last = *values++) == 0) {
+        i++;
+    }
+    gf8_muladd_00(dst, src[i++] + offset);
+    while (i < count) {
+        tmp = *values++;
+        if (tmp != 0) {
+            gf8_muladd[last](dst, src[i] + offset);
+            last = tmp;
+        }
+        i++;
+    }
+    gf8_muladd[last](dst, zero);
+}
diff --git a/xlators/cluster/ec/src/ec-code-c.h b/xlators/cluster/ec/src/ec-code-c.h
new file mode 100644
index 00000000000..42b5a064eb8
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-c.h
@@ -0,0 +1,27 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_CODE_C_H__
+#define __EC_CODE_C_H__
+
+#include "ec-types.h"
+
+void
+ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count);
+
+void
+ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
+                 uint32_t count);
+
+void
+ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values,
+                      uint32_t count);
+
+#endif /* __EC_CODE_C_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-intel.c b/xlators/cluster/ec/src/ec-code-intel.c
new file mode 100644
index 00000000000..f1c4e13e321
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-intel.c
@@ -0,0 +1,594 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <inttypes.h>
+#include <string.h>
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static void
+ec_code_intel_init(ec_code_intel_t *intel)
+{
+    memset(intel, 0, sizeof(ec_code_intel_t));
+}
+
+static void
+ec_code_intel_prefix(ec_code_intel_t *intel, uint8_t prefix)
+{
+    intel->prefix.data[intel->prefix.bytes++] = prefix;
+}
+
+static void
+ec_code_intel_rex(ec_code_intel_t *intel, gf_boolean_t w)
+{
+    gf_boolean_t present = _gf_false;
+
+    if (w) {
+        intel->rex.w = 1;
+        present = _gf_true;
+    }
+    if (intel->modrm.present) {
+        if (intel->modrm.reg > 7) {
+            intel->modrm.reg &= 7;
+            intel->rex.r = 1;
+            present = _gf_true;
+        }
+        if (intel->sib.present) {
+            if (intel->sib.index > 7) {
+                intel->sib.index &= 7;
+                intel->rex.x = 1;
+                present = _gf_true;
+            }
+            if (intel->sib.base > 7) {
+                intel->sib.base &= 7;
+                intel->rex.b = 1;
+                present = _gf_true;
+            }
+        } else if (intel->modrm.rm > 7) {
+            intel->modrm.rm &= 7;
+            intel->rex.b = 1;
+            present = _gf_true;
+        }
+    } else if (intel->reg > 7) {
+        intel->reg &= 7;
+        intel->rex.b = 1;
+        present = _gf_true;
+    }
+    intel->rex.present = present;
+}
+
+static void
+ec_code_intel_vex(ec_code_intel_t *intel, gf_boolean_t w, gf_boolean_t l,
+                  ec_code_vex_opcode_t opcode, ec_code_vex_prefix_t prefix,
+                  uint32_t reg)
+{
+    ec_code_intel_rex(intel, w);
+    if (((intel->rex.w == 1) || (intel->rex.x == 0) || (intel->rex.b == 0)) ||
+        ((opcode != VEX_OPCODE_NONE) && (opcode != VEX_OPCODE_0F))) {
+        intel->rex.present = _gf_false;
+
+        intel->vex.bytes = 3;
+        intel->vex.data[0] = 0xC4;
+        intel->vex.data[1] = ((intel->rex.r << 7) | (intel->rex.x << 6) |
+                              (intel->rex.b << 5) | opcode) ^
+                             0xE0;
+        intel->vex.data[2] = (intel->rex.w << 7) | ((~reg & 0x0F) << 3) |
+                             (l ? 0x04 : 0x00) | prefix;
+    } else {
+        intel->vex.bytes = 2;
+        intel->vex.data[0] = 0xC5;
+        intel->vex.data[1] = (intel->rex.r << 7) | ((~reg & 0x0F) << 3) |
+                             (l ? 0x04 : 0x00) | prefix;
+    }
+}
+
+static void
+ec_code_intel_modrm_reg(ec_code_intel_t *intel, uint32_t rm, uint32_t reg)
+{
+    intel->modrm.present = _gf_true;
+    intel->modrm.mod = 3;
+    intel->modrm.rm = rm;
+    intel->modrm.reg = reg;
+}
+
+static void
+ec_code_intel_modrm_mem(ec_code_intel_t *intel, uint32_t reg,
+                        ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                        uint32_t scale, int32_t offset)
+{
+    if (index == REG_SP) {
+        intel->invalid = _gf_true;
+        return;
+    }
+    if ((index != REG_NULL) && (scale != 1) && (scale != 2) && (scale != 4) &&
+        (scale != 8)) {
+        intel->invalid = _gf_true;
+        return;
+    }
+    scale >>= 1;
+    if (scale == 4) {
+        scale = 3;
+    }
+
+    intel->modrm.present = _gf_true;
+    intel->modrm.reg = reg;
+
+    intel->offset.value = offset;
+    if ((offset == 0) && (base != REG_BP)) {
+        intel->modrm.mod = 0;
+        intel->offset.bytes = 0;
+    } else if ((offset >= -128) && (offset <= 127)) {
+        intel->modrm.mod = 1;
+        intel->offset.bytes = 1;
+    } else {
+        intel->modrm.mod = 2;
+        intel->offset.bytes = 4;
+    }
+
+    intel->modrm.rm = base;
+    if ((index != REG_NULL) || (base == REG_SP)) {
+        intel->modrm.rm = 4;
+        intel->sib.present = _gf_true;
+        intel->sib.index = index;
+        if (index == REG_NULL) {
+            intel->sib.index = 4;
+        }
+        intel->sib.scale = scale;
+        intel->sib.base = base;
+        if (base == REG_NULL) {
+            intel->sib.base = 5;
+            intel->modrm.mod = 0;
+            intel->offset.bytes = 4;
+        }
+    } else if (base == REG_NULL) {
+        intel->modrm.mod = 0;
+        intel->modrm.rm = 5;
+        intel->offset.bytes = 4;
+    }
+}
+
+static void
+ec_code_intel_op_1(ec_code_intel_t *intel, uint8_t opcode, uint32_t reg)
+{
+    intel->reg = reg;
+    intel->opcode.bytes = 1;
+    intel->opcode.data[0] = opcode;
+}
+
+static void
+ec_code_intel_op_2(ec_code_intel_t *intel, uint8_t opcode1, uint8_t opcode2,
+                   uint32_t reg)
+{
+    intel->reg = reg;
+    intel->opcode.bytes = 2;
+    intel->opcode.data[0] = opcode1;
+    intel->opcode.data[1] = opcode2;
+}
+
+static void
+ec_code_intel_immediate_1(ec_code_intel_t *intel, uint32_t value)
+{
+    intel->immediate.bytes = 1;
+    intel->immediate.value = value;
+}
+
+static void
+ec_code_intel_immediate_2(ec_code_intel_t *intel, uint32_t value)
+{
+    intel->immediate.bytes = 2;
+    intel->immediate.value = value;
+}
+
+static void
+ec_code_intel_immediate_4(ec_code_intel_t *intel, uint32_t value)
+{
+    intel->immediate.bytes = 4;
+    intel->immediate.value = value;
+}
+
+static void
+ec_code_intel_emit(ec_code_builder_t *builder, ec_code_intel_t *intel)
+{
+    uint8_t insn[15];
+    uint32_t i, count;
+
+    if (intel->invalid) {
+        ec_code_error(builder, EINVAL);
+        return;
+    }
+
+    count = 0;
+    for (i = 0; i < intel->prefix.bytes; i++) {
+        insn[count++] = intel->prefix.data[i];
+    }
+    for (i = 0; i < intel->vex.bytes; i++) {
+        insn[count++] = intel->vex.data[i];
+    }
+    if (intel->rex.present) {
+        insn[count++] = 0x40 | (intel->rex.w << 3) | (intel->rex.r << 2) |
+                        (intel->rex.x << 1) | (intel->rex.b << 0);
+    }
+    for (i = 0; i < intel->opcode.bytes; i++) {
+        insn[count++] = intel->opcode.data[i];
+    }
+    if (intel->modrm.present) {
+        insn[count++] = (intel->modrm.mod << 6) | (intel->modrm.reg << 3) |
+                        (intel->modrm.rm << 0);
+        if (intel->sib.present) {
+            insn[count++] = (intel->sib.scale << 6) | (intel->sib.index << 3) |
+                            (intel->sib.base << 0);
+        }
+    }
+    for (i = 0; i < intel->offset.bytes; i++) {
+        insn[count++] = intel->offset.data[i];
+    }
+    for (i = 0; i < intel->immediate.bytes; i++) {
+        insn[count++] = intel->immediate.data[i];
+    }
+
+    ec_code_emit(builder, insn, count);
+}
+
+void
+ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_op_1(&intel, 0x50 | (reg & 7), reg);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_op_1(&intel, 0x58 | (reg & 7), reg);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    if (size == 0) {
+        ec_code_intel_op_1(&intel, 0xC3, 0);
+    } else {
+        ec_code_intel_immediate_2(&intel, size);
+        ec_code_intel_op_1(&intel, 0xC2, 0);
+    }
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, dst, src);
+    ec_code_intel_op_1(&intel, 0x89, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                         uint32_t scale, int32_t offset)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x89, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x8B, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, dst, src);
+    ec_code_intel_op_1(&intel, 0x31, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x33, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value,
+                         ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    if ((value >= -128) && (value < 128)) {
+        ec_code_intel_modrm_reg(&intel, reg, 0);
+        ec_code_intel_op_1(&intel, 0x83, 0);
+        ec_code_intel_immediate_1(&intel, value);
+    } else {
+        if (reg == REG_AX) {
+            ec_code_intel_op_1(&intel, 0x05, reg);
+        } else {
+            ec_code_intel_modrm_reg(&intel, reg, 0);
+            ec_code_intel_op_1(&intel, 0x81, 0);
+        }
+        ec_code_intel_immediate_4(&intel, value);
+    }
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value,
+                          ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    if (reg == REG_AX) {
+        ec_code_intel_op_1(&intel, 0xA9, reg);
+    } else {
+        ec_code_intel_modrm_reg(&intel, reg, 0);
+        ec_code_intel_op_1(&intel, 0xF7, 0);
+    }
+    ec_code_intel_immediate_4(&intel, value);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address)
+{
+    ec_code_intel_t intel;
+    int32_t rel;
+
+    ec_code_intel_init(&intel);
+
+    rel = address - builder->address - 2;
+    if ((rel >= -128) && (rel < 128)) {
+        ec_code_intel_op_1(&intel, 0x75, 0);
+        ec_code_intel_immediate_1(&intel, rel);
+    } else {
+        rel -= 4;
+        ec_code_intel_op_2(&intel, 0x0F, 0x85, 0);
+        ec_code_intel_immediate_4(&intel, rel);
+    }
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset);
+    ec_code_intel_op_2(&intel, 0x0F, 0x7F, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_1(&intel, 0x6F, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x7F, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x6F, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_1(&intel, 0xEF, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      dst);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0xEF, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      dst);
+
+    ec_code_intel_emit(builder, &intel);
+}
diff --git a/xlators/cluster/ec/src/ec-code-intel.h b/xlators/cluster/ec/src/ec-code-intel.h
new file mode 100644
index 00000000000..3fa4a174765
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-intel.h
@@ -0,0 +1,191 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_CODE_INTEL_H__
+#define __EC_CODE_INTEL_H__
+
+#include "ec-code.h"
+
+#define VEX_REG_NONE 0
+
+enum _ec_code_intel_reg;
+typedef enum _ec_code_intel_reg ec_code_intel_reg_t;
+
+enum _ec_code_vex_prefix;
+typedef enum _ec_code_vex_prefix ec_code_vex_prefix_t;
+
+enum _ec_code_vex_opcode;
+typedef enum _ec_code_vex_opcode ec_code_vex_opcode_t;
+
+struct _ec_code_intel_buffer;
+typedef struct _ec_code_intel_buffer ec_code_intel_buffer_t;
+
+struct _ec_code_intel_sib;
+typedef struct _ec_code_intel_sib ec_code_intel_sib_t;
+
+struct _ec_code_intel_modrm;
+typedef struct _ec_code_intel_modrm ec_code_intel_modrm_t;
+
+struct _ec_code_intel_rex;
+typedef struct _ec_code_intel_rex ec_code_intel_rex_t;
+
+struct _ec_code_intel;
+typedef struct _ec_code_intel ec_code_intel_t;
+
+enum _ec_code_intel_reg {
+    REG_NULL = -1,
+    REG_AX,
+    REG_CX,
+    REG_DX,
+    REG_BX,
+    REG_SP,
+    REG_BP,
+    REG_SI,
+    REG_DI,
+    REG_8,
+    REG_9,
+    REG_10,
+    REG_11,
+    REG_12,
+    REG_13,
+    REG_14,
+    REG_15
+};
+
+enum _ec_code_vex_prefix {
+    VEX_PREFIX_NONE = 0,
+    VEX_PREFIX_66,
+    VEX_PREFIX_F3,
+    VEX_PREFIX_F2
+};
+
+enum _ec_code_vex_opcode {
+    VEX_OPCODE_NONE = 0,
+    VEX_OPCODE_0F,
+    VEX_OPCODE_0F_38,
+    VEX_OPCODE_0F_3A
+};
+
+struct _ec_code_intel_buffer {
+    uint32_t bytes;
+    union {
+        uint8_t data[4];
+        uint32_t value;
+    };
+};
+
+struct _ec_code_intel_sib {
+    gf_boolean_t present;
+    uint32_t base;
+    uint32_t index;
+    uint32_t scale;
+};
+
+struct _ec_code_intel_modrm {
+    gf_boolean_t present;
+    uint32_t mod;
+    uint32_t rm;
+    uint32_t reg;
+};
+
+struct _ec_code_intel_rex {
+    gf_boolean_t present;
+    uint32_t w;
+    uint32_t r;
+    uint32_t x;
+    uint32_t b;
+};
+
+struct _ec_code_intel {
+    gf_boolean_t invalid;
+    ec_code_intel_buffer_t prefix;
+    ec_code_intel_buffer_t opcode;
+    ec_code_intel_buffer_t offset;
+    ec_code_intel_buffer_t immediate;
+    ec_code_intel_buffer_t vex;
+    ec_code_intel_rex_t rex;
+    ec_code_intel_modrm_t modrm;
+    ec_code_intel_sib_t sib;
+    uint32_t reg;
+};
+
+void
+ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size);
+
+void
+ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                         uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value,
+                         ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value,
+                          ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address);
+
+void
+ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+void
+ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+
+void
+ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+void
+ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+
+#endif /* __EC_CODE_INTEL_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-sse.c b/xlators/cluster/ec/src/ec-code-sse.c
new file mode 100644
index 00000000000..e11e7ff8400
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-sse.c
@@ -0,0 +1,101 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static void
+ec_code_sse_prolog(ec_code_builder_t *builder)
+{
+    builder->loop = builder->address;
+}
+
+static void
+ec_code_sse_epilog(ec_code_builder_t *builder)
+{
+    ec_code_intel_op_add_i2r(builder, 16, REG_DX);
+    ec_code_intel_op_add_i2r(builder, 16, REG_DI);
+    ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX);
+    ec_code_intel_op_jne(builder, builder->loop);
+
+    ec_code_intel_op_ret(builder, 0);
+}
+
+static void
+ec_code_sse_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_mov_m2sse(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_mov_m2sse(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static void
+ec_code_sse_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit)
+{
+    ec_code_intel_op_mov_sse2m(builder, src, REG_DI, REG_NULL, 0,
+                               bit * builder->width);
+}
+
+static void
+ec_code_sse_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_mov_sse2sse(builder, src, dst);
+}
+
+static void
+ec_code_sse_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_xor_sse2sse(builder, src, dst);
+}
+
+static void
+ec_code_sse_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_xor_m2sse(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_xor_m2sse(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static char *ec_code_sse_needed_flags[] = {"sse2", NULL};
+
+ec_code_gen_t ec_code_gen_sse = {.name = "sse",
+                                 .flags = ec_code_sse_needed_flags,
+                                 .width = 16,
+                                 .prolog = ec_code_sse_prolog,
+                                 .epilog = ec_code_sse_epilog,
+                                 .load = ec_code_sse_load,
+                                 .store = ec_code_sse_store,
+                                 .copy = ec_code_sse_copy,
+                                 .xor2 = ec_code_sse_xor2,
+                                 .xor3 = NULL,
+                                 .xorm = ec_code_sse_xorm};
diff --git a/xlators/performance/decompounder/src/decompounder-mem-types.h b/xlators/cluster/ec/src/ec-code-sse.h
index 5c211c1a907..f1acbcf894b 100644
--- a/xlators/performance/decompounder/src/decompounder-mem-types.h
+++ b/xlators/cluster/ec/src/ec-code-sse.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,13 +8,11 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef __DC_MEM_TYPES_H__
-#define __DC_MEM_TYPES_H__
+#ifndef __EC_CODE_SSE_H__
+#define __EC_CODE_SSE_H__
 
-#include "mem-types.h"
+#include "ec-code.h"
 
-enum gf_dc_mem_types_ {
-        gf_dc_mt_rsp_t              = gf_common_mt_end + 1,
-        gf_dc_mt_end
-};
-#endif
+extern ec_code_gen_t ec_code_gen_sse;
+
+#endif /* __EC_CODE_SSE_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-x64.c b/xlators/cluster/ec/src/ec-code-x64.c
new file mode 100644
index 00000000000..26565b4493f
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-x64.c
@@ -0,0 +1,144 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static ec_code_intel_reg_t ec_code_x64_regmap[] = {
+    REG_AX, REG_CX, REG_BP, REG_8,  REG_9, REG_10,
+    REG_11, REG_12, REG_13, REG_14, REG_15};
+
+static void
+ec_code_x64_prolog(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    ec_code_intel_op_push_r(builder, REG_BP);
+    if (!builder->linear) {
+        ec_code_intel_op_push_r(builder, REG_BX);
+    }
+    if (builder->regs > 11) {
+        ec_code_error(builder, EINVAL);
+        return;
+    }
+    for (i = 7; i < builder->regs; i++) {
+        ec_code_intel_op_push_r(builder, ec_code_x64_regmap[i]);
+    }
+
+    builder->loop = builder->address;
+}
+
+static void
+ec_code_x64_epilog(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    ec_code_intel_op_add_i2r(builder, 8, REG_DX);
+    ec_code_intel_op_add_i2r(builder, 8, REG_DI);
+    ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX);
+    ec_code_intel_op_jne(builder, builder->loop);
+
+    if (builder->regs > 11) {
+        ec_code_error(builder, EINVAL);
+        return;
+    }
+    for (i = builder->regs; i > 7; i--) {
+        ec_code_intel_op_pop_r(builder, ec_code_x64_regmap[i - 1]);
+    }
+    if (!builder->linear) {
+        ec_code_intel_op_pop_r(builder, REG_BX);
+    }
+    ec_code_intel_op_pop_r(builder, REG_BP);
+    ec_code_intel_op_ret(builder, 0);
+}
+
+static void
+ec_code_x64_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    dst = ec_code_x64_regmap[dst];
+
+    if (builder->linear) {
+        ec_code_intel_op_mov_m2r(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_BX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_mov_m2r(builder, REG_BX, REG_DX, 1,
+                                 bit * builder->width, dst);
+    }
+}
+
+static void
+ec_code_x64_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit)
+{
+    src = ec_code_x64_regmap[src];
+
+    ec_code_intel_op_mov_r2m(builder, src, REG_DI, REG_NULL, 0,
+                             bit * builder->width);
+}
+
+static void
+ec_code_x64_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    dst = ec_code_x64_regmap[dst];
+    src = ec_code_x64_regmap[src];
+
+    ec_code_intel_op_mov_r2r(builder, src, dst);
+}
+
+static void
+ec_code_x64_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    dst = ec_code_x64_regmap[dst];
+    src = ec_code_x64_regmap[src];
+
+    ec_code_intel_op_xor_r2r(builder, src, dst);
+}
+
+static void
+ec_code_x64_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    dst = ec_code_x64_regmap[dst];
+
+    if (builder->linear) {
+        ec_code_intel_op_xor_m2r(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_BX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_xor_m2r(builder, REG_BX, REG_DX, 1,
+                                 bit * builder->width, dst);
+    }
+}
+
+static char *ec_code_x64_needed_flags[] = {NULL};
+
+ec_code_gen_t ec_code_gen_x64 = {.name = "x64",
+                                 .flags = ec_code_x64_needed_flags,
+                                 .width = sizeof(uint64_t),
+                                 .prolog = ec_code_x64_prolog,
+                                 .epilog = ec_code_x64_epilog,
+                                 .load = ec_code_x64_load,
+                                 .store = ec_code_x64_store,
+                                 .copy = ec_code_x64_copy,
+                                 .xor2 = ec_code_x64_xor2,
+                                 .xor3 = NULL,
+                                 .xorm = ec_code_x64_xorm};
diff --git a/xlators/experimental/posix2/common/src/posix2-common.c b/xlators/cluster/ec/src/ec-code-x64.h
index 14b51d538b2..bd8174e4bf5 100644
--- a/xlators/experimental/posix2/common/src/posix2-common.c
+++ b/xlators/cluster/ec/src/ec-code-x64.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,11 +8,11 @@
   cases as published by the Free Software Foundation.
 */
 
-/* File: posix2-common.c
- * This file contains common routines across ds and mds posix xlators
- * The entire functionality including comments is TODO.
- */
+#ifndef __EC_CODE_X64_H__
+#define __EC_CODE_X64_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "statedump.h"
+#include "ec-code.h"
+
+extern ec_code_gen_t ec_code_gen_x64;
+
+#endif /* __EC_CODE_X64_H__ */
diff --git a/xlators/cluster/ec/src/ec-code.c b/xlators/cluster/ec/src/ec-code.c
new file mode 100644
index 00000000000..03162ae05a9
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code.c
@@ -0,0 +1,1060 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <ctype.h>
+
+#include <glusterfs/syscall.h>
+
+#include "ec-mem-types.h"
+#include "ec-code.h"
+#include "ec-messages.h"
+#include "ec-code-c.h"
+#include "ec-helpers.h"
+
+#ifdef USE_EC_DYNAMIC_X64
+#include "ec-code-x64.h"
+#endif
+
+#ifdef USE_EC_DYNAMIC_SSE
+#include "ec-code-sse.h"
+#endif
+
+#ifdef USE_EC_DYNAMIC_AVX
+#include "ec-code-avx.h"
+#endif
+
+#define EC_CODE_SIZE (1024 * 64)
+#define EC_CODE_ALIGN 4096
+
+#define EC_CODE_CHUNK_MIN_SIZE 512
+
+#define EC_PROC_BUFFER_SIZE 4096
+
+#define PROC_CPUINFO "/proc/cpuinfo"
+
+struct _ec_code_proc;
+typedef struct _ec_code_proc ec_code_proc_t;
+
+struct _ec_code_proc {
+    int32_t fd;
+    gf_boolean_t eof;
+    gf_boolean_t error;
+    gf_boolean_t skip;
+    ssize_t size;
+    ssize_t pos;
+    char buffer[EC_PROC_BUFFER_SIZE];
+};
+
+static ec_code_gen_t *ec_code_gen_table[] = {
+#ifdef USE_EC_DYNAMIC_AVX
+    &ec_code_gen_avx,
+#endif
+#ifdef USE_EC_DYNAMIC_SSE
+    &ec_code_gen_sse,
+#endif
+#ifdef USE_EC_DYNAMIC_X64
+    &ec_code_gen_x64,
+#endif
+    NULL};
+
+static void
+ec_code_arg_set(ec_code_arg_t *arg, uint32_t value)
+{
+    arg->value = value;
+}
+
+static void
+ec_code_arg_assign(ec_code_builder_t *builder, ec_code_op_t *op,
+                   ec_code_arg_t *arg, uint32_t reg)
+{
+    arg->value = reg;
+
+    if (builder->regs <= reg) {
+        builder->regs = reg + 1;
+    }
+}
+
+static void
+ec_code_arg_use(ec_code_builder_t *builder, ec_code_op_t *op,
+                ec_code_arg_t *arg, uint32_t reg)
+{
+    arg->value = reg;
+}
+
+static void
+ec_code_arg_update(ec_code_builder_t *builder, ec_code_op_t *op,
+                   ec_code_arg_t *arg, uint32_t reg)
+{
+    arg->value = reg;
+}
+
+static ec_code_op_t *
+ec_code_op_next(ec_code_builder_t *builder)
+{
+    ec_code_op_t *op;
+
+    op = &builder->ops[builder->count++];
+    memset(op, 0, sizeof(ec_code_op_t));
+
+    return op;
+}
+
+static void
+ec_code_load(ec_code_builder_t *builder, uint32_t bit, uint32_t offset)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_LOAD;
+    ec_code_arg_assign(builder, op, &op->arg1, builder->map[bit]);
+    ec_code_arg_set(&op->arg2, offset);
+    ec_code_arg_set(&op->arg3, bit);
+}
+
+static void
+ec_code_store(ec_code_builder_t *builder, uint32_t reg, uint32_t bit)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_STORE;
+    ec_code_arg_use(builder, op, &op->arg1, builder->map[reg]);
+    ec_code_arg_set(&op->arg2, 0);
+    ec_code_arg_set(&op->arg3, bit);
+}
+
+static void
+ec_code_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_COPY;
+    ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]);
+    ec_code_arg_use(builder, op, &op->arg2, builder->map[src]);
+    ec_code_arg_set(&op->arg3, 0);
+}
+
+static void
+ec_code_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_XOR2;
+    ec_code_arg_update(builder, op, &op->arg1, builder->map[dst]);
+    ec_code_arg_use(builder, op, &op->arg2, builder->map[src]);
+    ec_code_arg_set(&op->arg3, 0);
+}
+
+static void
+ec_code_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1,
+             uint32_t src2)
+{
+    ec_code_op_t *op;
+
+    if (builder->code->gen->xor3 == NULL) {
+        ec_code_copy(builder, dst, src1);
+        ec_code_xor2(builder, dst, src2);
+
+        return;
+    }
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_XOR3;
+    ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]);
+    ec_code_arg_use(builder, op, &op->arg2, builder->map[src1]);
+    ec_code_arg_use(builder, op, &op->arg3, builder->map[src2]);
+}
+
+static void
+ec_code_xorm(ec_code_builder_t *builder, uint32_t bit, uint32_t offset)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_XORM;
+    ec_code_arg_update(builder, op, &op->arg1, builder->map[bit]);
+    ec_code_arg_set(&op->arg2, offset);
+    ec_code_arg_set(&op->arg3, bit);
+}
+
+static void
+ec_code_dup(ec_code_builder_t *builder, ec_gf_op_t *op)
+{
+    switch (op->op) {
+        case EC_GF_OP_COPY:
+            ec_code_copy(builder, op->arg1, op->arg2);
+            break;
+        case EC_GF_OP_XOR2:
+            ec_code_xor2(builder, op->arg1, op->arg2);
+            break;
+        case EC_GF_OP_XOR3:
+            ec_code_xor3(builder, op->arg1, op->arg2, op->arg3);
+            break;
+        default:
+            break;
+    }
+}
+
+static void
+ec_code_gf_load(ec_code_builder_t *builder, uint32_t offset)
+{
+    uint32_t i;
+
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_load(builder, i, offset);
+    }
+}
+
+static void
+ec_code_gf_load_xor(ec_code_builder_t *builder, uint32_t offset)
+{
+    uint32_t i;
+
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_xorm(builder, i, offset);
+    }
+}
+
+static void
+ec_code_gf_store(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_store(builder, i, i);
+    }
+}
+
+static void
+ec_code_gf_clear(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    ec_code_xor2(builder, 0, 0);
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_store(builder, 0, i);
+    }
+}
+
+static void
+ec_code_gf_mul(ec_code_builder_t *builder, uint32_t value)
+{
+    ec_gf_mul_t *mul;
+    ec_gf_op_t *op;
+    uint32_t map[EC_GF_MAX_REGS];
+    int32_t i;
+
+    mul = builder->code->gf->table[value];
+    for (op = mul->ops; op->op != EC_GF_OP_END; op++) {
+        ec_code_dup(builder, op);
+    }
+
+    for (i = 0; i < mul->regs; i++) {
+        map[i] = builder->map[mul->map[i]];
+    }
+    memcpy(builder->map, map, sizeof(uint32_t) * mul->regs);
+}
+
+static ec_code_builder_t *
+ec_code_prepare(ec_code_t *code, uint32_t count, uint32_t width,
+                gf_boolean_t linear)
+{
+    ec_code_builder_t *builder;
+    uint32_t i;
+
+    count *= code->gf->bits + code->gf->max_ops;
+    count += code->gf->bits;
+    builder = GF_MALLOC(
+        sizeof(ec_code_builder_t) + sizeof(ec_code_op_t) * count,
+        ec_mt_ec_code_builder_t);
+    if (builder == NULL) {
+        return EC_ERR(ENOMEM);
+    }
+
+    builder->address = 0;
+    builder->code = code;
+    builder->size = 0;
+    builder->count = 0;
+    builder->regs = 0;
+    builder->error = 0;
+    builder->bits = code->gf->bits;
+    builder->width = width;
+    builder->data = NULL;
+    builder->linear = linear;
+    builder->base = -1;
+
+    for (i = 0; i < EC_GF_MAX_REGS; i++) {
+        builder->map[i] = i;
+    }
+
+    return builder;
+}
+
+static size_t
+ec_code_space_size(void)
+{
+    return (sizeof(ec_code_space_t) + 15) & ~15;
+}
+
+static size_t
+ec_code_chunk_size(void)
+{
+    return (sizeof(ec_code_chunk_t) + 15) & ~15;
+}
+
+static ec_code_chunk_t *
+ec_code_chunk_from_space(ec_code_space_t *space)
+{
+    return (ec_code_chunk_t *)((uintptr_t)space + ec_code_space_size());
+}
+
+static void *
+ec_code_to_executable(ec_code_space_t *space, void *addr)
+{
+    return (void *)((uintptr_t)addr - (uintptr_t)space +
+                    (uintptr_t)space->exec);
+}
+
+static void *
+ec_code_from_executable(ec_code_space_t *space, void *addr)
+{
+    return (void *)((uintptr_t)addr - (uintptr_t)space->exec +
+                    (uintptr_t)space);
+}
+
+static void *
+ec_code_func_from_chunk(ec_code_chunk_t *chunk, void **exec)
+{
+    void *addr;
+
+    addr = (void *)((uintptr_t)chunk + ec_code_chunk_size());
+
+    *exec = ec_code_to_executable(chunk->space, addr);
+
+    return addr;
+}
+
+static ec_code_chunk_t *
+ec_code_chunk_from_func(ec_code_func_linear_t func)
+{
+    ec_code_chunk_t *chunk;
+
+    chunk = (ec_code_chunk_t *)((uintptr_t)func - ec_code_chunk_size());
+
+    return ec_code_from_executable(chunk->space, chunk);
+}
+
+static ec_code_chunk_t *
+ec_code_chunk_split(ec_code_chunk_t *chunk, size_t size)
+{
+    ec_code_chunk_t *extra;
+    ssize_t avail;
+
+    avail = chunk->size - size - ec_code_chunk_size();
+    if (avail > 0) {
+        extra = (ec_code_chunk_t *)((uintptr_t)chunk + chunk->size - avail);
+        extra->space = chunk->space;
+        extra->size = avail;
+        list_add(&extra->list, &chunk->list);
+        chunk->size = size;
+    }
+    list_del_init(&chunk->list);
+
+    return chunk;
+}
+
+static gf_boolean_t
+ec_code_chunk_touch(ec_code_chunk_t *prev, ec_code_chunk_t *next)
+{
+    uintptr_t end;
+
+    end = (uintptr_t)prev + ec_code_chunk_size() + prev->size;
+    return (end == (uintptr_t)next);
+}
+
+static ec_code_space_t *
+ec_code_space_create(ec_code_t *code, size_t size)
+{
+    char path[] = GLUSTERFS_LIBEXECDIR "/ec-code-dynamic.XXXXXX";
+    ec_code_space_t *space;
+    void *exec;
+    int32_t fd, err;
+
+    /* We need to create memory areas to store the generated dynamic code.
+     * Obviously these areas need to be written to be able to create the
+     * code and they also need to be executable to execute it.
+     *
+     * However it's a bad practice to have a memory region that is both
+     * writable *and* executable. In fact, selinux forbids this and causes
+     * attempts to do so to fail (unless specifically configured).
+     *
+     * To solve the problem we'll use two distinct memory areas mapped to
+     * the same physical storage. One of the memory areas will have write
+     * permission, and the other will have execute permission. Both areas
+     * will have the same contents. The physical storage will be a regular
+     * file that will be mmapped to both areas.
+     */
+
+    /* We need to create a temporary file as the backend storage for the
+     * memory mapped areas. */
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    fd = mkstemp(path);
+    if (fd < 0) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to create a temporary file for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+        goto done;
+    }
+    /* Once created we don't need to keep it in the file system. It will
+     * still exist until we close the last file descriptor or unmap the
+     * memory areas bound to the file. */
+    sys_unlink(path);
+
+    size = (size + EC_CODE_ALIGN - 1) & ~(EC_CODE_ALIGN - 1);
+    if (sys_ftruncate(fd, size) < 0) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to resize the file for the ec dynamic code");
+        space = EC_ERR(err);
+        goto done_close;
+    }
+
+    /* This creates an executable memory area to be able to run the
+     * generated fragments of code. */
+    exec = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0);
+    if (exec == MAP_FAILED) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to map the executable area for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+        goto done_close;
+    }
+    /* It's not important to check the return value of mlock(). If it fails
+     * everything will continue to work normally. */
+    mlock(exec, size);
+
+    /* This maps a read/write memory area to be able to create the dynamici
+     * code. */
+    space = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (space == MAP_FAILED) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to map the writable area for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+
+        munmap(exec, size);
+
+        goto done_close;
+    }
+
+    space->exec = exec;
+    space->size = size;
+    space->code = code;
+    list_add_tail(&space->list, &code->spaces);
+    INIT_LIST_HEAD(&space->chunks);
+
+done_close:
+    /* If everything has succeeded, we already have the memory areas
+     * mapped. We don't need the file descriptor anymore because the
+     * backend storage will be there until the mmap()'d regions are
+     * unmapped. */
+    sys_close(fd);
+done:
+    return space;
+}
+
+static void
+ec_code_space_destroy(ec_code_space_t *space)
+{
+    list_del_init(&space->list);
+
+    munmap(space->exec, space->size);
+    munmap(space, space->size);
+}
+
+static void
+ec_code_chunk_merge(ec_code_chunk_t *chunk)
+{
+    ec_code_chunk_t *item, *tmp;
+
+    list_for_each_entry_safe(item, tmp, &chunk->space->chunks, list)
+    {
+        if ((uintptr_t)item > (uintptr_t)chunk) {
+            list_add_tail(&chunk->list, &item->list);
+            if (ec_code_chunk_touch(chunk, item)) {
+                chunk->size += item->size + ec_code_chunk_size();
+                list_del_init(&item->list);
+            }
+
+            goto check;
+        }
+        if (ec_code_chunk_touch(item, chunk)) {
+            item->size += chunk->size + ec_code_chunk_size();
+            list_del_init(&item->list);
+            chunk = item;
+        }
+    }
+    list_add_tail(&chunk->list, &chunk->space->chunks);
+
+check:
+    if (chunk->size ==
+        chunk->space->size - ec_code_space_size() - ec_code_chunk_size()) {
+        ec_code_space_destroy(chunk->space);
+    }
+}
+
+static ec_code_chunk_t *
+ec_code_space_alloc(ec_code_t *code, size_t size)
+{
+    ec_code_space_t *space;
+    ec_code_chunk_t *chunk;
+    size_t map_size;
+
+    /* To minimize fragmentation, we only allocate chunks of sizes multiples
+     * of EC_CODE_CHUNK_MIN_SIZE. */
+    size = ((size + ec_code_chunk_size() + EC_CODE_CHUNK_MIN_SIZE - 1) &
+            ~(EC_CODE_CHUNK_MIN_SIZE - 1)) -
+           ec_code_chunk_size();
+    list_for_each_entry(space, &code->spaces, list)
+    {
+        list_for_each_entry(chunk, &space->chunks, list)
+        {
+            if (chunk->size >= size) {
+                goto out;
+            }
+        }
+    }
+
+    map_size = EC_CODE_SIZE - ec_code_space_size() - ec_code_chunk_size();
+    if (map_size < size) {
+        map_size = size;
+    }
+    space = ec_code_space_create(code, map_size);
+    if (EC_IS_ERR(space)) {
+        return (ec_code_chunk_t *)space;
+    }
+
+    chunk = ec_code_chunk_from_space(space);
+    chunk->size = map_size - ec_code_space_size() - ec_code_chunk_size();
+    list_add(&chunk->list, &space->chunks);
+
+out:
+    chunk->space = space;
+
+    return ec_code_chunk_split(chunk, size);
+}
+
+static ec_code_chunk_t *
+ec_code_alloc(ec_code_t *code, uint32_t size)
+{
+    ec_code_chunk_t *chunk;
+
+    LOCK(&code->lock);
+
+    chunk = ec_code_space_alloc(code, size);
+
+    UNLOCK(&code->lock);
+
+    return chunk;
+}
+
+static void
+ec_code_free(ec_code_chunk_t *chunk)
+{
+    gf_lock_t *lock;
+
+    lock = &chunk->space->code->lock;
+    LOCK(lock);
+
+    ec_code_chunk_merge(chunk);
+
+    UNLOCK(lock);
+}
+
+static int32_t
+ec_code_write(ec_code_builder_t *builder)
+{
+    ec_code_gen_t *gen;
+    ec_code_op_t *op;
+    uint32_t i;
+
+    builder->error = 0;
+    builder->size = 0;
+    builder->address = 0;
+    builder->base = -1;
+
+    gen = builder->code->gen;
+    gen->prolog(builder);
+    for (i = 0; i < builder->count; i++) {
+        op = &builder->ops[i];
+        switch (op->op) {
+            case EC_GF_OP_LOAD:
+                gen->load(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            case EC_GF_OP_STORE:
+                gen->store(builder, op->arg1.value, op->arg3.value);
+                break;
+            case EC_GF_OP_COPY:
+                gen->copy(builder, op->arg1.value, op->arg2.value);
+                break;
+            case EC_GF_OP_XOR2:
+                gen->xor2(builder, op->arg1.value, op->arg2.value);
+                break;
+            case EC_GF_OP_XOR3:
+                gen->xor3(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            case EC_GF_OP_XORM:
+                gen->xorm(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            default:
+                break;
+        }
+    }
+    gen->epilog(builder);
+
+    return builder->error;
+}
+
+static void *
+ec_code_compile(ec_code_builder_t *builder)
+{
+    ec_code_chunk_t *chunk;
+    void *func;
+    int32_t err;
+
+    err = ec_code_write(builder);
+    if (err != 0) {
+        return EC_ERR(err);
+    }
+
+    chunk = ec_code_alloc(builder->code, builder->size);
+    if (EC_IS_ERR(chunk)) {
+        return chunk;
+    }
+    builder->data = ec_code_func_from_chunk(chunk, &func);
+
+    err = ec_code_write(builder);
+    if (err != 0) {
+        ec_code_free(chunk);
+
+        return EC_ERR(err);
+    }
+
+    GF_FREE(builder);
+
+    return func;
+}
+
+ec_code_t *
+ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen)
+{
+    ec_code_t *code;
+
+    code = GF_MALLOC(sizeof(ec_code_t), ec_mt_ec_code_t);
+    if (code == NULL) {
+        return EC_ERR(ENOMEM);
+    }
+    memset(code, 0, sizeof(ec_code_t));
+    INIT_LIST_HEAD(&code->spaces);
+    LOCK_INIT(&code->lock);
+
+    code->gf = gf;
+    code->gen = gen;
+
+    return code;
+}
+
+void
+ec_code_destroy(ec_code_t *code)
+{
+    if (!list_empty(&code->spaces)) {
+    }
+
+    LOCK_DESTROY(&code->lock);
+
+    GF_FREE(code);
+}
+
+static uint32_t
+ec_code_value_next(uint32_t *values, uint32_t count, uint32_t *offset)
+{
+    uint32_t i, next;
+
+    next = 0;
+    for (i = *offset + 1; i < count; i++) {
+        next = values[i];
+        if (next != 0) {
+            break;
+        }
+    }
+    *offset = i;
+
+    return next;
+}
+
+static void *
+ec_code_build_dynamic(ec_code_t *code, uint32_t width, uint32_t *values,
+                      uint32_t count, gf_boolean_t linear)
+{
+    ec_code_builder_t *builder;
+    uint32_t offset, val, next;
+
+    builder = ec_code_prepare(code, count, width, linear);
+    if (EC_IS_ERR(builder)) {
+        return builder;
+    }
+
+    offset = -1;
+    next = ec_code_value_next(values, count, &offset);
+    if (next != 0) {
+        ec_code_gf_load(builder, offset);
+        do {
+            val = next;
+            next = ec_code_value_next(values, count, &offset);
+            if (next != 0) {
+                ec_code_gf_mul(builder, ec_gf_div(code->gf, val, next));
+                ec_code_gf_load_xor(builder, offset);
+            }
+        } while (next != 0);
+        ec_code_gf_mul(builder, val);
+        ec_code_gf_store(builder);
+    } else {
+        ec_code_gf_clear(builder);
+    }
+
+    return ec_code_compile(builder);
+}
+
+static void *
+ec_code_build(ec_code_t *code, uint32_t width, uint32_t *values, uint32_t count,
+              gf_boolean_t linear)
+{
+    void *func;
+
+    if (code->gen != NULL) {
+        func = ec_code_build_dynamic(code, width, values, count, linear);
+        if (!EC_IS_ERR(func)) {
+            return func;
+        }
+
+        gf_msg_debug(THIS->name, GF_LOG_DEBUG,
+                     "Unable to generate dynamic code. Falling back "
+                     "to precompiled code");
+
+        /* The dynamic code generation shouldn't fail in normal
+         * conditions, but if it fails at some point, it's very
+         * probable that it will fail again, so we completely disable
+         * dynamic code generation. */
+        code->gen = NULL;
+    }
+
+    ec_code_c_prepare(code->gf, values, count);
+
+    if (linear) {
+        return ec_code_c_linear;
+    }
+
+    return ec_code_c_interleaved;
+}
+
+ec_code_func_linear_t
+ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values,
+                     uint32_t count)
+{
+    return (ec_code_func_linear_t)ec_code_build(code, width, values, count,
+                                                _gf_true);
+}
+
+ec_code_func_interleaved_t
+ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values,
+                          uint32_t count)
+{
+    return (ec_code_func_interleaved_t)ec_code_build(code, width, values, count,
+                                                     _gf_false);
+}
+
+void
+ec_code_release(ec_code_t *code, ec_code_func_t *func)
+{
+    if ((func->linear != ec_code_c_linear) &&
+        (func->interleaved != ec_code_c_interleaved)) {
+        ec_code_free(ec_code_chunk_from_func(func->linear));
+    }
+}
+
+void
+ec_code_error(ec_code_builder_t *builder, int32_t error)
+{
+    if (builder->error == 0) {
+        gf_msg(THIS->name, GF_LOG_ERROR, error, EC_MSG_DYN_CODEGEN_FAILED,
+               "Failed to generate dynamic code");
+        builder->error = error;
+    }
+}
+
+void
+ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count)
+{
+    if (builder->error != 0) {
+        return;
+    }
+
+    if (builder->data != NULL) {
+        memcpy(builder->data + builder->size, bytes, count);
+    }
+
+    builder->size += count;
+    builder->address += count;
+}
+
+static char *
+ec_code_proc_trim_left(char *text, ssize_t *length)
+{
+    ssize_t len;
+
+    for (len = *length; (len > 0) && isspace(*text); len--) {
+        text++;
+    }
+    *length = len;
+
+    return text;
+}
+
+static char *
+ec_code_proc_trim_right(char *text, ssize_t *length, char sep)
+{
+    char *last;
+    ssize_t len;
+
+    len = *length;
+
+    last = text;
+    for (len = *length; (len > 0) && (*text != sep); len--) {
+        if (!isspace(*text)) {
+            last = text + 1;
+        }
+        text++;
+    }
+    *last = 0;
+    *length = len;
+
+    return text;
+}
+
+static char *
+ec_code_proc_line_parse(ec_code_proc_t *file, ssize_t *length)
+{
+    char *text, *end;
+    ssize_t len;
+
+    len = file->size - file->pos;
+    text = ec_code_proc_trim_left(file->buffer + file->pos, &len);
+    end = ec_code_proc_trim_right(text, &len, '\n');
+    if (len == 0) {
+        if (!file->eof) {
+            if (text == file->buffer) {
+                file->size = file->pos = 0;
+                file->skip = _gf_true;
+            } else {
+                file->size = file->pos = end - text;
+                memmove(file->buffer, text, file->pos + 1);
+            }
+            len = sys_read(file->fd, file->buffer + file->pos,
+                           sizeof(file->buffer) - file->pos - 1);
+            if (len > 0) {
+                file->size += len;
+            }
+            file->error = len < 0;
+            file->eof = len <= 0;
+
+            return NULL;
+        }
+        file->size = file->pos = 0;
+    } else {
+        file->pos = end - file->buffer + 1;
+    }
+
+    *length = end - text;
+
+    if (file->skip) {
+        file->skip = _gf_false;
+        text = NULL;
+    }
+
+    return text;
+}
+
+static char *
+ec_code_proc_line(ec_code_proc_t *file, ssize_t *length)
+{
+    char *text;
+
+    text = NULL;
+    while (!file->eof) {
+        text = ec_code_proc_line_parse(file, length);
+        if (text != NULL) {
+            break;
+        }
+    }
+
+    return text;
+}
+
+static char *
+ec_code_proc_split(char *text, ssize_t *length, char sep)
+{
+    text = ec_code_proc_trim_right(text, length, sep);
+    if (*length == 0) {
+        return NULL;
+    }
+    (*length)--;
+    text++;
+
+    return ec_code_proc_trim_left(text, length);
+}
+
+static uint32_t
+ec_code_cpu_check(uint32_t idx, char *list, uint32_t count)
+{
+    ec_code_gen_t *gen;
+    char **ptr;
+    char *table[count + 1];
+    uint32_t i;
+
+    for (i = 0; i < count; i++) {
+        table[i] = list;
+        list += strlen(list) + 1;
+    }
+
+    gen = ec_code_gen_table[idx];
+    while (gen != NULL) {
+        for (ptr = gen->flags; *ptr != NULL; ptr++) {
+            for (i = 0; i < count; i++) {
+                if (strcmp(*ptr, table[i]) == 0) {
+                    break;
+                }
+            }
+            if (i >= count) {
+                gen = ec_code_gen_table[++idx];
+                break;
+            }
+        }
+        if (*ptr == NULL) {
+            break;
+        }
+    }
+
+    return idx;
+}
+
+ec_code_gen_t *
+ec_code_detect(xlator_t *xl, const char *def)
+{
+    ec_code_proc_t file;
+    ec_code_gen_t *gen = NULL;
+    char *line, *data, *list;
+    ssize_t length;
+    uint32_t count, base, select;
+
+    if (strcmp(def, "none") == 0) {
+        gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE,
+               "Not using any cpu extensions");
+
+        return NULL;
+    }
+
+    file.fd = sys_open(PROC_CPUINFO, O_RDONLY, 0);
+    if (file.fd < 0) {
+        goto out;
+    }
+    file.size = file.pos = 0;
+    file.eof = file.error = file.skip = _gf_false;
+
+    select = 0;
+    if (strcmp(def, "auto") != 0) {
+        while (ec_code_gen_table[select] != NULL) {
+            if (strcmp(ec_code_gen_table[select]->name, def) == 0) {
+                break;
+            }
+            select++;
+        }
+        if (ec_code_gen_table[select] == NULL) {
+            gf_msg(xl->name, GF_LOG_WARNING, EINVAL, EC_MSG_EXTENSION_UNKNOWN,
+                   "CPU extension '%s' is not known. Not using any cpu "
+                   "extensions",
+                   def);
+
+            return NULL;
+        }
+    } else {
+        def = NULL;
+    }
+
+    while ((line = ec_code_proc_line(&file, &length)) != NULL) {
+        data = ec_code_proc_split(line, &length, ':');
+        if ((data != NULL) && (strcmp(line, "flags") == 0)) {
+            list = data;
+            count = 0;
+            while ((data != NULL) && (*data != 0)) {
+                count++;
+                data = ec_code_proc_split(data, &length, ' ');
+            }
+            base = select;
+            select = ec_code_cpu_check(select, list, count);
+            if ((base != select) && (def != NULL)) {
+                gf_msg(xl->name, GF_LOG_WARNING, ENOTSUP,
+                       EC_MSG_EXTENSION_UNSUPPORTED,
+                       "CPU extension '%s' is not supported", def);
+                def = NULL;
+            }
+        }
+    }
+
+    if (file.error) {
+        gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_EXTENSION_FAILED,
+               "Unable to determine supported CPU extensions. Not using any "
+               "cpu extensions");
+
+        gen = NULL;
+    } else {
+        gen = ec_code_gen_table[select];
+        if (gen == NULL) {
+            gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE,
+                   "Not using any cpu extensions");
+        } else {
+            gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION,
+                   "Using '%s' CPU extensions", gen->name);
+        }
+    }
+
+    sys_close(file.fd);
+
+out:
+    return gen;
+}
diff --git a/xlators/cluster/ec/src/ec-code.h b/xlators/cluster/ec/src/ec-code.h
new file mode 100644
index 00000000000..75fb35d93e3
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code.h
@@ -0,0 +1,44 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_CODE_H__
+#define __EC_CODE_H__
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/list.h>
+
+#include "ec-types.h"
+#include "ec-galois.h"
+
+ec_code_gen_t *
+ec_code_detect(xlator_t *xl, const char *def);
+
+ec_code_t *
+ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen);
+
+void
+ec_code_destroy(ec_code_t *code);
+
+ec_code_func_linear_t
+ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values,
+                     uint32_t count);
+ec_code_func_interleaved_t
+ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values,
+                          uint32_t count);
+void
+ec_code_release(ec_code_t *code, ec_code_func_t *func);
+
+void
+ec_code_error(ec_code_builder_t *builder, int32_t error);
+
+void
+ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count);
+
+#endif /* __EC_CODE_H__ */
diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c
index 2566cc0d0cf..703a30e2485 100644
--- a/xlators/cluster/ec/src/ec-combine.c
+++ b/xlators/cluster/ec/src/ec-combine.c
@@ -11,58 +11,57 @@
 #include <fnmatch.h>
 
 #include "libxlator.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
-#include "ec-data.h"
+#include "ec-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-messages.h"
-#include "quota-common-utils.h"
+#include <glusterfs/quota-common-utils.h>
 
 #define EC_QUOTA_PREFIX "trusted.glusterfs.quota."
 
+#define EC_MISSING_DATA ((data_t *)1ULL)
+
 struct _ec_dict_info;
 typedef struct _ec_dict_info ec_dict_info_t;
 
 struct _ec_dict_combine;
 typedef struct _ec_dict_combine ec_dict_combine_t;
 
-struct _ec_dict_info
-{
-    dict_t * dict;
-    int32_t  count;
+struct _ec_dict_info {
+    dict_t *dict;
+    int32_t count;
 };
 
-struct _ec_dict_combine
-{
-    ec_cbk_data_t * cbk;
-    int32_t         which;
+struct _ec_dict_combine {
+    ec_cbk_data_t *cbk;
+    int32_t which;
 };
 
 int32_t
-ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                  ec_cbk_data_t *src)
+ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-        int     valid = 0;
+    int valid = 0;
 
-        if (!fop || !dst || !src)
-                return 0;
+    if (!fop || !dst || !src)
+        return 0;
 
-        switch (fop->id) {
+    switch (fop->id) {
         case GF_FOP_REMOVEXATTR:
         case GF_FOP_FREMOVEXATTR:
         case GF_FOP_SETXATTR:
         case GF_FOP_FSETXATTR:
-                return 1;
+            return 1;
 
         case GF_FOP_SYMLINK:
         case GF_FOP_LINK:
         case GF_FOP_CREATE:
         case GF_FOP_MKNOD:
         case GF_FOP_MKDIR:
-                valid = 3;
-                break;
+            valid = 3;
+            break;
         case GF_FOP_UNLINK:
         case GF_FOP_RMDIR:
         case GF_FOP_SETATTR:
@@ -73,57 +72,40 @@ ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
         case GF_FOP_FALLOCATE:
         case GF_FOP_DISCARD:
         case GF_FOP_ZEROFILL:
-                valid = 2;
-                break;
+            valid = 2;
+            break;
         case GF_FOP_RENAME:
-                valid = 5;
-                break;
+            valid = 5;
+            break;
         default:
-                gf_msg_callingfn (fop->xl->name, GF_LOG_WARNING, EINVAL,
-                                  EC_MSG_INVALID_FOP,
-                                  "Invalid fop %d", fop->id);
-                return 0;
-                break;
-        }
+            gf_msg_callingfn(fop->xl->name, GF_LOG_WARNING, EINVAL,
+                             EC_MSG_INVALID_FOP, "Invalid fop %d", fop->id);
+            return 0;
+            break;
+    }
 
-        if (!ec_iatt_combine(fop, dst->iatt, src->iatt, valid)) {
-                gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                        EC_MSG_IATT_MISMATCH,
-                        "Mismatching iatt in "
-                        "answers of '%s'", gf_fop_list[fop->id]);
-                return 0;
-        }
-        return 1;
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, valid)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of '%s'",
+               gf_fop_list[fop->id]);
+        return 0;
+    }
+    return 1;
 }
 
-void ec_iatt_time_merge(uint32_t * dst_sec, uint32_t * dst_nsec,
-                        uint32_t src_sec, uint32_t src_nsec)
+void
+ec_iatt_time_merge(int64_t *dst_sec, uint32_t *dst_nsec, int64_t src_sec,
+                   uint32_t src_nsec)
 {
     if ((*dst_sec < src_sec) ||
-        ((*dst_sec == src_sec) && (*dst_nsec < src_nsec)))
-    {
+        ((*dst_sec == src_sec) && (*dst_nsec < src_nsec))) {
         *dst_sec = src_sec;
         *dst_nsec = src_nsec;
     }
 }
 
-static
-uint64_t
-gfid_to_ino(uuid_t gfid)
-{
-    uint64_t ino = 0;
-    int32_t i;
-
-    for (i = 8; i < 16; i++) {
-        ino <<= 8;
-        ino += (uint8_t)gfid[i];
-    }
-
-    return ino;
-}
-
-static
-gf_boolean_t
+static gf_boolean_t
 ec_iatt_is_trusted(ec_fop_data_t *fop, struct iatt *iatt)
 {
     uint64_t ino;
@@ -154,14 +136,14 @@ ec_iatt_is_trusted(ec_fop_data_t *fop, struct iatt *iatt)
     return _gf_false;
 }
 
-int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
-                        int32_t count)
+int32_t
+ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
+                int32_t count)
 {
     int32_t i;
     gf_boolean_t failed = _gf_false;
 
-    for (i = 0; i < count; i++)
-    {
+    for (i = 0; i < count; i++) {
         /* Check for basic fields. These fields must be equal always, even if
          * the inode is not locked because in these cases the parent inode
          * will be locked and differences in these fields require changes in
@@ -176,8 +158,6 @@ int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
          * inode is not locked. */
         if (!failed && ((dst[i].ia_uid != src[i].ia_uid) ||
                         (dst[i].ia_gid != src[i].ia_gid) ||
-                        ((dst[i].ia_type == IA_IFREG) &&
-                         (dst[i].ia_size != src[i].ia_size)) ||
                         (st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type) !=
                          st_mode_from_ia(src[i].ia_prot, src[i].ia_type)))) {
             if (ec_iatt_is_trusted(fop, dst)) {
@@ -187,33 +167,34 @@ int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
                  * data is returned. */
                 failed = _gf_true;
             } else {
-                gf_msg_debug (fop->xl->name, 0,
-                       "Ignoring iatt differences because inode is not "
-                       "locked");
+                gf_msg_debug(fop->xl->name, 0,
+                             "Ignoring iatt differences because inode is not "
+                             "locked");
             }
         }
         if (failed) {
-            gf_msg (fop->xl->name, GF_LOG_WARNING, 0,
-                    EC_MSG_IATT_COMBINE_FAIL,
-                    "Failed to combine iatt (inode: %lu-%lu, links: %u-%u, "
-                    "uid: %u-%u, gid: %u-%u, rdev: %lu-%lu, size: %lu-%lu, "
-                    "mode: %o-%o)",
-                    dst[i].ia_ino, src[i].ia_ino, dst[i].ia_nlink,
-                    src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid,
-                    dst[i].ia_gid, src[i].ia_gid, dst[i].ia_rdev,
-                    src[i].ia_rdev, dst[i].ia_size, src[i].ia_size,
-                    st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type),
-                    st_mode_from_ia(src[i].ia_prot, dst[i].ia_type));
+            gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_IATT_COMBINE_FAIL,
+                   "Failed to combine iatt (inode: %" PRIu64 "-%" PRIu64
+                   ", "
+                   "links: %u-%u, uid: %u-%u, gid: %u-%u, "
+                   "rdev: %" PRIu64 "-%" PRIu64 ", size: %" PRIu64 "-%" PRIu64
+                   ", "
+                   "mode: %o-%o), %s",
+                   dst[i].ia_ino, src[i].ia_ino, dst[i].ia_nlink,
+                   src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid, dst[i].ia_gid,
+                   src[i].ia_gid, dst[i].ia_rdev, src[i].ia_rdev,
+                   dst[i].ia_size, src[i].ia_size,
+                   st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type),
+                   st_mode_from_ia(src[i].ia_prot, dst[i].ia_type),
+                   ec_msg_str(fop));
 
             return 0;
         }
     }
 
-    while (count-- > 0)
-    {
+    while (count-- > 0) {
         dst[count].ia_blocks += src[count].ia_blocks;
-        if (dst[count].ia_blksize < src[count].ia_blksize)
-        {
+        if (dst[count].ia_blksize < src[count].ia_blksize) {
             dst[count].ia_blksize = src[count].ia_blksize;
         }
 
@@ -228,13 +209,12 @@ int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
     return 1;
 }
 
-void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
-                     int32_t answers)
+void
+ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers)
 {
     uint64_t blocks;
 
-    while (count-- > 0)
-    {
+    while (count-- > 0) {
         blocks = iatt[count].ia_blocks * ec->fragments + answers - 1;
         blocks /= answers;
         iatt[count].ia_blocks = blocks;
@@ -242,79 +222,93 @@ void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
 }
 
 gf_boolean_t
-ec_xattr_match (dict_t *dict, char *key, data_t *value, void *arg)
+ec_xattr_match(dict_t *dict, char *key, data_t *value, void *arg)
 {
-        if ((fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) ||
-            (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)) {
-                return _gf_false;
-        }
+    if ((fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) ||
+        (strcmp(key, GET_LINK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)) {
+        return _gf_false;
+    }
 
-        return _gf_true;
+    return _gf_true;
 }
 
 gf_boolean_t
-ec_value_ignore (char *key)
+ec_value_ignore(char *key)
 {
-        if ((strcmp(key, GF_CONTENT_KEY) == 0) ||
-            (strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
-            (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0) ||
-            (strcmp(key, GF_XATTR_LOCKINFO_KEY) == 0) ||
-            (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) ||
-            (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
-            (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
-            (strncmp(key, GF_XATTR_CLRLK_CMD,
-                     strlen (GF_XATTR_CLRLK_CMD)) == 0) ||
-            (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) ||
-            (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) ||
-            (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) ||
-            (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) ||
-            (XATTR_IS_NODE_UUID(key))) {
-                return _gf_true;
-        }
-        return _gf_false;
+    if ((strcmp(key, GF_CONTENT_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_LOCKINFO_KEY) == 0) ||
+        (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
+        (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) ||
+        (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) ||
+        (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) ||
+        (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) ||
+        (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) ||
+        (XATTR_IS_NODE_UUID(key))) {
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
 int32_t
-ec_dict_compare (dict_t *dict1, dict_t *dict2)
+ec_dict_compare(dict_t *dict1, dict_t *dict2)
 {
-        if (are_dicts_equal (dict1, dict2, ec_xattr_match, ec_value_ignore))
-                return 1;
-        return 0;
+    if (are_dicts_equal(dict1, dict2, ec_xattr_match, ec_value_ignore))
+        return 1;
+    return 0;
 }
 
-int32_t ec_dict_list(data_t ** list, int32_t * count, ec_cbk_data_t * cbk,
-                     int32_t which, char * key)
+static uint32_t
+ec_dict_list(data_t **list, ec_cbk_data_t *cbk, int32_t which, char *key,
+             gf_boolean_t global)
 {
-    ec_cbk_data_t * ans;
-    dict_t * dict;
-    int32_t i, max;
-
-    max = *count;
-    i = 0;
-    for (ans = cbk; ans != NULL; ans = ans->next) {
-        if (i >= max) {
-            gf_msg (cbk->fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_INVALID_DICT_NUMS,
-                    "Unexpected number of "
-                    "dictionaries");
+    ec_t *ec = cbk->fop->xl->private;
+    ec_cbk_data_t *ans = NULL;
+    dict_t *dict = NULL;
+    data_t *data;
+    uint32_t count;
+    int32_t i;
 
-            return -EINVAL;
+    for (i = 0; i < ec->nodes; i++) {
+        /* We initialize the list with EC_MISSING_DATA if we are
+         * returning a global list or the current subvolume belongs
+         * to the group of the accepted answer. Note that if some
+         * subvolume is known to be down before issuing the request,
+         * we won't have any answer from it, so we set here the
+         * appropriate default value. */
+        if (global || ((cbk->mask & (1ULL << i)) != 0)) {
+            list[i] = EC_MISSING_DATA;
+        } else {
+            list[i] = NULL;
         }
+    }
 
-        dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict;
-        list[i] = dict_get(dict, key);
-        if (list[i] != NULL) {
-            i++;
+    count = 0;
+    list_for_each_entry(ans, &cbk->fop->answer_list, answer_list)
+    {
+        if (global || ((cbk->mask & ans->mask) != 0)) {
+            dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict;
+            data = dict_get(dict, key);
+            if (data != NULL) {
+                list[ans->idx] = data;
+                count++;
+            }
         }
     }
 
-    *count = i;
-
-    return 0;
+    return count;
 }
 
-int32_t ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post,
-                          const char *fmt, va_list args)
+int32_t
+ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post,
+                  const char *fmt, va_list args)
 {
     char *tmp;
     int32_t len;
@@ -340,32 +334,29 @@ int32_t ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post,
     return 0;
 
 out:
-    gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-            EC_MSG_INVALID_FORMAT,
-            "Invalid concat format");
+    gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_FORMAT,
+           "Invalid concat format");
 
     GF_FREE(*str);
 
     return -EINVAL;
 }
 
-int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk,
-                            int32_t which, char * key, ...)
+static int32_t
+ec_dict_data_concat(ec_cbk_data_t *cbk, int32_t which, char *key, char *new_key,
+                    const char *def, gf_boolean_t global, const char *fmt, ...)
 {
-    data_t * data[cbk->count];
-    char * str = NULL, * pre = NULL, * sep, * post;
-    dict_t * dict;
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    char *str = NULL, *pre = NULL, *sep, *post;
+    dict_t *dict;
     va_list args;
-    int32_t i, num, len, prelen, postlen, seplen, tmp;
+    int32_t i, num, len, deflen, prelen, postlen, seplen, tmp;
     int32_t err;
 
-    num = cbk->count;
-    err = ec_dict_list(data, &num, cbk, which, key);
-    if (err != 0) {
-        return err;
-    }
+    ec_dict_list(data, cbk, which, key, global);
 
-    va_start(args, key);
+    va_start(args, fmt);
     err = ec_concat_prepare(cbk->fop->xl, &pre, &sep, &post, fmt, args);
     va_end(args);
 
@@ -377,9 +368,29 @@ int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk,
     seplen = strlen(sep);
     postlen = strlen(post);
 
-    len = prelen + (num - 1) * seplen + postlen + 1;
-    for (i = 0; i < num; i++) {
-        len += data[i]->len - 1;
+    deflen = 0;
+    if (def != NULL) {
+        deflen = strlen(def);
+    }
+
+    len = prelen + postlen + 1;
+    num = -1;
+    for (i = 0; i < ec->nodes; i++) {
+        if (data[i] == NULL) {
+            continue;
+        }
+        if (data[i] == EC_MISSING_DATA) {
+            if (def == NULL) {
+                continue;
+            }
+            len += deflen;
+        } else {
+            len += data[i]->len - 1;
+        }
+        if (num >= 0) {
+            len += seplen;
+        }
+        num++;
     }
 
     err = -ENOMEM;
@@ -391,18 +402,32 @@ int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk,
 
     memcpy(str, pre, prelen);
     len = prelen;
-    for (i = 0; i < num; i++) {
-        if (i > 0) {
+    for (i = 0; i < ec->nodes; i++) {
+        if (data[i] == NULL) {
+            continue;
+        }
+        if (data[i] == EC_MISSING_DATA) {
+            if (deflen == 0) {
+                continue;
+            }
+            tmp = deflen;
+            memcpy(str + len, def, tmp);
+        } else {
+            tmp = data[i]->len - 1;
+            memcpy(str + len, data[i]->data, tmp);
+        }
+        len += tmp;
+        if (i < num) {
             memcpy(str + len, sep, seplen);
             len += seplen;
         }
-        tmp = data[i]->len - 1;
-        memcpy(str + len, data[i]->data, tmp);
-        len += tmp;
     }
     memcpy(str + len, post, postlen + 1);
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
+    if (new_key) {
+        key = new_key;
+    }
     err = dict_set_dynstr(dict, key, str);
     if (err != 0) {
         goto out;
@@ -417,32 +442,28 @@ out:
     return err;
 }
 
-int32_t ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t *data[cbk->count];
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
     dict_t *dict, *lockinfo, *tmp = NULL;
     char *ptr = NULL;
-    int32_t i, num, len;
+    int32_t i, len;
     int32_t err;
 
-    num = cbk->count;
-    err = ec_dict_list(data, &num, cbk, which, key);
-    if (err != 0) {
-        return err;
-    }
+    ec_dict_list(data, cbk, which, key, _gf_false);
 
     lockinfo = dict_new();
     if (lockinfo == NULL) {
         return -ENOMEM;
     }
 
-    err = dict_unserialize(data[0]->data, data[0]->len, &lockinfo);
-    if (err != 0) {
-        goto out;
-    }
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
 
-    for (i = 1; i < num; i++)
-    {
         tmp = dict_new();
         if (tmp == NULL) {
             err = -ENOMEM;
@@ -464,22 +485,12 @@ int32_t ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
 
     tmp = NULL;
 
-    len = dict_serialized_length(lockinfo);
-    if (len < 0) {
-        err = len;
-
-        goto out;
-    }
-    ptr = GF_MALLOC(len, gf_common_mt_char);
-    if (ptr == NULL) {
-        err = -ENOMEM;
-
-        goto out;
-    }
-    err = dict_serialize(lockinfo, ptr);
+    err = dict_allocate_and_serialize(lockinfo, (char **)&ptr,
+                                      (unsigned int *)&len);
     if (err != 0) {
         goto out;
     }
+
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
     err = dict_set_dynptr(dict, key, ptr, len);
     if (err != 0) {
@@ -498,11 +509,12 @@ out:
     return err;
 }
 
-int32_t ec_dict_data_uuid(ec_cbk_data_t * cbk, int32_t which, char * key)
+int32_t
+ec_dict_data_uuid(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    ec_cbk_data_t * ans, * min;
-    dict_t * src, * dst;
-    data_t * data;
+    ec_cbk_data_t *ans, *min;
+    dict_t *src, *dst;
+    data_t *data;
 
     min = cbk;
     for (ans = cbk->next; ans != NULL; ans = ans->next) {
@@ -527,21 +539,82 @@ int32_t ec_dict_data_uuid(ec_cbk_data_t * cbk, int32_t which, char * key)
     return 0;
 }
 
-int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_iatt(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t * data[cbk->count];
-    dict_t * dict;
-    int32_t i, num, err;
-    uint32_t max, tmp;
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict;
+    struct iatt *stbuf, *tmp;
+    int32_t i, ret;
 
-    num = cbk->count;
-    err = ec_dict_list(data, &num, cbk, which, key);
-    if (err != 0) {
-        return err;
+    ec_dict_list(data, cbk, which, key, _gf_false);
+
+    stbuf = NULL;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
+        tmp = data_to_iatt(data[i], key);
+        if (tmp == NULL) {
+            ret = -EINVAL;
+            goto out;
+        }
+        if (stbuf == NULL) {
+            stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char);
+            if (stbuf == NULL) {
+                ret = -ENOMEM;
+                goto out;
+            }
+            *stbuf = *tmp;
+        } else {
+            if (!ec_iatt_combine(cbk->fop, stbuf, tmp, 1)) {
+                ret = -EINVAL;
+                goto out;
+            }
+        }
+    }
+
+    if ((stbuf != NULL) && (stbuf->ia_type == IA_IFREG)) {
+        ec_iatt_rebuild(ec, stbuf, 1, cbk->count);
+        /* TODO: not sure if an iatt could come in xdata from a fop that takes
+         *       no locks. */
+        if (!ec_get_inode_size(cbk->fop, cbk->fop->locks[0].lock->loc.inode,
+                               &stbuf->ia_size)) {
+            ret = -EINVAL;
+            goto out;
+        }
     }
 
-    max = data_to_uint32(data[0]);
-    for (i = 1; i < num; i++) {
+    dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
+    ret = dict_set_iatt(dict, key, stbuf, false);
+    if (ret >= 0) {
+        stbuf = NULL;
+    }
+
+out:
+    GF_FREE(stbuf);
+
+    return ret;
+}
+
+int32_t
+ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
+{
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict;
+    int32_t i;
+    uint32_t max, tmp;
+
+    ec_dict_list(data, cbk, which, key, _gf_false);
+
+    max = 0;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
+
         tmp = data_to_uint32(data[i]);
         if (max < tmp) {
             max = tmp;
@@ -552,21 +625,23 @@ int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
     return dict_set_uint32(dict, key, max);
 }
 
-int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t *data[cbk->count];
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
     dict_t *dict;
-    int32_t i, num, err;
+    int32_t i;
     uint64_t max, tmp;
 
-    num = cbk->count;
-    err = ec_dict_list(data, &num, cbk, which, key);
-    if (err != 0) {
-        return err;
-    }
+    ec_dict_list(data, cbk, which, key, _gf_false);
+
+    max = 0;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
 
-    max = data_to_uint64(data[0]);
-    for (i = 1; i < num; i++) {
         tmp = data_to_uint64(data[i]);
         if (max < tmp) {
             max = tmp;
@@ -577,24 +652,21 @@ int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
     return dict_set_uint64(dict, key, max);
 }
 
-int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t      *data[cbk->count];
-    dict_t      *dict             = NULL;
-    ec_t        *ec               = NULL;
-    int32_t      i                = 0;
-    int32_t      num              = 0;
-    int32_t      err              = 0;
-    quota_meta_t size             = {0, };
-    quota_meta_t max_size         = {0, };
-
-    num = cbk->count;
-    err = ec_dict_list(data, &num, cbk, which, key);
-    if (err != 0) {
-        return err;
-    }
-
-    if (num == 0) {
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict = NULL;
+    int32_t i = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
+
+    if (ec_dict_list(data, cbk, which, key, _gf_false) == 0) {
         return 0;
     }
 
@@ -603,44 +675,45 @@ int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
      * bricks and we can receive slightly different values. If that's the
      * case, we take the maximum of all received values.
      */
-    for (i = 0; i < num; i++) {
-        if (quota_data_to_meta (data[i], QUOTA_SIZE_KEY, &size) < 0) {
-                continue;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA) ||
+            (quota_data_to_meta(data[i], &size) < 0)) {
+            continue;
         }
 
         if (size.size > max_size.size)
-                max_size.size = size.size;
+            max_size.size = size.size;
         if (size.file_count > max_size.file_count)
-                max_size.file_count = size.file_count;
+            max_size.file_count = size.file_count;
         if (size.dir_count > max_size.dir_count)
-                max_size.dir_count = size.dir_count;
+            max_size.dir_count = size.dir_count;
     }
 
-    ec = cbk->fop->xl->private;
     max_size.size *= ec->fragments;
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    return quota_dict_set_meta (dict, key, &max_size, IA_IFDIR);
+    return quota_dict_set_meta(dict, key, &max_size, IA_IFDIR);
 }
 
-int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key)
+int32_t
+ec_dict_data_stime(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t * data[cbk->count];
-    dict_t * dict;
-    int32_t i, num, err;
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict;
+    int32_t i, err;
 
-    num = cbk->count;
-    err = ec_dict_list(data, &num, cbk, which, key);
-    if (err != 0) {
-        return err;
-    }
+    ec_dict_list(data, cbk, which, key, _gf_false);
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    for (i = 1; i < num; i++) {
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
         err = gf_get_max_stime(cbk->fop->xl, dict, key, data[i]);
         if (err != 0) {
-            gf_msg (cbk->fop->xl->name, GF_LOG_ERROR, -err,
-                    EC_MSG_STIME_COMBINE_FAIL, "STIME combination failed");
+            gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err,
+                   EC_MSG_STIME_COMBINE_FAIL, "STIME combination failed");
 
             return err;
         }
@@ -649,31 +722,32 @@ int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key)
     return 0;
 }
 
-int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
-                             void * arg)
+int32_t
+ec_dict_data_combine(dict_t *dict, char *key, data_t *value, void *arg)
 {
-    ec_dict_combine_t * data = arg;
+    ec_dict_combine_t *data = arg;
 
     if ((strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
-        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0))
-    {
-        return ec_dict_data_concat("(<EC:%s> { })", data->cbk, data->which,
-                                   key, data->cbk->fop->xl->name);
+        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) {
+        return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL,
+                                   _gf_false, _gf_false, "(<EC:%s> { })",
+                                   data->cbk->fop->xl->name);
     }
 
-    if (strncmp(key, GF_XATTR_CLRLK_CMD, strlen(GF_XATTR_CLRLK_CMD)) == 0)
-    {
-        return ec_dict_data_concat("{\n}", data->cbk, data->which, key);
+    if (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) {
+        return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL,
+                                   _gf_false, "{\n}");
     }
 
-    if (strncmp(key, GF_XATTR_LOCKINFO_KEY,
-                strlen(GF_XATTR_LOCKINFO_KEY)) == 0)
-    {
+    if (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) == 0) {
         return ec_dict_data_merge(data->cbk, data->which, key);
     }
 
-    if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)
-    {
+    if (strcmp(key, GET_LINK_COUNT) == 0) {
+        return ec_dict_data_max32(data->cbk, data->which, key);
+    }
+
+    if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) {
         return ec_dict_data_max32(data->cbk, data->which, key);
     }
     if ((strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
@@ -685,17 +759,22 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
         return ec_dict_data_quota(data->cbk, data->which, key);
     }
     /* Ignore all other quota attributes */
-    if (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) {
+    if (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) {
         return 0;
     }
 
-    if (XATTR_IS_NODE_UUID(key))
-    {
-        return ec_dict_data_uuid(data->cbk, data->which, key);
+    if (XATTR_IS_NODE_UUID(key)) {
+        if (data->cbk->fop->int32) {
+            /* List of node uuid is requested */
+            return ec_dict_data_concat(data->cbk, data->which, key,
+                                       GF_XATTR_LIST_NODE_UUIDS_KEY, UUID0_STR,
+                                       _gf_true, "{ }");
+        } else {
+            return ec_dict_data_uuid(data->cbk, data->which, key);
+        }
     }
 
-    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
-    {
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
         return ec_dict_data_stime(data->cbk, data->which, key);
     }
 
@@ -703,12 +782,17 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
         return ec_dict_data_max64(data->cbk, data->which, key);
     }
 
+    if (strcmp(key, GF_PRESTAT) == 0 || strcmp(key, GF_POSTSTAT) == 0) {
+        return ec_dict_data_iatt(data->cbk, data->which, key);
+    }
+
     return 0;
 }
 
-int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which)
+int32_t
+ec_dict_combine(ec_cbk_data_t *cbk, int32_t which)
 {
-    dict_t * dict;
+    dict_t *dict = NULL;
     ec_dict_combine_t data;
     int32_t err = 0;
 
@@ -719,9 +803,8 @@ int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which)
     if (dict != NULL) {
         err = dict_foreach(dict, ec_dict_data_combine, &data);
         if (err != 0) {
-            gf_msg (cbk->fop->xl->name, GF_LOG_ERROR, -err,
-                    EC_MSG_DICT_COMBINE_FAIL,
-                    "Dictionary combination failed");
+            gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err,
+                   EC_MSG_DICT_COMBINE_FAIL, "Dictionary combination failed");
 
             return err;
         }
@@ -730,47 +813,43 @@ int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which)
     return 0;
 }
 
-int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count,
-                          struct iovec * src_vector, int32_t src_count)
+int32_t
+ec_vector_compare(struct iovec *dst_vector, int32_t dst_count,
+                  struct iovec *src_vector, int32_t src_count)
 {
     int32_t dst_size = 0, src_size = 0;
 
-    if (dst_count > 0)
-    {
+    if (dst_count > 0) {
         dst_size = iov_length(dst_vector, dst_count);
     }
-    if (src_count > 0)
-    {
+    if (src_count > 0) {
         src_size = iov_length(src_vector, src_count);
     }
 
     return (dst_size == src_size);
 }
 
-int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src)
+int32_t
+ec_flock_compare(struct gf_flock *dst, struct gf_flock *src)
 {
-    if ((dst->l_type != src->l_type) ||
-        (dst->l_whence != src->l_whence) ||
-        (dst->l_start != src->l_start) ||
-        (dst->l_len != src->l_len) ||
+    if ((dst->l_type != src->l_type) || (dst->l_whence != src->l_whence) ||
+        (dst->l_start != src->l_start) || (dst->l_len != src->l_len) ||
         (dst->l_pid != src->l_pid) ||
-        !is_same_lkowner(&dst->l_owner, &src->l_owner))
-    {
+        !is_same_lkowner(&dst->l_owner, &src->l_owner)) {
         return 0;
     }
 
     return 1;
 }
 
-void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
+void
+ec_statvfs_combine(struct statvfs *dst, struct statvfs *src)
 {
-    if (dst->f_bsize < src->f_bsize)
-    {
+    if (dst->f_bsize < src->f_bsize) {
         dst->f_bsize = src->f_bsize;
     }
 
-    if (dst->f_frsize < src->f_frsize)
-    {
+    if (dst->f_frsize < src->f_frsize) {
         dst->f_blocks *= dst->f_frsize;
         dst->f_blocks /= src->f_frsize;
 
@@ -781,9 +860,7 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
         dst->f_bavail /= src->f_frsize;
 
         dst->f_frsize = src->f_frsize;
-    }
-    else if (dst->f_frsize > src->f_frsize)
-    {
+    } else if (dst->f_frsize > src->f_frsize) {
         src->f_blocks *= src->f_frsize;
         src->f_blocks /= dst->f_frsize;
 
@@ -793,90 +870,80 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
         src->f_bavail *= src->f_frsize;
         src->f_bavail /= dst->f_frsize;
     }
-    if (dst->f_blocks > src->f_blocks)
-    {
+    if (dst->f_blocks > src->f_blocks) {
         dst->f_blocks = src->f_blocks;
     }
-    if (dst->f_bfree > src->f_bfree)
-    {
+    if (dst->f_bfree > src->f_bfree) {
         dst->f_bfree = src->f_bfree;
     }
-    if (dst->f_bavail > src->f_bavail)
-    {
+    if (dst->f_bavail > src->f_bavail) {
         dst->f_bavail = src->f_bavail;
     }
 
-    if (dst->f_files < src->f_files)
-    {
+    if (dst->f_files < src->f_files) {
         dst->f_files = src->f_files;
     }
-    if (dst->f_ffree > src->f_ffree)
-    {
+    if (dst->f_ffree > src->f_ffree) {
         dst->f_ffree = src->f_ffree;
     }
-    if (dst->f_favail > src->f_favail)
-    {
+    if (dst->f_favail > src->f_favail) {
         dst->f_favail = src->f_favail;
     }
-    if (dst->f_namemax > src->f_namemax)
-    {
+    if (dst->f_namemax > src->f_namemax) {
         dst->f_namemax = src->f_namemax;
     }
 
-    if (dst->f_flag != src->f_flag)
-    {
-        gf_msg_debug (THIS->name, 0,
-                "Mismatching file system flags "
-                "(%lX, %lX)",
-                dst->f_flag, src->f_flag);
+    if (dst->f_flag != src->f_flag) {
+        gf_msg_debug(THIS->name, 0,
+                     "Mismatching file system flags "
+                     "(%lX, %lX)",
+                     dst->f_flag, src->f_flag);
     }
     dst->f_flag &= src->f_flag;
 }
 
-int32_t ec_combine_check(ec_cbk_data_t * dst, ec_cbk_data_t * src,
-                         ec_combine_f combine)
+int32_t
+ec_combine_check(ec_cbk_data_t *dst, ec_cbk_data_t *src, ec_combine_f combine)
 {
-    ec_fop_data_t * fop = dst->fop;
+    ec_fop_data_t *fop = dst->fop;
 
-    if (dst->op_ret != src->op_ret)
-    {
-        gf_msg_debug (fop->xl->name, 0, "Mismatching return code in "
-                                            "answers of '%s': %d <-> %d",
-               ec_fop_name(fop->id), dst->op_ret, src->op_ret);
+    if (dst->op_ret != src->op_ret) {
+        gf_msg_debug(fop->xl->name, 0,
+                     "Mismatching return code in "
+                     "answers of '%s': %d <-> %d",
+                     ec_fop_name(fop->id), dst->op_ret, src->op_ret);
 
         return 0;
     }
-    if (dst->op_ret < 0)
-    {
-        if (dst->op_errno != src->op_errno)
-        {
-            gf_msg_debug (fop->xl->name, 0, "Mismatching errno code in "
-                                                "answers of '%s': %d <-> %d",
-                   ec_fop_name(fop->id), dst->op_errno, src->op_errno);
+    if (dst->op_ret < 0) {
+        if (dst->op_errno != src->op_errno) {
+            gf_msg_debug(fop->xl->name, 0,
+                         "Mismatching errno code in "
+                         "answers of '%s': %d <-> %d",
+                         ec_fop_name(fop->id), dst->op_errno, src->op_errno);
 
             return 0;
         }
     }
 
-    if (!ec_dict_compare(dst->xdata, src->xdata))
-    {
-        gf_msg (fop->xl->name, GF_LOG_DEBUG, 0,
-                EC_MSG_XDATA_MISMATCH,
-                "Mismatching xdata in answers "
-                "of '%s'", ec_fop_name(fop->id));
+    if (!ec_dict_compare(dst->xdata, src->xdata)) {
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_XDATA_MISMATCH,
+               "Mismatching xdata in answers "
+               "of '%s'",
+               ec_fop_name(fop->id));
 
         return 0;
     }
 
-    if ((dst->op_ret >= 0) && (combine != NULL))
-    {
+    if ((dst->op_ret >= 0) && (combine != NULL)) {
         return combine(fop, dst, src);
     }
 
     return 1;
 }
 
-void ec_combine (ec_cbk_data_t *newcbk, ec_combine_f combine)
+void
+ec_combine(ec_cbk_data_t *newcbk, ec_combine_f combine)
 {
     ec_fop_data_t *fop = newcbk->fop;
     ec_cbk_data_t *cbk = NULL, *tmp = NULL;
@@ -891,17 +958,14 @@ void ec_combine (ec_cbk_data_t *newcbk, ec_combine_f combine)
     item = fop->cbk_list.prev;
     list_for_each_entry(cbk, &fop->cbk_list, list)
     {
-        if (ec_combine_check(newcbk, cbk, combine))
-        {
+        if (ec_combine_check(newcbk, cbk, combine)) {
             newcbk->count += cbk->count;
             newcbk->mask |= cbk->mask;
 
             item = cbk->list.prev;
-            while (item != &fop->cbk_list)
-            {
+            while (item != &fop->cbk_list) {
                 tmp = list_entry(item, ec_cbk_data_t, list);
-                if (tmp->count >= newcbk->count)
-                {
+                if (tmp->count >= newcbk->count) {
                     break;
                 }
                 item = item->prev;
diff --git a/xlators/cluster/ec/src/ec-combine.h b/xlators/cluster/ec/src/ec-combine.h
index 19a42ded706..1010cc3be26 100644
--- a/xlators/cluster/ec/src/ec-combine.h
+++ b/xlators/cluster/ec/src/ec-combine.h
@@ -11,28 +11,34 @@
 #ifndef __EC_COMBINE_H__
 #define __EC_COMBINE_H__
 
-#define EC_COMBINE_DICT  0
+#define EC_COMBINE_DICT 0
 #define EC_COMBINE_XDATA 1
 
-typedef int32_t (* ec_combine_f)(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                                 ec_cbk_data_t * src);
+typedef int32_t (*ec_combine_f)(ec_fop_data_t *fop, ec_cbk_data_t *dst,
+                                ec_cbk_data_t *src);
 
-void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
-                     int32_t answers);
+void
+ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers);
 
-int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
-                        int32_t count);
-int32_t ec_dict_compare(dict_t * dict1, dict_t * dict2);
-int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count,
-                          struct iovec * src_vector, int32_t src_count);
-int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src);
-void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src);
+int32_t
+ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
+                int32_t count);
+int32_t
+ec_dict_compare(dict_t *dict1, dict_t *dict2);
+int32_t
+ec_vector_compare(struct iovec *dst_vector, int32_t dst_count,
+                  struct iovec *src_vector, int32_t src_count);
+int32_t
+ec_flock_compare(struct gf_flock *dst, struct gf_flock *src);
+void
+ec_statvfs_combine(struct statvfs *dst, struct statvfs *src);
 
-int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which);
+int32_t
+ec_dict_combine(ec_cbk_data_t *cbk, int32_t which);
 
-void ec_combine(ec_cbk_data_t * cbk, ec_combine_f combine);
+void
+ec_combine(ec_cbk_data_t *cbk, ec_combine_f combine);
 
 int32_t
-ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                  ec_cbk_data_t *src);
+ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src);
 #endif /* __EC_COMBINE_H__ */
diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c
index 009fad2f896..b955efd8c2d 100644
--- a/xlators/cluster/ec/src/ec-common.c
+++ b/xlators/cluster/ec/src/ec-common.c
@@ -8,11 +8,11 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "byte-order.h"
-#include "hashfn.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/hashfn.h>
 
 #include "ec-mem-types.h"
-#include "ec-data.h"
+#include "ec-types.h"
 #include "ec-helpers.h"
 #include "ec-combine.h"
 #include "ec-common.h"
@@ -21,121 +21,332 @@
 #include "ec.h"
 #include "ec-messages.h"
 
+#define EC_INVALID_INDEX UINT32_MAX
+
+void
+ec_update_fd_status(fd_t *fd, xlator_t *xl, int idx, int32_t ret_status)
+{
+    ec_fd_t *fd_ctx;
+
+    if (fd == NULL)
+        return;
+
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __ec_fd_get(fd, xl);
+        if (fd_ctx) {
+            if (ret_status >= 0)
+                fd_ctx->fd_status[idx] = EC_FD_OPENED;
+            else
+                fd_ctx->fd_status[idx] = EC_FD_NOT_OPENED;
+        }
+    }
+    UNLOCK(&fd->lock);
+}
+
+static uintptr_t
+ec_fd_ctx_need_open(fd_t *fd, xlator_t *this, uintptr_t mask)
+{
+    int i = 0;
+    int count = 0;
+    ec_t *ec = NULL;
+    ec_fd_t *fd_ctx = NULL;
+    uintptr_t need_open = 0;
+
+    ec = this->private;
+
+    fd_ctx = ec_fd_get(fd, this);
+    if (!fd_ctx)
+        return count;
+
+    LOCK(&fd->lock);
+    {
+        for (i = 0; i < ec->nodes; i++) {
+            if ((fd_ctx->fd_status[i] == EC_FD_NOT_OPENED) &&
+                ((ec->xl_up & (1 << i)) != 0) && ((mask & (1 << i)) != 0)) {
+                fd_ctx->fd_status[i] = EC_FD_OPENING;
+                need_open |= (1 << i);
+                count++;
+            }
+        }
+    }
+    UNLOCK(&fd->lock);
+
+    /* If fd needs to open on minimum number of nodes
+     * then ignore fixing the fd as it has been
+     * requested from heal operation.
+     */
+    if (count >= ec->fragments) {
+        need_open = 0;
+    }
+
+    return need_open;
+}
+
+static gf_boolean_t
+ec_is_fd_fixable(fd_t *fd)
+{
+    if (!fd || !fd->inode)
+        return _gf_false;
+    else if (fd_is_anonymous(fd))
+        return _gf_false;
+    else if (gf_uuid_is_null(fd->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
+}
+
+static void
+ec_fix_open(ec_fop_data_t *fop, uintptr_t mask)
+{
+    uintptr_t need_open = 0;
+    int ret = 0;
+    int32_t flags = 0;
+    loc_t loc = {
+        0,
+    };
+
+    if (!ec_is_fd_fixable(fop->fd))
+        goto out;
+
+    /* Evaluate how many remote fd's to be opened */
+    need_open = ec_fd_ctx_need_open(fop->fd, fop->xl, mask);
+    if (need_open == 0) {
+        goto out;
+    }
+
+    loc.inode = inode_ref(fop->fd->inode);
+    gf_uuid_copy(loc.gfid, fop->fd->inode->gfid);
+    ret = loc_path(&loc, NULL);
+    if (ret < 0) {
+        goto out;
+    }
+
+    flags = fop->fd->flags & (~(O_TRUNC | O_APPEND | O_CREAT | O_EXCL));
+    if (IA_IFDIR == fop->fd->inode->ia_type) {
+        ec_opendir(fop->frame, fop->xl, need_open,
+                   EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL,
+                   &fop->loc[0], fop->fd, NULL);
+    } else {
+        ec_open(fop->frame, fop->xl, need_open,
+                EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL, &loc,
+                flags, fop->fd, NULL);
+    }
+
+out:
+    loc_wipe(&loc);
+}
+
+static off_t
+ec_range_end_get(off_t fl_start, uint64_t fl_size)
+{
+    if (fl_size > 0) {
+        if (fl_size >= EC_RANGE_FULL) {
+            /* Infinity */
+            fl_start = LLONG_MAX;
+        } else {
+            fl_start += fl_size - 1;
+            if (fl_start < 0) {
+                /* Overflow */
+                fl_start = LLONG_MAX;
+            }
+        }
+    }
+
+    return fl_start;
+}
+
+static gf_boolean_t
+ec_is_range_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2)
+{
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
+}
+
+static gf_boolean_t
+ec_lock_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2)
+{
+    ec_t *ec = l1->fop->xl->private;
+
+    /* Fops like access/stat won't have to worry what the other fops are
+     * modifying as the fop is wound only to one brick. So it can be
+     * executed in parallel*/
+    if (l1->fop->minimum == EC_MINIMUM_ONE ||
+        l2->fop->minimum == EC_MINIMUM_ONE)
+        return _gf_false;
+
+    if ((l1->fop->flags & EC_FLAG_LOCK_SHARED) &&
+        (l2->fop->flags & EC_FLAG_LOCK_SHARED))
+        return _gf_false;
+
+    if (!ec->parallel_writes) {
+        return _gf_true;
+    }
+
+    return ec_is_range_conflict(l1, l2);
+}
+
 uint32_t
-ec_select_first_by_read_policy (ec_t *ec, ec_fop_data_t *fop)
-{
-        if (ec->read_policy == EC_ROUND_ROBIN) {
-                return ec->idx;
-        } else if (ec->read_policy == EC_GFID_HASH) {
-                if (fop->use_fd) {
-                        return SuperFastHash((char *)fop->fd->inode->gfid,
-                                   sizeof(fop->fd->inode->gfid)) % ec->nodes;
-                } else {
-                        if (gf_uuid_is_null (fop->loc[0].gfid))
-                                loc_gfid (&fop->loc[0], fop->loc[0].gfid);
-                        return SuperFastHash((char *)fop->loc[0].gfid,
-                                   sizeof(fop->loc[0].gfid)) % ec->nodes;
-                }
+ec_select_first_by_read_policy(ec_t *ec, ec_fop_data_t *fop)
+{
+    if (ec->read_policy == EC_ROUND_ROBIN) {
+        return ec->idx;
+    } else if (ec->read_policy == EC_GFID_HASH) {
+        if (fop->use_fd) {
+            return SuperFastHash((char *)fop->fd->inode->gfid,
+                                 sizeof(fop->fd->inode->gfid)) %
+                   ec->nodes;
+        } else {
+            if (gf_uuid_is_null(fop->loc[0].gfid))
+                loc_gfid(&fop->loc[0], fop->loc[0].gfid);
+            return SuperFastHash((char *)fop->loc[0].gfid,
+                                 sizeof(fop->loc[0].gfid)) %
+                   ec->nodes;
         }
-        return 0;
+    }
+    return 0;
 }
 
-int32_t ec_child_valid(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+static gf_boolean_t
+ec_child_valid(ec_t *ec, ec_fop_data_t *fop, uint32_t idx)
 {
     return (idx < ec->nodes) && (((fop->remaining >> idx) & 1) == 1);
 }
 
-int32_t ec_child_next(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+static uint32_t
+ec_child_next(ec_t *ec, ec_fop_data_t *fop, uint32_t idx)
 {
-    while (!ec_child_valid(ec, fop, idx))
-    {
-        if (++idx >= ec->nodes)
-        {
+    while (!ec_child_valid(ec, fop, idx)) {
+        if (++idx >= ec->nodes) {
             idx = 0;
         }
-        if (idx == fop->first)
-        {
-            return -1;
+        if (idx == fop->first) {
+            return EC_INVALID_INDEX;
         }
     }
 
     return idx;
 }
 
-int32_t ec_heal_report(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                       uintptr_t good, uintptr_t bad, dict_t * xdata)
+int32_t
+ec_heal_report(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, uintptr_t mask, uintptr_t good,
+               uintptr_t bad, uint32_t pending, dict_t *xdata)
 {
     if (op_ret < 0) {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_HEAL_FAIL, "Heal failed");
+        gf_msg(this->name, GF_LOG_DEBUG, op_errno, EC_MSG_HEAL_FAIL,
+               "Heal failed");
     } else {
         if ((mask & ~good) != 0) {
-            gf_msg (this->name, GF_LOG_INFO, 0,
-                    EC_MSG_HEAL_SUCCESS, "Heal succeeded on %d/%d "
-                    "subvolumes",
-                    ec_bits_count(mask & ~(good | bad)),
-                    ec_bits_count(mask & ~good));
+            gf_msg(this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_SUCCESS,
+                   "Heal succeeded on %d/%d "
+                   "subvolumes",
+                   gf_bits_count(mask & ~(good | bad)),
+                   gf_bits_count(mask & ~good));
         }
     }
 
     return 0;
 }
 
-int32_t ec_fop_needs_heal(ec_fop_data_t *fop)
+static uintptr_t
+ec_fop_needs_name_heal(ec_fop_data_t *fop)
+{
+    ec_t *ec = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    ec_cbk_data_t *enoent_cbk = NULL;
+
+    ec = fop->xl->private;
+    if (fop->id != GF_FOP_LOOKUP)
+        return 0;
+
+    if (!fop->loc[0].name || strlen(fop->loc[0].name) == 0)
+        return 0;
+
+    list_for_each_entry(cbk, &fop->cbk_list, list)
+    {
+        if (cbk->op_ret < 0 && cbk->op_errno == ENOENT) {
+            enoent_cbk = cbk;
+            break;
+        }
+    }
+
+    if (!enoent_cbk)
+        return 0;
+
+    return ec->xl_up & ~enoent_cbk->mask;
+}
+
+int32_t
+ec_fop_needs_heal(ec_fop_data_t *fop)
 {
     ec_t *ec = fop->xl->private;
 
+    if (fop->lock_count == 0) {
+        /*
+         * if fop->lock_count is zero that means it saw version mismatch
+         * without any locks so it can't be trusted. If we launch a heal
+         * based on this it will lead to INODELKs which will affect I/O
+         * performance. Considering self-heal-daemon and operations on
+         * the inode from client which take locks can still trigger the
+         * heal we can choose to not attempt a heal when fop->lock_count
+         * is zero.
+         */
+        return 0;
+    }
     return (ec->xl_up & ~(fop->remaining | fop->good)) != 0;
 }
 
-void ec_check_status(ec_fop_data_t * fop)
+void
+ec_check_status(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     int32_t partial = 0;
+    char str1[32], str2[32], str3[32], str4[32], str5[32];
 
-    if (!ec_fop_needs_heal(fop)) {
+    if (!ec_fop_needs_name_heal(fop) && !ec_fop_needs_heal(fop)) {
         return;
     }
 
-    if (fop->answer->op_ret >= 0) {
-        if ((fop->id == GF_FOP_LOOKUP) ||
-            (fop->id == GF_FOP_STAT) || (fop->id == GF_FOP_FSTAT)) {
+    if (fop->answer && fop->answer->op_ret >= 0) {
+        if ((fop->id == GF_FOP_LOOKUP) || (fop->id == GF_FOP_STAT) ||
+            (fop->id == GF_FOP_FSTAT)) {
             partial = fop->answer->iatt[0].ia_type == IA_IFDIR;
         } else if (fop->id == GF_FOP_OPENDIR) {
             partial = 1;
         }
     }
 
-    gf_msg (fop->xl->name, GF_LOG_WARNING, 0,
-            EC_MSG_OP_FAIL_ON_SUBVOLS,
-            "Operation failed on some "
-            "subvolumes (up=%lX, mask=%lX, "
-            "remaining=%lX, good=%lX, bad=%lX)",
-            ec->xl_up, fop->mask, fop->remaining, fop->good,
-            ec->xl_up & ~(fop->remaining | fop->good));
-
-    if (fop->use_fd)
-    {
+    gf_msg(
+        fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_FAIL_ON_SUBVOLS,
+        "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, "
+        "remaining=%s, good=%s, bad=%s,"
+        "(Least significant bit represents first client/brick of subvol), %s)",
+        gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes,
+        ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+        ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
+        ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes),
+        ec_bin(str4, sizeof(str4), fop->good, ec->nodes),
+        ec_bin(str5, sizeof(str5), ec->xl_up & ~(fop->remaining | fop->good),
+               ec->nodes),
+        ec_msg_str(fop));
+    if (fop->use_fd) {
         if (fop->fd != NULL) {
             ec_fheal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                      fop->fd, partial, NULL);
         }
-    }
-    else
-    {
+    } else {
         ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                 &fop->loc[0], partial, NULL);
 
-        if (fop->loc[1].inode != NULL)
-        {
+        if (fop->loc[1].inode != NULL) {
             ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                     &fop->loc[1], partial, NULL);
         }
     }
 }
 
-void ec_update_good(ec_fop_data_t *fop, uintptr_t good)
+void
+ec_update_good(ec_fop_data_t *fop, uintptr_t good)
 {
     fop->good = good;
 
@@ -146,7 +357,8 @@ void ec_update_good(ec_fop_data_t *fop, uintptr_t good)
     }
 }
 
-void ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop)
+void
+ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop)
 {
     /* Fops that are executed only on one brick do not have enough information
      * to update the global mask of good bricks. */
@@ -162,15 +374,16 @@ void ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop)
     lock->good_mask &= fop->good | fop->remaining;
 }
 
-void __ec_fop_set_error(ec_fop_data_t * fop, int32_t error)
+void
+__ec_fop_set_error(ec_fop_data_t *fop, int32_t error)
 {
-    if ((error != 0) && (fop->error == 0))
-    {
+    if ((error != 0) && (fop->error == 0)) {
         fop->error = error;
     }
 }
 
-void ec_fop_set_error(ec_fop_data_t * fop, int32_t error)
+void
+ec_fop_set_error(ec_fop_data_t *fop, int32_t error)
 {
     LOCK(&fop->lock);
 
@@ -222,18 +435,20 @@ ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro)
     return cbk;
 }
 
-void ec_sleep(ec_fop_data_t *fop)
+void
+ec_sleep(ec_fop_data_t *fop)
 {
     LOCK(&fop->lock);
 
-    GF_ASSERT (fop->refs > 0);
+    GF_ASSERT(fop->refs > 0);
     fop->refs++;
     fop->jobs++;
 
     UNLOCK(&fop->lock);
 }
 
-int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
+int32_t
+ec_check_complete(ec_fop_data_t *fop, ec_resume_f resume)
 {
     int32_t error = -1;
 
@@ -241,14 +456,11 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
 
     GF_ASSERT(fop->resume == NULL);
 
-    if (--fop->jobs != 0)
-    {
+    if (--fop->jobs != 0) {
         ec_trace("WAIT", fop, "resume=%p", resume);
 
         fop->resume = resume;
-    }
-    else
-    {
+    } else {
         error = fop->error;
         fop->error = 0;
     }
@@ -258,7 +470,8 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
     return error;
 }
 
-void ec_resume(ec_fop_data_t * fop, int32_t error)
+void
+ec_resume(ec_fop_data_t *fop, int32_t error)
 {
     ec_resume_f resume = NULL;
 
@@ -266,16 +479,13 @@ void ec_resume(ec_fop_data_t * fop, int32_t error)
 
     __ec_fop_set_error(fop, error);
 
-    if (--fop->jobs == 0)
-    {
+    if (--fop->jobs == 0) {
         resume = fop->resume;
         fop->resume = NULL;
-        if (resume != NULL)
-        {
+        if (resume != NULL) {
             ec_trace("RESUME", fop, "error=%d", error);
 
-            if (fop->error != 0)
-            {
+            if (fop->error != 0) {
                 error = fop->error;
             }
             fop->error = 0;
@@ -284,21 +494,24 @@ void ec_resume(ec_fop_data_t * fop, int32_t error)
 
     UNLOCK(&fop->lock);
 
-    if (resume != NULL)
-    {
+    if (resume != NULL) {
         resume(fop, error);
     }
 
     ec_fop_data_release(fop);
 }
 
-void ec_resume_parent(ec_fop_data_t * fop, int32_t error)
+void
+ec_resume_parent(ec_fop_data_t *fop)
 {
-    ec_fop_data_t * parent;
+    ec_fop_data_t *parent;
+    int32_t error = 0;
 
     parent = fop->parent;
-    if (parent != NULL)
-    {
+    if (parent != NULL) {
+        if ((fop->fop_flags & EC_FOP_NO_PROPAGATE_ERROR) == 0) {
+            error = fop->error;
+        }
         ec_trace("RESUME_PARENT", fop, "error=%u", error);
         fop->parent = NULL;
         ec_resume(parent, error);
@@ -306,22 +519,23 @@ void ec_resume_parent(ec_fop_data_t * fop, int32_t error)
 }
 
 gf_boolean_t
-ec_is_recoverable_error (int32_t op_errno)
+ec_is_recoverable_error(int32_t op_errno)
 {
-        switch (op_errno) {
+    switch (op_errno) {
         case ENOTCONN:
         case ESTALE:
         case ENOENT:
-        case EBADFD:/*Opened fd but brick is disconnected*/
-        case EIO:/*Backend-fs crash like XFS/ext4 etc*/
-                return _gf_true;
-        }
-        return _gf_false;
+        case EBADFD: /*Opened fd but brick is disconnected*/
+        case EIO:    /*Backend-fs crash like XFS/ext4 etc*/
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-void ec_complete(ec_fop_data_t * fop)
+void
+ec_complete(ec_fop_data_t *fop)
 {
-    ec_cbk_data_t * cbk = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t resume = 0, update = 0;
     int healing_count = 0;
 
@@ -333,9 +547,9 @@ void ec_complete(ec_fop_data_t * fop)
         if (fop->answer == NULL) {
             if (!list_empty(&fop->cbk_list)) {
                 cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list);
-                healing_count = ec_bits_count (cbk->mask & fop->healing);
-                    /* fop shouldn't be treated as success if it is not
-                     * successful on at least fop->minimum good copies*/
+                healing_count = gf_bits_count(cbk->mask & fop->healing);
+                /* fop shouldn't be treated as success if it is not
+                 * successful on at least fop->minimum good copies*/
                 if ((cbk->count - healing_count) >= fop->minimum) {
                     fop->answer = cbk;
 
@@ -357,8 +571,7 @@ void ec_complete(ec_fop_data_t * fop)
         ec_update_good(fop, cbk->mask);
     }
 
-    if (resume)
-    {
+    if (resume) {
         ec_resume(fop, 0);
     }
 
@@ -368,36 +581,95 @@ void ec_complete(ec_fop_data_t * fop)
 /* There could be already granted locks sitting on the bricks, unlock for which
  * must be wound at all costs*/
 static gf_boolean_t
-ec_must_wind (ec_fop_data_t *fop)
+ec_must_wind(ec_fop_data_t *fop)
 {
-        if ((fop->id == GF_FOP_INODELK) || (fop->id == GF_FOP_FINODELK) ||
-            (fop->id == GF_FOP_LK)) {
-                if (fop->flock.l_type == F_UNLCK)
-                        return _gf_true;
-        } else if ((fop->id == GF_FOP_ENTRYLK) ||
-                   (fop->id == GF_FOP_FENTRYLK)) {
-                if (fop->entrylk_cmd == ENTRYLK_UNLOCK)
-                        return _gf_true;
-        }
+    if ((fop->id == GF_FOP_INODELK) || (fop->id == GF_FOP_FINODELK) ||
+        (fop->id == GF_FOP_LK)) {
+        if (fop->flock.l_type == F_UNLCK)
+            return _gf_true;
+    } else if ((fop->id == GF_FOP_ENTRYLK) || (fop->id == GF_FOP_FENTRYLK)) {
+        if (fop->entrylk_cmd == ENTRYLK_UNLOCK)
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-ec_internal_op (ec_fop_data_t *fop)
-{
-        if (ec_must_wind (fop))
-                return _gf_true;
-        if (fop->id == GF_FOP_XATTROP)
-                return _gf_true;
-        if (fop->id == GF_FOP_FXATTROP)
-                return _gf_true;
-        return _gf_false;
+ec_internal_op(ec_fop_data_t *fop)
+{
+    if (ec_must_wind(fop))
+        return _gf_true;
+    if (fop->id == GF_FOP_XATTROP)
+        return _gf_true;
+    if (fop->id == GF_FOP_FXATTROP)
+        return _gf_true;
+    if (fop->id == GF_FOP_OPEN)
+        return _gf_true;
+    return _gf_false;
+}
+
+char *
+ec_msg_str(ec_fop_data_t *fop)
+{
+    loc_t *loc1 = NULL;
+    loc_t *loc2 = NULL;
+    char gfid1[64] = {0};
+    char gfid2[64] = {0};
+    ec_fop_data_t *parent = fop->parent;
+
+    if (fop->errstr)
+        return fop->errstr;
+    if (!fop->use_fd) {
+        loc1 = &fop->loc[0];
+        loc2 = &fop->loc[1];
+
+        if (fop->id == GF_FOP_RENAME) {
+            gf_asprintf(&fop->errstr,
+                        "FOP : '%s' failed on '%s' and '%s' with gfids "
+                        "%s and %s respectively. Parent FOP: %s",
+                        ec_fop_name(fop->id), loc1->path, loc2->path,
+                        uuid_utoa_r(loc1->gfid, gfid1),
+                        uuid_utoa_r(loc2->gfid, gfid2),
+                        parent ? ec_fop_name(parent->id) : "No Parent");
+        } else {
+            gf_asprintf(
+                &fop->errstr,
+                "FOP : '%s' failed on '%s' with gfid %s. Parent FOP: %s",
+                ec_fop_name(fop->id), loc1->path,
+                uuid_utoa_r(loc1->gfid, gfid1),
+                parent ? ec_fop_name(parent->id) : "No Parent");
+        }
+    } else {
+        gf_asprintf(
+            &fop->errstr, "FOP : '%s' failed on gfid %s. Parent FOP: %s",
+            ec_fop_name(fop->id), uuid_utoa_r(fop->fd->inode->gfid, gfid1),
+            parent ? ec_fop_name(parent->id) : "No Parent");
+    }
+    return fop->errstr;
 }
 
-int32_t ec_child_select(ec_fop_data_t * fop)
+static void
+ec_log_insufficient_vol(ec_fop_data_t *fop, int32_t have, uint32_t need,
+                        int32_t loglevel)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
+    char str1[32], str2[32], str3[32];
+
+    gf_msg(ec->xl->name, loglevel, 0, EC_MSG_CHILDS_INSUFFICIENT,
+           "Insufficient available children for this request: "
+           "Have : %d, Need : %u : Child UP : %s "
+           "Mask: %s, Healing : %s : %s ",
+           have, need, ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
+           ec_bin(str3, sizeof(str3), fop->healing, ec->nodes),
+           ec_msg_str(fop));
+}
+
+static int32_t
+ec_child_select(ec_fop_data_t *fop)
+{
+    ec_t *ec = fop->xl->private;
     int32_t first = 0, num = 0;
 
     ec_fop_cleanup(fop);
@@ -406,27 +678,25 @@ int32_t ec_child_select(ec_fop_data_t * fop)
     /* Wind the fop on same subvols as parent for any internal extra fops like
      * head/tail read in case of writev fop. Unlocks shouldn't do this because
      * unlock should go on all subvols where lock is performed*/
-    if (fop->parent && !ec_internal_op (fop)) {
-            fop->mask &= (fop->parent->mask & ~fop->parent->healing);
+    if (fop->parent && !ec_internal_op(fop)) {
+        fop->mask &= (fop->parent->mask & ~fop->parent->healing);
+        if (ec_is_data_fop(fop->id)) {
+            fop->healing |= fop->parent->healing;
+        }
     }
 
-    if ((fop->mask & ~ec->xl_up) != 0)
-    {
-        gf_msg (fop->xl->name, GF_LOG_WARNING, 0,
-                EC_MSG_OP_EXEC_UNAVAIL,
-                "Executing operation with "
-                "some subvolumes unavailable "
-                "(%lX)", fop->mask & ~ec->xl_up);
-
+    if ((fop->mask & ~ec->xl_up) != 0) {
+        gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_EXEC_UNAVAIL,
+               "Executing operation with "
+               "some subvolumes unavailable. (%" PRIXPTR "). %s ",
+               fop->mask & ~ec->xl_up, ec_msg_str(fop));
         fop->mask &= ec->xl_up;
     }
 
-    switch (fop->minimum)
-    {
+    switch (fop->minimum) {
         case EC_MINIMUM_ALL:
-            fop->minimum = ec_bits_count(fop->mask);
-            if (fop->minimum >= ec->fragments)
-            {
+            fop->minimum = gf_bits_count(fop->mask);
+            if (fop->minimum >= ec->fragments) {
                 break;
             }
         case EC_MINIMUM_MIN:
@@ -437,13 +707,14 @@ int32_t ec_child_select(ec_fop_data_t * fop)
     }
 
     if (ec->read_policy == EC_ROUND_ROBIN) {
-            first = ec->idx;
-            if (++first >= ec->nodes) {
-                first = 0;
-            }
-            ec->idx = first;
+        first = ec->idx;
+        if (++first >= ec->nodes) {
+            first = 0;
+        }
+        ec->idx = first;
     }
 
+    num = gf_bits_count(fop->mask);
     /*Unconditionally wind on healing subvolumes*/
     fop->mask |= fop->healing;
     fop->remaining = fop->mask;
@@ -451,32 +722,35 @@ int32_t ec_child_select(ec_fop_data_t * fop)
 
     ec_trace("SELECT", fop, "");
 
-    num = ec_bits_count(fop->mask);
-    if ((num < fop->minimum) && (num < ec->fragments))
-    {
-        gf_msg (ec->xl->name, GF_LOG_ERROR, 0,
-                EC_MSG_CHILDS_INSUFFICIENT,
-                "Insufficient available children "
-                "for this request (have %d, need "
-                "%d)", num, fop->minimum);
-
+    if ((num < fop->minimum) && (num < ec->fragments)) {
+        ec_log_insufficient_vol(fop, num, fop->minimum, GF_LOG_ERROR);
         return 0;
     }
 
-    ec_sleep(fop);
+    if (!fop->parent && fop->lock_count &&
+        (fop->locks[0].update[EC_DATA_TXN] ||
+         fop->locks[0].update[EC_METADATA_TXN])) {
+        if (ec->quorum_count && (num < ec->quorum_count)) {
+            ec_log_insufficient_vol(fop, num, ec->quorum_count, GF_LOG_ERROR);
+            return 0;
+        }
+    }
 
     return 1;
 }
 
-int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx)
+void
+ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx)
 {
-    ec_t * ec = fop->xl->private;
+    uint32_t i = EC_INVALID_INDEX;
+    ec_t *ec = fop->xl->private;
 
     LOCK(&fop->lock);
 
-    idx = ec_child_next(ec, fop, idx);
-    if (idx >= 0)
-    {
+    i = ec_child_next(ec, fop, idx);
+    if (i < EC_MAX_NODES) {
+        idx = i;
+
         fop->remaining ^= 1ULL << idx;
 
         ec_trace("EXECUTE", fop, "idx=%d", idx);
@@ -487,20 +761,18 @@ int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx)
 
     UNLOCK(&fop->lock);
 
-    if (idx >= 0)
-    {
+    if (i < EC_MAX_NODES) {
         fop->wind(ec, fop, idx);
     }
-
-    return idx;
 }
 
-void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
+void
+ec_dispatch_mask(ec_fop_data_t *fop, uintptr_t mask)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     int32_t count, idx;
 
-    count = ec_bits_count(mask);
+    count = gf_bits_count(mask);
 
     LOCK(&fop->lock);
 
@@ -514,10 +786,8 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
     UNLOCK(&fop->lock);
 
     idx = 0;
-    while (mask != 0)
-    {
-        if ((mask & 1) != 0)
-        {
+    while (mask != 0) {
+        if ((mask & 1) != 0) {
             fop->wind(ec, fop, idx);
         }
         idx++;
@@ -525,27 +795,29 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
     }
 }
 
-void ec_dispatch_start(ec_fop_data_t * fop)
+void
+ec_dispatch_start(ec_fop_data_t *fop)
 {
     fop->answer = NULL;
     fop->good = 0;
 
     INIT_LIST_HEAD(&fop->cbk_list);
 
-    if (fop->lock_count > 0)
-    {
+    if (fop->lock_count > 0) {
         ec_owner_copy(fop->frame, &fop->req_frame->root->lk_owner);
     }
 }
 
-void ec_dispatch_one(ec_fop_data_t * fop)
+void
+ec_dispatch_one(ec_fop_data_t *fop)
 {
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
         fop->expected = 1;
-        fop->first = ec_select_first_by_read_policy (fop->xl->private, fop);
+        fop->first = ec_select_first_by_read_policy(fop->xl->private, fop);
 
         ec_dispatch_next(fop, fop->first);
     }
@@ -561,8 +833,8 @@ ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk)
         *cbk = tmp;
     }
     if ((tmp != NULL) && (tmp->op_ret < 0) &&
-        ec_is_recoverable_error (tmp->op_errno)) {
-        GF_ASSERT (fop->mask & (1ULL << tmp->idx));
+        ec_is_recoverable_error(tmp->op_errno)) {
+        GF_ASSERT(fop->mask & (1ULL << tmp->idx));
         fop->mask ^= (1ULL << tmp->idx);
         if (fop->mask) {
             return _gf_true;
@@ -572,13 +844,15 @@ ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk)
     return _gf_false;
 }
 
-void ec_dispatch_inc(ec_fop_data_t * fop)
+void
+ec_dispatch_inc(ec_fop_data_t *fop)
 {
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
-        fop->expected = ec_bits_count(fop->remaining);
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
+        fop->expected = gf_bits_count(fop->remaining);
         fop->first = 0;
 
         ec_dispatch_next(fop, 0);
@@ -586,55 +860,76 @@ void ec_dispatch_inc(ec_fop_data_t * fop)
 }
 
 void
-ec_dispatch_all (ec_fop_data_t *fop)
+ec_dispatch_all(ec_fop_data_t *fop)
 {
-        ec_dispatch_start(fop);
+    ec_dispatch_start(fop);
 
-        if (ec_child_select(fop)) {
-                fop->expected = ec_bits_count(fop->remaining);
-                fop->first = 0;
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
 
-                ec_dispatch_mask(fop, fop->remaining);
-        }
+        fop->expected = gf_bits_count(fop->remaining);
+        fop->first = 0;
+
+        ec_dispatch_mask(fop, fop->remaining);
+    }
 }
 
-void ec_dispatch_min(ec_fop_data_t * fop)
+void
+ec_dispatch_min(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     uintptr_t mask;
-    int32_t idx, count;
+    uint32_t idx;
+    int32_t count;
 
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
         fop->expected = count = ec->fragments;
-        fop->first = ec_select_first_by_read_policy (fop->xl->private, fop);
+        fop->first = ec_select_first_by_read_policy(fop->xl->private, fop);
         idx = fop->first - 1;
         mask = 0;
-        while (count-- > 0)
-        {
+        while (count-- > 0) {
             idx = ec_child_next(ec, fop, idx + 1);
-            mask |= 1ULL << idx;
+            if (idx < EC_MAX_NODES)
+                mask |= 1ULL << idx;
         }
 
         ec_dispatch_mask(fop, mask);
     }
 }
 
-ec_lock_t *ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
+void
+ec_succeed_all(ec_fop_data_t *fop)
+{
+    ec_dispatch_start(fop);
+
+    if (ec_child_select(fop)) {
+        fop->expected = gf_bits_count(fop->remaining);
+        fop->first = 0;
+
+        /* Simulate a successful execution on all bricks */
+        ec_trace("SUCCEED", fop, "");
+
+        fop->good = fop->remaining;
+        fop->remaining = 0;
+    }
+}
+
+ec_lock_t *
+ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
 {
     ec_t *ec = fop->xl->private;
-    ec_lock_t * lock;
+    ec_lock_t *lock;
     int32_t err;
 
     if ((loc->inode == NULL) ||
-        (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)))
-    {
-        gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_INVALID_INODE,
-                "Trying to lock based on an invalid "
-                "inode");
+        (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid))) {
+        gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_INODE,
+               "Trying to lock based on an invalid "
+               "inode");
 
         __ec_fop_set_error(fop, EINVAL);
 
@@ -642,9 +937,8 @@ ec_lock_t *ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
     }
 
     lock = mem_get0(ec->lock_pool);
-    if (lock != NULL)
-    {
-        lock->good_mask = -1ULL;
+    if (lock != NULL) {
+        lock->good_mask = UINTPTR_MAX;
         INIT_LIST_HEAD(&lock->owners);
         INIT_LIST_HEAD(&lock->waiting);
         INIT_LIST_HEAD(&lock->frozen);
@@ -660,7 +954,8 @@ ec_lock_t *ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
     return lock;
 }
 
-void ec_lock_destroy(ec_lock_t * lock)
+void
+ec_lock_destroy(ec_lock_t *lock)
 {
     loc_wipe(&lock->loc);
     if (lock->fd != NULL) {
@@ -670,13 +965,15 @@ void ec_lock_destroy(ec_lock_t * lock)
     mem_put(lock);
 }
 
-int32_t ec_lock_compare(ec_lock_t * lock1, ec_lock_t * lock2)
+int32_t
+ec_lock_compare(ec_lock_t *lock1, ec_lock_t *lock2)
 {
     return gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid);
 }
 
-void ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags,
-                    loc_t *base)
+static void
+ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags, loc_t *base,
+               off_t fl_start, uint64_t fl_size)
 {
     ec_lock_link_t *link;
 
@@ -710,12 +1007,15 @@ void ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags,
     link->update[EC_DATA_TXN] = (flags & EC_UPDATE_DATA) != 0;
     link->update[EC_METADATA_TXN] = (flags & EC_UPDATE_META) != 0;
     link->base = base;
+    link->fl_start = fl_start;
+    link->fl_end = ec_range_end_get(fl_start, fl_size);
 
     lock->refs_pending++;
 }
 
-void ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc,
-                                    uint32_t flags, loc_t *base)
+static void
+ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                               loc_t *base, off_t fl_start, uint64_t fl_size)
 {
     ec_lock_t *lock = NULL;
     ec_inode_t *ctx;
@@ -744,8 +1044,8 @@ void ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc,
         if ((fop->lock_count > 0) && (fop->locks[0].lock == lock)) {
             /* Combine data/meta updates */
             fop->locks[0].update[EC_DATA_TXN] |= (flags & EC_UPDATE_DATA) != 0;
-            fop->locks[0].update[EC_METADATA_TXN] |=
-                                                 (flags & EC_UPDATE_META) != 0;
+            fop->locks[0].update[EC_METADATA_TXN] |= (flags & EC_UPDATE_META) !=
+                                                     0;
 
             /* Only one base inode is allowed per fop, so there shouldn't be
              * overwrites here. */
@@ -756,8 +1056,10 @@ void ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc,
             goto update_query;
         }
 
-        ec_trace("LOCK_INODELK", fop, "lock=%p, inode=%p. Lock already "
-                                      "acquired", lock, loc->inode);
+        ec_trace("LOCK_INODELK", fop,
+                 "lock=%p, inode=%p. Lock already "
+                 "acquired",
+                 lock, loc->inode);
 
         goto insert;
     }
@@ -776,22 +1078,25 @@ void ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc,
     ctx->inode_lock = lock;
 
 insert:
-    ec_lock_insert(fop, lock, flags, base);
+    ec_lock_insert(fop, lock, flags, base, fl_start, fl_size);
 update_query:
     lock->query |= (flags & EC_QUERY_INFO) != 0;
 unlock:
     UNLOCK(&loc->inode->lock);
 }
 
-void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags)
+void
+ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                      off_t fl_start, uint64_t fl_size)
 {
-    ec_lock_prepare_inode_internal(fop, loc, flags, NULL);
+    ec_lock_prepare_inode_internal(fop, loc, flags, NULL, fl_start, fl_size);
 }
 
-void ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc,
-                                  uint32_t flags)
+void
+ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
+                             uint32_t flags)
 {
-    loc_t tmp, *base = NULL;
+    loc_t tmp;
     int32_t err;
 
     if (fop->error != 0) {
@@ -806,16 +1111,19 @@ void ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc,
     }
 
     if ((flags & EC_INODE_SIZE) != 0) {
-        base = loc;
         flags ^= EC_INODE_SIZE;
+    } else {
+        base = NULL;
     }
 
-    ec_lock_prepare_inode_internal(fop, &tmp, flags, base);
+    ec_lock_prepare_inode_internal(fop, &tmp, flags, base, 0, EC_RANGE_FULL);
 
     loc_wipe(&tmp);
 }
 
-void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags)
+void
+ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start,
+                   uint64_t fl_size)
 {
     loc_t loc;
     int32_t err;
@@ -831,21 +1139,20 @@ void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags)
         return;
     }
 
-    ec_lock_prepare_inode_internal(fop, &loc, flags, NULL);
+    ec_lock_prepare_inode_internal(fop, &loc, flags, NULL, fl_start, fl_size);
 
     loc_wipe(&loc);
 }
 
 gf_boolean_t
-ec_config_check (ec_fop_data_t *fop, ec_config_t *config)
+ec_config_check(xlator_t *xl, ec_config_t *config)
 {
     ec_t *ec;
 
-    ec = fop->xl->private;
+    ec = xl->private;
     if ((config->version != EC_CONFIG_VERSION) ||
         (config->algorithm != EC_CONFIG_ALGORITHM) ||
-        (config->gf_word_size != EC_GF_BITS) ||
-        (config->bricks != ec->nodes) ||
+        (config->gf_word_size != EC_GF_BITS) || (config->bricks != ec->nodes) ||
         (config->redundancy != ec->redundancy) ||
         (config->chunk_size != EC_METHOD_CHUNK_SIZE)) {
         uint32_t data_bricks;
@@ -864,20 +1171,17 @@ ec_config_check (ec_fop_data_t *fop, ec_config_t *config)
         if ((config->redundancy < 1) ||
             (config->redundancy * 2 >= config->bricks) ||
             !ec_is_power_of_2(config->gf_word_size) ||
-            ((config->chunk_size * 8) % (config->gf_word_size * data_bricks)
-                                                                       != 0)) {
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_INVALID_CONFIG,
-                    "Invalid or corrupted config");
+            ((config->chunk_size * 8) % (config->gf_word_size * data_bricks) !=
+             0)) {
+            gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG,
+                   "Invalid or corrupted config");
         } else {
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_INVALID_CONFIG,
-                    "Unsupported config "
-                    "(V=%u, A=%u, W=%u, "
-                    "N=%u, R=%u, S=%u)",
-                   config->version, config->algorithm,
-                   config->gf_word_size, config->bricks,
-                   config->redundancy, config->chunk_size);
+            gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG,
+                   "Unsupported config "
+                   "(V=%u, A=%u, W=%u, "
+                   "N=%u, R=%u, S=%u)",
+                   config->version, config->algorithm, config->gf_word_size,
+                   config->bricks, config->redundancy, config->chunk_size);
         }
 
         return _gf_false;
@@ -886,121 +1190,164 @@ ec_config_check (ec_fop_data_t *fop, ec_config_t *config)
     return _gf_true;
 }
 
+gf_boolean_t
+ec_set_dirty_flag(ec_lock_link_t *link, ec_inode_t *ctx, uint64_t *dirty)
+{
+    gf_boolean_t set_dirty = _gf_false;
+
+    if (link->update[EC_DATA_TXN] && !ctx->dirty[EC_DATA_TXN]) {
+        if (!link->optimistic_changelog)
+            dirty[EC_DATA_TXN] = 1;
+    }
+
+    if (link->update[EC_METADATA_TXN] && !ctx->dirty[EC_METADATA_TXN]) {
+        if (!link->optimistic_changelog)
+            dirty[EC_METADATA_TXN] = 1;
+    }
+
+    if (dirty[EC_METADATA_TXN] || dirty[EC_DATA_TXN]) {
+        set_dirty = _gf_true;
+    }
+
+    return set_dirty;
+}
+
 int32_t
-ec_prepare_update_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *dict, dict_t *xdata)
+ec_prepare_update_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
     struct list_head list;
     ec_fop_data_t *fop = cookie, *parent, *tmp;
-    ec_lock_link_t *link = fop->data;
+    ec_lock_link_t *parent_link = fop->data;
+    ec_lock_link_t *link = NULL;
     ec_lock_t *lock = NULL;
     ec_inode_t *ctx;
-
-    lock = link->lock;
-    parent = link->fop;
+    gf_boolean_t release = _gf_false;
+    uint64_t provided_flags = 0;
+    uint64_t dirty[EC_VERSION_SIZE] = {0, 0};
+    lock = parent_link->lock;
+    parent = parent_link->fop;
     ctx = lock->ctx;
 
     INIT_LIST_HEAD(&list);
+    provided_flags = EC_PROVIDED_FLAGS(parent_link->waiting_flags);
 
     LOCK(&lock->loc.inode->lock);
 
-    list_for_each_entry(link, &lock->owners, owner_list) {
-        if ((link->fop->flags & EC_FLAG_WAITING_SIZE) != 0) {
-            link->fop->flags ^= EC_FLAG_WAITING_SIZE;
-
-            list_add_tail(&link->fop->cbk_list, &list);
+    list_for_each_entry(link, &lock->owners, owner_list)
+    {
+        if ((link->waiting_flags & provided_flags) != 0) {
+            link->waiting_flags ^= (link->waiting_flags & provided_flags);
+            if (EC_NEEDED_FLAGS(link->waiting_flags) == 0)
+                list_add_tail(&link->fop->cbk_list, &list);
         }
     }
-
     if (op_ret < 0) {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_SIZE_VERS_GET_FAIL,
-                "Failed to get size and version");
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_SIZE_VERS_GET_FAIL,
+               "Failed to get size and version :  %s", ec_msg_str(fop));
 
         goto unlock;
     }
 
-    op_errno = -ec_dict_del_array(dict, EC_XATTR_VERSION, ctx->pre_version,
-                                  EC_VERSION_SIZE);
-    if (op_errno != 0) {
-        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                EC_MSG_VER_XATTR_GET_FAIL,
-                "Unable to get version xattr");
-
-        goto unlock;
-    }
-    ctx->post_version[0] += ctx->pre_version[0];
-    ctx->post_version[1] += ctx->pre_version[1];
-
-    ctx->have_version = _gf_true;
-
-    if (lock->loc.inode->ia_type == IA_IFREG ||
-        lock->loc.inode->ia_type == IA_INVAL) {
-        op_errno = -ec_dict_del_number(dict, EC_XATTR_SIZE, &ctx->pre_size);
+    if (EC_FLAGS_HAVE(provided_flags, EC_FLAG_XATTROP)) {
+        op_errno = -ec_dict_del_array(dict, EC_XATTR_VERSION, ctx->pre_version,
+                                      EC_VERSION_SIZE);
         if (op_errno != 0) {
-            if (lock->loc.inode->ia_type == IA_IFREG) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        EC_MSG_SIZE_XATTR_GET_FAIL,
-                        "Unable to get size xattr");
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   EC_MSG_VER_XATTR_GET_FAIL, "Unable to get version xattr. %s",
+                   ec_msg_str(fop));
+            goto unlock;
+        }
+        ctx->post_version[0] += ctx->pre_version[0];
+        ctx->post_version[1] += ctx->pre_version[1];
+
+        ctx->have_version = _gf_true;
+
+        if (lock->loc.inode->ia_type == IA_IFREG ||
+            lock->loc.inode->ia_type == IA_INVAL) {
+            op_errno = -ec_dict_del_number(dict, EC_XATTR_SIZE, &ctx->pre_size);
+            if (op_errno != 0) {
+                if (lock->loc.inode->ia_type == IA_IFREG) {
+                    gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                           EC_MSG_SIZE_XATTR_GET_FAIL,
+                           "Unable to get size xattr. %s", ec_msg_str(fop));
+                    goto unlock;
+                }
+            } else {
+                ctx->post_size = ctx->pre_size;
 
-                goto unlock;
+                ctx->have_size = _gf_true;
             }
-        } else {
-            ctx->post_size = ctx->pre_size;
 
-            ctx->have_size = _gf_true;
-        }
+            op_errno = -ec_dict_del_config(dict, EC_XATTR_CONFIG, &ctx->config);
+            if (op_errno != 0) {
+                if ((lock->loc.inode->ia_type == IA_IFREG) ||
+                    (op_errno != ENODATA)) {
+                    gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                           EC_MSG_CONFIG_XATTR_GET_FAIL,
+                           "Unable to get config xattr. %s", ec_msg_str(fop));
 
-        op_errno = -ec_dict_del_config(dict, EC_XATTR_CONFIG, &ctx->config);
-        if (op_errno != 0) {
-            if ((lock->loc.inode->ia_type == IA_IFREG) ||
-                (op_errno != ENODATA)) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        EC_MSG_CONFIG_XATTR_GET_FAIL,
-                        "Unable to get config xattr");
-
-                goto unlock;
-            }
-        } else {
-            if (!ec_config_check(parent, &ctx->config)) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        EC_MSG_CONFIG_XATTR_INVALID,
-                        "Invalid config xattr");
+                    goto unlock;
+                }
+            } else {
+                if (!ec_config_check(parent->xl, &ctx->config)) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           EC_MSG_CONFIG_XATTR_INVALID, "Invalid config xattr");
 
-                op_errno = EINVAL;
+                    op_errno = EINVAL;
 
-                goto unlock;
+                    goto unlock;
+                }
+                ctx->have_config = _gf_true;
             }
-
-            ctx->have_config = _gf_true;
         }
+        ctx->have_info = _gf_true;
     }
 
-    ctx->have_info = _gf_true;
+    ec_set_dirty_flag(fop->data, ctx, dirty);
+    if (dirty[EC_METADATA_TXN] &&
+        (EC_FLAGS_HAVE(provided_flags, EC_FLAG_METADATA_DIRTY))) {
+        GF_ASSERT(!ctx->dirty[EC_METADATA_TXN]);
+        ctx->dirty[EC_METADATA_TXN] = 1;
+    }
 
+    if (dirty[EC_DATA_TXN] &&
+        (EC_FLAGS_HAVE(provided_flags, EC_FLAG_DATA_DIRTY))) {
+        GF_ASSERT(!ctx->dirty[EC_DATA_TXN]);
+        ctx->dirty[EC_DATA_TXN] = 1;
+    }
     op_errno = 0;
-
 unlock:
-    lock->getting_size = _gf_false;
 
-    UNLOCK(&lock->loc.inode->lock);
+    lock->waiting_flags ^= provided_flags;
 
     if (op_errno == 0) {
+        /* If the fop fails on any of the good bricks, it is important to mark
+         * it dirty and update versions right away if dirty was not set before.
+         */
+        if (lock->good_mask & ~(fop->good | fop->remaining)) {
+            release = _gf_true;
+        }
+
+        if (parent_link->update[0] && !parent_link->dirty[0]) {
+            lock->release |= release;
+        }
+
+        if (parent_link->update[1] && !parent_link->dirty[1]) {
+            lock->release |= release;
+        }
+
         /* We don't allow the main fop to be executed on bricks that have not
          * succeeded the initial xattrop. */
-        parent->mask &= fop->good;
-        ec_lock_update_good (lock, fop);
+        ec_lock_update_good(lock, fop);
 
         /*As of now only data healing marks bricks as healing*/
         lock->healing |= fop->healing;
-        if (ec_is_data_fop (parent->id)) {
-            parent->healing |= fop->healing;
-        }
-    } else {
-        ec_fop_set_error(parent, op_errno);
     }
 
+    UNLOCK(&lock->loc.inode->lock);
+
     while (!list_empty(&list)) {
         tmp = list_entry(list.next, ec_fop_data_t, cbk_list);
         list_del_init(&tmp->cbk_list);
@@ -1009,93 +1356,163 @@ unlock:
             tmp->mask &= fop->good;
 
             /*As of now only data healing marks bricks as healing*/
-            if (ec_is_data_fop (tmp->id)) {
+            if (ec_is_data_fop(tmp->id)) {
                 tmp->healing |= fop->healing;
             }
-        } else {
-            ec_fop_set_error(tmp, op_errno);
         }
 
-        ec_resume(tmp, 0);
+        ec_resume(tmp, op_errno);
     }
 
     return 0;
 }
 
-void ec_get_size_version(ec_lock_link_t *link)
+static gf_boolean_t
+ec_set_needed_flag(ec_lock_t *lock, ec_lock_link_t *link, uint64_t flag)
+{
+    uint64_t current;
+
+    link->waiting_flags |= EC_FLAG_NEEDS(flag);
+
+    current = EC_NEEDED_FLAGS(lock->waiting_flags);
+    if (!EC_FLAGS_HAVE(current, flag)) {
+        lock->waiting_flags |= EC_FLAG_NEEDS(flag);
+        link->waiting_flags |= EC_FLAG_PROVIDES(flag);
+
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static uint64_t
+ec_set_xattrop_flags_and_params(ec_lock_t *lock, ec_lock_link_t *link,
+                                uint64_t *dirty)
+{
+    uint64_t oldflags = 0;
+    uint64_t newflags = 0;
+    ec_inode_t *ctx = lock->ctx;
+
+    oldflags = EC_NEEDED_FLAGS(lock->waiting_flags);
+
+    if (lock->query && !ctx->have_info) {
+        ec_set_needed_flag(lock, link, EC_FLAG_XATTROP);
+    }
+
+    if (dirty[EC_DATA_TXN]) {
+        if (!ec_set_needed_flag(lock, link, EC_FLAG_DATA_DIRTY)) {
+            dirty[EC_DATA_TXN] = 0;
+        }
+    }
+
+    if (dirty[EC_METADATA_TXN]) {
+        if (!ec_set_needed_flag(lock, link, EC_FLAG_METADATA_DIRTY)) {
+            dirty[EC_METADATA_TXN] = 0;
+        }
+    }
+    newflags = EC_NEEDED_FLAGS(lock->waiting_flags);
+
+    return oldflags ^ newflags;
+}
+
+void
+ec_get_size_version(ec_lock_link_t *link)
 {
     loc_t loc;
     ec_lock_t *lock;
     ec_inode_t *ctx;
     ec_fop_data_t *fop;
     dict_t *dict = NULL;
-    int32_t error = -ENOMEM;
-    gf_boolean_t getting_size;
+    dict_t *xdata = NULL;
+    ec_t *ec = NULL;
+    int32_t error = 0;
+    gf_boolean_t set_dirty = _gf_false;
     uint64_t allzero[EC_VERSION_SIZE] = {0, 0};
-
+    uint64_t dirty[EC_VERSION_SIZE] = {0, 0};
     lock = link->lock;
     ctx = lock->ctx;
     fop = link->fop;
+    ec = fop->xl->private;
+    uint64_t changed_flags = 0;
+
+    if (ec->optimistic_changelog && !(ec->node_mask & ~link->lock->good_mask) &&
+        !ec_is_data_fop(fop->id))
+        link->optimistic_changelog = _gf_true;
+
+    memset(&loc, 0, sizeof(loc));
+
+    LOCK(&lock->loc.inode->lock);
+
+    set_dirty = ec_set_dirty_flag(link, ctx, dirty);
 
     /* If ec metadata has already been retrieved, do not try again. */
     if (ctx->have_info) {
-        if (ec_is_data_fop (fop->id)) {
+        if (ec_is_data_fop(fop->id)) {
             fop->healing |= lock->healing;
         }
-        return;
+        if (!set_dirty)
+            goto unlock;
     }
 
     /* Determine if there's something we need to retrieve for the current
      * operation. */
-    if (!lock->query &&
-        (lock->loc.inode->ia_type != IA_IFREG) &&
+    if (!set_dirty && !lock->query && (lock->loc.inode->ia_type != IA_IFREG) &&
         (lock->loc.inode->ia_type != IA_INVAL)) {
-        return;
+        goto unlock;
     }
 
-    memset(&loc, 0, sizeof(loc));
-
-    LOCK(&lock->loc.inode->lock);
-
-    getting_size = lock->getting_size;
-    lock->getting_size = _gf_true;
-    if (getting_size) {
-        fop->flags |= EC_FLAG_WAITING_SIZE;
-
+    changed_flags = ec_set_xattrop_flags_and_params(lock, link, dirty);
+    if (link->waiting_flags) {
+        /* This fop needs to wait until all its flags are cleared which
+         * potentially can be cleared by other xattrops that are already
+         * wound*/
         ec_sleep(fop);
+    } else {
+        GF_ASSERT(!changed_flags);
     }
 
+unlock:
     UNLOCK(&lock->loc.inode->lock);
 
-    if (getting_size) {
-        error = 0;
-
+    if (!changed_flags)
         goto out;
-    }
 
     dict = dict_new();
     if (dict == NULL) {
+        error = -ENOMEM;
         goto out;
     }
 
-    /* Once we know that an xattrop will be needed, we try to get all available
-     * information in a single call. */
-    error = ec_dict_set_array(dict, EC_XATTR_VERSION, allzero,
-                              EC_VERSION_SIZE);
-    if (error == 0) {
-        error = ec_dict_set_array(dict, EC_XATTR_DIRTY, allzero,
+    if (EC_FLAGS_HAVE(changed_flags, EC_FLAG_XATTROP)) {
+        /* Once we know that an xattrop will be needed,
+         * we try to get all available information in a
+         * single call. */
+        error = ec_dict_set_array(dict, EC_XATTR_VERSION, allzero,
                                   EC_VERSION_SIZE);
-    }
-    if (error != 0) {
-        goto out;
-    }
+        if (error != 0) {
+            goto out;
+        }
 
-    if (lock->loc.inode->ia_type == IA_IFREG ||
-        lock->loc.inode->ia_type == IA_INVAL) {
-        error = ec_dict_set_number(dict, EC_XATTR_SIZE, 0);
-        if (error == 0) {
-            error = ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
+        if (lock->loc.inode->ia_type == IA_IFREG ||
+            lock->loc.inode->ia_type == IA_INVAL) {
+            error = ec_dict_set_number(dict, EC_XATTR_SIZE, 0);
+            if (error == 0) {
+                error = ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
+            }
+            if (error != 0) {
+                goto out;
+            }
+
+            xdata = dict_new();
+            if (xdata == NULL || dict_set_int32(xdata, GF_GET_SIZE, 1)) {
+                error = -ENOMEM;
+                goto out;
+            }
         }
+    }
+
+    if (memcmp(allzero, dirty, sizeof(allzero))) {
+        error = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
         if (error != 0) {
             goto out;
         }
@@ -1126,13 +1543,13 @@ void ec_get_size_version(ec_lock_link_t *link)
             loc.name = NULL;
         }
 
-        ec_xattrop (fop->frame, fop->xl, fop->mask, fop->minimum,
-                    ec_prepare_update_cbk, link, &loc,
-                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ec_xattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
+                   ec_prepare_update_cbk, link, &loc, GF_XATTROP_ADD_ARRAY64,
+                   dict, xdata);
     } else {
         ec_fxattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
-                ec_prepare_update_cbk, link, lock->fd,
-                GF_XATTROP_ADD_ARRAY64, dict, NULL);
+                    ec_prepare_update_cbk, link, lock->fd,
+                    GF_XATTROP_ADD_ARRAY64, dict, xdata);
     }
 
     error = 0;
@@ -1147,22 +1564,24 @@ out:
         dict_unref(dict);
     }
 
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
+
     if (error != 0) {
         ec_fop_set_error(fop, -error);
     }
 }
 
-gf_boolean_t ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                               uint64_t *size)
+gf_boolean_t
+__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size)
 {
     ec_inode_t *ctx;
     gf_boolean_t found = _gf_false;
 
-    LOCK(&inode->lock);
-
     ctx = __ec_inode_get(inode, fop->xl);
     if (ctx == NULL) {
-        goto unlock;
+        goto out;
     }
 
     if (ctx->have_size) {
@@ -1170,23 +1589,33 @@ gf_boolean_t ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
         found = _gf_true;
     }
 
-unlock:
+out:
+    return found;
+}
+
+gf_boolean_t
+ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size)
+{
+    gf_boolean_t found = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        found = __ec_get_inode_size(fop, inode, size);
+    }
     UNLOCK(&inode->lock);
 
     return found;
 }
 
-gf_boolean_t ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                               uint64_t size)
+gf_boolean_t
+__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size)
 {
     ec_inode_t *ctx;
     gf_boolean_t found = _gf_false;
 
-    LOCK(&inode->lock);
-
     ctx = __ec_inode_get(inode, fop->xl);
     if (ctx == NULL) {
-        goto unlock;
+        goto out;
     }
 
     /* Normal fops always have ctx->have_size set. However self-heal calls this
@@ -1201,13 +1630,42 @@ gf_boolean_t ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode,
 
     found = _gf_true;
 
-unlock:
+out:
+    return found;
+}
+
+gf_boolean_t
+ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size)
+{
+    gf_boolean_t found = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        found = __ec_set_inode_size(fop, inode, size);
+    }
     UNLOCK(&inode->lock);
 
     return found;
 }
 
-void ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode)
+static void
+ec_release_stripe_cache(ec_inode_t *ctx)
+{
+    ec_stripe_list_t *stripe_cache = NULL;
+    ec_stripe_t *stripe = NULL;
+
+    stripe_cache = &ctx->stripe_cache;
+    while (!list_empty(&stripe_cache->lru)) {
+        stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru);
+        list_del(&stripe->lru);
+        GF_FREE(stripe);
+    }
+    stripe_cache->count = 0;
+    stripe_cache->max = 0;
+}
+
+void
+ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode)
 {
     ec_inode_t *ctx;
 
@@ -1218,6 +1676,7 @@ void ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode)
         goto unlock;
     }
 
+    ec_release_stripe_cache(ctx);
     ctx->have_info = _gf_false;
     ctx->have_config = _gf_false;
     ctx->have_version = _gf_false;
@@ -1233,10 +1692,10 @@ unlock:
     UNLOCK(&inode->lock);
 }
 
-int32_t ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, inode_t *inode,
-                             struct iatt *buf, dict_t *xdata,
-                             struct iatt *postparent)
+int32_t
+ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link;
@@ -1253,14 +1712,15 @@ int32_t ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 /* This function is used to get the trusted.ec.size xattr from a file when
- * no lock is needed on the inode. This is only required to maintan iatt
+ * no lock is needed on the inode. This is only required to maintain iatt
  * structs on fops that manipulate directory entries but do not operate
  * directly on the inode, like link, rename, ...
  *
  * Any error processing this request is ignored. In the worst case, an invalid
  * or not up to date value in the iatt could cause some cache invalidation.
  */
-void ec_get_real_size(ec_lock_link_t *link)
+void
+ec_get_real_size(ec_lock_link_t *link)
 {
     ec_fop_data_t *fop;
     dict_t *xdata;
@@ -1304,42 +1764,59 @@ ec_lock_update_fd(ec_lock_t *lock, ec_fop_data_t *fop)
 {
     /* If the fop has an fd available, attach it to the lock structure to be
      * able to do fxattrop calls instead of xattrop. */
-    if (fop->use_fd) {
-        if (lock->fd != NULL) {
-            __fd_unref(lock->fd);
-        }
+    if (fop->use_fd && (lock->fd == NULL)) {
         lock->fd = __fd_ref(fop->fd);
     }
 }
 
+static gf_boolean_t
+ec_link_has_lock_conflict(ec_lock_link_t *link, gf_boolean_t waitlist_check)
+{
+    ec_lock_link_t *trav_link = NULL;
+
+    list_for_each_entry(trav_link, &link->lock->owners, owner_list)
+    {
+        if (ec_lock_conflict(trav_link, link))
+            return _gf_true;
+    }
+
+    if (!waitlist_check)
+        return _gf_false;
+
+    list_for_each_entry(trav_link, &link->lock->waiting, wait_list)
+    {
+        if (ec_lock_conflict(trav_link, link))
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
+
 static void
 ec_lock_wake_shared(ec_lock_t *lock, struct list_head *list)
 {
     ec_fop_data_t *fop;
     ec_lock_link_t *link;
-    gf_boolean_t exclusive = _gf_false;
+    gf_boolean_t conflict = _gf_false;
 
-    while (!exclusive && !list_empty(&lock->waiting)) {
+    while (!conflict && !list_empty(&lock->waiting)) {
         link = list_entry(lock->waiting.next, ec_lock_link_t, wait_list);
         fop = link->fop;
 
         /* If lock is not acquired, at most one fop can be assigned as owner.
          * The following fops will need to wait in the lock->waiting queue
          * until the lock has been fully acquired. */
-        exclusive = !lock->acquired;
+        conflict = !lock->acquired;
 
         /* If the fop is not shareable, only this fop can be assigned as owner.
          * Other fops will need to wait until this one finishes. */
-        if ((fop->flags & EC_FLAG_LOCK_SHARED) == 0) {
-            exclusive = _gf_true;
-
-            /* Avoid other requests to be assigned as owners. */
-            lock->exclusive = 1;
+        if (ec_link_has_lock_conflict(link, _gf_false)) {
+            conflict = _gf_true;
         }
 
         /* If only one fop is allowed, it can be assigned as the owner of the
          * lock only if there weren't any other owner. */
-        if (exclusive && !list_empty(&lock->owners)) {
+        if (conflict && !list_empty(&lock->owners)) {
             break;
         }
 
@@ -1364,7 +1841,8 @@ ec_lock_apply(ec_lock_link_t *link)
     ec_get_real_size(link);
 }
 
-gf_boolean_t ec_lock_acquire(ec_lock_link_t *link);
+gf_boolean_t
+ec_lock_acquire(ec_lock_link_t *link);
 
 static void
 ec_lock_resume_shared(struct list_head *list)
@@ -1388,7 +1866,8 @@ ec_lock_resume_shared(struct list_head *list)
     }
 }
 
-void ec_lock_acquired(ec_lock_link_t *link)
+void
+ec_lock_acquired(ec_lock_link_t *link)
 {
     struct list_head list;
     ec_lock_t *lock;
@@ -1404,61 +1883,76 @@ void ec_lock_acquired(ec_lock_link_t *link)
     LOCK(&lock->loc.inode->lock);
 
     lock->acquired = _gf_true;
+    if (lock->contention) {
+        lock->release = _gf_true;
+        lock->contention = _gf_false;
+    }
 
     ec_lock_update_fd(lock, fop);
-    if ((fop->flags & EC_FLAG_LOCK_SHARED) != 0) {
-        ec_lock_wake_shared(lock, &list);
-    }
+    ec_lock_wake_shared(lock, &list);
 
     UNLOCK(&lock->loc.inode->lock);
 
     ec_lock_apply(link);
 
+    if (fop->use_fd &&
+        (link->update[EC_DATA_TXN] || link->update[EC_METADATA_TXN])) {
+        /* Try to reopen closed fd's only if lock has succeeded. */
+        ec_fix_open(fop, lock->mask);
+    }
+
     ec_lock_resume_shared(&list);
 }
 
-int32_t ec_locked(call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_locked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link = NULL;
     ec_lock_t *lock = NULL;
 
+    link = fop->data;
+    lock = link->lock;
     if (op_ret >= 0) {
-        link = fop->data;
-        lock = link->lock;
         lock->mask = lock->good_mask = fop->good;
         lock->healing = 0;
 
         ec_lock_acquired(link);
         ec_lock(fop->parent);
     } else {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_PREOP_LOCK_FAILED,
-                "Failed to complete preop lock");
+        LOCK(&lock->loc.inode->lock);
+        {
+            lock->contention = _gf_false;
+        }
+        UNLOCK(&lock->loc.inode->lock);
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_PREOP_LOCK_FAILED,
+               "Failed to complete preop lock");
     }
 
     return 0;
 }
 
-gf_boolean_t ec_lock_acquire(ec_lock_link_t *link)
+gf_boolean_t
+ec_lock_acquire(ec_lock_link_t *link)
 {
     ec_lock_t *lock;
     ec_fop_data_t *fop;
+    gf_lkowner_t lk_owner;
 
     lock = link->lock;
     fop = link->fop;
 
     if (!lock->acquired) {
-        ec_owner_set(fop->frame, lock);
+        set_lk_owner_from_ptr(&lk_owner, lock);
 
         ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p", lock,
                  lock->loc.inode);
 
         lock->flock.l_type = F_WRLCK;
-        ec_inodelk(fop->frame, fop->xl, -1, EC_MINIMUM_ALL, ec_locked,
-                   link, fop->xl->name, &lock->loc, F_SETLKW, &lock->flock,
-                   NULL);
+        ec_inodelk(fop->frame, fop->xl, &lk_owner, -1, EC_MINIMUM_ALL,
+                   ec_locked, link, fop->xl->name, &lock->loc, F_SETLKW,
+                   &lock->flock, NULL);
 
         return _gf_false;
     }
@@ -1470,6 +1964,67 @@ gf_boolean_t ec_lock_acquire(ec_lock_link_t *link)
     return _gf_true;
 }
 
+static ec_lock_link_t *
+ec_lock_timer_cancel(xlator_t *xl, ec_lock_t *lock)
+{
+    ec_lock_link_t *timer_link;
+
+    /* If we don't have any timer, there's nothing to cancel. */
+    if (lock->timer == NULL) {
+        return NULL;
+    }
+
+    /* We are trying to access a lock that has an unlock timer active.
+     * This means that the lock must be idle, i.e. no fop can be in the
+     * owner, waiting or frozen lists. It also means that the lock cannot
+     * have been marked as being released (this is done without timers).
+     * There should only be one owner reference, but it's possible that
+     * some fops are being prepared to use this lock. */
+    GF_ASSERT((lock->refs_owners == 1) && list_empty(&lock->owners) &&
+              list_empty(&lock->waiting));
+
+    /* We take the timer_link before cancelling the timer, since a
+     * successful cancellation will destroy it. It must not be NULL
+     * because it references the fop responsible for the delayed unlock
+     * that we are currently trying to cancel. */
+    timer_link = lock->timer->data;
+    GF_ASSERT(timer_link != NULL);
+
+    if (gf_timer_call_cancel(xl->ctx, lock->timer) < 0) {
+        /* It's too late to avoid the execution of the timer callback.
+         * Since we need to be sure that the callback has access to all
+         * needed resources, we cannot resume the execution of the
+         * timer fop now. This will be done in the callback. */
+        timer_link = NULL;
+    } else {
+        /* The timer has been cancelled. The fop referenced by
+         * timer_link holds the last reference. The caller is
+         * responsible to release it when not needed anymore. */
+        ec_trace("UNLOCK_CANCELLED", timer_link->fop, "lock=%p", lock);
+    }
+
+    /* We have two options here:
+     *
+     * 1. The timer has been successfully cancelled.
+     *
+     *    This is the easiest case and we can continue with the currently
+     *    acquired lock.
+     *
+     * 2. The timer callback has already been fired.
+     *
+     *    In this case we have not been able to cancel the timer before
+     *    the timer callback has been fired, but we also know that
+     *    lock->timer != NULL. This means that the timer callback is still
+     *    trying to acquire the inode mutex that we currently own. We are
+     *    safe until we release it. In this case we can safely clear
+     *    lock->timer. This will cause that the timer callback does nothing
+     *    once it acquires the mutex.
+     */
+    lock->timer = NULL;
+
+    return timer_link;
+}
+
 static gf_boolean_t
 ec_lock_assign_owner(ec_lock_link_t *link)
 {
@@ -1489,7 +2044,7 @@ ec_lock_assign_owner(ec_lock_link_t *link)
 
     /* Since the link has just been prepared but it's not active yet, the
      * refs_pending must be one at least (the ref owned by this link). */
-    GF_ASSERT (lock->refs_pending > 0);
+    GF_ASSERT(lock->refs_pending > 0);
     /* The link is not pending any more. It will be assigned to the owner,
      * waiting or frozen list. */
     lock->refs_pending--;
@@ -1514,59 +2069,15 @@ ec_lock_assign_owner(ec_lock_link_t *link)
      * empty. */
     GF_ASSERT(list_empty(&lock->frozen));
 
-    if (lock->timer != NULL) {
-        /* We are trying to acquire a lock that has an unlock timer active.
-         * This means that the lock must be idle, i.e. no fop can be in the
-         * owner, waiting or frozen lists. It also means that the lock cannot
-         * have been marked as being released (this is done without timers)
-         * and it must not be exclusive. There should only be one owner
-         * reference, but it's possible that some fops are being prepared to
-         * use this lock. */
-        GF_ASSERT ((lock->exclusive == 0) && (lock->refs_owners == 1) &&
-                   list_empty(&lock->owners) && list_empty(&lock->waiting));
-
-        /* We take the timer_link before cancelling the timer, since a
-         * successful cancellation will destroy it. It must not be NULL
-         * because it references the fop responsible for the delayed unlock
-         * that we are currently trying to cancel. */
-        timer_link = lock->timer->data;
-        GF_ASSERT(timer_link != NULL);
-
-        gf_timer_call_cancel(fop->xl->ctx, lock->timer);
-        ec_trace("UNLOCK_CANCELLED", timer_link->fop, "lock=%p", lock);
-
-        /* We have two options here:
-         *
-         * 1. The timer has been successfully cancelled.
-         *
-         *    This is the easiest case and we can continue with the currently
-         *    acquired lock.
-         *
-         * 2. The timer callback has already been fired.
-         *
-         *    In this case we have not been able to cancel the timer before
-         *    the timer callback has been fired, but we also know that
-         *    lock->timer != NULL. This means that the timer callback is still
-         *    trying to acquire the inode mutex that we currently own. We are
-         *    safe until we release it. In this case we can safely clear
-         *    lock->timer. This will cause that the timer callback does nothing
-         *    once it acquires the mutex.
-         *
-         * In both cases we must release the owner reference assigned to the
-         * fop that was handling the unlock because ec_unlock_now() won't be
-         * called for that fop.
-         */
-        lock->timer = NULL;
-        lock->refs_owners--;
-    }
-
-    lock->exclusive |= (fop->flags & EC_FLAG_LOCK_SHARED) == 0;
+    timer_link = ec_lock_timer_cancel(fop->xl, lock);
 
     if (!list_empty(&lock->owners)) {
         /* There are other owners of this lock. We can only take ownership if
-         * the lock is already acquired and can be shared. Otherwise we need
-         * to wait. */
-        if (!lock->acquired || (lock->exclusive != 0)) {
+         * the lock is already acquired and doesn't have conflict with existing
+         * owners, or waiters(to prevent starvation).
+         * Otherwise we need to wait.
+         */
+        if (!lock->acquired || ec_link_has_lock_conflict(link, _gf_true)) {
             ec_trace("LOCK_QUEUE_WAIT", fop, "lock=%p", lock);
 
             list_add_tail(&link->wait_list, &lock->waiting);
@@ -1576,7 +2087,13 @@ ec_lock_assign_owner(ec_lock_link_t *link)
     }
 
     list_add_tail(&link->owner_list, &lock->owners);
-    lock->refs_owners++;
+
+    /* If timer_link is not NULL, it means that we have inherited the owner
+     * reference assigned to the timer fop. In this case we simply reuse it.
+     * Otherwise we need to increase the number of owners. */
+    if (timer_link == NULL) {
+        lock->refs_owners++;
+    }
 
     assigned = _gf_true;
 
@@ -1606,7 +2123,6 @@ ec_lock_next_owner(ec_lock_link_t *link, ec_cbk_data_t *cbk,
     ec_lock_t *lock = link->lock;
     ec_fop_data_t *fop = link->fop;
     ec_inode_t *ctx = lock->ctx;
-    ec_t *ec = fop->xl->private;
 
     INIT_LIST_HEAD(&list);
 
@@ -1627,31 +2143,33 @@ ec_lock_next_owner(ec_lock_link_t *link, ec_cbk_data_t *cbk,
     if ((fop->error == 0) && (cbk != NULL) && (cbk->op_ret >= 0)) {
         if (link->update[0]) {
             ctx->post_version[0]++;
-            if (ec->node_mask & ~fop->good) {
-                ctx->dirty[0]++;
-            }
         }
         if (link->update[1]) {
             ctx->post_version[1]++;
-            if (ec->node_mask & ~fop->good) {
-                ctx->dirty[1]++;
+        }
+        /* If the fop fails on any of the good bricks, it is important to mark
+         * it dirty and update versions right away. */
+        if (link->update[0] || link->update[1]) {
+            if (lock->good_mask & ~(fop->good | fop->remaining)) {
+                lock->release = _gf_true;
             }
         }
     }
 
+    if (fop->healing) {
+        lock->healing = fop->healing & (fop->good | fop->remaining);
+    }
     ec_lock_update_good(lock, fop);
 
-    lock->exclusive -= (fop->flags & EC_FLAG_LOCK_SHARED) == 0;
-    if (list_empty(&lock->owners)) {
-        ec_lock_wake_shared(lock, &list);
-    }
+    ec_lock_wake_shared(lock, &list);
 
     UNLOCK(&lock->loc.inode->lock);
 
     ec_lock_resume_shared(&list);
 }
 
-void ec_lock(ec_fop_data_t *fop)
+void
+ec_lock(ec_fop_data_t *fop)
 {
     ec_lock_link_t *link;
 
@@ -1659,7 +2177,7 @@ void ec_lock(ec_fop_data_t *fop)
      * Which can result in refs == 0 for fop leading to use after free in this
      * function when it calls ec_sleep so do ec_sleep at start and ec_resume at
      * the end of this function.*/
-    ec_sleep (fop);
+    ec_sleep(fop);
 
     while (fop->locked < fop->lock_count) {
         /* Since there are only up to 2 locks per fop, this xor will change
@@ -1698,11 +2216,11 @@ ec_lock_unfreeze(ec_lock_link_t *link)
     lock->acquired = _gf_false;
 
     /* We are unfreezing a lock. This means that the lock has already been
-     * released. In this state it shouldn't be exclusive nor have a pending
-     * timer nor have any owner, and the waiting list should be empty. Only
-     * the frozen list can contain some fop. */
-    GF_ASSERT((lock->exclusive == 0) && (lock->timer == NULL) &&
-              list_empty(&lock->waiting) && list_empty(&lock->owners));
+     * released. In this state it shouldn't have a pending timer nor have any
+     * owner, and the waiting list should be empty. Only the frozen list can
+     * contain some fop. */
+    GF_ASSERT((lock->timer == NULL) && list_empty(&lock->waiting) &&
+              list_empty(&lock->owners));
 
     /* We move all frozen fops to the waiting list. */
     list_splice_init(&lock->frozen, &lock->waiting);
@@ -1729,17 +2247,16 @@ ec_lock_unfreeze(ec_lock_link_t *link)
     }
 }
 
-int32_t ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link = fop->data;
 
     if (op_ret < 0) {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_UNLOCK_FAILED,
-                "entry/inode unlocking failed (%s)",
-                ec_fop_name(link->fop->id));
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED,
+               "entry/inode unlocking failed :(%s)", ec_msg_str(link->fop));
     } else {
         ec_trace("UNLOCKED", link->fop, "lock=%p", link->lock);
     }
@@ -1749,24 +2266,26 @@ int32_t ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
     return 0;
 }
 
-void ec_unlock_lock(ec_lock_link_t *link)
+void
+ec_unlock_lock(ec_lock_link_t *link)
 {
     ec_lock_t *lock;
     ec_fop_data_t *fop;
+    gf_lkowner_t lk_owner;
 
     lock = link->lock;
     fop = link->fop;
 
+    lock->unlock_now = _gf_false;
     ec_clear_inode_info(fop, lock->loc.inode);
 
     if ((lock->mask != 0) && lock->acquired) {
-        ec_owner_set(fop->frame, lock);
-
+        set_lk_owner_from_ptr(&lk_owner, lock);
         lock->flock.l_type = F_UNLCK;
         ec_trace("UNLOCK_INODELK", fop, "lock=%p, inode=%p", lock,
                  lock->loc.inode);
 
-        ec_inodelk(fop->frame, fop->xl, lock->mask, EC_MINIMUM_ONE,
+        ec_inodelk(fop->frame, fop->xl, &lk_owner, lock->mask, EC_MINIMUM_ONE,
                    ec_unlocked, link, fop->xl->name, &lock->loc, F_SETLK,
                    &lock->flock, NULL);
     } else {
@@ -1774,25 +2293,49 @@ void ec_unlock_lock(ec_lock_link_t *link)
     }
 }
 
-int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie,
-                                    xlator_t * this, int32_t op_ret,
-                                    int32_t op_errno, dict_t * xattr,
-                                    dict_t * xdata)
+void
+ec_inode_bad_inc(inode_t *inode, xlator_t *xl)
+{
+    ec_inode_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __ec_inode_get(inode, xl);
+        if (ctx == NULL) {
+            goto unlock;
+        }
+        ctx->bad_version++;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+}
+
+int32_t
+ec_update_size_version_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                            dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link;
     ec_lock_t *lock;
     ec_inode_t *ctx;
 
+    link = fop->data;
+    lock = link->lock;
+    ctx = lock->ctx;
+
     if (op_ret < 0) {
-        gf_msg(fop->xl->name, fop_log_level (fop->id, op_errno), op_errno,
+        if (link->lock->fd == NULL) {
+            ec_inode_bad_inc(link->lock->loc.inode, this);
+        } else {
+            ec_inode_bad_inc(link->lock->fd->inode, this);
+        }
+
+        gf_msg(fop->xl->name, fop_log_level(fop->id, op_errno), op_errno,
                EC_MSG_SIZE_VERS_UPDATE_FAIL,
-               "Failed to update version and size");
+               "Failed to update version and size. %s", ec_msg_str(fop));
     } else {
         fop->parent->good &= fop->good;
-        link = fop->data;
-        lock = link->lock;
-        ctx = lock->ctx;
 
         ec_lock_update_good(lock, fop);
 
@@ -1809,16 +2352,17 @@ int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie,
             ctx->have_size = _gf_true;
         }
         if ((ec_dict_del_config(xdata, EC_XATTR_CONFIG, &ctx->config) == 0) &&
-            ec_config_check(fop->parent, &ctx->config)) {
+            ec_config_check(fop->xl, &ctx->config)) {
             ctx->have_config = _gf_true;
         }
 
         ctx->have_info = _gf_true;
     }
-
-    if ((fop->parent->id != GF_FOP_FLUSH) &&
-        (fop->parent->id != GF_FOP_FSYNC) &&
-        (fop->parent->id != GF_FOP_FSYNCDIR)) {
+    /* If we are here because of fop's and other than unlock request,
+     * that means we are still holding a lock. That make sure
+     * lock->unlock_now can not be modified.
+     */
+    if (lock->unlock_now) {
         ec_unlock_lock(fop->data);
     }
 
@@ -1826,16 +2370,19 @@ int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie,
 }
 
 void
-ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
-                       uint64_t size, uint64_t *dirty)
+ec_update_size_version(ec_lock_link_t *link, uint64_t *version, uint64_t size,
+                       uint64_t *dirty)
 {
     ec_fop_data_t *fop;
     ec_lock_t *lock;
     ec_inode_t *ctx;
-    dict_t * dict;
+    dict_t *dict = NULL;
+    uintptr_t update_on = 0;
     int32_t err = -ENOMEM;
 
     fop = link->fop;
+    lock = link->lock;
+    ctx = lock->ctx;
 
     ec_trace("UPDATE", fop, "version=%ld/%ld, size=%ld, dirty=%ld/%ld",
              version[0], version[1], size, dirty[0], dirty[1]);
@@ -1845,9 +2392,6 @@ ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
         goto out;
     }
 
-    lock = link->lock;
-    ctx = lock->ctx;
-
     /* If we don't have version information or it has been modified, we
      * update it. */
     if (!ctx->have_version || (version[0] != 0) || (version[1] != 0)) {
@@ -1859,8 +2403,8 @@ ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
     }
 
     if (size != 0) {
-        /* If size has been changed, we should already know the previous size
-         * of the file. */
+        /* If size has been changed, we should already
+         * know the previous size of the file. */
         GF_ASSERT(ctx->have_size);
 
         err = ec_dict_set_number(dict, EC_XATTR_SIZE, size);
@@ -1869,9 +2413,7 @@ ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
         }
     }
 
-    /* If we don't have dirty information or it has been modified, we update
-     * it. */
-    if ((dirty[0] != 0) || (dirty[1] != 0)) {
+    if (dirty[0] || dirty[1]) {
         err = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
         if (err != 0) {
             goto out;
@@ -1882,20 +2424,22 @@ ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
     if ((lock->loc.inode->ia_type == IA_IFREG) && !ctx->have_config) {
         /* A failure requesting this xattr is ignored because it's not
          * absolutely required right now. */
-        ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
+        (void)ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
     }
 
     fop->frame->root->uid = 0;
     fop->frame->root->gid = 0;
 
+    update_on = lock->good_mask | lock->healing;
+
     if (link->lock->fd == NULL) {
-            ec_xattrop(fop->frame, fop->xl, lock->good_mask, EC_MINIMUM_MIN,
-                       ec_update_size_version_done, link, &link->lock->loc,
-                       GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ec_xattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
+                   ec_update_size_version_done, link, &link->lock->loc,
+                   GF_XATTROP_ADD_ARRAY64, dict, NULL);
     } else {
-            ec_fxattrop(fop->frame, fop->xl, lock->good_mask, EC_MINIMUM_MIN,
-                       ec_update_size_version_done, link, link->lock->fd,
-                       GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ec_fxattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
+                    ec_update_size_version_done, link, link->lock->fd,
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
     }
 
     fop->frame->root->uid = fop->uid;
@@ -1912,15 +2456,12 @@ out:
 
     ec_fop_set_error(fop, -err);
 
-    gf_msg (fop->xl->name, GF_LOG_ERROR, -err, EC_MSG_SIZE_VERS_UPDATE_FAIL,
-            "Unable to update version and size");
+    gf_msg(fop->xl->name, GF_LOG_ERROR, -err, EC_MSG_SIZE_VERS_UPDATE_FAIL,
+           "Unable to update version and size. %s", ec_msg_str(fop));
 
-    if ((fop->parent->id != GF_FOP_FLUSH) &&
-        (fop->parent->id != GF_FOP_FSYNC) &&
-        (fop->parent->id != GF_FOP_FSYNCDIR)) {
+    if (lock->unlock_now) {
         ec_unlock_lock(fop->data);
     }
-
 }
 
 gf_boolean_t
@@ -1928,28 +2469,62 @@ ec_update_info(ec_lock_link_t *link)
 {
     ec_lock_t *lock;
     ec_inode_t *ctx;
-    uint64_t version[2];
-    uint64_t dirty[2];
+    uint64_t version[2] = {0, 0};
+    uint64_t dirty[2] = {0, 0};
     uint64_t size;
+    ec_t *ec = NULL;
+    uintptr_t mask;
 
     lock = link->lock;
     ctx = lock->ctx;
+    ec = link->fop->xl->private;
 
     /* pre_version[*] will be 0 if have_version is false */
-    version[0] = ctx->post_version[0] - ctx->pre_version[0];
-    version[1] = ctx->post_version[1] - ctx->pre_version[1];
+    version[EC_DATA_TXN] = ctx->post_version[EC_DATA_TXN] -
+                           ctx->pre_version[EC_DATA_TXN];
+    version[EC_METADATA_TXN] = ctx->post_version[EC_METADATA_TXN] -
+                               ctx->pre_version[EC_METADATA_TXN];
 
     size = ctx->post_size - ctx->pre_size;
+    /* If we set the dirty flag for update fop, we have to unset it.
+     * If fop has failed on some bricks, leave the dirty as marked. */
+
+    if (lock->unlock_now) {
+        if (version[EC_DATA_TXN]) {
+            /*A data fop will have difference in post and pre version
+             *and for data fop we send writes on healing bricks also */
+            mask = lock->good_mask | lock->healing;
+        } else {
+            mask = lock->good_mask;
+        }
+        /* Ensure that nodes are up while doing final
+         * metadata update.*/
+        if (!(ec->node_mask & ~(mask)) && !(ec->node_mask & ~ec->xl_up)) {
+            if (ctx->dirty[EC_DATA_TXN] != 0) {
+                dirty[EC_DATA_TXN] = -1;
+            }
+            if (ctx->dirty[EC_METADATA_TXN] != 0) {
+                dirty[EC_METADATA_TXN] = -1;
+            }
+            /*If everything is fine and we already
+             *have version xattr set on entry, there
+             *is no need to update version again*/
+            if (ctx->pre_version[EC_DATA_TXN]) {
+                version[EC_DATA_TXN] = 0;
+            }
+            if (ctx->pre_version[EC_METADATA_TXN]) {
+                version[EC_METADATA_TXN] = 0;
+            }
+        } else {
+            link->optimistic_changelog = _gf_false;
+            ec_set_dirty_flag(link, ctx, dirty);
+        }
+        memset(ctx->dirty, 0, sizeof(ctx->dirty));
+    }
 
-    dirty[0] = ctx->dirty[0];
-    dirty[1] = ctx->dirty[1];
-    /*Dirty is not combined so just reset it right here*/
-    memset(ctx->dirty, 0, sizeof(ctx->dirty));
-
-    if ((version[0] != 0) || (version[1] != 0) ||
-        (dirty[0] != 0) || (dirty[1] != 0)) {
+    if ((version[EC_DATA_TXN] != 0) || (version[EC_METADATA_TXN] != 0) ||
+        (dirty[EC_DATA_TXN] != 0) || (dirty[EC_METADATA_TXN] != 0)) {
         ec_update_size_version(link, version, size, dirty);
-
         return _gf_true;
     }
 
@@ -1959,7 +2534,15 @@ ec_update_info(ec_lock_link_t *link)
 void
 ec_unlock_now(ec_lock_link_t *link)
 {
+    ec_lock_t *lock;
+    lock = link->lock;
+
     ec_trace("UNLOCK_NOW", link->fop, "lock=%p", link->lock);
+    /*At this point, lock is not being used by any fop and
+     *can not be reused by any fop as it is going to be released.
+     *lock->unlock_now can not be modified at any other place.
+     */
+    lock->unlock_now = _gf_true;
 
     if (!ec_update_info(link)) {
         ec_unlock_lock(link);
@@ -1969,66 +2552,197 @@ ec_unlock_now(ec_lock_link_t *link)
 }
 
 void
+ec_lock_release(ec_t *ec, inode_t *inode)
+{
+    ec_lock_t *lock;
+    ec_inode_t *ctx;
+    ec_lock_link_t *timer_link = NULL;
+
+    LOCK(&inode->lock);
+
+    ctx = __ec_inode_get(inode, ec->xl);
+    if (ctx == NULL) {
+        goto done;
+    }
+    lock = ctx->inode_lock;
+    if ((lock == NULL) || lock->release) {
+        goto done;
+    }
+
+    gf_msg_debug(ec->xl->name, 0, "Releasing inode %p due to lock contention",
+                 inode);
+
+    if (!lock->acquired) {
+        /* This happens if some bricks already got the lock while inodelk is in
+         * progress.  Set release to true after lock is acquired*/
+        lock->contention = _gf_true;
+        goto done;
+    }
+
+    /* The lock is not marked to be released, so the frozen list should be
+     * empty. */
+    GF_ASSERT(list_empty(&lock->frozen));
+
+    timer_link = ec_lock_timer_cancel(ec->xl, lock);
+
+    /* We mark the lock to be released as soon as possible. */
+    lock->release = _gf_true;
+
+done:
+    UNLOCK(&inode->lock);
+
+    /* If we have cancelled the timer, we need to start the unlock of the
+     * inode. If there was a timer but we have been unable to cancel it
+     * because it was just triggered, the timer callback will take care
+     * of releasing the inode. */
+    if (timer_link != NULL) {
+        ec_unlock_now(timer_link);
+    }
+}
+
+void
+ec_unlock_timer_add(ec_lock_link_t *link);
+
+void
 ec_unlock_timer_del(ec_lock_link_t *link)
 {
-        ec_lock_t *lock;
-        inode_t *inode;
-        gf_boolean_t now = _gf_false;
+    ec_lock_t *lock;
+    inode_t *inode;
+    gf_boolean_t now = _gf_false;
+
+    /* If we are here, it means that the timer has expired before having
+     * been cancelled. This guarantees that 'link' is still valid because
+     * the fop that contains it must be pending (if timer cancellation in
+     * ec_lock_assign_owner() fails, the fop is left sleeping).
+     *
+     * At the same time, the fop still has a reference to the lock, so
+     * it must also be valid.
+     */
+    lock = link->lock;
 
-        lock = link->lock;
+    /* 'lock' must have a valid inode since it can only be destroyed
+     * when the lock itself is destroyed, but we have a reference to the
+     * lock to avoid this.
+     */
+    inode = lock->loc.inode;
 
-        /* A race condition can happen if timer expires, calls this function
-         * and the lock is released (lock->loc is wiped) but the fop is not
-         * fully completed yet (it's still on the list of pending fops). In
-         * this case, this function can also be called if ec_unlock_force() is
-         * called. */
-        inode = lock->loc.inode;
-        if (inode == NULL) {
-                return;
-        }
+    LOCK(&inode->lock);
+
+    if (lock->timer != NULL) {
+        ec_trace("UNLOCK_DELAYED", link->fop, "lock=%p", lock);
+
+        /* The unlock timer has expired without anyone cancelling it.
+         * This means that it shouldn't have any owner, and the waiting
+         * and frozen lists should be empty.  It must have only one
+         * owner reference, but there can be fops being prepared
+         * though.
+         * */
+        GF_ASSERT(!lock->release && (lock->refs_owners == 1) &&
+                  list_empty(&lock->owners) && list_empty(&lock->waiting) &&
+                  list_empty(&lock->frozen));
+
+        gf_timer_call_cancel(link->fop->xl->ctx, lock->timer);
+        lock->timer = NULL;
 
-        LOCK(&inode->lock);
+        /* Any fop being processed from now on, will need to wait
+         * until the next unlock/lock cycle. */
+        lock->release = now = _gf_true;
+    }
 
-        if (lock->timer != NULL) {
-                ec_trace("UNLOCK_DELAYED", link->fop, "lock=%p", lock);
+    UNLOCK(&inode->lock);
 
-                /* The unlock timer has expired without anyone cancelling it.
-                 * This means that it shouldn't have any owner, and the
-                 * waiting and frozen lists should be empty. It shouldn't have
-                 * been marked as release nor be exclusive either. It must have
-                 * only one owner reference, but there can be fops being
-                 * prepared though. */
-                GF_ASSERT(!lock->release && (lock->exclusive == 0) &&
-                          (lock->refs_owners == 1) &&
-                          list_empty(&lock->owners) &&
-                          list_empty(&lock->waiting) &&
-                          list_empty(&lock->frozen));
+    if (now) {
+        ec_unlock_now(link);
+    } else {
+        /* The timer has been cancelled just after firing it but before
+         * getting here. This means that another fop has used the lock
+         * and everything should be handled as if this callback were
+         * have not been executed. However we still have an owner
+         * reference.
+         *
+         * We need to release our reference. If this is not the last
+         * reference (the most common case because another fop has
+         * taken another ref) we only need to decrement the counter.
+         * Otherwise we have been delayed enough so that the other fop
+         * has had time to acquire the reference, do its operation and
+         * release it. At the time of releasing it, the fop did found
+         * that the ref counter was > 1 (our reference), so the delayed
+         * unlock timer wasn't started. We need to start it again if we
+         * are the last reference.
+         *
+         * ec_unlock_timer_add() handles both cases.
+         */
+        ec_unlock_timer_add(link);
 
-                gf_timer_call_cancel(link->fop->xl->ctx, lock->timer);
-                lock->timer = NULL;
+        /* We need to resume the fop that was waiting for the delayed
+         * unlock.
+         */
+        ec_resume(link->fop, 0);
+    }
+}
 
-                /* Any fop being processed from now on, will need to wait
-                 * until the next unlock/lock cycle. */
-                lock->release = now = _gf_true;
-        }
+void
+ec_unlock_timer_cbk(void *data)
+{
+    ec_unlock_timer_del(data);
+}
 
-        UNLOCK(&inode->lock);
+static gf_boolean_t
+ec_eager_lock_used(ec_t *ec, ec_fop_data_t *fop)
+{
+    /* Fops with no locks at this point mean that they are sent as sub-fops
+     * of other higher level fops. In this case we simply assume that the
+     * parent fop will take correct care of the eager lock. */
+    if (fop->lock_count == 0) {
+        return _gf_true;
+    }
 
-        if (now) {
-                ec_unlock_now(link);
-        }
+    /* We may have more than one lock, but this only happens in the rename
+     * fop, and both locks will reference an inode of the same type (a
+     * directory in this case), so we only need to check the first lock. */
+    if (fop->locks[0].lock->loc.inode->ia_type == IA_IFREG) {
+        return ec->eager_lock;
+    }
+
+    return ec->other_eager_lock;
 }
 
-void ec_unlock_timer_cbk(void *data)
+static uint32_t
+ec_eager_lock_timeout(ec_t *ec, ec_lock_t *lock)
 {
-        ec_unlock_timer_del(data);
+    if (lock->loc.inode->ia_type == IA_IFREG) {
+        return ec->eager_lock_timeout;
+    }
+
+    return ec->other_eager_lock_timeout;
 }
 
-void ec_unlock_timer_add(ec_lock_link_t *link)
+static gf_boolean_t
+ec_lock_delay_create(ec_lock_link_t *link)
 {
     struct timespec delay;
     ec_fop_data_t *fop = link->fop;
     ec_lock_t *lock = link->lock;
+
+    delay.tv_sec = ec_eager_lock_timeout(fop->xl->private, lock);
+    delay.tv_nsec = 0;
+    lock->timer = gf_timer_call_after(fop->xl->ctx, delay, ec_unlock_timer_cbk,
+                                      link);
+    if (lock->timer == NULL) {
+        gf_msg(fop->xl->name, GF_LOG_WARNING, ENOMEM,
+               EC_MSG_UNLOCK_DELAY_FAILED, "Unable to delay an unlock");
+
+        return _gf_false;
+    }
+
+    return _gf_true;
+}
+
+void
+ec_unlock_timer_add(ec_lock_link_t *link)
+{
+    ec_fop_data_t *fop = link->fop;
+    ec_lock_t *lock = link->lock;
     gf_boolean_t now = _gf_false;
 
     LOCK(&lock->loc.inode->lock);
@@ -2080,19 +2794,12 @@ void ec_unlock_timer_add(ec_lock_link_t *link)
             ec_trace("UNLOCK_DELAY", fop, "lock=%p, release=%d", lock,
                      lock->release);
 
-            delay.tv_sec = 1;
-            delay.tv_nsec = 0;
-            lock->timer = gf_timer_call_after(fop->xl->ctx, delay,
-                                              ec_unlock_timer_cbk, link);
-            if (lock->timer == NULL) {
-                gf_msg(fop->xl->name, GF_LOG_WARNING, ENOMEM,
-                       EC_MSG_UNLOCK_DELAY_FAILED,
-                       "Unable to delay an unlock");
-
+            if (!ec_lock_delay_create(link)) {
                 /* We are unable to create a new timer. We immediately release
                  * the lock. */
                 lock->release = now = _gf_true;
             }
+
         } else {
             ec_trace("UNLOCK_FORCE", fop, "lock=%p, release=%d", lock,
                      lock->release);
@@ -2122,7 +2829,8 @@ void ec_unlock_timer_add(ec_lock_link_t *link)
     }
 }
 
-void ec_unlock(ec_fop_data_t *fop)
+void
+ec_unlock(ec_fop_data_t *fop)
 {
     int32_t i;
 
@@ -2131,13 +2839,109 @@ void ec_unlock(ec_fop_data_t *fop)
     }
 }
 
-void ec_flush_size_version(ec_fop_data_t * fop)
+void
+ec_flush_size_version(ec_fop_data_t *fop)
 {
     GF_ASSERT(fop->lock_count == 1);
     ec_update_info(&fop->locks[0]);
 }
 
-void ec_lock_reuse(ec_fop_data_t *fop)
+static void
+ec_update_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache, ec_stripe_t *stripe,
+                 ec_fop_data_t *fop)
+{
+    off_t base;
+
+    /* On write fops, we only update existing fragments if the write has
+     * succeeded. Otherwise, we remove them from the cache. */
+    if ((fop->id == GF_FOP_WRITE) && (fop->answer != NULL) &&
+        (fop->answer->op_ret >= 0)) {
+        base = stripe->frag_offset - fop->frag_range.first;
+        base *= ec->fragments;
+
+        /* We check if the stripe offset falls inside the real region
+         * modified by the write fop (a write request is allowed,
+         * though uncommon, to write less bytes than requested). The
+         * current write fop implementation doesn't allow partial
+         * writes of fragments, so if there's no error, we are sure
+         * that a full stripe has been completely modified or not
+         * touched at all. The value of op_ret may not be a multiple
+         * of the stripe size because it depends on the requested
+         * size by the user, so we update the stripe if the write has
+         * modified at least one byte (meaning ec has written the full
+         * stripe). */
+        if (base < fop->answer->op_ret + fop->head) {
+            memcpy(stripe->data, fop->vector[0].iov_base + base,
+                   ec->stripe_size);
+            list_move_tail(&stripe->lru, &stripe_cache->lru);
+
+            GF_ATOMIC_INC(ec->stats.stripe_cache.updates);
+        }
+    } else {
+        stripe->frag_offset = -1;
+        list_move(&stripe->lru, &stripe_cache->lru);
+
+        GF_ATOMIC_INC(ec->stats.stripe_cache.invals);
+    }
+}
+
+static void
+ec_update_cached_stripes(ec_fop_data_t *fop)
+{
+    uint64_t first;
+    uint64_t last;
+    ec_stripe_t *stripe = NULL;
+    ec_inode_t *ctx = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+    inode_t *inode = NULL;
+    struct list_head *temp;
+    struct list_head sentinel;
+
+    first = fop->frag_range.first;
+    /* 'last' represents the first stripe not touched by the operation */
+    last = fop->frag_range.last;
+
+    /* If there are no modified stripes, we don't need to do anything
+     * else. */
+    if (last <= first) {
+        return;
+    }
+
+    if (!fop->use_fd) {
+        inode = fop->loc[0].inode;
+    } else {
+        inode = fop->fd->inode;
+    }
+
+    LOCK(&inode->lock);
+
+    ctx = __ec_inode_get(inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
+    }
+    stripe_cache = &ctx->stripe_cache;
+
+    /* Since we'll be moving elements of the list to the tail, we might
+     * end in an infinite loop. To avoid it, we insert a sentinel element
+     * into the list, so that it will be used to detect when we have
+     * traversed all existing elements once. */
+    list_add_tail(&sentinel, &stripe_cache->lru);
+    temp = stripe_cache->lru.next;
+    while (temp != &sentinel) {
+        stripe = list_entry(temp, ec_stripe_t, lru);
+        temp = temp->next;
+        if ((first <= stripe->frag_offset) && (stripe->frag_offset < last)) {
+            ec_update_stripe(fop->xl->private, stripe_cache, stripe, fop);
+        }
+    }
+    list_del(&sentinel);
+
+out:
+    UNLOCK(&inode->lock);
+}
+
+void
+ec_lock_reuse(ec_fop_data_t *fop)
 {
     ec_cbk_data_t *cbk;
     ec_t *ec = NULL;
@@ -2146,15 +2950,15 @@ void ec_lock_reuse(ec_fop_data_t *fop)
     ec = fop->xl->private;
     cbk = fop->answer;
 
-    if (ec->eager_lock && cbk != NULL) {
+    if (ec_eager_lock_used(ec, fop) && cbk != NULL) {
         if (cbk->xdata != NULL) {
-            if ((dict_get_int32(cbk->xdata, GLUSTERFS_INODELK_COUNT,
-                                &count) == 0) && (count > 1)) {
+            if ((dict_get_int32(cbk->xdata, GLUSTERFS_INODELK_COUNT, &count) ==
+                 0) &&
+                (count > 1)) {
                 release = _gf_true;
             }
             if (release) {
-                gf_msg_debug (fop->xl->name, 0,
-                       "Lock contention detected");
+                gf_msg_debug(fop->xl->name, 0, "Lock contention detected");
             }
         }
     } else {
@@ -2163,23 +2967,25 @@ void ec_lock_reuse(ec_fop_data_t *fop)
          * the lock. */
         release = _gf_true;
     }
+    ec_update_cached_stripes(fop);
 
     for (i = 0; i < fop->lock_count; i++) {
         ec_lock_next_owner(&fop->locks[i], cbk, release);
     }
 }
 
-void __ec_manager(ec_fop_data_t * fop, int32_t error)
+void
+__ec_manager(ec_fop_data_t *fop, int32_t error)
 {
     ec_t *ec = fop->xl->private;
 
     do {
         ec_trace("MANAGER", fop, "error=%d", error);
 
-        if (!ec_must_wind (fop)) {
-                if (ec->xl_up_count < ec->fragments) {
-                    error = ENOTCONN;
-                }
+        if (!ec_must_wind(fop)) {
+            if (ec->xl_up_count < ec->fragments) {
+                error = ENOTCONN;
+            }
         }
 
         if (error != 0) {
@@ -2205,22 +3011,32 @@ void __ec_manager(ec_fop_data_t * fop, int32_t error)
         fop->jobs = 1;
 
         fop->state = fop->handler(fop, fop->state);
-        GF_ASSERT (fop->state >= 0);
+        GF_ASSERT(fop->state >= 0);
 
         error = ec_check_complete(fop, __ec_manager);
     } while (error >= 0);
 }
 
-void ec_manager(ec_fop_data_t * fop, int32_t error)
+void
+ec_manager(ec_fop_data_t *fop, int32_t error)
 {
     GF_ASSERT(fop->jobs == 0);
     GF_ASSERT(fop->winds == 0);
     GF_ASSERT(fop->error == 0);
 
-    if (fop->state == EC_STATE_START)
-    {
+    if (fop->state == EC_STATE_START) {
         fop->state = EC_STATE_INIT;
     }
 
     __ec_manager(fop, error);
 }
+
+gf_boolean_t
+__ec_is_last_fop(ec_t *ec)
+{
+    if ((list_empty(&ec->pending_fops)) &&
+        (GF_ATOMIC_GET(ec->async_fop_count) == 0)) {
+        return _gf_true;
+    }
+    return _gf_false;
+}
diff --git a/xlators/cluster/ec/src/ec-common.h b/xlators/cluster/ec/src/ec-common.h
index 8e724a81380..51493612ac6 100644
--- a/xlators/cluster/ec/src/ec-common.h
+++ b/xlators/cluster/ec/src/ec-common.h
@@ -11,110 +11,224 @@
 #ifndef __EC_COMMON_H__
 #define __EC_COMMON_H__
 
-#include "xlator.h"
-
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
 #include "ec-data.h"
 
-typedef enum {
-        EC_DATA_TXN,
-        EC_METADATA_TXN
-} ec_txn_t;
+typedef enum { EC_DATA_TXN, EC_METADATA_TXN } ec_txn_t;
 
-#define EC_FOP_HEAL     -1
-#define EC_FOP_FHEAL    -2
+#define EC_FOP_HEAL -1
+#define EC_FOP_FHEAL -2
 
 #define EC_CONFIG_VERSION 0
 
 #define EC_CONFIG_ALGORITHM 0
 
-#define EC_FLAG_LOCK_SHARED       0x0001
-#define EC_FLAG_WAITING_SIZE      0x0002
+#define EC_FLAG_LOCK_SHARED 0x0001
+
+#define QUORUM_CBK(fn, fop, frame, cookie, this, op_ret, op_errno, params...)  \
+    do {                                                                       \
+        ec_t *__ec = fop->xl->private;                                         \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+        int32_t __success_count = gf_bits_count(fop->good);                    \
+                                                                               \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (!fop->parent && frame &&                                           \
+            (GF_CLIENT_PID_SELF_HEALD != frame->root->pid) &&                  \
+            __ec->quorum_count && (__success_count < __ec->quorum_count) &&    \
+            op_ret >= 0) {                                                     \
+            __op_ret = -1;                                                     \
+            __op_errno = EIO;                                                  \
+            gf_msg(__ec->xl->name, GF_LOG_ERROR, 0,                            \
+                   EC_MSG_CHILDS_INSUFFICIENT,                                 \
+                   "Insufficient available children for this request "         \
+                   "(have %d, need %d). %s",                                   \
+                   __success_count, __ec->quorum_count, ec_msg_str(fop));      \
+        }                                                                      \
+        fn(frame, cookie, this, __op_ret, __op_errno, params);                 \
+    } while (0)
+
+enum _ec_xattrop_flags {
+    EC_FLAG_XATTROP,
+    EC_FLAG_DATA_DIRTY,
+    EC_FLAG_METADATA_DIRTY,
+
+    /* Add any new flag here, before EC_FLAG_MAX. The maximum number of
+     * flags that can be defined is 16. */
+
+    EC_FLAG_MAX
+};
+
+/* We keep two sets of flags. One to determine what's really providing the
+ * current xattrop and the other to know what the parent fop of the xattrop
+ * needs to proceed. It might happen that a fop needs some information that
+ * is being already requested by a previous fop. The two sets are stored
+ * contiguously. */
+
+#define EC_FLAG_NEEDS(_flag) (1 << (_flag))
+#define EC_FLAG_PROVIDES(_flag) (1 << ((_flag) + EC_FLAG_MAX))
+
+#define EC_NEEDED_FLAGS(_flags) ((_flags) & ((1 << EC_FLAG_MAX) - 1))
+
+#define EC_PROVIDED_FLAGS(_flags) EC_NEEDED_FLAGS((_flags) >> EC_FLAG_MAX)
+
+#define EC_FLAGS_HAVE(_flags, _flag) (((_flags) & (1 << (_flag))) != 0)
 
 #define EC_SELFHEAL_BIT 62
 
-#define EC_MINIMUM_ONE   -1
-#define EC_MINIMUM_MIN   -2
-#define EC_MINIMUM_ALL   -3
-
-#define EC_UPDATE_DATA   1
-#define EC_UPDATE_META   2
-#define EC_QUERY_INFO    4
-#define EC_INODE_SIZE    8
-
-#define EC_STATE_START                        0
-#define EC_STATE_END                          0
-#define EC_STATE_INIT                         1
-#define EC_STATE_LOCK                         2
-#define EC_STATE_DISPATCH                     3
-#define EC_STATE_PREPARE_ANSWER               4
-#define EC_STATE_REPORT                       5
-#define EC_STATE_LOCK_REUSE                   6
-#define EC_STATE_UNLOCK                       7
-
-#define EC_STATE_DELAYED_START              100
-
-#define EC_STATE_HEAL_ENTRY_LOOKUP          200
-#define EC_STATE_HEAL_ENTRY_PREPARE         201
-#define EC_STATE_HEAL_PRE_INODELK_LOCK      202
-#define EC_STATE_HEAL_PRE_INODE_LOOKUP      203
-#define EC_STATE_HEAL_XATTRIBUTES_REMOVE    204
-#define EC_STATE_HEAL_XATTRIBUTES_SET       205
-#define EC_STATE_HEAL_ATTRIBUTES            206
-#define EC_STATE_HEAL_OPEN                  207
-#define EC_STATE_HEAL_REOPEN_FD             208
-#define EC_STATE_HEAL_UNLOCK                209
-#define EC_STATE_HEAL_UNLOCK_ENTRY          210
-#define EC_STATE_HEAL_DATA_LOCK             211
-#define EC_STATE_HEAL_DATA_COPY             212
-#define EC_STATE_HEAL_DATA_UNLOCK           213
-#define EC_STATE_HEAL_POST_INODELK_LOCK     214
-#define EC_STATE_HEAL_POST_INODE_LOOKUP     215
-#define EC_STATE_HEAL_SETATTR               216
-#define EC_STATE_HEAL_POST_INODELK_UNLOCK   217
-#define EC_STATE_HEAL_DISPATCH              218
-
-gf_boolean_t ec_dispatch_one_retry (ec_fop_data_t *fop, ec_cbk_data_t **cbk);
-int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx);
-
-void ec_complete(ec_fop_data_t *fop);
-
-void ec_update_good(ec_fop_data_t *fop, uintptr_t good);
-
-void ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
-
-ec_cbk_data_t *
-ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro);
+#define EC_MINIMUM_ONE (1 << 6)
+#define EC_MINIMUM_MIN (2 << 6)
+#define EC_MINIMUM_ALL (3 << 6)
+#define EC_FOP_NO_PROPAGATE_ERROR (1 << 8)
+#define EC_FOP_MINIMUM(_flags) ((_flags)&255)
+#define EC_FOP_FLAGS(_flags) ((_flags) & ~255)
+
+#define EC_UPDATE_DATA 1
+#define EC_UPDATE_META 2
+#define EC_QUERY_INFO 4
+#define EC_INODE_SIZE 8
+
+#define EC_STATE_START 0
+#define EC_STATE_END 0
+#define EC_STATE_INIT 1
+#define EC_STATE_LOCK 2
+#define EC_STATE_DISPATCH 3
+#define EC_STATE_PREPARE_ANSWER 4
+#define EC_STATE_REPORT 5
+#define EC_STATE_LOCK_REUSE 6
+#define EC_STATE_UNLOCK 7
+
+#define EC_STATE_DELAYED_START 100
+
+#define EC_STATE_HEAL_ENTRY_LOOKUP 200
+#define EC_STATE_HEAL_ENTRY_PREPARE 201
+#define EC_STATE_HEAL_PRE_INODELK_LOCK 202
+#define EC_STATE_HEAL_PRE_INODE_LOOKUP 203
+#define EC_STATE_HEAL_XATTRIBUTES_REMOVE 204
+#define EC_STATE_HEAL_XATTRIBUTES_SET 205
+#define EC_STATE_HEAL_ATTRIBUTES 206
+#define EC_STATE_HEAL_OPEN 207
+#define EC_STATE_HEAL_REOPEN_FD 208
+#define EC_STATE_HEAL_UNLOCK 209
+#define EC_STATE_HEAL_UNLOCK_ENTRY 210
+#define EC_STATE_HEAL_DATA_LOCK 211
+#define EC_STATE_HEAL_DATA_COPY 212
+#define EC_STATE_HEAL_DATA_UNLOCK 213
+#define EC_STATE_HEAL_POST_INODELK_LOCK 214
+#define EC_STATE_HEAL_POST_INODE_LOOKUP 215
+#define EC_STATE_HEAL_SETATTR 216
+#define EC_STATE_HEAL_POST_INODELK_UNLOCK 217
+#define EC_STATE_HEAL_DISPATCH 218
+
+/* Value to cover the full range of a file */
+#define EC_RANGE_FULL ((uint64_t)LLONG_MAX + 1)
 
 gf_boolean_t
-ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro);
+ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk);
+void
+ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx);
 
-void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags);
-void ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc,
-                                  uint32_t flags);
-void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags);
-void ec_lock(ec_fop_data_t * fop);
-void ec_lock_reuse(ec_fop_data_t *fop);
-void ec_unlock(ec_fop_data_t * fop);
+void
+ec_complete(ec_fop_data_t *fop);
 
-gf_boolean_t ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                               uint64_t *size);
-gf_boolean_t ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                               uint64_t size);
-void ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode);
+void
+ec_update_good(ec_fop_data_t *fop, uintptr_t good);
 
-void ec_flush_size_version(ec_fop_data_t * fop);
+void
+ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
 
-void ec_dispatch_all(ec_fop_data_t * fop);
-void ec_dispatch_inc(ec_fop_data_t * fop);
-void ec_dispatch_min(ec_fop_data_t * fop);
-void ec_dispatch_one(ec_fop_data_t * fop);
+void
+__ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
 
-void ec_sleep(ec_fop_data_t *fop);
-void ec_resume(ec_fop_data_t * fop, int32_t error);
-void ec_resume_parent(ec_fop_data_t * fop, int32_t error);
+ec_cbk_data_t *
+ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro);
 
-void ec_manager(ec_fop_data_t * fop, int32_t error);
-gf_boolean_t ec_is_recoverable_error (int32_t op_errno);
-void ec_handle_healers_done (ec_fop_data_t *fop);
+gf_boolean_t
+ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro);
 
+void
+ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                      off_t fl_start, uint64_t fl_size);
+void
+ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
+                             uint32_t flags);
+void
+ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start,
+                   uint64_t fl_size);
+void
+ec_lock(ec_fop_data_t *fop);
+void
+ec_lock_reuse(ec_fop_data_t *fop);
+void
+ec_unlock(ec_fop_data_t *fop);
+void
+ec_lock_release(ec_t *ec, inode_t *inode);
+
+gf_boolean_t
+ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size);
+gf_boolean_t
+__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size);
+gf_boolean_t
+ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size);
+gf_boolean_t
+__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size);
+void
+ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode);
+
+void
+ec_flush_size_version(ec_fop_data_t *fop);
+
+void
+ec_dispatch_all(ec_fop_data_t *fop);
+void
+ec_dispatch_inc(ec_fop_data_t *fop);
+void
+ec_dispatch_min(ec_fop_data_t *fop);
+void
+ec_dispatch_one(ec_fop_data_t *fop);
+
+void
+ec_succeed_all(ec_fop_data_t *fop);
+
+void
+ec_sleep(ec_fop_data_t *fop);
+void
+ec_resume(ec_fop_data_t *fop, int32_t error);
+void
+ec_resume_parent(ec_fop_data_t *fop);
+
+void
+ec_manager(ec_fop_data_t *fop, int32_t error);
+gf_boolean_t
+ec_is_recoverable_error(int32_t op_errno);
+void
+ec_handle_healers_done(ec_fop_data_t *fop);
+
+int32_t
+ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *locked_on, gf_boolean_t self_locked,
+                gf_boolean_t thorough, ec_heal_need_t *need_heal);
+int32_t
+ec_get_heal_info(xlator_t *this, loc_t *loc, dict_t **dict);
+
+int32_t
+ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+void
+ec_update_fd_status(fd_t *fd, xlator_t *xl, int child_index,
+                    int32_t ret_status);
+gf_boolean_t
+ec_is_entry_healing(ec_fop_data_t *fop);
+void
+ec_set_entry_healing(ec_fop_data_t *fop);
+void
+ec_reset_entry_healing(ec_fop_data_t *fop);
+char *
+ec_msg_str(ec_fop_data_t *fop);
+gf_boolean_t
+__ec_is_last_fop(ec_t *ec);
+void
+ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop);
 #endif /* __EC_COMMON_H__ */
diff --git a/xlators/cluster/ec/src/ec-data.c b/xlators/cluster/ec/src/ec-data.c
index 28bf988d09d..06388833546 100644
--- a/xlators/cluster/ec/src/ec-data.c
+++ b/xlators/cluster/ec/src/ec-data.c
@@ -8,53 +8,49 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "ec-mem-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-data.h"
 #include "ec-messages.h"
 
-ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     ec_fop_data_t * fop, int32_t id,
-                                     int32_t idx, int32_t op_ret,
-                                     int32_t op_errno)
+ec_cbk_data_t *
+ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop,
+                     int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno)
 {
-    ec_cbk_data_t * cbk;
-    ec_t * ec = this->private;
+    ec_cbk_data_t *cbk;
+    ec_t *ec = this->private;
 
-    if (fop->xl != this)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_XLATOR_MISMATCH, "Mismatching xlators between request "
-                "and answer (req=%s, ans=%s).", fop->xl->name, this->name);
+    if (fop->xl != this) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_MISMATCH,
+               "Mismatching xlators between request "
+               "and answer (req=%s, ans=%s).",
+               fop->xl->name, this->name);
 
         return NULL;
     }
-    if (fop->frame != frame)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_FRAME_MISMATCH, "Mismatching frames between request "
-                                         "and answer (req=%p, ans=%p).",
-                                         fop->frame, frame);
+    if (fop->frame != frame) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FRAME_MISMATCH,
+               "Mismatching frames between request "
+               "and answer (req=%p, ans=%p).",
+               fop->frame, frame);
 
         return NULL;
     }
-    if (fop->id != id)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_FOP_MISMATCH, "Mismatching fops between request "
-                                         "and answer (req=%d, ans=%d).",
-                                         fop->id, id);
+    if (fop->id != id) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FOP_MISMATCH,
+               "Mismatching fops between request "
+               "and answer (req=%d, ans=%d).",
+               fop->id, id);
 
         return NULL;
     }
 
     cbk = mem_get0(ec->cbk_pool);
-    if (cbk == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to allocate memory for an "
-                                         "answer.");
+    if (cbk == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate memory for an "
+               "answer.");
+        return NULL;
     }
 
     cbk->fop = fop;
@@ -63,7 +59,7 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
     cbk->count = 1;
     cbk->op_ret = op_ret;
     cbk->op_errno = op_errno;
-    INIT_LIST_HEAD (&cbk->entries.list);
+    INIT_LIST_HEAD(&cbk->entries.list);
 
     LOCK(&fop->lock);
 
@@ -74,63 +70,45 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
     return cbk;
 }
 
-void ec_cbk_data_destroy(ec_cbk_data_t * cbk)
+void
+ec_cbk_data_destroy(ec_cbk_data_t *cbk)
 {
-    if (cbk->xdata != NULL)
-    {
+    if (cbk->xdata != NULL) {
         dict_unref(cbk->xdata);
     }
-    if (cbk->dict != NULL)
-    {
+    if (cbk->dict != NULL) {
         dict_unref(cbk->dict);
     }
-    if (cbk->inode != NULL)
-    {
+    if (cbk->inode != NULL) {
         inode_unref(cbk->inode);
     }
-    if (cbk->fd != NULL)
-    {
+    if (cbk->fd != NULL) {
         fd_unref(cbk->fd);
     }
-    if (cbk->buffers != NULL)
-    {
+    if (cbk->buffers != NULL) {
         iobref_unref(cbk->buffers);
     }
     GF_FREE(cbk->vector);
-    gf_dirent_free (&cbk->entries);
-    GF_FREE (cbk->str);
+    gf_dirent_free(&cbk->entries);
+    GF_FREE(cbk->str);
 
     mem_put(cbk);
 }
 
-/* PARENT_DOWN will be notified to children only after these fops are complete
- * when graph switch happens.  We do not want graph switch to be waiting on
- * heal to complete as healing big file/directory could take a while. Which
- * will lead to hang on the mount.
- */
-static gf_boolean_t
-ec_needs_graceful_completion (ec_fop_data_t *fop)
+ec_fop_data_t *
+ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id,
+                     uint32_t flags, uintptr_t target, uint32_t fop_flags,
+                     ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks,
+                     void *data)
 {
-        if ((fop->id != EC_FOP_HEAL) && (fop->id != EC_FOP_FHEAL))
-                return _gf_true;
-        return _gf_false;
-}
-
-ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     int32_t id, uint32_t flags,
-                                     uintptr_t target, int32_t minimum,
-                                     ec_wind_f wind, ec_handler_f handler,
-                                     ec_cbk_t cbks, void * data)
-{
-    ec_fop_data_t * fop, * parent;
-    ec_t * ec = this->private;
+    ec_fop_data_t *fop, *parent;
+    ec_t *ec = this->private;
 
     fop = mem_get0(ec->fop_pool);
-    if (fop == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to allocate memory for a "
-                                         "request.");
+    if (fop == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate memory for a "
+               "request.");
 
         return NULL;
     }
@@ -154,19 +132,15 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
      * TODO: minimize usage of private frames. Reuse req_frame as much as
      *       possible.
      */
-    if (frame != NULL)
-    {
+    if (frame != NULL) {
         fop->frame = copy_frame(frame);
-    }
-    else
-    {
+    } else {
         fop->frame = create_frame(this, this->ctx->pool);
     }
-    if (fop->frame == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to create a private frame "
-                                         "for a request");
+    if (fop->frame == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to create a private frame "
+               "for a request");
 
         mem_put(fop);
 
@@ -176,7 +150,8 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
     fop->refs = 1;
 
     fop->flags = flags;
-    fop->minimum = minimum;
+    fop->minimum = EC_FOP_MINIMUM(fop_flags);
+    fop->fop_flags = EC_FOP_FLAGS(fop_flags);
     fop->mask = target;
 
     fop->wind = wind;
@@ -191,29 +166,26 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
 
     fop->frame->local = fop;
 
-    if (frame != NULL)
-    {
+    if (frame != NULL) {
         parent = frame->local;
-        if (parent != NULL)
-        {
+        if (parent != NULL) {
             ec_sleep(parent);
         }
 
         fop->parent = parent;
     }
 
-    if (ec_needs_graceful_completion (fop)) {
-            LOCK(&ec->lock);
+    LOCK(&ec->lock);
 
-            list_add_tail(&fop->pending_list, &ec->pending_fops);
+    list_add_tail(&fop->pending_list, &ec->pending_fops);
 
-            UNLOCK(&ec->lock);
-    }
+    UNLOCK(&ec->lock);
 
     return fop;
 }
 
-void ec_fop_data_acquire(ec_fop_data_t * fop)
+void
+ec_fop_data_acquire(ec_fop_data_t *fop)
 {
     LOCK(&fop->lock);
 
@@ -225,36 +197,40 @@ void ec_fop_data_acquire(ec_fop_data_t * fop)
 }
 
 static void
-ec_handle_last_pending_fop_completion (ec_fop_data_t *fop, gf_boolean_t *notify)
+ec_handle_last_pending_fop_completion(ec_fop_data_t *fop, gf_boolean_t *notify)
 {
-        ec_t *ec = fop->xl->private;
-
-        if (!list_empty (&fop->pending_list)) {
-                LOCK(&ec->lock);
-                {
-                        list_del_init (&fop->pending_list);
-                        *notify = list_empty (&ec->pending_fops);
-                }
-                UNLOCK(&ec->lock);
+    ec_t *ec = fop->xl->private;
+
+    *notify = _gf_false;
+
+    if (!list_empty(&fop->pending_list)) {
+        LOCK(&ec->lock);
+        {
+            list_del_init(&fop->pending_list);
+            *notify = __ec_is_last_fop(ec);
         }
+        UNLOCK(&ec->lock);
+    }
 }
 
 void
 ec_fop_cleanup(ec_fop_data_t *fop)
 {
-        ec_cbk_data_t *cbk, *tmp;
+    ec_cbk_data_t *cbk, *tmp;
 
-        list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list) {
-            list_del_init(&cbk->answer_list);
+    list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list)
+    {
+        list_del_init(&cbk->answer_list);
 
-            ec_cbk_data_destroy(cbk);
-        }
-        INIT_LIST_HEAD(&fop->cbk_list);
+        ec_cbk_data_destroy(cbk);
+    }
+    INIT_LIST_HEAD(&fop->cbk_list);
 
-        fop->answer = NULL;
+    fop->answer = NULL;
 }
 
-void ec_fop_data_release(ec_fop_data_t * fop)
+void
+ec_fop_data_release(ec_fop_data_t *fop)
 {
     ec_t *ec = NULL;
     int32_t refs;
@@ -264,36 +240,30 @@ void ec_fop_data_release(ec_fop_data_t * fop)
 
     ec_trace("RELEASE", fop, "");
 
-    GF_ASSERT (fop->refs > 0);
+    GF_ASSERT(fop->refs > 0);
     refs = --fop->refs;
 
     UNLOCK(&fop->lock);
 
-    if (refs == 0)
-    {
+    if (refs == 0) {
         fop->frame->local = NULL;
         STACK_DESTROY(fop->frame->root);
 
         LOCK_DESTROY(&fop->lock);
 
-        if (fop->xdata != NULL)
-        {
+        if (fop->xdata != NULL) {
             dict_unref(fop->xdata);
         }
-        if (fop->dict != NULL)
-        {
+        if (fop->dict != NULL) {
             dict_unref(fop->dict);
         }
-        if (fop->inode != NULL)
-        {
+        if (fop->inode != NULL) {
             inode_unref(fop->inode);
         }
-        if (fop->fd != NULL)
-        {
+        if (fop->fd != NULL) {
             fd_unref(fop->fd);
         }
-        if (fop->buffers != NULL)
-        {
+        if (fop->buffers != NULL) {
             iobref_unref(fop->buffers);
         }
         GF_FREE(fop->vector);
@@ -301,14 +271,15 @@ void ec_fop_data_release(ec_fop_data_t * fop)
         GF_FREE(fop->str[1]);
         loc_wipe(&fop->loc[0]);
         loc_wipe(&fop->loc[1]);
+        GF_FREE(fop->errstr);
 
-        ec_resume_parent(fop, fop->error);
+        ec_resume_parent(fop);
 
         ec_fop_cleanup(fop);
 
         ec = fop->xl->private;
-        ec_handle_last_pending_fop_completion (fop, &notify);
-        ec_handle_healers_done (fop);
+        ec_handle_last_pending_fop_completion(fop, &notify);
+        ec_handle_healers_done(fop);
         mem_put(fop);
         if (notify) {
             ec_pending_fops_completed(ec);
diff --git a/xlators/cluster/ec/src/ec-data.h b/xlators/cluster/ec/src/ec-data.h
index 4a2a11f4ccd..c8a74ffe1ed 100644
--- a/xlators/cluster/ec/src/ec-data.h
+++ b/xlators/cluster/ec/src/ec-data.h
@@ -11,325 +11,25 @@
 #ifndef __EC_DATA_H__
 #define __EC_DATA_H__
 
-#include "xlator.h"
-
-#include "ec.h"
-
-struct _ec_config;
-typedef struct _ec_config ec_config_t;
-
-struct _ec_fd;
-typedef struct _ec_fd ec_fd_t;
-
-struct _ec_inode;
-typedef struct _ec_inode ec_inode_t;
-
-union _ec_cbk;
-typedef union _ec_cbk ec_cbk_t;
-
-struct _ec_lock;
-typedef struct _ec_lock ec_lock_t;
-
-struct _ec_lock_link;
-typedef struct _ec_lock_link ec_lock_link_t;
-
-struct _ec_fop_data;
-typedef struct _ec_fop_data ec_fop_data_t;
-
-struct _ec_cbk_data;
-typedef struct _ec_cbk_data ec_cbk_data_t;
-
-struct _ec_heal;
-typedef struct _ec_heal ec_heal_t;
-
-typedef void (* ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t);
-typedef int32_t (* ec_handler_f)(ec_fop_data_t *, int32_t);
-typedef void (* ec_resume_f)(ec_fop_data_t *, int32_t);
-
-struct _ec_config
-{
-    uint32_t version;
-    uint8_t  algorithm;
-    uint8_t  gf_word_size;
-    uint8_t  bricks;
-    uint8_t  redundancy;
-    uint32_t chunk_size;
-};
-
-struct _ec_fd
-{
-    loc_t     loc;
-    uintptr_t open;
-    int32_t   flags;
-};
-
-struct _ec_inode
-{
-    ec_lock_t        *inode_lock;
-    gf_boolean_t      have_info;
-    gf_boolean_t      have_config;
-    gf_boolean_t      have_version;
-    gf_boolean_t      have_size;
-    ec_config_t       config;
-    uint64_t          pre_version[2];
-    uint64_t          post_version[2];
-    uint64_t          pre_size;
-    uint64_t          post_size;
-    uint64_t          dirty[2];
-    struct list_head  heal;
-};
-
-typedef int32_t (* fop_heal_cbk_t)(call_frame_t *, void * cookie, xlator_t *,
-                                   int32_t, int32_t, uintptr_t, uintptr_t,
-                                   uintptr_t, dict_t *);
-typedef int32_t (* fop_fheal_cbk_t)(call_frame_t *, void * cookie, xlator_t *,
-                                    int32_t, int32_t, uintptr_t, uintptr_t,
-                                    uintptr_t, dict_t *);
-
-union _ec_cbk
-{
-    fop_access_cbk_t       access;
-    fop_create_cbk_t       create;
-    fop_discard_cbk_t      discard;
-    fop_entrylk_cbk_t      entrylk;
-    fop_fentrylk_cbk_t     fentrylk;
-    fop_fallocate_cbk_t    fallocate;
-    fop_flush_cbk_t        flush;
-    fop_fsync_cbk_t        fsync;
-    fop_fsyncdir_cbk_t     fsyncdir;
-    fop_getxattr_cbk_t     getxattr;
-    fop_fgetxattr_cbk_t    fgetxattr;
-    fop_heal_cbk_t         heal;
-    fop_fheal_cbk_t        fheal;
-    fop_inodelk_cbk_t      inodelk;
-    fop_finodelk_cbk_t     finodelk;
-    fop_link_cbk_t         link;
-    fop_lk_cbk_t           lk;
-    fop_lookup_cbk_t       lookup;
-    fop_mkdir_cbk_t        mkdir;
-    fop_mknod_cbk_t        mknod;
-    fop_open_cbk_t         open;
-    fop_opendir_cbk_t      opendir;
-    fop_readdir_cbk_t      readdir;
-    fop_readdirp_cbk_t     readdirp;
-    fop_readlink_cbk_t     readlink;
-    fop_readv_cbk_t        readv;
-    fop_removexattr_cbk_t  removexattr;
-    fop_fremovexattr_cbk_t fremovexattr;
-    fop_rename_cbk_t       rename;
-    fop_rmdir_cbk_t        rmdir;
-    fop_setattr_cbk_t      setattr;
-    fop_fsetattr_cbk_t     fsetattr;
-    fop_setxattr_cbk_t     setxattr;
-    fop_fsetxattr_cbk_t    fsetxattr;
-    fop_stat_cbk_t         stat;
-    fop_fstat_cbk_t        fstat;
-    fop_statfs_cbk_t       statfs;
-    fop_symlink_cbk_t      symlink;
-    fop_truncate_cbk_t     truncate;
-    fop_ftruncate_cbk_t    ftruncate;
-    fop_unlink_cbk_t       unlink;
-    fop_writev_cbk_t       writev;
-    fop_xattrop_cbk_t      xattrop;
-    fop_fxattrop_cbk_t     fxattrop;
-    fop_zerofill_cbk_t     zerofill;
-    fop_seek_cbk_t         seek;
-};
-
-struct _ec_lock
-{
-    ec_inode_t        *ctx;
-    gf_timer_t        *timer;
-
-    /* List of owners of this lock. All fops added to this list are running
-     * concurrently. */
-    struct list_head   owners;
-
-    /* List of fops waiting to be an owner of the lock. Fops are added to this
-     * list when the current owner has an incompatible access (shared vs
-     * exclusive) or the lock is not acquired yet. */
-    struct list_head   waiting;
-
-    /* List of fops that will wait until the next unlock/lock cycle. This
-     * happens when the currently acquired lock is decided to be released as
-     * soon as possible. In this case, all frozen fops will be continued only
-     * after the lock is reacquired. */
-    struct list_head   frozen;
-
-    int32_t            exclusive;
-    uintptr_t          mask;
-    uintptr_t          good_mask;
-    uintptr_t          healing;
-    uint32_t           refs_owners;  /* Refs for fops owning the lock */
-    uint32_t           refs_pending; /* Refs assigned to fops being prepared */
-    gf_boolean_t       acquired;
-    gf_boolean_t       getting_size;
-    gf_boolean_t       release;
-    gf_boolean_t       query;
-    fd_t              *fd;
-    loc_t              loc;
-    union
-    {
-        entrylk_type     type;
-        struct gf_flock  flock;
-    };
-};
-
-struct _ec_lock_link
-{
-    ec_lock_t        *lock;
-    ec_fop_data_t    *fop;
-    struct list_head  owner_list;
-    struct list_head  wait_list;
-    gf_boolean_t      update[2];
-    loc_t            *base;
-    uint64_t          size;
-};
-
-struct _ec_fop_data
-{
-    int32_t            id;
-    int32_t            refs;
-    int32_t            state;
-    int32_t            minimum;
-    int32_t            expected;
-    int32_t            winds;
-    int32_t            jobs;
-    int32_t            error;
-    ec_fop_data_t     *parent;
-    xlator_t          *xl;
-    call_frame_t      *req_frame;    /* frame of the calling xlator */
-    call_frame_t      *frame;        /* frame used by this fop */
-    struct list_head   cbk_list;     /* sorted list of groups of answers */
-    struct list_head   answer_list;  /* list of answers */
-    struct list_head   pending_list; /* member of ec_t.pending_fops */
-    ec_cbk_data_t     *answer;       /* accepted answer */
-    int32_t            lock_count;
-    int32_t            locked;
-    ec_lock_link_t     locks[2];
-    int32_t            first_lock;
-    gf_lock_t          lock;
-
-    uint32_t           flags;
-    uint32_t           first;
-    uintptr_t          mask;
-    uintptr_t          healing; /*Dispatch is done but call is successful only
-                                  if fop->minimum number of subvolumes succeed
-                                  which are not healing*/
-    uintptr_t          remaining;
-    uintptr_t          received; /* Mask of responses */
-    uintptr_t          good;
-
-    uid_t              uid;
-    gid_t              gid;
-
-    ec_wind_f          wind;
-    ec_handler_f       handler;
-    ec_resume_f        resume;
-    ec_cbk_t           cbks;
-    void              *data;
-    ec_heal_t         *heal;
-    struct list_head   healer;
-
-    uint64_t           user_size;
-    uint32_t           head;
-
-    int32_t            use_fd;
-
-    dict_t            *xdata;
-    dict_t            *dict;
-    int32_t            int32;
-    uint32_t           uint32;
-    uint64_t           size;
-    off_t              offset;
-    mode_t             mode[2];
-    entrylk_cmd        entrylk_cmd;
-    entrylk_type       entrylk_type;
-    gf_xattrop_flags_t xattrop_flags;
-    dev_t              dev;
-    inode_t           *inode;
-    fd_t              *fd;
-    struct iatt        iatt;
-    char              *str[2];
-    loc_t              loc[2];
-    struct gf_flock    flock;
-    struct iovec      *vector;
-    struct iobref     *buffers;
-    gf_seek_what_t     seek;
-};
-
-struct _ec_cbk_data
-{
-    struct list_head list;        // item in the sorted list of groups
-    struct list_head answer_list; // item in the list of answers
-    ec_fop_data_t *  fop;
-    ec_cbk_data_t *  next;        // next answer in the same group
-    int32_t          idx;
-    int32_t          op_ret;
-    int32_t          op_errno;
-    int32_t          count;
-    uintptr_t        mask;
-    uint64_t         dirty[2];
-
-    dict_t *         xdata;
-    dict_t *         dict;
-    int32_t          int32;
-    uintptr_t        uintptr[3];
-    uint64_t         size;
-    uint64_t         version[2];
-    inode_t *        inode;
-    fd_t *           fd;
-    struct statvfs   statvfs;
-    struct iatt      iatt[5];
-    struct gf_flock  flock;
-    struct iovec *   vector;
-    struct iobref *  buffers;
-    char            *str;
-    gf_dirent_t      entries;
-    off_t            offset;
-    gf_seek_what_t   what;
-};
-
-struct _ec_heal
-{
-    struct list_head  list;
-    gf_lock_t         lock;
-    xlator_t         *xl;
-    ec_fop_data_t    *fop;
-    void             *data;
-    ec_fop_data_t    *lookup;
-    loc_t             loc;
-    struct iatt       iatt;
-    char             *symlink;
-    fd_t             *fd;
-    int32_t           partial;
-    int32_t           done;
-    int32_t           error;
-    gf_boolean_t      nameheal;
-    uintptr_t         available;
-    uintptr_t         good;
-    uintptr_t         bad;
-    uintptr_t         open;
-    uintptr_t         fixed;
-    uint64_t          offset;
-    uint64_t          size;
-    uint64_t          total_size;
-    uint64_t          version[2];
-    uint64_t          raw_size;
-};
-
-ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     ec_fop_data_t * fop, int32_t id,
-                                     int32_t idx, int32_t op_ret,
-                                     int32_t op_errno);
-ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     int32_t id, uint32_t flags,
-                                     uintptr_t target, int32_t minimum,
-                                     ec_wind_f wind, ec_handler_f handler,
-                                     ec_cbk_t cbks, void * data);
-void ec_fop_data_acquire(ec_fop_data_t * fop);
-void ec_fop_data_release(ec_fop_data_t * fop);
-
-void ec_fop_cleanup(ec_fop_data_t *fop);
+#include "ec-types.h"
+
+ec_cbk_data_t *
+ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop,
+                     int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno);
+ec_fop_data_t *
+ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id,
+                     uint32_t flags, uintptr_t target, uint32_t fop_flags,
+                     ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks,
+                     void *data);
+void
+ec_fop_data_acquire(ec_fop_data_t *fop);
+void
+ec_fop_data_release(ec_fop_data_t *fop);
+
+void
+ec_fop_cleanup(ec_fop_data_t *fop);
+
+void
+ec_pending_fops_completed(ec_t *ec);
 
 #endif /* __EC_DATA_H__ */
diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c
index fc8b38b22a4..f71dcfac293 100644
--- a/xlators/cluster/ec/src/ec-dir-read.c
+++ b/xlators/cluster/ec/src/ec-dir-read.c
@@ -8,27 +8,27 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
-#include "ec-messages.h"
 
-/* FOP: opendir */
+/****************************************************************
+ *
+ * File Operation: opendir
+ *
+ ***************************************************************/
 
-int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                           ec_cbk_data_t * src)
+int32_t
+ec_combine_opendir(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (dst->fd != src->fd)
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_FD_MISMATCH, "Mismatching fd in answers "
-                "of 'GF_FOP_OPENDIR': %p <-> %p",
-                dst->fd, src->fd);
+    if (dst->fd != src->fd) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH,
+               "Mismatching fd in answers "
+               "of 'GF_FOP_OPENDIR': %p <-> %p",
+               dst->fd, src->fd);
 
         return 0;
     }
@@ -36,12 +36,12 @@ int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, fd_t * fd,
-                       dict_t * xdata)
+int32_t
+ec_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -51,54 +51,51 @@ int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPENDIR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (fd != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (fd != NULL) {
                 cbk->fd = fd_ref(fd);
-                if (cbk->fd == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                            "file descriptor.");
+                if (cbk->fd == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           EC_MSG_FILE_DESC_REF_FAIL,
+                           "Failed to reference a "
+                           "file descriptor.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
         ec_combine(cbk, ec_combine_opendir);
+
+        ec_update_fd_status(fd, this, idx, op_ret);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_opendir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -107,14 +104,14 @@ void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->fd, fop->xdata);
 }
 
-int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_opendir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
     ec_fd_t *ctx;
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
@@ -126,18 +123,28 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
                 return EC_STATE_REPORT;
             }
-            err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
-            if (err != 0) {
-                UNLOCK(&fop->fd->lock);
+            if (!ctx->loc.inode) {
+                err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
+                if (err != 0) {
+                    UNLOCK(&fop->fd->lock);
 
-                fop->error = -err;
+                    fop->error = -err;
 
-                return EC_STATE_REPORT;
+                    return EC_STATE_REPORT;
+                }
             }
 
             UNLOCK(&fop->fd->lock);
 
             /* Fall through */
+
+        case EC_STATE_LOCK:
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
+            ec_lock(fop);
+
+            return EC_STATE_DISPATCH;
+
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
 
@@ -167,63 +174,73 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.opendir != NULL)
-            {
+            if (fop->cbks.opendir != NULL) {
                 fop->cbks.opendir(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                   cbk->op_errno, cbk->fd, cbk->xdata);
             }
 
-            return EC_STATE_END;
+            return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.opendir != NULL)
-            {
+            if (fop->cbks.opendir != NULL) {
                 fop->cbks.opendir(fop->req_frame, fop, fop->xl, -1, fop->error,
                                   NULL, NULL);
             }
 
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
+
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_opendir_cbk_t func, void * data,
-                loc_t * loc, fd_t * fd, dict_t * xdata)
+void
+ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc,
+           fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .opendir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.opendir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(OPENDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(OPENDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_opendir, ec_manager_opendir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_opendir,
+                               ec_manager_opendir, callback, data);
     if (fop == NULL) {
         goto out;
     }
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -231,9 +248,9 @@ void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -241,9 +258,9 @@ void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -261,50 +278,49 @@ out:
 
 /* Returns -1 if client_id is invalid else index of child subvol in xl_list */
 int
-ec_deitransform (xlator_t *this, off_t offset)
+ec_deitransform(xlator_t *this, off_t offset)
 {
-        int  idx       = -1;
-        int  client_id = -1;
-        ec_t *ec       = this->private;
-        char id[32]    = {0};
-        int err;
-
-        client_id = gf_deitransform (this, offset);
-        sprintf (id, "%d", client_id);
-        err = dict_get_int32 (ec->leaf_to_subvolid, id, &idx);
-        if (err < 0) {
-                idx = err;
-                goto out;
-        }
+    int idx = -1;
+    int client_id = -1;
+    ec_t *ec = this->private;
+    char id[32] = {0};
+    int err;
+
+    client_id = gf_deitransform(this, offset);
+    sprintf(id, "%d", client_id);
+    err = dict_get_int32(ec->leaf_to_subvolid, id, &idx);
+    if (err < 0) {
+        idx = err;
+        goto out;
+    }
 
 out:
-        if (idx < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        EC_MSG_INVALID_REQUEST,
-                        "Invalid index %d in readdirp request", client_id);
-                idx = -EINVAL;
-        }
-        return idx;
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REQUEST,
+               "Invalid index %d in readdirp request", client_id);
+        idx = -EINVAL;
+    }
+    return idx;
 }
 
 /* FOP: readdir */
 
-void ec_adjust_readdirp (ec_t *ec, int32_t idx, gf_dirent_t *entries)
+void
+ec_adjust_readdirp(ec_t *ec, int32_t idx, gf_dirent_t *entries)
 {
-    gf_dirent_t * entry;
+    gf_dirent_t *entry;
 
     list_for_each_entry(entry, &entries->list, list)
     {
         if (!entry->inode)
-                continue;
+            continue;
 
-        if (entry->d_stat.ia_type == IA_IFREG)
-        {
+        if (entry->d_stat.ia_type == IA_IFREG) {
             if ((entry->dict == NULL) ||
                 (ec_dict_del_number(entry->dict, EC_XATTR_SIZE,
                                     &entry->d_stat.ia_size) != 0)) {
-                    inode_unref (entry->inode);
-                    entry->inode = NULL;
+                inode_unref(entry->inode);
+                entry->inode = NULL;
             } else {
                 ec_iatt_rebuild(ec, &entry->d_stat, 1, 1);
             }
@@ -313,9 +329,9 @@ void ec_adjust_readdirp (ec_t *ec, int32_t idx, gf_dirent_t *entries)
 }
 
 int32_t
-ec_common_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       gf_dirent_t *entries, dict_t *xdata)
+ec_common_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
     ec_fop_data_t *fop = NULL;
     ec_cbk_data_t *cbk = NULL;
@@ -328,30 +344,29 @@ ec_common_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate (frame, this, fop, fop->id,
-                                idx, op_ret, op_errno);
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
     if (cbk) {
         if (xdata)
-                cbk->xdata = dict_ref (xdata);
+            cbk->xdata = dict_ref(xdata);
         if (cbk->op_ret >= 0)
-                list_splice_init (&entries->list,
-                                  &cbk->entries.list);
-        ec_combine (cbk, NULL);
+            list_splice_init(&entries->list, &cbk->entries.list);
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_readdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -360,59 +375,66 @@ void ec_wind_readdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->size, fop->offset, fop->xdata);
 }
 
-int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readdir(ec_fop_data_t *fop, int32_t state)
 {
     ec_fd_t *ctx = NULL;
     ec_cbk_data_t *cbk = NULL;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             /* Return error if opendir has not been successfully called on
              * any subvolume. */
             ctx = ec_fd_get(fop->fd, fop->xl);
-            if ((ctx == NULL) || (ctx->open == 0)) {
-                fop->error = EINVAL;
+            if (ctx == NULL) {
+                fop->error = ENOMEM;
+            } else if (ctx->open == 0) {
+                fop->error = EBADFD;
+            }
 
+            if (fop->error) {
+                gf_msg(fop->xl->name, GF_LOG_ERROR, fop->error,
+                       EC_MSG_INVALID_REQUEST, "EC is not winding readdir: %s",
+                       ec_msg_str(fop));
                 return EC_STATE_REPORT;
             }
 
             if (fop->id == GF_FOP_READDIRP) {
-                    int32_t err;
+                int32_t err;
 
+                if (fop->xdata == NULL) {
+                    fop->xdata = dict_new();
                     if (fop->xdata == NULL) {
-                        fop->xdata = dict_new();
-                        if (fop->xdata == NULL) {
-                            fop->error = ENOMEM;
+                        fop->error = ENOMEM;
 
-                            return EC_STATE_REPORT;
-                        }
+                        return EC_STATE_REPORT;
                     }
+                }
 
-                    err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
-                    if (err != 0) {
-                        fop->error = -err;
+                err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
+                if (err != 0) {
+                    fop->error = -err;
 
-                        return EC_STATE_REPORT;
-                    }
+                    return EC_STATE_REPORT;
+                }
             }
 
-            if (fop->offset != 0)
-            {
-            /* Non-zero offset is irrecoverable error as the offset may not be
-             * valid on other bricks*/
+            if (fop->offset != 0) {
+                /* Non-zero offset is irrecoverable error as the offset may not
+                 * be valid on other bricks*/
                 int32_t idx = -1;
 
-                idx = ec_deitransform (fop->xl, fop->offset);
+                idx = ec_deitransform(fop->xl, fop->offset);
 
                 if (idx < 0) {
-                        fop->error = -idx;
-                        return EC_STATE_REPORT;
+                    fop->error = -idx;
+                    return EC_STATE_REPORT;
                 }
                 fop->mask &= 1ULL << idx;
             } else {
-                    ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO);
-                    ec_lock(fop);
+                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                   EC_RANGE_FULL);
+                ec_lock(fop);
             }
 
             return EC_STATE_DISPATCH;
@@ -429,14 +451,14 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
 
             if ((cbk != NULL) && (cbk->op_ret > 0) &&
                 (fop->id == GF_FOP_READDIRP)) {
-                ec_adjust_readdirp (fop->xl->private, cbk->idx, &cbk->entries);
+                ec_adjust_readdirp(fop->xl->private, cbk->idx, &cbk->entries);
             }
 
             return EC_STATE_REPORT;
 
         case EC_STATE_REPORT:
             cbk = fop->answer;
-            GF_ASSERT (cbk);
+            GF_ASSERT(cbk);
             if (fop->id == GF_FOP_READDIR) {
                 if (fop->cbks.readdir != NULL) {
                     fop->cbks.readdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
@@ -450,9 +472,9 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
                 }
             }
             if (fop->offset == 0)
-                    return EC_STATE_LOCK_REUSE;
+                return EC_STATE_LOCK_REUSE;
             else
-                    return EC_STATE_END;
+                return EC_STATE_END;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
@@ -471,50 +493,49 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
                 }
             }
             if (fop->offset == 0)
-                    return EC_STATE_LOCK_REUSE;
+                return EC_STATE_LOCK_REUSE;
             else
-                    return EC_STATE_END;
+                return EC_STATE_END;
 
         case -EC_STATE_LOCK_REUSE:
         case EC_STATE_LOCK_REUSE:
-            GF_ASSERT (fop->offset == 0);
+            GF_ASSERT(fop->offset == 0);
             ec_lock_reuse(fop);
 
             return EC_STATE_UNLOCK;
 
         case -EC_STATE_UNLOCK:
         case EC_STATE_UNLOCK:
-            GF_ASSERT (fop->offset == 0);
+            GF_ASSERT(fop->offset == 0);
             ec_unlock(fop);
 
             return EC_STATE_END;
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_readdir_cbk_t func, void * data,
-                fd_t * fd, size_t size, off_t offset, dict_t * xdata)
+void
+ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd,
+           size_t size, off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_readdir, ec_manager_readdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_readdir,
+                               ec_manager_readdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -527,9 +548,9 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -537,9 +558,9 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -557,7 +578,8 @@ out:
 
 /* FOP: readdirp */
 
-void ec_wind_readdirp(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readdirp(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -566,24 +588,24 @@ void ec_wind_readdirp(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->size, fop->offset, fop->xdata);
 }
 
-void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readdirp_cbk_t func, void * data,
-                 fd_t * fd, size_t size, off_t offset, dict_t * xdata)
+void
+ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd,
+            size_t size, off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readdirp = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readdirp = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READDIRP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READDIRP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIRP,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_readdirp, ec_manager_readdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_READDIRP, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_readdirp, ec_manager_readdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -596,9 +618,9 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -606,9 +628,9 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-dir-write.c b/xlators/cluster/ec/src/ec-dir-write.c
index 68741137619..53d27d895c3 100644
--- a/xlators/cluster/ec/src/ec-dir-write.c
+++ b/xlators/cluster/ec/src/ec-dir-write.c
@@ -8,86 +8,84 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-method.h"
 #include "ec-fops.h"
-#include "ec-messages.h"
 
 int
-ec_dir_write_cbk (call_frame_t *frame, xlator_t *this,
-                  void *cookie, int op_ret, int op_errno,
-                  struct iatt *poststat, struct iatt *preparent,
-                  struct iatt *postparent, struct iatt *preparent2,
-                  struct iatt *postparent2, dict_t *xdata)
+ec_dir_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie, int op_ret,
+                 int op_errno, struct iatt *poststat, struct iatt *preparent,
+                 struct iatt *postparent, struct iatt *preparent2,
+                 struct iatt *postparent2, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        int           i    = 0;
-        int           idx  = 0;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int i = 0;
+    int idx = 0;
 
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        fop = frame->local;
-        idx = (long) cookie;
+    fop = frame->local;
+    idx = (long)cookie;
 
-        ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                 frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-        if (op_ret < 0)
-                goto out;
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        if (xdata)
-                cbk->xdata = dict_ref (xdata);
+    if (op_ret < 0)
+        goto out;
 
-        if (poststat)
-                cbk->iatt[i++] = *poststat;
+    if (poststat)
+        cbk->iatt[i++] = *poststat;
 
-        if (preparent)
-                cbk->iatt[i++] = *preparent;
+    if (preparent)
+        cbk->iatt[i++] = *preparent;
 
-        if (postparent)
-                cbk->iatt[i++] = *postparent;
+    if (postparent)
+        cbk->iatt[i++] = *postparent;
 
-        if (preparent2)
-                cbk->iatt[i++] = *preparent2;
+    if (preparent2)
+        cbk->iatt[i++] = *preparent2;
 
-        if (postparent2)
-                cbk->iatt[i++] = *postparent2;
+    if (postparent2)
+        cbk->iatt[i++] = *postparent2;
 
 out:
-        if (cbk)
-                ec_combine (cbk, ec_combine_write);
-        if (fop)
-                ec_complete (fop);
-        return 0;
+    if (cbk)
+        ec_combine(cbk, ec_combine_write);
+
+    if (fop)
+        ec_complete(fop);
+    return 0;
 }
 
 /* FOP: create */
 
-int32_t ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+int32_t
+ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_create(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -97,7 +95,8 @@ void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->xdata);
 }
 
-int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_create(ec_fop_data_t *fop, int32_t state)
 {
     ec_config_t config;
     ec_t *ec;
@@ -106,8 +105,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
     uint64_t version[2] = {0, 0};
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
@@ -174,10 +172,10 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
              * need to remove O_APPEND from flags (if present) */
             fop->int32 &= ~O_APPEND;
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0],
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
@@ -216,12 +214,11 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.create != NULL)
-            {
-                fop->cbks.create (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                  cbk->op_errno, fop->fd, fop->loc[0].inode,
-                                  &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
-                                  cbk->xdata);
+            if (fop->cbks.create != NULL) {
+                QUORUM_CBK(fop->cbks.create, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->fd,
+                           fop->loc[0].inode, &cbk->iatt[0], &cbk->iatt[1],
+                           &cbk->iatt[2], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -233,8 +230,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.create != NULL)
-            {
+            if (fop->cbks.create != NULL) {
                 fop->cbks.create(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL, NULL, NULL);
             }
@@ -254,30 +250,29 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_create_cbk_t func, void * data,
-               loc_t * loc, int32_t flags, mode_t mode, mode_t umask,
-               fd_t * fd, dict_t * xdata)
+void
+ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc,
+          int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .create = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.create = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(CREATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(CREATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, 0, target, fop_flags,
                                ec_wind_create, ec_manager_create, callback,
                                data);
     if (fop == NULL) {
@@ -290,8 +285,8 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -299,19 +294,19 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -329,16 +324,17 @@ out:
 
 /* FOP: link */
 
-int32_t ec_link_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, inode_t * inode,
-                    struct iatt * buf, struct iatt * preparent,
-                    struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *buf,
+            struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_link(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -347,17 +343,17 @@ void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->loc[1], fop->xdata);
 }
 
-int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_link(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[1], EC_UPDATE_DATA |
-                                                            EC_UPDATE_META |
-                                                            EC_INODE_SIZE);
+            ec_lock_prepare_parent_inode(
+                fop, &fop->loc[1], &fop->loc[0],
+                EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -390,11 +386,11 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.link != NULL)
-            {
-                fop->cbks.link(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                               cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                               &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.link != NULL) {
+                QUORUM_CBK(fop->cbks.link, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -406,8 +402,7 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.link != NULL)
-            {
+            if (fop->cbks.link != NULL) {
                 fop->cbks.link(fop->req_frame, fop, fop->xl, -1, fop->error,
                                NULL, NULL, NULL, NULL, NULL);
             }
@@ -427,29 +422,29 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_link_cbk_t func, void * data, loc_t * oldloc,
-             loc_t * newloc, dict_t * xdata)
+void
+ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc,
+        loc_t *newloc, dict_t *xdata)
 {
-    ec_cbk_t callback = { .link = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.link = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(LINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, fop_flags,
                                ec_wind_link, ec_manager_link, callback, data);
     if (fop == NULL) {
         goto out;
@@ -457,26 +452,26 @@ void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (oldloc != NULL) {
         if (loc_copy(&fop->loc[0], oldloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (newloc != NULL) {
         if (loc_copy(&fop->loc[1], newloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -494,16 +489,17 @@ out:
 
 /* FOP: mkdir */
 
-int32_t ec_mkdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, inode_t * inode,
-                     struct iatt * buf, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_mkdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -512,14 +508,14 @@ void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->mode[0], fop->mode[1], fop->xdata);
 }
 
-int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_mkdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
     uint64_t version[2] = {0, 0};
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (fop->xdata == NULL) {
                 fop->xdata = dict_new();
@@ -537,10 +533,10 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
                 return EC_STATE_REPORT;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0],
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
@@ -570,11 +566,11 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.mkdir != NULL)
-            {
-                fop->cbks.mkdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                                &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.mkdir != NULL) {
+                QUORUM_CBK(fop->cbks.mkdir, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -584,12 +580,13 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
+            cbk = fop->answer;
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.mkdir != NULL)
-            {
+            if (fop->cbks.mkdir != NULL) {
                 fop->cbks.mkdir(fop->req_frame, fop, fop->xl, -1, fop->error,
-                                NULL, NULL, NULL, NULL, NULL);
+                                NULL, NULL, NULL, NULL,
+                                ((cbk) ? cbk->xdata : NULL));
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -607,31 +604,30 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mkdir_cbk_t func, void * data, loc_t * loc,
-              mode_t mode, mode_t umask, dict_t * xdata)
+void
+ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .mkdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.mkdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(MKDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(MKDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, 0, target, minimum,
-                               ec_wind_mkdir, ec_manager_mkdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, 0, target, fop_flags,
+                               ec_wind_mkdir, ec_manager_mkdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -641,18 +637,18 @@ void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -670,16 +666,17 @@ out:
 
 /* FOP: mknod */
 
-int32_t ec_mknod_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, inode_t * inode,
-                     struct iatt * buf, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_mknod(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -689,15 +686,15 @@ void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_mknod(ec_fop_data_t *fop, int32_t state)
 {
     ec_config_t config;
     ec_t *ec;
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
     uint64_t version[2] = {0, 0};
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (S_ISREG(fop->mode[0])) {
                 int32_t err;
@@ -741,10 +738,10 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
                 }
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0],
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
@@ -774,11 +771,11 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.mknod != NULL)
-            {
-                fop->cbks.mknod(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                                &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.mknod != NULL) {
+                QUORUM_CBK(fop->cbks.mknod, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -790,8 +787,7 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.mknod != NULL)
-            {
+            if (fop->cbks.mknod != NULL) {
                 fop->cbks.mknod(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL, NULL, NULL);
             }
@@ -811,31 +807,30 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mknod_cbk_t func, void * data, loc_t * loc,
-              mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata)
+void
+ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .mknod = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.mknod = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(MKNOD) %p", frame);
+    gf_msg_trace("ec", 0, "EC(MKNOD) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, 0, target, minimum,
-                               ec_wind_mknod, ec_manager_mknod, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, 0, target, fop_flags,
+                               ec_wind_mknod, ec_manager_mknod, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -846,18 +841,18 @@ void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -875,18 +870,19 @@ out:
 
 /* FOP: rename */
 
-int32_t ec_rename_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                      struct iatt * preoldparent, struct iatt * postoldparent,
-                      struct iatt * prenewparent, struct iatt * postnewparent,
-                      dict_t * xdata)
+int32_t
+ec_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, struct iatt *preoldparent,
+              struct iatt *postoldparent, struct iatt *prenewparent,
+              struct iatt *postnewparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preoldparent, postoldparent, prenewparent,
-                                 postnewparent, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preoldparent, postoldparent, prenewparent,
+                            postnewparent, xdata);
 }
 
-void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_rename(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -895,18 +891,18 @@ void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->loc[1], fop->xdata);
 }
 
-int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_rename(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0], EC_UPDATE_DATA |
-                                                            EC_UPDATE_META |
-                                                            EC_INODE_SIZE);
-            ec_lock_prepare_parent_inode(fop, &fop->loc[1],
+            ec_lock_prepare_parent_inode(
+                fop, &fop->loc[0], &fop->loc[0],
+                EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[1], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
@@ -934,12 +930,11 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.rename != NULL)
-            {
-                fop->cbks.rename(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 &cbk->iatt[2], &cbk->iatt[3], &cbk->iatt[4],
-                                 cbk->xdata);
+            if (fop->cbks.rename != NULL) {
+                QUORUM_CBK(fop->cbks.rename, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], &cbk->iatt[2], &cbk->iatt[3],
+                           &cbk->iatt[4], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -951,8 +946,7 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.rename != NULL)
-            {
+            if (fop->cbks.rename != NULL) {
                 fop->cbks.rename(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL, NULL, NULL);
             }
@@ -972,29 +966,29 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_rename_cbk_t func, void * data,
-               loc_t * oldloc, loc_t * newloc, dict_t * xdata)
+void
+ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc,
+          loc_t *newloc, dict_t *xdata)
 {
-    ec_cbk_t callback = { .rename = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.rename = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(RENAME) %p", frame);
+    gf_msg_trace("ec", 0, "EC(RENAME) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, 0, target, fop_flags,
                                ec_wind_rename, ec_manager_rename, callback,
                                data);
     if (fop == NULL) {
@@ -1003,26 +997,26 @@ void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (oldloc != NULL) {
         if (loc_copy(&fop->loc[0], oldloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (newloc != NULL) {
         if (loc_copy(&fop->loc[1], newloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1040,15 +1034,17 @@ out:
 
 /* FOP: rmdir */
 
-int32_t ec_rmdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+             dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno, NULL,
-                                 preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_rmdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1057,15 +1053,15 @@ void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_rmdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0],
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
@@ -1086,11 +1082,10 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.rmdir != NULL)
-            {
-                fop->cbks.rmdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                cbk->xdata);
+            if (fop->cbks.rmdir != NULL) {
+                QUORUM_CBK(fop->cbks.rmdir, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1102,8 +1097,7 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.rmdir != NULL)
-            {
+            if (fop->cbks.rmdir != NULL) {
                 fop->cbks.rmdir(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL);
             }
@@ -1123,31 +1117,30 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_rmdir_cbk_t func, void * data, loc_t * loc,
-              int xflags, dict_t * xdata)
+void
+ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc,
+         int xflags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .rmdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.rmdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(RMDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(RMDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, 0, target, minimum,
-                               ec_wind_rmdir, ec_manager_rmdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, 0, target, fop_flags,
+                               ec_wind_rmdir, ec_manager_rmdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1156,18 +1149,18 @@ void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1185,16 +1178,18 @@ out:
 
 /* FOP: symlink */
 
-int32_t ec_symlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, inode_t * inode,
-                       struct iatt * buf, struct iatt * preparent,
-                       struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_symlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1203,15 +1198,15 @@ void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->str[0], &fop->loc[0], fop->mode[0], fop->xdata);
 }
 
-int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_symlink(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0],
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
@@ -1241,12 +1236,11 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.symlink != NULL)
-            {
-                fop->cbks.symlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                  cbk->op_errno, fop->loc[0].inode,
-                                  &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
-                                  cbk->xdata);
+            if (fop->cbks.symlink != NULL) {
+                QUORUM_CBK(fop->cbks.symlink, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1258,8 +1252,7 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.symlink != NULL)
-            {
+            if (fop->cbks.symlink != NULL) {
                 fop->cbks.symlink(fop->req_frame, fop, fop->xl, -1, fop->error,
                                   NULL, NULL, NULL, NULL, NULL);
             }
@@ -1279,32 +1272,31 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_symlink_cbk_t func, void * data,
-                const char * linkname, loc_t * loc, mode_t umask,
-                dict_t * xdata)
+void
+ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_symlink_cbk_t func, void *data,
+           const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .symlink = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.symlink = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(SYMLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SYMLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, 0, target, minimum,
-                               ec_wind_symlink, ec_manager_symlink, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, 0, target,
+                               fop_flags, ec_wind_symlink, ec_manager_symlink,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1314,16 +1306,16 @@ void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (linkname != NULL) {
         fop->str[0] = gf_strdup(linkname);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY, "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1331,9 +1323,9 @@ void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1351,16 +1343,17 @@ out:
 
 /* FOP: unlink */
 
-int32_t ec_unlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt * preparent, struct iatt * postparent,
-                      dict_t * xdata)
+int32_t
+ec_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno, NULL,
-                                 preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_unlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1369,15 +1362,15 @@ void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_unlink(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0],
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
@@ -1398,11 +1391,10 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.unlink != NULL)
-            {
-                fop->cbks.unlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 cbk->xdata);
+            if (fop->cbks.unlink != NULL) {
+                QUORUM_CBK(fop->cbks.unlink, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1414,8 +1406,7 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.unlink != NULL)
-            {
+            if (fop->cbks.unlink != NULL) {
                 fop->cbks.unlink(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL);
             }
@@ -1435,29 +1426,29 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_unlink_cbk_t func, void * data,
-               loc_t * loc, int xflags, dict_t * xdata)
+void
+ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc,
+          int xflags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .unlink = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.unlink = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(UNLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(UNLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, 0, target, fop_flags,
                                ec_wind_unlink, ec_manager_unlink, callback,
                                data);
     if (fop == NULL) {
@@ -1468,18 +1459,18 @@ void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-fops.h b/xlators/cluster/ec/src/ec-fops.h
index 8d938427a18..07edf8a7fec 100644
--- a/xlators/cluster/ec/src/ec-fops.h
+++ b/xlators/cluster/ec/src/ec-fops.h
@@ -11,192 +11,244 @@
 #ifndef __EC_FOPS_H__
 #define __EC_FOPS_H__
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
-#include "ec-data.h"
+#include "ec-types.h"
 #include "ec-common.h"
 
-void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_access_cbk_t func, void *data, loc_t * loc,
-               int32_t mask, dict_t * xdata);
-
-void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_create_cbk_t func, void *data, loc_t * loc,
-               int32_t flags, mode_t mode, mode_t umask, fd_t * fd,
-               dict_t * xdata);
-
-void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_entrylk_cbk_t func, void *data,
-                const char * volume, loc_t * loc, const char * basename,
-                entrylk_cmd cmd, entrylk_type type, dict_t * xdata);
-
-void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fentrylk_cbk_t func, void *data,
-                 const char * volume, fd_t * fd, const char * basename,
-                 entrylk_cmd cmd, entrylk_type type, dict_t * xdata);
-
-void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_flush_cbk_t func, void *data, fd_t * fd,
-              dict_t * xdata);
-
-void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fsync_cbk_t func, void *data, fd_t * fd,
-              int32_t datasync, dict_t * xdata);
-
-void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsyncdir_cbk_t func, void *data,
-                 fd_t * fd, int32_t datasync, dict_t * xdata);
-
-void ec_getxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_getxattr_cbk_t func, void *data,
-                 loc_t * loc, const char * name, dict_t * xdata);
-
-void ec_fgetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_fgetxattr_cbk_t func, void *data,
-                  fd_t * fd, const char * name, dict_t * xdata);
-
-void ec_heal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_heal_cbk_t func, void *data, loc_t * loc,
-             int32_t partial, dict_t *xdata);
-
-void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fheal_cbk_t func, void *data, fd_t * fd,
-              int32_t partial, dict_t *xdata);
-
-void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_inodelk_cbk_t func, void *data,
-                const char * volume, loc_t * loc, int32_t cmd,
-                struct gf_flock * flock, dict_t * xdata);
-
-void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_finodelk_cbk_t func, void *data,
-                 const char * volume, fd_t * fd, int32_t cmd,
-                 struct gf_flock * flock, dict_t * xdata);
-
-void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_link_cbk_t func, void *data, loc_t * oldloc,
-             loc_t * newloc, dict_t * xdata);
-
-void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-           int32_t minimum, fop_lk_cbk_t func, void *data, fd_t * fd,
-           int32_t cmd, struct gf_flock * flock, dict_t * xdata);
-
-void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_lookup_cbk_t func, void *data, loc_t * loc,
-               dict_t * xdata);
-
-void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mkdir_cbk_t func, void *data, loc_t * loc,
-              mode_t mode, mode_t umask, dict_t * xdata);
-
-void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mknod_cbk_t func, void *data, loc_t * loc,
-              mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata);
-
-void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_open_cbk_t func, void *data, loc_t * loc,
-             int32_t flags, fd_t * fd, dict_t * xdata);
-
-void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_opendir_cbk_t func, void *data,
-                loc_t * loc, fd_t * fd, dict_t * xdata);
-
-void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_readdir_cbk_t func, void *data, fd_t * fd,
-                size_t size, off_t offset, dict_t * xdata);
-
-void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readdirp_cbk_t func, void *data,
-                 fd_t * fd, size_t size, off_t offset, dict_t * xdata);
-
-void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readlink_cbk_t func, void *data,
-                 loc_t * loc, size_t size, dict_t * xdata);
-
-void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_readv_cbk_t func, void *data, fd_t * fd,
-              size_t size, off_t offset, uint32_t flags, dict_t * xdata);
-
-void ec_removexattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                    int32_t minimum, fop_removexattr_cbk_t func, void *data,
-                    loc_t * loc, const char * name, dict_t * xdata);
-
-void ec_fremovexattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                     int32_t minimum, fop_fremovexattr_cbk_t func, void *data,
-                     fd_t * fd, const char * name, dict_t * xdata);
-
-void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_rename_cbk_t func, void *data,
-               loc_t * oldloc, loc_t * newloc, dict_t * xdata);
-
-void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_rmdir_cbk_t func, void *data, loc_t * loc,
-              int xflags, dict_t * xdata);
-
-void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_setattr_cbk_t func, void *data,
-                loc_t * loc, struct iatt * stbuf, int32_t valid,
-                dict_t * xdata);
-
-void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsetattr_cbk_t func, void *data,
-                 fd_t * fd, struct iatt * stbuf, int32_t valid,
-                 dict_t * xdata);
-
-void ec_setxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_setxattr_cbk_t func, void *data,
-                 loc_t * loc, dict_t * dict, int32_t flags, dict_t * xdata);
-
-void ec_fsetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_fsetxattr_cbk_t func, void *data,
-                  fd_t * fd, dict_t * dict, int32_t flags, dict_t * xdata);
-
-void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_stat_cbk_t func, void *data, loc_t * loc,
-             dict_t * xdata);
-
-void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fstat_cbk_t func, void *data, fd_t * fd,
-              dict_t * xdata);
-
-void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_statfs_cbk_t func, void *data, loc_t * loc,
-               dict_t * xdata);
-
-void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_symlink_cbk_t func, void *data,
-                const char * linkname, loc_t * loc, mode_t umask,
-                dict_t * xdata);
-
-void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_truncate_cbk_t func, void *data,
-                 loc_t * loc, off_t offset, dict_t * xdata);
-
-void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_ftruncate_cbk_t func, void *data,
-                  fd_t * fd, off_t offset, dict_t * xdata);
-
-void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_unlink_cbk_t func, void *data, loc_t * loc,
-               int xflags, dict_t * xdata);
-
-void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_writev_cbk_t func, void *data, fd_t * fd,
-               struct iovec * vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref * iobref, dict_t * xdata);
-
-void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_xattrop_cbk_t func, void *data,
-                loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr,
-                dict_t * xdata);
-
-void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fxattrop_cbk_t func, void *data,
-                 fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr,
-                 dict_t * xdata);
-
-void ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_seek_cbk_t func, void *data, fd_t *fd,
-             off_t offset, gf_seek_what_t what, dict_t *xdata);
+void
+ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc,
+          int32_t mask, dict_t *xdata);
+
+void
+ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc,
+          int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+void
+ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_entrylk_cbk_t func, void *data,
+           const char *volume, loc_t *loc, const char *basename,
+           entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
+
+void
+ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data,
+            const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata);
+
+void
+ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata);
+
+void
+ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd,
+         int32_t datasync, dict_t *xdata);
+
+void
+ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd,
+            int32_t datasync, dict_t *xdata);
+
+void
+ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc,
+            const char *name, dict_t *xdata);
+
+void
+ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd,
+             const char *name, dict_t *xdata);
+
+void
+ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc,
+        int32_t partial, dict_t *xdata);
+
+void
+ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd,
+         int32_t partial, dict_t *xdata);
+
+void
+ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+           uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func,
+           void *data, const char *volume, loc_t *loc, int32_t cmd,
+           struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+            uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func,
+            void *data, const char *volume, fd_t *fd, int32_t cmd,
+            struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc,
+        loc_t *newloc, dict_t *xdata);
+
+void
+ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags,
+      fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata);
+
+void
+ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, mode_t umask, dict_t *xdata);
+
+void
+ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
+
+void
+ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc,
+        int32_t flags, fd_t *fd, dict_t *xdata);
+
+void
+ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc,
+           fd_t *fd, dict_t *xdata);
+
+void
+ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd,
+           size_t size, off_t offset, dict_t *xdata);
+
+void
+ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd,
+            size_t size, off_t offset, dict_t *xdata);
+
+void
+ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc,
+            size_t size, dict_t *xdata);
+
+void
+ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd,
+         size_t size, off_t offset, uint32_t flags, dict_t *xdata);
+
+void
+ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+               uint32_t fop_flags, fop_removexattr_cbk_t func, void *data,
+               loc_t *loc, const char *name, dict_t *xdata);
+
+void
+ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+                uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data,
+                fd_t *fd, const char *name, dict_t *xdata);
+
+void
+ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc,
+          loc_t *newloc, dict_t *xdata);
+
+void
+ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc,
+         int xflags, dict_t *xdata);
+
+void
+ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc,
+           struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+void
+ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+void
+ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc,
+            dict_t *dict, int32_t flags, dict_t *xdata);
+
+void
+ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd,
+             dict_t *dict, int32_t flags, dict_t *xdata);
+
+void
+ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc,
+        dict_t *xdata);
+
+void
+ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata);
+
+void
+ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata);
+
+void
+ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_symlink_cbk_t func, void *data,
+           const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata);
+
+void
+ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd,
+             int32_t mode, off_t offset, size_t len, dict_t *xdata);
+
+void
+ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd,
+           off_t offset, size_t len, dict_t *xdata);
+
+void
+ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc,
+            off_t offset, dict_t *xdata);
+
+void
+ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd,
+             off_t offset, dict_t *xdata);
+
+void
+ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc,
+          int xflags, dict_t *xdata);
+
+void
+ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd,
+          struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+          struct iobref *iobref, dict_t *xdata);
+
+void
+ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+void
+ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+void
+ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd,
+        off_t offset, gf_seek_what_t what, dict_t *xdata);
+
+void
+ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
+       uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op,
+       dict_t *xdata);
 
 #endif /* __EC_FOPS_H__ */
diff --git a/xlators/cluster/ec/src/ec-galois.c b/xlators/cluster/ec/src/ec-galois.c
new file mode 100644
index 00000000000..6e4990c71f5
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-galois.c
@@ -0,0 +1,183 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <string.h>
+
+#include "ec-mem-types.h"
+#include "ec-gf8.h"
+#include "ec-helpers.h"
+
+static ec_gf_t *
+ec_gf_alloc(uint32_t bits, uint32_t mod)
+{
+    ec_gf_t *gf;
+
+    gf = GF_MALLOC(sizeof(ec_gf_t), ec_mt_ec_gf_t);
+    if (gf == NULL) {
+        goto failed;
+    }
+
+    gf->bits = bits;
+    gf->size = 1 << bits;
+    gf->mod = mod;
+
+    gf->log = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1),
+                        gf_common_mt_int);
+    if (gf->log == NULL) {
+        goto failed_gf;
+    }
+    gf->pow = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1),
+                        gf_common_mt_int);
+    if (gf->pow == NULL) {
+        goto failed_log;
+    }
+
+    return gf;
+
+failed_log:
+    GF_FREE(gf->log);
+failed_gf:
+    GF_FREE(gf);
+failed:
+    return EC_ERR(ENOMEM);
+}
+
+static void
+ec_gf_init_tables(ec_gf_t *gf)
+{
+    uint32_t i, tmp;
+
+    memset(gf->log, -1, sizeof(uint32_t) * gf->size);
+
+    gf->pow[0] = 1;
+    gf->log[0] = gf->size;
+    gf->log[1] = 0;
+    for (i = 1; i < gf->size; i++) {
+        tmp = gf->pow[i - 1] << 1;
+        if (tmp >= gf->size) {
+            tmp ^= gf->mod;
+        }
+        gf->pow[i + gf->size - 1] = gf->pow[i] = tmp;
+        gf->log[tmp + gf->size - 1] = gf->log[tmp] = i;
+    }
+}
+
+ec_gf_t *
+ec_gf_prepare(uint32_t bits, uint32_t mod)
+{
+    ec_gf_mul_t **tbl;
+    ec_gf_t *gf;
+    uint32_t i, j;
+
+    if (bits != 8) {
+        return EC_ERR(EINVAL);
+    }
+
+    tbl = ec_gf8_mul;
+    if (mod == 0) {
+        mod = 0x11d;
+    }
+
+    gf = ec_gf_alloc(bits, mod);
+    if (EC_IS_ERR(gf)) {
+        return gf;
+    }
+    ec_gf_init_tables(gf);
+
+    gf->table = tbl;
+    gf->min_ops = bits * bits;
+    gf->max_ops = 0;
+    gf->avg_ops = 0;
+    for (i = 1; i < gf->size; i++) {
+        for (j = 0; tbl[i]->ops[j].op != EC_GF_OP_END; j++) {
+        }
+        if (gf->max_ops < j) {
+            gf->max_ops = j;
+        }
+        if (gf->min_ops > j) {
+            gf->min_ops = j;
+        }
+        gf->avg_ops += j;
+    }
+    gf->avg_ops /= gf->size;
+
+    return gf;
+}
+
+void
+ec_gf_destroy(ec_gf_t *gf)
+{
+    GF_FREE(gf->pow);
+    GF_FREE(gf->log);
+    GF_FREE(gf);
+}
+
+uint32_t
+ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    if ((a >= gf->size) || (b >= gf->size)) {
+        return gf->size;
+    }
+
+    return a ^ b;
+}
+
+uint32_t
+ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    if ((a >= gf->size) || (b >= gf->size)) {
+        return gf->size;
+    }
+
+    if ((a != 0) && (b != 0)) {
+        return gf->pow[gf->log[a] + gf->log[b]];
+    }
+
+    return 0;
+}
+
+uint32_t
+ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    if ((a >= gf->size) || (b >= gf->size)) {
+        return gf->size;
+    }
+
+    if (b != 0) {
+        if (a != 0) {
+            return gf->pow[gf->size - 1 + gf->log[a] - gf->log[b]];
+        }
+
+        return 0;
+    }
+
+    return gf->size;
+}
+
+uint32_t
+ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    uint32_t r;
+
+    if ((a >= gf->size) || ((a == 0) && (b == 0))) {
+        return gf->size;
+    }
+
+    r = 1;
+    while (b != 0) {
+        if ((b & 1) != 0) {
+            r = ec_gf_mul(gf, r, a);
+        }
+        a = ec_gf_mul(gf, a, a);
+        b >>= 1;
+    }
+
+    return r;
+}
diff --git a/xlators/cluster/ec/src/ec-galois.h b/xlators/cluster/ec/src/ec-galois.h
new file mode 100644
index 00000000000..ed55d53e419
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-galois.h
@@ -0,0 +1,32 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_GALOIS_H__
+#define __EC_GALOIS_H__
+
+#include <inttypes.h>
+
+#include "ec-types.h"
+
+ec_gf_t *
+ec_gf_prepare(uint32_t bits, uint32_t mod);
+void
+ec_gf_destroy(ec_gf_t *gf);
+
+uint32_t
+ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b);
+
+#endif /* __EC_GALOIS_H__ */
diff --git a/xlators/cluster/ec/src/ec-generic.c b/xlators/cluster/ec/src/ec-generic.c
index 0ad514908aa..884deb93669 100644
--- a/xlators/cluster/ec/src/ec-generic.c
+++ b/xlators/cluster/ec/src/ec-generic.c
@@ -8,24 +8,23 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/byte-order.h>
 
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
-#include "ec-messages.h"
-#include "byte-order.h"
 
 /* FOP: flush */
 
-int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -35,21 +34,18 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FLUSH, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -59,15 +55,15 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_flush(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -76,15 +72,15 @@ void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_flush(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0);
+            ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -109,8 +105,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.flush != NULL)
-            {
+            if (fop->cbks.flush != NULL) {
                 fop->cbks.flush(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, cbk->xdata);
             }
@@ -125,8 +120,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.flush != NULL)
-            {
+            if (fop->cbks.flush != NULL) {
                 fop->cbks.flush(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL);
             }
@@ -146,31 +140,71 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_flush_cbk_t func, void * data, fd_t * fd,
-              dict_t * xdata)
+static int32_t
+ec_validate_fd(fd_t *fd, xlator_t *xl)
+{
+    uint64_t iversion = 0;
+    uint64_t fversion = 0;
+    ec_inode_t *inode_ctx = NULL;
+    ec_fd_t *fd_ctx = NULL;
+
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __ec_fd_get(fd, xl);
+        if (fd_ctx) {
+            fversion = fd_ctx->bad_version;
+        }
+    }
+    UNLOCK(&fd->lock);
+
+    LOCK(&fd->inode->lock);
+    {
+        inode_ctx = __ec_inode_get(fd->inode, xl);
+        if (inode_ctx) {
+            iversion = inode_ctx->bad_version;
+        }
+    }
+    UNLOCK(&fd->inode->lock);
+    if (fversion < iversion) {
+        return EBADF;
+    }
+    return 0;
+}
+
+void
+ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata)
 {
-    ec_cbk_t callback = { .flush = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.flush = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FLUSH) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FLUSH) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, 0, target, minimum,
-                               ec_wind_flush, ec_manager_flush, callback,
-                               data);
+    if (fd) {
+        error = ec_validate_fd(fd, this);
+        if (error) {
+            gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD,
+                   "Failing %s on %s", gf_fop_list[GF_FOP_FLUSH],
+                   fd->inode ? uuid_utoa(fd->inode->gfid) : "");
+            goto out;
+        }
+    }
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, 0, target, fop_flags,
+                               ec_wind_flush, ec_manager_flush, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -180,9 +214,9 @@ void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -190,9 +224,9 @@ void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -210,13 +244,13 @@ out:
 
 /* FOP: fsync */
 
-int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                         ec_cbk_data_t * src)
+int32_t
+ec_combine_fsync(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_FSYNC'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_FSYNC'");
 
         return 0;
     }
@@ -224,12 +258,13 @@ int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * prebuf,
-                     struct iatt * postbuf, dict_t * xdata)
+int32_t
+ec_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+             dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -239,32 +274,26 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNC, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (prebuf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (prebuf != NULL) {
                 cbk->iatt[0] = *prebuf;
             }
-            if (postbuf != NULL)
-            {
+            if (postbuf != NULL) {
                 cbk->iatt[1] = *postbuf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -274,15 +303,15 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsync(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -291,15 +320,15 @@ void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_fsync(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO);
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -317,8 +346,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
         case EC_STATE_PREPARE_ANSWER:
             cbk = ec_fop_prepare_answer(fop, _gf_false);
             if (cbk != NULL) {
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
                 GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode,
@@ -333,8 +361,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.fsync != NULL)
-            {
+            if (fop->cbks.fsync != NULL) {
                 fop->cbks.fsync(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
                                 cbk->xdata);
@@ -350,8 +377,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_DELAYED_START:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.fsync != NULL)
-            {
+            if (fop->cbks.fsync != NULL) {
                 fop->cbks.fsync(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL);
             }
@@ -371,31 +397,40 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fsync_cbk_t func, void * data, fd_t * fd,
-              int32_t datasync, dict_t * xdata)
+void
+ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd,
+         int32_t datasync, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsync = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsync = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSYNC) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSYNC) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, 0, target, minimum,
-                               ec_wind_fsync, ec_manager_fsync, callback,
-                               data);
+    if (fd) {
+        error = ec_validate_fd(fd, this);
+        if (error) {
+            gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD,
+                   "Failing %s on %s", gf_fop_list[GF_FOP_FSYNC],
+                   fd->inode ? uuid_utoa(fd->inode->gfid) : "");
+            goto out;
+        }
+    }
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, 0, target, fop_flags,
+                               ec_wind_fsync, ec_manager_fsync, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -407,9 +442,9 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -417,9 +452,9 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -437,11 +472,12 @@ out:
 
 /* FOP: fsyncdir */
 
-int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -451,21 +487,18 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNCDIR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -475,15 +508,15 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsyncdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -492,15 +525,15 @@ void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_fsyncdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0);
+            ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -525,8 +558,7 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.fsyncdir != NULL)
-            {
+            if (fop->cbks.fsyncdir != NULL) {
                 fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, cbk->xdata);
             }
@@ -541,10 +573,9 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_DELAYED_START:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.fsyncdir != NULL)
-            {
-                fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL);
+            if (fop->cbks.fsyncdir != NULL) {
+                fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -562,30 +593,30 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsyncdir_cbk_t func, void * data,
-                 fd_t * fd, int32_t datasync, dict_t * xdata)
+void
+ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd,
+            int32_t datasync, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsyncdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsyncdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSYNCDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSYNCDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNCDIR, 0, target,
-                               minimum, ec_wind_fsyncdir, ec_manager_fsyncdir,
+                               fop_flags, ec_wind_fsyncdir, ec_manager_fsyncdir,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -598,9 +629,9 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -608,9 +639,9 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -628,9 +659,10 @@ out:
 
 /* FOP: lookup */
 
-void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
+void
+ec_lookup_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk)
 {
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
     uint64_t size = 0;
     int32_t have_size = 0, err;
 
@@ -649,8 +681,7 @@ void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
     LOCK(&cbk->inode->lock);
 
     ctx = __ec_inode_get(cbk->inode, fop->xl);
-    if (ctx != NULL)
-    {
+    if (ctx != NULL) {
         if (ctx->have_version) {
             cbk->version[0] = ctx->post_version[0];
             cbk->version[1] = ctx->post_version[1];
@@ -663,24 +694,22 @@ void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
 
     UNLOCK(&cbk->inode->lock);
 
-    if (cbk->iatt[0].ia_type == IA_IFREG)
-    {
+    if (cbk->iatt[0].ia_type == IA_IFREG) {
         cbk->size = cbk->iatt[0].ia_size;
         ec_dict_del_number(cbk->xdata, EC_XATTR_SIZE, &cbk->iatt[0].ia_size);
-        if (have_size)
-        {
+        if (have_size) {
             cbk->iatt[0].ia_size = size;
         }
     }
 }
 
-int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                          ec_cbk_data_t * src)
+int32_t
+ec_combine_lookup(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_LOOKUP'");
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_LOOKUP'");
 
         return 0;
     }
@@ -688,14 +717,15 @@ int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, inode_t * inode,
-                      struct iatt * buf, dict_t * xdata,
-                      struct iatt * postparent)
+int32_t
+ec_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
+    uint64_t dirty[2] = {0};
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -704,64 +734,54 @@ int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LOOKUP, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (inode != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (inode != NULL) {
                 cbk->inode = inode_ref(inode);
-                if (cbk->inode == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_INODE_REF_FAIL,
-                            "Failed to reference an inode.");
+                if (cbk->inode == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_INODE_REF_FAIL,
+                           "Failed to reference an inode.");
 
                     goto out;
                 }
             }
-            if (buf != NULL)
-            {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
-            if (postparent != NULL)
-            {
+            if (postparent != NULL) {
                 cbk->iatt[1] = *postparent;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
-            ec_dict_del_array (xdata, EC_XATTR_DIRTY, cbk->dirty,
-                               EC_VERSION_SIZE);
+            ec_dict_del_array(xdata, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
         }
 
         ec_combine(cbk, ec_combine_lookup);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_lookup(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -770,20 +790,21 @@ void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_lookup(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk;
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (fop->xdata == NULL) {
                 fop->xdata = dict_new();
                 if (fop->xdata == NULL) {
-                    gf_msg (fop->xl->name, GF_LOG_ERROR, ENOMEM,
-                            EC_MSG_LOOKUP_REQ_PREP_FAIL, "Unable to prepare "
-                            "lookup request");
+                    gf_msg(fop->xl->name, GF_LOG_ERROR, ENOMEM,
+                           EC_MSG_LOOKUP_REQ_PREP_FAIL,
+                           "Unable to prepare "
+                           "lookup request");
 
                     fop->error = ENOMEM;
 
@@ -791,7 +812,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
                 }
             } else {
                 /*TODO: To be handled once we have 'syndromes' */
-                dict_del (fop->xdata, GF_CONTENT_KEY);
+                dict_del(fop->xdata, GF_CONTENT_KEY);
             }
             err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
             if (err == 0) {
@@ -801,16 +822,17 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
                 err = dict_set_uint64(fop->xdata, EC_XATTR_DIRTY, 0);
             }
             if (err != 0) {
-                gf_msg (fop->xl->name, GF_LOG_ERROR, -err,
-                        EC_MSG_LOOKUP_REQ_PREP_FAIL, "Unable to prepare lookup "
-                                                    "request");
+                gf_msg(fop->xl->name, GF_LOG_ERROR, -err,
+                       EC_MSG_LOOKUP_REQ_PREP_FAIL,
+                       "Unable to prepare lookup "
+                       "request");
 
                 fop->error = -err;
 
                 return EC_STATE_REPORT;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -826,8 +848,8 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
              */
 
             if (!fop->answer && !list_empty(&fop->cbk_list)) {
-                fop->answer = list_entry (fop->cbk_list.next, ec_cbk_data_t,
-                                          list);
+                fop->answer = list_entry(fop->cbk_list.next, ec_cbk_data_t,
+                                         list);
             }
 
             cbk = ec_fop_prepare_answer(fop, _gf_true);
@@ -844,8 +866,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.lookup != NULL)
-            {
+            if (fop->cbks.lookup != NULL) {
                 fop->cbks.lookup(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                  cbk->op_errno, cbk->inode, &cbk->iatt[0],
                                  cbk->xdata, &cbk->iatt[1]);
@@ -859,8 +880,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.lookup != NULL)
-            {
+            if (fop->cbks.lookup != NULL) {
                 fop->cbks.lookup(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL);
             }
@@ -868,30 +888,30 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_lookup_cbk_t func, void * data,
-               loc_t * loc, dict_t * xdata)
+void
+ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata)
 {
-    ec_cbk_t callback = { .lookup = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.lookup = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(LOOKUP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LOOKUP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_LOOKUP, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_lookup,
+                               target, fop_flags, ec_wind_lookup,
                                ec_manager_lookup, callback, data);
     if (fop == NULL) {
         goto out;
@@ -899,14 +919,14 @@ void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         /* Do not log failures here as a memory problem would have already
          * been logged by the corresponding alloc functions */
         if (fop->xdata == NULL)
@@ -925,20 +945,20 @@ out:
 
 /* FOP: statfs */
 
-int32_t ec_combine_statfs(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                          ec_cbk_data_t * src)
+int32_t
+ec_combine_statfs(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     ec_statvfs_combine(&dst->statvfs, &src->statvfs);
 
     return 1;
 }
 
-int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, struct statvfs * buf,
-                      dict_t * xdata)
+int32_t
+ec_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -948,28 +968,23 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STATFS, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->statvfs = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -979,15 +994,15 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_statfs(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -996,14 +1011,14 @@ void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t   *cbk                   = NULL;
-    gf_boolean_t     deem_statfs_enabled   = _gf_false;
-    int32_t          err                   = 0;
+    ec_cbk_data_t *cbk = NULL;
+    gf_boolean_t deem_statfs_enabled = _gf_false;
+    int32_t err = 0;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -1016,8 +1031,8 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
                 ec_t *ec = fop->xl->private;
 
                 if (cbk->xdata) {
-                    err = dict_get_int8 (cbk->xdata, "quota-deem-statfs",
-                                          (int8_t *)&deem_statfs_enabled);
+                    err = dict_get_int8(cbk->xdata, "quota-deem-statfs",
+                                        (int8_t *)&deem_statfs_enabled);
                     if (err != -ENOENT) {
                         ec_cbk_set_error(cbk, -err, _gf_true);
                     }
@@ -1037,8 +1052,7 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.statfs != NULL)
-            {
+            if (fop->cbks.statfs != NULL) {
                 fop->cbks.statfs(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                  cbk->op_errno, &cbk->statvfs, cbk->xdata);
             }
@@ -1051,8 +1065,7 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.statfs != NULL)
-            {
+            if (fop->cbks.statfs != NULL) {
                 fop->cbks.statfs(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL);
             }
@@ -1060,30 +1073,30 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_statfs_cbk_t func, void * data,
-               loc_t * loc, dict_t * xdata)
+void
+ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata)
 {
-    ec_cbk_t callback = { .statfs = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.statfs = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(STATFS) %p", frame);
+    gf_msg_trace("ec", 0, "EC(STATFS) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_STATFS, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_statfs,
+                               target, fop_flags, ec_wind_statfs,
                                ec_manager_statfs, callback, data);
     if (fop == NULL) {
         goto out;
@@ -1091,8 +1104,8 @@ void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1100,9 +1113,9 @@ void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1120,14 +1133,13 @@ out:
 
 /* FOP: xattrop */
 
-int32_t ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                           ec_cbk_data_t *src)
+int32_t
+ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_dict_compare(dst->dict, src->dict))
-    {
-        gf_msg (fop->xl->name, GF_LOG_DEBUG, 0,
-                EC_MSG_DICT_MISMATCH, "Mismatching dictionary in "
-                                             "answers of 'GF_FOP_XATTROP'");
+    if (!ec_dict_compare(dst->dict, src->dict)) {
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_DICT_MISMATCH,
+               "Mismatching dictionary in "
+               "answers of 'GF_FOP_XATTROP'");
 
         return 0;
     }
@@ -1136,64 +1148,71 @@ int32_t ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst,
 }
 
 int32_t
-ec_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                dict_t *xdata)
+ec_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        data_t *data;
-        uint64_t *version;
-        int32_t idx = (int32_t)(uintptr_t)cookie;
+    ec_fop_data_t *fop = NULL;
+    ec_lock_link_t *link = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    uint64_t dirty[2] = {0};
+    data_t *data;
+    uint64_t *version;
+    int32_t idx = (int32_t)(uintptr_t)cookie;
 
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        fop = frame->local;
+    fop = frame->local;
 
-        ec_trace ("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                  frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-        if (op_ret >= 0) {
-                cbk->dict = dict_ref (xattr);
+    if (op_ret >= 0) {
+        cbk->dict = dict_ref(xattr);
 
-                data = dict_get(cbk->dict, EC_XATTR_VERSION);
-                if ((data != NULL) && (data->len >= sizeof(uint64_t))) {
-                    version = (uint64_t *)data->data;
+        data = dict_get(cbk->dict, EC_XATTR_VERSION);
+        if ((data != NULL) && (data->len >= sizeof(uint64_t))) {
+            version = (uint64_t *)data->data;
 
-                    if (((ntoh64(version[0]) >> EC_SELFHEAL_BIT) & 1) != 0) {
-                            LOCK(&fop->lock);
+            if (((ntoh64(version[0]) >> EC_SELFHEAL_BIT) & 1) != 0) {
+                LOCK(&fop->lock);
 
-                            fop->healing |= 1ULL << idx;
+                fop->healing |= 1ULL << idx;
 
-                            UNLOCK(&fop->lock);
-                    }
-                }
+                UNLOCK(&fop->lock);
+            }
+        }
 
-                ec_dict_del_array (xattr, EC_XATTR_DIRTY, cbk->dirty,
-                                   EC_VERSION_SIZE);
+        ec_dict_del_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+        link = fop->data;
+        if (link) {
+            /*Keep a note of if the dirty is already set or not*/
+            link->dirty[0] |= (dirty[0] != 0);
+            link->dirty[1] |= (dirty[1] != 0);
         }
+    }
 
-        if (xdata)
-                cbk->xdata = dict_ref(xdata);
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        ec_combine (cbk, ec_combine_xattrop);
+    ec_combine(cbk, ec_combine_xattrop);
 
 out:
-        if (fop)
-            ec_complete(fop);
+    if (fop)
+        ec_complete(fop);
 
-        return 0;
+    return 0;
 }
 
-void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_xattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1202,18 +1221,20 @@ void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xattrop_flags, fop->dict, fop->xdata);
 }
 
-int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_xattrop(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             if (fop->fd == NULL) {
-                ec_lock_prepare_inode(fop, &fop->loc[0], EC_UPDATE_META);
+                ec_lock_prepare_inode(fop, &fop->loc[0], EC_UPDATE_META, 0,
+                                      EC_RANGE_FULL);
             } else {
-                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META);
+                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META, 0,
+                                   EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -1240,19 +1261,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_XATTROP)
-            {
-                if (fop->cbks.xattrop != NULL)
-                {
-                    fop->cbks.xattrop(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->dict,
-                                      cbk->xdata);
+            if (fop->id == GF_FOP_XATTROP) {
+                if (fop->cbks.xattrop != NULL) {
+                    fop->cbks.xattrop(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->dict, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fxattrop != NULL)
-                {
+            } else {
+                if (fop->cbks.fxattrop != NULL) {
                     fop->cbks.fxattrop(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->dict,
                                        cbk->xdata);
@@ -1268,18 +1283,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_XATTROP)
-            {
-                if (fop->cbks.xattrop != NULL)
-                {
+            if (fop->id == GF_FOP_XATTROP) {
+                if (fop->cbks.xattrop != NULL) {
                     fop->cbks.xattrop(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fxattrop != NULL)
-                {
+            } else {
+                if (fop->cbks.fxattrop != NULL) {
                     fop->cbks.fxattrop(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL);
                 }
@@ -1300,32 +1310,31 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_xattrop_cbk_t func, void * data,
-                loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr,
-                dict_t * xdata)
+void
+ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_cbk_t callback = { .xattrop = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.xattrop = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(XATTROP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(XATTROP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, 0, target, minimum,
-                               ec_wind_xattrop, ec_manager_xattrop, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, 0, target,
+                               fop_flags, ec_wind_xattrop, ec_manager_xattrop,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1334,8 +1343,8 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1343,9 +1352,9 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xattr != NULL) {
         fop->dict = dict_ref(xattr);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1353,9 +1362,9 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1371,7 +1380,8 @@ out:
     }
 }
 
-void ec_wind_fxattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fxattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1380,23 +1390,23 @@ void ec_wind_fxattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->xattrop_flags, fop->dict, fop->xdata);
 }
 
-void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fxattrop_cbk_t func, void * data,
-                 fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr,
-                 dict_t * xdata)
+void
+ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fxattrop = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fxattrop = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FXATTROP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FXATTROP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FXATTROP, 0, target,
-                               minimum, ec_wind_fxattrop, ec_manager_xattrop,
+                               fop_flags, ec_wind_fxattrop, ec_manager_xattrop,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -1409,9 +1419,9 @@ void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1419,9 +1429,9 @@ void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xattr != NULL) {
         fop->dict = dict_ref(xattr);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1429,9 +1439,9 @@ void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1446,3 +1456,136 @@ out:
         func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
+
+/* FOP: IPC */
+
+int32_t
+ec_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, dict_t *xdata)
+{
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int32_t idx = (int32_t)(uintptr_t)cookie;
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = frame->local;
+
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
+
+    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_IPC, idx, op_ret,
+                               op_errno);
+
+    if (cbk != NULL) {
+        if (xdata != NULL) {
+            cbk->xdata = dict_ref(xdata);
+        }
+
+        ec_combine(cbk, NULL);
+    }
+
+out:
+    if (fop != NULL) {
+        ec_complete(fop);
+    }
+
+    return 0;
+}
+
+void
+ec_wind_ipc(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
+
+    STACK_WIND_COOKIE(fop->frame, ec_ipc_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->ipc, fop->int32,
+                      fop->xdata);
+}
+
+int32_t
+ec_manager_ipc(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk;
+
+    switch (state) {
+        case EC_STATE_INIT:
+        case EC_STATE_DISPATCH:
+            ec_dispatch_all(fop);
+
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            ec_fop_prepare_answer(fop, _gf_true);
+
+            return EC_STATE_REPORT;
+
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+
+            GF_ASSERT(cbk != NULL);
+            if (fop->cbks.ipc != NULL) {
+                fop->cbks.ipc(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                              cbk->op_errno, cbk->xdata);
+            }
+
+            return EC_STATE_END;
+
+        case -EC_STATE_INIT:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
+
+            if (fop->cbks.ipc != NULL) {
+                fop->cbks.ipc(fop->req_frame, fop, fop->xl, -1, fop->error,
+                              NULL);
+            }
+
+            return EC_STATE_END;
+
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
+    }
+}
+
+void
+ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
+       uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op,
+       dict_t *xdata)
+{
+    ec_cbk_t callback = {.ipc = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
+
+    gf_msg_trace("ec", 0, "EC(IPC) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_IPC, 0, target, fop_flags,
+                               ec_wind_ipc, ec_manager_ipc, callback, data);
+    if (fop == NULL) {
+        goto out;
+    }
+    if (xdata != NULL) {
+        fop->xdata = dict_ref(xdata);
+    }
+    fop->int32 = op;
+
+    error = 0;
+
+out:
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
+    }
+}
diff --git a/xlators/cluster/ec/src/ec-gf.c b/xlators/cluster/ec/src/ec-gf.c
deleted file mode 100644
index 1ae8928f20b..00000000000
--- a/xlators/cluster/ec/src/ec-gf.c
+++ /dev/null
@@ -1,11635 +0,0 @@
-/*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <inttypes.h>
-#include <string.h>
-
-#include "ec-gf.h"
-
-static void gf8_muladd_00(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    memcpy(out, in, sizeof(uint64_t) * 8 * width);
-}
-
-static void gf8_muladd_01(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        out_ptr[0] ^= in_ptr[0];
-        out_ptr[width] ^= in_ptr[width];
-        out_ptr[width * 2] ^= in_ptr[width * 2];
-        out_ptr[width * 3] ^= in_ptr[width * 3];
-        out_ptr[width * 4] ^= in_ptr[width * 4];
-        out_ptr[width * 5] ^= in_ptr[width * 5];
-        out_ptr[width * 6] ^= in_ptr[width * 6];
-        out_ptr[width * 7] ^= in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_02(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in7;
-        out1 = in0;
-        out7 = in6;
-        out5 = in4;
-        out6 = in5;
-        out3 = in2 ^ in7;
-        out4 = in3 ^ in7;
-        out2 = in1 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_03(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in7;
-        tmp0 = in2 ^ in7;
-        out1 = in0 ^ in1;
-        out7 = in6 ^ in7;
-        out5 = in4 ^ in5;
-        out6 = in5 ^ in6;
-        out4 = in3 ^ in4 ^ in7;
-        out2 = tmp0 ^ in1;
-        out3 = tmp0 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_04(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in6;
-        out1 = in7;
-        out7 = in5;
-        out6 = in4;
-        tmp0 = in6 ^ in7;
-        out2 = in0 ^ in6;
-        out5 = in3 ^ in7;
-        out3 = tmp0 ^ in1;
-        out4 = tmp0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_05(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in6;
-        out1 = in1 ^ in7;
-        out7 = in5 ^ in7;
-        out6 = in4 ^ in6;
-        out2 = out0 ^ in2;
-        out3 = out1 ^ in3 ^ in6;
-        out5 = out7 ^ in3;
-        out4 = out6 ^ in2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_06(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in6 ^ in7;
-        tmp0 = in1 ^ in6;
-        out1 = in0 ^ in7;
-        out7 = in5 ^ in6;
-        out6 = in4 ^ in5;
-        out4 = in2 ^ in3 ^ in6;
-        out5 = in3 ^ in4 ^ in7;
-        out3 = tmp0 ^ in2;
-        out2 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_07(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        tmp1 = in5 ^ in6;
-        tmp2 = in0 ^ in7;
-        tmp3 = tmp0 ^ in3;
-        out6 = tmp1 ^ in4;
-        out7 = tmp1 ^ in7;
-        out0 = tmp2 ^ in6;
-        out1 = tmp2 ^ in1;
-        out3 = tmp3 ^ in1;
-        out4 = tmp3 ^ in4;
-        out5 = out4 ^ out7 ^ in2;
-        out2 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_08(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in5;
-        out1 = in6;
-        out7 = in4;
-        out6 = in3 ^ in7;
-        out3 = in0 ^ in5 ^ in6;
-        out5 = in2 ^ in6 ^ in7;
-        out2 = in5 ^ in7;
-        out4 = out2 ^ in1 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_09(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in5;
-        tmp0 = in3 ^ in6;
-        out1 = in1 ^ in6;
-        out7 = in4 ^ in7;
-        out2 = in2 ^ in5 ^ in7;
-        out3 = tmp0 ^ out0;
-        out6 = tmp0 ^ in7;
-        out4 = out1 ^ out7 ^ in5;
-        out5 = out2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in5 ^ in7;
-        out1 = in0 ^ in6;
-        out7 = in4 ^ in6;
-        out2 = in1 ^ in5;
-        out6 = out0 ^ in3;
-        out3 = out0 ^ out1 ^ in2;
-        out5 = out7 ^ in2 ^ in7;
-        out4 = out2 ^ in3 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in0 ^ in6;
-        tmp2 = in4 ^ in7;
-        out0 = in0 ^ in5 ^ in7;
-        out2 = tmp0 ^ in1;
-        out1 = tmp1 ^ in1;
-        out6 = tmp1 ^ out0 ^ in3;
-        out7 = tmp2 ^ in6;
-        out4 = tmp2 ^ out6 ^ in1;
-        out3 = out6 ^ in0 ^ in2;
-        out5 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in5 ^ in6;
-        out1 = in6 ^ in7;
-        out7 = in4 ^ in5;
-        tmp0 = in1 ^ in5;
-        tmp1 = in0 ^ in7;
-        out5 = in2 ^ in3 ^ in6;
-        out6 = in3 ^ in4 ^ in7;
-        out2 = tmp1 ^ out0;
-        out4 = tmp0 ^ in2;
-        out3 = tmp0 ^ tmp1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in5;
-        tmp1 = in5 ^ in6;
-        out1 = in1 ^ in6 ^ in7;
-        out7 = tmp0 ^ in7;
-        out4 = tmp0 ^ in1 ^ in2;
-        out0 = tmp1 ^ in0;
-        tmp2 = tmp1 ^ in3;
-        out6 = tmp2 ^ out7;
-        out2 = out0 ^ in2 ^ in7;
-        out3 = out0 ^ out1 ^ in3;
-        out5 = tmp2 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in5;
-        tmp2 = in5 ^ in6;
-        out1 = in0 ^ in6 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out2 = tmp0 ^ tmp2;
-        tmp3 = tmp1 ^ in3;
-        out7 = tmp2 ^ in4;
-        out0 = tmp2 ^ in7;
-        out4 = tmp3 ^ in1 ^ in7;
-        out5 = tmp3 ^ out7;
-        out6 = out0 ^ out5 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in6 ^ in7;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp0 ^ in5;
-        out1 = tmp1 ^ in0;
-        out7 = tmp2 ^ in4;
-        out0 = tmp2 ^ in0;
-        out6 = out7 ^ in3;
-        out5 = out6 ^ in2 ^ in7;
-        tmp3 = tmp1 ^ out0 ^ in2;
-        out4 = tmp1 ^ out5;
-        out2 = tmp3 ^ in6;
-        out3 = tmp3 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_10(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4;
-        out1 = in5;
-        out7 = in3 ^ in7;
-        tmp0 = in6 ^ in7;
-        out2 = in4 ^ in6;
-        tmp1 = out2 ^ in5;
-        out6 = tmp0 ^ in2;
-        out3 = tmp0 ^ tmp1;
-        out5 = out2 ^ out3 ^ in1;
-        out4 = tmp1 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_11(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in3;
-        out0 = in0 ^ in4;
-        out1 = in1 ^ in5;
-        out6 = in2 ^ in7;
-        out4 = in0 ^ in5 ^ in6;
-        out5 = in1 ^ in6 ^ in7;
-        out2 = in2 ^ in4 ^ in6;
-        out3 = in3 ^ in4 ^ in5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_12(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4 ^ in7;
-        out1 = in0 ^ in5;
-        out3 = in2 ^ in4 ^ in5;
-        tmp0 = out0 ^ in6;
-        out2 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in3;
-        out6 = tmp0 ^ out3;
-        out5 = out2 ^ in5;
-        out7 = tmp1 ^ in4;
-        out4 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_13(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in3 ^ in6;
-        tmp0 = in0 ^ in5;
-        tmp1 = in4 ^ in7;
-        out6 = in2 ^ in5 ^ in7;
-        out4 = tmp0 ^ out7 ^ in7;
-        out1 = tmp0 ^ in1;
-        out0 = tmp1 ^ in0;
-        out5 = tmp1 ^ in1 ^ in6;
-        out3 = tmp1 ^ out6 ^ in3;
-        out2 = out5 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_14(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4 ^ in6;
-        out1 = in5 ^ in7;
-        out2 = in0 ^ in4;
-        tmp0 = out0 ^ in5;
-        out7 = out1 ^ in3;
-        tmp1 = out1 ^ in2;
-        out3 = tmp0 ^ in1;
-        out6 = tmp0 ^ tmp1;
-        out4 = tmp1 ^ out2;
-        out5 = out3 ^ in3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_15(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in3 ^ in5;
-        tmp0 = in0 ^ in4;
-        out1 = in1 ^ in5 ^ in7;
-        out5 = in1 ^ in3 ^ in6;
-        out0 = tmp0 ^ in6;
-        out2 = tmp0 ^ in2;
-        out3 = out5 ^ in4 ^ in5;
-        out6 = out2 ^ in0 ^ in7;
-        out4 = tmp0 ^ out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_16(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in4 ^ in7;
-        tmp2 = in2 ^ in3 ^ in4;
-        out1 = tmp0 ^ in7;
-        out4 = tmp0 ^ tmp2;
-        out0 = tmp1 ^ in6;
-        tmp3 = tmp1 ^ in1;
-        out6 = out0 ^ in2 ^ in5;
-        out2 = tmp3 ^ in0;
-        out3 = out6 ^ in1;
-        out7 = tmp2 ^ out6;
-        out5 = tmp3 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_17(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in3 ^ in6;
-        tmp2 = tmp0 ^ in4;
-        out4 = tmp0 ^ in0 ^ in3;
-        out7 = tmp1 ^ in5;
-        tmp3 = tmp1 ^ in1;
-        out6 = tmp2 ^ in7;
-        out5 = tmp3 ^ in4;
-        out3 = tmp3 ^ out6;
-        out0 = out3 ^ out4 ^ in1;
-        out2 = out3 ^ out7 ^ in0;
-        out1 = tmp2 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_18(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4 ^ in5;
-        out1 = in5 ^ in6;
-        tmp0 = in4 ^ in7;
-        out5 = in1 ^ in2 ^ in5;
-        out6 = in2 ^ in3 ^ in6;
-        out2 = tmp0 ^ out1;
-        out7 = tmp0 ^ in3;
-        tmp1 = tmp0 ^ in0;
-        out3 = tmp1 ^ in6;
-        out4 = tmp1 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_19(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in1 ^ in2;
-        out7 = in3 ^ in4;
-        tmp0 = in0 ^ in7;
-        out6 = in2 ^ in3;
-        out1 = in1 ^ in5 ^ in6;
-        out0 = in0 ^ in4 ^ in5;
-        out4 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in6;
-        out2 = tmp1 ^ out0 ^ in2;
-        out3 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in5;
-        tmp1 = in5 ^ in6;
-        tmp2 = tmp0 ^ in1;
-        out0 = tmp0 ^ in7;
-        out1 = tmp1 ^ in0;
-        tmp3 = tmp1 ^ in3;
-        out5 = tmp2 ^ in2;
-        out2 = tmp2 ^ in6;
-        out7 = tmp3 ^ out0;
-        out6 = tmp3 ^ in2;
-        out4 = tmp3 ^ out2 ^ in0;
-        out3 = tmp0 ^ out1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = in2 ^ in5;
-        tmp2 = in3 ^ in6;
-        out5 = tmp0 ^ in1;
-        tmp3 = tmp0 ^ in0;
-        out6 = tmp1 ^ in3;
-        out0 = tmp1 ^ tmp3 ^ in7;
-        out7 = tmp2 ^ in4;
-        tmp4 = out5 ^ in6;
-        out3 = tmp2 ^ tmp3;
-        out2 = tmp4 ^ in5;
-        out4 = tmp4 ^ out3;
-        out1 = tmp3 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in4 ^ in6;
-        tmp2 = in5 ^ in7;
-        out6 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ in5;
-        out1 = tmp2 ^ in6;
-        tmp3 = tmp2 ^ in1;
-        tmp4 = tmp2 ^ in4;
-        out2 = tmp4 ^ in0;
-        out7 = tmp4 ^ in3;
-        out5 = tmp0 ^ tmp3;
-        out3 = tmp3 ^ out2;
-        out4 = out3 ^ in2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in0 ^ in4;
-        tmp2 = in3 ^ in4;
-        tmp3 = in2 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out5 = tmp0 ^ tmp3;
-        tmp4 = tmp1 ^ in5;
-        out6 = tmp2 ^ in2;
-        out7 = tmp2 ^ in5;
-        out2 = tmp3 ^ tmp4;
-        out4 = out3 ^ out6 ^ in6;
-        out0 = tmp4 ^ in6;
-        out1 = out2 ^ out4 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in1;
-        out3 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ in5;
-        out4 = out3 ^ in3 ^ in6;
-        tmp3 = out4 ^ in7;
-        out6 = tmp3 ^ out2 ^ in4;
-        out7 = tmp1 ^ out6;
-        out0 = out7 ^ in3;
-        out1 = tmp0 ^ out0;
-        out5 = tmp3 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in6;
-        tmp1 = tmp0 ^ in5;
-        out7 = tmp1 ^ in3;
-        out0 = tmp1 ^ in0 ^ in7;
-        out6 = out7 ^ in2 ^ in6;
-        out1 = out0 ^ in1 ^ in4;
-        out4 = out0 ^ out6 ^ in1;
-        out3 = tmp0 ^ out4;
-        out2 = out4 ^ out7 ^ in7;
-        out5 = out3 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_20(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in4;
-        out0 = in3 ^ in7;
-        tmp0 = in3 ^ in4;
-        tmp1 = in6 ^ in7;
-        out2 = out0 ^ in5;
-        out4 = tmp0 ^ in5;
-        out3 = tmp0 ^ tmp1;
-        out7 = tmp1 ^ in2;
-        out6 = tmp1 ^ in1 ^ in5;
-        out5 = out2 ^ out3 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_21(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in1 ^ in4;
-        tmp0 = in4 ^ in6;
-        out4 = in3 ^ in5;
-        out7 = in2 ^ in6;
-        out0 = in0 ^ in3 ^ in7;
-        out6 = in1 ^ in5 ^ in7;
-        out3 = tmp0 ^ in7;
-        out5 = tmp0 ^ in0;
-        out2 = out4 ^ in2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_22(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3;
-        out1 = in0 ^ in4;
-        out7 = in2 ^ in7;
-        out4 = in4 ^ in5 ^ in7;
-        out5 = in0 ^ in5 ^ in6;
-        out6 = in1 ^ in6 ^ in7;
-        out3 = in2 ^ in3 ^ in4 ^ in6;
-        out2 = in1 ^ in3 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_23(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in2;
-        out0 = in0 ^ in3;
-        out4 = in5 ^ in7;
-        out5 = in0 ^ in6;
-        out6 = in1 ^ in7;
-        out3 = in2 ^ in4 ^ in6;
-        out1 = in0 ^ in1 ^ in4;
-        out2 = out4 ^ out6 ^ in2 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_24(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in4 ^ in7;
-        tmp0 = in3 ^ in4;
-        out0 = in3 ^ in6 ^ in7;
-        out3 = tmp0 ^ in1;
-        tmp1 = out0 ^ in5;
-        out6 = tmp1 ^ out3;
-        out2 = tmp1 ^ in0;
-        out7 = tmp1 ^ in2 ^ in3;
-        out5 = out2 ^ in4;
-        out4 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_25(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in4;
-        tmp0 = in2 ^ in5;
-        out1 = out3 ^ in7;
-        out7 = tmp0 ^ in6;
-        out6 = out1 ^ in5;
-        out4 = out7 ^ in3 ^ in7;
-        out2 = out4 ^ in0;
-        out0 = tmp0 ^ out2;
-        out5 = out0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_26(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3 ^ in6;
-        tmp0 = in4 ^ in7;
-        out7 = in2 ^ in5 ^ in7;
-        tmp1 = out0 ^ in0 ^ in5;
-        out1 = tmp0 ^ in0;
-        tmp2 = tmp0 ^ in6;
-        out2 = tmp1 ^ in1;
-        out5 = tmp1 ^ in7;
-        out6 = tmp2 ^ in1;
-        out4 = tmp2 ^ out7;
-        out3 = out0 ^ out6 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_27(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in2 ^ in5;
-        out0 = in0 ^ in3 ^ in6;
-        out6 = in1 ^ in4 ^ in7;
-        out4 = out7 ^ in6;
-        out2 = out0 ^ out7 ^ in1;
-        out5 = out0 ^ in7;
-        out1 = out6 ^ in0;
-        out3 = out6 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_28(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3;
-        out1 = in4 ^ in6;
-        out0 = in3 ^ in5 ^ in7;
-        tmp0 = out1 ^ in7;
-        tmp1 = out0 ^ in4;
-        out7 = tmp0 ^ in2;
-        tmp2 = tmp0 ^ in1;
-        out3 = tmp1 ^ in0;
-        out6 = tmp1 ^ tmp2;
-        out4 = tmp2 ^ in3;
-        out5 = out3 ^ in2 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_29(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in3;
-        tmp0 = in1 ^ in3;
-        tmp1 = in4 ^ in6;
-        tmp2 = in0 ^ in4 ^ in7;
-        out6 = tmp0 ^ in5;
-        out4 = tmp0 ^ in6 ^ in7;
-        out1 = tmp1 ^ in1;
-        out7 = tmp1 ^ in2;
-        out3 = tmp2 ^ in5;
-        out5 = tmp2 ^ in2;
-        out0 = out3 ^ in3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3 ^ in5;
-        tmp0 = in1 ^ in3;
-        tmp1 = in0 ^ in4;
-        out7 = in2 ^ in4 ^ in7;
-        out3 = tmp1 ^ out0 ^ in2;
-        out2 = tmp0 ^ in7;
-        out6 = tmp0 ^ in6;
-        out1 = tmp1 ^ in6;
-        out5 = tmp1 ^ out7 ^ in5;
-        out4 = out1 ^ in0 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in6;
-        out7 = in2 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in2 ^ in7;
-        out6 = in1 ^ in3;
-        out1 = out4 ^ in0 ^ in4;
-        out3 = tmp0 ^ out7;
-        out0 = tmp0 ^ in3;
-        out5 = tmp1 ^ in0;
-        out2 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in2 ^ in3 ^ in4;
-        tmp2 = tmp0 ^ in6;
-        out4 = tmp1 ^ in1;
-        out5 = tmp1 ^ in0 ^ in5;
-        tmp3 = tmp2 ^ in4;
-        out6 = tmp2 ^ out4;
-        out7 = tmp3 ^ in7;
-        out2 = tmp3 ^ out5;
-        out3 = out6 ^ in0;
-        out0 = tmp1 ^ out7;
-        out1 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out4 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in0;
-        out2 = tmp1 ^ in6;
-        out5 = tmp1 ^ in4;
-        tmp2 = out2 ^ in2;
-        tmp3 = tmp2 ^ in5;
-        out0 = tmp3 ^ in7;
-        out7 = tmp3 ^ out5;
-        out6 = out4 ^ out7 ^ in6;
-        out3 = tmp2 ^ out6;
-        out1 = out0 ^ out6 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in7;
-        out0 = in3 ^ in5 ^ in6;
-        tmp1 = tmp0 ^ in0;
-        tmp2 = tmp0 ^ in2;
-        out1 = tmp1 ^ in6;
-        out4 = tmp2 ^ in1;
-        out7 = tmp2 ^ in5;
-        out3 = out0 ^ out4 ^ in0;
-        out2 = out3 ^ out7 ^ in7;
-        out6 = tmp1 ^ out2;
-        out5 = tmp1 ^ out7 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in2 ^ in5;
-        out4 = in1 ^ in2 ^ in7;
-        out6 = in1 ^ in3 ^ in4;
-        out5 = tmp0 ^ in2;
-        tmp2 = tmp0 ^ in6;
-        out7 = tmp1 ^ in4;
-        out0 = tmp2 ^ in5;
-        out2 = tmp2 ^ out4;
-        out1 = tmp2 ^ out6 ^ in7;
-        out3 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_30(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in4 ^ in5;
-        tmp0 = in3 ^ in6;
-        tmp1 = in4 ^ in7;
-        out6 = in1 ^ in2 ^ in5;
-        out3 = tmp0 ^ in5;
-        out4 = tmp0 ^ in0;
-        out7 = tmp0 ^ in2;
-        out0 = tmp1 ^ in3;
-        out2 = tmp1 ^ out3;
-        out5 = tmp1 ^ in0 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_31(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in5 ^ in6;
-        tmp0 = in4 ^ in5;
-        tmp1 = in0 ^ in3 ^ in4;
-        tmp2 = out3 ^ in2;
-        out1 = tmp0 ^ in1;
-        out0 = tmp1 ^ in7;
-        out4 = tmp1 ^ in6;
-        out6 = tmp2 ^ in1;
-        out2 = tmp2 ^ out0 ^ in0;
-        out5 = out1 ^ in0 ^ in7;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_32(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3 ^ in4;
-        out7 = in2 ^ in3;
-        tmp0 = in5 ^ in6;
-        tmp1 = in0 ^ in7;
-        out6 = in1 ^ in2;
-        out1 = in0 ^ in4 ^ in5;
-        out2 = tmp0 ^ out0 ^ in1;
-        out3 = tmp0 ^ out7 ^ in7;
-        out4 = tmp1 ^ in6;
-        out5 = tmp1 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_33(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in0 ^ in4;
-        tmp2 = in1 ^ in5;
-        out6 = in1 ^ in2 ^ in6;
-        out7 = tmp0 ^ in7;
-        out0 = tmp1 ^ in3;
-        out1 = tmp1 ^ tmp2;
-        tmp3 = tmp2 ^ in7;
-        tmp4 = tmp2 ^ in4 ^ in6;
-        out5 = tmp3 ^ in0;
-        out3 = tmp3 ^ out6;
-        out4 = tmp4 ^ out5;
-        out2 = tmp0 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_34(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in4 ^ in5;
-        tmp2 = tmp0 ^ in1;
-        tmp3 = tmp0 ^ in6;
-        out1 = tmp1 ^ in7;
-        tmp4 = tmp1 ^ in2;
-        out5 = tmp2 ^ in0;
-        out3 = tmp2 ^ out1;
-        out0 = tmp3 ^ in7;
-        out7 = tmp3 ^ tmp4;
-        out6 = tmp4 ^ in1;
-        out2 = out3 ^ out5 ^ in3;
-        out4 = tmp4 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_35(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        tmp1 = in5 ^ in7;
-        out7 = tmp0 ^ tmp1 ^ in3;
-        out3 = tmp1 ^ in1;
-        out1 = out3 ^ in4;
-        tmp2 = out1 ^ in7;
-        out5 = tmp2 ^ in0 ^ in3;
-        out6 = tmp0 ^ tmp2;
-        out0 = out3 ^ out5 ^ in6;
-        out4 = tmp0 ^ out0;
-        out2 = out4 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_36(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in2;
-        tmp0 = in1 ^ in3;
-        out0 = in3 ^ in4 ^ in6;
-        out6 = in1 ^ in2 ^ in4;
-        out5 = tmp0 ^ in0;
-        tmp1 = out5 ^ in5;
-        out2 = tmp1 ^ in4;
-        out3 = tmp1 ^ out4;
-        out1 = tmp0 ^ out2 ^ in7;
-        out7 = out3 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_37(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in4;
-        tmp2 = tmp0 ^ in6;
-        out3 = tmp0 ^ in5;
-        out4 = tmp1 ^ in0;
-        out6 = tmp2 ^ in4;
-        out1 = out3 ^ out4 ^ in7;
-        tmp3 = out4 ^ in1 ^ in3;
-        out7 = tmp3 ^ out1;
-        out2 = tmp3 ^ in5;
-        out5 = tmp1 ^ out2;
-        out0 = tmp2 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_38(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in3;
-        tmp0 = in3 ^ in4;
-        tmp1 = in5 ^ in7;
-        tmp2 = out3 ^ in1;
-        out2 = tmp0 ^ in6;
-        out0 = tmp0 ^ tmp1;
-        out4 = tmp1 ^ tmp2;
-        out7 = out2 ^ in2;
-        out1 = out2 ^ in3 ^ in5;
-        out6 = out4 ^ in0 ^ in2;
-        out5 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_39(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0;
-        tmp0 = in1 ^ in5;
-        tmp1 = tmp0 ^ in4;
-        out1 = tmp1 ^ in6;
-        out5 = out1 ^ in0 ^ in2;
-        tmp2 = tmp0 ^ out5;
-        out2 = tmp2 ^ in0 ^ in3;
-        out7 = out2 ^ in7;
-        out6 = tmp1 ^ out7;
-        out4 = tmp2 ^ out6;
-        out0 = out4 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in0 ^ in2;
-        tmp2 = in3 ^ in4;
-        tmp3 = in1 ^ in6;
-        tmp4 = in3 ^ in7;
-        out4 = tmp0 ^ in5;
-        out5 = tmp1 ^ tmp3;
-        out3 = tmp1 ^ tmp4;
-        out0 = tmp2 ^ in5;
-        out7 = tmp2 ^ in2;
-        tmp5 = tmp3 ^ in4;
-        out2 = tmp4 ^ tmp5;
-        out1 = tmp5 ^ out4;
-        out6 = tmp0 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in6;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in3;
-        out3 = tmp1 ^ in0;
-        out6 = tmp1 ^ tmp2;
-        out2 = out6 ^ in4;
-        out7 = tmp0 ^ out2;
-        out0 = out3 ^ out7 ^ in5;
-        out5 = out0 ^ out2 ^ in7;
-        out1 = tmp2 ^ out0;
-        out4 = out1 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in2 ^ in7;
-        tmp2 = in1 ^ in6 ^ in7;
-        out2 = tmp0 ^ in4;
-        out3 = tmp0 ^ tmp2;
-        out4 = tmp1 ^ out3 ^ in5;
-        out5 = tmp2 ^ out2 ^ in2;
-        out1 = out4 ^ out5 ^ in6;
-        out0 = out1 ^ in3;
-        out7 = tmp1 ^ out0;
-        out6 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = tmp0 ^ in3;
-        out2 = tmp1 ^ in4;
-        tmp2 = out2 ^ in5;
-        out4 = tmp2 ^ in1 ^ in6;
-        out5 = out4 ^ in7;
-        out6 = out5 ^ in0;
-        out7 = out6 ^ in1;
-        out0 = tmp0 ^ out7;
-        out1 = tmp1 ^ out5;
-        out3 = tmp2 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = tmp0 ^ in4;
-        out0 = tmp1 ^ in6;
-        out7 = tmp1 ^ in2;
-        out6 = out7 ^ in1 ^ in5 ^ in7;
-        out2 = out6 ^ in0 ^ in2;
-        out4 = out0 ^ out6 ^ in0;
-        out5 = tmp0 ^ out4;
-        out3 = out5 ^ in7;
-        out1 = out3 ^ out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        out3 = tmp0 ^ in2 ^ in6;
-        tmp1 = out3 ^ in5 ^ in7;
-        out4 = tmp1 ^ in4;
-        out5 = tmp1 ^ in3;
-        out1 = out4 ^ in2;
-        out7 = out1 ^ out3 ^ in3;
-        out2 = tmp0 ^ out7 ^ in5;
-        tmp2 = out2 ^ in0;
-        out6 = tmp2 ^ in6;
-        out0 = tmp1 ^ tmp2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_40(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3 ^ in7;
-        tmp0 = in3 ^ in4;
-        tmp1 = in6 ^ in7;
-        out4 = tmp0 ^ in2;
-        out5 = tmp0 ^ in5;
-        out0 = tmp1 ^ in2;
-        out7 = tmp1 ^ in1 ^ in5;
-        out2 = out0 ^ in4;
-        out3 = out2 ^ out5 ^ in7;
-        out6 = out3 ^ out4 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_41(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in3;
-        tmp0 = in5 ^ in6;
-        tmp1 = in6 ^ in7;
-        out5 = in3 ^ in4;
-        out1 = in1 ^ in3 ^ in7;
-        out6 = in0 ^ in4 ^ in5;
-        out3 = tmp0 ^ in2;
-        out7 = tmp0 ^ in1;
-        out2 = tmp1 ^ in4;
-        out0 = tmp1 ^ in0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_42(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in6;
-        out5 = in3 ^ in5;
-        out1 = in0 ^ in3 ^ in7;
-        out7 = in1 ^ in5 ^ in7;
-        out4 = in2 ^ in4 ^ in7;
-        out6 = in0 ^ in4 ^ in6;
-        out2 = out0 ^ in1 ^ in4;
-        out3 = out5 ^ in6 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_43(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in3;
-        out7 = in1 ^ in5;
-        out4 = in2 ^ in7;
-        out6 = in0 ^ in4;
-        out0 = in0 ^ in2 ^ in6;
-        out3 = in5 ^ in6 ^ in7;
-        out2 = in1 ^ in4 ^ in6;
-        out1 = in0 ^ in1 ^ in3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_44(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3;
-        out0 = in2 ^ in7;
-        tmp0 = in4 ^ in7;
-        out7 = in1 ^ in6 ^ in7;
-        out6 = in0 ^ in5 ^ in6;
-        out4 = tmp0 ^ in3 ^ in6;
-        out3 = out0 ^ in1 ^ in3 ^ in5;
-        out2 = out0 ^ in0 ^ in4;
-        out5 = tmp0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_45(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in1 ^ in3;
-        out7 = in1 ^ in6;
-        out5 = in4 ^ in7;
-        out6 = in0 ^ in5;
-        out0 = in0 ^ in2 ^ in7;
-        out4 = in3 ^ in6 ^ in7;
-        out2 = out5 ^ in0;
-        out3 = out0 ^ out6 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_46(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2;
-        out1 = in0 ^ in3;
-        out7 = in1 ^ in7;
-        out4 = in4 ^ in6;
-        out5 = in5 ^ in7;
-        out6 = in0 ^ in6;
-        out3 = in1 ^ in3 ^ in5;
-        out2 = out4 ^ out6 ^ in1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_47(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in6;
-        out7 = in1;
-        out5 = in7;
-        out6 = in0;
-        tmp0 = in0 ^ in1;
-        out3 = in1 ^ in5;
-        out0 = in0 ^ in2;
-        out1 = tmp0 ^ in3;
-        out2 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_48(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out1 = in3 ^ in6 ^ in7;
-        out3 = tmp0 ^ in0;
-        out0 = tmp0 ^ out1 ^ in5;
-        tmp1 = out0 ^ in4;
-        out2 = tmp1 ^ in7;
-        out5 = tmp1 ^ in3;
-        out4 = out5 ^ in1;
-        out7 = tmp0 ^ out4;
-        out6 = tmp1 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_49(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in2;
-        tmp0 = in2 ^ in5;
-        out2 = in4 ^ in5 ^ in6;
-        tmp1 = tmp0 ^ out2 ^ in3;
-        out7 = out2 ^ in1;
-        out5 = tmp1 ^ in7;
-        out4 = out5 ^ out7 ^ in6;
-        out1 = tmp0 ^ out4;
-        out6 = out1 ^ out7 ^ in0;
-        out0 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        tmp1 = in3 ^ in7;
-        out0 = tmp0 ^ in5;
-        out3 = tmp1 ^ in0;
-        out5 = tmp1 ^ out0;
-        out4 = out0 ^ in1 ^ in4;
-        out1 = out3 ^ in6;
-        out2 = out4 ^ in7;
-        out6 = out1 ^ in4;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in7;
-        tmp0 = in1 ^ in5;
-        tmp1 = in2 ^ in6;
-        tmp2 = out3 ^ in3;
-        out7 = tmp0 ^ in4;
-        out4 = tmp0 ^ tmp1;
-        tmp3 = tmp1 ^ in0;
-        out6 = tmp2 ^ in4;
-        out5 = tmp2 ^ tmp3;
-        out1 = tmp2 ^ in1 ^ in6;
-        out2 = out7 ^ in6 ^ in7;
-        out0 = tmp3 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3 ^ in6;
-        tmp0 = in2 ^ in5;
-        tmp1 = out1 ^ in5 ^ in7;
-        out0 = tmp0 ^ in7;
-        tmp2 = tmp0 ^ in4;
-        out6 = tmp1 ^ in0;
-        out2 = tmp2 ^ in0;
-        out5 = tmp2 ^ in6;
-        out3 = tmp0 ^ out6 ^ in1;
-        out7 = out0 ^ out5 ^ in1;
-        out4 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in6;
-        out4 = in1 ^ in3 ^ in5;
-        tmp2 = tmp0 ^ in7;
-        out2 = tmp0 ^ in4;
-        out1 = tmp1 ^ in3;
-        out7 = tmp1 ^ in4;
-        out0 = tmp2 ^ in2;
-        out6 = tmp2 ^ in3;
-        out5 = out7 ^ in1 ^ in2;
-        out3 = tmp1 ^ out0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in5;
-        out7 = in1 ^ in4 ^ in7;
-        out1 = in0 ^ in3 ^ in6;
-        out5 = out0 ^ in6;
-        out4 = out7 ^ in5;
-        out3 = out1 ^ in1;
-        out6 = out1 ^ in7;
-        out2 = out4 ^ in0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in2 ^ in6;
-        out7 = in1 ^ in4;
-        out3 = in0 ^ in1 ^ in6;
-        out4 = in1 ^ in5 ^ in7;
-        out0 = in0 ^ in2 ^ in5;
-        out6 = in0 ^ in3 ^ in7;
-        out1 = out3 ^ in3;
-        out2 = out4 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_50(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in7;
-        tmp0 = in3 ^ in5;
-        out0 = out2 ^ in4 ^ in6;
-        out1 = tmp0 ^ in7;
-        tmp1 = tmp0 ^ in6;
-        out3 = out0 ^ in3;
-        out7 = tmp1 ^ in1;
-        tmp2 = tmp1 ^ in0;
-        out5 = out3 ^ in1 ^ in2;
-        out4 = tmp2 ^ in2;
-        out6 = tmp2 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_51(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in7;
-        out3 = in2 ^ in4 ^ in6 ^ in7;
-        out0 = out3 ^ in0;
-        out6 = out0 ^ in5;
-        out4 = out6 ^ in3 ^ in7;
-        out1 = out0 ^ out4 ^ in1;
-        out7 = out1 ^ in6;
-        out5 = out7 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_52(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in2;
-        tmp0 = in2 ^ in4;
-        tmp1 = in3 ^ in5;
-        tmp2 = in3 ^ in6;
-        tmp3 = in0 ^ in7;
-        out0 = tmp0 ^ in6;
-        out6 = tmp0 ^ tmp3;
-        out7 = tmp1 ^ in1;
-        out1 = tmp1 ^ tmp3;
-        out3 = tmp2 ^ in4;
-        out5 = tmp2 ^ in1 ^ in7;
-        out4 = tmp2 ^ out1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_53(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1;
-        out3 = in4 ^ in6;
-        out0 = out3 ^ in0 ^ in2;
-        out6 = out0 ^ in7;
-        out4 = out6 ^ in5;
-        out7 = out0 ^ out4 ^ in1 ^ in3;
-        out1 = out7 ^ in0;
-        out5 = out7 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_54(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3 ^ in5;
-        tmp0 = in1 ^ in3;
-        tmp1 = in2 ^ in4;
-        tmp2 = in0 ^ in7;
-        out5 = in1 ^ in4 ^ in6;
-        out4 = tmp2 ^ out1;
-        out7 = tmp0 ^ in6;
-        out3 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ in7;
-        tmp3 = tmp2 ^ in2;
-        out2 = tmp3 ^ in6;
-        out6 = tmp3 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_55(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in1 ^ in4;
-        tmp2 = in6 ^ in7;
-        out7 = tmp0 ^ tmp2;
-        out1 = tmp0 ^ in5;
-        out3 = tmp1 ^ in2;
-        out5 = tmp1 ^ in5 ^ in6;
-        out2 = tmp2 ^ in0;
-        out4 = out5 ^ out7 ^ in0;
-        out6 = out2 ^ in2 ^ in5;
-        out0 = out5 ^ out6 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_56(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in4;
-        tmp0 = in0 ^ in2;
-        out4 = in0 ^ in5;
-        out7 = in1 ^ in3;
-        out5 = in1 ^ in6;
-        out6 = tmp0 ^ in7;
-        out2 = tmp0 ^ out5;
-        out1 = out4 ^ in3;
-        out3 = out7 ^ in4 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_57(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in7;
-        out0 = in0 ^ in2 ^ in4;
-        out5 = in1 ^ in5 ^ in6;
-        out4 = tmp0 ^ in4;
-        out1 = tmp0 ^ in1 ^ in3;
-        out2 = tmp0 ^ out5;
-        out3 = tmp1 ^ in4;
-        out7 = tmp1 ^ in3;
-        out6 = tmp1 ^ out2 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_58(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in5;
-        tmp0 = in2 ^ in3 ^ in4;
-        out5 = tmp0 ^ in1;
-        out6 = tmp0 ^ in0 ^ in5;
-        out3 = out6 ^ in7;
-        tmp1 = out2 ^ out5;
-        out7 = tmp1 ^ in6;
-        out4 = tmp1 ^ out3 ^ in3;
-        out0 = out4 ^ out7 ^ in0;
-        out1 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_59(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in5;
-        tmp0 = in0 ^ in5 ^ in7;
-        out3 = tmp0 ^ in2 ^ in4;
-        out0 = out3 ^ in6;
-        tmp1 = out0 ^ in7;
-        out6 = tmp1 ^ in3;
-        out5 = out6 ^ in0 ^ in1 ^ in6;
-        out4 = tmp0 ^ out5;
-        out1 = tmp1 ^ out4;
-        out7 = out1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in5;
-        out5 = tmp0 ^ in3;
-        out4 = tmp0 ^ in0;
-        tmp2 = tmp1 ^ in4;
-        out2 = tmp1 ^ in1 ^ in7;
-        out7 = tmp2 ^ out5;
-        out6 = out4 ^ out7 ^ in5;
-        out0 = tmp2 ^ in6;
-        out1 = out0 ^ out6 ^ in7;
-        out3 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in0 ^ in4;
-        tmp2 = in1 ^ in5;
-        out5 = tmp0 ^ tmp2;
-        tmp3 = tmp1 ^ in6;
-        out3 = tmp1 ^ in5;
-        out2 = tmp2 ^ in7;
-        tmp4 = out3 ^ in2;
-        out7 = out2 ^ in3 ^ in4;
-        out0 = tmp4 ^ in6;
-        out6 = tmp0 ^ tmp3;
-        out4 = tmp2 ^ tmp4;
-        out1 = tmp3 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in6;
-        tmp1 = in0 ^ in2 ^ in5;
-        out1 = tmp0 ^ in5;
-        tmp2 = tmp0 ^ in1;
-        out2 = tmp1 ^ in6;
-        out6 = tmp1 ^ in3;
-        out4 = tmp2 ^ in0;
-        out7 = tmp2 ^ in4;
-        out3 = tmp1 ^ out7;
-        out0 = out3 ^ out4 ^ in7;
-        out5 = out0 ^ in1 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in0 ^ in6;
-        out2 = tmp1 ^ in5;
-        tmp2 = out2 ^ in3;
-        out6 = tmp2 ^ in2;
-        out1 = tmp0 ^ tmp2;
-        tmp3 = out1 ^ in4 ^ in5;
-        out4 = tmp3 ^ in0;
-        out7 = tmp3 ^ in7;
-        tmp4 = out4 ^ out6;
-        out5 = tmp4 ^ in7;
-        out0 = tmp0 ^ out5;
-        out3 = tmp1 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in3 ^ in5;
-        tmp2 = in1 ^ in7;
-        out7 = in1 ^ in3 ^ in4;
-        out0 = tmp0 ^ in4;
-        tmp3 = tmp1 ^ in0;
-        out5 = tmp2 ^ in2;
-        out1 = tmp3 ^ in6;
-        out6 = tmp0 ^ tmp3;
-        tmp4 = tmp2 ^ out1;
-        out3 = tmp4 ^ in4;
-        out4 = tmp1 ^ tmp4;
-        out2 = tmp0 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in5;
-        tmp1 = in0 ^ in6;
-        tmp2 = tmp0 ^ in7;
-        tmp3 = tmp1 ^ in3;
-        out2 = tmp1 ^ tmp2;
-        out5 = tmp2 ^ in2;
-        out6 = tmp3 ^ in2;
-        out3 = out2 ^ in4;
-        out4 = out3 ^ in5;
-        out1 = tmp0 ^ tmp3;
-        out7 = tmp3 ^ out4;
-        out0 = out4 ^ out5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_60(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in5;
-        tmp0 = in3 ^ in6;
-        out1 = in3 ^ in4 ^ in7;
-        out7 = out4 ^ in1;
-        tmp1 = out4 ^ in4;
-        out0 = tmp0 ^ in2;
-        out5 = tmp0 ^ in0;
-        out2 = tmp0 ^ tmp1;
-        out3 = tmp1 ^ in7;
-        out6 = out3 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_61(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        out4 = tmp0 ^ in4;
-        tmp1 = out4 ^ in3;
-        out3 = tmp1 ^ in7;
-        out2 = tmp1 ^ in2 ^ in6;
-        out1 = tmp0 ^ out3 ^ in1;
-        out0 = out2 ^ out4 ^ in0;
-        out7 = tmp1 ^ out1;
-        out6 = out0 ^ out1 ^ in2;
-        out5 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_62(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in4 ^ in5;
-        tmp0 = in0 ^ in3 ^ in4;
-        out1 = tmp0 ^ in7;
-        out5 = tmp0 ^ in6;
-        tmp1 = out1 ^ in0;
-        tmp2 = tmp1 ^ out3;
-        out4 = tmp2 ^ in2;
-        tmp3 = tmp2 ^ in1;
-        out0 = out4 ^ in5 ^ in6;
-        out7 = tmp3 ^ out0;
-        out6 = tmp0 ^ tmp3;
-        out2 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_63(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in1 ^ in7;
-        out3 = tmp0 ^ in5;
-        tmp2 = out3 ^ in6;
-        out4 = out3 ^ in2 ^ in7;
-        out5 = tmp2 ^ in0;
-        tmp3 = out5 ^ in3;
-        out0 = tmp3 ^ out4;
-        out2 = tmp1 ^ tmp2;
-        out6 = tmp1 ^ tmp3;
-        tmp4 = tmp0 ^ out2;
-        out1 = tmp4 ^ out5;
-        out7 = tmp4 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_64(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in3;
-        out1 = in3 ^ in4;
-        out7 = in1 ^ in2;
-        tmp0 = in4 ^ in5;
-        tmp1 = in0 ^ in7;
-        out4 = in5 ^ in6 ^ in7;
-        out2 = tmp0 ^ out0 ^ in0;
-        out3 = tmp0 ^ out7 ^ in6;
-        out5 = tmp1 ^ in6;
-        out6 = tmp1 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_65(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in4 ^ in5;
-        tmp2 = in6 ^ in7;
-        out7 = in1 ^ in2 ^ in7;
-        out1 = in1 ^ in3 ^ in4;
-        out0 = tmp0 ^ in2;
-        out2 = tmp0 ^ tmp1;
-        out4 = tmp1 ^ tmp2;
-        tmp3 = tmp2 ^ in0;
-        out3 = out4 ^ out7 ^ in3;
-        out5 = tmp3 ^ in5;
-        out6 = tmp3 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_66(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in3;
-        tmp2 = in0 ^ in4;
-        out7 = tmp0 ^ in6;
-        out0 = tmp1 ^ in7;
-        out1 = tmp2 ^ in3;
-        tmp3 = tmp2 ^ in6;
-        tmp4 = out1 ^ in5;
-        out5 = tmp3 ^ in7;
-        out4 = tmp3 ^ tmp4;
-        out2 = tmp0 ^ tmp4 ^ in7;
-        out6 = tmp1 ^ out2 ^ in4;
-        out3 = tmp3 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_67(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp0 ^ in7;
-        out1 = tmp1 ^ in4;
-        out0 = tmp2 ^ in2;
-        tmp3 = out1 ^ in7;
-        out2 = tmp3 ^ in5;
-        out3 = out2 ^ in0 ^ in6;
-        out7 = tmp1 ^ out0 ^ in6;
-        out5 = tmp1 ^ out3;
-        out4 = tmp2 ^ out5;
-        out6 = tmp3 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_68(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in2 ^ in3 ^ in5;
-        tmp2 = tmp0 ^ in1;
-        tmp3 = tmp0 ^ in6;
-        out0 = tmp1 ^ in6;
-        out6 = tmp2 ^ in0;
-        out7 = tmp1 ^ tmp2;
-        out1 = tmp3 ^ in7;
-        out2 = out1 ^ in2;
-        out4 = tmp2 ^ out2;
-        out3 = out4 ^ out6 ^ in3;
-        out5 = tmp3 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_69(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in6 ^ in7;
-        out2 = tmp0 ^ in3 ^ in4;
-        out1 = out2 ^ in1;
-        out3 = out2 ^ in0 ^ in2;
-        out4 = out1 ^ in2 ^ in3;
-        out6 = out1 ^ in0 ^ in7;
-        out7 = out4 ^ in5 ^ in6;
-        out5 = out4 ^ out6 ^ in5;
-        out0 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        out3 = in0 ^ in4 ^ in6;
-        tmp1 = tmp0 ^ in3;
-        out4 = tmp1 ^ in1;
-        tmp2 = tmp1 ^ in7;
-        out2 = out4 ^ in4;
-        out0 = tmp2 ^ in5;
-        out5 = tmp2 ^ out3;
-        out7 = out2 ^ in3 ^ in5;
-        out1 = tmp0 ^ out5;
-        out6 = tmp1 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in6;
-        out2 = tmp0 ^ in1 ^ in3;
-        out4 = out2 ^ in2;
-        tmp1 = out2 ^ in0;
-        out7 = out4 ^ in3 ^ in5 ^ in7;
-        out1 = tmp1 ^ in7;
-        out3 = tmp1 ^ in1;
-        out6 = tmp1 ^ in5;
-        out0 = tmp1 ^ out7 ^ in6;
-        out5 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1;
-        tmp0 = in2 ^ in3;
-        out5 = in0 ^ in2;
-        out1 = in3 ^ in4 ^ in6;
-        tmp1 = out5 ^ in1;
-        out0 = tmp0 ^ in5;
-        out6 = tmp0 ^ tmp1;
-        out3 = tmp1 ^ in4;
-        out7 = out3 ^ in0;
-        out2 = out6 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in4;
-        tmp0 = in0 ^ in2;
-        tmp1 = out4 ^ in3;
-        out7 = out4 ^ in2 ^ in7;
-        out5 = tmp0 ^ in5;
-        out3 = tmp0 ^ tmp1;
-        out1 = tmp1 ^ in6;
-        out0 = out5 ^ in3;
-        out2 = out3 ^ out7 ^ in4;
-        out6 = out1 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in0 ^ in4;
-        out4 = tmp0 ^ in7;
-        out6 = tmp0 ^ in0 ^ in5;
-        out5 = tmp1 ^ in2;
-        tmp2 = tmp1 ^ in3;
-        out3 = tmp2 ^ out4;
-        out1 = tmp2 ^ in6;
-        out2 = tmp0 ^ out5;
-        out0 = out2 ^ out3 ^ in5;
-        out7 = out1 ^ out2 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in7;
-        tmp1 = tmp0 ^ in4;
-        tmp2 = tmp0 ^ in0 ^ in2;
-        out4 = tmp1 ^ in1;
-        out0 = tmp2 ^ in5;
-        out3 = out4 ^ in0;
-        out2 = out3 ^ in7;
-        out1 = out2 ^ in6;
-        out6 = out1 ^ in4 ^ in5;
-        out7 = tmp2 ^ out1;
-        out5 = tmp1 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_70(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2;
-        tmp0 = in2 ^ in4;
-        out2 = in2 ^ in3 ^ in5;
-        tmp1 = tmp0 ^ in6;
-        tmp2 = out2 ^ in7;
-        out0 = tmp1 ^ in3;
-        out4 = tmp1 ^ in0;
-        out7 = tmp2 ^ in1;
-        out6 = out4 ^ in1;
-        out5 = out7 ^ in0 ^ in2;
-        out1 = tmp0 ^ tmp2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_71(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in5;
-        out3 = in2 ^ in3;
-        tmp0 = in0 ^ in2;
-        tmp1 = out2 ^ in1;
-        out4 = tmp0 ^ in6;
-        tmp2 = tmp0 ^ in1;
-        out7 = tmp1 ^ in2;
-        out1 = tmp1 ^ in4 ^ in7;
-        out0 = out4 ^ in3 ^ in4;
-        out6 = tmp2 ^ in4;
-        out5 = tmp2 ^ out3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_72(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in7;
-        tmp0 = in0 ^ in4;
-        tmp1 = tmp0 ^ in3 ^ in7;
-        out1 = tmp1 ^ in5;
-        out5 = out1 ^ in1;
-        tmp2 = tmp0 ^ out5;
-        out2 = tmp2 ^ in2;
-        out7 = out2 ^ in6;
-        out6 = tmp1 ^ out7;
-        out4 = tmp2 ^ out6;
-        out0 = out4 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_73(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in7;
-        out2 = out3 ^ in1 ^ in5;
-        out1 = out2 ^ in0 ^ in4;
-        out5 = out1 ^ in5;
-        out6 = out1 ^ out3 ^ in2;
-        out0 = out2 ^ out6 ^ in6;
-        out7 = out0 ^ out1 ^ in3;
-        out4 = out0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_74(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in1 ^ in2 ^ in6;
-        out4 = in0 ^ in4 ^ in7;
-        out5 = in0 ^ in1 ^ in5;
-        out0 = tmp0 ^ in2;
-        out1 = tmp0 ^ in5;
-        out3 = tmp1 ^ in7;
-        out6 = tmp1 ^ in0;
-        out2 = tmp1 ^ out5 ^ in3;
-        out7 = out3 ^ in3 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_75(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in7;
-        tmp0 = in1 ^ in3;
-        out5 = in0 ^ in1;
-        out7 = tmp0 ^ in2;
-        tmp1 = tmp0 ^ in4;
-        out6 = out5 ^ in2;
-        tmp2 = out7 ^ in6;
-        out1 = tmp1 ^ in5;
-        out0 = tmp1 ^ out6;
-        out3 = tmp2 ^ in7;
-        out2 = tmp2 ^ out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_76(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in6;
-        tmp0 = in0 ^ in5;
-        tmp1 = in3 ^ in7;
-        tmp2 = tmp0 ^ in4;
-        tmp3 = tmp1 ^ in2;
-        out5 = tmp2 ^ in1;
-        out1 = tmp2 ^ in3;
-        out0 = tmp3 ^ in4;
-        out4 = out1 ^ in5;
-        out7 = tmp3 ^ out3;
-        out2 = tmp0 ^ out7;
-        out6 = tmp1 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_77(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in3;
-        tmp0 = in1 ^ in4;
-        tmp1 = in1 ^ in6;
-        tmp2 = out4 ^ in5;
-        out5 = tmp0 ^ in0;
-        out1 = tmp0 ^ tmp2;
-        out3 = tmp1 ^ in3;
-        out2 = tmp1 ^ tmp2 ^ in7;
-        out7 = out3 ^ in2;
-        tmp3 = out7 ^ in6;
-        out6 = tmp2 ^ tmp3;
-        out0 = tmp3 ^ out5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_78(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in2 ^ in7;
-        tmp2 = in0 ^ in5 ^ in6;
-        out2 = tmp1 ^ in3;
-        out3 = tmp2 ^ in2;
-        out5 = out3 ^ in1 ^ in3;
-        out0 = tmp0 ^ out3 ^ in4;
-        out1 = tmp1 ^ out0;
-        out4 = out1 ^ out5 ^ in5;
-        out7 = tmp0 ^ out4;
-        out6 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_79(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in7;
-        tmp0 = in3 ^ in4;
-        tmp1 = in1 ^ in5;
-        tmp2 = tmp1 ^ in2;
-        out4 = tmp2 ^ in0 ^ in7;
-        tmp3 = out4 ^ in5;
-        out5 = tmp3 ^ out2 ^ in6;
-        out7 = tmp0 ^ tmp2;
-        out6 = tmp0 ^ tmp3;
-        out3 = tmp1 ^ out5;
-        out0 = out3 ^ in4;
-        out1 = tmp3 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        out2 = tmp0 ^ in3;
-        tmp1 = out2 ^ in4;
-        out4 = tmp1 ^ in0 ^ in5;
-        out5 = out4 ^ in6;
-        out6 = out5 ^ in7;
-        out7 = out6 ^ in0;
-        out0 = out7 ^ in1;
-        out1 = tmp0 ^ out6;
-        out3 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in3;
-        tmp0 = in0 ^ in5;
-        out4 = tmp0 ^ out2 ^ in2;
-        tmp1 = out4 ^ in4;
-        out6 = tmp1 ^ in7;
-        out5 = tmp1 ^ in5 ^ in6;
-        out0 = out6 ^ in1 ^ in6;
-        tmp2 = out0 ^ in2;
-        out1 = tmp2 ^ in1;
-        out3 = tmp2 ^ in4;
-        out7 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = tmp0 ^ in4;
-        out0 = tmp1 ^ in2;
-        out1 = tmp1 ^ in6;
-        out7 = out0 ^ in1 ^ in5 ^ in7;
-        out5 = out1 ^ out7 ^ in0;
-        out3 = out5 ^ in6;
-        out6 = tmp0 ^ out5;
-        out2 = out6 ^ in1;
-        out4 = out2 ^ out7 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = tmp0 ^ in3;
-        tmp2 = tmp0 ^ in6;
-        out7 = tmp1 ^ in4;
-        tmp3 = tmp2 ^ in0;
-        out5 = tmp3 ^ in7;
-        out4 = tmp3 ^ in2 ^ in5;
-        out2 = tmp1 ^ out5;
-        out6 = tmp2 ^ out2;
-        out0 = out4 ^ out7 ^ in6;
-        out1 = tmp3 ^ out0;
-        out3 = out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in0 ^ in5;
-        out1 = tmp0 ^ tmp1 ^ in6;
-        out3 = tmp1 ^ in1;
-        out4 = out1 ^ in1 ^ in7;
-        tmp2 = out4 ^ in3;
-        out5 = tmp2 ^ in2;
-        out6 = tmp0 ^ out5;
-        out7 = tmp1 ^ out4 ^ in2;
-        out2 = out6 ^ in5 ^ in7;
-        out0 = tmp2 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in7;
-        tmp1 = tmp0 ^ in3 ^ in5;
-        tmp2 = tmp1 ^ in0;
-        out0 = tmp2 ^ in4;
-        out6 = tmp2 ^ in1;
-        out3 = tmp0 ^ out6;
-        tmp3 = out3 ^ in6;
-        out1 = tmp3 ^ in4;
-        out2 = tmp3 ^ in5;
-        out4 = tmp3 ^ in7;
-        out5 = tmp1 ^ out1;
-        out7 = out0 ^ out4 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_80(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in4 ^ in5;
-        out1 = in2 ^ in6 ^ in7;
-        out5 = tmp0 ^ in4;
-        tmp2 = tmp0 ^ in1;
-        out6 = tmp1 ^ in3;
-        out7 = tmp1 ^ in0 ^ in6;
-        out4 = tmp2 ^ in7;
-        out3 = tmp2 ^ out6;
-        out2 = out3 ^ out5 ^ in6;
-        out0 = out2 ^ in3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_81(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in6;
-        tmp1 = tmp0 ^ in3;
-        out6 = tmp1 ^ in5;
-        out5 = out6 ^ in2 ^ in6;
-        out3 = out5 ^ in1;
-        out2 = tmp0 ^ out3;
-        out1 = out3 ^ out6 ^ in7;
-        out4 = tmp1 ^ out1;
-        out7 = out2 ^ out4 ^ in0;
-        out0 = out7 ^ in1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_82(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in2;
-        tmp0 = in6 ^ in7;
-        out5 = in2 ^ in3;
-        out6 = in3 ^ in4;
-        out7 = in0 ^ in4 ^ in5;
-        out0 = in1 ^ in5 ^ in6;
-        out1 = tmp0 ^ in0 ^ in2;
-        out2 = tmp0 ^ in3 ^ in5;
-        out3 = tmp0 ^ out0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_83(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in5;
-        tmp2 = in3 ^ in6;
-        out4 = in1 ^ in2 ^ in4;
-        out0 = tmp0 ^ in5 ^ in6;
-        out5 = tmp1 ^ in3;
-        tmp3 = tmp1 ^ in7;
-        out6 = tmp2 ^ in4;
-        out2 = tmp2 ^ tmp3;
-        tmp4 = tmp3 ^ out4;
-        out1 = tmp3 ^ out0;
-        out3 = tmp4 ^ in3;
-        out7 = tmp0 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_84(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2 ^ in6;
-        out6 = in3 ^ in5;
-        out0 = in1 ^ in5 ^ in7;
-        out7 = in0 ^ in4 ^ in6;
-        out4 = in1 ^ in3 ^ in6;
-        out5 = in2 ^ in4 ^ in7;
-        out2 = out6 ^ in0 ^ in1;
-        out3 = out5 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_85(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in6;
-        tmp1 = in3 ^ in6;
-        tmp2 = tmp0 ^ in4;
-        out1 = tmp0 ^ in2;
-        out6 = tmp1 ^ in5;
-        out4 = tmp2 ^ in3;
-        tmp3 = out1 ^ out6;
-        out2 = tmp3 ^ in0;
-        out3 = tmp2 ^ tmp3 ^ in7;
-        out7 = out2 ^ out3 ^ in1;
-        out5 = tmp1 ^ out3;
-        out0 = tmp2 ^ out7 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_86(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in3;
-        out7 = in0 ^ in4;
-        out0 = in1 ^ in5;
-        out5 = in2 ^ in7;
-        out3 = in4 ^ in5 ^ in6;
-        out1 = in0 ^ in2 ^ in6;
-        out4 = in1 ^ in6 ^ in7;
-        out2 = in0 ^ in3 ^ in5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_87(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in3 ^ in6;
-        tmp0 = in0 ^ in1;
-        out7 = in0 ^ in4 ^ in7;
-        out5 = in2 ^ in5 ^ in7;
-        out3 = out6 ^ in4 ^ in5;
-        out0 = tmp0 ^ in5;
-        tmp1 = tmp0 ^ in6;
-        out2 = out5 ^ in0 ^ in3;
-        out1 = tmp1 ^ in2;
-        out4 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_88(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2 ^ in7;
-        tmp0 = in5 ^ in6;
-        out0 = in1 ^ in6 ^ in7;
-        out6 = in4 ^ in5 ^ in7;
-        out3 = out0 ^ out1 ^ in0 ^ in4;
-        out7 = tmp0 ^ in0;
-        tmp1 = tmp0 ^ in3;
-        out2 = out0 ^ in3;
-        out4 = tmp1 ^ in2;
-        out5 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_89(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in7;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in6;
-        out1 = tmp1 ^ in1;
-        out7 = tmp2 ^ in5;
-        out0 = tmp2 ^ in1;
-        out2 = out1 ^ in3 ^ in6;
-        out6 = out7 ^ in0 ^ in4;
-        out5 = out6 ^ in3;
-        out3 = tmp0 ^ out2 ^ in4;
-        out4 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in6;
-        out7 = in0 ^ in5;
-        out2 = in3 ^ in6;
-        out6 = in4 ^ in7;
-        out1 = in0 ^ in2 ^ in7;
-        out3 = out0 ^ out6 ^ in0;
-        out4 = out1 ^ out7 ^ in6;
-        out5 = out2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in3 ^ in6;
-        tmp2 = in5 ^ in7;
-        tmp3 = tmp0 ^ in7;
-        out0 = tmp0 ^ in6;
-        out2 = tmp1 ^ in2;
-        out5 = tmp1 ^ tmp2;
-        out7 = tmp2 ^ in0;
-        tmp4 = tmp3 ^ in4;
-        out1 = tmp3 ^ in2;
-        out6 = tmp4 ^ out0;
-        out4 = out6 ^ in2 ^ in5;
-        out3 = tmp1 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2;
-        out0 = in1 ^ in7;
-        out7 = in0 ^ in6;
-        out5 = in4 ^ in6;
-        out6 = in5 ^ in7;
-        out2 = out0 ^ in0 ^ in3;
-        out3 = out5 ^ out7 ^ in2 ^ in7;
-        out4 = out6 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in1 ^ in2;
-        tmp0 = in6 ^ in7;
-        out0 = in0 ^ in1 ^ in7;
-        out5 = in4 ^ in5 ^ in6;
-        out6 = tmp0 ^ in5;
-        out7 = tmp0 ^ in0;
-        out4 = tmp0 ^ out5 ^ in3;
-        out2 = out0 ^ in2 ^ in3;
-        out3 = out2 ^ in1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1;
-        out4 = in5;
-        out7 = in0;
-        out5 = in6;
-        out6 = in7;
-        out3 = in0 ^ in4;
-        out1 = in0 ^ in2;
-        out2 = in0 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in1;
-        tmp0 = in0 ^ in3;
-        out4 = in4 ^ in5;
-        out7 = in0 ^ in7;
-        out5 = in5 ^ in6;
-        out6 = in6 ^ in7;
-        out1 = out0 ^ in2;
-        out2 = tmp0 ^ in2;
-        out3 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_90(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in6 ^ in7;
-        out3 = tmp0 ^ in7;
-        out1 = tmp1 ^ in5;
-        tmp2 = out1 ^ in4;
-        out6 = tmp2 ^ in3;
-        out5 = out6 ^ in1;
-        out4 = out5 ^ in0;
-        out0 = tmp0 ^ tmp2;
-        out7 = tmp0 ^ out4;
-        out2 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_91(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = tmp0 ^ in3 ^ in5;
-        out2 = tmp1 ^ in1;
-        out6 = tmp1 ^ in7;
-        tmp2 = out2 ^ in5 ^ in7;
-        out3 = tmp2 ^ in4;
-        out5 = tmp2 ^ in6;
-        out1 = tmp1 ^ out5 ^ in2;
-        tmp3 = out1 ^ in0;
-        out4 = tmp3 ^ in3;
-        out0 = tmp0 ^ tmp3;
-        out7 = tmp2 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_92(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1;
-        tmp0 = in4 ^ in5;
-        tmp1 = tmp0 ^ in1;
-        out2 = tmp0 ^ in3 ^ in7;
-        out0 = tmp1 ^ in6;
-        out7 = out2 ^ in0;
-        out4 = out0 ^ in0 ^ in2;
-        out5 = out4 ^ out7 ^ in5;
-        out6 = tmp1 ^ out5;
-        out1 = out6 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_93(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in3;
-        tmp0 = in2 ^ in7;
-        tmp1 = out3 ^ in6;
-        tmp2 = tmp0 ^ in4;
-        out5 = tmp0 ^ tmp1;
-        out6 = tmp2 ^ in3;
-        out2 = out6 ^ in5;
-        out0 = out2 ^ out5 ^ in0;
-        out7 = tmp1 ^ out0;
-        out1 = tmp2 ^ out0;
-        out4 = out1 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_94(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in6;
-        tmp0 = in1 ^ in4 ^ in5;
-        out1 = out3 ^ in5;
-        out5 = tmp0 ^ out3;
-        out0 = tmp0 ^ in7;
-        out4 = tmp0 ^ in0 ^ in3;
-        out6 = out1 ^ in3 ^ in7;
-        out2 = out4 ^ in6;
-        out7 = out0 ^ out2 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_95(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out3 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in7;
-        tmp2 = out3 ^ in0;
-        out6 = tmp1 ^ in5;
-        tmp3 = tmp2 ^ in4;
-        out7 = tmp3 ^ in2;
-        tmp4 = tmp3 ^ in5;
-        out2 = tmp4 ^ in1;
-        tmp5 = out2 ^ in6;
-        out0 = tmp1 ^ tmp5;
-        out1 = tmp5 ^ out7;
-        out4 = tmp2 ^ out1;
-        out5 = tmp4 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_96(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in6 ^ in7;
-        tmp0 = in1 ^ in5;
-        tmp1 = in5 ^ in6;
-        out6 = out3 ^ in2 ^ in3;
-        out0 = tmp0 ^ in4;
-        tmp2 = tmp1 ^ in2;
-        out4 = out0 ^ in0 ^ in7;
-        out1 = tmp2 ^ in0;
-        out5 = tmp2 ^ in1;
-        out7 = tmp0 ^ out4 ^ in3;
-        out2 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_97(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        tmp1 = in2 ^ in6;
-        out3 = in3 ^ in6 ^ in7;
-        out7 = tmp0 ^ in3;
-        tmp2 = tmp0 ^ in5;
-        out5 = tmp1 ^ in1;
-        out6 = tmp1 ^ out3;
-        out0 = tmp2 ^ in1;
-        out2 = tmp2 ^ out3 ^ in2;
-        tmp3 = out0 ^ in4;
-        out4 = tmp3 ^ in7;
-        out1 = tmp1 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_98(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in7;
-        tmp1 = in1 ^ in4 ^ in7;
-        out1 = tmp0 ^ in2;
-        out0 = tmp1 ^ in6;
-        out2 = tmp1 ^ in3;
-        out6 = out0 ^ out1 ^ in1;
-        out5 = tmp0 ^ out2;
-        out3 = tmp1 ^ out6 ^ in0;
-        out7 = out0 ^ out5 ^ in0;
-        out4 = out6 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_99(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        out5 = in1 ^ in3 ^ in4;
-        out6 = in2 ^ in4 ^ in5;
-        out4 = tmp0 ^ in2;
-        tmp1 = tmp0 ^ in6;
-        tmp2 = out5 ^ in7;
-        out7 = tmp1 ^ in5;
-        out0 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ in2;
-        out3 = out0 ^ out6 ^ in3;
-        out1 = tmp1 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in6;
-        out5 = in1 ^ in3 ^ in5;
-        tmp2 = tmp0 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ in4;
-        out7 = tmp2 ^ in3;
-        out1 = tmp2 ^ in2;
-        out6 = out0 ^ in1 ^ in2;
-        out4 = out1 ^ in4 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in1 ^ in3;
-        tmp0 = in3 ^ in5;
-        out6 = in2 ^ in4;
-        out4 = in0 ^ in2 ^ in7;
-        out7 = tmp0 ^ in0;
-        out2 = out6 ^ in3;
-        out1 = out4 ^ in1 ^ in5;
-        out3 = out7 ^ in1 ^ in6;
-        out0 = tmp0 ^ out3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2 ^ in5;
-        tmp0 = in0 ^ in3 ^ in6;
-        out3 = out1 ^ in0;
-        out6 = out1 ^ in6;
-        out7 = tmp0 ^ in7;
-        out4 = out7 ^ in4;
-        out2 = out4 ^ in1;
-        out0 = tmp0 ^ out2;
-        out5 = out0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in2 ^ in5;
-        tmp0 = in0 ^ in3;
-        out5 = in1 ^ in4 ^ in7;
-        out1 = out6 ^ in1;
-        out3 = tmp0 ^ out6;
-        out7 = tmp0 ^ in6;
-        out0 = out5 ^ in0;
-        out4 = out7 ^ in7;
-        out2 = out5 ^ out7 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in4;
-        tmp0 = in0 ^ in5;
-        out6 = in2 ^ in6;
-        out7 = in0 ^ in3 ^ in7;
-        out4 = in0 ^ in4 ^ in6;
-        out5 = in1 ^ in5 ^ in7;
-        out1 = tmp0 ^ in2;
-        out3 = tmp0 ^ in7;
-        out2 = out4 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in2;
-        out7 = in0 ^ in3;
-        tmp0 = in0 ^ in1;
-        out4 = in0 ^ in6;
-        out5 = in1 ^ in7;
-        out1 = tmp0 ^ in2 ^ in5;
-        out2 = out7 ^ in2 ^ in4 ^ in6;
-        out3 = out7 ^ in5 ^ in7;
-        out0 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in6;
-        out2 = tmp0 ^ in7;
-        tmp1 = tmp0 ^ in5;
-        out6 = out2 ^ in3 ^ in4;
-        out0 = tmp1 ^ in3;
-        tmp2 = out0 ^ in2;
-        out3 = tmp2 ^ in7;
-        tmp3 = tmp2 ^ in1;
-        out5 = tmp3 ^ in0;
-        out4 = tmp3 ^ out6;
-        out7 = out5 ^ out6 ^ in1;
-        out1 = tmp1 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp0 ^ in4;
-        out4 = tmp1 ^ in7;
-        out7 = tmp2 ^ in0;
-        out6 = tmp2 ^ out4 ^ in3;
-        out3 = out4 ^ in6;
-        out2 = out3 ^ in5;
-        out1 = out2 ^ in4;
-        out5 = out1 ^ out6 ^ in0;
-        out0 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in6;
-        tmp0 = in1 ^ in3 ^ in5;
-        out3 = tmp0 ^ in6;
-        out4 = tmp0 ^ in2 ^ in4;
-        out0 = out3 ^ in7;
-        out6 = out0 ^ in4;
-        out1 = out0 ^ out4 ^ in0;
-        out7 = out1 ^ in5;
-        out5 = out7 ^ in3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in6;
-        out3 = in1 ^ in5 ^ in6;
-        tmp0 = out2 ^ in0;
-        out4 = out2 ^ out3 ^ in3;
-        tmp1 = tmp0 ^ in4;
-        out0 = tmp0 ^ out4 ^ in7;
-        out5 = tmp1 ^ in3;
-        out7 = tmp1 ^ in5;
-        out1 = tmp1 ^ in1 ^ in7;
-        out6 = tmp1 ^ out0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in2 ^ in4;
-        tmp2 = in2 ^ in5;
-        tmp3 = in0 ^ in7;
-        out0 = tmp0 ^ in5;
-        out6 = tmp0 ^ in6 ^ in7;
-        out1 = tmp1 ^ in6;
-        out7 = tmp1 ^ tmp3;
-        out3 = tmp2 ^ in3;
-        tmp4 = tmp2 ^ out1;
-        out2 = tmp3 ^ in1;
-        out5 = tmp4 ^ out7;
-        out4 = tmp4 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in5;
-        tmp0 = in1 ^ in6;
-        tmp1 = in0 ^ in1;
-        tmp2 = in2 ^ in4;
-        out6 = in1 ^ in3 ^ in7;
-        out4 = tmp0 ^ in5;
-        out1 = tmp0 ^ tmp2;
-        out0 = tmp1 ^ in3 ^ in5;
-        out2 = tmp1 ^ in2 ^ in7;
-        out7 = tmp2 ^ in0;
-        out5 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0;
-        out3 = in3 ^ in5 ^ in7;
-        out1 = in0 ^ in2 ^ in4 ^ in6;
-        out0 = out3 ^ in1;
-        out7 = out1 ^ in7;
-        out6 = out0 ^ in6;
-        out5 = out7 ^ in5;
-        out4 = out6 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in2;
-        out3 = in5 ^ in7;
-        out7 = out2 ^ in4 ^ in6;
-        out6 = out3 ^ in1 ^ in3;
-        out1 = out7 ^ in1;
-        out5 = out7 ^ in7;
-        out0 = out6 ^ in0;
-        out4 = out6 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = in1 ^ in6;
-        tmp2 = in0 ^ in2 ^ in7;
-        out1 = tmp0 ^ in7;
-        out4 = tmp0 ^ in6;
-        out0 = tmp1 ^ in3;
-        out2 = tmp1 ^ in5;
-        out6 = tmp1 ^ in4;
-        out7 = tmp2 ^ in5;
-        out3 = tmp2 ^ out0 ^ in6;
-        out5 = out7 ^ in2 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in6;
-        out6 = in1 ^ in4;
-        out7 = in0 ^ in2 ^ in5;
-        out5 = in0 ^ in3 ^ in7;
-        out2 = out4 ^ in1 ^ in5;
-        out1 = out6 ^ in2 ^ in7;
-        out0 = out2 ^ out7 ^ in3;
-        out3 = out1 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in1 ^ in3;
-        tmp2 = in6 ^ in7;
-        out1 = tmp0 ^ in4 ^ in7;
-        out3 = tmp1 ^ in0;
-        out0 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ in5;
-        out7 = tmp0 ^ out2;
-        out6 = out1 ^ out7 ^ in1;
-        out5 = out0 ^ out6 ^ in0;
-        out4 = out5 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in1;
-        tmp0 = in1 ^ in4;
-        tmp1 = in0 ^ in7;
-        out6 = tmp0 ^ in5;
-        out1 = tmp0 ^ tmp1 ^ in2;
-        out5 = tmp1 ^ in3 ^ in4;
-        out0 = tmp0 ^ out5 ^ in6;
-        out4 = out0 ^ out3 ^ in2;
-        out2 = out4 ^ in3 ^ in5;
-        out7 = tmp1 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in3;
-        out1 = in2 ^ in4;
-        tmp0 = in0 ^ in2;
-        out4 = in4 ^ in7;
-        out5 = in0 ^ in5;
-        out6 = in1 ^ in6;
-        out7 = tmp0 ^ in7;
-        out3 = tmp0 ^ in3 ^ in6;
-        out2 = out5 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in7;
-        out5 = in0;
-        out6 = in1;
-        out7 = in0 ^ in2;
-        out0 = in0 ^ in1 ^ in3;
-        out2 = out7 ^ in1 ^ in5;
-        out1 = in1 ^ in2 ^ in4;
-        out3 = out7 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in3 ^ in4;
-        tmp0 = in0 ^ in4;
-        tmp1 = in0 ^ in7;
-        out0 = in1 ^ in3 ^ in7;
-        out1 = tmp0 ^ in2;
-        out5 = tmp0 ^ in5;
-        tmp2 = tmp1 ^ in6;
-        out2 = tmp1 ^ in5;
-        out3 = tmp2 ^ in3;
-        out7 = tmp2 ^ in2;
-        out6 = tmp2 ^ out2 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in3;
-        tmp0 = in0 ^ in7;
-        out5 = in0 ^ in4;
-        out6 = in1 ^ in5;
-        out7 = in0 ^ in2 ^ in6;
-        out0 = tmp0 ^ in1 ^ in3;
-        out3 = tmp0 ^ in6;
-        out2 = tmp0 ^ in2 ^ in5;
-        out1 = out5 ^ in1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in4;
-        tmp1 = in3 ^ in6;
-        out2 = tmp0 ^ in7;
-        tmp2 = tmp0 ^ tmp1;
-        out0 = tmp2 ^ in5;
-        out3 = tmp2 ^ in2;
-        out6 = out3 ^ in6;
-        tmp3 = out6 ^ in0 ^ in1;
-        out7 = tmp3 ^ in5;
-        out5 = tmp3 ^ out2;
-        out1 = out0 ^ out5 ^ in0;
-        out4 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in4;
-        out2 = tmp0 ^ in2 ^ in7;
-        tmp1 = out2 ^ in6;
-        out1 = tmp1 ^ in5;
-        out3 = tmp1 ^ in7;
-        out4 = tmp1 ^ in0;
-        out6 = out3 ^ in3;
-        out0 = out6 ^ in0 ^ in2 ^ in5;
-        out5 = tmp1 ^ out0 ^ in1;
-        out7 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in4;
-        tmp0 = in4 ^ in7;
-        tmp1 = in1 ^ in3 ^ in6;
-        out3 = tmp0 ^ tmp1;
-        tmp2 = tmp1 ^ in0;
-        out0 = out3 ^ in5;
-        out4 = tmp2 ^ in2;
-        tmp3 = out4 ^ in6;
-        out5 = tmp0 ^ tmp3;
-        out1 = tmp3 ^ out0;
-        tmp4 = out1 ^ in7;
-        out7 = tmp4 ^ in3;
-        out6 = tmp2 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in6;
-        out3 = tmp1 ^ in4 ^ in7;
-        tmp2 = tmp0 ^ out3;
-        out0 = tmp2 ^ in3;
-        out1 = tmp2 ^ in2;
-        out5 = out0 ^ in2 ^ in6;
-        out7 = tmp1 ^ out5;
-        out4 = out7 ^ in1 ^ in5 ^ in7;
-        out6 = tmp0 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in1;
-        out5 = out4 ^ in2;
-        tmp0 = out4 ^ in4;
-        out6 = out5 ^ in0 ^ in3;
-        out7 = tmp0 ^ out6;
-        out2 = tmp0 ^ in6 ^ in7;
-        out3 = out7 ^ in0 ^ in7;
-        out0 = out5 ^ out7 ^ in5;
-        out1 = out0 ^ out6 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in4;
-        out4 = tmp0 ^ in4;
-        out3 = tmp1 ^ in7;
-        tmp2 = out4 ^ in5;
-        out7 = out3 ^ in0 ^ in3;
-        out0 = tmp2 ^ in3;
-        out2 = tmp0 ^ out3 ^ in6;
-        out5 = tmp1 ^ tmp2;
-        out6 = out2 ^ out7 ^ in2;
-        out1 = tmp0 ^ out0 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in4;
-        tmp0 = in1 ^ in2;
-        tmp1 = in0 ^ in4;
-        tmp2 = in3 ^ in5;
-        tmp3 = out3 ^ in1 ^ in7;
-        out5 = tmp0 ^ tmp1;
-        out6 = tmp0 ^ tmp2;
-        out2 = tmp1 ^ in6;
-        out4 = tmp1 ^ tmp3;
-        out0 = tmp3 ^ in5;
-        out1 = out2 ^ in2 ^ in5;
-        out7 = tmp2 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in4;
-        tmp0 = in0 ^ in4;
-        out2 = tmp0 ^ in2 ^ in6;
-        tmp1 = out2 ^ in7;
-        out1 = out2 ^ in1 ^ in5;
-        out7 = tmp1 ^ in3;
-        out5 = out1 ^ in6;
-        out6 = tmp0 ^ out1 ^ in3;
-        out0 = tmp1 ^ out6;
-        out4 = out0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in4;
-        tmp1 = in2 ^ in5;
-        out2 = tmp0 ^ in5;
-        out4 = tmp1 ^ in0;
-        tmp2 = tmp1 ^ in7;
-        out6 = tmp2 ^ out2;
-        out7 = out4 ^ in3;
-        out1 = tmp2 ^ in4;
-        out3 = tmp0 ^ out7;
-        out0 = out3 ^ out4 ^ in6;
-        out5 = out0 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in4 ^ in5;
-        out4 = tmp0 ^ tmp1;
-        tmp2 = tmp0 ^ in3 ^ in7;
-        out3 = out4 ^ in1;
-        out7 = tmp2 ^ in5;
-        out2 = out3 ^ in0;
-        out1 = out2 ^ in7;
-        out6 = out1 ^ in5 ^ in6;
-        out0 = tmp2 ^ out6;
-        out5 = tmp1 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in7;
-        out2 = tmp0 ^ in4;
-        tmp1 = out2 ^ in2;
-        out1 = tmp1 ^ in0;
-        out6 = tmp1 ^ in1;
-        out4 = out1 ^ in3 ^ in4;
-        tmp2 = out4 ^ out6;
-        out7 = out4 ^ in6 ^ in7;
-        out5 = tmp2 ^ in6;
-        out3 = tmp0 ^ tmp2;
-        out0 = out6 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in4 ^ in5 ^ in7;
-        tmp0 = out2 ^ in1;
-        out4 = out2 ^ in0 ^ in3;
-        out1 = tmp0 ^ in0;
-        out6 = tmp0 ^ in6;
-        out3 = out1 ^ in2;
-        tmp1 = out4 ^ out6 ^ in4;
-        out0 = tmp1 ^ in7;
-        out5 = tmp1 ^ in5;
-        out7 = tmp0 ^ tmp1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in2 ^ in4;
-        out0 = in1 ^ in3 ^ in4;
-        out6 = in1 ^ in2 ^ in7;
-        out7 = tmp0 ^ in3;
-        out5 = tmp0 ^ out6 ^ in6;
-        out1 = tmp1 ^ in5;
-        tmp2 = out1 ^ out5 ^ in1;
-        out3 = tmp2 ^ in3;
-        out4 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in1 ^ in4;
-        out0 = tmp0 ^ tmp1;
-        out7 = tmp0 ^ in2 ^ in7;
-        out1 = tmp1 ^ in2 ^ in5;
-        tmp2 = out1 ^ in0;
-        out2 = tmp2 ^ in6;
-        out3 = out2 ^ in1 ^ in7;
-        out4 = out3 ^ in2;
-        out5 = tmp1 ^ out4;
-        out6 = tmp2 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3 ^ in6;
-        out4 = tmp0 ^ in5;
-        out7 = tmp0 ^ in2;
-        out3 = out4 ^ in4;
-        out1 = out3 ^ out7 ^ in0;
-        out2 = out3 ^ in3 ^ in7;
-        out0 = out2 ^ out4 ^ in1;
-        out5 = tmp0 ^ out0;
-        out6 = out1 ^ out5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        out3 = tmp0 ^ in5 ^ in6;
-        out4 = out3 ^ in3;
-        tmp1 = out3 ^ in7;
-        out2 = tmp1 ^ in2;
-        out5 = tmp1 ^ in1;
-        tmp2 = out2 ^ in5;
-        out7 = tmp2 ^ in3 ^ in4;
-        tmp3 = tmp0 ^ out5;
-        out0 = tmp3 ^ out4;
-        out1 = tmp2 ^ tmp3;
-        out6 = tmp3 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in2 ^ in5;
-        tmp0 = in1 ^ in4;
-        tmp1 = in3 ^ in6;
-        out0 = out5 ^ in1;
-        out4 = tmp0 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out1 = tmp1 ^ in2;
-        out6 = tmp1 ^ in0;
-        out7 = out4 ^ in0;
-        out2 = out4 ^ out5 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in2;
-        tmp0 = in0 ^ in1;
-        out4 = in1 ^ in7;
-        out6 = in0 ^ in3;
-        out3 = in1 ^ in4 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out7 = tmp0 ^ in4;
-        out0 = tmp1 ^ in5;
-        out1 = tmp1 ^ out6 ^ in6;
-        out2 = out6 ^ out7 ^ in5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in3 ^ in4;
-        tmp0 = in0 ^ in3 ^ in6;
-        out5 = in2 ^ in4 ^ in5;
-        tmp1 = out4 ^ in7;
-        out1 = tmp0 ^ in2;
-        out6 = tmp0 ^ in5;
-        out2 = out5 ^ in3;
-        out7 = tmp0 ^ tmp1;
-        out3 = tmp1 ^ in2 ^ in6;
-        out0 = tmp1 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in3;
-        tmp0 = in0 ^ in2;
-        tmp1 = in3 ^ in5;
-        out5 = in2 ^ in4;
-        tmp2 = tmp0 ^ out4;
-        out2 = tmp1 ^ in4;
-        out6 = tmp1 ^ in0;
-        out0 = tmp1 ^ tmp2 ^ in7;
-        out1 = tmp2 ^ in6;
-        out7 = out1 ^ out5 ^ in3;
-        out3 = tmp0 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in7;
-        out3 = tmp0 ^ in4;
-        tmp1 = tmp0 ^ in2;
-        out1 = tmp1 ^ in6;
-        out5 = tmp1 ^ in5;
-        out4 = out1 ^ out3 ^ in1;
-        out0 = out4 ^ in4 ^ in5;
-        out2 = out0 ^ out3 ^ in0;
-        out7 = out1 ^ out2 ^ in7;
-        out6 = tmp1 ^ out0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in4 ^ in7;
-        tmp0 = in3 ^ in7;
-        out4 = in1 ^ in2 ^ in6;
-        out6 = in0 ^ in3 ^ in4;
-        out5 = tmp0 ^ in2;
-        out1 = tmp0 ^ out4;
-        out0 = out4 ^ in0 ^ in5;
-        out2 = out0 ^ out5 ^ in4;
-        out7 = tmp0 ^ out2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in6;
-        tmp1 = in1 ^ in7;
-        tmp2 = tmp0 ^ in0;
-        tmp3 = tmp0 ^ tmp1;
-        tmp4 = tmp2 ^ in4;
-        out0 = tmp3 ^ in2;
-        out6 = tmp4 ^ in3;
-        out2 = out6 ^ in2;
-        out7 = tmp1 ^ tmp4;
-        out3 = tmp2 ^ out2;
-        tmp5 = out3 ^ in5;
-        out5 = tmp5 ^ in7;
-        out4 = tmp3 ^ tmp5;
-        out1 = tmp4 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in4;
-        tmp0 = in3 ^ in5;
-        tmp1 = out3 ^ in7;
-        out6 = tmp0 ^ in0 ^ in4;
-        out5 = tmp1 ^ in3;
-        out2 = out6 ^ in6;
-        out7 = out2 ^ in1 ^ in3;
-        out0 = tmp1 ^ out7;
-        out1 = tmp0 ^ out0;
-        out4 = out1 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in2;
-        out1 = in2 ^ in3;
-        tmp0 = in5 ^ in6;
-        tmp1 = in0 ^ in7;
-        out2 = out1 ^ in1 ^ in4;
-        out4 = tmp0 ^ in4;
-        out5 = tmp0 ^ in7;
-        out6 = tmp1 ^ in6;
-        out7 = tmp1 ^ in1;
-        out3 = out2 ^ in0 ^ in2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in5 ^ in6;
-        out7 = in0 ^ in1;
-        tmp0 = in1 ^ in3;
-        out5 = in6 ^ in7;
-        out6 = in0 ^ in7;
-        out0 = out7 ^ in2;
-        out3 = out7 ^ in4 ^ in5;
-        out1 = tmp0 ^ in2;
-        out2 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in7;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in6;
-        out0 = tmp1 ^ in1;
-        tmp3 = tmp1 ^ in3;
-        out6 = tmp2 ^ in5;
-        out7 = tmp2 ^ in1;
-        out2 = tmp3 ^ in4;
-        out5 = out6 ^ in0 ^ in4;
-        out4 = out5 ^ in3;
-        out1 = tmp0 ^ tmp3;
-        out3 = tmp3 ^ out5 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in7;
-        tmp1 = in5 ^ in7;
-        out7 = in0 ^ in1 ^ in6;
-        out5 = tmp0 ^ in6;
-        out2 = tmp0 ^ in3;
-        out6 = tmp1 ^ in0;
-        out4 = tmp1 ^ in3 ^ in6;
-        tmp2 = out5 ^ out7 ^ in2;
-        out1 = tmp2 ^ out2;
-        out0 = tmp2 ^ in4;
-        out3 = tmp2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = in1 ^ in6;
-        out1 = in2 ^ in3 ^ in7;
-        out5 = tmp0 ^ in6;
-        out0 = tmp1 ^ in2;
-        tmp2 = out5 ^ in0 ^ in7;
-        out3 = tmp2 ^ in4;
-        out6 = tmp0 ^ out3;
-        out7 = tmp1 ^ tmp2 ^ in3;
-        tmp3 = out1 ^ out6;
-        out4 = tmp2 ^ tmp3;
-        out2 = tmp3 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in3 ^ in6;
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in7;
-        out6 = in0 ^ in4 ^ in7;
-        out2 = tmp0 ^ out5 ^ in4;
-        out7 = tmp0 ^ in5;
-        out0 = tmp0 ^ in2 ^ in6;
-        out4 = tmp1 ^ in5;
-        out1 = tmp1 ^ in1 ^ in3;
-        out3 = out6 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = tmp0 ^ in3;
-        out4 = tmp1 ^ in4;
-        tmp2 = out4 ^ in6;
-        out3 = tmp2 ^ in0;
-        out5 = tmp2 ^ in2;
-        out2 = out3 ^ in5 ^ in7;
-        out6 = tmp1 ^ out2;
-        out7 = out2 ^ out4 ^ in1;
-        out1 = tmp2 ^ out6;
-        out0 = tmp0 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in6;
-        tmp1 = in0 ^ in1 ^ in5;
-        out4 = in2 ^ in3 ^ in5;
-        out5 = tmp0 ^ in4;
-        out7 = tmp1 ^ in6;
-        out1 = tmp1 ^ out4 ^ in7;
-        tmp2 = out5 ^ in0;
-        out2 = tmp2 ^ in7;
-        out3 = tmp2 ^ out4;
-        out6 = tmp0 ^ out2 ^ in5;
-        out0 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in1 ^ in4;
-        tmp2 = in2 ^ in5;
-        out7 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ tmp2;
-        tmp3 = tmp2 ^ in3;
-        out1 = tmp3 ^ in6;
-        tmp4 = out1 ^ in1;
-        out2 = tmp4 ^ in7;
-        out3 = out2 ^ in2;
-        out4 = tmp0 ^ out3;
-        out5 = tmp3 ^ out3;
-        out6 = tmp4 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5 ^ in6;
-        tmp1 = tmp0 ^ in1;
-        out1 = tmp1 ^ in2;
-        out2 = tmp1 ^ in7;
-        out3 = out2 ^ in3;
-        out5 = out3 ^ in2;
-        tmp2 = out3 ^ in0;
-        out4 = tmp2 ^ in4;
-        out7 = tmp0 ^ out4;
-        out6 = tmp2 ^ out1 ^ in6;
-        out0 = out2 ^ out6 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in6;
-        out2 = tmp0 ^ in2 ^ in3;
-        out1 = out2 ^ in0;
-        out3 = out2 ^ in1;
-        out4 = out1 ^ in1 ^ in2;
-        out6 = out1 ^ in6 ^ in7;
-        out7 = out4 ^ in4 ^ in5;
-        out5 = out4 ^ out6 ^ in4;
-        out0 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in5 ^ in6;
-        tmp0 = out2 ^ in2;
-        tmp1 = tmp0 ^ in1;
-        out1 = tmp1 ^ in0;
-        out3 = tmp1 ^ in3;
-        out4 = out1 ^ in2 ^ in4;
-        tmp2 = out4 ^ in5;
-        out7 = tmp2 ^ in7;
-        out0 = tmp0 ^ out7;
-        tmp3 = out0 ^ in0;
-        out5 = tmp3 ^ in6;
-        out6 = tmp2 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in5;
-        tmp0 = in1 ^ in5;
-        tmp1 = tmp0 ^ in2;
-        out4 = tmp1 ^ in0;
-        tmp2 = tmp1 ^ in6;
-        out2 = out4 ^ in3 ^ in7;
-        out0 = tmp2 ^ in4;
-        out5 = tmp2 ^ out3;
-        out1 = tmp0 ^ out5 ^ in7;
-        out6 = tmp0 ^ out2 ^ in4;
-        out7 = tmp1 ^ out6 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in5;
-        tmp0 = in0 ^ in4;
-        tmp1 = tmp0 ^ in1 ^ in5;
-        out4 = tmp1 ^ in2;
-        out0 = out4 ^ in6;
-        tmp2 = tmp0 ^ out0;
-        out5 = tmp2 ^ in3;
-        out1 = out5 ^ in7;
-        out6 = tmp1 ^ out1;
-        out7 = tmp2 ^ out6;
-        out2 = out7 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2 ^ in4 ^ in6;
-        out5 = tmp0 ^ in3;
-        out0 = tmp0 ^ in5 ^ in7;
-        out3 = out0 ^ out5 ^ in2;
-        tmp1 = out3 ^ in0;
-        out1 = tmp1 ^ in6;
-        out2 = tmp1 ^ in7;
-        out4 = tmp1 ^ in1;
-        out6 = tmp1 ^ in4;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        out3 = in2 ^ in5 ^ in7;
-        out2 = tmp0 ^ in5;
-        tmp1 = tmp0 ^ out3 ^ in1;
-        out1 = tmp1 ^ in6;
-        out4 = tmp1 ^ in4;
-        tmp2 = out1 ^ in4;
-        out6 = tmp2 ^ in1;
-        out7 = tmp2 ^ in2;
-        out0 = tmp2 ^ in3;
-        out5 = tmp2 ^ in0 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0;
-        out5 = in1;
-        tmp0 = in1 ^ in2;
-        out6 = in0 ^ in2;
-        out0 = tmp0 ^ in4;
-        tmp1 = tmp0 ^ in3;
-        out7 = tmp1 ^ out6;
-        out2 = tmp1 ^ in6;
-        out3 = out7 ^ in7;
-        out1 = tmp1 ^ in1 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in4;
-        out5 = in1 ^ in5;
-        out2 = in1 ^ in3 ^ in6;
-        out3 = in0 ^ in1 ^ in7;
-        out6 = in0 ^ in2 ^ in6;
-        out0 = out4 ^ in1 ^ in2;
-        out1 = out5 ^ in2 ^ in3;
-        out7 = out3 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in1 ^ in4;
-        tmp0 = in2 ^ in7;
-        tmp1 = in0 ^ in2 ^ in3;
-        out0 = tmp0 ^ out5;
-        out4 = tmp0 ^ tmp1;
-        out2 = tmp0 ^ in3 ^ in6;
-        out1 = tmp1 ^ in5;
-        out3 = tmp1 ^ in1;
-        out6 = out1 ^ in3;
-        out7 = out3 ^ in2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in1 ^ in5;
-        tmp2 = in3 ^ in7;
-        out3 = tmp0 ^ in2;
-        out5 = tmp1 ^ in4;
-        out6 = tmp1 ^ out3 ^ in6;
-        out2 = tmp2 ^ in6;
-        tmp3 = tmp2 ^ in4;
-        tmp4 = out3 ^ in3;
-        out4 = tmp3 ^ in0;
-        out1 = tmp4 ^ in5;
-        out0 = tmp3 ^ tmp4;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in0 ^ in3;
-        out6 = tmp0 ^ in4;
-        tmp2 = tmp0 ^ in7;
-        out3 = tmp1 ^ in6;
-        tmp3 = tmp1 ^ in1;
-        out1 = tmp1 ^ tmp2 ^ in5;
-        out4 = tmp2 ^ in6;
-        out2 = tmp3 ^ in2;
-        out7 = tmp3 ^ in5;
-        out5 = tmp2 ^ out2;
-        out0 = out2 ^ out3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in6;
-        out2 = in0 ^ in1 ^ in3;
-        out6 = out3 ^ in2 ^ in4;
-        out7 = out2 ^ in5 ^ in7;
-        out0 = out6 ^ in1;
-        out4 = out6 ^ in7;
-        out5 = out7 ^ in0;
-        out1 = out5 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3 ^ in6;
-        tmp1 = in3 ^ in4 ^ in7;
-        out4 = tmp0 ^ in0;
-        out5 = tmp1 ^ in1;
-        out3 = out4 ^ in7;
-        out2 = out3 ^ in6;
-        out1 = out2 ^ in5;
-        out6 = tmp1 ^ out1;
-        out0 = tmp0 ^ out5;
-        out7 = out0 ^ out1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in3 ^ in7;
-        tmp0 = out2 ^ in1 ^ in5;
-        out1 = tmp0 ^ in2;
-        out7 = tmp0 ^ in6;
-        out5 = tmp0 ^ in0 ^ in4;
-        tmp1 = out1 ^ out5 ^ in6;
-        out4 = tmp1 ^ in3;
-        out6 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in7;
-        out0 = tmp2 ^ in1;
-        out3 = tmp2 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in7;
-        tmp0 = in2 ^ in4;
-        out4 = out3 ^ in3 ^ in5;
-        out2 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in6;
-        out0 = out4 ^ in2;
-        out6 = out4 ^ in0;
-        out1 = tmp1 ^ in3;
-        out5 = tmp1 ^ in0;
-        out7 = out5 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in4;
-        tmp0 = in1 ^ in7;
-        out3 = tmp0 ^ in3;
-        tmp1 = out3 ^ in5;
-        out4 = tmp1 ^ in4;
-        tmp2 = tmp1 ^ in0;
-        out0 = tmp2 ^ in2;
-        out6 = tmp2 ^ in6;
-        tmp3 = out0 ^ out4 ^ in6;
-        out5 = tmp3 ^ in5;
-        out7 = tmp0 ^ tmp3;
-        out1 = tmp2 ^ out5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in2;
-        out4 = in1 ^ in5;
-        out2 = in2 ^ in4 ^ in7;
-        out5 = in0 ^ in2 ^ in6;
-        out0 = out3 ^ in3 ^ in5;
-        out7 = out3 ^ in0 ^ in4;
-        out6 = out2 ^ out7 ^ in3;
-        out1 = out5 ^ in3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in4 ^ in7;
-        tmp0 = in1 ^ in3;
-        out3 = tmp0 ^ in2;
-        tmp1 = out3 ^ in0;
-        out0 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in4;
-        out1 = tmp2 ^ in6;
-        tmp3 = tmp2 ^ in3;
-        out7 = tmp3 ^ in7;
-        out6 = out1 ^ out2 ^ in2;
-        tmp4 = tmp0 ^ out0;
-        out5 = tmp4 ^ in6;
-        out4 = tmp3 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in6;
-        tmp0 = in0 ^ in4;
-        tmp1 = tmp0 ^ in2 ^ in6;
-        out2 = tmp1 ^ in1;
-        out7 = out2 ^ in5;
-        tmp2 = tmp0 ^ out7;
-        out4 = tmp2 ^ in3;
-        out0 = out4 ^ in7;
-        out6 = tmp1 ^ out0;
-        out5 = tmp2 ^ out6;
-        out1 = out5 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in6;
-        tmp0 = in0 ^ in1;
-        tmp1 = in5 ^ in7;
-        out2 = tmp0 ^ in4 ^ in6;
-        tmp2 = tmp1 ^ out2;
-        out6 = tmp2 ^ in3;
-        out7 = tmp2 ^ in2;
-        out0 = out6 ^ in2 ^ in4;
-        out5 = out6 ^ in1 ^ in2;
-        out1 = tmp0 ^ out5 ^ in5;
-        out4 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in6 ^ in7;
-        out2 = out3 ^ in0 ^ in4;
-        out4 = out3 ^ in1 ^ in5;
-        out1 = out2 ^ in3;
-        out7 = out2 ^ out4 ^ in2;
-        out0 = out4 ^ in3 ^ in7;
-        out5 = out1 ^ in4;
-        out6 = out0 ^ out2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out3 = tmp0 ^ in6 ^ in7;
-        tmp1 = out3 ^ in0;
-        out5 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in4;
-        tmp3 = out5 ^ in7;
-        out1 = tmp2 ^ in1;
-        out0 = tmp3 ^ in1;
-        out6 = out1 ^ in2;
-        out2 = tmp0 ^ tmp2;
-        tmp4 = tmp3 ^ out6;
-        out4 = tmp4 ^ in6;
-        out7 = tmp4 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in3 ^ in6;
-        tmp0 = in4 ^ in7;
-        out1 = in2 ^ in3 ^ in4;
-        out5 = tmp0 ^ in0;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp1 ^ in5;
-        out0 = tmp1 ^ out1;
-        out2 = tmp2 ^ in2;
-        out6 = tmp2 ^ out5;
-        tmp3 = out6 ^ in6;
-        out3 = tmp3 ^ in7;
-        out7 = tmp3 ^ in2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in3 ^ in6;
-        tmp2 = tmp0 ^ in6;
-        out4 = tmp1 ^ in4;
-        out6 = tmp2 ^ in5;
-        out7 = tmp2 ^ in2 ^ in7;
-        out3 = out6 ^ in3 ^ in7;
-        out0 = tmp1 ^ out7;
-        out2 = out3 ^ out4 ^ in0;
-        out5 = tmp0 ^ out2;
-        out1 = out0 ^ out5 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in6 ^ in7;
-        out5 = in0 ^ in7;
-        out6 = in0 ^ in1;
-        out0 = in1 ^ in2 ^ in3;
-        out2 = in2 ^ in4 ^ in5;
-        out7 = out6 ^ in2;
-        out1 = out0 ^ out6 ^ in4;
-        out3 = out7 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in4 ^ in5;
-        tmp0 = in0 ^ in1;
-        out4 = in4 ^ in6 ^ in7;
-        out5 = in0 ^ in5 ^ in7;
-        out6 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out0 = tmp1 ^ in3;
-        out7 = tmp1 ^ in7;
-        out1 = out0 ^ in4;
-        out3 = out0 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in5;
-        out4 = in2 ^ in3 ^ in7;
-        out5 = in0 ^ in3 ^ in4;
-        out6 = out3 ^ in1 ^ in4;
-        out1 = out4 ^ in4;
-        out0 = out4 ^ in1 ^ in6;
-        out2 = out0 ^ out5 ^ in5;
-        out7 = out2 ^ in4 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_ED(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = in3 ^ in5;
-        out4 = tmp0 ^ in3 ^ in7;
-        out3 = tmp1 ^ in0;
-        out1 = out4 ^ in1;
-        out5 = out3 ^ in4;
-        out7 = out1 ^ out5 ^ in6;
-        out2 = tmp0 ^ out7;
-        out0 = tmp1 ^ out7;
-        out6 = out2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2;
-        tmp0 = in0 ^ in1;
-        out5 = in0 ^ in3;
-        tmp1 = tmp0 ^ in2;
-        out6 = tmp0 ^ in4;
-        tmp2 = tmp1 ^ out5;
-        out7 = tmp1 ^ in5;
-        out1 = tmp2 ^ out6 ^ in7;
-        out0 = tmp2 ^ in6;
-        tmp3 = out7 ^ in1;
-        out3 = tmp3 ^ in7;
-        out2 = tmp3 ^ in4 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in4 ^ in6;
-        out5 = tmp0 ^ in3;
-        out2 = tmp0 ^ tmp1;
-        out6 = tmp1 ^ in0 ^ in1;
-        out3 = out5 ^ in2 ^ in7;
-        out7 = out3 ^ in1 ^ in3;
-        out0 = out4 ^ out6 ^ in3;
-        out1 = tmp1 ^ out0 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in4 ^ in5;
-        out2 = tmp0 ^ in6;
-        out3 = tmp1 ^ in1;
-        tmp2 = tmp1 ^ in7;
-        out1 = out2 ^ out3 ^ in3;
-        tmp3 = tmp0 ^ tmp2;
-        out0 = tmp3 ^ in3;
-        out5 = tmp3 ^ in0;
-        out4 = out1 ^ out5 ^ in4;
-        out7 = out4 ^ in2;
-        out6 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in6;
-        tmp0 = in3 ^ in5;
-        out3 = tmp0 ^ in1 ^ in4;
-        tmp1 = out3 ^ in2;
-        out1 = tmp1 ^ in6;
-        tmp2 = tmp1 ^ in0;
-        tmp3 = out1 ^ in5;
-        out0 = tmp2 ^ in7;
-        out6 = tmp2 ^ in4;
-        out7 = tmp3 ^ in0;
-        out5 = tmp0 ^ out0;
-        out4 = tmp3 ^ out5 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in5;
-        out2 = in2 ^ in6 ^ in7;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp1 ^ in2;
-        out0 = tmp2 ^ in3;
-        out3 = tmp2 ^ in7;
-        out5 = out3 ^ in0 ^ in4;
-        tmp3 = tmp0 ^ out5;
-        out7 = tmp3 ^ in3;
-        out4 = tmp3 ^ out2;
-        out1 = out0 ^ out4 ^ in4;
-        out6 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in6 ^ in7;
-        tmp0 = in0 ^ in1;
-        out4 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out5 = tmp1 ^ in7;
-        out6 = tmp1 ^ in3;
-        out7 = out6 ^ in4;
-        out0 = out7 ^ in5;
-        out1 = out0 ^ in6;
-        out3 = out0 ^ in0 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in1 ^ in2;
-        tmp0 = out2 ^ in3;
-        out4 = tmp0 ^ in4;
-        out5 = out4 ^ in5;
-        out6 = out5 ^ in6;
-        out7 = out6 ^ in7;
-        out0 = out7 ^ in0;
-        out1 = out0 ^ in1;
-        out3 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in1;
-        tmp0 = out2 ^ in2;
-        out4 = tmp0 ^ in3;
-        out5 = out4 ^ in4;
-        out6 = out5 ^ in5;
-        out7 = out6 ^ in6;
-        out0 = out7 ^ in7;
-        out1 = out0 ^ in0;
-        out3 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in7;
-        out2 = tmp0 ^ in2;
-        out4 = out2 ^ in1 ^ in4;
-        out7 = out4 ^ in3 ^ in5;
-        out5 = out7 ^ in4 ^ in7;
-        out0 = tmp0 ^ out7 ^ in6;
-        tmp1 = out0 ^ in1;
-        out6 = out0 ^ in0 ^ in5;
-        out3 = tmp1 ^ in3;
-        out1 = tmp0 ^ tmp1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in7;
-        tmp0 = out2 ^ in1;
-        out4 = tmp0 ^ in2;
-        out5 = out4 ^ in3 ^ in7;
-        out6 = out5 ^ in4;
-        out7 = out6 ^ in5;
-        out0 = out7 ^ in6;
-        out1 = out0 ^ in7;
-        out3 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        tmp1 = in3 ^ in5;
-        tmp2 = tmp0 ^ in6;
-        out4 = tmp0 ^ tmp1;
-        out1 = tmp1 ^ in2 ^ in4;
-        out3 = tmp2 ^ in1;
-        out5 = out3 ^ in5;
-        out7 = out1 ^ out5 ^ in7;
-        out6 = tmp1 ^ out7;
-        out0 = tmp2 ^ out7;
-        out2 = out6 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = in0 ^ in6;
-        out4 = tmp0 ^ in0;
-        tmp2 = tmp1 ^ in4;
-        tmp3 = tmp1 ^ in2;
-        out5 = tmp2 ^ in1;
-        out3 = out5 ^ in3;
-        tmp4 = tmp3 ^ out3;
-        out1 = tmp4 ^ in5;
-        out0 = tmp4 ^ in0 ^ in7;
-        out6 = tmp0 ^ out0 ^ in4;
-        out7 = tmp2 ^ tmp4;
-        out2 = tmp3 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = tmp0 ^ in2;
-        tmp2 = tmp0 ^ in5;
-        tmp3 = tmp1 ^ in7;
-        out5 = tmp2 ^ in6;
-        out6 = tmp3 ^ in6;
-        out7 = tmp3 ^ in3;
-        out3 = out6 ^ in4;
-        out2 = tmp1 ^ out5;
-        out4 = out2 ^ out3 ^ in1;
-        out0 = out4 ^ out7 ^ in5;
-        out1 = tmp2 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in5 ^ in6;
-        tmp0 = in0 ^ in1;
-        out4 = in0 ^ in5 ^ in7;
-        out5 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out6 = tmp1 ^ in7;
-        out7 = tmp1 ^ in3;
-        out0 = out7 ^ in4;
-        out1 = out0 ^ in5;
-        out3 = out0 ^ in6 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in0 ^ in7;
-        out2 = tmp0 ^ tmp1 ^ in5;
-        out3 = tmp1 ^ in4;
-        tmp2 = out2 ^ in6;
-        out6 = tmp2 ^ in4;
-        out7 = tmp2 ^ in3;
-        out4 = out6 ^ in1 ^ in3;
-        tmp3 = out4 ^ in0;
-        out1 = tmp3 ^ in6;
-        out0 = tmp3 ^ in1 ^ in5;
-        out5 = tmp0 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in7;
-        out2 = tmp0 ^ tmp1;
-        out6 = out2 ^ in2 ^ in4;
-        tmp2 = out6 ^ in0;
-        out1 = tmp2 ^ in3;
-        out0 = tmp0 ^ out1 ^ in6;
-        out5 = out0 ^ in2;
-        tmp3 = out5 ^ in1;
-        out3 = tmp3 ^ in6;
-        out7 = tmp2 ^ tmp3;
-        out4 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        out2 = tmp0 ^ in5;
-        out3 = tmp0 ^ in4;
-        tmp1 = out3 ^ in6;
-        out4 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in1;
-        out6 = tmp2 ^ in7;
-        tmp3 = tmp2 ^ in0;
-        out0 = tmp3 ^ in3;
-        tmp4 = out0 ^ out4 ^ in7;
-        out5 = tmp4 ^ in6;
-        out7 = tmp4 ^ in2;
-        out1 = tmp3 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in5;
-        tmp0 = in4 ^ in7;
-        tmp1 = out2 ^ in2;
-        out4 = tmp1 ^ in6;
-        out7 = tmp1 ^ in1 ^ in3;
-        out1 = tmp0 ^ out7;
-        tmp2 = out1 ^ in5;
-        out6 = tmp2 ^ in3;
-        tmp3 = tmp2 ^ in7;
-        out0 = tmp3 ^ in6;
-        out3 = tmp3 ^ in1;
-        out5 = tmp0 ^ out0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-void (* ec_gf_muladd[])(uint8_t * out, uint8_t * in, unsigned int width) =
-{
-    gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03,
-    gf8_muladd_04, gf8_muladd_05, gf8_muladd_06, gf8_muladd_07,
-    gf8_muladd_08, gf8_muladd_09, gf8_muladd_0A, gf8_muladd_0B,
-    gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E, gf8_muladd_0F,
-    gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13,
-    gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17,
-    gf8_muladd_18, gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B,
-    gf8_muladd_1C, gf8_muladd_1D, gf8_muladd_1E, gf8_muladd_1F,
-    gf8_muladd_20, gf8_muladd_21, gf8_muladd_22, gf8_muladd_23,
-    gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27,
-    gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B,
-    gf8_muladd_2C, gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F,
-    gf8_muladd_30, gf8_muladd_31, gf8_muladd_32, gf8_muladd_33,
-    gf8_muladd_34, gf8_muladd_35, gf8_muladd_36, gf8_muladd_37,
-    gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B,
-    gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F,
-    gf8_muladd_40, gf8_muladd_41, gf8_muladd_42, gf8_muladd_43,
-    gf8_muladd_44, gf8_muladd_45, gf8_muladd_46, gf8_muladd_47,
-    gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A, gf8_muladd_4B,
-    gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F,
-    gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53,
-    gf8_muladd_54, gf8_muladd_55, gf8_muladd_56, gf8_muladd_57,
-    gf8_muladd_58, gf8_muladd_59, gf8_muladd_5A, gf8_muladd_5B,
-    gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E, gf8_muladd_5F,
-    gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63,
-    gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67,
-    gf8_muladd_68, gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B,
-    gf8_muladd_6C, gf8_muladd_6D, gf8_muladd_6E, gf8_muladd_6F,
-    gf8_muladd_70, gf8_muladd_71, gf8_muladd_72, gf8_muladd_73,
-    gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77,
-    gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B,
-    gf8_muladd_7C, gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F,
-    gf8_muladd_80, gf8_muladd_81, gf8_muladd_82, gf8_muladd_83,
-    gf8_muladd_84, gf8_muladd_85, gf8_muladd_86, gf8_muladd_87,
-    gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B,
-    gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F,
-    gf8_muladd_90, gf8_muladd_91, gf8_muladd_92, gf8_muladd_93,
-    gf8_muladd_94, gf8_muladd_95, gf8_muladd_96, gf8_muladd_97,
-    gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A, gf8_muladd_9B,
-    gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F,
-    gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3,
-    gf8_muladd_A4, gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7,
-    gf8_muladd_A8, gf8_muladd_A9, gf8_muladd_AA, gf8_muladd_AB,
-    gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE, gf8_muladd_AF,
-    gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3,
-    gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7,
-    gf8_muladd_B8, gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB,
-    gf8_muladd_BC, gf8_muladd_BD, gf8_muladd_BE, gf8_muladd_BF,
-    gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2, gf8_muladd_C3,
-    gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7,
-    gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB,
-    gf8_muladd_CC, gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF,
-    gf8_muladd_D0, gf8_muladd_D1, gf8_muladd_D2, gf8_muladd_D3,
-    gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6, gf8_muladd_D7,
-    gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB,
-    gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF,
-    gf8_muladd_E0, gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3,
-    gf8_muladd_E4, gf8_muladd_E5, gf8_muladd_E6, gf8_muladd_E7,
-    gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA, gf8_muladd_EB,
-    gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF,
-    gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3,
-    gf8_muladd_F4, gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7,
-    gf8_muladd_F8, gf8_muladd_F9, gf8_muladd_FA, gf8_muladd_FB,
-    gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE, gf8_muladd_FF
-};
diff --git a/xlators/cluster/ec/src/ec-gf8.c b/xlators/cluster/ec/src/ec-gf8.c
new file mode 100644
index 00000000000..039adae5929
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-gf8.c
@@ -0,0 +1,5882 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "ec-gf8.h"
+
+static ec_gf_op_t ec_gf8_mul_00_ops[] = {{EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_00 = {0,
+                                    {
+                                        0,
+                                    },
+                                    ec_gf8_mul_00_ops};
+
+static ec_gf_op_t ec_gf8_mul_01_ops[] = {{EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_01 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_01_ops};
+
+static ec_gf_op_t ec_gf8_mul_02_ops[] = {{EC_GF_OP_XOR2, 1, 7, 0},
+                                         {EC_GF_OP_XOR2, 2, 7, 0},
+                                         {EC_GF_OP_XOR2, 3, 7, 0},
+                                         {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_02 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_02_ops};
+
+static ec_gf_op_t ec_gf8_mul_03_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_03 = {9,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_03_ops};
+
+static ec_gf_op_t ec_gf8_mul_04_ops[] = {
+    {EC_GF_OP_XOR3, 8, 6, 7}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_04 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_04_ops};
+
+static ec_gf_op_t ec_gf8_mul_05_ops[] = {
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_05 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_05_ops};
+
+static ec_gf_op_t ec_gf8_mul_06_ops[] = {
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_06 = {9,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        8,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_06_ops};
+
+static ec_gf_op_t ec_gf8_mul_07_ops[] = {
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_07 = {8,
+                                    {
+                                        6,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        7,
+                                    },
+                                    ec_gf8_mul_07_ops};
+
+static ec_gf_op_t ec_gf8_mul_08_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 6, 7},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_08 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_08_ops};
+
+static ec_gf_op_t ec_gf8_mul_09_ops[] = {
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_09 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_09_ops};
+
+static ec_gf_op_t ec_gf8_mul_0A_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0A = {8,
+                                    {
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_0A_ops};
+
+static ec_gf_op_t ec_gf8_mul_0B_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 5, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR3, 3, 8, 6}, {EC_GF_OP_XOR2, 1, 9, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0B = {10,
+                                    {
+                                        7,
+                                        1,
+                                        5,
+                                        2,
+                                        4,
+                                        3,
+                                        0,
+                                        6,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_0B_ops};
+
+static ec_gf_op_t ec_gf8_mul_0C_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0C = {9,
+                                    {
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        8,
+                                        2,
+                                        3,
+                                        4,
+                                        6,
+                                    },
+                                    ec_gf8_mul_0C_ops};
+
+static ec_gf_op_t ec_gf8_mul_0D_ops[] = {
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR3, 2, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0D = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        0,
+                                        2,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_0D_ops};
+
+static ec_gf_op_t ec_gf8_mul_0E_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0E = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_0E_ops};
+
+static ec_gf_op_t ec_gf8_mul_0F_ops[] = {
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0F = {8,
+                                    {
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_0F_ops};
+
+static ec_gf_op_t ec_gf8_mul_10_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_10 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_10_ops};
+
+static ec_gf_op_t ec_gf8_mul_11_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_11 = {8,
+                                    {
+                                        4,
+                                        1,
+                                        2,
+                                        6,
+                                        0,
+                                        5,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_11_ops};
+
+static ec_gf_op_t ec_gf8_mul_12_ops[] = {
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_12 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                    },
+                                    ec_gf8_mul_12_ops};
+
+static ec_gf_op_t ec_gf8_mul_13_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 3, 7},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_13 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_13_ops};
+
+static ec_gf_op_t ec_gf8_mul_14_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_14 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        3,
+                                    },
+                                    ec_gf8_mul_14_ops};
+
+static ec_gf_op_t ec_gf8_mul_15_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR3, 5, 8, 7},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_15 = {9,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        7,
+                                        6,
+                                        5,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_15_ops};
+
+static ec_gf_op_t ec_gf8_mul_16_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_16 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_16_ops};
+
+static ec_gf_op_t ec_gf8_mul_17_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_17 = {8,
+                                    {
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        6,
+                                    },
+                                    ec_gf8_mul_17_ops};
+
+static ec_gf_op_t ec_gf8_mul_18_ops[] = {
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_18 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_18_ops};
+
+static ec_gf_op_t ec_gf8_mul_19_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_19 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_19_ops};
+
+static ec_gf_op_t ec_gf8_mul_1A_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1A = {8,
+                                    {
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                        3,
+                                        1,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1A_ops};
+
+static ec_gf_op_t ec_gf8_mul_1B_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1B = {8,
+                                    {
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        1,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_1B_ops};
+
+static ec_gf_op_t ec_gf8_mul_1C_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1C = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1C_ops};
+
+static ec_gf_op_t ec_gf8_mul_1D_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 4, 2},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1D = {9,
+                                    {
+                                        0,
+                                        7,
+                                        5,
+                                        8,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1D_ops};
+
+static ec_gf_op_t ec_gf8_mul_1E_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1E = {8,
+                                    {
+                                        4,
+                                        7,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_1E_ops};
+
+static ec_gf_op_t ec_gf8_mul_1F_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR3, 8, 3, 7},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1F = {9,
+                                    {
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        3,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_1F_ops};
+
+static ec_gf_op_t ec_gf8_mul_20_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_20 = {8,
+                                    {
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_20_ops};
+
+static ec_gf_op_t ec_gf8_mul_21_ops[] = {
+    {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 8, 7, 5}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_21 = {10,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        7,
+                                        5,
+                                        4,
+                                        3,
+                                        6,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_21_ops};
+
+static ec_gf_op_t ec_gf8_mul_22_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_22 = {8,
+                                    {
+                                        3,
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_22_ops};
+
+static ec_gf_op_t ec_gf8_mul_23_ops[] = {
+    {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_23 = {9,
+                                    {
+                                        0,
+                                        4,
+                                        3,
+                                        2,
+                                        5,
+                                        6,
+                                        1,
+                                        8,
+                                        7,
+                                    },
+                                    ec_gf8_mul_23_ops};
+
+static ec_gf_op_t ec_gf8_mul_24_ops[] = {
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_24 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        3,
+                                    },
+                                    ec_gf8_mul_24_ops};
+
+static ec_gf_op_t ec_gf8_mul_25_ops[] = {
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_25 = {8,
+                                    {
+                                        2,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_25_ops};
+
+static ec_gf_op_t ec_gf8_mul_26_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_26 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        1,
+                                        2,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_26_ops};
+
+static ec_gf_op_t ec_gf8_mul_27_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_27 = {8,
+                                    {
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_27_ops};
+
+static ec_gf_op_t ec_gf8_mul_28_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_28 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        7,
+                                    },
+                                    ec_gf8_mul_28_ops};
+
+static ec_gf_op_t ec_gf8_mul_29_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_29 = {8,
+                                    {
+                                        4,
+                                        6,
+                                        3,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_29_ops};
+
+static ec_gf_op_t ec_gf8_mul_2A_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 6, 8, 4}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2A = {9,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        2,
+                                        6,
+                                        5,
+                                        1,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2A_ops};
+
+static ec_gf_op_t ec_gf8_mul_2B_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2B = {8,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_2B_ops};
+
+static ec_gf_op_t ec_gf8_mul_2C_ops[] = {
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2C = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_2C_ops};
+
+static ec_gf_op_t ec_gf8_mul_2D_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 6},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2D = {9,
+                                    {
+                                        7,
+                                        0,
+                                        3,
+                                        5,
+                                        1,
+                                        4,
+                                        2,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2D_ops};
+
+static ec_gf_op_t ec_gf8_mul_2E_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2E = {9,
+                                    {
+                                        5,
+                                        0,
+                                        7,
+                                        3,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2E_ops};
+
+static ec_gf_op_t ec_gf8_mul_2F_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR3, 8, 7, 6}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2F = {9,
+                                    {
+                                        6,
+                                        3,
+                                        2,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2F_ops};
+
+static ec_gf_op_t ec_gf8_mul_30_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 7},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_30 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        0,
+                                        6,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_30_ops};
+
+static ec_gf_op_t ec_gf8_mul_31_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_31 = {8,
+                                    {
+                                        7,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_31_ops};
+
+static ec_gf_op_t ec_gf8_mul_32_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_32 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        7,
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_32_ops};
+
+static ec_gf_op_t ec_gf8_mul_33_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_33 = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        0,
+                                        2,
+                                        1,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_33_ops};
+
+static ec_gf_op_t ec_gf8_mul_34_ops[] = {
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_34 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        3,
+                                        0,
+                                        2,
+                                        4,
+                                        1,
+                                        6,
+                                    },
+                                    ec_gf8_mul_34_ops};
+
+static ec_gf_op_t ec_gf8_mul_35_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_35 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        5,
+                                        4,
+                                        2,
+                                        0,
+                                        1,
+                                        3,
+                                    },
+                                    ec_gf8_mul_35_ops};
+
+static ec_gf_op_t ec_gf8_mul_36_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_36 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_36_ops};
+
+static ec_gf_op_t ec_gf8_mul_37_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 1},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_37 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        2,
+                                        1,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_37_ops};
+
+static ec_gf_op_t ec_gf8_mul_38_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 7},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_38 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_38_ops};
+
+static ec_gf_op_t ec_gf8_mul_39_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_39 = {8,
+                                    {
+                                        1,
+                                        6,
+                                        3,
+                                        0,
+                                        5,
+                                        2,
+                                        4,
+                                        7,
+                                    },
+                                    ec_gf8_mul_39_ops};
+
+static ec_gf_op_t ec_gf8_mul_3A_ops[] = {
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3A = {8,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        0,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_3A_ops};
+
+static ec_gf_op_t ec_gf8_mul_3B_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR3, 8, 7, 3}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3B = {9,
+                                    {
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        6,
+                                        2,
+                                        4,
+                                        8,
+                                        5,
+                                    },
+                                    ec_gf8_mul_3B_ops};
+
+static ec_gf_op_t ec_gf8_mul_3C_ops[] = {
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3C = {8,
+                                    {
+                                        3,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                        2,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_3C_ops};
+
+static ec_gf_op_t ec_gf8_mul_3D_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3D = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_3D_ops};
+
+static ec_gf_op_t ec_gf8_mul_3E_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3E = {8,
+                                    {
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        0,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_3E_ops};
+
+static ec_gf_op_t ec_gf8_mul_3F_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0},  {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_COPY, 10, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0},  {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0},  {EC_GF_OP_COPY, 9, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0},  {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0},  {EC_GF_OP_XOR3, 4, 9, 7},
+    {EC_GF_OP_XOR2, 3, 4, 0},  {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0},  {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 3, 10, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3F = {11,
+                                    {
+                                        1,
+                                        7,
+                                        6,
+                                        2,
+                                        4,
+                                        3,
+                                        5,
+                                        0,
+                                        8,
+                                        9,
+                                        10,
+                                    },
+                                    ec_gf8_mul_3F_ops};
+
+static ec_gf_op_t ec_gf8_mul_40_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 7, 6},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_40 = {9,
+                                    {
+                                        5,
+                                        7,
+                                        4,
+                                        6,
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_40_ops};
+
+static ec_gf_op_t ec_gf8_mul_41_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_41 = {9,
+                                    {
+                                        0,
+                                        7,
+                                        6,
+                                        5,
+                                        3,
+                                        4,
+                                        8,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_41_ops};
+
+static ec_gf_op_t ec_gf8_mul_42_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_42 = {9,
+                                    {
+                                        2,
+                                        7,
+                                        1,
+                                        6,
+                                        4,
+                                        3,
+                                        0,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_42_ops};
+
+static ec_gf_op_t ec_gf8_mul_43_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_43 = {8,
+                                    {
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                        3,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_43_ops};
+
+static ec_gf_op_t ec_gf8_mul_44_ops[] = {
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_44 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        6,
+                                        5,
+                                        0,
+                                        7,
+                                    },
+                                    ec_gf8_mul_44_ops};
+
+static ec_gf_op_t ec_gf8_mul_45_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_45 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_45_ops};
+
+static ec_gf_op_t ec_gf8_mul_46_ops[] = {
+    {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_46 = {9,
+                                    {
+                                        2,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_46_ops};
+
+static ec_gf_op_t ec_gf8_mul_47_ops[] = {
+    {EC_GF_OP_XOR3, 8, 0, 1}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_47 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_47_ops};
+
+static ec_gf_op_t ec_gf8_mul_48_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_48 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                        3,
+                                        7,
+                                        2,
+                                    },
+                                    ec_gf8_mul_48_ops};
+
+static ec_gf_op_t ec_gf8_mul_49_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 6},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR3, 1, 8, 5},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_49 = {9,
+                                    {
+                                        7,
+                                        2,
+                                        4,
+                                        0,
+                                        3,
+                                        5,
+                                        1,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_49_ops};
+
+static ec_gf_op_t ec_gf8_mul_4A_ops[] = {
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4A = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        2,
+                                    },
+                                    ec_gf8_mul_4A_ops};
+
+static ec_gf_op_t ec_gf8_mul_4B_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR3, 8, 3, 7}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4B = {9,
+                                    {
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_4B_ops};
+
+static ec_gf_op_t ec_gf8_mul_4C_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4C = {8,
+                                    {
+                                        5,
+                                        3,
+                                        4,
+                                        7,
+                                        0,
+                                        6,
+                                        2,
+                                        1,
+                                    },
+                                    ec_gf8_mul_4C_ops};
+
+static ec_gf_op_t ec_gf8_mul_4D_ops[] = {
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR3, 9, 3, 1},
+    {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 0, 8, 2},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4D = {10,
+                                    {
+                                        0,
+                                        9,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                        7,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_4D_ops};
+
+static ec_gf_op_t ec_gf8_mul_4E_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4E = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_4E_ops};
+
+static ec_gf_op_t ec_gf8_mul_4F_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4F = {8,
+                                    {
+                                        0,
+                                        3,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_4F_ops};
+
+static ec_gf_op_t ec_gf8_mul_50_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_50 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_50_ops};
+
+static ec_gf_op_t ec_gf8_mul_51_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_51 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_51_ops};
+
+static ec_gf_op_t ec_gf8_mul_52_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR3, 3, 5, 8},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 9, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_52 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        6,
+                                        7,
+                                        0,
+                                        5,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_52_ops};
+
+static ec_gf_op_t ec_gf8_mul_53_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_53 = {8,
+                                    {
+                                        2,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_53_ops};
+
+static ec_gf_op_t ec_gf8_mul_54_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_54 = {8,
+                                    {
+                                        7,
+                                        3,
+                                        0,
+                                        4,
+                                        2,
+                                        6,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_54_ops};
+
+static ec_gf_op_t ec_gf8_mul_55_ops[] = {
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_55 = {8,
+                                    {
+                                        1,
+                                        5,
+                                        6,
+                                        4,
+                                        3,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_55_ops};
+
+static ec_gf_op_t ec_gf8_mul_56_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_56 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_56_ops};
+
+static ec_gf_op_t ec_gf8_mul_57_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_57 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_57_ops};
+
+static ec_gf_op_t ec_gf8_mul_58_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_58 = {8,
+                                    {
+                                        4,
+                                        3,
+                                        2,
+                                        7,
+                                        0,
+                                        1,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_58_ops};
+
+static ec_gf_op_t ec_gf8_mul_59_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_59 = {8,
+                                    {
+                                        7,
+                                        3,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_59_ops};
+
+static ec_gf_op_t ec_gf8_mul_5A_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5A = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_5A_ops};
+
+static ec_gf_op_t ec_gf8_mul_5B_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5B = {8,
+                                    {
+                                        6,
+                                        0,
+                                        7,
+                                        5,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_5B_ops};
+
+static ec_gf_op_t ec_gf8_mul_5C_ops[] = {
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5C = {9,
+                                    {
+                                        7,
+                                        5,
+                                        2,
+                                        4,
+                                        1,
+                                        0,
+                                        6,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_5C_ops};
+
+static ec_gf_op_t ec_gf8_mul_5D_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5D = {8,
+                                    {
+                                        1,
+                                        3,
+                                        5,
+                                        4,
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_5D_ops};
+
+static ec_gf_op_t ec_gf8_mul_5E_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5E = {8,
+                                    {
+                                        4,
+                                        3,
+                                        6,
+                                        2,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_5E_ops};
+
+static ec_gf_op_t ec_gf8_mul_5F_ops[] = {
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5F = {8,
+                                    {
+                                        6,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_5F_ops};
+
+static ec_gf_op_t ec_gf8_mul_60_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_60 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_60_ops};
+
+static ec_gf_op_t ec_gf8_mul_61_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_61 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                        2,
+                                        1,
+                                        3,
+                                    },
+                                    ec_gf8_mul_61_ops};
+
+static ec_gf_op_t ec_gf8_mul_62_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_62 = {8,
+                                    {
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_62_ops};
+
+static ec_gf_op_t ec_gf8_mul_63_ops[] = {
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_63 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_63_ops};
+
+static ec_gf_op_t ec_gf8_mul_64_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 8, 7, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_64 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        7,
+                                        8,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_64_ops};
+
+static ec_gf_op_t ec_gf8_mul_65_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_65 = {8,
+                                    {
+                                        2,
+                                        5,
+                                        1,
+                                        3,
+                                        4,
+                                        0,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_65_ops};
+
+static ec_gf_op_t ec_gf8_mul_66_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_66 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        5,
+                                        7,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_66_ops};
+
+static ec_gf_op_t ec_gf8_mul_67_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_67 = {8,
+                                    {
+                                        2,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_67_ops};
+
+static ec_gf_op_t ec_gf8_mul_68_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_68 = {8,
+                                    {
+                                        5,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_68_ops};
+
+static ec_gf_op_t ec_gf8_mul_69_ops[] = {
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_69 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                    },
+                                    ec_gf8_mul_69_ops};
+
+static ec_gf_op_t ec_gf8_mul_6A_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6A = {8,
+                                    {
+                                        5,
+                                        7,
+                                        4,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_6A_ops};
+
+static ec_gf_op_t ec_gf8_mul_6B_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6B = {9,
+                                    {
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        5,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6B_ops};
+
+static ec_gf_op_t ec_gf8_mul_6C_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6C = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_6C_ops};
+
+static ec_gf_op_t ec_gf8_mul_6D_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR3, 8, 3, 4}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6D = {9,
+                                    {
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6D_ops};
+
+static ec_gf_op_t ec_gf8_mul_6E_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6E = {8,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        1,
+                                        7,
+                                        2,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_6E_ops};
+
+static ec_gf_op_t ec_gf8_mul_6F_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 0, 8, 7}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6F = {9,
+                                    {
+                                        2,
+                                        6,
+                                        3,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6F_ops};
+
+static ec_gf_op_t ec_gf8_mul_70_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_70 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_70_ops};
+
+static ec_gf_op_t ec_gf8_mul_71_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_71 = {8,
+                                    {
+                                        4,
+                                        7,
+                                        5,
+                                        3,
+                                        6,
+                                        0,
+                                        2,
+                                        1,
+                                    },
+                                    ec_gf8_mul_71_ops};
+
+static ec_gf_op_t ec_gf8_mul_72_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_72 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        7,
+                                        4,
+                                        1,
+                                        3,
+                                        6,
+                                    },
+                                    ec_gf8_mul_72_ops};
+
+static ec_gf_op_t ec_gf8_mul_73_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_73 = {8,
+                                    {
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_73_ops};
+
+static ec_gf_op_t ec_gf8_mul_74_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_74 = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_74_ops};
+
+static ec_gf_op_t ec_gf8_mul_75_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_75 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_75_ops};
+
+static ec_gf_op_t ec_gf8_mul_76_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 8, 6, 2},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 8, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_76 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        5,
+                                        1,
+                                        7,
+                                        8,
+                                        4,
+                                    },
+                                    ec_gf8_mul_76_ops};
+
+static ec_gf_op_t ec_gf8_mul_77_ops[] = {
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_77 = {8,
+                                    {
+                                        7,
+                                        4,
+                                        3,
+                                        6,
+                                        0,
+                                        1,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_77_ops};
+
+static ec_gf_op_t ec_gf8_mul_78_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 0, 2},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_78 = {9,
+                                    {
+                                        4,
+                                        7,
+                                        3,
+                                        2,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_78_ops};
+
+static ec_gf_op_t ec_gf8_mul_79_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 7},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_79 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        1,
+                                        6,
+                                        2,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_79_ops};
+
+static ec_gf_op_t ec_gf8_mul_7A_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7A = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_7A_ops};
+
+static ec_gf_op_t ec_gf8_mul_7B_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR3, 8, 5, 3},
+    {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 4, 1, 9},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7B = {10,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        8,
+                                        5,
+                                        6,
+                                        0,
+                                        7,
+                                        9,
+                                    },
+                                    ec_gf8_mul_7B_ops};
+
+static ec_gf_op_t ec_gf8_mul_7C_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7C = {8,
+                                    {
+                                        2,
+                                        4,
+                                        1,
+                                        6,
+                                        3,
+                                        5,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_7C_ops};
+
+static ec_gf_op_t ec_gf8_mul_7D_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7D = {8,
+                                    {
+                                        1,
+                                        0,
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        4,
+                                    },
+                                    ec_gf8_mul_7D_ops};
+
+static ec_gf_op_t ec_gf8_mul_7E_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_COPY, 8, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 6, 2, 7},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7E = {9,
+                                    {
+                                        5,
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                        4,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_7E_ops};
+
+static ec_gf_op_t ec_gf8_mul_7F_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR3, 9, 7, 5}, {EC_GF_OP_XOR2, 2, 9, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 9, 0},
+    {EC_GF_OP_XOR3, 9, 6, 4}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7F = {10,
+                                    {
+                                        4,
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_7F_ops};
+
+static ec_gf_op_t ec_gf8_mul_80_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_80 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        6,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_80_ops};
+
+static ec_gf_op_t ec_gf8_mul_81_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_81 = {8,
+                                    {
+                                        2,
+                                        7,
+                                        4,
+                                        1,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_81_ops};
+
+static ec_gf_op_t ec_gf8_mul_82_ops[] = {
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR3, 5, 8, 7}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_82 = {9,
+                                    {
+                                        6,
+                                        2,
+                                        7,
+                                        5,
+                                        1,
+                                        3,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_82_ops};
+
+static ec_gf_op_t ec_gf8_mul_83_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_83 = {8,
+                                    {
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_83_ops};
+
+static ec_gf_op_t ec_gf8_mul_84_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_84 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        0,
+                                        4,
+                                        1,
+                                        5,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_84_ops};
+
+static ec_gf_op_t ec_gf8_mul_85_ops[] = {
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_85 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        0,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_85_ops};
+
+static ec_gf_op_t ec_gf8_mul_86_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_86 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_86_ops};
+
+static ec_gf_op_t ec_gf8_mul_87_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR3, 5, 8, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_87 = {9,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_87_ops};
+
+static ec_gf_op_t ec_gf8_mul_88_ops[] = {
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_88 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_88_ops};
+
+static ec_gf_op_t ec_gf8_mul_89_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR3, 8, 5, 2},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_89 = {9,
+                                    {
+                                        2,
+                                        1,
+                                        6,
+                                        5,
+                                        7,
+                                        3,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_89_ops};
+
+static ec_gf_op_t ec_gf8_mul_8A_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8A = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_8A_ops};
+
+static ec_gf_op_t ec_gf8_mul_8B_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8B = {8,
+                                    {
+                                        6,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        7,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8B_ops};
+
+static ec_gf_op_t ec_gf8_mul_8C_ops[] = {
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8C = {8,
+                                    {
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_8C_ops};
+
+static ec_gf_op_t ec_gf8_mul_8D_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8D = {8,
+                                    {
+                                        7,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_8D_ops};
+
+static ec_gf_op_t ec_gf8_mul_8E_ops[] = {{EC_GF_OP_XOR2, 2, 0, 0},
+                                         {EC_GF_OP_XOR2, 3, 0, 0},
+                                         {EC_GF_OP_XOR2, 4, 0, 0},
+                                         {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8E = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8E_ops};
+
+static ec_gf_op_t ec_gf8_mul_8F_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8F = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8F_ops};
+
+static ec_gf_op_t ec_gf8_mul_90_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_90 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_90_ops};
+
+static ec_gf_op_t ec_gf8_mul_91_ops[] = {
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_COPY, 9, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR3, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_91 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        0,
+                                        6,
+                                        7,
+                                        5,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_91_ops};
+
+static ec_gf_op_t ec_gf8_mul_92_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_92 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_92_ops};
+
+static ec_gf_op_t ec_gf8_mul_93_ops[] = {
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_93 = {8,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        1,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_93_ops};
+
+static ec_gf_op_t ec_gf8_mul_94_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_94 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        0,
+                                        2,
+                                        6,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_94_ops};
+
+static ec_gf_op_t ec_gf8_mul_95_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_95 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        1,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_95_ops};
+
+static ec_gf_op_t ec_gf8_mul_96_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR3, 8, 0, 4}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_96 = {9,
+                                    {
+                                        4,
+                                        0,
+                                        1,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_96_ops};
+
+static ec_gf_op_t ec_gf8_mul_97_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 8, 6, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_97 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_97_ops};
+
+static ec_gf_op_t ec_gf8_mul_98_ops[] = {
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_98 = {8,
+                                    {
+                                        4,
+                                        2,
+                                        3,
+                                        6,
+                                        7,
+                                        5,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_98_ops};
+
+static ec_gf_op_t ec_gf8_mul_99_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_99 = {8,
+                                    {
+                                        6,
+                                        5,
+                                        3,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        2,
+                                    },
+                                    ec_gf8_mul_99_ops};
+
+static ec_gf_op_t ec_gf8_mul_9A_ops[] = {
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9A = {9,
+                                    {
+                                        6,
+                                        3,
+                                        4,
+                                        0,
+                                        5,
+                                        1,
+                                        2,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_9A_ops};
+
+static ec_gf_op_t ec_gf8_mul_9B_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_COPY, 9, 5, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 8, 3, 2}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 9, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9B = {10,
+                                    {
+                                        4,
+                                        5,
+                                        8,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                        9,
+                                    },
+                                    ec_gf8_mul_9B_ops};
+
+static ec_gf_op_t ec_gf8_mul_9C_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9C = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_9C_ops};
+
+static ec_gf_op_t ec_gf8_mul_9D_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9D = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_9D_ops};
+
+static ec_gf_op_t ec_gf8_mul_9E_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_COPY, 8, 7, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9E = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        8,
+                                        6,
+                                        0,
+                                        2,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_9E_ops};
+
+static ec_gf_op_t ec_gf8_mul_9F_ops[] = {
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9F = {9,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_9F_ops};
+
+static ec_gf_op_t ec_gf8_mul_A0_ops[] = {
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A0 = {8,
+                                    {
+                                        3,
+                                        1,
+                                        6,
+                                        7,
+                                        5,
+                                        2,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_A0_ops};
+
+static ec_gf_op_t ec_gf8_mul_A1_ops[] = {
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 0, 6},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A1 = {9,
+                                    {
+                                        7,
+                                        4,
+                                        1,
+                                        5,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A1_ops};
+
+static ec_gf_op_t ec_gf8_mul_A2_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A2 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        3,
+                                        2,
+                                        1,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_A2_ops};
+
+static ec_gf_op_t ec_gf8_mul_A3_ops[] = {
+    {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A3 = {9,
+                                    {
+                                        3,
+                                        7,
+                                        2,
+                                        6,
+                                        1,
+                                        4,
+                                        0,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A3_ops};
+
+static ec_gf_op_t ec_gf8_mul_A4_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A4 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        4,
+                                        3,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_A4_ops};
+
+static ec_gf_op_t ec_gf8_mul_A5_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR3, 8, 5, 6}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A5 = {9,
+                                    {
+                                        1,
+                                        4,
+                                        2,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A5_ops};
+
+static ec_gf_op_t ec_gf8_mul_A6_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A6 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_A6_ops};
+
+static ec_gf_op_t ec_gf8_mul_A7_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A7 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_A7_ops};
+
+static ec_gf_op_t ec_gf8_mul_A8_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 9, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A8 = {10,
+                                    {
+                                        1,
+                                        7,
+                                        5,
+                                        8,
+                                        6,
+                                        3,
+                                        4,
+                                        0,
+                                        2,
+                                        9,
+                                    },
+                                    ec_gf8_mul_A8_ops};
+
+static ec_gf_op_t ec_gf8_mul_A9_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A9 = {8,
+                                    {
+                                        3,
+                                        7,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_A9_ops};
+
+static ec_gf_op_t ec_gf8_mul_AA_ops[] = {
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AA = {8,
+                                    {
+                                        0,
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_AA_ops};
+
+static ec_gf_op_t ec_gf8_mul_AB_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_COPY, 9, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR3, 3, 9, 7},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AB = {10,
+                                    {
+                                        2,
+                                        3,
+                                        8,
+                                        0,
+                                        5,
+                                        6,
+                                        1,
+                                        4,
+                                        7,
+                                        9,
+                                    },
+                                    ec_gf8_mul_AB_ops};
+
+static ec_gf_op_t ec_gf8_mul_AC_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AC = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_AC_ops};
+
+static ec_gf_op_t ec_gf8_mul_AD_ops[] = {
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AD = {9,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_AD_ops};
+
+static ec_gf_op_t ec_gf8_mul_AE_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_COPY, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AE = {9,
+                                    {
+                                        7,
+                                        0,
+                                        5,
+                                        6,
+                                        3,
+                                        4,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_AE_ops};
+
+static ec_gf_op_t ec_gf8_mul_AF_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AF = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_AF_ops};
+
+static ec_gf_op_t ec_gf8_mul_B0_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B0 = {8,
+                                    {
+                                        4,
+                                        0,
+                                        7,
+                                        2,
+                                        3,
+                                        1,
+                                        6,
+                                        5,
+                                    },
+                                    ec_gf8_mul_B0_ops};
+
+static ec_gf_op_t ec_gf8_mul_B1_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR3, 5, 8, 1}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B1 = {9,
+                                    {
+                                        2,
+                                        6,
+                                        4,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B1_ops};
+
+static ec_gf_op_t ec_gf8_mul_B2_ops[] = {
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 4, 5},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B2 = {9,
+                                    {
+                                        0,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B2_ops};
+
+static ec_gf_op_t ec_gf8_mul_B3_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 9, 5, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 4},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR3, 1, 9, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B3 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        7,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_B3_ops};
+
+static ec_gf_op_t ec_gf8_mul_B4_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B4 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_B4_ops};
+
+static ec_gf_op_t ec_gf8_mul_B5_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR3, 4, 8, 3}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B5 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        0,
+                                        7,
+                                        1,
+                                        5,
+                                        6,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B5_ops};
+
+static ec_gf_op_t ec_gf8_mul_B6_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B6 = {8,
+                                    {
+                                        5,
+                                        3,
+                                        6,
+                                        4,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_B6_ops};
+
+static ec_gf_op_t ec_gf8_mul_B7_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B7 = {8,
+                                    {
+                                        5,
+                                        0,
+                                        1,
+                                        4,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_B7_ops};
+
+static ec_gf_op_t ec_gf8_mul_B8_ops[] = {
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B8 = {8,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_B8_ops};
+
+static ec_gf_op_t ec_gf8_mul_B9_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR3, 0, 8, 2}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B9 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        1,
+                                        4,
+                                        5,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B9_ops};
+
+static ec_gf_op_t ec_gf8_mul_BA_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BA = {8,
+                                    {
+                                        1,
+                                        2,
+                                        4,
+                                        3,
+                                        5,
+                                        6,
+                                        0,
+                                        7,
+                                    },
+                                    ec_gf8_mul_BA_ops};
+
+static ec_gf_op_t ec_gf8_mul_BB_ops[] = {
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BB = {9,
+                                    {
+                                        7,
+                                        2,
+                                        1,
+                                        8,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_BB_ops};
+
+static ec_gf_op_t ec_gf8_mul_BC_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR3, 2, 8, 4},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BC = {9,
+                                    {
+                                        2,
+                                        6,
+                                        3,
+                                        4,
+                                        5,
+                                        1,
+                                        7,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_BC_ops};
+
+static ec_gf_op_t ec_gf8_mul_BD_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BD = {8,
+                                    {
+                                        4,
+                                        5,
+                                        0,
+                                        2,
+                                        7,
+                                        1,
+                                        6,
+                                        3,
+                                    },
+                                    ec_gf8_mul_BD_ops};
+
+static ec_gf_op_t ec_gf8_mul_BE_ops[] = {
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BE = {8,
+                                    {
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        1,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_BE_ops};
+
+static ec_gf_op_t ec_gf8_mul_BF_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BF = {8,
+                                    {
+                                        5,
+                                        6,
+                                        1,
+                                        7,
+                                        3,
+                                        0,
+                                        2,
+                                        4,
+                                    },
+                                    ec_gf8_mul_BF_ops};
+
+static ec_gf_op_t ec_gf8_mul_C0_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C0 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                    },
+                                    ec_gf8_mul_C0_ops};
+
+static ec_gf_op_t ec_gf8_mul_C1_ops[] = {
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C1 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_C1_ops};
+
+static ec_gf_op_t ec_gf8_mul_C2_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C2 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_C2_ops};
+
+static ec_gf_op_t ec_gf8_mul_C3_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR3, 0, 2, 6}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 9, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C3 = {10,
+                                    {
+                                        5,
+                                        6,
+                                        4,
+                                        7,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_C3_ops};
+
+static ec_gf_op_t ec_gf8_mul_C4_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C4 = {8,
+                                    {
+                                        0,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_C4_ops};
+
+static ec_gf_op_t ec_gf8_mul_C5_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C5 = {8,
+                                    {
+                                        4,
+                                        3,
+                                        5,
+                                        7,
+                                        6,
+                                        2,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C5_ops};
+
+static ec_gf_op_t ec_gf8_mul_C6_ops[] = {
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 9, 5, 4},
+    {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C6 = {10,
+                                    {
+                                        6,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        1,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_C6_ops};
+
+static ec_gf_op_t ec_gf8_mul_C7_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C7 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        2,
+                                        5,
+                                        3,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C7_ops};
+
+static ec_gf_op_t ec_gf8_mul_C8_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C8 = {8,
+                                    {
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        6,
+                                        7,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_C8_ops};
+
+static ec_gf_op_t ec_gf8_mul_C9_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C9 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C9_ops};
+
+static ec_gf_op_t ec_gf8_mul_CA_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CA = {8,
+                                    {
+                                        1,
+                                        2,
+                                        5,
+                                        7,
+                                        3,
+                                        4,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_CA_ops};
+
+static ec_gf_op_t ec_gf8_mul_CB_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CB = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_CB_ops};
+
+static ec_gf_op_t ec_gf8_mul_CC_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CC = {8,
+                                    {
+                                        2,
+                                        7,
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_CC_ops};
+
+static ec_gf_op_t ec_gf8_mul_CD_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CD = {8,
+                                    {
+                                        0,
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_CD_ops};
+
+static ec_gf_op_t ec_gf8_mul_CE_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_COPY, 8, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 3, 6, 8},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 2, 3},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CE = {9,
+                                    {
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_CE_ops};
+
+static ec_gf_op_t ec_gf8_mul_CF_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CF = {8,
+                                    {
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        4,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_CF_ops};
+
+static ec_gf_op_t ec_gf8_mul_D0_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D0 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        4,
+                                    },
+                                    ec_gf8_mul_D0_ops};
+
+static ec_gf_op_t ec_gf8_mul_D1_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D1 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        2,
+                                        0,
+                                        7,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D1_ops};
+
+static ec_gf_op_t ec_gf8_mul_D2_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D2 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                    },
+                                    ec_gf8_mul_D2_ops};
+
+static ec_gf_op_t ec_gf8_mul_D3_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D3 = {9,
+                                    {
+                                        0,
+                                        3,
+                                        2,
+                                        8,
+                                        4,
+                                        6,
+                                        7,
+                                        1,
+                                        5,
+                                    },
+                                    ec_gf8_mul_D3_ops};
+
+static ec_gf_op_t ec_gf8_mul_D4_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 1, 7, 8},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D4 = {9,
+                                    {
+                                        4,
+                                        1,
+                                        7,
+                                        5,
+                                        0,
+                                        6,
+                                        3,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D4_ops};
+
+static ec_gf_op_t ec_gf8_mul_D5_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D5 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_D5_ops};
+
+static ec_gf_op_t ec_gf8_mul_D6_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D6 = {9,
+                                    {
+                                        0,
+                                        6,
+                                        2,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D6_ops};
+
+static ec_gf_op_t ec_gf8_mul_D7_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR3, 8, 3, 5}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR3, 6, 7, 8}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D7 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        0,
+                                        7,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D7_ops};
+
+static ec_gf_op_t ec_gf8_mul_D8_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D8 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_D8_ops};
+
+static ec_gf_op_t ec_gf8_mul_D9_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D9 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_D9_ops};
+
+static ec_gf_op_t ec_gf8_mul_DA_ops[] = {
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR3, 8, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DA = {9,
+                                    {
+                                        2,
+                                        5,
+                                        7,
+                                        1,
+                                        0,
+                                        4,
+                                        3,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_DA_ops};
+
+static ec_gf_op_t ec_gf8_mul_DB_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DB = {9,
+                                    {
+                                        7,
+                                        5,
+                                        6,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_DB_ops};
+
+static ec_gf_op_t ec_gf8_mul_DC_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DC = {8,
+                                    {
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_DC_ops};
+
+static ec_gf_op_t ec_gf8_mul_DD_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DD = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_DD_ops};
+
+static ec_gf_op_t ec_gf8_mul_DE_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DE = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_DE_ops};
+
+static ec_gf_op_t ec_gf8_mul_DF_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR3, 1, 9, 2}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DF = {10,
+                                    {
+                                        7,
+                                        2,
+                                        8,
+                                        4,
+                                        3,
+                                        1,
+                                        0,
+                                        6,
+                                        5,
+                                        9,
+                                    },
+                                    ec_gf8_mul_DF_ops};
+
+static ec_gf_op_t ec_gf8_mul_E0_ops[] = {
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E0 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_E0_ops};
+
+static ec_gf_op_t ec_gf8_mul_E1_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 7, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR3, 9, 5, 3},
+    {EC_GF_OP_XOR2, 0, 9, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E1 = {10,
+                                    {
+                                        0,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        2,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_E1_ops};
+
+static ec_gf_op_t ec_gf8_mul_E2_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E2 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        7,
+                                        1,
+                                        5,
+                                        6,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_E2_ops};
+
+static ec_gf_op_t ec_gf8_mul_E3_ops[] = {
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR3, 8, 2, 7}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 4},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E3 = {9,
+                                    {
+                                        5,
+                                        4,
+                                        7,
+                                        2,
+                                        1,
+                                        3,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E3_ops};
+
+static ec_gf_op_t ec_gf8_mul_E4_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E4 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        6,
+                                        3,
+                                        4,
+                                        2,
+                                        5,
+                                    },
+                                    ec_gf8_mul_E4_ops};
+
+static ec_gf_op_t ec_gf8_mul_E5_ops[] = {
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E5 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        0,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E5_ops};
+
+static ec_gf_op_t ec_gf8_mul_E6_ops[] = {
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E6 = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_E6_ops};
+
+static ec_gf_op_t ec_gf8_mul_E7_ops[] = {
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 9, 0, 6}, {EC_GF_OP_XOR2, 4, 9, 0},
+    {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E7 = {10,
+                                    {
+                                        1,
+                                        4,
+                                        3,
+                                        6,
+                                        7,
+                                        5,
+                                        2,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_E7_ops};
+
+static ec_gf_op_t ec_gf8_mul_E8_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E8 = {8,
+                                    {
+                                        1,
+                                        4,
+                                        2,
+                                        7,
+                                        3,
+                                        0,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_E8_ops};
+
+static ec_gf_op_t ec_gf8_mul_E9_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E9 = {9,
+                                    {
+                                        6,
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        1,
+                                        5,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E9_ops};
+
+static ec_gf_op_t ec_gf8_mul_EA_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EA = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_EA_ops};
+
+static ec_gf_op_t ec_gf8_mul_EB_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EB = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_EB_ops};
+
+static ec_gf_op_t ec_gf8_mul_EC_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EC = {9,
+                                    {
+                                        7,
+                                        4,
+                                        3,
+                                        0,
+                                        2,
+                                        5,
+                                        1,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EC_ops};
+
+static ec_gf_op_t ec_gf8_mul_ED_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_ED = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_ED_ops};
+
+static ec_gf_op_t ec_gf8_mul_EE_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 2, 3},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EE = {9,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EE_ops};
+
+static ec_gf_op_t ec_gf8_mul_EF_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_COPY, 8, 0, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EF = {9,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EF_ops};
+
+static ec_gf_op_t ec_gf8_mul_F0_ops[] = {
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 8, 3, 6},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F0 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        5,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F0_ops};
+
+static ec_gf_op_t ec_gf8_mul_F1_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_COPY, 9, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 9, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 4, 9, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR3, 9, 8, 7},
+    {EC_GF_OP_XOR2, 1, 9, 0}, {EC_GF_OP_XOR2, 5, 9, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F1 = {10,
+                                    {
+                                        7,
+                                        2,
+                                        6,
+                                        3,
+                                        5,
+                                        1,
+                                        4,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_F1_ops};
+
+static ec_gf_op_t ec_gf8_mul_F2_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 4},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F2 = {9,
+                                    {
+                                        1,
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F2_ops};
+
+static ec_gf_op_t ec_gf8_mul_F3_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F3 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_F3_ops};
+
+static ec_gf_op_t ec_gf8_mul_F4_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F4 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_F4_ops};
+
+static ec_gf_op_t ec_gf8_mul_F5_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F5 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_F5_ops};
+
+static ec_gf_op_t ec_gf8_mul_F6_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 9, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR3, 7, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F6 = {10,
+                                    {
+                                        0,
+                                        6,
+                                        2,
+                                        7,
+                                        4,
+                                        3,
+                                        5,
+                                        9,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F6_ops};
+
+static ec_gf_op_t ec_gf8_mul_F7_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F7 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_F7_ops};
+
+static ec_gf_op_t ec_gf8_mul_F8_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F8 = {8,
+                                    {
+                                        6,
+                                        2,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        3,
+                                        7,
+                                    },
+                                    ec_gf8_mul_F8_ops};
+
+static ec_gf_op_t ec_gf8_mul_F9_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR3, 8, 7, 1}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F9 = {9,
+                                    {
+                                        4,
+                                        1,
+                                        7,
+                                        6,
+                                        0,
+                                        3,
+                                        5,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F9_ops};
+
+static ec_gf_op_t ec_gf8_mul_FA_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FA = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_FA_ops};
+
+static ec_gf_op_t ec_gf8_mul_FB_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FB = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_FB_ops};
+
+static ec_gf_op_t ec_gf8_mul_FC_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_COPY, 9, 3, 0},
+    {EC_GF_OP_XOR3, 8, 5, 7}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR3, 0, 9, 2}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FC = {10,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        7,
+                                        1,
+                                        8,
+                                        0,
+                                        4,
+                                        2,
+                                        9,
+                                    },
+                                    ec_gf8_mul_FC_ops};
+
+static ec_gf_op_t ec_gf8_mul_FD_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_COPY, 8, 7, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FD = {9,
+                                    {
+                                        5,
+                                        3,
+                                        7,
+                                        6,
+                                        1,
+                                        2,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_FD_ops};
+
+static ec_gf_op_t ec_gf8_mul_FE_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FE = {9,
+                                    {
+                                        3,
+                                        4,
+                                        8,
+                                        2,
+                                        5,
+                                        0,
+                                        6,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_FE_ops};
+
+static ec_gf_op_t ec_gf8_mul_FF_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 9, 0, 0},
+    {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 9, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 9, 4, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR3, 3, 8, 5}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FF = {10,
+                                    {
+                                        6,
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        9,
+                                        3,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_FF_ops};
+
+ec_gf_mul_t *ec_gf8_mul[] = {
+    &ec_gf8_mul_00, &ec_gf8_mul_01, &ec_gf8_mul_02, &ec_gf8_mul_03,
+    &ec_gf8_mul_04, &ec_gf8_mul_05, &ec_gf8_mul_06, &ec_gf8_mul_07,
+    &ec_gf8_mul_08, &ec_gf8_mul_09, &ec_gf8_mul_0A, &ec_gf8_mul_0B,
+    &ec_gf8_mul_0C, &ec_gf8_mul_0D, &ec_gf8_mul_0E, &ec_gf8_mul_0F,
+    &ec_gf8_mul_10, &ec_gf8_mul_11, &ec_gf8_mul_12, &ec_gf8_mul_13,
+    &ec_gf8_mul_14, &ec_gf8_mul_15, &ec_gf8_mul_16, &ec_gf8_mul_17,
+    &ec_gf8_mul_18, &ec_gf8_mul_19, &ec_gf8_mul_1A, &ec_gf8_mul_1B,
+    &ec_gf8_mul_1C, &ec_gf8_mul_1D, &ec_gf8_mul_1E, &ec_gf8_mul_1F,
+    &ec_gf8_mul_20, &ec_gf8_mul_21, &ec_gf8_mul_22, &ec_gf8_mul_23,
+    &ec_gf8_mul_24, &ec_gf8_mul_25, &ec_gf8_mul_26, &ec_gf8_mul_27,
+    &ec_gf8_mul_28, &ec_gf8_mul_29, &ec_gf8_mul_2A, &ec_gf8_mul_2B,
+    &ec_gf8_mul_2C, &ec_gf8_mul_2D, &ec_gf8_mul_2E, &ec_gf8_mul_2F,
+    &ec_gf8_mul_30, &ec_gf8_mul_31, &ec_gf8_mul_32, &ec_gf8_mul_33,
+    &ec_gf8_mul_34, &ec_gf8_mul_35, &ec_gf8_mul_36, &ec_gf8_mul_37,
+    &ec_gf8_mul_38, &ec_gf8_mul_39, &ec_gf8_mul_3A, &ec_gf8_mul_3B,
+    &ec_gf8_mul_3C, &ec_gf8_mul_3D, &ec_gf8_mul_3E, &ec_gf8_mul_3F,
+    &ec_gf8_mul_40, &ec_gf8_mul_41, &ec_gf8_mul_42, &ec_gf8_mul_43,
+    &ec_gf8_mul_44, &ec_gf8_mul_45, &ec_gf8_mul_46, &ec_gf8_mul_47,
+    &ec_gf8_mul_48, &ec_gf8_mul_49, &ec_gf8_mul_4A, &ec_gf8_mul_4B,
+    &ec_gf8_mul_4C, &ec_gf8_mul_4D, &ec_gf8_mul_4E, &ec_gf8_mul_4F,
+    &ec_gf8_mul_50, &ec_gf8_mul_51, &ec_gf8_mul_52, &ec_gf8_mul_53,
+    &ec_gf8_mul_54, &ec_gf8_mul_55, &ec_gf8_mul_56, &ec_gf8_mul_57,
+    &ec_gf8_mul_58, &ec_gf8_mul_59, &ec_gf8_mul_5A, &ec_gf8_mul_5B,
+    &ec_gf8_mul_5C, &ec_gf8_mul_5D, &ec_gf8_mul_5E, &ec_gf8_mul_5F,
+    &ec_gf8_mul_60, &ec_gf8_mul_61, &ec_gf8_mul_62, &ec_gf8_mul_63,
+    &ec_gf8_mul_64, &ec_gf8_mul_65, &ec_gf8_mul_66, &ec_gf8_mul_67,
+    &ec_gf8_mul_68, &ec_gf8_mul_69, &ec_gf8_mul_6A, &ec_gf8_mul_6B,
+    &ec_gf8_mul_6C, &ec_gf8_mul_6D, &ec_gf8_mul_6E, &ec_gf8_mul_6F,
+    &ec_gf8_mul_70, &ec_gf8_mul_71, &ec_gf8_mul_72, &ec_gf8_mul_73,
+    &ec_gf8_mul_74, &ec_gf8_mul_75, &ec_gf8_mul_76, &ec_gf8_mul_77,
+    &ec_gf8_mul_78, &ec_gf8_mul_79, &ec_gf8_mul_7A, &ec_gf8_mul_7B,
+    &ec_gf8_mul_7C, &ec_gf8_mul_7D, &ec_gf8_mul_7E, &ec_gf8_mul_7F,
+    &ec_gf8_mul_80, &ec_gf8_mul_81, &ec_gf8_mul_82, &ec_gf8_mul_83,
+    &ec_gf8_mul_84, &ec_gf8_mul_85, &ec_gf8_mul_86, &ec_gf8_mul_87,
+    &ec_gf8_mul_88, &ec_gf8_mul_89, &ec_gf8_mul_8A, &ec_gf8_mul_8B,
+    &ec_gf8_mul_8C, &ec_gf8_mul_8D, &ec_gf8_mul_8E, &ec_gf8_mul_8F,
+    &ec_gf8_mul_90, &ec_gf8_mul_91, &ec_gf8_mul_92, &ec_gf8_mul_93,
+    &ec_gf8_mul_94, &ec_gf8_mul_95, &ec_gf8_mul_96, &ec_gf8_mul_97,
+    &ec_gf8_mul_98, &ec_gf8_mul_99, &ec_gf8_mul_9A, &ec_gf8_mul_9B,
+    &ec_gf8_mul_9C, &ec_gf8_mul_9D, &ec_gf8_mul_9E, &ec_gf8_mul_9F,
+    &ec_gf8_mul_A0, &ec_gf8_mul_A1, &ec_gf8_mul_A2, &ec_gf8_mul_A3,
+    &ec_gf8_mul_A4, &ec_gf8_mul_A5, &ec_gf8_mul_A6, &ec_gf8_mul_A7,
+    &ec_gf8_mul_A8, &ec_gf8_mul_A9, &ec_gf8_mul_AA, &ec_gf8_mul_AB,
+    &ec_gf8_mul_AC, &ec_gf8_mul_AD, &ec_gf8_mul_AE, &ec_gf8_mul_AF,
+    &ec_gf8_mul_B0, &ec_gf8_mul_B1, &ec_gf8_mul_B2, &ec_gf8_mul_B3,
+    &ec_gf8_mul_B4, &ec_gf8_mul_B5, &ec_gf8_mul_B6, &ec_gf8_mul_B7,
+    &ec_gf8_mul_B8, &ec_gf8_mul_B9, &ec_gf8_mul_BA, &ec_gf8_mul_BB,
+    &ec_gf8_mul_BC, &ec_gf8_mul_BD, &ec_gf8_mul_BE, &ec_gf8_mul_BF,
+    &ec_gf8_mul_C0, &ec_gf8_mul_C1, &ec_gf8_mul_C2, &ec_gf8_mul_C3,
+    &ec_gf8_mul_C4, &ec_gf8_mul_C5, &ec_gf8_mul_C6, &ec_gf8_mul_C7,
+    &ec_gf8_mul_C8, &ec_gf8_mul_C9, &ec_gf8_mul_CA, &ec_gf8_mul_CB,
+    &ec_gf8_mul_CC, &ec_gf8_mul_CD, &ec_gf8_mul_CE, &ec_gf8_mul_CF,
+    &ec_gf8_mul_D0, &ec_gf8_mul_D1, &ec_gf8_mul_D2, &ec_gf8_mul_D3,
+    &ec_gf8_mul_D4, &ec_gf8_mul_D5, &ec_gf8_mul_D6, &ec_gf8_mul_D7,
+    &ec_gf8_mul_D8, &ec_gf8_mul_D9, &ec_gf8_mul_DA, &ec_gf8_mul_DB,
+    &ec_gf8_mul_DC, &ec_gf8_mul_DD, &ec_gf8_mul_DE, &ec_gf8_mul_DF,
+    &ec_gf8_mul_E0, &ec_gf8_mul_E1, &ec_gf8_mul_E2, &ec_gf8_mul_E3,
+    &ec_gf8_mul_E4, &ec_gf8_mul_E5, &ec_gf8_mul_E6, &ec_gf8_mul_E7,
+    &ec_gf8_mul_E8, &ec_gf8_mul_E9, &ec_gf8_mul_EA, &ec_gf8_mul_EB,
+    &ec_gf8_mul_EC, &ec_gf8_mul_ED, &ec_gf8_mul_EE, &ec_gf8_mul_EF,
+    &ec_gf8_mul_F0, &ec_gf8_mul_F1, &ec_gf8_mul_F2, &ec_gf8_mul_F3,
+    &ec_gf8_mul_F4, &ec_gf8_mul_F5, &ec_gf8_mul_F6, &ec_gf8_mul_F7,
+    &ec_gf8_mul_F8, &ec_gf8_mul_F9, &ec_gf8_mul_FA, &ec_gf8_mul_FB,
+    &ec_gf8_mul_FC, &ec_gf8_mul_FD, &ec_gf8_mul_FE, &ec_gf8_mul_FF};
diff --git a/xlators/cluster/ec/src/ec-gf.h b/xlators/cluster/ec/src/ec-gf8.h
index 23bca91e3b5..4aca91127fc 100644
--- a/xlators/cluster/ec/src/ec-gf.h
+++ b/xlators/cluster/ec/src/ec-gf8.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,13 +11,8 @@
 #ifndef __EC_GF8_H__
 #define __EC_GF8_H__
 
-#define EC_GF_BITS 8
-#define EC_GF_MOD 0x11D
+#include "ec-galois.h"
 
-#define EC_GF_SIZE (1 << EC_GF_BITS)
-#define EC_GF_WORD_SIZE sizeof(uint64_t)
-
-extern void (* ec_gf_muladd[])(uint8_t * out, uint8_t * in,
-                               unsigned int width);
+extern ec_gf_mul_t *ec_gf8_mul[];
 
 #endif /* __EC_GF8_H__ */
diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c
index 94ff4757b4d..7d991f04aac 100644
--- a/xlators/cluster/ec/src/ec-heal.c
+++ b/xlators/cluster/ec/src/ec-heal.c
@@ -8,103 +8,164 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
-
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
+#include <glusterfs/cluster-syncop.h>
+
+#include "ec.h"
+#include "ec-types.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-method.h"
 #include "ec-fops.h"
-
-#include "ec-mem-types.h"
-#include "ec-data.h"
-#include "byte-order.h"
-#include "ec-messages.h"
-#include "syncop.h"
-#include "syncop-utils.h"
-#include "cluster-syncop.h"
-
-#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr; })
-#define EC_COUNT(array, max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res; })
-#define EC_INTERSECT(dst, src1, src2, max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i]; })
-#define EC_ADJUST_SOURCE(source, sources, max) ({int __i; if (sources[source] == 0) {source = -1; for (__i = 0; __i < max; __i++) if (sources[__i]) source = __i; } })
-#define IA_EQUAL(f, s, field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
-#define EC_REPLIES_ALLOC(replies, numsubvols) do {              \
-        int __i = 0;                                            \
-        replies = alloca0(numsubvols * sizeof (*replies));      \
-        for (__i = 0; __i < numsubvols; __i++)                  \
-                INIT_LIST_HEAD (&replies[__i].entries.list);    \
-        } while (0)
-
+#include "ec-heald.h"
+
+#define EC_COUNT(array, max)                                                   \
+    ({                                                                         \
+        int __i;                                                               \
+        int __res = 0;                                                         \
+        for (__i = 0; __i < max; __i++)                                        \
+            if (array[__i])                                                    \
+                __res++;                                                       \
+        __res;                                                                 \
+    })
+#define EC_INTERSECT(dst, src1, src2, max)                                     \
+    ({                                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < max; __i++)                                        \
+            dst[__i] = src1[__i] && src2[__i];                                 \
+    })
+#define EC_ADJUST_SOURCE(source, sources, max)                                 \
+    ({                                                                         \
+        int __i;                                                               \
+        if (sources[source] == 0) {                                            \
+            source = -1;                                                       \
+            for (__i = 0; __i < max; __i++)                                    \
+                if (sources[__i])                                              \
+                    source = __i;                                              \
+        }                                                                      \
+    })
+#define IA_EQUAL(f, s, field)                                                  \
+    (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0)
+#define EC_REPLIES_ALLOC(replies, numsubvols)                                  \
+    do {                                                                       \
+        int __i = 0;                                                           \
+        replies = alloca0(numsubvols * sizeof(*replies));                      \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+    } while (0)
 
 struct ec_name_data {
-        call_frame_t *frame;
-        unsigned char *participants;
-        unsigned char *failed_on;
-        unsigned char *gfidless;
-        unsigned char *enoent;
-        unsigned char *same;
-        char *name;
-        inode_t *parent;
-        default_args_cbk_t *replies;
+    call_frame_t *frame;
+    unsigned char *participants;
+    unsigned char *failed_on;
+    unsigned char *gfidless;
+    unsigned char *enoent;
+    unsigned char *same;
+    char *name;
+    inode_t *parent;
+    default_args_cbk_t *replies;
+    uint32_t heal_pending;
 };
 
-static char *ec_ignore_xattrs[] = {
-        GF_SELINUX_XATTR_KEY,
-        QUOTA_SIZE_KEY,
-        NULL
-};
+static char *ec_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
 
 static gf_boolean_t
-ec_ignorable_key_match (dict_t *dict, char *key, data_t *val, void *mdata)
+ec_ignorable_key_match(dict_t *dict, char *key, data_t *val, void *mdata)
 {
-        int i = 0;
+    int i = 0;
 
-        if (!key)
-                goto out;
+    if (!key)
+        goto out;
 
-        if (strncmp (key, EC_XATTR_PREFIX, strlen (EC_XATTR_PREFIX)) == 0)
-                        return _gf_true;
+    if (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0)
+        return _gf_true;
 
-        for (i = 0; ec_ignore_xattrs[i]; i++) {
-                if (!strcmp (key, ec_ignore_xattrs[i]))
-                       return _gf_true;
-        }
+    for (i = 0; ec_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, ec_ignore_xattrs[i]))
+            return _gf_true;
+    }
 
 out:
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-ec_sh_key_match (dict_t *dict, char *key, data_t *val, void *mdata)
+ec_sh_key_match(dict_t *dict, char *key, data_t *val, void *mdata)
 {
-        return !ec_ignorable_key_match (dict, key, val, mdata);
+    return !ec_ignorable_key_match(dict, key, val, mdata);
 }
 /* FOP: heal */
 
-uintptr_t ec_heal_check(ec_fop_data_t * fop, uintptr_t * pgood)
+void
+ec_set_entry_healing(ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    loc_t *loc = NULL;
+
+    if (!fop)
+        return;
+
+    loc = &fop->loc[0];
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            ctx->heal_count += 1;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+}
+
+void
+ec_reset_entry_healing(ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    loc_t *loc = NULL;
+    int32_t heal_count = 0;
+    if (!fop)
+        return;
+
+    loc = &fop->loc[0];
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            ctx->heal_count += -1;
+            heal_count = ctx->heal_count;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+    GF_ASSERT(heal_count >= 0);
+}
+
+uintptr_t
+ec_heal_check(ec_fop_data_t *fop, uintptr_t *pgood)
 {
-    ec_cbk_data_t * cbk;
-    uintptr_t mask[2] = { 0, 0 };
+    ec_cbk_data_t *cbk;
+    uintptr_t mask[2] = {0, 0};
 
     list_for_each_entry(cbk, &fop->cbk_list, list)
     {
         mask[cbk->op_ret >= 0] |= cbk->mask;
     }
 
-    if (pgood != NULL)
-    {
+    if (pgood != NULL) {
         *pgood = mask[1];
     }
 
     return mask[0];
 }
 
-void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
+void
+ec_heal_update(ec_fop_data_t *fop, int32_t is_open)
 {
-    ec_heal_t * heal = fop->data;
+    ec_heal_t *heal = fop->data;
     uintptr_t good, bad;
 
     bad = ec_heal_check(fop, &good);
@@ -112,8 +173,7 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
     LOCK(&heal->lock);
 
     heal->bad &= ~bad;
-    if (is_open)
-    {
+    if (is_open) {
         heal->open |= good;
     }
 
@@ -122,9 +182,10 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
     fop->error = 0;
 }
 
-void ec_heal_avoid(ec_fop_data_t * fop)
+void
+ec_heal_avoid(ec_fop_data_t *fop)
 {
-    ec_heal_t * heal = fop->data;
+    ec_heal_t *heal = fop->data;
     uintptr_t bad;
 
     bad = ec_heal_check(fop, NULL);
@@ -136,22 +197,24 @@ void ec_heal_avoid(ec_fop_data_t * fop)
     UNLOCK(&heal->lock);
 }
 
-int32_t ec_heal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_heal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_heal_t *heal = fop->data;
 
     if (op_ret >= 0) {
-        GF_ASSERT(ec_set_inode_size(heal->fop, heal->fd->inode,
-                                    heal->total_size));
+        GF_ASSERT(
+            ec_set_inode_size(heal->fop, heal->fd->inode, heal->total_size));
     }
 
     return 0;
 }
 
-void ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc,
-                  off_t offset, size_t size)
+void
+ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc, off_t offset,
+             size_t size)
 {
     struct gf_flock flock;
     fop_inodelk_cbk_t cbk = NULL;
@@ -170,98 +233,116 @@ void ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc,
         } else {
             ec_clear_inode_info(heal->fop, heal->fd->inode);
         }
+        cbk = ec_lock_unlocked;
     } else {
         /* Otherwise use the callback to update size information. */
         cbk = ec_heal_lock_cbk;
     }
 
-    if (fd != NULL)
-    {
-        ec_finodelk(heal->fop->frame, heal->xl, heal->fop->mask,
+    if (fd != NULL) {
+        ec_finodelk(heal->fop->frame, heal->xl,
+                    &heal->fop->frame->root->lk_owner, heal->fop->mask,
                     EC_MINIMUM_ALL, cbk, heal, heal->xl->name, fd, F_SETLKW,
                     &flock, NULL);
-    }
-    else
-    {
-        ec_inodelk(heal->fop->frame, heal->xl, heal->fop->mask, EC_MINIMUM_ALL,
-                   cbk, heal, heal->xl->name, loc, F_SETLKW, &flock, NULL);
+    } else {
+        ec_inodelk(heal->fop->frame, heal->xl,
+                   &heal->fop->frame->root->lk_owner, heal->fop->mask,
+                   EC_MINIMUM_ALL, cbk, heal, heal->xl->name, loc, F_SETLKW,
+                   &flock, NULL);
     }
 }
 
-void ec_heal_inodelk(ec_heal_t *heal, int32_t type, int32_t use_fd,
-                     off_t offset, size_t size)
+void
+ec_heal_inodelk(ec_heal_t *heal, int32_t type, int32_t use_fd, off_t offset,
+                size_t size)
 {
     ec_heal_lock(heal, type, use_fd ? heal->fd : NULL, &heal->loc, offset,
                  size);
 }
 
 int32_t
-ec_heal_xattr_clean (dict_t *dict, char *key, data_t *data,
-                     void *arg)
+ec_heal_xattr_clean(dict_t *dict, char *key, data_t *data, void *arg)
 {
-        dict_t *base = arg;
+    dict_t *base = arg;
 
-        if (ec_ignorable_key_match (NULL, key, NULL, NULL)) {
-                dict_del (dict, key);
-                return 0;
-        }
+    if (ec_ignorable_key_match(NULL, key, NULL, NULL)) {
+        dict_del(dict, key);
+        return 0;
+    }
 
-        if (dict_get (base, key) != NULL)
-                dict_del (dict, key);
+    if (dict_get(base, key) != NULL)
+        dict_del(dict, key);
 
-        return 0;
+    return 0;
+}
+
+/********************************************************************
+ * ec_wind_xattrop_parallel:
+ *              Helper function to update the extended attributes
+ *    in parallel.
+ *
+ *******************************************************************/
+void
+ec_wind_xattrop_parallel(call_frame_t *frame, xlator_t *subvol, int child_index,
+                         loc_t *loc, gf_xattrop_flags_t flags, dict_t **dict,
+                         dict_t *xdata)
+{
+    gf_msg_debug("EC", 0, "WIND: on child %d ", child_index);
+    STACK_WIND_COOKIE(
+        frame, cluster_xattrop_cbk, (void *)(uintptr_t)child_index, subvol,
+        subvol->fops->xattrop, loc, flags, dict[child_index], xdata);
 }
 
 int32_t
-ec_heal_writev_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf,
-                    dict_t *xdata)
+ec_heal_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_heal_t *heal = fop->data;
 
     ec_trace("WRITE_CBK", cookie, "ret=%d, errno=%d", op_ret, op_errno);
 
-    gf_msg_debug (fop->xl->name, 0, "%s: write op_ret %d, op_errno %s"
-            " at %"PRIu64, uuid_utoa (heal->fd->inode->gfid), op_ret,
-            strerror (op_errno), heal->offset);
+    gf_msg_debug(fop->xl->name, 0,
+                 "%s: write op_ret %d, op_errno %s"
+                 " at %" PRIu64,
+                 uuid_utoa(heal->fd->inode->gfid), op_ret, strerror(op_errno),
+                 heal->offset);
 
     ec_heal_update(cookie, 0);
 
     return 0;
 }
 
-int32_t ec_heal_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iovec * vector, int32_t count,
-                          struct iatt * stbuf, struct iobref * iobref,
-                          dict_t * xdata)
+int32_t
+ec_heal_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_heal_t * heal = fop->data;
+    ec_fop_data_t *fop = cookie;
+    ec_heal_t *heal = fop->data;
 
     ec_trace("READ_CBK", fop, "ret=%d, errno=%d", op_ret, op_errno);
 
     ec_heal_avoid(fop);
 
-    if (op_ret > 0)
-    {
-        gf_msg_debug (fop->xl->name, 0, "%s: read succeeded, proceeding "
-                "to write at %"PRIu64, uuid_utoa (heal->fd->inode->gfid),
-                heal->offset);
+    if (op_ret > 0) {
+        gf_msg_debug(fop->xl->name, 0,
+                     "%s: read succeeded, proceeding "
+                     "to write at %" PRIu64,
+                     uuid_utoa(heal->fd->inode->gfid), heal->offset);
         ec_writev(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE,
                   ec_heal_writev_cbk, heal, heal->fd, vector, count,
                   heal->offset, 0, iobref, NULL);
-    }
-    else
-    {
+    } else {
         if (op_ret < 0) {
-                gf_msg_debug (fop->xl->name, 0, "%s: read failed %s, failing "
-                        "to heal block at %"PRIu64,
-                        uuid_utoa (heal->fd->inode->gfid), strerror (op_errno),
-                        heal->offset);
-                heal->bad = 0;
+            gf_msg_debug(fop->xl->name, 0,
+                         "%s: read failed %s, failing "
+                         "to heal block at %" PRIu64,
+                         uuid_utoa(heal->fd->inode->gfid), strerror(op_errno),
+                         heal->offset);
+            heal->bad = 0;
         }
         heal->done = 1;
     }
@@ -269,1542 +350,1657 @@ int32_t ec_heal_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     return 0;
 }
 
-void ec_heal_data_block(ec_heal_t *heal)
+void
+ec_heal_data_block(ec_heal_t *heal)
 {
     ec_trace("DATA", heal->fop, "good=%lX, bad=%lX", heal->good, heal->bad);
 
     if ((heal->good != 0) && (heal->bad != 0) &&
-        (heal->iatt.ia_type == IA_IFREG))
-    {
+        (heal->iatt.ia_type == IA_IFREG)) {
         ec_readv(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN,
-                 ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset,
-                 0, NULL);
+                 ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset, 0,
+                 NULL);
     }
 }
 
 /* FOP: fheal */
 
-void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fheal_cbk_t func, void * data, fd_t * fd,
-              int32_t partial, dict_t *xdata)
+void
+ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd,
+         int32_t partial, dict_t *xdata)
 {
-    ec_fd_t * ctx = ec_fd_get(fd, this);
+    ec_fd_t *ctx = ec_fd_get(fd, this);
 
-    if (ctx != NULL)
-    {
-        gf_msg_trace ("ec", 0, "FHEAL ctx: flags=%X, open=%lX", ctx->flags,
-                      ctx->open);
-        ec_heal(frame, this, target, minimum, func, data, &ctx->loc, partial,
+    if (ctx != NULL) {
+        gf_msg_trace("ec", 0, "FHEAL ctx: flags=%X, open=%" PRIXPTR, ctx->flags,
+                     ctx->open);
+        ec_heal(frame, this, target, fop_flags, func, data, &ctx->loc, partial,
                 xdata);
     }
 }
 
 /* Common heal code */
 void
-ec_mask_to_char_array (uintptr_t mask, unsigned char *array, int numsubvols)
+ec_mask_to_char_array(uintptr_t mask, unsigned char *array, int numsubvols)
 {
-        int     i = 0;
+    int i = 0;
 
-        for (i = 0; i < numsubvols; i++)
-                array[i] = ((mask >> i) & 1);
+    for (i = 0; i < numsubvols; i++)
+        array[i] = ((mask >> i) & 1);
 }
 
 uintptr_t
-ec_char_array_to_mask (unsigned char *array, int numsubvols)
+ec_char_array_to_mask(unsigned char *array, int numsubvols)
 {
-        int       i    = 0;
-        uintptr_t mask = 0;
+    int i = 0;
+    uintptr_t mask = 0;
+
+    if (array == NULL)
+        goto out;
 
-        for (i = 0; i < numsubvols; i++)
-                if (array[i])
-                        mask |= (1ULL<<i);
-        return mask;
+    for (i = 0; i < numsubvols; i++)
+        if (array[i])
+            mask |= (1ULL << i);
+out:
+    return mask;
 }
 
 int
-ec_heal_entry_find_direction (ec_t *ec, default_args_cbk_t *replies,
-                        uint64_t *versions, uint64_t *dirty,
-                        unsigned char *sources, unsigned char *healed_sinks)
+ec_heal_entry_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                             uint64_t *versions, uint64_t *dirty,
+                             unsigned char *sources,
+                             unsigned char *healed_sinks)
 {
-        uint64_t    xattr[EC_VERSION_SIZE] = {0};
-        int         source      = -1;
-        uint64_t    max_version = 0;
-        int         ret         = 0;
-        int         i           = 0;
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    int source = -1;
+    uint64_t max_version = 0;
+    int ret = 0;
+    int i = 0;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret == -1)
-                        continue;
-
-                if (source == -1)
-                        source = i;
-
-                ret = ec_dict_del_array (replies[i].xdata, EC_XATTR_VERSION,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        versions[i] = xattr[EC_DATA_TXN];
-                        if (max_version < versions[i]) {
-                                max_version = versions[i];
-                                source = i;
-                        }
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                memset (xattr, 0, sizeof(xattr));
-                ret = ec_dict_del_array (replies[i].xdata, EC_XATTR_DIRTY,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        dirty[i] = xattr[EC_DATA_TXN];
-                }
+        if (replies[i].op_ret == -1)
+            continue;
+
+        if (source == -1)
+            source = i;
+
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            versions[i] = xattr[EC_DATA_TXN];
+            if (max_version < versions[i]) {
+                max_version = versions[i];
+                source = i;
+            }
         }
 
-        if (source < 0)
-                goto out;
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_DATA_TXN];
+        }
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
+    if (source < 0)
+        goto out;
 
-                if (replies[i].op_ret == -1)
-                        continue;
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (versions[i] == versions[source])
-                        sources[i] = 1;
-                else
-                        healed_sinks[i] = 1;
-        }
+        if (replies[i].op_ret == -1)
+            continue;
+
+        if (versions[i] == versions[source])
+            sources[i] = 1;
+        else
+            healed_sinks[i] = 1;
+    }
 
 out:
-        return source;
+    return source;
 }
 
 int
-ec_adjust_versions (call_frame_t *frame, ec_t *ec, ec_txn_t type,
-                    inode_t *inode, int source, unsigned char *sources,
-                    unsigned char *healed_sinks, uint64_t *versions,
-                    uint64_t *dirty)
-{
-        int                        i                 = 0;
-        int                        ret               = 0;
-        dict_t                     *xattr            = NULL;
-        int                        op_ret            = 0;
-        loc_t                      loc               = {0};
-        gf_boolean_t               erase_dirty       = _gf_false;
-        uint64_t                   versions_xattr[2] = {0};
-        uint64_t                   dirty_xattr[2]    = {0};
-        uint64_t                   allzero[2]        = {0};
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        xattr = dict_new ();
-        if (!xattr)
-                goto out;
+ec_adjust_versions(call_frame_t *frame, ec_t *ec, ec_txn_t type, inode_t *inode,
+                   int source, unsigned char *sources,
+                   unsigned char *healed_sinks, uint64_t *versions,
+                   uint64_t *dirty)
+{
+    int i = 0;
+    int ret = 0;
+    int call_count = 0;
+    dict_t **xattr = NULL;
+    int op_ret = 0;
+    loc_t loc = {0};
+    gf_boolean_t erase_dirty = _gf_false;
+    uint64_t *versions_xattr = NULL;
+    uint64_t *dirty_xattr = NULL;
+    uint64_t allzero[2] = {0};
+    unsigned char *on = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+
+    /* Allocate the required memory */
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer);
+    if (!xattr) {
+        op_ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        xattr[i] = dict_new();
+        if (!xattr[i]) {
+            op_ret = -ENOMEM;
+            goto out;
+        }
+    }
 
-        /* dirty xattr represents if the file/dir needs heal. Unless all the
-         * copies are healed, don't erase it */
-        if (EC_COUNT (sources, ec->nodes) +
-            EC_COUNT (healed_sinks, ec->nodes) == ec->nodes)
-                erase_dirty = _gf_true;
+    /* dirty xattr represents if the file/dir needs heal. Unless all the
+     * copies are healed, don't erase it */
+    if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) ==
+        ec->nodes)
+        erase_dirty = _gf_true;
+    else
+        op_ret = -ENOTCONN;
+
+    /* Populate the xattr array */
+    for (i = 0; i < ec->nodes; i++) {
+        if (!sources[i] && !healed_sinks[i])
+            continue;
+        versions_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*versions_xattr),
+                                   gf_common_mt_pointer);
+        if (!versions_xattr) {
+            op_ret = -ENOMEM;
+            continue;
+        }
+
+        versions_xattr[type] = hton64(versions[source] - versions[i]);
+        ret = dict_set_bin(xattr[i], EC_XATTR_VERSION, versions_xattr,
+                           (sizeof(*versions_xattr) * EC_VERSION_SIZE));
+        if (ret < 0) {
+            op_ret = -ENOMEM;
+            continue;
+        }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!sources[i] && !healed_sinks[i])
-                        continue;
-                versions_xattr[type] = hton64(versions[source] - versions[i]);
-                ret = dict_set_static_bin (xattr, EC_XATTR_VERSION,
-                                           versions_xattr,
-                                           sizeof (versions_xattr));
-                if (ret < 0) {
-                        op_ret = -ENOTCONN;
-                        continue;
-                }
+        if (erase_dirty) {
+            dirty_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*dirty_xattr),
+                                    gf_common_mt_pointer);
+            if (!dirty_xattr) {
+                op_ret = -ENOMEM;
+                continue;
+            }
 
-                if (erase_dirty) {
-                        dirty_xattr[type] = hton64(-dirty[i]);
-                        ret = dict_set_static_bin (xattr, EC_XATTR_DIRTY,
-                                                   dirty_xattr,
-                                                   sizeof (dirty_xattr));
-                        if (ret < 0) {
-                                op_ret = -ENOTCONN;
-                                continue;
-                        }
-                }
+            dirty_xattr[type] = hton64(-dirty[i]);
+            ret = dict_set_bin(xattr[i], EC_XATTR_DIRTY, dirty_xattr,
+                               (sizeof(*dirty_xattr) * EC_VERSION_SIZE));
+            if (ret < 0) {
+                op_ret = -ENOMEM;
+                continue;
+            }
+        }
 
-                if ((memcmp (versions_xattr, allzero, sizeof (allzero)) == 0) &&
-                    (memcmp (dirty_xattr, allzero, sizeof (allzero)) == 0))
-                        continue;
+        if (memcmp(versions_xattr, allzero,
+                   (sizeof(*versions_xattr) * EC_VERSION_SIZE)) == 0) {
+            if (!erase_dirty) {
+                continue;
+            }
 
-                ret = syncop_xattrop (ec->xl_list[i], &loc,
-                                      GF_XATTROP_ADD_ARRAY64, xattr, NULL,
-                                      NULL);
-                if (ret < 0) {
-                        op_ret = -ret;
-                        continue;
-                }
+            if (memcmp(dirty_xattr, allzero,
+                       (sizeof(*dirty_xattr) * EC_VERSION_SIZE)) == 0) {
+                continue;
+            }
         }
 
-out:
-        if (xattr)
-                dict_unref (xattr);
-        loc_wipe (&loc);
-        return op_ret;
-}
-int
-ec_heal_metadata_find_direction (ec_t *ec, default_args_cbk_t *replies,
-                                 uint64_t *versions, uint64_t *dirty,
-                            unsigned char *sources, unsigned char *healed_sinks)
-{
-        uint64_t xattr[EC_VERSION_SIZE] = {0};
-        uint64_t max_version    = 0;
-        int      same_count     = 0;
-        int      max_same_count = 0;
-        int      same_source    = -1;
-        int      ret            = 0;
-        int      i              = 0;
-        int      j              = 0;
-        int      *groups        = NULL;
-        struct iatt source_ia   = {0};
-        struct iatt child_ia    = {0};
-
-        groups = alloca0 (ec->nodes * sizeof(*groups));
-        for (i = 0; i < ec->nodes; i++)
-                groups[i] = -1;
+        on[i] = 1;
+        call_count++;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-                if (replies[i].op_ret < 0)
-                        continue;
-                ret = ec_dict_del_array (replies[i].xdata, EC_XATTR_VERSION,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        versions[i] = xattr[EC_METADATA_TXN];
-                }
+    /* Update the bricks with xattr */
+    if (call_count) {
+        PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame,
+                            ec_wind_xattrop_parallel, &loc,
+                            GF_XATTROP_ADD_ARRAY64, xattr, NULL);
+        ret = cluster_fop_success_fill(replies, ec->nodes, output);
+    }
 
-                memset (xattr, 0, sizeof (xattr));
-                ret = ec_dict_del_array (replies[i].xdata, EC_XATTR_DIRTY,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        dirty[i] = xattr[EC_METADATA_TXN];
-                }
-                if (groups[i] >= 0) /*Already part of group*/
-                        continue;
-                groups[i] = i;
-                same_count = 1;
-                source_ia = replies[i].stat;
-                for (j = i + 1; j < ec->nodes; j++) {
-                        if (!replies[j].valid || replies[j].op_ret < 0)
-                                continue;
-                        child_ia = replies[j].stat;
-                        if (!IA_EQUAL(source_ia, child_ia, gfid) ||
-                            !IA_EQUAL(source_ia, child_ia, type) ||
-                            !IA_EQUAL(source_ia, child_ia, prot) ||
-                            !IA_EQUAL(source_ia, child_ia, uid) ||
-                            !IA_EQUAL(source_ia, child_ia, gid))
-                                continue;
-                        if (!are_dicts_equal(replies[i].xdata, replies[j].xdata,
-                                             ec_sh_key_match, NULL))
-                                continue;
-                        groups[j] = i; /*If iatts match put them into a group*/
-                        same_count++;
-                }
+    if (ret < call_count) {
+        op_ret = -ENOTCONN;
+        goto out;
+    }
 
-                if (max_same_count < same_count) {
-                        max_same_count = same_count;
-                        same_source = i;
-                }
+out:
+    /* Cleanup */
+    if (xattr) {
+        for (i = 0; i < ec->nodes; i++) {
+            if (xattr[i])
+                dict_unref(xattr[i]);
         }
+        GF_FREE(xattr);
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return op_ret;
+}
 
-        if (max_same_count < ec->fragments) {
-                ret = -EIO;
-                goto out;
+int
+ec_heal_metadata_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                                uint64_t *versions, uint64_t *dirty,
+                                unsigned char *sources,
+                                unsigned char *healed_sinks)
+{
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    uint64_t max_version = 0;
+    int same_count = 0;
+    int max_same_count = 0;
+    int same_source = -1;
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+    int *groups = NULL;
+    struct iatt source_ia = {0};
+    struct iatt child_ia = {0};
+
+    groups = alloca0(ec->nodes * sizeof(*groups));
+    for (i = 0; i < ec->nodes; i++)
+        groups[i] = -1;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret < 0)
+            continue;
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            versions[i] = xattr[EC_METADATA_TXN];
         }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (groups[i] == groups[same_source])
-                        sources[i] = 1;
-                else if (replies[i].valid && replies[i].op_ret >= 0)
-                        healed_sinks[i] = 1;
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_METADATA_TXN];
+        }
+        if (groups[i] >= 0) /*Already part of group*/
+            continue;
+        groups[i] = i;
+        same_count = 1;
+        source_ia = replies[i].stat;
+        for (j = i + 1; j < ec->nodes; j++) {
+            if (!replies[j].valid || replies[j].op_ret < 0)
+                continue;
+            child_ia = replies[j].stat;
+            if (!IA_EQUAL(source_ia, child_ia, gfid) ||
+                !IA_EQUAL(source_ia, child_ia, type) ||
+                !IA_EQUAL(source_ia, child_ia, prot) ||
+                !IA_EQUAL(source_ia, child_ia, uid) ||
+                !IA_EQUAL(source_ia, child_ia, gid))
+                continue;
+            if (!are_dicts_equal(replies[i].xdata, replies[j].xdata,
+                                 ec_sh_key_match, NULL))
+                continue;
+            groups[j] = i;
+            same_count++;
+        }
+
+        if (max_same_count < same_count) {
+            max_same_count = same_count;
+            same_source = i;
         }
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i] && (versions[i] > max_version)) {
-                         same_source = i;
-                         max_version = versions[i];
-                }
+    }
+
+    if (max_same_count < ec->fragments) {
+        ret = -EIO;
+        goto out;
+    }
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (groups[i] == groups[same_source])
+            sources[i] = 1;
+        else if (replies[i].valid && replies[i].op_ret >= 0)
+            healed_sinks[i] = 1;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i] && (versions[i] > max_version)) {
+            same_source = i;
+            max_version = versions[i];
         }
-        ret = same_source;
+    }
+    ret = same_source;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-__ec_heal_metadata_prepare (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                   unsigned char *locked_on, default_args_cbk_t *replies,
-                   uint64_t *versions, uint64_t *dirty, unsigned char *sources,
-                   unsigned char *healed_sinks)
+__ec_heal_metadata_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                           unsigned char *locked_on,
+                           default_args_cbk_t *replies, uint64_t *versions,
+                           uint64_t *dirty, unsigned char *sources,
+                           unsigned char *healed_sinks)
 {
-        loc_t              loc        = {0};
-        unsigned char      *output    = NULL;
-        unsigned char      *lookup_on = NULL;
-        int                ret        = 0;
-        int                source     = 0;
-        default_args_cbk_t *greplies  = NULL;
-        int                i          = 0;
-        EC_REPLIES_ALLOC (greplies, ec->nodes);
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        output = alloca0 (ec->nodes);
-        lookup_on = alloca0 (ec->nodes);
-        ret = cluster_lookup (ec->xl_list, locked_on, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, NULL);
-        if (ret <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+    unsigned char *lookup_on = NULL;
+    int ret = 0;
+    int source = 0;
+    default_args_cbk_t *greplies = NULL;
+    int i = 0;
+    EC_REPLIES_ALLOC(greplies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    output = alloca0(ec->nodes);
+    lookup_on = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, NULL);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        memcpy (lookup_on, output, ec->nodes);
-        /*Use getxattr to get the filtered xattrs which filter internal xattrs*/
-        ret = cluster_getxattr (ec->xl_list, lookup_on, ec->nodes, greplies,
-                                output, frame, ec->xl, &loc, NULL, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (lookup_on[i] && !output[i]) {
-                        replies[i].valid = 0;
-                        continue;
-                }
-                if (replies[i].xdata) {
-                        dict_unref (replies[i].xdata);
-                        replies[i].xdata = NULL;
-                        if (greplies[i].xattr)
-                                replies[i].xdata = dict_ref (greplies[i].xattr);
-                }
+    memcpy(lookup_on, output, ec->nodes);
+    /*Use getxattr to get the filtered xattrs which filter internal xattrs*/
+    ret = cluster_getxattr(ec->xl_list, lookup_on, ec->nodes, greplies, output,
+                           frame, ec->xl, &loc, NULL, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (lookup_on[i] && !output[i]) {
+            replies[i].valid = 0;
+            continue;
+        }
+        if (replies[i].xdata) {
+            dict_unref(replies[i].xdata);
+            replies[i].xdata = NULL;
+            if (greplies[i].xattr)
+                replies[i].xdata = dict_ref(greplies[i].xattr);
         }
+    }
 
-        source = ec_heal_metadata_find_direction (ec, replies, versions,
-                                         dirty, sources, healed_sinks);
-        if (source < 0) {
-                ret = -EIO;
-                goto out;
-        }
-        ret = source;
+    source = ec_heal_metadata_find_direction(ec, replies, versions, dirty,
+                                             sources, healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
+    ret = source;
 out:
-        cluster_replies_wipe (greplies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    cluster_replies_wipe(greplies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 /* Metadata heal */
 int
-__ec_removexattr_sinks (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                        int source, unsigned char *sources,
-                        unsigned char *healed_sinks,
-                        default_args_cbk_t *replies)
+__ec_removexattr_sinks(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                       int source, unsigned char *sources,
+                       unsigned char *healed_sinks, default_args_cbk_t *replies)
 {
-        int   i   = 0;
-        int   ret = 0;
-        loc_t loc = {0};
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-
-        for (i = 0; i < ec->nodes; i++) {
-                if (i == source)
-                        continue;
-                if (!sources[i] && !healed_sinks[i])
-                        continue;
-                ret = dict_foreach (replies[i].xdata, ec_heal_xattr_clean,
-                                    replies[source].xdata);
-                if (ret < 0) {
-                        sources[i] = 0;
-                        healed_sinks[i] = 0;
-                        continue;
-                }
-
-                if (replies[i].xdata->count == 0) {
-                        continue;
-                } else if (sources[i]) {
-                        /* This can happen if setxattr/removexattr succeeds on
-                         * the bricks but fails to update the version. This
-                         * will make sure that the xattrs are made equal after
-                         * heal*/
-                        sources[i] = 0;
-                        healed_sinks[i] = 1;
-                }
+    int i = 0;
+    int ret = 0;
+    loc_t loc = {0};
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (i == source)
+            continue;
+        if (!sources[i] && !healed_sinks[i])
+            continue;
+        ret = dict_foreach(replies[i].xdata, ec_heal_xattr_clean,
+                           replies[source].xdata);
+        if (ret < 0) {
+            sources[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
+        }
 
-                ret = syncop_removexattr (ec->xl_list[i], &loc, "",
-                                          replies[i].xdata, NULL);
-                if (ret < 0)
-                        healed_sinks[i] = 0;
+        if (replies[i].xdata->count == 0) {
+            continue;
+        } else if (sources[i]) {
+            /* This can happen if setxattr/removexattr succeeds on
+             * the bricks but fails to update the version. This
+             * will make sure that the xattrs are made equal after
+             * heal*/
+            sources[i] = 0;
+            healed_sinks[i] = 1;
         }
 
-        loc_wipe (&loc);
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0)
-                return -ENOTCONN;
-        return 0;
+        ret = syncop_removexattr(ec->xl_list[i], &loc, "", replies[i].xdata,
+                                 NULL);
+        if (ret < 0)
+            healed_sinks[i] = 0;
+    }
+
+    loc_wipe(&loc);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0)
+        return -ENOTCONN;
+    return 0;
 }
 
 int
-__ec_heal_metadata (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                    unsigned char *locked_on, unsigned char *sources,
-                    unsigned char *healed_sinks)
-{
-        loc_t              loc           = {0};
-        int                ret           = 0;
-        int                source        = 0;
-        default_args_cbk_t *replies      = NULL;
-        default_args_cbk_t *sreplies     = NULL;
-        uint64_t           *versions     = NULL;
-        uint64_t           *dirty        = NULL;
-        unsigned char      *output       = NULL;
-        dict_t             *source_dict  = NULL;
-        struct iatt        source_buf    = {0};
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        EC_REPLIES_ALLOC (sreplies, ec->nodes);
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        output = alloca0 (ec->nodes);
-        versions = alloca0 (ec->nodes * sizeof (*versions));
-        dirty = alloca0 (ec->nodes * sizeof (*dirty));
-        source = __ec_heal_metadata_prepare (frame, ec, inode, locked_on, replies,
-                                    versions, dirty, sources, healed_sinks);
-        if (source < 0) {
-                ret = -EIO;
-                goto out;
-        }
+__ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                   unsigned char *locked_on, unsigned char *sources,
+                   unsigned char *healed_sinks)
+{
+    loc_t loc = {0};
+    int ret = 0;
+    int source = 0;
+    default_args_cbk_t *replies = NULL;
+    default_args_cbk_t *sreplies = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    unsigned char *output = NULL;
+    dict_t *source_dict = NULL;
+    struct iatt source_buf = {0};
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    EC_REPLIES_ALLOC(sreplies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    output = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    source = __ec_heal_metadata_prepare(frame, ec, inode, locked_on, replies,
+                                        versions, dirty, sources, healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
 
-        if (EC_COUNT (sources, ec->nodes) == ec->nodes) {
-                ret = 0;
-                goto erase_dirty;
-        }
+    if ((EC_COUNT(sources, ec->nodes) == ec->nodes) ||
+        (EC_COUNT(healed_sinks, ec->nodes) == 0)) {
+        ret = 0;
+        goto erase_dirty;
+    }
 
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
-        source_buf = replies[source].stat;
-        ret = cluster_setattr (ec->xl_list, healed_sinks, ec->nodes, sreplies,
-                               output, frame, ec->xl, &loc,
-                               &source_buf, GF_SET_ATTR_MODE |
-                               GF_SET_ATTR_UID | GF_SET_ATTR_GID, NULL);
-        /*In case the operation fails on some of the subvols*/
-        memcpy (healed_sinks, output, ec->nodes);
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    source_buf = replies[source].stat;
+    ret = cluster_setattr(ec->xl_list, healed_sinks, ec->nodes, sreplies,
+                          output, frame, ec->xl, &loc, &source_buf,
+                          GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID,
+                          NULL);
+    /*In case the operation fails on some of the subvols*/
+    memcpy(healed_sinks, output, ec->nodes);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        ret = __ec_removexattr_sinks (frame, ec, inode, source, sources,
-                                      healed_sinks, replies);
-        if (ret < 0)
-                goto out;
+    ret = __ec_removexattr_sinks(frame, ec, inode, source, sources,
+                                 healed_sinks, replies);
+    if (ret < 0)
+        goto out;
 
-        source_dict = dict_ref (replies[source].xdata);
-        if (dict_foreach_match (source_dict, ec_ignorable_key_match, NULL,
-                                dict_remove_foreach_fn, NULL) == -1) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    source_dict = dict_ref(replies[source].xdata);
+    if (dict_foreach_match(source_dict, ec_ignorable_key_match, NULL,
+                           dict_remove_foreach_fn, NULL) == -1) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = cluster_setxattr (ec->xl_list, healed_sinks, ec->nodes,
-                                replies, output, frame, ec->xl, &loc,
-                                source_dict, 0, NULL);
+    ret = cluster_setxattr(ec->xl_list, healed_sinks, ec->nodes, replies,
+                           output, frame, ec->xl, &loc, source_dict, 0, NULL);
 
-        EC_INTERSECT (healed_sinks, healed_sinks, output, ec->nodes);
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
 erase_dirty:
-        ret = ec_adjust_versions (frame, ec, EC_METADATA_TXN, inode, source,
-                                  sources, healed_sinks, versions, dirty);
+    ret = ec_adjust_versions(frame, ec, EC_METADATA_TXN, inode, source, sources,
+                             healed_sinks, versions, dirty);
 out:
-        if (source_dict)
-                dict_unref (source_dict);
+    if (source_dict)
+        dict_unref(source_dict);
 
-        loc_wipe (&loc);
-        cluster_replies_wipe (replies, ec->nodes);
-        cluster_replies_wipe (sreplies, ec->nodes);
-        return ret;
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    cluster_replies_wipe(sreplies, ec->nodes);
+    return ret;
 }
 
 int
-ec_heal_metadata (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                  unsigned char *sources, unsigned char *healed_sinks)
-{
-        unsigned char      *locked_on  = NULL;
-        unsigned char      *up_subvols = NULL;
-        unsigned char      *output     = NULL;
-        int                ret         = 0;
-        default_args_cbk_t *replies    = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        locked_on = alloca0(ec->nodes);
-        output = alloca0(ec->nodes);
-        up_subvols = alloca0(ec->nodes);
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, up_subvols, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name, inode, 0,
-                               0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_metadata (frame, ec, inode, locked_on, sources,
-                                          healed_sinks);
-        }
+ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                 unsigned char *sources, unsigned char *healed_sinks)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_metadata(frame, ec, inode, locked_on, sources,
+                                 healed_sinks);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, inode, 0, 0);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 /*entry heal*/
 int
-__ec_heal_entry_prepare (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                         unsigned char *locked_on, uint64_t *versions,
-                         uint64_t *dirty, unsigned char *sources,
-                         unsigned char *healed_sinks)
-{
-        loc_t              loc      = {0};
-        int                source   = 0;
-        int                ret      = 0;
-        default_args_cbk_t *replies = NULL;
-        unsigned char      *output  = NULL;
-        dict_t             *xdata   = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        xdata = dict_new ();
-        if (!xdata) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_entry_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                        unsigned char *locked_on, uint64_t *versions,
+                        uint64_t *dirty, unsigned char *sources,
+                        unsigned char *healed_sinks)
+{
+    loc_t loc = {0};
+    int source = 0;
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    dict_t *xdata = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        if (dict_set_uint64(xdata, EC_XATTR_VERSION, 0) ||
-            dict_set_uint64(xdata, EC_XATTR_DIRTY, 0)) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (dict_set_uint64(xdata, EC_XATTR_VERSION, 0) ||
+        dict_set_uint64(xdata, EC_XATTR_DIRTY, 0)) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        output = alloca0 (ec->nodes);
-        ret = cluster_lookup (ec->xl_list, locked_on, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, xdata);
-        if (ret <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    output = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, xdata);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        source = ec_heal_entry_find_direction (ec, replies, versions,
-                                         dirty, sources, healed_sinks);
-        if (source < 0) {
-                ret = -EIO;
-                goto out;
-        }
-        ret = source;
+    source = ec_heal_entry_find_direction(ec, replies, versions, dirty, sources,
+                                          healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
+    ret = source;
 out:
-        if (xdata)
-                dict_unref (xdata);
-        loc_wipe (&loc);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 int32_t
-ec_set_new_entry_dirty (ec_t *ec, loc_t *loc, struct iatt *ia,
-                        call_frame_t *frame, xlator_t *this, unsigned char *on)
-{
-        dict_t              *xattr = NULL;
-        int32_t             ret    = -1;
-        default_args_cbk_t  *replies = NULL;
-        unsigned char       *output  = NULL;
-        uint64_t            dirty[EC_VERSION_SIZE] = {1, 1};
-        loc_t               newloc = {0};
-
-        /*Symlinks don't have any data to be healed*/
-        if (ia->ia_type == IA_IFLNK)
-                dirty[EC_DATA_TXN] = 0;
-
-        newloc.inode = inode_ref (loc->inode);
-        gf_uuid_copy (newloc.gfid, ia->ia_gfid);
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output = alloca0 (ec->nodes);
-        xattr = dict_new();
-        if (!xattr) {
-                ret = -ENOMEM;
-                goto out;
-        }
+ec_set_new_entry_dirty(ec_t *ec, loc_t *loc, struct iatt *ia,
+                       call_frame_t *frame, xlator_t *this, unsigned char *on)
+{
+    dict_t *xattr = NULL;
+    int32_t ret = -1;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    uint64_t dirty[EC_VERSION_SIZE] = {1, 1};
+    loc_t newloc = {0};
+
+    /*Symlinks don't have any data to be healed*/
+    if (ia->ia_type == IA_IFLNK)
+        dirty[EC_DATA_TXN] = 0;
+
+    newloc.inode = inode_ref(loc->inode);
+    gf_uuid_copy(newloc.gfid, ia->ia_gfid);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = ec_dict_set_array (xattr, EC_XATTR_DIRTY, dirty,
-                                 EC_VERSION_SIZE);
-        if (ret)
-                goto out;
+    ret = ec_dict_set_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+    if (ret)
+        goto out;
 
-        ret = cluster_xattrop (ec->xl_list, on, ec->nodes, replies, output,
-                               frame, ec->xl, &newloc,
-                               GF_XATTROP_ADD_ARRAY64, xattr, NULL);
+    ret = cluster_xattrop(ec->xl_list, on, ec->nodes, replies, output, frame,
+                          ec->xl, &newloc, GF_XATTROP_ADD_ARRAY64, xattr, NULL);
+
+    if (ret < ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        if (ret < ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
 out:
-        if (xattr)
-                dict_unref (xattr);
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&newloc);
-        return ret;
+    if (xattr)
+        dict_unref(xattr);
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&newloc);
+    return ret;
 }
 
 /*Name heal*/
 int
-ec_delete_stale_name (dict_t *gfid_db, char *key, data_t *d, void *data)
-{
-        struct ec_name_data *name_data   = data;
-        struct iatt         *ia          = NULL;
-        ec_t                *ec          = NULL;
-        loc_t               loc          = {0};
-        unsigned char       *same        = data_to_bin (d);
-        default_args_cbk_t  *replies     = NULL;
-        unsigned char       *output      = NULL;
-        int                 ret          = 0;
-        int                 estale_count = 0;
-        int                 i            = 0;
-        call_frame_t        *frame       = name_data->frame;
-
-        ec = name_data->frame->this->private;
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        if (EC_COUNT (same, ec->nodes) >= ec->fragments) {
-                ret = 0;
-                goto out;
-        }
+ec_delete_stale_name(dict_t *gfid_db, char *key, data_t *d, void *data)
+{
+    struct ec_name_data *name_data = data;
+    struct iatt *ia = NULL;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    unsigned char *same = data_to_bin(d);
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    int estale_count = 0;
+    int i = 0;
+    call_frame_t *frame = name_data->frame;
+    uuid_t gfid;
+
+    ec = name_data->frame->this->private;
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    if (EC_COUNT(same, ec->nodes) >= ec->fragments) {
+        ret = 0;
+        goto out;
+    }
 
-        loc.inode = inode_new (name_data->parent->table);
-        if (!loc.inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        gf_uuid_parse (key, loc.gfid);
-        output = alloca0(ec->nodes);
-        ret = cluster_lookup (ec->xl_list, name_data->participants, ec->nodes,
-                              replies, output, name_data->frame, ec->xl, &loc,
-                              NULL);
+    loc.parent = inode_ref(name_data->parent);
+    loc.inode = inode_new(name_data->parent->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-                if (replies[i].op_ret == -1) {
-                        if (replies[i].op_errno == ESTALE ||
-                            replies[i].op_errno == ENOENT)
-                                estale_count++;
-                        else
-                                name_data->participants[i] = 0;
-                }
+    gf_uuid_parse(key, gfid);
+    gf_uuid_copy(loc.pargfid, name_data->parent->gfid);
+    loc.name = name_data->name;
+    output = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, name_data->participants, ec->nodes,
+                         replies, output, name_data->frame, ec->xl, &loc, NULL);
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1) {
+            if (replies[i].op_errno == ESTALE || replies[i].op_errno == ENOENT)
+                estale_count++;
+            else
+                name_data->participants[i] = 0;
+        } else if (gf_uuid_compare(gfid, replies[i].stat.ia_gfid)) {
+            estale_count++;
+            gf_msg_debug(ec->xl->name, 0, "%s/%s: different gfid as %s",
+                         uuid_utoa(name_data->parent->gfid), name_data->name,
+                         key);
         }
+    }
 
-        if (estale_count <= ec->redundancy) {
-                /* We have at least ec->fragments number of fragments, so the
-                 * file is recoverable, so don't delete it*/
+    if (estale_count <= ec->redundancy) {
+        /* We have at least ec->fragments number of fragments, so the
+         * file is recoverable, so don't delete it*/
 
-                /* Please note that the lookup call above could fail with
-                 * ENOTCONN on all subvoumes and still this branch will be
-                 * true, but in those cases conservatively we decide to not
-                 * delete the file until we are sure*/
-                ret = 0;
-                goto out;
-        }
+        /* Please note that the lookup call above could fail with
+         * ENOTCONN on all subvoumes and still this branch will be
+         * true, but in those cases conservatively we decide to not
+         * delete the file until we are sure*/
+        ret = 0;
+        goto out;
+    }
 
-        /*Noway to recover, delete the name*/
-        loc_wipe (&loc);
-        loc.parent = inode_ref (name_data->parent);
-        gf_uuid_copy (loc.pargfid, loc.parent->gfid);
-        loc.name = name_data->name;
-        for (i = 0; i < ec->nodes; i++) {
-                if (same[i] && replies[i].valid && (replies[i].op_ret == 0)) {
-                        ia = &replies[i].stat;
-                        break;
-                }
+    /*Noway to recover, delete the name*/
+    loc_wipe(&loc);
+    loc.parent = inode_ref(name_data->parent);
+    gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+    loc.name = name_data->name;
+    for (i = 0; i < ec->nodes; i++) {
+        if (same[i] && replies[i].valid && (replies[i].op_ret == 0)) {
+            ia = &replies[i].stat;
+            break;
         }
+    }
 
-        if (!ia) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    if (!ia) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        if (IA_ISDIR (ia->ia_type)) {
-                ret = cluster_rmdir (ec->xl_list, same, ec->nodes, replies,
-                                     output, frame, ec->xl, &loc, 1, NULL);
-        } else {
-                ret = cluster_unlink (ec->xl_list, same, ec->nodes, replies,
-                                      output, frame, ec->xl, &loc, 0, NULL);
-        }
+    if (IA_ISDIR(ia->ia_type)) {
+        ret = cluster_rmdir(ec->xl_list, same, ec->nodes, replies, output,
+                            frame, ec->xl, &loc, 1, NULL);
+        gf_msg_debug(ec->xl->name, 0,
+                     "cluster rmdir succeeded on %d "
+                     "nodes",
+                     ret);
+    } else {
+        ret = cluster_unlink(ec->xl_list, same, ec->nodes, replies, output,
+                             frame, ec->xl, &loc, 0, NULL);
+        gf_msg_debug(ec->xl->name, 0,
+                     "cluster unlink succeeded on %d "
+                     "nodes",
+                     ret);
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (output[i]) {
-                        same[i] = 0;
-                        name_data->enoent[i] = 1;
-                } else {
-                        /*op failed*/
-                        if (same[i])
-                                name_data->participants[i] = 0;
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (output[i]) {
+            same[i] = 0;
+            name_data->enoent[i] = 1;
+        } else {
+            /*op failed*/
+            if (same[i])
+                name_data->participants[i] = 0;
         }
-        ret = 0;
-        /*This will help in making decisions about creating names*/
-        dict_del (gfid_db, key);
+    }
+    ret = 0;
+    /*This will help in making decisions about creating names*/
+    dict_del(gfid_db, key);
 out:
-        if (ret < 0) {
-                gf_msg_debug (ec->xl->name, 0, "%s/%s: heal failed %s",
-                        uuid_utoa (name_data->parent->gfid), name_data->name,
-                        strerror (-ret));
-        }
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s",
+                     uuid_utoa(name_data->parent->gfid), name_data->name,
+                     strerror(-ret));
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-ec_delete_stale_names (call_frame_t *frame, ec_t *ec, inode_t *parent,
-                       char *name, default_args_cbk_t *replies, dict_t *gfid_db,
-                       unsigned char *enoent, unsigned char *gfidless,
-                       unsigned char *participants)
+ec_delete_stale_names(call_frame_t *frame, ec_t *ec, inode_t *parent,
+                      char *name, default_args_cbk_t *replies, dict_t *gfid_db,
+                      unsigned char *enoent, unsigned char *gfidless,
+                      unsigned char *participants)
 {
-        struct ec_name_data name_data = {0};
-
-        name_data.enoent = enoent;
-        name_data.gfidless = gfidless;
-        name_data.participants = participants;
-        name_data.name = name;
-        name_data.parent = parent;
-        name_data.frame = frame;
-        name_data.replies = replies;
-        return dict_foreach (gfid_db, ec_delete_stale_name, &name_data);
+    struct ec_name_data name_data = {0};
+
+    name_data.enoent = enoent;
+    name_data.gfidless = gfidless;
+    name_data.participants = participants;
+    name_data.name = name;
+    name_data.parent = parent;
+    name_data.frame = frame;
+    name_data.replies = replies;
+    return dict_foreach(gfid_db, ec_delete_stale_name, &name_data);
 }
 
 int
-_assign_same (dict_t *dict, char *key, data_t *value, void *data)
+_assign_same(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct ec_name_data *name_data = data;
+    struct ec_name_data *name_data = data;
 
-        name_data->same = data_to_bin (value);
-        return 0;
+    name_data->same = data_to_bin(value);
+    return 0;
 }
 
 int
-ec_create_name (call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
-                default_args_cbk_t *lookup_replies, dict_t *gfid_db,
-                unsigned char *enoent, unsigned char *participants)
-{
-        int                 ret       = 0;
-        int                 i         = 0;
-        struct ec_name_data name_data = {0};
-        struct iatt         *ia       = NULL;
-        unsigned char       *output   = 0;
-        unsigned char       *output1  = 0;
-        unsigned char       *on       = NULL;
-        default_args_cbk_t  *replies  = NULL;
-        loc_t               loc       = {0};
-        loc_t               srcloc    = {0};
-        unsigned char       *link     = NULL;
-        unsigned char       *create   = NULL;
-        dict_t              *xdata    = NULL;
-        char                *linkname = NULL;
-        ec_config_t         config;
-        /* There should be just one gfid key */
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        if (gfid_db->count != 1) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        ret = dict_foreach (gfid_db, _assign_same, &name_data);
-        if (ret < 0)
-                goto out;
-        /*There should at least be one valid success reply with gfid*/
-        for (i = 0; i < ec->nodes; i++)
-                if (name_data.same[i])
-                        break;
+ec_create_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+               default_args_cbk_t *lookup_replies, dict_t *gfid_db,
+               unsigned char *enoent, unsigned char *participants)
+{
+    int ret = 0;
+    int i = 0;
+    struct ec_name_data name_data = {0};
+    struct iatt *ia = NULL;
+    unsigned char *output = 0;
+    unsigned char *output1 = 0;
+    unsigned char *on = NULL;
+    default_args_cbk_t *replies = NULL;
+    loc_t loc = {0};
+    loc_t srcloc = {0};
+    unsigned char *link = NULL;
+    unsigned char *create = NULL;
+    dict_t *xdata = NULL;
+    char *linkname = NULL;
+    ec_config_t config;
+
+    /* There should be just one gfid key */
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    if (gfid_db->count != 1) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-        if (i == ec->nodes) {
-                ret = -EINVAL;
-                goto out;
-        }
+    ret = dict_foreach(gfid_db, _assign_same, &name_data);
+    if (ret < 0)
+        goto out;
+    /*There should at least be one valid success reply with gfid*/
+    for (i = 0; i < ec->nodes; i++)
+        if (name_data.same[i])
+            break;
 
-        ia = &lookup_replies[i].stat;
-        xdata = dict_new ();
-        loc.parent = inode_ref (parent);
-        gf_uuid_copy (loc.pargfid, parent->gfid);
-        loc.inode = inode_new (parent->table);
-        if (loc.inode)
-                srcloc.inode = inode_ref (loc.inode);
-        gf_uuid_copy (srcloc.gfid, ia->ia_gfid);
-        if (!loc.inode || !xdata || dict_set_static_bin (xdata, "gfid-req",
-                                                         ia->ia_gfid,
-                                                        sizeof (ia->ia_gfid))) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        loc.name = name;
-        link = alloca0 (ec->nodes);
-        create = alloca0 (ec->nodes);
-        on = alloca0 (ec->nodes);
-        output = alloca0 (ec->nodes);
-        output1 = alloca0 (ec->nodes);
+    if (i == ec->nodes) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!lookup_replies[i].valid)
-                        continue;
-                if (lookup_replies[i].op_ret)
-                        continue;
-                on[i] = 1;
-        }
-        switch (ia->ia_type) {
+    ia = &lookup_replies[i].stat;
+    xdata = dict_new();
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.inode = inode_new(parent->table);
+    if (loc.inode)
+        srcloc.inode = inode_ref(loc.inode);
+    gf_uuid_copy(srcloc.gfid, ia->ia_gfid);
+    if (!loc.inode || !xdata ||
+        dict_set_static_bin(xdata, "gfid-req", ia->ia_gfid,
+                            sizeof(ia->ia_gfid))) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    loc.name = name;
+    link = alloca0(ec->nodes);
+    create = alloca0(ec->nodes);
+    on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    output1 = alloca0(ec->nodes);
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!lookup_replies[i].valid)
+            continue;
+        if (lookup_replies[i].op_ret)
+            continue;
+        on[i] = 1;
+    }
+    switch (ia->ia_type) {
         case IA_IFDIR:
-                ec_set_new_entry_dirty (ec, &loc, ia, frame, ec->xl, on);
-                ret = cluster_mkdir (ec->xl_list, enoent, ec->nodes,
-                                   replies, output, frame, ec->xl, &loc,
-                                   st_mode_from_ia (ia->ia_prot,
-                                                ia->ia_type), 0, xdata);
-                break;
+            ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+            (void)cluster_mkdir(
+                ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl,
+                &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type), 0, xdata);
+            break;
 
         case IA_IFLNK:
-                /*Check for hard links and create/link*/
-                ret = cluster_lookup (ec->xl_list, enoent, ec->nodes,
-                                      replies, output, frame, ec->xl,
-                                      &srcloc, NULL);
-                for (i = 0; i < ec->nodes; i++) {
-                        if (output[i]) {
-                                link[i] = 1;
-                        } else {
-                                if (replies[i].op_errno == ENOENT ||
-                                    replies[i].op_errno == ESTALE) {
-                                        create[i] = 1;
-                                }
-                        }
+            /*Check for hard links and create/link*/
+            ret = cluster_lookup(ec->xl_list, enoent, ec->nodes, replies,
+                                 output, frame, ec->xl, &srcloc, NULL);
+            for (i = 0; i < ec->nodes; i++) {
+                if (output[i]) {
+                    link[i] = 1;
+                } else {
+                    if (replies[i].op_errno == ENOENT ||
+                        replies[i].op_errno == ESTALE) {
+                        create[i] = 1;
+                    }
                 }
-
-                if (EC_COUNT (link, ec->nodes)) {
-                        cluster_link (ec->xl_list, link, ec->nodes,
-                                      replies, output1, frame, ec->xl,
-                                      &srcloc, &loc, NULL);
+            }
+
+            if (EC_COUNT(link, ec->nodes)) {
+                cluster_link(ec->xl_list, link, ec->nodes, replies, output1,
+                             frame, ec->xl, &srcloc, &loc, NULL);
+            }
+
+            if (EC_COUNT(create, ec->nodes)) {
+                cluster_readlink(ec->xl_list, name_data.same, ec->nodes,
+                                 replies, output, frame, ec->xl, &srcloc, 4096,
+                                 NULL);
+                if (EC_COUNT(output, ec->nodes) == 0) {
+                    ret = -ENOTCONN;
+                    goto out;
                 }
 
-                if (EC_COUNT (create, ec->nodes)) {
-                        cluster_readlink (ec->xl_list, name_data.same,
-                                          ec->nodes, replies, output,
-                                          frame, ec->xl, &srcloc, 4096,
-                                          NULL);
-                        if (EC_COUNT (output, ec->nodes) == 0) {
-                                ret = -ENOTCONN;
-                                goto out;
-                        }
-
-                        for (i = 0; i < ec->nodes; i++) {
-                                if (output[i])
-                                        break;
-                        }
-                        linkname = alloca0 (strlen(replies[i].buf) + 1);
-                        strcpy (linkname, replies[i].buf);
-                        ec_set_new_entry_dirty (ec, &loc, ia, frame,
-                                                ec->xl, on);
-                        cluster_symlink (ec->xl_list, create, ec->nodes,
-                                         replies, output, frame, ec->xl,
-                                         linkname, &loc, 0, xdata);
+                for (i = 0; i < ec->nodes; i++) {
+                    if (output[i])
+                        break;
                 }
-                for (i = 0; i < ec->nodes; i++)
-                        if (output1[i])
-                                output[i] = 1;
-                break;
+                linkname = alloca0(strlen(replies[i].buf) + 1);
+                strcpy(linkname, replies[i].buf);
+                ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+                cluster_symlink(ec->xl_list, create, ec->nodes, replies, output,
+                                frame, ec->xl, linkname, &loc, 0, xdata);
+            }
+            for (i = 0; i < ec->nodes; i++)
+                if (output1[i])
+                    output[i] = 1;
+            break;
         case IA_IFREG:
-                ec_set_new_entry_dirty (ec, &loc, ia,
-                                        frame, ec->xl, on);
-                config.version = EC_CONFIG_VERSION;
-                config.algorithm = EC_CONFIG_ALGORITHM;
-                config.gf_word_size = EC_GF_BITS;
-                config.bricks = ec->nodes;
-                config.redundancy = ec->redundancy;
-                config.chunk_size = EC_METHOD_CHUNK_SIZE;
-
-                ret = ec_dict_set_config(xdata, EC_XATTR_CONFIG, &config);
-                if (ret != 0) {
-                        goto out;
-                }
+            ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+            config.version = EC_CONFIG_VERSION;
+            config.algorithm = EC_CONFIG_ALGORITHM;
+            config.gf_word_size = EC_GF_BITS;
+            config.bricks = ec->nodes;
+            config.redundancy = ec->redundancy;
+            config.chunk_size = EC_METHOD_CHUNK_SIZE;
+
+            ret = ec_dict_set_config(xdata, EC_XATTR_CONFIG, &config);
+            if (ret != 0) {
+                goto out;
+            }
+
+            /* Fall through */
+
         default:
-                ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY,
-                                      1);
-                if (ret)
-                        goto out;
-                ret = cluster_mknod (ec->xl_list, enoent, ec->nodes,
-                                     replies, output, frame, ec->xl,
-                                     &loc, st_mode_from_ia (ia->ia_prot,
-                                                           ia->ia_type),
-                                     ia->ia_rdev, 0, xdata);
-                break;
-        }
+            ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+            if (ret)
+                goto out;
+            ret = cluster_mknod(
+                ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl,
+                &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type),
+                makedev(ia_major(ia->ia_rdev), ia_minor(ia->ia_rdev)), 0,
+                xdata);
+            break;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (enoent[i] && !output[i])
-                        participants[i] = 0;
-        }
+    for (i = 0; i < ec->nodes; i++) {
+        if (enoent[i] && !output[i])
+            participants[i] = 0;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s/%s: heal failed %s",
-                        uuid_utoa (parent->gfid), name, strerror (-ret));
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        loc_wipe (&srcloc);
-        if (xdata)
-                dict_unref (xdata);
-        return ret;
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s",
+                     uuid_utoa(parent->gfid), name, strerror(-ret));
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    loc_wipe(&srcloc);
+    if (xdata)
+        dict_unref(xdata);
+    return ret;
 }
 
 int
-__ec_heal_name (call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
-                unsigned char *participants)
-{
-        unsigned char      *output   = NULL;
-        unsigned char      *enoent   = NULL;
-        default_args_cbk_t *replies  = NULL;
-        dict_t             *xdata    = NULL;
-        dict_t             *gfid_db  = NULL;
-        int                ret       = 0;
-        loc_t              loc       = {0};
-        int                i         = 0;
-        struct iatt        *ia       = NULL;
-        char               gfid[64]  = {0};
-        unsigned char      *same     = NULL;
-        unsigned char      *gfidless = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        loc.parent = inode_ref (parent);
-        loc.inode = inode_new (parent->table);
-        gf_uuid_copy (loc.pargfid, parent->gfid);
-        loc.name = name;
-        xdata = dict_new ();
-        gfid_db = dict_new ();
-        if (!xdata || !gfid_db || !loc.inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+               unsigned char *participants)
+{
+    unsigned char *output = NULL;
+    unsigned char *enoent = NULL;
+    default_args_cbk_t *replies = NULL;
+    dict_t *xdata = NULL;
+    dict_t *gfid_db = NULL;
+    int ret = 0;
+    loc_t loc = {0};
+    int i = 0;
+    struct iatt *ia = NULL;
+    char gfid[64] = {0};
+    unsigned char *same = NULL;
+    unsigned char *gfidless = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    loc.parent = inode_ref(parent);
+    loc.inode = inode_new(parent->table);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    xdata = dict_new();
+    gfid_db = dict_new();
+    if (!xdata || !gfid_db || !loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = dict_set_int32 (xdata, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    ret = dict_set_int32(xdata, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        output = alloca0 (ec->nodes);
-        gfidless = alloca0 (ec->nodes);
-        enoent = alloca0 (ec->nodes);
-        ret = cluster_lookup (ec->xl_list, participants, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret == -1) {
-                        /*If ESTALE comes here, that means parent dir is not
-                         * present, nothing to do there, so reset participants
-                         * for that brick*/
-                        if (replies[i].op_errno == ENOENT)
-                                enoent[i] = 1;
-                        else
-                                participants[i] = 0;
-                        continue;
-                }
-                ia = &replies[i].stat;
-                if (gf_uuid_is_null (ia->ia_gfid)) {
-                        if (IA_ISDIR (ia->ia_type) || ia->ia_size == 0)
-                                gfidless[i] = 1;
-                        else
-                                participants[i] = 0;
-                } else {
-                        uuid_utoa_r (ia->ia_gfid, gfid);
-                        ret = dict_get_bin (gfid_db, gfid, (void **)&same);
-                        if (ret < 0) {
-                                same = alloca0(ec->nodes);
-                        }
-                        same[i] = 1;
-                        if (ret < 0) {
-                                ret = dict_set_static_bin (gfid_db, gfid, same,
-                                                           ec->nodes);
-                        }
-                        if (ret < 0)
-                                goto out;
-                }
+    output = alloca0(ec->nodes);
+    gfidless = alloca0(ec->nodes);
+    enoent = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, participants, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == -1) {
+            /*If ESTALE comes here, that means parent dir is not
+             * present, nothing to do there, so reset participants
+             * for that brick*/
+            if (replies[i].op_errno == ENOENT)
+                enoent[i] = 1;
+            else
+                participants[i] = 0;
+            continue;
+        }
+        ia = &replies[i].stat;
+        if (gf_uuid_is_null(ia->ia_gfid)) {
+            if (IA_ISDIR(ia->ia_type) || ia->ia_size == 0)
+                gfidless[i] = 1;
+            else
+                participants[i] = 0;
+        } else {
+            uuid_utoa_r(ia->ia_gfid, gfid);
+            ret = dict_get_bin(gfid_db, gfid, (void **)&same);
+            if (ret < 0) {
+                same = alloca0(ec->nodes);
+            }
+            same[i] = 1;
+            if (ret < 0) {
+                ret = dict_set_static_bin(gfid_db, gfid, same, ec->nodes);
+            }
+            if (ret < 0)
+                goto out;
         }
+    }
 
-        ret = ec_delete_stale_names (frame, ec, parent, name, replies, gfid_db,
-                                     enoent, gfidless, participants);
+    ret = ec_delete_stale_names(frame, ec, parent, name, replies, gfid_db,
+                                enoent, gfidless, participants);
 
-        if (gfid_db->count == 0) {
-                /* All entries seem to be stale entries and deleted,
-                 * nothing more to do.*/
-                goto out;
-        }
+    if (gfid_db->count == 0) {
+        /* All entries seem to be stale entries and deleted,
+         * nothing more to do.*/
+        goto out;
+    }
 
-        if (gfid_db->count > 1) {
-                gf_msg (ec->xl->name, GF_LOG_INFO, 0,
-                        EC_MSG_HEAL_FAIL, "%s/%s: Not able to heal",
-                        uuid_utoa (parent->gfid), name);
-                memset (participants, 0, ec->nodes);
-                goto out;
-        }
+    if (gfid_db->count > 1) {
+        gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL,
+               "%s/%s: Not able to heal", uuid_utoa(parent->gfid), name);
+        memset(participants, 0, ec->nodes);
+        goto out;
+    }
 
-        EC_INTERSECT (enoent, enoent, participants, ec->nodes);
-        if (EC_COUNT (enoent, ec->nodes) == 0) {
-                ret = 0;
-                goto out;
-        }
+    EC_INTERSECT(enoent, enoent, participants, ec->nodes);
+    if (EC_COUNT(enoent, ec->nodes) == 0) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = ec_create_name (frame, ec, parent, name, replies, gfid_db, enoent,
-                              participants);
+    ret = ec_create_name(frame, ec, parent, name, replies, gfid_db, enoent,
+                         participants);
+    if (ret >= 0) {
+        /* If ec_create_name() succeeded we return 1 to indicate that a new
+         * file has been created and it will need to be healed. */
+        ret = 1;
+    }
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        if (xdata)
-                dict_unref (xdata);
-        if (gfid_db)
-                dict_unref (gfid_db);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    if (xdata)
+        dict_unref(xdata);
+    if (gfid_db)
+        dict_unref(gfid_db);
+    return ret;
 }
 
 int
-ec_heal_name (call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
-              unsigned char *participants)
-{
-        int                ret        = 0;
-        default_args_cbk_t *replies   = NULL;
-        unsigned char      *output    = NULL;
-        unsigned char      *locked_on = NULL;
-        loc_t              loc        = {0};
-
-        loc.parent = inode_ref (parent);
-        loc.name = name;
-        loc.inode = inode_new (parent->table);
-        if (!loc.inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+             unsigned char *participants)
+{
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    unsigned char *locked_on = NULL;
+    loc_t loc = {0};
+
+    loc.parent = inode_ref(parent);
+    loc.name = name;
+    loc.inode = inode_new(parent->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output = alloca0 (ec->nodes);
-        locked_on = alloca0 (ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, participants, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name, parent,
-                               0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s/%s: Skipping "
-                                "heal as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (parent->gfid), name,
-                                ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                EC_INTERSECT (participants, participants, locked_on, ec->nodes);
-                ret = __ec_heal_name (frame, ec, parent, name, participants);
-        }
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, parent, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s/%s: Skipping "
+                         "heal as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(parent->gfid), name, ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        EC_INTERSECT(participants, participants, locked_on, ec->nodes);
+        ret = __ec_heal_name(frame, ec, parent, name, participants);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, parent, 0, 0);
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, parent, 0, 0);
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-ec_name_heal_handler (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                      void *data)
+ec_name_heal_handler(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                     void *data)
 {
-        struct ec_name_data *name_data = data;
-        xlator_t            *this      = THIS;
-        ec_t                *ec        = this->private;
-        unsigned char       *name_on   = alloca0 (ec->nodes);
-        int                 i          = 0;
-        int                 ret        = 0;
+    struct ec_name_data *name_data = data;
+    xlator_t *this = THIS;
+    ec_t *ec = this->private;
+    unsigned char *name_on = alloca0(ec->nodes);
+    int i = 0;
+    int ret = 0;
+
+    if (ec->shutdown) {
+        gf_msg_debug(this->name, 0,
+                     "Cancelling directory heal "
+                     "because EC is stopping.");
+        return -ENOTCONN;
+    }
 
-        memcpy (name_on, name_data->participants, ec->nodes);
-        ret = ec_heal_name (name_data->frame, ec, parent->inode,
-                            entry->d_name, name_on);
+    memcpy(name_on, name_data->participants, ec->nodes);
+    ret = ec_heal_name(name_data->frame, ec, parent->inode, entry->d_name,
+                       name_on);
 
-        if (ret < 0)
-                memset (name_on, 0, ec->nodes);
+    if (ret < 0) {
+        memset(name_on, 0, ec->nodes);
+    } else {
+        name_data->heal_pending += ret;
+    }
 
-        for (i = 0; i < ec->nodes; i++)
-                if (name_data->participants[i] && !name_on[i])
-                        name_data->failed_on[i] = 1;
-        return 0;
+    for (i = 0; i < ec->nodes; i++)
+        if (name_data->participants[i] && !name_on[i])
+            name_data->failed_on[i] = 1;
+
+    return 0;
 }
 
 int
-ec_heal_names (call_frame_t *frame, ec_t *ec, inode_t *inode,
-               unsigned char *participants)
+ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode,
+              unsigned char *participants, uint32_t *pending)
 {
-        int i = 0;
-        int j = 0;
-        loc_t loc = {0};
-        struct ec_name_data name_data = {0};
+    int i = 0;
+    int j = 0;
+    loc_t loc = {0};
+    struct ec_name_data name_data = {0};
+    int ret = 0;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    name_data.frame = frame;
+    name_data.participants = participants;
+    name_data.failed_on = alloca0(ec->nodes);
+    name_data.heal_pending = 0;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!participants[i])
+            continue;
+        ret = syncop_dir_scan(ec->xl_list[i], &loc, GF_CLIENT_PID_SELF_HEALD,
+                              &name_data, ec_name_heal_handler);
+        if (ret < 0) {
+            break;
+        }
+        for (j = 0; j < ec->nodes; j++)
+            if (name_data.failed_on[j])
+                participants[j] = 0;
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        name_data.frame = frame;
-        name_data.participants = participants;
-        name_data.failed_on = alloca0(ec->nodes);;
+        if (EC_COUNT(participants, ec->nodes) <= ec->fragments) {
+            ret = -ENOTCONN;
+            break;
+        }
+    }
+    *pending += name_data.heal_pending;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!participants[i])
-                        continue;
-                syncop_dir_scan (ec->xl_list[i], &loc,
-                                GF_CLIENT_PID_SELF_HEALD, &name_data,
-                                ec_name_heal_handler);
-                for (j = 0; j < ec->nodes; j++)
-                        if (name_data.failed_on[j])
-                                participants[j] = 0;
-
-                if (EC_COUNT (participants, ec->nodes) <= ec->fragments)
-                        return -ENOTCONN;
-        }
-        loc_wipe (&loc);
-        return 0;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-__ec_heal_entry (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                 unsigned char *heal_on, unsigned char *sources,
-                 unsigned char *healed_sinks)
-{
-        unsigned char      *locked_on    = NULL;
-        unsigned char      *output       = NULL;
-        uint64_t           *versions     = NULL;
-        uint64_t           *dirty        = NULL;
-        unsigned char      *participants = NULL;
-        default_args_cbk_t *replies      = NULL;
-        int                ret           = 0;
-        int                source        = 0;
-        int                i             = 0;
-
-        locked_on = alloca0(ec->nodes);
-        output = alloca0(ec->nodes);
-        versions = alloca0 (ec->nodes * sizeof (*versions));
-        dirty = alloca0 (ec->nodes * sizeof (*dirty));
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, heal_on, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name, inode,
-                               0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_entry_prepare (frame, ec, inode, locked_on,
-                                               versions, dirty, sources,
-                                               healed_sinks);
-                source = ret;
-        }
+__ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *heal_on, unsigned char *sources,
+                unsigned char *healed_sinks, uint32_t *pending)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *output = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    unsigned char *participants = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+    int source = 0;
+    int i = 0;
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on,
+                          frame, ec->xl, ec->xl->name, inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_entry_prepare(frame, ec, inode, locked_on, versions,
+                                      dirty, sources, healed_sinks);
+        source = ret;
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, inode, 0, 0);
-        if (ret < 0)
-                goto out;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    if (ret < 0)
+        goto out;
 
-        participants = alloca0 (ec->nodes);
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i] || healed_sinks[i])
-                        participants[i] = 1;
-        }
-        ret = ec_heal_names (frame, ec, inode, participants);
+    participants = alloca0(ec->nodes);
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i] || healed_sinks[i])
+            participants[i] = 1;
+    }
+    ret = ec_heal_names(frame, ec, inode, participants, pending);
 
-        if (EC_COUNT (participants, ec->nodes) <= ec->fragments)
-                goto out;
+    if (EC_COUNT(participants, ec->nodes) <= ec->fragments)
+        goto out;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!participants[i]) {
-                        sources[i] = 0;
-                        healed_sinks[i] = 0;
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (!participants[i]) {
+            sources[i] = 0;
+            healed_sinks[i] = 0;
         }
+    }
 
-        ec_adjust_versions (frame, ec, EC_DATA_TXN, inode, source,
-                            sources, healed_sinks, versions, dirty);
+    ec_adjust_versions(frame, ec, EC_DATA_TXN, inode, source, sources,
+                       healed_sinks, versions, dirty);
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 int
-ec_heal_entry (call_frame_t *frame, ec_t *ec, inode_t *inode,
-               unsigned char *sources, unsigned char *healed_sinks)
+ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode,
+              unsigned char *sources, unsigned char *healed_sinks,
+              uint32_t *pending)
 {
-        unsigned char      *locked_on            = NULL;
-        unsigned char      *up_subvols           = NULL;
-        unsigned char      *output               = NULL;
-        char               selfheal_domain[1024] = {0};
-        int                ret                   = 0;
-        default_args_cbk_t *replies              = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        locked_on = alloca0(ec->nodes);
-        output = alloca0(ec->nodes);
-        up_subvols = alloca0(ec->nodes);
-
-        sprintf (selfheal_domain, "%s:self-heal", ec->xl->name);
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
-        /*If other processes are already doing the heal, don't block*/
-        ret = cluster_inodelk (ec->xl_list, up_subvols, ec->nodes, replies,
-                               locked_on, frame, ec->xl, selfheal_domain, inode,
-                               0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_entry (frame, ec, inode, locked_on,
-                                       sources, healed_sinks);
-        }
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    char selfheal_domain[1024] = {0};
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+
+    sprintf(selfheal_domain, "%s:self-heal", ec->xl->name);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+    /*If other processes are already doing the heal, don't block*/
+    ret = cluster_tiebreaker_inodelk(ec->xl_list, up_subvols, ec->nodes,
+                                     replies, locked_on, frame, ec->xl,
+                                     selfheal_domain, inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_entry(frame, ec, inode, locked_on, sources,
+                              healed_sinks, pending);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, selfheal_domain, inode, 0, 0);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, selfheal_domain, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
-/*Data heal*/
+/*Find direction for data heal and heal info*/
 int
-ec_heal_data_find_direction (ec_t *ec, default_args_cbk_t *replies,
-                             uint64_t *versions, uint64_t *dirty,
-                             uint64_t *size, unsigned char *sources,
-                             unsigned char *healed_sinks)
+ec_heal_data_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                            uint64_t *data_versions, uint64_t *dirty,
+                            uint64_t *size, unsigned char *sources,
+                            unsigned char *healed_sinks,
+                            gf_boolean_t check_ondisksize, int which)
 {
-        uint64_t        xattr[EC_VERSION_SIZE] = {0};
-        char            version_size[64] = {0};
-        dict_t          *version_size_db = NULL;
-        unsigned char   *same            = NULL;
-        int             max_same_count   = 0;
-        int             source           = 0;
-        int             i                = 0;
-        int             ret              = 0;
-
-        version_size_db = dict_new ();
-        if (!version_size_db) {
-                ret = -ENOMEM;
-                goto out;
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    char version_size[128] = {0};
+    dict_t *version_size_db = NULL;
+    unsigned char *same = NULL;
+    int max_same_count = 0;
+    int source = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *dict = NULL;
+    uint64_t source_size = 0;
+
+    version_size_db = dict_new();
+    if (!version_size_db) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret < 0)
+            continue;
+        dict = (which == EC_COMBINE_XDATA) ? replies[i].xdata
+                                           : replies[i].xattr;
+
+        ret = ec_dict_get_array(dict, EC_XATTR_VERSION, xattr, EC_VERSION_SIZE);
+        if (ret == 0) {
+            data_versions[i] = xattr[EC_DATA_TXN];
         }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-                if (replies[i].op_ret < 0)
-                        continue;
-                ret = ec_dict_del_array (replies[i].xattr, EC_XATTR_VERSION,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        versions[i] = xattr[EC_DATA_TXN];
-                }
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(dict, EC_XATTR_DIRTY, xattr, EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_DATA_TXN];
+        }
+        ret = ec_dict_del_number(dict, EC_XATTR_SIZE, &size[i]);
+        /*Build a db of same metadata and data version and size*/
+        snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64,
+                 data_versions[i], size[i]);
 
-                memset (xattr, 0, sizeof (xattr));
-                ret = ec_dict_del_array (replies[i].xattr, EC_XATTR_DIRTY,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        dirty[i] = xattr[EC_DATA_TXN];
-                }
-                ret = ec_dict_del_number (replies[i].xattr, EC_XATTR_SIZE,
-                                          &size[i]);
-                /*Build a db of same version, size*/
-                snprintf (version_size, sizeof (version_size),
-                          "%"PRIu64"-%"PRIu64, versions[i], size[i]);
-                ret = dict_get_bin (version_size_db, version_size,
-                                    (void **)&same);
-                if (ret < 0) {
-                        same = alloca0 (ec->nodes);
-                }
+        ret = dict_get_bin(version_size_db, version_size, (void **)&same);
+        if (ret < 0) {
+            same = alloca0(ec->nodes);
+        }
 
-                same[i] = 1;
-                if (max_same_count < EC_COUNT (same, ec->nodes)) {
-                        max_same_count = EC_COUNT (same, ec->nodes);
-                        source = i;
-                }
+        same[i] = 1;
+        if (max_same_count < EC_COUNT(same, ec->nodes)) {
+            max_same_count = EC_COUNT(same, ec->nodes);
+            source = i;
+        }
 
-                if (ret < 0) {
-                        ret = dict_set_static_bin (version_size_db,
-                                                version_size, same, ec->nodes);
-                }
+        if (ret < 0) {
+            ret = dict_set_static_bin(version_size_db, version_size, same,
+                                      ec->nodes);
+        }
+
+        if (ret < 0) {
+            ret = -ENOMEM;
+            goto out;
+        }
+    }
+    /* If we don't have ec->fragments number of same version,size it is not
+     * recoverable*/
+    if (max_same_count < ec->fragments) {
+        ret = -EIO;
+        goto out;
+    } else {
+        snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64,
+                 data_versions[source], size[source]);
 
-                if (ret < 0) {
-                        ret = -ENOMEM;
-                        goto out;
+        ret = dict_get_bin(version_size_db, version_size, (void **)&same);
+        if (ret < 0)
+            goto out;
+        memcpy(sources, same, ec->nodes);
+        for (i = 0; i < ec->nodes; i++) {
+            if (replies[i].valid && (replies[i].op_ret == 0) && !sources[i])
+                healed_sinks[i] = 1;
+        }
+    }
+
+    /* There could be files with versions, size same but on disk ia_size
+     * could be different because of disk crashes, mark them as sinks as
+     * well*/
+
+    if (check_ondisksize) {
+        source_size = size[source];
+        ec_adjust_size_up(ec, &source_size, _gf_true);
+
+        for (i = 0; i < ec->nodes; i++) {
+            if (sources[i]) {
+                if (replies[i].stat.ia_size != source_size) {
+                    sources[i] = 0;
+                    healed_sinks[i] = 1;
+                    max_same_count--;
+                } else {
+                    source = i;
                 }
+            }
         }
-        /* If we don't have ec->fragments number of same version,size it is not
-         * recoverable*/
         if (max_same_count < ec->fragments) {
-                ret = -EIO;
-                goto out;
-        } else {
-                snprintf (version_size, sizeof (version_size),
-                          "%"PRIu64"-%"PRIu64, versions[source], size[source]);
-                ret = dict_get_bin (version_size_db, version_size,
-                                    (void **)&same);
-                if (ret < 0)
-                        goto out;
-                memcpy (sources, same, ec->nodes);
-                for (i = 0; i < ec->nodes; i++) {
-                        if (replies[i].valid && (replies[i].op_ret == 0) &&
-                            !sources[i])
-                                healed_sinks[i] = 1;
-                }
+            ret = -EIO;
+            goto out;
         }
+    }
 
-        ret = source;
+    ret = source;
 out:
-        if (version_size_db)
-                dict_unref (version_size_db);
-        return ret;
+    if (version_size_db)
+        dict_unref(version_size_db);
+    return ret;
 }
 
 int
-__ec_heal_data_prepare (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                        unsigned char *locked_on, uint64_t *versions,
-                        uint64_t *dirty, uint64_t *size, unsigned char *sources,
-                        unsigned char *healed_sinks, unsigned char *trim,
-                        struct iatt *stbuf)
-{
-        default_args_cbk_t *replies = NULL;
-        unsigned char      *output  = NULL;
-        dict_t             *xattrs  = NULL;
-        uint64_t           zero_array[2] = {0};
-        int                source   = 0;
-        int                ret      = 0;
-        uint64_t           zero_value = 0;
-        uint64_t           source_size = 0;
-        int                i = 0;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output = alloca0(ec->nodes);
-        xattrs = dict_new ();
-        if (!xattrs ||
-            dict_set_static_bin (xattrs, EC_XATTR_VERSION, zero_array,
-                                 sizeof (zero_array)) ||
-            dict_set_static_bin (xattrs, EC_XATTR_DIRTY, zero_array,
-                                 sizeof (zero_array)) ||
-            dict_set_static_bin (xattrs, EC_XATTR_SIZE, &zero_value,
-                                 sizeof (zero_value))) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_data_prepare(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                       unsigned char *locked_on, uint64_t *versions,
+                       uint64_t *dirty, uint64_t *size, unsigned char *sources,
+                       unsigned char *healed_sinks, unsigned char *trim,
+                       struct iatt *stbuf)
+{
+    default_args_cbk_t *replies = NULL;
+    default_args_cbk_t *fstat_replies = NULL;
+    unsigned char *output = NULL;
+    unsigned char *fstat_output = NULL;
+    dict_t *xattrs = NULL;
+    uint64_t zero_array[2] = {0};
+    int source = 0;
+    int ret = 0;
+    uint64_t zero_value = 0;
+    int i = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    EC_REPLIES_ALLOC(fstat_replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    fstat_output = alloca0(ec->nodes);
+    xattrs = dict_new();
+    if (!xattrs ||
+        dict_set_static_bin(xattrs, EC_XATTR_VERSION, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xattrs, EC_XATTR_DIRTY, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xattrs, EC_XATTR_SIZE, &zero_value,
+                            sizeof(zero_value))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = cluster_fxattrop (ec->xl_list, locked_on, ec->nodes,
-                                replies, output, frame, ec->xl, fd,
-                                GF_XATTROP_ADD_ARRAY64, xattrs, NULL);
-        if (EC_COUNT (output, ec->nodes) <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    ret = cluster_fxattrop(ec->xl_list, locked_on, ec->nodes, replies, output,
+                           frame, ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs,
+                           NULL);
 
-        source = ec_heal_data_find_direction (ec, replies, versions, dirty,
-                                              size, sources, healed_sinks);
-        ret = source;
-        if (ret < 0)
-                goto out;
+    ret = cluster_fstat(ec->xl_list, locked_on, ec->nodes, fstat_replies,
+                        fstat_output, frame, ec->xl, fd, NULL);
 
-        /* There could be files with versions, size same but on disk ia_size
-         * could be different because of disk crashes, mark them as sinks as
-         * well*/
-        ret = cluster_fstat (ec->xl_list, locked_on, ec->nodes, replies,
-                             output, frame, ec->xl, fd, NULL);
-        EC_INTERSECT (sources, sources, output, ec->nodes);
-        EC_INTERSECT (healed_sinks, healed_sinks, output, ec->nodes);
-        if (EC_COUNT (sources, ec->nodes) < ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    for (i = 0; i < ec->nodes; i++) {
+        output[i] = output[i] && fstat_output[i];
+        replies[i].valid = output[i];
+        if (output[i])
+            replies[i].stat = fstat_replies[i].stat;
+    }
 
-        source_size = ec_adjust_size (ec, size[source], 1);
+    if (EC_COUNT(output, ec->nodes) <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i]) {
-                        if (replies[i].stat.ia_size != source_size) {
-                                sources[i] = 0;
-                                healed_sinks[i] = 1;
-                        } else if (stbuf) {
-                                source = i;
-                                *stbuf = replies[i].stat;
-                        }
-                }
+    source = ec_heal_data_find_direction(ec, replies, versions, dirty, size,
+                                         sources, healed_sinks, _gf_true,
+                                         EC_COMBINE_DICT);
+    ret = source;
+    if (ret < 0)
+        goto out;
 
-                if (healed_sinks[i]) {
-                        if (replies[i].stat.ia_size)
-                                trim[i] = 1;
-                }
-        }
+    if (stbuf)
+        *stbuf = replies[source].stat;
 
-        if (EC_COUNT(sources, ec->nodes) < ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i]) {
+            if (replies[i].stat.ia_size)
+                trim[i] = 1;
         }
+    }
 
-        ret = source;
+    if (EC_COUNT(sources, ec->nodes) < ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
+
+    ret = source;
 out:
-        if (xattrs)
-                dict_unref (xattrs);
-        cluster_replies_wipe (replies, ec->nodes);
-        if (ret < 0) {
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        } else {
-                gf_msg_debug (ec->xl->name, 0, "%s: sources: %d, sinks: "
-                        "%d", uuid_utoa (fd->inode->gfid),
-                        EC_COUNT (sources, ec->nodes),
-                        EC_COUNT (healed_sinks, ec->nodes));
-        }
-        return ret;
+    if (xattrs)
+        dict_unref(xattrs);
+    cluster_replies_wipe(replies, ec->nodes);
+    cluster_replies_wipe(fstat_replies, ec->nodes);
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    } else {
+        gf_msg_debug(ec->xl->name, 0,
+                     "%s: sources: %d, sinks: "
+                     "%d",
+                     uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                     EC_COUNT(healed_sinks, ec->nodes));
+    }
+    return ret;
 }
 
 int
-__ec_heal_mark_sinks (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                      uint64_t *versions, unsigned char *healed_sinks)
-{
-        int                i        = 0;
-        int                ret      = 0;
-        unsigned char      *mark    = NULL;
-        dict_t             *xattrs  = NULL;
-        default_args_cbk_t *replies = NULL;
-        unsigned char      *output  = NULL;
-        uint64_t versions_xattr[2]  = {0};
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        xattrs = dict_new ();
-        if (!xattrs) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_mark_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                     uint64_t *versions, unsigned char *healed_sinks)
+{
+    int i = 0;
+    int ret = 0;
+    unsigned char *mark = NULL;
+    dict_t *xattrs = NULL;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    uint64_t versions_xattr[2] = {0};
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattrs = dict_new();
+    if (!xattrs) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        mark = alloca0 (ec->nodes);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!healed_sinks[i])
-                        continue;
-                if ((versions[i] >> EC_SELFHEAL_BIT) & 1)
-                        continue;
-                mark[i] = 1;
-        }
+    mark = alloca0(ec->nodes);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!healed_sinks[i])
+            continue;
+        if ((versions[i] >> EC_SELFHEAL_BIT) & 1)
+            continue;
+        mark[i] = 1;
+    }
 
-        if (EC_COUNT (mark, ec->nodes) == 0)
-                return 0;
+    if (EC_COUNT(mark, ec->nodes) == 0)
+        return 0;
 
-        versions_xattr[EC_DATA_TXN] = hton64(1ULL<<EC_SELFHEAL_BIT);
-        if (dict_set_static_bin (xattrs, EC_XATTR_VERSION, versions_xattr,
-                                 sizeof (versions_xattr))) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    versions_xattr[EC_DATA_TXN] = hton64(1ULL << EC_SELFHEAL_BIT);
+    if (dict_set_static_bin(xattrs, EC_XATTR_VERSION, versions_xattr,
+                            sizeof(versions_xattr))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        output = alloca0 (ec->nodes);
-        ret = cluster_fxattrop (ec->xl_list, mark, ec->nodes,
-                                replies, output, frame, ec->xl, fd,
-                                GF_XATTROP_ADD_ARRAY64, xattrs, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!output[i]) {
-                        if (mark[i])
-                                healed_sinks[i] = 0;
-                        continue;
-                }
-                versions[i] |= (1ULL<<EC_SELFHEAL_BIT);
+    output = alloca0(ec->nodes);
+    ret = cluster_fxattrop(ec->xl_list, mark, ec->nodes, replies, output, frame,
+                           ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i]) {
+            if (mark[i])
+                healed_sinks[i] = 0;
+            continue;
         }
+        versions[i] |= (1ULL << EC_SELFHEAL_BIT);
+    }
 
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
-        ret = 0;
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
+    ret = 0;
 
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        if (xattrs)
-                dict_unref (xattrs);
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    if (xattrs)
+        dict_unref(xattrs);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
 int32_t
-ec_manager_heal_block (ec_fop_data_t *fop, int32_t state)
+ec_manager_heal_block(ec_fop_data_t *fop, int32_t state)
 {
     ec_heal_t *heal = fop->data;
     heal->fop = fop;
 
     switch (state) {
-    case EC_STATE_INIT:
-        ec_owner_set(fop->frame, fop->frame->root);
+        case EC_STATE_INIT:
+            ec_owner_set(fop->frame, fop->frame->root);
 
-        ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0);
+            ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0);
 
-        return EC_STATE_HEAL_DATA_COPY;
+            return EC_STATE_HEAL_DATA_COPY;
 
-    case EC_STATE_HEAL_DATA_COPY:
-        gf_msg_debug (fop->xl->name, 0, "%s: read/write starting",
-                uuid_utoa (heal->fd->inode->gfid));
-        ec_heal_data_block (heal);
+        case EC_STATE_HEAL_DATA_COPY:
+            gf_msg_debug(fop->xl->name, 0, "%s: read/write starting",
+                         uuid_utoa(heal->fd->inode->gfid));
+            ec_heal_data_block(heal);
 
-        return EC_STATE_HEAL_DATA_UNLOCK;
+            return EC_STATE_HEAL_DATA_UNLOCK;
 
-    case -EC_STATE_HEAL_DATA_COPY:
-    case -EC_STATE_HEAL_DATA_UNLOCK:
-    case EC_STATE_HEAL_DATA_UNLOCK:
-        ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0);
+        case -EC_STATE_HEAL_DATA_COPY:
+        case -EC_STATE_HEAL_DATA_UNLOCK:
+        case EC_STATE_HEAL_DATA_UNLOCK:
+            ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0);
 
-         return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
-    case EC_STATE_REPORT:
-        if (fop->cbks.heal) {
-            fop->cbks.heal (fop->req_frame, fop, fop->xl, 0,
-                            0, (heal->good | heal->bad),
-                            heal->good, heal->bad, NULL);
-        }
+        case EC_STATE_REPORT:
+            if (fop->cbks.heal) {
+                fop->cbks.heal(fop->req_frame, fop->data, fop->xl, 0, 0,
+                               (heal->good | heal->bad), heal->good, heal->bad,
+                               0, NULL);
+            }
 
-        return EC_STATE_END;
-    case -EC_STATE_REPORT:
-        if (fop->cbks.heal) {
-            fop->cbks.heal (fop->req_frame, fop, fop->xl, -1,
-                            fop->error, 0, 0, 0, NULL);
-        }
+            return EC_STATE_END;
+        case -EC_STATE_REPORT:
+            if (fop->cbks.heal) {
+                fop->cbks.heal(fop->req_frame, fop->data, fop->xl, -1,
+                               fop->error, 0, 0, 0, 0, NULL);
+            }
 
-        return EC_STATE_END;
-    default:
-        gf_msg (fop->xl->name, GF_LOG_ERROR, 0,
-                EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                state, ec_fop_name(fop->id));
+            return EC_STATE_END;
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-        return EC_STATE_END;
+            return EC_STATE_END;
     }
 }
 
 /*Takes lock */
 void
-ec_heal_block (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_heal_cbk_t func, ec_heal_t *heal)
+ec_heal_block(call_frame_t *frame, xlator_t *this, uintptr_t target,
+              uint32_t fop_flags, fop_heal_cbk_t func, ec_heal_t *heal)
 {
-    ec_cbk_t callback = { .heal = func };
+    ec_cbk_t callback = {.heal = func};
     ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
@@ -1813,9 +2009,8 @@ ec_heal_block (call_frame_t *frame, xlator_t *this, uintptr_t target,
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate (frame, this, EC_FOP_HEAL, 0, target, minimum,
-                                NULL, ec_manager_heal_block, callback,
-                                heal);
+    fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags,
+                               NULL, ec_manager_heal_block, callback, heal);
     if (fop == NULL)
         goto out;
 
@@ -1825,709 +2020,878 @@ out:
     if (fop != NULL) {
         ec_manager(fop, error);
     } else {
-        func(frame, NULL, this, -1, error, 0, 0, 0, NULL);
+        func(frame, heal, this, -1, error, 0, 0, 0, 0, NULL);
     }
 }
 
 int32_t
-ec_heal_block_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                    uintptr_t good, uintptr_t bad, dict_t *xdata)
+ec_heal_block_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, uintptr_t mask,
+                   uintptr_t good, uintptr_t bad, uint32_t pending,
+                   dict_t *xdata)
 {
-        ec_fop_data_t *fop = cookie;
-        ec_heal_t *heal = fop->data;
+    ec_heal_t *heal = cookie;
 
-        fop->heal = NULL;
-        heal->fop = NULL;
-        heal->error = op_ret < 0 ? op_errno : 0;
-        syncbarrier_wake (heal->data);
-        return 0;
+    if (heal->fop) {
+        heal->fop->heal = NULL;
+    }
+    heal->fop = NULL;
+    heal->error = op_ret < 0 ? op_errno : 0;
+    syncbarrier_wake(heal->data);
+    return 0;
 }
 
 int
-ec_sync_heal_block (call_frame_t *frame, xlator_t *this, ec_heal_t *heal)
+ec_sync_heal_block(call_frame_t *frame, xlator_t *this, ec_heal_t *heal)
 {
-        ec_heal_block (frame, this, heal->bad|heal->good, EC_MINIMUM_ONE,
-                       ec_heal_block_done, heal);
-        syncbarrier_wait (heal->data, 1);
-        if (heal->error != 0) {
-                return -heal->error;
-        }
-        if (heal->bad == 0)
-                return -ENOTCONN;
-        return 0;
+    ec_heal_block(frame, this, heal->bad | heal->good, EC_MINIMUM_ONE,
+                  ec_heal_block_done, heal);
+    syncbarrier_wait(heal->data, 1);
+    if (heal->error != 0) {
+        return -heal->error;
+    }
+    if (heal->bad == 0)
+        return -ENOTCONN;
+    return 0;
 }
 
 int
-ec_rebuild_data (call_frame_t *frame, ec_t *ec, fd_t *fd, uint64_t size,
-                 unsigned char *sources, unsigned char *healed_sinks)
+ec_rebuild_data(call_frame_t *frame, ec_t *ec, fd_t *fd, uint64_t size,
+                unsigned char *sources, unsigned char *healed_sinks)
 {
-        ec_heal_t        *heal = NULL;
-        int              ret = 0;
-        syncbarrier_t    barrier;
-        struct iobuf_pool *pool = NULL;
-
-        if (syncbarrier_init (&barrier))
-                return -ENOMEM;
-
-        heal = alloca0(sizeof (*heal));
-        heal->fd = fd_ref (fd);
-        heal->xl = ec->xl;
-        heal->data = &barrier;
-        syncbarrier_init (heal->data);
-        pool = ec->xl->ctx->iobuf_pool;
-        heal->total_size = size;
-        heal->size = iobpool_default_pagesize (pool);
-        /* We need to adjust the size to a multiple of the stripe size of the
-         * volume. Otherwise writes would need to fill gaps (head and/or tail)
-         * with existent data from the bad bricks. This could be garbage on a
-         * damaged file or it could fail if there aren't enough bricks. */
-        heal->size -= heal->size % ec->stripe_size;
-        heal->bad       = ec_char_array_to_mask (healed_sinks, ec->nodes);
-        heal->good      = ec_char_array_to_mask (sources, ec->nodes);
-        heal->iatt.ia_type = IA_IFREG;
-        LOCK_INIT(&heal->lock);
-
-        for (heal->offset = 0; (heal->offset < size) && !heal->done;
-                                                   heal->offset += heal->size) {
-                gf_msg_debug (ec->xl->name, 0, "%s: sources: %d, sinks: "
-                        "%d, offset: %"PRIu64" bsize: %"PRIu64,
-                        uuid_utoa (fd->inode->gfid),
-                        EC_COUNT (sources, ec->nodes),
-                        EC_COUNT (healed_sinks, ec->nodes), heal->offset,
-                        heal->size);
-                ret = ec_sync_heal_block (frame, ec->xl, heal);
-                if (ret < 0)
-                        break;
-
-        }
-        memset (healed_sinks, 0, ec->nodes);
-        ec_mask_to_char_array (heal->bad, healed_sinks, ec->nodes);
-        fd_unref (heal->fd);
-        LOCK_DESTROY (&heal->lock);
-        syncbarrier_destroy (heal->data);
+    ec_heal_t *heal = NULL;
+    int ret = 0;
+    syncbarrier_t barrier;
+
+    if (syncbarrier_init(&barrier))
+        return -ENOMEM;
+
+    heal = alloca0(sizeof(*heal));
+    heal->fd = fd_ref(fd);
+    heal->xl = ec->xl;
+    heal->data = &barrier;
+    ec_adjust_size_up(ec, &size, _gf_false);
+    heal->total_size = size;
+    heal->size = (128 * GF_UNIT_KB * (ec->self_heal_window_size));
+    /* We need to adjust the size to a multiple of the stripe size of the
+     * volume. Otherwise writes would need to fill gaps (head and/or tail)
+     * with existent data from the bad bricks. This could be garbage on a
+     * damaged file or it could fail if there aren't enough bricks. */
+    heal->size -= heal->size % ec->stripe_size;
+    heal->bad = ec_char_array_to_mask(healed_sinks, ec->nodes);
+    heal->good = ec_char_array_to_mask(sources, ec->nodes);
+    heal->iatt.ia_type = IA_IFREG;
+    LOCK_INIT(&heal->lock);
+
+    for (heal->offset = 0; (heal->offset < size) && !heal->done;
+         heal->offset += heal->size) {
+        /* We immediately abort any heal if a shutdown request has been
+         * received to avoid delays. The healing of this file will be
+         * restarted by another SHD or other client that accesses the
+         * file. */
+        if (ec->shutdown) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "Cancelling heal because "
+                         "EC is stopping.");
+            ret = -ENOTCONN;
+            break;
+        }
+
+        gf_msg_debug(ec->xl->name, 0,
+                     "%s: sources: %d, sinks: "
+                     "%d, offset: %" PRIu64 " bsize: %" PRIu64,
+                     uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                     EC_COUNT(healed_sinks, ec->nodes), heal->offset,
+                     heal->size);
+        ret = ec_sync_heal_block(frame, ec->xl, heal);
         if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        return ret;
+            break;
+    }
+    memset(healed_sinks, 0, ec->nodes);
+    ec_mask_to_char_array(heal->bad, healed_sinks, ec->nodes);
+    fd_unref(heal->fd);
+    LOCK_DESTROY(&heal->lock);
+    syncbarrier_destroy(heal->data);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
 int
-__ec_heal_trim_sinks (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                      unsigned char *healed_sinks, unsigned char *trim)
+__ec_heal_trim_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                     unsigned char *healed_sinks, unsigned char *trim,
+                     uint64_t size)
 {
-        default_args_cbk_t *replies = NULL;
-        unsigned char      *output  = NULL;
-        int                ret      = 0;
-        int                i        = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    int i = 0;
+    off_t trim_offset = 0;
 
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output       = alloca0 (ec->nodes);
-
-        if (EC_COUNT (trim, ec->nodes) == 0) {
-                ret = 0;
-                goto out;
-        }
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
 
-        ret = cluster_ftruncate (ec->xl_list, trim, ec->nodes, replies, output,
-                                 frame, ec->xl, fd, 0, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!output[i] && trim[i])
-                        healed_sinks[i] = 0;
-        }
+    if (EC_COUNT(trim, ec->nodes) == 0) {
+        ret = 0;
+        goto out;
+    }
+    trim_offset = size;
+    ec_adjust_offset_up(ec, &trim_offset, _gf_true);
+    ret = cluster_ftruncate(ec->xl_list, trim, ec->nodes, replies, output,
+                            frame, ec->xl, fd, trim_offset, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i] && trim[i])
+            healed_sinks[i] = 0;
+    }
 
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
 int
-ec_data_undo_pending (call_frame_t *frame, ec_t *ec, fd_t *fd, dict_t *xattr,
-                      uint64_t *versions, uint64_t *dirty, uint64_t *size,
-                      int source, gf_boolean_t erase_dirty, int idx)
-{
-        uint64_t versions_xattr[2] = {0};
-        uint64_t dirty_xattr[2]    = {0};
-        uint64_t allzero[2]        = {0};
-        uint64_t size_xattr        = 0;
-        int      ret               = 0;
-
-        versions_xattr[EC_DATA_TXN] = hton64(versions[source] - versions[idx]);
-        ret = dict_set_static_bin (xattr, EC_XATTR_VERSION,
-                                   versions_xattr,
-                                   sizeof (versions_xattr));
-        if (ret < 0)
-                goto out;
+ec_data_undo_pending(call_frame_t *frame, ec_t *ec, fd_t *fd, dict_t *xattr,
+                     uint64_t *versions, uint64_t *dirty, uint64_t *size,
+                     int source, gf_boolean_t erase_dirty, int idx)
+{
+    uint64_t versions_xattr[2] = {0};
+    uint64_t dirty_xattr[2] = {0};
+    uint64_t allzero[2] = {0};
+    uint64_t size_xattr = 0;
+    int ret = 0;
+
+    versions_xattr[EC_DATA_TXN] = hton64(versions[source] - versions[idx]);
+    ret = dict_set_static_bin(xattr, EC_XATTR_VERSION, versions_xattr,
+                              sizeof(versions_xattr));
+    if (ret < 0)
+        goto out;
+
+    size_xattr = hton64(size[source] - size[idx]);
+    ret = dict_set_static_bin(xattr, EC_XATTR_SIZE, &size_xattr,
+                              sizeof(size_xattr));
+    if (ret < 0)
+        goto out;
 
-        size_xattr = hton64(size[source] - size[idx]);
-        ret = dict_set_static_bin (xattr, EC_XATTR_SIZE,
-                                   &size_xattr, sizeof (size_xattr));
+    if (erase_dirty) {
+        dirty_xattr[EC_DATA_TXN] = hton64(-dirty[idx]);
+        ret = dict_set_static_bin(xattr, EC_XATTR_DIRTY, dirty_xattr,
+                                  sizeof(dirty_xattr));
         if (ret < 0)
-                goto out;
+            goto out;
+    }
 
-        if (erase_dirty) {
-                dirty_xattr[EC_DATA_TXN] = hton64(-dirty[idx]);
-                ret = dict_set_static_bin (xattr, EC_XATTR_DIRTY,
-                                           dirty_xattr,
-                                           sizeof (dirty_xattr));
-                if (ret < 0)
-                        goto out;
-        }
-
-        if ((memcmp (versions_xattr, allzero, sizeof (allzero)) == 0) &&
-            (memcmp (dirty_xattr, allzero, sizeof (allzero)) == 0) &&
-             (size == 0)) {
-                ret = 0;
-                goto out;
-        }
+    if ((memcmp(versions_xattr, allzero, sizeof(allzero)) == 0) &&
+        (memcmp(dirty_xattr, allzero, sizeof(allzero)) == 0) &&
+        (size_xattr == 0)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = syncop_fxattrop (ec->xl_list[idx], fd,
-                               GF_XATTROP_ADD_ARRAY64, xattr, NULL, NULL);
+    ret = syncop_fxattrop(ec->xl_list[idx], fd, GF_XATTROP_ADD_ARRAY64, xattr,
+                          NULL, NULL, NULL);
 out:
-        return ret;
+    return ret;
 }
 
 int
-__ec_fd_data_adjust_versions (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                            unsigned char *sources, unsigned char *healed_sinks,
-                            uint64_t *versions, uint64_t *dirty, uint64_t *size)
-{
-        dict_t                     *xattr            = NULL;
-        int                        i                 = 0;
-        int                        ret               = 0;
-        int                        op_ret            = 0;
-        int                        source            = -1;
-        gf_boolean_t               erase_dirty       = _gf_false;
-
-        xattr = dict_new ();
-        if (!xattr) {
-                op_ret = -ENOMEM;
-                goto out;
-        }
+__ec_fd_data_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                             unsigned char *sources,
+                             unsigned char *healed_sinks, uint64_t *versions,
+                             uint64_t *dirty, uint64_t *size)
+{
+    dict_t *xattr = NULL;
+    int i = 0;
+    int ret = 0;
+    int op_ret = 0;
+    int source = -1;
+    gf_boolean_t erase_dirty = _gf_false;
+
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -ENOMEM;
+        goto out;
+    }
 
-        /* dirty xattr represents if the file needs heal. Unless all the
-         * copies are healed, don't erase it */
-        if (EC_COUNT (sources, ec->nodes) +
-            EC_COUNT (healed_sinks, ec->nodes) == ec->nodes)
-                erase_dirty = _gf_true;
+    /* dirty xattr represents if the file needs heal. Unless all the
+     * copies are healed, don't erase it */
+    if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) ==
+        ec->nodes)
+        erase_dirty = _gf_true;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i]) {
-                        source = i;
-                        break;
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
         }
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (healed_sinks[i]) {
-                        ret = ec_data_undo_pending (frame, ec, fd, xattr,
-                                                    versions, dirty, size,
-                                                    source, erase_dirty, i);
-                        if (ret < 0)
-                                goto out;
-                }
-
-        }
+    if (source == -1) {
+        op_ret = -ENOTCONN;
+        goto out;
+    }
 
-        if (!erase_dirty)
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i]) {
+            ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty,
+                                       size, source, erase_dirty, i);
+            if (ret < 0)
                 goto out;
+        }
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i]) {
-                        ret = ec_data_undo_pending (frame, ec, fd, xattr,
-                                                    versions, dirty, size,
-                                                    source, erase_dirty, i);
-                        if (ret < 0)
-                                continue;
-                }
+    if (!erase_dirty)
+        goto out;
 
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i]) {
+            ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty,
+                                       size, source, erase_dirty, i);
+            if (ret < 0)
+                continue;
         }
+    }
 out:
-        if (xattr)
-                dict_unref (xattr);
-        return op_ret;
+    if (xattr)
+        dict_unref(xattr);
+    return op_ret;
 }
 
 int
-ec_restore_time_and_adjust_versions (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                                     unsigned char *sources,
-                                     unsigned char *healed_sinks,
-                                     uint64_t *versions, uint64_t *dirty,
-                                     uint64_t *size)
-{
-        unsigned char      *locked_on           = NULL;
-        unsigned char      *participants        = NULL;
-        unsigned char      *output              = NULL;
-        default_args_cbk_t *replies             = NULL;
-        unsigned char      *postsh_sources      = NULL;
-        unsigned char      *postsh_healed_sinks = NULL;
-        unsigned char      *postsh_trim         = NULL;
-        uint64_t           *postsh_versions     = NULL;
-        uint64_t           *postsh_dirty        = NULL;
-        uint64_t           *postsh_size         = NULL;
-        int                ret                  = 0;
-        int                i                    = 0;
-        struct iatt        source_buf           = {0};
-        loc_t              loc                  = {0};
-
-        locked_on           = alloca0(ec->nodes);
-        output              = alloca0(ec->nodes);
-        participants        = alloca0(ec->nodes);
-        postsh_sources      = alloca0(ec->nodes);
-        postsh_healed_sinks = alloca0(ec->nodes);
-        postsh_trim         = alloca0(ec->nodes);
-        postsh_versions     = alloca0(ec->nodes * sizeof (*postsh_versions));
-        postsh_dirty        = alloca0(ec->nodes * sizeof (*postsh_dirty));
-        postsh_size         = alloca0(ec->nodes * sizeof (*postsh_size));
-
-        for (i = 0; i < ec->nodes; i++) {
-                if (healed_sinks[i] || sources[i])
-                        participants[i] = 1;
-        }
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, participants, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name,
-                               fd->inode, 0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (fd->inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
+ec_restore_time_and_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                                    unsigned char *sources,
+                                    unsigned char *healed_sinks,
+                                    uint64_t *versions, uint64_t *dirty,
+                                    uint64_t *size)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *participants = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *postsh_sources = NULL;
+    unsigned char *postsh_healed_sinks = NULL;
+    unsigned char *postsh_trim = NULL;
+    uint64_t *postsh_versions = NULL;
+    uint64_t *postsh_dirty = NULL;
+    uint64_t *postsh_size = NULL;
+    int ret = 0;
+    int i = 0;
+    struct iatt source_buf = {0};
+    loc_t loc = {0};
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    participants = alloca0(ec->nodes);
+    postsh_sources = alloca0(ec->nodes);
+    postsh_healed_sinks = alloca0(ec->nodes);
+    postsh_trim = alloca0(ec->nodes);
+    postsh_versions = alloca0(ec->nodes * sizeof(*postsh_versions));
+    postsh_dirty = alloca0(ec->nodes * sizeof(*postsh_dirty));
+    postsh_size = alloca0(ec->nodes * sizeof(*postsh_size));
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i] || sources[i])
+            participants[i] = 1;
+    }
 
-                ret = __ec_heal_data_prepare (frame, ec, fd, locked_on,
-                                              postsh_versions, postsh_dirty,
-                                              postsh_size, postsh_sources,
-                                              postsh_healed_sinks, postsh_trim,
-                                              &source_buf);
-                if (ret < 0)
-                        goto unlock;
-
-                loc.inode = inode_ref (fd->inode);
-                gf_uuid_copy (loc.gfid, fd->inode->gfid);
-                ret = cluster_setattr (ec->xl_list, healed_sinks, ec->nodes,
-                                       replies, output, frame, ec->xl, &loc,
-                                       &source_buf,
-                                       GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME,
-                                       NULL);
-                EC_INTERSECT (healed_sinks, healed_sinks, output, ec->nodes);
-                if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_fd_data_adjust_versions (frame, ec, fd, sources,
-                                           healed_sinks, versions, dirty, size);
-        }
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, fd->inode, 0,
+                          0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, postsh_versions,
+                                     postsh_dirty, postsh_size, postsh_sources,
+                                     postsh_healed_sinks, postsh_trim,
+                                     &source_buf);
+        if (ret < 0)
+            goto unlock;
+
+        loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(loc.gfid, fd->inode->gfid);
+        ret = cluster_setattr(
+            ec->xl_list, healed_sinks, ec->nodes, replies, output, frame,
+            ec->xl, &loc, &source_buf,
+            GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME, NULL);
+        EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
+        if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_fd_data_adjust_versions(frame, ec, fd, sources, healed_sinks,
+                                           versions, dirty, size);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, fd->inode, 0, 0);
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, fd->inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-__ec_heal_data (call_frame_t *frame, ec_t *ec, fd_t *fd, unsigned char *heal_on,
-                unsigned char *sources, unsigned char *healed_sinks)
+__ec_heal_data(call_frame_t *frame, ec_t *ec, fd_t *fd, unsigned char *heal_on,
+               unsigned char *sources, unsigned char *healed_sinks)
 {
-        unsigned char      *locked_on    = NULL;
-        unsigned char      *output       = NULL;
-        uint64_t           *versions     = NULL;
-        uint64_t           *dirty        = NULL;
-        uint64_t           *size         = NULL;
-        unsigned char      *trim         = NULL;
-        default_args_cbk_t *replies      = NULL;
-        int                ret           = 0;
-        int                source        = 0;
-
-        locked_on    = alloca0(ec->nodes);
-        output       = alloca0(ec->nodes);
-        trim         = alloca0 (ec->nodes);
-        versions     = alloca0 (ec->nodes * sizeof (*versions));
-        dirty        = alloca0 (ec->nodes * sizeof (*dirty));
-        size         = alloca0 (ec->nodes * sizeof (*size));
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, heal_on, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name,
-                               fd->inode, 0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (fd->inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-
-                ret = __ec_heal_data_prepare (frame, ec, fd, locked_on,
-                                              versions, dirty, size, sources,
-                                              healed_sinks, trim, NULL);
-                if (ret < 0)
-                        goto unlock;
-
-                if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
-                        ret = __ec_fd_data_adjust_versions (frame, ec, fd,
-                                                            sources,
-                                        healed_sinks, versions, dirty, size);
-                        goto unlock;
-                }
+    unsigned char *locked_on = NULL;
+    unsigned char *output = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    uint64_t *size = NULL;
+    unsigned char *trim = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+    int source = 0;
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    trim = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    size = alloca0(ec->nodes * sizeof(*size));
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on,
+                          frame, ec->xl, ec->xl->name, fd->inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-                source = ret;
-                ret = __ec_heal_mark_sinks (frame, ec, fd, versions,
-                                            healed_sinks);
-                if (ret < 0)
-                        goto unlock;
+        ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, versions, dirty,
+                                     size, sources, healed_sinks, trim, NULL);
+        if (ret < 0)
+            goto unlock;
 
-                ret = __ec_heal_trim_sinks (frame, ec, fd, healed_sinks, trim);
+        if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+            ret = __ec_fd_data_adjust_versions(
+                frame, ec, fd, sources, healed_sinks, versions, dirty, size);
+            goto unlock;
         }
-unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, fd->inode, 0, 0);
+
+        source = ret;
+        ret = __ec_heal_mark_sinks(frame, ec, fd, versions, healed_sinks);
         if (ret < 0)
-                goto out;
+            goto unlock;
 
-        if (EC_COUNT(healed_sinks, ec->nodes) == 0)
-                goto out;
+        ret = __ec_heal_trim_sinks(frame, ec, fd, healed_sinks, trim,
+                                   size[source]);
+    }
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, fd->inode, 0, 0);
+    if (ret < 0)
+        goto out;
 
-        gf_msg_debug (ec->xl->name, 0, "%s: sources: %d, sinks: "
-                "%d", uuid_utoa (fd->inode->gfid),
-                EC_COUNT (sources, ec->nodes),
-                EC_COUNT (healed_sinks, ec->nodes));
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0)
+        goto out;
 
-        ret = ec_rebuild_data (frame, ec, fd, size[source], sources,
-                               healed_sinks);
-        if (ret < 0)
-                goto out;
+    gf_msg_debug(ec->xl->name, 0,
+                 "%s: sources: %d, sinks: "
+                 "%d",
+                 uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                 EC_COUNT(healed_sinks, ec->nodes));
 
-        ret = ec_restore_time_and_adjust_versions (frame, ec, fd, sources,
-                                                   healed_sinks, versions,
-                                                   dirty, size);
+    ret = ec_rebuild_data(frame, ec, fd, size[source], sources, healed_sinks);
+    if (ret < 0)
+        goto out;
+
+    ret = ec_restore_time_and_adjust_versions(
+        frame, ec, fd, sources, healed_sinks, versions, dirty, size);
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 int
-ec_heal_data (call_frame_t *frame, ec_t *ec, gf_boolean_t block, inode_t *inode,
-              unsigned char *sources, unsigned char *healed_sinks)
-{
-        unsigned char      *locked_on            = NULL;
-        unsigned char      *up_subvols           = NULL;
-        unsigned char      *output               = NULL;
-        default_args_cbk_t *replies              = NULL;
-        fd_t               *fd                   = NULL;
-        loc_t               loc                  = {0};
-        char               selfheal_domain[1024] = {0};
-        int                ret                   = 0;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-
-        locked_on  = alloca0(ec->nodes);
-        output     = alloca0(ec->nodes);
-        up_subvols = alloca0(ec->nodes);
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-
-        fd = fd_create (inode, 0);
-        if (!fd) {
-                ret = -ENOMEM;
-                goto out;
-        }
+ec_heal_data(call_frame_t *frame, ec_t *ec, gf_boolean_t block, inode_t *inode,
+             unsigned char *sources, unsigned char *healed_sinks)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    fd_t *fd = NULL;
+    loc_t loc = {0};
+    char selfheal_domain[1024] = {0};
+    int ret = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
 
-        ret = cluster_open (ec->xl_list, up_subvols, ec->nodes, replies, output,
-                            frame, ec->xl, &loc, O_RDWR|O_LARGEFILE, fd, NULL);
-        if (ret <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    ret = cluster_open(ec->xl_list, up_subvols, ec->nodes, replies, output,
+                       frame, ec->xl, &loc, O_RDWR | O_LARGEFILE, fd, NULL);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        fd_bind (fd);
-        sprintf (selfheal_domain, "%s:self-heal", ec->xl->name);
-        /*If other processes are already doing the heal, don't block*/
-        if (block) {
-                ret = cluster_inodelk (ec->xl_list, output, ec->nodes, replies,
-                                       locked_on, frame, ec->xl,
-                                       selfheal_domain, inode, 0, 0);
-        } else {
-                ret = cluster_tryinodelk (ec->xl_list, output, ec->nodes,
-                                          replies, locked_on, frame, ec->xl,
-                                          selfheal_domain, inode, 0, 0);
-        }
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_data (frame, ec, fd, locked_on, sources,
-                                      healed_sinks);
-        }
+    fd_bind(fd);
+    sprintf(selfheal_domain, "%s:self-heal", ec->xl->name);
+    /*If other processes are already doing the heal, don't block*/
+    if (block) {
+        ret = cluster_inodelk(ec->xl_list, output, ec->nodes, replies,
+                              locked_on, frame, ec->xl, selfheal_domain, inode,
+                              0, 0);
+    } else {
+        ret = cluster_tiebreaker_inodelk(ec->xl_list, output, ec->nodes,
+                                         replies, locked_on, frame, ec->xl,
+                                         selfheal_domain, inode, 0, 0);
+    }
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_data(frame, ec, fd, locked_on, sources, healed_sinks);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, selfheal_domain, inode, 0, 0);
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, selfheal_domain, inode, 0, 0);
+out:
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
+}
+
+int
+ec_heal_purge_stale_index(call_frame_t *frame, ec_t *ec, inode_t *inode)
+{
+    int i = 0;
+    int ret = 0;
+    dict_t **xattr = NULL;
+    loc_t loc = {0};
+    uint64_t dirty_xattr[EC_VERSION_SIZE] = {0};
+    unsigned char *on = NULL;
+    default_args_cbk_t *replies = NULL;
+    dict_t *dict = NULL;
+
+    /* Allocate the required memory */
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    on = alloca0(ec->nodes);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer);
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        xattr[i] = dict;
+        on[i] = 1;
+    }
+    ret = dict_set_static_bin(dict, EC_XATTR_DIRTY, dirty_xattr,
+                              (sizeof(*dirty_xattr) * EC_VERSION_SIZE));
+    if (ret < 0) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame,
+                        ec_wind_xattrop_parallel, &loc, GF_XATTROP_ADD_ARRAY64,
+                        xattr, NULL);
 out:
-        if (fd)
-                fd_unref (fd);
-        loc_wipe (&loc);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    if (dict) {
+        dict_unref(dict);
+    }
+    if (xattr) {
+        GF_FREE(xattr);
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 void
-ec_heal_do (xlator_t *this, void *data, loc_t *loc, int32_t partial)
-{
-        call_frame_t  *frame         = NULL;
-        unsigned char *participants  = NULL;
-        unsigned char *msources      = NULL;
-        unsigned char *mhealed_sinks = NULL;
-        unsigned char *sources       = NULL;
-        unsigned char *healed_sinks  = NULL;
-        ec_t          *ec            = NULL;
-        int           ret            = 0;
-        int           op_ret         = 0;
-        int           op_errno       = 0;
-        intptr_t      mgood          = 0;
-        intptr_t      mbad           = 0;
-        intptr_t      good           = 0;
-        intptr_t      bad            = 0;
-        ec_fop_data_t *fop           = data;
-        gf_boolean_t  blocking       = _gf_false;
-
-        ec = this->private;
-
-        /* If it is heal request from getxattr, complete the heal and then
-         * unwind, if it is ec_heal with NULL as frame then no need to block
-         * the heal as the caller doesn't care about its completion*/
-        if (fop->req_frame)
-                blocking = _gf_true;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                return;
-
-        ec_owner_set(frame, frame->root);
-        /*Do heal as root*/
-        frame->root->uid = 0;
-        frame->root->gid = 0;
-        /*Mark the fops as internal*/
-        frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
-        participants = alloca0(ec->nodes);
-        ec_mask_to_char_array (ec->xl_up, participants, ec->nodes);
-        if (loc->name && strlen (loc->name)) {
-                ret = ec_heal_name (frame, ec, loc->parent, (char *)loc->name,
-                                    participants);
-                if (ret == 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                EC_MSG_HEAL_SUCCESS, "%s: name heal "
-                                "successful on %lX", loc->path,
-                                ec_char_array_to_mask (participants,
-                                    ec->nodes));
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, -ret,
-                                EC_MSG_HEAL_FAIL, "%s: name heal "
-                                "failed", loc->path);
-                }
-        }
+ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial)
+{
+    call_frame_t *frame = NULL;
+    unsigned char *participants = NULL;
+    unsigned char *msources = NULL;
+    unsigned char *mhealed_sinks = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    ec_t *ec = NULL;
+    int ret = 0;
+    int op_ret = 0;
+    int op_errno = 0;
+    intptr_t mgood = 0;
+    intptr_t mbad = 0;
+    intptr_t good = 0;
+    intptr_t bad = 0;
+    uint32_t pending = 0;
+    ec_fop_data_t *fop = data;
+    gf_boolean_t blocking = _gf_false;
+    ec_heal_need_t need_heal = EC_HEAL_NONEED;
+    unsigned char *up_subvols = NULL;
+    char up_bricks[32];
+
+    ec = this->private;
+
+    /* If it is heal request from getxattr, complete the heal and then
+     * unwind, if it is ec_heal with NULL as frame then no need to block
+     * the heal as the caller doesn't care about its completion. In case
+     * of heald whichever gets tiebreaking inodelk will take care of the
+     * heal, so no need to block*/
+    if (fop->req_frame && !ec->shd.iamshd)
+        blocking = _gf_true;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-        msources = alloca0(ec->nodes);
-        mhealed_sinks = alloca0(ec->nodes);
-        ret = ec_heal_metadata (frame, ec, loc->inode, msources, mhealed_sinks);
-        if (ret == 0) {
-                mgood = ec_char_array_to_mask (msources, ec->nodes);
-                mbad  = ec_char_array_to_mask (mhealed_sinks, ec->nodes);
-        } else {
-                op_ret = -1;
-                op_errno = -ret;
-        }
-        sources = alloca0(ec->nodes);
-        healed_sinks = alloca0(ec->nodes);
-        if (IA_ISREG (loc->inode->ia_type)) {
-                ret = ec_heal_data (frame, ec, blocking, loc->inode, sources,
-                                    healed_sinks);
-        } else if (IA_ISDIR (loc->inode->ia_type) && !partial) {
-                ret = ec_heal_entry (frame, ec, loc->inode, sources,
-                                     healed_sinks);
+    ec_owner_set(frame, frame->root);
+    /*Do heal as root*/
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    /*Mark the fops as internal*/
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+    participants = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, participants, ec->nodes);
+
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+
+    if (loc->name && strlen(loc->name)) {
+        ret = ec_heal_name(frame, ec, loc->parent, (char *)loc->name,
+                           participants);
+        if (ret >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s: name heal "
+                         "successful on %" PRIXPTR,
+                         loc->path,
+                         ec_char_array_to_mask(participants, ec->nodes));
         } else {
-                ret = 0;
-                memcpy (sources, participants, ec->nodes);
-                memcpy (healed_sinks, participants, ec->nodes);
+            gf_msg_debug(
+                this->name, 0,
+                "%s: name heal "
+                "failed. ret = %d, subvolumes up = %s",
+                loc->path, ret,
+                ec_bin(up_bricks, sizeof(up_bricks), ec->xl_up, ec->nodes));
         }
+    }
 
-        if (ret == 0) {
-                good = ec_char_array_to_mask (sources, ec->nodes);
-                bad  = ec_char_array_to_mask (healed_sinks, ec->nodes);
-        } else {
-                op_ret = -1;
-                op_errno = -ret;
+    /* Mount triggers heal only when it detects that it must need heal, shd
+     * triggers heals periodically which need not be thorough*/
+    if (ec->shd.iamshd && (ret <= 0)) {
+        ec_heal_inspect(frame, ec, loc->inode, up_subvols, _gf_false, _gf_false,
+                        &need_heal);
+
+        if (need_heal == EC_HEAL_PURGE_INDEX) {
+            gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL,
+                   "Index entry needs to be purged for: %s ",
+                   uuid_utoa(loc->gfid));
+            /* We need to send zero-xattrop so that stale index entry could be
+             * removed. We need not take lock on this entry to do so as
+             * xattrop on a brick is atomic. */
+            ec_heal_purge_stale_index(frame, ec, loc->inode);
+            goto out;
+        } else if (need_heal == EC_HEAL_NONEED) {
+            gf_msg(ec->xl->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL,
+                   "Heal is not required for : %s ", uuid_utoa(loc->gfid));
+            goto out;
         }
+    }
 
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    if (IA_ISREG(loc->inode->ia_type)) {
+        ret = ec_heal_data(frame, ec, blocking, loc->inode, sources,
+                           healed_sinks);
+    } else if (IA_ISDIR(loc->inode->ia_type) && !partial) {
+        ret = ec_heal_entry(frame, ec, loc->inode, sources, healed_sinks,
+                            &pending);
+    } else {
+        ret = 0;
+        memcpy(sources, participants, ec->nodes);
+        memcpy(healed_sinks, participants, ec->nodes);
+    }
 
-        if (fop->cbks.heal) {
-                fop->cbks.heal (fop->req_frame, fop, fop->xl, op_ret,
-                                op_errno, ec_char_array_to_mask (participants,
-                                                                 ec->nodes),
-                                mgood & good, mbad & bad, NULL);
-        }
-        STACK_DESTROY (frame->root);
-        return;
+    if (ret == 0) {
+        good = ec_char_array_to_mask(sources, ec->nodes);
+        bad = ec_char_array_to_mask(healed_sinks, ec->nodes);
+    } else {
+        op_ret = -1;
+        op_errno = -ret;
+    }
+    msources = alloca0(ec->nodes);
+    mhealed_sinks = alloca0(ec->nodes);
+    ret = ec_heal_metadata(frame, ec, loc->inode, msources, mhealed_sinks);
+    if (ret == 0) {
+        mgood = ec_char_array_to_mask(msources, ec->nodes);
+        mbad = ec_char_array_to_mask(mhealed_sinks, ec->nodes);
+    } else {
+        op_ret = -1;
+        op_errno = -ret;
+    }
+
+out:
+    ec_reset_entry_healing(fop);
+    if (fop->cbks.heal) {
+        fop->cbks.heal(fop->req_frame, fop->data, fop->xl, op_ret, op_errno,
+                       ec_char_array_to_mask(participants, ec->nodes),
+                       mgood & good, mbad & bad, pending, NULL);
+    }
+    if (frame)
+        STACK_DESTROY(frame->root);
+
+    return;
 }
 
 int
-ec_synctask_heal_wrap (void *opaque)
+ec_synctask_heal_wrap(void *opaque)
 {
-        ec_fop_data_t *fop = opaque;
-        ec_heal_do (fop->xl, fop, &fop->loc[0], fop->int32);
-        return 0;
+    ec_fop_data_t *fop = opaque;
+    ec_heal_do(fop->xl, fop, &fop->loc[0], fop->int32);
+    return 0;
 }
 
 int
-ec_heal_done (int ret, call_frame_t *heal, void *opaque)
+ec_heal_done(int ret, call_frame_t *heal, void *opaque)
 {
-        if (opaque)
-                ec_fop_data_release (opaque);
-        return 0;
+    if (opaque)
+        ec_fop_data_release(opaque);
+    return 0;
 }
 
-ec_fop_data_t*
-__ec_dequeue_heals (ec_t *ec)
+ec_fop_data_t *
+__ec_dequeue_heals(ec_t *ec)
 {
-        ec_fop_data_t *fop = NULL;
+    ec_fop_data_t *fop = NULL;
 
-        if (list_empty (&ec->heal_waiting))
-                goto none;
+    if (list_empty(&ec->heal_waiting))
+        goto none;
 
-        if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals))
-                goto none;
+    if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals))
+        goto none;
 
-        fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer);
-        ec->heal_waiters--;
-        list_del_init(&fop->healer);
-        list_add(&fop->healer, &ec->healing);
-        ec->healers++;
-        return fop;
+    fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer);
+    ec->heal_waiters--;
+    list_del_init(&fop->healer);
+    list_add(&fop->healer, &ec->healing);
+    ec->healers++;
+    return fop;
 none:
-        gf_msg_debug (ec->xl->name, 0, "Num healers: %d, Num Waiters: %d",
-                      ec->healers, ec->heal_waiters);
-        return NULL;
+    gf_msg_debug(ec->xl->name, 0, "Num healers: %d, Num Waiters: %d",
+                 ec->healers, ec->heal_waiters);
+    return NULL;
 }
 
 void
-ec_heal_fail (ec_t *ec, ec_fop_data_t *fop)
+ec_heal_fail(ec_t *ec, ec_fop_data_t *fop)
 {
-        if (fop->cbks.heal) {
-            fop->cbks.heal (fop->req_frame, NULL, ec->xl, -1, fop->error, 0, 0,
-                            0, NULL);
-        }
-        if (fop)
-            ec_fop_data_release (fop);
+    if (fop->cbks.heal) {
+        fop->cbks.heal(fop->req_frame, fop->data, ec->xl, -1, fop->error, 0, 0,
+                       0, 0, NULL);
+    }
+    ec_fop_data_release(fop);
 }
 
 void
-ec_launch_heal (ec_t *ec, ec_fop_data_t *fop)
+ec_launch_heal(ec_t *ec, ec_fop_data_t *fop)
 {
-        int     ret = 0;
+    int ret = 0;
+    call_frame_t *frame = NULL;
 
-        ret = synctask_new (ec->xl->ctx->env, ec_synctask_heal_wrap,
-                            ec_heal_done, NULL, fop);
-        if (ret < 0) {
-                ec_fop_set_error(fop, ENOMEM);
-                ec_heal_fail (ec, fop);
-        }
+    frame = create_frame(ec->xl, ec->xl->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ec_owner_set(frame, frame->root);
+    /*Do heal as root*/
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    /*Mark the fops as internal*/
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+
+    ret = synctask_new(ec->xl->ctx->env, ec_synctask_heal_wrap, ec_heal_done,
+                       frame, fop);
+out:
+    if (ret < 0) {
+        ec_fop_set_error(fop, ENOMEM);
+        ec_heal_fail(ec, fop);
+    }
+
+    if (frame)
+        STACK_DESTROY(frame->root);
 }
 
 void
-ec_handle_healers_done (ec_fop_data_t *fop)
+ec_handle_healers_done(ec_fop_data_t *fop)
 {
-        ec_t *ec = fop->xl->private;
-        ec_fop_data_t *heal_fop = NULL;
+    ec_t *ec = fop->xl->private;
+    ec_fop_data_t *heal_fop = NULL;
 
-        if (list_empty (&fop->healer))
-                return;
+    if (list_empty(&fop->healer))
+        return;
 
-        LOCK (&ec->lock);
-        {
-                list_del_init (&fop->healer);
-                ec->healers--;
-                heal_fop = __ec_dequeue_heals (ec);
+    LOCK(&ec->lock);
+
+    list_del_init(&fop->healer);
+
+    do {
+        ec->healers--;
+        heal_fop = __ec_dequeue_heals(ec);
+
+        if ((heal_fop != NULL) && ec->shutdown) {
+            /* This will prevent ec_handle_healers_done() to be
+             * called recursively. That would be problematic if
+             * the queue is too big. */
+            list_del_init(&heal_fop->healer);
+
+            UNLOCK(&ec->lock);
+
+            ec_fop_set_error(fop, ENOTCONN);
+            ec_heal_fail(ec, heal_fop);
+
+            LOCK(&ec->lock);
         }
-        UNLOCK (&ec->lock);
+    } while ((heal_fop != NULL) && ec->shutdown);
 
-        if (heal_fop)
-                ec_launch_heal (ec, heal_fop);
+    UNLOCK(&ec->lock);
 
+    if (heal_fop)
+        ec_launch_heal(ec, heal_fop);
+}
+
+gf_boolean_t
+ec_is_entry_healing(ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    int32_t heal_count = 0;
+    loc_t *loc = NULL;
+
+    loc = &fop->loc[0];
+
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            heal_count = ctx->heal_count;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+    GF_ASSERT(heal_count >= 0);
+    return heal_count;
 }
 
 void
-ec_heal_throttle (xlator_t *this, ec_fop_data_t *fop)
-{
-        gf_boolean_t can_heal = _gf_true;
-        ec_t         *ec      = this->private;
-
-        if (fop->req_frame == NULL) {
-
-                LOCK (&ec->lock);
-                {
-                        if ((ec->background_heals > 0) &&
-                            (ec->heal_wait_qlen + ec->background_heals) >
-                                             (ec->heal_waiters + ec->healers)) {
-                                list_add_tail(&fop->healer, &ec->heal_waiting);
-                                ec->heal_waiters++;
-                                fop = __ec_dequeue_heals (ec);
-                        } else {
-                                can_heal = _gf_false;
-                        }
+ec_heal_throttle(xlator_t *this, ec_fop_data_t *fop)
+{
+    gf_boolean_t can_heal = _gf_true;
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop_rel = NULL;
+
+    if (fop->req_frame == NULL) {
+        LOCK(&ec->lock);
+        {
+            if ((ec->background_heals > 0) &&
+                (ec->heal_wait_qlen + ec->background_heals) >
+                    (ec->heal_waiters + ec->healers)) {
+                if (!ec_is_entry_healing(fop)) {
+                    list_add_tail(&fop->healer, &ec->heal_waiting);
+                    ec->heal_waiters++;
+                    ec_set_entry_healing(fop);
+                } else {
+                    fop_rel = fop;
                 }
-                UNLOCK (&ec->lock);
+                fop = __ec_dequeue_heals(ec);
+            } else {
+                can_heal = _gf_false;
+            }
         }
+        UNLOCK(&ec->lock);
+    }
 
-        if (can_heal) {
-                if (fop)
-                        ec_launch_heal (ec, fop);
-        } else {
-                gf_msg_debug (this->name, 0, "Max number of heals are "
-                              "pending, background self-heal rejected");
-                ec_fop_set_error(fop, EBUSY);
-                ec_heal_fail (ec, fop);
+    if (can_heal) {
+        if (fop) {
+            if (fop->req_frame != NULL) {
+                ec_set_entry_healing(fop);
+            }
+            ec_launch_heal(ec, fop);
         }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Max number of heals are "
+                     "pending, background self-heal rejected");
+        ec_fop_set_error(fop, EBUSY);
+        ec_heal_fail(ec, fop);
+    }
+    if (fop_rel) {
+        ec_heal_done(0, NULL, fop_rel);
+    }
 }
 
 void
-ec_heal (call_frame_t *frame, xlator_t *this, uintptr_t target,
-         int32_t minimum, fop_heal_cbk_t func, void *data, loc_t *loc,
-         int32_t partial, dict_t *xdata)
+ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc,
+        int32_t partial, dict_t *xdata)
 {
-    ec_cbk_t callback = { .heal = func };
+    ec_cbk_t callback = {.heal = func};
     ec_fop_data_t *fop = NULL;
     int32_t err = EINVAL;
 
-    gf_msg_trace ("ec", 0, "EC(HEAL) %p", frame);
+    gf_msg_trace("ec", 0, "EC(HEAL) %p", frame);
 
     VALIDATE_OR_GOTO(this, fail);
     GF_VALIDATE_OR_GOTO(this->name, this->private, fail);
 
-    if (!loc || !loc->inode || gf_uuid_is_null (loc->inode->gfid))
-            goto fail;
+    if (!loc || !loc->inode || gf_uuid_is_null(loc->inode->gfid))
+        goto fail;
 
     if (frame && frame->local)
-            goto fail;
-    fop = ec_fop_data_allocate (frame, this, EC_FOP_HEAL, 0, target, minimum,
-                                NULL, NULL, callback, data);
+        goto fail;
+    fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags,
+                               NULL, NULL, callback, data);
 
     err = ENOMEM;
 
@@ -2544,73 +2908,460 @@ ec_heal (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata)
         fop->xdata = dict_ref(xdata);
 
-    ec_heal_throttle (this, fop);
+    ec_heal_throttle(this, fop);
 
     return;
 
 fail:
     if (fop)
-            ec_fop_data_release (fop);
+        ec_fop_data_release(fop);
     if (func)
-            func (frame, NULL, this, -1, err, 0, 0, 0, NULL);
+        func(frame, data, this, -1, err, 0, 0, 0, 0, NULL);
 }
 
 int
-ec_replace_heal_done (int ret, call_frame_t *heal, void *opaque)
+ec_replace_heal_done(int ret, call_frame_t *heal, void *opaque)
 {
-        ec_t *ec = opaque;
+    ec_t *ec = opaque;
+    gf_boolean_t last_fop = _gf_false;
 
-        gf_msg_debug (ec->xl->name, 0,
-                "getxattr on bricks is done ret %d", ret);
-        return 0;
+    if (GF_ATOMIC_DEC(ec->async_fop_count) == 0) {
+        LOCK(&ec->lock);
+        {
+            last_fop = __ec_is_last_fop(ec);
+        }
+        UNLOCK(&ec->lock);
+    }
+    gf_msg_debug(ec->xl->name, 0, "getxattr on bricks is done ret %d", ret);
+
+    if (last_fop)
+        ec_pending_fops_completed(ec);
+
+    return 0;
 }
 
 int32_t
-ec_replace_heal (ec_t *ec, inode_t *inode)
+ec_replace_heal(ec_t *ec, inode_t *inode)
 {
-        loc_t        loc = {0};
-        int          ret = 0;
+    loc_t loc = {0};
+    int ret = 0;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = syncop_getxattr(ec->xl, &loc, NULL, EC_XATTR_HEAL, NULL, NULL);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d",
+                     ret);
+
+    /* Once the root inode has been checked, it might have triggered a
+     * self-heal on it after a replace brick command or for some other
+     * reason. It can also happen that the volume already had damaged
+     * files in the index, even if the heal on the root directory failed.
+     * In both cases we need to wake all index healers to continue
+     * healing remaining entries that are marked as dirty. */
+    ec_shd_index_healer_wake(ec);
+
+    loc_wipe(&loc);
+    return ret;
+}
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        ret = syncop_getxattr (ec->xl, &loc, NULL, EC_XATTR_HEAL,
-                               NULL, NULL);
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0,
-                        "Heal failed for replace brick ret = %d", ret);
+int32_t
+ec_replace_brick_heal_wrap(void *opaque)
+{
+    ec_t *ec = opaque;
+    inode_table_t *itable = NULL;
+    int32_t ret = -1;
+
+    if (ec->xl->itable)
+        itable = ec->xl->itable;
+    else
+        goto out;
+
+    if (xlator_is_cleanup_starting(ec->xl))
+        goto out;
 
-        loc_wipe (&loc);
-        return ret;
+    ret = ec_replace_heal(ec, itable->root);
+out:
+    return ret;
 }
 
 int32_t
-ec_replace_brick_heal_wrap (void  *opaque)
+ec_launch_replace_heal(ec_t *ec)
+{
+    int ret = -1;
+
+    ret = synctask_new(ec->xl->ctx->env, ec_replace_brick_heal_wrap,
+                       ec_replace_heal_done, NULL, ec);
+
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d",
+                     ret);
+        ec_replace_heal_done(-1, NULL, ec);
+    }
+
+    return ret;
+}
+
+int32_t
+ec_set_heal_info(dict_t **dict_rsp, char *status)
+{
+    dict_t *dict = NULL;
+    int ret = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = dict_set_str(dict, "heal-info", status);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_WARNING, -ret, EC_MSG_HEAL_FAIL,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
+        dict_unref(dict);
+        dict = NULL;
+    }
+    *dict_rsp = dict;
+out:
+    return ret;
+}
+
+static int32_t
+_need_heal_calculate(ec_t *ec, uint64_t *dirty, unsigned char *sources,
+                     gf_boolean_t self_locked, int32_t lock_count,
+                     ec_heal_need_t *need_heal, uint64_t *versions)
+{
+    int i = 0;
+    int source_count = 0;
+
+    source_count = EC_COUNT(sources, ec->nodes);
+    if (source_count == ec->nodes) {
+        *need_heal = EC_HEAL_NONEED;
+        if (self_locked || lock_count == 0) {
+            for (i = 0; i < ec->nodes; i++) {
+                if (dirty[i] || (versions[i] != versions[0])) {
+                    *need_heal = EC_HEAL_MUST;
+                    goto out;
+                }
+            }
+            /* If lock count is 0, all dirty flags are 0 and all the
+             * versions are macthing then why are we here. It looks
+             * like something went wrong while removing the index entries
+             * after completing a successful heal or fop. In this case
+             * we need to remove this index entry to avoid triggering heal
+             * in a loop and causing lookups again and again*/
+            *need_heal = EC_HEAL_PURGE_INDEX;
+        } else {
+            for (i = 0; i < ec->nodes; i++) {
+                /* Since each lock can only increment the dirty
+                 * count once, if dirty is > 1 it means that
+                 * another operation has left the dirty count
+                 * set and this indicates a problem in the
+                 * inode.*/
+                if (dirty[i] > 1) {
+                    *need_heal = EC_HEAL_MUST;
+                    goto out;
+                }
+                if (dirty[i] != dirty[0] || (versions[i] != versions[0])) {
+                    *need_heal = EC_HEAL_MAYBE;
+                }
+            }
+        }
+    } else {
+        *need_heal = EC_HEAL_MUST;
+    }
+
+out:
+    return source_count;
+}
+
+static int32_t
+ec_need_metadata_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                      int32_t lock_count, gf_boolean_t self_locked,
+                      gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *meta_versions = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    meta_versions = alloca0(ec->nodes * sizeof(*meta_versions));
+    ret = ec_heal_metadata_find_direction(ec, replies, meta_versions, dirty,
+                                          sources, healed_sinks);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
+
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, meta_versions);
+out:
+    return ret;
+}
+
+static int32_t
+ec_need_data_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                  int32_t lock_count, gf_boolean_t self_locked,
+                  gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *data_versions = NULL;
+    uint64_t *size = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    data_versions = alloca0(ec->nodes * sizeof(*data_versions));
+    size = alloca0(ec->nodes * sizeof(*size));
+
+    /* When dd is going on and heal info is called there is a very good
+     * chance for on disk sizes to mismatch even though nothing is wrong
+     * we don't need ondisk size check there. But if the file is either
+     * self-locked or the caller wants a thorough check then make sure to
+     * perform on disk check also. */
+    ret = ec_heal_data_find_direction(
+        ec, replies, data_versions, dirty, size, sources, healed_sinks,
+        self_locked || thorough, EC_COMBINE_XDATA);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
+
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, data_versions);
+out:
+    return ret;
+}
+
+static int32_t
+ec_need_entry_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                   int32_t lock_count, gf_boolean_t self_locked,
+                   gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *data_versions = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    data_versions = alloca0(ec->nodes * sizeof(*data_versions));
+
+    ret = ec_heal_entry_find_direction(ec, replies, data_versions, dirty,
+                                       sources, healed_sinks);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
+
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, data_versions);
+out:
+    return ret;
+}
+
+static int32_t
+ec_need_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+             int32_t lock_count, gf_boolean_t self_locked,
+             gf_boolean_t thorough, ec_heal_need_t *need_heal)
 {
-         ec_t *ec = opaque;
-         inode_table_t   *itable = NULL;
-         int32_t         ret     = -1;
+    int ret = 0;
+
+    ret = ec_need_metadata_heal(ec, inode, replies, lock_count, self_locked,
+                                thorough, need_heal);
+    if (ret < 0)
+        goto out;
+
+    if (*need_heal == EC_HEAL_MUST)
+        goto out;
+
+    if (inode->ia_type == IA_IFREG) {
+        ret = ec_need_data_heal(ec, inode, replies, lock_count, self_locked,
+                                thorough, need_heal);
+    } else if (inode->ia_type == IA_IFDIR) {
+        ret = ec_need_entry_heal(ec, inode, replies, lock_count, self_locked,
+                                 thorough, need_heal);
+    }
 
-         if (ec->xl->itable)
-                 itable = ec->xl->itable;
-         else
-                 goto out;
-         ret = ec_replace_heal (ec, itable->root);
 out:
-         return ret;
+    return ret;
 }
 
 int32_t
-ec_launch_replace_heal (ec_t *ec)
+ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *locked_on, gf_boolean_t self_locked,
+                gf_boolean_t thorough, ec_heal_need_t *need_heal)
 {
-	int ret = -1;
+    loc_t loc = {0};
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    uint64_t zero_array[2] = {0};
+    uint64_t zero_value = 0;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    int32_t lock_count = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    xdata = dict_new();
+    if (!xdata ||
+        dict_set_static_bin(xdata, EC_XATTR_VERSION, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xdata, EC_XATTR_DIRTY, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xdata, EC_XATTR_SIZE, &zero_value,
+                            sizeof(zero_value))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        if (!ec)
-                return ret;
-        ret = synctask_new (ec->xl->ctx->env, ec_replace_brick_heal_wrap,
-                            ec_replace_heal_done, NULL, ec);
-        if (ret < 0) {
-                gf_msg_debug (ec->xl->name, 0,
-                        "Heal failed for replace brick ret = %d", ret);
+    if (!self_locked) {
+        ret = dict_set_str(xdata, GLUSTERFS_INODELK_DOM_COUNT, ec->xl->name);
+        if (ret) {
+            ret = -ENOMEM;
+            goto out;
+        }
+    }
+
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, xdata);
+
+    if (ret != ec->nodes) {
+        ret = ec->nodes;
+        *need_heal = EC_HEAL_MUST;
+        goto out;
+    }
+
+    if (self_locked)
+        goto need_heal;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i] || !replies[i].xdata) {
+            continue;
+        }
+        if ((dict_get_int32(replies[i].xdata, GLUSTERFS_INODELK_COUNT,
+                            &lock_count) == 0) &&
+            lock_count > 0) {
+            break;
         }
-        return ret;
+    }
+need_heal:
+    ret = ec_need_heal(ec, inode, replies, lock_count, self_locked, thorough,
+                       need_heal);
+out:
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    if (xdata) {
+        dict_unref(xdata);
+    }
+    return ret;
+}
+
+int32_t
+ec_heal_locked_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                       ec_heal_need_t *need_heal)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+
+    ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0);
+    if (ret != ec->nodes) {
+        *need_heal = EC_HEAL_MUST;
+        goto unlock;
+    }
+    ret = ec_heal_inspect(frame, ec, inode, locked_on, _gf_true, _gf_true,
+                          need_heal);
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
+}
+
+int32_t
+ec_get_heal_info(xlator_t *this, loc_t *entry_loc, dict_t **dict_rsp)
+{
+    int ret = -ENOMEM;
+    ec_heal_need_t need_heal = EC_HEAL_NONEED;
+    call_frame_t *frame = NULL;
+    ec_t *ec = NULL;
+    unsigned char *up_subvols = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry_loc, out);
+
+    ec = this->private;
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+
+    if (EC_COUNT(up_subvols, ec->nodes) != ec->nodes) {
+        need_heal = EC_HEAL_MUST;
+        goto set_heal;
+    }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        goto out;
+    }
+    ec_owner_set(frame, frame->root);
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+
+    if (loc_copy(&loc, entry_loc) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+               "Failed to copy a location.");
+        goto out;
+    }
+    if (!loc.inode) {
+        ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL);
+        if (ret < 0)
+            goto out;
+    }
+
+    ret = ec_heal_inspect(frame, ec, loc.inode, up_subvols, _gf_false,
+                          _gf_false, &need_heal);
+    if (ret == ec->nodes && need_heal != EC_HEAL_MAYBE) {
+        goto set_heal;
+    }
+    need_heal = EC_HEAL_NONEED;
+    ret = ec_heal_locked_inspect(frame, ec, loc.inode, &need_heal);
+    if (ret < 0)
+        goto out;
+set_heal:
+    if (need_heal == EC_HEAL_MUST) {
+        ret = ec_set_heal_info(dict_rsp, "heal");
+    } else {
+        ret = ec_set_heal_info(dict_rsp, "no-heal");
+    }
+out:
+    if (frame) {
+        STACK_DESTROY(frame->root);
+    }
+    loc_wipe(&loc);
+    return ret;
 }
diff --git a/xlators/cluster/ec/src/ec-heald.c b/xlators/cluster/ec/src/ec-heald.c
index 0e8076826c6..5c1586bc9c5 100644
--- a/xlators/cluster/ec/src/ec-heald.c
+++ b/xlators/cluster/ec/src/ec-heald.c
@@ -8,600 +8,674 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-errno.h>
 #include "ec.h"
 #include "ec-messages.h"
 #include "ec-heald.h"
 #include "ec-mem-types.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 #include "protocol-common.h"
 
-#define ASSERT_LOCAL(this, healer)				        \
-        do {                                                            \
-                if (!ec_shd_is_subvol_local (this, healer->subvol)) {	\
-                        healer->local = _gf_false;			\
-                        if (safe_break (healer)) {			\
-                                break;					\
-                        } else {					\
-                                continue;				\
-                        }						\
-                } else {						\
-                        healer->local = _gf_true;			\
-                }                                                       \
-        } while (0);
-
-
-#define NTH_INDEX_HEALER(this, n) (&((((ec_t *)this->private))->shd.index_healers[n]))
-#define NTH_FULL_HEALER(this, n) (&((((ec_t *)this->private))->shd.full_healers[n]))
+#define NTH_INDEX_HEALER(this, n)                                              \
+    (&((((ec_t *)this->private))->shd.index_healers[n]))
+#define NTH_FULL_HEALER(this, n)                                               \
+    (&((((ec_t *)this->private))->shd.full_healers[n]))
 
 gf_boolean_t
-ec_shd_is_subvol_local (xlator_t *this, int subvol)
+ec_shd_is_subvol_local(xlator_t *this, int subvol)
 {
-        ec_t *ec = NULL;
-        gf_boolean_t is_local = _gf_false;
-        loc_t loc = {0, };
-
-        ec = this->private;
-        loc.inode = this->itable->root;
-        syncop_is_subvol_local (ec->xl_list[subvol], &loc, &is_local);
-        return is_local;
+    ec_t *ec = NULL;
+    gf_boolean_t is_local = _gf_false;
+    loc_t loc = {
+        0,
+    };
+
+    ec = this->private;
+    loc.inode = this->itable->root;
+    syncop_is_subvol_local(ec->xl_list[subvol], &loc, &is_local);
+    return is_local;
 }
 
 char *
-ec_subvol_name (xlator_t *this, int subvol)
+ec_subvol_name(xlator_t *this, int subvol)
 {
-        ec_t *ec = NULL;
+    ec_t *ec = NULL;
 
-        ec = this->private;
-        if (subvol < 0 || subvol > ec->nodes)
-                return NULL;
+    ec = this->private;
+    if (subvol < 0 || subvol > ec->nodes)
+        return NULL;
 
-        return ec->xl_list[subvol]->name;
+    return ec->xl_list[subvol]->name;
 }
 
 int
-__ec_shd_healer_wait (struct subvol_healer *healer)
+__ec_shd_healer_wait(struct subvol_healer *healer)
 {
-        ec_t *ec = NULL;
-        struct timespec wait_till = {0, };
-        int ret = 0;
+    ec_t *ec = NULL;
+    struct timespec wait_till = {
+        0,
+    };
+    int ret = 0;
 
-        ec = healer->this->private;
+    ec = healer->this->private;
 
 disabled_loop:
-        wait_till.tv_sec = time (NULL) + 60;
-
-        while (!healer->rerun) {
-                ret = pthread_cond_timedwait (&healer->cond,
-                                              &healer->mutex,
-                                              &wait_till);
-                if (ret == ETIMEDOUT)
-                        break;
-        }
+    wait_till.tv_sec = gf_time() + ec->shd.timeout;
 
-        ret = healer->rerun;
-        healer->rerun = 0;
-
-        if (!ec->shd.enabled || !ec->up)
-                goto disabled_loop;
-
-        return ret;
-}
-
-
-int
-ec_shd_healer_wait (struct subvol_healer *healer)
-{
-        int ret = 0;
-
-        pthread_mutex_lock (&healer->mutex);
-        {
-                ret = __ec_shd_healer_wait (healer);
-        }
-        pthread_mutex_unlock (&healer->mutex);
+    while (!healer->rerun) {
+        ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
+        if (ret == ETIMEDOUT)
+            break;
+    }
 
-        return ret;
-}
-
-
-gf_boolean_t
-safe_break (struct subvol_healer *healer)
-{
-        gf_boolean_t ret = _gf_false;
+    if (ec->shutdown) {
+        healer->running = _gf_false;
+        return -1;
+    }
 
-        pthread_mutex_lock (&healer->mutex);
-        {
-                if (healer->rerun)
-                        goto unlock;
+    ret = healer->rerun;
+    healer->rerun = 0;
 
-                healer->running = _gf_false;
-                ret = _gf_true;
-        }
-unlock:
-        pthread_mutex_unlock (&healer->mutex);
+    if (!ec->shd.enabled || !ec->up)
+        goto disabled_loop;
 
-        return ret;
+    return ret;
 }
 
-
 int
-ec_shd_inode_find (xlator_t *this, xlator_t *subvol,
-                   uuid_t gfid, inode_t **inode)
+ec_shd_healer_wait(struct subvol_healer *healer)
 {
-        int         ret    = 0;
-        loc_t       loc    = {0, };
-        struct iatt iatt   = {0, };
-	*inode =  NULL;
-
-        *inode = inode_find (this->itable, gfid);
-        if (*inode)
-                goto out;
-
-        loc.inode = inode_new (this->itable);
-        if (!loc.inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        gf_uuid_copy (loc.gfid, gfid);
+    int ret = 0;
 
-        ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
-        if (ret < 0)
-                goto out;
+    pthread_mutex_lock(&healer->mutex);
+    {
+        ret = __ec_shd_healer_wait(healer);
+    }
+    pthread_mutex_unlock(&healer->mutex);
 
-        *inode = inode_link (loc.inode, NULL, NULL, &iatt);
-        if (!*inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
-out:
-        loc_wipe (&loc);
-        return ret;
+    return ret;
 }
 
-
 int
-ec_shd_index_inode (xlator_t *this, xlator_t *subvol, inode_t **inode)
+ec_shd_index_inode(xlator_t *this, xlator_t *subvol, inode_t **inode)
 {
-        loc_t   rootloc     = {0, };
-        int     ret         = 0;
-        dict_t  *xattr      = NULL;
-        void    *index_gfid = NULL;
-
-        *inode = NULL;
-        rootloc.inode = inode_ref (this->itable->root);
-        gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
-
-        ret = syncop_getxattr (subvol, &rootloc, &xattr,
-                               GF_XATTROP_INDEX_GFID, NULL, NULL);
-        if (ret < 0)
-                goto out;
-        if (!xattr) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
-        if (ret)
-                goto out;
-
-        gf_msg_debug (this->name, 0, "index-dir gfid for %s: %s",
-                subvol->name, uuid_utoa (index_gfid));
-
-        ret = ec_shd_inode_find (this, subvol, index_gfid, inode);
+    loc_t rootloc = {
+        0,
+    };
+    int ret = 0;
+    dict_t *xattr = NULL;
+    void *index_gfid = NULL;
+
+    *inode = NULL;
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
+
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_GFID, NULL,
+                          NULL);
+    if (ret < 0)
+        goto out;
+    if (!xattr) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+    if (ret)
+        goto out;
+
+    gf_msg_debug(this->name, 0, "index-dir gfid for %s: %s", subvol->name,
+                 uuid_utoa(index_gfid));
+
+    ret = syncop_inode_find(this, subvol, index_gfid, inode, NULL, NULL);
 
 out:
-        loc_wipe (&rootloc);
+    loc_wipe(&rootloc);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+ec_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name)
 {
-        loc_t loc = {0, };
-        int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
 
-        loc.parent = inode_ref (inode);
-        loc.name = name;
+    loc.parent = inode_ref(inode);
+    loc.name = name;
 
-        ret = syncop_unlink (subvol, &loc, NULL, NULL);
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
 
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
-int
-ec_shd_selfheal (struct subvol_healer *healer, int child, loc_t *loc)
+static gf_boolean_t
+ec_is_heal_completed(char *status)
 {
-        return syncop_getxattr (healer->this, loc, NULL, EC_XATTR_HEAL, NULL,
-                                NULL);
+    char *bad_pos = NULL;
+    char *zero_pos = NULL;
+
+    if (!status) {
+        return _gf_false;
+    }
+
+    /*Logic:
+     * Status will be of the form Good: <binary>, Bad: <binary>
+     * If heal completes, if we do strchr for '0' it should be present after
+     * 'Bad:' i.e. strRchr for ':'
+     * */
+
+    zero_pos = strchr(status, '0');
+    bad_pos = strrchr(status, ':');
+    if (!zero_pos || !bad_pos) {
+        /*malformed status*/
+        return _gf_false;
+    }
+
+    if (zero_pos > bad_pos) {
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
-
 int
-ec_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+ec_shd_selfheal(struct subvol_healer *healer, int child, loc_t *loc,
+                gf_boolean_t full)
 {
-        struct subvol_healer *healer = data;
-        ec_t                 *ec     = NULL;
-        loc_t                loc     = {0};
-        int                  ret     = 0;
+    dict_t *xdata = NULL;
+    dict_t *dict = NULL;
+    uint32_t count;
+    int32_t ret;
+    char *heal_status = NULL;
+    ec_t *ec = healer->this->private;
+
+    GF_ATOMIC_INC(ec->stats.shd.attempted);
+    ret = syncop_getxattr(healer->this, loc, &dict, EC_XATTR_HEAL, NULL,
+                          &xdata);
+    if (ret == 0) {
+        if (dict && (dict_get_str(dict, EC_XATTR_HEAL, &heal_status) == 0)) {
+            if (ec_is_heal_completed(heal_status)) {
+                GF_ATOMIC_INC(ec->stats.shd.completed);
+            }
+        }
+    }
+
+    if (!full && (loc->inode->ia_type == IA_IFDIR)) {
+        /* If we have just healed a directory, it's possible that
+         * other index entries have appeared to be healed. */
+        if ((xdata != NULL) &&
+            (dict_get_uint32(xdata, EC_XATTR_HEAL_NEW, &count) == 0) &&
+            (count > 0)) {
+            /* Force a rerun of the index healer. */
+            gf_msg_debug(healer->this->name, 0, "%d more entries to heal",
+                         count);
+
+            healer->rerun = _gf_true;
+        }
+    }
 
-        ec = healer->this->private;
-        if (!ec->shd.enabled)
-                return -EBUSY;
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
-        gf_msg_debug (healer->this->name, 0, "got entry: %s",
-                entry->d_name);
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        ret = gf_uuid_parse (entry->d_name, loc.gfid);
-        if (ret)
-                return 0;
+    return ret;
+}
 
-        /* If this fails with ENOENT/ESTALE index is stale */
-        ret = syncop_gfid_to_path (healer->this->itable, subvol, loc.gfid,
-                                   (char **)&loc.path);
-        if (ret < 0)
-                goto out;
+int
+ec_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                  void *data)
+{
+    struct subvol_healer *healer = data;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = 0;
+
+    ec = healer->this->private;
+    if (ec->xl_up_count <= ec->fragments) {
+        return -ENOTCONN;
+    }
+    if (!ec->shd.enabled)
+        return -EBUSY;
+
+    gf_msg_debug(healer->this->name, 0, "got entry: %s", entry->d_name);
+
+    ret = gf_uuid_parse(entry->d_name, loc.gfid);
+    if (ret)
+        return 0;
 
-        ret = ec_shd_inode_find (healer->this, healer->this, loc.gfid,
-                                  &loc.inode);
-        if (ret < 0)
-                goto out;
+    /* If this fails with ENOENT/ESTALE index is stale */
+    ret = syncop_gfid_to_path(healer->this->itable, subvol, loc.gfid,
+                              (char **)&loc.path);
+    if (ret < 0)
+        goto out;
 
-        ec_shd_selfheal (healer, healer->subvol, &loc);
-out:
-        if (ret == -ENOENT || ret == -ESTALE) {
-                gf_msg (healer->this->name, GF_LOG_DEBUG, 0,
-                        EC_MSG_HEAL_FAIL, "Purging index for gfid %s:",
-                        uuid_utoa(loc.gfid));
-                ec_shd_index_purge (subvol, parent->inode, entry->d_name);
-        }
-        loc_wipe (&loc);
+    ret = syncop_inode_find(healer->this, healer->this, loc.gfid, &loc.inode,
+                            NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-        return 0;
+    ec_shd_selfheal(healer, healer->subvol, &loc, _gf_false);
+out:
+    if (ret == -ENOENT || ret == -ESTALE) {
+        gf_msg(healer->this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL,
+               "Purging index for gfid %s:", uuid_utoa(loc.gfid));
+        ec_shd_index_purge(subvol, parent->inode, entry->d_name);
+    }
+    loc_wipe(&loc);
+
+    return 0;
 }
 
 int
-ec_shd_index_sweep (struct subvol_healer *healer)
+ec_shd_index_sweep(struct subvol_healer *healer)
 {
-        loc_t         loc     = {0};
-        ec_t          *ec     = NULL;
-        int           ret     = 0;
-        xlator_t      *subvol = NULL;
-
-        ec = healer->this->private;
-        subvol = ec->xl_list[healer->subvol];
-
-        ret = ec_shd_index_inode (healer->this, subvol, &loc.inode);
-        if (ret < 0) {
-                gf_msg (healer->this->name, GF_LOG_WARNING, errno,
-                        EC_MSG_INDEX_DIR_GET_FAIL,
-                        "unable to get index-dir on %s", subvol->name);
-                goto out;
-        }
-
-        ret = syncop_dir_scan (subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
-                               healer, ec_shd_index_heal);
+    loc_t loc = {0};
+    ec_t *ec = NULL;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    dict_t *xdata = NULL;
+
+    ec = healer->this->private;
+    subvol = ec->xl_list[healer->subvol];
+
+    ret = ec_shd_index_inode(healer->this, subvol, &loc.inode);
+    if (ret < 0) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, errno,
+               EC_MSG_INDEX_DIR_GET_FAIL, "unable to get index-dir on %s",
+               subvol->name);
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int32(xdata, "get-gfid-type", 1)) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    _mask_cancellation();
+    ret = syncop_mt_dir_scan(NULL, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
+                             healer, ec_shd_index_heal, xdata,
+                             ec->shd.max_threads, ec->shd.wait_qlength);
+    _unmask_cancellation();
 out:
-        loc_wipe (&loc);
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_full_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+ec_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                 void *data)
 {
-        struct subvol_healer *healer = data;
-        xlator_t             *this   = healer->this;
-        ec_t                 *ec     = NULL;
-        loc_t                loc    = {0};
-        int                  ret    = 0;
-
-        ec = this->private;
-        if (!ec->shd.enabled)
-                return -EBUSY;
+    struct subvol_healer *healer = data;
+    xlator_t *this = healer->this;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = 0;
+
+    ec = this->private;
+
+    if (this->cleanup_starting) {
+        return -ENOTCONN;
+    }
+
+    if (ec->xl_up_count <= ec->fragments) {
+        return -ENOTCONN;
+    }
+    if (!ec->shd.enabled)
+        return -EBUSY;
+
+    if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
+        /* It's possible that an entry has been removed just after
+         * being seen in a directory but before getting its stat info.
+         * In this case we'll receive a NULL gfid here. Since the file
+         * doesn't exist anymore, we can safely ignore it. */
+        return 0;
+    }
 
-        loc.parent = inode_ref (parent->inode);
-        loc.name   = entry->d_name;
-        gf_uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
+    loc.parent = inode_ref(parent->inode);
+    loc.name = entry->d_name;
+    gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid);
 
-        /* If this fails with ENOENT/ESTALE index is stale */
-        ret = syncop_gfid_to_path (this->itable, subvol, loc.gfid,
-                                   (char **)&loc.path);
-        if (ret < 0)
-                goto out;
+    /* If this fails with ENOENT/ESTALE index is stale */
+    ret = syncop_gfid_to_path(this->itable, subvol, loc.gfid,
+                              (char **)&loc.path);
+    if (ret < 0)
+        goto out;
 
-        ret = ec_shd_inode_find (this, this, loc.gfid, &loc.inode);
-        if (ret < 0)
-                goto out;
+    ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-        ec_shd_selfheal (healer, healer->subvol, &loc);
+    ec_shd_selfheal(healer, healer->subvol, &loc, _gf_true);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-ec_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
+ec_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
 {
-        ec_t           *ec  = NULL;
-        loc_t          loc  = {0};
-
-        ec = healer->this->private;
-        loc.inode = inode;
-        return syncop_ftw (ec->xl_list[healer->subvol], &loc,
-                           GF_CLIENT_PID_SELF_HEALD, healer,
-                           ec_shd_full_heal);
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = -1;
+
+    ec = healer->this->private;
+    loc.inode = inode;
+    _mask_cancellation();
+    ret = syncop_ftw(ec->xl_list[healer->subvol], &loc,
+                     GF_CLIENT_PID_SELF_HEALD, healer, ec_shd_full_heal);
+    _unmask_cancellation();
+    return ret;
 }
 
-
 void *
-ec_shd_index_healer (void *data)
+ec_shd_index_healer(void *data)
 {
-        struct subvol_healer *healer = NULL;
-        xlator_t *this = NULL;
-
-        healer = data;
-        THIS = this = healer->this;
-
-        for (;;) {
-                ec_shd_healer_wait (healer);
-
-                ASSERT_LOCAL(this, healer);
-
-                gf_msg_debug (this->name, 0,
-                        "starting index sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
-
-                ec_shd_index_sweep (healer);
-
-                gf_msg_debug (this->name, 0,
-                        "finished index sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int run = 0;
+
+    healer = data;
+    THIS = this = healer->this;
+    ec_t *ec = this->private;
+
+    for (;;) {
+        run = ec_shd_healer_wait(healer);
+        if (run == -1)
+            break;
+
+        if (ec->xl_up_count > ec->fragments) {
+            gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
+                         ec_subvol_name(this, healer->subvol));
+            ec_shd_index_sweep(healer);
         }
+        gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
+                     ec_subvol_name(this, healer->subvol));
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 void *
-ec_shd_full_healer (void *data)
+ec_shd_full_healer(void *data)
 {
-        struct subvol_healer *healer = NULL;
-        xlator_t *this = NULL;
-        loc_t rootloc = {0};
-
-        int run = 0;
-
-        healer = data;
-        THIS = this = healer->this;
-
-        rootloc.inode = this->itable->root;
-        for (;;) {
-                pthread_mutex_lock (&healer->mutex);
-                {
-                        run = __ec_shd_healer_wait (healer);
-                        if (!run)
-                                healer->running = _gf_false;
-                }
-                pthread_mutex_unlock (&healer->mutex);
-
-                if (!run)
-                        break;
-
-                ASSERT_LOCAL(this, healer);
-
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        EC_MSG_FULL_SWEEP_START,
-                        "starting full sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
-
-                ec_shd_selfheal (healer, healer->subvol, &rootloc);
-                ec_shd_full_sweep (healer, this->itable->root);
-
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        EC_MSG_FULL_SWEEP_STOP,
-                        "finished full sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    loc_t rootloc = {0};
+
+    int run = 0;
+
+    healer = data;
+    THIS = this = healer->this;
+    ec_t *ec = this->private;
+
+    rootloc.inode = this->itable->root;
+    for (;;) {
+        run = ec_shd_healer_wait(healer);
+        if (run < 0) {
+            break;
+        } else if (run == 0) {
+            continue;
         }
 
-        return NULL;
-}
+        if (ec->xl_up_count > ec->fragments) {
+            gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_START,
+                   "starting full sweep on subvol %s",
+                   ec_subvol_name(this, healer->subvol));
+
+            ec_shd_selfheal(healer, healer->subvol, &rootloc, _gf_true);
+            ec_shd_full_sweep(healer, this->itable->root);
+        }
 
+        gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_STOP,
+               "finished full sweep on subvol %s",
+               ec_subvol_name(this, healer->subvol));
+    }
+
+    return NULL;
+}
 
 int
-ec_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
+ec_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = pthread_mutex_init (&healer->mutex, NULL);
-        if (ret)
-                goto out;
+    ret = pthread_mutex_init(&healer->mutex, NULL);
+    if (ret)
+        goto out;
 
-        ret = pthread_cond_init (&healer->cond, NULL);
-        if (ret)
-                goto out;
+    ret = pthread_cond_init(&healer->cond, NULL);
+    if (ret)
+        goto out;
 
-        healer->this = this;
-        healer->running = _gf_false;
-        healer->rerun = _gf_false;
-        healer->local = _gf_false;
+    healer->this = this;
+    healer->running = _gf_false;
+    healer->rerun = _gf_false;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-ec_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
-                      void *(threadfn)(void *))
+ec_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
+                    void *(threadfn)(void *))
 {
-        int ret = 0;
-
-        pthread_mutex_lock (&healer->mutex);
-        {
-                if (healer->running) {
-                        pthread_cond_signal (&healer->cond);
-                } else {
-                        ret = gf_thread_create (&healer->thread, NULL,
-                                                threadfn, healer);
-                        if (ret)
-                                goto unlock;
-                        healer->running = 1;
-                }
-
-                healer->rerun = 1;
+    int ret = 0;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->running) {
+            pthread_cond_signal(&healer->cond);
+        } else {
+            ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
+                                   "ecshd");
+            if (ret)
+                goto unlock;
+            healer->running = 1;
         }
+
+        healer->rerun = 1;
+    }
 unlock:
-        pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_unlock(&healer->mutex);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_full_healer_spawn (xlator_t *this, int subvol)
+ec_shd_full_healer_spawn(xlator_t *this, int subvol)
 {
-        return ec_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
-                                    ec_shd_full_healer);
-}
+    if (xlator_is_cleanup_starting(this))
+        return -1;
 
+    return ec_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
+                               ec_shd_full_healer);
+}
 
 int
-ec_shd_index_healer_spawn (xlator_t *this, int subvol)
+ec_shd_index_healer_spawn(xlator_t *this, int subvol)
 {
-        return ec_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
-                                    ec_shd_index_healer);
+    if (xlator_is_cleanup_starting(this))
+        return -1;
+
+    return ec_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
+                               ec_shd_index_healer);
 }
 
 void
-ec_selfheal_childup (ec_t *ec, int child)
+ec_shd_index_healer_wake(ec_t *ec)
 {
-        if (!ec->shd.iamshd)
-                return;
-        ec_shd_index_healer_spawn (ec->xl, child);
+    int32_t i;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (((ec->xl_up >> i) & 1) != 0) {
+            ec_shd_index_healer_spawn(ec->xl, i);
+        }
+    }
 }
 
 int
-ec_selfheal_daemon_init (xlator_t *this)
+ec_selfheal_daemon_init(xlator_t *this)
 {
-        ec_t    *ec = NULL;
-        ec_self_heald_t *shd = NULL;
-        int ret = -1;
-        int i = 0;
-
-        ec = this->private;
-        shd = &ec->shd;
-
-        shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
-                                        ec->nodes,
-                                        ec_mt_subvol_healer_t);
-        if (!shd->index_healers)
-                goto out;
-
-        for (i = 0; i < ec->nodes; i++) {
-                shd->index_healers[i].subvol = i;
-                ret = ec_shd_healer_init (this, &shd->index_healers[i]);
-                if (ret)
-                        goto out;
-        }
+    ec_t *ec = NULL;
+    ec_self_heald_t *shd = NULL;
+    int ret = -1;
+    int i = 0;
+
+    ec = this->private;
+    shd = &ec->shd;
+
+    shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers), ec->nodes,
+                                   ec_mt_subvol_healer_t);
+    if (!shd->index_healers)
+        goto out;
+
+    for (i = 0; i < ec->nodes; i++) {
+        shd->index_healers[i].subvol = i;
+        ret = ec_shd_healer_init(this, &shd->index_healers[i]);
+        if (ret)
+            goto out;
+    }
 
-        shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
-                                       ec->nodes,
-                                       ec_mt_subvol_healer_t);
-        if (!shd->full_healers)
-                goto out;
-
-        for (i = 0; i < ec->nodes; i++) {
-                shd->full_healers[i].subvol = i;
-                ret = ec_shd_healer_init (this, &shd->full_healers[i]);
-                if (ret)
-                        goto out;
-        }
+    shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), ec->nodes,
+                                  ec_mt_subvol_healer_t);
+    if (!shd->full_healers)
+        goto out;
 
-        ret = 0;
+    for (i = 0; i < ec->nodes; i++) {
+        shd->full_healers[i].subvol = i;
+        ret = ec_shd_healer_init(this, &shd->full_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-ec_heal_op (xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id)
+ec_heal_op(xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id)
 {
-        char key[64] = {0};
-        int op_ret = 0;
-        ec_t *ec = NULL;
-        int     i = 0;
-        GF_UNUSED int     ret = 0;
-
-        ec = this->private;
-
-        op_ret = -1;
-        for (i = 0; i < ec->nodes; i++) {
-                snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-                if (((ec->xl_up >> i) & 1) == 0) {
-                        ret = dict_set_str (output, key, "Brick is not connected");
-                } else if (!ec->up) {
-                        ret = dict_set_str (output, key,
-                                            "Disperse subvolume is not up");
-                } else if (!ec_shd_is_subvol_local (this, i)) {
-                        ret = dict_set_str (output, key, "Brick is remote");
-                } else {
-                        ret = dict_set_str (output, key, "Started self-heal");
-                        if (op == GF_SHD_OP_HEAL_FULL) {
-                                ec_shd_full_healer_spawn (this, i);
-                        } else if (op == GF_SHD_OP_HEAL_INDEX) {
-                                ec_shd_index_healer_spawn (this, i);
-                        }
-                        op_ret = 0;
-                }
+    char key[64] = {0};
+    int op_ret = 0;
+    ec_t *ec = NULL;
+    int i = 0;
+    GF_UNUSED int ret = 0;
+
+    ec = this->private;
+
+    op_ret = -1;
+    for (i = 0; i < ec->nodes; i++) {
+        snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+        if (((ec->xl_up >> i) & 1) == 0) {
+            ret = dict_set_str(output, key, "Brick is not connected");
+        } else if (!ec->up) {
+            ret = dict_set_str(output, key, "Disperse subvolume is not up");
+        } else if (!ec_shd_is_subvol_local(this, i)) {
+            ret = dict_set_str(output, key, "Brick is remote");
+        } else {
+            ret = dict_set_str(output, key, "Started self-heal");
+            if (op == GF_SHD_OP_HEAL_FULL) {
+                ec_shd_full_healer_spawn(this, i);
+            } else if (op == GF_SHD_OP_HEAL_INDEX) {
+                ec_shd_index_healer_spawn(this, i);
+            }
+            op_ret = 0;
         }
-        return op_ret;
+    }
+    return op_ret;
 }
 
 int
-ec_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+ec_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 {
-        gf_xl_afr_op_t op     = GF_SHD_OP_INVALID;
-        int            ret    = 0;
-        int            xl_id  = 0;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    int xl_id = 0;
 
-        ret = dict_get_int32 (input, "xl-op", (int32_t *)&op);
-        if (ret)
-                goto out;
+    ret = dict_get_int32(input, "xl-op", (int32_t *)&op);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
+    ret = dict_get_int32(input, this->name, &xl_id);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(output, this->name, xl_id);
+    if (ret)
+        goto out;
 
-        switch (op) {
+    switch (op) {
         case GF_SHD_OP_HEAL_FULL:
-                ret = ec_heal_op (this, output, op, xl_id);
-                break;
+            ret = ec_heal_op(this, output, op, xl_id);
+            break;
 
         case GF_SHD_OP_HEAL_INDEX:
-                ret = ec_heal_op (this, output, op, xl_id);
-                break;
+            ret = ec_heal_op(this, output, op, xl_id);
+            break;
 
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 out:
-        dict_del (output, this->name);
-        return ret;
+    dict_del(output, this->name);
+    return ret;
+}
+
+void
+ec_destroy_healer_object(xlator_t *this, struct subvol_healer *healer)
+{
+    if (!healer)
+        return;
+
+    pthread_cond_destroy(&healer->cond);
+    pthread_mutex_destroy(&healer->mutex);
+}
+
+void
+ec_selfheal_daemon_fini(xlator_t *this)
+{
+    struct subvol_healer *healer = NULL;
+    ec_self_heald_t *shd = NULL;
+    ec_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+    if (!priv)
+        return;
+
+    shd = &priv->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < priv->nodes; i++) {
+        healer = &shd->index_healers[i];
+        ec_destroy_healer_object(this, healer);
+
+        healer = &shd->full_healers[i];
+        ec_destroy_healer_object(this, healer);
+    }
+
+    GF_FREE(shd->index_healers);
+    GF_FREE(shd->full_healers);
 }
diff --git a/xlators/cluster/ec/src/ec-heald.h b/xlators/cluster/ec/src/ec-heald.h
index 0f27a8ec776..6c7da4edc10 100644
--- a/xlators/cluster/ec/src/ec-heald.h
+++ b/xlators/cluster/ec/src/ec-heald.h
@@ -11,37 +11,20 @@
 #ifndef __EC_HEALD_H__
 #define __EC_HEALD_H__
 
-#include "xlator.h"
-
-struct _ec;
-typedef struct _ec ec_t;
-
-struct subvol_healer {
-        xlator_t        *this;
-        int              subvol;
-        gf_boolean_t     local;
-        gf_boolean_t     running;
-        gf_boolean_t     rerun;
-        pthread_mutex_t  mutex;
-        pthread_cond_t   cond;
-        pthread_t        thread;
-};
-
-struct _ec_self_heald;
-typedef struct _ec_self_heald ec_self_heald_t;
-
-struct _ec_self_heald {
-        gf_boolean_t            iamshd;
-        gf_boolean_t            enabled;
-        int                     timeout;
-        struct subvol_healer   *index_healers;
-        struct subvol_healer   *full_healers;
-};
+#include "ec-types.h"           // for ec_t
+#include "glusterfs/dict.h"     // for dict_t
+#include "glusterfs/globals.h"  // for xlator_t
 
 int
-ec_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+ec_xl_op(xlator_t *this, dict_t *input, dict_t *output);
 
 int
-ec_selfheal_daemon_init (xlator_t *this);
-void ec_selfheal_childup (ec_t *ec, int child);
+ec_selfheal_daemon_init(xlator_t *this);
+
+void
+ec_shd_index_healer_wake(ec_t *ec);
+
+void
+ec_selfheal_daemon_fini(xlator_t *this);
+
 #endif /* __EC_HEALD_H__ */
diff --git a/xlators/cluster/ec/src/ec-helpers.c b/xlators/cluster/ec/src/ec-helpers.c
index c8f904ac51d..48f54475e01 100644
--- a/xlators/cluster/ec/src/ec-helpers.c
+++ b/xlators/cluster/ec/src/ec-helpers.c
@@ -10,36 +10,29 @@
 
 #include <libgen.h>
 
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
+#include "ec.h"
 #include "ec-mem-types.h"
+#include "ec-messages.h"
 #include "ec-fops.h"
+#include "ec-method.h"
 #include "ec-helpers.h"
-#include "ec-messages.h"
-
-#ifndef ffsll
-#define ffsll(x) __builtin_ffsll(x)
-#endif
 
-static const char * ec_fop_list[] =
-{
-    [-EC_FOP_HEAL] = "HEAL"
-};
+static const char *ec_fop_list[] = {[-EC_FOP_HEAL] = "HEAL"};
 
-const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits)
+const char *
+ec_bin(char *str, size_t size, uint64_t value, int32_t digits)
 {
     str += size;
 
-    if (size-- < 1)
-    {
+    if (size-- < 1) {
         goto failed;
     }
     *--str = 0;
 
-    while ((value != 0) || (digits > 0))
-    {
-        if (size-- < 1)
-        {
+    while ((value != 0) || (digits > 0)) {
+        if (size-- < 1) {
             goto failed;
         }
         *--str = '0' + (value & 1);
@@ -53,21 +46,22 @@ failed:
     return "<buffer too small>";
 }
 
-const char * ec_fop_name(int32_t id)
+const char *
+ec_fop_name(int32_t id)
 {
-    if (id >= 0)
-    {
+    if (id >= 0) {
         return gf_fop_list[id];
     }
 
     return ec_fop_list[-id];
 }
 
-void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...)
+void
+ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...)
 {
     char str1[32], str2[32], str3[32];
-    char * msg;
-    ec_t * ec = fop->xl->private;
+    char *msg;
+    ec_t *ec = fop->xl->private;
     va_list args;
     int32_t ret;
 
@@ -75,45 +69,28 @@ void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...)
     ret = vasprintf(&msg, fmt, args);
     va_end(args);
 
-    if (ret < 0)
-    {
+    if (ret < 0) {
         msg = "<memory allocation error>";
     }
 
-    gf_msg_trace ("ec", 0, "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] "
-                               "frame=%p/%p, min/exp=%d/%d, err=%d state=%d "
-                               "{%s:%s:%s} %s",
-           event, ec_fop_name(fop->id), fop, fop->parent, fop->refs,
-           fop->winds, fop->jobs, fop->req_frame, fop->frame, fop->minimum,
-           fop->expected, fop->error, fop->state,
-           ec_bin(str1, sizeof(str1), fop->mask, ec->nodes),
-           ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes),
-           ec_bin(str3, sizeof(str3), fop->good, ec->nodes), msg);
+    gf_msg_trace("ec", 0,
+                 "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] "
+                 "frame=%p/%p, min/exp=%d/%d, err=%d state=%d "
+                 "{%s:%s:%s} %s",
+                 event, ec_fop_name(fop->id), fop, fop->parent, fop->refs,
+                 fop->winds, fop->jobs, fop->req_frame, fop->frame,
+                 fop->minimum, fop->expected, fop->error, fop->state,
+                 ec_bin(str1, sizeof(str1), fop->mask, ec->nodes),
+                 ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes),
+                 ec_bin(str3, sizeof(str3), fop->good, ec->nodes), msg);
 
-    if (ret >= 0)
-    {
+    if (ret >= 0) {
         free(msg);
     }
 }
 
-int32_t ec_bits_count(uint64_t n)
-{
-    n -= (n >> 1) & 0x5555555555555555ULL;
-    n = ((n >> 2) & 0x3333333333333333ULL) + (n & 0x3333333333333333ULL);
-    n = (n + (n >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
-    n += n >> 8;
-    n += n >> 16;
-    n += n >> 32;
-
-    return n & 0xFF;
-}
-
-int32_t ec_bits_index(uint64_t n)
-{
-    return ffsll(n) - 1;
-}
-
-int32_t ec_bits_consume(uint64_t * n)
+int32_t
+ec_bits_consume(uint64_t *n)
 {
     uint64_t tmp;
 
@@ -121,24 +98,21 @@ int32_t ec_bits_consume(uint64_t * n)
     tmp &= -tmp;
     *n ^= tmp;
 
-    return ffsll(tmp) - 1;
+    return gf_bits_index(tmp);
 }
 
-size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
-                      off_t offset, size_t size)
+size_t
+ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset,
+               size_t size)
 {
     int32_t i = 0;
     size_t total = 0, len = 0;
 
-    while (i < count)
-    {
-        if (offset < vector[i].iov_len)
-        {
-            while ((i < count) && (size > 0))
-            {
+    while (i < count) {
+        if (offset < vector[i].iov_len) {
+            while ((i < count) && (size > 0)) {
                 len = size;
-                if (len > vector[i].iov_len - offset)
-                {
+                if (len > vector[i].iov_len - offset) {
                     len = vector[i].iov_len - offset;
                 }
                 memcpy(dst, vector[i++].iov_base + offset, len);
@@ -158,12 +132,59 @@ size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
     return total;
 }
 
-int32_t ec_dict_set_array(dict_t *dict, char *key, uint64_t value[],
-                          int32_t size)
+int32_t
+ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr)
 {
-    int         ret = -1;
-    uint64_t   *ptr = NULL;
-    int32_t     vindex;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    int32_t ret = -ENOMEM;
+
+    iobuf = iobuf_get_page_aligned(xl->ctx->iobuf_pool, size,
+                                   EC_METHOD_WORD_SIZE);
+    if (iobuf == NULL) {
+        goto out;
+    }
+
+    iobref = *piobref;
+    if (iobref == NULL) {
+        iobref = iobref_new();
+        if (iobref == NULL) {
+            goto out;
+        }
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret != 0) {
+        if (iobref != *piobref) {
+            iobref_unref(iobref);
+        }
+        iobref = NULL;
+
+        goto out;
+    }
+
+    GF_ASSERT(EC_ALIGN_CHECK(iobuf->ptr, EC_METHOD_WORD_SIZE));
+
+    *ptr = iobuf->ptr;
+
+out:
+    if (iobuf != NULL) {
+        iobuf_unref(iobuf);
+    }
+
+    if (iobref != NULL) {
+        *piobref = iobref;
+    }
+
+    return ret;
+}
+
+int32_t
+ec_dict_set_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
+{
+    int ret = -1;
+    uint64_t *ptr = NULL;
+    int32_t vindex;
 
     if (value == NULL) {
         return -EINVAL;
@@ -174,19 +195,18 @@ int32_t ec_dict_set_array(dict_t *dict, char *key, uint64_t value[],
         return -ENOMEM;
     }
     for (vindex = 0; vindex < size; vindex++) {
-         ptr[vindex] = hton64(value[vindex]);
+        ptr[vindex] = hton64(value[vindex]);
     }
     ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t) * size);
     if (ret)
-         GF_FREE (ptr);
+        GF_FREE(ptr);
     return ret;
 }
 
-
-int32_t ec_dict_del_array(dict_t *dict, char *key, uint64_t value[],
-                          int32_t size)
+int32_t
+ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
 {
-    void    *ptr;
+    void *ptr;
     int32_t len;
     int32_t vindex;
     int32_t old_size = 0;
@@ -200,34 +220,43 @@ int32_t ec_dict_del_array(dict_t *dict, char *key, uint64_t value[],
         return err;
     }
 
-    if (len > (size * sizeof(uint64_t)) || (len % sizeof (uint64_t))) {
+    if (len > (size * sizeof(uint64_t)) || (len % sizeof(uint64_t))) {
         return -EINVAL;
     }
 
-    memset (value, 0, size * sizeof(uint64_t));
     /* 3.6 version ec would have stored version in 64 bit. In that case treat
      * metadata versions same as data*/
-    old_size = min (size, len/sizeof(uint64_t));
+    old_size = min(size, len / sizeof(uint64_t));
     for (vindex = 0; vindex < old_size; vindex++) {
-         value[vindex] = ntoh64(*((uint64_t *)ptr + vindex));
+        value[vindex] = ntoh64(*((uint64_t *)ptr + vindex));
     }
 
     if (old_size < size) {
-            for (vindex = old_size; vindex < size; vindex++) {
-                 value[vindex] = value[old_size-1];
-            }
+        for (vindex = old_size; vindex < size; vindex++) {
+            value[vindex] = value[old_size - 1];
+        }
     }
 
-    dict_del(dict, key);
-
     return 0;
 }
 
+int32_t
+ec_dict_del_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
+{
+    int ret = 0;
+
+    ret = ec_dict_get_array(dict, key, value, size);
+    if (ret == 0)
+        dict_del(dict, key);
 
-int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value)
+    return ret;
+}
+
+int32_t
+ec_dict_set_number(dict_t *dict, char *key, uint64_t value)
 {
-    int        ret = -1;
-    uint64_t * ptr;
+    int ret = -1;
+    uint64_t *ptr;
 
     ptr = GF_MALLOC(sizeof(value), gf_common_mt_char);
     if (ptr == NULL) {
@@ -238,14 +267,15 @@ int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value)
 
     ret = dict_set_bin(dict, key, ptr, sizeof(value));
     if (ret)
-        GF_FREE (ptr);
+        GF_FREE(ptr);
 
     return ret;
 }
 
-int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value)
+int32_t
+ec_dict_del_number(dict_t *dict, char *key, uint64_t *value)
 {
-    void * ptr;
+    void *ptr;
     int32_t len, err;
 
     if (dict == NULL) {
@@ -266,24 +296,23 @@ int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value)
     return 0;
 }
 
-int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config)
+int32_t
+ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config)
 {
     int ret = -1;
-    uint64_t * ptr, data;
+    uint64_t *ptr, data;
 
-    if (config->version > EC_CONFIG_VERSION)
-    {
-        gf_msg ("ec", GF_LOG_ERROR, EINVAL,
-                EC_MSG_UNSUPPORTED_VERSION,
-                "Trying to store an unsupported config "
-                "version (%u)", config->version);
+    if (config->version > EC_CONFIG_VERSION) {
+        gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION,
+               "Trying to store an unsupported config "
+               "version (%u)",
+               config->version);
 
         return -EINVAL;
     }
 
     ptr = GF_MALLOC(sizeof(uint64_t), gf_common_mt_char);
-    if (ptr == NULL)
-    {
+    if (ptr == NULL) {
         return -ENOMEM;
     }
 
@@ -298,14 +327,15 @@ int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config)
 
     ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t));
     if (ret)
-        GF_FREE (ptr);
+        GF_FREE(ptr);
 
     return ret;
 }
 
-int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
+int32_t
+ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config)
 {
-    void * ptr;
+    void *ptr;
     uint64_t data;
     int32_t len, err;
 
@@ -328,20 +358,16 @@ int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
      * instead of saying that it doesn't exist.
      *
      * We need to filter out this case and consider that a config xattr == 0 is
-     * the same than a non-existant xattr. Otherwise ec_config_check() will
-     * fail.
+     * the same as a non-existent xattr. Otherwise ec_config_check() will fail.
      */
     if (data == 0) {
         return -ENODATA;
     }
 
     config->version = (data >> 56) & 0xff;
-    if (config->version > EC_CONFIG_VERSION)
-    {
-        gf_msg ("ec", GF_LOG_ERROR, EINVAL,
-                EC_MSG_UNSUPPORTED_VERSION,
-                "Found an unsupported config version (%u)",
-                config->version);
+    if (config->version > EC_CONFIG_VERSION) {
+        gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION,
+               "Found an unsupported config version (%u)", config->version);
 
         return -EINVAL;
     }
@@ -357,7 +383,8 @@ int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
     return 0;
 }
 
-gf_boolean_t ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
+gf_boolean_t
+ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
 {
     if (gf_uuid_is_null(src)) {
         return _gf_true;
@@ -370,9 +397,8 @@ gf_boolean_t ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
     }
 
     if (gf_uuid_compare(dst, src) != 0) {
-        gf_msg (xl->name, GF_LOG_WARNING, 0,
-                EC_MSG_GFID_MISMATCH,
-                "Mismatching GFID's in loc");
+        gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_GFID_MISMATCH,
+               "Mismatching GFID's in loc");
 
         return _gf_false;
     }
@@ -380,7 +406,8 @@ gf_boolean_t ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
     return _gf_true;
 }
 
-int32_t ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc)
+int32_t
+ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc)
 {
     int32_t ret = -EINVAL;
 
@@ -391,7 +418,7 @@ int32_t ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc)
     } else if (table != NULL) {
         if (!gf_uuid_is_null(loc->gfid)) {
             loc->inode = inode_find(table, loc->gfid);
-        } else if (loc->path && strchr (loc->path, '/')) {
+        } else if (loc->path && strchr(loc->path, '/')) {
             loc->inode = inode_resolve(table, (char *)loc->path);
         }
     }
@@ -402,7 +429,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc)
+int32_t
+ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc)
 {
     char *path, *parent;
     int32_t ret = -EINVAL;
@@ -414,13 +442,11 @@ int32_t ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc)
     } else if (table != NULL) {
         if (!gf_uuid_is_null(loc->pargfid)) {
             loc->parent = inode_find(table, loc->pargfid);
-        } else if (loc->path && strchr (loc->path, '/')) {
+        } else if (loc->path && strchr(loc->path, '/')) {
             path = gf_strdup(loc->path);
             if (path == NULL) {
-                gf_msg (xl->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_NO_MEMORY,
-                        "Unable to duplicate path '%s'",
-                        loc->path);
+                gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                       "Unable to duplicate path '%s'", loc->path);
 
                 ret = -ENOMEM;
 
@@ -447,9 +473,10 @@ out:
     return ret;
 }
 
-int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc)
+int32_t
+ec_loc_setup_path(xlator_t *xl, loc_t *loc)
 {
-    uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
     char *name;
     int32_t ret = -EINVAL;
 
@@ -477,10 +504,8 @@ int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc)
 
         if (loc->name != NULL) {
             if (strcmp(loc->name, name) != 0) {
-                gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-                        EC_MSG_INVALID_LOC_NAME,
-                        "Invalid name '%s' in loc",
-                        loc->name);
+                gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_LOC_NAME,
+                       "Invalid name '%s' in loc", loc->name);
 
                 goto out;
             }
@@ -495,7 +520,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
+int32_t
+ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
 {
     inode_table_t *table = NULL;
     char *str = NULL;
@@ -512,24 +538,20 @@ int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
     if (!gf_uuid_is_null(loc->pargfid)) {
         gf_uuid_copy(parent->gfid, loc->pargfid);
     }
-    if (loc->path && strchr (loc->path, '/')) {
+    if (loc->path && strchr(loc->path, '/')) {
         str = gf_strdup(loc->path);
         if (str == NULL) {
-                gf_msg (xl->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_NO_MEMORY,
-                        "Unable to duplicate path '%s'",
-                        loc->path);
+            gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Unable to duplicate path '%s'", loc->path);
 
-                goto out;
+            goto out;
         }
         parent->path = gf_strdup(dirname(str));
         if (parent->path == NULL) {
-                gf_msg (xl->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_NO_MEMORY,
-                        "Unable to duplicate path '%s'",
-                        dirname(str));
+            gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Unable to duplicate path '%s'", dirname(str));
 
-                goto out;
+            goto out;
         }
     }
 
@@ -546,9 +568,8 @@ int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
 
     if ((parent->inode == NULL) && (parent->path == NULL) &&
         gf_uuid_is_null(parent->gfid)) {
-        gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_LOC_PARENT_INODE_MISSING,
-                "Parent inode missing for loc_t");
+        gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_LOC_PARENT_INODE_MISSING,
+               "Parent inode missing for loc_t");
 
         ret = -EINVAL;
 
@@ -567,8 +588,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode,
-                      struct iatt *iatt)
+int32_t
+ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt)
 {
     inode_table_t *table = NULL;
     int32_t ret = -EINVAL;
@@ -609,9 +630,10 @@ out:
     return ret;
 }
 
-int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd)
+int32_t
+ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd)
 {
-    ec_fd_t * ctx;
+    ec_fd_t *ctx;
     int32_t ret = -ENOMEM;
 
     memset(loc, 0, sizeof(*loc));
@@ -636,7 +658,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src)
+int32_t
+ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src)
 {
     int32_t ret = -ENOMEM;
 
@@ -659,50 +682,62 @@ out:
     return ret;
 }
 
-void ec_owner_set(call_frame_t * frame, void * owner)
+void
+ec_owner_set(call_frame_t *frame, void *owner)
 {
     set_lk_owner_from_ptr(&frame->root->lk_owner, owner);
 }
 
-void ec_owner_copy(call_frame_t * frame, gf_lkowner_t * owner)
+void
+ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner)
 {
-    frame->root->lk_owner.len = owner->len;
-    memcpy(frame->root->lk_owner.data, owner->data, owner->len);
+    lk_owner_copy(&frame->root->lk_owner, owner);
 }
 
-ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl)
+static void
+ec_stripe_cache_init(ec_t *ec, ec_inode_t *ctx)
 {
-    ec_inode_t * ctx = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+
+    stripe_cache = &(ctx->stripe_cache);
+    if (stripe_cache->max == 0) {
+        stripe_cache->max = ec->stripe_cache;
+    }
+}
+
+ec_inode_t *
+__ec_inode_get(inode_t *inode, xlator_t *xl)
+{
+    ec_inode_t *ctx = NULL;
     uint64_t value = 0;
 
-    if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0))
-    {
+    if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0)) {
         ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_inode_t);
-        if (ctx != NULL)
-        {
+        if (ctx != NULL) {
             memset(ctx, 0, sizeof(*ctx));
             INIT_LIST_HEAD(&ctx->heal);
-
+            INIT_LIST_HEAD(&ctx->stripe_cache.lru);
+            ctx->heal_count = 0;
             value = (uint64_t)(uintptr_t)ctx;
-            if (__inode_ctx_set(inode, xl, &value) != 0)
-            {
+            if (__inode_ctx_set(inode, xl, &value) != 0) {
                 GF_FREE(ctx);
 
                 return NULL;
             }
         }
-    }
-    else
-    {
+    } else {
         ctx = (ec_inode_t *)(uintptr_t)value;
     }
+    if (ctx)
+        ec_stripe_cache_init(xl->private, ctx);
 
     return ctx;
 }
 
-ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl)
+ec_inode_t *
+ec_inode_get(inode_t *inode, xlator_t *xl)
 {
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
 
     LOCK(&inode->lock);
 
@@ -713,34 +748,47 @@ ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl)
     return ctx;
 }
 
-ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl)
+ec_fd_t *
+__ec_fd_get(fd_t *fd, xlator_t *xl)
 {
-    ec_fd_t * ctx = NULL;
+    int i = 0;
+    ec_fd_t *ctx = NULL;
+    ec_inode_t *ictx = NULL;
     uint64_t value = 0;
+    ec_t *ec = xl->private;
 
-    if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0))
-    {
-        ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_fd_t);
-        if (ctx != NULL)
-        {
+    if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0)) {
+        ctx = GF_MALLOC(sizeof(*ctx) + (sizeof(ec_fd_status_t) * ec->nodes),
+                        ec_mt_ec_fd_t);
+        if (ctx != NULL) {
             memset(ctx, 0, sizeof(*ctx));
 
+            for (i = 0; i < ec->nodes; i++) {
+                if (fd_is_anonymous(fd)) {
+                    ctx->fd_status[i] = EC_FD_OPENED;
+                } else {
+                    ctx->fd_status[i] = EC_FD_NOT_OPENED;
+                }
+            }
+
             value = (uint64_t)(uintptr_t)ctx;
-            if (__fd_ctx_set(fd, xl, value) != 0)
-            {
+            if (__fd_ctx_set(fd, xl, value) != 0) {
                 GF_FREE(ctx);
-
                 return NULL;
             }
+            /* Only refering bad-version so no need for lock
+             * */
+            ictx = __ec_inode_get(fd->inode, xl);
+            if (ictx) {
+                ctx->bad_version = ictx->bad_version;
+            }
         }
-    }
-    else
-    {
+    } else {
         ctx = (ec_fd_t *)(uintptr_t)value;
     }
 
     /* Treat anonymous fd specially */
-    if (fd->anonymous) {
+    if (fd->anonymous && ctx) {
         /* Mark the fd open for all subvolumes. */
         ctx->open = -1;
         /* Try to populate ctx->loc with fd->inode information. */
@@ -750,9 +798,10 @@ ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl)
     return ctx;
 }
 
-ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl)
+ec_fd_t *
+ec_fd_get(fd_t *fd, xlator_t *xl)
 {
-    ec_fd_t * ctx = NULL;
+    ec_fd_t *ctx = NULL;
 
     LOCK(&fd->lock);
 
@@ -763,67 +812,37 @@ ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl)
     return ctx;
 }
 
-uint32_t ec_adjust_offset(ec_t * ec, off_t * offset, int32_t scale)
-{
-    off_t head, tmp;
-
-    tmp = *offset;
-    head = tmp % ec->stripe_size;
-    tmp -= head;
-    if (scale)
-    {
-        tmp /= ec->fragments;
-    }
-
-    *offset = tmp;
-
-    return head;
-}
-
-uint64_t ec_adjust_size(ec_t * ec, uint64_t size, int32_t scale)
-{
-    size += ec->stripe_size - 1;
-    size -= size % ec->stripe_size;
-    if (scale)
-    {
-        size /= ec->fragments;
-    }
-
-    return size;
-}
-
 gf_boolean_t
-ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data)
+ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data)
 {
-        if (key &&
-            (strncmp (key, EC_XATTR_PREFIX, strlen (EC_XATTR_PREFIX)) == 0))
-                return _gf_true;
+    if (key && (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0))
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 void
-ec_filter_internal_xattrs (dict_t *xattr)
+ec_filter_internal_xattrs(dict_t *xattr)
 {
-        dict_foreach_match (xattr, ec_is_internal_xattr, NULL,
-                            dict_remove_foreach_fn, NULL);
+    dict_foreach_match(xattr, ec_is_internal_xattr, NULL,
+                       dict_remove_foreach_fn, NULL);
 }
 
 gf_boolean_t
-ec_is_data_fop (glusterfs_fop_t fop)
+ec_is_data_fop(glusterfs_fop_t fop)
 {
-        switch (fop) {
+    switch (fop) {
         case GF_FOP_WRITE:
         case GF_FOP_TRUNCATE:
         case GF_FOP_FTRUNCATE:
         case GF_FOP_FALLOCATE:
         case GF_FOP_DISCARD:
         case GF_FOP_ZEROFILL:
-                return _gf_true;
+            return _gf_true;
         default:
-                return _gf_false;
-        }
-        return _gf_false;
+            return _gf_false;
+    }
+    return _gf_false;
 }
 /*
 gf_boolean_t
diff --git a/xlators/cluster/ec/src/ec-helpers.h b/xlators/cluster/ec/src/ec-helpers.h
index 1f39da2c09f..015db514e05 100644
--- a/xlators/cluster/ec/src/ec-helpers.h
+++ b/xlators/cluster/ec/src/ec-helpers.h
@@ -11,62 +11,190 @@
 #ifndef __EC_HELPERS_H__
 #define __EC_HELPERS_H__
 
-#include "ec-data.h"
-
-const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits);
-const char * ec_fop_name(int32_t id);
-void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...);
-int32_t ec_bits_count(uint64_t n);
-int32_t ec_bits_index(uint64_t n);
-int32_t ec_bits_consume(uint64_t * n);
-size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
-                      off_t offset, size_t size);
-
-int32_t ec_dict_set_array(dict_t *dict, char *key,
-                          uint64_t *value, int32_t size);
-int32_t ec_dict_del_array(dict_t *dict, char *key,
-                          uint64_t *value, int32_t size);
-int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value);
-int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value);
-int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config);
-int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config);
-
-int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent);
-int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode,
-                      struct iatt *iatt);
-
-int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd);
-int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src);
-
-void ec_owner_set(call_frame_t * frame, void * owner);
-void ec_owner_copy(call_frame_t * frame, gf_lkowner_t * owner);
-
-ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl);
-ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl);
-ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl);
-ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl);
-
-uint32_t ec_adjust_offset(ec_t * ec, off_t * offset, int32_t scale);
-uint64_t ec_adjust_size(ec_t * ec, uint64_t size, int32_t scale);
-
-static inline int32_t ec_is_power_of_2(uint32_t value)
+#include "ec-types.h"
+
+#define EC_ERR(_x) ((void *)-(intptr_t)(_x))
+#define EC_IS_ERR(_x) (((uintptr_t)(_x) & ~0xfffULL) == ~0xfffULL)
+#define EC_GET_ERR(_x) ((int32_t)(intptr_t)(_x))
+
+#define EC_ALIGN_CHECK(_ptr, _align) ((((uintptr_t)(_ptr)) & ((_align)-1)) == 0)
+
+const char *
+ec_bin(char *str, size_t size, uint64_t value, int32_t digits);
+const char *
+ec_fop_name(int32_t id);
+void
+ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...);
+int32_t
+ec_bits_consume(uint64_t *n);
+size_t
+ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset,
+               size_t size);
+int32_t
+ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr);
+int32_t
+ec_dict_set_array(dict_t *dict, char *key, uint64_t *value, int32_t size);
+int32_t
+ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size);
+
+int32_t
+ec_dict_del_array(dict_t *dict, char *key, uint64_t *value, int32_t size);
+int32_t
+ec_dict_set_number(dict_t *dict, char *key, uint64_t value);
+int32_t
+ec_dict_del_number(dict_t *dict, char *key, uint64_t *value);
+int32_t
+ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config);
+int32_t
+ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config);
+
+int32_t
+ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent);
+int32_t
+ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt);
+
+int32_t
+ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd);
+int32_t
+ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src);
+
+void
+ec_owner_set(call_frame_t *frame, void *owner);
+void
+ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner);
+
+ec_inode_t *
+__ec_inode_get(inode_t *inode, xlator_t *xl);
+ec_inode_t *
+ec_inode_get(inode_t *inode, xlator_t *xl);
+ec_fd_t *
+__ec_fd_get(fd_t *fd, xlator_t *xl);
+ec_fd_t *
+ec_fd_get(fd_t *fd, xlator_t *xl);
+
+static inline uint32_t
+ec_adjust_size_down(ec_t *ec, uint64_t *value, gf_boolean_t scale)
+{
+    uint64_t head, tmp;
+
+    tmp = *value;
+    head = tmp % ec->stripe_size;
+    tmp -= head;
+
+    if (scale) {
+        tmp /= ec->fragments;
+    }
+
+    *value = tmp;
+
+    return (uint32_t)head;
+}
+
+/* This function can cause an overflow if the passed value is too near to the
+ * uint64_t limit. If this happens, it returns the tail in negative form and
+ * the value is set to UINT64_MAX. */
+static inline int32_t
+ec_adjust_size_up(ec_t *ec, uint64_t *value, gf_boolean_t scale)
+{
+    uint64_t tmp;
+    int32_t tail;
+
+    tmp = *value;
+    /* We first adjust the value down. This never causes overflow. */
+    tail = ec_adjust_size_down(ec, &tmp, scale);
+
+    /* If the value was already aligned, tail will be 0 and nothing else
+     * needs to be done. */
+    if (tail != 0) {
+        /* Otherwise, we need to compute the real tail and adjust the
+         * returned value to the next stripe. */
+        tail = ec->stripe_size - tail;
+        if (scale) {
+            tmp += ec->fragment_size;
+        } else {
+            tmp += ec->stripe_size;
+            /* If no scaling is requested there's a possibility of
+             * overflow. */
+            if (tmp < ec->stripe_size) {
+                tmp = UINT64_MAX;
+                tail = -tail;
+            }
+        }
+    }
+
+    *value = tmp;
+
+    return tail;
+}
+
+/* This function is equivalent to ec_adjust_size_down() but with a potentially
+ * different parameter size (off_t vs uint64_t). */
+static inline uint32_t
+ec_adjust_offset_down(ec_t *ec, off_t *value, gf_boolean_t scale)
+{
+    off_t head, tmp;
+
+    tmp = *value;
+    head = tmp % ec->stripe_size;
+    tmp -= head;
+
+    if (scale) {
+        tmp /= ec->fragments;
+    }
+
+    *value = tmp;
+
+    return (uint32_t)head;
+}
+
+/* This function is equivalent to ec_adjust_size_up() but with a potentially
+ * different parameter size (off_t vs uint64_t). */
+static inline int32_t
+ec_adjust_offset_up(ec_t *ec, off_t *value, gf_boolean_t scale)
+{
+    uint64_t tail, tmp;
+
+    /* An offset is a signed type that can only have positive values, so
+     * we take advantage of this to avoid overflows. We simply convert it
+     * to an unsigned integer and operate normally. This won't cause an
+     * overflow. Overflow is only checked when converting back to an
+     * off_t. */
+    tmp = *value;
+    tail = ec->stripe_size;
+    tail -= (tmp + tail - 1) % tail + 1;
+    tmp += tail;
+    if (scale) {
+        /* If we are scaling, we'll never get an overflow. */
+        tmp /= ec->fragments;
+    } else {
+        /* Check if there has been an overflow. */
+        if ((off_t)tmp < 0) {
+            tmp = GF_OFF_MAX;
+            tail = -tail;
+        }
+    }
+
+    *value = (off_t)tmp;
+
+    return (int32_t)tail;
+}
+
+static inline int32_t
+ec_is_power_of_2(uint32_t value)
 {
     return (value != 0) && ((value & (value - 1)) == 0);
 }
 
 gf_boolean_t
-ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data);
+ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data);
 
 void
-ec_filter_internal_xattrs (dict_t *xattr);
+ec_filter_internal_xattrs(dict_t *xattr);
 
 gf_boolean_t
-ec_is_data_fop (glusterfs_fop_t fop);
+ec_is_data_fop(glusterfs_fop_t fop);
 
 int32_t
-ec_launch_replace_heal (ec_t *ec);
-/*
-gf_boolean_t
-ec_is_metadata_fop (glusterfs_fop_t fop);
-*/
+ec_launch_replace_heal(ec_t *ec);
+
 #endif /* __EC_HELPERS_H__ */
diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c
index c3d9c879eb7..dad5f4d7018 100644
--- a/xlators/cluster/ec/src/ec-inode-read.c
+++ b/xlators/cluster/ec/src/ec-inode-read.c
@@ -8,20 +8,19 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-method.h"
 #include "ec-fops.h"
-#include "ec-messages.h"
 
 /* FOP: access */
 
-int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = NULL;
     ec_cbk_data_t *cbk = NULL;
@@ -34,27 +33,27 @@ int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate (frame, this, fop, GF_FOP_ACCESS,
-                                idx, op_ret, op_errno);
+    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ACCESS, idx, op_ret,
+                               op_errno);
     if (cbk) {
         if (xdata)
-               cbk->xdata = dict_ref (xdata);
-        ec_combine (cbk, NULL);
+            cbk->xdata = dict_ref(xdata);
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
-        ec_complete (fop);
+    if (fop != NULL) {
+        ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_access(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -66,18 +65,19 @@ void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 int32_t
 ec_manager_access(ec_fop_data_t *fop, int32_t state)
 {
-        ec_cbk_data_t *cbk = NULL;
+    ec_cbk_data_t *cbk = NULL;
 
-        switch (state) {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_inode (fop, &fop->loc[0], EC_QUERY_INFO);
-            ec_lock (fop);
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
+            ec_lock(fop);
 
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-            ec_dispatch_one (fop);
+            ec_dispatch_one(fop);
 
             return EC_STATE_PREPARE_ANSWER;
 
@@ -90,12 +90,11 @@ ec_manager_access(ec_fop_data_t *fop, int32_t state)
 
         case EC_STATE_REPORT:
             cbk = fop->answer;
-            GF_ASSERT (cbk);
+            GF_ASSERT(cbk);
             if (fop->cbks.access != NULL) {
                 if (cbk) {
-                    fop->cbks.access(fop->req_frame, fop, fop->xl,
-                                     cbk->op_ret, cbk->op_errno,
-                                     cbk->xdata);
+                    fop->cbks.access(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                     cbk->op_errno, cbk->xdata);
                 }
             }
             return EC_STATE_LOCK_REUSE;
@@ -106,8 +105,8 @@ ec_manager_access(ec_fop_data_t *fop, int32_t state)
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             if (fop->cbks.access != NULL) {
-                fop->cbks.access(fop->req_frame, fop, fop->xl, -1,
-                                 fop->error, NULL);
+                fop->cbks.access(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                 NULL);
             }
             return EC_STATE_LOCK_REUSE;
 
@@ -124,31 +123,30 @@ ec_manager_access(ec_fop_data_t *fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
-        }
+    }
 }
 
-void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_access_cbk_t func, void * data,
-               loc_t * loc, int32_t mask, dict_t * xdata)
+void
+ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc,
+          int32_t mask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .access = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.access = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(ACCESS) %p", frame);
+    gf_msg_trace("ec", 0, "EC(ACCESS) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_ACCESS, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_access,
+                               target, fop_flags, ec_wind_access,
                                ec_manager_access, callback, data);
     if (fop == NULL) {
         goto out;
@@ -158,9 +156,8 @@ void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -168,10 +165,9 @@ void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -189,14 +185,13 @@ out:
 
 /* FOP: getxattr */
 
-int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                            ec_cbk_data_t * src)
+int32_t
+ec_combine_getxattr(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_dict_compare(dst->dict, src->dict))
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_DICT_MISMATCH, "Mismatching dictionary in "
-                                             "answers of 'GF_FOP_GETXATTR'");
+    if (!ec_dict_compare(dst->dict, src->dict)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_DICT_MISMATCH,
+               "Mismatching dictionary in "
+               "answers of 'GF_FOP_GETXATTR'");
 
         return 0;
     }
@@ -204,12 +199,12 @@ int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * dict,
-                        dict_t * xdata)
+int32_t
+ec_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -219,38 +214,30 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_GETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (dict != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (dict != NULL) {
                 cbk->dict = dict_ref(dict);
-                if (cbk->dict == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_DICT_REF_FAIL,
-                            "Failed to reference a "
-                            "dictionary.");
+                if (cbk->dict == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                           "Failed to reference a "
+                           "dictionary.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -260,15 +247,15 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_getxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -278,41 +265,43 @@ void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_handle_special_xattrs (ec_fop_data_t *fop)
+ec_handle_special_xattrs(ec_fop_data_t *fop)
 {
-        ec_cbk_data_t *cbk = NULL;
-        /* Stime may not be available on all the bricks, so even if some of the
-         * subvols succeed the operation, treat it as answer.*/
-        if (fop->str[0] &&
-            fnmatch (GF_XATTR_STIME_PATTERN, fop->str[0], 0) == 0) {
-                if (!fop->answer || (fop->answer->op_ret < 0)) {
-                        list_for_each_entry (cbk, &fop->cbk_list, list) {
-                                if (cbk->op_ret >= 0) {
-                                        fop->answer = cbk;
-                                        break;
-                                }
-                        }
+    ec_cbk_data_t *cbk = NULL;
+    /* Stime may not be available on all the bricks, so even if some of the
+     * subvols succeed the operation, treat it as answer.*/
+    if (fop->str[0] && fnmatch(GF_XATTR_STIME_PATTERN, fop->str[0], 0) == 0) {
+        if (!fop->answer || (fop->answer->op_ret < 0)) {
+            list_for_each_entry(cbk, &fop->cbk_list, list)
+            {
+                if (cbk->op_ret >= 0) {
+                    fop->answer = cbk;
+                    break;
                 }
+            }
         }
+    }
 }
 
-int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_getxattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             /* clear-locks commands must be done without any locks acquired
                to avoid interferences. */
             if ((fop->str[0] == NULL) ||
                 (strncmp(fop->str[0], GF_XATTR_CLRLK_CMD,
-                         strlen(GF_XATTR_CLRLK_CMD)) != 0)) {
+                         SLEN(GF_XATTR_CLRLK_CMD)) != 0)) {
                 if (fop->fd == NULL) {
-                    ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO);
+                    ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                          EC_RANGE_FULL);
                 } else {
-                    ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO);
+                    ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                       EC_RANGE_FULL);
                 }
                 ec_lock(fop);
             }
@@ -320,23 +309,33 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-            ec_dispatch_all(fop);
+            if (fop->minimum == EC_MINIMUM_ALL) {
+                ec_dispatch_all(fop);
+            } else {
+                ec_dispatch_one(fop);
+            }
 
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            ec_handle_special_xattrs (fop);
-            cbk = ec_fop_prepare_answer(fop, _gf_true);
+            ec_handle_special_xattrs(fop);
+            if (fop->minimum == EC_MINIMUM_ALL) {
+                cbk = ec_fop_prepare_answer(fop, _gf_true);
+            } else {
+                if (ec_dispatch_one_retry(fop, &cbk)) {
+                    return EC_STATE_DISPATCH;
+                }
+            }
             if (cbk != NULL) {
                 int32_t err;
 
                 err = ec_dict_combine(cbk, EC_COMBINE_DICT);
                 if (!ec_cbk_set_error(cbk, -err, _gf_true)) {
                     if (cbk->xdata != NULL)
-                            ec_filter_internal_xattrs (cbk->xdata);
+                        ec_filter_internal_xattrs(cbk->xdata);
 
                     if (cbk->dict != NULL)
-                            ec_filter_internal_xattrs (cbk->dict);
+                        ec_filter_internal_xattrs(cbk->dict);
                 }
             }
 
@@ -347,8 +346,7 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.getxattr != NULL)
-            {
+            if (fop->cbks.getxattr != NULL) {
                 fop->cbks.getxattr(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, cbk->dict, cbk->xdata);
             }
@@ -362,10 +360,9 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.getxattr != NULL)
-            {
-                fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL);
+            if (fop->cbks.getxattr != NULL) {
+                fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -383,26 +380,44 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
-                             int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                             uintptr_t good, uintptr_t bad, dict_t *xdata)
+int32_t
+ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
+                     int32_t op_ret, int32_t op_errno, uintptr_t mask,
+                     uintptr_t good, uintptr_t bad, uint32_t pending,
+                     dict_t *xdata)
 {
-    ec_fop_data_t *fop = cookie;
-    fop_getxattr_cbk_t func = fop->data;
+    fop_getxattr_cbk_t func = cookie;
     ec_t *ec = xl->private;
     dict_t *dict = NULL;
     char *str;
     char bin1[65], bin2[65];
 
+    /* We try to return the 'pending' information in xdata, but if this cannot
+     * be set, we will ignore it silently. We prefer to report the success or
+     * failure of the heal itself. */
+    if (xdata == NULL) {
+        xdata = dict_new();
+    } else {
+        dict_ref(xdata);
+    }
+    if (xdata != NULL) {
+        if (dict_set_uint32(xdata, EC_XATTR_HEAL_NEW, pending) != 0) {
+            /* dict_set_uint32() is marked as 'warn_unused_result' and gcc
+             * enforces to check the result in this case. However we don't
+             * really care if it succeeded or not. We'll just do the same.
+             *
+             * This empty 'if' avoids the warning, and it will be removed by
+             * the optimizer. */
+        }
+    }
+
     if (op_ret >= 0) {
         dict = dict_new();
         if (dict == NULL) {
@@ -436,25 +451,28 @@ int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
     }
 
 out:
-    func(frame, NULL, xl, op_ret, op_errno, dict, NULL);
+    func(frame, NULL, xl, op_ret, op_errno, dict, xdata);
 
     if (dict != NULL) {
         dict_unref(dict);
     }
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
     return 0;
 }
 
 void
-ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_getxattr_cbk_t func, void *data,
-             loc_t *loc, const char *name, dict_t *xdata)
+ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc,
+            const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .getxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.getxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(GETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(GETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -462,35 +480,39 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     /* Special handling of an explicit self-heal request */
     if ((name != NULL) && (strcmp(name, EC_XATTR_HEAL) == 0)) {
-        ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk,
-                func, loc, 0, NULL);
+        ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk, func,
+                loc, 0, NULL);
 
         return;
     }
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_GETXATTR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_getxattr, ec_manager_getxattr, callback,
-                               data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_GETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_getxattr, ec_manager_getxattr, callback, data);
     if (fop == NULL) {
         goto out;
     }
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (name != NULL) {
-        fop->str[0] = gf_strdup(name);
+        /* In case of list-node-uuids xattr, set flag to indicate
+         * the same and use node-uuid xattr for winding fop */
+        if (XATTR_IS_NODE_UUID_LIST(name)) {
+            fop->int32 = 1;
+            fop->str[0] = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+        } else {
+            fop->str[0] = gf_strdup(name);
+        }
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -498,10 +520,9 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -511,20 +532,20 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL, NULL);
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: fgetxattr */
 
-int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                         int32_t op_ret, int32_t op_errno, dict_t * dict,
-                         dict_t * xdata)
+int32_t
+ec_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -534,38 +555,30 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FGETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (dict != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (dict != NULL) {
                 cbk->dict = dict_ref(dict);
-                if (cbk->dict == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_DICT_REF_FAIL,
-                            "Failed to reference a "
-                            "dictionary.");
+                if (cbk->dict == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                           "Failed to reference a "
+                           "dictionary.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -575,8 +588,7 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
@@ -584,7 +596,7 @@ out:
 }
 
 void
-ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+ec_wind_fgetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -594,24 +606,23 @@ ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 }
 
 void
-ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fgetxattr_cbk_t func, void *data,
-              fd_t *fd, const char *name, dict_t *xdata)
+ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd,
+             const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fgetxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fgetxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FGETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FGETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FGETXATTR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_fgetxattr, ec_manager_getxattr,
-                               callback, data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_FGETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_fgetxattr, ec_manager_getxattr, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -621,10 +632,9 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -632,8 +642,8 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (name != NULL) {
         fop->str[0] = gf_strdup(name);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY, "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -641,9 +651,9 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -653,22 +663,21 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL, NULL);
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: open */
 
-int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                        ec_cbk_data_t * src)
+int32_t
+ec_combine_open(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (dst->fd != src->fd)
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_FD_MISMATCH, "Mismatching fd in answers "
-                                             "of 'GF_FOP_OPEN': %p <-> %p",
+    if (dst->fd != src->fd) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH,
+               "Mismatching fd in answers "
+               "of 'GF_FOP_OPEN': %p <-> %p",
                dst->fd, src->fd);
 
         return 0;
@@ -677,12 +686,12 @@ int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -692,54 +701,51 @@ int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPEN, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (fd != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (fd != NULL) {
                 cbk->fd = fd_ref(fd);
-                if (cbk->fd == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                                     "file descriptor.");
+                if (cbk->fd == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           EC_MSG_FILE_DESC_REF_FAIL,
+                           "Failed to reference a "
+                           "file descriptor.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
         ec_combine(cbk, ec_combine_open);
+
+        ec_update_fd_status(fd, this, idx, op_ret);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_open(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -748,10 +754,10 @@ void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->fd, fop->xdata);
 }
 
-int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *prebuf, struct iatt *postbuf,
-                             dict_t *xdata)
+int32_t
+ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     int32_t error = 0;
@@ -768,14 +774,14 @@ int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
     return 0;
 }
 
-int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_open(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
-    ec_fd_t * ctx;
+    ec_cbk_data_t *cbk;
+    ec_fd_t *ctx;
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
@@ -787,13 +793,15 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
 
                 return EC_STATE_REPORT;
             }
-            err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
-            if (err != 0) {
-                UNLOCK(&fop->fd->lock);
+            if (!ctx->loc.inode) {
+                err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
+                if (err != 0) {
+                    UNLOCK(&fop->fd->lock);
 
-                fop->error = -err;
+                    fop->error = -err;
 
-                return EC_STATE_REPORT;
+                    return EC_STATE_REPORT;
+                }
             }
 
             ctx->flags = fop->int32;
@@ -809,7 +817,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
             fop->uint32 = fop->int32 & O_TRUNC;
             fop->int32 &= ~(O_APPEND | O_TRUNC);
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -840,8 +848,8 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
                     if (fop->uint32 != 0) {
                         ec_sleep(fop);
                         ec_ftruncate(fop->req_frame, fop->xl, cbk->mask,
-                                     fop->minimum, ec_open_truncate_cbk,
-                                     fop, cbk->fd, 0, NULL);
+                                     fop->minimum, ec_open_truncate_cbk, fop,
+                                     cbk->fd, 0, NULL);
                     }
                 }
             }
@@ -853,8 +861,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.open != NULL)
-            {
+            if (fop->cbks.open != NULL) {
                 fop->cbks.open(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                cbk->op_errno, cbk->fd, cbk->xdata);
             }
@@ -867,8 +874,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.open != NULL)
-            {
+            if (fop->cbks.open != NULL) {
                 fop->cbks.open(fop->req_frame, fop, fop->xl, -1, fop->error,
                                NULL, NULL);
             }
@@ -876,30 +882,30 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_open_cbk_t func, void * data, loc_t * loc,
-             int32_t flags, fd_t * fd, dict_t * xdata)
+void
+ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc,
+        int32_t flags, fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .open = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.open = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(OPEN) %p", frame);
+    gf_msg_trace("ec", 0, "EC(OPEN) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_OPEN, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_open, ec_manager_open,
+                               target, fop_flags, ec_wind_open, ec_manager_open,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -909,8 +915,8 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -918,9 +924,9 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -928,9 +934,9 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -948,13 +954,13 @@ out:
 
 /* FOP: readlink */
 
-int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                            ec_cbk_data_t * src)
+int32_t
+ec_combine_readlink(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_READLINK'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_READLINK'");
 
         return 0;
     }
@@ -963,13 +969,13 @@ int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst,
 }
 
 int32_t
-ec_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, const char *path,
-                 struct iatt *buf, dict_t *xdata)
+ec_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, const char *path,
+                struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t   *fop = NULL;
-    ec_cbk_data_t   *cbk = NULL;
-    int32_t         idx  = (int32_t)(uintptr_t)cookie;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -978,23 +984,23 @@ ec_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate (frame, this, fop, fop->id,
-                                idx, op_ret, op_errno);
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
     if (cbk) {
-            if (xdata)
-                    cbk->xdata = dict_ref (xdata);
-
-            if (cbk->op_ret >= 0) {
-                    cbk->iatt[0] = *buf;
-                    cbk->str = gf_strdup (path);
-                    if (!cbk->str) {
-                            ec_cbk_set_error(cbk, ENOMEM, _gf_true);
-                    }
+        if (xdata)
+            cbk->xdata = dict_ref(xdata);
+
+        if (cbk->op_ret >= 0) {
+            cbk->iatt[0] = *buf;
+            cbk->str = gf_strdup(path);
+            if (!cbk->str) {
+                ec_cbk_set_error(cbk, ENOMEM, _gf_true);
             }
-            ec_combine (cbk, NULL);
+        }
+        ec_combine(cbk, NULL);
     }
 
 out:
@@ -1004,7 +1010,8 @@ out:
     return 0;
 }
 
-void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1013,20 +1020,21 @@ void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->size, fop->xdata);
 }
 
-int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readlink(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk = NULL;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_inode (fop, &fop->loc[0], EC_QUERY_INFO);
-            ec_lock (fop);
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
+            ec_lock(fop);
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-            ec_dispatch_one (fop);
+            ec_dispatch_one(fop);
 
             return EC_STATE_PREPARE_ANSWER;
 
@@ -1043,11 +1051,11 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
 
         case EC_STATE_REPORT:
             cbk = fop->answer;
-            GF_ASSERT (cbk);
+            GF_ASSERT(cbk);
             if (fop->cbks.readlink != NULL) {
-                fop->cbks.readlink (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                    cbk->op_errno, cbk->str, &cbk->iatt[0],
-                                    cbk->xdata);
+                fop->cbks.readlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                   cbk->op_errno, cbk->str, &cbk->iatt[0],
+                                   cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1058,8 +1066,8 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             if (fop->cbks.readlink != NULL) {
-                fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL, NULL);
+                fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL, NULL);
             }
             return EC_STATE_LOCK_REUSE;
 
@@ -1075,32 +1083,31 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readlink_cbk_t func, void * data,
-                 loc_t * loc, size_t size, dict_t * xdata)
+void
+ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc,
+            size_t size, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readlink = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readlink = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READLINK,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_readlink, ec_manager_readlink, callback,
-                               data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_READLINK, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_readlink, ec_manager_readlink, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1109,8 +1116,8 @@ void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1118,9 +1125,9 @@ void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1138,14 +1145,15 @@ out:
 
 /* FOP: readv */
 
-int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
+int32_t
+ec_readv_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk)
 {
-    ec_cbk_data_t * ans = NULL;
-    struct iobref * iobref = NULL;
-    struct iobuf * iobuf = NULL;
-    uint8_t * buff = NULL, * ptr;
-    size_t fsize = 0, size = 0, max = 0;
-    int32_t i = 0, err = -ENOMEM;
+    struct iovec vector[1];
+    ec_cbk_data_t *ans = NULL;
+    struct iobref *iobref = NULL;
+    void *ptr;
+    uint64_t fsize = 0, size = 0, max = 0;
+    int32_t pos, err = -ENOMEM;
 
     if (cbk->op_ret < 0) {
         err = -cbk->op_errno;
@@ -1157,47 +1165,42 @@ int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
     GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, &cbk->iatt[0].ia_size));
 
     if (cbk->op_ret > 0) {
-        struct iovec vector[1];
-        uint8_t * blocks[cbk->count];
+        void *blocks[cbk->count];
         uint32_t values[cbk->count];
 
         fsize = cbk->op_ret;
         size = fsize * ec->fragments;
-        buff = GF_MALLOC(size, gf_common_mt_char);
-        if (buff == NULL) {
-            goto out;
-        }
-        ptr = buff;
-        for (i = 0, ans = cbk; ans != NULL; i++, ans = ans->next) {
-            values[i] = ans->idx;
-            blocks[i] = ptr;
-            ptr += ec_iov_copy_to(ptr, ans->vector, ans->int32, 0, fsize);
+        for (ans = cbk; ans != NULL; ans = ans->next) {
+            pos = gf_bits_count(cbk->mask & ((1 << ans->idx) - 1));
+            values[pos] = ans->idx + 1;
+            blocks[pos] = ans->vector[0].iov_base;
+            if ((ans->int32 != 1) ||
+                !EC_ALIGN_CHECK(blocks[pos], EC_METHOD_WORD_SIZE)) {
+                if (iobref == NULL) {
+                    err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr);
+                    if (err != 0) {
+                        goto out;
+                    }
+                }
+                ec_iov_copy_to(ptr, ans->vector, ans->int32, 0, fsize);
+                blocks[pos] = ptr;
+                ptr += fsize;
+            }
         }
 
-        iobref = iobref_new();
-        if (iobref == NULL) {
-            goto out;
-        }
-        iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, size);
-        if (iobuf == NULL) {
+        err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr);
+        if (err != 0) {
             goto out;
         }
-        err = iobref_add(iobref, iobuf);
+
+        err = ec_method_decode(&ec->matrix, fsize, cbk->mask, values, blocks,
+                               ptr);
         if (err != 0) {
             goto out;
         }
 
-        vector[0].iov_base = iobuf->ptr;
-        vector[0].iov_len = ec_method_decode(fsize, ec->fragments, values,
-                                             blocks, iobuf->ptr);
-
-        iobuf_unref(iobuf);
-
-        GF_FREE(buff);
-        buff = NULL;
-
-        vector[0].iov_base += fop->head;
-        vector[0].iov_len -= fop->head;
+        vector[0].iov_base = ptr + fop->head;
+        vector[0].iov_len = size - fop->head;
 
         max = fop->offset * ec->fragments + size;
         if (max > cbk->iatt[0].ia_size) {
@@ -1229,33 +1232,28 @@ int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
     return 0;
 
 out:
-    if (iobuf != NULL) {
-        iobuf_unref(iobuf);
-    }
     if (iobref != NULL) {
         iobref_unref(iobref);
     }
-    GF_FREE(buff);
 
     return err;
 }
 
-int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                         ec_cbk_data_t * src)
+int32_t
+ec_combine_readv(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32))
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_VECTOR_MISMATCH, "Mismatching vector in "
-                                             "answers of 'GF_FOP_READ'");
+    if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_VECTOR_MISMATCH,
+               "Mismatching vector in "
+               "answers of 'GF_FOP_READ'");
 
         return 0;
     }
 
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_READ'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_READ'");
 
         return 0;
     }
@@ -1263,14 +1261,14 @@ int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iovec * vector,
-                     int32_t count, struct iatt * stbuf,
-                     struct iobref * iobref, dict_t * xdata)
+int32_t
+ec_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
-    ec_t * ec = this->private;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    ec_t *ec = this->private;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1280,8 +1278,8 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_READ, idx, op_ret,
                                op_errno);
@@ -1292,9 +1290,9 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
             if (count > 0) {
                 cbk->vector = iov_dup(vector, count);
                 if (cbk->vector == NULL) {
-                    gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                            EC_MSG_NO_MEMORY, "Failed to duplicate a "
-                                                     "vector list.");
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                           "Failed to duplicate a "
+                           "vector list.");
 
                     goto out;
                 }
@@ -1306,9 +1304,9 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
             if (iobref != NULL) {
                 cbk->buffers = iobref_ref(iobref);
                 if (cbk->buffers == NULL) {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_BUF_REF_FAIL, "Failed to reference a "
-                                                     "buffer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL,
+                           "Failed to reference a "
+                           "buffer.");
 
                     goto out;
                 }
@@ -1317,9 +1315,9 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
         if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
             if (cbk->xdata == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1340,7 +1338,8 @@ out:
     return 0;
 }
 
-void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readv(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1349,27 +1348,33 @@ void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->size, fop->offset, fop->uint32, fop->xdata);
 }
 
-int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readv(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    ec_t *ec = fop->xl->private;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             fop->user_size = fop->size;
-            fop->head = ec_adjust_offset(fop->xl->private, &fop->offset, 1);
-            fop->size = ec_adjust_size(fop->xl->private, fop->size + fop->head,
-                                       1);
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
+            fop->size += fop->head;
+            ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true);
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO);
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset,
+                               fop->size);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
+            if (ec->read_mask) {
+                fop->mask &= ec->read_mask;
+            }
             ec_dispatch_min(fop);
 
             return EC_STATE_PREPARE_ANSWER;
@@ -1379,8 +1384,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
             if (cbk != NULL) {
                 int32_t err;
 
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count);
 
                 err = ec_readv_rebuild(fop->xl->private, fop, cbk);
                 if (err != 0) {
@@ -1395,8 +1399,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.readv != NULL)
-            {
+            if (fop->cbks.readv != NULL) {
                 fop->cbks.readv(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, cbk->vector, cbk->int32,
                                 &cbk->iatt[0], cbk->buffers, cbk->xdata);
@@ -1411,8 +1414,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.readv != NULL)
-            {
+            if (fop->cbks.readv != NULL) {
                 fop->cbks.readv(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, 0, NULL, NULL, NULL);
             }
@@ -1432,30 +1434,30 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_readv_cbk_t func, void * data, fd_t * fd,
-              size_t size, off_t offset, uint32_t flags, dict_t * xdata)
+void
+ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd,
+         size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readv = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readv = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READ) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READ) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_READ, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_readv,
+                               target, fop_flags, ec_wind_readv,
                                ec_manager_readv, callback, data);
     if (fop == NULL) {
         goto out;
@@ -1470,9 +1472,9 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1480,9 +1482,9 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1500,9 +1502,9 @@ out:
 
 /* FOP: seek */
 
-int32_t ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, off_t offset,
-                    dict_t *xdata)
+int32_t
+ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, off_t offset, dict_t *xdata)
 {
     ec_fop_data_t *fop = NULL;
     ec_cbk_data_t *cbk = NULL;
@@ -1516,8 +1518,8 @@ int32_t ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SEEK, idx, op_ret,
                                op_errno);
@@ -1545,7 +1547,8 @@ out:
     return 0;
 }
 
-void ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+void
+ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1554,114 +1557,128 @@ void ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
                       fop->offset, fop->seek, fop->xdata);
 }
 
-int32_t ec_manager_seek(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_seek(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk;
+    uint64_t size;
 
     switch (state) {
-    case EC_STATE_INIT:
-        fop->user_size = fop->offset;
-        fop->head = ec_adjust_offset(fop->xl->private, &fop->offset, 1);
+        case EC_STATE_INIT:
+            fop->user_size = fop->offset;
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
 
-    /* Fall through */
+            /* Fall through */
 
-    case EC_STATE_LOCK:
-        ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO);
-        ec_lock(fop);
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset,
+                               EC_RANGE_FULL);
+            ec_lock(fop);
 
-        return EC_STATE_DISPATCH;
+            return EC_STATE_DISPATCH;
 
-    case EC_STATE_DISPATCH:
-        ec_dispatch_one(fop);
+        case EC_STATE_DISPATCH:
+            /* This shouldn't fail because we have the inode locked. */
+            GF_ASSERT(
+                ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, &size));
 
-        return EC_STATE_PREPARE_ANSWER;
+            if (fop->user_size >= size) {
+                ec_fop_set_error(fop, ENXIO);
 
-    case EC_STATE_PREPARE_ANSWER:
-        cbk = fop->answer;
-        if (cbk != NULL) {
+                return EC_STATE_REPORT;
+            }
+
+            ec_dispatch_one(fop);
+
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
             if (ec_dispatch_one_retry(fop, &cbk)) {
                 return EC_STATE_DISPATCH;
             }
-            if (cbk->op_ret >= 0) {
+            if ((cbk != NULL) && (cbk->op_ret >= 0)) {
                 ec_t *ec = fop->xl->private;
 
+                /* This shouldn't fail because we have the inode locked. */
+                GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
+                                            &size));
+
                 cbk->offset *= ec->fragments;
                 if (cbk->offset < fop->user_size) {
                     cbk->offset = fop->user_size;
                 }
-            } else {
-                ec_fop_set_error(fop, cbk->op_errno);
+                if (cbk->offset > size) {
+                    cbk->offset = size;
+                }
             }
-        } else {
-            ec_fop_set_error(fop, EIO);
-        }
 
-        return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
-    case EC_STATE_REPORT:
-        cbk = fop->answer;
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
 
-        GF_ASSERT(cbk != NULL);
+            GF_ASSERT(cbk != NULL);
 
-        if (fop->cbks.seek != NULL) {
-            fop->cbks.seek(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                           cbk->op_errno, cbk->offset, cbk->xdata);
-        }
+            if (fop->cbks.seek != NULL) {
+                fop->cbks.seek(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                               cbk->op_errno, cbk->offset, cbk->xdata);
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_INIT:
-    case -EC_STATE_LOCK:
-    case -EC_STATE_DISPATCH:
-    case -EC_STATE_PREPARE_ANSWER:
-    case -EC_STATE_REPORT:
-        GF_ASSERT(fop->error != 0);
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
 
-        if (fop->cbks.seek != NULL) {
-            fop->cbks.seek(fop->req_frame, fop, fop->xl, -1, fop->error, 0,
-                           NULL);
-        }
+            if (fop->cbks.seek != NULL) {
+                fop->cbks.seek(fop->req_frame, fop, fop->xl, -1, fop->error, 0,
+                               NULL);
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_LOCK_REUSE:
-    case EC_STATE_LOCK_REUSE:
-        ec_lock_reuse(fop);
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
 
-        return EC_STATE_UNLOCK;
+            return EC_STATE_UNLOCK;
 
-    case -EC_STATE_UNLOCK:
-    case EC_STATE_UNLOCK:
-        ec_unlock(fop);
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
 
-        return EC_STATE_END;
+            return EC_STATE_END;
 
-    default:
-        gf_msg (fop->xl->name, GF_LOG_ERROR, 0,
-                EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s", state,
-                ec_fop_name(fop->id));
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-        return EC_STATE_END;
+            return EC_STATE_END;
     }
 }
 
-void ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_seek_cbk_t func, void *data, fd_t *fd,
-              off_t offset, gf_seek_what_t what, dict_t *xdata)
+void
+ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd,
+        off_t offset, gf_seek_what_t what, dict_t *xdata)
 {
-    ec_cbk_t callback = { .seek = func };
+    ec_cbk_t callback = {.seek = func};
     ec_fop_data_t *fop = NULL;
     int32_t error = EIO;
 
-    gf_msg_trace ("ec", 0, "EC(SEEK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SEEK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_SEEK, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_seek,
-                               ec_manager_seek, callback, data);
+                               target, fop_flags, ec_wind_seek, ec_manager_seek,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1690,13 +1707,13 @@ out:
 
 /* FOP: stat */
 
-int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                        ec_cbk_data_t * src)
+int32_t
+ec_combine_stat(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_STAT'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_STAT'");
 
         return 0;
     }
@@ -1704,12 +1721,12 @@ int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                    dict_t * xdata)
+int32_t
+ec_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1719,28 +1736,23 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STAT, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1750,15 +1762,15 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_stat(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1767,18 +1779,20 @@ void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_stat(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             if (fop->fd == NULL) {
-                ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO);
+                ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
             } else {
-                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO);
+                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                   EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -1791,10 +1805,10 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
 
         case EC_STATE_PREPARE_ANSWER:
             cbk = ec_fop_prepare_answer(fop, _gf_true);
+
             if (cbk != NULL) {
                 if (cbk->iatt[0].ia_type == IA_IFREG) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1,
-                                    cbk->count);
+                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count);
 
                     /* This shouldn't fail because we have the inode locked. */
                     GF_ASSERT(ec_get_inode_size(fop,
@@ -1810,18 +1824,13 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_STAT)
-            {
-                if (fop->cbks.stat != NULL)
-                {
+            if (fop->id == GF_FOP_STAT) {
+                if (fop->cbks.stat != NULL) {
                     fop->cbks.stat(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, &cbk->iatt[0], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fstat != NULL)
-                {
+            } else {
+                if (fop->cbks.fstat != NULL) {
                     fop->cbks.fstat(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                     cbk->op_errno, &cbk->iatt[0], cbk->xdata);
                 }
@@ -1836,18 +1845,13 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_STAT)
-            {
-                if (fop->cbks.stat != NULL)
-                {
-                    fop->cbks.stat(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL);
+            if (fop->id == GF_FOP_STAT) {
+                if (fop->cbks.stat != NULL) {
+                    fop->cbks.stat(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fstat != NULL)
-                {
+            } else {
+                if (fop->cbks.fstat != NULL) {
                     fop->cbks.fstat(fop->req_frame, fop, fop->xl, -1,
                                     fop->error, NULL, NULL);
                 }
@@ -1868,30 +1872,30 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_stat_cbk_t func, void * data, loc_t * loc,
-             dict_t * xdata)
+void
+ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc,
+        dict_t *xdata)
 {
-    ec_cbk_t callback = { .stat = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.stat = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(STAT) %p", frame);
+    gf_msg_trace("ec", 0, "EC(STAT) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_STAT, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_stat, ec_manager_stat,
+                               target, fop_flags, ec_wind_stat, ec_manager_stat,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -1899,8 +1903,8 @@ void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1908,9 +1912,9 @@ void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1928,12 +1932,12 @@ out:
 
 /* FOP: fstat */
 
-int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                     dict_t * xdata)
+int32_t
+ec_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1943,28 +1947,23 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSTAT, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1974,15 +1973,15 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fstat(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1991,23 +1990,24 @@ void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fstat_cbk_t func, void * data, fd_t * fd,
-              dict_t * xdata)
+void
+ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata)
 {
-    ec_cbk_t callback = { .fstat = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fstat = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSTAT) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSTAT) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSTAT, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_fstat, ec_manager_stat,
-                               callback, data);
+                               target, fop_flags, ec_wind_fstat,
+                               ec_manager_stat, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -2017,9 +2017,9 @@ void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -2027,9 +2027,9 @@ void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-inode-write.c b/xlators/cluster/ec/src/ec-inode-write.c
index 6aeda5a2481..9b5fe2a7fdc 100644
--- a/xlators/cluster/ec/src/ec-inode-write.c
+++ b/xlators/cluster/ec/src/ec-inode-write.c
@@ -8,73 +8,163 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-method.h"
 #include "ec-fops.h"
-#include "ec-messages.h"
+#include "ec-mem-types.h"
+
+int32_t
+ec_update_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    ec_fop_data_t *fop = cookie;
+    ec_cbk_data_t *cbk = NULL;
+    ec_fop_data_t *parent = fop->parent;
+    int i = 0;
+
+    ec_trace("UPDATE_WRITEV_CBK", cookie, "ret=%d, errno=%d, parent-fop=%s",
+             op_ret, op_errno, ec_fop_name(parent->id));
+
+    if (op_ret < 0) {
+        ec_fop_set_error(parent, op_errno);
+        goto out;
+    }
+    cbk = ec_cbk_data_allocate(parent->frame, this, parent, parent->id, 0,
+                               op_ret, op_errno);
+    if (!cbk) {
+        ec_fop_set_error(parent, ENOMEM);
+        goto out;
+    }
+
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
+
+    if (prebuf)
+        cbk->iatt[i++] = *prebuf;
+
+    if (postbuf)
+        cbk->iatt[i++] = *postbuf;
+
+    LOCK(&parent->lock);
+    {
+        parent->good &= fop->good;
+
+        if (gf_bits_count(parent->good) < parent->minimum) {
+            __ec_fop_set_error(parent, EIO);
+        } else if (fop->error == 0 && parent->answer == NULL) {
+            parent->answer = cbk;
+        }
+    }
+    UNLOCK(&parent->lock);
+out:
+    return 0;
+}
+
+static int32_t
+ec_update_write(ec_fop_data_t *fop, uintptr_t mask, off_t offset, uint64_t size)
+{
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec vector;
+    int32_t err = -ENOMEM;
+
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        goto out;
+    }
+    iobuf = iobuf_get(fop->xl->ctx->iobuf_pool);
+    if (iobuf == NULL) {
+        goto out;
+    }
+    err = iobref_add(iobref, iobuf);
+    if (err != 0) {
+        goto out;
+    }
+
+    if (fop->locks[0].lock)
+        ec_lock_update_good(fop->locks[0].lock, fop);
+    vector.iov_base = iobuf->ptr;
+    vector.iov_len = size;
+    memset(vector.iov_base, 0, vector.iov_len);
+
+    ec_writev(fop->frame, fop->xl, mask, fop->minimum, ec_update_writev_cbk,
+              NULL, fop->fd, &vector, 1, offset, 0, iobref, NULL);
+
+    err = 0;
+
+out:
+    if (iobuf != NULL) {
+        iobuf_unref(iobuf);
+    }
+    if (iobref != NULL) {
+        iobref_unref(iobref);
+    }
+
+    return err;
+}
 
 int
-ec_inode_write_cbk (call_frame_t *frame, xlator_t *this, void *cookie,
-                    int op_ret, int op_errno, struct iatt *prestat,
-                    struct iatt *poststat, dict_t *xdata)
+ec_inode_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie,
+                   int op_ret, int op_errno, struct iatt *prestat,
+                   struct iatt *poststat, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        int           i    = 0;
-        int           idx  = 0;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int i = 0;
+    int idx = 0;
 
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        fop = frame->local;
-        idx = (int32_t)(uintptr_t) cookie;
+    fop = frame->local;
+    idx = (int32_t)(uintptr_t)cookie;
 
-        ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                 frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        if (xdata)
-                cbk->xdata = dict_ref (xdata);
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        if (prestat)
-                cbk->iatt[i++] = *prestat;
+    if (prestat)
+        cbk->iatt[i++] = *prestat;
 
-        if (poststat)
-                cbk->iatt[i++] = *poststat;
+    if (poststat)
+        cbk->iatt[i++] = *poststat;
 
 out:
-        if (cbk)
-                ec_combine (cbk, ec_combine_write);
+    if (cbk)
+        ec_combine(cbk, ec_combine_write);
 
-        if (fop)
-                ec_complete (fop);
-        return 0;
+    if (fop)
+        ec_complete(fop);
+    return 0;
 }
 /* FOP: removexattr */
 
-int32_t ec_removexattr_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *xdata)
+int32_t
+ec_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_removexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -84,128 +174,126 @@ void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, dict_t *xdata)
+ec_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        ec_fop_data_t *fop = cookie;
-        switch (fop->id) {
+    ec_fop_data_t *fop = cookie;
+    switch (fop->id) {
         case GF_FOP_SETXATTR:
-                if (fop->cbks.setxattr) {
-                        fop->cbks.setxattr (frame, cookie, this, op_ret,
-                                            op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.setxattr) {
+                QUORUM_CBK(fop->cbks.setxattr, fop, frame, cookie, this, op_ret,
+                           op_errno, xdata);
+            }
+            break;
         case GF_FOP_REMOVEXATTR:
-                if (fop->cbks.removexattr) {
-                        fop->cbks.removexattr (frame, cookie, this, op_ret,
-                                               op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.removexattr) {
+                QUORUM_CBK(fop->cbks.removexattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
         case GF_FOP_FSETXATTR:
-                if (fop->cbks.fsetxattr) {
-                        fop->cbks.fsetxattr (frame, cookie, this, op_ret,
-                                             op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.fsetxattr) {
+                QUORUM_CBK(fop->cbks.fsetxattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
         case GF_FOP_FREMOVEXATTR:
-                if (fop->cbks.fremovexattr) {
-                        fop->cbks.fremovexattr (frame, cookie, this, op_ret,
-                                                op_errno, xdata);
-                }
-                break;
-        }
+            if (fop->cbks.fremovexattr) {
+                QUORUM_CBK(fop->cbks.fremovexattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
+    }
 }
 
 int32_t
-ec_manager_xattr (ec_fop_data_t *fop, int32_t state)
+ec_manager_xattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-        switch (state) {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-                if (fop->fd == NULL) {
-                        ec_lock_prepare_inode(fop, &fop->loc[0],
-                                              EC_UPDATE_META | EC_QUERY_INFO);
-                } else {
-                        ec_lock_prepare_fd(fop, fop->fd,
-                                           EC_UPDATE_META | EC_QUERY_INFO);
-                }
-                ec_lock(fop);
+            if (fop->fd == NULL) {
+                ec_lock_prepare_inode(fop, &fop->loc[0],
+                                      EC_UPDATE_META | EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
+            } else {
+                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO,
+                                   0, EC_RANGE_FULL);
+            }
+            ec_lock(fop);
 
-                return EC_STATE_DISPATCH;
+            return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-                ec_dispatch_all(fop);
+            ec_dispatch_all(fop);
 
-                return EC_STATE_PREPARE_ANSWER;
+            return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-                ec_fop_prepare_answer(fop, _gf_false);
+            ec_fop_prepare_answer(fop, _gf_false);
 
-                return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
         case EC_STATE_REPORT:
-                cbk = fop->answer;
+            cbk = fop->answer;
 
-                GF_ASSERT(cbk != NULL);
+            GF_ASSERT(cbk != NULL);
 
-                ec_xattr_cbk (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                              cbk->op_errno, cbk->xdata);
+            ec_xattr_cbk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                         cbk->op_errno, cbk->xdata);
 
-                    return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
-                GF_ASSERT(fop->error != 0);
+            GF_ASSERT(fop->error != 0);
 
-                ec_xattr_cbk (fop->req_frame, fop, fop->xl, -1, fop->error,
-                              NULL);
+            ec_xattr_cbk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL);
 
-                return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_LOCK_REUSE:
         case EC_STATE_LOCK_REUSE:
-                ec_lock_reuse(fop);
+            ec_lock_reuse(fop);
 
-                return EC_STATE_UNLOCK;
+            return EC_STATE_UNLOCK;
 
         case -EC_STATE_UNLOCK:
         case EC_STATE_UNLOCK:
-                ec_unlock(fop);
+            ec_unlock(fop);
 
-                return EC_STATE_END;
+            return EC_STATE_END;
 
         default:
-                gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                        EC_MSG_UNHANDLED_STATE,
-                        "Unhandled state %d for %s",
-                        state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-                return EC_STATE_END;
-        }
+            return EC_STATE_END;
+    }
 }
 
 void
-ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-                int32_t minimum, fop_removexattr_cbk_t func, void *data,
-                loc_t *loc, const char *name, dict_t *xdata)
+ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+               uint32_t fop_flags, fop_removexattr_cbk_t func, void *data,
+               loc_t *loc, const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .removexattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.removexattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(REMOVEXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(REMOVEXATTR) %p", frame);
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_REMOVEXATTR, 0, target,
-                               minimum, ec_wind_removexattr, ec_manager_xattr,
+                               fop_flags, ec_wind_removexattr, ec_manager_xattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -213,9 +301,8 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -223,20 +310,18 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (name != NULL) {
         fop->str[0] = gf_strdup(name);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -246,23 +331,24 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fremovexattr */
 
-int32_t ec_fremovexattr_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *xdata)
+int32_t
+ec_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fremovexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -272,23 +358,23 @@ void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-                 int32_t minimum, fop_fremovexattr_cbk_t func, void *data,
-                 fd_t *fd, const char *name, dict_t *xdata)
+ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+                uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data,
+                fd_t *fd, const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fremovexattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fremovexattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FREMOVEXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FREMOVEXATTR) %p", frame);
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FREMOVEXATTR, 0, target,
-                               minimum, ec_wind_fremovexattr, ec_manager_xattr,
-                               callback, data);
+                               fop_flags, ec_wind_fremovexattr,
+                               ec_manager_xattr, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -298,10 +384,9 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -309,9 +394,8 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (name != NULL) {
         fop->str[0] = gf_strdup(name);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -319,10 +403,9 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -332,24 +415,25 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: setattr */
 
-int32_t ec_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,
-                        struct iatt *prestat, struct iatt *poststat,
-                        dict_t *xdata)
+int32_t
+ec_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+               struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_setattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -358,20 +442,21 @@ void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->iatt, fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_setattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             if (fop->fd == NULL) {
                 ec_lock_prepare_inode(fop, &fop->loc[0],
-                                      EC_UPDATE_META | EC_QUERY_INFO);
+                                      EC_UPDATE_META | EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
             } else {
-                ec_lock_prepare_fd(fop, fop->fd,
-                                   EC_UPDATE_META | EC_QUERY_INFO);
+                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO,
+                                   0, EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -386,8 +471,7 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
             cbk = ec_fop_prepare_answer(fop, _gf_false);
             if (cbk != NULL) {
                 if (cbk->iatt[0].ia_type == IA_IFREG) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                    cbk->count);
+                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                     /* This shouldn't fail because we have the inode locked. */
                     GF_ASSERT(ec_get_inode_size(fop,
@@ -404,24 +488,17 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_SETATTR)
-            {
-                if (fop->cbks.setattr != NULL)
-                {
-                    fop->cbks.setattr(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno,
-                                      &cbk->iatt[0], &cbk->iatt[1],
-                                      cbk->xdata);
+            if (fop->id == GF_FOP_SETATTR) {
+                if (fop->cbks.setattr != NULL) {
+                    QUORUM_CBK(fop->cbks.setattr, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fsetattr != NULL)
-                {
-                    fop->cbks.fsetattr(fop->req_frame, fop, fop->xl,
-                                       cbk->op_ret, cbk->op_errno,
-                                       &cbk->iatt[0], &cbk->iatt[1],
-                                       cbk->xdata);
+            } else {
+                if (fop->cbks.fsetattr != NULL) {
+                    QUORUM_CBK(fop->cbks.fsetattr, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
             }
 
@@ -434,18 +511,13 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_SETATTR)
-            {
-                if (fop->cbks.setattr != NULL)
-                {
+            if (fop->id == GF_FOP_SETATTR) {
+                if (fop->cbks.setattr != NULL) {
                     fop->cbks.setattr(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fsetattr != NULL)
-                {
+            } else {
+                if (fop->cbks.fsetattr != NULL) {
                     fop->cbks.fsetattr(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL, NULL);
                 }
@@ -466,33 +538,31 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_setattr_cbk_t func, void * data,
-                loc_t * loc, struct iatt * stbuf, int32_t valid,
-                dict_t * xdata)
+void
+ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc,
+           struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_cbk_t callback = { .setattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.setattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(SETATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SETATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, 0, target, minimum,
-                               ec_wind_setattr, ec_manager_setattr, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, 0, target,
+                               fop_flags, ec_wind_setattr, ec_manager_setattr,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -501,9 +571,8 @@ void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -514,10 +583,9 @@ void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -535,16 +603,17 @@ out:
 
 /* FOP: fsetattr */
 
-int32_t ec_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *prestat, struct iatt *poststat,
-                         dict_t *xdata)
+int32_t
+ec_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_fsetattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsetattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -553,22 +622,23 @@ void ec_wind_fsetattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, &fop->iatt, fop->int32, fop->xdata);
 }
 
-void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsetattr_cbk_t func, void * data,
-                 fd_t * fd, struct iatt * stbuf, int32_t valid, dict_t * xdata)
+void
+ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsetattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsetattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSETATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSETATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETATTR, 0, target,
-                               minimum, ec_wind_fsetattr, ec_manager_setattr,
+                               fop_flags, ec_wind_fsetattr, ec_manager_setattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -581,10 +651,9 @@ void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -595,10 +664,9 @@ void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -616,14 +684,16 @@ out:
 
 /* FOP: setxattr */
 
-int32_t ec_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_setxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_setxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -633,22 +703,22 @@ void ec_wind_setxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_setxattr_cbk_t func, void *data,
-             loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc,
+            dict_t *dict, int32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .setxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.setxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(SETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_SETXATTR, 0, target,
-                               minimum, ec_wind_setxattr, ec_manager_xattr,
+                               fop_flags, ec_wind_setxattr, ec_manager_xattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -658,9 +728,8 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -668,10 +737,9 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (dict != NULL) {
         fop->dict = dict_copy_with_ref(dict, NULL);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -679,10 +747,9 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -692,45 +759,41 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fsetxattr */
 
 int32_t
-ec_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+ec_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -740,15 +803,15 @@ ec_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsetxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -758,22 +821,22 @@ void ec_wind_fsetxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fsetxattr_cbk_t func, void *data,
-              fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd,
+             dict_t *dict, int32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsetxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsetxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETXATTR, 0, target,
-                               minimum, ec_wind_fsetxattr, ec_manager_xattr,
+                               fop_flags, ec_wind_fsetxattr, ec_manager_xattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -786,10 +849,9 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -797,10 +859,9 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (dict != NULL) {
         fop->dict = dict_copy_with_ref(dict, NULL);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -808,10 +869,9 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -821,77 +881,488 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
-/* FOP: truncate */
+/*********************************************************************
+ *
+ * File Operation : fallocate
+ *
+ *********************************************************************/
 
-int32_t ec_truncate_write(ec_fop_data_t * fop, uintptr_t mask)
+int32_t
+ec_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-    ec_t * ec = fop->xl->private;
-    struct iobref * iobref = NULL;
-    struct iobuf * iobuf = NULL;
-    struct iovec vector;
-    int32_t err = -ENOMEM;
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf,
+                              postbuf, xdata);
+}
 
-    iobref = iobref_new();
-    if (iobref == NULL) {
-        goto out;
+void
+ec_wind_fallocate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
+
+    STACK_WIND_COOKIE(fop->frame, ec_fallocate_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->fallocate,
+                      fop->fd, fop->int32, fop->offset, fop->size, fop->xdata);
+}
+
+int32_t
+ec_manager_fallocate(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk = NULL;
+
+    switch (state) {
+        case EC_STATE_INIT:
+            if (fop->size == 0) {
+                ec_fop_set_error(fop, EINVAL);
+                return EC_STATE_REPORT;
+            }
+            if (fop->int32 &
+                (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE |
+                 FALLOC_FL_ZERO_RANGE | FALLOC_FL_PUNCH_HOLE)) {
+                ec_fop_set_error(fop, ENOTSUP);
+                return EC_STATE_REPORT;
+            }
+            fop->user_size = fop->offset + fop->size;
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
+            fop->size += fop->head;
+            ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true);
+
+            /* Fall through */
+
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd,
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fop->offset, fop->size);
+            ec_lock(fop);
+
+            return EC_STATE_DISPATCH;
+
+        case EC_STATE_DISPATCH:
+
+            ec_dispatch_all(fop);
+
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                /* This shouldn't fail because we have the inode locked. */
+                LOCK(&fop->locks[0].lock->loc.inode->lock);
+                {
+                    GF_ASSERT(__ec_get_inode_size(fop,
+                                                  fop->locks[0].lock->loc.inode,
+                                                  &cbk->iatt[0].ia_size));
+
+                    /*If mode has FALLOC_FL_KEEP_SIZE keep the size */
+                    if (fop->int32 & FALLOC_FL_KEEP_SIZE) {
+                        cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    } else if (fop->user_size > cbk->iatt[0].ia_size) {
+                        cbk->iatt[1].ia_size = fop->user_size;
+
+                        /* This shouldn't fail because we have the inode
+                         * locked. */
+                        GF_ASSERT(__ec_set_inode_size(
+                            fop, fop->locks[0].lock->loc.inode,
+                            cbk->iatt[1].ia_size));
+                    } else {
+                        cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    }
+                }
+                UNLOCK(&fop->locks[0].lock->loc.inode->lock);
+            }
+
+            return EC_STATE_REPORT;
+
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+
+            GF_ASSERT(cbk != NULL);
+
+            if (fop->cbks.fallocate != NULL) {
+                QUORUM_CBK(fop->cbks.fallocate, fop, fop->req_frame, fop,
+                           fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
+
+            if (fop->cbks.fallocate != NULL) {
+                fop->cbks.fallocate(fop->req_frame, fop, fop->xl, -1,
+                                    fop->error, NULL, NULL, NULL);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
+
+            return EC_STATE_END;
+
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
     }
-    iobuf = iobuf_get(fop->xl->ctx->iobuf_pool);
-    if (iobuf == NULL) {
+}
+
+void
+ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd,
+             int32_t mode, off_t offset, size_t len, dict_t *xdata)
+{
+    ec_cbk_t callback = {.fallocate = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
+
+    gf_msg_trace("ec", 0, "EC(FALLOCATE) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FALLOCATE, 0, target,
+                               fop_flags, ec_wind_fallocate,
+                               ec_manager_fallocate, callback, data);
+    if (fop == NULL) {
         goto out;
     }
-    err = iobref_add(iobref, iobuf);
-    if (err != 0) {
+
+    fop->use_fd = 1;
+    fop->int32 = mode;
+    fop->offset = offset;
+    fop->size = len;
+
+    if (fd != NULL) {
+        fop->fd = fd_ref(fd);
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
+            goto out;
+        }
+    }
+
+    if (xdata != NULL) {
+        fop->xdata = dict_ref(xdata);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
+            goto out;
+        }
+    }
+
+    error = 0;
+
+out:
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
+    }
+}
+
+/*********************************************************************
+ *
+ * File Operation : Discard
+ *
+ *********************************************************************/
+void
+ec_update_discard_write(ec_fop_data_t *fop, uintptr_t mask)
+{
+    ec_t *ec = fop->xl->private;
+    off_t off_head = 0;
+    off_t off_tail = 0;
+    uint64_t size_head = 0;
+    uint64_t size_tail = 0;
+    int error = 0;
+
+    off_head = fop->offset * ec->fragments - fop->int32;
+    if (fop->size == 0) {
+        error = ec_update_write(fop, mask, off_head, fop->user_size);
+    } else {
+        size_head = fop->int32;
+        size_tail = (off_head + fop->user_size) % ec->stripe_size;
+        off_tail = off_head + fop->user_size - size_tail;
+        if (size_head) {
+            error = ec_update_write(fop, mask, off_head, size_head);
+            if (error) {
+                goto out;
+            }
+        }
+        if (size_tail) {
+            error = ec_update_write(fop, mask, off_tail, size_tail);
+        }
+    }
+out:
+    if (error)
+        ec_fop_set_error(fop, -error);
+}
+
+void
+ec_discard_adjust_offset_size(ec_fop_data_t *fop)
+{
+    ec_t *ec = fop->xl->private;
+
+    fop->user_size = fop->size;
+    /* If discard length covers at least a fragment on brick, we will
+     * perform discard operation(when fop->size is non-zero) else we just
+     * write zeros.
+     */
+    fop->int32 = ec_adjust_offset_up(ec, &fop->offset, _gf_true);
+    fop->frag_range.first = fop->offset;
+    if (fop->size < fop->int32) {
+        fop->size = 0;
+    } else {
+        fop->size -= fop->int32;
+        ec_adjust_size_down(ec, &fop->size, _gf_true);
+    }
+    fop->frag_range.last = fop->offset + fop->size;
+}
+
+int32_t
+ec_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf,
+                              postbuf, xdata);
+}
+
+void
+ec_wind_discard(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
+
+    STACK_WIND_COOKIE(fop->frame, ec_discard_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->discard,
+                      fop->fd, fop->offset, fop->size, fop->xdata);
+}
+
+int32_t
+ec_manager_discard(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk = NULL;
+    off_t fl_start = 0;
+    uint64_t fl_size = 0;
+
+    switch (state) {
+        case EC_STATE_INIT:
+            if ((fop->size <= 0) || (fop->offset < 0)) {
+                ec_fop_set_error(fop, EINVAL);
+                return EC_STATE_REPORT;
+            }
+            /* Because of the head/tail writes, "discard" happens on the
+             * remaining regions, but we need to compute region including
+             * head/tail writes so compute them separately*/
+            fl_start = fop->offset;
+            fl_size = fop->size;
+            fl_size += ec_adjust_offset_down(fop->xl->private, &fl_start,
+                                             _gf_true);
+            ec_adjust_size_up(fop->xl->private, &fl_size, _gf_true);
+
+            ec_discard_adjust_offset_size(fop);
+
+            /* Fall through */
+
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd,
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fl_start, fl_size);
+            ec_lock(fop);
+
+            return EC_STATE_DISPATCH;
+
+        case EC_STATE_DISPATCH:
+
+            /* Dispatch discard fop only if we have whole fragment
+             * to deallocate */
+            if (fop->size) {
+                ec_dispatch_all(fop);
+                return EC_STATE_DELAYED_START;
+            } else {
+                /* Assume discard to have succeeded on all bricks */
+                ec_succeed_all(fop);
+            }
+
+            /* Fall through */
+
+        case EC_STATE_DELAYED_START:
+
+            if (fop->size) {
+                if (fop->answer && fop->answer->op_ret == 0)
+                    ec_update_discard_write(fop, fop->answer->mask);
+            } else {
+                ec_update_discard_write(fop, fop->mask);
+            }
+
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                /* This shouldn't fail because we have the inode locked. */
+                GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
+                                            &cbk->iatt[0].ia_size));
+
+                cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+            }
+            return EC_STATE_REPORT;
+
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+
+            GF_ASSERT(cbk != NULL);
+
+            if (fop->cbks.discard != NULL) {
+                QUORUM_CBK(fop->cbks.discard, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_DELAYED_START:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
+
+            if (fop->cbks.discard != NULL) {
+                fop->cbks.discard(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                  NULL, NULL, NULL);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
+
+            return EC_STATE_END;
+
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
+    }
+}
+
+void
+ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd,
+           off_t offset, size_t len, dict_t *xdata)
+{
+    ec_cbk_t callback = {.discard = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
+
+    gf_msg_trace("ec", 0, "EC(DISCARD) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_DISCARD, 0, target,
+                               fop_flags, ec_wind_discard, ec_manager_discard,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
-    vector.iov_base = iobuf->ptr;
-    vector.iov_len = fop->offset * ec->fragments - fop->user_size;
-    memset(vector.iov_base, 0, vector.iov_len);
+    fop->use_fd = 1;
+    fop->offset = offset;
+    fop->size = len;
 
-    iobuf_unref (iobuf);
-    iobuf = NULL;
+    if (fd != NULL) {
+        fop->fd = fd_ref(fd);
+    }
 
-    ec_writev(fop->frame, fop->xl, mask, fop->minimum, NULL, NULL, fop->fd,
-              &vector, 1, fop->user_size, 0, iobref, NULL);
+    if (xdata != NULL) {
+        fop->xdata = dict_ref(xdata);
+    }
 
-    err = 0;
+    error = 0;
 
 out:
-    if (iobuf != NULL) {
-        iobuf_unref(iobuf);
-    }
-    if (iobref != NULL) {
-        iobref_unref(iobref);
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
+}
 
-    return err;
+/*********************************************************************
+ *
+ * File Operation : truncate
+ *
+ *********************************************************************/
+
+int32_t
+ec_update_truncate_write(ec_fop_data_t *fop, uintptr_t mask)
+{
+    ec_t *ec = fop->xl->private;
+    uint64_t size = fop->offset * ec->fragments - fop->user_size;
+    return ec_update_write(fop, mask, fop->user_size, size);
 }
 
-int32_t ec_truncate_open_cbk(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             fd_t * fd, dict_t * xdata)
+int32_t
+ec_truncate_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = cookie;
+    ec_fop_data_t *fop = cookie;
     int32_t err;
 
+    fop->parent->good &= fop->good;
     if (op_ret >= 0) {
-        fd_bind (fd);
-        err = ec_truncate_write(fop->parent, fop->answer->mask);
+        fd_bind(fd);
+        err = ec_update_truncate_write(fop->parent, fop->answer->mask);
         if (err != 0) {
-            fop->error = -err;
+            ec_fop_set_error(fop->parent, -err);
         }
     }
 
     return 0;
 }
 
-int32_t ec_truncate_clean(ec_fop_data_t * fop)
+int32_t
+ec_truncate_clean(ec_fop_data_t *fop)
 {
     if (fop->fd == NULL) {
         fop->fd = fd_create(fop->loc[0].inode, fop->frame->root->pid);
@@ -900,24 +1371,25 @@ int32_t ec_truncate_clean(ec_fop_data_t * fop)
         }
 
         ec_open(fop->frame, fop->xl, fop->answer->mask, fop->minimum,
-                ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd,
-                NULL);
+                ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd, NULL);
 
         return 0;
     } else {
-        return ec_truncate_write(fop, fop->answer->mask);
+        return ec_update_truncate_write(fop, fop->answer->mask);
     }
 }
 
-int32_t ec_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *prestat,
-                         struct iatt *poststat, dict_t *xdata)
+int32_t
+ec_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_truncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -926,27 +1398,35 @@ void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->offset, fop->xdata);
 }
 
-int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_truncate(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    off_t offset_down;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             fop->user_size = fop->offset;
-            fop->offset = ec_adjust_size(fop->xl->private, fop->offset, 1);
+            ec_adjust_offset_up(fop->xl->private, &fop->offset, _gf_true);
+            fop->frag_range.first = fop->offset;
+            fop->frag_range.last = UINT64_MAX;
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
+            offset_down = fop->user_size;
+            ec_adjust_offset_down(fop->xl->private, &offset_down, _gf_true);
+
             if (fop->id == GF_FOP_TRUNCATE) {
-                ec_lock_prepare_inode(fop, &fop->loc[0],
-                                      EC_UPDATE_DATA | EC_UPDATE_META |
-                                      EC_QUERY_INFO);
+                ec_lock_prepare_inode(
+                    fop, &fop->loc[0],
+                    EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                    offset_down, EC_RANGE_FULL);
             } else {
-                ec_lock_prepare_fd(fop, fop->fd,
-                                   EC_UPDATE_DATA | EC_UPDATE_META |
-                                   EC_QUERY_INFO);
+                ec_lock_prepare_fd(
+                    fop, fop->fd,
+                    EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                    offset_down, EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -962,10 +1442,12 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
             if (cbk != NULL) {
                 int32_t err;
 
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
+                /* Inode size doesn't need to be updated under locks, because
+                 * conflicting operations won't be in-flight
+                 */
                 GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
                                             &cbk->iatt[0].ia_size));
                 cbk->iatt[1].ia_size = fop->user_size;
@@ -988,24 +1470,17 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_TRUNCATE)
-            {
-                if (fop->cbks.truncate != NULL)
-                {
-                    fop->cbks.truncate(fop->req_frame, fop, fop->xl,
-                                       cbk->op_ret, cbk->op_errno,
-                                       &cbk->iatt[0], &cbk->iatt[1],
-                                       cbk->xdata);
+            if (fop->id == GF_FOP_TRUNCATE) {
+                if (fop->cbks.truncate != NULL) {
+                    QUORUM_CBK(fop->cbks.truncate, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.ftruncate != NULL)
-                {
-                    fop->cbks.ftruncate(fop->req_frame, fop, fop->xl,
-                                        cbk->op_ret, cbk->op_errno,
-                                        &cbk->iatt[0], &cbk->iatt[1],
-                                        cbk->xdata);
+            } else {
+                if (fop->cbks.ftruncate != NULL) {
+                    QUORUM_CBK(fop->cbks.ftruncate, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
             }
 
@@ -1018,18 +1493,13 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_TRUNCATE)
-            {
-                if (fop->cbks.truncate != NULL)
-                {
+            if (fop->id == GF_FOP_TRUNCATE) {
+                if (fop->cbks.truncate != NULL) {
                     fop->cbks.truncate(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.ftruncate != NULL)
-                {
+            } else {
+                if (fop->cbks.ftruncate != NULL) {
                     fop->cbks.ftruncate(fop->req_frame, fop, fop->xl, -1,
                                         fop->error, NULL, NULL, NULL);
                 }
@@ -1050,31 +1520,30 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_truncate_cbk_t func, void * data,
-                 loc_t * loc, off_t offset, dict_t * xdata)
+void
+ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc,
+            off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .truncate = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.truncate = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(TRUNCATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(TRUNCATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_TRUNCATE, 0, target,
-                               minimum, ec_wind_truncate, ec_manager_truncate,
+                               fop_flags, ec_wind_truncate, ec_manager_truncate,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -1084,9 +1553,8 @@ void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1094,10 +1562,9 @@ void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1115,16 +1582,17 @@ out:
 
 /* FOP: ftruncate */
 
-int32_t ec_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *prestat, struct iatt *poststat,
-                          dict_t *xdata)
+int32_t
+ec_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                 struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_ftruncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1133,23 +1601,24 @@ void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->offset, fop->xdata);
 }
 
-void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_ftruncate_cbk_t func, void * data,
-                  fd_t * fd, off_t offset, dict_t * xdata)
+void
+ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd,
+             off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .ftruncate = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.ftruncate = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FTRUNCATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FTRUNCATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FTRUNCATE, 0, target,
-                               minimum, ec_wind_ftruncate, ec_manager_truncate,
-                               callback, data);
+                               fop_flags, ec_wind_ftruncate,
+                               ec_manager_truncate, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1161,10 +1630,9 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1172,10 +1640,9 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1192,24 +1659,92 @@ out:
 }
 
 /* FOP: writev */
+static ec_stripe_t *
+ec_allocate_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache)
+{
+    ec_stripe_t *stripe = NULL;
+
+    if (stripe_cache->count >= stripe_cache->max) {
+        GF_ASSERT(!list_empty(&stripe_cache->lru));
+        stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru);
+        list_move_tail(&stripe->lru, &stripe_cache->lru);
+        GF_ATOMIC_INC(ec->stats.stripe_cache.evicts);
+    } else {
+        stripe = GF_MALLOC(sizeof(ec_stripe_t) + ec->stripe_size,
+                           ec_mt_ec_stripe_t);
+        if (stripe != NULL) {
+            stripe_cache->count++;
+            list_add_tail(&stripe->lru, &stripe_cache->lru);
+            GF_ATOMIC_INC(ec->stats.stripe_cache.allocs);
+        } else {
+            GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
+        }
+    }
+
+    return stripe;
+}
 
-int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             struct iovec * vector, int32_t count,
-                             struct iatt * stbuf, struct iobref * iobref,
-                             dict_t * xdata)
+static void
+ec_write_stripe_data(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
 {
-    ec_t * ec = this->private;
-    ec_fop_data_t * fop = frame->local;
-    size_t size, base, tmp;
+    off_t base;
 
-    if (op_ret >= 0)
-    {
+    base = fop->size - ec->stripe_size;
+    memcpy(stripe->data, fop->vector[0].iov_base + base, ec->stripe_size);
+    stripe->frag_offset = fop->frag_range.last - ec->fragment_size;
+}
+
+static void
+ec_add_stripe_in_cache(ec_t *ec, ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    ec_stripe_t *stripe = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+    gf_boolean_t failed = _gf_true;
+
+    LOCK(&fop->fd->inode->lock);
+
+    ctx = __ec_inode_get(fop->fd->inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
+    }
+
+    stripe_cache = &ctx->stripe_cache;
+    if (stripe_cache->max > 0) {
+        stripe = ec_allocate_stripe(ec, stripe_cache);
+        if (stripe == NULL) {
+            goto out;
+        }
+
+        ec_write_stripe_data(ec, fop, stripe);
+    }
+
+    failed = _gf_false;
+
+out:
+    UNLOCK(&fop->fd->inode->lock);
+
+    if (failed) {
+        gf_msg(ec->xl->name, GF_LOG_DEBUG, ENOMEM, EC_MSG_FILE_DESC_REF_FAIL,
+               "Failed to create and add stripe in cache");
+    }
+}
+
+int32_t
+ec_writev_merge_tail(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                     int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                     dict_t *xdata)
+{
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop = frame->local;
+    uint64_t size, base, tmp;
+
+    if (op_ret >= 0) {
         tmp = 0;
         size = fop->size - fop->user_size - fop->head;
         base = ec->stripe_size - size;
-        if (op_ret > base)
-        {
+        if (op_ret > base) {
             tmp = min(op_ret - base, size);
             ec_iov_copy_to(fop->vector[0].iov_base + fop->size - size, vector,
                            count, base, tmp);
@@ -1217,46 +1752,44 @@ int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie,
             size -= tmp;
         }
 
-        if (size > 0)
-        {
+        if (size > 0) {
             memset(fop->vector[0].iov_base + fop->size - size, 0, size);
         }
-    }
 
+        if (ec->stripe_cache) {
+            ec_add_stripe_in_cache(ec, fop);
+        }
+    }
     return 0;
 }
 
-int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             struct iovec * vector, int32_t count,
-                             struct iatt * stbuf, struct iobref * iobref,
-                             dict_t * xdata)
+int32_t
+ec_writev_merge_head(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                     int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                     dict_t *xdata)
 {
-    ec_t * ec = this->private;
-    ec_fop_data_t * fop = frame->local;
-    size_t size, base;
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop = frame->local;
+    uint64_t size, base;
 
-    if (op_ret >= 0)
-    {
+    if (op_ret >= 0) {
         size = fop->head;
         base = 0;
 
-        if (op_ret > 0)
-        {
+        if (op_ret > 0) {
             base = min(op_ret, size);
             ec_iov_copy_to(fop->vector[0].iov_base, vector, count, 0, base);
 
             size -= base;
         }
 
-        if (size > 0)
-        {
+        if (size > 0) {
             memset(fop->vector[0].iov_base + base, 0, size);
         }
 
         size = fop->size - fop->user_size - fop->head;
-        if ((size > 0) && (fop->size == ec->stripe_size))
-        {
+        if ((size > 0) && (fop->size == ec->stripe_size)) {
             ec_writev_merge_tail(frame, cookie, this, op_ret, op_errno, vector,
                                  count, stbuf, iobref, xdata);
         }
@@ -1266,46 +1799,213 @@ int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie,
 }
 
 static int
-ec_make_internal_fop_xdata (dict_t **xdata)
+ec_make_internal_fop_xdata(dict_t **xdata)
 {
     dict_t *dict = NULL;
 
+    if (*xdata)
+        return 0;
+
     dict = dict_new();
     if (!dict)
-       goto out;
+        goto out;
 
-    if (dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes"))
-       goto out;
+    if (dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes"))
+        goto out;
 
     *xdata = dict;
     return 0;
 out:
     if (dict)
-            dict_unref (dict);
+        dict_unref(dict);
     return -1;
 }
 
-void ec_writev_start(ec_fop_data_t *fop)
+static int32_t
+ec_writev_prepare_buffers(ec_t *ec, ec_fop_data_t *fop)
 {
-    ec_t *ec = fop->xl->private;
     struct iobref *iobref = NULL;
-    struct iobuf *iobuf = NULL;
-    void *ptr = NULL;
+    struct iovec *iov;
+    void *ptr;
+    int32_t err;
+
+    fop->user_size = iov_length(fop->vector, fop->int32);
+    fop->head = ec_adjust_offset_down(ec, &fop->offset, _gf_false);
+    fop->frag_range.first = fop->offset / ec->fragments;
+    fop->size = fop->user_size + fop->head;
+    ec_adjust_size_up(ec, &fop->size, _gf_false);
+    fop->frag_range.last = fop->frag_range.first + fop->size / ec->fragments;
+
+    if ((fop->int32 != 1) || (fop->head != 0) || (fop->size > fop->user_size) ||
+        !EC_ALIGN_CHECK(fop->vector[0].iov_base, EC_METHOD_WORD_SIZE)) {
+        err = ec_buffer_alloc(ec->xl, fop->size, &iobref, &ptr);
+        if (err != 0) {
+            goto out;
+        }
+
+        ec_iov_copy_to(ptr + fop->head, fop->vector, fop->int32, 0,
+                       fop->user_size);
+
+        fop->vector[0].iov_base = ptr;
+        fop->vector[0].iov_len = fop->size;
+
+        iobref_unref(fop->buffers);
+        fop->buffers = iobref;
+    }
+
+    if (fop->int32 != 2) {
+        iov = GF_MALLOC(VECTORSIZE(2), gf_common_mt_iovec);
+        if (iov == NULL) {
+            err = -ENOMEM;
+
+            goto out;
+        }
+        iov[0].iov_base = fop->vector[0].iov_base;
+        iov[0].iov_len = fop->vector[0].iov_len;
+
+        GF_FREE(fop->vector);
+        fop->vector = iov;
+    }
+
+    fop->vector[1].iov_len = fop->size / ec->fragments;
+    err = ec_buffer_alloc(ec->xl, fop->vector[1].iov_len * ec->nodes,
+                          &fop->buffers, &fop->vector[1].iov_base);
+    if (err != 0) {
+        goto out;
+    }
+
+    err = 0;
+
+out:
+    return err;
+}
+
+static void
+ec_merge_stripe_head_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
+{
+    uint32_t head, size;
+
+    head = fop->head;
+    memcpy(fop->vector[0].iov_base, stripe->data, head);
+
+    size = ec->stripe_size - head;
+    if (size > fop->user_size) {
+        head += fop->user_size;
+        size = ec->stripe_size - head;
+        memcpy(fop->vector[0].iov_base + head, stripe->data + head, size);
+    }
+}
+
+static void
+ec_merge_stripe_tail_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
+{
+    uint32_t head, tail;
+    off_t offset;
+
+    offset = fop->user_size + fop->head;
+    tail = fop->size - offset;
+    head = ec->stripe_size - tail;
+
+    memcpy(fop->vector[0].iov_base + offset, stripe->data + head, tail);
+}
+
+static ec_stripe_t *
+ec_get_stripe_from_cache_locked(ec_t *ec, ec_fop_data_t *fop,
+                                uint64_t frag_offset)
+{
+    ec_inode_t *ctx = NULL;
+    ec_stripe_t *stripe = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+
+    ctx = __ec_inode_get(fop->fd->inode, fop->xl);
+    if (ctx == NULL) {
+        GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
+        return NULL;
+    }
+
+    stripe_cache = &ctx->stripe_cache;
+    list_for_each_entry(stripe, &stripe_cache->lru, lru)
+    {
+        if (stripe->frag_offset == frag_offset) {
+            list_move_tail(&stripe->lru, &stripe_cache->lru);
+            GF_ATOMIC_INC(ec->stats.stripe_cache.hits);
+            return stripe;
+        }
+    }
+
+    GF_ATOMIC_INC(ec->stats.stripe_cache.misses);
+
+    return NULL;
+}
+
+static gf_boolean_t
+ec_get_and_merge_stripe(ec_t *ec, ec_fop_data_t *fop, ec_stripe_part_t which)
+{
+    uint64_t frag_offset;
+    ec_stripe_t *stripe = NULL;
+    gf_boolean_t found = _gf_false;
+
+    if (!ec->stripe_cache) {
+        return found;
+    }
+
+    LOCK(&fop->fd->inode->lock);
+    if (which == EC_STRIPE_HEAD) {
+        frag_offset = fop->frag_range.first;
+        stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
+        if (stripe) {
+            ec_merge_stripe_head_locked(ec, fop, stripe);
+            found = _gf_true;
+        }
+    }
+
+    if (which == EC_STRIPE_TAIL) {
+        frag_offset = fop->frag_range.last - ec->fragment_size;
+        stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
+        if (stripe) {
+            ec_merge_stripe_tail_locked(ec, fop, stripe);
+            found = _gf_true;
+        }
+    }
+    UNLOCK(&fop->fd->inode->lock);
+
+    return found;
+}
+
+static uintptr_t
+ec_get_lock_good_mask(inode_t *inode, xlator_t *xl)
+{
+    ec_lock_t *lock = NULL;
+    ec_inode_t *ictx = NULL;
+    LOCK(&inode->lock);
+    {
+        ictx = __ec_inode_get(inode, xl);
+        if (ictx)
+            lock = ictx->inode_lock;
+    }
+    UNLOCK(&inode->lock);
+    if (lock)
+        return lock->good_mask;
+    return 0;
+}
+
+void
+ec_writev_start(ec_fop_data_t *fop)
+{
+    ec_t *ec = fop->xl->private;
     ec_fd_t *ctx;
     fd_t *fd;
-    size_t tail;
-    uint64_t current;
+    dict_t *xdata = NULL;
+    uint64_t tail, current;
     int32_t err = -ENOMEM;
-    dict_t      *xdata = NULL;
+    gf_boolean_t found_stripe = _gf_false;
 
     /* This shouldn't fail because we have the inode locked. */
     GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, &current));
 
     fd = fd_anonymous(fop->fd->inode);
     if (fd == NULL) {
-        ec_fop_set_error(fop, ENOMEM);
-
-        return;
+        goto failed;
     }
 
     fop->frame->root->uid = 0;
@@ -1314,165 +2014,157 @@ void ec_writev_start(ec_fop_data_t *fop)
     ctx = ec_fd_get(fop->fd, fop->xl);
     if (ctx != NULL) {
         if ((ctx->flags & O_APPEND) != 0) {
+            /* Appending writes take full locks so size won't change because
+             * of any parallel operations
+             */
             fop->offset = current;
         }
     }
 
-    fop->user_size = iov_length(fop->vector, fop->int32);
-    fop->head = ec_adjust_offset(ec, &fop->offset, 0);
-    fop->size = ec_adjust_size(ec, fop->user_size + fop->head, 0);
-
-    iobref = iobref_new();
-    if (iobref == NULL) {
-        goto out;
-    }
-    iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, fop->size);
-    if (iobuf == NULL) {
-        goto out;
-    }
-    err = iobref_add(iobref, iobuf);
+    err = ec_writev_prepare_buffers(ec, fop);
     if (err != 0) {
-        goto out;
+        goto failed_fd;
     }
-
-    ptr = iobuf->ptr + fop->head;
-    ec_iov_copy_to(ptr, fop->vector, fop->int32, 0, fop->user_size);
-
-    fop->vector[0].iov_base = iobuf->ptr;
-    fop->vector[0].iov_len = fop->size;
-
-    iobuf_unref(iobuf);
-
-    iobref_unref(fop->buffers);
-    fop->buffers = iobref;
-
+    tail = fop->size - fop->user_size - fop->head;
     if (fop->head > 0) {
-        if (ec_make_internal_fop_xdata (&xdata)) {
-                err = -ENOMEM;
-                goto out;
+        if (current > fop->offset) {
+            found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_HEAD);
+            if (!found_stripe) {
+                if (ec_make_internal_fop_xdata(&xdata)) {
+                    err = -ENOMEM;
+                    goto failed_xdata;
+                }
+                ec_readv(fop->frame, fop->xl,
+                         ec_get_lock_good_mask(fop->fd->inode, fop->xl),
+                         EC_MINIMUM_MIN, ec_writev_merge_head, NULL, fd,
+                         ec->stripe_size, fop->offset, 0, xdata);
+            }
+        } else {
+            memset(fop->vector[0].iov_base, 0, fop->head);
+            memset(fop->vector[0].iov_base + fop->size - tail, 0, tail);
+            if (ec->stripe_cache && (fop->size <= ec->stripe_size)) {
+                ec_add_stripe_in_cache(ec, fop);
+            }
         }
-        ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN, ec_writev_merge_head,
-                 NULL, fd, ec->stripe_size, fop->offset, 0, xdata);
     }
-    tail = fop->size - fop->user_size - fop->head;
+
     if ((tail > 0) && ((fop->head == 0) || (fop->size > ec->stripe_size))) {
+        /* Current locking scheme will make sure the 'current' below will
+         * never decrease while the fop is in progress, so the checks will
+         * work as expected
+         */
         if (current > fop->offset + fop->head + fop->user_size) {
-            if (ec_make_internal_fop_xdata (&xdata)) {
+            found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_TAIL);
+            if (!found_stripe) {
+                if (ec_make_internal_fop_xdata(&xdata)) {
                     err = -ENOMEM;
-                    goto out;
+                    goto failed_xdata;
+                }
+                ec_readv(fop->frame, fop->xl,
+                         ec_get_lock_good_mask(fop->fd->inode, fop->xl),
+                         EC_MINIMUM_MIN, ec_writev_merge_tail, NULL, fd,
+                         ec->stripe_size,
+                         fop->offset + fop->size - ec->stripe_size, 0, xdata);
             }
-            ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN,
-                     ec_writev_merge_tail, NULL, fd, ec->stripe_size,
-                     fop->offset + fop->size - ec->stripe_size, 0, xdata);
         } else {
             memset(fop->vector[0].iov_base + fop->size - tail, 0, tail);
+            if (ec->stripe_cache) {
+                ec_add_stripe_in_cache(ec, fop);
+            }
         }
     }
 
-    fd_unref(fd);
-    if (xdata)
-            dict_unref (xdata);
-
-    return;
+    err = 0;
 
-out:
-    if (iobuf != NULL) {
-        iobuf_unref(iobuf);
-    }
-    if (iobref != NULL) {
-        iobref_unref(iobref);
+failed_xdata:
+    if (xdata) {
+        dict_unref(xdata);
     }
-
+failed_fd:
     fd_unref(fd);
-    if (xdata)
-            dict_unref (xdata);
-
+failed:
     ec_fop_set_error(fop, -err);
 }
 
-int32_t ec_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prestat,
-                       struct iatt *poststat, dict_t *xdata)
+int32_t
+ec_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prestat, struct iatt *poststat,
+              dict_t *xdata)
 {
-        ec_t    *ec = NULL;
-        if (this && this->private) {
-                ec = this->private;
-                if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) {
-                        op_ret = -1;
-                        op_errno = EIO;
-                }
+    ec_t *ec = NULL;
+    if (this && this->private) {
+        ec = this->private;
+        if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) {
+            op_ret = -1;
+            op_errno = EIO;
         }
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    }
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_writev(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_writev(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
     struct iovec vector[1];
-    struct iobref * iobref = NULL;
-    struct iobuf * iobuf = NULL;
-    ssize_t size = 0, bufsize = 0;
-    int32_t err = -ENOMEM;
+    size_t size;
 
-    iobref = iobref_new();
-    if (iobref == NULL) {
-        goto out;
-    }
-
-    size = fop->vector[0].iov_len;
-    bufsize = size / ec->fragments;
-
-    iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, bufsize);
-    if (iobuf == NULL) {
-        goto out;
-    }
-    err = iobref_add(iobref, iobuf);
-    if (err != 0) {
-        goto out;
-    }
-
-    ec_method_encode(size, ec->fragments, idx, fop->vector[0].iov_base,
-                     iobuf->ptr);
-
-    vector[0].iov_base = iobuf->ptr;
-    vector[0].iov_len = bufsize;
+    size = fop->vector[1].iov_len;
 
-    iobuf_unref(iobuf);
+    vector[0].iov_base = fop->vector[1].iov_base + idx * size;
+    vector[0].iov_len = size;
 
     STACK_WIND_COOKIE(fop->frame, ec_writev_cbk, (void *)(uintptr_t)idx,
-                      ec->xl_list[idx], ec->xl_list[idx]->fops->writev,
-                      fop->fd, vector, 1, fop->offset / ec->fragments,
-                      fop->uint32, iobref, fop->xdata);
-
-    iobref_unref(iobref);
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->writev, fop->fd,
+                      vector, 1, fop->offset / ec->fragments, fop->uint32,
+                      fop->buffers, fop->xdata);
+}
 
-    return;
+static void
+ec_writev_encode(ec_fop_data_t *fop)
+{
+    ec_t *ec = fop->xl->private;
+    void *blocks[ec->nodes];
+    uint32_t i;
 
-out:
-    if (iobuf != NULL) {
-        iobuf_unref(iobuf);
-    }
-    if (iobref != NULL) {
-        iobref_unref(iobref);
+    blocks[0] = fop->vector[1].iov_base;
+    for (i = 1; i < ec->nodes; i++) {
+        blocks[i] = blocks[i - 1] + fop->vector[1].iov_len;
     }
-
-    ec_writev_cbk(fop->frame, (void *)(uintptr_t)idx, fop->xl, -1, -err, NULL,
-                  NULL, NULL);
+    ec_method_encode(&ec->matrix, fop->vector[0].iov_len,
+                     fop->vector[0].iov_base, blocks);
 }
 
-int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_writev(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk;
+    ec_fd_t *ctx = NULL;
+    ec_t *ec = fop->xl->private;
+    off_t fl_start = 0;
+    uint64_t fl_size = LONG_MAX;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
+            ctx = ec_fd_get(fop->fd, fop->xl);
+            if (ctx != NULL) {
+                if ((ctx->flags & O_APPEND) == 0) {
+                    off_t user_size = 0;
+                    off_t head = 0;
+
+                    fl_start = fop->offset;
+                    user_size = iov_length(fop->vector, fop->int32);
+                    head = ec_adjust_offset_down(ec, &fl_start, _gf_true);
+                    fl_size = user_size + head;
+                    ec_adjust_size_up(ec, &fl_size, _gf_true);
+                }
+            }
             ec_lock_prepare_fd(fop, fop->fd,
-                               EC_UPDATE_DATA | EC_UPDATE_META |
-                               EC_QUERY_INFO);
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fl_start, fl_size);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -1488,6 +2180,8 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
             fop->frame->root->uid = fop->uid;
             fop->frame->root->gid = fop->gid;
 
+            ec_writev_encode(fop);
+
             ec_dispatch_all(fop);
 
             return EC_STATE_PREPARE_ANSWER;
@@ -1496,29 +2190,33 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
             cbk = ec_fop_prepare_answer(fop, _gf_false);
             if (cbk != NULL) {
                 ec_t *ec = fop->xl->private;
-                size_t size;
+                uint64_t size;
 
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
-                GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode,
-                                            &cbk->iatt[0].ia_size));
-                cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
-                size = fop->offset + fop->head + fop->user_size;
-                if (size > cbk->iatt[0].ia_size) {
-                    /* Only update inode size if this is a top level fop.
-                     * Otherwise this is an internal write and the top
-                     * level fop should take care of the real inode size.
-                     */
-                    if (fop->parent == NULL) {
-                        /* This shouldn't fail because we have the inode
-                         * locked. */
-                        GF_ASSERT(ec_set_inode_size(fop, fop->fd->inode,
-                                                    size));
+                LOCK(&fop->fd->inode->lock);
+                {
+                    GF_ASSERT(__ec_get_inode_size(fop, fop->fd->inode,
+                                                  &cbk->iatt[0].ia_size));
+                    cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    size = fop->offset + fop->head + fop->user_size;
+                    if (size > cbk->iatt[0].ia_size) {
+                        /* Only update inode size if this is a top level fop.
+                         * Otherwise this is an internal write and the top
+                         * level fop should take care of the real inode size.
+                         */
+                        if (fop->parent == NULL) {
+                            /* This shouldn't fail because we have the inode
+                             * locked. */
+                            GF_ASSERT(
+                                __ec_set_inode_size(fop, fop->fd->inode, size));
+                        }
+                        cbk->iatt[1].ia_size = size;
                     }
-                    cbk->iatt[1].ia_size = size;
                 }
+                UNLOCK(&fop->fd->inode->lock);
+
                 if (fop->error == 0) {
                     cbk->op_ret *= ec->fragments;
                     if (cbk->op_ret < fop->head) {
@@ -1539,11 +2237,10 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.writev != NULL)
-            {
-                fop->cbks.writev(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 cbk->xdata);
+            if (fop->cbks.writev != NULL) {
+                QUORUM_CBK(fop->cbks.writev, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1554,7 +2251,7 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
             fop->frame->root->uid = fop->uid;
             fop->frame->root->gid = fop->gid;
 
-        /* Fall through */
+            /* Fall through */
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
@@ -1563,8 +2260,7 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.writev != NULL)
-            {
+            if (fop->cbks.writev != NULL) {
                 fop->cbks.writev(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL);
             }
@@ -1584,31 +2280,30 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_writev_cbk_t func, void * data, fd_t * fd,
-               struct iovec * vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref * iobref, dict_t * xdata)
+void
+ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd,
+          struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+          struct iobref *iobref, dict_t *xdata)
 {
-    ec_cbk_t callback = { .writev = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.writev = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(WRITE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(WRITE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, 0, target, fop_flags,
                                ec_wind_writev, ec_manager_writev, callback,
                                data);
     if (fop == NULL) {
@@ -1624,10 +2319,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1635,10 +2329,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (count > 0) {
         fop->vector = iov_dup(vector, count);
         if (fop->vector == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a "
-                    "vector list.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a "
+                   "vector list.");
 
             goto out;
         }
@@ -1647,10 +2340,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (iobref != NULL) {
         fop->buffers = iobref_ref(iobref);
         if (fop->buffers == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_BUF_REF_FAIL,
-                    "Failed to reference a "
-                    "buffer.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL,
+                   "Failed to reference a "
+                   "buffer.");
 
             goto out;
         }
@@ -1658,10 +2350,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-locks.c b/xlators/cluster/ec/src/ec-locks.c
index 7902a99ba84..601960d6154 100644
--- a/xlators/cluster/ec/src/ec-locks.c
+++ b/xlators/cluster/ec/src/ec-locks.c
@@ -8,52 +8,76 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
 #include "ec-messages.h"
 
 #define EC_LOCK_MODE_NONE 0
-#define EC_LOCK_MODE_INC  1
-#define EC_LOCK_MODE_ALL  2
+#define EC_LOCK_MODE_INC 1
+#define EC_LOCK_MODE_ALL 2
 
-int32_t ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
+int32_t
+ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
 {
     ec_t *ec = fop->xl->private;
     ec_cbk_data_t *ans = NULL;
     ec_cbk_data_t *cbk = NULL;
-    uintptr_t locked = 0, notlocked = 0;
+    uintptr_t locked = 0;
+    int32_t good = 0;
+    int32_t eagain = 0;
+    int32_t estale = 0;
     int32_t error = -1;
 
-    list_for_each_entry(ans, &fop->cbk_list, list) {
+    /* There are some errors that we'll handle in an special way while trying
+     * to acquire a lock.
+     *
+     *   EAGAIN:  If it's found during a parallel non-blocking lock request, we
+     *            consider that there's contention on the inode, so we consider
+     *            the acquisition a failure and try again with a sequential
+     *            blocking lock request. This will ensure that we get a lock on
+     *            as many bricks as possible (ignoring EAGAIN here would cause
+     *            unnecessary triggers of self-healing).
+     *
+     *            If it's found during a sequential blocking lock request, it's
+     *            considered an error. Lock will only succeed if there are
+     *            enough other bricks locked.
+     *
+     *   ESTALE:  This can appear during parallel or sequential lock request if
+     *            the inode has just been unlinked. We consider this error is
+     *            not recoverable, but we also don't consider it as fatal. So,
+     *            if it happens during parallel lock, we won't attempt a
+     *            sequential one unless there are EAGAIN errors on other
+     *            bricks (and are enough to form a quorum), but if we reach
+     *            quorum counting the ESTALE bricks, we consider the whole
+     *            result of the operation is ESTALE instead of EIO.
+     */
+
+    list_for_each_entry(ans, &fop->cbk_list, list)
+    {
         if (ans->op_ret >= 0) {
             if (locked != 0) {
                 error = EIO;
             }
             locked |= ans->mask;
+            good = ans->count;
             cbk = ans;
-        } else {
-                if (ans->op_errno == EAGAIN) {
-                        switch (fop->uint32) {
-                        case EC_LOCK_MODE_NONE:
-                        case EC_LOCK_MODE_ALL:
-                                /* Goal is to treat non-blocking lock as failure
-                                 * even if there is a signle EAGAIN*/
-                                notlocked |= ans->mask;
-                                break;
-                        }
-                }
+        } else if (ans->op_errno == ESTALE) {
+            estale += ans->count;
+        } else if ((ans->op_errno == EAGAIN) &&
+                   (fop->uint32 != EC_LOCK_MODE_INC)) {
+            eagain += ans->count;
         }
     }
 
     if (error == -1) {
-        if (ec_bits_count(locked | notlocked) >= ec->fragments) {
-            if (notlocked == 0) {
+        /* If we have enough quorum with succeeded and EAGAIN answers, we
+         * ignore for now any ESTALE answer. If there are EAGAIN answers,
+         * we retry with a sequential blocking lock request if needed.
+         * Otherwise we succeed. */
+        if ((good + eagain) >= ec->fragments) {
+            if (eagain == 0) {
                 if (fop->answer == NULL) {
                     fop->answer = cbk;
                 }
@@ -63,24 +87,31 @@ int32_t ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
                 error = 0;
             } else {
                 switch (fop->uint32) {
-                case EC_LOCK_MODE_NONE:
-                    error = EAGAIN;
-                    break;
-
-                case EC_LOCK_MODE_ALL:
-                    fop->uint32 = EC_LOCK_MODE_INC;
-                    break;
-
-                default:
-                    error = EIO;
-                    break;
+                    case EC_LOCK_MODE_NONE:
+                        error = EAGAIN;
+                        break;
+                    case EC_LOCK_MODE_ALL:
+                        fop->uint32 = EC_LOCK_MODE_INC;
+                        break;
+                    default:
+                        /* This shouldn't happen because eagain cannot be > 0
+                         * when fop->uint32 is EC_LOCK_MODE_INC. */
+                        error = EIO;
+                        break;
                 }
             }
         } else {
-            if (fop->answer && fop->answer->op_ret < 0)
-                    error = fop->answer->op_errno;
-            else
-                    error = EIO;
+            /* We have been unable to find enough candidates that will be able
+             * to take the lock. If we have quorum on some answer, we return
+             * it. Otherwise we check if ESTALE answers allow us to reach
+             * quorum. If so, we return ESTALE. */
+            if (fop->answer && fop->answer->op_ret < 0) {
+                error = fop->answer->op_errno;
+            } else if ((good + eagain + estale) >= ec->fragments) {
+                error = ESTALE;
+            } else {
+                error = EIO;
+            }
         }
     }
 
@@ -89,28 +120,25 @@ int32_t ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
     return error;
 }
 
-int32_t ec_lock_unlocked(call_frame_t * frame, void * cookie,
-                         xlator_t * this, int32_t op_ret, int32_t op_errno,
-                         dict_t * xdata)
+int32_t
+ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    if (op_ret < 0)
-    {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_UNLOCK_FAILED,
-                "Failed to unlock an entry/inode");
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED,
+               "Failed to unlock an entry/inode");
     }
 
     return 0;
 }
 
-int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_lock_lk_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                    dict_t *xdata)
 {
-    if (op_ret < 0)
-    {
-        gf_msg(this->name, GF_LOG_WARNING, op_errno,
-               EC_MSG_LK_UNLOCK_FAILED,
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_LK_UNLOCK_FAILED,
                "Failed to unlock an lk");
     }
 
@@ -119,11 +147,12 @@ int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie,
 
 /* FOP: entrylk */
 
-int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -133,20 +162,16 @@ int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ENTRYLK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       EC_MSG_DICT_REF_FAIL,
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
                        "Failed to reference a "
                        "dictionary.");
 
@@ -158,15 +183,15 @@ int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_entrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -176,20 +201,19 @@ void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->entrylk_type, fop->xdata);
 }
 
-int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_entrylk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            if (fop->entrylk_cmd == ENTRYLK_LOCK)
-            {
+            if (fop->entrylk_cmd == ENTRYLK_LOCK) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->entrylk_cmd = ENTRYLK_LOCK_NB;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -197,18 +221,18 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_PREPARE_ANSWER:
             if (fop->entrylk_cmd != ENTRYLK_UNLOCK) {
                 uintptr_t mask;
 
-                fop->error = ec_lock_check(fop, &mask);
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
                 if (fop->error != 0) {
                     if (mask != 0) {
                         if (fop->id == GF_FOP_ENTRYLK) {
-                            ec_entrylk(fop->frame, fop->xl, mask, 1,
-                                       ec_lock_unlocked, NULL, fop->str[0],
-                                       &fop->loc[0], fop->str[1],
-                                       ENTRYLK_UNLOCK, fop->entrylk_type,
-                                       fop->xdata);
+                            ec_entrylk(
+                                fop->frame, fop->xl, mask, 1, ec_lock_unlocked,
+                                NULL, fop->str[0], &fop->loc[0], fop->str[1],
+                                ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata);
                         } else {
                             ec_fentrylk(fop->frame, fop->xl, mask, 1,
                                         ec_lock_unlocked, NULL, fop->str[0],
@@ -237,18 +261,13 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_ENTRYLK)
-            {
-                if (fop->cbks.entrylk != NULL)
-                {
-                    fop->cbks.entrylk(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->xdata);
+            if (fop->id == GF_FOP_ENTRYLK) {
+                if (fop->cbks.entrylk != NULL) {
+                    fop->cbks.entrylk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fentrylk != NULL)
-                {
+            } else {
+                if (fop->cbks.fentrylk != NULL) {
                     fop->cbks.fentrylk(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->xdata);
                 }
@@ -258,22 +277,16 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
-        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_ENTRYLK)
-            {
-                if (fop->cbks.entrylk != NULL)
-                {
+            if (fop->id == GF_FOP_ENTRYLK) {
+                if (fop->cbks.entrylk != NULL) {
                     fop->cbks.entrylk(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fentrylk != NULL)
-                {
+            } else {
+                if (fop->cbks.fentrylk != NULL) {
                     fop->cbks.fentrylk(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL);
                 }
@@ -282,33 +295,31 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_entrylk_cbk_t func, void * data,
-                const char * volume, loc_t * loc, const char * basename,
-                entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+void
+ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_entrylk_cbk_t func, void *data,
+           const char *volume, loc_t *loc, const char *basename,
+           entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-    ec_cbk_t callback = { .entrylk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.entrylk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(ENTRYLK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(ENTRYLK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, 0, target, minimum,
-                               ec_wind_entrylk, ec_manager_entrylk, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, 0, target,
+                               fop_flags, ec_wind_entrylk, ec_manager_entrylk,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -319,18 +330,16 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -338,9 +347,8 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (basename != NULL) {
         fop->str[1] = gf_strdup(basename);
         if (fop->str[1] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -348,10 +356,9 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -369,11 +376,12 @@ out:
 
 /* FOP: fentrylk */
 
-int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -383,22 +391,18 @@ int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FENTRYLK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                         "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -408,15 +412,15 @@ int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fentrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -426,23 +430,23 @@ void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->entrylk_type, fop->xdata);
 }
 
-void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fentrylk_cbk_t func, void * data,
-                 const char * volume, fd_t * fd, const char * basename,
-                 entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+void
+ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data,
+            const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fentrylk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fentrylk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FENTRYLK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FENTRYLK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FENTRYLK, 0, target,
-                               minimum, ec_wind_fentrylk, ec_manager_entrylk,
+                               fop_flags, ec_wind_fentrylk, ec_manager_entrylk,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -456,9 +460,8 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -466,10 +469,9 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -477,9 +479,8 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (basename != NULL) {
         fop->str[1] = gf_strdup(basename);
         if (fop->str[1] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -487,10 +488,9 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -508,11 +508,12 @@ out:
 
 /* FOP: inodelk */
 
-int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -522,22 +523,18 @@ int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_INODELK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -547,15 +544,15 @@ int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_inodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -565,24 +562,22 @@ void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_inodelk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            fop->flock.l_len += ec_adjust_offset(fop->xl->private,
-                                                 &fop->flock.l_start, 1);
-            fop->flock.l_len = ec_adjust_size(fop->xl->private,
-                                              fop->flock.l_len, 1);
-            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK))
-            {
+            fop->flock.l_len += ec_adjust_offset_down(
+                fop->xl->private, &fop->flock.l_start, _gf_true);
+            ec_adjust_offset_up(fop->xl->private, &fop->flock.l_len, _gf_true);
+            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->int32 = F_SETLK;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -590,10 +585,11 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_PREPARE_ANSWER:
             if (fop->flock.l_type != F_UNLCK) {
                 uintptr_t mask;
 
-                fop->error = ec_lock_check(fop, &mask);
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
                 if (fop->error != 0) {
                     if (mask != 0) {
                         ec_t *ec = fop->xl->private;
@@ -607,12 +603,14 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
                         flock.l_owner.len = 0;
 
                         if (fop->id == GF_FOP_INODELK) {
-                            ec_inodelk(fop->frame, fop->xl, mask, 1,
+                            ec_inodelk(fop->frame, fop->xl,
+                                       &fop->frame->root->lk_owner, mask, 1,
                                        ec_lock_unlocked, NULL, fop->str[0],
                                        &fop->loc[0], F_SETLK, &flock,
                                        fop->xdata);
                         } else {
-                            ec_finodelk(fop->frame, fop->xl, mask, 1,
+                            ec_finodelk(fop->frame, fop->xl,
+                                        &fop->frame->root->lk_owner, mask, 1,
                                         ec_lock_unlocked, NULL, fop->str[0],
                                         fop->fd, F_SETLK, &flock, fop->xdata);
                         }
@@ -638,18 +636,13 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_INODELK)
-            {
-                if (fop->cbks.inodelk != NULL)
-                {
-                    fop->cbks.inodelk(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->xdata);
+            if (fop->id == GF_FOP_INODELK) {
+                if (fop->cbks.inodelk != NULL) {
+                    fop->cbks.inodelk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.finodelk != NULL)
-                {
+            } else {
+                if (fop->cbks.finodelk != NULL) {
                     fop->cbks.finodelk(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->xdata);
                 }
@@ -659,22 +652,16 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
-        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_INODELK)
-            {
-                if (fop->cbks.inodelk != NULL)
-                {
+            if (fop->id == GF_FOP_INODELK) {
+                if (fop->cbks.inodelk != NULL) {
                     fop->cbks.inodelk(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.finodelk != NULL)
-                {
+            } else {
+                if (fop->cbks.finodelk != NULL) {
                     fop->cbks.finodelk(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL);
                 }
@@ -683,54 +670,52 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_inodelk_cbk_t func, void * data,
-                const char * volume, loc_t * loc, int32_t cmd,
-                struct gf_flock * flock, dict_t * xdata)
+void
+ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+           uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func,
+           void *data, const char *volume, loc_t *loc, int32_t cmd,
+           struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .inodelk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.inodelk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(INODELK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(INODELK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, 0, target, minimum,
-                               ec_wind_inodelk, ec_manager_inodelk, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, 0, target,
+                               fop_flags, ec_wind_inodelk, ec_manager_inodelk,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = cmd;
+    ec_owner_copy(fop->frame, owner);
 
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -750,10 +735,9 @@ void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -771,11 +755,12 @@ out:
 
 /* FOP: finodelk */
 
-int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -785,22 +770,18 @@ int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FINODELK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -810,15 +791,15 @@ int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_finodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -828,23 +809,24 @@ void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_finodelk_cbk_t func, void * data,
-                 const char * volume, fd_t * fd, int32_t cmd,
-                 struct gf_flock * flock, dict_t * xdata)
+void
+ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+            uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func,
+            void *data, const char *volume, fd_t *fd, int32_t cmd,
+            struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .finodelk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.finodelk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FINODELK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FINODELK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FINODELK, 0, target,
-                               minimum, ec_wind_finodelk, ec_manager_inodelk,
+                               fop_flags, ec_wind_finodelk, ec_manager_inodelk,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -853,13 +835,13 @@ void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     fop->use_fd = 1;
 
     fop->int32 = cmd;
+    ec_owner_copy(fop->frame, owner);
 
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -867,10 +849,9 @@ void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -890,10 +871,9 @@ void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -911,15 +891,13 @@ out:
 
 /* FOP: lk */
 
-int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                      ec_cbk_data_t * src)
+int32_t
+ec_combine_lk(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_flock_compare(&dst->flock, &src->flock))
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_LOCK_MISMATCH,
-                "Mismatching lock in "
-                "answers of 'GF_FOP_LK'");
+    if (!ec_flock_compare(&dst->flock, &src->flock)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_LOCK_MISMATCH,
+               "Mismatching lock in "
+               "answers of 'GF_FOP_LK'");
 
         return 0;
     }
@@ -927,12 +905,12 @@ int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                  int32_t op_ret, int32_t op_errno, struct gf_flock * flock,
-                  dict_t * xdata)
+int32_t
+ec_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -942,39 +920,32 @@ int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (flock != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (flock != NULL) {
                 cbk->flock.l_type = flock->l_type;
                 cbk->flock.l_whence = flock->l_whence;
                 cbk->flock.l_start = flock->l_start;
                 cbk->flock.l_len = flock->l_len;
                 cbk->flock.l_pid = flock->l_pid;
                 cbk->flock.l_owner.len = flock->l_owner.len;
-                if (flock->l_owner.len > 0)
-                {
+                if (flock->l_owner.len > 0) {
                     memcpy(cbk->flock.l_owner.data, flock->l_owner.data,
                            flock->l_owner.len);
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -984,15 +955,15 @@ int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_lk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1001,24 +972,19 @@ void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->int32, &fop->flock, fop->xdata);
 }
 
-int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_lk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            fop->flock.l_len += ec_adjust_offset(fop->xl->private,
-                                                 &fop->flock.l_start, 1);
-            fop->flock.l_len = ec_adjust_size(fop->xl->private,
-                                              fop->flock.l_len, 1);
-            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK))
-            {
+            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->int32 = F_SETLK;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -1026,26 +992,26 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_PREPARE_ANSWER:
             if (fop->flock.l_type != F_UNLCK) {
                 uintptr_t mask;
 
-                fop->error = ec_lock_check(fop, &mask);
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
                 if (fop->error != 0) {
                     if (mask != 0) {
-                        ec_t *ec = fop->xl->private;
-                        struct gf_flock flock;
+                        struct gf_flock flock = {0};
 
                         flock.l_type = F_UNLCK;
                         flock.l_whence = fop->flock.l_whence;
-                        flock.l_start = fop->flock.l_start * ec->fragments;
-                        flock.l_len = fop->flock.l_len * ec->fragments;
-                        flock.l_pid = 0;
-                        flock.l_owner.len = 0;
+                        flock.l_start = fop->flock.l_start;
+                        flock.l_len = fop->flock.l_len;
+                        flock.l_pid = fop->flock.l_pid;
+                        lk_owner_copy(&flock.l_owner, &fop->flock.l_owner);
 
-                        ec_lk(fop->frame, fop->xl, mask, 1,
-                              ec_lock_lk_unlocked, NULL, fop->fd, F_SETLK,
-                              &flock, fop->xdata);
+                        ec_lk(fop->frame, fop->xl, mask, 1, ec_lock_lk_unlocked,
+                              NULL, fop->fd, F_SETLK, &flock, fop->xdata);
                     }
+
                     if (fop->error < 0) {
                         fop->error = 0;
 
@@ -1067,8 +1033,7 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.lk != NULL)
-            {
+            if (fop->cbks.lk != NULL) {
                 fop->cbks.lk(fop->req_frame, fop, fop->xl, cbk->op_ret,
                              cbk->op_errno, &cbk->flock, cbk->xdata);
             }
@@ -1077,43 +1042,39 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
-        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.lk != NULL)
-            {
-                fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error,
-                             NULL, NULL);
+            if (fop->cbks.lk != NULL) {
+                fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL,
+                             NULL);
             }
 
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-           int32_t minimum, fop_lk_cbk_t func, void * data, fd_t * fd,
-           int32_t cmd, struct gf_flock * flock, dict_t * xdata)
+void
+ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags,
+      fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .lk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.lk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(LK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, 0, target, fop_flags,
                                ec_wind_lk, ec_manager_lk, callback, data);
     if (fop == NULL) {
         goto out;
@@ -1126,10 +1087,9 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1149,10 +1109,9 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-mem-types.h b/xlators/cluster/ec/src/ec-mem-types.h
index df65a031590..3252c4c1c58 100644
--- a/xlators/cluster/ec/src/ec-mem-types.h
+++ b/xlators/cluster/ec/src/ec-mem-types.h
@@ -11,16 +11,19 @@
 #ifndef __EC_MEM_TYPES_H__
 #define __EC_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
-enum gf_ec_mem_types_
-{
+enum gf_ec_mem_types_ {
     ec_mt_ec_t = gf_common_mt_end + 1,
     ec_mt_xlator_t,
     ec_mt_ec_inode_t,
     ec_mt_ec_fd_t,
-    ec_mt_ec_heal_t,
     ec_mt_subvol_healer_t,
+    ec_mt_ec_gf_t,
+    ec_mt_ec_code_t,
+    ec_mt_ec_code_builder_t,
+    ec_mt_ec_matrix_t,
+    ec_mt_ec_stripe_t,
     ec_mt_end
 };
 
diff --git a/xlators/cluster/ec/src/ec-messages.h b/xlators/cluster/ec/src/ec-messages.h
index 76678f8f836..72e98f11286 100644
--- a/xlators/cluster/ec/src/ec-messages.h
+++ b/xlators/cluster/ec/src/ec-messages.h
@@ -11,516 +11,51 @@
 #ifndef _EC_MESSAGES_H_
 #define _EC_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs-message-id.h"
-
-/*! \file ec-messages.h
- *  \brief Glusterd log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_EC_COMP_BASE       GLFS_MSGID_COMP_EC
-#define GLFS_NUM_MESSAGES       66
-#define GLFS_MSGID_END          (GLFS_EC_COMP_BASE + GLFS_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x GLFS_EC_COMP_BASE, "Invalid: Start of messages"
-/*------------*/
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_CONFIG           (GLFS_EC_COMP_BASE + 1)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_HEAL_FAIL                (GLFS_EC_COMP_BASE + 2)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_DICT_COMBINE_FAIL        (GLFS_EC_COMP_BASE + 3)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_STIME_COMBINE_FAIL       (GLFS_EC_COMP_BASE + 4)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_DICT_NUMS        (GLFS_EC_COMP_BASE + 5)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_IATT_COMBINE_FAIL        (GLFS_EC_COMP_BASE + 6)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_FORMAT           (GLFS_EC_COMP_BASE + 7)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_DICT_GET_FAILED          (GLFS_EC_COMP_BASE + 8)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_UNHANDLED_STATE          (GLFS_EC_COMP_BASE + 9)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FILE_DESC_REF_FAIL       (GLFS_EC_COMP_BASE + 10)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_LOC_COPY_FAIL            (GLFS_EC_COMP_BASE + 11)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_BUF_REF_FAIL             (GLFS_EC_COMP_BASE + 12)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_DICT_REF_FAIL            (GLFS_EC_COMP_BASE + 13)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_LK_UNLOCK_FAILED         (GLFS_EC_COMP_BASE + 14)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_UNLOCK_FAILED            (GLFS_EC_COMP_BASE + 15)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_LOC_PARENT_INODE_MISSING         (GLFS_EC_COMP_BASE + 16)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_LOC_NAME         (GLFS_EC_COMP_BASE + 17)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_NO_MEMORY                (GLFS_EC_COMP_BASE + 18)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_GFID_MISMATCH            (GLFS_EC_COMP_BASE + 19)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_UNSUPPORTED_VERSION      (GLFS_EC_COMP_BASE + 20)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FD_CREATE_FAIL           (GLFS_EC_COMP_BASE + 21)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_READDIRP_REQ_PREP_FAIL   (GLFS_EC_COMP_BASE + 22)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_LOOKUP_REQ_PREP_FAIL     (GLFS_EC_COMP_BASE + 23)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INODE_REF_FAIL           (GLFS_EC_COMP_BASE + 24)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_LOOKUP_READAHEAD_FAIL    (GLFS_EC_COMP_BASE + 25)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FRAME_MISMATCH           (GLFS_EC_COMP_BASE + 26)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_XLATOR_MISMATCH          (GLFS_EC_COMP_BASE + 27)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_VECTOR_MISMATCH          (GLFS_EC_COMP_BASE + 28)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_IATT_MISMATCH            (GLFS_EC_COMP_BASE + 29)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FD_MISMATCH              (GLFS_EC_COMP_BASE + 30)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_DICT_MISMATCH            (GLFS_EC_COMP_BASE + 31)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INDEX_DIR_GET_FAIL       (GLFS_EC_COMP_BASE + 32)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_PREOP_LOCK_FAILED        (GLFS_EC_COMP_BASE + 33)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_CHILDS_INSUFFICIENT      (GLFS_EC_COMP_BASE + 34)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_OP_EXEC_UNAVAIL          (GLFS_EC_COMP_BASE + 35)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_UNLOCK_DELAY_FAILED      (GLFS_EC_COMP_BASE + 36)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_SIZE_VERS_UPDATE_FAIL    (GLFS_EC_COMP_BASE + 37)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_REQUEST          (GLFS_EC_COMP_BASE + 38)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_LOCK_TYPE        (GLFS_EC_COMP_BASE + 39)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_SIZE_VERS_GET_FAIL       (GLFS_EC_COMP_BASE + 40)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FILE_SIZE_GET_FAIL       (GLFS_EC_COMP_BASE + 41)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FOP_MISMATCH             (GLFS_EC_COMP_BASE + 42)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_SUBVOL_ID_DICT_SET_FAIL          (GLFS_EC_COMP_BASE + 43)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_SUBVOL_BUILD_FAIL        (GLFS_EC_COMP_BASE + 44)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_XLATOR_INIT_FAIL         (GLFS_EC_COMP_BASE + 45)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_NO_PARENTS               (GLFS_EC_COMP_BASE + 46)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_TIMER_CREATE_FAIL        (GLFS_EC_COMP_BASE + 47)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_TOO_MANY_SUBVOLS         (GLFS_EC_COMP_BASE + 48)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_DATA_UNAVAILABLE         (GLFS_EC_COMP_BASE + 49)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INODE_REMOVE_FAIL        (GLFS_EC_COMP_BASE + 50)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_REDUNDANCY        (GLFS_EC_COMP_BASE + 51)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_XLATOR_PARSE_OPT_FAIL        (GLFS_EC_COMP_BASE + 52)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_OP_FAIL_ON_SUBVOLS         (GLFS_EC_COMP_BASE + 53)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_INODE             (GLFS_EC_COMP_BASE + 54)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_LOCK_MISMATCH             (GLFS_EC_COMP_BASE + 55)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_XDATA_MISMATCH             (GLFS_EC_COMP_BASE + 56)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_HEALING_INFO               (GLFS_EC_COMP_BASE + 57)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_HEAL_SUCCESS               (GLFS_EC_COMP_BASE + 58)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FULL_SWEEP_START           (GLFS_EC_COMP_BASE + 59)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_FULL_SWEEP_STOP            (GLFS_EC_COMP_BASE + 59)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_INVALID_FOP                 (GLFS_EC_COMP_BASE + 60)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_EC_UP                       (GLFS_EC_COMP_BASE + 61)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_EC_DOWN                     (GLFS_EC_COMP_BASE + 62)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_SIZE_XATTR_GET_FAIL         (GLFS_EC_COMP_BASE + 63)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_VER_XATTR_GET_FAIL           (GLFS_EC_COMP_BASE + 64)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_CONFIG_XATTR_GET_FAIL        (GLFS_EC_COMP_BASE + 65)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define EC_MSG_CONFIG_XATTR_INVALID         (GLFS_EC_COMP_BASE + 66)
-
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(EC, EC_MSG_INVALID_CONFIG, EC_MSG_HEAL_FAIL,
+           EC_MSG_DICT_COMBINE_FAIL, EC_MSG_STIME_COMBINE_FAIL,
+           EC_MSG_INVALID_DICT_NUMS, EC_MSG_IATT_COMBINE_FAIL,
+           EC_MSG_INVALID_FORMAT, EC_MSG_DICT_GET_FAILED,
+           EC_MSG_UNHANDLED_STATE, EC_MSG_FILE_DESC_REF_FAIL,
+           EC_MSG_LOC_COPY_FAIL, EC_MSG_BUF_REF_FAIL, EC_MSG_DICT_REF_FAIL,
+           EC_MSG_LK_UNLOCK_FAILED, EC_MSG_UNLOCK_FAILED,
+           EC_MSG_LOC_PARENT_INODE_MISSING, EC_MSG_INVALID_LOC_NAME,
+           EC_MSG_NO_MEMORY, EC_MSG_GFID_MISMATCH, EC_MSG_UNSUPPORTED_VERSION,
+           EC_MSG_FD_CREATE_FAIL, EC_MSG_READDIRP_REQ_PREP_FAIL,
+           EC_MSG_LOOKUP_REQ_PREP_FAIL, EC_MSG_INODE_REF_FAIL,
+           EC_MSG_LOOKUP_READAHEAD_FAIL, EC_MSG_FRAME_MISMATCH,
+           EC_MSG_XLATOR_MISMATCH, EC_MSG_VECTOR_MISMATCH, EC_MSG_IATT_MISMATCH,
+           EC_MSG_FD_MISMATCH, EC_MSG_DICT_MISMATCH, EC_MSG_INDEX_DIR_GET_FAIL,
+           EC_MSG_PREOP_LOCK_FAILED, EC_MSG_CHILDS_INSUFFICIENT,
+           EC_MSG_OP_EXEC_UNAVAIL, EC_MSG_UNLOCK_DELAY_FAILED,
+           EC_MSG_SIZE_VERS_UPDATE_FAIL, EC_MSG_INVALID_REQUEST,
+           EC_MSG_INVALID_LOCK_TYPE, EC_MSG_SIZE_VERS_GET_FAIL,
+           EC_MSG_FILE_SIZE_GET_FAIL, EC_MSG_FOP_MISMATCH,
+           EC_MSG_SUBVOL_ID_DICT_SET_FAIL, EC_MSG_SUBVOL_BUILD_FAIL,
+           EC_MSG_XLATOR_INIT_FAIL, EC_MSG_NO_PARENTS, EC_MSG_TIMER_CREATE_FAIL,
+           EC_MSG_TOO_MANY_SUBVOLS, EC_MSG_DATA_UNAVAILABLE,
+           EC_MSG_INODE_REMOVE_FAIL, EC_MSG_INVALID_REDUNDANCY,
+           EC_MSG_XLATOR_PARSE_OPT_FAIL, EC_MSG_OP_FAIL_ON_SUBVOLS,
+           EC_MSG_INVALID_INODE, EC_MSG_LOCK_MISMATCH, EC_MSG_XDATA_MISMATCH,
+           EC_MSG_HEALING_INFO, EC_MSG_HEAL_SUCCESS, EC_MSG_FULL_SWEEP_START,
+           EC_MSG_FULL_SWEEP_STOP, EC_MSG_INVALID_FOP, EC_MSG_EC_UP,
+           EC_MSG_EC_DOWN, EC_MSG_SIZE_XATTR_GET_FAIL,
+           EC_MSG_VER_XATTR_GET_FAIL, EC_MSG_CONFIG_XATTR_GET_FAIL,
+           EC_MSG_CONFIG_XATTR_INVALID, EC_MSG_EXTENSION, EC_MSG_EXTENSION_NONE,
+           EC_MSG_EXTENSION_UNKNOWN, EC_MSG_EXTENSION_UNSUPPORTED,
+           EC_MSG_EXTENSION_FAILED, EC_MSG_NO_GF, EC_MSG_MATRIX_FAILED,
+           EC_MSG_DYN_CREATE_FAILED, EC_MSG_DYN_CODEGEN_FAILED,
+           EC_MSG_THREAD_CLEANUP_FAILED, EC_MSG_FD_BAD);
 
 #endif /* !_EC_MESSAGES_H_ */
diff --git a/xlators/cluster/ec/src/ec-method.c b/xlators/cluster/ec/src/ec-method.c
index faab0115cdd..55faed0b193 100644
--- a/xlators/cluster/ec/src/ec-method.c
+++ b/xlators/cluster/ec/src/ec-method.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,149 +11,423 @@
 #include <string.h>
 #include <inttypes.h>
 
-#include "ec-gf.h"
+#include "ec-types.h"
+#include "ec-mem-types.h"
+#include "ec-galois.h"
+#include "ec-code.h"
 #include "ec-method.h"
+#include "ec-helpers.h"
 
-static uint32_t GfPow[EC_GF_SIZE << 1];
-static uint32_t GfLog[EC_GF_SIZE << 1];
+static void
+ec_method_matrix_normal(ec_gf_t *gf, uint32_t *matrix, uint32_t columns,
+                        uint32_t *values, uint32_t count)
+{
+    uint32_t i, j, v, tmp;
+
+    columns--;
+    for (i = 0; i < count; i++) {
+        v = *values++;
+        *matrix++ = tmp = ec_gf_exp(gf, v, columns);
+        for (j = 0; j < columns; j++) {
+            *matrix++ = tmp = ec_gf_div(gf, tmp, v);
+        }
+    }
+}
+
+static void
+ec_method_matrix_inverse(ec_gf_t *gf, uint32_t *matrix, uint32_t *values,
+                         uint32_t count)
+{
+    uint32_t a[count];
+    uint32_t i, j, p, last, tmp;
+
+    last = count - 1;
+    for (i = 0; i < last; i++) {
+        a[i] = 1;
+    }
+    a[i] = values[0];
+    for (i = last; i > 0; i--) {
+        for (j = i - 1; j < last; j++) {
+            a[j] = a[j + 1] ^ ec_gf_mul(gf, values[i], a[j]);
+        }
+        a[j] = ec_gf_mul(gf, values[i], a[j]);
+    }
+    for (i = 0; i < count; i++) {
+        p = a[0];
+        matrix += count;
+        *matrix = tmp = p ^ values[i];
+        for (j = 1; j < last; j++) {
+            matrix += count;
+            *matrix = tmp = a[j] ^ ec_gf_mul(gf, values[i], tmp);
+            p = tmp ^ ec_gf_mul(gf, values[i], p);
+        }
+        for (j = 0; j < last; j++) {
+            *matrix = ec_gf_div(gf, *matrix, p);
+            matrix -= count;
+        }
+        *matrix = ec_gf_div(gf, 1, p);
+        matrix++;
+    }
+}
 
-void ec_method_initialize(void)
+static void
+ec_method_matrix_init(ec_matrix_list_t *list, ec_matrix_t *matrix,
+                      uintptr_t mask, uint32_t *rows, gf_boolean_t inverse)
 {
     uint32_t i;
 
-    GfPow[0] = 1;
-    GfLog[0] = EC_GF_SIZE;
-    for (i = 1; i < EC_GF_SIZE; i++)
-    {
-        GfPow[i] = GfPow[i - 1] << 1;
-        if (GfPow[i] >= EC_GF_SIZE)
-        {
-            GfPow[i] ^= EC_GF_MOD;
+    matrix->refs = 1;
+    matrix->mask = mask;
+    matrix->code = list->code;
+    matrix->columns = list->columns;
+    INIT_LIST_HEAD(&matrix->lru);
+
+    if (inverse) {
+        matrix->rows = list->columns;
+        ec_method_matrix_inverse(matrix->code->gf, matrix->values, rows,
+                                 matrix->rows);
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].values = matrix->values + i * matrix->columns;
+            matrix->row_data[i].func.interleaved = ec_code_build_interleaved(
+                matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values,
+                matrix->columns);
+        }
+    } else {
+        matrix->rows = list->rows;
+        ec_method_matrix_normal(matrix->code->gf, matrix->values,
+                                matrix->columns, rows, matrix->rows);
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].values = matrix->values + i * matrix->columns;
+            matrix->row_data[i].func.linear = ec_code_build_linear(
+                matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values,
+                matrix->columns);
         }
-        GfPow[i + EC_GF_SIZE - 1] = GfPow[i];
-        GfLog[GfPow[i] + EC_GF_SIZE - 1] = GfLog[GfPow[i]] = i;
     }
 }
 
-static uint32_t ec_method_mul(uint32_t a, uint32_t b)
+static void
+ec_method_matrix_release(ec_matrix_t *matrix)
 {
-    if (a && b)
-    {
-        return GfPow[GfLog[a] + GfLog[b]];
+    uint32_t i;
+
+    for (i = 0; i < matrix->rows; i++) {
+        if (matrix->row_data[i].func.linear != NULL) {
+            ec_code_release(matrix->code, &matrix->row_data[i].func);
+            matrix->row_data[i].func.linear = NULL;
+        }
     }
-    return 0;
 }
 
-static uint32_t ec_method_div(uint32_t a, uint32_t b)
+static void
+ec_method_matrix_destroy(ec_matrix_list_t *list, ec_matrix_t *matrix)
+{
+    list_del_init(&matrix->lru);
+
+    ec_method_matrix_release(matrix);
+
+    mem_put(matrix);
+
+    list->count--;
+}
+
+static void
+ec_method_matrix_unref(ec_matrix_list_t *list, ec_matrix_t *matrix)
 {
-    if (b)
-    {
-        if (a)
-        {
-            return GfPow[EC_GF_SIZE - 1 + GfLog[a] - GfLog[b]];
+    if (--matrix->refs == 0) {
+        list_add_tail(&matrix->lru, &list->lru);
+        if (list->count > list->max) {
+            matrix = list_first_entry(&list->lru, ec_matrix_t, lru);
+            ec_method_matrix_destroy(list, matrix);
         }
-        return 0;
     }
-    return EC_GF_SIZE;
 }
 
-size_t ec_method_encode(size_t size, uint32_t columns, uint32_t row,
-                        uint8_t * in, uint8_t * out)
+static ec_matrix_t *
+ec_method_matrix_lookup(ec_matrix_list_t *list, uintptr_t mask, uint32_t *pos)
 {
-    uint32_t i, j;
+    ec_matrix_t *matrix;
+    uint32_t i, j, k;
 
-    size /= EC_METHOD_CHUNK_SIZE * columns;
-    row++;
-    for (j = 0; j < size; j++)
-    {
-        ec_gf_muladd[0](out, in, EC_METHOD_WIDTH);
-        in += EC_METHOD_CHUNK_SIZE;
-        for (i = 1; i < columns; i++)
-        {
-            ec_gf_muladd[row](out, in, EC_METHOD_WIDTH);
-            in += EC_METHOD_CHUNK_SIZE;
+    i = 0;
+    j = list->count;
+    while (i < j) {
+        k = (i + j) >> 1;
+        matrix = list->objects[k];
+        if (matrix->mask == mask) {
+            *pos = k;
+            return matrix;
+        }
+        if (matrix->mask < mask) {
+            i = k + 1;
+        } else {
+            j = k;
         }
-        out += EC_METHOD_CHUNK_SIZE;
     }
+    *pos = i;
 
-    return size * EC_METHOD_CHUNK_SIZE;
+    return NULL;
 }
 
-size_t ec_method_decode(size_t size, uint32_t columns, uint32_t * rows,
-                        uint8_t ** in, uint8_t * out)
+static void
+ec_method_matrix_remove(ec_matrix_list_t *list, uintptr_t mask)
 {
-    uint32_t i, j, k, off, last, value;
-    uint32_t f;
-    uint8_t inv[EC_METHOD_MAX_FRAGMENTS][EC_METHOD_MAX_FRAGMENTS + 1];
-    uint8_t mtx[EC_METHOD_MAX_FRAGMENTS][EC_METHOD_MAX_FRAGMENTS];
-    uint8_t dummy[EC_METHOD_CHUNK_SIZE];
+    uint32_t pos;
 
-    size /= EC_METHOD_CHUNK_SIZE;
+    if (ec_method_matrix_lookup(list, mask, &pos) != NULL) {
+        list->count--;
+        if (pos < list->count) {
+            memmove(list->objects + pos, list->objects + pos + 1,
+                    sizeof(ec_matrix_t *) * (list->count - pos));
+        }
+    }
+}
+
+static void
+ec_method_matrix_insert(ec_matrix_list_t *list, ec_matrix_t *matrix)
+{
+    uint32_t pos;
 
-    memset(inv, 0, sizeof(inv));
-    memset(mtx, 0, sizeof(mtx));
-    memset(dummy, 0, sizeof(dummy));
-    for (i = 0; i < columns; i++)
-    {
-        inv[i][i] = 1;
-        inv[i][columns] = 1;
+    GF_ASSERT(ec_method_matrix_lookup(list, matrix->mask, &pos) == NULL);
+
+    if (pos < list->count) {
+        memmove(list->objects + pos + 1, list->objects + pos,
+                sizeof(ec_matrix_t *) * (list->count - pos));
     }
-    for (i = 0; i < columns; i++)
-    {
-        mtx[i][columns - 1] = 1;
-        for (j = columns - 1; j > 0; j--)
-        {
-            mtx[i][j - 1] = ec_method_mul(mtx[i][j], rows[i] + 1);
-        }
+    list->objects[pos] = matrix;
+    list->count++;
+}
+
+static ec_matrix_t *
+ec_method_matrix_get(ec_matrix_list_t *list, uintptr_t mask, uint32_t *rows)
+{
+    ec_matrix_t *matrix;
+    uint32_t pos;
+
+    LOCK(&list->lock);
+
+    matrix = ec_method_matrix_lookup(list, mask, &pos);
+    if (matrix != NULL) {
+        list_del_init(&matrix->lru);
+        matrix->refs++;
+
+        goto out;
     }
 
-    for (i = 0; i < columns; i++)
-    {
-        f = mtx[i][i];
-        for (j = 0; j < columns; j++)
-        {
-            mtx[i][j] = ec_method_div(mtx[i][j], f);
-            inv[i][j] = ec_method_div(inv[i][j], f);
+    if ((list->count >= list->max) && !list_empty(&list->lru)) {
+        matrix = list_first_entry(&list->lru, ec_matrix_t, lru);
+        list_del_init(&matrix->lru);
+
+        ec_method_matrix_remove(list, matrix->mask);
+
+        ec_method_matrix_release(matrix);
+    } else {
+        matrix = mem_get0(list->pool);
+        if (matrix == NULL) {
+            matrix = EC_ERR(ENOMEM);
+            goto out;
         }
-        for (j = 0; j < columns; j++)
-        {
-            if (i != j)
-            {
-                f = mtx[j][i];
-                for (k = 0; k < columns; k++)
-                {
-                    mtx[j][k] ^= ec_method_mul(mtx[i][k], f);
-                    inv[j][k] ^= ec_method_mul(inv[i][k], f);
-                }
-            }
+        matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) +
+                                      sizeof(ec_matrix_row_t) * list->columns);
+    }
+
+    ec_method_matrix_init(list, matrix, mask, rows, _gf_true);
+
+    if (list->count < list->max) {
+        ec_method_matrix_insert(list, matrix);
+    } else {
+        matrix->mask = 0;
+    }
+
+out:
+    UNLOCK(&list->lock);
+
+    return matrix;
+}
+
+static void
+ec_method_matrix_put(ec_matrix_list_t *list, ec_matrix_t *matrix)
+{
+    LOCK(&list->lock);
+
+    ec_method_matrix_unref(list, matrix);
+
+    UNLOCK(&list->lock);
+}
+
+static int32_t
+ec_method_setup(xlator_t *xl, ec_matrix_list_t *list, const char *gen)
+{
+    ec_matrix_t *matrix;
+    uint32_t values[list->rows];
+    uint32_t i;
+    int32_t err;
+
+    matrix = GF_MALLOC(sizeof(ec_matrix_t) +
+                           sizeof(ec_matrix_row_t) * list->rows +
+                           sizeof(uint32_t) * list->columns * list->rows,
+                       ec_mt_ec_matrix_t);
+    if (matrix == NULL) {
+        err = -ENOMEM;
+        goto failed;
+    }
+    memset(matrix, 0, sizeof(ec_matrix_t));
+    matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) +
+                                  sizeof(ec_matrix_row_t) * list->rows);
+
+    list->code = ec_code_create(list->gf, ec_code_detect(xl, gen));
+    if (EC_IS_ERR(list->code)) {
+        err = EC_GET_ERR(list->code);
+        list->code = NULL;
+        goto failed_matrix;
+    }
+
+    for (i = 0; i < list->rows; i++) {
+        values[i] = i + 1;
+    }
+    ec_method_matrix_init(list, matrix, 0, values, _gf_false);
+
+    list->encode = matrix;
+
+    return 0;
+
+failed_matrix:
+    GF_FREE(matrix);
+failed:
+    return err;
+}
+
+int32_t
+ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,
+               uint32_t rows, uint32_t max, const char *gen)
+{
+    list->columns = columns;
+    list->rows = rows;
+    list->max = max;
+    list->stripe = EC_METHOD_CHUNK_SIZE * list->columns;
+    INIT_LIST_HEAD(&list->lru);
+    int32_t err;
+
+    list->pool = mem_pool_new_fn(xl->ctx,
+                                 sizeof(ec_matrix_t) +
+                                     sizeof(ec_matrix_row_t) * columns +
+                                     sizeof(uint32_t) * columns * columns,
+                                 128, "ec_matrix_t");
+    if (list->pool == NULL) {
+        err = -ENOMEM;
+        goto failed;
+    }
+
+    list->objects = GF_MALLOC(sizeof(ec_matrix_t *) * max, ec_mt_ec_matrix_t);
+    if (list->objects == NULL) {
+        err = -ENOMEM;
+        goto failed_pool;
+    }
+
+    list->gf = ec_gf_prepare(EC_GF_BITS, EC_GF_MOD);
+    if (EC_IS_ERR(list->gf)) {
+        err = EC_GET_ERR(list->gf);
+        goto failed_objects;
+    }
+
+    err = ec_method_setup(xl, list, gen);
+    if (err != 0) {
+        goto failed_gf;
+    }
+
+    LOCK_INIT(&list->lock);
+
+    return 0;
+
+failed_gf:
+    ec_gf_destroy(list->gf);
+failed_objects:
+    GF_FREE(list->objects);
+failed_pool:
+    mem_pool_destroy(list->pool);
+failed:
+    list->pool = NULL;
+    list->objects = NULL;
+    list->gf = NULL;
+
+    return err;
+}
+
+void
+ec_method_fini(ec_matrix_list_t *list)
+{
+    ec_matrix_t *matrix;
+
+    if (list->encode == NULL) {
+        return;
+    }
+
+    while (!list_empty(&list->lru)) {
+        matrix = list_first_entry(&list->lru, ec_matrix_t, lru);
+        ec_method_matrix_destroy(list, matrix);
+    }
+
+    GF_ASSERT(list->count == 0);
+
+    if (list->pool) /*Init was successful*/
+        LOCK_DESTROY(&list->lock);
+
+    ec_method_matrix_release(list->encode);
+    GF_FREE(list->encode);
+
+    ec_code_destroy(list->code);
+    ec_gf_destroy(list->gf);
+    GF_FREE(list->objects);
+
+    if (list->pool)
+        mem_pool_destroy(list->pool);
+}
+
+int32_t
+ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen)
+{
+    /* TODO: Allow changing code generator */
+
+    return 0;
+}
+
+void
+ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out)
+{
+    ec_matrix_t *matrix;
+    uint64_t pos;
+    uint32_t i;
+
+    matrix = list->encode;
+    for (pos = 0; pos < size; pos += list->stripe) {
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].func.linear(
+                out[i], in, pos, matrix->row_data[i].values, list->columns);
+            out[i] += EC_METHOD_CHUNK_SIZE;
         }
     }
-    off = 0;
-    for (f = 0; f < size; f++)
-    {
-        for (i = 0; i < columns; i++)
-        {
-            last = 0;
-            j = 0;
-            do
-            {
-                while (inv[i][j] == 0)
-                {
-                    j++;
-                }
-                if (j < columns)
-                {
-                    value = ec_method_div(last, inv[i][j]);
-                    last = inv[i][j];
-                    ec_gf_muladd[value](out, in[j] + off, EC_METHOD_WIDTH);
-                    j++;
-                }
-            } while (j < columns);
-            ec_gf_muladd[last](out, dummy, EC_METHOD_WIDTH);
+}
+
+int32_t
+ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask,
+                 uint32_t *rows, void **in, void *out)
+{
+    ec_matrix_t *matrix;
+    uint64_t pos;
+    uint32_t i;
+
+    matrix = ec_method_matrix_get(list, mask, rows);
+    if (EC_IS_ERR(matrix)) {
+        return EC_GET_ERR(matrix);
+    }
+    for (pos = 0; pos < size; pos += EC_METHOD_CHUNK_SIZE) {
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].func.interleaved(
+                out, in, pos, matrix->row_data[i].values, list->columns);
             out += EC_METHOD_CHUNK_SIZE;
         }
-        off += EC_METHOD_CHUNK_SIZE;
     }
 
-    return size * EC_METHOD_CHUNK_SIZE * columns;
+    ec_method_matrix_put(list, matrix);
+
+    return 0;
 }
diff --git a/xlators/cluster/ec/src/ec-method.h b/xlators/cluster/ec/src/ec-method.h
index 29b46e10443..f91233b2f88 100644
--- a/xlators/cluster/ec/src/ec-method.h
+++ b/xlators/cluster/ec/src/ec-method.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,22 +11,38 @@
 #ifndef __EC_METHOD_H__
 #define __EC_METHOD_H__
 
-#include "ec-gf.h"
+#include "ec-types.h"
+#include "ec-galois.h"
+
+#define EC_GF_BITS 8
+#define EC_GF_MOD 0x11D
+
+#define EC_GF_SIZE (1 << EC_GF_BITS)
 
 /* Determines the maximum size of the matrix used to encode/decode data */
 #define EC_METHOD_MAX_FRAGMENTS 16
 /* Determines the maximum number of usable elements in the Galois Field */
-#define EC_METHOD_MAX_NODES     (EC_GF_SIZE - 1)
+#define EC_METHOD_MAX_NODES (EC_GF_SIZE - 1)
 
 #define EC_METHOD_WORD_SIZE 64
 
 #define EC_METHOD_CHUNK_SIZE (EC_METHOD_WORD_SIZE * EC_GF_BITS)
-#define EC_METHOD_WIDTH (EC_METHOD_WORD_SIZE / EC_GF_WORD_SIZE)
 
-void ec_method_initialize(void);
-size_t ec_method_encode(size_t size, uint32_t columns, uint32_t row,
-                        uint8_t * in, uint8_t * out);
-size_t ec_method_decode(size_t size, uint32_t columns, uint32_t * rows,
-                        uint8_t ** in, uint8_t * out);
+int32_t
+ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,
+               uint32_t rows, uint32_t max, const char *gen);
+
+void
+ec_method_fini(ec_matrix_list_t *list);
+
+int32_t
+ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen);
+
+void
+ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out);
+
+int32_t
+ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask,
+                 uint32_t *rows, void **in, void *out);
 
 #endif /* __EC_METHOD_H__ */
diff --git a/xlators/cluster/ec/src/ec-types.h b/xlators/cluster/ec/src/ec-types.h
new file mode 100644
index 00000000000..de9b89bb2c9
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-types.h
@@ -0,0 +1,690 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_TYPES_H__
+#define __EC_TYPES_H__
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
+#include "libxlator.h"
+#include <glusterfs/atomic.h>
+
+#define EC_GF_MAX_REGS 16
+
+enum _ec_heal_need;
+typedef enum _ec_heal_need ec_heal_need_t;
+
+enum _ec_stripe_part;
+typedef enum _ec_stripe_part ec_stripe_part_t;
+
+enum _ec_read_policy;
+typedef enum _ec_read_policy ec_read_policy_t;
+
+struct _ec_config;
+typedef struct _ec_config ec_config_t;
+
+struct _ec_fd;
+typedef struct _ec_fd ec_fd_t;
+
+struct _ec_fragment_range;
+typedef struct _ec_fragment_range ec_fragment_range_t;
+
+struct _ec_inode;
+typedef struct _ec_inode ec_inode_t;
+
+union _ec_cbk;
+typedef union _ec_cbk ec_cbk_t;
+
+struct _ec_lock;
+typedef struct _ec_lock ec_lock_t;
+
+struct _ec_lock_link;
+typedef struct _ec_lock_link ec_lock_link_t;
+
+struct _ec_fop_data;
+typedef struct _ec_fop_data ec_fop_data_t;
+
+struct _ec_cbk_data;
+typedef struct _ec_cbk_data ec_cbk_data_t;
+
+enum _ec_gf_opcode;
+typedef enum _ec_gf_opcode ec_gf_opcode_t;
+
+struct _ec_gf_op;
+typedef struct _ec_gf_op ec_gf_op_t;
+
+struct _ec_gf_mul;
+typedef struct _ec_gf_mul ec_gf_mul_t;
+
+struct _ec_gf;
+typedef struct _ec_gf ec_gf_t;
+
+struct _ec_code_gen;
+typedef struct _ec_code_gen ec_code_gen_t;
+
+struct _ec_code;
+typedef struct _ec_code ec_code_t;
+
+struct _ec_code_arg;
+typedef struct _ec_code_arg ec_code_arg_t;
+
+struct _ec_code_op;
+typedef struct _ec_code_op ec_code_op_t;
+
+struct _ec_code_builder;
+typedef struct _ec_code_builder ec_code_builder_t;
+
+struct _ec_code_chunk;
+typedef struct _ec_code_chunk ec_code_chunk_t;
+
+struct _ec_stripe;
+typedef struct _ec_stripe ec_stripe_t;
+
+struct _ec_stripe_list;
+typedef struct _ec_stripe_list ec_stripe_list_t;
+
+struct _ec_code_space;
+typedef struct _ec_code_space ec_code_space_t;
+
+typedef void (*ec_code_func_linear_t)(void *dst, void *src, uint64_t offset,
+                                      uint32_t *values, uint32_t count);
+
+typedef void (*ec_code_func_interleaved_t)(void *dst, void **src,
+                                           uint64_t offset, uint32_t *values,
+                                           uint32_t count);
+
+union _ec_code_func;
+typedef union _ec_code_func ec_code_func_t;
+
+struct _ec_matrix_row;
+typedef struct _ec_matrix_row ec_matrix_row_t;
+
+struct _ec_matrix;
+typedef struct _ec_matrix ec_matrix_t;
+
+struct _ec_matrix_list;
+typedef struct _ec_matrix_list ec_matrix_list_t;
+
+struct _ec_heal;
+typedef struct _ec_heal ec_heal_t;
+
+struct _ec_self_heald;
+typedef struct _ec_self_heald ec_self_heald_t;
+
+struct _ec_statistics;
+typedef struct _ec_statistics ec_statistics_t;
+
+struct _ec;
+typedef struct _ec ec_t;
+
+typedef void (*ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t);
+typedef int32_t (*ec_handler_f)(ec_fop_data_t *, int32_t);
+typedef void (*ec_resume_f)(ec_fop_data_t *, int32_t);
+
+enum _ec_read_policy { EC_ROUND_ROBIN, EC_GFID_HASH, EC_READ_POLICY_MAX };
+
+enum _ec_heal_need {
+    EC_HEAL_NONEED,
+    EC_HEAL_MAYBE,
+    EC_HEAL_MUST,
+    EC_HEAL_PURGE_INDEX
+};
+
+enum _ec_stripe_part { EC_STRIPE_HEAD, EC_STRIPE_TAIL };
+
+/* Enumartions to indicate FD status. */
+typedef enum { EC_FD_NOT_OPENED, EC_FD_OPENED, EC_FD_OPENING } ec_fd_status_t;
+
+struct _ec_config {
+    uint32_t version;
+    uint8_t algorithm;
+    uint8_t gf_word_size;
+    uint8_t bricks;
+    uint8_t redundancy;
+    uint32_t chunk_size;
+};
+
+struct _ec_fd {
+    loc_t loc;
+    uintptr_t open;
+    int32_t flags;
+    uint64_t bad_version;
+    ec_fd_status_t fd_status[0];
+};
+
+struct _ec_stripe {
+    struct list_head lru; /* LRU list member */
+    uint64_t frag_offset; /* Fragment offset of this stripe */
+    char data[];          /* Contents of the stripe */
+};
+
+struct _ec_stripe_list {
+    struct list_head lru;
+    uint32_t count;
+    uint32_t max;
+};
+
+struct _ec_inode {
+    ec_lock_t *inode_lock;
+    gf_boolean_t have_info;
+    gf_boolean_t have_config;
+    gf_boolean_t have_version;
+    gf_boolean_t have_size;
+    int32_t heal_count;
+    ec_config_t config;
+    uint64_t pre_version[2];
+    uint64_t post_version[2];
+    uint64_t pre_size;
+    uint64_t post_size;
+    uint64_t dirty[2];
+    struct list_head heal;
+    ec_stripe_list_t stripe_cache;
+    uint64_t bad_version;
+};
+
+typedef int32_t (*fop_heal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t,
+                                  int32_t, uintptr_t, uintptr_t, uintptr_t,
+                                  uint32_t, dict_t *);
+typedef int32_t (*fop_fheal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t,
+                                   int32_t, uintptr_t, uintptr_t, uintptr_t,
+                                   uint32_t, dict_t *);
+
+union _ec_cbk {
+    fop_access_cbk_t access;
+    fop_create_cbk_t create;
+    fop_discard_cbk_t discard;
+    fop_entrylk_cbk_t entrylk;
+    fop_fentrylk_cbk_t fentrylk;
+    fop_fallocate_cbk_t fallocate;
+    fop_flush_cbk_t flush;
+    fop_fsync_cbk_t fsync;
+    fop_fsyncdir_cbk_t fsyncdir;
+    fop_getxattr_cbk_t getxattr;
+    fop_fgetxattr_cbk_t fgetxattr;
+    fop_heal_cbk_t heal;
+    fop_fheal_cbk_t fheal;
+    fop_inodelk_cbk_t inodelk;
+    fop_finodelk_cbk_t finodelk;
+    fop_link_cbk_t link;
+    fop_lk_cbk_t lk;
+    fop_lookup_cbk_t lookup;
+    fop_mkdir_cbk_t mkdir;
+    fop_mknod_cbk_t mknod;
+    fop_open_cbk_t open;
+    fop_opendir_cbk_t opendir;
+    fop_readdir_cbk_t readdir;
+    fop_readdirp_cbk_t readdirp;
+    fop_readlink_cbk_t readlink;
+    fop_readv_cbk_t readv;
+    fop_removexattr_cbk_t removexattr;
+    fop_fremovexattr_cbk_t fremovexattr;
+    fop_rename_cbk_t rename;
+    fop_rmdir_cbk_t rmdir;
+    fop_setattr_cbk_t setattr;
+    fop_fsetattr_cbk_t fsetattr;
+    fop_setxattr_cbk_t setxattr;
+    fop_fsetxattr_cbk_t fsetxattr;
+    fop_stat_cbk_t stat;
+    fop_fstat_cbk_t fstat;
+    fop_statfs_cbk_t statfs;
+    fop_symlink_cbk_t symlink;
+    fop_truncate_cbk_t truncate;
+    fop_ftruncate_cbk_t ftruncate;
+    fop_unlink_cbk_t unlink;
+    fop_writev_cbk_t writev;
+    fop_xattrop_cbk_t xattrop;
+    fop_fxattrop_cbk_t fxattrop;
+    fop_zerofill_cbk_t zerofill;
+    fop_seek_cbk_t seek;
+    fop_ipc_cbk_t ipc;
+};
+
+struct _ec_lock {
+    ec_inode_t *ctx;
+    gf_timer_t *timer;
+
+    /* List of owners of this lock. All fops added to this list are running
+     * concurrently. */
+    struct list_head owners;
+
+    /* List of fops waiting to be an owner of the lock. Fops are added to this
+     * list when the current owner has an incompatible access (conflicting lock)
+     * or the lock is not acquired yet. */
+    struct list_head waiting;
+
+    /* List of fops that will wait until the next unlock/lock cycle. This
+     * happens when the currently acquired lock is decided to be released as
+     * soon as possible. In this case, all frozen fops will be continued only
+     * after the lock is reacquired. */
+    struct list_head frozen;
+
+    uintptr_t mask;
+    uintptr_t good_mask;
+    uintptr_t healing;
+    uint32_t refs_owners;   /* Refs for fops owning the lock */
+    uint32_t refs_pending;  /* Refs assigned to fops being prepared */
+    uint32_t waiting_flags; /*Track xattrop/dirty marking*/
+    gf_boolean_t acquired;
+    gf_boolean_t contention;
+    gf_boolean_t unlock_now;
+    gf_boolean_t release;
+    gf_boolean_t query;
+    fd_t *fd;
+    loc_t loc;
+    union {
+        entrylk_type type;
+        struct gf_flock flock;
+    };
+};
+
+struct _ec_lock_link {
+    ec_lock_t *lock;
+    ec_fop_data_t *fop;
+    struct list_head owner_list;
+    struct list_head wait_list;
+    gf_boolean_t update[2];
+    gf_boolean_t dirty[2];
+    gf_boolean_t optimistic_changelog;
+    loc_t *base;
+    uint64_t size;
+    uint32_t waiting_flags;
+    off_t fl_start;
+    off_t fl_end;
+};
+
+/* This structure keeps a range of fragment offsets affected by a fop. Since
+ * real file offsets can be difficult to handle correctly because of overflows,
+ * we use the 'scaled' offset, which corresponds to the offset of the fragment
+ * seen by the bricks, which is always smaller and cannot overflow. */
+struct _ec_fragment_range {
+    uint64_t first; /* Address of the first affected fragment as seen by the
+                       bricks (offset on brick) */
+    uint64_t last;  /* Address of the first non affected fragment as seen by
+                       the bricks (offset on brick) */
+};
+
+/* EC xlator data structure to collect all the data required to perform
+ * the file operation.*/
+struct _ec_fop_data {
+    int32_t id; /* ID of the file operation */
+    int32_t refs;
+    int32_t state;
+    uint32_t minimum; /* Minimum number of successful
+                         operation required to conclude a
+                         fop as successful */
+    int32_t expected;
+    int32_t winds;
+    int32_t jobs;
+    int32_t error;
+    ec_fop_data_t *parent;
+    xlator_t *xl;                  /* points to EC xlator */
+    call_frame_t *req_frame;       /* frame of the calling xlator */
+    call_frame_t *frame;           /* frame used by this fop */
+    struct list_head cbk_list;     /* sorted list of groups of answers */
+    struct list_head answer_list;  /* list of answers */
+    struct list_head pending_list; /* member of ec_t.pending_fops */
+    ec_cbk_data_t *answer;         /* accepted answer */
+    int32_t lock_count;
+    int32_t locked;
+    gf_lock_t lock;
+    ec_lock_link_t locks[2];
+    int32_t first_lock;
+
+    uint32_t fop_flags; /* Flags passed by the caller. */
+    uint32_t flags;     /* Internal flags. */
+    uint32_t first;
+    uintptr_t mask;
+    uintptr_t healing; /*Dispatch is done but call is successful only
+                         if fop->minimum number of subvolumes succeed
+                         which are not healing*/
+    uintptr_t remaining;
+    uintptr_t received; /* Mask of responses */
+    uintptr_t good;
+
+    uid_t uid;
+    gid_t gid;
+
+    ec_wind_f wind;       /* Function to wind to */
+    ec_handler_f handler; /* FOP manager function */
+    ec_resume_f resume;
+    ec_cbk_t cbks; /* Callback function for this FOP */
+    void *data;
+    ec_heal_t *heal;
+    struct list_head healer;
+
+    uint64_t user_size;
+    uint32_t head;
+
+    int32_t use_fd; /* Indicates whether this FOP uses FD or
+                       not */
+
+    dict_t *xdata;
+    dict_t *dict;
+    int32_t int32;
+    uint32_t uint32;
+    uint64_t size;
+    off_t offset;
+    mode_t mode[2];
+    entrylk_cmd entrylk_cmd;
+    entrylk_type entrylk_type;
+    gf_xattrop_flags_t xattrop_flags;
+    dev_t dev;
+    inode_t *inode;
+    fd_t *fd; /* FD of the file on which FOP is
+                 being carried upon */
+    struct iatt iatt;
+    char *str[2];
+    loc_t loc[2]; /* Holds the location details for
+                     the file */
+    struct gf_flock flock;
+    struct iovec *vector;
+    struct iobref *buffers;
+    gf_seek_what_t seek;
+    ec_fragment_range_t frag_range; /* This will hold the range of stripes
+                                        affected by the fop. */
+    char *errstr;                   /*String of fop name, path and gfid
+                                     to be used in gf_msg. */
+};
+
+struct _ec_cbk_data {
+    struct list_head list;        /* item in the sorted list of groups */
+    struct list_head answer_list; /* item in the list of answers */
+    ec_fop_data_t *fop;
+    ec_cbk_data_t *next; /* next answer in the same group */
+    uint32_t idx;
+    int32_t op_ret;
+    int32_t op_errno;
+    int32_t count;
+    uintptr_t mask;
+
+    dict_t *xdata;
+    dict_t *dict;
+    int32_t int32;
+    uintptr_t uintptr[3];
+    uint64_t size;
+    uint64_t version[2];
+    inode_t *inode;
+    fd_t *fd;
+    struct statvfs statvfs;
+    struct iatt iatt[5];
+    struct gf_flock flock;
+    struct iovec *vector;
+    struct iobref *buffers;
+    char *str;
+    gf_dirent_t entries;
+    off_t offset;
+    gf_seek_what_t what;
+};
+
+enum _ec_gf_opcode {
+    EC_GF_OP_LOAD,
+    EC_GF_OP_STORE,
+    EC_GF_OP_COPY,
+    EC_GF_OP_XOR2,
+    EC_GF_OP_XOR3,
+    EC_GF_OP_XORM,
+    EC_GF_OP_END
+};
+
+struct _ec_gf_op {
+    ec_gf_opcode_t op;
+    uint32_t arg1;
+    uint32_t arg2;
+    uint32_t arg3;
+};
+
+struct _ec_gf_mul {
+    uint32_t regs;
+    uint32_t map[EC_GF_MAX_REGS];
+    ec_gf_op_t *ops;
+};
+
+struct _ec_gf {
+    uint32_t bits;
+    uint32_t size;
+    uint32_t mod;
+    uint32_t min_ops;
+    uint32_t max_ops;
+    uint32_t avg_ops;
+    uint32_t *log;
+    uint32_t *pow;
+    ec_gf_mul_t **table;
+};
+
+struct _ec_code_gen {
+    char *name;
+    char **flags;
+    uint32_t width;
+
+    void (*prolog)(ec_code_builder_t *builder);
+    void (*epilog)(ec_code_builder_t *builder);
+    void (*load)(ec_code_builder_t *builder, uint32_t reg, uint32_t offset,
+                 uint32_t bit);
+    void (*store)(ec_code_builder_t *builder, uint32_t reg, uint32_t bit);
+    void (*copy)(ec_code_builder_t *builder, uint32_t dst, uint32_t src);
+    void (*xor2)(ec_code_builder_t *builder, uint32_t dst, uint32_t src);
+    void (*xor3)(ec_code_builder_t *builder, uint32_t dst, uint32_t src1,
+                 uint32_t src2);
+    void (*xorm)(ec_code_builder_t *builder, uint32_t dst, uint32_t offset,
+                 uint32_t bit);
+};
+
+struct _ec_code {
+    gf_lock_t lock;
+    struct list_head spaces;
+    ec_gf_t *gf;
+    ec_code_gen_t *gen;
+};
+
+struct _ec_code_arg {
+    uint32_t value;
+};
+
+struct _ec_code_op {
+    ec_gf_opcode_t op;
+    ec_code_arg_t arg1;
+    ec_code_arg_t arg2;
+    ec_code_arg_t arg3;
+};
+
+struct _ec_code_builder {
+    ec_code_t *code;
+    uint64_t address;
+    uint8_t *data;
+    uint32_t size;
+    int32_t error;
+    uint32_t regs;
+    uint32_t bits;
+    uint32_t width;
+    uint32_t count;
+    uint32_t base;
+    uint32_t map[EC_GF_MAX_REGS];
+    gf_boolean_t linear;
+    uint64_t loop;
+    ec_code_op_t ops[0];
+};
+
+struct _ec_code_chunk {
+    struct list_head list;
+    size_t size;
+    ec_code_space_t *space;
+};
+
+struct _ec_code_space {
+    struct list_head list;
+    struct list_head chunks;
+    ec_code_t *code;
+    void *exec;
+    size_t size;
+};
+
+union _ec_code_func {
+    ec_code_func_linear_t linear;
+    ec_code_func_interleaved_t interleaved;
+};
+
+struct _ec_matrix_row {
+    ec_code_func_t func;
+    uint32_t *values;
+};
+
+struct _ec_matrix {
+    struct list_head lru;
+    uint32_t refs;
+    uint32_t columns;
+    uint32_t rows;
+    uintptr_t mask;
+    ec_code_t *code;
+    uint32_t *values;
+    ec_matrix_row_t row_data[0];
+};
+
+struct _ec_matrix_list {
+    struct list_head lru;
+    gf_lock_t lock;
+    uint32_t columns;
+    uint32_t rows;
+    uint32_t max;
+    uint32_t count;
+    uint32_t stripe;
+    struct mem_pool *pool;
+    ec_gf_t *gf;
+    ec_code_t *code;
+    ec_matrix_t *encode;
+    ec_matrix_t **objects;
+};
+
+struct _ec_heal {
+    struct list_head list;
+    gf_lock_t lock;
+    xlator_t *xl;
+    ec_fop_data_t *fop;
+    void *data;
+    ec_fop_data_t *lookup;
+    loc_t loc;
+    struct iatt iatt;
+    char *symlink;
+    fd_t *fd;
+    int32_t partial;
+    int32_t done;
+    int32_t error;
+    gf_boolean_t nameheal;
+    uintptr_t available;
+    uintptr_t good;
+    uintptr_t bad;
+    uintptr_t open;
+    uintptr_t fixed;
+    uint64_t offset;
+    uint64_t size;
+    uint64_t total_size;
+    uint64_t version[2];
+    uint64_t raw_size;
+};
+
+struct subvol_healer {
+    xlator_t *this;
+    int subvol;
+    gf_boolean_t running;
+    gf_boolean_t rerun;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t thread;
+};
+
+struct _ec_self_heald {
+    gf_boolean_t iamshd;
+    gf_boolean_t enabled;
+    int timeout;
+    uint32_t max_threads;
+    uint32_t wait_qlength;
+    struct subvol_healer *index_healers;
+    struct subvol_healer *full_healers;
+};
+
+struct _ec_statistics {
+    struct {
+        gf_atomic_t hits;    /* Cache hits. */
+        gf_atomic_t misses;  /* Cache misses. */
+        gf_atomic_t updates; /* Number of times an existing stripe has
+                                been updated with new content. */
+        gf_atomic_t invals;  /* Number of times an existing stripe has
+                                been invalidated because of truncates
+                                or discards. */
+        gf_atomic_t evicts;  /* Number of times that an existing entry
+                                has been evicted to make room for newer
+                                entries. */
+        gf_atomic_t allocs;  /* Number of memory allocations made to
+                                store stripes. */
+        gf_atomic_t errors;  /* Number of errors that have caused extra
+                                requests. (Basically memory allocation
+                                errors). */
+    } stripe_cache;
+    struct {
+        gf_atomic_t attempted; /*Number of heals attempted on
+                                files/directories*/
+        gf_atomic_t completed; /*Number of heals complted on files/directories*/
+    } shd;
+};
+
+struct _ec {
+    xlator_t *xl;
+    int32_t healers;
+    int32_t heal_waiters;
+    int32_t nodes; /* Total number of bricks(n) */
+    int32_t bits_for_nodes;
+    int32_t fragments;      /* Data bricks(k) */
+    int32_t redundancy;     /* Redundant bricks(m) */
+    uint32_t fragment_size; /* Size of fragment/chunk on a
+                               brick. */
+    uint32_t stripe_size;   /* (fragment_size * fragments)
+                               maximum size of user data
+                               stored in one stripe. */
+    int32_t up;             /* Represents whether EC volume is
+                               up or not. */
+    uint32_t idx;
+    uint32_t xl_up_count;     /* Number of UP bricks. */
+    uintptr_t xl_up;          /* Bit flag representing UP
+                                 bricks */
+    uint32_t xl_notify_count; /* Number of notifications. */
+    uintptr_t xl_notify;      /* Bit flag representing
+                                 notification for bricks. */
+    uintptr_t node_mask;
+    uintptr_t read_mask;         /*Stores user defined read-mask*/
+    gf_atomic_t async_fop_count; /* Number of on going asynchronous fops. */
+    xlator_t **xl_list;
+    gf_lock_t lock;
+    gf_timer_t *timer;
+    gf_boolean_t shutdown;
+    gf_boolean_t eager_lock;
+    gf_boolean_t other_eager_lock;
+    gf_boolean_t optimistic_changelog;
+    gf_boolean_t parallel_writes;
+    uint32_t stripe_cache;
+    uint32_t quorum_count;
+    uint32_t background_heals;
+    uint32_t heal_wait_qlen;
+    uint32_t self_heal_window_size; /* max size of read/writes */
+    uint32_t eager_lock_timeout;
+    uint32_t other_eager_lock_timeout;
+    struct list_head pending_fops;
+    struct list_head heal_waiting;
+    struct list_head healing;
+    struct mem_pool *fop_pool;
+    struct mem_pool *cbk_pool;
+    struct mem_pool *lock_pool;
+    ec_self_heald_t shd;
+    char vol_uuid[UUID_SIZE + 1];
+    dict_t *leaf_to_subvolid;
+    ec_read_policy_t read_policy;
+    ec_matrix_list_t matrix;
+    ec_statistics_t stats;
+};
+
+#endif /* __EC_TYPES_H__ */
diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c
index c803ebfa796..7344be4968d 100644
--- a/xlators/cluster/ec/src/ec.c
+++ b/xlators/cluster/ec/src/ec.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,67 +8,63 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "defaults.h"
-#include "statedump.h"
-#include "compat-errno.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/upcall-utils.h>
 
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-mem-types.h"
+#include "ec-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-fops.h"
 #include "ec-method.h"
-#include "ec.h"
-#include "ec-messages.h"
+#include "ec-code.h"
 #include "ec-heald.h"
+#include <glusterfs/events.h>
 
 static char *ec_read_policies[EC_READ_POLICY_MAX + 1] = {
-        [EC_ROUND_ROBIN] = "round-robin",
-        [EC_GFID_HASH] = "gfid-hash",
-        [EC_READ_POLICY_MAX] = NULL
-};
-#define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS
-/* The maximum number of nodes is derived from the maximum allowed fragments
- * using the rule that redundancy cannot be equal or greater than the number
- * of fragments.
- */
-#define EC_MAX_NODES min(EC_MAX_FRAGMENTS * 2 - 1, EC_METHOD_MAX_NODES)
+    [EC_ROUND_ROBIN] = "round-robin",
+    [EC_GFID_HASH] = "gfid-hash",
+    [EC_READ_POLICY_MAX] = NULL};
 
 #define EC_INTERNAL_XATTR_OR_GOTO(name, xattr, op_errno, label)                \
-        do {                                                                   \
-                if (ec_is_internal_xattr (NULL, (char *)name, NULL, NULL)) {   \
-                        op_errno = EPERM;                                      \
-                        goto label;                                            \
-                }                                                              \
-                if (name && (strlen (name) == 0) && xattr) {                   \
-                        /* Bulk [f]removexattr/[f]setxattr */                  \
-                        GF_IF_INTERNAL_XATTR_GOTO (EC_XATTR_PREFIX"*", xattr,  \
-                                                   op_errno, label);           \
-                }                                                              \
-        } while (0)
-
-int32_t ec_parse_options(xlator_t * this)
-{
-    ec_t * ec = this->private;
+    do {                                                                       \
+        if (ec_is_internal_xattr(NULL, (char *)name, NULL, NULL)) {            \
+            op_errno = EPERM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+        if (name && (strlen(name) == 0) && xattr) {                            \
+            /* Bulk [f]removexattr/[f]setxattr */                              \
+            GF_IF_INTERNAL_XATTR_GOTO(EC_XATTR_PREFIX "*", xattr, op_errno,    \
+                                      label);                                  \
+        }                                                                      \
+    } while (0)
+
+int32_t
+ec_parse_options(xlator_t *this)
+{
+    ec_t *ec = this->private;
     int32_t error = EINVAL;
     uintptr_t mask;
 
     GF_OPTION_INIT("redundancy", ec->redundancy, int32, out);
     ec->fragments = ec->nodes - ec->redundancy;
     if ((ec->redundancy < 1) || (ec->redundancy >= ec->fragments) ||
-        (ec->fragments > EC_MAX_FRAGMENTS))
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_INVALID_REDUNDANCY,
-                "Invalid redundancy (must be between "
-                "1 and %d)", (ec->nodes - 1) / 2);
+        (ec->fragments > EC_MAX_FRAGMENTS)) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REDUNDANCY,
+               "Invalid redundancy (must be between "
+               "1 and %d)",
+               (ec->nodes - 1) / 2);
 
         goto out;
     }
 
     ec->bits_for_nodes = 1;
     mask = 2;
-    while (ec->nodes > mask)
-    {
+    while (ec->nodes > mask) {
         ec->bits_for_nodes++;
         mask <<= 1;
     }
@@ -76,9 +72,10 @@ int32_t ec_parse_options(xlator_t * this)
     ec->fragment_size = EC_METHOD_CHUNK_SIZE;
     ec->stripe_size = ec->fragment_size * ec->fragments;
 
-    gf_msg_debug ("ec", 0, "Initialized with: nodes=%u, fragments=%u, "
-                               "stripe_size=%u, node_mask=%lX",
-           ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask);
+    gf_msg_debug("ec", 0,
+                 "Initialized with: nodes=%u, fragments=%u, "
+                 "stripe_size=%u, node_mask=%" PRIxFAST32,
+                 ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask);
 
     error = 0;
 
@@ -86,30 +83,28 @@ out:
     return error;
 }
 
-int32_t ec_prepare_childs(xlator_t * this)
+int32_t
+ec_prepare_childs(xlator_t *this)
 {
-    ec_t * ec = this->private;
-    xlator_list_t * child = NULL;
+    ec_t *ec = this->private;
+    xlator_list_t *child = NULL;
     int32_t count = 0;
 
-    for (child = this->children; child != NULL; child = child->next)
-    {
+    for (child = this->children; child != NULL; child = child->next) {
         count++;
     }
-    if (count > EC_MAX_NODES)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_TOO_MANY_SUBVOLS, "Too many subvolumes");
+    if (count > EC_MAX_NODES) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_TOO_MANY_SUBVOLS,
+               "Too many subvolumes");
 
         return EINVAL;
     }
     ec->nodes = count;
 
     ec->xl_list = GF_CALLOC(count, sizeof(ec->xl_list[0]), ec_mt_xlator_t);
-    if (ec->xl_list == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Allocation of xlator list failed");
+    if (ec->xl_list == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Allocation of xlator list failed");
 
         return ENOMEM;
     }
@@ -117,8 +112,7 @@ int32_t ec_prepare_childs(xlator_t * this)
     ec->xl_up_count = 0;
 
     count = 0;
-    for (child = this->children; child != NULL; child = child->next)
-    {
+    for (child = this->children; child != NULL; child = child->next) {
         ec->xl_list[count++] = child->xlator;
     }
 
@@ -127,43 +121,42 @@ int32_t ec_prepare_childs(xlator_t * this)
 
 /* This function transforms the subvol to subvol-id*/
 static int
-_subvol_to_subvolid (dict_t *this, char *key, data_t *value, void *data)
-{
-        ec_t *ec = data;
-        xlator_t *subvol = NULL;
-        int      i = 0;
-        int     ret = -1;
-
-        subvol = data_to_ptr (value);
-        for (i = 0; i < ec->nodes; i++) {
-                if (ec->xl_list[i] == subvol) {
-                        ret = dict_set_int32 (this, key, i);
-                        /* -1 stops dict_foreach and returns -1*/
-                        if (ret < 0)
-                                ret = -1;
-                        goto out;
-                }
+_subvol_to_subvolid(dict_t *this, char *key, data_t *value, void *data)
+{
+    ec_t *ec = data;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = -1;
+
+    subvol = data_to_ptr(value);
+    for (i = 0; i < ec->nodes; i++) {
+        if (ec->xl_list[i] == subvol) {
+            ret = dict_set_int32(this, key, i);
+            /* -1 stops dict_foreach and returns -1*/
+            if (ret < 0)
+                ret = -1;
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-ec_subvol_to_subvol_id_transform (ec_t *ec, dict_t *leaf_to_subvolid)
+ec_subvol_to_subvol_id_transform(ec_t *ec, dict_t *leaf_to_subvolid)
 {
-        return dict_foreach (leaf_to_subvolid, _subvol_to_subvolid, ec);
+    return dict_foreach(leaf_to_subvolid, _subvol_to_subvolid, ec);
 }
 
-void __ec_destroy_private(xlator_t * this)
+void
+__ec_destroy_private(xlator_t *this)
 {
-    ec_t * ec = this->private;
+    ec_t *ec = this->private;
 
-    if (ec != NULL)
-    {
+    if (ec != NULL) {
         LOCK(&ec->lock);
 
-        if (ec->timer != NULL)
-        {
+        if (ec->timer != NULL) {
             gf_timer_call_cancel(this->ctx, ec->timer);
             ec->timer = NULL;
         }
@@ -181,42 +174,41 @@ void __ec_destroy_private(xlator_t * this)
         sleep(2);
 
         this->private = NULL;
-        if (ec->xl_list != NULL)
-        {
+        if (ec->xl_list != NULL) {
             GF_FREE(ec->xl_list);
             ec->xl_list = NULL;
         }
 
-        if (ec->fop_pool != NULL)
-        {
+        if (ec->fop_pool != NULL) {
             mem_pool_destroy(ec->fop_pool);
         }
 
-        if (ec->cbk_pool != NULL)
-        {
+        if (ec->cbk_pool != NULL) {
             mem_pool_destroy(ec->cbk_pool);
         }
 
-        if (ec->lock_pool != NULL)
-        {
+        if (ec->lock_pool != NULL) {
             mem_pool_destroy(ec->lock_pool);
         }
 
         LOCK_DESTROY(&ec->lock);
 
         if (ec->leaf_to_subvolid)
-                dict_unref (ec->leaf_to_subvolid);
+            dict_unref(ec->leaf_to_subvolid);
+
+        ec_method_fini(&ec->matrix);
+
         GF_FREE(ec);
     }
 }
 
-int32_t mem_acct_init(xlator_t * this)
+int32_t
+mem_acct_init(xlator_t *this)
 {
-    if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Memory accounting initialization "
-                                         "failed.");
+    if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Memory accounting initialization "
+               "failed.");
 
         return -1;
     }
@@ -225,340 +217,582 @@ int32_t mem_acct_init(xlator_t * this)
 }
 
 void
-ec_configure_background_heal_opts (ec_t *ec, int background_heals,
-                                   int heal_wait_qlen)
+ec_configure_background_heal_opts(ec_t *ec, int background_heals,
+                                  int heal_wait_qlen)
 {
-        if (background_heals == 0) {
-                ec->heal_wait_qlen = 0;
-        } else {
-                ec->heal_wait_qlen = heal_wait_qlen;
-        }
-        ec->background_heals = background_heals;
+    if (background_heals == 0) {
+        ec->heal_wait_qlen = 0;
+    } else {
+        ec->heal_wait_qlen = heal_wait_qlen;
+    }
+    ec->background_heals = background_heals;
 }
 
 int
-ec_assign_read_policy (ec_t *ec, char *read_policy)
+ec_assign_read_policy(ec_t *ec, char *read_policy)
 {
-        int read_policy_idx = -1;
+    int read_policy_idx = -1;
 
-        read_policy_idx = gf_get_index_by_elem (ec_read_policies, read_policy);
-        if (read_policy_idx < 0 || read_policy_idx >= EC_READ_POLICY_MAX)
-                return -1;
+    read_policy_idx = gf_get_index_by_elem(ec_read_policies, read_policy);
+    if (read_policy_idx < 0 || read_policy_idx >= EC_READ_POLICY_MAX)
+        return -1;
 
-        ec->read_policy = read_policy_idx;
-        return 0;
+    ec->read_policy = read_policy_idx;
+    return 0;
 }
 
 int32_t
-reconfigure (xlator_t *this, dict_t *options)
-{
-        ec_t     *ec              = this->private;
-        char     *read_policy     = NULL;
-        uint32_t heal_wait_qlen   = 0;
-        uint32_t background_heals = 0;
-
-        GF_OPTION_RECONF ("self-heal-daemon", ec->shd.enabled, options, bool,
-                          failed);
-        GF_OPTION_RECONF ("iam-self-heal-daemon", ec->shd.iamshd, options,
-                          bool, failed);
-        GF_OPTION_RECONF ("eager-lock", ec->eager_lock, options,
-                          bool, failed);
-        GF_OPTION_RECONF ("background-heals", background_heals, options,
-                          uint32, failed);
-        GF_OPTION_RECONF ("heal-wait-qlength", heal_wait_qlen, options,
-                          uint32, failed);
-        GF_OPTION_RECONF ("heal-timeout", ec->shd.timeout, options,
-                          int32, failed);
-        ec_configure_background_heal_opts (ec, background_heals,
-                                           heal_wait_qlen);
-        GF_OPTION_RECONF ("read-policy", read_policy, options, str, failed);
-        if (ec_assign_read_policy (ec, read_policy))
-                goto failed;
+reconfigure(xlator_t *this, dict_t *options)
+{
+    ec_t *ec = this->private;
+    char *read_policy = NULL;
+    char *extensions = NULL;
+    uint32_t heal_wait_qlen = 0;
+    uint32_t background_heals = 0;
+    int32_t ret = -1;
+    int32_t err;
+
+    GF_OPTION_RECONF("cpu-extensions", extensions, options, str, failed);
+
+    GF_OPTION_RECONF("self-heal-daemon", ec->shd.enabled, options, bool,
+                     failed);
+    GF_OPTION_RECONF("iam-self-heal-daemon", ec->shd.iamshd, options, bool,
+                     failed);
+    GF_OPTION_RECONF("eager-lock", ec->eager_lock, options, bool, failed);
+    GF_OPTION_RECONF("other-eager-lock", ec->other_eager_lock, options, bool,
+                     failed);
+    GF_OPTION_RECONF("eager-lock-timeout", ec->eager_lock_timeout, options,
+                     uint32, failed);
+    GF_OPTION_RECONF("other-eager-lock-timeout", ec->other_eager_lock_timeout,
+                     options, uint32, failed);
+    GF_OPTION_RECONF("background-heals", background_heals, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("heal-wait-qlength", heal_wait_qlen, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("self-heal-window-size", ec->self_heal_window_size,
+                     options, uint32, failed);
+    GF_OPTION_RECONF("heal-timeout", ec->shd.timeout, options, int32, failed);
+    ec_configure_background_heal_opts(ec, background_heals, heal_wait_qlen);
+    GF_OPTION_RECONF("shd-max-threads", ec->shd.max_threads, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("shd-wait-qlength", ec->shd.wait_qlength, options, uint32,
+                     failed);
+
+    GF_OPTION_RECONF("read-policy", read_policy, options, str, failed);
+
+    GF_OPTION_RECONF("optimistic-change-log", ec->optimistic_changelog, options,
+                     bool, failed);
+    GF_OPTION_RECONF("parallel-writes", ec->parallel_writes, options, bool,
+                     failed);
+    GF_OPTION_RECONF("stripe-cache", ec->stripe_cache, options, uint32, failed);
+    GF_OPTION_RECONF("quorum-count", ec->quorum_count, options, uint32, failed);
+    ret = 0;
+    if (ec_assign_read_policy(ec, read_policy)) {
+        ret = -1;
+    }
+
+    err = ec_method_update(this, &ec->matrix, extensions);
+    if (err != 0) {
+        ret = -1;
+    }
 
-        return 0;
 failed:
-        return -1;
+    return ret;
 }
 
 glusterfs_event_t
-ec_get_event_from_state (ec_t *ec)
-{
-        int               down_count = 0;
-
-        if (ec->xl_up_count >= ec->fragments) {
-                /* If ec is up but some subvolumes are yet to notify, give
-                 * grace time for other subvols to notify to prevent start of
-                 * I/O which may result in self-heals */
-                if (ec->timer && ec->xl_notify_count < ec->nodes)
-                        return GF_EVENT_MAXVAL;
-
-                return GF_EVENT_CHILD_UP;
-        } else {
-                down_count = ec->xl_notify_count - ec->xl_up_count;
-                if (down_count > ec->redundancy)
-                        return GF_EVENT_CHILD_DOWN;
-        }
+ec_get_event_from_state(ec_t *ec)
+{
+    int down_count = 0;
+
+    if (ec->xl_up_count >= ec->fragments) {
+        /* If ec is up but some subvolumes are yet to notify, give
+         * grace time for other subvols to notify to prevent start of
+         * I/O which may result in self-heals */
+        if (ec->xl_notify_count < ec->nodes)
+            return GF_EVENT_MAXVAL;
+
+        return GF_EVENT_CHILD_UP;
+    } else {
+        down_count = ec->xl_notify_count - ec->xl_up_count;
+        if (down_count > ec->redundancy)
+            return GF_EVENT_CHILD_DOWN;
+    }
 
-        return GF_EVENT_MAXVAL;
+    return GF_EVENT_MAXVAL;
 }
 
 void
-ec_up (xlator_t *this, ec_t *ec)
+ec_up(xlator_t *this, ec_t *ec)
 {
-        if (ec->timer != NULL) {
-                gf_timer_call_cancel (this->ctx, ec->timer);
-                ec->timer = NULL;
-        }
+    char str1[32], str2[32];
 
-        ec->up = 1;
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                EC_MSG_EC_UP, "Going UP");
+    if (ec->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, ec->timer);
+        ec->timer = NULL;
+    }
+
+    ec->up = 1;
+    gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP,
+           "Going UP : Child UP = %s Child Notify = %s",
+           ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes));
+
+    gf_event(EVENT_EC_MIN_BRICKS_UP, "subvol=%s", this->name);
 }
 
 void
-ec_down (xlator_t *this, ec_t *ec)
+ec_down(xlator_t *this, ec_t *ec)
 {
-        if (ec->timer != NULL) {
-                gf_timer_call_cancel(this->ctx, ec->timer);
-                ec->timer = NULL;
-        }
+    char str1[32], str2[32];
+
+    if (ec->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, ec->timer);
+        ec->timer = NULL;
+    }
 
-        ec->up = 0;
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                EC_MSG_EC_DOWN, "Going DOWN");
+    ec->up = 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN,
+           "Going DOWN : Child UP = %s Child Notify = %s",
+           ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes));
+
+    gf_event(EVENT_EC_MIN_BRICKS_NOT_UP, "subvol=%s", this->name);
 }
 
 void
-ec_notify_cbk (void *data)
+ec_notify_cbk(void *data)
 {
-        ec_t *ec = data;
-        glusterfs_event_t event = GF_EVENT_MAXVAL;
-        gf_boolean_t propagate = _gf_false;
+    ec_t *ec = data;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+    gf_boolean_t launch_heal = _gf_false;
 
-        LOCK(&ec->lock);
-        {
-                if (!ec->timer) {
-                        /*
-                         * Either child_up/child_down is already sent to parent
-                         * This is a spurious wake up.
-                         */
-                        goto unlock;
-                }
-
-                gf_timer_call_cancel (ec->xl->ctx, ec->timer);
-                ec->timer = NULL;
-
-                event = ec_get_event_from_state (ec);
-                /* If event is still MAXVAL then enough subvolumes didn't
-                 * notify, treat it as CHILD_DOWN. */
-                if (event == GF_EVENT_MAXVAL) {
-                        event = GF_EVENT_CHILD_DOWN;
-                        ec->xl_notify = (1ULL << ec->nodes) - 1ULL;
-                        ec->xl_notify_count = ec->nodes;
-                } else if (event == GF_EVENT_CHILD_UP) {
-                        /* Rest of the bricks are still not coming up,
-                         * notify that ec is up. Files/directories will be
-                         * healed as in when they come up. */
-                        ec_up (ec->xl, ec);
-                }
-
-                /* CHILD_DOWN should not come here as no grace period is given
-                 * for notifying CHILD_DOWN. */
-
-                propagate = _gf_true;
+    LOCK(&ec->lock);
+    {
+        if (!ec->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent
+             * This is a spurious wake up.
+             */
+            goto unlock;
         }
-unlock:
-        UNLOCK(&ec->lock);
 
-        if (propagate) {
-                default_notify (ec->xl, event, NULL);
+        gf_timer_call_cancel(ec->xl->ctx, ec->timer);
+        ec->timer = NULL;
+
+        /* The timeout has expired, so any subvolume that has not
+         * already reported its state, will be considered to be down.
+         * We mark as if all bricks had reported. */
+        ec->xl_notify = (1ULL << ec->nodes) - 1ULL;
+        ec->xl_notify_count = ec->nodes;
+
+        /* Since we have marked all subvolumes as notified, it's
+         * guaranteed that ec_get_event_from_state() will return
+         * CHILD_UP or CHILD_DOWN, but not MAXVAL. */
+        event = ec_get_event_from_state(ec);
+        if (event == GF_EVENT_CHILD_UP) {
+            /* We are ready to bring the volume up. If there are
+             * still bricks DOWN, they will be healed when they
+             * come up. */
+            ec_up(ec->xl, ec);
+
+            if (ec->shd.iamshd && !ec->shutdown) {
+                launch_heal = _gf_true;
+                GF_ATOMIC_INC(ec->async_fop_count);
+            }
         }
+
+        propagate = _gf_true;
+    }
+unlock:
+    UNLOCK(&ec->lock);
+
+    if (launch_heal) {
+        /* We have just brought the volume UP, so we trigger
+         * a self-heal check on the root directory. */
+        ec_launch_replace_heal(ec);
+    }
+    if (propagate) {
+        default_notify(ec->xl, event, NULL);
+    }
 }
 
 void
-ec_launch_notify_timer (xlator_t *this, ec_t *ec)
-{
-        struct timespec delay = {0, };
-
-        gf_msg_debug (this->name, 0, "Initiating child-down timer");
-        delay.tv_sec = 10;
-        delay.tv_nsec = 0;
-        ec->timer = gf_timer_call_after (this->ctx, delay, ec_notify_cbk, ec);
-        if (ec->timer == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_TIMER_CREATE_FAIL, "Cannot create timer "
-                        "for delayed initialization");
-        }
+ec_launch_notify_timer(xlator_t *this, ec_t *ec)
+{
+    struct timespec delay = {
+        0,
+    };
+
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    ec->timer = gf_timer_call_after(this->ctx, delay, ec_notify_cbk, ec);
+    if (ec->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer "
+               "for delayed initialization");
+    }
 }
 
-void
-ec_handle_up (xlator_t *this, ec_t *ec, int32_t idx)
+gf_boolean_t
+ec_disable_delays(ec_t *ec)
 {
-        if (((ec->xl_notify >> idx) & 1) == 0) {
-                ec->xl_notify |= 1ULL << idx;
-                ec->xl_notify_count++;
-        }
+    ec->shutdown = _gf_true;
 
-        if (((ec->xl_up >> idx) & 1) == 0) { /* Duplicate event */
-                ec->xl_up |= 1ULL << idx;
-                ec->xl_up_count++;
-        }
+    return __ec_is_last_fop(ec);
 }
 
 void
-ec_handle_down (xlator_t *this, ec_t *ec, int32_t idx)
+ec_cleanup_healer_object(ec_t *ec)
 {
-        if (((ec->xl_notify >> idx) & 1) == 0) {
-                ec->xl_notify |= 1ULL << idx;
-                ec->xl_notify_count++;
-        }
+    struct subvol_healer *healer = NULL;
+    ec_self_heald_t *shd = NULL;
+    void *res = NULL;
+    int i = 0;
+    gf_boolean_t is_join = _gf_false;
 
-        if (((ec->xl_up >> idx) & 1) != 0) { /* Duplicate event */
-                gf_msg_debug (this->name, 0, "Child %d is DOWN", idx);
+    shd = &ec->shd;
+    if (!shd->iamshd)
+        return;
 
-                ec->xl_up ^= 1ULL << idx;
-                ec->xl_up_count--;
+    for (i = 0; i < ec->nodes; i++) {
+        healer = &shd->index_healers[i];
+        pthread_mutex_lock(&healer->mutex);
+        {
+            healer->rerun = 1;
+            if (healer->running) {
+                pthread_cond_signal(&healer->cond);
+                is_join = _gf_true;
+            }
+        }
+        pthread_mutex_unlock(&healer->mutex);
+        if (is_join) {
+            pthread_join(healer->thread, &res);
+            is_join = _gf_false;
         }
-}
-
-gf_boolean_t
-ec_disable_delays(ec_t *ec)
-{
-        ec->shutdown = _gf_true;
 
-        return list_empty (&ec->pending_fops);
+        healer = &shd->full_healers[i];
+        pthread_mutex_lock(&healer->mutex);
+        {
+            healer->rerun = 1;
+            if (healer->running) {
+                pthread_cond_signal(&healer->cond);
+                is_join = _gf_true;
+            }
+        }
+        pthread_mutex_unlock(&healer->mutex);
+        if (is_join) {
+            pthread_join(healer->thread, &res);
+            is_join = _gf_false;
+        }
+    }
 }
-
 void
 ec_pending_fops_completed(ec_t *ec)
 {
-        if (ec->shutdown) {
-                default_notify(ec->xl, GF_EVENT_PARENT_DOWN, NULL);
-        }
+    if (ec->shutdown) {
+        default_notify(ec->xl, GF_EVENT_PARENT_DOWN, NULL);
+    }
+}
+
+static gf_boolean_t
+ec_set_up_state(ec_t *ec, uintptr_t index_mask, uintptr_t new_state)
+{
+    uintptr_t current_state = 0;
+
+    if (xlator_is_cleanup_starting(ec->xl))
+        return _gf_false;
+
+    if ((ec->xl_notify & index_mask) == 0) {
+        ec->xl_notify |= index_mask;
+        ec->xl_notify_count++;
+    }
+    current_state = ec->xl_up & index_mask;
+    if (current_state != new_state) {
+        ec->xl_up ^= index_mask;
+        ec->xl_up_count += (current_state ? -1 : 1);
+
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+ec_upcall(ec_t *ec, struct gf_upcall *upcall)
+{
+    struct gf_upcall_cache_invalidation *ci = NULL;
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    inode_t *inode;
+    inode_table_t *table;
+
+    switch (upcall->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ci = upcall->data;
+            ci->flags |= UP_INVAL_ATTR;
+            return _gf_true;
+
+        case GF_UPCALL_INODELK_CONTENTION:
+            lc = upcall->data;
+            if (strcmp(lc->domain, ec->xl->name) != 0) {
+                /* The lock is not owned by EC, ignore it. */
+                return _gf_true;
+            }
+            table = ((xlator_t *)ec->xl->graph->top)->itable;
+            if (table == NULL) {
+                /* Self-heal daemon doesn't have an inode table on the top
+                 * xlator because it doesn't need it. In this case we should
+                 * use the inode table managed by EC itself where all inodes
+                 * being healed should be present. However self-heal doesn't
+                 * use eager-locking and inodelk's are already released as
+                 * soon as possible. In this case we can safely ignore these
+                 * notifications. */
+                return _gf_false;
+            }
+            inode = inode_find(table, upcall->gfid);
+            /* If inode is not found, it means that it's already released,
+             * so we can ignore it. Probably it has been released and
+             * destroyed while the contention notification was being sent.
+             */
+            if (inode != NULL) {
+                ec_lock_release(ec, inode);
+                inode_unref(inode);
+            }
+
+            return _gf_false;
+
+        default:
+            return _gf_true;
+    }
 }
 
 int32_t
-ec_notify (xlator_t *this, int32_t event, void *data, void *data2)
-{
-        ec_t             *ec        = this->private;
-        int32_t           idx       = 0;
-        int32_t           error     = 0;
-        glusterfs_event_t old_event = GF_EVENT_MAXVAL;
-        dict_t            *input    = NULL;
-        dict_t            *output   = NULL;
-        gf_boolean_t      propagate = _gf_true;
-
-        gf_msg_trace (this->name, 0, "NOTIFY(%d): %p, %p",
-                event, data, data2);
-
-        if (event == GF_EVENT_TRANSLATOR_OP) {
-                if (!ec->up) {
-                        error = -1;
-                } else {
-                        input = data;
-                        output = data2;
-                        error = ec_xl_op (this, input, output);
-                }
-                goto out;
+ec_notify(xlator_t *this, int32_t event, void *data, void *data2)
+{
+    ec_t *ec = this->private;
+    int32_t idx = 0;
+    int32_t error = 0;
+    glusterfs_event_t old_event = GF_EVENT_MAXVAL;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t propagate = _gf_true;
+    gf_boolean_t needs_shd_check = _gf_false;
+    int32_t orig_event = event;
+    uintptr_t mask = 0;
+
+    gf_msg_trace(this->name, 0, "NOTIFY(%d): %p, %p", event, data, data2);
+
+    if (event == GF_EVENT_UPCALL) {
+        propagate = ec_upcall(ec, data);
+        goto done;
+    }
+
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        if (!ec->up) {
+            error = -1;
+        } else {
+            input = data;
+            output = data2;
+            error = ec_xl_op(this, input, output);
         }
+        goto out;
+    }
 
-        for (idx = 0; idx < ec->nodes; idx++) {
-                if (ec->xl_list[idx] == data) {
-                        if (event == GF_EVENT_CHILD_UP)
-                                ec_selfheal_childup (ec, idx);
-                        break;
-                }
+    for (idx = 0; idx < ec->nodes; idx++) {
+        if (ec->xl_list[idx] == data) {
+            break;
         }
+    }
+
+    LOCK(&ec->lock);
+
+    if (event == GF_EVENT_PARENT_UP) {
+        /*
+         * Start a timer which sends appropriate event to parent
+         * xlator to prevent the 'mount' syscall from hanging.
+         */
+        ec_launch_notify_timer(this, ec);
+        goto unlock;
+    } else if (event == GF_EVENT_PARENT_DOWN) {
+        /* If there aren't pending fops running after we have waken up
+         * them, we immediately propagate the notification. */
+        propagate = ec_disable_delays(ec);
+        ec_cleanup_healer_object(ec);
+        goto unlock;
+    }
 
-        LOCK (&ec->lock);
-
-        if (event == GF_EVENT_PARENT_UP) {
-                /*
-                 * Start a timer which sends appropriate event to parent
-                 * xlator to prevent the 'mount' syscall from hanging.
-                 */
-                ec_launch_notify_timer (this, ec);
-                goto unlock;
-        } else if (event == GF_EVENT_PARENT_DOWN) {
-                /* If there aren't pending fops running after we have waken up
-                 * them, we immediately propagate the notification. */
-                propagate = ec_disable_delays(ec);
-                goto unlock;
+    if (idx < ec->nodes) { /* CHILD_* events */
+        old_event = ec_get_event_from_state(ec);
+
+        mask = 1ULL << idx;
+        if (event == GF_EVENT_CHILD_UP) {
+            /* We need to trigger a selfheal if a brick changes
+             * to UP state. */
+            if (ec_set_up_state(ec, mask, mask) && ec->shd.iamshd &&
+                !ec->shutdown) {
+                needs_shd_check = _gf_true;
+            }
+        } else if (event == GF_EVENT_CHILD_DOWN) {
+            ec_set_up_state(ec, mask, 0);
         }
 
-        if (idx < ec->nodes) { /* CHILD_* events */
-                old_event = ec_get_event_from_state (ec);
-
-                if (event == GF_EVENT_CHILD_UP) {
-                        ec_handle_up (this, ec, idx);
-                } else if (event == GF_EVENT_CHILD_DOWN) {
-                        ec_handle_down (this, ec, idx);
-                }
-
-                event = ec_get_event_from_state (ec);
-
-                if (event == GF_EVENT_CHILD_UP && !ec->up) {
-                        ec_up (this, ec);
-                } else if (event == GF_EVENT_CHILD_DOWN && ec->up) {
-                        ec_down (this, ec);
-                }
-
-                if (event != GF_EVENT_MAXVAL) {
-                        if (event == old_event) {
-                                event = GF_EVENT_CHILD_MODIFIED;
-                        }
-                } else {
-                        propagate = _gf_false;
-                }
+        event = ec_get_event_from_state(ec);
+
+        if (event == GF_EVENT_CHILD_UP) {
+            if (!ec->up) {
+                ec_up(this, ec);
+            }
+        } else {
+            /* If the volume is not UP, it's irrelevant if one
+             * brick has come up. We cannot heal anything. */
+            needs_shd_check = _gf_false;
+
+            if ((event == GF_EVENT_CHILD_DOWN) && ec->up) {
+                ec_down(this, ec);
+            }
         }
-unlock:
-        UNLOCK (&ec->lock);
 
-        if (propagate) {
-                error = default_notify (this, event, data);
+        if (event != GF_EVENT_MAXVAL) {
+            if (event == old_event) {
+                if (orig_event == GF_EVENT_CHILD_UP)
+                    event = GF_EVENT_SOME_DESCENDENT_UP;
+                else /* orig_event has to be GF_EVENT_CHILD_DOWN */
+                    event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            }
+        } else {
+            propagate = _gf_false;
+            needs_shd_check = _gf_false;
         }
 
-        if (ec->shd.iamshd &&
-            ec->xl_notify_count == ec->nodes &&
-            event == GF_EVENT_CHILD_UP) {
-                ec_launch_replace_heal (ec);
+        if (needs_shd_check) {
+            GF_ATOMIC_INC(ec->async_fop_count);
         }
+    }
+unlock:
+    UNLOCK(&ec->lock);
+
+done:
+    if (needs_shd_check) {
+        ec_launch_replace_heal(ec);
+    }
+    if (propagate) {
+        error = default_notify(this, event, data);
+    }
+
 out:
-        return error;
+    return error;
 }
 
 int32_t
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int ret = -1;
-        va_list         ap;
-        void *data2 = NULL;
+    int ret = -1;
+    va_list ap;
+    void *data2 = NULL;
 
-        va_start (ap, data);
-        data2 = va_arg (ap, dict_t*);
-        va_end (ap);
-        ret = ec_notify (this, event, data, data2);
+    va_start(ap, data);
+    data2 = va_arg(ap, dict_t *);
+    va_end(ap);
+    ret = ec_notify(this, event, data, data2);
 
-        return ret;
+    return ret;
+}
+
+static void
+ec_statistics_init(ec_t *ec)
+{
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.hits, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.misses, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.updates, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.invals, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.evicts, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.allocs, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.errors, 0);
+    GF_ATOMIC_INIT(ec->stats.shd.attempted, 0);
+    GF_ATOMIC_INIT(ec->stats.shd.completed, 0);
+}
+
+static int
+ec_assign_read_mask(ec_t *ec, char *read_mask_str)
+{
+    char *mask = NULL;
+    char *maskptr = NULL;
+    char *saveptr = NULL;
+    char *id_str = NULL;
+    int id = 0;
+    int ret = 0;
+    uintptr_t read_mask = 0;
+
+    if (!read_mask_str) {
+        ec->read_mask = 0;
+        ret = 0;
+        goto out;
+    }
+
+    mask = gf_strdup(read_mask_str);
+    if (!mask) {
+        ret = -1;
+        goto out;
+    }
+    maskptr = mask;
+
+    for (;;) {
+        id_str = strtok_r(maskptr, ":", &saveptr);
+        if (id_str == NULL)
+            break;
+        if (gf_string2int(id_str, &id)) {
+            gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+                   "In read-mask \"%s\" id %s is not a valid integer",
+                   read_mask_str, id_str);
+            ret = -1;
+            goto out;
+        }
+
+        if ((id < 0) || (id >= ec->nodes)) {
+            gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+                   "In read-mask \"%s\" id %d is not in range [0 - %d]",
+                   read_mask_str, id, ec->nodes - 1);
+            ret = -1;
+            goto out;
+        }
+        read_mask |= (1UL << id);
+        maskptr = NULL;
+    }
+
+    if (gf_bits_count(read_mask) < ec->fragments) {
+        gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+               "read-mask \"%s\" should contain at least %d ids", read_mask_str,
+               ec->fragments);
+        ret = -1;
+        goto out;
+    }
+    ec->read_mask = read_mask;
+    ret = 0;
+out:
+    GF_FREE(mask);
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-    ec_t *ec          = NULL;
+    ec_t *ec = NULL;
     char *read_policy = NULL;
+    char *extensions = NULL;
+    int32_t err;
+    char *read_mask_str = NULL;
 
-    if (this->parents == NULL)
-    {
-        gf_msg (this->name, GF_LOG_WARNING, 0,
-                EC_MSG_NO_PARENTS, "Volume does not have parents.");
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, EC_MSG_NO_PARENTS,
+               "Volume does not have parents.");
     }
 
     ec = GF_MALLOC(sizeof(*ec), ec_mt_ec_t);
-    if (ec == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to allocate private memory.");
+    if (ec == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate private memory.");
 
         return -1;
     }
@@ -569,6 +803,7 @@ init (xlator_t *this)
     ec->xl = this;
     LOCK_INIT(&ec->lock);
 
+    GF_ATOMIC_INIT(ec->async_fop_count, 0);
     INIT_LIST_HEAD(&ec->pending_fops);
     INIT_LIST_HEAD(&ec->heal_waiting);
     INIT_LIST_HEAD(&ec->healing);
@@ -577,67 +812,96 @@ init (xlator_t *this)
     ec->cbk_pool = mem_pool_new(ec_cbk_data_t, 4096);
     ec->lock_pool = mem_pool_new(ec_lock_t, 1024);
     if ((ec->fop_pool == NULL) || (ec->cbk_pool == NULL) ||
-        (ec->lock_pool == NULL))
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to create memory pools.");
+        (ec->lock_pool == NULL)) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to create memory pools.");
 
         goto failed;
     }
 
-    if (ec_prepare_childs(this) != 0)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                EC_MSG_XLATOR_INIT_FAIL, "Failed to initialize xlator");
+    if (ec_prepare_childs(this) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+               "Failed to initialize xlator");
 
         goto failed;
     }
 
-    if (ec_parse_options(this) != 0)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_XLATOR_PARSE_OPT_FAIL, "Failed to parse xlator options");
+    if (ec_parse_options(this) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_PARSE_OPT_FAIL,
+               "Failed to parse xlator options");
+
+        goto failed;
+    }
+
+    GF_OPTION_INIT("cpu-extensions", extensions, str, failed);
+
+    err = ec_method_init(this, &ec->matrix, ec->fragments, ec->nodes,
+                         ec->nodes * 2, extensions);
+    if (err != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -err, EC_MSG_MATRIX_FAILED,
+               "Failed to initialize matrix management");
 
         goto failed;
     }
 
-    ec_method_initialize();
-    GF_OPTION_INIT ("self-heal-daemon", ec->shd.enabled, bool, failed);
-    GF_OPTION_INIT ("iam-self-heal-daemon", ec->shd.iamshd, bool, failed);
-    GF_OPTION_INIT ("eager-lock", ec->eager_lock, bool, failed);
-    GF_OPTION_INIT ("background-heals", ec->background_heals, uint32, failed);
-    GF_OPTION_INIT ("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed);
-    ec_configure_background_heal_opts (ec, ec->background_heals,
-                                       ec->heal_wait_qlen);
-    GF_OPTION_INIT ("read-policy", read_policy, str, failed);
-    if (ec_assign_read_policy (ec, read_policy))
-            goto failed;
-
-    this->itable = inode_table_new (EC_SHD_INODE_LRU_LIMIT, this);
+    GF_OPTION_INIT("self-heal-daemon", ec->shd.enabled, bool, failed);
+    GF_OPTION_INIT("iam-self-heal-daemon", ec->shd.iamshd, bool, failed);
+    GF_OPTION_INIT("eager-lock", ec->eager_lock, bool, failed);
+    GF_OPTION_INIT("other-eager-lock", ec->other_eager_lock, bool, failed);
+    GF_OPTION_INIT("eager-lock-timeout", ec->eager_lock_timeout, uint32,
+                   failed);
+    GF_OPTION_INIT("other-eager-lock-timeout", ec->other_eager_lock_timeout,
+                   uint32, failed);
+    GF_OPTION_INIT("background-heals", ec->background_heals, uint32, failed);
+    GF_OPTION_INIT("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed);
+    GF_OPTION_INIT("self-heal-window-size", ec->self_heal_window_size, uint32,
+                   failed);
+    ec_configure_background_heal_opts(ec, ec->background_heals,
+                                      ec->heal_wait_qlen);
+    GF_OPTION_INIT("read-policy", read_policy, str, failed);
+    if (ec_assign_read_policy(ec, read_policy))
+        goto failed;
+
+    GF_OPTION_INIT("heal-timeout", ec->shd.timeout, int32, failed);
+    GF_OPTION_INIT("shd-max-threads", ec->shd.max_threads, uint32, failed);
+    GF_OPTION_INIT("shd-wait-qlength", ec->shd.wait_qlength, uint32, failed);
+    GF_OPTION_INIT("optimistic-change-log", ec->optimistic_changelog, bool,
+                   failed);
+    GF_OPTION_INIT("parallel-writes", ec->parallel_writes, bool, failed);
+    GF_OPTION_INIT("stripe-cache", ec->stripe_cache, uint32, failed);
+    GF_OPTION_INIT("quorum-count", ec->quorum_count, uint32, failed);
+    GF_OPTION_INIT("ec-read-mask", read_mask_str, str, failed);
+
+    if (ec_assign_read_mask(ec, read_mask_str))
+        goto failed;
+
+    this->itable = inode_table_new(EC_SHD_INODE_LRU_LIMIT, this);
     if (!this->itable)
-            goto failed;
+        goto failed;
 
     if (ec->shd.iamshd)
-            ec_selfheal_daemon_init (this);
-    gf_msg_debug (this->name, 0, "Disperse translator initialized.");
+        ec_selfheal_daemon_init(this);
+    gf_msg_debug(this->name, 0, "Disperse translator initialized.");
 
-    ec->leaf_to_subvolid = dict_new ();
+    ec->leaf_to_subvolid = dict_new();
     if (!ec->leaf_to_subvolid)
-            goto failed;
-    if (glusterfs_reachable_leaves (this, ec->leaf_to_subvolid)) {
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                EC_MSG_SUBVOL_BUILD_FAIL, "Failed to build subvol "
-                "dictionary");
+        goto failed;
+    if (glusterfs_reachable_leaves(this, ec->leaf_to_subvolid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_BUILD_FAIL,
+               "Failed to build subvol "
+               "dictionary");
         goto failed;
     }
 
-    if (ec_subvol_to_subvol_id_transform (ec, ec->leaf_to_subvolid) < 0) {
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                EC_MSG_SUBVOL_ID_DICT_SET_FAIL, "Failed to build subvol-id "
-                "dictionary");
+    if (ec_subvol_to_subvol_id_transform(ec, ec->leaf_to_subvolid) < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_ID_DICT_SET_FAIL,
+               "Failed to build subvol-id "
+               "dictionary");
         goto failed;
     }
 
+    ec_statistics_init(ec);
+
     return 0;
 
 failed:
@@ -646,13 +910,16 @@ failed:
     return -1;
 }
 
-void fini(xlator_t * this)
+void
+fini(xlator_t *this)
 {
+    ec_selfheal_daemon_fini(this);
     __ec_destroy_private(this);
 }
 
-int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int32_t mask, dict_t * xdata)
+int32_t
+ec_gf_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+             dict_t *xdata)
 {
     ec_access(frame, this, -1, EC_MINIMUM_ONE, default_access_cbk, NULL, loc,
               mask, xdata);
@@ -660,9 +927,9 @@ int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int32_t flags, mode_t mode, mode_t umask, fd_t * fd,
-                     dict_t * xdata)
+int32_t
+ec_gf_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
     ec_create(frame, this, -1, EC_MINIMUM_MIN, default_create_cbk, NULL, loc,
               flags, mode, umask, fd, xdata);
@@ -670,51 +937,58 @@ int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_discard(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                      off_t offset, size_t len, dict_t * xdata)
+int32_t
+ec_gf_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
 {
-    default_discard_failure_cbk(frame, ENOTSUP);
+    ec_discard(frame, this, -1, EC_MINIMUM_MIN, default_discard_cbk, NULL, fd,
+               offset, len, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_entrylk(call_frame_t * frame, xlator_t * this,
-                      const char * volume, loc_t * loc, const char * basename,
-                      entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+int32_t
+ec_gf_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    uint32_t fop_flags = EC_MINIMUM_ALL;
+
     if (cmd == ENTRYLK_UNLOCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_entrylk(frame, this, -1, minimum, default_entrylk_cbk, NULL,
-               volume, loc, basename, cmd, type, xdata);
+        fop_flags = EC_MINIMUM_ONE;
+    ec_entrylk(frame, this, -1, fop_flags, default_entrylk_cbk, NULL, volume,
+               loc, basename, cmd, type, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_fentrylk(call_frame_t * frame, xlator_t * this,
-                       const char * volume, fd_t * fd, const char * basename,
-                       entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+int32_t
+ec_gf_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, const char *basename, entrylk_cmd cmd,
+               entrylk_type type, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    uint32_t fop_flags = EC_MINIMUM_ALL;
+
     if (cmd == ENTRYLK_UNLOCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_fentrylk(frame, this, -1, minimum, default_fentrylk_cbk, NULL,
-                volume, fd, basename, cmd, type, xdata);
+        fop_flags = EC_MINIMUM_ONE;
+    ec_fentrylk(frame, this, -1, fop_flags, default_fentrylk_cbk, NULL, volume,
+                fd, basename, cmd, type, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_fallocate(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                        int32_t keep_size, off_t offset, size_t len,
-                        dict_t * xdata)
+int32_t
+ec_gf_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+                off_t offset, size_t len, dict_t *xdata)
 {
-    default_fallocate_failure_cbk(frame, ENOTSUP);
+    ec_fallocate(frame, this, -1, EC_MINIMUM_MIN, default_fallocate_cbk, NULL,
+                 fd, mode, offset, len, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_gf_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
     ec_flush(frame, this, -1, EC_MINIMUM_MIN, default_flush_cbk, NULL, fd,
              xdata);
@@ -722,8 +996,9 @@ int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    int32_t datasync, dict_t * xdata)
+int32_t
+ec_gf_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
 {
     ec_fsync(frame, this, -1, EC_MINIMUM_MIN, default_fsync_cbk, NULL, fd,
              datasync, xdata);
@@ -731,135 +1006,139 @@ int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_fsyncdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       int32_t datasync, dict_t * xdata)
+int32_t
+ec_gf_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-    ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL,
-                fd, datasync, xdata);
+    ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL, fd,
+                datasync, xdata);
 
     return 0;
 }
 
 int
-ec_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                         xlator_t **subvols)
+ec_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                        xlator_t **subvols)
 {
-        xlator_t *this = frame->this;
-        ec_t *ec = this->private;
+    xlator_t *this = frame->this;
+    ec_t *ec = this->private;
 
-        memcpy (subvols, ec->xl_list, sizeof (*subvols) * ec->nodes);
+    memcpy(subvols, ec->xl_list, sizeof(*subvols) * ec->nodes);
 
-        if (type == MARKER_XTIME_TYPE) {
-                /*Don't error out on ENOENT/ENOTCONN */
-                gauge[MCNT_NOTFOUND] = 0;
-                gauge[MCNT_ENOTCONN] = 0;
-        }
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
 
-        return ec->nodes;
+    return ec->nodes;
 }
 
 int32_t
-ec_handle_heal_commands (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         const char *name, dict_t *xdata)
+ec_handle_heal_commands(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        const char *name, dict_t *xdata)
 {
-        dict_t  *dict_rsp = NULL;
-        int     op_ret = -1;
-        int     op_errno = ENOMEM;
+    dict_t *dict_rsp = NULL;
+    int op_ret = -1;
+    int op_errno = ENOMEM;
 
-        if (!name || strcmp (name, GF_HEAL_INFO))
-                return -1;
+    if (!name || strcmp(name, GF_HEAL_INFO))
+        return -1;
 
-        dict_rsp = dict_new ();
-        if (dict_rsp == NULL)
-                goto out;
+    op_errno = -ec_get_heal_info(this, loc, &dict_rsp);
+    if (op_errno <= 0) {
+        op_errno = op_ret = 0;
+    }
 
-        if (dict_set_str (dict_rsp, "heal-info", "heal") == 0)
-                op_ret = 0;
-out:
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict_rsp, NULL);
-        if (dict_rsp)
-                dict_unref (dict_rsp);
-        return 0;
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict_rsp, NULL);
+    if (dict_rsp)
+        dict_unref(dict_rsp);
+    return 0;
 }
 
 int32_t
-ec_gf_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+ec_gf_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int     error = 0;
-        ec_t    *ec = this->private;
-        int32_t minimum = EC_MINIMUM_MIN;
+    int error = 0;
+    ec_t *ec = this->private;
+    int32_t fop_flags = EC_MINIMUM_ONE;
 
-        if (name && strcmp (name, EC_XATTR_HEAL) != 0) {
-                EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
-        }
+    if (name && strcmp(name, EC_XATTR_HEAL) != 0) {
+        EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
+    }
 
-        if (ec_handle_heal_commands (frame, this, loc, name, xdata) == 0)
-                return 0;
+    if (ec_handle_heal_commands(frame, this, loc, name, xdata) == 0)
+        return 0;
 
-        if (cluster_handle_marker_getxattr (frame, loc, name, ec->vol_uuid,
-                                            NULL, ec_marker_populate_args) == 0)
-                return 0;
+    if (cluster_handle_marker_getxattr(frame, loc, name, ec->vol_uuid, NULL,
+                                       ec_marker_populate_args) == 0)
+        return 0;
 
-        if (name && (fnmatch (GF_XATTR_STIME_PATTERN, name, 0) == 0))
-                minimum = EC_MINIMUM_ALL;
+    if (name && ((fnmatch(GF_XATTR_STIME_PATTERN, name, 0) == 0) ||
+                 XATTR_IS_NODE_UUID(name) || XATTR_IS_NODE_UUID_LIST(name))) {
+        fop_flags = EC_MINIMUM_ALL;
+    }
 
-        ec_getxattr (frame, this, -1, minimum, default_getxattr_cbk,
-                     NULL, loc, name, xdata);
+    ec_getxattr(frame, this, -1, fop_flags, default_getxattr_cbk, NULL, loc,
+                name, xdata);
 
-        return 0;
+    return 0;
 out:
-        error = ENODATA;
-        STACK_UNWIND_STRICT (getxattr, frame, -1, error, NULL, NULL);
-        return 0;
+    error = ENODATA;
+    STACK_UNWIND_STRICT(getxattr, frame, -1, error, NULL, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+ec_gf_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
 
-        ec_fgetxattr (frame, this, -1, EC_MINIMUM_MIN, default_fgetxattr_cbk,
-                      NULL, fd, name, xdata);
-        return 0;
+    ec_fgetxattr(frame, this, -1, EC_MINIMUM_ONE, default_fgetxattr_cbk, NULL,
+                 fd, name, xdata);
+    return 0;
 out:
-        error = ENODATA;
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, error, NULL, NULL);
-        return 0;
+    error = ENODATA;
+    STACK_UNWIND_STRICT(fgetxattr, frame, -1, error, NULL, NULL);
+    return 0;
 }
 
-int32_t ec_gf_inodelk(call_frame_t * frame, xlator_t * this,
-                      const char * volume, loc_t * loc, int32_t cmd,
-                      struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
     if (flock->l_type == F_UNLCK)
-            minimum = EC_MINIMUM_ONE;
+        fop_flags = EC_MINIMUM_ONE;
 
-    ec_inodelk(frame, this, -1, minimum, default_inodelk_cbk, NULL,
-               volume, loc, cmd, flock, xdata);
+    ec_inodelk(frame, this, &frame->root->lk_owner, -1, fop_flags,
+               default_inodelk_cbk, NULL, volume, loc, cmd, flock, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_finodelk(call_frame_t * frame, xlator_t * this,
-                       const char * volume, fd_t * fd, int32_t cmd,
-                       struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
     if (flock->l_type == F_UNLCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_finodelk(frame, this, -1, minimum, default_finodelk_cbk, NULL,
-                volume, fd, cmd, flock, xdata);
+        fop_flags = EC_MINIMUM_ONE;
+    ec_finodelk(frame, this, &frame->root->lk_owner, -1, fop_flags,
+                default_finodelk_cbk, NULL, volume, fd, cmd, flock, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
-                   loc_t * newloc, dict_t * xdata)
+int32_t
+ec_gf_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
     ec_link(frame, this, -1, EC_MINIMUM_MIN, default_link_cbk, NULL, oldloc,
             newloc, xdata);
@@ -867,20 +1146,22 @@ int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
     return 0;
 }
 
-int32_t ec_gf_lk(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                 int32_t cmd, struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+         struct gf_flock *flock, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
     if (flock->l_type == F_UNLCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_lk(frame, this, -1, minimum, default_lk_cbk, NULL, fd, cmd,
-          flock, xdata);
+        fop_flags = EC_MINIMUM_ONE;
+    ec_lk(frame, this, -1, fop_flags, default_lk_cbk, NULL, fd, cmd, flock,
+          xdata);
 
     return 0;
 }
 
-int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     dict_t * xdata)
+int32_t
+ec_gf_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_lookup(frame, this, -1, EC_MINIMUM_MIN, default_lookup_cbk, NULL, loc,
               xdata);
@@ -888,8 +1169,9 @@ int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    mode_t mode, mode_t umask, dict_t * xdata)
+int32_t
+ec_gf_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
     ec_mkdir(frame, this, -1, EC_MINIMUM_MIN, default_mkdir_cbk, NULL, loc,
              mode, umask, xdata);
@@ -897,8 +1179,9 @@ int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata)
+int32_t
+ec_gf_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t rdev, mode_t umask, dict_t *xdata)
 {
     ec_mknod(frame, this, -1, EC_MINIMUM_MIN, default_mknod_cbk, NULL, loc,
              mode, rdev, umask, xdata);
@@ -906,17 +1189,19 @@ int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_open(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                   int32_t flags, fd_t * fd, dict_t * xdata)
+int32_t
+ec_gf_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
 {
-    ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc,
-            flags, fd, xdata);
+    ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc, flags,
+            fd, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      fd_t * fd, dict_t * xdata)
+int32_t
+ec_gf_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+              dict_t *xdata)
 {
     ec_opendir(frame, this, -1, EC_MINIMUM_MIN, default_opendir_cbk, NULL, loc,
                fd, xdata);
@@ -924,8 +1209,9 @@ int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                      size_t size, off_t offset, dict_t * xdata)
+int32_t
+ec_gf_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
     ec_readdir(frame, this, -1, EC_MINIMUM_ONE, default_readdir_cbk, NULL, fd,
                size, offset, xdata);
@@ -933,17 +1219,19 @@ int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_readdirp(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       size_t size, off_t offset, dict_t * xdata)
+int32_t
+ec_gf_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
 {
-    ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL,
-                fd, size, offset, xdata);
+    ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL, fd,
+                size, offset, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                       size_t size, dict_t * xdata)
+int32_t
+ec_gf_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
     ec_readlink(frame, this, -1, EC_MINIMUM_ONE, default_readlink_cbk, NULL,
                 loc, size, xdata);
@@ -951,60 +1239,63 @@ int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_readv(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    size_t size, off_t offset, uint32_t flags, dict_t * xdata)
+int32_t
+ec_gf_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-    ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd,
-             size, offset, flags, xdata);
+    ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd, size,
+             offset, flags, xdata);
 
     return 0;
 }
 
 int32_t
-ec_gf_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   const char *name, dict_t *xdata)
+ec_gf_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        int     error  = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out);
 
-        ec_removexattr (frame, this, -1, EC_MINIMUM_MIN,
-                        default_removexattr_cbk, NULL, loc, name, xdata);
+    ec_removexattr(frame, this, -1, EC_MINIMUM_MIN, default_removexattr_cbk,
+                   NULL, loc, name, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(removexattr, frame, -1, error, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    const char *name, dict_t *xdata)
+ec_gf_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        int     error  = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out);
 
-        ec_fremovexattr (frame, this, -1, EC_MINIMUM_MIN,
-                         default_fremovexattr_cbk, NULL, fd, name, xdata);
+    ec_fremovexattr(frame, this, -1, EC_MINIMUM_MIN, default_fremovexattr_cbk,
+                    NULL, fd, name, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (fremovexattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fremovexattr, frame, -1, error, NULL);
+    return 0;
 }
 
-int32_t ec_gf_rename(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
-                     loc_t * newloc, dict_t * xdata)
+int32_t
+ec_gf_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-    ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL,
-              oldloc, newloc, xdata);
+    ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL, oldloc,
+              newloc, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    int xflags, dict_t * xdata)
+int32_t
+ec_gf_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+            dict_t *xdata)
 {
     ec_rmdir(frame, this, -1, EC_MINIMUM_MIN, default_rmdir_cbk, NULL, loc,
              xflags, xdata);
@@ -1012,8 +1303,9 @@ int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      struct iatt * stbuf, int32_t valid, dict_t * xdata)
+int32_t
+ec_gf_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
     ec_setattr(frame, this, -1, EC_MINIMUM_MIN, default_setattr_cbk, NULL, loc,
                stbuf, valid, xdata);
@@ -1021,51 +1313,52 @@ int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fsetattr(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       struct iatt * stbuf, int32_t valid, dict_t * xdata)
+int32_t
+ec_gf_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL,
-                fd, stbuf, valid, xdata);
+    ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL, fd,
+                stbuf, valid, xdata);
 
     return 0;
 }
 
 int32_t
-ec_gf_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                dict_t *dict, int32_t flags, dict_t *xdata)
+ec_gf_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out);
 
-        ec_setxattr (frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk,
-                     NULL, loc, dict, flags, xdata);
+    ec_setxattr(frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk, NULL,
+                loc, dict, flags, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, -1, error, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
+ec_gf_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out);
 
-        ec_fsetxattr (frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk,
-                      NULL, fd, dict, flags, xdata);
+    ec_fsetxattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk, NULL,
+                 fd, dict, flags, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, -1, error, NULL);
+    return 0;
 }
 
-int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                   dict_t * xdata)
+int32_t
+ec_gf_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_stat(frame, this, -1, EC_MINIMUM_MIN, default_stat_cbk, NULL, loc,
             xdata);
@@ -1073,8 +1366,8 @@ int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_gf_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
     ec_fstat(frame, this, -1, EC_MINIMUM_MIN, default_fstat_cbk, NULL, fd,
              xdata);
@@ -1082,8 +1375,8 @@ int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     dict_t * xdata)
+int32_t
+ec_gf_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_statfs(frame, this, -1, EC_MINIMUM_MIN, default_statfs_cbk, NULL, loc,
               xdata);
@@ -1091,9 +1384,9 @@ int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this,
-                      const char * linkname, loc_t * loc, mode_t umask,
-                      dict_t * xdata)
+int32_t
+ec_gf_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
     ec_symlink(frame, this, -1, EC_MINIMUM_MIN, default_symlink_cbk, NULL,
                linkname, loc, umask, xdata);
@@ -1101,8 +1394,9 @@ int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this,
     return 0;
 }
 
-int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                       off_t offset, dict_t * xdata)
+int32_t
+ec_gf_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
     ec_truncate(frame, this, -1, EC_MINIMUM_MIN, default_truncate_cbk, NULL,
                 loc, offset, xdata);
@@ -1110,8 +1404,9 @@ int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                        off_t offset, dict_t * xdata)
+int32_t
+ec_gf_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
     ec_ftruncate(frame, this, -1, EC_MINIMUM_MIN, default_ftruncate_cbk, NULL,
                  fd, offset, xdata);
@@ -1119,8 +1414,9 @@ int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int xflags, dict_t * xdata)
+int32_t
+ec_gf_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+             dict_t *xdata)
 {
     ec_unlink(frame, this, -1, EC_MINIMUM_MIN, default_unlink_cbk, NULL, loc,
               xflags, xdata);
@@ -1128,9 +1424,10 @@ int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                     struct iovec * vector, int32_t count, off_t offset,
-                     uint32_t flags, struct iobref * iobref, dict_t * xdata)
+int32_t
+ec_gf_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
     ec_writev(frame, this, -1, EC_MINIMUM_MIN, default_writev_cbk, NULL, fd,
               vector, count, offset, flags, iobref, xdata);
@@ -1138,9 +1435,9 @@ int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      gf_xattrop_flags_t optype, dict_t * xattr,
-                      dict_t * xdata)
+int32_t
+ec_gf_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
     ec_xattrop(frame, this, -1, EC_MINIMUM_MIN, default_xattrop_cbk, NULL, loc,
                optype, xattr, xdata);
@@ -1148,79 +1445,94 @@ int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fxattrop(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       gf_xattrop_flags_t optype, dict_t * xattr,
-                       dict_t * xdata)
+int32_t
+ec_gf_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL,
-                fd, optype, xattr, xdata);
+    ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL, fd,
+                optype, xattr, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_zerofill(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       off_t offset, off_t len, dict_t * xdata)
+int32_t
+ec_gf_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
 {
     default_zerofill_failure_cbk(frame, ENOTSUP);
 
     return 0;
 }
 
-int32_t ec_gf_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   gf_seek_what_t what, dict_t *xdata)
+int32_t
+ec_gf_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
 {
-    ec_seek(frame, this, -1, EC_MINIMUM_ONE, default_seek_cbk, NULL, fd,
-            offset, what, xdata);
+    ec_seek(frame, this, -1, EC_MINIMUM_ONE, default_seek_cbk, NULL, fd, offset,
+            what, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_forget(xlator_t * this, inode_t * inode)
+int32_t
+ec_gf_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    ec_ipc(frame, this, -1, EC_MINIMUM_MIN, default_ipc_cbk, NULL, op, xdata);
+    return 0;
+}
+
+int32_t
+ec_gf_forget(xlator_t *this, inode_t *inode)
 {
     uint64_t value = 0;
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
 
-    if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0))
-    {
+    if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) {
         ctx = (ec_inode_t *)(uintptr_t)value;
+        /* We can only forget an inode if it has been unlocked, so the stripe
+         * cache should also be empty. */
+        GF_ASSERT(list_empty(&ctx->stripe_cache.lru));
         GF_FREE(ctx);
     }
 
     return 0;
 }
 
-void ec_gf_release_fd(xlator_t * this, fd_t * fd)
+void
+ec_gf_release_fd(xlator_t *this, fd_t *fd)
 {
     uint64_t value = 0;
-    ec_fd_t * ctx = NULL;
+    ec_fd_t *ctx = NULL;
 
-    if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0))
-    {
+    if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0)) {
         ctx = (ec_fd_t *)(uintptr_t)value;
         loc_wipe(&ctx->loc);
         GF_FREE(ctx);
     }
 }
 
-int32_t ec_gf_release(xlator_t * this, fd_t * fd)
+int32_t
+ec_gf_release(xlator_t *this, fd_t *fd)
 {
     ec_gf_release_fd(this, fd);
 
     return 0;
 }
 
-int32_t ec_gf_releasedir(xlator_t * this, fd_t * fd)
+int32_t
+ec_gf_releasedir(xlator_t *this, fd_t *fd)
 {
     ec_gf_release_fd(this, fd);
 
     return 0;
 }
 
-int32_t ec_dump_private(xlator_t *this)
+int32_t
+ec_dump_private(xlator_t *this)
 {
     ec_t *ec = NULL;
-    char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-    char  tmp[65];
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char tmp[65];
 
     GF_ASSERT(this);
 
@@ -1228,7 +1540,8 @@ int32_t ec_dump_private(xlator_t *this)
     GF_ASSERT(ec);
 
     snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-    gf_proc_dump_add_section(key_prefix);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("up", "%u", ec->up);
     gf_proc_dump_write("nodes", "%u", ec->nodes);
     gf_proc_dump_write("redundancy", "%u", ec->redundancy);
     gf_proc_dump_write("fragment_size", "%u", ec->fragment_size);
@@ -1236,143 +1549,325 @@ int32_t ec_dump_private(xlator_t *this)
     gf_proc_dump_write("childs_up", "%u", ec->xl_up_count);
     gf_proc_dump_write("childs_up_mask", "%s",
                        ec_bin(tmp, sizeof(tmp), ec->xl_up, ec->nodes));
+    if (ec->read_mask) {
+        gf_proc_dump_write("read-mask", "%s",
+                           ec_bin(tmp, sizeof(tmp), ec->read_mask, ec->nodes));
+    }
     gf_proc_dump_write("background-heals", "%d", ec->background_heals);
     gf_proc_dump_write("heal-wait-qlength", "%d", ec->heal_wait_qlen);
+    gf_proc_dump_write("self-heal-window-size", "%" PRIu32,
+                       ec->self_heal_window_size);
     gf_proc_dump_write("healers", "%d", ec->healers);
     gf_proc_dump_write("heal-waiters", "%d", ec->heal_waiters);
     gf_proc_dump_write("read-policy", "%s", ec_read_policies[ec->read_policy]);
+    gf_proc_dump_write("parallel-writes", "%d", ec->parallel_writes);
+    gf_proc_dump_write("quorum-count", "%u", ec->quorum_count);
+
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s.stats.stripe_cache",
+             this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    gf_proc_dump_write("hits", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.hits));
+    gf_proc_dump_write("misses", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.misses));
+    gf_proc_dump_write("updates", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.updates));
+    gf_proc_dump_write("invalidations", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.invals));
+    gf_proc_dump_write("evicts", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.evicts));
+    gf_proc_dump_write("allocations", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.allocs));
+    gf_proc_dump_write("errors", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.errors));
+    gf_proc_dump_write("heals-attempted", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.shd.attempted));
+    gf_proc_dump_write("heals-completed", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.shd.completed));
 
     return 0;
 }
 
-struct xlator_fops fops =
-{
-    .lookup       = ec_gf_lookup,
-    .stat         = ec_gf_stat,
-    .fstat        = ec_gf_fstat,
-    .truncate     = ec_gf_truncate,
-    .ftruncate    = ec_gf_ftruncate,
-    .access       = ec_gf_access,
-    .readlink     = ec_gf_readlink,
-    .mknod        = ec_gf_mknod,
-    .mkdir        = ec_gf_mkdir,
-    .unlink       = ec_gf_unlink,
-    .rmdir        = ec_gf_rmdir,
-    .symlink      = ec_gf_symlink,
-    .rename       = ec_gf_rename,
-    .link         = ec_gf_link,
-    .create       = ec_gf_create,
-    .open         = ec_gf_open,
-    .readv        = ec_gf_readv,
-    .writev       = ec_gf_writev,
-    .flush        = ec_gf_flush,
-    .fsync        = ec_gf_fsync,
-    .opendir      = ec_gf_opendir,
-    .readdir      = ec_gf_readdir,
-    .readdirp     = ec_gf_readdirp,
-    .fsyncdir     = ec_gf_fsyncdir,
-    .statfs       = ec_gf_statfs,
-    .setxattr     = ec_gf_setxattr,
-    .getxattr     = ec_gf_getxattr,
-    .fsetxattr    = ec_gf_fsetxattr,
-    .fgetxattr    = ec_gf_fgetxattr,
-    .removexattr  = ec_gf_removexattr,
-    .fremovexattr = ec_gf_fremovexattr,
-    .lk           = ec_gf_lk,
-    .inodelk      = ec_gf_inodelk,
-    .finodelk     = ec_gf_finodelk,
-    .entrylk      = ec_gf_entrylk,
-    .fentrylk     = ec_gf_fentrylk,
-    .xattrop      = ec_gf_xattrop,
-    .fxattrop     = ec_gf_fxattrop,
-    .setattr      = ec_gf_setattr,
-    .fsetattr     = ec_gf_fsetattr,
-    .fallocate    = ec_gf_fallocate,
-    .discard      = ec_gf_discard,
-    .zerofill     = ec_gf_zerofill,
-    .seek         = ec_gf_seek
-};
-
-struct xlator_cbks cbks =
-{
-    .forget            = ec_gf_forget,
-    .release           = ec_gf_release,
-    .releasedir        = ec_gf_releasedir
-};
-
-struct xlator_dumpops dumpops = {
-    .priv = ec_dump_private
-};
-
-struct volume_options options[] =
-{
+struct xlator_fops fops = {.lookup = ec_gf_lookup,
+                           .stat = ec_gf_stat,
+                           .fstat = ec_gf_fstat,
+                           .truncate = ec_gf_truncate,
+                           .ftruncate = ec_gf_ftruncate,
+                           .access = ec_gf_access,
+                           .readlink = ec_gf_readlink,
+                           .mknod = ec_gf_mknod,
+                           .mkdir = ec_gf_mkdir,
+                           .unlink = ec_gf_unlink,
+                           .rmdir = ec_gf_rmdir,
+                           .symlink = ec_gf_symlink,
+                           .rename = ec_gf_rename,
+                           .link = ec_gf_link,
+                           .create = ec_gf_create,
+                           .open = ec_gf_open,
+                           .readv = ec_gf_readv,
+                           .writev = ec_gf_writev,
+                           .flush = ec_gf_flush,
+                           .fsync = ec_gf_fsync,
+                           .opendir = ec_gf_opendir,
+                           .readdir = ec_gf_readdir,
+                           .readdirp = ec_gf_readdirp,
+                           .fsyncdir = ec_gf_fsyncdir,
+                           .statfs = ec_gf_statfs,
+                           .setxattr = ec_gf_setxattr,
+                           .getxattr = ec_gf_getxattr,
+                           .fsetxattr = ec_gf_fsetxattr,
+                           .fgetxattr = ec_gf_fgetxattr,
+                           .removexattr = ec_gf_removexattr,
+                           .fremovexattr = ec_gf_fremovexattr,
+                           .lk = ec_gf_lk,
+                           .inodelk = ec_gf_inodelk,
+                           .finodelk = ec_gf_finodelk,
+                           .entrylk = ec_gf_entrylk,
+                           .fentrylk = ec_gf_fentrylk,
+                           .xattrop = ec_gf_xattrop,
+                           .fxattrop = ec_gf_fxattrop,
+                           .setattr = ec_gf_setattr,
+                           .fsetattr = ec_gf_fsetattr,
+                           .fallocate = ec_gf_fallocate,
+                           .discard = ec_gf_discard,
+                           .zerofill = ec_gf_zerofill,
+                           .seek = ec_gf_seek,
+                           .ipc = ec_gf_ipc};
+
+struct xlator_cbks cbks = {.forget = ec_gf_forget,
+                           .release = ec_gf_release,
+                           .releasedir = ec_gf_releasedir};
+
+struct xlator_dumpops dumpops = {.priv = ec_dump_private};
+
+struct volume_options options[] = {
+    {.key = {"redundancy"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "{{ volume.redundancy }}",
+     .description = "Maximum number of bricks that can fail "
+                    "simultaneously without losing data."},
     {
-        .key = { "redundancy" },
-        .type = GF_OPTION_TYPE_INT,
-        .description = "Maximum number of bricks that can fail "
-                       "simultaneously without losing data."
-    },
-    {
-        .key = { "self-heal-daemon" },
+        .key = {"self-heal-daemon"},
         .type = GF_OPTION_TYPE_BOOL,
         .description = "self-heal daemon enable/disable",
         .default_value = "enable",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"disperse"},
     },
-    { .key = {"iam-self-heal-daemon"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "off",
-      .description = "This option differentiates if the disperse "
-                     "translator is running as part of self-heal-daemon "
-                     "or not."
+    {.key = {"iam-self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the disperse "
+                    "translator is running as part of self-heal-daemon "
+                    "or not."},
+    {.key = {"eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_7_10},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Enable/Disable eager lock for regular files on a "
+                    "disperse volume. If a fop takes a lock and completes "
+                    "its operation, it waits for next 1 second before "
+                    "releasing the lock, to see if the lock can be reused "
+                    "for next fop from the same client. If ec finds any lock "
+                    "contention within 1 second it releases the lock "
+                    "immediately before time expires. This improves the "
+                    "performance of file operations. However, as it takes "
+                    "lock on first brick, for few operations like read, "
+                    "discovery of lock contention might take long time and "
+                    "can actually degrade the performance. If eager lock is "
+                    "disabled, lock will be released as soon as fop "
+                    "completes."},
+    {.key = {"other-eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_13_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "It's equivalent to the eager-lock option but for non "
+                    "regular files."},
+    {.key = {"eager-lock-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 60,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse", "locks", "timeout"},
+     .description = "Maximum time (in seconds) that a lock on an inode is "
+                    "kept held if no new operations on the inode are "
+                    "received."},
+    {.key = {"other-eager-lock-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 60,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse", "locks", "timeout"},
+     .description = "It's equivalent to eager-lock-timeout option but for "
+                    "non regular files."},
+    {
+        .key = {"background-heals"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0, /*Disabling background heals*/
+        .max = 256,
+        .default_value = "8",
+        .op_version = {GD_OP_VERSION_3_7_3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description = "This option can be used to control number of parallel"
+                       " heals",
     },
-    { .key = {"eager-lock"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "on",
-      .description = "Enable/Disable eager lock for disperse volume. "
-                     "If a fop takes a lock and completes its operation, "
-                     "it waits for next 1 second before releasing the lock, "
-                     "to see if the lock can be reused for next fop from "
-                     "the same client. If ec finds any lock contention within "
-                     "1 second it releases the lock immediately before time "
-                     "expires. This improves the performance of file operations."
-                     "However, as it takes lock on first brick, for few operations "
-                     "like read, discovery of lock contention might take long time "
-                     "and can actually degrade the performance. "
-                     "If eager lock is disabled, lock will be released as soon as fop "
-                     "completes. "
+    {
+        .key = {"heal-wait-qlength"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max =
+            65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/
+        .default_value = "128",
+        .op_version = {GD_OP_VERSION_3_7_3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description = "This option can be used to control number of heals"
+                       " that can wait",
     },
-    { .key = {"background-heals"},
-      .type = GF_OPTION_TYPE_INT,
-      .min = 0,/*Disabling background heals*/
-      .max = 256,
-      .default_value = "8",
-      .description = "This option can be used to control number of parallel"
-                     " heals",
+    {.key = {"heal-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 60,
+     .max = INT_MAX,
+     .default_value = "600",
+     .op_version = {GD_OP_VERSION_3_7_3},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"disperse"},
+     .description = "time interval for checking the need to self-heal "
+                    "in self-heal-daemon"},
+    {
+        .key = {"read-policy"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"round-robin", "gfid-hash"},
+        .default_value = "gfid-hash",
+        .op_version = {GD_OP_VERSION_3_7_6},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description =
+            "inode-read fops happen only on 'k' number of bricks in"
+            " n=k+m disperse subvolume. 'round-robin' selects the read"
+            " subvolume using round-robin algo. 'gfid-hash' selects read"
+            " subvolume based on hash of the gfid of that file/directory.",
     },
-    { .key = {"heal-wait-qlength"},
-      .type = GF_OPTION_TYPE_INT,
-      .min = 0,
-      .max = 65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/
-      .default_value = "128",
-      .description = "This option can be used to control number of heals"
-                     " that can wait",
+    {.key = {"shd-max-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 64,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Maximum number of parallel heals SHD can do per local "
+                    "brick.  This can substantially lower heal times, "
+                    "but can also crush your bricks if you don't have "
+                    "the storage hardware to support this."},
+    {.key = {"shd-wait-qlength"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 65536,
+     .default_value = "1024",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "This option can be used to control number of heals"
+                    " that can wait in SHD per subvolume"},
+    {.key = {"cpu-extensions"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"none", "auto", "x64", "sse", "avx"},
+     .default_value = "auto",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "force the cpu extensions to be used to accelerate the "
+                    "galois field computations."},
+    {.key = {"self-heal-window-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Maximum number blocks(128KB) per file for which "
+                    "self-heal process would be applied simultaneously."},
+    {.key = {"optimistic-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_10_1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+     .tags = {"disperse"},
+     .description = "Set/Unset dirty flag for every update fop at the start"
+                    "of the fop. If OFF, this option impacts performance of"
+                    "entry  operations or metadata operations as it will"
+                    "set dirty flag at the start and unset it at the end of"
+                    "ALL update fop. If ON and all the bricks are good,"
+                    "dirty flag will be set at the start only for file fops"
+                    "For metadata and entry fops dirty flag will not be set"
+                    "at the start, if all the bricks are good. This does"
+                    "not impact performance for metadata operations and"
+                    "entry operation but has a very small window to miss"
+                    "marking entry as dirty in case it is required to be"
+                    "healed"},
+    {.key = {"parallel-writes"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This controls if writes can be wound in parallel as long"
+                    "as it doesn't modify same stripes"},
+    {.key = {"stripe-cache"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0, /*Disabling stripe_cache*/
+     .max = EC_STRIPE_CACHE_MAX_SIZE,
+     .default_value = "4",
+     .description = "This option will keep the last stripe of write fop"
+                    "in memory. If next write falls in this stripe, we need"
+                    "not to read it again from backend and we can save READ"
+                    "fop going over the network. This will improve performance,"
+                    "specially for sequential writes. However, this will also"
+                    "lead to extra memory consumption, maximum "
+                    "(cache size * stripe size) Bytes per open file."},
+    {
+        .key = {"quorum-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "0",
+        .description =
+            "This option can be used to define how many successes on"
+            "the bricks constitute a success to the application. This"
+            " count should be in the range"
+            "[disperse-data-count,  disperse-count] (inclusive)",
     },
-    { .key  = {"heal-timeout"},
-      .type = GF_OPTION_TYPE_INT,
-      .min  = 60,
-      .max  = INT_MAX,
-      .default_value = "600",
-      .description = "time interval for checking the need to self-heal "
-                     "in self-heal-daemon"
+    {
+        .key = {"ec-read-mask"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = NULL,
+        .description = "This option can be used to choose which bricks can be"
+                       " used for reading data/metadata of a file/directory",
     },
-    { .key = {"read-policy" },
-      .type = GF_OPTION_TYPE_STR,
-      .value = {"round-robin", "gfid-hash"},
-      .default_value = "round-robin",
-      .description = "inode-read fops happen only on 'k' number of bricks in"
-              " n=k+m disperse subvolume. 'round-robin' selects the read"
-              " subvolume using round-robin algo. 'gfid-hash' selects read"
-              " subvolume based on hash of the gfid of that file/directory.",
+    {
+        .key = {NULL},
     },
-    { }
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "disperse",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h
index 49af5c2daf2..6f6de6d5981 100644
--- a/xlators/cluster/ec/src/ec.h
+++ b/xlators/cluster/ec/src/ec.h
@@ -11,64 +11,24 @@
 #ifndef __EC_H__
 #define __EC_H__
 
-#include "xlator.h"
-#include "timer.h"
-#include "ec-heald.h"
-#include "libxlator.h"
-
-#define EC_XATTR_PREFIX  "trusted.ec."
-#define EC_XATTR_CONFIG  EC_XATTR_PREFIX"config"
-#define EC_XATTR_SIZE    EC_XATTR_PREFIX"size"
-#define EC_XATTR_VERSION EC_XATTR_PREFIX"version"
-#define EC_XATTR_HEAL    EC_XATTR_PREFIX"heal"
-#define EC_XATTR_DIRTY   EC_XATTR_PREFIX"dirty"
-
+#include "ec-method.h"
+
+#define EC_XATTR_PREFIX "trusted.ec."
+#define EC_XATTR_CONFIG EC_XATTR_PREFIX "config"
+#define EC_XATTR_SIZE EC_XATTR_PREFIX "size"
+#define EC_XATTR_VERSION EC_XATTR_PREFIX "version"
+#define EC_XATTR_HEAL EC_XATTR_PREFIX "heal"
+#define EC_XATTR_HEAL_NEW EC_XATTR_PREFIX "heal-new"
+#define EC_XATTR_DIRTY EC_XATTR_PREFIX "dirty"
+#define EC_STRIPE_CACHE_MAX_SIZE 10
 #define EC_VERSION_SIZE 2
-#define EC_SHD_INODE_LRU_LIMIT          10
-
-typedef enum {
-        EC_ROUND_ROBIN,
-        EC_GFID_HASH,
-        EC_READ_POLICY_MAX
-} ec_read_policy_t;
-
-struct _ec
-{
-    xlator_t *        xl;
-    int32_t           healers;
-    int32_t           heal_waiters;
-    int32_t           nodes;
-    int32_t           bits_for_nodes;
-    int32_t           fragments;
-    int32_t           redundancy;
-    uint32_t          fragment_size;
-    uint32_t          stripe_size;
-    int32_t           up;
-    uint32_t          idx;
-    uint32_t          xl_up_count;
-    uintptr_t         xl_up;
-    uint32_t          xl_notify_count;
-    uintptr_t         xl_notify;
-    uintptr_t         node_mask;
-    xlator_t **       xl_list;
-    gf_lock_t         lock;
-    gf_timer_t *      timer;
-    gf_boolean_t      shutdown;
-    gf_boolean_t      eager_lock;
-    uint32_t          background_heals;
-    uint32_t          heal_wait_qlen;
-    struct list_head  pending_fops;
-    struct list_head  heal_waiting;
-    struct list_head  healing;
-    struct mem_pool * fop_pool;
-    struct mem_pool * cbk_pool;
-    struct mem_pool * lock_pool;
-    ec_self_heald_t   shd;
-    char              vol_uuid[UUID_SIZE + 1];
-    dict_t           *leaf_to_subvolid;
-    ec_read_policy_t  read_policy;
-};
-
-void ec_pending_fops_completed(ec_t *ec);
+#define EC_SHD_INODE_LRU_LIMIT 10
+
+#define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS
+/* The maximum number of nodes is derived from the maximum allowed fragments
+ * using the rule that redundancy cannot be equal or greater than the number
+ * of fragments.
+ */
+#define EC_MAX_NODES min(EC_MAX_FRAGMENTS * 2 - 1, EC_METHOD_MAX_NODES)
 
 #endif /* __EC_H__ */
diff --git a/xlators/cluster/ha/src/Makefile.am b/xlators/cluster/ha/src/Makefile.am
deleted file mode 100644
index 740a6b840d7..00000000000
--- a/xlators/cluster/ha/src/Makefile.am
+++ /dev/null
@@ -1,17 +0,0 @@
-xlator_LTLIBRARIES = ha.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
-
-ha_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-
-ha_la_SOURCES = ha-helpers.c ha.c
-ha_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = ha.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/ha/src/ha-helpers.c b/xlators/cluster/ha/src/ha-helpers.c
deleted file mode 100644
index 19be1ed2773..00000000000
--- a/xlators/cluster/ha/src/ha-helpers.c
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "dict.h"
-#include "compat-errno.h"
-#include "ha.h"
-
-#define HA_TRANSPORT_NOTCONN(_ret, _errno, _fd) \
-	((_ret == -1) && (_fd ? (_errno == EBADFD):(_errno == ENOTCONN)))
-
-int ha_alloc_init_fd (call_frame_t *frame, fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	int i = -1;
-	ha_private_t *pvt = NULL;
-	int child_count = 0;
-	int ret = -1;
-	hafd_t *hafdp = NULL;
-	xlator_t *this = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	this = frame->this;
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	if (local == NULL) {
-		ret = fd_ctx_get (fd, this, &tmp_hafdp);
-		if (ret < 0) {
-			goto out;
-		}
-		hafdp = (hafd_t *)(long)tmp_hafdp;
-                local = frame->local = GF_CALLOC (1, sizeof (*local), 
-                                                  gf_ha_mt_ha_local_t);
-		if (local == NULL) {
-			ret = -ENOMEM;
-			goto out;
-		}
-                local->state = GF_CALLOC (1, child_count, 
-                                          gf_ha_mt_child_count);
-		if (local->state == NULL) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		/* take care of the preferred subvolume */
-		if (pvt->pref_subvol == -1)
-			local->active = hafdp->active;
-		else
-			local->active = pvt->pref_subvol;
-
-		LOCK (&hafdp->lock);
-		memcpy (local->state, hafdp->fdstate, child_count);
-		UNLOCK (&hafdp->lock);
-
-		/* in case the preferred subvolume is down */
-		if ((local->active != -1) && (local->state[local->active] == 0))
-			local->active = -1;
-
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i]) {
-				if (local->active == -1)
-					local->active = i;
-				local->tries++;
-			}
-		}
-		if (local->active == -1) {
-			ret = -ENOTCONN;
-			goto out;
-		}
-		local->fd = fd_ref (fd);
-	}
-	ret = 0;
-out:
-	return ret;
-}
-
-int ha_handle_cbk (call_frame_t *frame, void *cookie, int op_ret, int op_errno) 
-{
-	xlator_t *xl = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int prev_child = -1;
-	hafd_t *hafdp = NULL;
-	int ret = -1;
-	call_stub_t *stub = NULL;
-	ha_local_t *local = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	xl = frame->this;
-	pvt = xl->private;
-	children = pvt->children;
-	prev_child = (long) cookie;
-	local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (xl->name, GF_LOG_ERROR ,"(child=%s) (op_ret=%d op_errno=%s)",
-			children[prev_child]->name, op_ret, strerror (op_errno));
-	}
-
-	if (HA_TRANSPORT_NOTCONN (op_ret, op_errno, (local->fd))) {
-		ret = 0;
-		if (local->fd) {
-			ret = fd_ctx_get (local->fd, xl, &tmp_hafdp);
-		}
-		hafdp = (hafd_t *)(long)tmp_hafdp;		
-		if (ret == 0) {
-			if (local->fd) {
-				LOCK(&hafdp->lock);
-				hafdp->fdstate[prev_child] = 0;
-				UNLOCK(&hafdp->lock);
-			}
-			local->tries--;
-			if (local->tries != 0) {
-				while (1) {
-					local->active = (local->active + 1) % pvt->child_count;
-					if (local->state[local->active])
-						break;
-				}
-				stub = local->stub;
-				local->stub = NULL;
-				call_resume (stub);
-				return -1;
-			}
-		}
-	}
-	if (local->stub) {
-		call_stub_destroy (local->stub);
-                local->stub = NULL;
-        }
-
-	if (local->fd) {
-		GF_FREE (local->state);
-                local->state = NULL;
-
-		fd_unref (local->fd);
-                local->fd = NULL;
-	}
-	return 0;
-}
-
-int ha_alloc_init_inode (call_frame_t *frame, inode_t *inode)
-{
-	int i = -1;
-	ha_private_t *pvt = NULL;
-	xlator_t *xl = NULL;
-	int ret = -1;
-	ha_local_t *local = NULL;
-	uint64_t tmp_state = 0;
-
-	xl = frame->this;
-	pvt = xl->private;
-	local = frame->local;
-
-	if (local == NULL) {
-                local = frame->local = GF_CALLOC (1, sizeof (*local), 
-                                                  gf_ha_mt_ha_local_t);
-		if (local == NULL) {
-			ret = -ENOMEM;
-			goto out;
-		}
-		local->active = pvt->pref_subvol;
-		ret = inode_ctx_get (inode, xl, &tmp_state);
-		if (ret < 0) {
-			goto out;
-		}
-		local->state = (char *)(long)tmp_state;
-		if (local->active != -1 && local->state[local->active] == 0)
-			local->active = -1;
-		for (i = 0; i < pvt->child_count; i++) {
-			if (local->state[i]) {
-				if (local->active == -1)
-					local->active = i;
-				local->tries++;
-			}
-		}
-		if (local->active == -1) {
-			ret = -ENOTCONN;
-			goto out;
-		}
-	}
-	ret = 0;
-out:
-	return ret;
-}
diff --git a/xlators/cluster/ha/src/ha-mem-types.h b/xlators/cluster/ha/src/ha-mem-types.h
deleted file mode 100644
index e5e97d237dc..00000000000
--- a/xlators/cluster/ha/src/ha-mem-types.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __HA_MEM_TYPES_H__
-#define __HA_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_ha_mem_types_ {
-        gf_ha_mt_ha_local_t = gf_common_mt_end + 1,
-        gf_ha_mt_hafd_t,
-        gf_ha_mt_char,
-        gf_ha_mt_child_count,
-        gf_ha_mt_xlator_t,
-        gf_ha_mt_ha_private_t,
-        gf_ha_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/ha/src/ha.c b/xlators/cluster/ha/src/ha.c
deleted file mode 100644
index 6071eab55ee..00000000000
--- a/xlators/cluster/ha/src/ha.c
+++ /dev/null
@@ -1,4008 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-/* generate errors randomly, code is simple now, better alogorithm
- * can be written to decide what error to be returned and when
- */
-
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "dict.h"
-#include "compat-errno.h"
-#include "ha.h"
-
-/*
- * TODO:
- * - dbench fails if ha over server side afr
- * - lock calls - lock on all subvols.
- * - support preferred-subvolume option. code already there.
- * - do not alloc the call-stub in case only one subvol is up.
- */
-
-void
-ha_local_wipe (ha_local_t *local)
-{
-        if (local->stub) {
-                call_stub_destroy (local->stub);
-                local->stub = NULL;
-        }
-
-        if (local->state) {
-                GF_FREE (local->state);
-                local->state = NULL;
-        }
-
-        if (local->dict) {
-                dict_unref (local->dict);
-                local->dict = NULL;
-        }
-
-        loc_wipe (&local->loc);
-
-        if (local->fd) {
-                fd_unref (local->fd);
-                local->fd = NULL;
-        }
-
-        if (local->inode) {
-                inode_unref (local->inode);
-                local->inode = NULL;
-        }
-
-        GF_FREE (local);
-        return;
-}
-
-
-int
-ha_forget (xlator_t *this,
-	   inode_t *inode)
-{
-	uint64_t stateino = 0;
-	char *state = NULL;
-	if (!inode_ctx_del (inode, this, &stateino)) {
-		state =  ((char *)(long)stateino);
-		GF_FREE (state);
-	}
-
-	return 0;
-
-}
-
-int32_t 
-ha_lookup_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       inode_t *inode,
-	       struct iatt *buf,
-               dict_t *dict,
-               struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0, callcnt = 0;
-	char *state = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_state = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++) {
-		if (pvt->children[i] == prev_frame->this)
-			break;
-	}
-	if ((op_ret == -1) && (op_errno != ENOENT)) {
-		gf_log (this->name, GF_LOG_ERROR, "(child=%s) (op_ret=%d op_errno=%s)", 
-			  children[i]->name, op_ret, strerror (op_errno));
-	}
-	inode_ctx_get (local->inode, this, &tmp_state);
-	state = (char *)(long)tmp_state;
-
-	LOCK (&frame->lock);
-	if (local->revalidate == 1) {
-		if ((!op_ret) != state[i]) {
-			local->revalidate_error = 1;
-			gf_log (this->name, GF_LOG_DEBUG, "revalidate error on %s", 
-				pvt->children[i]->name);
-		}
-	} else {
-		if (op_ret == 0) {
-			state[i] = 1;
-		}
-	}
-	if (local->op_ret == -1 && op_ret == 0) {
-		local->op_ret = 0;
-		local->buf = *buf;
-                local->postparent = *postparent;
-		if (dict)
-			local->dict = dict_ref (dict);
-	}
-	if (op_ret == -1 && op_ret != ENOTCONN)
-		local->op_errno = op_errno;
-	callcnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		dict_t *ctx = local->dict;
-		inode_t *inode = local->inode;
-		if (local->revalidate_error == 1) {
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			gf_log (this->name, GF_LOG_DEBUG, "revalidate error, returning EIO");
-		}
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      inode,
-			      &local->buf,
-			      ctx,
-                              &local->postparent);
-		if (inode)
-			inode_unref (inode);
-		if (ctx)
-			dict_unref (ctx);
-	}
-	return 0;
-}
-
-int32_t
-ha_lookup (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   dict_t *xattr_req)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *state = NULL;
-	xlator_t **children = NULL;
-	int ret = -1;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	children = pvt->children;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-	child_count = pvt->child_count;
-	local->inode = inode_ref (loc->inode);
-
-	ret = inode_ctx_get (loc->inode, this, NULL);
-	if (ret) {
-		state = GF_CALLOC (1, child_count, gf_ha_mt_child_count);
-                if (state == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-		inode_ctx_put (loc->inode, this, (uint64_t)(long)state);
-	} else
-		local->revalidate = 1;
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->call_count = child_count;
-
-	for (i = 0; i < child_count; i++) {
-		STACK_WIND (frame,
-			    ha_lookup_cbk,
-			    children[i],
-			    children[i]->fops->lookup,
-			    loc,
-			    xattr_req);
-	}
-	return 0;
-
-unwind:
-        local = frame->local;
-        frame->local = NULL;
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_stat_cbk (call_frame_t *frame,
-	     void *cookie,
-	     xlator_t *this,
-	     int32_t op_ret,
-	     int32_t op_errno,
-	     struct iatt *buf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
-
-int32_t
-ha_stat (call_frame_t *frame,
-	 xlator_t *this,
-	 loc_t *loc)
-{
-	ha_local_t *local = NULL;
-	int op_errno = ENOTCONN;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_stat_stub (frame, ha_stat, loc);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_stat_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->stat,
-			   loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;	
-}
-
-int32_t
-ha_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                struct iatt *statpost)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
-	}
-	return 0;
-}
-
-
-int32_t
-ha_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
-            int32_t valid)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_setattr_stub (frame, ha_setattr, loc, stbuf, valid);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_setattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->setattr,
-			   loc, stbuf, valid);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-
-int32_t
-ha_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-             int32_t valid)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fsetattr_stub (frame, ha_fsetattr, fd, stbuf, valid);
-
-	STACK_WIND_COOKIE (frame,	      
-			   ha_setattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fsetattr,
-			   fd, stbuf, valid);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-
-int32_t
-ha_truncate_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      prebuf,
-                              postbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_truncate (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     off_t offset)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_truncate_stub (frame, ha_truncate, loc, offset);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_truncate_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->truncate,
-			   loc,
-			   offset);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
- int32_t
-ha_ftruncate_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      prebuf, postbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_ftruncate (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      off_t offset)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_ftruncate_stub (frame, ha_ftruncate, fd, offset);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_ftruncate_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->ftruncate,
-			   fd,
-			   offset);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_access_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_access (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   int32_t mask)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_access_stub (frame, ha_access, loc, mask);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_access_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->access,
-			   loc,
-			   mask);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
- int32_t
-ha_readlink_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 const char *path,
-                 struct iatt *sbuf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      path,
-                              sbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_readlink (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     size_t size)
-{
-	ha_local_t *local = frame->local;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_readlink_stub (frame, ha_readlink, loc, size);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_readlink_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->readlink,
-			   loc,
-			   size);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-int
-ha_mknod_lookup_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"(path=%s) (op_ret=%d op_errno=%d)", 
-			local->stub->args.mknod.loc.path, op_ret, op_errno);
-	}
-	ret = inode_ctx_get (local->stub->args.mknod.loc.inode, 
-			     this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"unwind(-1), inode_ctx_get() error");
-		/* It is difficult to handle this error at this stage
-		 * as we still expect more cbks, we can't return as
-		 * of now
-		 */
-	} else if (op_ret == 0) {
-		stateino[i] = 1;
-	}
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.mknod.loc.inode,
-			      &local->buf, &local->preparent,
-                              &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_mknod_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      inode_t *inode,
-              struct iatt *buf,
-              struct iatt *preparent,
-              struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.mknod.loc.path, op_ret, op_errno);
-	}
-
-	ret = inode_ctx_get (local->stub->args.mknod.loc.inode, 
-			     this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "inode_ctx_get() error");
-		/* FIXME: handle the case */
-	}
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              local->stub->args.mknod.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_mknod_cbk,
-			    children[i],
-			    children[i]->fops->mknod,
-			    &local->stub->args.mknod.loc,
-			    local->stub->args.mknod.mode,
-			    local->stub->args.mknod.rdev);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_mknod_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.mknod.loc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_mknod (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  mode_t mode,
-	  dev_t rdev)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->stub = fop_mknod_stub (frame, ha_mknod, loc, mode, rdev);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!stateino) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1) 
-				local->active = i;
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_mknod_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->mknod,
-		    loc, mode, rdev);
-	return 0;
-
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-        return 0;
-}
-
-
-int
-ha_mkdir_lookup_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.mkdir.loc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.mkdir.loc.inode, 
-		       this, &tmp_stateino);  
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0)
-		stateino[i] = 1;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.mkdir.loc.inode, &local->buf,
-			      &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_mkdir_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      inode_t *inode,
-              struct iatt *buf,
-              struct iatt *preparent,
-              struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-	
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.mkdir.loc.path, op_ret, op_errno);
-	}
-
-	inode_ctx_get (local->stub->args.mkdir.loc.inode, 
-		       this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              local->stub->args.mkdir.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_mkdir_cbk,
-			    children[i],
-			    children[i]->fops->mkdir,
-			    &local->stub->args.mkdir.loc,
-			    local->stub->args.mkdir.mode);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_mkdir_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.mkdir.loc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_mkdir (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  mode_t mode)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!frame->local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->stub = fop_mkdir_stub (frame, ha_mkdir, loc, mode);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!stateino) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1)
-				local->active = i;
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_mkdir_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->mkdir,
-		    loc, mode);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_unlink_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-               struct iatt *preparent,
-               struct iatt *postparent)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	}
-	return 0;
-}
-
-int32_t
-ha_unlink (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_unlink_stub (frame, ha_unlink, loc);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_unlink_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->unlink,
-			   loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
- int32_t
-ha_rmdir_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-              struct iatt *preparent,
-              struct iatt *postparent)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-                              preparent,
-                              postparent);
-	}
-	return 0;
-}
-
-int32_t
-ha_rmdir (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc)
-{
-	ha_local_t *local = frame->local;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_rmdir_stub (frame, ha_rmdir, loc);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_rmdir_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->rmdir,
-			   loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-
-int
-ha_symlink_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-                       dict_t *dict,
-                       struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.symlink.loc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.symlink.loc.inode,
-		       this, &tmp_stateino);  
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0)
-		stateino[i] = 1;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.symlink.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_symlink_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.symlink.loc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.symlink.loc.inode, 
-		       this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->stub->args.symlink.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_symlink_cbk,
-			    children[i],
-			    children[i]->fops->symlink,
-			    local->stub->args.symlink.linkname,
-			    &local->stub->args.symlink.loc);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_symlink_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.symlink.loc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_symlink (call_frame_t *frame,
-	    xlator_t *this,
-	    const char *linkname,
-	    loc_t *loc)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->stub = fop_symlink_stub (frame, ha_symlink, linkname, loc);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!stateino) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1) {
-				local->active = i;
-			}
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_symlink_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->symlink,
-		    linkname, loc);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_rename_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct iatt *buf,
-               struct iatt *preoldparent,
-               struct iatt *postoldparent,
-               struct iatt *prenewparent,
-               struct iatt *postnewparent)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, buf, preoldparent,
-                              postoldparent, prenewparent, postnewparent);
-	}
-	return 0;
-}
-
-int32_t
-ha_rename (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *oldloc,
-	   loc_t *newloc)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, oldloc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_rename_stub (frame, ha_rename, oldloc, newloc);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_rename_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->rename,
-			   oldloc, newloc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-	return 0;
-}
-
-int
-ha_link_lookup_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct iatt *buf,
-                    dict_t *dict,
-                    struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.link.newloc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.link.newloc.inode, 
-		       this, &tmp_stateino);  
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0)
-		stateino[i] = 1;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.link.oldloc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_link_cbk (call_frame_t *frame,
-	     void *cookie,
-	     xlator_t *this,
-	     int32_t op_ret,
-	     int32_t op_errno,
-	     inode_t *inode,
-             struct iatt *buf,
-             struct iatt *preparent,
-             struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.link.newloc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.link.newloc.inode, 
-		       this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              local->stub->args.link.oldloc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_link_cbk,
-			    children[i],
-			    children[i]->fops->link,
-			    &local->stub->args.link.oldloc,
-			    &local->stub->args.link.newloc);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_link_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.link.newloc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_link (call_frame_t *frame,
-	 xlator_t *this,
-	 loc_t *oldloc,
-	 loc_t *newloc)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-	int32_t ret = 0, op_errno = 0;
-	uint64_t tmp_stateino = 0;
-
-	ret = inode_ctx_get (newloc->inode, this, &tmp_stateino);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_ptr_error()");
-	}
-	stateino = (char *)(long)tmp_stateino;
-
-	if (stateino == NULL) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"newloc->inode's ctx is NULL, returning EINVAL");
-		STACK_UNWIND (frame, -1, EINVAL, oldloc->inode, NULL, NULL,
-                              NULL);
-		return 0;
-	}
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!frame->local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->stub = fop_link_stub (frame, ha_link, oldloc, newloc);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1)
-				local->active = i;
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_link_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->link,
-		    oldloc,
-		    newloc);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-ha_create_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       fd_t *fd,
-	       inode_t *inode,
-	       struct iatt *buf,
-               struct iatt *preparent,
-               struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int i, child_count = 0, cnt = 0, ret = 0;
-	char *stateino = NULL;
-	hafd_t *hafdp = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-	uint64_t tmp_hafdp = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	ret = inode_ctx_get (local->stub->args.create.loc.inode, 
-			     this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_to_ptr() error");
-		/* FIXME: handle */
-	}
-	ret = fd_ctx_get (local->stub->args.create.fd, this, &tmp_hafdp);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_to_ptr() error");
-		/* FIXME: handle */
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	for (i = 0; i < child_count; i++) {
-		if (prev_frame->this == children[i])
-			break;
-	}
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.create.loc.path, op_ret, op_errno);
-	}
-	if (op_ret != -1) {
-		stateino[i] = 1;
-		hafdp->fdstate[i] = 1;
-		if (local->op_ret == -1) {
-			local->op_ret = 0;
-			local->buf = *buf;
-                        local->preparent = *preparent;
-                        local->postparent = *postparent;
-			local->first_success = 1;
-		}
-		local->stub->args.create.flags &= (~O_EXCL);
-	}
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		char *state = local->state;
-		call_stub_t *stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      stub->args.create.fd,
-			      stub->args.create.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		GF_FREE (state);
-		call_stub_destroy (stub);
-		return 0;
-	}
-	local->active = i;
-	cnt = local->call_count;
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_create_cbk,
-				    children[i],
-				    children[i]->fops->create,
-				    &local->stub->args.create.loc,
-				    local->stub->args.create.flags,
-				    local->stub->args.create.mode,
-				    local->stub->args.create.fd);
-			if ((local->first_success == 0) || (cnt == 0))
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_create (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   int32_t flags,
-	   mode_t mode, fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int i, child_count = 0;
-	char *stateino = NULL;
-	xlator_t **children = NULL;
-	hafd_t *hafdp = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	children = pvt->children;
-
-	if (local == NULL) {
-                frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                                  gf_ha_mt_ha_local_t);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		local->stub = fop_create_stub (frame, ha_create, loc, flags, mode, fd);
-                if (!local->stub) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-                local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-                if (!local->state) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		local->active = -1;
-		local->op_ret = -1;
-		local->op_errno = ENOTCONN;
-		memcpy (local->state, pvt->state, child_count);
-
-		for (i = 0; i < pvt->child_count; i++) {
-			if (local->state[i]) {
-				local->call_count++;
-				if (local->active == -1)
-					local->active = i;
-			}
-		}
-		/* FIXME handle active -1 */
-		stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-                if (!stateino) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
-                if (!hafdp) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
-                if (!hafdp->fdstate) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		hafdp->path = gf_strdup(loc->path);
-                if (!hafdp->path) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		LOCK_INIT (&hafdp->lock);
-		fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
-		inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-	}
-
-	STACK_WIND (frame,
-		    ha_create_cbk,
-		    children[local->active],
-		    children[local->active]->fops->create,
-		    loc, flags, mode, fd);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-
-        if (stateino) {
-                GF_FREE (stateino);
-                stateino = NULL;
-        }
-
-        if (hafdp) {
-                GF_FREE (hafdp->fdstate);
-
-                GF_FREE (hafdp->path);
-
-                GF_FREE (hafdp);
-        }
-
-        return 0;
-}
-
- int32_t
-ha_open_cbk (call_frame_t *frame,
-	     void *cookie,
-	     xlator_t *this,
-	     int32_t op_ret,
-	     int32_t op_errno,
-	     fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0, child_count = 0, callcnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-
-	ret = fd_ctx_get (local->fd, this, &tmp_hafdp);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_ptr_error()");
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	for (i = 0; i < child_count; i++)
-		if (children[i] == prev_frame->this)
-			break;
-	LOCK (&frame->lock);
-	if (op_ret != -1) {
-		hafdp->fdstate[i] = 1;
-		local->op_ret = 0;
-	}
-	if (op_ret == -1 && op_errno != ENOTCONN)
-		local->op_errno = op_errno;
-	callcnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->fd);
-	}
-	return 0;
-}
-
-int32_t
-ha_open (call_frame_t *frame,
-	 xlator_t *this,
-	 loc_t *loc,
-	 int32_t flags, fd_t *fd, int wbflags)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	xlator_t **children = NULL;
-	int cnt = 0, i, child_count = 0, ret = 0;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_stateino = 0;
-        int32_t op_errno = ENOMEM;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-                
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->fd = fd;
-
-	hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
-        if (!hafdp) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!hafdp->fdstate) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->path = gf_strdup (loc->path);
-        if (!hafdp->path) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->active = -1;
-	if (pvt->pref_subvol == -1) {
-		hafdp->active = fd->inode->ino % child_count;
-	}
-
-	LOCK_INIT (&hafdp->lock);
-	fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
-	ret = inode_ctx_get (loc->inode, this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	for (i = 0; i < child_count; i++)
-		if (stateino[i])
-			cnt++;
-	local->call_count = cnt;
-	for (i = 0; i < child_count; i++) {
-		if (stateino[i]) {
-			STACK_WIND (frame,
-				    ha_open_cbk,
-				    children[i],
-				    children[i]->fops->open,
-				    loc, flags, fd, wbflags);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, fd);
-        if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                        hafdp->fdstate = NULL;
-                }
-
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                        hafdp->path = NULL;
-                }
-
-                GF_FREE (hafdp);
-        }
-
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_readv_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct iovec *vector,
-	      int32_t count,
-	      struct iatt *stbuf,
-              struct iobref *iobref)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      vector,
-			      count,
-			      stbuf,
-                              iobref);
-	}
-	return 0;
-}
-
-int32_t
-ha_readv (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  size_t size,
-	  off_t offset)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_readv_stub (frame, ha_readv, fd, size, offset);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_readv_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->readv,
-			   fd,
-			   size,
-			   offset);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_writev_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-               struct iatt *prebuf,
-	       struct iatt *postbuf)
-{
-	int ret = 0;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-                              prebuf,
-			      postbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_writev (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   struct iovec *vector,
-	   int32_t count,
-	   off_t off,
-           struct iobref *iobref)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_writev_stub (frame, ha_writev, fd, vector, count, off,
-                                       iobref);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_writev_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->writev,
-			   fd,
-			   vector,
-			   count,
-			   off,
-                           iobref);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;	
-}
-
- int32_t
-ha_flush_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno)
-{
-	int ret = 0;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_flush (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_flush_stub (frame, ha_flush, fd);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-                
-	STACK_WIND_COOKIE (frame,
-			   ha_flush_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->flush,
-			   fd);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
- int32_t
-ha_fsync_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-              struct iatt *prebuf,
-              struct iatt *postbuf)
-{
-	int ret = 0;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_fsync (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t flags)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fsync_stub (frame, ha_fsync, fd, flags);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fsync_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fsync,
-			   fd,
-			   flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_fstat_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct iatt *buf)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
-
-int32_t
-ha_fstat (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fstat_stub (frame, ha_fstat, fd);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fstat_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fstat,
-			   fd);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_opendir_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0, child_count = 0, callcnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-
-	ret = fd_ctx_get (local->fd, this, &tmp_hafdp);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_ptr_error()");
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	for (i = 0; i < child_count; i++)
-		if (children[i] == prev_frame->this)
-			break;
-	LOCK (&frame->lock);
-	if (op_ret != -1) {
-		hafdp->fdstate[i] = 1;
-		local->op_ret = 0;
-	}
-	if (op_ret == -1 && op_errno != ENOTCONN)
-		local->op_errno = op_errno;
-	callcnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->fd);
-	}
-	return 0;
-}
-
-int32_t
-ha_opendir (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc, fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	xlator_t **children = NULL;
-	int cnt = 0, i, child_count = 0, ret = 0;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_stateino = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->fd = fd;
-
-	hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
-        if (!hafdp) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!hafdp->fdstate) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->path = gf_strdup (loc->path);
-        if (!hafdp->path) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	LOCK_INIT (&hafdp->lock);
-	fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
-	ret = inode_ctx_get (loc->inode, this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-	
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "inode_ctx_get() error");
-	}
-	for (i = 0; i < child_count; i++)
-		if (stateino[i])
-			cnt++;
-	local->call_count = cnt;
-	for (i = 0; i < child_count; i++) {
-		if (stateino[i]) {
-			STACK_WIND (frame,
-				    ha_opendir_cbk,
-				    children[i],
-				    children[i]->fops->opendir,
-				    loc, fd);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-        ha_local_wipe (local);
-        if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                        hafdp->fdstate = NULL;
-                }
-
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                        hafdp->path = NULL;
-                }
-
-                GF_FREE (hafdp);
-        }
-        return 0;
-}
-
- int32_t
-ha_getdents_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 dir_entry_t *entries,
-		 int32_t count)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      entries,
-			      count);
-	}
-	return 0;
-}
-
-int32_t
-ha_getdents (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset,
-	     int32_t flag)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_getdents_stub (frame, ha_getdents, fd, size, offset,
-                                         flag);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_getdents_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->getdents,
-			   fd,
-			   size,
-			   offset,
-			   flag);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, 0);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_setdents_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_setdents (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags,
-	     dir_entry_t *entries,
-	     int32_t count)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-
-	local->stub = fop_setdents_stub (frame, ha_setdents, fd, flags, entries,
-                                         count);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_setdents_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->setdents,
-			   fd,
-			   flags,
-			   entries,
-			   count);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_fsyncdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_fsyncdir (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fsyncdir_stub (frame, ha_fsyncdir, fd, flags);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fsyncdir_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fsyncdir,
-			   fd,
-			   flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
- 
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
- int32_t
-ha_statfs_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct statvfs *buf)
-{
-        ha_local_t   *local = NULL;
-        ha_private_t *priv  = NULL;
-
-        GF_ASSERT (this);
-
-        local = frame->local;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (-1 == op_ret) {
-                local->active = (local->active + 1) % priv->child_count;
-                local->tries--;
-                if (!local->tries)
-                        goto out;
-                
-                STACK_WIND (frame, ha_statfs_cbk,
-                            HA_ACTIVE_CHILD(this, local),
-                            HA_ACTIVE_CHILD(this, local)->fops->statfs, 
-                            &local->loc);
-                return 0;
-        }
-
- out:
-        loc_wipe (&local->loc);
-        STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t
-ha_statfs (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-        ha_private_t *priv = NULL;
-	ha_local_t   *local = NULL;
-	int           op_errno = 0;
-
-        /* The normal way of handling failover doesn't work here
-         * as loc->inode may be null in this case.
-         */
-        local = GF_CALLOC (1, sizeof (*local), 
-                           gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        priv = this->private;
-        frame->local  = local;
-        local->active = priv->pref_subvol;
-        if (-1 == local->active)
-                local->active = 0;
-        local->tries  = priv->child_count;
-        loc_copy (&local->loc, loc);
-
-	STACK_WIND (frame, ha_statfs_cbk, HA_ACTIVE_CHILD(this, local),
-                    HA_ACTIVE_CHILD(this, local)->fops->statfs, loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;
-}
-
- int32_t
-ha_setxattr_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_setxattr (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     dict_t *dict,
-	     int32_t flags)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_setxattr_stub (frame, ha_setxattr, loc, dict, flags);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_setxattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->setxattr,
-			   loc,
-			   dict,
-			   flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_getxattr_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 dict_t *dict)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      dict);
-	}
-	return 0;
-}
-
-int32_t
-ha_getxattr (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     const char *name)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_getxattr_stub (frame, ha_getxattr, loc, name);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_getxattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->getxattr,
-			   loc,
-			   name);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_xattrop_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		dict_t *dict)
-{
-	int ret = -1;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, dict);
-	}
-	return 0;
-}
-
-
-int32_t
-ha_xattrop (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    gf_xattrop_flags_t flags,
-	    dict_t *dict)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-
-	local->stub = fop_xattrop_stub (frame, ha_xattrop, loc, flags, dict);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_xattrop_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->xattrop,
-			   loc,
-			   flags,
-			   dict);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, dict);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_fxattrop_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 dict_t *dict)
-{
-	int ret = -1;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0)
-		STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t
-ha_fxattrop (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     gf_xattrop_flags_t flags,
-	     dict_t *dict)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fxattrop_stub (frame, ha_fxattrop, fd, flags, dict);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fxattrop_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fxattrop,
-			   fd,
-			   flags,
-			   dict);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, dict);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_removexattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int ret = -1;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_removexattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	
-	local->stub = fop_removexattr_stub (frame, ha_removexattr, loc, name);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_removexattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->removexattr,
-			   loc,
-			   name);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_lk_setlk_unlck_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	int cnt = 0;
-	call_stub_t *stub = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	if (op_ret == 0)
-		local->op_ret = 0;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		stub = local->stub;
-		GF_FREE (local->state);
-		if (stub->args.lk.lock.l_type == F_UNLCK) {
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, &stub->args.lk.lock);
-		} else {
-			STACK_UNWIND (frame, -1, EIO, NULL);
-		}
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_lk_setlk_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0, cnt = 0, j = 0;
-	int child_count = 0;
-	call_frame_t *prev_frame = NULL;
-	char *state = NULL;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	state = local->state;
-
-	if (op_ret == 0)
-		local->op_ret = 0;
-
-	if ((op_ret == 0) || (op_ret == -1 && op_errno == ENOTCONN)) {
-		for (i = 0; i < child_count; i++) {
-			if (prev_frame->this == cookie)
-				break;
-		}
-		i++;
-		for (; i < child_count; i++) {
-			if (local->state[i])
-				break;
-		}
-		if (i == child_count) {
-			call_stub_t *stub = local->stub;
-			GF_FREE (local->state);
-			STACK_UNWIND (frame, 0, op_errno, &stub->args.lk.lock);
-			call_stub_destroy (stub);
-			return 0;
-		}
-		STACK_WIND (frame,
-			    ha_lk_setlk_cbk,
-			    children[i],
-			    children[i]->fops->lk,
-			    local->stub->args.lk.fd,
-			    local->stub->args.lk.cmd,
-			    &local->stub->args.lk.lock);
-		return 0;
-	} else {
-		for (i = 0; i < child_count; i++) {
-			if (prev_frame->this == cookie)
-				break;
-		}
-		cnt = 0;
-		for (j = 0; j < i; j++) {
-			if (state[i])
-				cnt++;
-		}
-		if (cnt) {
-			struct gf_flock lock;
-			lock = local->stub->args.lk.lock;
-			for (i = 0; i < child_count; i++) {
-				if (state[i]) {
-					STACK_WIND (frame,
-						    ha_lk_setlk_unlck_cbk,
-						    children[i],
-						    children[i]->fops->lk,
-						    local->stub->args.lk.fd,
-						    local->stub->args.lk.cmd,
-						    &lock);
-					if (--cnt == 0)
-						break;
-				}
-			}
-			return 0;
-		} else {
-			GF_FREE (local->state);
-			call_stub_destroy (local->stub);
-			STACK_UNWIND (frame,
-				      op_ret,
-				      op_errno,
-				      lock);
-			return 0;
-		}
-	}
-}
-
-int32_t
-ha_lk_getlk_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	fd_t *fd = NULL;
-	int child_count = 0, i = 0;
-	xlator_t **children = NULL;
-	call_frame_t *prev_frame = NULL;
-
-	local = frame->local;
-	pvt = this->private;
-	fd = local->stub->args.lk.fd;
-	child_count = pvt->child_count;
-	children = pvt->children;
-	prev_frame = cookie;
-
-	if (op_ret == 0) {
-		GF_FREE (local->state);
-		call_stub_destroy (local->stub);
-		STACK_UNWIND (frame, 0, 0, lock);
-		return 0;
-	}
-
-	for (i = 0; i < child_count; i++) {
-		if (prev_frame->this == children[i])
-			break;
-	}
-
-	for (; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (i == child_count) {
-		GF_FREE (local->state);
-		call_stub_destroy (local->stub);
-		STACK_UNWIND (frame, op_ret, op_errno, lock);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    ha_lk_getlk_cbk,
-		    children[i],
-		    children[i]->fops->lk,
-		    fd,
-		    local->stub->args.lk.cmd,
-		    &local->stub->args.lk.lock);
-	return 0;
-}
-
-int32_t
-ha_lk (call_frame_t *frame,
-       xlator_t *this,
-       fd_t *fd,
-       int32_t cmd,
-       struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	hafd_t *hafdp = NULL;
-	char *state = NULL;
-	int child_count = 0, i = 0, cnt = 0, ret = 0;
-	xlator_t **children = NULL;
-	uint64_t tmp_hafdp = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	children = pvt->children;
-	ret = fd_ctx_get (fd, this, &tmp_hafdp);
-	if (ret < 0)
-		gf_log (this->name, GF_LOG_ERROR, "fd_ctx_get failed");
-
-	if (local == NULL) {
-                local = frame->local = GF_CALLOC (1, sizeof (*local),
-                                                  gf_ha_mt_ha_local_t);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-                        
-		local->active = -1;
-		local->op_ret = -1;
-		local->op_errno = ENOTCONN;
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	if (local->active == -1) {
-                op_errno = ENOTCONN;
-                goto err;
-	}
-
-	local->stub = fop_lk_stub (frame, ha_lk, fd, cmd, lock);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	state = hafdp->fdstate;
-	LOCK (&hafdp->lock);
-	memcpy (local->state, state, child_count);
-	UNLOCK (&hafdp->lock);
-	if (cmd == F_GETLK) {
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i])
-				break;
-		}
-		STACK_WIND (frame,
-			    ha_lk_getlk_cbk,
-			    children[i],
-			    children[i]->fops->lk,
-			    fd,
-			    cmd,
-			    lock);
-	} else if (cmd == F_SETLK && lock->l_type == F_UNLCK) {
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i])
-				local->call_count++;
-		}
-		cnt = local->call_count;
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i]) {
-				STACK_WIND (frame,
-					    ha_lk_setlk_unlck_cbk,
-					    children[i],
-					    children[i]->fops->lk,
-					    fd, cmd, lock);
-				if (--cnt == 0)
-					break;
-			}
-		}
-	} else {
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i])
-				break;
-		}
-		STACK_WIND (frame,
-			    ha_lk_setlk_cbk,
-			    children[i],
-			    children[i]->fops->lk,
-			    fd,
-			    cmd,
-			    lock);
-	}
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-        STACK_UNWIND (frame, -1, op_errno, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_inode_entry_lk_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_inodelk (call_frame_t *frame,
-	    xlator_t *this,
-            const char *volume,
-	    loc_t *loc,
-	    int32_t cmd,
-	    struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_inodelk_stub (frame, ha_inodelk, volume,
-                                        loc, cmd, lock);
-	STACK_WIND_COOKIE (frame,
-			   ha_inode_entry_lk_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->inodelk,
-			   volume,
-                           loc,
-			   cmd,
-			   lock);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno);
-	return 0;
-}
-
-int32_t
-ha_entrylk (call_frame_t *frame,
-	    xlator_t *this,
-            const char *volume,
-	    loc_t *loc,
-	    const char *basename,
-	    entrylk_cmd cmd,
-	    entrylk_type type)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_entrylk_stub (frame, ha_entrylk, volume,
-                                        loc, basename, cmd, type);
-	STACK_WIND_COOKIE (frame,
-			   ha_inode_entry_lk_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->entrylk,
-			   volume, loc, basename, cmd, type);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno);
-	return 0;
-}
-
- int32_t
-ha_checksum_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 uint8_t *file_checksum,
-		 uint8_t *dir_checksum)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      file_checksum,
-			      dir_checksum);
-	}
-	return 0;
-}
-
-int32_t
-ha_checksum (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t flag)
-{
-	int op_errno = 0;
-	ha_local_t *local = NULL;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_checksum_stub (frame, ha_checksum, loc, flag);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_checksum_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->checksum,
-			   loc,
-			   flag);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_common_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0)
-		STACK_UNWIND (frame, op_ret, op_errno, entries);
-	return 0;
-}
-
-int32_t
-ha_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-            off_t off);
-
-int32_t
-ha_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off);
-int32_t
-ha_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t off, int whichop)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-        if (whichop == GF_FOP_READDIR)
-                local->stub = fop_readdir_stub (frame, ha_readdir, fd, size,
-                                                off);
-        else
-                local->stub = fop_readdirp_stub (frame, ha_readdirp, fd, size,
-                                                 off);
-        if (!local->stub) {
-                op_errno = ENOMEM; 
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, ha_common_readdir_cbk,
-                                   (void *)(long)local->active,
-                                   HA_ACTIVE_CHILD(this, local),
-                                   HA_ACTIVE_CHILD(this, local)->fops->readdir,
-                                   fd, size, off);
-        else
-                STACK_WIND_COOKIE (frame, ha_common_readdir_cbk,
-                                   (void *)(long)local->active,
-                                   HA_ACTIVE_CHILD(this, local),
-                                   HA_ACTIVE_CHILD(this, local)->fops->readdirp,
-                                   fd, size, off);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
-int32_t
-ha_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-            off_t off)
-{
-        ha_do_readdir (frame, this, fd, size, off, GF_FOP_READDIR);
-        return 0;
-}
-
-int32_t
-ha_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off)
-{
-        ha_do_readdir (frame, this, fd, size, off, GF_FOP_READDIRP);
-        return 0;
-}
-
-/* Management operations */
-
- int32_t
-ha_stats_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct xlator_stats *stats)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	if (op_ret == -1 && op_errno == ENOTCONN) {
-		for (i = 0; i < pvt->child_count; i++) {
-			if (prev_frame->this == children[i])
-				break;
-		}
-		i++;
-		for (; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				break;
-		}
-
-		if (i == pvt->child_count) {
-			STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-			return 0;
-		}
-		STACK_WIND (frame,
-			    ha_stats_cbk,
-			    children[i],
-			    children[i]->mops->stats,
-			    local->flags);
-		return 0;
-	}
-
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      stats);
-	return 0;
-}
-
-int32_t
-ha_stats (call_frame_t *frame,
-	  xlator_t *this,
-	  int32_t flags)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local = GF_CALLOC (1, sizeof (*local),
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	pvt = this->private;
-	children = pvt->children;
-	for (i = 0; i < pvt->child_count; i++) {
-		if (pvt->state[i])
-			break;
-	}
-
-	if (i == pvt->child_count) {
-                op_errno = ENOTCONN;
-                goto err;
-	}
-	local->flags = flags;
-
-	STACK_WIND (frame,
-		    ha_stats_cbk,
-		    children[i],
-		    children[i]->mops->stats,
-		    flags);
-	return 0;
-
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-        STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-
-}
-
-
-int32_t
-ha_getspec_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		char *spec_data)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	if (op_ret == -1 && op_errno == ENOTCONN) {
-		for (i = 0; i < pvt->child_count; i++) {
-			if (prev_frame->this == children[i])
-				break;
-		}
-		i++;
-		for (; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				break;
-		}
-
-		if (i == pvt->child_count) {
-			STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-			return 0;
-		}
-		STACK_WIND (frame,
-			    ha_getspec_cbk,
-			    children[i],
-			    children[i]->mops->getspec,
-			    local->pattern,
-			    local->flags);
-		return 0;
-	}
-
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      spec_data);
-	return 0;
-}
-
-int32_t
-ha_getspec (call_frame_t *frame,
-	    xlator_t *this,
-	    const char *key,
-	    int32_t flags)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	pvt = this->private;
-	children = pvt->children;
-
-	for (i = 0; i < pvt->child_count; i++) {
-		if (pvt->state[i])
-			break;
-	}
-
-	if (i == pvt->child_count) {
-                op_errno = ENOTCONN;
-                goto err;
-	}
-	local->flags = flags;
-	local->pattern = (char *)key;
-
-	STACK_WIND (frame,
-		    ha_getspec_cbk,
-		    children[i],
-		    children[i]->mops->getspec,
-		    key, flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-        STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-
-}
-
-int32_t
-ha_closedir (xlator_t *this,
-	     fd_t *fd)
-{
-	hafd_t *hafdp = NULL;
-	int op_errno = 0;
-	uint64_t tmp_hafdp = 0;
-
-	op_errno = fd_ctx_del (fd, this, &tmp_hafdp);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "fd_ctx_del() error");
-		return 0;
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	GF_FREE (hafdp->fdstate);
-	GF_FREE (hafdp->path);
-	LOCK_DESTROY (&hafdp->lock);
-	return 0;
-}
-
-int32_t
-ha_close (xlator_t *this,
-	  fd_t *fd)
-{
-	hafd_t *hafdp = NULL;
-	int op_errno = 0;
-	uint64_t tmp_hafdp = 0;
-
-	op_errno = fd_ctx_del (fd, this, &tmp_hafdp);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "fd_ctx_del() error");
-		return 0;
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	GF_FREE (hafdp->fdstate);
-	GF_FREE (hafdp->path);
-	LOCK_DESTROY (&hafdp->lock);
-	return 0;
-}
-
-/* notify */
-int32_t
-notify (xlator_t *this,
-	int32_t event,
-	void *data,
-	...)
-{
-	ha_private_t *pvt = NULL;
-	int32_t i = 0, upcnt = 0;
-
-	pvt = this->private;
-	if (pvt == NULL) {
-		gf_log (this->name, GF_LOG_DEBUG, "got notify before init()");
-		return 0;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_DOWN:
-	{
-		for (i = 0; i < pvt->child_count; i++) {
-			if (data == pvt->children[i])
-				break;
-		}
-	        gf_log (this->name, GF_LOG_DEBUG, "GF_EVENT_CHILD_DOWN from %s", pvt->children[i]->name);
-		pvt->state[i] = 0;
-		for (i = 0; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				break;
-		}
-		if (i == pvt->child_count) {
-			default_notify (this, event, data);
-		}
-	}
-	break;
-	case GF_EVENT_CHILD_UP:
-	{
-		for (i = 0; i < pvt->child_count; i++) {
-			if (data == pvt->children[i])
-				break;
-		}
-
-		gf_log (this->name, GF_LOG_DEBUG, "GF_EVENT_CHILD_UP from %s", pvt->children[i]->name);
-
-		pvt->state[i] = 1;
-
-		for (i = 0; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				upcnt++;
-		}
-
-		if (upcnt == 1) {
-			default_notify (this, event, data);
-		}
-	}
-	break;
-
-	default:
-	{
-		default_notify (this, event, data);
-	}
-	}
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_ha_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int
-init (xlator_t *this)
-{
-	ha_private_t *pvt = NULL;
-	xlator_list_t *trav = NULL;
-	int count = 0, ret = 0;
-
-
-	if (!this->children) {
-		gf_log (this->name,GF_LOG_ERROR, 
-			"FATAL: ha should have one or more child defined");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	trav = this->children;
-	pvt = GF_CALLOC (1, sizeof (ha_private_t), gf_ha_mt_ha_private_t);
-
-	ret = dict_get_int32 (this->options, "preferred-subvolume", 
-			      &pvt->pref_subvol);
-	if (ret < 0) {
-		pvt->pref_subvol = -1;
-	}
-
-	trav = this->children;
-	while (trav) {
-		count++;
-		trav = trav->next;
-	}
-
-	pvt->child_count = count;
-	pvt->children = GF_CALLOC (count, sizeof (xlator_t*), 
-                                   gf_ha_mt_xlator_t);
-
-	trav = this->children;
-	count = 0;
-	while (trav) {
-		pvt->children[count] = trav->xlator;
-		count++;
-		trav = trav->next;
-	}
-
-	pvt->state = GF_CALLOC (1, count, gf_ha_mt_char);
-	this->private = pvt;
-	return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-	ha_private_t *priv = NULL;
-	priv = this->private;
-	GF_FREE (priv);
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = ha_lookup,
-	.stat        = ha_stat,
-	.readlink    = ha_readlink,
-	.mknod       = ha_mknod,
-	.mkdir       = ha_mkdir,
-	.unlink      = ha_unlink,
-	.rmdir       = ha_rmdir,
-	.symlink     = ha_symlink,
-	.rename      = ha_rename,
-	.link        = ha_link,
-	.truncate    = ha_truncate,
-	.create      = ha_create,
-	.open        = ha_open,
-	.readv       = ha_readv,
-	.writev      = ha_writev,
-	.statfs      = ha_statfs,
-	.flush       = ha_flush,
-	.fsync       = ha_fsync,
-	.setxattr    = ha_setxattr,
-	.getxattr    = ha_getxattr,
-	.removexattr = ha_removexattr,
-	.opendir     = ha_opendir,
-	.readdir     = ha_readdir,
-	.readdirp    = ha_readdirp,
-	.getdents    = ha_getdents,
-	.fsyncdir    = ha_fsyncdir,
-	.access      = ha_access,
-	.ftruncate   = ha_ftruncate,
-	.fstat       = ha_fstat,
-	.lk          = ha_lk,
-	.setdents    = ha_setdents,
-	.lookup_cbk  = ha_lookup_cbk,
-	.checksum    = ha_checksum,
-	.xattrop     = ha_xattrop,
-	.fxattrop    = ha_fxattrop,
-        .setattr     = ha_setattr,
-        .fsetattr    = ha_fsetattr,
-};
-
-struct xlator_cbks cbks = {
-	.release    = ha_close,
-	.releasedir = ha_closedir,
-	.forget     = ha_forget,
-};
diff --git a/xlators/cluster/ha/src/ha.h b/xlators/cluster/ha/src/ha.h
deleted file mode 100644
index e2ed7eaa68a..00000000000
--- a/xlators/cluster/ha/src/ha.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __HA_H_
-#define __HA_H_
-
-#include "ha-mem-types.h"
-
-typedef struct {
-	call_stub_t *stub;
-	int32_t op_ret, op_errno;
-	int32_t active, tries, revalidate, revalidate_error;
-	int32_t call_count;
-	char *state, *pattern;
-	dict_t *dict;
-	loc_t loc;
-	struct iatt buf;
-        struct iatt postparent;
-        struct iatt preparent;
-	fd_t *fd;
-	inode_t *inode;
-	int32_t flags;
-	int32_t first_success;
-} ha_local_t;
-
-typedef struct {
-	char *state;
-	xlator_t **children;
-	int child_count, pref_subvol;
-} ha_private_t;
-
-typedef struct {
-	char *fdstate;
-	char *path;
-	gf_lock_t lock;
-	int active;
-} hafd_t;
-
-#define HA_ACTIVE_CHILD(this, local) (((ha_private_t *)this->private)->children[local->active])
-
-extern int ha_alloc_init_fd (call_frame_t *frame, fd_t *fd);
-
-extern int ha_handle_cbk (call_frame_t *frame, void *cookie, int op_ret, int op_errno) ;
-
-extern int ha_alloc_init_inode (call_frame_t *frame, inode_t *inode);
-
-#endif
diff --git a/xlators/cluster/map/Makefile.am b/xlators/cluster/map/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/cluster/map/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/cluster/map/src/Makefile.am b/xlators/cluster/map/src/Makefile.am
deleted file mode 100644
index 209cafa7c1b..00000000000
--- a/xlators/cluster/map/src/Makefile.am
+++ /dev/null
@@ -1,17 +0,0 @@
-xlator_LTLIBRARIES = map.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
-
-map_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-
-map_la_SOURCES = map.c map-helper.c
-map_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = map.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/map/src/map-helper.c b/xlators/cluster/map/src/map-helper.c
deleted file mode 100644
index c5f828ae6d1..00000000000
--- a/xlators/cluster/map/src/map-helper.c
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
-   Copyright (c) 2009-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "map.h"
-
-
-xlator_t *
-map_subvol_next (xlator_t *this, xlator_t *prev)
-{
-	map_private_t *priv = NULL;
-	xlator_t      *next = NULL;
-	int            i = 0;
-
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (priv->xlarray[i].xl == prev) {
-			if ((i + 1) < priv->child_count)
-				next = priv->xlarray[i + 1].xl;
-			break;
-		}
-	}
-
-	return next;
-}
-
-int
-map_subvol_cnt (xlator_t *this, xlator_t *subvol)
-{
-	int i = 0;
-	int ret = -1;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (subvol == priv->xlarray[i].xl) {
-			ret = i;
-			break;
-		}
-	}
-
-	return ret;
-}
-
-int
-map_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
-{
-	map_private_t *priv = NULL;
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    y = 0;
-
-	if (x == ((uint64_t) -1)) {
-		y = (uint64_t) -1;
-		goto out;
-	}
-
-	priv = this->private;
-
-	max = priv->child_count;
-	cnt = map_subvol_cnt (this, subvol);
-
-	y = ((x * max) + cnt);
-
-out:
-	if (y_p)
-		*y_p = y;
-
-	return 0;
-}
-
-
-int
-map_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
-		  uint64_t *x_p)
-{
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    x = 0;
-	xlator_t   *subvol = 0;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-	max = priv->child_count;
-
-	cnt = y % max;
-	x   = y / max;
-
-	subvol = priv->xlarray[cnt].xl;
-
-	if (subvol_p)
-		*subvol_p = subvol;
-
-	if (x_p)
-		*x_p = x;
-
-	return 0;
-}
-
-
-xlator_t *
-get_mapping_subvol_from_path (xlator_t *this, const char *path) 
-{
-	map_private_t      *priv = NULL;
-	struct map_pattern *map = NULL;
-
-	/* To make sure we handle '/' properly */
-	if (!strcmp (path, "/"))
-		return NULL;
-
-	priv = this->private;
-
-	map = priv->map;
-	while (map) {
-		if (!strncmp (map->directory, path, map->dir_len)) {
-			if ((path[map->dir_len] == '/') ||
-			    (path[map->dir_len] == '\0')) {
-				return map->xl;
-			}
-		}
-		
-		map = map->next;
-	}
-
-	return priv->default_xl;
-}
-
-xlator_t *
-get_mapping_subvol_from_ctx (xlator_t *this, inode_t *inode)
-{
-	uint64_t subvol = 0;
-	int      ret    = -1;
-
-	ret = inode_ctx_get (inode, this, &subvol);
-	if (ret != 0) 
-		return NULL;
-
-	return (xlator_t *)(long)subvol;
-}
-
-int
-check_multiple_volume_entry (xlator_t *this, 
-			     xlator_t *subvol)
-{
-	int ret = -1;
-	int idx = 0;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-	
-	for (idx = 0; idx < priv->child_count; idx++) {
-		if (priv->xlarray[idx].xl == subvol) {
-			if (priv->xlarray[idx].mapped) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume '%s' is already mapped",
-					subvol->name);
-				goto out;
-			}
-			priv->xlarray[idx].mapped = 1;
-			ret = 0;
-			goto out;
-		}
-	}
-
-	gf_log (this->name, GF_LOG_ERROR,
-		"subvolume '%s' is not found",
-		subvol->name);
-	
- out:
-	return ret;
-}
-
-int
-verify_dir_and_assign_subvol (xlator_t *this, 
-			      const char *directory, 
-			      const char *subvol)
-{
-	int            default_flag = 0;
-	int            ret  = -1;
-	int            idx  = 0;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-	struct map_pattern *tmp_map = NULL;
-
-	priv = this->private;
-
-	/* check if directory is valid, ie, its a top level dir, and 
-	 * not includes a '*' in it.
-	 */
-	if (!strcmp ("*", directory)) {
-		default_flag = 1;
-	} else {
-		if (directory[0] != '/') {
-			gf_log (this->name, GF_LOG_ERROR,
-				"map takes absolute path, starting with '/'. "
-				"not '%s'", directory);
-			goto out;
-		}
-		for (idx = 1; idx < (strlen (directory) - 1); idx++) {
-			if (directory[idx] == '/') {
-				gf_log (this->name, GF_LOG_ERROR,
-					"map takes only top level directory, "
-					"not '%s'", directory);
-				goto out;
-			}
-		}
-	}
-
-	/* Assign proper subvolume */
-	trav = this->children;
-	while (trav) {
-		if (!strcmp (trav->xlator->name, subvol)) {
-			
-			/* Check if there is another directory for 
-			 * same volume, if yes, return error.
-			 */
-			ret = check_multiple_volume_entry (this, 
-							   trav->xlator);
-			if (ret != 0) {
-				goto out;
-			}
-
-			ret = 0;
-			if (default_flag) {
-				if (priv->default_xl) {
-					ret = -1;
-					gf_log (this->name, GF_LOG_ERROR,
-						"'*' specified more than "
-						"once. don't confuse me!!!");
-				}
-
-				priv->default_xl = trav->xlator;
-				goto out;
-			}
-
-			tmp_map = GF_CALLOC (1, sizeof (struct map_pattern),
-                                             gf_map_mt_map_pattern);
-			tmp_map->xl = trav->xlator;
-			tmp_map->dir_len = strlen (directory);
-
-			/* make sure that the top level directory starts 
-			 * with '/' and ends without '/'
-			 */
-			tmp_map->directory = gf_strdup (directory);
-			if (directory[tmp_map->dir_len - 1] == '/') {
-				tmp_map->dir_len--;
-			}
-
-			if (!priv->map) 
-				priv->map = tmp_map;
-			else {
-				struct map_pattern *trav_map = NULL;
-				trav_map = priv->map;
-				while (trav_map->next)
-					trav_map = trav_map->next;
-				trav_map->next = tmp_map;
-			}
-			
-			goto out;
-		}
-
-		trav = trav->next;
-	}
-
-	gf_log (this->name, GF_LOG_ERROR, 
-		"map volume '%s' is not proper subvolume", subvol);
-
- out:
-	return ret;
-}
-
-int 
-assign_default_subvol (xlator_t *this, const char *default_xl)
-{
-	int ret = -1;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-	priv = this->private;
-	trav = this->children;
-
-	while (trav) {
-		if (!strcmp (trav->xlator->name, default_xl)) {
-			ret = check_multiple_volume_entry (this, 
-							   trav->xlator);
-			if (ret != 0) {
-				goto out;
-			}
-			if (priv->default_xl)
-				gf_log (this->name, GF_LOG_WARNING,
-					"default-volume option provided, "
-					"overriding earlier '*' option");
-			priv->default_xl = trav->xlator;
-			return 0;
-		}
-		trav = trav->next;
-	}
-
-	gf_log (this->name, GF_LOG_ERROR,
-		"default-volume value is not an valid subvolume. check again");
- out:
-	return -1;
-}
-
-void
-verify_if_all_subvolumes_got_used (xlator_t *this)
-{
-	int idx = 0;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-	
-	for (idx = 0; idx < priv->child_count; idx++) {
-		if (!priv->xlarray[idx].mapped) {
-			if (!priv->default_xl) {
-				priv->default_xl = priv->xlarray[idx].xl;
-				priv->xlarray[idx].mapped = 1;
-			} else {
-				gf_log (this->name, GF_LOG_WARNING,
-					"subvolume '%s' is not mapped to "
-					"any directory",
-					priv->xlarray[idx].xl->name);
-			}
-		}
-	}
-
-	if (!priv->default_xl) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"default subvolume not specified, filesystem "
-			"may not work properly. Check 'map' translator "
-			"documentation for more info");
-	}
-
-	return ;
-}
diff --git a/xlators/cluster/map/src/map-mem-types.h b/xlators/cluster/map/src/map-mem-types.h
deleted file mode 100644
index 3e89f4736e4..00000000000
--- a/xlators/cluster/map/src/map-mem-types.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __MAP_MEM_TYPES_H__
-#define __MAP_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_map_mem_types_ {
-        gf_map_mt_map_private_t = gf_common_mt_end + 1,
-        gf_map_mt_map_local_t,
-        gf_map_mt_map_xlator_array,
-        gf_map_mt_map_pattern,
-        gf_map_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/map/src/map.c b/xlators/cluster/map/src/map.c
deleted file mode 100644
index fbb56dbc554..00000000000
--- a/xlators/cluster/map/src/map.c
+++ /dev/null
@@ -1,2561 +0,0 @@
-/*
-   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "map.h"
-
-/* TODO : 
- *     -> support for 'get' 'put' API in through xattrs.
- *     -> define the behavior of notify()
- */
-
-static int32_t
-map_stat_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf)
-
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-static int32_t
-map_setattr_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct iatt *statpre,
-                 struct iatt *statpost)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-
-	map_itransform (this, prev->this, statpre->ia_ino, &statpre->ia_ino);
-	map_itransform (this, prev->this, statpost->ia_ino, &statpost->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
-	return 0;
-}
-
-static int32_t
-map_fsetattr_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  struct iatt *statpre,
-                  struct iatt *statpost)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-
-	map_itransform (this, prev->this, statpre->ia_ino, &statpre->ia_ino);
-	map_itransform (this, prev->this, statpost->ia_ino, &statpost->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
-	return 0;
-}
-
-static int32_t
-map_truncate_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-static int32_t
-map_ftruncate_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-                       struct iatt *postbuf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-static int32_t
-map_access_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-static int32_t
-map_readlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      const char *path,
-                      struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-static int32_t
-map_unlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-static int32_t
-map_rmdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-
-static int32_t
-map_rename_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *buf,
-                    struct iatt *preoldparent,
-                    struct iatt *postoldparent,
-                    struct iatt *prenewparent,
-                    struct iatt *postnewparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-static int32_t
-map_link_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode,	buf);
-	return 0;
-}
-
-static int32_t
-map_open_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-static int32_t
-map_readv_cbk (call_frame_t *frame,
-               void *cookie,
-               xlator_t *this,
-               int32_t op_ret,
-               int32_t op_errno,
-               struct iovec *vector,
-               int32_t count,
-               struct iatt *stbuf,
-               struct iobref *iobref)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, stbuf->ia_ino, &stbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf, iobref);
-	return 0;
-}
-
-static int32_t
-map_writev_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *prebuf,
-		    struct iatt *postbuf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-static int32_t
-map_flush_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fsync_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-static int32_t
-map_fstat_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct iatt *buf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-static int32_t
-map_getdents_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      dir_entry_t *entries,
-		      int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entries, count);
-	return 0;
-}
-
-
-static int32_t
-map_setdents_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-static int32_t
-map_fsyncdir_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_setxattr_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fsetxattr_cbk (call_frame_t *frame,
-                       void *cookie,
-                       xlator_t *this,
-                       int32_t op_ret,
-                       int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fgetxattr_cbk (call_frame_t *frame,
-                       void *cookie,
-                       xlator_t *this,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-
-
-static int32_t
-map_getxattr_cbk (call_frame_t *frame,
-                      void *cookie,
-                      xlator_t *this,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t
-map_xattrop_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t
-map_fxattrop_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-static int32_t
-map_removexattr_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_lk_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		struct gf_flock *lock)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-
-static int32_t
-map_inodelk_cbk (call_frame_t *frame, void *cookie,
-		     xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-
-static int32_t
-map_finodelk_cbk (call_frame_t *frame, void *cookie,
-		      xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_entrylk_cbk (call_frame_t *frame, void *cookie,
-		     xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fentrylk_cbk (call_frame_t *frame, void *cookie,
-		      xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-static int32_t
-map_newentry_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-
-}
-
-
-static int32_t
-map_create_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd,
-		inode_t *inode,
-		struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
-}
-
-
-/*
- * map_normalize_stats -
- */
-void
-map_normalize_stats (struct statvfs *buf,
-		     unsigned long bsize,
-		     unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-int32_t
-map_statfs_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		struct statvfs *stbuf)
-{
-	struct statvfs *dict_buf = NULL;
-	map_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	unsigned long bsize;
-	unsigned long frsize;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		this_call_cnt = --local->call_count;
-
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-		local->op_ret = 0;
-		
-		/* when a call is successful, add it to local->dict */
-		dict_buf = &local->statvfs;
-		
-		if (dict_buf->f_bsize != 0) {
-			bsize  = max (dict_buf->f_bsize, 
-				      stbuf->f_bsize);
-			
-			frsize = max (dict_buf->f_frsize, 
-				      stbuf->f_frsize);
-			map_normalize_stats(dict_buf, bsize, frsize);
-			map_normalize_stats(stbuf, bsize, frsize);
-		} else {
-			dict_buf->f_bsize   = stbuf->f_bsize;
-			dict_buf->f_frsize  = stbuf->f_frsize;
-		}
-		
-		dict_buf->f_blocks += stbuf->f_blocks;
-		dict_buf->f_bfree  += stbuf->f_bfree;
-		dict_buf->f_bavail += stbuf->f_bavail;
-		dict_buf->f_files  += stbuf->f_files;
-		dict_buf->f_ffree  += stbuf->f_ffree;
-		dict_buf->f_favail += stbuf->f_favail;
-		dict_buf->f_fsid    = stbuf->f_fsid;
-		dict_buf->f_flag    = stbuf->f_flag;
-		dict_buf->f_namemax = stbuf->f_namemax;
-	}
-unlock:
-	UNLOCK (&frame->lock);
-
-	if (!this_call_cnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      &local->statvfs);
-	}
-
-	return 0;
-}
-
-int32_t
-map_single_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-                       dict_t *dict,
-                       struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict);
-
-	return 0;
-}
-
-int32_t
-map_root_lookup_cbk (call_frame_t *frame,
-                     void *cookie,
-                     xlator_t *this,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     inode_t *inode,
-                     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
-{
-	int          callcnt = 0;
-	map_local_t *local = NULL;
-	inode_t     *tmp_inode = NULL;
-	dict_t      *tmp_dict = NULL;
-
-	local = frame->local;
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == 0) && (local->op_ret == -1)) {
-			local->op_ret = 0;
-			local->stbuf = *buf;
-			if (dict)
-				local->dict = dict_ref (dict);
-			local->inode = inode_ref (inode);
-		}
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-		
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		tmp_dict = local->dict;
-		tmp_inode = local->inode;
-
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->inode, 
-			      &local->stbuf, local->dict);
-
-		inode_unref (local->inode);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-
-int32_t
-map_opendir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd)
-{
-	int callcnt = 0;
-	map_local_t *local = NULL;
-	fd_t *local_fd = NULL;
-
-	local = frame->local;
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-
-		local->op_ret = 0;
-	}
- unlock:
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_fd = local->fd;
-		local->fd = NULL;
-
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local_fd);
-
-		fd_unref (local_fd);
-	}
-	return 0;
-}
-
-int32_t
-map_single_readdir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			gf_dirent_t *entries)
-{
-        call_frame_t *prev = NULL;
-	gf_dirent_t  *orig_entry = NULL;
-
-        prev  = cookie;
-
-	list_for_each_entry (orig_entry, &entries->list, list) {
-		map_itransform (this, prev->this, orig_entry->d_ino, 
-				&orig_entry->d_ino);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, entries);
-	
-	return 0;
-}
-
-
-int32_t
-map_single_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        call_frame_t *prev = NULL;
-	gf_dirent_t  *orig_entry = NULL;
-
-        prev  = cookie;
-
-	list_for_each_entry (orig_entry, &entries->list, list) {
-		map_itransform (this, prev->this, orig_entry->d_ino,
-				&orig_entry->d_ino);
-                orig_entry->d_stat.ia_ino = orig_entry->d_ino;
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, entries);
-	return 0;
-}
-
-int
-map_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries);
-
-int
-map_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries);
-
-int
-map_generic_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                         int whichop)
-{
-	map_local_t  *local = NULL;
-	gf_dirent_t   entries;
-	gf_dirent_t  *orig_entry = NULL;
-	gf_dirent_t  *entry = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *subvol = NULL;
-	xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-	int           count = 0;
-	fd_t         *local_fd = NULL;
-
-	INIT_LIST_HEAD (&entries.list);
-	prev = cookie;
-	local = frame->local;
-
-	if (op_ret < 0)
-		goto done;
-
-	list_for_each_entry (orig_entry, &orig_entries->list, list) {
-		subvol = prev->this;
-
-		entry = gf_dirent_for_name (orig_entry->d_name);
-		if (!entry) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"memory allocation failed :(");
-			goto unwind;
-		}
-		
-		map_itransform (this, subvol, orig_entry->d_ino,
-				&entry->d_ino);
-		map_itransform (this, subvol, orig_entry->d_off,
-				&entry->d_off);
-		
-                if (whichop == GF_FOP_READDIRP)
-                        entry->d_stat.ia_ino = entry->d_ino;
-		entry->d_type = orig_entry->d_type;
-		entry->d_len  = orig_entry->d_len;
-		
-		list_add_tail (&entry->list, &entries.list);
-		count++;
-                next_offset = orig_entry->d_off;
-	}
-
-	op_ret = count;
-
-done:
-	if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = map_subvol_next (this, prev->this);
-                } else {
-                        next_subvol = prev->this;
-                }
-
-		if (!next_subvol) {
-			goto unwind;
-		}
-
-                if (whichop == GF_FOP_READDIR)
-                        STACK_WIND (frame, map_readdir_cbk, next_subvol,
-                                    next_subvol->fops->readdir, local->fd,
-                                    local->size, 0);
-                else
-                        STACK_WIND (frame, map_readdirp_cbk, next_subvol,
-                                    next_subvol->fops->readdirp, local->fd,
-                                    local->size, 0);
-		return 0;
-	}
-
-unwind:
-	if (op_ret < 0)
-		op_ret = 0;
-
-	local_fd = local->fd;
-	local->fd = NULL;
-
-	STACK_UNWIND (frame, op_ret, op_errno, &entries);
-
-	fd_unref (local_fd);
-
-	gf_dirent_free (&entries);
-
-        return 0;
-}
-
-
-int
-map_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries)
-{
-        map_generic_readdir_cbk (frame, cookie, this, op_ret, op_errno,
-                                 orig_entries, GF_FOP_READDIR);
-        return 0;
-}
-
-
-int
-map_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, gf_dirent_t *orig_entries)
-{
-        map_generic_readdir_cbk (frame, cookie, this, op_ret, op_errno,
-                                 orig_entries, GF_FOP_READDIRP);
-        return 0;
-}
-
-
-/* Management operations */
-
-static int32_t
-map_checksum_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      uint8_t *file_checksum,
-		      uint8_t *dir_checksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, file_checksum, dir_checksum);
-	return 0;
-}
-
-
-/* Fops starts here */
-
-int32_t
-map_stat (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_stat_cbk, subvol, subvol->fops->stat, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_setattr (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc,
-             struct iatt *stbuf,
-             int32_t valid)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("map", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        GF_VALIDATE_OR_GOTO (this->name, loc, err);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, err);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, err);
-        GF_VALIDATE_OR_GOTO (this->name, stbuf, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_setattr_cbk, subvol,
-		    subvol->fops->setattr, loc, stbuf, valid);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fsetattr (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iatt *stbuf,
-              int32_t valid)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("map", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        GF_VALIDATE_OR_GOTO (this->name, fd, err);
-        GF_VALIDATE_OR_GOTO (this->name, stbuf, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsetattr_cbk, subvol,
-		    subvol->fops->fsetattr, fd, stbuf, valid);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_truncate (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      off_t offset)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_truncate_cbk, subvol, 
-                    subvol->fops->truncate, loc, offset);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_ftruncate (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       off_t offset)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_ftruncate_cbk, subvol,
-		    subvol->fops->ftruncate, fd, offset);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_access (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t mask)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_access_cbk, subvol,
-		    subvol->fops->access, loc, mask);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_readlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      size_t size)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_readlink_cbk, subvol,
-		    subvol->fops->readlink, loc, size);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_unlink (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_unlink_cbk, subvol, subvol->fops->unlink, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_rmdir (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_rmdir_cbk, subvol, subvol->fops->rmdir, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_rename (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	int32_t op_errno = 1;
-	xlator_t *old_subvol = NULL;
-	xlator_t *new_subvol = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-	old_subvol = get_mapping_subvol_from_ctx (this, oldloc->inode);
-	if (!old_subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	if (newloc->path) {
-		new_subvol = get_mapping_subvol_from_path (this, newloc->path);
-		if (new_subvol && (new_subvol != old_subvol)) {
-			op_errno = EXDEV;
-			goto err;
-		}
-	}
-
-	STACK_WIND (frame, map_rename_cbk, old_subvol,
-		    old_subvol->fops->rename, oldloc, newloc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_link (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *oldloc,
-	  loc_t *newloc)
-{
-	int32_t op_errno = 1;
-	xlator_t *old_subvol = NULL;
-	xlator_t *new_subvol = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-	old_subvol = get_mapping_subvol_from_ctx (this, oldloc->inode);
-	if (!old_subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	if (newloc->path) {
-		new_subvol = get_mapping_subvol_from_path (this, newloc->path);
-		if (new_subvol && (new_subvol != old_subvol)) {
-			op_errno = EXDEV;
-			goto err;
-		}
-	}
-
-	STACK_WIND (frame, map_link_cbk, old_subvol,
-		    old_subvol->fops->link, oldloc, newloc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_open (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  int32_t flags, fd_t *fd, int wbflags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_open_cbk, subvol, 
-                    subvol->fops->open, loc, flags, fd, wbflags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_readv (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   size_t size,
-	   off_t offset)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_readv_cbk, subvol,
-		    subvol->fops->readv, fd, size, offset);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_writev (call_frame_t *frame,
-	    xlator_t *this,
-	    fd_t *fd,
-	    struct iovec *vector,
-	    int32_t count,
-	    off_t off,
-            struct iobref *iobref)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_writev_cbk, subvol,
-		    subvol->fops->writev, fd, vector, count, off, iobref);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_flush (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_flush_cbk, subvol, subvol->fops->flush, fd);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_fsync (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   int32_t flags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsync_cbk, subvol,
-		    subvol->fops->fsync, fd, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fstat (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fstat_cbk, subvol, subvol->fops->fstat, fd);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_getdents (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      size_t size,
-	      off_t offset,
-	      int32_t flag)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_getdents_cbk, subvol,
-		    subvol->fops->getdents, fd, size, offset, flag);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_setdents (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      int32_t flags,
-	      dir_entry_t *entries,
-	      int32_t count)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_setdents_cbk, subvol,
-		    subvol->fops->setdents, fd, flags, entries, count);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_fsyncdir (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      int32_t flags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsyncdir_cbk, subvol,
-		    subvol->fops->fsyncdir, fd, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-
-
-int32_t
-map_setxattr (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *dict,
-	      int32_t flags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_setxattr_cbk, subvol,
-		    subvol->fops->setxattr, loc, dict, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_getxattr (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      const char *name)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_getxattr_cbk, subvol,
-		    subvol->fops->getxattr, loc, name);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_fsetxattr (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               dict_t *dict,
-               int32_t flags)
-{
-	int32_t   op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsetxattr_cbk, subvol, 
-                    subvol->fops->fsetxattr, fd, dict, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fgetxattr (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               const char *name)
-{
-	int32_t   op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fgetxattr_cbk, subvol,
-		    subvol->fops->fgetxattr, fd, name);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_xattrop (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     gf_xattrop_flags_t flags,
-	     dict_t *dict)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_xattrop_cbk, subvol,
-		    subvol->fops->xattrop, loc, flags, dict);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fxattrop (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      gf_xattrop_flags_t flags,
-	      dict_t *dict)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fxattrop_cbk, subvol,
-		    subvol->fops->fxattrop, fd, flags, dict);
-
-        return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_removexattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 const char *name)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_removexattr_cbk, subvol,
-		    subvol->fops->removexattr, loc, name);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_lk (call_frame_t *frame,
-	xlator_t *this,
-	fd_t *fd,
-	int32_t cmd,
-	struct gf_flock *lock)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_lk_cbk, subvol,
-		    subvol->fops->lk, fd, cmd, lock);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_inodelk (call_frame_t *frame, xlator_t *this,
-	     const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_inodelk_cbk, subvol,
-		    subvol->fops->inodelk, volume, loc, cmd, lock);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_finodelk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_finodelk_cbk, subvol,
-		    subvol->fops->finodelk, volume, fd, cmd, lock);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_entrylk (call_frame_t *frame, xlator_t *this,
-	     const char *volume, loc_t *loc, const char *basename,
-	     entrylk_cmd cmd, entrylk_type type)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_entrylk_cbk, subvol,
-		    subvol->fops->entrylk, volume, loc, basename, cmd, type);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fentrylk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, fd_t *fd, const char *basename,
-	      entrylk_cmd cmd, entrylk_type type)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fentrylk_cbk, subvol,
-		    subvol->fops->fentrylk, volume, fd, basename, cmd, type);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_checksum (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flag)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_checksum_cbk, subvol,
-		    subvol->fops->checksum, loc, flag);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_mknod (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   mode_t mode,
-	   dev_t rdev)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-	
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_newentry_cbk, subvol,
-		    subvol->fops->mknod, loc, mode, rdev);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_mkdir (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   mode_t mode)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_newentry_cbk, subvol,
-		    subvol->fops->mkdir, loc, mode);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_symlink (call_frame_t *frame,
-	     xlator_t *this,
-	     const char *linkpath,
-	     loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-	
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_newentry_cbk, subvol,
-		    subvol->fops->symlink, linkpath, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_create (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    mode_t mode, fd_t *fd)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_create_cbk, subvol,
-		    subvol->fops->create, loc, flags, mode, fd);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_lookup (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    dict_t *xattr_req)
-{
-	int32_t op_errno = EINVAL;
-	xlator_t *subvol   = NULL;
-	map_local_t *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	priv = this->private;
-
-	if (loc->inode->ino == 1)
-		goto root_inode;
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		subvol = get_mapping_subvol_from_path (this, loc->path);
-		if (!subvol) {
-			goto err;
-		}
-
-		op_errno = inode_ctx_put (loc->inode, this, 
-					  (uint64_t)(long)subvol);
-		if (op_errno != 0) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: failed to set subvolume in inode ctx",
-				loc->path);
-		}
-	}
-
-	/* Just one callback */
-	STACK_WIND (frame, map_single_lookup_cbk, subvol,
-		    subvol->fops->lookup, loc, xattr_req);
-
-	return 0;
-
- root_inode:
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-
-	frame->local = local;
-	local->call_count = priv->child_count;
-	local->op_ret = -1;
-
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame, map_root_lookup_cbk, trav->xlator,
-			    trav->xlator->fops->lookup, loc, xattr_req);
-		trav = trav->next;
-	}
-
-	return 0;
-
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_statfs (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	int32_t op_errno = EINVAL;
-	xlator_t *subvol   = NULL;
-	map_local_t *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	if (loc->inode->ino == 1)
-		goto root_inode;
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		goto err;
-	}
-	
-	/* Just one callback */
-	STACK_WIND (frame, map_statfs_cbk, subvol, subvol->fops->statfs, loc);
-
-	return 0;
-
- root_inode:
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-
-	priv = this->private;
-	frame->local = local;
-	local->call_count = priv->child_count;
-	local->op_ret = -1;
-
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame, map_statfs_cbk, trav->xlator,
-			    trav->xlator->fops->statfs, loc);
-		trav = trav->next;
-	}
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-int32_t
-map_opendir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc, fd_t *fd)
-{
-	int32_t op_errno = EINVAL;
-	xlator_t *subvol   = NULL;
-	map_local_t *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	if (loc->inode->ino == 1) 
-		goto root_inode;
-	
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		goto err;
-	}
-	
-	/* Just one callback */
-	STACK_WIND (frame, map_opendir_cbk, subvol,
-		    subvol->fops->opendir, loc, fd);
-
-	return 0;
-
- root_inode:
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-
-	priv = this->private;
-	frame->local = local;
-	local->call_count = priv->child_count;
-	local->op_ret = -1;
-	local->fd = fd_ref (fd);
-
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame, map_opendir_cbk, trav->xlator,
-			    trav->xlator->fops->opendir, loc, fd);
-		trav = trav->next;
-	}
-	
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop)
-{
-	int32_t        op_errno = EINVAL;
-	xlator_t      *subvol = NULL;
-	map_local_t   *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_t      *xvol = NULL;
-	off_t          xoff = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	if (fd->inode->ino == 1) 
-		goto root_inode;
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		goto err;
-	}
-	
-	/* Just one callback */
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND (frame, map_single_readdir_cbk, subvol,
-                            subvol->fops->readdir, fd, size, yoff);
-        else
-                STACK_WIND (frame, map_single_readdirp_cbk, subvol,
-                            subvol->fops->readdirp, fd, size, yoff);
-
-	return 0;
-
- root_inode:
-	/* readdir on '/' */
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
-	
-	priv = this->private;
-	frame->local = local;
-	local->op_errno = ENOENT;
-	local->op_ret = -1;
-
-	local->fd = fd_ref (fd);
-	local->size = size;
-
-	map_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
-
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND (frame, map_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, xoff);
-        else
-                STACK_WIND (frame, map_readdirp_cbk, xvol, xvol->fops->readdirp,
-                            fd, size, xoff);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-
-
-int32_t
-map_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff)
-{
-        map_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIR);
-        return 0;
-}
-
-int32_t
-map_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff)
-{
-        map_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP);
-        return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	map_private_t *priv = NULL;
-	struct map_pattern *trav_map = NULL;
-	struct map_pattern *tmp_map  = NULL;
-
-	priv = this->private;
-
-	if (priv) {
-		GF_FREE (priv->xlarray);
-
-		trav_map = priv->map;
-		while (trav_map) {
-			tmp_map = trav_map;
-			trav_map = trav_map->next;
-			GF_FREE (tmp_map);
-		}
-
-		GF_FREE(priv);
-	}
-
-	return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_map_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int
-init (xlator_t *this)
-{
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-	int   count = 0;
-	int   ret = -1;
-	char *pattern_string = NULL;
-	char *map_pair_str = NULL;
-	char *tmp_str = NULL;
-	char *tmp_str1 = NULL;
-	char *dup_map_pair = NULL;
-	char *dir_str = NULL;
-	char *subvol_str = NULL;
-	char *map_xl = NULL;
-
-
-	if (!this->children) {
-		gf_log (this->name,GF_LOG_ERROR,
-			"FATAL: map should have one or more child defined");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	priv = GF_CALLOC (1, sizeof (map_private_t),
-                          gf_map_mt_map_private_t);
-	this->private = priv;
-
-	/* allocate xlator array */
-	trav = this->children;
-	while (trav) {
-		count++;
-		trav = trav->next;
-	}
-	priv->xlarray = GF_CALLOC (1, sizeof (struct map_xlator_array) * count,
-                                   gf_map_mt_map_xlator_array);
-	priv->child_count = count;
-
-	/* build xlator array */
-	count = 0;
-	trav = this->children;
-	while (trav) {
-		priv->xlarray[count++].xl = trav->xlator;
-		trav = trav->next;
-	}
-
-	/* map dir1:brick1;dir2:brick2;dir3:brick3;*:brick4 */
-	ret = dict_get_str (this->options, "map-directory", &pattern_string);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"map.pattern not given, can't continue");
-		goto err;
-	}
-	map_pair_str = strtok_r (pattern_string, ";", &tmp_str);
-	while (map_pair_str) {
-		dup_map_pair = gf_strdup (map_pair_str);
-		dir_str = strtok_r (dup_map_pair, ":", &tmp_str1);
-		if (!dir_str) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"directory string invalid");
-			goto err;
-		}
-		subvol_str = strtok_r (NULL, ":", &tmp_str1);
-		if (!subvol_str) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"mapping subvolume string invalid");
-			goto err;
-		}
-		ret = verify_dir_and_assign_subvol (this, 
-						    dir_str, 
-						    subvol_str);
-		if (ret != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"verification failed");
-			goto err;
-		}
-		
-		GF_FREE (dup_map_pair);
-
-		map_pair_str = strtok_r (NULL, ";", &tmp_str);
-	}
-
-	/* default-volume brick4 */
-	ret = dict_get_str (this->options, "default-volume", &map_xl);
-	if (ret == 0) {
-		ret = assign_default_subvol (this, map_xl);
-		if (ret != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"assigning default failed");
-			goto err;
-		}
-	}
-
-	verify_if_all_subvolumes_got_used (this);
-	
-	return 0;
- err:
-	fini (this);
-	return -1;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = map_lookup,
-	.mknod       = map_mknod,
-	.create      = map_create,
-
-	.stat        = map_stat,
-	.fstat       = map_fstat,
-	.truncate    = map_truncate,
-	.ftruncate   = map_ftruncate,
-	.access      = map_access,
-	.readlink    = map_readlink,
-	.setxattr    = map_setxattr,
-	.getxattr    = map_getxattr,
-	.fsetxattr   = map_fsetxattr,
-	.fgetxattr   = map_fgetxattr,
-	.removexattr = map_removexattr,
-	.open        = map_open,
-	.readv       = map_readv,
-	.writev      = map_writev,
-	.flush       = map_flush,
-	.fsync       = map_fsync,
-	.statfs      = map_statfs,
-	.lk          = map_lk,
-	.opendir     = map_opendir,
-	.readdir     = map_readdir,
-	.readdirp    = map_readdirp,
-	.fsyncdir    = map_fsyncdir,
-	.symlink     = map_symlink,
-	.unlink      = map_unlink,
-	.link        = map_link,
-	.mkdir       = map_mkdir,
-	.rmdir       = map_rmdir,
-	.rename      = map_rename,
-	.inodelk     = map_inodelk,
-	.finodelk    = map_finodelk,
-	.entrylk     = map_entrylk,
-	.fentrylk    = map_fentrylk,
-	.xattrop     = map_xattrop,
-	.fxattrop    = map_fxattrop,
-	.setdents    = map_setdents,
-	.getdents    = map_getdents,
-	.checksum    = map_checksum,
-        .setattr     = map_setattr,
-        .fsetattr    = map_fsetattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key   = {"map-directory"},  
-	  .type  = GF_OPTION_TYPE_ANY 
-	},
-	{ .key   = {"default-volume"},  
-	  .type  = GF_OPTION_TYPE_XLATOR 
-	},
-	
-	{ .key = {NULL} }
-};
diff --git a/xlators/cluster/map/src/map.h b/xlators/cluster/map/src/map.h
deleted file mode 100644
index 7703a543e28..00000000000
--- a/xlators/cluster/map/src/map.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __MAP_H__
-#define __MAP_H__
-
-#include "xlator.h"
-#include "map-mem-types.h"
-
-struct map_pattern {
-	struct map_pattern *next;
-	xlator_t           *xl;
-	char               *directory;
-	int                 dir_len;
-};
-
-struct map_xlator_array {
-	xlator_t *xl;
-	int       mapped; /* yes/no */
-};
-
-typedef struct {
-	struct map_pattern      *map;
-	xlator_t                *default_xl;
-	struct map_xlator_array *xlarray;
-	int                      child_count;
-} map_private_t;
-
-typedef struct {
-	int32_t        op_ret;
-	int32_t        op_errno;
-	int            call_count;
-	struct statvfs statvfs;
-	struct iatt    stbuf;
-	inode_t       *inode;
-	dict_t        *dict;
-	fd_t          *fd;
-
-	size_t        size;
-} map_local_t;
-
-xlator_t *map_subvol_next (xlator_t *this, xlator_t *prev);
-int map_subvol_cnt (xlator_t *this, xlator_t *subvol);
-
-int map_itransform (xlator_t *this, xlator_t *subvol, 
-		    uint64_t x, uint64_t *y_p);
-int map_deitransform (xlator_t *this, uint64_t y, 
-		      xlator_t **subvol_p, uint64_t *x_p);
-
-
-xlator_t *get_mapping_subvol_from_path (xlator_t *this, const char *path);
-xlator_t *get_mapping_subvol_from_ctx (xlator_t *this, inode_t *inode);
-
-int check_multiple_volume_entry (xlator_t *this, xlator_t *subvol);
-int verify_dir_and_assign_subvol (xlator_t *this, 
-				  const char *directory, const char *subvol);
-int assign_default_subvol (xlator_t *this, const char *default_xl);
-void verify_if_all_subvolumes_got_used (xlator_t *this);
-
-
-#endif /* __MAP_H__ */
diff --git a/xlators/cluster/stripe/Makefile.am b/xlators/cluster/stripe/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/cluster/stripe/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
deleted file mode 100644
index e732c52423c..00000000000
--- a/xlators/cluster/stripe/src/Makefile.am
+++ /dev/null
@@ -1,20 +0,0 @@
-xlator_LTLIBRARIES = stripe.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
-
-stripe_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-
-stripe_la_SOURCES = stripe.c stripe-helpers.c \
-	$(top_builddir)/xlators/lib/src/libxlator.c
-
-stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = stripe.h stripe-mem-types.h $(top_builddir)/xlators/lib/src/libxlator.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/lib/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
deleted file mode 100644
index 02ee6a43d7c..00000000000
--- a/xlators/cluster/stripe/src/stripe-helpers.c
+++ /dev/null
@@ -1,677 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <fnmatch.h>
-
-#include "stripe.h"
-#include "byte-order.h"
-#include "mem-types.h"
-#include "logging.h"
-
-void
-stripe_local_wipe (stripe_local_t *local)
-{
-        if (!local)
-                goto out;
-
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
-
-        if (local->fd)
-                fd_unref (local->fd);
-
-        if (local->inode)
-                inode_unref (local->inode);
-
-        if (local->xattr)
-                dict_unref (local->xattr);
-
-        if (local->xdata)
-                dict_unref (local->xdata);
-
-out:
-        return;
-}
-
-
-
-int
-stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
-{
-        dict_t  *dst  = NULL;
-        int64_t *ptr  = 0, *size = NULL;
-        int32_t  ret  = -1;
-
-        dst = data;
-
-        if (strcmp (key, QUOTA_SIZE_KEY) == 0) {
-                ret = dict_get_bin (dst, key, (void **)&size);
-                if (ret < 0) {
-                        size = GF_CALLOC (1, sizeof (int64_t),
-                                          gf_common_mt_char);
-                        if (size == NULL) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "memory allocation failed");
-                                goto out;
-                        }
-                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
-                        if (ret < 0) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "stripe aggregate dict set failed");
-                                GF_FREE (size);
-                                goto out;
-                        }
-                }
-
-                ptr = data_to_bin (value);
-                if (ptr == NULL) {
-                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
-                        goto out;
-                }
-
-                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
-        } else if (strcmp (key, GF_CONTENT_KEY)) {
-                /* No need to aggregate 'CONTENT' data */
-                ret = dict_set (dst, key, value);
-                if (ret)
-                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
-        }
-
-out:
-        return 0;
-}
-
-
-void
-stripe_aggregate_xattr (dict_t *dst, dict_t *src)
-{
-        if ((dst == NULL) || (src == NULL)) {
-                goto out;
-        }
-
-        dict_foreach (src, stripe_aggregate, dst);
-out:
-        return;
-}
-
-
-int32_t
-stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        int32_t              len   = 0;
-        char                *sbuf  = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!buffer || !local || !local->xattr_list)
-                goto out;
-
-        sbuf = buffer;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->xattr_len;
-
-                if (len && xattr && xattr->xattr_value) {
-                        memcpy (buffer, xattr->xattr_value, len);
-                        buffer += len;
-                        *buffer++ = ' ';
-                }
-        }
-
-        *--buffer = '\0';
-        if (total)
-                *total = buffer - sbuf;
-        ret = 0;
-
- out:
-        return ret;
-}
-
-int32_t
-stripe_free_xattr_str (stripe_local_t *local)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!local || !local->xattr_list)
-                goto out;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-
-                if (xattr && xattr->xattr_value)
-                        GF_FREE (xattr->xattr_value);
-        }
-
-        ret = 0;
- out:
-        return ret;
-}
-
-
-int32_t
-stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            void **xattr_serz)
-{
-        int32_t              ret   = -1, i = 0, len = 0;
-        dict_t              *tmp1  = NULL, *tmp2 = NULL;
-        char                *buf   = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (xattr_serz == NULL) {
-                goto out;
-        }
-
-        tmp2 = dict_new ();
-
-        if (tmp2 == NULL) {
-                goto out;
-        }
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->xattr_len;
-
-                if (len && xattr && xattr->xattr_value) {
-                        ret = dict_reset (tmp2);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "dict_reset failed (%s)",
-                                        strerror (-ret));
-                        }
-
-                        ret = dict_unserialize (xattr->xattr_value,
-                                                xattr->xattr_len,
-                                                &tmp2);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "dict_unserialize failed (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-
-                        tmp1 = dict_copy (tmp2, tmp1);
-                        if (tmp1 == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "dict_copy failed (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-                }
-        }
-
-        len = dict_serialized_length (tmp1);
-        if (len > 0) {
-                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
-                if (buf == NULL) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_serialize (tmp1, buf);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "dict_serialize failed (%s)", strerror (-ret));
-                        GF_FREE(buf);
-                        ret = -1;
-                        goto out;
-                }
-
-                *xattr_serz = buf;
-        }
-
-        ret = 0;
-out:
-        if (tmp1 != NULL) {
-                dict_unref (tmp1);
-        }
-
-        if (tmp2 != NULL) {
-                dict_unref (tmp2);
-        }
-
-        return ret;
-}
-
-
-int32_t
-stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            char **xattr_serz)
-{
-        int      ret             = -1;
-        int32_t  padding         = 0;
-        int32_t  tlen            = 0;
-        char stripe_size_str[20] = {0,};
-        char    *pathinfo_serz   = NULL;
-
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
-                goto out;
-        }
-
-        (void) snprintf (stripe_size_str, 20, "%"PRId64,
-                         (long long) (local->fctx) ? local->fctx->stripe_size : 0);
-
-        /* extra bytes for decorations (brackets and <>'s) */
-        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
-                + strlen (stripe_size_str) + 7;
-        local->xattr_total_len += (padding + 2);
-
-        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
-                                   gf_common_mt_char);
-        if (!pathinfo_serz)
-                goto out;
-
-        /* xlator info */
-        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
-                        this->name, stripe_size_str);
-
-        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot aggregate pathinfo list");
-                GF_FREE(pathinfo_serz);
-                goto out;
-        }
-
-        *(pathinfo_serz + padding + tlen) = ')';
-        *(pathinfo_serz + padding + tlen + 1) = '\0';
-
-        *xattr_serz = pathinfo_serz;
-
-        ret = 0;
- out:
-        return ret;
-}
-
-/**
- * stripe_get_matching_bs - Get the matching block size for the given path.
- */
-int32_t
-stripe_get_matching_bs (const char *path, stripe_private_t *priv)
-{
-        struct stripe_options *trav       = NULL;
-        uint64_t               block_size = 0;
-
-        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
-        GF_VALIDATE_OR_GOTO ("stripe", path, out);
-
-        LOCK (&priv->lock);
-        {
-                block_size = priv->block_size;
-                trav = priv->pattern;
-                while (trav) {
-                        if (!fnmatch (trav->path_pattern, path, FNM_NOESCAPE)) {
-                                block_size = trav->block_size;
-                                break;
-                        }
-                        trav = trav->next;
-                }
-        }
-        UNLOCK (&priv->lock);
-
-out:
-        return block_size;
-}
-
-int32_t
-stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
-                   dict_t *dict)
-{
-        char            key[256]       = {0,};
-        data_t         *data            = NULL;
-        int32_t         index           = 0;
-        stripe_private_t *priv          = NULL;
-
-        priv = this->private;
-
-
-        if (!local->fctx) {
-                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                         gf_stripe_mt_stripe_fd_ctx_t);
-                if (!local->fctx) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret = -1;
-                        goto out;
-                }
-
-                local->fctx->static_array = 0;
-        }
-        /* Stripe block size */
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-size");
-                goto out;
-        } else {
-                if (!local->fctx->stripe_size) {
-                        local->fctx->stripe_size =
-                                     data_to_int64 (data);
-                }
-
-                if (local->fctx->stripe_size != data_to_int64 (data)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "stripe-size mismatch in blocks");
-                        local->xattr_self_heal_needed = 1;
-                }
-        }
-
-        /* Stripe count */
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        data = dict_get (dict, key);
-
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-count");
-                goto out;
-        }
-        if (!local->fctx->xl_array) {
-                local->fctx->stripe_count = data_to_int32 (data);
-                if (!local->fctx->stripe_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error with stripe-count xattr");
-                        local->op_ret   = -1;
-                        local->op_errno = EIO;
-                        goto out;
-                }
-
-                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
-                                                   sizeof (xlator_t *),
-                                                   gf_stripe_mt_xlator_t);
-
-                if (!local->fctx->xl_array) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret   = -1;
-                        goto out;
-                }
-        }
-        if (local->fctx->stripe_count != data_to_int32 (data)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-count xattr (%d != %d)",
-                        local->fctx->stripe_count, data_to_int32 (data));
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-
-        /* index */
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-index");
-                goto out;
-        }
-        index = data_to_int32 (data);
-        if (index > priv->child_count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-index xattr (%d)", index);
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-        if (local->fctx->xl_array) {
-                if (!local->fctx->xl_array[index])
-                        local->fctx->xl_array[index] = prev->this;
-        }
-
-	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
-	data = dict_get(dict, key);
-	if (!data) {
-		/*
-		 * The file was probably created prior to coalesce support.
-		 * Assume non-coalesce mode for this file to maintain backwards
-		 * compatibility.
-		 */
-		gf_log(this->name, GF_LOG_DEBUG, "missing stripe-coalesce "
-			"attr, assume non-coalesce mode");
-		local->fctx->stripe_coalesce = 0;
-	} else {
-		local->fctx->stripe_coalesce = data_to_int32(data);
-	}
-
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
-                            uint32_t stripe_count, uint32_t stripe_index,
-			    uint32_t stripe_coalesce)
-{
-        char            key[256]       = {0,};
-        int32_t         ret             = -1;
-
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        ret = dict_set_int64 (dict, key, stripe_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        ret = dict_set_int32 (dict, key, stripe_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        ret = dict_set_int32 (dict, key, stripe_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
-	ret = dict_set_int32(dict, key, stripe_coalesce);
-	if (ret) {
-		gf_log(this->name, GF_LOG_WARNING,
-			"failed to set %s in xattr_req_dict", key);
-		goto out;
-	}
-out:
-        return ret;
-}
-
-
-static int
-set_default_block_size (stripe_private_t *priv, char *num)
-{
-
-        int             ret = -1;
-        GF_VALIDATE_OR_GOTO ("stripe", THIS, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, num, out);
-
-
-        if (gf_string2bytesize_uint64 (num, &priv->block_size) != 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "invalid number format \"%s\"", num);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
-
-}
-
-
-int
-set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
-{
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *stripe_str = NULL;
-        char                  *pattern = NULL;
-        char                  *num = NULL;
-        struct stripe_options *temp_stripeopt = NULL;
-        struct stripe_options *stripe_opt = NULL;
-
-        if (!this || !priv || !data)
-                goto out;
-
-        /* Get the pattern for striping.
-           "option block-size *avi:10MB" etc */
-        stripe_str = strtok_r (data, ",", &tmp_str);
-        while (stripe_str) {
-                dup_str = gf_strdup (stripe_str);
-                stripe_opt = GF_CALLOC (1, sizeof (struct stripe_options),
-                                        gf_stripe_mt_stripe_options);
-                if (!stripe_opt) {
-                        goto out;
-                }
-
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!num) {
-                        num = pattern;
-                        pattern = "*";
-                        ret = set_default_block_size (priv, num);
-                        if (ret)
-                                goto out;
-                }
-                if (gf_string2bytesize_uint64 (num, &stripe_opt->block_size) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid number format \"%s\"", num);
-                        goto out;
-                }
-
-                if (stripe_opt->block_size < STRIPE_MIN_BLOCK_SIZE) {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
-                                "%s. Should be atleast %llu bytes", num,
-                                STRIPE_MIN_BLOCK_SIZE);
-                        goto out;
-                }
-                if (stripe_opt->block_size % 512) {
-                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
-                                " be a multiple of 512 bytes", num);
-                        goto out;
-                }
-
-                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "block-size : pattern %s : size %"PRId64,
-                        stripe_opt->path_pattern, stripe_opt->block_size);
-
-                if (priv->pattern)
-                        temp_stripeopt = NULL;
-                else
-                        temp_stripeopt = priv->pattern;
-
-                stripe_opt->next = temp_stripeopt;
-
-                priv->pattern = stripe_opt;
-                stripe_opt = NULL;
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
-        }
-
-        ret = 0;
-out:
-
-        GF_FREE (dup_str);
-
-        GF_FREE (stripe_opt);
-
-        return ret;
-}
-
-int32_t
-stripe_iatt_merge (struct iatt *from, struct iatt *to)
-{
-        if (to->ia_size < from->ia_size)
-                to->ia_size = from->ia_size;
-        if (to->ia_mtime < from->ia_mtime)
-                to->ia_mtime = from->ia_mtime;
-        if (to->ia_ctime < from->ia_ctime)
-                to->ia_ctime = from->ia_ctime;
-        if (to->ia_atime < from->ia_atime)
-                to->ia_atime = from->ia_atime;
-        return 0;
-}
-
-off_t
-coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count)
-{
-	size_t line_size = 0;
-	uint64_t stripe_num = 0;
-	off_t coalesced_offset = 0;
-
-	line_size = stripe_size * stripe_count;
-	stripe_num = offset / line_size;
-
-	coalesced_offset = (stripe_num * stripe_size) +
-		(offset % stripe_size);
-
-	return coalesced_offset;
-}
-
-off_t
-uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
-		int stripe_index)
-{
-        uint64_t nr_full_stripe_chunks = 0, mod = 0;
-
-        if (!size)
-                return size;
-
-	/*
-	 * Estimate the number of fully written stripes from the
-	 * local file size. Each stripe_size chunk corresponds to
-	 * a stripe.
-	 */
-        nr_full_stripe_chunks = (size / stripe_size) * stripe_count;
-        mod = size % stripe_size;
-
-        if (!mod) {
-                /*
-		 * There is no remainder, thus we could have overestimated
-		 * the size of the file in terms of chunks. Trim the number
-		 * of chunks by the following stripe members and leave it
-		 * up to those nodes to respond with a larger size (if
-		 * necessary).
-		 */
-                nr_full_stripe_chunks -= stripe_count -
-                        (stripe_index + 1);
-                size = nr_full_stripe_chunks * stripe_size;
-        } else {
-		/*
-		 * There is a remainder and thus we own the last chunk of the
-		 * file. Add the preceding stripe members of the final stripe
-		 * along with the remainder to calculate the exact size.
-		 */
-                nr_full_stripe_chunks += stripe_index;
-                size = nr_full_stripe_chunks * stripe_size + mod;
-        }
-
-        return size;
-}
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
deleted file mode 100644
index e9ac9cf4648..00000000000
--- a/xlators/cluster/stripe/src/stripe-mem-types.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __STRIPE_MEM_TYPES_H__
-#define __STRIPE_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_stripe_mem_types_ {
-        gf_stripe_mt_iovec = gf_common_mt_end + 1,
-        gf_stripe_mt_stripe_replies,
-        gf_stripe_mt_stripe_fd_ctx_t,
-        gf_stripe_mt_char,
-        gf_stripe_mt_int8_t,
-        gf_stripe_mt_int32_t,
-        gf_stripe_mt_xlator_t,
-        gf_stripe_mt_stripe_private_t,
-        gf_stripe_mt_stripe_options,
-        gf_stripe_mt_xattr_sort_t,
-        gf_stripe_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
deleted file mode 100644
index ae175faf811..00000000000
--- a/xlators/cluster/stripe/src/stripe.c
+++ /dev/null
@@ -1,5775 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/**
- * xlators/cluster/stripe:
- *    Stripe translator, stripes the data across its child nodes,
- *    as per the options given in the volfile. The striping works
- *    fairly simple. It writes files at different offset as per
- *    calculation. So, 'ls -l' output at the real posix level will
- *    show file size bigger than the actual size. But when one does
- *    'df' or 'du <file>', real size of the file on the server is shown.
- *
- * WARNING:
- *  Stripe translator can't regenerate data if a child node gets disconnected.
- *  So, no 'self-heal' for stripe. Hence the advice, use stripe only when its
- *  very much necessary, or else, use it in combination with AFR, to have a
- *  backup copy.
- */
-#include <fnmatch.h>
-
-#include "stripe.h"
-#include "libxlator.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-struct volume_options options[];
-
-int32_t
-stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        int             callcnt = -1;
-        stripe_local_t *local   = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_DESTROY (frame);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        call_frame_t    *prev = NULL;
-
-        if (!frame || !frame->local || !cookie || !this) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        STACK_WIND (frame, stripe_sh_chown_cbk, prev->this,
-                    prev->this->fops->setattr, &local->loc,
-                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
-                        stripe_local_t *local)
-{
-        xlator_list_t    *trav   = NULL;
-        call_frame_t     *rframe = NULL;
-        stripe_local_t   *rlocal = NULL;
-        stripe_private_t *priv   = NULL;
-        dict_t           *xdata  = NULL;
-        int               ret    = 0;
-
-        if (!local || !this || !frame) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        if (!(IA_ISREG (local->stbuf.ia_type) ||
-              IA_ISDIR (local->stbuf.ia_type)))
-                return 0;
-
-        priv = this->private;
-        trav = this->children;
-        rframe = copy_frame (frame);
-        if (!rframe) {
-                goto out;
-        }
-        rlocal = mem_get0 (this->local_pool);
-        if (!rlocal) {
-                goto out;
-        }
-        rframe->local = rlocal;
-        rlocal->call_count = priv->child_count;
-        loc_copy (&rlocal->loc, &local->loc);
-        memcpy (&rlocal->stbuf, &local->stbuf, sizeof (struct iatt));
-
-        xdata = dict_new ();
-        if (!xdata)
-                goto out;
-
-        ret = dict_set_static_bin (xdata, "gfid-req", local->stbuf.ia_gfid, 16);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set gfid-req", local->loc.path);
-
-        while (trav) {
-                if (IA_ISREG (local->stbuf.ia_type)) {
-                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
-                                    trav->xlator, trav->xlator->fops->mknod,
-                                    &local->loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, 0, xdata);
-                }
-                if (IA_ISDIR (local->stbuf.ia_type)) {
-                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
-                                    trav->xlator, trav->xlator->fops->mkdir,
-                                    &local->loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, xdata);
-                }
-                trav = trav->next;
-        }
-
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
-
-out:
-        if (rframe)
-                STRIPE_STACK_DESTROY (rframe);
-        if (xdata)
-                dict_unref (xdata);
-
-        return 0;
-}
-
-
-int32_t
-stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        int32_t         callcnt     = 0;
-        stripe_local_t *local       = NULL;
-        call_frame_t   *prev        = NULL;
-        int             ret         = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        if ((op_errno != ENOENT) || (op_errno != ESTALE))
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "%s returned error %s",
-                                        prev->this->name,
-                                        strerror (op_errno));
-                        if (local->op_errno != ESTALE)
-                                local->op_errno = op_errno;
-                        if (((op_errno != ENOENT) && (op_errno != ENOTCONN)
-                              && (op_errno != ESTALE)) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        if (op_errno == ENOENT)
-                                local->entry_self_heal_needed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-                        if (IA_ISREG (buf->ia_type)) {
-                                ret = stripe_ctx_handle (this, prev, local,
-                                                         xdata);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                 "Error getting fctx info from"
-                                                 " dict");
-                        }
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf      = *buf;
-                                local->postparent = *postparent;
-                                local->inode = inode_ref (inode);
-                                if (xdata)
-                                        local->xdata = dict_ref (xdata);
-                                if (local->xattr) {
-                                        stripe_aggregate_xattr (local->xdata,
-                                                                local->xattr);
-                                        dict_unref (local->xattr);
-                                        local->xattr = NULL;
-                                }
-                        }
-
-                        if (!local->xdata && !local->xattr) {
-                                local->xattr = dict_ref (xdata);
-                        } else if (local->xdata) {
-                                stripe_aggregate_xattr (local->xdata, xdata);
-                        } else if (local->xattr) {
-                                stripe_aggregate_xattr (local->xattr, xdata);
-                        }
-
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-
-                        if (gf_uuid_is_null (local->ia_gfid))
-                                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-
-                        /* Make sure the gfid on all the nodes are same */
-                        if (gf_uuid_compare (local->ia_gfid, buf->ia_gfid)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "%s: gfid different on subvolume %s",
-                                        local->loc.path, prev->this->name);
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->op_ret == 0 && local->entry_self_heal_needed &&
-                    !gf_uuid_is_null (local->loc.inode->gfid))
-                        stripe_entry_self_heal (frame, this, local);
-
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        inode_ctx_put (local->inode, this,
-                                       (uint64_t) (long)local->fctx);
-                }
-
-                STRIPE_STACK_UNWIND (lookup, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, local->xdata,
-                                     &local->postparent);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xdata)
-{
-        stripe_local_t   *local    = NULL;
-        xlator_list_t    *trav     = NULL;
-        stripe_private_t *priv     = NULL;
-        int32_t           op_errno = EINVAL;
-        int64_t           filesize = 0;
-        int               ret      = 0;
-        uint64_t          tmpctx   = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        inode_ctx_get (local->inode, this, &tmpctx);
-        if (tmpctx)
-                local->fctx = (stripe_fd_ctx_t*) (long)tmpctx;
-
-        /* quick-read friendly changes */
-        if (xdata && dict_get (xdata, GF_CONTENT_KEY)) {
-                ret = dict_get_int64 (xdata, GF_CONTENT_KEY, &filesize);
-                if (!ret && (filesize > priv->block_size))
-                        dict_del (xdata, GF_CONTENT_KEY);
-        }
-
-        /* get stripe-size xattr on lookup. This would be required for
-         * open/read/write/pathinfo calls. Hence we send down the request
-         * even when type == IA_INVAL */
-
-	/*
-	 * We aren't guaranteed to have xdata here. We need the format info for
-	 * the file, so allocate xdata if necessary.
-	 */
-	if (!xdata)
-		xdata = dict_new();
-	else
-		xdata = dict_ref(xdata);
-
-        if (xdata && (IA_ISREG (loc->inode->ia_type) ||
-            (loc->inode->ia_type == IA_INVAL))) {
-                ret = stripe_xattr_request_build (this, xdata, 8, 4, 4, 0);
-                if (ret)
-                        gf_log (this->name , GF_LOG_ERROR, "Failed to build"
-                                " xattr request for %s", loc->path);
-
-        }
-
-        /* Every time in stripe lookup, all child nodes
-           should be looked up */
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_lookup_cbk, trav->xlator,
-                            trav->xlator->fops->lookup, loc, xdata);
-                trav = trav->next;
-        }
-
-	dict_unref(xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf = *buf;
-                        }
-
-                        local->stbuf_blocks += buf->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_size   = local->stbuf_size;
-                        local->stbuf.ia_blocks = local->stbuf_blocks;
-                }
-
-                STRIPE_STACK_UNWIND (stat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(loc->inode->ia_type)) {
-		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        while (trav) {
-                STACK_WIND (frame, stripe_stat_cbk, trav->xlator,
-                            trav->xlator->fops->stat, loc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-
-err:
-        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        int32_t         callcnt = 0;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        local = frame->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret && (op_errno != ENOTCONN)) {
-                        local->op_errno = op_errno;
-                }
-                if (op_ret == 0) {
-                        struct statvfs *dict_buf = &local->statvfs_buf;
-                        dict_buf->f_bsize   = stbuf->f_bsize;
-                        dict_buf->f_frsize  = stbuf->f_frsize;
-                        dict_buf->f_blocks += stbuf->f_blocks;
-                        dict_buf->f_bfree  += stbuf->f_bfree;
-                        dict_buf->f_bavail += stbuf->f_bavail;
-                        dict_buf->f_files  += stbuf->f_files;
-                        dict_buf->f_ffree  += stbuf->f_ffree;
-                        dict_buf->f_favail += stbuf->f_favail;
-                        dict_buf->f_fsid    = stbuf->f_fsid;
-                        dict_buf->f_flag    = stbuf->f_flag;
-                        dict_buf->f_namemax = stbuf->f_namemax;
-                        local->op_ret = 0;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (statfs, frame, local->op_ret,
-                                     local->op_errno, &local->statvfs_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        trav = this->children;
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->op_errno = ENOTCONN;
-        frame->local = local;
-
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_statfs_cbk, trav->xlator,
-                            trav->xlator->fops->statfs, loc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-
-int32_t
-stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *prebuf;
-                                local->post_buf = *postbuf;
-                        }
-
-                        local->prebuf_blocks  += prebuf->ia_blocks;
-                        local->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, local->fctx, prev);
-			correct_file_size(postbuf, local->fctx, prev);
-
-                        if (local->prebuf_size < prebuf->ia_size)
-                                local->prebuf_size = prebuf->ia_size;
-
-                        if (local->postbuf_size < postbuf->ia_size)
-                                local->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (truncate, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-	int		  i, eof_idx;
-	off_t		  dest_offset, tmp_offset;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->fctx = fctx;
-	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (!fctx->xl_array[i]) {
-			gf_log(this->name, GF_LOG_ERROR,
-				"no xlator at index %d", i);
-			op_errno = EINVAL;
-			goto err;
-		}
-
-		if (fctx->stripe_coalesce) {
-			/*
-	 		 * The node that owns EOF is truncated to the exact
-	 		 * coalesced offset. Nodes prior to this index should
-	 		 * be rounded up to the size of the complete stripe,
-	 		 * while nodes after this index should be rounded down
-			 * to the size of the previous stripe.
-			 */
-			if (i < eof_idx)
-				tmp_offset = roof(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else if (i > eof_idx)
-				tmp_offset = floor(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else
-				tmp_offset = offset;
-
-			dest_offset = coalesced_offset(tmp_offset,
-					fctx->stripe_size, fctx->stripe_count);
-		} else {
-			dest_offset = offset;
-		}
-
-		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
-			fctx->xl_array[i]->fops->truncate, loc, dest_offset,
-			NULL);
-	}
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *preop;
-                                local->post_buf = *postop;
-                        }
-
-                        local->prebuf_blocks  += preop->ia_blocks;
-                        local->postbuf_blocks += postop->ia_blocks;
-
-			correct_file_size(preop, local->fctx, prev);
-			correct_file_size(postop, local->fctx, prev);
-
-                        if (local->prebuf_size < preop->ia_size)
-                                local->prebuf_size = preop->ia_size;
-                        if (local->postbuf_size < postop->ia_size)
-                                local->postbuf_size = postop->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (setattr, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t	 *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        if (!IA_ISDIR (loc->inode->ia_type) &&
-            !IA_ISREG (loc->inode->ia_type)) {
-                local->call_count = 1;
-                STACK_WIND (frame, stripe_setattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, NULL);
-                return 0;
-        }
-
-	if (IA_ISREG(loc->inode->ia_type)) {
-		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_setattr_cbk,
-                            trav->xlator, trav->xlator->fops->setattr,
-                            loc, stbuf, valid, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_setattr_cbk, trav->xlator,
-                            trav->xlator->fops->fsetattr, fd, stbuf, valid, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        local->stbuf.ia_blocks      += buf->ia_blocks;
-                        local->preparent.ia_blocks  += preoldparent->ia_blocks;
-                        local->postparent.ia_blocks += postoldparent->ia_blocks;
-                        local->pre_buf.ia_blocks    += prenewparent->ia_blocks;
-                        local->post_buf.ia_blocks   += postnewparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf.ia_size < buf->ia_size)
-                                local->stbuf.ia_size =  buf->ia_size;
-
-                        if (local->preparent.ia_size < preoldparent->ia_size)
-                                local->preparent.ia_size = preoldparent->ia_size;
-
-                        if (local->postparent.ia_size < postoldparent->ia_size)
-                                local->postparent.ia_size = postoldparent->ia_size;
-
-                        if (local->pre_buf.ia_size < prenewparent->ia_size)
-                                local->pre_buf.ia_size = prenewparent->ia_size;
-
-                        if (local->post_buf.ia_size < postnewparent->ia_size)
-                                local->post_buf.ia_size = postnewparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent,  &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        xlator_list_t  *trav = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
-
-        local = frame->local;
-        trav = this->children;
-
-        local->stbuf      = *buf;
-        local->preparent  = *preoldparent;
-        local->postparent = *postoldparent;
-        local->pre_buf    = *prenewparent;
-        local->post_buf   = *postnewparent;
-
-        local->op_ret = 0;
-        local->call_count--;
-
-        trav = trav->next; /* Skip first child */
-        while (trav) {
-                STACK_WIND (frame, stripe_stack_rename_cbk,
-                            trav->xlator, trav->xlator->fops->rename,
-                            &local->loc, &local->loc2, NULL);
-                trav = trav->next;
-        }
-        return 0;
-
-unwind:
-        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, NULL);
-        return 0;
-}
-
-int32_t
-stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t	 *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* If any one node is down, don't allow rename */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        local->op_ret = -1;
-        loc_copy (&local->loc, oldloc);
-        loc_copy (&local->loc2, newloc);
-
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(oldloc->inode->ia_type)) {
-		inode_ctx_get(oldloc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
-                    trav->xlator->fops->rename, oldloc, newloc, NULL);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
-        return 0;
-}
-int32_t
-stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
-        local->op_ret = 0;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        STRIPE_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
-                            &local->preparent, &local->postparent, xdata);
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-
-
-
-int32_t
-stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if (op_errno != ENOENT) {
-                                local->failed = 1;
-                                local->op_ret = op_ret;
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (callcnt == 1) {
-                if (local->failed) {
-                        op_errno = local->op_errno;
-                        goto out;
-                }
-                STACK_WIND(frame, stripe_first_unlink_cbk, FIRST_CHILD (this),
-                           FIRST_CHILD (this)->fops->unlink, &local->loc,
-                           local->xflag, local->xdata);
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-int32_t
-stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int xflag, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Don't unlink a file if a node is down */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-
-        if (xdata)
-            local->xdata = dict_ref (xdata);
-
-        frame->local = local;
-        local->call_count = priv->child_count;
-        trav = trav->next; /* Skip the first child */
-
-        while (trav) {
-                STACK_WIND (frame, stripe_unlink_cbk,
-                            trav->xlator, trav->xlator->fops->unlink,
-                            loc, xflag, xdata);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (op_ret == -1) {
-                goto err;
-        }
-
-        local = frame->local;
-        local->op_ret = 0;
-
-        local->call_count--; /* First child successful */
-
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-        local->preparent_size  = preparent->ia_size;
-        local->postparent_size = postparent->ia_size;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        STRIPE_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                             &local->preparent, &local->postparent, xdata);
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
-        return 0;
-
-}
-
-int32_t
-stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        if (op_errno != ENOENT)
-                                local->failed = 1;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (callcnt == 1) {
-                if (local->failed)
-                        goto out;
-                STACK_WIND (frame, stripe_first_rmdir_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->rmdir, &local->loc,
-                            local->flags, NULL);
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* don't delete a directory if any of the subvolume is down */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-        local->flags = flags;
-        local->call_count = priv->child_count;
-        trav = trav->next; /* skip the first child */
-
-        while (trav) {
-                STACK_WIND (frame, stripe_rmdir_cbk,  trav->xlator,
-                            trav->xlator->fops->rmdir, loc, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int32_t op_ret,
-                                    int32_t op_errno, struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-/**
- */
-int32_t
-stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        call_frame_t     *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-	LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->op_ret == -1) {
-                        local->call_count = priv->child_count;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_mknod_ifreg_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-                        return 0;
-                }
-
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-
-                        /* Can be used as a mechanism to understand if mknod
-                           was successful in at least one place */
-                        if (gf_uuid_is_null (local->ia_gfid))
-                                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-
-			if (stripe_ctx_handle(this, prev, local, xdata))
-				gf_log(this->name, GF_LOG_ERROR,
-					"Error getting fctx info from dict");
-
-                        local->stbuf_blocks += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if ((local->op_ret == -1) && !gf_uuid_is_null (local->ia_gfid)) {
-                        /* ia_gfid set means, at least on one node 'mknod'
-                           is successful */
-                        local->call_count = priv->child_count;
-                        trav = this->children;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_mknod_ifreg_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-                        return 0;
-                }
-
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        inode_ctx_put (local->inode, this,
-                                       (uint64_t)(long) local->fctx);
-
-                }
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-        int               i    = 1;
-        dict_t           *dict           = NULL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-        trav = this->children;
-
-        local->call_count--;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                local->failed = 1;
-                local->op_errno = op_errno;
-                goto out;
-        }
-
-        local->op_ret = op_ret;
-
-        local->stbuf      = *buf;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-
-        if (gf_uuid_is_null (local->ia_gfid))
-                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-        local->preparent.ia_blocks  = local->preparent_blocks;
-        local->preparent.ia_size    = local->preparent_size;
-        local->postparent.ia_blocks = local->postparent_blocks;
-        local->postparent.ia_size   = local->postparent_size;
-        local->stbuf.ia_size        = local->stbuf_size;
-        local->stbuf.ia_blocks      = local->stbuf_blocks;
-
-        trav = trav->next;
-        while (trav) {
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", local->loc.path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (local->xattr, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count, i,
-							  priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to build xattr request");
-
-                } else {
-                        dict = local->xattr;
-                }
-
-                STACK_WIND (frame, stripe_mknod_ifreg_cbk,
-                            trav->xlator, trav->xlator->fops->mknod,
-                            &local->loc, local->mode, local->rdev, 0, dict);
-                trav = trav->next;
-                i++;
-
-                if (dict && need_unref)
-                        dict_unref (dict);
-        }
-
-        return 0;
-
-out:
-
-       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL, NULL);
-       return 0;
-}
-
-
-int32_t
-stripe_single_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-
-int
-stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        stripe_private_t *priv           = NULL;
-        stripe_local_t   *local          = NULL;
-        int32_t           op_errno       = EINVAL;
-        int32_t           i              = 0;
-        dict_t           *dict           = NULL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        if (S_ISREG(mode)) {
-                /* NOTE: on older kernels (older than 2.6.9),
-                   creat() fops is sent as mknod() + open(). Hence handling
-                   S_IFREG files is necessary */
-                if (priv->nodes_down) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Some node down, returning EIO");
-                        op_errno = EIO;
-                        goto err;
-                }
-
-                /* Initialization */
-                local = mem_get0 (this->local_pool);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-                local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-                frame->local = local;
-                local->inode = inode_ref (loc->inode);
-                loc_copy (&local->loc, loc);
-                local->xattr = dict_copy_with_ref (xdata, NULL);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-
-                /* Every time in stripe lookup, all child nodes should
-                   be looked up */
-                local->call_count = priv->child_count;
-
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", loc->path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (xdata, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count,
-                                                          i, priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to build xattr request");
-                } else {
-                        dict = xdata;
-                }
-
-                STACK_WIND (frame, stripe_mknod_first_ifreg_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                            loc, mode, rdev, umask, dict);
-
-                        if (dict && need_unref)
-                                dict_unref (dict);
-                return 0;
-        }
-
-        STACK_WIND (frame, stripe_single_mknod_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                }
-                STRIPE_STACK_UNWIND (mkdir, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-        xlator_list_t        *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-        trav = this->children;
-
-        local->call_count--; /* first child is successful */
-        trav = trav->next;   /* skip first child */
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                local->op_errno = op_errno;
-                goto out;
-        }
-
-        local->op_ret = 0;
-
-        local->inode      = inode_ref (inode);
-        local->stbuf      = *buf;
-        local->postparent = *postparent;
-        local->preparent  = *preparent;
-
-        local->stbuf_blocks      += buf->ia_blocks;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        local->stbuf_size = buf->ia_size;
-        local->preparent_size = preparent->ia_size;
-        local->postparent_size = postparent->ia_size;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_mkdir_cbk, trav->xlator,
-                            trav->xlator->fops->mkdir, &local->loc, local->mode,
-                            local->umask, local->xdata);
-                trav = trav->next;
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
-
-        return 0;
-
-}
-
-
-int
-stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              mode_t umask, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->call_count = priv->child_count;
-        if (xdata)
-                local->xdata = dict_ref (xdata);
-        local->mode  = mode;
-        local->umask = umask;
-        loc_copy (&local->loc, loc);
-        frame->local = local;
-
-        /* Every time in stripe lookup, all child nodes should be looked up */
-        STACK_WIND (frame, stripe_first_mkdir_cbk, trav->xlator,
-                    trav->xlator->fops->mkdir, loc, mode, umask, xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-	stripe_fd_ctx_t *fctx = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-
-			if (IA_ISREG(inode->ia_type)) {
-				inode_ctx_get(inode, this, (uint64_t *) &fctx);
-				if (!fctx) {
-					gf_log(this->name, GF_LOG_ERROR,
-						"failed to get stripe context");
-					op_ret = -1;
-					op_errno = EINVAL;
-				}
-			}
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->inode      = inode_ref (inode);
-                                local->stbuf      = *buf;
-                                local->postparent = *postparent;
-                                local->preparent  = *preparent;
-                        }
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                }
-                STRIPE_STACK_UNWIND (link, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* If any one node is down, don't allow link operation */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        /* Every time in stripe lookup, all child
-           nodes should be looked up */
-        while (trav) {
-                STACK_WIND (frame, stripe_link_cbk,
-                            trav->xlator, trav->xlator->fops->link,
-                            oldloc, newloc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, struct iatt *preparent,
-                               struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (create, frame, local->op_ret, local->op_errno,
-                                     local->fd, local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0) {
-			if (IA_ISREG(buf->ia_type)) {
-				if (stripe_ctx_handle(this, prev, local, xdata))
-					gf_log(this->name, GF_LOG_ERROR,
-						"Error getting fctx info from "
-						"dict");
-			}
-
-                        local->op_ret = op_ret;
-
-                        local->stbuf_blocks += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret == -1) {
-                        local->call_count = priv->child_count;
-                        trav = this->children;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_create_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-
-                        return 0;
-                }
-
-                if (local->op_ret >= 0) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-
-			stripe_copy_xl_array(local->fctx->xl_array,
-				             priv->xl_array,
-					     local->fctx->stripe_count);
-			inode_ctx_put(local->inode, this,
-					(uint64_t) local->fctx);
-                }
-
-                /* Create itself has failed.. so return
-                   without setxattring */
-                STRIPE_STACK_UNWIND (create, frame, local->op_ret,
-                                     local->op_errno, local->fd,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-
-out:
-        return 0;
-}
-
-
-
-int32_t
-stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-        int               i    = 1;
-        dict_t           *dict = NULL;
-        loc_t            *loc  = NULL;
-        int32_t           need_unref = 0;
-        int32_t           ret  = -1;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-        trav = this->children;
-        loc = &local->loc;
-
-        --local->call_count;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                 local->failed = 1;
-                 local->op_errno = op_errno;
-        }
-
-        local->op_ret = 0;
-        /* Get the mapping in inode private */
-        /* Get the stat buf right */
-        local->stbuf      = *buf;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-
-        local->stbuf_blocks += buf->ia_blocks;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        if (local->stbuf_size < buf->ia_size)
-              local->stbuf_size = buf->ia_size;
-        if (local->preparent_size < preparent->ia_size)
-              local->preparent_size = preparent->ia_size;
-        if (local->postparent_size < postparent->ia_size)
-              local->postparent_size = postparent->ia_size;
-
-        if (local->failed)
-                local->op_ret = -1;
-
-        if (local->op_ret == -1) {
-                local->call_count = 1;
-                STACK_WIND (frame, stripe_create_fail_unlink_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                            &local->loc, 0, NULL);
-                return 0;
-        }
-
-        if (local->op_ret >= 0) {
-                local->preparent.ia_blocks  = local->preparent_blocks;
-                local->preparent.ia_size    = local->preparent_size;
-                local->postparent.ia_blocks = local->postparent_blocks;
-                local->postparent.ia_size   = local->postparent_size;
-                local->stbuf.ia_size        = local->stbuf_size;
-                local->stbuf.ia_blocks      = local->stbuf_blocks;
-        }
-
-        /* Send a setxattr request to nodes where the
-           files are created */
-        trav = trav->next;
-        while (trav) {
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", loc->path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (local->xattr, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count,
-                                                          i, priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to build xattr request");
-                } else {
-                        dict = local->xattr;
-                }
-
-                STACK_WIND (frame, stripe_create_cbk, trav->xlator,
-                            trav->xlator->fops->create, &local->loc,
-                            local->flags, local->mode, local->umask, local->fd,
-                            dict);
-                trav = trav->next;
-                if (need_unref && dict)
-                        dict_unref (dict);
-                i++;
-        }
-
-out:
-        return 0;
-}
-
-
-
-/**
- * stripe_create - If a block-size is specified for the 'name', create the
- *    file in all the child nodes. If not, create it in only first child.
- *
- * @name- complete path of the file to be created.
- */
-int32_t
-stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        int32_t           op_errno = EINVAL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-        int               i              = 0;
-        dict_t           *dict           = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        /* files created in O_APPEND mode does not allow lseek() on fd */
-        flags &= ~O_APPEND;
-
-        if (priv->first_child_down || priv->nodes_down) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "First node down, returning EIO");
-                op_errno = EIO;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->op_errno = ENOTCONN;
-        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-        frame->local = local;
-        local->inode = inode_ref (loc->inode);
-        loc_copy (&local->loc, loc);
-        local->fd = fd_ref (fd);
-        local->flags = flags;
-        local->mode = mode;
-        local->umask = umask;
-        if (xdata)
-                local->xattr = dict_ref (xdata);
-
-        local->call_count = priv->child_count;
-        /* Send a setxattr request to nodes where the
-           files are created */
-
-        if (priv->xattr_supported) {
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to allocate dict %s", loc->path);
-                }
-                need_unref = 1;
-
-                dict_copy (xdata, dict);
-
-                ret = stripe_xattr_request_build (this, dict,
-                                                  local->stripe_size,
-                                                  priv->child_count,
-                                                  i, priv->coalesce);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to build xattr request");
-        } else {
-                        dict = xdata;
-        }
-
-
-        STACK_WIND (frame, stripe_first_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create, loc, flags, mode,
-                    umask, fd, dict);
-
-        if (need_unref && dict)
-                dict_unref (dict);
-
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, xdata);
-        return 0;
-}
-
-int32_t
-stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (open, frame, local->op_ret,
-                                     local->op_errno, local->fd, xdata);
-        }
-out:
-        return 0;
-}
-
-
-/**
- * stripe_open -
- */
-int32_t
-stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        /* files opened in O_APPEND mode does not allow lseek() on fd */
-        flags &= ~O_APPEND;
-
-        local->fd = fd_ref (fd);
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        /* Striped files */
-        local->flags = flags;
-        local->call_count = priv->child_count;
-        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-
-        while (trav) {
-                STACK_WIND (frame, stripe_open_cbk, trav->xlator,
-                            trav->xlator->fops->open,
-                            &local->loc, local->flags, local->fd,
-                            xdata);
-                trav = trav->next;
-        }
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (opendir, frame, local->op_ret,
-                                     local->op_errno, local->fd, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->call_count = priv->child_count;
-        local->fd = fd_ref (fd);
-
-        while (trav) {
-                STACK_WIND (frame, stripe_opendir_cbk, trav->xlator,
-                            trav->xlator->fops->opendir, loc, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0) {
-                        if (FIRST_CHILD(this) == prev->this) {
-                                /* First successful call, copy the *lock */
-                                local->op_ret = op_ret;
-                                local->lock = *lock;
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-                STRIPE_STACK_UNWIND (lk, frame, local->op_ret,
-                                     local->op_errno, &local->lock, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        trav = this->children;
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_lk_cbk, trav->xlator,
-                            trav->xlator->fops->lk, fd, cmd, lock, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (flush, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_flush_cbk,  trav->xlator,
-                            trav->xlator->fops->flush, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-
-int32_t
-stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *prebuf;
-                                local->post_buf = *postbuf;
-                        }
-                        local->prebuf_blocks  += prebuf->ia_blocks;
-                        local->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, local->fctx, prev);
-			correct_file_size(postbuf, local->fctx, prev);
-
-                        if (local->prebuf_size < prebuf->ia_size)
-                                local->prebuf_size = prebuf->ia_size;
-
-                        if (local->postbuf_size < postbuf->ia_size)
-                                local->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (fsync, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		op_errno = EINVAL;
-		goto err;
-	}
-	local->fctx = fctx;
-        local->op_ret = -1;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsync_cbk, trav->xlator,
-                            trav->xlator->fops->fsync, fd, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this)
-                                local->stbuf = *buf;
-
-                        local->stbuf_blocks += buf->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_size   = local->stbuf_size;
-                        local->stbuf.ia_blocks = local->stbuf_blocks;
-                }
-
-                STRIPE_STACK_UNWIND (fstat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf, NULL);
-        }
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_fstat (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(fd->inode->ia_type)) {
-		inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fstat_cbk, trav->xlator,
-                            trav->xlator->fops->fstat, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-	int		  i, eof_idx;
-	off_t		  dest_offset, tmp_offset;
-        int32_t		  op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
-		op_errno = EINVAL;
-		goto err;
-	}
-	if (!fctx->stripe_count) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe count");
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->fctx = fctx;
-	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (!fctx->xl_array[i]) {
-			gf_log(this->name, GF_LOG_ERROR, "no xlator at index "
-				"%d", i);
-			op_errno = EINVAL;
-			goto err;
-		}
-
-		if (fctx->stripe_coalesce) {
-			if (i < eof_idx)
-				tmp_offset = roof(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else if (i > eof_idx)
-				tmp_offset = floor(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else
-				tmp_offset = offset;
-
-			dest_offset = coalesced_offset(tmp_offset,
-				fctx->stripe_size, fctx->stripe_count);
-		} else {
-			dest_offset = offset;
-		}
-
-		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
-			fctx->xl_array[i]->fops->ftruncate, fd, dest_offset,
-			NULL);
-	}
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsyncdir_cbk, trav->xlator,
-                            trav->xlator->fops->fsyncdir, fd, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         i = 0;
-        int32_t         callcnt = 0;
-        int32_t         count = 0;
-        stripe_local_t *local = NULL;
-        struct iovec   *vec = NULL;
-        struct iatt     tmp_stbuf = {0,};
-        struct iobref  *tmp_iobref = NULL;
-        struct iobuf   *iobuf = NULL;
-	call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-	prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret != -1) {
-			correct_file_size(buf, local->fctx, prev);
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                op_ret = 0;
-
-                /* Keep extra space for filling in '\0's */
-                vec = GF_CALLOC ((local->count * 2), sizeof (struct iovec),
-                                 gf_stripe_mt_iovec);
-                if (!vec) {
-                        op_ret = -1;
-                        goto done;
-                }
-
-                for (i = 0; i < local->wind_count; i++) {
-                        if (local->replies[i].op_ret) {
-                                memcpy ((vec + count), local->replies[i].vector,
-                                        (local->replies[i].count * sizeof (struct iovec)));
-                                count +=  local->replies[i].count;
-                                op_ret += local->replies[i].op_ret;
-                        }
-                        if ((local->replies[i].op_ret <
-                             local->replies[i].requested_size) &&
-                            (local->stbuf_size > (local->offset + op_ret))) {
-                                /* Fill in 0s here */
-                                vec[count].iov_len  =
-                                        (local->replies[i].requested_size -
-                                         local->replies[i].op_ret);
-                                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
-                                                    vec[count].iov_len);
-                                if (!iobuf) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Out of memory.");
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        goto done;
-                                }
-                                memset (iobuf->ptr, 0, vec[count].iov_len);
-                                vec[count].iov_base = iobuf->ptr;
-
-                                iobref_add (local->iobref, iobuf);
-                                iobuf_unref(iobuf);
-
-                                op_ret += vec[count].iov_len;
-                                count++;
-                        }
-                        GF_FREE (local->replies[i].vector);
-                }
-
-                /* ENOENT signals EOF to the NFS-server */
-                if (op_ret != -1 && op_ret < local->readv_size &&
-                    (local->offset + op_ret == buf->ia_size))
-                        op_errno = ENOENT;
-
-                /* FIXME: notice that st_ino, and st_dev (gen) will be
-                 * different than what inode will have. Make sure this doesn't
-                 * cause any bugs at higher levels */
-                memcpy (&tmp_stbuf, &local->replies[0].stbuf,
-                        sizeof (struct iatt));
-                tmp_stbuf.ia_size = local->stbuf_size;
-
-        done:
-                GF_FREE (local->replies);
-                tmp_iobref = local->iobref;
-                STRIPE_STACK_UNWIND (readv, frame, op_ret, op_errno, vec,
-                                     count, &tmp_stbuf, tmp_iobref, NULL);
-
-                iobref_unref (tmp_iobref);
-                GF_FREE (vec);
-        }
-out:
-        return 0;
-}
-
-/**
- * stripe_readv_cbk - get all the striped reads, and order it properly, send it
- *        to above layer after putting it in a single vector.
- */
-int32_t
-stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
-{
-        int32_t         index = 0;
-        int32_t         callcnt = 0;
-        int32_t         final_count = 0;
-        int32_t         need_to_check_proper_size = 0;
-        call_frame_t   *mframe = NULL;
-        stripe_local_t *mlocal = NULL;
-        stripe_local_t *local = NULL;
-        struct iovec   *final_vec = NULL;
-        struct iatt     tmp_stbuf = {0,};
-        struct iatt    *tmp_stbuf_p = NULL; //need it for a warning
-        struct iobref  *tmp_iobref = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-	call_frame_t	*prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto end;
-        }
-
-        local  = frame->local;
-        index  = local->node_index;
-	prev = cookie;
-        mframe = local->orig_frame;
-        if (!mframe)
-                goto out;
-
-        mlocal = mframe->local;
-        if (!mlocal)
-                goto out;
-
-        fctx = mlocal->fctx;
-
-        LOCK (&mframe->lock);
-        {
-                mlocal->replies[index].op_ret = op_ret;
-                mlocal->replies[index].op_errno = op_errno;
-                mlocal->replies[index].requested_size = local->readv_size;
-                if (op_ret >= 0) {
-                        mlocal->replies[index].stbuf  = *stbuf;
-                        mlocal->replies[index].count  = count;
-                        mlocal->replies[index].vector = iov_dup (vector, count);
-
-			correct_file_size(stbuf, fctx, prev);
-
-                        if (local->stbuf_size < stbuf->ia_size)
-                                local->stbuf_size = stbuf->ia_size;
-                        local->stbuf_blocks += stbuf->ia_blocks;
-
-                        if (!mlocal->iobref)
-                                mlocal->iobref = iobref_new ();
-                        iobref_merge (mlocal->iobref, iobref);
-                }
-                callcnt = ++mlocal->call_count;
-        }
-        UNLOCK(&mframe->lock);
-
-        if (callcnt == mlocal->wind_count) {
-                op_ret = 0;
-
-                for (index=0; index < mlocal->wind_count; index++) {
-                        /* check whether each stripe returned
-                         * 'expected' number of bytes */
-                        if (mlocal->replies[index].op_ret == -1) {
-                                op_ret = -1;
-                                op_errno = mlocal->replies[index].op_errno;
-                                break;
-                        }
-                        /* TODO: handle the 'holes' within the read range
-                           properly */
-                        if (mlocal->replies[index].op_ret <
-                            mlocal->replies[index].requested_size) {
-                                need_to_check_proper_size = 1;
-                        }
-
-                        op_ret       += mlocal->replies[index].op_ret;
-                        mlocal->count += mlocal->replies[index].count;
-                }
-                if (op_ret == -1)
-                        goto done;
-                if (need_to_check_proper_size)
-                        goto check_size;
-
-                final_vec = GF_CALLOC (mlocal->count, sizeof (struct iovec),
-                                       gf_stripe_mt_iovec);
-
-                if (!final_vec) {
-                        op_ret = -1;
-                        goto done;
-                }
-
-                for (index = 0; index < mlocal->wind_count; index++) {
-                        memcpy ((final_vec + final_count),
-                                mlocal->replies[index].vector,
-                                (mlocal->replies[index].count *
-                                 sizeof (struct iovec)));
-                        final_count +=  mlocal->replies[index].count;
-                        GF_FREE (mlocal->replies[index].vector);
-                }
-
-                /* FIXME: notice that st_ino, and st_dev (gen) will be
-                 * different than what inode will have. Make sure this doesn't
-                 * cause any bugs at higher levels */
-                memcpy (&tmp_stbuf, &mlocal->replies[0].stbuf,
-                        sizeof (struct iatt));
-                tmp_stbuf.ia_size = local->stbuf_size;
-                tmp_stbuf.ia_blocks = local->stbuf_blocks;
-
-        done:
-                /* */
-                GF_FREE (mlocal->replies);
-                tmp_iobref = mlocal->iobref;
-                /* work around for nfs truncated read. Bug 3774 */
-                tmp_stbuf_p = &tmp_stbuf;
-                WIPE (tmp_stbuf_p);
-                STRIPE_STACK_UNWIND (readv, mframe, op_ret, op_errno, final_vec,
-                                     final_count, &tmp_stbuf, tmp_iobref, NULL);
-
-                iobref_unref (tmp_iobref);
-                GF_FREE (final_vec);
-        }
-
-        goto out;
-
-check_size:
-        mlocal->call_count = fctx->stripe_count;
-
-        for (index = 0; index < fctx->stripe_count; index++) {
-                STACK_WIND (mframe, stripe_readv_fstat_cbk,
-                            (fctx->xl_array[index]),
-                            (fctx->xl_array[index])->fops->fstat,
-                            mlocal->fd, NULL);
-        }
-
-out:
-        STRIPE_STACK_DESTROY (frame);
-end:
-        return 0;
-}
-
-
-int32_t
-stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-        int32_t           op_errno = EINVAL;
-        int32_t           idx = 0;
-        int32_t           index = 0;
-        int32_t           num_stripe = 0;
-        int32_t           off_index = 0;
-        size_t            frame_size = 0;
-        off_t             rounded_end = 0;
-        uint64_t          tmp_fctx = 0;
-        uint64_t          stripe_size = 0;
-        off_t             rounded_start = 0;
-        off_t             frame_offset = offset;
-	off_t		  dest_offset = 0;
-        stripe_local_t   *local = NULL;
-        call_frame_t     *rframe = NULL;
-        stripe_local_t   *rlocal = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EBADFD;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                goto err;
-        }
-        /* The file is stripe across the child nodes. Send the read request
-         * to the child nodes appropriately after checking which region of
-         * the file is in which child node. Always '0-<stripe_size>' part of
-         * the file resides in the first child.
-         */
-        rounded_start = floor (offset, stripe_size);
-        rounded_end = roof (offset+size, stripe_size);
-        num_stripe = (rounded_end- rounded_start)/stripe_size;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-
-        /* This is where all the vectors should be copied. */
-        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
-                                    gf_stripe_mt_stripe_replies);
-        if (!local->replies) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        off_index = (offset / stripe_size) % fctx->stripe_count;
-        local->wind_count = num_stripe;
-        local->readv_size = size;
-        local->offset     = offset;
-        local->fd         = fd_ref (fd);
-        local->fctx       = fctx;
-
-        for (index = off_index; index < (num_stripe + off_index); index++) {
-                rframe = copy_frame (frame);
-                rlocal = mem_get0 (this->local_pool);
-                if (!rlocal) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                frame_size = min (roof (frame_offset+1, stripe_size),
-                                  (offset + size)) - frame_offset;
-
-                rlocal->node_index = index - off_index;
-                rlocal->orig_frame = frame;
-                rlocal->readv_size = frame_size;
-                rframe->local = rlocal;
-                idx = (index % fctx->stripe_count);
-
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(frame_offset,
-				stripe_size, fctx->stripe_count);
-		else
-			dest_offset = frame_offset;
-
-                STACK_WIND (rframe, stripe_readv_cbk, fctx->xl_array[idx],
-                            fctx->xl_array[idx]->fops->readv,
-                            fd, frame_size, dest_offset, flags, xdata);
-
-                frame_offset += frame_size;
-        }
-
-        return 0;
-err:
-        if (rframe)
-                STRIPE_STACK_DESTROY (rframe);
-
-        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-	struct stripe_replies *reply = NULL;
-	int32_t		i = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-		mlocal->replies[local->node_index].op_ret = op_ret;
-		mlocal->replies[local->node_index].op_errno = op_errno;
-
-                if (op_ret >= 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-		/*
-		 * Only return the number of consecutively written bytes up until
-		 * the first error. Only return an error if it occurs first.
-		 *
-		 * When a short write occurs, the application should retry at the
-		 * appropriate offset, at which point we'll potentially pass back
-		 * the error.
-		 */
-		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
-			i++, reply++) {
-			if (reply->op_ret == -1) {
-				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
-					"returned error %s", i,
-					strerror(reply->op_errno));
-				if (!mlocal->op_ret) {
-					mlocal->op_ret = -1;
-					mlocal->op_errno = reply->op_errno;
-				}
-				break;
-			}
-
-			mlocal->op_ret += reply->op_ret;
-
-			if (reply->op_ret < reply->requested_size)
-				break;
-		}
-
-		GF_FREE(mlocal->replies);
-
-                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iovec *vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        struct iovec     *tmp_vec = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           total_size = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        int32_t           tmp_count = count;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	off_t		  rounded_start = 0;
-	off_t		  rounded_end = 0;
-	int32_t		  total_chunks = 0;
-	call_frame_t	  *wframe = NULL;
-	stripe_local_t	  *wlocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        /* File has to be stripped across the child nodes */
-        for (idx = 0; idx< count; idx ++) {
-                total_size += vector[idx].iov_len;
-        }
-        remaining_size = total_size;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-	rounded_start = floor(offset, stripe_size);
-	rounded_end = roof(offset + total_size, stripe_size);
-	total_chunks = (rounded_end - rounded_start) / stripe_size;
-	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
-				gf_stripe_mt_stripe_replies);
-	if (!local->replies) {
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	total_chunks = 0;
-        while (1) {
-		wframe = copy_frame(frame);
-		wlocal = mem_get0(this->local_pool);
-		if (!wlocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		wlocal->orig_frame = frame;
-		wframe->local = wlocal;
-
-                /* Send striped chunk of the vector to child
-                   nodes appropriately. */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                tmp_count = iov_subset (vector, count, offset_offset,
-                                        offset_offset + fill_size, NULL);
-                tmp_vec = GF_CALLOC (tmp_count, sizeof (struct iovec),
-                                     gf_stripe_mt_iovec);
-                if (!tmp_vec) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                tmp_count = iov_subset (vector, count, offset_offset,
-                                        offset_offset + fill_size, tmp_vec);
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		/*
-		 * Store off the request index (with respect to the chunk of the
-		 * initial offset) and the size of the request. This is required
-		 * in the callback to calculate an appropriate return value in
-		 * the event of a write failure in one or more requests.
-		 */
-		wlocal->node_index = total_chunks;
-		local->replies[total_chunks].requested_size = fill_size;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
-                            fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
-                            tmp_count, dest_offset, flags, iobref,
-                            xdata);
-
-                GF_FREE (tmp_vec);
-                offset_offset += fill_size;
-		total_chunks++;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (wframe)
-		STRIPE_STACK_DESTROY(wframe);
-
-        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-		/* return the first failure */
-		if (mlocal->op_ret == 0) {
-			mlocal->op_ret = op_ret;
-			mlocal->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		 off_t offset, size_t len, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	call_frame_t	  *fframe = NULL;
-	stripe_local_t	  *flocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-		fframe = copy_frame(frame);
-		flocal = mem_get0(this->local_pool);
-		if (!flocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		flocal->orig_frame = frame;
-		fframe->local = flocal;
-
-		/* send fallocate request to the associated child node */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-		/*
-		 * TODO: Create a separate handler for coalesce mode that sends a
-		 * single fallocate per-child (since the ranges are linear).
-		 */
-		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
-			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
-			   dest_offset, fill_size, xdata);
-
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (fframe)
-		STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-		/* return the first failure */
-		if (mlocal->op_ret == 0) {
-			mlocal->op_ret = op_ret;
-			mlocal->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       size_t len, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	call_frame_t	  *fframe = NULL;
-	stripe_local_t	  *flocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-		fframe = copy_frame(frame);
-		flocal = mem_get0(this->local_pool);
-		if (!flocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		flocal->orig_frame = frame;
-		fframe->local = flocal;
-
-		/* send discard request to the associated child node */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-		/*
-		 * TODO: Create a separate handler for coalesce mode that sends a
-		 * single discard per-child (since the ranges are linear).
-		 */
-		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
-			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
-			   fill_size, xdata);
-
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (fframe)
-		STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt    = 0;
-        stripe_local_t *local      = NULL;
-        stripe_local_t *mlocal     = NULL;
-        call_frame_t   *prev       = NULL;
-        call_frame_t   *mframe     = NULL;
-
-        GF_ASSERT (frame);
-
-        if (!this || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-        mframe = local->orig_frame;
-        mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
-                        mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-                        correct_file_size(prebuf, mlocal->fctx, prev);
-                        correct_file_size(postbuf, mlocal->fctx, prev);
-
-                        if (mlocal->prebuf_size < prebuf->ia_size)
-                                mlocal->prebuf_size = prebuf->ia_size;
-                        if (mlocal->postbuf_size < postbuf->ia_size)
-                                mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-                /* return the first failure */
-                if (mlocal->op_ret == 0) {
-                        mlocal->op_ret = op_ret;
-                        mlocal->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-                mlocal->post_buf.ia_size = mlocal->postbuf_size;
-                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-        STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               off_t len, dict_t *xdata)
-{
-        stripe_local_t   *local            = NULL;
-        stripe_fd_ctx_t  *fctx             = NULL;
-        int32_t           op_errno         = 1;
-        int32_t           idx              = 0;
-        int32_t           offset_offset    = 0;
-        int32_t           remaining_size   = 0;
-        off_t             fill_size        = 0;
-        uint64_t          stripe_size      = 0;
-        uint64_t          tmp_fctx         = 0;
-        off_t             dest_offset      = 0;
-        call_frame_t     *fframe           = NULL;
-        stripe_local_t   *flocal           = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-        local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-                fframe = copy_frame(frame);
-                flocal = mem_get0(this->local_pool);
-                if (!flocal) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                flocal->orig_frame = frame;
-                fframe->local = flocal;
-
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-                dest_offset = offset + offset_offset;
-                if (fctx->stripe_coalesce)
-                        dest_offset = coalesced_offset(dest_offset,
-                                                       local->stripe_size,
-                                                       fctx->stripe_count);
-
-                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
-                           fctx->xl_array[idx]->fops->zerofill, fd,
-                           dest_offset, fill_size, xdata);
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-        if (fframe)
-                STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             gf_seek_what_t what, dict_t *xdata)
-{
-        /* TBD */
-        gf_log (this->name, GF_LOG_INFO, "seek called on %s.",
-                uuid_utoa (fd->inode->gfid));
-        STRIPE_STACK_UNWIND (seek, frame, -1, ENOTSUP, 0, NULL);
-        return 0;
-}
-
-int32_t
-stripe_release (xlator_t *this, fd_t *fd)
-{
-	return 0;
-}
-
-int
-stripe_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t          tmp_fctx = 0;
-        stripe_fd_ctx_t  *fctx = NULL;
-
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (inode, err);
-
-        (void) inode_ctx_del (inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                goto err;
-        }
-
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-
-        if (!fctx->static_array)
-                GF_FREE (fctx->xl_array);
-
-        GF_FREE (fctx);
-err:
-        return 0;
-}
-
-int32_t
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        stripe_private_t *priv = NULL;
-        int               down_client = 0;
-        int               i = 0;
-        gf_boolean_t      heard_from_all_children = _gf_false;
-
-        if (!this)
-                return 0;
-
-        priv = this->private;
-        if (!priv)
-                return 0;
-
-        switch (event)
-        {
-        case GF_EVENT_CHILD_UP:
-        {
-                /* get an index number to set */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (data == priv->xl_array[i])
-                                break;
-                }
-
-                if (priv->child_count == i) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "got GF_EVENT_CHILD_UP bad subvolume %s",
-                                data? ((xlator_t *)data)->name: NULL);
-                        break;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        if (data == FIRST_CHILD (this))
-                                priv->first_child_down = 0;
-                        priv->last_event[i] = event;
-                }
-                UNLOCK (&priv->lock);
-        }
-        break;
-        case GF_EVENT_CHILD_CONNECTING:
-        {
-                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
-                goto out;
-        }
-        case GF_EVENT_CHILD_DOWN:
-        {
-                /* get an index number to set */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (data == priv->xl_array[i])
-                                break;
-                }
-
-                if (priv->child_count == i) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
-                                data? ((xlator_t *)data)->name: NULL);
-                        break;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        if (data == FIRST_CHILD (this))
-                                priv->first_child_down = 1;
-                        priv->last_event[i] = event;
-                }
-                UNLOCK (&priv->lock);
-        }
-        break;
-
-        default:
-        {
-                /* */
-                default_notify (this, event, data);
-                goto out;
-        }
-        break;
-        }
-
-        // Consider child as down if it's last_event is not CHILD_UP
-        for (i = 0, down_client = 0; i < priv->child_count; i++)
-                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
-                        down_client++;
-
-        LOCK (&priv->lock);
-        {
-                priv->nodes_down = down_client;
-        }
-        UNLOCK (&priv->lock);
-
-        heard_from_all_children = _gf_true;
-        for (i = 0; i < priv->child_count; i++)
-                if (!priv->last_event[i])
-                        heard_from_all_children = _gf_false;
-
-        if (heard_from_all_children)
-                default_notify (this, event, data);
-out:
-        return 0;
-}
-
-int
-stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
-{
-        int ret = -1;
-        int call_cnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
-                return ret;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_cnt = --local->wind_count;
-
-                /**
-                 * We overwrite ->op_* values here for subsequent faliure
-                 * conditions, hence we propagate the last errno down the
-                 * stack.
-                 */
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-        }
-
- unlock:
-        UNLOCK (&frame->lock);
-
-        if (!call_cnt) {
-                STRIPE_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                     local->op_errno, xdata);
-        }
-
-        return 0;
-}
-
-#ifdef HAVE_BD_XLATOR
-int
-stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
-{
-        gf_boolean_t *is_bd = data;
-
-        if (data == NULL)
-                return 0;
-
-        if (XATTR_IS_BD (key))
-            *is_bd = _gf_true;
-
-        return 0;
-}
-
-static gf_boolean_t
-stripe_setxattr_is_bd (dict_t *dict)
-{
-        gf_boolean_t is_bd = _gf_false;
-
-        if (dict == NULL)
-                goto out;
-
-        dict_foreach (dict, stripe_is_bd, &is_bd);
-out:
-        return is_bd;
-}
-#else
-#define stripe_setxattr_is_bd(dict) _gf_false
-#endif
-
-int
-stripe_setxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t           op_errno = EINVAL;
-        xlator_list_t    *trav     = NULL;
-        stripe_private_t *priv     = NULL;
-        stripe_local_t   *local    = NULL;
-        int               i        = 0;
-        gf_boolean_t      is_bd    = _gf_false;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   op_errno, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-        local->wind_count = priv->child_count;
-        local->op_ret = local->op_errno = 0;
-
-        is_bd = stripe_setxattr_is_bd (dict);
-
-        /**
-         * Set xattrs for directories on all subvolumes. Additionally
-         * this power is only given to a special client. Bd xlator
-         * also needs xattrs for regular files (ie LVs)
-         */
-        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
-             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
-                for (i = 0; i < priv->child_count; i++, trav = trav->next) {
-                        STACK_WIND (frame, stripe_setxattr_cbk,
-                                    trav->xlator, trav->xlator->fops->setxattr,
-                                    loc, dict, flags, xdata);
-                }
-        } else {
-                local->wind_count = 1;
-                STACK_WIND (frame, stripe_setxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setxattr,
-                            loc, dict, flags, xdata);
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno, NULL);
-        return 0;
-}
-
-
-int
-stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-stripe_is_special_key (dict_t *this,
-                       char *key,
-                       data_t *value,
-                       void *data)
-{
-        gf_boolean_t *is_special = NULL;
-
-        if (data == NULL) {
-                goto out;
-        }
-
-        is_special = data;
-
-        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
-                *is_special = _gf_true;
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xdata)
-{
-        int             call_count = 0;
-        stripe_local_t *local      = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_count = --local->wind_count;
-
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-        return 0;
-}
-
-int
-stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                              dict_t *dict, int flags, dict_t *xdata)
-{
-        xlator_list_t    *trav  = NULL;
-        stripe_private_t *priv  = NULL;
-        int               ret   = -1;
-        stripe_local_t   *local = NULL;
-
-        priv = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                goto out;
-        }
-
-        frame->local = local;
-
-        local->wind_count = priv->child_count;
-
-        trav = this->children;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
-                            trav->xlator, trav->xlator->fops->fsetxattr,
-                            fd, dict, flags, xdata);
-                trav = trav->next;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-static gf_boolean_t
-stripe_fsetxattr_is_special (dict_t *dict)
-{
-        gf_boolean_t is_spl = _gf_false;
-
-        if (dict == NULL) {
-                goto out;
-        }
-
-        dict_foreach (dict, stripe_is_special_key, &is_spl);
-
-out:
-        return is_spl;
-}
-
-int
-stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
-        gf_boolean_t is_spl = _gf_false;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   op_errno, err);
-
-        is_spl = stripe_fsetxattr_is_special (dict);
-        if (is_spl) {
-                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
-                                                    flags, xdata);
-                if (ret < 0) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                goto out;
-        }
-
-        STACK_WIND (frame, stripe_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-out:
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
-}
-
-int
-stripe_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-stripe_removexattr (call_frame_t *frame, xlator_t *this,
-                    loc_t *loc, const char *name, dict_t *xdata)
-{
-        int32_t         op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (this, err);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
-                                 name, op_errno, err);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        STACK_WIND (frame, stripe_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
-        return 0;
-}
-
-
-int
-stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
-                     fd_t *fd, const char *name, dict_t *xdata)
-{
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
-                                 name, op_errno, err);
-
-        STACK_WIND (frame, stripe_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
- err:
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-stripe_readdirp_lookup_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int op_ret, int op_errno,
-                            inode_t *inode, struct iatt *stbuf,
-                            dict_t *xattr, struct iatt *parent)
-{
-        stripe_local_t          *local          = NULL;
-        call_frame_t            *main_frame     = NULL;
-        stripe_local_t          *main_local     = NULL;
-        gf_dirent_t             *entry          = NULL;
-        call_frame_t            *prev           = NULL;
-        int                      done           = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        entry = local->dirent;
-
-        main_frame = local->orig_frame;
-        main_local = main_frame->local;
-        LOCK (&frame->lock);
-        {
-
-                local->call_count--;
-                if (!local->call_count)
-                        done = 1;
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret = op_ret;
-                        goto unlock;
-                }
-
-		if (stripe_ctx_handle(this, prev, local, xattr))
-			gf_log(this->name, GF_LOG_ERROR,
-				"Error getting fctx info from dict.");
-
-		correct_file_size(stbuf, local->fctx, prev);
-
-                stripe_iatt_merge (stbuf, &entry->d_stat);
-                local->stbuf_blocks += stbuf->ia_blocks;
-        }
-unlock:
-        UNLOCK(&frame->lock);
-
-        if (done) {
-                inode_ctx_put (entry->inode, this,
-                               (uint64_t) (long)local->fctx);
-
-                done = 0;
-                LOCK (&main_frame->lock);
-                {
-                        main_local->wind_count--;
-                        if (!main_local->wind_count)
-                                done = 1;
-                        if (local->op_ret == -1) {
-                                main_local->op_errno = local->op_errno;
-                                main_local->op_ret = local->op_ret;
-                        }
-                        entry->d_stat.ia_blocks = local->stbuf_blocks;
-                }
-                UNLOCK (&main_frame->lock);
-                if (done) {
-                        main_frame->local = NULL;
-                        STRIPE_STACK_UNWIND (readdir, main_frame,
-                                             main_local->op_ret,
-                                             main_local->op_errno,
-                                             &main_local->entries, NULL);
-                        gf_dirent_free (&main_local->entries);
-                        stripe_local_wipe (main_local);
-                        mem_put (main_local);
-                }
-                frame->local = NULL;
-                stripe_local_wipe (local);
-                mem_put (local);
-                STRIPE_STACK_DESTROY (frame);
-        }
-
-        return 0;
-}
-
-int32_t
-stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-        gf_dirent_t    *local_entry = NULL;
-        gf_dirent_t    *tmp_entry = NULL;
-        xlator_list_t  *trav = NULL;
-        loc_t          loc = {0, };
-        int32_t        count = 0;
-        stripe_private_t *priv = NULL;
-        int32_t        subvols = 0;
-        dict_t         *xattrs = NULL;
-        call_frame_t   *local_frame = NULL;
-        stripe_local_t *local_ent = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        prev  = cookie;
-        local = frame->local;
-        trav = this->children;
-        priv = this->private;
-
-        subvols = priv->child_count;
-
-        LOCK (&frame->lock);
-        {
-                local->op_errno = op_errno;
-                local->op_ret = op_ret;
-
-                if (op_ret != -1) {
-                        list_splice_init (&orig_entries->list,
-                                          &local->entries.list);
-                        local->wind_count = op_ret;
-                }
-
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
-
-        xattrs = dict_new ();
-        if (xattrs)
-                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0, 0);
-        count = op_ret;
-        list_for_each_entry_safe (local_entry, tmp_entry,
-                                  (&local->entries.list), list) {
-
-                if (!local_entry)
-                        break;
-                if (!IA_ISREG (local_entry->d_stat.ia_type) || !local_entry->inode) {
-                        LOCK (&frame->lock);
-                        {
-                                local->wind_count--;
-                                count = local->wind_count;
-                        }
-                        UNLOCK (&frame->lock);
-                        continue;
-                }
-
-                local_frame = copy_frame (frame);
-
-                if (!local_frame) {
-                        op_errno = ENOMEM;
-                        op_ret = -1;
-                        goto out;
-                }
-
-                local_ent = mem_get0 (this->local_pool);
-                if (!local_ent) {
-                        op_errno = ENOMEM;
-                        op_ret = -1;
-                        goto out;
-                }
-
-                loc.inode = inode_ref (local_entry->inode);
-
-                gf_uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
-
-                local_ent->orig_frame = frame;
-
-                local_ent->call_count = subvols;
-
-                local_ent->dirent = local_entry;
-
-                local_frame->local = local_ent;
-
-                trav = this->children;
-                while (trav) {
-                        STACK_WIND (local_frame, stripe_readdirp_lookup_cbk,
-                                    trav->xlator, trav->xlator->fops->lookup,
-                                    &loc, xattrs);
-                        trav = trav->next;
-                }
-                loc_wipe (&loc);
-        }
-out:
-        if (!count) {
-                /* all entries are directories */
-                frame->local = NULL;
-                STRIPE_STACK_UNWIND (readdir, frame,
-                                     (local ? local->op_ret : -1),
-                                     (local ? local->op_errno : EINVAL),
-                                     (local ? &local->entries : NULL),
-                                     NULL);
-                gf_dirent_free (&local->entries);
-                stripe_local_wipe (local);
-                mem_put (local);
-        }
-        if (xattrs)
-                dict_unref (xattrs);
-        return 0;
-
-}
-int32_t
-stripe_readdirp (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        stripe_local_t  *local  = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t   *trav = NULL;
-        int             op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        local->fd = fd_ref (fd);
-
-        local->wind_count = 0;
-
-        local->count = 0;
-        local->op_ret = -1;
-        INIT_LIST_HEAD(&local->entries);
-
-        if (!trav)
-                goto err;
-
-        STACK_WIND (frame, stripe_readdirp_cbk, trav->xlator,
-                    trav->xlator->fops->readdirp, fd, size, off, xdata);
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                goto out;
-
-        ret = xlator_mem_acct_init (this, gf_stripe_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-static int
-clear_pattern_list (stripe_private_t *priv)
-{
-        struct stripe_options *prev = NULL;
-        struct stripe_options *trav = NULL;
-        int                    ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
-
-        trav = priv->pattern;
-        priv->pattern = NULL;
-        while (trav) {
-                prev = trav;
-                trav = trav->next;
-                GF_FREE (prev);
-        }
-
-        ret = 0;
- out:
-        return ret;
-
-
-}
-
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-
-        stripe_private_t *priv = NULL;
-        data_t           *data = NULL;
-        int               ret = -1;
-        volume_option_t  *opt = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (this->private);
-
-        priv = this->private;
-
-
-        ret = 0;
-        LOCK (&priv->lock);
-        {
-                ret = clear_pattern_list (priv);
-                if (ret)
-                         goto unlock;
-
-                data = dict_get (options, "block-size");
-                if (data) {
-                        ret = set_stripe_block_size (this, priv, data->data);
-                        if (ret)
-                                goto unlock;
-                } else {
-                        opt = xlator_volume_option_get (this, "block-size");
-                        if (!opt) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "option 'block-size' not found");
-                                ret = -1;
-                                goto unlock;
-                        }
-
-                        if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to set default block-size ");
-                                ret = -1;
-                                goto unlock;
-                        }
-                }
-
-		GF_OPTION_RECONF("coalesce", priv->coalesce, options, bool,
-				unlock);
-        }
- unlock:
-        UNLOCK (&priv->lock);
-        if (ret)
-                goto out;
-
-        ret = 0;
- out:
-        return ret;
-
-}
-
-/**
- * init - This function is called when xlator-graph gets initialized.
- *     The option given in volfiles are parsed here.
- * @this -
- */
-int32_t
-init (xlator_t *this)
-{
-        stripe_private_t *priv = NULL;
-        volume_option_t  *opt = NULL;
-        xlator_list_t    *trav = NULL;
-        data_t           *data = NULL;
-        int32_t           count = 0;
-        int               ret = -1;
-
-        if (!this)
-                goto out;
-
-        trav = this->children;
-        while (trav) {
-                count++;
-                trav = trav->next;
-        }
-
-        if (!count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stripe configured without \"subvolumes\" option. "
-                        "exiting");
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        if (count == 1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stripe configured with only one \"subvolumes\" option."
-                        " please check the volume. exiting");
-                goto out;
-        }
-
-        priv = GF_CALLOC (1, sizeof (stripe_private_t),
-                          gf_stripe_mt_stripe_private_t);
-
-        if (!priv)
-                goto out;
-        priv->xl_array = GF_CALLOC (count, sizeof (xlator_t *),
-                                    gf_stripe_mt_xlator_t);
-        if (!priv->xl_array)
-                goto out;
-
-        priv->last_event = GF_CALLOC (count, sizeof (int),
-                                      gf_stripe_mt_int32_t);
-        if (!priv->last_event)
-                goto out;
-
-        priv->child_count = count;
-        LOCK_INIT (&priv->lock);
-
-        trav = this->children;
-        count = 0;
-        while (trav) {
-                priv->xl_array[count++] = trav->xlator;
-                trav = trav->next;
-        }
-
-        if (count > 256) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "maximum number of stripe subvolumes supported "
-                        "is 256");
-                goto out;
-        }
-
-        ret = 0;
-        LOCK (&priv->lock);
-        {
-                opt = xlator_volume_option_get (this, "block-size");
-                if (!opt) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "option 'block-size' not found");
-                        ret = -1;
-                        goto unlock;
-                }
-                if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set default block-size ");
-                        ret = -1;
-                        goto unlock;
-                }
-                /* option stripe-pattern *avi:1GB,*pdf:16K */
-                data = dict_get (this->options, "block-size");
-                if (data) {
-                        ret = set_stripe_block_size (this, priv, data->data);
-                        if (ret)
-                                goto unlock;
-                }
-        }
- unlock:
-        UNLOCK (&priv->lock);
-        if (ret)
-                goto out;
-
-        GF_OPTION_INIT ("use-xattr", priv->xattr_supported, bool, out);
-        /* notify related */
-        priv->nodes_down = priv->child_count;
-
-	GF_OPTION_INIT("coalesce", priv->coalesce, bool, out);
-
-        this->local_pool = mem_pool_new (stripe_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-        this->private = priv;
-
-        ret = 0;
-out:
-        if (ret) {
-                if (priv) {
-                        GF_FREE (priv->xl_array);
-                        GF_FREE (priv);
-                }
-        }
-        return ret;
-}
-
-/**
- * fini -   Free all the private variables
- * @this -
- */
-void
-fini (xlator_t *this)
-{
-        stripe_private_t      *priv = NULL;
-        struct stripe_options *prev = NULL;
-        struct stripe_options *trav = NULL;
-
-        if (!this)
-                goto out;
-
-        priv = this->private;
-        if (priv) {
-                this->private = NULL;
-                GF_FREE (priv->xl_array);
-
-                trav = priv->pattern;
-                while (trav) {
-                        prev = trav;
-                        trav = trav->next;
-                        GF_FREE (prev);
-                }
-                GF_FREE (priv->last_event);
-                LOCK_DESTROY (&priv->lock);
-                GF_FREE (priv);
-        }
-
-out:
-        return;
-}
-
-int32_t
-stripe_getxattr_unwind (call_frame_t *frame,
-                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
-
-{
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
-
-int
-stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, dict_t *xattr,
-                              dict_t *xdata)
-{
-
-        char        size_key[256]  = {0,};
-        char        index_key[256] = {0,};
-        char        count_key[256] = {0,};
-        char        coalesce_key[256] = {0,};
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-
-        if (!xattr || (op_ret == -1))
-            goto out;
-
-        sprintf (size_key, "trusted.%s.stripe-size", this->name);
-        sprintf (count_key, "trusted.%s.stripe-count", this->name);
-        sprintf (index_key, "trusted.%s.stripe-index", this->name);
-	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
-
-        dict_del (xattr, size_key);
-        dict_del (xattr, count_key);
-        dict_del (xattr, index_key);
-        dict_del (xattr, coalesce_key);
-
-out:
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
-
-        return 0;
-
-}
-
-int
-stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int                     call_cnt = 0;
-        stripe_local_t         *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_cnt = --local->wind_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!xattr || (op_ret < 0))
-                goto out;
-
-        local->op_ret = 0;
-
-        if (!local->xattr) {
-                local->xattr = dict_ref (xattr);
-        } else {
-                stripe_aggregate_xattr (local->xattr, xattr);
-        }
-
-out:
-        if (!call_cnt) {
-                STRIPE_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                     local->xattr, xdata);
-        }
-
-        return 0;
-}
-
-int32_t
-stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int32_t op_ret, int32_t op_errno,
-                      dict_t *dict, dict_t *xdata)
-{
-        stripe_local_t      *local         = NULL;
-        int32_t              callcnt       = 0;
-        int32_t              ret           = -1;
-        long                 cky           = 0;
-        void                *xattr_val     = NULL;
-        void                *xattr_serz    = NULL;
-        stripe_xattr_sort_t *xattr         = NULL;
-        dict_t              *stripe_xattr  = NULL;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
-                return ret;
-        }
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        if (local->xsel[0] == '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "Empty xattr in cbk");
-                return ret;
-        }
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->wind_count;
-
-                if (!dict || (op_ret < 0))
-                        goto out;
-
-                if (!local->xattr_list)
-                        local->xattr_list = (stripe_xattr_sort_t *)
-                                GF_CALLOC (local->nallocs,
-                                           sizeof (stripe_xattr_sort_t),
-                                           gf_stripe_mt_xattr_sort_t);
-
-                if (local->xattr_list) {
-                        xattr = local->xattr_list + (int32_t) cky;
-
-                        ret = dict_get_ptr_and_len (dict, local->xsel,
-                                                    &xattr_val,
-                                                    &xattr->xattr_len);
-                        if (xattr->xattr_len == 0)
-                                goto out;
-
-                        xattr->pos = cky;
-                        xattr->xattr_value = gf_memdup (xattr_val,
-                                                        xattr->xattr_len);
-
-                        if (xattr->xattr_value != NULL)
-                                local->xattr_total_len += xattr->xattr_len + 1;
-                }
-        }
- out:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->xattr_total_len)
-                        goto unwind;
-
-                stripe_xattr = dict_new ();
-                if (!stripe_xattr)
-                        goto unwind;
-
-                /* select filler based on ->xsel */
-                if (XATTR_IS_PATHINFO (local->xsel))
-                        ret = stripe_fill_pathinfo_xattr (this, local,
-                                                          (char **)&xattr_serz);
-                else if (XATTR_IS_LOCKINFO (local->xsel)) {
-                        ret = stripe_fill_lockinfo_xattr (this, local,
-                                                          &xattr_serz);
-                } else {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unknown xattr in xattr request");
-                        goto unwind;
-                }
-
-                if (!ret) {
-                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
-                                               xattr_serz,
-                                               local->xattr_total_len);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Can't set %s key in dict",
-                                        local->xsel);
-                }
-
-        unwind:
-                /*
-                 * Among other things, STRIPE_STACK_UNWIND will free "local"
-                 * for us.  That means we can't dereference it afterward.
-                 * Fortunately, the actual result is in stripe_xattr now, so we
-                 * can simply clean up before unwinding.
-                 */
-                ret = stripe_free_xattr_str (local);
-                GF_FREE (local->xattr_list);
-                local->xattr_list = NULL;
-
-                STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
-                                     stripe_xattr, NULL);
-
-                if (stripe_xattr)
-                        dict_unref (stripe_xattr);
-        }
-
-        return ret;
-}
-
-int
-stripe_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                            xlator_t **subvols)
-{
-        xlator_t         *this  = frame->this;
-        stripe_private_t *priv  = this->private;
-        stripe_local_t   *local = frame->local;
-        int              count  = 0;
-
-        count = priv->child_count;
-        if (MARKER_XTIME_TYPE == type) {
-                if (!IA_FILE_OR_DIR (local->loc.inode->ia_type))
-                        count = 1;
-        }
-        memcpy (subvols, priv->xl_array, sizeof (*subvols) * count);
-
-        return count;
-}
-
-int32_t
-stripe_getxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
-{
-        stripe_local_t    *local    = NULL;
-        xlator_list_t     *trav     = NULL;
-        stripe_private_t  *priv     = NULL;
-        int32_t            op_errno = EINVAL;
-        int                i        = 0;
-        int                ret      = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-
-        if (name && strncmp (name, QUOTA_SIZE_KEY,
-                             strlen (QUOTA_SIZE_KEY)) == 0) {
-                local->wind_count = priv->child_count;
-
-                for (i = 0, trav=this->children; i < priv->child_count; i++,
-                             trav = trav->next) {
-                        STACK_WIND (frame, stripe_getxattr_cbk,
-                                    trav->xlator, trav->xlator->fops->getxattr,
-                                    loc, name, xdata);
-                }
-
-                return 0;
-        }
-
-        if (name && (XATTR_IS_PATHINFO (name))) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        ret = inode_ctx_get (loc->inode, this,
-                                             (uint64_t *) &local->fctx);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "stripe size unavailable from fctx"
-                                        " relying on pathinfo could lead to"
-                                        " wrong results");
-                }
-
-                local->nallocs = local->wind_count = priv->child_count;
-                (void) strncpy (local->xsel, name, strlen (name));
-
-                /**
-                 * for xattrs that need info from all childs, fill ->xsel
-                 * as above and call the filler function in cbk based on
-                 * it
-                 */
-                for (i = 0, trav = this->children; i < priv->child_count; i++,
-                     trav = trav->next) {
-                        STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
-                                           (void *) (long) i, trav->xlator,
-                                           trav->xlator->fops->getxattr,
-                                           loc, name, xdata);
-                }
-
-                return 0;
-        }
-
-        if (cluster_handle_marker_getxattr (frame, loc, name, priv->vol_uuid,
-                                            stripe_getxattr_unwind,
-                                            stripe_marker_populate_args) == 0)
-                return 0;
-
-        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-
-        return 0;
-
-err:
-        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-static gf_boolean_t
-stripe_is_special_xattr (const char *name)
-{
-        gf_boolean_t    is_spl = _gf_false;
-
-        if (!name) {
-                goto out;
-        }
-
-        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
-                      strlen (GF_XATTR_LOCKINFO_KEY))
-            || XATTR_IS_PATHINFO (name))
-                is_spl = _gf_true;
-out:
-        return is_spl;
-}
-
-int32_t
-stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                                const char *name, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv  = NULL;
-        int32_t           ret   = -1, op_errno = 0;
-        int               i     = 0;
-        xlator_list_t    *trav  = NULL;
-
-        priv = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->op_ret = -1;
-        frame->local = local;
-
-        strncpy (local->xsel, name, strlen (name));
-        local->nallocs = local->wind_count = priv->child_count;
-
-        for (i = 0, trav = this->children; i < priv->child_count; i++,
-                     trav = trav->next) {
-                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
-                                   (void *) (long) i, trav->xlator,
-                                   trav->xlator->fops->fgetxattr,
-                                   fd, name, xdata);
-        }
-
-        return 0;
-
-err:
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
-        return ret;
-}
-
-int32_t
-stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name, dict_t *xdata)
-{
-        if (stripe_is_special_xattr (name)) {
-                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
-                goto out;
-        }
-
-        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
-
-out:
-        return 0;
-}
-
-
-
-int32_t
-stripe_priv_dump (xlator_t *this)
-{
-        char                    key[GF_DUMP_MAX_BUF_LEN];
-        int                     i = 0;
-        stripe_private_t       *priv = NULL;
-        int                     ret = -1;
-        struct stripe_options  *options = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stripe", this, out);
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        ret = TRY_LOCK (&priv->lock);
-        if (ret != 0)
-                goto out;
-
-        gf_proc_dump_add_section("xlator.cluster.stripe.%s.priv", this->name);
-        gf_proc_dump_write("child_count","%d", priv->child_count);
-
-        for (i = 0; i < priv->child_count; i++) {
-                sprintf (key, "subvolumes[%d]", i);
-                gf_proc_dump_write (key, "%s.%s", priv->xl_array[i]->type,
-                                    priv->xl_array[i]->name);
-        }
-
-        options = priv->pattern;
-        while (options != NULL) {
-                gf_proc_dump_write ("path_pattern", "%s", priv->pattern->path_pattern);
-                gf_proc_dump_write ("options_block_size", "%ul", options->block_size);
-
-                options = options->next;
-        }
-
-        gf_proc_dump_write ("block_size", "%ul", priv->block_size);
-        gf_proc_dump_write ("nodes-down", "%d", priv->nodes_down);
-        gf_proc_dump_write ("first-child_down", "%d", priv->first_child_down);
-        gf_proc_dump_write ("xattr_supported", "%d", priv->xattr_supported);
-
-        UNLOCK (&priv->lock);
-
-out:
-        return ret;
-}
-
-struct xlator_fops fops = {
-        .stat           = stripe_stat,
-        .unlink         = stripe_unlink,
-        .rename         = stripe_rename,
-        .link           = stripe_link,
-        .truncate       = stripe_truncate,
-        .create         = stripe_create,
-        .open           = stripe_open,
-        .readv          = stripe_readv,
-        .writev         = stripe_writev,
-        .statfs         = stripe_statfs,
-        .flush          = stripe_flush,
-        .fsync          = stripe_fsync,
-        .ftruncate      = stripe_ftruncate,
-        .fstat          = stripe_fstat,
-        .mkdir          = stripe_mkdir,
-        .rmdir          = stripe_rmdir,
-        .lk             = stripe_lk,
-        .opendir        = stripe_opendir,
-        .fsyncdir       = stripe_fsyncdir,
-        .setattr        = stripe_setattr,
-        .fsetattr       = stripe_fsetattr,
-        .lookup         = stripe_lookup,
-        .mknod          = stripe_mknod,
-        .setxattr       = stripe_setxattr,
-        .fsetxattr      = stripe_fsetxattr,
-        .getxattr       = stripe_getxattr,
-        .fgetxattr      = stripe_fgetxattr,
-        .removexattr    = stripe_removexattr,
-        .fremovexattr   = stripe_fremovexattr,
-        .readdirp       = stripe_readdirp,
-	.fallocate	= stripe_fallocate,
-	.discard	= stripe_discard,
-        .zerofill       = stripe_zerofill,
-        .seek           = stripe_seek,
-};
-
-struct xlator_cbks cbks = {
-        .release = stripe_release,
-        .forget  = stripe_forget,
-};
-
-struct xlator_dumpops dumpops = {
-        .priv = stripe_priv_dump,
-};
-
-struct volume_options options[] = {
-        { .key  = {"block-size"},
-          .type = GF_OPTION_TYPE_SIZE_LIST,
-          .default_value = "128KB",
-          .min = STRIPE_MIN_BLOCK_SIZE,
-          .description = "Size of the stripe unit that would be read "
-                         "from or written to the striped servers."
-        },
-        { .key  = {"use-xattr"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "true"
-        },
-	{ .key = {"coalesce"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "true",
-	  .description = "Enable/Disable coalesce mode to flatten striped "
-			 "files as stored on the server (i.e., eliminate holes "
-			 "caused by the traditional format)."
-	},
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
deleted file mode 100644
index 1e2fcb4e659..00000000000
--- a/xlators/cluster/stripe/src/stripe.h
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _STRIPE_H_
-#define _STRIPE_H_
-
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "stripe-mem-types.h"
-#include "libxlator.h"
-#include <fnmatch.h>
-#include <signal.h>
-
-#define STRIPE_PATHINFO_HEADER "STRIPE:"
-#define STRIPE_MIN_BLOCK_SIZE  (16*GF_UNIT_KB)
-
-#define STRIPE_STACK_UNWIND(fop, frame, params ...) do {           \
-                stripe_local_t *__local = NULL;                    \
-                if (frame) {                                       \
-                        __local = frame->local;                    \
-                        frame->local = NULL;                       \
-                }                                                  \
-                STACK_UNWIND_STRICT (fop, frame, params);          \
-                if (__local) {                                     \
-                        stripe_local_wipe(__local);                \
-                        mem_put (__local);       \
-                }                                                  \
-        } while (0)
-
-#define STRIPE_STACK_DESTROY(frame) do {                        \
-                stripe_local_t *__local = NULL;                 \
-                __local = frame->local;                         \
-                frame->local = NULL;                            \
-                STACK_DESTROY (frame->root);                    \
-                if (__local) {                                  \
-                        stripe_local_wipe (__local);            \
-                        mem_put (__local);    \
-                }                                               \
-        } while (0)
-
-#define STRIPE_VALIDATE_FCTX(fctx, label) do {                  \
-        int     idx = 0;                                        \
-        if (!fctx) {                                            \
-                op_errno = EINVAL;                              \
-                goto label;                                     \
-        }                                                       \
-        for (idx = 0; idx < fctx->stripe_count; idx++) {        \
-                if (!fctx->xl_array[idx]) {                     \
-                        gf_log (this->name, GF_LOG_ERROR,       \
-                                "fctx->xl_array[%d] is NULL",   \
-                                idx);                           \
-                        op_errno = ESTALE;                      \
-                        goto label;                             \
-                }                                               \
-        }                                                       \
-       } while (0)
-
-typedef struct stripe_xattr_sort {
-        int   pos;
-        int   xattr_len;
-        char *xattr_value;
-} stripe_xattr_sort_t;
-
-/**
- * struct stripe_options : This keeps the pattern and the block-size
- *     information, which is used for striping on a file.
- */
-struct stripe_options {
-        struct stripe_options *next;
-        char                   path_pattern[256];
-        uint64_t               block_size;
-};
-
-/**
- * Private structure for stripe translator
- */
-struct stripe_private {
-        struct stripe_options  *pattern;
-        xlator_t              **xl_array;
-        uint64_t                block_size;
-        gf_lock_t               lock;
-        uint8_t                 nodes_down;
-        int8_t                  first_child_down;
-        int                    *last_event;
-        int8_t                  child_count;
-        gf_boolean_t            xattr_supported;  /* default yes */
-	gf_boolean_t		coalesce;
-        char                    vol_uuid[UUID_SIZE + 1];
-};
-
-/**
- * Used to keep info about the replies received from readv/writev calls
- */
-struct stripe_replies {
-        struct iovec *vector;
-        int32_t       count;    //count of vector
-        int32_t       op_ret;   //op_ret of readv
-        int32_t       op_errno;
-        int32_t       requested_size;
-        struct iatt   stbuf;    /* 'stbuf' is also a part of reply */
-};
-
-typedef struct _stripe_fd_ctx {
-        off_t      stripe_size;
-        int        stripe_count;
-	int	   stripe_coalesce;
-        int        static_array;
-        xlator_t **xl_array;
-} stripe_fd_ctx_t;
-
-
-/**
- * Local structure to be passed with all the frames in case of STACK_WIND
- */
-struct stripe_local; /* this itself is used inside the structure; */
-
-struct stripe_local {
-        struct stripe_local *next;
-        call_frame_t        *orig_frame;
-
-        stripe_fd_ctx_t     *fctx;
-
-        /* Used by _cbk functions */
-        struct iatt          stbuf;
-        struct iatt          pre_buf;
-        struct iatt          post_buf;
-        struct iatt          preparent;
-        struct iatt          postparent;
-
-        off_t                stbuf_size;
-        off_t                prebuf_size;
-        off_t                postbuf_size;
-        off_t                preparent_size;
-        off_t                postparent_size;
-
-        blkcnt_t             stbuf_blocks;
-        blkcnt_t             prebuf_blocks;
-        blkcnt_t             postbuf_blocks;
-        blkcnt_t             preparent_blocks;
-        blkcnt_t             postparent_blocks;
-
-        struct stripe_replies *replies;
-        struct statvfs        statvfs_buf;
-        dir_entry_t          *entry;
-
-        int8_t               revalidate;
-        int8_t               failed;
-        int8_t               unwind;
-
-        size_t               readv_size;
-        int32_t              entry_count;
-        int32_t              node_index;
-        int32_t              call_count;
-        int32_t              wind_count; /* used instead of child_cound
-                                            in case of read and write */
-        int32_t              op_ret;
-        int32_t              op_errno;
-        int32_t              count;
-        int32_t              flags;
-        char                *name;
-        inode_t             *inode;
-
-        loc_t                loc;
-        loc_t                loc2;
-
-        mode_t               mode;
-        dev_t                rdev;
-        /* For File I/O fops */
-        dict_t              *xdata;
-
-        stripe_xattr_sort_t *xattr_list;
-        int32_t              xattr_total_len;
-        int32_t              nallocs;
-        char xsel[256];
-
-        /* General usage */
-        off_t                offset;
-        off_t                stripe_size;
-
-        int xattr_self_heal_needed;
-        int entry_self_heal_needed;
-
-        int8_t              *list;
-        struct gf_flock         lock;
-        fd_t                *fd;
-        void                *value;
-        struct iobref       *iobref;
-        gf_dirent_t          entries;
-        gf_dirent_t         *dirent;
-        dict_t              *xattr;
-        uuid_t               ia_gfid;
-
-        int                  xflag;
-        mode_t               umask;
-};
-
-typedef struct stripe_local   stripe_local_t;
-typedef struct stripe_private stripe_private_t;
-
-/*
- * Determine the stripe index of a particular frame based on the translator.
- */
-static inline int32_t stripe_get_frame_index(stripe_fd_ctx_t *fctx,
-					     call_frame_t *prev)
-{
-	int32_t i, idx = -1;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (fctx->xl_array[i] == prev->this) {
-			idx = i;
-			break;
-		}
-	}
-
-	return idx;
-}
-
-static inline void stripe_copy_xl_array(xlator_t **dst, xlator_t **src,
-					int count)
-{
-	int i;
-
-	for (i = 0; i < count; i++)
-		dst[i] = src[i];
-}
-
-void stripe_local_wipe (stripe_local_t *local);
-int32_t stripe_ctx_handle (xlator_t *this, call_frame_t *prev,
-                           stripe_local_t *local, dict_t *dict);
-void stripe_aggregate_xattr (dict_t *dst, dict_t *src);
-int32_t stripe_xattr_request_build (xlator_t *this, dict_t *dict,
-                                    uint64_t stripe_size, uint32_t stripe_count,
-                                    uint32_t stripe_index,
-				    uint32_t stripe_coalesce);
-int32_t stripe_get_matching_bs (const char *path, stripe_private_t *priv);
-int set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data);
-int32_t stripe_iatt_merge (struct iatt *from, struct iatt *to);
-int32_t stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
-                                    char **xattr_serz);
-int32_t stripe_free_xattr_str (stripe_local_t *local);
-int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
-                                int32_t *total);
-off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
-off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
-			int stripe_index);
-int32_t
-stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            void **xattr_serz);
-
-/*
- * Adjust the size attribute for files if coalesce is enabled.
- */
-static inline void correct_file_size(struct iatt *buf, stripe_fd_ctx_t *fctx,
-	call_frame_t *prev)
-{
-	int index;
-
-	if (!IA_ISREG(buf->ia_type))
-		return;
-
-	if (!fctx || !fctx->stripe_coalesce)
-		return;
-
-	index = stripe_get_frame_index(fctx, prev);
-	buf->ia_size = uncoalesced_size(buf->ia_size, fctx->stripe_size,
-		fctx->stripe_count, index);
-}
-
-#endif /* _STRIPE_H_ */
diff --git a/xlators/debug/Makefile.am b/xlators/debug/Makefile.am
index b655554efec..88fac1c6d9e 100644
--- a/xlators/debug/Makefile.am
+++ b/xlators/debug/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = trace error-gen io-stats
+SUBDIRS = error-gen io-stats sink trace delay-gen
 
 CLEANFILES = 
diff --git a/xlators/experimental/dht2/dht2-client/Makefile.am b/xlators/debug/delay-gen/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/dht2/dht2-client/Makefile.am
+++ b/xlators/debug/delay-gen/Makefile.am
diff --git a/xlators/debug/delay-gen/src/Makefile.am b/xlators/debug/delay-gen/src/Makefile.am
new file mode 100644
index 00000000000..8f758dec199
--- /dev/null
+++ b/xlators/debug/delay-gen/src/Makefile.am
@@ -0,0 +1,11 @@
+
+xlator_LTLIBRARIES = delay-gen.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
+delay_gen_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+delay_gen_la_SOURCES = delay-gen.c
+delay_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+noinst_HEADERS = delay-gen.h delay-gen-mem-types.h delay-gen-messages.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+CLEANFILES =
diff --git a/xlators/debug/delay-gen/src/delay-gen-mem-types.h b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
new file mode 100644
index 00000000000..c89a9217193
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
@@ -0,0 +1,21 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __DELAY_GEN_MEM_TYPES_H__
+#define __DELAY_GEN_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_delay_gen_mem_types_ {
+    gf_delay_gen_mt_dg_t = gf_common_mt_end + 1,
+    gf_delay_gen_mt_end
+};
+
+#endif /* __DELAY_GEN_MEM_TYPES_H__ */
diff --git a/xlators/debug/delay-gen/src/delay-gen-messages.h b/xlators/debug/delay-gen/src/delay-gen-messages.h
new file mode 100644
index 00000000000..bc98cec2885
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen-messages.h
@@ -0,0 +1,26 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __DELAY_GEN_MESSAGES_H__
+#define __DELAY_GEN_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+#endif /* __DELAY_GEN_MESSAGES_H__ */
diff --git a/xlators/debug/delay-gen/src/delay-gen.c b/xlators/debug/delay-gen/src/delay-gen.c
new file mode 100644
index 00000000000..4698f1fd785
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen.c
@@ -0,0 +1,697 @@
+/*
+ *  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+#include "delay-gen.h"
+
+#define DELAY_GRANULARITY (1 << 20)
+
+#define DG_FOP(fop, name, frame, this, args...)                                \
+    do {                                                                       \
+        delay_gen(this, fop);                                                  \
+        default_##name(frame, this, args);                                     \
+    } while (0)
+
+int
+delay_gen(xlator_t *this, int fop)
+{
+    dg_t *dg = this->private;
+
+    if (!dg->enable[fop] || !dg->delay_ppm)
+        return 0;
+
+    if ((rand() % DELAY_GRANULARITY) < dg->delay_ppm)
+        gf_nanosleep(dg->delay_duration * GF_US_IN_NS);
+
+    return 0;
+}
+
+int32_t
+dg_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_RENAME, rename, frame, this, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+dg_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_IPC, ipc, frame, this, op, xdata);
+    return 0;
+}
+
+int32_t
+dg_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               lock_migration_info_t *locklist, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SETACTIVELK, setactivelk, frame, this, loc, locklist, xdata);
+    return 0;
+}
+
+int32_t
+dg_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FLUSH, flush, frame, this, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t off, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READDIR, readdir, frame, this, fd, size, off, xdata);
+    return 0;
+}
+
+int32_t
+dg_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SETXATTR, setxattr, frame, this, loc, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_MKNOD, mknod, frame, this, loc, mode, rdev, umask, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSETXATTR, fsetxattr, frame, this, fd, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READ, readv, frame, this, fd, size, offset, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_INODELK, inodelk, frame, this, volume, loc, cmd, lock, xdata);
+    return 0;
+}
+
+int32_t
+dg_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FREMOVEXATTR, fremovexattr, frame, this, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_OPEN, open, frame, this, loc, flags, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_XATTROP, xattrop, frame, this, loc, flags, dict, xdata);
+    return 0;
+}
+
+int32_t
+dg_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
+{
+    DG_FOP(GF_FOP_ENTRYLK, entrylk, frame, this, volume, loc, basename, cmd,
+           type, xdata);
+    return 0;
+}
+
+int32_t
+dg_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_GETACTIVELK, getactivelk, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FINODELK, finodelk, frame, this, volume, fd, cmd, lock,
+           xdata);
+    return 0;
+}
+
+int32_t
+dg_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_CREATE, create, frame, this, loc, flags, mode, umask, fd,
+           xdata);
+    return 0;
+}
+
+int32_t
+dg_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_DISCARD, discard, frame, this, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+dg_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_MKDIR, mkdir, frame, this, loc, mode, umask, xdata);
+    return 0;
+}
+
+int32_t
+dg_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *lock, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LK, lk, frame, this, fd, cmd, lock, xdata);
+    return 0;
+}
+
+int32_t
+dg_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_WRITE, writev, frame, this, fd, vector, count, off, flags,
+           iobref, xdata);
+    return 0;
+}
+
+int32_t
+dg_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_ACCESS, access, frame, this, loc, mask, xdata);
+    return 0;
+}
+
+int32_t
+dg_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LOOKUP, lookup, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         dict_t *xdata)
+{
+    DG_FOP(GF_FOP_RMDIR, rmdir, frame, this, loc, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FALLOCATE, fallocate, frame, this, fd, keep_size, offset, len,
+           xdata);
+    return 0;
+}
+
+int32_t
+dg_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSTAT, fstat, frame, this, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+         struct gf_lease *lease, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LEASE, lease, frame, this, loc, lease, xdata);
+    return 0;
+}
+
+int32_t
+dg_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_STAT, stat, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_TRUNCATE, truncate, frame, this, loc, offset, xdata);
+    return 0;
+}
+
+int32_t
+dg_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_GETXATTR, getxattr, frame, this, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SYMLINK, symlink, frame, this, linkpath, loc, umask, xdata);
+    return 0;
+}
+
+int32_t
+dg_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_ZEROFILL, zerofill, frame, this, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSYNCDIR, fsyncdir, frame, this, fd, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FGETXATTR, fgetxattr, frame, this, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READDIRP, readdirp, frame, this, fd, size, off, xdata);
+    return 0;
+}
+
+int32_t
+dg_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LINK, link, frame, this, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+dg_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FXATTROP, fxattrop, frame, this, fd, flags, dict, xdata);
+    return 0;
+}
+
+int32_t
+dg_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FTRUNCATE, ftruncate, frame, this, fd, offset, xdata);
+    return 0;
+}
+
+int32_t
+dg_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             int32_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_RCHECKSUM, rchecksum, frame, this, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+dg_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_UNLINK, unlink, frame, this, loc, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FENTRYLK, fentrylk, frame, this, volume, fd, basename, cmd,
+           type, xdata);
+    return 0;
+}
+
+int32_t
+dg_getspec(call_frame_t *frame, xlator_t *this, const char *key, int32_t flags)
+{
+    DG_FOP(GF_FOP_GETSPEC, getspec, frame, this, key, flags);
+    return 0;
+}
+
+int32_t
+dg_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SETATTR, setattr, frame, this, loc, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+         dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSYNC, fsync, frame, this, fd, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_STATFS, statfs, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SEEK, seek, frame, this, fd, offset, what, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSETATTR, fsetattr, frame, this, fd, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+dg_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    DG_FOP(GF_FOP_OPENDIR, opendir, frame, this, loc, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READLINK, readlink, frame, this, loc, size, xdata);
+    return 0;
+}
+
+int32_t
+dg_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_REMOVEXATTR, removexattr, frame, this, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_forget(xlator_t *this, inode_t *inode)
+{
+    return 0;
+}
+
+int32_t
+dg_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+dg_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+static int
+delay_gen_parse_fill_fops(dg_t *dg, char *enable_fops)
+{
+    char *op_no_str = NULL;
+    int op_no = -1;
+    int i = 0;
+    int ret = 0;
+    xlator_t *this = THIS;
+    char *saveptr = NULL;
+    char *dup_enable_fops = NULL;
+
+    if (strlen(enable_fops) == 0) {
+        for (i = GF_FOP_NULL + 1; i < GF_FOP_MAXVALUE; i++)
+            dg->enable[i] = 1;
+    } else {
+        dup_enable_fops = gf_strdup(enable_fops);
+        if (!dup_enable_fops) {
+            ret = -1;
+            goto out;
+        }
+        op_no_str = strtok_r(dup_enable_fops, ",", &saveptr);
+        while (op_no_str) {
+            op_no = gf_fop_int(op_no_str);
+            if (op_no == -1) {
+                gf_log(this->name, GF_LOG_WARNING, "Wrong option value %s",
+                       op_no_str);
+                ret = -1;
+                goto out;
+            } else {
+                dg->enable[op_no] = 1;
+            }
+
+            op_no_str = strtok_r(NULL, ",", &saveptr);
+        }
+    }
+out:
+    GF_FREE(dup_enable_fops);
+    return ret;
+}
+
+void
+delay_gen_set_delay_ppm(dg_t *dg, double percent)
+{
+    double ppm;
+
+    ppm = (percent / 100.0) * (double)DELAY_GRANULARITY;
+    dg->delay_ppm = ppm;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    dg_t *dg = NULL;
+    int32_t ret = 0;
+    double delay_percent = 0;
+    char *delay_enable_fops = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "delay-gen not configured with one subvolume");
+        ret = -1;
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    dg = GF_CALLOC(1, sizeof(*dg), gf_delay_gen_mt_dg_t);
+
+    if (!dg) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+
+    GF_OPTION_INIT("delay-percentage", delay_percent, percent, out);
+    GF_OPTION_INIT("enable", delay_enable_fops, str, out);
+    GF_OPTION_INIT("delay-duration", dg->delay_duration, int32, out);
+
+    delay_gen_set_delay_ppm(dg, delay_percent);
+
+    ret = delay_gen_parse_fill_fops(dg, delay_enable_fops);
+    if (ret)
+        goto out;
+
+    this->private = dg;
+
+    ret = 0;
+out:
+    if (ret)
+        GF_FREE(dg);
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    GF_FREE(this->private);
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_delay_gen_mt_end + 1);
+
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
+        return ret;
+    }
+
+    return ret;
+}
+
+int32_t
+reconfigure(xlator_t *this, dict_t *dict)
+{
+    /*At the moment I don't see any need to implement this. In future
+     *if this is needed we can add code here.
+     */
+    return 0;
+}
+
+int
+notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+struct xlator_fops fops = {
+    .rename = dg_rename,
+    .ipc = dg_ipc,
+    .setactivelk = dg_setactivelk,
+    .flush = dg_flush,
+    .readdir = dg_readdir,
+    .setxattr = dg_setxattr,
+    .mknod = dg_mknod,
+    .fsetxattr = dg_fsetxattr,
+    .readv = dg_readv,
+    .inodelk = dg_inodelk,
+    .fremovexattr = dg_fremovexattr,
+    .open = dg_open,
+    .xattrop = dg_xattrop,
+    .entrylk = dg_entrylk,
+    .getactivelk = dg_getactivelk,
+    .finodelk = dg_finodelk,
+    .create = dg_create,
+    .discard = dg_discard,
+    .mkdir = dg_mkdir,
+    .lk = dg_lk,
+    .writev = dg_writev,
+    .access = dg_access,
+    .lookup = dg_lookup,
+    .rmdir = dg_rmdir,
+    .fallocate = dg_fallocate,
+    .fstat = dg_fstat,
+    .lease = dg_lease,
+    .stat = dg_stat,
+    .truncate = dg_truncate,
+    .getxattr = dg_getxattr,
+    .symlink = dg_symlink,
+    .zerofill = dg_zerofill,
+    .fsyncdir = dg_fsyncdir,
+    .fgetxattr = dg_fgetxattr,
+    .readdirp = dg_readdirp,
+    .link = dg_link,
+    .fxattrop = dg_fxattrop,
+    .ftruncate = dg_ftruncate,
+    .rchecksum = dg_rchecksum,
+    .unlink = dg_unlink,
+    .fentrylk = dg_fentrylk,
+    .getspec = dg_getspec,
+    .setattr = dg_setattr,
+    .fsync = dg_fsync,
+    .statfs = dg_statfs,
+    .seek = dg_seek,
+    .fsetattr = dg_fsetattr,
+    .opendir = dg_opendir,
+    .readlink = dg_readlink,
+    .removexattr = dg_removexattr,
+};
+
+struct xlator_cbks cbks = {
+    .forget = dg_forget,
+    .release = dg_release,
+    .releasedir = dg_releasedir,
+};
+
+struct volume_options options[] = {
+    {
+        .key = {"delay-percentage"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .default_value = "10%",
+        .description = "Percentage delay of operations when enabled.",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+    },
+
+    {
+        .key = {"delay-duration"},
+        .type = GF_OPTION_TYPE_INT,
+        .description = "Delay duration in micro seconds",
+        .default_value = "100000",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+    },
+
+    {
+        .key = {"enable"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Accepts a string which takes ',' separated fop "
+                       "strings to denote which fops are enabled for delay",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+        .default_value = "",
+    },
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "delay-gen",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/debug/delay-gen/src/delay-gen.h b/xlators/debug/delay-gen/src/delay-gen.h
new file mode 100644
index 00000000000..afa95e5eb2d
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen.h
@@ -0,0 +1,27 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __DELAY_GEN_H__
+#define __DELAY_GEN_H__
+
+#include "delay-gen-mem-types.h"
+#include "delay-gen-messages.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+typedef struct {
+    int enable[GF_FOP_MAXVALUE];
+    int op_count;
+    int delay_ppm;
+    int delay_duration;
+} dg_t;
+
+#endif /* __DELAY_GEN_H__ */
diff --git a/xlators/debug/error-gen/src/Makefile.am b/xlators/debug/error-gen/src/Makefile.am
index 8baf15612bd..038d2e8e66d 100644
--- a/xlators/debug/error-gen/src/Makefile.am
+++ b/xlators/debug/error-gen/src/Makefile.am
@@ -9,7 +9,8 @@ error_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = error-gen.h error-gen-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
index f02280535df..b9b713af8fc 100644
--- a/xlators/debug/error-gen/src/error-gen-mem-types.h
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __ERROR_GEN_MEM_TYPES_H__
 #define __ERROR_GEN_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_error_gen_mem_types_ {
-        gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
-        gf_error_gen_mt_end
+    gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
+    gf_error_gen_mt_end
 };
 #endif
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index b6b17baa87f..d45655ef4c3 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -7,2221 +7,1657 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "error-gen.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/defaults.h>
 
-sys_error_t error_no_list[] = {
-        [GF_FOP_LOOKUP]            = { .error_no_count = 4,
-                                    .error_no = {ENOENT,ENOTDIR,
-                                                 ENAMETOOLONG,EAGAIN}},
-        [GF_FOP_STAT]              = { .error_no_count = 7,
-                                    .error_no = {EACCES,EBADF,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR}},
-        [GF_FOP_READLINK]          = { .error_no_count = 8,
-                                    .error_no = {EACCES,EFAULT,EINVAL,EIO,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOTDIR}},
-        [GF_FOP_MKNOD]             = { .error_no_count = 11,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EINVAL,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOSPC,
-                                                 ENOTDIR,EPERM,EROFS}},
-        [GF_FOP_MKDIR]             = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOSPC,ENOTDIR,
-						 EPERM,EROFS}},
-        [GF_FOP_UNLINK]            = { .error_no_count = 10,
-                                    .error_no = {EACCES,EBUSY,EFAULT,EIO,
-                                                 EISDIR,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOTDIR,
-                                                 EPERM,EROFS}},
-        [GF_FOP_RMDIR]             = { .error_no_count = 8,
-                                    .error_no = {EACCES,EBUSY,EFAULT,
-                                                 ENOMEM,ENOTDIR,ENOTEMPTY,
-                                                 EPERM,EROFS}},
-        [GF_FOP_SYMLINK]           = { .error_no_count = 11,
-                                    .error_no = {EACCES,EEXIST,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOSPC,ENOTDIR,EPERM,
-                                                 EROFS}},
-        [GF_FOP_RENAME]            = { .error_no_count = 13,
-                                    .error_no = {EACCES,EBUSY,EFAULT,
-                                                 EINVAL,EISDIR,EMLINK,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOSPC,ENOTDIR,EEXIST,
-                                                 EXDEV}},
-        [GF_FOP_LINK]              = { .error_no_count = 13,
-                                    .error_no = {EACCES,EFAULT,EEXIST,EIO,
-                                                 EMLINK,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOSPC,
-                                                 ENOTDIR,EPERM,EROFS,
-                                                 EXDEV}},
-        [GF_FOP_TRUNCATE]          = { .error_no_count = 10,
-                                    .error_no = {EACCES,EFAULT,EFBIG,
-                                                 EINTR,EINVAL,EIO,EISDIR,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 EISDIR}},
-        [GF_FOP_CREATE]            = {.error_no_count = 10,
-                                   .error_no = {EACCES,EEXIST,EFAULT,
-                                                EISDIR,EMFILE,ENAMETOOLONG,
-                                                ENFILE,ENODEV,ENOENT,
-                                                ENODEV}},
-        [GF_FOP_OPEN]              = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV,ENOENT,ENOMEM}},
-        [GF_FOP_READ]              = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_WRITE]             = { .error_no_count = 7,
-                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG,ENOSPC,
-						 GF_ERROR_SHORT_WRITE}},
-        [GF_FOP_STATFS]            = {.error_no_count = 10,
-                                   .error_no = {EACCES,EBADF,EFAULT,EINTR,
-                                                EIO,ENAMETOOLONG,ENOENT,
-                                                ENOMEM,ENOSYS,ENOTDIR}},
-        [GF_FOP_FLUSH]             = { .error_no_count = 5,
-                                    .error_no = {EACCES,EFAULT,
-                                                 ENAMETOOLONG,ENOSYS,
-                                                 ENOENT}},
-        [GF_FOP_FSYNC]             = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_SETXATTR]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_GETXATTR]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}},
-        [GF_FOP_REMOVEXATTR]       = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}},
-        [GF_FOP_FSETXATTR]          = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,EINTR,
-                                                     ENAMETOOLONG}},
-        [GF_FOP_FGETXATTR]          = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                     EINTR}},
-        [GF_FOP_FREMOVEXATTR]       = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                     EINTR}},
-        [GF_FOP_OPENDIR]           = { .error_no_count = 8,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV}},
-        [GF_FOP_READDIR]           = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EACCES,EBADF,
-                                                 EMFILE,ENOENT}},
-        [GF_FOP_READDIRP]          = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EACCES,EBADF,
-                                                 EMFILE,ENOENT}},
-        [GF_FOP_FSYNCDIR]          = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_ACCESS]            = { .error_no_count = 8,
-                                    .error_no = {EACCES,ENAMETOOLONG,
-                                                 ENOENT,ENOTDIR,EROFS,
-                                                 EFAULT,EINVAL,EIO}},
-        [GF_FOP_FTRUNCATE]         = { .error_no_count = 9,
-                                    .error_no = {EACCES,EFAULT,EFBIG,
-                                                 EINTR,EINVAL,EIO,EISDIR,
-                                                 ENAMETOOLONG,ENOENT}},
-        [GF_FOP_FSTAT]             = { .error_no_count = 7,
-                                    .error_no = {EACCES,EBADF,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR}},
-        [GF_FOP_LK]                = { .error_no_count = 4,
-                                    .error_no = {EACCES,EFAULT,ENOENT,
-                                                 EINTR}},
-        [GF_FOP_XATTROP]           = { .error_no_count = 5,
-                                    .error_no = {EACCES,EFAULT,
-                                                 ENAMETOOLONG,ENOSYS,
-                                                 ENOENT}},
-        [GF_FOP_FXATTROP]          = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_INODELK]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_FINODELK]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_ENTRYLK]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,
-                                                 ENAMETOOLONG,EINTR}},
-        [GF_FOP_FENTRYLK]          = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV,ENOENT,ENOMEM}},
-        [GF_FOP_SETATTR]           = {.error_no_count = 11,
-                                    .error_no = {EACCES,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR,EPERM,
-                                                 EROFS,EBADF,EIO}},
-        [GF_FOP_FSETATTR]          = { .error_no_count = 11,
-                                    .error_no = {EACCES,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR,EPERM,
-                                                 EROFS,EBADF,EIO}},
-        [GF_FOP_GETSPEC]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}}
-};
-
-int
-generate_rand_no (int op_no)
-{
-        int             rand_no = 0;
-
-        if (op_no < GF_FOP_MAXVALUE)
-                rand_no = rand () % error_no_list[op_no].error_no_count;
-        return rand_no;
-}
-
-int
-conv_errno_to_int (char **error_no)
-{
-        if (!strcmp ((*error_no), "ENOENT"))
-                return ENOENT;
-        else if (!strcmp ((*error_no), "ENOTDIR"))
-                return ENOTDIR;
-        else if (!strcmp ((*error_no), "ENAMETOOLONG"))
-                return ENAMETOOLONG;
-        else if (!strcmp ((*error_no), "EACCES"))
-                return EACCES;
-        else if (!strcmp ((*error_no), "EBADF"))
-                return EBADF;
-        else if (!strcmp ((*error_no), "EFAULT"))
-                return EFAULT;
-        else if (!strcmp ((*error_no), "ENOMEM"))
-                return ENOMEM;
-        else if (!strcmp ((*error_no), "EINVAL"))
-                return EINVAL;
-        else if (!strcmp ((*error_no), "EIO"))
-                return EIO;
-        else if (!strcmp ((*error_no), "EEXIST"))
-                return EEXIST;
-        else if (!strcmp ((*error_no), "ENOSPC"))
-                return ENOSPC;
-        else if (!strcmp ((*error_no), "EPERM"))
-                return EPERM;
-        else if (!strcmp ((*error_no), "EROFS"))
-                return EROFS;
-        else if (!strcmp ((*error_no), "EBUSY"))
-                return EBUSY;
-        else if (!strcmp ((*error_no), "EISDIR"))
-                return EISDIR;
-        else if (!strcmp ((*error_no), "ENOTEMPTY"))
-                return ENOTEMPTY;
-        else if (!strcmp ((*error_no), "EMLINK"))
-                return EMLINK;
-        else if (!strcmp ((*error_no), "ENODEV"))
-                return ENODEV;
-        else if (!strcmp ((*error_no), "EXDEV"))
-                return EXDEV;
-        else if (!strcmp ((*error_no), "EMFILE"))
-                return EMFILE;
-        else if (!strcmp ((*error_no), "ENFILE"))
-                return ENFILE;
-        else if (!strcmp ((*error_no), "ENOSYS"))
-                return ENOSYS;
-        else if (!strcmp ((*error_no), "EINTR"))
-                return EINTR;
-        else if (!strcmp ((*error_no), "EFBIG"))
-                return EFBIG;
-	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
-		return GF_ERROR_SHORT_WRITE;
-        else
-                return EAGAIN;
-}
-
-int
-get_fop_int (char **op_no_str)
-{
-        if (!strcmp ((*op_no_str), "lookup"))
-                return GF_FOP_LOOKUP;
-        else if (!strcmp ((*op_no_str), "stat"))
-                return GF_FOP_STAT;
-        else if (!strcmp ((*op_no_str), "readlink"))
-                return GF_FOP_READLINK;
-        else if (!strcmp ((*op_no_str), "mknod"))
-                return GF_FOP_MKNOD;
-        else if (!strcmp ((*op_no_str), "mkdir"))
-                return GF_FOP_MKDIR;
-        else if (!strcmp ((*op_no_str), "unlink"))
-                return GF_FOP_UNLINK;
-        else if (!strcmp ((*op_no_str), "rmdir"))
-                return GF_FOP_RMDIR;
-        else if (!strcmp ((*op_no_str), "symlink"))
-                return GF_FOP_SYMLINK;
-        else if (!strcmp ((*op_no_str), "rename"))
-                return GF_FOP_RENAME;
-        else if (!strcmp ((*op_no_str), "link"))
-                return GF_FOP_LINK;
-        else if (!strcmp ((*op_no_str), "truncate"))
-                return GF_FOP_TRUNCATE;
-        else if (!strcmp ((*op_no_str), "create"))
-                return GF_FOP_CREATE;
-        else if (!strcmp ((*op_no_str), "open"))
-                return GF_FOP_OPEN;
-        else if (!strcmp ((*op_no_str), "readv"))
-                return GF_FOP_READ;
-        else if (!strcmp ((*op_no_str), "writev"))
-                return GF_FOP_WRITE;
-        else if (!strcmp ((*op_no_str), "statfs"))
-                return GF_FOP_STATFS;
-        else if (!strcmp ((*op_no_str), "flush"))
-                return GF_FOP_FLUSH;
-        else if (!strcmp ((*op_no_str), "fsync"))
-                return GF_FOP_FSYNC;
-        else if (!strcmp ((*op_no_str), "setxattr"))
-                return GF_FOP_SETXATTR;
-        else if (!strcmp ((*op_no_str), "getxattr"))
-                return GF_FOP_GETXATTR;
-        else if (!strcmp ((*op_no_str), "removexattr"))
-                return GF_FOP_REMOVEXATTR;
-        else if (!strcmp ((*op_no_str), "fsetxattr"))
-                return GF_FOP_FSETXATTR;
-        else if (!strcmp ((*op_no_str), "fgetxattr"))
-                return GF_FOP_FGETXATTR;
-        else if (!strcmp ((*op_no_str), "fremovexattr"))
-                return GF_FOP_FREMOVEXATTR;
-        else if (!strcmp ((*op_no_str), "opendir"))
-                return GF_FOP_OPENDIR;
-        else if (!strcmp ((*op_no_str), "readdir"))
-                return GF_FOP_READDIR;
-        else if (!strcmp ((*op_no_str), "readdirp"))
-                return GF_FOP_READDIRP;
-        else if (!strcmp ((*op_no_str), "fsyncdir"))
-                return GF_FOP_FSYNCDIR;
-        else if (!strcmp ((*op_no_str), "access"))
-                return GF_FOP_ACCESS;
-        else if (!strcmp ((*op_no_str), "ftruncate"))
-                return GF_FOP_FTRUNCATE;
-        else if (!strcmp ((*op_no_str), "fstat"))
-                return GF_FOP_FSTAT;
-        else if (!strcmp ((*op_no_str), "lk"))
-                return GF_FOP_LK;
-        else if (!strcmp ((*op_no_str), "xattrop"))
-                return GF_FOP_XATTROP;
-        else if (!strcmp ((*op_no_str), "fxattrop"))
-                return GF_FOP_FXATTROP;
-        else if (!strcmp ((*op_no_str), "inodelk"))
-                return GF_FOP_INODELK;
-        else if (!strcmp ((*op_no_str), "finodelk"))
-                return GF_FOP_FINODELK;
-        else if (!strcmp ((*op_no_str), "etrylk"))
-                return GF_FOP_ENTRYLK;
-        else if (!strcmp ((*op_no_str), "fentrylk"))
-                return GF_FOP_FENTRYLK;
-        else if (!strcmp ((*op_no_str), "setattr"))
-                return GF_FOP_SETATTR;
-        else if (!strcmp ((*op_no_str), "fsetattr"))
-                return GF_FOP_FSETATTR;
-        else if (!strcmp ((*op_no_str), "getspec"))
-                return GF_FOP_GETSPEC;
-	else
-                return -1;
-}
-
-int
-error_gen (xlator_t *this, int op_no)
-{
-        eg_t             *egp = NULL;
-        int              count = 0;
-        int              failure_iter_no = GF_FAILURE_DEFAULT;
-        char             *error_no = NULL;
-        int              rand_no = 0;
-        int              ret = 0;
-
-        egp = this->private;
+/*
+ * The user can specify an error probability as a float percentage, but we
+ * store it internally as a numerator with this as the denominator.  When it's
+ * used, it's like this:
+ *
+ *    (rand() % FAILURE_GRANULARITY) < error_rate
+ *
+ * To minimize rounding errors from the modulo operation, it's good for this to
+ * be a power of two.
+ *
+ * (BTW this is just the normal case.  If "random-failure" is set, that does
+ * something completely different and this number is irrelevant.  See error_gen
+ * for the legacy code.)
+ */
+#define FAILURE_GRANULARITY (1 << 20)
 
-        LOCK (&egp->lock);
+sys_error_t error_no_list[] = {
+    [GF_FOP_LOOKUP] = {.error_no_count = 4,
+                       .error_no = {ENOENT, ENOTDIR, ENAMETOOLONG, EAGAIN}},
+    [GF_FOP_STAT] = {.error_no_count = 6,
+                     .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOENT, ENOMEM,
+                                  ENOTDIR}},
+    [GF_FOP_READLINK] = {.error_no_count = 8,
+                         .error_no = {EACCES, EFAULT, EINVAL, EIO, ENAMETOOLONG,
+                                      ENOENT, ENOMEM, ENOTDIR}},
+    [GF_FOP_MKNOD] = {.error_no_count = 11,
+                      .error_no = {EACCES, EEXIST, EFAULT, EINVAL, ENAMETOOLONG,
+                                   ENOENT, ENOMEM, ENOSPC, ENOTDIR, EPERM,
+                                   EROFS}},
+    [GF_FOP_MKDIR] = {.error_no_count = 10,
+                      .error_no = {EACCES, EEXIST, EFAULT, ENAMETOOLONG, ENOENT,
+                                   ENOMEM, ENOSPC, ENOTDIR, EPERM, EROFS}},
+    [GF_FOP_UNLINK] = {.error_no_count = 10,
+                       .error_no = {EACCES, EBUSY, EFAULT, EIO, EISDIR,
+                                    ENAMETOOLONG, ENOENT, ENOMEM, ENOTDIR,
+                                    EPERM, EROFS}},
+    [GF_FOP_RMDIR] = {.error_no_count = 8,
+                      .error_no = {EACCES, EBUSY, EFAULT, ENOMEM, ENOTDIR,
+                                   ENOTEMPTY, EPERM, EROFS}},
+    [GF_FOP_SYMLINK] = {.error_no_count = 11,
+                        .error_no = {EACCES, EEXIST, EFAULT, EIO, ENAMETOOLONG,
+                                     ENOENT, ENOMEM, ENOSPC, ENOTDIR, EPERM,
+                                     EROFS}},
+    [GF_FOP_RENAME] = {.error_no_count = 13,
+                       .error_no = {EACCES, EBUSY, EFAULT, EINVAL, EISDIR,
+                                    EMLINK, ENAMETOOLONG, ENOENT, ENOMEM,
+                                    ENOSPC, ENOTDIR, EEXIST, EXDEV}},
+    [GF_FOP_LINK] = {.error_no_count = 13,
+                     .error_no = {EACCES, EFAULT, EEXIST, EIO, EMLINK,
+                                  ENAMETOOLONG, ENOENT, ENOMEM, ENOSPC, ENOTDIR,
+                                  EPERM, EROFS, EXDEV}},
+    [GF_FOP_TRUNCATE] = {.error_no_count = 10,
+                         .error_no = {EACCES, EFAULT, EFBIG, EINTR, EINVAL, EIO,
+                                      EISDIR, ENAMETOOLONG, ENOENT, EISDIR}},
+    [GF_FOP_CREATE] = {.error_no_count = 10,
+                       .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                    ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                    ENODEV}},
+    [GF_FOP_OPEN] = {.error_no_count = 10,
+                     .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                  ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                  ENOMEM}},
+    [GF_FOP_READ] = {.error_no_count = 5,
+                     .error_no = {EINVAL, EBADF, EFAULT, EISDIR, ENAMETOOLONG}},
+    [GF_FOP_WRITE] = {.error_no_count = 7,
+                      .error_no = {EINVAL, EBADF, EFAULT, EISDIR, ENAMETOOLONG,
+                                   ENOSPC, GF_ERROR_SHORT_WRITE}},
+    [GF_FOP_STATFS] = {.error_no_count = 9,
+                       .error_no = {EACCES, EFAULT, EINTR, EIO, ENAMETOOLONG,
+                                    ENOENT, ENOMEM, ENOSYS, ENOTDIR}},
+    [GF_FOP_FLUSH] = {.error_no_count = 5,
+                      .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOSYS,
+                                   ENOENT}},
+    [GF_FOP_FSYNC] = {.error_no_count = 4,
+                      .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_SETXATTR] = {.error_no_count = 3,
+                         .error_no = {EACCES, EINTR, ENAMETOOLONG}},
+    [GF_FOP_GETXATTR] = {.error_no_count = 3,
+                         .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_REMOVEXATTR] = {.error_no_count = 3,
+                            .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FSETXATTR] = {.error_no_count = 4,
+                          .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}},
+    [GF_FOP_FGETXATTR] = {.error_no_count = 4,
+                          .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FREMOVEXATTR] = {.error_no_count = 4,
+                             .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}},
+    [GF_FOP_OPENDIR] = {.error_no_count = 8,
+                        .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                     ENAMETOOLONG, ENFILE, ENODEV}},
+    [GF_FOP_READDIR] = {.error_no_count = 5,
+                        .error_no = {EINVAL, EACCES, EBADF, EMFILE, ENOENT}},
+    [GF_FOP_READDIRP] = {.error_no_count = 5,
+                         .error_no = {EINVAL, EACCES, EBADF, EMFILE, ENOENT}},
+    [GF_FOP_FSYNCDIR] = {.error_no_count = 4,
+                         .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_ACCESS] = {.error_no_count = 8,
+                       .error_no = {EACCES, ENAMETOOLONG, ENOENT, ENOTDIR,
+                                    EROFS, EFAULT, EINVAL, EIO}},
+    [GF_FOP_FTRUNCATE] = {.error_no_count = 9,
+                          .error_no = {EACCES, EFAULT, EFBIG, EINTR, EINVAL,
+                                       EIO, EISDIR, ENAMETOOLONG, ENOENT}},
+    [GF_FOP_FSTAT] = {.error_no_count = 7,
+                      .error_no = {EACCES, EBADF, EFAULT, ENAMETOOLONG, ENOENT,
+                                   ENOMEM, ENOTDIR}},
+    [GF_FOP_LK] = {.error_no_count = 4,
+                   .error_no = {EACCES, EFAULT, ENOENT, EINTR}},
+    [GF_FOP_XATTROP] = {.error_no_count = 5,
+                        .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOSYS,
+                                     ENOENT}},
+    [GF_FOP_FXATTROP] = {.error_no_count = 4,
+                         .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_INODELK] = {.error_no_count = 3,
+                        .error_no = {EACCES, EINTR, ENAMETOOLONG}},
+    [GF_FOP_FINODELK] = {.error_no_count = 4,
+                         .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}},
+    [GF_FOP_ENTRYLK] = {.error_no_count = 3,
+                        .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FENTRYLK] = {.error_no_count = 10,
+                         .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                      ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                      ENOMEM}},
+    [GF_FOP_SETATTR] = {.error_no_count = 10,
+                        .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT,
+                                     ENOMEM, ENOTDIR, EPERM, EROFS, EIO}},
+    [GF_FOP_FSETATTR] = {.error_no_count = 11,
+                         .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT,
+                                      ENOMEM, ENOTDIR, EPERM, EROFS, EBADF,
+                                      EIO}},
+    [GF_FOP_GETSPEC] = {.error_no_count = 3,
+                        .error_no = {EACCES, ENAMETOOLONG, EINTR}}};
+
+int
+generate_rand_no(int op_no)
+{
+    int rand_no = 0;
+    int error_no_list_size = 0;
+
+    error_no_list_size = sizeof(error_no_list) / sizeof(error_no_list[0]);
+
+    if (op_no < error_no_list_size)
+        /* coverity[DC.WEAK_CRYPTO] */
+        rand_no = rand() % error_no_list[op_no].error_no_count;
+    return rand_no;
+}
+
+int
+conv_errno_to_int(char **error_no)
+{
+    if (!strcmp((*error_no), "ENOENT"))
+        return ENOENT;
+    else if (!strcmp((*error_no), "ENOTDIR"))
+        return ENOTDIR;
+    else if (!strcmp((*error_no), "ENAMETOOLONG"))
+        return ENAMETOOLONG;
+    else if (!strcmp((*error_no), "EACCES"))
+        return EACCES;
+    else if (!strcmp((*error_no), "EBADF"))
+        return EBADF;
+    else if (!strcmp((*error_no), "EFAULT"))
+        return EFAULT;
+    else if (!strcmp((*error_no), "ENOMEM"))
+        return ENOMEM;
+    else if (!strcmp((*error_no), "EINVAL"))
+        return EINVAL;
+    else if (!strcmp((*error_no), "EIO"))
+        return EIO;
+    else if (!strcmp((*error_no), "EEXIST"))
+        return EEXIST;
+    else if (!strcmp((*error_no), "ENOSPC"))
+        return ENOSPC;
+    else if (!strcmp((*error_no), "EPERM"))
+        return EPERM;
+    else if (!strcmp((*error_no), "EROFS"))
+        return EROFS;
+    else if (!strcmp((*error_no), "EBUSY"))
+        return EBUSY;
+    else if (!strcmp((*error_no), "EISDIR"))
+        return EISDIR;
+    else if (!strcmp((*error_no), "ENOTEMPTY"))
+        return ENOTEMPTY;
+    else if (!strcmp((*error_no), "EMLINK"))
+        return EMLINK;
+    else if (!strcmp((*error_no), "ENODEV"))
+        return ENODEV;
+    else if (!strcmp((*error_no), "EXDEV"))
+        return EXDEV;
+    else if (!strcmp((*error_no), "EMFILE"))
+        return EMFILE;
+    else if (!strcmp((*error_no), "ENFILE"))
+        return ENFILE;
+    else if (!strcmp((*error_no), "ENOSYS"))
+        return ENOSYS;
+    else if (!strcmp((*error_no), "EINTR"))
+        return EINTR;
+    else if (!strcmp((*error_no), "EFBIG"))
+        return EFBIG;
+    else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+        return GF_ERROR_SHORT_WRITE;
+    else
+        return EAGAIN;
+}
+
+int
+error_gen(xlator_t *this, int op_no)
+{
+    eg_t *egp = NULL;
+    int count = 0;
+    int error_no_int = 0;
+    int rand_no = 0;
+    int ret = 0;
+    gf_boolean_t should_err = _gf_false;
+    int error_no_list_size = 0;
+
+    egp = this->private;
+
+    if (egp->random_failure) {
+        /*
+         * I honestly don't know why anyone would use this "feature"
+         * but I'll try to preserve its functionality anyway.  Without
+         * locking twice to update failure_iter_no and egp->op_count
+         * separately, then not locking at all to update
+         * egp->failure_iter_no.  That's not needed for compatibility,
+         * and it's abhorrently wrong.  I have *some* standards.
+         */
+        LOCK(&egp->lock);
         {
-                count = ++egp->op_count;
-                failure_iter_no = egp->failure_iter_no;
-                error_no = egp->error_no;
+            count = ++(egp->op_count);
+            error_no_int = egp->error_no_int;
+            if ((count % egp->failure_iter_no) == 0) {
+                egp->op_count = 0;
+                /* coverity[DC.WEAK_CRYPTO] */
+                egp->failure_iter_no = 3 + (rand() % GF_UNIVERSAL_ANSWER);
+                should_err = _gf_true;
+            }
         }
-        UNLOCK (&egp->lock);
-
-        if((count % failure_iter_no) == 0) {
-                LOCK (&egp->lock);
-                {
-                        egp->op_count = 0;
-                }
-                UNLOCK (&egp->lock);
-
-                if (error_no)
-                        ret = conv_errno_to_int (&error_no);
-                else {
-
-                        rand_no = generate_rand_no (op_no);
-                        if (op_no >= GF_FOP_MAXVALUE)
-                                op_no = 0;
-                        if (rand_no >= error_no_list[op_no].error_no_count)
-                                rand_no = 0;
-                        ret = error_no_list[op_no].error_no[rand_no];
-                }
-                if (egp->random_failure == _gf_true)
-                        egp->failure_iter_no = 3 + (rand () % GF_UNIVERSAL_ANSWER);
+        UNLOCK(&egp->lock);
+    } else {
+        /*
+         * It turns out that rand() is almost universally implemented
+         * as a linear congruential PRNG, which is about as cheap as
+         * it gets.  This gets us real random behavior, including
+         * phenomena like streaks and dry spells, with controllable
+         * long-term probability, cheaply.
+         */
+        if ((rand() % FAILURE_GRANULARITY) < egp->failure_iter_no) {
+            should_err = _gf_true;
         }
-        return ret;
-}
-
+    }
+
+    error_no_list_size = sizeof(error_no_list) / sizeof(error_no_list[0]);
+    if (should_err) {
+        if (error_no_int)
+            ret = error_no_int;
+        else {
+            rand_no = generate_rand_no(op_no);
+            if (op_no >= error_no_list_size)
+                op_no = 0;
+            if (rand_no >= error_no_list[op_no].error_no_count)
+                rand_no = 0;
+            ret = error_no_list[op_no].error_no[rand_no];
+        }
+    }
 
-int
-error_gen_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, inode_t *inode,
-		      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-	STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode,
-                             buf, xdata, postparent);
-        return 0;
+    return ret;
 }
 
-
 int
-error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  dict_t *xdata)
+error_gen_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LOOKUP];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LOOKUP);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LOOKUP];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                                     NULL);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LOOKUP);
 
-	STACK_WIND (frame, error_gen_lookup_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup,
-		    loc, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL);
         return 0;
-}
-
+    }
 
-int
-error_gen_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
+                    loc, xdata);
+    return 0;
 }
 
 int
-error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+error_gen_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_STAT];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_STAT];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_STAT);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_STAT);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(stat, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_stat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->stat,
-		    loc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 }
 
-
 int
-error_gen_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+error_gen_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SETATTR];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SETATTR);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(setattr, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+    return 0;
+}
 
 int
-error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+error_gen_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SETATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSETATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SETATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSETATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr,
-		    loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+error_gen_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSETATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_TRUNCATE];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSETATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_TRUNCATE);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(truncate, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+    return 0;
 }
 
-
 int
-error_gen_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno,
-			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+error_gen_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FTRUNCATE];
 
-int
-error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_TRUNCATE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_TRUNCATE);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (truncate, frame, -1, op_errno,
-                                     NULL, NULL, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FTRUNCATE);
 
-	STACK_WIND (frame, error_gen_truncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     off_t offset, dict_t *xdata)
+error_gen_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+                 dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp =NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FTRUNCATE];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FTRUNCATE);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_ACCESS];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno,
-                                     NULL, NULL, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_ACCESS);
 
-	STACK_WIND (frame, error_gen_ftruncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(access, frame, -1, op_errno, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->access,
+                    loc, mask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t mask, dict_t *xdata)
+error_gen_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                   dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_ACCESS];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READLINK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_ACCESS);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READLINK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (access, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readlink, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_access_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->access,
-		    loc, mask, xdata);
-        return 0;
-}
-
-
-int
-error_gen_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno,
-			const char *path, struct iatt *sbuf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink,
+                    loc, size, xdata);
+    return 0;
 }
 
-
 int
-error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size, dict_t *xdata)
+error_gen_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READLINK];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_MKNOD];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READLINK);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_MKNOD);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_readlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readlink,
-		    loc, size, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, rdev, umask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+error_gen_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                mode_t umask, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
-                             inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_MKDIR];
 
-int
-error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		 mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_MKNOD];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_MKNOD);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_MKDIR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_mknod_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev, umask, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+error_gen_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                 dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
-                             inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-int
-error_gen_mkdir (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_MKDIR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_MKDIR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, error_gen_mkdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode, umask, xdata);
-        return 0;
-}
+    egp = this->private;
+    enable = egp->enable[GF_FOP_UNLINK];
 
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_UNLINK);
 
-int
-error_gen_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+    return 0;
+}
 
 int
-error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-                  dict_t *xdata)
+error_gen_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_UNLINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_UNLINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
-                                     xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, error_gen_unlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    loc, xflag, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_RMDIR];
 
-int
-error_gen_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_RMDIR);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+    return 0;
+}
 
 int
-error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-                 dict_t *xdata)
+error_gen_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_RMDIR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SYMLINK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_RMDIR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SYMLINK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(symlink, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL, NULL); /* pre & post parent attr */
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_rmdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rmdir,
-		    loc, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, inode_t *inode,
-                       struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+error_gen_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_RENAME];
 
-int
-error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		   loc_t *loc, mode_t umask, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SYMLINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SYMLINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL); /* pre & post parent attr */
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_symlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc, umask, xdata);
-        return 0;
-}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_RENAME);
 
-
-int
-error_gen_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                      struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent,
-                      dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
         return 0;
-}
-
+    }
 
-int
-error_gen_rename (call_frame_t *frame, xlator_t *this,
-		  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_RENAME];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_RENAME);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_rename_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-error_gen_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+error_gen_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LINK];
 
-int
-error_gen_link (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_link_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc, xdata);
-        return 0;
-}
-
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LINK);
 
-int
-error_gen_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
-		      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+    return 0;
+}
 
 int
-error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-                  dict_t *xdata)
+error_gen_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                 mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_CREATE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_CREATE);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_create_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, umask, fd, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_CREATE];
 
-int
-error_gen_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_CREATE);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+    return 0;
+}
 
 int
-error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		int32_t flags, fd_t *fd, dict_t *xdata)
+error_gen_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_OPEN];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_OPEN];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_OPEN);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_OPEN);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(open, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_open_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-error_gen_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno,
-		     struct iovec *vector, int32_t count,
-		     struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+error_gen_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READ];
 
-int
-error_gen_readv (call_frame_t *frame, xlator_t *this,
-		 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READ];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READ);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READ);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
-                                     NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                            xdata);
         return 0;
-	}
+    }
 
-
-	STACK_WIND (frame, error_gen_readv_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+error_gen_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+                 struct iobref *iobref, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+    struct iovec *shortvec = NULL;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_WRITE];
 
-int
-error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  struct iovec *vector, int32_t count,
-		  off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_WRITE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_WRITE);
-
-	if (op_errno == GF_ERROR_SHORT_WRITE) {
-		struct iovec *shortvec;
-
-		/*
-		 * A short write error returns some value less than what was
-		 * requested from a write. To simulate this, replace the vector
-		 * with one half the size;
-		 */
-		shortvec = iov_dup(vector, 1);
-		shortvec->iov_len /= 2;
-
-		STACK_WIND(frame, error_gen_writev_cbk, FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->writev, fd, shortvec, count,
-			   off, flags, iobref, xdata);
-		GF_FREE(shortvec);
-		return 0;
-	} else if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
-        	return 0;
-	}
-
-	STACK_WIND (frame, error_gen_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, off, flags, iobref, xdata);
-        return 0;
-}
-
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_WRITE);
 
-int
-error_gen_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+    if (op_errno == GF_ERROR_SHORT_WRITE) {
+        /*
+         * A short write error returns some value less than what was
+         * requested from a write. To simulate this, replace the vector
+         * with one half the size;
+         */
+        shortvec = iov_dup(vector, 1);
+        shortvec->iov_len /= 2;
+        count = 1;
+        goto wind;
+    } else if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(writev, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    fd, shortvec ? shortvec : vector, count, off, flags, iobref,
+                    xdata);
 
+    if (shortvec)
+        GF_FREE(shortvec);
+    return 0;
+}
 
 int
-error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+error_gen_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FLUSH];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FLUSH];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FLUSH);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FLUSH);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (flush, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(flush, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_flush_cbk,
-                    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->flush,
-		    fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->flush,
+                    fd, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fsync_cbk (call_frame_t *frame, void *cookie,
-		     xlator_t *this, int32_t op_ret,
-		     int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+error_gen_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSYNC];
 
-int
-error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSYNC];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSYNC);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSYNC);
 
-	STACK_WIND (frame, error_gen_fsync_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsync,
-		    fd, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsync, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+error_gen_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSTAT];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSTAT];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSTAT);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSTAT);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fstat, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fstat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat,
-		    fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 }
 
-
 int
-error_gen_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-
-int
-error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
+error_gen_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                  dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_OPENDIR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_OPENDIR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_OPENDIR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_OPENDIR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(opendir, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_opendir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+    return 0;
 }
 
 int
-error_gen_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    int32_t flags, dict_t *xdata)
+error_gen_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                   dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSYNCDIR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSYNCDIR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSYNCDIR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSYNCDIR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fsyncdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsyncdir,
-		    fd, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsyncdir,
+                    fd, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+error_gen_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_STATFS];
 
-int
-error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_STATFS);
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_STATFS];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_STATFS);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(statfs, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
-
-	STACK_WIND (frame, error_gen_statfs_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->statfs,
-		    loc, xdata);
-        return 0;
-}
+    }
 
-
-int
-error_gen_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs,
+                    loc, xdata);
+    return 0;
 }
 
-
 int
-error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *dict, int32_t flags, dict_t *xdata)
+error_gen_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SETXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SETXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SETXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SETXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(setxattr, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_setxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setxattr,
-		    loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+error_gen_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_GETXATTR];
 
-int
-error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    const char *name, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_GETXATTR);
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_GETXATTR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_GETXATTR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(getxattr, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_getxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
+                    loc, name, xdata);
+    return 0;
 }
 
 int
-error_gen_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     dict_t *dict, int32_t flags, dict_t *xdata)
+error_gen_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSETXATTR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSETXATTR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSETXATTR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSETXATTR);
 
-	STACK_WIND (frame, error_gen_fsetxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetxattr,
-		    fd, dict, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, op_errno, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name, dict_t *xdata)
+error_gen_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    const char *name, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FGETXATTR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FGETXATTR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FGETXATTR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FGETXATTR);
 
-	STACK_WIND (frame, error_gen_fgetxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fgetxattr,
-		    fd, name, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fgetxattr, frame, -1, op_errno, NULL, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+error_gen_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_XATTROP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_XATTROP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_XATTROP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_XATTROP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(xattrop, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_xattrop_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->xattrop,
-		    loc, flags, dict, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->xattrop,
+                    loc, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+error_gen_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FXATTROP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FXATTROP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FXATTROP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FXATTROP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fxattrop_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fxattrop,
-		    fd, flags, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fxattrop,
+                    fd, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-error_gen_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_REMOVEXATTR];
 
-int
-error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_REMOVEXATTR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_REMOVEXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_REMOVEXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(removexattr, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_removexattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->removexattr,
-		    loc, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 int
-error_gen_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        const char *name, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FREMOVEXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FREMOVEXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FREMOVEXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FREMOVEXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fremovexattr, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fremovexattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fremovexattr,
-		    fd, name, xdata);
-        return 0;
-}
-
-
-int
-error_gen_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	      struct gf_flock *lock, dict_t *xdata)
+error_gen_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+             struct gf_flock *lock, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LK];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LK);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(lk, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_lk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk, fd,
+                    cmd, lock, xdata);
+    return 0;
 }
 
-
 int
-error_gen_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_inodelk (call_frame_t *frame, xlator_t *this,
-		   const char *volume, loc_t *loc, int32_t cmd,
-                   struct gf_flock *lock, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_INODELK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_INODELK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_INODELK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_INODELK);
 
-	STACK_WIND (frame, error_gen_inodelk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(inodelk, frame, -1, op_errno, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->inodelk,
+                    volume, loc, cmd, lock, xdata);
+    return 0;
 }
 
-
 int
-error_gen_finodelk (call_frame_t *frame, xlator_t *this,
-		    const char *volume, fd_t *fd, int32_t cmd,
-                    struct gf_flock *lock, dict_t *xdata)
+error_gen_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FINODELK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FINODELK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FINODELK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FINODELK);
 
-	STACK_WIND (frame, error_gen_finodelk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->finodelk,
-		    volume, fd, cmd, lock, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(finodelk, frame, -1, op_errno, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
+    return 0;
 }
 
-
 int
-error_gen_entrylk (call_frame_t *frame, xlator_t *this,
-		   const char *volume, loc_t *loc, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+error_gen_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  loc_t *loc, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_ENTRYLK];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_ENTRYLK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_ENTRYLK);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_ENTRYLK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(entrylk, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_entrylk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, const char *basename, entrylk_cmd cmd,
+                   entrylk_type type, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FENTRYLK];
 
-int
-error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
-		    const char *volume, fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
-{
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FENTRYLK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FENTRYLK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno, xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, error_gen_fentrylk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fentrylk,
-		    volume, fd, basename, cmd, type, xdata);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FENTRYLK);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fentrylk, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
+    return 0;
 }
 
+int
+error_gen_getspec(call_frame_t *frame, xlator_t *this, const char *key,
+                  int32_t flags)
+{
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-/* Management operations */
+    egp = this->private;
+    enable = egp->enable[GF_FOP_GETSPEC];
 
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_GETSPEC);
 
-int
-error_gen_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, char *spec_data)
-{
-	STACK_UNWIND_STRICT (getspec, frame, op_ret, op_errno, spec_data);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(getspec, frame, -1, op_errno, NULL);
         return 0;
-}
+    }
 
-
-int
-error_gen_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-		   int32_t flags)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_GETSPEC];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_GETSPEC);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getspec, frame, -1, op_errno, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_getspec_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getspec,
-		    key, flags);
-	return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->getspec,
+                    key, flags);
+    return 0;
 }
 
-
 int
-error_gen_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                       dict_t *xdata)
+error_gen_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
-	return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READDIR];
 
-int
-error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		   size_t size, off_t off, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READDIR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READDIR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_readdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readdir,
-		    fd, size, off, xdata);
-	return 0;
-}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READDIR);
 
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readdir, frame, -1, op_errno, NULL, xdata);
+        return 0;
+    }
 
-int
-error_gen_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdir,
+                    fd, size, off, xdata);
+    return 0;
 }
 
-
 int
-error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                    off_t off, dict_t *dict)
+error_gen_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                   off_t off, dict_t *dict)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READDIRP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READDIRP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READDIRP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READDIRP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
-		return 0;
-	}
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+        return 0;
+    }
 
-	STACK_WIND (frame, error_gen_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
                     fd, size, off, dict);
-	return 0;
+    return 0;
 }
 
 static void
-error_gen_set_failure (eg_t *pvt, int percent)
+error_gen_set_failure(eg_t *pvt, double percent)
 {
-        GF_ASSERT (pvt);
+    double ppm;
+
+    GF_ASSERT(pvt);
 
-        if (percent)
-                pvt->failure_iter_no = 100/percent;
-        else
-                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
+    ppm = (percent / 100.0) * (double)FAILURE_GRANULARITY;
+    pvt->failure_iter_no = (int)ppm;
 }
 
 static void
-error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
+error_gen_parse_fill_fops(eg_t *pvt, char *enable_fops)
 {
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
-        xlator_t        *this = THIS;
-        char            *saveptr = NULL;
+    char *op_no_str = NULL;
+    int op_no = -1;
+    int i = 0;
+    xlator_t *this = THIS;
+    char *saveptr = NULL;
 
-        GF_ASSERT (pvt);
-        GF_ASSERT (this);
+    GF_ASSERT(pvt);
+    GF_ASSERT(this);
 
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        pvt->enable[i] = 0;
+
+    if (!enable_fops) {
+        gf_log(this->name, GF_LOG_WARNING, "All fops are enabled.");
         for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                pvt->enable[i] = 0;
-
-        if (!enable_fops) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "All fops are enabled.");
-                for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                        pvt->enable[i] = 1;
-        } else {
-                op_no_str = strtok_r (enable_fops, ",", &saveptr);
-                while (op_no_str) {
-                        op_no = get_fop_int (&op_no_str);
-                        if (op_no == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Wrong option value %s", op_no_str);
-                        } else
-                                pvt->enable[op_no] = 1;
-
-                        op_no_str = strtok_r (NULL, ",", &saveptr);
-                }
+            pvt->enable[i] = 1;
+    } else {
+        op_no_str = strtok_r(enable_fops, ",", &saveptr);
+        while (op_no_str) {
+            op_no = gf_fop_int(op_no_str);
+            if (op_no == -1) {
+                gf_log(this->name, GF_LOG_WARNING, "Wrong option value %s",
+                       op_no_str);
+            } else
+                pvt->enable[op_no] = 1;
+
+            op_no_str = strtok_r(NULL, ",", &saveptr);
         }
+    }
 }
 
 int32_t
-error_gen_priv_dump (xlator_t *this)
+error_gen_priv_dump(xlator_t *this)
 {
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        int             ret = -1;
-        eg_t            *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    int ret = -1;
+    eg_t *conf = NULL;
 
-        if (!this)
-            goto out;
+    if (!this)
+        goto out;
 
-        conf = this->private;
-        if (!conf)
-            goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        ret = TRY_LOCK(&conf->lock);
-        if (ret != 0) {
-                return ret;
-        }
+    ret = TRY_LOCK(&conf->lock);
+    if (ret != 0) {
+        return ret;
+    }
 
-        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
-                               this->name);
+    gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+    gf_proc_dump_build_key(key_prefix, "xlator.debug.error-gen", "%s.priv",
+                           this->name);
 
-        gf_proc_dump_write("op_count", "%d", conf->op_count);
-        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
-        gf_proc_dump_write("error_no", "%s", conf->error_no);
-        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+    gf_proc_dump_write("op_count", "%d", conf->op_count);
+    gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+    gf_proc_dump_write("error_no", "%d", conf->error_no_int);
+    gf_proc_dump_write("random_failure", "%d", conf->random_failure);
 
-        UNLOCK(&conf->lock);
+    UNLOCK(&conf->lock);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_error_gen_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_error_gen_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        eg_t            *pvt = NULL;
-        int32_t          ret = 0;
-        char            *error_enable_fops = NULL;
-        int32_t          failure_percent_int = 0;
+    eg_t *pvt = NULL;
+    int32_t ret = 0;
+    char *error_enable_fops = NULL;
+    char *error_no = NULL;
+    double failure_percent_dbl = 0.0;
 
-        if (!this || !this->private)
-                goto out;
+    if (!this || !this->private)
+        goto out;
 
-        pvt = this->private;
+    pvt = this->private;
 
-        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+    ret = -1;
 
-        GF_OPTION_RECONF ("failure", failure_percent_int, options, int32,
-                          out);
+    GF_OPTION_RECONF("error-no", error_no, options, str, out);
 
-        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+    if (error_no)
+        pvt->error_no_int = conv_errno_to_int(&error_no);
 
-        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
-                          bool, out);
+    GF_OPTION_RECONF("failure", failure_percent_dbl, options, percent, out);
 
-        error_gen_parse_fill_fops (pvt, error_enable_fops);
-        error_gen_set_failure (pvt, failure_percent_int);
+    GF_OPTION_RECONF("enable", error_enable_fops, options, str, out);
 
-        ret = 0;
+    GF_OPTION_RECONF("random-failure", pvt->random_failure, options, bool, out);
+
+    error_gen_parse_fill_fops(pvt, error_enable_fops);
+    error_gen_set_failure(pvt, failure_percent_dbl);
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
-        return ret;
+    gf_log(this ? this->name : "error-gen", GF_LOG_DEBUG,
+           "reconfigure returning %d", ret);
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        eg_t            *pvt = NULL;
-        int32_t          ret = 0;
-        char            *error_enable_fops = NULL;
-        int32_t          failure_percent_int = 0;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error-gen not configured with one subvolume");
-                ret = -1;
-                goto out;
-        }
+    eg_t *pvt = NULL;
+    int32_t ret = 0;
+    char *error_enable_fops = NULL;
+    char *error_no = NULL;
+    double failure_percent_dbl = 0.0;
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "error-gen not configured with one subvolume");
+        ret = -1;
+        goto out;
+    }
 
-        pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-        if (!pvt) {
-                ret = -1;
-                goto out;
-        }
+    pvt = GF_CALLOC(1, sizeof(eg_t), gf_error_gen_mt_eg_t);
+
+    if (!pvt) {
+        ret = -1;
+        goto out;
+    }
+
+    LOCK_INIT(&pvt->lock);
 
-        LOCK_INIT (&pvt->lock);
+    ret = -1;
 
-        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
+    GF_OPTION_INIT("error-no", error_no, str, out);
 
-        GF_OPTION_INIT ("failure", failure_percent_int, int32, out);
+    if (error_no)
+        pvt->error_no_int = conv_errno_to_int(&error_no);
 
-        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
+    GF_OPTION_INIT("failure", failure_percent_dbl, percent, out);
 
-        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
+    GF_OPTION_INIT("enable", error_enable_fops, str, out);
 
+    GF_OPTION_INIT("random-failure", pvt->random_failure, bool, out);
 
-        error_gen_parse_fill_fops (pvt, error_enable_fops);
-        error_gen_set_failure (pvt, failure_percent_int);
+    error_gen_parse_fill_fops(pvt, error_enable_fops);
+    error_gen_set_failure(pvt, failure_percent_dbl);
 
-        this->private = pvt;
+    this->private = pvt;
 
-        /* Give some seed value here */
-        srand (time(NULL));
+    /* Give some seed value here. */
+    srand(gf_time());
+
+    ret = 0;
 out:
-        if (ret)
-                GF_FREE (pvt);
-        return ret;
+    if (ret)
+        GF_FREE(pvt);
+    return ret;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        eg_t            *pvt = NULL;
+    eg_t *pvt = NULL;
 
-        if (!this)
-                 return;
-        pvt = this->private;
-
-        if (pvt) {
-                LOCK_DESTROY (&pvt->lock);
-                GF_FREE (pvt);
-                gf_log (this->name, GF_LOG_DEBUG, "fini called");
-        }
+    if (!this)
         return;
+    pvt = this->private;
+
+    if (pvt) {
+        LOCK_DESTROY(&pvt->lock);
+        GF_FREE(pvt);
+        gf_log(this->name, GF_LOG_DEBUG, "fini called");
+    }
+    return;
 }
 
 struct xlator_dumpops dumpops = {
-        .priv = error_gen_priv_dump,
+    .priv = error_gen_priv_dump,
 };
 
-struct xlator_fops cbks;
+struct xlator_cbks cbks;
 
 struct xlator_fops fops = {
-	.lookup      = error_gen_lookup,
-	.stat        = error_gen_stat,
-	.readlink    = error_gen_readlink,
-	.mknod       = error_gen_mknod,
-	.mkdir       = error_gen_mkdir,
-	.unlink      = error_gen_unlink,
-	.rmdir       = error_gen_rmdir,
-	.symlink     = error_gen_symlink,
-	.rename      = error_gen_rename,
-	.link        = error_gen_link,
-	.truncate    = error_gen_truncate,
-	.create      = error_gen_create,
-	.open        = error_gen_open,
-	.readv       = error_gen_readv,
-	.writev      = error_gen_writev,
-	.statfs      = error_gen_statfs,
-	.flush       = error_gen_flush,
-	.fsync       = error_gen_fsync,
-	.setxattr    = error_gen_setxattr,
-	.getxattr    = error_gen_getxattr,
-	.removexattr = error_gen_removexattr,
-	.fsetxattr    = error_gen_fsetxattr,
-	.fgetxattr    = error_gen_fgetxattr,
-	.fremovexattr = error_gen_fremovexattr,
-	.opendir     = error_gen_opendir,
-	.readdir     = error_gen_readdir,
-	.readdirp    = error_gen_readdirp,
-	.fsyncdir    = error_gen_fsyncdir,
-	.access      = error_gen_access,
-	.ftruncate   = error_gen_ftruncate,
-	.fstat       = error_gen_fstat,
-	.lk          = error_gen_lk,
-	.lookup_cbk  = error_gen_lookup_cbk,
-	.xattrop     = error_gen_xattrop,
-	.fxattrop    = error_gen_fxattrop,
-	.inodelk     = error_gen_inodelk,
-	.finodelk    = error_gen_finodelk,
-	.entrylk     = error_gen_entrylk,
-	.fentrylk    = error_gen_fentrylk,
-        .setattr     = error_gen_setattr,
-        .fsetattr    = error_gen_fsetattr,
-	.getspec     = error_gen_getspec,
+    .lookup = error_gen_lookup,
+    .stat = error_gen_stat,
+    .readlink = error_gen_readlink,
+    .mknod = error_gen_mknod,
+    .mkdir = error_gen_mkdir,
+    .unlink = error_gen_unlink,
+    .rmdir = error_gen_rmdir,
+    .symlink = error_gen_symlink,
+    .rename = error_gen_rename,
+    .link = error_gen_link,
+    .truncate = error_gen_truncate,
+    .create = error_gen_create,
+    .open = error_gen_open,
+    .readv = error_gen_readv,
+    .writev = error_gen_writev,
+    .statfs = error_gen_statfs,
+    .flush = error_gen_flush,
+    .fsync = error_gen_fsync,
+    .setxattr = error_gen_setxattr,
+    .getxattr = error_gen_getxattr,
+    .removexattr = error_gen_removexattr,
+    .fsetxattr = error_gen_fsetxattr,
+    .fgetxattr = error_gen_fgetxattr,
+    .fremovexattr = error_gen_fremovexattr,
+    .opendir = error_gen_opendir,
+    .readdir = error_gen_readdir,
+    .readdirp = error_gen_readdirp,
+    .fsyncdir = error_gen_fsyncdir,
+    .access = error_gen_access,
+    .ftruncate = error_gen_ftruncate,
+    .fstat = error_gen_fstat,
+    .lk = error_gen_lk,
+    .xattrop = error_gen_xattrop,
+    .fxattrop = error_gen_fxattrop,
+    .inodelk = error_gen_inodelk,
+    .finodelk = error_gen_finodelk,
+    .entrylk = error_gen_entrylk,
+    .fentrylk = error_gen_fentrylk,
+    .setattr = error_gen_setattr,
+    .fsetattr = error_gen_fsetattr,
+    .getspec = error_gen_getspec,
 };
 
 struct volume_options options[] = {
-        { .key  = {"failure"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Percentage failure of operations when enabled.",
-        },
-
-        { .key  = {"error-no"},
-          .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
-                    "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
-                    "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
-                    "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
-                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key = {"random-failure"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-
-        { .key  = {"enable"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key  = {NULL} }
+    {
+        .key = {"failure"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .description = "Percentage failure of operations when enabled.",
+    },
+
+    {
+        .key = {"error-no"},
+        .value = {"ENOENT",
+                  "ENOTDIR",
+                  "ENAMETOOLONG",
+                  "EACCES",
+                  "EBADF",
+                  "EFAULT",
+                  "ENOMEM",
+                  "EINVAL",
+                  "EIO",
+                  "EEXIST",
+                  "ENOSPC",
+                  "EPERM",
+                  "EROFS",
+                  "EBUSY",
+                  "EISDIR",
+                  "ENOTEMPTY",
+                  "EMLINK"
+                  "ENODEV",
+                  "EXDEV",
+                  "EMFILE",
+                  "ENFILE",
+                  "ENOSYS",
+                  "EINTR",
+                  "EFBIG",
+                  "EAGAIN",
+                  "GF_ERROR_SHORT_WRITE"},
+        .type = GF_OPTION_TYPE_STR,
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+
+    },
+
+    {
+        .key = {"random-failure"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+
+    {
+        .key = {"enable", "error-fops"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Accepts a string which takes ',' separated fop "
+                       "strings to denote which fops are enabled for error",
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "error-gen",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
index 351f5dc99d6..2478cd5b21c 100644
--- a/xlators/debug/error-gen/src/error-gen.h
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -22,22 +22,28 @@
  * range.
  */
 enum GF_PSEUDO_ERRORS {
-	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
-	GF_ERROR_MAX
+    GF_ERROR_SHORT_WRITE = 1000, /* short writev return value */
+    GF_ERROR_MAX
 };
 
 typedef struct {
-        int enable[GF_FOP_MAXVALUE];
-        int op_count;
-        int failure_iter_no;
-        char *error_no;
-        gf_boolean_t random_failure;
-        gf_lock_t lock;
+    int enable[GF_FOP_MAXVALUE];
+    int op_count;
+    /*
+     * This is only an iteration number in the random-failure case.  For
+     * the normal controlled-probability case, it's actually a numerator
+     * for the failure probability (see FAILURE_GRANULARITY declaration).
+     * It's just not worth blowing up the diff by changing it.
+     */
+    int failure_iter_no;
+    int error_no_int;
+    gf_boolean_t random_failure;
+    gf_lock_t lock;
 } eg_t;
 
 typedef struct {
-        int error_no_count;
-	int error_no[20];
+    int error_no_count;
+    int error_no[20];
 } sys_error_t;
 
 #endif
diff --git a/xlators/debug/io-stats/src/Makefile.am b/xlators/debug/io-stats/src/Makefile.am
index c5df598549a..c69f3caf0fe 100644
--- a/xlators/debug/io-stats/src/Makefile.am
+++ b/xlators/debug/io-stats/src/Makefile.am
@@ -10,9 +10,9 @@ io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 noinst_HEADERS = io-stats-mem-types.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(top_srcdir)/rpc/rpc-lib/src \
-        -DDATADIR=\"$(localstatedir)\"
+	-DDATADIR=\"$(localstatedir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index 9dde9373264..51d38d8b97c 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -11,18 +11,17 @@
 #ifndef __IO_STATS_MEM_TYPES_H__
 #define __IO_STATS_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 extern const char *__progname;
 
 enum gf_io_stats_mem_types_ {
-        gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
-        gf_io_stats_mt_ios_fd,
-        gf_io_stats_mt_ios_stat,
-        gf_io_stats_mt_ios_stat_list,
-        gf_io_stats_mt_ios_sample_buf,
-        gf_io_stats_mt_ios_sample,
-        gf_io_stats_mt_end
+    gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
+    gf_io_stats_mt_ios_fd,
+    gf_io_stats_mt_ios_stat,
+    gf_io_stats_mt_ios_stat_list,
+    gf_io_stats_mt_ios_sample_buf,
+    gf_io_stats_mt_ios_sample,
+    gf_io_stats_mt_end
 };
 #endif
-
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 62e85bbb99b..aa00c446e5a 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -7,8 +7,8 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "xlator.h"
-#include "syscall.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/syscall.h>
 
 /**
  * xlators/debug/io_stats :
@@ -17,179 +17,210 @@
  *
  *  a) total read data - since process start, last interval and per fd
  *  b) total write data - since process start, last interval and per fd
- *  c) counts of read IO block size - since process start, last interval and per fd
- *  d) counts of write IO block size - since process start, last interval and per fd
- *  e) counts of all FOP types passing through it
+ *  c) counts of read IO block size - since process start, last interval and per
+ * fd d) counts of write IO block size - since process start, last interval and
+ * per fd e) counts of all FOP types passing through it
  *
- *  Usage: setfattr -n io-stats-dump /tmp/filename /mnt/gluster
+ *  Usage: setfattr -n trusted.io-stats-dump /tmp/filename /mnt/gluster
+ *      output is written to /tmp/filename.<iostats xlator instance name>
  *
  */
 
 #include <fnmatch.h>
 #include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
 #include "io-stats-mem-types.h"
 #include <stdarg.h>
-#include "defaults.h"
-#include "logging.h"
-#include "cli1-xdr.h"
-#include "statedump.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/syncop.h>
 #include <pwd.h>
 #include <grp.h>
+#include <glusterfs/upcall-utils.h>
+#include <glusterfs/async.h>
 
 #define MAX_LIST_MEMBERS 100
 #define DEFAULT_PWD_BUF_SZ 16384
 #define DEFAULT_GRP_BUF_SZ 16384
+#define IOS_BLOCK_COUNT_SIZE 32
+
+#define IOS_STATS_DUMP_DIR DEFAULT_VAR_RUN_DIRECTORY
 
 typedef enum {
-        IOS_STATS_TYPE_NONE,
-        IOS_STATS_TYPE_OPEN,
-        IOS_STATS_TYPE_READ,
-        IOS_STATS_TYPE_WRITE,
-        IOS_STATS_TYPE_OPENDIR,
-        IOS_STATS_TYPE_READDIRP,
-        IOS_STATS_TYPE_READ_THROUGHPUT,
-        IOS_STATS_TYPE_WRITE_THROUGHPUT,
-        IOS_STATS_TYPE_MAX
-}ios_stats_type_t;
+    IOS_STATS_TYPE_NONE,
+    IOS_STATS_TYPE_OPEN,
+    IOS_STATS_TYPE_READ,
+    IOS_STATS_TYPE_WRITE,
+    IOS_STATS_TYPE_OPENDIR,
+    IOS_STATS_TYPE_READDIRP,
+    IOS_STATS_TYPE_READ_THROUGHPUT,
+    IOS_STATS_TYPE_WRITE_THROUGHPUT,
+    IOS_STATS_TYPE_MAX
+} ios_stats_type_t;
 
 typedef enum {
-        IOS_STATS_THRU_READ,
-        IOS_STATS_THRU_WRITE,
-        IOS_STATS_THRU_MAX,
-}ios_stats_thru_t;
+    IOS_STATS_THRU_READ,
+    IOS_STATS_THRU_WRITE,
+    IOS_STATS_THRU_MAX,
+} ios_stats_thru_t;
+
+/* This is same as gf1_cli_info_op */
+/* had to be defined here again, so we have modularity between
+ xdr, xlator, and library functions */
+typedef enum ios_info_op {
+    GF_IOS_INFO_NONE = 0,
+    GF_IOS_INFO_ALL = 1,
+    GF_IOS_INFO_INCREMENTAL = 2,
+    GF_IOS_INFO_CUMULATIVE = 3,
+    GF_IOS_INFO_CLEAR = 4,
+} ios_info_op_t;
 
 struct ios_stat_lat {
-        struct timeval  time;
-        double          throughput;
+    struct timeval time;
+    double throughput;
 };
 
 struct ios_stat {
-        gf_lock_t       lock;
-        uuid_t          gfid;
-        char           *filename;
-        uint64_t        counters [IOS_STATS_TYPE_MAX];
-        struct ios_stat_lat thru_counters [IOS_STATS_THRU_MAX];
-        int             refcnt;
+    gf_lock_t lock;
+    uuid_t gfid;
+    char *filename;
+    gf_atomic_t counters[IOS_STATS_TYPE_MAX];
+    struct ios_stat_lat thru_counters[IOS_STATS_THRU_MAX];
+    gf_atomic_t refcnt;
 };
 
 struct ios_stat_list {
-        struct list_head  list;
-        struct ios_stat  *iosstat;
-        double            value;
+    struct list_head list;
+    struct ios_stat *iosstat;
+    double value;
 };
 
 struct ios_stat_head {
-       gf_lock_t                lock;
-       double                   min_cnt;
-       uint64_t                 members;
-       struct ios_stat_list    *iosstats;
+    gf_lock_t lock;
+    double min_cnt;
+    uint64_t members;
+    struct ios_stat_list *iosstats;
 };
 
 typedef struct _ios_sample_t {
-        uid_t  uid;
-        gid_t  gid;
-        char   identifier[UNIX_PATH_MAX];
-        glusterfs_fop_t fop_type;
-        struct timeval timestamp;
-        double elapsed;
+    uid_t uid;
+    gid_t gid;
+    char identifier[UNIX_PATH_MAX];
+    glusterfs_fop_t fop_type;
+    struct timeval timestamp;
+    double elapsed;
 } ios_sample_t;
 
-
 typedef struct _ios_sample_buf_t {
-        uint64_t        pos;  /* Position in write buffer */
-        uint64_t        size;  /* Size of ring buffer */
-        uint64_t        collected;  /* Number of samples we've collected */
-        uint64_t        observed;  /* Number of FOPs we've observed */
-        ios_sample_t    *ios_samples;  /* Our list of samples */
+    uint64_t pos;              /* Position in write buffer */
+    uint64_t size;             /* Size of ring buffer */
+    uint64_t collected;        /* Number of samples we've collected */
+    uint64_t observed;         /* Number of FOPs we've observed */
+    ios_sample_t *ios_samples; /* Our list of samples */
 } ios_sample_buf_t;
 
-
 struct ios_lat {
-        double      min;
-        double      max;
-        double      avg;
-        uint64_t    total;
+    double min;
+    double max;
+    double avg;
+    uint64_t total;
 };
 
 struct ios_global_stats {
-        uint64_t        data_written;
-        uint64_t        data_read;
-        uint64_t        block_count_write[32];
-        uint64_t        block_count_read[32];
-        uint64_t        fop_hits[GF_FOP_MAXVALUE];
-        struct timeval  started_at;
-        struct ios_lat  latency[GF_FOP_MAXVALUE];
-        uint64_t        nr_opens;
-        uint64_t        max_nr_opens;
-        struct timeval  max_openfd_time;
+    gf_atomic_t data_written;
+    gf_atomic_t data_read;
+    gf_atomic_t block_count_write[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t block_count_read[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t fop_hits[GF_FOP_MAXVALUE];
+    gf_atomic_t upcall_hits[GF_UPCALL_FLAGS_MAXVALUE];
+    time_t started_at;
+    struct ios_lat latency[GF_FOP_MAXVALUE];
+    uint64_t nr_opens;
+    uint64_t max_nr_opens;
+    struct timeval max_openfd_time;
 };
 
+typedef enum {
+    IOS_DUMP_TYPE_NONE = 0,
+    IOS_DUMP_TYPE_FILE = 1,
+    IOS_DUMP_TYPE_DICT = 2,
+    IOS_DUMP_TYPE_JSON_FILE = 3,
+    IOS_DUMP_TYPE_SAMPLES = 4,
+    IOS_DUMP_TYPE_MAX = 5
+} ios_dump_type_t;
+
 struct ios_conf {
-        gf_lock_t                 lock;
-        struct ios_global_stats   cumulative;
-        uint64_t                  increment;
-        struct ios_global_stats   incremental;
-        gf_boolean_t              dump_fd_stats;
-        gf_boolean_t              count_fop_hits;
-        gf_boolean_t              measure_latency;
-        struct ios_stat_head      list[IOS_STATS_TYPE_MAX];
-        struct ios_stat_head      thru_list[IOS_STATS_THRU_MAX];
-        int32_t                   ios_dump_interval;
-        pthread_t                 dump_thread;
-        gf_boolean_t              dump_thread_should_die;
-        gf_lock_t                 ios_sampling_lock;
-        int32_t                   ios_sample_interval;
-        int32_t                   ios_sample_buf_size;
-        ios_sample_buf_t          *ios_sample_buf;
-        struct dnscache           *dnscache;
-        int32_t                   ios_dnscache_ttl_sec;
+    gf_lock_t lock;
+    struct ios_global_stats cumulative;
+    uint64_t increment;
+    struct ios_global_stats incremental;
+    gf_boolean_t dump_fd_stats;
+    gf_boolean_t count_fop_hits;
+    gf_boolean_t measure_latency;
+    struct ios_stat_head list[IOS_STATS_TYPE_MAX];
+    struct ios_stat_head thru_list[IOS_STATS_THRU_MAX];
+    int32_t ios_dump_interval;
+    pthread_t dump_thread;
+    gf_boolean_t dump_thread_should_die;
+    gf_boolean_t dump_thread_running;
+    gf_lock_t ios_sampling_lock;
+    int32_t ios_sample_interval;
+    int32_t ios_sample_buf_size;
+    ios_sample_buf_t *ios_sample_buf;
+    struct dnscache *dnscache;
+    int32_t ios_dnscache_ttl_sec;
+    /*
+     * What we really need here is just a unique value to keep files
+     * created by this instance distinct from those created by any other.
+     * On the client side this isn't a problem, so we just use the
+     * translator name.  On the server side conflicts can occur, so the
+     * volfile-generation code automatically sets this (via an option)
+     * to be the brick path.
+     *
+     * NB While the *field* name has changed, it didn't seem worth changing
+     * all of the cases where "xlator_name" is used as a *variable* name.
+     */
+    char *unique_id;
+    ios_dump_type_t dump_format;
 };
 
-
 struct ios_fd {
-        char           *filename;
-        uint64_t        data_written;
-        uint64_t        data_read;
-        uint64_t        block_count_write[32];
-        uint64_t        block_count_read[32];
-        struct timeval  opened_at;
+    char *filename;
+    gf_atomic_t data_written;
+    gf_atomic_t data_read;
+    gf_atomic_t block_count_write[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t block_count_read[IOS_BLOCK_COUNT_SIZE];
+    struct timeval opened_at;
 };
 
-typedef enum {
-        IOS_DUMP_TYPE_NONE      = 0,
-        IOS_DUMP_TYPE_FILE      = 1,
-        IOS_DUMP_TYPE_DICT      = 2,
-        IOS_DUMP_TYPE_JSON_FILE = 3,
-        IOS_DUMP_TYPE_SAMPLES   = 4,
-        IOS_DUMP_TYPE_MAX       = 5
-} ios_dump_type_t;
-
 struct ios_dump_args {
-        ios_dump_type_t type;
-        union {
-                FILE *logfp;
-                dict_t *dict;
-        } u;
+    ios_dump_type_t type;
+    union {
+        FILE *logfp;
+        dict_t *dict;
+    } u;
 };
 
-typedef int (*block_dump_func) (xlator_t *, struct ios_dump_args*,
-                                    int , int , uint64_t ) ;
+typedef int (*block_dump_func)(xlator_t *, struct ios_dump_args *, int, int,
+                               uint64_t);
 
 struct ios_local {
-        struct timeval  wind_at;
-        struct timeval  unwind_at;
+    struct timeval wind_at;
+    struct timeval unwind_at;
 };
 
 struct volume_options options[];
 
 static int
-is_fop_latency_started (call_frame_t *frame)
+is_fop_latency_started(call_frame_t *frame)
 {
-        GF_ASSERT (frame);
-        struct timeval epoch = {0,};
-        return memcmp (&frame->begin, &epoch, sizeof (epoch));
+    GF_ASSERT(frame);
+    struct timeval epoch = {
+        0,
+    };
+    return memcmp(&frame->begin, &epoch, sizeof(epoch));
 }
 
 #define _IOS_SAMP_DIR DEFAULT_LOG_FILE_DIRECTORY "/samples"
@@ -199,265 +230,337 @@ is_fop_latency_started (call_frame_t *frame)
 #define _IOS_DUMP_DIR DATADIR "/db/glusterd/stats"
 #endif
 
-#define END_FOP_LATENCY(frame, op)                                      \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                                                                        \
-                conf = this->private;                                   \
-                if (conf && conf->measure_latency) {                    \
-                        gettimeofday (&frame->end, NULL);               \
-                        update_ios_latency (conf, frame, GF_FOP_##op);  \
-                }                                                       \
-        } while (0)
-
-#define START_FOP_LATENCY(frame)                                         \
-        do {                                                             \
-                struct ios_conf  *conf = NULL;                           \
-                                                                         \
-                conf = this->private;                                    \
-                if (conf && conf->measure_latency) {                     \
-                        gettimeofday (&frame->begin, NULL);              \
-                } else {                                                 \
-                        memset (&frame->begin, 0, sizeof (frame->begin));\
-                }                                                        \
-        } while (0)
-
-
-#define BUMP_FOP(op)                                                    \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                                                                        \
-                conf = this->private;                                   \
-                if (!conf)                                              \
-                        break;                                          \
-                conf->cumulative.fop_hits[GF_FOP_##op]++;               \
-                conf->incremental.fop_hits[GF_FOP_##op]++;              \
-        } while (0)
-
-#if defined(HAVE_ATOMIC_BUILTINS)
-#define STATS_LOCK(x)
-#define STATS_UNLOCK(x)
-#define STATS_ADD(x,i)  __sync_add_and_fetch (&x, i)
-#else
-#define STATS_LOCK(x)   LOCK (x)
-#define STATS_UNLOCK(x) UNLOCK (x)
-#define STATS_ADD(x,i)  (x) += (i)
-#endif
-
-#define UPDATE_PROFILE_STATS(frame, op)                                       \
-        do {                                                                  \
-                struct ios_conf  *conf = NULL;                                \
-                                                                              \
-                if (!is_fop_latency_started (frame))                          \
-                        break;                                                \
-                conf = this->private;                                         \
-                STATS_LOCK (&conf->lock);                                     \
-                {                                                             \
-                        if (conf && conf->measure_latency &&                  \
-                            conf->count_fop_hits) {                           \
-                                BUMP_FOP(op);                                 \
-                                gettimeofday (&frame->end, NULL);             \
-                                update_ios_latency (conf, frame, GF_FOP_##op);\
-                        }                                                     \
-                }                                                             \
-                STATS_UNLOCK (&conf->lock);                                   \
-        } while (0)
-
-#define BUMP_READ(fd, len)                                                     \
-        do {                                                                   \
-                struct ios_conf  *conf = NULL;                                 \
-                struct ios_fd    *iosfd = NULL;                                \
-                int               lb2 = 0;                                     \
-                                                                               \
-                conf = this->private;                                          \
-                lb2 = log_base2 (len);                                         \
-                ios_fd_ctx_get (fd, this, &iosfd);                             \
-                if (!conf)                                                     \
-                        break;                                                 \
+#define END_FOP_LATENCY(frame, op)                                             \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
                                                                                \
-                STATS_LOCK (&conf->lock);                                      \
-                {                                                              \
-                        STATS_ADD (conf->cumulative.data_read, len);           \
-                        STATS_ADD (conf->incremental.data_read, len);          \
-                        STATS_ADD (conf->cumulative.block_count_read[lb2], 1); \
-                        STATS_ADD (conf->incremental.block_count_read[lb2], 1);\
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency) {                                   \
+            timespec_now(&frame->end);                                         \
+            update_ios_latency(conf, frame, GF_FOP_##op);                      \
+        }                                                                      \
+    } while (0)
+
+#define START_FOP_LATENCY(frame)                                               \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
                                                                                \
-                        if (iosfd) {                                           \
-                                STATS_ADD (iosfd->data_read, len);             \
-                                STATS_ADD (iosfd->block_count_read[lb2], 1);   \
-                        }                                                      \
-                }                                                              \
-                STATS_UNLOCK (&conf->lock);                                    \
-        } while (0)
-
-#define BUMP_WRITE(fd, len)                                                    \
-        do {                                                                   \
-                struct ios_conf  *conf = NULL;                                 \
-                struct ios_fd    *iosfd = NULL;                                \
-                int               lb2 = 0;                                     \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency) {                                   \
+            timespec_now(&frame->begin);                                       \
+        } else {                                                               \
+            memset(&frame->begin, 0, sizeof(frame->begin));                    \
+        }                                                                      \
+    } while (0)
+
+#define BUMP_FOP(op)                                                           \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
                                                                                \
-                conf = this->private;                                          \
-                lb2 = log_base2 (len);                                         \
-                ios_fd_ctx_get (fd, this, &iosfd);                             \
-                if (!conf)                                                     \
-                        break;                                                 \
-                STATS_LOCK (&conf->lock);                                      \
-                {                                                              \
-                        STATS_ADD (conf->cumulative.data_written, len);        \
-                        STATS_ADD (conf->incremental.data_written, len);       \
-                        STATS_ADD (conf->cumulative.block_count_write[lb2], 1);\
-                        STATS_ADD (conf->incremental.block_count_write[lb2], 1);\
+        conf = this->private;                                                  \
+        if (!conf)                                                             \
+            break;                                                             \
+        GF_ATOMIC_INC(conf->cumulative.fop_hits[GF_FOP_##op]);                 \
+        GF_ATOMIC_INC(conf->incremental.fop_hits[GF_FOP_##op]);                \
+    } while (0)
+
+#define UPDATE_PROFILE_STATS(frame, op)                                        \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
                                                                                \
-                        if (iosfd) {                                           \
-                                STATS_ADD (iosfd->data_written, len);          \
-                                STATS_ADD (iosfd->block_count_write[lb2], 1);  \
-                        }                                                      \
-                }                                                              \
-                STATS_UNLOCK (&conf->lock);                                    \
-        } while (0)
-
-#define BUMP_STATS(iosstat, type)                                       \
-        do {                                                            \
-                struct ios_conf         *conf = NULL;                   \
-                uint64_t                 value = 0;                     \
-                                                                        \
-                conf = this->private;                                   \
-                                                                        \
-                LOCK(&iosstat->lock);                                   \
-                {                                                       \
-                        value = STATS_ADD (iosstat->counters[type], 1); \
-                }                                                       \
-                UNLOCK (&iosstat->lock);                                \
-                ios_stat_add_to_list (&conf->list[type],                \
-                                     value, iosstat);                   \
-        } while (0)
+        if (!is_fop_latency_started(frame))                                    \
+            break;                                                             \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency && conf->count_fop_hits) {           \
+            BUMP_FOP(op);                                                      \
+            timespec_now(&frame->end);                                         \
+            update_ios_latency(conf, frame, GF_FOP_##op);                      \
+        }                                                                      \
+    } while (0)
 
 #define BUMP_THROUGHPUT(iosstat, type)                                         \
-        do {                                                                   \
-                struct ios_conf         *conf = NULL;                          \
-                double                   elapsed;                              \
-                struct timeval          *begin, *end;                          \
-                double                   throughput;                           \
-                int                      flag = 0;                             \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+        double elapsed;                                                        \
+        struct timespec *begin, *end;                                          \
+        double throughput;                                                     \
+        int flag = 0;                                                          \
+        struct timeval tv = {                                                  \
+            0,                                                                 \
+        };                                                                     \
                                                                                \
-                begin = &frame->begin;                                         \
-                end   = &frame->end;                                           \
+        begin = &frame->begin;                                                 \
+        end = &frame->end;                                                     \
                                                                                \
-                elapsed = (end->tv_sec - begin->tv_sec) * 1e6                  \
-                        + (end->tv_usec - begin->tv_usec);                     \
-                throughput = op_ret / elapsed;                                 \
+        elapsed = gf_tsdiff(begin, end) / 1000.0;                              \
+        throughput = op_ret / elapsed;                                         \
                                                                                \
-                conf = this->private;                                          \
-                STATS_LOCK (&iosstat->lock);                                   \
-                {                                                              \
-                        if (iosstat->thru_counters[type].throughput            \
-                                <= throughput) {                               \
-                                iosstat->thru_counters[type].throughput =      \
-                                                                throughput;    \
-                                gettimeofday (&iosstat->                       \
-                                             thru_counters[type].time, NULL);  \
-                               flag = 1;                                       \
-                        }                                                      \
-                }                                                              \
-                STATS_UNLOCK (&iosstat->lock);                                 \
-                if (flag)                                                      \
-                       ios_stat_add_to_list (&conf->thru_list[type],           \
-                                               throughput, iosstat);           \
-        } while (0)
-
-int
-ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
-{
-        uint64_t      iosfd64 = 0;
-        unsigned long iosfdlong = 0;
-        int           ret = 0;
-
-        ret = fd_ctx_get (fd, this, &iosfd64);
-        iosfdlong = iosfd64;
-        if (ret != -1)
-                *iosfd = (void *) iosfdlong;
+        conf = this->private;                                                  \
+        gettimeofday(&tv, NULL);                                               \
+        LOCK(&iosstat->lock);                                                  \
+        {                                                                      \
+            if (iosstat->thru_counters[type].throughput <= throughput) {       \
+                iosstat->thru_counters[type].throughput = throughput;          \
+                memcpy(&iosstat->thru_counters[type].time, &tv,                \
+                       sizeof(struct timeval));                                \
+                flag = 1;                                                      \
+            }                                                                  \
+        }                                                                      \
+        UNLOCK(&iosstat->lock);                                                \
+        if (flag)                                                              \
+            ios_stat_add_to_list(&conf->thru_list[type], throughput, iosstat); \
+    } while (0)
 
-        return ret;
-}
+static int
+ios_fd_ctx_get(fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
+{
+    uint64_t iosfd64 = 0;
+    unsigned long iosfdlong = 0;
+    int ret = 0;
 
+    ret = fd_ctx_get(fd, this, &iosfd64);
+    iosfdlong = iosfd64;
+    if (ret != -1)
+        *iosfd = (void *)iosfdlong;
 
+    return ret;
+}
 
-int
-ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
+static int
+ios_fd_ctx_set(fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
 {
-        uint64_t   iosfd64 = 0;
-        int        ret = 0;
+    uint64_t iosfd64 = 0;
+    int ret = 0;
 
-        iosfd64 = (unsigned long) iosfd;
-        ret = fd_ctx_set (fd, this, iosfd64);
+    iosfd64 = (unsigned long)iosfd;
+    ret = fd_ctx_set(fd, this, iosfd64);
 
-        return ret;
+    return ret;
 }
 
-int
-ios_stat_ref (struct ios_stat *iosstat)
+static int
+ios_stat_ref(struct ios_stat *iosstat)
 {
-        LOCK (&iosstat->lock);
-        {
-                iosstat->refcnt++;
-        }
-        UNLOCK (&iosstat->lock);
+    uint64_t refcnt = 0;
+    refcnt = GF_ATOMIC_INC(iosstat->refcnt);
 
-        return iosstat->refcnt;
+    return refcnt;
 }
 
-int
-ios_stat_unref (struct ios_stat *iosstat)
+static int
+ios_stat_unref(struct ios_stat *iosstat)
 {
-        int cleanup = 0;
-        LOCK (&iosstat->lock);
+    int cleanup = 0;
+    uint64_t refcnt = 0;
+
+    refcnt = GF_ATOMIC_DEC(iosstat->refcnt);
+    if (refcnt == 0) {
+        if (iosstat->filename) {
+            GF_FREE(iosstat->filename);
+            iosstat->filename = NULL;
+        }
+        cleanup = 1;
+    }
+
+    if (cleanup) {
+        LOCK_DESTROY(&iosstat->lock);
+        GF_FREE(iosstat);
+        iosstat = NULL;
+    }
+
+    return 0;
+}
+
+static int
+ios_stat_add_to_list(struct ios_stat_head *list_head, uint64_t value,
+                     struct ios_stat *iosstat)
+{
+    struct ios_stat_list *new = NULL;
+    struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *t = NULL;
+    struct ios_stat_list *list_entry = NULL;
+    struct ios_stat_list *tmp = NULL;
+    struct ios_stat_list *last = NULL;
+    struct ios_stat *stat = NULL;
+    int cnt = 0;
+    int found = 0;
+    int reposition = 0;
+    double min_count = 0;
+
+    LOCK(&list_head->lock);
+    {
+        if (list_head->min_cnt == 0)
+            list_head->min_cnt = value;
+        if ((list_head->members == MAX_LIST_MEMBERS) &&
+            (list_head->min_cnt > value))
+            goto out;
+
+        list_for_each_entry_safe(entry, t, &list_head->iosstats->list, list)
         {
-                iosstat->refcnt--;
-                if (iosstat->refcnt == 0) {
-                        if (iosstat->filename) {
-                                GF_FREE (iosstat->filename);
-                                iosstat->filename = NULL;
-                        }
-                        cleanup = 1;
+            cnt++;
+            if (cnt == list_head->members)
+                last = entry;
+
+            if (!gf_uuid_compare(iosstat->gfid, entry->iosstat->gfid)) {
+                list_entry = entry;
+                found = cnt;
+                entry->value = value;
+                if (!reposition) {
+                    if (cnt == list_head->members)
+                        list_head->min_cnt = value;
+                    goto out;
                 }
+                break;
+            } else if (entry->value <= value && !reposition) {
+                reposition = cnt;
+                tmp = entry;
+                if (cnt == list_head->members - 1)
+                    min_count = entry->value;
+            }
         }
-        UNLOCK (&iosstat->lock);
-
-        if (cleanup) {
-                LOCK_DESTROY (&iosstat->lock);
-                GF_FREE (iosstat);
-                iosstat = NULL;
+        if (found) {
+            list_del(&list_entry->list);
+            list_add_tail(&list_entry->list, &tmp->list);
+            if (min_count)
+                list_head->min_cnt = min_count;
+            goto out;
+        } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
+            new = GF_CALLOC(1, sizeof(*new), gf_io_stats_mt_ios_stat_list);
+            new->iosstat = iosstat;
+            new->value = value;
+            ios_stat_ref(iosstat);
+            list_add_tail(&new->list, &tmp->list);
+            if (last) {
+                stat = last->iosstat;
+                last->iosstat = NULL;
+                ios_stat_unref(stat);
+                list_del(&last->list);
+                GF_FREE(last);
+            }
+            if (reposition == MAX_LIST_MEMBERS)
+                list_head->min_cnt = value;
+            else if (min_count) {
+                list_head->min_cnt = min_count;
+            }
+        } else if (list_head->members < MAX_LIST_MEMBERS) {
+            new = GF_CALLOC(1, sizeof(*new), gf_io_stats_mt_ios_stat_list);
+            new->iosstat = iosstat;
+            new->value = value;
+            ios_stat_ref(iosstat);
+            if (reposition) {
+                list_add_tail(&new->list, &tmp->list);
+            } else {
+                list_add_tail(&new->list, &entry->list);
+            }
+            list_head->members++;
+            if (list_head->min_cnt > value)
+                list_head->min_cnt = value;
         }
+    }
+out:
+    UNLOCK(&list_head->lock);
+    return 0;
+}
 
-        return 0;
+static void
+ios_bump_read(xlator_t *this, fd_t *fd, size_t len)
+{
+    struct ios_conf *conf = NULL;
+    struct ios_fd *iosfd = NULL;
+    int lb2 = 0;
+
+    conf = this->private;
+    lb2 = log_base2(len);
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (!conf)
+        return;
+
+    GF_ATOMIC_ADD(conf->cumulative.data_read, len);
+    GF_ATOMIC_ADD(conf->incremental.data_read, len);
+    GF_ATOMIC_INC(conf->cumulative.block_count_read[lb2]);
+    GF_ATOMIC_INC(conf->incremental.block_count_read[lb2]);
+
+    if (iosfd) {
+        GF_ATOMIC_ADD(iosfd->data_read, len);
+        GF_ATOMIC_INC(iosfd->block_count_read[lb2]);
+    }
 }
 
-int
-ios_inode_ctx_set (inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
+static void
+ios_bump_write(xlator_t *this, fd_t *fd, size_t len)
 {
-        uint64_t   iosstat64 = 0;
-        int        ret     = 0;
+    struct ios_conf *conf = NULL;
+    struct ios_fd *iosfd = NULL;
+    int lb2 = 0;
 
-        ios_stat_ref (iosstat);
-        iosstat64 = (unsigned long )iosstat;
-        ret = inode_ctx_put (inode, this, iosstat64);
-        return ret;
+    conf = this->private;
+    lb2 = log_base2(len);
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (!conf)
+        return;
+
+    GF_ATOMIC_ADD(conf->cumulative.data_written, len);
+    GF_ATOMIC_ADD(conf->incremental.data_written, len);
+    GF_ATOMIC_INC(conf->cumulative.block_count_write[lb2]);
+    GF_ATOMIC_INC(conf->incremental.block_count_write[lb2]);
+
+    if (iosfd) {
+        GF_ATOMIC_ADD(iosfd->data_written, len);
+        GF_ATOMIC_INC(iosfd->block_count_write[lb2]);
+    }
+}
+
+static void
+ios_bump_upcall(xlator_t *this, gf_upcall_flags_t event)
+{
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+    if (!conf)
+        return;
+    if (conf->count_fop_hits) {
+        GF_ATOMIC_INC(conf->cumulative.upcall_hits[event]);
+        GF_ATOMIC_INC(conf->incremental.upcall_hits[event]);
+    }
+}
+
+static void
+ios_bump_stats(xlator_t *this, struct ios_stat *iosstat, ios_stats_type_t type)
+{
+    struct ios_conf *conf = NULL;
+    uint64_t value = 0;
+
+    conf = this->private;
+
+    value = GF_ATOMIC_INC(iosstat->counters[type]);
+    ios_stat_add_to_list(&conf->list[type], value, iosstat);
 }
 
 int
-ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
+ios_inode_ctx_set(inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
 {
-        uint64_t      iosstat64 = 0;
-        unsigned long iosstatlong = 0;
-        int           ret = 0;
+    uint64_t iosstat64 = 0;
+    int ret = 0;
 
-        ret = inode_ctx_get (inode, this, &iosstat64);
-        iosstatlong = iosstat64;
-        if (ret != -1)
-                *iosstat = (void *) iosstatlong;
+    ios_stat_ref(iosstat);
+    iosstat64 = (unsigned long)iosstat;
+    ret = inode_ctx_put(inode, this, iosstat64);
+    return ret;
+}
 
-        return ret;
+int
+ios_inode_ctx_get(inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
+{
+    uint64_t iosstat64 = 0;
+    unsigned long iosstatlong = 0;
+    int ret = 0;
+
+    ret = inode_ctx_get(inode, this, &iosstat64);
+    iosstatlong = iosstat64;
+    if (ret != -1)
+        *iosstat = (void *)iosstatlong;
 
+    return ret;
 }
 
 /*
@@ -472,604 +575,561 @@ ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
  *
  */
 ios_sample_buf_t *
-ios_create_sample_buf (size_t buf_size)
+ios_create_sample_buf(size_t buf_size)
 {
-        ios_sample_buf_t *ios_sample_buf = NULL;
-        ios_sample_t     *ios_samples = NULL;
+    ios_sample_buf_t *ios_sample_buf = NULL;
+    ios_sample_t *ios_samples = NULL;
 
-        ios_sample_buf = GF_CALLOC (1,
-                sizeof (*ios_sample_buf),
-                gf_io_stats_mt_ios_sample_buf);
-        if (!ios_sample_buf)
-                goto err;
+    ios_sample_buf = GF_CALLOC(1, sizeof(*ios_sample_buf),
+                               gf_io_stats_mt_ios_sample_buf);
+    if (!ios_sample_buf)
+        goto err;
 
-        ios_samples = GF_CALLOC (buf_size,
-                sizeof (*ios_samples),
-                gf_io_stats_mt_ios_sample);
+    ios_samples = GF_CALLOC(buf_size, sizeof(*ios_samples),
+                            gf_io_stats_mt_ios_sample);
 
-        if (!ios_samples)
-                goto err;
+    if (!ios_samples)
+        goto err;
 
-        ios_sample_buf->ios_samples = ios_samples;
-        ios_sample_buf->size = buf_size;
-        ios_sample_buf->pos = 0;
-        ios_sample_buf->observed = 0;
-        ios_sample_buf->collected = 0;
+    ios_sample_buf->ios_samples = ios_samples;
+    ios_sample_buf->size = buf_size;
+    ios_sample_buf->pos = 0;
+    ios_sample_buf->observed = 0;
+    ios_sample_buf->collected = 0;
 
-        return ios_sample_buf;
+    return ios_sample_buf;
 err:
-        GF_FREE (ios_sample_buf);
-        return NULL;
+    GF_FREE(ios_sample_buf);
+    return NULL;
 }
 
 void
-ios_destroy_sample_buf (ios_sample_buf_t *ios_sample_buf)
+ios_destroy_sample_buf(ios_sample_buf_t *ios_sample_buf)
 {
-        GF_FREE (ios_sample_buf->ios_samples);
-        GF_FREE (ios_sample_buf);
+    GF_FREE(ios_sample_buf->ios_samples);
+    GF_FREE(ios_sample_buf);
 }
 
 static int
-ios_init_sample_buf (struct ios_conf *conf)
-{
-        int32_t        ret = -1;
-
-        GF_ASSERT (conf);
-        LOCK (&conf->lock);
-        conf->ios_sample_buf = ios_create_sample_buf (
-                conf->ios_sample_buf_size);
-        if (!conf->ios_sample_buf)
-                goto out;
-        ret = 0;
-out:
-        UNLOCK (&conf->lock);
-        return ret;
-}
-
-int
-ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
-                            struct ios_stat *iosstat)
+ios_init_sample_buf(struct ios_conf *conf)
 {
-        struct ios_stat_list *new = NULL;
-        struct ios_stat_list *entry = NULL;
-        struct ios_stat_list *t = NULL;
-        struct ios_stat_list *list_entry = NULL;
-        struct ios_stat_list *tmp = NULL;
-        struct ios_stat_list *last = NULL;
-        struct ios_stat      *stat = NULL;
-        int                   cnt = 0;
-        int                   found = 0;
-        int                   reposition = 0;
-        double                min_count = 0;
-
-        LOCK (&list_head->lock);
-        {
+    int32_t ret = -1;
 
-                if (list_head->min_cnt == 0)
-                        list_head->min_cnt = value;
-                if ((list_head->members == MAX_LIST_MEMBERS) &&
-                     (list_head->min_cnt > value))
-                        goto out;
-
-                list_for_each_entry_safe (entry, t,
-                              &list_head->iosstats->list, list) {
-                        cnt++;
-                        if (cnt == list_head->members)
-                                last = entry;
-
-                        if (!gf_uuid_compare (iosstat->gfid,
-                            entry->iosstat->gfid)) {
-                                list_entry = entry;
-                                found = cnt;
-                                entry->value = value;
-                                if (!reposition) {
-                                        if (cnt == list_head->members)
-                                                list_head->min_cnt = value;
-                                        goto out;
-                                }
-                                break;
-                        } else if (entry->value <= value && !reposition) {
-                                reposition = cnt;
-                                tmp = entry;
-                                if (cnt == list_head->members - 1)
-                                        min_count = entry->value;
-                        }
-                }
-                if (found) {
-                        list_del (&list_entry->list);
-                        list_add_tail (&list_entry->list, &tmp->list);
-                        if (min_count)
-                                list_head->min_cnt = min_count;
-                        goto out;
-                } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
-                        new = GF_CALLOC (1, sizeof (*new),
-                                        gf_io_stats_mt_ios_stat_list);
-                        new->iosstat = iosstat;
-                        new->value = value;
-                        ios_stat_ref (iosstat);
-                        list_add_tail (&new->list, &tmp->list);
-                        if (last) {
-                                stat = last->iosstat;
-                                last->iosstat = NULL;
-                                ios_stat_unref (stat);
-                                list_del (&last->list);
-                                GF_FREE (last);
-                        }
-                        if (reposition == MAX_LIST_MEMBERS)
-                                list_head->min_cnt = value;
-                        else if (min_count) {
-                                list_head->min_cnt = min_count;
-                        }
-                } else if (list_head->members < MAX_LIST_MEMBERS) {
-                        new  = GF_CALLOC (1, sizeof (*new),
-                                          gf_io_stats_mt_ios_stat_list);
-                        new->iosstat = iosstat;
-                        new->value = value;
-                        ios_stat_ref (iosstat);
-                        if (reposition) {
-                                list_add_tail (&new->list, &tmp->list);
-                        } else {
-                                list_add_tail (&new->list, &entry->list);
-                        }
-                        list_head->members++;
-                        if (list_head->min_cnt > value)
-                                list_head->min_cnt = value;
-                }
-        }
+    GF_ASSERT(conf);
+    LOCK(&conf->lock);
+    conf->ios_sample_buf = ios_create_sample_buf(conf->ios_sample_buf_size);
+    if (!conf->ios_sample_buf)
+        goto out;
+    ret = 0;
 out:
-        UNLOCK (&list_head->lock);
-        return 0;
+    UNLOCK(&conf->lock);
+    return ret;
 }
 
 static int
-ios_stats_cleanup (xlator_t *this, inode_t *inode)
+ios_stats_cleanup(xlator_t *this, inode_t *inode)
 {
+    struct ios_stat *iosstat = NULL;
+    uint64_t iosstat64 = 0;
 
-        struct ios_stat *iosstat = NULL;
-        uint64_t         iosstat64 = 0;
-
-        inode_ctx_del (inode, this, &iosstat64);
-        if (!iosstat64) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not get inode ctx");
-                return 0;
-        }
-        iosstat = (void *) (long)iosstat64;
-        if (iosstat) {
-                ios_stat_unref (iosstat);
-        }
+    inode_ctx_del(inode, this, &iosstat64);
+    if (!iosstat64) {
+        gf_log(this->name, GF_LOG_WARNING, "could not get inode ctx");
         return 0;
-}
-
-#define ios_log(this, logfp, fmt ...)                           \
-        do {                                                    \
-                if (logfp) {                                    \
-                        fprintf (logfp, fmt);                   \
-                        fprintf (logfp, "\n");                  \
-                }                                               \
-                gf_log (this->name, GF_LOG_DEBUG, fmt);        \
-        } while (0)
+    }
+    iosstat = (void *)(long)iosstat64;
+    if (iosstat) {
+        ios_stat_unref(iosstat);
+    }
+    return 0;
+}
+
+#define ios_log(this, logfp, fmt...)                                           \
+    do {                                                                       \
+        if (logfp) {                                                           \
+            fprintf(logfp, fmt);                                               \
+            fprintf(logfp, "\n");                                              \
+        }                                                                      \
+        gf_log(this->name, GF_LOG_DEBUG, fmt);                                 \
+    } while (0)
 
 int
-ios_dump_file_stats (struct ios_stat_head *list_head, xlator_t *this,
-                     FILE *logfp)
+ios_dump_file_stats(struct ios_stat_head *list_head, xlator_t *this,
+                    FILE *logfp)
 {
-        struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *entry = NULL;
 
-        LOCK (&list_head->lock);
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
         {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        ios_log (this, logfp, "%-12.0f %s",
-                                entry->value, entry->iosstat->filename);
-                }
+            ios_log(this, logfp, "%-12.0f %s", entry->value,
+                    entry->iosstat->filename);
         }
-        UNLOCK (&list_head->lock);
-        return 0;
+    }
+    UNLOCK(&list_head->lock);
+    return 0;
 }
 
 int
-ios_dump_throughput_stats (struct ios_stat_head *list_head, xlator_t *this,
-                           FILE *logfp, ios_stats_thru_t type)
+ios_dump_throughput_stats(struct ios_stat_head *list_head, xlator_t *this,
+                          FILE *logfp, ios_stats_thru_t type)
 {
-        struct ios_stat_list *entry = NULL;
-        struct timeval        time  = {0, };
-        char                  timestr[256] = {0, };
+    struct ios_stat_list *entry = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
 
-        LOCK (&list_head->lock);
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
         {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        gf_time_fmt (timestr, sizeof timestr,
-                                     entry->iosstat->thru_counters[type].time.tv_sec,
-                                     gf_timefmt_FT);
-                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, time.tv_usec);
-
-                        ios_log (this, logfp, "%s \t %-10.2f  \t  %s",
-                                 timestr, entry->value, entry->iosstat->filename);
-                }
+            gf_time_fmt_tv(timestr, sizeof timestr,
+                           &entry->iosstat->thru_counters[type].time,
+                           gf_timefmt_FT);
+
+            ios_log(this, logfp, "%s \t %-10.2f  \t  %s", timestr, entry->value,
+                    entry->iosstat->filename);
         }
-        UNLOCK (&list_head->lock);
-        return 0;
+    }
+    UNLOCK(&list_head->lock);
+    return 0;
 }
 
 int
-_io_stats_get_key_prefix (xlator_t *this, char **key_prefix) {
-        char                  *key_root = "gluster";
-        char                  *xlator_name = NULL;
-        char                  *instance_name = NULL;
-        size_t                key_len = 0;
-        int                   bytes_written = 0;
-        int                   i = 0;
-        int                   ret = 0;
-
-        xlator_name = strdupa (this->name);
-        for (i = 0; i < strlen (xlator_name); i++) {
-                if (xlator_name[i] == '/')
-                        xlator_name[i] = '_';
+_io_stats_get_key_prefix(xlator_t *this, char **key_prefix)
+{
+    char *key_root = "gluster";
+    char *xlator_name = NULL;
+    char *instance_name = NULL;
+    size_t key_len = 0;
+    int bytes_written = 0;
+    int i = 0;
+    int ret = 0;
+    struct ios_conf *conf = this->private;
+
+    xlator_name = strdupa(conf->unique_id);
+    for (i = 0; i < strlen(xlator_name); i++) {
+        if (xlator_name[i] == '/')
+            xlator_name[i] = '_';
+    }
+
+    instance_name = this->instance_name;
+    if (this->name && strcmp(this->name, "glustershd") == 0) {
+        xlator_name = "shd";
+    } else if (this->prev && strcmp(this->prev->name, "nfs-server") == 0) {
+        xlator_name = "nfsd";
+        if (this->prev->instance_name)
+            instance_name = strdupa(this->prev->instance_name);
+    }
+
+    if (strcmp(__progname, "glusterfsd") == 0)
+        key_root = "gluster.brick";
+
+    if (instance_name) {
+        /* +3 for 2 x "." + NULL */
+        key_len = strlen(key_root) + strlen(xlator_name) +
+                  strlen(instance_name) + 3;
+        *key_prefix = GF_CALLOC(key_len, sizeof(char), gf_common_mt_char);
+        if (!*key_prefix) {
+            ret = -ENOMEM;
+            goto err;
         }
-
-        instance_name = this->instance_name;
-        if (this->name && strcmp (this->name, "glustershd") == 0) {
-                xlator_name = "shd";
-        } else if (this->prev &&
-                   strcmp (this->prev->name, "nfs-server") == 0) {
-                xlator_name = "nfsd";
-                if (this->prev->instance_name)
-                        instance_name = strdupa (this->prev->instance_name);
+        bytes_written = snprintf(*key_prefix, key_len, "%s.%s.%s", key_root,
+                                 xlator_name, instance_name);
+        if (bytes_written != key_len - 1) {
+            ret = -EINVAL;
+            goto err;
         }
-
-        if (strcmp (__progname, "glusterfsd") == 0)
-                key_root = "gluster.brick";
-
-        if (instance_name) {
-                /* +3 for 2 x "." + NULL */
-                key_len = strlen (key_root) + strlen (xlator_name) +
-                        strlen (instance_name) + 3;
-                *key_prefix = GF_CALLOC (key_len, sizeof (char),
-                        gf_common_mt_char);
-                if (!key_prefix) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                bytes_written = snprintf (*key_prefix, key_len, "%s.%s.%s",
-                        key_root, xlator_name, instance_name);
-                if (bytes_written != key_len - 1) {
-                        ret = -EINVAL;
-                        goto err;
-                }
-        } else {
-                /* +2 for 1 x "." + NULL */
-                key_len = strlen (key_root) + strlen (xlator_name) + 2;
-                *key_prefix = GF_CALLOC (key_len, sizeof (char),
-                        gf_common_mt_char);
-                if (!key_prefix) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                bytes_written = snprintf (*key_prefix, key_len, "%s.%s",
-                        key_root, xlator_name);
-                if (bytes_written != key_len - 1) {
-                        ret = -EINVAL;
-                        goto err;
-                }
+    } else {
+        /* +2 for 1 x "." + NULL */
+        key_len = strlen(key_root) + strlen(xlator_name) + 2;
+        *key_prefix = GF_CALLOC(key_len, sizeof(char), gf_common_mt_char);
+        if (!*key_prefix) {
+            ret = -ENOMEM;
+            goto err;
         }
-        return 0;
+        bytes_written = snprintf(*key_prefix, key_len, "%s.%s", key_root,
+                                 xlator_name);
+        if (bytes_written != key_len - 1) {
+            ret = -EINVAL;
+            goto err;
+        }
+    }
+    return 0;
 err:
-        GF_FREE (*key_prefix);
-        *key_prefix = NULL;
-        return ret;
+    GF_FREE(*key_prefix);
+    *key_prefix = NULL;
+    return ret;
 }
 
 int
-io_stats_dump_global_to_json_logfp (xlator_t *this,
-    struct ios_global_stats *stats, struct timeval *now, int interval,
-    FILE *logfp)
-{
-        int                   i = 0;
-        int                   j = 0;
-        struct ios_conf       *conf = NULL;
-        char                  *key_prefix = NULL;
-        char                  *str_prefix = NULL;
-        char                  *lc_fop_name = NULL;
-        int                   ret = 1;  /* Default to error */
-        int                   rw_size;
-        char                  *rw_unit = NULL;
-        long                  fop_hits;
-        float                 fop_lat_ave;
-        float                 fop_lat_min;
-        float                 fop_lat_max;
-        double                interval_sec;
-
-        interval_sec = ((now->tv_sec * 1000000.0 + now->tv_usec) -
-                (stats->started_at.tv_sec * 1000000.0 +
-                 stats->started_at.tv_usec)) / 1000000.0;
-
-        conf = this->private;
-
-        ret = _io_stats_get_key_prefix (this, &key_prefix);
-        if (ret) {
-                goto out;
+io_stats_dump_global_to_json_logfp(xlator_t *this,
+                                   struct ios_global_stats *stats, time_t now,
+                                   int interval, FILE *logfp)
+{
+    int i = 0;
+    int j = 0;
+    struct ios_conf *conf = NULL;
+    char *key_prefix = NULL;
+    char *str_prefix = NULL;
+    char *lc_fop_name = NULL;
+    int ret = 1; /* Default to error */
+    int rw_size;
+    char *rw_unit = NULL;
+    uint64_t fop_hits;
+    float fop_lat_ave;
+    float fop_lat_min;
+    float fop_lat_max;
+    double interval_sec;
+    double fop_ave_usec = 0.0;
+    double fop_ave_usec_sum = 0.0;
+    double weighted_fop_ave_usec = 0.0;
+    double weighted_fop_ave_usec_sum = 0.0;
+    long total_fop_hits = 0;
+    loc_t unused_loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    interval_sec = (double)(now - stats->started_at);
+
+    conf = this->private;
+
+    ret = _io_stats_get_key_prefix(this, &key_prefix);
+    if (ret) {
+        goto out;
+    }
+
+    if (interval == -1) {
+        str_prefix = "aggr";
+
+    } else {
+        str_prefix = "inter";
+    }
+    ios_log(this, logfp, "{");
+
+    for (i = 0; i < 31; i++) {
+        rw_size = (1 << i);
+        if (rw_size >= 1024 * 1024) {
+            rw_size = rw_size / (1024 * 1024);
+            rw_unit = "mb";
+        } else if (rw_size >= 1024) {
+            rw_size = rw_size / 1024;
+            rw_unit = "kb";
+        } else {
+            rw_unit = "b";
         }
 
         if (interval == -1) {
-                str_prefix = "aggr";
-
+            ios_log(this, logfp, "\"%s.%s.read_%d%s\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    GF_ATOMIC_GET(stats->block_count_read[i]));
+            ios_log(this, logfp, "\"%s.%s.write_%d%s\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    GF_ATOMIC_GET(stats->block_count_write[i]));
         } else {
-                str_prefix = "inter";
+            ios_log(this, logfp, "\"%s.%s.read_%d%s_per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    (double)(GF_ATOMIC_GET(stats->block_count_read[i]) /
+                             interval_sec));
+            ios_log(this, logfp, "\"%s.%s.write_%d%s_per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    (double)(GF_ATOMIC_GET(stats->block_count_write[i]) /
+                             interval_sec));
         }
-        ios_log (this, logfp, "{");
-
-        for (i = 0; i < 31; i++) {
-                rw_size = (1 << i);
-                if (rw_size >= 1024 * 1024) {
-                        rw_size = rw_size / (1024 * 1024);
-                        rw_unit = "mb";
-                } else if (rw_size >= 1024) {
-                        rw_size = rw_size / 1024;
-                        rw_unit = "kb";
-                } else {
-                        rw_unit = "b";
-                }
-
-                if (interval == -1) {
-                        ios_log (this, logfp,
-                                "\"%s.%s.read_%d%s\": \"%"PRId64"\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                stats->block_count_read[i]);
-                        ios_log (this, logfp,
-                                "\"%s.%s.write_%d%s\": \"%"PRId64"\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                stats->block_count_write[i]);
-                } else {
-                        ios_log (this, logfp,
-                                "\"%s.%s.read_%d%s_per_sec\": \"%0.2lf\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                (double)(stats->block_count_read[i] /
-                                        interval_sec));
-                        ios_log (this, logfp,
-                                "\"%s.%s.write_%d%s_per_sec\": \"%0.2lf\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                (double)(stats->block_count_write[i] /
-                                        interval_sec));
-                }
+    }
+
+    if (interval == -1) {
+        ios_log(this, logfp, "\"%s.%s.fds.open_count\": %" PRId64 ",",
+                key_prefix, str_prefix, conf->cumulative.nr_opens);
+        ios_log(this, logfp, "\"%s.%s.fds.max_open_count\": %" PRId64 ",",
+                key_prefix, str_prefix, conf->cumulative.max_nr_opens);
+    }
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        lc_fop_name = strdupa(gf_fop_list[i]);
+        for (j = 0; lc_fop_name[j]; j++) {
+            lc_fop_name[j] = tolower(lc_fop_name[j]);
         }
 
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        fop_lat_ave = 0.0;
+        fop_lat_min = 0.0;
+        fop_lat_max = 0.0;
+        if (fop_hits) {
+            if (stats->latency[i].avg) {
+                fop_lat_ave = stats->latency[i].avg;
+                fop_lat_min = stats->latency[i].min;
+                fop_lat_max = stats->latency[i].max;
+            }
+        }
         if (interval == -1) {
-                ios_log (this, logfp, "\"%s.%s.fds.open_count\": \"%"PRId64
-                        "\",", key_prefix, str_prefix,
-                        conf->cumulative.nr_opens);
-                ios_log (this, logfp,
-                        "\"%s.%s.fds.max_open_count\": \"%"PRId64"\",",
-                        key_prefix, str_prefix, conf->cumulative.max_nr_opens);
+            ios_log(this, logfp, "\"%s.%s.fop.%s.count\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, lc_fop_name, fop_hits);
+        } else {
+            ios_log(this, logfp, "\"%s.%s.fop.%s.per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, lc_fop_name,
+                    (double)(fop_hits / interval_sec));
         }
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                lc_fop_name = strdupa (gf_fop_list[i]);
-                for (j = 0; lc_fop_name[j]; j++) {
-                        lc_fop_name[j] = tolower (lc_fop_name[j]);
-                }
-
-                fop_hits = 0;
-                fop_lat_ave = 0.0;
-                fop_lat_min = 0.0;
-                fop_lat_max = 0.0;
-                if (stats->fop_hits[i]) {
-                        fop_hits = stats->fop_hits[i];
-                        if (stats->latency[i].avg) {
-                                fop_lat_ave = stats->latency[i].avg;
-                                fop_lat_min = stats->latency[i].min;
-                                fop_lat_max = stats->latency[i].max;
-                        }
-                }
-                if (interval == -1) {
-                        ios_log (this, logfp,
-                                "\"%s.%s.fop.%s.count\": \"%"PRId64"\",",
-                                key_prefix, str_prefix, lc_fop_name,
-                                fop_hits);
-                } else {
-                        ios_log (this, logfp,
-                                "\"%s.%s.fop.%s.per_sec\": \"%0.2lf\",",
-                                key_prefix, str_prefix, lc_fop_name,
-                                (double)(fop_hits / interval_sec));
-                }
-
-                ios_log (this, logfp,
-                        "\"%s.%s.fop.%s.latency_ave_usec\": \"%0.2lf\",",
-                         key_prefix, str_prefix, lc_fop_name, fop_lat_ave);
-                ios_log (this, logfp,
-                        "\"%s.%s.fop.%s.latency_min_usec\": \"%0.2lf\",",
-                         key_prefix, str_prefix, lc_fop_name, fop_lat_min);
-                ios_log (this, logfp,
-                        "\"%s.%s.fop.%s.latency_max_usec\": \"%0.2lf\",",
-                        key_prefix, str_prefix, lc_fop_name, fop_lat_max);
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_ave_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_ave);
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_min_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_min);
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_max_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_max);
+
+        fop_ave_usec_sum += fop_lat_ave;
+        weighted_fop_ave_usec_sum += fop_hits * fop_lat_ave;
+        total_fop_hits += fop_hits;
+    }
+
+    if (total_fop_hits) {
+        weighted_fop_ave_usec = weighted_fop_ave_usec_sum / total_fop_hits;
+        /* Extra key that does not print out an entry w/ 0.00 for
+         * intervals with no data
+         */
+        ios_log(this, logfp,
+                "\"%s.%s.fop.weighted_latency_ave_usec_nozerofill\": "
+                "%0.4lf,",
+                key_prefix, str_prefix, weighted_fop_ave_usec);
+    }
+    ios_log(this, logfp, "\"%s.%s.fop.weighted_latency_ave_usec\": %0.4lf,",
+            key_prefix, str_prefix, weighted_fop_ave_usec);
+    ios_log(this, logfp, "\"%s.%s.fop.weighted_fop_count\": %ld,", key_prefix,
+            str_prefix, total_fop_hits);
+
+    fop_ave_usec = fop_ave_usec_sum / GF_FOP_MAXVALUE;
+    ios_log(this, logfp, "\"%s.%s.fop.unweighted_latency_ave_usec\":%0.4lf,",
+            key_prefix, str_prefix, fop_ave_usec);
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        lc_fop_name = strdupa(gf_upcall_list[i]);
+        for (j = 0; lc_fop_name[j]; j++) {
+            lc_fop_name[j] = tolower(lc_fop_name[j]);
         }
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
         if (interval == -1) {
-                ios_log (this, logfp, "\"%s.%s.uptime\": \"%"PRId64"\",",
-                         key_prefix, str_prefix,
-                         (uint64_t) (now->tv_sec - stats->started_at.tv_sec));
-                ios_log (this, logfp, "\"%s.%s.bytes_read\": \"%"PRId64"\",",
-                         key_prefix, str_prefix, stats->data_read);
-                ios_log (this, logfp, "\"%s.%s.bytes_written\": \"%"PRId64"\"",
-                         key_prefix, str_prefix, stats->data_written);
+            ios_log(this, logfp, "\"%s.%s.fop.%s.count\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, lc_fop_name, fop_hits);
         } else {
-                ios_log (this, logfp,
-                         "\"%s.%s.sample_interval_sec\": \"%0.2lf\",",
-                         key_prefix, str_prefix,
-                         interval_sec);
-                ios_log (this, logfp,
-                         "\"%s.%s.bytes_read_per_sec\": \"%0.2lf\",",
-                         key_prefix, str_prefix,
-                         (double)(stats->data_read / interval_sec));
-                ios_log (this, logfp,
-                         "\"%s.%s.bytes_written_per_sec\": \"%0.2lf\"",
-                         key_prefix, str_prefix,
-                         (double)(stats->data_written / interval_sec));
+            ios_log(this, logfp, "\"%s.%s.fop.%s.per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, lc_fop_name,
+                    (double)(fop_hits / interval_sec));
         }
+    }
 
-        ios_log (this, logfp, "}");
-        ret = 0;
+    ret = syncop_getxattr(this, &unused_loc, &xattr, IO_THREADS_QUEUE_SIZE_KEY,
+                          NULL, NULL);
+    if (xattr) {
+        /*
+         * Iterate over the dictionary returned to us by io-threads and
+         * dump the results to the stats file.
+         */
+        data_pair_t *curr = NULL;
+
+        dict_foreach_inline(xattr, curr)
+        {
+            ios_log(this, logfp, "\"%s.%s.%s.queue_size\": %d,", key_prefix,
+                    str_prefix, curr->key, data_to_int32(curr->value));
+        }
+
+        /* Free the dictionary */
+        dict_unref(xattr);
+    } else {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Unable to get queue size counts from "
+               "the io-threads translator!");
+    }
+
+    if (interval == -1) {
+        ios_log(this, logfp, "\"%s.%s.uptime\": %" PRIu64 ",", key_prefix,
+                str_prefix, (uint64_t)(now - stats->started_at));
+        ios_log(this, logfp,
+                "\"%s.%s.bytes_read\": "
+                "%" GF_PRI_ATOMIC ",",
+                key_prefix, str_prefix, GF_ATOMIC_GET(stats->data_read));
+        ios_log(this, logfp,
+                "\"%s.%s.bytes_written\": "
+                "%" GF_PRI_ATOMIC "",
+                key_prefix, str_prefix, GF_ATOMIC_GET(stats->data_written));
+    } else {
+        ios_log(this, logfp, "\"%s.%s.sample_interval_sec\": %0.2lf,",
+                key_prefix, str_prefix, interval_sec);
+        ios_log(this, logfp, "\"%s.%s.bytes_read_per_sec\": %0.2lf,",
+                key_prefix, str_prefix,
+                (double)(GF_ATOMIC_GET(stats->data_read) / interval_sec));
+        ios_log(this, logfp, "\"%s.%s.bytes_written_per_sec\": %0.2lf",
+                key_prefix, str_prefix,
+                (double)(GF_ATOMIC_GET(stats->data_written) / interval_sec));
+    }
+
+    ios_log(this, logfp, "}");
+    ret = 0;
 out:
-        GF_FREE (key_prefix);
-        return ret;
+    GF_FREE(key_prefix);
+    return ret;
 }
 
 char *
-_resolve_username (xlator_t *this, uid_t uid)
+_resolve_username(xlator_t *this, uid_t uid)
 {
-        struct passwd pwd;
-        struct passwd *pwd_result = NULL;
-        size_t pwd_buf_len;
-        char   *pwd_buf = NULL;
-        char   *ret = NULL;
+    struct passwd pwd;
+    struct passwd *pwd_result = NULL;
+    size_t pwd_buf_len;
+    char *pwd_buf = NULL;
+    char *ret = NULL;
 
-        /* Prepare our buffer for the uid->username translation */
+    /* Prepare our buffer for the uid->username translation */
 #ifdef _SC_GETGR_R_SIZE_MAX
-        pwd_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+    pwd_buf_len = sysconf(_SC_GETGR_R_SIZE_MAX);
 #else
-        pwd_buf_len = -1;
+    pwd_buf_len = -1;
 #endif
-        if (pwd_buf_len == -1) {
-                pwd_buf_len = DEFAULT_PWD_BUF_SZ;  /* per the man page */
-        }
+    if (pwd_buf_len == -1) {
+        pwd_buf_len = DEFAULT_PWD_BUF_SZ; /* per the man page */
+    }
 
-        pwd_buf = alloca (pwd_buf_len);
-        if (!pwd_buf)
-                goto err;
+    pwd_buf = alloca(pwd_buf_len);
+    if (!pwd_buf)
+        goto err;
 
-        getpwuid_r (uid, &pwd, pwd_buf, pwd_buf_len,
-                   &pwd_result);
-        if (!pwd_result)
-                goto err;
+    getpwuid_r(uid, &pwd, pwd_buf, pwd_buf_len, &pwd_result);
+    if (!pwd_result)
+        goto err;
 
-        ret = gf_strdup (pwd.pw_name);
-        if (ret)
-                return ret;
-        else
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gf_strdup failed, failing username "
-                        "resolution.");
-err:
+    ret = gf_strdup(pwd.pw_name);
+    if (ret)
         return ret;
+    else
+        gf_log(this->name, GF_LOG_ERROR,
+               "gf_strdup failed, failing username "
+               "resolution.");
+err:
+    return ret;
 }
 
 char *
-_resolve_group_name (xlator_t *this, gid_t gid)
+_resolve_group_name(xlator_t *this, gid_t gid)
 {
-        struct group grp;
-        struct group *grp_result = NULL;
-        size_t grp_buf_len;
-        char   *grp_buf = NULL;
-        char   *ret = NULL;
+    struct group grp;
+    struct group *grp_result = NULL;
+    size_t grp_buf_len;
+    char *grp_buf = NULL;
+    char *ret = NULL;
 
-        /* Prepare our buffer for the gid->group name translation */
+    /* Prepare our buffer for the gid->group name translation */
 #ifdef _SC_GETGR_R_SIZE_MAX
-        grp_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+    grp_buf_len = sysconf(_SC_GETGR_R_SIZE_MAX);
 #else
-        grp_buf_len = -1;
+    grp_buf_len = -1;
 #endif
-        if (grp_buf_len == -1) {
-                grp_buf_len = DEFAULT_GRP_BUF_SZ;  /* per the man page */
-        }
+    if (grp_buf_len == -1) {
+        grp_buf_len = DEFAULT_GRP_BUF_SZ; /* per the man page */
+    }
 
-        grp_buf = alloca (grp_buf_len);
-        if (!grp_buf) {
-                goto err;
-        }
+    grp_buf = alloca(grp_buf_len);
+    if (!grp_buf) {
+        goto err;
+    }
 
-        getgrgid_r (gid, &grp, grp_buf, grp_buf_len,
-                   &grp_result);
-        if (!grp_result)
-                goto err;
+    if (getgrgid_r(gid, &grp, grp_buf, grp_buf_len, &grp_result) != 0)
+        goto err;
 
-        ret = gf_strdup (grp.gr_name);
-        if (ret)
-                return ret;
-        else
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gf_strdup failed, failing username "
-                        "resolution.");
-err:
+    if (!grp_result)
+        goto err;
+
+    ret = gf_strdup(grp.gr_name);
+    if (ret)
         return ret;
+    else
+        gf_log(this->name, GF_LOG_ERROR,
+               "gf_strdup failed, failing username "
+               "resolution.");
+err:
+    return ret;
 }
 
-
 /*
  * This function writes out a latency sample to a given file descriptor
  * and beautifies the output in the process.
  */
 void
-_io_stats_write_latency_sample (xlator_t *this, ios_sample_t *sample,
-                                FILE *logfp)
-{
-        double epoch_time = 0.00;
-        char   *xlator_name = NULL;
-        char   *instance_name = NULL;
-        char   *hostname = NULL;
-        char   *identifier = NULL;
-        char   *port = NULL;
-        char   *port_pos = NULL;
-        char   *group_name = NULL;
-        char   *username = NULL;
-        struct ios_conf *conf = NULL;
-
-        conf = this->private;
-
-        epoch_time = (sample->timestamp).tv_sec +
-          ((sample->timestamp).tv_usec / 1000000.0);
-
-        if (strlen (sample->identifier) == 0) {
-                hostname = "Unknown";
-                port = "Unknown";
-        } else {
-                identifier = strdupa (sample->identifier);
-                port_pos = strrchr (identifier, ':');
-                if (!port_pos || strlen(port_pos) < 2)
-                        goto err;
-                port = strdupa (port_pos + 1);
-                if (!port)
-                        goto err;
-                *port_pos = '\0';
-                hostname = gf_rev_dns_lookup_cached (identifier,
-                                                     conf->dnscache);
-                if (!hostname)
-                        hostname = "Unknown";
-        }
-
-        xlator_name = this->name;
-        if (!xlator_name || strlen (xlator_name) == 0)
-                xlator_name = "Unknown";
-
-        instance_name = this->instance_name;
-        if (!instance_name || strlen (instance_name) == 0)
-                instance_name = "N/A";
-
-        /* Resolve the UID to a string username */
-        username = _resolve_username (this, sample->uid);
+_io_stats_write_latency_sample(xlator_t *this, ios_sample_t *sample,
+                               FILE *logfp)
+{
+    double epoch_time = 0.00;
+    char *xlator_name = NULL;
+    char *instance_name = NULL;
+    char *hostname = NULL;
+    char *identifier = NULL;
+    char *port = NULL;
+    char *port_pos = NULL;
+    char *group_name = NULL;
+    char *username = NULL;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+
+    epoch_time = (sample->timestamp).tv_sec +
+                 ((sample->timestamp).tv_usec / 1000000.0);
+
+    if (strlen(sample->identifier) == 0) {
+        hostname = "Unknown";
+        port = "Unknown";
+    } else {
+        identifier = strdupa(sample->identifier);
+        port_pos = strrchr(identifier, ':');
+        if (!port_pos || strlen(port_pos) < 2)
+            goto err;
+        port = strdupa(port_pos + 1);
+        if (!port)
+            goto err;
+        *port_pos = '\0';
+        hostname = gf_rev_dns_lookup_cached(identifier, conf->dnscache);
+        if (!hostname)
+            hostname = "Unknown";
+    }
+
+    xlator_name = conf->unique_id;
+    if (!xlator_name || strlen(xlator_name) == 0)
+        xlator_name = "Unknown";
+
+    instance_name = this->instance_name;
+    if (!instance_name || strlen(instance_name) == 0)
+        instance_name = "N/A";
+
+    /* Resolve the UID to a string username */
+    username = _resolve_username(this, sample->uid);
+    if (!username) {
+        username = GF_MALLOC(30, gf_common_mt_char);
         if (!username) {
-                username = GF_MALLOC (30, gf_common_mt_char);
-                sprintf (username, "%d", (int32_t)sample->uid);
+            goto out;
         }
+        sprintf(username, "%d", (int32_t)sample->uid);
+    }
 
-        /* Resolve the GID to a string group name */
-        group_name = _resolve_group_name (this, sample->gid);
+    /* Resolve the GID to a string group name */
+    group_name = _resolve_group_name(this, sample->gid);
+    if (!group_name) {
+        group_name = GF_MALLOC(30, gf_common_mt_char);
         if (!group_name) {
-                group_name = GF_MALLOC (30, gf_common_mt_char);
-                sprintf (group_name, "%d", (int32_t)sample->gid);
+            goto out;
         }
-
-        ios_log (this, logfp,
-                 "%0.6lf,%s,%s,%0.4lf,%s,%s,%s,%s,%s,%s",
-                 epoch_time, fop_enum_to_pri_string (sample->fop_type),
-                 fop_enum_to_string (sample->fop_type),
-                 sample->elapsed, xlator_name, instance_name, username,
-                 group_name, hostname, port);
-        goto out;
+        sprintf(group_name, "%d", (int32_t)sample->gid);
+    }
+
+    ios_log(this, logfp, "%0.6lf,%s,%s,%0.4lf,%s,%s,%s,%s,%s,%s", epoch_time,
+            fop_enum_to_pri_string(sample->fop_type),
+            gf_fop_string(sample->fop_type), sample->elapsed, xlator_name,
+            instance_name, username, group_name, hostname, port);
+    goto out;
 err:
-        gf_log (this->name, GF_LOG_ERROR,
-                "Error parsing socket identifier");
+    gf_log(this->name, GF_LOG_ERROR, "Error parsing socket identifier");
 out:
-        GF_FREE (group_name);
-        GF_FREE (username);
+    GF_FREE(group_name);
+    GF_FREE(username);
 }
 
 /*
@@ -1078,3005 +1138,3343 @@ out:
  * contents of the saved reference.
  */
 int
-io_stats_dump_latency_samples_logfp (xlator_t *this, FILE *logfp)
-{
-        uint64_t              i = 0;
-        struct ios_conf       *conf = NULL;
-        ios_sample_buf_t      *sample_buf = NULL;
-        int                   ret = 1;  /* Default to error */
-
-        conf = this->private;
-
-        /* Save pointer to old buffer; the CS equivalent of
-         * Indiana Jones: https://www.youtube.com/watch?v=Pr-8AP0To4k,
-         * though ours will end better I hope!
-         */
-        sample_buf = conf->ios_sample_buf;
-        if (!sample_buf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Sampling buffer is null, bailing!");
-                goto out;
-        }
-
-        /* Empty case, nothing to do, exit. */
-        if (sample_buf->collected == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No samples, dump not required.");
-                ret = 0;
-                goto out;
-        }
-
-        /* Init a new buffer, so we are free to work on the one we saved a
-         * reference to above.
-         */
-        if (ios_init_sample_buf (conf) != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Failed to init new sampling buffer, out of memory?");
-                goto out;
-        }
+io_stats_dump_latency_samples_logfp(xlator_t *this, FILE *logfp)
+{
+    uint64_t i = 0;
+    struct ios_conf *conf = NULL;
+    ios_sample_buf_t *sample_buf = NULL;
+    int ret = 1; /* Default to error */
+
+    conf = this->private;
+
+    /* Save pointer to old buffer; the CS equivalent of
+     * Indiana Jones: https://www.youtube.com/watch?v=Pr-8AP0To4k,
+     * though ours will end better I hope!
+     */
+    sample_buf = conf->ios_sample_buf;
+    if (!sample_buf) {
+        gf_log(this->name, GF_LOG_WARNING, "Sampling buffer is null, bailing!");
+        goto out;
+    }
 
-        /* Wrap-around case, dump from pos to sample_buf->size -1
-         * and then from 0 to sample_buf->pos (covered off by
-         * "simple case")
-         */
-        if (sample_buf->collected > sample_buf->pos + 1) {
-                for (i = sample_buf->pos; i < sample_buf->size; i++) {
-                        _io_stats_write_latency_sample (this,
-                                &(sample_buf->ios_samples[i]), logfp);
-                }
+    /* Empty case, nothing to do, exit. */
+    if (sample_buf->collected == 0) {
+        gf_log(this->name, GF_LOG_DEBUG, "No samples, dump not required.");
+        ret = 0;
+        goto out;
+    }
+
+    /* Init a new buffer, so we are free to work on the one we saved a
+     * reference to above.
+     */
+    if (ios_init_sample_buf(conf) != 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Failed to init new sampling buffer, out of memory?");
+        goto out;
+    }
+
+    /* Wrap-around case, dump from pos to sample_buf->size -1
+     * and then from 0 to sample_buf->pos (covered off by
+     * "simple case")
+     */
+    if (sample_buf->collected > sample_buf->pos + 1) {
+        for (i = sample_buf->pos; i < sample_buf->size; i++) {
+            _io_stats_write_latency_sample(this, &(sample_buf->ios_samples[i]),
+                                           logfp);
         }
+    }
 
-        /* Simple case: Dump from 0 to sample_buf->pos */
-        for (i = 0; i < sample_buf->pos; i++) {
-                _io_stats_write_latency_sample (this,
-                        &(sample_buf->ios_samples[i]), logfp);
-        }
-        ios_destroy_sample_buf (sample_buf);
+    /* Simple case: Dump from 0 to sample_buf->pos */
+    for (i = 0; i < sample_buf->pos; i++) {
+        _io_stats_write_latency_sample(this, &(sample_buf->ios_samples[i]),
+                                       logfp);
+    }
+    ios_destroy_sample_buf(sample_buf);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
-                               struct timeval *now, int interval, FILE *logfp)
-{
-        int                   i = 0;
-        int                   per_line = 0;
-        int                   index = 0;
-        struct ios_stat_head *list_head = NULL;
-        struct ios_conf      *conf = NULL;
-        char                  timestr[256] = {0, };
-        char                  str_header[128] = {0};
-        char                  str_read[128] = {0};
-        char                  str_write[128] = {0};
-
-        conf = this->private;
-
-        if (interval == -1)
-                ios_log (this, logfp, "\n=== Cumulative stats ===");
+io_stats_dump_global_to_logfp(xlator_t *this, struct ios_global_stats *stats,
+                              time_t now, int interval, FILE *logfp)
+{
+    int i = 0;
+    int per_line = 0;
+    int index = 0;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_conf *conf = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char str_header[128] = {0};
+    char str_read[128] = {0};
+    char str_write[128] = {0};
+    uint64_t fop_hits = 0;
+    uint64_t block_count_read = 0;
+    uint64_t block_count_write = 0;
+
+    conf = this->private;
+
+    if (interval == -1)
+        ios_log(this, logfp, "\n=== Cumulative stats ===");
+    else
+        ios_log(this, logfp, "\n=== Interval %d stats ===", interval);
+    ios_log(this, logfp, "      Duration : %" PRIu64 " secs",
+            (uint64_t)(now - stats->started_at));
+    ios_log(this, logfp, "     BytesRead : %" GF_PRI_ATOMIC,
+            GF_ATOMIC_GET(stats->data_read));
+    ios_log(this, logfp, "  BytesWritten : %" GF_PRI_ATOMIC "\n",
+            GF_ATOMIC_GET(stats->data_written));
+
+    snprintf(str_header, sizeof(str_header), "%-12s %c", "Block Size", ':');
+    snprintf(str_read, sizeof(str_read), "%-12s %c", "Read Count", ':');
+    snprintf(str_write, sizeof(str_write), "%-12s %c", "Write Count", ':');
+    index = 14;
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        block_count_read = GF_ATOMIC_GET(stats->block_count_read[i]);
+        block_count_write = GF_ATOMIC_GET(stats->block_count_write[i]);
+        if ((block_count_read == 0) && (block_count_write == 0))
+            continue;
+        per_line++;
+
+        snprintf(str_header + index, sizeof(str_header) - index, "%16dB+",
+                 (1 << i));
+        if (block_count_read)
+            snprintf(str_read + index, sizeof(str_read) - index, "%18" PRId64,
+                     block_count_read);
         else
-                ios_log (this, logfp, "\n=== Interval %d stats ===",
-                         interval);
-        ios_log (this, logfp, "      Duration : %"PRId64" secs",
-                 (uint64_t) (now->tv_sec - stats->started_at.tv_sec));
-        ios_log (this, logfp, "     BytesRead : %"PRId64,
-                 stats->data_read);
-        ios_log (this, logfp, "  BytesWritten : %"PRId64"\n",
-                 stats->data_written);
-
-        snprintf (str_header, sizeof (str_header), "%-12s %c", "Block Size", ':');
-        snprintf (str_read, sizeof (str_read), "%-12s %c", "Read Count", ':');
-        snprintf (str_write, sizeof (str_write), "%-12s %c", "Write Count", ':');
-        index = 14;
-        for (i = 0; i < 32; i++) {
-                if ((stats->block_count_read[i] == 0) &&
-                    (stats->block_count_write[i] == 0))
-                        continue;
-                per_line++;
-
-                snprintf (str_header+index, sizeof (str_header)-index,
-                          "%16dB+", (1<<i));
-                if (stats->block_count_read[i])
-                        snprintf (str_read+index, sizeof (str_read)-index,
-                                  "%18"PRId64, stats->block_count_read[i]);
-                else    snprintf (str_read+index, sizeof (str_read)-index,
-                                  "%18s", "0");
-                if (stats->block_count_write[i])
-                        snprintf (str_write+index, sizeof (str_write)-index,
-                                  "%18"PRId64, stats->block_count_write[i]);
-                else    snprintf (str_write+index, sizeof (str_write)-index,
-                                  "%18s", "0");
-
-                index += 18;
-                if (per_line == 3) {
-                        ios_log (this, logfp, "%s", str_header);
-                        ios_log (this, logfp, "%s", str_read);
-                        ios_log (this, logfp, "%s\n", str_write);
-
-                        memset (str_header, 0, sizeof (str_header));
-                        memset (str_read, 0, sizeof (str_read));
-                        memset (str_write, 0, sizeof (str_write));
-
-                        snprintf (str_header, sizeof (str_header), "%-12s %c",
-                                  "Block Size", ':');
-                        snprintf (str_read, sizeof (str_read), "%-12s %c",
-                                  "Read Count", ':');
-                        snprintf (str_write, sizeof (str_write), "%-12s %c",
-                                  "Write Count", ':');
-
-                        index = 14;
-                        per_line = 0;
-                }
-        }
-
-        if (per_line != 0) {
-                ios_log (this, logfp, "%s", str_header);
-                ios_log (this, logfp, "%s", str_read);
-                ios_log (this, logfp, "%s\n", str_write);
-        }
-
-        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "Fop",
-                 "Call Count", "Avg-Latency", "Min-Latency",
-                 "Max-Latency");
-        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
-                 "-----------", "-----------", "-----------");
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                if (stats->fop_hits[i] && !stats->latency[i].avg)
-                        ios_log (this, logfp, "%-13s %10"PRId64" %11s "
-                                 "us %11s us %11s us", gf_fop_list[i],
-                                 stats->fop_hits[i], "0", "0", "0");
-                else if (stats->fop_hits[i] && stats->latency[i].avg)
-                        ios_log (this, logfp, "%-13s %10"PRId64" %11.2lf us "
-                                 "%11.2lf us %11.2lf us", gf_fop_list[i],
-                                 stats->fop_hits[i], stats->latency[i].avg,
-                                 stats->latency[i].min, stats->latency[i].max);
+            snprintf(str_read + index, sizeof(str_read) - index, "%18s", "0");
+        if (block_count_write)
+            snprintf(str_write + index, sizeof(str_write) - index,
+                     "%18" GF_PRI_ATOMIC, block_count_write);
+        else
+            snprintf(str_write + index, sizeof(str_write) - index, "%18s", "0");
+
+        index += 18;
+        if (per_line == 3) {
+            ios_log(this, logfp, "%s", str_header);
+            ios_log(this, logfp, "%s", str_read);
+            ios_log(this, logfp, "%s\n", str_write);
+
+            snprintf(str_header, sizeof(str_header), "%-12s %c", "Block Size",
+                     ':');
+            snprintf(str_read, sizeof(str_read), "%-12s %c", "Read Count", ':');
+            snprintf(str_write, sizeof(str_write), "%-12s %c", "Write Count",
+                     ':');
+
+            index = 14;
+            per_line = 0;
         }
-        ios_log (this, logfp, "------ ----- ----- ----- ----- ----- ----- ----- "
-                 " ----- ----- ----- -----\n");
-
-        if (interval == -1) {
-                LOCK (&conf->lock);
-                {
-                        gf_time_fmt (timestr, sizeof timestr,
-                                     conf->cumulative.max_openfd_time.tv_sec,
-                                     gf_timefmt_FT);
-                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                  ".%"GF_PRI_SUSECONDS,
-                                  conf->cumulative.max_openfd_time.tv_usec);
-                        ios_log (this, logfp, "Current open fd's: %"PRId64
-                                 " Max open fd's: %"PRId64" time %s",
-                                 conf->cumulative.nr_opens,
-                                 conf->cumulative.max_nr_opens, timestr);
-                }
-                UNLOCK (&conf->lock);
-                ios_log (this, logfp, "\n==========Open File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_OPEN];
-                ios_dump_file_stats (list_head, this, logfp);
-
-
-                ios_log (this, logfp, "\n==========Read File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_READ];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n==========Write File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_WRITE];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n==========Directory open stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n========Directory readdirp Stats=======");
-                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n========Read Throughput File Stats=====");
-                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
-                         "\tFILE NAME");
-                list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                ios_dump_throughput_stats(list_head, this, logfp,
-                                          IOS_STATS_THRU_READ);
-
-                ios_log (this, logfp, "\n======Write Throughput File Stats======");
-                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
-                         "\tFILE NAME");
-                list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-                ios_dump_throughput_stats (list_head, this, logfp,
-                                           IOS_STATS_THRU_WRITE);
+    }
+
+    if (per_line != 0) {
+        ios_log(this, logfp, "%s", str_header);
+        ios_log(this, logfp, "%s", str_read);
+        ios_log(this, logfp, "%s\n", str_write);
+    }
+
+    ios_log(this, logfp, "%-13s %10s %14s %14s %14s", "Fop", "Call Count",
+            "Avg-Latency", "Min-Latency", "Max-Latency");
+    ios_log(this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
+            "-----------", "-----------", "-----------");
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        if (fop_hits && !stats->latency[i].avg)
+            ios_log(this, logfp,
+                    "%-13s %10" GF_PRI_ATOMIC
+                    " %11s "
+                    "us %11s us %11s us",
+                    gf_fop_list[i], fop_hits, "0", "0", "0");
+        else if (fop_hits && stats->latency[i].avg)
+            ios_log(this, logfp,
+                    "%-13s %10" GF_PRI_ATOMIC
+                    " "
+                    "%11.2lf us %11.2lf us %11.2lf us",
+                    gf_fop_list[i], fop_hits, stats->latency[i].avg,
+                    stats->latency[i].min, stats->latency[i].max);
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
+        if (fop_hits)
+            ios_log(this, logfp,
+                    "%-13s %10" PRId64
+                    " %11s "
+                    "us %11s us %11s us",
+                    gf_upcall_list[i], fop_hits, "0", "0", "0");
+    }
+
+    ios_log(this, logfp,
+            "------ ----- ----- ----- ----- ----- ----- ----- "
+            " ----- ----- ----- -----\n");
+
+    if (interval == -1) {
+        LOCK(&conf->lock);
+        {
+            gf_time_fmt_tv(timestr, sizeof timestr,
+                           &conf->cumulative.max_openfd_time, gf_timefmt_FT);
+            ios_log(this, logfp,
+                    "Current open fd's: %" PRId64 " Max open fd's: %" PRId64
+                    " time %s",
+                    conf->cumulative.nr_opens, conf->cumulative.max_nr_opens,
+                    timestr);
         }
-        return 0;
+        UNLOCK(&conf->lock);
+        ios_log(this, logfp, "\n==========Open File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Read File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_READ];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Write File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Directory open stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n========Directory readdirp Stats=======");
+        ios_log(this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n========Read Throughput File Stats=====");
+        ios_log(this, logfp,
+                "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                "\tFILE NAME");
+        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+        ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+
+        ios_log(this, logfp, "\n======Write Throughput File Stats======");
+        ios_log(this, logfp,
+                "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                "\tFILE NAME");
+        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+        ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_WRITE);
+    }
+    return 0;
 }
 
 int
-io_stats_dump_global_to_dict (xlator_t *this, struct ios_global_stats *stats,
-                              struct timeval *now, int interval, dict_t *dict)
-{
-        int             ret = 0;
-        char            key[256] = {0};
-        uint64_t        sec = 0;
-        int             i = 0;
-        uint64_t        count = 0;
-
-        GF_ASSERT (stats);
-        GF_ASSERT (now);
-        GF_ASSERT (dict);
-        GF_ASSERT (this);
-
-        if (interval == -1)
-                snprintf (key, sizeof (key), "cumulative");
-        else
-                snprintf (key, sizeof (key), "interval");
-        ret = dict_set_int32 (dict, key, interval);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set "
-                        "interval %d", interval);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-duration", interval);
-        sec = (uint64_t) (now->tv_sec - stats->started_at.tv_sec);
-        ret = dict_set_uint64 (dict, key, sec);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set "
-                        "duration(%d) - %"PRId64, interval, sec);
+io_stats_dump_global_to_dict(xlator_t *this, struct ios_global_stats *stats,
+                             time_t now, int interval, dict_t *dict)
+{
+    int ret = 0;
+    char key[64] = {0};
+    uint64_t sec = 0;
+    int i = 0;
+    uint64_t count = 0;
+    uint64_t fop_hits = 0;
+
+    GF_ASSERT(stats);
+    GF_ASSERT(now);
+    GF_ASSERT(dict);
+    GF_ASSERT(this);
+
+    if (interval == -1)
+        snprintf(key, sizeof(key), "cumulative");
+    else
+        snprintf(key, sizeof(key), "interval");
+    ret = dict_set_int32(dict, key, interval);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set "
+               "interval %d",
+               interval);
+
+    snprintf(key, sizeof(key), "%d-duration", interval);
+    sec = now - stats->started_at;
+    ret = dict_set_uint64(dict, key, sec);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set "
+               "duration(%d) - %" PRId64,
+               interval, sec);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%d-total-read", interval);
+    ret = dict_set_uint64(dict, key, GF_ATOMIC_GET(stats->data_read));
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set total "
+               "read(%d) - %" GF_PRI_ATOMIC,
+               interval, GF_ATOMIC_GET(stats->data_read));
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%d-total-write", interval);
+    ret = dict_set_uint64(dict, key, GF_ATOMIC_GET(stats->data_written));
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set total "
+               "write(%d) - %" GF_PRI_ATOMIC,
+               interval, GF_ATOMIC_GET(stats->data_written));
+        goto out;
+    }
+    for (i = 0; i < 32; i++) {
+        count = GF_ATOMIC_GET(stats->block_count_read[i]);
+        if (count) {
+            snprintf(key, sizeof(key), "%d-read-%d", interval, (1 << i));
+            ret = dict_set_uint64(dict, key, count);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to "
+                       "set read-%db+, with: %" PRId64,
+                       (1 << i), count);
                 goto out;
+            }
         }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-total-read", interval);
-        ret = dict_set_uint64 (dict, key, stats->data_read);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
-                       "read(%d) - %"PRId64, interval, stats->data_read);
+    }
+
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        count = GF_ATOMIC_GET(stats->block_count_write[i]);
+        if (count) {
+            snprintf(key, sizeof(key), "%d-write-%d", interval, (1 << i));
+            ret = dict_set_uint64(dict, key, count);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to "
+                       "set write-%db+, with: %" PRId64,
+                       (1 << i), count);
                 goto out;
+            }
+        }
+    }
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        if (fop_hits == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-hits", interval, i);
+        ret = dict_set_uint64(dict, key, fop_hits);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "%s-fop-hits: %" GF_PRI_ATOMIC,
+                   gf_fop_list[i], fop_hits);
+            goto out;
         }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-total-write", interval);
-        ret = dict_set_uint64 (dict, key, stats->data_written);
+        if (stats->latency[i].avg == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-avglatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].avg);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
-                        "write(%d) - %"PRId64, interval, stats->data_written);
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "avglatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].avg);
+            goto out;
         }
-        for (i = 0; i < 32; i++) {
-                if (stats->block_count_read[i]) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-read-%d", interval,
-                                  (1 << i));
-                        count = stats->block_count_read[i];
-                        ret = dict_set_uint64 (dict, key, count);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set read-%db+, with: %"PRId64,
-                                        (1<<i), count);
-                                goto out;
-                        }
-                }
+        snprintf(key, sizeof(key), "%d-%d-minlatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].min);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "minlatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].min);
+            goto out;
         }
-
-        for (i = 0; i < 32; i++) {
-                if (stats->block_count_write[i]) {
-                        snprintf (key, sizeof (key), "%d-write-%d", interval,
-                                  (1<<i));
-                        count = stats->block_count_write[i];
-                        ret = dict_set_uint64 (dict, key, count);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set write-%db+, with: %"PRId64,
-                                        (1<<i), count);
-                                goto out;
-                        }
-                }
+        snprintf(key, sizeof(key), "%d-%d-maxlatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].max);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "maxlatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].max);
+            goto out;
         }
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                if (stats->fop_hits[i] == 0)
-                        continue;
-                snprintf (key, sizeof (key), "%d-%d-hits", interval, i);
-                ret = dict_set_uint64 (dict, key, stats->fop_hits[i]);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "set %s-fop-hits: %"PRIu64, gf_fop_list[i],
-                                stats->fop_hits[i]);
-                        goto out;
-                }
-
-                if (stats->latency[i].avg == 0)
-                        continue;
-                snprintf (key, sizeof (key), "%d-%d-avglatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].avg);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "avglatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].avg);
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-%d-minlatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].min);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "minlatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].min);
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-%d-maxlatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].max);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "maxlatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].max);
-                        goto out;
-                }
+    }
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
+        if (fop_hits == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-upcall-hits", interval, i);
+        ret = dict_set_uint64(dict, key, fop_hits);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to "
+                   "set %s-upcall-hits: %" PRIu64,
+                   gf_upcall_list[i], fop_hits);
+            goto out;
         }
+    }
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_log(this->name, GF_LOG_DEBUG, "returning %d", ret);
+    return ret;
 }
 
 int
-io_stats_dump_global (xlator_t *this, struct ios_global_stats *stats,
-                      struct timeval *now, int interval,
-                      struct ios_dump_args *args)
+io_stats_dump_global(xlator_t *this, struct ios_global_stats *stats, time_t now,
+                     int interval, struct ios_dump_args *args)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (args);
-        GF_ASSERT (now);
-        GF_ASSERT (stats);
-        GF_ASSERT (this);
+    GF_ASSERT(args);
+    GF_ASSERT(now);
+    GF_ASSERT(stats);
+    GF_ASSERT(this);
 
-
-
-        switch (args->type) {
+    switch (args->type) {
         case IOS_DUMP_TYPE_JSON_FILE:
-                ret = io_stats_dump_global_to_json_logfp (
-                    this, stats, now, interval, args->u.logfp);
-        break;
+            ret = io_stats_dump_global_to_json_logfp(this, stats, now, interval,
+                                                     args->u.logfp);
+            break;
         case IOS_DUMP_TYPE_FILE:
-                ret = io_stats_dump_global_to_logfp (this, stats, now,
-                                                     interval, args->u.logfp);
-        break;
+            ret = io_stats_dump_global_to_logfp(this, stats, now, interval,
+                                                args->u.logfp);
+            break;
         case IOS_DUMP_TYPE_DICT:
-                ret = io_stats_dump_global_to_dict (this, stats, now,
-                                                    interval, args->u.dict);
-        break;
+            ret = io_stats_dump_global_to_dict(this, stats, now, interval,
+                                               args->u.dict);
+            break;
         default:
-                GF_ASSERT (0);
-                ret = -1;
-        break;
-        }
-        return ret;
+            GF_ASSERT(0);
+            ret = -1;
+            break;
+    }
+    return ret;
 }
 
 int
-ios_dump_args_init (struct ios_dump_args *args, ios_dump_type_t type,
-                    void *output)
+ios_dump_args_init(struct ios_dump_args *args, ios_dump_type_t type,
+                   void *output)
 {
-        int             ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (args);
-        GF_ASSERT (type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
-        GF_ASSERT (output);
+    GF_ASSERT(args);
+    GF_ASSERT(type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
+    GF_ASSERT(output);
 
-        args->type = type;
-        switch (args->type) {
+    args->type = type;
+    switch (args->type) {
         case IOS_DUMP_TYPE_JSON_FILE:
         case IOS_DUMP_TYPE_FILE:
-                args->u.logfp = output;
-                break;
+            args->u.logfp = output;
+            break;
         case IOS_DUMP_TYPE_DICT:
-                args->u.dict = output;
-                break;
+            args->u.dict = output;
+            break;
         default:
-                GF_ASSERT (0);
-                ret = -1;
-        }
+            GF_ASSERT(0);
+            ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 static void
-ios_global_stats_clear (struct ios_global_stats *stats, struct timeval *now)
+ios_global_stats_clear(struct ios_global_stats *stats, time_t now)
 {
-        GF_ASSERT (stats);
-        GF_ASSERT (now);
+    GF_ASSERT(stats);
+    GF_ASSERT(now);
 
-        memset (stats, 0, sizeof (*stats));
-        stats->started_at = *now;
+    memset(stats, 0, sizeof(*stats));
+    stats->started_at = now;
 }
 
 int
-io_stats_dump (xlator_t *this, struct ios_dump_args *args,
-               gf1_cli_info_op op, gf_boolean_t is_peek)
+io_stats_dump(xlator_t *this, struct ios_dump_args *args, ios_info_op_t op,
+              gf_boolean_t is_peek)
 {
-        struct ios_conf         *conf = NULL;
-        struct ios_global_stats  cumulative = {0, };
-        struct ios_global_stats  incremental = {0, };
-        int                      increment = 0;
-        struct timeval           now;
+    struct ios_conf *conf = NULL;
+    struct ios_global_stats cumulative = {};
+    struct ios_global_stats incremental = {};
+    int increment = 0;
+    time_t now = 0;
 
-        GF_ASSERT (this);
-        GF_ASSERT (args);
-        GF_ASSERT (args->type > IOS_DUMP_TYPE_NONE);
-        GF_ASSERT (args->type < IOS_DUMP_TYPE_MAX);
+    GF_ASSERT(this);
+    GF_ASSERT(args);
+    GF_ASSERT(args->type > IOS_DUMP_TYPE_NONE);
+    GF_ASSERT(args->type < IOS_DUMP_TYPE_MAX);
 
-        conf = this->private;
+    conf = this->private;
+    now = gf_time();
 
-        gettimeofday (&now, NULL);
-        LOCK (&conf->lock);
-        {
-                if (op == GF_CLI_INFO_ALL ||
-                    op == GF_CLI_INFO_CUMULATIVE)
-                        cumulative  = conf->cumulative;
+    LOCK(&conf->lock);
+    {
+        if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE)
+            cumulative = conf->cumulative;
 
-                if (op == GF_CLI_INFO_ALL ||
-                    op == GF_CLI_INFO_INCREMENTAL) {
-                        incremental = conf->incremental;
-                        increment = conf->increment;
+        if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_INCREMENTAL) {
+            incremental = conf->incremental;
+            increment = conf->increment;
 
-                        if (!is_peek) {
-                                increment = conf->increment++;
+            if (!is_peek) {
+                increment = conf->increment++;
 
-                                ios_global_stats_clear (&conf->incremental,
-                                                        &now);
-                        }
-                }
+                ios_global_stats_clear(&conf->incremental, now);
+            }
         }
-        UNLOCK (&conf->lock);
+    }
+    UNLOCK(&conf->lock);
 
-        if (op == GF_CLI_INFO_ALL ||
-            op == GF_CLI_INFO_CUMULATIVE)
-                io_stats_dump_global (this, &cumulative, &now, -1, args);
+    if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE)
+        io_stats_dump_global(this, &cumulative, now, -1, args);
 
-        if (op == GF_CLI_INFO_ALL ||
-            op == GF_CLI_INFO_INCREMENTAL)
-                io_stats_dump_global (this, &incremental, &now, increment, args);
+    if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_INCREMENTAL)
+        io_stats_dump_global(this, &incremental, now, increment, args);
 
-        return 0;
+    return 0;
 }
 
-
 int
-io_stats_dump_fd (xlator_t *this, struct ios_fd *iosfd)
+io_stats_dump_fd(xlator_t *this, struct ios_fd *iosfd)
 {
-        struct ios_conf         *conf = NULL;
-        struct timeval           now;
-        uint64_t                 sec = 0;
-        uint64_t                 usec = 0;
-        int                      i = 0;
+    struct ios_conf *conf = NULL;
+    struct timeval now;
+    int i = 0;
+    double usecs = 0;
+    uint64_t data_read = 0;
+    uint64_t data_written = 0;
+    uint64_t block_count_read = 0;
+    uint64_t block_count_write = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->dump_fd_stats)
-                return 0;
+    if (!conf->dump_fd_stats)
+        return 0;
 
-        if (!iosfd)
-                return 0;
+    if (!iosfd)
+        return 0;
 
-        gettimeofday (&now, NULL);
+    gettimeofday(&now, NULL);
+    usecs = gf_tvdiff(&iosfd->opened_at, &now);
 
-        if (iosfd->opened_at.tv_usec > now.tv_usec) {
-                now.tv_usec += 1000000;
-                now.tv_usec--;
-        }
+    gf_log(this->name, GF_LOG_INFO, "--- fd stats ---");
 
-        sec = now.tv_sec - iosfd->opened_at.tv_sec;
-        usec = now.tv_usec - iosfd->opened_at.tv_usec;
-
-        gf_log (this->name, GF_LOG_INFO,
-                "--- fd stats ---");
-
-        if (iosfd->filename)
-                gf_log (this->name, GF_LOG_INFO,
-                        "      Filename : %s",
-                        iosfd->filename);
-
-        if (sec)
-                gf_log (this->name, GF_LOG_INFO,
-                        "      Lifetime : %"PRId64"secs, %"PRId64"usecs",
-                        sec, usec);
-
-        if (iosfd->data_read)
-                gf_log (this->name, GF_LOG_INFO,
-                        "     BytesRead : %"PRId64" bytes",
-                        iosfd->data_read);
-
-        if (iosfd->data_written)
-                gf_log (this->name, GF_LOG_INFO,
-                        "  BytesWritten : %"PRId64" bytes",
-                        iosfd->data_written);
-
-        for (i = 0; i < 32; i++) {
-                if (iosfd->block_count_read[i])
-                        gf_log (this->name, GF_LOG_INFO,
-                                " Read %06db+ : %"PRId64,
-                                (1 << i), iosfd->block_count_read[i]);
-        }
-        for (i = 0; i < 32; i++) {
-                if (iosfd->block_count_write[i])
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Write %06db+ : %"PRId64,
-                                (1 << i), iosfd->block_count_write[i]);
-        }
-        return 0;
-}
+    if (iosfd->filename)
+        gf_log(this->name, GF_LOG_INFO, "      Filename : %s", iosfd->filename);
 
-void collect_ios_latency_sample (struct ios_conf *conf,
-                glusterfs_fop_t fop_type, double elapsed,
-                call_frame_t *frame)
-{
-        ios_sample_buf_t *ios_sample_buf = NULL;
-        ios_sample_t     *ios_sample = NULL;
-        struct timeval   *timestamp = NULL;
-        call_stack_t     *root = NULL;
+    if (usecs)
+        gf_log(this->name, GF_LOG_INFO, "      Lifetime : %lf secs", usecs);
 
+    data_read = GF_ATOMIC_GET(iosfd->data_read);
+    if (data_read)
+        gf_log(this->name, GF_LOG_INFO, "     BytesRead : %" PRId64 " bytes",
+               data_read);
 
-        ios_sample_buf = conf->ios_sample_buf;
-        LOCK (&conf->ios_sampling_lock);
-        if (conf->ios_sample_interval == 0 ||
-            ios_sample_buf->observed % conf->ios_sample_interval != 0)
-                goto out;
+    data_written = GF_ATOMIC_GET(iosfd->data_written);
+    if (data_written)
+        gf_log(this->name, GF_LOG_INFO, "  BytesWritten : %" PRId64 " bytes",
+               data_written);
 
-        timestamp = &frame->begin;
-        root = frame->root;
-
-        ios_sample = &(ios_sample_buf->ios_samples[ios_sample_buf->pos]);
-        ios_sample->elapsed = elapsed;
-        ios_sample->fop_type = fop_type;
-        ios_sample->uid = root->uid;
-        ios_sample->gid = root->gid;
-        (ios_sample->timestamp).tv_sec = timestamp->tv_sec;
-        (ios_sample->timestamp).tv_usec = timestamp->tv_usec;
-        memcpy (&ios_sample->identifier, &root->identifier,
-                sizeof (root->identifier));
-
-        /* We've reached the end of the circular buffer, start from the
-         * beginning. */
-        if (ios_sample_buf->pos == (ios_sample_buf->size - 1))
-                ios_sample_buf->pos = 0;
-        else
-                ios_sample_buf->pos++;
-        ios_sample_buf->collected++;
+    for (i = 0; i < 32; i++) {
+        block_count_read = GF_ATOMIC_GET(iosfd->block_count_read[i]);
+        if (block_count_read)
+            gf_log(this->name, GF_LOG_INFO,
+                   " Read %06db+ :"
+                   "%" PRId64,
+                   (1 << i), block_count_read);
+    }
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        block_count_write = GF_ATOMIC_GET(iosfd->block_count_write[i]);
+        if (block_count_write)
+            gf_log(this->name, GF_LOG_INFO, "Write %06db+ : %" PRId64, (1 << i),
+                   block_count_write);
+    }
+    return 0;
+}
+
+void
+collect_ios_latency_sample(struct ios_conf *conf, glusterfs_fop_t fop_type,
+                           double elapsed, call_frame_t *frame)
+{
+    ios_sample_buf_t *ios_sample_buf = NULL;
+    ios_sample_t *ios_sample = NULL;
+    struct timespec *timestamp = NULL;
+    call_stack_t *root = NULL;
+
+    ios_sample_buf = conf->ios_sample_buf;
+    LOCK(&conf->ios_sampling_lock);
+    if (conf->ios_sample_interval == 0 ||
+        ios_sample_buf->observed % conf->ios_sample_interval != 0)
+        goto out;
+
+    timestamp = &frame->begin;
+    root = frame->root;
+
+    ios_sample = &(ios_sample_buf->ios_samples[ios_sample_buf->pos]);
+    ios_sample->elapsed = elapsed;
+    ios_sample->fop_type = fop_type;
+    ios_sample->uid = root->uid;
+    ios_sample->gid = root->gid;
+    (ios_sample->timestamp).tv_sec = timestamp->tv_sec;
+    (ios_sample->timestamp).tv_usec = timestamp->tv_nsec / 1000;
+    memcpy(&ios_sample->identifier, &root->identifier,
+           sizeof(root->identifier));
+
+    /* We've reached the end of the circular buffer, start from the
+     * beginning. */
+    if (ios_sample_buf->pos == (ios_sample_buf->size - 1))
+        ios_sample_buf->pos = 0;
+    else
+        ios_sample_buf->pos++;
+    ios_sample_buf->collected++;
 out:
-        ios_sample_buf->observed++;
-        UNLOCK (&conf->ios_sampling_lock);
-        return;
+    ios_sample_buf->observed++;
+    UNLOCK(&conf->ios_sampling_lock);
+    return;
 }
 
 static void
-update_ios_latency_stats (struct ios_global_stats   *stats, double elapsed,
-                          glusterfs_fop_t op)
+update_ios_latency_stats(struct ios_global_stats *stats, double elapsed,
+                         glusterfs_fop_t op)
 {
-        double avg;
+    double avg;
 
-        GF_ASSERT (stats);
+    GF_ASSERT(stats);
 
-        stats->latency[op].total += elapsed;
+    stats->latency[op].total += elapsed;
 
-        if (!stats->latency[op].min)
-                stats->latency[op].min = elapsed;
-        if (stats->latency[op].min > elapsed)
-                stats->latency[op].min = elapsed;
-        if (stats->latency[op].max < elapsed)
-                stats->latency[op].max = elapsed;
+    if (!stats->latency[op].min)
+        stats->latency[op].min = elapsed;
+    if (stats->latency[op].min > elapsed)
+        stats->latency[op].min = elapsed;
+    if (stats->latency[op].max < elapsed)
+        stats->latency[op].max = elapsed;
 
-        avg = stats->latency[op].avg;
+    avg = stats->latency[op].avg;
 
-        stats->latency[op].avg = avg + (elapsed - avg) / stats->fop_hits[op];
+    stats->latency[op].avg = avg + (elapsed - avg) /
+                                       GF_ATOMIC_GET(stats->fop_hits[op]);
 }
 
 int
-update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
-                    glusterfs_fop_t op)
+update_ios_latency(struct ios_conf *conf, call_frame_t *frame,
+                   glusterfs_fop_t op)
 {
-        double elapsed;
-        struct timeval *begin, *end;
+    double elapsed;
+    struct timespec *begin, *end;
 
-        begin = &frame->begin;
-        end   = &frame->end;
+    begin = &frame->begin;
+    end = &frame->end;
 
-        elapsed = (end->tv_sec - begin->tv_sec) * 1e6
-                + (end->tv_usec - begin->tv_usec);
+    elapsed = gf_tsdiff(begin, end) / 1000.0;
 
-        update_ios_latency_stats (&conf->cumulative, elapsed, op);
-        update_ios_latency_stats (&conf->incremental, elapsed, op);
-        collect_ios_latency_sample (conf, op, elapsed, frame);
+    update_ios_latency_stats(&conf->cumulative, elapsed, op);
+    update_ios_latency_stats(&conf->incremental, elapsed, op);
+    collect_ios_latency_sample(conf, op, elapsed, frame);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
-                             ios_stats_type_t flags, int32_t list_cnt)
-{
-        struct ios_conf         *conf = NULL;
-        int                      cnt  = 0;
-        char                     key[256];
-        struct ios_stat_head    *list_head = NULL;
-        struct ios_stat_list    *entry = NULL;
-        int                      ret = -1;
-        ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
-        char                     timestr[256] = {0, };
-        char                    *dict_timestr = NULL;
-
-        conf = this->private;
-
-        switch (flags) {
-                case IOS_STATS_TYPE_OPEN:
-                        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
-                        LOCK (&conf->lock);
-                        {
-                                ret = dict_set_uint64 (resp, "current-open",
-                                                     conf->cumulative.nr_opens);
-                                if (ret)
-                                        goto unlock;
-                                ret = dict_set_uint64 (resp, "max-open",
-                                                       conf->cumulative.max_nr_opens);
-
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             conf->cumulative.max_openfd_time.tv_sec,
-                                             gf_timefmt_FT);
-                                if (conf->cumulative.max_openfd_time.tv_sec)
-                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                                  ".%"GF_PRI_SUSECONDS,
-                                                  conf->cumulative.max_openfd_time.tv_usec);
-
-                                dict_timestr = gf_strdup (timestr);
-                                if (!dict_timestr)
-                                        goto unlock;
-                                ret = dict_set_dynstr (resp, "max-openfd-time",
-                                                       dict_timestr);
-                                if (ret)
-                                        goto unlock;
-                        }
+io_stats_dump_stats_to_dict(xlator_t *this, dict_t *resp,
+                            ios_stats_type_t flags, int32_t list_cnt)
+{
+    struct ios_conf *conf = NULL;
+    int cnt = 0;
+    char key[32];
+    int keylen;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_stat_list *entry = NULL;
+    int ret = -1;
+    ios_stats_thru_t index = IOS_STATS_THRU_MAX;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *dict_timestr = NULL;
+
+    conf = this->private;
+
+    switch (flags) {
+        case IOS_STATS_TYPE_OPEN:
+            list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+            LOCK(&conf->lock);
+            {
+                ret = dict_set_uint64(resp, "current-open",
+                                      conf->cumulative.nr_opens);
+                if (ret)
+                    goto unlock;
+                ret = dict_set_uint64(resp, "max-open",
+                                      conf->cumulative.max_nr_opens);
+
+                gf_time_fmt_tv(timestr, sizeof timestr,
+                               &conf->cumulative.max_openfd_time,
+                               gf_timefmt_FT);
+
+                dict_timestr = gf_strdup(timestr);
+                if (!dict_timestr)
+                    goto unlock;
+                ret = dict_set_dynstr(resp, "max-openfd-time", dict_timestr);
+                if (ret)
+                    goto unlock;
+            }
         unlock:
-                        UNLOCK (&conf->lock);
-                        /* Do not proceed if we came here because of some error
-                         * during the dict operation */
-                        if (ret)
-                                goto out;
-                        break;
-                case IOS_STATS_TYPE_READ:
-                        list_head = &conf->list[IOS_STATS_TYPE_READ];
-                        break;
-                case IOS_STATS_TYPE_WRITE:
-                        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
-                        break;
-                case IOS_STATS_TYPE_OPENDIR:
-                        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
-                        break;
-                case IOS_STATS_TYPE_READDIRP:
-                        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
-                        break;
-                case IOS_STATS_TYPE_READ_THROUGHPUT:
-                        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                        index = IOS_STATS_THRU_READ;
-                        break;
-                case IOS_STATS_TYPE_WRITE_THROUGHPUT:
-                        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-                        index = IOS_STATS_THRU_WRITE;
-                        break;
-
-                default:
-                       goto out;
-        }
-        ret = dict_set_int32 (resp, "top-op", flags);
-        if (!list_cnt)
+            UNLOCK(&conf->lock);
+            /* Do not proceed if we came here because of some error
+             * during the dict operation */
+            if (ret)
                 goto out;
-        LOCK (&list_head->lock);
-        {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-
-                        cnt++;
-                        snprintf (key, 256, "%s-%d", "filename", cnt);
-                        ret = dict_set_str (resp, key, entry->iosstat->filename);
-                        if (ret)
-                                goto unlock_list_head;
-                         snprintf (key, 256, "%s-%d", "value",cnt);
-                         ret = dict_set_uint64 (resp, key, entry->value);
-                         if (ret)
-                                 goto unlock_list_head;
-                         if (index != IOS_STATS_THRU_MAX) {
-                                 snprintf (key, 256, "%s-%d", "time-sec", cnt);
-                                 ret = dict_set_int32 (resp, key,
-                                         entry->iosstat->thru_counters[index].time.tv_sec);
-                                 if (ret)
-                                         goto unlock_list_head;
-                                 snprintf (key, 256, "%s-%d", "time-usec", cnt);
-                                 ret = dict_set_int32 (resp, key,
-                                         entry->iosstat->thru_counters[index].time.tv_usec);
-                                 if (ret)
-                                         goto unlock_list_head;
-                         }
-                         if (cnt == list_cnt)
-                                 break;
+            break;
+        case IOS_STATS_TYPE_READ:
+            list_head = &conf->list[IOS_STATS_TYPE_READ];
+            break;
+        case IOS_STATS_TYPE_WRITE:
+            list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+            break;
+        case IOS_STATS_TYPE_OPENDIR:
+            list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+            break;
+        case IOS_STATS_TYPE_READDIRP:
+            list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+            break;
+        case IOS_STATS_TYPE_READ_THROUGHPUT:
+            list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+            index = IOS_STATS_THRU_READ;
+            break;
+        case IOS_STATS_TYPE_WRITE_THROUGHPUT:
+            list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+            index = IOS_STATS_THRU_WRITE;
+            break;
 
-                }
+        default:
+            goto out;
+    }
+    ret = dict_set_int32_sizen(resp, "top-op", flags);
+    if (!list_cnt)
+        goto out;
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
+        {
+            cnt++;
+            keylen = snprintf(key, sizeof(key), "filename-%d", cnt);
+            ret = dict_set_strn(resp, key, keylen, entry->iosstat->filename);
+            if (ret)
+                goto unlock_list_head;
+            snprintf(key, sizeof(key), "value-%d", cnt);
+            ret = dict_set_uint64(resp, key, entry->value);
+            if (ret)
+                goto unlock_list_head;
+            if (index != IOS_STATS_THRU_MAX) {
+                keylen = snprintf(key, sizeof(key), "time-sec-%d", cnt);
+                ret = dict_set_int32n(
+                    resp, key, keylen,
+                    entry->iosstat->thru_counters[index].time.tv_sec);
+                if (ret)
+                    goto unlock_list_head;
+                keylen = snprintf(key, sizeof(key), "time-usec-%d", cnt);
+                ret = dict_set_int32n(
+                    resp, key, keylen,
+                    entry->iosstat->thru_counters[index].time.tv_usec);
+                if (ret)
+                    goto unlock_list_head;
+            }
+            if (cnt == list_cnt)
+                break;
         }
+    }
 unlock_list_head:
-        UNLOCK (&list_head->lock);
-        /* ret is !=0 if some dict operation in the above critical region
-         * failed. */
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (resp, "members", cnt);
- out:
-        return ret;
+    UNLOCK(&list_head->lock);
+    /* ret is !=0 if some dict operation in the above critical region
+     * failed. */
+    if (ret)
+        goto out;
+    ret = dict_set_int32_sizen(resp, "members", cnt);
+out:
+    return ret;
 }
 
-int
-io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd,
-                     inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+static struct ios_stat *
+ios_init_iosstat(xlator_t *this, char *path, uuid_t gfid, inode_t *inode)
 {
-        struct ios_fd *iosfd = NULL;
-        char          *path = NULL;
-        struct ios_stat *iosstat = NULL;
-        struct ios_conf   *conf = NULL;
+    struct ios_stat *iosstat = NULL;
+    int i = 0;
 
-        conf = this->private;
+    iosstat = GF_CALLOC(1, sizeof(*iosstat), gf_io_stats_mt_ios_stat);
+    if (!iosstat)
+        goto out;
 
-        path = frame->local;
-        frame->local = NULL;
+    iosstat->filename = gf_strdup(path);
+    gf_uuid_copy(iosstat->gfid, gfid);
+    LOCK_INIT(&iosstat->lock);
 
-        if (!path)
-                goto unwind;
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++)
+        GF_ATOMIC_INIT(iosstat->counters[i], 0);
 
-        if (op_ret < 0) {
-                GF_FREE (path);
-                goto unwind;
-        }
+    ios_inode_ctx_set(inode, this, iosstat);
 
-        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
-        if (!iosfd) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd->filename = path;
-        gettimeofday (&iosfd->opened_at, NULL);
+out:
+    return iosstat;
+}
 
-        ios_fd_ctx_set (fd, this, iosfd);
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens++;
-                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
-                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
-                        conf->cumulative.max_openfd_time = iosfd->opened_at;
-                }
+int
+io_stats_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    struct ios_fd *iosfd = NULL;
+    char *path = NULL;
+    struct ios_stat *iosstat = NULL;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+
+    path = frame->local;
+    frame->local = NULL;
+
+    if (!path)
+        goto unwind;
+
+    if (op_ret < 0) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd = GF_CALLOC(1, sizeof(*iosfd), gf_io_stats_mt_ios_fd);
+    if (!iosfd) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd->filename = path;
+    gettimeofday(&iosfd->opened_at, NULL);
+
+    ios_fd_ctx_set(fd, this, iosfd);
+    LOCK(&conf->lock);
+    {
+        conf->cumulative.nr_opens++;
+        if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+            conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+            conf->cumulative.max_openfd_time = iosfd->opened_at;
         }
-        UNLOCK (&conf->lock);
+    }
+    UNLOCK(&conf->lock);
 
-        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
-        if (!iosstat) {
-                GF_FREE (path);
-                goto unwind;
-        }
-        iosstat->filename = gf_strdup (path);
-        gf_uuid_copy (iosstat->gfid, buf->ia_gfid);
-        LOCK_INIT (&iosstat->lock);
-        ios_inode_ctx_set (fd->inode, this, iosstat);
+    iosstat = ios_init_iosstat(this, path, buf->ia_gfid, inode);
+    if (!iosstat)
+        GF_FREE(path);
 
 unwind:
-        UPDATE_PROFILE_STATS (frame, CREATE);
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, CREATE);
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        struct ios_fd *iosfd = NULL;
-        char          *path = NULL;
-        struct   ios_stat *iosstat = NULL;
-        struct ios_conf   *conf = NULL;
-
-        conf = this->private;
-        path = frame->local;
-        frame->local = NULL;
-
-        if (!path)
-                goto unwind;
-
-        if (op_ret < 0) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
-        if (!iosfd) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd->filename = path;
-        gettimeofday (&iosfd->opened_at, NULL);
-
-        ios_fd_ctx_set (fd, this, iosfd);
-
-        ios_inode_ctx_get (fd->inode, this, &iosstat);
-        if (!iosstat) {
-                iosstat = GF_CALLOC (1, sizeof (*iosstat),
-                                     gf_io_stats_mt_ios_stat);
-                if (iosstat) {
-                        iosstat->filename = gf_strdup (path);
-                        gf_uuid_copy (iosstat->gfid, fd->inode->gfid);
-                        LOCK_INIT (&iosstat->lock);
-                        ios_inode_ctx_set (fd->inode, this, iosstat);
-                }
-        }
-
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens++;
-                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
-                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
-                        conf->cumulative.max_openfd_time = iosfd->opened_at;
-                }
-        }
-        UNLOCK (&conf->lock);
-        if (iosstat) {
-              BUMP_STATS (iosstat, IOS_STATS_TYPE_OPEN);
-              iosstat = NULL;
+io_stats_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    struct ios_fd *iosfd = NULL;
+    char *path = NULL;
+    struct ios_stat *iosstat = NULL;
+    struct ios_conf *conf = NULL;
+    int i = 0;
+
+    conf = this->private;
+    path = frame->local;
+    frame->local = NULL;
+
+    if (!path)
+        goto unwind;
+
+    if (op_ret < 0) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd = GF_CALLOC(1, sizeof(*iosfd), gf_io_stats_mt_ios_fd);
+    if (!iosfd) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd->filename = path;
+    GF_ATOMIC_INIT(iosfd->data_read, 0);
+    GF_ATOMIC_INIT(iosfd->data_written, 0);
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        GF_ATOMIC_INIT(iosfd->block_count_write[i], 0);
+        GF_ATOMIC_INIT(iosfd->block_count_read[i], 0);
+    }
+    gettimeofday(&iosfd->opened_at, NULL);
+
+    ios_fd_ctx_set(fd, this, iosfd);
+
+    ios_inode_ctx_get(fd->inode, this, &iosstat);
+    if (!iosstat) {
+        iosstat = ios_init_iosstat(this, path, fd->inode->gfid, fd->inode);
+    }
+
+    LOCK(&conf->lock);
+    {
+        conf->cumulative.nr_opens++;
+        if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+            conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+            conf->cumulative.max_openfd_time = iosfd->opened_at;
         }
+    }
+    UNLOCK(&conf->lock);
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_OPEN);
+        iosstat = NULL;
+    }
 unwind:
-        UPDATE_PROFILE_STATS (frame, OPEN);
-
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, OPEN);
 
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+io_stats_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, STAT);
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, STAT);
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iovec *vector, int32_t count,
-                    struct iatt *buf, struct iobref *iobref, dict_t *xdata)
+io_stats_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                   int32_t count, struct iatt *buf, struct iobref *iobref,
+                   dict_t *xdata)
 {
-        int              len = 0;
-        fd_t            *fd = NULL;
-        struct ios_stat *iosstat = NULL;
+    int len = 0;
+    fd_t *fd = NULL;
+    struct ios_stat *iosstat = NULL;
 
-        fd = frame->local;
-        frame->local = NULL;
-
-        if (op_ret > 0) {
-                len = iov_length (vector, count);
-                BUMP_READ (fd, len);
-        }
+    fd = frame->local;
+    frame->local = NULL;
 
-        UPDATE_PROFILE_STATS (frame, READ);
-        ios_inode_ctx_get (fd->inode, this, &iosstat);
+    if (op_ret > 0) {
+        len = iov_length(vector, count);
+        ios_bump_read(this, fd, len);
+    }
 
-        if (iosstat) {
-              BUMP_STATS (iosstat, IOS_STATS_TYPE_READ);
-              BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_READ);
-              iosstat = NULL;
-        }
+    UPDATE_PROFILE_STATS(frame, READ);
+    ios_inode_ctx_get(fd->inode, this, &iosstat);
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, buf, iobref, xdata);
-        return 0;
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_READ);
+        BUMP_THROUGHPUT(iosstat, IOS_STATS_THRU_READ);
+        iosstat = NULL;
+    }
 
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, buf,
+                        iobref, xdata);
+    return 0;
 }
 
-
 int
-io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        inode_t         *inode   = NULL;
+    struct ios_stat *iosstat = NULL;
+    inode_t *inode = NULL;
 
-        UPDATE_PROFILE_STATS (frame, WRITE);
-        if (frame->local){
-                inode = frame->local;
-                frame->local = NULL;
-                ios_inode_ctx_get (inode, this, &iosstat);
-                if (iosstat) {
-                        BUMP_STATS (iosstat, IOS_STATS_TYPE_WRITE);
-                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
-                        inode = NULL;
-                        iosstat = NULL;
-                }
+    UPDATE_PROFILE_STATS(frame, WRITE);
+    if (frame->local) {
+        inode = frame->local;
+        frame->local = NULL;
+        ios_inode_ctx_get(inode, this, &iosstat);
+        if (iosstat) {
+            ios_bump_stats(this, iosstat, IOS_STATS_TYPE_WRITE);
+            BUMP_THROUGHPUT(iosstat, IOS_STATS_THRU_WRITE);
+            inode = NULL;
+            iosstat = NULL;
         }
+    }
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
-
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
+int
+io_stats_copy_file_range_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *stbuf, struct iatt *prebuf_dst,
+                             struct iatt *postbuf_dst, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, COPY_FILE_RANGE);
 
-
+    STACK_UNWIND_STRICT(copy_file_range, frame, op_ret, op_errno, stbuf,
+                        prebuf_dst, postbuf_dst, xdata);
+    return 0;
+}
 
 int
-io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+io_stats_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                      dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        inode_t         *inode   = frame->local;
+    struct ios_stat *iosstat = NULL;
+    inode_t *inode = frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        UPDATE_PROFILE_STATS (frame, READDIRP);
+    UPDATE_PROFILE_STATS(frame, READDIRP);
 
-        ios_inode_ctx_get (inode, this, &iosstat);
+    ios_inode_ctx_get(inode, this, &iosstat);
 
-        if (iosstat) {
-              BUMP_STATS (iosstat, IOS_STATS_TYPE_READDIRP);
-               iosstat = NULL;
-        }
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_READDIRP);
+        iosstat = NULL;
+    }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+io_stats_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                     dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, READDIR);
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, READDIR);
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSYNC);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSYNC);
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+io_stats_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SETATTR);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETATTR);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, preop, postop, xdata);
+    return 0;
 }
 
-
 int
-io_stats_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, UNLINK);
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-        return 0;
-
+    UPDATE_PROFILE_STATS(frame, UNLINK);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
+io_stats_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, RENAME);
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, RENAME);
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, const char *buf,
-                       struct iatt *sbuf, dict_t *xdata)
+io_stats_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, const char *buf,
+                      struct iatt *sbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, READLINK);
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, READLINK);
+    STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     inode_t *inode, struct iatt *buf,
-                     dict_t *xdata, struct iatt *postparent)
+io_stats_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        UPDATE_PROFILE_STATS (frame, LOOKUP);
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
-                             postparent);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LOOKUP);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
-
 int
-io_stats_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SYMLINK);
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SYMLINK);
+    STACK_UNWIND_STRICT(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, MKNOD);
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, MKNOD);
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata)
+io_stats_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        char   *path = frame->local;
+    char *path = frame->local;
 
-        if (!path)
-                goto unwind;
+    if (!path)
+        goto unwind;
 
-        UPDATE_PROFILE_STATS (frame, MKDIR);
-        if (op_ret < 0)
-                goto unwind;
+    UPDATE_PROFILE_STATS(frame, MKDIR);
+    if (op_ret < 0)
+        goto unwind;
 
-        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
-        if (iosstat) {
-                LOCK_INIT (&iosstat->lock);
-                iosstat->filename = gf_strdup(path);
-                gf_uuid_copy (iosstat->gfid, buf->ia_gfid);
-                ios_inode_ctx_set (inode, this, iosstat);
-        }
+    /* allocate a struct ios_stat and set the inode ctx */
+    ios_init_iosstat(this, path, buf->ia_gfid, inode);
 
 unwind:
-        /* local is assigned with path */
-        GF_FREE (frame->local);
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    /* local is assigned with path */
+    GF_FREE(frame->local);
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, LINK);
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LINK);
+    STACK_UNWIND_STRICT(link, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FLUSH);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FLUSH);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+io_stats_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        int              ret     = -1;
+    struct ios_stat *iosstat = NULL;
+    int ret = -1;
 
-        UPDATE_PROFILE_STATS (frame, OPENDIR);
-        if (op_ret < 0)
-                goto unwind;
+    UPDATE_PROFILE_STATS(frame, OPENDIR);
+    if (op_ret < 0)
+        goto unwind;
 
-        ios_fd_ctx_set (fd, this, 0);
+    ios_fd_ctx_set(fd, this, 0);
 
-        ret = ios_inode_ctx_get (fd->inode, this, &iosstat);
-        if (!ret)
-                BUMP_STATS (iosstat, IOS_STATS_TYPE_OPENDIR);
+    ret = ios_inode_ctx_get(fd->inode, this, &iosstat);
+    if (!ret)
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_OPENDIR);
 
 unwind:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
+    UPDATE_PROFILE_STATS(frame, RMDIR);
 
-        UPDATE_PROFILE_STATS (frame, RMDIR);
-
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, TRUNCATE);
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, TRUNCATE);
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+io_stats_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                    dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, STATFS);
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, STATFS);
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SETXATTR);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETXATTR);
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, GETXATTR);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, GETXATTR);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, REMOVEXATTR);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, REMOVEXATTR);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSETXATTR);
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSETXATTR);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FGETXATTR);
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FGETXATTR);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FREMOVEXATTR);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FREMOVEXATTR);
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSYNCDIR);
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSYNCDIR);
+    STACK_UNWIND_STRICT(fsyncdir, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, ACCESS);
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, ACCESS);
+    STACK_UNWIND_STRICT(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,
-                        struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FTRUNCATE);
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FTRUNCATE);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+io_stats_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSTAT);
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSTAT);
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
 io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		       struct iatt *postbuf, dict_t *xdata)
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	UPDATE_PROFILE_STATS(frame, FALLOCATE);
-	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
-			    xdata);
-	return 0;
+    UPDATE_PROFILE_STATS(frame, FALLOCATE);
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
 io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		     struct iatt *postbuf, dict_t *xdata)
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	UPDATE_PROFILE_STATS(frame, DISCARD);
-	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
-			    xdata);
-	return 0;
+    UPDATE_PROFILE_STATS(frame, DISCARD);
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int
 io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS(frame, ZEROFILL);
-        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, ZEROFILL);
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
+}
+
+int32_t
+io_stats_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, IPC);
+    STACK_UNWIND_STRICT(ipc, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+io_stats_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, LK);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LK);
+    STACK_UNWIND_STRICT(lk, frame, op_ret, op_errno, lock, xdata);
+    return 0;
 }
 
+int
+io_stats_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, ENTRYLK);
+    STACK_UNWIND_STRICT(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
 int
-io_stats_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+io_stats_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, ENTRYLK);
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FENTRYLK);
+    STACK_UNWIND_STRICT(fentrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                       uint8_t *strong_checksum, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, XATTROP);
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, RCHECKSUM);
+    STACK_UNWIND_STRICT(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                        strong_checksum, xdata);
+    return 0;
 }
 
+int
+io_stats_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, SEEK);
+    STACK_UNWIND_STRICT(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
 
 int
-io_stats_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                   dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FXATTROP);
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LEASE);
+    STACK_UNWIND_STRICT(lease, frame, op_ret, op_errno, lease, xdata);
+    return 0;
 }
 
+int
+io_stats_getactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         lock_migration_info_t *locklist, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, GETACTIVELK);
+    STACK_UNWIND_STRICT(getactivelk, frame, op_ret, op_errno, locklist, xdata);
+    return 0;
+}
 
 int
-io_stats_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_setactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, INODELK);
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETACTIVELK);
+    STACK_UNWIND_STRICT(setactivelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_entrylk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+io_stats_compound_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, void *data,
+                      dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    UPDATE_PROFILE_STATS(frame, COMPOUND);
+    STACK_UNWIND_STRICT(compound, frame, op_ret, op_errno, data, xdata);
+    return 0;
+}
 
-        STACK_WIND (frame, io_stats_entrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
-        return 0;
+int
+io_stats_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, XATTROP);
+    STACK_UNWIND_STRICT(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
+int
+io_stats_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, FXATTROP);
+    STACK_UNWIND_STRICT(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
 
 int
-io_stats_inodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+io_stats_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+    UPDATE_PROFILE_STATS(frame, INODELK);
+    STACK_UNWIND_STRICT(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        START_FOP_LATENCY (frame);
+int
+io_stats_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_inodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
-
 int
-io_stats_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  fd_t *fd, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
 {
+    START_FOP_LATENCY(frame);
 
-        UPDATE_PROFILE_STATS (frame, FINODELK);
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
-
 int
-io_stats_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                   fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+io_stats_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_finodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
 }
 
+int
+io_stats_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, FINODELK);
+    STACK_UNWIND_STRICT(finodelk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
 int
-io_stats_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+io_stats_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_xattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
 }
 
-
 int
-io_stats_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+io_stats_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fxattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_lookup (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xdata)
+io_stats_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+io_stats_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readlink (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, size_t size, dict_t *xdata)
+io_stats_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+io_stats_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                  dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_mknod_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
 }
 
+int
+io_stats_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+               dev_t dev, mode_t umask, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
+}
 
 int
-io_stats_mkdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+io_stats_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+               mode_t umask, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_mkdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_unlink (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int xflag, dict_t *xdata)
+io_stats_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_unlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rmdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags, dict_t *xdata)
+io_stats_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_rmdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                  loc_t *loc, mode_t umask, dict_t *xdata)
+io_stats_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                 loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_symlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rename (call_frame_t *frame, xlator_t *this,
-                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+io_stats_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_link (call_frame_t *frame, xlator_t *this,
-               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+io_stats_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_link_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+io_stats_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-io_stats_truncate (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, off_t offset, dict_t *xdata)
+io_stats_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                  dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_truncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, fd_t *fd, dict_t *xdata)
+io_stats_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_open_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_create (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t flags, mode_t mode,
-                 mode_t umask, fd_t *fd, dict_t *xdata)
+io_stats_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
-
 int
-io_stats_readv (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+io_stats_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
-        frame->local = fd;
+    frame->local = fd;
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readv_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_writev (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, struct iovec *vector,
-                 int32_t count, off_t offset,
-                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
+io_stats_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iovec *vector, int32_t count, off_t offset,
+                uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        int                 len = 0;
+    int len = 0;
 
-        if (fd->inode)
-                frame->local = fd->inode;
-        len = iov_length (vector, count);
+    if (fd->inode)
+        frame->local = fd->inode;
+    len = iov_length(vector, count);
 
-        BUMP_WRITE (fd, len);
-        START_FOP_LATENCY (frame);
-
-        STACK_WIND (frame, io_stats_writev_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+    ios_bump_write(this, fd, len);
+    START_FOP_LATENCY(frame);
 
+    STACK_WIND(frame, io_stats_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
-
 int
-io_stats_statfs (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xdata)
+io_stats_copy_file_range(call_frame_t *frame, xlator_t *this, fd_t *fd_in,
+                         off_t off_in, fd_t *fd_out, off_t off_out, size_t len,
+                         uint32_t flags, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_statfs_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_copy_file_range_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->copy_file_range, fd_in, off_in, fd_out,
+               off_out, len, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_flush (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, dict_t *xdata)
+io_stats_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_flush_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsync (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t flags, dict_t *xdata)
+io_stats_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsync_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
-
 int
-conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
+io_stats_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata)
 {
-        struct {
-                xlator_t       *this;
-                inode_t        *inode;
-                const char     *path;
-        } *stub;
-        xlator_t             *this = NULL;
-        char                 *filename = NULL;
-        FILE                 *logfp = NULL;
-        struct ios_dump_args args = {0};
-        int                   pid;
-        char                  dump_key[100];
+    START_FOP_LATENCY(frame);
 
-        stub  = data;
-        this  = stub->this;
-
-        filename = alloca (value->len + 1);
-        memset (filename, 0, value->len + 1);
-        memcpy (filename, data_to_str (value), value->len);
-
-        pid = getpid ();
+    STACK_WIND(frame, io_stats_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
 
-        if (!strncmp (filename, "", 1)) {
-                gf_log (this->name, GF_LOG_ERROR, "No filename given");
-                return -1;
-        }
-        logfp = fopen (filename, "w+");
-        if (!logfp) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
-                                "for writing", filename);
-                return -1;
-        }
-        sprintf (dump_key, "*io*stat*%d_json_dump", pid);
-        if (fnmatch (dump_key, key, 0) == 0) {
-                (void) ios_dump_args_init (
-                                &args, IOS_DUMP_TYPE_JSON_FILE,
-                                logfp);
-        } else {
-                (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
-                                logfp);
-        }
-        io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
-        fclose (logfp);
-        return 0;
+int
+conditional_dump(dict_t *dict, char *key, data_t *value, void *data)
+{
+    struct {
+        xlator_t *this;
+        inode_t *inode;
+        const char *path;
+    } * stub;
+    xlator_t *this = NULL;
+    char *filename = NULL;
+    FILE *logfp = NULL;
+    struct ios_dump_args args = {0};
+    int pid, namelen, dirlen;
+    char dump_key[100];
+    char *slash_ptr = NULL;
+    char *path_in_value = NULL;
+    char *identifier = NULL;
+    struct ios_conf *conf = NULL;
+
+    stub = data;
+    this = stub->this;
+    conf = this->private;
+
+    /* Don't do this on 'brick-side', only do this on client side */
+    /* Addresses CVE-2018-14659 */
+    if (this->ctx->process_mode != GF_CLIENT_PROCESS) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "taking io-stats dump using setxattr not permitted on brick."
+               " Use 'gluster profile' instead");
+        return -1;
+    }
+
+    /* Create a file name that is appended with the io-stats instance
+    name as well. This helps when there is more than a single io-stats
+    instance in the graph, or the client and server processes are running
+    on the same node */
+    /* For the sanity of where the file should be located, we should make
+       sure file is written only inside RUNDIR (ie, /var/run/gluster) */
+    /* TODO: provide an option to dump it to different directory of
+       choice, based on options */
+    /* name format: /var/run/gluster/<passed in path/filename>.<xlator name
+     * slashes to -> */
+
+    path_in_value = alloca0(value->len + 1);
+
+    /* We need a memcpy here because of the way dict_unserialize works */
+
+    memcpy(path_in_value, data_to_str(value), value->len);
+    path_in_value[value->len] = '\0';
+
+    if (strstr(path_in_value, "../")) {
+        gf_log(this->name, GF_LOG_ERROR, "%s: no \"../\" allowed in path",
+               path_in_value);
+        return -1;
+    }
+
+    if (path_in_value[0] == '/') {
+        path_in_value = path_in_value + 1;
+    }
+
+    dirlen = strlen(IOS_STATS_DUMP_DIR);
+    if (conf->unique_id) {
+        /* this->name will be the same for all bricks of the volume */
+        identifier = conf->unique_id;
+    } else {
+        identifier = this->name;
+    }
+
+    namelen = (dirlen + value->len + strlen(identifier) + 3);
+    /* +3 for '/', '.' and '\0' added in snprintf below*/
+
+    filename = alloca0(namelen);
+    snprintf(filename, namelen, "%s/%s.%s", IOS_STATS_DUMP_DIR, path_in_value,
+             identifier);
+
+    /* convert any slashes to '-' so that fopen works correctly */
+    slash_ptr = strchr(filename + dirlen + 1, '/');
+    while (slash_ptr) {
+        *slash_ptr = '-';
+        slash_ptr = strchr(slash_ptr, '/');
+    }
+
+    pid = getpid();
+
+    if (!strncmp(filename, "", 1)) {
+        gf_log(this->name, GF_LOG_ERROR, "No filename given");
+        return -1;
+    }
+    logfp = fopen(filename, "w+");
+    if (!logfp) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to open %s "
+               "for writing",
+               filename);
+        return -1;
+    }
+    sprintf(dump_key, "*io*stat*%d_json_dump", pid);
+    if (fnmatch(dump_key, key, 0) == 0) {
+        (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_JSON_FILE, logfp);
+    } else {
+        (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_FILE, logfp);
+    }
+    io_stats_dump(this, &args, GF_IOS_INFO_ALL, _gf_false);
+    fclose(logfp);
+    return 0;
 }
 
 int
-_ios_destroy_dump_thread (struct ios_conf *conf) {
-        conf->dump_thread_should_die = _gf_true;
-        if (conf->ios_dump_interval > 0) {
-                (void) pthread_cancel (conf->dump_thread);
-                (void) pthread_join (conf->dump_thread, NULL);
-        }
-        return 0;
+_ios_destroy_dump_thread(struct ios_conf *conf)
+{
+    conf->dump_thread_should_die = _gf_true;
+    if (conf->dump_thread_running) {
+        (void)pthread_cancel(conf->dump_thread);
+        (void)pthread_join(conf->dump_thread, NULL);
+    }
+    return 0;
 }
 
 void *
-_ios_dump_thread (xlator_t *this) {
-        struct ios_conf         *conf = NULL;
-        FILE                    *stats_logfp = NULL;
-        FILE                    *samples_logfp = NULL;
-        struct ios_dump_args args = {0};
-        int                     i;
-        int                     stats_bytes_written = 0;
-        int                     samples_bytes_written = 0;
-        char                    stats_filename[PATH_MAX];
-        char                    samples_filename[PATH_MAX];
-        char                    *xlator_name;
-        char                    *instance_name;
-        gf_boolean_t            log_stats_fopen_failure = _gf_true;
-        gf_boolean_t            log_samples_fopen_failure = _gf_true;
-        int                     old_cancel_type;
-
-        conf = this->private;
-        gf_log (this->name, GF_LOG_INFO, "IO stats dump thread started, "
-                "polling IO stats every %d seconds", conf->ios_dump_interval);
-        xlator_name = strdupa (this->name);
-        for (i = 0; i < strlen (xlator_name); i++) {
-                if (xlator_name[i] == '/')
-                        xlator_name[i] = '_';
+_ios_dump_thread(xlator_t *this)
+{
+    struct ios_conf *conf = NULL;
+    FILE *stats_logfp = NULL;
+    FILE *samples_logfp = NULL;
+    struct ios_dump_args args = {0};
+    int i;
+    int stats_bytes_written = 0;
+    int samples_bytes_written = 0;
+    char stats_filename[PATH_MAX];
+    char samples_filename[PATH_MAX];
+    char *xlator_name;
+    char *instance_name;
+    gf_boolean_t log_stats_fopen_failure = _gf_true;
+    gf_boolean_t log_samples_fopen_failure = _gf_true;
+    int old_cancel_type;
+
+    conf = this->private;
+    gf_log(this->name, GF_LOG_INFO,
+           "IO stats dump thread started, "
+           "polling IO stats every %d seconds",
+           conf->ios_dump_interval);
+    xlator_name = strdupa(conf->unique_id);
+    for (i = 0; i < strlen(xlator_name); i++) {
+        if (xlator_name[i] == '/')
+            xlator_name[i] = '_';
+    }
+    instance_name = this->instance_name;
+    if (this->name && strcmp(this->name, "glustershd") == 0) {
+        xlator_name = "shd";
+    } else if (this->prev && strcmp(this->prev->name, "nfs-server") == 0) {
+        xlator_name = "nfsd";
+        instance_name = this->prev->instance_name;
+    }
+    if (sys_mkdir(_IOS_DUMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+        if (errno != EEXIST) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not create stats-dump directory %s", _IOS_DUMP_DIR);
+            goto out;
         }
-        instance_name = this->instance_name;
-        if (this->name && strcmp (this->name, "glustershd") == 0) {
-                xlator_name = "shd";
-        } else if (this->prev &&
-                   strcmp (this->prev->name, "nfs-server") == 0) {
-                xlator_name = "nfsd";
-                instance_name = this->prev->instance_name;
+    }
+    if (sys_mkdir(_IOS_SAMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+        if (errno != EEXIST) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not create stats-sample directory %s", _IOS_SAMP_DIR);
+            goto out;
         }
-        if (sys_mkdir (_IOS_DUMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
-                if (errno != EEXIST) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not create stats-dump directory %s",
-                                _IOS_DUMP_DIR);
-                        goto out;
-                }
-        }
-        if (sys_mkdir (_IOS_SAMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
-                if (errno != EEXIST) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not create stats-sample directory %s",
-                                _IOS_SAMP_DIR);
-                        goto out;
-                }
-        }
-        if (instance_name) {
-                stats_bytes_written = snprintf (stats_filename, PATH_MAX,
-                        "%s/%s_%s_%s.dump", _IOS_DUMP_DIR,
-                        __progname, xlator_name, instance_name);
-                samples_bytes_written = snprintf (samples_filename, PATH_MAX,
-                        "%s/%s_%s_%s.samp", _IOS_SAMP_DIR,
-                        __progname, xlator_name, instance_name);
-        } else {
-                stats_bytes_written = snprintf (stats_filename, PATH_MAX,
-                        "%s/%s_%s.dump", _IOS_DUMP_DIR, __progname,
-                        xlator_name);
-                samples_bytes_written = snprintf (samples_filename, PATH_MAX,
-                        "%s/%s_%s.samp", _IOS_SAMP_DIR, __progname,
-                        xlator_name);
+    }
+    if (instance_name) {
+        stats_bytes_written = snprintf(stats_filename, PATH_MAX,
+                                       "%s/%s_%s_%s.dump", _IOS_DUMP_DIR,
+                                       __progname, xlator_name, instance_name);
+        samples_bytes_written = snprintf(
+            samples_filename, PATH_MAX, "%s/%s_%s_%s.samp", _IOS_SAMP_DIR,
+            __progname, xlator_name, instance_name);
+    } else {
+        stats_bytes_written = snprintf(stats_filename, PATH_MAX,
+                                       "%s/%s_%s.dump", _IOS_DUMP_DIR,
+                                       __progname, xlator_name);
+        samples_bytes_written = snprintf(samples_filename, PATH_MAX,
+                                         "%s/%s_%s.samp", _IOS_SAMP_DIR,
+                                         __progname, xlator_name);
+    }
+    if ((stats_bytes_written >= PATH_MAX) ||
+        (samples_bytes_written >= PATH_MAX)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Invalid path for stats dump (%s) and/or latency "
+               "samples (%s)",
+               stats_filename, samples_filename);
+        goto out;
+    }
+    while (1) {
+        if (conf->dump_thread_should_die)
+            break;
+        (void)pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS,
+                                    &old_cancel_type);
+        sleep(conf->ios_dump_interval);
+        (void)pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, &old_cancel_type);
+        /*
+         * It's not clear whether we should reopen this each time, or
+         * just hold it open and rewind/truncate on each iteration.
+         * Leaving it alone for now.
+         */
+        stats_logfp = fopen(stats_filename, "w+");
+        if (stats_logfp) {
+            (void)ios_dump_args_init(&args, conf->dump_format, stats_logfp);
+            io_stats_dump(this, &args, GF_IOS_INFO_ALL, _gf_false);
+            fclose(stats_logfp);
+            log_stats_fopen_failure = _gf_true;
+        } else if (log_stats_fopen_failure) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not open stats-dump file %s (%s)", stats_filename,
+                   strerror(errno));
+            log_stats_fopen_failure = _gf_false;
         }
-        if ((stats_bytes_written >= PATH_MAX) ||
-            (samples_bytes_written >= PATH_MAX)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid path for stats dump (%s) and/or latency "
-                        "samples (%s)", stats_filename, samples_filename);
-                goto out;
-        }
-        while (1) {
-                if (conf->dump_thread_should_die)
-                        break;
-                (void) pthread_setcanceltype (PTHREAD_CANCEL_ASYNCHRONOUS,
-                                              &old_cancel_type);
-                sleep (conf->ios_dump_interval);
-                (void) pthread_setcanceltype (PTHREAD_CANCEL_DEFERRED,
-                                              &old_cancel_type);
-                /*
-                 * It's not clear whether we should reopen this each time, or
-                 * just hold it open and rewind/truncate on each iteration.
-                 * Leaving it alone for now.
-                 */
-                stats_logfp = fopen (stats_filename, "w+");
-                if (stats_logfp) {
-                        (void) ios_dump_args_init (&args,
-                                                   IOS_DUMP_TYPE_JSON_FILE,
-                                                   stats_logfp);
-                        io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
-                        fclose (stats_logfp);
-                        log_stats_fopen_failure = _gf_true;
-                } else if (log_stats_fopen_failure) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not open stats-dump file %s (%s)",
-                                stats_filename, strerror(errno));
-                        log_stats_fopen_failure = _gf_false;
-                }
-                samples_logfp = fopen (samples_filename, "w+");
-                if (samples_logfp) {
-                        io_stats_dump_latency_samples_logfp (this,
-                                                             samples_logfp);
-                        fclose (samples_logfp);
-                        log_samples_fopen_failure = _gf_true;
-                } else if (log_samples_fopen_failure) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not open samples-dump file %s (%s)",
-                                samples_filename, strerror(errno));
-                        log_samples_fopen_failure = _gf_false;
-                }
+        samples_logfp = fopen(samples_filename, "w+");
+        if (samples_logfp) {
+            io_stats_dump_latency_samples_logfp(this, samples_logfp);
+            fclose(samples_logfp);
+            log_samples_fopen_failure = _gf_true;
+        } else if (log_samples_fopen_failure) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not open samples-dump file %s (%s)", samples_filename,
+                   strerror(errno));
+            log_samples_fopen_failure = _gf_false;
         }
+    }
 out:
-        gf_log (this->name, GF_LOG_INFO, "IO stats dump thread terminated");
-        return NULL;
+    conf->dump_thread_running = _gf_false;
+    gf_log(this->name, GF_LOG_INFO, "IO stats dump thread terminated");
+    return NULL;
 }
 
 static gf_boolean_t
-match_special_xattr (dict_t *d, char *k, data_t *val, void *mdata)
+match_special_xattr(dict_t *d, char *k, data_t *val, void *mdata)
 {
-        gf_boolean_t ret = _gf_false;
-        if (fnmatch ("*io*stat*dump", k, 0) == 0) {
-                ret = _gf_true;
-        }
+    gf_boolean_t ret = _gf_false;
+    if (fnmatch("*io*stat*dump", k, 0) == 0) {
+        ret = _gf_true;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-io_stats_setxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+io_stats_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
 {
-        int ret = 0;
-        struct {
-                xlator_t     *this;
-                inode_t      *inode;
-                const char   *path;
-        } stub;
-
-        stub.this  = this;
-        stub.inode = loc->inode;
-        stub.path  = loc->path;
-
-        ret = dict_foreach_match (dict, match_special_xattr, NULL,
-                                  conditional_dump, &stub);
-        if (ret > 0) {
-                /* Setxattr was on key 'io-stat-dump', hence dump and unwind
-                 * from here */
-                goto out;
-        }
+    struct {
+        xlator_t *this;
+        inode_t *inode;
+        const char *path;
+    } stub;
 
-        START_FOP_LATENCY (frame);
+    stub.this = this;
+    stub.inode = loc->inode;
+    stub.path = loc->path;
 
-        STACK_WIND (frame, io_stats_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+    (void)dict_foreach_match(dict, match_special_xattr, NULL, conditional_dump,
+                             &stub);
 
-out:
-        STACK_UNWIND_STRICT (setxattr, frame, 0, 0, NULL);
-        return 0;
-}
+    START_FOP_LATENCY(frame);
 
+    STACK_WIND(frame, io_stats_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+}
 
 int
-io_stats_getxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+io_stats_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_removexattr (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, const char *name, dict_t *xdata)
+io_stats_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, dict_t *dict,
-                    int32_t flags, dict_t *xdata)
+io_stats_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                   int32_t flags, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fgetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name, dict_t *xdata)
+io_stats_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fgetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fremovexattr (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, const char *name, dict_t *xdata)
+io_stats_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_opendir (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, fd_t *fd, dict_t *xdata)
+io_stats_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                 dict_t *xdata)
 {
+    START_FOP_LATENCY(frame);
 
-        START_FOP_LATENCY (frame);
-
-        STACK_WIND (frame, io_stats_opendir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-io_stats_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                   off_t offset, dict_t *dict)
+io_stats_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t offset, dict_t *dict)
 {
-        frame->local = fd->inode;
-        START_FOP_LATENCY (frame);
+    frame->local = fd->inode;
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
-        return 0;
+    STACK_WIND(frame, io_stats_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    return 0;
 }
 
-
 int
-io_stats_readdir (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset, dict_t *xdata)
+io_stats_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsyncdir (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, int32_t datasync, dict_t *xdata)
+io_stats_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  int32_t datasync, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsyncdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
 }
 
-
 int
-io_stats_access (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t mask, dict_t *xdata)
+io_stats_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+                dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_access_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access,
-                    loc, mask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_ftruncate (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, off_t offset, dict_t *xdata)
+io_stats_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                   dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_ftruncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsetattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+io_stats_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fstat (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, dict_t *xdata)
+io_stats_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fstat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
-
 int
 io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		   off_t offset, size_t len, dict_t *xdata)
+                   off_t offset, size_t len, dict_t *xdata)
 {
-	START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		   xdata);
+    STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
 io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-		 size_t len, dict_t *xdata)
+                 size_t len, dict_t *xdata)
 {
-	START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
 io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 off_t len, dict_t *xdata)
+                  off_t len, dict_t *xdata)
 {
-        START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+io_stats_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
 
 int
-io_stats_lk (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+io_stats_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+            struct gf_flock *lock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_lk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
+    return 0;
 }
 
+int
+io_stats_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                   int32_t len, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+    return 0;
+}
 
 int
-io_stats_release (xlator_t *this, fd_t *fd)
+io_stats_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              gf_seek_what_t what, dict_t *xdata)
 {
-        struct ios_fd  *iosfd = NULL;
-        struct ios_conf *conf = NULL;
+    START_FOP_LATENCY(frame);
 
-        BUMP_FOP (RELEASE);
+    STACK_WIND(frame, io_stats_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+    return 0;
+}
 
-        conf = this->private;
+int
+io_stats_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct gf_lease *lease, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
 
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens--;
-        }
-        UNLOCK (&conf->lock);
+    STACK_WIND(frame, io_stats_lease_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lease, loc, lease, xdata);
+    return 0;
+}
 
-        ios_fd_ctx_get (fd, this, &iosfd);
-        if (iosfd) {
-                io_stats_dump_fd (this, iosfd);
+int
+io_stats_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
 
-                GF_FREE (iosfd->filename);
-                GF_FREE (iosfd);
-        }
+    STACK_WIND(frame, io_stats_getactivelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getactivelk, loc, xdata);
+    return 0;
+}
 
-        return 0;
+int
+io_stats_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     lock_migration_info_t *locklist, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_setactivelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setactivelk, loc, locklist, xdata);
+    return 0;
 }
 
+int
+io_stats_compound(call_frame_t *frame, xlator_t *this, void *args,
+                  dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_compound_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->compound, args, xdata);
+    return 0;
+}
 
 int
-io_stats_releasedir (xlator_t *this, fd_t *fd)
+io_stats_release(xlator_t *this, fd_t *fd)
 {
-        BUMP_FOP (RELEASEDIR);
+    struct ios_fd *iosfd = NULL;
+    struct ios_conf *conf = NULL;
 
-        return 0;
+    BUMP_FOP(RELEASE);
+
+    conf = this->private;
+    if (conf) {
+        LOCK(&conf->lock);
+        {
+            conf->cumulative.nr_opens--;
+        }
+        UNLOCK(&conf->lock);
+    }
+
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (iosfd) {
+        io_stats_dump_fd(this, iosfd);
+
+        GF_FREE(iosfd->filename);
+        GF_FREE(iosfd);
+    }
+
+    return 0;
 }
 
+int
+io_stats_releasedir(xlator_t *this, fd_t *fd)
+{
+    BUMP_FOP(RELEASEDIR);
+
+    return 0;
+}
 
 int
-io_stats_forget (xlator_t *this, inode_t *inode)
+io_stats_forget(xlator_t *this, inode_t *inode)
 {
-        BUMP_FOP (FORGET);
-        ios_stats_cleanup (this, inode);
-        return 0;
+    BUMP_FOP(FORGET);
+    ios_stats_cleanup(this, inode);
+    return 0;
 }
 
 static int
-ios_init_top_stats (struct ios_conf *conf)
+ios_init_top_stats(struct ios_conf *conf)
 {
-        int     i = 0;
+    int i = 0;
 
-        GF_ASSERT (conf);
+    GF_ASSERT(conf);
 
-        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
-                conf->list[i].iosstats = GF_CALLOC (1,
-                                         sizeof(*conf->list[i].iosstats),
-                                         gf_io_stats_mt_ios_stat);
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+        conf->list[i].iosstats = GF_CALLOC(1, sizeof(*conf->list[i].iosstats),
+                                           gf_io_stats_mt_ios_stat);
 
-                if (!conf->list[i].iosstats)
-                        return -1;
+        if (!conf->list[i].iosstats)
+            return -1;
 
-                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
-                LOCK_INIT (&conf->list[i].lock);
-        }
+        INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+        LOCK_INIT(&conf->list[i].lock);
+    }
 
-        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
-                conf->thru_list[i].iosstats = GF_CALLOC (1,
-                                 sizeof (*conf->thru_list[i].iosstats),
-                                 gf_io_stats_mt_ios_stat);
+    for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+        conf->thru_list[i].iosstats = GF_CALLOC(
+            1, sizeof(*conf->thru_list[i].iosstats), gf_io_stats_mt_ios_stat);
 
-                if (!conf->thru_list[i].iosstats)
-                        return -1;
+        if (!conf->thru_list[i].iosstats)
+            return -1;
 
-                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
-                LOCK_INIT (&conf->thru_list[i].lock);
-        }
+        INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+        LOCK_INIT(&conf->thru_list[i].lock);
+    }
 
-        return 0;
+    return 0;
 }
 
 static void
-ios_destroy_top_stats (struct ios_conf *conf)
-{
-        int                     i = 0;
-        struct ios_stat_head    *list_head = NULL;
-        struct ios_stat_list    *entry     = NULL;
-        struct ios_stat_list    *tmp       = NULL;
-        struct ios_stat_list    *list      = NULL;
-        struct ios_stat         *stat      = NULL;
-
-        GF_ASSERT (conf);
-
-        LOCK (&conf->lock);
-
-        conf->cumulative.nr_opens = 0;
-        conf->cumulative.max_nr_opens = 0;
-        conf->cumulative.max_openfd_time.tv_sec = 0;
-        conf->cumulative.max_openfd_time.tv_usec = 0;
-
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
-                list_head = &conf->list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        GF_FREE (list);
-                        list_head->members--;
-                }
-        }
+ios_destroy_top_stats(struct ios_conf *conf)
+{
+    int i = 0;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *tmp = NULL;
+    struct ios_stat_list *list = NULL;
+    struct ios_stat *stat = NULL;
 
-        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
-                list_head = &conf->thru_list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        GF_FREE (list);
-                        list_head->members--;
-                }
-        }
+    GF_ASSERT(conf);
 
-        UNLOCK (&conf->lock);
+    LOCK(&conf->lock);
 
-        return;
-}
+    conf->cumulative.nr_opens = 0;
+    conf->cumulative.max_nr_opens = 0;
+    conf->cumulative.max_openfd_time.tv_sec = 0;
+    conf->cumulative.max_openfd_time.tv_usec = 0;
 
-static int
-io_stats_clear (struct ios_conf *conf)
-{
-        struct timeval      now;
-        int                 ret = -1;
-
-        GF_ASSERT (conf);
-
-        if (!gettimeofday (&now, NULL))
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+        list_head = &conf->list[i];
+        if (!list_head)
+            continue;
+        list_for_each_entry_safe(entry, tmp, &list_head->iosstats->list, list)
         {
-            LOCK (&conf->lock);
-            {
-                    ios_global_stats_clear (&conf->cumulative, &now);
-                    ios_global_stats_clear (&conf->incremental, &now);
-                    conf->increment = 0;
-            }
-            UNLOCK (&conf->lock);
-            ret = 0;
+            list = entry;
+            stat = list->iosstat;
+            ios_stat_unref(stat);
+            list_del(&list->list);
+            GF_FREE(list);
+            list_head->members--;
         }
+        GF_FREE(list_head->iosstats);
+    }
+
+    for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+        list_head = &conf->thru_list[i];
+        if (!list_head)
+            continue;
+        list_for_each_entry_safe(entry, tmp, &list_head->iosstats->list, list)
+        {
+            list = entry;
+            stat = list->iosstat;
+            ios_stat_unref(stat);
+            list_del(&list->list);
+            GF_FREE(list);
+            list_head->members--;
+        }
+        GF_FREE(list_head->iosstats);
+    }
 
-        return ret;
+    UNLOCK(&conf->lock);
+
+    return;
 }
 
-int32_t
-io_priv (xlator_t *this)
-{
-        int                 i;
-        char                key[GF_DUMP_MAX_BUF_LEN];
-        char                key_prefix_cumulative[GF_DUMP_MAX_BUF_LEN];
-        char                key_prefix_incremental[GF_DUMP_MAX_BUF_LEN];
-        double              min, max, avg;
-        uint64_t            count, total;
-        struct ios_conf    *conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                return -1;
-
-        if(!conf->count_fop_hits || !conf->measure_latency)
-                return -1;
-
-        gf_proc_dump_write("cumulative.data_read", "%"PRIu64,
-                                                conf->cumulative.data_read);
-        gf_proc_dump_write("cumulative.data_written", "%"PRIu64,
-                                                conf->cumulative.data_written);
-
-        gf_proc_dump_write("incremental.data_read", "%"PRIu64,
-                                                conf->incremental.data_read);
-        gf_proc_dump_write("incremental.data_written", "%"PRIu64,
-                                                conf->incremental.data_written);
-
-        snprintf (key_prefix_cumulative, GF_DUMP_MAX_BUF_LEN, "%s.cumulative",
-                                                                    this->name);
-        snprintf (key_prefix_incremental, GF_DUMP_MAX_BUF_LEN, "%s.incremental",
-                                                                    this->name);
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                count = conf->cumulative.fop_hits[i];
-                total = conf->cumulative.latency[i].total;
-                min = conf->cumulative.latency[i].min;
-                max = conf->cumulative.latency[i].max;
-                avg = conf->cumulative.latency[i].avg;
-
-                gf_proc_dump_build_key (key, key_prefix_cumulative,
-                                        (char *)gf_fop_list[i]);
-
-                gf_proc_dump_write (key,"%"PRId64",%"PRId64",%.03f,%.03f,%.03f",
-                                    count, total, min, max, avg);
-
-                count = conf->incremental.fop_hits[i];
-                total = conf->incremental.latency[i].total;
-                min = conf->incremental.latency[i].min;
-                max = conf->incremental.latency[i].max;
-                avg = conf->incremental.latency[i].avg;
-
-                gf_proc_dump_build_key (key, key_prefix_incremental,
-                                        (char *)gf_fop_list[i]);
-
-                gf_proc_dump_write (key,"%"PRId64",%"PRId64",%.03f,%.03f,%.03f",
-                                    count, total, min, max, avg);
+static void
+io_stats_clear(struct ios_conf *conf)
+{
+    time_t now = 0;
 
-        }
+    GF_ASSERT(conf);
+    now = gf_time();
 
-        return 0;
+    LOCK(&conf->lock);
+    {
+        ios_global_stats_clear(&conf->cumulative, now);
+        ios_global_stats_clear(&conf->incremental, now);
+        conf->increment = 0;
+    }
+    UNLOCK(&conf->lock);
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
+int32_t
+io_priv(xlator_t *this)
 {
-        struct ios_conf    *conf = NULL;
-        int                 ret = -1;
-        char               *sys_log_str = NULL;
-        char               *log_format_str = NULL;
-        char               *logger_str    = NULL;
-        int                 sys_log_level = -1;
-        char               *log_str = NULL;
-        int                 log_level = -1;
-        int                 log_format = -1;
-        int                 logger = -1;
-        uint32_t            log_buf_size = 0;
-        uint32_t            log_flush_timeout = 0;
-        int32_t             old_dump_interval;
+    int i;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    char key_prefix_cumulative[GF_DUMP_MAX_BUF_LEN];
+    char key_prefix_incremental[GF_DUMP_MAX_BUF_LEN];
+    double min, max, avg;
+    uint64_t count, total;
+    struct ios_conf *conf = NULL;
 
-        if (!this || !this->private)
-                goto out;
-
-        conf = this->private;
+    conf = this->private;
+    if (!conf)
+        return -1;
 
-        GF_OPTION_RECONF ("dump-fd-stats", conf->dump_fd_stats, options, bool,
-                          out);
+    if (!conf->count_fop_hits || !conf->measure_latency)
+        return -1;
 
-        GF_OPTION_RECONF ("count-fop-hits", conf->count_fop_hits, options, bool,
-                          out);
+    gf_proc_dump_write("cumulative.data_read", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->cumulative.data_read));
+    gf_proc_dump_write("cumulative.data_written", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->cumulative.data_written));
 
-        GF_OPTION_RECONF ("latency-measurement", conf->measure_latency,
-                          options, bool, out);
+    gf_proc_dump_write("incremental.data_read", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->incremental.data_read));
+    gf_proc_dump_write("incremental.data_written", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->incremental.data_written));
 
-        old_dump_interval = conf->ios_dump_interval;
-        GF_OPTION_RECONF ("ios-dump-interval", conf->ios_dump_interval, options,
-                         int32, out);
-        if ((old_dump_interval <= 0) && (conf->ios_dump_interval > 0)) {
-                pthread_create (&conf->dump_thread, NULL,
-                                (void *) &_ios_dump_thread, this);
-        }
+    snprintf(key_prefix_cumulative, GF_DUMP_MAX_BUF_LEN, "%s.cumulative",
+             this->name);
+    snprintf(key_prefix_incremental, GF_DUMP_MAX_BUF_LEN, "%s.incremental",
+             this->name);
 
-        GF_OPTION_RECONF ("ios-sample-interval", conf->ios_sample_interval,
-                         options, int32, out);
-        GF_OPTION_RECONF ("ios-sample-buf-size", conf->ios_sample_buf_size,
-                         options, int32, out);
-        GF_OPTION_RECONF ("sys-log-level", sys_log_str, options, str, out);
-        if (sys_log_str) {
-                sys_log_level = glusterd_check_log_level (sys_log_str);
-                set_sys_log_level (sys_log_level);
-        }
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        count = GF_ATOMIC_GET(conf->cumulative.fop_hits[i]);
+        total = conf->cumulative.latency[i].total;
+        min = conf->cumulative.latency[i].min;
+        max = conf->cumulative.latency[i].max;
+        avg = conf->cumulative.latency[i].avg;
 
-        GF_OPTION_RECONF ("log-level", log_str, options, str, out);
-        if (log_str) {
-                log_level = glusterd_check_log_level (log_str);
-                gf_log_set_loglevel (log_level);
-        }
+        gf_proc_dump_build_key(key, key_prefix_cumulative, "%s",
+                               (char *)gf_fop_list[i]);
 
-        GF_OPTION_RECONF ("logger", logger_str, options, str, out);
-        if (logger_str) {
-                logger = gf_check_logger (logger_str);
-                gf_log_set_logger (logger);
-        }
+        gf_proc_dump_write(key, "%" PRId64 ",%" PRId64 ",%.03f,%.03f,%.03f",
+                           count, total, min, max, avg);
 
-        GF_OPTION_RECONF ("log-format", log_format_str, options, str, out);
-        if (log_format_str) {
-                log_format = gf_check_log_format (log_format_str);
-                gf_log_set_logformat (log_format);
-        }
+        count = GF_ATOMIC_GET(conf->incremental.fop_hits[i]);
+        total = conf->incremental.latency[i].total;
+        min = conf->incremental.latency[i].min;
+        max = conf->incremental.latency[i].max;
+        avg = conf->incremental.latency[i].avg;
 
-        GF_OPTION_RECONF ("log-buf-size", log_buf_size, options, uint32, out);
-        gf_log_set_log_buf_size (log_buf_size);
+        gf_proc_dump_build_key(key, key_prefix_incremental, "%s",
+                               (char *)gf_fop_list[i]);
 
-        GF_OPTION_RECONF ("log-flush-timeout", log_flush_timeout, options,
-                          time, out);
-        gf_log_set_log_flush_timeout (log_flush_timeout);
+        gf_proc_dump_write(key, "%" PRId64 ",%" PRId64 ",%.03f,%.03f,%.03f",
+                           count, total, min, max, avg);
+    }
 
-        ret = 0;
-out:
-        gf_log (this ? this->name : "io-stats",
-                        GF_LOG_DEBUG, "reconfigure returning %d", ret);
-        return ret;
+    return 0;
 }
 
-
-int32_t
-mem_acct_init (xlator_t *this)
+static void
+ios_set_log_format_code(struct ios_conf *conf, char *dump_format_str)
 {
-        int     ret = -1;
+    if (strcmp(dump_format_str, "json") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_JSON_FILE;
+    else if (strcmp(dump_format_str, "text") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_FILE;
+    else if (strcmp(dump_format_str, "dict") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_DICT;
+    else if (strcmp(dump_format_str, "samples") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_SAMPLES;
+}
 
-        if (!this)
-                return ret;
+void
+xlator_set_loglevel(xlator_t *this, int log_level)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *top = NULL;
+    xlator_t *trav = this;
 
-        ret = xlator_mem_acct_init (this, gf_io_stats_mt_end + 1);
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    top = active->first;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    if (log_level == -1)
+        return;
 
-        return ret;
-}
+    if (ctx->cmd_args.brick_mux) {
+        /* Set log-level for all brick xlators */
+        top->loglevel = log_level;
 
-void
-ios_conf_destroy (struct ios_conf *conf)
-{
-        if (!conf)
-                return;
+        /* Set log-level for parent xlator */
+        if (this->parents)
+            this->parents->xlator->loglevel = log_level;
 
-        ios_destroy_top_stats (conf);
-        _ios_destroy_dump_thread (conf);
-        LOCK_DESTROY (&conf->lock);
-        GF_FREE(conf);
+        while (trav) {
+            trav->loglevel = log_level;
+            trav = trav->next;
+        }
+    } else {
+        gf_log_set_loglevel(this->ctx, log_level);
+    }
 }
 
 int
-init (xlator_t *this)
-{
-        struct ios_conf    *conf = NULL;
-        char               *sys_log_str = NULL;
-        char               *logger_str  = NULL;
-        char               *log_format_str  = NULL;
-        int                 logger = -1;
-        int                 log_format = -1;
-        int                 sys_log_level = -1;
-        char               *log_str = NULL;
-        int                 log_level = -1;
-        int                 ret = -1;
-        uint32_t            log_buf_size = 0;
-        uint32_t            log_flush_timeout = 0;
+reconfigure(xlator_t *this, dict_t *options)
+{
+    struct ios_conf *conf = NULL;
+    int ret = -1;
+    char *sys_log_str = NULL;
+    char *log_format_str = NULL;
+    char *logger_str = NULL;
+    char *dump_format_str = NULL;
+    int sys_log_level = -1;
+    char *log_str = NULL;
+    int log_level = -1;
+    int log_format = -1;
+    int logger = -1;
+    uint32_t log_buf_size = 0;
+    uint32_t log_flush_timeout = 0;
+    int32_t old_dump_interval;
+    int32_t threads;
+
+    if (!this || !this->private)
+        goto out;
 
-        if (!this)
-                return -1;
+    conf = this->private;
 
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "io_stats translator requires atleast one subvolume");
-                return -1;
-        }
+    GF_OPTION_RECONF("dump-fd-stats", conf->dump_fd_stats, options, bool, out);
 
-        if (!this->parents) {
-                /* This is very much valid as io-stats currently is loaded
-                 * on top of volumes on both client and server, hence this is
-                 * not an warning message */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
+    GF_OPTION_RECONF("count-fop-hits", conf->count_fop_hits, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("latency-measurement", conf->measure_latency, options,
+                     bool, out);
+
+    old_dump_interval = conf->ios_dump_interval;
+    GF_OPTION_RECONF("ios-dump-interval", conf->ios_dump_interval, options,
+                     int32, out);
+    if ((old_dump_interval <= 0) && (conf->ios_dump_interval > 0)) {
+        conf->dump_thread_running = _gf_true;
+        conf->dump_thread_should_die = _gf_false;
+        ret = gf_thread_create(&conf->dump_thread, NULL,
+                               (void *)&_ios_dump_thread, this, "iosdump");
+        if (ret) {
+            conf->dump_thread_running = _gf_false;
+            gf_log(this ? this->name : "io-stats", GF_LOG_ERROR,
+                   "Failed to start thread"
+                   "while reconfigure. Returning %d",
+                   ret);
+            goto out;
         }
+    } else if ((old_dump_interval > 0) && (conf->ios_dump_interval == 0)) {
+        _ios_destroy_dump_thread(conf);
+    }
+
+    GF_OPTION_RECONF("ios-sample-interval", conf->ios_sample_interval, options,
+                     int32, out);
+    GF_OPTION_RECONF("ios-dump-format", dump_format_str, options, str, out);
+    ios_set_log_format_code(conf, dump_format_str);
+    GF_OPTION_RECONF("ios-sample-buf-size", conf->ios_sample_buf_size, options,
+                     int32, out);
+    GF_OPTION_RECONF("sys-log-level", sys_log_str, options, str, out);
+    if (sys_log_str) {
+        sys_log_level = glusterd_check_log_level(sys_log_str);
+        set_sys_log_level(sys_log_level);
+    }
+
+    GF_OPTION_RECONF("log-level", log_str, options, str, out);
+    if (log_str) {
+        log_level = glusterd_check_log_level(log_str);
+        /* Set loglevel for all children and server xlators */
+        xlator_set_loglevel(this, log_level);
+    }
+
+    GF_OPTION_RECONF("logger", logger_str, options, str, out);
+    if (logger_str) {
+        logger = gf_check_logger(logger_str);
+        gf_log_set_logger(logger);
+    }
+
+    GF_OPTION_RECONF("log-format", log_format_str, options, str, out);
+    if (log_format_str) {
+        log_format = gf_check_log_format(log_format_str);
+        gf_log_set_logformat(log_format);
+    }
+
+    GF_OPTION_RECONF("log-buf-size", log_buf_size, options, uint32, out);
+    gf_log_set_log_buf_size(log_buf_size);
+
+    GF_OPTION_RECONF("log-flush-timeout", log_flush_timeout, options, time,
+                     out);
+    gf_log_set_log_flush_timeout(log_flush_timeout);
+
+    GF_OPTION_RECONF("threads", threads, options, int32, out);
+    gf_async_adjust_threads(threads);
+
+    ret = 0;
+out:
+    gf_log(this ? this->name : "io-stats", GF_LOG_DEBUG,
+           "reconfigure returning %d", ret);
+    return ret;
+}
 
-        conf = GF_CALLOC (1, sizeof(*conf), gf_io_stats_mt_ios_conf);
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-        if (!conf)
-                goto out;
+    if (!this)
+        return ret;
 
-        /*
-         * Init it just after calloc, so that we are sure the lock is inited
-         * in case of error paths.
-         */
-        LOCK_INIT (&conf->lock);
-        LOCK_INIT (&conf->ios_sampling_lock);
+    ret = xlator_mem_acct_init(this, gf_io_stats_mt_end + 1);
 
-        gettimeofday (&conf->cumulative.started_at, NULL);
-        gettimeofday (&conf->incremental.started_at, NULL);
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
+        return ret;
+    }
 
-        ret = ios_init_top_stats (conf);
-        if (ret)
-                goto out;
+    return ret;
+}
+
+void
+ios_conf_destroy(struct ios_conf *conf)
+{
+    if (!conf)
+        return;
+
+    ios_destroy_top_stats(conf);
+    _ios_destroy_dump_thread(conf);
+    ios_destroy_sample_buf(conf->ios_sample_buf);
+    LOCK_DESTROY(&conf->lock);
+    gf_dnscache_deinit(conf->dnscache);
+    GF_FREE(conf);
+}
 
-        GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
+static void
+ios_init_stats(struct ios_global_stats *stats)
+{
+    int i = 0;
 
-        GF_OPTION_INIT ("count-fop-hits", conf->count_fop_hits, bool, out);
+    GF_ATOMIC_INIT(stats->data_read, 0);
+    GF_ATOMIC_INIT(stats->data_written, 0);
 
-        GF_OPTION_INIT ("latency-measurement", conf->measure_latency,
-                        bool, out);
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        GF_ATOMIC_INIT(stats->block_count_write[i], 0);
+        GF_ATOMIC_INIT(stats->block_count_read[i], 0);
+    }
 
-        GF_OPTION_INIT ("ios-dump-interval", conf->ios_dump_interval,
-                        int32, out);
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        GF_ATOMIC_INIT(stats->fop_hits[i], 0);
 
-        GF_OPTION_INIT ("ios-sample-interval", conf->ios_sample_interval,
-                        int32, out);
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++)
+        GF_ATOMIC_INIT(stats->upcall_hits[i], 0);
 
-        GF_OPTION_INIT ("ios-sample-buf-size", conf->ios_sample_buf_size,
-                        int32, out);
+    stats->started_at = gf_time();
+}
 
-        if (ios_init_sample_buf (conf) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                return -1;
-        }
+int
+init(xlator_t *this)
+{
+    struct ios_conf *conf = NULL;
+    char *volume_id = NULL;
+    char *sys_log_str = NULL;
+    char *logger_str = NULL;
+    char *log_format_str = NULL;
+    char *dump_format_str = NULL;
+    int logger = -1;
+    int log_format = -1;
+    int sys_log_level = -1;
+    char *log_str = NULL;
+    int log_level = -1;
+    int ret = -1;
+    uint32_t log_buf_size = 0;
+    uint32_t log_flush_timeout = 0;
+    int32_t threads;
+
+    if (!this)
+        return -1;
+
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "io_stats translator requires at least one subvolume");
+        return -1;
+    }
+
+    if (!this->parents) {
+        /* This is very much valid as io-stats currently is loaded
+         * on top of volumes on both client and server, hence this is
+         * not an warning message */
+        gf_log(this->name, GF_LOG_DEBUG, "dangling volume. check volfile ");
+    }
+
+    conf = GF_CALLOC(1, sizeof(*conf), gf_io_stats_mt_ios_conf);
+
+    if (!conf)
+        goto out;
 
-        GF_OPTION_INIT ("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec,
-                        int32, out);
-        conf->dnscache = gf_dnscache_init (conf->ios_dnscache_ttl_sec);
+    if (dict_get_str(this->options, "unique-id", &conf->unique_id) != 0) {
+        /* This is always set on servers, so we must be a client. */
+        conf->unique_id = this->name;
+    }
+
+    ret = dict_get_strn(this->options, "volume-id", SLEN("volume-id"),
+                        &volume_id);
+    if (!ret) {
+        strncpy(this->graph->volume_id, volume_id, GF_UUID_BUF_SIZE);
+    }
+    /*
+     * Init it just after calloc, so that we are sure the lock is inited
+     * in case of error paths.
+     */
+    LOCK_INIT(&conf->lock);
+    LOCK_INIT(&conf->ios_sampling_lock);
+
+    ios_init_stats(&conf->cumulative);
+    ios_init_stats(&conf->incremental);
+
+    ret = ios_init_top_stats(conf);
+    if (ret)
+        goto out;
 
-        GF_OPTION_INIT ("sys-log-level", sys_log_str, str, out);
-        if (sys_log_str) {
-                sys_log_level = glusterd_check_log_level (sys_log_str);
-                set_sys_log_level (sys_log_level);
-        }
+    GF_OPTION_INIT("dump-fd-stats", conf->dump_fd_stats, bool, out);
 
-        GF_OPTION_INIT ("log-level", log_str, str, out);
-        if (log_str) {
-                log_level = glusterd_check_log_level (log_str);
-                gf_log_set_loglevel (log_level);
-        }
+    GF_OPTION_INIT("count-fop-hits", conf->count_fop_hits, bool, out);
 
-        GF_OPTION_INIT ("logger", logger_str, str, out);
-        if (logger_str) {
-                logger = gf_check_logger (logger_str);
-                gf_log_set_logger (logger);
-        }
+    GF_OPTION_INIT("latency-measurement", conf->measure_latency, bool, out);
 
-        GF_OPTION_INIT ("log-format", log_format_str, str, out);
-        if (log_format_str) {
-                log_format = gf_check_log_format (log_format_str);
-                gf_log_set_logformat (log_format);
-        }
+    GF_OPTION_INIT("ios-dump-interval", conf->ios_dump_interval, int32, out);
 
-        GF_OPTION_INIT ("log-buf-size", log_buf_size, uint32, out);
-        gf_log_set_log_buf_size (log_buf_size);
+    GF_OPTION_INIT("ios-sample-interval", conf->ios_sample_interval, int32,
+                   out);
 
-        GF_OPTION_INIT ("log-flush-timeout", log_flush_timeout, time, out);
-        gf_log_set_log_flush_timeout (log_flush_timeout);
+    GF_OPTION_INIT("ios-dump-format", dump_format_str, str, out);
+    ios_set_log_format_code(conf, dump_format_str);
 
+    GF_OPTION_INIT("ios-sample-buf-size", conf->ios_sample_buf_size, int32,
+                   out);
 
-        this->private = conf;
-        if (conf->ios_dump_interval > 0) {
-                pthread_create (&conf->dump_thread, NULL,
-                                (void *) &_ios_dump_thread, this);
+    ret = ios_init_sample_buf(conf);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Out of memory.");
+        goto out;
+    }
+
+    GF_OPTION_INIT("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec, int32,
+                   out);
+    conf->dnscache = gf_dnscache_init(conf->ios_dnscache_ttl_sec);
+    if (!conf->dnscache) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_INIT("sys-log-level", sys_log_str, str, out);
+    if (sys_log_str) {
+        sys_log_level = glusterd_check_log_level(sys_log_str);
+        set_sys_log_level(sys_log_level);
+    }
+
+    GF_OPTION_INIT("log-level", log_str, str, out);
+    if (log_str) {
+        log_level = glusterd_check_log_level(log_str);
+        if (DEFAULT_LOG_LEVEL != log_level)
+            gf_log_set_loglevel(this->ctx, log_level);
+    }
+
+    GF_OPTION_INIT("logger", logger_str, str, out);
+    if (logger_str) {
+        logger = gf_check_logger(logger_str);
+        gf_log_set_logger(logger);
+    }
+
+    GF_OPTION_INIT("log-format", log_format_str, str, out);
+    if (log_format_str) {
+        log_format = gf_check_log_format(log_format_str);
+        gf_log_set_logformat(log_format);
+    }
+
+    GF_OPTION_INIT("log-buf-size", log_buf_size, uint32, out);
+    gf_log_set_log_buf_size(log_buf_size);
+
+    GF_OPTION_INIT("log-flush-timeout", log_flush_timeout, time, out);
+    gf_log_set_log_flush_timeout(log_flush_timeout);
+
+    GF_OPTION_INIT("threads", threads, int32, out);
+    gf_async_adjust_threads(threads);
+
+    this->private = conf;
+    if (conf->ios_dump_interval > 0) {
+        conf->dump_thread_running = _gf_true;
+        conf->dump_thread_should_die = _gf_false;
+        ret = gf_thread_create(&conf->dump_thread, NULL,
+                               (void *)&_ios_dump_thread, this, "iosdump");
+        if (ret) {
+            conf->dump_thread_running = _gf_false;
+            gf_log(this ? this->name : "io-stats", GF_LOG_ERROR,
+                   "Failed to start thread"
+                   "in init. Returning %d",
+                   ret);
+            goto out;
         }
-        ret = 0;
+    }
+    return 0;
 out:
-        if (!this->private) {
-                ios_conf_destroy (conf);
-                ret = -1;
-        }
-
-        return ret;
+    ios_conf_destroy(conf);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        struct ios_conf *conf = NULL;
+    struct ios_conf *conf = NULL;
 
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        conf = this->private;
+    conf = this->private;
 
-        ios_conf_destroy (conf);
-        this->private = NULL;
-        gf_log (this->name, GF_LOG_INFO,
-                "io-stats translator unloaded");
-        return;
+    ios_conf_destroy(conf);
+    this->private = NULL;
+    gf_log(this->name, GF_LOG_INFO, "io-stats translator unloaded");
+    return;
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int          ret = 0;
-        struct ios_dump_args args = {0};
-        dict_t       *output = NULL;
-        dict_t       *dict = NULL;
-        int32_t       op = 0;
-        int32_t       list_cnt = 0;
-        double        throughput = 0;
-        double        time = 0;
-        gf_boolean_t  is_peek = _gf_false;
-        va_list ap;
-
-        dict = data;
-        va_start (ap, data);
-        output = va_arg (ap, dict_t*);
-        va_end (ap);
-        switch (event) {
+notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    int ret = 0;
+    struct ios_dump_args args = {0};
+    dict_t *output = NULL;
+    dict_t *dict = NULL;
+    int32_t op = 0;
+    int32_t list_cnt = 0;
+    double throughput = 0;
+    double time = 0;
+    gf_boolean_t is_peek = _gf_false;
+    va_list ap;
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+
+    dict = data;
+    va_start(ap, data);
+    output = va_arg(ap, dict_t *);
+    va_end(ap);
+    switch (event) {
         case GF_EVENT_TRANSLATOR_INFO:
-                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+            ret = dict_get_str_boolean(dict, "clear-stats", _gf_false);
+            if (ret) {
+                ret = dict_set_int32(output, "top-op", op);
                 if (ret) {
-                        ret = dict_set_int32 (output, "top-op", op);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set top-op in dict");
-                                goto out;
-                        }
-                        ios_destroy_top_stats (this->private);
-                        ret = ios_init_top_stats (this->private);
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set top-op in dict");
+                    goto out;
+                }
+                ios_destroy_top_stats(this->private);
+                ret = ios_init_top_stats(this->private);
+                if (ret)
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to reset top stats");
+                ret = dict_set_int32(output, "stats-cleared", ret ? 0 : 1);
+                if (ret)
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set stats-cleared"
+                           " in dict");
+                goto out;
+            }
+
+            ret = dict_get_int32(dict, "top-op", &op);
+            if (!ret) {
+                ret = dict_get_int32(dict, "list-cnt", &list_cnt);
+                if (op > IOS_STATS_TYPE_NONE && op < IOS_STATS_TYPE_MAX)
+                    ret = io_stats_dump_stats_to_dict(this, output, op,
+                                                      list_cnt);
+                if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
+                    op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
+                    ret = dict_get_double(dict, "throughput", &throughput);
+                    if (!ret) {
+                        ret = dict_get_double(dict, "time", &time);
+                        if (ret)
+                            goto out;
+                        ret = dict_set_double(output, "throughput", throughput);
                         if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to reset top stats");
-                        ret = dict_set_int32 (output, "stats-cleared",
-                                              ret ? 0 : 1);
+                            goto out;
+                        ret = dict_set_double(output, "time", time);
                         if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set stats-cleared"
-                                        " in dict");
-                        goto out;
+                            goto out;
+                    }
+                    ret = 0;
                 }
+            } else {
+                ret = dict_get_int32(dict, "info-op", &op);
+                if (ret || op < GF_IOS_INFO_ALL || GF_IOS_INFO_CLEAR < op)
+                    op = GF_IOS_INFO_ALL;
 
-                ret = dict_get_int32 (dict, "top-op", &op);
-                if (!ret) {
-                        ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
-                        if (op > IOS_STATS_TYPE_NONE &&
-                            op < IOS_STATS_TYPE_MAX)
-                                ret = io_stats_dump_stats_to_dict (this, output,
-                                                             op, list_cnt);
-                        if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
-                                op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
-                                ret = dict_get_double (dict, "throughput",
-                                                        &throughput);
-                                if (!ret) {
-                                        ret = dict_get_double (dict, "time",
-                                                                &time);
-                                        if (ret)
-                                                goto out;
-                                        ret = dict_set_double (output,
-                                                "throughput", throughput);
-                                        if (ret)
-                                                goto out;
-                                        ret = dict_set_double (output, "time",
-                                                        time);
-                                        if (ret)
-                                                goto out;
-                                }
-                                ret = 0;
-
-                        }
+                ret = dict_set_int32(output, "info-op", op);
+                if (ret) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set info-op in dict");
+                    goto out;
+                }
+
+                if (GF_IOS_INFO_CLEAR == op) {
+                    io_stats_clear(this->private);
+
+                    ret = dict_set_int32(output, "stats-cleared", 1);
+                    if (ret)
+                        gf_log(this->name, GF_LOG_ERROR,
+                               "Failed to set stats-cleared"
+                               " in dict");
                 } else {
-                        ret = dict_get_int32 (dict, "info-op", &op);
-                        if (ret || op < GF_CLI_INFO_ALL ||
-                            GF_CLI_INFO_CLEAR < op)
-                            op = GF_CLI_INFO_ALL;
-
-                        ret = dict_set_int32 (output, "info-op", op);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set info-op in dict");
-                                goto out;
-                        }
-
-                        if (GF_CLI_INFO_CLEAR == op) {
-                                ret = io_stats_clear (this->private);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to clear info stats");
-
-                                ret = dict_set_int32 (output, "stats-cleared",
-                                              ret ? 0 : 1);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set stats-cleared"
-                                                " in dict");
-                        }
-                        else {
-                                ret = dict_get_str_boolean (dict, "peek",
-                                                            _gf_false);
-                                if (-1 != ret)
-                                        is_peek = ret;
-
-                                (void) ios_dump_args_init (&args,
-                                                IOS_DUMP_TYPE_DICT, output);
-                                ret = io_stats_dump (this, &args, op, is_peek);
-                        }
+                    ret = dict_get_str_boolean(dict, "peek", _gf_false);
+                    if (-1 != ret)
+                        is_peek = ret;
+
+                    (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_DICT, output);
+                    ret = io_stats_dump(this, &args, op, is_peek);
                 }
-                break;
-        default:
-                default_notify (this, event, data);
-                break;
+            }
+            break;
+        case GF_EVENT_UPCALL:
+            up_data = (struct gf_upcall *)data;
+            ios_bump_upcall(this, GF_UPCALL);
+
+            switch (up_data->event_type) {
+                case GF_UPCALL_RECALL_LEASE:
+                    ios_bump_upcall(this, GF_UPCALL_LEASE_RECALL);
+                    break;
+                case GF_UPCALL_CACHE_INVALIDATION:
+                    up_ci = (struct gf_upcall_cache_invalidation *)
+                                up_data->data;
+                    if (up_ci->flags & (UP_XATTR | UP_XATTR_RM))
+                        ios_bump_upcall(this, GF_UPCALL_CI_XATTR);
+                    if (up_ci->flags & IATT_UPDATE_FLAGS)
+                        ios_bump_upcall(this, GF_UPCALL_CI_STAT);
+                    if (up_ci->flags & UP_RENAME_FLAGS)
+                        ios_bump_upcall(this, GF_UPCALL_CI_RENAME);
+                    if (up_ci->flags & UP_FORGET)
+                        ios_bump_upcall(this, GF_UPCALL_CI_FORGET);
+                    if (up_ci->flags & UP_NLINK)
+                        ios_bump_upcall(this, GF_UPCALL_CI_NLINK);
+                    break;
+                default:
+                    gf_msg_debug(this->name, 0,
+                                 "Unknown upcall event "
+                                 "type :%d",
+                                 up_data->event_type);
+                    break;
+            }
 
-        }
+            default_notify(this, event, data);
+            break;
+        default:
+            default_notify(this, event, data);
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-struct xlator_dumpops dumpops = {
-        .priv    = io_priv
-};
+struct xlator_dumpops dumpops = {.priv = io_priv};
 
 struct xlator_fops fops = {
-        .stat        = io_stats_stat,
-        .readlink    = io_stats_readlink,
-        .mknod       = io_stats_mknod,
-        .mkdir       = io_stats_mkdir,
-        .unlink      = io_stats_unlink,
-        .rmdir       = io_stats_rmdir,
-        .symlink     = io_stats_symlink,
-        .rename      = io_stats_rename,
-        .link        = io_stats_link,
-        .truncate    = io_stats_truncate,
-        .open        = io_stats_open,
-        .readv       = io_stats_readv,
-        .writev      = io_stats_writev,
-        .statfs      = io_stats_statfs,
-        .flush       = io_stats_flush,
-        .fsync       = io_stats_fsync,
-        .setxattr    = io_stats_setxattr,
-        .getxattr    = io_stats_getxattr,
-        .removexattr = io_stats_removexattr,
-        .fsetxattr    = io_stats_fsetxattr,
-        .fgetxattr    = io_stats_fgetxattr,
-        .fremovexattr = io_stats_fremovexattr,
-        .opendir     = io_stats_opendir,
-        .readdir     = io_stats_readdir,
-        .readdirp    = io_stats_readdirp,
-        .fsyncdir    = io_stats_fsyncdir,
-        .access      = io_stats_access,
-        .ftruncate   = io_stats_ftruncate,
-        .fstat       = io_stats_fstat,
-        .create      = io_stats_create,
-        .lk          = io_stats_lk,
-        .inodelk     = io_stats_inodelk,
-        .finodelk    = io_stats_finodelk,
-        .entrylk     = io_stats_entrylk,
-        .lookup      = io_stats_lookup,
-        .xattrop     = io_stats_xattrop,
-        .fxattrop    = io_stats_fxattrop,
-        .setattr     = io_stats_setattr,
-        .fsetattr    = io_stats_fsetattr,
-	.fallocate   = io_stats_fallocate,
-	.discard     = io_stats_discard,
-        .zerofill    = io_stats_zerofill,
+    .stat = io_stats_stat,
+    .readlink = io_stats_readlink,
+    .mknod = io_stats_mknod,
+    .mkdir = io_stats_mkdir,
+    .unlink = io_stats_unlink,
+    .rmdir = io_stats_rmdir,
+    .symlink = io_stats_symlink,
+    .rename = io_stats_rename,
+    .link = io_stats_link,
+    .truncate = io_stats_truncate,
+    .open = io_stats_open,
+    .readv = io_stats_readv,
+    .writev = io_stats_writev,
+    .statfs = io_stats_statfs,
+    .flush = io_stats_flush,
+    .fsync = io_stats_fsync,
+    .setxattr = io_stats_setxattr,
+    .getxattr = io_stats_getxattr,
+    .removexattr = io_stats_removexattr,
+    .fsetxattr = io_stats_fsetxattr,
+    .fgetxattr = io_stats_fgetxattr,
+    .fremovexattr = io_stats_fremovexattr,
+    .opendir = io_stats_opendir,
+    .readdir = io_stats_readdir,
+    .readdirp = io_stats_readdirp,
+    .fsyncdir = io_stats_fsyncdir,
+    .access = io_stats_access,
+    .ftruncate = io_stats_ftruncate,
+    .fstat = io_stats_fstat,
+    .create = io_stats_create,
+    .lk = io_stats_lk,
+    .inodelk = io_stats_inodelk,
+    .finodelk = io_stats_finodelk,
+    .entrylk = io_stats_entrylk,
+    .fentrylk = io_stats_fentrylk,
+    .lookup = io_stats_lookup,
+    .xattrop = io_stats_xattrop,
+    .fxattrop = io_stats_fxattrop,
+    .setattr = io_stats_setattr,
+    .fsetattr = io_stats_fsetattr,
+    .fallocate = io_stats_fallocate,
+    .discard = io_stats_discard,
+    .zerofill = io_stats_zerofill,
+    .ipc = io_stats_ipc,
+    .rchecksum = io_stats_rchecksum,
+    .seek = io_stats_seek,
+    .lease = io_stats_lease,
+    .getactivelk = io_stats_getactivelk,
+    .setactivelk = io_stats_setactivelk,
+    .compound = io_stats_compound,
+    .copy_file_range = io_stats_copy_file_range,
 };
 
 struct xlator_cbks cbks = {
-        .release     = io_stats_release,
-        .releasedir  = io_stats_releasedir,
-        .forget      = io_stats_forget,
+    .release = io_stats_release,
+    .releasedir = io_stats_releasedir,
+    .forget = io_stats_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"dump-fd-stats"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "If on stats related to file-operations would be "
-                         "tracked inside GlusterFS data-structures."
-        },
-        { .key  = { "ios-dump-interval" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 3600,
-          .default_value = "0",
-          .description = "Interval (in seconds) at which to auto-dump "
-                         "statistics. Zero disables automatic dumping."
-        },
-        { .key  = { "ios-sample-interval" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 65535,
-          .default_value = "0",
-          .description = "Interval in which we want to collect FOP latency "
-                         "samples.  2 means collect a sample every 2nd FOP."
-        },
-        { .key  = { "ios-sample-buf-size" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 1024,
-          .max = 1024*1024,
-          .default_value = "65535",
-          .description = "The maximum size of our FOP sampling ring buffer."
-        },
-        { .key  = { "ios-dnscache-ttl-sec" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 1,
-          .max = 3600 * 72,
-          .default_value = "86400",
-          .description = "The interval after wish a cached DNS entry will be "
-                         "re-validated.  Default: 24 hrs"
-        },
-        { .key  = { "latency-measurement" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "If on stats related to the latency of each operation "
-                         "would be tracked inside GlusterFS data-structures. "
-        },
-        { .key  = {"count-fop-hits"},
-          .type = GF_OPTION_TYPE_BOOL,
-        },
-        { .key = {"log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-
-        /* These are synthetic entries to assist validation of CLI's  *
-         *  volume set  command                                       */
-        { .key = {"client-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "INFO",
-          .description = "Changes the log-level of the clients",
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-        { .key = {"sys-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "CRITICAL",
-          .description = "Gluster's syslog log-level",
-          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
-        },
-        { .key = {"brick-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "INFO",
-          .description = "Changes the log-level of the bricks",
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-        { .key = {"logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"client-logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOGGER_GLUSTER_LOG,
-          .description = "Changes the logging sub-system to log to, for the "
-                         "clients",
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"brick-logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOGGER_GLUSTER_LOG,
-          .description = "Changes the logging sub-system to log to, for the "
-                         "bricks",
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key = {"client-log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
-          .description = "Changes log format for the clients",
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key = {"brick-log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
-          .description = "Changes the log format for the bricks",
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key  = {"log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-        },
-        { .key  = {"client-log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option client-log-flush-timeout."
-        },
-        { .key  = {"brick-log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option brick-log-flush-timeout."
-        },
-        { .key  = {"log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-        },
-        { .key  = {"client-log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option client-log-flush-timeout."
-        },
-        { .key  = {"brick-log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option brick-log-flush-timeout."
-        },
-        { .key  = {NULL} },
+    {.key = {"dump-fd-stats"},
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "If on stats related to file-operations would be "
+                    "tracked inside GlusterFS data-structures."},
+    {.key = {"ios-dump-interval"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 0,
+     .max = 3600,
+     .default_value = "0",
+     .description = "Interval (in seconds) at which to auto-dump "
+                    "statistics. Zero disables automatic dumping."},
+    {.key = {"ios-sample-interval"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 0,
+     .max = 65535,
+     .default_value = "0",
+     .description = "Interval in which we want to collect FOP latency "
+                    "samples.  2 means collect a sample every 2nd FOP."},
+    {.key = {"ios-dump-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_12_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "json",
+     .description = " The dump-format option specifies the format in which"
+                    " to dump the statistics. Select between \"text\", "
+                    "\"json\", \"dict\" and \"samples\". Default is "
+                    "\"json\".",
+     .value = {"text", "json", "dict", "samples"}},
+    {.key = {"ios-sample-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 1024,
+     .max = 1024 * 1024,
+     .default_value = "65535",
+     .description = "The maximum size of our FOP sampling ring buffer."},
+    {.key = {"ios-dnscache-ttl-sec"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 1,
+     .max = 3600 * 72,
+     .default_value = "86400",
+     .description = "The interval after wish a cached DNS entry will be "
+                    "re-validated.  Default: 24 hrs"},
+    {.key = {"latency-measurement"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "off",
+     .description = "If on stats related to the latency of each operation "
+                    "would be tracked inside GlusterFS data-structures. "},
+    {
+        .key = {"count-fop-hits"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"io-stats"},
+    },
+    {.key = {"log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+
+    /* These are synthetic entries to assist validation of CLI's  *
+     *  volume set  command                                       */
+    {.key = {"client-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "INFO",
+     .description = "Changes the log-level of the clients",
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+    {.key = {"sys-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "CRITICAL",
+     .description = "Gluster's syslog log-level",
+     .value = {"WARNING", "ERROR", "INFO", "CRITICAL"}},
+    {.key = {"brick-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "INFO",
+     .description = "Changes the log-level of the bricks",
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+    {.key = {"logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"client-logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOGGER_GLUSTER_LOG,
+     .description = "Changes the logging sub-system to log to, for the "
+                    "clients",
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"brick-logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOGGER_GLUSTER_LOG,
+     .description = "Changes the logging sub-system to log to, for the "
+                    "bricks",
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {.key = {"client-log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+     .description = "Changes log format for the clients",
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {.key = {"brick-log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+     .description = "Changes the log format for the bricks",
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {
+        .key = {"log-buf-size"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = GF_LOG_LRU_BUFSIZE_MIN,
+        .max = GF_LOG_LRU_BUFSIZE_MAX,
+        .default_value = "5",
+    },
+    {.key = {"client-log-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_LRU_BUFSIZE_MIN,
+     .max = GF_LOG_LRU_BUFSIZE_MAX,
+     .default_value = "5",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option client-log-flush-timeout."},
+    {.key = {"brick-log-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_LRU_BUFSIZE_MIN,
+     .max = GF_LOG_LRU_BUFSIZE_MAX,
+     .default_value = "5",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option brick-log-flush-timeout."},
+    {
+        .key = {"log-flush-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+        .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+        .default_value = "120",
+    },
+    {.key = {"client-log-flush-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+     .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+     .default_value = "120",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option client-log-flush-timeout."},
+    {.key = {"brick-log-flush-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+     .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+     .default_value = "120",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option brick-log-flush-timeout."},
+    {.key = {"unique-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "/no/such/path",
+     .description = "Unique ID for our files."},
+    {.key = {"global-threading"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"io-stats", "threading"},
+     .description = "This option enables the global threading support for "
+                    "bricks. If enabled, it's recommended to also enable "
+                    "'performance.iot-pass-through'"},
+    {.key = {"threads"}, .type = GF_OPTION_TYPE_INT},
+    {.key = {"brick-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "16",
+     .min = 0,
+     .max = GF_ASYNC_MAX_THREADS,
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats", "threading"},
+     .description = "When global threading is used, this value determines the "
+                    "maximum amount of threads that can be created on bricks"},
+    {.key = {"client-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "16",
+     .min = 0,
+     .max = GF_ASYNC_MAX_THREADS,
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"io-stats", "threading"},
+     .description = "When global threading is used, this value determines the "
+                    "maximum amount of threads that can be created on clients"},
+    {.key = {"volume-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_7_1},
+     .tags = {"global", "volume-id"},
+     .description =
+         "This option points to the 'unique' UUID particular to this "
+         "volume, which would be set in 'graph->volume_id'"},
+    {.key = {NULL}},
+};
 
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "io-stats",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/performance/decompounder/Makefile.am b/xlators/debug/sink/Makefile.am
index af437a64d6d..f2689244371 100644
--- a/xlators/performance/decompounder/Makefile.am
+++ b/xlators/debug/sink/Makefile.am
@@ -1 +1,2 @@
 SUBDIRS = src
+
diff --git a/xlators/debug/sink/src/Makefile.am b/xlators/debug/sink/src/Makefile.am
new file mode 100644
index 00000000000..f952c2ce6bc
--- /dev/null
+++ b/xlators/debug/sink/src/Makefile.am
@@ -0,0 +1,14 @@
+xlator_LTLIBRARIES = sink.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_builddir)/rpc/xdr/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+sink_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+sink_la_SOURCES = sink.c
+sink_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+CLEANFILES =
+
diff --git a/xlators/debug/sink/src/sink.c b/xlators/debug/sink/src/sink.c
new file mode 100644
index 00000000000..9822bbb732e
--- /dev/null
+++ b/xlators/debug/sink/src/sink.c
@@ -0,0 +1,94 @@
+/*
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+int32_t
+init(xlator_t *this)
+{
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    return;
+}
+
+/*
+ * notify - when parent sends PARENT_UP, send CHILD_UP event from here
+ */
+int32_t
+notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    switch (event) {
+        case GF_EVENT_PARENT_UP:
+            /* Tell the parent that this xlator is up */
+            default_notify(this, GF_EVENT_CHILD_UP, data);
+            break;
+        case GF_EVENT_PARENT_DOWN:
+            /* Tell the parent that this xlator is down */
+            default_notify(this, GF_EVENT_CHILD_DOWN, data);
+            break;
+        default:
+            break;
+    }
+
+    return 0;
+}
+
+/*
+ * A lookup on "/" is done while mounting or glfs_init() is performed. This
+ * needs to return a valid directory for the root of the mountpoint.
+ *
+ * In case this xlator is used for more advanced debugging, it will need to be
+ * extended to support different LOOKUPs too.
+ */
+static int32_t
+sink_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+
+    /* the root of the volume always need to be a directory */
+    stbuf.ia_type = IA_IFDIR;
+
+    STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc ? loc->inode : NULL, &stbuf,
+                        xdata, &postparent);
+
+    return 0;
+}
+
+struct xlator_fops fops = {
+    .lookup = sink_lookup,
+};
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "sink",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/debug/trace/src/Makefile.am b/xlators/debug/trace/src/Makefile.am
index 9bd53c89bfe..a37ea63af04 100644
--- a/xlators/debug/trace/src/Makefile.am
+++ b/xlators/debug/trace/src/Makefile.am
@@ -8,7 +8,8 @@ trace_la_SOURCES = trace.c
 trace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = trace.h trace-mem-types.h
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
index 9fa7d97c2ca..18a7e0414a6 100644
--- a/xlators/debug/trace/src/trace-mem-types.h
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -8,14 +8,13 @@
    cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __TRACE_MEM_TYPES_H__
 #define __TRACE_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_trace_mem_types_ {
-        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
-        gf_trace_mt_end
+    gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+    gf_trace_mt_end
 };
 #endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index c809254eb17..6ed0ca00342 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -17,3287 +17,3518 @@
  *    their _cbk functions, which later passes the call to next layer.
  *    Very helpful translator for debugging.
  */
-#define TRACE_STAT_TO_STR(buf, str) trace_stat_to_str (buf, str, sizeof (str))
+#define TRACE_STAT_TO_STR(buf, str) trace_stat_to_str(buf, str, sizeof(str))
 
 static void
 trace_stat_to_str(struct iatt *buf, char *str, size_t len)
 {
-        char     atime_buf[256]    = {0,};
-        char     mtime_buf[256]    = {0,};
-        char     ctime_buf[256]    = {0,};
-
-        if (!buf)
-                return;
+    char atime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char mtime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char ctime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    if (!buf)
+        return;
 
-        gf_time_fmt (atime_buf, sizeof atime_buf, buf->ia_atime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(atime_buf, sizeof atime_buf, buf->ia_atime, gf_timefmt_dirent);
 
-        gf_time_fmt (mtime_buf, sizeof mtime_buf, buf->ia_mtime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(mtime_buf, sizeof mtime_buf, buf->ia_mtime, gf_timefmt_dirent);
 
-        gf_time_fmt (ctime_buf, sizeof ctime_buf, buf->ia_ctime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(ctime_buf, sizeof ctime_buf, buf->ia_ctime, gf_timefmt_dirent);
 
-        snprintf (str, len, "gfid=%s ino=%"PRIu64", mode=%o, "
-                  "nlink=%"GF_PRI_NLINK", uid=%u, gid=%u, size=%"PRIu64", "
-                  "blocks=%"PRIu64", atime=%s mtime=%s ctime=%s "
-                  "atime_sec=%"PRIu32", atime_nsec=%"PRIu32","
-                  " mtime_sec=%"PRIu32", mtime_nsec=%"PRIu32", "
-                  "ctime_sec=%"PRIu32", ctime_nsec=%"PRIu32"",
-                  uuid_utoa (buf->ia_gfid), buf->ia_ino,
-                  st_mode_from_ia (buf->ia_prot, buf->ia_type), buf->ia_nlink,
-                  buf->ia_uid, buf->ia_gid, buf->ia_size, buf->ia_blocks,
-                  atime_buf, mtime_buf, ctime_buf,
-                  buf->ia_atime, buf->ia_atime_nsec,
-                  buf->ia_mtime, buf->ia_mtime_nsec,
-                  buf->ia_ctime, buf->ia_ctime_nsec);
+    snprintf(str, len,
+             "gfid=%s ino=%" PRIu64
+             ", mode=%o, "
+             "nlink=%" GF_PRI_NLINK ", uid=%u, gid=%u, size=%" PRIu64
+             ", "
+             "blocks=%" PRIu64
+             ", atime=%s mtime=%s ctime=%s "
+             "atime_sec=%" PRId64 ", atime_nsec=%" PRIu32
+             ","
+             " mtime_sec=%" PRId64 ", mtime_nsec=%" PRIu32
+             ", "
+             "ctime_sec=%" PRId64 ", ctime_nsec=%" PRIu32 "",
+             uuid_utoa(buf->ia_gfid), buf->ia_ino,
+             st_mode_from_ia(buf->ia_prot, buf->ia_type), buf->ia_nlink,
+             buf->ia_uid, buf->ia_gid, buf->ia_size, buf->ia_blocks, atime_buf,
+             mtime_buf, ctime_buf, buf->ia_atime, buf->ia_atime_nsec,
+             buf->ia_mtime, buf->ia_mtime_nsec, buf->ia_ctime,
+             buf->ia_ctime_nsec);
 }
 
-
 int
-dump_history_trace (circular_buffer_t *cb, void *data)
+dump_history_trace(circular_buffer_t *cb, void *data)
 {
-        char     timestr[256] = {0,};
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
 
-        /* Since we are continuing with adding entries to the buffer even when
-           gettimeofday () fails, it's safe to check tm and then dump the time
-           at which the entry was added to the buffer */
+    /* Since we are continuing with adding entries to the buffer even when
+       gettimeofday () fails, it's safe to check tm and then dump the time
+       at which the entry was added to the buffer */
 
-        gf_time_fmt (timestr, sizeof timestr, cb->tv.tv_sec, gf_timefmt_Ymd_T);
-        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
-        gf_proc_dump_write ("TIME", "%s", timestr);
+    gf_time_fmt_tv(timestr, sizeof timestr, &cb->tv, gf_timefmt_Ymd_T);
+    gf_proc_dump_write("TIME", "%s", timestr);
 
-        gf_proc_dump_write ("FOP", "%s\n", cb->data);
-
-        return 0;
-}
+    gf_proc_dump_write("FOP", "%s\n", (char *)cb->data);
 
-int
-trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd,
-                  inode_t *inode, struct iatt *buf,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
-                                  "*stbuf {%s}, *preparent {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid), op_ret, fd,
-                                  statstr, preparentstr, postparentstr);
-
-                        /* for 'release' log */
-                        fd_ctx_set (fd, this, 0);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+trace_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        trace_conf_t      *conf = NULL;
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                char     string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
-                          "*fd=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
 
-                LOG_ELEMENT (conf, string);
-        }
-
-out:
-        /* for 'release' log */
-        if (op_ret >= 0)
-                fd_ctx_set (fd, this, 0);
-
-        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d, fd=%p"
+                     "*stbuf {%s}, *preparent {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret, fd,
+                     statstr, preparentstr, postparentstr);
 
-int
-trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STAT].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
+            /* for 'release' log */
+            fd_ctx_set(fd, this, 0);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
         }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                       preparent, postparent, xdata);
+    return 0;
 }
 
 int
-trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref,
-                 dict_t *xdata)
+trace_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READ].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                            buf, iobref, xdata);
-        return 0;
-}
-
-int
-trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char         preopstr[4096]  = {0, };
-        char         postopstr[4096] = {0, };
-        trace_conf_t *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
-}
+    trace_conf_t *conf = NULL;
 
-int
-trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
-                   dict_t *xdata)
-{
-        trace_conf_t  *conf = NULL;
+    conf = this->private;
 
-        conf = this->private;
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d, "
+                 "*fd=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 fd);
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                char    string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
+        LOG_ELEMENT(conf, string);
+    }
 
-                LOG_ELEMENT (conf, string);
-        }
 out:
-        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
+    /* for 'release' log */
+    if (op_ret >= 0)
+        fd_ctx_set(fd, this, 0);
 
-        return 0;
+    TRACE_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int
-trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
-                    dict_t *xdata)
+trace_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                char        string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-
-out:
-        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int
-trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            (void)snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s op_ret=%d buf=%s",
+                frame->root->unique, uuid_utoa(frame->local), op_ret, statstr);
+        } else {
+            (void)snprintf(string, sizeof(string),
+                           "%" PRId64
+                           ": gfid=%s op_ret=%d, "
+                           "op_errno=%d)",
+                           frame->root->unique, uuid_utoa(frame->local), op_ret,
+                           op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                int32_t count, struct iatt *buf, struct iobref *iobref,
+                dict_t *xdata)
 {
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s}",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-
-        return 0;
-}
-
-int
-trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READ].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s op_ret=%d buf=%s",
+                frame->root->unique, uuid_utoa(frame->local), op_ret, statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf,
+                       iobref, xdata);
+    return 0;
+}
+
+int
+trace_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_WRITE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int
+trace_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                  dict_t *xdata)
 {
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (statpre, preopstr);
-                        TRACE_STAT_TO_STR (statpost, postopstr);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 " : gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
-                            statpost, xdata);
-        return 0;
-}
-
-int
-trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
-{
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (statpre, preopstr);
-                        TRACE_STAT_TO_STR (statpost, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
-                            statpre, statpost, xdata);
-        return 0;
-}
+    TRACE_STACK_UNWIND(readdir, frame, op_ret, op_errno, buf, xdata);
 
-int
-trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
-{
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  " *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                            preparent, postparent, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent,
-                  dict_t *xdata)
-{
-        char           statstr[4096]          = {0, };
-        char           preoldparentstr[4096]  = {0, };
-        char           postoldparentstr[4096] = {0, };
-        char           prenewparentstr[4096]  = {0, };
-        char           postnewparentstr[4096] = {0, };
-        trace_conf_t   *conf                  = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preoldparent, preoldparentstr);
-                        TRACE_STAT_TO_STR (postoldparent, postoldparentstr);
-                        TRACE_STAT_TO_STR (prenewparent, prenewparentstr);
-                        TRACE_STAT_TO_STR (postnewparent, postnewparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*stbuf = {%s}, *preoldparent = {%s},"
-                                  " *postoldparent = {%s}"
-                                  " *prenewparent = {%s}, "
-                                  "*postnewparent = {%s})",
-                                  frame->root->unique, op_ret, statstr,
-                                  preoldparentstr, postoldparentstr,
-                                  prenewparentstr, postnewparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
-                            preoldparent, postoldparent,
-                            prenewparent, postnewparent, xdata);
-        return 0;
-}
-
-int
-trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    const char *buf, struct iatt *stbuf, dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (stbuf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
-                                  "buf=%s, stbuf = { %s })",
-                                  frame->root->unique, op_ret, op_errno,
-                                  buf, statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
-                            xdata);
-        return 0;
-}
-
-int
-trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  inode_t *inode, struct iatt *buf,
-                  dict_t *xdata, struct iatt *postparent)
-{
-        char          statstr[4096]       = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-                        /* print buf->ia_gfid instead of inode->gfid,
-                         * since if the inode is not yet linked to the
-                         * inode table (fresh lookup) then null gfid
-                         * will be printed.
-                         */
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*buf {%s}, *postparent {%s}",
-                                  frame->root->unique,
-                                  uuid_utoa (buf->ia_gfid),
-                                  op_ret, statstr, postparentstr);
-
-                        /* For 'forget' */
-                        inode_ctx_put (inode, this, 0);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                            xdata, postparent);
-        return 0;
-}
-
-int
-trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent,
+trace_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
                    dict_t *xdata)
 {
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*stbuf = {%s}, *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": op_ret=%d, op_errno=%d",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char string[4096]  = {0,};
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*stbuf = {%s}, *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  ", *stbuf = {%s}, *prebuf = {%s}, "
-                                  "*postbuf = {%s} )",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char  string[4096]  = {0,};
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*stbuf = {%s},  *prebuf = {%s},"
-                                  " *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  statstr, preparentstr, postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    int count = 0;
+    char statstr[1024] = {
+        0,
+    };
+    char string[4096] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+    gf_dirent_t *entry = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64 " : gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+    if (op_ret < 0)
+        goto out;
+
+    list_for_each_entry(entry, &buf->list, list)
+    {
+        count++;
+        TRACE_STAT_TO_STR(&entry->d_stat, statstr);
+        snprintf(string, sizeof(string),
+                 "entry no. %d, pargfid=%s, "
+                 "bname=%s *buf {%s}",
+                 count, uuid_utoa(frame->local), entry->d_name, statstr);
+        LOG_ELEMENT(conf, string);
+    }
+
+out:
+    TRACE_STACK_UNWIND(readdirp, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s}",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
+}
+
+int
+trace_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                  struct iatt *statpost, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(statpre, preopstr);
+            TRACE_STAT_TO_STR(statpost, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
+}
+
+int
+trace_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(statpre, preopstr);
+            TRACE_STAT_TO_STR(statpost, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
+}
+
+int
+trace_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     " *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
+}
+
+int
+trace_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 struct iatt *preoldparent, struct iatt *postoldparent,
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preoldparentstr[1024] = {
+        0,
+    };
+    char postoldparentstr[1024] = {
+        0,
+    };
+    char prenewparentstr[1024] = {
+        0,
+    };
+    char postnewparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RENAME].enabled) {
+        char string[6044] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preoldparent, preoldparentstr);
+            TRACE_STAT_TO_STR(postoldparent, postoldparentstr);
+            TRACE_STAT_TO_STR(prenewparent, prenewparentstr);
+            TRACE_STAT_TO_STR(postnewparent, postnewparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*stbuf = {%s}, *preoldparent = {%s},"
+                     " *postoldparent = {%s}"
+                     " *prenewparent = {%s}, "
+                     "*postnewparent = {%s})",
+                     frame->root->unique, op_ret, statstr, preoldparentstr,
+                     postoldparentstr, prenewparentstr, postnewparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                       postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
+}
+
+int
+trace_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, const char *buf,
+                   struct iatt *stbuf, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(stbuf, statstr);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, op_errno=%d,"
+                     "buf=%s, stbuf = { %s })",
+                     frame->root->unique, op_ret, op_errno, buf, statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, stbuf, xdata);
+    return 0;
+}
+
+int
+trace_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+            /* print buf->ia_gfid instead of inode->gfid,
+             * since if the inode is not yet linked to the
+             * inode table (fresh lookup) then null gfid
+             * will be printed.
+             */
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*buf {%s}, *postparent {%s}",
+                     frame->root->unique, uuid_utoa(buf->ia_gfid), op_ret,
+                     statstr, postparentstr);
+
+            /* For 'forget' */
+            inode_ctx_put(inode, this, 0);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                       postparent);
+    return 0;
+}
+
+int
+trace_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*stbuf = {%s}, *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": op_ret=%d, op_errno=%d", frame->root->unique,
+                     op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*stbuf = {%s}, *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     ", *stbuf = {%s}, *prebuf = {%s}, "
+                     "*postbuf = {%s} )",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_LINK].enabled) {
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*stbuf = {%s},  *prebuf = {%s},"
+                     " *postbuf = {%s})",
+                     frame->root->unique, op_ret, statstr, preparentstr,
+                     postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int
+trace_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " fd=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 fd);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    /* for 'releasedir' log */
+    if (op_ret >= 0)
+        fd_ctx_set(fd, this, 0);
+
+    TRACE_STACK_UNWIND(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int
+trace_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "*prebuf={%s},  *postbuf={%s}",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
+}
+
+int
+trace_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s} )",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
+}
+
+int
+trace_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STATFS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": ({f_bsize=%lu, "
+                     "f_frsize=%lu, "
+                     "f_blocks=%" GF_PRI_FSBLK ", f_bfree=%" GF_PRI_FSBLK
+                     ", "
+                     "f_bavail=%" GF_PRI_FSBLK
+                     ", "
+                     "f_files=%" GF_PRI_FSBLK
+                     ", "
+                     "f_ffree=%" GF_PRI_FSBLK
+                     ", "
+                     "f_favail=%" GF_PRI_FSBLK
+                     ", "
+                     "f_fsid=%lu, f_flag=%lu, "
+                     "f_namemax=%lu}) => ret=%d",
+                     frame->root->unique, buf->f_bsize, buf->f_frsize,
+                     buf->f_blocks, buf->f_bfree, buf->f_bavail, buf->f_files,
+                     buf->f_ffree, buf->f_favail, buf->f_fsid, buf->f_flag,
+                     buf->f_namemax, op_ret);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char      string[4096] = {0,};
-        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+trace_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char    string[4096] = {0,};
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " fd=%p",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " dict=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 dict);
 
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        /* for 'releasedir' log */
-        if (op_ret >= 0)
-                fd_ctx_set (fd, this, 0);
-
-        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-int
-trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char           preparentstr[4096]  = {0, };
-        char           postparentstr[4096] = {0, };
-        trace_conf_t   *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "*prebuf={%s},  *postbuf={%s}",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, preparentstr, postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char           preopstr[4096]  = {0, };
-        char           postopstr[4096] = {0, };
-        trace_conf_t   *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                char   string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s} )",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
-}
+    TRACE_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
 
-int
-trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                  dict_t *xdata)
-{
-        trace_conf_t  *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                char   string[4096] = {0,};
-                if (op_ret == 0) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": ({f_bsize=%lu, "
-                                  "f_frsize=%lu, "
-                                  "f_blocks=%"GF_PRI_FSBLK
-                                  ", f_bfree=%"GF_PRI_FSBLK", "
-                                  "f_bavail=%"GF_PRI_FSBLK", "
-                                  "f_files=%"GF_PRI_FSBLK", "
-                                  "f_ffree=%"GF_PRI_FSBLK", "
-                                  "f_favail=%"GF_PRI_FSBLK", "
-                                  "f_fsid=%lu, f_flag=%lu, "
-                                  "f_namemax=%lu}) => ret=%d",
-                                  frame->root->unique, buf->f_bsize,
-                                  buf->f_frsize, buf->f_blocks,
-                                  buf->f_bfree, buf->f_bavail,
-                                  buf->f_files, buf->f_ffree,
-                                  buf->f_favail, buf->f_fsid,
-                                  buf->f_flag, buf->f_namemax, op_ret);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret,
-                          op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *dict,
                     dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " dict=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " dict=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 dict);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-
-        return 0;
-}
+    TRACE_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-int
-trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                char    string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
+trace_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                char      string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " dict=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    TRACE_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
+    TRACE_STACK_UNWIND(fsyncdir, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                char       string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, "
+                 "op_errno=%d)",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int
+trace_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+    char prebufstr[1024] = {
+        0,
+    };
+    char postbufstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, prebufstr);
+            TRACE_STAT_TO_STR(postbuf, postbufstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s} )",
+                     frame->root->unique, op_ret, prebufstr, postbufstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
+}
+
+int
+trace_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+        char string[4096] = {0.};
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d "
+                     "buf=%s",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "{l_type=%d, l_whence=%d, "
+                     "l_start=%" PRId64
+                     ", "
+                     "l_len=%" PRId64 ", l_pid=%u})",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     lock->l_type, lock->l_whence, lock->l_start, lock->l_len,
+                     lock->l_pid);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(lk, frame, op_ret, op_errno, lock, xdata);
+    return 0;
+}
+
+int
+trace_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, "
-                          "op_errno=%d)", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char          prebufstr[4096]  = {0, };
-        char          postbufstr[4096] = {0, };
-        trace_conf_t  *conf            = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, prebufstr);
-                        TRACE_STAT_TO_STR (postbuf, postbufstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s} )",
-                                  frame->root->unique, op_ret,
-                                  prebufstr, postbufstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
-}
-
-int
-trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                char string[4096]  = {0.};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d "
-                                  "buf=%s", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-              dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LK].enabled) {
-                char      string[4096] = {0,};
-                if (op_ret == 0) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "{l_type=%d, l_whence=%d, "
-                                  "l_start=%"PRId64", "
-                                  "l_len=%"PRId64", l_pid=%u})",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, lock->l_type, lock->l_whence,
-                                  lock->l_start, lock->l_len,
-                                  lock->l_pid);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
-}
-
-int
-trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                char   string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int
-trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *dict,
                    dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                char    string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int
-trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict,
-                    dict_t *xdata)
+trace_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_INODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local),op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     uint32_t weak_checksum, uint8_t *strong_checksum,
-                     dict_t *xdata)
+trace_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                    uint8_t *strong_checksum, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                            strong_checksum, xdata);
+    TRACE_STACK_UNWIND(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                       strong_checksum, xdata);
 
-        return 0;
+    return 0;
 }
 
 /* *_cbk section over <----------> fop section start */
 
 int
-trace_entrylk (call_frame_t *frame, xlator_t *this,
-               const char *volume, loc_t *loc, const char *basename,
-               entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+trace_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                char     string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (path=%s "
-                          "basename=%s, cmd=%s, type=%s)",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid),
-                          volume, loc->path, basename,
-                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
-                           "ENTRYLK_UNLOCK"),
-                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
-                           "ENTRYLK_WRLCK"));
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (path=%s "
+                 "basename=%s, cmd=%s, type=%s)",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), volume,
+                 loc->path, basename,
+                 ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
+                 ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_entrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
-        return 0;
+    STACK_WIND(frame, trace_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
 int
-trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-               loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+trace_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char         *cmd_str  = NULL;
-        char         *type_str = NULL;
-        trace_conf_t *conf     = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                char string[4096]  = {0,};
-                switch (cmd) {
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_INODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        switch (cmd) {
 #if F_GETLK != F_GETLK64
-                case F_GETLK64:
+            case F_GETLK64:
 #endif
-                case F_GETLK:
-                        cmd_str = "GETLK";
-                        break;
+            case F_GETLK:
+                cmd_str = "GETLK";
+                break;
 
 #if F_SETLK != F_SETLK64
-                case F_SETLK64:
+            case F_SETLK64:
 #endif
-                case F_SETLK:
-                        cmd_str = "SETLK";
-                        break;
+            case F_SETLK:
+                cmd_str = "SETLK";
+                break;
 
 #if F_SETLKW != F_SETLKW64
-                case F_SETLKW64:
+            case F_SETLKW64:
 #endif
-                case F_SETLKW:
-                        cmd_str = "SETLKW";
-                        break;
-
-                default:
-                        cmd_str = "UNKNOWN";
-                        break;
-                }
-
-                switch (flock->l_type) {
-                case F_RDLCK:
-                        type_str = "READ";
-                        break;
-                case F_WRLCK:
-                        type_str = "WRITE";
-                        break;
-                case F_UNLCK:
-                        type_str = "UNLOCK";
-                        break;
-                default:
-                        type_str = "UNKNOWN";
-                        break;
-                }
-
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (path=%s "
-                          "cmd=%s, type=%s, start=%llu, len=%llu, "
-                          "pid=%llu)", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), volume,
-                          loc->path, cmd_str, type_str,
-                          (unsigned long long)flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid);
-
-                frame->local = loc->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
+            case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+            default:
+                cmd_str = "UNKNOWN";
+                break;
+        }
+
+        switch (flock->l_type) {
+            case F_RDLCK:
+                type_str = "READ";
+                break;
+            case F_WRLCK:
+                type_str = "WRITE";
+                break;
+            case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+            default:
+                type_str = "UNKNOWN";
+                break;
         }
 
+        snprintf(
+            string, sizeof(string),
+            "%" PRId64
+            ": gfid=%s volume=%s, (path=%s "
+            "cmd=%s, type=%s, start=%llu, len=%llu, "
+            "pid=%llu)",
+            frame->root->unique, uuid_utoa(loc->inode->gfid), volume, loc->path,
+            cmd_str, type_str, (unsigned long long)flock->l_start,
+            (unsigned long long)flock->l_len, (unsigned long long)flock->l_pid);
+
+        frame->local = loc->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
+
 out:
-        STACK_WIND (frame, trace_inodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
 }
 
 int
-trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+trace_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char          *cmd_str  = NULL;
-        char          *type_str = NULL;
-        trace_conf_t  *conf     = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                char  string[4096] = {0,};
-                switch (cmd) {
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        switch (cmd) {
 #if F_GETLK != F_GETLK64
-                case F_GETLK64:
+            case F_GETLK64:
 #endif
-                case F_GETLK:
-                        cmd_str = "GETLK";
-                        break;
+            case F_GETLK:
+                cmd_str = "GETLK";
+                break;
 
 #if F_SETLK != F_SETLK64
-                case F_SETLK64:
+            case F_SETLK64:
 #endif
-                case F_SETLK:
-                        cmd_str = "SETLK";
-                        break;
+            case F_SETLK:
+                cmd_str = "SETLK";
+                break;
 
 #if F_SETLKW != F_SETLKW64
-                case F_SETLKW64:
+            case F_SETLKW64:
 #endif
-                case F_SETLKW:
-                        cmd_str = "SETLKW";
-                        break;
-
-                default:
-                        cmd_str = "UNKNOWN";
-                        break;
-                }
-
-                switch (flock->l_type) {
-                case F_RDLCK:
-                        type_str = "READ";
-                        break;
-                case F_WRLCK:
-                        type_str = "WRITE";
-                        break;
-                case F_UNLCK:
-                        type_str = "UNLOCK";
-                        break;
-                default:
-                        type_str = "UNKNOWN";
-                        break;
-                }
-
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
-                          "cmd=%s, type=%s, start=%llu, len=%llu, "
-                          "pid=%llu)", frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), volume, fd,
-                          cmd_str, type_str,
-                          (unsigned long long) flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid);
-
-                frame->local = fd->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
+            case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+            default:
+                cmd_str = "UNKNOWN";
+                break;
         }
+
+        switch (flock->l_type) {
+            case F_RDLCK:
+                type_str = "READ";
+                break;
+            case F_WRLCK:
+                type_str = "WRITE";
+                break;
+            case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+            default:
+                type_str = "UNKNOWN";
+                break;
+        }
+
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (fd =%p "
+                 "cmd=%s, type=%s, start=%llu, len=%llu, "
+                 "pid=%llu)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), volume, fd,
+                 cmd_str, type_str, (unsigned long long)flock->l_start,
+                 (unsigned long long)flock->l_len,
+                 (unsigned long long)flock->l_pid);
+
+        frame->local = fd->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_finodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
 }
 
 int
-trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+trace_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s (path=%s flags=%d)",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s (path=%s flags=%d)", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_xattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict, xdata);
+    STACK_WIND(frame, trace_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+trace_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                char    string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p, flags=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fxattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict, xdata);
+    STACK_WIND(frame, trace_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xdata)
+trace_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char string[4096] = {0,};
-                /* TODO: print all the keys mentioned in xattr_req */
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        /* TODO: print all the keys mentioned in xattr_req */
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
+    STACK_WIND(frame, trace_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+trace_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_STAT].enabled) {
-                char  string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
+    STACK_WIND(frame, trace_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                dict_t *xdata)
+trace_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, "
-                          "size=%"GF_PRI_SIZET")", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          size);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, "
+                 "size=%" GF_PRI_SIZET ")",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 size);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
+    STACK_WIND(frame, trace_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+trace_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t dev, mode_t umask, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mode=%d "
-                          "umask=0%o, dev=%"GF_PRI_DEV")",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          mode, umask, dev);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s mode=%d "
+                 "umask=0%o, dev=%" GF_PRI_DEV ")",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 mode, umask, dev);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_mknod_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
+    STACK_WIND(frame, trace_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             mode_t umask, dict_t *xdata)
+trace_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mode=%d"
-                          " umask=0%o", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          mode, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s mode=%d"
+                 " umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 mode, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_mkdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    STACK_WIND(frame, trace_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 }
 
 int
-trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+trace_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flag=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          xflag);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s flag=%d",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 xflag);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_unlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, trace_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
 int
-trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-             dict_t *xdata)
+trace_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rmdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
+    STACK_WIND(frame, trace_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, mode_t umask, dict_t *xdata)
+trace_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                char     string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
-                          " umask=0%o", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), linkpath,
-                          loc->path, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s linkpath=%s, path=%s"
+                 " umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), linkpath,
+                 loc->path, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_symlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
+    STACK_WIND(frame, trace_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata)
+trace_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        char         oldgfid[50] = {0,};
-        char         newgfid[50] = {0,};
-        trace_conf_t *conf       = NULL;
+    char oldgfid[50] = {
+        0,
+    };
+    char newgfid[50] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                char string[4096] = {0,};
-                if (newloc->inode)
-                        uuid_utoa_r (newloc->inode->gfid, newgfid);
-                else
-                        strcpy (newgfid, "0");
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RENAME].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (newloc->inode)
+            uuid_utoa_r(newloc->inode->gfid, newgfid);
+        else
+            strcpy(newgfid, "0");
 
-                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+        uuid_utoa_r(oldloc->inode->gfid, oldgfid);
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": oldgfid=%s oldpath=%s --> "
-                          "newgfid=%s newpath=%s",
-                          frame->root->unique, oldgfid,
-                          oldloc->path, newgfid, newloc->path);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": oldgfid=%s oldpath=%s --> "
+                 "newgfid=%s newpath=%s",
+                 frame->root->unique, oldgfid, oldloc->path, newgfid,
+                 newloc->path);
 
-                frame->local = oldloc->inode->gfid;
+        frame->local = oldloc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
+    STACK_WIND(frame, trace_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-            dict_t *xdata)
+trace_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        char         oldgfid[50] = {0,};
-        char         newgfid[50] = {0,};
-        trace_conf_t *conf       = NULL;
+    char oldgfid[50] = {
+        0,
+    };
+    char newgfid[50] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                char string[4096]  = {0,};
-                if (newloc->inode)
-                        uuid_utoa_r (newloc->inode->gfid, newgfid);
-                else
-                        strcpy (newgfid, "0");
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (newloc->inode)
+            uuid_utoa_r(newloc->inode->gfid, newgfid);
+        else
+            strcpy(newgfid, "0");
 
-                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+        uuid_utoa_r(oldloc->inode->gfid, oldgfid);
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": oldgfid=%s oldpath=%s --> "
-                          "newgfid=%s newpath=%s", frame->root->unique,
-                          oldgfid, oldloc->path, newgfid,
-                          newloc->path);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": oldgfid=%s oldpath=%s --> "
+                 "newgfid=%s newpath=%s",
+                 frame->root->unique, oldgfid, oldloc->path, newgfid,
+                 newloc->path);
 
-                frame->local = oldloc->inode->gfid;
+        frame->local = oldloc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_link_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, trace_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+trace_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char         actime_str[256]  = {0,};
-        char         modtime_str[256] = {0,};
-        trace_conf_t *conf            = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                char     string[4096]  =  {0,};
-                if (valid & GF_SET_ATTR_MODE) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s mode=%o)",
-                                  frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path,
-                                  st_mode_from_ia (stbuf->ia_prot,
-                                                   stbuf->ia_type));
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-
-                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s uid=%o,"
-                                  " gid=%o", frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, stbuf->ia_uid,
-                                  stbuf->ia_gid);
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-
-                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
-
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s "
-                                  "ia_atime=%s, ia_mtime=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, actime_str, modtime_str);
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-                frame->local = loc->inode->gfid;
+    char actime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char modtime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (valid & GF_SET_ATTR_MODE) {
+            snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s path=%s mode=%o)",
+                frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                st_mode_from_ia(stbuf->ia_prot, stbuf->ia_type));
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s path=%s uid=%o,"
+                     " gid=%o",
+                     frame->root->unique, uuid_utoa(loc->inode->gfid),
+                     loc->path, stbuf->ia_uid, stbuf->ia_gid);
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
         }
 
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            gf_time_fmt(actime_str, sizeof actime_str, stbuf->ia_atime,
+                        gf_timefmt_bdT);
+
+            gf_time_fmt(modtime_str, sizeof modtime_str, stbuf->ia_mtime,
+                        gf_timefmt_bdT);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s path=%s "
+                     "ia_atime=%s, ia_mtime=%s",
+                     frame->root->unique, uuid_utoa(loc->inode->gfid),
+                     loc->path, actime_str, modtime_str);
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+        frame->local = loc->inode->gfid;
+    }
+
 out:
-        STACK_WIND (frame, trace_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
+    STACK_WIND(frame, trace_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+trace_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char           actime_str[256]  = {0,};
-        char           modtime_str[256] = {0,};
-        trace_conf_t   *conf            = NULL;
+    char actime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char modtime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (valid & GF_SET_ATTR_MODE) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": gfid=%s fd=%p, mode=%o", frame->root->unique,
+                     uuid_utoa(fd->inode->gfid), fd,
+                     st_mode_from_ia(stbuf->ia_prot, stbuf->ia_type));
 
-        conf = this->private;
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                char     string[4096]  =  {0,};
-                if (valid & GF_SET_ATTR_MODE) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
-                                  frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid), fd,
-                                  st_mode_from_ia (stbuf->ia_prot,
-                                                   stbuf->ia_type));
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s fd=%p, uid=%o, "
+                     "gid=%o",
+                     frame->root->unique, uuid_utoa(fd->inode->gfid), fd,
+                     stbuf->ia_uid, stbuf->ia_gid);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
 
-                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
-                                  "gid=%o", frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid),
-                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            gf_time_fmt(actime_str, sizeof actime_str, stbuf->ia_atime,
+                        gf_timefmt_bdT);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
+            gf_time_fmt(modtime_str, sizeof modtime_str, stbuf->ia_mtime,
+                        gf_timefmt_bdT);
 
-                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s fd=%p "
+                     "ia_atime=%s, ia_mtime=%s",
+                     frame->root->unique, uuid_utoa(fd->inode->gfid), fd,
+                     actime_str, modtime_str);
 
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+        frame->local = fd->inode->gfid;
+    }
 
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p "
-                                  "ia_atime=%s, ia_mtime=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid),
-                                  fd, actime_str, modtime_str);
+out:
+    STACK_WIND(frame, trace_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
-                frame->local = fd->inode->gfid;
-        }
+    return 0;
+}
+
+static int
+trace_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    trace_conf_t *conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SEEK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d op_errno=%d, "
+                 "offset=%" PRId64 "",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 offset);
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
+
+static int
+trace_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
+{
+    trace_conf_t *conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SEEK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p "
+                 "offset=%" PRId64 " what=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, offset,
+                 what);
 
+        frame->local = fd->inode->gfid;
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_fsetattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
+    STACK_WIND(frame, trace_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset, dict_t *xdata)
+trace_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, "
-                          "offset=%"PRId64"", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          offset);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, "
+                 "offset=%" PRId64 "",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 offset);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_truncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
+    STACK_WIND(frame, trace_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, dict_t *xdata)
+trace_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                char      string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d fd=%p",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 flags, fd);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_open_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 int
-trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-              dict_t *xdata)
+trace_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, fd=%p, "
-                          "flags=0%o mode=0%o umask=0%o",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          fd, flags, mode, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, fd=%p, "
+                 "flags=0%o mode=0%o umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 fd, flags, mode, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
 int
-trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+trace_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READ].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"
-                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READ].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 "offset=%" PRId64 " flags=0%x)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readv_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int
-trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count,
-              off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
+trace_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
-        int i = 0;
-        size_t total_size = 0;
+    trace_conf_t *conf = NULL;
+    int i = 0;
+    size_t total_size = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                char     string[4096]  =  {0,};
-                for (i = 0; i < count; i++)
-                        total_size += vector[i].iov_len;
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_WRITE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        for (i = 0; i < count; i++)
+            total_size += vector[i].iov_len;
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, count=%d, "
-                          " offset=%"PRId64" flags=0%x write_size=%lu",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, count,
-                          offset, flags, total_size);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p, count=%d, "
+                 " offset=%" PRId64 " flags=0%x write_size=%zu",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, count,
+                 offset, flags, total_size);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_writev_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, trace_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 int
-trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+trace_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                char  string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique, (loc->inode)?
-                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STATFS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique,
+                 (loc->inode) ? uuid_utoa(loc->inode->gfid) : "0", loc->path);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_statfs_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, trace_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
 int
-trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+trace_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_flush_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
 int
-trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+trace_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s flags=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), flags, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s flags=%d fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), flags, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsync_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 }
 
 int
-trace_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+trace_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
 int
-trace_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name, dict_t *xdata)
+trace_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s name=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s name=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 name);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, trace_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
 int
-trace_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+trace_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s name=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s name=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 name);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
+    STACK_WIND(frame, trace_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-               dict_t *xdata)
+trace_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+              dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s fd=%p",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 fd);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_opendir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset, dict_t *dict)
+trace_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *dict)
 {
-        trace_conf_t *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
-                          ", offset=%"PRId64" dict=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset, dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 ", offset=%" PRId64 " dict=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset, dict);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
+    STACK_WIND(frame, trace_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset, dict_t *xdata)
+trace_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
-                          ", offset=%"PRId64,
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 ", offset=%" PRId64,
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset, xdata);
+    STACK_WIND(frame, trace_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fsyncdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t datasync, dict_t *xdata)
+trace_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s datasync=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), datasync, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s datasync=%d fd=%p", frame->root->unique,
+                 uuid_utoa(fd->inode->gfid), datasync, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsyncdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
 }
 
 int
-trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-              dict_t *xdata)
+trace_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+             dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mask=0%o",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid),
-                          loc->path, mask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s mask=0%o", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, mask);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_access_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access,
-                    loc, mask, xdata);
-        return 0;
+    STACK_WIND(frame, trace_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
 }
 
 int32_t
-trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 int32_t len, dict_t *xdata)
+trace_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                int32_t len, dict_t *xdata)
 {
+    trace_conf_t *conf = NULL;
 
-        trace_conf_t *conf = NULL;
-
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s offset=%"PRId64
-                          "len=%u fd=%p", frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s offset=%" PRId64 "len=%u fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), offset, len,
+                 fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rchecksum_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum,
-                    fd, offset, len, xdata);
-
-        return 0;
+    STACK_WIND(frame, trace_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
 
+    return 0;
 }
 
 int32_t
-trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, const char *basename, entrylk_cmd cmd,
-                entrylk_type type, dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                char      string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
-                          "basename=%s, cmd=%s, type=%s)",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), volume, fd,
-                          basename,
-                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
-                           "ENTRYLK_UNLOCK"),
-                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
-                           "ENTRYLK_WRLCK"));
-
-                frame->local = fd->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
-        }
+trace_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, const char *basename, entrylk_cmd cmd,
+               entrylk_type type, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
 
-out:
-        STACK_WIND (frame, trace_fentrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type, xdata);
-        return 0;
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (fd=%p "
+                 "basename=%s, cmd=%s, type=%s)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), volume, fd,
+                 basename,
+                 ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
+                 ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+
+        frame->local = fd->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
 
+out:
+    STACK_WIND(frame, trace_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
 int32_t
-trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+trace_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p name=%s",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p name=%s",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, name);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fgetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
+trace_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p flags=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
 int
-trace_ftruncate (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, off_t offset, dict_t *xdata)
+trace_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), offset, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s offset=%" PRId64 " fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), offset, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_ftruncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
+    STACK_WIND(frame, trace_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+trace_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fstat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
 int
-trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-          int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+trace_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+         struct gf_flock *lock, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LK].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
-                          "lock {l_type=%d, "
-                          "l_whence=%d, l_start=%"PRId64", "
-                          "l_len=%"PRId64", l_pid=%u})",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, cmd,
-                          lock->l_type, lock->l_whence,
-                          lock->l_start, lock->l_len, lock->l_pid);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p, cmd=%d, "
+                 "lock {l_type=%d, "
+                 "l_whence=%d, l_start=%" PRId64
+                 ", "
+                 "l_len=%" PRId64 ", l_pid=%u})",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, cmd,
+                 lock->l_type, lock->l_whence, lock->l_start, lock->l_len,
+                 lock->l_pid);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_lk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
+    return 0;
 }
 
 int32_t
-trace_forget (xlator_t *this, inode_t *inode)
+trace_forget(xlator_t *this, inode_t *inode)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
-        /* If user want to understand when a lookup happens,
-           he should know about 'forget' too */
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s", uuid_utoa (inode->gfid));
+    conf = this->private;
+    /* If user want to understand when a lookup happens,
+       he should know about 'forget' too */
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s", uuid_utoa(inode->gfid));
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-trace_releasedir (xlator_t *this, fd_t *fd)
+trace_releasedir(xlator_t *this, fd_t *fd)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s fd=%p",
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s fd=%p",
+                 uuid_utoa(fd->inode->gfid), fd);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-trace_release (xlator_t *this, fd_t *fd)
+trace_release(xlator_t *this, fd_t *fd)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled ||
-            trace_fop_names[GF_FOP_CREATE].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s fd=%p",
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled ||
+        trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s fd=%p",
+                 uuid_utoa(fd->inode->gfid), fd);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 void
-enable_all_calls (int enabled)
+enable_all_calls(int enabled)
 {
-        int i;
+    int i;
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                trace_fop_names[i].enabled = enabled;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        trace_fop_names[i].enabled = enabled;
 }
 
 void
-enable_call (const char *name, int enabled)
+enable_call(const char *name, int enabled)
 {
-        int i;
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                if (!strcasecmp(trace_fop_names[i].name, name))
-                        trace_fop_names[i].enabled = enabled;
+    int i;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        if (!strcasecmp(trace_fop_names[i].name, name))
+            trace_fop_names[i].enabled = enabled;
 }
 
-
 /*
   include = 1 for "include-ops"
   = 0 for "exclude-ops"
 */
 void
-process_call_list (const char *list, int include)
+process_call_list(const char *list, int include)
 {
-        enable_all_calls (include ? 0 : 1);
+    enable_all_calls(include ? 0 : 1);
 
-        char *call = strsep ((char **)&list, ",");
+    char *call = strsep((char **)&list, ",");
 
-        while (call) {
-                enable_call (call, include);
-                call = strsep ((char **)&list, ",");
-        }
+    while (call) {
+        enable_call(call, include);
+        call = strsep((char **)&list, ",");
+    }
 }
 
 int32_t
-trace_dump_history (xlator_t *this)
-{
-        int ret = -1;
-        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
-        trace_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("trace", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
-
-        conf = this->private;
-        // Is it ok to return silently if log-history option his off?
-        if (conf && conf->log_history == _gf_true) {
-                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
-                                        "history");
-                gf_proc_dump_add_section (key_prefix);
-                eh_dump (this->history, NULL, dump_history_trace);
-        }
-        ret = 0;
+trace_dump_history(xlator_t *this)
+{
+    int ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("trace", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->history, out);
+
+    conf = this->private;
+    // Is it ok to return silently if log-history option his off?
+    if (conf && conf->log_history == _gf_true) {
+        gf_proc_dump_build_key(key_prefix, "xlator.debug.trace", "history");
+        gf_proc_dump_add_section("%s", key_prefix);
+        eh_dump(this->history, NULL, dump_history_trace);
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_trace_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t       ret            = -1;
-        trace_conf_t    *conf           = NULL;
-        char            *includes = NULL, *excludes = NULL;
+    int32_t ret = -1;
+    trace_conf_t *conf = NULL;
+    char *includes = NULL, *excludes = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
+    GF_VALIDATE_OR_GOTO("quick-read", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
 
-        conf = this->private;
+    conf = this->private;
 
-        includes = data_to_str (dict_get (options, "include-ops"));
-        excludes = data_to_str (dict_get (options, "exclude-ops"));
+    includes = data_to_str(dict_get(options, "include-ops"));
+    excludes = data_to_str(dict_get(options, "exclude-ops"));
 
-        {
-                int i;
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        if (gf_fop_list[i])
-                                strncpy (trace_fop_names[i].name,
-                                         gf_fop_list[i],
-                                         strlen (gf_fop_list[i]));
-                        else
-                                strncpy (trace_fop_names[i].name, ":O",
-                                         strlen (":O"));
-                        trace_fop_names[i].enabled = 1;
-                }
+    {
+        int i;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            if (gf_fop_list[i])
+                strncpy(trace_fop_names[i].name, gf_fop_list[i],
+                        sizeof(trace_fop_names[i].name));
+            else
+                strncpy(trace_fop_names[i].name, ":0",
+                        sizeof(trace_fop_names[i].name));
+            trace_fop_names[i].enabled = 1;
+            trace_fop_names[i].name[sizeof(trace_fop_names[i].name) - 1] = 0;
         }
+    }
 
-        if (includes && excludes) {
-                gf_log (this->name,
-                        GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and "
-                        "'exclude-ops'");
-                goto out;
-        }
+    if (includes && excludes) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "must specify only one of 'include-ops' and "
+               "'exclude-ops'");
+        goto out;
+    }
 
-        if (includes)
-                process_call_list (includes, 1);
-        if (excludes)
-                process_call_list (excludes, 0);
+    if (includes)
+        process_call_list(includes, 1);
+    if (excludes)
+        process_call_list(excludes, 0);
 
-        /* Should resizing of the event-history be allowed in reconfigure?
-         * for which a new event_history might have to be allocated and the
-         * older history has to be freed.
-         */
-        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+    /* Should resizing of the event-history be allowed in reconfigure?
+     * for which a new event_history might have to be allocated and the
+     * older history has to be freed.
+     */
+    GF_OPTION_RECONF("log-file", conf->log_file, options, bool, out);
 
-        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
+    GF_OPTION_RECONF("log-history", conf->log_history, options, bool, out);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
-{
-        dict_t *options = NULL;
-        char *includes = NULL, *excludes = NULL;
-        char *forced_loglevel = NULL;
-        eh_t *history = NULL;
-        int  ret = -1;
-        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
-        trace_conf_t    *conf = NULL;
-
-        if (!this)
-                return -1;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "trace translator requires one subvolume");
-                return -1;
-        }
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
-                        "xl->private");
-                return -1;
-        }
-
-        options = this->options;
-        includes = data_to_str (dict_get (options, "include-ops"));
-        excludes = data_to_str (dict_get (options, "exclude-ops"));
-
-        {
-                int i;
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        if (gf_fop_list[i])
-                                strncpy (trace_fop_names[i].name,
-                                         gf_fop_list[i],
-                                         strlen (gf_fop_list[i]));
-                        else
-                                strncpy (trace_fop_names[i].name, ":O",
-                                         strlen (":O"));
-                        trace_fop_names[i].enabled = 1;
-                }
-        }
-
-        if (includes && excludes) {
-                gf_log (this->name,
-                        GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and "
-                        "'exclude-ops'");
-                return -1;
-        }
-
-        if (includes)
-                process_call_list (includes, 1);
-        if (excludes)
-                process_call_list (excludes, 0);
-
-
-        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
-
-        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
-                history_size);
-
-        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
-
-        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
-                (conf->log_file == _gf_true)?"enabled":"disabled");
-
-        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
-
-        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
-                (conf->log_history == _gf_true)?"enabled":"disabled");
-
-        history = eh_new (history_size, _gf_false, NULL);
-        if (!history) {
-                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
-                        "initialized");
-                return -1;
-        }
-
-        this->history = history;
-
-        conf->trace_log_level = GF_LOG_INFO;
-
-        if (dict_get (options, "force-log-level")) {
-                forced_loglevel = data_to_str (dict_get (options,
-                                                         "force-log-level"));
-                if (!forced_loglevel)
-                        goto setloglevel;
-
-                if (strcmp (forced_loglevel, "INFO") == 0)
-                        conf->trace_log_level = GF_LOG_INFO;
-                else if (strcmp (forced_loglevel, "TRACE") == 0)
-                        conf->trace_log_level = GF_LOG_TRACE;
-                else if (strcmp (forced_loglevel, "ERROR") == 0)
-                        conf->trace_log_level = GF_LOG_ERROR;
-                else if (strcmp (forced_loglevel, "DEBUG") == 0)
-                        conf->trace_log_level = GF_LOG_DEBUG;
-                else if (strcmp (forced_loglevel, "WARNING") == 0)
-                        conf->trace_log_level = GF_LOG_WARNING;
-                else if (strcmp (forced_loglevel, "CRITICAL") == 0)
-                        conf->trace_log_level = GF_LOG_CRITICAL;
-                else if (strcmp (forced_loglevel, "NONE") == 0)
-                        conf->trace_log_level = GF_LOG_NONE;
-        }
+init(xlator_t *this)
+{
+    dict_t *options = NULL;
+    char *includes = NULL, *excludes = NULL;
+    char *forced_loglevel = NULL;
+    eh_t *history = NULL;
+    int ret = -1;
+    uint64_t history_size = TRACE_DEFAULT_HISTORY_SIZE;
+    trace_conf_t *conf = NULL;
+
+    if (!this)
+        return -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "trace translator requires one subvolume");
+        return -1;
+    }
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    conf = GF_CALLOC(1, sizeof(trace_conf_t), gf_trace_mt_trace_conf_t);
+    if (!conf) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "cannot allocate "
+               "xl->private");
+        return -1;
+    }
+
+    options = this->options;
+    includes = data_to_str(dict_get(options, "include-ops"));
+    excludes = data_to_str(dict_get(options, "exclude-ops"));
+
+    {
+        int i;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            if (gf_fop_list[i])
+                strncpy(trace_fop_names[i].name, gf_fop_list[i],
+                        sizeof(trace_fop_names[i].name));
+            else
+                strncpy(trace_fop_names[i].name, ":O",
+                        sizeof(trace_fop_names[i].name));
+            trace_fop_names[i].enabled = 1;
+            trace_fop_names[i].name[sizeof(trace_fop_names[i].name) - 1] = 0;
+        }
+    }
+
+    if (includes && excludes) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "must specify only one of 'include-ops' and "
+               "'exclude-ops'");
+        return -1;
+    }
+
+    if (includes)
+        process_call_list(includes, 1);
+    if (excludes)
+        process_call_list(excludes, 0);
+
+    GF_OPTION_INIT("history-size", history_size, size, out);
+    conf->history_size = history_size;
+
+    gf_log(this->name, GF_LOG_INFO, "history size %" PRIu64, history_size);
+
+    GF_OPTION_INIT("log-file", conf->log_file, bool, out);
+
+    gf_log(this->name, GF_LOG_INFO, "logging to file %s",
+           (conf->log_file == _gf_true) ? "enabled" : "disabled");
+
+    GF_OPTION_INIT("log-history", conf->log_history, bool, out);
+
+    gf_log(this->name, GF_LOG_DEBUG, "logging to history %s",
+           (conf->log_history == _gf_true) ? "enabled" : "disabled");
+
+    history = eh_new(history_size, _gf_false, NULL);
+    if (!history) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "event history cannot be "
+               "initialized");
+        return -1;
+    }
+
+    this->history = history;
+
+    conf->trace_log_level = GF_LOG_INFO;
+
+    if (dict_get(options, "force-log-level")) {
+        forced_loglevel = data_to_str(dict_get(options, "force-log-level"));
+        if (!forced_loglevel)
+            goto setloglevel;
+
+        if (strcmp(forced_loglevel, "INFO") == 0)
+            conf->trace_log_level = GF_LOG_INFO;
+        else if (strcmp(forced_loglevel, "TRACE") == 0)
+            conf->trace_log_level = GF_LOG_TRACE;
+        else if (strcmp(forced_loglevel, "ERROR") == 0)
+            conf->trace_log_level = GF_LOG_ERROR;
+        else if (strcmp(forced_loglevel, "DEBUG") == 0)
+            conf->trace_log_level = GF_LOG_DEBUG;
+        else if (strcmp(forced_loglevel, "WARNING") == 0)
+            conf->trace_log_level = GF_LOG_WARNING;
+        else if (strcmp(forced_loglevel, "CRITICAL") == 0)
+            conf->trace_log_level = GF_LOG_CRITICAL;
+        else if (strcmp(forced_loglevel, "NONE") == 0)
+            conf->trace_log_level = GF_LOG_NONE;
+    }
 
 setloglevel:
-        gf_log_set_loglevel (conf->trace_log_level);
-        this->private = conf;
-        ret = 0;
-out:
-        if (ret == -1) {
-                if (history)
-                        GF_FREE (history);
-                if (conf)
-                        GF_FREE (conf);
-        }
+    gf_log_set_loglevel(this->ctx, conf->trace_log_level);
+    this->private = conf;
+    ret = 0;
+out:
+    if (ret == -1) {
+        if (history)
+            GF_FREE(history);
+        if (conf)
+            GF_FREE(conf);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        if (this->history)
-                eh_destroy (this->history);
+    if (this->history)
+        eh_destroy(this->history);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "trace translator unloaded");
-        return;
+    gf_log(this->name, GF_LOG_INFO, "trace translator unloaded");
+    return;
 }
 
 struct xlator_fops fops = {
-        .stat        = trace_stat,
-        .readlink    = trace_readlink,
-        .mknod       = trace_mknod,
-        .mkdir       = trace_mkdir,
-        .unlink      = trace_unlink,
-        .rmdir       = trace_rmdir,
-        .symlink     = trace_symlink,
-        .rename      = trace_rename,
-        .link        = trace_link,
-        .truncate    = trace_truncate,
-        .open        = trace_open,
-        .readv       = trace_readv,
-        .writev      = trace_writev,
-        .statfs      = trace_statfs,
-        .flush       = trace_flush,
-        .fsync       = trace_fsync,
-        .setxattr    = trace_setxattr,
-        .getxattr    = trace_getxattr,
-        .fsetxattr   = trace_fsetxattr,
-        .fgetxattr   = trace_fgetxattr,
-        .removexattr = trace_removexattr,
-        .opendir     = trace_opendir,
-        .readdir     = trace_readdir,
-        .readdirp    = trace_readdirp,
-        .fsyncdir    = trace_fsyncdir,
-        .access      = trace_access,
-        .ftruncate   = trace_ftruncate,
-        .fstat       = trace_fstat,
-        .create      = trace_create,
-        .lk          = trace_lk,
-        .inodelk     = trace_inodelk,
-        .finodelk    = trace_finodelk,
-        .entrylk     = trace_entrylk,
-        .fentrylk    = trace_fentrylk,
-        .lookup      = trace_lookup,
-        .rchecksum   = trace_rchecksum,
-        .xattrop     = trace_xattrop,
-        .fxattrop    = trace_fxattrop,
-        .setattr     = trace_setattr,
-        .fsetattr    = trace_fsetattr,
+    .stat = trace_stat,
+    .readlink = trace_readlink,
+    .mknod = trace_mknod,
+    .mkdir = trace_mkdir,
+    .unlink = trace_unlink,
+    .rmdir = trace_rmdir,
+    .symlink = trace_symlink,
+    .rename = trace_rename,
+    .link = trace_link,
+    .truncate = trace_truncate,
+    .open = trace_open,
+    .readv = trace_readv,
+    .writev = trace_writev,
+    .statfs = trace_statfs,
+    .flush = trace_flush,
+    .fsync = trace_fsync,
+    .setxattr = trace_setxattr,
+    .getxattr = trace_getxattr,
+    .fsetxattr = trace_fsetxattr,
+    .fgetxattr = trace_fgetxattr,
+    .removexattr = trace_removexattr,
+    .opendir = trace_opendir,
+    .readdir = trace_readdir,
+    .readdirp = trace_readdirp,
+    .fsyncdir = trace_fsyncdir,
+    .access = trace_access,
+    .ftruncate = trace_ftruncate,
+    .fstat = trace_fstat,
+    .create = trace_create,
+    .lk = trace_lk,
+    .inodelk = trace_inodelk,
+    .finodelk = trace_finodelk,
+    .entrylk = trace_entrylk,
+    .fentrylk = trace_fentrylk,
+    .lookup = trace_lookup,
+    .rchecksum = trace_rchecksum,
+    .xattrop = trace_xattrop,
+    .fxattrop = trace_fxattrop,
+    .setattr = trace_setattr,
+    .fsetattr = trace_fsetattr,
+    .seek = trace_seek,
 };
 
 struct xlator_cbks cbks = {
-        .release     = trace_release,
-        .releasedir  = trace_releasedir,
-        .forget      = trace_forget,
+    .release = trace_release,
+    .releasedir = trace_releasedir,
+    .forget = trace_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"include-ops", "include"},
-          .type = GF_OPTION_TYPE_STR,
-          /*.value = { ""} */
-        },
-        { .key  = {"exclude-ops", "exclude"},
-          .type = GF_OPTION_TYPE_STR
-          /*.value = { ""} */
-        },
-        { .key  = {"history-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .default_value = "1024",
-        },
-        { .key  = {"log-file"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-        },
-        { .key  = {"log-history"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-        },
-        { .key  = {NULL} },
+    {
+        .key = {"include-ops", "include"},
+        .type = GF_OPTION_TYPE_STR,
+        /*.value = { ""} */
+    },
+    {
+        .key = {"exclude-ops", "exclude"},
+        .type = GF_OPTION_TYPE_STR
+        /*.value = { ""} */
+    },
+    {
+        .key = {"history-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .default_value = "1024",
+    },
+    {
+        .key = {"log-file"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+    },
+    {
+        .key = {"log-history"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+    },
+    {.key = {NULL}},
 };
 
-struct xlator_dumpops dumpops = {
-        .history = trace_dump_history
+struct xlator_dumpops dumpops = {.history = trace_dump_history};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "trace",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
index 3b5f7891d00..b16304799da 100644
--- a/xlators/debug/trace/src/trace.h
+++ b/xlators/debug/trace/src/trace.h
@@ -10,47 +10,46 @@
 
 #include <time.h>
 #include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-#include "event-history.h"
-#include "logging.h"
-#include "circ-buff.h"
-#include "statedump.h"
-#include "options.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/event-history.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/circ-buff.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/options.h>
 
 #define TRACE_DEFAULT_HISTORY_SIZE 1024
 
 typedef struct {
-        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
-         * is kept 24, i.e double of the maximum.
-         */
-        char name[24];
-        int enabled;
+    /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+     * is kept 24, i.e double of the maximum.
+     */
+    char name[24];
+    int enabled;
 } trace_fop_name_t;
 
 trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
 
 typedef struct {
-        gf_boolean_t log_file;
-        gf_boolean_t log_history;
-        size_t       history_size;
-        int trace_log_level;
+    gf_boolean_t log_file;
+    gf_boolean_t log_history;
+    uint64_t history_size;
+    int trace_log_level;
 } trace_conf_t;
 
-#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
-        do {                                                            \
-                frame->local = NULL;                                    \
-                STACK_UNWIND_STRICT (op, frame, params);                \
-        } while (0);
-
-#define LOG_ELEMENT(_conf, _string)                                     \
-        do {                                                            \
-                if (_conf) {                                            \
-                        if ((_conf->log_history) == _gf_true)           \
-                                gf_log_eh ("%s", _string);              \
-                        if ((_conf->log_file) == _gf_true)              \
-                                gf_log (THIS->name, _conf->trace_log_level, \
-                                        "%s", _string);                 \
-                }                                                       \
-        } while (0);
+#define TRACE_STACK_UNWIND(op, frame, params...)                               \
+    do {                                                                       \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(op, frame, params);                                \
+    } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                            \
+    do {                                                                       \
+        if (_conf) {                                                           \
+            if ((_conf->log_history) == _gf_true)                              \
+                gf_log_eh("%s", _string);                                      \
+            if ((_conf->log_file) == _gf_true)                                 \
+                gf_log(THIS->name, _conf->trace_log_level, "%s", _string);     \
+        }                                                                      \
+    } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
deleted file mode 100644
index 36efc6698bd..00000000000
--- a/xlators/encryption/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = rot-13 crypt
-
-CLEANFILES = 
diff --git a/xlators/encryption/crypt/Makefile.am b/xlators/encryption/crypt/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/encryption/crypt/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
deleted file mode 100644
index 5e45a5da98a..00000000000
--- a/xlators/encryption/crypt/src/Makefile.am
+++ /dev/null
@@ -1,24 +0,0 @@
-if ENABLE_CRYPT_XLATOR
-
-xlator_LTLIBRARIES = crypt.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
-
-crypt_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
-crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la -lssl -lcrypto
-
-noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
-
-else
-
-noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
-noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
-
-endif
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
deleted file mode 100644
index 21d63e5d6d6..00000000000
--- a/xlators/encryption/crypt/src/atom.c
+++ /dev/null
@@ -1,957 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-/*
- *                              Glossary
- *
- *
- * cblock            (or cipher block). A logical unit in a file.
- *                   cblock size is defined as the number of bits
- *                   in an input (or output) block of the block
- *                   cipher (*). Cipher block size is a property of
- *                   cipher algorithm. E.g. cblock size is 64 bits
- *                   for DES, 128 bits for AES, etc.
- *
- * atomic cipher     A cipher algorithm, which requires some chunks of
- * algorithm         text to be padded at left and(or) right sides before
- *                   cipher transaform.
- *
- *
- * block (atom)      Minimal chunk of file's data, which doesn't require
- *                   padding. We'll consider logical units in a file of
- *                   block size (atom size).
- *
- * cipher algorithm  Atomic cipher algorithm, which requires the last
- * with EOF issue    incomplete cblock in a file to be padded with some
- *                   data (usually zeros).
- *
- *
- * operation, which  reading/writing from offset, which is not aligned to
- * forms a gap at    to atom size
- * the beginning
- *
- *
- * operation, which  reading/writing count bytes starting from offset off,
- * forms a gap at    so that off+count is not aligned to atom_size
- * the end
- *
- * head block        the first atom affected by an operation, which forms
- *                   a gap at the beginning, or(and) at the end.
- *                   Сomment. Head block has at least one gap (either at
- *                   the beginning, or at the end)
- *
- *
- * tail block        the last atom different from head, affected by an
- *                   operation, which forms a gap at the end.
- *                   Сomment: Tail block has exactly one gap (at the end).
- *
- *
- * partial block     head or tail block
- *
- *
- * full block        block without gaps.
- *
- *
- * (*) Recommendation for Block Cipher Modes of Operation
- *     Methods and Techniques
- *     NIST Special Publication 800-38A Edition 2001
- */
-
-/*
- * atom->offset_at()
- */
-static off_t offset_at_head(struct avec_config *conf)
-{
-	return conf->aligned_offset;
-}
-
-static off_t offset_at_hole_head(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_head(get_hole_conf(frame));
-}
-
-static off_t offset_at_data_head(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_head(get_data_conf(frame));
-}
-
-
-static off_t offset_at_tail(struct avec_config *conf,
-				    struct object_cipher_info *object)
-{
-	return conf->aligned_offset +
-		(conf->off_in_head ? get_atom_size(object) : 0) +
-		(conf->nr_full_blocks << get_atom_bits(object));
-}
-
-static off_t offset_at_hole_tail(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_tail(get_hole_conf(frame), object);
-}
-
-
-static off_t offset_at_data_tail(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_tail(get_data_conf(frame), object);
-}
-
-static off_t offset_at_full(struct avec_config *conf,
-				    struct object_cipher_info *object)
-{
-	return conf->aligned_offset +
-		(conf->off_in_head ? get_atom_size(object) : 0);
-}
-
-static off_t offset_at_data_full(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_full(get_data_conf(frame), object);
-}
-
-static off_t offset_at_hole_full(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_full(get_hole_conf(frame), object);
-}
-
-/*
- * atom->io_size_nopad()
- */
-
-static uint32_t io_size_nopad_head(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	uint32_t gap_at_beg;
-	uint32_t gap_at_end;
-
-	check_head_block(conf);
-
-	gap_at_beg = conf->off_in_head;
-
-	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
-		gap_at_end = 0;
-	else
-		gap_at_end = get_atom_size(object) - conf->off_in_tail;
-
-	return get_atom_size(object) - (gap_at_beg + gap_at_end);
-}
-
-static uint32_t io_size_nopad_tail(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	check_tail_block(conf);
-	return conf->off_in_tail;
-}
-
-static uint32_t io_size_nopad_full(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	check_full_block(conf);
-	return get_atom_size(object);
-}
-
-static uint32_t io_size_nopad_data_head(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_head(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_head(get_hole_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_tail(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_tail(get_hole_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_data_full(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_full(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_full(get_hole_conf(frame), object);
-}
-
-static uint32_t offset_in_head(struct avec_config *conf)
-{
-	check_cursor_head(conf);
-
-	return conf->off_in_head;
-}
-
-static uint32_t offset_in_tail(call_frame_t *frame,
-			       struct object_cipher_info *object)
-{
-	return 0;
-}
-
-static uint32_t offset_in_full(struct avec_config *conf,
-			       struct object_cipher_info *object)
-{
-	check_cursor_full(conf);
-
-	if (has_head_block(conf))
-		return (conf->cursor - 1) << get_atom_bits(object);
-	else
-		return conf->cursor << get_atom_bits(object);
-}
-
-static uint32_t offset_in_data_head(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_head(get_data_conf(frame));
-}
-
-static uint32_t offset_in_hole_head(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_head(get_hole_conf(frame));
-}
-
-static uint32_t offset_in_data_full(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_full(get_data_conf(frame), object);
-}
-
-static uint32_t offset_in_hole_full(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_full(get_hole_conf(frame), object);
-}
-
-/*
- * atom->rmw()
- */
-/*
- * Pre-conditions:
- * @vec contains plain text of the latest
- * version.
- *
- * Uptodate gaps of the @partial block with
- * this plain text, encrypt the whole block
- * and write the result to disk.
- */
-static int32_t rmw_partial_block(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno,
-				 struct iovec *vec,
-				 int32_t count,
-				 struct iatt *stbuf,
-				 struct iobref *iobref,
-				 struct rmw_atom *atom)
-{
-	size_t was_read = 0;
-	uint64_t file_size;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	struct iovec *partial = atom->get_iovec(frame, 0);
-	struct avec_config *conf = atom->get_config(frame);
-	end_writeback_handler_t end_writeback_partial_block;
-#if DEBUG_CRYPT
-	gf_boolean_t check_last_cblock = _gf_false;
-#endif
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto exit;
-
-	file_size = local->cur_file_size;
-	was_read = op_ret;
-
-	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
-		/*
-		 * head atom with a non-uptodate gap
-		 * at the beginning
-		 *
-		 * fill the gap with plain text of the
-		 * latest version. Convert a part of hole
-		 * (if any) to zeros.
-		 */
-		int32_t i;
-		int32_t copied = 0;
-		int32_t to_gap; /* amount of data needed to uptodate
-				   the gap at the beginning */
-#if 0
-		int32_t hole = 0; /* The part of the hole which
-				   * got in the head block */
-#endif /* 0 */
-		to_gap = conf->off_in_head;
-
-		if (was_read < to_gap) {
-			if (file_size >
-			    offset_at_head(conf) + was_read) {
-				/*
-				 * It is impossible to uptodate
-				 * head block: too few bytes have
-				 * been read from disk, so that
-				 * partial write is impossible.
-				 *
-				 * It could happen because of many
-				 * reasons: IO errors, (meta)data
-				 * corruption in the local file system,
-				 * etc.
-				 */
-				gf_log(this->name, GF_LOG_WARNING,
-				     "Can not uptodate a gap at the beginning");
-				local->op_ret = -1;
-				local->op_errno = EIO;
-				goto exit;
-			}
-#if 0
-			hole = to_gap - was_read;
-#endif /* 0 */
-			to_gap = was_read;
-		}
-		/*
-		 * uptodate the gap at the beginning
-		 */
-		for (i = 0; i < count && copied < to_gap; i++) {
-			int32_t to_copy;
-
-			to_copy = vec[i].iov_len;
-			if (to_copy > to_gap - copied)
-				to_copy = to_gap - copied;
-
-			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
-			copied += to_copy;
-		}
-#if 0
-		/*
-		 * If possible, convert part of the
-		 * hole, which got in the head block
-		 */
-		ret = TRY_LOCK(&local->hole_lock);
-		if (!ret) {
-			if (local->hole_handled)
-				/*
-				 * already converted by
-				 * crypt_writev_cbk()
-				 */
-				UNLOCK(&local->hole_lock);
-			else {
-				/*
-				 * convert the part of the hole
-				 * which got in the head block
-				 * to zeros.
-				 *
-				 * Update the orig_offset to make
-				 * sure writev_cbk() won't care
-				 * about this part of the hole.
-				 *
-				 */
-				memset(partial->iov_base + to_gap, 0, hole);
-
-				conf->orig_offset -= hole;
-				conf->orig_size += hole;
-				UNLOCK(&local->hole_lock);
-			}
-		}
-		else /*
-		      * conversion is being performed
-		      * by crypt_writev_cbk()
-		      */
-			;
-#endif /* 0 */
-	}
-	if (atom->locality == TAIL_ATOM ||
-	    (!has_tail_block(conf) && conf->off_in_tail)) {
-		/*
-		 * tail atom, or head atom with a non-uptodate
-		 * gap at the end.
-		 *
-		 * fill the gap at the end of the block
-		 * with plain text of the latest version.
-		 * Pad the result, (if needed)
-		 */
-		int32_t i;
-		int32_t to_gap;
-		int copied;
-		off_t off_in_tail;
-		int32_t to_copy;
-
-		off_in_tail = conf->off_in_tail;
-		to_gap = conf->gap_in_tail;
-
-		if (to_gap && was_read < off_in_tail + to_gap) {
-			/*
-			 * It is impossible to uptodate
-			 * the gap at the end: too few bytes
-			 * have been read from disk, so that
-			 * partial write is impossible.
-			 *
-			 * It could happen because of many
-			 * reasons: IO errors, (meta)data
-			 * corruption in the local file system,
-			 * etc.
-			 */
-			gf_log(this->name, GF_LOG_WARNING,
-			       "Can not uptodate a gap at the end");
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-		/*
-		 * uptodate the gap at the end
-		 */
-		copied = 0;
-		to_copy = to_gap;
-		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
-			uint32_t from_vec, off_in_vec;
-
-			off_in_vec = 0;
-			from_vec = vec[i].iov_len;
-			if (from_vec > to_copy) {
-				off_in_vec = from_vec - to_copy;
-				from_vec = to_copy;
-			}
-			memcpy(partial->iov_base +
-			       off_in_tail + to_gap - copied - from_vec,
-			       vec[i].iov_base + off_in_vec,
-			       from_vec);
-
-			gf_log(this->name, GF_LOG_DEBUG,
-			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
-			       (int)from_vec,
-			       (int)off_in_tail + to_gap - copied - from_vec,
-			       (int)off_in_vec);
-
-			copied += from_vec;
-			to_copy -= from_vec;
-		}
-		partial->iov_len = off_in_tail + to_gap;
-
-		if (object_alg_should_pad(object)) {
-			int32_t resid = 0;
-			resid = partial->iov_len & (object_alg_blksize(object) - 1);
-			if (resid) {
-				/*
-				 * append a new EOF padding
-				 */
-				local->eof_padding_size =
-					object_alg_blksize(object) - resid;
-
-				gf_log(this->name, GF_LOG_DEBUG,
-				       "set padding size %d",
-				       local->eof_padding_size);
-
-				memset(partial->iov_base + partial->iov_len,
-				       1,
-				       local->eof_padding_size);
-				partial->iov_len += local->eof_padding_size;
-#if DEBUG_CRYPT
-				gf_log(this->name, GF_LOG_DEBUG,
-				       "pad cblock with %d zeros:",
-				       local->eof_padding_size);
-				dump_cblock(this,
-					    (unsigned char *)partial->iov_base +
-					    partial->iov_len - object_alg_blksize(object));
-				check_last_cblock = _gf_true;
-#endif
-			}
-		}
-	}
-	/*
-	 * encrypt the whole block
-	 */
-	encrypt_aligned_iov(object,
-			    partial,
-			    1,
-			    atom->offset_at(frame, object));
-#if DEBUG_CRYPT
-	if (check_last_cblock == _gf_true) {
-		gf_log(this->name, GF_LOG_DEBUG,
-		       "encrypt last cblock with offset %llu",
-		       (unsigned long long)atom->offset_at(frame, object));
-		dump_cblock(this, (unsigned char *)partial->iov_base +
-			    partial->iov_len - object_alg_blksize(object));
-	}
-#endif
-	set_local_io_params_writev(frame, object, atom,
-				   atom->offset_at(frame, object),
-				   iovec_get_size(partial, 1));
-	/*
-	 * write the whole block to disk
-	 */
-	end_writeback_partial_block = dispatch_end_writeback(local->fop);
-	conf->cursor ++;
-	STACK_WIND(frame,
-		   end_writeback_partial_block,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->writev,
-		   local->fd,
-		   partial,
-		   1,
-		   atom->offset_at(frame, object),
-		   local->flags,
-		   local->iobref_data,
-		   local->xdata);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "submit partial block: %d bytes from %d offset",
-	       (int)iovec_get_size(partial, 1),
-	       (int)atom->offset_at(frame, object));
- exit:
-	return 0;
-}
-
-/*
- * Perform a (read-)modify-write sequence.
- * This should be performed only after approval
- * of upper server-side manager, i.e. the caller
- * needs to make sure this is his turn to rmw.
- */
-void submit_partial(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    atom_locality_type ltype)
-{
-	int32_t ret;
-	dict_t *dict;
-	struct rmw_atom *atom;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	atom = atom_by_types(local->active_setup, ltype);
-	/*
-	 * To perform the "read" component of the read-modify-write
-	 * sequence the crypt translator does stack_wind to itself.
-	 *
-	 * Pass current file size to crypt_readv()
-	 */
-	dict = dict_new();
-	if (!dict) {
-		/*
-		 * FIXME: Handle the error
-		 */
-		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
-		return;
-	}
-	ret = dict_set(dict,
-		       FSIZE_XATTR_PREFIX,
-		       data_from_uint64(local->cur_file_size));
-	if (ret) {
-		/*
-		 * FIXME: Handle the error
-		 */
-		dict_unref(dict);
-		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   atom->rmw,
-		   this,
-		   this->fops->readv, /* crypt_readv */
-		   fd,
-		   atom->count_to_uptodate(frame, object), /* count */
-		   atom->offset_at(frame, object), /* offset to read from */
-		   0,
-		   dict);
- exit:
-	dict_unref(dict);
-}
-
-/*
- * submit blocks of FULL_ATOM type
- */
-void submit_full(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
-	uint32_t count; /* total number of full blocks to submit */
-	uint32_t granularity; /* number of blocks to submit in one iteration */
-
-	uint64_t off_in_file; /* start offset in the file, bytes */
-	uint32_t off_in_atom; /* start offset in the atom, blocks */
-	uint32_t blocks_written = 0; /* blocks written for this submit */
-
-	struct avec_config *conf = atom->get_config(frame);
-	end_writeback_handler_t end_writeback_full_block;
-	/*
-	 * Write full blocks by groups of granularity size.
-	 */
-	end_writeback_full_block = dispatch_end_writeback(local->fop);
-
-	if (is_ordered_mode(frame)) {
-		uint32_t skip = has_head_block(conf) ? 1 : 0;
-		count = 1;
-		granularity = 1;
-		/*
-		 * calculate start offset using cursor value;
-		 * here we should take into accout head block,
-		 * which corresponds to cursor value 0.
-		 */
-		off_in_file = atom->offset_at(frame, object) +
-			((conf->cursor - skip) << get_atom_bits(object));
-		off_in_atom = conf->cursor - skip;
-	}
-	else {
-		/*
-		 * in parallel mode
-		 */
-		count = conf->nr_full_blocks;
-		granularity = MAX_IOVEC;
-		off_in_file = atom->offset_at(frame, object);
-		off_in_atom = 0;
-	}
-	while (count) {
-		uint32_t blocks_to_write = count;
-
-		if (blocks_to_write > granularity)
-			blocks_to_write = granularity;
-		if (conf->type == HOLE_ATOM)
-			/*
-			 * reset iovec before encryption
-			 */
-			memset(atom->get_iovec(frame, 0)->iov_base,
-			       0,
-			       get_atom_size(object));
-		/*
-		 * encrypt the group
-		 */
-		encrypt_aligned_iov(object,
-				    atom->get_iovec(frame,
-						    off_in_atom +
-						    blocks_written),
-				    blocks_to_write,
-				    off_in_file + (blocks_written <<
-						   get_atom_bits(object)));
-
-		set_local_io_params_writev(frame, object, atom,
-		        off_in_file + (blocks_written << get_atom_bits(object)),
-			blocks_to_write <<  get_atom_bits(object));
-
-		conf->cursor += blocks_to_write;
-
-		STACK_WIND(frame,
-			   end_writeback_full_block,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->writev,
-			   local->fd,
-			   atom->get_iovec(frame, off_in_atom + blocks_written),
-			   blocks_to_write,
-			   off_in_file + (blocks_written << get_atom_bits(object)),
-			   local->flags,
-			   local->iobref_data ? local->iobref_data : local->iobref,
-			   local->xdata);
-
-		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
-		       blocks_to_write,
-		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
-
-		count -= blocks_to_write;
-		blocks_written += blocks_to_write;
-	}
-	return;
-}
-
-static int32_t rmw_data_head(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(DATA_ATOM, HEAD_ATOM));
-}
-
-static int32_t rmw_data_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(DATA_ATOM, TAIL_ATOM));
-}
-
-static int32_t rmw_hole_head(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
-}
-
-static int32_t rmw_hole_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
-}
-
-/*
- * atom->count_to_uptodate()
- */
-static uint32_t count_to_uptodate_head(struct avec_config *conf,
-				       struct object_cipher_info *object)
-{
-	if (conf->acount == 1 && conf->off_in_tail)
-		return get_atom_size(object);
-	else
-		/* there is no need to read the whole head block */
-		return conf->off_in_head;
-}
-
-static uint32_t count_to_uptodate_tail(struct avec_config *conf,
-				       struct object_cipher_info *object)
-{
-	/* we need to read the whole tail block */
-	return get_atom_size(object);
-}
-
-static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_head(get_data_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_tail(get_data_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_head(get_hole_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_tail(get_hole_conf(frame), object);
-}
-
-/* atom->get_config() */
-
-static struct avec_config *get_config_data(call_frame_t *frame)
-{
-	return &((crypt_local_t *)frame->local)->data_conf;
-}
-
-static struct avec_config *get_config_hole(call_frame_t *frame)
-{
-	return &((crypt_local_t *)frame->local)->hole_conf;
-}
-
-/*
- * atom->get_iovec()
- */
-static struct iovec *get_iovec_hole_head(call_frame_t *frame,
-					 uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec;
-}
-
-static struct iovec *get_iovec_hole_full(call_frame_t *frame,
-					 uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec + (conf->off_in_head ? 1 : 0);
-}
-
-static struct iovec *get_iovec_hole_tail(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec + (conf->blocks_in_pool - 1);
-}
-
-static struct iovec *get_iovec_data_head(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec;
-}
-
-static struct iovec *get_iovec_data_full(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
-}
-
-static struct iovec *get_iovec_data_tail(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec +
-		(conf->off_in_head ? 1 : 0) +
-		conf->nr_full_blocks;
-}
-
-static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
-	[DATA_ATOM][HEAD_ATOM] =
-	{ .locality = HEAD_ATOM,
-	  .rmw = rmw_data_head,
-	  .offset_at = offset_at_data_head,
-	  .offset_in = offset_in_data_head,
-	  .get_iovec = get_iovec_data_head,
-	  .io_size_nopad = io_size_nopad_data_head,
-	  .count_to_uptodate = count_to_uptodate_data_head,
-	  .get_config = get_config_data
-	},
-	[DATA_ATOM][TAIL_ATOM] =
-	{ .locality = TAIL_ATOM,
-	  .rmw = rmw_data_tail,
-	  .offset_at = offset_at_data_tail,
-	  .offset_in = offset_in_tail,
-	  .get_iovec = get_iovec_data_tail,
-	  .io_size_nopad = io_size_nopad_data_tail,
-	  .count_to_uptodate = count_to_uptodate_data_tail,
-	  .get_config = get_config_data
-	},
-	[DATA_ATOM][FULL_ATOM] =
-	{ .locality = FULL_ATOM,
-	  .offset_at = offset_at_data_full,
-	  .offset_in = offset_in_data_full,
-	  .get_iovec = get_iovec_data_full,
-	  .io_size_nopad = io_size_nopad_data_full,
-	  .get_config = get_config_data
-	},
-	[HOLE_ATOM][HEAD_ATOM] =
-	{ .locality = HEAD_ATOM,
-	  .rmw = rmw_hole_head,
-	  .offset_at = offset_at_hole_head,
-	  .offset_in = offset_in_hole_head,
-	  .get_iovec = get_iovec_hole_head,
-	  .io_size_nopad = io_size_nopad_hole_head,
-	  .count_to_uptodate = count_to_uptodate_hole_head,
-	  .get_config = get_config_hole
-	},
-	[HOLE_ATOM][TAIL_ATOM] =
-	{ .locality = TAIL_ATOM,
-	  .rmw = rmw_hole_tail,
-	  .offset_at = offset_at_hole_tail,
-	  .offset_in = offset_in_tail,
-	  .get_iovec = get_iovec_hole_tail,
-	  .io_size_nopad = io_size_nopad_hole_tail,
-	  .count_to_uptodate = count_to_uptodate_hole_tail,
-	  .get_config = get_config_hole
-	},
-	[HOLE_ATOM][FULL_ATOM] =
-	{ .locality = FULL_ATOM,
-	  .offset_at = offset_at_hole_full,
-	  .offset_in = offset_in_hole_full,
-	  .get_iovec = get_iovec_hole_full,
-	  .io_size_nopad = io_size_nopad_hole_full,
-	  .get_config = get_config_hole
-	}
-};
-
-struct rmw_atom *atom_by_types(atom_data_type data,
-			       atom_locality_type locality)
-{
-	return &atoms[data][locality];
-}
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
deleted file mode 100644
index 7c212ad5d25..00000000000
--- a/xlators/encryption/crypt/src/crypt-common.h
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CRYPT_COMMON_H__
-#define __CRYPT_COMMON_H__
-
-#define INVAL_SUBVERSION_NUMBER (0xff)
-#define CRYPT_INVAL_OP   (GF_FOP_NULL)
-
-#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
-#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
-#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
-#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
-#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
-#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
-#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
-
-
-/* messages for crypt_open() */
-#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
-#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
-
-#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
-
-#define noop   do {; } while (0)
-#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
-#define __round_mask(x, y) ((__typeof__(x))((y)-1))
-#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
-
-/*
- * Format of file's metadata
- */
-struct crypt_format {
-	uint8_t loader_id;  /* version of metadata loader */
-	uint8_t versioned[0]; /* file's metadata of specific version */
-} __attribute__((packed));
-
-typedef enum {
-	AES_CIPHER_ALG,
-	LAST_CIPHER_ALG
-} cipher_alg_t;
-
-typedef enum {
-	XTS_CIPHER_MODE,
-	LAST_CIPHER_MODE
-} cipher_mode_t;
-
-typedef enum {
-	MTD_LOADER_V1,
-	LAST_MTD_LOADER
-} mtd_loader_id;
-
-static inline void msgflags_set_mtd_rlock(uint32_t *flags)
-{
-	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
-}
-
-static inline void msgflags_set_mtd_wlock(uint32_t *flags)
-{
-	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
-{
-	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
-{
-	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
-{
-	return  msgflags_check_mtd_rlock(flags) ||
-		msgflags_check_mtd_wlock(flags);
-}
-
-/*
- * returns number of logical blocks occupied
- * (maybe partially) by @count bytes
- * at offset @start.
- */
-static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
-					    int blkbits)
-{
-	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
-}
-
-/*
- * are two bytes (represented by offsets @off1
- * and @off2 respectively) in the same logical
- * block.
- */
-static inline int in_same_lblock(uint64_t off1, uint64_t off2,
-				 int blkbits)
-{
-	return off1 >> blkbits == off2 >> blkbits;
-}
-
-static inline void dump_cblock(xlator_t *this, unsigned char *buf)
-{
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
-	       (buf)[0],
-	       (buf)[1],
-	       (buf)[2],
-	       (buf)[3],
-	       (buf)[4],
-	       (buf)[5],
-	       (buf)[6],
-	       (buf)[7],
-	       (buf)[8],
-	       (buf)[9],
-	       (buf)[10],
-	       (buf)[11],
-	       (buf)[12],
-	       (buf)[13],
-	       (buf)[14],
-	       (buf)[15]);
-}
-
-#endif /* __CRYPT_COMMON_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
deleted file mode 100644
index 1954c579423..00000000000
--- a/xlators/encryption/crypt/src/crypt-mem-types.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __CRYPT_MEM_TYPES_H__
-#define __CRYPT_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_crypt_mem_types_ {
-	gf_crypt_mt_priv = gf_common_mt_end + 1,
-	gf_crypt_mt_inode,
-	gf_crypt_mt_data,
-	gf_crypt_mt_mtd,
-	gf_crypt_mt_loc,
-	gf_crypt_mt_iatt,
-	gf_crypt_mt_key,
-	gf_crypt_mt_iovec,
-	gf_crypt_mt_char,
-	gf_crypt_mt_local,
-	gf_crypt_mt_end,
-};
-
-#endif /* __CRYPT_MEM_TYPES_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
-
-
-
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
deleted file mode 100644
index 2982bb26db0..00000000000
--- a/xlators/encryption/crypt/src/crypt.c
+++ /dev/null
@@ -1,4525 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-#include "crypt-common.h"
-#include "crypt.h"
-
-static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
-static int32_t init_inode_info_tail(struct crypt_inode_info *info,
-				    struct master_cipher_info *master);
-static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
-				       uint64_t from, off_t size);
-static int32_t load_file_size(call_frame_t *frame, void *cookie,
-			      xlator_t *this, int32_t op_ret, int32_t op_errno,
-			      dict_t *dict, dict_t *xdata);
-static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
-			      atom_data_type dtype);
-static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
-			       atom_data_type dtype);
-static void put_one_call_open(call_frame_t *frame);
-static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
-static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
-static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
-static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
-static void free_avec_data(crypt_local_t *local);
-static void free_avec_hole(crypt_local_t *local);
-
-static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
-					glusterfs_fop_t fop)
-{
-	crypt_local_t *local = NULL;
-
-	local = GF_CALLOC (1, sizeof (*local), gf_crypt_mt_local);
-	if (!local) {
-		gf_log(this->name, GF_LOG_ERROR, "out of memory");
-		return NULL;
-	}
-	local->fop = fop;
-	LOCK_INIT(&local->hole_lock);
-	LOCK_INIT(&local->call_lock);
-	LOCK_INIT(&local->rw_count_lock);
-
-	frame->local = local;
-	return local;
-}
-
-struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
-{
-	int ret;
-	uint64_t value = 0;
-	struct crypt_inode_info *info;
-
-	ret = inode_ctx_get(inode, this, &value);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Can not get inode info");
-		return NULL;
-	}
-	info = (struct crypt_inode_info *)(long)value;
-	if (info == NULL) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Can not obtain inode info");
-		return NULL;
-	}
-	return info;
-}
-
-static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
-						      xlator_t *this)
-{
-	if (local->info)
-		return local->info;
-	local->info = get_crypt_inode_info(local->fd->inode, this);
-	return local->info;
-}
-
-static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
-						 loc_t *loc)
-{
-	struct crypt_inode_info *info;
-
-	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
-	if (!info) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		gf_log ("crypt", GF_LOG_WARNING,
-			"Can not allocate inode info");
-		return NULL;
-	}
-	memset(info, 0, sizeof(*info));
-#if DEBUG_CRYPT
-	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!info->loc) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
-		GF_FREE(info);
-		return NULL;
-	}
-	if (loc_copy(info->loc, loc)){
-		GF_FREE(info->loc);
-		GF_FREE(info);
-		return NULL;
-	}
-#endif /* DEBUG_CRYPT */
-
-	local->info = info;
-	return info;
-}
-
-static void free_inode_info(struct crypt_inode_info *info)
-{
-#if DEBUG_CRYPT
-	loc_wipe(info->loc);
-	GF_FREE(info->loc);
-#endif
-	memset(info, 0, sizeof(*info));
-	GF_FREE(info);
-}
-
-int crypt_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t ctx_addr = 0;
-        if (!inode_ctx_del (inode, this, &ctx_addr))
-                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
-        return 0;
-}
-
-#if DEBUG_CRYPT
-static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
-		       struct iovec *vec, int32_t count, struct iatt *stbuf)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = get_object_cinfo(local->info);
-	struct avec_config *conf = &local->data_conf;
-	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
-
-	if (read <= 0)
-		return;
-	if (read != iovec_get_size(vec, count))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"op_ret differs from amount of read bytes");
-
-	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad amount of read bytes (!= 0 mod(cblock size))");
-
-	if (conf->aligned_offset + read >
-	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad amount of read bytes (too large))");
-
-}
-
-#define PT_BYTES_TO_DUMP (32)
-static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
-{
-	int32_t to_dump;
-	char str[PT_BYTES_TO_DUMP + 1];
-
-	if (!avec)
-		return;
-	to_dump = avec->iov_len;
-	if (to_dump > PT_BYTES_TO_DUMP)
-		to_dump = PT_BYTES_TO_DUMP;
-	memcpy(str, avec->iov_base, to_dump);
-	memset(str + to_dump, '0', 1);
-	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
-}
-
-static int32_t data_conf_invariant(struct avec_config *conf)
-{
-	return conf->acount ==
-		!!has_head_block(conf) +
-		!!has_tail_block(conf)+
-		conf->nr_full_blocks;
-}
-
-static int32_t hole_conf_invariant(struct avec_config *conf)
-{
-	return conf->blocks_in_pool ==
-		!!has_head_block(conf) +
-		!!has_tail_block(conf)+
-		!!has_full_blocks(conf);
-}
-
-static void crypt_check_conf(struct avec_config *conf)
-{
-	int32_t ret = 0;
-	const char *msg;
-
-	switch (conf->type) {
-	case DATA_ATOM:
-		msg = "data";
-		ret = data_conf_invariant(conf);
-		break;
-	case HOLE_ATOM:
-		msg = "hole";
-		ret = hole_conf_invariant(conf);
-		break;
-	default:
-		msg = "unknown";
-	}
-	if (!ret)
-		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
-}
-
-static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	uint64_t local_file_size;
-
-	switch(local->fop) {
-	case GF_FOP_FTRUNCATE:
-		return;
-	case GF_FOP_WRITE:
-		local_file_size = local->new_file_size;
-		break;
-	case GF_FOP_READ:
-		if (parent_is_crypt_xlator(frame, this))
-			return;
-		local_file_size = local->cur_file_size;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
-		return;
-	}
-	if (buf->ia_size != round_up(local_file_size,
-					 object_alg_blksize(object)))
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "bad ia_size in buf (%llu), should be %llu",
-		       (unsigned long long)buf->ia_size,
-		       (unsigned long long)round_up(local_file_size,
-						    object_alg_blksize(object)));
-}
-
-#else
-#define check_read(frame, this, op_ret, vec, count, stbuf) noop
-#define dump_plain_text(local, avec) noop
-#define crypt_check_conf(conf) noop
-#define check_buf(frame, this, buf) noop
-#endif /* DEBUG_CRYPT */
-
-/*
- * Pre-conditions:
- * @vec represents a ciphertext of expanded size and
- * aligned offset.
- *
- * Compound a temporal vector @avec with block-aligned
- * components, decrypt and fix it up to represent a chunk
- * of data corresponding to the original size and offset.
- * Pass the result to the next translator.
- */
-int32_t crypt_readv_cbk(call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct iovec *vec,
-			int32_t count,
-			struct iatt *stbuf,
-			struct iobref *iobref,
-			dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	struct iovec *avec;
-	uint32_t i;
-	uint32_t to_vec;
-	uint32_t to_user;
-
-	check_buf(frame, this, stbuf);
-	check_read(frame, this, op_ret, vec, count, stbuf);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->iobref = iobref_ref(iobref);
-
-	local->buf = *stbuf;
-	local->buf.ia_size = local->cur_file_size;
-
-	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
-		goto put_one_call;
-
-	if (conf->orig_offset >= local->cur_file_size) {
-		local->op_ret = 0;
-		goto put_one_call;
-	}
-	/*
-	 * correct config params with real file size
-	 * and actual amount of bytes read
-	 */
-	set_config_offsets(frame, this,
-			   conf->orig_offset, op_ret, DATA_ATOM, 0);
-
-	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
-		conf->orig_size = local->cur_file_size - conf->orig_offset;
-	/*
-	 * calculate amount of data to be returned
-	 * to user.
-	 */
-	to_user = op_ret;
-	if (conf->aligned_offset + to_user <= conf->orig_offset) {
-		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto put_one_call;
-	}
-	to_user -= (conf->aligned_offset - conf->orig_offset);
-
-	if (to_user > conf->orig_size)
-		to_user = conf->orig_size;
-	local->rw_count = to_user;
-
-	op_errno = set_config_avec_data(this, local,
-					conf, object, vec, count);
-	if (op_errno) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
-		goto put_one_call;
-	}
-	avec = conf->avec;
-#if DEBUG_CRYPT
-	if (conf->off_in_tail != 0 &&
-	    conf->off_in_tail < object_alg_blksize(object) &&
-	    object_alg_should_pad(object))
-		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
-		       conf->off_in_tail);
-	if (iovec_get_size(vec, count) != 0 &&
-	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
-			   local->cur_file_size - 1,
-			   object_alg_blkbits(object))) {
-		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
-		dump_cblock(this,
-			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
-			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
-		dump_cblock(this,
-			    (unsigned char *)(vec[count - 1].iov_base) +
-			    vec[count - 1].iov_len - object_alg_blksize(object));
-	}
-#endif
-	decrypt_aligned_iov(object, avec,
-			    conf->acount, conf->aligned_offset);
-	/*
-	 * pass proper plain data to user
-	 */
-	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
-	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
-
-	to_vec = to_user;
-	for (i = 0; i < conf->acount; i++) {
-		if (avec[i].iov_len > to_vec)
-			avec[i].iov_len = to_vec;
-		to_vec -= avec[i].iov_len;
-	}
- put_one_call:
-	put_one_call_readv(frame, this);
-	return 0;
-}
-
-static int32_t do_readv(call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			dict_t *dict,
-			dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_errno = EIO;
-		goto error;
-	}
-	local->cur_file_size = data_to_uint64(data);
-
-	get_one_call(frame);
-	STACK_WIND(frame,
-		   crypt_readv_cbk,
-		   FIRST_CHILD (this),
-		   FIRST_CHILD (this)->fops->readv,
-		   local->fd,
-		   /*
-		    * FIXME: read amount can be reduced
-		    */
-		   local->data_conf.expanded_size,
-		   local->data_conf.aligned_offset,
-		   local->flags,
-		   local->xdata);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-
-	get_one_call(frame);
-	put_one_call_readv(frame, this);
-	return 0;
-}
-
-static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
-					void *cookie,
-					xlator_t *this,
-					int32_t op_ret,
-					int32_t op_errno,
-					dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size
-	 */
-	STACK_WIND(frame,
-		   do_readv,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return  0;
- error:
-	fd_unref(local->fd);
-	if (local->xdata)
-		dict_unref(local->xdata);
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    -1,
-			    op_errno,
-			    NULL,
-			    0,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t readv_trivial_completion(call_frame_t *frame,
-					void *cookie,
-					xlator_t *this,
-					int32_t op_ret,
-					int32_t op_errno,
-					struct iatt *buf,
-					dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "stat failed (%d)", op_errno);
-		goto error;
-	}
-	local->buf = *buf;
-	STACK_WIND(frame,
-		   load_file_size,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->getxattr,
-		   local->loc,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
-			    NULL, 0, NULL, NULL, NULL);
-	return 0;
-}
-
-int32_t crypt_readv(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    size_t size,
-		    off_t offset,
-		    uint32_t flags, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-
-#if DEBUG_CRYPT
-	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
-	       (int)size, (long long)offset);
-	if (parent_is_crypt_xlator(frame, this))
-		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_READ);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	if (size == 0)
-		goto trivial;
-
-	local->fd = fd_ref(fd);
-	local->flags = flags;
-
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		fd_unref(fd);
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		fd_unref(fd);
-		goto error;
-	}
-	set_config_offsets(frame, this, offset, size,
-			   DATA_ATOM, 0);
-	if (parent_is_crypt_xlator(frame, this)) {
-		data_t *data;
-		/*
-		 * We are called by crypt_writev (or cypt_ftruncate)
-		 * to perform the "read" component of the read-modify-write
-		 * (or read-prune-write) sequence for some atom;
-		 *
-		 * don't ask for access:
-		 * it has already been acquired
-		 *
-		 * Retrieve current file size
-		 */
-		if (!xdata) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size hasn't been passed");
-			ret = EIO;
-			goto error;
-		}
-		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
-		if (!data) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size not found");
-			ret = EIO;
-			goto error;
-		}
-		local->old_file_size =
-			local->cur_file_size = data_to_uint64(data);
-
-		get_one_call(frame);
-		STACK_WIND(frame,
-			   crypt_readv_cbk,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->readv,
-			   local->fd,
-			   /*
-			    * FIXME: read amount can be reduced
-			    */
-			   local->data_conf.expanded_size,
-			   local->data_conf.aligned_offset,
-			   flags,
-			   NULL);
-		return 0;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_RDLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_readv_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   readv_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    -1,
-			    ret,
-			    NULL,
-			    0,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-void set_local_io_params_writev(call_frame_t *frame,
-				struct object_cipher_info *object,
-				struct rmw_atom *atom,
-				off_t io_offset,
-				uint32_t io_size)
-{
-	crypt_local_t *local = frame->local;
-
-	local->io_offset = io_offset;
-	local->io_size = io_size;
-
-	local->io_offset_nopad =
-		atom->offset_at(frame, object) + atom->offset_in(frame, object);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "set nopad offset to %llu",
-	       (unsigned long long)local->io_offset_nopad);
-
-	local->io_size_nopad = atom->io_size_nopad(frame, object);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "set nopad size to %llu",
-	       (unsigned long long)local->io_size_nopad);
-
-	local->update_disk_file_size = 0;
-	/*
-	 * NOTE: eof_padding_size is 0 for all full atoms;
-	 * For head and tail atoms it will be set up at rmw_partial block()
-	 */
-	local->new_file_size = local->cur_file_size;
-
-	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
-
-		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
-
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "set new file size to %llu",
-		       (unsigned long long)local->new_file_size);
-
-		local->update_disk_file_size = 1;
-	}
-}
-
-void set_local_io_params_ftruncate(call_frame_t *frame,
-				   struct object_cipher_info *object)
-{
-	uint32_t resid;
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
-	if (resid) {
-		local->eof_padding_size =
-			object_alg_blksize(object) - resid;
-		local->new_file_size = conf->aligned_offset;
-		local->update_disk_file_size = 0;
-		/*
-		 * file size will be updated
-		 * in the ->writev() stack,
-		 * when submitting file tail
-		 */
-	} else {
-		local->eof_padding_size = 0;
-		local->new_file_size = conf->orig_offset;
-		local->update_disk_file_size = 1;
-		/*
-		 * file size will be updated
-		 * in this ->ftruncate stack
-		 */
-	}
-}
-
-static void submit_head(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	submit_partial(frame, this, local->fd, HEAD_ATOM);
-}
-
-static void submit_tail(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	submit_partial(frame, this, local->fd, TAIL_ATOM);
-}
-
-static void submit_hole(call_frame_t *frame, xlator_t *this)
-{
-	/*
-	 * hole conversion always means
-	 * appended write and goes in ordered fashion
-	 */
-	do_ordered_submit(frame, this, HOLE_ATOM);
-}
-
-static void submit_data(call_frame_t *frame, xlator_t *this)
-{
-	if (is_ordered_mode(frame)) {
-		do_ordered_submit(frame, this, DATA_ATOM);
-		return;
-	}
-	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
-	get_nr_calls(frame, nr_calls_data(frame));
-	do_parallel_submit(frame, this, DATA_ATOM);
-	return;
-}
-
-/*
- * heplers called by writev_cbk, fruncate_cbk in ordered mode
- */
-
-static int32_t should_submit_hole(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-
-	return conf->avec != NULL;
-}
-
-static int32_t should_resume_submit_hole(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-
-	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
-		/*
-		 * Don't submit a part of hole, which
-		 * fits into a data block:
- 		 * this part of hole will be converted
-		 * as a gap filled by zeros in data head
-		 * block.
-		 */
-		return conf->cursor < conf->acount - 1;
-	else
-		return conf->cursor < conf->acount;
-}
-
-static int32_t should_resume_submit_data(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	if (is_ordered_mode(frame))
-		return conf->cursor < conf->acount;
-	/*
-	 * parallel writes
-	 */
-	return 0;
-}
-
-static int32_t should_submit_data_after_hole(crypt_local_t *local)
-{
-	return local->data_conf.avec != NULL;
-}
-
-static void update_local_file_params(call_frame_t *frame,
-				     xlator_t *this,
-				     struct iatt *prebuf,
-				     struct iatt *postbuf)
-{
-	crypt_local_t *local = frame->local;
-
-	check_buf(frame, this, postbuf);
-
-	local->prebuf = *prebuf;
-	local->postbuf = *postbuf;
-
-	local->prebuf.ia_size  = local->cur_file_size;
-	local->postbuf.ia_size = local->new_file_size;
-
-	local->cur_file_size = local->new_file_size;
-}
-
-static int32_t end_writeback_writev(call_frame_t *frame,
-				    void *cookie,
-				    xlator_t *this,
-				    int32_t op_ret,
-				    int32_t op_errno,
-				    struct iatt *prebuf,
-				    struct iatt *postbuf,
-				    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret <= 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "writev iteration failed");
-		goto put_one_call;
-	}
-	/*
-	 * op_ret includes paddings (atom's head, atom's tail and EOF)
-	 */
-	if (op_ret < local->io_size) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Incomplete writev iteration");
-		goto put_one_call;
-	}
-	op_ret -= local->eof_padding_size;
-	local->op_ret = op_ret;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	if (data_write_in_progress(local)) {
-
-		LOCK(&local->rw_count_lock);
-		local->rw_count += op_ret;
-		UNLOCK(&local->rw_count_lock);
-
-		if (should_resume_submit_data(frame))
-			submit_data(frame, this);
-	}
-	else {
-		/*
-		 * hole conversion is going on;
-		 * don't take into account written zeros
-		 */
-		if (should_resume_submit_hole(local))
-			submit_hole(frame, this);
-
-		else if (should_submit_data_after_hole(local))
-			submit_data(frame, this);
-	}
- put_one_call:
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-#define crypt_writev_cbk end_writeback_writev
-
-#define HOLE_WRITE_CHUNK_BITS 12
-#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
-
-/*
- * Convert hole of size @size at offset @off to
- * zeros and prepare respective iovecs for submit.
- * The hole lock should be held.
- *
- * Pre-conditions:
- * @local->file_size is set and valid.
- */
-int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
-				uint64_t off, off_t size)
-{
-	int32_t ret;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
-
-	ret = set_config_avec_hole(this, local,
-				   &local->hole_conf, object, local->fop);
-	crypt_check_conf(&local->hole_conf);
-
-	return ret;
-}
-
-/*
- * prepare for submit @count bytes at offset @from
- */
-int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
-				off_t from, int32_t size, struct iovec *vec,
-				int32_t vec_count, int32_t setup_gap)
-{
-	uint32_t ret;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_config_offsets(frame, this, from, size,
-			   DATA_ATOM, setup_gap);
-
-	ret = set_config_avec_data(this, local,
-				   &local->data_conf, object, vec, vec_count);
-	crypt_check_conf(&local->data_conf);
-
-	return ret;
-}
-
-static void free_avec(struct iovec *avec,
-		      char **pool, int blocks_in_pool)
-{
-	if (!avec)
-		return;
-	GF_FREE(pool);
-	GF_FREE(avec);
-}
-
-static void free_avec_data(crypt_local_t *local)
-{
-	return free_avec(local->data_conf.avec,
-			 local->data_conf.pool,
-			 local->data_conf.blocks_in_pool);
-}
-
-static void free_avec_hole(crypt_local_t *local)
-{
-	return free_avec(local->hole_conf.avec,
-			 local->hole_conf.pool,
-			 local->hole_conf.blocks_in_pool);
-}
-
-
-static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
-			       atom_data_type dtype)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf;
-
-	local->active_setup = dtype;
-	conf = conf_by_type(frame, dtype);
-
-	if (has_head_block(conf))
-		submit_head(frame, this);
-
-	if (has_full_blocks(conf))
-		submit_full(frame, this);
-
-	if (has_tail_block(conf))
-		submit_tail(frame, this);
-	return;
-}
-
-static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
-			      atom_data_type dtype)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf;
-
-	local->active_setup = dtype;
-	conf = conf_by_type(frame, dtype);
-
-	if (should_submit_head_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_head(frame, this);
-	}
-	else if (should_submit_full_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_full(frame, this);
-	}
-	else if (should_submit_tail_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_tail(frame, this);
-	}
-	else
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "nothing has been submitted in ordered mode");
-	return;
-}
-
-static int32_t do_writev(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *dict,
-			 dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_ret = -1;
-		op_errno = EIO;
-		goto error;
-	}
-	local->old_file_size = local->cur_file_size = data_to_uint64(data);
-
-	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
-
-	if (local->cur_file_size < local->data_conf.orig_offset) {
-		/*
-		 * Set up hole config
-		 */
-		op_errno = prepare_for_submit_hole(frame,
-			       this,
-			       local->cur_file_size,
-			       local->data_conf.orig_offset - local->cur_file_size);
-		if (op_errno) {
-			local->op_ret = -1;
-			local->op_errno = op_errno;
-			goto error;
-		}
-	}
-	if (should_submit_hole(local))
-		submit_hole(frame, this);
-	else
-		submit_data(frame, this);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size first
-	 */
-	STACK_WIND(frame,
-		   do_writev,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-static int32_t writev_trivial_completion(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 struct iatt *buf,
-					 dict_t *dict)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->prebuf = *buf;
-	local->postbuf = *buf;
-
-	local->prebuf.ia_size = local->cur_file_size;
-	local->postbuf.ia_size = local->cur_file_size;
-
-	get_one_call(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-int crypt_writev(call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 struct iovec *vec,
-		 int32_t count,
-		 off_t offset,
-		 uint32_t flags,
-		 struct iobref *iobref,
-		 dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-#if DEBUG_CRYPT
-	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
-		(int)iovec_get_size(vec, count), (long long)offset);
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	if (iobref)
-		local->iobref = iobref_ref(iobref);
-	/*
-	 * to update real file size on the server
-	 */
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->flags = flags;
-
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (iovec_get_size(vec, count) == 0)
-		goto trivial;
-
-	ret = prepare_for_submit_data(frame, this, offset,
-				      iovec_get_size(vec, count),
-				      vec, count, 0 /* don't setup gup
-						       in tail: we don't
-						       know file size yet */);
-	if (ret)
-		goto error;
-
-	if (parent_is_crypt_xlator(frame, this)) {
-		data_t *data;
-		/*
-		 * we are called by shinking crypt_ftruncate(),
-		 * which doesn't perform hole conversion;
-		 *
-		 * don't ask for access:
-		 * it has already been acquired
-		 */
-
-		/*
-		 * extract file size
-		 */
-		if (!xdata) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size hasn't been passed");
-			ret = EIO;
-			goto error;
-		}
-		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
-		if (!data) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size not found");
-			ret = EIO;
-			goto error;
-		}
-		local->old_file_size =
-			local->cur_file_size = data_to_uint64(data);
-
-		submit_data(frame, this);
-		return 0;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	/*
-	 * lock the file and retrieve its size
-	 */
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_writev_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   writev_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	if (local && local->fd)
-		fd_unref(fd);
-	if (local && local->iobref)
-		iobref_unref(iobref);
-	if (local && local->xdata)
-		dict_unref(xdata);
-	if (local && local->xattr)
-		dict_unref(local->xattr);
-	if (local && local->info)
-		free_inode_info(local->info);
-
-	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
-	return 0;
-}
-
-int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
-{
-	set_config_offsets(frame, this,
-			   offset,
-			   0, /* count */
-			   DATA_ATOM,
-			   0 /* since we prune, there is no
-				gap in tail to uptodate */);
-	return 0;
-}
-
-/*
- * Finish the read-prune-modify sequence
- *
- * Can be invoked as
- * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
- * 2) ->writev_cbk() for non-cblock-aligned prune
- */
-
-static int32_t prune_complete(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      struct iatt *prebuf,
-			      struct iatt *postbuf,
-			      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * This is called as ->ftruncate_cbk()
- *
- * Perform the "write" component of the
- * read-prune-write sequence.
- *
- * submuit the rest of the file
- */
-static int32_t prune_submit_file_tail(call_frame_t *frame,
-				      void *cookie,
-				      xlator_t *this,
-				      int32_t op_ret,
-				      int32_t op_errno,
-				      struct iatt *prebuf,
-				      struct iatt *postbuf,
-				      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	dict_t *dict;
-
-	if (op_ret < 0)
-		goto put_one_call;
-
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	dict = dict_new();
-	if (!dict) {
-		op_errno = ENOMEM;
-		goto error;
-	}
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-	local->new_file_size = conf->orig_offset;
-
-	/*
-	 * The rest of the file is a partial block and, hence,
-	 * should be written via RMW sequence, so the crypt xlator
-	 * does STACK_WIND to itself.
-	 *
-	 * Pass current file size to crypt_writev()
-	 */
-	op_errno = dict_set(dict,
-			    FSIZE_XATTR_PREFIX,
-			    data_from_uint64(local->cur_file_size));
-	if (op_errno) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "can not set key to update file size");
-		dict_unref(dict);
-		goto error;
-	}
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "passing current file size (%llu) to crypt_writev",
-	       (unsigned long long)local->cur_file_size);
-	/*
-	 * Padding will be filled with
-	 * zeros by rmw_partial_block()
-	 */
-	STACK_WIND(frame,
-		   prune_complete,
-		   this,
-		   this->fops->writev, /* crypt_writev */
-		   local->fd,
-		   &local->vec,
-		   1,
-		   conf->aligned_offset, /* offset to write from */
-		   0,
-		   local->iobref,
-		   dict);
-
-	dict_unref(dict);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * This is called as a callback of ->writev() invoked in behalf
- * of ftruncate(): it can be
- * 1) ordered writes issued by hole conversion in the case of
- *    expanded truncate, or
- * 2) an rmw partial data block issued by non-cblock-aligned
- * prune.
- */
-int32_t end_writeback_ftruncate(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				struct iatt *prebuf,
-				struct iatt *postbuf,
-				dict_t *xdata)
-{
-	crypt_local_t  *local = frame->local;
-	/*
-	 * if nothing has been written,
-	 * then it must be an error
-	 */
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto put_one_call;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	if (data_write_in_progress(local))
-		/* case (2) */
-		goto put_one_call;
-	/* case (1) */
-	if (should_resume_submit_hole(local))
-		submit_hole(frame, this);
-	/*
-	 * case of hole, when we should't resume
-	 */
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * Perform prune and write components of the
- * read-prune-write sequence.
- *
- * Called as ->readv_cbk()
- *
- * Pre-conditions:
- * @vec contains the latest atom of the file
- * (plain text)
- */
-static int32_t prune_write(call_frame_t *frame,
-			   void *cookie,
-			   xlator_t *this,
-			   int32_t op_ret,
-			   int32_t op_errno,
-			   struct iovec *vec,
-			   int32_t count,
-			   struct iatt *stbuf,
-			   struct iobref *iobref,
-			   dict_t *xdata)
-{
-	int32_t i;
-	size_t to_copy;
-	size_t copied = 0;
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	if (op_ret == -1)
-		goto put_one_call;
-
-	/*
-	 * At first, uptodate head block
-	 */
-	if (iovec_get_size(vec, count) < conf->off_in_head) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Failed to uptodate head block for prune");
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto put_one_call;
-	}
-	local->vec.iov_len = conf->off_in_head;
-	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
-					gf_crypt_mt_data);
-
-	if (local->vec.iov_base == NULL) {
-	        gf_log(this->name, GF_LOG_WARNING,
-                       "Failed to calloc head block for prune");
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-	        goto put_one_call;
-	}
-	for (i = 0; i < count; i++) {
-		to_copy = vec[i].iov_len;
-		if (to_copy > local->vec.iov_len - copied)
-			to_copy = local->vec.iov_len - copied;
-
-		memcpy((char *)local->vec.iov_base + copied,
-		       vec[i].iov_base,
-		       to_copy);
-		copied += to_copy;
-		if (copied == local->vec.iov_len)
-			break;
-	}
-	/*
-	 * perform prune with aligned offset
-	 * (i.e. at this step we prune a bit
-	 * more then it is needed
-	 */
-	STACK_WIND(frame,
-		   prune_submit_file_tail,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->ftruncate,
-		   local->fd,
-		   conf->aligned_offset,
-		   local->xdata);
-	return 0;
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * Perform a read-prune-write sequence
- */
-int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
-{
-	int32_t ret = 0;
-	dict_t *dict = NULL;
-	crypt_local_t  *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_local_io_params_ftruncate(frame, object);
-	get_one_call_nolock(frame);
-
-	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
-		/*
-		 * cblock-aligned prune:
-		 * we don't need read and write components,
-		 * just cut file body
-		 */
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "prune without RMW (at offset %llu",
-		       (unsigned long long)conf->orig_offset);
-
-		STACK_WIND(frame,
-			   prune_complete,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->ftruncate,
-			   local->fd,
-			   conf->orig_offset,
-			   local->xdata);
-		return 0;
-	}
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "prune with RMW (at offset %llu",
-	       (unsigned long long)conf->orig_offset);
-	/*
-	 * We are about to perform the "read" component of the
-	 * read-prune-write sequence. It means that we need to
-	 * read encrypted data from disk and decrypt it.
-	 * So, the crypt translator does STACK_WIND to itself.
-	 *
-	 * Pass current file size to crypt_readv()
-
-	 */
-	dict = dict_new();
-	if (!dict) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
-		ret = ENOMEM;
-		goto exit;
-	}
-	ret = dict_set(dict,
-		       FSIZE_XATTR_PREFIX,
-		       data_from_uint64(local->cur_file_size));
-	if (ret) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   prune_write,
-		   this,
-		   this->fops->readv, /* crypt_readv */
-		   local->fd,
-		   get_atom_size(object), /* bytes to read */
-		   conf->aligned_offset, /* offset to read from */
-		   0,
-		   dict);
- exit:
-	if (dict)
-		dict_unref(dict);
-	return ret;
-}
-
-/*
- * File prune is more complicated than expand.
- * First we need to read the latest atom to not lose info
- * needed for proper update. Also we need to make sure that
- * every component of read-prune-write sequence leaves data
- * consistent
- *
- * Non-cblock aligned prune is performed as read-prune-write
- * sequence:
- *
- * 1) read the latest atom;
- * 2) perform cblock-aligned prune
- * 3) issue a write request for the end-of-file
- */
-int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
-{
-	int32_t ret;
-
-	ret = prepare_for_prune(frame, this, offset);
-	if (ret)
-		return ret;
-	return read_prune_write(frame, this);
-}
-
-int32_t expand_file(call_frame_t *frame, xlator_t *this,
-		    uint64_t offset)
-{
-	int32_t ret;
-	crypt_local_t *local = frame->local;
-
-	ret = prepare_for_submit_hole(frame, this,
-				      local->old_file_size,
-				      offset - local->old_file_size);
-	if (ret)
-		return ret;
-	submit_hole(frame, this);
-	return 0;
-}
-
-static int32_t ftruncate_trivial_completion(call_frame_t *frame,
-					    void *cookie,
-					    xlator_t *this,
-					    int32_t op_ret,
-					    int32_t op_errno,
-					    struct iatt *buf,
-					    dict_t *dict)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->prebuf = *buf;
-	local->postbuf = *buf;
-
-	local->prebuf.ia_size = local->cur_file_size;
-	local->postbuf.ia_size = local->cur_file_size;
-
-	get_one_call(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-static int32_t do_ftruncate(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dict_t *dict,
-			    dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	if (op_ret)
-		goto error;
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_errno = EIO;
-		goto error;
-	}
-	local->old_file_size = local->cur_file_size = data_to_uint64(data);
-
-	if (local->data_conf.orig_offset == local->cur_file_size) {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "trivial ftruncate (current file size %llu)",
-		       (unsigned long long)local->cur_file_size);
-#endif
-		goto trivial;
-	}
-	else if (local->data_conf.orig_offset < local->cur_file_size) {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
-		       (unsigned long long)local->cur_file_size,
-		       (unsigned long long)local->data_conf.orig_offset);
-#endif
-		op_errno = prune_file(frame,
-				      this,
-				      local->data_conf.orig_offset);
-	}
-	else {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
-		       (unsigned long long)local->cur_file_size,
-		       (unsigned long long)local->data_conf.orig_offset);
-#endif
-		op_errno = expand_file(frame,
-				       this,
-				       local->data_conf.orig_offset);
-	}
-	if (op_errno)
-		goto error;
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   ftruncate_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	/*
-	 * finish with ftruncate
-	 */
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-
-	get_one_call_nolock(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
-					    void *cookie,
-					    xlator_t *this,
-					    int32_t op_ret,
-					    int32_t op_errno,
-					    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size first
-	 */
-	STACK_WIND(frame,
-		   do_ftruncate,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * ftruncate is performed in 2 steps:
- * . receive file size;
- * . expand or prune file.
- */
-static int32_t crypt_ftruncate(call_frame_t *frame,
-			       xlator_t *this,
-			       fd_t *fd,
-			       off_t offset,
-			       dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-
-	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		goto error;
-	}
-	local->data_conf.orig_offset = offset;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_ftruncate_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	if (local && local->fd)
-		fd_unref(fd);
-	if (local && local->xdata)
-		dict_unref(xdata);
-	if (local && local->xattr)
-		dict_unref(local->xattr);
-	if (local && local->info)
-		free_inode_info(local->info);
-
-	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
-	return 0;
-}
-
-/* ->flush_cbk() */
-int32_t truncate_end(call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_UNWIND_STRICT(truncate,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local->xdata);
-	return 0;
-}
-
-/* ftruncate_cbk() */
-int32_t truncate_flush(call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-		       struct iatt *postbuf,
-		       dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *fd = local->fd;
-	local->prebuf = *prebuf;
-	local->postbuf = *postbuf;
-
-	STACK_WIND(frame,
-		   truncate_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   fd,
-		   NULL);
-	fd_unref(fd);
-	return 0;
-}
-
-/*
- * is called as ->open_cbk()
- */
-static int32_t truncate_begin(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      fd_t *fd,
-			      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0) {
-		fd_unref(fd);
-		STACK_UNWIND_STRICT(truncate,
-				    frame,
-				    op_ret,
-				    op_errno, NULL, NULL, NULL);
-		return 0;
-	} else {
-	        fd_bind (fd);
-        }
-	/*
-	 * crypt_truncate() is implemented via crypt_ftruncate(),
-	 * so the crypt xlator does STACK_WIND to itself here
-	 */
-	STACK_WIND(frame,
-		   truncate_flush,
-		   this,
-		   this->fops->ftruncate, /* crypt_ftruncate */
-		   fd,
-		   local->offset,
-		   NULL);
-	return 0;
-}
-
-/*
- * crypt_truncate() is implemented via crypt_ftruncate() as a
- * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
- */
-int32_t crypt_truncate(call_frame_t *frame,
-		       xlator_t *this,
-		       loc_t *loc,
-		       off_t offset,
-		       dict_t *xdata)
-{
-	fd_t *fd;
-	crypt_local_t *local;
-
-#if DEBUG_CRYPT
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "truncate file %s at offset %llu",
-	       loc->path, (unsigned long long)offset);
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
-	if (!local)
-		goto error;
-
-	fd = fd_create(loc->inode, frame->root->pid);
-	if (!fd) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
-		goto error;
-	}
-	local->fd = fd;
-	local->offset = offset;
-	local->xdata = xdata;
-	STACK_WIND(frame,
-		   truncate_begin,
-		   this,
-		   this->fops->open, /* crypt_open() */
-		   loc,
-		   O_RDWR,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
-	return 0;
-}
-
-end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
-{
-	switch (fop) {
-	case GF_FOP_WRITE:
-		return end_writeback_writev;
-	case GF_FOP_FTRUNCATE:
-		return end_writeback_ftruncate;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
-		return NULL;
-	}
-}
-
-/*
- * true, if the caller needs metadata string
- */
-static int32_t is_custom_mtd(dict_t *xdata)
-{
-	data_t *data;
-	uint32_t flags;
-
-	if (!xdata)
-		return 0;
-
-	data = dict_get(xdata, MSGFLAGS_PREFIX);
-	if (!data)
-		return 0;
-	if (data->len != sizeof(uint32_t)) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "Bad msgflags size (%d)", data->len);
-		return -1;
-	}
-	flags = *((uint32_t *)data->data);
-	return msgflags_check_mtd_lock(&flags);
-}
-
-static int32_t crypt_open_done(call_frame_t *frame,
-			       void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret,
-			       int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	if (op_ret < 0)
-		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
-		       op_errno);
-	put_one_call_open(frame);
-	return 0;
-}
-
-static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   crypt_open_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-}
-
-/*
- * load private inode info at open time
- * called as ->fgetxattr_cbk()
- */
-static int load_mtd_open(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *dict,
-			 dict_t *xdata)
-{
-	int32_t ret;
-	gf_boolean_t upload_info;
-	data_t *mtd;
-	uint64_t value = 0;
-	struct crypt_inode_info *info;
-	crypt_local_t *local = frame->local;
-	crypt_private_t *priv = this->private;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (local->fd->inode->ia_type == IA_IFLNK)
-		goto exit;
-	if (op_ret < 0)
-		goto exit;
-	/*
-	 * first, check for cached info
-	 */
-	ret = inode_ctx_get(local->fd->inode, this, &value);
-	if (ret != -1) {
-		info = (struct crypt_inode_info *)(long)value;
-		if (info == NULL) {
-			gf_log(this->name, GF_LOG_WARNING,
-			       "Inode info expected, but not found");
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-		/*
-		 * info has been found in the cache
-		 */
-		upload_info = _gf_false;
-	}
-	else {
-		/*
-		 * info hasn't been found in the cache.
-		 */
-		info = alloc_inode_info(local, local->loc);
-		if (!info) {
-			local->op_ret = -1;
-			local->op_errno = ENOMEM;
-			goto exit;
-		}
-		init_inode_info_head(info, local->fd);
-		upload_info = _gf_true;
-	}
-	/*
-	 * extract metadata
-	 */
-	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
-	if (!mtd) {
-		local->op_ret = -1;
-		local->op_errno = ENOENT;
-		gf_log (this->name, GF_LOG_WARNING,
-			"Format string wasn't found");
-		goto exit;
-	}
-	/*
-	 * authenticate metadata against the path
-	 */
-	ret = open_format((unsigned char *)mtd->data,
-			  mtd->len,
-			  local->loc,
-			  info,
-			  get_master_cinfo(priv),
-			  local,
-			  upload_info);
-	if (ret) {
-		local->op_ret = -1;
-		local->op_errno = ret;
-		goto exit;
-	}
-	if (upload_info) {
-		ret = init_inode_info_tail(info, get_master_cinfo(priv));
-		if (ret) {
-			local->op_ret = -1;
-			local->op_errno = ret;
-			goto exit;
-		}
-		ret = inode_ctx_put(local->fd->inode,
-				    this, (uint64_t)(long)info);
-		if (ret == -1) {
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-	}
-	if (local->custom_mtd) {
-		/*
-		 * pass the metadata string to the customer
-		 */
-		ret = dict_set_static_bin(local->xdata,
-					  CRYPTO_FORMAT_PREFIX,
-					  mtd->data,
-					  mtd->len);
-		if (ret) {
-			local->op_ret = -1;
-			local->op_errno = ret;
-			goto exit;
-		}
-	}
- exit:
-	if (!local->custom_mtd)
-		crypt_open_tail(frame, this);
-	else
-		put_one_call_open(frame);
-	return 0;
-}
-
-static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
-				       void *cookie,
-				       xlator_t *this,
-				       int32_t op_ret,
-				       int32_t op_errno,
-				       dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   load_mtd_open,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   CRYPTO_FORMAT_PREFIX,
-		   NULL);
-	return 0;
- exit:
-	put_one_call_open(frame);
-	return 0;
-}
-
-/*
- * verify metadata against the specified pathname
- */
-static int32_t crypt_open_cbk(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      fd_t *fd,
-			      dict_t *xdata)
-{
-	struct gf_flock lock = {0, };
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (local->fd->inode->ia_type == IA_IFLNK)
-		goto exit;
-	if (op_ret < 0)
-		goto exit;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	else if (local->custom_mtd){
-		local->xdata = dict_new();
-		if (!local->xdata) {
-			local->op_ret = -1;
-			local->op_errno = ENOMEM;
-			gf_log ("crypt", GF_LOG_ERROR,
-				"Can not get new dict for mtd string");
-			goto exit;
-		}
-	}
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_open_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- exit:
-	put_one_call_open(frame);
-	return 0;
-}
-
-static int32_t crypt_open(call_frame_t *frame,
-			  xlator_t *this,
-			  loc_t *loc,
-			  int32_t flags,
-			  fd_t *fd,
-			  dict_t *xdata)
-{
-	int32_t ret = ENOMEM;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc) {
-		ret = ENOMEM;
-		goto error;
-	}
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	ret = is_custom_mtd(xdata);
-	if (ret < 0) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-		ret = EINVAL;
-		goto error;
-	}
-	local->custom_mtd = ret;
-
-	if ((flags & O_ACCMODE) == O_WRONLY)
-		/*
-		 * we can't open O_WRONLY, because
-		 * we need to do read-modify-write
-		 */
-		flags = (flags & ~O_ACCMODE) | O_RDWR;
-	/*
-	 * Make sure that out translated offsets
-	 * and counts won't be ignored
-	 */
-	flags &= ~O_APPEND;
-	get_one_call_nolock(frame);
-	STACK_WIND(frame,
-		   crypt_open_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->open,
-		   loc,
-		   flags,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(open,
-			    frame,
-			    -1,
-			    ret,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t init_inode_info_tail(struct crypt_inode_info *info,
-				    struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct object_cipher_info *object = &info->cinfo;
-
-#if DEBUG_CRYPT
-	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
-	       uuid_utoa(info->oid));
-#endif
-	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
-									master);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
-		return ret;
-	}
-	return 0;
-}
-
-/*
- * Init inode info at ->create() time
- */
-static void init_inode_info_create(struct crypt_inode_info *info,
-				   struct master_cipher_info *master,
-				   data_t *data)
-{
-	struct object_cipher_info *object;
-
-	info->nr_minor = CRYPT_XLATOR_ID;
-	memcpy(info->oid, data->data, data->len);
-
-	object = &info->cinfo;
-
-	object->o_alg        = master->m_alg;
-	object->o_mode       = master->m_mode;
-	object->o_block_bits = master->m_block_bits;
-	object->o_dkey_size  = master->m_dkey_size;
-}
-
-static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
-{
-	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
-}
-
-static int32_t crypt_create_done(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno, dict_t *xdata)
-{
-	crypt_private_t *priv = this->private;
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	if (op_ret < 0) {
-		free_inode_info(info);
-		goto unwind;
-	}
-	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
-	if (op_errno) {
-		op_ret = -1;
-		free_inode_info(info);
-		goto unwind;
-	}
-	/*
-	 * FIXME: drop major subversion number
-	 */
-	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
-	if (op_ret == -1) {
-		op_errno = EIO;
-		free_inode_info(info);
-		goto unwind;
-	}
- unwind:
-	free_format(local);
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    local_fd,
-			    local_inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	inode_unref(local_inode);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int crypt_create_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     dict_t *xdata)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	dict_unref(local->xattr);
-
-	if (op_ret < 0)
-		goto error;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   crypt_create_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	free_inode_info(local->info);
-	free_format(local);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    local_fd,
-			    local_inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-
-	fd_unref(local_fd);
-	inode_unref(local_inode);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-
-	if (op_ret < 0)
-		goto error;
-
-	STACK_WIND(frame,
-		   crypt_create_tail,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fsetxattr,
-		   local->fd,
-		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-		   0,
-		   NULL);
-	return 0;
- error:
-	free_inode_info(info);
-	free_format(local);
-	fd_unref(local->fd);
-	dict_unref(local->xattr);
-	if (local->xdata)
-		dict_unref(local->xdata);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-/*
- * Create and store crypt-specific format on disk;
- * Populate cache with private inode info
- */
-static int32_t crypt_create_cbk(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				fd_t *fd,
-				inode_t *inode,
-				struct iatt *buf,
-				struct iatt *preparent,
-				struct iatt *postparent,
-				dict_t *xdata)
-{
-	struct gf_flock lock = {0, };
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-
-	if (op_ret < 0)
-		goto error;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-        STACK_WIND(frame,
-		   crypt_create_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	free_inode_info(info);
-	free_format(local);
-	fd_unref(local->fd);
-	dict_unref(local->xattr);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    NULL, NULL, NULL,
-			    NULL, NULL, NULL);
-	return 0;
-}
-
-static int32_t crypt_create(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc,
-			    int32_t flags,
-			    mode_t mode,
-			    mode_t umask,
-			    fd_t *fd,
-			    dict_t *xdata)
-{
-	int ret;
-	data_t *data;
-	crypt_local_t *local;
-	crypt_private_t *priv;
-	struct master_cipher_info *master;
-	struct crypt_inode_info *info;
-
-	priv = this->private;
-	master = get_master_cinfo(priv);
-
-	if (master_alg_atomic(master)) {
-		/*
-		 * We can't open O_WRONLY, because we
-		 * need to do read-modify-write.
-		 */
-		if ((flags & O_ACCMODE) == O_WRONLY)
-			flags = (flags & ~O_ACCMODE) | O_RDWR;
-		/*
-		 * Make sure that out translated offsets
-		 * and counts won't be ignored
-		 */
-		flags &= ~O_APPEND;
-	}
-	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	data = dict_get(xdata, "gfid-req");
-	if (!data) {
-		ret = EINVAL;
-		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
-		goto error;
-	}
-	if (data->len != sizeof(uuid_t)) {
-		ret = EINVAL;
-		gf_log("crypt", GF_LOG_WARNING,
-		       "bad gfid size (%d), should be %d",
-		       (int)data->len, (int)sizeof(uuid_t));
-		goto error;
-	}
-	info = alloc_inode_info(local, loc);
-	if (!info){
-		ret = ENOMEM;
-		goto error;
-	}
-	/*
-	 * NOTE:
-	 * format has to be created BEFORE
-	 * proceeding to the untrusted server
-	 */
-	ret = alloc_format_create(local);
-	if (ret) {
-		free_inode_info(info);
-		goto error;
-	}
-	init_inode_info_create(info, master, data);
-
-	ret = create_format(local->format,
-			    loc,
-			    info,
-			    master);
-	if (ret) {
-		free_inode_info(info);
-		goto error;
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	ret = dict_set_static_bin(local->xattr,
-				  CRYPTO_FORMAT_PREFIX,
-				  local->format,
-				  new_format_size());
-	if (ret) {
-		dict_unref(local->xattr);
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
-	if (ret) {
-		dict_unref(local->xattr);
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	STACK_WIND(frame,
-		   crypt_create_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->create,
-		   loc,
-		   flags,
-		   mode,
-		   umask,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	gf_log("crypt", GF_LOG_WARNING, "can not create file");
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    -1,
-			    ret,
-			    NULL, NULL, NULL,
-			    NULL, NULL, NULL);
-	return 0;
-}
-
-/*
- * FIXME: this should depends on the version of format string
- */
-static int32_t filter_crypt_xattr(dict_t *dict,
-				  char *key, data_t *value, void *data)
-{
-        dict_del(dict, key);
-        return 0;
-}
-
-static int32_t crypt_fsetxattr(call_frame_t *frame,
-			       xlator_t *this,
-			       fd_t *fd,
-			       dict_t *dict,
-			       int32_t flags, dict_t *xdata)
-{
-	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
-			     filter_crypt_xattr, NULL);
-	STACK_WIND(frame,
-		   default_fsetxattr_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fsetxattr,
-		   fd,
-		   dict,
-		   flags,
-		   xdata);
-	return 0;
-}
-
-/*
- * TBD: verify file metadata before wind
- */
-static int32_t crypt_setxattr(call_frame_t *frame,
-			      xlator_t *this,
-			      loc_t *loc,
-			      dict_t *dict,
-			      int32_t flags, dict_t *xdata)
-{
-	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
-			     filter_crypt_xattr, NULL);
-	STACK_WIND(frame,
-		   default_setxattr_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->setxattr,
-		   loc,
-		   dict,
-		   flags,
-		   xdata);
-	return 0;
-}
-
-/*
- * called as flush_cbk()
- */
-static int32_t linkop_end(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	linkop_unwind_handler_t unwind_fn;
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0 &&
-	    op_errno == ENOENT &&
-	    local->loc->inode->ia_type == IA_IFLNK) {
-		local->op_ret = 0;
-		local->op_errno = 0;
-	}
-	unwind_fn(frame);
-	return 0;
-}
-
-/*
- * unpin inode on the server
- */
-static int32_t link_flush(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  inode_t *inode,
-			  struct iatt *buf,
-			  struct iatt *preparent,
-			  struct iatt *postparent, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-
-	STACK_WIND(frame,
-		   linkop_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	link_unwind(frame);
-	return 0;
-}
-
-void link_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-	inode_t *inode;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(link,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	inode = local->inode;
-
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->newloc) {
-		loc_wipe(local->newloc);
-		GF_FREE(local->newloc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(link,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-	if (inode)
-		inode_unref(inode);
-}
-
-void link_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   link_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->link,
-		   local->loc,
-		   local->newloc,
-		   local->xdata);
-}
-
-/*
- * unlink()
- */
-static int32_t unlink_flush(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    struct iatt *preparent,
-			    struct iatt *postparent, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	STACK_WIND(frame,
-		   linkop_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	unlink_unwind(frame);
-	return 0;
-}
-
-void unlink_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(unlink,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(unlink,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-}
-
-void unlink_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   unlink_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->unlink,
-		   local->loc,
-		   local->flags,
-		   local->xdata);
-}
-
-void rename_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-	struct iatt *prenewparent;
-	struct iatt *postnewparent;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(rename,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	prenewparent = local->prenewparent;
-	postnewparent = local->postnewparent;
-
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->newloc){
-		loc_wipe(local->newloc);
-		GF_FREE(local->newloc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(rename,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    prenewparent,
-			    postnewparent,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-	if (prenewparent)
-		GF_FREE(prenewparent);
-	if (postnewparent)
-		GF_FREE(postnewparent);
-}
-
-/*
- * called as flush_cbk()
- */
-static int32_t rename_end(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	rename_unwind(frame);
-	return 0;
-}
-
-static int32_t rename_flush(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    struct iatt *buf,
-			    struct iatt *preoldparent,
-			    struct iatt *postoldparent,
-			    struct iatt *prenewparent,
-			    struct iatt *postnewparent,
-			    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	dict_unref(local->xdata);
-	local->xdata = NULL;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	local->buf = *buf;
-	local->prebuf = *preoldparent;
-	local->postbuf = *postoldparent;
-	if (prenewparent) {
-		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
-						gf_crypt_mt_iatt);
-		if (!local->prenewparent) {
-			op_errno = ENOMEM;
-			goto error;
-		}
-		*local->prenewparent = *prenewparent;
-	}
-	if (postnewparent) {
-		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
-						 gf_crypt_mt_iatt);
-		if (!local->postnewparent) {
-			op_errno = ENOMEM;
-			goto error;
-		}
-		*local->postnewparent = *postnewparent;
-	}
-	STACK_WIND(frame,
-		   rename_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	rename_unwind(frame);
-	return 0;
-}
-
-void rename_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   rename_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->rename,
-		   local->loc,
-		   local->newloc,
-		   local->xdata);
-}
-
-static int32_t __do_linkop(call_frame_t *frame,
-			   void *cookie,
-			   xlator_t *this,
-			   int32_t op_ret,
-			   int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	linkop_wind_handler_t wind_fn;
-	linkop_unwind_handler_t unwind_fn;
-
-	wind_fn = linkop_wind_dispatch(local->fop);
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret >= 0)
-		wind_fn(frame, this);
-	else {
-		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
-		       op_errno);
-		unwind_fn(frame);
-	}
-	return 0;
-}
-
-static int32_t do_linkop(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *xdata)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-	linkop_unwind_handler_t unwind_fn;
-
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if(op_ret < 0)
-		goto error;
-
-	lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   __do_linkop,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	unwind_fn(frame);
-	return 0;
-}
-
-/*
- * Update the metadata string (against the new pathname);
- * submit the result
- */
-static int32_t linkop_begin(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    fd_t *fd,
-			    dict_t *xdata)
-{
-	gf_boolean_t upload_info;
-	crypt_local_t *local = frame->local;
-	crypt_private_t *priv = this->private;
-	struct crypt_inode_info *info;
-	data_t *old_mtd;
-	uint32_t new_mtd_size;
-	uint64_t value = 0;
-	void (*unwind_fn)(call_frame_t *frame);
-	mtd_op_t mop;
-
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-	mop = linkop_mtdop_dispatch(local->fop);
-
-	if (op_ret < 0) {
-		/*
-		 * verification failed
-		 */
-		goto error;
-        } else {
-                fd_bind (fd);
-        }
-
-	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
-	if (!old_mtd) {
-		op_errno = EIO;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Metadata string wasn't found");
-		goto error;
-	}
-	new_mtd_size = format_size(mop, old_mtd->len);
-	op_errno = alloc_format(local, new_mtd_size);
-	if (op_errno)
-		goto error;
-	/*
-	 * check for cached info
-	 */
-	op_ret = inode_ctx_get(fd->inode, this, &value);
-	if (op_ret != -1) {
-		info = (struct crypt_inode_info *)(long)value;
-		if (info == NULL) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Inode info was not found");
-			op_errno = EINVAL;
-			goto error;
-		}
-		/*
-		 * info was found in the cache
-		 */
-		local->info = info;
-		upload_info = _gf_false;
-	}
-	else {
-		/*
-		 * info wasn't found in the cache;
-		 */
-		info = alloc_inode_info(local, local->loc);
-		if (!info)
-			goto error;
-		init_inode_info_head(info, fd);
-		local->info = info;
-		upload_info = _gf_true;
-	}
-	op_errno = open_format((unsigned char *)old_mtd->data,
-			       old_mtd->len,
-			       local->loc,
-			       info,
-			       get_master_cinfo(priv),
-			       local,
-			       upload_info);
-	if (op_errno)
-		goto error;
-	if (upload_info == _gf_true) {
-		op_errno = init_inode_info_tail(info,
-						get_master_cinfo(priv));
-		if (op_errno)
-			goto error;
-		op_errno = inode_ctx_put(fd->inode, this,
-					 (uint64_t)(long)(info));
-		if (op_errno == -1) {
-			op_errno = EIO;
-			goto error;
-		}
-	}
-	/*
-	 * update the format string (append/update/cup a MAC)
-	 */
-	op_errno = update_format(local->format,
-				 (unsigned char *)old_mtd->data,
-				 old_mtd->len,
-				 local->mac_idx,
-				 mop,
-				 local->newloc,
-				 info,
-				 get_master_cinfo(priv),
-				 local);
-	if (op_errno)
-		goto error;
-	/*
-	 * store the new format string on the server
-	 */
-	if (new_mtd_size) {
-		op_errno = dict_set_static_bin(local->xattr,
-					       CRYPTO_FORMAT_PREFIX,
-					       local->format,
-					       new_mtd_size);
-		if (op_errno)
-			goto error;
-	}
-	STACK_WIND(frame,
-		   do_linkop,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->setxattr,
-		   local->loc,
-		   local->xattr,
-		   0,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	unwind_fn(frame);
-	return 0;
-}
-
-static int32_t linkop_grab_local(call_frame_t *frame,
-				 xlator_t *this,
-				 loc_t *oldloc,
-				 loc_t *newloc,
-				 int flags, dict_t *xdata,
-				 glusterfs_fop_t op)
-{
-	int32_t ret = ENOMEM;
-	fd_t *fd;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, op);
-	if (!local)
-		goto error;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	fd = fd_create(oldloc->inode, frame->root->pid);
-	if (!fd) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
-		goto error;
-	}
-	local->fd = fd;
-	local->flags = flags;
-	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, oldloc);
-	if (ret) {
-		GF_FREE(local->loc);
-		local->loc = NULL;
-		goto error;
-	}
-	if (newloc) {
-		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
-		if (!local->newloc) {
-			loc_wipe(local->loc);
-			GF_FREE(local->loc);
-			goto error;
-		}
-		memset(local->newloc, 0, sizeof(*local->newloc));
-		ret = loc_copy(local->newloc, newloc);
-		if (ret) {
-			loc_wipe(local->loc);
-			GF_FREE(local->loc);
-			GF_FREE(local->newloc);
-			goto error;
-		}
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
-		ret = ENOMEM;
-		goto error;
-	}
-	return 0;
-
-error:
-        if (local) {
-                if (local->xdata)
-                        dict_unref(local->xdata);
-                if (local->fd)
-                        fd_unref(local->fd);
-                local->fd = 0;
-                local->loc = NULL;
-                local->newloc = NULL;
-                local->op_ret = -1;
-                local->op_errno = ret;
-        }
-
-        return ret;
-}
-
-/*
- * read and verify locked metadata against the old pathname (via open);
- * update the metadata string in accordance with the new pathname;
- * submit modified metadata;
- * wind;
- */
-static int32_t linkop(call_frame_t *frame,
-		      xlator_t *this,
-		      loc_t *oldloc,
-		      loc_t *newloc,
-		      int flags,
-		      dict_t *xdata,
-		      glusterfs_fop_t op)
-{
-	int32_t ret;
-	dict_t *dict;
-	crypt_local_t *local;
-	void (*unwind_fn)(call_frame_t *frame);
-        void (*wind_fn)(call_frame_t *frame, xlator_t *this);
-
-        wind_fn = linkop_wind_dispatch(op);
-	unwind_fn = linkop_unwind_dispatch(op);
-
-	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
-	local = frame->local;
-	if (ret)
-		goto error;
-
-        if (local->fd->inode->ia_type == IA_IFLNK)
-                goto wind;
-
-	dict = dict_new();
-	if (!dict) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
-		ret = ENOMEM;
-		goto error;
-	}
-	/*
-	 * Set a message to crypt_open() that we need
-	 * locked metadata string.
-	 * All link operations (link, unlink, rename)
-	 * need write lock
-	 */
-	msgflags_set_mtd_wlock(&local->msgflags);
-	ret = dict_set_static_bin(dict,
-				  MSGFLAGS_PREFIX,
-				  &local->msgflags,
-				  sizeof(local->msgflags));
-	if (ret) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
-		dict_unref(dict);
-		goto error;
-	}
-	/*
-	 * verify metadata against the old pathname
-	 * and retrieve locked metadata string
-	 */
-	STACK_WIND(frame,
-		   linkop_begin,
-		   this,
-		   this->fops->open, /* crypt_open() */
-		   oldloc,
-		   O_RDWR,
-		   local->fd,
-		   dict);
-	dict_unref(dict);
-	return 0;
-
-wind:
-        wind_fn(frame, this);
-        return 0;
-
-error:
-	local->op_ret = -1;
-	local->op_errno = ret;
-	unwind_fn(frame);
-	return 0;
-}
-
-static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
-			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
-}
-
-static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
-			    loc_t *loc, int flags, dict_t *xdata)
-{
-	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
-}
-
-static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
-			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
-}
-
-static void put_one_call_open(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		fd_t *fd = local->fd;
-		loc_t *loc = local->loc;
-		dict_t *xdata = local->xdata;
-
-		STACK_UNWIND_STRICT(open,
-				    frame,
-				    local->op_ret,
-				    local->op_errno,
-				    fd,
-				    xdata);
-		fd_unref(fd);
-		if (xdata)
-			dict_unref(xdata);
-		loc_wipe(loc);
-		GF_FREE(loc);
-	}
-}
-
-static int32_t __crypt_readv_done(call_frame_t *frame,
-				  void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret,
-				  int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	/* read deals with data configs only */
-	struct iovec *avec = local->data_conf.avec;
-	char **pool = local->data_conf.pool;
-	int blocks_in_pool = local->data_conf.blocks_in_pool;
-	struct iobref *iobref = local->iobref;
-	struct iobref *iobref_data = local->iobref_data;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "readv unlock failed (%d)", op_errno);
-		if (local->op_ret >= 0) {
-			local->op_ret = op_ret;
-			local->op_errno = op_errno;
-		}
-	}
-	dump_plain_text(local, avec);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
-	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
-	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
-	       (unsigned long long)local->buf.ia_size);
-
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    local->rw_count > 0 ? local->rw_count : local->op_ret,
-			    local->op_errno,
-			    avec,
-			    avec ? local->data_conf.acount : 0,
-			    &local->buf,
-			    local->iobref,
-			    local_xdata);
-
-	free_avec(avec, pool, blocks_in_pool);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (iobref)
-		iobref_unref(iobref);
-	if (iobref_data)
-		iobref_unref(iobref_data);
-	return 0;
-}
-
-static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
-{
-	if (parent_is_crypt_xlator(frame, this))
-		/*
-		 *  don't unlock (it will be done by the parent)
-		 */
-		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
-	else {
-		crypt_local_t *local = frame->local;
-		struct gf_flock  lock  = {0, };
-
-		lock.l_type   = F_UNLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start  = 0;
-		lock.l_len    = 0;
-		lock.l_pid    = 0;
-
-		STACK_WIND(frame,
-			   __crypt_readv_done,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->finodelk,
-			   this->name,
-			   local->fd,
-			   F_SETLKW,
-			   &lock,
-			   NULL);
-	}
-}
-
-static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local))
-		crypt_readv_done(frame, this);
-}
-
-static int32_t __crypt_writev_done(call_frame_t *frame,
-				   void *cookie,
-				   xlator_t *this,
-				   int32_t op_ret,
-				   int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	int32_t ret_to_user;
-
-	if (local->xattr)
-		dict_unref(local->xattr);
-	/*
-	 * Calculate amout of butes to be returned
-	 * to user. We need to subtract paddings that
-	 * have been written as a part of atom.
-	 */
-	/*
-	 * subtract head padding
-	 */
-	if (local->rw_count == 0)
-		/*
-		 * Nothing has been written, it must be an error
-		 */
-		ret_to_user = local->op_ret;
-	else if (local->rw_count <= local->data_conf.off_in_head) {
-		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
-		ret_to_user = 0;
-	}
-	else
-		ret_to_user = local->rw_count -
-			local->data_conf.off_in_head;
-	/*
-	 * subtract tail padding
-	 */
-	if (ret_to_user > local->data_conf.orig_size)
-		ret_to_user = local->data_conf.orig_size;
-
-	if (local->iobref)
-		iobref_unref(local->iobref);
-	if (local->iobref_data)
-		iobref_unref(local->iobref_data);
-	free_avec_data(local);
-	free_avec_hole(local);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "writev: ret_to_user: %d", ret_to_user);
-
-	STACK_UNWIND_STRICT(writev,
-			    frame,
-			    ret_to_user,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int32_t crypt_writev_done(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno,
-				 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
-
-	if (parent_is_crypt_xlator(frame, this))
-		/*
-		 * don't unlock (it will be done by the parent)
-		 */
-		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
-	else {
-		struct gf_flock  lock  = {0, };
-
-		lock.l_type   = F_UNLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start  = 0;
-		lock.l_len    = 0;
-		lock.l_pid    = 0;
-
-		STACK_WIND(frame,
-			   __crypt_writev_done,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->finodelk,
-			   this->name,
-			   local->fd,
-			   F_SETLKW,
-			   &lock,
-			   NULL);
-	}
-	return 0;
-}
-
-static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		if (local->update_disk_file_size) {
-			int32_t ret;
-			/*
-			 * update file size, unlock the file and unwind
-			 */
-			ret = dict_set(local->xattr,
-				       FSIZE_XATTR_PREFIX,
-				       data_from_uint64(local->cur_file_size));
-			if (ret) {
-				gf_log("crypt", GF_LOG_WARNING,
-				       "can not set key to update file size");
-				crypt_writev_done(frame, NULL,
-						  this, 0, 0, NULL);
-				return;
-			}
-			gf_log("crypt", GF_LOG_DEBUG,
-			       "Updating disk file size to %llu",
-			       (unsigned long long)local->cur_file_size);
-			STACK_WIND(frame,
-				   crypt_writev_done,
-				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->fsetxattr,
-				   local->fd,
-				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-				   0,
-				   NULL);
-		}
-		else
-			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
-	}
-}
-
-static int32_t __crypt_ftruncate_done(call_frame_t *frame,
-				      void *cookie,
-				      xlator_t *this,
-				      int32_t op_ret,
-				      int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	char *iobase = local->vec.iov_base;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "ftruncate unlock failed (%d)", op_errno);
-		if (local->op_ret >= 0) {
-			local->op_ret = op_ret;
-			local->op_errno = op_errno;
-		}
-	}
-	if (local->iobref_data)
-		iobref_unref(local->iobref_data);
-	free_avec_data(local);
-	free_avec_hole(local);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "ftruncate, return to user: presize=%llu, postsize=%llu",
-	       (unsigned long long)local->prebuf.ia_size,
-	       (unsigned long long)local->postbuf.ia_size);
-
-	STACK_UNWIND_STRICT(ftruncate,
-			    frame,
-			    local->op_ret < 0 ? -1 : 0,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (iobase)
-		GF_FREE(iobase);
-	return 0;
-}
-
-static int32_t crypt_ftruncate_done(call_frame_t *frame,
-				    void *cookie,
-				    xlator_t *this,
-				    int32_t op_ret,
-				    int32_t op_errno,
-				    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct gf_flock  lock  = {0, };
-
-	dict_unref(local->xattr);
-	if (op_ret < 0)
-		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
-
-	lock.l_type   = F_UNLCK;
-	lock.l_whence = SEEK_SET;
-	lock.l_start  = 0;
-	lock.l_len    = 0;
-	lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   __crypt_ftruncate_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
-}
-
-static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		if (local->update_disk_file_size) {
-			int32_t ret;
-			/*
-			 * update file size, unlock the file and unwind
-			 */
-			ret = dict_set(local->xattr,
-				       FSIZE_XATTR_PREFIX,
-				       data_from_uint64(local->cur_file_size));
-			if (ret) {
-				gf_log("crypt", GF_LOG_WARNING,
-				       "can not set key to update file size");
-				crypt_ftruncate_done(frame, NULL,
-						     this, 0, 0, NULL);
-				return;
-			}
-			gf_log("crypt", GF_LOG_DEBUG,
-			       "Updating disk file size to %llu",
-			       (unsigned long long)local->cur_file_size);
-			STACK_WIND(frame,
-				   crypt_ftruncate_done,
-				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->fsetxattr,
-				   local->fd,
-				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-				   0,
-				   NULL);
-		}
-		else
-			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
-	}
-}
-
-/*
- * load regular file size for some FOPs
- */
-static int32_t load_file_size(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      dict_t *dict,
-			      dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	if (op_ret < 0)
-		goto unwind;
-	/*
-	 * load regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		if (local->xdata)
-			dict_unref(local->xdata);
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_ret = -1;
-		op_errno = EIO;
-		goto unwind;
-	}
-	local->buf.ia_size = data_to_uint64(data);
-
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "FOP %d: Translate regular file to %llu",
-	       local->fop,
-	       (unsigned long long)local->buf.ia_size);
- unwind:
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->loc) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_UNWIND_STRICT(fstat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata);
-		break;
-	case GF_FOP_STAT:
-		STACK_UNWIND_STRICT(stat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata);
-		break;
-	case GF_FOP_LOOKUP:
-		STACK_UNWIND_STRICT(lookup,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? local->inode : NULL,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata,
-				    op_ret >= 0 ? &local->postbuf : NULL);
-		break;
-	case GF_FOP_READ:
-		STACK_UNWIND_STRICT(readv,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    NULL,
-				    0,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    NULL,
-				    NULL);
-		break;
-	default:
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Improper file operation %d", local->fop);
-	}
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (local_inode)
-		inode_unref(local_inode);
-	return 0;
-}
-
-static int32_t crypt_stat_common_cbk(call_frame_t *frame,
-				     void *cookie,
-				     xlator_t *this,
-				     int32_t op_ret,
-				     int32_t op_errno,
-				     struct iatt *buf, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto unwind;
-	if (!IA_ISREG(buf->ia_type))
-		goto unwind;
-
-	local->buf = *buf;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_WIND(frame,
-			   load_file_size,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->fgetxattr,
-			   local->fd,
-			   FSIZE_XATTR_PREFIX,
-			   NULL);
-		break;
-	case GF_FOP_STAT:
-		STACK_WIND(frame,
-			   load_file_size,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->getxattr,
-			   local->loc,
-			   FSIZE_XATTR_PREFIX,
-			   NULL);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_WARNING,
-			"Improper file operation %d", local->fop);
-	}
-	return 0;
- unwind:
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->loc) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_UNWIND_STRICT(fstat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? buf : NULL,
-				    op_ret >= 0 ? xdata : NULL);
-		break;
-	case GF_FOP_STAT:
-		STACK_UNWIND_STRICT(stat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? buf : NULL,
-				    op_ret >= 0 ? xdata : NULL);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_WARNING,
-			"Improper file operation %d", local->fop);
-	}
-	return 0;
-}
-
-static int32_t crypt_fstat(call_frame_t *frame,
-			   xlator_t *this,
-			   fd_t *fd, dict_t *xdata)
-{
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
-	if (!local)
-		goto error;
-	local->fd = fd_ref(fd);
-	STACK_WIND(frame,
-		   crypt_stat_common_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(fstat,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t crypt_stat(call_frame_t *frame,
-			  xlator_t *this,
-			  loc_t *loc, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	STACK_WIND(frame,
-		   crypt_stat_common_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->stat,
-		   loc,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(stat,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t crypt_lookup_cbk(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				inode_t *inode,
-				struct iatt *buf, dict_t *xdata,
-				struct iatt *postparent)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto unwind;
-	if (!IA_ISREG(buf->ia_type))
-		goto unwind;
-
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->postbuf = *postparent;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	gf_uuid_copy(local->loc->gfid, buf->ia_gfid);
-
-	STACK_WIND(frame,
-		   load_file_size,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->getxattr,
-		   local->loc,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- unwind:
-	loc_wipe(local->loc);
-	GF_FREE(local->loc);
-	STACK_UNWIND_STRICT(lookup,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    inode,
-			    buf,
-			    xdata,
-			    postparent);
-	return 0;
-}
-
-static int32_t crypt_lookup(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
-	STACK_WIND(frame,
-		   crypt_lookup_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->lookup,
-		   loc,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(lookup,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-/*
- * for every regular directory entry find its real file size
- * and update stat's buf properly
- */
-static int32_t crypt_readdirp_cbk(call_frame_t *frame,
-				  void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret,
-				  int32_t op_errno,
-				  gf_dirent_t *entries, dict_t *xdata)
-{
-	gf_dirent_t *entry = NULL;
-
-	if (op_ret < 0)
-		goto unwind;
-
-	list_for_each_entry (entry, (&entries->list), list) {
-		data_t *data;
-
-		if (!IA_ISREG(entry->d_stat.ia_type))
-			continue;
-		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
-		if (!data){
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size of direntry not found");
-			op_errno = EIO;
-			op_ret = -1;
-			break;
-		}
-		entry->d_stat.ia_size = data_to_uint64(data);
-	}
- unwind:
-	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
-}
-
-/*
- * ->readdirp() fills in-core inodes, so we need to set proper
- * file sizes for all directory entries of the parent @fd.
- * Actual updates take place in ->crypt_readdirp_cbk()
- */
-static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
-			      fd_t *fd, size_t size, off_t offset,
-			      dict_t *xdata)
-{
-	int32_t ret = ENOMEM;
-
-	if (!xdata) {
-		xdata = dict_new();
-		if (!xdata)
-			goto error;
-	}
-	else
-		dict_ref(xdata);
-	/*
-	 * make sure that we'll have real file sizes at ->readdirp_cbk()
-	 */
-	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
-	if (ret) {
-		dict_unref(xdata);
-		goto error;
-	}
-	STACK_WIND(frame,
-		   crypt_readdirp_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->readdirp,
-		   fd,
-		   size,
-		   offset,
-		   xdata);
-	dict_unref(xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
-	return 0;
-}
-
-static int32_t crypt_access(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc,
-			    int32_t mask, dict_t *xdata)
-{
-	gf_log(this->name, GF_LOG_WARNING,
-	       "NFS mounts of encrypted volumes are unsupported");
-	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
-	return 0;
-}
-
-int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
-			       dict_t *options)
-{
-	uint64_t block_size = 0;
-	struct master_cipher_info *master = get_master_cinfo(priv);
-
-	if (options != NULL)
-		GF_OPTION_RECONF("block-size", block_size, options,
-				 size_uint64, error);
-	else
-		GF_OPTION_INIT("block-size", block_size, size_uint64, error);
-
-	switch (block_size) {
-	case 512:
-		master->m_block_bits = 9;
-		break;
-	case 1024:
-		master->m_block_bits = 10;
-		break;
-	case 2048:
-		master->m_block_bits = 11;
-		break;
-	case 4096:
-		master->m_block_bits = 12;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_ERROR,
-		       "FATAL: unsupported block size %llu",
-		       (unsigned long long)block_size);
-		goto error;
-	}
-	return 0;
- error:
-	return -1;
-}
-
-int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
-{
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	master->m_alg = AES_CIPHER_ALG;
-	return 0;
-}
-
-int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
-{
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	master->m_mode = XTS_CIPHER_MODE;
-	return 0;
-}
-
-/*
- * set key size in bits to the master info
- * Pre-conditions: cipher mode in the master info is uptodate.
- */
-static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
-				     dict_t *options)
-{
-	int32_t ret;
-	uint64_t key_size = 0;
-	struct master_cipher_info *master = get_master_cinfo(priv);
-
-	if (options != NULL)
-		GF_OPTION_RECONF("data-key-size", key_size, options,
-				 uint64, error);
-	else
-		GF_OPTION_INIT("data-key-size", key_size, uint64, error);
-
-	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "FATAL: wrong bin key size %llu for alg %d mode %d",
-		       (unsigned long long)key_size,
-		       (int)master->m_alg,
-		       (int)master->m_mode);
-		goto error;
-	}
-	master->m_dkey_size = key_size;
-	return 0;
- error:
-	return -1;
-}
-
-static int is_hex(char *s) {
-	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
-}
-
-static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
-			 int hex_size)
-{
-	int i;
-	int hex_byte = 0;
-
-	for (i = 0; i < (hex_size / 2); i++) {
-		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "FATAL: not hex symbol in key");
-			return -1;
-		}
-		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "FATAL: can not parse hex key");
-			return -1;
-		}
-		dst[i] = hex_byte & 0xff;
-	}
-	return 0;
-}
-
-/*
- * Parse options;
- * install master volume key
- */
-int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
-{
-	int32_t ret;
-	FILE *file = NULL;
-
-	int32_t key_size;
-	char *opt_key_file_pathname = NULL;
-
-	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
-	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
-
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	/*
-	 * extract master key passed via option
-	 */
-	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
-
-	if (!opt_key_file_pathname) {
-		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
-		return -1;
-	}
-	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
-	       opt_key_file_pathname);
-
-	file = fopen(opt_key_file_pathname, "r");
-	if (file == NULL) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "FATAL: can not open file with master key");
-		return -1;
-	}
-	/*
-	 * extract hex key
-	 */
-	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
-	if (key_size < sizeof(hex_buf)) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "FATAL: master key is too short");
-		goto bad_key;
-	}
-	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
-	if (ret)
-		goto bad_key;
-	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
-	memset(hex_buf, 0, sizeof(hex_buf));
-	fclose(file);
-
- 	memset(bin_buf, 0, sizeof(bin_buf));
-	return 0;
- bad_key:
-	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
-	if (file)
-		fclose(file);
-	memset(bin_buf, 0, sizeof(bin_buf));
-	return -1;
-}
-
-/*
- * Derive volume key for object-id authentication
- */
-int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
-{
-	return get_nmtd_vol_key(get_master_cinfo(priv));
-}
-
-int32_t crypt_init_xlator(xlator_t *this)
-{
-	int32_t ret;
-	crypt_private_t *priv = this->private;
-
-	ret = master_set_alg(this, priv);
-	if (ret)
-		return ret;
-	ret = master_set_mode(this, priv);
-	if (ret)
-		return ret;
-	ret = master_set_block_size(this, priv, NULL);
-	if (ret)
-		return ret;
-	ret = master_set_data_key_size(this, priv, NULL);
-	if (ret)
-		return ret;
-	ret = master_set_master_vol_key(this, priv);
-	if (ret)
-		return ret;
-	return master_set_nmtd_vol_key(this, priv);
-}
-
-static int32_t crypt_alloc_private(xlator_t *this)
-{
-	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
-	if (!this->private) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Can not allocate memory for private data");
-		return ENOMEM;
-	}
-	return 0;
-}
-
-static void crypt_free_private(xlator_t *this)
-{
-	crypt_private_t *priv = this->private;
-	if (priv) {
-		memset(priv, 0, sizeof(*priv));
-		GF_FREE(priv);
-	}
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_crypt_mt_end);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t reconfigure (xlator_t *this, dict_t *options)
-{
-	int32_t ret = -1;
-	crypt_private_t *priv = NULL;
-
-	GF_VALIDATE_OR_GOTO ("crypt", this, error);
-	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
-	GF_VALIDATE_OR_GOTO (this->name, options, error);
-
-	priv = this->private;
-
-	ret = master_set_block_size(this, priv, options);
-	if (ret) {
-		gf_log("this->name", GF_LOG_ERROR,
-		       "Failed to reconfure block size");
-		goto error;
-	}
-	ret = master_set_data_key_size(this, priv, options);
-	if (ret) {
-		gf_log("this->name", GF_LOG_ERROR,
-		       "Failed to reconfure data key size");
-		goto error;
-	}
-	return 0;
- error:
-	return ret;
-}
-
-int32_t init(xlator_t *this)
-{
-	int32_t ret;
-
-	if (!this->children || this->children->next) {
-		gf_log ("crypt", GF_LOG_ERROR,
-			"FATAL: crypt should have exactly one child");
-		return EINVAL;
-	}
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-	ret = crypt_alloc_private(this);
-	if (ret)
-		return ret;
-	ret = crypt_init_xlator(this);
-	if (ret)
-		goto error;
-	this->local_pool = mem_pool_new(crypt_local_t, 64);
-        if (!this->local_pool) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "failed to create local_t's memory pool");
-		ret = ENOMEM;
-                goto error;
-        }
-	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
-	return 0;
- error:
-	crypt_free_private(this);
-	return ret;
-}
-
-void fini (xlator_t *this)
-{
-	crypt_free_private(this);
-}
-
-struct xlator_fops fops = {
-	.readv        = crypt_readv,
-	.writev       = crypt_writev,
-	.truncate     = crypt_truncate,
-	.ftruncate    = crypt_ftruncate,
-	.setxattr     = crypt_setxattr,
-	.fsetxattr    = crypt_fsetxattr,
-	.link         = crypt_link,
-	.unlink       = crypt_unlink,
-	.rename       = crypt_rename,
-	.open         = crypt_open,
-	.create       = crypt_create,
-	.stat         = crypt_stat,
-	.fstat        = crypt_fstat,
-	.lookup       = crypt_lookup,
-	.readdirp     = crypt_readdirp,
-	.access       = crypt_access
-};
-
-struct xlator_cbks cbks = {
-	.forget       = crypt_forget
-};
-
-struct volume_options options[] = {
-	{ .key = {"master-key"},
-	  .type = GF_OPTION_TYPE_PATH,
-	  .description = "Pathname of regular file which contains master volume key"
-	},
-	{ .key = {"data-key-size"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .description = "Data key size (bits)",
-	  .min = 256,
-	  .max = 512,
-	  .default_value = "256",
-	},
-	{ .key = {"block-size"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .description = "Atom size (bits)",
-	  .min = 512,
-	  .max = 4096,
-	  .default_value = "4096"
-	},
-	{ .key  = {NULL} },
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
deleted file mode 100644
index c1bfe3fcd0c..00000000000
--- a/xlators/encryption/crypt/src/crypt.h
+++ /dev/null
@@ -1,900 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CRYPT_H__
-#define __CRYPT_H__
-
-#include <openssl/aes.h>
-#include <openssl/evp.h>
-#include <openssl/sha.h>
-#include <openssl/hmac.h>
-#include <openssl/cmac.h>
-#include <openssl/modes.h>
-#include "crypt-mem-types.h"
-#include "compat.h"
-
-#define CRYPT_XLATOR_ID  (0)
-
-#define MAX_IOVEC_BITS (3)
-#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
-#define KEY_FACTOR_BITS  (6)
-
-#define DEBUG_CRYPT (0)
-#define TRIVIAL_TFM (0)
-
-#define CRYPT_MIN_BLOCK_BITS (9)
-#define CRYPT_MAX_BLOCK_BITS (12)
-
-#define MASTER_VOL_KEY_SIZE (32)
-#define NMTD_VOL_KEY_SIZE (16)
-
-#if !defined(GF_LINUX_HOST_OS)
-typedef off_t loff_t;
-#endif
-
-struct crypt_key {
-	uint32_t len;
-	const char *label;
-};
-
-/*
- * Add new key types to the end of this
- * enumeration but before LAST_KEY_TYPE
- */
-typedef enum {
-	MASTER_VOL_KEY,
-	NMTD_VOL_KEY,
-	NMTD_LINK_KEY,
-	EMTD_FILE_KEY,
-	DATA_FILE_KEY_256,
-	DATA_FILE_KEY_512,
-	LAST_KEY_TYPE
-}crypt_key_type;
-
-struct kderive_context {
-	const unsigned char *pkey;/* parent key */
-	uint32_t pkey_len;        /* parent key size, bits */
-	uint32_t ckey_len;        /* child key size, bits */
-	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
-	uint32_t fid_len;         /* fid len, bytes */
-	unsigned char *out;       /* contains child keying material */
-	uint32_t out_len;         /* out len, bytes */
-};
-
-typedef enum {
-	DATA_ATOM,
-	HOLE_ATOM,
-	LAST_DATA_TYPE
-}atom_data_type;
-
-typedef enum {
-	HEAD_ATOM,
-	TAIL_ATOM,
-	FULL_ATOM,
-	LAST_LOCALITY_TYPE
-}atom_locality_type;
-
-typedef enum {
-	MTD_CREATE,
-	MTD_APPEND,
-	MTD_OVERWRITE,
-	MTD_CUT,
-	MTD_LAST_OP
-} mtd_op_t;
-
-struct xts128_context {
-	void      *key1, *key2;
-	block128_f block1,block2;
-};
-
-struct object_cipher_info {
-	cipher_alg_t  o_alg;
-        cipher_mode_t o_mode;
-	uint32_t      o_block_bits;
-        uint32_t      o_dkey_size; /* raw data key size in bits */
-	union {
-		struct {
-			unsigned char ivec[16];
-			AES_KEY dkey[2];
-			AES_KEY tkey;    /* key used for tweaking */
-			XTS128_CONTEXT xts;
-		} aes_xts;
-	} u;
-};
-
-struct master_cipher_info {
-	/*
-	 * attributes inherited by newly created regular files
-	 */
-	cipher_alg_t  m_alg;
-        cipher_mode_t m_mode;
-	uint32_t      m_block_bits;
-        uint32_t      m_dkey_size; /* raw key size in bits */
-	/*
-	 * master key
-	 */
-	unsigned char m_key[MASTER_VOL_KEY_SIZE];
-	/*
-	 * volume key for oid authentication
-	 */
-	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
-};
-
-/*
-* This info is not changed during file's life
- */
-struct crypt_inode_info {
-#if DEBUG_CRYPT
-	loc_t *loc; /* pathname that the file has been
-		       opened, or created with */
-#endif
-	uint16_t nr_minor;
-	uuid_t oid;
-	struct object_cipher_info cinfo;
-};
-
-/*
- * this should locate in secure memory
- */
-typedef struct {
-	struct master_cipher_info master;
-} crypt_private_t;
-
-static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
-{
-	return &priv->master;
-}
-
-static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
-							 *info)
-{
-	return &info->cinfo;
-}
-
-/*
- * this describes layouts and properties
- * of atoms in an aligned vector
- */
-struct avec_config {
-	uint32_t atom_size;
-	atom_data_type type;
-	size_t orig_size;
-	off_t orig_offset;
-	size_t expanded_size;
-	off_t aligned_offset;
-
-	uint32_t off_in_head;
-	uint32_t off_in_tail;
-	uint32_t gap_in_tail;
-	uint32_t nr_full_blocks;
-
-	struct iovec *avec; /* aligned vector */
-	uint32_t acount; /* number of avec components. The same
-			  * as number of occupied logical blocks */
-	char **pool;
-	uint32_t blocks_in_pool;
-	uint32_t cursor; /* makes sense only for ordered writes,
-			  * so there is no races on this counter.
-			  *
-			  * Cursor is per-config object, we don't
-			  * reset cursor for atoms of different
-			  * localities (head, tail, full)
-			  */
-};
-
-
-typedef struct {
-	glusterfs_fop_t fop; /* code of FOP this local info built for */
-	fd_t *fd;
-	inode_t *inode;
-	loc_t *loc;
-	int32_t mac_idx;
-	loc_t *newloc;
-	int32_t flags;
-	int32_t wbflags;
-	struct crypt_inode_info *info;
-	struct iobref *iobref;
-	struct iobref *iobref_data;
-	off_t offset;
-
-	uint64_t old_file_size; /* per FOP, retrieved under lock held */
-	uint64_t cur_file_size; /* per iteration, before issuing IOs */
-	uint64_t new_file_size; /* per iteration, after issuing IOs */
-
-	uint64_t io_offset; /* offset of IOs issued per iteration */
-	uint64_t io_offset_nopad; /* offset of user's data in the atom */
-	uint32_t io_size; /* size of IOs issued per iteration */
-	uint32_t io_size_nopad; /* size of user's data in the IOs */
-	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
-
-	gf_lock_t call_lock; /* protect nr_calls from many cbks */
-	int32_t nr_calls;
-
-	atom_data_type active_setup; /* which setup (hole or date)
-					is currently active */
-	/* data setup */
-	struct avec_config data_conf;
-
-	/* hole setup */
-	int hole_conv_in_proggress;
-	gf_lock_t hole_lock; /* protect hole config from many cbks */
-	int hole_handled;
-	struct avec_config hole_conf;
-	struct iatt buf;
-	struct iatt prebuf;
-	struct iatt postbuf;
-	struct iatt *prenewparent;
-	struct iatt *postnewparent;
-	int32_t op_ret;
-	int32_t op_errno;
-	int32_t rw_count; /* total read or written */
-	gf_lock_t rw_count_lock; /* protect the counter above */
-	unsigned char *format; /* for create, update format string */
-	uint32_t format_size;
-	uint32_t msgflags; /* messages for crypt_open() */
-	dict_t *xdata;
-	dict_t *xattr;
-	struct iovec vec; /* contains last file's atom for
-			     read-prune-write sequence */
-	gf_boolean_t custom_mtd;
-	/*
-	 * the next 3 fields are used by readdir and friends
-	 */
-	gf_dirent_t *de; /* directory entry */
-	char *de_path; /* pathname of directory entry */
-	uint32_t de_prefix_len; /* length of the parent's pathname */
-	gf_dirent_t *entries;
-
-	uint32_t update_disk_file_size:1;
-} crypt_local_t;
-
-/* This represents a (read)modify-write atom */
-struct rmw_atom {
-	atom_locality_type locality;
-	/*
-	 * read-modify-write sequence of the atom
-	 */
-	int32_t (*rmw)(call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iovec *vec,
-		       int32_t count,
-		       struct iatt *stbuf,
-		       struct iobref *iobref,
-		       dict_t *xdata);
-	/*
-	 * offset of the logical block in a file
-	 */
-	loff_t (*offset_at)(call_frame_t *frame,
-			    struct object_cipher_info *object);
-	/*
-	 * IO offset in an atom
-	 */
-	uint32_t (*offset_in)(call_frame_t *frame,
-			      struct object_cipher_info *object);
-	/*
-	 * number of bytes of plain text of this atom that user
-	 * wants to read/write.
-	 * It can be smaller than atom_size in the case of head
-	 * or tail atoms.
-	 */
-	uint32_t (*io_size_nopad)(call_frame_t *frame,
-				  struct object_cipher_info *object);
-	/*
-	 * which iovec represents the atom
-	 */
-	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
-	/*
-	 * how many bytes of partial block should be uptodated by
-	 * reading from disk.
-	 * This is used to perform a read component of RMW (read-modify-write).
-	 */
-	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
-	struct avec_config *(*get_config)(call_frame_t *frame);
-};
-
-struct data_cipher_alg {
-	gf_boolean_t atomic; /* true means that algorithm requires
-				to pad data before cipher transform */
-	gf_boolean_t should_pad; /* true means that algorithm requires
-				    to pad the end of file with extra-data */
-	uint32_t blkbits; /* blksize = 1 << blkbits */
-	/*
-	 * any preliminary sanity checks goes here
-	 */
-	int32_t (*init)(void);
-	/*
-	 * set alg-mode specific inode info
-	 */
-	int32_t (*set_private)(struct crypt_inode_info *info,
-			       struct master_cipher_info *master);
-	/*
-	 * check alg-mode specific data key
-	 */
-	int32_t (*check_key)(uint32_t key_size);
-	void (*set_iv)(off_t offset, struct object_cipher_info *object);
-	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
-			   size_t length, off_t offset, const int enc,
-			   struct object_cipher_info *object);
-};
-
-/*
- * version-dependent metadata loader
- */
-struct crypt_mtd_loader {
-	/*
-	 * return core format size
-	 */
-	size_t (*format_size)(mtd_op_t op, size_t old_size);
-	/*
-	 * pack version-specific metadata of an object
-	 * at ->create()
-	 */
-	int32_t (*create_format)(unsigned char *wire,
-				 loc_t *loc,
-				 struct crypt_inode_info *info,
-				 struct master_cipher_info *master);
-	/*
-	 * extract version-specific metadata of an object
-	 * at ->open() time
-	 */
-	int32_t (*open_format)(unsigned char *wire,
-			       int32_t len,
-			       loc_t *loc,
-			       struct crypt_inode_info *info,
-			       struct master_cipher_info *master,
-			       crypt_local_t *local,
-			       gf_boolean_t load_info);
-	int32_t (*update_format)(unsigned char *new,
-				 unsigned char *old,
-				 size_t old_len,
-				 int32_t mac_idx,
-				 mtd_op_t op,
-				 loc_t *loc,
-				 struct crypt_inode_info *info,
-				 struct master_cipher_info *master,
-				 crypt_local_t *local);
-};
-
-typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
-					   void *cookie,
-					   xlator_t *this,
-					   int32_t op_ret,
-					   int32_t op_errno,
-					   struct iatt *prebuf,
-					   struct iatt *postbuf,
-					   dict_t *xdata);
-typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
-typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
-
-
-/* Declarations */
-
-/* keys.c */
-extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
-int32_t get_nmtd_vol_key(struct master_cipher_info *master);
-int32_t get_nmtd_link_key(loc_t *loc,
-			  struct master_cipher_info *master,
-			  unsigned char *result);
-int32_t get_emtd_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  unsigned char *result);
-int32_t get_data_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  uint32_t keysize,
-			  unsigned char *key);
-/* data.c */
-extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
-void encrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off);
-void decrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off);
-int32_t align_iov_by_atoms(xlator_t *this,
-			   crypt_local_t *local,
-			   struct object_cipher_info *object,
-			   struct iovec *vec /* input vector */,
-			   int32_t count /* number of vec components */,
-			   struct iovec *avec /* aligned vector */,
-			   char **blocks /* pool of blocks */,
-			   uint32_t *blocks_allocated,
-			   struct avec_config *conf);
-int32_t set_config_avec_data(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     struct iovec *vec,
-			     int32_t vec_count);
-int32_t set_config_avec_hole(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     glusterfs_fop_t fop);
-void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
-		    struct avec_config *conf, atom_data_type dtype);
-void set_config_offsets(call_frame_t *frame,
-			xlator_t *this,
-			uint64_t offset,
-			uint64_t count,
-			atom_data_type dtype,
-			int32_t setup_gap_in_tail);
-
-/* metadata.c */
-extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
-
-int32_t alloc_format(crypt_local_t *local, size_t size);
-int32_t alloc_format_create(crypt_local_t *local);
-void free_format(crypt_local_t *local);
-size_t format_size(mtd_op_t op, size_t old_size);
-size_t new_format_size(void);
-int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
-		    struct crypt_inode_info *info,
-		    struct master_cipher_info *master, crypt_local_t *local,
-		    gf_boolean_t load_info);
-int32_t update_format(unsigned char *new, unsigned char *old,
-		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master,
-		      crypt_local_t *local);
-int32_t create_format(unsigned char *wire,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master);
-
-/* atom.c */
-struct rmw_atom *atom_by_types(atom_data_type data,
-			       atom_locality_type locality);
-void submit_partial(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    atom_locality_type ltype);
-void submit_full(call_frame_t *frame, xlator_t *this);
-
-/* crypt.c */
-
-end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
-static size_t iovec_get_size(struct iovec *vec, uint32_t count);
-void set_local_io_params_writev(call_frame_t *frame,
-				struct object_cipher_info *object,
-				struct rmw_atom *atom, off_t io_offset,
-				uint32_t io_size);
-void link_wind(call_frame_t *frame, xlator_t *this);
-void unlink_wind(call_frame_t *frame, xlator_t *this);
-void link_unwind(call_frame_t *frame);
-void unlink_unwind(call_frame_t *frame);
-void rename_wind(call_frame_t *frame, xlator_t *this);
-void rename_unwind(call_frame_t *frame);
-
-/* Inline functions */
-
-static inline size_t iovec_get_size(struct iovec *vec, uint32_t count)
-{
-	int i;
-	size_t size = 0;
-	for (i = 0; i < count; i++)
-		size += vec[i].iov_len;
-	return size;
-}
-
-static inline int32_t crypt_xlator_id(void)
-{
-	return CRYPT_XLATOR_ID;
-}
-
-static inline mtd_loader_id current_mtd_loader(void)
-{
-	return MTD_LOADER_V1;
-}
-
-static inline uint32_t master_key_size (void)
-{
-	return crypt_keys[MASTER_VOL_KEY].len >> 3;
-}
-
-static inline uint32_t nmtd_vol_key_size (void)
-{
-	return crypt_keys[NMTD_VOL_KEY].len >> 3;
-}
-
-static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
-					cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].blkbits;
-}
-
-static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
-					cipher_mode_t mode)
-{
-	return 1 << alg_mode_blkbits(alg, mode);
-}
-
-static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
-					   cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].atomic;
-}
-
-static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
-					       cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].should_pad;
-}
-
-static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
-{
-	return alg_mode_blksize(mr->m_alg, mr->m_mode);
-}
-
-static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
-{
-	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
-}
-
-static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
-{
-	return alg_mode_atomic(mr->m_alg, mr->m_mode);
-}
-
-static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
-{
-	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
-}
-
-static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
-{
-	return alg_mode_blksize(ob->o_alg, ob->o_mode);
-}
-
-static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
-{
-	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
-}
-
-static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
-{
-	return alg_mode_atomic(ob->o_alg, ob->o_mode);
-}
-
-static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
-{
-	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
-}
-
-static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
-{
-	return master->m_dkey_size >> 3;
-}
-
-static inline struct avec_config *get_hole_conf(call_frame_t *frame)
-{
-	return &(((crypt_local_t *)frame->local)->hole_conf);
-}
-
-static inline struct avec_config *get_data_conf(call_frame_t *frame)
-{
-	return &(((crypt_local_t *)frame->local)->data_conf);
-}
-
-static inline int32_t get_atom_bits (struct object_cipher_info *object)
-{
-	return object->o_block_bits;
-}
-
-static inline int32_t get_atom_size (struct object_cipher_info *object)
-{
-	return 1 << get_atom_bits(object);
-}
-
-static inline int32_t has_head_block(struct avec_config *conf)
-{
-	return conf->off_in_head ||
-		(conf->acount == 1 && conf->off_in_tail);
-}
-
-static inline int32_t has_tail_block(struct avec_config *conf)
-{
-	return conf->off_in_tail && conf->acount > 1;
-}
-
-static inline int32_t has_full_blocks(struct avec_config *conf)
-{
-	return conf->nr_full_blocks;
-}
-
-static inline int32_t should_submit_head_block(struct avec_config *conf)
-{
-	return has_head_block(conf) && (conf->cursor == 0);
-}
-
-static inline int32_t should_submit_tail_block(struct avec_config *conf)
-{
-	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
-}
-
-static inline int32_t should_submit_full_block(struct avec_config *conf)
-{
-	uint32_t start = has_head_block(conf) ? 1 : 0;
-
-	return has_full_blocks(conf) &&
-		conf->cursor >= start &&
-		conf->cursor < start + conf->nr_full_blocks;
-}
-
-#if DEBUG_CRYPT
-static inline void crypt_check_input_len(size_t len,
-					 struct object_cipher_info *object)
-{
-	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
-		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
-}
-
-static inline void check_head_block(struct avec_config *conf)
-{
-	if (!has_head_block(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
-}
-
-static inline void check_tail_block(struct avec_config *conf)
-{
-	if (!has_tail_block(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
-}
-
-static inline void check_full_block(struct avec_config *conf)
-{
-	if (!has_full_blocks(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
-}
-
-static inline void check_cursor_head(struct avec_config *conf)
-{
-	if (!has_head_block(conf))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Illegal call of head atom method");
-	else if (conf->cursor != 0)
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
-		       conf->cursor);
-}
-
-static inline void check_cursor_full(struct avec_config *conf)
-{
-	if (!has_full_blocks(conf))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Illegal call of full atom method");
-	if (has_head_block(conf) && (conf->cursor == 0))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Cursor is not at full atom");
-}
-
-/*
- * FIXME: use avec->iov_len to check setup
- */
-static inline int data_local_invariant(crypt_local_t *local)
-{
-	return 0;
-}
-
-#else
-#define crypt_check_input_len(len, object) noop
-#define check_head_block(conf) noop
-#define check_tail_block(conf) noop
-#define check_full_block(conf) noop
-#define check_cursor_head(conf) noop
-#define check_cursor_full(conf) noop
-
-#endif /* DEBUG_CRYPT */
-
-static inline struct avec_config *conf_by_type(call_frame_t *frame,
-					       atom_data_type dtype)
-{
-	struct avec_config *conf = NULL;
-
-	switch (dtype) {
-	case HOLE_ATOM:
-		conf = get_hole_conf(frame);
-		break;
-	case DATA_ATOM:
-		conf = get_data_conf(frame);
-		break;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
-	}
-	return conf;
-}
-
-static inline uint32_t nr_calls_head(struct avec_config *conf)
-{
-	return has_head_block(conf) ? 1 : 0;
-}
-
-static inline uint32_t nr_calls_tail(struct avec_config *conf)
-{
-	return has_tail_block(conf) ? 1 : 0;
-}
-
-static inline uint32_t nr_calls_full(struct avec_config *conf)
-{
-	switch(conf->type) {
-	case HOLE_ATOM:
-		return has_full_blocks(conf);
-	case DATA_ATOM:
-		return has_full_blocks(conf) ?
-			logical_blocks_occupied(0,
-						conf->nr_full_blocks,
-						MAX_IOVEC_BITS)	: 0;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
-		return 0;
-	}
-}
-
-static inline uint32_t nr_calls(struct avec_config *conf)
-{
-	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
-}
-
-static inline uint32_t nr_calls_data(call_frame_t *frame)
-{
-	return nr_calls(get_data_conf(frame));
-}
-
-static inline uint32_t nr_calls_hole(call_frame_t *frame)
-{
-	return nr_calls(get_hole_conf(frame));
-}
-
-static inline void get_one_call_nolock(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-
-	++local->nr_calls;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
-}
-
-static inline void get_one_call(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-
-	LOCK(&local->call_lock);
-	get_one_call_nolock(frame);
-	UNLOCK(&local->call_lock);
-}
-
-static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
-{
-	crypt_local_t *local = frame->local;
-
-	local->nr_calls += nr;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
-}
-
-static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
-{
-	crypt_local_t *local = frame->local;
-
-	LOCK(&local->call_lock);
-	get_nr_calls_nolock(frame, nr);
-	UNLOCK(&local->call_lock);
-}
-
-static inline int put_one_call(crypt_local_t *local)
-{
-	uint32_t last = 0;
-
-	LOCK(&local->call_lock);
-	if (--local->nr_calls == 0)
-		last = 1;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
-
-	UNLOCK(&local->call_lock);
-	return last;
-}
-
-static inline int is_appended_write(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->orig_offset + conf->orig_size > local->old_file_size;
-}
-
-static inline int is_ordered_mode(call_frame_t *frame)
-{
-#if 0
-	crypt_local_t *local = frame->local;
-	return local->fop == GF_FOP_FTRUNCATE ||
-		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
-#endif
-	return 1;
-}
-
-static inline int32_t hole_conv_completed(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-	return conf->cursor == conf->acount;
-}
-
-static inline int32_t data_write_in_progress(crypt_local_t *local)
-{
-	return local->active_setup == DATA_ATOM;
-}
-
-static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
-					     xlator_t *this)
-{
-	return frame->parent->this == this;
-}
-
-static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
-{
-	switch(fop){
-	case GF_FOP_LINK:
-		return link_wind;
-	case GF_FOP_UNLINK:
-		return unlink_wind;
-	case GF_FOP_RENAME:
-		return rename_wind;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
-		return NULL;
-	}
-}
-
-static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
-{
-	switch(fop){
-	case GF_FOP_LINK:
-		return link_unwind;
-	case GF_FOP_UNLINK:
-		return unlink_unwind;
-	case GF_FOP_RENAME:
-		return rename_unwind;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
-		return NULL;
-	}
-}
-
-static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
-{
-	switch (fop) {
-	case GF_FOP_LINK:
-		return MTD_APPEND;
-	case GF_FOP_UNLINK:
-		return MTD_CUT;
-	case GF_FOP_RENAME:
-		return MTD_OVERWRITE;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
-		return MTD_LAST_OP;
-	}
-}
-
-#endif /* __CRYPT_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
deleted file mode 100644
index 2f96ed2bab5..00000000000
--- a/xlators/encryption/crypt/src/data.c
+++ /dev/null
@@ -1,764 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
-{
-	unsigned char *ivec;
-
-	ivec = object->u.aes_xts.ivec;
-
-	/* convert the tweak into a little-endian byte
-	 * array (IEEE P1619/D16, May 2007, section 5.1)
-	 */
-
-	*((uint64_t *)ivec) = htole64(offset);
-
-	/* ivec is padded with zeroes */
-}
-
-static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
-				   AES_KEY *keys)
-{
-	int32_t ret;
-
-	ret = AES_set_encrypt_key(raw_key,
-				  key_size,
-				  &keys[AES_ENCRYPT]);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
-		return ret;
-	}
-	ret = AES_set_decrypt_key(raw_key,
-				  key_size,
-				  &keys[AES_DECRYPT]);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
-		return ret;
-	}
-	return 0;
-}
-
-/*
- * set private cipher info for xts mode
- */
-static int32_t set_private_aes_xts(struct crypt_inode_info *info,
-				   struct master_cipher_info *master)
-{
-	int ret;
-	struct object_cipher_info *object = get_object_cinfo(info);
-	unsigned char *data_key;
-	uint32_t subkey_size;
-
-	/* init tweak value */
-	memset(object->u.aes_xts.ivec, 0, 16);
-
-	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
-	if (!data_key)
-		return ENOMEM;
-
-	/*
-	 * retrieve data keying meterial
-	 */
-	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
-		GF_FREE(data_key);
-		return ret;
-	}
-	/*
-	 * parse compound xts key
-	 */
-	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
-	/*
-	 * install key for data encryption
-	 */
-	ret = aes_set_keys_common(data_key,
-				  subkey_size << 3, object->u.aes_xts.dkey);
-	if (ret) {
-		GF_FREE(data_key);
-		return ret;
-	}
-	/*
-	 * set up key used to encrypt tweaks
-	 */
-	ret = AES_set_encrypt_key(data_key + subkey_size,
-				  object->o_dkey_size / 2,
-				  &object->u.aes_xts.tkey);
-	if (ret < 0)
-		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
-
-	GF_FREE(data_key);
-	return ret;
-}
-
-static int32_t aes_xts_init(void)
-{
-	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
-	return 0;
-}
-
-static int32_t check_key_aes_xts(uint32_t keysize)
-{
-	switch(keysize) {
-	case 256:
-	case 512:
-		return 0;
-	default:
-		break;
-	}
-	return -1;
-}
-
-static int32_t encrypt_aes_xts(const unsigned char *from,
-			       unsigned char *to, size_t length,
-			       off_t offset, const int enc,
-			       struct object_cipher_info *object)
-{
-	XTS128_CONTEXT ctx;
-	if (enc) {
-		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
-		ctx.block1 = (block128_f)AES_encrypt;
-	}
-	else {
-		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
-		ctx.block1 = (block128_f)AES_decrypt;
-	}
-	ctx.key2 = &object->u.aes_xts.tkey;
-	ctx.block2 = (block128_f)AES_encrypt;
-
-	return CRYPTO_xts128_encrypt(&ctx,
-				     object->u.aes_xts.ivec,
-				     from,
-				     to,
-				     length, enc);
-}
-
-/*
- * Cipher input chunk @from of length @len;
- * @to: result of cipher transform;
- * @off: offset in a file (must be cblock-aligned);
- */
-static void cipher_data(struct object_cipher_info *object,
-			char *from,
-			char *to,
-			off_t off,
-			size_t len,
-			const int enc)
-{
-	crypt_check_input_len(len, object);
-
-#if TRIVIAL_TFM && DEBUG_CRYPT
-	return;
-#endif
-	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
-	data_cipher_algs[object->o_alg][object->o_mode].encrypt
-		((const unsigned char *)from,
-		 (unsigned char *)to,
-		 len,
-		 off,
-		 enc,
-		 object);
-}
-
-#define MAX_CIPHER_CHUNK (1 << 30)
-
-/*
- * Do cipher (encryption/decryption) transform of a
- * continuous region of memory.
- *
- * @len: a number of bytes to transform;
- * @buf: data to transform;
- * @off: offset in a file, should be block-aligned
- *       for atomic cipher modes and ksize-aligned
- *       for other modes).
- * @dir: direction of transform (encrypt/decrypt).
- */
-static void cipher_region(struct object_cipher_info *object,
-			  char *from,
-			  char *to,
-			  off_t off,
-			  size_t len,
-			  int dir)
-{
-	while (len > 0) {
-		size_t to_cipher;
-
-		to_cipher = len;
-		if (to_cipher > MAX_CIPHER_CHUNK)
-			to_cipher = MAX_CIPHER_CHUNK;
-
-		/* this will reset IV */
-		cipher_data(object,
-			    from,
-			    to,
-			    off,
-			    to_cipher,
-			    dir);
-		from += to_cipher;
-		to   += to_cipher;
-		off  += to_cipher;
-		len  -= to_cipher;
-	}
-}
-
-/*
- * Do cipher transform (encryption/decryption) of
- * plaintext/ciphertext represented by @vec.
- *
- * Pre-conditions: @vec represents a continuous piece
- * of data in a file at offset @off to be ciphered
- * (encrypted/decrypted).
- * @count is the number of vec's components. All the
- * components must be block-aligned, the caller is
- * responsible for this. @dir is "direction" of
- * transform (encrypt/decrypt).
- */
-static void cipher_aligned_iov(struct object_cipher_info *object,
-			       struct iovec *vec,
-			       int count,
-			       off_t off,
-			       int32_t dir)
-{
-	int i;
-	int len = 0;
-
-	for (i = 0; i < count; i++) {
-		cipher_region(object,
-			      vec[i].iov_base,
-			      vec[i].iov_base,
-			      off + len,
-			      vec[i].iov_len,
-			      dir);
-		len += vec[i].iov_len;
-	}
-}
-
-void encrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off)
-{
-	cipher_aligned_iov(object, vec, count, off, 1);
-}
-
-void decrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off)
-{
-	cipher_aligned_iov(object, vec, count, off, 0);
-}
-
-#if DEBUG_CRYPT
-static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
-{
-	int i;
-	int off = 0;
-	for (i = 0; i < count; i++) {
-		memcpy(buf + off,
-		       vec[i].iov_base + skip,
-		       vec[i].iov_len - skip);
-
-		off += (vec[i].iov_len - skip);
-		skip = 0;
-	}
-}
-
-static void check_iovecs(struct iovec *vec, int cnt,
-			 struct iovec *avec, int acnt, uint32_t off_in_head)
-{
-	char *s1, *s2;
-	uint32_t size, asize;
-
-	size = iovec_get_size(vec, cnt);
-	asize = iovec_get_size(avec, acnt) - off_in_head;
-	if (size != asize) {
-		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
-		       size, asize);
-		return;
-	}
-	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
-	if (!s1) {
-		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
-		return;
-	}
-	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
-	if (!s2) {
-		GF_FREE(s1);
-		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
-		return;
-	}
-	compound_stream(vec, cnt, s1, 0);
-	compound_stream(avec, acnt, s2, off_in_head);
-	if (memcmp(s1, s2, size))
-		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
-	GF_FREE(s1);
-	GF_FREE(s2);
-}
-
-#else
-#define check_iovecs(vec, count, avec, avecn, off) noop
-#endif /* DEBUG_CRYPT */
-
-static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
-			      int32_t block_size)
-{
-	struct iobuf *iobuf = NULL;
-
-	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
-	if (!iobuf) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Failed to get iobuf");
-		return NULL;
-	}
-	if (!local->iobref_data) {
-		local->iobref_data = iobref_new();
-		if (!local->iobref_data) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "Failed to get iobref");
-			iobuf_unref(iobuf);
-			return NULL;
-		}
-	}
-	iobref_add(local->iobref_data, iobuf);
-	return iobuf->ptr;
-}
-
-/*
- * Compound @avec, which represent the same data
- * chunk as @vec, but has aligned components of
- * specified block size. Alloc blocks, if needed.
- * In particular, incomplete head and tail blocks
- * must be allocated.
- * Put number of allocated blocks to @num_blocks.
- *
- * Example:
- *
- * input: data chunk represented by 4 components
- * [AB],[BC],[CD],[DE];
- * output: 5 logical blocks (0, 1, 2, 3, 4).
- *
- *   A     B       C     D               E
- *   *-----*+------*-+---*----+--------+-*
- *   |     ||      | |   |    |        | |
- * *-+-----+*------+-*---+----*--------*-+------*
- * 0        1        2        3        4
- *
- * 0 - incomplete compound (head);
- * 1, 2 - full compound;
- * 3 - full non-compound (the case of reuse);
- * 4 - incomplete non-compound (tail).
- */
-int32_t align_iov_by_atoms(xlator_t *this,
-			   crypt_local_t *local,
-			   struct object_cipher_info *object,
-			   struct iovec *vec /* input vector */,
-			   int32_t count /* number of vec components */,
-			   struct iovec *avec /* aligned vector */,
-			   char **blocks /* pool of blocks */,
-			   uint32_t *blocks_allocated,
-			   struct avec_config *conf)
-{
-	int vecn = 0;      /* number of the current component in vec */
-	int avecn = 0;     /* number of the current component in avec */
-	off_t vec_off = 0; /* offset in the current vec component,
-			    * i.e. the number of bytes have already
-			    * been copied */
-	int32_t block_size = get_atom_size(object);
-	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
-	int32_t off_in_head = conf->off_in_head;
-
-	to_process = iovec_get_size(vec, count);
-
-	while (to_process > 0) {
-		if (off_in_head ||
-		    vec[vecn].iov_len - vec_off < block_size) {
-			/*
-			 * less than block_size:
-			 * the case of incomplete (head or tail),
-			 * or compound block
-			 */
-			size_t copied = 0;
-			/*
-			 * populate the pool with a new block
-			 */
-			blocks[*blocks_allocated] = data_alloc_block(this,
-								    local,
-								    block_size);
-			if (!blocks[*blocks_allocated])
-				return -ENOMEM;
-			memset(blocks[*blocks_allocated], 0, off_in_head);
-			/*
-			 * fill the block with vec components
-			 */
-			do {
-				size_t to_copy;
-
-				to_copy = vec[vecn].iov_len - vec_off;
-				if (to_copy > block_size - off_in_head)
-					to_copy = block_size - off_in_head;
-
-				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
-				       vec[vecn].iov_base + vec_off,
-				       to_copy);
-
-				copied += to_copy;
-				to_process -= to_copy;
-
-				vec_off += to_copy;
-				if (vec_off == vec[vecn].iov_len) {
-					/* finished with this vecn */
-					vec_off = 0;
-					vecn++;
-				}
-			} while (copied < (block_size - off_in_head) && to_process > 0);
-			/*
-			 * update avec
-			 */
-			avec[avecn].iov_len = off_in_head + copied;
-			avec[avecn].iov_base = blocks[*blocks_allocated];
-
-			(*blocks_allocated)++;
-			off_in_head = 0;
-		} else {
-			/*
-			 * the rest of the current vec component
-			 * is not less than block_size, so reuse
-			 * the memory buffer of the component.
-			 */
-			size_t to_reuse;
-			to_reuse = (to_process > block_size ?
-				    block_size :
-				    to_process);
-			avec[avecn].iov_len = to_reuse;
-			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
-
-			vec_off += to_reuse;
-			if (vec_off == vec[vecn].iov_len) {
-				/* finished with this vecn */
-				vec_off = 0;
-				vecn++;
-			}
-			to_process -= to_reuse;
-		}
-		avecn++;
-	}
-	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
-	return 0;
-}
-
-/*
- * allocate and setup aligned vector for data submission
- * Pre-condition: @conf is set.
- */
-int32_t set_config_avec_data(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     struct iovec *vec,
-			     int32_t vec_count)
-{
-	int32_t ret = ENOMEM;
-	struct iovec *avec;
-	char **pool;
-	uint32_t blocks_in_pool = 0;
-
-	conf->type = DATA_ATOM;
-
-	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
-	if (!avec)
-		return ret;
-	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
-	if (!pool) {
-		GF_FREE(avec);
-		return ret;
-	}
-	if (!vec) {
-		/*
-		 * degenerated case: no data
-		 */
-		pool[0] = data_alloc_block(this, local, get_atom_size(object));
-		if (!pool[0])
-			goto free;
-		blocks_in_pool = 1;
-		avec->iov_base = pool[0];
-		avec->iov_len = conf->off_in_tail;
-	}
-	else {
-		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
-					 avec, pool, &blocks_in_pool, conf);
-		if (ret)
-			goto free;
-	}
-	conf->avec = avec;
-	conf->pool = pool;
-	conf->blocks_in_pool = blocks_in_pool;
-	return 0;
- free:
-	GF_FREE(avec);
-	GF_FREE(pool);
-	return ret;
-}
-
-/*
- * allocate and setup aligned vector for hole submission
- */
-int32_t set_config_avec_hole(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     glusterfs_fop_t fop)
-{
-	uint32_t i, idx;
-	struct iovec *avec;
-	char **pool;
-	uint32_t num_blocks;
-	uint32_t blocks_in_pool = 0;
-
-	conf->type = HOLE_ATOM;
-
-	num_blocks = conf->acount -
-		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
-
-	switch (fop) {
-	case GF_FOP_WRITE:
-		/*
-		 * hole goes before data
-		 */
-		if (num_blocks == 1 && conf->off_in_tail != 0)
-		/*
-		 * we won't submit a hole which fits into
-		 * a data atom: this part of hole will be
-		 * submitted with data write
-		 */
-			return 0;
-		break;
-	case GF_FOP_FTRUNCATE:
-		/*
-		 * expanding truncate, hole goes after data,
-		 * and will be submited in any case.
- 		 */
-		break;
-	default:
-		gf_log("crypt", GF_LOG_WARNING,
-		       "bad file operation %d", fop);
-		return 0;
-	}
-	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
-	if (!avec)
-		return ENOMEM;
-	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
-	if (!pool) {
-		GF_FREE(avec);
-		return ENOMEM;
-	}
-	for (i = 0; i < num_blocks; i++) {
-		pool[i] = data_alloc_block(this, local, get_atom_size(object));
-		if (pool[i] == NULL)
-			goto free;
-		blocks_in_pool++;
-	}
-	if (has_head_block(conf)) {
-		/* set head block */
-		idx = 0;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		memset(avec[idx].iov_base + conf->off_in_head,
-		       0,
-		       get_atom_size(object) - conf->off_in_head);
-	}
-	if (has_tail_block(conf)) {
-		/* set tail block */
-		idx = num_blocks - 1;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		memset(avec[idx].iov_base, 0, conf->off_in_tail);
-	}
-	if (has_full_blocks(conf)) {
-		/* set full block */
-		idx = conf->off_in_head ? 1 : 0;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		/*
-		 * since we re-use the buffer,
-		 * zeroes will be set every time
-		 * before encryption, see submit_full()
-		 */
-	}
-	conf->avec = avec;
-	conf->pool = pool;
-	conf->blocks_in_pool = blocks_in_pool;
-	return 0;
- free:
-	GF_FREE(avec);
-	GF_FREE(pool);
-	return ENOMEM;
-}
-
-/* A helper for setting up config of partial atoms (which
- * participate in read-modify-write sequence).
- *
- * Calculate and setup precise amount of "extra-bytes"
- * that should be uptodated at the end of partial (not
- * necessarily tail!) block.
- *
- * Pre-condition: local->old_file_size is valid!
- * @conf contains setup, which is enough for correct calculation
- * of has_tail_block(), ->get_offset().
- */
-void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
-		    struct avec_config *conf, atom_data_type dtype)
-{
-	uint32_t to_block;
-	crypt_local_t *local = frame->local;
-	uint64_t old_file_size = local->old_file_size;
-	struct rmw_atom *partial = atom_by_types(dtype,
-						 has_tail_block(conf) ?
-						 TAIL_ATOM : HEAD_ATOM);
-
-	if (old_file_size <= partial->offset_at(frame, object))
-		to_block = 0;
-	else {
-		to_block = old_file_size - partial->offset_at(frame, object);
-		if (to_block > get_atom_size(object))
-			to_block = get_atom_size(object);
-	}
-	if (to_block > conf->off_in_tail)
-		conf->gap_in_tail = to_block - conf->off_in_tail;
-	else
-		/*
-		 * nothing to uptodate
-		 */
-		conf->gap_in_tail = 0;
-}
-
-/*
- * fill struct avec_config with offsets layouts
- */
-void set_config_offsets(call_frame_t *frame,
-			xlator_t *this,
-			uint64_t offset,
-			uint64_t count,
-			atom_data_type dtype,
-			int32_t set_gap)
-{
-	crypt_local_t *local;
-	struct object_cipher_info *object;
-	struct avec_config *conf;
-	uint32_t resid;
-
-	uint32_t atom_size;
-	uint32_t atom_bits;
-
-	size_t orig_size;
-	off_t orig_offset;
-	size_t expanded_size;
-	off_t aligned_offset;
-
-	uint32_t off_in_head = 0;
-	uint32_t off_in_tail = 0;
-	uint32_t nr_full_blocks;
-	int32_t size_full_blocks;
-
-	uint32_t acount; /* number of alifned components to write.
-			  * The same as number of occupied logical
-			  * blocks (atoms)
-			  */
-	local = frame->local;
-	object = &local->info->cinfo;
-	conf = (dtype == DATA_ATOM ?
-		get_data_conf(frame) : get_hole_conf(frame));
-
-	orig_offset = offset;
-	orig_size = count;
-
-	atom_size = get_atom_size(object);
-	atom_bits = get_atom_bits(object);
-
-	/*
-	 * Round-down the start,
-	 * round-up the end.
-	 */
-	resid = offset & (uint64_t)(atom_size - 1);
-
-	if (resid)
-		off_in_head = resid;
-	aligned_offset = offset - off_in_head;
-	expanded_size = orig_size + off_in_head;
-
-	/* calculate tail,
-	   expand size forward  */
-	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
-
-	if (resid) {
-		off_in_tail = resid;
-		expanded_size += (atom_size - off_in_tail);
-	}
-	/*
-	 * calculate number of occupied blocks
-	 */
-	acount = expanded_size >> atom_bits;
-	/*
-	 * calculate number of full blocks
-	 */
-	size_full_blocks = expanded_size;
-	if (off_in_head)
-		size_full_blocks -= atom_size;
-	if (off_in_tail && size_full_blocks > 0)
-		size_full_blocks -= atom_size;
-	nr_full_blocks = size_full_blocks >> atom_bits;
-
-	conf->atom_size = atom_size;
-	conf->orig_size = orig_size;
-	conf->orig_offset = orig_offset;
-	conf->expanded_size = expanded_size;
-	conf->aligned_offset = aligned_offset;
-
-	conf->off_in_head = off_in_head;
-	conf->off_in_tail = off_in_tail;
-	conf->nr_full_blocks = nr_full_blocks;
-	conf->acount = acount;
-	/*
-	 * Finally, calculate precise amount of
-	 * "extra-bytes" that should be uptodated
-	 * at the end.
-	 * Only if RMW is expected.
-	 */
-	if (off_in_tail && set_gap)
-		set_gap_at_end(frame, object, conf, dtype);
-}
-
-struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
-	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
-	{ .atomic = _gf_true,
-	  .should_pad = _gf_true,
-	  .blkbits = AES_BLOCK_BITS,
-	  .init = aes_xts_init,
-	  .set_private = set_private_aes_xts,
-	  .check_key = check_key_aes_xts,
-	  .set_iv = set_iv_aes_xts,
-	  .encrypt = encrypt_aes_xts
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
deleted file mode 100644
index 0b243d3e827..00000000000
--- a/xlators/encryption/crypt/src/keys.c
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-/* Key hierarchy
-
-                   +----------------+
-                   | MASTER_VOL_KEY |
-                   +-------+--------+
-                           |
-                           |
-          +----------------+----------------+
-          |                |                |
-          |                |                |
-  +-------+------+ +-------+-------+ +------+--------+
-  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
-  +-------+------+ +---------------+ +---------------+
-          |
-          |
-  +-------+-------+
-  | NMTD_LINK_KEY |
-  +---------------+
-
- */
-
-#if DEBUG_CRYPT
-static void check_prf_iters(uint32_t num_iters)
-{
-	if (num_iters == 0)
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad number of prf iterations : %d", num_iters);
-}
-#else
-#define check_prf_iters(num_iters) noop
-#endif /* DEBUG_CRYPT */
-
-unsigned char crypt_fake_oid[16] =
-	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/*
- * derive key in the counter mode using
- * sha256-based HMAC as PRF, see
- * NIST Special Publication 800-108, 5.1)
- */
-
-#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
-
-static int32_t kderive_init(struct kderive_context *ctx,
-			    const unsigned char *pkey, /* parent key */
-			    uint32_t pkey_size,  /* parent key size */
-			    const unsigned char *idctx, /* id-context */
-			    uint32_t idctx_size,
-			    crypt_key_type type  /* type of child key */)
-{
-	unsigned char *pos;
-	uint32_t llen = strlen(crypt_keys[type].label);
-	/*
-	 * Compoud the fixed input data for KDF:
-	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
-	 * NIST SP 800-108, 5.1
-	 */
-	ctx->fid_len =
-		sizeof(uint32_t) +
-		llen +
-		1 +
-		idctx_size +
-		sizeof(uint32_t);
-
-	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
-	if (!ctx->fid)
-		return ENOMEM;
-	ctx->out_len = round_up(crypt_keys[type].len >> 3,
-				PRF_OUTPUT_SIZE);
-	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
-	if (!ctx->out) {
-		GF_FREE(ctx->fid);
-		return ENOMEM;
-	}
-	ctx->pkey = pkey;
-	ctx->pkey_len = pkey_size;
-	ctx->ckey_len = crypt_keys[type].len;
-
-	pos = ctx->fid;
-
-	/* counter will be set up in kderive_rfn() */
-	pos += sizeof(uint32_t);
-
-	memcpy(pos, crypt_keys[type].label, llen);
-	pos += llen;
-
-	/* set up zero octet */
-	*pos = 0;
-	pos += 1;
-
-	memcpy(pos, idctx, idctx_size);
-	pos += idctx_size;
-
-	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
-
-	return 0;
-}
-
-static void kderive_update(struct kderive_context *ctx)
-{
-	uint32_t i;
-	HMAC_CTX hctx;
-	unsigned char *pos = ctx->out;
-	uint32_t *p_iter = (uint32_t *)ctx->fid;
-	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
-
-	check_prf_iters(num_iters);
-
-	HMAC_CTX_init(&hctx);
-	for (i = 0; i < num_iters; i++) {
-		/*
-		 * update the iteration number in the fid
-		 */
-		*p_iter = htobe32(i);
-		HMAC_Init_ex(&hctx,
-			     ctx->pkey, ctx->pkey_len >> 3,
-			     EVP_sha256(),
-			     NULL);
-		HMAC_Update(&hctx, ctx->fid, ctx->fid_len);
-		HMAC_Final(&hctx, pos, NULL);
-
-		pos += PRF_OUTPUT_SIZE;
-	}
-	HMAC_CTX_cleanup(&hctx);
-}
-
-static void kderive_final(struct kderive_context *ctx, unsigned char *child)
-{
-	memcpy(child, ctx->out, ctx->ckey_len >> 3);
-	GF_FREE(ctx->fid);
-	GF_FREE(ctx->out);
-	memset(ctx, 0, sizeof(*ctx));
-}
-
-/*
- * derive per-volume key for object ids aithentication
- */
-int32_t get_nmtd_vol_key(struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, master->m_nmtd_key);
-	return 0;
-}
-
-/*
- * derive per-link key for aithentication of non-encrypted
- * meta-data (nmtd)
- */
-int32_t get_nmtd_link_key(loc_t *loc,
-			  struct master_cipher_info *master,
-			  unsigned char *result)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_nmtd_key,
-			   nmtd_vol_key_size(),
-			   (const unsigned char *)loc->path,
-			   strlen(loc->path), NMTD_LINK_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, result);
-	return 0;
-}
-
-/*
- * derive per-file key for encryption and authentication
- * of encrypted part of metadata (emtd)
- */
-int32_t get_emtd_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  unsigned char *result)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, result);
-	return 0;
-}
-
-static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
-{
-	int32_t ret = 0;
-	switch (keysize) {
-	case 256:
-		*type = DATA_FILE_KEY_256;
-		break;
-	case 512:
-		*type = DATA_FILE_KEY_512;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
-		       keysize);
-		ret = ENOTSUP;
-		break;
-	}
-	return ret;
-}
-
-/*
- * derive per-file key for data encryption
- */
-int32_t get_data_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  uint32_t keysize,
-			  unsigned char *key)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-	crypt_key_type type;
-
-	ret = data_key_type_by_size(keysize, &type);
-	if (ret)
-		return ret;
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   info->oid, sizeof(uuid_t), type);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, key);
-	return 0;
-}
-
-/*
- * NOTE: Don't change existing keys: it will break compatibility;
- */
-struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
-	[MASTER_VOL_KEY] =
-	{ .len = MASTER_VOL_KEY_SIZE << 3,
-	  .label = "volume-master",
-	},
-	[NMTD_VOL_KEY] =
-	{ .len = NMTD_VOL_KEY_SIZE << 3,
-	  .label = "volume-nmtd-key-generation"
-	},
-	[NMTD_LINK_KEY] =
-	{ .len = 128,
-	  .label = "link-nmtd-authentication"
-	},
-	[EMTD_FILE_KEY] =
-	{ .len = 128,
-	  .label = "file-emtd-encryption-and-auth"
-	},
-	[DATA_FILE_KEY_256] =
-	{ .len = 256,
-	  .label = "file-data-encryption-256"
-	},
-	[DATA_FILE_KEY_512] =
-	{ .len = 512,
-	  .label = "file-data-encryption-512"
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
deleted file mode 100644
index 1364f825a98..00000000000
--- a/xlators/encryption/crypt/src/metadata.c
+++ /dev/null
@@ -1,614 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-#include "metadata.h"
-
-int32_t alloc_format(crypt_local_t *local, size_t size)
-{
-	if (size > 0) {
-		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
-		if (!local->format)
-			return ENOMEM;
-	}
-	local->format_size = size;
-	return 0;
-}
-
-int32_t alloc_format_create(crypt_local_t *local)
-{
-	return alloc_format(local, new_format_size());
-}
-
-void free_format(crypt_local_t *local)
-{
-	GF_FREE(local->format);
-}
-
-/*
- * Check compatibility with extracted metadata
- */
-static int32_t check_file_metadata(struct crypt_inode_info *info)
-{
-	struct object_cipher_info *object = &info->cinfo;
-
-	if (info->nr_minor != CRYPT_XLATOR_ID) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported minor subversion %d", info->nr_minor);
-		return EINVAL;
-	}
-	if (object->o_alg > LAST_CIPHER_ALG) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported cipher algorithm %d",
-		       object->o_alg);
-		return EINVAL;
-	}
-	if (object->o_mode > LAST_CIPHER_MODE) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported cipher mode %d",
-		       object->o_mode);
-		return EINVAL;
-	}
-	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
-	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
-		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
-		       object->o_block_bits);
-		return EINVAL;
-	}
-	/* TBD: check data key size */
-	return 0;
-}
-
-static size_t format_size_v1(mtd_op_t op, size_t old_size)
-{
-
-	switch (op) {
-	case MTD_CREATE:
-		return sizeof(struct mtd_format_v1);
-	case MTD_OVERWRITE:
-		return old_size;
-	case MTD_APPEND:
-		return old_size + NMTD_8_MAC_SIZE;
-	case MTD_CUT:
-		if (old_size > sizeof(struct mtd_format_v1))
-			return old_size - NMTD_8_MAC_SIZE;
-		else
-			return 0;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
-		return 0;
-	}
-}
-
-/*
- * Calculate size of the updated format string.
- * Returned zero means that we don't need to update the format string.
- */
-size_t format_size(mtd_op_t op, size_t old_size)
-{
-	size_t versioned;
-
-	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
-				 old_size - sizeof(struct crypt_format));
-	if (versioned != 0)
-		return versioned + sizeof(struct crypt_format);
-	return 0;
-}
-
-/*
- * size of the format string of newly created file (nr_links = 1)
- */
-size_t new_format_size(void)
-{
-	return format_size(MTD_CREATE, 0);
-}
-
-/*
- * Calculate per-link MAC by pathname
- */
-static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
-				loc_t *loc,
-				unsigned char *result,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char nmtd_link_key[16];
-	CMAC_CTX *cctx;
-	size_t len;
-
-	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
-		return -1;
-	}
-	cctx = CMAC_CTX_new();
-	if (!cctx) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
-		return -1;
-	}
-	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
-			EVP_aes_128_cbc(), 0);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
-		CMAC_CTX_free(cctx);
-		return -1;
-	}
-	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
-		CMAC_CTX_free(cctx);
-		return -1;
-	}
-	ret = CMAC_Final(cctx, result, &len);
-	CMAC_CTX_free(cctx);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
-		return -1;
-	}
-	return 0;
-}
-
-/*
- * Create per-link MAC of index @idx by pathname
- */
-static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t idx,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char *mac;
-	unsigned char cmac[16];
-
-	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
-
-	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
-	if (ret)
-		return -1;
-	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
-	return 0;
-}
-
-static int32_t create_format_v1(unsigned char *wire,
-				loc_t *loc,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct mtd_format_v1 *fmt;
-	unsigned char mtd_key[16];
-	AES_KEY EMTD_KEY;
-	unsigned char nmtd_link_key[16];
-	uint32_t ad;
-	GCM128_CONTEXT *gctx;
-
-	fmt = (struct mtd_format_v1 *)wire;
-
-	fmt->minor_id = info->nr_minor;
-	fmt->alg_id = AES_CIPHER_ALG;
-	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
-	fmt->block_bits = master->m_block_bits;
-	fmt->mode_id = master->m_mode;
-	/*
-	 * retrieve keys for the parts of metadata
-	 */
-	ret = get_emtd_file_key(info, master, mtd_key);
-	if (ret)
-		return ret;
-	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
-	if (ret)
-		return ret;
-
-	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
-
-	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
-
-	/* TBD: Check return values */
-
-	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
-
-	ad = htole32(MTD_LOADER_V1);
-	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
-		CRYPTO_gcm128_release(gctx);
-		return ret;
-	}
-	ret = CRYPTO_gcm128_encrypt(gctx,
-				    get_EMTD_V1(fmt),
-				    get_EMTD_V1(fmt),
-				    SIZE_OF_EMTD_V1);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
-		CRYPTO_gcm128_release(gctx);
-		return ret;
-	}
-	/*
-	 * set MAC of encrypted part of metadata
-	 */
-	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
-	CRYPTO_gcm128_release(gctx);
-	/*
-	 * set the first MAC of non-encrypted part of metadata
-	 */
-	return create_link_mac_v1(fmt, 0, loc, info, master);
-}
-
-/*
- * Called by fops:
- * ->create();
- * ->link();
- *
- * Pack common and version-specific parts of file's metadata
- * Pre-conditions: @info contains valid object-id.
- */
-int32_t create_format(unsigned char *wire,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master)
-{
-	struct crypt_format *fmt = (struct crypt_format *)wire;
-
-	fmt->loader_id = current_mtd_loader();
-
-	wire += sizeof(struct crypt_format);
-	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
-							       info, master);
-}
-
-/*
- * Append or overwrite per-link mac of @mac_idx index
- * in accordance with the new pathname
- */
-int32_t appov_link_mac_v1(unsigned char *new,
-			  unsigned char *old,
-			  uint32_t old_size,
-			  int32_t mac_idx,
-			  loc_t *loc,
-			  struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  crypt_local_t *local)
-{
-	memcpy(new, old, old_size);
-	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
-				  loc, info, master);
-}
-
-/*
- * Cut per-link mac of @mac_idx index
- */
-static int32_t cut_link_mac_v1(unsigned char *new,
-			       unsigned char *old,
-			       uint32_t old_size,
-			       int32_t mac_idx,
-			       loc_t *loc,
-			       struct crypt_inode_info *info,
-			       struct master_cipher_info *master,
-			       crypt_local_t *local)
-{
-	memcpy(new,
-	       old,
-	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
-
-	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
-	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
-	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
-	return 0;
-}
-
-int32_t update_format_v1(unsigned char *new,
-			 unsigned char *old,
-				size_t old_len,
-				int32_t mac_idx, /* of old name */
-				mtd_op_t op,
-				loc_t *loc,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master,
-				crypt_local_t *local)
-{
-	switch (op) {
-	case MTD_APPEND:
-		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
-	case MTD_OVERWRITE:
-		return appov_link_mac_v1(new, old, old_len, mac_idx,
-					 loc, info, master, local);
-	case MTD_CUT:
-		return cut_link_mac_v1(new, old, old_len, mac_idx,
-				       loc, info, master, local);
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
-		return -1;
-	}
-}
-
-/*
- * Called by fops:
- *
- * ->link()
- * ->unlink()
- * ->rename()
- *
- */
-int32_t update_format(unsigned char *new,
-		      unsigned char *old,
-		      size_t old_len,
-		      int32_t mac_idx,
-		      mtd_op_t op,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master,
-		      crypt_local_t *local)
-{
-	if (!new)
-		return 0;
-	memcpy(new, old, sizeof(struct crypt_format));
-
-	old += sizeof(struct crypt_format);
-	new += sizeof(struct crypt_format);
-	old_len -= sizeof(struct crypt_format);
-
-	return mtd_loaders[current_mtd_loader()].update_format(new, old,
-							       old_len,
-							       mac_idx, op,
-							       loc, info,
-							       master, local);
-}
-
-/*
- * Perform preliminary checks of found metadata
- * Return < 0 on errors;
- * Return number of object-id MACs (>= 1) on success
- */
-int32_t check_format_v1(uint32_t len, unsigned char *wire)
-{
-	uint32_t nr_links;
-
-	if (len < sizeof(struct mtd_format_v1)) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "v1-loader: bad metadata size %d", len);
-		goto error;
-	}
-	len -= sizeof(struct mtd_format_v1);
-	if (len % sizeof(nmtd_8_mac_t)) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "v1-loader: bad metadata format");
-		goto error;
-	}
-	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
-	if (nr_links > _POSIX_LINK_MAX)
-		goto error;
-	return nr_links;
- error:
-	return EIO;
-}
-
-/*
- * Verify per-link MAC specified by index @idx
- *
- * return:
- * -1 on errors;
- *  0 on failed verification;
- *  1 on successful verification
- */
-static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t idx /* index of the mac to verify */,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char *mac;
-	unsigned char cmac[16];
-
-	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
-
-	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
-	if (ret)
-		return -1;
-	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
-		return 0;
-	return 1;
-}
-
-/*
- * Lookup per-link MAC by pathname.
- *
- * return index of the MAC, if it was found;
- * return < 0 on errors, or if the MAC wasn't found
- */
-static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t nr_macs,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	uint32_t idx;
-
-	for (idx = 0; idx < nr_macs; idx++) {
-		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
-		if (ret < 0)
-			return ret;
-		if (ret > 0)
-			return idx;
-	}
-	return -ENOENT;
-}
-
-/*
- * Extract version-specific part of metadata
- */
-static int32_t open_format_v1(unsigned char *wire,
-			      int32_t len,
-			      loc_t *loc,
-			      struct crypt_inode_info *info,
-			      struct master_cipher_info *master,
-			      crypt_local_t *local,
-			      gf_boolean_t load_info)
-{
-	int32_t ret;
-	int32_t num_nmtd_macs;
-	struct mtd_format_v1 *fmt;
-	unsigned char mtd_key[16];
-	AES_KEY EMTD_KEY;
-	GCM128_CONTEXT *gctx;
-	uint32_t ad;
-	emtd_8_mac_t gmac;
-	struct object_cipher_info *object;
-
-	num_nmtd_macs = check_format_v1(len, wire);
-	if (num_nmtd_macs <= 0)
-		return EIO;
-
-	ret = lookup_link_mac_v1((struct mtd_format_v1 *)wire,
-				 num_nmtd_macs, loc, info, master);
-	if (ret < 0) {
-		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
-		return EINVAL;
-	}
-
-	local->mac_idx = ret;
-	if (load_info == _gf_false)
-		/* the case of partial open */
-		return 0;
-
-	fmt = GF_CALLOC(1, len, gf_crypt_mt_mtd);
-	if (!fmt)
-		return ENOMEM;
-	memcpy(fmt, wire, len);
-
-	object = &info->cinfo;
-
-	ret = get_emtd_file_key(info, master, mtd_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
-		goto out;
-	}
-	/*
-	 * decrypt encrypted meta-data
-	 */
-	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
-	if (ret < 0) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
-		ret = EIO;
-		goto out;
-	}
-	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
-	if (!gctx) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
-		ret = ENOMEM;
-		goto out;
-	}
-	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
-
-	ad = htole32(MTD_LOADER_V1);
-	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
-		CRYPTO_gcm128_release(gctx);
-		ret = EIO;
-		goto out;
-	}
-	ret = CRYPTO_gcm128_decrypt(gctx,
-				    get_EMTD_V1(fmt),
-				    get_EMTD_V1(fmt),
-				    SIZE_OF_EMTD_V1);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
-		CRYPTO_gcm128_release(gctx);
-		ret = EIO;
-		goto out;
-	}
-	/*
-	 * verify metadata
-	 */
-	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
-	CRYPTO_gcm128_release(gctx);
-	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
-		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
-		ret = EINVAL;
-		goto out;
-	}
-	/*
-	 * load verified metadata to the private part of inode
-	 */
-	info->nr_minor = fmt->minor_id;
-
-	object->o_alg = fmt->alg_id;
-	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
-	object->o_block_bits = fmt->block_bits;
-	object->o_mode = fmt->mode_id;
-
-	ret = check_file_metadata(info);
- out:
-	GF_FREE(fmt);
-	return ret;
-}
-
-/*
- * perform metadata authentication against @loc->path;
- * extract crypt-specific attribtes and populate @info
- * with them (optional)
- */
-int32_t open_format(unsigned char *str,
-		    int32_t len,
-		    loc_t *loc,
-		    struct crypt_inode_info *info,
-		    struct master_cipher_info *master,
-		    crypt_local_t *local,
-		    gf_boolean_t load_info)
-{
-	struct crypt_format *fmt;
-	if (len < sizeof(*fmt)) {
-		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
-		return EIO;
-	}
-	fmt = (struct crypt_format *)str;
-
-	if (fmt->loader_id >= LAST_MTD_LOADER) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Unsupported loader id %d", fmt->loader_id);
-		return EINVAL;
-	}
-	str += sizeof(*fmt);
-	len -= sizeof(*fmt);
-
-	return mtd_loaders[fmt->loader_id].open_format(str,
-						       len,
-						       loc,
-						       info,
-						       master,
-						       local,
-						       load_info);
-}
-
-struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
-	[MTD_LOADER_V1] =
-	{.format_size = format_size_v1,
-	 .create_format = create_format_v1,
-	 .open_format = open_format_v1,
-	 .update_format = update_format_v1
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
deleted file mode 100644
index b67ae25b58c..00000000000
--- a/xlators/encryption/crypt/src/metadata.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __METADATA_H__
-#define __METADATA_H__
-
-#define NMTD_8_MAC_SIZE   (8)
-#define EMTD_8_MAC_SIZE   (8)
-
-typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
-typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
-
-/*
- * Version "v1" of file's metadata.
- * Metadata of this version has 4 components:
- *
- * 1) EMTD (Encrypted part of MeTaData);
- * 2) NMTD (Non-encrypted part of MeTaData);
- * 3) EMTD_MAC; (EMTD Message Authentication Code);
- * 4) Array of per-link NMTD MACs (for every (hard)link it includes
- *    exactly one MAC)
- */
-struct mtd_format_v1 {
-	/* EMTD, encrypted part of meta-data */
-	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
-        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
-	uint8_t            block_bits; /* encoded block size */
-	uint8_t            minor_id; /* client translator id */
-	uint8_t            dkey_factor; /* encoded size of the data key */
-	/* MACs */
-	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
-	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
-				  * meta-data: at least one such MAC is always
-				  * present */
-} __attribute__((packed));
-
-/*
- * NMTD, the non-encrypted part of metadata of version "v1"
- * is file's gfid, which is generated on trusted machines.
- */
-#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
-#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
-			 offsetof(struct mtd_format_v1, alg_id))
-#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
-#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
-
-static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
-{
-        return &format->alg_id;
-}
-
-static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
-{
-        return info->oid;
-}
-
-static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
-{
-        return format->gmac;
-}
-
-static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
-{
-        return format->omac;
-}
-
-#endif /* __METADATA_H__ */
diff --git a/xlators/encryption/rot-13/src/rot-13.c b/xlators/encryption/rot-13/src/rot-13.c
deleted file mode 100644
index 6ec1b47c87b..00000000000
--- a/xlators/encryption/rot-13/src/rot-13.c
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-
-#include "rot-13.h"
-
-/*
- * This is a rot13 ``encryption'' xlator. It rot13's data when
- * writing to disk and rot13's it back when reading it.
- * This xlator is meant as an example, NOT FOR PRODUCTION
- * USE ;) (hence no error-checking)
- */
-
-void
-rot13 (char *buf, int len)
-{
-	int i;
-	for (i = 0; i < len; i++) {
-		if (buf[i] >= 'a' && buf[i] <= 'z')
-			buf[i] = 'a' + ((buf[i] - 'a' + 13) % 26);
-		else if (buf[i] >= 'A' && buf[i] <= 'Z')
-			buf[i] = 'A' + ((buf[i] - 'A' + 13) % 26);
-	}
-}
-
-void
-rot13_iovec (struct iovec *vector, int count)
-{
-	int i;
-	for (i = 0; i < count; i++) {
-		rot13 (vector[i].iov_base, vector[i].iov_len);
-	}
-}
-
-int32_t
-rot13_readv_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct iovec *vector,
-                 int32_t count,
-		 struct iatt *stbuf,
-                 struct iobref *iobref, dict_t *xdata)
-{
-	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-
-	if (priv->decrypt_read)
-		rot13_iovec (vector, count);
-
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref, xdata);
-	return 0;
-}
-
-int32_t
-rot13_readv (call_frame_t *frame,
-             xlator_t *this,
-             fd_t *fd,
-             size_t size,
-             off_t offset, uint32_t flags, dict_t *xdata)
-{
-	STACK_WIND (frame,
-		    rot13_readv_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset, flags, xdata);
-	return 0;
-}
-
-int32_t
-rot13_writev_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  struct iatt *prebuf,
-		  struct iatt *postbuf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-	return 0;
-}
-
-int32_t
-rot13_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count,
-              off_t offset, uint32_t flags,
-              struct iobref *iobref, dict_t *xdata)
-{
-	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-	if (priv->encrypt_write)
-		rot13_iovec (vector, count);
-
-	STACK_WIND (frame,
-		    rot13_writev_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset, flags,
-                    iobref, xdata);
-	return 0;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	data_t *data = NULL;
-	rot_13_private_t *priv = NULL;
-
-	if (!this->children || this->children->next) {
-		gf_log ("rot13", GF_LOG_ERROR,
-			"FATAL: rot13 should have exactly one child");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	priv = GF_CALLOC (sizeof (rot_13_private_t), 1, 0);
-        if (!priv)
-                return -1;
-
-	priv->decrypt_read = 1;
-	priv->encrypt_write = 1;
-
-	data = dict_get (this->options, "encrypt-write");
-	if (data) {
-		if (gf_string2boolean (data->data, &priv->encrypt_write) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"encrypt-write takes only boolean options");
-			GF_FREE (priv);
-			return -1;
-		}
-	}
-
-	data = dict_get (this->options, "decrypt-read");
-	if (data) {
-		if (gf_string2boolean (data->data, &priv->decrypt_read) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"decrypt-read takes only boolean options");
-			GF_FREE (priv);
-			return -1;
-		}
-	}
-
-	this->private = priv;
-	gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded");
-	return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-	rot_13_private_t *priv = this->private;
-
-        if (!priv)
-                return;
-        this->private = NULL;
-	GF_FREE (priv);
-
-	return;
-}
-
-struct xlator_fops fops = {
-	.readv        = rot13_readv,
-	.writev       = rot13_writev
-};
-
-struct xlator_cbks cbks;
-
-struct volume_options options[] = {
-	{ .key  = {"encrypt-write"},
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"decrypt-read"},
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {NULL} },
-};
diff --git a/xlators/experimental/Makefile.am b/xlators/experimental/Makefile.am
deleted file mode 100644
index a530845c4c0..00000000000
--- a/xlators/experimental/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = jbr-client jbr-server fdl dht2 posix2
-
-CLEANFILES =
diff --git a/xlators/experimental/README.md b/xlators/experimental/README.md
deleted file mode 100644
index b00f24e114b..00000000000
--- a/xlators/experimental/README.md
+++ /dev/null
@@ -1,107 +0,0 @@
-# Purpose of this directory
-
-This directory is created to host experimental gluster translators. A new
-translator that is *experimental* in nature, would need to create its own
-subdirectory under this directory, to host/publish its work.
-
-Example:
-  The first commit should include the following changes
-    1. xlators/experimental/Makefile.am
-      NOTE: Add foobar to the list of SUBDIRS here
-    2. xlators/experimental/foobar
-    3. xlators/experimental/foobar/Makefle.am
-      NOTE: Can be empty initially in the first commit
-    4. configure.ac
-      NOTE: Include your experimental Makefile under AC_CONFIG_FILES
-    5. xlators/experimental/foobar/README.md
-      NOTE: The readme should cover details as required for the translator to be
-      accepted as experimental, primarily including a link to the specification
-      under the gluster-specs repository [1]. Later the readme should suffice
-      as an entry point for developers and users alike, who wish to experiment
-      with the xlator under development
-    6. xlators/experimental/foobar/TODO.md
-      NOTE: This is a list of TODO's identified during the development process
-      that needs addressing over time. These include exceptions granted during
-      the review process, for things not addressed when commits are merged into
-      the repository
-
-# Why is it provided
-
-Quite often translator development that happens out of tree, does not get
-enough eyeballs early in its development phase, has not undergone CI
-(regression/continuous integration testing), and at times is not well integrated
-with the rest of gluster stack.
-
-Also, when such out of tree translators are submitted for acceptance, it is a
-bulk commit that makes review difficult and inefficient. Such submissions also
-have to be merged forward, and depending on the time spent in developing the
-translator the master branch could have moved far ahead, making this a painful
-activity.
-
-Experimental is born out of such needs, to provide xlator developers,
-  - Early access to CI
-  - Ability to adapt to ongoing changes in other parts of gluster
-  - More eye balls on the code and design aspects of the translator
-  - TBD: What else?
-
-and for maintainers,
-  - Ability to look at smaller change sets in the review process
-  - Ability to verify/check implementation against the specification provided
-
-# General rules
-
-1. If a new translator is added under here it should, at the very least, pass
-compilation.
-
-2. All translators under the experimental directory are shipped as a part of
-gluster-experimental RPMs.
-TBD: Spec file and other artifacts for the gluster-experimental RPM needs to be
-fleshed out.
-
-3. Experimental translators can leverage the CI framework as needed. Tests need
-to be hosted under xlators/experimental/tests initially, and later moved to the
-appropriate tests/ directory as the xlator matures. It is encouraged to provide
-tests for each commit or series of commits, so that code and tests can be
-inspected together.
-
-4. If any experimental translator breaks CI, it is quarantined till demonstrable
-proof towards the contrary is provided. This is applicable as tests are moved
-out of experimental tests directory to the CI framework directory, as otherwise
-experimental tests are not a part of regular CI regression runs.
-
-5. An experimental translator need not function at all, as a result commits can
-be merged pretty much at will as long as other rules as stated are not violated.
-
-6. Experimental submissions will be assigned a existing maintainer, to aid
-merging commits and ensure aspects of gluster code submissions are respected.
-When an experimental xlator is proposed and the first commit posted
-a mail to gluster-devel@gluster.org requesting attention, will assign the
-maintainer buddy for the submission.
-NOTE: As we scale, this may change.
-
-6. More?
-
-# Getting out of the experimental jail
-
-So you now think your xlator is ready to leave experimental and become part of
-mainline!
-- TBD: guidelines pending.
-
-# FAQs
-
-1. How do I submit/commit experimental framework changes outside of my
-experimental xlator?
-  - Provide such framework changes as a separate commit
-  - Conditionally ensure these are built or activated only when the experimental
-  feature is activated, so as to prevent normal gluster workflow to function as
-  before
-  - TBD: guidelines and/or examples pending.
-
-2. Ask your question either on gluster-devel@gluster.org or as a change request
-to this file in gluster gerrit [2] for an answer that will be assimilated into
-this readme.
-
-# Links
-[1] http://review.gluster.org/#/q/project:glusterfs-specs
-
-[2] http://review.gluster.org/#/q/project:glusterfs
diff --git a/xlators/experimental/dht2/Makefile.am b/xlators/experimental/dht2/Makefile.am
deleted file mode 100644
index 9d910a66056..00000000000
--- a/xlators/experimental/dht2/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = dht2-client dht2-server
-
-CLEANFILES =
diff --git a/xlators/experimental/dht2/README.md b/xlators/experimental/dht2/README.md
deleted file mode 100644
index 8f249a83673..00000000000
--- a/xlators/experimental/dht2/README.md
+++ /dev/null
@@ -1,47 +0,0 @@
-# DHT2 Experimental README
-
-DHT2 is the new distribution scheme being developed for Gluster, that
-aims to remove the subdirectory spread across all DHT subvolumes.
-
-As a result of this work, the Gluster backend file layouts and on disk
-representation of directories and files are modified, thus making DHT2
-volumes incompatible to existing DHT based Gluster deployments.
-
-This document presents interested users with relevant data to play around
-with DHT2 volumes and provide feedback towards the same.
-
-REMOVEME: Design details currently under review here,
-	- http://review.gluster.org/#/c/13395/
-
-TODO: Add more information as relevant code is pulled in
-
-# Directory strucutre elaborated
-
-## dht2-server
-This directory contains code for the server side DHT2 xlator. This xlator is
-intended to run on the brick graph, and is responsible for FOP synchronization,
-redirection, transactions, and journal replays.
-
-NOTE: The server side code also handles changes to volume/cluster map and
-also any rebalance activities.
-
-## dht2-client
-This directory contains code for the client side DHT2 xlator. This xlator is
-intended to run on the client/access protocol/mount graph, and is responsible
-for FOP routing to the right DHT2 subvolume. It uses a volume/cluster wide map
-of the routing (layout), to achieve the same.
-
-## dht2-common
-This directory contains code that is used in common across other parts of DHT2.
-For example, FOP routing store/consult abstractions that are common across the
-client and server side of DHT2.
-
-## Issue: How to build dht2-common?
-  1. Build a shared object
-    - We cannot ship this as a part of both the client xlator RPM
-  2. Build an archive
-    - Symbol clashes? when both the client and server xlators are loaded as a
-    part of the same graph
-  3. Compile with other parts of the code that needs it 
-    - Not a very different from (2) above
-    - This is what is chosen at present, and maybe would be revised later
diff --git a/xlators/experimental/dht2/TODO.md b/xlators/experimental/dht2/TODO.md
deleted file mode 100644
index 1e2c53c5b36..00000000000
--- a/xlators/experimental/dht2/TODO.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# DHT2 TODO list
-
-<Items will be added as code is pulled into the repository>
diff --git a/xlators/experimental/dht2/dht2-client/src/Makefile.am b/xlators/experimental/dht2/dht2-client/src/Makefile.am
deleted file mode 100644
index 39132994d08..00000000000
--- a/xlators/experimental/dht2/dht2-client/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-xlator_LTLIBRARIES = dht2c.la
-
-dht2c_sources = dht2-client-main.c
-
-dht2common_sources = $(top_srcdir)/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
-
-dht2c_la_SOURCES = $(dht2c_sources) $(dht2common_sources)
-dht2c_la_LDFLAGS = -module -avoid-version
-dht2c_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/experimental/dht2/dht2-common/src/
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-
-CLEANFILES =
diff --git a/xlators/experimental/dht2/dht2-client/src/dht2-client-main.c b/xlators/experimental/dht2/dht2-client/src/dht2-client-main.c
deleted file mode 100644
index bd1d446e2b5..00000000000
--- a/xlators/experimental/dht2/dht2-client/src/dht2-client-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: dht2-client-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-dht2_client_init (xlator_t *this)
-{
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Missing children in volume graph, this (%s) is"
-                        " not a leaf translator", this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-dht2_client_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = dht2_client_init,
-        .fini           = dht2_client_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c b/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
deleted file mode 100644
index d959483b8a4..00000000000
--- a/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: dht2-common-map.c
- * This file contains helper routines to store, consult, the volume map
- * for subvolume to GFID relations.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "statedump.h"
diff --git a/xlators/experimental/dht2/dht2-server/src/Makefile.am b/xlators/experimental/dht2/dht2-server/src/Makefile.am
deleted file mode 100644
index 4f721551020..00000000000
--- a/xlators/experimental/dht2/dht2-server/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-xlator_LTLIBRARIES = dht2s.la
-
-dht2s_sources = dht2-server-main.c
-
-dht2common_sources = $(top_srcdir)/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
-
-dht2s_la_SOURCES = $(dht2s_sources) $(dht2common_sources)
-dht2s_la_LDFLAGS = -module -avoid-version
-dht2s_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/experimental/dht2/dht2-common/src/
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-
-CLEANFILES =
diff --git a/xlators/experimental/dht2/dht2-server/src/dht2-server-main.c b/xlators/experimental/dht2/dht2-server/src/dht2-server-main.c
deleted file mode 100644
index 1f232cc3430..00000000000
--- a/xlators/experimental/dht2/dht2-server/src/dht2-server-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: dht2-server-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-dht2_server_init (xlator_t *this)
-{
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Missing children in volume graph, this (%s) is"
-                        " not a leaf translator", this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-dht2_server_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = dht2_server_init,
-        .fini           = dht2_server_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/experimental/fdl/src/Makefile.am b/xlators/experimental/fdl/src/Makefile.am
deleted file mode 100644
index aed0204284f..00000000000
--- a/xlators/experimental/fdl/src/Makefile.am
+++ /dev/null
@@ -1,43 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-xlator_LTLIBRARIES = fdl.la
-
-noinst_HEADERS = jnl-types.h
-
-nodist_fdl_la_SOURCES = fdl.c
-fdl_la_LDFLAGS = -module -avoid-version
-fdl_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-sbin_PROGRAMS = gf_logdump gf_recon
-gf_logdump_SOURCES = logdump.c
-nodist_gf_logdump_SOURCES = libfdl.c
-gf_logdump_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
-                   $(top_builddir)/api/src/libgfapi.la
-
-# Eventually recon(ciliation) code will move elsewhere, but for now it's
-# easier to have it next to the similar logdump code.
-gf_recon_SOURCES = recon.c
-nodist_gf_recon_SOURCES = librecon.c
-gf_recon_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
-                   $(top_builddir)/api/src/libgfapi.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	      -I$(top_srcdir)/api/src -fPIC \
-	      -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
-	      -DDATADIR=\"$(localstatedir)\"
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-noinst_PYTHON = gen_fdl.py gen_dumper.py gen_recon.py
-EXTRA_DIST = fdl-tmpl.c dump-tmpl.c recon-tmpl.c
-
-CLEANFILES = $(nodist_fdl_la_SOURCES) $(nodist_gf_logdump_SOURCES) \
-	$(nodist_gf_recon_SOURCES)
-
-fdl.c: fdl-tmpl.c gen_fdl.py
-	$(PYTHON) $(srcdir)/gen_fdl.py $(srcdir)/fdl-tmpl.c > $@
-
-libfdl.c: dump-tmpl.c gen_dumper.py
-	$(PYTHON) $(srcdir)/gen_dumper.py $(srcdir)/dump-tmpl.c > $@
-
-librecon.c: recon-tmpl.c gen_recon.py
-	$(PYTHON) $(srcdir)/gen_recon.py $(srcdir)/recon-tmpl.c > $@
diff --git a/xlators/experimental/fdl/src/dump-tmpl.c b/xlators/experimental/fdl/src/dump-tmpl.c
deleted file mode 100644
index cac1071a9c1..00000000000
--- a/xlators/experimental/fdl/src/dump-tmpl.c
+++ /dev/null
@@ -1,156 +0,0 @@
-#pragma fragment PROLOG
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs.h"
-#include "iatt.h"
-#include "xlator.h"
-#include "jnl-types.h"
-
-#pragma fragment DICT
-        {
-                int key_len, data_len;
-                char *key_ptr;
-                printf ("@ARGNAME@ = dict {\n");
-                for (;;) {
-                        key_len = *((int *)new_meta);
-                        new_meta += sizeof(int);
-                        if (!key_len) {
-                                break;
-                        }
-                        key_ptr = new_meta;
-                        new_meta += key_len;
-                        data_len = *((int *)new_meta);
-                        new_meta += sizeof(int) + data_len;
-                        printf (" %s = <%d bytes>\n", key_ptr, data_len);
-                }
-                printf ("}\n");
-        }
-
-#pragma fragment DOUBLE
-        printf ("@ARGNAME@ = @FORMAT@\n", *((uint64_t *)new_meta),
-                *((uint64_t *)new_meta));
-        new_meta += sizeof(uint64_t);
-
-#pragma fragment GFID
-        printf ("@ARGNAME@ = <gfid %s>\n", uuid_utoa(*((uuid_t *)new_meta)));
-        new_meta += 16;
-
-#pragma fragment INTEGER
-        printf ("@ARGNAME@ = @FORMAT@\n", *((uint32_t *)new_meta),
-                *((uint32_t *)new_meta));
-        new_meta += sizeof(uint32_t);
-
-#pragma fragment LOC
-        printf ("@ARGNAME@ = loc {\n");
-        printf ("  gfid = %s\n", uuid_utoa(*((uuid_t *)new_meta)));
-        new_meta += 16;
-        printf ("  pargfid = %s\n", uuid_utoa(*((uuid_t *)new_meta)));
-        new_meta += 16;
-        if (*(new_meta++)) {
-                printf ("  name = %s\n", new_meta);
-                new_meta += (strlen(new_meta) + 1);
-        }
-        printf ("}\n");
-
-#pragma fragment STRING
-        if (*(new_meta++)) {
-                printf ("@ARGNAME@ = %s\n", new_meta);
-                new_meta += (strlen(new_meta) + 1);
-        }
-
-#pragma fragment VECTOR
-        {
-                size_t len = *((size_t *)new_meta);
-                new_meta += sizeof(len);
-                printf ("@ARGNAME@ = <%zu bytes>\n", len);
-                new_data += len;
-        }
-
-#pragma fragment IATT
-        {
-                ia_prot_t *myprot = ((ia_prot_t *)new_meta);
-                printf ("@ARGNAME@ = iatt {\n");
-                printf ("  ia_prot = %c%c%c",
-                        myprot->suid ? 'S' : '-',
-                        myprot->sgid ? 'S' : '-',
-                        myprot->sticky ? 'T' : '-');
-                printf ("%c%c%c",
-                        myprot->owner.read ? 'r' : '-',
-                        myprot->owner.write ? 'w' : '-',
-                        myprot->owner.exec ? 'x' : '-');
-                printf ("%c%c%c",
-                        myprot->group.read ? 'r' : '-',
-                        myprot->group.write ? 'w' : '-',
-                        myprot->group.exec ? 'x' : '-');
-                printf ("%c%c%c\n",
-                        myprot->other.read ? 'r' : '-',
-                        myprot->other.write ? 'w' : '-',
-                        myprot->other.exec ? 'x' : '-');
-                new_meta += sizeof(ia_prot_t);
-                uint32_t *myints = (uint32_t *)new_meta;
-                printf ("  ia_uid = %u\n", myints[0]);
-                printf ("  ia_gid = %u\n", myints[1]);
-                printf ("  ia_atime = %u.%09u\n", myints[2], myints[3]);
-                printf ("  ia_mtime = %u.%09u\n", myints[4], myints[5]);
-                new_meta += sizeof(*myints) * 6;
-        }
-
-#pragma fragment FOP
-void
-fdl_dump_@NAME@ (char **old_meta, char **old_data)
-{
-        char    *new_meta	= *old_meta;
-        char	*new_data	= *old_data;
-
-        /* TBD: word size/endianness */
-@FUNCTION_BODY@
-
-        *old_meta = new_meta;
-        *old_data = new_data;
-}
-
-#pragma fragment CASE
-        case GF_FOP_@UPNAME@:
-                printf ("=== GF_FOP_@UPNAME@\n");
-                fdl_dump_@NAME@ (&new_meta, &new_data);
-                break;
-
-#pragma fragment EPILOG
-int
-fdl_dump (char **old_meta, char **old_data)
-{
-        char            *new_meta       = *old_meta;
-        char            *new_data       = *old_data;
-        static glfs_t   *fs             = NULL;
-        int             recognized      = 1;
-        event_header_t  *eh;
-
-        /*
-         * We don't really call anything else in GFAPI, but this is the most
-         * convenient way to satisfy all of the spurious dependencies on how it
-         * or glusterfsd initialize (e.g. setting up THIS).
-         */
-        if (!fs) {
-                fs = glfs_new ("dummy");
-        }
-
-        eh = (event_header_t *)new_meta;
-        new_meta += sizeof (*eh);
-
-        /* TBD: check event_type instead of assuming NEW_REQUEST */
-
-        switch (eh->fop_type) {
-@SWITCH_BODY@
-
-        default:
-                printf ("unknown fop %u\n", eh->fop_type);
-                recognized = 0;
-        }
-
-        *old_meta = new_meta;
-        *old_data = new_data;
-        return recognized;
-}
diff --git a/xlators/experimental/fdl/src/fdl-tmpl.c b/xlators/experimental/fdl/src/fdl-tmpl.c
deleted file mode 100644
index fdcfafbac31..00000000000
--- a/xlators/experimental/fdl/src/fdl-tmpl.c
+++ /dev/null
@@ -1,506 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include "call-stub.h"
-#include "iatt.h"
-#include "defaults.h"
-#include "syscall.h"
-#include "xlator.h"
-#include "jnl-types.h"
-
-/* TBD: make tunable */
-#define META_FILE_SIZE  (1 << 20)
-#define DATA_FILE_SIZE  (1 << 24)
-
-enum gf_fdl {
-        gf_fdl_mt_fdl_private_t = gf_common_mt_end + 1,
-        gf_fdl_mt_end
-};
-
-typedef struct {
-        char            *type;
-        off_t           size;
-        char            *path;
-        int             fd;
-        void *          ptr;
-        off_t           max_offset;
-} log_obj_t;
-
-typedef struct {
-        struct list_head        reqs;
-        pthread_mutex_t         req_lock;
-        pthread_cond_t          req_cond;
-        char                    *log_dir;
-        pthread_t               worker;
-        gf_boolean_t            should_stop;
-        gf_boolean_t            change_term;
-        log_obj_t               meta_log;
-        log_obj_t               data_log;
-        int                     term;
-        int                     first_term;
-} fdl_private_t;
-
-void
-fdl_enqueue (xlator_t *this, call_stub_t *stub)
-{
-        fdl_private_t   *priv   = this->private;
-
-        pthread_mutex_lock (&priv->req_lock);
-        list_add_tail (&stub->list, &priv->reqs);
-        pthread_mutex_unlock (&priv->req_lock);
-
-        pthread_cond_signal (&priv->req_cond);
-}
-
-#pragma generate
-
-char *
-fdl_open_term_log (xlator_t *this, log_obj_t *obj, int term)
-{
-        fdl_private_t   *priv   = this->private;
-        int             ret;
-        char *          ptr     = NULL;
-
-        /*
-         * Use .jnl instead of .log so that we don't get test info (mistakenly)
-         * appended to our journal files.
-         */
-        if (this->ctx->cmd_args.log_ident) {
-                ret = gf_asprintf (&obj->path, "%s/%s-%s-%d.jnl",
-                                   priv->log_dir, this->ctx->cmd_args.log_ident,
-                                   obj->type, term);
-        }
-        else {
-                ret = gf_asprintf (&obj->path, "%s/fubar-%s-%d.jnl",
-                                   priv->log_dir, obj->type, term);
-        }
-        if ((ret <= 0) || !obj->path) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to construct log-file path");
-                goto err;
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "opening %s (size %ld)",
-                obj->path, obj->size);
-
-        obj->fd = open (obj->path, O_RDWR|O_CREAT|O_TRUNC, 0666);
-        if (obj->fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to open log file (%s)", strerror(errno));
-                goto err;
-        }
-
-#if !defined(GF_BSD_HOST_OS)
-        /*
-         * NetBSD can just go die in a fire.  Even though it claims to support
-         * fallocate/posix_fallocate they don't actually *do* anything so the
-         * file size remains zero.  Then mmap succeeds anyway, but any access
-         * to the mmap'ed region will segfault.  It would be acceptable for
-         * fallocate to do what it says, for mmap to fail, or for access to
-         * extend the file.  NetBSD managed to hit the trifecta of Getting
-         * Everything Wrong, and debugging in that environment to get this far
-         * has already been painful enough (systems I worked on in 1990 were
-         * better that way).  We'll fall through to the lseek/write method, and
-         * performance will be worse, and TOO BAD.
-         */
-        if (sys_fallocate(obj->fd,0,0,obj->size) < 0)
-#endif
-        {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to fallocate space for log file");
-                /* Have to do this the ugly page-faulty way. */
-                (void) sys_lseek (obj->fd, obj->size-1, SEEK_SET);
-                (void) sys_write (obj->fd, "", 1);
-        }
-
-        ptr = mmap (NULL, obj->size, PROT_WRITE, MAP_SHARED, obj->fd, 0);
-        if (ptr == MAP_FAILED) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to mmap log (%s)",
-                        strerror(errno));
-                goto err;
-        }
-
-        obj->ptr = ptr;
-        obj->max_offset = 0;
-        return ptr;
-
-err:
-        if (obj->fd >= 0) {
-                sys_close (obj->fd);
-                obj->fd = (-1);
-        }
-        if (obj->path) {
-                GF_FREE (obj->path);
-                obj->path = NULL;
-        }
-        return ptr;
-}
-
-void
-fdl_close_term_log (xlator_t *this, log_obj_t *obj)
-{
-        fdl_private_t   *priv           = this->private;
-
-        if (obj->ptr) {
-                (void) munmap (obj->ptr, obj->size);
-                obj->ptr = NULL;
-        }
-
-        if (obj->fd >= 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "truncating term %d %s journal to %ld",
-                        priv->term, obj->type, obj->max_offset);
-                if (sys_ftruncate(obj->fd,obj->max_offset) < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to truncate journal (%s)",
-                                strerror(errno));
-                }
-                sys_close (obj->fd);
-                obj->fd = (-1);
-        }
-
-        if (obj->path) {
-                GF_FREE (obj->path);
-                obj->path = NULL;
-        }
-}
-
-gf_boolean_t
-fdl_change_term (xlator_t *this, char **meta_ptr, char **data_ptr)
-{
-        fdl_private_t   *priv           = this->private;
-
-        fdl_close_term_log (this, &priv->meta_log);
-        fdl_close_term_log (this, &priv->data_log);
-
-        ++(priv->term);
-
-        *meta_ptr = fdl_open_term_log (this, &priv->meta_log, priv->term);
-        if (!*meta_ptr) {
-                return _gf_false;
-        }
-
-        *data_ptr = fdl_open_term_log (this, &priv->data_log, priv->term);
-        if (!*data_ptr) {
-                return _gf_false;
-        }
-
-        return _gf_true;
-}
-
-void *
-fdl_worker (void *arg)
-{
-        xlator_t        *this           = arg;
-        fdl_private_t   *priv           = this->private;
-        call_stub_t     *stub;
-        char *          meta_ptr        = NULL;
-        off_t           *meta_offset    = &priv->meta_log.max_offset;
-        char *          data_ptr        = NULL;
-        off_t           *data_offset    = &priv->data_log.max_offset;
-        unsigned long   base_as_ul;
-        void *          msync_ptr;
-        size_t          msync_len;
-        gf_boolean_t    recycle;
-        void            *err_label      = &&err_unlocked;
-
-        priv->meta_log.type = "meta";
-        priv->meta_log.size = META_FILE_SIZE;
-        priv->meta_log.path = NULL;
-        priv->meta_log.fd = (-1);
-        priv->meta_log.ptr = NULL;
-
-        priv->data_log.type = "data";
-        priv->data_log.size = DATA_FILE_SIZE;
-        priv->data_log.path = NULL;
-        priv->data_log.fd = (-1);
-        priv->data_log.ptr = NULL;
-
-        /* TBD: initial term should come from persistent storage (e.g. etcd) */
-        priv->first_term = ++(priv->term);
-        meta_ptr = fdl_open_term_log (this, &priv->meta_log, priv->term);
-        if (!meta_ptr) {
-                goto *err_label;
-        }
-        data_ptr = fdl_open_term_log (this, &priv->data_log, priv->term);
-        if (!data_ptr) {
-                fdl_close_term_log (this, &priv->meta_log);
-                goto *err_label;
-        }
-
-        for (;;) {
-                pthread_mutex_lock (&priv->req_lock);
-                err_label = &&err_locked;
-                while (list_empty(&priv->reqs)) {
-                        pthread_cond_wait (&priv->req_cond, &priv->req_lock);
-                        if (priv->should_stop) {
-                                goto *err_label;
-                        }
-                        if (priv->change_term) {
-                                if (!fdl_change_term(this, &meta_ptr,
-                                                           &data_ptr)) {
-                                        goto *err_label;
-                                }
-                                priv->change_term = _gf_false;
-                                continue;
-                        }
-                }
-                stub = list_entry (priv->reqs.next, call_stub_t, list);
-                list_del_init (&stub->list);
-                pthread_mutex_unlock (&priv->req_lock);
-                err_label = &&err_unlocked;
-                /*
-                 * TBD: batch requests
-                 *
-                 * What we should do here is gather up *all* of the requests
-                 * that have accumulated since we were last at this point,
-                 * blast them all out in one big writev, and then dispatch them
-                 * all before coming back for more.  That maximizes throughput,
-                 * at some cost to latency (due to queuing effects at the log
-                 * stage).  Note that we're likely to be above io-threads, so
-                 * the dispatch itself will be parallelized (at further cost to
-                 * latency).  For now, we just do the simplest thing and handle
-                 * one request all the way through before fetching the next.
-                 *
-                 * So, why mmap/msync instead of writev/fdatasync?  Because it's
-                 * faster.  Much faster.  So much faster that I half-suspect
-                 * cheating, but it's more convenient for now than having to
-                 * ensure that everything's page-aligned for O_DIRECT (the only
-                 * alternative that still might avoid ridiculous levels of
-                 * local-FS overhead).
-                 *
-                 * TBD: check that msync really does get our data to disk.
-                 */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "logging %u+%u bytes for op %d",
-                        stub->jnl_meta_len, stub->jnl_data_len, stub->fop);
-                recycle = _gf_false;
-                if ((*meta_offset + stub->jnl_meta_len) > priv->meta_log.size) {
-                        recycle = _gf_true;
-                }
-                if ((*data_offset + stub->jnl_data_len) > priv->data_log.size) {
-                        recycle = _gf_true;
-                }
-                if (recycle && !fdl_change_term(this,&meta_ptr,&data_ptr)) {
-                        goto *err_label;
-                }
-                meta_ptr = priv->meta_log.ptr;
-                data_ptr = priv->data_log.ptr;
-                gf_log (this->name, GF_LOG_DEBUG, "serializing to %p/%p",
-                        meta_ptr + *meta_offset, data_ptr + *data_offset);
-                stub->serialize (stub, meta_ptr + *meta_offset,
-                                       data_ptr + *data_offset);
-                if (stub->jnl_meta_len > 0) {
-                        base_as_ul = (unsigned long) (meta_ptr + *meta_offset);
-                        msync_ptr = (void *) (base_as_ul & ~0x0fff);
-                        msync_len = (size_t) (base_as_ul &  0x0fff);
-                        if (msync (msync_ptr, msync_len+stub->jnl_meta_len,
-                                              MS_SYNC) < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to log request meta (%s)",
-                                        strerror(errno));
-                        }
-                        *meta_offset += stub->jnl_meta_len;
-                }
-                if (stub->jnl_data_len > 0) {
-                        base_as_ul = (unsigned long) (data_ptr + *data_offset);
-                        msync_ptr = (void *) (base_as_ul & ~0x0fff);
-                        msync_len = (size_t) (base_as_ul &  0x0fff);
-                        if (msync (msync_ptr, msync_len+stub->jnl_data_len,
-                                              MS_SYNC) < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to log request data (%s)",
-                                        strerror(errno));
-                        }
-                        *data_offset += stub->jnl_data_len;
-                }
-                call_resume (stub);
-        }
-
-err_locked:
-        pthread_mutex_unlock (&priv->req_lock);
-err_unlocked:
-        fdl_close_term_log (this, &priv->meta_log);
-        fdl_close_term_log (this, &priv->data_log);
-        return NULL;
-}
-
-int32_t
-fdl_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
-{
-        fdl_private_t   *priv   = this->private;
-        dict_t          *tdict;
-        int32_t         gt_err  = EIO;
-
-        switch (op) {
-
-        case FDL_IPC_CHANGE_TERM:
-                gf_log (this->name, GF_LOG_INFO, "got CHANGE_TERM op");
-                priv->change_term = _gf_true;
-                pthread_cond_signal (&priv->req_cond);
-                STACK_UNWIND_STRICT (ipc, frame, 0, 0, NULL);
-                break;
-
-        case FDL_IPC_GET_TERMS:
-                gf_log (this->name, GF_LOG_INFO, "got GET_TERMS op");
-                tdict = dict_new ();
-                if (!tdict) {
-                        gt_err = ENOMEM;
-                        goto gt_done;
-                }
-                if (dict_set_int32(tdict,"first",priv->first_term) != 0) {
-                        goto gt_done;
-                }
-                if (dict_set_int32(tdict,"last",priv->term) != 0) {
-                        goto gt_done;
-                }
-                gt_err = 0;
-        gt_done:
-                if (gt_err) {
-                        STACK_UNWIND_STRICT (ipc, frame, -1, gt_err, NULL);
-                } else {
-                        STACK_UNWIND_STRICT (ipc, frame, 0, 0, tdict);
-                }
-                if (tdict) {
-                        dict_unref (tdict);
-                }
-                break;
-
-        default:
-                STACK_WIND_TAIL (frame,
-                                 FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->ipc,
-                                 op, xdata);
-        }
-
-        return 0;
-}
-
-int
-fdl_init (xlator_t *this)
-{
-        fdl_private_t   *priv   = NULL;
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_fdl_mt_fdl_private_t);
-        if (!priv) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to allocate fdl_private");
-                goto err;
-        }
-
-        INIT_LIST_HEAD (&priv->reqs);
-        if (pthread_mutex_init (&priv->req_lock, NULL) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to initialize req_lock");
-                goto err;
-        }
-        if (pthread_cond_init (&priv->req_cond, NULL) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to initialize req_cond");
-                goto err;
-        }
-
-        GF_OPTION_INIT ("log-path", priv->log_dir, path, err);
-
-        this->private = priv;
-        /*
-         * The rest of the fop table is automatically generated, so this is a
-         * bit cleaner than messing with the generation to add a hand-written
-         * exception.
-         */
-        this->fops->ipc = fdl_ipc;
-
-        if (pthread_create(&priv->worker,NULL,fdl_worker,this) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to start fdl_worker");
-                goto err;
-        }
-
-        return 0;
-
-err:
-        if (priv) {
-                GF_FREE(priv);
-        }
-        return -1;
-}
-
-void
-fdl_fini (xlator_t *this)
-{
-        fdl_private_t   *priv   = this->private;
-
-        if (priv) {
-                priv->should_stop = _gf_true;
-                pthread_cond_signal (&priv->req_cond);
-                pthread_join (priv->worker, NULL);
-                GF_FREE(priv);
-        }
-}
-
-int
-fdl_reconfigure (xlator_t *this, dict_t *options)
-{
-        fdl_private_t   *priv   = this->private;
-
-	GF_OPTION_RECONF ("log_dir", priv->log_dir, options, path, out);
-        /* TBD: react if it changed */
-
-out:
-        return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("fdl", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_fdl_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-class_methods_t class_methods = {
-        .init           = fdl_init,
-        .fini           = fdl_fini,
-        .reconfigure    = fdl_reconfigure,
-        .notify         = default_notify,
-};
-
-struct volume_options options[] = {
-        { .key = {"log-path"},
-          .type = GF_OPTION_TYPE_PATH,
-          .default_value = DEFAULT_LOG_FILE_DIRECTORY,
-          .description = "Directory for FDL files."
-        },
-        { .key  = {NULL} },
-};
-
-struct xlator_cbks cbks = {
-        .release        = default_release,
-        .releasedir     = default_releasedir,
-        .forget         = default_forget,
-};
diff --git a/xlators/experimental/fdl/src/gen_dumper.py b/xlators/experimental/fdl/src/gen_dumper.py
deleted file mode 100755
index 42db55d2cb3..00000000000
--- a/xlators/experimental/fdl/src/gen_dumper.py
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/python
-
-import os
-import re
-import sys
-
-curdir = os.path.dirname (sys.argv[0])
-gendir = os.path.join (curdir, '../../../../libglusterfs/src')
-sys.path.append (gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# See the big header comment at the start of gen_fdl.py to see how the stages
-# fit together.  The big difference here is that *all* of the C code is in the
-# template file as labelled fragments, instead of as Python strings.  That
-# makes it much easier to edit in one place, with proper syntax highlighting
-# and indentation.
-#
-#   Stage 1 uses type-specific fragments to generate FUNCTION_BODY, instead of
-#   LEN_*_TEMPLATE and SERLZ_*_TEMPLATE to generate LEN_CODE and SER_CODE.
-#
-#   Stage 2 uses the FOP and CASE fragments instead of RECON_TEMPLATE and
-#   FOP_TEMPLATE.  The expanded FOP code (including FUNCTION_BODY substitution
-#   in the middle of each function) is emitted immediately; the expanded CASE
-#   code is saved for the next stage.
-#
-#   Stage 3 uses the PROLOG and EPILOG fragments, with the expanded CASE code
-#   in the middle of EPILOG, to generate the whole output file.
-#
-# Another way of looking at it is to consider how the fragments appear in
-# the final output:
-#
-#   PROLOG
-#   FOP (expanded for CREATE)
-#       FOP before FUNCTION_BODY
-#       LOC, INTEGER, GFID, etc. (one per arg, by type)
-#       FOP after FUNCTION_BODY
-#   FOP (expanded for WRITEV)
-#       FOP before FUNCTION_BODY
-#       GFID, VECTOR, etc. (on per arg, by type)
-#       FOP after FUNCTION_BODY
-#   (more FOPs)
-#   EPILOG
-#       EPILOG before CASE
-#       CASE statements (one per fop)
-#       EPILOG after CASE
-
-typemap = {
-	'dict_t *':				( "DICT",		""),
-	'fd_t *':				( "GFID",		""),
-	'dev_t':				( "DOUBLE",		"%ld (0x%lx)"),
-	'gf_xattrop_flags_t':	( "INTEGER",	"%d (0x%x)"),
-	'int32_t':				( "INTEGER",	"%d (0x%x)"),
-	'mode_t':				( "INTEGER",	"%d (0x%x)"),
-	'off_t':				( "DOUBLE",		"%ld (0x%lx)"),
-	'size_t':				( "DOUBLE",		"%ld (0x%lx)"),
-	'uint32_t':				( "INTEGER",	"%d (0x%x)"),
-	'loc_t *':				( "LOC",		""),
-	'const char *':			( "STRING",		""),
-	'struct iovec *':		( "VECTOR",		""),
-	'struct iatt *':		( "IATT",		""),
-}
-
-def get_special_subs (args):
-	code = ""
-	for arg in args:
-		if (arg[0] != 'fop-arg') or (len(arg) < 4):
-			continue
-		recon_type, recon_fmt = typemap[arg[2]]
-		code += fragments[recon_type].replace("@ARGNAME@",arg[3])		\
-									 .replace("@FORMAT@",recon_fmt)
-	return code
-
-def gen_functions ():
-	code = ""
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-		fop_subs[name]["@FUNCTION_BODY@"] = get_special_subs(value)
-		# Print the FOP fragment with @FUNCTION_BODY@ in the middle.
-		code += generate(fragments["FOP"],name,fop_subs)
-	return code
-
-def gen_cases ():
-	code = ""
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-		# Add the CASE fragment for this fop.
-		code += generate(fragments["CASE"],name,fop_subs)
-	return code
-
-def load_fragments (path="recon-tmpl.c"):
-	pragma_re = re.compile('pragma fragment (.*)')
-	cur_symbol = None
-	cur_value = ""
-	result = {}
-	for line in open(path,"r").readlines():
-		m = pragma_re.search(line)
-		if m:
-			if cur_symbol:
-				result[cur_symbol] = cur_value
-			cur_symbol = m.group(1)
-			cur_value = ""
-		else:
-			cur_value += line
-	if cur_symbol:
-		result[cur_symbol] = cur_value
-	return result
-
-if __name__ == "__main__":
-	fragments = load_fragments(sys.argv[1])
-	print "/* BEGIN GENERATED CODE - DO NOT MODIFY */"
-	print fragments["PROLOG"]
-	print gen_functions()
-	print fragments["EPILOG"].replace("@SWITCH_BODY@",gen_cases())
-	print "/* END GENERATED CODE */"
diff --git a/xlators/experimental/fdl/src/gen_fdl.py b/xlators/experimental/fdl/src/gen_fdl.py
deleted file mode 100755
index 7f6b1aaaeaa..00000000000
--- a/xlators/experimental/fdl/src/gen_fdl.py
+++ /dev/null
@@ -1,328 +0,0 @@
-#!/usr/bin/python
-
-import os
-import sys
-
-curdir = os.path.dirname (sys.argv[0])
-gendir = os.path.join (curdir, '../../../../libglusterfs/src')
-sys.path.append (gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# Generation occurs in three stages.  In this case, it actually makes more
-# sense to discuss them in the *opposite* order of that in which they
-# actually happen.
-#
-#   Stage 3 is to insert all of the generated code into a file, replacing the
-#   "#pragma generate" that's already there.  The file can thus contain all
-#   sorts of stuff that's not specific to one fop, either before or after the
-#   generated code as appropriate.
-#
-#   Stage 2 is to generate all of the code *for a particular fop*, using a
-#   string-valued template plus a table of substitution values.  Most of these
-#   are built in to the generator itself.  However, we also add a couple that
-#   are specific to this particular translator - LEN_CODE and SER_CODE.  These
-#   are per-fop functions to get the length or the contents (respectively) of
-#   what we'll put in the log.  As with stage 3 allowing per-file boilerplate
-#   before and after generated code, this allows per-fop boilerplate before and
-#   after generated code.
-#
-#   Stage 1, therefore, is to create the LEN_CODE and SER_CODE substitutions for
-#   each fop, and put them in the same table where e.g. NAME and SHORT_ARGS
-#   already are.  We do this by looking at the fop-description table in the
-#   generator module, then doing out own template substitution to plug each
-#   specific argument name into another string-valued template.
-#
-# So, what does this leave us with in terms of variables and files?
-#
-#   For stage 1, we have a series of LEN_*_TEMPLATE and SERLZ_*_TEMPLATE
-#   strings, which are used to generate the length and serialization code for
-#   each argument type.
-#
-#   For stage 2, we have a bunch of *_TEMPLATE strings (no LEN_ or SERLZ_
-#   prefix), which are used (along with the output from stage 1) to generate
-#   whole functions.
-#
-#   For stage 3, we have a whole separate file (fdl_tmpl.c) into which we insert
-#   the collection of all functions defined in stage 2.
-
-
-LEN_TEMPLATE = """
-void
-fdl_len_@NAME@ (call_stub_t *stub)
-{
-        uint32_t    meta_len    = sizeof (event_header_t);
-		uint32_t	data_len	= 0;
-
-        /* TBD: global stuff, e.g. uid/gid */
-@LEN_CODE@
-
-		/* TBD: pad extension length */
-		stub->jnl_meta_len = meta_len;
-		stub->jnl_data_len = data_len;
-}
-"""
-
-SER_TEMPLATE = """
-void
-fdl_serialize_@NAME@ (call_stub_t *stub, char *meta_buf, char *data_buf)
-{
-		event_header_t	*eh;
-		unsigned long	offset = 0;
-
-        /* TBD: word size/endianness */
-		eh = (event_header_t *)meta_buf;
-		eh->event_type = NEW_REQUEST;
-		eh->fop_type = GF_FOP_@UPNAME@;
-		eh->request_id = 0;	// TBD
-		meta_buf += sizeof (*eh);
-@SER_CODE@
-		/* TBD: pad extension length */
-		eh->ext_length = offset;
-}
-"""
-
-CBK_TEMPLATE = """
-int32_t
-fdl_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                @LONG_ARGS@)
-{
-        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
-                             @SHORT_ARGS@);
-        return 0;
-}
-"""
-
-CONTINUE_TEMPLATE = """
-int32_t
-fdl_@NAME@_continue (call_frame_t *frame, xlator_t *this,
-                     @LONG_ARGS@)
-{
-        STACK_WIND (frame, fdl_@NAME@_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                    @SHORT_ARGS@);
-        return 0;
-}
-
-"""
-
-FOP_TEMPLATE = """
-int32_t
-fdl_@NAME@ (call_frame_t *frame, xlator_t *this,
-            @LONG_ARGS@)
-{
-        call_stub_t     *stub;
-
-        stub = fop_@NAME@_stub (frame, default_@NAME@,
-                                @SHORT_ARGS@);
-		fdl_len_@NAME@ (stub);
-        stub->serialize = fdl_serialize_@NAME@;
-        fdl_enqueue (this, stub);
-
-        return 0;
-}
-"""
-
-LEN_DICT_TEMPLATE = """
-		if (@SRC@) {
-			data_pair_t *memb;
-			for (memb = @SRC@->members_list; memb; memb = memb->next) {
-				meta_len += sizeof(int);
-				meta_len += strlen(memb->key) + 1;
-				meta_len += sizeof(int);
-				meta_len += memb->value->len;
-			}
-		}
-		meta_len += sizeof(int);
-"""
-
-LEN_GFID_TEMPLATE = """
-        meta_len += 16;
-"""
-
-LEN_INTEGER_TEMPLATE = """
-        meta_len += sizeof (@SRC@);
-"""
-
-# 16 for gfid, 16 for pargfid, 1 for flag, 0/1 for terminating NUL
-LEN_LOC_TEMPLATE = """
-        if (@SRC@.name) {
-                meta_len += (strlen (@SRC@.name) + 34);
-        } else {
-                meta_len += 33;
-        }
-"""
-
-LEN_STRING_TEMPLATE = """
-        if (@SRC@) {
-                meta_len += (strlen (@SRC@) + 1);
-        } else {
-                meta_len += 1;
-        }
-"""
-
-LEN_VECTOR_TEMPLATE = """
-        meta_len += sizeof(size_t);
-        data_len += iov_length (@VEC@, @CNT@);
-"""
-
-LEN_IATT_TEMPLATE = """
-		meta_len += sizeof(@SRC@.ia_prot);
-		meta_len += sizeof(@SRC@.ia_uid);
-		meta_len += sizeof(@SRC@.ia_gid);
-		meta_len += sizeof(@SRC@.ia_atime);
-		meta_len += sizeof(@SRC@.ia_atime_nsec);
-		meta_len += sizeof(@SRC@.ia_mtime);
-		meta_len += sizeof(@SRC@.ia_mtime_nsec);
-"""
-
-SERLZ_DICT_TEMPLATE = """
-        if (@SRC@) {
-			data_pair_t *memb;
-			for (memb = @SRC@->members_list; memb; memb = memb->next) {
-				*((int *)(meta_buf+offset)) = strlen(memb->key) + 1;
-				offset += sizeof(int);
-				strcpy (meta_buf+offset, memb->key);
-				offset += strlen(memb->key) + 1;
-				*((int *)(meta_buf+offset)) = memb->value->len;
-				offset += sizeof(int);
-				memcpy (meta_buf+offset, memb->value->data, memb->value->len);
-				offset += memb->value->len;
-			}
-        }
-		*((int *)(meta_buf+offset)) = 0;
-		offset += sizeof(int);
-"""
-
-SERLZ_GFID_TEMPLATE = """
-        memcpy (meta_buf+offset, @SRC@->inode->gfid, 16);
-        offset += 16;
-"""
-
-SERLZ_INTEGER_TEMPLATE = """
-        memcpy (meta_buf+offset, &@SRC@, sizeof(@SRC@));
-        offset += sizeof(@SRC@);
-"""
-
-SERLZ_LOC_TEMPLATE = """
-        memcpy (meta_buf+offset, @SRC@.gfid, 16);
-        offset += 16;
-        memcpy (meta_buf+offset, @SRC@.pargfid, 16);
-        offset += 16;
-        if (@SRC@.name) {
-                *(meta_buf+offset) = 1;
-				++offset;
-                strcpy (meta_buf+offset, @SRC@.name);
-                offset += (strlen (@SRC@.name) + 1);
-        } else {
-                *(meta_buf+offset) = 0;
-				++offset;
-        }
-"""
-
-SERLZ_STRING_TEMPLATE = """
-        if (@SRC@) {
-                *(meta_buf+offset) = 1;
-				++offset;
-                strcpy (meta_buf+offset, @SRC@);
-                offset += strlen(@SRC@);
-        } else {
-                *(meta_buf+offset) = 0;
-				++offset;
-        }
-"""
-
-SERLZ_VECTOR_TEMPLATE = """
-        *((size_t *)(meta_buf+offset)) = iov_length (@VEC@, @CNT@);
-        offset += sizeof(size_t);
-        int32_t i;
-        for (i = 0; i < @CNT@; ++i) {
-                memcpy (data_buf, @VEC@[i].iov_base, @VEC@[i].iov_len);
-                data_buf += @VEC@[i].iov_len;
-        }
-"""
-
-# We don't need to save all of the fields - only those affected by chown,
-# chgrp, chmod, and utime.
-SERLZ_IATT_TEMPLATE = """
-		*((ia_prot_t *)(meta_buf+offset)) = @SRC@.ia_prot;
-		offset += sizeof(@SRC@.ia_prot);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_uid;
-		offset += sizeof(@SRC@.ia_uid);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_gid;
-		offset += sizeof(@SRC@.ia_gid);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_atime;
-		offset += sizeof(@SRC@.ia_atime);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_atime_nsec;
-		offset += sizeof(@SRC@.ia_atime_nsec);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_mtime;
-		offset += sizeof(@SRC@.ia_mtime);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_mtime_nsec;
-		offset += sizeof(@SRC@.ia_mtime_nsec);
-"""
-
-typemap = {
-	'dict_t *':				( LEN_DICT_TEMPLATE,	SERLZ_DICT_TEMPLATE),
-	'fd_t *':				( LEN_GFID_TEMPLATE,	SERLZ_GFID_TEMPLATE),
-	'dev_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'gf_xattrop_flags_t':	( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'int32_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'mode_t':				( LEN_INTEGER_TEMPLATE, SERLZ_INTEGER_TEMPLATE),
-	'off_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'size_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'uint32_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'loc_t *':				( LEN_LOC_TEMPLATE,		SERLZ_LOC_TEMPLATE),
-	'const char *':			( LEN_STRING_TEMPLATE,	SERLZ_STRING_TEMPLATE),
-	'struct iatt *':		( LEN_IATT_TEMPLATE,	SERLZ_IATT_TEMPLATE),
-}
-
-def get_special_subs (args):
-	len_code = ""
-	ser_code = ""
-	for arg in args:
-		if (arg[0] != 'fop-arg') or (len(arg) < 4):
-			continue
-		# Let this throw an exception if we get an unknown field name.  The
-		# broken build will remind whoever messed with the stub code that a
-		# corresponding update is needed here.
-		if arg[3] == "vector":
-			# Make it as obvious as possible that this is a special case.
-			len_code += LEN_VECTOR_TEMPLATE \
-				.replace("@VEC@","stub->args.vector") \
-				.replace("@CNT@","stub->args.count")
-			ser_code += SERLZ_VECTOR_TEMPLATE \
-				.replace("@VEC@","stub->args.vector") \
-				.replace("@CNT@","stub->args.count")
-		else:
-			len_tmpl, ser_tmpl = typemap[arg[2]]
-			src = "stub->args.%s" % arg[3]
-			len_code += len_tmpl.replace("@SRC@",src)
-			ser_code += ser_tmpl.replace("@SRC@",src)
-	return len_code, ser_code
-
-def gen_fdl ():
-	entrypoints = []
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-		len_code, ser_code = get_special_subs(value)
-		fop_subs[name]["@LEN_CODE@"] = len_code[:-1]
-		fop_subs[name]["@SER_CODE@"] = ser_code[:-1]
-		print generate(LEN_TEMPLATE,name,fop_subs)
-		print generate(SER_TEMPLATE,name,fop_subs)
-		print generate(CBK_TEMPLATE,name,cbk_subs)
-		print generate(CONTINUE_TEMPLATE,name,fop_subs)
-		print generate(FOP_TEMPLATE,name,fop_subs)
-		entrypoints.append(name)
-	print "struct xlator_fops fops = {"
-	for ep in entrypoints:
-		print "\t.%s = fdl_%s," % (ep, ep)
-	print "};"
-
-for l in open(sys.argv[1],'r').readlines():
-	if l.find('#pragma generate') != -1:
-		print "/* BEGIN GENERATED CODE - DO NOT MODIFY */"
-		gen_fdl()
-		print "/* END GENERATED CODE */"
-	else:
-		print l[:-1]
diff --git a/xlators/experimental/fdl/src/gen_recon.py b/xlators/experimental/fdl/src/gen_recon.py
deleted file mode 100755
index 67f9ea9ebd3..00000000000
--- a/xlators/experimental/fdl/src/gen_recon.py
+++ /dev/null
@@ -1,213 +0,0 @@
-#!/usr/bin/python
-
-import os
-import re
-import string
-import sys
-
-curdir = os.path.dirname (sys.argv[0])
-gendir = os.path.join (curdir, '../../../../libglusterfs/src')
-sys.path.append (gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# See the big header comment at the start of gen_fdl.py to see how the stages
-# fit together.  The big difference here is that *all* of the C code is in the
-# template file as labelled fragments, instead of as Python strings.  That
-# makes it much easier to edit in one place, with proper syntax highlighting
-# and indentation.
-#
-#   Stage 1 uses type-specific fragments to generate FUNCTION_BODY, instead of
-#   LEN_*_TEMPLATE and SERLZ_*_TEMPLATE to generate LEN_CODE and SER_CODE.
-#
-#   Stage 2 uses the FOP and CASE fragments instead of RECON_TEMPLATE and
-#   FOP_TEMPLATE.  The expanded FOP code (including FUNCTION_BODY substitution
-#   in the middle of each function) is emitted immediately; the expanded CASE
-#   code is saved for the next stage.
-#
-#   Stage 3 uses the PROLOG and EPILOG fragments, with the expanded CASE code
-#   in the middle of EPILOG, to generate the whole output file.
-#
-# Another way of looking at it is to consider how the fragments appear in
-# the final output:
-#
-#   PROLOG
-#   FOP (expanded for CREATE)
-#       FOP before FUNCTION_BODY
-#       LOC, INTEGER, GFID, etc. (one per arg, by type)
-#       FOP after FUNCTION_BODY
-#   FOP (expanded for WRITEV)
-#       FOP before FUNCTION_BODY
-#       GFID, VECTOR, etc. (one per arg, by type)
-#       FOP after FUNCTION_BODY
-#   (more FOPs)
-#   EPILOG
-#       EPILOG before CASE
-#       CASE statements (one per fop)
-#       EPILOG after CASE
-
-typemap = {
-	'dict_t *':				"DICT",
-	'fd_t *':				"FD",
-	'dev_t':				"DOUBLE",
-	'gf_xattrop_flags_t':	"INTEGER",
-	'int32_t':				"INTEGER",
-	'mode_t':				"INTEGER",
-	'off_t':				"DOUBLE",
-	'size_t':				"DOUBLE",
-	'uint32_t':				"INTEGER",
-	'loc_t *':				"LOC",
-	'const char *':			"STRING",
-	'struct iovec *':		"VECTOR",
-	'struct iatt *':		"IATT",
-	'struct iobref *':		"IOBREF",
-}
-
-def get_special_subs (name, args, fop_type):
-	code = ""
-	cleanups = ""
-	links = ""
-	s_args = []
-	for arg in args:
-		if arg[0] == 'extra':
-			code += "\t%s %s;\n\n" % (arg[2], arg[1])
-			s_args.append(arg[3])
-			continue
-		if arg[0] == 'link':
-			links += fragments["LINK"].replace("@INODE_ARG@",arg[1])	\
-									  .replace("@IATT_ARG@",arg[2])
-			continue
-		if arg[0] != 'fop-arg':
-			continue
-		if (name, arg[1]) == ('writev', 'count'):
-			# Special case: just skip this.  We can't mark it as 'nosync'
-			# because of the way the translator and dumper generators look for
-			# that after 'stub-name' which we don't define.  Instead of adding a
-			# bunch of generic infrastructure for this one case, just pound it
-			# here.
-			continue
-		recon_type = typemap[arg[2]]
-		# print "/* %s.%s => %s (%s)*/" % (name, arg[1], recon_type, fop_type)
-		if (name == "create") and (arg[1] == "fd"):
-			# Special case: fd for create is new, not looked up.
-			# print "/* change to NEW_FD */"
-			recon_type = "NEW_FD"
-		elif (recon_type == "LOC") and (fop_type == "entry-op"):
-			# Need to treat this differently for inode vs. entry ops.
-			# Special case: link source is treated like inode-op.
-			if (name != "link") or (arg[1] != "oldloc"):
-				# print "/* change to PARENT_LOC */"
-				recon_type = "PARENT_LOC"
-		code += fragments[recon_type].replace("@ARGNAME@",arg[1])		\
-									 .replace("@ARGTYPE@",arg[2])
-		cleanup_key = recon_type + "_CLEANUP"
-		if fragments.has_key(cleanup_key):
-			new_frag = fragments[cleanup_key].replace("@ARGNAME@",arg[1])
-			# Make sure these get added in *reverse* order.  Otherwise, a
-			# failure for an earlier argument might goto a label that falls
-			# through to the cleanup code for a variable associated with a
-			# later argument, but that variable might not even have been
-			# *declared* (let alone initialized) yet.  Consider the following
-			# case.
-			#
-			#         process argument A (on failure goto cleanup_A)
-			#         set error label to cleanup_A
-			#
-			#         declare pointer variable for argument B
-			#         process argument B (on failure goto cleanup_B)
-			#
-			#     cleanup_A:
-			#         /* whatever */
-			#     cleanup_B:
-			#         free pointer variable <= "USED BUT NOT SET" error here
-			#
-			# By adding these in reverse order, we ensure that cleanup_B is
-			# actually *before* cleanup_A, and nothing will try to do the free
-			# until we've actually attempted processing of B.
-			cleanups = new_frag + cleanups
-		if 'nosync' in arg[4:]:
-			code += "\t(void)%s;\n" % arg[1];
-			continue
-		if arg[2] in ("loc_t *", "struct iatt *"):
-			# These are passed as pointers to the syncop, but they're actual
-			# structures in the generated code.
-			s_args.append("&"+arg[1]);
-		else:
-			s_args.append(arg[1])
-	# We have to handle a couple of special cases here, because some n00b
-	# defined the syncops with a different argument order than the fops they're
-	# based on.
-	if name == 'writev':
-		# Swap 'flags' and 'iobref'.  Also, we need to add the iov count, which
-		# is not stored in or read from the journal.  There are other ways to
-		# do that, but this is the only place we need anything similar and we
-		# already have to treat it as a special case so this is simplest.
-		s_args_str = 'fd, &vector, 1, off, iobref, flags, xdata'
-	elif name == 'symlink':
-		# Swap 'linkpath' and 'loc'.
-		s_args_str = '&loc, linkpath, &iatt, xdata'
-	else:
-		s_args_str = string.join (s_args, ", ")
-	return code, links, s_args_str, cleanups
-
-# TBD: probably need to generate type-specific cleanup code as well - e.g.
-# fd_unref for an fd_t, loc_wipe for a loc_t, and so on.  All of these
-# generated CLEANUP fragments will go at the end of the function, with goto
-# labels.  Meanwhile, the error-checking part of each type-specific fragment
-# (e.g. LOC or FD) will need to update the indirect label that we jump to when
-# an error is detected.  This will probably get messy.
-def gen_functions ():
-	code = ""
-	for name, value in ops.iteritems():
-		fop_type = [ x[1] for x in value if x[0] == "journal" ]
-		if not fop_type:
-			continue
-		body, links, syncop_args, cleanups = get_special_subs (name, value,
-															   fop_type[0])
-		fop_subs[name]["@FUNCTION_BODY@"] = body
-		fop_subs[name]["@LINKS@"] = links
-		fop_subs[name]["@SYNCOP_ARGS@"] = syncop_args
-		fop_subs[name]["@CLEANUPS@"] = cleanups
-		if name == "writev":
-			# Take advantage of the fact that, *during reconciliation*, the
-			# vector is always a single element.  In normal I/O it's not.
-			fop_subs[name]["@SUCCESS_VALUE@"] = "vector.iov_len"
-		else:
-			fop_subs[name]["@SUCCESS_VALUE@"] = "GFAPI_SUCCESS"
-		# Print the FOP fragment with @FUNCTION_BODY@ in the middle.
-		code += generate(fragments["FOP"],name,fop_subs)
-	return code
-
-def gen_cases ():
-	code = ""
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-		# Add the CASE fragment for this fop.
-		code += generate(fragments["CASE"],name,fop_subs)
-	return code
-
-def load_fragments (path="recon-tmpl.c"):
-	pragma_re = re.compile('pragma fragment (.*)')
-	cur_symbol = None
-	cur_value = ""
-	result = {}
-	for line in open(path,"r").readlines():
-		m = pragma_re.search(line)
-		if m:
-			if cur_symbol:
-				result[cur_symbol] = cur_value
-			cur_symbol = m.group(1)
-			cur_value = ""
-		else:
-			cur_value += line
-	if cur_symbol:
-		result[cur_symbol] = cur_value
-	return result
-
-if __name__ == "__main__":
-	fragments = load_fragments(sys.argv[1])
-	print "/* BEGIN GENERATED CODE - DO NOT MODIFY */"
-	print fragments["PROLOG"]
-	print gen_functions()
-	print fragments["EPILOG"].replace("@SWITCH_BODY@",gen_cases())
-	print "/* END GENERATED CODE */"
diff --git a/xlators/experimental/fdl/src/jnl-types.h b/xlators/experimental/fdl/src/jnl-types.h
deleted file mode 100644
index 8cb39d01a25..00000000000
--- a/xlators/experimental/fdl/src/jnl-types.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#define NEW_REQUEST     (uint8_t)'N'
-
-typedef struct {
-        uint8_t         event_type;     /* e.g. NEW_REQUEST */
-        uint8_t         fop_type;       /* e.g. GF_FOP_SETATTR */
-        uint16_t        request_id;
-        uint32_t        ext_length;
-} event_header_t;
-
-enum {
-        FDL_IPC_BASE = 0xfeedbee5,       /* ... and they make honey */
-        FDL_IPC_CHANGE_TERM,
-        FDL_IPC_GET_TERMS,
-};
diff --git a/xlators/experimental/fdl/src/logdump.c b/xlators/experimental/fdl/src/logdump.c
deleted file mode 100644
index 7c979c32a04..00000000000
--- a/xlators/experimental/fdl/src/logdump.c
+++ /dev/null
@@ -1,50 +0,0 @@
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-extern int fdl_dump (char **, char **);
-
-int
-main (int argc, char **argv)
-{
-        int     meta_fd         = (-1);
-        char    *meta_buf       = NULL;
-        int     data_fd         = (-1);
-        char    *data_buf       = NULL;
-
-        meta_fd = open (argv[1], O_RDONLY);
-        if (meta_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        meta_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, meta_fd, 0);
-        if (meta_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        data_fd = open (argv[2], O_RDONLY);
-        if (data_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        data_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, data_fd, 0);
-        if (data_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        for (;;) {
-                if (!fdl_dump(&meta_buf,&data_buf)) {
-                        break;
-                }
-        }
-
-        return EXIT_SUCCESS;
-}
diff --git a/xlators/experimental/fdl/src/recon-tmpl.c b/xlators/experimental/fdl/src/recon-tmpl.c
deleted file mode 100644
index 523bda39418..00000000000
--- a/xlators/experimental/fdl/src/recon-tmpl.c
+++ /dev/null
@@ -1,305 +0,0 @@
-#pragma fragment PROLOG
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "fd.h"
-#include "iatt.h"
-#include "syncop.h"
-#include "xlator.h"
-#include "glfs-internal.h"
-
-#include "jnl-types.h"
-
-#define GFAPI_SUCCESS 0
-
-inode_t *
-recon_get_inode (glfs_t *fs, uuid_t gfid)
-{
-        inode_t         *inode;
-        loc_t           loc     = {NULL,};
-        struct iatt     iatt;
-        int             ret;
-        inode_t         *newinode;
-
-        inode = inode_find (fs->active_subvol->itable, gfid);
-        if (inode) {
-                printf ("=== FOUND %s IN TABLE\n", uuid_utoa(gfid));
-                return inode;
-        }
-
-        loc.inode = inode_new (fs->active_subvol->itable);
-        if (!loc.inode) {
-                return NULL;
-        }
-        gf_uuid_copy (loc.inode->gfid, gfid);
-        gf_uuid_copy (loc.gfid, gfid);
-
-        printf ("=== DOING LOOKUP FOR %s\n", uuid_utoa(gfid));
-
-        ret = syncop_lookup (fs->active_subvol, &loc, &iatt,
-                             NULL, NULL, NULL);
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "syncop_lookup failed (%d)\n", ret);
-                return NULL;
-        }
-
-        newinode = inode_link (loc.inode, NULL, NULL, &iatt);
-        if (newinode) {
-                inode_lookup (newinode);
-        }
-
-        return newinode;
-}
-
-#pragma fragment DICT
-        dict_t  *@ARGNAME@;
-
-        @ARGNAME@ = dict_new();
-        if (!@ARGNAME@) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-
-        {
-                int     key_len, data_len;
-                char    *key_ptr;
-                int     garbage;
-                for (;;) {
-                        key_len = *((int *)new_meta);
-                        new_meta += sizeof(int);
-                        if (!key_len) {
-                                break;
-                        }
-                        key_ptr = new_meta;
-                        new_meta += key_len;
-                        data_len = *((int *)new_meta);
-                        new_meta += sizeof(int);
-                        garbage = dict_set_static_bin (@ARGNAME@, key_ptr,
-                                                       new_meta, data_len);
-                        /* TBD: check error from dict_set_static_bin */
-                        (void)garbage;
-                        new_meta += data_len;
-                }
-        }
-
-#pragma fragment DICT_CLEANUP
-cleanup_@ARGNAME@:
-        dict_unref (@ARGNAME@);
-
-#pragma fragment DOUBLE
-        @ARGTYPE@       @ARGNAME@       = *((@ARGTYPE@ *)new_meta);
-        new_meta += sizeof(uint64_t);
-
-#pragma fragment FD
-        inode_t *@ARGNAME@_ino;
-        fd_t    *@ARGNAME@;
-
-        @ARGNAME@_ino = recon_get_inode (fs, *((uuid_t *)new_meta));
-        new_meta += 16;
-        if (!@ARGNAME@_ino) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@_ino;
-
-        @ARGNAME@ = fd_anonymous (@ARGNAME@_ino);
-        if (!@ARGNAME@) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-
-#pragma fragment FD_CLEANUP
-cleanup_@ARGNAME@:
-        fd_unref (@ARGNAME@);
-cleanup_@ARGNAME@_ino:
-        inode_unref (@ARGNAME@_ino);
-
-#pragma fragment NEW_FD
-        /*
-         * This pseudo-type is only used for create, and in that case we know
-         * we'll be using loc.inode, so it's not worth generalizing to take an
-         * extra argument.
-         */
-        fd_t    *@ARGNAME@      = fd_anonymous (loc.inode);
-
-        if (!fd) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-        new_meta += 16;
-
-#pragma fragment NEW_FD_CLEANUP
-cleanup_@ARGNAME@:
-        fd_unref (@ARGNAME@);
-
-#pragma fragment INTEGER
-        @ARGTYPE@       @ARGNAME@       = *((@ARGTYPE@ *)new_meta);
-
-        new_meta += sizeof(@ARGTYPE@);
-
-#pragma fragment LOC
-        loc_t           @ARGNAME@       = { NULL, };
-
-        @ARGNAME@.inode = recon_get_inode (fs, *((uuid_t *)new_meta));
-        if (!@ARGNAME@.inode) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-        gf_uuid_copy (@ARGNAME@.gfid, @ARGNAME@.inode->gfid);
-        new_meta += 16;
-        new_meta += 16; /* skip over pargfid */
-        if (*(new_meta++)) {
-                @ARGNAME@.name = new_meta;
-                new_meta += strlen(new_meta) + 1;
-        }
-
-#pragma fragment LOC_CLEANUP
-cleanup_@ARGNAME@:
-        loc_wipe (&@ARGNAME@);
-
-#pragma fragment PARENT_LOC
-        loc_t           @ARGNAME@       = { NULL, };
-
-        new_meta += 16; /* skip over gfid */
-        @ARGNAME@.parent = recon_get_inode (fs, *((uuid_t *)new_meta));
-        if (!@ARGNAME@.parent) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-        gf_uuid_copy (@ARGNAME@.pargfid, @ARGNAME@.parent->gfid);
-        new_meta += 16;
-        if (!*(new_meta++)) {
-                goto *err_label;
-        }
-        @ARGNAME@.name = new_meta;
-        new_meta += strlen(new_meta) + 1;
-
-        @ARGNAME@.inode = inode_new (fs->active_subvol->itable);
-        if (!@ARGNAME@.inode) {
-                goto *err_label;
-        }
-
-#pragma fragment PARENT_LOC_CLEANUP
-cleanup_@ARGNAME@:
-        loc_wipe (&@ARGNAME@);
-
-#pragma fragment STRING
-        char    *@ARGNAME@;
-        if (*(new_meta++)) {
-                @ARGNAME@ = new_meta;
-                new_meta += (strlen(new_meta) + 1);
-        }
-        else {
-                goto *err_label;
-        }
-
-#pragma fragment VECTOR
-        struct iovec    @ARGNAME@;
-
-        @ARGNAME@.iov_len = *((size_t *)new_meta);
-        new_meta += sizeof(@ARGNAME@.iov_len);
-        @ARGNAME@.iov_base = new_data;
-        new_data += @ARGNAME@.iov_len;
-
-#pragma fragment IATT
-        struct iatt     @ARGNAME@;
-        {
-                @ARGNAME@.ia_prot = *((ia_prot_t *)new_meta);
-                new_meta += sizeof(ia_prot_t);
-                uint32_t *myints = (uint32_t *)new_meta;
-                @ARGNAME@.ia_uid = myints[0];
-                @ARGNAME@.ia_gid = myints[1];
-                @ARGNAME@.ia_atime = myints[2];
-                @ARGNAME@.ia_atime_nsec = myints[3];
-                @ARGNAME@.ia_mtime = myints[4];
-                @ARGNAME@.ia_mtime_nsec = myints[5];
-                new_meta += sizeof(*myints) * 6;
-        }
-
-#pragma fragment IOBREF
-        struct iobref   *@ARGNAME@;
-
-        @ARGNAME@ = iobref_new();
-        if (!@ARGNAME@) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-
-#pragma fragment IOBREF_CLEANUP
-cleanup_@ARGNAME@:
-        iobref_unref (@ARGNAME@);
-
-#pragma fragment LINK
-        /* TBD: check error */
-        inode_t *new_inode = inode_link (@INODE_ARG@, NULL, NULL, @IATT_ARG@);
-        if (new_inode) {
-                inode_lookup (new_inode);
-        }
-
-#pragma fragment FOP
-int
-fdl_replay_@NAME@ (glfs_t *fs, char **old_meta, char **old_data)
-{
-        char    *new_meta	= *old_meta;
-        char	*new_data	= *old_data;
-        int     ret;
-        int     status          = 0xbad;
-        void    *err_label      = &&done;
-
-@FUNCTION_BODY@
-
-        ret = syncop_@NAME@ (fs->active_subvol, @SYNCOP_ARGS@, NULL);
-        if (ret != @SUCCESS_VALUE@) {
-                fprintf (stderr, "syncop_@NAME@ returned %d", ret);
-                goto *err_label;
-        }
-
-@LINKS@
-
-        status = 0;
-
-@CLEANUPS@
-
-done:
-        *old_meta = new_meta;
-        *old_data = new_data;
-        return status;
-}
-
-#pragma fragment CASE
-        case GF_FOP_@UPNAME@:
-                printf ("=== GF_FOP_@UPNAME@\n");
-                if (fdl_replay_@NAME@ (fs, &new_meta, &new_data) != 0) {
-                        goto done;
-                }
-                recognized = 1;
-                break;
-
-#pragma fragment EPILOG
-int
-recon_execute (glfs_t *fs, char **old_meta, char **old_data)
-{
-        char            *new_meta       = *old_meta;
-        char            *new_data       = *old_data;
-        int             recognized      = 0;
-        event_header_t  *eh;
-
-        eh = (event_header_t *)new_meta;
-        new_meta += sizeof (*eh);
-
-        /* TBD: check event_type instead of assuming NEW_REQUEST */
-
-        switch (eh->fop_type) {
-@SWITCH_BODY@
-
-        default:
-                printf ("unknown fop %u\n", eh->fop_type);
-        }
-
-done:
-        *old_meta = new_meta;
-        *old_data = new_data;
-        return recognized;
-}
diff --git a/xlators/experimental/fdl/src/recon.c b/xlators/experimental/fdl/src/recon.c
deleted file mode 100644
index 14168a011e0..00000000000
--- a/xlators/experimental/fdl/src/recon.c
+++ /dev/null
@@ -1,89 +0,0 @@
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-#include "glusterfs.h"
-#include "fd.h"
-#include "syncop.h"
-#include "glfs-internal.h"
-
-#define GFAPI_SUCCESS 0
-
-extern int recon_execute (glfs_t *, char **, char **);
-
-int
-main (int argc, char **argv)
-{
-        glfs_t  *fs;
-        int     ret;
-        int     meta_fd         = (-1);
-        char    *meta_buf       = NULL;
-        int     data_fd         = (-1);
-        char    *data_buf       = NULL;
-
-        fs = glfs_new ("whocares");
-        if (!fs) {
-                fprintf (stderr, "glfs_new failed\n");
-                return EXIT_FAILURE;
-        }
-
-        if (getenv("RECON_DEBUG")) {
-                ret = glfs_set_logging (fs, "/dev/stderr", 7);
-        }
-        else {
-                ret = glfs_set_logging (fs, "/dev/null", 0);
-        }
-
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "glfs_set_logging failed (%d)\n", errno);
-                return EXIT_FAILURE;
-        }
-
-        ret = glfs_set_volfile (fs, argv[1]);
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "glfs_set_volfile failed (%d)\n", errno);
-                return EXIT_FAILURE;
-        }
-
-        ret = glfs_init (fs);
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "glfs_init failed (%d)\n", errno);
-                return EXIT_FAILURE;
-        }
-
-        meta_fd = open (argv[2], O_RDONLY);
-        if (meta_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        meta_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, meta_fd, 0);
-        if (meta_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        data_fd = open (argv[3], O_RDONLY);
-        if (data_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        data_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, data_fd, 0);
-        if (data_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        for (;;) {
-                if (!recon_execute(fs,&meta_buf,&data_buf)) {
-                        break;
-                }
-        }
-
-        return EXIT_SUCCESS;
-}
diff --git a/xlators/experimental/jbr-client/src/Makefile.am b/xlators/experimental/jbr-client/src/Makefile.am
deleted file mode 100644
index 58f399f0607..00000000000
--- a/xlators/experimental/jbr-client/src/Makefile.am
+++ /dev/null
@@ -1,32 +0,0 @@
-xlator_LTLIBRARIES = jbrc.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-
-nodist_jbrc_la_SOURCES = jbrc-cg.c
-CLEANFILES = $(nodist_jbrc_la_SOURCES)
-
-jbrc_la_LDFLAGS = -module -avoid-version
-jbrc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = \
-	 $(top_srcdir)/xlators/lib/src/libxlator.h \
-	 $(top_srcdir)/glusterfsd/src/glusterfsd.h \
-	 jbrc.h jbr-messages.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-JBRC_PREFIX	= $(top_srcdir)/xlators/experimental/jbr-client/src
-JBRC_GEN_FOPS	= $(JBRC_PREFIX)/gen-fops.py
-JBRC_TEMPLATES	= $(JBRC_PREFIX)/fop-template.c
-JBRC_WRAPPER	= $(JBRC_PREFIX)/jbrc.c
-noinst_PYTHON	= $(JBRC_GEN_FOPS)
-EXTRA_DIST	= $(JBRC_TEMPLATES) $(JBRC_WRAPPER)
-
-jbrc-cg.c: $(JBRC_GEN_FOPS) $(JBRC_TEMPLATES) $(JBRC_WRAPPER)
-	$(PYTHON) $(JBRC_GEN_FOPS) $(JBRC_TEMPLATES) $(JBRC_WRAPPER) > $@
-
-uninstall-local:
-	rm -f $(DESTDIR)$(xlatordir)/jbr.so
diff --git a/xlators/experimental/jbr-client/src/fop-template.c b/xlators/experimental/jbr-client/src/fop-template.c
deleted file mode 100644
index 7719f511f01..00000000000
--- a/xlators/experimental/jbr-client/src/fop-template.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/* template-name fop */
-int32_t
-jbrc_@NAME@ (call_frame_t *frame, xlator_t *this,
-             @LONG_ARGS@)
-{
-        jbrc_local_t    *local          = NULL;
-        xlator_t        *target_xl      = ACTIVE_CHILD(this);
-
-        local = mem_get(this->local_pool);
-        if (!local) {
-                goto err;
-        }
-
-        local->stub = fop_@NAME@_stub (frame, jbrc_@NAME@_continue,
-                                       @SHORT_ARGS@);
-        if (!local->stub) {
-                goto err;
-        }
-        local->curr_xl = target_xl;
-        local->scars = 0;
-
-        frame->local = local;
-        STACK_WIND_COOKIE (frame, jbrc_@NAME@_cbk, target_xl,
-                    target_xl, target_xl->fops->@NAME@,
-                    @SHORT_ARGS@);
-        return 0;
-
-err:
-        if (local) {
-                mem_put(local);
-        }
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, ENOMEM,
-                             @ERROR_ARGS@);
-        return 0;
-}
-
-/* template-name cbk */
-int32_t
-jbrc_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 @LONG_ARGS@)
-{
-        jbrc_local_t    *local          = frame->local;
-        xlator_t        *last_xl        = cookie;
-        xlator_t        *next_xl;
-        jbrc_private_t  *priv           = this->private;
-        struct timespec spec;
-
-        if (op_ret != (-1)) {
-                if (local->scars) {
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_RETRY_MSG,
-                                HILITE("retried %p OK"), frame->local);
-                }
-                priv->active = last_xl;
-                goto unwind;
-        }
-        if ((op_errno != EREMOTE) && (op_errno != ENOTCONN)) {
-                goto unwind;
-        }
-
-        /* TBD: get leader ID from xdata? */
-        next_xl = next_xlator(this, last_xl);
-        /*
-         * We can't just give up after we've tried all bricks, because it's
-         * quite likely that a new leader election just hasn't finished yet.
-         * We also shouldn't retry endlessly, and especially not at a high
-         * rate, but that's good enough while we work on other things.
-         *
-         * TBD: implement slow/finite retry via a worker thread
-         */
-        if (!next_xl || (local->scars >= SCAR_LIMIT)) {
-                gf_msg (this->name, GF_LOG_DEBUG, 0, J_MSG_RETRY_MSG,
-                        HILITE("ran out of retries for %p"), frame->local);
-                goto unwind;
-        }
-
-        local->curr_xl = next_xl;
-        local->scars += 1;
-        spec.tv_sec = 1;
-        spec.tv_nsec = 0;
-        /*
-         * WARNING
-         *
-         * Just calling gf_timer_call_after like this leaves open the
-         * possibility that writes will get reordered, if a first write is
-         * rescheduled and then a second comes along to find an updated
-         * priv->active before the first actually executes.  We might need to
-         * implement a stricter (and more complicated) queuing mechanism to
-         * ensure absolute consistency in this case.
-         */
-        if (gf_timer_call_after(this->ctx, spec, jbrc_retry_cb, local)) {
-                return 0;
-        }
-
-unwind:
-        call_stub_destroy(local->stub);
-        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
-                                     @SHORT_ARGS@);
-        return 0;
-}
-
-/* template-name cont-func */
-int32_t
-jbrc_@NAME@_continue (call_frame_t *frame, xlator_t *this,
-                      @LONG_ARGS@)
-{
-        jbrc_local_t    *local  = frame->local;
-
-        STACK_WIND_COOKIE (frame, jbrc_@NAME@_cbk, local->curr_xl,
-                           local->curr_xl, local->curr_xl->fops->@NAME@,
-                           @SHORT_ARGS@);
-        return 0;
-}
diff --git a/xlators/experimental/jbr-client/src/gen-fops.py b/xlators/experimental/jbr-client/src/gen-fops.py
deleted file mode 100755
index 4d9451f7177..00000000000
--- a/xlators/experimental/jbr-client/src/gen-fops.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/python
-
-import os
-import re
-import string
-import sys
-
-curdir = os.path.dirname(sys.argv[0])
-gendir = os.path.join(curdir,'../../../../libglusterfs/src')
-sys.path.append(gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# We really want the callback argument list, even when we're generating fop
-# code, so we propagate here.
-# TBD: this should probably be right in generate.py
-for k, v in cbk_subs.iteritems():
-	fop_subs[k]['@ERROR_ARGS@'] = v['@ERROR_ARGS@']
-
-# Stolen from old codegen.py
-def load_templates (path):
-	templates = {}
-	tmpl_re = re.compile("/\* template-name (.*) \*/")
-	templates = {}
-	t_name = None
-	for line in open(path,"r").readlines():
-		if not line:
-			break
-		m = tmpl_re.match(line)
-		if m:
-			if t_name:
-				templates[t_name] = string.join(t_contents,'')
-			t_name = m.group(1).strip()
-			t_contents = []
-		elif t_name:
-			t_contents.append(line)
-	if t_name:
-		templates[t_name] = string.join(t_contents,'')
-	return templates
-
-# Stolen from gen_fdl.py
-def gen_client (templates):
-	for name, value in ops.iteritems():
-		if name == 'getspec':
-			# It's not real if it doesn't have a stub function.
-			continue
-		print generate(templates['cbk'],name,cbk_subs)
-		print generate(templates['cont-func'],name,fop_subs)
-		print generate(templates['fop'],name,fop_subs)
-
-tmpl = load_templates(sys.argv[1])
-for l in open(sys.argv[2],'r').readlines():
-	if l.find('#pragma generate') != -1:
-		print "/* BEGIN GENERATED CODE - DO NOT MODIFY */"
-		gen_client(tmpl)
-		print "/* END GENERATED CODE */"
-	else:
-		print l[:-1]
diff --git a/xlators/experimental/jbr-client/src/jbr-messages.h b/xlators/experimental/jbr-client/src/jbr-messages.h
deleted file mode 100644
index 626c4fd3eaa..00000000000
--- a/xlators/experimental/jbr-client/src/jbr-messages.h
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _JBR_MESSAGES_H_
-#define _JBR_MESSAGES_H_
-
-#include "glfs-message-id.h"
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define JBR_COMP_BASE           GLFS_MSGID_COMP_JBR
-#define GLFS_NUM_MESSAGES       1
-#define GLFS_MSGID_END          (JBR_COMP_BASE + GLFS_NUM_MESSAGES + 1)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_INIT_FAIL                      (JBR_COMP_BASE + 1)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_RETRY_MSG                      (JBR_COMP_BASE + 2)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_MEM_ERR                        (JBR_COMP_BASE + 3)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_DICT_FLR                       (JBR_COMP_BASE + 4)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_GENERIC                        (JBR_COMP_BASE + 5)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_INVALID                        (JBR_COMP_BASE + 6)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_NO_DATA                        (JBR_COMP_BASE + 7)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_SYS_CALL_FAILURE               (JBR_COMP_BASE + 8)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_QUORUM_NOT_MET                 (JBR_COMP_BASE + 9)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define J_MSG_LOCK_FAILURE                  (JBR_COMP_BASE + 10)
-
-
-#endif /* _JBR_MESSAGES_H_ */
diff --git a/xlators/experimental/jbr-client/src/jbrc.c b/xlators/experimental/jbr-client/src/jbrc.c
deleted file mode 100644
index 9bb9346c5c0..00000000000
--- a/xlators/experimental/jbr-client/src/jbrc.c
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "call-stub.h"
-#include "defaults.h"
-#include "timer.h"
-#include "xlator.h"
-#include "jbr-messages.h"
-#include "jbrc.h"
-#include "statedump.h"
-
-#define SCAR_LIMIT      20
-#define HILITE(x)       (""x"")
-
-/*
- * The fops are actually generated by gen-fops.py; the rest was mostly copied
- * from defaults.c (commit cd253754 on 27 August 2013).
- */
-
-enum gf_dht_mem_types_ {
-        gf_mt_jbrc_private_t = gf_common_mt_end + 1,
-        gf_mt_jbrc_end
-};
-
-char            *JBRC_XATTR     = "user.jbr.active";
-
-static inline
-xlator_t *
-ACTIVE_CHILD (xlator_t *parent)
-{
-        jbrc_private_t  *priv   = parent->private;
-
-        return priv ? priv->active : FIRST_CHILD(parent);
-}
-
-xlator_t *
-next_xlator (xlator_t *this, xlator_t *prev)
-{
-        xlator_list_t   *trav;
-
-        for (trav = this->children; trav; trav = trav->next) {
-                if (trav->xlator == prev) {
-                        return trav->next ? trav->next->xlator
-                                          : this->children->xlator;
-                }
-        }
-
-        return NULL;
-}
-
-void
-jbrc_retry_cb (void *cb_arg)
-{
-        jbrc_local_t    *local  = cb_arg;
-
-        gf_msg (__func__, GF_LOG_INFO, 0, J_MSG_RETRY_MSG,
-                HILITE("retrying %p"), local);
-        call_resume_wind(local->stub);
-}
-
-#pragma generate
-
-int32_t
-jbrc_forget (xlator_t *this, inode_t *inode)
-{
-        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL,
-                          "xlator does not implement forget_cbk");
-        return 0;
-}
-
-
-int32_t
-jbrc_releasedir (xlator_t *this, fd_t *fd)
-{
-        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL,
-                          "xlator does not implement releasedir_cbk");
-        return 0;
-}
-
-int32_t
-jbrc_release (xlator_t *this, fd_t *fd)
-{
-        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL,
-                          "xlator does not implement release_cbk");
-        return 0;
-}
-
-struct xlator_fops fops = {
-        .lookup         = jbrc_lookup,
-        .stat           = jbrc_stat,
-        .fstat          = jbrc_fstat,
-        .truncate       = jbrc_truncate,
-        .ftruncate      = jbrc_ftruncate,
-        .access         = jbrc_access,
-        .readlink       = jbrc_readlink,
-        .mknod          = jbrc_mknod,
-        .mkdir          = jbrc_mkdir,
-        .unlink         = jbrc_unlink,
-        .rmdir          = jbrc_rmdir,
-        .symlink        = jbrc_symlink,
-        .rename         = jbrc_rename,
-        .link           = jbrc_link,
-        .create         = jbrc_create,
-        .open           = jbrc_open,
-        .readv          = jbrc_readv,
-        .writev         = jbrc_writev,
-        .flush          = jbrc_flush,
-        .fsync          = jbrc_fsync,
-        .opendir        = jbrc_opendir,
-        .readdir        = jbrc_readdir,
-        .readdirp       = jbrc_readdirp,
-        .fsyncdir       = jbrc_fsyncdir,
-        .statfs         = jbrc_statfs,
-        .setxattr       = jbrc_setxattr,
-        .getxattr       = jbrc_getxattr,
-        .fsetxattr      = jbrc_fsetxattr,
-        .fgetxattr      = jbrc_fgetxattr,
-        .removexattr    = jbrc_removexattr,
-        .fremovexattr   = jbrc_fremovexattr,
-        .lk             = jbrc_lk,
-        .inodelk        = jbrc_inodelk,
-        .finodelk       = jbrc_finodelk,
-        .entrylk        = jbrc_entrylk,
-        .fentrylk       = jbrc_fentrylk,
-        .rchecksum      = jbrc_rchecksum,
-        .xattrop        = jbrc_xattrop,
-        .fxattrop       = jbrc_fxattrop,
-        .setattr        = jbrc_setattr,
-        .fsetattr       = jbrc_fsetattr,
-	.fallocate	= jbrc_fallocate,
-	.discard        = jbrc_discard,
-};
-
-struct xlator_cbks cbks = {
-};
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbrc", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_mt_jbrc_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, J_MSG_MEM_ERR,
-                        "Memory accounting init failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-int32_t
-jbrc_init (xlator_t *this)
-{
-        jbrc_private_t  *priv   = NULL;
-        xlator_list_t   *trav   = NULL;
-
-        this->local_pool = mem_pool_new (jbrc_local_t, 128);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, J_MSG_MEM_ERR,
-                        "failed to create jbrc_local_t pool");
-                goto err;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_mt_jbrc_private_t);
-        if (!priv) {
-                goto err;
-        }
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++(priv->n_children);
-        }
-
-        priv->active = FIRST_CHILD(this);
-        this->private = priv;
-        return 0;
-
-err:
-        if (priv) {
-                GF_FREE(priv);
-        }
-        return -1;
-}
-
-void
-jbrc_fini (xlator_t *this)
-{
-        GF_FREE(this->private);
-}
-
-int
-jbrc_get_child_index (xlator_t *this, xlator_t *kid)
-{
-        xlator_list_t   *trav;
-        int             retval = -1;
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++retval;
-                if (trav->xlator == kid) {
-                        return retval;
-                }
-        }
-
-        return -1;
-}
-
-uint8_t
-jbrc_count_up_kids (jbrc_private_t *priv)
-{
-        uint8_t         retval  = 0;
-        uint8_t         i;
-
-        for (i = 0; i < priv->n_children; ++i) {
-                if (priv->kid_state & (1 << i)) {
-                        ++retval;
-                }
-        }
-
-        return retval;
-}
-
-int32_t
-jbrc_notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int32_t           ret        = 0;
-        int32_t           index      = 0;
-        jbrc_private_t   *priv       = NULL;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        switch (event) {
-        case GF_EVENT_CHILD_UP:
-                index = jbrc_get_child_index(this, data);
-                if (index >= 0) {
-                        priv->kid_state |= (1 << index);
-                        priv->up_children = jbrc_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_UP for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                }
-                ret = default_notify (this, event, data);
-                break;
-        case GF_EVENT_CHILD_DOWN:
-                index = jbrc_get_child_index(this, data);
-                if (index >= 0) {
-                        priv->kid_state &= ~(1 << index);
-                        priv->up_children = jbrc_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_DOWN for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                }
-                break;
-        default:
-                ret = default_notify (this, event, data);
-        }
-
-out:
-        return ret;
-}
-
-int
-jbrc_priv_dump (xlator_t *this)
-{
-        jbrc_private_t     *priv = NULL;
-        char                key_prefix[GF_DUMP_MAX_BUF_LEN];
-        xlator_list_t      *trav = NULL;
-        int32_t             i    = -1;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s",
-                 this->type, this->name);
-        gf_proc_dump_add_section(key_prefix);
-
-        gf_proc_dump_write("up_children", "%u", priv->up_children);
-
-        for (trav = this->children, i = 0; trav; trav = trav->next, i++) {
-                snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "child_%d", i);
-                gf_proc_dump_write(key_prefix, "%s", trav->xlator->name);
-        }
-
-out:
-        return 0;
-}
-
-struct xlator_dumpops dumpops = {
-        .priv       = jbrc_priv_dump,
-};
-
-class_methods_t class_methods = {
-        .init           = jbrc_init,
-        .fini           = jbrc_fini,
-        .notify         = jbrc_notify,
-};
-
-struct volume_options options[] = {
-	{ .key = {NULL} },
-};
diff --git a/xlators/experimental/jbr-client/src/jbrc.h b/xlators/experimental/jbr-client/src/jbrc.h
deleted file mode 100644
index c83259ca1bd..00000000000
--- a/xlators/experimental/jbr-client/src/jbrc.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _JBRC_H_
-#define _JBRC_H_
-
-typedef struct {
-        xlator_t        *active;
-        uint8_t          up_children;
-        uint8_t          n_children;
-        uint32_t         kid_state;
-} jbrc_private_t;
-
-typedef struct {
-        call_stub_t     *stub;
-        xlator_t        *curr_xl;
-        uint16_t        scars;
-} jbrc_local_t;
-
-#endif /* _JBRC_H_ */
diff --git a/xlators/experimental/jbr-server/src/Makefile.am b/xlators/experimental/jbr-server/src/Makefile.am
deleted file mode 100644
index 66f73ba8c96..00000000000
--- a/xlators/experimental/jbr-server/src/Makefile.am
+++ /dev/null
@@ -1,35 +0,0 @@
-xlator_LTLIBRARIES = jbr.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-
-nodist_jbr_la_SOURCES = jbr-cg.c
-CLEANFILES = $(nodist_jbr_la_SOURCES)
-
-jbr_la_LDFLAGS = -module -avoid-version
-jbr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-		$(top_builddir)/api/src/libgfapi.la
-
-noinst_HEADERS = jbr-internal.h \
-	$(top_srcdir)/xlators/lib/src/libxlator.h \
-	$(top_srcdir)/glusterfsd/src/glusterfsd.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src -DSBIN_DIR=\"$(sbindir)\" \
-	-I$(top_srcdir)/api/src -DJBR_SCRIPT_PREFIX=\"$(jbrdir)\" \
-        -I$(top_srcdir)/xlators/experimental/jbr-client/src/
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-JBR_PREFIX	= $(top_srcdir)/xlators/experimental/jbr-server/src
-JBR_GEN_FOPS	= $(JBR_PREFIX)/gen-fops.py
-JBR_TEMPLATES	= $(JBR_PREFIX)/all-templates.c
-JBR_WRAPPER	= $(JBR_PREFIX)/jbr.c
-noinst_PYTHON	= $(JBR_GEN_FOPS)
-EXTRA_DIST	= $(JBR_TEMPLATES) $(JBR_WRAPPER)
-
-jbr-cg.c: $(JBR_GEN_FOPS) $(JBR_TEMPLATES) $(JBR_WRAPPER)
-	$(PYTHON) $(JBR_GEN_FOPS) $(JBR_TEMPLATES) $(JBR_WRAPPER) > $@
-
-uninstall-local:
-	rm -f $(DESTDIR)$(xlatordir)/jbr.so
diff --git a/xlators/experimental/jbr-server/src/all-templates.c b/xlators/experimental/jbr-server/src/all-templates.c
deleted file mode 100644
index 7314701029c..00000000000
--- a/xlators/experimental/jbr-server/src/all-templates.c
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * You can put anything here - it doesn't even have to be a comment - and it
- * will be ignored until we reach the first template-name comment.
- */
-
-
-/* template-name read-fop */
-int32_t
-jbr_@NAME@ (call_frame_t *frame, xlator_t *this,
-            @LONG_ARGS@)
-{
-        jbr_private_t   *priv     = NULL;
-        gf_boolean_t     in_recon = _gf_false;
-        int32_t          op_errno = 0;
-        int32_t          recon_term, recon_index;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-
-        op_errno = EREMOTE;
-
-        /* allow reads during reconciliation       *
-         * TBD: allow "dirty" reads on non-leaders *
-         */
-        if (xdata &&
-            (dict_get_int32(xdata, RECON_TERM_XATTR, &recon_term) == 0) &&
-            (dict_get_int32(xdata, RECON_INDEX_XATTR, &recon_index) == 0)) {
-                in_recon = _gf_true;
-        }
-
-        if ((!priv->leader) && (in_recon == _gf_false)) {
-                goto err;
-        }
-
-        STACK_WIND (frame, default_@NAME@_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                    @SHORT_ARGS@);
-        return 0;
-
-err:
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, op_errno,
-                             @ERROR_ARGS@);
-        return 0;
-}
-
-/* template-name read-perform_local_op */
-/* No "perform_local_op" function needed for @NAME@ */
-
-/* template-name read-dispatch */
-/* No "dispatch" function needed for @NAME@ */
-
-/* template-name read-call_dispatch */
-/* No "call_dispatch" function needed for @NAME@ */
-
-/* template-name read-fan-in */
-/* No "fan-in" function needed for @NAME@ */
-
-/* template-name read-continue */
-/* No "continue" function needed for @NAME@ */
-
-/* template-name read-complete */
-/* No "complete" function needed for @NAME@ */
-
-/* template-name write-fop */
-int32_t
-jbr_@NAME@ (call_frame_t *frame, xlator_t *this,
-            @LONG_ARGS@)
-{
-        jbr_local_t     *local         = NULL;
-        jbr_private_t   *priv          = NULL;
-        int32_t          ret           = -1;
-        int              op_errno      = ENOMEM;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-
-#if defined(JBR_CG_NEED_FD)
-        ret = jbr_leader_checks_and_init (frame, this, &op_errno, xdata, fd);
-#else
-        ret = jbr_leader_checks_and_init (frame, this, &op_errno, xdata, NULL);
-#endif
-        if (ret)
-                goto err;
-
-        local = frame->local;
-
-        /*
-         * If we let it through despite not being the leader, then we just want
-         * to pass it on down without all of the additional xattrs, queuing, and
-         * so on.  However, jbr_*_complete does depend on the initialization
-         * immediately above this.
-         */
-        if (!priv->leader) {
-                STACK_WIND (frame, jbr_@NAME@_complete,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                            @SHORT_ARGS@);
-                return 0;
-        }
-
-        ret = jbr_initialize_xdata_set_attrs (this, &xdata);
-        if (ret)
-                goto err;
-
-        local->stub = fop_@NAME@_stub (frame, jbr_@NAME@_continue,
-                                       @SHORT_ARGS@);
-        if (!local->stub) {
-                goto err;
-        }
-
-        /*
-         * Can be used to just call_dispatch or be customised per fop to *
-         * perform ops specific to that particular fop.                  *
-         */
-        ret = jbr_@NAME@_perform_local_op (frame, this, &op_errno,
-                                           @SHORT_ARGS@);
-        if (ret)
-                goto err;
-
-        return ret;
-err:
-        if (local) {
-                if (local->stub) {
-                        call_stub_destroy(local->stub);
-                }
-                if (local->qstub) {
-                        call_stub_destroy(local->qstub);
-                }
-                if (local->fd) {
-                        fd_unref(local->fd);
-                }
-                mem_put(local);
-        }
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, op_errno,
-                             @ERROR_ARGS@);
-        return 0;
-}
-
-/* template-name write-perform_local_op */
-int32_t
-jbr_@NAME@_perform_local_op (call_frame_t *frame, xlator_t *this, int *op_errno,
-                             @LONG_ARGS@)
-{
-        int32_t          ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        ret = jbr_@NAME@_call_dispatch (frame, this, op_errno,
-                                        @SHORT_ARGS@);
-
-out:
-        return ret;
-}
-
-/* template-name write-call_dispatch */
-int32_t
-jbr_@NAME@_call_dispatch (call_frame_t *frame, xlator_t *this, int *op_errno,
-                          @LONG_ARGS@)
-{
-        jbr_local_t     *local  = NULL;
-        jbr_private_t   *priv   = NULL;
-        int32_t          ret    = -1;
-        xlator_list_t   *trav   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-#if defined(JBR_CG_QUEUE)
-        jbr_inode_ctx_t  *ictx  = jbr_get_inode_ctx(this, fd->inode);
-        if (!ictx) {
-                *op_errno = EIO;
-                goto out;
-        }
-
-        LOCK(&ictx->lock);
-                if (ictx->active) {
-                        gf_msg_debug (this->name, 0,
-                                      "queuing request due to conflict");
-                        /*
-                         * TBD: enqueue only for real conflict
-                         *
-                         * Currently we just act like all writes are in
-                         * conflict with one another.  What we should really do
-                         * is check the active/pending queues and defer only if
-                         * there's a conflict there.
-                         *
-                         * It's important to check the pending queue because we
-                         * might have an active request X which conflicts with
-                         * a pending request Y, and this request Z might
-                         * conflict with Y but not X.  If we checked only the
-                         * active queue then Z could jump ahead of Y, which
-                         * would be incorrect.
-                         */
-                        local->qstub = fop_@NAME@_stub (frame,
-                                                        jbr_@NAME@_dispatch,
-                                                        @SHORT_ARGS@);
-                        if (!local->qstub) {
-                                UNLOCK(&ictx->lock);
-                                goto out;
-                        }
-                        list_add_tail(&local->qlinks, &ictx->pqueue);
-                        ++(ictx->pending);
-                        UNLOCK(&ictx->lock);
-                        ret = 0;
-                        goto out;
-                } else {
-                        list_add_tail(&local->qlinks, &ictx->aqueue);
-                        ++(ictx->active);
-                }
-        UNLOCK(&ictx->lock);
-#endif
-        ret = jbr_@NAME@_dispatch (frame, this, @SHORT_ARGS@);
-
-out:
-        return ret;
-}
-
-/* template-name write-dispatch */
-int32_t
-jbr_@NAME@_dispatch (call_frame_t *frame, xlator_t *this,
-                     @LONG_ARGS@)
-{
-        jbr_local_t     *local  = NULL;
-        jbr_private_t   *priv   = NULL;
-        int32_t          ret    = -1;
-        xlator_list_t   *trav;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        /*
-         * TBD: unblock pending request(s) if we fail after this point but
-         * before we get to jbr_@NAME@_complete (where that code currently
-         * resides).
-         */
-
-        local->call_count = priv->n_children - 1;
-        local->successful_acks = 0;
-        for (trav = this->children->next; trav; trav = trav->next) {
-                STACK_WIND (frame, jbr_@NAME@_fan_in,
-                            trav->xlator, trav->xlator->fops->@NAME@,
-                            @SHORT_ARGS@);
-        }
-
-        /* TBD: variable Issue count */
-        ret = 0;
-out:
-        return ret;
-}
-
-/* template-name write-fan-in */
-int32_t
-jbr_@NAME@_fan_in (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   @LONG_ARGS@)
-{
-        jbr_local_t   *local  = NULL;
-        int32_t        ret    = -1;
-        uint8_t        call_count;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        gf_msg_trace (this->name, 0, "op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno);
-
-        LOCK(&frame->lock);
-        call_count = --(local->call_count);
-        if (op_ret != -1) {
-                /* Increment the number of successful acks *
-                 * received for the operation.             *
-                 */
-                (local->successful_acks)++;
-                local->successful_op_ret = op_ret;
-        }
-        gf_msg_debug (this->name, 0, "succ_acks = %d, op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno, local->successful_acks);
-        UNLOCK(&frame->lock);
-
-        /* TBD: variable Completion count */
-        if (call_count == 0) {
-                call_resume(local->stub);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/* template-name write-continue */
-int32_t
-jbr_@NAME@_continue (call_frame_t *frame, xlator_t *this,
-                     @LONG_ARGS@)
-{
-        int32_t          ret = -1;
-        gf_boolean_t     result   = _gf_false;
-        jbr_local_t     *local    = NULL;
-        jbr_private_t   *priv     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        /* Perform quorum check to see if the leader needs     *
-         * to perform the operation. If the operation will not *
-         * meet quorum irrespective of the leader's result     *
-         * there is no point in the leader performing the fop  *
-         */
-        result = fop_quorum_check (this, (double)priv->n_children,
-                                   (double)local->successful_acks + 1);
-        if (result == _gf_false) {
-                gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                        J_MSG_QUORUM_NOT_MET, "Didn't receive enough acks "
-                        "to meet quorum. Failing the operation without trying "
-                        "it on the leader.");
-                STACK_UNWIND_STRICT (@NAME@, frame, -1, EROFS,
-                                     @ERROR_ARGS@);
-        } else {
-                STACK_WIND (frame, jbr_@NAME@_complete,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                            @SHORT_ARGS@);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/* template-name write-complete */
-int32_t
-jbr_@NAME@_complete (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     @LONG_ARGS@)
-{
-        int32_t          ret       = -1;
-        gf_boolean_t     result    = _gf_false;
-        jbr_private_t   *priv      = NULL;
-        jbr_local_t     *local     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-
-        /* If the fop failed on the leader, then reduce one succesful ack
-         * before calculating the fop quorum
-         */
-        LOCK(&frame->lock);
-        if (op_ret == -1)
-                (local->successful_acks)--;
-        UNLOCK(&frame->lock);
-
-#if defined(JBR_CG_QUEUE)
-        ret = jbr_remove_from_queue (frame, this);
-        if (ret)
-                goto err;
-#endif
-
-#if defined(JBR_CG_FSYNC)
-        jbr_mark_fd_dirty(this, local);
-#endif
-
-#if defined(JBR_CG_NEED_FD)
-        fd_unref(local->fd);
-#endif
-
-        /* After the leader completes the fop, a quorum check is      *
-         * performed, taking into account the outcome of the fop      *
-         * on the leader. Irrespective of the fop being successful    *
-         * or failing on the leader, the result of the quorum will    *
-         * determine if the overall fop is successful or not. For     *
-         * example, a fop might have succeeded on every node except   *
-         * the leader, in which case as quorum is being met, the fop  *
-         * will be treated as a successful fop, even though it failed *
-         * on the leader. On follower nodes, no quorum check should   *
-         * be done, and the result is returned to the leader as is.   *
-         */
-        if (priv->leader) {
-                result = fop_quorum_check (this, (double)priv->n_children,
-                                           (double)local->successful_acks + 1);
-                if (result == _gf_false) {
-                        op_ret = -1;
-                        op_errno = EROFS;
-                        gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                J_MSG_QUORUM_NOT_MET, "Quorum is not met. "
-                                "The operation has failed.");
-                } else {
-#if defined(JBR_CG_NEED_FD)
-                        op_ret = local->successful_op_ret;
-#else
-                        op_ret = 0;
-#endif
-                        op_errno = 0;
-                        gf_msg_debug (this->name, 0,
-                                      "Quorum has met. The operation has succeeded.");
-                }
-        }
-
-        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
-                             @SHORT_ARGS@);
-
-
-        return 0;
-
-err:
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, 0,
-                             @SHORT_ARGS@);
-
-        return 0;
-}
diff --git a/xlators/experimental/jbr-server/src/gen-fops.py b/xlators/experimental/jbr-server/src/gen-fops.py
deleted file mode 100755
index 8a2b47c5345..00000000000
--- a/xlators/experimental/jbr-server/src/gen-fops.py
+++ /dev/null
@@ -1,178 +0,0 @@
-#!/usr/bin/python
-
-# This script generates the boilerplate versions of most fops and cbks in the
-# server.  This allows the details of leadership-status checking, sequencing
-# between leader and followers (including fan-out), and basic error checking
-# to be centralized one place, with per-operation code kept to a minimum.
-
-import os
-import re
-import string
-import sys
-
-curdir = os.path.dirname(sys.argv[0])
-gendir = os.path.join(curdir,'../../../../libglusterfs/src')
-sys.path.append(gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# We really want the callback argument list, even when we're generating fop
-# code, so we propagate here.
-# TBD: this should probably be right in generate.py
-for k, v in cbk_subs.iteritems():
-	fop_subs[k]['@ERROR_ARGS@'] = v['@ERROR_ARGS@']
-
-# Stolen from old codegen.py
-def load_templates (path):
-	templates = {}
-	tmpl_re = re.compile("/\* template-name (.*) \*/")
-	templates = {}
-	t_name = None
-	for line in open(path,"r").readlines():
-		if not line:
-			break
-		m = tmpl_re.match(line)
-		if m:
-			if t_name:
-				templates[t_name] = string.join(t_contents,'')
-			t_name = m.group(1).strip()
-			t_contents = []
-		elif t_name:
-			t_contents.append(line)
-	if t_name:
-		templates[t_name] = string.join(t_contents,'')
-	return templates
-
-# We need two types of templates.  The first, for pure read operations, just
-# needs to do a simple am-i-leader check (augmented to allow dirty reads).
-# The second, for pure writes, needs to do fan-out to followers between those
-# initial checks and local execution.  There are other operations that don't
-# fit neatly into either category - e.g. lock ops or fsync - so we'll just have
-# to handle those manually.  The table thus includes entries only for those we
-# can categorize.  The special cases, plus any new operations we've never even
-# heard of, aren't in there.
-#
-# Various keywords can be used to define/undefine preprocessor symbols used
-# in the templates, on a per-function basis.  For example, if the keyword here
-# is "fsync" (lowercase word or abbreviation) that will cause JBR_CG_FSYNC
-# (prefix plus uppercase version) to be defined above all of the generated code
-# for that fop.
-
-fop_table = {
-	"access":		"read",
-	"create":		"write",
-	"discard":		"write",
-#	"entrylk":		"read",
-	"fallocate":	"write",
-#	"fentrylk":		"read",
-	"fgetxattr":	"read",
-#	"finodelk":		"read",
-#	"flush":		"read",
-	"fremovexattr":	"write",
-	"fsetattr":		"write",
-	"fsetxattr":	"write",
-	"fstat":		"read",
-#	"fsync":		"read",
-#	"fsyncdir":		"read",
-	"ftruncate":	"write",
-	"fxattrop":		"write",
-	"getxattr":		"read",
-#	"inodelk":		"read",
-	"link":			"write",
-	"lk":			"write,queue",
-#	"lookup":		"read",
-	"mkdir":		"write",
-	"mknod":		"write",
-	"open":			"write",
-	"opendir":		"read",
-	"rchecksum":	"read",
-	"readdir":		"read",
-	"readdirp":		"read",
-	"readlink":		"read",
-	"readv":		"read",
-	"removexattr":	"write",
-	"rename":		"write",
-	"rmdir":		"write",
-	"setattr":		"write",
-	"setxattr":		"write",
-	"stat":			"read",
-	"statfs":		"read",
-	"symlink":		"write",
-	"truncate":		"write",
-	"unlink":		"write",
-	"writev":		"write,fsync,queue",
-	"xattrop":		"write",
-}
-
-# Mention those fops in the selective_generate table, for which
-# only a few common functions will be generated, and mention those
-# functions. Rest of the functions can be customized
-selective_generate = {
-	"lk":			"fop,dispatch,call_dispatch",
-}
-
-# Stolen from gen_fdl.py
-def gen_server (templates):
-	fops_done = []
-	for name in fop_table.keys():
-		info = fop_table[name].split(",")
-		kind = info[0]
-		flags = info[1:]
-
-		# generate all functions for the fops in fop_table
-		# except for the ones in selective_generate for which
-		# generate only the functions mentioned in the
-		# selective_generate table
-		gen_funcs = "fop,complete,continue,fan-in,dispatch, \
-			call_dispatch,perform_local_op"
-		if name in selective_generate:
-			gen_funcs = selective_generate[name].split(",")
-
-		if ("fsync" in flags) or ("queue" in flags):
-			flags.append("need_fd")
-		for fname in flags:
-			print "#define JBR_CG_%s" % fname.upper()
-
-		if 'complete' in gen_funcs:
-			print generate(templates[kind+"-complete"],
-					name,cbk_subs)
-
-		if 'continue' in gen_funcs:
-			print generate(templates[kind+"-continue"],
-					name,fop_subs)
-
-		if 'fan-in' in gen_funcs:
-			print generate(templates[kind+"-fan-in"],
-					name,cbk_subs)
-
-		if 'dispatch' in gen_funcs:
-			print generate(templates[kind+"-dispatch"],
-					name,fop_subs)
-
-		if 'call_dispatch' in gen_funcs:
-			print generate(templates[kind+"-call_dispatch"],
-					name,fop_subs)
-
-		if 'perform_local_op' in gen_funcs:
-			print generate(templates[kind+"-perform_local_op"],
-					name, fop_subs)
-
-		if 'fop' in gen_funcs:
-			print generate(templates[kind+"-fop"],name,fop_subs)
-
-		for fname in flags:
-			print "#undef JBR_CG_%s" % fname.upper()
-		fops_done.append(name)
-	# Just for fun, emit the fops table too.
-	print("struct xlator_fops fops = {")
-	for x in fops_done:
-		print("	.%s = jbr_%s,"%(x,x))
-	print("};")
-
-tmpl = load_templates(sys.argv[1])
-for l in open(sys.argv[2],'r').readlines():
-	if l.find('#pragma generate') != -1:
-		print "/* BEGIN GENERATED CODE - DO NOT MODIFY */"
-		gen_server(tmpl)
-		print "/* END GENERATED CODE */"
-	else:
-		print l[:-1]
diff --git a/xlators/experimental/jbr-server/src/jbr-internal.h b/xlators/experimental/jbr-server/src/jbr-internal.h
deleted file mode 100644
index ab1dfc16de2..00000000000
--- a/xlators/experimental/jbr-server/src/jbr-internal.h
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#define LEADER_XATTR            "user.jbr.leader"
-#define SECOND_CHILD(xl)        (xl->children->next->xlator)
-#define RECONCILER_PATH         JBR_SCRIPT_PREFIX"/reconciler.py"
-#define CHANGELOG_ENTRY_SIZE    128
-
-enum {
-        gf_mt_jbr_private_t = gf_common_mt_end + 1,
-        gf_mt_jbr_fd_ctx_t,
-        gf_mt_jbr_inode_ctx_t,
-        gf_mt_jbr_dirty_t,
-        gf_mt_jbr_end
-};
-
-typedef enum jbr_recon_notify_ev_id_t {
-        JBR_RECON_SET_LEADER = 1,
-        JBR_RECON_ADD_CHILD = 2
-} jbr_recon_notify_ev_id_t;
-
-typedef struct _jbr_recon_notify_ev_s {
-        jbr_recon_notify_ev_id_t id;
-        uint32_t index; /* in case of add */
-        struct list_head list;
-} jbr_recon_notify_ev_t;
-
-typedef struct {
-        /*
-         * This is a hack to allow a non-leader to accept requests while the
-         * leader is down, and it only works for n=2.  The way it works is that
-         * "config_leader" indicates the state from our options (via init or
-         * reconfigure) but "leader" is what the fop code actually looks at.  If
-         * config_leader is true, then leader will *always* be true as well,
-         * giving that brick precedence.  If config_leader is false, then
-         * leader will only be true if there is no connection to the other
-         * brick (tracked in jbr_notify).
-         *
-         * TBD: implement real leader election
-         */
-        gf_boolean_t            config_leader;
-        gf_boolean_t            leader;
-        uint8_t                 up_children;
-        uint8_t                 n_children;
-        char                    *vol_file;
-        uint32_t                current_term;
-        uint32_t                kid_state;
-        gf_lock_t               dirty_lock;
-        struct list_head        dirty_fds;
-	uint32_t                index;
-	gf_lock_t               index_lock;
-        double                  quorum_pct;
-        int                     term_fd;
-        long                    term_total;
-        long                    term_read;
-        /*
-         * This is a super-duper hack, but it will do for now.  The reason it's
-         * a hack is that we pass this to dict_set_static_bin, so we don't have
-         * to mess around with allocating and freeing it on every single IPC
-         * request, but it's totally not thread-safe.  On the other hand, there
-         * should only be one reconciliation thread running and calling these
-         * functions at a time, so maybe that doesn't matter.
-         *
-         * TBD: re-evaluate how to manage this
-         */
-        char                    term_buf[CHANGELOG_ENTRY_SIZE];
-        gf_boolean_t            child_up; /* To maintain the state of *
-                                           * the translator */
-} jbr_private_t;
-
-typedef struct {
-        call_stub_t             *stub;
-        call_stub_t             *qstub;
-        uint32_t                call_count;
-        uint32_t                successful_acks;
-        uint32_t                successful_op_ret;
-        fd_t                    *fd;
-        struct list_head        qlinks;
-} jbr_local_t;
-
-/*
- * This should match whatever changelog returns on the pre-op for us to pass
- * when we're ready for our post-op.
- */
-typedef uint32_t log_id_t;
-
-typedef struct {
-        struct list_head        links;
-        log_id_t                id;
-} jbr_dirty_list_t;
-
-typedef struct {
-        fd_t                    *fd;
-        struct list_head        dirty_list;
-        struct list_head        fd_list;
-} jbr_fd_ctx_t;
-
-typedef struct {
-        gf_lock_t               lock;
-        uint32_t                active;
-        struct list_head        aqueue;
-        uint32_t                pending;
-        struct list_head        pqueue;
-} jbr_inode_ctx_t;
-
-void jbr_start_reconciler (xlator_t *this);
diff --git a/xlators/experimental/jbr-server/src/jbr.c b/xlators/experimental/jbr-server/src/jbr.c
deleted file mode 100644
index e1eed3f6094..00000000000
--- a/xlators/experimental/jbr-server/src/jbr.c
+++ /dev/null
@@ -1,1676 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <fnmatch.h>
-#include "call-stub.h"
-#include "defaults.h"
-#include "xlator.h"
-#include "glfs.h"
-#include "glfs-internal.h"
-#include "run.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "syscall.h"
-#include "compat-errno.h"
-
-#include "jbr-internal.h"
-#include "jbr-messages.h"
-
-#define JBR_FLUSH_INTERVAL      5
-
-enum {
-        /* echo "cluster/jbr-server" | md5sum | cut -c 1-8 */
-        JBR_SERVER_IPC_BASE = 0x0e2d66a5,
-        JBR_SERVER_TERM_RANGE,
-        JBR_SERVER_OPEN_TERM,
-        JBR_SERVER_NEXT_ENTRY
-};
-
-/*
- * Need to declare jbr_lk_call_dispatch as jbr_lk_continue and *
- * jbr_lk_perform_local_op call it, before code is generated.  *
- */
-int32_t
-jbr_lk_call_dispatch (call_frame_t *frame, xlator_t *this, int *op_errno,
-                      fd_t *fd, int32_t cmd, struct gf_flock *lock,
-                      dict_t *xdata);
-
-int32_t
-jbr_lk_dispatch (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, int32_t cmd, struct gf_flock *lock,
-                 dict_t *xdata);
-
-/* Used to check the quorum of acks received after the fop
- * confirming the status of the fop on all the brick processes
- * for this particular subvolume
- */
-gf_boolean_t
-fop_quorum_check (xlator_t *this, double n_children,
-                  double current_state)
-{
-        jbr_private_t   *priv           = NULL;
-        gf_boolean_t     result         = _gf_false;
-        double           required       = 0;
-        double           current        = 0;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        required = n_children * priv->quorum_pct;
-
-        /*
-         * Before performing the fop on the leader, we need to check,
-         * if there is any merit in performing the fop on the leader.
-         * In a case, where even a successful write on the leader, will
-         * not meet quorum, there is no point in trying the fop on the
-         * leader.
-         * When this function is called after the leader has tried
-         * performing the fop, this check will calculate quorum taking into
-         * account the status of the fop on the leader. If the leader's
-         * op_ret was -1, the complete function would account that by
-         * decrementing successful_acks by 1
-         */
-
-        current = current_state * 100.0;
-
-        if (current < required) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_QUORUM_NOT_MET,
-                        "Quorum not met. quorum_pct = %f "
-                        "Current State = %f, Required State = %f",
-                        priv->quorum_pct, current,
-                        required);
-        } else
-                result = _gf_true;
-
-out:
-        return result;
-}
-
-jbr_inode_ctx_t *
-jbr_get_inode_ctx (xlator_t *this, inode_t *inode)
-{
-        uint64_t                ctx_int         = 0LL;
-        jbr_inode_ctx_t         *ctx_ptr;
-
-        if (__inode_ctx_get(inode, this, &ctx_int) == 0) {
-                ctx_ptr = (jbr_inode_ctx_t *)(long)ctx_int;
-        } else {
-                ctx_ptr = GF_CALLOC (1, sizeof(*ctx_ptr),
-                                     gf_mt_jbr_inode_ctx_t);
-                if (ctx_ptr) {
-                        ctx_int = (uint64_t)(long)ctx_ptr;
-                        if (__inode_ctx_set(inode, this, &ctx_int) == 0) {
-                                LOCK_INIT(&ctx_ptr->lock);
-                                INIT_LIST_HEAD(&ctx_ptr->aqueue);
-                                INIT_LIST_HEAD(&ctx_ptr->pqueue);
-                        } else {
-                                GF_FREE(ctx_ptr);
-                                ctx_ptr = NULL;
-                        }
-                }
-
-        }
-
-        return ctx_ptr;
-}
-
-jbr_fd_ctx_t *
-jbr_get_fd_ctx (xlator_t *this, fd_t *fd)
-{
-        uint64_t                ctx_int         = 0LL;
-        jbr_fd_ctx_t            *ctx_ptr;
-
-        if (__fd_ctx_get(fd, this, &ctx_int) == 0) {
-                ctx_ptr = (jbr_fd_ctx_t *)(long)ctx_int;
-        } else {
-                ctx_ptr = GF_CALLOC (1, sizeof(*ctx_ptr), gf_mt_jbr_fd_ctx_t);
-                if (ctx_ptr) {
-                        if (__fd_ctx_set(fd, this, (uint64_t)ctx_ptr) == 0) {
-                                INIT_LIST_HEAD(&ctx_ptr->dirty_list);
-                                INIT_LIST_HEAD(&ctx_ptr->fd_list);
-                        } else {
-                                GF_FREE(ctx_ptr);
-                                ctx_ptr = NULL;
-                        }
-                }
-
-        }
-
-        return ctx_ptr;
-}
-
-void
-jbr_mark_fd_dirty (xlator_t *this, jbr_local_t *local)
-{
-        fd_t                    *fd             = local->fd;
-        jbr_fd_ctx_t            *ctx_ptr;
-        jbr_dirty_list_t        *dirty;
-        jbr_private_t           *priv           = this->private;
-
-        /*
-         * TBD: don't do any of this for O_SYNC/O_DIRECT writes.
-         * Unfortunately, that optimization requires that we distinguish
-         * between writev and other "write" calls, saving the original flags
-         * and checking them in the callback.  Too much work for too little
-         * gain right now.
-         */
-
-        LOCK(&fd->lock);
-                ctx_ptr = jbr_get_fd_ctx(this, fd);
-                dirty = GF_CALLOC(1, sizeof(*dirty), gf_mt_jbr_dirty_t);
-                if (ctx_ptr && dirty) {
-                        gf_msg_trace (this->name, 0,
-                                      "marking fd %p as dirty (%p)", fd, dirty);
-                        /* TBD: fill dirty->id from what changelog gave us */
-                        list_add_tail(&dirty->links, &ctx_ptr->dirty_list);
-                        if (list_empty(&ctx_ptr->fd_list)) {
-                                /* Add a ref so _release doesn't get called. */
-                                ctx_ptr->fd = fd_ref(fd);
-                                LOCK(&priv->dirty_lock);
-                                        list_add_tail (&ctx_ptr->fd_list,
-                                                       &priv->dirty_fds);
-                                UNLOCK(&priv->dirty_lock);
-                        }
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                J_MSG_MEM_ERR, "could not mark %p dirty", fd);
-                        if (ctx_ptr) {
-                                GF_FREE(ctx_ptr);
-                        }
-                        if (dirty) {
-                                GF_FREE(dirty);
-                        }
-                }
-        UNLOCK(&fd->lock);
-}
-
-#define JBR_TERM_XATTR          "trusted.jbr.term"
-#define JBR_INDEX_XATTR         "trusted.jbr.index"
-#define JBR_REP_COUNT_XATTR     "trusted.jbr.rep-count"
-#define RECON_TERM_XATTR        "trusted.jbr.recon-term"
-#define RECON_INDEX_XATTR       "trusted.jbr.recon-index"
-
-int32_t
-jbr_leader_checks_and_init (call_frame_t *frame, xlator_t *this, int *op_errno,
-                            dict_t *xdata, fd_t *fd)
-{
-        jbr_local_t     *local         = NULL;
-        jbr_private_t   *priv          = NULL;
-        int32_t          ret           = -1;
-        gf_boolean_t     result        = _gf_false;
-        int              from_leader   = _gf_false;
-        int              from_recon    = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-
-        /*
-         * Our first goal here is to avoid "split brain surprise" for users who
-         * specify exactly 50% with two- or three-way replication.  That means
-         * either a more-than check against half the total replicas or an
-         * at-least check against half of our peers (one less).  Of the two,
-         * only an at-least check supports the intuitive use of 100% to mean
-         * all replicas must be present, because "more than 100%" will never
-         * succeed regardless of which count we use.  This leaves us with a
-         * slightly non-traditional definition of quorum ("at least X% of peers
-         * not including ourselves") but one that's useful enough to be worth
-         * it.
-         *
-         * Note that n_children and up_children *do* include the local
-         * subvolume, so we need to subtract one in each case.
-         */
-        if (priv->leader) {
-                result = fop_quorum_check (this, (double)(priv->n_children - 1),
-                                   (double)(priv->up_children - 1));
-
-                if (result == _gf_false) {
-                        /* Emulate the AFR client-side-quorum behavior. */
-                        gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                J_MSG_QUORUM_NOT_MET, "Sufficient number of "
-                                "subvolumes are not up to meet quorum.");
-                        *op_errno = EROFS;
-                        goto out;
-                }
-        } else {
-                if (xdata) {
-                        from_leader = !!dict_get(xdata, JBR_TERM_XATTR);
-                        from_recon = !!dict_get(xdata, RECON_TERM_XATTR)
-                                  && !!dict_get(xdata, RECON_INDEX_XATTR);
-                } else {
-                        from_leader = from_recon = _gf_false;
-                }
-
-                /* follower/recon path        *
-                 * just send it to local node *
-                 */
-                if (!from_leader && !from_recon) {
-                        *op_errno = EREMOTE;
-                        goto out;
-                }
-        }
-
-        local = mem_get0(this->local_pool);
-        if (!local) {
-                goto out;
-        }
-
-        if (fd)
-                local->fd = fd_ref(fd);
-        else
-                local->fd = NULL;
-
-        INIT_LIST_HEAD(&local->qlinks);
-        frame->local = local;
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_initialize_xdata_set_attrs (xlator_t *this, dict_t **xdata)
-{
-        jbr_local_t     *local         = NULL;
-        jbr_private_t   *priv          = NULL;
-        int32_t          ret           = -1;
-        uint32_t         ti            = 0;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, out);
-
-        if (!*xdata) {
-                *xdata = dict_new();
-                if (!*xdata) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                J_MSG_MEM_ERR, "failed to allocate xdata");
-                        goto out;
-                }
-        }
-
-        if (dict_set_int32(*xdata, JBR_TERM_XATTR, priv->current_term) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR, "failed to set jbr-term");
-                goto out;
-        }
-
-        LOCK(&priv->index_lock);
-        ti = ++(priv->index);
-        UNLOCK(&priv->index_lock);
-        if (dict_set_int32(*xdata, JBR_INDEX_XATTR, ti) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR, "failed to set index");
-                goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_remove_from_queue (call_frame_t *frame, xlator_t *this)
-{
-        int32_t          ret       = -1;
-        jbr_inode_ctx_t *ictx      = NULL;
-        jbr_local_t     *local     = NULL;
-        jbr_local_t     *next      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        if (local->qlinks.next != &local->qlinks) {
-                list_del(&local->qlinks);
-                ictx = jbr_get_inode_ctx(this, local->fd->inode);
-                if (ictx) {
-                        LOCK(&ictx->lock);
-                                if (ictx->pending) {
-                                        /*
-                                         * TBD: dequeue *all* non-conflicting
-                                         * reqs
-                                         *
-                                         * With the stub implementation there
-                                         * can only be one request active at a
-                                         * time (zero here) so it's not an
-                                         * issue.  In a real implementation
-                                         * there might still be other active
-                                         * requests to check against, and
-                                         * multiple pending requests that could
-                                         * continue.
-                                         */
-                                        gf_msg_debug (this->name, 0,
-                                                     "unblocking next request");
-                                        --(ictx->pending);
-                                        next = list_entry (ictx->pqueue.next,
-                                                           jbr_local_t, qlinks);
-                                        list_del(&next->qlinks);
-                                        list_add_tail(&next->qlinks,
-                                                      &ictx->aqueue);
-                                        call_resume(next->qstub);
-                                } else {
-                                        --(ictx->active);
-                                }
-                        UNLOCK(&ictx->lock);
-                }
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-int32_t
-jbr_lk_complete (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct gf_flock *flock, dict_t *xdata)
-{
-        int32_t          ret       = -1;
-        jbr_private_t   *priv      = NULL;
-        jbr_local_t     *local     = NULL;
-        gf_boolean_t     result    = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-        GF_VALIDATE_OR_GOTO (this->name, flock, err);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, err);
-
-        /*
-         * Remove from queue for unlock operation only   *
-         * For lock operation, it will be done in fan-in *
-         */
-        if (flock->l_type == F_UNLCK) {
-                ret = jbr_remove_from_queue (frame, this);
-                if (ret)
-                        goto err;
-        }
-
-        /*
-         * On a follower, unwind with the op_ret and op_errno. On a *
-         * leader, if the fop is a locking fop, and its a failure,  *
-         * send fail, else call stub which will dispatch the fop to *
-         * the followers.                                           *
-         *                                                          *
-         * If the fop is a unlocking fop, check quorum. If quorum   *
-         * is met, then send success. Else Rollback on leader,      *
-         * followed by followers, and then send -ve ack to client.  *
-         */
-        if (priv->leader) {
-
-                /* Increase the successful acks if it's a success. */
-                LOCK(&frame->lock);
-                if (op_ret != -1)
-                        (local->successful_acks)++;
-                UNLOCK(&frame->lock);
-
-                if (flock->l_type == F_UNLCK) {
-                        result = fop_quorum_check (this,
-                                            (double)priv->n_children,
-                                            (double)local->successful_acks);
-                        if (result == _gf_false) {
-                                op_ret = -1;
-                                op_errno = EROFS;
-                                gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                        J_MSG_QUORUM_NOT_MET,
-                                        "Quorum is not met. "
-                                        "The operation has failed.");
-
-                                /* TODO: PERFORM UNLOCK ROLLBACK ON LEADER *
-                                 * FOLLOWED BY FOLLOWERS. */
-                        } else {
-                                op_ret = 0;
-                                op_errno = 0;
-                        }
-
-                        fd_unref(local->fd);
-                        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno,
-                                             flock, xdata);
-                } else {
-                        if (op_ret == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        J_MSG_LOCK_FAILURE,
-                                        "The lock operation failed on "
-                                        "the leader.");
-
-                                fd_unref(local->fd);
-                                STACK_UNWIND_STRICT (lk, frame, op_ret,
-                                                     op_errno, flock, xdata);
-                        } else {
-                                if (!local->stub) {
-                                        goto err;
-                                }
-
-                                call_resume(local->stub);
-                        }
-                }
-        } else {
-                fd_unref(local->fd);
-                STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno,
-                                     flock, xdata);
-        }
-
-        return 0;
-
-err:
-        if (local) {
-                if (local->stub) {
-                        call_stub_destroy(local->stub);
-                }
-                if (local->qstub) {
-                        call_stub_destroy(local->qstub);
-                }
-                if (local->fd) {
-                        fd_unref(local->fd);
-                }
-                mem_put(local);
-        }
-        STACK_UNWIND_STRICT (lk, frame, -1, op_errno,
-                             flock, xdata);
-        return 0;
-}
-
-int32_t
-jbr_lk_fan_in (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
-               dict_t *xdata)
-{
-        uint8_t          call_count = -1;
-        int32_t          ret        = -1;
-        gf_boolean_t     result     = _gf_false;
-        jbr_local_t     *local      = NULL;
-        jbr_private_t   *priv       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        gf_msg_trace (this->name, 0, "op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno);
-
-        LOCK(&frame->lock);
-        call_count = --(local->call_count);
-        if (op_ret != -1) {
-                /* Increment the number of successful acks *
-                 * received for the operation.             *
-                 */
-                (local->successful_acks)++;
-                local->successful_op_ret = op_ret;
-        }
-        gf_msg_debug (this->name, 0, "succ_acks = %d, op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno, local->successful_acks);
-        UNLOCK(&frame->lock);
-
-        if (call_count == 0) {
-                /*
-                 * If the fop is a locking fop, then check quorum. If quorum *
-                 * is met, send successful ack to the client. If quorum is   *
-                 * not met, then rollback locking on followers, followed by  *
-                 * rollback of locking on leader, and then sending -ve ack   *
-                 * to the client.                                            *
-                 *                                                           *
-                 * If the fop is a unlocking fop, then call stub.            *
-                 */
-                if (flock->l_type == F_UNLCK) {
-                        call_resume(local->stub);
-                } else {
-                        /*
-                         * Remove from queue for locking fops, for unlocking *
-                         * fops, it is taken care of in jbr_lk_complete      *
-                         */
-                        ret = jbr_remove_from_queue (frame, this);
-                        if (ret)
-                                goto out;
-
-                        fd_unref(local->fd);
-
-                        result = fop_quorum_check (this,
-                                          (double)priv->n_children,
-                                          (double)local->successful_acks);
-                        if (result == _gf_false) {
-                                gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                        J_MSG_QUORUM_NOT_MET,
-                                        "Didn't receive enough acks to meet "
-                                        "quorum. Failing the locking "
-                                        "operation and initiating rollback on "
-                                        "followers and the leader "
-                                        "respectively.");
-
-                                /* TODO: PERFORM ROLLBACK OF LOCKING ON
-                                 * FOLLOWERS, FOLLOWED BY ROLLBACK ON
-                                 * LEADER.
-                                 */
-
-                                STACK_UNWIND_STRICT (lk, frame, -1, EROFS,
-                                                     flock, xdata);
-                        } else {
-                                STACK_UNWIND_STRICT (lk, frame, 0, 0,
-                                                     flock, xdata);
-                        }
-                }
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/*
- * Called from leader for locking fop, being writen as a separate *
- * function so as to support queues.                              *
- */
-int32_t
-jbr_perform_lk_on_leader (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, int32_t cmd, struct gf_flock *flock,
-                         dict_t *xdata)
-{
-        int32_t          ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, flock, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        STACK_WIND (frame, jbr_lk_complete,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, flock, xdata);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_lk_perform_local_op (call_frame_t *frame, xlator_t *this, int *op_errno,
-                         fd_t *fd, int32_t cmd, struct gf_flock *flock,
-                         dict_t *xdata)
-{
-        int32_t          ret    = -1;
-        jbr_local_t     *local  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-        GF_VALIDATE_OR_GOTO (this->name, flock, out);
-
-        /*
-         * Check if the fop is a locking fop or unlocking fop, and
-         * handle it accordingly. If it is a locking fop, take the
-         * lock on leader first, and then send it to the followers.
-         * If it is a unlocking fop, unlock the followers first,
-         * and then on meeting quorum perform the unlock on the leader.
-         */
-        if (flock->l_type == F_UNLCK) {
-                ret = jbr_lk_call_dispatch (frame, this, op_errno,
-                                            fd, cmd, flock, xdata);
-                if (ret)
-                        goto out;
-        } else {
-                jbr_inode_ctx_t  *ictx  = jbr_get_inode_ctx(this, fd->inode);
-
-                if (!ictx) {
-                        *op_errno = EIO;
-                        goto out;
-                }
-
-                LOCK(&ictx->lock);
-                        if (ictx->active) {
-                                gf_msg_debug (this->name, 0,
-                                              "queuing request due to conflict");
-
-                                local->qstub = fop_lk_stub (frame,
-                                                       jbr_perform_lk_on_leader,
-                                                       fd, cmd, flock, xdata);
-                                if (!local->qstub) {
-                                        UNLOCK(&ictx->lock);
-                                        goto out;
-                                }
-                                list_add_tail(&local->qlinks, &ictx->pqueue);
-                                ++(ictx->pending);
-                                UNLOCK(&ictx->lock);
-                                ret = 0;
-                                goto out;
-                        } else {
-                                list_add_tail(&local->qlinks, &ictx->aqueue);
-                                ++(ictx->active);
-                        }
-                UNLOCK(&ictx->lock);
-                ret = jbr_perform_lk_on_leader (frame, this, fd, cmd,
-                                                flock, xdata);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_lk_continue (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
-{
-        int32_t          ret      = -1;
-        jbr_local_t     *local    = NULL;
-        jbr_private_t   *priv     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-        GF_VALIDATE_OR_GOTO (this->name, flock, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, out);
-
-        /*
-         * If it's a locking fop, then call dispatch to followers  *
-         * If it's a unlock fop, then perform the unlock operation *
-         */
-        if (flock->l_type == F_UNLCK) {
-                STACK_WIND (frame, jbr_lk_complete,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk,
-                            fd, cmd, flock, xdata);
-        } else {
-                /*
-                 * Directly call jbr_lk_dispatch instead of appending *
-                 * in queue, which is done at jbr_lk_perform_local_op *
-                 * for locking fops                                   *
-                 */
-                ret = jbr_lk_dispatch (frame, this, fd, cmd,
-                                       flock, xdata);
-                if (ret) {
-                        STACK_UNWIND_STRICT (lk, frame, -1, 0,
-                                             flock, xdata);
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-#pragma generate
-
-uint8_t
-jbr_count_up_kids (jbr_private_t *priv)
-{
-        uint8_t         retval  = 0;
-        uint8_t         i;
-
-        for (i = 0; i < priv->n_children; ++i) {
-                if (priv->kid_state & (1 << i)) {
-                        ++retval;
-                }
-        }
-
-        return retval;
-}
-
-/*
- * The fsync machinery looks a lot like that for any write call, but there are
- * some important differences that are easy to miss.  First, we don't care
- * about the xdata that shows whether the call came from a leader or
- * reconciliation process.  If we're the leader we fan out; if we're not we
- * don't.  Second, we don't wait for followers before we issue the local call.
- * The code generation system could be updated to handle this, and still might
- * if we need to implement other "almost identical" paths (e.g. for open), but
- * a copy is more readable as long as it's just one.
- */
-
-int32_t
-jbr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
-{
-        jbr_local_t     *local  = frame->local;
-        gf_boolean_t    unwind;
-
-        LOCK(&frame->lock);
-                unwind = !--(local->call_count);
-        UNLOCK(&frame->lock);
-
-        if (unwind) {
-                STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf,
-                                     postbuf, xdata);
-        }
-        return 0;
-}
-
-int32_t
-jbr_fsync_local_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        jbr_dirty_list_t        *dirty;
-        jbr_dirty_list_t        *dtmp;
-        jbr_local_t             *local  = frame->local;
-
-        list_for_each_entry_safe (dirty, dtmp, &local->qlinks, links) {
-                gf_msg_trace (this->name, 0,
-                              "sending post-op on %p (%p)", local->fd, dirty);
-                GF_FREE(dirty);
-        }
-
-        return jbr_fsync_cbk (frame, cookie, this, op_ret, op_errno,
-                              prebuf, postbuf, xdata);
-}
-
-int32_t
-jbr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-           dict_t *xdata)
-{
-        jbr_private_t   *priv   = this->private;
-        jbr_local_t     *local;
-        uint64_t        ctx_int         = 0LL;
-        jbr_fd_ctx_t    *ctx_ptr;
-        xlator_list_t   *trav;
-
-        local = mem_get0(this->local_pool);
-        if (!local) {
-                STACK_UNWIND_STRICT(fsync, frame, -1, ENOMEM,
-                                    NULL, NULL, xdata);
-                return 0;
-        }
-        INIT_LIST_HEAD(&local->qlinks);
-        frame->local = local;
-
-        /* Move the dirty list from the fd to the fsync request. */
-        LOCK(&fd->lock);
-                if (__fd_ctx_get(fd, this, &ctx_int) == 0) {
-                        ctx_ptr = (jbr_fd_ctx_t *)(long)ctx_int;
-                        list_splice_init (&ctx_ptr->dirty_list,
-                                          &local->qlinks);
-                }
-        UNLOCK(&fd->lock);
-
-        /* Issue the local call. */
-        local->call_count = priv->leader ? priv->n_children : 1;
-        STACK_WIND (frame, jbr_fsync_local_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-
-        /* Issue remote calls if we're the leader. */
-        if (priv->leader) {
-                for (trav = this->children->next; trav; trav = trav->next) {
-                        STACK_WIND (frame, jbr_fsync_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->fsync,
-                                    fd, flags, xdata);
-                }
-        }
-
-        return 0;
-}
-
-int32_t
-jbr_getxattr_special (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      const char *name, dict_t *xdata)
-{
-        dict_t          *result;
-        jbr_private_t   *priv   = this->private;
-
-        if (!priv->leader) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, EREMOTE, NULL, NULL);
-                return 0;
-        }
-
-        if (!name || (strcmp(name, JBR_REP_COUNT_XATTR) != 0)) {
-                STACK_WIND_TAIL (frame,
-                                 FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->getxattr,
-                                 loc, name, xdata);
-                return 0;
-        }
-
-        result = dict_new();
-        if (!result) {
-                goto dn_failed;
-        }
-
-        priv->up_children = jbr_count_up_kids(this->private);
-        if (dict_set_uint32(result, JBR_REP_COUNT_XATTR,
-                            priv->up_children) != 0) {
-                goto dsu_failed;
-        }
-
-        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, result, NULL);
-        dict_destroy(result);
-        return 0;
-
-dsu_failed:
-        dict_destroy(result);
-dn_failed:
-        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
-}
-
-void
-jbr_flush_fd (xlator_t *this, jbr_fd_ctx_t *fd_ctx)
-{
-        jbr_dirty_list_t        *dirty;
-        jbr_dirty_list_t        *dtmp;
-
-        list_for_each_entry_safe (dirty, dtmp, &fd_ctx->dirty_list, links) {
-                gf_msg_trace (this->name, 0,
-                              "sending post-op on %p (%p)", fd_ctx->fd, dirty);
-                GF_FREE(dirty);
-        }
-
-        INIT_LIST_HEAD(&fd_ctx->dirty_list);
-}
-
-void *
-jbr_flush_thread (void *ctx)
-{
-        xlator_t                *this   = ctx;
-        jbr_private_t           *priv   = this->private;
-        struct list_head        dirty_fds;
-        jbr_fd_ctx_t            *fd_ctx;
-        jbr_fd_ctx_t            *fd_tmp;
-        int                     ret;
-
-        for (;;) {
-                /*
-                 * We have to be very careful to avoid lock inversions here, so
-                 * we can't just hold priv->dirty_lock while we take and
-                 * release locks for each fd.  Instead, we only hold dirty_lock
-                 * at the beginning of each iteration, as we (effectively) make
-                 * a copy of the current list head and then clear the original.
-                 * This leads to four scenarios for adding the first entry to
-                 * an fd and potentially putting it on the global list.
-                 *
-                 * (1) While we're asleep.  No lock contention, it just gets
-                 *     added and will be processed on the next iteration.
-                 *
-                 * (2) After we've made a local copy, but before we've started
-                 *     processing that fd.  The new entry will be added to the
-                 *     fd (under its lock), and we'll process it on the current
-                 *     iteration.
-                 *
-                 * (3) While we're processing the fd.  They'll block on the fd
-                 *     lock, then see that the list is empty and put it on the
-                 *     global list.  We'll process it here on the next
-                 *     iteration.
-                 *
-                 * (4) While we're working, but after we've processed that fd.
-                 *     Same as (1) as far as that fd is concerned.
-                 */
-                INIT_LIST_HEAD(&dirty_fds);
-                LOCK(&priv->dirty_lock);
-                list_splice_init(&priv->dirty_fds, &dirty_fds);
-                UNLOCK(&priv->dirty_lock);
-
-                list_for_each_entry_safe (fd_ctx, fd_tmp, &dirty_fds, fd_list) {
-                        ret = syncop_fsync(FIRST_CHILD(this), fd_ctx->fd, 0,
-                                           NULL, NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        J_MSG_SYS_CALL_FAILURE,
-                                        "failed to fsync %p (%d)",
-                                        fd_ctx->fd, -ret);
-                        }
-
-                        LOCK(&fd_ctx->fd->lock);
-                                jbr_flush_fd(this, fd_ctx);
-                                list_del_init(&fd_ctx->fd_list);
-                        UNLOCK(&fd_ctx->fd->lock);
-                        fd_unref(fd_ctx->fd);
-                }
-
-                sleep(JBR_FLUSH_INTERVAL);
-        }
-
-        return NULL;
-}
-
-
-int32_t
-jbr_get_changelog_dir (xlator_t *this, char **cl_dir_p)
-{
-        xlator_t        *cl_xl;
-
-        /* Find our changelog translator. */
-        cl_xl = this;
-        while (cl_xl) {
-                if (strcmp(cl_xl->type, "features/changelog") == 0) {
-                        break;
-                }
-                cl_xl = cl_xl->children->xlator;
-        }
-        if (!cl_xl) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_INIT_FAIL,
-                        "failed to find changelog translator");
-                return ENOENT;
-        }
-
-        /* Find the actual changelog directory. */
-        if (dict_get_str(cl_xl->options, "changelog-dir", cl_dir_p) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_INIT_FAIL,
-                        "failed to find changelog-dir for %s", cl_xl->name);
-                return ENODATA;
-        }
-
-        return 0;
-}
-
-
-void
-jbr_get_terms (call_frame_t *frame, xlator_t *this)
-{
-        int32_t         op_errno;
-        char            *cl_dir;
-        DIR             *fp             = NULL;
-        struct dirent   *rd_entry;
-        struct dirent   *rd_result;
-        int32_t         term_first      = -1;
-        int32_t         term_contig     = -1;
-        int32_t         term_last       = -1;
-        int             term_num;
-        char            *probe_str;
-        dict_t          *my_xdata       = NULL;
-
-        op_errno = jbr_get_changelog_dir(this, &cl_dir);
-        if (op_errno) {
-                goto err;       /* Error was already logged. */
-        }
-        op_errno = ENODATA;     /* Most common error after this. */
-
-        rd_entry = alloca (offsetof(struct dirent, d_name) +
-                           pathconf(cl_dir, _PC_NAME_MAX) + 1);
-        if (!rd_entry) {
-                goto err;
-        }
-
-        fp = sys_opendir (cl_dir);
-        if (!fp) {
-                op_errno = errno;
-                goto err;
-        }
-
-        /* Find first and last terms. */
-        for (;;) {
-                if (readdir_r(fp, rd_entry, &rd_result) != 0) {
-                        op_errno = errno;
-                        goto err;
-                }
-                if (!rd_result) {
-                        break;
-                }
-                if (fnmatch("TERM.*", rd_entry->d_name, FNM_PATHNAME) != 0) {
-                        continue;
-                }
-                /* +5 points to the character after the period */
-                term_num = atoi(rd_entry->d_name+5);
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        J_MSG_GENERIC,
-                        "%s => %d", rd_entry->d_name, term_num);
-                if (term_num < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_INVALID,
-                                "invalid term file name %s", rd_entry->d_name);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-                if ((term_first < 0) || (term_first > term_num)) {
-                        term_first = term_num;
-                }
-                if ((term_last < 0) || (term_last < term_num)) {
-                        term_last = term_num;
-                }
-        }
-        if ((term_first < 0) || (term_last < 0)) {
-                /* TBD: are we *sure* there should always be at least one? */
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_NO_DATA, "no terms found");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        sys_closedir (fp);
-        fp = NULL;
-
-        /*
-         * Find term_contig, which is the earliest term for which there are
-         * no gaps between it and term_last.
-         */
-        for (term_contig = term_last; term_contig > 0; --term_contig) {
-                if (gf_asprintf(&probe_str, "%s/TERM.%d",
-                                cl_dir, term_contig-1) <= 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_MEM_ERR,
-                                "failed to format term %d", term_contig-1);
-                        goto err;
-                }
-                if (sys_access(probe_str, F_OK) != 0) {
-                        GF_FREE(probe_str);
-                        break;
-                }
-                GF_FREE(probe_str);
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                J_MSG_GENERIC,
-                "found terms %d-%d (%d)",
-                term_first, term_last, term_contig);
-
-        /* Return what we've found */
-        my_xdata = dict_new();
-        if (!my_xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_MEM_ERR,
-                        "failed to allocate reply dictionary");
-                goto err;
-        }
-        if (dict_set_int32(my_xdata, "term-first", term_first) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR,
-                        "failed to set term-first");
-                goto err;
-        }
-        if (dict_set_int32(my_xdata, "term-contig", term_contig) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR,
-                        "failed to set term-contig");
-                goto err;
-        }
-        if (dict_set_int32(my_xdata, "term-last", term_last) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR,
-                        "failed to set term-last");
-                goto err;
-        }
-
-        /* Finally! */
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, my_xdata);
-        dict_unref(my_xdata);
-        return;
-
-err:
-        if (fp) {
-                sys_closedir (fp);
-        }
-        if (my_xdata) {
-                dict_unref(my_xdata);
-        }
-        STACK_UNWIND_STRICT (ipc, frame, -1, op_errno, NULL);
-}
-
-
-long
-get_entry_count (xlator_t *this, int fd)
-{
-        struct stat     buf;
-        long            min;            /* last entry not known to be empty */
-        long            max;            /* first entry known to be empty */
-        long            curr;
-        char            entry[CHANGELOG_ENTRY_SIZE];
-
-        if (sys_fstat (fd, &buf) < 0) {
-                return -1;
-        }
-
-        min = 0;
-        max = buf.st_size / CHANGELOG_ENTRY_SIZE;
-
-        while ((min+1) < max) {
-                curr = (min + max) / 2;
-                if (sys_lseek(fd, curr*CHANGELOG_ENTRY_SIZE, SEEK_SET) < 0) {
-                        return -1;
-                }
-                if (sys_read(fd, entry, sizeof(entry)) != sizeof(entry)) {
-                        return -1;
-                }
-                if ((entry[0] == '_') && (entry[1] == 'P')) {
-                        min = curr;
-                } else {
-                        max = curr;
-                }
-        }
-
-        if (sys_lseek(fd, 0, SEEK_SET) < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        J_MSG_SYS_CALL_FAILURE,
-                        "failed to reset offset");
-        }
-        return max;
-}
-
-
-void
-jbr_open_term (call_frame_t *frame, xlator_t *this, dict_t *xdata)
-{
-        int32_t         op_errno;
-        char            *cl_dir;
-        char            *term;
-        char            *path;
-        jbr_private_t   *priv           = this->private;
-
-        op_errno = jbr_get_changelog_dir(this, &cl_dir);
-        if (op_errno) {
-                goto err;
-        }
-
-        if (dict_get_str(xdata, "term", &term) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_NO_DATA, "missing term");
-                op_errno = ENODATA;
-                goto err;
-        }
-
-        if (gf_asprintf(&path, "%s/TERM.%s", cl_dir, term) < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_MEM_ERR, "failed to construct path");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (priv->term_fd >= 0) {
-                sys_close (priv->term_fd);
-        }
-        priv->term_fd = open(path, O_RDONLY);
-        if (priv->term_fd < 0) {
-                op_errno = errno;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_SYS_CALL_FAILURE,
-                        "failed to open term file");
-                goto err;
-        }
-
-        priv->term_total = get_entry_count(this, priv->term_fd);
-        if (priv->term_total < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_NO_DATA, "failed to get entry count");
-                sys_close (priv->term_fd);
-                priv->term_fd = -1;
-                op_errno = EIO;
-                goto err;
-        }
-        priv->term_read = 0;
-
-        /* Success! */
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, NULL);
-        return;
-
-err:
-        STACK_UNWIND_STRICT (ipc, frame, -1, op_errno, NULL);
-}
-
-
-void
-jbr_next_entry (call_frame_t *frame, xlator_t *this)
-{
-        int32_t         op_errno        = ENOMEM;
-        jbr_private_t   *priv           = this->private;
-        ssize_t          nbytes;
-        dict_t          *my_xdata;
-
-        if (priv->term_fd < 0) {
-                op_errno = EBADFD;
-                goto err;
-        }
-
-        if (priv->term_read >= priv->term_total) {
-                op_errno = ENODATA;
-                goto err;
-        }
-
-        nbytes = sys_read (priv->term_fd, priv->term_buf, CHANGELOG_ENTRY_SIZE);
-        if (nbytes < CHANGELOG_ENTRY_SIZE) {
-                if (nbytes < 0) {
-                        op_errno = errno;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_SYS_CALL_FAILURE,
-                                "error reading next entry: %s",
-                                strerror(errno));
-                } else {
-                        op_errno = EIO;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_SYS_CALL_FAILURE,
-                                "got %ld/%d bytes for next entry",
-                                nbytes, CHANGELOG_ENTRY_SIZE);
-                }
-                goto err;
-        }
-        ++(priv->term_read);
-
-        my_xdata = dict_new();
-        if (!my_xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_MEM_ERR, "failed to allocate reply xdata");
-                goto err;
-        }
-
-        if (dict_set_static_bin(my_xdata, "data",
-                                priv->term_buf, CHANGELOG_ENTRY_SIZE) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR, "failed to assign reply xdata");
-                goto err;
-        }
-
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, my_xdata);
-        dict_unref(my_xdata);
-        return;
-
-err:
-        STACK_UNWIND_STRICT (ipc, frame, -1, op_errno, NULL);
-}
-
-
-int32_t
-jbr_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
-{
-        switch (op) {
-        case JBR_SERVER_TERM_RANGE:
-                jbr_get_terms(frame, this);
-                break;
-        case JBR_SERVER_OPEN_TERM:
-                jbr_open_term(frame, this, xdata);
-                break;
-        case JBR_SERVER_NEXT_ENTRY:
-                jbr_next_entry(frame, this);
-                break;
-        default:
-                STACK_WIND_TAIL (frame,
-                                 FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->ipc,
-                                 op, xdata);
-        }
-
-        return 0;
-}
-
-
-int32_t
-jbr_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t        ctx     = 0LL;
-
-        if ((inode_ctx_del(inode, this, &ctx) == 0) && ctx) {
-                GF_FREE((void *)(long)ctx);
-        }
-
-        return 0;
-}
-
-int32_t
-jbr_release (xlator_t *this, fd_t *fd)
-{
-        uint64_t        ctx     = 0LL;
-
-        if ((fd_ctx_del(fd, this, &ctx) == 0) && ctx) {
-                GF_FREE((void *)(long)ctx);
-        }
-
-        return 0;
-}
-
-struct xlator_cbks cbks = {
-        .forget  = jbr_forget,
-        .release = jbr_release,
-};
-
-int
-jbr_reconfigure (xlator_t *this, dict_t *options)
-{
-        jbr_private_t   *priv   = this->private;
-
-        GF_OPTION_RECONF ("leader",
-                          priv->config_leader, options, bool, err);
-        GF_OPTION_RECONF ("quorum-percent",
-                          priv->quorum_pct, options, percent, err);
-        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                "reconfigure called, config_leader = %d, quorum_pct = %.1f\n",
-                priv->leader, priv->quorum_pct);
-
-        priv->leader = priv->config_leader;
-
-        return 0;
-
-err:
-        return -1;
-}
-
-int
-jbr_get_child_index (xlator_t *this, xlator_t *kid)
-{
-        xlator_list_t   *trav;
-        int             retval = -1;
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++retval;
-                if (trav->xlator == kid) {
-                        return retval;
-                }
-        }
-
-        return -1;
-}
-
-/*
- * Child notify handling is unreasonably FUBAR.  Sometimes we'll get a
- * CHILD_DOWN for a protocol/client child before we ever got a CHILD_UP for it.
- * Other times we won't.  Because it's effectively random (probably racy), we
- * can't just maintain a count.  We actually have to keep track of the state
- * for each child separately, to filter out the bogus CHILD_DOWN events, and
- * then generate counts on demand.
- */
-int
-jbr_notify (xlator_t *this, int event, void *data, ...)
-{
-        jbr_private_t   *priv         = this->private;
-        int             index         = -1;
-        int             ret           = -1;
-        gf_boolean_t    result        = _gf_false;
-        gf_boolean_t    relevant      = _gf_false;
-
-        switch (event) {
-        case GF_EVENT_CHILD_UP:
-                index = jbr_get_child_index(this, data);
-                if (index >= 0) {
-                        /* Check if the child was previously down
-                         * and it's not a false CHILD_UP
-                         */
-                        if (!(priv->kid_state & (1 << index))) {
-                                relevant = _gf_true;
-                        }
-
-                        priv->kid_state |= (1 << index);
-                        priv->up_children = jbr_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_UP for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                        if (!priv->config_leader && (priv->up_children > 1)) {
-                                priv->leader = _gf_false;
-                        }
-
-                        /* If it's not relevant, or we have already *
-                         * sent CHILD_UP just break */
-                        if (!relevant || priv->child_up)
-                                break;
-
-                        /* If it's not a leader, just send the notify up */
-                        if (!priv->leader) {
-                                ret = default_notify(this, event, data);
-                                if (!ret)
-                                        priv->child_up = _gf_true;
-                                break;
-                        }
-
-                        result = fop_quorum_check (this,
-                                                (double)(priv->n_children - 1),
-                                               (double)(priv->up_children - 1));
-                        if (result == _gf_false) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Not enough children "
-                                        "are up to meet quorum. Waiting to "
-                                        "send CHILD_UP from leader");
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Enough children are up "
-                                        "to meet quorum. Sending CHILD_UP "
-                                        "from leader");
-                                ret = default_notify(this, event, data);
-                                if (!ret)
-                                        priv->child_up = _gf_true;
-                        }
-                }
-                break;
-        case GF_EVENT_CHILD_DOWN:
-                index = jbr_get_child_index(this, data);
-                if (index >= 0) {
-                        /* Check if the child was previously up
-                         * and it's not a false CHILD_DOWN
-                         */
-                        if (priv->kid_state & (1 << index)) {
-                                relevant = _gf_true;
-                        }
-                        priv->kid_state &= ~(1 << index);
-                        priv->up_children = jbr_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_DOWN for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                        if (!priv->config_leader && (priv->up_children < 2)
-                            && relevant) {
-                                priv->leader = _gf_true;
-                        }
-
-                        /* If it's not relevant, or we have already *
-                         * sent CHILD_DOWN just break */
-                        if (!relevant || !priv->child_up)
-                                break;
-
-                        /* If it's not a leader, just break coz we shouldn't  *
-                         * propagate the failure from the failure till it     *
-                         * itself goes down                                   *
-                         */
-                        if (!priv->leader) {
-                                break;
-                        }
-
-                        result = fop_quorum_check (this,
-                                           (double)(priv->n_children - 1),
-                                           (double)(priv->up_children - 1));
-                        if (result == _gf_false) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Enough children are "
-                                        "to down to fail quorum. "
-                                        "Sending CHILD_DOWN from leader");
-                                ret = default_notify(this, event, data);
-                                if (!ret)
-                                        priv->child_up = _gf_false;
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Not enough children "
-                                        "are down to fail quorum. Waiting to "
-                                        "send CHILD_DOWN from leader");
-                        }
-                }
-                break;
-        default:
-                ret = default_notify(this, event, data);
-        }
-
-        return ret;
-}
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_mt_jbr_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_MEM_ERR,
-                        "Memory accounting init" "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-void
-jbr_deallocate_priv (jbr_private_t *priv)
-{
-        if (!priv) {
-                return;
-        }
-
-        GF_FREE(priv);
-}
-
-
-int32_t
-jbr_init (xlator_t *this)
-{
-        xlator_list_t   *remote;
-        xlator_list_t   *local;
-        jbr_private_t   *priv           = NULL;
-        xlator_list_t   *trav;
-        pthread_t       kid;
-        extern xlator_t global_xlator;
-        glusterfs_ctx_t *oldctx         = global_xlator.ctx;
-
-        /*
-         * Any fop that gets special treatment has to be patched in here,
-         * because the compiled-in table is produced by the code generator and
-         * only contains generated functions.  Note that we have to go through
-         * this->fops because of some dynamic-linking strangeness; modifying
-         * the static table doesn't work.
-         */
-        this->fops->getxattr = jbr_getxattr_special;
-        this->fops->fsync = jbr_fsync;
-        this->fops->ipc = jbr_ipc;
-
-        local = this->children;
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_NO_DATA,
-                        "no local subvolume");
-                goto err;
-        }
-
-        remote = local->next;
-        if (!remote) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_NO_DATA,
-                        "no remote subvolumes");
-                goto err;
-        }
-
-        this->local_pool = mem_pool_new (jbr_local_t, 128);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_MEM_ERR,
-                        "failed to create jbr_local_t pool");
-                goto err;
-        }
-
-        priv = GF_CALLOC (1, sizeof(*priv), gf_mt_jbr_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_MEM_ERR,
-                        "could not allocate priv");
-                goto err;
-        }
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++(priv->n_children);
-        }
-
-        LOCK_INIT(&priv->dirty_lock);
-	LOCK_INIT(&priv->index_lock);
-        INIT_LIST_HEAD(&priv->dirty_fds);
-        priv->term_fd = -1;
-
-        this->private = priv;
-
-        GF_OPTION_INIT ("leader", priv->config_leader, bool, err);
-        GF_OPTION_INIT ("quorum-percent", priv->quorum_pct, percent, err);
-
-        priv->leader = priv->config_leader;
-        priv->child_up = _gf_false;
-
-        if (pthread_create(&kid, NULL, jbr_flush_thread,
-                           this) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_SYS_CALL_FAILURE,
-                        "could not start flush thread");
-                /* TBD: treat this as a fatal error? */
-        }
-
-        /*
-         * Calling glfs_new changes old->ctx, even if THIS still points
-         * to global_xlator.  That causes problems later in the main
-         * thread, when gf_log_dump_graph tries to use the FILE after
-         * we've mucked with it and gets a segfault in __fprintf_chk.
-         * We can avoid all that by undoing the damage before we
-         * continue.
-         */
-        global_xlator.ctx = oldctx;
-
-	return 0;
-
-err:
-        jbr_deallocate_priv(priv);
-        return -1;
-}
-
-
-void
-jbr_fini (xlator_t *this)
-{
-        jbr_deallocate_priv(this->private);
-}
-
-class_methods_t class_methods = {
-        .init           = jbr_init,
-        .fini           = jbr_fini,
-        .reconfigure    = jbr_reconfigure,
-        .notify         = jbr_notify,
-};
-
-struct volume_options options[] = {
-        { .key = {"leader"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false",
-          .description = "Start in the leader role.  This is only for "
-                         "bootstrapping the code, and should go away when we "
-                         "have real leader election."
-        },
-        { .key = {"vol-name"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "volume name"
-        },
-        { .key = {"my-name"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "brick name in form of host:/path"
-        },
-        { .key = {"etcd-servers"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "list of comma separated etc servers"
-        },
-        { .key = {"subvol-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "UUID for this JBR (sub)volume"
-        },
-        { .key = {"quorum-percent"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "50.0",
-          .description = "percentage of rep_count-1 that must be up"
-        },
-	{ .key = {NULL} },
-};
diff --git a/xlators/experimental/posix2/Makefile.am b/xlators/experimental/posix2/Makefile.am
deleted file mode 100644
index 74e5ab0f5bc..00000000000
--- a/xlators/experimental/posix2/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = common mds ds
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/README.md b/xlators/experimental/posix2/README.md
deleted file mode 100644
index 955a98d061e..00000000000
--- a/xlators/experimental/posix2/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# POSIX2 Experimental README
-
-POSIX2 is an implementation of modified storage translator to cater to DHT2
-on disk needs.
-
-For further understanding, refer to xlators/experimental/dht2/README.md for
-details regarding POSIX2
diff --git a/xlators/experimental/posix2/TODO.md b/xlators/experimental/posix2/TODO.md
deleted file mode 100644
index 20cd1e89c1d..00000000000
--- a/xlators/experimental/posix2/TODO.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# POSIX2 TODO List
-
-<Items will be added as code is pulled into the repository>
\ No newline at end of file
diff --git a/xlators/experimental/posix2/common/src/Makefile.am b/xlators/experimental/posix2/common/src/Makefile.am
deleted file mode 100644
index 465f2f2ba32..00000000000
--- a/xlators/experimental/posix2/common/src/Makefile.am
+++ /dev/null
@@ -1,13 +0,0 @@
-lib_LTLIBRARIES = libposix2common.la
-
-posix2_common_sources = posix2-common.c
-
-libposix2common_la_SOURCES = $(posix2_common_sources)
-libposix2common_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-libposix2common_la_CFLAGS = -Wall $(GF_CFLAGS)
-
-libposix2common_la_CPPFLAGS = $(GF_CPPFLAGS)
-libposix2common_la_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-
-CLEANFILES =
\ No newline at end of file
diff --git a/xlators/experimental/posix2/ds/src/Makefile.am b/xlators/experimental/posix2/ds/src/Makefile.am
deleted file mode 100644
index d29c5e135a2..00000000000
--- a/xlators/experimental/posix2/ds/src/Makefile.am
+++ /dev/null
@@ -1,18 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-xlator_LTLIBRARIES = posix2-ds.la
-
-posix2_ds_sources = posix2-ds-main.c
-
-posix2_ds_la_SOURCES = $(posix2_ds_sources)
-posix2_ds_la_LDFLAGS = -module -avoid-version
-posix2_ds_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-posix2_ds_la_LIBADD += $(top_builddir)/xlators/experimental/posix2/common/src/libposix2common.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/storage/posix2/common/src
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-
-CLEANFILES =
\ No newline at end of file
diff --git a/xlators/experimental/posix2/ds/src/posix2-ds-main.c b/xlators/experimental/posix2/ds/src/posix2-ds-main.c
deleted file mode 100644
index 675c4d7c9da..00000000000
--- a/xlators/experimental/posix2/ds/src/posix2-ds-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: posix2-ds-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-posix2_ds_init (xlator_t *this)
-{
-        if (this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "This (%s) is a leaf xlator, but found children",
-                        this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-posix2_ds_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = posix2_ds_init,
-        .fini           = posix2_ds_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/experimental/posix2/mds/src/Makefile.am b/xlators/experimental/posix2/mds/src/Makefile.am
deleted file mode 100644
index ddd49ef0012..00000000000
--- a/xlators/experimental/posix2/mds/src/Makefile.am
+++ /dev/null
@@ -1,18 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-xlator_LTLIBRARIES = posix2-mds.la
-
-posix2_mds_sources = posix2-mds-main.c
-
-posix2_mds_la_SOURCES = $(posix2_mds_sources)
-posix2_mds_la_LDFLAGS = -module -avoid-version
-posix2_mds_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-posix2_mds_la_LIBADD += $(top_builddir)/xlators/experimental/posix2/common/src/libposix2common.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/storage/posix2/common/src
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-
-CLEANFILES =
\ No newline at end of file
diff --git a/xlators/experimental/posix2/mds/src/posix2-mds-main.c b/xlators/experimental/posix2/mds/src/posix2-mds-main.c
deleted file mode 100644
index 71ff4e0089c..00000000000
--- a/xlators/experimental/posix2/mds/src/posix2-mds-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: posix2-mds-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-posix2_mds_init (xlator_t *this)
-{
-        if (this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "This (%s) is a leaf xlator, but found children",
-                        this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-posix2_mds_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = posix2_mds_init,
-        .fini           = posix2_mds_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index c63eb75a7c3..c57897f11ea 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,6 +1,14 @@
-SUBDIRS = locks quota read-only mac-compat quiesce marker index barrier \
-	  arbiter protect compress changelog changetimerecorder ganesha \
-	  gfid-access $(GLUPY_SUBDIR) upcall snapview-client snapview-server \
-	  trash shard bit-rot leases
+if BUILD_CLOUDSYNC
+  CLOUDSYNC_DIR = cloudsync
+endif
+
+if BUILD_METADISP
+  METADISP_DIR = metadisp
+endif
+
+SUBDIRS = locks quota read-only quiesce marker index barrier arbiter upcall \
+	compress changelog gfid-access snapview-client snapview-server trash \
+	shard bit-rot leases selinux sdfs namespace $(CLOUDSYNC_DIR) thin-arbiter \
+	utime $(METADISP_DIR)
 
 CLEANFILES =
diff --git a/xlators/features/arbiter/src/Makefile.am b/xlators/features/arbiter/src/Makefile.am
index dd262c3d6dc..badc42f37be 100644
--- a/xlators/features/arbiter/src/Makefile.am
+++ b/xlators/features/arbiter/src/Makefile.am
@@ -1,4 +1,7 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = arbiter.la
+endif
+
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 arbiter_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
@@ -8,7 +11,8 @@ arbiter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = arbiter.h arbiter-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/arbiter/src/arbiter-mem-types.h b/xlators/features/arbiter/src/arbiter-mem-types.h
index 200b59de695..05d18374c46 100644
--- a/xlators/features/arbiter/src/arbiter-mem-types.h
+++ b/xlators/features/arbiter/src/arbiter-mem-types.h
@@ -9,11 +9,10 @@
 
 #ifndef __ARBITER_MEM_TYPES_H__
 #define __ARBITER_MEM_TYPES_H__
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 typedef enum gf_arbiter_mem_types_ {
-        gf_arbiter_mt_inode_ctx_t =  gf_common_mt_end + 1,
-        gf_arbiter_mt_iatt,
-        gf_arbiter_mt_end
+    gf_arbiter_mt_inode_ctx_t = gf_common_mt_end + 1,
+    gf_arbiter_mt_end
 } gf_arbiter_mem_types_t;
 #endif
diff --git a/xlators/features/arbiter/src/arbiter.c b/xlators/features/arbiter/src/arbiter.c
index 786f60b7bc9..83a97e3354b 100644
--- a/xlators/features/arbiter/src/arbiter.c
+++ b/xlators/features/arbiter/src/arbiter.c
@@ -10,351 +10,371 @@
 
 #include "arbiter.h"
 #include "arbiter-mem-types.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-
-void
-arbiter_inode_ctx_destroy (arbiter_inode_ctx_t *ctx)
-{
-        if (!ctx)
-                return;
-        GF_FREE (ctx->iattbuf);
-        GF_FREE (ctx);
-}
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
 
 static arbiter_inode_ctx_t *
-__arbiter_inode_ctx_get (inode_t *inode, xlator_t *this)
+__arbiter_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-
-        arbiter_inode_ctx_t *ctx = NULL;
-        int ret = 0;
-        uint64_t ctx_addr = 0;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret == 0) {
-                ctx = (arbiter_inode_ctx_t *) (long) ctx_addr;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_arbiter_mt_inode_ctx_t);
-        if (!ctx)
-                goto fail;
-        ctx->iattbuf = GF_CALLOC (1, sizeof (*ctx->iattbuf),
-                                  gf_arbiter_mt_iatt);
-        if (!ctx->iattbuf)
-                goto fail;
-        ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-        if (ret) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                  "set the inode ctx (%s)",
-                                  uuid_utoa (inode->gfid));
-                goto fail;
-        }
+    arbiter_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+    uint64_t ctx_addr = 0;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret == 0) {
+        ctx = (arbiter_inode_ctx_t *)(long)ctx_addr;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_arbiter_mt_inode_ctx_t);
+    if (!ctx)
+        goto out;
+
+    ret = __inode_ctx_put(inode, this, (uint64_t)(uintptr_t)ctx);
+    if (ret) {
+        GF_FREE(ctx);
+        ctx = NULL;
+        gf_log_callingfn(this->name, GF_LOG_ERROR,
+                         "failed to "
+                         "set the inode ctx (%s)",
+                         uuid_utoa(inode->gfid));
+    }
 out:
-        return ctx;
-fail:
-        arbiter_inode_ctx_destroy (ctx);
-        return NULL;
+    return ctx;
 }
 
 static arbiter_inode_ctx_t *
-arbiter_inode_ctx_get (inode_t *inode, xlator_t *this)
+arbiter_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-
-        LOCK(&inode->lock);
-        {
-                ctx = __arbiter_inode_ctx_get (inode, this);
-        }
-        UNLOCK(&inode->lock);
-        return ctx;
+    arbiter_inode_ctx_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __arbiter_inode_ctx_get(inode, this);
+    }
+    UNLOCK(&inode->lock);
+    return ctx;
 }
 
 int32_t
-arbiter_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+arbiter_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-
-        if (op_ret != 0)
-                goto unwind;
-        ctx = arbiter_inode_ctx_get (inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        memcpy (ctx->iattbuf, buf, sizeof (*ctx->iattbuf));
+    arbiter_inode_ctx_t *ctx = NULL;
+
+    if (op_ret != 0)
+        goto unwind;
+    ctx = arbiter_inode_ctx_get(inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    memcpy(&ctx->iattbuf, buf, sizeof(ctx->iattbuf));
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
 int32_t
-arbiter_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+arbiter_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, arbiter_lookup_cbk, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
+    STACK_WIND(frame, arbiter_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
 }
 
 int32_t
-arbiter_readv (call_frame_t *frame,  xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, uint32_t flags, dict_t *xdata)
+arbiter_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readv, frame, -1, ENOTCONN, NULL, 0, NULL, NULL,
-                             NULL);
-        return 0;
-}
-
-int32_t
-arbiter_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                  dict_t *xdata)
-{
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int32_t              op_ret   = 0;
-        int32_t              op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (loc->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(loc->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   dict_t *xdata)
+arbiter_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int32_t              op_ret   = 0;
-        int32_t              op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, buf, buf,
-                             NULL);
-        return 0;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
-dict_t*
-arbiter_fill_writev_xdata (fd_t *fd, dict_t *xdata, xlator_t *this)
+dict_t *
+arbiter_fill_writev_xdata(fd_t *fd, dict_t *xdata, xlator_t *this)
 {
-        dict_t  *rsp_xdata = NULL;
-        int32_t ret = 0;
-        int is_append = 1;
-
-        if (!fd || !fd->inode || gf_uuid_is_null (fd->inode->gfid)) {
-                goto out;
+    dict_t *rsp_xdata = NULL;
+    int32_t ret = 0;
+    int is_append = 1;
+
+    if (!fd || !fd->inode || gf_uuid_is_null(fd->inode->gfid)) {
+        goto out;
+    }
+
+    if (!xdata)
+        goto out;
+
+    rsp_xdata = dict_new();
+    if (!rsp_xdata)
+        goto out;
+
+    if (dict_get(xdata, GLUSTERFS_OPEN_FD_COUNT)) {
+        ret = dict_set_uint32(rsp_xdata, GLUSTERFS_OPEN_FD_COUNT,
+                              fd->inode->fd_count);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dict value"
+                         " for GLUSTERFS_OPEN_FD_COUNT");
         }
-
-        if (!xdata)
-                goto out;
-
-        rsp_xdata = dict_new();
-        if (!rsp_xdata)
-                goto out;
-
-        if (dict_get (xdata, GLUSTERFS_OPEN_FD_COUNT)) {
-                ret = dict_set_uint32 (rsp_xdata, GLUSTERFS_OPEN_FD_COUNT,
-                                       fd->inode->fd_count);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0, "Failed to set dict value"
-                                      " for GLUSTERFS_OPEN_FD_COUNT");
-                }
-        }
-        if (dict_get (xdata, GLUSTERFS_WRITE_IS_APPEND)) {
-                ret = dict_set_uint32 (rsp_xdata, GLUSTERFS_WRITE_IS_APPEND,
-                                       is_append);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0, "Failed to set dict value"
-                                      " for GLUSTERFS_WRITE_IS_APPEND");
-                }
+    }
+    if (dict_get(xdata, GLUSTERFS_WRITE_IS_APPEND)) {
+        ret = dict_set_uint32(rsp_xdata, GLUSTERFS_WRITE_IS_APPEND, is_append);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dict value"
+                         " for GLUSTERFS_WRITE_IS_APPEND");
         }
+    }
 out:
-        return rsp_xdata;
+    return rsp_xdata;
 }
 
 int32_t
-arbiter_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
+arbiter_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        dict_t           *rsp_xdata   = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
-        op_ret = iov_length (vector, count);
-        rsp_xdata = arbiter_fill_writev_xdata (fd, xdata, this);
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    dict_t *rsp_xdata = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
+    op_ret = iov_length(vector, count);
+    rsp_xdata = arbiter_fill_writev_xdata(fd, xdata, this);
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, buf, buf,
-                             rsp_xdata);
-        if (rsp_xdata)
-                dict_unref (rsp_xdata);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, buf, buf, rsp_xdata);
+    if (rsp_xdata)
+        dict_unref(rsp_xdata);
+    return 0;
 }
 
 int32_t
-arbiter_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+arbiter_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
                   int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_discard (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                off_t offset, size_t len, dict_t *xdata)
+arbiter_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  off_t offset, off_t len, dict_t *xdata)
+arbiter_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
-int32_t
-mem_acct_init (xlator_t *this)
+static int32_t
+arbiter_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(readv, frame, -1, ENOSYS, NULL, 0, NULL, NULL, NULL);
+    return 0;
+}
+
+static int32_t
+arbiter_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             gf_seek_what_t what, dict_t *xdata)
 {
-        int ret = -1;
+    STACK_UNWIND_STRICT(seek, frame, -1, ENOSYS, 0, xdata);
+    return 0;
+}
 
-        ret = xlator_mem_acct_init (this, gf_arbiter_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
-        return ret;
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_arbiter_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-
-        return 0;
+    return 0;
 }
 
 int
-arbiter_forget (xlator_t *this, inode_t *inode)
+arbiter_forget(xlator_t *this, inode_t *inode)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-        uint64_t ctx_addr = 0;
-
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
-        ctx = (arbiter_inode_ctx_t *) (long) ctx_addr;
-        GF_FREE (ctx);
+    arbiter_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
         return 0;
+    ctx = (arbiter_inode_ctx_t *)(long)ctx_addr;
+    GF_FREE(ctx);
+    return 0;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'arbiter' not configured with exactly one child");
+        return -1;
+    }
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "'arbiter' not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dangling volume. check volfile ");
+    if (!this->parents)
+        gf_log(this->name, GF_LOG_ERROR, "dangling volume. check volfile ");
 
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        return;
+    return;
 }
 
 struct xlator_fops fops = {
-        .lookup = arbiter_lookup,
-        .readv  = arbiter_readv,
-        .truncate = arbiter_truncate,
-        .writev = arbiter_writev,
-        .ftruncate = arbiter_ftruncate,
-        .fallocate = arbiter_fallocate,
-        .discard = arbiter_discard,
-        .zerofill = arbiter_zerofill,
+    .lookup = arbiter_lookup,
+
+    /* Return success for these inode write FOPS without winding it down to
+     * posix; this is needed for AFR write transaction logic to work.*/
+    .truncate = arbiter_truncate,
+    .writev = arbiter_writev,
+    .ftruncate = arbiter_ftruncate,
+    .fallocate = arbiter_fallocate,
+    .discard = arbiter_discard,
+    .zerofill = arbiter_zerofill,
+
+    /* AFR is not expected to wind these inode read FOPS initiated by the
+     * application to the arbiter brick. But in case a bug causes them
+     * to be called, we return ENOSYS. */
+    .readv = arbiter_readv,
+    .seek = arbiter_seek,
+
+    /* The following inode read FOPS initiated by the application are not
+     * wound by AFR either but internal logic like  shd, glfsheal and
+     * client side healing in AFR will send them for selfheal/ inode refresh
+     * operations etc.,so we need to wind them down to posix:
+     *
+     * (f)stat, readdir(p), readlink, (f)getxattr.*/
+
+    /* All other FOPs not listed here are safe to be wound down to posix.*/
 };
 
 struct xlator_cbks cbks = {
-        .forget = arbiter_forget,
+    .forget = arbiter_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {NULL} },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "arbiter",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/arbiter/src/arbiter.h b/xlators/features/arbiter/src/arbiter.h
index 6ccc3add3b3..546db7b751a 100644
--- a/xlators/features/arbiter/src/arbiter.h
+++ b/xlators/features/arbiter/src/arbiter.h
@@ -11,11 +11,11 @@
 #ifndef _ARBITER_H
 #define _ARBITER_H
 
-#include "locking.h"
-#include "common-utils.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/common-utils.h>
 
 typedef struct arbiter_inode_ctx_ {
-        struct iatt *iattbuf;
+    struct iatt iattbuf;
 } arbiter_inode_ctx_t;
 
 #endif /* _ARBITER_H */
diff --git a/xlators/features/barrier/src/Makefile.am b/xlators/features/barrier/src/Makefile.am
index 4e909c8aad8..25099bc56e5 100644
--- a/xlators/features/barrier/src/Makefile.am
+++ b/xlators/features/barrier/src/Makefile.am
@@ -9,7 +9,8 @@ barrier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = barrier.h barrier-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/barrier/src/barrier-mem-types.h b/xlators/features/barrier/src/barrier-mem-types.h
index 36647a66966..71ed7898d9c 100644
--- a/xlators/features/barrier/src/barrier-mem-types.h
+++ b/xlators/features/barrier/src/barrier-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __BARRIER_MEM_TYPES_H__
 #define __BARRIER_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_barrier_mem_types_ {
-        gf_barrier_mt_priv_t = gf_common_mt_end + 1,
-        gf_barrier_mt_end
+    gf_barrier_mt_priv_t = gf_common_mt_end + 1,
+    gf_barrier_mt_end
 };
 #endif
diff --git a/xlators/features/barrier/src/barrier.c b/xlators/features/barrier/src/barrier.c
index ce3a255d93e..852bbacb99d 100644
--- a/xlators/features/barrier/src/barrier.c
+++ b/xlators/features/barrier/src/barrier.c
@@ -9,791 +9,801 @@
 */
 
 #include "barrier.h"
-#include "defaults.h"
-#include "call-stub.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
 
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 
 void
-barrier_local_set_gfid (call_frame_t *frame, uuid_t gfid, xlator_t *this)
-{
-        if (gfid) {
-                uuid_t *id = GF_MALLOC (sizeof (uuid_t), gf_common_mt_uuid_t);
-                if (!id) {
-                        gf_log (this->name, GF_LOG_WARNING, "Could not set gfid"
-                                ". gfid will not be dumped in statedump file.");
-                        return;
-                }
-                gf_uuid_copy (*id, gfid);
-                frame->local = id;
+barrier_local_set_gfid(call_frame_t *frame, uuid_t gfid, xlator_t *this)
+{
+    if (gfid) {
+        uuid_t *id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!id) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Could not set gfid"
+                   ". gfid will not be dumped in statedump file.");
+            return;
         }
+        gf_uuid_copy(*id, gfid);
+        frame->local = id;
+    }
 }
 
 void
-barrier_local_free_gfid (call_frame_t *frame)
+barrier_local_free_gfid(call_frame_t *frame)
 {
-        if (frame->local) {
-                GF_FREE (frame->local);
-                frame->local = NULL;
-        }
+    if (frame->local) {
+        GF_FREE(frame->local);
+        frame->local = NULL;
+    }
 }
 
 int32_t
-barrier_truncate_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_truncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *prebuf, struct iatt *postbuf,
+                            dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
+}
+
+int32_t
+barrier_ftruncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
                              struct iatt *prebuf, struct iatt *postbuf,
                              dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_ftruncate_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno,
-                              struct iatt *prebuf, struct iatt *postbuf,
-                              dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+barrier_unlink_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_unlink_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+barrier_rmdir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_rmdir_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *preparent, struct iatt *postparent,
+barrier_rename_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
                           dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
 int32_t
-barrier_rename_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                           struct iatt *preoldparent, struct iatt *postoldparent,
-                           struct iatt *prenewparent, struct iatt *postnewparent,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, xdata);
-        return 0;
+barrier_writev_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_writev_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+barrier_fsync_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fsync_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                          struct iatt *postbuf, dict_t *xdata)
+barrier_removexattr_cbk_resume(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-barrier_removexattr_cbk_resume (call_frame_t *frame, void *cookie,
+barrier_fremovexattr_cbk_resume(call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
                                 int32_t op_errno, dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr_cbk_resume (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+barrier_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    BARRIER_FOP_CBK(writev, out, frame, this, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+out:
+    return 0;
 }
 
 int32_t
-barrier_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
+barrier_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (writev, out, frame, this, op_ret, op_errno,
-                         prebuf, postbuf, xdata);
+    BARRIER_FOP_CBK(fremovexattr, out, frame, this, op_ret, op_errno, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+barrier_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (fremovexattr, out, frame, this, op_ret, op_errno,
-                         xdata);
+    BARRIER_FOP_CBK(removexattr, out, frame, this, op_ret, op_errno, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+barrier_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (removexattr, out, frame, this, op_ret, op_errno,
-                         xdata);
+    BARRIER_FOP_CBK(truncate, out, frame, this, op_ret, op_errno, prebuf,
+                    postbuf, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                       struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (truncate, out, frame, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+    BARRIER_FOP_CBK(ftruncate, out, frame, this, op_ret, op_errno, prebuf,
+                    postbuf, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
-{
-        BARRIER_FOP_CBK (ftruncate, out, frame, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+barrier_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+    BARRIER_FOP_CBK(rename, out, frame, this, op_ret, op_errno, buf,
+                    preoldparent, postoldparent, prenewparent, postnewparent,
+                    xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
-        BARRIER_FOP_CBK (rename, out, frame, this, op_ret, op_errno, buf,
-                         preoldparent, postoldparent, prenewparent,
-                         postnewparent, xdata);
+barrier_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    BARRIER_FOP_CBK(rmdir, out, frame, this, op_ret, op_errno, preparent,
+                    postparent, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (rmdir, out, frame, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
+    BARRIER_FOP_CBK(unlink, out, frame, this, op_ret, op_errno, preparent,
+                    postparent, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+barrier_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (unlink, out, frame, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
+    BARRIER_FOP_CBK(fsync, out, frame, this, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+barrier_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (fsync, out, frame, this, op_ret, op_errno,
-                         prebuf, postbuf, xdata);
-out:
-        return 0;
-}
-
-int32_t
-barrier_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
-{
-        if (!((flags | fd->flags) & (O_SYNC | O_DSYNC))) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->writev,
-                                 fd, vector, count, off, flags, iobref, xdata);
-
-                return 0;
-        }
+    if (!((flags | fd->flags) & (O_SYNC | O_DSYNC))) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                        flags, iobref, xdata);
 
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    off, flags, iobref, xdata);
         return 0;
+    }
+
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
+barrier_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-barrier_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
+barrier_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 int32_t
-barrier_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
+barrier_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int32_t
-barrier_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
+barrier_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, oldloc->inode->gfid, this);
-        STACK_WIND (frame, barrier_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, oldloc->inode->gfid, this);
+    STACK_WIND(frame, barrier_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-barrier_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-               dict_t *xdata)
+barrier_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_rmdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rmdir,
-                    loc, flags, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 }
 
 int32_t
-barrier_unlink (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int xflag, dict_t *xdata)
+barrier_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
 int32_t
-barrier_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
+barrier_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_ftruncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               int32_t flags, dict_t *xdata)
+barrier_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 }
 
 call_stub_t *
-__barrier_dequeue (xlator_t *this, struct list_head *queue)
+__barrier_dequeue(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t    *stub            = NULL;
-        barrier_priv_t *priv            = NULL;
+    call_stub_t *stub = NULL;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (list_empty (queue))
-                goto out;
+    if (list_empty(queue))
+        goto out;
 
-        stub = list_entry (queue->next, call_stub_t, list);
-        list_del_init (&stub->list);
+    stub = list_entry(queue->next, call_stub_t, list);
+    list_del_init(&stub->list);
 
 out:
-        return stub;
+    return stub;
 }
 
 void
-barrier_dequeue_all (xlator_t *this, struct list_head *queue)
+barrier_dequeue_all(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t            *stub    = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_log (this->name, GF_LOG_INFO, "Dequeuing all the barriered fops");
+    gf_log(this->name, GF_LOG_INFO, "Dequeuing all the barriered fops");
 
-        /* TODO: Start the below task in a new thread */
-        while ((stub = __barrier_dequeue (this, queue)))
-                call_resume (stub);
+    /* TODO: Start the below task in a new thread */
+    while ((stub = __barrier_dequeue(this, queue)))
+        call_resume(stub);
 
-        gf_log (this->name, GF_LOG_INFO, "Dequeuing the barriered fops is "
-                                         "finished");
-        return;
+    gf_log(this->name, GF_LOG_INFO,
+           "Dequeuing the barriered fops is "
+           "finished");
+    return;
 }
 
 void
-barrier_timeout (void *data)
+barrier_timeout(void *data)
 {
-        xlator_t               *this    = NULL;
-        barrier_priv_t         *priv    = NULL;
-        struct list_head        queue   = {0,};
+    xlator_t *this = NULL;
+    barrier_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        this = data;
-        THIS = this;
-        priv = this->private;
+    this = data;
+    THIS = this;
+    priv = this->private;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_log (this->name, GF_LOG_CRITICAL, "Disabling barrier because of "
-                                             "the barrier timeout.");
+    gf_log(this->name, GF_LOG_CRITICAL,
+           "Disabling barrier because of "
+           "the barrier timeout.");
 
-        LOCK (&priv->lock);
-        {
-                __barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        __barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        barrier_dequeue_all (this, &queue);
+    barrier_dequeue_all(this, &queue);
 
-        return;
+    return;
 }
 
 void
-__barrier_enqueue (xlator_t *this, call_stub_t *stub)
+__barrier_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        barrier_priv_t *priv    = NULL;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        list_add_tail (&stub->list, &priv->queue);
-        priv->queue_size++;
+    list_add_tail(&stub->list, &priv->queue);
+    priv->queue_size++;
 
-        return;
+    return;
 }
 
 void
-__barrier_disable (xlator_t *this, struct list_head *queue)
+__barrier_disable(xlator_t *this, struct list_head *queue)
 {
-        GF_UNUSED int   ret     = 0;
-        barrier_priv_t *priv    = NULL;
+    GF_UNUSED int ret = 0;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->timer) {
-                ret = gf_timer_call_cancel (this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
+    if (priv->timer) {
+        ret = gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
 
-        list_splice_init (&priv->queue, queue);
-        priv->queue_size = 0;
-        priv->barrier_enabled = _gf_false;
+    list_splice_init(&priv->queue, queue);
+    priv->queue_size = 0;
+    priv->barrier_enabled = _gf_false;
 }
 
 int
-__barrier_enable (xlator_t *this, barrier_priv_t *priv)
+__barrier_enable(xlator_t *this, barrier_priv_t *priv)
 {
-        int             ret     = -1;
+    int ret = -1;
 
-        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
-                                           barrier_timeout, (void *) this);
-        if (!priv->timer) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Couldn't add barrier "
-                                                     "timeout event.");
-                goto out;
-        }
+    priv->timer = gf_timer_call_after(this->ctx, priv->timeout, barrier_timeout,
+                                      (void *)this);
+    if (!priv->timer) {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "Couldn't add barrier "
+               "timeout event.");
+        goto out;
+    }
 
-        priv->barrier_enabled = _gf_true;
-        ret = 0;
+    priv->barrier_enabled = _gf_true;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        barrier_priv_t  *priv                   = NULL;
-        dict_t          *dict                   = NULL;
-        gf_boolean_t     past                   = _gf_false;
-        int              ret                    = -1;
-        int              barrier_enabled        = _gf_false;
-        struct list_head queue                  = {0,};
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        INIT_LIST_HEAD (&queue);
-
-        switch (event) {
-        case GF_EVENT_TRANSLATOR_OP:
-        {
-                dict = data;
-                barrier_enabled = dict_get_str_boolean (dict, "barrier", -1);
-
-                if (barrier_enabled == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not fetch "
-                                " barrier key from the dictionary.");
-                        goto out;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        past = priv->barrier_enabled;
-
-                        switch (past) {
-                        case _gf_false:
-                                if (barrier_enabled) {
-                                        ret = __barrier_enable (this,priv);
-                                        if (ret)
-                                                goto unlock;
-                                } else {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Already disabled.");
-                                        goto unlock;
-                                }
-                                break;
-
-                        case _gf_true:
-                                if (!barrier_enabled) {
-                                        __barrier_disable(this, &queue);
-                                } else {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Already enabled");
-                                        goto unlock;
-                                }
-                                break;
-                        }
+notify(xlator_t *this, int event, void *data, ...)
+{
+    barrier_priv_t *priv = this->private;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int barrier_enabled = _gf_false;
+    struct list_head queue = {
+        0,
+    };
+
+    GF_ASSERT(priv);
+    INIT_LIST_HEAD(&queue);
+
+    switch (event) {
+        case GF_EVENT_TRANSLATOR_OP: {
+            dict = data;
+            barrier_enabled = dict_get_str_boolean(dict, "barrier", -1);
+
+            if (barrier_enabled == -1) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Could not fetch "
+                       " barrier key from the dictionary.");
+                goto out;
+            }
+
+            LOCK(&priv->lock);
+            {
+                if (!priv->barrier_enabled) {
+                    if (barrier_enabled) {
+                        ret = __barrier_enable(this, priv);
+                    } else {
+                        UNLOCK(&priv->lock);
+                        gf_log(this->name, GF_LOG_ERROR, "Already disabled.");
+                        goto post_unlock;
+                    }
+                } else {
+                    if (!barrier_enabled) {
+                        __barrier_disable(this, &queue);
                         ret = 0;
+                    } else {
+                        UNLOCK(&priv->lock);
+                        gf_log(this->name, GF_LOG_ERROR, "Already enabled");
+                        goto post_unlock;
+                    }
                 }
-unlock:
-                UNLOCK (&priv->lock);
+            }
+            UNLOCK(&priv->lock);
+        post_unlock:
+            if (!list_empty(&queue))
+                barrier_dequeue_all(this, &queue);
 
-                if (!list_empty (&queue))
-                        barrier_dequeue_all (this, &queue);
-
-                break;
-        }
-        default:
-        {
-                default_notify (this, event, data);
-                ret = 0;
-                goto out;
+            break;
         }
+        default: {
+            default_notify(this, event, data);
+            ret = 0;
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        barrier_priv_t  *priv                   = NULL;
-        gf_boolean_t     past                   = _gf_false;
-        int              ret                    = -1;
-        gf_boolean_t     barrier_enabled        = _gf_false;
-        uint32_t         timeout                = {0,};
-        struct list_head queue                  = {0,};
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_OPTION_RECONF ("barrier", barrier_enabled, options, bool, out);
-        GF_OPTION_RECONF ("barrier-timeout", timeout, options, time, out);
-
-        INIT_LIST_HEAD (&queue);
-
-        LOCK (&priv->lock);
-        {
-                past = priv->barrier_enabled;
-
-                switch (past) {
-                case _gf_false:
-                        if (barrier_enabled) {
-                                ret = __barrier_enable (this, priv);
-                                if (ret) {
-                                        goto unlock;
-                                }
-                        }
-                        break;
-
-                case _gf_true:
-                        if (!barrier_enabled) {
-                                __barrier_disable (this, &queue);
-
-                        }
-                        break;
+reconfigure(xlator_t *this, dict_t *options)
+{
+    barrier_priv_t *priv = NULL;
+    int ret = -1;
+    gf_boolean_t barrier_enabled = _gf_false;
+    uint32_t timeout = {
+        0,
+    };
+    struct list_head queue = {
+        0,
+    };
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_OPTION_RECONF("barrier", barrier_enabled, options, bool, out);
+    GF_OPTION_RECONF("barrier-timeout", timeout, options, time, out);
+
+    INIT_LIST_HEAD(&queue);
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->barrier_enabled) {
+            if (barrier_enabled) {
+                ret = __barrier_enable(this, priv);
+                if (ret) {
+                    goto unlock;
                 }
-                priv->timeout.tv_sec = timeout;
-                ret = 0;
+            }
+        } else {
+            if (!barrier_enabled) {
+                __barrier_disable(this, &queue);
+            }
         }
+        priv->timeout.tv_sec = timeout;
+        ret = 0;
+    }
 unlock:
-        UNLOCK (&priv->lock);
+    UNLOCK(&priv->lock);
 
-        if (!list_empty (&queue))
-                barrier_dequeue_all (this, &queue);
+    if (!list_empty(&queue))
+        barrier_dequeue_all(this, &queue);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_barrier_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
+    ret = xlator_mem_acct_init(this, gf_barrier_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
 
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int                     ret     = -1;
-        barrier_priv_t         *priv    = NULL;
-        uint32_t                timeout = {0,};
+    int ret = -1;
+    barrier_priv_t *priv = NULL;
+    uint32_t timeout = {
+        0,
+    };
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "'barrier' not configured with exactly one child");
-                goto out;
-        }
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'barrier' not configured with exactly one child");
+        goto out;
+    }
 
-        if (!this->parents)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
+    if (!this->parents)
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_barrier_mt_priv_t);
-        if (!priv)
-                goto out;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_barrier_mt_priv_t);
+    if (!priv)
+        goto out;
 
-        LOCK_INIT (&priv->lock);
+    LOCK_INIT(&priv->lock);
 
-        GF_OPTION_INIT ("barrier", priv->barrier_enabled, bool, out);
-        GF_OPTION_INIT ("barrier-timeout", timeout, time, out);
-        priv->timeout.tv_sec = timeout;
+    GF_OPTION_INIT("barrier", priv->barrier_enabled, bool, out);
+    GF_OPTION_INIT("barrier-timeout", timeout, time, out);
+    priv->timeout.tv_sec = timeout;
 
-        INIT_LIST_HEAD (&priv->queue);
+    INIT_LIST_HEAD(&priv->queue);
 
-        if (priv->barrier_enabled) {
-                ret = __barrier_enable (this, priv);
-                if (ret == -1)
-                        goto out;
-        }
+    if (priv->barrier_enabled) {
+        ret = __barrier_enable(this, priv);
+        if (ret == -1)
+            goto out;
+    }
 
-        this->private = priv;
-        ret = 0;
+    this->private = priv;
+    ret = 0;
 out:
-        return ret;
+    if (ret && priv)
+        GF_FREE(priv);
+
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        barrier_priv_t         *priv    = NULL;
-        struct list_head        queue   = {0,};
+    barrier_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+    priv = this->private;
+    if (!priv)
+        goto out;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_log (this->name, GF_LOG_INFO, "Disabling barriering and dequeuing "
-                                         "all the queued fops");
-        LOCK (&priv->lock);
-        {
-                __barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "Disabling barriering and dequeuing "
+           "all the queued fops");
+    LOCK(&priv->lock);
+    {
+        __barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        if (!list_empty (&queue))
-                barrier_dequeue_all (this, &queue);
+    if (!list_empty(&queue))
+        barrier_dequeue_all(this, &queue);
 
-        this->private = NULL;
+    this->private = NULL;
 
-        LOCK_DESTROY (&priv->lock);
-        GF_FREE (priv);
+    LOCK_DESTROY(&priv->lock);
+    GF_FREE(priv);
 out:
-        return;
+    return;
 }
 
 static void
-barrier_dump_stub (call_stub_t *stub, char *prefix)
+barrier_dump_stub(call_stub_t *stub, char *prefix)
 {
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
 
-        gf_proc_dump_build_key (key, prefix, "fop");
-        gf_proc_dump_write (key, "%s", gf_fop_list[stub->fop]);
+    gf_proc_dump_build_key(key, prefix, "fop");
+    gf_proc_dump_write(key, "%s", gf_fop_list[stub->fop]);
 
-        if (stub->frame->local) {
-                gf_proc_dump_build_key (key, prefix, "gfid");
-                gf_proc_dump_write (key, "%s",
-                                    uuid_utoa (*(uuid_t*)(stub->frame->local)));
-        }
-        if (stub->args.loc.path) {
-                gf_proc_dump_build_key (key, prefix, "path");
-                gf_proc_dump_write (key, "%s", stub->args.loc.path);
-        }
-        if (stub->args.loc.name) {
-                gf_proc_dump_build_key (key, prefix, "name");
-                gf_proc_dump_write (key, "%s", stub->args.loc.name);
-        }
+    if (stub->frame->local) {
+        gf_proc_dump_build_key(key, prefix, "gfid");
+        gf_proc_dump_write(key, "%s",
+                           uuid_utoa(*(uuid_t *)(stub->frame->local)));
+    }
+    if (stub->args.loc.path) {
+        gf_proc_dump_build_key(key, prefix, "path");
+        gf_proc_dump_write(key, "%s", stub->args.loc.path);
+    }
+    if (stub->args.loc.name) {
+        gf_proc_dump_build_key(key, prefix, "name");
+        gf_proc_dump_write(key, "%s", stub->args.loc.name);
+    }
 
-        return;
+    return;
 }
 
 static void
-__barrier_dump_queue (barrier_priv_t *priv)
+__barrier_dump_queue(barrier_priv_t *priv)
 {
-        call_stub_t *stub = NULL;
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        int i = 0;
+    call_stub_t *stub = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("barrier", priv, out);
+    GF_VALIDATE_OR_GOTO("barrier", priv, out);
 
-        list_for_each_entry (stub, &priv->queue, list) {
-                snprintf (key, sizeof (key), "stub.%d", i++);
-                gf_proc_dump_add_section (key);
-                barrier_dump_stub(stub, key);
-        }
+    list_for_each_entry(stub, &priv->queue, list)
+    {
+        snprintf(key, sizeof(key), "stub.%d", i++);
+        gf_proc_dump_add_section("%s", key);
+        barrier_dump_stub(stub, key);
+    }
 
 out:
-        return;
+    return;
 }
 
 int
-barrier_dump_priv (xlator_t *this)
-{
-        int ret = -1;
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        barrier_priv_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("barrier", this, out);
-
-        priv = this->private;
-        if (!priv)
-                return 0;
-
-        gf_proc_dump_build_key (key, "xlator.features.barrier", "priv");
-        gf_proc_dump_add_section (key);
-
-        LOCK (&priv->lock);
-        {
-                gf_proc_dump_build_key (key, "barrier", "enabled");
-                gf_proc_dump_write (key, "%d", priv->barrier_enabled);
-                gf_proc_dump_build_key (key, "barrier", "timeout");
-                gf_proc_dump_write (key, "%"PRId64, priv->timeout.tv_sec);
-                if (priv->barrier_enabled) {
-                        gf_proc_dump_build_key (key, "barrier", "queue_size");
-                        gf_proc_dump_write (key, "%d", priv->queue_size);
-                        __barrier_dump_queue (priv);
-                }
+barrier_dump_priv(xlator_t *this)
+{
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    barrier_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("barrier", this, out);
+
+    priv = this->private;
+    if (!priv)
+        return 0;
+
+    gf_proc_dump_build_key(key, "xlator.features.barrier", "priv");
+    gf_proc_dump_add_section("%s", key);
+    gf_proc_dump_build_key(key, "barrier", "enabled");
+
+    LOCK(&priv->lock);
+    {
+        gf_proc_dump_write(key, "%d", priv->barrier_enabled);
+        gf_proc_dump_build_key(key, "barrier", "timeout");
+        gf_proc_dump_write(key, "%ld", priv->timeout.tv_sec);
+        if (priv->barrier_enabled) {
+            gf_proc_dump_build_key(key, "barrier", "queue_size");
+            gf_proc_dump_write(key, "%d", priv->queue_size);
+            __barrier_dump_queue(priv);
         }
-        UNLOCK (&priv->lock);
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops = {
 
-        /* Barrier Class fops */
-        .rmdir          = barrier_rmdir,
-        .unlink         = barrier_unlink,
-        .rename         = barrier_rename,
-        .removexattr    = barrier_removexattr,
-        .fremovexattr   = barrier_fremovexattr,
-        .truncate       = barrier_truncate,
-        .ftruncate      = barrier_ftruncate,
-        .fsync          = barrier_fsync,
-
-        /* Writes with only O_SYNC flag */
-        .writev         = barrier_writev,
+    /* Barrier Class fops */
+    .rmdir = barrier_rmdir,
+    .unlink = barrier_unlink,
+    .rename = barrier_rename,
+    .removexattr = barrier_removexattr,
+    .fremovexattr = barrier_fremovexattr,
+    .truncate = barrier_truncate,
+    .ftruncate = barrier_ftruncate,
+    .fsync = barrier_fsync,
+
+    /* Writes with only O_SYNC flag */
+    .writev = barrier_writev,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = barrier_dump_priv,
+    .priv = barrier_dump_priv,
 };
 
 struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-        { .key  = {"barrier"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", blocks acknowledgements to application "
-                         "for file operations such as rmdir, rename, unlink, "
-                         "removexattr, fremovexattr, truncate, ftruncate, "
-                         "write (with O_SYNC), fsync. It is turned \"off\" by "
-                         "default."
-        },
-        { .key = {"barrier-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = BARRIER_TIMEOUT,
-          .description = "After 'timeout' seconds since the time 'barrier' "
-                         "option was set to \"on\", acknowledgements to file "
-                         "operations are no longer blocked and previously "
-                         "blocked acknowledgements are sent to the application"
-        },
-        { .key  = {NULL} },
+    {.key = {"barrier"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "disable",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"enabled\", blocks acknowledgements to application "
+                    "for file operations such as rmdir, rename, unlink, "
+                    "removexattr, fremovexattr, truncate, ftruncate, "
+                    "write (with O_SYNC), fsync. It is turned \"off\" by "
+                    "default."},
+    {.key = {"barrier-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = BARRIER_TIMEOUT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "After 'timeout' seconds since the time 'barrier' "
+                    "option was set to \"on\", acknowledgements to file "
+                    "operations are no longer blocked and previously "
+                    "blocked acknowledgements are sent to the application"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "barrier",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/barrier/src/barrier.h b/xlators/features/barrier/src/barrier.h
index 0d646f90474..1337f311f7d 100644
--- a/xlators/features/barrier/src/barrier.h
+++ b/xlators/features/barrier/src/barrier.h
@@ -12,71 +12,78 @@
 #define __BARRIER_H__
 
 #include "barrier-mem-types.h"
-#include "xlator.h"
-#include "timer.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/call-stub.h>
 
-#define BARRIER_FOP_CBK(fop_name, label, frame, this, params ...)       \
-        do {                                                            \
-                barrier_priv_t         *_priv           = NULL;         \
-                call_stub_t            *_stub           = NULL;         \
-                gf_boolean_t            _barrier_enabled= _gf_false;    \
-                struct list_head        queue           = {0, };        \
-                                                                        \
-                INIT_LIST_HEAD (&queue);                                \
-                                                                        \
-                _priv = this->private;                                  \
-                GF_ASSERT (_priv);                                      \
-                                                                        \
-                LOCK (&_priv->lock);                                    \
-                {                                                       \
-                        if (_priv->barrier_enabled) {                   \
-                                _barrier_enabled = _priv->barrier_enabled;\
-                                                                        \
-                                _stub = fop_##fop_name##_cbk_stub       \
-                                        (frame,                         \
-                                         barrier_##fop_name##_cbk_resume,\
-                                         params);                       \
-                                if (!_stub) {                           \
-                                        __barrier_disable (this, &queue);\
-                                        goto unlock;                    \
-                                }                                       \
-                                                                        \
-                                __barrier_enqueue (this, _stub);        \
-                        }                                               \
-                }                                                       \
-unlock:                                                                 \
-                UNLOCK (&_priv->lock);                                  \
-                                                                        \
-                if (_stub)                                              \
-                        goto label;                                     \
-                                                                        \
-                if (_barrier_enabled && !_stub) {                       \
-                        gf_log (this->name, GF_LOG_CRITICAL,            \
-                                "Failed to barrier FOPs, disabling "    \
-                                "barrier. FOP: %s, ERROR: %s",          \
-                                #fop_name, strerror (ENOMEM));          \
-                        barrier_dequeue_all (this, &queue);             \
-                }                                                       \
-                barrier_local_free_gfid (frame);                        \
-                STACK_UNWIND_STRICT (fop_name, frame, params);          \
-                goto label;                                             \
-        } while (0)
+#define BARRIER_FOP_CBK(fop_name, label, frame, this, params...)               \
+    do {                                                                       \
+        barrier_priv_t *_priv = NULL;                                          \
+        call_stub_t *_stub = NULL;                                             \
+        gf_boolean_t _barrier_enabled = _gf_false;                             \
+        struct list_head queue = {                                             \
+            0,                                                                 \
+        };                                                                     \
+                                                                               \
+        INIT_LIST_HEAD(&queue);                                                \
+                                                                               \
+        _priv = this->private;                                                 \
+        GF_ASSERT(_priv);                                                      \
+                                                                               \
+        LOCK(&_priv->lock);                                                    \
+        {                                                                      \
+            if (_priv->barrier_enabled) {                                      \
+                _barrier_enabled = _priv->barrier_enabled;                     \
+                                                                               \
+                _stub = fop_##fop_name##_cbk_stub(                             \
+                    frame, barrier_##fop_name##_cbk_resume, params);           \
+                if (!_stub) {                                                  \
+                    __barrier_disable(this, &queue);                           \
+                    goto unlock;                                               \
+                }                                                              \
+                                                                               \
+                __barrier_enqueue(this, _stub);                                \
+            }                                                                  \
+        }                                                                      \
+    unlock:                                                                    \
+        UNLOCK(&_priv->lock);                                                  \
+                                                                               \
+        if (_stub)                                                             \
+            goto label;                                                        \
+                                                                               \
+        if (_barrier_enabled && !_stub) {                                      \
+            gf_log(this->name, GF_LOG_CRITICAL,                                \
+                   "Failed to barrier FOPs, disabling "                        \
+                   "barrier. FOP: %s, ERROR: %s",                              \
+                   #fop_name, strerror(ENOMEM));                               \
+            barrier_dequeue_all(this, &queue);                                 \
+        }                                                                      \
+        barrier_local_free_gfid(frame);                                        \
+        STACK_UNWIND_STRICT(fop_name, frame, params);                          \
+        goto label;                                                            \
+    } while (0)
 
 typedef struct {
-        gf_timer_t       *timer;
-        gf_boolean_t      barrier_enabled;
-        gf_lock_t         lock;
-        struct list_head  queue;
-        struct timespec   timeout;
-        uint32_t          queue_size;
+    gf_timer_t *timer;
+    gf_lock_t lock;
+    struct list_head queue;
+    struct timespec timeout;
+    uint32_t queue_size;
+    gf_boolean_t barrier_enabled;
+    char _pad[3]; /* manual padding */
 } barrier_priv_t;
 
-int __barrier_enable (xlator_t *this, barrier_priv_t *priv);
-void __barrier_enqueue (xlator_t *this, call_stub_t *stub);
-void __barrier_disable (xlator_t *this, struct list_head *queue);
-void barrier_timeout (void *data);
-void barrier_dequeue_all (xlator_t *this, struct list_head *queue);
-call_stub_t *__barrier_dequeue (xlator_t *this, struct list_head *queue);
+int
+__barrier_enable(xlator_t *this, barrier_priv_t *priv);
+void
+__barrier_enqueue(xlator_t *this, call_stub_t *stub);
+void
+__barrier_disable(xlator_t *this, struct list_head *queue);
+void
+barrier_timeout(void *data);
+void
+barrier_dequeue_all(xlator_t *this, struct list_head *queue);
+call_stub_t *
+__barrier_dequeue(xlator_t *this, struct list_head *queue);
 
 #endif
diff --git a/xlators/features/bit-rot/src/bitd/Makefile.am b/xlators/features/bit-rot/src/bitd/Makefile.am
index a915f2d34b8..6db800e6565 100644
--- a/xlators/features/bit-rot/src/bitd/Makefile.am
+++ b/xlators/features/bit-rot/src/bitd/Makefile.am
@@ -1,20 +1,21 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = bit-rot.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 bit_rot_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-		 -I$(top_srcdir)/rpc/xdr/src/ \
-		 -I$(top_srcdir)/rpc/rpc-lib/src \
-                 -I$(CONTRIBDIR)/timer-wheel \
-                 -I$(top_srcdir)/xlators/features/bit-rot/src/stub
+	-I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \
+	-I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/timer-wheel \
+	-I$(top_srcdir)/xlators/features/bit-rot/src/stub
 
-bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-tbf.c bit-rot-ssm.c \
+bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-ssm.c \
 		     bit-rot-scrub-status.c
 bit_rot_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-                    $(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la
+	$(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la
 
-noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-tbf.h bit-rot-bitd-messages.h bit-rot-ssm.h \
+noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-bitd-messages.h bit-rot-ssm.h \
 		 bit-rot-scrub-status.h
 
 AM_CFLAGS = -Wall -DBR_RATE_LIMIT_SIGNER $(GF_CFLAGS)
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
index c6b6a4afa05..5bc5103a27c 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
@@ -11,438 +11,91 @@
 #ifndef _BITROT_BITD_MESSAGES_H_
 #define _BITROT_BITD_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
-/* file bit-rot-bitd-messages.h
- * brief BIT-ROT log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_BITROT_BITD_BASE                   GLFS_MSGID_COMP_BITROT_BITD
-#define GLFS_BITROT_BITD_NUM_MESSAGES           55
-#define GLFS_MSGID_END                          (GLFS_BITROT_BITD_BASE + \
-                                           GLFS_BITROT_BITD_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x  GLFS_BITROT_BITD_BASE, "Invalid: Start of messages"
-/*------------*/
-
-
-#define BRB_MSG_FD_CREATE_FAILED               (GLFS_BITROT_BITD_BASE + 1)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define BRB_MSG_READV_FAILED                (GLFS_BITROT_BITD_BASE + 2)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define BRB_MSG_BLOCK_READ_FAILED           (GLFS_BITROT_BITD_BASE + 3)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_CALC_CHECKSUM_FAILED        (GLFS_BITROT_BITD_BASE + 4)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_NO_MEMORY                   (GLFS_BITROT_BITD_BASE + 5)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_GET_SIGN_FAILED             (GLFS_BITROT_BITD_BASE + 6)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SET_SIGN_FAILED             (GLFS_BITROT_BITD_BASE + 7)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_OP_FAILED                   (GLFS_BITROT_BITD_BASE + 8)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_READ_AND_SIGN_FAILED        (GLFS_BITROT_BITD_BASE + 9)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SIGN_FAILED                 (GLFS_BITROT_BITD_BASE + 10)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_GET_SUBVOL_FAILED           (GLFS_BITROT_BITD_BASE + 11)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SET_TIMER_FAILED            (GLFS_BITROT_BITD_BASE + 12)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_GET_INFO_FAILED             (GLFS_BITROT_BITD_BASE + 13)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_PATH_FAILED                 (GLFS_BITROT_BITD_BASE + 14)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_MARK_BAD_FILE               (GLFS_BITROT_BITD_BASE + 15)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_TRIGGER_SIGN                (GLFS_BITROT_BITD_BASE + 16)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_REGISTER_FAILED             (GLFS_BITROT_BITD_BASE + 17)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_CRAWLING_START              (GLFS_BITROT_BITD_BASE + 18)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SPAWN_FAILED                (GLFS_BITROT_BITD_BASE + 19)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_INVALID_SUBVOL_CHILD        (GLFS_BITROT_BITD_BASE + 20)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SKIP_OBJECT                 (GLFS_BITROT_BITD_BASE + 21)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_NO_CHILD                    (GLFS_BITROT_BITD_BASE + 22)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_CHECKSUM_MISMATCH           (GLFS_BITROT_BITD_BASE + 23)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_MARK_CORRUPTED              (GLFS_BITROT_BITD_BASE + 24)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_CRAWLING_FINISH               (GLFS_BITROT_BITD_BASE + 25)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_CALC_ERROR                  (GLFS_BITROT_BITD_BASE + 26)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_LOOKUP_FAILED               (GLFS_BITROT_BITD_BASE + 27)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_PARTIAL_VERSION_PRESENCE    (GLFS_BITROT_BITD_BASE + 28)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_MEM_ACNT_FAILED             (GLFS_BITROT_BITD_BASE + 29)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_TIMER_WHEEL_UNAVAILABLE     (GLFS_BITROT_BITD_BASE + 30)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_BITROT_LOADED               (GLFS_BITROT_BITD_BASE + 31)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCALE_DOWN_FAILED           (GLFS_BITROT_BITD_BASE + 32)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCALE_UP_FAILED             (GLFS_BITROT_BITD_BASE + 33)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
+/* To add new message IDs, append new identifiers at the end of the list.
  *
- */
-#define BRB_MSG_SCALE_DOWN_SCRUBBER         (GLFS_BITROT_BITD_BASE + 34)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCALING_UP_SCRUBBER         (GLFS_BITROT_BITD_BASE + 35)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define BRB_MSG_UNKNOWN_THROTTLE            (GLFS_BITROT_BITD_BASE + 36)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_RATE_LIMIT_INFO             (GLFS_BITROT_BITD_BASE + 37)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCRUB_INFO                  (GLFS_BITROT_BITD_BASE + 38)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_CONNECTED_TO_BRICK          (GLFS_BITROT_BITD_BASE + 39)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_BRICK_INFO                  (GLFS_BITROT_BITD_BASE + 40)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SUBVOL_CONNECT_FAILED       (GLFS_BITROT_BITD_BASE + 41)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_INVALID_SUBVOL              (GLFS_BITROT_BITD_BASE + 42)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_RESCHEDULE_SCRUBBER_FAILED  (GLFS_BITROT_BITD_BASE + 43)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
  *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
 
-#define BRB_MSG_SCRUB_START                 (GLFS_BITROT_BITD_BASE + 44)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCRUB_FINISH                (GLFS_BITROT_BITD_BASE + 45)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCRUB_RUNNING               (GLFS_BITROT_BITD_BASE + 46)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCRUB_RESCHEDULED           (GLFS_BITROT_BITD_BASE + 47)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRB_MSG_SCRUB_TUNABLE              (GLFS_BITROT_BITD_BASE + 48)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
-#define BRB_MSG_SCRUB_THREAD_CLEANUP       (GLFS_BITROT_BITD_BASE + 49)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
-#define BRB_MSG_SCRUBBER_CLEANED           (GLFS_BITROT_BITD_BASE + 50)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
-#define BRB_MSG_GENERIC_SSM_INFO           (GLFS_BITROT_BITD_BASE + 51)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
-#define BRB_MSG_ZERO_TIMEOUT_BUG           (GLFS_BITROT_BITD_BASE + 52)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
-#define BRB_MSG_BAD_OBJ_READDIR_FAIL       (GLFS_BITROT_BITD_BASE + 53)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
-#define BRB_MSG_SSM_FAILED                 (GLFS_BITROT_BITD_BASE + 54)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
-#define BRB_MSG_SCRUB_WAIT_FAILED          (GLFS_BITROT_BITD_BASE + 55)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
+GLFS_MSGID(BITROT_BITD, BRB_MSG_FD_CREATE_FAILED, BRB_MSG_READV_FAILED,
+           BRB_MSG_BLOCK_READ_FAILED, BRB_MSG_CALC_CHECKSUM_FAILED,
+           BRB_MSG_NO_MEMORY, BRB_MSG_GET_SIGN_FAILED, BRB_MSG_SET_SIGN_FAILED,
+           BRB_MSG_OP_FAILED, BRB_MSG_READ_AND_SIGN_FAILED, BRB_MSG_SIGN_FAILED,
+           BRB_MSG_GET_SUBVOL_FAILED, BRB_MSG_SET_TIMER_FAILED,
+           BRB_MSG_GET_INFO_FAILED, BRB_MSG_PATH_FAILED, BRB_MSG_MARK_BAD_FILE,
+           BRB_MSG_TRIGGER_SIGN, BRB_MSG_REGISTER_FAILED,
+           BRB_MSG_CRAWLING_START, BRB_MSG_SPAWN_FAILED,
+           BRB_MSG_INVALID_SUBVOL_CHILD, BRB_MSG_SKIP_OBJECT, BRB_MSG_NO_CHILD,
+           BRB_MSG_CHECKSUM_MISMATCH, BRB_MSG_MARK_CORRUPTED,
+           BRB_MSG_CRAWLING_FINISH, BRB_MSG_CALC_ERROR, BRB_MSG_LOOKUP_FAILED,
+           BRB_MSG_PARTIAL_VERSION_PRESENCE, BRB_MSG_MEM_ACNT_FAILED,
+           BRB_MSG_TIMER_WHEEL_UNAVAILABLE, BRB_MSG_BITROT_LOADED,
+           BRB_MSG_SCALE_DOWN_FAILED, BRB_MSG_SCALE_UP_FAILED,
+           BRB_MSG_SCALE_DOWN_SCRUBBER, BRB_MSG_SCALING_UP_SCRUBBER,
+           BRB_MSG_UNKNOWN_THROTTLE, BRB_MSG_RATE_LIMIT_INFO,
+           BRB_MSG_SCRUB_INFO, BRB_MSG_CONNECTED_TO_BRICK, BRB_MSG_BRICK_INFO,
+           BRB_MSG_SUBVOL_CONNECT_FAILED, BRB_MSG_INVALID_SUBVOL,
+           BRB_MSG_RESCHEDULE_SCRUBBER_FAILED, BRB_MSG_SCRUB_START,
+           BRB_MSG_SCRUB_FINISH, BRB_MSG_SCRUB_RUNNING,
+           BRB_MSG_SCRUB_RESCHEDULED, BRB_MSG_SCRUB_TUNABLE,
+           BRB_MSG_SCRUB_THREAD_CLEANUP, BRB_MSG_SCRUBBER_CLEANED,
+           BRB_MSG_GENERIC_SSM_INFO, BRB_MSG_ZERO_TIMEOUT_BUG,
+           BRB_MSG_BAD_OBJ_READDIR_FAIL, BRB_MSG_SSM_FAILED,
+           BRB_MSG_SCRUB_WAIT_FAILED, BRB_MSG_TRIGGER_SIGN_FAILED,
+           BRB_MSG_EVENT_UNHANDLED, BRB_MSG_COULD_NOT_SCHEDULE_SCRUB,
+           BRB_MSG_THREAD_CREATION_FAILED, BRB_MSG_MEM_POOL_ALLOC,
+           BRB_MSG_SAVING_HASH_FAILED);
 
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#define BRB_MSG_FD_CREATE_FAILED_STR "failed to create fd for the inode"
+#define BRB_MSG_READV_FAILED_STR "readv failed"
+#define BRB_MSG_BLOCK_READ_FAILED_STR "reading block failed"
+#define BRB_MSG_NO_MEMORY_STR "failed to allocate memory"
+#define BRB_MSG_CALC_CHECKSUM_FAILED_STR "calculating checksum failed"
+#define BRB_MSG_GET_SIGN_FAILED_STR "failed to get the signature"
+#define BRB_MSG_SET_SIGN_FAILED_STR "signing failed"
+#define BRB_MSG_OP_FAILED_STR "failed on object"
+#define BRB_MSG_TRIGGER_SIGN_FAILED_STR "Could not trigger signing"
+#define BRB_MSG_READ_AND_SIGN_FAILED_STR "reading and signing of object failed"
+#define BRB_MSG_SET_TIMER_FAILED_STR "Failed to allocate object expiry timer"
+#define BRB_MSG_GET_SUBVOL_FAILED_STR                                          \
+    "failed to get the subvolume for the brick"
+#define BRB_MSG_PATH_FAILED_STR "path failed"
+#define BRB_MSG_SKIP_OBJECT_STR "Entry is marked corrupted. skipping"
+#define BRB_MSG_PARTIAL_VERSION_PRESENCE_STR                                   \
+    "PArtial version xattr presence detected, ignoring"
+#define BRB_MSG_TRIGGER_SIGN_STR "Triggering signing"
+#define BRB_MSG_CRAWLING_START_STR                                             \
+    "Crawling brick, scanning for unsigned objects"
+#define BRB_MSG_CRAWLING_FINISH_STR "Completed crawling brick"
+#define BRB_MSG_REGISTER_FAILED_STR "Register to changelog failed"
+#define BRB_MSG_SPAWN_FAILED_STR "failed to spawn"
+#define BRB_MSG_CONNECTED_TO_BRICK_STR "Connected to brick"
+#define BRB_MSG_LOOKUP_FAILED_STR "lookup on root failed"
+#define BRB_MSG_GET_INFO_FAILED_STR "failed to get stub info"
+#define BRB_MSG_SCRUB_THREAD_CLEANUP_STR "Error cleaning up scanner thread"
+#define BRB_MSG_SCRUBBER_CLEANED_STR "clened up scrubber for brick"
+#define BRB_MSG_SUBVOL_CONNECT_FAILED_STR                                      \
+    "callback handler for subvolume failed"
+#define BRB_MSG_MEM_ACNT_FAILED_STR "Memory accounting init failed"
+#define BRB_MSG_EVENT_UNHANDLED_STR "Event unhandled for child"
+#define BRB_MSG_INVALID_SUBVOL_STR "Got event from invalid subvolume"
+#define BRB_MSG_RESCHEDULE_SCRUBBER_FAILED_STR                                 \
+    "on demand scrub schedule failed. Scrubber is not in pending state."
+#define BRB_MSG_COULD_NOT_SCHEDULE_SCRUB_STR                                   \
+    "Could not schedule ondemand scrubbing. Scrubbing will continue "          \
+    "according to old frequency."
+#define BRB_MSG_THREAD_CREATION_FAILED_STR "thread creation failed"
+#define BRB_MSG_RATE_LIMIT_INFO_STR "Rate Limit Info"
+#define BRB_MSG_MEM_POOL_ALLOC_STR "failed to allocate mem-pool for timer"
+#define BRB_MSG_NO_CHILD_STR "FATAL: no children"
+#define BRB_MSG_TIMER_WHEEL_UNAVAILABLE_STR "global timer wheel unavailable"
+#define BRB_MSG_BITROT_LOADED_STR "bit-rot xlator loaded"
+#define BRB_MSG_SAVING_HASH_FAILED_STR                                         \
+    "failed to allocate memory for saving hash of the object"
 #endif /* !_BITROT_BITD_MESSAGES_H_ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
index 0afd7ea05b1..5cef2ffa5e5 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
@@ -9,65 +9,70 @@
 */
 
 #include <string.h>
+#include <stdio.h>
 
 #include "bit-rot-scrub-status.h"
 
 void
-br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat)
+br_inc_unsigned_file_count(br_scrub_stats_t *scrub_stat)
 {
-        if (!scrub_stat)
-                return;
+    if (!scrub_stat)
+        return;
 
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->unsigned_files++;
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->unsigned_files++;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
 
 void
-br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat)
+br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat)
 {
-        if (!scrub_stat)
-                return;
+    if (!scrub_stat)
+        return;
 
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->scrubbed_files++;
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrubbed_files++;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
 
 void
-br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv)
+br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time)
 {
-        if (!scrub_stat)
-                return;
+    if (!scrub_stat)
+        return;
 
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->scrub_start_tv.tv_sec = tv->tv_sec;
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrub_start_time = time;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
 
 void
-br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr,
-                             struct timeval *tv)
+br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr,
+                            time_t time)
 {
-        if (!scrub_stat)
-                return;
+    int lst_size = 0;
 
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->scrub_end_tv.tv_sec = tv->tv_sec;
+    if (!scrub_stat)
+        return;
 
-                scrub_stat->scrub_duration =
-                                 scrub_stat->scrub_end_tv.tv_sec -
-                                 scrub_stat->scrub_start_tv.tv_sec;
+    lst_size = sizeof(scrub_stat->last_scrub_time);
+    if (strlen(timestr) >= lst_size)
+        return;
 
-                strncpy (scrub_stat->last_scrub_time, timestr,
-                         sizeof (scrub_stat->last_scrub_time));
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrub_end_time = time;
+
+        scrub_stat->scrub_duration = scrub_stat->scrub_end_time -
+                                     scrub_stat->scrub_start_time;
+
+        snprintf(scrub_stat->last_scrub_time, lst_size, "%s", timestr);
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
index 694ba0acbe3..f022aa831eb 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
@@ -15,32 +15,36 @@
 #include <sys/time.h>
 #include <pthread.h>
 
+#include <glusterfs/common-utils.h>
+
 struct br_scrub_stats {
-        uint64_t       scrubbed_files;       /* Total number of scrubbed file */
+    uint64_t scrubbed_files; /* Total number of scrubbed files. */
+
+    uint64_t unsigned_files; /* Total number of unsigned files. */
 
-        uint64_t       unsigned_files;       /* Total number of unsigned file */
+    uint64_t scrub_duration; /* Duration of last scrub. */
 
-        uint64_t       scrub_duration;            /* Duration of last scrub */
+    char last_scrub_time[GF_TIMESTR_SIZE]; /* Last scrub completion time. */
 
-        char           last_scrub_time[1024];    /*last scrub completion time */
+    time_t scrub_start_time; /* Scrubbing starting time. */
 
-        struct         timeval scrub_start_tv;   /* Scrubbing starting time*/
+    time_t scrub_end_time; /* Scrubbing finishing time. */
 
-        struct         timeval scrub_end_tv;     /* Scrubbing finishing time */
+    int8_t scrub_running; /* Whether scrub running or not. */
 
-        pthread_mutex_t  lock;
+    pthread_mutex_t lock;
 };
 
 typedef struct br_scrub_stats br_scrub_stats_t;
 
 void
-br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat);
+br_inc_unsigned_file_count(br_scrub_stats_t *scrub_stat);
 void
-br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat);
+br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat);
 void
-br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv);
+br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time);
 void
-br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr,
-                             struct timeval *tv);
+br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr,
+                            time_t time);
 
 #endif /* __BIT_ROT_SCRUB_STATUS_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index 8c88d62eceb..289dd53f610 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -12,31 +12,32 @@
 #include <ctype.h>
 #include <sys/uio.h>
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
 #include "bit-rot-scrub.h"
 #include <pthread.h>
 #include "bit-rot-bitd-messages.h"
 #include "bit-rot-scrub-status.h"
+#include <glusterfs/events.h>
 
 struct br_scrubbers {
-        pthread_t scrubthread;
+    pthread_t scrubthread;
 
-        struct list_head list;
+    struct list_head list;
 };
 
 struct br_fsscan_entry {
-        void *data;
+    void *data;
 
-        loc_t parent;
+    loc_t parent;
 
-        gf_dirent_t *entry;
+    gf_dirent_t *entry;
 
-        struct br_scanfs *fsscan;  /* backpointer to subvolume scanner */
+    struct br_scanfs *fsscan; /* backpointer to subvolume scanner */
 
-        struct list_head list;
+    struct list_head list;
 };
 
 /**
@@ -45,34 +46,32 @@ struct br_fsscan_entry {
  * to the dictionary value.
  */
 static int32_t
-bitd_fetch_signature (xlator_t *this, br_child_t *child,
-                      fd_t *fd, dict_t **xattr, br_isignature_out_t **sign)
+bitd_fetch_signature(xlator_t *this, br_child_t *child, fd_t *fd,
+                     dict_t **xattr, br_isignature_out_t **sign)
 {
-       int32_t ret = -1;
-
-        ret = syncop_fgetxattr (child->xl, fd, xattr,
-                               GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
-        if (ret < 0) {
-                br_log_object (this, "fgetxattr", fd->inode->gfid, -ret);
-                goto out;
-        }
-
-        ret = dict_get_ptr
-                (*xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **) sign);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
-                        "failed to extract signature info [GFID: %s]",
-                        uuid_utoa (fd->inode->gfid));
-                goto unref_dict;
-        }
-
-        return 0;
-
- unref_dict:
-        dict_unref (*xattr);
- out:
-        return -1;
-
+    int32_t ret = -1;
+
+    ret = syncop_fgetxattr(child->xl, fd, xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                           NULL, NULL);
+    if (ret < 0) {
+        br_log_object(this, "fgetxattr", fd->inode->gfid, -ret);
+        goto out;
+    }
+
+    ret = dict_get_ptr(*xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **)sign);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+               "failed to extract signature info [GFID: %s]",
+               uuid_utoa(fd->inode->gfid));
+        goto unref_dict;
+    }
+
+    return 0;
+
+unref_dict:
+    dict_unref(*xattr);
+out:
+    return -1;
 }
 
 /**
@@ -85,84 +84,87 @@ bitd_fetch_signature (xlator_t *this, br_child_t *child,
  */
 
 int32_t
-bitd_scrub_post_compute_check (xlator_t *this,
-                               br_child_t *child,
-                               fd_t *fd, unsigned long version,
-                               br_isignature_out_t **signature,
-                               br_scrub_stats_t *scrub_stat)
+bitd_scrub_post_compute_check(xlator_t *this, br_child_t *child, fd_t *fd,
+                              unsigned long version,
+                              br_isignature_out_t **signature,
+                              br_scrub_stats_t *scrub_stat,
+                              gf_boolean_t skip_stat)
 {
-        int32_t              ret     = 0;
-        size_t               signlen = 0;
-        dict_t              *xattr   = NULL;
-        br_isignature_out_t *signptr = NULL;
-
-        ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0) {
-                br_inc_unsigned_file_count (scrub_stat);
-                goto out;
-        }
-
-        /**
-         * Either the object got dirtied during the time the signature was
-         * calculated OR the version we saved during pre-compute check does
-         * not match now, implying that the object got dirtied and signed in
-         * between scrubs pre & post compute checks (checksum window).
-         *
-         * The log entry looks pretty ugly, but helps in debugging..
-         */
-        if (signptr->stale || (signptr->version != version)) {
-                br_inc_unsigned_file_count (scrub_stat);
-                gf_msg_debug (this->name, 0, "<STAGE: POST> Object [GFID: %s] "
-                              "either has a stale signature OR underwent "
-                              "signing during checksumming {Stale: %d | "
-                              "Version: %lu,%lu}", uuid_utoa (fd->inode->gfid),
-                              (signptr->stale) ? 1 : 0, version,
-                              signptr->version);
-                ret = -1;
-                goto unref_dict;
-        }
-
-        signlen = signptr->signaturelen;
-        *signature = GF_CALLOC (1, sizeof (br_isignature_out_t) + signlen,
-                                gf_common_mt_char);
-
-        (void) memcpy (*signature, signptr,
-                       sizeof (br_isignature_out_t) + signlen);
-
- unref_dict:
-        dict_unref (xattr);
- out:
-        return ret;
-
+    int32_t ret = 0;
+    size_t signlen = 0;
+    dict_t *xattr = NULL;
+    br_isignature_out_t *signptr = NULL;
+
+    ret = bitd_fetch_signature(this, child, fd, &xattr, &signptr);
+    if (ret < 0) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        goto out;
+    }
+
+    /**
+     * Either the object got dirtied during the time the signature was
+     * calculated OR the version we saved during pre-compute check does
+     * not match now, implying that the object got dirtied and signed in
+     * between scrubs pre & post compute checks (checksum window).
+     *
+     * The log entry looks pretty ugly, but helps in debugging..
+     */
+    if (signptr->stale || (signptr->version != version)) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        gf_msg_debug(this->name, 0,
+                     "<STAGE: POST> Object [GFID: %s] "
+                     "either has a stale signature OR underwent "
+                     "signing during checksumming {Stale: %d | "
+                     "Version: %lu,%lu}",
+                     uuid_utoa(fd->inode->gfid), (signptr->stale) ? 1 : 0,
+                     version, signptr->version);
+        ret = -1;
+        goto unref_dict;
+    }
+
+    signlen = signptr->signaturelen;
+    *signature = GF_MALLOC(sizeof(br_isignature_out_t) + signlen,
+                           gf_common_mt_char);
+
+    (void)memcpy(*signature, signptr, sizeof(br_isignature_out_t) + signlen);
+
+    (*signature)->signaturelen = signlen;
+
+unref_dict:
+    dict_unref(xattr);
+out:
+    return ret;
 }
 
 static int32_t
-bitd_signature_staleness (xlator_t *this,
-                          br_child_t *child, fd_t *fd,
-                          int *stale, unsigned long *version,
-                          br_scrub_stats_t *scrub_stat)
+bitd_signature_staleness(xlator_t *this, br_child_t *child, fd_t *fd,
+                         int *stale, unsigned long *version,
+                         br_scrub_stats_t *scrub_stat, gf_boolean_t skip_stat)
 {
-        int32_t ret = -1;
-        dict_t *xattr = NULL;
-        br_isignature_out_t *signptr = NULL;
+    int32_t ret = -1;
+    dict_t *xattr = NULL;
+    br_isignature_out_t *signptr = NULL;
+
+    ret = bitd_fetch_signature(this, child, fd, &xattr, &signptr);
+    if (ret < 0) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        goto out;
+    }
+
+    /**
+     * save version for validation in post compute stage
+     * c.f. bitd_scrub_post_compute_check()
+     */
+    *stale = signptr->stale ? 1 : 0;
+    *version = signptr->version;
+
+    dict_unref(xattr);
 
-        ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0) {
-                br_inc_unsigned_file_count (scrub_stat);
-                goto out;
-        }
-
-        /**
-         * save verison for validation in post compute stage
-         * c.f. bitd_scrub_post_compute_check()
-         */
-        *stale = signptr->stale ? 1 : 0;
-        *version = signptr->version;
-
-        dict_unref (xattr);
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /**
@@ -174,97 +176,102 @@ bitd_signature_staleness (xlator_t *this,
  *  - has stale signature
  */
 int32_t
-bitd_scrub_pre_compute_check (xlator_t *this, br_child_t *child,
-                              fd_t *fd, unsigned long *version,
-                              br_scrub_stats_t *scrub_stat)
+bitd_scrub_pre_compute_check(xlator_t *this, br_child_t *child, fd_t *fd,
+                             unsigned long *version,
+                             br_scrub_stats_t *scrub_stat,
+                             gf_boolean_t skip_stat)
 {
-        int     stale = 0;
-        int32_t ret   = -1;
-
-        if (bitd_is_bad_file (this, child, NULL, fd)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT,
-                        "Object [GFID: %s] is marked corrupted, skipping..",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
+    int stale = 0;
+    int32_t ret = -1;
+
+    if (bitd_is_bad_file(this, child, NULL, fd)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT,
+               "Object [GFID: %s] is marked corrupted, skipping..",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    ret = bitd_signature_staleness(this, child, fd, &stale, version, scrub_stat,
+                                   skip_stat);
+    if (!ret && stale) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        gf_msg_debug(this->name, 0,
+                     "<STAGE: PRE> Object [GFID: %s] "
+                     "has stale signature",
+                     uuid_utoa(fd->inode->gfid));
+        ret = -1;
+    }
 
-        ret = bitd_signature_staleness (this, child, fd, &stale, version,
-                                        scrub_stat);
-        if (!ret && stale) {
-                br_inc_unsigned_file_count (scrub_stat);
-                gf_msg_debug (this->name, 0, "<STAGE: PRE> Object [GFID: %s] "
-                              "has stale signature",
-                              uuid_utoa (fd->inode->gfid));
-                ret = -1;
-        }
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /* static int */
 int
-bitd_compare_ckum (xlator_t *this,
-                   br_isignature_out_t *sign,
-                   unsigned char *md, inode_t *linked_inode,
-                   gf_dirent_t *entry, fd_t *fd, br_child_t *child, loc_t *loc)
+bitd_compare_ckum(xlator_t *this, br_isignature_out_t *sign, unsigned char *md,
+                  inode_t *linked_inode, gf_dirent_t *entry, fd_t *fd,
+                  br_child_t *child, loc_t *loc)
 {
-        int   ret = -1;
-        dict_t *xattr = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, sign, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, linked_inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, md, out);
-        GF_VALIDATE_OR_GOTO (this->name, entry, out);
-
-        if (strncmp
-            (sign->signature, (char *) md, strlen (sign->signature)) == 0) {
-                gf_msg_debug (this->name, 0, "%s [GFID: %s | Brick: %s] "
-                              "matches calculated checksum", loc->path,
-                              uuid_utoa (linked_inode->gfid),
-                              child->brick_path);
-                return 0;
-        }
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0, BRB_MSG_CHECKSUM_MISMATCH,
-                "Object checksum mismatch: %s [GFID: %s | Brick: %s]",
-                loc->path, uuid_utoa (linked_inode->gfid), child->brick_path);
-        gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_CHECKSUM_MISMATCH,
-                "CORRUPTION DETECTED: Object %s {Brick: %s | GFID: %s}",
-                loc->path, child->brick_path, uuid_utoa (linked_inode->gfid));
-
-        /* Perform bad-file marking */
-        xattr = dict_new ();
-        if (!xattr) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_int32 (xattr, BITROT_OBJECT_BAD_KEY, _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
-                        "Error setting bad-file marker for %s [GFID: %s | "
-                        "Brick: %s]", loc->path, uuid_utoa (linked_inode->gfid),
-                        child->brick_path);
-                goto dictfree;
-        }
-
-        gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_MARK_CORRUPTED, "Marking"
-                " %s [GFID: %s | Brick: %s] as corrupted..", loc->path,
-                uuid_utoa (linked_inode->gfid), child->brick_path);
-        ret = syncop_fsetxattr (child->xl, fd, xattr, 0, NULL, NULL);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
-                        "Error marking object %s [GFID: %s] as corrupted",
-                        loc->path, uuid_utoa (linked_inode->gfid));
-
- dictfree:
-        dict_unref (xattr);
- out:
-        return ret;
+    int ret = -1;
+    dict_t *xattr = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, sign, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, linked_inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, md, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry, out);
+
+    if (strncmp(sign->signature, (char *)md, sign->signaturelen) == 0) {
+        gf_msg_debug(this->name, 0,
+                     "%s [GFID: %s | Brick: %s] "
+                     "matches calculated checksum",
+                     loc->path, uuid_utoa(linked_inode->gfid),
+                     child->brick_path);
+        return 0;
+    }
+
+    gf_msg(this->name, GF_LOG_DEBUG, 0, BRB_MSG_CHECKSUM_MISMATCH,
+           "Object checksum mismatch: %s [GFID: %s | Brick: %s]", loc->path,
+           uuid_utoa(linked_inode->gfid), child->brick_path);
+    gf_msg(this->name, GF_LOG_ALERT, 0, BRB_MSG_CHECKSUM_MISMATCH,
+           "CORRUPTION DETECTED: Object %s {Brick: %s | GFID: %s}", loc->path,
+           child->brick_path, uuid_utoa(linked_inode->gfid));
+
+    /* Perform bad-file marking */
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32(xattr, BITROT_OBJECT_BAD_KEY, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
+               "Error setting bad-file marker for %s [GFID: %s | "
+               "Brick: %s]",
+               loc->path, uuid_utoa(linked_inode->gfid), child->brick_path);
+        goto dictfree;
+    }
+
+    gf_msg(this->name, GF_LOG_ALERT, 0, BRB_MSG_MARK_CORRUPTED,
+           "Marking"
+           " %s [GFID: %s | Brick: %s] as corrupted..",
+           loc->path, uuid_utoa(linked_inode->gfid), child->brick_path);
+    gf_event(EVENT_BITROT_BAD_FILE, "gfid=%s;path=%s;brick=%s",
+             uuid_utoa(linked_inode->gfid), loc->path, child->brick_path);
+    ret = syncop_fsetxattr(child->xl, fd, xattr, 0, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
+               "Error marking object %s [GFID: %s] as corrupted", loc->path,
+               uuid_utoa(linked_inode->gfid));
+
+dictfree:
+    dict_unref(xattr);
+out:
+    return ret;
 }
 
 /**
@@ -275,522 +282,550 @@ bitd_compare_ckum (xlator_t *this,
  * signs with SHA256).
  */
 int
-br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry)
+br_scrubber_scrub_begin(xlator_t *this, struct br_fsscan_entry *fsentry)
 {
-        int32_t                ret           = -1;
-        fd_t                  *fd            = NULL;
-        loc_t                  loc           = {0, };
-        struct iatt            iatt          = {0, };
-        struct iatt            parent_buf    = {0, };
-        pid_t                  pid           = 0;
-        br_child_t            *child         = NULL;
-        unsigned char         *md            = NULL;
-        inode_t               *linked_inode  = NULL;
-        br_isignature_out_t   *sign          = NULL;
-        unsigned long          signedversion = 0;
-        gf_dirent_t           *entry         = NULL;
-        br_private_t          *priv          = NULL;
-        loc_t                 *parent        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", fsentry, out);
-
-        entry = fsentry->entry;
-        parent = &fsentry->parent;
-        child = fsentry->data;
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", entry, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", parent, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", child, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", priv, out);
-
-        pid = GF_CLIENT_PID_SCRUB;
-
-        ret = br_prepare_loc (this, child, parent, entry, &loc);
-        if (!ret)
-                goto out;
-
-        syncopctx_setfspid (&pid);
-
-        ret = syncop_lookup (child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
-        if (ret) {
-                br_log_object_path (this, "lookup", loc.path, -ret);
-                goto out;
-        }
-
-        linked_inode = inode_link (loc.inode, parent->inode, loc.name, &iatt);
-        if (linked_inode)
-                inode_lookup (linked_inode);
-
-        gf_msg_debug (this->name, 0, "Scrubbing object %s [GFID: %s]",
-                      entry->d_name, uuid_utoa (linked_inode->gfid));
-
-        if (iatt.ia_type != IA_IFREG) {
-                gf_msg_debug (this->name, 0, "%s is not a regular file",
-                              entry->d_name);
-                ret = 0;
-                goto unref_inode;
-        }
-
-        /**
-         * open() an fd for subsequent opertaions
-         */
-        fd = fd_create (linked_inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
-                        "failed to create fd for inode %s",
-                        uuid_utoa (linked_inode->gfid));
-                goto unref_inode;
-        }
-
-        ret = syncop_open (child->xl, &loc, O_RDWR, fd, NULL, NULL);
-        if (ret) {
-                br_log_object (this, "open", linked_inode->gfid, -ret);
-                ret = -1;
-                goto unrefd;
-        }
-
-        fd_bind (fd);
-
-        /**
-         * perform pre compute checks before initiating checksum
-         * computation
-         *  - presence of bad object
-         *  - signature staleness
-         */
-        ret = bitd_scrub_pre_compute_check (this, child, fd, &signedversion,
-                                            &priv->scrub_stat);
-        if (ret)
-                goto unrefd; /* skip this object */
-
-        /* if all's good, proceed to calculate the hash */
-        md = GF_CALLOC (SHA256_DIGEST_LENGTH, sizeof (*md),
-                        gf_common_mt_char);
-        if (!md)
-                goto unrefd;
-
-        ret = br_calculate_obj_checksum (md, child, fd, &iatt);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_ERROR,
-                        "error calculating hash for object [GFID: %s]",
-                        uuid_utoa (fd->inode->gfid));
-                ret = -1;
-                goto free_md;
-        }
-
-        /**
-         * perform post compute checks as an object's signature may have
-         * become stale while scrubber calculated checksum.
-         */
-        ret = bitd_scrub_post_compute_check (this, child, fd, signedversion,
-                                             &sign, &priv->scrub_stat);
-        if (ret)
-                goto free_md;
-
-        ret = bitd_compare_ckum (this, sign, md,
-                                 linked_inode, entry, fd, child, &loc);
-
-        /* Increment of total number of scrubbed file counter */
-        br_inc_scrubbed_file (&priv->scrub_stat);
-
-        GF_FREE (sign); /* alloced on post-compute */
-
-        /** fd_unref() takes care of closing fd.. like syncop_close() */
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent_buf = {
+        0,
+    };
+    pid_t pid = 0;
+    br_child_t *child = NULL;
+    unsigned char *md = NULL;
+    inode_t *linked_inode = NULL;
+    br_isignature_out_t *sign = NULL;
+    unsigned long signedversion = 0;
+    gf_dirent_t *entry = NULL;
+    br_private_t *priv = NULL;
+    loc_t *parent = NULL;
+    gf_boolean_t skip_stat = _gf_false;
+    uuid_t shard_root_gfid = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("bit-rot", fsentry, out);
+
+    entry = fsentry->entry;
+    parent = &fsentry->parent;
+    child = fsentry->data;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", entry, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", parent, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", priv, out);
+
+    pid = GF_CLIENT_PID_SCRUB;
+
+    ret = br_prepare_loc(this, child, parent, entry, &loc);
+    if (!ret)
+        goto out;
+
+    syncopctx_setfspid(&pid);
+
+    ret = syncop_lookup(child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
+    if (ret) {
+        br_log_object_path(this, "lookup", loc.path, -ret);
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, parent->inode, loc.name, &iatt);
+    if (linked_inode)
+        inode_lookup(linked_inode);
+
+    gf_msg_debug(this->name, 0, "Scrubbing object %s [GFID: %s]", entry->d_name,
+                 uuid_utoa(linked_inode->gfid));
+
+    if (iatt.ia_type != IA_IFREG) {
+        gf_msg_debug(this->name, 0, "%s is not a regular file", entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
 
- free_md:
-        GF_FREE (md);
- unrefd:
-        fd_unref (fd);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        loc_wipe (&loc);
-        return ret;
+    if (IS_DHT_LINKFILE_MODE((&iatt))) {
+        gf_msg_debug(this->name, 0, "%s is a dht sticky bit file",
+                     entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
+
+    /* skip updating scrub statistics for shard entries */
+    gf_uuid_parse(SHARD_ROOT_GFID, shard_root_gfid);
+    if (gf_uuid_compare(loc.pargfid, shard_root_gfid) == 0)
+        skip_stat = _gf_true;
+
+    /**
+     * open() an fd for subsequent operations
+     */
+    fd = fd_create(linked_inode, 0);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+               "failed to create fd for inode %s",
+               uuid_utoa(linked_inode->gfid));
+        goto unref_inode;
+    }
+
+    ret = syncop_open(child->xl, &loc, O_RDWR, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", linked_inode->gfid, -ret);
+        ret = -1;
+        goto unrefd;
+    }
+
+    fd_bind(fd);
+
+    /**
+     * perform pre compute checks before initiating checksum
+     * computation
+     *  - presence of bad object
+     *  - signature staleness
+     */
+    ret = bitd_scrub_pre_compute_check(this, child, fd, &signedversion,
+                                       &priv->scrub_stat, skip_stat);
+    if (ret)
+        goto unrefd; /* skip this object */
+
+    /* if all's good, proceed to calculate the hash */
+    md = GF_MALLOC(SHA256_DIGEST_LENGTH, gf_common_mt_char);
+    if (!md)
+        goto unrefd;
+
+    ret = br_calculate_obj_checksum(md, child, fd, &iatt);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_ERROR,
+               "error calculating hash for object [GFID: %s]",
+               uuid_utoa(fd->inode->gfid));
+        ret = -1;
+        goto free_md;
+    }
+
+    /**
+     * perform post compute checks as an object's signature may have
+     * become stale while scrubber calculated checksum.
+     */
+    ret = bitd_scrub_post_compute_check(this, child, fd, signedversion, &sign,
+                                        &priv->scrub_stat, skip_stat);
+    if (ret)
+        goto free_md;
+
+    ret = bitd_compare_ckum(this, sign, md, linked_inode, entry, fd, child,
+                            &loc);
+
+    if (!skip_stat)
+        br_inc_scrubbed_file(&priv->scrub_stat);
+
+    GF_FREE(sign); /* allocated on post-compute */
+
+    /** fd_unref() takes care of closing fd.. like syncop_close() */
+
+free_md:
+    GF_FREE(md);
+unrefd:
+    fd_unref(fd);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    loc_wipe(&loc);
+    return ret;
 }
 
 static void
-_br_lock_cleaner (void *arg)
+_br_lock_cleaner(void *arg)
 {
-        pthread_mutex_t *mutex = arg;
+    pthread_mutex_t *mutex = arg;
 
-        pthread_mutex_unlock (mutex);
+    pthread_mutex_unlock(mutex);
 }
 
 static void
-wait_for_scrubbing (xlator_t *this, struct br_scanfs *fsscan)
+wait_for_scrubbing(xlator_t *this, struct br_scanfs *fsscan)
 {
-        br_private_t *priv = NULL;
-        struct br_scrubber *fsscrub = NULL;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
 
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
 
-        pthread_cleanup_push (_br_lock_cleaner, &fsscan->waitlock);
-        pthread_mutex_lock (&fsscan->waitlock);
+    pthread_cleanup_push(_br_lock_cleaner, &fsscan->waitlock);
+    pthread_mutex_lock(&fsscan->waitlock);
+    {
+        pthread_cleanup_push(_br_lock_cleaner, &fsscrub->mutex);
+        pthread_mutex_lock(&fsscrub->mutex);
         {
-                pthread_cleanup_push (_br_lock_cleaner, &fsscrub->mutex);
-                pthread_mutex_lock (&fsscrub->mutex);
-                {
-                        list_replace_init (&fsscan->queued, &fsscan->ready);
-
-                        /* wake up scrubbers */
-                        pthread_cond_broadcast (&fsscrub->cond);
-                }
-                pthread_mutex_unlock (&fsscrub->mutex);
-                pthread_cleanup_pop (0);
-
-                while (fsscan->entries != 0)
-                        pthread_cond_wait
-                                    (&fsscan->waitcond, &fsscan->waitlock);
+            list_replace_init(&fsscan->queued, &fsscan->ready);
+
+            /* wake up scrubbers */
+            pthread_cond_broadcast(&fsscrub->cond);
         }
-        pthread_mutex_unlock (&fsscan->waitlock);
-        pthread_cleanup_pop (0);
+        pthread_mutex_unlock(&fsscrub->mutex);
+        pthread_cleanup_pop(0);
+
+        while (fsscan->entries != 0)
+            pthread_cond_wait(&fsscan->waitcond, &fsscan->waitlock);
+    }
+    pthread_mutex_unlock(&fsscan->waitlock);
+    pthread_cleanup_pop(0);
 }
 
 static void
-_br_fsscan_inc_entry_count (struct br_scanfs *fsscan)
+_br_fsscan_inc_entry_count(struct br_scanfs *fsscan)
 {
-        fsscan->entries++;
+    fsscan->entries++;
 }
 
 static void
-_br_fsscan_dec_entry_count (struct br_scanfs *fsscan)
+_br_fsscan_dec_entry_count(struct br_scanfs *fsscan)
 {
-        if (--fsscan->entries == 0) {
-                pthread_mutex_lock (&fsscan->waitlock);
-                {
-                        pthread_cond_signal (&fsscan->waitcond);
-                }
-                pthread_mutex_unlock (&fsscan->waitlock);
+    if (--fsscan->entries == 0) {
+        pthread_mutex_lock(&fsscan->waitlock);
+        {
+            pthread_cond_signal(&fsscan->waitcond);
         }
+        pthread_mutex_unlock(&fsscan->waitlock);
+    }
 }
 
 static void
-_br_fsscan_collect_entry (struct br_scanfs *fsscan,
-                           struct br_fsscan_entry *fsentry)
+_br_fsscan_collect_entry(struct br_scanfs *fsscan,
+                         struct br_fsscan_entry *fsentry)
 {
-        list_add_tail (&fsentry->list, &fsscan->queued);
-        _br_fsscan_inc_entry_count (fsscan);
+    list_add_tail(&fsentry->list, &fsscan->queued);
+    _br_fsscan_inc_entry_count(fsscan);
 }
 
-#define NR_ENTRIES (1<<7) /* ..bulk scrubbing */
+#define NR_ENTRIES (1 << 7) /* ..bulk scrubbing */
 
 int
-br_fsscanner_handle_entry (xlator_t *subvol,
-                           gf_dirent_t *entry, loc_t *parent, void *data)
+br_fsscanner_handle_entry(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                          void *data)
 {
-        int32_t                 ret     = -1;
-        int                     scrub   = 0;
-        br_child_t             *child   = NULL;
-        xlator_t               *this    = NULL;
-        struct br_scanfs       *fsscan  = NULL;
-        struct br_fsscan_entry *fsentry = NULL;
+    int32_t ret = -1;
+    int scrub = 0;
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", subvol, error_return);
-        GF_VALIDATE_OR_GOTO ("bit-rot", data, error_return);
+    GF_VALIDATE_OR_GOTO("bit-rot", subvol, error_return);
+    GF_VALIDATE_OR_GOTO("bit-rot", data, error_return);
 
-        child = data;
-        this = child->this;
-        fsscan = &child->fsscan;
+    child = data;
+    this = child->this;
+    fsscan = &child->fsscan;
 
-        _mask_cancellation ();
+    _mask_cancellation();
 
-        fsentry = GF_CALLOC (1, sizeof (*fsentry), gf_br_mt_br_fsscan_entry_t);
-        if (!fsentry)
-                goto error_return;
+    fsentry = GF_CALLOC(1, sizeof(*fsentry), gf_br_mt_br_fsscan_entry_t);
+    if (!fsentry)
+        goto error_return;
 
-        {
-                fsentry->data = data;
-                fsentry->fsscan = &child->fsscan;
+    {
+        fsentry->data = data;
+        fsentry->fsscan = &child->fsscan;
 
-                /* copy parent loc */
-                ret = loc_copy (&fsentry->parent, parent);
-                if (ret)
-                        goto dealloc;
+        /* copy parent loc */
+        ret = loc_copy(&fsentry->parent, parent);
+        if (ret)
+            goto dealloc;
 
-                /* copy child entry */
-                fsentry->entry = entry_copy (entry);
-                if (!fsentry->entry)
-                        goto locwipe;
+        /* copy child entry */
+        fsentry->entry = entry_copy(entry);
+        if (!fsentry->entry)
+            goto locwipe;
 
-                INIT_LIST_HEAD (&fsentry->list);
-        }
+        INIT_LIST_HEAD(&fsentry->list);
+    }
 
-        LOCK (&fsscan->entrylock);
-        {
-                _br_fsscan_collect_entry (fsscan, fsentry);
-
-                /**
-                 * need not be a equality check as entries may be pushed
-                 * back onto the scanned queue when thread(s) are cleaned.
-                 */
-                if (fsscan->entries >= NR_ENTRIES)
-                        scrub = 1;
-        }
-        UNLOCK (&fsscan->entrylock);
+    LOCK(&fsscan->entrylock);
+    {
+        _br_fsscan_collect_entry(fsscan, fsentry);
 
-        _unmask_cancellation ();
+        /**
+         * need not be a equality check as entries may be pushed
+         * back onto the scanned queue when thread(s) are cleaned.
+         */
+        if (fsscan->entries >= NR_ENTRIES)
+            scrub = 1;
+    }
+    UNLOCK(&fsscan->entrylock);
 
-        if (scrub)
-                wait_for_scrubbing (this, fsscan);
+    _unmask_cancellation();
 
-        return 0;
+    if (scrub)
+        wait_for_scrubbing(this, fsscan);
 
- locwipe:
-        loc_wipe (&fsentry->parent);
- dealloc:
-        GF_FREE (fsentry);
- error_return:
-        return -1;
+    return 0;
+
+locwipe:
+    loc_wipe(&fsentry->parent);
+dealloc:
+    GF_FREE(fsentry);
+error_return:
+    return -1;
 }
 
 int32_t
-br_fsscan_deactivate (xlator_t *this)
+br_fsscan_deactivate(xlator_t *this)
 {
-        int ret = 0;
-        br_private_t *priv = NULL;
-        br_scrub_state_t nstate = 0;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        ret = gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer);
-        if (ret == 0) {
-                nstate = BR_SCRUB_STATE_STALLED;
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Volume is under active scrubbing. Pausing scrub..");
-        } else {
-                nstate = BR_SCRUB_STATE_PAUSED;
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubber paused");
-        }
-
-        _br_monitor_set_scrub_state (scrub_monitor, nstate);
-
-        return 0;
+    int ret = 0;
+    br_private_t *priv = NULL;
+    br_scrub_state_t nstate = 0;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    ret = gf_tw_del_timer(priv->timer_wheel, scrub_monitor->timer);
+    if (ret == 0) {
+        nstate = BR_SCRUB_STATE_STALLED;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Volume is under active scrubbing. Pausing scrub..");
+    } else {
+        nstate = BR_SCRUB_STATE_PAUSED;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber paused");
+    }
+
+    _br_monitor_set_scrub_state(scrub_monitor, nstate);
+
+    return 0;
 }
 
 static void
-br_scrubber_log_time (xlator_t *this, const char *sfx)
+br_scrubber_log_time(xlator_t *this, const char *sfx)
 {
-        char           timestr[1024] = {0,};
-        struct         timeval tv    = {0,};
-        br_private_t  *priv          = NULL;
-
-        priv = this->private;
-
-        gettimeofday (&tv, NULL);
-        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
-
-        if (strcasecmp (sfx, "started") == 0) {
-                br_update_scrub_start_time (&priv->scrub_stat, &tv);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START,
-                        "Scrubbing %s at %s", sfx, timestr);
-        } else {
-                br_update_scrub_finish_time (&priv->scrub_stat, timestr, &tv);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH,
-                        "Scrubbing %s at %s", sfx, timestr);
-        }
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    br_private_t *priv = NULL;
+    time_t now = 0;
+
+    now = gf_time();
+    priv = this->private;
+
+    gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT);
+
+    if (strcasecmp(sfx, "started") == 0) {
+        br_update_scrub_start_time(&priv->scrub_stat, now);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START,
+               "Scrubbing %s at %s", sfx, timestr);
+    } else {
+        br_update_scrub_finish_time(&priv->scrub_stat, timestr, now);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH,
+               "Scrubbing %s at %s", sfx, timestr);
+    }
 }
 
 static void
-br_fsscanner_log_time (xlator_t *this, br_child_t *child, const char *sfx)
+br_fsscanner_log_time(xlator_t *this, br_child_t *child, const char *sfx)
 {
-        char           timestr[1024] = {0,};
-        struct         timeval tv    = {0,};
-
-        gettimeofday (&tv, NULL);
-        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
-
-        if (strcasecmp (sfx, "started") == 0) {
-                gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s",
-                              child->brick_path, sfx, timestr);
-        } else {
-                gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s",
-                              child->brick_path, sfx, timestr);
-        }
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+
+    now = gf_time();
+    gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT);
+
+    if (strcasecmp(sfx, "started") == 0) {
+        gf_msg_debug(this->name, 0, "Scrubbing \"%s\" %s at %s",
+                     child->brick_path, sfx, timestr);
+    } else {
+        gf_msg_debug(this->name, 0, "Scrubbing \"%s\" %s at %s",
+                     child->brick_path, sfx, timestr);
+    }
 }
 
 void
-br_child_set_scrub_state (br_child_t *child, gf_boolean_t state)
+br_child_set_scrub_state(br_child_t *child, gf_boolean_t state)
 {
-        child->active_scrubbing = state;
+    child->active_scrubbing = state;
 }
 
 static void
-br_fsscanner_wait_until_kicked (xlator_t *this, br_child_t *child)
+br_fsscanner_wait_until_kicked(xlator_t *this, br_child_t *child)
 {
-        br_private_t      *priv          = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock);
-        pthread_mutex_lock (&scrub_monitor->wakelock);
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->wakelock);
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        while (!scrub_monitor->kick)
+            pthread_cond_wait(&scrub_monitor->wakecond,
+                              &scrub_monitor->wakelock);
+
+        /* Child lock is to synchronize with disconnect events */
+        pthread_cleanup_push(_br_lock_cleaner, &child->lock);
+        pthread_mutex_lock(&child->lock);
         {
-                while (!scrub_monitor->kick)
-                        pthread_cond_wait (&scrub_monitor->wakecond,
-                                           &scrub_monitor->wakelock);
-
-                /* Child lock is to synchronize with disconnect events */
-                pthread_cleanup_push (_br_lock_cleaner, &child->lock);
-                pthread_mutex_lock (&child->lock);
-                {
-                        scrub_monitor->active_child_count++;
-                        br_child_set_scrub_state (child, _gf_true);
-                }
-                pthread_mutex_unlock (&child->lock);
-                pthread_cleanup_pop (0);
+            scrub_monitor->active_child_count++;
+            br_child_set_scrub_state(child, _gf_true);
         }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
-        pthread_cleanup_pop (0);
+        pthread_mutex_unlock(&child->lock);
+        pthread_cleanup_pop(0);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+    pthread_cleanup_pop(0);
 }
 
 static void
-br_scrubber_entry_control (xlator_t *this)
+br_scrubber_entry_control(xlator_t *this)
 {
-        br_private_t      *priv          = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        LOCK (&scrub_monitor->lock);
-        {
-                /* Move the state to BR_SCRUB_STATE_ACTIVE */
-                if (scrub_monitor->state == BR_SCRUB_STATE_PENDING)
-                        scrub_monitor->state = BR_SCRUB_STATE_ACTIVE;
-                br_scrubber_log_time (this, "started");
-        }
-        UNLOCK (&scrub_monitor->lock);
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    LOCK(&scrub_monitor->lock);
+    {
+        /* Move the state to BR_SCRUB_STATE_ACTIVE */
+        if (scrub_monitor->state == BR_SCRUB_STATE_PENDING)
+            scrub_monitor->state = BR_SCRUB_STATE_ACTIVE;
+        br_scrubber_log_time(this, "started");
+        priv->scrub_stat.scrub_running = 1;
+    }
+    UNLOCK(&scrub_monitor->lock);
 }
 
 static void
-br_scrubber_exit_control (xlator_t *this)
+br_scrubber_exit_control(xlator_t *this)
 {
-        br_private_t      *priv          = NULL;
-        struct br_monitor *scrub_monitor = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        LOCK (&scrub_monitor->lock);
-        {
-                br_scrubber_log_time (this, "finished");
-
-                if (scrub_monitor->state == BR_SCRUB_STATE_ACTIVE) {
-                        (void) br_fsscan_activate (this);
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                                "Volume waiting to get rescheduled..");
-                }
+    LOCK(&scrub_monitor->lock);
+    {
+        br_scrubber_log_time(this, "finished");
+        priv->scrub_stat.scrub_running = 0;
+
+        if (scrub_monitor->state == BR_SCRUB_STATE_ACTIVE) {
+            (void)br_fsscan_activate(this);
+        } else {
+            UNLOCK(&scrub_monitor->lock);
+            gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+                   "Volume waiting to get rescheduled..");
+            return;
         }
-        UNLOCK (&scrub_monitor->lock);
+    }
+    UNLOCK(&scrub_monitor->lock);
 }
 
 static void
-br_fsscanner_entry_control (xlator_t *this, br_child_t *child)
+br_fsscanner_entry_control(xlator_t *this, br_child_t *child)
 {
-                br_fsscanner_log_time (this, child, "started");
+    br_fsscanner_log_time(this, child, "started");
 }
 
 static void
-br_fsscanner_exit_control (xlator_t *this, br_child_t *child)
+br_fsscanner_exit_control(xlator_t *this, br_child_t *child)
 {
-        br_private_t *priv = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        if (!_br_is_child_connected (child)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SCRUB_INFO,
-                        "Brick [%s] disconnected while scrubbing. Scrubbing "
-                        "might be incomplete", child->brick_path);
-        }
-
-        br_fsscanner_log_time (this, child, "finished");
-
-        pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock);
-        pthread_mutex_lock (&scrub_monitor->wakelock);
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (!_br_is_child_connected(child)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCRUB_INFO,
+               "Brick [%s] disconnected while scrubbing. Scrubbing "
+               "might be incomplete",
+               child->brick_path);
+    }
+
+    br_fsscanner_log_time(this, child, "finished");
+
+    pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->wakelock);
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        scrub_monitor->active_child_count--;
+        pthread_cleanup_push(_br_lock_cleaner, &child->lock);
+        pthread_mutex_lock(&child->lock);
         {
-                scrub_monitor->active_child_count--;
-                pthread_cleanup_push (_br_lock_cleaner, &child->lock);
-                pthread_mutex_lock (&child->lock);
-                {
-                        br_child_set_scrub_state (child, _gf_false);
-                }
-                pthread_mutex_unlock (&child->lock);
-                pthread_cleanup_pop (0);
-
-                if (scrub_monitor->active_child_count == 0) {
-                        /* The last child has finished scrubbing.
-                         * Set the kick to false and  wake up other
-                         * children who are waiting for the last
-                         * child to complete scrubbing.
-                         */
-                        scrub_monitor->kick = _gf_false;
-                        pthread_cond_broadcast (&scrub_monitor->wakecond);
-
-                        /* Signal monitor thread waiting for the all
-                         * the children to finish scrubbing.
-                         */
-                        pthread_cleanup_push (_br_lock_cleaner,
-                                              &scrub_monitor->donelock);
-                        pthread_mutex_lock (&scrub_monitor->donelock);
-                        {
-                                scrub_monitor->done = _gf_true;
-                                pthread_cond_signal (&scrub_monitor->donecond);
-                        }
-                        pthread_mutex_unlock (&scrub_monitor->donelock);
-                        pthread_cleanup_pop (0);
-                } else {
-                        while (scrub_monitor->active_child_count)
-                                pthread_cond_wait (&scrub_monitor->wakecond,
-                                                   &scrub_monitor->wakelock);
-                }
+            br_child_set_scrub_state(child, _gf_false);
         }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
-        pthread_cleanup_pop (0);
+        pthread_mutex_unlock(&child->lock);
+        pthread_cleanup_pop(0);
+
+        if (scrub_monitor->active_child_count == 0) {
+            /* The last child has finished scrubbing.
+             * Set the kick to false and  wake up other
+             * children who are waiting for the last
+             * child to complete scrubbing.
+             */
+            scrub_monitor->kick = _gf_false;
+            pthread_cond_broadcast(&scrub_monitor->wakecond);
+
+            /* Signal monitor thread waiting for the all
+             * the children to finish scrubbing.
+             */
+            pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->donelock);
+            pthread_mutex_lock(&scrub_monitor->donelock);
+            {
+                scrub_monitor->done = _gf_true;
+                pthread_cond_signal(&scrub_monitor->donecond);
+            }
+            pthread_mutex_unlock(&scrub_monitor->donelock);
+            pthread_cleanup_pop(0);
+        } else {
+            while (scrub_monitor->active_child_count)
+                pthread_cond_wait(&scrub_monitor->wakecond,
+                                  &scrub_monitor->wakelock);
+        }
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+    pthread_cleanup_pop(0);
 }
 
 void *
-br_fsscanner (void *arg)
+br_fsscanner(void *arg)
 {
-        loc_t               loc     = {0,};
-        br_child_t         *child   = NULL;
-        xlator_t           *this    = NULL;
-        struct br_scanfs   *fsscan  = NULL;
-
-        child = arg;
-        this = child->this;
-        fsscan = &child->fsscan;
-
-        THIS = this;
-        loc.inode = child->table->root;
-
-        while (1) {
-                br_fsscanner_wait_until_kicked (this, child);
-                {
-                        /* precursor for scrub */
-                        br_fsscanner_entry_control (this, child);
-
-                        /* scrub */
-                        (void) syncop_ftw (child->xl,
-                                           &loc, GF_CLIENT_PID_SCRUB,
-                                           child, br_fsscanner_handle_entry);
-                        if (!list_empty (&fsscan->queued))
-                                wait_for_scrubbing (this, fsscan);
-
-                        /* scrub exit criteria */
-                        br_fsscanner_exit_control (this, child);
-                }
+    loc_t loc = {
+        0,
+    };
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    struct br_scanfs *fsscan = NULL;
+
+    child = arg;
+    this = child->this;
+    fsscan = &child->fsscan;
+
+    THIS = this;
+    loc.inode = child->table->root;
+
+    while (1) {
+        br_fsscanner_wait_until_kicked(this, child);
+        {
+            /* precursor for scrub */
+            br_fsscanner_entry_control(this, child);
+
+            /* scrub */
+            (void)syncop_ftw(child->xl, &loc, GF_CLIENT_PID_SCRUB, child,
+                             br_fsscanner_handle_entry);
+            if (!list_empty(&fsscan->queued))
+                wait_for_scrubbing(this, fsscan);
+
+            /* scrub exit criteria */
+            br_fsscanner_exit_control(this, child);
         }
+    }
 
-        return NULL;
+    return NULL;
 }
 
 /**
@@ -801,216 +836,268 @@ br_fsscanner (void *arg)
  * non-pending timer.
  */
 void
-br_kickstart_scanner (struct gf_tw_timer_list *timer,
-                      void *data, unsigned long calltime)
+br_kickstart_scanner(struct gf_tw_timer_list *timer, void *data,
+                     unsigned long calltime)
 {
-        xlator_t *this = NULL;
-        struct br_monitor *scrub_monitor = data;
-        br_private_t *priv = NULL;
-
-        THIS = this = scrub_monitor->this;
-        priv = this->private;
-
-        /* Reset scrub statistics */
-        priv->scrub_stat.scrubbed_files = 0;
-        priv->scrub_stat.unsigned_files = 0;
-
-        /* Moves state from PENDING to ACTIVE */
-        (void) br_scrubber_entry_control (this);
-
-        /* kickstart scanning.. */
-        pthread_mutex_lock (&scrub_monitor->wakelock);
-        {
-                scrub_monitor->kick = _gf_true;
-                GF_ASSERT (scrub_monitor->active_child_count == 0);
-                pthread_cond_broadcast (&scrub_monitor->wakecond);
-        }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
-
-        return;
+    xlator_t *this = NULL;
+    struct br_monitor *scrub_monitor = data;
+    br_private_t *priv = NULL;
+
+    THIS = this = scrub_monitor->this;
+    priv = this->private;
+
+    /* Reset scrub statistics */
+    priv->scrub_stat.scrubbed_files = 0;
+    priv->scrub_stat.unsigned_files = 0;
+
+    /* Moves state from PENDING to ACTIVE */
+    (void)br_scrubber_entry_control(this);
+
+    /* kickstart scanning.. */
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        scrub_monitor->kick = _gf_true;
+        GF_ASSERT(scrub_monitor->active_child_count == 0);
+        pthread_cond_broadcast(&scrub_monitor->wakecond);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+
+    return;
 }
 
 static uint32_t
-br_fsscan_calculate_delta (uint32_t times)
+br_fsscan_calculate_delta(uint32_t times)
 {
-        return times;
+    return times;
 }
 
-#define BR_SCRUB_HOURLY     (60 * 60)
-#define BR_SCRUB_DAILY      (1 * 24 * 60 * 60)
-#define BR_SCRUB_WEEKLY     (7 * 24 * 60 * 60)
-#define BR_SCRUB_BIWEEKLY   (14 * 24 * 60 * 60)
-#define BR_SCRUB_MONTHLY    (30 * 24 * 60 * 60)
+#define BR_SCRUB_ONDEMAND (1)
+#define BR_SCRUB_MINUTE (60)
+#define BR_SCRUB_HOURLY (60 * 60)
+#define BR_SCRUB_DAILY (1 * 24 * 60 * 60)
+#define BR_SCRUB_WEEKLY (7 * 24 * 60 * 60)
+#define BR_SCRUB_BIWEEKLY (14 * 24 * 60 * 60)
+#define BR_SCRUB_MONTHLY (30 * 24 * 60 * 60)
 
 static unsigned int
-br_fsscan_calculate_timeout (scrub_freq_t freq)
+br_fsscan_calculate_timeout(scrub_freq_t freq)
 {
-        uint32_t timo = 0;
+    uint32_t timo = 0;
 
-        switch (freq) {
+    switch (freq) {
+        case BR_FSSCRUB_FREQ_MINUTE:
+            timo = br_fsscan_calculate_delta(BR_SCRUB_MINUTE);
+            break;
         case BR_FSSCRUB_FREQ_HOURLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_HOURLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_HOURLY);
+            break;
         case BR_FSSCRUB_FREQ_DAILY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_DAILY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_DAILY);
+            break;
         case BR_FSSCRUB_FREQ_WEEKLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_WEEKLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_WEEKLY);
+            break;
         case BR_FSSCRUB_FREQ_BIWEEKLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_BIWEEKLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_BIWEEKLY);
+            break;
         case BR_FSSCRUB_FREQ_MONTHLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_MONTHLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_MONTHLY);
+            break;
         default:
-                timo = 0;
-        }
+            timo = 0;
+    }
 
-        return timo;
+    return timo;
 }
 
 int32_t
-br_fsscan_schedule (xlator_t *this)
+br_fsscan_schedule(xlator_t *this)
 {
-        uint32_t timo = 0;
-        br_private_t *priv = NULL;
-        struct timeval tv = {0,};
-        char timestr[1024] = {0,};
-        struct br_scrubber *fsscrub = NULL;
-        struct gf_tw_timer_list *timer = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        (void) gettimeofday (&tv, NULL);
-        scrub_monitor->boot = tv.tv_sec;
-
-        timo = br_fsscan_calculate_timeout (fsscrub->frequency);
-        if (timo == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
-                        "BUG: Zero schedule timeout");
-                goto error_return;
-        }
-
-        scrub_monitor->timer = GF_CALLOC (1, sizeof (*scrub_monitor->timer),
-                                   gf_br_stub_mt_br_scanner_freq_t);
-        if (!scrub_monitor->timer)
-                goto error_return;
-
-        timer = scrub_monitor->timer;
-        INIT_LIST_HEAD (&timer->entry);
+    uint32_t timo = 0;
+    br_private_t *priv = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    struct br_scrubber *fsscrub = NULL;
+    struct gf_tw_timer_list *timer = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    scrub_monitor->boot = gf_time();
+
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        goto error_return;
+    }
+
+    scrub_monitor->timer = GF_CALLOC(1, sizeof(*scrub_monitor->timer),
+                                     gf_br_stub_mt_br_scanner_freq_t);
+    if (!scrub_monitor->timer)
+        goto error_return;
+
+    timer = scrub_monitor->timer;
+    INIT_LIST_HEAD(&timer->entry);
+
+    timer->data = scrub_monitor;
+    timer->expires = timo;
+    timer->function = br_kickstart_scanner;
+
+    gf_tw_add_timer(priv->timer_wheel, timer);
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+
+    gf_time_fmt(timestr, sizeof(timestr), (scrub_monitor->boot + timo),
+                gf_timefmt_FT);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Scrubbing is "
+           "scheduled to run at %s",
+           timestr);
+
+    return 0;
+
+error_return:
+    return -1;
+}
 
-        timer->data = scrub_monitor;
-        timer->expires = timo;
-        timer->function = br_kickstart_scanner;
+int32_t
+br_fsscan_activate(xlator_t *this)
+{
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    now = gf_time();
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        return -1;
+    }
 
-        gf_tw_add_timer (priv->timer_wheel, timer);
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
 
-        gf_time_fmt (timestr, sizeof (timestr),
-                     (scrub_monitor->boot + timo), gf_timefmt_FT);
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is "
-                "scheduled to run at %s", timestr);
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
+    (void)gf_tw_mod_timer(priv->timer_wheel, scrub_monitor->timer, timo);
 
-        return 0;
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Scrubbing is "
+           "rescheduled to run at %s",
+           timestr);
 
- error_return:
-        return -1;
+    return 0;
 }
 
 int32_t
-br_fsscan_activate (xlator_t *this)
+br_fsscan_reschedule(xlator_t *this)
 {
-        uint32_t            timo    = 0;
-        char timestr[1024]          = {0,};
-        struct timeval      now     = {0,};
-        br_private_t       *priv    = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_monitor  *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        (void) gettimeofday (&now, NULL);
-        timo = br_fsscan_calculate_timeout (fsscrub->frequency);
-        if (timo == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
-                        "BUG: Zero schedule timeout");
-                return -1;
-        }
-
-        pthread_mutex_lock (&scrub_monitor->donelock);
-        {
-                scrub_monitor->done = _gf_false;
-        }
-        pthread_mutex_unlock (&scrub_monitor->donelock);
+    int32_t ret = 0;
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (!fsscrub->frequency_reconf)
+        return 0;
 
-        gf_time_fmt (timestr, sizeof (timestr),
-                     (now.tv_sec + timo), gf_timefmt_FT);
-        (void) gf_tw_mod_timer (priv->timer_wheel, scrub_monitor->timer, timo);
+    now = gf_time();
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        return -1;
+    }
 
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is "
-                "rescheduled to run at %s", timestr);
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
 
-        return 0;
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+
+    ret = gf_tw_mod_timer_pending(priv->timer_wheel, scrub_monitor->timer,
+                                  timo);
+    if (ret == 0)
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber is currently running and would be "
+               "rescheduled after completion");
+    else {
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubbing rescheduled to run at %s", timestr);
+    }
+
+    return 0;
 }
 
 int32_t
-br_fsscan_reschedule (xlator_t *this)
+br_fsscan_ondemand(xlator_t *this)
 {
-        int32_t             ret     = 0;
-        uint32_t            timo    = 0;
-        char timestr[1024]          = {0,};
-        struct timeval      now     = {0,};
-        br_private_t       *priv    = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_monitor  *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        if (!fsscrub->frequency_reconf)
-                return 0;
-
-        (void) gettimeofday (&now, NULL);
-        timo = br_fsscan_calculate_timeout (fsscrub->frequency);
-        if (timo == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
-                        "BUG: Zero schedule timeout");
-                return -1;
-        }
-
-        gf_time_fmt (timestr, sizeof (timestr),
-                     (now.tv_sec + timo), gf_timefmt_FT);
-
-        pthread_mutex_lock (&scrub_monitor->donelock);
-        {
-                scrub_monitor->done = _gf_false;
-        }
-        pthread_mutex_unlock (&scrub_monitor->donelock);
-
-        ret = gf_tw_mod_timer_pending (priv->timer_wheel, scrub_monitor->timer, timo);
-        if (ret == 0)
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubber is currently running and would be "
-                        "rescheduled after completion");
-        else {
-                _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubbing rescheduled to run at %s", timestr);
-        }
-
-        return 0;
+    int32_t ret = 0;
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    now = gf_time();
+    timo = BR_SCRUB_ONDEMAND;
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+
+    ret = gf_tw_mod_timer_pending(priv->timer_wheel, scrub_monitor->timer,
+                                  timo);
+    if (ret == 0)
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber is currently running and would be "
+               "rescheduled after completion");
+    else {
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Ondemand Scrubbing scheduled to run at %s", timestr);
+    }
+
+    return 0;
 }
 
-#define BR_SCRUB_THREAD_SCALE_LAZY       0
-#define BR_SCRUB_THREAD_SCALE_NORMAL     0.4
+#define BR_SCRUB_THREAD_SCALE_LAZY 0
+#define BR_SCRUB_THREAD_SCALE_NORMAL 0.4
 #define BR_SCRUB_THREAD_SCALE_AGGRESSIVE 1.0
 
 #ifndef M_E
@@ -1023,111 +1110,105 @@ br_fsscan_reschedule (xlator_t *this)
  * the scale based on the number of processor cores too.
  */
 static unsigned int
-br_scrubber_calc_scale (xlator_t *this,
-                        br_private_t *priv, scrub_throttle_t throttle)
+br_scrubber_calc_scale(xlator_t *this, br_private_t *priv,
+                       scrub_throttle_t throttle)
 {
-        unsigned int scale = 0;
+    unsigned int scale = 0;
 
-        switch (throttle) {
+    switch (throttle) {
         case BR_SCRUB_THROTTLE_VOID:
         case BR_SCRUB_THROTTLE_STALLED:
-                scale = 0;
-                break;
+            scale = 0;
+            break;
         case BR_SCRUB_THROTTLE_LAZY:
-                scale = priv->child_count *
-                              pow (M_E, BR_SCRUB_THREAD_SCALE_LAZY);
-                break;
+            scale = priv->child_count * pow(M_E, BR_SCRUB_THREAD_SCALE_LAZY);
+            break;
         case BR_SCRUB_THROTTLE_NORMAL:
-                scale = priv->child_count *
-                              pow (M_E, BR_SCRUB_THREAD_SCALE_NORMAL);
-                break;
+            scale = priv->child_count * pow(M_E, BR_SCRUB_THREAD_SCALE_NORMAL);
+            break;
         case BR_SCRUB_THROTTLE_AGGRESSIVE:
-                scale = priv->child_count *
-                              pow (M_E, BR_SCRUB_THREAD_SCALE_AGGRESSIVE);
-                break;
+            scale = priv->child_count *
+                    pow(M_E, BR_SCRUB_THREAD_SCALE_AGGRESSIVE);
+            break;
         default:
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_UNKNOWN_THROTTLE,
-                        "Unknown throttle %d", throttle);
-        }
-
-        return scale;
+            gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_UNKNOWN_THROTTLE,
+                   "Unknown throttle %d", throttle);
+    }
 
+    return scale;
 }
 
 static br_child_t *
-_br_scrubber_get_next_child (struct br_scrubber *fsscrub)
+_br_scrubber_get_next_child(struct br_scrubber *fsscrub)
 {
-        br_child_t *child = NULL;
+    br_child_t *child = NULL;
 
-        child = list_first_entry (&fsscrub->scrublist, br_child_t, list);
-        list_rotate_left (&fsscrub->scrublist);
+    child = list_first_entry(&fsscrub->scrublist, br_child_t, list);
+    list_rotate_left(&fsscrub->scrublist);
 
-        return child;
+    return child;
 }
 
 static void
-_br_scrubber_get_entry (br_child_t *child, struct br_fsscan_entry **fsentry)
+_br_scrubber_get_entry(br_child_t *child, struct br_fsscan_entry **fsentry)
 {
-        struct br_scanfs *fsscan = &child->fsscan;
+    struct br_scanfs *fsscan = &child->fsscan;
 
-        if (list_empty (&fsscan->ready))
-                return;
-        *fsentry = list_first_entry
-                            (&fsscan->ready, struct br_fsscan_entry, list);
-        list_del_init (&(*fsentry)->list);
+    if (list_empty(&fsscan->ready))
+        return;
+    *fsentry = list_first_entry(&fsscan->ready, struct br_fsscan_entry, list);
+    list_del_init(&(*fsentry)->list);
 }
 
 static void
-_br_scrubber_find_scrubbable_entry (struct br_scrubber *fsscrub,
-                                     struct br_fsscan_entry **fsentry)
+_br_scrubber_find_scrubbable_entry(struct br_scrubber *fsscrub,
+                                   struct br_fsscan_entry **fsentry)
 {
-        br_child_t *child = NULL;
-        br_child_t *firstchild = NULL;
-
-        while (1) {
-                while (list_empty (&fsscrub->scrublist))
-                        pthread_cond_wait (&fsscrub->cond, &fsscrub->mutex);
+    br_child_t *child = NULL;
+    br_child_t *firstchild = NULL;
 
-                firstchild = NULL;
-                for (child = _br_scrubber_get_next_child (fsscrub);
-                     child != firstchild;
-                     child = _br_scrubber_get_next_child (fsscrub)) {
+    while (1) {
+        while (list_empty(&fsscrub->scrublist))
+            pthread_cond_wait(&fsscrub->cond, &fsscrub->mutex);
 
-                        if (!firstchild)
-                                firstchild = child;
+        firstchild = NULL;
+        for (child = _br_scrubber_get_next_child(fsscrub); child != firstchild;
+             child = _br_scrubber_get_next_child(fsscrub)) {
+            if (!firstchild)
+                firstchild = child;
 
-                        _br_scrubber_get_entry (child, fsentry);
-                        if (*fsentry)
-                                break;
-                }
+            _br_scrubber_get_entry(child, fsentry);
+            if (*fsentry)
+                break;
+        }
 
-                if (*fsentry)
-                        break;
+        if (*fsentry)
+            break;
 
-                /* nothing to work on.. wait till available */
-                pthread_cond_wait (&fsscrub->cond, &fsscrub->mutex);
-        }
+        /* nothing to work on.. wait till available */
+        pthread_cond_wait(&fsscrub->cond, &fsscrub->mutex);
+    }
 }
 
 static void
-br_scrubber_pick_entry (struct br_scrubber *fsscrub,
-                        struct br_fsscan_entry **fsentry)
+br_scrubber_pick_entry(struct br_scrubber *fsscrub,
+                       struct br_fsscan_entry **fsentry)
 {
-        pthread_cleanup_push (_br_lock_cleaner, &fsscrub->mutex);
+    pthread_cleanup_push(_br_lock_cleaner, &fsscrub->mutex);
 
-        pthread_mutex_lock (&fsscrub->mutex);
-        {
-                *fsentry = NULL;
-                _br_scrubber_find_scrubbable_entry (fsscrub, fsentry);
-        }
-        pthread_mutex_unlock (&fsscrub->mutex);
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        *fsentry = NULL;
+        _br_scrubber_find_scrubbable_entry(fsscrub, fsentry);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
 
-        pthread_cleanup_pop (0);
+    pthread_cleanup_pop(0);
 }
 
 struct br_scrub_entry {
-        gf_boolean_t scrubbed;
-        struct br_fsscan_entry *fsentry;
+    gf_boolean_t scrubbed;
+    struct br_fsscan_entry *fsentry;
 };
 
 /**
@@ -1137,664 +1218,702 @@ struct br_scrub_entry {
  * in the ->pending queue or when an object is undergoing scrubbing.
  */
 static void
-br_scrubber_entry_handle (void *arg)
+br_scrubber_entry_handle(void *arg)
 {
-        struct br_scanfs       *fsscan  = NULL;
-        struct br_scrub_entry  *sentry  = NULL;
-        struct br_fsscan_entry *fsentry = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_scrub_entry *sentry = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
 
-        sentry = arg;
+    sentry = arg;
 
-        fsentry = sentry->fsentry;
-        fsscan  = fsentry->fsscan;
+    fsentry = sentry->fsentry;
+    fsscan = fsentry->fsscan;
 
-        LOCK (&fsscan->entrylock);
-        {
-                if (sentry->scrubbed) {
-                        _br_fsscan_dec_entry_count (fsscan);
-
-                        /* cleanup ->entry */
-                        fsentry->data   = NULL;
-                        fsentry->fsscan = NULL;
-                        loc_wipe (&fsentry->parent);
-                        gf_dirent_entry_free (fsentry->entry);
-
-                        GF_FREE (sentry->fsentry);
-                } else {
-                        /* (re)queue the entry again for scrub */
-                        _br_fsscan_collect_entry (fsscan, sentry->fsentry);
-                }
+    LOCK(&fsscan->entrylock);
+    {
+        if (sentry->scrubbed) {
+            _br_fsscan_dec_entry_count(fsscan);
+
+            /* cleanup ->entry */
+            fsentry->data = NULL;
+            fsentry->fsscan = NULL;
+            loc_wipe(&fsentry->parent);
+            gf_dirent_entry_free(fsentry->entry);
+
+            GF_FREE(sentry->fsentry);
+        } else {
+            /* (re)queue the entry again for scrub */
+            _br_fsscan_collect_entry(fsscan, sentry->fsentry);
         }
-        UNLOCK (&fsscan->entrylock);
+    }
+    UNLOCK(&fsscan->entrylock);
 }
 
 static void
-br_scrubber_scrub_entry (xlator_t *this, struct br_fsscan_entry *fsentry)
+br_scrubber_scrub_entry(xlator_t *this, struct br_fsscan_entry *fsentry)
 {
-        struct br_scrub_entry sentry = {0, };
-
-        sentry.scrubbed = 0;
-        sentry.fsentry = fsentry;
-
-        pthread_cleanup_push (br_scrubber_entry_handle, &sentry);
-        {
-                (void) br_scrubber_scrub_begin (this, fsentry);
-                sentry.scrubbed = 1;
-        }
-        pthread_cleanup_pop (1);
+    struct br_scrub_entry sentry = {
+        0,
+    };
+
+    sentry.scrubbed = 0;
+    sentry.fsentry = fsentry;
+
+    pthread_cleanup_push(br_scrubber_entry_handle, &sentry);
+    {
+        (void)br_scrubber_scrub_begin(this, fsentry);
+        sentry.scrubbed = 1;
+    }
+    pthread_cleanup_pop(1);
 }
 
-void *br_scrubber_proc (void *arg)
+void *
+br_scrubber_proc(void *arg)
 {
-        xlator_t *this = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_fsscan_entry *fsentry = NULL;
+    xlator_t *this = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
 
-        fsscrub = arg;
-        THIS = this = fsscrub->this;
+    fsscrub = arg;
+    THIS = this = fsscrub->this;
 
-        while (1) {
-                br_scrubber_pick_entry (fsscrub, &fsentry);
-                br_scrubber_scrub_entry (this, fsentry);
-                sleep (1);
-        }
+    while (1) {
+        br_scrubber_pick_entry(fsscrub, &fsentry);
+        br_scrubber_scrub_entry(this, fsentry);
+        sleep(1);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 static int32_t
-br_scrubber_scale_up (xlator_t *this,
-                      struct br_scrubber *fsscrub,
-                      unsigned int v1, unsigned int v2)
+br_scrubber_scale_up(xlator_t *this, struct br_scrubber *fsscrub,
+                     unsigned int v1, unsigned int v2)
 {
-        int i = 0;
-        int32_t ret = -1;
-        int diff = 0;
-        struct br_scrubbers *scrub = NULL;
-
-        diff = (int)(v2 - v1);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCALING_UP_SCRUBBER,
-                "Scaling up scrubbers [%d => %d]", v1, v2);
-
-        for (i = 0; i < diff; i++) {
-                scrub = GF_CALLOC (diff, sizeof (*scrub),
-                                   gf_br_mt_br_scrubber_t);
-                if (!scrub)
-                        break;
-
-                INIT_LIST_HEAD (&scrub->list);
-                ret = gf_thread_create (&scrub->scrubthread,
-                                        NULL, br_scrubber_proc, fsscrub);
-                if (ret)
-                        break;
-
-                fsscrub->nr_scrubbers++;
-                list_add_tail (&scrub->list, &fsscrub->scrubbers);
-        }
+    int i = 0;
+    int32_t ret = -1;
+    int diff = 0;
+    struct br_scrubbers *scrub = NULL;
 
-        if ((i != diff) && !scrub)
-                goto error_return;
+    diff = (int)(v2 - v1);
 
-        if (i != diff) /* degraded scaling.. */
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_UP_FAILED,
-                        "Could not fully scale up to %d scrubber(s). Spawned "
-                        "%d/%d [total scrubber(s): %d]", v2, i, diff, (v1 + i));
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCALING_UP_SCRUBBER,
+           "Scaling up scrubbers [%d => %d]", v1, v2);
 
-        return 0;
+    for (i = 0; i < diff; i++) {
+        scrub = GF_CALLOC(diff, sizeof(*scrub), gf_br_mt_br_scrubber_t);
+        if (!scrub)
+            break;
 
- error_return:
-        return -1;
+        INIT_LIST_HEAD(&scrub->list);
+        ret = gf_thread_create(&scrub->scrubthread, NULL, br_scrubber_proc,
+                               fsscrub, "brsproc");
+        if (ret)
+            break;
+
+        fsscrub->nr_scrubbers++;
+        list_add_tail(&scrub->list, &fsscrub->scrubbers);
+    }
+
+    if ((i != diff) && !scrub)
+        goto error_return;
+
+    if (i != diff) /* degraded scaling.. */
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_UP_FAILED,
+               "Could not fully scale up to %d scrubber(s). Spawned "
+               "%d/%d [total scrubber(s): %d]",
+               v2, i, diff, (v1 + i));
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
 static int32_t
-br_scrubber_scale_down (xlator_t *this,
-                        struct br_scrubber *fsscrub,
-                        unsigned int v1, unsigned int v2)
+br_scrubber_scale_down(xlator_t *this, struct br_scrubber *fsscrub,
+                       unsigned int v1, unsigned int v2)
 {
-        int i = 0;
-        int diff = 0;
-        int32_t ret = -1;
-        struct br_scrubbers *scrub = NULL;
-
-        diff = (int)(v1 - v2);
+    int i = 0;
+    int diff = 0;
+    int32_t ret = -1;
+    struct br_scrubbers *scrub = NULL;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCALE_DOWN_SCRUBBER,
-                "Scaling down scrubbers [%d => %d]", v1, v2);
+    diff = (int)(v1 - v2);
 
-        for (i = 0 ; i < diff; i++) {
-                scrub = list_first_entry
-                            (&fsscrub->scrubbers, struct br_scrubbers, list);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCALE_DOWN_SCRUBBER,
+           "Scaling down scrubbers [%d => %d]", v1, v2);
 
-                list_del_init (&scrub->list);
-                ret = gf_thread_cleanup_xint (scrub->scrubthread);
-                if (ret)
-                        break;
-                GF_FREE (scrub);
-
-                fsscrub->nr_scrubbers--;
-        }
+    for (i = 0; i < diff; i++) {
+        scrub = list_first_entry(&fsscrub->scrubbers, struct br_scrubbers,
+                                 list);
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRB_MSG_SCALE_DOWN_FAILED, "Could not fully scale down "
-                        "to %d scrubber(s). Terminated %d/%d [total "
-                        "scrubber(s): %d]", v1, i, diff, (v2 - i));
-                ret = 0;
-        }
+        list_del_init(&scrub->list);
+        ret = gf_thread_cleanup_xint(scrub->scrubthread);
+        if (ret)
+            break;
+        GF_FREE(scrub);
+
+        fsscrub->nr_scrubbers--;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_DOWN_FAILED,
+               "Could not fully scale down "
+               "to %d scrubber(s). Terminated %d/%d [total "
+               "scrubber(s): %d]",
+               v1, i, diff, (v2 - i));
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-br_scrubber_configure (xlator_t *this, br_private_t *priv,
-                       struct br_scrubber *fsscrub, scrub_throttle_t nthrottle)
+br_scrubber_configure(xlator_t *this, br_private_t *priv,
+                      struct br_scrubber *fsscrub, scrub_throttle_t nthrottle)
 {
-        int32_t ret = 0;
-        unsigned int v1 = 0;
-        unsigned int v2 = 0;
+    int32_t ret = 0;
+    unsigned int v1 = 0;
+    unsigned int v2 = 0;
 
-        v1 = fsscrub->nr_scrubbers;
-        v2 = br_scrubber_calc_scale (this, priv, nthrottle);
+    v1 = fsscrub->nr_scrubbers;
+    v2 = br_scrubber_calc_scale(this, priv, nthrottle);
 
-        if (v1 == v2)
-                return 0;
+    if (v1 == v2)
+        return 0;
 
-        if (v1 > v2)
-                ret = br_scrubber_scale_down (this, fsscrub, v1, v2);
-        else
-                ret = br_scrubber_scale_up (this, fsscrub, v1, v2);
+    if (v1 > v2)
+        ret = br_scrubber_scale_down(this, fsscrub, v1, v2);
+    else
+        ret = br_scrubber_scale_up(this, fsscrub, v1, v2);
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-br_scrubber_fetch_option (xlator_t *this,
-                          char *opt, dict_t *options, char **value)
+br_scrubber_fetch_option(xlator_t *this, char *opt, dict_t *options,
+                         char **value)
 {
-        if (options)
-                GF_OPTION_RECONF (opt, *value, options, str, error_return);
-        else
-                GF_OPTION_INIT (opt, *value, str, error_return);
+    if (options)
+        GF_OPTION_RECONF(opt, *value, options, str, error_return);
+    else
+        GF_OPTION_INIT(opt, *value, str, error_return);
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 /* internal "throttle" override */
-#define BR_SCRUB_STALLED  "STALLED"
+#define BR_SCRUB_STALLED "STALLED"
 
 /* TODO: token buket spec */
 static int32_t
-br_scrubber_handle_throttle (xlator_t *this, br_private_t *priv,
-                             dict_t *options, gf_boolean_t scrubstall)
+br_scrubber_handle_throttle(xlator_t *this, br_private_t *priv, dict_t *options,
+                            gf_boolean_t scrubstall)
 {
-        int32_t ret = 0;
-        char *tmp = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        scrub_throttle_t nthrottle = BR_SCRUB_THROTTLE_VOID;
-
-        fsscrub = &priv->fsscrub;
-        fsscrub->throttle_reconf = _gf_false;
-
-        ret = br_scrubber_fetch_option (this, "scrub-throttle", options, &tmp);
-        if (ret)
-                goto error_return;
-
-        if (scrubstall)
-                tmp = BR_SCRUB_STALLED;
-
-        if (strcasecmp (tmp, "lazy") == 0)
-                nthrottle = BR_SCRUB_THROTTLE_LAZY;
-        else if (strcasecmp (tmp, "normal") == 0)
-                nthrottle = BR_SCRUB_THROTTLE_NORMAL;
-        else if (strcasecmp (tmp, "aggressive") == 0)
-                nthrottle = BR_SCRUB_THROTTLE_AGGRESSIVE;
-        else if (strcasecmp (tmp, BR_SCRUB_STALLED) == 0)
-                nthrottle = BR_SCRUB_THROTTLE_STALLED;
-        else
-                goto error_return;
-
-        /* on failure old throttling value is preserved */
-        ret = br_scrubber_configure (this, priv, fsscrub, nthrottle);
-        if (ret)
-                goto error_return;
-
-        if (fsscrub->throttle != nthrottle)
-                fsscrub->throttle_reconf = _gf_true;
-
-        fsscrub->throttle = nthrottle;
-        return 0;
-
- error_return:
-        return -1;
+    int32_t ret = 0;
+    char *tmp = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    scrub_throttle_t nthrottle = BR_SCRUB_THROTTLE_VOID;
+
+    fsscrub = &priv->fsscrub;
+    fsscrub->throttle_reconf = _gf_false;
+
+    ret = br_scrubber_fetch_option(this, "scrub-throttle", options, &tmp);
+    if (ret)
+        goto error_return;
+
+    if (scrubstall)
+        tmp = BR_SCRUB_STALLED;
+
+    if (strcasecmp(tmp, "lazy") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_LAZY;
+    else if (strcasecmp(tmp, "normal") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_NORMAL;
+    else if (strcasecmp(tmp, "aggressive") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_AGGRESSIVE;
+    else if (strcasecmp(tmp, BR_SCRUB_STALLED) == 0)
+        nthrottle = BR_SCRUB_THROTTLE_STALLED;
+    else
+        goto error_return;
+
+    /* on failure old throttling value is preserved */
+    ret = br_scrubber_configure(this, priv, fsscrub, nthrottle);
+    if (ret)
+        goto error_return;
+
+    if (fsscrub->throttle != nthrottle)
+        fsscrub->throttle_reconf = _gf_true;
+
+    fsscrub->throttle = nthrottle;
+    return 0;
+
+error_return:
+    return -1;
 }
 
 static int32_t
-br_scrubber_handle_stall (xlator_t *this, br_private_t *priv,
-                          dict_t *options, gf_boolean_t *scrubstall)
+br_scrubber_handle_stall(xlator_t *this, br_private_t *priv, dict_t *options,
+                         gf_boolean_t *scrubstall)
 {
-        int32_t ret = 0;
-        char *tmp = NULL;
+    int32_t ret = 0;
+    char *tmp = NULL;
 
-        ret = br_scrubber_fetch_option (this, "scrub-state", options, &tmp);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_fetch_option(this, "scrub-state", options, &tmp);
+    if (ret)
+        goto error_return;
 
-        if (strcasecmp (tmp, "pause") == 0) /* anything else is active */
-                *scrubstall = _gf_true;
+    if (strcasecmp(tmp, "pause") == 0) /* anything else is active */
+        *scrubstall = _gf_true;
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static int32_t
-br_scrubber_handle_freq (xlator_t *this, br_private_t *priv,
-                         dict_t *options, gf_boolean_t scrubstall)
+br_scrubber_handle_freq(xlator_t *this, br_private_t *priv, dict_t *options,
+                        gf_boolean_t scrubstall)
 {
-        int32_t ret  = -1;
-        char *tmp = NULL;
-        scrub_freq_t frequency = BR_FSSCRUB_FREQ_HOURLY;
-        struct br_scrubber *fsscrub = NULL;
-
-        fsscrub = &priv->fsscrub;
-        fsscrub->frequency_reconf = _gf_true;
-
-        ret = br_scrubber_fetch_option (this, "scrub-freq", options, &tmp);
-        if (ret)
-                goto error_return;
-
-        if (scrubstall)
-                tmp = BR_SCRUB_STALLED;
-
-        if (strcasecmp (tmp, "hourly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_HOURLY;
-        } else if (strcasecmp (tmp, "daily") == 0) {
-                frequency = BR_FSSCRUB_FREQ_DAILY;
-        } else if (strcasecmp (tmp, "weekly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_WEEKLY;
-        } else if (strcasecmp (tmp, "biweekly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_BIWEEKLY;
-        } else if (strcasecmp (tmp, "monthly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_MONTHLY;
-        } else if (strcasecmp (tmp, BR_SCRUB_STALLED) == 0) {
-                frequency = BR_FSSCRUB_FREQ_STALLED;
-        } else
-                goto error_return;
-
-        if (fsscrub->frequency == frequency)
-                fsscrub->frequency_reconf = _gf_false;
-        else
-                fsscrub->frequency = frequency;
-
-        return 0;
-
- error_return:
-        return -1;
+    int32_t ret = -1;
+    char *tmp = NULL;
+    scrub_freq_t frequency = BR_FSSCRUB_FREQ_HOURLY;
+    struct br_scrubber *fsscrub = NULL;
+
+    fsscrub = &priv->fsscrub;
+    fsscrub->frequency_reconf = _gf_true;
+
+    ret = br_scrubber_fetch_option(this, "scrub-freq", options, &tmp);
+    if (ret)
+        goto error_return;
+
+    if (scrubstall)
+        tmp = BR_SCRUB_STALLED;
+
+    if (strcasecmp(tmp, "hourly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_HOURLY;
+    } else if (strcasecmp(tmp, "daily") == 0) {
+        frequency = BR_FSSCRUB_FREQ_DAILY;
+    } else if (strcasecmp(tmp, "weekly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_WEEKLY;
+    } else if (strcasecmp(tmp, "biweekly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_BIWEEKLY;
+    } else if (strcasecmp(tmp, "monthly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_MONTHLY;
+    } else if (strcasecmp(tmp, "minute") == 0) {
+        frequency = BR_FSSCRUB_FREQ_MINUTE;
+    } else if (strcasecmp(tmp, BR_SCRUB_STALLED) == 0) {
+        frequency = BR_FSSCRUB_FREQ_STALLED;
+    } else
+        goto error_return;
+
+    if (fsscrub->frequency == frequency)
+        fsscrub->frequency_reconf = _gf_false;
+    else
+        fsscrub->frequency = frequency;
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
-static void br_scrubber_log_option (xlator_t *this,
-                                    br_private_t *priv, gf_boolean_t scrubstall)
+static void
+br_scrubber_log_option(xlator_t *this, br_private_t *priv,
+                       gf_boolean_t scrubstall)
 {
-        struct br_scrubber *fsscrub = &priv->fsscrub;
-        char *scrub_throttle_str[] = {
-                [BR_SCRUB_THROTTLE_LAZY]       = "lazy",
-                [BR_SCRUB_THROTTLE_NORMAL]     = "normal",
-                [BR_SCRUB_THROTTLE_AGGRESSIVE] = "aggressive",
-        };
-
-        char *scrub_freq_str[] = {
-                [BR_FSSCRUB_FREQ_HOURLY]   = "hourly",
-                [BR_FSSCRUB_FREQ_DAILY]    = "daily",
-                [BR_FSSCRUB_FREQ_WEEKLY]   = "weekly",
-                [BR_FSSCRUB_FREQ_BIWEEKLY] = "biweekly",
-                [BR_FSSCRUB_FREQ_MONTHLY]  = "monthly (30 days)",
-        };
-
-        if (scrubstall)
-                return; /* logged as pause */
-
-        if (fsscrub->frequency_reconf || fsscrub->throttle_reconf) {
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_TUNABLE,
-                        "SCRUB TUNABLES:: [Frequency: %s, Throttle: %s]",
-                        scrub_freq_str[fsscrub->frequency],
-                        scrub_throttle_str[fsscrub->throttle]);
-        }
+    struct br_scrubber *fsscrub = &priv->fsscrub;
+    char *scrub_throttle_str[] = {
+        [BR_SCRUB_THROTTLE_LAZY] = "lazy",
+        [BR_SCRUB_THROTTLE_NORMAL] = "normal",
+        [BR_SCRUB_THROTTLE_AGGRESSIVE] = "aggressive",
+        [BR_SCRUB_THROTTLE_STALLED] = "stalled",
+    };
+
+    char *scrub_freq_str[] = {
+        [0] = "",
+        [BR_FSSCRUB_FREQ_HOURLY] = "hourly",
+        [BR_FSSCRUB_FREQ_DAILY] = "daily",
+        [BR_FSSCRUB_FREQ_WEEKLY] = "weekly",
+        [BR_FSSCRUB_FREQ_BIWEEKLY] = "biweekly",
+        [BR_FSSCRUB_FREQ_MONTHLY] = "monthly (30 days)",
+        [BR_FSSCRUB_FREQ_MINUTE] = "every minute",
+    };
+
+    if (scrubstall)
+        return; /* logged as pause */
+
+    if (fsscrub->frequency_reconf || fsscrub->throttle_reconf) {
+        if (fsscrub->throttle == BR_SCRUB_THROTTLE_VOID)
+            return;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_TUNABLE,
+               "SCRUB TUNABLES:: [Frequency: %s, Throttle: %s]",
+               scrub_freq_str[fsscrub->frequency],
+               scrub_throttle_str[fsscrub->throttle]);
+    }
 }
 
 int32_t
-br_scrubber_handle_options (xlator_t *this, br_private_t *priv, dict_t *options)
+br_scrubber_handle_options(xlator_t *this, br_private_t *priv, dict_t *options)
 {
-        int32_t ret = 0;
-        gf_boolean_t scrubstall = _gf_false; /* not as dangerous as it sounds */
+    int32_t ret = 0;
+    gf_boolean_t scrubstall = _gf_false; /* not as dangerous as it sounds */
 
-        ret = br_scrubber_handle_stall (this, priv, options, &scrubstall);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_handle_stall(this, priv, options, &scrubstall);
+    if (ret)
+        goto error_return;
 
-        ret = br_scrubber_handle_throttle (this, priv, options, scrubstall);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_handle_throttle(this, priv, options, scrubstall);
+    if (ret)
+        goto error_return;
 
-        ret = br_scrubber_handle_freq (this, priv, options, scrubstall);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_handle_freq(this, priv, options, scrubstall);
+    if (ret)
+        goto error_return;
 
-        br_scrubber_log_option (this, priv, scrubstall);
+    br_scrubber_log_option(this, priv, scrubstall);
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 inode_t *
-br_lookup_bad_obj_dir (xlator_t *this, br_child_t *child, uuid_t gfid)
+br_lookup_bad_obj_dir(xlator_t *this, br_child_t *child, uuid_t gfid)
 {
-        struct  iatt statbuf   = {0, };
-        inode_table_t *table = NULL;
-        int32_t      ret     = -1;
-        loc_t        loc     = {0, };
-        inode_t     *linked_inode = NULL;
-        int32_t      op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-scrubber", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-
-        table = child->table;
-
-        loc.inode = inode_new (table);
-        if (!loc.inode) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        BRB_MSG_NO_MEMORY, "failed to allocate a new inode for"
-                        "bad object directory");
-                goto out;
-        }
+    struct iatt statbuf = {
+        0,
+    };
+    inode_table_t *table = NULL;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    int32_t op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+
+    table = child->table;
+
+    loc.inode = inode_new(table);
+    if (!loc.inode) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+               "failed to allocate a new inode for"
+               "bad object directory");
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, gfid);
+
+    ret = syncop_lookup(child->xl, &loc, &statbuf, NULL, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_LOOKUP_FAILED,
+               "failed to lookup the bad "
+               "objects directory (gfid: %s (%s))",
+               uuid_utoa(gfid), strerror(op_errno));
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, NULL, NULL, &statbuf);
+    if (linked_inode)
+        inode_lookup(linked_inode);
 
-        gf_uuid_copy (loc.gfid, gfid);
+out:
+    loc_wipe(&loc);
+    return linked_inode;
+}
 
-        ret = syncop_lookup (child->xl, &loc, &statbuf, NULL, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_LOOKUP_FAILED, "failed to lookup the bad "
-                        "objects directory (gfid: %s (%s))", uuid_utoa (gfid),
-                        strerror (op_errno));
-                goto out;
+int32_t
+br_read_bad_object_dir(xlator_t *this, br_child_t *child, fd_t *fd,
+                       dict_t *dict)
+{
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    int32_t ret = -1;
+    off_t offset = 0;
+    int32_t count = 0;
+    char key[32] = {
+        0,
+    };
+    dict_t *out_dict = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    while ((ret = syncop_readdir(child->xl, fd, 131072, offset, &entries, NULL,
+                                 &out_dict))) {
+        if (ret < 0)
+            goto out;
+
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
+
+            snprintf(key, sizeof(key), "quarantine-%d", count);
+
+            /*
+             * ignore the dict_set errors for now. The intention is
+             * to get as many bad objects as possible instead of
+             * erroring out at the first failure.
+             */
+            ret = dict_set_dynstr_with_alloc(dict, key, entry->d_name);
+            if (!ret)
+                count++;
+
+            if (out_dict) {
+                dict_copy(out_dict, dict);
+                dict_unref(out_dict);
+                out_dict = NULL;
+            }
         }
 
-        linked_inode = inode_link (loc.inode, NULL, NULL, &statbuf);
-        if (linked_inode)
-                inode_lookup (linked_inode);
+        gf_dirent_free(&entries);
+    }
+
+    ret = count;
+    ret = dict_set_int32_sizen(dict, "count", count);
 
 out:
-        loc_wipe (&loc);
-        return linked_inode;
+    return ret;
 }
 
 int32_t
-br_read_bad_object_dir (xlator_t *this, br_child_t *child, fd_t *fd,
-                        dict_t *dict)
+br_get_bad_objects_from_child(xlator_t *this, dict_t *dict, br_child_t *child)
 {
-        gf_dirent_t  entries;
-        gf_dirent_t *entry   = NULL;
-        int32_t      ret     = -1;
-        off_t        offset  = 0;
-        int32_t      count   = 0;
-        char         key[PATH_MAX] = {0, };
-
-        INIT_LIST_HEAD (&entries.list);
-
-        while ((ret = syncop_readdir (child->xl, fd, 131072, offset, &entries,
-                                      NULL, NULL))) {
-                if (ret < 0)
-                        goto out;
-		if (ret == 0)
-			break;
-		list_for_each_entry (entry, &entries.list, list) {
-			offset = entry->d_off;
-
-                         snprintf (key, sizeof (key), "quarantine-%d", count);
-
-                        /*
-                         * ignore the dict_set errors for now. The intention is
-                         * to get as many bad objects as possible instead of
-                         * erroring out at the first failure.
-                         */
-                         ret = dict_set_dynstr_with_alloc (dict, key,
-                                                           entry->d_name);
-                        if (!ret)
-                                count++;
-		}
-
-		gf_dirent_free (&entries);
-	}
-
-        ret = count;
-        ret = dict_set_int32 (dict, "count", count);
+    inode_t *inode = NULL;
+    inode_table_t *table = NULL;
+    fd_t *fd = NULL;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int32_t op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    table = child->table;
+
+    inode = inode_find(table, BR_BAD_OBJ_CONTAINER);
+    if (!inode) {
+        inode = br_lookup_bad_obj_dir(this, child, BR_BAD_OBJ_CONTAINER);
+        if (!inode)
+            goto out;
+    }
+
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_FD_CREATE_FAILED,
+               "fd creation for the bad "
+               "objects directory failed (gfid: %s)",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    loc.inode = inode;
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_opendir(child->xl, &loc, fd, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        fd_unref(fd);
+        fd = NULL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_FD_CREATE_FAILED,
+               "failed to open the bad "
+               "objects directory %s",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    fd_bind(fd);
+
+    ret = br_read_bad_object_dir(this, child, fd, dict);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_BAD_OBJ_READDIR_FAIL,
+               "readdir of the bad "
+               "objects directory (%s) failed ",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    loc_wipe(&loc);
+    if (fd)
+        fd_unref(fd);
+    return ret;
 }
 
 int32_t
-br_get_bad_objects_from_child (xlator_t *this, dict_t *dict, br_child_t *child)
+br_collect_bad_objects_of_child(xlator_t *this, br_child_t *child, dict_t *dict,
+                                dict_t *child_dict, int32_t total_count)
 {
-        inode_t     *inode   = NULL;
-        inode_table_t *table = NULL;
-        fd_t        *fd      = NULL;
-        int32_t      ret     = -1;
-        loc_t        loc     = {0, };
-        int32_t      op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-scrubber", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        table = child->table;
-
-        inode = inode_find (table, BR_BAD_OBJ_CONTAINER);
-        if (!inode) {
-                inode = br_lookup_bad_obj_dir (this, child,
-                                               BR_BAD_OBJ_CONTAINER);
-                if (!inode)
-                        goto out;
-        }
-
-        fd = fd_create (inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        BRB_MSG_FD_CREATE_FAILED, "fd creation for the bad "
-                        "objects directory failed (gfid: %s)",
-                        uuid_utoa (BR_BAD_OBJ_CONTAINER));
-                goto out;
-        }
-
-        loc.inode = inode;
-        gf_uuid_copy (loc.gfid, inode->gfid);
+    int32_t ret = -1;
+    int32_t count = 0;
+    char key[32] = {
+        0,
+    };
+    char main_key[32] = {
+        0,
+    };
+    int32_t j = 0;
+    int32_t tmp_count = 0;
+    char *entry = NULL;
+    char tmp[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    int32_t len = 0;
+
+    ret = dict_get_int32_sizen(child_dict, "count", &count);
+    if (ret)
+        goto out;
+
+    tmp_count = total_count;
+
+    for (j = 0; j < count; j++) {
+        len = snprintf(key, sizeof(key), "quarantine-%d", j);
+        ret = dict_get_strn(child_dict, key, len, &entry);
+        if (ret)
+            continue;
 
-        ret = syncop_opendir (child->xl, &loc, fd, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                fd_unref (fd);
-                fd = NULL;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        BRB_MSG_FD_CREATE_FAILED, "failed to open the bad "
-                        "objects directory %s",
-                        uuid_utoa (BR_BAD_OBJ_CONTAINER));
-                goto out;
+        ret = dict_get_str(child_dict, entry, &path);
+        len = snprintf(tmp, PATH_MAX, "%s ==> BRICK: %s\n path: %s", entry,
+                       child->brick_path, path);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            continue;
         }
+        snprintf(main_key, sizeof(main_key), "quarantine-%d", tmp_count);
 
-        fd_bind (fd);
+        ret = dict_set_dynstr_with_alloc(dict, main_key, tmp);
+        if (!ret)
+            tmp_count++;
+        path = NULL;
+    }
 
-        ret = br_read_bad_object_dir (this, child, fd, dict);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_BAD_OBJ_READDIR_FAIL, "readdir of the bad "
-                        "objects directory (%s) failed ",
-                        uuid_utoa (BR_BAD_OBJ_CONTAINER));
-                goto out;
-        }
-
-        ret = 0;
+    ret = tmp_count;
 
 out:
-        loc_wipe (&loc);
-        if (fd)
-                fd_unref (fd);
-        return ret;
+    return ret;
 }
 
 int32_t
-br_collect_bad_objects_of_child (xlator_t *this, br_child_t *child,
-                                 dict_t *dict, dict_t *child_dict,
-                                 int32_t total_count)
+br_collect_bad_objects_from_children(xlator_t *this, dict_t *dict)
 {
-
-        int32_t    ret = -1;
-        int32_t    count = 0;
-        char       key[PATH_MAX] = {0, };
-        char       main_key[PATH_MAX] = {0, };
-        int32_t     j = 0;
-        int32_t    tmp_count = 0;
-        char       *entry = NULL;
-
-        ret = dict_get_int32 (child_dict, "count", &count);
-        if (ret)
-                goto out;
-
-        tmp_count = total_count;
-
-        for (j = 0; j < count; j++) {
-                snprintf (key, PATH_MAX, "quarantine-%d", j);
-                ret = dict_get_str (child_dict, key, &entry);
-                if (ret)
-                        continue;
-                snprintf (main_key, PATH_MAX, "quarantine-%d",
-                          tmp_count);
-                ret = dict_set_dynstr_with_alloc (dict, main_key, entry);
-                if (!ret)
-                        tmp_count++;
+    int32_t ret = -1;
+    dict_t *child_dict = NULL;
+    int32_t i = 0;
+    int32_t total_count = 0;
+    br_child_t *child = NULL;
+    br_private_t *priv = NULL;
+    dict_t *tmp_dict = NULL;
+
+    priv = this->private;
+    tmp_dict = dict;
+
+    for (i = 0; i < priv->child_count; i++) {
+        child = &priv->children[i];
+        GF_ASSERT(child);
+        if (!_br_is_child_connected(child))
+            continue;
+
+        child_dict = dict_new();
+        if (!child_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            continue;
+        }
+        ret = br_get_bad_objects_from_child(this, child_dict, child);
+        /*
+         * Continue asking the remaining children for the list of
+         * bad objects even though getting the list from one of them
+         * fails.
+         */
+        if (ret) {
+            dict_unref(child_dict);
+            continue;
         }
 
-        ret = tmp_count;
-
-out:
-        return ret;
-}
-
-int32_t
-br_collect_bad_objects_from_children (xlator_t *this, dict_t *dict)
-{
-        int32_t    ret = -1;
-        dict_t    *child_dict = NULL;
-        int32_t    i = 0;
-        int32_t    total_count = 0;
-        br_child_t *child = NULL;
-        br_private_t *priv = NULL;
-        dict_t     *tmp_dict = NULL;
-
-        priv = this->private;
-        tmp_dict = dict;
-
-        for (i = 0; i < priv->child_count; i++) {
-                child = &priv->children[i];
-                GF_ASSERT (child);
-                if (!_br_is_child_connected (child))
-                        continue;
-
-                child_dict = dict_new ();
-                if (!child_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                BRB_MSG_NO_MEMORY, "failed to allocate dict");
-                        continue;
-                }
-                ret = br_get_bad_objects_from_child (this, child_dict, child);
-                /*
-                 * Continue asking the remaining children for the list of
-                 * bad objects even though getting the list from one of them
-                 * fails.
-                 */
-                if (ret) {
-                        dict_unref (child_dict);
-                        continue;
-                }
-
-                ret = br_collect_bad_objects_of_child (this, child, tmp_dict,
-                                                       child_dict, total_count);
-                if (ret < 0) {
-                        dict_unref (child_dict);
-                        continue;
-                }
-
-                total_count = ret;
-                dict_unref (child_dict);
-                child_dict = NULL;
+        ret = br_collect_bad_objects_of_child(this, child, tmp_dict, child_dict,
+                                              total_count);
+        if (ret < 0) {
+            dict_unref(child_dict);
+            continue;
         }
 
-        ret = dict_set_int32 (tmp_dict, "total-count", total_count);
+        total_count = ret;
+        dict_unref(child_dict);
+        child_dict = NULL;
+    }
+
+    ret = dict_set_int32(tmp_dict, "total-count", total_count);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_get_bad_objects_list (xlator_t *this, dict_t **dict)
+br_get_bad_objects_list(xlator_t *this, dict_t **dict)
 {
-        int32_t      ret     = -1;
-        dict_t      *tmp_dict = NULL;
+    int32_t ret = -1;
+    dict_t *tmp_dict = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bir-rot-scrubber", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+    GF_VALIDATE_OR_GOTO("bir-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
 
-        tmp_dict = *dict;
+    tmp_dict = *dict;
+    if (!tmp_dict) {
+        tmp_dict = dict_new();
         if (!tmp_dict) {
-                tmp_dict = dict_new ();
-                if (!tmp_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                BRB_MSG_NO_MEMORY, "failed to allocate dict");
-                        goto out;
-                }
-                *dict = tmp_dict;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            goto out;
         }
+        *dict = tmp_dict;
+    }
 
-        ret = br_collect_bad_objects_from_children (this, tmp_dict);
+    ret = br_collect_bad_objects_from_children(this, tmp_dict);
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-wait_for_scrub_to_finish (xlator_t *this)
+wait_for_scrub_to_finish(xlator_t *this)
 {
-        int                  ret               = -1;
-        br_private_t         *priv             = NULL;
-        struct br_monitor    *scrub_monitor    = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", scrub_monitor, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                "Waiting for all children to start and finish scrub");
-
-        pthread_mutex_lock (&scrub_monitor->donelock);
-        {
-                while (!scrub_monitor->done)
-                        pthread_cond_wait (&scrub_monitor->donecond,
-                                           &scrub_monitor->donelock);
-        }
-        pthread_mutex_unlock (&scrub_monitor->donelock);
-        ret = 0;
+    int ret = -1;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", scrub_monitor, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Waiting for all children to start and finish scrub");
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        while (!scrub_monitor->done)
+            pthread_cond_wait(&scrub_monitor->donecond,
+                              &scrub_monitor->donelock);
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1802,152 +1921,150 @@ out:
  * thread that takes care of state machine.
  */
 void *
-br_monitor_thread (void *arg)
+br_monitor_thread(void *arg)
 {
-        int32_t              ret               = 0;
-        xlator_t            *this              = NULL;
-        br_private_t        *priv              = NULL;
-        struct br_monitor   *scrub_monitor     = NULL;
-
-        this = arg;
-        priv = this->private;
-
-        /*
-         * Since, this is the topmost xlator, THIS has to be set by bit-rot
-         * xlator itself (STACK_WIND wont help in this case). Also it has
-         * to be done for each thread that gets spawned. Otherwise, a new
-         * thread will get global_xlator's pointer when it does "THIS".
-         */
-        THIS = this;
-
-        scrub_monitor = &priv->scrub_monitor;
-
-        pthread_mutex_lock (&scrub_monitor->mutex);
-        {
-                while (!scrub_monitor->inited)
-                        pthread_cond_wait (&scrub_monitor->cond,
-                                           &scrub_monitor->mutex);
-        }
-        pthread_mutex_unlock (&scrub_monitor->mutex);
-
-        /* this needs to be serialized with reconfigure() */
-        pthread_mutex_lock (&priv->lock);
-        {
-                ret = br_scrub_state_machine (this);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    this = arg;
+    priv = this->private;
+
+    /*
+     * Since, this is the topmost xlator, THIS has to be set by bit-rot
+     * xlator itself (STACK_WIND won't help in this case). Also it has
+     * to be done for each thread that gets spawned. Otherwise, a new
+     * thread will get global_xlator's pointer when it does "THIS".
+     */
+    THIS = this;
+
+    scrub_monitor = &priv->scrub_monitor;
+
+    pthread_mutex_lock(&scrub_monitor->mutex);
+    {
+        while (!scrub_monitor->inited)
+            pthread_cond_wait(&scrub_monitor->cond, &scrub_monitor->mutex);
+    }
+    pthread_mutex_unlock(&scrub_monitor->mutex);
+
+    /* this needs to be serialized with reconfigure() */
+    pthread_mutex_lock(&priv->lock);
+    {
+        ret = br_scrub_state_machine(this, _gf_false);
+    }
+    pthread_mutex_unlock(&priv->lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SSM_FAILED,
+               "Scrub state machine failed");
+        goto out;
+    }
+
+    while (1) {
+        /* Wait for all children to finish scrubbing */
+        ret = wait_for_scrub_to_finish(this);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        BRB_MSG_SSM_FAILED,
-                        "Scrub state machine failed");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SCRUB_WAIT_FAILED,
+                   "Scrub wait failed");
+            goto out;
         }
 
-        while (1) {
-                /* Wait for all children to finish scrubbing */
-                ret = wait_for_scrub_to_finish (this);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                BRB_MSG_SCRUB_WAIT_FAILED,
-                                "Scrub wait failed");
-                        goto out;
-                }
-
-                /* scrub exit criteria: Move the state to PENDING */
-                br_scrubber_exit_control (this);
-        }
+        /* scrub exit criteria: Move the state to PENDING */
+        br_scrubber_exit_control(this);
+    }
 
 out:
-        return NULL;
+    return NULL;
 }
 
 static void
-br_set_scrub_state (struct br_monitor *scrub_monitor, br_scrub_state_t state)
+br_set_scrub_state(struct br_monitor *scrub_monitor, br_scrub_state_t state)
 {
-        LOCK (&scrub_monitor->lock);
-        {
-                _br_monitor_set_scrub_state (scrub_monitor, state);
-        }
-        UNLOCK (&scrub_monitor->lock);
+    LOCK(&scrub_monitor->lock);
+    {
+        _br_monitor_set_scrub_state(scrub_monitor, state);
+    }
+    UNLOCK(&scrub_monitor->lock);
 }
 
 int32_t
-br_scrubber_monitor_init (xlator_t *this, br_private_t *priv)
+br_scrubber_monitor_init(xlator_t *this, br_private_t *priv)
 {
-        struct br_monitor *scrub_monitor = NULL;
-        int                ret           = 0;
+    struct br_monitor *scrub_monitor = NULL;
+    int ret = 0;
 
-        scrub_monitor = &priv->scrub_monitor;
+    scrub_monitor = &priv->scrub_monitor;
 
-        LOCK_INIT (&scrub_monitor->lock);
-        scrub_monitor->this = this;
+    LOCK_INIT(&scrub_monitor->lock);
+    scrub_monitor->this = this;
 
-        scrub_monitor->inited = _gf_false;
-        pthread_mutex_init (&scrub_monitor->mutex, NULL);
-        pthread_cond_init (&scrub_monitor->cond, NULL);
+    scrub_monitor->inited = _gf_false;
+    pthread_mutex_init(&scrub_monitor->mutex, NULL);
+    pthread_cond_init(&scrub_monitor->cond, NULL);
 
-        scrub_monitor->kick = _gf_false;
-        scrub_monitor->active_child_count = 0;
-        pthread_mutex_init (&scrub_monitor->wakelock, NULL);
-        pthread_cond_init (&scrub_monitor->wakecond, NULL);
+    scrub_monitor->kick = _gf_false;
+    scrub_monitor->active_child_count = 0;
+    pthread_mutex_init(&scrub_monitor->wakelock, NULL);
+    pthread_cond_init(&scrub_monitor->wakecond, NULL);
 
-        scrub_monitor->done = _gf_false;
-        pthread_mutex_init (&scrub_monitor->donelock, NULL);
-        pthread_cond_init (&scrub_monitor->donecond, NULL);
-
-        /* Set the state to INACTIVE */
-        br_set_scrub_state (&priv->scrub_monitor, BR_SCRUB_STATE_INACTIVE);
-
-        /* Start the monitor thread */
-        ret = gf_thread_create (&scrub_monitor->thread, NULL, br_monitor_thread, this);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        BRB_MSG_SPAWN_FAILED, "monitor thread creation failed");
-                ret = -1;
-                goto err;
-        }
+    scrub_monitor->done = _gf_false;
+    pthread_mutex_init(&scrub_monitor->donelock, NULL);
+    pthread_cond_init(&scrub_monitor->donecond, NULL);
 
-        return 0;
+    /* Set the state to INACTIVE */
+    br_set_scrub_state(&priv->scrub_monitor, BR_SCRUB_STATE_INACTIVE);
+
+    /* Start the monitor thread */
+    ret = gf_thread_create(&scrub_monitor->thread, NULL, br_monitor_thread,
+                           this, "brmon");
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SPAWN_FAILED,
+               "monitor thread creation failed");
+        ret = -1;
+        goto err;
+    }
+
+    return 0;
 err:
-        pthread_mutex_destroy (&scrub_monitor->mutex);
-        pthread_cond_destroy (&scrub_monitor->cond);
+    pthread_mutex_destroy(&scrub_monitor->mutex);
+    pthread_cond_destroy(&scrub_monitor->cond);
 
-        pthread_mutex_destroy (&scrub_monitor->wakelock);
-        pthread_cond_destroy (&scrub_monitor->wakecond);
+    pthread_mutex_destroy(&scrub_monitor->wakelock);
+    pthread_cond_destroy(&scrub_monitor->wakecond);
 
-        pthread_mutex_destroy (&scrub_monitor->donelock);
-        pthread_cond_destroy (&scrub_monitor->donecond);
+    pthread_mutex_destroy(&scrub_monitor->donelock);
+    pthread_cond_destroy(&scrub_monitor->donecond);
 
-        LOCK_DESTROY (&scrub_monitor->lock);
+    LOCK_DESTROY(&scrub_monitor->lock);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_scrubber_init (xlator_t *this, br_private_t *priv)
+br_scrubber_init(xlator_t *this, br_private_t *priv)
 {
-        struct br_scrubber *fsscrub = NULL;
-        int                 ret     = 0;
+    struct br_scrubber *fsscrub = NULL;
+    int ret = 0;
 
-        priv->tbf = br_tbf_init (NULL, 0);
-        if (!priv->tbf)
-                return -1;
+    priv->tbf = tbf_init(NULL, 0);
+    if (!priv->tbf)
+        return -1;
 
-        ret = br_scrubber_monitor_init (this, priv);
-        if (ret)
-                return -1;
+    ret = br_scrubber_monitor_init(this, priv);
+    if (ret)
+        return -1;
 
-        fsscrub = &priv->fsscrub;
+    fsscrub = &priv->fsscrub;
 
-        fsscrub->this = this;
-        fsscrub->throttle = BR_SCRUB_THROTTLE_VOID;
+    fsscrub->this = this;
+    fsscrub->throttle = BR_SCRUB_THROTTLE_VOID;
 
-        pthread_mutex_init (&fsscrub->mutex, NULL);
-        pthread_cond_init (&fsscrub->cond, NULL);
+    pthread_mutex_init(&fsscrub->mutex, NULL);
+    pthread_cond_init(&fsscrub->cond, NULL);
 
-        fsscrub->nr_scrubbers = 0;
-        INIT_LIST_HEAD (&fsscrub->scrubbers);
-        INIT_LIST_HEAD (&fsscrub->scrublist);
+    fsscrub->nr_scrubbers = 0;
+    INIT_LIST_HEAD(&fsscrub->scrubbers);
+    INIT_LIST_HEAD(&fsscrub->scrublist);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
index 63169068ed4..4e5f67bc021 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
@@ -11,26 +11,36 @@
 #ifndef __BIT_ROT_SCRUB_H__
 #define __BIT_ROT_SCRUB_H__
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "bit-rot.h"
 
-void *br_fsscanner (void *);
+void *
+br_fsscanner(void *);
 
-int32_t br_fsscan_schedule (xlator_t *);
-int32_t br_fsscan_reschedule (xlator_t *);
-int32_t br_fsscan_activate (xlator_t *);
-int32_t br_fsscan_deactivate (xlator_t *);
+int32_t
+br_fsscan_schedule(xlator_t *);
+int32_t
+br_fsscan_reschedule(xlator_t *);
+int32_t
+br_fsscan_activate(xlator_t *);
+int32_t
+br_fsscan_deactivate(xlator_t *);
+int32_t
+br_fsscan_ondemand(xlator_t *);
 
-int32_t br_scrubber_handle_options (xlator_t *, br_private_t *, dict_t *);
+int32_t
+br_scrubber_handle_options(xlator_t *, br_private_t *, dict_t *);
 
 int32_t
-br_scrubber_monitor_init (xlator_t *, br_private_t *);
+br_scrubber_monitor_init(xlator_t *, br_private_t *);
 
-int32_t br_scrubber_init (xlator_t *, br_private_t *);
+int32_t
+br_scrubber_init(xlator_t *, br_private_t *);
 
-int32_t br_collect_bad_objects_from_children (xlator_t *this, dict_t *dict);
+int32_t
+br_collect_bad_objects_from_children(xlator_t *this, dict_t *dict);
 
 void
-br_child_set_scrub_state (br_child_t *, gf_boolean_t);
+br_child_set_scrub_state(br_child_t *, gf_boolean_t);
 
 #endif /* __BIT_ROT_SCRUB_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
index d304fc804ee..753e31a3b23 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
@@ -12,103 +12,113 @@
 #include "bit-rot-scrub.h"
 #include "bit-rot-bitd-messages.h"
 
-int br_scrub_ssm_noop (xlator_t *this)
+int
+br_scrub_ssm_noop(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
 int
-br_scrub_ssm_state_pause (xlator_t *this)
+br_scrub_ssm_state_pause(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Scrubber paused");
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PAUSED);
-        return 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Scrubber paused");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PAUSED);
+    return 0;
 }
 
 int
-br_scrub_ssm_state_ipause (xlator_t *this)
+br_scrub_ssm_state_ipause(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Scrubber paused");
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_IPAUSED);
-        return 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Scrubber paused");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_IPAUSED);
+    return 0;
 }
 
 int
-br_scrub_ssm_state_active (xlator_t *this)
+br_scrub_ssm_state_active(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        if (scrub_monitor->done) {
-                (void) br_fsscan_activate (this);
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                        "Scrubbing resumed");
-                _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_ACTIVE);
-        }
+    if (scrub_monitor->done) {
+        (void)br_fsscan_activate(this);
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+               "Scrubbing resumed");
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_ACTIVE);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-br_scrub_ssm_state_stall (xlator_t *this)
+br_scrub_ssm_state_stall(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Volume is under active scrubbing. Pausing scrub..");
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_STALLED);
-        return 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Volume is under active scrubbing. Pausing scrub..");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_STALLED);
+    return 0;
 }
 
-static br_scrub_ssm_call *
-br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] = {
-        {br_fsscan_schedule, br_scrub_ssm_state_ipause},    /* INACTIVE */
-        {br_fsscan_reschedule, br_fsscan_deactivate},       /* PENDING  */
-        {br_scrub_ssm_noop, br_scrub_ssm_state_stall},      /* ACTIVE   */
-        {br_fsscan_activate, br_scrub_ssm_noop},            /* PAUSED   */
-        {br_fsscan_schedule, br_scrub_ssm_noop},            /* IPAUSED  */
-        {br_scrub_ssm_state_active, br_scrub_ssm_noop},     /* STALLED  */
+static br_scrub_ssm_call *br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] =
+    {
+        /* INACTIVE */
+        {br_fsscan_schedule, br_scrub_ssm_state_ipause, br_scrub_ssm_noop},
+        /* PENDING  */
+        {br_fsscan_reschedule, br_fsscan_deactivate, br_fsscan_ondemand},
+        /* ACTIVE   */
+        {br_scrub_ssm_noop, br_scrub_ssm_state_stall, br_scrub_ssm_noop},
+        /* PAUSED   */
+        {br_fsscan_activate, br_scrub_ssm_noop, br_scrub_ssm_noop},
+        /* IPAUSED  */
+        {br_fsscan_schedule, br_scrub_ssm_noop, br_scrub_ssm_noop},
+        /* STALLED  */
+        {br_scrub_ssm_state_active, br_scrub_ssm_noop, br_scrub_ssm_noop},
 };
 
 int32_t
-br_scrub_state_machine (xlator_t *this)
+br_scrub_state_machine(xlator_t *this, gf_boolean_t scrub_ondemand)
 {
-        br_private_t       *priv      = NULL;
-        br_scrub_ssm_call  *call      = NULL;
-        struct br_scrubber *fsscrub   = NULL;
-        br_scrub_state_t    currstate = 0;
-        br_scrub_event_t    event     = 0;
-        struct br_monitor  *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        currstate = scrub_monitor->state;
-        event = _br_child_get_scrub_event (fsscrub);
-
-        call = br_scrub_ssm[currstate][event];
-        return call (this);
+    br_private_t *priv = NULL;
+    br_scrub_ssm_call *call = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    br_scrub_state_t currstate = 0;
+    br_scrub_event_t event = 0;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    currstate = scrub_monitor->state;
+    if (scrub_ondemand)
+        event = BR_SCRUB_EVENT_ONDEMAND;
+    else
+        event = _br_child_get_scrub_event(fsscrub);
+
+    call = br_scrub_ssm[currstate][event];
+    return call(this);
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
index 936ee4d837c..37b45a42eac 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
@@ -11,26 +11,28 @@
 #ifndef __BIT_ROT_SSM_H__
 #define __BIT_ROT_SSM_H__
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 typedef enum br_scrub_state {
-        BR_SCRUB_STATE_INACTIVE = 0,
-        BR_SCRUB_STATE_PENDING,
-        BR_SCRUB_STATE_ACTIVE,
-        BR_SCRUB_STATE_PAUSED,
-        BR_SCRUB_STATE_IPAUSED,
-        BR_SCRUB_STATE_STALLED,
-        BR_SCRUB_MAXSTATES,
+    BR_SCRUB_STATE_INACTIVE = 0,
+    BR_SCRUB_STATE_PENDING,
+    BR_SCRUB_STATE_ACTIVE,
+    BR_SCRUB_STATE_PAUSED,
+    BR_SCRUB_STATE_IPAUSED,
+    BR_SCRUB_STATE_STALLED,
+    BR_SCRUB_MAXSTATES,
 } br_scrub_state_t;
 
 typedef enum br_scrub_event {
-        BR_SCRUB_EVENT_SCHEDULE = 0,
-        BR_SCRUB_EVENT_PAUSE,
-        BR_SCRUB_MAXEVENTS,
+    BR_SCRUB_EVENT_SCHEDULE = 0,
+    BR_SCRUB_EVENT_PAUSE,
+    BR_SCRUB_EVENT_ONDEMAND,
+    BR_SCRUB_MAXEVENTS,
 } br_scrub_event_t;
 
 struct br_monitor;
 
-int32_t br_scrub_state_machine (xlator_t *);
+int32_t
+br_scrub_state_machine(xlator_t *, gf_boolean_t);
 
 #endif /* __BIT_ROT_SSM_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c b/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c
deleted file mode 100644
index f8b9b75d575..00000000000
--- a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-/**
- *
- * Basic token bucket implementation for rate limiting. As of now interfaces
- * to throttle disk read request, directory entry scan and hash calculation
- * are available. To throttle a particular request (operation), the call needs
- * to be wrapped in-between throttling APIs, for e.g.
- *
- *  TBF_THROTTLE_BEGIN (...);  <-- induces "delays" if required
- *  {
- *      call (...);
- *  }
- *  TBF_THROTTLE_END (...);  <-- not used atm, maybe needed later
- *
- */
-
-#include "mem-pool.h"
-#include "bit-rot-tbf.h"
-#include "bit-rot-stub-mem-types.h"
-
-typedef struct br_tbf_throttle {
-        char done;
-
-        pthread_mutex_t mutex;
-        pthread_cond_t  cond;
-
-        unsigned long tokens;
-
-        struct list_head list;
-} br_tbf_throttle_t;
-
-/**
- * OK. Most implementations of TBF I've come across generate tokens
- * every second (UML, etc..) and some chose sub-second granularity
- * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
- * any logic for choosing generation interval and it seems pretty
- * logical as one could jack up token count per interval w.r.t.
- * generation rate.
- *
- * Value used here is chosen based on a series of test(s) performed
- * to balance object signing time and not maxing out on all available
- * CPU cores. It's obvious to have seconds granularity and jack up
- * token count per interval, thereby achieving close to similar
- * results. Let's stick to this as it seems to be working fine for
- * the set of ops that are throttled.
- */
-#define BR_TBF_TOKENGEN_INTERVAL_USEC  600000
-
-static br_tbf_throttle_t *
-br_tbf_init_throttle (unsigned long tokens_required)
-{
-        br_tbf_throttle_t *throttle = NULL;
-
-        throttle = GF_CALLOC (1, sizeof (*throttle),
-                              gf_br_mt_br_tbf_throttle_t);
-        if (!throttle)
-                return NULL;
-
-        throttle->done = 0;
-        throttle->tokens = tokens_required;
-        INIT_LIST_HEAD (&throttle->list);
-
-        (void) pthread_mutex_init (&throttle->mutex, NULL);
-        (void) pthread_cond_init (&throttle->cond, NULL);
-
-        return throttle;
-}
-
-void
-_br_tbf_dispatch_queued (br_tbf_bucket_t *bucket)
-{
-        gf_boolean_t xcont = _gf_false;
-        br_tbf_throttle_t *tmp = NULL;
-        br_tbf_throttle_t *throttle = NULL;
-
-        list_for_each_entry_safe (throttle, tmp, &bucket->queued, list) {
-
-                pthread_mutex_lock (&throttle->mutex);
-                {
-                        if (bucket->tokens < throttle->tokens) {
-                                xcont = _gf_true;
-                                goto unblock;
-                        }
-
-                        /* this request can now be serviced */
-                        throttle->done = 1;
-                        list_del_init (&throttle->list);
-
-                        bucket->tokens -= throttle->tokens;
-                        pthread_cond_signal (&throttle->cond);
-                }
-        unblock:
-                pthread_mutex_unlock (&throttle->mutex);
-                if (xcont)
-                        break;
-        }
-}
-
-void *br_tbf_tokengenerator (void *arg)
-{
-        unsigned long tokenrate = 0;
-        unsigned long maxtokens = 0;
-        br_tbf_bucket_t *bucket = arg;
-
-        tokenrate = bucket->tokenrate;
-        maxtokens = bucket->maxtokens;
-
-        while (1) {
-                usleep (BR_TBF_TOKENGEN_INTERVAL_USEC);
-
-                LOCK (&bucket->lock);
-                {
-                        bucket->tokens += tokenrate;
-                        if (bucket->tokens > maxtokens)
-                                bucket->tokens = maxtokens;
-
-                        if (!list_empty (&bucket->queued))
-                                _br_tbf_dispatch_queued (bucket);
-                }
-                UNLOCK (&bucket->lock);
-        }
-
-        return NULL;
-}
-
-/**
- * There is lazy synchronization between this routine (when invoked
- * under br_tbf_mod() context) and br_tbf_throttle(). *bucket is
- * updated _after_ all the required variables are initialized.
- */
-static int32_t
-br_tbf_init_bucket (br_tbf_t *tbf, br_tbf_opspec_t *spec)
-{
-        int ret = 0;
-        br_tbf_bucket_t *curr = NULL;
-        br_tbf_bucket_t **bucket = NULL;
-
-        GF_ASSERT (spec->op >= BR_TBF_OP_MIN);
-        GF_ASSERT (spec->op <= BR_TBF_OP_MAX);
-
-        /* no rate? no throttling. */
-        if (!spec->rate)
-                return 0;
-
-        bucket = tbf->bucket + spec->op;
-
-        curr = GF_CALLOC (1, sizeof (*curr), gf_br_mt_br_tbf_bucket_t);
-        if (!curr)
-                goto error_return;
-
-        LOCK_INIT (&curr->lock);
-        INIT_LIST_HEAD (&curr->queued);
-
-        curr->tokens = 0;
-        curr->tokenrate = spec->rate;
-        curr->maxtokens = spec->maxlimit;
-
-        ret = gf_thread_create (&curr->tokener,
-                                NULL, br_tbf_tokengenerator, curr);
-        if (ret != 0)
-                goto freemem;
-
-        *bucket = curr;
-        return 0;
-
- freemem:
-        LOCK_DESTROY (&curr->lock);
-        GF_FREE (curr);
- error_return:
-        return -1;
-}
-
-#define BR_TBF_ALLOC_SIZE                                               \
-        (sizeof (br_tbf_t) + (BR_TBF_OP_MAX * sizeof (br_tbf_bucket_t)))
-
-br_tbf_t *
-br_tbf_init (br_tbf_opspec_t *tbfspec, unsigned int count)
-{
-        int32_t i = 0;
-        int32_t ret = 0;
-        br_tbf_t *tbf = NULL;
-        br_tbf_opspec_t *opspec = NULL;
-
-        tbf = GF_CALLOC (1, BR_TBF_ALLOC_SIZE, gf_br_mt_br_tbf_t);
-        if (!tbf)
-                goto error_return;
-
-        tbf->bucket = (br_tbf_bucket_t **) ((char *)tbf + sizeof (*tbf));
-        for (i = 0; i < BR_TBF_OP_MAX; i++) {
-                *(tbf->bucket + i) = NULL;
-        }
-
-        for (i = 0; i < count; i++) {
-                opspec = tbfspec + i;
-
-                ret = br_tbf_init_bucket (tbf, opspec);
-                if (ret)
-                        break;
-        }
-
-        if (ret)
-                goto error_return;
-
-        return tbf;
-
- error_return:
-        return NULL;
-}
-
-static void
-br_tbf_mod_bucket (br_tbf_bucket_t *bucket, br_tbf_opspec_t *spec)
-{
-        LOCK (&bucket->lock);
-        {
-                bucket->tokens = 0;
-                bucket->tokenrate = spec->rate;
-                bucket->maxtokens = spec->maxlimit;
-        }
-        UNLOCK (&bucket->lock);
-
-        /* next token tick would unqueue pending operations */
-}
-
-int
-br_tbf_mod (br_tbf_t *tbf, br_tbf_opspec_t *tbfspec)
-{
-        int              ret    = 0;
-        br_tbf_bucket_t *bucket = NULL;
-        br_tbf_ops_t     op     = BR_TBF_OP_MIN;
-
-        if (!tbf || !tbfspec)
-                return -1;
-
-        op = tbfspec->op;
-
-        GF_ASSERT (op >= BR_TBF_OP_MIN);
-        GF_ASSERT (op <= BR_TBF_OP_MAX);
-
-        bucket = *(tbf->bucket + op);
-        if (bucket) {
-                br_tbf_mod_bucket (bucket, tbfspec);
-        } else {
-                ret = br_tbf_init_bucket (tbf, tbfspec);
-        }
-
-        return ret;
-}
-
-void
-br_tbf_throttle (br_tbf_t *tbf, br_tbf_ops_t op, unsigned long tokens_requested)
-{
-        char waitq = 0;
-        br_tbf_bucket_t *bucket = NULL;
-        br_tbf_throttle_t *throttle = NULL;
-
-        GF_ASSERT (op >= BR_TBF_OP_MIN);
-        GF_ASSERT (op <= BR_TBF_OP_MAX);
-
-        bucket = *(tbf->bucket + op);
-        if (!bucket)
-                return;
-
-        LOCK (&bucket->lock);
-        {
-                /**
-                 * if there are enough tokens in the bucket there is no need
-                 * to throttle the request: therefore, consume the required
-                 * number of tokens and continue.
-                 */
-                if (tokens_requested <= bucket->tokens) {
-                        bucket->tokens -= tokens_requested;
-                } else {
-                        throttle = br_tbf_init_throttle (tokens_requested);
-                        if (!throttle) /* let it slip through for now.. */
-                                goto unblock;
-
-                        waitq = 1;
-                        pthread_mutex_lock (&throttle->mutex);
-                        list_add_tail (&throttle->list, &bucket->queued);
-                }
-        }
- unblock:
-        UNLOCK (&bucket->lock);
-
-        if (waitq) {
-                while (!throttle->done) {
-                        pthread_cond_wait (&throttle->cond, &throttle->mutex);
-                }
-
-                pthread_mutex_unlock (&throttle->mutex);
-
-                pthread_mutex_destroy (&throttle->mutex);
-                pthread_cond_destroy (&throttle->cond);
-
-                GF_FREE (throttle);
-        }
-}
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h b/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h
deleted file mode 100644
index 5a41be4fd95..00000000000
--- a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "list.h"
-#include "xlator.h"
-#include "locking.h"
-
-#ifndef __BIT_ROT_TBF_H__
-#define __BIT_ROT_TBF_H__
-
-typedef enum br_tbf_ops {
-        BR_TBF_OP_MIN     = -1,
-        BR_TBF_OP_HASH    = 0,    /* checksum calculation  */
-        BR_TBF_OP_READ    = 1,    /* inode read(s)         */
-        BR_TBF_OP_READDIR = 2,    /* dentry read(s)        */
-        BR_TBF_OP_MAX     = 3,
-} br_tbf_ops_t;
-
-/**
- * Operation rate specification
- */
-typedef struct br_tbf_opspec {
-        br_tbf_ops_t op;
-
-        unsigned long rate;
-
-        unsigned long maxlimit;
-} br_tbf_opspec_t;
-
-/**
- * Token bucket for each operation type
- */
-typedef struct br_tbf_bucket {
-        gf_lock_t lock;
-
-        pthread_t tokener;         /* token generator thread          */
-
-        unsigned long tokenrate;   /* token generation rate           */
-
-        unsigned long tokens;      /* number of current tokens        */
-
-        unsigned long maxtokens;   /* maximum token in the bucket     */
-
-        struct list_head queued;   /* list of non-conformant requests */
-} br_tbf_bucket_t;
-
-typedef struct br_tbf {
-        br_tbf_bucket_t **bucket;
-} br_tbf_t;
-
-br_tbf_t *
-br_tbf_init (br_tbf_opspec_t *, unsigned int);
-
-int
-br_tbf_mod (br_tbf_t *, br_tbf_opspec_t *);
-
-void
-br_tbf_throttle (br_tbf_t *, br_tbf_ops_t, unsigned long);
-
-#define TBF_THROTTLE_BEGIN(tbf, op, tokens) (br_tbf_throttle (tbf, op, tokens))
-#define TBF_THROTTLE_END(tbf, op, tokens) (void)
-
-#endif /** __BIT_ROT_TBF_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index ed436c34dc4..a2f1c343a1d 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -9,86 +9,81 @@
 */
 
 #include <ctype.h>
-#include <sys/uio.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "compat-errno.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/compat-errno.h>
 
 #include "bit-rot.h"
 #include "bit-rot-scrub.h"
 #include <pthread.h>
 #include "bit-rot-bitd-messages.h"
 
-#include "tw.h"
+#define BR_HASH_CALC_READ_SIZE (128 * 1024)
 
-#define BR_HASH_CALC_READ_SIZE  (128 * 1024)
-
-typedef int32_t (br_child_handler)(xlator_t *, br_child_t *);
+typedef int32_t(br_child_handler)(xlator_t *, br_child_t *);
 
 struct br_child_event {
-        xlator_t *this;
+    xlator_t *this;
 
-        br_child_t *child;
+    br_child_t *child;
 
-        br_child_handler *call;
+    br_child_handler *call;
 
-        struct list_head list;
+    struct list_head list;
 };
 
 static int
-br_find_child_index (xlator_t *this, xlator_t *child)
+br_find_child_index(xlator_t *this, xlator_t *child)
 {
-        br_private_t *priv   = NULL;
-        int           i      = -1;
-        int           index  = -1;
+    br_private_t *priv = NULL;
+    int i = -1;
+    int index = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (child == priv->children[i].xl) {
-                        index = i;
-                        break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (child == priv->children[i].xl) {
+            index = i;
+            break;
         }
+    }
 
 out:
-        return index;
+    return index;
 }
 
 br_child_t *
-br_get_child_from_brick_path (xlator_t *this, char *brick_path)
+br_get_child_from_brick_path(xlator_t *this, char *brick_path)
 {
-        br_private_t *priv  = NULL;
-        br_child_t   *child = NULL;
-        br_child_t   *tmp   = NULL;
-        int           i     = 0;
+    br_private_t *priv = NULL;
+    br_child_t *child = NULL;
+    br_child_t *tmp = NULL;
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, brick_path, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, brick_path, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        pthread_mutex_lock (&priv->lock);
-        {
-                for (i = 0; i < priv->child_count; i++) {
-                        tmp = &priv->children[i];
-                        if (!strcmp (tmp->brick_path, brick_path)) {
-                                child = tmp;
-                                break;
-                        }
-                }
+    pthread_mutex_lock(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            tmp = &priv->children[i];
+            if (!strcmp(tmp->brick_path, brick_path)) {
+                child = tmp;
+                break;
+            }
         }
-        pthread_mutex_unlock (&priv->lock);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
 out:
-        return child;
+    return child;
 }
 
 /**
@@ -96,18 +91,18 @@ out:
  * needed -- later.
  */
 void *
-br_brick_init (void *xl, struct gf_brick_spec *brick)
+br_brick_init(void *xl, struct gf_brick_spec *brick)
 {
-        return brick;
+    return brick;
 }
 
 /**
  * and cleanup things here when allocated br_brick_init().
  */
 void
-br_brick_fini (void *xl, char *brick, void *data)
+br_brick_fini(void *xl, char *brick, void *data)
 {
-        return;
+    return;
 }
 
 /**
@@ -119,109 +114,109 @@ br_brick_fini (void *xl, char *brick, void *data)
  *      change stub to handle this change.
  */
 static br_isignature_t *
-br_prepare_signature (const unsigned char *sign,
-                      unsigned long hashlen,
-                      int8_t hashtype, br_object_t *object)
+br_prepare_signature(const unsigned char *sign, unsigned long hashlen,
+                     int8_t hashtype, br_object_t *object)
 {
-        br_isignature_t *signature = NULL;
+    br_isignature_t *signature = NULL;
 
-        /* TODO: use mem-pool */
-        signature = GF_CALLOC (1, signature_size (hashlen + 1),
-                               gf_br_stub_mt_signature_t);
-        if (!signature)
-                return NULL;
+    /* TODO: use mem-pool */
+    signature = GF_CALLOC(1, signature_size(hashlen + 1),
+                          gf_br_stub_mt_signature_t);
+    if (!signature)
+        return NULL;
 
-        /* object version */
-        signature->signedversion = object->signedversion;
+    /* object version */
+    signature->signedversion = object->signedversion;
 
-        /* signature length & type */
-        signature->signaturelen = hashlen;
-        signature->signaturetype = hashtype;
+    /* signature length & type */
+    signature->signaturelen = hashlen;
+    signature->signaturetype = hashtype;
 
-        /* signature itself */
-        memcpy (signature->signature, (char *)sign, hashlen);
-        signature->signature[hashlen+1] = '\0';
+    /* signature itself */
+    memcpy(signature->signature, (char *)sign, hashlen);
+    signature->signature[hashlen + 1] = '\0';
 
-        return signature;
+    return signature;
 }
 
 gf_boolean_t
-bitd_is_bad_file (xlator_t *this, br_child_t *child, loc_t *loc, fd_t *fd)
+bitd_is_bad_file(xlator_t *this, br_child_t *child, loc_t *loc, fd_t *fd)
 {
-        int32_t       ret      = -1;
-        dict_t       *xattr    = NULL;
-        inode_t      *inode    = NULL;
-        gf_boolean_t  bad_file = _gf_false;
+    int32_t ret = -1;
+    dict_t *xattr = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t bad_file = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
 
-        inode = (loc) ? loc->inode : fd->inode;
+    inode = (loc) ? loc->inode : fd->inode;
 
-        if (fd)
-                ret = syncop_fgetxattr (child->xl, fd, &xattr,
-                                        BITROT_OBJECT_BAD_KEY, NULL, NULL);
-        else if (loc)
-                ret = syncop_getxattr (child->xl, loc,
-                                       &xattr, BITROT_OBJECT_BAD_KEY, NULL,
-                                       NULL);
+    if (fd)
+        ret = syncop_fgetxattr(child->xl, fd, &xattr, BITROT_OBJECT_BAD_KEY,
+                               NULL, NULL);
+    else if (loc)
+        ret = syncop_getxattr(child->xl, loc, &xattr, BITROT_OBJECT_BAD_KEY,
+                              NULL, NULL);
 
-        if (!ret) {
-                gf_msg_debug (this->name, 0, "[GFID: %s] is marked corrupted",
-                              uuid_utoa (inode->gfid));
-                bad_file = _gf_true;
-        }
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "[GFID: %s] is marked corrupted",
+                     uuid_utoa(inode->gfid));
+        bad_file = _gf_true;
+    }
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
 out:
-        return bad_file;
+    return bad_file;
 }
 
 /**
  * Do a lookup on the gfid present within the object.
  */
 static int32_t
-br_object_lookup (xlator_t *this, br_object_t *object,
-                  struct iatt *iatt, inode_t **linked_inode)
+br_object_lookup(xlator_t *this, br_object_t *object, struct iatt *iatt,
+                 inode_t **linked_inode)
 {
-	int      ret          = -EINVAL;
-	loc_t    loc          = {0, };
-	inode_t *inode        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, object, out);
-
-	inode = inode_find (object->child->table, object->gfid);
-
-        if (inode)
-                loc.inode = inode;
-        else
-                loc.inode = inode_new (object->child->table);
-
-	if (!loc.inode) {
-                ret = -ENOMEM;
-		goto out;
-        }
-
-	gf_uuid_copy (loc.gfid, object->gfid);
-
-	ret = syncop_lookup (object->child->xl, &loc, iatt, NULL, NULL, NULL);
-	if (ret < 0)
-		goto out;
-
-        /*
-         * The file might have been deleted by the application
-         * after getting the event, but before doing a lookup.
-         * So use linked_inode after inode_link is done.
-         */
-	*linked_inode = inode_link (loc.inode, NULL, NULL, iatt);
-	if (*linked_inode)
-		inode_lookup (*linked_inode);
+    int ret = -EINVAL;
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, object, out);
+
+    inode = inode_find(object->child->table, object->gfid);
+
+    if (inode)
+        loc.inode = inode;
+    else
+        loc.inode = inode_new(object->child->table);
+
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, object->gfid);
+
+    ret = syncop_lookup(object->child->xl, &loc, iatt, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+
+    /*
+     * The file might have been deleted by the application
+     * after getting the event, but before doing a lookup.
+     * So use linked_inode after inode_link is done.
+     */
+    *linked_inode = inode_link(loc.inode, NULL, NULL, iatt);
+    if (*linked_inode)
+        inode_lookup(*linked_inode);
 
 out:
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 /**
@@ -230,43 +225,44 @@ out:
  * passing xdata -- may be use frame->root->pid itself.
  */
 static int32_t
-br_object_open (xlator_t *this,
-                br_object_t *object, inode_t *inode, fd_t **openfd)
+br_object_open(xlator_t *this, br_object_t *object, inode_t *inode,
+               fd_t **openfd)
 {
-        int32_t      ret   = -1;
-        fd_t        *fd   = NULL;
-        loc_t        loc   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, object, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = -EINVAL;
-        fd = fd_create (inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
-                        "failed to create fd for the inode %s",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-        ret = syncop_open (object->child->xl, &loc, O_RDONLY, fd, NULL, NULL);
-	if (ret) {
-                br_log_object (this, "open", inode->gfid, -ret);
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-                *openfd = fd;
-	}
-
-        loc_wipe (&loc);
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, object, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = -EINVAL;
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_open(object->child->xl, &loc, O_RDONLY, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", inode->gfid, -ret);
+        fd_unref(fd);
+        fd = NULL;
+    } else {
+        fd_bind(fd);
+        *openfd = fd;
+    }
+
+    loc_wipe(&loc);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -274,287 +270,282 @@ out:
  * and return the buffer.
  */
 static int32_t
-br_object_read_block_and_sign (xlator_t *this, fd_t *fd, br_child_t *child,
-                               off_t offset, size_t size, SHA256_CTX *sha256)
+br_object_read_block_and_sign(xlator_t *this, fd_t *fd, br_child_t *child,
+                              off_t offset, size_t size, SHA256_CTX *sha256)
 {
-        int32_t        ret    = -1;
-        br_tbf_t      *tbf    = NULL;
-        struct iovec  *iovec  = NULL;
-        struct iobref *iobref = NULL;
-        br_private_t  *priv   = NULL;
-        int            count  = 0;
-        int            i      = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO (this->name, priv->tbf, out);
-        tbf = priv->tbf;
-
-        ret = syncop_readv (child->xl, fd,
-                            size, offset, 0, &iovec, &count, &iobref, NULL,
-                            NULL);
-
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno, BRB_MSG_READV_FAILED,
-                        "readv on %s failed", uuid_utoa (fd->inode->gfid));
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    tbf_t *tbf = NULL;
+    struct iovec *iovec = NULL;
+    struct iobref *iobref = NULL;
+    br_private_t *priv = NULL;
+    int count = 0;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO(this->name, priv->tbf, out);
+    tbf = priv->tbf;
+
+    ret = syncop_readv(child->xl, fd, size, offset, 0, &iovec, &count, &iobref,
+                       NULL, NULL, NULL);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRB_MSG_READV_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if (ret == 0)
-                goto out;
+    if (ret == 0)
+        goto out;
 
-        for (i = 0; i < count; i++) {
-                TBF_THROTTLE_BEGIN (tbf, BR_TBF_OP_HASH, iovec[i].iov_len);
-                {
-                        SHA256_Update (sha256, (const unsigned char *)
-                                       (iovec[i].iov_base), iovec[i].iov_len);
-                }
-                TBF_THROTTLE_BEGIN (tbf, BR_TBF_OP_HASH, iovec[i].iov_len);
+    for (i = 0; i < count; i++) {
+        TBF_THROTTLE_BEGIN(tbf, TBF_OP_HASH, iovec[i].iov_len);
+        {
+            SHA256_Update(sha256, (const unsigned char *)(iovec[i].iov_base),
+                          iovec[i].iov_len);
         }
+        TBF_THROTTLE_BEGIN(tbf, TBF_OP_HASH, iovec[i].iov_len);
+    }
 
- out:
-        if (iovec)
-                GF_FREE (iovec);
+out:
+    if (iovec)
+        GF_FREE(iovec);
 
-        if (iobref)
-                iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_calculate_obj_checksum (unsigned char *md,
-                           br_child_t *child, fd_t *fd, struct iatt *iatt)
+br_calculate_obj_checksum(unsigned char *md, br_child_t *child, fd_t *fd,
+                          struct iatt *iatt)
 {
-        int32_t   ret    = -1;
-        off_t     offset = 0;
-        size_t    block  = BR_HASH_CALC_READ_SIZE;
-        xlator_t *this   = NULL;
+    int32_t ret = -1;
+    off_t offset = 0;
+    size_t block = BR_HASH_CALC_READ_SIZE;
+    xlator_t *this = NULL;
 
-        SHA256_CTX       sha256;
+    SHA256_CTX sha256;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", child, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", iatt, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", iatt, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", fd, out);
 
-        this = child->this;
+    this = child->this;
 
-        SHA256_Init (&sha256);
+    SHA256_Init(&sha256);
 
-        while (1) {
-                ret = br_object_read_block_and_sign (this, fd, child,
-                                                     offset, block, &sha256);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_BLOCK_READ_FAILED, "reading block with "
-                                "offset %lu of object %s failed", offset,
-                                uuid_utoa (fd->inode->gfid));
-                        break;
-                }
-
-                if (ret == 0)
-                        break;
-
-                offset += ret;
+    while (1) {
+        ret = br_object_read_block_and_sign(this, fd, child, offset, block,
+                                            &sha256);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_BLOCK_READ_FAILED,
+                    "offset=%" PRIu64, offset, "object-gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            break;
         }
 
         if (ret == 0)
-                SHA256_Final (md, &sha256);
+            break;
 
- out:
-        return ret;
+        offset += ret;
+    }
+
+    if (ret == 0)
+        SHA256_Final(md, &sha256);
+
+out:
+    return ret;
 }
 
 static int32_t
-br_object_checksum (unsigned char *md,
-                    br_object_t *object, fd_t *fd, struct iatt *iatt)
+br_object_checksum(unsigned char *md, br_object_t *object, fd_t *fd,
+                   struct iatt *iatt)
 {
-        return br_calculate_obj_checksum (md, object->child, fd,  iatt);
+    return br_calculate_obj_checksum(md, object->child, fd, iatt);
 }
 
 static int32_t
-br_object_read_sign (inode_t *linked_inode, fd_t *fd, br_object_t *object,
-                     struct iatt *iatt)
+br_object_read_sign(inode_t *linked_inode, fd_t *fd, br_object_t *object,
+                    struct iatt *iatt)
 {
-        int32_t          ret           = -1;
-        xlator_t        *this          = NULL;
-        dict_t          *xattr         = NULL;
-        unsigned char   *md            = NULL;
-        br_isignature_t *sign          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", object, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", linked_inode, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", fd, out);
-
-        this = object->this;
-
-        md = GF_CALLOC (SHA256_DIGEST_LENGTH, sizeof (*md), gf_common_mt_char);
-        if (!md) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "failed to allocate memory for saving hash of the "
-                        "object %s", uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        ret = br_object_checksum (md, object, fd, iatt);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_CALC_CHECKSUM_FAILED, "calculating checksum "
-                        "for the object %s failed",
-                        uuid_utoa (linked_inode->gfid));
-                goto free_signature;
-        }
-
-        sign = br_prepare_signature (md, SHA256_DIGEST_LENGTH,
-                                     BR_SIGNATURE_TYPE_SHA256, object);
-        if (!sign) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
-                        "failed to get the signature for the object %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto free_signature;
-        }
-
-        xattr = dict_for_key_value
-                (GLUSTERFS_SET_OBJECT_SIGNATURE,
-                 (void *)sign, signature_size (SHA256_DIGEST_LENGTH));
-
-        if (!xattr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
-                        "dict allocation for signing failed for the object %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto free_isign;
-        }
-
-        ret = syncop_fsetxattr (object->child->xl, fd, xattr, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
-                        "fsetxattr of signature to the object %s failed",
-                        uuid_utoa (fd->inode->gfid));
-                goto unref_dict;
-        }
-
-        ret = 0;
-
- unref_dict:
-        dict_unref (xattr);
- free_isign:
-        GF_FREE (sign);
- free_signature:
-        GF_FREE (md);
- out:
-        return ret;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    dict_t *xattr = NULL;
+    unsigned char *md = NULL;
+    br_isignature_t *sign = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", object, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", linked_inode, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", fd, out);
+
+    this = object->this;
+
+    md = GF_MALLOC(SHA256_DIGEST_LENGTH, gf_common_mt_char);
+    if (!md) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_SAVING_HASH_FAILED,
+                "object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    ret = br_object_checksum(md, object, fd, iatt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_CHECKSUM_FAILED,
+                "object-gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto free_signature;
+    }
+
+    sign = br_prepare_signature(md, SHA256_DIGEST_LENGTH,
+                                BR_SIGNATURE_TYPE_SHA256, object);
+    if (!sign) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+                "object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto free_signature;
+    }
+
+    xattr = dict_for_key_value(GLUSTERFS_SET_OBJECT_SIGNATURE, (void *)sign,
+                               signature_size(SHA256_DIGEST_LENGTH), _gf_true);
+
+    if (!xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                "dict-allocation object-gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto free_isign;
+    }
+
+    ret = syncop_fsetxattr(object->child->xl, fd, xattr, 0, NULL, NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                "fsetxattr object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unref_dict;
+    }
+
+    ret = 0;
+
+unref_dict:
+    dict_unref(xattr);
+free_isign:
+    GF_FREE(sign);
+free_signature:
+    GF_FREE(md);
+out:
+    return ret;
 }
 
-static int br_object_sign_softerror (int32_t op_errno)
+static int
+br_object_sign_softerror(int32_t op_errno)
 {
-        return ((op_errno == ENOENT) || (op_errno == ESTALE)
-                || (op_errno == ENODATA));
+    return ((op_errno == ENOENT) || (op_errno == ESTALE) ||
+            (op_errno == ENODATA));
 }
 
 void
-br_log_object (xlator_t *this, char *op, uuid_t gfid, int32_t op_errno)
+br_log_object(xlator_t *this, char *op, uuid_t gfid, int32_t op_errno)
 {
-        int softerror = br_object_sign_softerror (op_errno);
-        if (softerror) {
-                gf_msg_debug (this->name, 0, "%s() failed on object %s "
-                              "[reason: %s]", op, uuid_utoa (gfid),
-                              strerror (op_errno));
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED,
-                        "%s() failed on object %s", op, uuid_utoa (gfid));
-        }
+    int softerror = br_object_sign_softerror(op_errno);
+    if (softerror) {
+        gf_msg_debug(this->name, 0,
+                     "%s() failed on object %s "
+                     "[reason: %s]",
+                     op, uuid_utoa(gfid), strerror(op_errno));
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED, "op=%s",
+                op, "gfid=%s", uuid_utoa(gfid), NULL);
+    }
 }
 
 void
-br_log_object_path (xlator_t *this, char *op,
-                    const char *path, int32_t op_errno)
+br_log_object_path(xlator_t *this, char *op, const char *path, int32_t op_errno)
 {
-        int softerror = br_object_sign_softerror (op_errno);
-        if (softerror) {
-                gf_msg_debug (this->name, 0, "%s() failed on object %s "
-                              "[reason: %s]", op, path, strerror (op_errno));
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED,
-                        "%s() failed on object %s", op, path);
-        }
+    int softerror = br_object_sign_softerror(op_errno);
+    if (softerror) {
+        gf_msg_debug(this->name, 0,
+                     "%s() failed on object %s "
+                     "[reason: %s]",
+                     op, path, strerror(op_errno));
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED, "op=%s",
+                op, "path=%s", path, NULL);
+    }
 }
 
 static void
-br_trigger_sign (xlator_t *this, br_child_t *child,
-                 inode_t *linked_inode, loc_t *loc, gf_boolean_t need_reopen)
+br_trigger_sign(xlator_t *this, br_child_t *child, inode_t *linked_inode,
+                loc_t *loc, gf_boolean_t need_reopen)
 {
-        fd_t     *fd   = NULL;
-        int32_t   ret  = -1;
-        uint32_t  val  = 0;
-        dict_t   *dict = NULL;
-        pid_t     pid  = GF_CLIENT_PID_BITD;
-
-        syncopctx_setfspid (&pid);
-
-        val = (need_reopen == _gf_true) ? BR_OBJECT_REOPEN : BR_OBJECT_RESIGN;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_uint32 (dict, BR_REOPEN_SIGN_HINT_KEY, val);
-        if (ret)
-                goto cleanup_dict;
-
-        ret = -1;
-        fd = fd_create (linked_inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
-                        "Failed to create fd [GFID %s]",
-                        uuid_utoa (linked_inode->gfid));
-                goto cleanup_dict;
-        }
-
-        ret = syncop_open (child->xl, loc, O_RDWR, fd, NULL, NULL);
-	if (ret) {
-                br_log_object (this, "open", linked_inode->gfid, -ret);
-                goto unref_fd;
-	}
-
-        fd_bind (fd);
-
-        ret = syncop_fsetxattr (child->xl, fd, dict, 0, NULL, NULL);
-        if (ret)
-                br_log_object (this, "fsetxattr", linked_inode->gfid, -ret);
-
-        /* passthough: fd_unref() */
-
- unref_fd:
-        fd_unref (fd);
- cleanup_dict:
-        dict_unref (dict);
- out:
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_TRIGGER_SIGN,
-                        "Could not trigger signingd for %s (reopen hint: %d)",
-                        uuid_utoa (linked_inode->gfid), val);
-        }
+    fd_t *fd = NULL;
+    int32_t ret = -1;
+    uint32_t val = 0;
+    dict_t *dict = NULL;
+    pid_t pid = GF_CLIENT_PID_BITD;
+
+    syncopctx_setfspid(&pid);
+
+    val = (need_reopen == _gf_true) ? BR_OBJECT_REOPEN : BR_OBJECT_RESIGN;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_uint32(dict, BR_REOPEN_SIGN_HINT_KEY, val);
+    if (ret)
+        goto cleanup_dict;
+
+    ret = -1;
+    fd = fd_create(linked_inode, 0);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto cleanup_dict;
+    }
+
+    ret = syncop_open(child->xl, loc, O_RDWR, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", linked_inode->gfid, -ret);
+        goto unref_fd;
+    }
+
+    fd_bind(fd);
+
+    ret = syncop_fsetxattr(child->xl, fd, dict, 0, NULL, NULL);
+    if (ret)
+        br_log_object(this, "fsetxattr", linked_inode->gfid, -ret);
+
+    /* passthough: fd_unref() */
+
+unref_fd:
+    fd_unref(fd);
+cleanup_dict:
+    dict_unref(dict);
+out:
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_TRIGGER_SIGN_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), "reopen-hint-val=%d",
+                val, NULL);
+    }
 }
 
 static void
-br_object_resign (xlator_t *this,
-                  br_object_t *object, inode_t *linked_inode)
+br_object_resign(xlator_t *this, br_object_t *object, inode_t *linked_inode)
 {
-        loc_t loc = {0, };
+    loc_t loc = {
+        0,
+    };
 
-        loc.inode = inode_ref (linked_inode);
-        gf_uuid_copy (loc.gfid, linked_inode->gfid);
+    loc.inode = inode_ref(linked_inode);
+    gf_uuid_copy(loc.gfid, linked_inode->gfid);
 
-        br_trigger_sign (this, object->child, linked_inode, &loc, _gf_false);
+    br_trigger_sign(this, object->child, linked_inode, &loc, _gf_false);
 
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 }
 
 /**
@@ -562,122 +553,123 @@ br_object_resign (xlator_t *this,
  * some form of priority scheduling and/or read burstness to avoid starving
  * (or kicking) client I/O's.
  */
-static int32_t br_sign_object (br_object_t *object)
+static int32_t
+br_sign_object(br_object_t *object)
 {
-        int32_t         ret           = -1;
-        inode_t        *linked_inode  = NULL;
-        xlator_t       *this          = NULL;
-        fd_t           *fd            = NULL;
-        struct iatt     iatt          = {0, };
-        pid_t           pid           = GF_CLIENT_PID_BITD;
-        br_sign_state_t sign_info     = BR_SIGN_NORMAL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", object, out);
-
-        this = object->this;
-
-        /**
-         * FIXME: This is required as signing an object is restricted to
-         * clients with special frame->root->pid. Change the way client
-         * pid is set.
-         */
-        syncopctx_setfspid (&pid);
-
-        ret = br_object_lookup (this, object, &iatt, &linked_inode);
-        if (ret) {
-                br_log_object (this, "lookup", object->gfid, -ret);
-                goto out;
-        }
-
-        /**
-         * For fd's that have notified for reopening, we send an explicit
-         * open() followed by a dummy write() call. This triggers the
-         * actual signing of the object.
-         */
-        sign_info = ntohl (object->sign_info);
-        if (sign_info == BR_SIGN_REOPEN_WAIT) {
-                br_object_resign (this, object, linked_inode);
-                goto unref_inode;
-        }
-
-        ret = br_object_open (this, object, linked_inode, &fd);
-        if (!fd) {
-                br_log_object (this, "open", object->gfid, -ret);
-                goto unref_inode;
-        }
-
-        /**
-         * we have an open file descriptor on the object. from here on,
-         * do not be generous to file operation errors.
-         */
-        gf_msg_debug (this->name, 0, "Signing object [%s]",
-                      uuid_utoa (linked_inode->gfid));
-
-        ret = br_object_read_sign (linked_inode, fd, object, &iatt);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_READ_AND_SIGN_FAILED, "reading and signing of "
-                        "the object %s failed", uuid_utoa (linked_inode->gfid));
-                goto unref_fd;
-        }
-
-        ret = 0;
-
- unref_fd:
-        fd_unref (fd);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        return ret;
+    int32_t ret = -1;
+    inode_t *linked_inode = NULL;
+    xlator_t *this = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    pid_t pid = GF_CLIENT_PID_BITD;
+    br_sign_state_t sign_info = BR_SIGN_NORMAL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", object, out);
+
+    this = object->this;
+
+    /**
+     * FIXME: This is required as signing an object is restricted to
+     * clients with special frame->root->pid. Change the way client
+     * pid is set.
+     */
+    syncopctx_setfspid(&pid);
+
+    ret = br_object_lookup(this, object, &iatt, &linked_inode);
+    if (ret) {
+        br_log_object(this, "lookup", object->gfid, -ret);
+        goto out;
+    }
+
+    /**
+     * For fd's that have notified for reopening, we send an explicit
+     * open() followed by a dummy write() call. This triggers the
+     * actual signing of the object.
+     */
+    sign_info = ntohl(object->sign_info);
+    if (sign_info == BR_SIGN_REOPEN_WAIT) {
+        br_object_resign(this, object, linked_inode);
+        goto unref_inode;
+    }
+
+    ret = br_object_open(this, object, linked_inode, &fd);
+    if (!fd) {
+        br_log_object(this, "open", object->gfid, -ret);
+        goto unref_inode;
+    }
+
+    /**
+     * we have an open file descriptor on the object. from here on,
+     * do not be generous to file operation errors.
+     */
+    gf_msg_debug(this->name, 0, "Signing object [%s]",
+                 uuid_utoa(linked_inode->gfid));
+
+    ret = br_object_read_sign(linked_inode, fd, object, &iatt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_READ_AND_SIGN_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto unref_fd;
+    }
+
+    ret = 0;
+
+unref_fd:
+    fd_unref(fd);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    return ret;
 }
 
-static br_object_t *__br_pick_object (br_private_t *priv)
+static br_object_t *
+__br_pick_object(br_private_t *priv)
 {
-        br_object_t *object = NULL;
+    br_object_t *object = NULL;
 
-        while (list_empty (&priv->obj_queue->objects)) {
-                pthread_cond_wait (&priv->object_cond, &priv->lock);
-        }
+    while (list_empty(&priv->obj_queue->objects)) {
+        pthread_cond_wait(&priv->object_cond, &priv->lock);
+    }
 
-        object = list_first_entry
-                (&priv->obj_queue->objects, br_object_t, list);
-        list_del_init (&object->list);
+    object = list_first_entry(&priv->obj_queue->objects, br_object_t, list);
+    list_del_init(&object->list);
 
-        return object;
+    return object;
 }
 
 /**
  * This is the place where the signing of the objects is triggered.
  */
 void *
-br_process_object (void *arg)
+br_process_object(void *arg)
 {
-        xlator_t     *this   = NULL;
-        br_object_t  *object = NULL;
-        br_private_t *priv   = NULL;
-        int32_t       ret    = -1;
-
-        this = arg;
-        priv = this->private;
-
-        THIS = this;
-
-        for (;;) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        object = __br_pick_object (priv);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                ret = br_sign_object (object);
-                if (ret && !br_object_sign_softerror (-ret))
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_SIGN_FAILED, "SIGNING FAILURE [%s]",
-                                uuid_utoa (object->gfid));
-                GF_FREE (object);
+    xlator_t *this = NULL;
+    br_object_t *object = NULL;
+    br_private_t *priv = NULL;
+    int32_t ret = -1;
+
+    this = arg;
+    priv = this->private;
+
+    THIS = this;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            object = __br_pick_object(priv);
         }
+        pthread_mutex_unlock(&priv->lock);
 
-        return NULL;
+        ret = br_sign_object(object);
+        if (ret && !br_object_sign_softerror(-ret))
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                    "gfid=%s", uuid_utoa(object->gfid), NULL);
+        GF_FREE(object);
+    }
+
+    return NULL;
 }
 
 /**
@@ -694,98 +686,97 @@ br_process_object (void *arg)
  * NOTE: use call_time to instrument signing time in br_sign_object().
  */
 void
-br_add_object_to_queue (struct gf_tw_timer_list *timer,
-                        void *data, unsigned long call_time)
+br_add_object_to_queue(struct gf_tw_timer_list *timer, void *data,
+                       unsigned long call_time)
 {
-        br_object_t   *object = NULL;
-        xlator_t      *this   = NULL;
-        br_private_t  *priv   = NULL;
-
-        object = data;
-        this   = object->this;
-        priv   = this->private;
-
-        THIS = this;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_tail (&object->list, &priv->obj_queue->objects);
-                pthread_cond_broadcast (&priv->object_cond);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (timer)
-                mem_put (timer);
-        return;
+    br_object_t *object = NULL;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+
+    object = data;
+    this = object->this;
+    priv = this->private;
+
+    THIS = this;
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_tail(&object->list, &priv->obj_queue->objects);
+        pthread_cond_broadcast(&priv->object_cond);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    if (timer)
+        mem_put(timer);
+    return;
 }
 
 static br_object_t *
-br_initialize_object (xlator_t *this, br_child_t *child, changelog_event_t *ev)
+br_initialize_object(xlator_t *this, br_child_t *child, changelog_event_t *ev)
 {
-        br_object_t *object = NULL;
+    br_object_t *object = NULL;
 
-        object = GF_CALLOC (1, sizeof (*object), gf_br_mt_br_object_t);
-        if (!object)
-                goto out;
-        INIT_LIST_HEAD (&object->list);
+    object = GF_CALLOC(1, sizeof(*object), gf_br_mt_br_object_t);
+    if (!object)
+        goto out;
+    INIT_LIST_HEAD(&object->list);
 
-        object->this  = this;
-        object->child = child;
-        gf_uuid_copy (object->gfid, ev->u.releasebr.gfid);
+    object->this = this;
+    object->child = child;
+    gf_uuid_copy(object->gfid, ev->u.releasebr.gfid);
 
-        /* NOTE: it's BE, but no worry */
-        object->signedversion = ev->u.releasebr.version;
-        object->sign_info = ev->u.releasebr.sign_info;
+    /* NOTE: it's BE, but no worry */
+    object->signedversion = ev->u.releasebr.version;
+    object->sign_info = ev->u.releasebr.sign_info;
 
 out:
-        return object;
+    return object;
 }
 
 static struct gf_tw_timer_list *
-br_initialize_timer (xlator_t *this, br_object_t *object, br_child_t *child,
-                     changelog_event_t *ev)
+br_initialize_timer(xlator_t *this, br_object_t *object, br_child_t *child,
+                    changelog_event_t *ev)
 {
-        br_private_t  *priv = NULL;
-        struct gf_tw_timer_list *timer = NULL;
+    br_private_t *priv = NULL;
+    struct gf_tw_timer_list *timer = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        timer = mem_get0 (child->timer_pool);
-        if (!timer)
-                goto out;
-        INIT_LIST_HEAD (&timer->entry);
+    timer = mem_get0(child->timer_pool);
+    if (!timer)
+        goto out;
+    INIT_LIST_HEAD(&timer->entry);
 
-        timer->expires = priv->expiry_time;
-        if (!timer->expires)
-                timer->expires = 1;
+    timer->expires = priv->expiry_time;
+    if (!timer->expires)
+        timer->expires = 1;
 
-        timer->data     = object;
-        timer->function = br_add_object_to_queue;
-        gf_tw_add_timer (priv->timer_wheel, timer);
+    timer->data = object;
+    timer->function = br_add_object_to_queue;
+    gf_tw_add_timer(priv->timer_wheel, timer);
 
 out:
-        return timer;
+    return timer;
 }
 
 static int32_t
-br_schedule_object_reopen (xlator_t *this, br_object_t *object,
-                           br_child_t *child, changelog_event_t *ev)
+br_schedule_object_reopen(xlator_t *this, br_object_t *object,
+                          br_child_t *child, changelog_event_t *ev)
 {
-        struct gf_tw_timer_list *timer = NULL;
-
-        timer = br_initialize_timer (this, object, child, ev);
-        if (!timer)
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_TIMER_FAILED,
-                        "Failed to allocate object expiry timer [GFID: %s]",
-                        uuid_utoa (object->gfid));
-        return timer ? 0 : -1;
+    struct gf_tw_timer_list *timer = NULL;
+
+    timer = br_initialize_timer(this, object, child, ev);
+    if (!timer)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_TIMER_FAILED,
+                "gfid=%s", uuid_utoa(object->gfid), NULL);
+    return timer ? 0 : -1;
 }
 
 static int32_t
-br_object_quicksign (xlator_t *this, br_object_t *object)
+br_object_quicksign(xlator_t *this, br_object_t *object)
 {
-        br_add_object_to_queue (NULL, object, 0ULL);
-        return 0;
+    br_add_object_to_queue(NULL, object, 0ULL);
+    return 0;
 }
 
 /**
@@ -798,148 +789,146 @@ br_object_quicksign (xlator_t *this, br_object_t *object)
  * object as a single alloc and bifurcate their respective pointers.
  */
 void
-br_brick_callback (void *xl, char *brick,
-                   void *data, changelog_event_t *ev)
+br_brick_callback(void *xl, char *brick, void *data, changelog_event_t *ev)
 {
-        int32_t          ret       = 0;
-        uuid_t           gfid      = {0,};
-        xlator_t        *this      = NULL;
-        br_object_t     *object    = NULL;
-        br_child_t      *child     = NULL;
-        br_sign_state_t  sign_info = BR_SIGN_INVALID;
-
-        this = xl;
-
-        GF_VALIDATE_OR_GOTO (this->name, ev, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        GF_ASSERT (ev->ev_type == CHANGELOG_OP_TYPE_BR_RELEASE);
-        GF_ASSERT (!gf_uuid_is_null (ev->u.releasebr.gfid));
-
-        gf_uuid_copy (gfid, ev->u.releasebr.gfid);
-
-        gf_msg_debug (this->name, 0, "RELEASE EVENT [GFID %s]",
-                      uuid_utoa (gfid));
-
-        child = br_get_child_from_brick_path (this, brick);
-        if (!child) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SUBVOL_FAILED,
-                        "failed to get the subvolume for the brick %s", brick);
-                goto out;
-        }
-
-        object = br_initialize_object (this, child, ev);
-        if (!object) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "failed to allocate object memory [GFID: %s]",
-                        uuid_utoa (gfid));
-                goto out;
-        }
-
-        /* sanity check */
-        sign_info = ntohl (object->sign_info);
-        GF_ASSERT (sign_info != BR_SIGN_NORMAL);
-
-        if (sign_info == BR_SIGN_REOPEN_WAIT)
-                ret = br_schedule_object_reopen (this, object, child, ev);
-        else
-                ret = br_object_quicksign (this, object);
-
-        if (ret)
-                goto free_object;
-
-        gf_msg_debug (this->name, 0, "->callback: brick [%s], type [%d]\n",
-                      brick, ev->ev_type);
-        return;
-
- free_object:
-        GF_FREE (object);
- out:
-        return;
+    int32_t ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    br_object_t *object = NULL;
+    br_child_t *child = NULL;
+    br_sign_state_t sign_info = BR_SIGN_INVALID;
+
+    this = xl;
+
+    GF_VALIDATE_OR_GOTO(this->name, ev, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    GF_ASSERT(ev->ev_type == CHANGELOG_OP_TYPE_BR_RELEASE);
+    GF_ASSERT(!gf_uuid_is_null(ev->u.releasebr.gfid));
+
+    gf_uuid_copy(gfid, ev->u.releasebr.gfid);
+
+    gf_msg_debug(this->name, 0, "RELEASE EVENT [GFID %s]", uuid_utoa(gfid));
+
+    child = br_get_child_from_brick_path(this, brick);
+    if (!child) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SUBVOL_FAILED,
+                "brick=%s", brick, NULL);
+        goto out;
+    }
+
+    object = br_initialize_object(this, child, ev);
+    if (!object) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                "object-gfid=%s", uuid_utoa(gfid), NULL);
+        goto out;
+    }
+
+    /* sanity check */
+    sign_info = ntohl(object->sign_info);
+    GF_ASSERT(sign_info != BR_SIGN_NORMAL);
+
+    if (sign_info == BR_SIGN_REOPEN_WAIT)
+        ret = br_schedule_object_reopen(this, object, child, ev);
+    else
+        ret = br_object_quicksign(this, object);
+
+    if (ret)
+        goto free_object;
+
+    gf_msg_debug(this->name, 0, "->callback: brick [%s], type [%d]\n", brick,
+                 ev->ev_type);
+    return;
+
+free_object:
+    GF_FREE(object);
+out:
+    return;
 }
 
 void
-br_fill_brick_spec (struct gf_brick_spec *brick, char *path)
+br_fill_brick_spec(struct gf_brick_spec *brick, char *path)
 {
-        brick->brick_path = gf_strdup (path);
-        brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
-
-        brick->init         = br_brick_init;
-        brick->fini         = br_brick_fini;
-        brick->callback     = br_brick_callback;
-        brick->connected    = NULL;
-        brick->disconnected = NULL;
+    brick->brick_path = gf_strdup(path);
+    brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
+
+    brick->init = br_brick_init;
+    brick->fini = br_brick_fini;
+    brick->callback = br_brick_callback;
+    brick->connected = NULL;
+    brick->disconnected = NULL;
 }
 
 static gf_boolean_t
-br_check_object_need_sign (xlator_t *this, dict_t *xattr, br_child_t *child)
+br_check_object_need_sign(xlator_t *this, dict_t *xattr, br_child_t *child)
 {
-        int32_t              ret       = -1;
-        gf_boolean_t         need_sign = _gf_false;
-        br_isignature_out_t *sign      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, xattr, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-
-        ret = dict_get_ptr (xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                            (void **)&sign);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
-                        "failed to get object signature info");
-                goto out;
-        }
+    int32_t ret = -1;
+    gf_boolean_t need_sign = _gf_false;
+    br_isignature_out_t *sign = NULL;
 
-        /* Object has been opened and hence dirty. Do not sign it */
-        if (sign->stale)
-                need_sign = _gf_true;
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, xattr, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
 
-out:
-        return need_sign;
-}
+    ret = dict_get_ptr(xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **)&sign);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+                "object-info", NULL);
+        goto out;
+    }
 
+    /* Object has been opened and hence dirty. Do not sign it */
+    if (sign->stale)
+        need_sign = _gf_true;
 
+out:
+    return need_sign;
+}
 
 int32_t
-br_prepare_loc (xlator_t *this, br_child_t *child, loc_t *parent,
-                gf_dirent_t *entry, loc_t *loc)
+br_prepare_loc(xlator_t *this, br_child_t *child, loc_t *parent,
+               gf_dirent_t *entry, loc_t *loc)
 {
-        int32_t  ret   = -1;
-        inode_t *inode = NULL;
-
-        inode = inode_grep (child->table, parent->inode, entry->d_name);
-        if (!inode)
-                loc->inode = inode_new (child->table);
-        else {
-                loc->inode = inode;
-                if (loc->inode->ia_type != IA_IFREG) {
-                        gf_msg_debug (this->name, 0, "%s is not a regular "
-                                      "file", entry->d_name);
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        loc->parent = inode_ref (parent->inode);
-        gf_uuid_copy (loc->pargfid, parent->inode->gfid);
-
-        ret = inode_path (parent->inode, entry->d_name, (char **)&loc->path);
-        if (ret < 0 || !loc->path) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_PATH_FAILED,
-                        "inode_path on %s (parent: %s) failed", entry->d_name,
-                        uuid_utoa (parent->inode->gfid));
-                goto out;
-        }
-
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-
-        ret = 1;
+    int32_t ret = -1;
+    inode_t *inode = NULL;
+
+    inode = inode_grep(child->table, parent->inode, entry->d_name);
+    if (!inode)
+        loc->inode = inode_new(child->table);
+    else {
+        loc->inode = inode;
+        if (loc->inode->ia_type != IA_IFREG) {
+            gf_msg_debug(this->name, 0,
+                         "%s is not a regular "
+                         "file",
+                         entry->d_name);
+            ret = 0;
+            goto out;
+        }
+    }
+
+    loc->parent = inode_ref(parent->inode);
+    gf_uuid_copy(loc->pargfid, parent->inode->gfid);
+
+    ret = inode_path(parent->inode, entry->d_name, (char **)&loc->path);
+    if (ret < 0 || !loc->path) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_PATH_FAILED,
+                "inode_path=%s", entry->d_name, "parent-gfid=%s",
+                uuid_utoa(parent->inode->gfid), NULL);
+        goto out;
+    }
+
+    loc->name = strrchr(loc->path, '/');
+    if (loc->name)
+        loc->name++;
+
+    ret = 1;
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -949,154 +938,174 @@ out:
  * last run for whatever reason (node crashes, reboots, etc..) become
  * candidates for signing. This allows the signature to "catch up" with
  * the current state of the object. Triggering signing is easy: perform
- * an open() followed by a close() therby resulting in call boomerang.
+ * an open() followed by a close() thereby resulting in call boomerang.
  * (though not back to itself :))
  */
 int
-bitd_oneshot_crawl (xlator_t *subvol,
-                    gf_dirent_t *entry, loc_t *parent, void *data)
+bitd_oneshot_crawl(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                   void *data)
 {
-        int           op_errno     = 0;
-        br_child_t   *child        = NULL;
-        xlator_t     *this         = NULL;
-        loc_t         loc          = {0, };
-        struct iatt   iatt         = {0, };
-        struct iatt   parent_buf   = {0, };
-        dict_t       *xattr        = NULL;
-        int32_t       ret          = -1;
-        inode_t      *linked_inode = NULL;
-        gf_boolean_t  need_signing = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", subvol, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", data, out);
-
-        child = data;
-        this = child->this;
-
-        ret = br_prepare_loc (this, child, parent, entry, &loc);
-        if (!ret)
-                goto out;
-
-        ret = syncop_lookup (child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
-        if (ret) {
-                br_log_object_path (this, "lookup", loc.path, -ret);
-                goto out;
-        }
-
-        linked_inode = inode_link (loc.inode, parent->inode, loc.name, &iatt);
-        if (linked_inode)
-                inode_lookup (linked_inode);
-
-        if (iatt.ia_type != IA_IFREG) {
-                gf_msg_debug (this->name, 0,  "%s is not a regular file, "
-                              "skipping..", entry->d_name);
-                ret = 0;
-                goto unref_inode;
-        }
+    int op_errno = 0;
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent_buf = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int32_t ret = -1;
+    inode_t *linked_inode = NULL;
+    gf_boolean_t need_signing = _gf_false;
+    gf_boolean_t need_reopen = _gf_true;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", subvol, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", data, out);
+
+    child = data;
+    this = child->this;
+
+    ret = br_prepare_loc(this, child, parent, entry, &loc);
+    if (!ret)
+        goto out;
+
+    ret = syncop_lookup(child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
+    if (ret) {
+        br_log_object_path(this, "lookup", loc.path, -ret);
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, parent->inode, loc.name, &iatt);
+    if (linked_inode)
+        inode_lookup(linked_inode);
+
+    if (iatt.ia_type != IA_IFREG) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not a regular file, "
+                     "skipping..",
+                     entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
+
+    /**
+     * As of now, 2 cases  are possible and handled.
+     * 1) GlusterFS is upgraded from a previous version which does not
+     *    have any idea about bit-rot and have data in the filesystem.
+     *    In this case syncop_getxattr fails with ENODATA and the object
+     *    is signed. (In real, when crawler sends lookup, bit-rot-stub
+     *    creates the xattrs before returning lookup reply)
+     * 2) Bit-rot was not enabled or BitD was does for some reasons, during
+     *    which some files were created, but since BitD was down, were not
+     *    signed.
+     * If the file was just created and was being written some data when
+     * the down BitD came up, then bit-rot stub should be intelligent to
+     * identify this case (by comparing the ongoing version or by checking
+     * if there are any fds present for that inode) and handle properly.
+     */
+
+    if (bitd_is_bad_file(this, child, &loc, NULL)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT, "path=%s",
+                loc.path, NULL);
+        goto unref_inode;
+    }
+
+    ret = syncop_getxattr(child->xl, &loc, &xattr,
+                          GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        br_log_object(this, "getxattr", linked_inode->gfid, op_errno);
 
         /**
-         * As of now, 2 cases  are possible and handled.
-         * 1) GlusterFS is upgraded from a previous version which does not
-         *    have any idea about bit-rot and have data in the filesystem.
-         *    In this case syncop_getxattr fails with ENODATA and the object
-         *    is signed. (In real, when crawler sends lookup, bit-rot-stub
-         *    creates the xattrs before returning lookup reply)
-         * 2) Bit-rot was not enabled or BitD was dows for some reasons, during
-         *    which some files were created, but since BitD was down, were not
-         *    signed.
-         * If the file was just created and was being written some data when
-         * the down BitD came up, then bit-rot stub should be intelligent to
-         * identify this case (by comparing the ongoing version or by checking
-         * if there are any fds present for that inode) and handle properly.
+         * No need to sign the zero byte objects as the signing
+         * happens upon first modification of the object.
          */
+        if (op_errno == ENODATA && (iatt.ia_size != 0))
+            need_signing = _gf_true;
+        if (op_errno == EINVAL)
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    BRB_MSG_PARTIAL_VERSION_PRESENCE, "gfid=%s",
+                    uuid_utoa(linked_inode->gfid), NULL);
+    } else {
+        need_signing = br_check_object_need_sign(this, xattr, child);
 
-        if (bitd_is_bad_file (this, child, &loc, NULL)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT,
-                        "Entry [%s] is marked corrupted.. skipping.", loc.path);
-                goto unref_inode;
-        }
-
-        ret = syncop_getxattr (child->xl, &loc, &xattr,
-                               GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                br_log_object (this, "getxattr", linked_inode->gfid, op_errno);
-
-                /**
-                 * No need to sign the zero byte objects as the signing
-                 * happens upon first modification of the object.
-                 */
-                if (op_errno == ENODATA && (iatt.ia_size != 0))
-                        need_signing = _gf_true;
-                if (op_errno == EINVAL)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                BRB_MSG_PARTIAL_VERSION_PRESENCE, "Partial "
-                                "version xattr presence detected, ignoring "
-                                "[GFID: %s]", uuid_utoa (linked_inode->gfid));
-        } else {
-                need_signing = br_check_object_need_sign (this, xattr, child);
+        /*
+         * If we are here means, bitrot daemon has started. Is it just
+         * a simple restart of the daemon or is it started because the
+         * feature is enabled is something hard to determine. Hence,
+         * if need_signing is false (because bit-rot version and signature
+         * are present), then still go ahead and sign it.
+         */
+        if (!need_signing) {
+            need_signing = _gf_true;
+            need_reopen = _gf_true;
         }
+    }
 
-        if (!need_signing)
-                goto unref_dict;
+    if (!need_signing)
+        goto unref_dict;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_TRIGGER_SIGN,
-                "Triggering signing for %s [GFID: %s | Brick: %s]",
-                loc.path, uuid_utoa (linked_inode->gfid), child->brick_path);
-        br_trigger_sign (this, child, linked_inode, &loc, _gf_true);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_TRIGGER_SIGN, "path=%s",
+            loc.path, "gfid=%s", uuid_utoa(linked_inode->gfid), "Brick-path=%s",
+            child->brick_path, NULL);
+    br_trigger_sign(this, child, linked_inode, &loc, need_reopen);
 
-        ret = 0;
+    ret = 0;
 
- unref_dict:
-        if (xattr)
-                dict_unref (xattr);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        loc_wipe (&loc);
+unref_dict:
+    if (xattr)
+        dict_unref(xattr);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
 #define BR_CRAWL_THROTTLE_COUNT 50
-#define BR_CRAWL_THROTTLE_ZZZ   5
+#define BR_CRAWL_THROTTLE_ZZZ 5
 
 void *
-br_oneshot_signer (void *arg)
+br_oneshot_signer(void *arg)
 {
-        loc_t       loc   = {0,};
-        xlator_t   *this  = NULL;
-        br_child_t *child = NULL;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *this = NULL;
+    br_child_t *child = NULL;
 
-        child = arg;
-        this = child->this;
+    child = arg;
+    this = child->this;
 
-        THIS = this;
+    THIS = this;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_START,
-                "Crawling brick [%s], scanning for unsigned objects",
-                child->brick_path);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_START, "brick-path=%s",
+            child->brick_path, NULL);
 
-        loc.inode = child->table->root;
-        (void) syncop_ftw_throttle
-                         (child->xl, &loc,
-                         GF_CLIENT_PID_BITD, child, bitd_oneshot_crawl,
-                         BR_CRAWL_THROTTLE_COUNT, BR_CRAWL_THROTTLE_ZZZ);
+    loc.inode = child->table->root;
+    (void)syncop_ftw_throttle(child->xl, &loc, GF_CLIENT_PID_BITD, child,
+                              bitd_oneshot_crawl, BR_CRAWL_THROTTLE_COUNT,
+                              BR_CRAWL_THROTTLE_ZZZ);
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_FINISH,
-                "Completed crawling brick [%s]", child->brick_path);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_FINISH,
+            "brick-path=%s", child->brick_path, NULL);
 
-        return NULL;
+    return NULL;
 }
 
 static void
-br_set_child_state (br_child_t *child, br_child_state_t state)
+br_set_child_state(br_child_t *child, br_child_state_t state)
 {
-        pthread_mutex_lock (&child->lock);
-        {
-                _br_set_child_state (child, state);
-        }
-        pthread_mutex_unlock (&child->lock);
+    pthread_mutex_lock(&child->lock);
+    {
+        _br_set_child_state(child, state);
+    }
+    pthread_mutex_unlock(&child->lock);
 }
 
 /**
@@ -1111,158 +1120,157 @@ br_set_child_state (br_child_t *child, br_child_state_t state)
  * notifications.
  */
 static int32_t
-br_enact_signer (xlator_t *this, br_child_t *child, br_stub_init_t *stub)
+br_enact_signer(xlator_t *this, br_child_t *child, br_stub_init_t *stub)
 {
-        int32_t ret = 0;
-        br_private_t *priv = NULL;
-        struct gf_brick_spec *brick = NULL;
-
-        priv = this->private;
-
-        brick = GF_CALLOC (1, sizeof (struct gf_brick_spec),
-                           gf_common_mt_gf_brick_spec_t);
-        if (!brick)
-                goto error_return;
-
-        br_fill_brick_spec (brick, stub->export);
-        ret = gf_changelog_register_generic
-                         (brick, 1, 1, this->ctx->cmd_args.log_file, -1, this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        BRB_MSG_REGISTER_FAILED, "Register to changelog "
-                        "failed");
-                goto dealloc;
-        }
-
-        child->threadrunning = 0;
-        ret = gf_thread_create (&child->thread, NULL, br_oneshot_signer, child);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SPAWN_FAILED,
-                        "failed to spawn FS crawler thread");
-        else
-                child->threadrunning = 1;
-
-        /* it's OK to continue, "old" objects would be signed when modified */
-        list_add_tail (&child->list, &priv->signing);
-        return 0;
-
- dealloc:
-        GF_FREE (brick);
- error_return:
-        return -1;
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct gf_brick_spec *brick = NULL;
+
+    priv = this->private;
+
+    brick = GF_CALLOC(1, sizeof(struct gf_brick_spec),
+                      gf_common_mt_gf_brick_spec_t);
+    if (!brick)
+        goto error_return;
+
+    br_fill_brick_spec(brick, stub->export);
+    ret = gf_changelog_register_generic(brick, 1, 1,
+                                        this->ctx->cmd_args.log_file, -1, this);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRB_MSG_REGISTER_FAILED, NULL);
+        goto dealloc;
+    }
+
+    child->threadrunning = 0;
+    ret = gf_thread_create(&child->thread, NULL, br_oneshot_signer, child,
+                           "brosign");
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SPAWN_FAILED,
+                "FS-crawler-thread", NULL);
+    else
+        child->threadrunning = 1;
+
+    /* it's OK to continue, "old" objects would be signed when modified */
+    list_add_tail(&child->list, &priv->signing);
+    return 0;
+
+dealloc:
+    GF_FREE(brick);
+error_return:
+    return -1;
 }
 
 static int32_t
-br_launch_scrubber (xlator_t *this, br_child_t *child,
-                    struct br_scanfs *fsscan, struct br_scrubber *fsscrub)
+br_launch_scrubber(xlator_t *this, br_child_t *child, struct br_scanfs *fsscan,
+                   struct br_scrubber *fsscrub)
 {
-        int32_t ret = -1;
-        br_private_t *priv = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-
-        scrub_monitor = &priv->scrub_monitor;
-        ret = gf_thread_create (&child->thread, NULL, br_fsscanner, child);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_SPAWN_FAILED,
-                        "failed to spawn bitrot scrubber daemon [Brick: %s]",
-                        child->brick_path);
-                goto error_return;
-        }
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+
+    scrub_monitor = &priv->scrub_monitor;
+    ret = gf_thread_create(&child->thread, NULL, br_fsscanner, child,
+                           "brfsscan");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ALERT, 0, BRB_MSG_SPAWN_FAILED,
+                "bitrot-scrubber-daemon Brick-path=%s", child->brick_path,
+                NULL);
+        goto error_return;
+    }
+
+    /* Signal monitor to kick off state machine*/
+    pthread_mutex_lock(&scrub_monitor->mutex);
+    {
+        if (!scrub_monitor->inited)
+            pthread_cond_signal(&scrub_monitor->cond);
+        scrub_monitor->inited = _gf_true;
+    }
+    pthread_mutex_unlock(&scrub_monitor->mutex);
+
+    /**
+     * Everything has been setup.. add this subvolume to scrubbers
+     * list.
+     */
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        list_add_tail(&child->list, &fsscrub->scrublist);
+        pthread_cond_broadcast(&fsscrub->cond);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
+
+    return 0;
 
-        /* Signal monitor to kick off state machine*/
-        pthread_mutex_lock (&scrub_monitor->mutex);
-        {
-                if (!scrub_monitor->inited)
-                        pthread_cond_signal (&scrub_monitor->cond);
-                scrub_monitor->inited = _gf_true;
-        }
-        pthread_mutex_unlock (&scrub_monitor->mutex);
-
-        /**
-         * Everything has been setup.. add this subvolume to scrubbers
-         * list.
-         */
-        pthread_mutex_lock (&fsscrub->mutex);
-        {
-                list_add_tail (&child->list, &fsscrub->scrublist);
-                pthread_cond_broadcast (&fsscrub->cond);
-        }
-        pthread_mutex_unlock (&fsscrub->mutex);
-
-        return 0;
-
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static int32_t
-br_enact_scrubber (xlator_t *this, br_child_t *child)
+br_enact_scrubber(xlator_t *this, br_child_t *child)
 {
-        int32_t ret = 0;
-        br_private_t *priv = NULL;
-        struct br_scanfs *fsscan = NULL;
-        struct br_scrubber *fsscrub = NULL;
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_scrubber *fsscrub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        fsscan = &child->fsscan;
-        fsscrub = &priv->fsscrub;
+    fsscan = &child->fsscan;
+    fsscrub = &priv->fsscrub;
 
-        /**
-         * if this child already witnesses a successfull connection earlier
-         * there's no need to initialize mutexes, condvars, etc..
-         */
-        if (_br_child_witnessed_connection (child))
-                return br_launch_scrubber (this, child, fsscan, fsscrub);
+    /**
+     * if this child already witnesses a successful connection earlier
+     * there's no need to initialize mutexes, condvars, etc..
+     */
+    if (_br_child_witnessed_connection(child))
+        return br_launch_scrubber(this, child, fsscan, fsscrub);
 
-        LOCK_INIT (&fsscan->entrylock);
-        pthread_mutex_init (&fsscan->waitlock, NULL);
-        pthread_cond_init (&fsscan->waitcond, NULL);
+    LOCK_INIT(&fsscan->entrylock);
+    pthread_mutex_init(&fsscan->waitlock, NULL);
+    pthread_cond_init(&fsscan->waitcond, NULL);
 
-        fsscan->entries = 0;
-        INIT_LIST_HEAD (&fsscan->queued);
-        INIT_LIST_HEAD (&fsscan->ready);
+    fsscan->entries = 0;
+    INIT_LIST_HEAD(&fsscan->queued);
+    INIT_LIST_HEAD(&fsscan->ready);
 
-        ret = br_launch_scrubber (this, child, fsscan, fsscrub);
-        if (ret)
-                goto error_return;
+    ret = br_launch_scrubber(this, child, fsscan, fsscrub);
+    if (ret)
+        goto error_return;
 
-        return 0;
+    return 0;
 
- error_return:
-        LOCK_DESTROY (&fsscan->entrylock);
-        pthread_mutex_destroy (&fsscan->waitlock);
-        pthread_cond_destroy (&fsscan->waitcond);
+error_return:
+    LOCK_DESTROY(&fsscan->entrylock);
+    pthread_mutex_destroy(&fsscan->waitlock);
+    pthread_cond_destroy(&fsscan->waitcond);
 
-        return -1;
+    return -1;
 }
 
 static int32_t
-br_child_enaction (xlator_t *this, br_child_t *child, br_stub_init_t *stub)
+br_child_enaction(xlator_t *this, br_child_t *child, br_stub_init_t *stub)
 {
-        int32_t ret = -1;
-        br_private_t *priv = this->private;
+    int32_t ret = -1;
+    br_private_t *priv = this->private;
 
-        pthread_mutex_lock (&child->lock);
-        {
-                if (priv->iamscrubber)
-                        ret = br_enact_scrubber (this, child);
-                else
-                        ret = br_enact_signer (this, child, stub);
-
-                if (!ret) {
-                        child->witnessed = 1;
-                        _br_set_child_state (child, BR_CHILD_STATE_CONNECTED);
-                        gf_msg (this->name, GF_LOG_INFO,
-                                0, BRB_MSG_CONNECTED_TO_BRICK,
-                                "Connected to brick %s..", child->brick_path);
-                }
+    pthread_mutex_lock(&child->lock);
+    {
+        if (priv->iamscrubber)
+            ret = br_enact_scrubber(this, child);
+        else
+            ret = br_enact_signer(this, child, stub);
+
+        if (!ret) {
+            child->witnessed = 1;
+            _br_set_child_state(child, BR_CHILD_STATE_CONNECTED);
+            gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CONNECTED_TO_BRICK,
+                    "brick-path=%s", child->brick_path, NULL);
         }
-        pthread_mutex_unlock (&child->lock);
+    }
+    pthread_mutex_unlock(&child->lock);
 
-        return ret;
+    return ret;
 }
 
 /**
@@ -1273,129 +1281,130 @@ br_child_enaction (xlator_t *this, br_child_t *child, br_stub_init_t *stub)
  * process either acts as a signer or a scrubber.
  */
 int32_t
-br_brick_connect (xlator_t *this, br_child_t *child)
+br_brick_connect(xlator_t *this, br_child_t *child)
 {
-        int32_t         ret      = -1;
-        loc_t           loc      = {0, };
-        struct iatt     buf      = {0, };
-        struct iatt     parent   = {0, };
-        br_stub_init_t *stub     = NULL;
-        dict_t         *xattr    = NULL;
-        int             op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        br_child_set_scrub_state (child, _gf_false);
-        br_set_child_state (child, BR_CHILD_STATE_INITIALIZING);
-
-        loc.inode = inode_ref (child->table->root);
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-        loc.path = gf_strdup ("/");
-
-        ret = syncop_lookup (child->xl, &loc, &buf, &parent, NULL, NULL);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        BRB_MSG_LOOKUP_FAILED, "lookup on root failed");
-                goto wipeloc;
-        }
-
-        ret = syncop_getxattr (child->xl, &loc, &xattr,
-                               GLUSTERFS_GET_BR_STUB_INIT_TIME, NULL, NULL);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        BRB_MSG_GET_INFO_FAILED, "failed to get stub info");
-                goto wipeloc;
-        }
-
-        ret = dict_get_ptr (xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                            (void **)&stub);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_INFO_FAILED,
-                        "failed to extract stub information");
-                goto free_dict;
-        }
-
-        memcpy (child->brick_path, stub->export, strlen (stub->export) + 1);
-        child->tv.tv_sec = ntohl (stub->timebuf[0]);
-        child->tv.tv_usec = ntohl (stub->timebuf[1]);
-
-        ret = br_child_enaction (this, child, stub);
-
- free_dict:
-        dict_unref (xattr);
- wipeloc:
-        loc_wipe (&loc);
- out:
-        if (ret)
-                br_set_child_state (child, BR_CHILD_STATE_CONNFAILED);
-        return ret;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    struct iatt parent = {
+        0,
+    };
+    br_stub_init_t *stub = NULL;
+    dict_t *xattr = NULL;
+    int op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    br_child_set_scrub_state(child, _gf_false);
+    br_set_child_state(child, BR_CHILD_STATE_INITIALIZING);
+
+    loc.inode = inode_ref(child->table->root);
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    loc.path = gf_strdup("/");
+
+    ret = syncop_lookup(child->xl, &loc, &buf, &parent, NULL, NULL);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_LOOKUP_FAILED,
+                NULL);
+        goto wipeloc;
+    }
+
+    ret = syncop_getxattr(child->xl, &loc, &xattr,
+                          GLUSTERFS_GET_BR_STUB_INIT_TIME, NULL, NULL);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_GET_INFO_FAILED,
+                NULL);
+        goto wipeloc;
+    }
+
+    ret = dict_get_ptr(xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME, (void **)&stub);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_INFO_FAILED, NULL);
+        goto free_dict;
+    }
+
+    memcpy(child->brick_path, stub->export, strlen(stub->export) + 1);
+    child->tv.tv_sec = ntohl(stub->timebuf[0]);
+    child->tv.tv_usec = ntohl(stub->timebuf[1]);
+
+    ret = br_child_enaction(this, child, stub);
+
+free_dict:
+    dict_unref(xattr);
+wipeloc:
+    loc_wipe(&loc);
+out:
+    if (ret)
+        br_set_child_state(child, BR_CHILD_STATE_CONNFAILED);
+    return ret;
 }
 
 /* TODO: cleanup signer */
 static int32_t
-br_cleanup_signer (xlator_t *this, br_child_t *child)
+br_cleanup_signer(xlator_t *this, br_child_t *child)
 {
-        return 0;
+    return 0;
 }
 
 static int32_t
-br_cleanup_scrubber (xlator_t *this, br_child_t *child)
+br_cleanup_scrubber(xlator_t *this, br_child_t *child)
 {
-        int32_t ret = 0;
-        br_private_t *priv = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv    = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        if (_br_is_child_scrub_active (child)) {
-                scrub_monitor->active_child_count--;
-                br_child_set_scrub_state (child, _gf_false);
-        }
-
-        /**
-         * 0x0: child (brick) goes out of rotation
-         *
-         * This is fully safe w.r.t. entries for this child being actively
-         * scrubbed. Each of the scrubber thread(s) would finish scrubbing
-         * the entry (probably failing due to disconnection) and either
-         * putting the entry back into the queue or continuing further.
-         * Either way, pending entries for this child's queue need not be
-         * drained; entries just sit there in the queued/ready list to be
-         * consumed later upon re-connection.
-         */
-        pthread_mutex_lock (&fsscrub->mutex);
-        {
-                list_del_init (&child->list);
-        }
-        pthread_mutex_unlock (&fsscrub->mutex);
-
-        /**
-         * 0x1: cleanup scanner thread
-         *
-         * The pending timer needs to be removed _after_ cleaning up the
-         * filesystem scanner (scheduling the next scrub time is not a
-         * cancellation point).
-         */
-        ret = gf_thread_cleanup_xint (child->thread);
-        if (ret)
-                gf_msg (this->name, GF_LOG_INFO,
-                        0, BRB_MSG_SCRUB_THREAD_CLEANUP,
-                        "Error cleaning up scanner thread");
-
-        gf_msg (this->name, GF_LOG_INFO,
-                0, BRB_MSG_SCRUBBER_CLEANED,
-                "Cleaned up scrubber for brick [%s]", child->brick_path);
-
-        return 0;
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (_br_is_child_scrub_active(child)) {
+        scrub_monitor->active_child_count--;
+        br_child_set_scrub_state(child, _gf_false);
+    }
+
+    /**
+     * 0x0: child (brick) goes out of rotation
+     *
+     * This is fully safe w.r.t. entries for this child being actively
+     * scrubbed. Each of the scrubber thread(s) would finish scrubbing
+     * the entry (probably failing due to disconnection) and either
+     * putting the entry back into the queue or continuing further.
+     * Either way, pending entries for this child's queue need not be
+     * drained; entries just sit there in the queued/ready list to be
+     * consumed later upon re-connection.
+     */
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        list_del_init(&child->list);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
+
+    /**
+     * 0x1: cleanup scanner thread
+     *
+     * The pending timer needs to be removed _after_ cleaning up the
+     * filesystem scanner (scheduling the next scrub time is not a
+     * cancellation point).
+     */
+    ret = gf_thread_cleanup_xint(child->thread);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_THREAD_CLEANUP, NULL);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUBBER_CLEANED,
+            "brick-path=%s", child->brick_path, NULL);
+
+    return 0;
 }
 
 /**
@@ -1404,38 +1413,38 @@ br_cleanup_scrubber (xlator_t *this, br_child_t *child)
  * the inode table, it's just reused taking care of stale inodes)
  */
 int32_t
-br_brick_disconnect (xlator_t *this, br_child_t *child)
+br_brick_disconnect(xlator_t *this, br_child_t *child)
 {
-        int32_t ret = 0;
-        struct br_monitor *scrub_monitor = NULL;
-        br_private_t *priv = this->private;
+    int32_t ret = 0;
+    struct br_monitor *scrub_monitor = NULL;
+    br_private_t *priv = this->private;
+
+    scrub_monitor = &priv->scrub_monitor;
+
+    /* Lock order should be wakelock and then child lock to
+     * dead locks.
+     */
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        pthread_mutex_lock(&child->lock);
+        {
+            if (!_br_is_child_connected(child))
+                goto unblock;
 
-        scrub_monitor = &priv->scrub_monitor;
+            /* child is on death row.. */
+            _br_set_child_state(child, BR_CHILD_STATE_DISCONNECTED);
 
-        /* Lock order should be wakelock and then child lock to
-         * dead locks.
-         */
-        pthread_mutex_lock (&scrub_monitor->wakelock);
-        {
-                pthread_mutex_lock (&child->lock);
-                {
-                        if (!_br_is_child_connected (child))
-                                goto unblock;
-
-                        /* child is on death row.. */
-                        _br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED);
-
-                        if (priv->iamscrubber)
-                                ret = br_cleanup_scrubber (this, child);
-                        else
-                                ret = br_cleanup_signer (this, child);
-                }
- unblock:
-                pthread_mutex_unlock (&child->lock);
+            if (priv->iamscrubber)
+                ret = br_cleanup_scrubber(this, child);
+            else
+                ret = br_cleanup_signer(this, child);
         }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
+    unblock:
+        pthread_mutex_unlock(&child->lock);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
 
-         return ret;
+    return ret;
 }
 
 /**
@@ -1444,306 +1453,341 @@ br_brick_disconnect (xlator_t *this, br_child_t *child)
  * information regarding that brick (such as brick path).
  */
 void *
-br_handle_events (void *arg)
+br_handle_events(void *arg)
 {
-        int32_t       ret   = 0;
-        xlator_t     *this  = NULL;
-        br_private_t *priv  = NULL;
-        br_child_t *child = NULL;
-        struct br_child_event *childev = NULL;
-
-        this = arg;
-        priv = this->private;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+    br_child_t *child = NULL;
+    struct br_child_event *childev = NULL;
+
+    this = arg;
+    priv = this->private;
+
+    /*
+     * Since, this is the topmost xlator, THIS has to be set by bit-rot
+     * xlator itself (STACK_WIND won't help in this case). Also it has
+     * to be done for each thread that gets spawned. Otherwise, a new
+     * thread will get global_xlator's pointer when it does "THIS".
+     */
+    THIS = this;
+
+    while (1) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->bricks))
+                pthread_cond_wait(&priv->cond, &priv->lock);
 
-        /*
-         * Since, this is the topmost xlator, THIS has to be set by bit-rot
-         * xlator itself (STACK_WIND wont help in this case). Also it has
-         * to be done for each thread that gets spawned. Otherwise, a new
-         * thread will get global_xlator's pointer when it does "THIS".
-         */
-        THIS = this;
-
-        while (1) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->bricks))
-                                pthread_cond_wait (&priv->cond, &priv->lock);
-
-                        childev = list_first_entry
-                                   (&priv->bricks, struct br_child_event, list);
-                        list_del_init (&childev->list);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                child = childev->child;
-                ret = childev->call (this, child);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_SUBVOL_CONNECT_FAILED,
-                                "callback handler for subvolume [%s] failed",
-                                child->xl->name);
-                GF_FREE (childev);
+            childev = list_first_entry(&priv->bricks, struct br_child_event,
+                                       list);
+            list_del_init(&childev->list);
         }
+        pthread_mutex_unlock(&priv->lock);
 
-        return NULL;
+        child = childev->child;
+        ret = childev->call(this, child);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SUBVOL_CONNECT_FAILED,
+                    "name=%s", child->xl->name, NULL);
+        GF_FREE(childev);
+    }
+
+    return NULL;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int32_t ret = -1;
+    int32_t ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_br_stub_mt_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_MEM_ACNT_FAILED,
-                        "Memory accounting init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_br_stub_mt_end + 1);
 
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_MEM_ACNT_FAILED, NULL);
         return ret;
+    }
+
+    return ret;
 }
 
 static void
-_br_qchild_event (xlator_t *this, br_child_t *child, br_child_handler *call)
+_br_qchild_event(xlator_t *this, br_child_t *child, br_child_handler *call)
 {
-        br_private_t *priv = NULL;
-        struct br_child_event *childev = NULL;
+    br_private_t *priv = NULL;
+    struct br_child_event *childev = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        childev = GF_CALLOC (1, sizeof (*childev), gf_br_mt_br_child_event_t);
-        if (!childev) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "Event unhandled for child.. [Brick: %s]",
-                        child->xl->name);
-                return;
-        }
+    childev = GF_CALLOC(1, sizeof(*childev), gf_br_mt_br_child_event_t);
+    if (!childev) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_EVENT_UNHANDLED,
+                "Brick-name=%s", child->xl->name, NULL);
+        return;
+    }
 
-        INIT_LIST_HEAD (&childev->list);
-        childev->this  = this;
-        childev->child = child;
-        childev->call  = call;
+    INIT_LIST_HEAD(&childev->list);
+    childev->this = this;
+    childev->child = child;
+    childev->call = call;
 
-        list_add_tail (&childev->list, &priv->bricks);
+    list_add_tail(&childev->list, &priv->bricks);
 }
 
 int
-br_scrubber_status_get (xlator_t *this, dict_t **dict)
+br_scrubber_status_get(xlator_t *this, dict_t **dict)
 {
-
-        int                    ret          = -1;
-        char                   key[256]     = {0,};
-        br_private_t          *priv         = NULL;
-        struct br_scrub_stats *scrub_stats  = NULL;
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", priv, out);
-
-        scrub_stats = &priv->scrub_stat;
-
-        ret = br_get_bad_objects_list (this, dict);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to collect corrupt "
-                              "files");
-        }
-
-        memset (key, 0, 256);
-        snprintf (key, 256, "scrubbed-files");
-        ret = dict_set_uint64 (*dict, key, scrub_stats->scrubbed_files);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to setting scrubbed file "
-                              "entry to the dictionary");
-        }
-
-        memset (key, 0, 256);
-        snprintf (key, 256, "unsigned-files");
-        ret = dict_set_uint64 (*dict, key, scrub_stats->unsigned_files);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to set unsigned file count"
-                              " entry to the dictionary");
-        }
-
-        memset (key, 0, 256);
-        snprintf (key, 256, "scrub-duration");
-        ret = dict_set_uint64 (*dict, key, scrub_stats->scrub_duration);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to set scrub duration"
-                              " entry to the dictionary");
-        }
-
-        memset (key, 0, 256);
-        snprintf (key, 256, "last-scrub-time");
-        ret = dict_set_dynstr_with_alloc (*dict, key,
-                                          scrub_stats->last_scrub_time);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to set "
-                                      "last scrub time value");
-        }
+    int ret = -1;
+    br_private_t *priv = NULL;
+    struct br_scrub_stats *scrub_stats = NULL;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", priv, out);
+
+    scrub_stats = &priv->scrub_stat;
+
+    ret = br_get_bad_objects_list(this, dict);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to collect corrupt "
+                     "files");
+    }
+
+    ret = dict_set_int8(*dict, "scrub-running", scrub_stats->scrub_running);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed setting scrub_running "
+                     "entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "scrubbed-files", scrub_stats->scrubbed_files);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to setting scrubbed file "
+                     "entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "unsigned-files", scrub_stats->unsigned_files);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set unsigned file count"
+                     " entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "scrub-duration", scrub_stats->scrub_duration);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set scrub duration"
+                     " entry to the dictionary");
+    }
+
+    ret = dict_set_dynstr_with_alloc(*dict, "last-scrub-time",
+                                     scrub_stats->last_scrub_time);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set "
+                     "last scrub time value");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int           idx    = -1;
-        int           ret    = -1;
-        xlator_t     *subvol = NULL;
-        br_child_t   *child  = NULL;
-        br_private_t *priv   = NULL;
-        dict_t       *output = NULL;
-        va_list       ap;
+    int idx = -1;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    br_child_t *child = NULL;
+    br_private_t *priv = NULL;
+    dict_t *output = NULL;
+    va_list ap;
+    struct br_monitor *scrub_monitor = NULL;
 
-        subvol = (xlator_t *)data;
-        priv = this->private;
+    subvol = (xlator_t *)data;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg_trace (this->name, 0, "Notification received: %d", event);
+    gf_msg_trace(this->name, 0, "Notification received: %d", event);
 
-        idx = br_find_child_index (this, subvol);
+    idx = br_find_child_index(this, subvol);
 
-        switch (event) {
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                if (idx < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_INVALID_SUBVOL, "Got event %d from "
-                                "invalid subvolume", event);
-                        goto out;
-                }
-
-                pthread_mutex_lock (&priv->lock);
-                {
-                        child = &priv->children[idx];
-                        if (child->child_up == 1)
-                                goto unblock_0;
-                        priv->up_children++;
-
-                        child->child_up = 1;
-                        child->xl = subvol;
-                        if (!child->table)
-                                child->table = inode_table_new (4096, subvol);
-
-                        _br_qchild_event (this, child, br_brick_connect);
-                        pthread_cond_signal (&priv->cond);
-                }
+            if (idx < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_INVALID_SUBVOL,
+                        "event=%d", event, NULL);
+                goto out;
+            }
+
+            pthread_mutex_lock(&priv->lock);
+            {
+                child = &priv->children[idx];
+                if (child->child_up == 1)
+                    goto unblock_0;
+                priv->up_children++;
+
+                child->child_up = 1;
+                child->xl = subvol;
+                if (!child->table)
+                    child->table = inode_table_new(4096, subvol);
+
+                _br_qchild_event(this, child, br_brick_connect);
+                pthread_cond_signal(&priv->cond);
+            }
         unblock_0:
-                pthread_mutex_unlock (&priv->lock);
+            pthread_mutex_unlock(&priv->lock);
 
-                if (priv->up_children == priv->child_count)
-                        default_notify (this, event, data);
-                break;
+            if (priv->up_children == priv->child_count)
+                default_notify(this, event, data);
+            break;
 
         case GF_EVENT_CHILD_DOWN:
-                if (idx < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_INVALID_SUBVOL_CHILD,
-                                "Got event %d from invalid subvolume", event);
-                        goto out;
-                }
-
-                pthread_mutex_lock (&priv->lock);
-                {
-                        child = &priv->children[idx];
-                        if (child->child_up == 0)
-                                goto unblock_1;
-
-                        child->child_up = 0;
-                        priv->up_children--;
-
-                        _br_qchild_event (this, child, br_brick_disconnect);
-                        pthread_cond_signal (&priv->cond);
-                }
+            if (idx < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_INVALID_SUBVOL,
+                        "event=%d", event, NULL);
+                goto out;
+            }
+
+            pthread_mutex_lock(&priv->lock);
+            {
+                child = &priv->children[idx];
+                if (child->child_up == 0)
+                    goto unblock_1;
+
+                child->child_up = 0;
+                priv->up_children--;
+
+                _br_qchild_event(this, child, br_brick_disconnect);
+                pthread_cond_signal(&priv->cond);
+            }
         unblock_1:
-                pthread_mutex_unlock (&priv->lock);
+            pthread_mutex_unlock(&priv->lock);
 
-                if (priv->up_children == 0)
-                        default_notify (this, event, data);
-                break;
+            if (priv->up_children == 0)
+                default_notify(this, event, data);
+            break;
 
         case GF_EVENT_SCRUB_STATUS:
-                gf_msg_debug (this->name, GF_LOG_INFO, "BitRot scrub status "
-                              "called");
-                va_start (ap, data);
-                output = va_arg (ap, dict_t *);
-                va_end (ap);
-
-                ret = br_scrubber_status_get (this, &output);
-                gf_msg_debug (this->name, 0, "returning %d", ret);
-                break;
+            gf_msg_debug(this->name, GF_LOG_INFO,
+                         "BitRot scrub status "
+                         "called");
+            va_start(ap, data);
+            output = va_arg(ap, dict_t *);
+            va_end(ap);
+
+            ret = br_scrubber_status_get(this, &output);
+            gf_msg_debug(this->name, 0, "returning %d", ret);
+            break;
+
+        case GF_EVENT_SCRUB_ONDEMAND:
+            gf_log(this->name, GF_LOG_INFO,
+                   "BitRot scrub ondemand "
+                   "called");
+
+            if (scrub_monitor->state != BR_SCRUB_STATE_PENDING) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRB_MSG_RESCHEDULE_SCRUBBER_FAILED, "Current-state=%d",
+                        scrub_monitor->state, NULL);
+                return -2;
+            }
+
+            /* Needs synchronization with reconfigure thread */
+            pthread_mutex_lock(&priv->lock);
+            {
+                ret = br_scrub_state_machine(this, _gf_true);
+            }
+            pthread_mutex_unlock(&priv->lock);
+
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRB_MSG_COULD_NOT_SCHEDULE_SCRUB, NULL);
+            }
+            gf_msg_debug(this->name, 0, "returning %d", ret);
+            break;
         default:
-                default_notify (this, event, data);
-        }
+            default_notify(this, event, data);
+    }
 
- out:
-        return 0;
+out:
+    return 0;
 }
 
-/**
- * Initialize signer specific structures, spawn worker threads.
- */
-
 static void
-br_fini_signer (xlator_t *this, br_private_t *priv)
+br_fini_signer(xlator_t *this, br_private_t *priv)
 {
-        int i = 0;
+    int i = 0;
 
-        for (; i < BR_WORKERS; i++) {
-                (void) gf_thread_cleanup_xint (priv->obj_queue->workers[i]);
-        }
+    if (priv == NULL)
+        return;
 
-        pthread_cond_destroy (&priv->object_cond);
+    for (; i < priv->signer_th_count; i++) {
+        (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]);
+    }
+    GF_FREE(priv->obj_queue->workers);
+
+    pthread_cond_destroy(&priv->object_cond);
 }
 
+/**
+ * Initialize signer specific structures, spawn worker threads.
+ */
+
 static int32_t
-br_init_signer (xlator_t *this, br_private_t *priv)
+br_init_signer(xlator_t *this, br_private_t *priv)
 {
-        int i = 0;
-        int32_t ret = -1;
-
-        /* initialize gfchangelog xlator context */
-        ret = gf_changelog_init (this);
-        if (ret)
-                goto out;
-
-        pthread_cond_init (&priv->object_cond, NULL);
-
-        priv->obj_queue = GF_CALLOC (1, sizeof (*priv->obj_queue),
-                                     gf_br_mt_br_ob_n_wk_t);
-        if (!priv->obj_queue)
-                goto cleanup_cond;
-        INIT_LIST_HEAD (&priv->obj_queue->objects);
-
-        for (i = 0; i < BR_WORKERS; i++) {
-                ret = gf_thread_create (&priv->obj_queue->workers[i], NULL,
-                                        br_process_object, this);
-                if (ret != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                BRB_MSG_SPAWN_FAILED, "thread creation"
-                                " failed");
-                        ret = -1;
-                        goto cleanup_threads;
-                }
+    int i = 0;
+    int32_t ret = -1;
+
+    /* initialize gfchangelog xlator context */
+    ret = gf_changelog_init(this);
+    if (ret)
+        goto out;
+
+    pthread_cond_init(&priv->object_cond, NULL);
+
+    priv->obj_queue = GF_CALLOC(1, sizeof(*priv->obj_queue),
+                                gf_br_mt_br_ob_n_wk_t);
+    if (!priv->obj_queue)
+        goto cleanup_cond;
+    INIT_LIST_HEAD(&priv->obj_queue->objects);
+
+    priv->obj_queue->workers = GF_CALLOC(
+        priv->signer_th_count, sizeof(pthread_t), gf_br_mt_br_worker_t);
+    if (!priv->obj_queue->workers)
+        goto cleanup_obj_queue;
+
+    for (i = 0; i < priv->signer_th_count; i++) {
+        ret = gf_thread_create(&priv->obj_queue->workers[i], NULL,
+                               br_process_object, this, "brpobj");
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    BRB_MSG_THREAD_CREATION_FAILED, NULL);
+            ret = -1;
+            goto cleanup_threads;
         }
+    }
 
-        return 0;
+    return 0;
 
- cleanup_threads:
-        for (i--; i >= 0; i--) {
-                (void) gf_thread_cleanup_xint (priv->obj_queue->workers[i]);
-        }
+cleanup_threads:
+    for (i--; i >= 0; i--) {
+        (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]);
+    }
+    GF_FREE(priv->obj_queue->workers);
 
-        GF_FREE (priv->obj_queue);
+cleanup_obj_queue:
+    GF_FREE(priv->obj_queue);
 
- cleanup_cond:
-        /* that's explicit */
-        pthread_cond_destroy (&priv->object_cond);
- out:
-        return -1;
+cleanup_cond:
+    /* that's explicit */
+    pthread_cond_destroy(&priv->object_cond);
+out:
+    return -1;
 }
 
 /**
@@ -1752,339 +1796,358 @@ br_init_signer (xlator_t *this, br_private_t *priv)
  * throttle.
  */
 static int32_t
-br_rate_limit_signer (xlator_t *this, int child_count, int numbricks)
+br_rate_limit_signer(xlator_t *this, int child_count, int numbricks)
 {
-        br_private_t *priv = NULL;
-        br_tbf_opspec_t spec = {0,};
-
-        priv = this->private;
-
-        spec.op       = BR_TBF_OP_HASH;
-        spec.rate     = 0;
-        spec.maxlimit = 0;
+    br_private_t *priv = NULL;
+    tbf_opspec_t spec = {
+        0,
+    };
+
+    priv = this->private;
+
+    spec.op = TBF_OP_HASH;
+    spec.rate = 0;
+    spec.maxlimit = 0;
+
+    /**
+     * OK. Most implementations of TBF I've come across generate tokens
+     * every second (UML, etc..) and some chose sub-second granularity
+     * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
+     * any logic for choosing generation interval and it seems pretty
+     * logical as one could jack up token count per interval w.r.t.
+     * generation rate.
+     *
+     * Value used here is chosen based on a series of test(s) performed
+     * to balance object signing time and not maxing out on all available
+     * CPU cores. It's obvious to have seconds granularity and jack up
+     * token count per interval, thereby achieving close to similar
+     * results. Let's stick to this as it seems to be working fine for
+     * the set of ops that are throttled.
+     **/
+    spec.token_gen_interval = 600000; /* In usec */
 
 #ifdef BR_RATE_LIMIT_SIGNER
 
-        double contribution = 0;
-        contribution = ((double)1 - ((double)child_count / (double)numbricks));
-        if (contribution == 0)
-                contribution = 1;
-        spec.rate = BR_HASH_CALC_READ_SIZE * contribution;
-        spec.maxlimit = BR_WORKERS * BR_HASH_CALC_READ_SIZE;
+    double contribution = 0;
+    contribution = ((double)1 - ((double)child_count / (double)numbricks));
+    if (contribution == 0)
+        contribution = 1;
+    spec.rate = BR_HASH_CALC_READ_SIZE * contribution;
+    spec.maxlimit = priv->signer_th_count * BR_HASH_CALC_READ_SIZE;
 
 #endif
 
-        if (!spec.rate)
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
-                        "[Rate Limit Info] \"FULL THROTTLE\"");
-        else
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
-                        "[Rate Limit Info] \"tokens/sec (rate): %lu, "
-                        "maxlimit: %lu\"", spec.rate, spec.maxlimit);
+    if (!spec.rate)
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
+                "FULL THROTTLE", NULL);
+    else
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
+                "tokens/sec-rate=%lu", spec.rate, "maxlimit=%lu", spec.maxlimit,
+                NULL);
 
-        priv->tbf = br_tbf_init (&spec, 1);
-        return priv->tbf ? 0 : -1;
+    priv->tbf = tbf_init(&spec, 1);
+    return priv->tbf ? 0 : -1;
 }
 
 static int32_t
-br_signer_handle_options (xlator_t *this, br_private_t *priv, dict_t *options)
+br_signer_handle_options(xlator_t *this, br_private_t *priv, dict_t *options)
 {
-        if (options)
-                GF_OPTION_RECONF ("expiry-time", priv->expiry_time,
-                                  options, uint32, error_return);
-        else
-                GF_OPTION_INIT ("expiry-time", priv->expiry_time,
-                                uint32, error_return);
-
-        return 0;
+    if (options) {
+        GF_OPTION_RECONF("expiry-time", priv->expiry_time, options, uint32,
+                         error_return);
+        GF_OPTION_RECONF("signer-threads", priv->signer_th_count, options,
+                         uint32, error_return);
+    } else {
+        GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return);
+        GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32,
+                       error_return);
+    }
+
+    return 0;
 
 error_return:
-        return -1;
+    return -1;
 }
 
 static int32_t
-br_signer_init (xlator_t *this, br_private_t *priv)
+br_signer_init(xlator_t *this, br_private_t *priv)
 {
-        int32_t ret = 0;
-        int numbricks = 0;
+    int32_t ret = 0;
+    int numbricks = 0;
 
-        GF_OPTION_INIT ("expiry-time", priv->expiry_time, uint32, error_return);
-        GF_OPTION_INIT ("brick-count", numbricks, int32, error_return);
+    GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return);
+    GF_OPTION_INIT("brick-count", numbricks, int32, error_return);
+    GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32,
+                   error_return);
 
-        ret = br_rate_limit_signer (this, priv->child_count, numbricks);
-        if (ret)
-                goto error_return;
+    ret = br_rate_limit_signer(this, priv->child_count, numbricks);
+    if (ret)
+        goto error_return;
 
-        ret = br_init_signer (this, priv);
-        if (ret)
-                goto cleanup_tbf;
+    ret = br_init_signer(this, priv);
+    if (ret)
+        goto cleanup_tbf;
 
-        return 0;
-
- cleanup_tbf:
-        /* cleanup TBF */
- error_return:
-        return -1;
+    return 0;
 
+cleanup_tbf:
+    /* cleanup TBF */
+error_return:
+    return -1;
 }
 
 static void
-br_free_scrubber_monitor (xlator_t *this, br_private_t *priv)
+br_free_scrubber_monitor(xlator_t *this, br_private_t *priv)
 {
-        struct br_monitor *scrub_monitor = &priv->scrub_monitor;
+    struct br_monitor *scrub_monitor = &priv->scrub_monitor;
 
-        if (scrub_monitor->timer) {
-                (void) gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer);
+    if (scrub_monitor->timer) {
+        (void)gf_tw_del_timer(priv->timer_wheel, scrub_monitor->timer);
 
-                GF_FREE (scrub_monitor->timer);
-                scrub_monitor->timer = NULL;
-        }
+        GF_FREE(scrub_monitor->timer);
+        scrub_monitor->timer = NULL;
+    }
 
-        (void) gf_thread_cleanup_xint (scrub_monitor->thread);
+    (void)gf_thread_cleanup_xint(scrub_monitor->thread);
 
-        /* Clean up cond and mutex variables */
-        pthread_mutex_destroy (&scrub_monitor->mutex);
-        pthread_cond_destroy (&scrub_monitor->cond);
+    /* Clean up cond and mutex variables */
+    pthread_mutex_destroy(&scrub_monitor->mutex);
+    pthread_cond_destroy(&scrub_monitor->cond);
 
-        pthread_mutex_destroy (&scrub_monitor->wakelock);
-        pthread_cond_destroy (&scrub_monitor->wakecond);
+    pthread_mutex_destroy(&scrub_monitor->wakelock);
+    pthread_cond_destroy(&scrub_monitor->wakecond);
 
-        pthread_mutex_destroy (&scrub_monitor->donelock);
-        pthread_cond_destroy (&scrub_monitor->donecond);
+    pthread_mutex_destroy(&scrub_monitor->donelock);
+    pthread_cond_destroy(&scrub_monitor->donecond);
 
-        LOCK_DESTROY (&scrub_monitor->lock);
+    LOCK_DESTROY(&scrub_monitor->lock);
 }
 
 static void
-br_free_children (xlator_t *this, br_private_t *priv, int count)
+br_free_children(xlator_t *this, br_private_t *priv, int count)
 {
-        br_child_t *child = NULL;
+    br_child_t *child = NULL;
 
-        for (--count; count >= 0; count--) {
-                child = &priv->children[count];
-                mem_pool_destroy (child->timer_pool);
-                pthread_mutex_destroy (&child->lock);
-        }
+    for (--count; count >= 0; count--) {
+        child = &priv->children[count];
+        mem_pool_destroy(child->timer_pool);
+        pthread_mutex_destroy(&child->lock);
+    }
 
-        GF_FREE (priv->children);
-        priv->children = NULL;
+    GF_FREE(priv->children);
+    priv->children = NULL;
 }
 
 static int
-br_init_children (xlator_t *this, br_private_t *priv)
+br_init_children(xlator_t *this, br_private_t *priv)
 {
-        int i = 0;
-        br_child_t *child = NULL;
-        xlator_list_t *trav = NULL;
-
-        priv->child_count = xlator_subvolume_count (this);
-        priv->children = GF_CALLOC (priv->child_count, sizeof (*priv->children),
-                                    gf_br_mt_br_child_t);
-        if (!priv->children)
-                goto err;
-
-        trav = this->children;
-        while (trav) {
-                child = &priv->children[i];
-
-                pthread_mutex_init (&child->lock, NULL);
-                child->witnessed = 0;
-
-                br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED);
-
-                child->this = this;
-                child->xl = trav->xlator;
-
-                child->timer_pool = mem_pool_new
-                                    (struct gf_tw_timer_list,  4096);
-                if (!child->timer_pool) {
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                ENOMEM, BRB_MSG_NO_MEMORY,
-                                "failed to allocate mem-pool for timer");
-                        errno = ENOMEM;
-                        goto freechild;
-                }
-
-                INIT_LIST_HEAD (&child->list);
-
-                i++;
-                trav = trav->next;
+    int i = 0;
+    br_child_t *child = NULL;
+    xlator_list_t *trav = NULL;
+
+    priv->child_count = xlator_subvolume_count(this);
+    priv->children = GF_CALLOC(priv->child_count, sizeof(*priv->children),
+                               gf_br_mt_br_child_t);
+    if (!priv->children)
+        goto err;
+
+    trav = this->children;
+    while (trav) {
+        child = &priv->children[i];
+
+        pthread_mutex_init(&child->lock, NULL);
+        child->witnessed = 0;
+
+        br_set_child_state(child, BR_CHILD_STATE_DISCONNECTED);
+
+        child->this = this;
+        child->xl = trav->xlator;
+
+        child->timer_pool = mem_pool_new(struct gf_tw_timer_list, 4096);
+        if (!child->timer_pool) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_MEM_POOL_ALLOC,
+                    NULL);
+            errno = ENOMEM;
+            goto freechild;
         }
 
-        return 0;
+        INIT_LIST_HEAD(&child->list);
+
+        i++;
+        trav = trav->next;
+    }
+
+    return 0;
 
- freechild:
-        br_free_children (this, priv, i);
- err:
-        return -1;
+freechild:
+    br_free_children(this, priv, i);
+err:
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int32_t       ret  = -1;
-	br_private_t *priv = NULL;
-
-	if (!this->children) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_NO_CHILD,
-                        "FATAL: no children");
-		goto out;
-	}
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_br_mt_br_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "failed to allocate memory (->priv)");
-                goto out;
-        }
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
 
-        GF_OPTION_INIT ("scrubber", priv->iamscrubber, bool, out);
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_NO_CHILD, NULL);
+        goto out;
+    }
 
-        ret = br_init_children (this, priv);
-        if (ret)
-                goto free_priv;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_br_mt_br_private_t);
+    if (!priv) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        pthread_mutex_init (&priv->lock, NULL);
-        pthread_cond_init (&priv->cond, NULL);
+    GF_OPTION_INIT("scrubber", priv->iamscrubber, bool, free_priv);
 
-        INIT_LIST_HEAD (&priv->bricks);
-        INIT_LIST_HEAD (&priv->signing);
+    ret = br_init_children(this, priv);
+    if (ret)
+        goto free_priv;
 
-        priv->timer_wheel = glusterfs_global_timer_wheel (this);
-        if (!priv->timer_wheel) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_TIMER_WHEEL_UNAVAILABLE,
-                        "global timer wheel unavailable");
-                goto cleanup;
-        }
+    pthread_mutex_init(&priv->lock, NULL);
+    pthread_cond_init(&priv->cond, NULL);
 
-	this->private = priv;
+    INIT_LIST_HEAD(&priv->bricks);
+    INIT_LIST_HEAD(&priv->signing);
 
-        if (!priv->iamscrubber) {
-                ret = br_signer_init (this, priv);
-                if (!ret)
-                        ret = br_signer_handle_options (this, priv, NULL);
-        } else {
-                ret = br_scrubber_init (this, priv);
-                if (!ret)
-                        ret = br_scrubber_handle_options (this, priv, NULL);
-        }
+    priv->timer_wheel = glusterfs_ctx_tw_get(this->ctx);
+    if (!priv->timer_wheel) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_TIMER_WHEEL_UNAVAILABLE,
+                NULL);
+        goto cleanup;
+    }
 
-        if (ret)
-                goto cleanup;
+    this->private = priv;
 
-        ret = gf_thread_create (&priv->thread, NULL, br_handle_events, this);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        BRB_MSG_SPAWN_FAILED, "thread creation failed");
-                ret = -1;
-        }
+    if (!priv->iamscrubber) {
+        ret = br_signer_init(this, priv);
+        if (!ret)
+            ret = br_signer_handle_options(this, priv, NULL);
+    } else {
+        ret = br_scrubber_init(this, priv);
+        if (!ret)
+            ret = br_scrubber_handle_options(this, priv, NULL);
+    }
 
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_BITROT_LOADED,
-                        "bit-rot xlator loaded in \"%s\" mode",
-                        (priv->iamscrubber) ? "SCRUBBER" : "SIGNER");
-                return 0;
-        }
+    if (ret)
+        goto cleanup;
+
+    ret = gf_thread_create(&priv->thread, NULL, br_handle_events, this,
+                           "brhevent");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_THREAD_CREATION_FAILED,
+                NULL);
+        ret = -1;
+    }
 
- cleanup:
-        (void) pthread_cond_destroy (&priv->cond);
-        (void) pthread_mutex_destroy (&priv->lock);
+    if (!ret) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_BITROT_LOADED, "mode=%s",
+                (priv->iamscrubber) ? "SCRUBBER" : "SIGNER", NULL);
+        return 0;
+    }
+
+cleanup:
+    (void)pthread_cond_destroy(&priv->cond);
+    (void)pthread_mutex_destroy(&priv->lock);
 
-        br_free_children (this, priv, priv->child_count);
+    br_free_children(this, priv, priv->child_count);
 
- free_priv:
-        GF_FREE (priv);
- out:
-        this->private = NULL;
-        return -1;
+free_priv:
+    GF_FREE(priv);
+out:
+    this->private = NULL;
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-	br_private_t *priv = this->private;
+    br_private_t *priv = this->private;
 
-        if (!priv)
-                return;
+    if (!priv)
+        return;
 
-        if (!priv->iamscrubber)
-                br_fini_signer (this, priv);
-        else
-                (void) br_free_scrubber_monitor (this, priv);
+    if (!priv->iamscrubber)
+        br_fini_signer(this, priv);
+    else
+        (void)br_free_scrubber_monitor(this, priv);
+
+    br_free_children(this, priv, priv->child_count);
 
-        br_free_children (this, priv, priv->child_count);
+    this->private = NULL;
+    GF_FREE(priv);
 
-        this->private = NULL;
-	GF_FREE (priv);
+    glusterfs_ctx_tw_put(this->ctx);
 
-	return;
+    return;
 }
 
 static void
-br_reconfigure_monitor (xlator_t *this)
+br_reconfigure_monitor(xlator_t *this)
 {
-        int32_t ret = 0;
-
-        ret = br_scrub_state_machine (this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
-                        "Could not reschedule scrubber for the volume. Scrubbing "
-                        "will continue according to old frequency.");
-        }
+    int32_t ret = 0;
+
+    ret = br_scrub_state_machine(this, _gf_false);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_COULD_NOT_SCHEDULE_SCRUB,
+                NULL);
+    }
 }
 
 static int
-br_reconfigure_scrubber (xlator_t *this, dict_t *options)
+br_reconfigure_scrubber(xlator_t *this, dict_t *options)
 {
-        int32_t       ret   = -1;
-        br_private_t *priv  = NULL;
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        pthread_mutex_lock (&priv->lock);
-        {
-                ret = br_scrubber_handle_options (this, priv, options);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    pthread_mutex_lock(&priv->lock);
+    {
+        ret = br_scrubber_handle_options(this, priv, options);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
-        if (ret)
-                goto err;
+    if (ret)
+        goto err;
 
-        /* change state for all _up_ subvolume(s) */
-        pthread_mutex_lock (&priv->lock);
-        {
-                br_reconfigure_monitor (this);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    /* change state for all _up_ subvolume(s) */
+    pthread_mutex_lock(&priv->lock);
+    {
+        br_reconfigure_monitor(this);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
- err:
-        return ret;
+err:
+    return ret;
 }
 
 static int
-br_reconfigure_signer (xlator_t *this, dict_t *options)
+br_reconfigure_signer(xlator_t *this, dict_t *options)
 {
-        br_private_t *priv = this->private;
+    br_private_t *priv = this->private;
 
-        return br_signer_handle_options (this, priv, options);
+    return br_signer_handle_options(this, priv, options);
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int ret = 0;
-        br_private_t *priv = NULL;
+    int ret = 0;
+    br_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->iamscrubber)
-                ret = br_reconfigure_scrubber (this, options);
-        else
-                ret = br_reconfigure_signer (this, options);
+    if (priv->iamscrubber)
+        ret = br_reconfigure_scrubber(this, options);
+    else
+        ret = br_reconfigure_signer(this, options);
 
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops;
@@ -2092,38 +2155,78 @@ struct xlator_fops fops;
 struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-        { .key = {"expiry-time"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = SIGNING_TIMEOUT,
-          .description = "Waiting time for an object on which it waits "
-                         "before it is signed",
-        },
-        { .key = {"brick-count"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Total number of bricks for the current node for "
-                         "all volumes in the trusted storage pool.",
-        },
-        { .key = {"scrubber"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false",
-          .description = "option to run as a scrubber",
-        },
-        { .key = {"scrub-throttle"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Scrub-throttle value is a measure of how fast "
-                         "or slow the scrubber scrubs the filesystem for "
-                         "volume <VOLNAME>",
-        },
-        { .key = {"scrub-freq"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "biweekly",
-          .description = "Scrub frequency for volume <VOLNAME>",
-        },
-        { .key = {"scrub-state"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "active",
-          .description = "Pause/Resume scrub. Upon resume, scrubber "
-                         "continues from where it left off.",
-        },
-	{ .key  = {NULL} },
+    {
+        .key = {"expiry-time"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = SIGNING_TIMEOUT,
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Waiting time for an object on which it waits "
+                       "before it is signed",
+    },
+    {
+        .key = {"brick-count"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Total number of bricks for the current node for "
+                       "all volumes in the trusted storage pool.",
+    },
+    {
+        .key = {"scrubber", "scrub"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+        .description = "option to run as a scrubber",
+    },
+    {
+        .key = {"scrub-throttle"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "lazy",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Scrub-throttle value is a measure of how fast "
+                       "or slow the scrubber scrubs the filesystem for "
+                       "volume <VOLNAME>",
+    },
+    {
+        .key = {"scrub-freq"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "biweekly",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Scrub frequency for volume <VOLNAME>",
+    },
+    {
+        .key = {"scrub-state"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "active",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Pause/Resume scrub. Upon resume, scrubber "
+                       "continues from where it left off.",
+    },
+    {
+        .key = {"signer-threads"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = BR_WORKERS,
+        .op_version = {GD_OP_VERSION_8_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Number of signing process threads. As a best "
+                       "practice, set this to the number of processor cores",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "bit-rot",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index 4153d0e4724..8ac7dcdac3d 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -11,17 +11,17 @@
 #ifndef __BIT_ROT_H__
 #define __BIT_ROT_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 #include "changelog.h"
 #include "timer-wheel.h"
 
-#include "bit-rot-tbf.h"
+#include <glusterfs/throttle-tbf.h>
 #include "bit-rot-ssm.h"
 
 #include "bit-rot-common.h"
@@ -30,277 +30,273 @@
 
 #include <openssl/sha.h>
 
-/**
- * TODO: make this configurable. As a best practice, set this to the
- * number of processor cores.
- */
-#define BR_WORKERS 4
-
 typedef enum scrub_throttle {
-        BR_SCRUB_THROTTLE_VOID       = -1,
-        BR_SCRUB_THROTTLE_LAZY       = 0,
-        BR_SCRUB_THROTTLE_NORMAL     = 1,
-        BR_SCRUB_THROTTLE_AGGRESSIVE = 2,
-        BR_SCRUB_THROTTLE_STALLED    = 3,
+    BR_SCRUB_THROTTLE_VOID = -1,
+    BR_SCRUB_THROTTLE_LAZY = 0,
+    BR_SCRUB_THROTTLE_NORMAL = 1,
+    BR_SCRUB_THROTTLE_AGGRESSIVE = 2,
+    BR_SCRUB_THROTTLE_STALLED = 3,
 } scrub_throttle_t;
 
 typedef enum scrub_freq {
-        BR_FSSCRUB_FREQ_HOURLY = 1,
-        BR_FSSCRUB_FREQ_DAILY,
-        BR_FSSCRUB_FREQ_WEEKLY,
-        BR_FSSCRUB_FREQ_BIWEEKLY,
-        BR_FSSCRUB_FREQ_MONTHLY,
-        BR_FSSCRUB_FREQ_STALLED,
+    BR_FSSCRUB_FREQ_HOURLY = 1,
+    BR_FSSCRUB_FREQ_DAILY,
+    BR_FSSCRUB_FREQ_WEEKLY,
+    BR_FSSCRUB_FREQ_BIWEEKLY,
+    BR_FSSCRUB_FREQ_MONTHLY,
+    BR_FSSCRUB_FREQ_MINUTE,
+    BR_FSSCRUB_FREQ_STALLED,
 } scrub_freq_t;
 
-#define signature_size(hl) (sizeof (br_isignature_t) + hl + 1)
+#define signature_size(hl) (sizeof(br_isignature_t) + hl + 1)
 
 struct br_scanfs {
-        gf_lock_t entrylock;
+    gf_lock_t entrylock;
 
-        pthread_mutex_t waitlock;
-        pthread_cond_t  waitcond;
+    pthread_mutex_t waitlock;
+    pthread_cond_t waitcond;
 
-        unsigned int     entries;
-        struct list_head queued;
-        struct list_head ready;
+    unsigned int entries;
+    struct list_head queued;
+    struct list_head ready;
 };
 
 /* just need three states to track child status */
 typedef enum br_child_state {
-        BR_CHILD_STATE_CONNECTED = 1,
-        BR_CHILD_STATE_INITIALIZING,
-        BR_CHILD_STATE_CONNFAILED,
-        BR_CHILD_STATE_DISCONNECTED,
+    BR_CHILD_STATE_CONNECTED = 1,
+    BR_CHILD_STATE_INITIALIZING,
+    BR_CHILD_STATE_CONNFAILED,
+    BR_CHILD_STATE_DISCONNECTED,
 } br_child_state_t;
 
 struct br_child {
-        pthread_mutex_t lock;         /* protects child state */
-        char witnessed;               /* witnessed at least one succesfull
-                                         connection */
-        br_child_state_t c_state;     /* current state of this child */
+    pthread_mutex_t lock;     /* protects child state */
+    char witnessed;           /* witnessed at least one successful
+                                 connection */
+    br_child_state_t c_state; /* current state of this child */
 
-        char child_up;                /* Indicates whether this child is
-                                         up or not */
-        xlator_t *xl;                 /* client xlator corresponding to
-                                         this child */
-        inode_table_t *table;         /* inode table for this child */
-        char brick_path[PATH_MAX];    /* brick export directory of this
-                                         child */
-        struct list_head list;        /* hook to attach to the list of
-                                         UP children */
-        xlator_t *this;               /* Bit rot xlator */
+    char child_up;             /* Indicates whether this child is
+                                  up or not */
+    xlator_t *xl;              /* client xlator corresponding to
+                                  this child */
+    inode_table_t *table;      /* inode table for this child */
+    char brick_path[PATH_MAX]; /* brick export directory of this
+                                  child */
+    struct list_head list;     /* hook to attach to the list of
+                                  UP children */
+    xlator_t *this;            /* Bit rot xlator */
 
-        pthread_t thread;             /* initial crawler for unsigned
-                                         object(s) or scrub crawler */
-        int threadrunning;            /* active thread */
+    pthread_t thread;  /* initial crawler for unsigned
+                          object(s) or scrub crawler */
+    int threadrunning; /* active thread */
 
-        struct mem_pool *timer_pool;  /* timer-wheel's timer mem-pool */
+    struct mem_pool *timer_pool; /* timer-wheel's timer mem-pool */
 
-        struct timeval tv;
+    struct timeval tv;
 
-        struct br_scanfs fsscan;      /* per subvolume FS scanner */
+    struct br_scanfs fsscan; /* per subvolume FS scanner */
 
-        gf_boolean_t active_scrubbing; /* Actively scrubbing or not */
+    gf_boolean_t active_scrubbing; /* Actively scrubbing or not */
 };
 
 typedef struct br_child br_child_t;
 
 struct br_obj_n_workers {
-        struct list_head objects;         /* queue of objects expired from the
-                                             timer wheel and ready to be picked
-                                             up for signing */
-        pthread_t workers[BR_WORKERS];    /* Threads which pick up the objects
-                                             from the above queue and start
-                                             signing each object */
+    struct list_head objects; /* queue of objects expired from the
+                                 timer wheel and ready to be picked
+                                 up for signing */
+    pthread_t *workers;       /* Threads which pick up the objects
+                                 from the above queue and start
+                                 signing each object */
 };
 
 struct br_scrubber {
-        xlator_t *this;
+    xlator_t *this;
 
-        scrub_throttle_t throttle;
+    scrub_throttle_t throttle;
 
-        /**
-         * frequency of scanning for this subvolume. this should
-         * normally be per-child, but since all childs follow the
-         * same frequency for a volume, this option ends up here
-         * instead of br_child_t.
-         */
-        scrub_freq_t frequency;
+    /**
+     * frequency of scanning for this subvolume. this should
+     * normally be per-child, but since all children follow the
+     * same frequency for a volume, this option ends up here
+     * instead of br_child_t.
+     */
+    scrub_freq_t frequency;
 
-        gf_boolean_t frequency_reconf;
-        gf_boolean_t throttle_reconf;
+    gf_boolean_t frequency_reconf;
+    gf_boolean_t throttle_reconf;
 
-        pthread_mutex_t mutex;
-        pthread_cond_t  cond;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
 
-        unsigned int nr_scrubbers;
-        struct list_head scrubbers;
+    unsigned int nr_scrubbers;
+    struct list_head scrubbers;
 
-        /**
-         * list of "rotatable" subvolume(s) undergoing scrubbing
-         */
-        struct list_head scrublist;
+    /**
+     * list of "rotatable" subvolume(s) undergoing scrubbing
+     */
+    struct list_head scrublist;
 };
 
 struct br_monitor {
-        gf_lock_t lock;
-        pthread_t thread;         /* Monitor thread */
-
-        gf_boolean_t  inited;
-        pthread_mutex_t mutex;
-        pthread_cond_t cond;      /* Thread starts and will be waiting on cond.
-                                     First child which is up wakes this up */
-
-        xlator_t *this;
-        /* scheduler */
-        uint32_t boot;
-
-        int32_t active_child_count; /* Number of children currently scrubbing */
-        gf_boolean_t kick;          /* This variable tracks the scrubber is
-                                     * kicked or not. Both 'kick' and
-                                     * 'active_child_count' uses the same pair
-                                     * of mutex-cond variable, i.e, wakelock and
-                                     * wakecond. */
-
-        pthread_mutex_t wakelock;
-        pthread_cond_t  wakecond;
-
-        gf_boolean_t done;
-        pthread_mutex_t donelock;
-        pthread_cond_t  donecond;
-
-        struct gf_tw_timer_list *timer;
-        br_scrub_state_t state;   /* current scrub state */
+    gf_lock_t lock;
+    pthread_t thread; /* Monitor thread */
+
+    gf_boolean_t inited;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond; /* Thread starts and will be waiting on cond.
+                            First child which is up wakes this up */
+
+    xlator_t *this;
+    /* scheduler */
+    uint32_t boot;
+
+    int32_t active_child_count; /* Number of children currently scrubbing */
+    gf_boolean_t kick;          /* This variable tracks the scrubber is
+                                 * kicked or not. Both 'kick' and
+                                 * 'active_child_count' uses the same pair
+                                 * of mutex-cond variable, i.e, wakelock and
+                                 * wakecond. */
+
+    pthread_mutex_t wakelock;
+    pthread_cond_t wakecond;
+
+    gf_boolean_t done;
+    pthread_mutex_t donelock;
+    pthread_cond_t donecond;
+
+    struct gf_tw_timer_list *timer;
+    br_scrub_state_t state; /* current scrub state */
 };
 
 typedef struct br_obj_n_workers br_obj_n_workers_t;
 
 typedef struct br_private br_private_t;
 
-typedef void (*br_scrubbed_file_update) (br_private_t *priv);
+typedef void (*br_scrubbed_file_update)(br_private_t *priv);
 
 struct br_private {
-        pthread_mutex_t lock;
+    pthread_mutex_t lock;
+
+    struct list_head bricks; /* list of bricks from which enents
+                                have been received */
 
-        struct list_head bricks;          /* list of bricks from which enents
-                                             have been received */
+    struct list_head signing;
 
-        struct list_head signing;
+    pthread_cond_t object_cond; /* handling signing of objects */
+    int child_count;
+    br_child_t *children; /* list of subvolumes */
+    int up_children;
 
-        pthread_cond_t object_cond;       /* handling signing of objects */
-        int child_count;
-        br_child_t *children;             /* list of subvolumes */
-        int up_children;
+    pthread_cond_t cond; /* handling CHILD_UP notifications */
+    pthread_t thread;    /* thread for connecting each UP
+                            child with changelog */
 
-        pthread_cond_t cond;              /* handling CHILD_UP notifications */
-        pthread_t thread;                 /* thread for connecting each UP
-                                             child with changelog */
+    struct tvec_base *timer_wheel; /* timer wheel where the objects which
+                                      changelog has sent sits and waits
+                                      for expiry */
+    br_obj_n_workers_t *obj_queue; /* place holder for all the objects
+                                      that are expired from timer wheel
+                                      and ready to be picked up for
+                                      signing and the workers which sign
+                                      the objects */
 
-        struct tvec_base *timer_wheel;    /* timer wheel where the objects which
-                                             changelog has sent sits and waits
-                                             for expiry */
-        br_obj_n_workers_t *obj_queue;    /* place holder for all the objects
-                                             that are expired from timer wheel
-                                             and ready to be picked up for
-                                             signing and the workers which sign
-                                             the objects */
+    uint32_t expiry_time; /* objects "wait" time */
 
-        uint32_t expiry_time;              /* objects "wait" time */
+    uint32_t signer_th_count; /* Number of signing process threads */
 
-        br_tbf_t *tbf;                    /* token bucket filter */
+    tbf_t *tbf; /* token bucket filter */
 
-        gf_boolean_t iamscrubber;         /* function as a fs scrubber */
+    gf_boolean_t iamscrubber; /* function as a fs scrubber */
 
-        struct br_scrub_stats scrub_stat; /* statistics of scrub*/
+    struct br_scrub_stats scrub_stat; /* statistics of scrub*/
 
-        struct br_scrubber fsscrub;       /* scrubbers for this subvolume */
+    struct br_scrubber fsscrub; /* scrubbers for this subvolume */
 
-        struct br_monitor scrub_monitor;  /* scrubber monitor */
+    struct br_monitor scrub_monitor; /* scrubber monitor */
 };
 
 struct br_object {
-        xlator_t *this;
+    xlator_t *this;
 
-        uuid_t gfid;
+    uuid_t gfid;
 
-        unsigned long signedversion;    /* version aginst which this object will
-                                           be signed */
-        br_child_t *child;              /* object's subvolume */
+    unsigned long signedversion; /* version against which this object will
+                                    be signed */
+    br_child_t *child;           /* object's subvolume */
 
-        int sign_info;
+    int sign_info;
 
-        struct list_head list;          /* hook to add to the queue once the
-                                           object is expired from timer wheel */
-        void *data;
+    struct list_head list; /* hook to add to the queue once the
+                              object is expired from timer wheel */
+    void *data;
 };
 
 typedef struct br_object br_object_t;
-typedef int32_t (br_scrub_ssm_call) (xlator_t *);
+typedef int32_t(br_scrub_ssm_call)(xlator_t *);
 
 void
-br_log_object (xlator_t *, char *, uuid_t, int32_t);
+br_log_object(xlator_t *, char *, uuid_t, int32_t);
 
 void
-br_log_object_path (xlator_t *, char *, const char *, int32_t);
+br_log_object_path(xlator_t *, char *, const char *, int32_t);
 
 int32_t
-br_calculate_obj_checksum (unsigned char *,
-                           br_child_t *, fd_t *, struct iatt *);
+br_calculate_obj_checksum(unsigned char *, br_child_t *, fd_t *, struct iatt *);
 
 int32_t
-br_prepare_loc (xlator_t *, br_child_t *, loc_t *, gf_dirent_t *, loc_t *);
+br_prepare_loc(xlator_t *, br_child_t *, loc_t *, gf_dirent_t *, loc_t *);
 
 gf_boolean_t
-bitd_is_bad_file (xlator_t *, br_child_t *, loc_t *, fd_t *);
+bitd_is_bad_file(xlator_t *, br_child_t *, loc_t *, fd_t *);
 
 static inline void
-_br_set_child_state (br_child_t *child, br_child_state_t state)
+_br_set_child_state(br_child_t *child, br_child_state_t state)
 {
-        child->c_state = state;
+    child->c_state = state;
 }
 
 static inline int
-_br_is_child_connected (br_child_t *child)
+_br_is_child_connected(br_child_t *child)
 {
-        return (child->c_state == BR_CHILD_STATE_CONNECTED);
+    return (child->c_state == BR_CHILD_STATE_CONNECTED);
 }
 
 static inline int
-_br_is_child_scrub_active (br_child_t *child)
+_br_is_child_scrub_active(br_child_t *child)
 {
-        return child->active_scrubbing;
+    return child->active_scrubbing;
 }
 
 static inline int
-_br_child_failed_conn (br_child_t *child)
+_br_child_failed_conn(br_child_t *child)
 {
-        return (child->c_state == BR_CHILD_STATE_CONNFAILED);
+    return (child->c_state == BR_CHILD_STATE_CONNFAILED);
 }
 
 static inline int
-_br_child_witnessed_connection (br_child_t *child)
+_br_child_witnessed_connection(br_child_t *child)
 {
-        return (child->witnessed == 1);
+    return (child->witnessed == 1);
 }
 
 /* scrub state */
 static inline void
-_br_monitor_set_scrub_state (struct br_monitor *scrub_monitor,
-                           br_scrub_state_t state)
+_br_monitor_set_scrub_state(struct br_monitor *scrub_monitor,
+                            br_scrub_state_t state)
 {
-        scrub_monitor->state = state;
+    scrub_monitor->state = state;
 }
 
 static inline br_scrub_event_t
-_br_child_get_scrub_event (struct br_scrubber *fsscrub)
+_br_child_get_scrub_event(struct br_scrubber *fsscrub)
 {
-        return (fsscrub->frequency == BR_FSSCRUB_FREQ_STALLED)
-                ? BR_SCRUB_EVENT_PAUSE : BR_SCRUB_EVENT_SCHEDULE;
+    return (fsscrub->frequency == BR_FSSCRUB_FREQ_STALLED)
+               ? BR_SCRUB_EVENT_PAUSE
+               : BR_SCRUB_EVENT_SCHEDULE;
 }
 
 int32_t
-br_get_bad_objects_list (xlator_t *this, dict_t **dict);
-
+br_get_bad_objects_list(xlator_t *this, dict_t **dict);
 
 #endif /* __BIT_ROT_H__ */
diff --git a/xlators/features/bit-rot/src/stub/Makefile.am b/xlators/features/bit-rot/src/stub/Makefile.am
index 7e4b6837eec..f13de7145fc 100644
--- a/xlators/features/bit-rot/src/stub/Makefile.am
+++ b/xlators/features/bit-rot/src/stub/Makefile.am
@@ -1,4 +1,6 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = bitrot-stub.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 bitrot_stub_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
@@ -7,10 +9,11 @@ bitrot_stub_la_SOURCES = bit-rot-stub-helpers.c bit-rot-stub.c
 bitrot_stub_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = bit-rot-stub.h bit-rot-common.h bit-rot-stub-mem-types.h \
-                 bit-rot-object-version.h bit-rot-stub-messages.h
+	bit-rot-object-version.h bit-rot-stub-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	      -I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-common.h b/xlators/features/bit-rot/src/stub/bit-rot-common.h
index 2afc9f47c29..20561aa7764 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-common.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-common.h
@@ -11,74 +11,74 @@
 #ifndef __BIT_ROT_COMMON_H__
 #define __BIT_ROT_COMMON_H__
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "bit-rot-object-version.h"
 
-#define BR_VXATTR_VERSION   (1 << 0)
+#define BR_VXATTR_VERSION (1 << 0)
 #define BR_VXATTR_SIGNATURE (1 << 1)
 
 #define BR_VXATTR_SIGN_MISSING (BR_VXATTR_SIGNATURE)
-#define BR_VXATTR_ALL_MISSING                           \
-        (BR_VXATTR_VERSION | BR_VXATTR_SIGNATURE)
+#define BR_VXATTR_ALL_MISSING (BR_VXATTR_VERSION | BR_VXATTR_SIGNATURE)
 
-#define BR_BAD_OBJ_CONTAINER (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8}
+#define BR_BAD_OBJ_CONTAINER                                                   \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8 }
 
 typedef enum br_vxattr_state {
-        BR_VXATTR_STATUS_FULL     = 0,
-        BR_VXATTR_STATUS_MISSING  = 1,
-        BR_VXATTR_STATUS_UNSIGNED = 2,
-        BR_VXATTR_STATUS_INVALID  = 3,
+    BR_VXATTR_STATUS_FULL = 0,
+    BR_VXATTR_STATUS_MISSING = 1,
+    BR_VXATTR_STATUS_UNSIGNED = 2,
+    BR_VXATTR_STATUS_INVALID = 3,
 } br_vxattr_status_t;
 
 typedef enum br_sign_state {
-        BR_SIGN_INVALID     = -1,
-        BR_SIGN_NORMAL      = 0,
-        BR_SIGN_REOPEN_WAIT = 1,
-        BR_SIGN_QUICK       = 2,
+    BR_SIGN_INVALID = -1,
+    BR_SIGN_NORMAL = 0,
+    BR_SIGN_REOPEN_WAIT = 1,
+    BR_SIGN_QUICK = 2,
 } br_sign_state_t;
 
 static inline br_vxattr_status_t
-br_version_xattr_state (dict_t *xattr, br_version_t **obuf,
-                        br_signature_t **sbuf, gf_boolean_t *objbad)
+br_version_xattr_state(dict_t *xattr, br_version_t **obuf,
+                       br_signature_t **sbuf, gf_boolean_t *objbad)
 {
-        int32_t             ret    = 0;
-        int32_t             vxattr = 0;
-        br_vxattr_status_t  status;
-        void               *data   = NULL;
-
-        /**
-         * The key being present in the dict indicates the xattr was set on
-         * disk. The presence of xattr itself as of now is suffecient to say
-         * the the object is bad.
-         */
-        *objbad = _gf_false;
-        ret = dict_get_bin (xattr, BITROT_OBJECT_BAD_KEY, (void **)&data);
-        if (!ret)
-                *objbad = _gf_true;
-
-        ret = dict_get_bin (xattr, BITROT_CURRENT_VERSION_KEY, (void **)obuf);
-        if (ret)
-                vxattr |= BR_VXATTR_VERSION;
-
-        ret = dict_get_bin (xattr, BITROT_SIGNING_VERSION_KEY, (void **)sbuf);
-        if (ret)
-                vxattr |= BR_VXATTR_SIGNATURE;
-
-        switch (vxattr) {
+    int32_t ret = 0;
+    int32_t vxattr = 0;
+    br_vxattr_status_t status;
+    void *data = NULL;
+
+    /**
+     * The key being present in the dict indicates the xattr was set on
+     * disk. The presence of xattr itself as of now is suffecient to say
+     * the the object is bad.
+     */
+    *objbad = _gf_false;
+    ret = dict_get_bin(xattr, BITROT_OBJECT_BAD_KEY, (void **)&data);
+    if (!ret)
+        *objbad = _gf_true;
+
+    ret = dict_get_bin(xattr, BITROT_CURRENT_VERSION_KEY, (void **)obuf);
+    if (ret)
+        vxattr |= BR_VXATTR_VERSION;
+
+    ret = dict_get_bin(xattr, BITROT_SIGNING_VERSION_KEY, (void **)sbuf);
+    if (ret)
+        vxattr |= BR_VXATTR_SIGNATURE;
+
+    switch (vxattr) {
         case 0:
-                status = BR_VXATTR_STATUS_FULL;
-                break;
+            status = BR_VXATTR_STATUS_FULL;
+            break;
         case BR_VXATTR_SIGN_MISSING:
-                status = BR_VXATTR_STATUS_UNSIGNED;
-                break;
+            status = BR_VXATTR_STATUS_UNSIGNED;
+            break;
         case BR_VXATTR_ALL_MISSING:
-                status = BR_VXATTR_STATUS_MISSING;
-                break;
+            status = BR_VXATTR_STATUS_MISSING;
+            break;
         default:
-                status = BR_VXATTR_STATUS_INVALID;
-        }
+            status = BR_VXATTR_STATUS_INVALID;
+    }
 
-        return status;
+    return status;
 }
 
 /**
@@ -86,13 +86,13 @@ br_version_xattr_state (dict_t *xattr, br_version_t **obuf,
  * signing.
  */
 typedef struct br_isignature_in {
-        int8_t signaturetype;            /* signature type            */
+    int8_t signaturetype; /* signature type            */
 
-        unsigned long signedversion;     /* version against which the
-                                            object was signed         */
+    unsigned long signedversion; /* version against which the
+                                    object was signed         */
 
-        size_t signaturelen;             /* signature length          */
-        char signature[0];               /* object signature          */
+    size_t signaturelen; /* signature length          */
+    char signature[0];   /* object signature          */
 } br_isignature_t;
 
 /**
@@ -100,80 +100,79 @@ typedef struct br_isignature_in {
  * verification.
  */
 typedef struct br_isignature_out {
-        char stale;                      /* stale signature?          */
+    char stale; /* stale signature?          */
 
-        unsigned long version;           /* current signed version    */
+    unsigned long version; /* current signed version    */
 
-        uint32_t time[2];                /* time when the object
-                                            got dirtied               */
+    uint32_t time[2]; /* time when the object
+                         got dirtied               */
 
-        int8_t signaturetype;            /* hash type                 */
-        size_t signaturelen;             /* signature length          */
-        char   signature[0];             /* signature (hash)          */
+    int8_t signaturetype; /* hash type                 */
+    size_t signaturelen;  /* signature length          */
+    char signature[0];    /* signature (hash)          */
 } br_isignature_out_t;
 
 typedef struct br_stub_init {
-        uint32_t timebuf[2];
-        char export[PATH_MAX];
+    uint32_t timebuf[2];
+    char export[PATH_MAX];
 } br_stub_init_t;
 
 typedef enum {
-        BR_SIGNATURE_TYPE_VOID   = -1,   /* object is not signed       */
-        BR_SIGNATURE_TYPE_ZERO   = 0,    /* min boundary               */
-        BR_SIGNATURE_TYPE_SHA256 = 1,    /* signed with SHA256         */
-        BR_SIGNATURE_TYPE_MAX    = 2,    /* max boundary               */
+    BR_SIGNATURE_TYPE_VOID = -1,  /* object is not signed       */
+    BR_SIGNATURE_TYPE_ZERO = 0,   /* min boundary               */
+    BR_SIGNATURE_TYPE_SHA256 = 1, /* signed with SHA256         */
+    BR_SIGNATURE_TYPE_MAX = 2,    /* max boundary               */
 } br_signature_type;
 
 /* BitRot stub start time (virtual xattr) */
-#define GLUSTERFS_GET_BR_STUB_INIT_TIME  "trusted.glusterfs.bit-rot.stub-init"
+#define GLUSTERFS_GET_BR_STUB_INIT_TIME "trusted.glusterfs.bit-rot.stub-init"
 
 /* signing/reopen hint */
 #define BR_OBJECT_RESIGN 0
-#define BR_OBJECT_REOPEN  1
-#define BR_REOPEN_SIGN_HINT_KEY  "trusted.glusterfs.bit-rot.reopen-hint"
+#define BR_OBJECT_REOPEN 1
+#define BR_REOPEN_SIGN_HINT_KEY "trusted.glusterfs.bit-rot.reopen-hint"
 
 static inline int
-br_is_signature_type_valid (int8_t signaturetype)
+br_is_signature_type_valid(int8_t signaturetype)
 {
-        return ((signaturetype > BR_SIGNATURE_TYPE_ZERO)
-                 && (signaturetype < BR_SIGNATURE_TYPE_MAX));
+    return ((signaturetype > BR_SIGNATURE_TYPE_ZERO) &&
+            (signaturetype < BR_SIGNATURE_TYPE_MAX));
 }
 
 static inline void
-br_set_default_ongoingversion (br_version_t *buf, uint32_t *tv)
+br_set_default_ongoingversion(br_version_t *buf, uint32_t *tv)
 {
-        buf->ongoingversion = BITROT_DEFAULT_CURRENT_VERSION;
-        buf->timebuf[0] = tv[0];
-        buf->timebuf[1] = tv[1];
+    buf->ongoingversion = BITROT_DEFAULT_CURRENT_VERSION;
+    buf->timebuf[0] = tv[0];
+    buf->timebuf[1] = tv[1];
 }
 
 static inline void
-br_set_default_signature (br_signature_t *buf, size_t *size)
+br_set_default_signature(br_signature_t *buf, size_t *size)
 {
-        buf->signaturetype = (int8_t) BR_SIGNATURE_TYPE_VOID;
-        buf->signedversion = BITROT_DEFAULT_SIGNING_VERSION;
+    buf->signaturetype = (int8_t)BR_SIGNATURE_TYPE_VOID;
+    buf->signedversion = BITROT_DEFAULT_SIGNING_VERSION;
 
-        *size = sizeof (br_signature_t); /* no signature */
+    *size = sizeof(br_signature_t); /* no signature */
 }
 
 static inline void
-br_set_ongoingversion (br_version_t *buf,
-                       unsigned long version, uint32_t *tv)
+br_set_ongoingversion(br_version_t *buf, unsigned long version, uint32_t *tv)
 {
-        buf->ongoingversion = version;
-        buf->timebuf[0] = tv[0];
-        buf->timebuf[1] = tv[1];
+    buf->ongoingversion = version;
+    buf->timebuf[0] = tv[0];
+    buf->timebuf[1] = tv[1];
 }
 
 static inline void
-br_set_signature (br_signature_t *buf,
-                  br_isignature_t *sign, size_t signaturelen, size_t *size)
+br_set_signature(br_signature_t *buf, br_isignature_t *sign,
+                 size_t signaturelen, size_t *size)
 {
-        buf->signaturetype  = sign->signaturetype;
-        buf->signedversion  = ntohl (sign->signedversion);
+    buf->signaturetype = sign->signaturetype;
+    buf->signedversion = ntohl(sign->signedversion);
 
-        memcpy (buf->signature, sign->signature, signaturelen);
-        *size = sizeof (br_signature_t) + signaturelen;
+    memcpy(buf->signature, sign->signature, signaturelen);
+    *size = sizeof(br_signature_t) + signaturelen;
 }
 
 #endif /* __BIT_ROT_COMMON_H__ */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-object-version.h b/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
index 1f2497aebe9..7ae6a5200df 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
@@ -15,16 +15,16 @@
  * on-disk formats for ongoing version and object signature.
  */
 typedef struct br_version {
-        unsigned long ongoingversion;
-        uint32_t timebuf[2];
+    unsigned long ongoingversion;
+    uint32_t timebuf[2];
 } br_version_t;
 
-typedef struct __attribute__ ((__packed__)) br_signature {
-        int8_t signaturetype;
+typedef struct __attribute__((__packed__)) br_signature {
+    int8_t signaturetype;
 
-        unsigned long signedversion;
+    unsigned long signedversion;
 
-        char signature[0];
+    char signature[0];
 } br_signature_t;
 
 #endif
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c b/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
index 7bd270d1513..8ac13a09941 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
@@ -11,140 +11,88 @@
 #include "bit-rot-stub.h"
 
 br_stub_fd_t *
-br_stub_fd_new (void)
+br_stub_fd_new(void)
 {
-        br_stub_fd_t    *br_stub_fd = NULL;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        br_stub_fd = GF_CALLOC (1, sizeof (*br_stub_fd),
-                                gf_br_stub_mt_br_stub_fd_t);
+    br_stub_fd = GF_CALLOC(1, sizeof(*br_stub_fd), gf_br_stub_mt_br_stub_fd_t);
 
-        return br_stub_fd;
+    return br_stub_fd;
 }
 
 int
-__br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
+__br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, br_stub_fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, br_stub_fd, out);
 
-        value = (uint64_t)(long) br_stub_fd;
+    value = (uint64_t)(long)br_stub_fd;
 
-        ret = __fd_ctx_set (fd, this, value);
+    ret = __fd_ctx_set(fd, this, value);
 
 out:
-        return ret;
+    return ret;
 }
 
 br_stub_fd_t *
-__br_stub_fd_ctx_get (xlator_t *this, fd_t *fd)
+__br_stub_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        br_stub_fd_t *br_stub_fd = NULL;
-        uint64_t  value  = 0;
-        int       ret    = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = __fd_ctx_get (fd, this, &value);
-        if (ret)
-                return NULL;
+    ret = __fd_ctx_get(fd, this, &value);
+    if (ret)
+        return NULL;
 
-        br_stub_fd = (br_stub_fd_t *) ((long) value);
+    br_stub_fd = (br_stub_fd_t *)((long)value);
 
 out:
-        return br_stub_fd;
+    return br_stub_fd;
 }
 
 br_stub_fd_t *
-br_stub_fd_ctx_get (xlator_t *this, fd_t *fd)
+br_stub_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        br_stub_fd_t *br_stub_fd = NULL;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                br_stub_fd = __br_stub_fd_ctx_get (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        br_stub_fd = __br_stub_fd_ctx_get(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return br_stub_fd;
+    return br_stub_fd;
 }
 
 int32_t
-br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
+br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, br_stub_fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, br_stub_fd, out);
 
-        LOCK (&fd->lock);
-        {
-                ret = __br_stub_fd_ctx_set (this, fd, br_stub_fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __br_stub_fd_ctx_set(this, fd, br_stub_fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return ret;
-}
-
-
-/**
- * prints the path to the bad object's entry into the buffer provided.
- * @priv: xlator private
- * @filename: gfid of the bad object.
- * @file_path: buffer provided into which path of the bad object is printed
- *            using above 2 arguments.
- */
-static void
-br_stub_link_path (br_stub_private_t *priv, const char *filename,
-                        char *file_path, size_t len)
-{
-        snprintf (file_path, len, "%s/%s", priv->stub_basepath, filename);
-}
-
-/**
- * Prints the path of the object which acts as a container for all the bad
- * objects. Each new entry corresponding to a bad object is a hard link to
- * the object with name "stub-0000000000000008".
- * @priv: xlator's private
- * @stub_gfid_path: buffer into which the path to the container of bad objects
- *                  is printed.
- */
-static void
-br_stub_container_entry (br_stub_private_t *priv, char *stub_gfid_path,
-                         size_t len)
-{
-
-        snprintf (stub_gfid_path, len, "%s/stub-%s", priv->stub_basepath,
-                  uuid_utoa (priv->bad_object_dir_gfid));
-}
-
-/**
- * Prints the path to the bad object's entry into the buffer provided.
- * @priv: xlator private
- * @gfid: gfid of the bad object.
- * @gfid_path: buffer provided into which path of the bad object is printed
- *            using above 2 arguments.
- * This function is same as br_stub_link_path. But in this function the
- * gfid of the bad object is obtained as an argument (i.e. uuid_t gfid),
- * where as in br_stub_link_path, the gfid is received as filename
- * (i.e. char *filename)
- */
-static void
-br_stub_linked_entry (br_stub_private_t *priv, char *gfid_path, uuid_t gfid,
-                        size_t len)
-{
-        snprintf (gfid_path, len, "%s/%s", priv->stub_basepath,
-                  uuid_utoa (gfid));
+    return ret;
 }
 
 /**
@@ -152,103 +100,112 @@ br_stub_linked_entry (br_stub_private_t *priv, char *gfid_path, uuid_t gfid,
  * @gfid: gfid of the bad object being added to the bad objects directory
  */
 int
-br_stub_add (xlator_t *this, uuid_t gfid)
+br_stub_add(xlator_t *this, uuid_t gfid)
 {
-        char              gfid_path[PATH_MAX] = {0};
-        char              bad_gfid_path[PATH_MAX] = {0};
-        int               ret = 0;
-        uuid_t            index = {0};
-        br_stub_private_t *priv = NULL;
-        struct stat       st = {0};
-        int               fd = 0;
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, errno, EINVAL);
-
-        br_stub_linked_entry (priv, gfid_path, gfid, sizeof (gfid_path));
-
-        ret = sys_stat (gfid_path, &st);
-        if (!ret)
-                goto out;
-        br_stub_container_entry (priv, bad_gfid_path, sizeof (bad_gfid_path));
-
-        ret = sys_link (bad_gfid_path, gfid_path);
-        if (ret) {
-                if ((errno != ENOENT) && (errno != EMLINK) && (errno != EEXIST))
-                        goto out;
-
-                /*
-                 * Continue with success. At least we'll have half of the
-                 * functionality, in the sense, object is marked bad and
-                 * would be inaccessible. It's only scrub status that would
-                 * show up less number of objects. That's fine as we'll have
-                 * the log files that will have the missing information.
-                 */
-                gf_msg (this->name, GF_LOG_WARNING, errno, BRS_MSG_LINK_FAIL,
-                        "failed to record  gfid [%s]", uuid_utoa (gfid));
-        }
-
-        return 0;
+    char gfid_path[BR_PATH_MAX_PLUS] = {0};
+    char bad_gfid_path[BR_PATH_MAX_PLUS] = {0};
+    int ret = 0;
+    br_stub_private_t *priv = NULL;
+    struct stat st = {0};
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  errno, EINVAL);
+
+    snprintf(gfid_path, sizeof(gfid_path), "%s/%s", priv->stub_basepath,
+             uuid_utoa(gfid));
+
+    ret = sys_stat(gfid_path, &st);
+    if (!ret)
+        goto out;
+    snprintf(bad_gfid_path, sizeof(bad_gfid_path), "%s/stub-%s",
+             priv->stub_basepath, uuid_utoa(priv->bad_object_dir_gfid));
+
+    ret = sys_link(bad_gfid_path, gfid_path);
+    if (ret) {
+        if ((errno != ENOENT) && (errno != EMLINK) && (errno != EEXIST))
+            goto out;
+
+        /*
+         * Continue with success. At least we'll have half of the
+         * functionality, in the sense, object is marked bad and
+         * would be inaccessible. It's only scrub status that would
+         * show up less number of objects. That's fine as we'll have
+         * the log files that will have the missing information.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, errno, BRS_MSG_LINK_FAIL, "gfid=%s",
+                uuid_utoa(gfid), NULL);
+    }
+
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int
-br_stub_del (xlator_t *this, uuid_t gfid)
+br_stub_del(xlator_t *this, uuid_t gfid)
 {
-        int32_t      op_errno __attribute__((unused)) = 0;
-        br_stub_private_t *priv = NULL;
-        int          ret = 0;
-        char         gfid_path[PATH_MAX] = {0};
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, op_errno, EINVAL);
-        br_stub_linked_entry (priv, gfid_path, gfid,
-                              sizeof (gfid_path));
-        ret = sys_unlink (gfid_path);
-        if (ret && (errno != ENOENT)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        BRS_MSG_BAD_OBJ_UNLINK_FAIL,
-                        "%s: failed to delete bad object link from quarantine "
-                        "directory", gfid_path);
-                ret = -errno;
-                goto out;
-        }
-
-        ret = 0;
+    int32_t op_errno __attribute__((unused)) = 0;
+    br_stub_private_t *priv = NULL;
+    int ret = 0;
+    char gfid_path[BR_PATH_MAX_PLUS] = {0};
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  op_errno, EINVAL);
+    snprintf(gfid_path, sizeof(gfid_path), "%s/%s", priv->stub_basepath,
+             uuid_utoa(gfid));
+    ret = sys_unlink(gfid_path);
+    if (ret && (errno != ENOENT)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJ_UNLINK_FAIL,
+                "path=%s", gfid_path, NULL);
+        ret = -errno;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-br_stub_check_stub_directory (xlator_t *this, char *fullpath)
+br_stub_check_stub_directory(xlator_t *this, char *fullpath)
 {
-        int ret = 0;
-        struct stat st = {0,};
-
-        ret = sys_stat (fullpath, &st);
-        if (!ret && !S_ISDIR (st.st_mode))
-                goto error_return;
-        if (ret) {
-                if (errno != ENOENT)
-                        goto error_return;
-                ret = mkdir_p (fullpath, 0600, _gf_true);
-        }
-
+    int ret = 0;
+    struct stat st = {
+        0,
+    };
+    char oldpath[BR_PATH_MAX_PLUS] = {0};
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    snprintf(oldpath, sizeof(oldpath), "%s/%s", priv->export,
+             OLD_BR_STUB_QUARANTINE_DIR);
+
+    ret = sys_stat(fullpath, &st);
+    if (!ret && !S_ISDIR(st.st_mode))
+        goto error_return;
+    if (ret) {
+        if (errno != ENOENT)
+            goto error_return;
+        ret = sys_stat(oldpath, &st);
         if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        BRS_MSG_BAD_OBJECT_DIR_FAIL,
-                        "failed to create stub directory [%s]", fullpath);
-        return ret;
+            ret = mkdir_p(fullpath, 0600, _gf_true);
+        else
+            ret = sys_rename(oldpath, fullpath);
+    }
+
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+                "create-path=%s", fullpath, NULL);
+    return ret;
 
 error_return:
-        gf_msg (this->name, GF_LOG_ERROR, errno,
-                BRS_MSG_BAD_OBJECT_DIR_FAIL,
-                "Failed to verify stub directory [%s]", fullpath);
-        return -1;
+    gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+            "verify-path=%s", fullpath, NULL);
+    return -1;
 }
 
 /**
@@ -256,380 +213,584 @@ error_return:
  * directory.
  */
 static int
-br_stub_check_stub_file (xlator_t *this, char *path)
+br_stub_check_stub_file(xlator_t *this, char *path)
 {
-        int ret = 0;
-        int fd = -1;
-        struct stat st = {0,};
-
-        ret = sys_stat (path, &st);
-        if (!ret && !S_ISREG (st.st_mode))
-                goto error_return;
-        if (ret) {
-                if (errno != ENOENT)
-                        goto error_return;
-                fd = sys_creat (path, 0);
-                if (fd < 0)
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                BRS_MSG_BAD_OBJECT_DIR_FAIL,
-                                "Failed ot create stub file [%s]", path);
-        }
-
-        if (fd >= 0) {
-                sys_close (fd);
-                ret = 0;
-        }
+    int ret = 0;
+    int fd = -1;
+    struct stat st = {
+        0,
+    };
+
+    ret = sys_stat(path, &st);
+    if (!ret && !S_ISREG(st.st_mode))
+        goto error_return;
+    if (ret) {
+        if (errno != ENOENT)
+            goto error_return;
+        fd = sys_creat(path, 0);
+        if (fd < 0)
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    BRS_MSG_BAD_OBJECT_DIR_FAIL, "create-path=%s", path, NULL);
+    }
+
+    if (fd >= 0) {
+        sys_close(fd);
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 
 error_return:
-        gf_msg (this->name, GF_LOG_ERROR, errno,
-                BRS_MSG_BAD_OBJECT_DIR_FAIL, "Failed ot verify stub file [%s]", path);
-        return -1;
+    gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+            "verify-path=%s", path, NULL);
+    return -1;
 }
 
 int
-br_stub_dir_create (xlator_t *this, br_stub_private_t *priv)
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv)
 {
-        int          ret = -1;
-        char         fullpath[PATH_MAX] = {0};
-        char         stub_gfid_path[PATH_MAX] = {0, };
-        char         path[PATH_MAX] = {0};
-        size_t       len = 0;
-
-        gf_uuid_copy (priv->bad_object_dir_gfid, BR_BAD_OBJ_CONTAINER);
-
-        snprintf (fullpath, sizeof (fullpath), "%s", priv->stub_basepath);
-
-        br_stub_container_entry (priv, stub_gfid_path, sizeof (stub_gfid_path));
-
-        ret = br_stub_check_stub_directory (this, fullpath);
-        if (ret)
-                goto out;
-        ret = br_stub_check_stub_file (this, stub_gfid_path);
-        if (ret)
-                goto out;
-
-        return 0;
+    int ret = -1;
+    char fullpath[BR_PATH_MAX_PLUS] = {
+        0,
+    };
+    char stub_gfid_path[BR_PATH_MAX_PLUS] = {
+        0,
+    };
+
+    gf_uuid_copy(priv->bad_object_dir_gfid, BR_BAD_OBJ_CONTAINER);
+
+    if (snprintf(fullpath, sizeof(fullpath), "%s", priv->stub_basepath) >=
+        sizeof(fullpath))
+        goto out;
+
+    if (snprintf(stub_gfid_path, sizeof(stub_gfid_path), "%s/stub-%s",
+                 priv->stub_basepath, uuid_utoa(priv->bad_object_dir_gfid)) >=
+        sizeof(stub_gfid_path))
+        goto out;
+
+    ret = br_stub_check_stub_directory(this, fullpath);
+    if (ret)
+        goto out;
+    ret = br_stub_check_stub_file(this, stub_gfid_path);
+    if (ret)
+        goto out;
+
+    return 0;
 
 out:
-        return -1;
+    return -1;
 }
 
 call_stub_t *
-__br_stub_dequeue (struct list_head *callstubs)
+__br_stub_dequeue(struct list_head *callstubs)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (!list_empty (callstubs)) {
-                stub = list_entry (callstubs->next, call_stub_t, list);
-                list_del_init (&stub->list);
-        }
+    if (!list_empty(callstubs)) {
+        stub = list_entry(callstubs->next, call_stub_t, list);
+        list_del_init(&stub->list);
+    }
 
-        return stub;
+    return stub;
 }
 
 void
-__br_stub_enqueue (struct list_head *callstubs, call_stub_t *stub)
+__br_stub_enqueue(struct list_head *callstubs, call_stub_t *stub)
 {
-        list_add_tail (&stub->list, callstubs);
+    list_add_tail(&stub->list, callstubs);
 }
 
 void
-br_stub_worker_enqueue (xlator_t *this, call_stub_t *stub)
+br_stub_worker_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        br_stub_private_t    *priv = NULL;
-
-        priv = this->private;
-        pthread_mutex_lock (&priv->container.bad_lock);
-        {
-                __br_stub_enqueue (&priv->container.bad_queue, stub);
-                pthread_cond_signal (&priv->container.bad_cond);
-        }
-        pthread_mutex_unlock (&priv->container.bad_lock);
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    pthread_mutex_lock(&priv->container.bad_lock);
+    {
+        __br_stub_enqueue(&priv->container.bad_queue, stub);
+        pthread_cond_signal(&priv->container.bad_cond);
+    }
+    pthread_mutex_unlock(&priv->container.bad_lock);
 }
 
 void *
-br_stub_worker (void *data)
+br_stub_worker(void *data)
 {
-        br_stub_private_t     *priv = NULL;
-        xlator_t         *this = NULL;
-        call_stub_t      *stub = NULL;
-        int               ret = 0;
-
-
-        THIS = data;
-        this = data;
-        priv = this->private;
-
-        for (;;) {
-                pthread_mutex_lock (&priv->container.bad_lock);
-                {
-                        while (list_empty (&priv->container.bad_queue)) {
-                              ret = pthread_cond_wait (&priv->container.bad_cond,
-                                                       &priv->container.bad_lock);
-                        }
-
-                        stub = __br_stub_dequeue (&priv->container.bad_queue);
-                }
-                pthread_mutex_unlock (&priv->container.bad_lock);
-
-                if (stub) /* guard against spurious wakeups */
-                        call_resume (stub);
+    br_stub_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    call_stub_t *stub = NULL;
+
+    THIS = data;
+    this = data;
+    priv = this->private;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->container.bad_lock);
+        {
+            while (list_empty(&priv->container.bad_queue)) {
+                (void)pthread_cond_wait(&priv->container.bad_cond,
+                                        &priv->container.bad_lock);
+            }
+
+            stub = __br_stub_dequeue(&priv->container.bad_queue);
         }
+        pthread_mutex_unlock(&priv->container.bad_lock);
 
-        return NULL;
+        if (stub) /* guard against spurious wakeups */
+            call_resume(stub);
+    }
+
+    return NULL;
 }
 
 int32_t
-br_stub_lookup_wrapper (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, dict_t *xattr_req)
+br_stub_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xattr_req)
 {
-        br_stub_private_t *priv        = NULL;
-        struct stat        lstatbuf    = {0};
-        int                ret         = 0;
-        int32_t            op_errno    = EINVAL;
-        int32_t            op_ret      = -1;
-        struct iatt        stbuf       = {0, };
-        struct iatt        postparent  = {0,};
-        dict_t            *xattr       = NULL;
-
-        priv = this->private;
-
-        VALIDATE_OR_GOTO (loc, done);
-        if (gf_uuid_compare (loc->gfid, priv->bad_object_dir_gfid))
-                goto done;
-
-        ret = sys_lstat (priv->stub_basepath, &lstatbuf);
-        if (ret) {
-                gf_msg_debug (this->name, errno, "Stat failed on stub bad "
-                              "object dir");
-                op_errno = errno;
-                goto done;
-        } else if (!S_ISDIR (lstatbuf.st_mode)) {
-                gf_msg_debug (this->name, errno, "bad object container is not "
-                              "a directory");
-                op_errno = ENOTDIR;
-                goto done;
-        }
-
-        iatt_from_stat (&stbuf, &lstatbuf);
-        gf_uuid_copy (stbuf.ia_gfid, priv->bad_object_dir_gfid);
-
-        op_ret = op_errno = 0;
-        xattr = dict_new ();
-        if (!xattr) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-        }
+    br_stub_private_t *priv = NULL;
+    struct stat lstatbuf = {0};
+    int ret = 0;
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), done);
+
+    VALIDATE_OR_GOTO(loc, done);
+    if (gf_uuid_compare(loc->gfid, priv->bad_object_dir_gfid))
+        goto done;
+
+    ret = sys_lstat(priv->stub_basepath, &lstatbuf);
+    if (ret) {
+        gf_msg_debug(this->name, errno,
+                     "Stat failed on stub bad "
+                     "object dir");
+        op_errno = errno;
+        goto done;
+    } else if (!S_ISDIR(lstatbuf.st_mode)) {
+        gf_msg_debug(this->name, errno,
+                     "bad object container is not "
+                     "a directory");
+        op_errno = ENOTDIR;
+        goto done;
+    }
+
+    iatt_from_stat(&stbuf, &lstatbuf);
+    gf_uuid_copy(stbuf.ia_gfid, priv->bad_object_dir_gfid);
+
+    op_ret = op_errno = 0;
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+    }
 
 done:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
-                             loc->inode, &stbuf, xattr, &postparent);
-        if (xattr)
-                dict_unref (xattr);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, loc->inode, &stbuf,
+                        xattr, &postparent);
+    if (xattr)
+        dict_unref(xattr);
+    return 0;
 }
 
 static int
-is_bad_gfid_file_current (char *filename, uuid_t gfid)
+is_bad_gfid_file_current(char *filename, uuid_t gfid)
 {
-        char current_stub_gfid[GF_UUID_BUF_SIZE + 16] = {0, };
+    char current_stub_gfid[GF_UUID_BUF_SIZE + 16] = {
+        0,
+    };
 
-        snprintf (current_stub_gfid, sizeof current_stub_gfid,
-                  "stub-%s", uuid_utoa(gfid));
-        return (!strcmp(filename, current_stub_gfid));
+    snprintf(current_stub_gfid, sizeof current_stub_gfid, "stub-%s",
+             uuid_utoa(gfid));
+    return (!strcmp(filename, current_stub_gfid));
 }
 
 static void
-check_delete_stale_bad_file (xlator_t *this, char *filename)
+check_delete_stale_bad_file(xlator_t *this, char *filename)
 {
-        int             ret = 0;
-        struct stat     st = {0};
-        char            filepath[PATH_MAX] = {0};
-        br_stub_private_t    *priv = NULL;
+    int ret = 0;
+    struct stat st = {0};
+    char filepath[BR_PATH_MAX_PLUS] = {0};
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (is_bad_gfid_file_current (filename, priv->bad_object_dir_gfid))
-                return;
+    if (is_bad_gfid_file_current(filename, priv->bad_object_dir_gfid))
+        return;
 
-        br_stub_link_path (priv, filename, filepath, sizeof (filepath));
+    snprintf(filepath, sizeof(filepath), "%s/%s", priv->stub_basepath,
+             filename);
 
-        ret = sys_stat (filepath, &st);
-        if (!ret && st.st_nlink == 1)
-                sys_unlink (filepath);
+    ret = sys_stat(filepath, &st);
+    if (!ret && st.st_nlink == 1)
+        sys_unlink(filepath);
 }
 
 static int
-br_stub_fill_readdir (fd_t *fd, br_stub_fd_t *fctx, DIR *dir, off_t off,
-                      size_t size, gf_dirent_t *entries)
+br_stub_fill_readdir(fd_t *fd, br_stub_fd_t *fctx, DIR *dir, off_t off,
+                     size_t size, gf_dirent_t *entries)
 {
-        off_t     in_case = -1;
-        off_t     last_off = 0;
-        size_t    filled = 0;
-        int       count = 0;
-        char      entrybuf[sizeof(struct dirent) + 256 + 8];
-        struct dirent  *entry          = NULL;
-        int32_t              this_size      = -1;
-        gf_dirent_t          *this_entry     = NULL;
-        xlator_t             *this = NULL;
-
-        this = THIS;
-        if (!off) {
-                rewinddir (dir);
-        } else {
-                seekdir (dir, off);
+    off_t in_case = -1;
+    off_t last_off = 0;
+    size_t filled = 0;
+    int count = 0;
+    int32_t this_size = -1;
+    gf_dirent_t *this_entry = NULL;
+    xlator_t *this = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+
+    this = THIS;
+    if (!off) {
+        rewinddir(dir);
+    } else {
+        seekdir(dir, off);
 #ifndef GF_LINUX_HOST_OS
-                if ((u_long)telldir(dir) != off &&
-                    off != fctx->bad_object.dir_eof) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL,
-                                "seekdir(0x%llx) failed on dir=%p: "
-				"Invalid argument (offset reused from "
-				"another DIR * structure?)", off, dir);
-                        errno = EINVAL;
-                        count = -1;
-                        goto out;
-                }
+        if ((u_long)telldir(dir) != off && off != fctx->bad_object.dir_eof) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, "off=(0x%llx)", off,
+                    "dir=%p", dir, NULL);
+            errno = EINVAL;
+            count = -1;
+            goto out;
+        }
 #endif /* GF_LINUX_HOST_OS */
+    }
+
+    while (filled <= size) {
+        in_case = (u_long)telldir(dir);
+
+        if (in_case == -1) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL, "dir=%p", dir, "err=%s",
+                    strerror(errno), NULL);
+            goto out;
         }
 
-        while (filled <= size) {
-                in_case = (u_long)telldir (dir);
-
-                if (in_case == -1) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL,
-                                "telldir failed on dir=%p: %s",
-                                dir, strerror (errno));
-                        goto out;
-                }
-
-                errno = 0;
-                entry = NULL;
-                readdir_r (dir, (struct dirent *)entrybuf, &entry);
-
-                if (!entry) {
-                        if (errno == EBADF) {
-                                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                        BRS_MSG_BAD_OBJECT_DIR_READ_FAIL,
-                                        "readdir failed on dir=%p: %s",
-                                        dir, strerror (errno));
-                                goto out;
-                        }
-                        break;
-                }
-
-                if (!strcmp (entry->d_name, ".") ||
-                    !strcmp (entry->d_name, ".."))
-                        continue;
-
-                if (!strncmp (entry->d_name, "stub-",
-                              strlen ("stub-"))) {
-                        check_delete_stale_bad_file (this, entry->d_name);
-                        continue;
-                }
-
-                this_size = max (sizeof (gf_dirent_t),
-                                 sizeof (gfs3_dirplist))
-                        + strlen (entry->d_name) + 1;
-
-                if (this_size + filled > size) {
-                        seekdir (dir, in_case);
+        errno = 0;
+        entry = sys_readdir(dir, scratch);
+        if (!entry || errno != 0) {
+            if (errno == EBADF) {
+                gf_smsg(THIS->name, GF_LOG_WARNING, 0,
+                        BRS_MSG_BAD_OBJECT_DIR_READ_FAIL, "dir=%p", dir,
+                        "err=%s", strerror(errno), NULL);
+                goto out;
+            }
+            break;
+        }
+
+        if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+            continue;
+
+        if (!strncmp(entry->d_name, "stub-", strlen("stub-"))) {
+            check_delete_stale_bad_file(this, entry->d_name);
+            continue;
+        }
+
+        this_size = max(sizeof(gf_dirent_t), sizeof(gfs3_dirplist)) +
+                    strlen(entry->d_name) + 1;
+
+        if (this_size + filled > size) {
+            seekdir(dir, in_case);
 #ifndef GF_LINUX_HOST_OS
-                        if ((u_long)telldir(dir) != in_case &&
-                            in_case != fctx->bad_object.dir_eof) {
-				gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                        BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL,
-					"seekdir(0x%llx) failed on dir=%p: "
-					"Invalid argument (offset reused from "
-					"another DIR * structure?)",
-					in_case, dir);
-				errno = EINVAL;
-				count = -1;
-				goto out;
-                        }
+            if ((u_long)telldir(dir) != in_case &&
+                in_case != fctx->bad_object.dir_eof) {
+                gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, "in_case=(0x%llx)",
+                        in_case, "dir=%p", dir, NULL);
+                errno = EINVAL;
+                count = -1;
+                goto out;
+            }
 #endif /* GF_LINUX_HOST_OS */
-                        break;
-                }
-
-                this_entry = gf_dirent_for_name (entry->d_name);
-
-                if (!this_entry) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_NO_MEMORY,
-                                "could not create gf_dirent for entry %s: (%s)",
-                                entry->d_name, strerror (errno));
-                        goto out;
-                }
-                /*
-                 * we store the offset of next entry here, which is
-                 * probably not intended, but code using syncop_readdir()
-                 * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
-                 * for directory read resumption.
-                 */
-                last_off = (u_long)telldir(dir);
-                this_entry->d_off = last_off;
-                this_entry->d_ino = entry->d_ino;
-
-                list_add_tail (&this_entry->list, &entries->list);
-
-                filled += this_size;
-                count++;
+            break;
         }
 
-        if ((!sys_readdir (dir) && (errno == 0))) {
-                /* Indicate EOF */
-                errno = ENOENT;
-                /* Remember EOF offset for later detection */
-                fctx->bad_object.dir_eof = last_off;
+        this_entry = gf_dirent_for_name(entry->d_name);
+
+        if (!this_entry) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CREATE_GF_DIRENT_FAILED, "entry-name=%s",
+                    entry->d_name, "err=%s", strerror(errno), NULL);
+            goto out;
         }
+        /*
+         * we store the offset of next entry here, which is
+         * probably not intended, but code using syncop_readdir()
+         * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
+         * for directory read resumption.
+         */
+        last_off = (u_long)telldir(dir);
+        this_entry->d_off = last_off;
+        this_entry->d_ino = entry->d_ino;
+
+        list_add_tail(&this_entry->list, &entries->list);
+
+        filled += this_size;
+        count++;
+    }
+
+    if ((!sys_readdir(dir, scratch) && (errno == 0))) {
+        /* Indicate EOF */
+        errno = ENOENT;
+        /* Remember EOF offset for later detection */
+        fctx->bad_object.dir_eof = last_off;
+    }
 out:
-        return count;
+    return count;
 }
 
 int32_t
-br_stub_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, size_t size, off_t off, dict_t *xdata)
+br_stub_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        size_t size, off_t off, dict_t *xdata)
 {
-        br_stub_fd_t         *fctx           = NULL;
-        DIR                  *dir            = NULL;
-        int                   ret            = -1;
-        int32_t               op_ret         = -1;
-        int32_t               op_errno       = 0;
-        int                   count          = 0;
-        gf_dirent_t           entries;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        fctx = br_stub_fd_ctx_get (this, fd);
-        if (!fctx) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_GET_FD_CONTEXT_FAILED,
-                        "pfd is NULL, fd=%p", fd);
-                op_errno = -ret;
-                goto done;
-        }
+    br_stub_fd_t *fctx = NULL;
+    DIR *dir = NULL;
+    int ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int count = 0;
+    gf_dirent_t entries;
+    gf_boolean_t xdata_unref = _gf_false;
+    dict_t *dict = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    fctx = br_stub_fd_ctx_get(this, fd);
+    if (!fctx) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_GET_FD_CONTEXT_FAILED,
+                "fd=%p", fd, NULL);
+        op_errno = -ret;
+        goto done;
+    }
+
+    dir = fctx->bad_object.dir;
+
+    if (!dir) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_BAD_HANDLE_DIR_NULL,
+                "fd=%p", fd, NULL);
+        op_errno = EINVAL;
+        goto done;
+    }
+
+    count = br_stub_fill_readdir(fd, fctx, dir, off, size, &entries);
+
+    /* pick ENOENT to indicate EOF */
+    op_errno = errno;
+    op_ret = count;
+
+    dict = xdata;
+    (void)br_stub_bad_objects_path(this, fd, &entries, &dict);
+    if (!xdata && dict) {
+        xdata = dict;
+        xdata_unref = _gf_true;
+    }
 
-        dir = fctx->bad_object.dir;
+done:
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, &entries, xdata);
+    gf_dirent_free(&entries);
+    if (xdata_unref)
+        dict_unref(xdata);
+    return 0;
+}
+
+/**
+ * This function is called to mainly obtain the paths of the corrupt
+ * objects (files as of now). Currently scrub status prints only the
+ * gfid of the corrupted files. Reason is, bitrot-stub maintains the
+ * list of the corrupted objects as entries inside the quarantine
+ * directory (<brick export>/.glusterfs/quarantine)
+ *
+ * And the name of each entry in the qurantine directory is the gfid
+ * of the corrupted object. So scrub status will just show that info.
+ * But it helps the users a lot if the actual path to the object is
+ * also reported. Hence the below function to get that information.
+ * The function allocates a new dict to be returned (if it does not
+ * get one from the caller of readdir i.e. scrubber as of now), and
+ * stores the paths of each corrupted gfid there. The gfid is used as
+ * the key and path is used as the value.
+ *
+ * NOTE: The path will be there in following situations
+ * 1) gfid2path option has been enabled (posix xlator option)
+ *    and the corrupted file contains the path as an extended
+ *    attribute.
+ * 2) If the gfid2path option is not enabled, OR if the xattr
+ *    is absent, then the inode table should have it.
+ *    The path will be there if a name based lookup has happened
+ *    on the file which has been corrupted. With lookup a inode and
+ *    dentry would be created in the inode table. And the path is
+ *    constructed using the in memory inode and dentry. If a lookup
+ *    has not happened OR the inode corresponding to the corrupted
+ *    file does not exist in the inode table (because it got purged
+ *    as lru limit of the inodes exceeded) OR a nameless lookup had
+ *    happened to populate the inode in the inode table, then the
+ *    path will not be printed in scrub and only the gfid will be there.
+ **/
+int
+br_stub_bad_objects_path(xlator_t *this, fd_t *fd, gf_dirent_t *entries,
+                         dict_t **dict)
+{
+    gf_dirent_t *entry = NULL;
+    inode_t *inode = NULL;
+    char *hpath = NULL;
+    uuid_t gfid = {0};
+    int ret = -1;
+    dict_t *tmp_dict = NULL;
+    char str_gfid[64] = {0};
+
+    if (list_empty(&entries->list))
+        return 0;
 
-        if (!dir) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_BAD_HANDLE_DIR_NULL,
-                        "dir is NULL for fd=%p", fd);
-                op_errno = EINVAL;
-                goto done;
+    tmp_dict = *dict;
+
+    if (!tmp_dict) {
+        tmp_dict = dict_new();
+        /*
+         * If the allocation of dict fails then no need treat it
+         * it as a error. This path (or function) is executed when
+         * "gluster volume bitrot <volume name> scrub status" is
+         * executed, to get the list of the corrupted objects.
+         * And the motive of this function is to get the paths of
+         * the corrupted objects. If the dict allocation fails, then
+         * the scrub status will only show the gfids of those corrupted
+         * objects (which is the behavior as of the time of this patch
+         * being worked upon). So just return and only the gfids will
+         * be shown.
+         */
+        if (!tmp_dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_FAILED, NULL);
+            goto out;
         }
+    }
 
-        count = br_stub_fill_readdir (fd, fctx, dir, off, size, &entries);
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        gf_uuid_clear(gfid);
+        gf_uuid_parse(entry->d_name, gfid);
 
-        /* pick ENOENT to indicate EOF */
-        op_errno = errno;
-        op_ret = count;
-done:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
-        gf_dirent_free (&entries);
-        return 0;
+        inode = inode_find(fd->inode->table, gfid);
+
+        /* No need to check the return value here.
+         * Because @hpath is examined.
+         */
+        (void)br_stub_get_path_of_gfid(this, fd->inode, inode, gfid, &hpath);
+
+        if (hpath) {
+            gf_msg_debug(this->name, 0,
+                         "path of the corrupted "
+                         "object (gfid: %s) is %s",
+                         uuid_utoa(gfid), hpath);
+            br_stub_entry_xattr_fill(this, hpath, entry, tmp_dict);
+        } else
+            gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                    "gfid=%s", uuid_utoa_r(gfid, str_gfid), NULL);
+
+        inode = NULL;
+        hpath = NULL;
+    }
+
+    ret = 0;
+    *dict = tmp_dict;
+
+out:
+    return ret;
 }
 
+int
+br_stub_get_path_of_gfid(xlator_t *this, inode_t *parent, inode_t *inode,
+                         uuid_t gfid, char **path)
+{
+    int32_t ret = -1;
+    char gfid_str[64] = {0};
+
+    GF_VALIDATE_OR_GOTO("bitrot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, path, out);
+
+    /* Above, No need to validate the @inode for hard resolution. Because
+     * inode can be NULL and if it is NULL, then syncop_gfid_to_path_hard
+     * will allocate a new inode and proceed. So no need to bother about
+     * @inode. Because we need it only to send a syncop_getxattr call
+     * from inside syncop_gfid_to_path_hard. And getxattr fetches the
+     * path from the backend.
+     */
+
+    ret = syncop_gfid_to_path_hard(parent->table, FIRST_CHILD(this), gfid,
+                                   inode, path, _gf_true);
+    if (ret < 0)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                "gfid=%s", uuid_utoa_r(gfid, gfid_str), NULL);
+
+    /*
+     * Try with soft resolution of path if hard resolve fails. Because
+     * checking the xattr on disk to get the path of a inode (or gfid)
+     * is dependent on whether that option is enabled in the posix
+     * xlator or not. If it is not enabled, then hard resolution by
+     * checking the on disk xattr fails.
+     *
+     * Thus in such situations fall back to the soft resolution which
+     * mainly depends on the inode_path() function. And for using
+     * inode_path, @inode has to be linked i.e. a successful lookup should
+     * have happened on the gfid (or the path) to link the inode to the
+     * inode table. And if @inode is NULL, means, the inode has not been
+     * found in the inode table and better not to do inode_path() on the
+     * inode which has not been linked.
+     */
+    if (ret < 0 && inode) {
+        ret = syncop_gfid_to_path_hard(parent->table, FIRST_CHILD(this), gfid,
+                                       inode, path, _gf_false);
+        if (ret < 0)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                    "from-memory  gfid=%s", uuid_utoa_r(gfid, gfid_str), NULL);
+    }
+
+out:
+    return ret;
+}
+
+/**
+ * NOTE: If the file has multiple hardlinks (in gluster volume
+ * namespace), the path would be one of the hardlinks. Its up to
+ * the user to find the remaining hardlinks (using find -samefile)
+ * and remove them.
+ **/
+void
+br_stub_entry_xattr_fill(xlator_t *this, char *hpath, gf_dirent_t *entry,
+                         dict_t *dict)
+{
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, hpath, out);
+
+    /*
+     * Use the entry->d_name (which is nothing but the gfid of the
+     * corrupted object) as the key. And the value will be the actual
+     * path of that object (or file).
+     *
+     * ALso ignore the dict_set errors. scrubber will get the gfid of
+     * the corrupted object for sure. So, for now lets just log the
+     * dict_set_dynstr failure and move on.
+     */
+
+    ret = dict_set_dynstr(dict, entry->d_name, hpath);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_DICT_SET_FAILED,
+                "path=%s", hpath, "object-name=%s", entry->d_name, NULL);
+out:
+    return;
+}
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
index f70fafbca49..9d93caf069f 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
@@ -11,28 +11,26 @@
 #ifndef _BR_MEM_TYPES_H
 #define _BR_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum br_mem_types {
-        gf_br_stub_mt_private_t   = gf_common_mt_end + 1,
-        gf_br_stub_mt_version_t,
-        gf_br_stub_mt_inode_ctx_t,
-        gf_br_stub_mt_signature_t,
-        gf_br_mt_br_private_t,
-        gf_br_mt_br_child_t,
-        gf_br_mt_br_object_t,
-        gf_br_mt_br_ob_n_wk_t,
-        gf_br_mt_br_tbf_t,
-        gf_br_mt_br_tbf_bucket_t,
-        gf_br_mt_br_tbf_throttle_t,
-        gf_br_mt_br_tbf_opspec_t,
-        gf_br_mt_br_scrubber_t,
-        gf_br_mt_br_fsscan_entry_t,
-        gf_br_stub_mt_br_stub_fd_t,
-        gf_br_stub_mt_br_scanner_freq_t,
-        gf_br_stub_mt_sigstub_t,
-        gf_br_mt_br_child_event_t,
-        gf_br_stub_mt_end,
+    gf_br_stub_mt_private_t = gf_common_mt_end + 1,
+    gf_br_stub_mt_version_t,
+    gf_br_stub_mt_inode_ctx_t,
+    gf_br_stub_mt_signature_t,
+    gf_br_mt_br_private_t,
+    gf_br_mt_br_child_t,
+    gf_br_mt_br_object_t,
+    gf_br_mt_br_ob_n_wk_t,
+    gf_br_mt_br_scrubber_t,
+    gf_br_mt_br_fsscan_entry_t,
+    gf_br_stub_mt_br_stub_fd_t,
+    gf_br_stub_mt_br_scanner_freq_t,
+    gf_br_stub_mt_sigstub_t,
+    gf_br_mt_br_child_event_t,
+    gf_br_stub_mt_misc,
+    gf_br_mt_br_worker_t,
+    gf_br_stub_mt_end,
 };
 
 #endif
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
index c0fcfd324a5..6c15a166f18 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
@@ -11,261 +11,107 @@
 #ifndef _BITROT_STUB_MESSAGES_H_
 #define _BITROT_STUB_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
-/* file bit-rot-stub-messages.h
- * brief BIT-ROT log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_BITROT_STUB_BASE                   GLFS_MSGID_COMP_BITROT_STUB
-#define GLFS_BITROT_STUB_NUM_MESSAGES           31
-#define GLFS_MSGID_END         (GLFS_BITROT_STUB_BASE + \
-                                GLFS_BITROT_STUB_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x   GLFS_BITROT_STUB_BASE, "Invalid: Start of messages"
-/*------------*/
-
-
-#define BRS_MSG_NO_MEMORY                   (GLFS_BITROT_STUB_BASE + 1)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_SET_EVENT_FAILED            (GLFS_BITROT_STUB_BASE + 2)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_MEM_ACNT_FAILED             (GLFS_BITROT_STUB_BASE + 3)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_CREATE_FRAME_FAILED         (GLFS_BITROT_STUB_BASE + 4)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_SET_CONTEXT_FAILED          (GLFS_BITROT_STUB_BASE + 5)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_CHANGE_VERSION_FAILED       (GLFS_BITROT_STUB_BASE + 6)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_ADD_FD_TO_LIST_FAILED       (GLFS_BITROT_STUB_BASE + 7)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_SET_FD_CONTEXT_FAILED       (GLFS_BITROT_STUB_BASE + 8)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_CREATE_ANONYMOUS_FD_FAILED  (GLFS_BITROT_STUB_BASE + 9)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_NO_CHILD                    (GLFS_BITROT_STUB_BASE + 10)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_STUB_ALLOC_FAILED           (GLFS_BITROT_STUB_BASE + 11)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_GET_INODE_CONTEXT_FAILED    (GLFS_BITROT_STUB_BASE + 12)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_CANCEL_SIGN_THREAD_FAILED   (GLFS_BITROT_STUB_BASE + 13)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_ADD_FD_TO_INODE             (GLFS_BITROT_STUB_BASE + 14)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_SIGN_VERSION_ERROR          (GLFS_BITROT_STUB_BASE + 15)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
+/* To add new message IDs, append new identifiers at the end of the list.
  *
- */
-#define BRS_MSG_BAD_OBJ_MARK_FAIL           (GLFS_BITROT_STUB_BASE + 16)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
  *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define BRS_MSG_NON_SCRUB_BAD_OBJ_MARK      (GLFS_BITROT_STUB_BASE + 17)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_REMOVE_INTERNAL_XATTR       (GLFS_BITROT_STUB_BASE + 18)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_SET_INTERNAL_XATTR          (GLFS_BITROT_STUB_BASE + 19)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJECT_ACCESS           (GLFS_BITROT_STUB_BASE + 20)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_CONTAINER_FAIL          (GLFS_BITROT_STUB_BASE + 21)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJECT_DIR_FAIL        (GLFS_BITROT_STUB_BASE + 22)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL   (GLFS_BITROT_STUB_BASE + 23)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL   (GLFS_BITROT_STUB_BASE + 24)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJECT_DIR_READ_FAIL   (GLFS_BITROT_STUB_BASE + 25)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_GET_FD_CONTEXT_FAILED      (GLFS_BITROT_STUB_BASE + 26)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_HANDLE_DIR_NULL        (GLFS_BITROT_STUB_BASE + 27)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJ_THREAD_FAIL        (GLFS_BITROT_STUB_BASE + 28)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL     (GLFS_BITROT_STUB_BASE + 29)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_LINK_FAIL                  (GLFS_BITROT_STUB_BASE + 30)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define BRS_MSG_BAD_OBJ_UNLINK_FAIL        (GLFS_BITROT_STUB_BASE + 31)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-/*------------*/
 
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+GLFS_MSGID(BITROT_STUB, BRS_MSG_NO_MEMORY, BRS_MSG_SET_EVENT_FAILED,
+           BRS_MSG_MEM_ACNT_FAILED, BRS_MSG_CREATE_FRAME_FAILED,
+           BRS_MSG_SET_CONTEXT_FAILED, BRS_MSG_CHANGE_VERSION_FAILED,
+           BRS_MSG_ADD_FD_TO_LIST_FAILED, BRS_MSG_SET_FD_CONTEXT_FAILED,
+           BRS_MSG_CREATE_ANONYMOUS_FD_FAILED, BRS_MSG_NO_CHILD,
+           BRS_MSG_STUB_ALLOC_FAILED, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+           BRS_MSG_CANCEL_SIGN_THREAD_FAILED, BRS_MSG_ADD_FD_TO_INODE,
+           BRS_MSG_SIGN_VERSION_ERROR, BRS_MSG_BAD_OBJ_MARK_FAIL,
+           BRS_MSG_NON_SCRUB_BAD_OBJ_MARK, BRS_MSG_REMOVE_INTERNAL_XATTR,
+           BRS_MSG_SET_INTERNAL_XATTR, BRS_MSG_BAD_OBJECT_ACCESS,
+           BRS_MSG_BAD_CONTAINER_FAIL, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+           BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL,
+           BRS_MSG_BAD_OBJECT_DIR_READ_FAIL, BRS_MSG_GET_FD_CONTEXT_FAILED,
+           BRS_MSG_BAD_HANDLE_DIR_NULL, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+           BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL, BRS_MSG_LINK_FAIL,
+           BRS_MSG_BAD_OBJ_UNLINK_FAIL, BRS_MSG_DICT_SET_FAILED,
+           BRS_MSG_PATH_GET_FAILED, BRS_MSG_NULL_LOCAL,
+           BRS_MSG_SPAWN_SIGN_THRD_FAILED, BRS_MSG_KILL_SIGN_THREAD,
+           BRS_MSG_NON_BITD_PID, BRS_MSG_SIGN_PREPARE_FAIL,
+           BRS_MSG_USING_DEFAULT_THREAD_SIZE, BRS_MSG_ALLOC_MEM_FAILED,
+           BRS_MSG_DICT_ALLOC_FAILED, BRS_MSG_CREATE_GF_DIRENT_FAILED,
+           BRS_MSG_ALLOC_FAILED, BRS_MSG_PATH_XATTR_GET_FAILED,
+           BRS_MSG_VERSION_PREPARE_FAIL);
+
+#define BRS_MSG_MEM_ACNT_FAILED_STR "Memory accounting init failed"
+#define BRS_MSG_BAD_OBJ_THREAD_FAIL_STR "pthread_init failed"
+#define BRS_MSG_USING_DEFAULT_THREAD_SIZE_STR "Using default thread stack size"
+#define BRS_MSG_NO_CHILD_STR "FATAL: no children"
+#define BRS_MSG_SPAWN_SIGN_THRD_FAILED_STR                                     \
+    "failed to create the new thread for signer"
+#define BRS_MSG_BAD_CONTAINER_FAIL_STR                                         \
+    "failed to launch the thread for storing bad gfids"
+#define BRS_MSG_CANCEL_SIGN_THREAD_FAILED_STR                                  \
+    "Could not cancel sign serializer thread"
+#define BRS_MSG_KILL_SIGN_THREAD_STR "killed the signer thread"
+#define BRS_MSG_GET_INODE_CONTEXT_FAILED_STR                                   \
+    "failed to init the inode context for the inode"
+#define BRS_MSG_ADD_FD_TO_INODE_STR "failed to add fd to the inode"
+#define BRS_MSG_NO_MEMORY_STR "local allocation failed"
+#define BRS_MSG_BAD_OBJECT_ACCESS_STR "bad object accessed. Returning"
+#define BRS_MSG_SIGN_VERSION_ERROR_STR "Signing version exceeds current version"
+#define BRS_MSG_NON_BITD_PID_STR                                               \
+    "PID from where signature request came, does not belong to bit-rot "       \
+    "daemon. Unwinding the fop"
+#define BRS_MSG_SIGN_PREPARE_FAIL_STR                                          \
+    "failed to prepare the signature. Unwinding the fop"
+#define BRS_MSG_VERSION_PREPARE_FAIL_STR                                       \
+    "failed to prepare the version. Unwinding the fop"
+#define BRS_MSG_STUB_ALLOC_FAILED_STR "failed to allocate stub fop, Unwinding"
+#define BRS_MSG_BAD_OBJ_MARK_FAIL_STR "failed to mark object as bad"
+#define BRS_MSG_NON_SCRUB_BAD_OBJ_MARK_STR                                     \
+    "bad object marking is not from the scrubber"
+#define BRS_MSG_ALLOC_MEM_FAILED_STR "failed to allocate memory"
+#define BRS_MSG_SET_INTERNAL_XATTR_STR "called on the internal xattr"
+#define BRS_MSG_REMOVE_INTERNAL_XATTR_STR "removexattr called on internal xattr"
+#define BRS_MSG_CREATE_ANONYMOUS_FD_FAILED_STR                                 \
+    "failed to create anonymous fd for the inode"
+#define BRS_MSG_ADD_FD_TO_LIST_FAILED_STR "failed add fd to the list"
+#define BRS_MSG_SET_FD_CONTEXT_FAILED_STR                                      \
+    "failed to set the fd context for the file"
+#define BRS_MSG_NULL_LOCAL_STR "local is NULL"
+#define BRS_MSG_DICT_ALLOC_FAILED_STR                                          \
+    "dict allocation failed: cannot send IPC FOP to changelog"
+#define BRS_MSG_SET_EVENT_FAILED_STR "cannot set release event in dict"
+#define BRS_MSG_CREATE_FRAME_FAILED_STR "create_frame() failure"
+#define BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL_STR "closedir error"
+#define BRS_MSG_LINK_FAIL_STR "failed to record gfid"
+#define BRS_MSG_BAD_OBJ_UNLINK_FAIL_STR                                        \
+    "failed to delete bad object link from quaratine directory"
+#define BRS_MSG_BAD_OBJECT_DIR_FAIL_STR "failed stub directory"
+#define BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL_STR                                   \
+    "seekdir failed. Invalid argument (offset reused from another DIR * "      \
+    "structure)"
+#define BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL_STR "telldir failed on dir"
+#define BRS_MSG_BAD_OBJECT_DIR_READ_FAIL_STR "readdir failed on dir"
+#define BRS_MSG_CREATE_GF_DIRENT_FAILED_STR "could not create gf_dirent"
+#define BRS_MSG_GET_FD_CONTEXT_FAILED_STR "pfd is NULL"
+#define BRS_MSG_BAD_HANDLE_DIR_NULL_STR "dir if NULL"
+#define BRS_MSG_ALLOC_FAILED_STR                                               \
+    "failed to allocate new dict for saving the paths of the corrupted "       \
+    "objects. Scrub status will only display the gfid"
+#define BRS_MSG_PATH_GET_FAILED_STR "failed to get the path"
+#define BRS_MSG_PATH_XATTR_GET_FAILED_STR                                      \
+    "failed to get the path xattr from disk for the gfid. Trying to get path " \
+    "from the memory"
+#define BRS_MSG_DICT_SET_FAILED_STR                                            \
+    "failed to set the actual path as the value in the dict for the "          \
+    "corrupted object"
+#define BRS_MSG_SET_CONTEXT_FAILED_STR                                         \
+    "could not set fd context for release callback"
+#define BRS_MSG_CHANGE_VERSION_FAILED_STR "change version failed"
 #endif /* !_BITROT_STUB_MESSAGES_H_ */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
index 67103f6b5e1..447dd47ff41 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
@@ -10,312 +10,433 @@
 
 #include <ctype.h>
 #include <sys/uio.h>
+#include <signal.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
 #include "changelog.h"
-#include "compat-errno.h"
-#include "call-stub.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/call-stub.h>
 
 #include "bit-rot-stub.h"
 #include "bit-rot-stub-mem-types.h"
 #include "bit-rot-stub-messages.h"
 #include "bit-rot-common.h"
 
-#define BR_STUB_REQUEST_COOKIE  0x1
+#define BR_STUB_REQUEST_COOKIE 0x1
 
-void *br_stub_signth (void *);
+void
+br_stub_lock_cleaner(void *arg)
+{
+    pthread_mutex_t *clean_mutex = arg;
+
+    pthread_mutex_unlock(clean_mutex);
+    return;
+}
+
+void *
+br_stub_signth(void *);
 
 struct br_stub_signentry {
-        unsigned long v;
+    unsigned long v;
 
-        call_stub_t *stub;
+    call_stub_t *stub;
 
-        struct list_head list;
+    struct list_head list;
 };
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int32_t ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_br_stub_mt_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRS_MSG_MEM_ACNT_FAILED,
-                        "Memory accounting init failed");
-                return ret;
-        }
+    int32_t ret = -1;
 
+    if (!this)
         return ret;
-}
-
-int32_t
-br_stub_bad_object_container_init (xlator_t *this, br_stub_private_t *priv)
-{
-        pthread_attr_t  w_attr;
-        int32_t         ret          = -1;
-
-        ret = pthread_cond_init(&priv->container.bad_cond, NULL);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "pthread_cond_init failed (%d)", ret);
-                goto out;
-        }
 
-        ret = pthread_mutex_init(&priv->container.bad_lock, NULL);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "pthread_mutex_init failed (%d)", ret);
-                goto cleanup_cond;
-        }
+    ret = xlator_mem_acct_init(this, gf_br_stub_mt_end + 1);
 
-        ret = pthread_attr_init (&w_attr);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "pthread_attr_init failed (%d)", ret);
-                goto cleanup_lock;
-        }
-
-        ret = pthread_attr_setstacksize (&w_attr, BAD_OBJECT_THREAD_STACK_SIZE);
-        if (ret == EINVAL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "Using default thread stack size");
-        }
-
-        INIT_LIST_HEAD (&priv->container.bad_queue);
-        ret = br_stub_dir_create (this, priv);
-        if (ret < 0)
-                goto cleanup_lock;
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_MEM_ACNT_FAILED, NULL);
+        return ret;
+    }
 
-        ret = gf_thread_create (&priv->container.thread, &w_attr, br_stub_worker, this);
-        if (ret)
-                goto cleanup_attr;
+    return ret;
+}
 
-        return 0;
+int
+br_stub_bad_object_container_init(xlator_t *this, br_stub_private_t *priv)
+{
+    pthread_attr_t w_attr;
+    int ret = -1;
+
+    ret = pthread_cond_init(&priv->container.bad_cond, NULL);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "cond_init ret=%d", ret, NULL);
+        goto out;
+    }
+
+    ret = pthread_mutex_init(&priv->container.bad_lock, NULL);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "mutex_init ret=%d", ret, NULL);
+        goto cleanup_cond;
+    }
+
+    ret = pthread_attr_init(&w_attr);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "attr_init ret=%d", ret, NULL);
+        goto cleanup_lock;
+    }
+
+    ret = pthread_attr_setstacksize(&w_attr, BAD_OBJECT_THREAD_STACK_SIZE);
+    if (ret == EINVAL) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                BRS_MSG_USING_DEFAULT_THREAD_SIZE, NULL);
+    }
+
+    INIT_LIST_HEAD(&priv->container.bad_queue);
+    ret = br_stub_dir_create(this, priv);
+    if (ret < 0)
+        goto cleanup_lock;
+
+    ret = gf_thread_create(&priv->container.thread, &w_attr, br_stub_worker,
+                           this, "brswrker");
+    if (ret)
+        goto cleanup_attr;
+
+    return 0;
 
 cleanup_attr:
-        pthread_attr_destroy (&w_attr);
+    pthread_attr_destroy(&w_attr);
 cleanup_lock:
-        pthread_mutex_destroy (&priv->container.bad_lock);
+    pthread_mutex_destroy(&priv->container.bad_lock);
 cleanup_cond:
-        pthread_cond_destroy (&priv->container.bad_cond);
+    pthread_cond_destroy(&priv->container.bad_cond);
 out:
-        return -1;
+    return -1;
 }
 
-#define BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH"/quanrantine"
-
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int32_t ret = 0;
-        char *tmp = NULL;
-        struct timeval tv = {0,};
-        br_stub_private_t *priv = NULL;
+    int ret = 0;
+    char *tmp = NULL;
+    struct timeval tv = {
+        0,
+    };
+    br_stub_private_t *priv = NULL;
 
-        if (!this->children) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_CHILD,
-                        "FATAL: no children");
-                goto error_return;
-        }
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_CHILD, NULL);
+        goto error_return;
+    }
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_br_stub_mt_private_t);
-        if (!priv)
-                goto error_return;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_br_stub_mt_private_t);
+    if (!priv)
+        goto error_return;
 
-        priv->local_pool = mem_pool_new (br_stub_local_t, 512);
-        if (!priv->local_pool)
-                goto free_priv;
+    priv->local_pool = mem_pool_new(br_stub_local_t, 512);
+    if (!priv->local_pool)
+        goto free_priv;
 
-        GF_OPTION_INIT ("bitrot", priv->go, bool, free_mempool);
+    GF_OPTION_INIT("bitrot", priv->do_versioning, bool, free_mempool);
 
-        GF_OPTION_INIT ("export", tmp, str, free_mempool);
-        memcpy (priv->export, tmp, strlen (tmp) + 1);
+    GF_OPTION_INIT("export", tmp, str, free_mempool);
 
-        (void) snprintf (priv->stub_basepath, PATH_MAX,
-                         "%s/%s", priv->export, BR_STUB_QUARANTINE_DIR);
+    if (snprintf(priv->export, PATH_MAX, "%s", tmp) >= PATH_MAX)
+        goto free_mempool;
 
-        (void) gettimeofday (&tv, NULL);
+    if (snprintf(priv->stub_basepath, sizeof(priv->stub_basepath), "%s/%s",
+                 priv->export,
+                 BR_STUB_QUARANTINE_DIR) >= sizeof(priv->stub_basepath))
+        goto free_mempool;
 
-        /* boot time is in network endian format */
-        priv->boot[0] = htonl (tv.tv_sec);
-        priv->boot[1] = htonl (tv.tv_usec);
+    (void)gettimeofday(&tv, NULL);
 
-        pthread_mutex_init (&priv->lock, NULL);
-        pthread_cond_init (&priv->cond, NULL);
-        INIT_LIST_HEAD (&priv->squeue);
+    /* boot time is in network endian format */
+    priv->boot[0] = htonl(tv.tv_sec);
+    priv->boot[1] = htonl(tv.tv_usec);
 
-        /* Thread creations need 'this' to be passed so that THIS can be
-         * assigned inside the thread. So setting this->private here.
-         */
-        this->private = priv;
+    pthread_mutex_init(&priv->lock, NULL);
+    pthread_cond_init(&priv->cond, NULL);
+    INIT_LIST_HEAD(&priv->squeue);
+
+    /* Thread creations need 'this' to be passed so that THIS can be
+     * assigned inside the thread. So setting this->private here.
+     */
+    this->private = priv;
+    if (!priv->do_versioning)
+        return 0;
 
-        ret = gf_thread_create (&priv->signth, NULL, br_stub_signth, this);
-        if (ret != 0)
-                goto cleanup_lock;
+    ret = gf_thread_create(&priv->signth, NULL, br_stub_signth, this,
+                           "brssign");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SPAWN_SIGN_THRD_FAILED,
+                NULL);
+        goto cleanup_lock;
+    }
 
-        ret = br_stub_bad_object_container_init (this, priv);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL,
-                        "failed to launch the thread for storing bad gfids");
-                goto cleanup_lock;
+    ret = br_stub_bad_object_container_init(this, priv);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL, NULL);
+        goto cleanup_lock;
+    }
+
+    gf_msg_debug(this->name, 0, "bit-rot stub loaded");
+
+    return 0;
+
+cleanup_lock:
+    pthread_cond_destroy(&priv->cond);
+    pthread_mutex_destroy(&priv->lock);
+free_mempool:
+    mem_pool_destroy(priv->local_pool);
+    priv->local_pool = NULL;
+free_priv:
+    GF_FREE(priv);
+    this->private = NULL;
+error_return:
+    return -1;
+}
+
+/* TODO:
+ * As of now enabling bitrot option does 2 things.
+ * 1) Start the Bitrot Daemon which signs the objects (currently files only)
+ *    upon getting notified by the stub.
+ * 2) Enable versioning of the objects. Object versions (again files only) are
+ *    incremented upon modification.
+ * So object versioning is tied to bitrot daemon's signing. In future, object
+ * versioning might be necessary for other things as well apart from bit-rot
+ * detection (well that's the objective of bringing in object-versioning :)).
+ * In that case, better to make versioning a new option and letting it to be
+ * enabled despite bit-rot detection is not needed.
+ * Ex: ICAP.
+ */
+int32_t
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("bitrot", priv->do_versioning, options, bool, err);
+    if (priv->do_versioning && !priv->signth) {
+        ret = gf_thread_create(&priv->signth, NULL, br_stub_signth, this,
+                               "brssign");
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    BRS_MSG_SPAWN_SIGN_THRD_FAILED, NULL);
+            goto err;
         }
 
-        gf_msg_debug (this->name, 0, "bit-rot stub loaded");
+        ret = br_stub_bad_object_container_init(this, priv);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL,
+                    NULL);
+            goto err;
+        }
+    } else {
+        if (priv->signth) {
+            if (gf_thread_cleanup_xint(priv->signth)) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, BRS_MSG_KILL_SIGN_THREAD,
+                        NULL);
+                priv->signth = 0;
+            }
+        }
+
+        if (priv->container.thread) {
+            if (gf_thread_cleanup_xint(priv->container.thread)) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+            }
+            priv->container.thread = 0;
+        }
+    }
+
+    ret = 0;
+    return ret;
+err:
+    if (priv->signth) {
+        if (gf_thread_cleanup_xint(priv->signth)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+        }
+        priv->signth = 0;
+    }
+
+    if (priv->container.thread) {
+        if (gf_thread_cleanup_xint(priv->container.thread)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+        }
+        priv->container.thread = 0;
+    }
+    ret = -1;
+    return ret;
+}
 
+int
+notify(xlator_t *this, int event, void *data, ...)
+{
+    br_stub_private_t *priv = NULL;
+
+    if (!this)
         return 0;
 
- cleanup_lock:
-        pthread_cond_destroy (&priv->cond);
-        pthread_mutex_destroy (&priv->lock);
- free_mempool:
-        mem_pool_destroy (priv->local_pool);
- free_priv:
-        GF_FREE (priv);
-        this->private = NULL;
- error_return:
-        return -1;
+    priv = this->private;
+    if (!priv)
+        return 0;
+
+    default_notify(this, event, data);
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        int32_t ret = 0;
-	br_stub_private_t *priv = this->private;
-        struct br_stub_signentry *sigstub = NULL;
-        call_stub_t *stub = NULL;
+    int32_t ret = 0;
+    br_stub_private_t *priv = this->private;
+    struct br_stub_signentry *sigstub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (!priv)
-                return;
+    if (!priv)
+        return;
 
-        ret = gf_thread_cleanup_xint (priv->signth);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
-                        "Could not cancel sign serializer thread");
-                goto out;
-        }
+    if (!priv->do_versioning)
+        goto cleanup;
 
-        while (!list_empty (&priv->squeue)) {
-                sigstub = list_first_entry (&priv->squeue,
-                                            struct br_stub_signentry, list);
-                list_del_init (&sigstub->list);
+    ret = gf_thread_cleanup_xint(priv->signth);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
+                NULL);
+        goto out;
+    }
+    priv->signth = 0;
 
-                call_stub_destroy (sigstub->stub);
-                GF_FREE (sigstub);
-        }
+    while (!list_empty(&priv->squeue)) {
+        sigstub = list_first_entry(&priv->squeue, struct br_stub_signentry,
+                                   list);
+        list_del_init(&sigstub->list);
 
-        pthread_mutex_destroy (&priv->lock);
-        pthread_cond_destroy (&priv->cond);
+        call_stub_destroy(sigstub->stub);
+        GF_FREE(sigstub);
+    }
 
-        ret = gf_thread_cleanup_xint (priv->container.thread);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
-                        "Could not cancel sign serializer thread");
-                goto out;
-        }
+    ret = gf_thread_cleanup_xint(priv->container.thread);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
+                NULL);
+        goto out;
+    }
 
-        while (!list_empty (&priv->container.bad_queue)) {
-                stub = list_first_entry (&priv->container.bad_queue, call_stub_t,
-                                         list);
-                list_del_init (&stub->list);
-                call_stub_destroy (stub);
-        };
+    priv->container.thread = 0;
 
-        pthread_mutex_destroy (&priv->container.bad_lock);
-        pthread_cond_destroy (&priv->container.bad_cond);
+    while (!list_empty(&priv->container.bad_queue)) {
+        stub = list_first_entry(&priv->container.bad_queue, call_stub_t, list);
+        list_del_init(&stub->list);
+        call_stub_destroy(stub);
+    }
 
-        this->private = NULL;
-	GF_FREE (priv);
+    pthread_mutex_destroy(&priv->container.bad_lock);
+    pthread_cond_destroy(&priv->container.bad_cond);
 
- out:
-	return;
+cleanup:
+    pthread_mutex_destroy(&priv->lock);
+    pthread_cond_destroy(&priv->cond);
+
+    if (priv->local_pool) {
+        mem_pool_destroy(priv->local_pool);
+        priv->local_pool = NULL;
+    }
+
+    this->private = NULL;
+    GF_FREE(priv);
+
+out:
+    return;
 }
 
 static int
-br_stub_alloc_versions (br_version_t **obuf,
-                        br_signature_t **sbuf, size_t signaturelen)
+br_stub_alloc_versions(br_version_t **obuf, br_signature_t **sbuf,
+                       size_t signaturelen)
 {
-        void *mem = NULL;
-        size_t size = 0;
+    void *mem = NULL;
+    size_t size = 0;
 
-        if (obuf)
-                size += sizeof (br_version_t);
-        if (sbuf)
-                size += sizeof (br_signature_t) + signaturelen;
+    if (obuf)
+        size += sizeof(br_version_t);
+    if (sbuf)
+        size += sizeof(br_signature_t) + signaturelen;
 
-        mem = GF_CALLOC (1, size, gf_br_stub_mt_version_t);
-        if (!mem)
-                goto error_return;
+    mem = GF_CALLOC(1, size, gf_br_stub_mt_version_t);
+    if (!mem)
+        goto error_return;
 
-        if (obuf) {
-                *obuf = (br_version_t *)mem;
-                mem = ((char *)mem + sizeof (br_version_t));
-        }
-        if (sbuf) {
-                *sbuf = (br_signature_t *)mem;
-        }
+    if (obuf) {
+        *obuf = (br_version_t *)mem;
+        mem = ((char *)mem + sizeof(br_version_t));
+    }
+    if (sbuf) {
+        *sbuf = (br_signature_t *)mem;
+    }
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static void
-br_stub_dealloc_versions (void *mem)
+br_stub_dealloc_versions(void *mem)
 {
-        GF_FREE (mem);
+    GF_FREE(mem);
 }
 
 static br_stub_local_t *
-br_stub_alloc_local (xlator_t *this)
+br_stub_alloc_local(xlator_t *this)
 {
-        br_stub_private_t *priv = this->private;
+    br_stub_private_t *priv = this->private;
 
-        return mem_get0 (priv->local_pool);
+    return mem_get0(priv->local_pool);
 }
 
 static void
-br_stub_dealloc_local (br_stub_local_t *ptr)
+br_stub_dealloc_local(br_stub_local_t *ptr)
 {
-        mem_put (ptr);
+    if (!ptr)
+        return;
+
+    mem_put(ptr);
 }
 
 static int
-br_stub_prepare_version_request (xlator_t *this, dict_t *dict,
+br_stub_prepare_version_request(xlator_t *this, dict_t *dict,
                                 br_version_t *obuf, unsigned long oversion)
 {
-        br_stub_private_t *priv = NULL;
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
-        br_set_ongoingversion (obuf, oversion, priv->boot);
+    priv = this->private;
+    br_set_ongoingversion(obuf, oversion, priv->boot);
 
-        return dict_set_static_bin (dict, BITROT_CURRENT_VERSION_KEY,
-                                    (void *)obuf, sizeof (br_version_t));
+    return dict_set_bin(dict, BITROT_CURRENT_VERSION_KEY, (void *)obuf,
+                        sizeof(br_version_t));
 }
 
 static int
-br_stub_prepare_signing_request (dict_t *dict,
-                                 br_signature_t *sbuf,
-                                 br_isignature_t *sign, size_t signaturelen)
+br_stub_prepare_signing_request(dict_t *dict, br_signature_t *sbuf,
+                                br_isignature_t *sign, size_t signaturelen)
 {
-        size_t size = 0;
+    size_t size = 0;
 
-        br_set_signature (sbuf, sign, signaturelen, &size);
+    br_set_signature(sbuf, sign, signaturelen, &size);
 
-        return dict_set_static_bin (dict, BITROT_SIGNING_VERSION_KEY,
-                                    (void *)sbuf, size);
+    return dict_set_bin(dict, BITROT_SIGNING_VERSION_KEY, (void *)sbuf, size);
 }
 
 /**
@@ -325,224 +446,245 @@ br_stub_prepare_signing_request (dict_t *dict,
  * initializes the transient inode version.
  */
 static int
-br_stub_init_inode_versions (xlator_t *this, fd_t *fd, inode_t *inode,
-                             unsigned long version, gf_boolean_t markdirty,
-                             gf_boolean_t bad_object)
-{
-        int32_t ret = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        ctx = GF_CALLOC (1, sizeof (br_stub_inode_ctx_t),
-                         gf_br_stub_mt_inode_ctx_t);
-        if (!ctx)
-                goto error_return;
-
-        INIT_LIST_HEAD (&ctx->fd_list);
-        (markdirty) ? __br_stub_mark_inode_dirty (ctx)
-                : __br_stub_mark_inode_synced (ctx);
-        __br_stub_set_ongoing_version (ctx, version);
-
-        if (bad_object)
-                __br_stub_mark_object_bad (ctx);
-
-        if (fd) {
-                ret = br_stub_add_fd_to_inode (this, fd, ctx);
-                if (ret)
-                        goto free_ctx;
-        }
+br_stub_init_inode_versions(xlator_t *this, fd_t *fd, inode_t *inode,
+                            unsigned long version, gf_boolean_t markdirty,
+                            gf_boolean_t bad_object, uint64_t *ctx_addr)
+{
+    int32_t ret = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    ctx = GF_CALLOC(1, sizeof(br_stub_inode_ctx_t), gf_br_stub_mt_inode_ctx_t);
+    if (!ctx)
+        goto error_return;
+
+    INIT_LIST_HEAD(&ctx->fd_list);
+    (markdirty) ? __br_stub_mark_inode_dirty(ctx)
+                : __br_stub_mark_inode_synced(ctx);
+    __br_stub_set_ongoing_version(ctx, version);
+
+    if (bad_object)
+        __br_stub_mark_object_bad(ctx);
 
-        ret = br_stub_set_inode_ctx (this, inode, ctx);
+    if (fd) {
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
         if (ret)
-                goto free_ctx;
-        return 0;
+            goto free_ctx;
+    }
+
+    ret = br_stub_set_inode_ctx(this, inode, ctx);
+    if (ret)
+        goto free_ctx;
+
+    if (ctx_addr)
+        *ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return 0;
 
 free_ctx:
-        GF_FREE (ctx);
- error_return:
-        return -1;
+    GF_FREE(ctx);
+error_return:
+    return -1;
 }
 
 /**
  * modify the ongoing version of an inode.
  */
 static int
-br_stub_mod_inode_versions (xlator_t *this,
-                            fd_t *fd, inode_t *inode, unsigned long version)
+br_stub_mod_inode_versions(xlator_t *this, fd_t *fd, inode_t *inode,
+                           unsigned long version)
 {
-        int32_t ret = -1;
-        br_stub_inode_ctx_t *ctx = 0;
+    int32_t ret = -1;
+    br_stub_inode_ctx_t *ctx = 0;
 
-        LOCK (&inode->lock);
-        {
-                ctx = __br_stub_get_ongoing_version_ctx (this, inode, NULL);
-                if (ctx == NULL)
-                        goto unblock;
-                if (__br_stub_is_inode_dirty (ctx)) {
-                        __br_stub_set_ongoing_version (ctx, version);
-                        __br_stub_mark_inode_synced (ctx);
-                }
-
-                ret = 0;
+    LOCK(&inode->lock);
+    {
+        ctx = __br_stub_get_ongoing_version_ctx(this, inode, NULL);
+        if (ctx == NULL)
+            goto unblock;
+        if (__br_stub_is_inode_dirty(ctx)) {
+            __br_stub_set_ongoing_version(ctx, version);
+            __br_stub_mark_inode_synced(ctx);
         }
+
+        ret = 0;
+    }
 unblock:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 static void
-br_stub_fill_local (br_stub_local_t *local,
-                    call_stub_t *stub, fd_t *fd, inode_t *inode, uuid_t gfid,
-                    int versioningtype, unsigned long memversion)
+br_stub_fill_local(br_stub_local_t *local, call_stub_t *stub, fd_t *fd,
+                   inode_t *inode, uuid_t gfid, int versioningtype,
+                   unsigned long memversion)
 {
-        local->fopstub = stub;
-        local->versioningtype = versioningtype;
-        local->u.context.version = memversion;
-        if (fd)
-                local->u.context.fd = fd_ref (fd);
-        if (inode)
-                local->u.context.inode = inode_ref (inode);
-        gf_uuid_copy (local->u.context.gfid, gfid);
+    local->fopstub = stub;
+    local->versioningtype = versioningtype;
+    local->u.context.version = memversion;
+    if (fd)
+        local->u.context.fd = fd_ref(fd);
+    if (inode)
+        local->u.context.inode = inode_ref(inode);
+    gf_uuid_copy(local->u.context.gfid, gfid);
 }
 
 static void
-br_stub_cleanup_local (br_stub_local_t *local)
-{
-        local->fopstub = NULL;
-        local->versioningtype = 0;
-        local->u.context.version = 0;
-        if (local->u.context.fd) {
-                fd_unref (local->u.context.fd);
-                local->u.context.fd = NULL;
-        }
-        if (local->u.context.inode) {
-                inode_unref (local->u.context.inode);
-                local->u.context.inode = NULL;
-        }
-        memset (local->u.context.gfid, '\0', sizeof (uuid_t));
+br_stub_cleanup_local(br_stub_local_t *local)
+{
+    if (!local)
+        return;
+
+    local->fopstub = NULL;
+    local->versioningtype = 0;
+    local->u.context.version = 0;
+    if (local->u.context.fd) {
+        fd_unref(local->u.context.fd);
+        local->u.context.fd = NULL;
+    }
+    if (local->u.context.inode) {
+        inode_unref(local->u.context.inode);
+        local->u.context.inode = NULL;
+    }
+    memset(local->u.context.gfid, '\0', sizeof(uuid_t));
 }
 
 static int
-br_stub_need_versioning (xlator_t *this,
-                         fd_t *fd, gf_boolean_t *versioning,
-                         gf_boolean_t *modified, br_stub_inode_ctx_t **ctx)
-{
-        int32_t ret      = -1;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *c = NULL;
-
-        *versioning = _gf_false;
-        *modified = _gf_false;
-
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED, "failed to get the "
-                        "inode context for the inode %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto error_return;
+br_stub_need_versioning(xlator_t *this, fd_t *fd, gf_boolean_t *versioning,
+                        gf_boolean_t *modified, br_stub_inode_ctx_t **ctx)
+{
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *c = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    *versioning = _gf_false;
+    *modified = _gf_false;
+
+    /* Bitrot stub inode context was initialized only in lookup, create
+     * and mknod cbk path. Object versioning was enabled by default
+     * irrespective of bitrot enabled or not. But it's made optional now.
+     * As a consequence there could be cases where getting inode ctx would
+     * fail because it's not set yet.
+     * e.g., If versioning (with bitrot enable) is enabled while I/O is
+     * happening, it could directly get other fops like writev without
+     * lookup, where getting inode ctx would fail. Hence initialize the
+     * inode ctx on failure to get ctx. This is done in all places where
+     * applicable.
+     */
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            goto error_return;
         }
+    }
 
-        c = (br_stub_inode_ctx_t *) (long) ctx_addr;
+    c = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&fd->inode->lock);
-        {
-                if (__br_stub_is_inode_dirty (c))
-                        *versioning = _gf_true;
-                if (__br_stub_is_inode_modified (c))
-                        *modified = _gf_true;
-        }
-        UNLOCK (&fd->inode->lock);
+    LOCK(&fd->inode->lock);
+    {
+        if (__br_stub_is_inode_dirty(c))
+            *versioning = _gf_true;
+        if (__br_stub_is_inode_modified(c))
+            *modified = _gf_true;
+    }
+    UNLOCK(&fd->inode->lock);
 
-        if (ctx)
-                *ctx = c;
-        return 0;
+    if (ctx)
+        *ctx = c;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static int32_t
-br_stub_anon_fd_ctx (xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
-{
-        int32_t  ret = -1;
-        br_stub_fd_t *br_stub_fd = NULL;
-
-        br_stub_fd = br_stub_fd_ctx_get (this, fd);
-        if (!br_stub_fd) {
-                ret = br_stub_add_fd_to_inode (this, fd, ctx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_ADD_FD_TO_INODE, "failed to add fd to "
-                                "the inode (gfid: %s)",
-                                uuid_utoa (fd->inode->gfid));
-                        goto out;
-                }
+br_stub_anon_fd_ctx(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    br_stub_fd = br_stub_fd_ctx_get(this, fd);
+    if (!br_stub_fd) {
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ADD_FD_TO_INODE,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-br_stub_versioning_prep (call_frame_t *frame,
-                         xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+br_stub_versioning_prep(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        br_stub_inode_ctx_t *ctx)
 {
-        int32_t          ret = -1;
-        br_stub_local_t *local   = NULL;
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        local = br_stub_alloc_local (this);
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_NO_MEMORY,
-                        "local allocation failed (gfid: %s)",
-                        uuid_utoa (fd->inode->gfid));
-                goto error_return;
-        }
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_NO_MEMORY, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), NULL);
+        goto error_return;
+    }
 
-        if (fd_is_anonymous (fd)) {
-                ret = br_stub_anon_fd_ctx (this, fd, ctx);
-                if (ret)
-                        goto free_local;
-        }
+    if (fd_is_anonymous(fd)) {
+        ret = br_stub_anon_fd_ctx(this, fd, ctx);
+        if (ret)
+            goto free_local;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        return 0;
+    return 0;
 
- free_local:
-        br_stub_dealloc_local (local);
- error_return:
-        return -1;
+free_local:
+    br_stub_dealloc_local(local);
+error_return:
+    return -1;
 }
 
 static int
-br_stub_mark_inode_modified (xlator_t *this, br_stub_local_t *local)
+br_stub_mark_inode_modified(xlator_t *this, br_stub_local_t *local)
 {
-        fd_t                *fd       = NULL;
-        int32_t              ret      = 0;
-        uint64_t             ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx      = NULL;
+    fd_t *fd = NULL;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
 
-        fd = local->u.context.fd;
+    fd = local->u.context.fd;
 
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0)
-                goto error_return;
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
+        if (ret)
+            goto error_return;
+    }
 
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&fd->inode->lock);
-        {
-                __br_stub_set_inode_modified (ctx);
-        }
-        UNLOCK (&fd->inode->lock);
+    LOCK(&fd->inode->lock);
+    {
+        __br_stub_set_inode_modified(ctx);
+    }
+    UNLOCK(&fd->inode->lock);
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 /**
@@ -554,65 +696,68 @@ br_stub_mark_inode_modified (xlator_t *this, br_stub_local_t *local)
  * and error is returned upwards.
  */
 static int
-br_stub_check_bad_object (xlator_t *this, inode_t *inode, int32_t *op_ret,
-                           int32_t *op_errno)
-{
-        int ret = -1;
-
-        ret = br_stub_is_bad_object (this, inode);
-        if (ret == -2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJECT_ACCESS,
-                        "%s is a bad object. Returning",
-                        uuid_utoa (inode->gfid));
-                *op_ret = -1;
-                *op_errno = EIO;
-        }
-
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED, "could not get inode"
-                        " context for %s", uuid_utoa (inode->gfid));
-                *op_ret = -1;
-                *op_errno = EINVAL;
+br_stub_check_bad_object(xlator_t *this, inode_t *inode, int32_t *op_ret,
+                         int32_t *op_errno)
+{
+    int ret = -1;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    ret = br_stub_is_bad_object(this, inode);
+    if (ret == -2) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJECT_ACCESS,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        *op_ret = -1;
+        *op_errno = EIO;
+    }
+
+    if (ret == -1) {
+        ret = br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                          _gf_false, NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                    uuid_utoa(inode->gfid), NULL);
+            *op_ret = -1;
+            *op_errno = EINVAL;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 /**
  * callback for inode/fd versioning
  */
 int
-br_stub_fd_incversioning_cbk (call_frame_t *frame,
-                              void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, dict_t *xdata)
-{
-        fd_t            *fd      = NULL;
-        inode_t         *inode   = NULL;
-        unsigned long    version = 0;
-        br_stub_local_t *local   = NULL;
-
-        local = (br_stub_local_t *)frame->local;
-        if (op_ret < 0)
-                goto done;
-        fd      = local->u.context.fd;
-        inode   = local->u.context.inode;
-        version = local->u.context.version;
-
-        op_ret = br_stub_mod_inode_versions (this, fd, inode, version);
-        if (op_ret < 0)
-                op_errno = EINVAL;
-
- done:
-        if (op_ret < 0) {
-                frame->local = NULL;
-                call_unwind_error (local->fopstub, -1, op_errno);
-                br_stub_cleanup_local (local);
-                br_stub_dealloc_local (local);
-        } else {
-                call_resume (local->fopstub);
-        }
-        return 0;
+br_stub_fd_incversioning_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata)
+{
+    fd_t *fd = NULL;
+    inode_t *inode = NULL;
+    unsigned long version = 0;
+    br_stub_local_t *local = NULL;
+
+    local = (br_stub_local_t *)frame->local;
+    if (op_ret < 0)
+        goto done;
+    fd = local->u.context.fd;
+    inode = local->u.context.inode;
+    version = local->u.context.version;
+
+    op_ret = br_stub_mod_inode_versions(this, fd, inode, version);
+    if (op_ret < 0)
+        op_errno = EINVAL;
+
+done:
+    if (op_ret < 0) {
+        frame->local = NULL;
+        call_unwind_error(local->fopstub, -1, op_errno);
+        br_stub_cleanup_local(local);
+        br_stub_dealloc_local(local);
+    } else {
+        call_resume(local->fopstub);
+    }
+    return 0;
 }
 
 /**
@@ -645,102 +790,101 @@ br_stub_fd_incversioning_cbk (call_frame_t *frame,
 /**
  * perform full or incremental versioning on an inode pointd by an
  * fd. incremental versioning is done when an inode is dirty and a
- * writeback is trigerred.
+ * writeback is triggered.
  */
 
 int
-br_stub_fd_versioning (xlator_t *this, call_frame_t *frame,
-                       call_stub_t *stub, dict_t *dict, fd_t *fd,
-                       br_stub_version_cbk *callback, unsigned long memversion,
-                       int versioningtype, int durable)
+br_stub_fd_versioning(xlator_t *this, call_frame_t *frame, call_stub_t *stub,
+                      dict_t *dict, fd_t *fd, br_stub_version_cbk *callback,
+                      unsigned long memversion, int versioningtype, int durable)
 {
-        int32_t          ret   = -1;
-        int              flags = 0;
-        dict_t          *xdata = NULL;
-        br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+    int flags = 0;
+    dict_t *xdata = NULL;
+    br_stub_local_t *local = NULL;
 
-        xdata = dict_new ();
-        if (!xdata)
-                goto done;
+    xdata = dict_new();
+    if (!xdata)
+        goto done;
 
-        ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
-        if (ret)
-                goto dealloc_xdata;
+    ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+    if (ret)
+        goto dealloc_xdata;
 
-        if (durable) {
-                ret = dict_set_int32 (xdata, GLUSTERFS_DURABLE_OP, 0);
-                if (ret)
-                        goto dealloc_xdata;
-        }
+    if (durable) {
+        ret = dict_set_int32(xdata, GLUSTERFS_DURABLE_OP, 0);
+        if (ret)
+            goto dealloc_xdata;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        br_stub_fill_local (local, stub, fd,
-                            fd->inode, fd->inode->gfid,
-                            versioningtype, memversion);
+    br_stub_fill_local(local, stub, fd, fd->inode, fd->inode->gfid,
+                       versioningtype, memversion);
 
-        STACK_WIND (frame, callback,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
+    STACK_WIND(frame, callback, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
 
-        ret = 0;
+    ret = 0;
 
- dealloc_xdata:
-        dict_unref (xdata);
- done:
-        return ret;
+dealloc_xdata:
+    dict_unref(xdata);
+done:
+    return ret;
 }
 
 static int
-br_stub_perform_incversioning (xlator_t *this,
-                               call_frame_t *frame, call_stub_t *stub,
-                               fd_t *fd, br_stub_inode_ctx_t *ctx)
-{
-        int32_t          ret               = -1;
-        dict_t          *dict              = NULL;
-        br_version_t    *obuf              = NULL;
-        unsigned long    writeback_version = 0;
-        int              op_errno          = 0;
-        br_stub_local_t *local             = NULL;
-
-        op_errno = EINVAL;
-        local = frame->local;
-
-        writeback_version = __br_stub_writeback_version (ctx);
-
-        op_errno = ENOMEM;
-        dict = dict_new ();
-        if (!dict)
-                goto done;
-        ret = br_stub_alloc_versions (&obuf, NULL, 0);
-        if (ret)
-                goto dealloc_dict;
-        ret = br_stub_prepare_version_request (this, dict,
-                                               obuf, writeback_version);
-        if (ret)
-                goto dealloc_versions;
-
-        ret = br_stub_fd_versioning
-                (this, frame, stub, dict,
-                 fd, br_stub_fd_incversioning_cbk, writeback_version,
-                 BR_STUB_INCREMENTAL_VERSIONING, !WRITEBACK_DURABLE);
-
- dealloc_versions:
-        br_stub_dealloc_versions (obuf);
- dealloc_dict:
-        dict_unref (dict);
- done:
-        if (ret) {
-                if (local)
-                        frame->local = NULL;
-                call_unwind_error (stub, -1, op_errno);
-                if (local) {
-                        br_stub_cleanup_local (local);
-                        br_stub_dealloc_local (local);
-                }
+br_stub_perform_incversioning(xlator_t *this, call_frame_t *frame,
+                              call_stub_t *stub, fd_t *fd,
+                              br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    br_version_t *obuf = NULL;
+    unsigned long writeback_version = 0;
+    int op_errno = 0;
+    br_stub_local_t *local = NULL;
+
+    op_errno = EINVAL;
+    local = frame->local;
+
+    writeback_version = __br_stub_writeback_version(ctx);
+
+    op_errno = ENOMEM;
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    ret = br_stub_alloc_versions(&obuf, NULL, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+    ret = br_stub_prepare_version_request(this, dict, obuf, writeback_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_VERSION_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        br_stub_dealloc_versions(obuf);
+        goto out;
+    }
+
+    ret = br_stub_fd_versioning(
+        this, frame, stub, dict, fd, br_stub_fd_incversioning_cbk,
+        writeback_version, BR_STUB_INCREMENTAL_VERSIONING, !WRITEBACK_DURABLE);
+out:
+    if (dict)
+        dict_unref(dict);
+    if (ret) {
+        if (local)
+            frame->local = NULL;
+        call_unwind_error(stub, -1, op_errno);
+        if (local) {
+            br_stub_cleanup_local(local);
+            br_stub_dealloc_local(local);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 /** {{{ */
@@ -748,232 +892,271 @@ br_stub_perform_incversioning (xlator_t *this,
 /* fsetxattr() */
 
 int32_t
-br_stub_perform_objsign (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, dict_t *dict, int flags, dict_t *xdata)
+br_stub_perform_objsign(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        dict_t *dict, int flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsetxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr, fd,
-                    dict, flags, xdata);
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
 
-        dict_unref (xdata);
-        return 0;
+    dict_unref(xdata);
+    return 0;
 }
 
 void *
-br_stub_signth (void *arg)
+br_stub_signth(void *arg)
 {
-        xlator_t *this = arg;
-        br_stub_private_t *priv = this->private;
-        struct br_stub_signentry *sigstub = NULL;
+    xlator_t *this = arg;
+    br_stub_private_t *priv = this->private;
+    struct br_stub_signentry *sigstub = NULL;
+
+    THIS = this;
+    while (1) {
+        /*
+         * Disabling bit-rot feature leads to this particular thread
+         * getting cleaned up by reconfigure via a call to the function
+         * gf_thread_cleanup_xint (which in turn calls pthread_cancel
+         * and pthread_join). But, if this thread had held the mutex
+         * &priv->lock at the time of cancellation, then it leads to
+         * deadlock in future when bit-rot feature is enabled (which
+         * again spawns this thread which cant hold the lock as the
+         * mutex is still held by the previous instance of the thread
+         * which got killed). Also, the br_stub_handle_object_signature
+         * function which is called whenever file has to be signed
+         * also gets blocked as it too attempts to acquire &priv->lock.
+         *
+         * So, arrange for the lock to be unlocked as part of the
+         * cleanup of this thread using pthread_cleanup_push and
+         * pthread_cleanup_pop.
+         */
+        pthread_cleanup_push(br_stub_lock_cleaner, &priv->lock);
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->squeue))
+                pthread_cond_wait(&priv->cond, &priv->lock);
 
-        THIS = this;
-        while (1) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->squeue))
-                                pthread_cond_wait (&priv->cond, &priv->lock);
+            sigstub = list_first_entry(&priv->squeue, struct br_stub_signentry,
+                                       list);
+            list_del_init(&sigstub->list);
+        }
+        pthread_mutex_unlock(&priv->lock);
+        pthread_cleanup_pop(0);
 
-                        sigstub = list_first_entry
-                                (&priv->squeue, struct br_stub_signentry, list);
-                        list_del_init (&sigstub->list);
-                }
-                pthread_mutex_unlock (&priv->lock);
+        call_resume(sigstub->stub);
 
-                call_resume (sigstub->stub);
+        GF_FREE(sigstub);
+    }
 
-                GF_FREE (sigstub);
-        }
+    return NULL;
+}
+
+static gf_boolean_t
+br_stub_internal_xattr(dict_t *dict)
+{
+    if (dict_get(dict, GLUSTERFS_SET_OBJECT_SIGNATURE) ||
+        dict_get(dict, GLUSTERFS_GET_OBJECT_SIGNATURE) ||
+        dict_get(dict, BR_REOPEN_SIGN_HINT_KEY) ||
+        dict_get(dict, BITROT_OBJECT_BAD_KEY) ||
+        dict_get(dict, BITROT_SIGNING_VERSION_KEY) ||
+        dict_get(dict, BITROT_CURRENT_VERSION_KEY))
+        return _gf_true;
 
-        return NULL;
+    return _gf_false;
 }
 
 int
-orderq (struct list_head *elem1, struct list_head *elem2)
+orderq(struct list_head *elem1, struct list_head *elem2)
 {
-        struct br_stub_signentry *s1 = NULL;
-        struct br_stub_signentry *s2 = NULL;
+    struct br_stub_signentry *s1 = NULL;
+    struct br_stub_signentry *s2 = NULL;
 
-        s1 = list_entry (elem1, struct br_stub_signentry, list);
-        s2 = list_entry (elem2, struct br_stub_signentry, list);
+    s1 = list_entry(elem1, struct br_stub_signentry, list);
+    s2 = list_entry(elem2, struct br_stub_signentry, list);
 
-        return (s1->v > s2->v);
+    return (s1->v > s2->v);
 }
 
 static int
-br_stub_compare_sign_version (xlator_t *this,
-                              inode_t *inode,
-                              br_signature_t *sbuf,
-                              dict_t *dict, int *fakesuccess)
-{
-        int32_t              ret     = -1;
-        uint64_t             tmp_ctx = 0;
-        gf_boolean_t         invalid = _gf_false;
-        br_stub_inode_ctx_t *ctx     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, sbuf, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        ret = br_stub_get_inode_ctx (this, inode, &tmp_ctx);
-        if (ret) {
-                dict_del (dict, BITROT_SIGNING_VERSION_KEY);
-                goto out;
-        }
-
-        ctx = (br_stub_inode_ctx_t *)(long)tmp_ctx;
-
-        LOCK (&inode->lock);
-        {
-                if (ctx->currentversion < sbuf->signedversion) {
-                        invalid = _gf_true;
-                } else if (ctx->currentversion > sbuf->signedversion) {
-                        gf_msg_debug (this->name, 0, "\"Signing version\" "
-                                      "(%lu) lower than \"Current version \" "
-                                      "(%lu)", ctx->currentversion,
-                                      sbuf->signedversion);
-                        *fakesuccess = 1;
-                }
-        }
-        UNLOCK (&inode->lock);
-
-        if (invalid) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_SIGN_VERSION_ERROR, "Signing version exceeds "
-                        "current version [%lu > %lu]", sbuf->signedversion,
-                        ctx->currentversion);
-        }
+br_stub_compare_sign_version(xlator_t *this, inode_t *inode,
+                             br_signature_t *sbuf, dict_t *dict,
+                             int *fakesuccess)
+{
+    int32_t ret = -1;
+    uint64_t tmp_ctx = 0;
+    gf_boolean_t invalid = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, sbuf, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    ret = br_stub_get_inode_ctx(this, inode, &tmp_ctx);
+    if (ret) {
+        dict_del(dict, BITROT_SIGNING_VERSION_KEY);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)tmp_ctx;
+
+    LOCK(&inode->lock);
+    {
+        if (ctx->currentversion < sbuf->signedversion) {
+            invalid = _gf_true;
+        } else if (ctx->currentversion > sbuf->signedversion) {
+            gf_msg_debug(this->name, 0,
+                         "\"Signing version\" "
+                         "(%lu) lower than \"Current version \" "
+                         "(%lu)",
+                         ctx->currentversion, sbuf->signedversion);
+            *fakesuccess = 1;
+        }
+    }
+    UNLOCK(&inode->lock);
+
+    if (invalid) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SIGN_VERSION_ERROR,
+                "Signing-ver=%lu", sbuf->signedversion, "current-ver=%lu",
+                ctx->currentversion, NULL);
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int
-br_stub_prepare_signature (xlator_t *this,
-                           dict_t *dict, inode_t *inode,
-                           br_isignature_t *sign, int *fakesuccess)
+br_stub_prepare_signature(xlator_t *this, dict_t *dict, inode_t *inode,
+                          br_isignature_t *sign, int *fakesuccess)
 {
-        int32_t ret = 0;
-        size_t signaturelen = 0;
-        br_signature_t *sbuf = NULL;
-
-        if (!br_is_signature_type_valid (sign->signaturetype))
-                goto error_return;
-
-        signaturelen = sign->signaturelen;
-        ret = br_stub_alloc_versions (NULL, &sbuf, signaturelen);
-        if (ret)
-                goto error_return;
-        ret = br_stub_prepare_signing_request (dict, sbuf, sign, signaturelen);
-        if (ret)
-                goto dealloc_versions;
+    int32_t ret = -1;
+    size_t signaturelen = 0;
+    br_signature_t *sbuf = NULL;
 
-        ret = br_stub_compare_sign_version (this, inode,
-                                            sbuf, dict, fakesuccess);
-        if (ret)
-                goto dealloc_versions;
+    if (!br_is_signature_type_valid(sign->signaturetype))
+        goto out;
 
-        return 0;
-
- dealloc_versions:
-        br_stub_dealloc_versions (sbuf);
- error_return:
-        return -1;
+    signaturelen = sign->signaturelen;
+    ret = br_stub_alloc_versions(NULL, &sbuf, signaturelen);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = br_stub_prepare_signing_request(dict, sbuf, sign, signaturelen);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SIGN_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+        br_stub_dealloc_versions(sbuf);
+        goto out;
+    }
+
+    /* At this point sbuf has been added to dict, so the memory will be freed
+     * when the data from the dict is destroyed
+     */
+    ret = br_stub_compare_sign_version(this, inode, sbuf, dict, fakesuccess);
+out:
+    return ret;
 }
 
 static void
-br_stub_handle_object_signature (call_frame_t *frame,
-                                 xlator_t *this, fd_t *fd, dict_t *dict,
-                                 br_isignature_t *sign, dict_t *xdata)
-{
-        int32_t                   ret         = -1;
-        int32_t                   op_ret      = -1;
-        int32_t                   op_errno    = EINVAL;
-        int                       fakesuccess = 0;
-        br_stub_private_t        *priv        = NULL;
-        struct br_stub_signentry *sigstub     = NULL;
-
-        priv = this->private;
-
-        if (frame->root->pid != GF_CLIENT_PID_BITD)
-                goto dofop;
-
-        ret = br_stub_prepare_signature (this, dict,
-                                         fd->inode, sign, &fakesuccess);
-        if (ret)
-                goto dofop;
-        if (fakesuccess) {
-                op_ret = op_errno = 0;
-                goto dofop;
-        }
-
-        dict_del (dict, GLUSTERFS_SET_OBJECT_SIGNATURE);
-
-        ret = -1;
-        if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata)
-                        goto dofop;
-        } else {
-                dict_ref (xdata);
-        }
+br_stub_handle_object_signature(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                dict_t *dict, br_isignature_t *sign,
+                                dict_t *xdata)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int fakesuccess = 0;
+    br_stub_private_t *priv = NULL;
+    struct br_stub_signentry *sigstub = NULL;
+
+    priv = this->private;
+
+    if (frame->root->pid != GF_CLIENT_PID_BITD) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, BRS_MSG_NON_BITD_PID,
+                "PID=%d", frame->root->pid, NULL);
+        goto dofop;
+    }
+
+    ret = br_stub_prepare_signature(this, dict, fd->inode, sign, &fakesuccess);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SIGN_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto dofop;
+    }
+    if (fakesuccess) {
+        op_ret = op_errno = 0;
+        goto dofop;
+    }
+
+    dict_del(dict, GLUSTERFS_SET_OBJECT_SIGNATURE);
+
+    ret = -1;
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto dofop;
+    } else {
+        dict_ref(xdata);
+    }
 
-        ret = dict_set_int32 (xdata, GLUSTERFS_DURABLE_OP, 0);
-        if (ret)
-                goto unref_dict;
+    ret = dict_set_int32(xdata, GLUSTERFS_DURABLE_OP, 0);
+    if (ret)
+        goto unref_dict;
 
-        /* prepare dispatch stub to order object signing */
-        sigstub = GF_CALLOC (1, sizeof (*sigstub), gf_br_stub_mt_sigstub_t);
-        if (!sigstub)
-                goto unref_dict;
+    /* prepare dispatch stub to order object signing */
+    sigstub = GF_CALLOC(1, sizeof(*sigstub), gf_br_stub_mt_sigstub_t);
+    if (!sigstub)
+        goto unref_dict;
 
-        INIT_LIST_HEAD (&sigstub->list);
-        sigstub->v = ntohl (sign->signedversion);
-        sigstub->stub = fop_fsetxattr_stub (frame, br_stub_perform_objsign,
-                                            fd, dict, 0, xdata);
-        if (!sigstub->stub)
-                goto cleanup_stub;
+    INIT_LIST_HEAD(&sigstub->list);
+    sigstub->v = ntohl(sign->signedversion);
+    sigstub->stub = fop_fsetxattr_stub(frame, br_stub_perform_objsign, fd, dict,
+                                       0, xdata);
+    if (!sigstub->stub)
+        goto cleanup_stub;
 
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_order (&sigstub->list, &priv->squeue, orderq);
-                pthread_cond_signal (&priv->cond);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_order(&sigstub->list, &priv->squeue, orderq);
+        pthread_cond_signal(&priv->cond);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
-        return;
+    return;
 
- cleanup_stub:
-        GF_FREE (sigstub);
- unref_dict:
-        dict_unref (xdata);
- dofop:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+cleanup_stub:
+    GF_FREE(sigstub);
+unref_dict:
+    dict_unref(xdata);
+dofop:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 }
 
 int32_t
-br_stub_fsetxattr_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+br_stub_fsetxattr_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int32_t ret = -1;
-        br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
-        return 0;
+    return 0;
 }
 
 /**
@@ -1011,65 +1194,62 @@ br_stub_fsetxattr_resume (call_frame_t *frame, void *cookie, xlator_t *this,
  * }
  */
 static void
-br_stub_handle_object_reopen (call_frame_t *frame,
-                              xlator_t *this, fd_t *fd, uint32_t val)
-{
-        int32_t              ret         = -1;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        call_stub_t         *stub        = NULL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        br_stub_local_t     *local       = NULL;
-        gf_boolean_t         goback      = _gf_true;
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto unwind;
-
-        LOCK (&fd->inode->lock);
-        {
-                if ((val == BR_OBJECT_REOPEN) && inc_version)
-                        goback = _gf_false;
-                if (val == BR_OBJECT_RESIGN &&
-                    ctx->info_sign == BR_SIGN_NORMAL) {
-                        __br_stub_mark_inode_synced (ctx);
-                        __br_stub_set_inode_modified (ctx);
-                }
-                (void) __br_stub_inode_sign_state (ctx, GF_FOP_FSETXATTR, fd);
-        }
-        UNLOCK (&fd->inode->lock);
-
-        if (goback) {
-                op_ret = op_errno = 0;
-                goto unwind;
-        }
-
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto unwind;
-        local = frame->local;
-
-        stub = fop_fsetxattr_cbk_stub (frame, br_stub_fsetxattr_resume,
-                                       0, 0, NULL);
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for fsetxattr fop (gfid: %s),"
-                        " unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
-
-        (void) br_stub_perform_incversioning (this, frame, stub, fd, ctx);
-        return;
+br_stub_handle_object_reopen(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                             uint32_t val)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    call_stub_t *stub = NULL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    br_stub_local_t *local = NULL;
+    gf_boolean_t goback = _gf_true;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    LOCK(&fd->inode->lock);
+    {
+        if ((val == BR_OBJECT_REOPEN) && inc_version)
+            goback = _gf_false;
+        if (val == BR_OBJECT_RESIGN && ctx->info_sign == BR_SIGN_NORMAL) {
+            __br_stub_mark_inode_synced(ctx);
+            __br_stub_set_inode_modified(ctx);
+        }
+        (void)__br_stub_inode_sign_state(ctx, GF_FOP_FSETXATTR, fd);
+    }
+    UNLOCK(&fd->inode->lock);
+
+    if (goback) {
+        op_ret = op_errno = 0;
+        goto unwind;
+    }
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+    local = frame->local;
+
+    stub = fop_fsetxattr_cbk_stub(frame, br_stub_fsetxattr_resume, 0, 0, NULL);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "fsetxattr gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    (void)br_stub_perform_incversioning(this, frame, stub, fd, ctx);
+    return;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 }
 
 /**
@@ -1080,89 +1260,83 @@ br_stub_handle_object_reopen (call_frame_t *frame,
  * to mark the object as bad.
  */
 int
-br_stub_fsetxattr_bad_object_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, dict_t *xdata)
-{
-        br_stub_local_t *local = NULL;
-        int32_t          ret   = -1;
-
-        local = frame->local;
-        frame->local = NULL;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        /*
-         * What to do if marking the object as bad fails? (i.e. in memory
-         * marking within the inode context. If we are here means fsetxattr
-         * fop has succeeded on disk and the bad object xattr has been set).
-         * We can return failure to scruber, but there is nothing the scrubber
-         * can do with it (it might assume that the on disk setxattr itself has
-         * failed). The main purpose of this operation is to help identify the
-         * bad object by checking the inode context itself (thus avoiding the
-         * necessity of doing a getxattr fop on the disk).
-         *
-         * So as of now, success itself is being returned even though inode
-         * context set operation fails.
-         * In future if there is any change in the policy which can handle this,
-         * then appropriate response should be sent (i.e. success or error).
-         */
-        ret = br_stub_mark_object_bad (this, local->u.context.inode);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_MARK_FAIL,
-                        "failed to mark object %s as bad",
-                        uuid_utoa (local->u.context.inode->gfid));
-
-        ret = br_stub_add (this, local->u.context.inode->gfid);
+br_stub_fsetxattr_bad_object_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    /*
+     * What to do if marking the object as bad fails? (i.e. in memory
+     * marking within the inode context. If we are here means fsetxattr
+     * fop has succeeded on disk and the bad object xattr has been set).
+     * We can return failure to scruber, but there is nothing the scrubber
+     * can do with it (it might assume that the on disk setxattr itself has
+     * failed). The main purpose of this operation is to help identify the
+     * bad object by checking the inode context itself (thus avoiding the
+     * necessity of doing a getxattr fop on the disk).
+     *
+     * So as of now, success itself is being returned even though inode
+     * context set operation fails.
+     * In future if there is any change in the policy which can handle this,
+     * then appropriate response should be sent (i.e. success or error).
+     */
+    ret = br_stub_mark_object_bad(this, local->u.context.inode);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_MARK_FAIL,
+                "gfid=%s", uuid_utoa(local->u.context.inode->gfid), NULL);
+
+    ret = br_stub_add(this, local->u.context.inode->gfid);
 
 unwind:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 static int32_t
-br_stub_handle_bad_object_key (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                               dict_t *dict, int flags, dict_t *xdata)
-{
-        br_stub_local_t *local    = NULL;
-        int32_t          op_ret   = -1;
-        int32_t         op_errno = EINVAL;
-
-        if (frame->root->pid != GF_CLIENT_PID_SCRUB) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_NON_SCRUB_BAD_OBJ_MARK, "bad object marking "
-                        "on %s is not from the scrubber",
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        local = br_stub_alloc_local (this);
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_MEMORY,
-                        "failed to allocate memory for fsetxattr on %s",
-                        uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+br_stub_handle_bad_object_key(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (frame->root->pid != GF_CLIENT_PID_SCRUB) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_NON_SCRUB_BAD_OBJ_MARK,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unwind;
+    }
+
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "fsetxattr gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        br_stub_fill_local (local, NULL, fd, fd->inode,
-                            fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-        frame->local = local;
+    br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                       BR_STUB_NO_VERSIONING, 0);
+    frame->local = local;
 
-        STACK_WIND (frame, br_stub_fsetxattr_bad_object_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_fsetxattr_bad_object_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-
 /**
  * As of now, versioning is done by the stub (though as a setxattr
  * operation) as part of inode modification operations such as writev,
@@ -1178,84 +1352,121 @@ unwind:
  *
  */
 static int32_t
-br_stub_handle_internal_xattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                               char *key)
+br_stub_handle_internal_xattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              char *key)
 {
-        int32_t          op_ret   = -1;
-        int32_t          op_errno = EINVAL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                BRS_MSG_SET_INTERNAL_XATTR, "setxattr called"
-                " on the internal xattr %s for inode %s", key,
-                uuid_utoa (fd->inode->gfid));
+    gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_INTERNAL_XATTR,
+            "setxattr key=%s", key, "inode-gfid=%s", uuid_utoa(fd->inode->gfid),
+            NULL);
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-int
-br_stub_fsetxattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t              ret      = 0;
-        uint32_t             val      = 0;
-        br_isignature_t     *sign     = NULL;
-
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
-
-        /* object signature request */
-        ret = dict_get_bin (dict, GLUSTERFS_SET_OBJECT_SIGNATURE,
-                            (void **) &sign);
-        if (!ret) {
-                br_stub_handle_object_signature (frame, this,
-                                                 fd, dict, sign, xdata);
-                goto done;
-        }
-
-        /* signing xattr */
-        if (dict_get(dict, BITROT_SIGNING_VERSION_KEY)) {
-                br_stub_handle_internal_xattr (frame, this, fd,
-                                               BITROT_SIGNING_VERSION_KEY);
-                goto done;
-        }
-
-        /* version xattr */
-        if (dict_get(dict, BITROT_CURRENT_VERSION_KEY)) {
-                br_stub_handle_internal_xattr (frame, this, fd,
-                                               BITROT_CURRENT_VERSION_KEY);
-                goto done;
-        }
+static void
+br_stub_dump_xattr(xlator_t *this, dict_t *dict, int *op_errno)
+{
+    char *format = "(%s:%s)";
+    char *dump = NULL;
+
+    dump = GF_CALLOC(1, BR_STUB_DUMP_STR_SIZE, gf_br_stub_mt_misc);
+    if (!dump) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    dict_dump_to_str(dict, dump, BR_STUB_DUMP_STR_SIZE, format);
+    gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_INTERNAL_XATTR,
+            "fsetxattr dump=%s", dump, NULL);
+out:
+    if (dump) {
+        GF_FREE(dump);
+    }
+    return;
+}
 
-        if (dict_get (dict, GLUSTERFS_GET_OBJECT_SIGNATURE)) {
-                br_stub_handle_internal_xattr (frame, this, fd,
-                                               GLUSTERFS_GET_OBJECT_SIGNATURE);
-                goto done;
-        }
+int
+br_stub_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int flags, dict_t *xdata)
+{
+    int32_t ret = 0;
+    uint32_t val = 0;
+    br_isignature_t *sign = NULL;
+    br_stub_private_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    priv = this->private;
+
+    if ((frame->root->pid != GF_CLIENT_PID_BITD &&
+         frame->root->pid != GF_CLIENT_PID_SCRUB) &&
+        br_stub_internal_xattr(dict)) {
+        br_stub_dump_xattr(this, dict, &op_errno);
+        goto unwind;
+    }
+
+    if (!priv->do_versioning)
+        goto wind;
+
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
+
+    /* object signature request */
+    ret = dict_get_bin(dict, GLUSTERFS_SET_OBJECT_SIGNATURE, (void **)&sign);
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "got SIGNATURE request on %s",
+                     uuid_utoa(fd->inode->gfid));
+        br_stub_handle_object_signature(frame, this, fd, dict, sign, xdata);
+        goto done;
+    }
+
+    /* signing xattr */
+    if (dict_get(dict, BITROT_SIGNING_VERSION_KEY)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      BITROT_SIGNING_VERSION_KEY);
+        goto done;
+    }
+
+    /* version xattr */
+    if (dict_get(dict, BITROT_CURRENT_VERSION_KEY)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      BITROT_CURRENT_VERSION_KEY);
+        goto done;
+    }
+
+    if (dict_get(dict, GLUSTERFS_GET_OBJECT_SIGNATURE)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      GLUSTERFS_GET_OBJECT_SIGNATURE);
+        goto done;
+    }
+
+    /* object reopen request */
+    ret = dict_get_uint32(dict, BR_REOPEN_SIGN_HINT_KEY, &val);
+    if (!ret) {
+        br_stub_handle_object_reopen(frame, this, fd, val);
+        goto done;
+    }
+
+    /* handle bad object */
+    if (dict_get(dict, BITROT_OBJECT_BAD_KEY)) {
+        br_stub_handle_bad_object_key(frame, this, fd, dict, flags, xdata);
+        goto done;
+    }
 
-        /* object reopen request */
-        ret = dict_get_uint32 (dict, BR_REOPEN_SIGN_HINT_KEY, &val);
-        if (!ret) {
-                br_stub_handle_object_reopen (frame, this, fd, val);
-                goto done;
-        }
+wind:
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 
-        /* handle bad object */
-        if (dict_get (dict, BITROT_OBJECT_BAD_KEY)) {
-                br_stub_handle_bad_object_key (frame, this, fd,
-                                               dict, flags, xdata);
-                goto done;
-        }
+unwind:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 
-wind:
-        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags,
-                    xdata);
 done:
-        return 0;
+    return 0;
 }
 
-
 /**
  * Currently BitD and scrubber are doing fsetxattr to either sign the object
  * or to mark it as bad. Hence setxattr on any of those keys is denied directly
@@ -1264,95 +1475,76 @@ done:
  * check has to be added below.
  */
 int
-br_stub_setxattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t  op_ret                    = -1;
-        int32_t  op_errno                  = EINVAL;
-        char     dump[64*1024]             = {0,};
-        char    *format                    = "(%s:%s)";
-
-        if (dict_get (dict, GLUSTERFS_SET_OBJECT_SIGNATURE) ||
-            dict_get (dict, GLUSTERFS_GET_OBJECT_SIGNATURE) ||
-            dict_get (dict, BR_REOPEN_SIGN_HINT_KEY) ||
-            dict_get (dict, BITROT_OBJECT_BAD_KEY) ||
-            dict_get (dict, BITROT_SIGNING_VERSION_KEY) ||
-            dict_get (dict, BITROT_CURRENT_VERSION_KEY)) {
-                dict_dump_to_str (dict, dump, sizeof(dump), format);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_SET_INTERNAL_XATTR, "setxattr called on "
-                        "internal xattr %s", dump);
-                goto unwind;
-        }
+br_stub_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int flags, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
+    if (br_stub_internal_xattr(dict)) {
+        br_stub_dump_xattr(this, dict, &op_errno);
+        goto unwind;
+    }
 
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->setxattr, loc, dict, flags,
-                         xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* {f}removexattr() */
 
 int32_t
-br_stub_removexattr (call_frame_t *frame, xlator_t *this,
-                     loc_t *loc, const char *name, dict_t *xdata)
-{
-        int32_t op_ret    = -1;
-        int32_t op_errno  = EINVAL;
-
-        if (!strcmp (BITROT_OBJECT_BAD_KEY, name) ||
-            !strcmp (BITROT_SIGNING_VERSION_KEY, name) ||
-            !strcmp (BITROT_CURRENT_VERSION_KEY, name)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_REMOVE_INTERNAL_XATTR, "removexattr called"
-                        " on internal xattr %s for file %s", name, loc->path);
-                goto unwind;
-        }
-
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->removexattr,
-                         loc, name, xdata);
-        return 0;
+br_stub_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (!strcmp(BITROT_OBJECT_BAD_KEY, name) ||
+        !strcmp(BITROT_SIGNING_VERSION_KEY, name) ||
+        !strcmp(BITROT_CURRENT_VERSION_KEY, name)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_REMOVE_INTERNAL_XATTR,
+                "name=%s", name, "file-path=%s", loc->path, NULL);
+        goto unwind;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 int32_t
-br_stub_fremovexattr (call_frame_t *frame, xlator_t *this,
-                      fd_t *fd, const char *name, dict_t *xdata)
-{
-        int32_t op_ret    = -1;
-        int32_t op_errno  = EINVAL;
-
-        if (!strcmp (BITROT_OBJECT_BAD_KEY, name) ||
-            !strcmp (BITROT_SIGNING_VERSION_KEY, name) ||
-            !strcmp (BITROT_CURRENT_VERSION_KEY, name)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_REMOVE_INTERNAL_XATTR, "removexattr called"
-                        " on internal xattr %s for inode %s", name,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fremovexattr,
-                         fd, name, xdata);
-        return 0;
+br_stub_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (!strcmp(BITROT_OBJECT_BAD_KEY, name) ||
+        !strcmp(BITROT_SIGNING_VERSION_KEY, name) ||
+        !strcmp(BITROT_CURRENT_VERSION_KEY, name)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_REMOVE_INTERNAL_XATTR,
+                "name=%s", name, "inode-gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto unwind;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 /** }}} */
@@ -1362,17 +1554,17 @@ unwind:
 /* {f}getxattr() */
 
 int
-br_stub_listxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+br_stub_listxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        br_stub_remove_vxattrs (xattr);
+    br_stub_remove_vxattrs(xattr, _gf_true);
 
- unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, xattr, xdata);
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
 }
 
 /**
@@ -1411,355 +1603,380 @@ br_stub_listxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  * then treat the object as stale.
  */
 char
-br_stub_is_object_stale (xlator_t *this, call_frame_t *frame, inode_t *inode,
-                         br_version_t *obuf, br_signature_t *sbuf)
-{
-        uint64_t   ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-        int32_t  ret = -1;
-        char stale = 0;
-
-        if (obuf->ongoingversion == sbuf->signedversion)
-                goto out;
-
-        if (frame->root->pid == GF_CLIENT_PID_SCRUB) {
-                stale = 1;
-                goto out;
-        }
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED, "failed to get the "
-                        "inode context for %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
-
-        LOCK (&inode->lock);
-        {
-                if ((!__br_stub_is_inode_dirty (ctx) &&
-                     ctx->info_sign != BR_SIGN_NORMAL) ||
-                    __br_stub_is_inode_dirty (ctx))
-                        stale = 1;
-        }
-        UNLOCK (&inode->lock);
+br_stub_is_object_stale(xlator_t *this, call_frame_t *frame, inode_t *inode,
+                        br_version_t *obuf, br_signature_t *sbuf)
+{
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    char stale = 0;
+
+    if (obuf->ongoingversion == sbuf->signedversion)
+        goto out;
+
+    if (frame->root->pid == GF_CLIENT_PID_SCRUB) {
+        stale = 1;
+        goto out;
+    }
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        if ((!__br_stub_is_inode_dirty(ctx) &&
+             ctx->info_sign != BR_SIGN_NORMAL) ||
+            __br_stub_is_inode_dirty(ctx))
+            stale = 1;
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return stale;
+    return stale;
 }
 
 int
-br_stub_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int32_t              ret          = 0;
-        size_t               totallen     = 0;
-        size_t               signaturelen = 0;
-        br_version_t        *obuf         = NULL;
-        br_signature_t      *sbuf         = NULL;
-        br_isignature_out_t *sign         = NULL;
-        br_vxattr_status_t   status;
-        br_stub_local_t     *local        = NULL;
-        inode_t             *inode        = NULL;
-        gf_boolean_t         bad_object   = _gf_false;
-
-        if (op_ret < 0)
-                goto unwind;
-        if (cookie != (void *) BR_STUB_REQUEST_COOKIE)
-                goto unwind;
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->u.context.inode;
-
-        op_ret   = -1;
-        status = br_version_xattr_state (xattr, &obuf, &sbuf, &bad_object);
-
-        op_errno = EIO;
-        if (bad_object)
-                goto delkeys;
-
-        op_errno = EINVAL;
-        if (status == BR_VXATTR_STATUS_INVALID)
-                goto delkeys;
-
-        op_errno = ENODATA;
-        if ((status == BR_VXATTR_STATUS_MISSING)
-            || (status == BR_VXATTR_STATUS_UNSIGNED))
-                goto delkeys;
-
-        /**
-         * okay.. we have enough information to satisfy the request,
-         * namely: version and signing extended attribute. what's
-         * pending is the signature length -- that's figured out
-         * indirectly via the size of the _whole_ xattr and the
-         * on-disk signing xattr header size.
-         */
-        op_errno = EINVAL;
-        ret = dict_get_uint32 (xattr, BITROT_SIGNING_XATTR_SIZE_KEY,
-                               (uint32_t *)&signaturelen);
-        if (ret)
-                goto delkeys;
-
-        signaturelen -= sizeof (br_signature_t);
-        totallen = sizeof (br_isignature_out_t) + signaturelen;
-
-        op_errno = ENOMEM;
-        sign = GF_CALLOC (1, totallen, gf_br_stub_mt_signature_t);
-        if (!sign)
-                goto delkeys;
-
-        sign->time[0] = obuf->timebuf[0];
-        sign->time[1] = obuf->timebuf[1];
-
-        /* Object's dirty state & current signed version */
-        sign->version = sbuf->signedversion;
-        sign->stale = br_stub_is_object_stale (this, frame, inode, obuf, sbuf);
-
-        /* Object's signature */
-        sign->signaturelen  = signaturelen;
-        sign->signaturetype = sbuf->signaturetype;
-        (void) memcpy (sign->signature, sbuf->signature, signaturelen);
-
+br_stub_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    int32_t ret = 0;
+    size_t totallen = 0;
+    size_t signaturelen = 0;
+    br_stub_private_t *priv = NULL;
+    br_version_t *obuf = NULL;
+    br_signature_t *sbuf = NULL;
+    br_isignature_out_t *sign = NULL;
+    br_vxattr_status_t status;
+    br_stub_local_t *local = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t bad_object = _gf_false;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), delkeys);
+
+    if (cookie != (void *)BR_STUB_REQUEST_COOKIE)
+        goto unwind;
+
+    local = frame->local;
+    frame->local = NULL;
+    if (!local) {
+        op_ret = -1;
         op_errno = EINVAL;
-        ret = dict_set_bin (xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                            (void *)sign, totallen);
-        if (ret < 0) {
-                GF_FREE (sign);
-                goto delkeys;
-        }
-        op_errno = 0;
-        op_ret = totallen;
+        goto unwind;
+    }
+    inode = local->u.context.inode;
+
+    op_ret = -1;
+    status = br_version_xattr_state(xattr, &obuf, &sbuf, &bad_object);
+
+    op_errno = EIO;
+    if (bad_object)
+        goto delkeys;
+
+    op_errno = EINVAL;
+    if (status == BR_VXATTR_STATUS_INVALID)
+        goto delkeys;
+
+    op_errno = ENODATA;
+    if ((status == BR_VXATTR_STATUS_MISSING) ||
+        (status == BR_VXATTR_STATUS_UNSIGNED))
+        goto delkeys;
+
+    /**
+     * okay.. we have enough information to satisfy the request,
+     * namely: version and signing extended attribute. what's
+     * pending is the signature length -- that's figured out
+     * indirectly via the size of the _whole_ xattr and the
+     * on-disk signing xattr header size.
+     */
+    op_errno = EINVAL;
+    ret = dict_get_uint32(xattr, BITROT_SIGNING_XATTR_SIZE_KEY,
+                          (uint32_t *)&signaturelen);
+    if (ret)
+        goto delkeys;
+
+    signaturelen -= sizeof(br_signature_t);
+    totallen = sizeof(br_isignature_out_t) + signaturelen;
+
+    op_errno = ENOMEM;
+    sign = GF_CALLOC(1, totallen, gf_br_stub_mt_signature_t);
+    if (!sign)
+        goto delkeys;
+
+    sign->time[0] = obuf->timebuf[0];
+    sign->time[1] = obuf->timebuf[1];
+
+    /* Object's dirty state & current signed version */
+    sign->version = sbuf->signedversion;
+    sign->stale = br_stub_is_object_stale(this, frame, inode, obuf, sbuf);
+
+    /* Object's signature */
+    sign->signaturelen = signaturelen;
+    sign->signaturetype = sbuf->signaturetype;
+    (void)memcpy(sign->signature, sbuf->signature, signaturelen);
+
+    op_errno = EINVAL;
+    ret = dict_set_bin(xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void *)sign,
+                       totallen);
+    if (ret < 0) {
+        GF_FREE(sign);
+        goto delkeys;
+    }
+    op_errno = 0;
+    op_ret = totallen;
+
+delkeys:
+    br_stub_remove_vxattrs(xattr, _gf_true);
 
- delkeys:
-        br_stub_remove_vxattrs (xattr);
-
- unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, xattr, xdata);
-        if (local) {
-                br_stub_cleanup_local (local);
-                br_stub_dealloc_local (local);
-        }
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 static void
-br_stub_send_stub_init_time (call_frame_t *frame, xlator_t *this)
+br_stub_send_stub_init_time(call_frame_t *frame, xlator_t *this)
 {
-        int op_ret = 0;
-        int op_errno = 0;
-        dict_t *xattr = NULL;
-        br_stub_init_t stub = {{0,},};
-        br_stub_private_t *priv = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+    dict_t *xattr = NULL;
+    br_stub_init_t stub = {
+        {
+            0,
+        },
+    };
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        xattr = dict_new ();
-        if (!xattr) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        stub.timebuf[0] = priv->boot[0];
-        stub.timebuf[1] = priv->boot[1];
-        memcpy (stub.export, priv->export, strlen (priv->export) + 1);
+    stub.timebuf[0] = priv->boot[0];
+    stub.timebuf[1] = priv->boot[1];
+    memcpy(stub.export, priv->export, strlen(priv->export) + 1);
 
-        op_ret = dict_set_static_bin (xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                                      (void *) &stub, sizeof (br_stub_init_t));
-        if (op_ret < 0) {
-                op_errno = EINVAL;
-                goto unwind;
-        }
+    op_ret = dict_set_static_bin(xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                                 (void *)&stub, sizeof(br_stub_init_t));
+    if (op_ret < 0) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
 
-        op_ret = sizeof (br_stub_init_t);
+    op_ret = sizeof(br_stub_init_t);
 
- unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, xattr, NULL);
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, NULL);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 }
 
 int
-br_stub_getxattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, const char *name, dict_t *xdata)
-{
-        void               *cookie   = NULL;
-        uuid_t              rootgfid = {0, };
-        fop_getxattr_cbk_t  cbk      = br_stub_getxattr_cbk;
-        int32_t             op_ret   = -1;
-        int32_t             op_errno = EINVAL;
-        br_stub_local_t    *local    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-
-        rootgfid[15] = 1;
-
-        if (!name) {
-                cbk = br_stub_listxattr_cbk;
-                goto wind;
-        }
-
-        /**
-         * If xattr is node-uuid and the inode is marked bad, return EIO.
-         * Returning EIO would result in AFR to choose correct node-uuid
-         * coresponding to the subvolume * where the good copy of the
-         * file resides.
-         */
-        if (IA_ISREG (loc->inode->ia_type) && XATTR_IS_NODE_UUID (name) &&
-            br_stub_check_bad_object (this, loc->inode, &op_ret, &op_errno)) {
-                goto unwind;
-        }
-
-        if (br_stub_is_internal_xattr (name))
-                goto unwind;
-
-        /**
-         * this special extended attribute is allowed only on root
-         */
-        if (name
-            && (strncmp (name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                          strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
-            && ((gf_uuid_compare (loc->gfid, rootgfid) == 0)
-                || (gf_uuid_compare (loc->inode->gfid, rootgfid) == 0))) {
-                br_stub_send_stub_init_time (frame, this);
-                return 0;
+br_stub_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
+{
+    void *cookie = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    fop_getxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    if (!name) {
+        cbk = br_stub_listxattr_cbk;
+        goto wind;
+    }
+
+    if (br_stub_is_internal_xattr(name))
+        goto unwind;
+
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    /**
+     * If xattr is node-uuid and the inode is marked bad, return EIO.
+     * Returning EIO would result in AFR to choose correct node-uuid
+     * corresponding to the subvolume * where the good copy of the
+     * file resides.
+     */
+    if (IA_ISREG(loc->inode->ia_type) && XATTR_IS_NODE_UUID(name) &&
+        br_stub_check_bad_object(this, loc->inode, &op_ret, &op_errno)) {
+        goto unwind;
+    }
+
+    /**
+     * this special extended attribute is allowed only on root
+     */
+    if (name &&
+        (strncmp(name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                 sizeof(GLUSTERFS_GET_BR_STUB_INIT_TIME) - 1) == 0) &&
+        ((gf_uuid_compare(loc->gfid, rootgfid) == 0) ||
+         (gf_uuid_compare(loc->inode->gfid, rootgfid) == 0))) {
+        BR_STUB_RESET_LOCAL_NULL(frame);
+        br_stub_send_stub_init_time(frame, this);
+        return 0;
+    }
+
+    if (!IA_ISREG(loc->inode->ia_type))
+        goto wind;
+
+    if (name && (strncmp(name, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                         sizeof(GLUSTERFS_GET_OBJECT_SIGNATURE) - 1) == 0)) {
+        cookie = (void *)BR_STUB_REQUEST_COOKIE;
+
+        local = br_stub_alloc_local(this);
+        if (!local) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        if (!IA_ISREG (loc->inode->ia_type))
-                goto wind;
-
-        if (name && (strncmp (name, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                              strlen (GLUSTERFS_GET_OBJECT_SIGNATURE)) == 0)) {
-                cookie = (void *) BR_STUB_REQUEST_COOKIE;
-
-                local = br_stub_alloc_local (this);
-                if (!local) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                br_stub_fill_local (local, NULL, NULL, loc->inode,
-                                    loc->inode->gfid,
-                                    BR_STUB_NO_VERSIONING, 0);
-                frame->local = local;
-        }
+        br_stub_fill_local(local, NULL, NULL, loc->inode, loc->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        frame->local = local;
+    }
 
- wind:
-        STACK_WIND_COOKIE
-                      (frame, cbk, cookie, FIRST_CHILD (this),
-                       FIRST_CHILD (this)->fops->getxattr, loc, name, xdata);
-        return 0;
+wind:
+    STACK_WIND_COOKIE(frame, cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, NULL, NULL);
-                return 0;
+    BR_STUB_RESET_LOCAL_NULL(frame);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-br_stub_fgetxattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, const char *name, dict_t *xdata)
-{
-        void *cookie = NULL;
-        uuid_t rootgfid = {0, };
-        fop_fgetxattr_cbk_t cbk = br_stub_getxattr_cbk;
-        int32_t op_ret = -1;
-        int32_t op_errno = EINVAL;
-        br_stub_local_t *local = NULL;
-
-        rootgfid[15] = 1;
-
-        if (!name) {
-                cbk = br_stub_listxattr_cbk;
-                goto wind;
-        }
-
-        /**
-         * If xattr is node-uuid and the inode is marked bad, return EIO.
-         * Returning EIO would result in AFR to choose correct node-uuid
-         * coresponding to the subvolume * where the good copy of the
-         * file resides.
-         */
-        if (IA_ISREG (fd->inode->ia_type) && XATTR_IS_NODE_UUID (name) &&
-            br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno)) {
-                goto unwind;
-        }
-
-        if (br_stub_is_internal_xattr (name))
-                goto unwind;
-
-        /**
-         * this special extended attribute is allowed only on root
-         */
-        if (name
-            && (strncmp (name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                         strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
-            && (gf_uuid_compare (fd->inode->gfid, rootgfid) == 0)) {
-                br_stub_send_stub_init_time (frame, this);
-                return 0;
+br_stub_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+    void *cookie = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    fop_fgetxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (!name) {
+        cbk = br_stub_listxattr_cbk;
+        goto wind;
+    }
+
+    if (br_stub_is_internal_xattr(name))
+        goto unwind;
+
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    /**
+     * If xattr is node-uuid and the inode is marked bad, return EIO.
+     * Returning EIO would result in AFR to choose correct node-uuid
+     * corresponding to the subvolume * where the good copy of the
+     * file resides.
+     */
+    if (IA_ISREG(fd->inode->ia_type) && XATTR_IS_NODE_UUID(name) &&
+        br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno)) {
+        goto unwind;
+    }
+
+    /**
+     * this special extended attribute is allowed only on root
+     */
+    if (name &&
+        (strncmp(name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                 sizeof(GLUSTERFS_GET_BR_STUB_INIT_TIME) - 1) == 0) &&
+        (gf_uuid_compare(fd->inode->gfid, rootgfid) == 0)) {
+        BR_STUB_RESET_LOCAL_NULL(frame);
+        br_stub_send_stub_init_time(frame, this);
+        return 0;
+    }
+
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
+
+    if (name && (strncmp(name, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                         sizeof(GLUSTERFS_GET_OBJECT_SIGNATURE) - 1) == 0)) {
+        cookie = (void *)BR_STUB_REQUEST_COOKIE;
+
+        local = br_stub_alloc_local(this);
+        if (!local) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
-
-        if (name && (strncmp (name, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                              strlen (GLUSTERFS_GET_OBJECT_SIGNATURE)) == 0)) {
-                cookie = (void *) BR_STUB_REQUEST_COOKIE;
-
-                local = br_stub_alloc_local (this);
-                if (!local) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid,
-                                    BR_STUB_NO_VERSIONING, 0);
-                frame->local = local;
-        }
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        frame->local = local;
+    }
 
- wind:
-        STACK_WIND_COOKIE
-                      (frame, cbk, cookie, FIRST_CHILD (this),
-                       FIRST_CHILD (this)->fops->fgetxattr, fd, name, xdata);
-        return 0;
+wind:
+    STACK_WIND_COOKIE(frame, cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    BR_STUB_RESET_LOCAL_NULL(frame);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 int32_t
-br_stub_readv (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+br_stub_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int32_t              op_ret   = -1;
-        int32_t              op_errno = EINVAL;
-        int32_t              ret      = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
 
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readv, fd, size, offset,
-                         flags, xdata);
-        return 0;
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, NULL, 0, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, NULL, 0, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
 /**
@@ -1770,287 +1987,294 @@ unwind:
  * fds.
  */
 int32_t
-br_stub_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata)
+br_stub_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        br_stub_local_t *local = NULL;
+    int32_t ret = 0;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame,
-                             op_ret, op_errno, prebuf, postbuf, xdata);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_writev_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       struct iovec *vector, int32_t count, off_t offset,
-                       uint32_t flags, struct iobref *iobref, dict_t *xdata)
+br_stub_writev_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iovec *vector, int32_t count, off_t offset,
+                      uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        STACK_WIND (frame, br_stub_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    offset, flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 /**
  * This is probably the most crucial part about the whole versioning thing.
  * There's absolutely no differentiation as such between an anonymous fd
  * and a regular fd except the fd context initialization. Object versioning
- * is perfomed when the inode is dirty. Parallel write operations are no
+ * is performed when the inode is dirty. Parallel write operations are no
  * special with each write performing object versioning followed by marking
  * the inode as non-dirty (synced). This is followed by the actual operation
  * (writev() in this case) which on a success marks the inode as modified.
  * This prevents signing of objects that have not been modified.
  */
 int32_t
-br_stub_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t offset,
-                uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        call_stub_t         *stub        = NULL;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        int32_t              ret         = -1;
-        fop_writev_cbk_t     cbk         = default_writev_cbk;
-        br_stub_local_t     *local       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto unwind;
+br_stub_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t offset,
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fop_writev_cbk_t cbk = default_writev_cbk;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    /**
+     * The inode is not dirty and also witnessed at least one successful
+     * modification operation. Therefore, subsequent operations need not
+     * perform any special tracking.
+     */
+    if (!inc_version && modified)
+        goto wind;
+
+    /**
+     * okay.. so, either the inode needs versioning or the modification
+     * needs to be tracked. ->cbk is set to the appropriate callback
+     * routine for this.
+     * NOTE: ->local needs to be deallocated on failures from here on.
+     */
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_writev_cbk;
+        goto wind;
+    }
+
+    stub = fop_writev_stub(frame, br_stub_writev_resume, fd, vector, count,
+                           offset, flags, iobref, xdata);
+
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "write  gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    /* Perform Versioning */
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
-
-        /**
-         * The inode is not dirty and also witnessed atleast one successful
-         * modification operation. Therefore, subsequent operations need not
-         * perform any special tracking.
-         */
-        if (!inc_version && modified)
-                goto wind;
-
-        /**
-         * okay.. so, either the inode needs versioning or the modification
-         * needs to be tracked. ->cbk is set to the appropriate callback
-         * routine for this.
-         * NOTE: ->local needs to be deallocated on failures from here on.
-         */
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto unwind;
-
-        local = frame->local;
-        if (!inc_version) {
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-                cbk = br_stub_writev_cbk;
-                goto wind;
-        }
-
-        stub = fop_writev_stub (frame, br_stub_writev_resume, fd, vector, count,
-                                offset, flags, iobref, xdata);
-
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for write fop (gfid: %s), "
-                        "unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
-
-        /* Perform Versioning */
-        return br_stub_perform_incversioning (this, frame, stub, fd, ctx);
-
- wind:
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+               fd, vector, count, offset, flags, iobref, xdata);
+    return 0;
 
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, NULL, NULL,
-                             NULL);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+br_stub_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t          ret   = -1;
-        br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame,
-                             op_ret, op_errno, prebuf, postbuf, xdata);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_ftruncate_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          off_t offset, dict_t *xdata)
+br_stub_ftruncate_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         off_t offset, dict_t *xdata)
 {
-        STACK_WIND (frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 /* c.f. br_stub_writev() for explanation */
 int32_t
-br_stub_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
-{
-        br_stub_local_t     *local       = NULL;
-        call_stub_t         *stub        = NULL;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        int32_t              ret         = -1;
-        fop_ftruncate_cbk_t  cbk         = default_ftruncate_cbk;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto unwind;
-
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
-
-        if (!inc_version && modified)
-                goto wind;
-
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto unwind;
-
-        local = frame->local;
-        if (!inc_version) {
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-                cbk = br_stub_ftruncate_cbk;
-                goto wind;
-        }
-
-        stub = fop_ftruncate_stub (frame, br_stub_ftruncate_resume, fd, offset,
-                                   xdata);
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for ftruncate fop (gfid: %s),"
-                        " unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
+br_stub_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fop_ftruncate_cbk_t cbk = default_ftruncate_cbk;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    if (!inc_version && modified)
+        goto wind;
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_ftruncate_cbk;
+        goto wind;
+    }
+
+    stub = fop_ftruncate_stub(frame, br_stub_ftruncate_resume, fd, offset,
+                              xdata);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "ftruncate gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
 
-        return br_stub_perform_incversioning (this, frame, stub, fd, ctx);
-
- wind:
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, NULL, NULL,
-                             NULL);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+br_stub_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        br_stub_local_t *local = NULL;
+    int32_t ret = 0;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 int32_t
-br_stub_truncate_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                          off_t offset, dict_t *xdata)
+br_stub_truncate_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        off_t offset, dict_t *xdata)
 {
-        br_stub_local_t *local = frame->local;
+    br_stub_local_t *local = frame->local;
 
-        fd_unref (local->u.context.fd);
-        STACK_WIND (frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    fd_unref(local->u.context.fd);
+    STACK_WIND(frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
 /**
@@ -2068,90 +2292,92 @@ br_stub_truncate_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
  * c.f. br_writev_cbk() for explanation
  */
 int32_t
-br_stub_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
-{
-        br_stub_local_t     *local       = NULL;
-        call_stub_t         *stub        = NULL;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        int32_t              ret         = -1;
-        fd_t                *fd          = NULL;
-        fop_truncate_cbk_t   cbk         = default_truncate_cbk;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-
-        fd = fd_anonymous (loc->inode);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_CREATE_ANONYMOUS_FD_FAILED, "failed to create "
-                        "anonymous fd for the inode %s",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto cleanup_fd;
-
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
-
-        if (!inc_version && modified)
-                goto wind;
-
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto cleanup_fd;
-
-        local = frame->local;
-        if (!inc_version) {
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-                cbk = br_stub_truncate_cbk;
-                goto wind;
-        }
-
-        stub = fop_truncate_stub (frame, br_stub_truncate_resume, loc, offset,
-                                  xdata);
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for truncate fop (gfid: %s), "
-                        "unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
+br_stub_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    fop_truncate_cbk_t cbk = default_truncate_cbk;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    fd = fd_anonymous(loc->inode);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CREATE_ANONYMOUS_FD_FAILED,
+                "inode-gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto unwind;
+    }
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto cleanup_fd;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    if (!inc_version && modified)
+        goto wind;
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto cleanup_fd;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_truncate_cbk;
+        goto wind;
+    }
+
+    stub = fop_truncate_stub(frame, br_stub_truncate_resume, loc, offset,
+                             xdata);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "truncate gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
 
-        return br_stub_perform_incversioning (this, frame, stub, fd, ctx);
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
 
- wind:
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        fd_unref (fd);
-        return 0;
-
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
- cleanup_fd:
-        fd_unref (fd);
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, NULL, NULL,
-                             NULL);
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+               loc, offset, xdata);
+    if (fd)
+        fd_unref(fd);
+    return 0;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+cleanup_fd:
+    fd_unref(fd);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* open() */
@@ -2171,62 +2397,70 @@ br_stub_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
  */
 
 int
-br_stub_open (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        int32_t              ret      = -1;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        uint64_t             ctx_addr = 0;
-        int32_t              op_ret   = -1;
-        int32_t              op_errno = EINVAL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
+br_stub_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata)
+{
+    int32_t ret = -1;
+    br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    priv = this->private;
+
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED, "failed to get the "
-                        "inode context for the file %s (gfid: %s)", loc->path,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "path=%s", loc->path,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto unwind;
         }
+    }
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
 
-        if (frame->root->pid == GF_CLIENT_PID_SCRUB)
-                goto wind;
+    if (frame->root->pid == GF_CLIENT_PID_SCRUB)
+        goto wind;
 
-        if (flags == O_RDONLY)
-                goto wind;
+    if (flags == O_RDONLY)
+        goto wind;
 
-        ret = br_stub_add_fd_to_inode (this, fd, ctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_ADD_FD_TO_LIST_FAILED,
-                        "failed add fd to the list (gfid: %s)",
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
+    ret = br_stub_add_fd_to_inode(this, fd, ctx);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ADD_FD_TO_LIST_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unwind;
+    }
 
 wind:
-        STACK_WIND (frame, default_open_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* creat() */
@@ -2236,130 +2470,137 @@ unwind:
  * fd to the inode context fd tracking list.
  */
 int32_t
-br_stub_add_fd_to_inode (xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+br_stub_add_fd_to_inode(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
 {
-        int32_t       ret        = -1;
-        br_stub_fd_t *br_stub_fd = NULL;
+    int32_t ret = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        ret = br_stub_require_release_call (this, fd, &br_stub_fd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_SET_FD_CONTEXT_FAILED, "failed to set the fd "
-                        "context for the file (gfid: %s)",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
+    ret = br_stub_require_release_call(this, fd, &br_stub_fd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
 
-        LOCK (&fd->inode->lock);
-        {
-                list_add_tail (&ctx->fd_list, &br_stub_fd->list);
-        }
-        UNLOCK (&fd->inode->lock);
+    LOCK(&fd->inode->lock);
+    {
+        list_add_tail(&ctx->fd_list, &br_stub_fd->list);
+    }
+    UNLOCK(&fd->inode->lock);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-br_stub_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        int32_t              ret      = 0;
-        uint64_t             ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0) {
-                ret = br_stub_init_inode_versions (this, fd, inode, version,
-                                                   _gf_true, _gf_false);
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                }
-        } else {
-                ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
-                ret = br_stub_add_fd_to_inode (this, fd, ctx);
+br_stub_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *stbuf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!priv->do_versioning)
+        goto unwind;
+
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, inode, version, _gf_true,
+                                          _gf_false, &ctx_addr);
+        if (ret) {
+            op_ret = -1;
+            op_errno = EINVAL;
         }
+    } else {
+        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno,
-                             fd, inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                        preparent, postparent, xdata);
+    return 0;
 }
 
 int
-br_stub_create (call_frame_t *frame,
-                xlator_t *this, loc_t *loc, int32_t flags,
-                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        STACK_WIND (frame, br_stub_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+br_stub_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    STACK_WIND(frame, br_stub_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (create, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
 int
-br_stub_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        int32_t              ret      = -1;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        ret = br_stub_init_inode_versions (this, NULL, inode, version,
-                                           _gf_true, _gf_false);
-        /**
-         * Like lookup, if init_inode_versions fail, return EINVAL
-         */
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+br_stub_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, inode_t *inode, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+    int32_t ret = -1;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!priv->do_versioning)
+        goto unwind;
+
+    ret = br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                      _gf_false, NULL);
+    /**
+     * Like lookup, if init_inode_versions fail, return EINVAL
+     */
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
-                             inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
 int
-br_stub_mknod (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+br_stub_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t dev, mode_t umask, dict_t *xdata)
 {
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
 
-        STACK_WIND (frame, br_stub_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (mknod, frame, -1, EINVAL, NULL, NULL, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(mknod, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
@@ -2389,210 +2630,245 @@ unwind:
  * creattion failure as the lookup failure.
  */
 static int32_t
-br_stub_lookup_version (xlator_t *this,
-                        uuid_t gfid, inode_t *inode, dict_t *xattr)
-{
-        unsigned long       version = 0;
-        br_version_t       *obuf    = NULL;
-        br_signature_t     *sbuf    = NULL;
-        br_vxattr_status_t  status;
-        gf_boolean_t        bad_object = _gf_false;
-
-        /**
-         * versioning xattrs were requested from POSIX. if available, figure
-         * out the correct version to use in the inode context (start with
-         * the default version if unavailable). As of now versions are not
-         * persisted on-disk. The inode is marked dirty, so that the first
-         * operation (such as write(), etc..) triggers synchronization to
-         * disk.
-         */
-        status = br_version_xattr_state (xattr, &obuf, &sbuf, &bad_object);
-        version = ((status == BR_VXATTR_STATUS_FULL)
-                   || (status == BR_VXATTR_STATUS_UNSIGNED))
-                        ? obuf->ongoingversion : BITROT_DEFAULT_CURRENT_VERSION;
-
-        /**
-         * If signature is there, but version is not therem then that status is
-         * is treated as INVALID. So in that case, we should not initialize the
-         * inode context with wrong version names etc.
-         */
-        if (status == BR_VXATTR_STATUS_INVALID)
-                return -1;
+br_stub_lookup_version(xlator_t *this, uuid_t gfid, inode_t *inode,
+                       dict_t *xattr)
+{
+    unsigned long version = 0;
+    br_version_t *obuf = NULL;
+    br_signature_t *sbuf = NULL;
+    br_vxattr_status_t status;
+    gf_boolean_t bad_object = _gf_false;
+
+    /**
+     * versioning xattrs were requested from POSIX. if available, figure
+     * out the correct version to use in the inode context (start with
+     * the default version if unavailable). As of now versions are not
+     * persisted on-disk. The inode is marked dirty, so that the first
+     * operation (such as write(), etc..) triggers synchronization to
+     * disk.
+     */
+    status = br_version_xattr_state(xattr, &obuf, &sbuf, &bad_object);
+    version = ((status == BR_VXATTR_STATUS_FULL) ||
+               (status == BR_VXATTR_STATUS_UNSIGNED))
+                  ? obuf->ongoingversion
+                  : BITROT_DEFAULT_CURRENT_VERSION;
+
+    /**
+     * If signature is there, but version is not there then that status is
+     * is treated as INVALID. So in that case, we should not initialize the
+     * inode context with wrong version names etc.
+     */
+    if (status == BR_VXATTR_STATUS_INVALID)
+        return -1;
 
-        return br_stub_init_inode_versions (this, NULL, inode, version,
-                                            _gf_true, bad_object);
+    return br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                       bad_object, NULL);
 }
 
-
 /** {{{ */
 
 int32_t
-br_stub_opendir (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        br_stub_private_t    *priv = NULL;
-        br_stub_fd_t         *fd_ctx = NULL;
-        int32_t               op_ret = -1;
-        int32_t               op_errno = EINVAL;
-
-        priv = this->private;
-        if (gf_uuid_compare (fd->inode->gfid, priv->bad_object_dir_gfid))
-                goto normal;
-
-        fd_ctx = br_stub_fd_new ();
-        if (!fd_ctx) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+br_stub_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
+{
+    br_stub_private_t *priv = NULL;
+    br_stub_fd_t *fd_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
-        fd_ctx->bad_object.dir_eof = -1;
-        fd_ctx->bad_object.dir = sys_opendir (priv->stub_basepath);
-        if (!fd_ctx->bad_object.dir) {
-                op_errno = errno;
-                goto err_freectx;
-        }
+    priv = this->private;
+    if (gf_uuid_compare(fd->inode->gfid, priv->bad_object_dir_gfid))
+        goto normal;
 
-        op_ret = br_stub_fd_ctx_set (this, fd, fd_ctx);
-        if (!op_ret)
-                goto unwind;
+    fd_ctx = br_stub_fd_new();
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        sys_closedir (fd_ctx->bad_object.dir);
+    fd_ctx->bad_object.dir_eof = -1;
+    fd_ctx->bad_object.dir = sys_opendir(priv->stub_basepath);
+    if (!fd_ctx->bad_object.dir) {
+        op_errno = errno;
+        goto err_freectx;
+    }
+
+    op_ret = br_stub_fd_ctx_set(this, fd, fd_ctx);
+    if (!op_ret)
+        goto unwind;
+
+    sys_closedir(fd_ctx->bad_object.dir);
 
 err_freectx:
-        GF_FREE (fd_ctx);
+    GF_FREE(fd_ctx);
 unwind:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, NULL);
+    return 0;
 
 normal:
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int32_t
-br_stub_readdir (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        call_stub_t     *stub = NULL;
-        br_stub_private_t    *priv = NULL;
-
-        priv = this->private;
-        if (gf_uuid_compare (fd->inode->gfid, priv->bad_object_dir_gfid))
-                goto out;
-        stub = fop_readdir_stub (frame, br_stub_readdir_wrapper, fd, size, off,
-                                 xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-        br_stub_worker_enqueue (this, stub);
-        return 0;
+br_stub_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto out;
+
+    if (gf_uuid_compare(fd->inode->gfid, priv->bad_object_dir_gfid))
+        goto out;
+    stub = fop_readdir_stub(frame, br_stub_readdir_wrapper, fd, size, off,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
+    br_stub_worker_enqueue(this, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
-        return 0;
+    STACK_WIND(frame, default_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+    return 0;
 }
 
 int
-br_stub_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, gf_dirent_t *entries,
-                      dict_t *dict)
-{
-        int32_t      ret     = 0;
-        uint64_t     ctxaddr = 0;
-        gf_dirent_t *entry   = NULL;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                if ((strcmp (entry->d_name, ".") == 0)
-                    || (strcmp (entry->d_name, "..") == 0))
-                        continue;
-
-                if (!IA_ISREG (entry->d_stat.ia_type))
-                        continue;
-
-                ret = br_stub_get_inode_ctx (this, entry->inode, &ctxaddr);
-                if (ret < 0)
-                        ctxaddr = 0;
-                if (ctxaddr) { /* already has the context */
-                        br_stub_remove_vxattrs (entry->dict);
-                        continue;
-                }
-
-                ret = br_stub_lookup_version
-                        (this, entry->inode->gfid, entry->inode, entry->dict);
-                br_stub_remove_vxattrs (entry->dict);
-                if (ret) {
-                        /**
-                         * there's no per-file granularity support in case of
-                         * failure. let's fail the entire request for now..
-                         */
-                        break;
-                }
+br_stub_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, gf_dirent_t *entries,
+                     dict_t *dict)
+{
+    int32_t ret = 0;
+    uint64_t ctxaddr = 0;
+    gf_dirent_t *entry = NULL;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), unwind);
+
+    if (op_ret < 0)
+        goto unwind;
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+            continue;
+
+        if (!IA_ISREG(entry->d_stat.ia_type))
+            continue;
+
+        /*
+         * Readdirp for most part is a bulk lookup for all the entries
+         * present in the directory being read. Ideally, for each
+         * entry, the handling should be similar to that of a lookup
+         * callback. But for now, just keeping this as it has been
+         * until now (which means, this comment has been added much
+         * later as part of a change that wanted to send the flag
+         * of true/false to br_stub_remove_vxattrs to indicate whether
+         * the bad-object xattr should be removed from the entry->dict
+         * or not). Until this change, the function br_stub_remove_vxattrs
+         * was just removing all the xattrs associated with bit-rot-stub
+         * (like version, bad-object, signature etc). But, there are
+         * scenarios where we only want to send bad-object xattr and not
+         * others. So this comment is part of that change which also
+         * mentions about another possible change that might be needed
+         * in future.
+         * But for now, adding _gf_true means functionally its same as
+         * what this function was doing before. Just remove all the stub
+         * related xattrs.
+         */
+        ret = br_stub_get_inode_ctx(this, entry->inode, &ctxaddr);
+        if (ret < 0)
+            ctxaddr = 0;
+        if (ctxaddr) { /* already has the context */
+            br_stub_remove_vxattrs(entry->dict, _gf_true);
+            continue;
         }
 
+        ret = br_stub_lookup_version(this, entry->inode->gfid, entry->inode,
+                                     entry->dict);
+        br_stub_remove_vxattrs(entry->dict, _gf_true);
         if (ret) {
-                op_ret   = -1;
-                op_errno = EINVAL;
+            /**
+             * there's no per-file granularity support in case of
+             * failure. let's fail the entire request for now..
+             */
+            break;
         }
+    }
+
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
- unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, dict);
+unwind:
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, dict);
 
-        return 0;
+    return 0;
 }
 
 int
-br_stub_readdirp (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset, dict_t *dict)
+br_stub_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, dict_t *dict)
 {
-        int32_t ret = -1;
-        int op_errno = 0;
-        gf_boolean_t xref = _gf_false;
-
-        op_errno = ENOMEM;
-        if (!dict) {
-                dict = dict_new ();
-                if (!dict)
-                        goto unwind;
-        } else {
-                dict = dict_ref (dict);
-        }
+    int32_t ret = -1;
+    int op_errno = 0;
+    gf_boolean_t xref = _gf_false;
+    br_stub_private_t *priv = NULL;
 
-        xref = _gf_true;
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        op_errno = EINVAL;
-        ret = dict_set_uint32 (dict, BITROT_CURRENT_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (dict, BITROT_SIGNING_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (dict, BITROT_OBJECT_BAD_KEY, 0);
-        if (ret)
-                goto unwind;
+    op_errno = ENOMEM;
+    if (!dict) {
+        dict = dict_new();
+        if (!dict)
+            goto unwind;
+    } else {
+        dict = dict_ref(dict);
+    }
+
+    xref = _gf_true;
+
+    op_errno = EINVAL;
+    ret = dict_set_uint32(dict, BITROT_CURRENT_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(dict, BITROT_SIGNING_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(dict, BITROT_OBJECT_BAD_KEY, 0);
+    if (ret)
+        goto unwind;
 
-        STACK_WIND (frame, br_stub_readdirp_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size,
-                    offset, dict);
-        goto unref_dict;
+wind:
+    STACK_WIND(frame, br_stub_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    goto unref_dict;
 
- unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
-        return 0;
+unwind:
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
 
- unref_dict:
-        if (xref)
-                dict_unref (dict);
-        return 0;
+unref_dict:
+    if (xref)
+        dict_unref(dict);
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* lookup() */
@@ -2606,181 +2882,213 @@ br_stub_readdirp (call_frame_t *frame, xlator_t *this,
  * either forget () or lookup () will take care of removing the link.
  */
 void
-br_stub_handle_lookup_error (xlator_t *this, inode_t *inode, int32_t op_errno)
+br_stub_handle_lookup_error(xlator_t *this, inode_t *inode, int32_t op_errno)
 {
-        int32_t     ret = -1;
-        uint64_t   ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
 
-        if (op_errno != ENOENT)
-                goto out;
+    if (op_errno != ENOENT)
+        goto out;
 
-        if (!inode_is_linked (inode))
-                goto out;
+    if (!inode_is_linked(inode))
+        goto out;
 
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret)
-                goto out;
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret)
+        goto out;
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&inode->lock);
-        {
-                if (__br_stub_is_bad_object (ctx))
-                        (void) br_stub_del (this, inode->gfid);
+    LOCK(&inode->lock);
+    {
+        if (__br_stub_is_bad_object(ctx))
+            (void)br_stub_del(this, inode->gfid);
+    }
+    UNLOCK(&inode->lock);
+
+    if (__br_stub_is_bad_object(ctx)) {
+        /* File is not present, might be deleted for recovery,
+         * del the bitrot inode context
+         */
+        ctx_addr = 0;
+        inode_ctx_del(inode, this, &ctx_addr);
+        if (ctx_addr) {
+            ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+            GF_FREE(ctx);
         }
-        UNLOCK (&inode->lock);
+    }
 
 out:
-        return;
+    return;
 }
 
 int
-br_stub_lookup_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int op_ret, int op_errno, inode_t *inode,
-                    struct iatt *stbuf, dict_t *xattr, struct iatt *postparent)
+br_stub_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
 {
-        int32_t ret = 0;
+    int32_t ret = 0;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+    gf_boolean_t remove_bad_file_marker = _gf_true;
 
-        if (op_ret < 0) {
-                (void) br_stub_handle_lookup_error (this, inode, op_errno);
-                goto unwind;
-        }
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
 
-        if (!IA_ISREG (stbuf->ia_type))
-                goto unwind;
+    if (op_ret < 0) {
+        (void)br_stub_handle_lookup_error(this, inode, op_errno);
 
-        /**
-         * If the object is bad, then "bad inode" marker has to be sent back
-         * in resoinse, for revalidated lookups as well. Some xlators such as
-         * quick-read might cache the data in revalidated lookup as fresh
-         * lookup would anyway have sent "bad inode" marker.
-         * In general send bad inode marker for every lookup operation on the
-         * bad object.
+        /*
+         * If the lookup error is not ENOENT, then it is better
+         * to send the bad file marker to the higher layer (if
+         * it has been set)
          */
-        if (cookie != (void *) BR_STUB_REQUEST_COOKIE) {
-                ret =  br_stub_mark_xdata_bad_object (this, inode, xattr);
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = EIO;
-                        goto unwind;
-                }
-
-                goto delkey;
-        }
-
-        ret = br_stub_lookup_version (this, stbuf->ia_gfid, inode, xattr);
-        if (ret < 0) {
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto delkey;
-        }
-
+        if (op_errno != ENOENT)
+            remove_bad_file_marker = _gf_false;
+        goto delkey;
+    }
+
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), delkey);
+
+    if (!IA_ISREG(stbuf->ia_type))
+        goto unwind;
+
+    /**
+     * If the object is bad, then "bad inode" marker has to be sent back
+     * in resoinse, for revalidated lookups as well. Some xlators such as
+     * quick-read might cache the data in revalidated lookup as fresh
+     * lookup would anyway have sent "bad inode" marker.
+     * In general send bad inode marker for every lookup operation on the
+     * bad object.
+     */
+    if (cookie != (void *)BR_STUB_REQUEST_COOKIE) {
+        ret = br_stub_mark_xdata_bad_object(this, inode, xattr);
+        if (ret) {
+            op_ret = -1;
+            op_errno = EIO;
+            /*
+             * This flag ensures that in the label @delkey below,
+             * bad file marker is not removed from the dictinary,
+             * but other virtual xattrs (such as version, signature)
+             * are removed.
+             */
+            remove_bad_file_marker = _gf_false;
+        }
+        goto delkey;
+    }
+
+    ret = br_stub_lookup_version(this, stbuf->ia_gfid, inode, xattr);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto delkey;
+    }
+
+    /**
+     * If the object is bad, send "bad inode" marker back in response
+     * for xlator(s) to act accordingly (such as quick-read, etc..)
+     */
+    ret = br_stub_mark_xdata_bad_object(this, inode, xattr);
+    if (ret) {
         /**
-         * If the object is bad, send "bad inode" marker back in response
-         * for xlator(s) to act accordingly (such as quick-read, etc..)
+         * aaha! bad object, but sorry we would not
+         * satisfy the request on allocation failures.
          */
-        ret = br_stub_mark_xdata_bad_object (this, inode, xattr);
-        if (ret) {
-                /**
-                 * aaha! bad object, but sorry we would not
-                 * satisfy the request on allocation failures.
-                 */
-                op_ret = -1;
-                op_errno = EIO;
-                goto unwind;
-        }
+        op_ret = -1;
+        op_errno = EIO;
+        goto delkey;
+    }
 
 delkey:
-        br_stub_remove_vxattrs (xattr);
+    br_stub_remove_vxattrs(xattr, remove_bad_file_marker);
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame,
-                             op_ret, op_errno, inode, stbuf, xattr, postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                        postparent);
 
-        return 0;
+    return 0;
 }
 
 int
-br_stub_lookup (call_frame_t *frame,
-                xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        int32_t ret = 0;
-        int op_errno = 0;
-        void *cookie = NULL;
-        uint64_t ctx_addr = 0;
-        gf_boolean_t xref = _gf_false;
-        br_stub_private_t *priv = NULL;
-        call_stub_t       *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-
-        priv = this->private;
-
-        if (!gf_uuid_compare (loc->gfid, priv->bad_object_dir_gfid) ||
-            !gf_uuid_compare (loc->pargfid, priv->bad_object_dir_gfid)) {
-
-                stub = fop_lookup_stub (frame, br_stub_lookup_wrapper, loc,
-                                        xdata);
-                if (!stub) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                br_stub_worker_enqueue (this, stub);
-                return 0;
-        }
+br_stub_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int32_t ret = 0;
+    int op_errno = 0;
+    void *cookie = NULL;
+    uint64_t ctx_addr = 0;
+    gf_boolean_t xref = _gf_false;
+    br_stub_private_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        ret = br_stub_get_inode_ctx (this, loc->inode, &ctx_addr);
-        if (ret < 0)
-                ctx_addr = 0;
-        if (ctx_addr != 0)
-                goto wind;
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
 
-        /**
-         * fresh lookup: request version keys from POSIX
-         */
-        op_errno = ENOMEM;
-        if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata)
-                        goto unwind;
-        } else {
-                xdata = dict_ref (xdata);
+    priv = this->private;
+
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    if (!gf_uuid_compare(loc->gfid, priv->bad_object_dir_gfid) ||
+        !gf_uuid_compare(loc->pargfid, priv->bad_object_dir_gfid)) {
+        stub = fop_lookup_stub(frame, br_stub_lookup_wrapper, loc, xdata);
+        if (!stub) {
+            op_errno = ENOMEM;
+            goto unwind;
         }
+        br_stub_worker_enqueue(this, stub);
+        return 0;
+    }
 
-        xref = _gf_true;
+    ret = br_stub_get_inode_ctx(this, loc->inode, &ctx_addr);
+    if (ret < 0)
+        ctx_addr = 0;
+    if (ctx_addr != 0)
+        goto wind;
 
-        /**
-         * Requesting both xattrs provides a way of sanity checking the
-         * object. Anomaly checking is done in cbk by examining absence
-         * of either or both xattrs.
-         */
-        op_errno = EINVAL;
-        ret = dict_set_uint32 (xdata, BITROT_CURRENT_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (xdata, BITROT_SIGNING_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (xdata, BITROT_OBJECT_BAD_KEY, 0);
-        if (ret)
-                goto unwind;
-        cookie = (void *) BR_STUB_REQUEST_COOKIE;
+    /**
+     * fresh lookup: request version keys from POSIX
+     */
+    op_errno = ENOMEM;
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto unwind;
+    } else {
+        xdata = dict_ref(xdata);
+    }
+
+    xref = _gf_true;
+
+    /**
+     * Requesting both xattrs provides a way of sanity checking the
+     * object. Anomaly checking is done in cbk by examining absence
+     * of either or both xattrs.
+     */
+    op_errno = EINVAL;
+    ret = dict_set_uint32(xdata, BITROT_CURRENT_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(xdata, BITROT_SIGNING_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(xdata, BITROT_OBJECT_BAD_KEY, 0);
+    if (ret)
+        goto unwind;
+    cookie = (void *)BR_STUB_REQUEST_COOKIE;
 
- wind:
-        STACK_WIND_COOKIE (frame, br_stub_lookup_cbk, cookie,
-                           FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
-                           loc, xdata);
-        goto dealloc_dict;
+wind:
+    STACK_WIND_COOKIE(frame, br_stub_lookup_cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    goto dealloc_dict;
 
- unwind:
-        STACK_UNWIND_STRICT (lookup, frame,
-                             -1, op_errno, NULL, NULL, NULL, NULL);
- dealloc_dict:
-        if (xref)
-                dict_unref (xdata);
-        return 0;
+unwind:
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+dealloc_dict:
+    if (xref)
+        dict_unref(xdata);
+    return 0;
 }
 
 /** }}} */
@@ -2789,52 +3097,64 @@ br_stub_lookup (call_frame_t *frame,
 
 /* stat */
 int
-br_stub_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+br_stub_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t ret      = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = EINVAL;
+    int32_t ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
 
-        if (!IA_ISREG (loc->inode->ia_type))
-                goto wind;
+    priv = this->private;
 
-        ret = br_stub_check_bad_object (this, loc->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    if (!priv->do_versioning)
+        goto wind;
 
- wind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->stat, loc, xdata);
-        return 0;
+    if (!IA_ISREG(loc->inode->ia_type))
+        goto wind;
+
+    ret = br_stub_check_bad_object(this, loc->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 /* fstat */
 int
-br_stub_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+br_stub_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t ret      = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = EINVAL;
+    int32_t ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
 
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
+    priv = this->private;
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    if (!priv->do_versioning)
+        goto wind;
 
- wind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
@@ -2844,101 +3164,114 @@ unwind:
 /* unlink() */
 
 int
-br_stub_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+br_stub_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        br_stub_local_t     *local    = NULL;
-        inode_t             *inode    = NULL;
-        uint64_t             ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        int32_t              ret      = -1;
-
-        local = frame->local;
-        frame->local = NULL;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        inode = local->u.context.inode;
-        if (!IA_ISREG (inode->ia_type))
-           goto unwind;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                /**
-                 * If the inode is bad AND context is not there, then there
-                 * is a possibility of the gfid of the object being listed
-                 * in the quarantine directory and will be shown in the
-                 * bad objects list. So continuing with the fop with a
-                 * warning log. The entry from the quarantine directory
-                 * has to be removed manually. Its not a good idea to fail
-                 * the fop, as the object has already been deleted.
-                 */
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED,
-                        "failed to get the context for the inode %s",
-                        uuid_utoa (inode->gfid));
-                goto unwind;
-        }
+    br_stub_local_t *local = NULL;
+    inode_t *inode = NULL;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), unwind);
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_NULL_LOCAL, NULL);
+        goto unwind;
+    }
+    inode = local->u.context.inode;
+    if (!IA_ISREG(inode->ia_type))
+        goto unwind;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        /**
+         * If the inode is bad AND context is not there, then there
+         * is a possibility of the gfid of the object being listed
+         * in the quarantine directory and will be shown in the
+         * bad objects list. So continuing with the fop with a
+         * warning log. The entry from the quarantine directory
+         * has to be removed manually. Its not a good idea to fail
+         * the fop, as the object has already been deleted.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto unwind;
+    }
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&inode->lock);
-        {
-                /**
-                 * Ignoring the return value of br_stub_del ().
-                 * There is not much that can be done if unlinking
-                 * of the entry in the quarantine directory fails.
-                 * The failure is logged.
-                 */
-                if (__br_stub_is_bad_object (ctx))
-                        (void) br_stub_del (this, inode->gfid);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        /**
+         * Ignoring the return value of br_stub_del ().
+         * There is not much that can be done if unlinking
+         * of the entry in the quarantine directory fails.
+         * The failure is logged.
+         */
+        if (__br_stub_is_bad_object(ctx))
+            (void)br_stub_del(this, inode->gfid);
+    }
+    UNLOCK(&inode->lock);
 
 unwind:
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-        return 0;
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 int
-br_stub_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-                dict_t *xdata)
+br_stub_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+               dict_t *xdata)
 {
-        br_stub_local_t *local = NULL;
-        int32_t          op_ret = -1;
-        int32_t          op_errno = 0;
+    br_stub_local_t *local = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    br_stub_private_t *priv = NULL;
 
-        local = br_stub_alloc_local (this);
-        if (!local) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_NO_MEMORY,
-                        "failed to allocate memory for local (path: %s, gfid: %s)",
-                        loc->path, uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_ALLOC_MEM_FAILED,
+                "local path=%s", loc->path, "gfid=%s",
+                uuid_utoa(loc->inode->gfid), NULL);
+        goto unwind;
+    }
 
-        br_stub_fill_local (local, NULL, NULL, loc->inode,
-                            loc->inode->gfid,
-                            BR_STUB_NO_VERSIONING, 0);
+    br_stub_fill_local(local, NULL, NULL, loc->inode, loc->inode->gfid,
+                       BR_STUB_NO_VERSIONING, 0);
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, br_stub_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink, loc, flag, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, br_stub_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flag, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
-        return 0;
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /** }}} */
 
 /** {{{ */
@@ -2946,20 +3279,20 @@ unwind:
 /* forget() */
 
 int
-br_stub_forget (xlator_t *this, inode_t *inode)
+br_stub_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
 
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
+        return 0;
 
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        GF_FREE (ctx);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
 /** }}} */
@@ -2967,60 +3300,58 @@ br_stub_forget (xlator_t *this, inode_t *inode)
 /** {{{ */
 
 int32_t
-br_stub_noop (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+br_stub_noop(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        STACK_DESTROY (frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static void
-br_stub_send_ipc_fop (xlator_t *this, fd_t *fd, unsigned long releaseversion,
-                      int sign_info)
-{
-        int32_t op = 0;
-        int32_t ret = 0;
-        dict_t *xdata = NULL;
-        call_frame_t *frame = NULL;
-        changelog_event_t ev = {0,};
-
-        ev.ev_type = CHANGELOG_OP_TYPE_BR_RELEASE;
-        ev.u.releasebr.version = releaseversion;
-        ev.u.releasebr.sign_info = sign_info;
-        gf_uuid_copy (ev.u.releasebr.gfid, fd->inode->gfid);
-
-        xdata = dict_new ();
-        if (!xdata) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, BRS_MSG_NO_MEMORY,
-                        "dict allocation failed: cannot send IPC FOP "
-                        "to changelog");
-                goto out;
-        }
-
-        ret = dict_set_static_bin (xdata,
-                                   "RELEASE-EVENT", &ev, CHANGELOG_EV_SIZE);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_EVENT_FAILED,
-                        "cannot set release event in dict");
-                goto dealloc_dict;
-        }
+br_stub_send_ipc_fop(xlator_t *this, fd_t *fd, unsigned long releaseversion,
+                     int sign_info)
+{
+    int32_t op = 0;
+    int32_t ret = 0;
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+
+    ev.ev_type = CHANGELOG_OP_TYPE_BR_RELEASE;
+    ev.u.releasebr.version = releaseversion;
+    ev.u.releasebr.sign_info = sign_info;
+    gf_uuid_copy(ev.u.releasebr.gfid, fd->inode->gfid);
+
+    xdata = dict_new();
+    if (!xdata) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, BRS_MSG_DICT_ALLOC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = dict_set_static_bin(xdata, "RELEASE-EVENT", &ev, CHANGELOG_EV_SIZE);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_EVENT_FAILED, NULL);
+        goto dealloc_dict;
+    }
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_CREATE_FRAME_FAILED,
-                        "create_frame() failure");
-                goto dealloc_dict;
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_CREATE_FRAME_FAILED,
+                NULL);
+        goto dealloc_dict;
+    }
 
-        op = GF_IPC_TARGET_CHANGELOG;
-        STACK_WIND (frame, br_stub_noop, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
+    op = GF_IPC_TARGET_CHANGELOG;
+    STACK_WIND(frame, br_stub_noop, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
 
- dealloc_dict:
-        dict_unref (xdata);
- out:
-        return;
+dealloc_dict:
+    dict_unref(xdata);
+out:
+    return;
 }
 
 /**
@@ -3031,7 +3362,7 @@ br_stub_send_ipc_fop (xlator_t *this, fd_t *fd, unsigned long releaseversion,
  * 3) BR_SIGN_QUICK => reopen has happened and this release should trigger sign
  * 2 events:
  * 1) GF_FOP_RELEASE
- * 2) GF_FOP_WRITE (actually a dummy write fro BitD)
+ * 2) GF_FOP_WRITE (actually a dummy write for BitD)
  *
  * This is how states are changed based on events:
  * EVENT: GF_FOP_RELEASE:
@@ -3044,115 +3375,113 @@ br_stub_send_ipc_fop (xlator_t *this, fd_t *fd, unsigned long releaseversion,
  *     set state = BR_SIGN_QUICK;
  */
 br_sign_state_t
-__br_stub_inode_sign_state (br_stub_inode_ctx_t *ctx,
-                            glusterfs_fop_t fop, fd_t *fd)
+__br_stub_inode_sign_state(br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
+                           fd_t *fd)
 {
-        br_sign_state_t sign_info = BR_SIGN_INVALID;
-
-        switch (fop) {
+    br_sign_state_t sign_info = BR_SIGN_INVALID;
 
+    switch (fop) {
         case GF_FOP_FSETXATTR:
-                sign_info = ctx->info_sign = BR_SIGN_QUICK;
-                break;
+            sign_info = ctx->info_sign = BR_SIGN_QUICK;
+            break;
 
         case GF_FOP_RELEASE:
-                GF_ASSERT (ctx->info_sign != BR_SIGN_REOPEN_WAIT);
+            GF_ASSERT(ctx->info_sign != BR_SIGN_REOPEN_WAIT);
 
-                if (ctx->info_sign == BR_SIGN_NORMAL) {
-                        sign_info = ctx->info_sign = BR_SIGN_REOPEN_WAIT;
-                } else {
-                        sign_info = ctx->info_sign;
-                        ctx->info_sign = BR_SIGN_NORMAL;
-                }
+            if (ctx->info_sign == BR_SIGN_NORMAL) {
+                sign_info = ctx->info_sign = BR_SIGN_REOPEN_WAIT;
+            } else {
+                sign_info = ctx->info_sign;
+                ctx->info_sign = BR_SIGN_NORMAL;
+            }
 
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        return sign_info;
+    return sign_info;
 }
 
 int32_t
-br_stub_release (xlator_t *this, fd_t *fd)
-{
-        int32_t              ret            = 0;
-        int32_t              flags          = 0;
-        inode_t             *inode          = NULL;
-        unsigned long        releaseversion = 0;
-        br_stub_inode_ctx_t *ctx            = NULL;
-        uint64_t             tmp            = 0;
-        br_stub_fd_t        *br_stub_fd     = NULL;
-        int32_t              signinfo       = 0;
-
-        inode = fd->inode;
-
-        LOCK (&inode->lock);
-        {
-                ctx = __br_stub_get_ongoing_version_ctx (this, inode, NULL);
-                if (ctx == NULL)
-                        goto unblock;
-                br_stub_fd = br_stub_fd_ctx_get (this, fd);
-                if (br_stub_fd) {
-                        list_del_init (&br_stub_fd->list);
-                }
-
-                ret = __br_stub_can_trigger_release
-                                    (inode, ctx, &releaseversion);
-                if (!ret)
-                        goto unblock;
-
-                signinfo = __br_stub_inode_sign_state (ctx, GF_FOP_RELEASE, fd);
-                signinfo = htonl (signinfo);
-
-                /* inode back to initital state: mark dirty */
-                if (ctx->info_sign == BR_SIGN_NORMAL) {
-                        __br_stub_mark_inode_dirty (ctx);
-                        __br_stub_unset_inode_modified (ctx);
-                }
-        }
- unblock:
-        UNLOCK (&inode->lock);
+br_stub_release(xlator_t *this, fd_t *fd)
+{
+    int32_t ret = 0;
+    int32_t flags = 0;
+    inode_t *inode = NULL;
+    unsigned long releaseversion = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t tmp = 0;
+    br_stub_fd_t *br_stub_fd = NULL;
+    int32_t signinfo = 0;
+
+    inode = fd->inode;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __br_stub_get_ongoing_version_ctx(this, inode, NULL);
+        if (ctx == NULL)
+            goto unblock;
+        br_stub_fd = br_stub_fd_ctx_get(this, fd);
+        if (br_stub_fd) {
+            list_del_init(&br_stub_fd->list);
+        }
+
+        ret = __br_stub_can_trigger_release(inode, ctx, &releaseversion);
+        if (!ret)
+            goto unblock;
+
+        signinfo = __br_stub_inode_sign_state(ctx, GF_FOP_RELEASE, fd);
+        signinfo = htonl(signinfo);
+
+        /* inode back to initital state: mark dirty */
+        if (ctx->info_sign == BR_SIGN_NORMAL) {
+            __br_stub_mark_inode_dirty(ctx);
+            __br_stub_unset_inode_modified(ctx);
+        }
+    }
+unblock:
+    UNLOCK(&inode->lock);
 
-        if (ret) {
-                gf_msg_debug (this->name, 0, "releaseversion: %lu | flags: %d "
-                              "| signinfo: %d",
-                              (unsigned long) ntohl (releaseversion), flags,
-                              ntohl(signinfo));
-                br_stub_send_ipc_fop (this, fd, releaseversion, signinfo);
-        }
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "releaseversion: %lu | flags: %d "
+                     "| signinfo: %d",
+                     (unsigned long)ntohl(releaseversion), flags,
+                     ntohl(signinfo));
+        br_stub_send_ipc_fop(this, fd, releaseversion, signinfo);
+    }
 
-        ret = fd_ctx_del (fd, this, &tmp);
-        br_stub_fd = (br_stub_fd_t *)(long)tmp;
+    ret = fd_ctx_del(fd, this, &tmp);
+    br_stub_fd = (br_stub_fd_t *)(long)tmp;
 
-        GF_FREE (br_stub_fd);
+    GF_FREE(br_stub_fd);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_releasedir (xlator_t *this, fd_t *fd)
+br_stub_releasedir(xlator_t *this, fd_t *fd)
 {
-        br_stub_fd_t   *fctx = NULL;
-        uint64_t        ctx  = 0;
-        int             ret  = 0;
+    br_stub_fd_t *fctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        ret = fd_ctx_del (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
-
-        fctx = (br_stub_fd_t *) (long) ctx;
-        if (fctx->bad_object.dir) {
-                ret = sys_closedir (fctx->bad_object.dir);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL,
-                                "closedir error: %s", strerror (errno));
-        }
+    ret = fd_ctx_del(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        GF_FREE (fctx);
+    fctx = (br_stub_fd_t *)(long)ctx;
+    if (fctx->bad_object.dir) {
+        ret = sys_closedir(fctx->bad_object.dir);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL,
+                    "error=%s", strerror(errno), NULL);
+    }
+
+    GF_FREE(fctx);
 out:
-        return 0;
+    return 0;
 }
 
 /** }}} */
@@ -3162,84 +3491,100 @@ out:
 /* ictxmerge */
 
 void
-br_stub_ictxmerge (xlator_t *this, fd_t *fd,
-                   inode_t *inode, inode_t *linked_inode)
-{
-        int32_t              ret        = 0;
-        uint64_t             ctxaddr    = 0;
-        uint64_t             lctxaddr   = 0;
-        br_stub_inode_ctx_t *ctx        = NULL;
-        br_stub_inode_ctx_t *lctx       = NULL;
-        br_stub_fd_t        *br_stub_fd = NULL;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctxaddr);
+br_stub_ictxmerge(xlator_t *this, fd_t *fd, inode_t *inode,
+                  inode_t *linked_inode)
+{
+    int32_t ret = 0;
+    uint64_t ctxaddr = 0;
+    uint64_t lctxaddr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    br_stub_inode_ctx_t *lctx = NULL;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctxaddr);
+    if (ret < 0)
+        goto done;
+    ctx = (br_stub_inode_ctx_t *)(uintptr_t)ctxaddr;
+
+    LOCK(&linked_inode->lock);
+    {
+        ret = __br_stub_get_inode_ctx(this, linked_inode, &lctxaddr);
         if (ret < 0)
-                goto done;
-        ctx = (br_stub_inode_ctx_t *) ctxaddr;
+            goto unblock;
+        lctx = (br_stub_inode_ctx_t *)(uintptr_t)lctxaddr;
 
-        LOCK (&linked_inode->lock);
-        {
-                ret = __br_stub_get_inode_ctx (this, linked_inode, &lctxaddr);
-                if (ret < 0)
-                        goto unblock;
-                lctx = (br_stub_inode_ctx_t *) lctxaddr;
-
-                GF_ASSERT (list_is_singular (&ctx->fd_list));
-                br_stub_fd = list_first_entry (&ctx->fd_list, br_stub_fd_t,
-                                               list);
-                if (br_stub_fd) {
-                        GF_ASSERT (br_stub_fd->fd == fd);
-                        list_move_tail (&br_stub_fd->list, &lctx->fd_list);
-                }
+        GF_ASSERT(list_is_singular(&ctx->fd_list));
+        br_stub_fd = list_first_entry(&ctx->fd_list, br_stub_fd_t, list);
+        if (br_stub_fd) {
+            GF_ASSERT(br_stub_fd->fd == fd);
+            list_move_tail(&br_stub_fd->list, &lctx->fd_list);
         }
+    }
 unblock:
-        UNLOCK (&linked_inode->lock);
+    UNLOCK(&linked_inode->lock);
 
- done:
-        return;
+done:
+    return;
 }
 
 /** }}} */
 
-
 struct xlator_fops fops = {
-        .lookup    = br_stub_lookup,
-        .stat      = br_stub_stat,
-        .fstat     = br_stub_fstat,
-        .open      = br_stub_open,
-        .create    = br_stub_create,
-        .readdirp  = br_stub_readdirp,
-        .getxattr  = br_stub_getxattr,
-        .fgetxattr = br_stub_fgetxattr,
-        .fsetxattr = br_stub_fsetxattr,
-        .writev    = br_stub_writev,
-        .truncate  = br_stub_truncate,
-        .ftruncate = br_stub_ftruncate,
-        .mknod     = br_stub_mknod,
-        .readv     = br_stub_readv,
-        .removexattr = br_stub_removexattr,
-        .fremovexattr = br_stub_fremovexattr,
-        .setxattr  = br_stub_setxattr,
-        .opendir   = br_stub_opendir,
-        .readdir   = br_stub_readdir,
-        .unlink    = br_stub_unlink,
+    .lookup = br_stub_lookup,
+    .stat = br_stub_stat,
+    .fstat = br_stub_fstat,
+    .open = br_stub_open,
+    .create = br_stub_create,
+    .readdirp = br_stub_readdirp,
+    .getxattr = br_stub_getxattr,
+    .fgetxattr = br_stub_fgetxattr,
+    .fsetxattr = br_stub_fsetxattr,
+    .writev = br_stub_writev,
+    .truncate = br_stub_truncate,
+    .ftruncate = br_stub_ftruncate,
+    .mknod = br_stub_mknod,
+    .readv = br_stub_readv,
+    .removexattr = br_stub_removexattr,
+    .fremovexattr = br_stub_fremovexattr,
+    .setxattr = br_stub_setxattr,
+    .opendir = br_stub_opendir,
+    .readdir = br_stub_readdir,
+    .unlink = br_stub_unlink,
 };
 
 struct xlator_cbks cbks = {
-        .forget    = br_stub_forget,
-        .release   = br_stub_release,
-        .ictxmerge = br_stub_ictxmerge,
+    .forget = br_stub_forget,
+    .release = br_stub_release,
+    .ictxmerge = br_stub_ictxmerge,
 };
 
 struct volume_options options[] = {
-        { .key = {"bitrot"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "enable/disable bitrot stub"
-        },
-        { .key = {"export"},
-          .type = GF_OPTION_TYPE_PATH,
-          .description = "brick path for versioning"
-        },
-	{ .key  = {NULL} },
+    {.key = {"bitrot"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+     .tags = {"bitrot"},
+     .description = "enable/disable bitrot stub"},
+    {.key = {"export"},
+     .type = GF_OPTION_TYPE_PATH,
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .tags = {"bitrot"},
+     .description = "brick path for versioning",
+     .default_value = "{{ brick.path }}"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "bitrot-stub",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.h b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
index 2d515417059..edd79a77e4f 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
@@ -1,288 +1,325 @@
- /*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 #ifndef __BIT_ROT_STUB_H__
 #define __BIT_ROT_STUB_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "call-stub.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
 #include "bit-rot-stub-mem-types.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
 #include "bit-rot-common.h"
 #include "bit-rot-stub-messages.h"
 #include "glusterfs3-xdr.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 
-#define BAD_OBJECT_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+#define BAD_OBJECT_THREAD_STACK_SIZE ((size_t)(1024 * 1024))
+#define BR_STUB_DUMP_STR_SIZE 65536
 
-typedef int (br_stub_version_cbk) (call_frame_t *, void *,
-                                   xlator_t *, int32_t, int32_t, dict_t *);
+#define BR_PATH_MAX_EXTRA (PATH_MAX + 1024)
+#define BR_PATH_MAX_PLUS (PATH_MAX + 2048)
+
+/*
+ * Oops. Spelling mistake. Correcting it
+ */
+#define OLD_BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH "/quanrantine"
+#define BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH "/quarantine"
+
+/* do not reference frame->local in cbk unless initialized.
+ * Assigned 0x1 marks verisoning flag between call path and
+ * cbk path.
+ */
+#define BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, label)                   \
+    do {                                                                       \
+        if (priv->do_versioning)                                               \
+            frame->local = (void *)0x1;                                        \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define BR_STUB_VER_COND_GOTO(priv, cond, label)                               \
+    do {                                                                       \
+        if (!priv->do_versioning || cond)                                      \
+            goto label;                                                        \
+    } while (0)
+
+#define BR_STUB_VER_ENABLED_IN_CALLPATH(frame, flag)                           \
+    do {                                                                       \
+        if (frame->local)                                                      \
+            flag = _gf_true;                                                   \
+        if (frame->local == (void *)0x1)                                       \
+            frame->local = NULL;                                               \
+    } while (0)
+
+#define BR_STUB_RESET_LOCAL_NULL(frame)                                        \
+    do {                                                                       \
+        if (frame->local == (void *)0x1)                                       \
+            frame->local = NULL;                                               \
+    } while (0)
+
+typedef int(br_stub_version_cbk)(call_frame_t *, void *, xlator_t *, int32_t,
+                                 int32_t, dict_t *);
 
 typedef struct br_stub_inode_ctx {
-        int need_writeback;                     /* does the inode need
-                                                      a writeback to disk? */
-        unsigned long currentversion;           /* ongoing version */
-
-        int            info_sign;
-        struct list_head fd_list; /* list of open fds or fds participating in
-                                     write operations */
-        gf_boolean_t bad_object;
+    int need_writeback;           /* does the inode need
+                                        a writeback to disk? */
+    unsigned long currentversion; /* ongoing version */
+
+    int info_sign;
+    struct list_head fd_list; /* list of open fds or fds participating in
+                                 write operations */
+    gf_boolean_t bad_object;
 } br_stub_inode_ctx_t;
 
 typedef struct br_stub_fd {
-        fd_t *fd;
-        struct list_head list;
-        struct bad_object_dir {
-                DIR *dir;
-                off_t dir_eof;
-        } bad_object;
+    fd_t *fd;
+    struct list_head list;
+    struct bad_object_dir {
+        DIR *dir;
+        off_t dir_eof;
+    } bad_object;
 } br_stub_fd_t;
 
-#define I_DIRTY  (1<<0)        /* inode needs writeback */
-#define I_MODIFIED (1<<1)
-#define WRITEBACK_DURABLE 1    /* writeback is durable */
+#define I_DIRTY (1 << 0) /* inode needs writeback */
+#define I_MODIFIED (1 << 1)
+#define WRITEBACK_DURABLE 1 /* writeback is durable */
 
 /**
  * This could just have been a plain struct without unions and all,
  * but we may need additional things in the future.
  */
 typedef struct br_stub_local {
-        call_stub_t *fopstub;   /* stub for original fop */
-
-        int versioningtype;     /* not much used atm */
-
-        union {
-                struct br_stub_ctx {
-                        fd_t          *fd;
-                        uuid_t         gfid;
-                        inode_t       *inode;
-                        unsigned long  version;
-                } context;
-        } u;
+    call_stub_t *fopstub; /* stub for original fop */
+
+    int versioningtype; /* not much used atm */
+
+    union {
+        struct br_stub_ctx {
+            fd_t *fd;
+            uuid_t gfid;
+            inode_t *inode;
+            unsigned long version;
+        } context;
+    } u;
 } br_stub_local_t;
 
 #define BR_STUB_NO_VERSIONING (1 << 0)
 #define BR_STUB_INCREMENTAL_VERSIONING (1 << 1)
 
 typedef struct br_stub_private {
-        gf_boolean_t go;
+    gf_boolean_t do_versioning;
 
-        uint32_t boot[2];
-        char export[PATH_MAX];
+    uint32_t boot[2];
+    char export[PATH_MAX];
 
-        pthread_mutex_t lock;
-        pthread_cond_t  cond;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
 
-        struct list_head squeue;      /* ordered signing queue */
-        pthread_t signth;
-        struct bad_objects_container {
-                pthread_t thread;
-                pthread_mutex_t bad_lock;
-                pthread_cond_t  bad_cond;
-                struct list_head bad_queue;
-        } container;
-        struct mem_pool *local_pool;
+    struct list_head squeue; /* ordered signing queue */
+    pthread_t signth;
+    struct bad_objects_container {
+        pthread_t thread;
+        pthread_mutex_t bad_lock;
+        pthread_cond_t bad_cond;
+        struct list_head bad_queue;
+    } container;
+    struct mem_pool *local_pool;
 
-        char stub_basepath[PATH_MAX];
+    char stub_basepath[BR_PATH_MAX_EXTRA];
 
-        uuid_t bad_object_dir_gfid;
+    uuid_t bad_object_dir_gfid;
 } br_stub_private_t;
 
 br_stub_fd_t *
-br_stub_fd_new (void);
-
+br_stub_fd_new(void);
 
 int
-__br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
+__br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
 
 br_stub_fd_t *
-__br_stub_fd_ctx_get (xlator_t *this, fd_t *fd);
+__br_stub_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 br_stub_fd_t *
-br_stub_fd_ctx_get (xlator_t *this, fd_t *fd);
+br_stub_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 int32_t
-br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
+br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
 
 static inline gf_boolean_t
-__br_stub_is_bad_object (br_stub_inode_ctx_t *ctx)
+__br_stub_is_bad_object(br_stub_inode_ctx_t *ctx)
 {
-        return ctx->bad_object;
+    return ctx->bad_object;
 }
 
 static inline void
-__br_stub_mark_object_bad (br_stub_inode_ctx_t *ctx)
+__br_stub_mark_object_bad(br_stub_inode_ctx_t *ctx)
 {
-        ctx->bad_object = _gf_true;
+    ctx->bad_object = _gf_true;
 }
 
 /* inode writeback helpers */
 static inline void
-__br_stub_mark_inode_dirty (br_stub_inode_ctx_t *ctx)
+__br_stub_mark_inode_dirty(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback |= I_DIRTY;
+    ctx->need_writeback |= I_DIRTY;
 }
 
 static inline void
-__br_stub_mark_inode_synced (br_stub_inode_ctx_t *ctx)
+__br_stub_mark_inode_synced(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback &= ~I_DIRTY;
+    ctx->need_writeback &= ~I_DIRTY;
 }
 
 static inline int
-__br_stub_is_inode_dirty (br_stub_inode_ctx_t *ctx)
+__br_stub_is_inode_dirty(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->need_writeback & I_DIRTY);
+    return (ctx->need_writeback & I_DIRTY);
 }
 
 /* inode mofification markers */
 static inline void
-__br_stub_set_inode_modified (br_stub_inode_ctx_t *ctx)
+__br_stub_set_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback |= I_MODIFIED;
+    ctx->need_writeback |= I_MODIFIED;
 }
 
 static inline void
-__br_stub_unset_inode_modified (br_stub_inode_ctx_t *ctx)
+__br_stub_unset_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback &= ~I_MODIFIED;
+    ctx->need_writeback &= ~I_MODIFIED;
 }
 
 static inline int
-__br_stub_is_inode_modified (br_stub_inode_ctx_t *ctx)
+__br_stub_is_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->need_writeback & I_MODIFIED);
+    return (ctx->need_writeback & I_MODIFIED);
 }
 
-
 static inline int
-br_stub_require_release_call (xlator_t *this, fd_t *fd, br_stub_fd_t **fd_ctx)
+br_stub_require_release_call(xlator_t *this, fd_t *fd, br_stub_fd_t **fd_ctx)
 {
-        int32_t ret = 0;
-        br_stub_fd_t *br_stub_fd = NULL;
+    int32_t ret = 0;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        br_stub_fd = br_stub_fd_new ();
-        if (!br_stub_fd)
-                return -1;
+    br_stub_fd = br_stub_fd_new();
+    if (!br_stub_fd)
+        return -1;
 
-        br_stub_fd->fd = fd;
-        INIT_LIST_HEAD (&br_stub_fd->list);
+    br_stub_fd->fd = fd;
+    INIT_LIST_HEAD(&br_stub_fd->list);
 
-        ret = br_stub_fd_ctx_set (this, fd, br_stub_fd);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_SET_CONTEXT_FAILED,
-                        "could not set fd context (for release callback");
-        else
-                *fd_ctx = br_stub_fd;
+    ret = br_stub_fd_ctx_set(this, fd, br_stub_fd);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_CONTEXT_FAILED,
+                NULL);
+    else
+        *fd_ctx = br_stub_fd;
 
-        return ret;
+    return ret;
 }
 
 /* get/set inode context helpers */
 
 static inline int
-__br_stub_get_inode_ctx (xlator_t *this,
-                         inode_t *inode, uint64_t *ctx)
+__br_stub_get_inode_ctx(xlator_t *this, inode_t *inode, uint64_t *ctx)
 {
-        return __inode_ctx_get (inode, this, ctx);
+    return __inode_ctx_get(inode, this, ctx);
 }
 
 static inline int
-br_stub_get_inode_ctx (xlator_t *this,
-                       inode_t *inode, uint64_t *ctx)
+br_stub_get_inode_ctx(xlator_t *this, inode_t *inode, uint64_t *ctx)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __br_stub_get_inode_ctx (this, inode, ctx);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __br_stub_get_inode_ctx(this, inode, ctx);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 static inline int
-br_stub_set_inode_ctx (xlator_t *this,
-                       inode_t *inode, br_stub_inode_ctx_t *ctx)
+br_stub_set_inode_ctx(xlator_t *this, inode_t *inode, br_stub_inode_ctx_t *ctx)
 {
-        uint64_t ctx_addr = (uint64_t) ctx;
-        return inode_ctx_set (inode, this, &ctx_addr);
+    uint64_t ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return inode_ctx_set(inode, this, &ctx_addr);
 }
 
 /* version get/set helpers */
 
 static inline unsigned long
-__br_stub_writeback_version (br_stub_inode_ctx_t *ctx)
+__br_stub_writeback_version(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->currentversion + 1);
+    return (ctx->currentversion + 1);
 }
 
 static inline void
-__br_stub_set_ongoing_version (br_stub_inode_ctx_t *ctx, unsigned long version)
+__br_stub_set_ongoing_version(br_stub_inode_ctx_t *ctx, unsigned long version)
 {
-        if (ctx->currentversion < version)
-                ctx->currentversion = version;
-        else
-                gf_msg ("bit-rot-stub", GF_LOG_WARNING, 0,
-                        BRS_MSG_CHANGE_VERSION_FAILED, "current version: %lu"
-                        "new version: %lu", ctx->currentversion, version);
+    if (ctx->currentversion < version)
+        ctx->currentversion = version;
+    else
+        gf_smsg("bit-rot-stub", GF_LOG_WARNING, 0,
+                BRS_MSG_CHANGE_VERSION_FAILED, "current version=%lu",
+                ctx->currentversion, "new version=%lu", version, NULL);
 }
 
 static inline int
-__br_stub_can_trigger_release (inode_t *inode,
-                               br_stub_inode_ctx_t *ctx, unsigned long *version)
+__br_stub_can_trigger_release(inode_t *inode, br_stub_inode_ctx_t *ctx,
+                              unsigned long *version)
 {
-        /**
-         * If the inode is modified, then it has to be dirty. An inode is
-         * marked dirty once version is increased. Its marked as modified
-         * when the modification call (write/truncate) which triggered
-         * the versioning is successful.
-         */
-        if (__br_stub_is_inode_modified (ctx)
-            && list_empty (&ctx->fd_list)
-            && (ctx->info_sign != BR_SIGN_REOPEN_WAIT)) {
-
-                GF_ASSERT (__br_stub_is_inode_dirty (ctx) == 0);
+    /**
+     * If the inode is modified, then it has to be dirty. An inode is
+     * marked dirty once version is increased. Its marked as modified
+     * when the modification call (write/truncate) which triggered
+     * the versioning is successful.
+     */
+    if (__br_stub_is_inode_modified(ctx) && list_empty(&ctx->fd_list) &&
+        (ctx->info_sign != BR_SIGN_REOPEN_WAIT)) {
+        GF_ASSERT(__br_stub_is_inode_dirty(ctx) == 0);
 
-                if (version)
-                        *version = htonl (ctx->currentversion);
-                return 1;
-        }
+        if (version)
+            *version = htonl(ctx->currentversion);
+        return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
 static inline int32_t
-br_stub_get_ongoing_version (xlator_t *this,
-                             inode_t *inode, unsigned long *version)
+br_stub_get_ongoing_version(xlator_t *this, inode_t *inode,
+                            unsigned long *version)
 {
-        int32_t ret = 0;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unblock;
-                ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-                *version = ctx->currentversion;
-        }
- unblock:
-        UNLOCK (&inode->lock);
-
-        return ret;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_addr);
+        if (ret < 0)
+            goto unblock;
+        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+        *version = ctx->currentversion;
+    }
+unblock:
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
 /**
@@ -291,45 +328,52 @@ br_stub_get_ongoing_version (xlator_t *this,
  * *needs* to be valid in the caller.
  */
 static inline br_stub_inode_ctx_t *
-__br_stub_get_ongoing_version_ctx (xlator_t *this,
-                                   inode_t *inode, unsigned long *version)
+__br_stub_get_ongoing_version_ctx(xlator_t *this, inode_t *inode,
+                                  unsigned long *version)
 {
-        int32_t ret = 0;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret < 0)
-                return NULL;
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-        if (version)
-                *version = ctx->currentversion;
-
-        return ctx;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret < 0)
+        return NULL;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    if (version)
+        *version = ctx->currentversion;
+
+    return ctx;
 }
 
 /* filter for xattr fetch */
 static inline int
-br_stub_is_internal_xattr (const char *name)
+br_stub_is_internal_xattr(const char *name)
 {
-        if (name
-            && ((strncmp (name, BITROT_CURRENT_VERSION_KEY,
-                          strlen (BITROT_CURRENT_VERSION_KEY)) == 0)
-                || (strncmp (name, BITROT_SIGNING_VERSION_KEY,
-                             strlen (BITROT_SIGNING_VERSION_KEY)) == 0)))
-                return 1;
-        return 0;
+    if (name && ((strncmp(name, BITROT_CURRENT_VERSION_KEY,
+                          SLEN(BITROT_CURRENT_VERSION_KEY)) == 0) ||
+                 (strncmp(name, BITROT_SIGNING_VERSION_KEY,
+                          SLEN(BITROT_SIGNING_VERSION_KEY)) == 0)))
+        return 1;
+    return 0;
 }
 
 static inline void
-br_stub_remove_vxattrs (dict_t *xattr)
+br_stub_remove_vxattrs(dict_t *xattr, gf_boolean_t remove_bad_marker)
 {
-        if (xattr) {
-                dict_del (xattr, BITROT_OBJECT_BAD_KEY);
-                dict_del (xattr, BITROT_CURRENT_VERSION_KEY);
-                dict_del (xattr, BITROT_SIGNING_VERSION_KEY);
-                dict_del (xattr, BITROT_SIGNING_XATTR_SIZE_KEY);
-        }
+    if (xattr) {
+        /*
+         * When a file is corrupted, bad-object should be
+         * set in the dict. But, other info such as version,
+         * signature etc should not be set. Hence the flag
+         * remove_bad_marker. The consumer should know whether
+         * to send the bad-object info in the dict or not.
+         */
+        if (remove_bad_marker)
+            dict_del(xattr, BITROT_OBJECT_BAD_KEY);
+        dict_del(xattr, BITROT_CURRENT_VERSION_KEY);
+        dict_del(xattr, BITROT_SIGNING_VERSION_KEY);
+        dict_del(xattr, BITROT_SIGNING_XATTR_SIZE_KEY);
+    }
 }
 
 /**
@@ -343,64 +387,60 @@ br_stub_remove_vxattrs (dict_t *xattr)
  * errors can be made into enums.
  */
 static inline int
-br_stub_is_bad_object (xlator_t *this, inode_t *inode)
+br_stub_is_bad_object(xlator_t *this, inode_t *inode)
 {
-        int                  bad_object = 0;
-        gf_boolean_t         tmp        = _gf_false;
-        uint64_t             ctx_addr   = 0;
-        br_stub_inode_ctx_t *ctx        = NULL;
-        int32_t              ret        = -1;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED,
-                        "failed to get the inode context for the inode %s",
-                        uuid_utoa (inode->gfid));
-                bad_object = -1;
-                goto out;
-        }
-
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
-
-        LOCK (&inode->lock);
-        {
-                tmp = __br_stub_is_bad_object (ctx);
-                if (tmp)
-                        bad_object = -2;
-        }
-        UNLOCK (&inode->lock);
+    int bad_object = 0;
+    gf_boolean_t tmp = _gf_false;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        bad_object = -1;
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        tmp = __br_stub_is_bad_object(ctx);
+        if (tmp)
+            bad_object = -2;
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return bad_object;
+    return bad_object;
 }
 
 static inline int32_t
-br_stub_mark_object_bad (xlator_t *this, inode_t *inode)
+br_stub_mark_object_bad(xlator_t *this, inode_t *inode)
 {
-        int32_t  ret = -1;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED, "failed to get the "
-                        "inode context for the inode %s",
-                        uuid_utoa (inode->gfid));
-               goto out;
-        }
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
 
-        LOCK (&inode->lock);
-        {
-                __br_stub_mark_object_bad (ctx);
-        }
-        UNLOCK (&inode->lock);
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        __br_stub_mark_object_bad(ctx);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -408,56 +448,68 @@ out:
  * given to the caller and the caller has to decide what to do.
  */
 static inline int32_t
-br_stub_mark_xdata_bad_object (xlator_t *this, inode_t *inode, dict_t *xdata)
+br_stub_mark_xdata_bad_object(xlator_t *this, inode_t *inode, dict_t *xdata)
 {
-        int32_t    ret = 0;
+    int32_t ret = 0;
 
-        if (br_stub_is_bad_object (this, inode) == -2)
-                ret = dict_set_int32 (xdata, GLUSTERFS_BAD_INODE, 1);
+    if (br_stub_is_bad_object(this, inode) == -2)
+        ret = dict_set_int32(xdata, GLUSTERFS_BAD_INODE, 1);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_stub_add_fd_to_inode (xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx);
+br_stub_add_fd_to_inode(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx);
 
 br_sign_state_t
-__br_stub_inode_sign_state (br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
-                            fd_t *fd);
+__br_stub_inode_sign_state(br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
+                           fd_t *fd);
 
 int
-br_stub_dir_create (xlator_t *this, br_stub_private_t *priv);
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv);
 
 int
-br_stub_add (xlator_t *this, uuid_t gfid);
+br_stub_add(xlator_t *this, uuid_t gfid);
 
 int32_t
-br_stub_create_stub_gfid (xlator_t *this, char *stub_gfid_path, uuid_t gfid);
+br_stub_create_stub_gfid(xlator_t *this, char *stub_gfid_path, uuid_t gfid);
 
 int
-br_stub_dir_create (xlator_t *this, br_stub_private_t *priv);
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv);
 
 call_stub_t *
-__br_stub_dequeue (struct list_head *callstubs);
+__br_stub_dequeue(struct list_head *callstubs);
 
 void
-__br_stub_enqueue (struct list_head *callstubs, call_stub_t *stub);
+__br_stub_enqueue(struct list_head *callstubs, call_stub_t *stub);
 
 void
-br_stub_worker_enqueue (xlator_t *this, call_stub_t *stub);
+br_stub_worker_enqueue(xlator_t *this, call_stub_t *stub);
 
 void *
-br_stub_worker (void *data);
+br_stub_worker(void *data);
 
 int32_t
-br_stub_lookup_wrapper (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, dict_t *xattr_req);
+br_stub_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xattr_req);
 
 int32_t
-br_stub_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, size_t size, off_t off, dict_t *xdata);
+br_stub_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        size_t size, off_t off, dict_t *xdata);
+
+int
+br_stub_del(xlator_t *this, uuid_t gfid);
+
+int
+br_stub_bad_objects_path(xlator_t *this, fd_t *fd, gf_dirent_t *entries,
+                         dict_t **dict);
+
+void
+br_stub_entry_xattr_fill(xlator_t *this, char *hpath, gf_dirent_t *entry,
+                         dict_t *dict);
 
 int
-br_stub_del (xlator_t *this, uuid_t gfid);
+br_stub_get_path_of_gfid(xlator_t *this, inode_t *parent, inode_t *inode,
+                         uuid_t gfid, char **path);
 
 #endif /* __BIT_ROT_STUB_H__ */
diff --git a/xlators/features/changelog/lib/examples/c/get-changes-multi.c b/xlators/features/changelog/lib/examples/c/get-changes-multi.c
index 3741bdf6edc..5ea5bbb6630 100644
--- a/xlators/features/changelog/lib/examples/c/get-changes-multi.c
+++ b/xlators/features/changelog/lib/examples/c/get-changes-multi.c
@@ -25,64 +25,66 @@
 
 #include "changelog.h"
 
-void *brick_init (void *xl, struct gf_brick_spec *brick)
+void *
+brick_init(void *xl, struct gf_brick_spec *brick)
 {
-        return brick;
+    return brick;
 }
 
-void brick_fini (void *xl, char *brick, void *data)
+void
+brick_fini(void *xl, char *brick, void *data)
 {
-        return;
+    return;
 }
 
-void brick_callback (void *xl, char *brick,
-                    void *data, changelog_event_t *ev)
+void
+brick_callback(void *xl, char *brick, void *data, changelog_event_t *ev)
 {
-        printf ("->callback: (brick,type) [%s:%d]\n", brick, ev->ev_type);
+    printf("->callback: (brick,type) [%s:%d]\n", brick, ev->ev_type);
 }
 
-void fill_brick_spec (struct gf_brick_spec *brick, char *path)
+void
+fill_brick_spec(struct gf_brick_spec *brick, char *path)
 {
-        brick->brick_path = strdup (path);
-        brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
-
-        brick->init         = brick_init;
-        brick->fini         = brick_fini;
-        brick->callback     = brick_callback;
-        brick->connected    = NULL;
-        brick->disconnected = NULL;
+    brick->brick_path = strdup(path);
+    brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
+
+    brick->init = brick_init;
+    brick->fini = brick_fini;
+    brick->callback = brick_callback;
+    brick->connected = NULL;
+    brick->disconnected = NULL;
 }
 
 int
-main (int argc, char **argv)
+main(int argc, char **argv)
 {
-        int ret = 0;
-        void *bricks = NULL;
-        struct gf_brick_spec *brick = NULL;
+    int ret = 0;
+    void *bricks = NULL;
+    struct gf_brick_spec *brick = NULL;
 
-        bricks = calloc (2, sizeof (struct gf_brick_spec));
-        if (!bricks)
-                goto error_return;
+    bricks = calloc(2, sizeof(struct gf_brick_spec));
+    if (!bricks)
+        goto error_return;
 
-        brick = (struct gf_brick_spec *)bricks;
-        fill_brick_spec (brick, "/export/z1/zwoop");
+    brick = (struct gf_brick_spec *)bricks;
+    fill_brick_spec(brick, "/export/z1/zwoop");
 
-        brick++;
-        fill_brick_spec (brick, "/export/z2/zwoop");
+    brick++;
+    fill_brick_spec(brick, "/export/z2/zwoop");
 
-        ret = gf_changelog_init (NULL);
-        if (ret)
-                goto error_return;
+    ret = gf_changelog_init(NULL);
+    if (ret)
+        goto error_return;
 
-        ret = gf_changelog_register_generic ((struct gf_brick_spec *)bricks, 2,
-                                             0, "/tmp/multi-changes.log", 9,
-                                             NULL);
-        if (ret)
-                goto error_return;
+    ret = gf_changelog_register_generic((struct gf_brick_spec *)bricks, 2, 0,
+                                        "/tmp/multi-changes.log", 9, NULL);
+    if (ret)
+        goto error_return;
 
-        /* let callbacks do the job */
-        select (0, NULL, NULL, NULL, NULL);
+    /* let callbacks do the job */
+    select(0, NULL, NULL, NULL, NULL);
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/examples/c/get-changes.c b/xlators/features/changelog/lib/examples/c/get-changes.c
index ef766c566b6..8bc651c24a4 100644
--- a/xlators/features/changelog/lib/examples/c/get-changes.c
+++ b/xlators/features/changelog/lib/examples/c/get-changes.c
@@ -27,67 +27,67 @@
 
 #include "changelog.h"
 
-#define handle_error(fn)                                \
-        printf ("%s (reason: %s)\n", fn, strerror (errno))
+#define handle_error(fn) printf("%s (reason: %s)\n", fn, strerror(errno))
 
 int
-main (int argc, char ** argv)
+main(int argc, char **argv)
 {
-        int     i           = 0;
-        int     ret         = 0;
-        ssize_t nr_changes  = 0;
-        ssize_t changes     = 0;
-        char fbuf[PATH_MAX] = {0,};
-
-        ret = gf_changelog_init (NULL);
-        if (ret) {
-                handle_error ("Init failed");
-                goto out;
+    int i = 0;
+    int ret = 0;
+    ssize_t nr_changes = 0;
+    ssize_t changes = 0;
+    char fbuf[PATH_MAX] = {
+        0,
+    };
+
+    ret = gf_changelog_init(NULL);
+    if (ret) {
+        handle_error("Init failed");
+        goto out;
+    }
+
+    /* get changes for brick "/home/vshankar/export/yow/yow-1" */
+    ret = gf_changelog_register("/export/z1/zwoop", "/tmp/scratch",
+                                "/tmp/change.log", 9, 5);
+    if (ret) {
+        handle_error("register failed");
+        goto out;
+    }
+
+    while (1) {
+        i = 0;
+        nr_changes = gf_changelog_scan();
+        if (nr_changes < 0) {
+            handle_error("scan(): ");
+            break;
         }
 
-        /* get changes for brick "/home/vshankar/export/yow/yow-1" */
-        ret = gf_changelog_register ("/export/z1/zwoop",
-                                     "/tmp/scratch", "/tmp/change.log", 9, 5);
-        if (ret) {
-                handle_error ("register failed");
-                goto out;
-        }
-
-        while (1) {
-                i = 0;
-                nr_changes = gf_changelog_scan ();
-                if (nr_changes < 0) {
-                        handle_error ("scan(): ");
-                        break;
-                }
-
-                if (nr_changes == 0)
-                        goto next;
+        if (nr_changes == 0)
+            goto next;
 
-                printf ("Got %ld changelog files\n", nr_changes);
+        printf("Got %ld changelog files\n", nr_changes);
 
-                while ( (changes =
-                         gf_changelog_next_change (fbuf, PATH_MAX)) > 0) {
-                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
+        while ((changes = gf_changelog_next_change(fbuf, PATH_MAX)) > 0) {
+            printf("changelog file [%d]: %s\n", ++i, fbuf);
 
-                        /* process changelog */
-                        /* ... */
-                        /* ... */
-                        /* ... */
-                        /* done processing */
+            /* process changelog */
+            /* ... */
+            /* ... */
+            /* ... */
+            /* done processing */
 
-                        ret = gf_changelog_done (fbuf);
-                        if (ret)
-                                handle_error ("gf_changelog_done");
-                }
+            ret = gf_changelog_done(fbuf);
+            if (ret)
+                handle_error("gf_changelog_done");
+        }
 
-                if (changes == -1)
-                        handle_error ("gf_changelog_next_change");
+        if (changes == -1)
+            handle_error("gf_changelog_next_change");
 
-        next:
-                sleep (10);
-        }
+    next:
+        sleep(10);
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
diff --git a/xlators/features/changelog/lib/examples/c/get-history.c b/xlators/features/changelog/lib/examples/c/get-history.c
index ee3ec0ad100..3e888d75ca6 100644
--- a/xlators/features/changelog/lib/examples/c/get-history.c
+++ b/xlators/features/changelog/lib/examples/c/get-history.c
@@ -27,90 +27,90 @@
 
 #include "changelog.h"
 
-#define handle_error(fn)                                \
-        printf ("%s (reason: %s)\n", fn, strerror (errno))
+#define handle_error(fn) printf("%s (reason: %s)\n", fn, strerror(errno))
 
 int
-main (int argc, char ** argv)
+main(int argc, char **argv)
 {
-        int     i            = 0;
-        int     ret          = 0;
-        ssize_t nr_changes   = 0;
-        ssize_t changes      = 0;
-        char fbuf[PATH_MAX]  = {0,};
-        unsigned long end_ts = 0;
-
-        ret = gf_changelog_init (NULL);
-        if (ret) {
-                handle_error ("init failed");
-                goto out;
+    int i = 0;
+    int ret = 0;
+    ssize_t nr_changes = 0;
+    ssize_t changes = 0;
+    char fbuf[PATH_MAX] = {
+        0,
+    };
+    unsigned long end_ts = 0;
+
+    ret = gf_changelog_init(NULL);
+    if (ret) {
+        handle_error("init failed");
+        goto out;
+    }
+
+    ret = gf_changelog_register("/export/z1/zwoop", "/tmp/scratch_v1",
+                                "/tmp/changes.log", 9, 5);
+    if (ret) {
+        handle_error("register failed");
+        goto out;
+    }
+
+    int a, b;
+    printf("give the two numbers start and end\t");
+    scanf("%d%d", &a, &b);
+    ret = gf_history_changelog("/export/z1/zwoop/.glusterfs/changelogs", a, b,
+                               3, &end_ts);
+    if (ret == -1) {
+        printf("history failed");
+        goto out;
+    }
+
+    printf("end time till when changelog available : %d , ret(%d) \t", end_ts,
+           ret);
+    fflush(stdout);
+
+    while (1) {
+        nr_changes = gf_history_changelog_scan();
+        printf("scanned, nr_changes : %d\n", nr_changes);
+        if (nr_changes < 0) {
+            handle_error("scan(): ");
+            break;
         }
 
-        ret = gf_changelog_register ("/export/z1/zwoop",
-                                     "/tmp/scratch_v1", "/tmp/changes.log",
-                                     9, 5);
-        if (ret) {
-                handle_error ("register failed");
-                goto out;
+        if (nr_changes == 0) {
+            printf("done scanning \n");
+            goto out;
         }
 
-        int a, b;
-        printf ("give the two numbers start and end\t");
-        scanf ("%d%d", &a, &b);
-        ret = gf_history_changelog ("/export/z1/zwoop/.glusterfs/changelogs",
-                                    a, b, 3, &end_ts);
-        if (ret == -1) {
-                printf ("history failed");
-                goto out;
-        }
+        printf("Got %ld changelog files\n", nr_changes);
+
+        while ((changes = gf_history_changelog_next_change(fbuf, PATH_MAX)) >
+               0) {
+            printf("changelog file [%d]: %s\n", ++i, fbuf);
 
-        printf ("end time till when changelog available : %d , ret(%d) \t", end_ts, ret);
-        fflush(stdout);
-
-        while (1) {
-                nr_changes = gf_history_changelog_scan ();
-                printf ("scanned, nr_changes : %d\n",nr_changes);
-                if (nr_changes < 0) {
-                        handle_error ("scan(): ");
-                        break;
-                }
-
-                if (nr_changes == 0) {
-                        printf ("done scanning \n");
-                        goto out;
-                }
-
-                printf ("Got %ld changelog files\n", nr_changes);
-
-                while ( (changes =
-                         gf_history_changelog_next_change (fbuf, PATH_MAX)) > 0) {
-                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
-
-                        /* process changelog */
-                        /* ... */
-                        /* ... */
-                        /* ... */
-                        /* done processing */
-
-                        ret = gf_history_changelog_done (fbuf);
-                        if (ret)
-                                handle_error ("gf_changelog_done");
-                }
-                /*
-                if (changes == -1)
-                        handle_error ("gf_changelog_next_change");
-                if (nr_changes ==1){
-                        printf("continue scanning\n");
-                }
-
-                if(nr_changes == 0){
-                        printf("done scanning \n");
-                        goto out;
-                }
-                */
+            /* process changelog */
+            /* ... */
+            /* ... */
+            /* ... */
+            /* done processing */
+
+            ret = gf_history_changelog_done(fbuf);
+            if (ret)
+                handle_error("gf_changelog_done");
+        }
+        /*
+        if (changes == -1)
+                handle_error ("gf_changelog_next_change");
+        if (nr_changes ==1){
+                printf("continue scanning\n");
         }
 
+        if(nr_changes == 0){
+                printf("done scanning \n");
+                goto out;
+        }
+        */
+    }
 
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/lib/examples/python/changes.py b/xlators/features/changelog/lib/examples/python/changes.py
index 221df642a36..c410d3b000d 100644..100755
--- a/xlators/features/changelog/lib/examples/python/changes.py
+++ b/xlators/features/changelog/lib/examples/python/changes.py
@@ -1,5 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
+from __future__ import print_function
 import os
 import sys
 import time
@@ -16,18 +17,18 @@ def get_changes(brick, scratch_dir, log_file, log_level, interval):
             cl.cl_scan()
             change_list = cl.cl_getchanges()
             if change_list:
-                print change_list
+                print(change_list)
             for change in change_list:
-                print('done with %s' % (change))
+                print(('done with %s' % (change)))
                 cl.cl_done(change)
             time.sleep(interval)
     except OSError:
         ex = sys.exc_info()[1]
-        print ex
+        print(ex)
 
 if __name__ == '__main__':
     if len(sys.argv) != 6:
-        print("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
-              % (sys.argv[0]))
+        print(("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
+              % (sys.argv[0])))
         sys.exit(1)
     get_changes(sys.argv[1], sys.argv[2], sys.argv[3], 9, int(sys.argv[4]))
diff --git a/xlators/features/changelog/lib/examples/python/libgfchangelog.py b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
index 10e73c02b34..2da9f2d2a8c 100644
--- a/xlators/features/changelog/lib/examples/python/libgfchangelog.py
+++ b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
@@ -3,7 +3,8 @@ from ctypes import *
 from ctypes.util import find_library
 
 class Changes(object):
-    libgfc = CDLL(find_library("gfchangelog"), mode=RTLD_GLOBAL, use_errno=True)
+    libgfc = CDLL(find_library("gfchangelog"), mode=RTLD_GLOBAL,
+                  use_errno=True)
 
     @classmethod
     def geterrno(cls):
diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
index 8d3edb4d63f..c933ec53ed2 100644
--- a/xlators/features/changelog/lib/src/Makefile.am
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -1,29 +1,33 @@
 libgfchangelog_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
-			-DDATADIR=\"$(localstatedir)\"
+	-DDATADIR=\"$(localstatedir)\"
 
-libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
-				-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
-				-I$(top_srcdir)/xlators/features/changelog/src \
-				-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
-				-I$(top_srcdir)/rpc/rpc-transport/socket/src \
-				-DDATADIR=\"$(localstatedir)\"
+libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64 -fpic \
+	-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/features/changelog/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/rpc-transport/socket/src \
+	-DDATADIR=\"$(localstatedir)\"
 
 libgfchangelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-			   $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-			   $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
-libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) -version-info $(LIBGFCHANGELOG_LT_VERSION)
+libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) \
+        -version-info $(LIBGFCHANGELOG_LT_VERSION) \
+        $(GF_NO_UNDEFINED)
 
-libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
 lib_LTLIBRARIES = libgfchangelog.la
 
 CONTRIB_BUILDDIR = $(top_builddir)/contrib
 
-libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-journal-handler.c gf-changelog-helpers.c \
-		gf-changelog-api.c gf-history-changelog.c gf-changelog-rpc.c gf-changelog-reborp.c \
-		$(top_srcdir)/xlators/features/changelog/src/changelog-rpc-common.c
+libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-journal-handler.c \
+	gf-changelog-helpers.c gf-changelog-api.c gf-history-changelog.c \
+	gf-changelog-rpc.c gf-changelog-reborp.c \
+	$(top_srcdir)/xlators/features/changelog/src/changelog-rpc-common.c
 
-noinst_HEADERS = gf-changelog-helpers.h gf-changelog-rpc.h gf-changelog-journal.h changelog-lib-messages.h
+noinst_HEADERS = gf-changelog-helpers.h gf-changelog-rpc.h \
+	gf-changelog-journal.h changelog-lib-messages.h
 
 CLEANFILES =
 
diff --git a/xlators/features/changelog/lib/src/changelog-lib-messages.h b/xlators/features/changelog/lib/src/changelog-lib-messages.h
index 976c67f61a9..d7fe7274353 100644
--- a/xlators/features/changelog/lib/src/changelog-lib-messages.h
+++ b/xlators/features/changelog/lib/src/changelog-lib-messages.h
@@ -11,277 +11,64 @@
 #ifndef _CHANGELOG_LIB_MESSAGES_H_
 #define _CHANGELOG_LIB_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs-message-id.h"
-
-/*! \file changelog-lib-messages.h
- *  \brief CHANGELOG_LIB log-message IDs and their descriptions.
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for readability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- * anywhere. If reused then then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
- */
-
-#define GLFS_COMP_BASE_CHANGELOG_LIB GLFS_MSGID_COMP_CHANGELOG_LIB
-#define GLFS_NUM_MESSAGES 28
-#define GLFS_MSGID_END (GLFS_COMP_BASE_CHANGELOG_LIB + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x GLFS_COMP_BASE_CHANGELOG_LIB,\
-        "Invalid: Start of messages"
-
-/*!
- * @messageid
- * @diagnosis open/opendir failed on a brick.
- * @recommended action Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define CHANGELOG_LIB_MSG_OPEN_FAILED        (GLFS_COMP_BASE_CHANGELOG_LIB + 1)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_FAILED_TO_RMDIR    (GLFS_COMP_BASE_CHANGELOG_LIB + 2)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_SCRATCH_DIR_ENTRIES_CREATION_ERROR                \
-(GLFS_COMP_BASE_CHANGELOG_LIB + 3)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED   \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 4)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_OPENDIR_ERROR    (GLFS_COMP_BASE_CHANGELOG_LIB + 5)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_RENAME_FAILED     (GLFS_COMP_BASE_CHANGELOG_LIB + 6)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_READ_ERROR       (GLFS_COMP_BASE_CHANGELOG_LIB + 7)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_HTIME_ERROR      (GLFS_COMP_BASE_CHANGELOG_LIB + 8)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_GET_TIME_ERROR   (GLFS_COMP_BASE_CHANGELOG_LIB + 9)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_WRITE_FAILED    (GLFS_COMP_BASE_CHANGELOG_LIB + 10)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_PTHREAD_ERROR    (GLFS_COMP_BASE_CHANGELOG_LIB + 11)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_MMAP_FAILED      (GLFS_COMP_BASE_CHANGELOG_LIB + 12)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_MUNMAP_FAILED    (GLFS_COMP_BASE_CHANGELOG_LIB + 13)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_ASCII_ERROR     (GLFS_COMP_BASE_CHANGELOG_LIB + 14)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_STAT_FAILED     (GLFS_COMP_BASE_CHANGELOG_LIB + 15)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_GET_XATTR_FAILED \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 16)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_PUBLISH_ERROR   (GLFS_COMP_BASE_CHANGELOG_LIB + 17)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_PARSE_ERROR     (GLFS_COMP_BASE_CHANGELOG_LIB + 18)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_TOTAL_LOG_INFO  (GLFS_COMP_BASE_CHANGELOG_LIB + 19)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_CLEANUP_ERROR   (GLFS_COMP_BASE_CHANGELOG_LIB + 20)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_UNLINK_FAILED   (GLFS_COMP_BASE_CHANGELOG_LIB + 21)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED\
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 22)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED\
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 23)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO\
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 24)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 25)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 26)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_XDR_DECODING_FAILED \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 27)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 28)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 29)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_COPY_FROM_BUFFER_FAILED \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 30)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED \
-        (GLFS_COMP_BASE_CHANGELOG_LIB + 20)
 
+GLFS_MSGID(
+    CHANGELOG_LIB, CHANGELOG_LIB_MSG_OPEN_FAILED,
+    CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
+    CHANGELOG_LIB_MSG_SCRATCH_DIR_ENTRIES_CREATION_ERROR,
+    CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+    CHANGELOG_LIB_MSG_RENAME_FAILED, CHANGELOG_LIB_MSG_READ_ERROR,
+    CHANGELOG_LIB_MSG_HTIME_ERROR, CHANGELOG_LIB_MSG_GET_TIME_ERROR,
+    CHANGELOG_LIB_MSG_WRITE_FAILED, CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+    CHANGELOG_LIB_MSG_MMAP_FAILED, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+    CHANGELOG_LIB_MSG_ASCII_ERROR, CHANGELOG_LIB_MSG_STAT_FAILED,
+    CHANGELOG_LIB_MSG_GET_XATTR_FAILED, CHANGELOG_LIB_MSG_PUBLISH_ERROR,
+    CHANGELOG_LIB_MSG_PARSE_ERROR, CHANGELOG_LIB_MSG_MIN_MAX_INFO,
+    CHANGELOG_LIB_MSG_CLEANUP_ERROR, CHANGELOG_LIB_MSG_UNLINK_FAILED,
+    CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
+    CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED, CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
+    CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO,
+    CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO, CHANGELOG_LIB_MSG_XDR_DECODING_FAILED,
+    CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO,
+    CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+    CHANGELOG_LIB_MSG_COPY_FROM_BUFFER_FAILED,
+    CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED, CHANGELOG_LIB_MSG_HIST_FAILED,
+    CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO, CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED,
+    CHANGELOG_LIB_MSG_REQUESTING_INFO, CHANGELOG_LIB_MSG_FINAL_INFO);
+
+#define CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO_STR "Registering brick"
+#define CHANGELOG_LIB_MSG_RENAME_FAILED_STR "error moving changelog file"
+#define CHANGELOG_LIB_MSG_OPEN_FAILED_STR "cannot open changelog file"
+#define CHANGELOG_LIB_MSG_UNLINK_FAILED_STR "failed to unlink"
+#define CHANGELOG_LIB_MSG_FAILED_TO_RMDIR_STR "failed to rmdir"
+#define CHANGELOG_LIB_MSG_STAT_FAILED_STR "stat failed on changelog file"
+#define CHANGELOG_LIB_MSG_PARSE_ERROR_STR "could not parse changelog"
+#define CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED_STR                               \
+    "parsing error, ceased publishing..."
+#define CHANGELOG_LIB_MSG_HTIME_ERROR_STR "fop failed on htime file"
+#define CHANGELOG_LIB_MSG_GET_XATTR_FAILED_STR                                 \
+    "error extracting max timstamp from htime file"
+#define CHANGELOG_LIB_MSG_MIN_MAX_INFO_STR "changelogs min max"
+#define CHANGELOG_LIB_MSG_REQUESTING_INFO_STR "Requesting historical changelogs"
+#define CHANGELOG_LIB_MSG_FINAL_INFO_STR "FINAL"
+#define CHANGELOG_LIB_MSG_HIST_FAILED_STR                                      \
+    "Requested changelog range is not available"
+#define CHANGELOG_LIB_MSG_GET_TIME_ERROR_STR "wrong result"
+#define CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO_STR                        \
+    "Cleaning brick entry for brick"
+#define CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO_STR "Draining event"
+#define CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO_STR "Drained event"
+#define CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO_STR "freeing entry"
 
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* !_CHANGELOG_MESSAGES_H_ */
diff --git a/xlators/features/changelog/lib/src/gf-changelog-api.c b/xlators/features/changelog/lib/src/gf-changelog-api.c
index f41b505a749..81a5cbfec10 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-api.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-api.c
@@ -8,10 +8,10 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "compat-uuid.h"
-#include "globals.h"
-#include "glusterfs.h"
-#include "syscall.h"
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
 
 #include "gf-changelog-helpers.h"
 #include "gf-changelog-journal.h"
@@ -19,55 +19,54 @@
 #include "changelog-lib-messages.h"
 
 int
-gf_changelog_done (char *file)
+gf_changelog_done(char *file)
 {
-        int                     ret    = -1;
-        char                   *buffer = NULL;
-        xlator_t               *this   = NULL;
-        gf_changelog_journal_t *jnl    = NULL;
-        char to_path[PATH_MAX]         = {0,};
-
-        errno = EINVAL;
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-
-        if (!file || !strlen (file))
-                goto out;
-
-        /* make sure 'file' is inside ->jnl_working_dir */
-        buffer = realpath (file, NULL);
-        if (!buffer)
-                goto out;
-
-        if (strncmp (jnl->jnl_working_dir,
-                     buffer, strlen (jnl->jnl_working_dir)))
-                goto out;
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_processed_dir, basename (buffer));
-        gf_msg_debug (this->name, 0,
-                      "moving %s to processed directory", file);
-        ret = sys_rename (buffer, to_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_RENAME_FAILED,
-                        "cannot move %s to %s",
-                        file, to_path);
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    errno = EINVAL;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+
+    if (!file || !strlen(file))
+        goto out;
+
+    /* make sure 'file' is inside ->jnl_working_dir */
+    buffer = realpath(file, NULL);
+    if (!buffer)
+        goto out;
+
+    if (strncmp(jnl->jnl_working_dir, buffer, strlen(jnl->jnl_working_dir)))
+        goto out;
+
+    (void)snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_processed_dir,
+                   basename(buffer));
+    gf_msg_debug(this->name, 0, "moving %s to processed directory", file);
+    ret = sys_rename(buffer, to_path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", file, "to=%s",
+                to_path, NULL);
+        goto out;
+    }
+
+    ret = 0;
 
- out:
-        if (buffer)
-                free (buffer); /* allocated by realpath() */
-        return ret;
+out:
+    if (buffer)
+        free(buffer); /* allocated by realpath() */
+    return ret;
 }
 
 /**
@@ -75,28 +74,28 @@ gf_changelog_done (char *file)
  *  for a set of changelogs, start from the beginning
  */
 int
-gf_changelog_start_fresh ()
+gf_changelog_start_fresh()
 {
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        if (gf_ftruncate (jnl->jnl_fd, 0))
-                goto out;
+    if (gf_ftruncate(jnl->jnl_fd, 0))
+        goto out;
 
-        return 0;
+    return 0;
 
- out:
-        return -1;
+out:
+    return -1;
 }
 
 /**
@@ -105,40 +104,42 @@ gf_changelog_start_fresh ()
  * consumed.
  */
 ssize_t
-gf_changelog_next_change (char *bufptr, size_t maxlen)
+gf_changelog_next_change(char *bufptr, size_t maxlen)
 {
-        ssize_t         size       = -1;
-        int             tracker_fd = 0;
-        xlator_t       *this       = NULL;
-        gf_changelog_journal_t *jnl = NULL;
-        char buffer[PATH_MAX]      = {0,};
+    ssize_t size = -1;
+    int tracker_fd = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    char buffer[PATH_MAX] = {
+        0,
+    };
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        tracker_fd = jnl->jnl_fd;
+    tracker_fd = jnl->jnl_fd;
 
-        size = gf_readline (tracker_fd, buffer, maxlen);
-        if (size < 0) {
-                size = -1;
-                goto out;
-        }
+    size = gf_readline(tracker_fd, buffer, maxlen);
+    if (size < 0) {
+        size = -1;
+        goto out;
+    }
 
-        if (size == 0)
-                goto out;
+    if (size == 0)
+        goto out;
 
-        memcpy (bufptr, buffer, size - 1);
-        bufptr[size - 1] = '\0';
+    memcpy(bufptr, buffer, size - 1);
+    bufptr[size - 1] = '\0';
 
 out:
-        return size;
+    return size;
 }
 
 /**
@@ -150,79 +151,74 @@ out:
  * This call also acts as a cancellation point for the consumer.
  */
 ssize_t
-gf_changelog_scan ()
+gf_changelog_scan()
 {
-        int             ret        = 0;
-        int             tracker_fd = 0;
-        size_t          len        = 0;
-        size_t          off        = 0;
-        xlator_t       *this       = NULL;
-        size_t          nr_entries = 0;
-        gf_changelog_journal_t *jnl = NULL;
-        struct dirent  *entryp     = NULL;
-        struct dirent  *result     = NULL;
-        char buffer[PATH_MAX]      = {0,};
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-        if (JNL_IS_API_DISCONNECTED (jnl)) {
-                errno = ENOTCONN;
-                goto out;
+    int tracker_fd = 0;
+    size_t off = 0;
+    xlator_t *this = NULL;
+    size_t nr_entries = 0;
+    gf_changelog_journal_t *jnl = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+    if (JNL_IS_API_DISCONNECTED(jnl)) {
+        errno = ENOTCONN;
+        goto out;
+    }
+
+    errno = EINVAL;
+
+    tracker_fd = jnl->jnl_fd;
+    if (gf_ftruncate(tracker_fd, 0))
+        goto out;
+
+    rewinddir(jnl->jnl_dir);
+
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(jnl->jnl_dir, scratch);
+        if (!entry || errno != 0)
+            break;
+
+        if (!strcmp(basename(entry->d_name), ".") ||
+            !strcmp(basename(entry->d_name), ".."))
+            continue;
+
+        nr_entries++;
+
+        GF_CHANGELOG_FILL_BUFFER(jnl->jnl_processing_dir, buffer, off,
+                                 strlen(jnl->jnl_processing_dir));
+        GF_CHANGELOG_FILL_BUFFER(entry->d_name, buffer, off,
+                                 strlen(entry->d_name));
+        GF_CHANGELOG_FILL_BUFFER("\n", buffer, off, 1);
+
+        if (gf_changelog_write(tracker_fd, buffer, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_WRITE_FAILED,
+                   "error writing changelog filename"
+                   " to tracker file");
+            break;
         }
+        off = 0;
+    }
 
-        errno = EINVAL;
-
-        tracker_fd = jnl->jnl_fd;
-        if (gf_ftruncate (tracker_fd, 0))
-                goto out;
-
-        len = offsetof(struct dirent, d_name)
-                + pathconf(jnl->jnl_processing_dir, _PC_NAME_MAX) + 1;
-        entryp = GF_CALLOC (1, len,
-                            gf_changelog_mt_libgfchangelog_dirent_t);
-        if (!entryp)
-                goto out;
-
-        rewinddir (jnl->jnl_dir);
-        while (1) {
-                ret = readdir_r (jnl->jnl_dir, entryp, &result);
-                if (ret || !result)
-                        break;
-
-                if (!strcmp (basename (entryp->d_name), ".")
-                     || !strcmp (basename (entryp->d_name), ".."))
-                        continue;
-
-                nr_entries++;
-
-                GF_CHANGELOG_FILL_BUFFER (jnl->jnl_processing_dir,
-                                          buffer, off,
-                                          strlen (jnl->jnl_processing_dir));
-                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
-                                          off, strlen (entryp->d_name));
-                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
-
-                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_WRITE_FAILED,
-                                "error writing changelog filename"
-                                " to tracker file");
-                        break;
-                }
-                off = 0;
-        }
-
-        GF_FREE (entryp);
-
-        if (!result) {
-                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
-                        return nr_entries;
-        }
- out:
-        return -1;
+    if (!entry) {
+        if (gf_lseek(tracker_fd, 0, SEEK_SET) != -1)
+            return nr_entries;
+    }
+out:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.c b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
index 8b35f4e9416..75f8a6dfc08 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-helpers.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
@@ -11,40 +11,36 @@
 #include "changelog-mem-types.h"
 #include "gf-changelog-helpers.h"
 #include "changelog-lib-messages.h"
-#include "syscall.h"
-
-ssize_t gf_changelog_read_path (int fd, char *buffer, size_t bufsize)
-{
-        return sys_read (fd, buffer, bufsize);
-}
+#include <glusterfs/syscall.h>
 
 size_t
-gf_changelog_write (int fd, char *buffer, size_t len)
+gf_changelog_write(int fd, char *buffer, size_t len)
 {
-        ssize_t size = 0;
-        size_t written = 0;
+    ssize_t size = 0;
+    size_t written = 0;
 
-        while (written < len) {
-                size = sys_write (fd, buffer + written, len - written);
-                if (size <= 0)
-                        break;
+    while (written < len) {
+        size = sys_write(fd, buffer + written, len - written);
+        if (size <= 0)
+            break;
 
-                written += size;
-        }
+        written += size;
+    }
 
-        return written;
+    return written;
 }
 
 void
-gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
+gf_rfc3986_encode_space_newline(unsigned char *s, char *enc, char *estr)
 {
-        for (; *s; s++) {
-                if (estr[*s])
-                        sprintf(enc, "%c", estr[*s]);
-                else
-                        sprintf(enc, "%%%02X", *s);
-                while (*++enc);
-        }
+    for (; *s; s++) {
+        if (estr[*s])
+            sprintf(enc, "%c", estr[*s]);
+        else
+            sprintf(enc, "%%%02X", *s);
+        while (*++enc)
+            ;
+    }
 }
 
 /**
@@ -57,163 +53,118 @@ gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
  * that can be done via @fflush(fp), @ftruncate(fd) and @fseek(fp),
  * but this involves mixing POSIX file descriptors and stream FILE *).
  *
- * NOTE: This implmentation still does work with more than one fd's
+ * NOTE: This implementation still does work with more than one fd's
  *       used to perform gf_readline(). For this very reason it's not
  *       made a part of libglusterfs.
  */
 
-static pthread_key_t rl_key;
-static pthread_once_t rl_once = PTHREAD_ONCE_INIT;
-
-static void
-readline_destructor (void *ptr)
-{
-        GF_FREE (ptr);
-}
-
-static void
-readline_once (void)
-{
-        pthread_key_create (&rl_key, readline_destructor);
-}
+static __thread read_line_t thread_tsd = {};
 
 static ssize_t
-my_read (read_line_t *tsd, int fd, char *ptr)
-{
-        if (tsd->rl_cnt <= 0) {
-                tsd->rl_cnt = sys_read (fd, tsd->rl_buf, MAXLINE);
-
-                if (tsd->rl_cnt < 0)
-                        return -1;
-                else if (tsd->rl_cnt == 0)
-                        return 0;
-                tsd->rl_bufptr = tsd->rl_buf;
-        }
-
-        tsd->rl_cnt--;
-        *ptr = *tsd->rl_bufptr++;
-        return 1;
-}
-
-static int
-gf_readline_init_once (read_line_t **tsd)
+my_read(read_line_t *tsd, int fd, char *ptr)
 {
-        if (pthread_once (&rl_once, readline_once) != 0)
-                return -1;
+    if (tsd->rl_cnt <= 0) {
+        tsd->rl_cnt = sys_read(fd, tsd->rl_buf, MAXLINE);
 
-        *tsd = pthread_getspecific (rl_key);
-        if (*tsd)
-                goto out;
-
-        *tsd = GF_CALLOC (1, sizeof (**tsd),
-                          gf_changelog_mt_libgfchangelog_rl_t);
-        if (!*tsd)
-                return -1;
-
-        if (pthread_setspecific (rl_key, *tsd) != 0)
-                return -1;
+        if (tsd->rl_cnt < 0)
+            return -1;
+        else if (tsd->rl_cnt == 0)
+            return 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+    }
 
- out:
-        return 0;
+    tsd->rl_cnt--;
+    *ptr = *tsd->rl_bufptr++;
+    return 1;
 }
 
 ssize_t
-gf_readline (int fd, void *vptr, size_t maxlen)
+gf_readline(int fd, void *vptr, size_t maxlen)
 {
-        size_t       n   = 0;
-        size_t       rc  = 0;
-        char         c   = ' ';
-        char        *ptr = NULL;
-        read_line_t *tsd = NULL;
-
-        if (gf_readline_init_once (&tsd))
-                return -1;
-
-        ptr = vptr;
-        for (n = 1; n < maxlen; n++) {
-                if ( (rc = my_read (tsd, fd, &c)) == 1 ) {
-                        *ptr++ = c;
-                        if (c == '\n')
-                                break;
-                } else if (rc == 0) {
-                        *ptr = '\0';
-                        return (n - 1);
-                } else
-                        return -1;
-        }
-
-        *ptr = '\0';
-        return n;
-
+    size_t n = 0;
+    size_t rc = 0;
+    char c = ' ';
+    char *ptr = NULL;
+    read_line_t *tsd = &thread_tsd;
+
+    ptr = vptr;
+    for (n = 1; n < maxlen; n++) {
+        if ((rc = my_read(tsd, fd, &c)) == 1) {
+            *ptr++ = c;
+            if (c == '\n')
+                break;
+        } else if (rc == 0) {
+            *ptr = '\0';
+            return (n - 1);
+        } else
+            return -1;
+    }
+
+    *ptr = '\0';
+    return n;
 }
 
 off_t
-gf_lseek (int fd, off_t offset, int whence)
+gf_lseek(int fd, off_t offset, int whence)
 {
-        off_t        off = 0;
-        read_line_t *tsd = NULL;
+    off_t off = 0;
+    read_line_t *tsd = &thread_tsd;
 
-        if (gf_readline_init_once (&tsd))
-                return -1;
+    off = sys_lseek(fd, offset, whence);
+    if (off == -1)
+        return -1;
 
-        off = sys_lseek (fd, offset, whence);
-        if (off == -1)
-                return -1;
+    tsd->rl_cnt = 0;
+    tsd->rl_bufptr = tsd->rl_buf;
 
-        tsd->rl_cnt = 0;
-        tsd->rl_bufptr = tsd->rl_buf;
-
-        return off;
+    return off;
 }
 
 int
-gf_ftruncate (int fd, off_t length)
+gf_ftruncate(int fd, off_t length)
 {
-        read_line_t *tsd = NULL;
+    read_line_t *tsd = &thread_tsd;
 
-        if (gf_readline_init_once (&tsd))
-                return -1;
+    if (sys_ftruncate(fd, 0))
+        return -1;
 
-        if (sys_ftruncate (fd, 0))
-                return -1;
+    tsd->rl_cnt = 0;
+    tsd->rl_bufptr = tsd->rl_buf;
 
-        tsd->rl_cnt = 0;
-        tsd->rl_bufptr = tsd->rl_buf;
-
-        return 0;
+    return 0;
 }
 
 int
-gf_thread_cleanup (xlator_t *this, pthread_t thread)
+gf_thread_cleanup(xlator_t *this, pthread_t thread)
 {
-        int ret = 0;
-        void *res = NULL;
-
-        ret = pthread_cancel (thread);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
-                        "Failed to send cancellation to thread");
-                goto error_return;
-        }
-
-        ret = pthread_join (thread, &res);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
-                        "failed to join thread");
-                goto error_return;
-        }
-
-        if (res != PTHREAD_CANCELED) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
-                        "Thread could not be cleaned up");
-                goto error_return;
-        }
-
-        return 0;
-
- error_return:
-        return -1;
+    int ret = 0;
+    void *res = NULL;
+
+    ret = pthread_cancel(thread);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "Failed to send cancellation to thread");
+        goto error_return;
+    }
+
+    ret = pthread_join(thread, &res);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "failed to join thread");
+        goto error_return;
+    }
+
+    if (res != PTHREAD_CANCELED) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "Thread could not be cleaned up");
+        goto error_return;
+    }
+
+    return 0;
+
+error_return:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
index bd21e4df035..9c609d33172 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-helpers.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -14,36 +14,37 @@
 #include <unistd.h>
 #include <dirent.h>
 #include <limits.h>
-#include "locking.h"
+#include <glusterfs/locking.h>
 
-#include <xlator.h>
+#include <glusterfs/xlator.h>
 
 #include "changelog.h"
 
 #include "changelog-rpc-common.h"
 #include "gf-changelog-journal.h"
 
-#define GF_CHANGELOG_TRACKER  "tracker"
+#define GF_CHANGELOG_TRACKER "tracker"
 
-#define GF_CHANGELOG_CURRENT_DIR    ".current"
-#define GF_CHANGELOG_PROCESSED_DIR  ".processed"
+#define GF_CHANGELOG_CURRENT_DIR ".current"
+#define GF_CHANGELOG_PROCESSED_DIR ".processed"
 #define GF_CHANGELOG_PROCESSING_DIR ".processing"
-#define GF_CHANGELOG_HISTORY_DIR    ".history"
+#define GF_CHANGELOG_HISTORY_DIR ".history"
 #define TIMESTAMP_LENGTH 10
 
 #ifndef MAXLINE
 #define MAXLINE 4096
 #endif
 
-#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len) do {     \
-                memcpy (ascii + off, ptr, len);                 \
-                off += len;                                     \
-        } while (0)
+#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len)                         \
+    do {                                                                       \
+        memcpy(ascii + off, ptr, len);                                         \
+        off += len;                                                            \
+    } while (0)
 
 typedef struct read_line {
-        int rl_cnt;
-        char *rl_bufptr;
-        char rl_buf[MAXLINE];
+    int rl_cnt;
+    char *rl_bufptr;
+    char rl_buf[MAXLINE];
 } read_line_t;
 
 struct gf_changelog;
@@ -55,51 +56,50 @@ struct gf_event;
  * ->next_seq holds the next _expected_ sequence number.
  */
 struct gf_event_list {
-        pthread_mutex_t lock;               /* protects this structure */
-        pthread_cond_t  cond;
+    pthread_mutex_t lock; /* protects this structure */
+    pthread_cond_t cond;
 
-        pthread_t invoker;
+    pthread_t invoker;
 
-        unsigned long next_seq;             /* next sequence number expected:
-                                               zero during bootstrap */
+    unsigned long next_seq; /* next sequence number expected:
+                               zero during bootstrap */
 
-        struct gf_changelog *entry;         /* backpointer to it's brick
-                                               encapsulator (entry) */
-        struct list_head events;            /* list of events */
+    struct gf_changelog *entry; /* backpointer to it's brick
+                                   encapsulator (entry) */
+    struct list_head events;    /* list of events */
 };
 
 /**
  * include a refcount if it's of use by additional layers
  */
 struct gf_event {
-        int count;
+    int count;
 
-        unsigned long seq;
+    unsigned long seq;
 
-        struct list_head list;
+    struct list_head list;
 
-        struct iovec iov[0];
+    struct iovec iov[0];
 };
-#define GF_EVENT_CALLOC_SIZE(cnt, len)                                  \
-        (sizeof (struct gf_event) + (cnt * sizeof (struct iovec)) + len)
+#define GF_EVENT_CALLOC_SIZE(cnt, len)                                         \
+    (sizeof(struct gf_event) + (cnt * sizeof(struct iovec)) + len)
 
 /**
  * assign the base address of the IO vector to the correct memory
 o * area and set it's addressable length.
  */
-#define GF_EVENT_ASSIGN_IOVEC(vec, event, len, pos)                     \
-        do {                                                            \
-                vec->iov_base = ((char *)event) +                       \
-                        sizeof (struct gf_event) +                      \
-                        (event->count * sizeof (struct iovec)) + pos;   \
-                vec->iov_len = len;                                     \
-                pos += len;                                             \
-        } while (0)
+#define GF_EVENT_ASSIGN_IOVEC(vec, event, len, pos)                            \
+    do {                                                                       \
+        vec->iov_base = ((char *)event) + sizeof(struct gf_event) +            \
+                        (event->count * sizeof(struct iovec)) + pos;           \
+        vec->iov_len = len;                                                    \
+        pos += len;                                                            \
+    } while (0)
 
 typedef enum gf_changelog_conn_state {
-        GF_CHANGELOG_CONN_STATE_PENDING = 0,
-        GF_CHANGELOG_CONN_STATE_ACCEPTED,
-        GF_CHANGELOG_CONN_STATE_DISCONNECTED,
+    GF_CHANGELOG_CONN_STATE_PENDING = 0,
+    GF_CHANGELOG_CONN_STATE_ACCEPTED,
+    GF_CHANGELOG_CONN_STATE_DISCONNECTED,
 } gf_changelog_conn_state_t;
 
 /**
@@ -107,153 +107,149 @@ typedef enum gf_changelog_conn_state {
  * notifications are streamed.
  */
 typedef struct gf_changelog {
-        gf_lock_t statelock;
-        gf_changelog_conn_state_t connstate;
+    gf_lock_t statelock;
+    gf_changelog_conn_state_t connstate;
 
-        xlator_t *this;
+    xlator_t *this;
 
-        struct list_head list;              /* list of instances */
+    struct list_head list; /* list of instances */
 
-        char brick[PATH_MAX];               /* brick path for this end-point */
+    char brick[PATH_MAX]; /* brick path for this end-point */
 
-        changelog_rpc_t grpc;               /* rpc{-clnt,svc} for this brick */
-#define RPC_PROBER(ent)  ent->grpc.rpc
-#define RPC_REBORP(ent)  ent->grpc.svc
-#define RPC_SOCK(ent)    ent->grpc.sock
+    changelog_rpc_t grpc; /* rpc{-clnt,svc} for this brick */
+#define RPC_PROBER(ent) ent->grpc.rpc
+#define RPC_REBORP(ent) ent->grpc.svc
+#define RPC_SOCK(ent) ent->grpc.sock
 
-        unsigned int notify;                /* notification flag(s) */
+    unsigned int notify; /* notification flag(s) */
 
-        FINI       *fini;                   /* destructor callback */
-        CALLBACK   *callback;               /* event callback dispatcher */
-        CONNECT    *connected;              /* connect callback */
-        DISCONNECT *disconnected;           /* disconnection callback */
+    FINI *fini;               /* destructor callback */
+    CALLBACK *callback;       /* event callback dispatcher */
+    CONNECT *connected;       /* connect callback */
+    DISCONNECT *disconnected; /* disconnection callback */
 
-        void *ptr;                          /* owner specific private data */
-        xlator_t *invokerxl;                /* consumers _this_, if valid,
-                                               assigned to THIS before cbk is
-                                               invoked */
+    void *ptr;           /* owner specific private data */
+    xlator_t *invokerxl; /* consumers _this_, if valid,
+                            assigned to THIS before cbk is
+                            invoked */
 
-        gf_boolean_t ordered;
+    gf_boolean_t ordered;
 
-        void (*queueevent) (struct gf_event_list *, struct gf_event *);
-        void (*pickevent) (struct gf_event_list *, struct gf_event **);
+    void (*queueevent)(struct gf_event_list *, struct gf_event *);
+    void (*pickevent)(struct gf_event_list *, struct gf_event **);
 
-        struct gf_event_list event;
+    struct gf_event_list event;
 } gf_changelog_t;
 
 static inline int
-gf_changelog_filter_check (gf_changelog_t *entry, changelog_event_t *event)
+gf_changelog_filter_check(gf_changelog_t *entry, changelog_event_t *event)
 {
-        if (event->ev_type & entry->notify)
-                return 1;
-        return 0;
+    if (event->ev_type & entry->notify)
+        return 1;
+    return 0;
 }
 
-#define GF_NEED_ORDERED_EVENTS(ent)  (ent->ordered == _gf_true)
+#define GF_NEED_ORDERED_EVENTS(ent) (ent->ordered == _gf_true)
 
 /** private structure */
 typedef struct gf_private {
-        pthread_mutex_t lock;            /* protects ->connections, cleanups */
-        pthread_cond_t  cond;
+    pthread_mutex_t lock; /* protects ->connections, cleanups */
+    pthread_cond_t cond;
 
-        void *api;                       /* pointer for API access */
+    void *api; /* pointer for API access */
 
-        pthread_t poller;                /* event poller thread */
-        pthread_t connectionjanitor;     /* connection cleaner */
+    pthread_t poller;            /* event poller thread */
+    pthread_t connectionjanitor; /* connection cleaner */
 
-        struct list_head connections;    /* list of connections */
-        struct list_head cleanups;       /* list of connection to be
-                                            cleaned up */
+    struct list_head connections; /* list of connections */
+    struct list_head cleanups;    /* list of connection to be
+                                     cleaned up */
 } gf_private_t;
 
-#define GF_CHANGELOG_GET_API_PTR(this) (((gf_private_t *) this->private)->api)
+#define GF_CHANGELOG_GET_API_PTR(this) (((gf_private_t *)this->private)->api)
 
 /**
  * upcall: invoke callback with _correct_ THIS
  */
-#define GF_CHANGELOG_INVOKE_CBK(this, cbk, brick, args ...)             \
-        do {                                                            \
-                xlator_t *old_this = NULL;                              \
-                xlator_t *invokerxl = NULL;                             \
-                                                                        \
-                invokerxl = entry->invokerxl;                           \
-                old_this = this;                                        \
-                                                                        \
-                if (invokerxl) {                                        \
-                        THIS = invokerxl;                               \
-                }                                                       \
-                                                                        \
-                cbk (invokerxl, brick, args);                           \
-                THIS = old_this;                                        \
-                                                                        \
-        } while (0)
-
-#define SAVE_THIS(xl)                           \
-        do {                                    \
-                old_this = xl;                  \
-                THIS = master;                  \
-        } while (0)
-
-#define RESTORE_THIS()                          \
-        do {                                    \
-                if (old_this)                   \
-                        THIS = old_this;        \
-        } while (0)
+#define GF_CHANGELOG_INVOKE_CBK(this, cbk, brick, args...)                     \
+    do {                                                                       \
+        xlator_t *old_this = NULL;                                             \
+        xlator_t *invokerxl = NULL;                                            \
+                                                                               \
+        invokerxl = entry->invokerxl;                                          \
+        old_this = this;                                                       \
+                                                                               \
+        if (invokerxl) {                                                       \
+            THIS = invokerxl;                                                  \
+        }                                                                      \
+                                                                               \
+        cbk(invokerxl, brick, args);                                           \
+        THIS = old_this;                                                       \
+                                                                               \
+    } while (0)
+
+#define SAVE_THIS(xl)                                                          \
+    do {                                                                       \
+        old_this = xl;                                                         \
+        THIS = master;                                                         \
+    } while (0)
+
+#define RESTORE_THIS()                                                         \
+    do {                                                                       \
+        if (old_this)                                                          \
+            THIS = old_this;                                                   \
+    } while (0)
 
 /** APIs and the rest */
 
 void *
-gf_changelog_process (void *data);
-
-ssize_t
-gf_changelog_read_path (int fd, char *buffer, size_t bufsize);
+gf_changelog_process(void *data);
 
 void
-gf_rfc3986_encode (unsigned char *s, char *enc, char *estr);
+gf_rfc3986_encode_space_newline(unsigned char *s, char *enc, char *estr);
 
 size_t
-gf_changelog_write (int fd, char *buffer, size_t len);
+gf_changelog_write(int fd, char *buffer, size_t len);
 
 ssize_t
-gf_readline (int fd, void *vptr, size_t maxlen);
+gf_readline(int fd, void *vptr, size_t maxlen);
 
 int
-gf_ftruncate (int fd, off_t length);
+gf_ftruncate(int fd, off_t length);
 
 off_t
-gf_lseek (int fd, off_t offset, int whence);
+gf_lseek(int fd, off_t offset, int whence);
 
 int
-gf_changelog_consume (xlator_t *this,
-                      gf_changelog_journal_t *jnl,
-                      char *from_path, gf_boolean_t no_publish);
+gf_changelog_consume(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path, gf_boolean_t no_publish);
 int
-gf_changelog_publish (xlator_t *this,
-                      gf_changelog_journal_t *jnl, char *from_path);
+gf_changelog_publish(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path);
 int
-gf_thread_cleanup (xlator_t *this, pthread_t thread);
+gf_thread_cleanup(xlator_t *this, pthread_t thread);
 void *
-gf_changelog_callback_invoker (void *arg);
+gf_changelog_callback_invoker(void *arg);
 
 int
-gf_cleanup_event (xlator_t *, struct gf_event_list *);
+gf_cleanup_event(xlator_t *, struct gf_event_list *);
 
 /* (un)ordered event queueing */
 void
-queue_ordered_event (struct gf_event_list *, struct gf_event *);
+queue_ordered_event(struct gf_event_list *, struct gf_event *);
 
 void
-queue_unordered_event (struct gf_event_list *, struct gf_event *);
+queue_unordered_event(struct gf_event_list *, struct gf_event *);
 
 /* (un)ordered event picking */
 void
-pick_event_ordered (struct gf_event_list *, struct gf_event **);
+pick_event_ordered(struct gf_event_list *, struct gf_event **);
 
 void
-pick_event_unordered (struct gf_event_list *, struct gf_event **);
+pick_event_unordered(struct gf_event_list *, struct gf_event **);
 
 /* connection janitor thread */
 void *
-gf_changelog_connection_janitor (void *);
+gf_changelog_connection_janitor(void *);
 
 #endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c b/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
index 6ea7cac88da..7f6e2329e71 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
@@ -8,11 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "compat-uuid.h"
-#include "globals.h"
-#include "glusterfs.h"
-#include "syscall.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat-errno.h>
 
 #include "gf-changelog-helpers.h"
 
@@ -25,112 +25,107 @@
 
 extern int byebye;
 
-enum changelog_versions {
-    VERSION_1_1 = 0,
-    VERSION_1_2 = 1
-};
+enum changelog_versions { VERSION_1_1 = 0, VERSION_1_2 = 1 };
 
 /**
  * number of gfid records after fop number
  */
-int nr_gfids[2][GF_FOP_MAXVALUE] = {
-    {
-        [GF_FOP_MKNOD]   = 1,
-        [GF_FOP_MKDIR]   = 1,
-        [GF_FOP_UNLINK]  = 1,
-        [GF_FOP_RMDIR]   = 1,
-        [GF_FOP_SYMLINK] = 1,
-        [GF_FOP_RENAME]  = 2,
-        [GF_FOP_LINK]    = 1,
-        [GF_FOP_CREATE]  = 1,
-    },
-    {
-        [GF_FOP_MKNOD]   = 1,
-        [GF_FOP_MKDIR]   = 1,
-        [GF_FOP_UNLINK]  = 2,
-        [GF_FOP_RMDIR]   = 2,
-        [GF_FOP_SYMLINK] = 1,
-        [GF_FOP_RENAME]  = 2,
-        [GF_FOP_LINK]    = 1,
-        [GF_FOP_CREATE]  = 1,
-    }
-};
-
-int nr_extra_recs[2][GF_FOP_MAXVALUE] = {
-    {
-        [GF_FOP_MKNOD]   = 3,
-        [GF_FOP_MKDIR]   = 3,
-        [GF_FOP_UNLINK]  = 0,
-        [GF_FOP_RMDIR]   = 0,
-        [GF_FOP_SYMLINK] = 0,
-        [GF_FOP_RENAME]  = 0,
-        [GF_FOP_LINK]    = 0,
-        [GF_FOP_CREATE]  = 3,
-    },
-    {
-        [GF_FOP_MKNOD]   = 3,
-        [GF_FOP_MKDIR]   = 3,
-        [GF_FOP_UNLINK]  = 0,
-        [GF_FOP_RMDIR]   = 0,
-        [GF_FOP_SYMLINK] = 0,
-        [GF_FOP_RENAME]  = 0,
-        [GF_FOP_LINK]    = 0,
-        [GF_FOP_CREATE]  = 3,
-    }
-};
+int nr_gfids[2][GF_FOP_MAXVALUE] = {{
+                                        [GF_FOP_MKNOD] = 1,
+                                        [GF_FOP_MKDIR] = 1,
+                                        [GF_FOP_UNLINK] = 1,
+                                        [GF_FOP_RMDIR] = 1,
+                                        [GF_FOP_SYMLINK] = 1,
+                                        [GF_FOP_RENAME] = 2,
+                                        [GF_FOP_LINK] = 1,
+                                        [GF_FOP_CREATE] = 1,
+                                    },
+                                    {
+                                        [GF_FOP_MKNOD] = 1,
+                                        [GF_FOP_MKDIR] = 1,
+                                        [GF_FOP_UNLINK] = 2,
+                                        [GF_FOP_RMDIR] = 2,
+                                        [GF_FOP_SYMLINK] = 1,
+                                        [GF_FOP_RENAME] = 2,
+                                        [GF_FOP_LINK] = 1,
+                                        [GF_FOP_CREATE] = 1,
+                                    }};
+
+int nr_extra_recs[2][GF_FOP_MAXVALUE] = {{
+                                             [GF_FOP_MKNOD] = 3,
+                                             [GF_FOP_MKDIR] = 3,
+                                             [GF_FOP_UNLINK] = 0,
+                                             [GF_FOP_RMDIR] = 0,
+                                             [GF_FOP_SYMLINK] = 0,
+                                             [GF_FOP_RENAME] = 0,
+                                             [GF_FOP_LINK] = 0,
+                                             [GF_FOP_CREATE] = 3,
+                                         },
+                                         {
+                                             [GF_FOP_MKNOD] = 3,
+                                             [GF_FOP_MKDIR] = 3,
+                                             [GF_FOP_UNLINK] = 0,
+                                             [GF_FOP_RMDIR] = 0,
+                                             [GF_FOP_SYMLINK] = 0,
+                                             [GF_FOP_RENAME] = 0,
+                                             [GF_FOP_LINK] = 0,
+                                             [GF_FOP_CREATE] = 3,
+                                         }};
 
 static char *
-binary_to_ascii (uuid_t uuid)
+binary_to_ascii(uuid_t uuid)
 {
-        return uuid_utoa (uuid);
+    return uuid_utoa(uuid);
 }
 
 static char *
-conv_noop (char *ptr) { return ptr; }
-
-#define VERIFY_SEPARATOR(ptr, plen, perr)       \
-        {                                       \
-                if (*(ptr + plen) != '\0') {    \
-                        perr = 1;               \
-                        break;                  \
-                }                               \
-        }
+conv_noop(char *ptr)
+{
+    return ptr;
+}
 
-#define MOVER_MOVE(mover, nleft, bytes)         \
-        {                                       \
-                mover += bytes;                 \
-                nleft -= bytes;                 \
-        }                                       \
-
-#define PARSE_GFID(mov, ptr, le, fn, perr)                      \
-        {                                                       \
-                VERIFY_SEPARATOR (mov, le, perr);               \
-                ptr = fn (mov);                                 \
-                if (!ptr) {                                     \
-                        perr = 1;                               \
-                        break;                                  \
-                }                                               \
-        }
+#define VERIFY_SEPARATOR(ptr, plen, perr)                                      \
+    {                                                                          \
+        if (*(ptr + plen) != '\0') {                                           \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+    }
 
-#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                          \
-        {                                                               \
-                GF_CHANGELOG_FILL_BUFFER (pt, buf, of, strlen (pt));    \
-                MOVER_MOVE (mo, nl, le);                                \
-        }
+#define MOVER_MOVE(mover, nleft, bytes)                                        \
+    {                                                                          \
+        mover += bytes;                                                        \
+        nleft -= bytes;                                                        \
+    }
+
+#define PARSE_GFID(mov, ptr, le, fn, perr)                                     \
+    {                                                                          \
+        VERIFY_SEPARATOR(mov, le, perr);                                       \
+        ptr = fn(mov);                                                         \
+        if (!ptr) {                                                            \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+    }
 
+#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                                 \
+    {                                                                          \
+        GF_CHANGELOG_FILL_BUFFER(pt, buf, of, strlen(pt));                     \
+        MOVER_MOVE(mo, nl, le);                                                \
+    }
 
-#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)          \
-        {                                                       \
-                memcpy (uuid, mover, sizeof (uuid_t));          \
-                ptr = binary_to_ascii (uuid);                   \
-                if (!ptr) {                                     \
-                        perr = 1;                               \
-                        break;                                  \
-                }                                               \
-                MOVER_MOVE (mover, nleft, sizeof (uuid_t));     \
-        }                                                       \
+#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)                         \
+    {                                                                          \
+        memcpy(uuid, mover, sizeof(uuid_t));                                   \
+        ptr = binary_to_ascii(uuid);                                           \
+        if (!ptr) {                                                            \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+        MOVER_MOVE(mover, nleft, sizeof(uuid_t));                              \
+    }
 
-#define LINE_BUFSIZE  (3*PATH_MAX) /* enough buffer for extra chars too */
+#define LINE_BUFSIZE (3 * PATH_MAX) /* enough buffer for extra chars too */
 
 /**
  * using mmap() makes parsing easy. fgets() cannot be used here as
@@ -145,107 +140,107 @@ conv_noop (char *ptr) { return ptr; }
  */
 
 static int
-gf_changelog_parse_binary (xlator_t *this,
-                           gf_changelog_journal_t *jnl,
-                           int from_fd, int to_fd,
-                           size_t start_offset, struct stat *stbuf,
-                           int version_idx)
+gf_changelog_parse_binary(xlator_t *this, gf_changelog_journal_t *jnl,
+                          int from_fd, int to_fd, size_t start_offset,
+                          struct stat *stbuf, int version_idx)
 
 {
-        int     ret              = -1;
-        off_t   off              = 0;
-        off_t   nleft            = 0;
-        uuid_t  uuid             = {0,};
-        char   *ptr              = NULL;
-        char   *bname_start      = NULL;
-        char   *bname_end        = NULL;
-        char   *mover            = NULL;
-        void   *start            = NULL;
-        char    current_mover    = ' ';
-        size_t  blen             = 0;
-        int     parse_err        = 0;
-        char ascii[LINE_BUFSIZE] = {0,};
-
-        nleft = stbuf->st_size;
-
-        start = mmap (NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
-        if (start == MAP_FAILED) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MMAP_FAILED,
-                        "mmap() error");
-                goto out;
-        }
-
-        mover = start;
-
-        MOVER_MOVE (mover, nleft, start_offset);
-
-        while (nleft > 0) {
+    int ret = -1;
+    off_t off = 0;
+    off_t nleft = 0;
+    uuid_t uuid = {
+        0,
+    };
+    char *ptr = NULL;
+    char *bname_start = NULL;
+    char *bname_end = NULL;
+    char *mover = NULL;
+    void *start = NULL;
+    char current_mover = ' ';
+    size_t blen = 0;
+    int parse_err = 0;
+    char *ascii = NULL;
+
+    ascii = GF_CALLOC(LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
+
+    nleft = stbuf->st_size;
+
+    start = mmap(NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
+    if (start == MAP_FAILED) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MMAP_FAILED,
+               "mmap() error");
+        goto out;
+    }
 
-                off = blen = 0;
-                ptr = bname_start = bname_end = NULL;
+    mover = start;
 
-                current_mover = *mover;
+    MOVER_MOVE(mover, nleft, start_offset);
 
-                switch (current_mover) {
-                case 'D':
-                case 'M':
-                        MOVER_MOVE (mover, nleft, 1);
-                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+    while (nleft > 0) {
+        off = blen = 0;
+        ptr = bname_start = bname_end = NULL;
 
-                        break;
+        current_mover = *mover;
 
-                case 'E':
-                        MOVER_MOVE (mover, nleft, 1);
-                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+        switch (current_mover) {
+            case 'D':
+            case 'M':
+                MOVER_MOVE(mover, nleft, 1);
+                PARSE_GFID_MOVE(ptr, uuid, mover, nleft, parse_err);
 
-                        bname_start = mover;
-                        bname_end = strchr (mover, '\n');
-                        if (bname_end == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
-
-                        blen = bname_end - bname_start;
-                        MOVER_MOVE (mover, nleft, blen);
+                break;
 
-                        break;
+            case 'E':
+                MOVER_MOVE(mover, nleft, 1);
+                PARSE_GFID_MOVE(ptr, uuid, mover, nleft, parse_err);
 
-                default:
-                        parse_err = 1;
+                bname_start = mover;
+                bname_end = strchr(mover, '\n');
+                if (bname_end == NULL) {
+                    parse_err = 1;
+                    break;
                 }
 
-                if (parse_err)
-                        break;
+                blen = bname_end - bname_start;
+                MOVER_MOVE(mover, nleft, blen);
 
-                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (ptr, ascii, off, strlen (ptr));
-                if (blen)
-                        GF_CHANGELOG_FILL_BUFFER (bname_start,
-                                                  ascii, off, blen);
-                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
-
-                if (gf_changelog_write (to_fd, ascii, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_LIB_MSG_ASCII_ERROR,
-                                "processing binary changelog failed due to "
-                                " error in writing ascii change");
-                        break;
-                }
+                break;
+
+            default:
+                parse_err = 1;
+        }
 
-                MOVER_MOVE (mover, nleft, 1);
+        if (parse_err)
+            break;
+
+        GF_CHANGELOG_FILL_BUFFER(&current_mover, ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, strlen(ptr));
+        if (blen)
+            GF_CHANGELOG_FILL_BUFFER(bname_start, ascii, off, blen);
+        GF_CHANGELOG_FILL_BUFFER("\n", ascii, off, 1);
+
+        if (gf_changelog_write(to_fd, ascii, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   CHANGELOG_LIB_MSG_ASCII_ERROR,
+                   "processing binary changelog failed due to "
+                   " error in writing ascii change");
+            break;
         }
 
-        if ((nleft == 0) && (!parse_err))
-                ret = 0;
+        MOVER_MOVE(mover, nleft, 1);
+    }
+
+    if ((nleft == 0) && (!parse_err))
+        ret = 0;
 
-        if (munmap (start, stbuf->st_size))
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MUNMAP_FAILED,
-                        "munmap() error");
- out:
-        return ret;
+    if (munmap(start, stbuf->st_size))
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+               "munmap() error");
+out:
+    if (ascii)
+        GF_FREE(ascii);
+    return ret;
 }
 
 /**
@@ -254,812 +249,781 @@ gf_changelog_parse_binary (xlator_t *this,
  *  - use fop name rather than fop number
  */
 static int
-gf_changelog_parse_ascii (xlator_t *this,
-                          gf_changelog_journal_t *jnl,
-                          int from_fd, int to_fd,
-                          size_t start_offset, struct stat *stbuf,
-                          int version_idx)
+gf_changelog_parse_ascii(xlator_t *this, gf_changelog_journal_t *jnl,
+                         int from_fd, int to_fd, size_t start_offset,
+                         struct stat *stbuf, int version_idx)
 {
-        int           ng            = 0;
-        int           ret           = -1;
-        int           fop           = 0;
-        int           len           = 0;
-        off_t         off           = 0;
-        off_t         nleft         = 0;
-        char         *ptr           = NULL;
-        char         *eptr          = NULL;
-        void         *start         = NULL;
-        char         *mover         = NULL;
-        int           parse_err     = 0;
-        char          current_mover = ' ';
-        char ascii[LINE_BUFSIZE]    = {0,};
-        const char   *fopname       = NULL;
-
-        nleft = stbuf->st_size;
-
-        start = mmap (NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
-        if (start == MAP_FAILED) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MMAP_FAILED,
-                        "mmap() error");
-                goto out;
-        }
+    int ng = 0;
+    int ret = -1;
+    int fop = 0;
+    int len = 0;
+    off_t off = 0;
+    off_t nleft = 0;
+    char *ptr = NULL;
+    char *eptr = NULL;
+    void *start = NULL;
+    char *mover = NULL;
+    int parse_err = 0;
+    char current_mover = ' ';
+    char *ascii = NULL;
+    const char *fopname = NULL;
+
+    ascii = GF_CALLOC(LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
+
+    nleft = stbuf->st_size;
+
+    start = mmap(NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
+    if (start == MAP_FAILED) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MMAP_FAILED,
+               "mmap() error");
+        goto out;
+    }
 
-        mover = start;
+    mover = start;
 
-        MOVER_MOVE (mover, nleft, start_offset);
+    MOVER_MOVE(mover, nleft, start_offset);
 
-        while (nleft > 0) {
-                off = 0;
-                current_mover = *mover;
+    while (nleft > 0) {
+        off = 0;
+        current_mover = *mover;
 
-                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(&current_mover, ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
 
-                switch (current_mover) {
-                case 'D':
-                        MOVER_MOVE (mover, nleft, 1);
+        switch (current_mover) {
+            case 'D':
+                MOVER_MOVE(mover, nleft, 1);
 
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE(ptr, ascii, off,
-                                      mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        break;
-                case 'M':
-                        MOVER_MOVE (mover, nleft, 1);
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                break;
+            case 'M':
+                MOVER_MOVE(mover, nleft, 1);
+
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                FILL_AND_MOVE(" ", ascii, off, mover, nleft, 1);
+
+                /* fop */
+                len = strlen(mover);
+                VERIFY_SEPARATOR(mover, len, parse_err);
+
+                fop = atoi(mover);
+                fopname = gf_fop_list[fop];
+                if (fopname == NULL) {
+                    parse_err = 1;
+                    break;
+                }
 
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE (ptr, ascii, off,
-                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        FILL_AND_MOVE (" ", ascii, off, mover, nleft, 1);
+                MOVER_MOVE(mover, nleft, len);
 
-                        /* fop */
-                        len = strlen (mover);
-                        VERIFY_SEPARATOR (mover, len, parse_err);
+                len = strlen(fopname);
+                GF_CHANGELOG_FILL_BUFFER(fopname, ascii, off, len);
 
-                        fop = atoi (mover);
-                        fopname = gf_fop_list[fop];
-                        if (fopname == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
+                break;
 
-                        MOVER_MOVE (mover, nleft, len);
+            case 'E':
+                MOVER_MOVE(mover, nleft, 1);
+
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                FILL_AND_MOVE(" ", ascii, off, mover, nleft, 1);
+
+                /* fop */
+                len = strlen(mover);
+                VERIFY_SEPARATOR(mover, len, parse_err);
+
+                fop = atoi(mover);
+                fopname = gf_fop_list[fop];
+                if (fopname == NULL) {
+                    parse_err = 1;
+                    break;
+                }
 
-                        len = strlen (fopname);
-                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+                MOVER_MOVE(mover, nleft, len);
 
-                        break;
+                len = strlen(fopname);
+                GF_CHANGELOG_FILL_BUFFER(fopname, ascii, off, len);
 
-                case 'E':
-                        MOVER_MOVE (mover, nleft, 1);
-
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE (ptr, ascii, off,
-                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        FILL_AND_MOVE (" ", ascii, off,
-                                       mover, nleft, 1);
-
-                        /* fop */
-                        len = strlen (mover);
-                        VERIFY_SEPARATOR (mover, len, parse_err);
-
-                        fop = atoi (mover);
-                        fopname = gf_fop_list[fop];
-                        if (fopname == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
-
-                        MOVER_MOVE (mover, nleft, len);
-
-                        len = strlen (fopname);
-                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
-
-                        ng = nr_extra_recs[version_idx][fop];
-                        for (; ng > 0; ng--) {
-                                MOVER_MOVE (mover, nleft, 1);
-                                len = strlen (mover);
-                                VERIFY_SEPARATOR (mover, len, parse_err);
-
-                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-                                FILL_AND_MOVE (mover, ascii,
-                                               off, mover, nleft, len);
-                        }
-
-                        /* pargfid + bname */
-                        ng = nr_gfids[version_idx][fop];
-                        while (ng-- > 0) {
-                                MOVER_MOVE (mover, nleft, 1);
-                                len = strlen (mover);
-                                if (!len) {
-                                        MOVER_MOVE (mover, nleft, 1);
-                                        continue;
-                                }
-
-                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-
-                                PARSE_GFID (mover, ptr, len,
-                                            conv_noop, parse_err);
-                                eptr = calloc (3, strlen (ptr));
-                                if (!eptr) {
-                                        parse_err = 1;
-                                        break;
-                                }
-
-                                gf_rfc3986_encode ((unsigned char *) ptr,
-                                                   eptr, jnl->rfc3986);
-                                FILL_AND_MOVE (eptr, ascii, off,
-                                               mover, nleft, len);
-                                free (eptr);
-                        }
+                ng = nr_extra_recs[version_idx][fop];
+                for (; ng > 0; ng--) {
+                    MOVER_MOVE(mover, nleft, 1);
+                    len = strlen(mover);
+                    VERIFY_SEPARATOR(mover, len, parse_err);
 
-                        break;
-                default:
-                        parse_err = 1;
+                    GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+                    FILL_AND_MOVE(mover, ascii, off, mover, nleft, len);
                 }
 
-                if (parse_err)
+                /* pargfid + bname */
+                ng = nr_gfids[version_idx][fop];
+                while (ng-- > 0) {
+                    MOVER_MOVE(mover, nleft, 1);
+                    len = strlen(mover);
+                    if (!len) {
+                        MOVER_MOVE(mover, nleft, 1);
+                        continue;
+                    }
+
+                    GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+
+                    PARSE_GFID(mover, ptr, len, conv_noop, parse_err);
+                    eptr = calloc(3, strlen(ptr));
+                    if (!eptr) {
+                        parse_err = 1;
                         break;
+                    }
 
-                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
-
-                if (gf_changelog_write (to_fd, ascii, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_LIB_MSG_ASCII_ERROR,
-                                "processing ascii changelog failed due to "
-                                " error in writing change");
-                        break;
+                    gf_rfc3986_encode_space_newline((unsigned char *)ptr, eptr,
+                                                    jnl->rfc3986_space_newline);
+                    FILL_AND_MOVE(eptr, ascii, off, mover, nleft, len);
+                    free(eptr);
                 }
 
-                MOVER_MOVE (mover, nleft, 1);
-
+                break;
+            default:
+                parse_err = 1;
         }
 
-        if ((nleft == 0) && (!parse_err))
-                ret = 0;
+        if (parse_err)
+            break;
 
-        if (munmap (start, stbuf->st_size))
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MUNMAP_FAILED,
-                        "munmap() error");
+        GF_CHANGELOG_FILL_BUFFER("\n", ascii, off, 1);
 
- out:
-        return ret;
-}
+        if (gf_changelog_write(to_fd, ascii, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   CHANGELOG_LIB_MSG_ASCII_ERROR,
+                   "processing ascii changelog failed due to "
+                   " error in writing change");
+            break;
+        }
 
-#define COPY_BUFSIZE  8192
-static int
-gf_changelog_copy (xlator_t *this, int from_fd, int to_fd)
-{
-        ssize_t size                  = 0;
-        char   buffer[COPY_BUFSIZE+1] = {0,};
+        MOVER_MOVE(mover, nleft, 1);
+    }
 
-        while (1) {
-                size = sys_read (from_fd, buffer, COPY_BUFSIZE);
-                if (size <= 0)
-                        break;
+    if ((nleft == 0) && (!parse_err))
+        ret = 0;
 
-                if (gf_changelog_write (to_fd,
-                                        buffer, size) != size) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_COPY_FROM_BUFFER_FAILED,
-                                "error processing ascii changlog");
-                        size = -1;
-                        break;
-                }
-        }
+    if (munmap(start, stbuf->st_size))
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+               "munmap() error");
 
-        return (size < 0 ? -1 : 0);
+out:
+    if (ascii)
+        GF_FREE(ascii);
+
+    return ret;
 }
 
 static int
-gf_changelog_decode (xlator_t *this, gf_changelog_journal_t *jnl,
-                     int from_fd, int to_fd, struct stat *stbuf, int *zerob)
+gf_changelog_decode(xlator_t *this, gf_changelog_journal_t *jnl, int from_fd,
+                    int to_fd, struct stat *stbuf, int *zerob)
 {
-        int    ret        = -1;
-        int    encoding   = -1;
-        int major_version = -1;
-        int minor_version = -1;
-        int version_idx   = -1;
-        size_t elen       = 0;
-        char buffer[1024] = {0,};
-
-        CHANGELOG_GET_HEADER_INFO (from_fd, buffer, 1024, encoding,
-                                   major_version, minor_version, elen);
-        if (encoding == -1) /* unknown encoding */
-                goto out;
-
-        if (major_version == -1) /* unknown major version */
-                goto out;
-
-        if (minor_version == -1) /* unknown minor version */
-                goto out;
-
-        if (!CHANGELOG_VALID_ENCODING (encoding))
-                goto out;
-
-        if (elen == stbuf->st_size) {
-                *zerob = 1;
-                goto out;
-        }
-
-        if (major_version == 1 && minor_version == 1) {
-                version_idx = VERSION_1_1;
-        } else if (major_version == 1 && minor_version == 2) {
-                version_idx = VERSION_1_2;
-        }
+    int ret = -1;
+    int encoding = -1;
+    int major_version = -1;
+    int minor_version = -1;
+    int version_idx = -1;
+    size_t elen = 0;
+    char buffer[1024] = {
+        0,
+    };
+
+    CHANGELOG_GET_HEADER_INFO(from_fd, buffer, sizeof(buffer), encoding,
+                              major_version, minor_version, elen);
+    if (encoding == -1) /* unknown encoding */
+        goto out;
+
+    if (major_version == -1) /* unknown major version */
+        goto out;
+
+    if (minor_version == -1) /* unknown minor version */
+        goto out;
+
+    if (!CHANGELOG_VALID_ENCODING(encoding))
+        goto out;
+
+    if (elen == stbuf->st_size) {
+        *zerob = 1;
+        goto out;
+    }
 
-        if (version_idx == -1) /* unknown version number */
-                goto out;
+    if (major_version == 1 && minor_version == 1) {
+        version_idx = VERSION_1_1;
+    } else if (major_version == 1 && minor_version == 2) {
+        version_idx = VERSION_1_2;
+    }
 
-        /**
-         * start processing after the header
-         */
-        sys_lseek (from_fd, elen, SEEK_SET);
+    if (version_idx == -1) /* unknown version number */
+        goto out;
 
-        switch (encoding) {
+    /**
+     * start processing after the header
+     */
+    if (sys_lseek(from_fd, elen, SEEK_SET) < 0) {
+        goto out;
+    }
+    switch (encoding) {
         case CHANGELOG_ENCODE_BINARY:
-                /**
-                 * this ideally should have been a part of changelog-encoders.c
-                 * (ie. part of the changelog translator).
-                 */
-                ret = gf_changelog_parse_binary (this, jnl, from_fd,
-                                                 to_fd, elen, stbuf,
-                                                 version_idx);
-                break;
+            /**
+             * this ideally should have been a part of changelog-encoders.c
+             * (ie. part of the changelog translator).
+             */
+            ret = gf_changelog_parse_binary(this, jnl, from_fd, to_fd, elen,
+                                            stbuf, version_idx);
+            break;
 
         case CHANGELOG_ENCODE_ASCII:
-                ret = gf_changelog_parse_ascii (this, jnl, from_fd,
-                                                to_fd, elen, stbuf,
-                                                version_idx);
-                break;
-        default:
-                ret = gf_changelog_copy (this, from_fd, to_fd);
-        }
+            ret = gf_changelog_parse_ascii(this, jnl, from_fd, to_fd, elen,
+                                           stbuf, version_idx);
+            break;
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-gf_changelog_publish (xlator_t *this,
-                      gf_changelog_journal_t *jnl, char *from_path)
+gf_changelog_publish(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path)
 {
-        int         ret        = 0;
-        char dest[PATH_MAX]    = {0,};
-        char to_path[PATH_MAX] = {0,};
-        struct stat stbuf      = {0,};
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_current_dir, basename (from_path));
-
-        /* handle zerob file that wont exist in current */
-        ret = sys_stat (to_path, &stbuf);
-        if (ret) {
-                if (errno == ENOENT)
-                        ret = 0;
-                goto out;
-        }
+    int ret = 0;
+    char dest[PATH_MAX] = {
+        0,
+    };
+    char to_path[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+
+    if (snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_current_dir,
+                 basename(from_path)) >= PATH_MAX)
+        return -1;
 
-        (void) snprintf (dest, PATH_MAX, "%s%s",
-                         jnl->jnl_processing_dir, basename (from_path));
+    /* handle zerob file that won't exist in current */
+    ret = sys_stat(to_path, &stbuf);
+    if (ret) {
+        if (errno == ENOENT)
+            ret = 0;
+        goto out;
+    }
 
-        ret = sys_rename (to_path, dest);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_RENAME_FAILED,
-                        "error moving %s to processing dir",
-                        to_path);
-        }
+    if (snprintf(dest, PATH_MAX, "%s%s", jnl->jnl_processing_dir,
+                 basename(from_path)) >= PATH_MAX)
+        return -1;
+
+    ret = sys_rename(to_path, dest);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", to_path, "to=%s",
+                dest, NULL);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-gf_changelog_consume (xlator_t *this,
-                      gf_changelog_journal_t *jnl,
-                      char *from_path, gf_boolean_t no_publish)
+gf_changelog_consume(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path, gf_boolean_t no_publish)
 {
-        int         ret        = -1;
-        int         fd1        = 0;
-        int         fd2        = 0;
-        int         zerob      = 0;
-        struct stat stbuf      = {0,};
-        char dest[PATH_MAX]    = {0,};
-        char to_path[PATH_MAX] = {0,};
-
-        ret = sys_stat (from_path, &stbuf);
-        if (ret || !S_ISREG(stbuf.st_mode)) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_STAT_FAILED,
-                        "stat failed on changelog file: %s", from_path);
-                goto out;
-        }
+    int ret = -1;
+    int fd1 = 0;
+    int fd2 = 0;
+    int zerob = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char dest[PATH_MAX] = {
+        0,
+    };
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    if (snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_current_dir,
+                 basename(from_path)) >= PATH_MAX)
+        goto out;
+    if (snprintf(dest, PATH_MAX, "%s%s", jnl->jnl_processing_dir,
+                 basename(from_path)) >= PATH_MAX)
+        goto out;
+
+    ret = sys_stat(from_path, &stbuf);
+    if (ret || !S_ISREG(stbuf.st_mode)) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_STAT_FAILED,
+                "path=%s", from_path, NULL);
+        goto out;
+    }
 
-        fd1 = open (from_path, O_RDONLY);
-        if (fd1 < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_OPEN_FAILED,
-                        "cannot open changelog file: %s",
-                        from_path);
-                goto out;
-        }
+    fd1 = open(from_path, O_RDONLY);
+    if (fd1 < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPEN_FAILED,
+                "path=%s", from_path, NULL);
+        goto out;
+    }
 
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_current_dir, basename (from_path));
-        (void) snprintf (dest, PATH_MAX, "%s%s",
-                         jnl->jnl_processing_dir, basename (from_path));
-
-        fd2 = open (to_path, O_CREAT | O_TRUNC | O_RDWR,
-                    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd2 < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_OPEN_FAILED,
-                        "cannot create ascii changelog file %s",
-                        to_path);
+    fd2 = open(to_path, O_CREAT | O_TRUNC | O_RDWR,
+               S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd2 < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPEN_FAILED,
+                "path=%s", to_path, NULL);
+        goto close_fd;
+    } else {
+        ret = gf_changelog_decode(this, jnl, fd1, fd2, &stbuf, &zerob);
+
+        sys_close(fd2);
+
+        if (!ret) {
+            /* move it to processing on a successful
+               decode */
+            if (no_publish == _gf_true)
                 goto close_fd;
-        } else {
-                ret = gf_changelog_decode (this, jnl, fd1,
-                                           fd2, &stbuf, &zerob);
-
-                sys_close (fd2);
-
-                if (!ret) {
-                        /* move it to processing on a successful
-                           decode */
-                        if (no_publish == _gf_true)
-                                goto close_fd;
-                        ret = sys_rename (to_path, dest);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        CHANGELOG_LIB_MSG_RENAME_FAILED,
-                                        "error moving %s to processing dir",
-                                        to_path);
-                }
+            ret = sys_rename(to_path, dest);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", to_path,
+                        "to=%s", dest, NULL);
+        }
 
-                /* remove it from .current if it's an empty file */
-                if (zerob) {
-                        /* zerob changelogs must be unlinked */
-                        ret = sys_unlink (to_path);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        CHANGELOG_LIB_MSG_UNLINK_FAILED,
-                                        "could not unlink %s",
-                                        to_path);
-                }
+        /* remove it from .current if it's an empty file */
+        if (zerob) {
+            /* zerob changelogs must be unlinked */
+            ret = sys_unlink(to_path);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_LIB_MSG_UNLINK_FAILED, "name=empty changelog",
+                        "path=%s", to_path, NULL);
         }
+    }
 
- close_fd:
-        sys_close (fd1);
+close_fd:
+    sys_close(fd1);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void *
-gf_changelog_process (void *data)
+gf_changelog_process(void *data)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
-        gf_changelog_entry_t *entry = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
-
-        jnl = data;
-        jnl_proc = jnl->jnl_proc;
-        THIS = jnl->this;
-        this = jnl->this;
-
-        while (1) {
-                pthread_mutex_lock (&jnl_proc->lock);
-                {
-                        while (list_empty (&jnl_proc->entries)) {
-                                jnl_proc->waiting = _gf_true;
-                                pthread_cond_wait
-                                        (&jnl_proc->cond, &jnl_proc->lock);
-                        }
-
-                        entry = list_first_entry (&jnl_proc->entries,
-                                                  gf_changelog_entry_t, list);
-                        list_del (&entry->list);
-                        jnl_proc->waiting = _gf_false;
-                }
-                pthread_mutex_unlock (&jnl_proc->lock);
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_entry_t *entry = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
+
+    jnl = data;
+    jnl_proc = jnl->jnl_proc;
+    THIS = jnl->this;
+    this = jnl->this;
+
+    while (1) {
+        pthread_mutex_lock(&jnl_proc->lock);
+        {
+            while (list_empty(&jnl_proc->entries)) {
+                jnl_proc->waiting = _gf_true;
+                pthread_cond_wait(&jnl_proc->cond, &jnl_proc->lock);
+            }
 
-                if (entry) {
-                        ret = gf_changelog_consume (this, jnl,
-                                                    entry->path, _gf_false);
-                        GF_FREE (entry);
-                }
+            entry = list_first_entry(&jnl_proc->entries, gf_changelog_entry_t,
+                                     list);
+            if (entry)
+                list_del(&entry->list);
+
+            jnl_proc->waiting = _gf_false;
         }
+        pthread_mutex_unlock(&jnl_proc->lock);
 
-        return NULL;
+        if (entry) {
+            (void)gf_changelog_consume(this, jnl, entry->path, _gf_false);
+            GF_FREE(entry);
+        }
+    }
+
+    return NULL;
 }
 
 void
-gf_changelog_queue_journal (gf_changelog_processor_t *jnl_proc,
-                            changelog_event_t *event)
+gf_changelog_queue_journal(gf_changelog_processor_t *jnl_proc,
+                           changelog_event_t *event)
 {
-        size_t len = 0;
-        gf_changelog_entry_t *entry = NULL;
+    size_t len = 0;
+    gf_changelog_entry_t *entry = NULL;
 
-        entry = GF_CALLOC (1, sizeof (gf_changelog_entry_t),
-                           gf_changelog_mt_libgfchangelog_entry_t);
-        if (!entry)
-                return;
-        INIT_LIST_HEAD (&entry->list);
+    entry = GF_CALLOC(1, sizeof(gf_changelog_entry_t),
+                      gf_changelog_mt_libgfchangelog_entry_t);
+    if (!entry)
+        return;
+    INIT_LIST_HEAD(&entry->list);
 
-        len = strlen (event->u.journal.path);
-        (void)memcpy (entry->path, event->u.journal.path, len+1);
+    len = strlen(event->u.journal.path);
+    (void)memcpy(entry->path, event->u.journal.path, len + 1);
+    entry->path[len] = '\0';
 
-        pthread_mutex_lock (&jnl_proc->lock);
-        {
-                list_add_tail (&entry->list, &jnl_proc->entries);
-                if (jnl_proc->waiting)
-                        pthread_cond_signal (&jnl_proc->cond);
-        }
-        pthread_mutex_unlock (&jnl_proc->lock);
+    pthread_mutex_lock(&jnl_proc->lock);
+    {
+        list_add_tail(&entry->list, &jnl_proc->entries);
+        if (jnl_proc->waiting)
+            pthread_cond_signal(&jnl_proc->cond);
+    }
+    pthread_mutex_unlock(&jnl_proc->lock);
 
-        return;
+    return;
 }
 
 void
-gf_changelog_handle_journal (void *xl, char *brick,
-                             void *cbkdata, changelog_event_t *event)
+gf_changelog_handle_journal(void *xl, char *brick, void *cbkdata,
+                            changelog_event_t *event)
 {
-        int                       ret      = 0;
-        gf_changelog_journal_t   *jnl      = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        jnl      = cbkdata;
-        jnl_proc = jnl->jnl_proc;
+    jnl = cbkdata;
+    jnl_proc = jnl->jnl_proc;
 
-        gf_changelog_queue_journal (jnl_proc, event);
+    gf_changelog_queue_journal(jnl_proc, event);
 }
 
 void
-gf_changelog_journal_disconnect (void *xl, char *brick, void *data)
+gf_changelog_journal_disconnect(void *xl, char *brick, void *data)
 {
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        jnl = data;
+    jnl = data;
 
-        pthread_spin_lock (&jnl->lock);
-        {
-                JNL_SET_API_STATE (jnl, JNL_API_DISCONNECTED);
-        };
-        pthread_spin_unlock (&jnl->lock);
+    pthread_spin_lock(&jnl->lock);
+    {
+        JNL_SET_API_STATE(jnl, JNL_API_DISCONNECTED);
+    };
+    pthread_spin_unlock(&jnl->lock);
 }
 
 void
-gf_changelog_journal_connect (void *xl, char *brick, void *data)
+gf_changelog_journal_connect(void *xl, char *brick, void *data)
 {
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        jnl = data;
+    jnl = data;
 
-        pthread_spin_lock (&jnl->lock);
-        {
-                JNL_SET_API_STATE (jnl, JNL_API_CONNECTED);
-        };
-        pthread_spin_unlock (&jnl->lock);
+    pthread_spin_lock(&jnl->lock);
+    {
+        JNL_SET_API_STATE(jnl, JNL_API_CONNECTED);
+    };
+    pthread_spin_unlock(&jnl->lock);
 
-        return;
+    return;
 }
 
 void
-gf_changelog_cleanup_processor (gf_changelog_journal_t *jnl)
+gf_changelog_cleanup_processor(gf_changelog_journal_t *jnl)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
-
-        this = THIS;
-        if (!this || !jnl || !jnl->jnl_proc)
-                goto error_return;
-
-        jnl_proc = jnl->jnl_proc;
-
-        ret = gf_thread_cleanup (this, jnl_proc->processor);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_CLEANUP_ERROR,
-                        "failed to cleanup processor thread");
-                goto error_return;
-        }
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        (void)pthread_mutex_destroy (&jnl_proc->lock);
-        (void)pthread_cond_destroy (&jnl_proc->cond);
+    this = THIS;
+    if (!this || !jnl || !jnl->jnl_proc)
+        goto error_return;
 
-        GF_FREE (jnl_proc);
+    jnl_proc = jnl->jnl_proc;
 
- error_return:
-        return;
+    ret = gf_thread_cleanup(this, jnl_proc->processor);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_CLEANUP_ERROR,
+               "failed to cleanup processor thread");
+        goto error_return;
+    }
+
+    (void)pthread_mutex_destroy(&jnl_proc->lock);
+    (void)pthread_cond_destroy(&jnl_proc->cond);
+
+    GF_FREE(jnl_proc);
+
+error_return:
+    return;
 }
 
 int
-gf_changelog_init_processor (gf_changelog_journal_t *jnl)
+gf_changelog_init_processor(gf_changelog_journal_t *jnl)
 {
-        int ret = -1;
-        gf_changelog_processor_t *jnl_proc = NULL;
+    int ret = -1;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        jnl_proc = GF_CALLOC (1, sizeof (gf_changelog_processor_t),
-                              gf_changelog_mt_libgfchangelog_t);
-        if (!jnl_proc)
-                goto error_return;
-
-        ret = pthread_mutex_init (&jnl_proc->lock, NULL);
-        if (ret != 0)
-                goto free_jnl_proc;
-        ret = pthread_cond_init (&jnl_proc->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-
-        INIT_LIST_HEAD (&jnl_proc->entries);
-        jnl_proc->waiting = _gf_false;
-        jnl->jnl_proc = jnl_proc;
-
-        ret = pthread_create (&jnl_proc->processor,
-                              NULL, gf_changelog_process, jnl);
-        if (ret != 0) {
-                jnl->jnl_proc = NULL;
-                goto cleanup_cond;
-        }
+    jnl_proc = GF_CALLOC(1, sizeof(gf_changelog_processor_t),
+                         gf_changelog_mt_libgfchangelog_t);
+    if (!jnl_proc)
+        goto error_return;
+
+    ret = pthread_mutex_init(&jnl_proc->lock, NULL);
+    if (ret != 0)
+        goto free_jnl_proc;
+    ret = pthread_cond_init(&jnl_proc->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+
+    INIT_LIST_HEAD(&jnl_proc->entries);
+    jnl_proc->waiting = _gf_false;
+    jnl->jnl_proc = jnl_proc;
+
+    ret = gf_thread_create(&jnl_proc->processor, NULL, gf_changelog_process,
+                           jnl, "clogproc");
+    if (ret != 0) {
+        jnl->jnl_proc = NULL;
+        goto cleanup_cond;
+    }
 
-        return 0;
+    return 0;
 
- cleanup_cond:
-        (void) pthread_cond_destroy (&jnl_proc->cond);
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&jnl_proc->lock);
- free_jnl_proc:
-        GF_FREE (jnl_proc);
- error_return:
-        return -1;
+cleanup_cond:
+    (void)pthread_cond_destroy(&jnl_proc->cond);
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&jnl_proc->lock);
+free_jnl_proc:
+    GF_FREE(jnl_proc);
+error_return:
+    return -1;
 }
 
 static void
-gf_changelog_cleanup_fds (gf_changelog_journal_t *jnl)
+gf_changelog_cleanup_fds(gf_changelog_journal_t *jnl)
 {
-        /* tracker fd */
-        if (jnl->jnl_fd != -1)
-                sys_close (jnl->jnl_fd);
-        /* processing dir */
-        if (jnl->jnl_dir)
-                sys_closedir (jnl->jnl_dir);
-
-        if (jnl->jnl_working_dir)
-                free (jnl->jnl_working_dir); /* allocated by realpath */
+    /* tracker fd */
+    if (jnl->jnl_fd != -1)
+        sys_close(jnl->jnl_fd);
+    /* processing dir */
+    if (jnl->jnl_dir)
+        sys_closedir(jnl->jnl_dir);
+
+    if (jnl->jnl_working_dir)
+        free(jnl->jnl_working_dir); /* allocated by realpath */
 }
 
 static int
-gf_changelog_open_dirs (xlator_t *this, gf_changelog_journal_t *jnl)
+gf_changelog_open_dirs(xlator_t *this, gf_changelog_journal_t *jnl)
 {
-        int  ret                    = -1;
-        DIR *dir                    = NULL;
-        int  tracker_fd             = 0;
-        char tracker_path[PATH_MAX] = {0,};
-
-        /* .current */
-        (void) snprintf (jnl->jnl_current_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_CURRENT_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = recursive_rmdir (jnl->jnl_current_dir);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
-                        "Failed to rmdir: %s",
-                        jnl->jnl_current_dir);
-                goto out;
-        }
-        ret = mkdir_p (jnl->jnl_current_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
+    int ret = -1;
+    DIR *dir = NULL;
+    int tracker_fd = 0;
+    char tracker_path[PATH_MAX] = {
+        0,
+    };
+
+    /* .current */
+    (void)snprintf(jnl->jnl_current_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_CURRENT_DIR "/", jnl->jnl_working_dir);
+    ret = recursive_rmdir(jnl->jnl_current_dir);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_FAILED_TO_RMDIR, "path=%s",
+                jnl->jnl_current_dir, NULL);
+        goto out;
+    }
+    ret = mkdir_p(jnl->jnl_current_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
+
+    /* .processed */
+    (void)snprintf(jnl->jnl_processed_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_PROCESSED_DIR "/", jnl->jnl_working_dir);
+    ret = mkdir_p(jnl->jnl_processed_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
+
+    /* .processing */
+    (void)snprintf(jnl->jnl_processing_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_PROCESSING_DIR "/", jnl->jnl_working_dir);
+    ret = recursive_rmdir(jnl->jnl_processing_dir);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_FAILED_TO_RMDIR, "path=%s",
+                jnl->jnl_processing_dir, NULL);
+        goto out;
+    }
 
-        /* .processed */
-        (void) snprintf (jnl->jnl_processed_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_PROCESSED_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = mkdir_p (jnl->jnl_processed_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
-
-        /* .processing */
-        (void) snprintf (jnl->jnl_processing_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_PROCESSING_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = recursive_rmdir (jnl->jnl_processing_dir);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
-                        "Failed to rmdir: %s",
-                        jnl->jnl_processing_dir);
-                goto out;
-        }
+    ret = mkdir_p(jnl->jnl_processing_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
 
-        ret = mkdir_p (jnl->jnl_processing_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
-
-        dir = sys_opendir (jnl->jnl_processing_dir);
-        if (!dir) {
-                gf_msg ("", GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_OPENDIR_ERROR,
-                        "opendir() error");
-                goto out;
-        }
+    dir = sys_opendir(jnl->jnl_processing_dir);
+    if (!dir) {
+        gf_msg("", GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "opendir() error");
+        goto out;
+    }
 
-        jnl->jnl_dir = dir;
+    jnl->jnl_dir = dir;
 
-        (void) snprintf (tracker_path, PATH_MAX,
-                         "%s/"GF_CHANGELOG_TRACKER, jnl->jnl_working_dir);
+    (void)snprintf(tracker_path, PATH_MAX, "%s/" GF_CHANGELOG_TRACKER,
+                   jnl->jnl_working_dir);
 
-        tracker_fd = open (tracker_path, O_CREAT | O_APPEND | O_RDWR,
-                           S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (tracker_fd < 0) {
-                sys_closedir (jnl->jnl_dir);
-                ret = -1;
-                goto out;
-        }
+    tracker_fd = open(tracker_path, O_CREAT | O_APPEND | O_RDWR,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (tracker_fd < 0) {
+        sys_closedir(jnl->jnl_dir);
+        ret = -1;
+        goto out;
+    }
 
-        jnl->jnl_fd = tracker_fd;
-        ret = 0;
- out:
-        return ret;
+    jnl->jnl_fd = tracker_fd;
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-gf_changelog_init_history (xlator_t *this,
-                           gf_changelog_journal_t *jnl,
-                           char *brick_path)
+gf_changelog_init_history(xlator_t *this, gf_changelog_journal_t *jnl,
+                          char *brick_path)
 {
-        int i   = 0;
-        int ret = 0;
-        char hist_scratch_dir[PATH_MAX] = {0,};
+    int i = 0;
+    int ret = 0;
+    char hist_scratch_dir[PATH_MAX] = {
+        0,
+    };
 
-        jnl->hist_jnl = GF_CALLOC (1, sizeof (*jnl),
-                         gf_changelog_mt_libgfchangelog_t);
-        if (!jnl->hist_jnl)
-                goto error_return;
+    jnl->hist_jnl = GF_CALLOC(1, sizeof(*jnl),
+                              gf_changelog_mt_libgfchangelog_t);
+    if (!jnl->hist_jnl)
+        goto error_return;
 
-        jnl->hist_jnl->jnl_dir = NULL;
-        jnl->hist_jnl->jnl_fd =  -1;
+    jnl->hist_jnl->jnl_dir = NULL;
+    jnl->hist_jnl->jnl_fd = -1;
 
-        (void) snprintf (hist_scratch_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_HISTORY_DIR"/",
-                         jnl->jnl_working_dir);
+    (void)snprintf(hist_scratch_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_HISTORY_DIR "/", jnl->jnl_working_dir);
 
-        ret = mkdir_p (hist_scratch_dir, 0600, _gf_false);
-        if (ret)
-                goto dealloc_hist;
-
-        jnl->hist_jnl->jnl_working_dir = realpath (hist_scratch_dir, NULL);
-        if (!jnl->hist_jnl->jnl_working_dir)
-                goto dealloc_hist;
-
-        ret = gf_changelog_open_dirs (this, jnl->hist_jnl);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_OPENDIR_ERROR,
-                        "could not create entries in history scratch dir");
-                goto dealloc_hist;
-        }
+    ret = mkdir_p(hist_scratch_dir, 0600, _gf_false);
+    if (ret)
+        goto dealloc_hist;
 
-        (void) strncpy (jnl->hist_jnl->jnl_brickpath, brick_path, PATH_MAX-1);
-        jnl->hist_jnl->jnl_brickpath[PATH_MAX-1] = 0;
+    jnl->hist_jnl->jnl_working_dir = realpath(hist_scratch_dir, NULL);
+    if (!jnl->hist_jnl->jnl_working_dir)
+        goto dealloc_hist;
 
-        for (i = 0; i < 256; i++) {
-                jnl->hist_jnl->rfc3986[i] =
-                        (isalnum(i) || i == '~' ||
-                        i == '-' || i == '.' || i == '_') ? i : 0;
-        }
+    ret = gf_changelog_open_dirs(this, jnl->hist_jnl);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "could not create entries in history scratch dir");
+        goto dealloc_hist;
+    }
 
-        return 0;
+    if (snprintf(jnl->hist_jnl->jnl_brickpath, PATH_MAX, "%s", brick_path) >=
+        PATH_MAX)
+        goto dealloc_hist;
 
- dealloc_hist:
-        GF_FREE (jnl->hist_jnl);
-        jnl->hist_jnl = NULL;
- error_return:
-        return -1;
+    for (i = 0; i < 256; i++) {
+        jnl->hist_jnl->rfc3986_space_newline[i] = (i == ' ' || i == '\n' ||
+                                                   i == '%')
+                                                      ? 0
+                                                      : i;
+    }
+
+    return 0;
+
+dealloc_hist:
+    GF_FREE(jnl->hist_jnl);
+    jnl->hist_jnl = NULL;
+error_return:
+    return -1;
 }
 
 void
-gf_changelog_journal_fini (void *xl, char *brick, void *data)
+gf_changelog_journal_fini(void *xl, char *brick, void *data)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        this = xl;
-        jnl = data;
+    jnl = data;
 
-        gf_changelog_cleanup_processor (jnl);
+    gf_changelog_cleanup_processor(jnl);
 
-        gf_changelog_cleanup_fds (jnl);
-        if (jnl->hist_jnl)
-                gf_changelog_cleanup_fds (jnl->hist_jnl);
+    gf_changelog_cleanup_fds(jnl);
+    if (jnl->hist_jnl)
+        gf_changelog_cleanup_fds(jnl->hist_jnl);
 
-        GF_FREE (jnl);
+    GF_FREE(jnl);
 }
 
 void *
-gf_changelog_journal_init (void *xl, struct gf_brick_spec *brick)
+gf_changelog_journal_init(void *xl, struct gf_brick_spec *brick)
 {
-        int                     i           = 0;
-        int                     ret         = 0;
-        xlator_t               *this        = NULL;
-        struct stat             buf         = {0,};
-        char                   *scratch_dir = NULL;
-        gf_changelog_journal_t *jnl         = NULL;
-
-        this = xl;
-        scratch_dir = (char *) brick->ptr;
-
-        jnl = GF_CALLOC (1, sizeof (gf_changelog_journal_t),
-                         gf_changelog_mt_libgfchangelog_t);
-        if (!jnl)
-                goto error_return;
-
-        if (sys_stat (scratch_dir, &buf) && errno == ENOENT) {
-                ret = mkdir_p (scratch_dir, 0600, _gf_true);
-                if (ret)
-                        goto dealloc_private;
-        }
-
-        jnl->jnl_working_dir = realpath (scratch_dir, NULL);
-        if (!jnl->jnl_working_dir)
-                goto dealloc_private;
-
-        ret = gf_changelog_open_dirs (this, jnl);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_OPENDIR_ERROR,
-                        "could not create entries in scratch dir");
-                goto dealloc_private;
-        }
+    int i = 0;
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct stat buf = {
+        0,
+    };
+    char *scratch_dir = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+
+    this = xl;
+    scratch_dir = (char *)brick->ptr;
+
+    jnl = GF_CALLOC(1, sizeof(gf_changelog_journal_t),
+                    gf_changelog_mt_libgfchangelog_t);
+    if (!jnl)
+        goto error_return;
+
+    if (snprintf(jnl->jnl_brickpath, PATH_MAX, "%s", brick->brick_path) >=
+        PATH_MAX)
+        goto dealloc_private;
+
+    if (sys_stat(scratch_dir, &buf) && errno == ENOENT) {
+        ret = mkdir_p(scratch_dir, 0600, _gf_true);
+        if (ret)
+            goto dealloc_private;
+    }
 
-        (void) strncpy (jnl->jnl_brickpath, brick->brick_path, PATH_MAX-1);
-        jnl->jnl_brickpath[PATH_MAX-1] = 0;
+    jnl->jnl_working_dir = realpath(scratch_dir, NULL);
+    if (!jnl->jnl_working_dir)
+        goto dealloc_private;
 
-        /* RFC 3986 {de,en}coding */
-        for (i = 0; i < 256; i++) {
-                jnl->rfc3986[i] =
-                        (isalnum(i) || i == '~' ||
-                        i == '-' || i == '.' || i == '_') ? i : 0;
-        }
+    ret = gf_changelog_open_dirs(this, jnl);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "could not create entries in scratch dir");
+        goto dealloc_private;
+    }
 
-        ret = gf_changelog_init_history (this, jnl, brick->brick_path);
-        if (ret)
-                goto cleanup_fds;
+    /* RFC 3986 {de,en}coding */
+    for (i = 0; i < 256; i++) {
+        jnl->rfc3986_space_newline[i] = (i == ' ' || i == '\n' || i == '%') ? 0
+                                                                            : i;
+    }
 
-        /* initialize journal processor */
-        jnl->this = this;
-        ret = gf_changelog_init_processor (jnl);
-        if (ret)
-                goto cleanup_fds;
-
-        JNL_SET_API_STATE (jnl, JNL_API_CONN_INPROGESS);
-        ret = pthread_spin_init (&jnl->lock, 0);
-        if (ret != 0)
-                goto cleanup_processor;
-        return jnl;
-
- cleanup_processor:
-        gf_changelog_cleanup_processor (jnl);
- cleanup_fds:
-        gf_changelog_cleanup_fds (jnl);
-        if (jnl->hist_jnl)
-                gf_changelog_cleanup_fds (jnl->hist_jnl);
- dealloc_private:
-        GF_FREE (jnl);
- error_return:
-        return NULL;
+    ret = gf_changelog_init_history(this, jnl, brick->brick_path);
+    if (ret)
+        goto cleanup_fds;
+
+    /* initialize journal processor */
+    jnl->this = this;
+    ret = gf_changelog_init_processor(jnl);
+    if (ret)
+        goto cleanup_fds;
+
+    JNL_SET_API_STATE(jnl, JNL_API_CONN_INPROGESS);
+    ret = pthread_spin_init(&jnl->lock, 0);
+    if (ret != 0)
+        goto cleanup_processor;
+    return jnl;
+
+cleanup_processor:
+    gf_changelog_cleanup_processor(jnl);
+cleanup_fds:
+    gf_changelog_cleanup_fds(jnl);
+    if (jnl->hist_jnl)
+        gf_changelog_cleanup_fds(jnl->hist_jnl);
+dealloc_private:
+    GF_FREE(jnl);
+error_return:
+    return NULL;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-journal.h b/xlators/features/changelog/lib/src/gf-changelog-journal.h
index e91807c80b6..ba5b9bf827e 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-journal.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-journal.h
@@ -17,91 +17,91 @@
 #include "changelog.h"
 
 enum api_conn {
-        JNL_API_CONNECTED,
-        JNL_API_CONN_INPROGESS,
-        JNL_API_DISCONNECTED,
+    JNL_API_CONNECTED,
+    JNL_API_CONN_INPROGESS,
+    JNL_API_DISCONNECTED,
 };
 
 typedef struct gf_changelog_entry {
-        char path[PATH_MAX];
+    char path[PATH_MAX];
 
-        struct list_head list;
+    struct list_head list;
 } gf_changelog_entry_t;
 
 typedef struct gf_changelog_processor {
-        pthread_mutex_t  lock;     /* protects ->entries */
-        pthread_cond_t   cond;     /* waiter during empty list */
-        gf_boolean_t     waiting;
+    pthread_mutex_t lock; /* protects ->entries */
+    pthread_cond_t cond;  /* waiter during empty list */
+    gf_boolean_t waiting;
 
-        pthread_t processor;       /* thread-id of journal processing thread */
+    pthread_t processor; /* thread-id of journal processing thread */
 
-        struct list_head entries;
+    struct list_head entries;
 } gf_changelog_processor_t;
 
 typedef struct gf_changelog_journal {
-        DIR *jnl_dir;                       /* 'processing' directory stream */
+    DIR *jnl_dir; /* 'processing' directory stream */
 
-        int jnl_fd;                         /* fd to the tracker file */
+    int jnl_fd; /* fd to the tracker file */
 
-        char jnl_brickpath[PATH_MAX];       /* brick path for this end-point */
+    char jnl_brickpath[PATH_MAX]; /* brick path for this end-point */
 
-        gf_changelog_processor_t *jnl_proc;
+    gf_changelog_processor_t *jnl_proc;
 
-        char *jnl_working_dir;              /* scratch directory */
+    char *jnl_working_dir; /* scratch directory */
 
-        char jnl_current_dir[PATH_MAX];
-        char jnl_processed_dir[PATH_MAX];
-        char jnl_processing_dir[PATH_MAX];
+    char jnl_current_dir[PATH_MAX];
+    char jnl_processed_dir[PATH_MAX];
+    char jnl_processing_dir[PATH_MAX];
 
-        char rfc3986[256];                  /* RFC 3986 string encoding */
+    char rfc3986_space_newline[256]; /* RFC 3986 string encoding */
 
-        struct gf_changelog_journal *hist_jnl;
-        int hist_done;                      /* holds 0 done scanning,
-                                               1 keep scanning and -1 error */
+    struct gf_changelog_journal *hist_jnl;
+    int hist_done; /* holds 0 done scanning,
+                      1 keep scanning and -1 error */
 
-        pthread_spinlock_t lock;
-        int connected;
-        xlator_t *this;
+    pthread_spinlock_t lock;
+    int connected;
+    xlator_t *this;
 } gf_changelog_journal_t;
 
-#define JNL_SET_API_STATE(jnl, state)  (jnl->connected = state)
-#define JNL_IS_API_DISCONNECTED(jnl)  (jnl->connected == JNL_API_DISCONNECTED)
+#define JNL_SET_API_STATE(jnl, state) (jnl->connected = state)
+#define JNL_IS_API_DISCONNECTED(jnl) (jnl->connected == JNL_API_DISCONNECTED)
 
 /* History API */
 typedef struct gf_changelog_history_data {
-        int           len;
+    int len;
 
-        int           htime_fd;
+    int htime_fd;
 
-        /* parallelism count */
-        int           n_parallel;
+    /* parallelism count */
+    int n_parallel;
 
-        /* history from, to indexes */
-        unsigned long from;
-        unsigned long to;
-        xlator_t      *this;
+    /* history from, to indexes */
+    unsigned long from;
+    unsigned long to;
+    xlator_t *this;
 } gf_changelog_history_data_t;
 
 typedef struct gf_changelog_consume_data {
-        /** set of inputs */
+    /** set of inputs */
 
-        /* fd to read from */
-        int             fd;
+    /* fd to read from */
+    int fd;
 
-        /* from @offset */
-        off_t           offset;
+    /* from @offset */
+    off_t offset;
 
-        xlator_t       *this;
+    xlator_t *this;
 
-        gf_changelog_journal_t *jnl;
+    gf_changelog_journal_t *jnl;
 
-        /** set of outputs */
+    /** set of outputs */
 
-        /* return value */
-        int retval;
+    /* return value */
+    int retval;
 
-        /* journal processed */
-        char changelog[PATH_MAX];
+    /* journal processed */
+    char changelog[PATH_MAX];
 } gf_changelog_consume_data_t;
 
 /* event handler */
diff --git a/xlators/features/changelog/lib/src/gf-changelog-reborp.c b/xlators/features/changelog/lib/src/gf-changelog-reborp.c
index 28ebcfd0146..56b11cbb705 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-reborp.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-reborp.c
@@ -15,139 +15,130 @@
 #include "changelog-rpc-common.h"
 #include "changelog-lib-messages.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
 /**
  * Reverse socket: actual data transfer handler. Connection
  * initiator is PROBER, data transfer is REBORP.
  */
 
-struct rpcsvc_program *gf_changelog_reborp_programs[];
+static struct rpcsvc_program *gf_changelog_reborp_programs[];
 
 void *
-gf_changelog_connection_janitor (void *arg)
+gf_changelog_connection_janitor(void *arg)
 {
-        int32_t ret = 0;
-        xlator_t *this = NULL;
-        gf_private_t *priv = NULL;
-        gf_changelog_t *entry = NULL;
-        struct gf_event *event = NULL;
-        struct gf_event_list *ev = NULL;
-        unsigned long drained = 0;
-
-        this = arg;
-        THIS = this;
-
-        priv = this->private;
-
-        while (1) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->cleanups))
-                                pthread_cond_wait (&priv->cond, &priv->lock);
-
-                        entry = list_first_entry (&priv->cleanups,
-                                                  gf_changelog_t, list);
-                        list_del_init (&entry->list);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                drained = 0;
-                ev = &entry->event;
-
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO,
-                        "Cleaning brick entry for brick %s", entry->brick);
-
-                /* 0x0: disbale rpc-clnt */
-                rpc_clnt_disable (RPC_PROBER (entry));
-
-                /* 0x1: cleanup callback invoker thread */
-                ret = gf_cleanup_event (this, ev);
-                if (ret)
-                        continue;
-
-                /* 0x2: drain pending events */
-                while (!list_empty (&ev->events)) {
-                        event = list_first_entry (&ev->events,
-                                                  struct gf_event, list);
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
-                                "Draining event [Seq: %lu, Payload: %d]",
-                                event->seq, event->count);
-
-                        GF_FREE (event);
-                        drained++;
-                }
-
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
-                        "Drained %lu events", drained);
-
-                /* 0x3: freeup brick entry */
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO,
-                        "freeing entry %p", entry);
-                LOCK_DESTROY (&entry->statelock);
-                GF_FREE (entry);
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    gf_private_t *priv = NULL;
+    gf_changelog_t *entry = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
+    unsigned long drained = 0;
+
+    this = arg;
+    THIS = this;
+
+    priv = this->private;
+
+    while (1) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->cleanups))
+                pthread_cond_wait(&priv->cond, &priv->lock);
+
+            entry = list_first_entry(&priv->cleanups, gf_changelog_t, list);
+            list_del_init(&entry->list);
+        }
+        pthread_mutex_unlock(&priv->lock);
+
+        drained = 0;
+        ev = &entry->event;
+
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO, "brick=%s",
+                entry->brick, NULL);
+
+        /* 0x0: disable rpc-clnt */
+        rpc_clnt_disable(RPC_PROBER(entry));
+
+        /* 0x1: cleanup callback invoker thread */
+        ret = gf_cleanup_event(this, ev);
+        if (ret)
+            continue;
+
+        /* 0x2: drain pending events */
+        while (!list_empty(&ev->events)) {
+            event = list_first_entry(&ev->events, struct gf_event, list);
+            gf_smsg(this->name, GF_LOG_INFO, 0,
+                    CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO, "seq=%lu",
+                    event->seq, "payload=%d", event->count, NULL);
+
+            GF_FREE(event);
+            drained++;
         }
 
-        return NULL;
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO, "num=%lu", drained, NULL);
+
+        /* 0x3: freeup brick entry */
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO, "entry=%p", entry, NULL);
+        LOCK_DESTROY(&entry->statelock);
+        GF_FREE(entry);
+    }
+
+    return NULL;
 }
 
 int
-gf_changelog_reborp_rpcsvc_notify (rpcsvc_t *rpc, void *mydata,
-                                   rpcsvc_event_t event, void *data)
+gf_changelog_reborp_rpcsvc_notify(rpcsvc_t *rpc, void *mydata,
+                                  rpcsvc_event_t event, void *data)
 {
-        int             ret      = 0;
-        xlator_t       *this     = NULL;
-        gf_private_t   *priv     = NULL;
-        gf_changelog_t *entry    = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_changelog_t *entry = NULL;
 
-        if (!(event == RPCSVC_EVENT_ACCEPT ||
-              event == RPCSVC_EVENT_DISCONNECT))
-                return 0;
+    if (!(event == RPCSVC_EVENT_ACCEPT || event == RPCSVC_EVENT_DISCONNECT))
+        return 0;
 
-        entry = mydata;
-        this = entry->this;
-        priv = this->private;
+    entry = mydata;
+    this = entry->this;
 
-        switch (event) {
+    switch (event) {
         case RPCSVC_EVENT_ACCEPT:
-                ret = sys_unlink (RPC_SOCK(entry));
-                if (ret != 0)
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                CHANGELOG_LIB_MSG_UNLINK_FAILED,
-                                "failed to unlink "
-                                "reverse socket %s", RPC_SOCK (entry));
-                if (entry->connected)
-                        GF_CHANGELOG_INVOKE_CBK (this, entry->connected,
-                                                 entry->brick, entry->ptr);
-                break;
+            ret = sys_unlink(RPC_SOCK(entry));
+            if (ret != 0)
+                gf_smsg(this->name, GF_LOG_WARNING, errno,
+                        CHANGELOG_LIB_MSG_UNLINK_FAILED, "name=reverse socket",
+                        "path=%s", RPC_SOCK(entry), NULL);
+            if (entry->connected)
+                GF_CHANGELOG_INVOKE_CBK(this, entry->connected, entry->brick,
+                                        entry->ptr);
+            break;
         case RPCSVC_EVENT_DISCONNECT:
-                if (entry->disconnected)
-                        GF_CHANGELOG_INVOKE_CBK (this, entry->disconnected,
-                                                 entry->brick, entry->ptr);
-                /* passthrough */
+            if (entry->disconnected)
+                GF_CHANGELOG_INVOKE_CBK(this, entry->disconnected, entry->brick,
+                                        entry->ptr);
+            /* passthrough */
         default:
-                break;
-        }
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 rpcsvc_t *
-gf_changelog_reborp_init_rpc_listner (xlator_t *this,
-                                      char *path, char *sock, void *cbkdata)
+gf_changelog_reborp_init_rpc_listner(xlator_t *this, char *path, char *sock,
+                                     void *cbkdata)
 {
-        CHANGELOG_MAKE_TMP_SOCKET_PATH (path, sock, UNIX_PATH_MAX);
-        return changelog_rpc_server_init (this, sock, cbkdata,
-                                          gf_changelog_reborp_rpcsvc_notify,
-                                          gf_changelog_reborp_programs);
+    CHANGELOG_MAKE_TMP_SOCKET_PATH(path, sock, UNIX_PATH_MAX);
+    return changelog_rpc_server_init(this, sock, cbkdata,
+                                     gf_changelog_reborp_rpcsvc_notify,
+                                     gf_changelog_reborp_programs);
 }
 
 /**
- * This is dirty and painful as of now untill there is event filtering in the
+ * This is dirty and painful as of now until there is event filtering in the
  * server. The entire event buffer is scanned and interested events are picked,
  * whereas we _should_ be notified with the events we were interested in
  * (selected at the time of probe). As of now this is complete BS and needs
@@ -156,29 +147,27 @@ gf_changelog_reborp_init_rpc_listner (xlator_t *this,
  * @FIXME: cleanup this bugger once server filters events.
  */
 void
-gf_changelog_invoke_callback (gf_changelog_t *entry,
-                              struct iovec **vec, int payloadcnt)
+gf_changelog_invoke_callback(gf_changelog_t *entry, struct iovec **vec,
+                             int payloadcnt)
 {
-        int i = 0;
-        int evsize = 0;
-        xlator_t *this = NULL;
-        changelog_event_t *event = NULL;
-
-        this = entry->this;
-
-        for (; i < payloadcnt; i++) {
-                event = (changelog_event_t *)vec[i]->iov_base;
-                evsize = vec[i]->iov_len / CHANGELOG_EV_SIZE;
-
-                for (; evsize > 0; evsize--, event++) {
-                        if (gf_changelog_filter_check (entry, event)) {
-                                GF_CHANGELOG_INVOKE_CBK (this,
-                                                         entry->callback,
-                                                         entry->brick,
-                                                         entry->ptr, event);
-                        }
-                }
+    int i = 0;
+    int evsize = 0;
+    xlator_t *this = NULL;
+    changelog_event_t *event = NULL;
+
+    this = entry->this;
+
+    for (; i < payloadcnt; i++) {
+        event = (changelog_event_t *)vec[i]->iov_base;
+        evsize = vec[i]->iov_len / CHANGELOG_EV_SIZE;
+
+        for (; evsize > 0; evsize--, event++) {
+            if (gf_changelog_filter_check(entry, event)) {
+                GF_CHANGELOG_INVOKE_CBK(this, entry->callback, entry->brick,
+                                        entry->ptr, event);
+            }
         }
+    }
 }
 
 /**
@@ -189,219 +178,218 @@ gf_changelog_invoke_callback (gf_changelog_t *entry,
  */
 
 int
-__is_expected_sequence (struct gf_event_list *ev, struct gf_event *event)
+__is_expected_sequence(struct gf_event_list *ev, struct gf_event *event)
 {
-        return (ev->next_seq == event->seq);
+    return (ev->next_seq == event->seq);
 }
 
 int
-__can_process_event (struct gf_event_list *ev, struct gf_event **event)
+__can_process_event(struct gf_event_list *ev, struct gf_event **event)
 {
-        *event = list_first_entry (&ev->events, struct gf_event, list);
+    *event = list_first_entry(&ev->events, struct gf_event, list);
 
-        if (__is_expected_sequence (ev, *event)) {
-                list_del (&(*event)->list);
-                ev->next_seq++;
-                return 1;
-        }
+    if (__is_expected_sequence(ev, *event)) {
+        list_del(&(*event)->list);
+        ev->next_seq++;
+        return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
 void
-pick_event_ordered (struct gf_event_list *ev, struct gf_event **event)
+pick_event_ordered(struct gf_event_list *ev, struct gf_event **event)
 {
-        pthread_mutex_lock (&ev->lock);
-        {
-                while (list_empty (&ev->events)
-                       || !__can_process_event (ev, event))
-                        pthread_cond_wait (&ev->cond, &ev->lock);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    pthread_mutex_lock(&ev->lock);
+    {
+        while (list_empty(&ev->events) || !__can_process_event(ev, event))
+            pthread_cond_wait(&ev->cond, &ev->lock);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void
-pick_event_unordered (struct gf_event_list *ev, struct gf_event **event)
+pick_event_unordered(struct gf_event_list *ev, struct gf_event **event)
 {
-        pthread_mutex_lock (&ev->lock);
-        {
-                while (list_empty (&ev->events))
-                        pthread_cond_wait (&ev->cond, &ev->lock);
-                *event = list_first_entry (&ev->events, struct gf_event, list);
-                list_del (&(*event)->list);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    pthread_mutex_lock(&ev->lock);
+    {
+        while (list_empty(&ev->events))
+            pthread_cond_wait(&ev->cond, &ev->lock);
+        *event = list_first_entry(&ev->events, struct gf_event, list);
+        list_del(&(*event)->list);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void *
-gf_changelog_callback_invoker (void *arg)
+gf_changelog_callback_invoker(void *arg)
 {
-        int                   ret    = 0;
-        xlator_t             *this   = NULL;
-        gf_changelog_t       *entry  = NULL;
-        struct iovec         *vec    = NULL;
-        struct gf_event      *event  = NULL;
-        struct gf_event_list *ev     = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_t *entry = NULL;
+    struct iovec *vec = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
 
-        ev    = arg;
-        entry = ev->entry;
-        THIS = this = entry->this;
+    ev = arg;
+    entry = ev->entry;
+    THIS = this = entry->this;
 
-        while (1) {
-                entry->pickevent (ev, &event);
+    while (1) {
+        entry->pickevent(ev, &event);
 
-                vec = (struct iovec *) &event->iov;
-                gf_changelog_invoke_callback (entry, &vec, event->count);
+        vec = (struct iovec *)&event->iov;
+        gf_changelog_invoke_callback(entry, &vec, event->count);
 
-                GF_FREE (event);
-        }
+        GF_FREE(event);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 static int
-orderfn (struct list_head *pos1, struct list_head *pos2)
+orderfn(struct list_head *pos1, struct list_head *pos2)
 {
-        struct gf_event *event1 = NULL;
-        struct gf_event *event2 = NULL;
+    struct gf_event *event1 = NULL;
+    struct gf_event *event2 = NULL;
 
-        event1 = list_entry (pos1, struct gf_event, list);
-        event2 = list_entry (pos2, struct gf_event, list);
+    event1 = list_entry(pos1, struct gf_event, list);
+    event2 = list_entry(pos2, struct gf_event, list);
 
-        if  (event1->seq > event2->seq)
-                return 1;
-        return -1;
+    if (event1->seq > event2->seq)
+        return 1;
+    return -1;
 }
 
 void
-queue_ordered_event (struct gf_event_list *ev, struct gf_event *event)
+queue_ordered_event(struct gf_event_list *ev, struct gf_event *event)
 {
-        /* add event to the ordered event list and wake up listner(s) */
-        pthread_mutex_lock (&ev->lock);
-        {
-                list_add_order (&event->list, &ev->events, orderfn);
-                if (!ev->next_seq)
-                        ev->next_seq = event->seq;
-                if (ev->next_seq == event->seq)
-                        pthread_cond_signal (&ev->cond);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    /* add event to the ordered event list and wake up listener(s) */
+    pthread_mutex_lock(&ev->lock);
+    {
+        list_add_order(&event->list, &ev->events, orderfn);
+        if (!ev->next_seq)
+            ev->next_seq = event->seq;
+        if (ev->next_seq == event->seq)
+            pthread_cond_signal(&ev->cond);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void
-queue_unordered_event (struct gf_event_list *ev, struct gf_event *event)
+queue_unordered_event(struct gf_event_list *ev, struct gf_event *event)
 {
-        /* add event to the tail of the queue and wake up listener(s) */
-        pthread_mutex_lock (&ev->lock);
-        {
-                list_add_tail (&event->list, &ev->events);
-                pthread_cond_signal (&ev->cond);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    /* add event to the tail of the queue and wake up listener(s) */
+    pthread_mutex_lock(&ev->lock);
+    {
+        list_add_tail(&event->list, &ev->events);
+        pthread_cond_signal(&ev->cond);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 int
-gf_changelog_event_handler (rpcsvc_request_t *req,
-                            xlator_t *this, gf_changelog_t *entry)
+gf_changelog_event_handler(rpcsvc_request_t *req, xlator_t *this,
+                           gf_changelog_t *entry)
 {
-        int                   i          = 0;
-        size_t                payloadlen = 0;
-        ssize_t               len        = 0;
-        int                   payloadcnt = 0;
-        changelog_event_req   rpc_req    = {0,};
-        changelog_event_rsp   rpc_rsp    = {0,};
-        struct iovec         *vec        = NULL;
-        struct gf_event      *event      = NULL;
-        struct gf_event_list *ev         = NULL;
-
-        ev = &entry->event;
-
-        len = xdr_to_generic (req->msg[0],
-                              &rpc_req, (xdrproc_t)xdr_changelog_event_req);
-        if (len < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_XDR_DECODING_FAILED,
-                        "xdr decoding failed");
-                req->rpc_err = GARBAGE_ARGS;
-                goto handle_xdr_error;
-        }
-
-        if (len < req->msg[0].iov_len) {
-                payloadcnt = 1;
-                payloadlen = (req->msg[0].iov_len - len);
-        }
-        for (i = 1; i < req->count; i++) {
-                payloadcnt++;
-                payloadlen += req->msg[i].iov_len;
-        }
-
-        event = GF_CALLOC (1, GF_EVENT_CALLOC_SIZE (payloadcnt, payloadlen),
-                           gf_changelog_mt_libgfchangelog_event_t);
-        if (!event)
-                goto handle_xdr_error;
-        INIT_LIST_HEAD (&event->list);
-
-        payloadlen   = 0;
-        event->seq   = rpc_req.seq;
-        event->count = payloadcnt;
-
-        /* deep copy IO vectors */
-        vec = &event->iov[0];
-        GF_EVENT_ASSIGN_IOVEC (vec, event,
-                               (req->msg[0].iov_len - len), payloadlen);
-        (void) memcpy (vec->iov_base,
-                       req->msg[0].iov_base + len, vec->iov_len);
-
-        for (i = 1; i < req->count; i++) {
-                vec = &event->iov[i];
-                GF_EVENT_ASSIGN_IOVEC (vec, event,
-                                       req->msg[i].iov_len, payloadlen);
-                (void) memcpy (event->iov[i].iov_base,
-                               req->msg[i].iov_base, req->msg[i].iov_len);
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "seq: %lu [%s] (time: %lu.%lu), (vec: %d, len: %ld)",
-                      rpc_req.seq, entry->brick, rpc_req.tv_sec,
-                      rpc_req.tv_usec, payloadcnt, payloadlen);
-
-        /* dispatch event */
-        entry->queueevent (ev, event);
-
-        /* ack sequence number */
-        rpc_rsp.op_ret = 0;
-        rpc_rsp.seq    = rpc_req.seq;
-
-        goto submit_rpc;
-
- handle_xdr_error:
-        rpc_rsp.op_ret = -1;
-        rpc_rsp.seq    = 0;     /* invalid */
- submit_rpc:
-        return changelog_rpc_sumbit_reply (req, &rpc_rsp, NULL, 0, NULL,
-                                           (xdrproc_t)xdr_changelog_event_rsp);
+    int i = 0;
+    size_t payloadlen = 0;
+    ssize_t len = 0;
+    int payloadcnt = 0;
+    changelog_event_req rpc_req = {
+        0,
+    };
+    changelog_event_rsp rpc_rsp = {
+        0,
+    };
+    struct iovec *vec = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
+
+    ev = &entry->event;
+
+    len = xdr_to_generic(req->msg[0], &rpc_req,
+                         (xdrproc_t)xdr_changelog_event_req);
+    if (len < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               CHANGELOG_LIB_MSG_XDR_DECODING_FAILED, "xdr decoding failed");
+        req->rpc_err = GARBAGE_ARGS;
+        goto handle_xdr_error;
+    }
+
+    if (len < req->msg[0].iov_len) {
+        payloadcnt = 1;
+        payloadlen = (req->msg[0].iov_len - len);
+    }
+    for (i = 1; i < req->count; i++) {
+        payloadcnt++;
+        payloadlen += req->msg[i].iov_len;
+    }
+
+    event = GF_CALLOC(1, GF_EVENT_CALLOC_SIZE(payloadcnt, payloadlen),
+                      gf_changelog_mt_libgfchangelog_event_t);
+    if (!event)
+        goto handle_xdr_error;
+    INIT_LIST_HEAD(&event->list);
+
+    payloadlen = 0;
+    event->seq = rpc_req.seq;
+    event->count = payloadcnt;
+
+    /* deep copy IO vectors */
+    vec = &event->iov[0];
+    GF_EVENT_ASSIGN_IOVEC(vec, event, (req->msg[0].iov_len - len), payloadlen);
+    (void)memcpy(vec->iov_base, req->msg[0].iov_base + len, vec->iov_len);
+
+    for (i = 1; i < req->count; i++) {
+        vec = &event->iov[i];
+        GF_EVENT_ASSIGN_IOVEC(vec, event, req->msg[i].iov_len, payloadlen);
+        (void)memcpy(event->iov[i].iov_base, req->msg[i].iov_base,
+                     req->msg[i].iov_len);
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "seq: %" PRIu64 " [%s] (time: %" PRIu64 ".%" PRIu64
+                 "), "
+                 "(vec: %d, len: %zd)",
+                 rpc_req.seq, entry->brick, rpc_req.tv_sec, rpc_req.tv_usec,
+                 payloadcnt, payloadlen);
+
+    /* dispatch event */
+    entry->queueevent(ev, event);
+
+    /* ack sequence number */
+    rpc_rsp.op_ret = 0;
+    rpc_rsp.seq = rpc_req.seq;
+
+    goto submit_rpc;
+
+handle_xdr_error:
+    rpc_rsp.op_ret = -1;
+    rpc_rsp.seq = 0; /* invalid */
+submit_rpc:
+    return changelog_rpc_sumbit_reply(req, &rpc_rsp, NULL, 0, NULL,
+                                      (xdrproc_t)xdr_changelog_event_rsp);
 }
 
 int
-gf_changelog_reborp_handle_event (rpcsvc_request_t *req)
+gf_changelog_reborp_handle_event(rpcsvc_request_t *req)
 {
-        xlator_t       *this  = NULL;
-        rpcsvc_t       *svc   = NULL;
-        gf_changelog_t *entry = NULL;
+    xlator_t *this = NULL;
+    rpcsvc_t *svc = NULL;
+    gf_changelog_t *entry = NULL;
 
-        svc = rpcsvc_request_service (req);
-        entry = svc->mydata;
+    svc = rpcsvc_request_service(req);
+    entry = svc->mydata;
 
-        this = THIS = entry->this;
+    this = THIS = entry->this;
 
-        return gf_changelog_event_handler (req, this, entry);
+    return gf_changelog_event_handler(req, this, entry);
 }
 
-rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
-        [CHANGELOG_REV_PROC_EVENT] = {
-                "CHANGELOG EVENT HANDLER", CHANGELOG_REV_PROC_EVENT,
-                gf_changelog_reborp_handle_event, NULL, 0, DRC_NA
-        },
+static rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
+    [CHANGELOG_REV_PROC_EVENT] = {"CHANGELOG EVENT HANDLER",
+                                  gf_changelog_reborp_handle_event, NULL,
+                                  CHANGELOG_REV_PROC_EVENT, DRC_NA, 0},
 };
 
 /**
@@ -410,16 +398,16 @@ rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
  * and that's required to invoke the callback with the appropriate
  * brick path and it's private data.
  */
-struct rpcsvc_program gf_changelog_reborp_prog = {
-        .progname  = "LIBGFCHANGELOG REBORP",
-        .prognum   = CHANGELOG_REV_RPC_PROCNUM,
-        .progver   = CHANGELOG_REV_RPC_PROCVER,
-        .numactors = CHANGELOG_REV_PROC_MAX,
-        .actors    = gf_changelog_reborp_actors,
-        .synctask  = _gf_false,
+static struct rpcsvc_program gf_changelog_reborp_prog = {
+    .progname = "LIBGFCHANGELOG REBORP",
+    .prognum = CHANGELOG_REV_RPC_PROCNUM,
+    .progver = CHANGELOG_REV_RPC_PROCVER,
+    .numactors = CHANGELOG_REV_PROC_MAX,
+    .actors = gf_changelog_reborp_actors,
+    .synctask = _gf_false,
 };
 
-struct rpcsvc_program *gf_changelog_reborp_programs[] = {
-        &gf_changelog_reborp_prog,
-        NULL,
+static struct rpcsvc_program *gf_changelog_reborp_programs[] = {
+    &gf_changelog_reborp_prog,
+    NULL,
 };
diff --git a/xlators/features/changelog/lib/src/gf-changelog-rpc.c b/xlators/features/changelog/lib/src/gf-changelog-rpc.c
index 270632bc71b..8ec6ffbcebc 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-rpc.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-rpc.c
@@ -16,31 +16,32 @@ struct rpc_clnt_program gf_changelog_clnt;
 
 /* TODO: piggyback reconnect to called (upcall) */
 int
-gf_changelog_rpc_notify (struct rpc_clnt *rpc,
-                         void *mydata, rpc_clnt_event_t event, void *data)
+gf_changelog_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                        rpc_clnt_event_t event, void *data)
 {
-        switch (event) {
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&rpc->conn);
-                break;
+            break;
         case RPC_CLNT_DISCONNECT:
         case RPC_CLNT_MSG:
         case RPC_CLNT_DESTROY:
-                break;
-        }
+        case RPC_CLNT_PING:
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 struct rpc_clnt *
-gf_changelog_rpc_init (xlator_t *this, gf_changelog_t *entry)
+gf_changelog_rpc_init(xlator_t *this, gf_changelog_t *entry)
 {
-        char sockfile[UNIX_PATH_MAX] = {0,};
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
 
-        CHANGELOG_MAKE_SOCKET_PATH (entry->brick,
-                                    sockfile, UNIX_PATH_MAX);
-        return changelog_rpc_client_init (this, entry,
-                                          sockfile, gf_changelog_rpc_notify);
+    CHANGELOG_MAKE_SOCKET_PATH(entry->brick, sockfile, UNIX_PATH_MAX);
+    return changelog_rpc_client_init(this, entry, sockfile,
+                                     gf_changelog_rpc_notify);
 }
 
 /**
@@ -48,52 +49,50 @@ gf_changelog_rpc_init (xlator_t *this, gf_changelog_t *entry)
  */
 
 int
-gf_probe_changelog_cbk (struct rpc_req *req,
-                        struct iovec *iovec, int count, void *myframe)
+gf_probe_changelog_cbk(struct rpc_req *req, struct iovec *iovec, int count,
+                       void *myframe)
 {
-        return 0;
+    return 0;
 }
 
 int
-gf_probe_changelog_filter (call_frame_t *frame, xlator_t *this, void *data)
+gf_probe_changelog_filter(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int ret = 0;
-        char *sock = NULL;
-        gf_changelog_t *entry = NULL;
-        changelog_probe_req req = {0,};
-
-        entry = data;
-        sock = RPC_SOCK (entry);
-
-        (void) memcpy (&req.sock, sock, strlen (sock));
-        req.filter = entry->notify;
-
-        /* invoke RPC */
-        return changelog_rpc_sumbit_req (RPC_PROBER (entry), (void *) &req,
-                                         frame, &gf_changelog_clnt,
-                                         CHANGELOG_RPC_PROBE_FILTER, NULL, 0,
-                                         NULL, this, gf_probe_changelog_cbk,
-                                         (xdrproc_t) xdr_changelog_probe_req);
+    char *sock = NULL;
+    gf_changelog_t *entry = NULL;
+    changelog_probe_req req = {
+        0,
+    };
+
+    entry = data;
+    sock = RPC_SOCK(entry);
+
+    (void)memcpy(&req.sock, sock, strlen(sock));
+    req.filter = entry->notify;
+
+    /* invoke RPC */
+    return changelog_rpc_sumbit_req(
+        RPC_PROBER(entry), (void *)&req, frame, &gf_changelog_clnt,
+        CHANGELOG_RPC_PROBE_FILTER, NULL, 0, NULL, this, gf_probe_changelog_cbk,
+        (xdrproc_t)xdr_changelog_probe_req);
 }
 
 int
-gf_changelog_invoke_rpc (xlator_t *this, gf_changelog_t *entry, int procidx)
+gf_changelog_invoke_rpc(xlator_t *this, gf_changelog_t *entry, int procidx)
 {
-        return changelog_invoke_rpc (this, RPC_PROBER (entry),
-                                     &gf_changelog_clnt, procidx, entry);
+    return changelog_invoke_rpc(this, RPC_PROBER(entry), &gf_changelog_clnt,
+                                procidx, entry);
 }
 
 struct rpc_clnt_procedure gf_changelog_procs[CHANGELOG_RPC_PROC_MAX] = {
-        [CHANGELOG_RPC_PROC_NULL] = {"NULL", NULL},
-        [CHANGELOG_RPC_PROBE_FILTER] = {
-                "PROBE FILTER", gf_probe_changelog_filter
-        },
+    [CHANGELOG_RPC_PROC_NULL] = {"NULL", NULL},
+    [CHANGELOG_RPC_PROBE_FILTER] = {"PROBE FILTER", gf_probe_changelog_filter},
 };
 
 struct rpc_clnt_program gf_changelog_clnt = {
-        .progname  = "LIBGFCHANGELOG",
-        .prognum   = CHANGELOG_RPC_PROGNUM,
-        .progver   = CHANGELOG_RPC_PROGVER,
-        .numproc   = CHANGELOG_RPC_PROC_MAX,
-        .proctable = gf_changelog_procs,
+    .progname = "LIBGFCHANGELOG",
+    .prognum = CHANGELOG_RPC_PROGNUM,
+    .progver = CHANGELOG_RPC_PROGVER,
+    .numproc = CHANGELOG_RPC_PROC_MAX,
+    .proctable = gf_changelog_procs,
 };
diff --git a/xlators/features/changelog/lib/src/gf-changelog-rpc.h b/xlators/features/changelog/lib/src/gf-changelog-rpc.h
index 1c982eef809..5c82d6f1c08 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-rpc.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-rpc.h
@@ -11,16 +11,18 @@
 #ifndef __GF_CHANGELOG_RPC_H
 #define __GF_CHANGELOG_RPC_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 #include "gf-changelog-helpers.h"
 #include "changelog-rpc-common.h"
 
-struct rpc_clnt *gf_changelog_rpc_init (xlator_t *, gf_changelog_t *);
+struct rpc_clnt *
+gf_changelog_rpc_init(xlator_t *, gf_changelog_t *);
 
-int gf_changelog_invoke_rpc (xlator_t *, gf_changelog_t *, int);
+int
+gf_changelog_invoke_rpc(xlator_t *, gf_changelog_t *, int);
 
 rpcsvc_t *
-gf_changelog_reborp_init_rpc_listner (xlator_t *, char *, char *, void *);
+gf_changelog_reborp_init_rpc_listner(xlator_t *, char *, char *, void *);
 
 #endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
index 75891635827..57c3d39ef76 100644
--- a/xlators/features/changelog/lib/src/gf-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -22,11 +22,11 @@
 #endif
 #include <string.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "defaults.h"
-#include "syncop.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
 
 #include "gf-changelog-rpc.h"
 #include "gf-changelog-helpers.h"
@@ -45,283 +45,315 @@
  */
 xlator_t *master = NULL;
 
-static inline
-gf_private_t *gf_changelog_alloc_priv ()
+static inline gf_private_t *
+gf_changelog_alloc_priv()
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
-        if (!priv)
-                goto error_return;
-        INIT_LIST_HEAD (&priv->connections);
-        INIT_LIST_HEAD (&priv->cleanups);
-
-        ret = pthread_mutex_init (&priv->lock, NULL);
-        if (ret != 0)
-                goto free_priv;
-        ret = pthread_cond_init (&priv->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-
-        priv->api = NULL;
-        return priv;
-
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&priv->lock);
- free_priv:
-        GF_FREE (priv);
- error_return:
-        return NULL;
+    int ret = 0;
+    gf_private_t *priv = NULL;
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_changelog_mt_priv_t);
+    if (!priv)
+        goto error_return;
+    INIT_LIST_HEAD(&priv->connections);
+    INIT_LIST_HEAD(&priv->cleanups);
+
+    ret = pthread_mutex_init(&priv->lock, NULL);
+    if (ret != 0)
+        goto free_priv;
+    ret = pthread_cond_init(&priv->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+
+    priv->api = NULL;
+    return priv;
+
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&priv->lock);
+free_priv:
+    GF_FREE(priv);
+error_return:
+    return NULL;
 }
 
-#define GF_CHANGELOG_EVENT_POOL_SIZE   16384
+#define GF_CHANGELOG_EVENT_POOL_SIZE 16384
 #define GF_CHANGELOG_EVENT_THREAD_COUNT 4
 
 static int
-gf_changelog_ctx_defaults_init (glusterfs_ctx_t *ctx)
+gf_changelog_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
-        int            ret         = -1;
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(THIS, gf_changelog_mt_end);
+    if (ret != 0)
+        return -1;
 
-        ret = xlator_mem_acct_init (THIS, gf_changelog_mt_end);
-        if (ret != 0)
-                return -1;
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid)
+        return -1;
 
-        ctx->process_uuid = generate_glusterfs_ctx_id ();
-        if (!ctx->process_uuid)
-                return -1;
+    ctx->page_size = 128 * GF_UNIT_KB;
 
-        ctx->page_size  = 128 * GF_UNIT_KB;
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool)
+        goto free_pool;
 
-        ctx->iobuf_pool = iobuf_pool_new ();
-        if (!ctx->iobuf_pool)
-                return -1;
+    ctx->event_pool = gf_event_pool_new(GF_CHANGELOG_EVENT_POOL_SIZE,
+                                        GF_CHANGELOG_EVENT_THREAD_COUNT);
+    if (!ctx->event_pool)
+        goto free_pool;
 
-        ctx->event_pool = event_pool_new (GF_CHANGELOG_EVENT_POOL_SIZE,
-                                          GF_CHANGELOG_EVENT_THREAD_COUNT);
-        if (!ctx->event_pool)
-                return -1;
+    pool = GF_CALLOC(1, sizeof(call_pool_t),
+                     gf_changelog_mt_libgfchangelog_call_pool_t);
+    if (!pool)
+        goto free_pool;
 
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          gf_changelog_mt_libgfchangelog_call_pool_t);
-        if (!pool)
-                return -1;
+    /* frame_mem_pool size 112 * 64 */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 32);
+    if (!pool->frame_mem_pool)
+        goto free_pool;
 
-        /* frame_mem_pool size 112 * 64 */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 32);
-        if (!pool->frame_mem_pool)
-                return -1;
+    /* stack_mem_pool size 256 * 128 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 16);
 
-        /* stack_mem_pool size 256 * 128 */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 16);
+    if (!pool->stack_mem_pool)
+        goto free_pool;
 
-        if (!pool->stack_mem_pool)
-                return -1;
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 16);
+    if (!ctx->stub_mem_pool)
+        goto free_pool;
 
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 16);
-        if (!ctx->stub_mem_pool)
-                return -1;
+    ctx->dict_pool = mem_pool_new(dict_t, 32);
+    if (!ctx->dict_pool)
+        goto free_pool;
 
-        ctx->dict_pool = mem_pool_new (dict_t, 32);
-        if (!ctx->dict_pool)
-                return -1;
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t, 512);
+    if (!ctx->dict_pair_pool)
+        goto free_pool;
 
-        ctx->dict_pair_pool = mem_pool_new (data_pair_t, 512);
-        if (!ctx->dict_pair_pool)
-                return -1;
+    ctx->dict_data_pool = mem_pool_new(data_t, 512);
+    if (!ctx->dict_data_pool)
+        goto free_pool;
 
-        ctx->dict_data_pool = mem_pool_new (data_t, 512);
-        if (!ctx->dict_data_pool)
-                return -1;
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, 256);
+    if (!ctx->logbuf_pool)
+        goto free_pool;
 
-        ctx->logbuf_pool = mem_pool_new (log_buf_t, 256);
-        if (!ctx->logbuf_pool)
-                return -1;
+    INIT_LIST_HEAD(&pool->all_frames);
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
 
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
+    LOCK_INIT(&ctx->lock);
 
-        LOCK_INIT (&ctx->lock);
+    cmd_args = &ctx->cmd_args;
 
-        cmd_args = &ctx->cmd_args;
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
 
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
 
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
+    return 0;
 
-        return 0;
+free_pool:
+    if (pool) {
+        GF_FREE(pool->frame_mem_pool);
+
+        GF_FREE(pool->stack_mem_pool);
+
+        GF_FREE(pool);
+    }
+
+    GF_FREE(ctx->stub_mem_pool);
+
+    GF_FREE(ctx->dict_pool);
+
+    GF_FREE(ctx->dict_pair_pool);
+
+    GF_FREE(ctx->dict_data_pool);
+
+    GF_FREE(ctx->logbuf_pool);
+
+    GF_FREE(ctx->iobuf_pool);
+
+    GF_FREE(ctx->event_pool);
+
+    return -1;
 }
 
 /* TODO: cleanup ctx defaults */
 void
-gf_changelog_cleanup_this (xlator_t *this)
+gf_changelog_cleanup_this(xlator_t *this)
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        ctx = this->ctx;
-        syncenv_destroy (ctx->env);
-        free (ctx);
+    ctx = this->ctx;
+    syncenv_destroy(ctx->env);
+    free(ctx);
 
-        this->private = NULL;
-        this->ctx = NULL;
+    this->private = NULL;
+    this->ctx = NULL;
+
+    mem_pools_fini();
 }
 
 static int
-gf_changelog_init_context ()
+gf_changelog_init_context()
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                goto error_return;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto error_return;
 
-        if (glusterfs_globals_init (ctx))
-                goto free_ctx;
+    if (glusterfs_globals_init(ctx))
+        goto free_ctx;
 
-        THIS->ctx = ctx;
-        if (gf_changelog_ctx_defaults_init (ctx))
-                goto free_ctx;
+    THIS->ctx = ctx;
+    if (gf_changelog_ctx_defaults_init(ctx))
+        goto free_ctx;
 
-        ctx->env = syncenv_new (0, 0, 0);
-        if (!ctx->env)
-                goto free_ctx;
-        return 0;
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env)
+        goto free_ctx;
+    return 0;
 
- free_ctx:
-        free (ctx);
-        THIS->ctx = NULL;
- error_return:
-        return -1;
+free_ctx:
+    free(ctx);
+    THIS->ctx = NULL;
+error_return:
+    return -1;
 }
 
 static int
-gf_changelog_init_master ()
+gf_changelog_init_master()
 {
-        return gf_changelog_init_context ();
+    int ret = 0;
+
+    ret = gf_changelog_init_context();
+    mem_pools_init();
+
+    return ret;
 }
 
 /* TODO: cleanup clnt/svc on failure */
 int
-gf_changelog_setup_rpc (xlator_t *this,
-                        gf_changelog_t *entry, int proc)
+gf_changelog_setup_rpc(xlator_t *this, gf_changelog_t *entry, int proc)
 {
-        int              ret = 0;
-        rpcsvc_t        *svc = NULL;
-        struct rpc_clnt *rpc = NULL;
-
-        /**
-         * Initialize a connect back socket. A probe() RPC call to the server
-         * triggers a reverse connect.
-         */
-        svc = gf_changelog_reborp_init_rpc_listner (this, entry->brick,
-                                                    RPC_SOCK (entry), entry);
-        if (!svc)
-                goto error_return;
-        RPC_REBORP (entry) = svc;
-
-        /* Initialize an RPC client */
-        rpc = gf_changelog_rpc_init (this, entry);
-        if (!rpc)
-                goto error_return;
-        RPC_PROBER (entry) = rpc;
-
-        /**
-         * @FIXME
-         * till we have connection state machine, let's delay the RPC call
-         * for now..
-         */
-        sleep (2);
-
-        /**
-         * Probe changelog translator for reverse connection. After a successful
-         * call, there's less use of the client and can be disconnected, but
-         * let's leave the connection active for any future RPC calls.
-         */
-        ret = gf_changelog_invoke_rpc (this, entry, proc);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED,
-                        "Could not initiate probe RPC, bailing out!!!");
-                goto error_return;
-        }
-
-        return 0;
-
- error_return:
-        return -1;
+    int ret = 0;
+    rpcsvc_t *svc = NULL;
+    struct rpc_clnt *rpc = NULL;
+
+    /**
+     * Initialize a connect back socket. A probe() RPC call to the server
+     * triggers a reverse connect.
+     */
+    svc = gf_changelog_reborp_init_rpc_listner(this, entry->brick,
+                                               RPC_SOCK(entry), entry);
+    if (!svc)
+        goto error_return;
+    RPC_REBORP(entry) = svc;
+
+    /* Initialize an RPC client */
+    rpc = gf_changelog_rpc_init(this, entry);
+    if (!rpc)
+        goto error_return;
+    RPC_PROBER(entry) = rpc;
+
+    /**
+     * @FIXME
+     * till we have connection state machine, let's delay the RPC call
+     * for now..
+     */
+    sleep(2);
+
+    /**
+     * Probe changelog translator for reverse connection. After a successful
+     * call, there's less use of the client and can be disconnected, but
+     * let's leave the connection active for any future RPC calls.
+     */
+    ret = gf_changelog_invoke_rpc(this, entry, proc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED,
+               "Could not initiate probe RPC, bailing out!!!");
+        goto error_return;
+    }
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
 int
-gf_cleanup_event (xlator_t *this, struct gf_event_list *ev)
+gf_cleanup_event(xlator_t *this, struct gf_event_list *ev)
 {
-        int ret = 0;
-
-        ret = gf_thread_cleanup (this, ev->invoker);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        CHANGELOG_LIB_MSG_CLEANUP_ERROR,
-                        "cannot cleanup callback invoker thread."
-                        " Not freeing resources");
-                return -1;
-        }
+    int ret = 0;
+
+    ret = gf_thread_cleanup(this, ev->invoker);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret,
+               CHANGELOG_LIB_MSG_CLEANUP_ERROR,
+               "cannot cleanup callback invoker thread."
+               " Not freeing resources");
+        return -1;
+    }
 
-        ev->entry = NULL;
+    ev->entry = NULL;
 
-        return 0;
+    return 0;
 }
 
 static int
-gf_init_event (gf_changelog_t *entry)
+gf_init_event(gf_changelog_t *entry)
 {
-        int ret = 0;
-        struct gf_event_list *ev = NULL;
-
-        ev = &entry->event;
-        ev->entry = entry;
-
-        ret = pthread_mutex_init (&ev->lock, NULL);
-        if (ret != 0)
-                goto error_return;
-        ret = pthread_cond_init (&ev->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-        INIT_LIST_HEAD (&ev->events);
-
-        ev->next_seq = 0;  /* bootstrap sequencing */
-
-        if (GF_NEED_ORDERED_EVENTS (entry)) {
-                entry->pickevent  = pick_event_ordered;
-                entry->queueevent = queue_ordered_event;
-        } else {
-                entry->pickevent  = pick_event_unordered;
-                entry->queueevent = queue_unordered_event;
-        }
-
-        ret = gf_thread_create (&ev->invoker, NULL,
-                                gf_changelog_callback_invoker, ev);
-        if (ret != 0) {
-                entry->pickevent = NULL;
-                entry->queueevent = NULL;
-                goto cleanup_cond;
-        }
-
-        return 0;
-
- cleanup_cond:
-        (void) pthread_cond_destroy (&ev->cond);
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&ev->lock);
- error_return:
-        return -1;
+    int ret = 0;
+    struct gf_event_list *ev = NULL;
+
+    ev = &entry->event;
+    ev->entry = entry;
+
+    ret = pthread_mutex_init(&ev->lock, NULL);
+    if (ret != 0)
+        goto error_return;
+    ret = pthread_cond_init(&ev->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+    INIT_LIST_HEAD(&ev->events);
+
+    ev->next_seq = 0; /* bootstrap sequencing */
+
+    if (GF_NEED_ORDERED_EVENTS(entry)) {
+        entry->pickevent = pick_event_ordered;
+        entry->queueevent = queue_ordered_event;
+    } else {
+        entry->pickevent = pick_event_unordered;
+        entry->queueevent = queue_unordered_event;
+    }
+
+    ret = gf_thread_create(&ev->invoker, NULL, gf_changelog_callback_invoker,
+                           ev, "clogcbki");
+    if (ret != 0) {
+        entry->pickevent = NULL;
+        entry->queueevent = NULL;
+        goto cleanup_cond;
+    }
+
+    return 0;
+
+cleanup_cond:
+    (void)pthread_cond_destroy(&ev->cond);
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&ev->lock);
+error_return:
+    return -1;
 }
 
 /**
@@ -331,246 +363,241 @@ gf_init_event (gf_changelog_t *entry)
  *  - destroy rpc{-clnt, svc}
  */
 int
-gf_cleanup_brick_connection (xlator_t *this, gf_changelog_t *entry)
+gf_cleanup_brick_connection(xlator_t *this, gf_changelog_t *entry)
 {
-        return 0;
+    return 0;
 }
 
 int
-gf_cleanup_connections (xlator_t *this)
+gf_cleanup_connections(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
 static int
-gf_setup_brick_connection (xlator_t *this,
-                           struct gf_brick_spec *brick,
-                           gf_boolean_t ordered, void *xl)
+gf_setup_brick_connection(xlator_t *this, struct gf_brick_spec *brick,
+                          gf_boolean_t ordered, void *xl)
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-        gf_changelog_t *entry = NULL;
-
-        priv = this->private;
-
-        if (!brick->callback || !brick->init || !brick->fini)
-                goto error_return;
-
-        entry = GF_CALLOC (1, sizeof (*entry),
-                           gf_changelog_mt_libgfchangelog_t);
-        if (!entry)
-                goto error_return;
-        INIT_LIST_HEAD (&entry->list);
-
-        LOCK_INIT (&entry->statelock);
-        entry->connstate = GF_CHANGELOG_CONN_STATE_PENDING;
-
-        entry->notify = brick->filter;
-        (void) strncpy (entry->brick, brick->brick_path, PATH_MAX-1);
-        entry->brick[PATH_MAX-1] = 0;
-
-        entry->this = this;
-        entry->invokerxl = xl;
-
-        entry->ordered = ordered;
-        ret = gf_init_event (entry);
-        if (ret)
-                goto free_entry;
-
-        entry->fini         = brick->fini;
-        entry->callback     = brick->callback;
-        entry->connected    = brick->connected;
-        entry->disconnected = brick->disconnected;
-
-        entry->ptr = brick->init (this, brick);
-        if (!entry->ptr)
-                goto cleanup_event;
-        priv->api = entry->ptr;  /* pointer to API, if required */
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_tail (&entry->list, &priv->connections);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        ret = gf_changelog_setup_rpc (this, entry, CHANGELOG_RPC_PROBE_FILTER);
-        if (ret)
-                goto cleanup_event;
-        return 0;
-
- cleanup_event:
-        (void) gf_cleanup_event (this, &entry->event);
- free_entry:
-        gf_msg_debug (this->name, 0, "freeing entry %p", entry);
-        list_del (&entry->list); /* FIXME: kludge for now */
-        GF_FREE (entry);
- error_return:
-        return -1;
+    int ret = 0;
+    gf_private_t *priv = NULL;
+    gf_changelog_t *entry = NULL;
+
+    priv = this->private;
+
+    if (!brick->callback || !brick->init || !brick->fini)
+        goto error_return;
+
+    entry = GF_CALLOC(1, sizeof(*entry), gf_changelog_mt_libgfchangelog_t);
+    if (!entry)
+        goto error_return;
+    INIT_LIST_HEAD(&entry->list);
+
+    LOCK_INIT(&entry->statelock);
+    entry->connstate = GF_CHANGELOG_CONN_STATE_PENDING;
+
+    entry->notify = brick->filter;
+    if (snprintf(entry->brick, PATH_MAX, "%s", brick->brick_path) >= PATH_MAX)
+        goto free_entry;
+
+    entry->this = this;
+    entry->invokerxl = xl;
+
+    entry->ordered = ordered;
+    ret = gf_init_event(entry);
+    if (ret)
+        goto free_entry;
+
+    entry->fini = brick->fini;
+    entry->callback = brick->callback;
+    entry->connected = brick->connected;
+    entry->disconnected = brick->disconnected;
+
+    entry->ptr = brick->init(this, brick);
+    if (!entry->ptr)
+        goto cleanup_event;
+    priv->api = entry->ptr; /* pointer to API, if required */
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_tail(&entry->list, &priv->connections);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    ret = gf_changelog_setup_rpc(this, entry, CHANGELOG_RPC_PROBE_FILTER);
+    if (ret)
+        goto cleanup_event;
+    return 0;
+
+cleanup_event:
+    (void)gf_cleanup_event(this, &entry->event);
+free_entry:
+    gf_msg_debug(this->name, 0, "freeing entry %p", entry);
+    list_del(&entry->list); /* FIXME: kludge for now */
+    GF_FREE(entry);
+error_return:
+    return -1;
 }
 
 int
-gf_changelog_register_brick (xlator_t *this,
-                             struct gf_brick_spec *brick,
-                             gf_boolean_t ordered, void *xl)
+gf_changelog_register_brick(xlator_t *this, struct gf_brick_spec *brick,
+                            gf_boolean_t ordered, void *xl)
 {
-        return gf_setup_brick_connection (this, brick, ordered, xl);
+    return gf_setup_brick_connection(this, brick, ordered, xl);
 }
 
 static int
-gf_changelog_setup_logging (xlator_t *this, char *logfile, int loglevel)
+gf_changelog_setup_logging(xlator_t *this, char *logfile, int loglevel)
 {
-        /* passing ident as NULL means to use default ident for syslog */
-        if (gf_log_init (this->ctx, logfile, NULL))
-                return -1;
+    /* passing ident as NULL means to use default ident for syslog */
+    if (gf_log_init(this->ctx, logfile, NULL))
+        return -1;
 
-        gf_log_set_loglevel ((loglevel == -1) ? GF_LOG_INFO :
-                             loglevel);
-        return 0;
+    gf_log_set_loglevel(this->ctx, (loglevel == -1) ? GF_LOG_INFO : loglevel);
+    return 0;
 }
 
 static int
-gf_changelog_set_master (xlator_t *master, void *xl)
+gf_changelog_set_master(xlator_t *master, void *xl)
 {
-        int32_t ret = 0;
-        xlator_t *this = NULL;
-        xlator_t *old_this = NULL;
-        gf_private_t *priv = NULL;
-
-        this = xl;
-        if (!this || !this->ctx) {
-                ret = gf_changelog_init_master ();
-                if (ret)
-                        return -1;
-                this = THIS;
-        }
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    xlator_t *old_this = NULL;
+    gf_private_t *priv = NULL;
+
+    this = xl;
+    if (!this || !this->ctx) {
+        ret = gf_changelog_init_master();
+        if (ret)
+            return -1;
+        this = THIS;
+    }
 
-        master->ctx = this->ctx;
+    master->ctx = this->ctx;
 
-        INIT_LIST_HEAD (&master->volume_options);
-        SAVE_THIS (THIS);
+    INIT_LIST_HEAD(&master->volume_options);
+    SAVE_THIS(THIS);
 
-        ret = xlator_mem_acct_init (THIS, gf_changelog_mt_end);
-        if (ret != 0)
-                goto restore_this;
+    ret = xlator_mem_acct_init(THIS, gf_changelog_mt_end);
+    if (ret != 0)
+        goto restore_this;
 
-        priv = gf_changelog_alloc_priv ();
-        if (!priv) {
-                ret = -1;
-                goto restore_this;
-        }
+    priv = gf_changelog_alloc_priv();
+    if (!priv) {
+        ret = -1;
+        goto restore_this;
+    }
 
-        if (!xl) {
-                /* poller thread */
-                ret = gf_thread_create (&priv->poller,
-                                        NULL, changelog_rpc_poller, THIS);
-                if (ret != 0) {
-                        GF_FREE (priv);
-                        gf_msg (master->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
-                                "failed to spawn poller thread");
-                        goto restore_this;
-                }
+    if (!xl) {
+        /* poller thread */
+        ret = gf_thread_create(&priv->poller, NULL, changelog_rpc_poller, THIS,
+                               "clogpoll");
+        if (ret != 0) {
+            GF_FREE(priv);
+            gf_msg(master->name, GF_LOG_ERROR, 0,
+                   CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                   "failed to spawn poller thread");
+            goto restore_this;
         }
+    }
 
-        master->private = priv;
+    master->private = priv;
 
- restore_this:
-        RESTORE_THIS ();
+restore_this:
+    RESTORE_THIS();
 
-        return ret;
+    return ret;
 }
 
 int
-gf_changelog_init (void *xl)
+gf_changelog_init(void *xl)
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-
-        if (master)
-                return 0;
-
-        master = calloc (1, sizeof (*master));
-        if (!master)
-                goto error_return;
-
-        master->name = strdup ("gfchangelog");
-        if (!master->name)
-                goto dealloc_master;
-
-        ret = gf_changelog_set_master (master, xl);
-        if (ret)
-                goto dealloc_name;
-
-        priv = master->private;
-        ret = gf_thread_create (&priv->connectionjanitor, NULL,
-                                gf_changelog_connection_janitor, master);
-        if (ret != 0) {
-                /* TODO: cleanup priv, mutex (poller thread for !xl) */
-                goto dealloc_name;
-        }
+    int ret = 0;
+    gf_private_t *priv = NULL;
 
+    if (master)
         return 0;
 
- dealloc_name:
-        free (master->name);
- dealloc_master:
-        free (master);
-        master = NULL;
- error_return:
-        return -1;
+    master = calloc(1, sizeof(*master));
+    if (!master)
+        goto error_return;
+
+    master->name = strdup("gfchangelog");
+    if (!master->name)
+        goto dealloc_master;
+
+    ret = gf_changelog_set_master(master, xl);
+    if (ret)
+        goto dealloc_name;
+
+    priv = master->private;
+    ret = gf_thread_create(&priv->connectionjanitor, NULL,
+                           gf_changelog_connection_janitor, master, "clogjan");
+    if (ret != 0) {
+        /* TODO: cleanup priv, mutex (poller thread for !xl) */
+        goto dealloc_name;
+    }
+
+    return 0;
+
+dealloc_name:
+    free(master->name);
+dealloc_master:
+    free(master);
+    master = NULL;
+error_return:
+    return -1;
 }
 
 int
-gf_changelog_register_generic (struct gf_brick_spec *bricks, int count,
-                               int ordered, char *logfile, int lvl, void *xl)
+gf_changelog_register_generic(struct gf_brick_spec *bricks, int count,
+                              int ordered, char *logfile, int lvl, void *xl)
 {
-        int                   ret        = 0;
-        xlator_t             *this       = NULL;
-        xlator_t             *old_this   = NULL;
-        struct gf_brick_spec *brick      = NULL;
-        gf_boolean_t          need_order = _gf_false;
+    int ret = 0;
+    xlator_t *this = NULL;
+    xlator_t *old_this = NULL;
+    struct gf_brick_spec *brick = NULL;
+    gf_boolean_t need_order = _gf_false;
 
-        SAVE_THIS (xl);
+    SAVE_THIS(xl);
 
-        this = THIS;
-        if (!this)
-                goto error_return;
+    this = THIS;
+    if (!this)
+        goto error_return;
 
-        ret = gf_changelog_setup_logging (this, logfile, lvl);
-        if (ret)
-                goto error_return;
-
-        need_order = (ordered) ? _gf_true : _gf_false;
-
-        brick = bricks;
-        while (count--) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO,
-                        "Registering brick: %s [notify filter: %d]",
-                        brick->brick_path, brick->filter);
-
-                ret = gf_changelog_register_brick (this, brick, need_order, xl);
-                if (ret != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
-                                "Error registering with changelog xlator");
-                        break;
-                }
-
-                brick++;
+    ret = gf_changelog_setup_logging(this, logfile, lvl);
+    if (ret)
+        goto error_return;
+
+    need_order = (ordered) ? _gf_true : _gf_false;
+
+    brick = bricks;
+    while (count--) {
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO, "brick=%s",
+                brick->brick_path, "notify_filter=%d", brick->filter, NULL);
+
+        ret = gf_changelog_register_brick(this, brick, need_order, xl);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
+                   "Error registering with changelog xlator");
+            break;
         }
 
-        if (ret != 0)
-                goto cleanup_inited_bricks;
+        brick++;
+    }
 
-        RESTORE_THIS();
-        return 0;
+    if (ret != 0)
+        goto cleanup_inited_bricks;
 
- cleanup_inited_bricks:
-        gf_cleanup_connections (this);
- error_return:
-        RESTORE_THIS();
-        return -1;
+    RESTORE_THIS();
+    return 0;
+
+cleanup_inited_bricks:
+    gf_cleanup_connections(this);
+error_return:
+    RESTORE_THIS();
+    return -1;
 }
 
 /**
@@ -597,27 +624,29 @@ gf_changelog_register_generic (struct gf_brick_spec *bricks, int count,
  * For generic API, refer gf_changelog_register_generic().
  */
 int
-gf_changelog_register (char *brick_path, char *scratch_dir,
-                       char *log_file, int log_level, int max_reconnects)
+gf_changelog_register(char *brick_path, char *scratch_dir, char *log_file,
+                      int log_level, int max_reconnects)
 {
-        struct gf_brick_spec brick = {0,};
+    struct gf_brick_spec brick = {
+        0,
+    };
 
-        if (master)
-                THIS = master;
-        else
-                return -1;
+    if (master)
+        THIS = master;
+    else
+        return -1;
 
-        brick.brick_path = brick_path;
-        brick.filter     = CHANGELOG_OP_TYPE_JOURNAL;
+    brick.brick_path = brick_path;
+    brick.filter = CHANGELOG_OP_TYPE_JOURNAL;
 
-        brick.init         = gf_changelog_journal_init;
-        brick.fini         = gf_changelog_journal_fini;
-        brick.callback     = gf_changelog_handle_journal;
-        brick.connected    = gf_changelog_journal_connect;
-        brick.disconnected = gf_changelog_journal_disconnect;
+    brick.init = gf_changelog_journal_init;
+    brick.fini = gf_changelog_journal_fini;
+    brick.callback = gf_changelog_handle_journal;
+    brick.connected = gf_changelog_journal_connect;
+    brick.disconnected = gf_changelog_journal_disconnect;
 
-        brick.ptr = scratch_dir;
+    brick.ptr = scratch_dir;
 
-        return gf_changelog_register_generic (&brick, 1, 1,
-                                              log_file, log_level, NULL);
+    return gf_changelog_register_generic(&brick, 1, 1, log_file, log_level,
+                                         NULL);
 }
diff --git a/xlators/features/changelog/lib/src/gf-history-changelog.c b/xlators/features/changelog/lib/src/gf-history-changelog.c
index 389b65f3b16..a16219f3664 100644
--- a/xlators/features/changelog/lib/src/gf-history-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-history-changelog.c
@@ -8,10 +8,10 @@
 #endif
 #include <string.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "syscall.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
 
 #include "gf-changelog-helpers.h"
 #include "gf-changelog-journal.h"
@@ -36,60 +36,60 @@
  *   -1: On error.
  */
 int
-gf_history_changelog_done (char *file)
+gf_history_changelog_done(char *file)
 {
-        int                     ret               = -1;
-        char                   *buffer            = NULL;
-        xlator_t               *this              = NULL;
-        gf_changelog_journal_t *jnl               = NULL;
-        gf_changelog_journal_t *hist_jnl          = NULL;
-        char                    to_path[PATH_MAX] = {0,};
+    int ret = -1;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    errno = EINVAL;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
+
+    if (!file || !strlen(file))
+        goto out;
+
+    /* make sure 'file' is inside ->jnl_working_dir */
+    buffer = realpath(file, NULL);
+    if (!buffer)
+        goto out;
+
+    if (strncmp(hist_jnl->jnl_working_dir, buffer,
+                strlen(hist_jnl->jnl_working_dir)))
+        goto out;
+
+    (void)snprintf(to_path, PATH_MAX, "%s%s", hist_jnl->jnl_processed_dir,
+                   basename(buffer));
+    gf_msg_debug(this->name, 0, "moving %s to processed directory", file);
+    ret = sys_rename(buffer, to_path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", file, "to=%s",
+                to_path, NULL);
+        goto out;
+    }
+
+    ret = 0;
 
-        errno = EINVAL;
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
-
-        if (!file || !strlen (file))
-                goto out;
-
-        /* make sure 'file' is inside ->jnl_working_dir */
-        buffer = realpath (file, NULL);
-        if (!buffer)
-                goto out;
-
-        if (strncmp (hist_jnl->jnl_working_dir,
-                     buffer, strlen (hist_jnl->jnl_working_dir)))
-                goto out;
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         hist_jnl->jnl_processed_dir, basename (buffer));
-        gf_msg_debug (this->name, 0,
-                      "moving %s to processed directory", file);
-        ret = sys_rename (buffer, to_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_RENAME_FAILED,
-                        "cannot move %s to %s",
-                        file, to_path);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        if (buffer)
-                free (buffer); /* allocated by realpath() */
-        return ret;
+out:
+    if (buffer)
+        free(buffer); /* allocated by realpath() */
+    return ret;
 }
 
 /**
@@ -103,33 +103,33 @@ gf_history_changelog_done (char *file)
  *    -1: On error.
  */
 int
-gf_history_changelog_start_fresh ()
+gf_history_changelog_start_fresh()
 {
-        xlator_t               *this     = NULL;
-        gf_changelog_journal_t *jnl      = NULL;
-        gf_changelog_journal_t *hist_jnl = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
 
-        if (gf_ftruncate (hist_jnl->jnl_fd, 0))
-                goto out;
+    if (gf_ftruncate(hist_jnl->jnl_fd, 0))
+        goto out;
 
-        return 0;
+    return 0;
 
- out:
-        return -1;
+out:
+    return -1;
 }
 
 /**
@@ -148,50 +148,52 @@ gf_history_changelog_start_fresh ()
  *     -1  : On error.
  */
 ssize_t
-gf_history_changelog_next_change (char *bufptr, size_t maxlen)
+gf_history_changelog_next_change(char *bufptr, size_t maxlen)
 {
-        ssize_t                 size             = -1;
-        int                     tracker_fd       = 0;
-        xlator_t               *this             = NULL;
-        gf_changelog_journal_t *jnl              = NULL;
-        gf_changelog_journal_t *hist_jnl         = NULL;
-        char                    buffer[PATH_MAX] = {0,};
-
-        if (maxlen > PATH_MAX) {
-                errno = ENAMETOOLONG;
-                goto out;
-        }
+    ssize_t size = -1;
+    int tracker_fd = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    char buffer[PATH_MAX] = {
+        0,
+    };
 
-        errno = EINVAL;
+    if (maxlen > PATH_MAX) {
+        errno = ENAMETOOLONG;
+        goto out;
+    }
 
-        this = THIS;
-        if (!this)
-                goto out;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        tracker_fd = hist_jnl->jnl_fd;
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
 
-        size = gf_readline (tracker_fd, buffer, maxlen);
-        if (size < 0) {
-                size = -1;
-                goto out;
-        }
+    tracker_fd = hist_jnl->jnl_fd;
 
-        if (size == 0)
-                goto out;
+    size = gf_readline(tracker_fd, buffer, maxlen);
+    if (size < 0) {
+        size = -1;
+        goto out;
+    }
+
+    if (size == 0)
+        goto out;
 
-        memcpy (bufptr, buffer, size - 1);
-        bufptr[size - 1] = '\0';
+    memcpy(bufptr, buffer, size - 1);
+    bufptr[size - 1] = '\0';
 
 out:
-        return size;
+    return size;
 }
 
 /**
@@ -212,106 +214,100 @@ out:
  *
  */
 ssize_t
-gf_history_changelog_scan ()
+gf_history_changelog_scan()
 {
-        int                     ret          = 0;
-        int                     tracker_fd   = 0;
-        size_t                  len          = 0;
-        size_t                  off          = 0;
-        xlator_t               *this         = NULL;
-        size_t                  nr_entries   = 0;
-        gf_changelog_journal_t *jnl          = NULL;
-        gf_changelog_journal_t *hist_jnl     = NULL;
-        struct dirent          *entryp       = NULL;
-        struct dirent          *result       = NULL;
-        char buffer[PATH_MAX]                = {0,};
-        static int              is_last_scan;
-
-        this = THIS;
-        if (!this)
-                goto out;
+    int tracker_fd = 0;
+    size_t off = 0;
+    xlator_t *this = NULL;
+    size_t nr_entries = 0;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+    static int is_last_scan;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+    if (JNL_IS_API_DISCONNECTED(jnl)) {
+        errno = ENOTCONN;
+        goto out;
+    }
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
+
+retry:
+    if (is_last_scan == 1)
+        return 0;
+    if (hist_jnl->hist_done == 0)
+        is_last_scan = 1;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-        if (JNL_IS_API_DISCONNECTED (jnl)) {
-                errno = ENOTCONN;
-                goto out;
-        }
+    errno = EINVAL;
+    if (hist_jnl->hist_done == -1)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    tracker_fd = hist_jnl->jnl_fd;
 
- retry:
-        if (is_last_scan == 1)
-                return 0;
-        if (hist_jnl->hist_done == 0)
-                is_last_scan = 1;
+    if (gf_ftruncate(tracker_fd, 0))
+        goto out;
 
-        errno = EINVAL;
-        if (hist_jnl->hist_done == -1)
-                goto out;
+    rewinddir(hist_jnl->jnl_dir);
 
-        tracker_fd = hist_jnl->jnl_fd;
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(hist_jnl->jnl_dir, scratch);
+        if (!entry || errno != 0)
+            break;
 
-        if (gf_ftruncate (tracker_fd, 0))
-                goto out;
+        if (strcmp(basename(entry->d_name), ".") == 0 ||
+            strcmp(basename(entry->d_name), "..") == 0)
+            continue;
 
-        len = offsetof (struct dirent, d_name)
-                + pathconf (hist_jnl->jnl_processing_dir, _PC_NAME_MAX) + 1;
-        entryp = GF_CALLOC (1, len,
-                            gf_changelog_mt_libgfchangelog_dirent_t);
-        if (!entryp)
-                goto out;
+        nr_entries++;
+
+        GF_CHANGELOG_FILL_BUFFER(hist_jnl->jnl_processing_dir, buffer, off,
+                                 strlen(hist_jnl->jnl_processing_dir));
+        GF_CHANGELOG_FILL_BUFFER(entry->d_name, buffer, off,
+                                 strlen(entry->d_name));
+        GF_CHANGELOG_FILL_BUFFER("\n", buffer, off, 1);
 
-        rewinddir (hist_jnl->jnl_dir);
-        while (1) {
-                ret = readdir_r (hist_jnl->jnl_dir, entryp, &result);
-                if (ret || !result)
-                        break;
-
-                if ( !strcmp (basename (entryp->d_name), ".")
-                     || !strcmp (basename (entryp->d_name), "..") )
-                        continue;
-
-                nr_entries++;
-
-                GF_CHANGELOG_FILL_BUFFER (hist_jnl->jnl_processing_dir,
-                                          buffer, off,
-                                          strlen (hist_jnl->jnl_processing_dir));
-                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
-                                          off, strlen (entryp->d_name));
-                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
-
-                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_WRITE_FAILED,
-                                "error writing changelog filename"
-                                " to tracker file");
-                        break;
-                }
-                off = 0;
+        if (gf_changelog_write(tracker_fd, buffer, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_WRITE_FAILED,
+                   "error writing changelog filename"
+                   " to tracker file");
+            break;
         }
+        off = 0;
+    }
 
-        GF_FREE (entryp);
-
-        gf_msg_debug (this->name, 0,
-                      "hist_done %d, is_last_scan: %d",
-                      hist_jnl->hist_done, is_last_scan);
-
-        if (!result) {
-                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1) {
-                        if (nr_entries > 0)
-                                return nr_entries;
-                        else {
-                                sleep(1);
-                                goto retry;
-                        }
-                }
+    gf_msg_debug(this->name, 0, "hist_done %d, is_last_scan: %d",
+                 hist_jnl->hist_done, is_last_scan);
+
+    if (!entry) {
+        if (gf_lseek(tracker_fd, 0, SEEK_SET) != -1) {
+            if (nr_entries > 0)
+                return nr_entries;
+            else {
+                sleep(1);
+                goto retry;
+            }
         }
- out:
-        return -1;
+    }
+out:
+    return -1;
 }
 
 /*
@@ -319,36 +315,36 @@ gf_history_changelog_scan ()
  * Returns 0 on success(updates given time-stamp), -1 on failure.
  */
 int
-gf_history_get_timestamp (int fd, int index, int len,
-                          unsigned long *ts)
+gf_history_get_timestamp(int fd, int index, int len, unsigned long *ts)
 {
-        xlator_t        *this             = NULL;
-        int             n_read            = -1;
-        char            path_buf[PATH_MAX]= {0,};
-        char            *iter             = path_buf;
-        size_t          offset            = index * (len+1);
-        unsigned long   value             = 0;
-        int             ret               = 0;
-
-        this = THIS;
-        if (!this) {
-                return -1;
-        }
-
-        n_read = sys_pread (fd, path_buf, len, offset);
-        if (n_read < 0 ) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_READ_ERROR,
-                         "could not read from htime file");
-                goto out;
-        }
-        iter+= len - TIMESTAMP_LENGTH;
-        sscanf (iter, "%lu",&value);
+    xlator_t *this = NULL;
+    int n_read = -1;
+    char path_buf[PATH_MAX] = {
+        0,
+    };
+    char *iter = path_buf;
+    size_t offset = index * (len + 1);
+    unsigned long value = 0;
+    int ret = 0;
+
+    this = THIS;
+    if (!this) {
+        return -1;
+    }
+
+    n_read = sys_pread(fd, path_buf, len, offset);
+    if (n_read < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_READ_ERROR,
+               "could not read from htime file");
+        goto out;
+    }
+    iter += len - TIMESTAMP_LENGTH;
+    sscanf(iter, "%lu", &value);
 out:
-        if(ret == 0)
-                *ts = value;
-        return ret;
+    if (ret == 0)
+        *ts = value;
+    return ret;
 }
 
 /*
@@ -356,38 +352,37 @@ out:
  * Checks whether @value is there next to @target_index or not
  */
 int
-gf_history_check ( int fd, int target_index, unsigned long value, int len)
+gf_history_check(int fd, int target_index, unsigned long value, int len)
 {
-        int             ret = 0;
-        unsigned long   ts1 = 0;
-        unsigned long   ts2 = 0;
-
-        if (target_index == 0) {
-                ret = gf_history_get_timestamp (fd, target_index, len, &ts1);
-                if (ret == -1)
-                        goto out;
-                if (value <= ts1)
-                        goto out;
-                else {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    int ret = 0;
+    unsigned long ts1 = 0;
+    unsigned long ts2 = 0;
 
-        ret = gf_history_get_timestamp (fd, target_index, len, &ts1);
-        if (ret ==-1)
-                goto out;
-        ret = gf_history_get_timestamp (fd, target_index -1, len, &ts2);
-        if (ret ==-1)
-                goto out;
-
-        if ( (value <= ts1) && (value > ts2) ) {
-                goto out;
-        }
-        else
-                ret = -1;
+    if (target_index == 0) {
+        ret = gf_history_get_timestamp(fd, target_index, len, &ts1);
+        if (ret == -1)
+            goto out;
+        if (value <= ts1)
+            goto out;
+        else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = gf_history_get_timestamp(fd, target_index, len, &ts1);
+    if (ret == -1)
+        goto out;
+    ret = gf_history_get_timestamp(fd, target_index - 1, len, &ts2);
+    if (ret == -1)
+        goto out;
+
+    if ((value <= ts1) && (value > ts2)) {
+        goto out;
+    } else
+        ret = -1;
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -407,83 +402,74 @@ out:
  */
 
 int
-gf_history_b_search (int fd, unsigned long value,
-                     unsigned long from, unsigned long to, int len)
+gf_history_b_search(int fd, unsigned long value, unsigned long from,
+                    unsigned long to, int len)
 {
-        int             m_index   = -1;
-        unsigned long   cur_value = 0;
-        unsigned long   ts1       = 0;
-        int             ret       = 0;
-
-        m_index = (from + to)/2;
-
-        if ( (to - from) <=1 ) {
-                /* either one or 2 changelogs left */
-                if ( to != from ) {
-                        /* check if value is less or greater than to
-                         * return accordingly
-                         */
-                        ret = gf_history_get_timestamp (fd, from, len, &ts1);
-                        if (ret ==-1)
-                                goto out;
-                        if ( ts1 >= value) {
-                                /* actually compatision should be
-                                 * exactly == but considering
-                                 *
-                                 * case of only 2 changelogs in htime file
-                                 */
-                                return from;
-                        }
-                        else
-                                return to;
-                }
-                else
-                        return to;
-        }
-
-        ret = gf_history_get_timestamp (fd, m_index, len, &cur_value);
+    int m_index = -1;
+    unsigned long cur_value = 0;
+    unsigned long ts1 = 0;
+    int ret = 0;
+
+    m_index = (from + to) / 2;
+
+    if ((to - from) <= 1) {
+        /* either one or 2 changelogs left */
+        if (to != from) {
+            /* check if value is less or greater than to
+             * return accordingly
+             */
+            ret = gf_history_get_timestamp(fd, from, len, &ts1);
+            if (ret == -1)
+                goto out;
+            if (ts1 >= value) {
+                /* actually compatision should be
+                 * exactly == but considering
+                 *
+                 * case of only 2 changelogs in htime file
+                 */
+                return from;
+            } else
+                return to;
+        } else
+            return to;
+    }
+
+    ret = gf_history_get_timestamp(fd, m_index, len, &cur_value);
+    if (ret == -1)
+        goto out;
+    if (cur_value == value) {
+        return m_index;
+    } else if (value > cur_value) {
+        ret = gf_history_get_timestamp(fd, m_index + 1, len, &cur_value);
         if (ret == -1)
-                goto out;
-        if (cur_value == value) {
+            goto out;
+        if (value < cur_value)
+            return m_index + 1;
+        else
+            return gf_history_b_search(fd, value, m_index + 1, to, len);
+    } else {
+        if (m_index == 0) {
+            /*  we are sure that values exists
+             *  in this htime file
+             */
+            return 0;
+        } else {
+            ret = gf_history_get_timestamp(fd, m_index - 1, len, &cur_value);
+            if (ret == -1)
+                goto out;
+            if (value > cur_value) {
                 return m_index;
+            } else
+                return gf_history_b_search(fd, value, from, m_index - 1, len);
         }
-        else if (value > cur_value) {
-                ret = gf_history_get_timestamp (fd, m_index+1, len, &cur_value);
-                if (ret == -1)
-                        goto out;
-                if (value < cur_value)
-                        return m_index + 1;
-                else
-                        return gf_history_b_search (fd, value,
-                                                    m_index+1, to, len);
-        }
-        else {
-                if (m_index ==0) {
-                       /*  we are sure that values exists
-                        *  in this htime file
-                        */
-                        return 0;
-                }
-                else {
-                        ret = gf_history_get_timestamp (fd, m_index-1, len,
-                                                        &cur_value);
-                        if (ret == -1)
-                                goto out;
-                        if (value > cur_value) {
-                                return m_index;
-                        }
-                        else
-                                return gf_history_b_search (fd, value, from,
-                                                            m_index-1, len);
-                }
-        }
+    }
 out:
-        return -1;
+    return -1;
 }
 
 /*
  * Description: Checks if the changelog path is usable or not,
- *              which is differenciated by checking for "changelog"
+ *              which is differentiated by checking for "changelog"
  *              in the path and not "CHANGELOG".
  *
  * Returns:
@@ -491,64 +477,59 @@ out:
  * 0 : No, Not usable ( contains, "changelog")
  */
 int
-gf_is_changelog_usable (char *cl_path)
+gf_is_changelog_usable(char *cl_path)
 {
-        int             ret             = -1;
-        const char      low_c[]         = "changelog";
-        char            *str_ret        = NULL;
-        char            *bname          = NULL;
+    int ret = -1;
+    const char low_c[] = "changelog";
+    char *str_ret = NULL;
+    char *bname = NULL;
 
-        bname = basename (cl_path);
+    bname = basename(cl_path);
 
-        str_ret = strstr (bname, low_c);
+    str_ret = strstr(bname, low_c);
 
-        if (str_ret != NULL)
-                ret = 0;
-        else
-                ret = 1;
-
-        return ret;
+    if (str_ret != NULL)
+        ret = 0;
+    else
+        ret = 1;
 
+    return ret;
 }
 
 void *
-gf_changelog_consume_wrap (void* data)
+gf_changelog_consume_wrap(void *data)
 {
-        int                          ret   = -1;
-        ssize_t                      nread = 0;
-        xlator_t                    *this  = NULL;
-        gf_changelog_consume_data_t *ccd   = NULL;
-
-        ccd = (gf_changelog_consume_data_t *) data;
-        this = ccd->this;
-
-        ccd->retval = -1;
-
-        nread = sys_pread (ccd->fd, ccd->changelog, PATH_MAX, ccd->offset);
-        if (nread < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_READ_ERROR,
-                        "cannot read from history metadata file");
-                goto out;
-        }
-
-        /* TODO: handle short reads and EOF. */
-        if (gf_is_changelog_usable (ccd->changelog) == 1) {
-
-                ret = gf_changelog_consume (ccd->this,
-                                            ccd->jnl, ccd->changelog, _gf_true);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                0, CHANGELOG_LIB_MSG_PARSE_ERROR,
-                                "could not parse changelog: %s",
-                                ccd->changelog);
-                        goto out;
-                }
+    int ret = -1;
+    ssize_t nread = 0;
+    xlator_t *this = NULL;
+    gf_changelog_consume_data_t *ccd = NULL;
+
+    ccd = (gf_changelog_consume_data_t *)data;
+    this = ccd->this;
+
+    ccd->retval = -1;
+
+    nread = sys_pread(ccd->fd, ccd->changelog, PATH_MAX - 1, ccd->offset);
+    if (nread < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_READ_ERROR,
+               "cannot read from history metadata file");
+        goto out;
+    }
+
+    /* TODO: handle short reads and EOF. */
+    if (gf_is_changelog_usable(ccd->changelog) == 1) {
+        ret = gf_changelog_consume(ccd->this, ccd->jnl, ccd->changelog,
+                                   _gf_true);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_PARSE_ERROR,
+                    "name=%s", ccd->changelog, NULL);
+            goto out;
         }
-        ccd->retval = 0;
+    }
+    ccd->retval = 0;
 
- out:
-        return NULL;
+out:
+    return NULL;
 }
 
 /**
@@ -557,130 +538,132 @@ gf_changelog_consume_wrap (void* data)
  * to index "to" in open htime file whose fd is "fd".
  */
 
-#define MAX_PARALLELS  10
+#define MAX_PARALLELS 10
 
 void *
-gf_history_consume (void * data)
+gf_history_consume(void *data)
 {
-        xlator_t                    *this              = NULL;
-        gf_changelog_journal_t              *jnl               = NULL;
-        gf_changelog_journal_t              *hist_jnl          = NULL;
-        int                          ret               = 0;
-        int                          iter              = 0;
-        int                          fd                = -1;
-        int                          from              = -1;
-        int                          to                = -1;
-        int                          len               = -1;
-        int                          n_parallel        = 0;
-        int                          n_envoked         = 0;
-        gf_boolean_t                 publish           = _gf_true;
-        pthread_t th_id[MAX_PARALLELS]                 = {0,};
-        gf_changelog_history_data_t *hist_data         = NULL;
-        gf_changelog_consume_data_t ccd[MAX_PARALLELS] = {{0},};
-        gf_changelog_consume_data_t *curr              = NULL;
-
-        hist_data = (gf_changelog_history_data_t *) data;
-        if (hist_data == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        fd         = hist_data->htime_fd;
-        from       = hist_data->from;
-        to         = hist_data->to;
-        len        = hist_data->len;
-        n_parallel = hist_data->n_parallel;
-
-        THIS = hist_data->this;
-        this = hist_data->this;
-        if (!this) {
-                ret = -1;
-                goto out;
-        }
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl) {
-                ret = -1;
-                goto out;
-        }
-
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl) {
-                ret = -1;
-                goto out;
-        }
-
-        while (from <= to) {
-                n_envoked = 0;
-
-                for (iter = 0 ; (iter < n_parallel) && (from <= to); iter++) {
-                        curr = &ccd[iter];
-
-                        curr->this   = this;
-                        curr->jnl    = hist_jnl;
-                        curr->fd     = fd;
-                        curr->offset = from * (len + 1);
-
-                        curr->retval = 0;
-                        memset (curr->changelog, '\0', PATH_MAX);
-
-                        ret = pthread_create (&th_id[iter], NULL,
-                                              gf_changelog_consume_wrap, curr);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED
-                                        , "could not create consume-thread");
-                                ret = -1;
-                                goto sync;
-                        } else
-                                n_envoked++;
-
-                        from++;
-                }
-
-        sync:
-                for (iter = 0; iter < n_envoked; iter++) {
-                        ret = pthread_join (th_id[iter], NULL);
-                        if (ret) {
-                                publish = _gf_false;
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED,
-                                        "pthread_join() error");
-                                /* try to join the rest */
-                                continue;
-                        }
-
-                        if (publish == _gf_false)
-                                continue;
-
-                        curr = &ccd[iter];
-                        if (ccd->retval) {
-                                publish = _gf_false;
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        0, CHANGELOG_LIB_MSG_PARSE_ERROR,
-                                        "parsing error, ceased publishing...");
-                                continue;
-                        }
-
-                        ret = gf_changelog_publish (curr->this,
-                                                    curr->jnl, curr->changelog);
-                        if (ret) {
-                                publish = _gf_false;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_LIB_MSG_PUBLISH_ERROR,
-                                        "publish error, ceased publishing...");
-                        }
-                }
-        }
-
-       /* informing "parsing done". */
-        hist_jnl->hist_done = (publish == _gf_true) ? 0 : -1;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    int ret = 0;
+    int iter = 0;
+    int fd = -1;
+    int from = -1;
+    int to = -1;
+    int len = -1;
+    int n_parallel = 0;
+    int n_envoked = 0;
+    gf_boolean_t publish = _gf_true;
+    pthread_t th_id[MAX_PARALLELS] = {
+        0,
+    };
+    gf_changelog_history_data_t *hist_data = NULL;
+    gf_changelog_consume_data_t ccd[MAX_PARALLELS] = {
+        {0},
+    };
+    gf_changelog_consume_data_t *curr = NULL;
+
+    hist_data = (gf_changelog_history_data_t *)data;
+    if (hist_data == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    fd = hist_data->htime_fd;
+    from = hist_data->from;
+    to = hist_data->to;
+    len = hist_data->len;
+    n_parallel = hist_data->n_parallel;
+
+    THIS = hist_data->this;
+    this = hist_data->this;
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    while (from <= to) {
+        n_envoked = 0;
+
+        for (iter = 0; (iter < n_parallel) && (from <= to); iter++) {
+            curr = &ccd[iter];
+
+            curr->this = this;
+            curr->jnl = hist_jnl;
+            curr->fd = fd;
+            curr->offset = from * (len + 1);
+
+            curr->retval = 0;
+            memset(curr->changelog, '\0', PATH_MAX);
+
+            ret = gf_thread_create(&th_id[iter], NULL,
+                                   gf_changelog_consume_wrap, curr,
+                                   "clogc%03hx", (iter + 1) & 0x3ff);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                       "could not create consume-thread");
+                goto sync;
+            } else
+                n_envoked++;
+
+            from++;
+        }
+
+    sync:
+        for (iter = 0; iter < n_envoked; iter++) {
+            ret = pthread_join(th_id[iter], NULL);
+            if (ret) {
+                publish = _gf_false;
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED,
+                       "pthread_join() error");
+                /* try to join the rest */
+                continue;
+            }
+
+            if (publish == _gf_false)
+                continue;
+
+            curr = &ccd[iter];
+            if (ccd->retval) {
+                publish = _gf_false;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED, NULL);
+                continue;
+            }
+
+            ret = gf_changelog_publish(curr->this, curr->jnl, curr->changelog);
+            if (ret) {
+                publish = _gf_false;
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       CHANGELOG_LIB_MSG_PUBLISH_ERROR,
+                       "publish error, ceased publishing...");
+            }
+        }
+    }
+
+    /* informing "parsing done". */
+    hist_jnl->hist_done = (publish == _gf_true) ? 0 : -1;
 
 out:
-        if (fd != -1)
-                sys_close (fd);
-        GF_FREE (hist_data);
-        return NULL;
+    if (fd != -1)
+        (void)sys_close(fd);
+    GF_FREE(hist_data);
+    return NULL;
 }
 
 /**
@@ -710,277 +693,328 @@ out:
  *   -2 : Ignore this metadata file and process next
  */
 int
-gf_changelog_extract_min_max (const char *dname, const char *htime_dir,
-                              int *fd, unsigned long *total,
-                              unsigned long *min_ts, unsigned long *max_ts)
+gf_changelog_extract_min_max(const char *dname, const char *htime_dir, int *fd,
+                             unsigned long *total, unsigned long *min_ts,
+                             unsigned long *max_ts)
 {
-        int          ret          = -1;
-        xlator_t    *this         = NULL;
-        char htime_file[PATH_MAX] = {0,};
-        struct stat  stbuf        = {0,};
-        char        *iter         = NULL;
-        char x_value[30]          = {0,};
-
-        this = THIS;
+    int ret = -1;
+    xlator_t *this = NULL;
+    char htime_file[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    char *iter = NULL;
+    char x_value[30] = {
+        0,
+    };
+
+    this = THIS;
+
+    snprintf(htime_file, PATH_MAX, "%s/%s", htime_dir, dname);
+
+    iter = (htime_file + strlen(htime_file) - TIMESTAMP_LENGTH);
+    sscanf(iter, "%lu", min_ts);
+
+    ret = sys_stat(htime_file, &stbuf);
+    if (ret) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=stat", "path=%s", htime_file, NULL);
+        goto out;
+    }
+
+    /* ignore everything except regular files */
+    if (!S_ISREG(stbuf.st_mode)) {
+        ret = -2;
+        goto out;
+    }
+
+    *fd = open(htime_file, O_RDONLY);
+    if (*fd < 0) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=open", "path=%s", htime_file, NULL);
+        goto out;
+    }
+
+    /* Looks good, extract max timestamp */
+    ret = sys_fgetxattr(*fd, HTIME_KEY, x_value, sizeof(x_value));
+    if (ret < 0) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_GET_XATTR_FAILED, "path=%s", htime_file,
+                NULL);
+        goto out;
+    }
+
+    sscanf(x_value, "%lu:%lu", max_ts, total);
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_MIN_MAX_INFO,
+            "min=%lu", *min_ts, "max=%lu", *max_ts, "total_changelogs=%lu",
+            *total, NULL);
+
+    ret = 0;
 
-        snprintf (htime_file, PATH_MAX, "%s/%s", htime_dir, dname);
-
-        iter = (htime_file + strlen (htime_file) - TIMESTAMP_LENGTH);
-        sscanf (iter ,"%lu",min_ts);
+out:
+    return ret;
+}
 
-        ret = sys_stat (htime_file, &stbuf);
+/* gf_history_changelog returns actual_end and spawns threads to
+ * parse historical changelogs. The return values are as follows.
+ *     0 : On success
+ *     1 : Successful, but partial historical changelogs available,
+ *         end time falls into different htime file or future time
+ *    -2 : Error, requested historical changelog not available, not
+ *         even partial
+ *    -1 : On any error
+ */
+int
+gf_history_changelog(char *changelog_dir, unsigned long start,
+                     unsigned long end, int n_parallel,
+                     unsigned long *actual_end)
+{
+    int ret = 0;
+    int len = -1;
+    int fd = -1;
+    int n_read = -1;
+    unsigned long min_ts = 0;
+    unsigned long max_ts = 0;
+    unsigned long end2 = 0;
+    unsigned long ts1 = 0;
+    unsigned long ts2 = 0;
+    unsigned long to = 0;
+    unsigned long from = 0;
+    unsigned long total_changelog = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    gf_changelog_history_data_t *hist_data = NULL;
+    DIR *dirp = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    pthread_t consume_th = 0;
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+    gf_boolean_t partial_history = _gf_false;
+
+    pthread_attr_t attr;
+
+    this = THIS;
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = pthread_attr_init(&attr);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+               "Pthread init failed");
+        return -1;
+    }
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    hist_jnl = (gf_changelog_journal_t *)jnl->hist_jnl;
+    if (!hist_jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_REQUESTING_INFO,
+            "start=%lu", start, "end=%lu", end, NULL);
+
+    /* basic sanity check */
+    if (start > end || n_parallel <= 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HIST_FAILED,
+                "start=%lu", start, "end=%lu", end, "thread_count=%d",
+                n_parallel, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* cap parallelism count */
+    if (n_parallel > MAX_PARALLELS)
+        n_parallel = MAX_PARALLELS;
+
+    CHANGELOG_FILL_HTIME_DIR(changelog_dir, htime_dir);
+
+    dirp = sys_opendir(htime_dir);
+    if (dirp == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=opendir", "path=%s", htime_dir, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    for (;;) {
+        errno = 0;
+
+        entry = sys_readdir(dirp, scratch);
+
+        if (!entry || errno != 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_LIB_MSG_HIST_FAILED, "start=%lu", start,
+                    "end=%lu", end, NULL);
+            ret = -2;
+            break;
+        }
+
+        ret = gf_changelog_extract_min_max(entry->d_name, htime_dir, &fd,
+                                           &total_changelog, &min_ts, &max_ts);
         if (ret) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_HTIME_ERROR,
-                        "stat() failed on htime file %s",
-                        htime_file);
-                goto out;
-        }
-
-        /* ignore everything except regular files */
-        if (!S_ISREG (stbuf.st_mode)) {
-                ret = -2;
-                goto out;
+            if (-2 == ret)
+                continue;
+            goto out;
         }
 
-        *fd = open (htime_file, O_RDONLY);
-        if (*fd < 0) {
+        if (start >= min_ts && start < max_ts) {
+            /**
+             * TODO: handle short reads later...
+             */
+            n_read = sys_read(fd, buffer, PATH_MAX);
+            if (n_read < 0) {
                 ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_HTIME_ERROR,
-                        "open() failed for htime %s",
-                        htime_file);
+                gf_msg(this->name, GF_LOG_ERROR, errno,
+                       CHANGELOG_LIB_MSG_READ_ERROR,
+                       "unable to read htime file");
                 goto out;
-        }
+            }
 
-        /* Looks good, extract max timestamp */
-        ret = sys_fgetxattr (*fd, HTIME_KEY, x_value, sizeof (x_value));
-        if (ret < 0) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_GET_XATTR_FAILED,
-                        "error extracting max timstamp from htime file"
-                        " %s", htime_file);
-                goto out;
-        }
+            len = strlen(buffer);
 
-        sscanf (x_value, "%lu:%lu", max_ts, total);
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_LIB_MSG_TOTAL_LOG_INFO,
-                "MIN: %lu, MAX: %lu, TOTAL CHANGELOGS: %lu",
-                *min_ts, *max_ts, *total);
-
-        ret = 0;
-
- out:
-        return ret;
-}
-
-int
-gf_history_changelog (char* changelog_dir, unsigned long start,
-                      unsigned long end, int n_parallel,
-                      unsigned long *actual_end)
-{
-        int                             ret                     = 0;
-        int                             len                     = -1;
-        int                             fd                      = -1;
-        int                             n_read                  = -1;
-        unsigned long                   min_ts                  = 0;
-        unsigned long                   max_ts                  = 0;
-        unsigned long                   end2                    = 0;
-        unsigned long                   ts1                     = 0;
-        unsigned long                   ts2                     = 0;
-        unsigned long                   to                      = 0;
-        unsigned long                   from                    = 0;
-        unsigned long                   total_changelog         = 0;
-        xlator_t                        *this                   = NULL;
-        gf_changelog_journal_t                  *jnl                    = NULL;
-        gf_changelog_journal_t                  *hist_jnl               = NULL;
-        gf_changelog_history_data_t     *hist_data              = NULL;
-        DIR                             *dirp                   = NULL;
-        struct dirent                   *dp                     = NULL;
-        pthread_t                       consume_th              = 0;
-        char                            htime_dir[PATH_MAX]     = {0,};
-        char                            buffer[PATH_MAX]        = {0,};
-
-        pthread_attr_t attr;
-
-        ret = pthread_attr_init (&attr);
-        if (ret != 0) {
-                return -1;
-        }
+            /**
+             * search @start in the htime file returning it's index
+             * (@from)
+             */
+            from = gf_history_b_search(fd, start, 0, total_changelog - 1, len);
 
-        this = THIS;
-        if (!this) {
+            /* ensuring correctness of gf_b_search */
+            if (gf_history_check(fd, from, start, len) != 0) {
                 ret = -1;
-                goto out;
-        }
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_GET_TIME_ERROR, "for=start",
+                        "start=%lu", start, "idx=%lu", from, NULL);
+                goto out;
+            }
+
+            end2 = (end <= max_ts) ? end : max_ts;
+
+            /* Check if end falls out of same HTIME file. The end
+             * falling to a different htime file or changelog
+             * disable-enable is detected only after 20 seconds.
+             * This is required because, applications generally
+             * asks historical changelogs till current time and
+             * it is possible changelog is not rolled over yet.
+             * So, buffer time of default rollover time plus 5
+             * seconds is subtracted.  If the application requests
+             * the end time with in half a minute of changelog
+             * disable, it's not detected as changelog disable and
+             * it's application's responsibility to retry after
+             * 20 seconds before confirming it as partial history.
+             */
+            if ((end - 20) > max_ts) {
+                partial_history = _gf_true;
+            }
+
+            /**
+             * search @end2 in htime file returning it's index (@to)
+             */
+            to = gf_history_b_search(fd, end2, 0, total_changelog - 1, len);
+
+            if (gf_history_check(fd, to, end2, len) != 0) {
                 ret = -1;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_GET_TIME_ERROR, "for=end",
+                        "start=%lu", end2, "idx=%lu", to, NULL);
                 goto out;
-        }
+            }
 
-        hist_jnl = (gf_changelog_journal_t *) jnl->hist_jnl;
-        if (!hist_jnl) {
-                ret = -1;
+            ret = gf_history_get_timestamp(fd, from, len, &ts1);
+            if (ret == -1)
                 goto out;
-        }
 
-        /* basic sanity check */
-        if (start > end || n_parallel <= 0) {
-                ret = -1;
+            ret = gf_history_get_timestamp(fd, to, len, &ts2);
+            if (ret == -1)
                 goto out;
-        }
 
-        /* cap parallelism count */
-        if (n_parallel > MAX_PARALLELS)
-                n_parallel = MAX_PARALLELS;
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_FINAL_INFO,
+                    "from=%lu", ts1, "to=%lu", ts2, "changes=%lu",
+                    (to - from + 1), NULL);
 
-        CHANGELOG_FILL_HTIME_DIR (changelog_dir, htime_dir);
+            hist_data = GF_CALLOC(1, sizeof(gf_changelog_history_data_t),
+                                  gf_changelog_mt_history_data_t);
 
-        dirp = sys_opendir (htime_dir);
-        if (dirp == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_HTIME_ERROR,
-                        "open dir on htime failed : %s",
-                        htime_dir);
+            hist_data->htime_fd = fd;
+            hist_data->from = from;
+            hist_data->to = to;
+            hist_data->len = len;
+            hist_data->n_parallel = n_parallel;
+            hist_data->this = this;
+
+            ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+            if (ret != 0) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+                       "unable to sets the detach"
+                       " state attribute");
                 ret = -1;
                 goto out;
-        }
+            }
 
-        while ((dp = sys_readdir (dirp)) != NULL) {
-                ret = gf_changelog_extract_min_max (dp->d_name, htime_dir,
-                                                    &fd, &total_changelog,
-                                                    &min_ts, &max_ts);
-                if (ret) {
-                        if (-2 == ret)
-                                continue;
-                        goto out;
-                }
-
-                if (start >= min_ts && start < max_ts) {
-                        /**
-                         * TODO: handle short reads later...
-                         */
-                        n_read = sys_read (fd, buffer, PATH_MAX);
-                        if (n_read < 0) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        CHANGELOG_LIB_MSG_READ_ERROR,
-                                        "unable to read htime file");
-                                goto out;
-                        }
-
-                        len = strlen (buffer);
-
-                        /**
-                         * search @start in the htime file returning it's index
-                         * (@from)
-                         */
-                        from = gf_history_b_search (fd, start, 0,
-                                                   total_changelog - 1, len);
-
-                        /* ensuring correctness of gf_b_search */
-                        if (gf_history_check (fd, from, start, len) != 0) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_LIB_MSG_GET_TIME_ERROR,
-                                        "wrong result for start: %lu idx: %lu",
-                                        start, from);
-                                goto out;
-                        }
-
-                        end2 = (end <= max_ts) ? end : max_ts;
-
-                        /**
-                         * search @end2 in htime file returning it's index (@to)
-                         */
-                        to = gf_history_b_search (fd, end2,
-                                                  0, total_changelog - 1, len);
-
-                        if (gf_history_check (fd, to, end2, len) != 0) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_LIB_MSG_GET_TIME_ERROR,
-                                        "wrong result for start: %lu idx: %lu",
-                                        end2, to);
-                                goto out;
-                        }
-
-                        ret = gf_history_get_timestamp (fd, from, len, &ts1);
-                        if (ret == -1)
-                                goto out;
-
-                        ret = gf_history_get_timestamp (fd, to, len, &ts2);
-                        if (ret == -1)
-                                goto out;
-
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_LIB_MSG_TOTAL_LOG_INFO,
-                                "FINAL: from: %lu, to: %lu, changes: %lu",
-                                ts1, ts2, (to - from + 1));
-
-                        hist_data =  GF_CALLOC (1,
-                                     sizeof (gf_changelog_history_data_t),
-                                     gf_changelog_mt_history_data_t);
-
-                        hist_data->htime_fd   = fd;
-                        hist_data->from       = from;
-                        hist_data->to         = to;
-                        hist_data->len        = len;
-                        hist_data->n_parallel = n_parallel;
-                        hist_data->this       = this;
-
-                        ret = pthread_attr_setdetachstate
-                                (&attr, PTHREAD_CREATE_DETACHED);
-                        if (ret != 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_PTHREAD_ERROR,
-                                        "unable to sets the detach"
-                                        " state attribute");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        /* spawn a thread for background parsing & publishing */
-                        ret = pthread_create (&consume_th, &attr,
-                                              gf_history_consume, hist_data);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED
-                                        , "creation of consume parent-thread"
-                                        " failed.");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        goto out;
-
-                } /* end of range check */
-
-        } /* end of readdir() */
-
-        if (!from || !to)
+            /* spawn a thread for background parsing & publishing */
+            ret = gf_thread_create(&consume_th, &attr, gf_history_consume,
+                                   hist_data, "cloghcon");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                       "creation of consume parent-thread"
+                       " failed.");
                 ret = -1;
+                goto out;
+            }
 
-out:
-        if (dirp != NULL)
-                sys_closedir (dirp);
-
-        if (ret < 0) {
-                if (fd != -1)
-                        sys_close (fd);
-                GF_FREE (hist_data);
-                (void) pthread_attr_destroy (&attr);
+            goto out;
 
-                return ret;
+        } else { /* end of range check */
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_LIB_MSG_HIST_FAILED, "start=%lu", start,
+                    "end=%lu", end, "chlog_min=%lu", min_ts, "chlog_max=%lu",
+                    max_ts, NULL);
         }
+    } /* end of readdir() */
 
-        hist_jnl->hist_done = 1;
-        *actual_end = ts2;
+out:
+    if (dirp != NULL)
+        (void)sys_closedir(dirp);
+
+    if (ret < 0) {
+        if (fd != -1)
+            (void)sys_close(fd);
+        GF_FREE(hist_data);
+        (void)pthread_attr_destroy(&attr);
 
         return ret;
+    }
+
+    hist_jnl->hist_done = 1;
+    *actual_end = ts2;
+
+    if (partial_history) {
+        ret = 1;
+    }
+
+    return ret;
 }
diff --git a/xlators/features/changelog/src/Makefile.am b/xlators/features/changelog/src/Makefile.am
index 27af7a5ebd3..eee7dfa238d 100644
--- a/xlators/features/changelog/src/Makefile.am
+++ b/xlators/features/changelog/src/Makefile.am
@@ -3,25 +3,26 @@ xlator_LTLIBRARIES = changelog.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 noinst_HEADERS = changelog-helpers.h changelog-mem-types.h changelog-rt.h \
-		 changelog-rpc-common.h changelog-misc.h changelog-encoders.h \
-		 changelog-rpc-common.h changelog-rpc.h changelog-ev-handle.h \
-		 changelog-messages.h
+	changelog-rpc-common.h changelog-misc.h changelog-encoders.h \
+	changelog-rpc-common.h changelog-rpc.h changelog-ev-handle.h \
+	changelog-messages.h
 
 changelog_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 changelog_la_SOURCES = changelog.c changelog-rt.c changelog-helpers.c \
-		changelog-encoders.c changelog-rpc.c changelog-barrier.c \
-		changelog-rpc-common.c changelog-ev-handle.c
+	changelog-encoders.c changelog-rpc.c changelog-barrier.c \
+	changelog-rpc-common.c changelog-ev-handle.c
 changelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-		      $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		      $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	      -I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
-	      -I$(top_srcdir)/rpc/rpc-transport/socket/src \
-	      -I$(top_srcdir)/xlators/features/changelog/lib/src/ \
-	      -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
-	      -DDATADIR=\"$(localstatedir)\"
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/rpc-transport/socket/src \
+	-I$(top_srcdir)/xlators/features/changelog/lib/src/ \
+	-fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+	-DDATADIR=\"$(localstatedir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/changelog/src/changelog-barrier.c b/xlators/features/changelog/src/changelog-barrier.c
index ac1eb0e4397..0fb89ddb127 100644
--- a/xlators/features/changelog/src/changelog-barrier.c
+++ b/xlators/features/changelog/src/changelog-barrier.c
@@ -10,125 +10,122 @@
 
 #include "changelog-helpers.h"
 #include "changelog-messages.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 
 /* Enqueue a stub*/
 void
-__chlog_barrier_enqueue (xlator_t *this, call_stub_t *stub)
+__chlog_barrier_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        changelog_priv_t *priv    = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        list_add_tail (&stub->list, &priv->queue);
-        priv->queue_size++;
+    list_add_tail(&stub->list, &priv->queue);
+    priv->queue_size++;
 
-        return;
+    return;
 }
 
 /* Dequeue a stub */
 call_stub_t *
-__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue)
+__chlog_barrier_dequeue(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t      *stub            = NULL;
-        changelog_priv_t *priv            = NULL;
+    call_stub_t *stub = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (list_empty (queue))
-                goto out;
+    if (list_empty(queue))
+        goto out;
 
-        stub = list_entry (queue->next, call_stub_t, list);
-        list_del_init (&stub->list);
+    stub = list_entry(queue->next, call_stub_t, list);
+    list_del_init(&stub->list);
 
 out:
-        return stub;
+    return stub;
 }
 
 /* Dequeue all the stubs and call corresponding resume functions */
 void
-chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue)
+chlog_barrier_dequeue_all(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t            *stub    = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_BARRIER_INFO,
-                "Dequeuing all the changelog barriered fops");
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS,
+            NULL);
 
-        while ((stub = __chlog_barrier_dequeue (this, queue)))
-                call_resume (stub);
+    while ((stub = __chlog_barrier_dequeue(this, queue)))
+        call_resume(stub);
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_BARRIER_INFO,
-                "Dequeuing changelog barriered fops is finished");
-        return;
+    gf_smsg(this->name, GF_LOG_INFO, 0,
+            CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED, NULL);
+    return;
 }
 
 /* Function called on changelog barrier timeout */
 void
-chlog_barrier_timeout (void *data)
+chlog_barrier_timeout(void *data)
 {
-        xlator_t               *this    = NULL;
-        changelog_priv_t       *priv    = NULL;
-        struct list_head        queue   = {0,};
+    xlator_t *this = NULL;
+    changelog_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        this = data;
-        THIS = this;
-        priv = this->private;
+    this = data;
+    THIS = this;
+    priv = this->private;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                CHANGELOG_MSG_BARRIER_ERROR,
-                "Disabling changelog barrier because of the timeout.");
+    gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_BARRIER_TIMEOUT, NULL);
 
-        LOCK (&priv->lock);
-        {
-                __chlog_barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        __chlog_barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        chlog_barrier_dequeue_all (this, &queue);
+    chlog_barrier_dequeue_all(this, &queue);
 
-        return;
+    return;
 }
 
 /* Disable changelog barrier enable flag */
 void
-__chlog_barrier_disable (xlator_t *this, struct list_head *queue)
+__chlog_barrier_disable(xlator_t *this, struct list_head *queue)
 {
-        changelog_priv_t  *priv   = this->private;
-        GF_ASSERT (priv);
+    changelog_priv_t *priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->timer) {
-                gf_timer_call_cancel (this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
+    if (priv->timer) {
+        gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
 
-        list_splice_init (&priv->queue, queue);
-        priv->queue_size = 0;
-        priv->barrier_enabled = _gf_false;
+    list_splice_init(&priv->queue, queue);
+    priv->queue_size = 0;
+    priv->barrier_enabled = _gf_false;
 }
 
 /* Enable chagelog barrier enable with timer */
 int
-__chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv)
+__chlog_barrier_enable(xlator_t *this, changelog_priv_t *priv)
 {
-        int             ret     = -1;
-
-        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
-                                           chlog_barrier_timeout, (void *)this);
-        if (!priv->timer) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        CHANGELOG_MSG_BARRIER_ERROR,
-                        "Couldn't add changelog barrier timeout event.");
-                goto out;
-        }
-
-        priv->barrier_enabled = _gf_true;
-        ret = 0;
+    int ret = -1;
+
+    priv->timer = gf_timer_call_after(this->ctx, priv->timeout,
+                                      chlog_barrier_timeout, (void *)this);
+    if (!priv->timer) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, 0,
+                CHANGELOG_MSG_TIMEOUT_ADD_FAILED, NULL);
+        goto out;
+    }
+
+    priv->barrier_enabled = _gf_true;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
index 95030236636..63754516c2e 100644
--- a/xlators/features/changelog/src/changelog-encoders.c
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -11,117 +11,117 @@
 #include "changelog-encoders.h"
 
 size_t
-entry_fn (void *data, char *buffer, gf_boolean_t encode)
+entry_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char    *tmpbuf = NULL;
-        size_t  bufsz  = 0;
-        struct changelog_entry_fields *ce = NULL;
-
-        ce = (struct changelog_entry_fields *) data;
-
-        if (encode) {
-                tmpbuf = uuid_utoa (ce->cef_uuid);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
-        } else {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                                       ce->cef_uuid, sizeof (uuid_t));
-        }
-
-        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
-        CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                               ce->cef_bname, strlen (ce->cef_bname));
-        return bufsz;
+    char *tmpbuf = NULL;
+    size_t bufsz = 0;
+    struct changelog_entry_fields *ce = NULL;
+
+    ce = (struct changelog_entry_fields *)data;
+
+    if (encode) {
+        tmpbuf = uuid_utoa(ce->cef_uuid);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, tmpbuf, strlen(tmpbuf));
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_uuid, sizeof(uuid_t));
+    }
+
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "/", 1);
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_bname, strlen(ce->cef_bname));
+    return bufsz;
 }
 
 size_t
-del_entry_fn (void *data, char *buffer, gf_boolean_t encode)
+del_entry_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char    *tmpbuf = NULL;
-        size_t  bufsz  = 0;
-        struct changelog_entry_fields *ce = NULL;
-
-        ce = (struct changelog_entry_fields *) data;
-
-        if (encode) {
-                tmpbuf = uuid_utoa (ce->cef_uuid);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
-        } else {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                                       ce->cef_uuid, sizeof (uuid_t));
-        }
-
-        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
-        CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                               ce->cef_bname, strlen (ce->cef_bname));
-        CHANGELOG_FILL_BUFFER (buffer, bufsz, "\0", 1);
-
-        if (ce->cef_path[0] == '\0') {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, "\0", 1);
-        } else {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                                       ce->cef_path, strlen (ce->cef_path));
-        }
-
-        return bufsz;
+    char *tmpbuf = NULL;
+    size_t bufsz = 0;
+    struct changelog_entry_fields *ce = NULL;
+
+    ce = (struct changelog_entry_fields *)data;
+
+    if (encode) {
+        tmpbuf = uuid_utoa(ce->cef_uuid);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, tmpbuf, strlen(tmpbuf));
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_uuid, sizeof(uuid_t));
+    }
+
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "/", 1);
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_bname, strlen(ce->cef_bname));
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "\0", 1);
+
+    if (ce->cef_path[0] == '\0') {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, "\0", 1);
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_path,
+                              strlen(ce->cef_path));
+    }
+
+    return bufsz;
 }
 
 size_t
-fop_fn (void *data, char *buffer, gf_boolean_t encode)
+fop_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char buf[10]          = {0,};
-        size_t         bufsz = 0;
-        glusterfs_fop_t fop   = 0;
+    char buf[10] = {
+        0,
+    };
+    size_t bufsz = 0;
+    glusterfs_fop_t fop = 0;
 
-        fop = *(glusterfs_fop_t *) data;
+    fop = *(glusterfs_fop_t *)data;
 
-        if (encode) {
-                (void) snprintf (buf, sizeof (buf), "%d", fop);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
-        } else
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, &fop, sizeof (fop));
+    if (encode) {
+        (void)snprintf(buf, sizeof(buf), "%d", fop);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, buf, strlen(buf));
+    } else
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, &fop, sizeof(fop));
 
-        return bufsz;
+    return bufsz;
 }
 
 size_t
-number_fn (void *data, char *buffer, gf_boolean_t encode)
+number_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        size_t       bufsz = 0;
-        unsigned int nr    = 0;
-        char buf[20]       = {0,};
+    size_t bufsz = 0;
+    unsigned int nr = 0;
+    char buf[20] = {
+        0,
+    };
 
-        nr = *(unsigned int *) data;
+    nr = *(unsigned int *)data;
 
-        if (encode) {
-                (void) snprintf (buf, sizeof (buf), "%u", nr);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
-        } else
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, &nr, sizeof (unsigned int));
+    if (encode) {
+        (void)snprintf(buf, sizeof(buf), "%u", nr);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, buf, strlen(buf));
+    } else
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, &nr, sizeof(unsigned int));
 
-        return bufsz;
+    return bufsz;
 }
 
 void
-entry_free_fn (void *data)
+entry_free_fn(void *data)
 {
-        changelog_opt_t *co = data;
+    changelog_opt_t *co = data;
 
-        if (!co)
-                return;
+    if (!co)
+        return;
 
-        GF_FREE (co->co_entry.cef_bname);
+    GF_FREE(co->co_entry.cef_bname);
 }
 
 void
-del_entry_free_fn (void *data)
+del_entry_free_fn(void *data)
 {
-        changelog_opt_t *co = data;
+    changelog_opt_t *co = data;
 
-        if (!co)
-                return;
+    if (!co)
+        return;
 
-        GF_FREE (co->co_entry.cef_bname);
-        GF_FREE (co->co_entry.cef_path);
+    GF_FREE(co->co_entry.cef_bname);
+    GF_FREE(co->co_entry.cef_path);
 }
 
 /**
@@ -129,108 +129,104 @@ del_entry_free_fn (void *data)
  */
 
 static void
-changelog_encode_write_xtra (changelog_log_data_t *cld,
-                             char *buffer, size_t *off, gf_boolean_t encode)
+changelog_encode_write_xtra(changelog_log_data_t *cld, char *buffer,
+                            size_t *off, gf_boolean_t encode)
 {
-        int              i      = 0;
-        size_t           offset = 0;
-        void            *data   = NULL;
-        changelog_opt_t *co     = NULL;
-
-        offset = *off;
-
-        co = (changelog_opt_t *) cld->cld_ptr;
-
-        for (; i < cld->cld_xtra_records; i++, co++) {
-                CHANGELOG_FILL_BUFFER (buffer, offset, "\0", 1);
-
-                switch (co->co_type) {
-                case CHANGELOG_OPT_REC_FOP:
-                        data = &co->co_fop;
-                        break;
-                case CHANGELOG_OPT_REC_ENTRY:
-                        data = &co->co_entry;
-                        break;
-                case CHANGELOG_OPT_REC_UINT32:
-                        data = &co->co_uint32;
-                        break;
-                }
-
-                if (co->co_convert)
-                        offset += co->co_convert (data,
-                                                  buffer + offset, encode);
-                else /* no coversion: write it out as it is */
-                        CHANGELOG_FILL_BUFFER (buffer, offset,
-                                               data, co->co_len);
+    int i = 0;
+    size_t offset = 0;
+    void *data = NULL;
+    changelog_opt_t *co = NULL;
+
+    offset = *off;
+
+    co = (changelog_opt_t *)cld->cld_ptr;
+
+    for (; i < cld->cld_xtra_records; i++, co++) {
+        CHANGELOG_FILL_BUFFER(buffer, offset, "\0", 1);
+
+        switch (co->co_type) {
+            case CHANGELOG_OPT_REC_FOP:
+                data = &co->co_fop;
+                break;
+            case CHANGELOG_OPT_REC_ENTRY:
+                data = &co->co_entry;
+                break;
+            case CHANGELOG_OPT_REC_UINT32:
+                data = &co->co_uint32;
+                break;
         }
 
-        *off = offset;
+        if (co->co_convert)
+            offset += co->co_convert(data, buffer + offset, encode);
+        else /* no coversion: write it out as it is */
+            CHANGELOG_FILL_BUFFER(buffer, offset, data, co->co_len);
+    }
+
+    *off = offset;
 }
 
 int
-changelog_encode_ascii (xlator_t *this, changelog_log_data_t *cld)
+changelog_encode_ascii(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off      = 0;
-        size_t            gfid_len = 0;
-        char             *gfid_str = NULL;
-        char             *buffer   = NULL;
-        changelog_priv_t *priv     = NULL;
+    size_t off = 0;
+    size_t gfid_len = 0;
+    char *gfid_str = NULL;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gfid_str = uuid_utoa (cld->cld_gfid);
-        gfid_len = strlen (gfid_str);
+    gfid_str = uuid_utoa(cld->cld_gfid);
+    gfid_len = strlen(gfid_str);
 
-        /* extra bytes for decorations */
-        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_ASCII (priv, buffer,
-                               off, gfid_str, gfid_len, cld);
+    /* extra bytes for decorations */
+    buffer = alloca(gfid_len + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_ASCII(priv, buffer, off, gfid_str, gfid_len, cld);
 
-        if (cld->cld_xtra_records)
-                changelog_encode_write_xtra (cld, buffer, &off, _gf_true);
+    if (cld->cld_xtra_records)
+        changelog_encode_write_xtra(cld, buffer, &off, _gf_true);
 
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
 
-        return changelog_write_change (priv, buffer, off);
+    return changelog_write_change(priv, buffer, off);
 }
 
 int
-changelog_encode_binary (xlator_t *this, changelog_log_data_t *cld)
+changelog_encode_binary(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off    = 0;
-        char             *buffer = NULL;
-        changelog_priv_t *priv   = NULL;
+    size_t off = 0;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* extra bytes for decorations */
-        buffer = alloca (sizeof (uuid_t) + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_BINARY (priv, buffer, off, cld->cld_gfid, cld);
+    /* extra bytes for decorations */
+    buffer = alloca(sizeof(uuid_t) + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_BINARY(priv, buffer, off, cld->cld_gfid, cld);
 
-        if (cld->cld_xtra_records)
-                changelog_encode_write_xtra (cld, buffer, &off, _gf_false);
+    if (cld->cld_xtra_records)
+        changelog_encode_write_xtra(cld, buffer, &off, _gf_false);
 
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
 
-        return changelog_write_change (priv, buffer, off);
+    return changelog_write_change(priv, buffer, off);
 }
 
-static struct changelog_encoder
-cb_encoder[] = {
-        [CHANGELOG_ENCODE_BINARY] =
+static struct changelog_encoder cb_encoder[] = {
+    [CHANGELOG_ENCODE_BINARY] =
         {
-                .encoder = CHANGELOG_ENCODE_BINARY,
-                .encode = changelog_encode_binary,
+            .encoder = CHANGELOG_ENCODE_BINARY,
+            .encode = changelog_encode_binary,
         },
-        [CHANGELOG_ENCODE_ASCII] =
+    [CHANGELOG_ENCODE_ASCII] =
         {
-                .encoder = CHANGELOG_ENCODE_ASCII,
-                .encode = changelog_encode_ascii,
+            .encoder = CHANGELOG_ENCODE_ASCII,
+            .encode = changelog_encode_ascii,
         },
 };
 
 void
 changelog_encode_change(changelog_priv_t *priv)
 {
-        priv->ce = &cb_encoder[priv->encode_mode];
+    priv->ce = &cb_encoder[priv->encode_mode];
 }
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
index d6a50cc9ef7..26252696d56 100644
--- a/xlators/features/changelog/src/changelog-encoders.h
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -11,41 +11,39 @@
 #ifndef _CHANGELOG_ENCODERS_H
 #define _CHANGELOG_ENCODERS_H
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "changelog-helpers.h"
 
-#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld) do { \
-                CHANGELOG_FILL_BUFFER (buffer, off,                     \
-                                       priv->maps[cld->cld_type], 1);   \
-                CHANGELOG_FILL_BUFFER (buffer,                          \
-                                       off, gfid, gfid_len);            \
-        } while (0)
+#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld)             \
+    do {                                                                       \
+        CHANGELOG_FILL_BUFFER(buffer, off, priv->maps[cld->cld_type], 1);      \
+        CHANGELOG_FILL_BUFFER(buffer, off, gfid, gfid_len);                    \
+    } while (0)
 
-#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld) do {          \
-                CHANGELOG_FILL_BUFFER (buffer, off,                     \
-                                       priv->maps[cld->cld_type], 1);   \
-                CHANGELOG_FILL_BUFFER (buffer,                          \
-                                       off, gfid, sizeof (uuid_t));     \
-        } while (0)
+#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld)                      \
+    do {                                                                       \
+        CHANGELOG_FILL_BUFFER(buffer, off, priv->maps[cld->cld_type], 1);      \
+        CHANGELOG_FILL_BUFFER(buffer, off, gfid, sizeof(uuid_t));              \
+    } while (0)
 
 size_t
-entry_fn (void *data, char *buffer, gf_boolean_t encode);
+entry_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-del_entry_fn (void *data, char *buffer, gf_boolean_t encode);
+del_entry_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-fop_fn (void *data, char *buffer, gf_boolean_t encode);
+fop_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-number_fn (void *data, char *buffer, gf_boolean_t encode);
+number_fn(void *data, char *buffer, gf_boolean_t encode);
 void
-entry_free_fn (void *data);
+entry_free_fn(void *data);
 void
-del_entry_free_fn (void *data);
+del_entry_free_fn(void *data);
 int
-changelog_encode_binary (xlator_t *, changelog_log_data_t *);
+changelog_encode_binary(xlator_t *, changelog_log_data_t *);
 int
-changelog_encode_ascii (xlator_t *, changelog_log_data_t *);
+changelog_encode_ascii(xlator_t *, changelog_log_data_t *);
 void
 changelog_encode_change(changelog_priv_t *);
 
diff --git a/xlators/features/changelog/src/changelog-ev-handle.c b/xlators/features/changelog/src/changelog-ev-handle.c
index 79652a969bd..aa94459de5a 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.c
+++ b/xlators/features/changelog/src/changelog-ev-handle.c
@@ -14,19 +14,19 @@
 
 struct rpc_clnt_program changelog_ev_program;
 
-#define NR_IOVEC  (MAX_IOVEC - 3)
+#define NR_IOVEC (MAX_IOVEC - 3)
 struct ev_rpc_vec {
-        int count;
-        struct iovec vector[NR_IOVEC];
+    int count;
+    struct iovec vector[NR_IOVEC];
 
-        /* sequence number */
-        unsigned long seq;
+    /* sequence number */
+    unsigned long seq;
 };
 
 struct ev_rpc {
-        rbuf_list_t     *rlist;
-        struct rpc_clnt *rpc;
-        struct ev_rpc_vec vec;
+    rbuf_list_t *rlist;
+    struct rpc_clnt *rpc;
+    struct ev_rpc_vec vec;
 };
 
 /**
@@ -35,205 +35,229 @@ struct ev_rpc {
  * intelligence can be built into the server.
  */
 int
-changelog_event_dispatch_cbk (struct rpc_req *req,
-                              struct iovec *iov, int count, void *myframe)
+changelog_event_dispatch_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        return 0;
+    return 0;
 }
 
 /* dispatcher RPC */
 int
-changelog_dispatch_vec (call_frame_t *frame, xlator_t *this,
-                        struct rpc_clnt *rpc, struct ev_rpc_vec *vec)
+changelog_dispatch_vec(call_frame_t *frame, xlator_t *this,
+                       struct rpc_clnt *rpc, struct ev_rpc_vec *vec)
 {
-         struct timeval      tv  = {0,};
-         changelog_event_req req = {0,};
-
-         (void) gettimeofday (&tv, NULL);
-
-         /**
-          * Event dispatch RPC header contains a sequence number for each
-          * dispatch. This allows the reciever to order the request before
-          * processing.
-          */
-         req.seq     = vec->seq;
-         req.tv_sec  = tv.tv_sec;
-         req.tv_usec = tv.tv_usec;
-
-         return changelog_rpc_sumbit_req (rpc, (void *)&req,
-                                          frame, &changelog_ev_program,
-                                          CHANGELOG_REV_PROC_EVENT,
-                                          vec->vector, vec->count, NULL,
-                                          this, changelog_event_dispatch_cbk,
-                                          (xdrproc_t) xdr_changelog_event_req);
- }
-
- int
- changelog_event_dispatch_rpc (call_frame_t *frame, xlator_t *this, void *data)
- {
-         int                idx      = 0;
-         int                count    = 0;
-         int                ret      = 0;
-         unsigned long      range    = 0;
-         unsigned long      sequence = 0;
-         rbuf_iovec_t      *rvec     = NULL;
-         struct ev_rpc     *erpc     = NULL;
-         struct rlist_iter  riter    = {{0,},};
-
-         /* dispatch NR_IOVEC IO vectors at a time. */
-
-         erpc = data;
-         RLIST_GET_SEQ (erpc->rlist, sequence, range);
-
-         rlist_iter_init (&riter, erpc->rlist);
-
-         rvec_for_each_entry (rvec, &riter) {
-                 idx = count % NR_IOVEC;
-                 if (++count == NR_IOVEC) {
-                         erpc->vec.vector[idx] = rvec->iov;
-                         erpc->vec.seq = sequence++;
-                         erpc->vec.count = NR_IOVEC;
-
-                         ret = changelog_dispatch_vec (frame, this,
-                                                       erpc->rpc, &erpc->vec);
-                         if (ret)
-                                 break;
-                         count = 0;
-                         continue;
-                 }
-
-                 erpc->vec.vector[idx] = rvec->iov;
-         }
-
-         if (ret)
-                 goto error_return;
-
-         idx = count % NR_IOVEC;
-         if (idx) {
-                 erpc->vec.seq = sequence;
-                 erpc->vec.count = idx;
-
-                 ret = changelog_dispatch_vec (frame, this,
-                                               erpc->rpc, &erpc->vec);
-         }
-
- error_return:
-         return ret;
+    struct timeval tv = {
+        0,
+    };
+    changelog_event_req req = {
+        0,
+    };
+
+    (void)gettimeofday(&tv, NULL);
+
+    /**
+     * Event dispatch RPC header contains a sequence number for each
+     * dispatch. This allows the receiver to order the request before
+     * processing.
+     */
+    req.seq = vec->seq;
+    req.tv_sec = tv.tv_sec;
+    req.tv_usec = tv.tv_usec;
+
+    return changelog_rpc_sumbit_req(
+        rpc, (void *)&req, frame, &changelog_ev_program,
+        CHANGELOG_REV_PROC_EVENT, vec->vector, vec->count, NULL, this,
+        changelog_event_dispatch_cbk, (xdrproc_t)xdr_changelog_event_req);
 }
 
 int
-changelog_rpc_notify (struct rpc_clnt *rpc,
-                      void *mydata, rpc_clnt_event_t event, void *data)
+changelog_event_dispatch_rpc(call_frame_t *frame, xlator_t *this, void *data)
 {
-        xlator_t                *this      = NULL;
-        changelog_rpc_clnt_t    *crpc      = NULL;
-        changelog_clnt_t        *c_clnt    = NULL;
-        changelog_priv_t        *priv      = NULL;
-        changelog_ev_selector_t *selection = NULL;
+    int idx = 0;
+    int count = 0;
+    int ret = 0;
+    unsigned long sequence = 0;
+    rbuf_iovec_t *rvec = NULL;
+    struct ev_rpc *erpc = NULL;
+    struct rlist_iter riter = {
+        {
+            0,
+        },
+    };
+
+    /* dispatch NR_IOVEC IO vectors at a time. */
+
+    erpc = data;
+    sequence = erpc->rlist->seq[0];
+
+    rlist_iter_init(&riter, erpc->rlist);
+
+    rvec_for_each_entry(rvec, &riter)
+    {
+        idx = count % NR_IOVEC;
+        if (++count == NR_IOVEC) {
+            erpc->vec.vector[idx] = rvec->iov;
+            erpc->vec.seq = sequence++;
+            erpc->vec.count = NR_IOVEC;
 
-        crpc = mydata;
-        this = crpc->this;
-        c_clnt = crpc->c_clnt;
+            ret = changelog_dispatch_vec(frame, this, erpc->rpc, &erpc->vec);
+            if (ret)
+                break;
+            count = 0;
+            continue;
+        }
+
+        erpc->vec.vector[idx] = rvec->iov;
+    }
 
-        priv = this->private;
+    if (ret)
+        goto error_return;
 
-        switch (event) {
+    idx = count % NR_IOVEC;
+    if (idx) {
+        erpc->vec.seq = sequence;
+        erpc->vec.count = idx;
+
+        ret = changelog_dispatch_vec(frame, this, erpc->rpc, &erpc->vec);
+    }
+
+error_return:
+    return ret;
+}
+
+int
+changelog_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                     void *data)
+{
+    xlator_t *this = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_ev_selector_t *selection = NULL;
+    uint64_t clntcnt = 0;
+    uint64_t xprtcnt = 0;
+
+    crpc = mydata;
+    this = crpc->this;
+    c_clnt = crpc->c_clnt;
+
+    priv = this->private;
+
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&rpc->conn);
-                selection = &priv->ev_selection;
+            selection = &priv->ev_selection;
+            GF_ATOMIC_INC(priv->clntcnt);
 
-                LOCK (&c_clnt->wait_lock);
+            LOCK(&c_clnt->wait_lock);
+            {
+                LOCK(&c_clnt->active_lock);
                 {
-                        LOCK (&c_clnt->active_lock);
-                        {
-                                changelog_select_event (this, selection,
-                                                        crpc->filter);
-                                list_move_tail (&crpc->list, &c_clnt->active);
-                        }
-                        UNLOCK (&c_clnt->active_lock);
+                    changelog_select_event(this, selection, crpc->filter);
+                    list_move_tail(&crpc->list, &c_clnt->active);
                 }
-                UNLOCK (&c_clnt->wait_lock);
+                UNLOCK(&c_clnt->active_lock);
+            }
+            UNLOCK(&c_clnt->wait_lock);
 
-                break;
+            break;
         case RPC_CLNT_DISCONNECT:
-                rpc_clnt_disable (crpc->rpc);
-                selection = &priv->ev_selection;
+            rpc_clnt_disable(crpc->rpc);
 
-                LOCK (&crpc->lock);
-                {
-                        changelog_deselect_event (this, selection,
-                                                  crpc->filter);
-                        changelog_set_disconnect_flag (crpc, _gf_true);
-                }
-                UNLOCK (&crpc->lock);
+            /* rpc_clnt_disable doesn't unref the rpc. It just marks
+             * the rpc as disabled and cancels reconnection timer.
+             * Hence unref the rpc object to free it.
+             */
+            rpc_clnt_unref(crpc->rpc);
 
-                break;
+            if (priv)
+                selection = &priv->ev_selection;
+
+            LOCK(&crpc->lock);
+            {
+                if (selection)
+                    changelog_deselect_event(this, selection, crpc->filter);
+                changelog_set_disconnect_flag(crpc, _gf_true);
+            }
+            UNLOCK(&crpc->lock);
+            LOCK(&c_clnt->active_lock);
+            {
+                list_del_init(&crpc->list);
+            }
+            UNLOCK(&c_clnt->active_lock);
+
+            break;
         case RPC_CLNT_MSG:
         case RPC_CLNT_DESTROY:
-                break;
-        }
-
-        return 0;
+            /* Free up mydata */
+            changelog_rpc_clnt_unref(crpc);
+            clntcnt = GF_ATOMIC_DEC(priv->clntcnt);
+            xprtcnt = GF_ATOMIC_GET(priv->xprtcnt);
+            if (this->cleanup_starting) {
+                if (!clntcnt && !xprtcnt)
+                    changelog_process_cleanup_event(this);
+            }
+            break;
+        case RPC_CLNT_PING:
+            break;
+    }
+
+    return 0;
 }
 
 void *
-changelog_ev_connector (void *data)
+changelog_ev_connector(void *data)
 {
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        changelog_rpc_clnt_t *crpc   = NULL;
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
 
-        c_clnt = data;
-        this = c_clnt->this;
+    c_clnt = data;
+    this = c_clnt->this;
 
-        while (1) {
-                pthread_mutex_lock (&c_clnt->pending_lock);
-                {
-                        while (list_empty (&c_clnt->pending))
-                                pthread_cond_wait (&c_clnt->pending_cond,
-                                                   &c_clnt->pending_lock);
-                        crpc = list_first_entry (&c_clnt->pending,
-                                                 changelog_rpc_clnt_t, list);
-                        crpc->rpc =
-                                changelog_rpc_client_init (this, crpc,
-                                                           crpc->sock,
-                                                           changelog_rpc_notify);
-                        if (!crpc->rpc) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_RPC_CONNECT_ERROR,
-                                        "failed to connect back.. <%s>",
-                                        crpc->sock);
-                                crpc->cleanup (crpc);
-                                goto mutex_unlock;
-                        }
-
-                        LOCK (&c_clnt->wait_lock);
-                        {
-                                list_move_tail (&crpc->list, &c_clnt->waitq);
-                        }
-                        UNLOCK (&c_clnt->wait_lock);
-                }
-        mutex_unlock:
-                pthread_mutex_unlock (&c_clnt->pending_lock);
+    while (1) {
+        pthread_mutex_lock(&c_clnt->pending_lock);
+        {
+            while (list_empty(&c_clnt->pending))
+                pthread_cond_wait(&c_clnt->pending_cond, &c_clnt->pending_lock);
+            crpc = list_first_entry(&c_clnt->pending, changelog_rpc_clnt_t,
+                                    list);
+            crpc->rpc = changelog_rpc_client_init(this, crpc, crpc->sock,
+                                                  changelog_rpc_notify);
+            if (!crpc->rpc) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_RPC_CONNECT_ERROR, "path=%s", crpc->sock,
+                        NULL);
+                crpc->cleanup(crpc);
+                goto mutex_unlock;
+            }
+
+            LOCK(&c_clnt->wait_lock);
+            {
+                list_move_tail(&crpc->list, &c_clnt->waitq);
+            }
+            UNLOCK(&c_clnt->wait_lock);
         }
+    mutex_unlock:
+        pthread_mutex_unlock(&c_clnt->pending_lock);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 void
-changelog_ev_cleanup_connections (xlator_t *this, changelog_clnt_t *c_clnt)
+changelog_ev_cleanup_connections(xlator_t *this, changelog_clnt_t *c_clnt)
 {
-        int ret = 0;
-        changelog_rpc_clnt_t *crpc = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
 
-        /* cleanup active connections */
-        LOCK (&c_clnt->active_lock);
+    /* cleanup active connections */
+    LOCK(&c_clnt->active_lock);
+    {
+        list_for_each_entry(crpc, &c_clnt->active, list)
         {
-                list_for_each_entry (crpc, &c_clnt->active, list) {
-                        rpc_clnt_disable (crpc->rpc);
-                }
+            rpc_clnt_disable(crpc->rpc);
         }
-        UNLOCK (&c_clnt->active_lock);
+    }
+    UNLOCK(&c_clnt->active_lock);
 }
 
 /**
@@ -244,143 +268,145 @@ changelog_ev_cleanup_connections (xlator_t *this, changelog_clnt_t *c_clnt)
  */
 
 static changelog_rpc_clnt_t *
-get_client (changelog_clnt_t *c_clnt, struct list_head **next)
+get_client(changelog_clnt_t *c_clnt, struct list_head **next)
 {
-        changelog_rpc_clnt_t *crpc = NULL;
-
-        LOCK (&c_clnt->active_lock);
-        {
-                if (*next == &c_clnt->active)
-                        goto unblock;
-                crpc = list_entry (*next, changelog_rpc_clnt_t, list);
-                changelog_rpc_clnt_ref (crpc);
-                *next = (*next)->next;
-        }
- unblock:
-        UNLOCK (&c_clnt->active_lock);
-
-        return crpc;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    LOCK(&c_clnt->active_lock);
+    {
+        if (*next == &c_clnt->active)
+            goto unblock;
+        crpc = list_entry(*next, changelog_rpc_clnt_t, list);
+        /* ref rpc as DISCONNECT might unref the rpc asynchronously */
+        changelog_rpc_clnt_ref(crpc);
+        rpc_clnt_ref(crpc->rpc);
+        *next = (*next)->next;
+    }
+unblock:
+    UNLOCK(&c_clnt->active_lock);
+
+    return crpc;
 }
 
 static void
-put_client (changelog_clnt_t *c_clnt, changelog_rpc_clnt_t *crpc)
+put_client(changelog_clnt_t *c_clnt, changelog_rpc_clnt_t *crpc)
 {
-        LOCK (&c_clnt->active_lock);
-        {
-                changelog_rpc_clnt_unref (crpc);
-        }
-        UNLOCK (&c_clnt->active_lock);
+    LOCK(&c_clnt->active_lock);
+    {
+        rpc_clnt_unref(crpc->rpc);
+        changelog_rpc_clnt_unref(crpc);
+    }
+    UNLOCK(&c_clnt->active_lock);
 }
 
 void
-_dispatcher (rbuf_list_t *rlist, void *arg)
+_dispatcher(rbuf_list_t *rlist, void *arg)
 {
-        int                   ret    = 0;
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        changelog_rpc_clnt_t *crpc   = NULL;
-        changelog_rpc_clnt_t *tmp    = NULL;
-        struct ev_rpc         erpc   = {0,};
-        struct list_head     *next   = NULL;
-
-        c_clnt = arg;
-        this = c_clnt->this;
-
-        erpc.rlist = rlist;
-        next = c_clnt->active.next;
-
-        while (1) {
-                crpc = get_client (c_clnt, &next);
-                if (!crpc)
-                        break;
-                erpc.rpc = crpc->rpc;
-                ret = changelog_invoke_rpc (this, crpc->rpc,
-                                            &changelog_ev_program,
-                                            CHANGELOG_REV_PROC_EVENT, &erpc);
-                put_client (c_clnt, crpc);
-        }
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+    struct ev_rpc erpc = {
+        0,
+    };
+    struct list_head *next = NULL;
+
+    c_clnt = arg;
+    this = c_clnt->this;
+
+    erpc.rlist = rlist;
+    next = c_clnt->active.next;
+
+    while (1) {
+        crpc = get_client(c_clnt, &next);
+        if (!crpc)
+            break;
+        erpc.rpc = crpc->rpc;
+        (void)changelog_invoke_rpc(this, crpc->rpc, &changelog_ev_program,
+                                   CHANGELOG_REV_PROC_EVENT, &erpc);
+        put_client(c_clnt, crpc);
+    }
 }
 
 /** this is called under rotbuff's lock */
 void
-sequencer (rbuf_list_t *rlist, void *mydata)
+sequencer(rbuf_list_t *rlist, void *mydata)
 {
-        unsigned long     range  = 0;
-        changelog_clnt_t *c_clnt = 0;
+    unsigned long range = 0;
+    changelog_clnt_t *c_clnt = 0;
 
-        c_clnt = mydata;
+    c_clnt = mydata;
 
-        range = (RLIST_ENTRY_COUNT (rlist)) / NR_IOVEC;
-        if ((RLIST_ENTRY_COUNT (rlist)) % NR_IOVEC)
-                range++;
-        RLIST_STORE_SEQ (rlist, c_clnt->sequence, range);
+    range = (RLIST_ENTRY_COUNT(rlist)) / NR_IOVEC;
+    if ((RLIST_ENTRY_COUNT(rlist)) % NR_IOVEC)
+        range++;
+    RLIST_STORE_SEQ(rlist, c_clnt->sequence, range);
 
-        c_clnt->sequence += range;
+    c_clnt->sequence += range;
 }
 
 void *
-changelog_ev_dispatch (void *data)
+changelog_ev_dispatch(void *data)
 {
-        int                   ret    = 0;
-        void                 *opaque = NULL;
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        struct timeval        tv     = {0,};
-
-        c_clnt = data;
-        this = c_clnt->this;
-
-        while (1) {
-                /* TODO: change this to be pthread cond based.. later */
-                tv.tv_sec = 1;
-                tv.tv_usec = 0;
-                select (0, NULL, NULL, NULL, &tv);
-
-                ret = rbuf_get_buffer (c_clnt->rbuf,
-                                       &opaque, sequencer, c_clnt);
-                if (ret != RBUF_CONSUMABLE) {
-                        if (ret != RBUF_EMPTY)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
-                                        "Failed to get buffer for RPC dispatch "
-                                        "[rbuf retval: %d]", ret);
-                        continue;
-                }
-
-                ret = rbuf_wait_for_completion (c_clnt->rbuf,
-                                                opaque, _dispatcher, c_clnt);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_PUT_BUFFER_FAILED,
-                                "failed to put buffer after consumption");
+    int ret = 0;
+    void *opaque = NULL;
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    struct timeval tv = {
+        0,
+    };
+
+    c_clnt = data;
+    this = c_clnt->this;
+
+    while (1) {
+        /* TODO: change this to be pthread cond based.. later */
+
+        tv.tv_sec = 1;
+        tv.tv_usec = 0;
+        select(0, NULL, NULL, NULL, &tv);
+
+        ret = rbuf_get_buffer(c_clnt->rbuf, &opaque, sequencer, c_clnt);
+        if (ret != RBUF_CONSUMABLE) {
+            if (ret != RBUF_EMPTY)
+                gf_smsg(this->name, GF_LOG_WARNING, 0,
+                        CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
+                        "Failed to get buffer for RPC dispatch",
+                        "rbuf_retval=%d", ret, NULL);
+            continue;
         }
 
-        return NULL;
+        ret = rbuf_wait_for_completion(c_clnt->rbuf, opaque, _dispatcher,
+                                       c_clnt);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    CHANGELOG_MSG_PUT_BUFFER_FAILED, NULL);
+    }
+
+    return NULL;
 }
 
 void
-changelog_ev_queue_connection (changelog_clnt_t *c_clnt,
-                               changelog_rpc_clnt_t *crpc)
+changelog_ev_queue_connection(changelog_clnt_t *c_clnt,
+                              changelog_rpc_clnt_t *crpc)
 {
-        pthread_mutex_lock (&c_clnt->pending_lock);
-        {
-                list_add_tail (&crpc->list, &c_clnt->pending);
-                pthread_cond_signal (&c_clnt->pending_cond);
-        }
-        pthread_mutex_unlock (&c_clnt->pending_lock);
+    pthread_mutex_lock(&c_clnt->pending_lock);
+    {
+        list_add_tail(&crpc->list, &c_clnt->pending);
+        pthread_cond_signal(&c_clnt->pending_cond);
+    }
+    pthread_mutex_unlock(&c_clnt->pending_lock);
 }
 
 struct rpc_clnt_procedure changelog_ev_procs[CHANGELOG_REV_PROC_MAX] = {
-        [CHANGELOG_REV_PROC_NULL]  = {"NULL", NULL},
-        [CHANGELOG_REV_PROC_EVENT] = {
-                "EVENT DISPATCH", changelog_event_dispatch_rpc
-        },
+    [CHANGELOG_REV_PROC_NULL] = {"NULL", NULL},
+    [CHANGELOG_REV_PROC_EVENT] = {"EVENT DISPATCH",
+                                  changelog_event_dispatch_rpc},
 };
 
 struct rpc_clnt_program changelog_ev_program = {
-        .progname  = "CHANGELOG EVENT DISPATCHER",
-        .prognum   = CHANGELOG_REV_RPC_PROCNUM,
-        .progver   = CHANGELOG_REV_RPC_PROCVER,
-        .numproc   = CHANGELOG_REV_PROC_MAX,
-        .proctable = changelog_ev_procs,
+    .progname = "CHANGELOG EVENT DISPATCHER",
+    .prognum = CHANGELOG_REV_RPC_PROCNUM,
+    .progver = CHANGELOG_REV_RPC_PROCVER,
+    .numproc = CHANGELOG_REV_PROC_MAX,
+    .proctable = changelog_ev_procs,
 };
diff --git a/xlators/features/changelog/src/changelog-ev-handle.h b/xlators/features/changelog/src/changelog-ev-handle.h
index eef0492a9ee..cc1af58a276 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.h
+++ b/xlators/features/changelog/src/changelog-ev-handle.h
@@ -11,74 +11,67 @@
 #ifndef __CHANGELOG_EV_HANDLE_H
 #define __CHANGELOG_EV_HANDLE_H
 
-#include "list.h"
-#include "xlator.h"
+#include <glusterfs/list.h>
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 
-#include "rot-buffs.h"
+#include <glusterfs/rot-buffs.h>
 
 struct changelog_clnt;
 
 typedef struct changelog_rpc_clnt {
-        xlator_t *this;
+    xlator_t *this;
 
-        gf_lock_t lock;
+    gf_lock_t lock;
 
-        unsigned long ref;
-        gf_boolean_t  disconnected;
+    gf_atomic_t ref;
+    gf_boolean_t disconnected;
 
-        unsigned int filter;
-        char sock[UNIX_PATH_MAX];
+    unsigned int filter;
+    char sock[UNIX_PATH_MAX];
 
-        struct changelog_clnt *c_clnt;   /* back pointer to list holder */
+    struct changelog_clnt *c_clnt; /* back pointer to list holder */
 
-        struct rpc_clnt *rpc;            /* RPC client endpoint */
+    struct rpc_clnt *rpc; /* RPC client endpoint */
 
-        struct list_head list;           /* ->pending, ->waitq, ->active */
+    struct list_head list; /* ->pending, ->waitq, ->active */
 
-        void (*cleanup)
-        (struct changelog_rpc_clnt *);   /* cleanup handler */
+    void (*cleanup)(struct changelog_rpc_clnt *); /* cleanup handler */
 } changelog_rpc_clnt_t;
 
 static inline void
-changelog_rpc_clnt_ref (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_ref(changelog_rpc_clnt_t *crpc)
 {
-        LOCK (&crpc->lock);
-        {
-                ++crpc->ref;
-        }
-        UNLOCK (&crpc->lock);
+    GF_ATOMIC_INC(crpc->ref);
 }
 
 static inline void
-changelog_set_disconnect_flag (changelog_rpc_clnt_t *crpc, gf_boolean_t flag)
+changelog_set_disconnect_flag(changelog_rpc_clnt_t *crpc, gf_boolean_t flag)
 {
-        crpc->disconnected = flag;
+    crpc->disconnected = flag;
 }
 
 static inline int
-changelog_rpc_clnt_is_disconnected (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_is_disconnected(changelog_rpc_clnt_t *crpc)
 {
-        return (crpc->disconnected == _gf_true);
+    return (crpc->disconnected == _gf_true);
 }
 
 static inline void
-changelog_rpc_clnt_unref (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_unref(changelog_rpc_clnt_t *crpc)
 {
-        gf_boolean_t gone = _gf_false;
-
-        LOCK (&crpc->lock);
-        {
-                if (!(--crpc->ref)
-                    && changelog_rpc_clnt_is_disconnected (crpc)) {
-                        list_del (&crpc->list);
-                        gone = _gf_true;
-                }
-        }
-        UNLOCK (&crpc->lock);
-
-        if (gone)
-                crpc->cleanup (crpc);
+    gf_boolean_t gone = _gf_false;
+    uint64_t ref = 0;
+
+    ref = GF_ATOMIC_DEC(crpc->ref);
+
+    if (!ref && changelog_rpc_clnt_is_disconnected(crpc)) {
+        list_del(&crpc->list);
+        gone = _gf_true;
+    }
+
+    if (gone)
+        crpc->cleanup(crpc);
 }
 
 /**
@@ -106,35 +99,38 @@ changelog_rpc_clnt_unref (changelog_rpc_clnt_t *crpc)
  */
 
 typedef struct changelog_clnt {
-        xlator_t *this;
+    xlator_t *this;
 
-        /* pending connections */
-        pthread_mutex_t pending_lock;
-        pthread_cond_t pending_cond;
-        struct list_head pending;
+    /* pending connections */
+    pthread_mutex_t pending_lock;
+    pthread_cond_t pending_cond;
+    struct list_head pending;
 
-        /* current active connections */
-        gf_lock_t active_lock;
-        struct list_head active;
+    /* current active connections */
+    gf_lock_t active_lock;
+    struct list_head active;
 
-        gf_lock_t wait_lock;
-        struct list_head waitq;
+    gf_lock_t wait_lock;
+    struct list_head waitq;
 
-        /* consumer part of rot-buffs */
-        rbuf_t *rbuf;
-        unsigned long sequence;
+    /* consumer part of rot-buffs */
+    rbuf_t *rbuf;
+    unsigned long sequence;
 } changelog_clnt_t;
 
-void *changelog_ev_connector (void *);
+void *
+changelog_ev_connector(void *);
 
-void *changelog_ev_dispatch (void *);
+void *
+changelog_ev_dispatch(void *);
 
 /* APIs */
 void
-changelog_ev_queue_connection (changelog_clnt_t *, changelog_rpc_clnt_t *);
+changelog_ev_queue_connection(changelog_clnt_t *, changelog_rpc_clnt_t *);
 
 void
-changelog_ev_cleanup_connections (xlator_t *, changelog_clnt_t *);
+changelog_ev_cleanup_connections(xlator_t *, changelog_clnt_t *);
 
+void
+changelog_process_cleanup_event(xlator_t *);
 #endif
-
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
index 0cb68587e57..e561997d858 100644
--- a/xlators/features/changelog/src/changelog-helpers.c
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -8,11 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
-#include "iobuf.h"
-#include "syscall.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/iobuf.h>
+#include <glusterfs/syscall.h>
 
 #include "changelog-helpers.h"
 #include "changelog-encoders.h"
@@ -22,312 +22,281 @@
 #include "changelog-encoders.h"
 #include "changelog-rpc-common.h"
 #include <pthread.h>
+#include <time.h>
 
 static void
-changelog_cleanup_free_mutex (void *arg_mutex)
+changelog_cleanup_free_mutex(void *arg_mutex)
 {
-    pthread_mutex_t *p_mutex = (pthread_mutex_t*) arg_mutex;
+    pthread_mutex_t *p_mutex = (pthread_mutex_t *)arg_mutex;
 
     if (p_mutex)
-            pthread_mutex_unlock(p_mutex);
+        pthread_mutex_unlock(p_mutex);
 }
 
 int
-changelog_thread_cleanup (xlator_t *this, pthread_t thr_id)
+changelog_thread_cleanup(xlator_t *this, pthread_t thr_id)
 {
-        int   ret    = 0;
-        void *retval = NULL;
-
-        /* send a cancel request to the thread */
-        ret = pthread_cancel (thr_id);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
-                        "could not cancel thread");
-                goto out;
-        }
+    int ret = 0;
+    void *retval = NULL;
+
+    /* send a cancel request to the thread */
+    ret = pthread_cancel(thr_id);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_CANCEL_FAILED, NULL);
+        goto out;
+    }
+
+    ret = pthread_join(thr_id, &retval);
+    if ((ret != 0) || (retval != PTHREAD_CANCELED)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_CANCEL_FAILED, NULL);
+    }
 
-        ret = pthread_join (thr_id, &retval);
-        if ((ret != 0) || (retval != PTHREAD_CANCELED)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
-                        "cancel request not adhered as expected");
-        }
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void *
-changelog_get_usable_buffer (changelog_local_t *local)
+changelog_get_usable_buffer(changelog_local_t *local)
 {
-        changelog_log_data_t *cld = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        if (!local)
-                return NULL;
+    if (!local)
+        return NULL;
 
-        cld = &local->cld;
-        if (!cld->cld_iobuf)
-                return NULL;
+    cld = &local->cld;
+    if (!cld->cld_iobuf)
+        return NULL;
 
-        return cld->cld_iobuf->ptr;
+    return cld->cld_iobuf->ptr;
 }
 
 static int
-changelog_selector_index (unsigned int selector)
+changelog_selector_index(unsigned int selector)
 {
-        return (ffs (selector) - 1);
+    return (ffs(selector) - 1);
 }
 
 int
-changelog_ev_selected (xlator_t *this,
-                       changelog_ev_selector_t *selection,
-                       unsigned int selector)
+changelog_ev_selected(xlator_t *this, changelog_ev_selector_t *selection,
+                      unsigned int selector)
 {
-        int idx = 0;
-
-        idx = changelog_selector_index (selector);
-        gf_msg_debug (this->name, 0,
-                      "selector ref count for %d (idx: %d): %d",
-                      selector, idx, selection->ref[idx]);
-        /* this can be lockless */
-        return (idx < CHANGELOG_EV_SELECTION_RANGE
-                 && (selection->ref[idx] > 0));
+    int idx = 0;
+
+    idx = changelog_selector_index(selector);
+    gf_msg_debug(this->name, 0, "selector ref count for %d (idx: %d): %d",
+                 selector, idx, selection->ref[idx]);
+    /* this can be lockless */
+    return (idx < CHANGELOG_EV_SELECTION_RANGE && (selection->ref[idx] > 0));
 }
 
 void
-changelog_select_event (xlator_t *this,
-                        changelog_ev_selector_t *selection,
-                        unsigned int selector)
+changelog_select_event(xlator_t *this, changelog_ev_selector_t *selection,
+                       unsigned int selector)
 {
-        int idx = 0;
-
-        LOCK (&selection->reflock);
-        {
-                while (selector) {
-                        idx = changelog_selector_index (selector);
-                        if (idx < CHANGELOG_EV_SELECTION_RANGE) {
-                                selection->ref[idx]++;
-                                gf_msg_debug (this->name, 0,
-                                              "selecting event %d", idx);
-                        }
-                        selector &= ~(1 << idx);
-                }
-        }
-        UNLOCK (&selection->reflock);
+    int idx = 0;
+
+    LOCK(&selection->reflock);
+    {
+        while (selector) {
+            idx = changelog_selector_index(selector);
+            if (idx < CHANGELOG_EV_SELECTION_RANGE) {
+                selection->ref[idx]++;
+                gf_msg_debug(this->name, 0, "selecting event %d", idx);
+            }
+            selector &= ~(1 << idx);
+        }
+    }
+    UNLOCK(&selection->reflock);
 }
 
 void
-changelog_deselect_event (xlator_t *this,
-                          changelog_ev_selector_t *selection,
-                          unsigned int selector)
+changelog_deselect_event(xlator_t *this, changelog_ev_selector_t *selection,
+                         unsigned int selector)
 {
-        int idx = 0;
-
-        LOCK (&selection->reflock);
-        {
-                while (selector) {
-                        idx = changelog_selector_index (selector);
-                        if (idx < CHANGELOG_EV_SELECTION_RANGE) {
-                                selection->ref[idx]--;
-                                gf_msg_debug (this->name, 0,
-                                              "de-selecting event %d", idx);
-                        }
-                        selector &= ~(1 << idx);
-                }
-        }
-        UNLOCK (&selection->reflock);
+    int idx = 0;
+
+    LOCK(&selection->reflock);
+    {
+        while (selector) {
+            idx = changelog_selector_index(selector);
+            if (idx < CHANGELOG_EV_SELECTION_RANGE) {
+                selection->ref[idx]--;
+                gf_msg_debug(this->name, 0, "de-selecting event %d", idx);
+            }
+            selector &= ~(1 << idx);
+        }
+    }
+    UNLOCK(&selection->reflock);
 }
 
 int
-changelog_init_event_selection (xlator_t *this,
-                                changelog_ev_selector_t *selection)
+changelog_init_event_selection(xlator_t *this,
+                               changelog_ev_selector_t *selection)
 {
-        int ret = 0;
-        int j = CHANGELOG_EV_SELECTION_RANGE;
-
-        ret = LOCK_INIT (&selection->reflock);
-        if (ret != 0)
-                return -1;
+    int ret = 0;
+    int j = CHANGELOG_EV_SELECTION_RANGE;
 
-        LOCK (&selection->reflock);
-        {
-                while (j--) {
-                        selection->ref[j] = 0;
-                }
-        }
-        UNLOCK (&selection->reflock);
-
-        return 0;
-}
-
-int
-changelog_cleanup_event_selection (xlator_t *this,
-                                   changelog_ev_selector_t *selection)
-{
-        int ret = 0;
-        int j = CHANGELOG_EV_SELECTION_RANGE;
+    ret = LOCK_INIT(&selection->reflock);
+    if (ret != 0)
+        return -1;
 
-        LOCK (&selection->reflock);
-        {
-                while (j--) {
-                        if (selection->ref[j] > 0)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        CHANGELOG_MSG_CLEANUP_ON_ACTIVE_REF,
-                                        "changelog event selection cleaning up "
-                                        " on active references");
-                }
+    LOCK(&selection->reflock);
+    {
+        while (j--) {
+            selection->ref[j] = 0;
         }
-        UNLOCK (&selection->reflock);
+    }
+    UNLOCK(&selection->reflock);
 
-        return LOCK_DESTROY (&selection->reflock);
+    return 0;
 }
 
 static void
-changelog_perform_dispatch (xlator_t *this,
-                            changelog_priv_t *priv, void *mem, size_t size)
+changelog_perform_dispatch(xlator_t *this, changelog_priv_t *priv, void *mem,
+                           size_t size)
 {
-        char *buf    = NULL;
-        void *opaque = NULL;
-
-        buf = rbuf_reserve_write_area (priv->rbuf, size, &opaque);
-        if (!buf) {
-                gf_msg_callingfn (this->name,
-                                  GF_LOG_WARNING, 0,
-                                  CHANGELOG_MSG_DISPATCH_EVENT_FAILED,
-                                  "failed to dispatch event");
-                return;
-        }
+    char *buf = NULL;
+    void *opaque = NULL;
+
+    buf = rbuf_reserve_write_area(priv->rbuf, size, &opaque);
+    if (!buf) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_DISPATCH_EVENT_FAILED,
+                         "failed to dispatch event");
+        return;
+    }
 
-        memcpy (buf, mem, size);
-        rbuf_write_complete (opaque);
+    memcpy(buf, mem, size);
+    rbuf_write_complete(opaque);
 }
 
 void
-changelog_dispatch_event (xlator_t *this,
-                          changelog_priv_t *priv, changelog_event_t *ev)
+changelog_dispatch_event(xlator_t *this, changelog_priv_t *priv,
+                         changelog_event_t *ev)
 {
-        changelog_ev_selector_t *selection = NULL;
+    changelog_ev_selector_t *selection = NULL;
 
-        selection = &priv->ev_selection;
-        if (changelog_ev_selected (this, selection, ev->ev_type)) {
-                changelog_perform_dispatch (this, priv, ev, CHANGELOG_EV_SIZE);
-        }
+    selection = &priv->ev_selection;
+    if (changelog_ev_selected(this, selection, ev->ev_type)) {
+        changelog_perform_dispatch(this, priv, ev, CHANGELOG_EV_SIZE);
+    }
 }
 
 void
-changelog_set_usable_record_and_length (changelog_local_t *local,
-                                        size_t len, int xr)
+changelog_set_usable_record_and_length(changelog_local_t *local, size_t len,
+                                       int xr)
 {
-        changelog_log_data_t *cld = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        cld = &local->cld;
+    cld = &local->cld;
 
-        cld->cld_ptr_len = len;
-        cld->cld_xtra_records = xr;
+    cld->cld_ptr_len = len;
+    cld->cld_xtra_records = xr;
 }
 
 void
-changelog_local_cleanup (xlator_t *xl, changelog_local_t *local)
+changelog_local_cleanup(xlator_t *xl, changelog_local_t *local)
 {
-        int                   i   = 0;
-        changelog_opt_t      *co  = NULL;
-        changelog_log_data_t *cld = NULL;
+    int i = 0;
+    changelog_opt_t *co = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        if (!local)
-                return;
+    if (!local)
+        return;
 
-        cld = &local->cld;
+    cld = &local->cld;
 
-        /* cleanup dynamic allocation for extra records */
-        if (cld->cld_xtra_records) {
-                co = (changelog_opt_t *) cld->cld_ptr;
-                for (; i < cld->cld_xtra_records; i++, co++)
-                        if (co->co_free)
-                                co->co_free (co);
-        }
+    /* cleanup dynamic allocation for extra records */
+    if (cld->cld_xtra_records) {
+        co = (changelog_opt_t *)cld->cld_ptr;
+        for (; i < cld->cld_xtra_records; i++, co++)
+            if (co->co_free)
+                co->co_free(co);
+    }
 
-        CHANGELOG_IOBUF_UNREF (cld->cld_iobuf);
+    CHANGELOG_IOBUF_UNREF(cld->cld_iobuf);
 
-        if (local->inode)
-                inode_unref (local->inode);
+    if (local->inode)
+        inode_unref(local->inode);
 
-        mem_put (local);
+    mem_put(local);
 }
 
 int
-changelog_write (int fd, char *buffer, size_t len)
+changelog_write(int fd, char *buffer, size_t len)
 {
-        ssize_t size = 0;
-        size_t written = 0;
+    ssize_t size = 0;
+    size_t written = 0;
 
-        while (written < len) {
-                size = sys_write (fd,
-                              buffer + written, len - written);
-                if (size <= 0)
-                        break;
+    while (written < len) {
+        size = sys_write(fd, buffer + written, len - written);
+        if (size <= 0)
+            break;
 
-                written += size;
-        }
+        written += size;
+    }
 
-        return (written != len);
+    return (written != len);
 }
 
 int
-htime_update (xlator_t *this,
-              changelog_priv_t *priv, unsigned long ts,
-              char * buffer)
+htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer)
 {
-        char changelog_path[PATH_MAX+1]   = {0,};
-        int len                           = -1;
-        char x_value[25]                  = {0,};
-        /* time stamp(10) + : (1) + rolltime (12 ) + buffer (2) */
-        int ret                           = 0;
-
-        if (priv->htime_fd ==-1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_HTIME_ERROR,
-                        "Htime fd not available for updation");
-                ret = -1;
-                goto out;
-        }
-        strncpy (changelog_path, buffer, PATH_MAX);
-        len = strlen (changelog_path);
-        changelog_path[len] = '\0'; /* redundant */
-
-        if (changelog_write (priv->htime_fd, (void*) changelog_path, len+1 ) < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_HTIME_ERROR,
-                        "Htime file content write failed");
-                ret =-1;
-                goto out;
-        }
-
-        snprintf (x_value, sizeof x_value, "%lu:%d",
-                 ts, priv->rollover_count);
-
-        if (sys_fsetxattr (priv->htime_fd, HTIME_KEY, x_value,
-                           strlen (x_value), XATTR_REPLACE)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_HTIME_ERROR,
-                        "Htime xattr updation failed with XATTR_REPLACE "
-                        "Changelog: %s", changelog_path);
-
-                if (sys_fsetxattr (priv->htime_fd, HTIME_KEY, x_value,
-                                   strlen (x_value), 0)) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_HTIME_ERROR,
-                                "Htime xattr updation failed "
-                                "Changelog: %s", changelog_path);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        priv->rollover_count +=1;
+    char changelog_path[PATH_MAX + 1] = {
+        0,
+    };
+    int len = -1;
+    char x_value[25] = {
+        0,
+    };
+    /* time stamp(10) + : (1) + rolltime (12 ) + buffer (2) */
+    int ret = 0;
+
+    if (priv->htime_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=fd not available", NULL);
+        ret = -1;
+        goto out;
+    }
+    len = snprintf(changelog_path, PATH_MAX, "%s", buffer);
+    if (len >= PATH_MAX) {
+        ret = -1;
+        goto out;
+    }
+    if (changelog_write(priv->htime_fd, (void *)changelog_path, len + 1) < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=write failed", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    len = snprintf(x_value, sizeof(x_value), "%ld:%d", ts,
+                   priv->rollover_count);
+    if (len >= sizeof(x_value)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (sys_fsetxattr(priv->htime_fd, HTIME_KEY, x_value, len, XATTR_REPLACE)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=xattr updation failed", "XATTR_REPLACE=true",
+                "changelog=%s", changelog_path, NULL);
+
+        if (sys_fsetxattr(priv->htime_fd, HTIME_KEY, x_value, len, 0)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_ERROR,
+                    "reason=xattr updation failed", "changelog=%s",
+                    changelog_path, NULL);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    priv->rollover_count += 1;
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -339,43 +308,45 @@ out:
  * 0 : If NOT empty, proceed usual.
  */
 int
-cl_is_empty (xlator_t *this, int fd)
+cl_is_empty(xlator_t *this, int fd)
 {
-        int             ret             = -1;
-        size_t          elen            = 0;
-        int             encoding        = -1;
-        char            buffer[1024]    = {0,};
-        struct stat     stbuf           = {0,};
-        int             major_version   = -1;
-        int             minor_version   = -1;
-
-        ret = sys_fstat (fd, &stbuf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSTAT_OP_FAILED,
-                        "Could not stat (CHANGELOG)");
-                goto out;
-        }
-
-        ret = sys_lseek (fd, 0, SEEK_SET);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_LSEEK_OP_FAILED,
-                        "Could not lseek (CHANGELOG)");
-                goto out;
-        }
-
-        CHANGELOG_GET_HEADER_INFO (fd, buffer, 1024, encoding,
-                                   major_version, minor_version, elen);
-
-        if (elen == stbuf.st_size) {
-                ret = 1;
-        } else {
-                ret = 0;
-        }
+    int ret = -1;
+    size_t elen = 0;
+    int encoding = -1;
+    char buffer[1024] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    int major_version = -1;
+    int minor_version = -1;
+
+    ret = sys_fstat(fd, &stbuf);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSTAT_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0, SEEK_SET);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_LSEEK_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    CHANGELOG_GET_HEADER_INFO(fd, buffer, sizeof(buffer), encoding,
+                              major_version, minor_version, elen);
+
+    if (elen == stbuf.st_size) {
+        ret = 1;
+    } else {
+        ret = 0;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -387,158 +358,172 @@ out:
  * -1 : Error
  */
 int
-update_path (xlator_t *this, char *cl_path)
+update_path(xlator_t *this, char *cl_path)
 {
-        char low_cl[]     = "changelog";
-        char up_cl[]      = "CHANGELOG";
-        char *found       = NULL;
-        int  iter         = 0;
-        int  ret          = -1;
-
-        found = strstr(cl_path, up_cl);
-
-        if (found == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_LSEEK_OP_FAILED,
-                        "Could not find CHANGELOG in changelog path");
-                goto out;
-        } else {
-                strncpy(found, low_cl, strlen(low_cl));
-        }
-
-        ret = 0;
+    const char low_cl[] = "changelog";
+    const char up_cl[] = "CHANGELOG";
+    char *found = NULL;
+    int ret = -1;
+
+    found = strstr(cl_path, up_cl);
+
+    if (found == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PATH_NOT_FOUND,
+                NULL);
+        goto out;
+    } else {
+        memcpy(found, low_cl, sizeof(low_cl) - 1);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-changelog_rollover_changelog (xlator_t *this,
-                              changelog_priv_t *priv, unsigned long ts)
+changelog_rollover_changelog(xlator_t *this, changelog_priv_t *priv, time_t ts)
 {
-        int   ret             = -1;
-        int   notify          = 0;
-        int   cl_empty_flag   = 0;
-        char  ofile[PATH_MAX] = {0,};
-        char  nfile[PATH_MAX] = {0,};
-        changelog_event_t ev  = {0,};
-
-        if (priv->changelog_fd != -1) {
-                ret = sys_fsync (priv->changelog_fd);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSYNC_OP_FAILED,
-                                "fsync failed");
-                }
-                ret = cl_is_empty (this, priv->changelog_fd);
-                if (ret == 1) {
-                        cl_empty_flag = 1;
-                } else if (ret == -1) {
-                        /* Log error but proceed as usual */
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED,
-                                "Error detecting empty changelog");
-                }
-                sys_close (priv->changelog_fd);
-                priv->changelog_fd = -1;
-        }
-
-        (void) snprintf (ofile, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME, priv->changelog_dir);
-        (void) snprintf (nfile, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME".%lu",
-                         priv->changelog_dir, ts);
-
-        if (cl_empty_flag == 1) {
-                ret = sys_unlink (ofile);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_UNLINK_OP_FAILED,
-                                "error unlinking(empty cl) %s)",
-                                ofile);
-                        ret = 0;  /* Error in unlinking empty changelog should
-                                     not break further changelog operation, so
-                                     reset return value to 0*/
-                }
-        } else {
-                ret = sys_rename (ofile, nfile);
+    int ret = -1;
+    int notify = 0;
+    int cl_empty_flag = 0;
+    struct tm *gmt;
+    char yyyymmdd[40];
+    char ofile[PATH_MAX] = {
+        0,
+    };
+    char nfile[PATH_MAX] = {
+        0,
+    };
+    char nfile_dir[PATH_MAX] = {
+        0,
+    };
+    changelog_event_t ev = {
+        0,
+    };
+
+    if (priv->changelog_fd != -1) {
+        ret = sys_fsync(priv->changelog_fd);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
+        }
+        ret = cl_is_empty(this, priv->changelog_fd);
+        if (ret == 1) {
+            cl_empty_flag = 1;
+        } else if (ret == -1) {
+            /* Log error but proceed as usual */
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED, NULL);
+        }
+        sys_close(priv->changelog_fd);
+        priv->changelog_fd = -1;
+    }
+
+    /* Get GMT time. */
+    gmt = gmtime(&ts);
+
+    strftime(yyyymmdd, sizeof(yyyymmdd), "%Y/%m/%d", gmt);
+
+    (void)snprintf(ofile, PATH_MAX, "%s/" CHANGELOG_FILE_NAME,
+                   priv->changelog_dir);
+    (void)snprintf(nfile, PATH_MAX, "%s/%s/" CHANGELOG_FILE_NAME ".%ld",
+                   priv->changelog_dir, yyyymmdd, ts);
+    (void)snprintf(nfile_dir, PATH_MAX, "%s/%s", priv->changelog_dir, yyyymmdd);
+
+    if (cl_empty_flag == 1) {
+        ret = sys_unlink(ofile);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_UNLINK_OP_FAILED, "path=%s", ofile, NULL);
+            ret = 0; /* Error in unlinking empty changelog should
+                        not break further changelog operation, so
+                        reset return value to 0*/
+        }
+    } else {
+        ret = sys_rename(ofile, nfile);
+
+        /* Changelog file rename gets ENOENT when parent dir doesn't exist */
+        if (errno == ENOENT) {
+            ret = mkdir_p(nfile_dir, 0600, _gf_true);
+
+            if ((ret == -1) && (EEXIST != errno)) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_MSG_MKDIR_ERROR, "%s", nfile_dir, NULL);
+                goto out;
+            }
 
-                if (ret && (errno == ENOENT)) {
-                        ret = 0;
-                        goto out;
-                }
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_RENAME_ERROR,
-                                "error renaming %s -> %s",
-                                ofile, nfile);
-                }
+            ret = sys_rename(ofile, nfile);
         }
 
-        if (!ret && (cl_empty_flag == 0)) {
-                        notify = 1;
+        if (ret && (errno == ENOENT)) {
+            ret = 0;
+            goto out;
         }
-
-        if (!ret) {
-                if (cl_empty_flag) {
-                        update_path (this, nfile);
-                }
-                ret = htime_update (this, priv, ts, nfile);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                0, CHANGELOG_MSG_HTIME_ERROR,
-                                "could not update htime file");
-                        goto out;
-                }
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_RENAME_ERROR,
+                    "from=%s", ofile, "to=%s", nfile, NULL);
         }
+    }
 
-        if (notify) {
-                ev.ev_type = CHANGELOG_OP_TYPE_JOURNAL;
-                memcpy (ev.u.journal.path, nfile, strlen (nfile) + 1);
-                changelog_dispatch_event (this, priv, &ev);
+    if (!ret && (cl_empty_flag == 0)) {
+        notify = 1;
+    }
+
+    if (!ret) {
+        if (cl_empty_flag) {
+            update_path(this, nfile);
         }
- out:
-        /* If this is explicit rollover initiated by snapshot,
-         * wakeup reconfigure thread waiting for changelog to
-         * rollover. This should happen even in failure cases as
-         * well otherwise snapshot will timeout and fail. Hence
-         * moved under out.
-         */
-        if (priv->explicit_rollover) {
-                priv->explicit_rollover = _gf_false;
-
-                pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                {
-                        if (ret) {
-                                priv->bn.bnotify_error = _gf_true;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED,
-                                        "Fail snapshot because of "
-                                        "previous errors");
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        CHANGELOG_MSG_BNOTIFY_INFO, "Explicit "
-                                        "rollover changelog: %s signaling "
-                                        "bnotify", nfile);
-                        }
-                        priv->bn.bnotify = _gf_false;
-                        pthread_cond_signal (&priv->bn.bnotify_cond);
-                }
-                pthread_mutex_unlock (&priv->bn.bnotify_mutex);
+        ret = htime_update(this, priv, ts, nfile);
+        if (ret == -1) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                    NULL);
+            goto out;
         }
-        return ret;
+    }
+
+    if (notify) {
+        ev.ev_type = CHANGELOG_OP_TYPE_JOURNAL;
+        memcpy(ev.u.journal.path, nfile, strlen(nfile) + 1);
+        changelog_dispatch_event(this, priv, &ev);
+    }
+out:
+    /* If this is explicit rollover initiated by snapshot,
+     * wakeup reconfigure thread waiting for changelog to
+     * rollover. This should happen even in failure cases as
+     * well otherwise snapshot will timeout and fail. Hence
+     * moved under out.
+     */
+    if (priv->explicit_rollover) {
+        priv->explicit_rollover = _gf_false;
+
+        pthread_mutex_lock(&priv->bn.bnotify_mutex);
+        {
+            if (ret) {
+                priv->bn.bnotify_error = _gf_true;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_BNOTIFY_INFO,
+                        "changelog=%s", nfile, NULL);
+            }
+            priv->bn.bnotify = _gf_false;
+            pthread_cond_signal(&priv->bn.bnotify_cond);
+        }
+        pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+    }
+    return ret;
 }
 
 int
-filter_cur_par_dirs (const struct dirent *entry)
+filter_cur_par_dirs(const struct dirent *entry)
 {
-        if (entry == NULL)
-                return 0;
+    if (entry == NULL)
+        return 0;
 
-        if ((strcmp(entry->d_name, ".") == 0) ||
-            (strcmp(entry->d_name, "..") == 0))
-                return 0;
-        else
-                return 1;
+    if ((strcmp(entry->d_name, ".") == 0) || (strcmp(entry->d_name, "..") == 0))
+        return 0;
+    else
+        return 1;
 }
 
 /*
@@ -551,252 +536,284 @@ filter_cur_par_dirs (const struct dirent *entry)
  */
 
 int
-find_current_htime (int ht_dir_fd, const char *ht_dir_path, char *ht_file_bname)
+find_current_htime(int ht_dir_fd, const char *ht_dir_path, char *ht_file_bname)
 {
-        struct dirent       **namelist = NULL;
-        int                   ret      = 0;
-        int                   cnt      = 0;
-        int                   i        = 0;
-        xlator_t             *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (ht_dir_path);
-
-        cnt = scandir (ht_dir_path, &namelist, filter_cur_par_dirs, alphasort);
-        if (cnt < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_SCAN_DIR_FAILED,
-                        "scandir failed");
-        } else if (cnt > 0) {
-                strncpy (ht_file_bname, namelist[cnt - 1]->d_name, NAME_MAX);
-                ht_file_bname[NAME_MAX - 1] = 0;
-
-                if (sys_fsetxattr (ht_dir_fd, HTIME_CURRENT, ht_file_bname,
-                    strlen (ht_file_bname), 0)) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSETXATTR_FAILED,
-                                "fsetxattr failed: HTIME_CURRENT");
-                        ret = -1;
-                        goto out;
-                }
+    struct dirent **namelist = NULL;
+    int ret = 0;
+    int cnt = 0;
+    int i = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(ht_dir_path);
+
+    cnt = scandir(ht_dir_path, &namelist, filter_cur_par_dirs, alphasort);
+    if (cnt < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_SCAN_DIR_FAILED,
+                NULL);
+    } else if (cnt > 0) {
+        if (snprintf(ht_file_bname, NAME_MAX, "%s",
+                     namelist[cnt - 1]->d_name) >= NAME_MAX) {
+            ret = -1;
+            goto out;
+        }
+        if (sys_fsetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                          strlen(ht_file_bname), 0)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSETXATTR_FAILED, "HTIME_CURRENT", NULL);
+            ret = -1;
+            goto out;
+        }
+
+        if (sys_fsync(ht_dir_fd) < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
+            ret = -1;
+            goto out;
+        }
+    }
 
-                if (sys_fsync (ht_dir_fd) < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSYNC_OP_FAILED,
-                                "fsync failed");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
- out:
-        for (i = 0; i < cnt; i++)
-                free (namelist[i]);
-        free (namelist);
+out:
+    for (i = 0; i < cnt; i++)
+        free(namelist[i]);
+    free(namelist);
 
-        if (ret)
-                cnt = ret;
+    if (ret)
+        cnt = ret;
 
-        return cnt;
+    return cnt;
 }
 
 /* Returns 0 on successful open of htime file
  * returns -1 on failure or error
  */
 int
-htime_open (xlator_t *this,
-              changelog_priv_t *priv, unsigned long ts)
+htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts)
 {
-        int ht_file_fd                  = -1;
-        int ht_dir_fd                   = -1;
-        int ret                         = 0;
-        int cnt                         = 0;
-        char ht_dir_path[PATH_MAX]      = {0,};
-        char ht_file_path[PATH_MAX]     = {0,};
-        char ht_file_bname[NAME_MAX]    = {0,};
-        char x_value[NAME_MAX]          = {0,};
-        int flags                       = 0;
-        unsigned long min_ts            = 0;
-        unsigned long max_ts            = 0;
-        unsigned long total             = 0;
-        ssize_t size                    = 0;
-
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
-
-        /* Open htime directory to get HTIME_CURRENT */
-        ht_dir_fd = open (ht_dir_path, O_RDONLY);
-        if (ht_dir_fd == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_OPEN_FAILED, "open failed: %s",
-                        ht_dir_path);
-                ret = -1;
-                goto out;
-        }
-
-        size = sys_fgetxattr (ht_dir_fd, HTIME_CURRENT, ht_file_bname,
-                             sizeof (ht_file_bname));
-        if (size < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FGETXATTR_FAILED, "Error extracting"
-                        " HTIME_CURRENT.");
-
-                /* If upgrade scenario, find the latest HTIME.TSTAMP file
-                 * and use the same. If error, create a new HTIME.TSTAMP
-                 * file.
-                 */
-                cnt = find_current_htime (ht_dir_fd, ht_dir_path,
-                                           ht_file_bname);
-                if (cnt <= 0) {
-                        gf_msg (this->name, GF_LOG_INFO, errno,
-                                CHANGELOG_MSG_HTIME_INFO,
-                                "HTIME_CURRENT not found. Changelog enabled"
-                                " before init");
-                        return htime_create (this, priv, ts);
-                }
-
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_HTIME_ERROR, "Error extracting"
-                        " HTIME_CURRENT.");
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_HTIME_INFO,
-                "HTIME_CURRENT: %s", ht_file_bname);
-        (void) snprintf (ht_file_path, PATH_MAX, "%s/%s",
-                         ht_dir_path, ht_file_bname);
-
-        /* Open in append mode as existing htime file is used */
-        flags |= (O_RDWR | O_SYNC | O_APPEND);
-        ht_file_fd = open (ht_file_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (ht_file_fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_OPEN_FAILED,
-                        "unable to open htime file: %s",
-                        ht_file_path);
-                ret = -1;
-                goto out;
-        }
-
-        /* save this htime_fd in priv->htime_fd */
-        priv->htime_fd = ht_file_fd;
-
-        /* Initialize rollover-number in priv to current number */
-        size = sys_fgetxattr (ht_file_fd, HTIME_KEY, x_value, sizeof (x_value));
-        if (size < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FGETXATTR_FAILED, "error extracting max"
-                        " timstamp from htime file %s",
-                        ht_file_path);
-                ret = -1;
-                goto out;
-        }
-
-        sscanf (x_value, "%lu:%lu", &max_ts, &total);
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_TOTAL_LOG_INFO,
-                "INIT CASE: MIN: %lu, MAX: %lu,"
-                " TOTAL CHANGELOGS: %lu", min_ts, max_ts, total);
+    int ht_file_fd = -1;
+    int ht_dir_fd = -1;
+    int ret = 0;
+    int cnt = 0;
+    char ht_dir_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_bname[NAME_MAX] = {
+        0,
+    };
+    char x_value[NAME_MAX] = {
+        0,
+    };
+    int flags = 0;
+    unsigned long min_ts = 0;
+    unsigned long max_ts = 0;
+    unsigned long total = 0;
+    unsigned long total1 = 0;
+    ssize_t size = 0;
+    struct stat stat_buf = {
+        0,
+    };
+    unsigned long record_len = 0;
+    int32_t len = 0;
+
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
+
+    /* Open htime directory to get HTIME_CURRENT */
+    ht_dir_fd = open(ht_dir_path, O_RDONLY);
+    if (ht_dir_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_dir_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    size = sys_fgetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                         sizeof(ht_file_bname));
+    if (size < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FGETXATTR_FAILED,
+                "name=HTIME_CURRENT", NULL);
+
+        /* If upgrade scenario, find the latest HTIME.TSTAMP file
+         * and use the same. If error, create a new HTIME.TSTAMP
+         * file.
+         */
+        cnt = find_current_htime(ht_dir_fd, ht_dir_path, ht_file_bname);
+        if (cnt <= 0) {
+            gf_smsg(this->name, GF_LOG_INFO, errno,
+                    CHANGELOG_MSG_NO_HTIME_CURRENT, NULL);
+            sys_close(ht_dir_fd);
+            return htime_create(this, priv, ts);
+        }
+
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_HTIME_CURRENT_ERROR, NULL);
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_HTIME_CURRENT, "path=%s",
+            ht_file_bname, NULL);
+    len = snprintf(ht_file_path, PATH_MAX, "%s/%s", ht_dir_path, ht_file_bname);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Open in append mode as existing htime file is used */
+    flags |= (O_RDWR | O_SYNC | O_APPEND);
+    ht_file_fd = open(ht_file_path, flags,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (ht_file_fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* save this htime_fd in priv->htime_fd */
+    priv->htime_fd = ht_file_fd;
+
+    ret = sys_fstat(ht_file_fd, &stat_buf);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_STAT_ERROR,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* Initialize rollover-number in priv to current number */
+    size = sys_fgetxattr(ht_file_fd, HTIME_KEY, x_value, sizeof(x_value));
+    if (size < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FGETXATTR_FAILED,
+                "name=%s", HTIME_KEY, "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    sscanf(x_value, "%lu:%lu", &max_ts, &total);
+
+    /* 22 = 1(/) + 20(CHANGELOG.TIMESTAMP) + 1(\x00) */
+    record_len = strlen(priv->changelog_dir) + 22;
+    total1 = stat_buf.st_size / record_len;
+    if (total != total1) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_TOTAL_LOG_INFO,
+                "xattr_total=%lu", total, "size_total=%lu", total1, NULL);
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_TOTAL_LOG_INFO, "min=%lu",
+            min_ts, "max=%lu", max_ts, "total_changelogs=%lu", total, NULL);
+
+    if (total < total1)
+        priv->rollover_count = total1 + 1;
+    else
         priv->rollover_count = total + 1;
 
 out:
-        if (ht_dir_fd != -1)
-                sys_close (ht_dir_fd);
-        return ret;
+    if (ht_dir_fd != -1)
+        sys_close(ht_dir_fd);
+    return ret;
 }
 
 /* Returns 0 on successful creation of htime file
  * returns -1 on failure or error
  */
 int
-htime_create (xlator_t *this,
-              changelog_priv_t *priv, unsigned long ts)
+htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts)
 {
-        int ht_file_fd                      = -1;
-        int ht_dir_fd                       = -1;
-        int ret                             = 0;
-        char ht_dir_path[PATH_MAX]          = {0,};
-        char ht_file_path[PATH_MAX]         = {0,};
-        char ht_file_bname[NAME_MAX + 1]    = {0,};
-        int flags                           = 0;
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_HTIME_INFO, "Changelog enable: Creating new "
-                "HTIME.%lu file", ts);
-
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
-
-        /* get the htime file name in ht_file_path */
-        (void) snprintf (ht_file_path,PATH_MAX,"%s/%s.%lu",ht_dir_path,
-                        HTIME_FILE_NAME, ts);
-
-        flags |= (O_CREAT | O_RDWR | O_SYNC);
-        ht_file_fd = open (ht_file_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (ht_file_fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_OPEN_FAILED,
-                        "unable to create htime file: %s",
-                        ht_file_path);
-                ret = -1;
-                goto out;
-        }
-
-        if (sys_fsetxattr (ht_file_fd, HTIME_KEY, HTIME_INITIAL_VALUE,
-                       sizeof (HTIME_INITIAL_VALUE)-1,  0)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSETXATTR_FAILED,
-                        "Htime xattr initialization failed");
-                ret = -1;
-                goto out;
-        }
-
-        ret = sys_fsync (ht_file_fd);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSYNC_OP_FAILED,
-                        "fsync failed");
-                goto out;
-        }
-
-        /* Set xattr HTIME_CURRENT on htime directory to htime filename */
-        ht_dir_fd = open (ht_dir_path, O_RDONLY);
-        if (ht_dir_fd == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_OPEN_FAILED, "open of %s failed",
-                        ht_dir_path);
-                ret = -1;
-                goto out;
-        }
-
-        (void) snprintf (ht_file_bname, sizeof (ht_file_bname), "%s.%lu",
-                         HTIME_FILE_NAME, ts);
-        if (sys_fsetxattr (ht_dir_fd, HTIME_CURRENT, ht_file_bname,
-            strlen (ht_file_bname), 0)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSETXATTR_FAILED, "fsetxattr failed:"
-                        " HTIME_CURRENT");
-                ret = -1;
-                goto out;
-        }
-
-        ret = sys_fsync (ht_dir_fd);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSYNC_OP_FAILED,
-                        "fsync failed");
-                goto out;
-        }
-
-        /* save this htime_fd in priv->htime_fd */
-        priv->htime_fd = ht_file_fd;
-        /* initialize rollover-number in priv to 1 */
-        priv->rollover_count = 1;
+    int ht_file_fd = -1;
+    int ht_dir_fd = -1;
+    int ret = 0;
+    char ht_dir_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_bname[NAME_MAX + 1] = {
+        0,
+    };
+    int flags = 0;
+    int32_t len = 0;
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_NEW_HTIME_FILE,
+            "name=%ld", ts, NULL);
+
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
+
+    /* get the htime file name in ht_file_path */
+    len = snprintf(ht_file_path, PATH_MAX, "%s/%s.%ld", ht_dir_path,
+                   HTIME_FILE_NAME, ts);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    flags |= (O_CREAT | O_RDWR | O_SYNC);
+    ht_file_fd = open(ht_file_path, flags,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (ht_file_fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (sys_fsetxattr(ht_file_fd, HTIME_KEY, HTIME_INITIAL_VALUE,
+                      sizeof(HTIME_INITIAL_VALUE) - 1, 0)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_XATTR_INIT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_fsync(ht_file_fd);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSYNC_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* save this htime_fd in priv->htime_fd */
+    priv->htime_fd = ht_file_fd;
+
+    ht_file_fd = -1;
+
+    /* Set xattr HTIME_CURRENT on htime directory to htime filename */
+    ht_dir_fd = open(ht_dir_path, O_RDONLY);
+    if (ht_dir_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_dir_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    (void)snprintf(ht_file_bname, sizeof(ht_file_bname), "%s.%ld",
+                   HTIME_FILE_NAME, ts);
+    if (sys_fsetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                      strlen(ht_file_bname), 0)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSETXATTR_FAILED,
+                " HTIME_CURRENT", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_fsync(ht_dir_fd);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSYNC_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* initialize rollover-number in priv to 1 */
+    priv->rollover_count = 1;
 
 out:
-        if (ht_dir_fd != -1)
-                sys_close (ht_dir_fd);
-        return ret;
+    if (ht_dir_fd != -1)
+        sys_close(ht_dir_fd);
+    if (ht_file_fd != -1)
+        sys_close(ht_file_fd);
+    return ret;
 }
 
 /* Description:
@@ -808,48 +825,53 @@ out:
  *      -1  : On failure.
  */
 int
-changelog_snap_open (xlator_t *this,
-                         changelog_priv_t *priv)
+changelog_snap_open(xlator_t *this, changelog_priv_t *priv)
 {
-        int fd                        = -1;
-        int ret                       = 0;
-        int flags                     = 0;
-        char buffer[1024]             = {0,};
-        char c_snap_path[PATH_MAX]    = {0,};
-        char csnap_dir_path[PATH_MAX] = {0,};
-
-        CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir_path);
-
-        (void) snprintf (c_snap_path, PATH_MAX,
-                        "%s/"CSNAP_FILE_NAME,
-                        csnap_dir_path);
-
-        flags |= (O_CREAT | O_RDWR | O_TRUNC);
-
-        fd = open (c_snap_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_OPEN_FAILED, "unable to open %s file ",
-                        c_snap_path);
-                ret = -1;
-                goto out;
-        }
-        priv->c_snap_fd = fd;
-
-        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
-                        CHANGELOG_VERSION_MAJOR,
-                        CHANGELOG_VERSION_MINOR,
-                        priv->ce->encoder);
-        ret = changelog_snap_write_change (priv, buffer, strlen (buffer));
-        if (ret < 0) {
-                sys_close (priv->c_snap_fd);
-                priv->c_snap_fd = -1;
-                goto out;
-        }
+    int fd = -1;
+    int ret = 0;
+    int flags = 0;
+    char buffer[1024] = {
+        0,
+    };
+    char c_snap_path[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir_path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir_path);
+
+    len = snprintf(c_snap_path, PATH_MAX, "%s/" CSNAP_FILE_NAME,
+                   csnap_dir_path);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    flags |= (O_CREAT | O_RDWR | O_TRUNC);
+
+    fd = open(c_snap_path, flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", c_snap_path, NULL);
+        ret = -1;
+        goto out;
+    }
+    priv->c_snap_fd = fd;
+
+    (void)snprintf(buffer, 1024, CHANGELOG_HEADER, CHANGELOG_VERSION_MAJOR,
+                   CHANGELOG_VERSION_MINOR, priv->ce->encoder);
+    ret = changelog_snap_write_change(priv, buffer, strlen(buffer));
+    if (ret < 0) {
+        sys_close(priv->c_snap_fd);
+        priv->c_snap_fd = -1;
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -860,17 +882,15 @@ out:
  *      -1 : On Failure.
  */
 int
-changelog_snap_logging_start (xlator_t *this,
-                                  changelog_priv_t *priv)
+changelog_snap_logging_start(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = changelog_snap_open (this, priv);
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                        CHANGELOG_MSG_SNAP_INFO,
-                        "Now starting to log in call path");
+    ret = changelog_snap_open(this, priv);
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO, "starting",
+            NULL);
 
-        return ret;
+    return ret;
 }
 
 /*
@@ -881,118 +901,104 @@ changelog_snap_logging_start (xlator_t *this,
  *      -1 : On Failure.
  */
 int
-changelog_snap_logging_stop (xlator_t *this,
-                changelog_priv_t *priv)
+changelog_snap_logging_stop(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret         = 0;
+    int ret = 0;
 
-        sys_close (priv->c_snap_fd);
-        priv->c_snap_fd = -1;
+    sys_close(priv->c_snap_fd);
+    priv->c_snap_fd = -1;
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_SNAP_INFO,
-                "Stopped to log in call path");
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO, "Stopped",
+            NULL);
 
-        return ret;
+    return ret;
 }
 
 int
-changelog_open_journal (xlator_t *this,
-                        changelog_priv_t *priv)
+changelog_open_journal(xlator_t *this, changelog_priv_t *priv)
 {
-        int fd                        = 0;
-        int ret                       = -1;
-        int flags                     = 0;
-        char buffer[1024]             = {0,};
-        char changelog_path[PATH_MAX] = {0,};
-
-        (void) snprintf (changelog_path, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME,
-                         priv->changelog_dir);
-
-        flags |= (O_CREAT | O_RDWR);
-        if (priv->fsync_interval == 0)
-                flags |= O_SYNC;
-
-        fd = open (changelog_path, flags,
-                   S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_OPEN_FAILED,
-                        "unable to open/create changelog file %s."
-                        " change-logging will be"
-                        " inactive", changelog_path);
-                goto out;
-        }
-
-        priv->changelog_fd = fd;
+    int fd = 0;
+    int ret = -1;
+    int flags = 0;
+    char buffer[1024] = {
+        0,
+    };
+    char changelog_path[PATH_MAX] = {
+        0,
+    };
+
+    (void)snprintf(changelog_path, PATH_MAX, "%s/" CHANGELOG_FILE_NAME,
+                   priv->changelog_dir);
+
+    flags |= (O_CREAT | O_RDWR);
+    if (priv->fsync_interval == 0)
+        flags |= O_SYNC;
+
+    fd = open(changelog_path, flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", changelog_path, NULL);
+        goto out;
+    }
+
+    priv->changelog_fd = fd;
+
+    (void)snprintf(buffer, 1024, CHANGELOG_HEADER, CHANGELOG_VERSION_MAJOR,
+                   CHANGELOG_VERSION_MINOR, priv->ce->encoder);
+    ret = changelog_write_change(priv, buffer, strlen(buffer));
+    if (ret) {
+        sys_close(priv->changelog_fd);
+        priv->changelog_fd = -1;
+        goto out;
+    }
+
+    ret = 0;
 
-        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
-                         CHANGELOG_VERSION_MAJOR,
-                         CHANGELOG_VERSION_MINOR,
-                         priv->ce->encoder);
-        ret = changelog_write_change (priv, buffer, strlen (buffer));
-        if (ret) {
-                sys_close (priv->changelog_fd);
-                priv->changelog_fd = -1;
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-changelog_start_next_change (xlator_t *this,
-                             changelog_priv_t *priv,
-                             unsigned long ts, gf_boolean_t finale)
+changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts,
+                            gf_boolean_t finale)
 {
-        int ret = -1;
+    int ret = -1;
 
-        ret = changelog_rollover_changelog (this, priv, ts);
+    ret = changelog_rollover_changelog(this, priv, ts);
 
-        if (!ret && !finale)
-                ret = changelog_open_journal (this, priv);
+    if (!ret && !finale)
+        ret = changelog_open_journal(this, priv);
 
-        return ret;
+    return ret;
 }
 
 /**
  * return the length of entry
  */
 size_t
-changelog_entry_length ()
+changelog_entry_length()
 {
-        return sizeof (changelog_log_data_t);
+    return sizeof(changelog_log_data_t);
 }
 
-int
-changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last)
+void
+changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last)
 {
-        struct timeval tv = {0,};
-
-        cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
-
-        if (gettimeofday (&tv, NULL))
-                return -1;
-
-        cld->cld_roll_time = (unsigned long) tv.tv_sec;
-        cld->cld_finale = is_last;
-        return 0;
+    cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
+    cld->cld_roll_time = gf_time();
+    cld->cld_finale = is_last;
 }
 
 int
-changelog_snap_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+changelog_snap_write_change(changelog_priv_t *priv, char *buffer, size_t len)
 {
-        return changelog_write (priv->c_snap_fd, buffer, len);
+    return changelog_write(priv->c_snap_fd, buffer, len);
 }
 
 int
-changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+changelog_write_change(changelog_priv_t *priv, char *buffer, size_t len)
 {
-        return changelog_write (priv->changelog_fd, buffer, len);
+    return changelog_write(priv->changelog_fd, buffer, len);
 }
 
 /*
@@ -1005,249 +1011,230 @@ changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
  *     -1 : On Failure.
  */
 int
-changelog_snap_handle_ascii_change (xlator_t *this,
-                                    changelog_log_data_t *cld)
+changelog_snap_handle_ascii_change(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off      = 0;
-        size_t            gfid_len = 0;
-        char             *gfid_str = NULL;
-        char             *buffer   = NULL;
-        changelog_priv_t *priv     = NULL;
-        int               ret      = 0;
-
-        if (this == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        priv = this->private;
-
-        if (priv == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        gfid_str = uuid_utoa (cld->cld_gfid);
-        gfid_len = strlen (gfid_str);
-
-        /*  extra bytes for decorations */
-        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_ASCII (priv, buffer,
-                        off, gfid_str, gfid_len, cld);
-
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
-
-        ret = changelog_snap_write_change (priv, buffer, off);
-
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_WRITE_FAILED,
-                        "error writing csnap to disk");
-        }
-        gf_msg (this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO,
-                        "Successfully wrote to csnap");
-        ret = 0;
+    size_t off = 0;
+    size_t gfid_len = 0;
+    char *gfid_str = NULL;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
+    int ret = 0;
+
+    if (this == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    priv = this->private;
+
+    if (priv == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    gfid_str = uuid_utoa(cld->cld_gfid);
+    gfid_len = strlen(gfid_str);
+
+    /*  extra bytes for decorations */
+    buffer = alloca(gfid_len + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_ASCII(priv, buffer, off, gfid_str, gfid_len, cld);
+
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
+
+    ret = changelog_snap_write_change(priv, buffer, off);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_WRITE_FAILED,
+                "csnap", NULL);
+    }
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_WROTE_TO_CSNAP, NULL);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-changelog_handle_change (xlator_t *this,
-                         changelog_priv_t *priv, changelog_log_data_t *cld)
+changelog_handle_change(xlator_t *this, changelog_priv_t *priv,
+                        changelog_log_data_t *cld)
 {
-        int ret = 0;
-
-        if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
-                changelog_encode_change (priv);
-                ret = changelog_start_next_change (this, priv,
-                                                   cld->cld_roll_time,
-                                                   cld->cld_finale);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_GET_TIME_OP_FAILED,
-                                "Problem rolling over changelog(s)");
-                goto out;
-        }
+    int ret = 0;
 
-        /**
-         * case when there is reconfigure done (disabling changelog) and there
-         * are still fops that have updates in prgress.
-         */
-        if (priv->changelog_fd == -1)
-                return 0;
-
-        if (CHANGELOG_TYPE_IS_FSYNC (cld->cld_type)) {
-                ret = sys_fsync (priv->changelog_fd);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSYNC_OP_FAILED,
-                                "fsync failed");
-                }
-                goto out;
-        }
+    if (CHANGELOG_TYPE_IS_ROLLOVER(cld->cld_type)) {
+        changelog_encode_change(priv);
+        ret = changelog_start_next_change(this, priv, cld->cld_roll_time,
+                                          cld->cld_finale);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_GET_TIME_OP_FAILED, NULL);
+        goto out;
+    }
+
+    /**
+     * case when there is reconfigure done (disabling changelog) and there
+     * are still fops that have updates in prgress.
+     */
+    if (priv->changelog_fd == -1)
+        return 0;
 
-        ret = priv->ce->encode (this, cld);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_WRITE_FAILED,
-                        "error writing changelog to disk");
+    if (CHANGELOG_TYPE_IS_FSYNC(cld->cld_type)) {
+        ret = sys_fsync(priv->changelog_fd);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
         }
+        goto out;
+    }
 
- out:
-        return ret;
+    ret = priv->ce->encode(this, cld);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_WRITE_FAILED,
+                "changelog", NULL);
+    }
+
+out:
+    return ret;
 }
 
 changelog_local_t *
-changelog_local_init (xlator_t *this, inode_t *inode,
-                      uuid_t gfid, int xtra_records,
-                      gf_boolean_t update_flag)
+changelog_local_init(xlator_t *this, inode_t *inode, uuid_t gfid,
+                     int xtra_records, gf_boolean_t update_flag)
 {
-        changelog_local_t *local = NULL;
-        struct iobuf      *iobuf = NULL;
+    changelog_local_t *local = NULL;
+    struct iobuf *iobuf = NULL;
 
-        /**
-         * We relax the presence of inode if @update_flag is true.
-         * The caller (implmentation of the fop) needs to be careful to
-         * not blindly use local->inode.
-         */
-        if (!update_flag && !inode) {
-                gf_msg_callingfn (this->name, GF_LOG_WARNING, 0,
-                                  CHANGELOG_MSG_INODE_NOT_FOUND,
-                                  "inode needed for version checking !!!");
-                goto out;
-        }
+    /**
+     * We relax the presence of inode if @update_flag is true.
+     * The caller (implementation of the fop) needs to be careful to
+     * not blindly use local->inode.
+     */
+    if (!update_flag && !inode) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_INODE_NOT_FOUND,
+                         "inode needed for version checking !!!");
 
-        if (xtra_records) {
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
-                                    xtra_records * CHANGELOG_OPT_RECORD_LEN);
-                if (!iobuf)
-                        goto out;
-        }
+        goto out;
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                CHANGELOG_IOBUF_UNREF (iobuf);
-                goto out;
-        }
+    if (xtra_records) {
+        iobuf = iobuf_get2(this->ctx->iobuf_pool,
+                           xtra_records * CHANGELOG_OPT_RECORD_LEN);
+        if (!iobuf)
+            goto out;
+    }
 
-        local->update_no_check = update_flag;
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        CHANGELOG_IOBUF_UNREF(iobuf);
+        goto out;
+    }
 
-        gf_uuid_copy (local->cld.cld_gfid, gfid);
+    local->update_no_check = update_flag;
 
-        local->cld.cld_iobuf = iobuf;
-        local->cld.cld_xtra_records = 0; /* set by the caller */
+    gf_uuid_copy(local->cld.cld_gfid, gfid);
 
-        if (inode)
-                local->inode = inode_ref (inode);
+    local->cld.cld_iobuf = iobuf;
+    local->cld.cld_xtra_records = 0; /* set by the caller */
 
- out:
-        return local;
+    if (inode)
+        local->inode = inode_ref(inode);
+
+out:
+    return local;
 }
 
 int
-changelog_forget (xlator_t *this, inode_t *inode)
+changelog_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t ctx_addr = 0;
-        changelog_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    changelog_inode_ctx_t *ctx = NULL;
 
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
+        return 0;
 
-        ctx = (changelog_inode_ctx_t *) (long) ctx_addr;
-        GF_FREE (ctx);
+    ctx = (changelog_inode_ctx_t *)(long)ctx_addr;
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_inject_single_event (xlator_t *this,
-                               changelog_priv_t *priv,
-                               changelog_log_data_t *cld)
+changelog_inject_single_event(xlator_t *this, changelog_priv_t *priv,
+                              changelog_log_data_t *cld)
 {
-        return priv->cd.dispatchfn (this, priv, priv->cd.cd_data, cld, NULL);
+    return priv->cd.dispatchfn(this, priv, priv->cd.cd_data, cld, NULL);
 }
 
 /* Wait till all the black fops are drained */
 void
-changelog_drain_black_fops (xlator_t *this, changelog_priv_t *priv)
+changelog_drain_black_fops(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-
-        /* clean up framework of pthread_mutex is required here as
-         * 'reconfigure' terminates the changelog_rollover thread
-         * on graph change.
-         */
-        pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                        &priv->dm.drain_black_mutex);
-        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        while (priv->dm.black_fop_cnt > 0) {
-                gf_msg_debug (this->name, 0,
-                              "Condtional wait on black fops: %ld",
-                              priv->dm.black_fop_cnt);
-                priv->dm.drain_wait_black = _gf_true;
-                ret = pthread_cond_wait (&priv->dm.drain_black_cond,
-                                         &priv->dm.drain_black_mutex);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED,
-                                "pthread cond wait failed: Error:%d",
-                                ret);
-        }
-        priv->dm.drain_wait_black = _gf_false;
-        ret = pthread_mutex_unlock (&priv->dm.drain_black_mutex);
+    int ret = 0;
+
+    /* clean up framework of pthread_mutex is required here as
+     * 'reconfigure' terminates the changelog_rollover thread
+     * on graph change.
+     */
+    pthread_cleanup_push(changelog_cleanup_free_mutex,
+                         &priv->dm.drain_black_mutex);
+    ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    while (priv->dm.black_fop_cnt > 0) {
+        gf_msg_debug(this->name, 0, "Conditional wait on black fops: %ld",
+                     priv->dm.black_fop_cnt);
+        priv->dm.drain_wait_black = _gf_true;
+        ret = pthread_cond_wait(&priv->dm.drain_black_cond,
+                                &priv->dm.drain_black_mutex);
         if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        pthread_cleanup_pop (0);
-        gf_msg_debug (this->name, 0,
-                      "Woke up: Conditional wait on black fops");
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, "error=%d", ret,
+                    NULL);
+    }
+    priv->dm.drain_wait_black = _gf_false;
+    ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    pthread_cleanup_pop(0);
+    gf_msg_debug(this->name, 0, "Woke up: Conditional wait on black fops");
 }
 
 /* Wait till all the white  fops are drained */
 void
-changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv)
+changelog_drain_white_fops(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-
-        /* clean up framework of pthread_mutex is required here as
-         * 'reconfigure' terminates the changelog_rollover thread
-         * on graph change.
-         */
-        pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                        &priv->dm.drain_white_mutex);
-        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
+    int ret = 0;
+
+    /* clean up framework of pthread_mutex is required here as
+     * 'reconfigure' terminates the changelog_rollover thread
+     * on graph change.
+     */
+    pthread_cleanup_push(changelog_cleanup_free_mutex,
+                         &priv->dm.drain_white_mutex);
+    ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    while (priv->dm.white_fop_cnt > 0) {
+        gf_msg_debug(this->name, 0, "Conditional wait on white fops : %ld",
+                     priv->dm.white_fop_cnt);
+        priv->dm.drain_wait_white = _gf_true;
+        ret = pthread_cond_wait(&priv->dm.drain_white_cond,
+                                &priv->dm.drain_white_mutex);
         if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        while (priv->dm.white_fop_cnt > 0) {
-                gf_msg_debug (this->name, 0,
-                              "Condtional wait on white fops : %ld",
-                        priv->dm.white_fop_cnt);
-                priv->dm.drain_wait_white = _gf_true;
-                ret = pthread_cond_wait (&priv->dm.drain_white_cond,
-                                         &priv->dm.drain_white_mutex);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED,
-                                "pthread cond wait failed: Error:%d",
-                                ret);
-        }
-        priv->dm.drain_wait_white = _gf_false;
-        ret = pthread_mutex_unlock (&priv->dm.drain_white_mutex);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        pthread_cleanup_pop (0);
-        gf_msg_debug (this->name, 0,
-                      "Woke up: Conditional wait on white fops");
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, "error=%d", ret,
+                    NULL);
+    }
+    priv->dm.drain_wait_white = _gf_false;
+    ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    pthread_cleanup_pop(0);
+    gf_msg_debug(this->name, 0, "Woke up: Conditional wait on white fops");
 }
 
 /**
@@ -1255,193 +1242,194 @@ changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv)
  * a certain time etc..). move them into separate routine.
  */
 void *
-changelog_rollover (void *data)
+changelog_rollover(void *data)
 {
-        int                     ret             = 0;
-        xlator_t               *this            = NULL;
-        struct timespec         tv              = {0,};
-        changelog_log_data_t    cld             = {0,};
-        changelog_time_slice_t *slice           = NULL;
-        changelog_priv_t       *priv            = data;
-
-        this = priv->cr.this;
-        slice = &priv->slice;
-
-        while (1) {
-                (void) pthread_testcancel();
-
-                tv.tv_sec  = time (NULL) + priv->rollover_time;
-                tv.tv_nsec = 0;
-                ret = 0; /* Reset ret to zero */
-
-               /* The race between actual rollover and explicit rollover is
-                * handled. If actual rollover is being done and the
-                * explicit rollover event comes, the event is not missed.
-                * Since explicit rollover sets 'cr.notify' to true, this
-                * thread doesn't wait on 'pthread_cond_timedwait'.
-                */
-                pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                      &priv->cr.lock);
-                pthread_mutex_lock (&priv->cr.lock);
-                {
-                        while (ret == 0 && !priv->cr.notify)
-                                ret = pthread_cond_timedwait (&priv->cr.cond,
-                                                              &priv->cr.lock,
-                                                              &tv);
-                        if (ret == 0)
-                                priv->cr.notify = _gf_false;
-                }
-                pthread_mutex_unlock (&priv->cr.lock);
-                pthread_cleanup_pop (0);
-
-                if (ret == 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BARRIER_INFO,
-                                "Explicit wakeup on barrier notify");
-                        priv->explicit_rollover = _gf_true;
-                } else if (ret && ret != ETIMEDOUT) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_SELECT_FAILED,
-                                "pthread_cond_timedwait failed");
-                        continue;
-                } else if (ret && ret == ETIMEDOUT) {
-                        gf_msg_debug (this->name, 0, "Wokeup on timeout");
-                }
-
-               /* Reading curent_color without lock is fine here
-                * as it is only modified here and is next to reading.
-                */
-                if (priv->current_color == FOP_COLOR_BLACK) {
-                        LOCK(&priv->lock);
-                                priv->current_color = FOP_COLOR_WHITE;
-                        UNLOCK(&priv->lock);
-                        gf_msg_debug (this->name, 0, "Black fops"
-                                      " to be drained:%ld",
-                                      priv->dm.black_fop_cnt);
-                        changelog_drain_black_fops (this, priv);
-                } else {
-                        LOCK(&priv->lock);
-                                priv->current_color = FOP_COLOR_BLACK;
-                        UNLOCK(&priv->lock);
-                        gf_msg_debug (this->name, 0, "White fops"
-                                      " to be drained:%ld",
-                                      priv->dm.white_fop_cnt);
-                        changelog_drain_white_fops (this, priv);
-                }
-
-                /* Adding delay of 1 second only during explicit rollover:
-                 *
-                 * Changelog rollover can happen either due to actual
-                 * or the explict rollover during snapshot. Actual
-                 * rollover is controlled by tuneable called 'rollover-time'.
-                 * The minimum granularity for rollover-time is 1 second.
-                 * Explicit rollover is asynchronous in nature and happens
-                 * during snapshot.
-                 *
-                 * Basically, rollover renames the current CHANGELOG file
-                 * to CHANGELOG.TIMESTAMP. Let's assume, at time 't1',
-                 * actual and explicit rollover raced against  each
-                 * other and actual rollover won the race renaming the
-                 * CHANGELOG file to CHANGELOG.t1 and opens a new
-                 * CHANGELOG file. There is high chance that, an immediate
-                 * explicit rollover at time 't1' can happen with in the same
-                 * second to rename CHANGELOG file to CHANGELOG.t1 resulting in
-                 * purging the earlier CHANGELOG.t1 file created by actual
-                 * rollover. So adding a delay of 1 second guarantees unique
-                 * CHANGELOG.TIMESTAMP during  explicit rollover.
-                 */
-                if (priv->explicit_rollover == _gf_true)
-                        sleep (1);
-
-                ret = changelog_fill_rollover_data (&cld, _gf_false);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_GET_TIME_OP_FAILED,
-                                "failed to fill rollover data");
-                        continue;
-                }
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct timespec tv = {
+        0,
+    };
+    changelog_log_data_t cld = {
+        0,
+    };
+    changelog_time_slice_t *slice = NULL;
+    changelog_priv_t *priv = data;
+
+    this = priv->cr.this;
+    slice = &priv->slice;
+
+    while (1) {
+        (void)pthread_testcancel();
+
+        tv.tv_sec = gf_time() + priv->rollover_time;
+        tv.tv_nsec = 0;
+        ret = 0; /* Reset ret to zero */
+
+        /* The race between actual rollover and explicit rollover is
+         * handled. If actual rollover is being done and the
+         * explicit rollover event comes, the event is not missed.
+         * Since explicit rollover sets 'cr.notify' to true, this
+         * thread doesn't wait on 'pthread_cond_timedwait'.
+         */
+        pthread_cleanup_push(changelog_cleanup_free_mutex, &priv->cr.lock);
+        pthread_mutex_lock(&priv->cr.lock);
+        {
+            while (ret == 0 && !priv->cr.notify)
+                ret = pthread_cond_timedwait(&priv->cr.cond, &priv->cr.lock,
+                                             &tv);
+            if (ret == 0)
+                priv->cr.notify = _gf_false;
+        }
+        pthread_mutex_unlock(&priv->cr.lock);
+        pthread_cleanup_pop(0);
+
+        if (ret == 0) {
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_BARRIER_INFO,
+                    NULL);
+            priv->explicit_rollover = _gf_true;
+        } else if (ret && ret != ETIMEDOUT) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_SELECT_FAILED, NULL);
+            continue;
+        } else if (ret && ret == ETIMEDOUT) {
+            gf_msg_debug(this->name, 0, "Wokeup on timeout");
+        }
+
+        /* Reading curent_color without lock is fine here
+         * as it is only modified here and is next to reading.
+         */
+        if (priv->current_color == FOP_COLOR_BLACK) {
+            LOCK(&priv->lock);
+            priv->current_color = FOP_COLOR_WHITE;
+            UNLOCK(&priv->lock);
+            gf_msg_debug(this->name, 0,
+                         "Black fops"
+                         " to be drained:%ld",
+                         priv->dm.black_fop_cnt);
+            changelog_drain_black_fops(this, priv);
+        } else {
+            LOCK(&priv->lock);
+            priv->current_color = FOP_COLOR_BLACK;
+            UNLOCK(&priv->lock);
+            gf_msg_debug(this->name, 0,
+                         "White fops"
+                         " to be drained:%ld",
+                         priv->dm.white_fop_cnt);
+            changelog_drain_white_fops(this, priv);
+        }
+
+        /* Adding delay of 1 second only during explicit rollover:
+         *
+         * Changelog rollover can happen either due to actual
+         * or the explicit rollover during snapshot. Actual
+         * rollover is controlled by tuneable called 'rollover-time'.
+         * The minimum granularity for rollover-time is 1 second.
+         * Explicit rollover is asynchronous in nature and happens
+         * during snapshot.
+         *
+         * Basically, rollover renames the current CHANGELOG file
+         * to CHANGELOG.TIMESTAMP. Let's assume, at time 't1',
+         * actual and explicit rollover raced against  each
+         * other and actual rollover won the race renaming the
+         * CHANGELOG file to CHANGELOG.t1 and opens a new
+         * CHANGELOG file. There is high chance that, an immediate
+         * explicit rollover at time 't1' can happen with in the same
+         * second to rename CHANGELOG file to CHANGELOG.t1 resulting in
+         * purging the earlier CHANGELOG.t1 file created by actual
+         * rollover. So adding a delay of 1 second guarantees unique
+         * CHANGELOG.TIMESTAMP during  explicit rollover.
+         */
+        if (priv->explicit_rollover == _gf_true)
+            sleep(1);
 
-                _mask_cancellation ();
+        changelog_fill_rollover_data(&cld, _gf_false);
 
-                LOCK (&priv->lock);
-                {
-                        ret = changelog_inject_single_event (this, priv, &cld);
-                        if (!ret)
-                                SLICE_VERSION_UPDATE (slice);
-                }
-                UNLOCK (&priv->lock);
+        _mask_cancellation();
 
-                _unmask_cancellation ();
+        LOCK(&priv->lock);
+        {
+            ret = changelog_inject_single_event(this, priv, &cld);
+            if (!ret)
+                SLICE_VERSION_UPDATE(slice);
         }
+        UNLOCK(&priv->lock);
 
-        return NULL;
+        _unmask_cancellation();
+    }
+
+    return NULL;
 }
 
 void *
-changelog_fsync_thread (void *data)
+changelog_fsync_thread(void *data)
 {
-        int                   ret  = 0;
-        xlator_t             *this = NULL;
-        struct timeval        tv   = {0,};
-        changelog_log_data_t  cld  = {0,};
-        changelog_priv_t     *priv = data;
-
-        this = priv->cf.this;
-        cld.cld_type = CHANGELOG_TYPE_FSYNC;
-
-        while (1) {
-                (void) pthread_testcancel();
-
-                tv.tv_sec  = priv->fsync_interval;
-                tv.tv_usec = 0;
-
-                ret = select (0, NULL, NULL, NULL, &tv);
-                if (ret)
-                        continue;
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct timeval tv = {
+        0,
+    };
+    changelog_log_data_t cld = {
+        0,
+    };
+    changelog_priv_t *priv = data;
+
+    this = priv->cf.this;
+    cld.cld_type = CHANGELOG_TYPE_FSYNC;
+
+    while (1) {
+        (void)pthread_testcancel();
+
+        tv.tv_sec = priv->fsync_interval;
+        tv.tv_usec = 0;
+
+        ret = select(0, NULL, NULL, NULL, &tv);
+        if (ret)
+            continue;
 
-                _mask_cancellation ();
+        _mask_cancellation();
 
-                ret = changelog_inject_single_event (this, priv, &cld);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_INJECT_FSYNC_FAILED,
-                                "failed to inject fsync event");
+        ret = changelog_inject_single_event(this, priv, &cld);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_INJECT_FSYNC_FAILED, NULL);
 
-                _unmask_cancellation ();
-        }
+        _unmask_cancellation();
+    }
 
-        return NULL;
+    return NULL;
 }
 
 /* macros for inode/changelog version checks */
 
-#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type) do {       \
-                LOCK (&inode->lock);                                    \
-                {                                                       \
-                        LOCK (&priv->lock);                             \
-                        {                                               \
-                                *iver = slice->changelog_version[type]; \
-                        }                                               \
-                        UNLOCK (&priv->lock);                           \
-                }                                                       \
-                UNLOCK (&inode->lock);                                  \
-        } while (0)
-
-#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd) do {    \
-                LOCK (&priv->lock);                                     \
-                {                                                       \
-                        upd = (ver == slice->changelog_version[type])   \
-                                ? _gf_false : _gf_true;                 \
-                }                                                       \
-                UNLOCK (&priv->lock);                                   \
-        } while (0)
+#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type)                   \
+    do {                                                                       \
+        LOCK(&inode->lock);                                                    \
+        {                                                                      \
+            LOCK(&priv->lock);                                                 \
+            {                                                                  \
+                *iver = slice->changelog_version[type];                        \
+            }                                                                  \
+            UNLOCK(&priv->lock);                                               \
+        }                                                                      \
+        UNLOCK(&inode->lock);                                                  \
+    } while (0)
+
+#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd)                \
+    do {                                                                       \
+        LOCK(&priv->lock);                                                     \
+        {                                                                      \
+            upd = (ver == slice->changelog_version[type]) ? _gf_false          \
+                                                          : _gf_true;          \
+        }                                                                      \
+        UNLOCK(&priv->lock);                                                   \
+    } while (0)
 
 static int
-__changelog_inode_ctx_set (xlator_t *this,
-                           inode_t *inode, changelog_inode_ctx_t *ctx)
+__changelog_inode_ctx_set(xlator_t *this, inode_t *inode,
+                          changelog_inode_ctx_t *ctx)
 {
-        uint64_t ctx_addr = (uint64_t) ctx;
-        return __inode_ctx_set (inode, this, &ctx_addr);
+    uint64_t ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return __inode_ctx_set(inode, this, &ctx_addr);
 }
 
 /**
@@ -1449,56 +1437,53 @@ __changelog_inode_ctx_set (xlator_t *this,
  * for a particular type.
  */
 changelog_inode_ctx_t *
-__changelog_inode_ctx_get (xlator_t *this,
-                           inode_t *inode, unsigned long **iver,
-                           unsigned long *version, changelog_log_type type)
+__changelog_inode_ctx_get(xlator_t *this, inode_t *inode, unsigned long **iver,
+                          unsigned long *version, changelog_log_type type)
 {
-        int                    ret      = 0;
-        uint64_t               ctx_addr = 0;
-        changelog_inode_ctx_t *ctx      = NULL;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret < 0)
-                ctx_addr = 0;
-        if (ctx_addr != 0) {
-                ctx = (changelog_inode_ctx_t *) (long)ctx_addr;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_changelog_mt_inode_ctx_t);
-        if (!ctx)
-                goto out;
+    int ret = 0;
+    uint64_t ctx_addr = 0;
+    changelog_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret < 0)
+        ctx_addr = 0;
+    if (ctx_addr != 0) {
+        ctx = (changelog_inode_ctx_t *)(long)ctx_addr;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_changelog_mt_inode_ctx_t);
+    if (!ctx)
+        goto out;
+
+    ret = __changelog_inode_ctx_set(this, inode, ctx);
+    if (ret) {
+        GF_FREE(ctx);
+        ctx = NULL;
+    }
 
-        ret = __changelog_inode_ctx_set (this, inode, ctx);
-        if (ret) {
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
-
- out:
-        if (ctx && iver && version) {
-                *iver = CHANGELOG_INODE_VERSION_TYPE (ctx, type);
-                *version = **iver;
-        }
+out:
+    if (ctx && iver && version) {
+        *iver = CHANGELOG_INODE_VERSION_TYPE(ctx, type);
+        *version = **iver;
+    }
 
-        return ctx;
+    return ctx;
 }
 
 static changelog_inode_ctx_t *
-changelog_inode_ctx_get (xlator_t *this,
-                         inode_t *inode, unsigned long **iver,
-                         unsigned long *version, changelog_log_type type)
+changelog_inode_ctx_get(xlator_t *this, inode_t *inode, unsigned long **iver,
+                        unsigned long *version, changelog_log_type type)
 {
-        changelog_inode_ctx_t *ctx = NULL;
+    changelog_inode_ctx_t *ctx = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ctx = __changelog_inode_ctx_get (this,
-                                                 inode, iver, version, type);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ctx = __changelog_inode_ctx_get(this, inode, iver, version, type);
+    }
+    UNLOCK(&inode->lock);
 
-        return ctx;
+    return ctx;
 }
 
 /**
@@ -1602,59 +1587,57 @@ changelog_inode_ctx_get (xlator_t *this,
  * signifies an update was recorded in the current time slice).
  */
 void
-changelog_update (xlator_t *this, changelog_priv_t *priv,
-                  changelog_local_t *local, changelog_log_type type)
+changelog_update(xlator_t *this, changelog_priv_t *priv,
+                 changelog_local_t *local, changelog_log_type type)
 {
-        int                     ret        = 0;
-        unsigned long          *iver       = NULL;
-        unsigned long           version    = 0;
-        inode_t                *inode      = NULL;
-        changelog_time_slice_t *slice      = NULL;
-        changelog_inode_ctx_t  *ctx        = NULL;
-        changelog_log_data_t   *cld_0      = NULL;
-        changelog_log_data_t   *cld_1      = NULL;
-        changelog_local_t      *next_local = NULL;
-        gf_boolean_t            need_upd   = _gf_true;
-
-        slice = &priv->slice;
-
-        /**
-         * for fops that do not require inode version checking
-         */
-        if (local->update_no_check)
-                goto update;
+    int ret = 0;
+    unsigned long *iver = NULL;
+    unsigned long version = 0;
+    inode_t *inode = NULL;
+    changelog_time_slice_t *slice = NULL;
+    changelog_inode_ctx_t *ctx = NULL;
+    changelog_log_data_t *cld_0 = NULL;
+    changelog_log_data_t *cld_1 = NULL;
+    changelog_local_t *next_local = NULL;
+    gf_boolean_t need_upd = _gf_true;
 
-        inode = local->inode;
+    slice = &priv->slice;
 
-        ctx = changelog_inode_ctx_get (this,
-                                       inode, &iver, &version, type);
-        if (!ctx)
-                goto update;
+    /**
+     * for fops that do not require inode version checking
+     */
+    if (local->update_no_check)
+        goto update;
 
-        INODE_VERSION_EQUALS_SLICE (priv, version, slice, type, need_upd);
+    inode = local->inode;
 
- update:
-        if (need_upd) {
-                cld_0 = &local->cld;
-                cld_0->cld_type = type;
+    ctx = changelog_inode_ctx_get(this, inode, &iver, &version, type);
+    if (!ctx)
+        goto update;
 
-                if ( (next_local = local->prev_entry) != NULL ) {
-                        cld_1 = &next_local->cld;
-                        cld_1->cld_type = type;
-                }
+    INODE_VERSION_EQUALS_SLICE(priv, version, slice, type, need_upd);
 
-                ret = priv->cd.dispatchfn (this, priv,
-                                           priv->cd.cd_data, cld_0, cld_1);
+update:
+    if (need_upd) {
+        cld_0 = &local->cld;
+        cld_0->cld_type = type;
 
-                /**
-                 * update after the dispatcher has successfully done
-                 * it's job.
-                 */
-                if (!local->update_no_check && iver && !ret)
-                        INODE_VERSION_UPDATE (priv, inode, iver, slice, type);
+        if ((next_local = local->prev_entry) != NULL) {
+            cld_1 = &next_local->cld;
+            cld_1->cld_type = type;
         }
 
-        return;
+        ret = priv->cd.dispatchfn(this, priv, priv->cd.cd_data, cld_0, cld_1);
+
+        /**
+         * update after the dispatcher has successfully done
+         * it's job.
+         */
+        if (!local->update_no_check && iver && !ret)
+            INODE_VERSION_UPDATE(priv, inode, iver, slice, type);
+    }
+
+    return;
 }
 
 /* Begin: Geo-rep snapshot dependency changes */
@@ -1670,226 +1653,221 @@ changelog_update (xlator_t *this, changelog_priv_t *priv,
  */
 
 void
-changelog_color_fop_and_inc_cnt (xlator_t *this, changelog_priv_t *priv,
-                                                 changelog_local_t *local)
+changelog_color_fop_and_inc_cnt(xlator_t *this, changelog_priv_t *priv,
+                                changelog_local_t *local)
 {
-        if (!priv || !local)
-                return;
+    if (!priv || !local)
+        return;
 
-        LOCK (&priv->lock);
-        {
-                local->color = priv->current_color;
-                changelog_inc_fop_cnt (this, priv, local);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        local->color = priv->current_color;
+        changelog_inc_fop_cnt(this, priv, local);
+    }
+    UNLOCK(&priv->lock);
 }
 
 /* Increments the respective fop counter based on the fop color */
 void
-changelog_inc_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local)
+changelog_inc_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local)
 {
-        int ret = 0;
-
-        if (local) {
-                if (local->color == FOP_COLOR_BLACK) {
-                        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.black_fop_cnt++;
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                } else {
-                        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.white_fop_cnt++;
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                }
-        }
- out:
-        return;
+    int ret = 0;
+
+    if (local) {
+        if (local->color == FOP_COLOR_BLACK) {
+            ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.black_fop_cnt++;
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        } else {
+            ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.white_fop_cnt++;
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        }
+    }
+out:
+    return;
 }
 
 /* Decrements the respective fop counter based on the fop color */
 void
-changelog_dec_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local)
+changelog_dec_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local)
 {
-        int ret = 0;
-
-        if (local) {
-                if (local->color == FOP_COLOR_BLACK) {
-                        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.black_fop_cnt--;
-                                if (priv->dm.black_fop_cnt == 0 &&
-                                    priv->dm.drain_wait_black == _gf_true) {
-                                        ret = pthread_cond_signal (
-                                                    &priv->dm.drain_black_cond);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret,
-                                                                           out);
-                                        gf_msg_debug (this->name, 0,
-                                                      "Signalled "
-                                                      "draining of black");
-                                }
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                } else {
-                        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.white_fop_cnt--;
-                                if (priv->dm.white_fop_cnt == 0 &&
-                                    priv->dm.drain_wait_white == _gf_true) {
-                                        ret = pthread_cond_signal (
-                                                    &priv->dm.drain_white_cond);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret,
-                                                                           out);
-                                        gf_msg_debug (this->name, 0,
-                                                      "Signalled "
-                                                      "draining of white");
-                                }
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
+    int ret = 0;
+
+    if (local) {
+        if (local->color == FOP_COLOR_BLACK) {
+            ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.black_fop_cnt--;
+                if (priv->dm.black_fop_cnt == 0 &&
+                    priv->dm.drain_wait_black == _gf_true) {
+                    ret = pthread_cond_signal(&priv->dm.drain_black_cond);
+                    CHANGELOG_PTHREAD_ERROR_HANDLE_2(
+                        ret, out, priv->dm.drain_black_mutex);
+                    gf_msg_debug(this->name, 0,
+                                 "Signalled "
+                                 "draining of black");
                 }
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        } else {
+            ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.white_fop_cnt--;
+                if (priv->dm.white_fop_cnt == 0 &&
+                    priv->dm.drain_wait_white == _gf_true) {
+                    ret = pthread_cond_signal(&priv->dm.drain_white_cond);
+                    CHANGELOG_PTHREAD_ERROR_HANDLE_2(
+                        ret, out, priv->dm.drain_white_mutex);
+                    gf_msg_debug(this->name, 0,
+                                 "Signalled "
+                                 "draining of white");
+                }
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
         }
- out:
-        return;
+    }
+out:
+    return;
 }
 
 /* Write to a pipe setup between changelog main thread and changelog
  * rollover thread to initiate explicit rollover of changelog journal.
  */
 int
-changelog_barrier_notify (changelog_priv_t *priv, char *buf)
+changelog_barrier_notify(changelog_priv_t *priv, char *buf)
 {
-        int ret = 0;
-
-        pthread_mutex_lock (&priv->cr.lock);
-        {
-                ret = pthread_cond_signal (&priv->cr.cond);
-                priv->cr.notify = _gf_true;
-        }
-        pthread_mutex_unlock (&priv->cr.lock);
-        return ret;
+    int ret = 0;
+
+    pthread_mutex_lock(&priv->cr.lock);
+    {
+        ret = pthread_cond_signal(&priv->cr.cond);
+        priv->cr.notify = _gf_true;
+    }
+    pthread_mutex_unlock(&priv->cr.lock);
+    return ret;
 }
 
 /* Clean up flags set on barrier notification */
 void
-changelog_barrier_cleanup (xlator_t *this, changelog_priv_t *priv,
-                                                struct list_head *queue)
+changelog_barrier_cleanup(xlator_t *this, changelog_priv_t *priv,
+                          struct list_head *queue)
 {
-        int ret = 0;
-
-        LOCK (&priv->bflags.lock);
-                priv->bflags.barrier_ext = _gf_false;
-        UNLOCK (&priv->bflags.lock);
-
-        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-        {
-                priv->bn.bnotify = _gf_false;
-        }
-        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-
-        /* Disable changelog barrier and dequeue fops */
-        LOCK (&priv->lock);
-        {
-                if (priv->barrier_enabled == _gf_true)
-                        __chlog_barrier_disable (this, queue);
-                else
-                        ret = -1;
-        }
-        UNLOCK (&priv->lock);
-        if (ret == 0)
-                chlog_barrier_dequeue_all(this, queue);
+    int ret = 0;
+
+    LOCK(&priv->bflags.lock);
+    priv->bflags.barrier_ext = _gf_false;
+    UNLOCK(&priv->bflags.lock);
+
+    ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+    CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+    {
+        priv->bn.bnotify = _gf_false;
+    }
+    ret = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+    CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+
+    /* Disable changelog barrier and dequeue fops */
+    LOCK(&priv->lock);
+    {
+        if (priv->barrier_enabled == _gf_true)
+            __chlog_barrier_disable(this, queue);
+        else
+            ret = -1;
+    }
+    UNLOCK(&priv->lock);
+    if (ret == 0)
+        chlog_barrier_dequeue_all(this, queue);
 
- out:
-        return;
+out:
+    return;
 }
 /* End: Geo-Rep snapshot dependency changes */
 
 int32_t
-changelog_fill_entry_buf (call_frame_t *frame, xlator_t *this,
-                          loc_t *loc, changelog_local_t **local)
+changelog_fill_entry_buf(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         changelog_local_t **local)
 {
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-        char             *dup_path = NULL;
-        char             *bname    = NULL;
-        inode_t          *parent   = NULL;
-
-        GF_ASSERT (this);
-
-        parent = inode_parent (loc->inode, 0, 0);
-        if (!parent) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_INODE_NOT_FOUND, "Parent inode not found"
-                        " for gfid: %s", uuid_utoa (loc->inode->gfid));
-                goto err;
-        }
-
-        CHANGELOG_INIT_NOCHECK (this, *local, loc->inode, loc->inode->gfid, 5);
-        if (!(*local)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_LOCAL_INIT_FAILED, "changelog local"
-                        " initiatilization failed");
-                goto err;
-        }
-
-        co = changelog_get_usable_buffer (*local);
-        if (!co) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to get buffer");
-                goto err;
-        }
-
-        if (loc->inode->ia_type == IA_IFDIR) {
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_MKDIR, fop_fn, xtra_len);
-                co++;
-                CHANGELOG_FILL_UINT32 (co, S_IFDIR|0755, number_fn, xtra_len);
-                co++;
-        } else {
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_CREATE, fop_fn, xtra_len);
-                co++;
-                CHANGELOG_FILL_UINT32 (co, S_IFREG|0644, number_fn, xtra_len);
-                co++;
-        }
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    char *dup_path = NULL;
+    char *bname = NULL;
+    inode_t *parent = NULL;
+
+    GF_ASSERT(this);
+
+    parent = inode_parent(loc->inode, 0, 0);
+    if (!parent) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_INODE_NOT_FOUND,
+                "type=parent", "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto err;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, *local, loc->inode, loc->inode->gfid, 5);
+    if (!(*local)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_LOCAL_INIT_FAILED,
+                NULL);
+        goto err;
+    }
+
+    co = changelog_get_usable_buffer(*local);
+    if (!co) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_GET_BUFFER_FAILED,
+                NULL);
+        goto err;
+    }
+
+    if (loc->inode->ia_type == IA_IFDIR) {
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_MKDIR, fop_fn, xtra_len);
         co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
+        CHANGELOG_FILL_UINT32(co, S_IFDIR | 0755, number_fn, xtra_len);
         co++;
+    } else {
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_CREATE, fop_fn, xtra_len);
+        co++;
+        CHANGELOG_FILL_UINT32(co, S_IFREG | 0644, number_fn, xtra_len);
+        co++;
+    }
 
-        dup_path = gf_strdup (loc->path);
-        bname = basename (dup_path);
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
 
-        CHANGELOG_FILL_ENTRY (co, parent->gfid, bname, entry_fn, entry_free_fn,
-                              xtra_len, err);
-        changelog_set_usable_record_and_length (*local, xtra_len, 5);
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
 
-        if (dup_path)
-                GF_FREE (dup_path);
-        if (parent)
-                inode_unref (parent);
-        return 0;
+    dup_path = gf_strdup(loc->path);
+    bname = basename(dup_path);
+
+    CHANGELOG_FILL_ENTRY(co, parent->gfid, bname, entry_fn, entry_free_fn,
+                         xtra_len, err);
+    changelog_set_usable_record_and_length(*local, xtra_len, 5);
+
+    if (dup_path)
+        GF_FREE(dup_path);
+    if (parent)
+        inode_unref(parent);
+    return 0;
 
 err:
-        if (dup_path)
-                GF_FREE (dup_path);
-        if (parent)
-                inode_unref (parent);
-        return -1;
+    if (dup_path)
+        GF_FREE(dup_path);
+    if (parent)
+        inode_unref(parent);
+    return -1;
 }
 
 /*
@@ -1902,78 +1880,98 @@ err:
  */
 
 int
-resolve_pargfid_to_path (xlator_t *this, const uuid_t pgfid,
-                         char **path, char *bname)
+resolve_pargfid_to_path(xlator_t *this, const uuid_t pgfid, char **path,
+                        char *bname)
 {
-        char             *linkname                  = NULL;
-        char             *dir_handle                = NULL;
-        char             *pgfidstr                  = NULL;
-        char             *saveptr                   = NULL;
-        ssize_t           len                       = 0;
-        int               ret                       = 0;
-        uuid_t            tmp_gfid                  = {0, };
-        uuid_t            pargfid                   = {0, };
-        changelog_priv_t *priv                      = NULL;
-        char              gpath[PATH_MAX]           = {0,};
-        char              result[PATH_MAX]          = {0,};
-        char             *dir_name                  = NULL;
-        char              pre_dir_name[PATH_MAX]    = {0,};
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        gf_uuid_copy (pargfid, pgfid);
-        if (!path || gf_uuid_is_null (pargfid)) {
-                ret = -1;
-                goto out;
-        }
+    char *linkname = NULL;
+    char *dir_handle = NULL;
+    char *pgfidstr = NULL;
+    char *saveptr = NULL;
+    ssize_t len = 0;
+    int ret = 0;
+    uuid_t tmp_gfid = {
+        0,
+    };
+    uuid_t pargfid = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
+    char gpath[PATH_MAX] = {
+        0,
+    };
+    char result[PATH_MAX] = {
+        0,
+    };
+    char *dir_name = NULL;
+    char pre_dir_name[PATH_MAX] = {
+        0,
+    };
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    gf_uuid_copy(pargfid, pgfid);
+    if (!path || gf_uuid_is_null(pargfid)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (__is_root_gfid(pargfid)) {
+        if (bname)
+            *path = gf_strdup(bname);
+        else
+            *path = gf_strdup(".");
+        return ret;
+    }
 
-        if (__is_root_gfid (pargfid)) {
-                if (bname)
-                        *path = gf_strdup (bname);
-                else
-                        *path = gf_strdup (".");
-                return ret;
-        }
+    dir_handle = alloca(PATH_MAX);
+    linkname = alloca(PATH_MAX);
+    (void)snprintf(gpath, PATH_MAX, "%s/.glusterfs/", priv->changelog_brick);
 
-        dir_handle = alloca (PATH_MAX);
-        linkname   = alloca (PATH_MAX);
-        (void) snprintf (gpath, PATH_MAX, "%s/.glusterfs/",
-                         priv->changelog_brick);
-
-        while (!(__is_root_gfid (pargfid))) {
-                snprintf (dir_handle, PATH_MAX, "%s/%02x/%02x/%s", gpath,
-                          pargfid[0], pargfid[1], uuid_utoa (pargfid));
-
-                len = sys_readlink (dir_handle, linkname, PATH_MAX);
-                if (len < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_READLINK_OP_FAILED,
-                                "could not read the "
-                                "link from the gfid handle %s", dir_handle);
-                        ret = -1;
-                        goto out;
-                }
+    while (!(__is_root_gfid(pargfid))) {
+        len = snprintf(dir_handle, PATH_MAX, "%s/%02x/%02x/%s", gpath,
+                       pargfid[0], pargfid[1], uuid_utoa(pargfid));
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
+        }
 
-                linkname[len] = '\0';
+        len = sys_readlink(dir_handle, linkname, PATH_MAX);
+        if (len < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_READLINK_OP_FAILED,
+                    "could not read the "
+                    "link from the gfid handle",
+                    "handle=%s", dir_handle, NULL);
+            ret = -1;
+            goto out;
+        }
 
-                pgfidstr = strtok_r (linkname + strlen("../../00/00/"), "/",
-                                     &saveptr);
-                dir_name = strtok_r (NULL, "/", &saveptr);
+        linkname[len] = '\0';
 
-                snprintf (result, PATH_MAX, "%s/%s", dir_name, pre_dir_name);
-                strncpy (pre_dir_name, result, sizeof(pre_dir_name));
+        pgfidstr = strtok_r(linkname + strlen("../../00/00/"), "/", &saveptr);
+        dir_name = strtok_r(NULL, "/", &saveptr);
 
-                gf_uuid_parse (pgfidstr, tmp_gfid);
-                gf_uuid_copy (pargfid, tmp_gfid);
+        len = snprintf(result, PATH_MAX, "%s/%s", dir_name, pre_dir_name);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
+        }
+        if (snprintf(pre_dir_name, len + 1, "%s", result) >= len + 1) {
+            ret = -1;
+            goto out;
         }
 
-        if (bname)
-                strncat (result, bname, strlen(bname) + 1);
+        gf_uuid_parse(pgfidstr, tmp_gfid);
+        gf_uuid_copy(pargfid, tmp_gfid);
+    }
 
-        *path = gf_strdup (result);
+    if (bname)
+        strncat(result, bname, strlen(bname) + 1);
+
+    *path = gf_strdup(result);
 
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
index 4fdba244aa1..38fa7590c32 100644
--- a/xlators/features/changelog/src/changelog-helpers.h
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -11,14 +11,14 @@
 #ifndef _CHANGELOG_HELPERS_H
 #define _CHANGELOG_HELPERS_H
 
-#include "locking.h"
-#include "timer.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/timer.h>
 #include "pthread.h"
-#include "iobuf.h"
-#include "rot-buffs.h"
+#include <glusterfs/iobuf.h>
+#include <glusterfs/rot-buffs.h>
 
 #include "changelog-misc.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 
 #include "rpcsvc.h"
 #include "changelog-ev-handle.h"
@@ -30,44 +30,44 @@
  * the changelog entry
  */
 typedef struct changelog_log_data {
-        /* rollover related */
-        unsigned long cld_roll_time;
+    /* rollover related */
+    time_t cld_roll_time;
 
-        /* reopen changelog? */
-        gf_boolean_t cld_finale;
+    /* reopen changelog? */
+    gf_boolean_t cld_finale;
 
-        changelog_log_type cld_type;
+    changelog_log_type cld_type;
 
-        /**
-         * sincd gfid is _always_ a necessity, it's not a part
-         * of the iobuf. by doing this we do not add any overhead
-         * for data and metadata related fops.
-         */
-        uuid_t        cld_gfid;
+    /**
+     * sincd gfid is _always_ a necessity, it's not a part
+     * of the iobuf. by doing this we do not add any overhead
+     * for data and metadata related fops.
+     */
+    uuid_t cld_gfid;
 
-        /**
-         * iobufs are used for optionals records: pargfid, path,
-         * write offsets etc.. It's the fop implementers job
-         * to allocate (iobuf_get() in the fop) and get unref'ed
-         * in the callback (CHANGELOG_STACK_UNWIND).
-         */
-        struct iobuf *cld_iobuf;
+    /**
+     * iobufs are used for optionals records: pargfid, path,
+     * write offsets etc.. It's the fop implementers job
+     * to allocate (iobuf_get() in the fop) and get unref'ed
+     * in the callback (CHANGELOG_STACK_UNWIND).
+     */
+    struct iobuf *cld_iobuf;
 
 #define cld_ptr cld_iobuf->ptr
 
-        /**
-         * after allocation you can point this to the length of
-         * usable data, but make sure it does not exceed the
-         * the size of the requested iobuf.
-         */
-        size_t        cld_iobuf_len;
+    /**
+     * after allocation you can point this to the length of
+     * usable data, but make sure it does not exceed the
+     * the size of the requested iobuf.
+     */
+    size_t cld_iobuf_len;
 
 #define cld_ptr_len cld_iobuf_len
 
-        /**
-         * number of optional records
-         */
-        int cld_xtra_records;
+    /**
+     * number of optional records
+     */
+    int cld_xtra_records;
 } changelog_log_data_t;
 
 /**
@@ -77,55 +77,48 @@ typedef struct changelog_log_data {
 typedef struct changelog_priv changelog_priv_t;
 
 typedef struct changelog_dispatcher {
-        void *cd_data;
-        int (*dispatchfn) (xlator_t *, changelog_priv_t *, void *,
-                           changelog_log_data_t *, changelog_log_data_t *);
+    void *cd_data;
+    int (*dispatchfn)(xlator_t *, changelog_priv_t *, void *,
+                      changelog_log_data_t *, changelog_log_data_t *);
 } changelog_dispatcher_t;
 
 struct changelog_bootstrap {
-        changelog_mode_t mode;
-        int (*ctor) (xlator_t *, changelog_dispatcher_t *);
-        int (*dtor) (xlator_t *, changelog_dispatcher_t *);
+    changelog_mode_t mode;
+    int (*ctor)(xlator_t *, changelog_dispatcher_t *);
+    int (*dtor)(xlator_t *, changelog_dispatcher_t *);
 };
 
 struct changelog_encoder {
-        changelog_encoder_t encoder;
-        int (*encode) (xlator_t *, changelog_log_data_t *);
+    changelog_encoder_t encoder;
+    int (*encode)(xlator_t *, changelog_log_data_t *);
 };
 
-
 /* xlator private */
 
 typedef struct changelog_time_slice {
-        /**
-         * just in case we need nanosecond granularity some day.
-         * field is unused as of now (maybe we'd need it later).
-         */
-        struct timeval tv_start;
-
-        /**
-         * version of changelog file, incremented each time changes
-         * rollover.
-         */
-        unsigned long changelog_version[CHANGELOG_MAX_TYPE];
+    /**
+     * version of changelog file, incremented each time changes
+     * rollover.
+     */
+    unsigned long changelog_version[CHANGELOG_MAX_TYPE];
 } changelog_time_slice_t;
 
 typedef struct changelog_rollover {
-        /* rollover thread */
-        pthread_t rollover_th;
+    /* rollover thread */
+    pthread_t rollover_th;
 
-        xlator_t *this;
+    xlator_t *this;
 
-        pthread_mutex_t lock;
-        pthread_cond_t cond;
-        gf_boolean_t notify;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+    gf_boolean_t notify;
 } changelog_rollover_t;
 
 typedef struct changelog_fsync {
-        /* fsync() thread */
-        pthread_t fsync_th;
+    /* fsync() thread */
+    pthread_t fsync_th;
 
-        xlator_t *this;
+    xlator_t *this;
 } changelog_fsync_t;
 
 /* Draining during changelog rollover (for geo-rep snapshot dependency):
@@ -145,198 +138,219 @@ typedef struct changelog_fsync {
  */
 
 typedef enum chlog_fop_color {
-         FOP_COLOR_BLACK,
-         FOP_COLOR_WHITE
+    FOP_COLOR_BLACK,
+    FOP_COLOR_WHITE
 } chlog_fop_color_t;
 
 /* Barrier notify variable */
 typedef struct barrier_notify {
-         pthread_mutex_t        bnotify_mutex;
-         pthread_cond_t         bnotify_cond;
-         gf_boolean_t           bnotify;
-         gf_boolean_t           bnotify_error;
+    pthread_mutex_t bnotify_mutex;
+    pthread_cond_t bnotify_cond;
+    gf_boolean_t bnotify;
+    gf_boolean_t bnotify_error;
 } barrier_notify_t;
 
 /* Two separate mutex and conditional variable set is used
  * to drain white and black fops. */
 
 typedef struct drain_mgmt {
-         pthread_mutex_t        drain_black_mutex;
-         pthread_cond_t         drain_black_cond;
-         pthread_mutex_t        drain_white_mutex;
-         pthread_cond_t         drain_white_cond;
-         /* Represents black fops count in-transit */
-         unsigned long          black_fop_cnt;
-         /* Represents white fops count in-transit */
-         unsigned long          white_fop_cnt;
-         gf_boolean_t           drain_wait_black;
-         gf_boolean_t           drain_wait_white;
+    pthread_mutex_t drain_black_mutex;
+    pthread_cond_t drain_black_cond;
+    pthread_mutex_t drain_white_mutex;
+    pthread_cond_t drain_white_cond;
+    /* Represents black fops count in-transit */
+    unsigned long black_fop_cnt;
+    /* Represents white fops count in-transit */
+    unsigned long white_fop_cnt;
+    gf_boolean_t drain_wait_black;
+    gf_boolean_t drain_wait_white;
 } drain_mgmt_t;
 
 /* External barrier as a result of snap on/off indicating flag*/
 typedef struct barrier_flags {
-        gf_lock_t lock;
-        gf_boolean_t barrier_ext;
+    gf_lock_t lock;
+    gf_boolean_t barrier_ext;
 } barrier_flags_t;
 
 /* Event selection */
 typedef struct changelog_ev_selector {
-        gf_lock_t reflock;
+    gf_lock_t reflock;
 
-        /**
-         * Array of references for each selection bit.
-         */
-        unsigned int ref[CHANGELOG_EV_SELECTION_RANGE];
+    /**
+     * Array of references for each selection bit.
+     */
+    unsigned int ref[CHANGELOG_EV_SELECTION_RANGE];
 } changelog_ev_selector_t;
 
-
 /* changelog's private structure */
 struct changelog_priv {
-        gf_boolean_t active;
+    /* changelog journalling */
+    gf_boolean_t active;
+
+    /* changelog live notifications */
+    gf_boolean_t rpc_active;
+
+    /* to generate unique socket file per brick */
+    char *changelog_brick;
+
+    /* logging directory */
+    char *changelog_dir;
 
-        /* to generate unique socket file per brick */
-        char *changelog_brick;
+    /* htime directory */
+    char *htime_dir;
 
-        /* logging directory */
-        char *changelog_dir;
+    /* one file for all changelog types */
+    int changelog_fd;
 
-        /* htime directory */
-        char *htime_dir;
+    /* htime fd for current changelog session */
+    int htime_fd;
 
-        /* one file for all changelog types */
-        int changelog_fd;
+    /*  c_snap_fd is fd for call-path changelog */
+    int c_snap_fd;
 
-        /* htime fd for current changelog session */
-        int htime_fd;
+    /* rollover_count used by htime */
+    int rollover_count;
 
-        /*  c_snap_fd is fd for call-path changelog */
-        int c_snap_fd;
+    gf_lock_t lock;
 
-        /* rollover_count used by htime */
-        int  rollover_count;
+    /*  lock to synchronize CSNAP updation */
+    gf_lock_t c_snap_lock;
 
-        gf_lock_t lock;
+    /* written end of the pipe */
+    int wfd;
 
-        /*  lock to synchronize CSNAP updation */
-        gf_lock_t c_snap_lock;
+    /* rollover time */
+    int32_t rollover_time;
 
-        /* written end of the pipe */
-        int wfd;
+    /* fsync() interval */
+    int32_t fsync_interval;
 
-        /* rollover time */
-        int32_t rollover_time;
+    /* changelog type maps */
+    const char *maps[CHANGELOG_MAX_TYPE];
 
-        /* fsync() interval */
-        int32_t fsync_interval;
+    /* time slicer */
+    changelog_time_slice_t slice;
 
-        /* changelog type maps */
-        const char *maps[CHANGELOG_MAX_TYPE];
+    /* context of the updater */
+    changelog_dispatcher_t cd;
 
-        /* time slicer */
-        changelog_time_slice_t slice;
+    /* context of the rollover thread */
+    changelog_rollover_t cr;
 
-        /* context of the updater */
-        changelog_dispatcher_t cd;
+    /* context of fsync thread */
+    changelog_fsync_t cf;
 
-        /* context of the rollover thread */
-        changelog_rollover_t cr;
+    /* operation mode */
+    changelog_mode_t op_mode;
 
-        /* context of fsync thread */
-        changelog_fsync_t cf;
+    /* bootstrap routine for 'current' logger */
+    struct changelog_bootstrap *cb;
 
-        /* operation mode */
-        changelog_mode_t op_mode;
+    /* encoder mode */
+    changelog_encoder_t encode_mode;
 
-        /* bootstrap routine for 'current' logger */
-        struct changelog_bootstrap *cb;
+    /* encoder */
+    struct changelog_encoder *ce;
 
-        /* encoder mode */
-        changelog_encoder_t encode_mode;
+    /**
+     * snapshot dependency changes
+     */
 
-        /* encoder */
-        struct changelog_encoder *ce;
+    /* Draining of fops*/
+    drain_mgmt_t dm;
 
-        /**
-         * snapshot dependency changes
-         */
+    /* Represents the active color. Initially by default black */
+    chlog_fop_color_t current_color;
 
-        /* Draining of fops*/
-        drain_mgmt_t dm;
+    /* flag to determine explicit rollover is triggered */
+    gf_boolean_t explicit_rollover;
 
-        /* Represents the active color. Initially by default black */
-        chlog_fop_color_t current_color;
+    /* barrier notification variable protected by mutex */
+    barrier_notify_t bn;
 
-        /* flag to determine explicit rollover is triggered */
-        gf_boolean_t explicit_rollover;
+    /* barrier on/off indicating flags */
+    barrier_flags_t bflags;
 
-        /* barrier notification variable protected by mutex */
-        barrier_notify_t bn;
+    /* changelog barrier on/off indicating flag */
+    gf_boolean_t barrier_enabled;
+    struct list_head queue;
+    uint32_t queue_size;
+    gf_timer_t *timer;
+    struct timespec timeout;
 
-        /* barrier on/off indicating flags */
-        barrier_flags_t bflags;
+    /**
+     * buffers, RPC, event selection, notifications and other
+     * beasts.
+     */
 
-        /* changelog barrier on/off indicating flag */
-        gf_boolean_t      barrier_enabled;
-        struct list_head  queue;
-        uint32_t          queue_size;
-        gf_timer_t       *timer;
-        struct timespec   timeout;
+    /* epoll pthread */
+    pthread_t poller;
 
-        /**
-         * buffers, RPC, event selection, notifications and other
-         * beasts.
-         */
+    /* rotational buffer */
+    rbuf_t *rbuf;
 
-        /* epoll pthread */
-        pthread_t poller;
+    /* changelog RPC server */
+    rpcsvc_t *rpc;
 
-        /* rotational buffer */
-        rbuf_t *rbuf;
+    /* event selection */
+    changelog_ev_selector_t ev_selection;
 
-        /* changelog RPC server */
-        rpcsvc_t *rpc;
+    /* client handling (reverse connection) */
+    pthread_t connector;
 
-        /* event selection */
-        changelog_ev_selector_t ev_selection;
+    int nr_dispatchers;
+    pthread_t *ev_dispatcher;
 
-        /* client handling (reverse connection) */
-        pthread_t connector;
+    changelog_clnt_t connections;
 
-        int nr_dispatchers;
-        pthread_t *ev_dispatcher;
+    /* glusterfind dependency to capture paths on deleted entries*/
+    gf_boolean_t capture_del_path;
 
-        changelog_clnt_t connections;
+    /* Save total no. of listners */
+    gf_atomic_t listnercnt;
 
-        /* glusterfind dependency to capture paths on deleted entries*/
-        gf_boolean_t capture_del_path;
+    /* Save total no. of xprt are associated with listner */
+    gf_atomic_t xprtcnt;
+
+    /* Save xprt list */
+    struct list_head xprt_list;
+
+    /* Save total no. of client connection */
+    gf_atomic_t clntcnt;
+
+    /* Save cleanup brick in victim */
+    xlator_t *victim;
+
+    /* Status to save cleanup notify status */
+    gf_boolean_t notify_down;
 };
 
 struct changelog_local {
-        inode_t              *inode;
-        gf_boolean_t          update_no_check;
+    inode_t *inode;
+    gf_boolean_t update_no_check;
 
-        changelog_log_data_t  cld;
+    changelog_log_data_t cld;
 
-        /**
-         * ->prev_entry is used in cases when there needs to be
-         * additional changelog entry for the parent (eg. rename)
-         * It's analogous to ->next in single linked list world,
-         * but we call it as ->prev_entry... ha ha ha
-         */
-        struct changelog_local *prev_entry;
+    /**
+     * ->prev_entry is used in cases when there needs to be
+     * additional changelog entry for the parent (eg. rename)
+     * It's analogous to ->next in single linked list world,
+     * but we call it as ->prev_entry... ha ha ha
+     */
+    struct changelog_local *prev_entry;
 
-        /* snap dependency changes */
-        chlog_fop_color_t color;
+    /* snap dependency changes */
+    chlog_fop_color_t color;
 };
 
 typedef struct changelog_local changelog_local_t;
 
 /* inode version is stored in inode ctx */
 typedef struct changelog_inode_ctx {
-        unsigned long iversion[CHANGELOG_MAX_TYPE];
+    unsigned long iversion[CHANGELOG_MAX_TYPE];
 } changelog_inode_ctx_t;
 
-#define CHANGELOG_INODE_VERSION_TYPE(ctx, type)  &(ctx->iversion[type])
+#define CHANGELOG_INODE_VERSION_TYPE(ctx, type) &(ctx->iversion[type])
 
 /**
  * Optional Records:
@@ -344,269 +358,276 @@ typedef struct changelog_inode_ctx {
  *  @changelog_opt_t struct. The array is allocated via @iobufs.
  */
 typedef enum {
-        CHANGELOG_OPT_REC_FOP,
-        CHANGELOG_OPT_REC_ENTRY,
-        CHANGELOG_OPT_REC_UINT32,
+    CHANGELOG_OPT_REC_FOP,
+    CHANGELOG_OPT_REC_ENTRY,
+    CHANGELOG_OPT_REC_UINT32,
 } changelog_optional_rec_type_t;
 
 struct changelog_entry_fields {
-        uuid_t  cef_uuid;
-        char   *cef_bname;
-        char   *cef_path;
+    uuid_t cef_uuid;
+    char *cef_bname;
+    char *cef_path;
 };
 
 typedef struct {
-        /**
-         * @co_covert can be used to do post-processing of the record before
-         * it's persisted to the CHANGELOG. If this is NULL, then the record
-         * is persisted as per it's in memory format.
-         */
-        size_t (*co_convert) (void *data, char *buffer, gf_boolean_t encode);
-
-        /* release routines */
-        void (*co_free) (void *data);
-
-        /* type of the field */
-        changelog_optional_rec_type_t co_type;
-
-        /**
-         * sizeof of the 'valid' field in the union. This field is not used if
-         * @co_convert is specified.
-         */
-        size_t co_len;
-
-        union {
-                unsigned int                  co_uint32;
-                glusterfs_fop_t               co_fop;
-                struct changelog_entry_fields co_entry;
-        };
+    /**
+     * @co_covert can be used to do post-processing of the record before
+     * it's persisted to the CHANGELOG. If this is NULL, then the record
+     * is persisted as per it's in memory format.
+     */
+    size_t (*co_convert)(void *data, char *buffer, gf_boolean_t encode);
+
+    /* release routines */
+    void (*co_free)(void *data);
+
+    /* type of the field */
+    changelog_optional_rec_type_t co_type;
+
+    /**
+     * sizeof of the 'valid' field in the union. This field is not used if
+     * @co_convert is specified.
+     */
+    size_t co_len;
+
+    union {
+        unsigned int co_uint32;
+        glusterfs_fop_t co_fop;
+        struct changelog_entry_fields co_entry;
+    };
 } changelog_opt_t;
 
-#define CHANGELOG_OPT_RECORD_LEN  sizeof (changelog_opt_t)
+#define CHANGELOG_OPT_RECORD_LEN sizeof(changelog_opt_t)
 
 /**
  * helpers routines
  */
 
 int
-changelog_thread_cleanup (xlator_t *this, pthread_t thr_id);
+changelog_thread_cleanup(xlator_t *this, pthread_t thr_id);
 
 void *
-changelog_get_usable_buffer (changelog_local_t *local);
+changelog_get_usable_buffer(changelog_local_t *local);
 
 void
-changelog_set_usable_record_and_length (changelog_local_t *local,
-                                        size_t len, int xr);
+changelog_set_usable_record_and_length(changelog_local_t *local, size_t len,
+                                       int xr);
 void
-changelog_local_cleanup (xlator_t *xl, changelog_local_t *local);
+changelog_local_cleanup(xlator_t *xl, changelog_local_t *local);
 changelog_local_t *
-changelog_local_init (xlator_t *this, inode_t *inode, uuid_t gfid,
-                      int xtra_records, gf_boolean_t update_flag);
-int
-changelog_start_next_change (xlator_t *this,
-                             changelog_priv_t *priv,
-                             unsigned long ts, gf_boolean_t finale);
+changelog_local_init(xlator_t *this, inode_t *inode, uuid_t gfid,
+                     int xtra_records, gf_boolean_t update_flag);
 int
-changelog_open_journal (xlator_t *this, changelog_priv_t *priv);
+changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts,
+                            gf_boolean_t finale);
 int
-changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last);
+changelog_open_journal(xlator_t *this, changelog_priv_t *priv);
+void
+changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last);
 int
-changelog_inject_single_event (xlator_t *this,
-                               changelog_priv_t *priv,
-                               changelog_log_data_t *cld);
+changelog_inject_single_event(xlator_t *this, changelog_priv_t *priv,
+                              changelog_log_data_t *cld);
 size_t
-changelog_entry_length ();
+changelog_entry_length();
 int
-changelog_write (int fd, char *buffer, size_t len);
+changelog_write(int fd, char *buffer, size_t len);
 int
-changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+changelog_write_change(changelog_priv_t *priv, char *buffer, size_t len);
 int
-changelog_handle_change (xlator_t *this,
-                         changelog_priv_t *priv, changelog_log_data_t *cld);
+changelog_handle_change(xlator_t *this, changelog_priv_t *priv,
+                        changelog_log_data_t *cld);
 void
-changelog_update (xlator_t *this, changelog_priv_t *priv,
-                  changelog_local_t *local, changelog_log_type type);
+changelog_update(xlator_t *this, changelog_priv_t *priv,
+                 changelog_local_t *local, changelog_log_type type);
 void *
-changelog_rollover (void *data);
+changelog_rollover(void *data);
 void *
-changelog_fsync_thread (void *data);
+changelog_fsync_thread(void *data);
 int
-changelog_forget (xlator_t *this, inode_t *inode);
+changelog_forget(xlator_t *this, inode_t *inode);
 int
-htime_update (xlator_t *this, changelog_priv_t *priv,
-              unsigned long ts, char * buffer);
+htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer);
 int
-htime_open (xlator_t *this, changelog_priv_t *priv, unsigned long ts);
+htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts);
 int
-htime_create (xlator_t *this, changelog_priv_t *priv, unsigned long ts);
+htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts);
 
 /* Geo-Rep snapshot dependency changes */
 void
-changelog_color_fop_and_inc_cnt (xlator_t *this, changelog_priv_t *priv,
-                                                 changelog_local_t *local);
+changelog_color_fop_and_inc_cnt(xlator_t *this, changelog_priv_t *priv,
+                                changelog_local_t *local);
 void
-changelog_inc_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local);
+changelog_inc_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local);
 void
-changelog_dec_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local);
+changelog_dec_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local);
 int
-changelog_barrier_notify (changelog_priv_t *priv, char* buf);
+changelog_barrier_notify(changelog_priv_t *priv, char *buf);
 void
-changelog_barrier_cleanup (xlator_t *this, changelog_priv_t *priv,
-                                                struct list_head *queue);
+changelog_barrier_cleanup(xlator_t *this, changelog_priv_t *priv,
+                          struct list_head *queue);
 void
-changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv);
+changelog_drain_white_fops(xlator_t *this, changelog_priv_t *priv);
 void
-changelog_drain_black_fops (xlator_t *this, changelog_priv_t *priv);
+changelog_drain_black_fops(xlator_t *this, changelog_priv_t *priv);
 
 /* Crash consistency of changelog wrt snapshot */
 int
-changelog_snap_logging_stop ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_logging_stop(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_logging_start ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_logging_start(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_open ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_open(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_handle_ascii_change (xlator_t *this,
-                changelog_log_data_t *cld);
+changelog_snap_handle_ascii_change(xlator_t *this, changelog_log_data_t *cld);
 int
-changelog_snap_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+changelog_snap_write_change(changelog_priv_t *priv, char *buffer, size_t len);
 
 /* Changelog barrier routines */
-void __chlog_barrier_enqueue (xlator_t *this, call_stub_t *stub);
-void __chlog_barrier_disable (xlator_t *this, struct list_head *queue);
-void chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue);
-call_stub_t *__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue);
-int __chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv);
+void
+__chlog_barrier_enqueue(xlator_t *this, call_stub_t *stub);
+void
+__chlog_barrier_disable(xlator_t *this, struct list_head *queue);
+void
+chlog_barrier_dequeue_all(xlator_t *this, struct list_head *queue);
+call_stub_t *
+__chlog_barrier_dequeue(xlator_t *this, struct list_head *queue);
+int
+__chlog_barrier_enable(xlator_t *this, changelog_priv_t *priv);
 
 int32_t
-changelog_fill_entry_buf (call_frame_t *frame, xlator_t *this,
-                          loc_t *loc, changelog_local_t **local);
+changelog_fill_entry_buf(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         changelog_local_t **local);
 
 /* event selection routines */
-void changelog_select_event (xlator_t *,
-                                    changelog_ev_selector_t *, unsigned int);
-void changelog_deselect_event (xlator_t *,
-                                      changelog_ev_selector_t *, unsigned int);
-int changelog_init_event_selection (xlator_t *,
-                                           changelog_ev_selector_t *);
-int changelog_cleanup_event_selection (xlator_t *,
-                                              changelog_ev_selector_t *);
-int changelog_ev_selected (xlator_t *,
-                                  changelog_ev_selector_t *, unsigned int);
 void
-changelog_dispatch_event (xlator_t *, changelog_priv_t *, changelog_event_t *);
+changelog_select_event(xlator_t *, changelog_ev_selector_t *, unsigned int);
+void
+changelog_deselect_event(xlator_t *, changelog_ev_selector_t *, unsigned int);
+int
+changelog_init_event_selection(xlator_t *, changelog_ev_selector_t *);
+int
+changelog_ev_selected(xlator_t *, changelog_ev_selector_t *, unsigned int);
+void
+changelog_dispatch_event(xlator_t *, changelog_priv_t *, changelog_event_t *);
 
 changelog_inode_ctx_t *
-__changelog_inode_ctx_get (xlator_t *, inode_t *, unsigned long **,
-                           unsigned long *, changelog_log_type);
+__changelog_inode_ctx_get(xlator_t *, inode_t *, unsigned long **,
+                          unsigned long *, changelog_log_type);
 int
-resolve_pargfid_to_path (xlator_t *this, const uuid_t gfid, char **path,
-                         char *bname);
+resolve_pargfid_to_path(xlator_t *this, const uuid_t gfid, char **path,
+                        char *bname);
 
 /* macros */
 
-#define CHANGELOG_STACK_UNWIND(fop, frame, params ...) do {             \
-                changelog_local_t *__local = NULL;                      \
-                xlator_t          *__xl    = NULL;                      \
-                if (frame) {                                            \
-                        __local      = frame->local;                    \
-                        __xl         = frame->this;                     \
-                        frame->local = NULL;                            \
-                }                                                       \
-                STACK_UNWIND_STRICT (fop, frame, params);               \
-                if (__local && __local->prev_entry)                     \
-                        changelog_local_cleanup (__xl,                  \
-                                                 __local->prev_entry);  \
-                changelog_local_cleanup (__xl, __local);                \
-        } while (0)
-
-#define CHANGELOG_IOBUF_REF(iobuf) do {         \
-                if (iobuf)                      \
-                        iobuf_ref (iobuf);      \
-        } while (0)
-
-#define CHANGELOG_IOBUF_UNREF(iobuf) do {       \
-                if (iobuf)                      \
-                        iobuf_unref (iobuf);    \
-        } while (0)
-
-#define CHANGELOG_FILL_BUFFER(buffer, off, val, len) do {       \
-                memcpy (buffer + off, val, len);                \
-                off += len;                                     \
-        } while (0)
-
-#define SLICE_VERSION_UPDATE(slice) do {                \
-                int i = 0;                              \
-                for (; i < CHANGELOG_MAX_TYPE; i++) {   \
-                        slice->changelog_version[i]++;  \
-                }                                       \
-        } while (0)
-
-#define CHANGELOG_FILL_UINT32(co, number, converter, xlen) do { \
-                co->co_convert = converter;                     \
-                co->co_free = NULL;                             \
-                co->co_type = CHANGELOG_OPT_REC_UINT32;         \
-                co->co_uint32 = number;                         \
-                xlen += sizeof (unsigned int);                 \
-        } while (0)
-
-#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen) do { \
-                co->co_convert = converter;                     \
-                co->co_free = NULL;                             \
-                co->co_type = CHANGELOG_OPT_REC_FOP;            \
-                co->co_fop = fop;                               \
-                xlen += sizeof (fop);                           \
-        } while (0)
-
-#define CHANGELOG_FILL_ENTRY(co, pargfid, bname,                        \
-                             converter, freefn, xlen, label)            \
-        do {                                                            \
-                co->co_convert = converter;                             \
-                co->co_free = freefn;                                   \
-                co->co_type = CHANGELOG_OPT_REC_ENTRY;                  \
-                gf_uuid_copy (co->co_entry.cef_uuid, pargfid);          \
-                co->co_entry.cef_bname = gf_strdup(bname);              \
-                if (!co->co_entry.cef_bname)                            \
-                        goto label;                                     \
-                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));     \
-        } while (0)
-
-#define CHANGELOG_FILL_ENTRY_DIR_PATH(co, pargfid, bname, converter,        \
-                                      del_freefn, xlen, label, capture_del) \
-        do {                                                                \
-                co->co_convert = converter;                                 \
-                co->co_free = del_freefn;                                   \
-                co->co_type = CHANGELOG_OPT_REC_ENTRY;                      \
-                gf_uuid_copy (co->co_entry.cef_uuid, pargfid);              \
-                co->co_entry.cef_bname = gf_strdup(bname);                  \
-                if (!co->co_entry.cef_bname)                                \
-                        goto label;                                         \
-                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));         \
-                if (!capture_del || resolve_pargfid_to_path (this, pargfid, \
-                    &(co->co_entry.cef_path), co->co_entry.cef_bname)) {    \
-                        co->co_entry.cef_path = gf_strdup ("\0");           \
-                        xlen += 1;                                          \
-                } else {                                                    \
-                        xlen += (strlen (co->co_entry.cef_path));           \
-                }                                                           \
-        } while (0)
-
-#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                  \
-        local = changelog_local_init (this, inode, gfid, xrec, _gf_false)
-
-#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)          \
-        local = changelog_local_init (this, inode, gfid, xrec, _gf_true)
-
-#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do {      \
-                if (!priv->active)                                   \
-                        goto label;                                  \
-                /* ignore rebalance process's activity. */           \
-                if ((frame->root->pid == GF_CLIENT_PID_DEFRAG) ||    \
-                    (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)) \
-                        goto label;                                  \
-        } while (0)
+#define CHANGELOG_STACK_UNWIND(fop, frame, params...)                          \
+    do {                                                                       \
+        changelog_local_t *__local = NULL;                                     \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __xl = frame->this;                                                \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local && __local->prev_entry)                                    \
+            changelog_local_cleanup(__xl, __local->prev_entry);                \
+        changelog_local_cleanup(__xl, __local);                                \
+    } while (0)
+
+#define CHANGELOG_IOBUF_REF(iobuf)                                             \
+    do {                                                                       \
+        if (iobuf)                                                             \
+            iobuf_ref(iobuf);                                                  \
+    } while (0)
+
+#define CHANGELOG_IOBUF_UNREF(iobuf)                                           \
+    do {                                                                       \
+        if (iobuf)                                                             \
+            iobuf_unref(iobuf);                                                \
+    } while (0)
+
+#define CHANGELOG_FILL_BUFFER(buffer, off, val, len)                           \
+    do {                                                                       \
+        memcpy(buffer + off, val, len);                                        \
+        off += len;                                                            \
+    } while (0)
+
+#define SLICE_VERSION_UPDATE(slice)                                            \
+    do {                                                                       \
+        int i = 0;                                                             \
+        for (; i < CHANGELOG_MAX_TYPE; i++) {                                  \
+            slice->changelog_version[i]++;                                     \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_FILL_UINT32(co, number, converter, xlen)                     \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = NULL;                                                    \
+        co->co_type = CHANGELOG_OPT_REC_UINT32;                                \
+        co->co_uint32 = number;                                                \
+        xlen += sizeof(unsigned int);                                          \
+    } while (0)
+
+#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen)                     \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = NULL;                                                    \
+        co->co_type = CHANGELOG_OPT_REC_FOP;                                   \
+        co->co_fop = fop;                                                      \
+        xlen += sizeof(fop);                                                   \
+    } while (0)
+
+#define CHANGELOG_FILL_ENTRY(co, pargfid, bname, converter, freefn, xlen,      \
+                             label)                                            \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = freefn;                                                  \
+        co->co_type = CHANGELOG_OPT_REC_ENTRY;                                 \
+        gf_uuid_copy(co->co_entry.cef_uuid, pargfid);                          \
+        co->co_entry.cef_bname = gf_strdup(bname);                             \
+        if (!co->co_entry.cef_bname)                                           \
+            goto label;                                                        \
+        xlen += (UUID_CANONICAL_FORM_LEN + strlen(bname));                     \
+    } while (0)
+
+#define CHANGELOG_FILL_ENTRY_DIR_PATH(co, pargfid, bname, converter,           \
+                                      del_freefn, xlen, label, capture_del)    \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = del_freefn;                                              \
+        co->co_type = CHANGELOG_OPT_REC_ENTRY;                                 \
+        gf_uuid_copy(co->co_entry.cef_uuid, pargfid);                          \
+        co->co_entry.cef_bname = gf_strdup(bname);                             \
+        if (!co->co_entry.cef_bname)                                           \
+            goto label;                                                        \
+        xlen += (UUID_CANONICAL_FORM_LEN + strlen(bname));                     \
+        if (!capture_del ||                                                    \
+            resolve_pargfid_to_path(this, pargfid, &(co->co_entry.cef_path),   \
+                                    co->co_entry.cef_bname)) {                 \
+            co->co_entry.cef_path = gf_strdup("\0");                           \
+            xlen += 1;                                                         \
+        } else {                                                               \
+            xlen += (strlen(co->co_entry.cef_path));                           \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                         \
+    local = changelog_local_init(this, inode, gfid, xrec, _gf_false)
+
+#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)                 \
+    local = changelog_local_init(this, inode, gfid, xrec, _gf_true)
+
+#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label)                     \
+    do {                                                                       \
+        if (!priv->active)                                                     \
+            goto label;                                                        \
+        /* ignore rebalance process's activity. */                             \
+        if ((frame->root->pid == GF_CLIENT_PID_DEFRAG) ||                      \
+            (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG))                   \
+            goto label;                                                        \
+    } while (0)
 
 /* If it is a METADATA entry and fop num being GF_FOP_NULL, don't
  * log in the changelog as it is of no use. And also if it is
@@ -615,66 +636,81 @@ resolve_pargfid_to_path (xlator_t *this, const uuid_t gfid, char **path,
  * to same changelog will be missed. Hence check for boundary
  * condition.
  */
-#define CHANGELOG_OP_BOUNDARY_CHECK(frame, label) do {          \
-                if (frame->root->op <= GF_FOP_NULL ||           \
-                    frame->root->op >= GF_FOP_MAXVALUE)         \
-                        goto label;                             \
-        } while (0)
+#define CHANGELOG_OP_BOUNDARY_CHECK(frame, label)                              \
+    do {                                                                       \
+        if (frame->root->op <= GF_FOP_NULL ||                                  \
+            frame->root->op >= GF_FOP_MAXVALUE)                                \
+            goto label;                                                        \
+    } while (0)
 
 /**
  * ignore internal fops for all clients except AFR self-heal daemon
  */
-#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label) do {    \
-                if ((frame->root->pid != GF_CLIENT_PID_SELF_HEALD)      \
-                    && dict                                             \
-                    && dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY))     \
-                        goto label;                                     \
-        } while (0)
-
-#define CHANGELOG_COND_GOTO(priv, cond, label) do {                    \
-                if (!priv->active || cond)                             \
-                        goto label;                                    \
-        } while (0)
+#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label)                \
+    do {                                                                       \
+        if ((frame->root->pid != GF_CLIENT_PID_SELF_HEALD) && dict &&          \
+            dict_get(dict, GLUSTERFS_INTERNAL_FOP_KEY))                        \
+            goto label;                                                        \
+    } while (0)
+
+#define CHANGELOG_COND_GOTO(priv, cond, label)                                 \
+    do {                                                                       \
+        if (!priv->active || cond)                                             \
+            goto label;                                                        \
+    } while (0)
 
 /* Begin: Geo-Rep snapshot dependency changes */
 
-#define DICT_ERROR         -1
-#define BARRIER_OFF         0
-#define BARRIER_ON          1
-#define DICT_DEFAULT        2
-
-#define CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, label) do {                      \
-                if (!priv->active) {                                           \
-                        gf_msg (this->name, GF_LOG_WARNING, 0,                 \
-                                CHANGELOG_MSG_NOT_ACTIVE,                      \
-                                "Changelog is not active, return success");    \
-                        ret = 0;                                               \
-                        goto label;                                            \
-                }                                                              \
-        } while (0)
+#define DICT_ERROR -1
+#define BARRIER_OFF 0
+#define BARRIER_ON 1
+#define DICT_DEFAULT 2
+
+#define CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, label)                           \
+    do {                                                                       \
+        if (!priv->active) {                                                   \
+            gf_smsg(this->name, GF_LOG_WARNING, 0,                             \
+                    CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE, NULL);                 \
+            ret = 0;                                                           \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 /* Log pthread error and goto label */
-#define CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, label) do {                      \
-                if (ret) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR,                      \
-                                0, CHANGELOG_MSG_PTHREAD_ERROR,                \
-                                "pthread error: Error: %d", ret);              \
-                        ret = -1;                                              \
-                        goto label;                                            \
-                }                                                              \
-        } while (0);
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, label)                           \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
 
 /* Log pthread error, set flag and goto label */
-#define CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, label, flag) do {                \
-                if (ret) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                   \
-                                CHANGELOG_MSG_PTHREAD_ERROR,                   \
-                                "pthread error: Error: %d", ret);              \
-                        ret = -1;                                              \
-                        flag = _gf_true;                                       \
-                        goto label;                                            \
-                }                                                              \
-        } while (0)
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, label, flag)                     \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            flag = _gf_true;                                                   \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+/* Log pthread error, unlock mutex and goto label */
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_2(ret, label, mutex)                    \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            pthread_mutex_unlock(&mutex);                                      \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
 /* End: Geo-Rep snapshot dependency changes */
 
 #endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog-mem-types.h b/xlators/features/changelog/src/changelog-mem-types.h
index 1618f722f6c..a2d8a9cbe93 100644
--- a/xlators/features/changelog/src/changelog-mem-types.h
+++ b/xlators/features/changelog/src/changelog-mem-types.h
@@ -11,25 +11,24 @@
 #ifndef _CHANGELOG_MEM_TYPES_H
 #define _CHANGELOG_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_changelog_mem_types {
-        gf_changelog_mt_priv_t                     = gf_common_mt_end + 1,
-        gf_changelog_mt_str_t                      = gf_common_mt_end + 2,
-        gf_changelog_mt_batch_t                    = gf_common_mt_end + 3,
-        gf_changelog_mt_rt_t                       = gf_common_mt_end + 4,
-        gf_changelog_mt_inode_ctx_t                = gf_common_mt_end + 5,
-        gf_changelog_mt_rpc_clnt_t                 = gf_common_mt_end + 6,
-        gf_changelog_mt_libgfchangelog_t           = gf_common_mt_end + 7,
-        gf_changelog_mt_libgfchangelog_entry_t     = gf_common_mt_end + 8,
-        gf_changelog_mt_libgfchangelog_rl_t        = gf_common_mt_end + 9,
-        gf_changelog_mt_libgfchangelog_dirent_t    = gf_common_mt_end + 10,
-        gf_changelog_mt_changelog_buffer_t         = gf_common_mt_end + 11,
-        gf_changelog_mt_history_data_t             = gf_common_mt_end + 12,
-        gf_changelog_mt_libgfchangelog_call_pool_t = gf_common_mt_end + 13,
-        gf_changelog_mt_libgfchangelog_event_t     = gf_common_mt_end + 14,
-        gf_changelog_mt_ev_dispatcher_t            = gf_common_mt_end + 15,
-        gf_changelog_mt_end
+    gf_changelog_mt_priv_t = gf_common_mt_end + 1,
+    gf_changelog_mt_str_t = gf_common_mt_end + 2,
+    gf_changelog_mt_batch_t = gf_common_mt_end + 3,
+    gf_changelog_mt_rt_t = gf_common_mt_end + 4,
+    gf_changelog_mt_inode_ctx_t = gf_common_mt_end + 5,
+    gf_changelog_mt_rpc_clnt_t = gf_common_mt_end + 6,
+    gf_changelog_mt_libgfchangelog_t = gf_common_mt_end + 7,
+    gf_changelog_mt_libgfchangelog_entry_t = gf_common_mt_end + 8,
+    gf_changelog_mt_libgfchangelog_rl_t = gf_common_mt_end + 9,
+    gf_changelog_mt_changelog_buffer_t = gf_common_mt_end + 10,
+    gf_changelog_mt_history_data_t = gf_common_mt_end + 11,
+    gf_changelog_mt_libgfchangelog_call_pool_t = gf_common_mt_end + 12,
+    gf_changelog_mt_libgfchangelog_event_t = gf_common_mt_end + 13,
+    gf_changelog_mt_ev_dispatcher_t = gf_common_mt_end + 14,
+    gf_changelog_mt_end
 };
 
 #endif
diff --git a/xlators/features/changelog/src/changelog-messages.h b/xlators/features/changelog/src/changelog-messages.h
index e65a457b7c0..cb0e16c85d8 100644
--- a/xlators/features/changelog/src/changelog-messages.h
+++ b/xlators/features/changelog/src/changelog-messages.h
@@ -11,440 +11,162 @@
 #ifndef _CHANGELOG_MESSAGES_H_
 #define _CHANGELOG_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs-message-id.h"
-
-/*! \file changelog-messages.h
- *  \brief CHANGELOG log-message IDs and their descriptions.
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for readability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- * anywhere. If reused then then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
- */
-
-#define GLFS_COMP_BASE_CHANGELOG GLFS_MSGID_COMP_CHANGELOG
-#define GLFS_NUM_MESSAGES 54
-#define GLFS_MSGID_END (GLFS_COMP_BASE_CHANGELOG + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x GLFS_COMP_BASE_CHANGELOG, "Invalid: Start of messages"
-
-/*!
- * @messageid
- * @diagnosis open/opendir failed on a brick.
- * @recommended action Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define CHANGELOG_MSG_OPEN_FAILED               (GLFS_COMP_BASE_CHANGELOG + 1)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_NO_MEMORY                 (GLFS_COMP_BASE_CHANGELOG + 2)
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_VOL_MISCONFIGURED         (GLFS_COMP_BASE_CHANGELOG + 3)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_RENAME_ERROR              (GLFS_COMP_BASE_CHANGELOG + 4)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_READ_ERROR                (GLFS_COMP_BASE_CHANGELOG + 5)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_HTIME_ERROR               (GLFS_COMP_BASE_CHANGELOG + 6)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED (GLFS_COMP_BASE_CHANGELOG + 7)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED  (GLFS_COMP_BASE_CHANGELOG + 8)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_CHILD_MISCONFIGURED       (GLFS_COMP_BASE_CHANGELOG + 9)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_DIR_OPTIONS_NOT_SET       (GLFS_COMP_BASE_CHANGELOG + 10)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_CLOSE_ERROR               (GLFS_COMP_BASE_CHANGELOG + 11)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PIPE_CREATION_ERROR       (GLFS_COMP_BASE_CHANGELOG + 12)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_DICT_GET_FAILED           (GLFS_COMP_BASE_CHANGELOG + 13)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_BARRIER_INFO              (GLFS_COMP_BASE_CHANGELOG + 14)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_BARRIER_ERROR             (GLFS_COMP_BASE_CHANGELOG + 15)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_GET_TIME_OP_FAILED        (GLFS_COMP_BASE_CHANGELOG + 16)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_WRITE_FAILED              (GLFS_COMP_BASE_CHANGELOG + 17)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PTHREAD_ERROR             (GLFS_COMP_BASE_CHANGELOG + 18)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_INODE_NOT_FOUND           (GLFS_COMP_BASE_CHANGELOG + 19)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_FSYNC_OP_FAILED           (GLFS_COMP_BASE_CHANGELOG + 20)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_TOTAL_LOG_INFO            (GLFS_COMP_BASE_CHANGELOG + 21)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_SNAP_INFO                 (GLFS_COMP_BASE_CHANGELOG + 22)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_SELECT_FAILED             (GLFS_COMP_BASE_CHANGELOG + 23)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_FCNTL_FAILED              (GLFS_COMP_BASE_CHANGELOG + 24)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_BNOTIFY_INFO              (GLFS_COMP_BASE_CHANGELOG + 25)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_ENTRY_BUF_INFO            (GLFS_COMP_BASE_CHANGELOG + 26)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_NOT_ACTIVE                (GLFS_COMP_BASE_CHANGELOG + 27)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_LOCAL_INIT_FAILED         (GLFS_COMP_BASE_CHANGELOG + 28)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_NOTIFY_REGISTER_FAILED    (GLFS_COMP_BASE_CHANGELOG + 28)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED   (GLFS_COMP_BASE_CHANGELOG + 29)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_HANDLE_PROBE_ERROR        (GLFS_COMP_BASE_CHANGELOG + 30)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_SET_FD_CONTEXT            (GLFS_COMP_BASE_CHANGELOG + 31)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_FREEUP_FAILED             (GLFS_COMP_BASE_CHANGELOG + 32)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_HTIME_INFO                (GLFS_COMP_BASE_CHANGELOG + 33)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED   (GLFS_COMP_BASE_CHANGELOG + 34)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_RPC_BUILD_ERROR           (GLFS_COMP_BASE_CHANGELOG + 35)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_RPC_CONNECT_ERROR         (GLFS_COMP_BASE_CHANGELOG + 36)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_RPC_START_ERROR           (GLFS_COMP_BASE_CHANGELOG + 37)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_BUFFER_STARVATION_ERROR   (GLFS_COMP_BASE_CHANGELOG + 3)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_SCAN_DIR_FAILED           (GLFS_COMP_BASE_CHANGELOG + 39)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_FSETXATTR_FAILED          (GLFS_COMP_BASE_CHANGELOG + 40)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_FGETXATTR_FAILED          (GLFS_COMP_BASE_CHANGELOG + 41)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_CLEANUP_ON_ACTIVE_REF                             \
-                                                (GLFS_COMP_BASE_CHANGELOG + 42)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_DISPATCH_EVENT_FAILED      (GLFS_COMP_BASE_CHANGELOG + 43)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PUT_BUFFER_FAILED      (GLFS_COMP_BASE_CHANGELOG + 44)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED  (GLFS_COMP_BASE_CHANGELOG + 45)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_PTHREAD_CANCEL_FAILED      (GLFS_COMP_BASE_CHANGELOG + 46)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_INJECT_FSYNC_FAILED      (GLFS_COMP_BASE_CHANGELOG + 47)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_CREATE_FRAME_FAILED      (GLFS_COMP_BASE_CHANGELOG + 48)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_FSTAT_OP_FAILED          (GLFS_COMP_BASE_CHANGELOG + 49)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_LSEEK_OP_FAILED          (GLFS_COMP_BASE_CHANGELOG + 50)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_STRSTR_OP_FAILED          (GLFS_COMP_BASE_CHANGELOG + 51)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_UNLINK_OP_FAILED          (GLFS_COMP_BASE_CHANGELOG + 52)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED \
-        (GLFS_COMP_BASE_CHANGELOG + 53)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_READLINK_OP_FAILED        (GLFS_COMP_BASE_CHANGELOG + 54)
-
-/*!
-  @messageid
- * @diagnosis
- * @recommended action
-*/
-#define CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED  (GLFS_COMP_BASE_CHANGELOG + 55)
-
-
 
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+GLFS_MSGID(
+    CHANGELOG, CHANGELOG_MSG_OPEN_FAILED, CHANGELOG_MSG_BARRIER_FOP_FAILED,
+    CHANGELOG_MSG_VOL_MISCONFIGURED, CHANGELOG_MSG_RENAME_ERROR,
+    CHANGELOG_MSG_READ_ERROR, CHANGELOG_MSG_HTIME_ERROR,
+    CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
+    CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, CHANGELOG_MSG_CHILD_MISCONFIGURED,
+    CHANGELOG_MSG_DIR_OPTIONS_NOT_SET, CHANGELOG_MSG_CLOSE_ERROR,
+    CHANGELOG_MSG_PIPE_CREATION_ERROR, CHANGELOG_MSG_DICT_GET_FAILED,
+    CHANGELOG_MSG_BARRIER_INFO, CHANGELOG_MSG_BARRIER_ERROR,
+    CHANGELOG_MSG_GET_TIME_OP_FAILED, CHANGELOG_MSG_WRITE_FAILED,
+    CHANGELOG_MSG_PTHREAD_ERROR, CHANGELOG_MSG_INODE_NOT_FOUND,
+    CHANGELOG_MSG_FSYNC_OP_FAILED, CHANGELOG_MSG_TOTAL_LOG_INFO,
+    CHANGELOG_MSG_SNAP_INFO, CHANGELOG_MSG_SELECT_FAILED,
+    CHANGELOG_MSG_FCNTL_FAILED, CHANGELOG_MSG_BNOTIFY_INFO,
+    CHANGELOG_MSG_ENTRY_BUF_INFO, CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE,
+    CHANGELOG_MSG_LOCAL_INIT_FAILED, CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
+    CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED, CHANGELOG_MSG_HANDLE_PROBE_ERROR,
+    CHANGELOG_MSG_SET_FD_CONTEXT, CHANGELOG_MSG_FREEUP_FAILED,
+    CHANGELOG_MSG_RECONFIGURE, CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
+    CHANGELOG_MSG_RPC_BUILD_ERROR, CHANGELOG_MSG_RPC_CONNECT_ERROR,
+    CHANGELOG_MSG_RPC_START_ERROR, CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
+    CHANGELOG_MSG_SCAN_DIR_FAILED, CHANGELOG_MSG_FSETXATTR_FAILED,
+    CHANGELOG_MSG_FGETXATTR_FAILED, CHANGELOG_MSG_CLEANUP_ON_ACTIVE_REF,
+    CHANGELOG_MSG_DISPATCH_EVENT_FAILED, CHANGELOG_MSG_PUT_BUFFER_FAILED,
+    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
+    CHANGELOG_MSG_INJECT_FSYNC_FAILED, CHANGELOG_MSG_CREATE_FRAME_FAILED,
+    CHANGELOG_MSG_FSTAT_OP_FAILED, CHANGELOG_MSG_LSEEK_OP_FAILED,
+    CHANGELOG_MSG_STRSTR_OP_FAILED, CHANGELOG_MSG_UNLINK_OP_FAILED,
+    CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED,
+    CHANGELOG_MSG_READLINK_OP_FAILED, CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED,
+    CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED, CHANGELOG_MSG_MEMORY_INIT_FAILED,
+    CHANGELOG_MSG_NO_MEMORY, CHANGELOG_MSG_HTIME_STAT_ERROR,
+    CHANGELOG_MSG_HTIME_CURRENT_ERROR, CHANGELOG_MSG_BNOTIFY_COND_INFO,
+    CHANGELOG_MSG_NO_HTIME_CURRENT, CHANGELOG_MSG_HTIME_CURRENT,
+    CHANGELOG_MSG_NEW_HTIME_FILE, CHANGELOG_MSG_MKDIR_ERROR,
+    CHANGELOG_MSG_PATH_NOT_FOUND, CHANGELOG_MSG_XATTR_INIT_FAILED,
+    CHANGELOG_MSG_WROTE_TO_CSNAP, CHANGELOG_MSG_UNUSED_0,
+    CHANGELOG_MSG_GET_BUFFER_FAILED, CHANGELOG_MSG_BARRIER_STATE_NOTIFY,
+    CHANGELOG_MSG_BARRIER_DISABLED, CHANGELOG_MSG_BARRIER_ALREADY_DISABLED,
+    CHANGELOG_MSG_BARRIER_ON_ERROR, CHANGELOG_MSG_BARRIER_ENABLE,
+    CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND, CHANGELOG_MSG_ERROR_IN_DICT_GET,
+    CHANGELOG_MSG_UNUSED_1, CHANGELOG_MSG_UNUSED_2,
+    CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS,
+    CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED,
+    CHANGELOG_MSG_BARRIER_TIMEOUT, CHANGELOG_MSG_TIMEOUT_ADD_FAILED,
+    CHANGELOG_MSG_CLEANUP_ALREADY_SET);
+
+#define CHANGELOG_MSG_BARRIER_FOP_FAILED_STR                                   \
+    "failed to barrier FOPs, disabling changelog barrier"
+#define CHANGELOG_MSG_MEMORY_INIT_FAILED_STR "memory accounting init failed"
+#define CHANGELOG_MSG_NO_MEMORY_STR "failed to create local memory pool"
+#define CHANGELOG_MSG_ENTRY_BUF_INFO_STR                                       \
+    "Entry cannot be captured for gfid, Capturing DATA entry."
+#define CHANGELOG_MSG_PTHREAD_ERROR_STR "pthread error"
+#define CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED_STR "pthread_mutex_init failed"
+#define CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED_STR "pthread_cond_init failed"
+#define CHANGELOG_MSG_HTIME_ERROR_STR "failed to update HTIME file"
+#define CHANGELOG_MSG_HTIME_STAT_ERROR_STR "unable to stat htime file"
+#define CHANGELOG_MSG_HTIME_CURRENT_ERROR_STR "Error extracting HTIME_CURRENT."
+#define CHANGELOG_MSG_UNLINK_OP_FAILED_STR "error unlinking empty changelog"
+#define CHANGELOG_MSG_RENAME_ERROR_STR "error renaming"
+#define CHANGELOG_MSG_MKDIR_ERROR_STR "unable to create directory"
+#define CHANGELOG_MSG_BNOTIFY_INFO_STR                                         \
+    "Explicit rollover changelog signaling bnotify"
+#define CHANGELOG_MSG_BNOTIFY_COND_INFO_STR "Woke up: bnotify conditional wait"
+#define CHANGELOG_MSG_RECONFIGURE_STR "Reconfigure: Changelog Enable"
+#define CHANGELOG_MSG_NO_HTIME_CURRENT_STR                                     \
+    "HTIME_CURRENT not found. Changelog enabled before init"
+#define CHANGELOG_MSG_HTIME_CURRENT_STR "HTIME_CURRENT"
+#define CHANGELOG_MSG_NEW_HTIME_FILE_STR                                       \
+    "Changelog enable: Creating new HTIME file"
+#define CHANGELOG_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define CHANGELOG_MSG_TOTAL_LOG_INFO_STR "changelog info"
+#define CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED_STR "pthread cond wait failed"
+#define CHANGELOG_MSG_INODE_NOT_FOUND_STR "inode not found"
+#define CHANGELOG_MSG_READLINK_OP_FAILED_STR                                   \
+    "could not read the link from the gfid handle"
+#define CHANGELOG_MSG_OPEN_FAILED_STR "unable to open file"
+#define CHANGELOG_MSG_RPC_CONNECT_ERROR_STR "failed to connect back"
+#define CHANGELOG_MSG_BUFFER_STARVATION_ERROR_STR                              \
+    "Failed to get buffer for RPC dispatch"
+#define CHANGELOG_MSG_PTHREAD_CANCEL_FAILED_STR "could not cancel thread"
+#define CHANGELOG_MSG_FSTAT_OP_FAILED_STR "Could not stat (CHANGELOG)"
+#define CHANGELOG_MSG_LSEEK_OP_FAILED_STR "Could not lseek (changelog)"
+#define CHANGELOG_MSG_PATH_NOT_FOUND_STR                                       \
+    "Could not find CHANGELOG in changelog path"
+#define CHANGELOG_MSG_FSYNC_OP_FAILED_STR "fsync failed"
+#define CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED_STR                        \
+    "Error detecting empty changelog"
+#define CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED_STR                             \
+    "Fail snapshot because of previous errors"
+#define CHANGELOG_MSG_SCAN_DIR_FAILED_STR "scandir failed"
+#define CHANGELOG_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define CHANGELOG_MSG_XATTR_INIT_FAILED_STR "Htime xattr initialization failed"
+#define CHANGELOG_MSG_SNAP_INFO_STR "log in call path"
+#define CHANGELOG_MSG_WRITE_FAILED_STR "error writing to disk"
+#define CHANGELOG_MSG_WROTE_TO_CSNAP_STR "Successfully wrote to csnap"
+#define CHANGELOG_MSG_GET_TIME_OP_FAILED_STR "Problem rolling over changelog(s)"
+#define CHANGELOG_MSG_BARRIER_INFO_STR "Explicit wakeup on barrier notify"
+#define CHANGELOG_MSG_SELECT_FAILED_STR "pthread_cond_timedwait failed"
+#define CHANGELOG_MSG_INJECT_FSYNC_FAILED_STR "failed to inject fsync event"
+#define CHANGELOG_MSG_LOCAL_INIT_FAILED_STR                                    \
+    "changelog local initialization failed"
+#define CHANGELOG_MSG_GET_BUFFER_FAILED_STR "Failed to get buffer"
+#define CHANGELOG_MSG_SET_FD_CONTEXT_STR                                       \
+    "could not set fd context(for release cbk)"
+#define CHANGELOG_MSG_DICT_GET_FAILED_STR "Barrier failed"
+#define CHANGELOG_MSG_BARRIER_STATE_NOTIFY_STR "Barrier notification"
+#define CHANGELOG_MSG_BARRIER_ERROR_STR                                        \
+    "Received another barrier off notification while already off"
+#define CHANGELOG_MSG_BARRIER_DISABLED_STR "disabled changelog barrier"
+#define CHANGELOG_MSG_BARRIER_ALREADY_DISABLED_STR                             \
+    "Changelog barrier already disabled"
+#define CHANGELOG_MSG_BARRIER_ON_ERROR_STR                                     \
+    "Received another barrier on notification when last one is not served yet"
+#define CHANGELOG_MSG_BARRIER_ENABLE_STR "Enabled changelog barrier"
+#define CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND_STR "barrier key not found"
+#define CHANGELOG_MSG_ERROR_IN_DICT_GET_STR                                    \
+    "Something went wrong in dict_get_str_boolean"
+#define CHANGELOG_MSG_DIR_OPTIONS_NOT_SET_STR "changelog-dir option is not set"
+#define CHANGELOG_MSG_FREEUP_FAILED_STR "could not cleanup bootstrapper"
+#define CHANGELOG_MSG_CHILD_MISCONFIGURED_STR                                  \
+    "translator needs a single subvolume"
+#define CHANGELOG_MSG_VOL_MISCONFIGURED_STR                                    \
+    "dangling volume. please check volfile"
+#define CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_STR                               \
+    "Dequeuing all the changelog barriered fops"
+#define CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED_STR                      \
+    "Dequeuing changelog barriered fops is finished"
+#define CHANGELOG_MSG_BARRIER_TIMEOUT_STR                                      \
+    "Disabling changelog barrier because of the timeout"
+#define CHANGELOG_MSG_TIMEOUT_ADD_FAILED_STR                                   \
+    "Couldn't add changelog barrier timeout event"
+#define CHANGELOG_MSG_RPC_BUILD_ERROR_STR "failed to build rpc options"
+#define CHANGELOG_MSG_NOTIFY_REGISTER_FAILED_STR "failed to register notify"
+#define CHANGELOG_MSG_RPC_START_ERROR_STR "failed to start rpc"
+#define CHANGELOG_MSG_CREATE_FRAME_FAILED_STR "failed to create frame"
+#define CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED_STR "failed to serialize reply"
+#define CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED_STR "cannot register program"
+#define CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE_STR                                 \
+    "Changelog is not active, return success"
+#define CHANGELOG_MSG_PUT_BUFFER_FAILED_STR                                    \
+    "failed to put buffer after consumption"
+#define CHANGELOG_MSG_CLEANUP_ALREADY_SET_STR                                  \
+    "cleanup_starting flag is already set for xl"
+#define CHANGELOG_MSG_HANDLE_PROBE_ERROR_STR "xdr decoding error"
 #endif /* !_CHANGELOG_MESSAGES_H_ */
diff --git a/xlators/features/changelog/src/changelog-misc.h b/xlators/features/changelog/src/changelog-misc.h
index 778f79c82c5..e2addc09414 100644
--- a/xlators/features/changelog/src/changelog-misc.h
+++ b/xlators/features/changelog/src/changelog-misc.h
@@ -11,10 +11,10 @@
 #ifndef _CHANGELOG_MISC_H
 #define _CHANGELOG_MISC_H
 
-#include "glusterfs.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/common-utils.h>
 
-#define CHANGELOG_MAX_TYPE  3
+#define CHANGELOG_MAX_TYPE 4
 #define CHANGELOG_FILE_NAME "CHANGELOG"
 #define HTIME_FILE_NAME "HTIME"
 #define CSNAP_FILE_NAME "CHANGELOG.SNAP"
@@ -22,110 +22,110 @@
 #define HTIME_CURRENT "trusted.glusterfs.current_htime"
 #define HTIME_INITIAL_VALUE "0:0"
 
-#define CHANGELOG_VERSION_MAJOR  1
-#define CHANGELOG_VERSION_MINOR  2
+#define CHANGELOG_VERSION_MAJOR 1
+#define CHANGELOG_VERSION_MINOR 2
 
-#define CHANGELOG_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/changelog-%s.sock"
-#define CHANGELOG_TMP_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/.%s%lu.sock"
+#define CHANGELOG_UNIX_SOCK DEFAULT_VAR_RUN_DIRECTORY "/changelog-%s.sock"
+#define CHANGELOG_TMP_UNIX_SOCK DEFAULT_VAR_RUN_DIRECTORY "/.%s%lu.sock"
 
 /**
  * header starts with the version and the format of the changelog.
  * 'version' not much of a use now.
  */
-#define CHANGELOG_HEADER                                                \
-        "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
-
-#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len) do {      \
-                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
-                md5_wrapper((unsigned char *) brick_path,               \
-                            strlen(brick_path),                         \
-                            md5_sum);                                   \
-                (void) snprintf (sockpath, len,                         \
-                                 CHANGELOG_UNIX_SOCK, md5_sum);         \
-        } while (0)
-
-#define CHANGELOG_MAKE_TMP_SOCKET_PATH(brick_path, sockpath, len) do {  \
-                unsigned long pid = 0;                                  \
-                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
-                pid = (unsigned long) getpid ();                        \
-                md5_wrapper((unsigned char *) brick_path,               \
-                            strlen(brick_path),                         \
-                            md5_sum);                                   \
-                (void) snprintf (sockpath,                              \
-                                 len, CHANGELOG_TMP_UNIX_SOCK,          \
-                                 md5_sum, pid);                         \
-        } while (0)
-
+#define CHANGELOG_HEADER                                                       \
+    "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
+
+#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len)                  \
+    do {                                                                       \
+        char xxh64[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {                         \
+            0,                                                                 \
+        };                                                                     \
+        gf_xxh64_wrapper((unsigned char *)brick_path, strlen(brick_path),      \
+                         GF_XXHSUM64_DEFAULT_SEED, xxh64);                     \
+        (void)snprintf(sockpath, len, CHANGELOG_UNIX_SOCK, xxh64);             \
+    } while (0)
+
+#define CHANGELOG_MAKE_TMP_SOCKET_PATH(brick_path, sockpath, len)              \
+    do {                                                                       \
+        unsigned long pid = 0;                                                 \
+        char xxh64[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {                         \
+            0,                                                                 \
+        };                                                                     \
+        pid = (unsigned long)getpid();                                         \
+        gf_xxh64_wrapper((unsigned char *)brick_path, strlen(brick_path),      \
+                         GF_XXHSUM64_DEFAULT_SEED, xxh64);                     \
+        (void)snprintf(sockpath, len, CHANGELOG_TMP_UNIX_SOCK, xxh64, pid);    \
+    } while (0)
 
 /**
  * ... used by libgfchangelog.
  */
-#define CHANGELOG_GET_HEADER_INFO(fd, buffer, len, enc, maj, min, elen) do { \
-                FILE *fp;                                               \
-                int fd_dup;                                             \
-                                                                        \
-                enc = -1;                                               \
-                maj = -1;                                               \
-                min = -1;                                               \
-                fd_dup = dup (fd);                                      \
-                                                                        \
-                if (fd_dup != -1) {                                     \
-                        fp = fdopen (fd_dup, "r");                      \
-                        if (fp) {                                       \
-                                if (fgets (buffer, len, fp)) {          \
-                                        elen = strlen (buffer);         \
-                                        sscanf (buffer,                 \
-                                                CHANGELOG_HEADER,       \
-                                                &maj, &min, &enc);      \
-                                }                                       \
-                                fclose (fp);                            \
-                        } else {                                        \
-                                sys_close (fd_dup);                     \
-                        }                                               \
-                }                                                       \
-        } while (0)
-
-#define CHANGELOG_FILL_HTIME_DIR(changelog_dir, path) do {              \
-                strncpy (path, changelog_dir, sizeof (path) - 1);       \
-                strcat (path, "/htime");                                \
-        } while(0)
-
-#define CHANGELOG_FILL_CSNAP_DIR(changelog_dir, path) do {              \
-                strncpy (path, changelog_dir, sizeof (path) - 1);       \
-                strcat (path, "/csnap");                                \
-        } while(0)
+#define CHANGELOG_GET_HEADER_INFO(fd, buffer, len, enc, maj, min, elen)        \
+    do {                                                                       \
+        FILE *fp;                                                              \
+        int fd_dup;                                                            \
+                                                                               \
+        enc = -1;                                                              \
+        maj = -1;                                                              \
+        min = -1;                                                              \
+        fd_dup = dup(fd);                                                      \
+                                                                               \
+        if (fd_dup != -1) {                                                    \
+            fp = fdopen(fd_dup, "r");                                          \
+            if (fp) {                                                          \
+                if (fgets(buffer, len, fp)) {                                  \
+                    elen = strlen(buffer);                                     \
+                    sscanf(buffer, CHANGELOG_HEADER, &maj, &min, &enc);        \
+                }                                                              \
+                fclose(fp);                                                    \
+            } else {                                                           \
+                sys_close(fd_dup);                                             \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_FILL_HTIME_DIR(changelog_dir, path)                          \
+    do {                                                                       \
+        snprintf(path, sizeof(path), "%s/htime", changelog_dir);               \
+    } while (0)
+
+#define CHANGELOG_FILL_CSNAP_DIR(changelog_dir, path)                          \
+    do {                                                                       \
+        snprintf(path, sizeof(path), "%s/csnap", changelog_dir);               \
+    } while (0)
 /**
- * everything after 'CHANGELOG_TYPE_ENTRY' are internal types
+ * everything after 'CHANGELOG_TYPE_METADATA_XATTR' are internal types
  * (ie. none of the fops trigger this type of event), hence
- * CHANGELOG_MAX_TYPE = 3
+ * CHANGELOG_MAX_TYPE = 4
  */
 typedef enum {
-        CHANGELOG_TYPE_DATA = 0,
-        CHANGELOG_TYPE_METADATA,
-        CHANGELOG_TYPE_ENTRY,
-        CHANGELOG_TYPE_ROLLOVER,
-        CHANGELOG_TYPE_FSYNC,
+    CHANGELOG_TYPE_DATA = 0,
+    CHANGELOG_TYPE_METADATA,
+    CHANGELOG_TYPE_ENTRY,
+    CHANGELOG_TYPE_METADATA_XATTR,
+    CHANGELOG_TYPE_ROLLOVER,
+    CHANGELOG_TYPE_FSYNC,
 } changelog_log_type;
 
 /* operation modes - RT for now */
 typedef enum {
-        CHANGELOG_MODE_RT = 0,
+    CHANGELOG_MODE_RT = 0,
 } changelog_mode_t;
 
 /* encoder types */
 
 typedef enum {
-        CHANGELOG_ENCODE_MIN = 0,
-        CHANGELOG_ENCODE_BINARY,
-        CHANGELOG_ENCODE_ASCII,
-        CHANGELOG_ENCODE_MAX,
+    CHANGELOG_ENCODE_MIN = 0,
+    CHANGELOG_ENCODE_BINARY,
+    CHANGELOG_ENCODE_ASCII,
+    CHANGELOG_ENCODE_MAX,
 } changelog_encoder_t;
 
-#define CHANGELOG_VALID_ENCODING(enc)                                   \
-        (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
+#define CHANGELOG_VALID_ENCODING(enc)                                          \
+    (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
 
-#define CHANGELOG_TYPE_IS_ENTRY(type)  (type == CHANGELOG_TYPE_ENTRY)
-#define CHANGELOG_TYPE_IS_ROLLOVER(type)  (type == CHANGELOG_TYPE_ROLLOVER)
-#define CHANGELOG_TYPE_IS_FSYNC(type)  (type == CHANGELOG_TYPE_FSYNC)
+#define CHANGELOG_TYPE_IS_ENTRY(type) (type == CHANGELOG_TYPE_ENTRY)
+#define CHANGELOG_TYPE_IS_ROLLOVER(type) (type == CHANGELOG_TYPE_ROLLOVER)
+#define CHANGELOG_TYPE_IS_FSYNC(type) (type == CHANGELOG_TYPE_FSYNC)
 
 #endif /* _CHANGELOG_MISC_H */
diff --git a/xlators/features/changelog/src/changelog-rpc-common.c b/xlators/features/changelog/src/changelog-rpc-common.c
index 4525923d34d..125246a17e1 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.c
+++ b/xlators/features/changelog/src/changelog-rpc-common.c
@@ -11,7 +11,7 @@
 #include "changelog-rpc-common.h"
 #include "changelog-messages.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 /**
 *****************************************************
                   Client Interface
@@ -24,65 +24,63 @@
  */
 
 void *
-changelog_rpc_poller (void *arg)
+changelog_rpc_poller(void *arg)
 {
-        xlator_t *this = arg;
+    xlator_t *this = arg;
 
-        (void) event_dispatch (this->ctx->event_pool);
-        return NULL;
+    (void)gf_event_dispatch(this->ctx->event_pool);
+    return NULL;
 }
 
 struct rpc_clnt *
-changelog_rpc_client_init (xlator_t *this, void *cbkdata,
-                           char *sockfile, rpc_clnt_notify_t fn)
+changelog_rpc_client_init(xlator_t *this, void *cbkdata, char *sockfile,
+                          rpc_clnt_notify_t fn)
 {
-        int              ret         = 0;
-        struct rpc_clnt *rpc         = NULL;
-        dict_t          *options     = NULL;
-
-        if (!cbkdata)
-                cbkdata = this;
-
-        options = dict_new ();
-        if (!options)
-                goto error_return;
-
-        ret = rpc_transport_unix_options_build (&options, sockfile, 0);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_BUILD_ERROR,
-                        "failed to build rpc options");
-                goto dealloc_dict;
-        }
-
-        rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!rpc)
-                goto dealloc_dict;
-
-        ret = rpc_clnt_register_notify (rpc, fn, cbkdata);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
-                        "failed to register notify");
-                goto dealloc_rpc_clnt;
-        }
-
-        ret = rpc_clnt_start (rpc);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_START_ERROR,
-                        "failed to start rpc");
-                goto dealloc_rpc_clnt;
-        }
-
-        return rpc;
-
- dealloc_rpc_clnt:
-        rpc_clnt_unref (rpc);
- dealloc_dict:
-        dict_unref (options);
- error_return:
-        return NULL;
+    int ret = 0;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+
+    if (!cbkdata)
+        cbkdata = this;
+
+    options = dict_new();
+    if (!options)
+        goto error_return;
+
+    ret = rpc_transport_unix_options_build(options, sockfile, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_BUILD_ERROR,
+                NULL);
+        goto dealloc_dict;
+    }
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto dealloc_dict;
+
+    ret = rpc_clnt_register_notify(rpc, fn, cbkdata);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                CHANGELOG_MSG_NOTIFY_REGISTER_FAILED, NULL);
+        goto dealloc_rpc_clnt;
+    }
+
+    ret = rpc_clnt_start(rpc);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_START_ERROR,
+                NULL);
+        goto dealloc_rpc_clnt;
+    }
+
+    dict_unref(options);
+    return rpc;
+
+dealloc_rpc_clnt:
+    rpc_clnt_unref(rpc);
+dealloc_dict:
+    dict_unref(options);
+error_return:
+    return NULL;
 }
 
 /**
@@ -90,96 +88,96 @@ changelog_rpc_client_init (xlator_t *this, void *cbkdata,
  * RPC server.
  */
 int
-changelog_rpc_sumbit_req (struct rpc_clnt *rpc, void *req,
-                          call_frame_t *frame, rpc_clnt_prog_t *prog,
-                          int procnum, struct iovec *payload, int payloadcnt,
-                          struct iobref *iobref, xlator_t *this,
-                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+changelog_rpc_sumbit_req(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                         rpc_clnt_prog_t *prog, int procnum,
+                         struct iovec *payload, int payloadcnt,
+                         struct iobref *iobref, xlator_t *this,
+                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = 0;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
+    int ret = 0;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
 
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
 
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
 
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
 
-                iobref_add (iobref, iobuf);
+            new_iobref = 1;
+        }
 
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
+        iobref_add(iobref, iobuf);
 
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
 
-                iov.iov_len = ret;
-                count = 1;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
 
-        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn, &iov, count,
-                               payload, payloadcnt, iobref, frame, NULL,
-                               0, NULL, 0, NULL);
+        iov.iov_len = ret;
+        count = 1;
+    }
 
- out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, payload,
+                          payloadcnt, iobref, frame, NULL, 0, NULL, 0, NULL);
+
+out:
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 /**
  * Entry point to perform a remote procedure call
  */
 int
-changelog_invoke_rpc (xlator_t *this, struct rpc_clnt *rpc,
-                      rpc_clnt_prog_t *prog, int procidx, void *arg)
+changelog_invoke_rpc(xlator_t *this, struct rpc_clnt *rpc,
+                     rpc_clnt_prog_t *prog, int procidx, void *arg)
 {
-        int                   ret   = 0;
-        call_frame_t         *frame = NULL;
-        rpc_clnt_procedure_t *proc  = NULL;
-
-        if (!this || !prog)
-                goto error_return;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_CREATE_FRAME_FAILED,
-                        "failed to create frame");
-                goto error_return;
-        }
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
 
-        proc = &prog->proctable[procidx];
-        if (proc->fn)
-                ret = proc->fn (frame, this, arg);
+    if (!this || !prog)
+        goto error_return;
 
-        STACK_DESTROY (frame->root);
-        return ret;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_CREATE_FRAME_FAILED,
+                NULL);
+        goto error_return;
+    }
 
- error_return:
-        return -1;
+    proc = &prog->proctable[procidx];
+    if (proc->fn)
+        ret = proc->fn(frame, this, arg);
+
+    STACK_DESTROY(frame->root);
+    return ret;
+
+error_return:
+    return -1;
 }
 
 /**
@@ -189,161 +187,173 @@ changelog_invoke_rpc (xlator_t *this, struct rpc_clnt *rpc,
 */
 
 struct iobuf *
-__changelog_rpc_serialize_reply (rpcsvc_request_t *req, void *arg,
-                                 struct iovec *outmsg, xdrproc_t xdrproc)
+__changelog_rpc_serialize_reply(rpcsvc_request_t *req, void *arg,
+                                struct iovec *outmsg, xdrproc_t xdrproc)
 {
-        struct iobuf *iob      = NULL;
-        ssize_t       retlen   = 0;
-        ssize_t       rsp_size = 0;
+    struct iobuf *iob = NULL;
+    ssize_t retlen = 0;
+    ssize_t rsp_size = 0;
 
-        rsp_size = xdr_sizeof (xdrproc, arg);
-        iob = iobuf_get2 (req->svc->ctx->iobuf_pool, rsp_size);
-        if (!iob)
-                goto error_return;
+    rsp_size = xdr_sizeof(xdrproc, arg);
+    iob = iobuf_get2(req->svc->ctx->iobuf_pool, rsp_size);
+    if (!iob)
+        goto error_return;
 
-        iobuf_to_iovec (iob, outmsg);
+    iobuf_to_iovec(iob, outmsg);
 
-        retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
-        if (retlen == -1)
-                goto unref_iob;
+    retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+    if (retlen == -1)
+        goto unref_iob;
 
-        outmsg->iov_len = retlen;
-        return iob;
+    outmsg->iov_len = retlen;
+    return iob;
 
- unref_iob:
-        iobuf_unref (iob);
- error_return:
-        return NULL;
+unref_iob:
+    iobuf_unref(iob);
+error_return:
+    return NULL;
 }
 
 int
-changelog_rpc_sumbit_reply (rpcsvc_request_t *req,
-                            void *arg, struct iovec *payload, int payloadcount,
-                            struct iobref *iobref, xdrproc_t xdrproc)
+changelog_rpc_sumbit_reply(rpcsvc_request_t *req, void *arg,
+                           struct iovec *payload, int payloadcount,
+                           struct iobref *iobref, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        struct iobuf *iob        = NULL;
-        struct iovec  iov        = {0,};
-        char          new_iobref = 0;
-
-        if (!req)
-                goto return_ret;
-
-        if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref)
-                        goto return_ret;
-                new_iobref = 1;
-        }
-
-        iob = __changelog_rpc_serialize_reply (req, arg, &iov, xdrproc);
-        if (!iob)
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
-                        "failed to serialize reply");
-        else
-                iobref_add (iobref, iob);
-
-        ret = rpcsvc_submit_generic (req, &iov,
-                                     1, payload, payloadcount, iobref);
-
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iob)
-                iobuf_unref (iob);
- return_ret:
-        return ret;
+    int ret = -1;
+    struct iobuf *iob = NULL;
+    struct iovec iov = {
+        0,
+    };
+    char new_iobref = 0;
+
+    if (!req)
+        goto return_ret;
+
+    if (!iobref) {
+        iobref = iobref_new();
+        if (!iobref)
+            goto return_ret;
+        new_iobref = 1;
+    }
+
+    iob = __changelog_rpc_serialize_reply(req, arg, &iov, xdrproc);
+    if (!iob)
+        gf_smsg("", GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
+                NULL);
+    else
+        iobref_add(iobref, iob);
+
+    ret = rpcsvc_submit_generic(req, &iov, 1, payload, payloadcount, iobref);
+
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iob)
+        iobuf_unref(iob);
+return_ret:
+    return ret;
 }
 
 void
-changelog_rpc_server_destroy (xlator_t *this, rpcsvc_t *rpc, char *sockfile,
-                              rpcsvc_notify_t fn, struct rpcsvc_program **progs)
+changelog_rpc_server_destroy(xlator_t *this, rpcsvc_t *rpc, char *sockfile,
+                             rpcsvc_notify_t fn, struct rpcsvc_program **progs)
 {
-        rpcsvc_listener_t      *listener = NULL;
-        rpcsvc_listener_t      *next     = NULL;
-        struct rpcsvc_program  *prog     = NULL;
-
-        while (*progs) {
-                prog = *progs;
-                (void) rpcsvc_program_unregister (rpc, prog);
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    struct rpcsvc_program *prog = NULL;
+    rpc_transport_t *trans = NULL;
+
+    if (!rpc)
+        return;
+
+    while (*progs) {
+        prog = *progs;
+        (void)rpcsvc_program_unregister(rpc, prog);
+        progs++;
+    }
+
+    list_for_each_entry_safe(listener, next, &rpc->listeners, list)
+    {
+        if (listener->trans) {
+            trans = listener->trans;
+            rpc_transport_disconnect(trans, _gf_false);
         }
-
-        list_for_each_entry_safe (listener, next, &rpc->listeners, list) {
-                rpcsvc_listener_destroy (listener);
+    }
+
+    (void)rpcsvc_unregister_notify(rpc, fn, this);
+
+    /* TODO Avoid freeing rpc object in case of brick multiplex
+       after freeing rpc object svc->rpclock corrupted and it takes
+       more time to detach a brick
+    */
+    if (!this->cleanup_starting) {
+        if (rpc->rxpool) {
+            mem_pool_destroy(rpc->rxpool);
+            rpc->rxpool = NULL;
         }
-
-        (void) rpcsvc_unregister_notify (rpc, fn, this);
-        sys_unlink (sockfile);
-
-        GF_FREE (rpc);
+        GF_FREE(rpc);
+    }
 }
 
 rpcsvc_t *
-changelog_rpc_server_init (xlator_t *this, char *sockfile, void *cbkdata,
-                           rpcsvc_notify_t fn, struct rpcsvc_program **progs)
+changelog_rpc_server_init(xlator_t *this, char *sockfile, void *cbkdata,
+                          rpcsvc_notify_t fn, struct rpcsvc_program **progs)
 {
-        int                    j       = 0;
-        int                    ret     = 0;
-        rpcsvc_t              *rpc     = NULL;
-        dict_t                *options = NULL;
-        struct rpcsvc_program *prog    = NULL;
-
-        if (!cbkdata)
-                cbkdata = this;
-
-        options = dict_new ();
-        if (!options)
-                goto error_return;
-
-        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
-        if (ret)
-                goto dealloc_dict;
-
-        rpc = rpcsvc_init (this, this->ctx, options, 8);
-        if (rpc == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_START_ERROR,
-                        "failed to init rpc");
-                goto dealloc_dict;
-        }
+    int ret = 0;
+    rpcsvc_t *rpc = NULL;
+    dict_t *options = NULL;
+    struct rpcsvc_program *prog = NULL;
 
-        ret = rpcsvc_register_notify (rpc, fn, cbkdata);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
-                        "failed to register notify function");
-                goto dealloc_rpc;
-        }
+    if (!cbkdata)
+        cbkdata = this;
 
-        ret = rpcsvc_create_listeners (rpc, options, this->name);
-        if (ret != 1) {
-                gf_msg_debug (this->name,
-                              0, "failed to create listeners");
-                goto dealloc_rpc;
-        }
+    options = dict_new();
+    if (!options)
+        return NULL;
 
-        while (*progs) {
-                prog = *progs;
-                ret = rpcsvc_program_register (rpc, prog);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED,
-                                "cannot register program "
-                                "(name: %s, prognum: %d, pogver: %d)",
-                                prog->progname, prog->prognum, prog->progver);
-                        goto dealloc_rpc;
-                }
-
-                progs++;
+    ret = rpcsvc_transport_unix_options_build(options, sockfile);
+    if (ret)
+        goto dealloc_dict;
+
+    rpc = rpcsvc_init(this, this->ctx, options, 8);
+    if (rpc == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_START_ERROR,
+                NULL);
+        goto dealloc_dict;
+    }
+
+    ret = rpcsvc_register_notify(rpc, fn, cbkdata);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                CHANGELOG_MSG_NOTIFY_REGISTER_FAILED, NULL);
+        goto dealloc_rpc;
+    }
+
+    ret = rpcsvc_create_listeners(rpc, options, this->name);
+    if (ret != 1) {
+        gf_msg_debug(this->name, 0, "failed to create listeners");
+        goto dealloc_rpc;
+    }
+
+    while (*progs) {
+        prog = *progs;
+        ret = rpcsvc_program_register(rpc, prog, _gf_false);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED, "name%s",
+                    prog->progname, "prognum=%d", prog->prognum, "pogver=%d",
+                    prog->progver, NULL);
+            goto dealloc_rpc;
         }
 
-        dict_unref (options);
-        return rpc;
+        progs++;
+    }
 
- dealloc_rpc:
-        GF_FREE (rpc);
- dealloc_dict:
-        dict_unref (options);
- error_return:
-        return NULL;
+    dict_unref(options);
+    return rpc;
+
+dealloc_rpc:
+    GF_FREE(rpc);
+dealloc_dict:
+    dict_unref(options);
+    return NULL;
 }
diff --git a/xlators/features/changelog/src/changelog-rpc-common.h b/xlators/features/changelog/src/changelog-rpc-common.h
index 95c850c9400..4d9aa2c694b 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.h
+++ b/xlators/features/changelog/src/changelog-rpc-common.h
@@ -13,8 +13,8 @@
 
 #include "rpcsvc.h"
 #include "rpc-clnt.h"
-#include "event.h"
-#include "call-stub.h"
+#include <glusterfs/gf-event.h>
+#include <glusterfs/call-stub.h>
 
 #include "changelog-xdr.h"
 #include "xdr-generic.h"
@@ -24,61 +24,62 @@
 /**
  * Let's keep this non-configurable for now.
  */
-#define NR_ROTT_BUFFS  4
+#define NR_ROTT_BUFFS 4
 #define NR_DISPATCHERS (NR_ROTT_BUFFS - 1)
 
 enum changelog_rpc_procnum {
-        CHANGELOG_RPC_PROC_NULL    = 0,
-        CHANGELOG_RPC_PROBE_FILTER = 1,
-        CHANGELOG_RPC_PROC_MAX     = 2,
+    CHANGELOG_RPC_PROC_NULL = 0,
+    CHANGELOG_RPC_PROBE_FILTER = 1,
+    CHANGELOG_RPC_PROC_MAX = 2,
 };
 
-#define CHANGELOG_RPC_PROGNUM   1885957735
-#define CHANGELOG_RPC_PROGVER   1
+#define CHANGELOG_RPC_PROGNUM 1885957735
+#define CHANGELOG_RPC_PROGVER 1
 
 /**
  * reverse connection: data xfer path
  */
 enum changelog_reverse_rpc_procnum {
-        CHANGELOG_REV_PROC_NULL  = 0,
-        CHANGELOG_REV_PROC_EVENT = 1,
-        CHANGELOG_REV_PROC_MAX   = 2,
+    CHANGELOG_REV_PROC_NULL = 0,
+    CHANGELOG_REV_PROC_EVENT = 1,
+    CHANGELOG_REV_PROC_MAX = 2,
 };
 
-#define CHANGELOG_REV_RPC_PROCNUM   1886350951
-#define CHANGELOG_REV_RPC_PROCVER   1
+#define CHANGELOG_REV_RPC_PROCNUM 1886350951
+#define CHANGELOG_REV_RPC_PROCVER 1
 
 typedef struct changelog_rpc {
-        rpcsvc_t        *svc;
-        struct rpc_clnt *rpc;
-        char             sock[UNIX_PATH_MAX];  /* tied to server */
+    rpcsvc_t *svc;
+    struct rpc_clnt *rpc;
+    char sock[UNIX_PATH_MAX]; /* tied to server */
 } changelog_rpc_t;
 
 /* event poller */
-void *changelog_rpc_poller (void *);
+void *
+changelog_rpc_poller(void *);
 
 /* CLIENT API */
 struct rpc_clnt *
-changelog_rpc_client_init (xlator_t *, void *, char *, rpc_clnt_notify_t);
+changelog_rpc_client_init(xlator_t *, void *, char *, rpc_clnt_notify_t);
 
 int
-changelog_rpc_sumbit_req (struct rpc_clnt *, void *, call_frame_t *,
-                          rpc_clnt_prog_t *, int , struct iovec *, int,
-                          struct iobref *, xlator_t *, fop_cbk_fn_t, xdrproc_t);
+changelog_rpc_sumbit_req(struct rpc_clnt *, void *, call_frame_t *,
+                         rpc_clnt_prog_t *, int, struct iovec *, int,
+                         struct iobref *, xlator_t *, fop_cbk_fn_t, xdrproc_t);
 
 int
-changelog_invoke_rpc (xlator_t *, struct rpc_clnt *,
-                      rpc_clnt_prog_t *, int , void *);
+changelog_invoke_rpc(xlator_t *, struct rpc_clnt *, rpc_clnt_prog_t *, int,
+                     void *);
 
 /* SERVER API */
 int
-changelog_rpc_sumbit_reply (rpcsvc_request_t *, void *,
-                            struct iovec *, int, struct iobref *, xdrproc_t);
+changelog_rpc_sumbit_reply(rpcsvc_request_t *, void *, struct iovec *, int,
+                           struct iobref *, xdrproc_t);
 rpcsvc_t *
-changelog_rpc_server_init (xlator_t *, char *, void*,
-                           rpcsvc_notify_t, struct rpcsvc_program **);
+changelog_rpc_server_init(xlator_t *, char *, void *, rpcsvc_notify_t,
+                          struct rpcsvc_program **);
 void
-changelog_rpc_server_destroy (xlator_t *, rpcsvc_t *, char *,
-                              rpcsvc_notify_t, struct rpcsvc_program **);
+changelog_rpc_server_destroy(xlator_t *, rpcsvc_t *, char *, rpcsvc_notify_t,
+                             struct rpcsvc_program **);
 
 #endif
diff --git a/xlators/features/changelog/src/changelog-rpc.c b/xlators/features/changelog/src/changelog-rpc.c
index 76addf18545..440b88091a6 100644
--- a/xlators/features/changelog/src/changelog-rpc.c
+++ b/xlators/features/changelog/src/changelog-rpc.c
@@ -8,216 +8,346 @@
    cases as published by the Free Software Foundation.
 */
 
+#include <glusterfs/syscall.h>
 #include "changelog-rpc.h"
 #include "changelog-mem-types.h"
 #include "changelog-ev-handle.h"
 
-struct rpcsvc_program *changelog_programs[];
+static struct rpcsvc_program *changelog_programs[];
 
 static void
-changelog_cleanup_dispatchers (xlator_t *this,
-                               changelog_priv_t *priv, int count)
+changelog_cleanup_dispatchers(xlator_t *this, changelog_priv_t *priv, int count)
 {
-        for (; count >= 0; count--) {
-                (void) changelog_thread_cleanup
-                        (this, priv->ev_dispatcher[count]);
-        }
+    for (count--; count >= 0; count--) {
+        (void)changelog_thread_cleanup(this, priv->ev_dispatcher[count]);
+        priv->ev_dispatcher[count] = 0;
+    }
 }
 
-static int
-changelog_cleanup_rpc_threads (xlator_t *this, changelog_priv_t *priv)
+int
+changelog_cleanup_rpc_threads(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-        changelog_clnt_t *conn = NULL;
-
-        conn = &priv->connections;
-        if (!conn)
-                return 0;
-
-        /** terminate RPC thread(s) */
-        ret = changelog_thread_cleanup (this, priv->connector);
-        if (ret != 0)
-                goto error_return;
-        /** terminate dispatcher thread(s) */
-        changelog_cleanup_dispatchers (this, priv, priv->nr_dispatchers);
-
-        /* TODO: what about pending and waiting connections? */
-        changelog_ev_cleanup_connections (this, conn);
-
-        /* destroy locks */
-        ret = pthread_mutex_destroy (&conn->pending_lock);
-        if (ret != 0)
-                goto error_return;
-        ret = pthread_cond_destroy (&conn->pending_cond);
-        if (ret != 0)
-                goto error_return;
-        ret = LOCK_DESTROY (&conn->active_lock);
-        if (ret != 0)
-                goto error_return;
-        ret = LOCK_DESTROY (&conn->wait_lock);
-        if (ret != 0)
-                goto error_return;
+    int ret = 0;
+    changelog_clnt_t *conn = NULL;
+
+    conn = &priv->connections;
+    if (!conn)
         return 0;
 
- error_return:
-        return -1;
+    /** terminate RPC thread(s) */
+    ret = changelog_thread_cleanup(this, priv->connector);
+    if (ret != 0)
+        goto error_return;
+    priv->connector = 0;
+
+    /** terminate dispatcher thread(s) */
+    changelog_cleanup_dispatchers(this, priv, priv->nr_dispatchers);
+
+    /* destroy locks */
+    ret = pthread_mutex_destroy(&conn->pending_lock);
+    if (ret != 0)
+        goto error_return;
+    ret = pthread_cond_destroy(&conn->pending_cond);
+    if (ret != 0)
+        goto error_return;
+    ret = LOCK_DESTROY(&conn->active_lock);
+    if (ret != 0)
+        goto error_return;
+    ret = LOCK_DESTROY(&conn->wait_lock);
+    if (ret != 0)
+        goto error_return;
+    return 0;
+
+error_return:
+    return -1;
 }
 
 static int
-changelog_init_rpc_threads (xlator_t *this, changelog_priv_t *priv,
-                            rbuf_t *rbuf, int nr_dispatchers)
+changelog_init_rpc_threads(xlator_t *this, changelog_priv_t *priv, rbuf_t *rbuf,
+                           int nr_dispatchers)
+{
+    int j = 0;
+    int ret = 0;
+    changelog_clnt_t *conn = NULL;
+
+    conn = &priv->connections;
+
+    conn->this = this;
+    conn->rbuf = rbuf;
+    conn->sequence = 1; /* start with sequence number one */
+
+    INIT_LIST_HEAD(&conn->pending);
+    INIT_LIST_HEAD(&conn->active);
+    INIT_LIST_HEAD(&conn->waitq);
+
+    ret = pthread_mutex_init(&conn->pending_lock, NULL);
+    if (ret)
+        goto error_return;
+    ret = pthread_cond_init(&conn->pending_cond, NULL);
+    if (ret)
+        goto cleanup_pending_lock;
+
+    ret = LOCK_INIT(&conn->active_lock);
+    if (ret)
+        goto cleanup_pending_cond;
+    ret = LOCK_INIT(&conn->wait_lock);
+    if (ret)
+        goto cleanup_active_lock;
+
+    /* spawn reverse connection thread */
+    ret = gf_thread_create(&priv->connector, NULL, changelog_ev_connector, conn,
+                           "clogecon");
+    if (ret != 0)
+        goto cleanup_wait_lock;
+
+    /* spawn dispatcher thread(s) */
+    priv->ev_dispatcher = GF_CALLOC(nr_dispatchers, sizeof(pthread_t),
+                                    gf_changelog_mt_ev_dispatcher_t);
+    if (!priv->ev_dispatcher)
+        goto cleanup_connector;
+
+    /* spawn dispatcher threads */
+    for (; j < nr_dispatchers; j++) {
+        ret = gf_thread_create(&priv->ev_dispatcher[j], NULL,
+                               changelog_ev_dispatch, conn, "clogd%03hx",
+                               j & 0x3ff);
+        if (ret != 0) {
+            changelog_cleanup_dispatchers(this, priv, j);
+            break;
+        }
+    }
+
+    if (ret != 0)
+        goto cleanup_connector;
+
+    priv->nr_dispatchers = nr_dispatchers;
+    return 0;
+
+cleanup_connector:
+    (void)pthread_cancel(priv->connector);
+cleanup_wait_lock:
+    LOCK_DESTROY(&conn->wait_lock);
+cleanup_active_lock:
+    LOCK_DESTROY(&conn->active_lock);
+cleanup_pending_cond:
+    (void)pthread_cond_destroy(&conn->pending_cond);
+cleanup_pending_lock:
+    (void)pthread_mutex_destroy(&conn->pending_lock);
+error_return:
+    return -1;
+}
+
+int
+changelog_rpcsvc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                        void *data)
 {
-        int               j    = 0;
-        int               ret  = 0;
-        changelog_clnt_t *conn = NULL;
-
-        conn = &priv->connections;
-
-        conn->this = this;
-        conn->rbuf = rbuf;
-        conn->sequence = 1; /* start with sequence number one */
-
-        INIT_LIST_HEAD (&conn->pending);
-        INIT_LIST_HEAD (&conn->active);
-        INIT_LIST_HEAD (&conn->waitq);
-
-        ret = pthread_mutex_init (&conn->pending_lock, NULL);
-        if (ret)
-                goto error_return;
-        ret = pthread_cond_init (&conn->pending_cond, NULL);
-        if (ret)
-                goto cleanup_pending_lock;
-
-        ret = LOCK_INIT (&conn->active_lock);
-        if (ret)
-                goto cleanup_pending_cond;
-        ret = LOCK_INIT (&conn->wait_lock);
-        if (ret)
-                goto cleanup_active_lock;
-
-        /* spawn reverse connection thread */
-        ret = pthread_create (&priv->connector,
-                              NULL, changelog_ev_connector, conn);
-        if (ret != 0)
-                goto cleanup_wait_lock;
-
-        /* spawn dispatcher thread(s) */
-        priv->ev_dispatcher = GF_CALLOC (nr_dispatchers, sizeof(pthread_t),
-                                         gf_changelog_mt_ev_dispatcher_t);
-        if (!priv->ev_dispatcher)
-                goto cleanup_connector;
-
-        /* spawn dispatcher threads */
-        for (; j < nr_dispatchers; j++) {
-                ret = pthread_create (&priv->ev_dispatcher[j],
-                                      NULL, changelog_ev_dispatch, conn);
-                if (ret != 0) {
-                        changelog_cleanup_dispatchers (this, priv, --j);
-                        break;
+    xlator_t *this = NULL;
+    rpc_transport_t *trans = NULL;
+    rpc_transport_t *xprt = NULL;
+    rpc_transport_t *xp_next = NULL;
+    changelog_priv_t *priv = NULL;
+    uint64_t listnercnt = 0;
+    uint64_t xprtcnt = 0;
+    uint64_t clntcnt = 0;
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    gf_boolean_t listner_found = _gf_false;
+    socket_private_t *sockpriv = NULL;
+
+    if (!xl || !data || !rpc) {
+        gf_msg_callingfn("changelog", GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED,
+                         "Calling rpc_notify without initializing");
+        goto out;
+    }
+
+    this = xl;
+    trans = data;
+    priv = this->private;
+
+    if (!priv) {
+        gf_msg_callingfn("changelog", GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED,
+                         "Calling rpc_notify without priv initializing");
+        goto out;
+    }
+
+    if (event == RPCSVC_EVENT_ACCEPT) {
+        GF_ATOMIC_INC(priv->xprtcnt);
+        LOCK(&priv->lock);
+        {
+            list_add_tail(&trans->list, &priv->xprt_list);
+        }
+        UNLOCK(&priv->lock);
+        goto out;
+    }
+
+    if (event == RPCSVC_EVENT_DISCONNECT) {
+        list_for_each_entry_safe(listener, next, &rpc->listeners, list)
+        {
+            if (listener && listener->trans) {
+                if (listener->trans == trans) {
+                    listnercnt = GF_ATOMIC_DEC(priv->listnercnt);
+                    listner_found = _gf_true;
+                    rpcsvc_listener_destroy(listener);
                 }
+            }
         }
 
-        if (ret != 0)
-                goto cleanup_connector;
+        if (listnercnt > 0) {
+            goto out;
+        }
+        if (listner_found) {
+            LOCK(&priv->lock);
+            list_for_each_entry_safe(xprt, xp_next, &priv->xprt_list, list)
+            {
+                sockpriv = (socket_private_t *)(xprt->private);
+                gf_log("changelog", GF_LOG_INFO,
+                       "Send disconnect"
+                       " on socket %d",
+                       sockpriv->sock);
+                rpc_transport_disconnect(xprt, _gf_false);
+            }
+            UNLOCK(&priv->lock);
+            goto out;
+        }
+        LOCK(&priv->lock);
+        {
+            list_del_init(&trans->list);
+        }
+        UNLOCK(&priv->lock);
 
-        priv->nr_dispatchers = nr_dispatchers;
-        return 0;
+        xprtcnt = GF_ATOMIC_DEC(priv->xprtcnt);
+        clntcnt = GF_ATOMIC_GET(priv->clntcnt);
+        if (!xprtcnt && !clntcnt) {
+            changelog_process_cleanup_event(this);
+        }
+    }
 
- cleanup_connector:
-        (void) pthread_cancel (priv->connector);
- cleanup_wait_lock:
-        LOCK_DESTROY (&conn->wait_lock);
- cleanup_active_lock:
-        LOCK_DESTROY (&conn->active_lock);
- cleanup_pending_cond:
-        (void) pthread_cond_destroy (&conn->pending_cond);
- cleanup_pending_lock:
-        (void) pthread_mutex_destroy (&conn->pending_lock);
- error_return:
-        return -1;
+out:
+    return 0;
 }
 
-int
-changelog_rpcsvc_notify (rpcsvc_t *rpc,
-                         void *xl, rpcsvc_event_t event, void *data)
+void
+changelog_process_cleanup_event(xlator_t *this)
 {
-        return 0;
+    gf_boolean_t cleanup_notify = _gf_false;
+    changelog_priv_t *priv = NULL;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+
+    if (!this)
+        return;
+    priv = this->private;
+    if (!priv)
+        return;
+
+    LOCK(&priv->lock);
+    {
+        cleanup_notify = priv->notify_down;
+        priv->notify_down = _gf_true;
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->victim && !cleanup_notify) {
+        default_notify(this, GF_EVENT_PARENT_DOWN, priv->victim);
+
+        if (priv->rpc) {
+            /* sockfile path could have been saved to avoid this */
+            CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile,
+                                       UNIX_PATH_MAX);
+            sys_unlink(sockfile);
+            (void)rpcsvc_unregister_notify(priv->rpc, changelog_rpcsvc_notify,
+                                           this);
+            if (priv->rpc->rxpool) {
+                mem_pool_destroy(priv->rpc->rxpool);
+                priv->rpc->rxpool = NULL;
+            }
+            GF_FREE(priv->rpc);
+            priv->rpc = NULL;
+        }
+    }
 }
 
 void
-changelog_destroy_rpc_listner (xlator_t *this, changelog_priv_t *priv)
+changelog_destroy_rpc_listner(xlator_t *this, changelog_priv_t *priv)
 {
-        char sockfile[UNIX_PATH_MAX] = {0,};
-
-        /* sockfile path could have been saved to avoid this */
-        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
-                                    sockfile, UNIX_PATH_MAX);
-        changelog_rpc_server_destroy (this,
-                                      priv->rpc, sockfile,
-                                      changelog_rpcsvc_notify,
-                                      changelog_programs);
-        (void) changelog_cleanup_rpc_threads (this, priv);
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+
+    /* sockfile path could have been saved to avoid this */
+    CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile, UNIX_PATH_MAX);
+    changelog_rpc_server_destroy(this, priv->rpc, sockfile,
+                                 changelog_rpcsvc_notify, changelog_programs);
 }
 
 rpcsvc_t *
-changelog_init_rpc_listner (xlator_t *this, changelog_priv_t *priv,
+changelog_init_rpc_listener(xlator_t *this, changelog_priv_t *priv,
                             rbuf_t *rbuf, int nr_dispatchers)
 {
-        int ret = 0;
-        char sockfile[UNIX_PATH_MAX] = {0,};
-
-        ret = changelog_init_rpc_threads (this, priv, rbuf, nr_dispatchers);
-        if (ret)
-                return NULL;
-
-        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
-                                    sockfile, UNIX_PATH_MAX);
-        return changelog_rpc_server_init (this, sockfile, NULL,
-                                          changelog_rpcsvc_notify,
-                                          changelog_programs);
+    int ret = 0;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+    rpcsvc_t *svcp;
+
+    ret = changelog_init_rpc_threads(this, priv, rbuf, nr_dispatchers);
+    if (ret)
+        return NULL;
+
+    CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile, UNIX_PATH_MAX);
+    (void)sys_unlink(sockfile);
+    svcp = changelog_rpc_server_init(
+        this, sockfile, NULL, changelog_rpcsvc_notify, changelog_programs);
+    return svcp;
 }
 
 void
-changelog_rpc_clnt_cleanup (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_cleanup(changelog_rpc_clnt_t *crpc)
 {
-        if (!crpc)
-                return;
-        crpc->c_clnt = NULL;
-        LOCK_DESTROY (&crpc->lock);
-        GF_FREE (crpc);
+    if (!crpc)
+        return;
+    crpc->c_clnt = NULL;
+    LOCK_DESTROY(&crpc->lock);
+    GF_FREE(crpc);
 }
 
 static changelog_rpc_clnt_t *
-changelog_rpc_clnt_init (xlator_t *this,
-                         changelog_probe_req *rpc_req, changelog_clnt_t *c_clnt)
+changelog_rpc_clnt_init(xlator_t *this, changelog_probe_req *rpc_req,
+                        changelog_clnt_t *c_clnt)
 {
-        int ret = 0;
-        changelog_rpc_clnt_t *crpc = NULL;
-
-        crpc = GF_CALLOC (1, sizeof (*crpc), gf_changelog_mt_rpc_clnt_t);
-        if (!crpc)
-                goto error_return;
-        INIT_LIST_HEAD (&crpc->list);
-
-        crpc->ref = 0;
-        changelog_set_disconnect_flag (crpc, _gf_false);
-
-        crpc->filter = rpc_req->filter;
-        (void) memcpy (crpc->sock, rpc_req->sock, strlen (rpc_req->sock));
-
-        crpc->this = this;
-        crpc->c_clnt = c_clnt;
-        crpc->cleanup = changelog_rpc_clnt_cleanup;
-
-        ret = LOCK_INIT (&crpc->lock);
-        if (ret != 0)
-                goto dealloc_crpc;
-        return crpc;
-
- dealloc_crpc:
-        GF_FREE (crpc);
- error_return:
-        return NULL;
+    int ret = 0;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    crpc = GF_CALLOC(1, sizeof(*crpc), gf_changelog_mt_rpc_clnt_t);
+    if (!crpc)
+        goto error_return;
+    INIT_LIST_HEAD(&crpc->list);
+
+    /* Take a ref, the last unref will be on RPC_CLNT_DESTROY
+     * which comes as a result of last rpc_clnt_unref.
+     */
+    GF_ATOMIC_INIT(crpc->ref, 1);
+    changelog_set_disconnect_flag(crpc, _gf_false);
+
+    crpc->filter = rpc_req->filter;
+    (void)memcpy(crpc->sock, rpc_req->sock, strlen(rpc_req->sock));
+
+    crpc->this = this;
+    crpc->c_clnt = c_clnt;
+    crpc->cleanup = changelog_rpc_clnt_cleanup;
+
+    ret = LOCK_INIT(&crpc->lock);
+    if (ret != 0)
+        goto dealloc_crpc;
+    return crpc;
+
+dealloc_crpc:
+    GF_FREE(crpc);
+error_return:
+    return NULL;
 }
 
 /**
@@ -231,72 +361,80 @@ changelog_rpc_clnt_init (xlator_t *this,
  */
 
 int
-changelog_handle_probe (rpcsvc_request_t *req)
+changelog_handle_probe(rpcsvc_request_t *req)
 {
-        int                   ret     = 0;
-        xlator_t             *this    = NULL;
-        rpcsvc_t             *svc     = NULL;
-        changelog_priv_t     *priv    = NULL;
-        changelog_clnt_t     *c_clnt  = NULL;
-        changelog_rpc_clnt_t *crpc    = NULL;
-
-        changelog_probe_req   rpc_req = {0,};
-        changelog_probe_rsp   rpc_rsp = {0,};
-
-        ret = xdr_to_generic (req->msg[0],
-                              &rpc_req, (xdrproc_t)xdr_changelog_probe_req);
-        if (ret < 0) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_HANDLE_PROBE_ERROR,
-                        "xdr decoding error");
-                req->rpc_err = GARBAGE_ARGS;
-                goto handle_xdr_error;
-        }
-
-        /* ->xl hidden in rpcsvc */
-        svc    = rpcsvc_request_service (req);
-        this   = svc->xl;
-        priv   = this->private;
-        c_clnt = &priv->connections;
-
-        crpc = changelog_rpc_clnt_init (this, &rpc_req, c_clnt);
-        if (!crpc)
-                goto handle_xdr_error;
-
-        changelog_ev_queue_connection (c_clnt, crpc);
-        rpc_rsp.op_ret = 0;
-
-        goto submit_rpc;
-
- handle_xdr_error:
-        rpc_rsp.op_ret = -1;
- submit_rpc:
-        (void) changelog_rpc_sumbit_reply (req, &rpc_rsp, NULL, 0, NULL,
-                                           (xdrproc_t)xdr_changelog_probe_rsp);
+    int ret = 0;
+    xlator_t *this = NULL;
+    rpcsvc_t *svc = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    changelog_probe_req rpc_req = {
+        0,
+    };
+    changelog_probe_rsp rpc_rsp = {
+        0,
+    };
+
+    this = req->trans->xl;
+    if (this->cleanup_starting) {
+        gf_smsg(this->name, GF_LOG_DEBUG, 0, CHANGELOG_MSG_CLEANUP_ALREADY_SET,
+                NULL);
         return 0;
+    }
+
+    ret = xdr_to_generic(req->msg[0], &rpc_req,
+                         (xdrproc_t)xdr_changelog_probe_req);
+    if (ret < 0) {
+        gf_smsg("", GF_LOG_ERROR, 0, CHANGELOG_MSG_HANDLE_PROBE_ERROR, NULL);
+        req->rpc_err = GARBAGE_ARGS;
+        goto handle_xdr_error;
+    }
+
+    /* ->xl hidden in rpcsvc */
+    svc = rpcsvc_request_service(req);
+    this = svc->xl;
+    priv = this->private;
+    c_clnt = &priv->connections;
+
+    crpc = changelog_rpc_clnt_init(this, &rpc_req, c_clnt);
+    if (!crpc)
+        goto handle_xdr_error;
+
+    changelog_ev_queue_connection(c_clnt, crpc);
+    rpc_rsp.op_ret = 0;
+
+    goto submit_rpc;
+
+handle_xdr_error:
+    rpc_rsp.op_ret = -1;
+submit_rpc:
+    (void)changelog_rpc_sumbit_reply(req, &rpc_rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_changelog_probe_rsp);
+    return 0;
 }
 
 /**
  * RPC declarations
  */
 
-rpcsvc_actor_t changelog_svc_actors[CHANGELOG_RPC_PROC_MAX] = {
-        [CHANGELOG_RPC_PROBE_FILTER] = {
-                "CHANGELOG PROBE FILTER", CHANGELOG_RPC_PROBE_FILTER,
-                changelog_handle_probe, NULL, 0, DRC_NA
-        },
+static rpcsvc_actor_t changelog_svc_actors[CHANGELOG_RPC_PROC_MAX] = {
+    [CHANGELOG_RPC_PROBE_FILTER] = {"CHANGELOG PROBE FILTER",
+                                    changelog_handle_probe, NULL,
+                                    CHANGELOG_RPC_PROBE_FILTER, DRC_NA, 0},
 };
 
-struct rpcsvc_program changelog_svc_prog = {
-        .progname  = CHANGELOG_RPC_PROGNAME,
-        .prognum   = CHANGELOG_RPC_PROGNUM,
-        .progver   = CHANGELOG_RPC_PROGVER,
-        .numactors = CHANGELOG_RPC_PROC_MAX,
-        .actors    = changelog_svc_actors,
-        .synctask  = _gf_true,
+static struct rpcsvc_program changelog_svc_prog = {
+    .progname = CHANGELOG_RPC_PROGNAME,
+    .prognum = CHANGELOG_RPC_PROGNUM,
+    .progver = CHANGELOG_RPC_PROGVER,
+    .numactors = CHANGELOG_RPC_PROC_MAX,
+    .actors = changelog_svc_actors,
+    .synctask = _gf_true,
 };
 
-struct rpcsvc_program *changelog_programs[] = {
-        &changelog_svc_prog,
-        NULL,
+static struct rpcsvc_program *changelog_programs[] = {
+    &changelog_svc_prog,
+    NULL,
 };
diff --git a/xlators/features/changelog/src/changelog-rpc.h b/xlators/features/changelog/src/changelog-rpc.h
index 0df96684b6c..b1707565249 100644
--- a/xlators/features/changelog/src/changelog-rpc.h
+++ b/xlators/features/changelog/src/changelog-rpc.h
@@ -11,19 +11,21 @@
 #ifndef __CHANGELOG_RPC_H
 #define __CHANGELOG_RPC_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "changelog-helpers.h"
 
 /* one time */
 #include "socket.h"
 #include "changelog-rpc-common.h"
 
-#define CHANGELOG_RPC_PROGNAME  "GlusterFS Changelog"
+#define CHANGELOG_RPC_PROGNAME "GlusterFS Changelog"
 
 rpcsvc_t *
-changelog_init_rpc_listner (xlator_t *, changelog_priv_t *, rbuf_t *, int);
+changelog_init_rpc_listener(xlator_t *, changelog_priv_t *, rbuf_t *, int);
 
 void
-changelog_destroy_rpc_listner (xlator_t *, changelog_priv_t *);
+changelog_destroy_rpc_listner(xlator_t *, changelog_priv_t *);
 
+int
+changelog_cleanup_rpc_threads(xlator_t *this, changelog_priv_t *priv);
 #endif
diff --git a/xlators/features/changelog/src/changelog-rt.c b/xlators/features/changelog/src/changelog-rt.c
index c262820c64c..841545ae359 100644
--- a/xlators/features/changelog/src/changelog-rt.c
+++ b/xlators/features/changelog/src/changelog-rt.c
@@ -8,60 +8,59 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
 
 #include "changelog-rt.h"
 #include "changelog-mem-types.h"
 
 int
-changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd)
+changelog_rt_init(xlator_t *this, changelog_dispatcher_t *cd)
 {
-        changelog_rt_t *crt = NULL;
+    changelog_rt_t *crt = NULL;
 
-        crt = GF_CALLOC (1, sizeof (*crt),
-                         gf_changelog_mt_rt_t);
-        if (!crt)
-                return -1;
+    crt = GF_CALLOC(1, sizeof(*crt), gf_changelog_mt_rt_t);
+    if (!crt)
+        return -1;
 
-        LOCK_INIT (&crt->lock);
+    LOCK_INIT(&crt->lock);
 
-        cd->cd_data = crt;
-        cd->dispatchfn = &changelog_rt_enqueue;
+    cd->cd_data = crt;
+    cd->dispatchfn = &changelog_rt_enqueue;
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd)
+changelog_rt_fini(xlator_t *this, changelog_dispatcher_t *cd)
 {
-        changelog_rt_t *crt = NULL;
+    changelog_rt_t *crt = NULL;
 
-        crt = cd->cd_data;
+    crt = cd->cd_data;
 
-        LOCK_DESTROY (&crt->lock);
-        GF_FREE (crt);
+    LOCK_DESTROY(&crt->lock);
+    GF_FREE(crt);
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
-                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
+changelog_rt_enqueue(xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                     changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
 {
-        int             ret = 0;
-        changelog_rt_t *crt = NULL;
+    int ret = 0;
+    changelog_rt_t *crt = NULL;
 
-        crt = (changelog_rt_t *) cbatch;
+    crt = (changelog_rt_t *)cbatch;
 
-        LOCK (&crt->lock);
-        {
-                ret = changelog_handle_change (this, priv, cld_0);
-                if (!ret && cld_1)
-                        ret = changelog_handle_change (this, priv, cld_1);
-        }
-        UNLOCK (&crt->lock);
+    LOCK(&crt->lock);
+    {
+        ret = changelog_handle_change(this, priv, cld_0);
+        if (!ret && cld_1)
+            ret = changelog_handle_change(this, priv, cld_1);
+    }
+    UNLOCK(&crt->lock);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-rt.h b/xlators/features/changelog/src/changelog-rt.h
index 1fc2bbc5bb9..28b9827d85b 100644
--- a/xlators/features/changelog/src/changelog-rt.h
+++ b/xlators/features/changelog/src/changelog-rt.h
@@ -11,23 +11,23 @@
 #ifndef _CHANGELOG_RT_H
 #define _CHANGELOG_RT_H
 
-#include "locking.h"
-#include "timer.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/timer.h>
 #include "pthread.h"
 
 #include "changelog-helpers.h"
 
 /* unused as of now - may be you would need it later */
 typedef struct changelog_rt {
-        gf_lock_t lock;
+    gf_lock_t lock;
 } changelog_rt_t;
 
 int
-changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd);
+changelog_rt_init(xlator_t *this, changelog_dispatcher_t *cd);
 int
-changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd);
+changelog_rt_fini(xlator_t *this, changelog_dispatcher_t *cd);
 int
-changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
-                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
+changelog_rt_enqueue(xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                     changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
 
 #endif /* _CHANGELOG_RT_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
index f8f95cf0e81..6a6e5af859e 100644
--- a/xlators/features/changelog/src/changelog.c
+++ b/xlators/features/changelog/src/changelog.c
@@ -8,11 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "syscall.h"
-#include "logging.h"
-#include "iobuf.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/iobuf.h>
 
 #include "changelog-rt.h"
 
@@ -21,19 +21,25 @@
 #include "changelog-messages.h"
 
 #include <pthread.h>
+#include <signal.h>
 
 #include "changelog-rpc.h"
 #include "errno.h"
 
-static struct changelog_bootstrap
-cb_bootstrap[] = {
-        {
-                .mode = CHANGELOG_MODE_RT,
-                .ctor = changelog_rt_init,
-                .dtor = changelog_rt_fini,
-        },
+static struct changelog_bootstrap cb_bootstrap[] = {
+    {
+        .mode = CHANGELOG_MODE_RT,
+        .ctor = changelog_rt_init,
+        .dtor = changelog_rt_fini,
+    },
 };
 
+static int
+changelog_init_rpc(xlator_t *this, changelog_priv_t *priv);
+
+static int
+changelog_init(xlator_t *this, changelog_priv_t *priv);
+
 /* Entry operations - TYPE III */
 
 /**
@@ -45,1109 +51,1052 @@ cb_bootstrap[] = {
 /* rmdir */
 
 int32_t
-changelog_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                                preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rmdir_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, int xflags, dict_t *xdata)
+changelog_rmdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int xflags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeue rmdir");
-                      changelog_color_fop_and_inc_cnt (this, priv,
-                      frame->local);
-        STACK_WIND (frame, changelog_rmdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
-                    loc, xflags, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue rmdir");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rmdir (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int xflags, dict_t *xdata)
-{
-        size_t                  xtra_len                = 0;
-        changelog_priv_t       *priv                    = NULL;
-        changelog_opt_t        *co                      = NULL;
-        call_stub_t            *stub                    = NULL;
-        struct list_head        queue                   = {0, };
-        gf_boolean_t            barrier_enabled         = _gf_false;
-
-        INIT_LIST_HEAD (&queue);
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                NULL, loc->inode->gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        if (priv->capture_del_path) {
-                CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid, loc->name,
-                                               del_entry_fn, del_entry_free_fn,
-                                              xtra_len, wind, _gf_true);
+changelog_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+                dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    if (priv->capture_del_path) {
+        CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name, del_entry_fn,
+                                      del_entry_free_fn, xtra_len, wind,
+                                      _gf_true);
+    } else {
+        CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name, del_entry_fn,
+                                      del_entry_free_fn, xtra_len, wind,
+                                      _gf_false);
+    }
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    /* changelog barrier */
+    /* Color assignment and increment of fop_cnt for rmdir/unlink/rename
+     * should be made with in priv lock if changelog barrier is not enabled.
+     * Because if counter is not incremented yet, draining wakes up and
+     * publishes the changelog but later these fops might hit the disk and
+     * present in snapped volume but where as the intention is these fops
+     * should not be present in snapped volume.
+     */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_rmdir_stub(frame, changelog_rmdir_resume, loc, xflags,
+                                  xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
         } else {
-                CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid, loc->name,
-                                               del_entry_fn, del_entry_free_fn,
-                                              xtra_len, wind, _gf_false);
-        }
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-/* changelog barrier */
-        /* Color assignment and increment of fop_cnt for rmdir/unlink/rename
-         * should be made with in priv lock if changelog barrier is not enabled.
-         * Because if counter is not incremented yet, draining wakes up and
-         * publishes the changelog but later these fops might hit the disk and
-         * present in snapped volume but where as the intention is these fops
-         * should not be present in snapped volume.
-         */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_rmdir_stub (frame, changelog_rmdir_resume,
-                                                loc, xflags, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueue rmdir");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: rmdir");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
-/* changelog barrier */
-
- wind:
-        STACK_WIND (frame, changelog_rmdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
-                    loc, xflags, xdata);
- out:
-        return 0;
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue rmdir");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=rmdir", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+out:
+    return 0;
 }
 
 /* unlink */
 
 int32_t
-changelog_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+changelog_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                                preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_unlink_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *loc, int xflags, dict_t *xdata)
+changelog_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        int xflags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeue unlink");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_unlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                    loc, xflags, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue unlink");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_unlink (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, int xflags, dict_t *xdata)
-{
-        size_t                       xtra_len        = 0;
-        changelog_priv_t             *priv           = NULL;
-        changelog_opt_t              *co             = NULL;
-        call_stub_t                  *stub           = NULL;
-        struct list_head             queue           = {0, };
-        gf_boolean_t                 barrier_enabled = _gf_false;
-        dht_changelog_rename_info_t  *info           = NULL;
-        int                          ret             = 0;
-        char                         old_name[NAME_MAX] = {0};
-        char                         new_name[NAME_MAX] = {0};
-        char                         *nname             = NULL;
-
-        INIT_LIST_HEAD (&queue);
-        priv = this->private;
-
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_bin (xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
-        if (!ret) {     /* special case: unlink considered as rename */
-                /* 3 == fop + oldloc + newloc */
-                CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                        NULL, loc->inode->gfid, 3);
-
-                co = changelog_get_usable_buffer (frame->local);
-                if (!co)
-                        goto wind;
-
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_RENAME, fop_fn, xtra_len);
-
-                co++;
-                strncpy (old_name, info->buffer, info->oldname_len);
-                CHANGELOG_FILL_ENTRY (co, info->old_pargfid, old_name,
-                                      entry_fn, entry_free_fn, xtra_len, wind);
-
-                co++;
-                /* new name resides just after old name */
-                nname = info->buffer + info->oldname_len;
-                strncpy (new_name, nname, info->newname_len);
-                CHANGELOG_FILL_ENTRY (co, info->new_pargfid, new_name,
-                                      entry_fn, entry_free_fn, xtra_len, wind);
-
-                changelog_set_usable_record_and_length (frame->local,
-                                                        xtra_len, 3);
-        } else {        /* default unlink */
-                CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-                CHANGELOG_INIT_NOCHECK (this, frame->local, NULL,
-                                                        loc->inode->gfid, 2);
-
-                co = changelog_get_usable_buffer (frame->local);
-                if (!co)
-                        goto wind;
-
-                CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op,
-                                                fop_fn, xtra_len);
-
-                co++;
-                if (priv->capture_del_path) {
-                        CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid,
-                                     loc->name, del_entry_fn, del_entry_free_fn,
-                                     xtra_len, wind, _gf_true);
-                } else {
-                        CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid,
-                                     loc->name, del_entry_fn, del_entry_free_fn,
-                                     xtra_len, wind, _gf_false);
-                }
+changelog_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+                 dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+    dht_changelog_rename_info_t *info = NULL;
+    int ret = 0;
+    char *old_name = NULL;
+    char *new_name = NULL;
+    char *nname = NULL;
+
+    INIT_LIST_HEAD(&queue);
+    priv = this->private;
+
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_bin(xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
+    if (!ret) { /* special case: unlink considered as rename */
+        /* 3 == fop + oldloc + newloc */
+        old_name = alloca(info->oldname_len);
+        new_name = alloca(info->newname_len);
+        CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 3);
 
-                changelog_set_usable_record_and_length (frame->local,
-                                                        xtra_len, 2);
-        }
+        co = changelog_get_usable_buffer(frame->local);
+        if (!co)
+            goto wind;
 
-/* changelog barrier */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_unlink_stub (frame, changelog_unlink_resume,
-                                                loc, xflags, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_RENAME, fop_fn, xtra_len);
 
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueue unlink");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: unlink");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
+        co++;
+        strncpy(old_name, info->buffer, info->oldname_len);
+        CHANGELOG_FILL_ENTRY(co, info->old_pargfid, old_name, entry_fn,
+                             entry_free_fn, xtra_len, wind);
 
-/* changelog barrier */
+        co++;
+        /* new name resides just after old name */
+        nname = info->buffer + info->oldname_len;
+        strncpy(new_name, nname, info->newname_len);
+        CHANGELOG_FILL_ENTRY(co, info->new_pargfid, new_name, entry_fn,
+                             entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length(frame->local, xtra_len, 3);
+    } else { /* default unlink */
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+        CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer(frame->local);
+        if (!co)
+            goto wind;
 
- wind:
-        STACK_WIND (frame, changelog_unlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                    loc, xflags, xdata);
- out:
-        return 0;
+        CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        if (priv->capture_del_path) {
+            CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name,
+                                          del_entry_fn, del_entry_free_fn,
+                                          xtra_len, wind, _gf_true);
+        } else {
+            CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name,
+                                          del_entry_fn, del_entry_free_fn,
+                                          xtra_len, wind, _gf_false);
+        }
+
+        changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+    }
+
+    /* changelog barrier */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_unlink_stub(frame, changelog_unlink_resume, loc, xflags,
+                                   xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue unlink");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=unlink", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+out:
+    return 0;
 }
 
 /* rename */
 
 int32_t
-changelog_rename_cbk (call_frame_t *frame,
-                      void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *buf, struct iatt *preoldparent,
-                      struct iatt *postoldparent, struct iatt *prenewparent,
-                      struct iatt *postnewparent, dict_t *xdata)
-{
-        changelog_priv_t                  *priv  = NULL;
-        changelog_local_t                 *local = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (rename, frame, op_ret, op_errno,
-                                buf, preoldparent, postoldparent,
-                                prenewparent, postnewparent, xdata);
-        return 0;
+changelog_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                           postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rename_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+changelog_rename_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                        loc_t *newloc, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeue rename");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_rename_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue rename");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rename (call_frame_t *frame, xlator_t *this,
-                  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        size_t               xtra_len        = 0;
-        changelog_priv_t     *priv           = NULL;
-        changelog_opt_t      *co             = NULL;
-        call_stub_t          *stub           = NULL;
-        struct list_head     queue           = {0, };
-        gf_boolean_t         barrier_enabled = _gf_false;
-        dht_changelog_rename_info_t  *info   = NULL;
-        int                  ret             = 0;
-
-        INIT_LIST_HEAD (&queue);
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_bin (xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
-        if (ret && oldloc->inode->ia_type != IA_IFDIR)  {
-                /* xdata "NOT" set for a non-directory,
-                 * Special rename => avoid logging */
-               goto wind;
-        }
-
-        /* 3 == fop + oldloc + newloc */
-        CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                NULL, oldloc->inode->gfid, 3);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 3);
-/* changelog barrier */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_rename_stub (frame, changelog_rename_resume,
-                                                oldloc, newloc, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueue rename");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: rename");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-/* changelog barrier */
-
- wind:
-        STACK_WIND (frame, changelog_rename_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
- out:
-        return 0;
+changelog_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+    dht_changelog_rename_info_t *info = NULL;
+    int ret = 0;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_bin(xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
+    if (ret && oldloc->inode->ia_type != IA_IFDIR) {
+        /* xdata "NOT" set for a non-directory,
+         * Special rename => avoid logging */
+        goto wind;
+    }
+
+    /* 3 == fop + oldloc + newloc */
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, oldloc->inode->gfid, 3);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, oldloc->pargfid, oldloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, newloc->pargfid, newloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 3);
+    /* changelog barrier */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_rename_stub(frame, changelog_rename_resume, oldloc,
+                                   newloc, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue rename");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=rename", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+out:
+    return 0;
 }
 
 /* link */
 
 int32_t
-changelog_link_cbk (call_frame_t *frame,
-                    void *cookie, xlator_t *this, int32_t op_ret,
-                    int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+changelog_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (link, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_link_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+changelog_link_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                      loc_t *newloc, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing link");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_link_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing link");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 int32_t
-changelog_link (call_frame_t *frame,
-                xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, oldloc->gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_link_stub (frame, changelog_link_resume,
-                                               oldloc, newloc, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued link");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: link");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
- wind:
-        STACK_WIND (frame, changelog_link_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
+changelog_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, oldloc->gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, newloc->pargfid, newloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_link_stub(frame, changelog_link_resume, oldloc, newloc,
+                                 xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued link");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_BARRIER_FOP_FAILED,
+                "fop=link", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+wind:
+    STACK_WIND(frame, changelog_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* mkdir */
 
 int32_t
-changelog_mkdir_cbk (call_frame_t *frame,
-                     void *cookie, xlator_t *this, int32_t op_ret,
-                     int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_mkdir_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, mode_t mode,
-                        mode_t umask, dict_t *xdata)
+changelog_mkdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, mode_t umask, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing mkdir");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_mkdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing mkdir");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_mkdir (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, S_IFDIR | mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_mkdir_stub (frame, changelog_mkdir_resume,
-                                               loc, mode, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued mkdir");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: mkdir");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
- wind:
-        STACK_WIND (frame, changelog_mkdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
-                    loc, mode, umask, xdata);
+changelog_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, S_IFDIR | mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_mkdir_stub(frame, changelog_mkdir_resume, loc, mode,
+                                  umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued mkdir");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=mkdir", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* symlink */
 
 int32_t
-changelog_symlink_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+changelog_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (symlink, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-changelog_symlink_resume (call_frame_t *frame, xlator_t *this,
-                          const char *linkname, loc_t *loc,
-                          mode_t umask, dict_t *xdata)
+changelog_symlink_resume(call_frame_t *frame, xlator_t *this,
+                         const char *linkname, loc_t *loc, mode_t umask,
+                         dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing symlink");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_symlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
-                    linkname, loc, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing symlink");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_symlink (call_frame_t *frame, xlator_t *this,
-                   const char *linkname, loc_t *loc,
-                   mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        size_t            xtra_len         = 0;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_symlink_stub (frame,
-                                                 changelog_symlink_resume,
-                                                 linkname, loc, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued symlink");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: symlink");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
- wind:
-        STACK_WIND (frame, changelog_symlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
-                    linkname, loc, umask, xdata);
+changelog_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    size_t xtra_len = 0;
+    uuid_t gfid = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_symlink_stub(frame, changelog_symlink_resume, linkname,
+                                    loc, umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued symlink");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=symlink", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* mknod */
 
 int32_t
-changelog_mknod_cbk (call_frame_t *frame,
-                     void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (mknod, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_mknod_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, mode_t mode, dev_t rdev,
-                        mode_t umask, dict_t *xdata)
+changelog_mknod_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing mknod");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_mknod_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing mknod");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_mknod (call_frame_t *frame,
-                 xlator_t *this, loc_t *loc,
-                 mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-
-        /* Check whether changelog active */
-        if (!(priv->active))
-                goto wind;
-
-        /* Check whether rebalance activity */
-        if (frame->root->pid == GF_CLIENT_PID_DEFRAG)
-                goto wind;
-
-        /* If tier-dht linkto is SET, ignore about verifiying :
-         * 1. Whether internal fop AND
-         * 2. Whether tier rebalance process activity (this will help in
-         * recording mknod if tier rebalance process calls this mknod) */
-        if (!(dict_get (xdata, "trusted.tier.tier-dht.linkto"))) {
-                CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-                if (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
-                        goto wind;
-        }
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_mknod_stub (frame, changelog_mknod_resume,
-                                               loc, mode, dev, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued mknod");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: mknod");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
- wind:
-        STACK_WIND (frame, changelog_mknod_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
+changelog_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                dev_t dev, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+
+    /* Check whether changelog active */
+    if (!(priv->active))
+        goto wind;
+
+    /* Check whether rebalance activity */
+    if (frame->root->pid == GF_CLIENT_PID_DEFRAG)
+        goto wind;
+
+    /* If tier-dht linkto is SET, ignore about verifiying :
+     * 1. Whether internal fop AND
+     * 2. Whether tier rebalance process activity (this will help in
+     * recording mknod if tier rebalance process calls this mknod) */
+    if (!(dict_get(xdata, "trusted.tier.tier-dht.linkto"))) {
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+        if (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
+            goto wind;
+    }
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_mknod_stub(frame, changelog_mknod_resume, loc, mode, dev,
+                                  umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued mknod");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=mknod", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
-/* creat */
+/* create */
 
 int32_t
-changelog_create_cbk (call_frame_t *frame,
-                      void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+changelog_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        int32_t ret = 0;
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
-        changelog_event_t  ev    = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
-
-        /* fill the event structure.. similar to open() */
-        ev.ev_type = CHANGELOG_OP_TYPE_CREATE;
-        gf_uuid_copy (ev.u.create.gfid, buf->ia_gfid);
-        ev.u.create.flags = fd->flags;
-        changelog_dispatch_event (this, priv, &ev);
-
-        if (changelog_ev_selected
-                   (this, &priv->ev_selection, CHANGELOG_OP_TYPE_RELEASE)) {
-                ret = fd_ctx_set (fd, this, (uint64_t)(long) 0x1);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_SET_FD_CONTEXT,
-                                "could not set fd context (for release cbk)");
-        }
+    int32_t ret = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+    changelog_event_t ev = {
+        0,
+    };
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    priv = this->private;
+    local = frame->local;
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (create, frame,
-                                op_ret, op_errno, fd, inode,
-                                buf, preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-changelog_create_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *loc, int32_t flags, mode_t mode,
-                         mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        changelog_priv_t *priv     = NULL;
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    /* fill the event structure.. similar to open() */
+    ev.ev_type = CHANGELOG_OP_TYPE_CREATE;
+    gf_uuid_copy(ev.u.create.gfid, buf->ia_gfid);
+    ev.u.create.flags = fd->flags;
+    changelog_dispatch_event(this, priv, &ev);
 
-        priv = this->private;
+    if (changelog_ev_selected(this, &priv->ev_selection,
+                              CHANGELOG_OP_TYPE_RELEASE)) {
+        ret = fd_ctx_set(fd, this, (uint64_t)(long)0x1);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, CHANGELOG_MSG_SET_FD_CONTEXT,
+                    NULL);
+    }
 
-        gf_msg_debug (this->name, 0, "Dequeuing create");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
-out:
-        return -1;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_create (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, int32_t flags, mode_t mode,
-                  mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        changelog_opt_t  *co               = NULL;
-        changelog_priv_t *priv             = NULL;
-        size_t            xtra_len         = 0;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        /* init with two extra records */
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-        if (!frame->local)
-                goto wind;
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
+changelog_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                        dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_create_stub (frame, changelog_create_resume,
-                                               loc, flags, mode, umask, fd,
-                                               xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
+    priv = this->private;
 
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued create");
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "Dequeuing create");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 
-        if (barrier_enabled && !stub) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: create");
-                chlog_barrier_dequeue_all (this, &queue);
-        }
+out:
+    return -1;
+}
 
- wind:
-        STACK_WIND (frame, changelog_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
+int32_t
+changelog_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                 mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    changelog_opt_t *co = NULL;
+    changelog_priv_t *priv = NULL;
+    size_t xtra_len = 0;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    /* init with two extra records */
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+    if (!frame->local)
+        goto wind;
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_create_stub(frame, changelog_create_resume, loc, flags,
+                                   mode, umask, fd, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued create");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=create", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* }}} */
 
-
 /* Metadata modification fops - TYPE II */
 
 /* {{{ */
@@ -1155,268 +1104,253 @@ out:
 /* {f}setattr */
 
 int32_t
-changelog_fsetattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *preop_stbuf,
-                        struct iatt *postop_stbuf, dict_t *xdata)
+changelog_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
-
-        priv  = this->private;
-        local = frame->local;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    priv = this->private;
+    local = frame->local;
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
-
-        return 0;
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, preop_stbuf,
+                           postop_stbuf, xdata);
 
+    return 0;
 }
 
 int32_t
-changelog_fsetattr (call_frame_t *frame,
-                    xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+changelog_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
-        if (!frame->local)
-                goto wind;
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
+    if (!frame->local)
+        goto wind;
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fsetattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
 int32_t
-changelog_setattr_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, struct iatt *preop_stbuf,
-                       struct iatt *postop_stbuf, dict_t *xdata)
+changelog_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                      dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(setattr, frame, op_ret, op_errno, preop_stbuf,
+                           postop_stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_setattr (call_frame_t *frame,
-                   xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+changelog_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-        uuid_t            shard_root_gfid = {0,};
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    uuid_t shard_root_gfid = {
+        0,
+    };
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
 
-        /* Do not record META on .shard */
-        gf_uuid_parse (SHARD_ROOT_GFID, shard_root_gfid);
-        if (gf_uuid_compare (loc->gfid, shard_root_gfid) == 0) {
-                goto wind;
-        }
+    /* Do not record META on .shard */
+    gf_uuid_parse(SHARD_ROOT_GFID, shard_root_gfid);
+    if (gf_uuid_compare(loc->gfid, shard_root_gfid) == 0) {
+        goto wind;
+    }
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
-        if (!frame->local)
-                goto wind;
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
+    if (!frame->local)
+        goto wind;
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_setattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
 /* {f}removexattr */
 
 int32_t
-changelog_fremovexattr_cbk (call_frame_t *frame,
-                            void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fremovexattr (call_frame_t *frame, xlator_t *this,
-                        fd_t *fd, const char *name, dict_t *xdata)
+changelog_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fremovexattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-changelog_removexattr_cbk (call_frame_t *frame,
-                           void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_removexattr (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, const char *name, dict_t *xdata)
+changelog_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_removexattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 /* {f}setxattr */
 
 int32_t
-changelog_setxattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 /* changelog_handle_virtual_xattr:
@@ -1429,274 +1363,255 @@ changelog_setxattr_cbk (call_frame_t *frame,
  *           any other value: ENOTSUP is returned.
  */
 static void
-changelog_handle_virtual_xattr (call_frame_t *frame, xlator_t *this,
-                                loc_t *loc, dict_t *dict)
+changelog_handle_virtual_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               dict_t *dict)
 {
-        changelog_priv_t     *priv         = NULL;
-        changelog_local_t    *local        = NULL;
-        int32_t               value        = 0;
-        int                   ret          = 0;
-        int                   dict_ret     = 0;
-        gf_boolean_t          valid        = _gf_false;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+    int32_t value = 0;
+    int ret = 0;
+    int dict_ret = 0;
+    gf_boolean_t valid = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        dict_ret = dict_get_int32 (dict, GF_XATTR_TRIGGER_SYNC, &value);
+    dict_ret = dict_get_int32(dict, GF_XATTR_TRIGGER_SYNC, &value);
 
-        if ((dict_ret == 0 && value == 1) && ((loc->inode->ia_type == IA_IFDIR)
-            || (loc->inode->ia_type == IA_IFREG)))
-                valid = _gf_true;
+    if ((dict_ret == 0 && value == 1) && ((loc->inode->ia_type == IA_IFDIR) ||
+                                          (loc->inode->ia_type == IA_IFREG)))
+        valid = _gf_true;
 
-        if (valid) {
-                ret = changelog_fill_entry_buf (frame, this, loc, &local);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_ENTRY_BUF_INFO,
-                                "Entry cannot be"
-                                " captured for gfid: %s. Capturing DATA"
-                                " entry.", uuid_utoa (loc->inode->gfid));
-                        goto unwind;
-                }
-                changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
-
- unwind:
-                /* Capture DATA only if it's a file. */
-                if (loc->inode->ia_type != IA_IFDIR)
-                        changelog_update (this, priv, frame->local,
-                                          CHANGELOG_TYPE_DATA);
-                /* Assign local to prev_entry, so unwind will take
-                 * care of cleanup. */
-                ((changelog_local_t *)(frame->local))->prev_entry = local;
-                CHANGELOG_STACK_UNWIND (setxattr, frame, 0, 0, NULL);
-                return;
-        } else {
-                CHANGELOG_STACK_UNWIND (setxattr, frame, -1, ENOTSUP, NULL);
-                return;
-        }
+    if (valid) {
+        ret = changelog_fill_entry_buf(frame, this, loc, &local);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_ENTRY_BUF_INFO,
+                    "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+            goto unwind;
+        }
+        changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+    unwind:
+        /* Capture DATA only if it's a file. */
+        if (loc->inode->ia_type != IA_IFDIR)
+            changelog_update(this, priv, frame->local, CHANGELOG_TYPE_DATA);
+        /* Assign local to prev_entry, so unwind will take
+         * care of cleanup. */
+        ((changelog_local_t *)(frame->local))->prev_entry = local;
+        CHANGELOG_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return;
+    } else {
+        CHANGELOG_STACK_UNWIND(setxattr, frame, -1, ENOTSUP, NULL);
+        return;
+    }
 }
 
 int32_t
-changelog_setxattr (call_frame_t *frame,
-                    xlator_t *this, loc_t *loc,
-                    dict_t *dict, int32_t flags, dict_t *xdata)
+changelog_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        /* On setting this virtual xattr on a file, an explicit data
-         * sync is triggered from geo-rep as CREATE|DATA entry is
-         * recorded in changelog based on xattr value.
-         */
-        if (dict_get (dict, GF_XATTR_TRIGGER_SYNC)) {
-                changelog_handle_virtual_xattr (frame, this, loc, dict);
-                return 0;
-        }
+    /* On setting this virtual xattr on a file, an explicit data
+     * sync is triggered from geo-rep as CREATE|DATA entry is
+     * recorded in changelog based on xattr value.
+     */
+    if (dict_get(dict, GF_XATTR_TRIGGER_SYNC)) {
+        changelog_handle_virtual_xattr(frame, this, loc, dict);
+        return 0;
+    }
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_setxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_fsetxattr_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, dict_t *xdata)
+changelog_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fsetxattr (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, dict_t *dict,
-                     int32_t flags, dict_t *xdata)
+changelog_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fsetxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_xattrop_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, dict_t *xattr, dict_t *xdata)
+changelog_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(xattrop, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+changelog_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        changelog_priv_t *priv      = NULL;
-        changelog_opt_t  *co        = NULL;
-        size_t            xtra_len  = 0;
-        int               ret       = 0;
-        void             *size_attr = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    int ret = 0;
+    void *size_attr = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        ret = dict_get_ptr (xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
-        if (ret)
-                goto wind;
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    ret = dict_get_ptr(xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret)
+        goto wind;
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_xattrop_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->xattrop,
-                    loc, optype, xattr, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    return 0;
 }
 
 int32_t
-changelog_fxattrop_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, dict_t *xattr, dict_t *xdata)
+changelog_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                       dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fxattrop, frame,
-                                op_ret, op_errno, xattr, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fxattrop (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, gf_xattrop_flags_t optype,
-                     dict_t *xattr, dict_t *xdata)
-{
-        changelog_priv_t *priv      = NULL;
-        changelog_opt_t  *co        = NULL;
-        size_t            xtra_len  = 0;
-        void             *size_attr = NULL;
-        int               ret       = 0;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        ret = dict_get_ptr (xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
-        if (ret)
-                goto wind;
+changelog_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    void *size_attr = NULL;
+    int ret = 0;
 
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    ret = dict_get_ptr(xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret)
+        goto wind;
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fxattrop_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fxattrop,
-                    fd, optype, xattr, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
+    return 0;
 }
 /* }}} */
 
-
 /* Data modification fops - TYPE I */
 
 /* {{{ */
@@ -1704,164 +1619,151 @@ changelog_fxattrop (call_frame_t *frame,
 /* {f}truncate() */
 
 int32_t
-changelog_truncate_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+changelog_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (truncate, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_truncate (call_frame_t *frame,
-                    xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+changelog_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
-
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_truncate_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
 int32_t
-changelog_ftruncate_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata)
+changelog_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (ftruncate, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_ftruncate (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+changelog_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_ftruncate_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 /* writev() */
 
 int32_t
-changelog_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
+changelog_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret <= 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret <= 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (writev, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_writev (call_frame_t *frame,
-                  xlator_t *this, fd_t *fd, struct iovec *vector,
-                  int32_t count, off_t offset, uint32_t flags,
-                  struct iobref *iobref, dict_t *xdata)
+changelog_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iovec *vector, int32_t count, off_t offset,
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_writev_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev, fd, vector,
-                    count, offset, flags, iobref, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 /* }}} */
@@ -1870,84 +1772,79 @@ changelog_writev (call_frame_t *frame,
 
 /* {{{ */
 
-
-
 int
-changelog_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-        int                ret    = 0;
-        void              *opaque = NULL;
-        char              *buf    = NULL;
-        ssize_t            buflen = 0;
-        changelog_priv_t  *priv   = NULL;
-        changelog_event_t  ev     = {0,};
-        gf_boolean_t logopen = _gf_false;
-
-        priv = this->private;
-        if (frame->local) {
-                frame->local = NULL;
-                logopen = _gf_true;
-        }
-
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !logopen), unwind);
-
-        /* fill the event structure */
-        ev.ev_type = CHANGELOG_OP_TYPE_OPEN;
-        gf_uuid_copy (ev.u.open.gfid, fd->inode->gfid);
-        ev.u.open.flags = fd->flags;
-        changelog_dispatch_event (this, priv, &ev);
-
-        if (changelog_ev_selected
-                   (this, &priv->ev_selection, CHANGELOG_OP_TYPE_RELEASE)) {
-                ret = fd_ctx_set (fd, this, (uint64_t)(long) 0x1);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_SET_FD_CONTEXT,
-                                "could not set fd context (for release cbk)");
-        }
+changelog_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+    int ret = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+    gf_boolean_t logopen = _gf_false;
+
+    priv = this->private;
+    if (frame->local) {
+        frame->local = NULL;
+        logopen = _gf_true;
+    }
+
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !logopen), unwind);
+
+    /* fill the event structure */
+    ev.ev_type = CHANGELOG_OP_TYPE_OPEN;
+    gf_uuid_copy(ev.u.open.gfid, fd->inode->gfid);
+    ev.u.open.flags = fd->flags;
+    changelog_dispatch_event(this, priv, &ev);
+
+    if (changelog_ev_selected(this, &priv->ev_selection,
+                              CHANGELOG_OP_TYPE_RELEASE)) {
+        ret = fd_ctx_set(fd, this, (uint64_t)(long)0x1);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, CHANGELOG_MSG_SET_FD_CONTEXT,
+                    NULL);
+    }
 
- unwind:
-        CHANGELOG_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+unwind:
+    CHANGELOG_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int
-changelog_open (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+changelog_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               fd_t *fd, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        frame->local = (void *)0x1; /* do not dereference in ->cbk */
+    frame->local = (void *)0x1; /* do not dereference in ->cbk */
 
- wind:
-        STACK_WIND (frame, changelog_open_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, changelog_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ */
 
-
 /* }}} */
 
 int32_t
-_changelog_generic_dispatcher (dict_t *dict,
-                               char *key, data_t *value, void *data)
+_changelog_generic_dispatcher(dict_t *dict, char *key, data_t *value,
+                              void *data)
 {
-        xlator_t *this = NULL;
-        changelog_priv_t *priv = NULL;
+    xlator_t *this = NULL;
+    changelog_priv_t *priv = NULL;
 
-        this = data;
-        priv = this->private;
+    this = data;
+    priv = this->private;
 
-        changelog_dispatch_event (this, priv, (changelog_event_t *)value->data);
-        return 0;
+    changelog_dispatch_event(this, priv, (changelog_event_t *)value->data);
+    return 0;
 }
 
 /**
@@ -1956,46 +1853,45 @@ _changelog_generic_dispatcher (dict_t *dict,
  * traverses the dictionary).
  */
 int32_t
-changelog_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+changelog_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        if (op != GF_IPC_TARGET_CHANGELOG)
-                goto wind;
+    if (op != GF_IPC_TARGET_CHANGELOG)
+        goto wind;
 
-        /* it's for us, do the job */
-        if (xdata)
-                (void) dict_foreach (xdata,
-                                     _changelog_generic_dispatcher, this);
+    /* it's for us, do the job */
+    if (xdata)
+        (void)dict_foreach(xdata, _changelog_generic_dispatcher, this);
 
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(ipc, frame, 0, 0, NULL);
+    return 0;
 
- wind:
-        STACK_WIND (frame, default_ipc_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
-
 /* {{{ */
 
 int32_t
-changelog_release (xlator_t *this, fd_t *fd)
+changelog_release(xlator_t *this, fd_t *fd)
 {
-        changelog_event_t ev = {0,};
-        changelog_priv_t *priv = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        ev.ev_type = CHANGELOG_OP_TYPE_RELEASE;
-        gf_uuid_copy (ev.u.release.gfid, fd->inode->gfid);
-        changelog_dispatch_event (this, priv, &ev);
+    ev.ev_type = CHANGELOG_OP_TYPE_RELEASE;
+    gf_uuid_copy(ev.u.release.gfid, fd->inode->gfid);
+    changelog_dispatch_event(this, priv, &ev);
 
-        (void) fd_ctx_del (fd, this, NULL);
+    (void)fd_ctx_del(fd, this, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /**
@@ -2010,979 +1906,1084 @@ changelog_release (xlator_t *this, fd_t *fd)
  * needed if there are more operation modes in the future.
  */
 static void
-changelog_assign_opmode (changelog_priv_t *priv, char *mode)
+changelog_assign_opmode(changelog_priv_t *priv, char *mode)
 {
-        if ( strncmp (mode, "realtime", 8) == 0 ) {
-                priv->op_mode = CHANGELOG_MODE_RT;
-        }
+    if (strncmp(mode, "realtime", 8) == 0) {
+        priv->op_mode = CHANGELOG_MODE_RT;
+    }
 }
 
 static void
-changelog_assign_encoding (changelog_priv_t *priv, char *enc)
+changelog_assign_encoding(changelog_priv_t *priv, char *enc)
 {
-        if ( strncmp (enc, "binary", 6) == 0 ) {
-                priv->encode_mode = CHANGELOG_ENCODE_BINARY;
-        } else if ( strncmp (enc, "ascii", 5) == 0 ) {
-                priv->encode_mode = CHANGELOG_ENCODE_ASCII;
-        }
+    if (strncmp(enc, "binary", 6) == 0) {
+        priv->encode_mode = CHANGELOG_ENCODE_BINARY;
+    } else if (strncmp(enc, "ascii", 5) == 0) {
+        priv->encode_mode = CHANGELOG_ENCODE_ASCII;
+    }
 }
 
 static void
 changelog_assign_barrier_timeout(changelog_priv_t *priv, uint32_t timeout)
 {
-       LOCK (&priv->lock);
-       {
-               priv->timeout.tv_sec = timeout;
-       }
-       UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        priv->timeout.tv_sec = timeout;
+    }
+    UNLOCK(&priv->lock);
 }
 
 /* cleanup any helper threads that are running */
 static void
-changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
+changelog_cleanup_helper_threads(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-
-        if (priv->cr.rollover_th) {
-                (void) changelog_thread_cleanup (this, priv->cr.rollover_th);
-                priv->cr.rollover_th = 0;
-        }
+    if (priv->cr.rollover_th) {
+        (void)changelog_thread_cleanup(this, priv->cr.rollover_th);
+        priv->cr.rollover_th = 0;
+    }
 
-        if (priv->cf.fsync_th) {
-                (void) changelog_thread_cleanup (this, priv->cf.fsync_th);
-                priv->cf.fsync_th = 0;
-        }
+    if (priv->cf.fsync_th) {
+        (void)changelog_thread_cleanup(this, priv->cf.fsync_th);
+        priv->cf.fsync_th = 0;
+    }
 }
 
 /* spawn helper thread; cleaning up in case of errors */
 static int
-changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
-{
-        int ret = 0;
-        int flags = 0;
-
-        /* Geo-Rep snapshot dependency:
-         *
-         * To implement explicit rollover of changlog journal on barrier
-         * notification, a pipe is created to communicate between
-         * 'changelog_rollover' thread and changelog main thread. The select
-         * call used to wait till roll-over time in changelog_rollover thread
-         * is modified to wait on read end of the pipe. When barrier
-         * notification comes (i.e, in 'reconfigure'), select in
-         * changelog_rollover thread is woken up explicitly by writing into
-         * the write end of the pipe in 'reconfigure'.
-         */
-
-        priv->cr.notify = _gf_false;
-        priv->cr.this = this;
-        ret = gf_thread_create (&priv->cr.rollover_th,
-				NULL, changelog_rollover, priv);
-        if (ret)
-                goto out;
+changelog_spawn_helper_threads(xlator_t *this, changelog_priv_t *priv)
+{
+    int ret = 0;
+
+    /* Geo-Rep snapshot dependency:
+     *
+     * To implement explicit rollover of changlog journal on barrier
+     * notification, a pipe is created to communicate between
+     * 'changelog_rollover' thread and changelog main thread. The select
+     * call used to wait till roll-over time in changelog_rollover thread
+     * is modified to wait on read end of the pipe. When barrier
+     * notification comes (i.e, in 'reconfigure'), select in
+     * changelog_rollover thread is woken up explicitly by writing into
+     * the write end of the pipe in 'reconfigure'.
+     */
+
+    priv->cr.notify = _gf_false;
+    priv->cr.this = this;
+    ret = gf_thread_create(&priv->cr.rollover_th, NULL, changelog_rollover,
+                           priv, "clogro");
+    if (ret)
+        goto out;
+
+    if (priv->fsync_interval) {
+        priv->cf.this = this;
+        ret = gf_thread_create(&priv->cf.fsync_th, NULL, changelog_fsync_thread,
+                               priv, "clogfsyn");
+    }
+
+    if (ret)
+        changelog_cleanup_helper_threads(this, priv);
 
-        if (priv->fsync_interval) {
-                priv->cf.this = this;
-                ret = gf_thread_create (&priv->cf.fsync_th,
-					NULL, changelog_fsync_thread, priv);
-        }
-
-        if (ret)
-                changelog_cleanup_helper_threads (this, priv);
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        changelog_priv_t       *priv            = NULL;
-        dict_t                 *dict            = NULL;
-        char                    buf[1]          = {1};
-        int                     barrier         = DICT_DEFAULT;
-        gf_boolean_t            bclean_req      = _gf_false;
-        int                     ret             = 0;
-        int                     ret1            = 0;
-        struct list_head        queue           = {0, };
+notify(xlator_t *this, int event, void *data, ...)
+{
+    changelog_priv_t *priv = NULL;
+    dict_t *dict = NULL;
+    char buf[1] = {1};
+    int barrier = DICT_DEFAULT;
+    gf_boolean_t bclean_req = _gf_false;
+    int ret = 0;
+    int ret1 = 0;
+    struct list_head queue = {
+        0,
+    };
+    uint64_t xprtcnt = 0;
+    uint64_t clntcnt = 0;
+    changelog_clnt_t *conn = NULL;
+    gf_boolean_t cleanup_notify = _gf_false;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    if (event == GF_EVENT_PARENT_DOWN) {
+        priv->victim = data;
+        gf_log(this->name, GF_LOG_INFO,
+               "cleanup changelog rpc connection of brick %s",
+               priv->victim->name);
+
+        if (priv->rpc_active) {
+            this->cleanup_starting = 1;
+            changelog_destroy_rpc_listner(this, priv);
+            conn = &priv->connections;
+            if (conn)
+                changelog_ev_cleanup_connections(this, conn);
+            xprtcnt = GF_ATOMIC_GET(priv->xprtcnt);
+            clntcnt = GF_ATOMIC_GET(priv->clntcnt);
+            if (!xprtcnt && !clntcnt) {
+                LOCK(&priv->lock);
+                {
+                    cleanup_notify = priv->notify_down;
+                    priv->notify_down = _gf_true;
+                }
+                UNLOCK(&priv->lock);
+                if (priv->rpc) {
+                    list_for_each_entry_safe(listener, next,
+                                             &priv->rpc->listeners, list)
+                    {
+                        if (listener->trans) {
+                            rpc_transport_unref(listener->trans);
+                        }
+                    }
+                    rpcsvc_destroy(priv->rpc);
+                    priv->rpc = NULL;
+                }
+                CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile,
+                                           UNIX_PATH_MAX);
+                sys_unlink(sockfile);
+                if (!cleanup_notify)
+                    default_notify(this, GF_EVENT_PARENT_DOWN, data);
+            }
+        } else {
+            default_notify(this, GF_EVENT_PARENT_DOWN, data);
+        }
+        goto out;
+    }
 
-        INIT_LIST_HEAD (&queue);
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        dict = data;
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+        barrier = dict_get_str_boolean(dict, "barrier", DICT_DEFAULT);
 
-        if (event == GF_EVENT_TRANSLATOR_OP) {
+        switch (barrier) {
+            case DICT_ERROR:
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_DICT_GET_FAILED, "dict_get_str_boolean",
+                        NULL);
+                ret = -1;
+                goto out;
 
-                dict = data;
+            case BARRIER_OFF:
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_STATE_NOTIFY, "off", NULL);
 
-                barrier = dict_get_str_boolean (dict, "barrier", DICT_DEFAULT);
+                CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
+                LOCK(&priv->c_snap_lock);
+                {
+                    changelog_snap_logging_stop(this, priv);
+                }
+                UNLOCK(&priv->c_snap_lock);
 
-                switch (barrier) {
-                case DICT_ERROR:
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_DICT_GET_FAILED,
-                                "Barrier dict_get_str_boolean failed");
+                LOCK(&priv->bflags.lock);
+                {
+                    if (priv->bflags.barrier_ext == _gf_false)
                         ret = -1;
-                        goto out;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                case BARRIER_OFF:
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BARRIER_INFO,
-                                "Barrier off notification");
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ERROR, NULL);
+                    goto out;
+                }
 
-                        CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
-                        LOCK(&priv->c_snap_lock);
-                        {
-                                changelog_snap_logging_stop (this, priv);
-                        }
-                        UNLOCK(&priv->c_snap_lock);
+                /* Stop changelog barrier and dequeue all fops */
+                LOCK(&priv->lock);
+                {
+                    if (priv->barrier_enabled == _gf_true)
+                        __chlog_barrier_disable(this, &queue);
+                    else
+                        ret = -1;
+                }
+                UNLOCK(&priv->lock);
+                /* If ret = -1, then changelog barrier is already
+                 * disabled because of error or timeout.
+                 */
+                if (ret == 0) {
+                    chlog_barrier_dequeue_all(this, &queue);
+                    gf_smsg(this->name, GF_LOG_INFO, 0,
+                            CHANGELOG_MSG_BARRIER_DISABLED, NULL);
+                } else {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ALREADY_DISABLED, NULL);
+                }
 
-                        LOCK (&priv->bflags.lock);
-                        {
-                                if (priv->bflags.barrier_ext == _gf_false)
-                                        ret = -1;
-                        }
-                        UNLOCK (&priv->bflags.lock);
-
-                        if (ret == -1 ) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_BARRIER_ERROR,
-                                        "Received another barrier off"
-                                        " notification while already off");
-                                goto out;
-                        }
+                LOCK(&priv->bflags.lock);
+                {
+                    priv->bflags.barrier_ext = _gf_false;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                        /* Stop changelog barrier and dequeue all fops */
-                        LOCK (&priv->lock);
-                        {
-                                if (priv->barrier_enabled == _gf_true)
-                                        __chlog_barrier_disable (this, &queue);
-                                else
-                                        ret = -1;
-                        }
-                        UNLOCK (&priv->lock);
-                        /* If ret = -1, then changelog barrier is already
-                         * disabled because of error or timeout.
-                         */
-                        if (ret == 0) {
-                                chlog_barrier_dequeue_all(this, &queue);
-                                gf_msg(this->name, GF_LOG_INFO, 0,
-                                       CHANGELOG_MSG_BARRIER_INFO,
-                                       "Disabled changelog barrier");
-                        } else {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_BARRIER_ERROR,
-                                        "Changelog barrier already disabled");
-                        }
+                goto out;
 
-                        LOCK (&priv->bflags.lock);
-                        {
-                                priv->bflags.barrier_ext = _gf_false;
-                        }
-                        UNLOCK (&priv->bflags.lock);
+            case BARRIER_ON:
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_STATE_NOTIFY, "on", NULL);
 
-                        goto out;
+                CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
+                LOCK(&priv->c_snap_lock);
+                {
+                    changelog_snap_logging_start(this, priv);
+                }
+                UNLOCK(&priv->c_snap_lock);
 
-                case BARRIER_ON:
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BARRIER_INFO,
-                                "Barrier on notification");
+                LOCK(&priv->bflags.lock);
+                {
+                    if (priv->bflags.barrier_ext == _gf_true)
+                        ret = -1;
+                    else
+                        priv->bflags.barrier_ext = _gf_true;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                        CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
-                        LOCK(&priv->c_snap_lock);
-                        {
-                                changelog_snap_logging_start (this, priv);
-                        }
-                        UNLOCK(&priv->c_snap_lock);
-
-                        LOCK (&priv->bflags.lock);
-                        {
-                                if (priv->bflags.barrier_ext == _gf_true)
-                                        ret = -1;
-                                else
-                                        priv->bflags.barrier_ext = _gf_true;
-                        }
-                        UNLOCK (&priv->bflags.lock);
-
-                        if (ret == -1 ) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_BARRIER_ERROR,
-                                        "Received another barrier on"
-                                        "notification when last one is"
-                                        "not served yet");
-                                goto out;
-                        }
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ON_ERROR, NULL);
+                    goto out;
+                }
 
-                        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-                        {
-                                priv->bn.bnotify = _gf_true;
-                        }
-                        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-
-                        /* Start changelog barrier */
-                        LOCK (&priv->lock);
-                        {
-                                ret = __chlog_barrier_enable (this, priv);
-                        }
-                        UNLOCK (&priv->lock);
-                        if (ret == -1) {
-                                changelog_barrier_cleanup (this, priv, &queue);
-                                goto out;
-                        }
+                ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                {
+                    priv->bn.bnotify = _gf_true;
+                }
+                ret = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
 
-                        gf_msg(this->name, GF_LOG_INFO, 0,
-                               CHANGELOG_MSG_BARRIER_INFO,
-                               "Enabled changelog barrier");
-
-                        ret = changelog_barrier_notify(priv, buf);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_WRITE_FAILED,
-                                        "Explicit roll over: write failed");
-                                changelog_barrier_cleanup (this, priv, &queue);
-                                ret = -1;
-                                goto out;
-                        }
+                /* Start changelog barrier */
+                LOCK(&priv->lock);
+                {
+                    ret = __chlog_barrier_enable(this, priv);
+                }
+                UNLOCK(&priv->lock);
+                if (ret == -1) {
+                    changelog_barrier_cleanup(this, priv, &queue);
+                    goto out;
+                }
 
-                        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-                        {
-                                /* The while condition check is required here to
-                                 * handle spurious wakeup of cond wait that can
-                                 * happen with pthreads. See man page */
-                                while (priv->bn.bnotify == _gf_true) {
-                                        ret = pthread_cond_wait (
-                                                       &priv->bn.bnotify_cond,
-                                                       &priv->bn.bnotify_mutex);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret,
-                                                                          out,
-                                                                    bclean_req);
-                                }
-                                if (priv->bn.bnotify_error == _gf_true) {
-                                        ret = -1;
-                                        priv->bn.bnotify_error = _gf_false;
-                                }
-                        }
-                        ret1 = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret1, out,
-                                                          bclean_req);
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BNOTIFY_INFO,
-                                "Woke up: bnotify conditional wait");
-
-                        goto out;
-
-                case DICT_DEFAULT:
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_DICT_GET_FAILED,
-                                "barrier key not found");
-                        ret = -1;
-                        goto out;
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_ENABLE, NULL);
 
-                default:
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                CHANGELOG_MSG_DICT_GET_FAILED,
-                                "Something went bad in dict_get_str_boolean");
+                ret = changelog_barrier_notify(priv, buf);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_WRITE_FAILED, "Explicit roll over",
+                            NULL);
+                    changelog_barrier_cleanup(this, priv, &queue);
+                    ret = -1;
+                    goto out;
+                }
+
+                ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                {
+                    /* The while condition check is required here to
+                     * handle spurious wakeup of cond wait that can
+                     * happen with pthreads. See man page */
+                    while (priv->bn.bnotify == _gf_true) {
+                        ret = pthread_cond_wait(&priv->bn.bnotify_cond,
+                                                &priv->bn.bnotify_mutex);
+                        CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                    }
+                    if (priv->bn.bnotify_error == _gf_true) {
                         ret = -1;
-                        goto out;
+                        priv->bn.bnotify_error = _gf_false;
+                    }
                 }
-        } else {
-                ret = default_notify (this, event, data);
+                ret1 = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret1, out, bclean_req);
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BNOTIFY_COND_INFO, NULL);
+
+                goto out;
+
+            case DICT_DEFAULT:
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND, NULL);
+                ret = -1;
+                goto out;
+
+            default:
+                gf_smsg(this->name, GF_LOG_ERROR, EINVAL,
+                        CHANGELOG_MSG_ERROR_IN_DICT_GET, NULL);
+                ret = -1;
+                goto out;
         }
+    } else {
+        ret = default_notify(this, event, data);
+    }
 
- out:
-        if (bclean_req)
-                changelog_barrier_cleanup (this, priv, &queue);
+out:
+    if (bclean_req)
+        changelog_barrier_cleanup(this, priv, &queue);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_changelog_mt_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_changelog_mt_end + 1);
 
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                CHANGELOG_MSG_MEMORY_INIT_FAILED, NULL);
         return ret;
+    }
+
+    return ret;
 }
 
 static int
-changelog_init (xlator_t *this, changelog_priv_t *priv)
+changelog_init(xlator_t *this, changelog_priv_t *priv)
 {
-        int                  i   = 0;
-        int                  ret = -1;
-        struct timeval       tv  = {0,};
-        changelog_log_data_t cld = {0,};
-
-        ret = gettimeofday (&tv, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_GET_TIME_OP_FAILED,
-                        "gettimeofday() failure");
-                goto out;
-        }
+    int i = 0;
+    int ret = 0;
+    changelog_log_data_t cld = {
+        0,
+    };
 
-        priv->slice.tv_start = tv;
+    priv->maps[CHANGELOG_TYPE_DATA] = "D ";
+    priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
+    priv->maps[CHANGELOG_TYPE_METADATA_XATTR] = "M ";
+    priv->maps[CHANGELOG_TYPE_ENTRY] = "E ";
 
-        priv->maps[CHANGELOG_TYPE_DATA]     = "D ";
-        priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
-        priv->maps[CHANGELOG_TYPE_ENTRY]    = "E ";
+    for (; i < CHANGELOG_MAX_TYPE; i++) {
+        /* start with version 1 */
+        priv->slice.changelog_version[i] = 1;
+    }
 
-        for (; i < CHANGELOG_MAX_TYPE; i++) {
-                /* start with version 1 */
-                priv->slice.changelog_version[i] = 1;
-        }
+    if (!priv->active)
+        return ret;
 
-        if (!priv->active)
-                return ret;
+    /**
+     * start with a fresh changelog file every time. this is done
+     * in case there was an encoding change. so... things are kept
+     * simple here.
+     */
+    changelog_fill_rollover_data(&cld, _gf_false);
 
-        /**
-         * start with a fresh changelog file every time. this is done
-         * in case there was an encoding change. so... things are kept
-         * simple here.
-         */
-        ret = changelog_fill_rollover_data (&cld, _gf_false);
-        if(ret)
-                goto out;
+    ret = htime_open(this, priv, cld.cld_roll_time);
+    /* call htime open with cld's rollover_time */
+    if (ret)
+        goto out;
 
-        ret = htime_open (this, priv, cld.cld_roll_time);
-        /* call htime open with cld's rollover_time */
-        if (ret)
-                goto out;
+    LOCK(&priv->lock);
+    {
+        ret = changelog_inject_single_event(this, priv, &cld);
+    }
+    UNLOCK(&priv->lock);
 
-        LOCK (&priv->lock);
-        {
-                ret = changelog_inject_single_event (this, priv, &cld);
-        }
-        UNLOCK (&priv->lock);
+    /* ... and finally spawn the helpers threads */
+    ret = changelog_spawn_helper_threads(this, priv);
 
-        /* ... and finally spawn the helpers threads */
-        ret = changelog_spawn_helper_threads (this, priv);
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /**
  * Init barrier related condition variables and locks
  */
 static int
-changelog_barrier_pthread_init (xlator_t *this, changelog_priv_t *priv)
-{
-        gf_boolean_t    bn_mutex_init         = _gf_false;
-        gf_boolean_t    bn_cond_init          = _gf_false;
-        gf_boolean_t    dm_mutex_black_init   = _gf_false;
-        gf_boolean_t    dm_cond_black_init    = _gf_false;
-        gf_boolean_t    dm_mutex_white_init   = _gf_false;
-        gf_boolean_t    dm_cond_white_init    = _gf_false;
-        gf_boolean_t    cr_mutex_init         = _gf_false;
-        gf_boolean_t    cr_cond_init          = _gf_false;
-        int             ret                   = 0;
-
-        if ((ret = pthread_mutex_init(&priv->bn.bnotify_mutex, NULL)) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                        "bnotify pthread_mutex_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        bn_mutex_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->bn.bnotify_cond, NULL)) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                        "bnotify pthread_cond_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        bn_cond_init = _gf_true;
-
-        if ((ret = pthread_mutex_init(&priv->dm.drain_black_mutex, NULL)) != 0)
-        {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                        "drain_black pthread_mutex_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_mutex_black_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->dm.drain_black_cond, NULL)) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                        "drain_black pthread_cond_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_cond_black_init = _gf_true;
-
-        if ((ret = pthread_mutex_init(&priv->dm.drain_white_mutex, NULL)) != 0)
-        {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                        "drain_white pthread_mutex_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_mutex_white_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->dm.drain_white_cond, NULL)) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                        "drain_white pthread_cond_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_cond_white_init = _gf_true;
-
-        if ((pthread_mutex_init(&priv->cr.lock, NULL)) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                        "changelog_rollover lock init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        cr_mutex_init = _gf_true;
-
-        if ((pthread_cond_init(&priv->cr.cond, NULL)) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                        "changelog_rollover cond init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        cr_cond_init = _gf_true;
- out:
-        if (ret) {
-                if (bn_mutex_init)
-                        pthread_mutex_destroy(&priv->bn.bnotify_mutex);
-                if (bn_cond_init)
-                        pthread_cond_destroy (&priv->bn.bnotify_cond);
-                if (dm_mutex_black_init)
-                        pthread_mutex_destroy(&priv->dm.drain_black_mutex);
-                if (dm_cond_black_init)
-                        pthread_cond_destroy (&priv->dm.drain_black_cond);
-                if (dm_mutex_white_init)
-                        pthread_mutex_destroy(&priv->dm.drain_white_mutex);
-                if (dm_cond_white_init)
-                        pthread_cond_destroy (&priv->dm.drain_white_cond);
-                if (cr_mutex_init)
-                        pthread_mutex_destroy(&priv->cr.lock);
-                if (cr_cond_init)
-                        pthread_cond_destroy (&priv->cr.cond);
-        }
-        return ret;
+changelog_barrier_pthread_init(xlator_t *this, changelog_priv_t *priv)
+{
+    gf_boolean_t bn_mutex_init = _gf_false;
+    gf_boolean_t bn_cond_init = _gf_false;
+    gf_boolean_t dm_mutex_black_init = _gf_false;
+    gf_boolean_t dm_cond_black_init = _gf_false;
+    gf_boolean_t dm_mutex_white_init = _gf_false;
+    gf_boolean_t dm_cond_white_init = _gf_false;
+    gf_boolean_t cr_mutex_init = _gf_false;
+    gf_boolean_t cr_cond_init = _gf_false;
+    int ret = 0;
+
+    if ((ret = pthread_mutex_init(&priv->bn.bnotify_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=bnotify",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    bn_mutex_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->bn.bnotify_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=bnotify",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    bn_cond_init = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->dm.drain_black_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=drain_black",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_mutex_black_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->dm.drain_black_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=drain_black",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_cond_black_init = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->dm.drain_white_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=drain_white",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_mutex_white_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->dm.drain_white_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=drain_white",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_cond_white_init = _gf_true;
+
+    if ((pthread_mutex_init(&priv->cr.lock, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
+                "name=changelog_rollover", "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    cr_mutex_init = _gf_true;
+
+    if ((pthread_cond_init(&priv->cr.cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
+                "changelog_rollover cond init failed", "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    cr_cond_init = _gf_true;
+out:
+    if (ret) {
+        if (bn_mutex_init)
+            pthread_mutex_destroy(&priv->bn.bnotify_mutex);
+        if (bn_cond_init)
+            pthread_cond_destroy(&priv->bn.bnotify_cond);
+        if (dm_mutex_black_init)
+            pthread_mutex_destroy(&priv->dm.drain_black_mutex);
+        if (dm_cond_black_init)
+            pthread_cond_destroy(&priv->dm.drain_black_cond);
+        if (dm_mutex_white_init)
+            pthread_mutex_destroy(&priv->dm.drain_white_mutex);
+        if (dm_cond_white_init)
+            pthread_cond_destroy(&priv->dm.drain_white_cond);
+        if (cr_mutex_init)
+            pthread_mutex_destroy(&priv->cr.lock);
+        if (cr_cond_init)
+            pthread_cond_destroy(&priv->cr.cond);
+    }
+    return ret;
 }
 
 /* Destroy barrier related condition variables and locks */
 static void
-changelog_barrier_pthread_destroy (changelog_priv_t *priv)
+changelog_barrier_pthread_destroy(changelog_priv_t *priv)
 {
-        pthread_mutex_destroy (&priv->bn.bnotify_mutex);
-        pthread_cond_destroy (&priv->bn.bnotify_cond);
-        pthread_mutex_destroy (&priv->dm.drain_black_mutex);
-        pthread_cond_destroy (&priv->dm.drain_black_cond);
-        pthread_mutex_destroy (&priv->dm.drain_white_mutex);
-        pthread_cond_destroy (&priv->dm.drain_white_cond);
-        pthread_mutex_destroy(&priv->cr.lock);
-        pthread_cond_destroy (&priv->cr.cond);
-        LOCK_DESTROY (&priv->bflags.lock);
+    pthread_mutex_destroy(&priv->bn.bnotify_mutex);
+    pthread_cond_destroy(&priv->bn.bnotify_cond);
+    pthread_mutex_destroy(&priv->dm.drain_black_mutex);
+    pthread_cond_destroy(&priv->dm.drain_black_cond);
+    pthread_mutex_destroy(&priv->dm.drain_white_mutex);
+    pthread_cond_destroy(&priv->dm.drain_white_cond);
+    pthread_mutex_destroy(&priv->cr.lock);
+    pthread_cond_destroy(&priv->cr.cond);
+    LOCK_DESTROY(&priv->bflags.lock);
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        int                     ret            = 0;
-        char                   *tmp            = NULL;
-        changelog_priv_t       *priv           = NULL;
-        gf_boolean_t            active_earlier = _gf_true;
-        gf_boolean_t            active_now     = _gf_true;
-        changelog_time_slice_t *slice          = NULL;
-        changelog_log_data_t    cld            = {0,};
-        char    htime_dir[PATH_MAX]            = {0,};
-        char    csnap_dir[PATH_MAX]            = {0,};
-        struct timeval          tv             = {0,};
-        uint32_t                timeout        = 0;
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        ret = -1;
-        active_earlier = priv->active;
-
-        /* first stop the rollover and the fsync thread */
-        changelog_cleanup_helper_threads (this, priv);
-
-        GF_OPTION_RECONF ("changelog-dir", tmp, options, str, out);
-        if (!tmp) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_DIR_OPTIONS_NOT_SET,
-                        "\"changelog-dir\" option is not set");
-                goto out;
-        }
-
-        GF_FREE (priv->changelog_dir);
-        priv->changelog_dir = gf_strdup (tmp);
-        if (!priv->changelog_dir)
-                goto out;
+static void
+changelog_cleanup_rpc(xlator_t *this, changelog_priv_t *priv)
+{
+    /* terminate rpc server */
+    if (!this->cleanup_starting)
+        changelog_destroy_rpc_listner(this, priv);
 
-        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+    (void)changelog_cleanup_rpc_threads(this, priv);
+    /* cleanup rot buffs */
+    rbuf_dtor(priv->rbuf);
 
-        if (ret)
-                goto out;
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
-        ret = mkdir_p (htime_dir, 0600, _gf_true);
+    /* cleanup poller thread */
+    if (priv->poller)
+        (void)changelog_thread_cleanup(this, priv->poller);
+}
 
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = 0;
+    char *tmp = NULL;
+    changelog_priv_t *priv = NULL;
+    gf_boolean_t active_earlier = _gf_true;
+    gf_boolean_t active_now = _gf_true;
+    gf_boolean_t rpc_active_earlier = _gf_true;
+    gf_boolean_t rpc_active_now = _gf_true;
+    gf_boolean_t iniate_rpc = _gf_false;
+    changelog_time_slice_t *slice = NULL;
+    changelog_log_data_t cld = {
+        0,
+    };
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir[PATH_MAX] = {
+        0,
+    };
+    uint32_t timeout = 0;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    ret = -1;
+    active_earlier = priv->active;
+    rpc_active_earlier = priv->rpc_active;
+
+    /* first stop the rollover and the fsync thread */
+    changelog_cleanup_helper_threads(this, priv);
+
+    GF_OPTION_RECONF("changelog-dir", tmp, options, str, out);
+    if (!tmp) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_DIR_OPTIONS_NOT_SET,
+                NULL);
+        goto out;
+    }
+
+    GF_FREE(priv->changelog_dir);
+    priv->changelog_dir = gf_strdup(tmp);
+    if (!priv->changelog_dir)
+        goto out;
+
+    ret = mkdir_p(priv->changelog_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
+    ret = mkdir_p(htime_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
+    ret = mkdir_p(csnap_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+
+    GF_OPTION_RECONF("changelog", active_now, options, bool, out);
+    GF_OPTION_RECONF("changelog-notification", rpc_active_now, options, bool,
+                     out);
+
+    /* If journalling is enabled, enable rpc notifications */
+    if (active_now && !active_earlier) {
+        if (!rpc_active_earlier)
+            iniate_rpc = _gf_true;
+    }
+
+    if (rpc_active_now && !rpc_active_earlier) {
+        iniate_rpc = _gf_true;
+    }
+
+    /* TODO: Disable of changelog-notifications is not supported for now
+     * as there is no clean way of cleaning up of rpc resources
+     */
+
+    if (iniate_rpc) {
+        ret = changelog_init_rpc(this, priv);
         if (ret)
-                goto out;
-
-        CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
-        ret = mkdir_p (csnap_dir, 0600, _gf_true);
+            goto out;
+        priv->rpc_active = _gf_true;
+    }
 
-        if (ret)
-                goto out;
+    /**
+     * changelog_handle_change() handles changes that could possibly
+     * have been submit changes before changelog deactivation.
+     */
+    if (!active_now)
+        priv->active = _gf_false;
 
-        GF_OPTION_RECONF ("changelog", active_now, options, bool, out);
+    GF_OPTION_RECONF("op-mode", tmp, options, str, out);
+    changelog_assign_opmode(priv, tmp);
 
-        /**
-         * changelog_handle_change() handles changes that could possibly
-         * have been submit changes before changelog deactivation.
-         */
-        if (!active_now)
-                priv->active = _gf_false;
+    tmp = NULL;
 
-        GF_OPTION_RECONF ("op-mode", tmp, options, str, out);
-        changelog_assign_opmode (priv, tmp);
+    GF_OPTION_RECONF("encoding", tmp, options, str, out);
+    changelog_assign_encoding(priv, tmp);
 
-        tmp = NULL;
+    GF_OPTION_RECONF("rollover-time", priv->rollover_time, options, int32, out);
+    GF_OPTION_RECONF("fsync-interval", priv->fsync_interval, options, int32,
+                     out);
+    GF_OPTION_RECONF("changelog-barrier-timeout", timeout, options, time, out);
+    changelog_assign_barrier_timeout(priv, timeout);
 
-        GF_OPTION_RECONF ("encoding", tmp, options, str, out);
-        changelog_assign_encoding (priv, tmp);
+    GF_OPTION_RECONF("capture-del-path", priv->capture_del_path, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("rollover-time",
-                          priv->rollover_time, options, int32, out);
-        GF_OPTION_RECONF ("fsync-interval",
-                          priv->fsync_interval, options, int32, out);
-        GF_OPTION_RECONF ("changelog-barrier-timeout",
-                          timeout, options, time, out);
-        changelog_assign_barrier_timeout (priv, timeout);
+    if (active_now || active_earlier) {
+        changelog_fill_rollover_data(&cld, !active_now);
 
-        GF_OPTION_RECONF ("capture-del-path", priv->capture_del_path, options,
-                          bool, out);
+        slice = &priv->slice;
 
-        if (active_now || active_earlier) {
-                ret = changelog_fill_rollover_data (&cld, !active_now);
-                if (ret)
-                        goto out;
+        LOCK(&priv->lock);
+        {
+            ret = changelog_inject_single_event(this, priv, &cld);
+            if (!ret && active_now)
+                SLICE_VERSION_UPDATE(slice);
+        }
+        UNLOCK(&priv->lock);
 
-                slice = &priv->slice;
+        if (ret)
+            goto out;
 
-                LOCK (&priv->lock);
-                {
-                        ret = changelog_inject_single_event (this, priv, &cld);
-                        if (!ret && active_now)
-                                SLICE_VERSION_UPDATE (slice);
-                }
-                UNLOCK (&priv->lock);
-
-                if (ret)
-                        goto out;
-
-                if (active_now) {
-                        if (!active_earlier) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        CHANGELOG_MSG_HTIME_INFO,
-                                        "Reconfigure: Changelog Enable");
-                                if (gettimeofday(&tv, NULL) ) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                CHANGELOG_MSG_HTIME_ERROR,
-                                                 "unable to fetch htime");
-                                        ret = -1;
-                                        goto out;
-                                }
-                                htime_create (this, priv, tv.tv_sec);
-                        }
-                        ret = changelog_spawn_helper_threads (this, priv);
-                }
+        if (active_now) {
+            if (!active_earlier) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_RECONFIGURE,
+                        NULL);
+                htime_create(this, priv, gf_time());
+            }
+            ret = changelog_spawn_helper_threads(this, priv);
         }
+    }
 
- out:
-        if (ret) {
-                /* TODO */
-        } else {
-                gf_msg_debug (this->name, 0,
-                              "changelog reconfigured");
-                if (active_now && priv)
-                        priv->active = _gf_true;
-        }
+out:
+    if (ret) {
+        /* TODO */
+    } else {
+        gf_msg_debug(this->name, 0, "changelog reconfigured");
+        if (active_now && priv)
+            priv->active = _gf_true;
+    }
 
-        return ret;
+    return ret;
 }
 
 static void
-changelog_freeup_options (xlator_t *this, changelog_priv_t *priv)
+changelog_freeup_options(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = priv->cb->dtor (this, &priv->cd);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_FREEUP_FAILED,
-                        "could not cleanup bootstrapper");
-        GF_FREE (priv->changelog_brick);
-        GF_FREE (priv->changelog_dir);
+    ret = priv->cb->dtor(this, &priv->cd);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_FREEUP_FAILED, NULL);
+    GF_FREE(priv->changelog_brick);
+    GF_FREE(priv->changelog_dir);
 }
 
 static int
-changelog_init_options (xlator_t *this, changelog_priv_t *priv)
+changelog_init_options(xlator_t *this, changelog_priv_t *priv)
 {
-        int       ret            = 0;
-        char     *tmp            = NULL;
-        uint32_t  timeout        = 0;
-        char htime_dir[PATH_MAX] = {0,};
-        char csnap_dir[PATH_MAX] = {0,};
-
-        GF_OPTION_INIT ("changelog-brick", tmp, str, error_return);
-        priv->changelog_brick = gf_strdup (tmp);
-        if (!priv->changelog_brick)
-                goto error_return;
-
-        tmp = NULL;
+    int ret = 0;
+    char *tmp = NULL;
+    uint32_t timeout = 0;
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir[PATH_MAX] = {
+        0,
+    };
 
-        GF_OPTION_INIT ("changelog-dir", tmp, str, dealloc_1);
-        priv->changelog_dir = gf_strdup (tmp);
-        if (!priv->changelog_dir)
-                goto dealloc_1;
+    GF_OPTION_INIT("changelog-brick", tmp, str, error_return);
+    priv->changelog_brick = gf_strdup(tmp);
+    if (!priv->changelog_brick)
+        goto error_return;
 
-        tmp = NULL;
+    tmp = NULL;
 
-        /**
-         * create the directory even if change-logging would be inactive
-         * so that consumers can _look_ into it (finding nothing...)
-         */
-        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+    GF_OPTION_INIT("changelog-dir", tmp, str, dealloc_1);
+    priv->changelog_dir = gf_strdup(tmp);
+    if (!priv->changelog_dir)
+        goto dealloc_1;
 
-        if (ret)
-                goto dealloc_2;
-
-        CHANGELOG_FILL_HTIME_DIR (priv->changelog_dir, htime_dir);
-        ret = mkdir_p (htime_dir, 0600, _gf_true);
-        if (ret)
-                goto dealloc_2;
+    tmp = NULL;
 
-        CHANGELOG_FILL_CSNAP_DIR (priv->changelog_dir, csnap_dir);
-        ret = mkdir_p (csnap_dir, 0600, _gf_true);
-        if (ret)
-                goto dealloc_2;
+    /**
+     * create the directory even if change-logging would be inactive
+     * so that consumers can _look_ into it (finding nothing...)
+     */
+    ret = mkdir_p(priv->changelog_dir, 0600, _gf_true);
 
-        GF_OPTION_INIT ("changelog", priv->active, bool, dealloc_2);
-        GF_OPTION_INIT ("capture-del-path", priv->capture_del_path,
-                        bool, dealloc_2);
+    if (ret)
+        goto dealloc_2;
 
-        GF_OPTION_INIT ("op-mode", tmp, str, dealloc_2);
-        changelog_assign_opmode (priv, tmp);
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
+    ret = mkdir_p(htime_dir, 0600, _gf_true);
+    if (ret)
+        goto dealloc_2;
 
-        tmp = NULL;
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
+    ret = mkdir_p(csnap_dir, 0600, _gf_true);
+    if (ret)
+        goto dealloc_2;
 
-        GF_OPTION_INIT ("encoding", tmp, str, dealloc_2);
-        changelog_assign_encoding (priv, tmp);
-        changelog_encode_change (priv);
+    GF_OPTION_INIT("changelog", priv->active, bool, dealloc_2);
+    GF_OPTION_INIT("changelog-notification", priv->rpc_active, bool, dealloc_2);
+    GF_OPTION_INIT("capture-del-path", priv->capture_del_path, bool, dealloc_2);
 
-        GF_OPTION_INIT ("rollover-time",
-                        priv->rollover_time, int32, dealloc_2);
+    GF_OPTION_INIT("op-mode", tmp, str, dealloc_2);
+    changelog_assign_opmode(priv, tmp);
 
-        GF_OPTION_INIT ("fsync-interval",
-                        priv->fsync_interval, int32, dealloc_2);
+    tmp = NULL;
 
-        GF_OPTION_INIT ("changelog-barrier-timeout",
-                        timeout, time, dealloc_2);
-        changelog_assign_barrier_timeout (priv, timeout);
+    GF_OPTION_INIT("encoding", tmp, str, dealloc_2);
+    changelog_assign_encoding(priv, tmp);
+    changelog_encode_change(priv);
 
-        GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
-        priv->cb = &cb_bootstrap[priv->op_mode];
+    GF_OPTION_INIT("rollover-time", priv->rollover_time, int32, dealloc_2);
 
-        /* ... now bootstrap the logger */
-        ret = priv->cb->ctor (this, &priv->cd);
-        if (ret)
-                goto dealloc_2;
+    GF_OPTION_INIT("fsync-interval", priv->fsync_interval, int32, dealloc_2);
 
-        priv->changelog_fd = -1;
+    GF_OPTION_INIT("changelog-barrier-timeout", timeout, time, dealloc_2);
+    changelog_assign_barrier_timeout(priv, timeout);
 
-        return 0;
+    GF_ASSERT(cb_bootstrap[priv->op_mode].mode == priv->op_mode);
+    priv->cb = &cb_bootstrap[priv->op_mode];
 
- dealloc_2:
-        GF_FREE (priv->changelog_dir);
- dealloc_1:
-        GF_FREE (priv->changelog_brick);
- error_return:
-        return -1;
-}
+    /* ... now bootstrap the logger */
+    ret = priv->cb->ctor(this, &priv->cd);
+    if (ret)
+        goto dealloc_2;
 
-static void
-changelog_cleanup_rpc (xlator_t *this, changelog_priv_t *priv)
-{
-        /* terminate rpc server */
-        changelog_destroy_rpc_listner (this, priv);
+    priv->changelog_fd = -1;
 
-        /* cleanup rot buffs */
-        rbuf_dtor (priv->rbuf);
+    return 0;
 
-        /* cleanup poller thread */
-        if (priv->poller)
-                (void) changelog_thread_cleanup (this, priv->poller);
+dealloc_2:
+    GF_FREE(priv->changelog_dir);
+dealloc_1:
+    GF_FREE(priv->changelog_brick);
+error_return:
+    return -1;
 }
 
 static int
-changelog_init_rpc (xlator_t *this, changelog_priv_t *priv)
+changelog_init_rpc(xlator_t *this, changelog_priv_t *priv)
 {
-        int        ret      = 0;
-        rpcsvc_t  *rpc      = NULL;
-        changelog_ev_selector_t *selection = NULL;
+    rpcsvc_t *rpc = NULL;
+    changelog_ev_selector_t *selection = NULL;
 
-        selection = &priv->ev_selection;
+    selection = &priv->ev_selection;
 
-        /* initialize event selection */
-        changelog_init_event_selection (this, selection);
+    /* initialize event selection */
+    changelog_init_event_selection(this, selection);
 
-        priv->rbuf = rbuf_init (NR_ROTT_BUFFS);
-        if (!priv->rbuf)
-                goto cleanup_thread;
+    priv->rbuf = rbuf_init(NR_ROTT_BUFFS);
+    if (!priv->rbuf)
+        goto cleanup_thread;
 
-        rpc = changelog_init_rpc_listner (this, priv,
-                                          priv->rbuf, NR_DISPATCHERS);
-        if (!rpc)
-                goto cleanup_rbuf;
-        priv->rpc = rpc;
+    rpc = changelog_init_rpc_listener(this, priv, priv->rbuf, NR_DISPATCHERS);
+    if (!rpc)
+        goto cleanup_rbuf;
+    priv->rpc = rpc;
 
-        return 0;
+    return 0;
 
- cleanup_rbuf:
-        rbuf_dtor (priv->rbuf);
- cleanup_thread:
-        if (priv->poller)
-                (void) changelog_thread_cleanup (this, priv->poller);
+cleanup_rbuf:
+    rbuf_dtor(priv->rbuf);
+cleanup_thread:
+    if (priv->poller)
+        (void)changelog_thread_cleanup(this, priv->poller);
 
-        return -1;
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
-{
-        int               ret  = -1;
-        char             *tmp  = NULL;
-        changelog_priv_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("changelog", this, error_return);
-
-        if (!this->children || this->children->next) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_CHILD_MISCONFIGURED,
-                        "translator needs a single subvolume");
-                goto error_return;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_VOL_MISCONFIGURED,
-                        "dangling volume. please check volfile");
-                goto error_return;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
-        if (!priv)
-                goto error_return;
-
-        this->local_pool = mem_pool_new (changelog_local_t, 64);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "failed to create local memory pool");
-                goto cleanup_priv;
-        }
-
-        LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->c_snap_lock);
-
-        ret = changelog_init_options (this, priv);
-        if (ret)
-                goto cleanup_mempool;
-
-        /* snap dependency changes */
-        priv->dm.black_fop_cnt = 0;
-        priv->dm.white_fop_cnt = 0;
-        priv->dm.drain_wait_black = _gf_false;
-        priv->dm.drain_wait_white = _gf_false;
-        priv->current_color = FOP_COLOR_BLACK;
-        priv->explicit_rollover = _gf_false;
-
-        priv->cr.notify = _gf_false;
-        /* Mutex is not needed as threads are not spawned yet */
-        priv->bn.bnotify = _gf_false;
-        priv->bn.bnotify_error = _gf_false;
-        ret = changelog_barrier_pthread_init (this, priv);
-        if (ret)
-                goto cleanup_options;
-        LOCK_INIT (&priv->bflags.lock);
-        priv->bflags.barrier_ext = _gf_false;
-
-        /* Changelog barrier init */
-        INIT_LIST_HEAD (&priv->queue);
-        priv->barrier_enabled = _gf_false;
-
+init(xlator_t *this)
+{
+    int ret = -1;
+    changelog_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("changelog", this, error_return);
+
+    if (!this->children || this->children->next) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_CHILD_MISCONFIGURED,
+                NULL);
+        goto error_return;
+    }
+
+    if (!this->parents) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_VOL_MISCONFIGURED,
+                NULL);
+        goto error_return;
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_changelog_mt_priv_t);
+    if (!priv)
+        goto error_return;
+
+    this->local_pool = mem_pool_new(changelog_local_t, 64);
+    if (!this->local_pool) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, CHANGELOG_MSG_NO_MEMORY,
+                NULL);
+        goto cleanup_priv;
+    }
+
+    LOCK_INIT(&priv->lock);
+    LOCK_INIT(&priv->c_snap_lock);
+    GF_ATOMIC_INIT(priv->listnercnt, 0);
+    GF_ATOMIC_INIT(priv->clntcnt, 0);
+    GF_ATOMIC_INIT(priv->xprtcnt, 0);
+    INIT_LIST_HEAD(&priv->xprt_list);
+    priv->htime_fd = -1;
+
+    ret = changelog_init_options(this, priv);
+    if (ret)
+        goto cleanup_mempool;
+
+    /* snap dependency changes */
+    priv->dm.black_fop_cnt = 0;
+    priv->dm.white_fop_cnt = 0;
+    priv->dm.drain_wait_black = _gf_false;
+    priv->dm.drain_wait_white = _gf_false;
+    priv->current_color = FOP_COLOR_BLACK;
+    priv->explicit_rollover = _gf_false;
+
+    priv->cr.notify = _gf_false;
+    /* Mutex is not needed as threads are not spawned yet */
+    priv->bn.bnotify = _gf_false;
+    priv->bn.bnotify_error = _gf_false;
+    ret = changelog_barrier_pthread_init(this, priv);
+    if (ret)
+        goto cleanup_options;
+    LOCK_INIT(&priv->bflags.lock);
+    priv->bflags.barrier_ext = _gf_false;
+
+    /* Changelog barrier init */
+    INIT_LIST_HEAD(&priv->queue);
+    priv->barrier_enabled = _gf_false;
+
+    if (priv->rpc_active || priv->active) {
         /* RPC ball rolling.. */
-        ret = changelog_init_rpc (this, priv);
+        ret = changelog_init_rpc(this, priv);
         if (ret)
-                goto cleanup_barrier;
-
-        ret = changelog_init (this, priv);
-        if (ret)
-                goto cleanup_rpc;
-
-        gf_msg_debug (this->name, 0, "changelog translator loaded");
-
-        this->private = priv;
-        return 0;
-
- cleanup_rpc:
-        changelog_cleanup_rpc (this, priv);
- cleanup_barrier:
-        changelog_barrier_pthread_destroy (priv);
- cleanup_options:
-        changelog_freeup_options (this, priv);
- cleanup_mempool:
-        mem_pool_destroy (this->local_pool);
- cleanup_priv:
-        GF_FREE (priv);
- error_return:
-        this->private = NULL;
-        return -1;
+            goto cleanup_barrier;
+        priv->rpc_active = _gf_true;
+    }
+
+    ret = changelog_init(this, priv);
+    if (ret)
+        goto cleanup_rpc;
+
+    gf_msg_debug(this->name, 0, "changelog translator loaded");
+
+    this->private = priv;
+    return 0;
+
+cleanup_rpc:
+    if (priv->rpc_active) {
+        changelog_cleanup_rpc(this, priv);
+    }
+cleanup_barrier:
+    changelog_barrier_pthread_destroy(priv);
+cleanup_options:
+    changelog_freeup_options(this, priv);
+cleanup_mempool:
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+cleanup_priv:
+    GF_FREE(priv);
+error_return:
+    this->private = NULL;
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
+
+    priv = this->private;
+
+    if (priv) {
+        if (priv->active || priv->rpc_active) {
+            /* terminate RPC server/threads */
+            changelog_cleanup_rpc(this, priv);
+            GF_FREE(priv->ev_dispatcher);
+        }
+        /* call barrier_disable to cancel timer */
+        if (priv->barrier_enabled)
+            __chlog_barrier_disable(this, &queue);
 
-        priv = this->private;
+        /* cleanup barrier related objects */
+        changelog_barrier_pthread_destroy(priv);
 
-        if (priv) {
-                /* terminate RPC server/threads */
-                changelog_cleanup_rpc (this, priv);
+        /* cleanup helper threads */
+        changelog_cleanup_helper_threads(this, priv);
 
-                /* cleanup barrier related objects */
-                changelog_barrier_pthread_destroy (priv);
+        /* cleanup allocated options */
+        changelog_freeup_options(this, priv);
 
-                /* cleanup allocated options */
-                changelog_freeup_options (this, priv);
+        /* deallocate mempool */
+        mem_pool_destroy(this->local_pool);
 
-                /* deallocate mempool */
-                mem_pool_destroy (this->local_pool);
-                /* finally, dealloac private variable */
-                GF_FREE (priv);
+        if (priv->htime_fd != -1) {
+            sys_close(priv->htime_fd);
         }
 
-        this->private = NULL;
+        /* finally, dealloac private variable */
+        GF_FREE(priv);
+    }
 
-        return;
+    this->private = NULL;
+    this->local_pool = NULL;
+
+    return;
 }
 
 struct xlator_fops fops = {
-        .open         = changelog_open,
-        .mknod        = changelog_mknod,
-        .mkdir        = changelog_mkdir,
-        .create       = changelog_create,
-        .symlink      = changelog_symlink,
-        .writev       = changelog_writev,
-        .truncate     = changelog_truncate,
-        .ftruncate    = changelog_ftruncate,
-        .link         = changelog_link,
-        .rename       = changelog_rename,
-        .unlink       = changelog_unlink,
-        .rmdir        = changelog_rmdir,
-        .setattr      = changelog_setattr,
-        .fsetattr     = changelog_fsetattr,
-        .setxattr     = changelog_setxattr,
-        .fsetxattr    = changelog_fsetxattr,
-        .removexattr  = changelog_removexattr,
-        .fremovexattr = changelog_fremovexattr,
-        .ipc          = changelog_ipc,
-        .xattrop      = changelog_xattrop,
-        .fxattrop     = changelog_fxattrop,
+    .open = changelog_open,
+    .mknod = changelog_mknod,
+    .mkdir = changelog_mkdir,
+    .create = changelog_create,
+    .symlink = changelog_symlink,
+    .writev = changelog_writev,
+    .truncate = changelog_truncate,
+    .ftruncate = changelog_ftruncate,
+    .link = changelog_link,
+    .rename = changelog_rename,
+    .unlink = changelog_unlink,
+    .rmdir = changelog_rmdir,
+    .setattr = changelog_setattr,
+    .fsetattr = changelog_fsetattr,
+    .setxattr = changelog_setxattr,
+    .fsetxattr = changelog_fsetxattr,
+    .removexattr = changelog_removexattr,
+    .fremovexattr = changelog_fremovexattr,
+    .ipc = changelog_ipc,
+    .xattrop = changelog_xattrop,
+    .fxattrop = changelog_fxattrop,
 };
 
 struct xlator_cbks cbks = {
-        .forget = changelog_forget,
-        .release = changelog_release,
+    .forget = changelog_forget,
+    .release = changelog_release,
 };
 
 struct volume_options options[] = {
-        {.key = {"changelog"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "enable/disable change-logging"
-        },
-        {.key = {"changelog-brick"},
-         .type = GF_OPTION_TYPE_PATH,
-         .description = "brick path to generate unique socket file name."
-                       " should be the export directory of the volume strictly."
-        },
-        {.key = {"changelog-dir"},
-         .type = GF_OPTION_TYPE_PATH,
-         .description = "directory for the changelog files"
-        },
-        {.key = {"op-mode"},
-         .type = GF_OPTION_TYPE_STR,
-         .default_value = "realtime",
-         .value = {"realtime"},
-         .description = "operation mode - futuristic operation modes"
-        },
-        {.key = {"encoding"},
-         .type = GF_OPTION_TYPE_STR,
-         .default_value = "ascii",
-         .value = {"binary", "ascii"},
-         .description = "encoding type for changelogs"
-        },
-        {.key = {"rollover-time"},
-         .default_value = "15",
-         .type = GF_OPTION_TYPE_TIME,
-         .description = "time to switch to a new changelog file (in seconds)"
-        },
-        {.key = {"fsync-interval"},
-         .type = GF_OPTION_TYPE_TIME,
-         .default_value = "5",
-         .description = "do not open CHANGELOG file with O_SYNC mode."
-                        " instead perform fsync() at specified intervals"
-        },
-        { .key = {"changelog-barrier-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = BARRIER_TIMEOUT,
-          .description = "After 'timeout' seconds since the time 'barrier' "
-                         "option was set to \"on\", unlink/rmdir/rename  "
-                         "operations are no longer blocked and previously "
-                         "blocked fops are allowed to go through"
-        },
-        {.key = {"capture-del-path"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "enable/disable capturing paths of deleted entries"
-        },
-        {.key = {NULL}
-        },
+    {.key = {"changelog"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable change-logging",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_BASIC,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"changelog-notification"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable changelog live notification",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .tags = {"bitrot", "georep"}},
+    {.key = {"changelog-brick"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "brick path to generate unique socket file name."
+                    " should be the export directory of the volume strictly.",
+     .default_value = "{{ brick.path }}",
+     .op_version = {3},
+     .tags = {"journal"}},
+    {.key = {"changelog-dir"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "directory for the changelog files",
+     .default_value = "{{ brick.path }}/.glusterfs/changelogs",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"op-mode"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "realtime",
+     .value = {"realtime"},
+     .description = "operation mode - futuristic operation modes",
+     .op_version = {3},
+     .tags = {"journal"}},
+    {.key = {"encoding"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "ascii",
+     .value = {"binary", "ascii"},
+     .description = "encoding type for changelogs",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"rollover-time"},
+     .default_value = "15",
+     .type = GF_OPTION_TYPE_TIME,
+     .description = "time to switch to a new changelog file (in seconds)",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"fsync-interval"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = "5",
+     .description = "do not open CHANGELOG file with O_SYNC mode."
+                    " instead perform fsync() at specified intervals",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"changelog-barrier-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = BARRIER_TIMEOUT,
+     .description = "After 'timeout' seconds since the time 'barrier' "
+                    "option was set to \"on\", unlink/rmdir/rename  "
+                    "operations are no longer blocked and previously "
+                    "blocked fops are allowed to go through",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"capture-del-path"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable capturing paths of deleted entries",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_BASIC,
+     .tags = {"journal", "glusterfind"}},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "changelog",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/changetimerecorder/src/Makefile.am b/xlators/features/changetimerecorder/src/Makefile.am
deleted file mode 100644
index 44cebd6aedf..00000000000
--- a/xlators/features/changetimerecorder/src/Makefile.am
+++ /dev/null
@@ -1,23 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-# changetimerecorder can only get build when libgfdb is enabled
-if BUILD_GFDB
-   xlator_LTLIBRARIES = changetimerecorder.la
-endif
-
-changetimerecorder_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-changetimerecorder_la_SOURCES = changetimerecorder.c ctr-helper.c ctr-xlator-ctx.c
-
-changetimerecorder_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
-			$(top_builddir)/libglusterfs/src/gfdb/libgfdb.la
-
-noinst_HEADERS = changetimerecorder.h ctr_mem_types.h ctr-helper.h ctr-xlator-ctx.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-		-I$(top_srcdir)/libglusterfs/src/gfdb \
-		-DDATADIR=\"$(localstatedir)\"
-
-AM_CFLAGS = -Wall $(GF_CFLAGS) $(SQLITE_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.c b/xlators/features/changetimerecorder/src/changetimerecorder.c
deleted file mode 100644
index 95ce08fea2b..00000000000
--- a/xlators/features/changetimerecorder/src/changetimerecorder.c
+++ /dev/null
@@ -1,2307 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "gfdb_sqlite3.h"
-#include "ctr-helper.h"
-#include "ctr-messages.h"
-#include "syscall.h"
-
-/*******************************inode forget***********************************/
-
-int
-ctr_forget (xlator_t *this, inode_t *inode)
-{
-        fini_ctr_xlator_ctx (this, inode);
-        return 0;
-}
-
-/************************** Look up heal **************************************/
-/*
-Problem: The CTR xlator records file meta (heat/hardlinks)
-into the data. This works fine for files which are created
-after ctr xlator is switched ON. But for files which were
-created before CTR xlator is ON, CTR xlator is not able to
-record either of the meta i.e heat or hardlinks. Thus making
-those files immune to promotions/demotions.
-
-Solution: The solution that is implemented in this patch is
-do ctr-db heal of all those pre-existent files, using named lookup.
-For this purpose we use the inode-xlator context variable option
-in gluster.
-The inode-xlator context variable for ctr xlator will have the
-following,
-    a. A Lock for the context variable
-    b. A hardlink list: This list represents the successful looked
-       up hardlinks.
-These are the scenarios when the hardlink list is updated:
-1) Named-Lookup: Whenever a named lookup happens on a file, in the
-   wind path we copy all required hardlink and inode information to
-   ctr_db_record structure, which resides in the frame->local variable.
-   We dont update the database in wind. During the unwind, we read the
-   information from the ctr_db_record and ,
-   Check if the inode context variable is created, if not we create it.
-   Check if the hard link is there in the hardlink list.
-      If its not there we add it to the list and send a update to the
-      database using libgfdb.
-      Please note: The database transaction can fail(and we ignore) as there
-      already might be a record in the db. This update to the db is to heal
-      if its not there.
-      If its there in the list we ignore it.
-2) Inode Forget: Whenever an inode forget hits we clear the hardlink list in
-   the inode context variable and delete the inode context variable.
-   Please note: An inode forget may happen for two reason,
-   a. when the inode is delete.
-   b. the in-memory inode is evicted from the inode table due to cache limits.
-3) create: whenever a create happens we create the inode context variable and
-   add the hardlink. The database updation is done as usual by ctr.
-4) link: whenever a hardlink is created for the inode, we create the inode
- context variable, if not present, and add the hardlink to the list.
-5) unlink: whenever a unlink happens we delete the hardlink from the list.
-6) mknod: same as create.
-7) rename: whenever a rename happens we update the hardlink in list. if the
-   hardlink was not present for updation, we add the hardlink to the list.
-
-What is pending:
-1) This solution will only work for named lookups.
-2) We dont track afr-self-heal/dht-rebalancer traffic for healing.
-
-*/
-
-
-/* This function doesnot write anything to the db,
- * just created the local variable
- * for the frame and sets values for the ctr_db_record */
-static int
-ctr_lookup_wind(call_frame_t                    *frame,
-                xlator_t                        *this,
-                gf_ctr_inode_context_t          *ctr_inode_cx)
-{
-        int ret                         = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-        GF_ASSERT(this);
-        IS_CTR_INODE_CX_SANE(ctr_inode_cx);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        if (_priv->ctr_record_wind && ctr_inode_cx->ia_type != IA_IFDIR) {
-
-                frame->local = init_ctr_local_t (this);
-                if (!frame->local) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_CREATE_CTR_LOCAL_ERROR_WIND,
-                                "WIND: Error while creating ctr local");
-                        goto out;
-                };
-                ctr_local = frame->local;
-                /*Definately no internal fops will reach here*/
-                ctr_local->is_internal_fop = _gf_false;
-                /*Dont record counters*/
-                CTR_DB_REC(ctr_local).do_record_counters = _gf_false;
-                /*Don't record time at all*/
-                CTR_DB_REC(ctr_local).do_record_times = _gf_false;
-
-                /* Copy gfid into db record*/
-                gf_uuid_copy (CTR_DB_REC(ctr_local).gfid,
-                                *(ctr_inode_cx->gfid));
-
-                /* Set fop_path and fop_type, required by libgfdb to make
-                * decision while inserting the record */
-                CTR_DB_REC(ctr_local).gfdb_fop_path = ctr_inode_cx->fop_path;
-                CTR_DB_REC(ctr_local).gfdb_fop_type = ctr_inode_cx->fop_type;
-
-                /* Copy hard link info*/
-                gf_uuid_copy (CTR_DB_REC(ctr_local).pargfid,
-                        *((NEW_LINK_CX(ctr_inode_cx))->pargfid));
-                strcpy (CTR_DB_REC(ctr_local).file_name,
-                        NEW_LINK_CX(ctr_inode_cx)->basename);
-
-                /* Since we are in lookup we can ignore errors while
-                 * Inserting in the DB, because there may be many
-                 * to write to the DB attempts for healing.
-                 * We dont want to log all failed attempts and
-                 * bloat the log*/
-                 ctr_local->gfdb_db_record.ignore_errors = _gf_true;
-        }
-
-        ret = 0;
-
-out:
-
-        if (ret) {
-                free_ctr_local (ctr_local);
-                frame->local = NULL;
-        }
-
-        return ret;
-}
-
-
-/* This function inserts the ctr_db_record populated by ctr_lookup_wind
- * in to the db. It also destroys the frame->local created by ctr_lookup_wind */
-static int
-ctr_lookup_unwind (call_frame_t          *frame,
-                   xlator_t              *this)
-{
-        int ret = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(this);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        ctr_local = frame->local;
-
-        if (ctr_local && (ctr_local->ia_inode_type != IA_IFDIR)) {
-
-                ret = insert_record(_priv->_db_conn,
-                                &ctr_local->gfdb_db_record);
-                if (ret == -1) {
-                        gf_msg (this->name,
-                                _gfdb_log_level (GF_LOG_ERROR,
-                                        ctr_local->
-                                        gfdb_db_record.ignore_errors),
-                                0, CTR_MSG_FILL_CTR_LOCAL_ERROR_UNWIND,
-                               "UNWIND: Error filling ctr local");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        free_ctr_local (ctr_local);
-        frame->local = NULL;
-        return ret;
-}
-
-/******************************************************************************
- *
- *                        FOPS HANDLING BELOW
- *
- * ***************************************************************************/
-
-/****************************LOOKUP********************************************/
-
-
-int32_t
-ctr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
-{
-        int ret                                 = -1;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-        gf_ctr_local_t *ctr_local               = NULL;
-        ctr_heal_ret_val_t  ret_val             = CTR_CTX_ERROR;
-        gf_boolean_t    _is_heal_needed         = _gf_false;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        /* if the lookup failed lookup dont do anything*/
-        if (op_ret == -1) {
-                gf_msg_trace (this->name, 0, "lookup failed with %s",
-                              strerror (op_errno));
-                goto out;
-        }
-
-        /* Ignore directory lookups */
-        if (inode->ia_type == IA_IFDIR) {
-                goto out;
-        }
-
-        /* if frame local was not set by the ctr_lookup()
-         * so dont so anything*/
-        if (!frame->local) {
-                goto out;
-        }
-
-        /* if the lookup is for dht link donot record*/
-        if (dht_is_linkfile (buf, dict)) {
-                gf_msg_trace (this->name, 0, "Ignoring Lookup "
-                                "for dht link file");
-                goto out;
-        }
-
-        ctr_local = frame->local;
-        /*Assign the proper inode type*/
-        ctr_local->ia_inode_type = inode->ia_type;
-
-        /* Copy gfid directly from inode */
-        gf_uuid_copy (CTR_DB_REC(ctr_local).gfid, inode->gfid);
-
-        /* Checking if gfid and parent gfid is valid */
-        if (gf_uuid_is_null(CTR_DB_REC(ctr_local).gfid) ||
-                gf_uuid_is_null(CTR_DB_REC(ctr_local).pargfid)) {
-                gf_msg_trace (this->name, 0,
-                        "Invalid GFID");
-                goto out;
-        }
-
-        /* if its a first entry
-         * then mark the ctr_record for create
-         * A create will attempt a file and a hard link created in the db*/
-        ctr_xlator_ctx = get_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                 /* This marks inode heal */
-                CTR_DB_REC(ctr_local).gfdb_fop_type = GFDB_FOP_CREATE_WRITE;
-                _is_heal_needed = _gf_true;
-        }
-
-        /* Copy the correct gfid from resolved inode */
-        gf_uuid_copy (CTR_DB_REC(ctr_local).gfid, inode->gfid);
-
-        /* Add hard link to the list */
-        ret_val = add_hard_link_ctx (frame, this, inode);
-        if (ret_val == CTR_CTX_ERROR) {
-                gf_msg_trace (this->name, 0,
-                        "Failed adding hardlink to list");
-                goto out;
-        }
-        /* If inode needs healing then heal the hardlink also */
-        else if (ret_val & CTR_TRY_INODE_HEAL) {
-                /* This marks inode heal */
-                CTR_DB_REC(ctr_local).gfdb_fop_type = GFDB_FOP_CREATE_WRITE;
-                _is_heal_needed = _gf_true;
-        }
-        /* If hardlink needs healing */
-        else if (ret_val & CTR_TRY_HARDLINK_HEAL) {
-                _is_heal_needed = _gf_true;
-        }
-
-        /* If lookup heal needed */
-        if (!_is_heal_needed)
-                goto out;
-
-        /* FINALLY HEAL : Inserts the ctr_db_record populated by ctr_lookup_wind
-        * in to the db. It also destroys the frame->local
-        * created by ctr_lookup_wind */
-        ret = ctr_lookup_unwind(frame, this);
-        if (ret) {
-                gf_msg_trace (this->name, 0,
-                        "Failed healing/inserting link");
-        }
-
-
-out:
-        free_ctr_local ((gf_ctr_local_t *)frame->local);
-        frame->local = NULL;
-
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             dict, postparent);
-
-        return 0;
-}
-
-
-
-int32_t
-ctr_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xdata)
-{
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx       = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx        = &ctr_link_cx;
-        int ret                                 = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /* Dont handle nameless lookups*/
-        if (!loc->parent || !loc->name)
-                goto out;
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->parent->gfid, loc->name, out);
-
-         /* Fill ctr inode context*/
-         /* IA_IFREG : We assume its a file in the wind
-          * but in the unwind we are sure what the inode is a file
-          * or directory
-          * gfid: we are just filling loc->gfid which is not correct.
-          * In unwind we fill the correct gfid for successful lookup*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, IA_IFREG,
-                loc->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-        /* Create the frame->local and populate ctr_db_record
-         * No writing to the db yet */
-        ret = ctr_lookup_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_LINK_WIND_FAILED,
-                        "Failed to insert link wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
-}
-
-
-
-
-/****************************WRITEV********************************************/
-int32_t
-ctr_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO (this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind (frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_WRITEV_UNWIND_FAILED,
-                        "Failed to insert writev unwind");
-        }
-
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
-                        postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t off,
-            uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                                fd->inode->gfid, NULL, NULL,
-                                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_WRITEV_WIND_FAILED,
-                        "Failed to insert writev wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    off, flags, iobref, xdata);
-
-        return 0;
-}
-
-/******************************setattr*****************************************/
-
-int32_t
-ctr_setattr_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, struct iatt *preop_stbuf,
-                       struct iatt *postop_stbuf, dict_t *xdata)
-{
-
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_UNWIND_FAILED,
-                        "Failed to insert setattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop_stbuf,
-                       postop_stbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_setattr (call_frame_t *frame,
-                   xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert setattr wind");
-        }
-out:
-
-        STACK_WIND (frame, ctr_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf,
-                    valid, xdata);
-
-        return 0;
-}
-
-/*************************** fsetattr ***************************************/
-int32_t
-ctr_fsetattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *preop_stbuf,
-                        struct iatt *postop_stbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_UNWIND_FAILED,
-                        "Failed to insert fsetattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
-
-        return 0;
-}
-
-
-int32_t
-ctr_fsetattr (call_frame_t *frame,
-                    xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert fsetattr wind");
-        }
-out:
-        STACK_WIND (frame, ctr_fsetattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-
-        return 0;
-}
-/****************************fremovexattr************************************/
-
-int32_t
-ctr_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FREMOVEXATTR_UNWIND_FAILED,
-                        "Failed to insert fremovexattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FREMOVEXATTR_WIND_FAILED,
-                        "Failed to insert fremovexattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
-}
-
-/****************************removexattr*************************************/
-
-int32_t
-ctr_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_REMOVEXATTR_UNWIND_FAILED,
-                        "Failed to insert removexattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_REMOVEXATTR_WIND_FAILED,
-                        "Failed to insert removexattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
-}
-
-/****************************truncate****************************************/
-
-int32_t
-ctr_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_TRUNCATE_UNWIND_FAILED,
-                        "Failed to insert truncate unwind");
-        }
-
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                      postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_TRUNCATE_WIND_FAILED,
-                        "Failed to insert truncate wind");
-        }
-out:
-        STACK_WIND (frame, ctr_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
-}
-
-/****************************ftruncate***************************************/
-
-int32_t
-ctr_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FTRUNCATE_UNWIND_FAILED,
-                        "Failed to insert ftruncate unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                      postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FTRUNCATE_WIND_FAILED,
-                        "Failed to insert ftruncate wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_ftruncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
-}
-
-/****************************rename******************************************/
-int32_t
-ctr_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
-        int ret                         = -1;
-        uint32_t remaining_links        = -1;
-        gf_ctr_local_t *ctr_local       = NULL;
-        gfdb_fop_type_t fop_type        = GFDB_FOP_INVALID_OP;
-        gfdb_fop_path_t fop_path        = GFDB_FOP_INVALID;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(this);
-
-        CTR_IS_DISABLED_THEN_GOTO (this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind (frame, this,
-                        GFDB_FOP_DENTRY_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RENAME_UNWIND_FAILED,
-                        "Failed to insert rename unwind");
-                goto out;
-        }
-
-        if (!xdata)
-                goto out;
-        /*
-         *
-         * Extracting GF_RESPONSE_LINK_COUNT_XDATA from POSIX Xlator
-         * This is only set when we are overwriting hardlinks.
-         *
-         * */
-        ret = dict_get_uint32 (xdata , GF_RESPONSE_LINK_COUNT_XDATA,
-                                &remaining_links);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_GET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed to getting GF_RESPONSE_LINK_COUNT_XDATA");
-                remaining_links = -1;
-                goto out;
-        }
-
-        ctr_local = frame->local;
-
-        /* This is not the only link */
-        if (remaining_links > 1) {
-                fop_type = GFDB_FOP_DENTRY_WRITE;
-                fop_path = GFDB_FOP_UNDEL;
-        }
-        /* Last link that was deleted */
-        else if (remaining_links == 1) {
-                fop_type = GFDB_FOP_DENTRY_WRITE;
-                fop_path = GFDB_FOP_UNDEL_ALL;
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RENAME_UNWIND_FAILED,
-                        "Invalid link count from posix");
-                goto out;
-        }
-
-        ret = ctr_delete_hard_link_from_db (this,
-                                    CTR_DB_REC(ctr_local).old_gfid,
-                                    CTR_DB_REC(ctr_local).pargfid,
-                                    CTR_DB_REC(ctr_local).file_name,
-                                    fop_type, fop_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                        "Failed to delete records of %s",
-                        CTR_DB_REC(ctr_local).old_file_name);
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent, prenewparent,
-                             postnewparent,
-                             xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        int ret                                         = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx               = &ctr_inode_cx;
-        gf_ctr_link_context_t new_link_cx, old_link_cx;
-        gf_ctr_link_context_t *_nlink_cx                = &new_link_cx;
-        gf_ctr_link_context_t *_olink_cx                = &old_link_cx;
-        int is_dict_created                             = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx                = NULL;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        /*Fill old link context*/
-        FILL_CTR_LINK_CX(_olink_cx, oldloc->pargfid, oldloc->name, out);
-
-        /*Fill new link context*/
-        FILL_CTR_LINK_CX(_nlink_cx, newloc->pargfid, newloc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, oldloc->inode->ia_type,
-                oldloc->inode->gfid, _nlink_cx, _olink_cx,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-
-        /* If the rename is a overwrite of hardlink
-         * rename ("file1", "file2")
-         * file1 is hardlink for gfid say 00000000-0000-0000-0000-00000000000A
-         * file2 is hardlink for gfid say 00000000-0000-0000-0000-00000000000B
-         * so we are saving file2 gfid in old_gfid so that we delete entries
-         * from the db during rename callback if the fop is successful
-         * */
-        if (newloc->inode) {
-                /* This is the GFID from where the newloc hardlink will be
-                 * unlinked */
-                _inode_cx->old_gfid = &newloc->inode->gfid;
-        }
-
-        /* Is a metatdata fop */
-        _inode_cx->is_metadata_fop = _gf_true;
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RENAME_WIND_FAILED,
-                        "Failed to insert rename wind");
-        } else {
-                /* We are doing updation of hard link in inode context in wind
-                 * As we dont get the "inode" in the call back for rename */
-                ret = update_hard_link_ctx (frame, this, oldloc->inode);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_UPDATE_HARDLINK_FAILED, "Failed "
-                                "updating hard link in ctr inode context");
-                        goto out;
-                }
-
-                /* If the newloc has an inode. i.e aquiring hardlink of an
-                 * exisitng file i.e overwritting a file.
-                 * */
-                if (newloc->inode) {
-
-                        /* Getting the ctr inode context variable for
-                         * inode whose hardlink will be aquired during
-                         * the rename
-                         * */
-                        ctr_xlator_ctx = get_ctr_xlator_ctx (this,
-                                                                newloc->inode);
-                        if (!ctr_xlator_ctx) {
-                                /* Since there is no ctr inode context
-                                 * so nothing more to do */
-                                ret = 0;
-                                goto out;
-                        }
-
-                        /* Deleting hardlink from context variable */
-                        ret = ctr_delete_hard_link (this, ctr_xlator_ctx,
-                                                newloc->pargfid, newloc->name);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CTR_MSG_DELETE_HARDLINK_FAILED,
-                                        "Failed to delete hard link");
-                                goto out;
-                        }
-
-                        /* Requesting for number of hardlinks on the newloc
-                         * inode from POSIX.
-                         * */
-                        is_dict_created = set_posix_link_request (this, &xdata);
-                        if (is_dict_created == -1) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
-        }
-
-out:
-        STACK_WIND (frame, ctr_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-
-        if (is_dict_created == 1) {
-                dict_unref (xdata);
-        }
-
-        return 0;
-}
-
-/****************************unlink******************************************/
-int32_t
-ctr_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        int ret                                 = -1;
-        uint32_t remaining_links                = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        if (!xdata)
-                goto out;
-
-        /*
-         *
-         * Extracting GF_RESPONSE_LINK_COUNT_XDATA from POSIX Xlator
-         *
-         * */
-        ret = dict_get_uint32 (xdata , GF_RESPONSE_LINK_COUNT_XDATA,
-                                &remaining_links);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_GET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed to getting GF_RESPONSE_LINK_COUNT_XDATA");
-                remaining_links = -1;
-        }
-
-        /*This is not the only link*/
-        if (remaining_links != 1) {
-
-                ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                        GFDB_FOP_UNDEL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                                "Failed to insert unlink unwind");
-                }
-        }
-        /*Last link that was deleted*/
-        else if (remaining_links == 1) {
-
-                ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                        GFDB_FOP_UNDEL_ALL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                                "Failed to insert unlink unwind");
-                }
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_unlink (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int xflag, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t ctr_link_cx;
-        gf_ctr_link_context_t *_link_cx = &ctr_link_cx;
-        gf_boolean_t is_xdata_created = _gf_false;
-        struct iatt dummy_stat        = {0};
-
-        GF_ASSERT (frame);
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        /*Fill link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->pargfid, loc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WDEL);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = is_internal_fop (frame, xdata);
-
-        /* Is a metadata FOP */
-        _inode_cx->is_metadata_fop = _gf_true;
-
-        /* If its a internal FOP and dht link file donot record*/
-        if (_inode_cx->is_internal_fop &&
-                        dht_is_linkfile (&dummy_stat, xdata)) {
-                goto out;
-        }
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                        "Failed to insert unlink wind");
-        } else {
-                /* We are doing delete of hard link in inode context in wind
-                 * As we dont get the "inode" in the call back for rename */
-                ret = delete_hard_link_ctx (frame, this, loc->inode);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_DELETE_HARDLINK_FAILED, "Failed "
-                                "deleting hard link from ctr inode context");
-                }
-        }
-
-        /*
-         *
-         * Sending GF_REQUEST_LINK_COUNT_XDATA
-         * to POSIX Xlator to send link count in unwind path
-         *
-         * */
-        /*create xdata if NULL*/
-        if (!xdata) {
-                xdata = dict_new();
-                is_xdata_created = (xdata) ? _gf_true : _gf_false;
-        }
-        if (!xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_XDATA_NULL, "xdata is NULL :Cannot send "
-                        "GF_REQUEST_LINK_COUNT_XDATA to posix");
-                goto out;
-        }
-
-        ret = dict_set_int32 (xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_SET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed setting GF_REQUEST_LINK_COUNT_XDATA");
-                if (is_xdata_created) {
-                        dict_unref (xdata);
-                }
-                goto out;
-        }
-
-out:
-        STACK_WIND (frame, ctr_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink,
-                    loc, xflag, xdata);
-
-        if (is_xdata_created)
-                dict_unref (xdata);
-
-        return 0;
-}
-
-/****************************fsync******************************************/
-int32_t
-ctr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_UNWIND_FAILED,
-                        "Failed to insert fsync unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                      xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               int32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_WIND_FAILED,
-                        "Failed to insert fsync wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
-}
-
-/****************************setxattr****************************************/
-
-int
-ctr_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_UNWIND_FAILED,
-                        "Failed to insert setxattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int
-ctr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert setxattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_setxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setxattr,
-                    loc, xattr, flags, xdata);
-        return 0;
-}
-/**************************** fsetxattr *************************************/
-int32_t
-ctr_fsetxattr_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_UNWIND_FAILED,
-                        "Failed to insert fsetxattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-               int32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert fsetxattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fsetxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
-}
-/****************************mknod*******************************************/
-
-
-int32_t
-ctr_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        int ret = -1;
-        ctr_heal_ret_val_t ret_val = CTR_CTX_ERROR;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        /* Add hard link to the list */
-        ret_val = add_hard_link_ctx (frame, this, inode);
-        if (ret_val == CTR_CTX_ERROR) {
-                gf_msg_trace (this->name, 0, "Failed adding hard link");
-        }
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_CREATE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_MKNOD_UNWIND_FAILED,
-                        "Failed to insert mknod unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                preparent, postparent, xdata);
-
-        return 0;
-}
-
-
-int
-ctr_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        void *uuid_req                          = NULL;
-        uuid_t gfid                             = {0,};
-        uuid_t *ptr_gfid                        = &gfid;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*get gfid from xdata dict*/
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "failed to get gfid from dict");
-                goto out;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX (_link_cx, loc->pargfid, loc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT (_inode_cx, loc->inode->ia_type,
-                *ptr_gfid, _link_cx, NULL,
-                GFDB_FOP_CREATE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_MKNOD_WIND_FAILED,
-                        "Failed to insert mknod wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
-}
-
-/****************************create******************************************/
-int
-ctr_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent,
-                dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = add_hard_link_ctx (frame, this, inode);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ADD_HARDLINK_FAILED,
-                        "Failed adding hard link");
-        }
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_CREATE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_UNWIND_FAILED,
-                        "Failed to insert create unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode,
-                             stbuf,
-                        preparent, postparent, xdata);
-
-        return 0;
-}
-
-int
-ctr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        void             *uuid_req              = NULL;
-        uuid_t            gfid                  = {0,};
-        uuid_t            *ptr_gfid             = &gfid;
-        struct iatt dummy_stat                  = {0};
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*Get GFID from Xdata dict*/
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_GET_GFID_FROM_DICT_FAILED,
-                        "failed to get gfid from dict");
-                goto out;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->pargfid, loc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                *ptr_gfid, _link_cx, NULL,
-                GFDB_FOP_CREATE_WRITE, GFDB_FOP_WIND);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = is_internal_fop (frame, xdata);
-
-        /* If its a internal FOP and dht link file donot record*/
-        if (_inode_cx->is_internal_fop &&
-                        dht_is_linkfile (&dummy_stat, xdata)) {
-                goto out;
-        }
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, &ctr_inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_WIND_FAILED,
-                        "Failed to insert create wind");
-        }
-out:
-        STACK_WIND (frame, ctr_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
-}
-
-/****************************link********************************************/
-
-int
-ctr_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        /* Add hard link to the list */
-        ret = add_hard_link_ctx (frame, this, inode);
-        if (ret) {
-                gf_msg_trace (this->name, 0, "Failed adding hard link");
-        }
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_UNWIND_FAILED,
-                        "Failed to insert create unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, stbuf,
-                       preparent, postparent, xdata);
-        return 0;
-}
-
-int
-ctr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        struct iatt dummy_stat          = {0};
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, newloc->pargfid, newloc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, oldloc->inode->ia_type,
-                oldloc->inode->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = is_internal_fop (frame, xdata);
-
-        /* Is a metadata fop */
-        _inode_cx->is_metadata_fop = _gf_true;
-
-        /* If its a internal FOP and dht link file donot record*/
-        if (_inode_cx->is_internal_fop &&
-                        dht_is_linkfile (&dummy_stat, xdata)) {
-                goto out;
-        }
-
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_LINK_WIND_FAILED,
-                        "Failed to insert link wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_link_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
-}
-
-/******************************readv*****************************************/
-int ctr_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata) {
-
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_READ,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_UNWIND_FAILED,
-                        "Failed to insert create unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                                stbuf, iobref, xdata);
-        return 0;
-}
-
-
-int
-ctr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_READ, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_READV_WIND_FAILED,
-                        "Failed to insert readv wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_readv_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readv,
-                    fd, size, off, flags, xdata);
-        return 0;
-}
-
-/*******************************ctr_ipc****************************************/
-
-/*This is the call back function per record/file from data base*/
-static int
-ctr_db_query_callback (gfdb_query_record_t *gfdb_query_record,
-                        void *args) {
-        int ret = -1;
-        ctr_query_cbk_args_t *query_cbk_args = args;
-
-        GF_VALIDATE_OR_GOTO ("ctr", query_cbk_args, out);
-
-        ret = gfdb_write_query_record (query_cbk_args->query_fd,
-                                       gfdb_query_record);
-        if (ret) {
-                gf_msg ("ctr", GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "Failed to write to query file");
-                goto out;
-        }
-
-        query_cbk_args->count++;
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/* This function does all the db queries related to tiering and
- * generates/populates new/existing query file
- * inputs:
- * xlator_t *this : CTR Translator
- * void *conn_node : Database connection
- * char *query_file: the query file that needs to be updated
- * gfdb_ipc_ctr_params_t *ipc_ctr_params: the query parameters
- * Return:
- * On success 0
- * On failure -1
- * */
-int
-ctr_db_query (xlator_t *this,
-              void *conn_node,
-              char *query_file,
-              gfdb_ipc_ctr_params_t *ipc_ctr_params)
-{
-        int ret = -1;
-        ctr_query_cbk_args_t query_cbk_args = {0};
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, conn_node, out);
-        GF_VALIDATE_OR_GOTO (this->name, query_file, out);
-        GF_VALIDATE_OR_GOTO (this->name, ipc_ctr_params, out);
-
-        /*Query for eligible files from db*/
-        query_cbk_args.query_fd = open (query_file,
-                        O_WRONLY | O_CREAT | O_APPEND,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (query_cbk_args.query_fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CTR_MSG_FATAL_ERROR,
-                        "Failed to open query file %s", query_file);
-                goto out;
-        }
-        if (!ipc_ctr_params->is_promote) {
-                if (ipc_ctr_params->write_freq_threshold == 0 &&
-                        ipc_ctr_params->read_freq_threshold == 0) {
-                                ret = find_unchanged_for_time (
-                                        conn_node,
-                                        ctr_db_query_callback,
-                                        (void *)&query_cbk_args,
-                                        &ipc_ctr_params->time_stamp);
-                } else {
-                                ret = find_unchanged_for_time_freq (
-                                        conn_node,
-                                        ctr_db_query_callback,
-                                        (void *)&query_cbk_args,
-                                        &ipc_ctr_params->time_stamp,
-                                        ipc_ctr_params->write_freq_threshold,
-                                        ipc_ctr_params->read_freq_threshold,
-                                        _gf_false);
-                }
-        } else {
-                if (ipc_ctr_params->write_freq_threshold == 0 &&
-                        ipc_ctr_params->read_freq_threshold == 0) {
-                        ret = find_recently_changed_files (
-                                conn_node,
-                                ctr_db_query_callback,
-                                (void *)&query_cbk_args,
-                                &ipc_ctr_params->time_stamp);
-                } else {
-                        ret = find_recently_changed_files_freq (
-                                conn_node,
-                                ctr_db_query_callback,
-                                (void *)&query_cbk_args,
-                                &ipc_ctr_params->time_stamp,
-                                ipc_ctr_params->write_freq_threshold,
-                                ipc_ctr_params->read_freq_threshold,
-                                _gf_false);
-                }
-        }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "FATAL: query from db failed");
-                        goto out;
-        }
-
-        ret = clear_files_heat (conn_node);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "FATAL: Failed to clear db entries");
-                        goto out;
-        }
-
-        ret = 0;
-out:
-
-        if (!ret)
-                ret = query_cbk_args.count;
-
-        if (query_cbk_args.query_fd >= 0) {
-                sys_close (query_cbk_args.query_fd);
-                query_cbk_args.query_fd = -1;
-        }
-
-        return ret;
-}
-
-
-int
-ctr_ipc_helper (xlator_t *this, dict_t *in_dict,
-                dict_t *out_dict)
-{
-        int ret = -1;
-        char *ctr_ipc_ops = NULL;
-        gf_ctr_private_t *priv = NULL;
-        char *db_version = NULL;
-        char *db_param_key = NULL;
-        char *db_param = NULL;
-        char *query_file = NULL;
-        gfdb_ipc_ctr_params_t *ipc_ctr_params = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv->_db_conn, out);
-        GF_VALIDATE_OR_GOTO (this->name, in_dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, out_dict, out);
-
-        GET_DB_PARAM_FROM_DICT(this->name, in_dict, GFDB_IPC_CTR_KEY,
-                                ctr_ipc_ops, out);
-
-        /*if its a db clear operation */
-        if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_CLEAR_OPS,
-                        strlen (GFDB_IPC_CTR_CLEAR_OPS)) == 0) {
-
-                ret = clear_files_heat (priv->_db_conn);
-                if (ret)
-                        goto out;
-
-        } /* if its a query operation, in  which case its query + clear db*/
-        else if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_QUERY_OPS,
-                                strlen (GFDB_IPC_CTR_QUERY_OPS)) == 0) {
-
-                ret = dict_get_str (in_dict, GFDB_IPC_CTR_GET_QFILE_PATH,
-                                                                &query_file);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting query file path");
-                        goto out;
-                }
-
-                ret = dict_get_bin (in_dict, GFDB_IPC_CTR_GET_QUERY_PARAMS,
-                                   (void *)&ipc_ctr_params);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting query parameters");
-                        goto out;
-                }
-
-                ret = ctr_db_query (this, priv->_db_conn, query_file,
-                                ipc_ctr_params);
-
-                ret = dict_set_int32 (out_dict,
-                                      GFDB_IPC_CTR_RET_QUERY_COUNT, ret);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed setting query reply");
-                        goto out;
-                }
-
-        } /* if its a query for db version */
-        else if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_GET_DB_VERSION_OPS,
-                        strlen (GFDB_IPC_CTR_GET_DB_VERSION_OPS)) == 0) {
-
-                ret = get_db_version (priv->_db_conn, &db_version);
-                if (ret == -1 || !db_version) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting db version ");
-                        goto out;
-                }
-
-                SET_DB_PARAM_TO_DICT(this->name, out_dict,
-                                        GFDB_IPC_CTR_RET_DB_VERSION,
-                                        db_version, ret, error);
-
-        } /* if its a query for a db setting */
-        else if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_GET_DB_PARAM_OPS,
-                                strlen (GFDB_IPC_CTR_GET_DB_PARAM_OPS)) == 0) {
-
-                ret = dict_get_str (in_dict, GFDB_IPC_CTR_GET_DB_KEY,
-                                &db_param_key);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting db param key");
-                        goto out;
-                }
-
-                ret = get_db_params (priv->_db_conn, db_param_key, &db_param);
-                if (ret == -1 || !db_param) {
-                        goto out;
-                }
-
-                SET_DB_PARAM_TO_DICT(this->name, out_dict,
-                                        db_param_key,
-                                        db_param, ret, error);
-        } /* default case */
-        else {
-                goto out;
-        }
-
-
-        ret = 0;
-        goto out;
-error:
-        GF_FREE (db_param_key);
-        GF_FREE (db_param);
-        GF_FREE (db_version);
-out:
-        return ret;
-}
-
-
-/* IPC Call from tier migrator to clear the heat on the DB */
-int32_t
-ctr_ipc (call_frame_t *frame, xlator_t *this, int32_t op,
-        dict_t *in_dict)
-{
-        int ret                         = -1;
-        gf_ctr_private_t *priv         = NULL;
-        dict_t *out_dict                = NULL;
-
-        GF_ASSERT(this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT(priv->_db_conn);
-        GF_VALIDATE_OR_GOTO (this->name, in_dict, wind);
-
-
-        if (op != GF_IPC_TARGET_CTR)
-                goto wind;
-
-        out_dict = dict_new();
-        if (!out_dict) {
-                goto out;
-        }
-
-        ret = ctr_ipc_helper (this, in_dict, out_dict);
-        if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                "Failed in ctr_ipc_helper");
-        }
-out:
-
-        STACK_UNWIND_STRICT (ipc, frame, ret, 0, out_dict);
-
-        if (out_dict)
-                dict_unref(out_dict);
-
-        return 0;
-
- wind:
-        STACK_WIND (frame, default_ipc_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, in_dict);
-
-
-
-        return 0;
-}
-
-
-/******************************************************************************/
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        char *temp_str = NULL;
-        int ret = 0;
-        gf_ctr_private_t *priv = NULL;
-
-        priv = this->private;
-        if (dict_get_str(options, "changetimerecorder.frequency",
-                         &temp_str)) {
-                gf_msg(this->name, GF_LOG_INFO, 0, CTR_MSG_SET, "set");
-        }
-
-        GF_OPTION_RECONF ("ctr-enabled", priv->enabled, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("record-counters", priv->ctr_record_counter, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("ctr-record-metadata-heat",
-                          priv->ctr_record_metadata_heat, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("ctr_link_consistency", priv->ctr_link_consistency,
-                        options, bool, out);
-
-        GF_OPTION_RECONF ("ctr_lookupheal_inode_timeout",
-                                priv->ctr_lookupheal_inode_timeout,
-                                options, uint64, out);
-
-        GF_OPTION_RECONF ("ctr_lookupheal_link_timeout",
-                                priv->ctr_lookupheal_link_timeout,
-                                options, uint64, out);
-
-        GF_OPTION_RECONF ("record-exit", priv->ctr_record_unwind, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("record-entry", priv->ctr_record_wind, options,
-                          bool, out);
-
-
-
-
-        /* If database is sqlite */
-        if (priv->gfdb_db_type == GFDB_SQLITE3) {
-
-                /* AUTOCHECKPOINT */
-                if (dict_get_str (options, GFDB_SQL_PARAM_WAL_AUTOCHECK,
-                                &temp_str) == 0) {
-                        ret = set_db_params (priv->_db_conn,
-                                        "wal_autocheckpoint", temp_str);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CTR_MSG_SET_VALUE_TO_SQL_PARAM_FAILED,
-                                        "Failed  to set %s",
-                                        GFDB_SQL_PARAM_WAL_AUTOCHECK);
-                        }
-                }
-
-                /* CACHE_SIZE */
-                if (dict_get_str (options, GFDB_SQL_PARAM_CACHE_SIZE, &temp_str)
-                                == 0) {
-                        ret = set_db_params (priv->_db_conn, "cache_size",
-                                        temp_str);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CTR_MSG_SET_VALUE_TO_SQL_PARAM_FAILED,
-                                        "Failed  to set %s",
-                                        GFDB_SQL_PARAM_CACHE_SIZE);
-                        }
-                }
-        }
-
-        ret = 0;
-
-out:
-
-        return ret;
-}
-
-/****************************init********************************************/
-
-int32_t
-init (xlator_t *this)
-{
-        gf_ctr_private_t *priv = NULL;
-        int ret_db              = -1;
-        dict_t *params_dict      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, error);
-
-        if (!this->children || this->children->next) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "FATAL: ctr should have exactly one child");
-                goto error;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CTR_MSG_DANGLING_VOLUME,
-                        "dangling volume. check volfile ");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_ctr_mt_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CTR_MSG_CALLOC_FAILED,
-                        "Calloc did not work!!!");
-                goto error;
-        }
-
-        /*Default values for the translator*/
-        priv->ctr_record_wind          = _gf_true;
-        priv->ctr_record_unwind        = _gf_false;
-        priv->ctr_hot_brick            = _gf_false;
-        priv->gfdb_db_type             = GFDB_SQLITE3;
-        priv->gfdb_sync_type           = GFDB_DB_SYNC;
-        priv->enabled                  = _gf_true;
-        priv->_db_conn                 = NULL;
-        priv->ctr_lookupheal_link_timeout =
-                                CTR_DEFAULT_HARDLINK_EXP_PERIOD;
-        priv->ctr_lookupheal_inode_timeout =
-                                CTR_DEFAULT_INODE_EXP_PERIOD;
-
-        /*Extract ctr xlator options*/
-        ret_db = extract_ctr_options (this, priv);
-        if (ret_db) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_EXTRACT_CTR_XLATOR_OPTIONS_FAILED,
-                        "Failed extracting ctr xlator options");
-                goto error;
-        }
-
-        params_dict = dict_new ();
-        if (!params_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INIT_DB_PARAMS_FAILED,
-                        "DB Params cannot initialized!");
-                goto error;
-        }
-
-        /*Extract db params options*/
-        ret_db = extract_db_params(this, params_dict, priv->gfdb_db_type);
-        if (ret_db) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_EXTRACT_DB_PARAM_OPTIONS_FAILED,
-                        "Failed extracting db params options");
-                goto error;
-        }
-
-        /*Create a memory pool for ctr xlator*/
-        this->local_pool = mem_pool_new (gf_ctr_local_t, 64);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_CREATE_LOCAL_MEMORY_POOL_FAILED,
-                        "failed to create local memory pool");
-                goto error;
-        }
-
-        /*Initialize Database Connection*/
-        priv->_db_conn = init_db(params_dict, priv->gfdb_db_type);
-        if (!priv->_db_conn) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                       CTR_MSG_FATAL_ERROR,
-                       "FATAL: Failed initializing data base");
-                        goto error;
-        }
-
-        ret_db = 0;
-        goto out;
-
-/*Error handling */
-error:
-
-        if (this)
-                mem_pool_destroy (this->local_pool);
-
-        if (priv) {
-                GF_FREE (priv->ctr_db_path);
-        }
-        GF_FREE (priv);
-
-        if (params_dict)
-                dict_unref (params_dict);
-
-        return -1;
-
-out:
-
-        if (params_dict)
-                dict_unref (params_dict);
-
-        this->private = (void *)priv;
-        return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_ctr_mt_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_MEM_ACC_INIT_FAILED, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        gf_ctr_private_t *priv = NULL;
-
-        priv = this->private;
-
-        if (priv) {
-                if (fini_db (priv->_db_conn)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CTR_MSG_CLOSE_DB_CONN_FAILED, "Failed closing "
-                                "db connection");
-                }
-                GF_FREE (priv->ctr_db_path);
-        }
-        GF_FREE (priv);
-        mem_pool_destroy (this->local_pool);
-
-        return;
-}
-
-struct xlator_fops fops = {
-        /*lookup*/
-        .lookup         = ctr_lookup,
-        /*write fops */
-        .mknod          = ctr_mknod,
-        .create         = ctr_create,
-        .truncate       = ctr_truncate,
-        .ftruncate      = ctr_ftruncate,
-        .setxattr       = ctr_setxattr,
-        .fsetxattr      = ctr_fsetxattr,
-        .removexattr    = ctr_removexattr,
-        .fremovexattr   = ctr_fremovexattr,
-        .unlink         = ctr_unlink,
-        .link           = ctr_link,
-        .rename         = ctr_rename,
-        .writev         = ctr_writev,
-        .setattr        = ctr_setattr,
-        .fsetattr       = ctr_fsetattr,
-        /*read fops*/
-        .readv          = ctr_readv,
-        /* IPC call*/
-        .ipc            = ctr_ipc
-};
-
-struct xlator_cbks cbks = {
-        .forget = ctr_forget
-};
-
-struct volume_options options[] = {
-        { .key  = {"ctr-enabled",},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off",
-          .description = "Enables the CTR"
-        },
-        { .key  = {"record-entry"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "on"
-        },
-        { .key  = {"record-exit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"record-counters"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"ctr-record-metadata-heat"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"ctr_link_consistency"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"ctr_lookupheal_link_timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "300"
-        },
-        { .key  = {"ctr_lookupheal_inode_timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "300"
-        },
-        { .key  = {"hot-brick"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"db-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"hashfile", "rocksdb", "changelog", "sqlite3",
-                    "hyperdex"},
-          .default_value = "sqlite3"
-        },
-        { .key  = {"db-sync"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"sync", "async"},
-          .default_value = "sync"
-        },
-        { .key  = {"db-path"},
-          .type = GF_OPTION_TYPE_PATH
-        },
-        { .key  = {"db-name"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {GFDB_SQL_PARAM_SYNC},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"off", "normal", "full"},
-          .default_value = "normal"
-        },
-        { .key  = {GFDB_SQL_PARAM_JOURNAL_MODE},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"delete", "truncate", "persist", "memory", "wal", "off"},
-          .default_value = "wal"
-        },
-        { .key  = {GFDB_SQL_PARAM_AUTO_VACUUM},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"off", "full", "incr"},
-          .default_value = "off"
-        },
-        { .key  = {GFDB_SQL_PARAM_WAL_AUTOCHECK},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "1000"
-        },
-        { .key  = {GFDB_SQL_PARAM_CACHE_SIZE},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "1000"
-        },
-        { .key  = {GFDB_SQL_PARAM_PAGE_SIZE},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "4096"
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.h b/xlators/features/changetimerecorder/src/changetimerecorder.h
deleted file mode 100644
index 2a8bbd18c5b..00000000000
--- a/xlators/features/changetimerecorder/src/changetimerecorder.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
-   Copyright (c) 2006-2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_H
-#define __CTR_H
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "ctr_mem_types.h"
-#include "ctr-helper.h"
-
-#endif /* __CTR_H */
diff --git a/xlators/features/changetimerecorder/src/ctr-helper.c b/xlators/features/changetimerecorder/src/ctr-helper.c
deleted file mode 100644
index 263eb58db6f..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-helper.c
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "gfdb_sqlite3.h"
-#include "ctr-helper.h"
-#include "ctr-messages.h"
-
-/*******************************************************************************
- *
- *                      Fill unwind into db record
- *
- ******************************************************************************/
-int
-fill_db_record_for_unwind(xlator_t              *this,
-                          gf_ctr_local_t        *ctr_local,
-                          gfdb_fop_type_t       fop_type,
-                          gfdb_fop_path_t       fop_path)
-{
-        int ret                         = -1;
-        gfdb_time_t *ctr_uwtime         = NULL;
-        gf_ctr_private_t *_priv                 = NULL;
-
-        GF_ASSERT (this);
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(ctr_local);
-
-        /*If not unwind path error*/
-        if (!isunwindpath(fop_path)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_WRONG_FOP_PATH,
-                        "Wrong fop_path. Should be unwind");
-                goto out;
-        }
-
-        ctr_uwtime = &CTR_DB_REC(ctr_local).gfdb_unwind_change_time;
-        CTR_DB_REC(ctr_local).gfdb_fop_path = fop_path;
-        CTR_DB_REC(ctr_local).gfdb_fop_type = fop_type;
-
-        ret = gettimeofday (ctr_uwtime, NULL);
-        if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CTR_MSG_FILL_UNWIND_TIME_REC_ERROR, "Error "
-                                "filling unwind time record %s",
-                                strerror(errno));
-                        goto out;
-                }
-
-        /* Special case i.e if its a tier rebalance
-         * + cold tier brick
-         * + its a create/mknod FOP
-         * we record unwind time as zero */
-        if (ctr_local->client_pid == GF_CLIENT_PID_TIER_DEFRAG
-                && (!_priv->ctr_hot_brick)
-                && isdentrycreatefop(fop_type)) {
-                memset(ctr_uwtime, 0, sizeof(*ctr_uwtime));
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-
-/*******************************************************************************
- *
- *                      Fill wind into db record
- *
- ******************************************************************************/
-int
-fill_db_record_for_wind (xlator_t               *this,
-                        gf_ctr_local_t          *ctr_local,
-                        gf_ctr_inode_context_t  *ctr_inode_cx)
-{
-        int ret                                 = -1;
-        gfdb_time_t *ctr_wtime                  = NULL;
-        gf_ctr_private_t *_priv                 = NULL;
-
-        GF_ASSERT (this);
-        _priv = this->private;
-        GF_ASSERT (_priv);
-        GF_ASSERT (ctr_local);
-        IS_CTR_INODE_CX_SANE (ctr_inode_cx);
-
-        /*if not wind path error!*/
-        if (!iswindpath(ctr_inode_cx->fop_path)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_WRONG_FOP_PATH,
-                        "Wrong fop_path. Should be wind");
-                goto out;
-        }
-
-        ctr_wtime = &CTR_DB_REC(ctr_local).gfdb_wind_change_time;
-        CTR_DB_REC(ctr_local).gfdb_fop_path = ctr_inode_cx->fop_path;
-        CTR_DB_REC(ctr_local).gfdb_fop_type = ctr_inode_cx->fop_type;
-        CTR_DB_REC(ctr_local).link_consistency = _priv->ctr_link_consistency;
-
-        ret = gettimeofday (ctr_wtime, NULL);
-        if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CTR_MSG_FILL_UNWIND_TIME_REC_ERROR,
-                                "Error filling wind time record %s",
-                                strerror(errno));
-                        goto out;
-        }
-
-        /* Special case i.e if its a tier rebalance
-         * + cold tier brick
-         * + its a create/mknod FOP
-         * we record wind time as zero */
-        if (ctr_local->client_pid == GF_CLIENT_PID_TIER_DEFRAG
-                && (!_priv->ctr_hot_brick)
-                && isdentrycreatefop(ctr_inode_cx->fop_type)) {
-                memset(ctr_wtime, 0, sizeof(*ctr_wtime));
-        }
-
-        /* Copy gfid into db record */
-        gf_uuid_copy (CTR_DB_REC(ctr_local).gfid, *(ctr_inode_cx->gfid));
-
-        /* Copy older gfid if any */
-        if (ctr_inode_cx->old_gfid &&
-                (!gf_uuid_is_null (*(ctr_inode_cx->old_gfid)))) {
-                gf_uuid_copy (CTR_DB_REC(ctr_local).old_gfid,
-                                *(ctr_inode_cx->old_gfid));
-        }
-
-        /*Hard Links*/
-        if (isdentryfop(ctr_inode_cx->fop_type)) {
-                /*new link fop*/
-                if (NEW_LINK_CX(ctr_inode_cx)) {
-                        gf_uuid_copy (CTR_DB_REC(ctr_local).pargfid,
-                                *((NEW_LINK_CX(ctr_inode_cx))->pargfid));
-                        strcpy (CTR_DB_REC(ctr_local).file_name,
-                                NEW_LINK_CX(ctr_inode_cx)->basename);
-                }
-                /*rename fop*/
-                if (OLD_LINK_CX(ctr_inode_cx)) {
-                        gf_uuid_copy (CTR_DB_REC(ctr_local).old_pargfid,
-                                *((OLD_LINK_CX(ctr_inode_cx))->pargfid));
-                        strcpy (CTR_DB_REC(ctr_local).old_file_name,
-                                OLD_LINK_CX(ctr_inode_cx)->basename);
-                }
-        }
-
-        ret = 0;
-out:
-        /*On error roll back and clean the record*/
-        if (ret == -1) {
-                CLEAR_CTR_DB_RECORD (ctr_local);
-        }
-        return ret;
-}
-
-
-/******************************************************************************
- *
- *                      CTR xlator init related functions
- *
- *
- * ****************************************************************************/
-static int
-extract_sql_params(xlator_t *this, dict_t *params_dict)
-{
-        int ret                         = -1;
-        char *db_path                   = NULL;
-        char *db_name                   = NULL;
-        char *db_full_path              = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (params_dict);
-
-        /*Extract the path of the db*/
-        db_path = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(this->name, this->options, "db-path",
-                                        db_path, "/var/run/gluster/");
-
-        /*Extract the name of the db*/
-        db_name = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(this->name, this->options, "db-name",
-                                        db_name, "gf_ctr_db.db");
-
-        /*Construct full path of the db*/
-        ret = gf_asprintf(&db_full_path, "%s/%s", db_path, db_name);
-        if (ret < 0) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        CTR_MSG_CONSTRUCT_DB_PATH_FAILED,
-                        "Construction of full db path failed!");
-                goto out;
-        }
-
-        /*Setting the SQL DB Path*/
-        SET_DB_PARAM_TO_DICT(this->name, params_dict, GFDB_SQL_PARAM_DBPATH,
-                                db_full_path, ret, out);
-
-        /*Extact rest of the sql params*/
-        ret = gfdb_set_sql_params(this->name, this->options, params_dict);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        CTR_MSG_SET_VALUE_TO_SQL_PARAM_FAILED,
-                        "Failed setting values to sql param dict!");
-        }
-
-        ret = 0;
-
-out:
-        if (ret)
-                GF_FREE (db_full_path);
-        return ret;
-}
-
-
-
-int extract_db_params(xlator_t *this, dict_t *params_dict,
-                                                gfdb_db_type_t db_type) {
-
-        int ret = -1;
-
-        GF_ASSERT (this);
-        GF_ASSERT (params_dict);
-
-        switch (db_type) {
-        case GFDB_SQLITE3:
-                ret = extract_sql_params(this, params_dict);
-                if (ret)
-                        goto out;
-                break;
-        case GFDB_ROCKS_DB:
-        case GFDB_HYPERDEX:
-        case GFDB_HASH_FILE_STORE:
-        case GFDB_INVALID_DB:
-        case GFDB_DB_END:
-                ret = -1;
-                break;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-int extract_ctr_options (xlator_t *this, gf_ctr_private_t *_priv) {
-        int ret         = -1;
-        char *_val_str  = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (_priv);
-
-        /*Checking if the CTR Translator is enabled. By default its disabled*/
-        _priv->enabled = _gf_false;
-        GF_OPTION_INIT ("ctr-enabled", _priv->enabled, bool, out);
-        if (!_priv->enabled) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_INFO, 0,
-                        CTR_MSG_XLATOR_DISABLED,
-                        "CTR Xlator is disabled.");
-                ret = 0;
-                goto out;
-        }
-
-        /*Extract db type*/
-        GF_OPTION_INIT ("db-type", _val_str, str, out);
-        _priv->gfdb_db_type = gf_string2gfdbdbtype(_val_str);
-
-        /*Extract flag for record on wind*/
-        GF_OPTION_INIT ("record-entry", _priv->ctr_record_wind, bool, out);
-
-        /*Extract flag for record on unwind*/
-        GF_OPTION_INIT ("record-exit", _priv->ctr_record_unwind, bool, out);
-
-        /*Extract flag for record on counters*/
-        GF_OPTION_INIT ("record-counters", _priv->ctr_record_counter, bool,
-                        out);
-
-        /* Extract flag for record metadata heat */
-        GF_OPTION_INIT ("ctr-record-metadata-heat",
-                        _priv->ctr_record_metadata_heat, bool,
-                        out);
-
-        /*Extract flag for link consistency*/
-        GF_OPTION_INIT ("ctr_link_consistency", _priv->ctr_link_consistency,
-                        bool, out);
-
-        /*Extract ctr_lookupheal_inode_timeout */
-        GF_OPTION_INIT ("ctr_lookupheal_inode_timeout",
-                        _priv->ctr_lookupheal_inode_timeout,
-                        uint64, out);
-
-        /*Extract ctr_lookupheal_link_timeout*/
-        GF_OPTION_INIT ("ctr_lookupheal_link_timeout",
-                        _priv->ctr_lookupheal_link_timeout,
-                        uint64, out);
-
-        /*Extract flag for hot tier brick*/
-        GF_OPTION_INIT ("hot-brick", _priv->ctr_hot_brick, bool, out);
-
-        /*Extract flag for sync mode*/
-        GF_OPTION_INIT ("db-sync", _val_str, str, out);
-        _priv->gfdb_sync_type = gf_string2gfdbdbsync(_val_str);
-
-        ret = 0;
-
-out:
-        return ret;
-}
diff --git a/xlators/features/changetimerecorder/src/ctr-helper.h b/xlators/features/changetimerecorder/src/ctr-helper.h
deleted file mode 100644
index d5615270184..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-helper.h
+++ /dev/null
@@ -1,923 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_HELPER_H
-#define __CTR_HELPER_H
-
-
-#include "xlator.h"
-#include "ctr_mem_types.h"
-#include "iatt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
-#include "common-utils.h"
-#include <time.h>
-#include <sys/time.h>
-
-#include "gfdb_data_store.h"
-#include "ctr-xlator-ctx.h"
-#include "ctr-messages.h"
-
-#define CTR_DEFAULT_HARDLINK_EXP_PERIOD 300  /* Five mins */
-#define CTR_DEFAULT_INODE_EXP_PERIOD    300 /* Five mins */
-
-
-typedef struct ctr_query_cbk_args {
-        int query_fd;
-        int count;
-} ctr_query_cbk_args_t;
-
-
-/*CTR Xlator Private structure*/
-typedef struct gf_ctr_private {
-        gf_boolean_t                    enabled;
-        char                            *ctr_db_path;
-        gf_boolean_t                    ctr_hot_brick;
-        gf_boolean_t                    ctr_record_wind;
-        gf_boolean_t                    ctr_record_unwind;
-        gf_boolean_t                    ctr_record_counter;
-        gf_boolean_t                    ctr_record_metadata_heat;
-        gf_boolean_t                    ctr_link_consistency;
-        gfdb_db_type_t                  gfdb_db_type;
-        gfdb_sync_type_t                gfdb_sync_type;
-        gfdb_conn_node_t                *_db_conn;
-        uint64_t                        ctr_lookupheal_link_timeout;
-        uint64_t                        ctr_lookupheal_inode_timeout;
-} gf_ctr_private_t;
-
-
-/*
- * gf_ctr_local_t is the ctr xlator local data structure that is stored in
- * the call_frame of each FOP.
- *
- * gfdb_db_record: The gf_ctr_local contains a gfdb_db_record object, which is
- * used by the insert_record() api from the libgfdb. The gfdb_db_record object
- * will contain all the inode and hardlink(only for dentry fops: create,
- * mknod,link, unlink, rename).The ctr_local is keep alive till the unwind
- * call and will be release during the unwind. The same gfdb_db_record will
- * used for the unwind insert_record() api, to record unwind in the database.
- *
- * ia_inode_type in gf_ctr_local will tell the type of the inode. This is
- * important for during the unwind path. As we will not have the inode during
- * the unwind path. We would have include this in the gfdb_db_record itself
- * but currently we record only file inode information.
- *
- * is_internal_fop in gf_ctr_local will tell us if this is a internal fop and
- * take special/no action. We dont record change/acces times or increement heat
- * counter for internal fops from rebalancer.
- * */
-typedef struct gf_ctr_local {
-        gfdb_db_record_t        gfdb_db_record;
-        ia_type_t               ia_inode_type;
-        gf_boolean_t            is_internal_fop;
-        gf_special_pid_t        client_pid;
-} gf_ctr_local_t;
-/*
- * Easy access of gfdb_db_record of ctr_local
- * */
-#define CTR_DB_REC(ctr_local)\
-        (ctr_local->gfdb_db_record)
-
-/*Clear db record*/
-#define CLEAR_CTR_DB_RECORD(ctr_local)\
-do {\
-        ctr_local->gfdb_db_record.gfdb_fop_path = GFDB_FOP_INVALID;\
-        memset(&(ctr_local->gfdb_db_record.gfdb_wind_change_time),\
-                0, sizeof(gfdb_time_t));\
-        memset(&(ctr_local->gfdb_db_record.gfdb_unwind_change_time),\
-                0, sizeof(gfdb_time_t));\
-        gf_uuid_clear (ctr_local->gfdb_db_record.gfid);\
-        gf_uuid_clear (ctr_local->gfdb_db_record.pargfid);\
-        memset(ctr_local->gfdb_db_record.file_name, 0, GF_NAME_MAX + 1);\
-        memset(ctr_local->gfdb_db_record.old_file_name, 0, GF_NAME_MAX + 1);\
-        ctr_local->gfdb_db_record.gfdb_fop_type = GFDB_FOP_INVALID_OP;\
-        ctr_local->ia_inode_type = IA_INVAL;\
-} while (0)
-
-
-static gf_ctr_local_t *
-init_ctr_local_t (xlator_t *this) {
-
-        gf_ctr_local_t  *ctr_local     = NULL;
-
-        GF_ASSERT(this);
-
-        ctr_local = mem_get0 (this->local_pool);
-        if (!ctr_local) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        CTR_MSG_CREATE_CTR_LOCAL_ERROR_WIND,
-                        "Error while creating ctr local");
-                goto out;
-        }
-
-        CLEAR_CTR_DB_RECORD (ctr_local);
-out:
-        return ctr_local;
-}
-
-static void
-free_ctr_local (gf_ctr_local_t *ctr_local)
-{
-        if (ctr_local)
-                mem_put (ctr_local);
-}
-
-
-
-/******************************************************************************
- *
- *
- *                      Context Carrier Structures
- *
- *
- * ****************************************************************************/
-
-/*
- * Context Carrier structures are used to carry relavent information about
- * inodes and links from the fops calls to the ctr_insert_wind.
- * These structure just have pointers to the original data and donot
- * do a deep copy of any data. This info is deep copied to
- * ctr_local->gfdb_db_record and passed to insert_record() api of libgfdb. This
- * info remains persistent for the unwind in  ctr_local->gfdb_db_record
- * and once used will be destroyed.
- *
- * gf_ctr_link_context_t : Context structure for hard links
- * gf_ctr_inode_context_t : Context structure for inodes
- *
- * */
-
- /*Context Carrier Structure for hard links*/
-typedef struct gf_ctr_link_context {
-        uuid_t                  *pargfid;
-        const char              *basename;
-} gf_ctr_link_context_t;
-
- /*Context Carrier Structure for inodes*/
-typedef struct gf_ctr_inode_context {
-        ia_type_t               ia_type;
-        uuid_t                  *gfid;
-        uuid_t                  *old_gfid;
-        gf_ctr_link_context_t   *new_link_cx;
-        gf_ctr_link_context_t   *old_link_cx;
-        gfdb_fop_type_t         fop_type;
-        gfdb_fop_path_t         fop_path;
-        gf_boolean_t            is_internal_fop;
-        /* Indicating metadata fops */
-        gf_boolean_t            is_metadata_fop;
-} gf_ctr_inode_context_t;
-
-
-/*******************Util Macros for Context Carrier Structures*****************/
-
-/*Checks if ctr_link_cx is sane!*/
-#define IS_CTR_LINK_CX_SANE(ctr_link_cx)\
-do {\
-        if (ctr_link_cx) {\
-                if (ctr_link_cx->pargfid)\
-                        GF_ASSERT (*(ctr_link_cx->pargfid));\
-                GF_ASSERT (ctr_link_cx->basename);\
-        };\
-} while (0)
-
-/*Clear and fill the ctr_link_context with values*/
-#define FILL_CTR_LINK_CX(ctr_link_cx, _pargfid, _basename, label)\
-do {\
-        GF_VALIDATE_OR_GOTO ("ctr", ctr_link_cx, label);\
-        GF_VALIDATE_OR_GOTO ("ctr", _pargfid, label);\
-        GF_VALIDATE_OR_GOTO ("ctr", _basename, label);\
-        memset (ctr_link_cx, 0, sizeof (*ctr_link_cx));\
-        ctr_link_cx->pargfid = &_pargfid;\
-        ctr_link_cx->basename = _basename;\
-} while (0)
-
-#define NEW_LINK_CX(ctr_inode_cx)\
-        ctr_inode_cx->new_link_cx\
-
-#define OLD_LINK_CX(ctr_inode_cx)\
-        ctr_inode_cx->old_link_cx\
-
-/*Checks if ctr_inode_cx is sane!*/
-#define IS_CTR_INODE_CX_SANE(ctr_inode_cx)\
-do {\
-        GF_ASSERT (ctr_inode_cx);\
-        GF_ASSERT (ctr_inode_cx->gfid);\
-        GF_ASSERT (*(ctr_inode_cx->gfid));\
-        GF_ASSERT (ctr_inode_cx->fop_type != GFDB_FOP_INVALID_OP);\
-        GF_ASSERT (ctr_inode_cx->fop_path != GFDB_FOP_INVALID);\
-        IS_CTR_LINK_CX_SANE (NEW_LINK_CX(ctr_inode_cx));\
-        IS_CTR_LINK_CX_SANE (OLD_LINK_CX(ctr_inode_cx));\
-} while (0)
-
-/*Clear and fill the ctr_inode_context with values*/
-#define FILL_CTR_INODE_CONTEXT(ctr_inode_cx,\
-                                _ia_type,\
-                                _gfid,\
-                                _new_link_cx,\
-                                _old_link_cx,\
-                                _fop_type,\
-                                _fop_path)\
-do {\
-        GF_ASSERT (ctr_inode_cx);\
-        GF_ASSERT (_gfid);\
-        GF_ASSERT (_fop_type != GFDB_FOP_INVALID_OP);\
-        GF_ASSERT (_fop_path != GFDB_FOP_INVALID);\
-        memset(ctr_inode_cx, 0, sizeof(*ctr_inode_cx));\
-        ctr_inode_cx->ia_type = _ia_type;\
-        ctr_inode_cx->gfid = &_gfid;\
-        IS_CTR_LINK_CX_SANE(NEW_LINK_CX(ctr_inode_cx));\
-        if (_new_link_cx)\
-                NEW_LINK_CX(ctr_inode_cx) = _new_link_cx;\
-        IS_CTR_LINK_CX_SANE(OLD_LINK_CX(ctr_inode_cx));\
-        if (_old_link_cx)\
-                OLD_LINK_CX(ctr_inode_cx) = _old_link_cx;\
-        ctr_inode_cx->fop_type = _fop_type;\
-        ctr_inode_cx->fop_path = _fop_path;\
-} while (0)
-
-
-/******************************************************************************
- *
- *                      Util functions or macros used by
- *                      insert wind and insert unwind
- *
- * ****************************************************************************/
-/* Free ctr frame local */
-static inline void
-ctr_free_frame_local (call_frame_t *frame) {
-        if (frame) {
-                free_ctr_local ((gf_ctr_local_t *) frame->local);
-                frame->local = NULL;
-        }
-}
-
-/* Setting GF_REQUEST_LINK_COUNT_XDATA in dict
- * that has to be sent to POSIX Xlator to send
- * link count in unwind path.
- * return 0 for success with not creation of dict
- * return 1 for success with creation of dict
- * return -1 for failure.
- * */
-static inline int
-set_posix_link_request (xlator_t        *this,
-                        dict_t          **xdata)
-{
-        int ret                         = -1;
-        gf_boolean_t is_created         = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, out);
-
-        /*create xdata if NULL*/
-        if (!*xdata) {
-                *xdata = dict_new();
-                is_created = _gf_true;
-                ret = 1;
-        } else {
-                ret = 0;
-        }
-
-        if (!*xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_XDATA_NULL,
-                        "xdata is NULL :Cannot send "
-                        "GF_REQUEST_LINK_COUNT_XDATA to posix");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_int32 (*xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_SET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed setting GF_REQUEST_LINK_COUNT_XDATA");
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
-out:
-        if (ret == -1) {
-                if (*xdata && is_created) {
-                        dict_unref (*xdata);
-                }
-        }
-        return ret;
-}
-
-
-/*
- * If a bitrot fop
- * */
-#define BITROT_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_BITD ||\
-        frame->root->pid == GF_CLIENT_PID_SCRUB)
-
-
-/*
- * If a rebalancer fop
- * */
-#define REBALANCE_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_DEFRAG)
-
-/*
- * If its a tiering rebalancer fop
- * */
-#define TIER_REBALANCE_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
-
-/*
- * If its a AFR SELF HEAL
- * */
- #define AFR_SELF_HEAL_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_SELF_HEALD)
-
-/*
- * if a rebalancer fop goto
- * */
-#define CTR_IF_REBALANCE_FOP_THEN_GOTO(frame, label)\
-do {\
-        if (REBALANCE_FOP (frame))\
-                goto label;\
-} while (0)
-
-/*
- * Internal fop
- *
- * */
-static inline gf_boolean_t
-is_internal_fop (call_frame_t *frame,
-                              dict_t       *xdata)
-{
-        gf_boolean_t ret = _gf_false;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        if (AFR_SELF_HEAL_FOP (frame)) {
-                ret = _gf_true;
-        }
-        if (BITROT_FOP (frame)) {
-                ret = _gf_true;
-        }
-        if (REBALANCE_FOP (frame) || TIER_REBALANCE_FOP (frame)) {
-                ret = _gf_true;
-                if (xdata && dict_get (xdata, CTR_ATTACH_TIER_LOOKUP)) {
-                        ret = _gf_false;
-                }
-        }
-        if (xdata && dict_get (xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
-                ret = _gf_true;
-        }
-
-        return ret;
-}
-
-#define CTR_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label)\
-do {\
-        if (is_internal_fop (frame, dict)) \
-                        goto label; \
-} while (0)
-
-/* if fop has failed exit */
-#define CTR_IF_FOP_FAILED_THEN_GOTO(this, op_ret, op_errno, label)\
-do {\
-        if (op_ret == -1) {\
-                gf_msg_trace (this->name, 0, "Failed fop with %s",\
-                              strerror (op_errno));\
-                goto label;\
-        };\
-} while (0)
-
-/*
- * IS CTR Xlator is disabled then goto to label
- * */
- #define CTR_IS_DISABLED_THEN_GOTO(this, label)\
- do {\
-        gf_ctr_private_t *_priv = NULL;\
-        GF_ASSERT (this);\
-        GF_ASSERT (this->private);\
-        _priv = this->private;\
-        if (!_priv->enabled)\
-                goto label;\
- } while (0)
-
-/*
- * IS CTR record metadata heat is disabled then goto to label
- * */
- #define CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO(this, label)\
- do {\
-        gf_ctr_private_t *_priv = NULL;\
-        GF_ASSERT (this);\
-        GF_ASSERT (this->private);\
-        _priv = this->private;\
-        if (!_priv->ctr_record_metadata_heat)\
-                goto label;\
- } while (0)
-
-int
-fill_db_record_for_unwind (xlator_t              *this,
-                          gf_ctr_local_t        *ctr_local,
-                          gfdb_fop_type_t       fop_type,
-                          gfdb_fop_path_t       fop_path);
-
-int
-fill_db_record_for_wind (xlator_t                *this,
-                        gf_ctr_local_t          *ctr_local,
-                        gf_ctr_inode_context_t  *ctr_inode_cx);
-
-/*******************************************************************************
- *                              CTR INSERT WIND
- * *****************************************************************************
- * Function used to insert/update record into the database during a wind fop
- * This function creates ctr_local structure into the frame of the fop
- * call.
- * ****************************************************************************/
-
-static inline int
-ctr_insert_wind (call_frame_t                    *frame,
-                xlator_t                        *this,
-                gf_ctr_inode_context_t          *ctr_inode_cx)
-{
-        int ret                         = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-        GF_ASSERT(this);
-        IS_CTR_INODE_CX_SANE(ctr_inode_cx);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        /*If record_wind option of CTR is on record wind for
-         * regular files only*/
-        if (_priv->ctr_record_wind && ctr_inode_cx->ia_type != IA_IFDIR) {
-                frame->local = init_ctr_local_t (this);
-                if (!frame->local) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_CREATE_CTR_LOCAL_ERROR_WIND,
-                                "WIND: Error while creating ctr local");
-                        goto out;
-                };
-                ctr_local = frame->local;
-                ctr_local->client_pid = frame->root->pid;
-                ctr_local->is_internal_fop = ctr_inode_cx->is_internal_fop;
-
-                /* Decide whether to record counters or not */
-                CTR_DB_REC(ctr_local).do_record_counters = _gf_false;
-                /* If record counter is enabled */
-                if (_priv->ctr_record_counter) {
-                        /* If not a internal fop */
-                        if (!(ctr_local->is_internal_fop)) {
-                                /* If its a metadata fop AND
-                                 * record metadata heat
-                                 * OR
-                                 * its NOT a metadata fop */
-                                if ((ctr_inode_cx->is_metadata_fop
-                                        && _priv->ctr_record_metadata_heat)
-                                        ||
-                                        (!ctr_inode_cx->is_metadata_fop)) {
-                                        CTR_DB_REC(ctr_local).do_record_counters
-                                                = _gf_true;
-                                }
-                        }
-                }
-
-                /* Decide whether to record times or not
-                 * For non internal FOPS record times as usual*/
-                CTR_DB_REC(ctr_local).do_record_times = _gf_false;
-                if (!ctr_local->is_internal_fop) {
-                        /* If its a metadata fop AND
-                        * record metadata heat
-                        * OR
-                        * its NOT a metadata fop */
-                        if ((ctr_inode_cx->is_metadata_fop &&
-                                _priv->ctr_record_metadata_heat)
-                                ||
-                                (!ctr_inode_cx->is_metadata_fop)) {
-                                CTR_DB_REC(ctr_local).do_record_times =
-                                        (_priv->ctr_record_wind
-                                        || _priv->ctr_record_unwind);
-                        }
-                }
-                /* when its a internal FOPS*/
-                else {
-                        /* Record times only for create
-                         * i.e when the inode is created */
-                        CTR_DB_REC(ctr_local).do_record_times =
-                                (isdentrycreatefop(ctr_inode_cx->fop_type)) ?
-                                        _gf_true : _gf_false;
-                }
-
-                /*Fill the db record for insertion*/
-                ret = fill_db_record_for_wind (this, ctr_local, ctr_inode_cx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_FILL_CTR_LOCAL_ERROR_WIND,
-                                "WIND: Error filling  ctr local");
-                        goto out;
-                }
-
-                /*Insert the db record*/
-                ret = insert_record (_priv->_db_conn,
-                                &ctr_local->gfdb_db_record);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_INSERT_RECORD_WIND_FAILED,
-                                "WIND: Inserting of record failed!");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-
-        if (ret) {
-                free_ctr_local (ctr_local);
-                frame->local = NULL;
-        }
-
-        return ret;
-}
-
-
-
-
-/*******************************************************************************
- *                             CTR INSERT UNWIND
- * *****************************************************************************
- * Function used to insert/update record into the database during a unwind fop
- * This function destroys ctr_local structure into the frame of the fop
- * call at the end.
- * ****************************************************************************/
-static inline int
-ctr_insert_unwind (call_frame_t          *frame,
-                  xlator_t              *this,
-                  gfdb_fop_type_t       fop_type,
-                  gfdb_fop_path_t       fop_path)
-{
-        int ret = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(this);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        ctr_local = frame->local;
-
-        if (ctr_local
-            && (_priv->ctr_record_unwind || isdentryfop(fop_type))
-            && (ctr_local->ia_inode_type != IA_IFDIR)) {
-
-                CTR_DB_REC(ctr_local).do_record_uwind_time =
-                                                _priv->ctr_record_unwind;
-
-                ret = fill_db_record_for_unwind(this, ctr_local, fop_type,
-                                               fop_path);
-                if (ret == -1) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0,
-                               CTR_MSG_FILL_CTR_LOCAL_ERROR_UNWIND,
-                               "UNWIND: Error filling ctr local");
-                        goto out;
-                }
-
-                ret = insert_record(_priv->_db_conn,
-                                        &ctr_local->gfdb_db_record);
-                if (ret == -1) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0,
-                               CTR_MSG_FILL_CTR_LOCAL_ERROR_UNWIND,
-                               "UNWIND: Error filling ctr local");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-/******************************************************************************
- *                          Delete file/flink record/s from db
- * ****************************************************************************/
-static inline int
-ctr_delete_hard_link_from_db (xlator_t               *this,
-                              uuid_t                 gfid,
-                              uuid_t                 pargfid,
-                              char                   *basename,
-                              gfdb_fop_type_t        fop_type,
-                              gfdb_fop_path_t        fop_path)
-{
-        int                     ret                = -1;
-        gfdb_db_record_t        gfdb_db_record;
-        gf_ctr_private_t        *_priv             = NULL;
-
-        _priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, _priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, (!gf_uuid_is_null (gfid)), out);
-        GF_VALIDATE_OR_GOTO (this->name, (!gf_uuid_is_null (pargfid)), out);
-        GF_VALIDATE_OR_GOTO (this->name, (fop_type == GFDB_FOP_DENTRY_WRITE),
-                             out);
-        GF_VALIDATE_OR_GOTO (this->name,
-                             (fop_path == GFDB_FOP_UNDEL || GFDB_FOP_UNDEL_ALL),
-                             out);
-
-        /* Set gfdb_db_record to 0 */
-        memset (&gfdb_db_record, 0, sizeof(gfdb_db_record));
-
-        /* Copy gfid into db record */
-        gf_uuid_copy (gfdb_db_record.gfid, gfid);
-
-        /* Copy pargid into db record */
-        gf_uuid_copy (gfdb_db_record.pargfid, pargfid);
-
-        /* Copy basename */
-        strncpy (gfdb_db_record.file_name, basename, GF_NAME_MAX - 1);
-
-        gfdb_db_record.gfdb_fop_path = fop_path;
-        gfdb_db_record.gfdb_fop_type = fop_type;
-
-        /*send delete request to db*/
-        ret = insert_record (_priv->_db_conn, &gfdb_db_record);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RECORD_WIND_FAILED,
-                        "Failed to delete record. %s", basename);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/******************************* Hard link function ***************************/
-
-static inline gf_boolean_t
-__is_inode_expired (ctr_xlator_ctx_t *ctr_xlator_ctx,
-                    gf_ctr_private_t *_priv,
-                    gfdb_time_t *current_time)
-{
-        gf_boolean_t    ret       = _gf_false;
-        uint64_t        time_diff = 0;
-
-        GF_ASSERT (ctr_xlator_ctx);
-        GF_ASSERT (_priv);
-        GF_ASSERT (current_time);
-
-        time_diff = current_time->tv_sec -
-                        ctr_xlator_ctx->inode_heal_period;
-
-        ret = (time_diff >= _priv->ctr_lookupheal_inode_timeout) ?
-                        _gf_true : _gf_false;
-        return ret;
-}
-
-static inline gf_boolean_t
-__is_hardlink_expired (ctr_hard_link_t *ctr_hard_link,
-                       gf_ctr_private_t *_priv,
-                       gfdb_time_t *current_time)
-{
-        gf_boolean_t    ret       = _gf_false;
-        uint64_t        time_diff = 0;
-
-        GF_ASSERT (ctr_hard_link);
-        GF_ASSERT (_priv);
-        GF_ASSERT (current_time);
-
-        time_diff = current_time->tv_sec -
-                        ctr_hard_link->hardlink_heal_period;
-
-        ret = ret || (time_diff >= _priv->ctr_lookupheal_link_timeout) ?
-                        _gf_true : _gf_false;
-
-        return ret;
-}
-
-
-/* Return values of heal*/
-typedef enum ctr_heal_ret_val {
-        CTR_CTX_ERROR = -1,
-        /* No healing required */
-        CTR_TRY_NO_HEAL = 0,
-        /* Try healing hard link */
-        CTR_TRY_HARDLINK_HEAL = 1,
-        /* Try healing inode */
-        CTR_TRY_INODE_HEAL = 2,
-} ctr_heal_ret_val_t;
-
-
-
-/**
- * @brief Function to add hard link to the inode context variable.
- *        The inode context maintainences a in-memory list. This is used
- *        smart healing of database.
- * @param frame of the FOP
- * @param this is the Xlator instant
- * @param inode
- * @return Return ctr_heal_ret_val_t
- */
-
-static inline ctr_heal_ret_val_t
-add_hard_link_ctx (call_frame_t *frame,
-                   xlator_t     *this,
-                   inode_t      *inode)
-{
-        ctr_heal_ret_val_t ret_val = CTR_TRY_NO_HEAL;
-        int ret = -1;
-        gf_ctr_local_t   *ctr_local       = NULL;
-        ctr_xlator_ctx_t *ctr_xlator_ctx  = NULL;
-        ctr_hard_link_t  *ctr_hard_link   = NULL;
-        gf_ctr_private_t *_priv           = NULL;
-        gfdb_time_t       current_time    = {0};
-
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-        GF_ASSERT (this->private);
-
-        _priv = this->private;
-
-        ctr_local = frame->local;
-        if (!ctr_local) {
-                goto out;
-        }
-
-        ctr_xlator_ctx  = init_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ACCESS_CTR_INODE_CONTEXT_FAILED,
-                        "Failed accessing ctr inode context");
-                goto out;
-        }
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        /* Check if the hard link already exists
-         * in the ctr inode context*/
-        ctr_hard_link = ctr_search_hard_link_ctx (this,
-                                ctr_xlator_ctx,
-                                CTR_DB_REC(ctr_local).pargfid,
-                                CTR_DB_REC(ctr_local).file_name);
-        /* if there then ignore */
-        if (ctr_hard_link) {
-
-                ret = gettimeofday (&current_time, NULL);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get current time");
-                        ret_val = CTR_CTX_ERROR;
-                        goto unlock;
-                }
-
-                if (__is_hardlink_expired (ctr_hard_link,
-                                           _priv, &current_time)) {
-                        ctr_hard_link->hardlink_heal_period =
-                                        current_time.tv_sec;
-                        ret_val = ret_val | CTR_TRY_HARDLINK_HEAL;
-                }
-
-                if (__is_inode_expired (ctr_xlator_ctx,
-                                           _priv, &current_time)) {
-                        ctr_xlator_ctx->inode_heal_period =
-                                                current_time.tv_sec;
-                        ret_val = ret_val | CTR_TRY_INODE_HEAL;
-                }
-
-                goto unlock;
-        }
-
-        /* Add the hard link to the list*/
-        ret = ctr_add_hard_link (this, ctr_xlator_ctx,
-                        CTR_DB_REC(ctr_local).pargfid,
-                        CTR_DB_REC(ctr_local).file_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ADD_HARDLINK_TO_CTR_INODE_CONTEXT_FAILED,
-                        "Failed to add hardlink to the ctr inode context");
-                ret_val = CTR_CTX_ERROR;
-                goto unlock;
-        }
-
-        ret_val = CTR_TRY_NO_HEAL;
-unlock:
-        UNLOCK (&ctr_xlator_ctx->lock);
-out:
-        return ret_val;
-}
-
-static inline int
-delete_hard_link_ctx (call_frame_t *frame,
-                      xlator_t     *this,
-                      inode_t      *inode)
-{
-        int ret = -1;
-        ctr_xlator_ctx_t *ctr_xlator_ctx  = NULL;
-        gf_ctr_local_t   *ctr_local       = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        ctr_local = frame->local;
-        if (!ctr_local) {
-                goto out;
-        }
-
-        ctr_xlator_ctx = get_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                /* Since there is no ctr inode context so nothing more to do */
-                ret = 0;
-                goto out;
-        }
-
-        ret = ctr_delete_hard_link (this, ctr_xlator_ctx,
-                        CTR_DB_REC(ctr_local).pargfid,
-                        CTR_DB_REC(ctr_local).file_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_DELETE_HARDLINK_FAILED,
-                        "Failed to delete hard link");
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static inline int
-update_hard_link_ctx (call_frame_t *frame,
-                      xlator_t     *this,
-                      inode_t      *inode)
-{
-        int ret = -1;
-        ctr_xlator_ctx_t *ctr_xlator_ctx  = NULL;
-        gf_ctr_local_t   *ctr_local       = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        ctr_local = frame->local;
-        if (!ctr_local) {
-                goto out;
-        }
-
-        ctr_xlator_ctx  = init_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ACCESS_CTR_INODE_CONTEXT_FAILED,
-                        "Failed accessing ctr inode context");
-                goto out;
-        }
-
-        ret = ctr_update_hard_link (this, ctr_xlator_ctx,
-                        CTR_DB_REC(ctr_local).pargfid,
-                        CTR_DB_REC(ctr_local).file_name,
-                        CTR_DB_REC(ctr_local).old_pargfid,
-                        CTR_DB_REC(ctr_local).old_file_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_DELETE_HARDLINK_FAILED,
-                        "Failed to delete hard link");
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
-/******************************************************************************
- *
- *                      CTR xlator init related functions
- *
- *
- * ****************************************************************************/
-int
-extract_db_params (xlator_t              *this,
-                  dict_t                *params_dict,
-                  gfdb_db_type_t        db_type);
-
-int
-extract_ctr_options (xlator_t            *this,
-                    gf_ctr_private_t    *_priv);
-
-#endif
diff --git a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.c b/xlators/features/changetimerecorder/src/ctr-xlator-ctx.c
deleted file mode 100644
index 7700ad40ba6..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.c
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "ctr-xlator-ctx.h"
-#include "ctr-messages.h"
-#include <time.h>
-#include <sys/time.h>
-
-#define IS_THE_ONLY_HARDLINK(ctr_hard_link)\
-        (ctr_hard_link->list.next == ctr_hard_link->list.prev)
-
-
-static void
-fini_ctr_hard_link (ctr_hard_link_t **ctr_hard_link) {
-
-        GF_ASSERT (ctr_hard_link);
-
-        if (*ctr_hard_link)
-                return;
-        GF_FREE ((*ctr_hard_link)->base_name);
-        GF_FREE (*ctr_hard_link);
-        *ctr_hard_link = NULL;
-}
-
-
-/* Please lock the ctr_xlator_ctx before using this function */
-ctr_hard_link_t *
-ctr_search_hard_link_ctx (xlator_t                  *this,
-                            ctr_xlator_ctx_t        *ctr_xlator_ctx,
-                            uuid_t                  pgfid,
-                            const char              *base_name)
-{
-        ctr_hard_link_t *_hard_link             = NULL;
-        ctr_hard_link_t *searched_hardlink      = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-        if (pgfid == NULL || base_name == NULL)
-                goto out;
-
-        /*linear search*/
-        list_for_each_entry (_hard_link, &ctr_xlator_ctx->hardlink_list, list) {
-                if (gf_uuid_compare (_hard_link->pgfid, pgfid) == 0
-                        && _hard_link->base_name
-                        && strcmp(_hard_link->base_name, base_name) == 0) {
-                        searched_hardlink = _hard_link;
-                        break;
-                }
-        }
-
-out:
-        return searched_hardlink;
-}
-
-
-
-
-/* Please lock the ctr_xlator_ctx before using this function */
-int
-ctr_add_hard_link (xlator_t             *this,
-               ctr_xlator_ctx_t         *ctr_xlator_ctx,
-               uuid_t                   pgfid,
-               const char               *base_name)
-{
-        int ret                                 = -1;
-        ctr_hard_link_t *ctr_hard_link          = NULL;
-        struct timeval  current_time                   = {0};
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-        if (pgfid == NULL || base_name == NULL)
-                goto out;
-
-        ctr_hard_link = GF_CALLOC (1, sizeof (*ctr_hard_link),
-                                        gf_ctr_mt_hard_link_t);
-        if (!ctr_hard_link) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CTR_MSG_CALLOC_FAILED, "Failed allocating "
-                        "ctr_hard_link");
-                goto out;
-        }
-
-        /*Initialize the ctr_hard_link object and
-         * Assign the values : parent GFID and basename*/
-        INIT_LIST_HEAD (&ctr_hard_link->list);
-        gf_uuid_copy (ctr_hard_link->pgfid, pgfid);
-        ret = gf_asprintf(&ctr_hard_link->base_name, "%s", base_name);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_COPY_FAILED, "Failed copying basename"
-                        "to ctr_hard_link");
-                goto error;
-        }
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get current time");
-                goto error;
-        }
-
-        /*Add the hard link to the list*/
-        list_add_tail (&ctr_hard_link->list,
-                        &ctr_xlator_ctx->hardlink_list);
-
-        ctr_hard_link->hardlink_heal_period = current_time.tv_sec;
-
-        /*aal izz well!*/
-        ret = 0;
-        goto out;
-error:
-        GF_FREE (ctr_hard_link);
-out:
-        return ret;
-}
-
-static void
-__delete_hard_link_from_list (ctr_hard_link_t  **ctr_hard_link)
-{
-        GF_ASSERT (ctr_hard_link);
-        GF_ASSERT (*ctr_hard_link);
-
-        /*Remove hard link from list*/
-        list_del(&(*ctr_hard_link)->list);
-        fini_ctr_hard_link (ctr_hard_link);
-}
-
-
-int
-ctr_delete_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name)
-{
-        int ret                         = -1;
-        ctr_hard_link_t *ctr_hard_link     = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        /*Check if the hard link is present */
-        ctr_hard_link = ctr_search_hard_link_ctx (this, ctr_xlator_ctx,
-                                                        pgfid, base_name);
-        if (!ctr_hard_link) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_HARDLINK_MISSING_IN_LIST,
-                        "Hard link doesn't exist in the list");
-                goto out;
-        }
-
-        __delete_hard_link_from_list (&ctr_hard_link);
-        ctr_hard_link = NULL;
-
-        ret = 0;
-out:
-        UNLOCK (&ctr_xlator_ctx->lock);
-
-        return ret;
-}
-
-
-
-
-int
-ctr_update_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name,
-                  uuid_t                old_pgfid,
-                  const char            *old_base_name)
-{
-        int ret                            = -1;
-        ctr_hard_link_t *ctr_hard_link     = NULL;
-        struct timeval current_time        = {0};
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        /*Check if the hard link is present */
-        ctr_hard_link = ctr_search_hard_link_ctx (this, ctr_xlator_ctx,
-                                                old_pgfid, old_base_name);
-        if (!ctr_hard_link) {
-                gf_msg_trace (this->name, 0, "Hard link doesn't exist"
-                              " in the list");
-                /* Since the hard link is not present in the list
-                 * we add it to the list */
-                ret = ctr_add_hard_link (this, ctr_xlator_ctx,
-                                        pgfid, base_name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_ADD_HARDLINK_TO_LIST_FAILED,
-                                "Failed adding hard link to the list");
-                        goto out;
-                }
-                ret = 0;
-                goto out;
-        }
-
-        /* update the hard link */
-        gf_uuid_copy (ctr_hard_link->pgfid, pgfid);
-        GF_FREE (ctr_hard_link->base_name);
-        ret = gf_asprintf(&ctr_hard_link->base_name, "%s", base_name);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_COPY_FAILED, "Failed copying basename"
-                                        "to ctr_hard_link");
-                /* delete the corrupted entry */
-                __delete_hard_link_from_list (&ctr_hard_link);
-                ctr_hard_link = NULL;
-                goto out;
-        }
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get current time");
-                ctr_hard_link->hardlink_heal_period = 0;
-        } else {
-                ctr_hard_link->hardlink_heal_period = current_time.tv_sec;
-        }
-
-        ret = 0;
-
-out:
-        UNLOCK (&ctr_xlator_ctx->lock);
-
-        return ret;
-}
-
-
-
-
-/* Delete all hardlinks */
-static int
-ctr_delete_all_hard_link (xlator_t *this,
-                          ctr_xlator_ctx_t *ctr_xlator_ctx)
-{
-        int ret = -1;
-        ctr_hard_link_t *ctr_hard_link     = NULL;
-        ctr_hard_link_t *tmp               = NULL;
-
-        GF_ASSERT (ctr_xlator_ctx);
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        list_for_each_entry_safe(ctr_hard_link, tmp,
-                        &ctr_xlator_ctx->hardlink_list, list)
-        {
-                /*Remove hard link from list*/
-                __delete_hard_link_from_list (&ctr_hard_link);
-                ctr_hard_link = NULL;
-
-        }
-
-
-        UNLOCK (&ctr_xlator_ctx->lock);
-
-        ret = 0;
-
-        return ret;
-}
-
-
-/* Please lock the inode before using this function */
-static ctr_xlator_ctx_t *
-__get_ctr_xlator_ctx (xlator_t  *this,
-                      inode_t   *inode)
-{
-        int ret                                 = 0;
-        uint64_t _addr                          = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        ret = __inode_ctx_get (inode, this, &_addr);
-        if (ret < 0)
-                _addr = 0;
-        if (_addr != 0) {
-                ctr_xlator_ctx = (ctr_xlator_ctx_t *) (long)_addr;
-        }
-
-        return ctr_xlator_ctx;
-}
-
-
-ctr_xlator_ctx_t *
-init_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode)
-{
-        int ret                                 = -1;
-        uint64_t _addr                          = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-        struct timeval current_time             = {0};
-
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        LOCK (&inode->lock);
-        {
-                ctr_xlator_ctx = __get_ctr_xlator_ctx (this, inode);
-                if (ctr_xlator_ctx) {
-                        ret = 0;
-                        goto out;
-                }
-                ctr_xlator_ctx = GF_CALLOC (1, sizeof (*ctr_xlator_ctx),
-                                                        gf_ctr_mt_xlator_ctx);
-                if (!ctr_xlator_ctx)
-                        goto out;
-
-                ret = LOCK_INIT (&ctr_xlator_ctx->lock);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ret,
-                                CTR_MSG_INIT_LOCK_FAILED,
-                                "Failed init lock %s", strerror(ret));
-                        goto out;
-                }
-                _addr = (uint64_t) ctr_xlator_ctx;
-
-                ret = __inode_ctx_set (inode, this, &_addr);
-                if (ret) {
-                        goto out;
-                }
-
-                INIT_LIST_HEAD (&ctr_xlator_ctx->hardlink_list);
-
-                ret = gettimeofday (&current_time, NULL);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get current time");
-                        goto out;
-                }
-
-                ctr_xlator_ctx->inode_heal_period = current_time.tv_sec;
-        }
-        ret = 0;
-out:
-        if (ret) {
-                GF_FREE (ctr_xlator_ctx);
-                ctr_xlator_ctx = NULL;
-        }
-
-        UNLOCK (&inode->lock);
-
-        return ctr_xlator_ctx;
-}
-
-
-
-
-void
-fini_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode)
-{
-        int ret                                 = 0;
-        uint64_t _addr                          = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-
-
-        inode_ctx_del (inode, this, &_addr);
-        if (!_addr)
-                return;
-
-        ctr_xlator_ctx = (ctr_xlator_ctx_t *) (long) _addr;
-
-        ret = ctr_delete_all_hard_link (this, ctr_xlator_ctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING , 0,
-                        CTR_MSG_DELETE_HARDLINK_FAILED, "Failed deleting all "
-                        "hard links from inode context");
-        }
-
-        LOCK_DESTROY (&ctr_xlator_ctx->lock);
-
-        GF_FREE (ctr_xlator_ctx);
-
-}
-
-
-
-
-ctr_xlator_ctx_t *
-get_ctr_xlator_ctx (xlator_t *this,
-                    inode_t *inode)
-{
-        ctr_xlator_ctx_t *ctr_xlator_ctx = NULL;
-
-        LOCK (&inode->lock);
-        ctr_xlator_ctx = __get_ctr_xlator_ctx (this, inode);
-        UNLOCK (&inode->lock);
-
-        return ctr_xlator_ctx;
-}
-
diff --git a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.h b/xlators/features/changetimerecorder/src/ctr-xlator-ctx.h
deleted file mode 100644
index 7f1c6cb1712..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_XLATOR_CTX_H
-#define __CTR_XLATOR_CTX_H
-
-#include "xlator.h"
-#include "ctr_mem_types.h"
-#include "iatt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "locking.h"
-#include "common-utils.h"
-#include <time.h>
-#include <sys/time.h>
-
-typedef struct ctr_hard_link {
-        uuid_t                  pgfid;
-        char                    *base_name;
-        /* Hardlink expiry : Defines the expiry period after which a
-         * database heal is attempted. */
-        uint64_t                  hardlink_heal_period;
-        struct list_head        list;
-} ctr_hard_link_t;
-
-typedef struct ctr_xlator_ctx {
-        /* This represents the looked up hardlinks
-         * NOTE: This doesn't represent all physical hardlinks of the inode*/
-        struct list_head        hardlink_list;
-        uint64_t                inode_heal_period;
-        gf_lock_t               lock;
-} ctr_xlator_ctx_t;
-
-
-ctr_hard_link_t *
-ctr_search_hard_link_ctx (xlator_t                  *this,
-                          ctr_xlator_ctx_t        *ctr_xlator_ctx,
-                          uuid_t                  pgfid,
-                          const char              *base_name);
-
-
-int
-ctr_add_hard_link (xlator_t          *this,
-               ctr_xlator_ctx_t         *ctr_xlator_ctx,
-               uuid_t                   pgfid,
-               const char               *base_name);
-
-
-
-int
-ctr_delete_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name);
-
-
-int
-ctr_update_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name,
-                  uuid_t                old_pgfid,
-                  const char            *old_base_name);
-
-
-ctr_xlator_ctx_t *
-get_ctr_xlator_ctx (xlator_t *this,
-                    inode_t *inode);
-
-
-
-
-ctr_xlator_ctx_t *
-init_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode);
-
-
-void
-fini_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode);
-
-#endif
diff --git a/xlators/experimental/dht2/dht2-server/Makefile.am b/xlators/features/cloudsync/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/dht2/dht2-server/Makefile.am
+++ b/xlators/features/cloudsync/Makefile.am
diff --git a/xlators/features/cloudsync/src/Makefile.am b/xlators/features/cloudsync/src/Makefile.am
new file mode 100644
index 00000000000..e2a277e372b
--- /dev/null
+++ b/xlators/features/cloudsync/src/Makefile.am
@@ -0,0 +1,46 @@
+SUBDIRS = cloudsync-plugins
+
+xlator_LTLIBRARIES = cloudsync.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+cloudsync_sources = cloudsync.c
+
+CLOUDSYNC_SRC = $(top_srcdir)/xlators/features/cloudsync/src
+CLOUDSYNC_BLD = $(top_builddir)/xlators/features/cloudsync/src
+
+cloudsynccommon_sources = $(CLOUDSYNC_SRC)/cloudsync-common.c
+
+noinst_HEADERS = $(CLOUDSYNC_BLD)/cloudsync.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-mem-types.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-messages.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-common.h
+
+cloudsync_la_SOURCES = $(cloudsync_sources) $(cloudsynccommon_sources)
+
+nodist_cloudsync_la_SOURCES = cloudsync-autogen-fops.c cloudsync-autogen-fops.h
+BUILT_SOURCES = cloudsync-autogen-fops.h
+
+cloudsync_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+cloudsync_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(LIB_DL)
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	      -DCS_PLUGINDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins\"
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+
+noinst_PYTHON = cloudsync-fops-c.py cloudsync-fops-h.py
+EXTRA_DIST = cloudsync-autogen-fops-tmpl.c cloudsync-autogen-fops-tmpl.h
+
+cloudsync-autogen-fops.c: cloudsync-fops-c.py cloudsync-autogen-fops-tmpl.c
+	$(PYTHON) $(CLOUDSYNC_SRC)/cloudsync-fops-c.py \
+	$(CLOUDSYNC_SRC)/cloudsync-autogen-fops-tmpl.c > $@
+
+cloudsync-autogen-fops.h: cloudsync-fops-h.py cloudsync-autogen-fops-tmpl.h
+	$(PYTHON) $(CLOUDSYNC_SRC)/cloudsync-fops-h.py \
+	$(CLOUDSYNC_SRC)/cloudsync-autogen-fops-tmpl.h > $@
+
+CLEANFILES = $(nodist_cloudsync_la_SOURCES)
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/cloudsync.so
diff --git a/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c
new file mode 100644
index 00000000000..ee63f983980
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c
@@ -0,0 +1,30 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* File: cloudsync-autogen-fops-tmpl.c
+ * This file contains the CLOUDSYNC autogenerated FOPs. This is run through
+ * the code generator, generator.py to generate the required FOPs.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <dlfcn.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include "cloudsync.h"
+#include "cloudsync-common.h"
+#include <glusterfs/call-stub.h>
+
+#pragma generate
diff --git a/xlators/features/changetimerecorder/src/ctr_mem_types.h b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h
index f408c028e24..d922c77d8aa 100644
--- a/xlators/features/changetimerecorder/src/ctr_mem_types.h
+++ b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h
@@ -8,17 +8,17 @@
   cases as published by the Free Software Foundation.
 */
 
+/* File: clousync-autogen-fops-tmpl.h
+ * This file contains the cloudsync autogenerated FOPs declarations.
+ */
 
-#ifndef __CTR_MEM_TYPES_H__
-#define __CTR_MEM_TYPES_H__
+#ifndef _CLOUDSYNC_AUTOGEN_FOPS_H
+#define _CLOUDSYNC_AUTOGEN_FOPS_H
 
-#include "gfdb_mem-types.h"
+#include <glusterfs/xlator.h>
+#include "cloudsync.h"
+#include "cloudsync-common.h"
 
-enum gf_ctr_mem_types_ {
-        gf_ctr_mt_private_t = gfdb_mt_end + 1,
-        gf_ctr_mt_xlator_ctx,
-        gf_ctr_mt_hard_link_t,
-        gf_ctr_mt_end
-};
-#endif
+#pragma generate
 
+#endif /* _CLOUDSYNC_AUTOGEN_FOPS_H */
diff --git a/xlators/features/cloudsync/src/cloudsync-common.c b/xlators/features/cloudsync/src/cloudsync-common.c
new file mode 100644
index 00000000000..445a31b90e7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-common.c
@@ -0,0 +1,60 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "cloudsync-common.h"
+
+void
+cs_xattrinfo_wipe(cs_local_t *local)
+{
+    if (local->xattrinfo.lxattr) {
+        if (local->xattrinfo.lxattr->file_path)
+            GF_FREE(local->xattrinfo.lxattr->file_path);
+
+        if (local->xattrinfo.lxattr->volname)
+            GF_FREE(local->xattrinfo.lxattr->volname);
+
+        GF_FREE(local->xattrinfo.lxattr);
+    }
+}
+
+void
+cs_local_wipe(xlator_t *this, cs_local_t *local)
+{
+    if (!local)
+        return;
+
+    loc_wipe(&local->loc);
+
+    if (local->fd) {
+        fd_unref(local->fd);
+        local->fd = NULL;
+    }
+
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
+
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
+
+    if (local->dlfd)
+        fd_unref(local->dlfd);
+
+    if (local->remotepath)
+        GF_FREE(local->remotepath);
+
+    cs_xattrinfo_wipe(local);
+
+    mem_put(local);
+}
diff --git a/xlators/features/cloudsync/src/cloudsync-common.h b/xlators/features/cloudsync/src/cloudsync-common.h
new file mode 100644
index 00000000000..11d233460a4
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-common.h
@@ -0,0 +1,134 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _CLOUDSYNC_COMMON_H
+#define _CLOUDSYNC_COMMON_H
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/compat-errno.h>
+#include "cloudsync-mem-types.h"
+#include "cloudsync-messages.h"
+
+typedef struct cs_loc_xattr {
+    char *file_path;
+    uuid_t uuid;
+    uuid_t gfid;
+    char *volname;
+} cs_loc_xattr_t;
+
+typedef struct cs_size_xattr {
+    uint64_t size;
+    uint64_t blksize;
+    uint64_t blocks;
+} cs_size_xattr_t;
+
+typedef struct cs_local {
+    loc_t loc;
+    fd_t *fd;
+    call_stub_t *stub;
+    call_frame_t *main_frame;
+    int op_errno;
+    int op_ret;
+    fd_t *dlfd;
+    off_t dloffset;
+    struct iatt stbuf;
+    dict_t *xattr_rsp;
+    dict_t *xattr_req;
+    glusterfs_fop_t fop;
+    gf_boolean_t locked;
+    int call_cnt;
+    inode_t *inode;
+    char *remotepath;
+
+    struct {
+        /* offset, flags and size are the information needed
+         * by read fop for remote read operation. These will be
+         * populated in cloudsync read fop, before being passed
+         * on to the plugin performing remote read.
+         */
+        off_t offset;
+        uint32_t flags;
+        size_t size;
+        cs_loc_xattr_t *lxattr;
+    } xattrinfo;
+
+} cs_local_t;
+
+typedef int (*fop_download_t)(call_frame_t *frame, void *config);
+
+typedef int (*fop_remote_read_t)(call_frame_t *, void *);
+
+typedef void *(*store_init)(xlator_t *this);
+
+typedef int (*store_reconfigure)(xlator_t *this, dict_t *options);
+
+typedef void (*store_fini)(void *config);
+
+struct cs_remote_stores {
+    char *name;                    /* store name */
+    void *config;                  /* store related information */
+    fop_download_t dlfop;          /* store specific download function */
+    fop_remote_read_t rdfop;       /* store specific read function */
+    store_init init;               /* store init to initialize store config */
+    store_reconfigure reconfigure; /* reconfigure store config */
+    store_fini fini;
+    void *handle; /* shared library handle*/
+};
+
+typedef struct cs_private {
+    xlator_t *this;
+    struct cs_remote_stores *stores;
+    gf_boolean_t abortdl;
+    pthread_spinlock_t lock;
+    gf_boolean_t remote_read;
+} cs_private_t;
+
+void
+cs_local_wipe(xlator_t *this, cs_local_t *local);
+
+void
+cs_xattrinfo_wipe(cs_local_t *local);
+
+#define CS_STACK_UNWIND(fop, frame, params...)                                 \
+    do {                                                                       \
+        cs_local_t *__local = NULL;                                            \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        cs_local_wipe(__xl, __local);                                          \
+    } while (0)
+
+#define CS_STACK_DESTROY(frame)                                                \
+    do {                                                                       \
+        cs_local_t *__local = NULL;                                            \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        cs_local_wipe(__xl, __local);                                          \
+    } while (0)
+
+typedef struct store_methods {
+    int (*fop_download)(call_frame_t *frame, void *config);
+    int (*fop_remote_read)(call_frame_t *, void *);
+    /* return type should be the store config */
+    void *(*fop_init)(xlator_t *this);
+    int (*fop_reconfigure)(xlator_t *this, dict_t *options);
+    void (*fop_fini)(void *config);
+} store_methods_t;
+
+#endif /* _CLOUDSYNC_COMMON_H */
diff --git a/xlators/features/cloudsync/src/cloudsync-fops-c.py b/xlators/features/cloudsync/src/cloudsync-fops-c.py
new file mode 100755
index 00000000000..c27df97ae58
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-fops-c.py
@@ -0,0 +1,324 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, cbk_subs, generate
+
+FD_DATA_MODIFYING_OP_FOP_TEMPLATE = """
+int32_t
+cs_@NAME@ (call_frame_t *frame, xlator_t *this,
+           @LONG_ARGS@)
+{
+        int                         op_errno        = EINVAL ;
+        cs_local_t                 *local           = NULL;
+        int                         ret             = 0;
+        cs_inode_ctx_t             *ctx             = NULL;
+        gf_cs_obj_state             state           = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = cs_local_init (this, frame, NULL, fd, GF_FOP_@UPNAME@);
+        if (!local) {
+
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        __cs_inode_ctx_get (this, fd->inode, &ctx);
+
+        if (ctx)
+                state = __cs_get_file_state (fd->inode, ctx);
+        else
+                state = GF_CS_LOCAL;
+
+        xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+        if (!xdata) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->xattr_req = xdata;
+
+        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
+                        " %s", GF_CS_OBJECT_STATUS);
+                goto err;
+        }
+
+        local->stub = fop_@NAME@_stub (frame, cs_resume_@NAME@,
+                                       @SHORT_ARGS@);
+        if (!local->stub) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+
+        if (state == GF_CS_LOCAL) {
+                STACK_WIND (frame, cs_@NAME@_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
+                            @SHORT_ARGS@);
+        } else {
+                local->call_cnt++;
+                ret = locate_and_execute (frame);
+                if (ret) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+        }
+
+        return 0;
+
+err:
+        CS_STACK_UNWIND (@NAME@, frame, -1, op_errno, @CBK_ERROR_ARGS@);
+
+        return 0;
+}
+"""
+
+FD_DATA_MODIFYING_RESUME_OP_FOP_TEMPLATE = """
+int32_t
+cs_resume_@NAME@ (call_frame_t *frame, xlator_t *this,
+                  @LONG_ARGS@)
+{
+        int              ret    = 0;
+
+        ret = cs_resume_postprocess (this, frame, fd->inode);
+        if (ret) {
+                goto unwind;
+        }
+
+        cs_inodelk_unlock (frame);
+
+        STACK_WIND (frame, cs_@NAME@_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
+                    @SHORT_ARGS@);
+
+        return 0;
+
+unwind:
+
+        cs_inodelk_unlock (frame);
+
+        cs_common_cbk (frame);
+
+        return 0;
+}
+"""
+FD_DATA_MODIFYING_OP_FOP_CBK_TEMPLATE = """
+int32_t
+cs_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               @LONG_ARGS@)
+{
+        cs_local_t      *local = NULL;
+        int              ret = 0;
+        uint64_t         val = 0;
+        fd_t            *fd = NULL;
+
+        local = frame->local;
+        fd = local->fd;
+
+        /* Do we need lock here? */
+        local->call_cnt++;
+
+        if (op_ret == -1) {
+                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
+                if (ret == 0) {
+                        if (val == GF_CS_ERROR) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+                                        "could not get file state, unwinding");
+                                op_ret = -1;
+                                op_errno = EIO;
+                                goto unwind;
+                        } else {
+                                __cs_inode_ctx_update (this, fd->inode, val);
+                                gf_msg (this->name, GF_LOG_INFO, 0, 0,
+                                        " state = %" PRIu64, val);
+
+                                if (local->call_cnt == 1 &&
+                                    (val == GF_CS_REMOTE ||
+                                     val == GF_CS_DOWNLOADING))  {
+                                        gf_msg (this->name, GF_LOG_INFO, 0,
+                                                0, " will repair and download "
+                                                "the file, current state : %"
+                                                PRIu64, val);
+                                        goto repair;
+                                } else {
+                                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+                                                "second @NAME@, Unwinding");
+                                        goto unwind;
+                                }
+                        }
+                } else {
+                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "file state "
+                                "could not be figured, unwinding");
+                        goto unwind;
+                }
+        } else {
+                /* successful @NAME@ => file is local */
+                __cs_inode_ctx_update (this, fd->inode, GF_CS_LOCAL);
+                gf_msg (this->name, GF_LOG_INFO, 0, 0, "state : GF_CS_LOCAL"
+                        ", @NAME@ successful");
+
+                goto unwind;
+        }
+
+repair:
+        ret = locate_and_execute (frame);
+        if (ret) {
+                goto unwind;
+        }
+
+        return 0;
+
+unwind:
+        CS_STACK_UNWIND (@NAME@, frame, op_ret, op_errno, @SHORT_ARGS@);
+
+        return 0;
+}
+"""
+
+LOC_STAT_OP_FOP_TEMPLATE = """
+int32_t
+cs_@NAME@ (call_frame_t *frame, xlator_t *this,
+           @LONG_ARGS@)
+{
+        int              op_errno = EINVAL;
+        cs_local_t      *local = NULL;
+        int              ret   = 0;
+
+        local = cs_local_init (this, frame, loc, NULL, GF_FOP_@UPNAME@);
+        if (!local) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local is NULL");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        if (loc->inode->ia_type == IA_IFDIR)
+                goto wind;
+
+        xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+        if (!xdata) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->xattr_req = xdata;
+
+        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
+                        " %s", GF_CS_OBJECT_STATUS);
+                goto err;
+        }
+
+wind:
+        STACK_WIND (frame, cs_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@,
+                    @SHORT_ARGS@);
+
+        return 0;
+err:
+        CS_STACK_UNWIND (@NAME@, frame, -1, op_errno, @CBK_ERROR_ARGS@);
+
+        return 0;
+}
+"""
+
+LOC_STAT_OP_FOP_CBK_TEMPLATE = """
+int32_t
+cs_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               @LONG_ARGS@)
+{
+        int              ret = 0;
+        uint64_t         val = 0;
+        loc_t           *loc = NULL;
+        cs_local_t      *local = NULL;
+
+        local = frame->local;
+
+        loc = &local->loc;
+
+        if (op_ret == 0) {
+                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
+                if (!ret) {
+                        ret = __cs_inode_ctx_update (this, loc->inode, val);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+                                        "ctx update failed");
+                        }
+                }
+        } else {
+                cs_inode_ctx_reset (this, loc->inode);
+        }
+
+        CS_STACK_UNWIND (@NAME@, frame, op_ret, op_errno, @SHORT_ARGS@);
+
+        return 0;
+}
+"""
+
+# All xlator FOPs are covered in the following section just to create a clarity
+# The lists themselves are not used.
+entry_ops = ['mknod', 'mkdir', 'unlink', 'rmdir', 'symlink', 'rename', 'link',
+             'create']
+special_ops = ['statfs', 'lookup', 'ipc', 'compound', 'icreate', 'namelink']
+ignored_ops = ['getspec']
+inode_ops = ['stat', 'readlink', 'truncate', 'open', 'setxattr', 'getxattr',
+             'removexattr', 'opendir', 'access', 'inodelk', 'entrylk',
+             'xattrop', 'setattr', 'lease', 'getactivelk', 'setactivelk',
+             'discover']
+fd_ops = ['readv', 'writev', 'flush', 'fsync', 'fsyncdir', 'ftruncate',
+          'fstat', 'lk', 'readdir', 'finodelk', 'fentrylk', 'fxattrop',
+          'fsetxattr', 'fgetxattr', 'rchecksum', 'fsetattr', 'readdirp',
+          'fremovexattr', 'fallocate', 'discard', 'zerofill', 'seek']
+
+
+# These are the current actual lists used to generate the code
+
+# The following list contains fops which are fd based that modifies data
+fd_data_modify_op_fop_template = ['writev', 'flush', 'fsync',
+                                  'ftruncate', 'rchecksum', 'fallocate',
+                                  'discard', 'zerofill', 'seek']
+
+# The following list contains fops which are entry based that does not change
+# data
+loc_stat_op_fop_template = ['lookup', 'stat', 'discover', 'access', 'setattr',
+                            'getattr']
+
+# These fops need a separate implementation
+special_fops = ['statfs', 'setxattr', 'unlink', 'getxattr',
+                'truncate', 'fstat', 'readv', 'readdirp']
+
+def gen_defaults():
+    for name in ops:
+        if name in fd_data_modify_op_fop_template:
+            print(generate(FD_DATA_MODIFYING_OP_FOP_CBK_TEMPLATE, name, cbk_subs))
+            print(generate(FD_DATA_MODIFYING_RESUME_OP_FOP_TEMPLATE, name, fop_subs))
+            print(generate(FD_DATA_MODIFYING_OP_FOP_TEMPLATE, name, fop_subs))
+        elif name in loc_stat_op_fop_template:
+            print(generate(LOC_STAT_OP_FOP_CBK_TEMPLATE, name, cbk_subs))
+            print(generate(LOC_STAT_OP_FOP_TEMPLATE, name, fop_subs))
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/cloudsync/src/cloudsync-fops-h.py b/xlators/features/cloudsync/src/cloudsync-fops-h.py
new file mode 100755
index 00000000000..faa2de651a7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-fops-h.py
@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, cbk_subs, generate
+
+OP_FOP_TEMPLATE = """
+int32_t
+cs_@NAME@ (call_frame_t *frame, xlator_t *this,
+                   @LONG_ARGS@);
+"""
+
+def gen_defaults():
+    for name, value in ops.items():
+        if name == 'getspec':
+            continue
+        print(generate(OP_FOP_TEMPLATE, name, fop_subs))
+
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/cloudsync/src/cloudsync-mem-types.h b/xlators/features/cloudsync/src/cloudsync-mem-types.h
new file mode 100644
index 00000000000..220346405d0
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-mem-types.h
@@ -0,0 +1,22 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __CLOUDSYNC_MEM_TYPES_H__
+#define __CLOUDSYNC_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum cs_mem_types_ {
+    gf_cs_mt_cs_private_t = gf_common_mt_end + 1,
+    gf_cs_mt_cs_remote_stores_t,
+    gf_cs_mt_cs_inode_ctx_t,
+    gf_cs_mt_cs_lxattr_t,
+    gf_cs_mt_end
+};
+#endif /* __CLOUDSYNC_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-messages.h b/xlators/features/cloudsync/src/cloudsync-messages.h
new file mode 100644
index 00000000000..fb08f72de7f
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-messages.h
@@ -0,0 +1,16 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __CLOUDSYNC_MESSAGES_H__
+#define __CLOUDSYNC_MESSAGES_H__
+
+/*TODO: define relevant message ids */
+
+#endif /* __CLOUDSYNC_MESSAGES_H__ */
diff --git a/xlators/experimental/fdl/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/fdl/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
new file mode 100644
index 00000000000..fb6b0580c6d
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
@@ -0,0 +1,11 @@
+if BUILD_AMAZONS3_PLUGIN
+  AMAZONS3_DIR = cloudsyncs3
+endif
+
+if BUILD_CVLT_PLUGIN
+  CVLT_DIR = cvlt
+endif
+
+SUBDIRS = ${AMAZONS3_DIR} ${CVLT_DIR}
+
+CLEANFILES =
diff --git a/xlators/experimental/jbr-client/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/jbr-client/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
new file mode 100644
index 00000000000..6509426ef87
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
@@ -0,0 +1,12 @@
+csp_LTLIBRARIES = cloudsyncs3.la
+cspdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins
+
+cloudsyncs3_la_SOURCES = libcloudsyncs3.c  $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-common.c
+cloudsyncs3_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+cloudsyncs3_la_LDFLAGS = -module -export-symbols $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym $(GF_XLATOR_LDFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src -lcurlpp -lcryptopp
+noinst_HEADERS = libcloudsyncs3.h libcloudsyncs3-mem-types.h
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) -lcurl -lcrypto -I$(top_srcdir)/xlators/features/cloudsync/src
+CLEANFILES =
+
+EXTRA_DIST = libcloudsyncs3.sym
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
new file mode 100644
index 00000000000..7ccfcc9f4b6
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
@@ -0,0 +1,19 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __LIBAWS_MEM_TYPES_H__
+#define __LIBAWS_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum libaws_mem_types_ {
+    gf_libaws_mt_aws_private_t = gf_common_mt_end + 1,
+    gf_libaws_mt_end
+};
+#endif /* __CLOUDSYNC_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
new file mode 100644
index 00000000000..23c3599825a
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
@@ -0,0 +1,584 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <stdlib.h>
+#include <openssl/hmac.h>
+#include <openssl/evp.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+#include <openssl/crypto.h>
+#include <curl/curl.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include "libcloudsyncs3.h"
+#include "cloudsync-common.h"
+
+#define RESOURCE_SIZE 4096
+
+store_methods_t store_ops = {
+    .fop_download = aws_download_s3,
+    .fop_init = aws_init,
+    .fop_reconfigure = aws_reconfigure,
+    .fop_fini = aws_fini,
+};
+
+typedef struct aws_private {
+    char *hostname;
+    char *bucketid;
+    char *awssekey;
+    char *awskeyid;
+    gf_boolean_t abortdl;
+    pthread_spinlock_t lock;
+} aws_private_t;
+
+void *
+aws_init(xlator_t *this)
+{
+    aws_private_t *priv = NULL;
+    char *temp_str = NULL;
+    int ret = 0;
+
+    priv = GF_CALLOC(1, sizeof(aws_private_t), gf_libaws_mt_aws_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        return NULL;
+    }
+
+    priv->abortdl = _gf_false;
+
+    pthread_spin_init(&priv->lock, PTHREAD_PROCESS_PRIVATE);
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        if (dict_get_str(this->options, "s3plugin-seckey", &temp_str) == 0) {
+            priv->awssekey = gf_strdup(temp_str);
+            if (!priv->awssekey) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws secret key failed");
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-keyid", &temp_str) == 0) {
+            priv->awskeyid = gf_strdup(temp_str);
+            if (!priv->awskeyid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws key ID failed");
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-bucketid", &temp_str) == 0) {
+            priv->bucketid = gf_strdup(temp_str);
+            if (!priv->bucketid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws bucketid failed");
+
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-hostname", &temp_str) == 0) {
+            priv->hostname = gf_strdup(temp_str);
+            if (!priv->hostname) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws hostname failed");
+
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        gf_msg_debug(this->name, 0,
+                     "stored key: %s id: %s "
+                     "bucketid %s hostname: %s",
+                     priv->awssekey, priv->awskeyid, priv->bucketid,
+                     priv->hostname);
+    }
+unlock:
+    pthread_spin_unlock(&(priv->lock));
+
+    if (ret == -1) {
+        GF_FREE(priv->awskeyid);
+        GF_FREE(priv->awssekey);
+        GF_FREE(priv->bucketid);
+        GF_FREE(priv->hostname);
+        GF_FREE(priv);
+        priv = NULL;
+    }
+
+    return (void *)priv;
+}
+
+int
+aws_reconfigure(xlator_t *this, dict_t *options)
+{
+    aws_private_t *priv = NULL;
+    char *temp_str = NULL;
+    int ret = 0;
+    cs_private_t *cspriv = NULL;
+
+    cspriv = this->private;
+
+    priv = cspriv->stores->config;
+
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null priv");
+        return -1;
+    }
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        if (dict_get_str(options, "s3plugin-seckey", &temp_str) == 0) {
+            priv->awssekey = gf_strdup(temp_str);
+            if (!priv->awssekey) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws secret key failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-keyid", &temp_str) == 0) {
+            priv->awskeyid = gf_strdup(temp_str);
+            if (!priv->awskeyid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws key ID failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-bucketid", &temp_str) == 0) {
+            priv->bucketid = gf_strdup(temp_str);
+            if (!priv->bucketid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws bucketid failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-hostname", &temp_str) == 0) {
+            priv->hostname = gf_strdup(temp_str);
+            if (!priv->hostname) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws hostname failed");
+                ret = -1;
+                goto out;
+            }
+        }
+    }
+out:
+    pthread_spin_unlock(&(priv->lock));
+
+    gf_msg_debug(this->name, 0,
+                 "stored key: %s id: %s "
+                 "bucketid %s hostname: %s",
+                 priv->awssekey, priv->awskeyid, priv->bucketid,
+                 priv->hostname);
+
+    return ret;
+}
+
+void
+aws_fini(void *config)
+{
+    aws_private_t *priv = NULL;
+
+    priv = (aws_private_t *)priv;
+
+    if (priv) {
+        GF_FREE(priv->hostname);
+        GF_FREE(priv->bucketid);
+        GF_FREE(priv->awssekey);
+        GF_FREE(priv->awskeyid);
+
+        pthread_spin_destroy(&priv->lock);
+        GF_FREE(priv);
+    }
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_libaws_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+char *
+aws_form_request(char *resource, char **date, char *reqtype, char *bucketid,
+                 char *filepath)
+{
+    char httpdate[256];
+    time_t ctime;
+    struct tm *gtime = NULL;
+    char *sign_req = NULL;
+    int signreq_len = -1;
+    int date_len = -1;
+    int res_len = -1;
+
+    ctime = gf_time();
+    gtime = gmtime(&ctime);
+
+    date_len = strftime(httpdate, sizeof(httpdate),
+                        "%a, %d %b %Y %H:%M:%S +0000", gtime);
+
+    *date = gf_strndup(httpdate, date_len);
+    if (*date == NULL) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "memory allocation "
+               "failure for date");
+        goto out;
+    }
+
+    res_len = snprintf(resource, RESOURCE_SIZE, "%s/%s", bucketid, filepath);
+
+    gf_msg_debug("CS", 0, "resource %s", resource);
+
+    /* 6 accounts for the 4 new line chars, one forward slash and
+     * one null char */
+    signreq_len = res_len + date_len + strlen(reqtype) + 6;
+
+    sign_req = GF_MALLOC(signreq_len, gf_common_mt_char);
+    if (sign_req == NULL) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "memory allocation "
+               "failure for sign_req");
+        goto out;
+    }
+
+    snprintf(sign_req, signreq_len, "%s\n\n%s\n%s\n/%s", reqtype, "", *date,
+             resource);
+
+out:
+    return sign_req;
+}
+
+char *
+aws_b64_encode(const unsigned char *input, int length)
+{
+    BIO *bio, *b64;
+    BUF_MEM *bptr;
+    char *buff = NULL;
+
+    b64 = BIO_new(BIO_f_base64());
+    bio = BIO_new(BIO_s_mem());
+    b64 = BIO_push(b64, bio);
+    BIO_write(b64, input, length);
+    BIO_flush(b64);
+    BIO_get_mem_ptr(b64, &bptr);
+
+    buff = GF_MALLOC(bptr->length, gf_common_mt_char);
+    memcpy(buff, bptr->data, bptr->length - 1);
+    buff[bptr->length - 1] = 0;
+
+    BIO_free_all(b64);
+
+    return buff;
+}
+
+char *
+aws_sign_request(char *const str, char *awssekey)
+{
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX ctx;
+#endif
+    HMAC_CTX *pctx = NULL;
+    ;
+
+    unsigned char md[256];
+    unsigned len;
+    char *base64 = NULL;
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX_init(&ctx);
+    pctx = &ctx;
+#else
+    pctx = HMAC_CTX_new();
+#endif
+    HMAC_Init_ex(pctx, awssekey, strlen(awssekey), EVP_sha1(), NULL);
+    HMAC_Update(pctx, (unsigned char *)str, strlen(str));
+    HMAC_Final(pctx, (unsigned char *)md, &len);
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX_cleanup(pctx);
+#else
+    HMAC_CTX_free(pctx);
+#endif
+    base64 = aws_b64_encode(md, len);
+
+    return base64;
+}
+
+int
+aws_dlwritev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
+{
+    aws_private_t *priv = NULL;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, op_errno,
+               "write failed "
+               ". Aborting Download");
+
+        priv = this->private;
+        pthread_spin_lock(&(priv->lock));
+        {
+            priv->abortdl = _gf_true;
+        }
+        pthread_spin_unlock(&(priv->lock));
+    }
+
+    CS_STACK_DESTROY(frame);
+
+    return op_ret;
+}
+
+size_t
+aws_write_callback(void *dlbuf, size_t size, size_t nitems, void *mainframe)
+{
+    call_frame_t *frame = NULL;
+    fd_t *dlfd = NULL;
+    int ret = 0;
+    cs_local_t *local = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec dliov = {
+        0,
+    };
+    size_t tsize = 0;
+    xlator_t *this = NULL;
+    cs_private_t *xl_priv = NULL;
+    aws_private_t *priv = NULL;
+    call_frame_t *dlframe = NULL;
+
+    frame = (call_frame_t *)mainframe;
+    this = frame->this;
+    xl_priv = this->private;
+    priv = xl_priv->stores->config;
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        /* returning size other than the size passed from curl will
+         * abort further download*/
+        if (priv->abortdl) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "aborting download");
+            pthread_spin_unlock(&(priv->lock));
+            return 0;
+        }
+    }
+    pthread_spin_unlock(&(priv->lock));
+
+    local = frame->local;
+    dlfd = local->dlfd;
+    tsize = size * nitems;
+
+    dliov.iov_base = (void *)dlbuf;
+    dliov.iov_len = tsize;
+
+    ret = iobuf_copy(this->ctx->iobuf_pool, &dliov, 1, &iobref, &iobuf, &iov);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "iobuf_copy failed");
+        goto out;
+    }
+
+    /* copy frame */
+    dlframe = copy_frame(frame);
+    if (!dlframe) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "copy_frame failed");
+        tsize = 0;
+        goto out;
+    }
+
+    STACK_WIND(dlframe, aws_dlwritev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, dlfd, &iov, 1, local->dloffset,
+               0, iobref, NULL);
+
+    local->dloffset += tsize;
+
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    return tsize;
+}
+
+int
+aws_download_s3(call_frame_t *frame, void *config)
+{
+    char *buf;
+    int bufsize = -1;
+    CURL *handle = NULL;
+    struct curl_slist *slist = NULL;
+    struct curl_slist *tmp = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+    int debug = 1;
+    CURLcode res;
+    char errbuf[CURL_ERROR_SIZE];
+    size_t len = 0;
+    long responsecode;
+    char *sign_req = NULL;
+    char *date = NULL;
+    char *const reqtype = "GET";
+    char *signature = NULL;
+    cs_local_t *local = NULL;
+    char resource[RESOURCE_SIZE] = {
+        0,
+    };
+    aws_private_t *priv = NULL;
+
+    this = frame->this;
+
+    local = frame->local;
+
+    priv = (aws_private_t *)config;
+
+    if (!priv->bucketid || !priv->hostname || !priv->awssekey ||
+        !priv->awskeyid) {
+        ret = -1;
+        goto out;
+    }
+
+    sign_req = aws_form_request(resource, &date, reqtype, priv->bucketid,
+                                local->remotepath);
+    if (!sign_req) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "null sign_req, "
+               "aborting download");
+        ret = -1;
+        goto out;
+    }
+
+    gf_msg_debug("CS", 0, "sign_req %s date %s", sign_req, date);
+
+    signature = aws_sign_request(sign_req, priv->awssekey);
+    if (!signature) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0,
+               "null signature, "
+               "aborting download");
+        ret = -1;
+        goto out;
+    }
+
+    handle = curl_easy_init();
+    this = frame->this;
+
+    /* special numbers 6, 20, 10 accounts for static characters in the
+     * below snprintf string format arguments*/
+    bufsize = strlen(date) + 6 + strlen(priv->awskeyid) + strlen(signature) +
+              20 + strlen(priv->hostname) + 10;
+
+    buf = (char *)alloca(bufsize);
+    if (!buf) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "mem allocation "
+               "failed for buf");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(buf, bufsize, "Date: %s", date);
+    slist = curl_slist_append(slist, buf);
+    snprintf(buf, bufsize, "Authorization: AWS %s:%s", priv->awskeyid,
+             signature);
+    slist = curl_slist_append(slist, buf);
+    snprintf(buf, bufsize, "https://%s/%s", priv->hostname, resource);
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG) {
+        tmp = slist;
+        while (tmp) {
+            gf_msg_debug(this->name, 0, "slist for curl - %s", tmp->data);
+            tmp = tmp->next;
+        }
+    }
+
+    curl_easy_setopt(handle, CURLOPT_HTTPHEADER, slist);
+    curl_easy_setopt(handle, CURLOPT_URL, buf);
+    curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, aws_write_callback);
+    curl_easy_setopt(handle, CURLOPT_WRITEDATA, frame);
+    curl_easy_setopt(handle, CURLOPT_VERBOSE, debug);
+    curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, errbuf);
+
+    res = curl_easy_perform(handle);
+    if (res != CURLE_OK) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "download failed. err: %s\n",
+               curl_easy_strerror(res));
+        ret = -1;
+        len = strlen(errbuf);
+        if (len) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "curl failure %s", errbuf);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "curl error "
+                   "%s\n",
+                   curl_easy_strerror(res));
+        }
+    }
+
+    if (res == CURLE_OK) {
+        curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responsecode);
+        gf_msg_debug(this->name, 0, "response code %ld", responsecode);
+        if (responsecode != 200) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "curl download failed");
+        }
+    }
+
+    curl_slist_free_all(slist);
+    curl_easy_cleanup(handle);
+
+out:
+    if (sign_req)
+        GF_FREE(sign_req);
+    if (date)
+        GF_FREE(date);
+    if (signature)
+        GF_FREE(signature);
+
+    return ret;
+}
+
+struct volume_options cs_options[] = {
+    {.key = {"s3plugin-seckey"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws secret key"},
+    {.key = {"s3plugin-keyid"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws key ID"
+
+    },
+    {.key = {"s3plugin-bucketid"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws bucketid"},
+    {.key = {"s3plugin-hostname"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws hostname e.g. s3.amazonaws.com"},
+    {.key = {NULL}},
+};
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
new file mode 100644
index 00000000000..85ae669486b
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
@@ -0,0 +1,50 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBAWS_H
+#define _LIBAWS_H
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
+#include <curl/curl.h>
+#include "cloudsync-common.h"
+#include "libcloudsyncs3-mem-types.h"
+
+char *
+aws_b64_encode(const unsigned char *input, int length);
+
+size_t
+aws_write_callback(void *dlbuf, size_t size, size_t nitems, void *mainframe);
+
+int
+aws_download_s3(call_frame_t *frame, void *config);
+
+int
+aws_dlwritev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
+
+void *
+aws_init(xlator_t *this);
+
+int
+aws_reconfigure(xlator_t *this, dict_t *options);
+
+char *
+aws_form_request(char *resource, char **date, char *reqtype, char *bucketid,
+                 char *filepath);
+char *
+aws_sign_request(char *const str, char *awssekey);
+
+void
+aws_fini(void *config);
+
+#endif
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
new file mode 100644
index 00000000000..0bc273670d5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
@@ -0,0 +1 @@
+store_ops
diff --git a/xlators/experimental/jbr-server/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/jbr-server/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile.am
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am
new file mode 100644
index 00000000000..b512464f157
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am
@@ -0,0 +1,12 @@
+csp_LTLIBRARIES = cloudsynccvlt.la
+cspdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins
+
+cloudsynccvlt_la_SOURCES = libcvlt.c  $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-common.c
+cloudsynccvlt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+cloudsynccvlt_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+noinst_HEADERS = archivestore.h libcvlt.h libcvlt-mem-types.h cvlt-messages.h
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) -I$(top_srcdir)/xlators/features/cloudsync/src
+CLEANFILES =
+
+EXTRA_DIST = libcloudsynccvlt.sym
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h
new file mode 100644
index 00000000000..7230ef77337
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h
@@ -0,0 +1,203 @@
+/*
+  Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __ARCHIVESTORE_H__
+#define __ARCHIVESTORE_H__
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <dlfcn.h>
+#include <uuid/uuid.h>
+
+#define CS_XATTR_ARCHIVE_UUID "trusted.cloudsync.uuid"
+#define CS_XATTR_PRODUCT_ID "trusted.cloudsync.product-id"
+#define CS_XATTR_STORE_ID "trusted.cloudsync.store-id"
+
+struct _archstore_methods;
+typedef struct _archstore_methods archstore_methods_t;
+
+struct _archstore_desc {
+    void *priv; /* Private field for store mgmt.   */
+                /* To be used only by archive store*/
+};
+typedef struct _archstore_desc archstore_desc_t;
+
+struct _archstore_info {
+    char *id;         /* Identifier for the archivestore */
+    uint32_t idlen;   /* Length of identifier string     */
+    char *prod;       /* Name of the data mgmt. product  */
+    uint32_t prodlen; /* Length of the product string    */
+};
+typedef struct _archstore_info archstore_info_t;
+
+struct _archstore_fileinfo {
+    uuid_t uuid;         /* uuid of the file                */
+    char *path;          /* file path                       */
+    uint32_t pathlength; /* length of file path             */
+};
+typedef struct _archstore_fileinfo archstore_fileinfo_t;
+
+struct _app_callback_info {
+    archstore_info_t *src_archstore;
+    archstore_fileinfo_t *src_archfile;
+    archstore_info_t *dest_archstore;
+    archstore_fileinfo_t *dest_archfile;
+};
+typedef struct _app_callback_info app_callback_info_t;
+
+typedef void (*app_callback_t)(archstore_desc_t *, app_callback_info_t *,
+                               void *, int64_t, int32_t);
+
+enum _archstore_scan_type { FULL = 1, INCREMENTAL = 2 };
+typedef enum _archstore_scan_type archstore_scan_type_t;
+
+typedef int32_t archstore_errno_t;
+
+/*
+ * Initialize archive store.
+ * arg1  pointer to structure containing archive store information
+ * arg2  error number if any generated during the initialization
+ * arg3  name of the log file
+ */
+typedef int32_t (*init_archstore_t)(archstore_desc_t *, archstore_errno_t *,
+                                    const char *);
+
+/*
+ * Clean up archive store.
+ * arg1  pointer to structure containing archive store information
+ * arg2  error number if any generated during the cleanup
+ */
+typedef int32_t (*term_archstore_t)(archstore_desc_t *, archstore_errno_t *);
+
+/*
+ * Read the contents of the file from archive store
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  pointer to structure containing information about file to be read
+ * arg4  offset in the file from which data should be read
+ * arg5  buffer where the data should be read
+ * arg6  number of bytes of data to be read
+ * arg7  error number if any generated during the read from file
+ * arg8  callback handler to be invoked after the data is read
+ * arg9  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*read_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                    archstore_fileinfo_t *, off_t, char *,
+                                    size_t, archstore_errno_t *, app_callback_t,
+                                    void *);
+
+/*
+ * Restore the contents of the file from archive store
+ * This is basically in-place restore
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  pointer to structure containing information about file to be restored
+ * arg4  error number if any generated during the file restore
+ * arg5  callback to be invoked after the file is restored
+ * arg6  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*recall_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_errno_t *, app_callback_t,
+                                      void *);
+
+/*
+ * Restore the contents of the file from archive store to a different store
+ * This is basically out-of-place restore
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about file to be restored
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about the location to
+         which the file will be restored
+ * arg6  error number if any generated during the file restore
+ * arg7  callback to be invoked after the file is restored
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*restore_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_errno_t *, app_callback_t,
+                                       void *);
+
+/*
+ * Archive the contents of the file to archive store
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about files to be archived
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about files that failed
+ *       to be archived
+ * arg6  error number if any generated during the file archival
+ * arg7  callback to be invoked after the file is archived
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*archive_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_errno_t *, app_callback_t,
+                                       void *);
+
+/*
+ * Backup list of files provided in the input file
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about files to be backed up
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about files that failed
+ *       to be backed up
+ * arg6  error number if any generated during the file archival
+ * arg7  callback to be invoked after the file is archived
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*backup_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_errno_t *, app_callback_t,
+                                      void *);
+
+/*
+ * Scan the contents of a store and determine the files which need to be
+ * backed up.
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  type of scan whether full or incremental
+ * arg4  path to file that contains list of files to be backed up
+ * arg5  error number if any generated during scan operation
+ */
+typedef int32_t (*scan_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                    archstore_scan_type_t, char *,
+                                    archstore_errno_t *);
+
+struct _archstore_methods {
+    init_archstore_t init;
+    term_archstore_t fini;
+    backup_archstore_t backup;
+    archive_archstore_t archive;
+    scan_archstore_t scan;
+    restore_archstore_t restore;
+    recall_archstore_t recall;
+    read_archstore_t read;
+};
+
+typedef int (*get_archstore_methods_t)(archstore_methods_t *);
+
+/*
+ * Single function that will be invoked by applications for extracting
+ * the function pointers to all data management functions.
+ */
+int32_t
+get_archstore_methods(archstore_methods_t *);
+
+#endif /* End of __ARCHIVESTORE_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h
new file mode 100644
index 00000000000..57c9aa77da0
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h
@@ -0,0 +1,30 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CVLT_MESSAGES_H_
+#define _CVLT_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(CVLT, CVLT_EXTRACTION_FAILED, CVLT_FREE,
+           CVLT_RESOURCE_ALLOCATION_FAILED, CVLT_RESTORE_FAILED,
+           CVLT_READ_FAILED, CVLT_NO_MEMORY, CVLT_DLOPEN_FAILED);
+
+#endif /* !_CVLT_MESSAGES_H_ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
new file mode 100644
index 00000000000..0bc273670d5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
@@ -0,0 +1 @@
+store_ops
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h
new file mode 100644
index 00000000000..c24fab8bfe7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h
@@ -0,0 +1,19 @@
+/*
+ *   Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __LIBCVLT_MEM_TYPES_H__
+#define __LIBCVLT_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum libcvlt_mem_types_ {
+    gf_libcvlt_mt_cvlt_private_t = gf_common_mt_end + 1,
+    gf_libcvlt_mt_end
+};
+#endif /* __LIBCVLT_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c
new file mode 100644
index 00000000000..5b7272bb448
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c
@@ -0,0 +1,842 @@
+#include <stdlib.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include "libcvlt.h"
+#include "cloudsync-common.h"
+#include "cvlt-messages.h"
+
+#define LIBARCHIVE_SO "libopenarchive.so"
+#define ALIGN_SIZE 4096
+#define CVLT_TRAILER "cvltv1"
+
+store_methods_t store_ops = {
+    .fop_download = cvlt_download,
+    .fop_init = cvlt_init,
+    .fop_reconfigure = cvlt_reconfigure,
+    .fop_fini = cvlt_fini,
+    .fop_remote_read = cvlt_read,
+};
+
+static const int32_t num_req = 32;
+static const int32_t num_iatt = 32;
+static char *plugin = "cvlt_cloudSync";
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_libcvlt_mt_end + 1);
+
+    if (ret != 0) {
+        return ret;
+    }
+
+    return ret;
+}
+
+static void
+cvlt_free_resources(archive_t *arch)
+{
+    /*
+     * We will release all the resources that were allocated by the xlator.
+     * Check whether there are any buffers which have not been released
+     * back to a mempool.
+     */
+
+    if (arch->handle) {
+        dlclose(arch->handle);
+    }
+
+    if (arch->iobuf_pool) {
+        iobuf_pool_destroy(arch->iobuf_pool);
+    }
+
+    if (arch->req_pool) {
+        mem_pool_destroy(arch->req_pool);
+        arch->req_pool = NULL;
+    }
+
+    return;
+}
+
+static int32_t
+cvlt_extract_store_fops(xlator_t *this, archive_t *arch)
+{
+    int32_t op_ret = -1;
+    get_archstore_methods_t get_archstore_methods;
+
+    /*
+     * libopenarchive.so defines methods for performing data management
+     * operations. We will extract the methods from library and these
+     * methods will be invoked for moving data between glusterfs volume
+     * and the data management product.
+     */
+
+    VALIDATE_OR_GOTO(arch, err);
+
+    arch->handle = dlopen(LIBARCHIVE_SO, RTLD_NOW);
+    if (!arch->handle) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_DLOPEN_FAILED,
+               " failed to open %s ", LIBARCHIVE_SO);
+        return op_ret;
+    }
+
+    dlerror(); /* Clear any existing error */
+
+    get_archstore_methods = dlsym(arch->handle, "get_archstore_methods");
+    if (!get_archstore_methods) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " Error extracting get_archstore_methods()");
+        dlclose(arch->handle);
+        arch->handle = NULL;
+        return op_ret;
+    }
+
+    op_ret = get_archstore_methods(&(arch->fops));
+    if (op_ret) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " Failed to extract methods in get_archstore_methods");
+        dlclose(arch->handle);
+        arch->handle = NULL;
+        return op_ret;
+    }
+
+err:
+    return op_ret;
+}
+
+static int32_t
+cvlt_alloc_resources(xlator_t *this, archive_t *arch, int num_req, int num_iatt)
+{
+    /*
+     * Initialize information about all the memory pools that will be
+     * used by this xlator.
+     */
+    arch->nreqs = 0;
+
+    arch->req_pool = NULL;
+
+    arch->handle = NULL;
+    arch->xl = this;
+
+    arch->req_pool = mem_pool_new(cvlt_request_t, num_req);
+    if (!arch->req_pool) {
+        goto err;
+    }
+
+    arch->iobuf_pool = iobuf_pool_new();
+    if (!arch->iobuf_pool) {
+        goto err;
+    }
+
+    if (cvlt_extract_store_fops(this, arch)) {
+        goto err;
+    }
+
+    return 0;
+
+err:
+
+    return -1;
+}
+
+static void
+cvlt_req_init(cvlt_request_t *req)
+{
+    sem_init(&(req->sem), 0, 0);
+
+    return;
+}
+
+static void
+cvlt_req_destroy(cvlt_request_t *req)
+{
+    if (req->iobuf) {
+        iobuf_unref(req->iobuf);
+    }
+
+    if (req->iobref) {
+        iobref_unref(req->iobref);
+    }
+
+    sem_destroy(&(req->sem));
+
+    return;
+}
+
+static cvlt_request_t *
+cvlt_alloc_req(archive_t *arch)
+{
+    cvlt_request_t *reqptr = NULL;
+
+    if (!arch) {
+        goto err;
+    }
+
+    if (arch->req_pool) {
+        reqptr = mem_get0(arch->req_pool);
+        if (reqptr) {
+            cvlt_req_init(reqptr);
+        }
+    }
+
+    if (reqptr) {
+        LOCK(&(arch->lock));
+        arch->nreqs++;
+        UNLOCK(&(arch->lock));
+    }
+
+err:
+    return reqptr;
+}
+
+static int32_t
+cvlt_free_req(archive_t *arch, cvlt_request_t *reqptr)
+{
+    if (!reqptr) {
+        goto err;
+    }
+
+    if (!arch) {
+        goto err;
+    }
+
+    if (arch->req_pool) {
+        /*
+         * Free the request resources if they exist.
+         */
+
+        cvlt_req_destroy(reqptr);
+        mem_put(reqptr);
+
+        LOCK(&(arch->lock));
+        arch->nreqs--;
+        UNLOCK(&(arch->lock));
+    }
+
+    return 0;
+
+err:
+    return -1;
+}
+
+static int32_t
+cvlt_init_xlator(xlator_t *this, archive_t *arch, int num_req, int num_iatt)
+{
+    int32_t ret = -1;
+    int32_t errnum = -1;
+    int32_t locked = 0;
+
+    /*
+     * Perform all the initializations needed for brining up the xlator.
+     */
+    if (!arch) {
+        goto err;
+    }
+
+    LOCK_INIT(&(arch->lock));
+    LOCK(&(arch->lock));
+
+    locked = 1;
+
+    ret = cvlt_alloc_resources(this, arch, num_req, num_iatt);
+
+    if (ret) {
+        goto err;
+    }
+
+    /*
+     * Now that the fops have been extracted initialize the store
+     */
+    ret = arch->fops.init(&(arch->descinfo), &errnum, plugin);
+    if (ret) {
+        goto err;
+    }
+
+    UNLOCK(&(arch->lock));
+    locked = 0;
+    ret = 0;
+
+    return ret;
+
+err:
+    if (arch) {
+        cvlt_free_resources(arch);
+
+        if (locked) {
+            UNLOCK(&(arch->lock));
+        }
+    }
+
+    return ret;
+}
+
+static int32_t
+cvlt_term_xlator(archive_t *arch)
+{
+    int32_t errnum = -1;
+
+    if (!arch) {
+        goto err;
+    }
+
+    LOCK(&(arch->lock));
+
+    /*
+     * Release the resources that have been allocated inside store
+     */
+    arch->fops.fini(&(arch->descinfo), &errnum);
+
+    cvlt_free_resources(arch);
+
+    UNLOCK(&(arch->lock));
+
+    GF_FREE(arch);
+
+    return 0;
+
+err:
+    return -1;
+}
+
+static int32_t
+cvlt_init_store_info(archive_t *priv, archstore_info_t *store_info)
+{
+    if (!store_info) {
+        return -1;
+    }
+
+    store_info->prod = priv->product_id;
+    store_info->prodlen = strlen(priv->product_id);
+
+    store_info->id = priv->store_id;
+    store_info->idlen = strlen(priv->store_id);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_file_info(cs_loc_xattr_t *xattr, archstore_fileinfo_t *file_info)
+{
+    if (!xattr || !file_info) {
+        return -1;
+    }
+
+    gf_uuid_copy(file_info->uuid, xattr->uuid);
+    file_info->path = xattr->file_path;
+    file_info->pathlength = strlen(xattr->file_path);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_gluster_store_info(cs_loc_xattr_t *xattr,
+                             archstore_info_t *store_info)
+{
+    static char *product = "glusterfs";
+
+    if (!xattr || !store_info) {
+        return -1;
+    }
+
+    store_info->prod = product;
+    store_info->prodlen = strlen(product);
+
+    store_info->id = xattr->volname;
+    store_info->idlen = strlen(xattr->volname);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_gluster_file_info(cs_loc_xattr_t *xattr,
+                            archstore_fileinfo_t *file_info)
+{
+    if (!xattr || !file_info) {
+        return -1;
+    }
+
+    gf_uuid_copy(file_info->uuid, xattr->gfid);
+    file_info->path = xattr->file_path;
+    file_info->pathlength = strlen(xattr->file_path);
+
+    return 0;
+}
+
+static void
+cvlt_copy_stat_info(struct iatt *buf, cs_size_xattr_t *xattrs)
+{
+    /*
+     * If the file was archived then the reported size will not be a
+     * correct one. We need to fix this.
+     */
+    if (buf && xattrs) {
+        buf->ia_size = xattrs->size;
+        buf->ia_blksize = xattrs->blksize;
+        buf->ia_blocks = xattrs->blocks;
+    }
+
+    return;
+}
+
+static void
+cvlt_readv_complete(archstore_desc_t *desc, app_callback_info_t *cbkinfo,
+                    void *cookie, int64_t op_ret, int32_t op_errno)
+{
+    struct iovec iov;
+    xlator_t *this = NULL;
+    struct iatt postbuf = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    cvlt_request_t *req = (cvlt_request_t *)cookie;
+    cs_local_t *local = NULL;
+    cs_private_t *cspriv = NULL;
+    archive_t *priv = NULL;
+
+    frame = req->frame;
+    this = frame->this;
+    local = frame->local;
+
+    cspriv = this->private;
+    priv = (archive_t *)cspriv->stores->config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_msg_debug(plugin, 0,
+                 " Read callback invoked offset:%" PRIu64 "bytes: %" PRIu64
+                 " op : %d ret : %" PRId64 " errno : %d",
+                 req->offset, req->bytes, req->op_type, op_ret, op_errno);
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    req->iobref = iobref_new();
+    if (!req->iobref) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    iobref_add(req->iobref, req->iobuf);
+    iov.iov_base = iobuf_ptr(req->iobuf);
+    iov.iov_len = op_ret;
+
+    cvlt_copy_stat_info(&postbuf, &(req->szxattr));
+
+    /*
+     * Hack to notify higher layers of EOF.
+     */
+    if (!postbuf.ia_size || (req->offset + iov.iov_len >= postbuf.ia_size)) {
+        gf_msg_debug(plugin, 0, " signalling end-of-file for uuid=%s",
+                     uuid_utoa(req->file_info.uuid));
+        op_errno = ENOENT;
+    }
+
+out:
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &iov, 1, &postbuf,
+                        req->iobref, local->xattr_rsp);
+
+    cvlt_free_req(priv, req);
+
+    return;
+}
+
+static void
+cvlt_download_complete(archstore_desc_t *store, app_callback_info_t *cbk_info,
+                       void *cookie, int64_t ret, int errcode)
+{
+    cvlt_request_t *req = (cvlt_request_t *)cookie;
+
+    gf_msg_debug(plugin, 0,
+                 " Download callback invoked  ret : %" PRId64 " errno : %d",
+                 ret, errcode);
+
+    req->op_ret = ret;
+    req->op_errno = errcode;
+    sem_post(&(req->sem));
+
+    return;
+}
+
+void *
+cvlt_init(xlator_t *this)
+{
+    int ret = 0;
+    archive_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0,
+               "should have exactly one child");
+        ret = -1;
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0,
+               "dangling volume. check volfile");
+        ret = -1;
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(archive_t), gf_libcvlt_mt_cvlt_private_t);
+    if (!priv) {
+        ret = -1;
+        goto out;
+    }
+
+    priv->trailer = CVLT_TRAILER;
+    if (cvlt_init_xlator(this, priv, num_req, num_iatt)) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0, "xlator init failed");
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_INIT("cloudsync-store-id", priv->store_id, str, out);
+    GF_OPTION_INIT("cloudsync-product-id", priv->product_id, str, out);
+
+    gf_msg(plugin, GF_LOG_INFO, 0, 0,
+           "store id is : %s "
+           "product id is : %s.",
+           priv->store_id, priv->product_id);
+out:
+    if (ret == -1) {
+        cvlt_term_xlator(priv);
+        return (NULL);
+    }
+    return priv;
+}
+
+int
+cvlt_reconfigure(xlator_t *this, dict_t *options)
+{
+    cs_private_t *cspriv = NULL;
+    archive_t *priv = NULL;
+
+    cspriv = this->private;
+    priv = (archive_t *)cspriv->stores->config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER))
+        goto out;
+
+    GF_OPTION_RECONF("cloudsync-store-id", priv->store_id, options, str, out);
+
+    GF_OPTION_RECONF("cloudsync-product-id", priv->product_id, options, str,
+                     out);
+    gf_msg_debug(plugin, 0,
+                 "store id is : %s "
+                 "product id is : %s.",
+                 priv->store_id, priv->product_id);
+    return 0;
+out:
+    return -1;
+}
+
+void
+cvlt_fini(void *config)
+{
+    archive_t *priv = NULL;
+
+    priv = (archive_t *)config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER))
+        return;
+
+    cvlt_term_xlator(priv);
+    gf_msg(plugin, GF_LOG_INFO, 0, CVLT_FREE, " released xlator resources");
+    return;
+}
+
+int
+cvlt_download(call_frame_t *frame, void *config)
+{
+    archive_t *parch = NULL;
+    cs_local_t *local = frame->local;
+    cs_loc_xattr_t *locxattr = local->xattrinfo.lxattr;
+    cvlt_request_t *req = NULL;
+    archstore_info_t dest_storeinfo;
+    archstore_fileinfo_t dest_fileinfo;
+    int32_t op_ret, op_errno;
+
+    parch = (archive_t *)config;
+
+    if (strcmp(parch->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    gf_msg_debug(plugin, 0, " download invoked for uuid = %s  gfid=%s ",
+                 locxattr->uuid, uuid_utoa(locxattr->gfid));
+
+    if (!(parch->fops.restore)) {
+        op_errno = ELIBBAD;
+        goto err;
+    }
+
+    /*
+     * Download needs to be processed. Allocate a request.
+     */
+    req = cvlt_alloc_req(parch);
+
+    if (!req) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, CVLT_RESOURCE_ALLOCATION_FAILED,
+               " failed to allocated request for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /*
+     * Initialize the request object.
+     */
+    req->op_type = CVLT_RESTORE_OP;
+    req->frame = frame;
+
+    /*
+     * The file is currently residing inside a data management store.
+     * To restore the file contents we need to provide the information
+     * about data management store.
+     */
+    op_ret = cvlt_init_store_info(parch, &(req->store_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract store info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    op_ret = cvlt_init_file_info(locxattr, &(req->file_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * We need to perform in-place restore of the file from data management
+     * store to gusterfs volume.
+     */
+    op_ret = cvlt_init_gluster_store_info(locxattr, &dest_storeinfo);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract destination store info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    op_ret = cvlt_init_gluster_file_info(locxattr, &dest_fileinfo);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * Submit the restore request.
+     */
+    op_ret = parch->fops.restore(&(parch->descinfo), &(req->store_info),
+                                 &(req->file_info), &dest_storeinfo,
+                                 &dest_fileinfo, &op_errno,
+                                 cvlt_download_complete, req);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_RESTORE_FAILED,
+               " failed to restore file gfid=%s from data management store",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * Wait for the restore to complete.
+     */
+    sem_wait(&(req->sem));
+
+    if (req->op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_RESTORE_FAILED,
+               " restored failed for gfid=%s", uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return 0;
+
+err:
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return -1;
+}
+
+int
+cvlt_read(call_frame_t *frame, void *config)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    archive_t *parch = NULL;
+    cvlt_request_t *req = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iobref *iobref;
+    size_t size = 0;
+    off_t off = 0;
+
+    cs_local_t *local = frame->local;
+    cs_loc_xattr_t *locxattr = local->xattrinfo.lxattr;
+
+    size = local->xattrinfo.size;
+    off = local->xattrinfo.offset;
+
+    parch = (archive_t *)config;
+
+    if (strcmp(parch->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    gf_msg_debug(plugin, 0,
+                 " read invoked for gfid = %s offset = %" PRIu64
+                 " file_size = %" PRIu64,
+                 uuid_utoa(locxattr->gfid), off, local->stbuf.ia_size);
+
+    if (off >= local->stbuf.ia_size) {
+        /*
+         * Hack to notify higher layers of EOF.
+         */
+
+        op_errno = ENOENT;
+        op_ret = 0;
+
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_READ_FAILED,
+               " reporting end-of-file for gfid=%s", uuid_utoa(locxattr->gfid));
+
+        goto err;
+    }
+
+    if (!size) {
+        op_errno = EINVAL;
+
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_READ_FAILED,
+               " zero size read attempted on gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    if (!(parch->fops.read)) {
+        op_errno = ELIBBAD;
+        goto err;
+    }
+
+    /*
+     * The read request need to be processed. Allocate a request.
+     */
+    req = cvlt_alloc_req(parch);
+
+    if (!req) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, CVLT_NO_MEMORY,
+               " failed to allocated request for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    req->iobuf = iobuf_get_page_aligned(parch->iobuf_pool, size, ALIGN_SIZE);
+    if (!req->iobuf) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /*
+     * Initialize the request object.
+     */
+    req->op_type = CVLT_READ_OP;
+    req->offset = off;
+    req->bytes = size;
+    req->frame = frame;
+    req->szxattr.size = local->stbuf.ia_size;
+    req->szxattr.blocks = local->stbuf.ia_blocks;
+    req->szxattr.blksize = local->stbuf.ia_blksize;
+
+    /*
+     * The file is currently residing inside a data management store.
+     * To read the file contents we need to provide the information
+     * about data management store.
+     */
+    op_ret = cvlt_init_store_info(parch, &(req->store_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract store info for gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    op_ret = cvlt_init_file_info(locxattr, &(req->file_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    /*
+     * Submit the read request.
+     */
+    op_ret = parch->fops.read(&(parch->descinfo), &(req->store_info),
+                              &(req->file_info), off, req->iobuf->ptr, size,
+                              &op_errno, cvlt_readv_complete, req);
+
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " read failed on gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    return 0;
+
+err:
+
+    iobref = iobref_new();
+    gf_msg_debug(plugin, 0, " read unwinding stack op_ret = %d, op_errno = %d",
+                 op_ret, op_errno);
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &iov, 1,
+                        &(local->stbuf), iobref, local->xattr_rsp);
+
+    if (iobref) {
+        iobref_unref(iobref);
+    }
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return 0;
+}
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h
new file mode 100644
index 00000000000..c45ac948f6c
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h
@@ -0,0 +1,84 @@
+/*
+  Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBCVLT_H
+#define _LIBCVLT_H
+
+#include <semaphore.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/compat-errno.h>
+#include "cloudsync-common.h"
+#include "libcvlt-mem-types.h"
+#include "archivestore.h"
+
+enum _cvlt_op {
+    CVLT_READ_OP = 1,
+    CVLT_WRITE_OP = 2,
+    CVLT_RESTORE_OP = 3,
+    CVLT_ARCHIVE_OP = 4,
+    CVLT_LOOKUP_OP = 5,
+    CVLT_XATTR_OP = 6,
+    CVLT_STAT_OP = 7,
+    CVLT_FSTAT_op = 8,
+    CVLT_UNDEF_OP = 127
+};
+typedef enum _cvlt_op cvlt_op_t;
+
+struct _archive;
+struct _cvlt_request {
+    uint64_t offset;
+    uint64_t bytes;
+    struct iobuf *iobuf;
+    struct iobref *iobref;
+    call_frame_t *frame;
+    cvlt_op_t op_type;
+    int32_t op_ret;
+    int32_t op_errno;
+    xlator_t *this;
+    sem_t sem;
+    archstore_info_t store_info;
+    archstore_fileinfo_t file_info;
+    cs_size_xattr_t szxattr;
+};
+typedef struct _cvlt_request cvlt_request_t;
+
+struct _archive {
+    gf_lock_t lock;                /* lock for controlling access   */
+    xlator_t *xl;                  /* xlator                        */
+    void *handle;                  /* handle returned from dlopen   */
+    int32_t nreqs;                 /* num requests active           */
+    struct mem_pool *req_pool;     /* pool for requests             */
+    struct iobuf_pool *iobuf_pool; /* iobuff pool                   */
+    archstore_desc_t descinfo;     /* Archive store descriptor info */
+    archstore_methods_t fops;      /* function pointers             */
+    char *product_id;
+    char *store_id;
+    char *trailer;
+};
+typedef struct _archive archive_t;
+
+void *
+cvlt_init(xlator_t *);
+
+int
+cvlt_reconfigure(xlator_t *, dict_t *);
+
+void
+cvlt_fini(void *);
+
+int
+cvlt_download(call_frame_t *, void *);
+
+int
+cvlt_read(call_frame_t *, void *);
+
+#endif
diff --git a/xlators/features/cloudsync/src/cloudsync.c b/xlators/features/cloudsync/src/cloudsync.c
new file mode 100644
index 00000000000..7f0b9e563b8
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync.c
@@ -0,0 +1,2076 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include "cloudsync.h"
+#include "cloudsync-common.h"
+#include <glusterfs/call-stub.h>
+#include "cloudsync-autogen-fops.h"
+
+#include <string.h>
+#include <dlfcn.h>
+
+static void
+cs_cleanup_private(cs_private_t *priv)
+{
+    if (priv) {
+        if (priv->stores) {
+            priv->stores->fini(priv->stores->config);
+            GF_FREE(priv->stores);
+        }
+
+        pthread_spin_destroy(&priv->lock);
+        GF_FREE(priv);
+    }
+
+    return;
+}
+
+static struct cs_plugin plugins[] = {
+    {.name = "cloudsyncs3",
+     .library = "cloudsyncs3.so",
+     .description = "cloudsync s3 store."},
+#if defined(__linux__)
+    {.name = "cvlt",
+     .library = "cloudsynccvlt.so",
+     .description = "Commvault content store."},
+#endif
+    {.name = NULL},
+};
+
+int
+cs_init(xlator_t *this)
+{
+    cs_private_t *priv = NULL;
+    gf_boolean_t per_vol = _gf_false;
+    int ret = 0;
+    char *libpath = NULL;
+    store_methods_t *store_methods = NULL;
+    void *handle = NULL;
+    char *temp_str = NULL;
+    int index = 0;
+    char *libname = NULL;
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_cs_mt_cs_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto out;
+    }
+
+    priv->this = this;
+
+    this->local_pool = mem_pool_new(cs_local_t, 512);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM, "initialisation failed.");
+        ret = -1;
+        goto out;
+    }
+
+    this->private = priv;
+
+    GF_OPTION_INIT("cloudsync-remote-read", priv->remote_read, bool, out);
+
+    /* temp workaround. Should be configurable through glusterd*/
+    per_vol = _gf_true;
+
+    if (per_vol) {
+        if (dict_get_str_sizen(this->options, "cloudsync-storetype",
+                               &temp_str) == 0) {
+            for (index = 0; plugins[index].name; index++) {
+                if (!strcmp(temp_str, plugins[index].name)) {
+                    libname = plugins[index].library;
+                    break;
+                }
+            }
+        } else {
+            ret = 0;
+        }
+
+        if (!libname) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0, "no plugin enabled");
+            ret = 0;
+            goto out;
+        }
+
+        ret = gf_asprintf(&libpath, "%s/%s", CS_PLUGINDIR, libname);
+        if (ret == -1) {
+            goto out;
+        }
+
+        handle = dlopen(libpath, RTLD_NOW);
+        if (!handle) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "could not "
+                   "load the required library. %s",
+                   dlerror());
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                   "loading library:%s successful", libname);
+        }
+
+        priv->stores = GF_CALLOC(1, sizeof(struct cs_remote_stores),
+                                 gf_cs_mt_cs_remote_stores_t);
+        if (!priv->stores) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "Could not "
+                   "allocate memory for priv->stores");
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror(); /* clear out previous error string */
+
+        /* load library methods */
+        store_methods = (store_methods_t *)dlsym(handle, "store_ops");
+        if (!store_methods) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null store_methods %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror();
+
+        if (priv->remote_read) {
+            priv->stores->rdfop = store_methods->fop_remote_read;
+            if (!priv->stores->rdfop) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "failed to get"
+                       " read fop %s",
+                       dlerror());
+                ret = -1;
+                goto out;
+            }
+        }
+
+        priv->stores->dlfop = store_methods->fop_download;
+        if (!priv->stores->dlfop) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " download fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror();
+        priv->stores->init = store_methods->fop_init;
+        if (!priv->stores->init) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " init fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror();
+        priv->stores->reconfigure = store_methods->fop_reconfigure;
+        if (!priv->stores->reconfigure) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " reconfigure fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        priv->stores->handle = handle;
+
+        priv->stores->config = (void *)((priv->stores->init)(this));
+        if (!priv->stores->config) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null config");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = 0;
+
+out:
+    if (ret == -1) {
+        if (this->local_pool) {
+            mem_pool_destroy(this->local_pool);
+            this->local_pool = NULL;
+        }
+
+        cs_cleanup_private(priv);
+
+        if (handle) {
+            dlclose(handle);
+        }
+    }
+
+    GF_FREE(libpath);
+
+    return ret;
+}
+
+int
+cs_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ctx_int = 0;
+    cs_inode_ctx_t *ctx = NULL;
+
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
+        return 0;
+
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctx_int;
+
+    GF_FREE(ctx);
+    return 0;
+}
+
+void
+cs_fini(xlator_t *this)
+{
+    cs_private_t *priv = NULL;
+    priv = this->private;
+
+    cs_cleanup_private(priv);
+}
+
+int
+cs_reconfigure(xlator_t *this, dict_t *options)
+{
+    cs_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    if (!priv) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_RECONF("cloudsync-remote-read", priv->remote_read, options, bool,
+                     out);
+
+    /* needed only for per volume configuration*/
+    ret = priv->stores->reconfigure(this, options);
+
+out:
+    return ret;
+}
+
+int32_t
+cs_mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("cloudsync", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_cs_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+
+int32_t
+cs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
+{
+    int ret = 0;
+    int op_errno = ENOMEM;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+                   "failed to create "
+                   "dict");
+            goto err;
+        }
+    }
+
+    ret = dict_set_uint32(xdata, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    uint64_t val = 0;
+
+    local = frame->local;
+
+    local->call_cnt++;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "truncate failed");
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "could not get file state, unwinding");
+                op_ret = -1;
+                op_errno = EIO;
+                goto unwind;
+            } else {
+                __cs_inode_ctx_update(this, local->loc.inode, val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+
+                if (local->call_cnt == 1 &&
+                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                    gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                           "will repair and download "
+                           "the file, current state : %" PRIu64,
+                           val);
+                    goto repair;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                           "second truncate, Unwinding");
+                    goto unwind;
+                }
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "file state "
+                   "could not be figured, unwinding");
+            goto unwind;
+        }
+    } else {
+        /* successful write => file is local */
+        __cs_inode_ctx_update(this, local->loc.inode, GF_CS_LOCAL);
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "state : GF_CS_LOCAL"
+               ", truncate successful");
+
+        goto unwind;
+    }
+
+repair:
+    ret = locate_and_execute(frame);
+    if (ret) {
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    CS_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+cs_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_TRUNCATE);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    __cs_inode_ctx_get(this, loc->inode, &ctx);
+
+    if (ctx)
+        state = __cs_get_file_state(loc->inode, ctx);
+    else
+        state = GF_CS_LOCAL;
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    local->stub = fop_truncate_stub(frame, cs_resume_truncate, loc, offset,
+                                    xdata);
+    if (!local->stub) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto err;
+    }
+
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_truncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+
+    } else {
+        local->call_cnt++;
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
+        }
+    }
+
+    return 0;
+err:
+    CS_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+cs_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    STACK_WIND(frame, cs_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
+}
+
+int32_t
+cs_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cs_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xattr_req)
+{
+    STACK_WIND(frame, cs_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xattr_req);
+    return 0;
+}
+
+int32_t
+cs_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->locked)
+        cs_inodelk_unlock(frame);
+
+    CS_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+
+    return 0;
+}
+
+int32_t
+cs_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    data_t *tmp = NULL;
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_SETXATTR);
+    if (!local) {
+        ret = -1;
+        goto err;
+    }
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    tmp = dict_get_sizen(dict, GF_CS_OBJECT_UPLOAD_COMPLETE);
+    if (tmp) {
+        /* Value of key should be the atime */
+        local->stub = fop_setxattr_stub(frame, cs_resume_setxattr, loc, dict,
+                                        flags, xdata);
+
+        if (!local->stub)
+            goto err;
+
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
+        }
+
+        return 0;
+    }
+
+    STACK_WIND(frame, cs_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+err:
+    CS_STACK_UNWIND(setxattr, frame, -1, errno, NULL);
+    return 0;
+}
+
+int32_t
+cs_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cs_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    STACK_WIND(frame, cs_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+cs_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cs_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    STACK_WIND(frame, cs_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+cs_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
+}
+
+int32_t
+cs_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_UNLINK);
+    if (!local)
+        goto err;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+    STACK_WIND(frame, cs_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flags, local->xattr_req);
+    return 0;
+err:
+    CS_STACK_UNWIND(unlink, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t val = 0;
+
+    if (op_ret == 0) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (!ret) {
+            ret = __cs_inode_ctx_update(this, fd->inode, val);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+            }
+        }
+    } else {
+        cs_inode_ctx_reset(this, fd->inode);
+    }
+
+    CS_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int32_t
+cs_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_OPEN);
+    if (!local)
+        goto err;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    STACK_WIND(frame, cs_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, local->xattr_req);
+    return 0;
+err:
+    CS_STACK_UNWIND(open, frame, -1, errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t val = 0;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+
+    fd = local->fd;
+
+    if (op_ret == 0) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (!ret) {
+            gf_msg_debug(this->name, 0, "state %" PRIu64, val);
+            ret = __cs_inode_ctx_update(this, fd->inode, val);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+            }
+        }
+    } else {
+        cs_inode_ctx_reset(this, fd->inode);
+    }
+
+    CS_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
+
+    return 0;
+}
+
+int32_t
+cs_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_FSTAT);
+    if (!local)
+        goto err;
+
+    if (fd->inode->ia_type == IA_IFDIR)
+        goto wind;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+wind:
+    STACK_WIND(frame, cs_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, local->xattr_req);
+    return 0;
+err:
+    CS_STACK_UNWIND(fstat, frame, -1, errno, NULL, NULL);
+    return 0;
+}
+
+cs_local_t *
+cs_local_init(xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
+              glusterfs_fop_t fop)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
+
+    if (loc) {
+        ret = loc_copy(&local->loc, loc);
+        if (ret)
+            goto out;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop = fop;
+    local->dloffset = 0;
+    frame->local = local;
+    local->locked = _gf_false;
+    local->call_cnt = 0;
+out:
+    if (ret) {
+        if (local)
+            mem_put(local);
+        local = NULL;
+    }
+
+    return local;
+}
+
+call_frame_t *
+cs_lock_frame(call_frame_t *parent_frame)
+{
+    call_frame_t *lock_frame = NULL;
+
+    lock_frame = copy_frame(parent_frame);
+
+    if (lock_frame == NULL)
+        goto out;
+
+    set_lk_owner_from_ptr(&lock_frame->root->lk_owner, parent_frame->root);
+
+out:
+    return lock_frame;
+}
+
+void
+cs_lock_wipe(call_frame_t *lock_frame)
+{
+    CS_STACK_DESTROY(lock_frame);
+}
+
+int32_t
+cs_inodelk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    cs_lock_wipe(frame);
+
+    return 0;
+}
+
+int
+cs_inodelk_unlock(call_frame_t *main_frame)
+{
+    xlator_t *this = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    call_frame_t *lock_frame = NULL;
+    cs_local_t *lock_local = NULL;
+    cs_local_t *main_local = NULL;
+    int ret = 0;
+
+    this = main_frame->this;
+    main_local = main_frame->local;
+
+    lock_frame = cs_lock_frame(main_frame);
+    if (!lock_frame)
+        goto out;
+
+    lock_local = cs_local_init(this, lock_frame, NULL, NULL, 0);
+    if (!lock_local)
+        goto out;
+
+    ret = cs_build_loc(&lock_local->loc, main_frame);
+    if (ret) {
+        goto out;
+    }
+
+    flock.l_type = F_UNLCK;
+
+    main_local->locked = _gf_false;
+
+    STACK_WIND(lock_frame, cs_inodelk_unlock_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, CS_LOCK_DOMAIN,
+               &lock_local->loc, F_SETLKW, &flock, NULL);
+
+    return 0;
+
+out:
+    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+           "Stale lock would be found on"
+           " server");
+
+    if (lock_frame)
+        cs_lock_wipe(lock_frame);
+
+    return 0;
+}
+
+int
+cs_download_task(void *arg)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    cs_private_t *priv = NULL;
+    int ret = -1;
+    char *sign_req = NULL;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    frame = (call_frame_t *)arg;
+
+    this = frame->this;
+
+    priv = this->private;
+
+    if (!priv->stores) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "No remote store "
+               "plugins found");
+        ret = -1;
+        goto out;
+    }
+
+    local = frame->local;
+
+    if (local->fd)
+        fd = fd_anonymous(local->fd->inode);
+    else
+        fd = fd_anonymous(local->loc.inode);
+
+    if (!fd) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    local->dlfd = fd;
+    local->dloffset = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "failed to create "
+               "dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_uint32(dict, GF_CS_OBJECT_DOWNLOADING, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(this, local->fd, dict, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "fsetxattr failed "
+               "key %s",
+               GF_CS_OBJECT_DOWNLOADING);
+        ret = -1;
+        goto out;
+    }
+    /*this calling method is for per volume setting */
+    ret = priv->stores->dlfop(frame, priv->stores->config);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "download failed"
+               ", remotepath: %s",
+               local->remotepath);
+
+        /*using dlfd as it is anonymous and have RDWR flag*/
+        ret = syncop_ftruncate(FIRST_CHILD(this), local->dlfd, 0, NULL, NULL,
+                               NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret, "ftruncate failed");
+        } else {
+            gf_msg_debug(this->name, 0, "ftruncate succeed");
+        }
+
+        ret = -1;
+        goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "download success, path"
+               " : %s",
+               local->remotepath);
+
+        ret = syncop_fremovexattr(this, local->fd, GF_CS_OBJECT_REMOTE, NULL,
+                                  NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret,
+                   "removexattr failed, remotexattr");
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "fremovexattr success, "
+                         "path : %s",
+                         local->remotepath);
+        }
+
+        ret = syncop_fremovexattr(this, local->fd, GF_CS_OBJECT_DOWNLOADING,
+                                  NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret,
+                   "removexattr failed, downloading xattr, path %s",
+                   local->remotepath);
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "fremovexattr success"
+                         " path  %s",
+                         local->remotepath);
+        }
+    }
+
+out:
+    GF_FREE(sign_req);
+
+    if (dict)
+        dict_unref(dict);
+
+    if (fd) {
+        fd_unref(fd);
+        local->dlfd = NULL;
+    }
+
+    return ret;
+}
+
+int
+cs_download(call_frame_t *frame)
+{
+    int ret = 0;
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    if (!local->remotepath) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "remote path not"
+               " available. Check posix logs to resolve");
+        goto out;
+    }
+
+    ret = cs_download_task((void *)frame);
+out:
+    return ret;
+}
+
+int
+cs_set_xattr_req(call_frame_t *frame)
+{
+    cs_local_t *local = NULL;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+
+    /* When remote reads are performed (i.e. reads on remote store),
+     * there needs to be a way to associate a file on gluster volume
+     * with its correspnding file on the remote store. In order to do
+     * that, a unique key can be maintained as an xattr
+     * (GF_CS_XATTR_ARCHIVE_UUID)on the stub file on gluster bricks.
+     * This xattr should be provided to the plugin to
+     * perform the read fop on the correct file. This assumes that the file
+     * hierarchy and name need not be the same on remote store as that of
+     * the gluster volume.
+     */
+    ret = dict_set_sizen_str_sizen(local->xattr_req, GF_CS_XATTR_ARCHIVE_UUID,
+                                   "1");
+
+    return 0;
+}
+
+int
+cs_update_xattrs(call_frame_t *frame, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int size = -1;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+    this = frame->this;
+
+    local->xattrinfo.lxattr = GF_CALLOC(1, sizeof(cs_loc_xattr_t),
+                                        gf_cs_mt_cs_lxattr_t);
+    if (!local->xattrinfo.lxattr) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    gf_uuid_copy(local->xattrinfo.lxattr->gfid, local->loc.gfid);
+
+    if (local->remotepath) {
+        local->xattrinfo.lxattr->file_path = gf_strdup(local->remotepath);
+        if (!local->xattrinfo.lxattr->file_path) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    ret = dict_get_gfuuid(xdata, GF_CS_XATTR_ARCHIVE_UUID,
+                          &(local->xattrinfo.lxattr->uuid));
+
+    if (ret) {
+        gf_uuid_clear(local->xattrinfo.lxattr->uuid);
+    }
+    size = strlen(this->name) - strlen("-cloudsync") + 1;
+    local->xattrinfo.lxattr->volname = GF_CALLOC(1, size, gf_common_mt_char);
+    if (!local->xattrinfo.lxattr->volname) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    strncpy(local->xattrinfo.lxattr->volname, this->name, size - 1);
+    local->xattrinfo.lxattr->volname[size - 1] = '\0';
+
+    return 0;
+err:
+    cs_xattrinfo_wipe(local);
+    return -1;
+}
+
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags)
+{
+    xlator_t *this = NULL;
+    cs_private_t *priv = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+
+    if (!local->remotepath) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "remote path not"
+               " available. Check posix logs to resolve");
+        goto out;
+    }
+
+    if (!priv->stores) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "No remote store "
+               "plugins found");
+        ret = -1;
+        goto out;
+    }
+
+    if (local->fd) {
+        fd = fd_anonymous(local->fd->inode);
+    } else {
+        fd = fd_anonymous(local->loc.inode);
+    }
+
+    local->xattrinfo.size = size;
+    local->xattrinfo.offset = offset;
+    local->xattrinfo.flags = flags;
+
+    if (!fd) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    local->dlfd = fd;
+    local->dloffset = offset;
+
+    /*this calling method is for per volume setting */
+    ret = priv->stores->rdfop(frame, priv->stores->config);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "read failed"
+               ", remotepath: %s",
+               local->remotepath);
+        ret = -1;
+        goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "read success, path"
+               " : %s",
+               local->remotepath);
+    }
+
+out:
+    if (fd) {
+        fd_unref(fd);
+        local->dlfd = NULL;
+    }
+    return ret;
+}
+
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    uint64_t val = 0;
+    fd_t *fd = NULL;
+
+    local = frame->local;
+    fd = local->fd;
+
+    local->call_cnt++;
+
+    if (op_ret == -1) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "could not get file state, unwinding");
+                op_ret = -1;
+                op_errno = EIO;
+                goto unwind;
+            } else {
+                __cs_inode_ctx_update(this, fd->inode, val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+
+                if (local->call_cnt == 1 &&
+                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                           " will read from remote : %" PRIu64, val);
+                    goto repair;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                           "second readv, Unwinding");
+                    goto unwind;
+                }
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "file state "
+                   "could not be figured, unwinding");
+            goto unwind;
+        }
+    } else {
+        /* successful readv => file is local */
+        __cs_inode_ctx_update(this, fd->inode, GF_CS_LOCAL);
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "state : GF_CS_LOCAL"
+               ", readv successful");
+
+        goto unwind;
+    }
+
+repair:
+    ret = locate_and_execute(frame);
+    if (ret) {
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    CS_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                    iobref, xdata);
+
+    return 0;
+}
+
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+
+    ret = cs_resume_postprocess(this, frame, fd->inode);
+    if (ret) {
+        goto unwind;
+    }
+
+    cs_inodelk_unlock(frame);
+
+    STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+
+    return 0;
+
+unwind:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_resume_remote_readv(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+    cs_local_t *local = NULL;
+    gf_cs_obj_state state = -1;
+    cs_inode_ctx_t *ctx = NULL;
+
+    cs_inodelk_unlock(frame);
+
+    local = frame->local;
+    if (!local) {
+        ret = -1;
+        goto unwind;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    state = __cs_get_file_state(fd->inode, ctx);
+    if (state == GF_CS_ERROR) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "status is GF_CS_ERROR."
+               " Aborting readv");
+        local->op_ret = -1;
+        local->op_errno = EREMOTE;
+        ret = -1;
+        goto unwind;
+    }
+
+    /* Serve readv from remote store only if it is remote. */
+    gf_msg_debug(this->name, 0, "status of file %s is %d",
+                 local->remotepath ? local->remotepath : "", state);
+
+    /* We will reach this condition if local inode ctx had REMOTE
+     * state when the control was in cs_readv but after stat
+     * we got an updated state saying that the file is LOCAL.
+     */
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else if (state == GF_CS_REMOTE) {
+        ret = cs_resume_remote_readv_postprocess(this, frame, fd->inode, offset,
+                                                 size, flags);
+        /* Failed to submit the remote readv fop to plugin */
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = EREMOTE;
+            goto unwind;
+        }
+        /* When the file is in any other intermediate state,
+         * we should not perform remote reads.
+         */
+    } else {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int op_errno = ENOMEM;
+    cs_local_t *local = NULL;
+    int ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = -1;
+    cs_private_t *priv = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    priv = this->private;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    if (ctx)
+        state = __cs_get_file_state(fd->inode, ctx);
+    else
+        state = GF_CS_LOCAL;
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    if (priv->remote_read) {
+        local->stub = fop_readv_stub(frame, cs_resume_remote_readv, fd, size,
+                                     offset, flags, xdata);
+    } else {
+        local->stub = fop_readv_stub(frame, cs_resume_readv, fd, size, offset,
+                                     flags, xdata);
+    }
+    if (!local->stub) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto err;
+    }
+
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else {
+        local->call_cnt++;
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
+        }
+    }
+
+    return 0;
+
+err:
+    CS_STACK_UNWIND(readv, frame, -1, op_errno, NULL, -1, NULL, NULL, NULL);
+
+    return 0;
+}
+
+int
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags)
+{
+    int ret = 0;
+
+    ret = cs_serve_readv(frame, offset, size, flags);
+
+    return ret;
+}
+
+int
+cs_stat_check_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    char *filepath = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    uint64_t val = 0;
+
+    local = frame->local;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg(this->name, GF_LOG_ERROR, 0, op_errno, "stat check failed");
+        goto err;
+    } else {
+        if (local->fd)
+            inode = local->fd->inode;
+        else
+            inode = local->loc.inode;
+
+        if (!inode) {
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "null inode "
+                   "returned");
+            goto err;
+        }
+
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                cs_inode_ctx_reset(this, inode);
+                local->op_ret = -1;
+                local->op_errno = EIO;
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "status = GF_CS_ERROR. failed to get "
+                       " file state");
+                goto err;
+            } else {
+                ret = __cs_inode_ctx_update(this, inode, val);
+                gf_msg_debug(this->name, 0, "status : %" PRIu64, val);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto err;
+                }
+            }
+        } else {
+            gf_msg_debug(this->name, 0, "status not found in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
+        }
+
+        ret = dict_get_str_sizen(xdata, GF_CS_OBJECT_REMOTE, &filepath);
+        if (filepath) {
+            gf_msg_debug(this->name, 0, "filepath returned %s", filepath);
+            local->remotepath = gf_strdup(filepath);
+            if (!local->remotepath) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto err;
+            }
+        } else {
+            gf_msg_debug(this->name, 0, "NULL filepath");
+        }
+
+        ret = cs_update_xattrs(frame, xdata);
+        if (ret)
+            goto err;
+
+        local->op_ret = 0;
+        local->xattr_rsp = dict_ref(xdata);
+        memcpy(&local->stbuf, stbuf, sizeof(struct iatt));
+    }
+
+    stub = local->stub;
+    local->stub = NULL;
+    call_resume(stub);
+
+    return 0;
+err:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int
+cs_do_stat_check(call_frame_t *main_frame)
+{
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+
+    local = main_frame->local;
+    this = main_frame->this;
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_REPAIR, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
+        goto err;
+    }
+
+    cs_set_xattr_req(main_frame);
+
+    if (local->fd) {
+        STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, local->fd, local->xattr_req);
+    } else {
+        STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, &local->loc,
+                   local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    cs_inodelk_unlock(main_frame);
+
+    cs_common_cbk(main_frame);
+
+    return 0;
+}
+
+void
+cs_common_cbk(call_frame_t *frame)
+{
+    glusterfs_fop_t fop = -1;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+
+    fop = local->fop;
+
+    /*Note: Only the failure case needs to be handled here. Since for
+     * successful stat check the fop will resume anyway. The unwind can
+     * happen from the fop_cbk and each cbk can unlock the inodelk in case
+     * a lock was taken before. The lock status can be stored in frame */
+
+    /* for failure case  */
+
+    /*TODO: add other fops*/
+    switch (fop) {
+        case GF_FOP_WRITE:
+            CS_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno, NULL,
+                            NULL, NULL);
+            break;
+
+        case GF_FOP_SETXATTR:
+            CS_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                            NULL);
+            break;
+        case GF_FOP_READ:
+            CS_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, NULL,
+                            0, NULL, NULL, NULL);
+            break;
+        case GF_FOP_FTRUNCATE:
+            CS_STACK_UNWIND(ftruncate, frame, local->op_ret, local->op_errno,
+                            NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_TRUNCATE:
+            CS_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                            NULL, NULL, NULL);
+            break;
+        default:
+            break;
+    }
+
+    return;
+}
+
+int
+cs_blocking_inodelk_cbk(call_frame_t *lock_frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    cs_local_t *main_local = NULL;
+    call_frame_t *main_frame = NULL;
+    cs_local_t *lock_local = NULL;
+
+    lock_local = lock_frame->local;
+
+    main_frame = lock_local->main_frame;
+    main_local = main_frame->local;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "inodelk failed");
+        main_local->op_errno = op_errno;
+        main_local->op_ret = op_ret;
+        goto err;
+    }
+
+    main_local->locked = _gf_true;
+
+    cs_lock_wipe(lock_frame);
+
+    cs_do_stat_check(main_frame);
+
+    return 0;
+err:
+    cs_common_cbk(main_frame);
+
+    cs_lock_wipe(lock_frame);
+
+    return 0;
+}
+
+int
+cs_build_loc(loc_t *loc, call_frame_t *frame)
+{
+    cs_local_t *local = NULL;
+    int ret = -1;
+
+    local = frame->local;
+
+    if (local->fd) {
+        loc->inode = inode_ref(local->fd->inode);
+        if (loc->inode) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
+            ret = 0;
+            goto out;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    } else {
+        loc->inode = inode_ref(local->loc.inode);
+        if (loc->inode) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
+            ret = 0;
+            goto out;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+int
+cs_blocking_inodelk(call_frame_t *parent_frame)
+{
+    call_frame_t *lock_frame = NULL;
+    cs_local_t *lock_local = NULL;
+    xlator_t *this = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    this = parent_frame->this;
+
+    lock_frame = cs_lock_frame(parent_frame);
+    if (!lock_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insuffcient memory");
+        goto err;
+    }
+
+    lock_local = cs_local_init(this, lock_frame, NULL, NULL, 0);
+    if (!lock_local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    lock_local->main_frame = parent_frame;
+
+    flock.l_type = F_WRLCK;
+
+    ret = cs_build_loc(&lock_local->loc, parent_frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "build_loc failed");
+        goto err;
+    }
+
+    STACK_WIND(lock_frame, cs_blocking_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, CS_LOCK_DOMAIN,
+               &lock_local->loc, F_SETLKW, &flock, NULL);
+
+    return 0;
+err:
+    if (lock_frame)
+        cs_lock_wipe(lock_frame);
+
+    return -1;
+}
+
+int
+locate_and_execute(call_frame_t *frame)
+{
+    int ret = 0;
+
+    ret = cs_blocking_inodelk(frame);
+
+    if (ret)
+        return -1;
+    else
+        return 0;
+}
+
+int32_t
+cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+
+    ret = cs_resume_postprocess(this, frame, loc->inode);
+    if (ret) {
+        goto unwind;
+    }
+
+    cs_inodelk_unlock(frame);
+
+    STACK_WIND(frame, cs_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset,
+               local->xattr_req);
+
+    return 0;
+
+unwind:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_resume_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = GF_CS_ERROR;
+
+    local = frame->local;
+
+    __cs_inode_ctx_get(this, loc->inode, &ctx);
+
+    state = __cs_get_file_state(loc->inode, ctx);
+
+    if (state == GF_CS_ERROR) {
+        /* file is already remote */
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+               "file %s , could not figure file state", loc->path);
+        goto unwind;
+    }
+
+    if (state == GF_CS_REMOTE) {
+        /* file is already remote */
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, 0, EINVAL,
+               "file %s is already remote", loc->path);
+        goto unwind;
+    }
+
+    if (state == GF_CS_DOWNLOADING) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+               " file is in downloading state.");
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    STACK_WIND(frame, cs_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+               local->xattr_req);
+
+    return 0;
+unwind:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+gf_cs_obj_state
+__cs_get_file_state(inode_t *inode, cs_inode_ctx_t *ctx)
+{
+    gf_cs_obj_state state = -1;
+
+    if (!ctx)
+        return GF_CS_ERROR;
+
+    LOCK(&inode->lock);
+    {
+        state = ctx->state;
+    }
+    UNLOCK(&inode->lock);
+
+    return state;
+}
+
+void
+__cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx)
+{
+    uint64_t ctxint = 0;
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctxint);
+    }
+    UNLOCK(&inode->lock);
+
+    if (ret)
+        *ctx = NULL;
+    else
+        *ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+
+    return;
+}
+
+int
+__cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val)
+{
+    cs_inode_ctx_t *ctx = NULL;
+    uint64_t ctxint = 0;
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctxint);
+        if (ret) {
+            ctx = GF_CALLOC(1, sizeof(*ctx), gf_cs_mt_cs_inode_ctx_t);
+            if (!ctx) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx allocation failed");
+                ret = -1;
+                goto out;
+            }
+
+            ctx->state = val;
+
+            ctxint = (uint64_t)(uintptr_t)ctx;
+
+            ret = __inode_ctx_set(inode, this, &ctxint);
+            if (ret) {
+                GF_FREE(ctx);
+                goto out;
+            }
+        } else {
+            ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+
+            ctx->state = val;
+        }
+    }
+
+out:
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
+
+int
+cs_inode_ctx_reset(xlator_t *this, inode_t *inode)
+{
+    cs_inode_ctx_t *ctx = NULL;
+    uint64_t ctxint = 0;
+
+    inode_ctx_del(inode, this, &ctxint);
+    if (!ctxint) {
+        return 0;
+    }
+
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+
+    GF_FREE(ctx);
+    return 0;
+}
+
+int
+cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode)
+{
+    cs_local_t *local = NULL;
+    gf_cs_obj_state state = -1;
+    cs_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    if (!local) {
+        ret = -1;
+        goto out;
+    }
+
+    __cs_inode_ctx_get(this, inode, &ctx);
+
+    state = __cs_get_file_state(inode, ctx);
+    if (state == GF_CS_ERROR) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "status is GF_CS_ERROR."
+               " Aborting write");
+        local->op_ret = -1;
+        local->op_errno = EREMOTE;
+        ret = -1;
+        goto out;
+    }
+
+    if (state == GF_CS_REMOTE || state == GF_CS_DOWNLOADING) {
+        gf_msg_debug(this->name, 0, "status is %d", state);
+        ret = cs_download(frame);
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0, "Winding for Final Write");
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   " download failed, unwinding writev");
+            local->op_ret = -1;
+            local->op_errno = EREMOTE;
+            ret = -1;
+        }
+    }
+out:
+    return ret;
+}
+
+int32_t
+cs_fdctx_to_dict(xlator_t *this, fd_t *fd, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_inode(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+cs_inode_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_history(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+cs_fd(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+cs_fd_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_fdctx(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+cs_inodectx(xlator_t *this, inode_t *ino)
+{
+    return 0;
+}
+
+int32_t
+cs_inodectx_to_dict(xlator_t *this, inode_t *ino, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname)
+{
+    return 0;
+}
+
+int32_t
+cs_priv(xlator_t *this)
+{
+    return 0;
+}
+
+int
+cs_notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+struct xlator_fops cs_fops = {
+    .stat = cs_stat,
+    .readdirp = cs_readdirp,
+    .truncate = cs_truncate,
+    .seek = cs_seek,
+    .statfs = cs_statfs,
+    .fallocate = cs_fallocate,
+    .discard = cs_discard,
+    .getxattr = cs_getxattr,
+    .writev = cs_writev,
+    .setxattr = cs_setxattr,
+    .fgetxattr = cs_fgetxattr,
+    .lookup = cs_lookup,
+    .fsetxattr = cs_fsetxattr,
+    .readv = cs_readv,
+    .ftruncate = cs_ftruncate,
+    .rchecksum = cs_rchecksum,
+    .unlink = cs_unlink,
+    .open = cs_open,
+    .fstat = cs_fstat,
+    .zerofill = cs_zerofill,
+};
+
+struct xlator_cbks cs_cbks = {
+    .forget = cs_forget,
+};
+
+struct xlator_dumpops cs_dumpops = {
+    .fdctx_to_dict = cs_fdctx_to_dict,
+    .inode = cs_inode,
+    .inode_to_dict = cs_inode_to_dict,
+    .history = cs_history,
+    .fd = cs_fd,
+    .fd_to_dict = cs_fd_to_dict,
+    .fdctx = cs_fdctx,
+    .inodectx = cs_inodectx,
+    .inodectx_to_dict = cs_inodectx_to_dict,
+    .priv_to_dict = cs_priv_to_dict,
+    .priv = cs_priv,
+};
+
+struct volume_options cs_options[] = {
+    {.key = {"cloudsync-storetype"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines which remote store is enabled"},
+    {.key = {"cloudsync-remote-read"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "Defines a remote read fop when on"},
+    {.key = {"cloudsync-store-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide store id"},
+    {.key = {"cloudsync-product-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide product id"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = cs_init,
+    .fini = cs_fini,
+    .notify = cs_notify,
+    .reconfigure = cs_reconfigure,
+    .mem_acct_init = cs_mem_acct_init,
+    .dumpops = &cs_dumpops,
+    .fops = &cs_fops,
+    .cbks = &cs_cbks,
+    .options = cs_options,
+    .identifier = "cloudsync",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/features/cloudsync/src/cloudsync.h b/xlators/features/cloudsync/src/cloudsync.h
new file mode 100644
index 00000000000..d24141978d6
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync.h
@@ -0,0 +1,123 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __CLOUDSYNC_H__
+#define __CLOUDSYNC_H__
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include "cloudsync-common.h"
+#include "cloudsync-autogen-fops.h"
+
+#define ALIGN_SIZE 4096
+#define CS_LOCK_DOMAIN "cs.protect.file.stat"
+typedef struct cs_dlstore {
+    off_t off;
+    struct iovec *vector;
+    int32_t count;
+    struct iobref *iobref;
+    uint32_t flags;
+} cs_dlstore;
+
+typedef struct cs_inode_ctx {
+    cs_loc_xattr_t locxattr;
+    gf_cs_obj_state state;
+} cs_inode_ctx_t;
+
+struct cs_plugin {
+    char *name;        /* store name */
+    char *library;     /* library to load for the given store */
+    char *description; /* description about the store */
+};
+
+cs_local_t *
+cs_local_init(xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
+              glusterfs_fop_t fop);
+
+int
+locate_and_execute(call_frame_t *frame);
+
+int32_t
+cs_resume_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata);
+
+int32_t
+cs_inodelk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+size_t
+cs_write_callback(void *lcurlbuf, size_t size, size_t nitems, void *frame);
+
+void
+cs_common_cbk(call_frame_t *frame);
+
+gf_boolean_t
+cs_is_file_remote(struct iatt *stbuf, dict_t *xattr);
+
+int32_t
+cs_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata);
+int
+cs_build_loc(loc_t *loc, call_frame_t *frame);
+
+int
+cs_blocking_inodelk_cbk(call_frame_t *lock_frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int
+cs_read_authinfo(xlator_t *this);
+
+int
+__cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val);
+
+int
+cs_inode_ctx_reset(xlator_t *this, inode_t *inode);
+
+void
+__cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx);
+
+gf_cs_obj_state
+__cs_get_file_state(inode_t *inode, cs_inode_ctx_t *ctx);
+
+int
+cs_inodelk_unlock(call_frame_t *main_frame);
+
+int
+cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode);
+
+int32_t
+cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata);
+int32_t
+cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xattr_req);
+
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata);
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata);
+
+int
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags);
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags);
+#endif /* __CLOUDSYNC_H__ */
diff --git a/xlators/features/compress/src/Makefile.am b/xlators/features/compress/src/Makefile.am
index b7c75e91b92..98271a9f3fc 100644
--- a/xlators/features/compress/src/Makefile.am
+++ b/xlators/features/compress/src/Makefile.am
@@ -9,8 +9,10 @@ cdc_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 cdc_la_SOURCES = cdc.c cdc-helper.c
 cdc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(ZLIB_LIBS)
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC \
-	-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) $(LIBZ_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+	$(LIBZ_CFLAGS)
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
index 0a9a0e3d29c..f973ff56cf5 100644
--- a/xlators/features/compress/src/cdc-helper.c
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -8,9 +8,9 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "syscall.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
 
 #include "cdc.h"
 #include "cdc-mem-types.h"
@@ -34,118 +34,110 @@
  * gzip_header is added only during debugging.
  * Refer to the function cdc_dump_iovec_to_disk
  */
-static const char gzip_header[10] =
-        {
-                '\037', '\213', Z_DEFLATED, 0,
-                0,      0,      0,          0,
-                0,      GF_CDC_OS_ID
-        };
+static const char gzip_header[10] = {'\037', '\213', Z_DEFLATED,  0, 0, 0, 0,
+                                     0,      0,      GF_CDC_OS_ID};
 
 static int32_t
-cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+cdc_next_iovec(xlator_t *this, cdc_info_t *ci)
 {
-        int ret = -1;
-
-        ci->ncount++;
-        /* check for iovec overflow -- should not happen */
-        if (ci->ncount == MAX_IOVEC) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Zlib output buffer overflow"
-                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
-                        ci->ncount, MAX_IOVEC);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
+    int ret = -1;
+
+    ci->ncount++;
+    /* check for iovec overflow -- should not happen */
+    if (ci->ncount == MAX_IOVEC) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Zlib output buffer overflow"
+               " ->ncount (%d) | ->MAX_IOVEC (%d)",
+               ci->ncount, MAX_IOVEC);
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    return ret;
 }
 
 static void
-cdc_put_long (unsigned char *string, unsigned long x)
+cdc_put_long(unsigned char *string, unsigned long x)
 {
-        string[0] = (unsigned char) (x & 0xff);
-        string[1] = (unsigned char) ((x & 0xff00) >> 8);
-        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
-        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+    string[0] = (unsigned char)(x & 0xff);
+    string[1] = (unsigned char)((x & 0xff00) >> 8);
+    string[2] = (unsigned char)((x & 0xff0000) >> 16);
+    string[3] = (unsigned char)((x & 0xff000000) >> 24);
 }
 
 static unsigned long
-cdc_get_long (unsigned char *buf)
+cdc_get_long(unsigned char *buf)
 {
-        return ((unsigned long) buf[0])
-                | (((unsigned long) buf[1]) << 8)
-                | (((unsigned long) buf[2]) << 16)
-                | (((unsigned long) buf[3]) << 24);
+    return ((unsigned long)buf[0]) | (((unsigned long)buf[1]) << 8) |
+           (((unsigned long)buf[2]) << 16) | (((unsigned long)buf[3]) << 24);
 }
 
 static int32_t
-cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+cdc_init_gzip_trailer(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
 {
-        int   ret = -1;
-        char *buf = NULL;
+    int ret = -1;
+    char *buf = NULL;
 
-        ret = cdc_next_iovec (this, ci);
-        if (ret)
-                goto out;
+    ret = cdc_next_iovec(this, ci);
+    if (ret)
+        goto out;
 
-        buf = CURR_VEC(ci).iov_base =
-                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
-                                    gf_cdc_mt_gzip_trailer_t);
+    buf = CURR_VEC(ci).iov_base = (char *)GF_CALLOC(1, GF_CDC_VALIDATION_SIZE,
+                                                    gf_cdc_mt_gzip_trailer_t);
 
-        if (!CURR_VEC(ci).iov_base)
-                goto out;
+    if (!CURR_VEC(ci).iov_base)
+        goto out;
 
-        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+    CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
 
-        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
-        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+    cdc_put_long((unsigned char *)&buf[0], ci->crc);
+    cdc_put_long((unsigned char *)&buf[4], ci->stream.total_in);
 
-        ret = 0;
+    ret = 0;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int32_t
-cdc_alloc_iobuf_and_init_vec (xlator_t *this,
-                              cdc_priv_t *priv, cdc_info_t *ci,
-                              int size)
+cdc_alloc_iobuf_and_init_vec(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                             int size)
 {
-        int           ret       = -1;
-        int           alloc_len = 0;
-        struct iobuf *iobuf     = NULL;
+    int ret = -1;
+    int alloc_len = 0;
+    struct iobuf *iobuf = NULL;
 
-        ret = cdc_next_iovec (this, ci);
-        if (ret)
-                goto out;
+    ret = cdc_next_iovec(this, ci);
+    if (ret)
+        goto out;
 
-        alloc_len = size ? size : ci->buffer_size;
+    alloc_len = size ? size : ci->buffer_size;
 
-        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
-        if (!iobuf)
-                goto out;
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, alloc_len);
+    if (!iobuf)
+        goto out;
 
-        ret = iobref_add (ci->iobref, iobuf);
-        if (ret)
-                goto out;
+    ret = iobref_add(ci->iobref, iobuf);
+    if (ret)
+        goto out;
 
-        /* Initialize this iovec */
-        CURR_VEC(ci).iov_base = iobuf->ptr;
-        CURR_VEC(ci).iov_len  = alloc_len;
+    /* Initialize this iovec */
+    CURR_VEC(ci).iov_base = iobuf->ptr;
+    CURR_VEC(ci).iov_len = alloc_len;
 
-        ret = 0;
+    ret = 0;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static void
-cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+cdc_init_zlib_output_stream(cdc_priv_t *priv, cdc_info_t *ci, int size)
 {
-        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
-        ci->stream.avail_out = size ? size : ci->buffer_size;
+    ci->stream.next_out = (unsigned char *)CURR_VEC(ci).iov_base;
+    ci->stream.avail_out = size ? size : ci->buffer_size;
 }
 
 /* This routine is for testing and debugging only.
@@ -153,391 +145,383 @@ cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
  * So each gzip dump file is at least 18 bytes in size.
  */
 void
-cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+cdc_dump_iovec_to_disk(xlator_t *this, cdc_info_t *ci, const char *file)
 {
-        int    i      = 0;
-        int    fd     = 0;
-        size_t written = 0;
-        size_t total_written = 0;
-
-        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot open file: %s", file);
-                return;
-        }
-
-        written = sys_write (fd, (char *) gzip_header, 10);
+    int i = 0;
+    int fd = 0;
+    size_t written = 0;
+    size_t total_written = 0;
+
+    fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0777);
+    if (fd < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "Cannot open file: %s", file);
+        return;
+    }
+
+    written = sys_write(fd, (char *)gzip_header, 10);
+    total_written += written;
+    for (i = 0; i < ci->ncount; i++) {
+        written = sys_write(fd, (char *)ci->vec[i].iov_base,
+                            ci->vec[i].iov_len);
         total_written += written;
-        for (i = 0; i < ci->ncount; i++) {
-                written = sys_write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
-                total_written += written;
-        }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                        "dump'd %zu bytes to %s", total_written, GF_CDC_DEBUG_DUMP_FILE );
+    gf_log(this->name, GF_LOG_DEBUG, "dump'd %zu bytes to %s", total_written,
+           GF_CDC_DEBUG_DUMP_FILE);
 
-        sys_close (fd);
+    sys_close(fd);
 }
 
 static int32_t
-cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
-                       int (*libz_func)(z_streamp, int),
-                       int flush)
+cdc_flush_libz_buffer(cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                      int (*libz_func)(z_streamp, int), int flush)
 {
-        int32_t ret = Z_OK;
-        int done = 0;
-        unsigned int deflate_len = 0;
+    int32_t ret = Z_OK;
+    int done = 0;
+    unsigned int deflate_len = 0;
 
-        for (;;) {
-                deflate_len = ci->buffer_size - ci->stream.avail_out;
+    for (;;) {
+        deflate_len = ci->buffer_size - ci->stream.avail_out;
 
-                if (deflate_len != 0) {
-                        CURR_VEC(ci).iov_len = deflate_len;
+        if (deflate_len != 0) {
+            CURR_VEC(ci).iov_len = deflate_len;
 
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret) {
-                                ret = Z_MEM_ERROR;
-                                break;
-                        }
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret) {
+                ret = Z_MEM_ERROR;
+                break;
+            }
 
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
+        }
 
-                if (done) {
-                        ci->ncount--;
-                        break;
-                }
+        if (done) {
+            ci->ncount--;
+            break;
+        }
 
-                ret = libz_func (&ci->stream, flush);
+        ret = libz_func(&ci->stream, flush);
 
-                if (ret == Z_BUF_ERROR) {
-                        ret = Z_OK;
-                        ci->ncount--;
-                        break;
-                }
+        if (ret == Z_BUF_ERROR) {
+            ret = Z_OK;
+            ci->ncount--;
+            break;
+        }
 
-                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+        done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
 
-                if (ret != Z_OK && ret != Z_STREAM_END)
-                        break;
-        }
+        if (ret != Z_OK && ret != Z_STREAM_END)
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
-                 cdc_info_t *ci)
+do_cdc_compress(struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                cdc_info_t *ci)
 {
-        int ret = -1;
+    int ret = -1;
 
-        /* Initialize defalte */
-        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
-                            priv->window_size, priv->mem_level,
-                            Z_DEFAULT_STRATEGY);
+    /* Initialize defalte */
+    ret = deflateInit2(&ci->stream, priv->cdc_level, Z_DEFLATED,
+                       priv->window_size, priv->mem_level, Z_DEFAULT_STRATEGY);
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to init Zlib (retval: %d)", ret);
-                goto out;
-        }
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "unable to init Zlib (retval: %d)",
+               ret);
+        goto out;
+    }
 
-        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-        if (ret)
-                goto out;
+    ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+    if (ret)
+        goto out;
 
-        /* setup output buffer */
-        cdc_init_zlib_output_stream (priv, ci, 0);
+    /* setup output buffer */
+    cdc_init_zlib_output_stream(priv, ci, 0);
 
-        /* setup input buffer */
-        ci->stream.next_in  = (unsigned char *) vec->iov_base;
-        ci->stream.avail_in = vec->iov_len;
+    /* setup input buffer */
+    ci->stream.next_in = (unsigned char *)vec->iov_base;
+    ci->stream.avail_in = vec->iov_len;
 
-        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+    ci->crc = crc32(ci->crc, (const Bytef *)vec->iov_base, vec->iov_len);
 
-        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
-                ci->crc, ci->stream.avail_in, ci->buffer_size);
+    gf_log(this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d", ci->crc,
+           ci->stream.avail_in, ci->buffer_size);
 
-        /* compress !! */
-        while (ci->stream.avail_in != 0) {
-                if (ci->stream.avail_out == 0) {
+    /* compress !! */
+    while (ci->stream.avail_in != 0) {
+        if (ci->stream.avail_out == 0) {
+            CURR_VEC(ci).iov_len = ci->buffer_size;
 
-                        CURR_VEC(ci).iov_len = ci->buffer_size;
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret)
+                break;
 
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret)
-                                break;
-
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
-
-                ret = deflate (&ci->stream, Z_NO_FLUSH);
-                if (ret != Z_OK)
-                        break;
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
         }
 
- out:
-        return ret;
+        ret = deflate(&ci->stream, Z_NO_FLUSH);
+        if (ret != Z_OK)
+            break;
+    }
+
+out:
+    return ret;
 }
 
 int32_t
-cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
-              dict_t **xdata)
+cdc_compress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t **xdata)
 {
-        int ret = -1;
-        int i   = 0;
+    int ret = -1;
+    int i = 0;
 
-        ci->iobref = iobref_new ();
-        if (!ci->iobref)
-                goto out;
+    ci->iobref = iobref_new();
+    if (!ci->iobref)
+        goto out;
 
+    if (!*xdata) {
+        *xdata = dict_new();
         if (!*xdata) {
-                *xdata = dict_new ();
-                if (!*xdata) {
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
-                                " dict");
-                        goto out;
-                }
-        }
-
-        /* data */
-        for (i = 0; i < ci->count; i++) {
-                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
-                if (ret != Z_OK)
-                        goto deflate_cleanup_out;
-        }
-
-        /* flush zlib buffer */
-        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
-        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Compression Error: ret (%d)", ret);
-                ret = -1;
-                goto deflate_cleanup_out;
-        }
-
-        /* trailer */
-        ret = cdc_init_gzip_trailer (this, priv, ci);
-        if (ret)
-                goto deflate_cleanup_out;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Compressed %ld to %ld bytes",
-                ci->stream.total_in, ci->stream.total_out);
-
-        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
-
-        /* set deflated canary value for identification */
-        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
-        if (ret) {
-                /* Send uncompressed data if we can't _tell_ the client
-                 * that deflated data is on it's way. So, we just log
-                 * the faliure and continue as usual.
-                 */
-                 gf_log (this->name, GF_LOG_ERROR,
-                 "Data deflated, but could not set canary"
-                 " value in dict for identification");
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Cannot allocate xdata"
+                   " dict");
+            goto out;
         }
+    }
+
+    /* data */
+    for (i = 0; i < ci->count; i++) {
+        ret = do_cdc_compress(&ci->vector[i], this, priv, ci);
+        if (ret != Z_OK)
+            goto deflate_cleanup_out;
+    }
+
+    /* flush zlib buffer */
+    ret = cdc_flush_libz_buffer(priv, this, ci, deflate, Z_FINISH);
+    if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+        gf_log(this->name, GF_LOG_ERROR, "Compression Error: ret (%d)", ret);
+        ret = -1;
+        goto deflate_cleanup_out;
+    }
+
+    /* trailer */
+    ret = cdc_init_gzip_trailer(this, priv, ci);
+    if (ret)
+        goto deflate_cleanup_out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "Compressed %ld to %ld bytes",
+           ci->stream.total_in, ci->stream.total_out);
+
+    ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+    /* set deflated canary value for identification */
+    ret = dict_set_int32(*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+    if (ret) {
+        /* Send uncompressed data if we can't _tell_ the client
+         * that deflated data is on it's way. So, we just log
+         * the failure and continue as usual.
+         */
+        gf_log(this->name, GF_LOG_ERROR,
+               "Data deflated, but could not set canary"
+               " value in dict for identification");
+    }
 
-        /* This is to be used in testing */
-        if ( priv->debug ) {
-                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
-        }
+    /* This is to be used in testing */
+    if (priv->debug) {
+        cdc_dump_iovec_to_disk(this, ci, GF_CDC_DEBUG_DUMP_FILE);
+    }
 
- deflate_cleanup_out:
-        (void) deflateEnd(&ci->stream);
+deflate_cleanup_out:
+    (void)deflateEnd(&ci->stream);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-
 /* deflate content is checked by the presence of a canary
  * value in the dict as the key
  */
 static int32_t
-cdc_check_content_for_deflate (dict_t *xdata)
+cdc_check_content_for_deflate(dict_t *xdata)
 {
-        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+    return dict_get(xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
 }
 
 static unsigned long
-cdc_extract_crc (char *trailer)
+cdc_extract_crc(char *trailer)
 {
-        return cdc_get_long ((unsigned char *) &trailer[0]);
+    return cdc_get_long((unsigned char *)&trailer[0]);
 }
 
 static unsigned long
-cdc_extract_size (char *trailer)
+cdc_extract_size(char *trailer)
 {
-        return cdc_get_long ((unsigned char *) &trailer[4]);
+    return cdc_get_long((unsigned char *)&trailer[4]);
 }
 
 static int32_t
-cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
-                      unsigned long len)
+cdc_validate_inflate(cdc_info_t *ci, unsigned long crc, unsigned long len)
 {
-        return !((crc == ci->crc)
-                 /* inflated length is hidden inside
-                  * Zlib stream struct */
-                 && (len == ci->stream.total_out));
+    return !((crc == ci->crc)
+             /* inflated length is hidden inside
+              * Zlib stream struct */
+             && (len == ci->stream.total_out));
 }
 
 static int32_t
-do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+do_cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
 {
-        int            ret          = -1;
-        int            i            = 0;
-        int            len          = 0;
-        char          *inflte       = NULL;
-        char          *trailer      = NULL;
-        struct iovec   vec          = {0,};
-        unsigned long  computed_crc = 0;
-        unsigned long  computed_len = 0;
-
-        ret = inflateInit2 (&ci->stream, priv->window_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Zlib: Unable to initialize inflate");
-                goto out;
+    int ret = -1;
+    int i = 0;
+    int len = 0;
+    char *inflte = NULL;
+    char *trailer = NULL;
+    struct iovec vec = {
+        0,
+    };
+    unsigned long computed_crc = 0;
+    unsigned long computed_len = 0;
+
+    ret = inflateInit2(&ci->stream, priv->window_size);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Zlib: Unable to initialize inflate");
+        goto out;
+    }
+
+    vec = THIS_VEC(ci, 0);
+
+    trailer = (char *)(((char *)vec.iov_base) + vec.iov_len -
+                       GF_CDC_VALIDATION_SIZE);
+
+    /* CRC of uncompressed data */
+    computed_crc = cdc_extract_crc(trailer);
+
+    /* size of uncomrpessed data */
+    computed_len = cdc_extract_size(trailer);
+
+    gf_log(this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+           computed_crc, computed_len, ci->buffer_size);
+
+    inflte = vec.iov_base;
+    len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+    /* allocate buffer of the original length of the data */
+    ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+    if (ret)
+        goto out;
+
+    /* setup output buffer */
+    cdc_init_zlib_output_stream(priv, ci, 0);
+
+    /* setup input buffer */
+    ci->stream.next_in = (unsigned char *)inflte;
+    ci->stream.avail_in = len;
+
+    while (ci->stream.avail_in != 0) {
+        if (ci->stream.avail_out == 0) {
+            CURR_VEC(ci).iov_len = ci->buffer_size;
+
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret)
+                break;
+
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
         }
 
-        vec = THIS_VEC(ci, 0);
-
-        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
-                - GF_CDC_VALIDATION_SIZE);
-
-        /* CRC of uncompressed data */
-        computed_crc = cdc_extract_crc (trailer);
-
-        /* size of uncomrpessed data */
-        computed_len = cdc_extract_size (trailer);
-
-        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
-                computed_crc, computed_len, ci->buffer_size);
-
-        inflte = vec.iov_base ;
-        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
-
-        /* allocate buffer of the original length of the data */
-        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-        if (ret)
-                goto out;
-
-        /* setup output buffer */
-        cdc_init_zlib_output_stream (priv, ci, 0);
-
-        /* setup input buffer */
-        ci->stream.next_in = (unsigned char *) inflte;
-        ci->stream.avail_in = len;
-
-        while (ci->stream.avail_in != 0) {
-                if (ci->stream.avail_out == 0) {
-                        CURR_VEC(ci).iov_len = ci->buffer_size;
-
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret)
-                                break;
-
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
-
-                ret = inflate (&ci->stream, Z_NO_FLUSH);
-                if (ret == Z_STREAM_ERROR)
-                        break;
-        }
-
-        /* flush zlib buffer */
-        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
-        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Decompression Error: ret (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-
-        /* compute CRC of the uncompresses data to check for
-         * correctness */
-
-        for (i = 0; i < ci->ncount; i++) {
-                ci->crc = crc32 (ci->crc,
-                                 (const Bytef *) ci->vec[i].iov_base,
-                                 ci->vec[i].iov_len);
-        }
-
-        /* validate inflated data */
-        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Checksum or length mismatched in inflated data");
-        }
-
- out:
-        return ret;
+        ret = inflate(&ci->stream, Z_NO_FLUSH);
+        if (ret == Z_STREAM_ERROR)
+            break;
+    }
+
+    /* flush zlib buffer */
+    ret = cdc_flush_libz_buffer(priv, this, ci, inflate, Z_SYNC_FLUSH);
+    if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+        gf_log(this->name, GF_LOG_ERROR, "Decompression Error: ret (%d)", ret);
+        ret = -1;
+        goto out;
+    }
+
+    /* compute CRC of the uncompresses data to check for
+     * correctness */
+
+    for (i = 0; i < ci->ncount; i++) {
+        ci->crc = crc32(ci->crc, (const Bytef *)ci->vec[i].iov_base,
+                        ci->vec[i].iov_len);
+    }
+
+    /* validate inflated data */
+    ret = cdc_validate_inflate(ci, computed_crc, computed_len);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Checksum or length mismatched in inflated data");
+    }
+
+out:
+    return ret;
 }
 
 int32_t
-cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
-                dict_t *xdata)
+cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t *xdata)
 {
-        int32_t ret = -1;
-
-        /* check for deflate content */
-        if (!cdc_check_content_for_deflate (xdata)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Content not deflated, passing through ...");
-                goto passthrough_out;
-        }
-
-        ci->iobref = iobref_new ();
-        if (!ci->iobref)
-                goto passthrough_out;
-
-        /* do we need to do this? can we assume that one iovec
-         * will hold per request data every time?
-         *
-         * server/client protocol seems to deal with a single
-         * iovec even if op_ret > 1M. So, it looks ok to
-         * assume that a single iovec will contain all the
-         * data (This saves us a lot from finding the trailer
-         * and the data since it could have been split-up onto
-         * two adjacent iovec's.
-         *
-         * But, in case this translator is loaded above quick-read
-         * for some reason, then it's entirely possible that we get
-         * multiple iovec's...
-         *
-         * This case (handled below) is not tested. (by loading the
-         * xlator below quick-read)
-         */
-
-        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
-        if (ci->count > 1) {
-                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
-                        " multiple iovecs (%d in number)", ci->count);
-                goto inflate_cleanup_out;
-                /* TODO: coallate all iovecs in one */
-        }
-
-        ret = do_cdc_decompress (this, priv, ci);
-        if (ret)
-                goto inflate_cleanup_out;
-
-        ci->nbytes = ci->stream.total_out;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Inflated %ld to %ld bytes",
-                ci->stream.total_in, ci->stream.total_out);
-
- inflate_cleanup_out:
-        (void) inflateEnd (&ci->stream);
-
- passthrough_out:
-        return ret;
+    int32_t ret = -1;
+
+    /* check for deflate content */
+    if (!cdc_check_content_for_deflate(xdata)) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Content not deflated, passing through ...");
+        goto passthrough_out;
+    }
+
+    ci->iobref = iobref_new();
+    if (!ci->iobref)
+        goto passthrough_out;
+
+    /* do we need to do this? can we assume that one iovec
+     * will hold per request data every time?
+     *
+     * server/client protocol seems to deal with a single
+     * iovec even if op_ret > 1M. So, it looks ok to
+     * assume that a single iovec will contain all the
+     * data (This saves us a lot from finding the trailer
+     * and the data since it could have been split-up onto
+     * two adjacent iovec's.
+     *
+     * But, in case this translator is loaded above quick-read
+     * for some reason, then it's entirely possible that we get
+     * multiple iovec's...
+     *
+     * This case (handled below) is not tested. (by loading the
+     * xlator below quick-read)
+     */
+
+    /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+    if (ci->count > 1) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "unable to handle"
+               " multiple iovecs (%d in number)",
+               ci->count);
+        goto inflate_cleanup_out;
+        /* TODO: coallate all iovecs in one */
+    }
+
+    ret = do_cdc_decompress(this, priv, ci);
+    if (ret)
+        goto inflate_cleanup_out;
+
+    ci->nbytes = ci->stream.total_out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "Inflated %ld to %ld bytes",
+           ci->stream.total_in, ci->stream.total_out);
+
+inflate_cleanup_out:
+    (void)inflateEnd(&ci->stream);
+
+passthrough_out:
+    return ret;
 }
 
 #endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
index ead2c70ba6e..928afdd2efe 100644
--- a/xlators/features/compress/src/cdc-mem-types.h
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -11,13 +11,13 @@
 #ifndef __CDC_MEM_TYPES_H
 #define __CDC_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_cdc_mem_types {
-        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
-        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
-        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
-        gf_cdc_mt_end            = gf_common_mt_end + 4,
+    gf_cdc_mt_priv_t = gf_common_mt_end + 1,
+    gf_cdc_mt_vec_t = gf_common_mt_end + 2,
+    gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+    gf_cdc_mt_end = gf_common_mt_end + 4,
 };
 
 #endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
index e33d4efc1a1..b0b51e914ed 100644
--- a/xlators/features/compress/src/cdc.c
+++ b/xlators/features/compress/src/cdc.c
@@ -10,347 +10,339 @@
 
 #include <sys/uio.h>
 
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
 
 #include "cdc.h"
 #include "cdc-mem-types.h"
 
 static void
-cdc_cleanup_iobref (cdc_info_t *ci)
+cdc_cleanup_iobref(cdc_info_t *ci)
 {
-        assert(ci->iobref != NULL);
-        iobref_clear (ci->iobref);
+    assert(ci->iobref != NULL);
+    iobref_clear(ci->iobref);
 }
 
 int32_t
-cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count,
-               struct iatt *stbuf, struct iobref *iobref,
-               dict_t *xdata)
+cdc_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        int         ret   = -1;
-        cdc_priv_t *priv  = NULL;
-        cdc_info_t  ci    = {0,};
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
-
-        priv = this->private;
-
-        if (op_ret <= 0)
-                goto default_out;
-
-        if ( (priv->min_file_size != 0)
-             && (op_ret < priv->min_file_size) )
-                goto default_out;
-
-        ci.count       = count;
-        ci.ibytes      = op_ret;
-        ci.vector      = vector;
-        ci.buf         = NULL;
-        ci.iobref      = NULL;
-        ci.ncount      = 0;
-        ci.crc         = 0;
-        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
-
-/* A readv compresses on the server side and decompresses on the client side
- */
-        if (priv->op_mode == GF_CDC_MODE_SERVER) {
-                ret = cdc_compress (this, priv, &ci, &xdata);
-        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
-                ret = cdc_decompress (this, priv, &ci, xdata);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid operation mode (%d)", priv->op_mode);
-        }
-
-        if (ret)
-                goto default_out;
-
-        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
-                             ci.vec, ci.ncount, stbuf, iobref,
-                             xdata);
-        cdc_cleanup_iobref (&ci);
-        return 0;
-
- default_out:
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref, xdata);
-        return 0;
+    int ret = -1;
+    cdc_priv_t *priv = NULL;
+    cdc_info_t ci = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("cdc", this, default_out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, default_out);
+
+    priv = this->private;
+
+    if (op_ret <= 0)
+        goto default_out;
+
+    if ((priv->min_file_size != 0) && (op_ret < priv->min_file_size))
+        goto default_out;
+
+    ci.count = count;
+    ci.ibytes = op_ret;
+    ci.vector = vector;
+    ci.buf = NULL;
+    ci.iobref = NULL;
+    ci.ncount = 0;
+    ci.crc = 0;
+    ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+    /* A readv compresses on the server side and decompresses on the client side
+     */
+    if (priv->op_mode == GF_CDC_MODE_SERVER) {
+        ret = cdc_compress(this, priv, &ci, &xdata);
+    } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+        ret = cdc_decompress(this, priv, &ci, xdata);
+    } else {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid operation mode (%d)",
+               priv->op_mode);
+    }
+
+    if (ret)
+        goto default_out;
+
+    STACK_UNWIND_STRICT(readv, frame, ci.nbytes, op_errno, ci.vec, ci.ncount,
+                        stbuf, iobref, xdata);
+    cdc_cleanup_iobref(&ci);
+    return 0;
+
+default_out:
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
+    return 0;
 }
 
 int32_t
-cdc_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags,
-           dict_t *xdata)
+cdc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        fop_readv_cbk_t cbk = NULL;
+    fop_readv_cbk_t cbk = NULL;
 
 #ifdef HAVE_LIB_Z
-        cbk = cdc_readv_cbk;
+    cbk = cdc_readv_cbk;
 #else
-        cbk = default_readv_cbk;
+    cbk = default_readv_cbk;
 #endif
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+               fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int32_t
-cdc_writev_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+cdc_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-cdc_writev (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd,
-            struct iovec *vector,
-            int32_t count,
-            off_t offset,
-            uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+cdc_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        int          ret   = -1;
-        cdc_priv_t  *priv  = NULL;
-        cdc_info_t   ci    = {0,};
-        size_t       isize = 0;
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
-
-        priv = this->private;
-
-        isize =  iov_length(vector, count);
-
-        if (isize <= 0)
-            goto default_out;
-
-        if ( (priv->min_file_size != 0) && (isize < priv->min_file_size) )
-            goto default_out;
-
-        ci.count       = count;
-        ci.ibytes      = isize;
-        ci.vector      = vector;
-        ci.buf         = NULL;
-        ci.iobref      = NULL;
-        ci.ncount      = 0;
-        ci.crc         = 0;
-        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
-
-/* A writev compresses on the client side and decompresses on the server side
- */
-	    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
-		    ret = cdc_compress (this, priv, &ci, &xdata);
-	    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
-		    ret = cdc_decompress (this, priv, &ci, xdata);
-	    } else {
-		    gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
-	    }
-
-	    if (ret)
-		    goto default_out;
-
-	    STACK_WIND (frame,
-                    cdc_writev_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev,
-                    fd, ci.vec, ci.ncount, offset, flags,
-                    iobref, xdata);
-
-        cdc_cleanup_iobref (&ci);
-        return 0;
-
- default_out:
-        STACK_WIND (frame,
-                    cdc_writev_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, flags,
-                    iobref, xdata);
-        return 0;
+    int ret = -1;
+    cdc_priv_t *priv = NULL;
+    cdc_info_t ci = {
+        0,
+    };
+    size_t isize = 0;
+
+    GF_VALIDATE_OR_GOTO("cdc", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, frame, err);
+
+    priv = this->private;
+
+    isize = iov_length(vector, count);
+
+    if (isize <= 0)
+        goto default_out;
+
+    if ((priv->min_file_size != 0) && (isize < priv->min_file_size))
+        goto default_out;
+
+    ci.count = count;
+    ci.ibytes = isize;
+    ci.vector = vector;
+    ci.buf = NULL;
+    ci.iobref = NULL;
+    ci.ncount = 0;
+    ci.crc = 0;
+    ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+    /* A writev compresses on the client side and decompresses on the server
+     * side
+     */
+    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+        ret = cdc_compress(this, priv, &ci, &xdata);
+    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+        ret = cdc_decompress(this, priv, &ci, xdata);
+    } else {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ",
+               priv->op_mode);
+    }
+
+    if (ret)
+        goto default_out;
+
+    STACK_WIND(frame, cdc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, ci.vec, ci.ncount, offset,
+               flags, iobref, xdata);
+
+    cdc_cleanup_iobref(&ci);
+    return 0;
+
+default_out:
+    STACK_WIND(frame, cdc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(writev, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_cdc_mt_end);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_cdc_mt_end);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int         ret      = -1;
-        char       *temp_str = NULL;
-        cdc_priv_t *priv     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, err);
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Need subvolume == 1");
-                goto err;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Dangling volume. Check volfile");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
-        if (!priv) {
-                goto err;
-        }
-
-        /* Check if debug mode is turned on */
-        GF_OPTION_INIT ("debug", priv->debug, bool, err);
-        if( priv->debug ) {
-                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
-        }
-
-        /* Set Gzip Window Size */
-        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
-        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
-             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip window size (%d), using default",
-                        priv->window_size);
-                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
-        }
-
-        /* Set Gzip (De)Compression Level */
-        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
-        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
-             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip (de)compression level (%d),"
-                        " using default", priv->cdc_level);
-                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
-        }
-
-        /* Set Gzip Memory Level */
-        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
-        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip memory level, using the default");
-                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
-        }
-
-        /* Set min file size to enable compression */
-        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
-
-        /* Mode of operation - Server/Client */
-        ret = dict_get_str (this->options, "mode", &temp_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Operation mode not specified !!");
-                goto err;
-        }
-
-        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
-                priv->op_mode = GF_CDC_MODE_CLIENT;
-        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
-                priv->op_mode = GF_CDC_MODE_SERVER;
-        } else {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Bogus operation mode (%s) specified", temp_str);
-                goto err;
-        }
-
-        this->private = priv;
-        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
-        return 0;
-
- err:
-        if (priv)
-                GF_FREE (priv);
-
-        return -1;
+    int ret = -1;
+    char *temp_str = NULL;
+    cdc_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("cdc", this, err);
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR, "Need subvolume == 1");
+        goto err;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "Dangling volume. Check volfile");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_cdc_mt_priv_t);
+    if (!priv) {
+        goto err;
+    }
+
+    /* Check if debug mode is turned on */
+    GF_OPTION_INIT("debug", priv->debug, bool, err);
+    if (priv->debug) {
+        gf_log(this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+    }
+
+    /* Set Gzip Window Size */
+    GF_OPTION_INIT("window-size", priv->window_size, int32, err);
+    if ((priv->window_size > GF_CDC_MAX_WINDOWSIZE) ||
+        (priv->window_size < GF_CDC_DEF_WINDOWSIZE)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip window size (%d), using default",
+               priv->window_size);
+        priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+    }
+
+    /* Set Gzip (De)Compression Level */
+    GF_OPTION_INIT("compression-level", priv->cdc_level, int32, err);
+    if (((priv->cdc_level < 1) || (priv->cdc_level > 9)) &&
+        (priv->cdc_level != GF_CDC_DEF_COMPRESSION)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip (de)compression level (%d),"
+               " using default",
+               priv->cdc_level);
+        priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+    }
+
+    /* Set Gzip Memory Level */
+    GF_OPTION_INIT("mem-level", priv->mem_level, int32, err);
+    if ((priv->mem_level < 1) || (priv->mem_level > 9)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip memory level, using the default");
+        priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+    }
+
+    /* Set min file size to enable compression */
+    GF_OPTION_INIT("min-size", priv->min_file_size, int32, err);
+
+    /* Mode of operation - Server/Client */
+    ret = dict_get_str(this->options, "mode", &temp_str);
+    if (ret) {
+        gf_log(this->name, GF_LOG_CRITICAL, "Operation mode not specified !!");
+        goto err;
+    }
+
+    if (GF_CDC_MODE_IS_CLIENT(temp_str)) {
+        priv->op_mode = GF_CDC_MODE_CLIENT;
+    } else if (GF_CDC_MODE_IS_SERVER(temp_str)) {
+        priv->op_mode = GF_CDC_MODE_SERVER;
+    } else {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "Bogus operation mode (%s) specified", temp_str);
+        goto err;
+    }
+
+    this->private = priv;
+    gf_log(this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",
+           temp_str);
+    return 0;
+
+err:
+    if (priv)
+        GF_FREE(priv);
+
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        cdc_priv_t *priv = this->private;
+    cdc_priv_t *priv = this->private;
 
-        if (priv)
-                GF_FREE (priv);
-        this->private = NULL;
-        return;
+    if (priv)
+        GF_FREE(priv);
+    this->private = NULL;
+    return;
 }
 
 struct xlator_fops fops = {
-        .readv  = cdc_readv,
-        .writev = cdc_writev,
+    .readv = cdc_readv,
+    .writev = cdc_writev,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-        { .key  = {"window-size"},
-          .default_value = "-15",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Size of the zlib history buffer."
-        },
-        { .key  = {"mem-level"},
-          .default_value = "8",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Memory allocated for internal compression state. "
-                         "1 uses minimum memory but is slow and reduces "
-                         "compression ratio; memLevel=9 uses maximum memory "
-                         "for optimal speed. The default value is 8."
-        },
-        { .key  = {"compression-level"},
-          .default_value = "-1",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Compression levels \n"
-                         "0 : no compression, 1 : best speed, \n"
-                         "9 : best compression, -1 : default compression "
-        },
-        { .key  = {"min-size"},
-          .default_value = "0",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Data is compressed only when its size exceeds this."
-        },
-        { .key  = {"mode"},
-          .value = {"server", "client"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Set on the basis of where the xlator is loaded. "
-                         "This option should NOT be configured by user."
-        },
-        { .key = {"debug"},
-          .default_value = "false",
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "This is used in testing. Will dump compressed data "
-                         "to disk as a gzip file."
-        },
-        { .key  = {NULL}
-        },
+    {.key = {"window-size"},
+     .default_value = "-15",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Size of the zlib history buffer."},
+    {.key = {"mem-level"},
+     .default_value = "8",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Memory allocated for internal compression state. "
+                    "1 uses minimum memory but is slow and reduces "
+                    "compression ratio; memLevel=9 uses maximum memory "
+                    "for optimal speed. The default value is 8."},
+    {.key = {"compression-level"},
+     .default_value = "-1",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Compression levels \n"
+                    "0 : no compression, 1 : best speed, \n"
+                    "9 : best compression, -1 : default compression "},
+    {.key = {"min-size"},
+     .default_value = "0",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Data is compressed only when its size exceeds this."},
+    {.key = {"mode"},
+     .value = {"server", "client"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Set on the basis of where the xlator is loaded. "
+                    "This option should NOT be configured by user."},
+    {.key = {"debug"},
+     .default_value = "false",
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "This is used in testing. Will dump compressed data "
+                    "to disk as a gzip file."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_9_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "cdc",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
index 71f4d2317bb..cb87b06a989 100644
--- a/xlators/features/compress/src/cdc.h
+++ b/xlators/features/compress/src/cdc.h
@@ -15,41 +15,41 @@
 #include "zlib.h"
 #endif
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 #ifndef MAX_IOVEC
 #define MAX_IOVEC 16
 #endif
 
 typedef struct cdc_priv {
-        int window_size;
-        int mem_level;
-        int cdc_level;
-        int min_file_size;
-        int op_mode;
-        gf_boolean_t debug;
-        gf_lock_t lock;
+    int window_size;
+    int mem_level;
+    int cdc_level;
+    int min_file_size;
+    int op_mode;
+    gf_boolean_t debug;
+    gf_lock_t lock;
 } cdc_priv_t;
 
 typedef struct cdc_info {
-        /* input bits */
-        int            count;
-        int32_t        ibytes;
-        struct iovec  *vector;
-        struct iatt   *buf;
-
-        /* output bits */
-        int            ncount;
-        int            nbytes;
-        int            buffer_size;
-        struct iovec   vec[MAX_IOVEC];
-        struct iobref *iobref;
-
-        /* zlib bits */
+    /* input bits */
+    int count;
+    int32_t ibytes;
+    struct iovec *vector;
+    struct iatt *buf;
+
+    /* output bits */
+    int ncount;
+    int nbytes;
+    int buffer_size;
+    struct iovec vec[MAX_IOVEC];
+    struct iobref *iobref;
+
+    /* zlib bits */
 #ifdef HAVE_LIB_Z
-        z_stream      stream;
+    z_stream stream;
 #endif
-        unsigned long crc;
+    unsigned long crc;
 } cdc_info_t;
 
 #define NVEC(ci) (ci->ncount - 1)
@@ -57,8 +57,8 @@ typedef struct cdc_info {
 #define THIS_VEC(ci, i) ci->vector[i]
 
 /* Gzip defaults */
-#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
-#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+#define GF_CDC_DEF_WINDOWSIZE -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE -8  /* max value     */
 
 #ifdef HAVE_LIB_Z
 #define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
@@ -66,15 +66,15 @@ typedef struct cdc_info {
 #define GF_CDC_DEF_COMPRESSION -1
 #endif
 
-#define GF_CDC_DEF_MEMLEVEL    8
-#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+#define GF_CDC_DEF_MEMLEVEL 8
+#define GF_CDC_DEF_BUFFERSIZE 262144  // 256K - default compression buffer size
 
 /* Operation mode
  * If xlator is loaded on client, readv decompresses and writev compresses
  * If xlator is loaded on server, readv compresses and writev decompresses
  */
-#define GF_CDC_MODE_CLIENT   0
-#define GF_CDC_MODE_SERVER   1
+#define GF_CDC_MODE_CLIENT 0
+#define GF_CDC_MODE_SERVER 1
 
 /* min size of data to do cmpression
  * 0 == compress even 1byte
@@ -87,21 +87,13 @@ typedef struct cdc_info {
 #define GF_CDC_DEFLATE_CANARY_VAL "deflate"
 #define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
 
-#define GF_CDC_MODE_IS_CLIENT(m) \
-        (strcmp (m, "client") == 0)
+#define GF_CDC_MODE_IS_CLIENT(m) (strcmp(m, "client") == 0)
 
-#define GF_CDC_MODE_IS_SERVER(m) \
-        (strcmp (m, "server") == 0)
+#define GF_CDC_MODE_IS_SERVER(m) (strcmp(m, "server") == 0)
 
 int32_t
-cdc_compress (xlator_t *this,
-              cdc_priv_t *priv,
-              cdc_info_t *ci,
-              dict_t **xdata);
+cdc_compress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t **xdata);
 int32_t
-cdc_decompress (xlator_t *this,
-                cdc_priv_t *priv,
-                cdc_info_t *ci,
-                dict_t *xdata);
+cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t *xdata);
 
 #endif
diff --git a/xlators/features/filter/Makefile.am b/xlators/features/filter/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/filter/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/filter/src/Makefile.am b/xlators/features/filter/src/Makefile.am
deleted file mode 100644
index 5bdc711ae07..00000000000
--- a/xlators/features/filter/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-xlator_LTLIBRARIES = filter.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
-
-filter_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-filter_la_SOURCES = filter.c
-filter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
-
-noinst_HEADERS = filter-mem-types.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/features/filter/src/filter.c b/xlators/features/filter/src/filter.c
deleted file mode 100644
index 3fd7dc8c8fb..00000000000
--- a/xlators/features/filter/src/filter.c
+++ /dev/null
@@ -1,1729 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "filter-mem-types.h"
-
-#define GF_FILTER_NOBODY_UID         65534
-#define GF_FILTER_NOBODY_GID         65534
-#define GF_FILTER_ROOT_UID           0
-#define GF_FILTER_ROOT_GID           0
-
-#define GF_MAXIMUM_FILTERING_ALLOWED 32
-
-/*
-  option root-filtering on (off by default)
-  option translate-uid <uid-range=newuid,uid=newuid>
-  option translate-gid <gid-range=newgid,gid=newgid>
-  option read-only <yes|true>
-  option fixed-uid <uid>
-  option fixed-gid <gid>
-  option filter-uid <uid-range,uid>
-  option filter-gid <gid-range,gid> // not supported yet
-
-*/
-
-struct gf_filter {
-	/* Flags */
-	gf_boolean_t complete_read_only;
-	char fixed_uid_set;
-	char fixed_gid_set;
-	char partial_filter;
-
-	/* Options */
-	/* Mapping/Filtering/Translate whatever you want to call */
-	int translate_num_uid_entries;
-	int translate_num_gid_entries;
-	int translate_input_uid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	int translate_output_uid[GF_MAXIMUM_FILTERING_ALLOWED];
-	int translate_input_gid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	int translate_output_gid[GF_MAXIMUM_FILTERING_ALLOWED];
-
-	/* Fixed uid/gid */
-	int fixed_uid;
-	int fixed_gid;
-
-	/* Filter */
-	int filter_num_uid_entries;
-	int filter_num_gid_entries;
-	int filter_input_uid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	int filter_input_gid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	
-};
-
-/* update_frame: The main logic of the whole translator.
-   Return values:
-   0: no change
-   // TRANSLATE
-   1: only uid changed 
-   2: only gid changed
-   3: both uid/gid changed
-   // FILTER
-   4: uid in filter range
-   5: gid in filter range  // not supported yet
-   6: complete fs is readonly
-*/
-
-#define GF_FILTER_NO_CHANGE    0
-#define	GF_FILTER_MAP_UID      1
-#define GF_FILTER_MAP_GID      2
-#define GF_FILTER_MAP_BOTH     3
-#define GF_FILTER_FILTER_UID   4
-#define GF_FILTER_FILTER_GID   5
-#define GF_FILTER_RO_FS        6
-
-static int32_t
-update_frame (call_frame_t *frame,
-	      inode_t *inode,
-	      struct gf_filter *filter)
-{
-	uid_t    uid = 0;
-	int32_t  idx = 0;
-	int32_t  ret = 0;
-	int32_t  dictret = 0;
-	uint64_t tmp_uid = 0;
-	
-	for (idx = 0; idx < filter->translate_num_uid_entries; idx++) {
-		if ((frame->root->uid >=filter->translate_input_uid[idx][0]) &&
-		    (frame->root->uid <=filter->translate_input_uid[idx][1])) {
-			dictret = inode_ctx_get (inode, frame->this, &tmp_uid);
-			uid = (uid_t)tmp_uid;
-			if (dictret == 0) {
-				if (frame->root->uid != uid)
-					ret = GF_FILTER_MAP_UID;
-			} else {
-				ret = GF_FILTER_MAP_UID;
-			}
-			break;
-		}
-	}
-	
-	for (idx = 0; idx < filter->translate_num_gid_entries; idx++) {
-		if ((frame->root->gid >=filter->translate_input_gid[idx][0]) &&
-		    (frame->root->gid <=filter->translate_input_gid[idx][1])) {
-			if (ret == GF_FILTER_NO_CHANGE) 
-				ret = GF_FILTER_MAP_GID;
-			else 
-				ret = GF_FILTER_MAP_BOTH;
-			break;
-		}
-	}
-
-
-	if (filter->complete_read_only)
-		return GF_FILTER_RO_FS;
-	
-	if (filter->partial_filter) {
-		dictret = inode_ctx_get (inode, frame->this, &tmp_uid);
-		uid = (uid_t)tmp_uid;
-		if (dictret != -1) {
-			for (idx = 0; idx < filter->filter_num_uid_entries; 
-			     idx++) {
-				if ((uid >=filter->filter_input_uid[idx][0]) &&
-				    (uid <=filter->filter_input_uid[idx][1])) {
-					return GF_FILTER_FILTER_UID;
-				}
-			}
-		}
-	}
-
-	return ret;
-}
-
-/* if 'root' don't change the uid/gid */
-static int32_t
-update_stat (struct iatt *stbuf,
-	     struct gf_filter *filter)
-{
-	int32_t idx = 0;
-	for (idx = 0; idx < filter->translate_num_uid_entries; idx++) {
-		if (stbuf->ia_uid == GF_FILTER_ROOT_UID)
-			continue;
-		if ((stbuf->ia_uid >= filter->translate_input_uid[idx][0]) &&
-		    (stbuf->ia_uid <= filter->translate_input_uid[idx][1])) {
-			stbuf->ia_uid = filter->translate_output_uid[idx];
-			break;
-		}
-	}
-	
-	for (idx = 0; idx < filter->translate_num_gid_entries; idx++) {
-		if (stbuf->ia_gid == GF_FILTER_ROOT_GID)
-			continue;
-		if ((stbuf->ia_gid >= filter->translate_input_gid[idx][0]) &&
-		    (stbuf->ia_gid <= filter->translate_input_gid[idx][1])) {
-			stbuf->ia_gid = filter->translate_output_gid[idx];
-			break;
-		}
-	}
-
-	if (filter->fixed_uid_set) {
-		stbuf->ia_uid = filter->fixed_uid;
-	}
-
-	if (filter->fixed_gid_set) {
-		stbuf->ia_gid = filter->fixed_gid;
-	}
-	
-	return 0;
-}
-
-static int32_t 
-filter_lookup_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-		   struct iatt *buf,
-		   dict_t *dict,
-                   struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict, postparent);
-	return 0;
-}
-
-int32_t 
-filter_lookup (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       dict_t *xattr_req)
-{
-	STACK_WIND (frame,
-		    filter_lookup_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup,
-		    loc,
-		    xattr_req);
-	return 0;
-}
-
-
-static int32_t
-filter_stat_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iatt *buf)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t
-filter_stat (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	STACK_WIND (frame,
-		    filter_stat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->stat,
-		    loc);
-	return 0;
-}
-
-static int32_t
-filter_setattr_cbk (call_frame_t *frame,
-                    void *cookie,
-                    xlator_t *this,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    struct iatt *preop,
-                    struct iatt *postop)
-{
-	if (op_ret >= 0) {
-		update_stat (preop, this->private);
-		update_stat (postop, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, preop, postop);
-	return 0;
-}
-
-int32_t
-filter_setattr (call_frame_t *frame,
-                xlator_t *this,
-                loc_t *loc,
-                struct iatt *stbuf,
-                int32_t valid)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	default:
-		break;
-	}
-
-	STACK_WIND (frame,
-		    filter_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr,
-		    loc,
-		    stbuf, valid);
-	return 0;
-}
-
-static int32_t
-filter_fsetattr_cbk (call_frame_t *frame,
-                     void *cookie,
-                     xlator_t *this,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     struct iatt *preop,
-                     struct iatt *postop)
-{
-	if (op_ret >= 0) {
-                update_stat (preop, this->private);
-		update_stat (postop, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      preop, postop);
-	return 0;
-}
-
-int32_t
-filter_fsetattr (call_frame_t *frame,
-                 xlator_t *this,
-                 fd_t *fd,
-                 struct iatt *stbuf,
-                 int32_t valid)
-{
-	STACK_WIND (frame,
-		    filter_fsetattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd,
-		    stbuf, valid);
-	return 0;
-}
-
-
-static int32_t
-filter_truncate_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *prebuf,
-                     struct iatt *postbuf)
-{
-	if (op_ret >= 0) {
-		update_stat (prebuf, this->private);
-		update_stat (postbuf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-int32_t
-filter_truncate (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 off_t offset)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
-		return 0;
-		
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_truncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->truncate,
-		    loc,
-		    offset);
-	return 0;
-}
-
-static int32_t
-filter_ftruncate_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-	if (op_ret >= 0) {
-		update_stat (prebuf, this->private);
-		update_stat (postbuf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-int32_t
-filter_ftruncate (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  off_t offset)
-{
-	STACK_WIND (frame,
-		    filter_ftruncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd,
-		    offset);
-	return 0;
-}
-
-
-static int32_t
-filter_readlink_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     const char *path,
-                     struct iatt *sbuf)
-{
-        if (op_ret >= 0)
-                update_stat (sbuf, this->private);
-
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-int32_t
-filter_readlink (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 size_t size)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IRGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IROTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
-		return 0;
-	}			
-	STACK_WIND (frame,
-		    filter_readlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readlink,
-		    loc,
-		    size);
-	return 0;
-}
-
-
-static int32_t
-filter_mknod_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_mknod (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode,
-	      dev_t rdev)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_mknod_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev);
-	return 0;
-}
-
-static int32_t
-filter_mkdir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_mkdir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_mkdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode);
-	return 0;
-}
-
-static int32_t
-filter_unlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        if (op_ret >= 0) {
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-        }
-
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_unlink (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc)
-{
-	int32_t ret = 0;
-	inode_t *parent = loc->parent;
-	if (!parent)
-		parent = inode_parent (loc->inode, 0, NULL);
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_unlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
-}
-
-static int32_t
-filter_rmdir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        if (op_ret >= 0) {
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-        }
-
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_rmdir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	int32_t ret = 0;
-	inode_t *parent = loc->parent;
-	if (!parent)
-		parent = inode_parent (loc->inode, 0, NULL);
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-                STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-                return 0;
-	}			
-	STACK_WIND (frame,
-		    filter_rmdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rmdir,
-		    loc);
-	return 0;
-}
-
-static int32_t
-filter_symlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_symlink (call_frame_t *frame,
-		xlator_t *this,
-		const char *linkpath,
-		loc_t *loc)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_symlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc);
-	return 0;
-}
-
-
-static int32_t
-filter_rename_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct iatt *buf,
-                   struct iatt *preoldparent,
-                   struct iatt *postoldparent,
-                   struct iatt *prenewparent,
-                   struct iatt *postnewparent)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-
-		update_stat (preoldparent, this->private);
-		update_stat (postoldparent, this->private);
-
-		update_stat (prenewparent, this->private);
-		update_stat (postnewparent, this->private);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf,
-                      preoldparent, postoldparent,
-                      prenewparent, postnewparent);
-	return 0;
-}
-
-int32_t
-filter_rename (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *oldloc,
-	       loc_t *newloc)
-{
-	int32_t ret = 0;
-	inode_t *parent = oldloc->parent;
-	if (!parent)
-		parent = inode_parent (oldloc->inode, 0, NULL);
-	ret = update_frame (frame, oldloc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-		if (oldloc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		if (oldloc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s -> %s: returning permission denied", oldloc->path, newloc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL,
-                              NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL,
-                              NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_rename_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc);
-	return 0;
-}
-
-
-static int32_t
-filter_link_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_link (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *oldloc,
-	     loc_t *newloc)
-{
-	int ret = 0;
-	ret = update_frame (frame, oldloc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_link_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc);
-	return 0;
-}
-
-
-static int32_t
-filter_create_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd,
-		   inode_t *inode,
-		   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_create (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       int32_t flags,
-	       mode_t mode, fd_t *fd)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame, filter_create_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd);
-	return 0;
-}
-
-static int32_t
-filter_open_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-int32_t
-filter_open (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t flags, 
-	     fd_t *fd,
-             int32_t wbflags)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-		if (!(((flags & O_ACCMODE) == O_WRONLY)
-                      || ((flags & O_ACCMODE) == O_RDWR))
-		    && (loc->inode->st_mode & S_IRGRP))
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-				break;
-		if (!(((flags & O_ACCMODE) == O_WRONLY)
-                      || ((flags & O_ACCMODE) == O_RDWR))
-		    && (loc->inode->st_mode & S_IROTH))
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s: returning permission denied (mode: 0%o, flag=0%o)", 
-			loc->path, loc->inode->st_mode, flags);
-		STACK_UNWIND (frame, -1, EPERM, fd);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		if (!(((flags & O_ACCMODE) == O_WRONLY)
-                      || ((flags & O_ACCMODE) == O_RDWR)))
-			break;
-		STACK_UNWIND (frame, -1, EROFS, NULL);
-		return 0;
-		
-	}
-	STACK_WIND (frame,
-		    filter_open_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, wbflags);
-	return 0;
-}
-
-static int32_t
-filter_readv_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iovec *vector,
-		  int32_t count,
-		  struct iatt *stbuf,
-                  struct iobref *iobref)
-{
-	if (op_ret >= 0) {
-		update_stat (stbuf, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      vector,
-		      count,
-		      stbuf,
-                      iobref);
-	return 0;
-}
-
-int32_t
-filter_readv (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      size_t size,
-	      off_t offset)
-{
-	STACK_WIND (frame,
-		    filter_readv_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readv,
-		    fd,
-		    size,
-		    offset);
-	return 0;
-}
-
-
-static int32_t
-filter_writev_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *prebuf,
-		   struct iatt *postbuf)
-{
-	if (op_ret >= 0) {
-		update_stat (prebuf, this->private);
-		update_stat (postbuf, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      prebuf,
-                      postbuf);
-	return 0;
-}
-
-int32_t
-filter_writev (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       struct iovec *vector,
-	       int32_t count,
-	       off_t off,
-               struct iobref *iobref)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, fd->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    filter_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off,
-                    iobref);
-	return 0;
-}
-
-static int32_t
-filter_fstat_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
-}
-
-int32_t
-filter_fstat (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd)
-{
-	STACK_WIND (frame,
-		    filter_fstat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
-	return 0;
-}
-
-static int32_t
-filter_opendir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    fd_t *fd)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      fd);
-	return 0;
-}
-
-int32_t
-filter_opendir (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc, fd_t *fd)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-		if (loc->inode->st_mode & S_IRGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		if (loc->inode->st_mode & S_IROTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, fd);
-		return 0;
-	}			
-	STACK_WIND (frame,
-		    filter_opendir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd);
-	return 0;
-}
-
-
-static int32_t
-filter_setxattr_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
-}
-
-int32_t
-filter_setxattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 dict_t *dict,
-		 int32_t flags)
-{
-
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_setxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setxattr,
-		    loc,
-		    dict,
-		    flags);
-	return 0;
-}
-
-static int32_t
-filter_getxattr_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *dict)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      dict);
-	return 0;
-}
-
-int32_t
-filter_getxattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 const char *name)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IRGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IROTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_getxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc,
-		    name);
-	return 0;
-}
-
-static int32_t
-filter_removexattr_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t
-filter_removexattr (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    const char *name)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_removexattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->removexattr,
-		    loc,
-		    name);
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_filter_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t 
-init (xlator_t *this)
-{
-	char *value = NULL;
-	char *tmp_str = NULL;
-	char *tmp_str1 = NULL;
-	char *tmp_str2 = NULL;
-	char *dup_str = NULL;
-	char *input_value_str1 = NULL;
-	char *input_value_str2 = NULL;
-	char *output_value_str = NULL;
-	int32_t input_value = 0;
-	int32_t output_value = 0;
-	data_t *option_data = NULL;
-	struct gf_filter *filter = NULL;
-	gf_boolean_t tmp_bool = 0;
-
-	if (!this->children || this->children->next) {
-		gf_log (this->name,
-			GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
-	
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	filter = GF_CALLOC (sizeof (*filter), 1, gf_filter_mt_gf_filter);
-	ERR_ABORT (filter);
-	
-	if (dict_get (this->options, "read-only")) {
-		value = data_to_str (dict_get (this->options, "read-only"));
-		if (gf_string2boolean (value, &filter->complete_read_only) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"wrong value provided for 'read-only'");
-			return -1;
-		}
-	}
-
-	if (dict_get (this->options, "root-squashing")) {
-		value = data_to_str (dict_get (this->options, "root-squashing"));
-		if (gf_string2boolean (value, &tmp_bool) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"wrong value provided for 'root-squashing'");
-			return -1;
-		}
-		if (tmp_bool) {
-			filter->translate_num_uid_entries = 1;
-			filter->translate_num_gid_entries = 1;
-			filter->translate_input_uid[0][0] = GF_FILTER_ROOT_UID; /* root */
-			filter->translate_input_uid[0][1] = GF_FILTER_ROOT_UID; /* root */
-			filter->translate_input_gid[0][0] = GF_FILTER_ROOT_GID; /* root */
-			filter->translate_input_gid[0][1] = GF_FILTER_ROOT_GID; /* root */
-			filter->translate_output_uid[0] = GF_FILTER_NOBODY_UID;
-			filter->translate_output_gid[0] = GF_FILTER_NOBODY_GID;
-		}
-	}
-
-	if (dict_get (this->options, "translate-uid")) {
-		option_data = dict_get (this->options, "translate-uid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			input_value_str1 = strtok_r (dup_str, "=", &tmp_str1);
-			if (input_value_str1) {
-				/* Check for n-m */
-				char *temp_string = gf_strdup (input_value_str1);
-				input_value_str2 = strtok_r (temp_string, "-", &tmp_str2);
-				if (gf_string2int (input_value_str2, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str2);
-					return -1;
-				}
-				filter->translate_input_uid[filter->translate_num_uid_entries][0] = input_value;
-				input_value_str2 = strtok_r (NULL, "-", &tmp_str2);
-				if (input_value_str2) {
-					if (gf_string2int (input_value_str2, &input_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							input_value_str2);
-						return -1;
-					}
-				}
-				filter->translate_input_uid[filter->translate_num_uid_entries][1] = input_value;
-				GF_FREE (temp_string);
-				output_value_str = strtok_r (NULL, "=", &tmp_str1);
-				if (output_value_str) {
-					if (gf_string2int (output_value_str, &output_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							output_value_str);
-						return -1;
-					}
-				} else {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"mapping string not valid");
-					return -1;
-				}
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"mapping string not valid");
-				return -1;
-			}
-			filter->translate_output_uid[filter->translate_num_uid_entries]   = output_value;
-			gf_log (this->name, 
-				GF_LOG_DEBUG, 
-				"pair %d: input uid '%d' will be changed to uid '%d'", 
-				filter->translate_num_uid_entries, input_value, output_value);
-
-			filter->translate_num_uid_entries++;
-			if (filter->translate_num_uid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-	}
-
-	tmp_str1 = NULL;
-	tmp_str2 = NULL;
-	tmp_str  = NULL;
-
-	if (dict_get (this->options, "translate-gid")) {
-		option_data = dict_get (this->options, "translate-gid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			input_value_str1 = strtok_r (dup_str, "=", &tmp_str1);
-			if (input_value_str1) {
-				/* Check for n-m */
-				char *temp_string = gf_strdup (input_value_str1);
-				input_value_str2 = strtok_r (temp_string, "-", &tmp_str2);
-				if (gf_string2int (input_value_str2, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str2);
-					return -1;
-				}
-				filter->translate_input_gid[filter->translate_num_gid_entries][0] = input_value;
-				input_value_str2 = strtok_r (NULL, "-", &tmp_str2);
-				if (input_value_str2) {
-					if (gf_string2int (input_value_str2, &input_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							input_value_str2);
-						return -1;
-					}
-				}
-				filter->translate_input_gid[filter->translate_num_gid_entries][1] = input_value;
-				GF_FREE (temp_string);
-				output_value_str = strtok_r (NULL, "=", &tmp_str1);
-				if (output_value_str) {
-					if (gf_string2int (output_value_str, &output_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							output_value_str);
-						return -1;
-					}
-				} else {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"translate-gid value not valid");
-					return -1;
-				}
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"translate-gid value not valid");
-				return -1;
-			}
-			
-			filter->translate_output_gid[filter->translate_num_gid_entries] = output_value;
-			
-			gf_log (this->name, GF_LOG_DEBUG, 
-				"pair %d: input gid '%d' will be changed to gid '%d'", 
-				filter->translate_num_gid_entries, input_value, output_value);
-			
-			filter->translate_num_gid_entries++;
-			if (filter->translate_num_gid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-	}
-
-	tmp_str  = NULL;
-	tmp_str1 = NULL;
-
-	if (dict_get (this->options, "filter-uid")) {
-		option_data = dict_get (this->options, "filter-uid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			/* Check for n-m */
-			input_value_str1 = strtok_r (dup_str, "-", &tmp_str1);
-			if (gf_string2int (input_value_str1, &input_value) != 0) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"invalid number format \"%s\"", 
-					input_value_str1);
-				return -1;
-			}
-			filter->filter_input_uid[filter->filter_num_uid_entries][0] = input_value;
-			input_value_str1 = strtok_r (NULL, "-", &tmp_str1);
-			if (input_value_str1) {
-				if (gf_string2int (input_value_str1, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str1);
-					return -1;
-				}
-			}
-			filter->filter_input_uid[filter->filter_num_uid_entries][1] = input_value;
-
-			gf_log (this->name, 
-				GF_LOG_DEBUG, 
-				"filter [%d]: input uid(s) '%s' will be filtered", 
-				filter->filter_num_uid_entries, dup_str);
-			
-			filter->filter_num_uid_entries++;
-			if (filter->filter_num_uid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-		filter->partial_filter = 1;
-	}
-
-	tmp_str  = NULL;
-	tmp_str1 = NULL;
-
-	if (dict_get (this->options, "filter-gid")) {
-		option_data = dict_get (this->options, "filter-gid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			/* Check for n-m */
-			input_value_str1 = strtok_r (dup_str, "-", &tmp_str1);
-			if (gf_string2int (input_value_str1, &input_value) != 0) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"invalid number format \"%s\"", 
-					input_value_str1);
-				return -1;
-			}
-			filter->filter_input_gid[filter->filter_num_gid_entries][0] = input_value;
-			input_value_str1 = strtok_r (NULL, "-", &tmp_str1);
-			if (input_value_str1) {
-				if (gf_string2int (input_value_str1, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str1);
-					return -1;
-				}
-			}
-			filter->filter_input_gid[filter->filter_num_gid_entries][1] = input_value;
-
-			gf_log (this->name, 
-				GF_LOG_DEBUG, 
-				"filter [%d]: input gid(s) '%s' will be filtered", 
-				filter->filter_num_gid_entries, dup_str);
-			
-			filter->filter_num_gid_entries++;
-			if (filter->filter_num_gid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-		gf_log (this->name, GF_LOG_ERROR, "this option is not supported currently.. exiting");
-		return -1;
-		filter->partial_filter = 1;
-	}
-
-	if (dict_get (this->options, "fixed-uid")) {
-		option_data = dict_get (this->options, "fixed-uid");
-		if (gf_string2int (option_data->data, &input_value) != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"invalid number format \"%s\"", 
-				option_data->data);
-			return -1;
-		}
-		filter->fixed_uid = input_value;
-		filter->fixed_uid_set = 1;
-	}
-
-	if (dict_get (this->options, "fixed-gid")) {
-		option_data = dict_get (this->options, "fixed-gid");
-		if (gf_string2int (option_data->data, &input_value) != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"invalid number format \"%s\"", 
-				option_data->data);
-			return -1;
-		}
-		filter->fixed_gid = input_value;
-		filter->fixed_gid_set = 1;
-	}
-
-	this->private = filter;
-	return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	struct gf_filter *filter = this->private;
-
-	GF_FREE (filter);
-
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = filter_lookup,
-	.stat        = filter_stat,
-	.fstat       = filter_fstat,
-	.readlink    = filter_readlink,
-	.mknod       = filter_mknod,
-	.mkdir       = filter_mkdir,
-	.unlink      = filter_unlink,
-	.rmdir       = filter_rmdir,
-	.symlink     = filter_symlink,
-	.rename      = filter_rename,
-	.link        = filter_link,
-	.truncate    = filter_truncate,
-	.ftruncate   = filter_ftruncate,
-	.create      = filter_create,
-	.open        = filter_open,
-	.readv       = filter_readv,
-	.writev      = filter_writev,
-	.setxattr    = filter_setxattr,
-	.getxattr    = filter_getxattr,
-	.removexattr = filter_removexattr,
-	.opendir     = filter_opendir,
-        .setattr     = filter_setattr,
-        .fsetattr    = filter_fsetattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key  = { "root-squashing" }, 
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = { "read-only" }, 
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = { "fixed-uid" },  
-	  .type = GF_OPTION_TYPE_INT
-	},
-	{ .key  = { "fixed-gid" },  
-	  .type = GF_OPTION_TYPE_INT
-	},
-	{ .key  = { "translate-uid" },  
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key  = { "translate-gid" },  
-	  .type = GF_OPTION_TYPE_ANY
-	},
-	{ .key  = { "filter-uid" },  
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key  = { "filter-gid" },  
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key = {NULL} },
-};
diff --git a/xlators/features/ganesha/src/Makefile.am b/xlators/features/ganesha/src/Makefile.am
deleted file mode 100644
index 3bf291b92c6..00000000000
--- a/xlators/features/ganesha/src/Makefile.am
+++ /dev/null
@@ -1,18 +0,0 @@
-xlator_LTLIBRARIES = ganesha.la
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-noinst_HEADERS = ganesha.h ganesha-mem-types.h
-
-ganesha_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-ganesha_la_SOURCES = ganesha.c
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-        -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
-        -DGANESHA_DIR=\"$(sysconfdir)/ganesha\" \
-        -DGYSNCD_PREFIX=\"$(libexecdir)/glusterfs\"
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/ganesha/src/ganesha.c b/xlators/features/ganesha/src/ganesha.c
deleted file mode 100644
index 859915420ac..00000000000
--- a/xlators/features/ganesha/src/ganesha.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-
-#include "ganesha.h"
-#include "ganesha-mem-types.h"
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_ganesha_mt_end + 1);
-
-        if (ret != 0)
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        "init failed");
-
-        return ret;
-}
-
-int32_t
-init (xlator_t *this)
-{
-        int        ret         = -1;
-        ganesha_priv_t   *priv = NULL;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Need subvolume == 1");
-                goto err;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Dangling volume. Check volfile");
-                goto err;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_ganesha_mt_priv_t);
-        if (!priv)
-                goto err;
-
-        this->private = priv;
-        ret = 0;
-
-err:
-        return ret;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        ganesha_priv_t *priv = this->private;
-
-        this->private = NULL;
-        if (priv)
-                GF_FREE (priv);
-
-        return;
-}
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-
-        { .key  = {"ganesha.enable"},
-          .default_value = "off",
-          .type =  GF_OPTION_TYPE_BOOL,
-          .description = "export volume via NFS-Ganesha"
-        },
-        { .key  = {NULL}
-        },
-};
diff --git a/xlators/features/ganesha/src/ganesha.h b/xlators/features/ganesha/src/ganesha.h
deleted file mode 100644
index 86320e9da28..00000000000
--- a/xlators/features/ganesha/src/ganesha.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "xlator.h"
-#include "ganesha-mem-types.h"
-
-typedef struct {
-        char *host_name;
-} ganesha_priv_t;
-
-
diff --git a/xlators/features/gfid-access/src/Makefile.am b/xlators/features/gfid-access/src/Makefile.am
index 3b25f099123..ff95604c4de 100644
--- a/xlators/features/gfid-access/src/Makefile.am
+++ b/xlators/features/gfid-access/src/Makefile.am
@@ -8,7 +8,8 @@ gfid_access_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = gfid-access.h gfid-access-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/gfid-access/src/gfid-access-mem-types.h b/xlators/features/gfid-access/src/gfid-access-mem-types.h
index 168d67b431f..1c4d0b93de2 100644
--- a/xlators/features/gfid-access/src/gfid-access-mem-types.h
+++ b/xlators/features/gfid-access/src/gfid-access-mem-types.h
@@ -11,13 +11,12 @@
 #ifndef _GFID_ACCESS_MEM_TYPES_H
 #define _GFID_ACCESS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_changelog_mem_types {
-        gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
-        gf_gfid_access_mt_gfid_t,
-        gf_gfid_access_mt_end
+    gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
+    gf_gfid_access_mt_gfid_t,
+    gf_gfid_access_mt_end
 };
 
 #endif
-
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
index a714b66a34d..3fea5672a21 100644
--- a/xlators/features/gfid-access/src/gfid-access.c
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -8,1420 +8,1413 @@
    cases as published by the Free Software Foundation.
 */
 #include "gfid-access.h"
-#include "inode.h"
-#include "byte-order.h"
-#include "statedump.h"
-
+#include <glusterfs/inode.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
 int
-ga_valid_inode_loc_copy (loc_t *dst, loc_t *src, xlator_t *this)
+ga_valid_inode_loc_copy(loc_t *dst, loc_t *src, xlator_t *this)
 {
-        int      ret        = 0;
-        uint64_t value      = 0;
-
-        /* if its an entry operation, on the virtual */
-        /* directory inode as parent, we need to handle */
-        /* it properly */
-        ret = loc_copy (dst, src);
-        if (ret < 0)
-                goto out;
-
-        /*
-         * Change ALL virtual inodes with real-inodes in loc
-         */
-        if (dst->parent) {
-                ret = inode_ctx_get (dst->parent, this, &value);
-                if (ret < 0) {
-                        ret = 0; //real-inode
-                        goto out;
-                }
-                inode_unref (dst->parent);
-                dst->parent = inode_ref ((inode_t*)value);
-                gf_uuid_copy (dst->pargfid, dst->parent->gfid);
+    int ret = 0;
+    uint64_t value = 0;
+
+    /* if its an entry operation, on the virtual */
+    /* directory inode as parent, we need to handle */
+    /* it properly */
+    ret = loc_copy(dst, src);
+    if (ret < 0)
+        goto out;
+
+    /*
+     * Change ALL virtual inodes with real-inodes in loc
+     */
+    if (dst->parent) {
+        ret = inode_ctx_get(dst->parent, this, &value);
+        if (ret < 0) {
+            ret = 0;  // real-inode
+            goto out;
         }
+        inode_unref(dst->parent);
+        dst->parent = inode_ref((inode_t *)(uintptr_t)value);
+        gf_uuid_copy(dst->pargfid, dst->parent->gfid);
+    }
 
-        if (dst->inode) {
-                ret = inode_ctx_get (dst->inode, this, &value);
-                if (ret < 0) {
-                        ret = 0; //real-inode
-                        goto out;
-                }
-                inode_unref (dst->inode);
-                dst->inode = inode_ref ((inode_t*)value);
-                gf_uuid_copy (dst->gfid, dst->inode->gfid);
+    if (dst->inode) {
+        ret = inode_ctx_get(dst->inode, this, &value);
+        if (ret < 0) {
+            ret = 0;  // real-inode
+            goto out;
         }
+        inode_unref(dst->inode);
+        dst->inode = inode_ref((inode_t *)(uintptr_t)value);
+        gf_uuid_copy(dst->gfid, dst->inode->gfid);
+    }
 out:
 
-        return ret;
+    return ret;
 }
 
 void
-ga_newfile_args_free (ga_newfile_args_t *args)
+ga_newfile_args_free(ga_newfile_args_t *args)
 {
-        if (!args)
-                goto out;
+    if (!args)
+        goto out;
 
-        GF_FREE (args->bname);
+    GF_FREE(args->bname);
 
-        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
-                GF_FREE (args->args.symlink.linkpath);
-                args->args.symlink.linkpath = NULL;
-        }
+    if (S_ISLNK(args->st_mode) && args->args.symlink.linkpath) {
+        GF_FREE(args->args.symlink.linkpath);
+        args->args.symlink.linkpath = NULL;
+    }
 
-        mem_put (args);
+    mem_put(args);
 out:
-        return;
+    return;
 }
 
-
 void
-ga_heal_args_free (ga_heal_args_t *args)
+ga_heal_args_free(ga_heal_args_t *args)
 {
-        if (!args)
-                goto out;
+    if (!args)
+        goto out;
 
-        GF_FREE (args->bname);
+    GF_FREE(args->bname);
 
-        mem_put (args);
+    mem_put(args);
 out:
-        return;
+    return;
 }
 
-
 ga_newfile_args_t *
-ga_newfile_parse_args (xlator_t *this, data_t *data)
+ga_newfile_parse_args(xlator_t *this, data_t *data)
 {
-        ga_newfile_args_t *args     = NULL;
-        ga_private_t      *priv     = NULL;
-        int                len      = 0;
-        int                blob_len = 0;
-        int                min_len  = 0;
-        void              *blob     = NULL;
-
-        priv = this->private;
-
-        blob = data->data;
-        blob_len = data->len;
-
-        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
-                + sizeof (args->st_mode) + 2 + 2;
-        if (blob_len < min_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid length: Total length is less "
-                        "than minimum length.");
-                goto err;
+    ga_newfile_args_t *args = NULL;
+    ga_private_t *priv = NULL;
+    int len = 0;
+    int blob_len = 0;
+    int min_len = 0;
+    void *blob = NULL;
+
+    priv = this->private;
+
+    blob = data->data;
+    blob_len = data->len;
+
+    min_len = sizeof(args->uid) + sizeof(args->gid) + sizeof(args->gfid) +
+              sizeof(args->st_mode) + 2 + 2;
+    if (blob_len < min_len) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Invalid length: Total length is less "
+               "than minimum length.");
+        goto err;
+    }
+
+    args = mem_get0(priv->newfile_args_pool);
+    if (args == NULL)
+        goto err;
+
+    args->uid = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    args->gid = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    memcpy(args->gfid, blob, sizeof(args->gfid));
+    blob += sizeof(args->gfid);
+    blob_len -= sizeof(args->gfid);
+
+    args->st_mode = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    len = strnlen(blob, blob_len);
+    if (len == blob_len) {
+        gf_log(this->name, GF_LOG_ERROR, "gfid: %s. No null byte present.",
+               args->gfid);
+        goto err;
+    }
+
+    args->bname = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (args->bname == NULL)
+        goto err;
+
+    memcpy(args->bname, blob, (len + 1));
+    blob += (len + 1);
+    blob_len -= (len + 1);
+
+    if (S_ISDIR(args->st_mode)) {
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
-
-        args = mem_get0 (priv->newfile_args_pool);
-        if (args == NULL)
-                goto err;
-
-        args->uid = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        args->gid = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        memcpy (args->gfid, blob, sizeof (args->gfid));
-        blob += sizeof (args->gfid);
-        blob_len -= sizeof (args->gfid);
-
-        args->st_mode = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        len = strnlen (blob, blob_len);
+        args->args.mkdir.mode = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+        args->args.mkdir.umask = ntoh32(*(uint32_t *)blob);
+        blob_len -= sizeof(uint32_t);
+        if (blob_len < 0) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+    } else if (S_ISLNK(args->st_mode)) {
+        len = strnlen(blob, blob_len);
         if (len == blob_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gfid: %s. No null byte present.",
-                        args->gfid);
-                goto err;
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
+        args->args.symlink.linkpath = GF_MALLOC(len + 1, gf_common_mt_char);
+        if (args->args.symlink.linkpath == NULL)
+            goto err;
 
-        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
-        if (args->bname == NULL)
-                goto err;
-
-        memcpy (args->bname, blob, (len + 1));
-        blob += (len + 1);
+        memcpy(args->args.symlink.linkpath, blob, (len + 1));
         blob_len -= (len + 1);
-
-        if (S_ISDIR (args->st_mode)) {
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
-                blob_len -= sizeof (uint32_t);
-                if (blob_len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-        } else if (S_ISLNK (args->st_mode)) {
-                len = strnlen (blob, blob_len);
-                if (len == blob_len) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
-                                                         gf_common_mt_char);
-                if (args->args.symlink.linkpath == NULL)
-                        goto err;
-
-                memcpy (args->args.symlink.linkpath, blob, (len + 1));
-                blob_len -= (len + 1);
-        } else {
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
-                blob_len -= sizeof (uint32_t);
+    } else {
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
-
-        if (blob_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gfid: %s. Invalid length",
-                        args->gfid);
-                goto err;
+        args->args.mknod.mode = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+        args->args.mknod.rdev = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
+        args->args.mknod.umask = ntoh32(*(uint32_t *)blob);
+        blob_len -= sizeof(uint32_t);
+    }
+
+    if (blob_len) {
+        gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+               args->gfid);
+        goto err;
+    }
 
-        return args;
+    return args;
 
 err:
-        if (args)
-                ga_newfile_args_free (args);
+    if (args)
+        ga_newfile_args_free(args);
 
-        return NULL;
+    return NULL;
 }
 
 ga_heal_args_t *
-ga_heal_parse_args (xlator_t *this, data_t *data)
+ga_heal_parse_args(xlator_t *this, data_t *data)
 {
-        ga_heal_args_t *args     = NULL;
-        ga_private_t   *priv     = NULL;
-        void           *blob     = NULL;
-        int             len      = 0;
-        int             blob_len = 0;
+    ga_heal_args_t *args = NULL;
+    ga_private_t *priv = NULL;
+    void *blob = NULL;
+    int len = 0;
+    int blob_len = 0;
 
-        blob = data->data;
-        blob_len = data->len;
+    blob = data->data;
+    blob_len = data->len;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* bname should at least contain a character */
-        if (blob_len < (sizeof (args->gfid) + 2))
-                goto err;
+    /* bname should at least contain a character */
+    if (blob_len < (sizeof(args->gfid) + 2))
+        goto err;
 
-        args = mem_get0 (priv->heal_args_pool);
-        if (!args)
-                goto err;
+    args = mem_get0(priv->heal_args_pool);
+    if (!args)
+        goto err;
 
-        memcpy (args->gfid, blob, sizeof (args->gfid));
-        blob += sizeof (args->gfid);
-        blob_len -= sizeof (args->gfid);
+    memcpy(args->gfid, blob, sizeof(args->gfid));
+    blob += sizeof(args->gfid);
+    blob_len -= sizeof(args->gfid);
 
-        len = strnlen (blob, blob_len);
-        if (len == blob_len)
-                goto err;
+    len = strnlen(blob, blob_len);
+    if (len == blob_len)
+        goto err;
 
-        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
-        if (!args->bname)
-                goto err;
+    args->bname = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (!args->bname)
+        goto err;
 
-        memcpy (args->bname, blob, len);
-        blob_len -= (len + 1);
+    memcpy(args->bname, blob, len);
+    args->bname[len] = '\0';
+    blob_len -= (len + 1);
 
-        if (blob_len)
-                goto err;
+    if (blob_len)
+        goto err;
 
-        return args;
+    return args;
 
 err:
-        if (args)
-                ga_heal_args_free (args);
+    if (args)
+        ga_heal_args_free(args);
 
-        return NULL;
+    return NULL;
 }
 
 static int32_t
-ga_fill_tmp_loc (loc_t *loc, xlator_t *this, uuid_t gfid,
-                 char *bname, dict_t *xdata, loc_t *new_loc)
+ga_fill_tmp_loc(loc_t *loc, xlator_t *this, uuid_t gfid, char *bname,
+                dict_t *xdata, loc_t *new_loc)
 {
-        int       ret    = -1;
-        uint64_t  value  = 0;
-        inode_t  *parent = NULL;
-        uuid_t *gfid_ptr = NULL;
-
-        parent = loc->inode;
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (!ret) {
-                parent = (void *)value;
-                if (gf_uuid_is_null (parent->gfid))
-                        parent = loc->inode;
-        }
-
-        /* parent itself should be looked up */
-        gf_uuid_copy (new_loc->pargfid, parent->gfid);
-        new_loc->parent = inode_ref (parent);
-
-        new_loc->inode = inode_grep (parent->table, parent, bname);
-        if (!new_loc->inode) {
-                new_loc->inode = inode_new (parent->table);
-                gf_uuid_copy (new_loc->inode->gfid, gfid);
-        }
-
-        loc_path (new_loc, bname);
-        if (new_loc->path) {
-                new_loc->name = strrchr (new_loc->path, '/');
-                if (new_loc->name)
-                        new_loc->name++;
-        }
-
-        gfid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!gfid_ptr) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (*gfid_ptr, gfid);
-        ret = dict_set_dynptr (xdata, "gfid-req", gfid_ptr, sizeof (uuid_t));
-        if (ret < 0)
-                goto out;
-
-        ret = 0;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *parent = NULL;
+    unsigned char *gfid_ptr = NULL;
+
+    parent = loc->inode;
+    ret = inode_ctx_get(loc->inode, this, &value);
+    if (!ret) {
+        parent = (void *)(uintptr_t)value;
+        if (gf_uuid_is_null(parent->gfid))
+            parent = loc->inode;
+    }
+
+    /* parent itself should be looked up */
+    gf_uuid_copy(new_loc->pargfid, parent->gfid);
+    new_loc->parent = inode_ref(parent);
+
+    new_loc->inode = inode_grep(parent->table, parent, bname);
+    if (!new_loc->inode) {
+        new_loc->inode = inode_new(parent->table);
+        gf_uuid_copy(new_loc->inode->gfid, gfid);
+    }
+
+    loc_path(new_loc, bname);
+    if (new_loc->path) {
+        new_loc->name = strrchr(new_loc->path, '/');
+        if (new_loc->name)
+            new_loc->name++;
+    }
+
+    gfid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!gfid_ptr) {
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(gfid_ptr, gfid);
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid_ptr, false);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
 
 out:
-        if (ret && gfid_ptr)
-                GF_FREE (gfid_ptr);
-        return ret;
+    if (ret && gfid_ptr)
+        GF_FREE(gfid_ptr);
+    return ret;
 }
 
-
-
 static gf_boolean_t
-__is_gfid_access_dir (uuid_t gfid)
+__is_gfid_access_dir(uuid_t gfid)
 {
-        uuid_t  aux_gfid;
+    static uuid_t aux_gfid = {0, 0, 0, 0, 0, 0, 0, 0,
+                              0, 0, 0, 0, 0, 0, 0, GF_AUX_GFID};
 
-        memset (aux_gfid, 0, 16);
-        aux_gfid[15] = GF_AUX_GFID;
+    if (gf_uuid_compare(gfid, aux_gfid) == 0)
+        return _gf_true;
 
-        if (gf_uuid_compare (gfid, aux_gfid) == 0)
-                return _gf_true;
-
-        return _gf_false;
+    return _gf_false;
 }
 
 int32_t
-ga_forget (xlator_t *this, inode_t *inode)
+ga_forget(xlator_t *this, inode_t *inode)
 {
-        int       ret = -1;
-        uint64_t  value = 0;
-        inode_t  *tmp_inode = NULL;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *tmp_inode = NULL;
 
-        ret = inode_ctx_del (inode, this, &value);
-        if (ret)
-                goto out;
+    ret = inode_ctx_del(inode, this, &value);
+    if (ret)
+        goto out;
 
-        tmp_inode = (void *)value;
-        inode_unref (tmp_inode);
+    tmp_inode = (void *)(uintptr_t)value;
+    inode_unref(tmp_inode);
 
 out:
-        return 0;
+    return 0;
 }
 
-
 static int
-ga_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno,
-             inode_t *inode, struct iatt *stat, dict_t *dict,
-             struct iatt *postparent)
+ga_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *stat, dict_t *dict,
+            struct iatt *postparent)
 {
-        call_frame_t *orig_frame = NULL;
+    call_frame_t *orig_frame = NULL;
 
-        orig_frame = frame->local;
-        frame->local = NULL;
+    orig_frame = frame->local;
+    frame->local = NULL;
 
-        /* don't worry about inode linking and other stuff. They'll happen on
-         * the next lookup.
-         */
-        STACK_DESTROY (frame->root);
+    /* don't worry about inode linking and other stuff. They'll happen on
+     * the next lookup.
+     */
+    STACK_DESTROY(frame->root);
 
-        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, dict);
+    STACK_UNWIND_STRICT(setxattr, orig_frame, op_ret, op_errno, dict);
 
-        return 0;
+    return 0;
 }
 
 static int
-ga_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata)
+ga_newentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        ga_local_t *local = NULL;
+    ga_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* don't worry about inode linking and other stuff. They'll happen on
-         * the next lookup.
-         */
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    /* don't worry about inode linking and other stuff. They'll happen on
+     * the next lookup.
+     */
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
-                             op_errno, xdata);
+    STACK_UNWIND_STRICT(setxattr, local->orig_frame, op_ret, op_errno, xdata);
 
-        if (local->xdata)
-                dict_unref (local->xdata);
-        loc_wipe (&local->loc);
-        mem_put (local);
+    if (local->xdata)
+        dict_unref(local->xdata);
+    loc_wipe(&local->loc);
+    mem_put(local);
 
-        return 0;
+    return 0;
 }
 
 static int
-ga_newentry_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *stat, dict_t *xdata,
-                        struct iatt *postparent)
+ga_newentry_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *stat, dict_t *xdata,
+                       struct iatt *postparent)
 
 {
-        ga_local_t *local = NULL;
+    ga_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if ((op_ret < 0) && ((op_errno != ENOENT) && (op_errno != ESTALE)))
-                goto err;
+    if ((op_ret < 0) && ((op_errno != ENOENT) && (op_errno != ESTALE)))
+        goto err;
 
-        STACK_WIND (frame, ga_newentry_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, &local->loc, local->mode,
-                    local->rdev, local->umask, local->xdata);
-        return 0;
+    STACK_WIND(frame, ga_newentry_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, &local->loc, local->mode,
+               local->rdev, local->umask, local->xdata);
+    return 0;
 
 err:
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret, op_errno,
-                             xdata);
-        if (local->xdata)
-                dict_unref (local->xdata);
-        loc_wipe (&local->loc);
-        mem_put (local);
-
-        return 0;
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    STACK_UNWIND_STRICT(setxattr, local->orig_frame, op_ret, op_errno, xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
+    loc_wipe(&local->loc);
+    mem_put(local);
+
+    return 0;
 }
 
 int32_t
-ga_new_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
-              dict_t *xdata)
+ga_new_entry(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+             dict_t *xdata)
 {
-        int                ret       = -1;
-        ga_newfile_args_t *args      = NULL;
-        loc_t              tmp_loc   = {0,};
-        call_frame_t      *new_frame = NULL;
-        ga_local_t        *local     = NULL;
-        uuid_t             gfid      = {0,};
-
-        args = ga_newfile_parse_args (this, data);
-        if (!args)
-                goto out;
-
-        ret = gf_uuid_parse (args->gfid, gfid);
-        if (ret)
-                goto out;
-
-        if (!xdata) {
-                xdata = dict_new ();
-        } else {
-                xdata = dict_ref (xdata);
-        }
-
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = ga_fill_tmp_loc (loc, this, gfid,
-                               args->bname, xdata, &tmp_loc);
-        if (ret)
-                goto out;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-        local = mem_get0 (this->local_pool);
-        local->orig_frame = frame;
-
-        loc_copy (&local->loc, &tmp_loc);
-
-        new_frame->local = local;
-        new_frame->root->uid = args->uid;
-        new_frame->root->gid = args->gid;
-
-        if (S_ISDIR (args->st_mode)) {
-                STACK_WIND (new_frame, ga_newentry_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                            &tmp_loc, args->args.mkdir.mode,
-                            args->args.mkdir.umask, xdata);
-        } else if (S_ISLNK (args->st_mode)) {
-                STACK_WIND (new_frame, ga_newentry_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                            args->args.symlink.linkpath,
-                            &tmp_loc, 0, xdata);
-        } else {
-                /* use 07777 (4 7s) for considering the Sticky bits etc) */
-                ((ga_local_t *)new_frame->local)->mode =
-                     (S_IFMT & args->st_mode) | (07777 & args->args.mknod.mode);
-
-                ((ga_local_t *)new_frame->local)->umask =
-                                                         args->args.mknod.umask;
-                ((ga_local_t *)new_frame->local)->rdev  = args->args.mknod.rdev;
-                ((ga_local_t *)new_frame->local)->xdata = dict_ref (xdata);
-
-                /* send a named lookup, so that dht can cleanup up stale linkto
-                 * files etc.
-                 */
-                STACK_WIND (new_frame, ga_newentry_lookup_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
-                            &tmp_loc, NULL);
-        }
+    int ret = -1;
+    ga_newfile_args_t *args = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+    ga_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    if (!xdata) {
+        xdata = dict_new();
+    } else {
+        xdata = dict_ref(xdata);
+    }
+
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    args = ga_newfile_parse_args(this, data);
+    if (!args)
+        goto out;
+
+    ret = gf_uuid_parse(args->gfid, gfid);
+    if (ret)
+        goto out;
+
+    ret = ga_fill_tmp_loc(loc, this, gfid, args->bname, xdata, &tmp_loc);
+    if (ret)
+        goto out;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    local = mem_get0(this->local_pool);
+    local->orig_frame = frame;
+
+    loc_copy(&local->loc, &tmp_loc);
+
+    new_frame->local = local;
+    new_frame->root->uid = args->uid;
+    new_frame->root->gid = args->gid;
+
+    if (S_ISDIR(args->st_mode)) {
+        STACK_WIND(new_frame, ga_newentry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, &tmp_loc,
+                   args->args.mkdir.mode, args->args.mkdir.umask, xdata);
+    } else if (S_ISLNK(args->st_mode)) {
+        STACK_WIND(new_frame, ga_newentry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->symlink,
+                   args->args.symlink.linkpath, &tmp_loc, 0, xdata);
+    } else {
+        /* use 07777 (4 7s) for considering the Sticky bits etc) */
+        ((ga_local_t *)new_frame->local)->mode = (S_IFMT & args->st_mode) |
+                                                 (07777 &
+                                                  args->args.mknod.mode);
+
+        ((ga_local_t *)new_frame->local)->umask = args->args.mknod.umask;
+        ((ga_local_t *)new_frame->local)->rdev = args->args.mknod.rdev;
+        ((ga_local_t *)new_frame->local)->xdata = dict_ref(xdata);
+
+        /* send a named lookup, so that dht can cleanup up stale linkto
+         * files etc.
+         */
+        STACK_WIND(new_frame, ga_newentry_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, &tmp_loc, NULL);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        ga_newfile_args_free (args);
+    ga_newfile_args_free(args);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-ga_heal_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
-               dict_t *xdata)
+ga_heal_entry(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+              dict_t *xdata)
 {
-        int             ret       = -1;
-        ga_heal_args_t *args      = NULL;
-        loc_t           tmp_loc   = {0,};
-        call_frame_t   *new_frame = NULL;
-        uuid_t          gfid      = {0,};
-
-        args = ga_heal_parse_args (this, data);
-        if (!args)
-                goto out;
-
-        ret = gf_uuid_parse (args->gfid, gfid);
-        if (ret)
-                goto out;
-
-        if (!xdata)
-                xdata = dict_new ();
-        else
-                xdata = dict_ref (xdata);
-
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = ga_fill_tmp_loc (loc, this, gfid, args->bname,
-                               xdata, &tmp_loc);
-        if (ret)
-                goto out;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-        new_frame->local = (void *)frame;
-
-        STACK_WIND (new_frame, ga_heal_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    &tmp_loc, xdata);
-
-        ret = 0;
+    int ret = -1;
+    ga_heal_args_t *args = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    args = ga_heal_parse_args(this, data);
+    if (!args)
+        goto out;
+
+    ret = gf_uuid_parse(args->gfid, gfid);
+    if (ret)
+        goto out;
+
+    if (!xdata)
+        xdata = dict_new();
+    else
+        xdata = dict_ref(xdata);
+
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = ga_fill_tmp_loc(loc, this, gfid, args->bname, xdata, &tmp_loc);
+    if (ret)
+        goto out;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_frame->local = (void *)frame;
+
+    STACK_WIND(new_frame, ga_heal_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+
+    ret = 0;
 out:
-        if (args)
-                ga_heal_args_free (args);
+    if (args)
+        ga_heal_args_free(args);
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-ga_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 dict_t *xdata)
+ga_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+ga_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        data_t  *data     = NULL;
-        int      op_errno = ENOMEM;
-        int      ret      = 0;
-        loc_t   ga_loc    = {0, };
-
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-
-        data = dict_get (dict, GF_FUSE_AUX_GFID_NEWFILE);
-        if (data) {
-                ret = ga_new_entry (frame, this, loc, data, xdata);
-                if (ret)
-                        goto err;
-                return 0;
-        }
+    data_t *data = NULL;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_loc = {
+        0,
+    };
+
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+
+    data = dict_get(dict, GF_FUSE_AUX_GFID_NEWFILE);
+    if (data) {
+        ret = ga_new_entry(frame, this, loc, data, xdata);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-        data = dict_get (dict, GF_FUSE_AUX_GFID_HEAL);
-        if (data) {
-                ret = ga_heal_entry (frame, this, loc, data, xdata);
-                if (ret)
-                        goto err;
-                return 0;
-        }
+    data = dict_get(dict, GF_FUSE_AUX_GFID_HEAL);
+    if (data) {
+        ret = ga_heal_entry(frame, this, loc, data, xdata);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-        //If the inode is a virtual inode change the inode otherwise perform
-        //the operation on same inode
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    // If the inode is a virtual inode change the inode otherwise perform
+    // the operation on same inode
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, ga_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, &ga_loc, dict, flags,
-                    xdata);
+    STACK_WIND(frame, ga_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, &ga_loc, dict, flags, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, -1, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-ga_virtual_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, inode_t *inode,
-                       struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+ga_virtual_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        int             j                       = 0;
-        int             i                       = 0;
-        int             ret                     = 0;
-        uint64_t        temp_ino                = 0;
-        inode_t         *cbk_inode              = NULL;
-        inode_t         *true_inode             = NULL;
-        uuid_t          random_gfid             = {0,};
-        inode_t         *linked_inode           = NULL;
-
-        if (frame->local)
-                cbk_inode = frame->local;
-        else
-                cbk_inode = inode_ref (inode);
-
-        frame->local = NULL;
-        if (op_ret)
-                goto unwind;
-
-        if (!IA_ISDIR (buf->ia_type))
+    int ret = 0;
+    inode_t *cbk_inode = NULL;
+    inode_t *true_inode = NULL;
+    uuid_t random_gfid = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+
+    if (frame->local)
+        cbk_inode = frame->local;
+    else
+        cbk_inode = inode_ref(inode);
+
+    frame->local = NULL;
+    if (op_ret)
+        goto unwind;
+
+    if (!IA_ISDIR(buf->ia_type))
+        goto unwind;
+
+    /* need to send back a different inode for linking in itable */
+    if (cbk_inode == inode) {
+        /* check if the inode is in the 'itable' or
+           if its just previously discover()'d inode */
+        true_inode = inode_find(inode->table, buf->ia_gfid);
+        if (!true_inode) {
+            /* This unref is for 'inode_ref()' done in beginning.
+               This is needed as cbk_inode is allocated new inode
+               whose unref is taken at the end*/
+            inode_unref(cbk_inode);
+            cbk_inode = inode_new(inode->table);
+
+            if (!cbk_inode) {
+                op_ret = -1;
+                op_errno = ENOMEM;
                 goto unwind;
+            }
+            /* the inode is not present in itable, ie, the actual
+               path is not yet looked up. Use the current inode
+               itself for now */
 
-        /* need to send back a different inode for linking in itable */
-        if (cbk_inode == inode) {
-                /* check if the inode is in the 'itable' or
-                   if its just previously discover()'d inode */
-                true_inode = inode_find (inode->table, buf->ia_gfid);
-                if (!true_inode) {
-                        /* This unref is for 'inode_ref()' done in beginning.
-                           This is needed as cbk_inode is allocated new inode
-                           whose unref is taken at the end*/
-                        inode_unref (cbk_inode);
-                        cbk_inode = inode_new (inode->table);
-
-                        if (!cbk_inode) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unwind;
-                        }
-                        /* the inode is not present in itable, ie, the actual
-                           path is not yet looked up. Use the current inode
-                           itself for now */
-
-                        linked_inode = inode_link (inode, NULL, NULL, buf);
-                        inode = linked_inode;
-                } else {
-                        /* 'inode_ref()' has been done in inode_find() */
-                        inode = true_inode;
-                }
-
-                ret = inode_ctx_put (cbk_inode, this, (uint64_t)inode);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set the inode ctx with"
-                                "the actual inode");
-                        if (inode)
-                                inode_unref (inode);
-                }
-                inode = NULL;
+            linked_inode = inode_link(inode, NULL, NULL, buf);
+            inode = linked_inode;
+        } else {
+            /* 'inode_ref()' has been done in inode_find() */
+            inode = true_inode;
         }
 
-        if (!gf_uuid_is_null (cbk_inode->gfid)) {
-                /* if the previous linked inode is used, use the
-                   same gfid */
-                gf_uuid_copy (random_gfid, cbk_inode->gfid);
-        } else {
-                /* replace the buf->ia_gfid to a random gfid
-                   for directory, for files, what we received is fine */
-                gf_uuid_generate (random_gfid);
+        ret = inode_ctx_put(cbk_inode, this, (uint64_t)(uintptr_t)inode);
+        if (ret) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "failed to set the inode ctx with"
+                   "the actual inode");
+            if (inode)
+                inode_unref(inode);
         }
+        inode = NULL;
+    }
 
-        gf_uuid_copy (buf->ia_gfid, random_gfid);
+    if (!gf_uuid_is_null(cbk_inode->gfid)) {
+        /* if the previous linked inode is used, use the
+           same gfid */
+        gf_uuid_copy(random_gfid, cbk_inode->gfid);
+    } else {
+        /* replace the buf->ia_gfid to a random gfid
+           for directory, for files, what we received is fine */
+        gf_uuid_generate(random_gfid);
+    }
 
-        for (i = 15; i > (15 - 8); i--) {
-                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
-                j += 8;
-        }
-        buf->ia_ino = temp_ino;
+    gf_uuid_copy(buf->ia_gfid, random_gfid);
+
+    buf->ia_ino = gfid_to_ino(buf->ia_gfid);
 
 unwind:
-        /* Lookup on non-existing gfid returns ESTALE.
-           Convert into ENOENT for virtual lookup*/
-        if (op_errno == ESTALE)
-               op_errno = ENOENT;
+    /* Lookup on non-existing gfid returns ESTALE.
+       Convert into ENOENT for virtual lookup*/
+    if (op_errno == ESTALE)
+        op_errno = ENOENT;
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, cbk_inode, buf,
-                             xdata, postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, cbk_inode, buf, xdata,
+                        postparent);
 
-        /* Also handles inode_unref of frame->local if done in ga_lookup */
-        if (cbk_inode)
-               inode_unref (cbk_inode);
+    /* Also handles inode_unref of frame->local if done in ga_lookup */
+    if (cbk_inode)
+        inode_unref(cbk_inode);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+ga_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-        ga_private_t *priv = NULL;
+    ga_private_t *priv = NULL;
 
-        /* if the entry in question is not 'root',
-           then follow the normal path */
-        if (op_ret || !__is_root_gfid(buf->ia_gfid))
-                goto unwind;
+    /* if the entry in question is not 'root',
+       then follow the normal path */
+    if (op_ret || !__is_root_gfid(buf->ia_gfid))
+        goto unwind;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* do we need to copy root stbuf everytime? */
-        /* mostly yes, as we want to have the 'stat' info show latest
-           in every _cbk() */
+    /* do we need to copy root stbuf every time? */
+    /* mostly yes, as we want to have the 'stat' info show latest
+       in every _cbk() */
 
-        /* keep the reference for root stat buf */
-        priv->root_stbuf = *buf;
-        priv->gfiddir_stbuf = priv->root_stbuf;
-        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
-        priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
+    /* keep the reference for root stat buf */
+    priv->root_stbuf = *buf;
+    priv->gfiddir_stbuf = priv->root_stbuf;
+    priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+    priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
 int32_t
-ga_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+ga_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        ga_private_t *priv     = NULL;
-        int           ret      = -1;
-        uuid_t        tmp_gfid = {0,};
-        loc_t         tmp_loc  = {0,};
-        uint64_t      value    = 0;
-        inode_t      *inode    = NULL;
-        inode_t      *true_inode    = NULL;
-        int32_t       op_errno = ENOENT;
-
-        /* if its discover(), no need for any action here */
-        if (!loc->name)
-                goto wind;
-
-        /* if its revalidate, and inode is not of type directory,
-           proceed with 'wind' */
-        if (loc->inode && loc->inode->ia_type &&
-            !IA_ISDIR (loc->inode->ia_type)) {
-
-                /* a revalidate on ".gfid/<dentry>" is possible, check for it */
-                if (((loc->parent &&
-                      __is_gfid_access_dir (loc->parent->gfid)) ||
-                     __is_gfid_access_dir (loc->pargfid))) {
-
-                        /* here, just send 'loc->gfid' and 'loc->inode' */
-                        tmp_loc.inode = inode_ref (loc->inode);
-                        gf_uuid_copy (tmp_loc.gfid, loc->inode->gfid);
-
-                        STACK_WIND (frame, default_lookup_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup,
-                                    &tmp_loc, xdata);
-
-                        inode_unref (tmp_loc.inode);
-
-                        return 0;
-                }
-
-                /* not something to bother, continue the flow */
-                goto wind;
-        }
+    ga_private_t *priv = NULL;
+    int ret = -1;
+    uuid_t tmp_gfid = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    uint64_t value = 0;
+    inode_t *inode = NULL;
+    inode_t *true_inode = NULL;
+    int32_t op_errno = ENOENT;
+
+    priv = this->private;
+
+    /* Handle nameless lookup on ".gfid" */
+    if (!loc->parent && __is_gfid_access_dir(loc->gfid)) {
+        STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc->inode,
+                            &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+        return 0;
+    }
 
-        priv = this->private;
+    /* if its discover(), no need for any action here */
+    if (!loc->name)
+        goto wind;
 
-        /* need to check if the lookup is on virtual dir */
-        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
-            ((loc->parent && __is_root_gfid (loc->parent->gfid)) ||
-             __is_root_gfid (loc->pargfid))) {
-                /* this means, the query is on '/.gfid', return the fake stat,
-                   and say success */
+    /* if its revalidate, and inode is not of type directory,
+       proceed with 'wind' */
+    if (loc->inode && loc->inode->ia_type && !IA_ISDIR(loc->inode->ia_type)) {
+        /* a revalidate on ".gfid/<dentry>" is possible, check for it */
+        if (((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||
+             __is_gfid_access_dir(loc->pargfid))) {
+            /* here, just send 'loc->gfid' and 'loc->inode' */
+            tmp_loc.inode = inode_ref(loc->inode);
+            gf_uuid_copy(tmp_loc.gfid, loc->inode->gfid);
 
-                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
-                                     &priv->gfiddir_stbuf, xdata,
-                                     &priv->root_stbuf);
-                return 0;
-        }
+            STACK_WIND(frame, default_lookup_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        /* now, check if the lookup() is on an existing entry,
-           but on gfid-path */
-        if (!((loc->parent && __is_gfid_access_dir (loc->parent->gfid)) ||
-              __is_gfid_access_dir (loc->pargfid))) {
-                if (!loc->parent)
-                        goto wind;
+            inode_unref(tmp_loc.inode);
 
-                ret = inode_ctx_get (loc->parent, this, &value);
-                if (ret)
-                        goto wind;
+            return 0;
+        }
 
-                inode = (inode_t *) value;
+        /* not something to bother, continue the flow */
+        goto wind;
+    }
 
-                ret = loc_copy_overload_parent (&tmp_loc, loc, inode);
-                if (ret)
-                        goto err;
+    /* need to check if the lookup is on virtual dir */
+    if ((loc->name && !strcmp(GF_GFID_DIR, loc->name)) &&
+        ((loc->parent && __is_root_gfid(loc->parent->gfid)) ||
+         __is_root_gfid(loc->pargfid))) {
+        /* this means, the query is on '/.gfid', return the fake stat,
+           and say success */
 
-                STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->lookup, &tmp_loc, xdata);
+        STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc->inode,
+                            &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+        return 0;
+    }
 
-                loc_wipe (&tmp_loc);
-                return 0;
-        }
+    /* now, check if the lookup() is on an existing entry,
+       but on gfid-path */
+    if (!((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||
+          __is_gfid_access_dir(loc->pargfid))) {
+        if (!loc->parent)
+            goto wind;
 
-        /* make sure the 'basename' is actually a 'canonical-gfid',
-           otherwise, return error */
-        ret = gf_uuid_parse (loc->name, tmp_gfid);
+        ret = inode_ctx_get(loc->parent, this, &value);
         if (ret)
-                goto err;
+            goto wind;
 
-        /* if its fresh lookup, go ahead and send it down, if not,
-           for directory, we need indirection to actual dir inode */
-        if (!(loc->inode && loc->inode->ia_type))
-                goto discover;
+        inode = (inode_t *)(uintptr_t)value;
 
-        /* revalidate on directory */
-        ret = inode_ctx_get (loc->inode, this, &value);
+        ret = loc_copy_overload_parent(&tmp_loc, loc, inode);
         if (ret)
-                goto err;
-
-        inode = (void *)value;
+            goto err;
 
-        /* valid inode, already looked up, work on that */
-        if (inode->ia_type)
-                goto discover;
+        STACK_WIND(frame, ga_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        /* check if the inode is in the 'itable' or
-           if its just previously discover()'d inode */
-        true_inode = inode_find (loc->inode->table, tmp_gfid);
-        if (true_inode) {
-                /* time do another lookup and update the context
-                   with proper inode */
-                op_errno = ESTALE;
-                /* 'inode_ref()' done in inode_find */
-                inode_unref (true_inode);
-                goto err;
-        }
+        loc_wipe(&tmp_loc);
+        return 0;
+    }
+
+    /* make sure the 'basename' is actually a 'canonical-gfid',
+       otherwise, return error */
+    ret = gf_uuid_parse(loc->name, tmp_gfid);
+    if (ret)
+        goto err;
+
+    /* if its fresh lookup, go ahead and send it down, if not,
+       for directory, we need indirection to actual dir inode */
+    if (!(loc->inode && loc->inode->ia_type))
+        goto discover;
+
+    /* revalidate on directory */
+    ret = inode_ctx_get(loc->inode, this, &value);
+    if (ret)
+        goto err;
+
+    inode = (void *)(uintptr_t)value;
+
+    /* valid inode, already looked up, work on that */
+    if (inode->ia_type)
+        goto discover;
+
+    /* check if the inode is in the 'itable' or
+       if its just previously discover()'d inode */
+    true_inode = inode_find(loc->inode->table, tmp_gfid);
+    if (true_inode) {
+        /* time do another lookup and update the context
+           with proper inode */
+        op_errno = ESTALE;
+        /* 'inode_ref()' done in inode_find */
+        inode_unref(true_inode);
+        goto err;
+    }
 
 discover:
-        /* for the virtual entries, we don't need to send 'gfid-req' key, as
-           for these entries, we don't want to 'set' a new gfid */
-        if (xdata)
-                dict_del (xdata, "gfid-req");
+    /* for the virtual entries, we don't need to send 'gfid-req' key, as
+       for these entries, we don't want to 'set' a new gfid */
+    if (xdata)
+        dict_del(xdata, "gfid-req");
 
-        gf_uuid_copy (tmp_loc.gfid, tmp_gfid);
+    gf_uuid_copy(tmp_loc.gfid, tmp_gfid);
 
-        /* if revalidate, then we need to have the proper reference */
-        if (inode) {
-                tmp_loc.inode = inode_ref (inode);
-                frame->local = inode_ref (loc->inode);
-        } else {
-                tmp_loc.inode = inode_ref (loc->inode);
-        }
+    /* if revalidate, then we need to have the proper reference */
+    if (inode) {
+        tmp_loc.inode = inode_ref(inode);
+        frame->local = inode_ref(loc->inode);
+    } else {
+        tmp_loc.inode = inode_ref(loc->inode);
+    }
 
-        STACK_WIND (frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+    STACK_WIND(frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        inode_unref (tmp_loc.inode);
+    inode_unref(tmp_loc.inode);
 
-        return 0;
+    return 0;
 
 wind:
-        /* used for all the normal lookup path */
-        STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    /* used for all the normal lookup path */
+    STACK_WIND(frame, ga_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, loc->inode,
-                             &priv->gfiddir_stbuf, xdata,
-                             &priv->root_stbuf);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, loc->inode,
+                        &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+    return 0;
 }
 
 int
-ga_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata)
+ga_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
-                    xdata);
+    STACK_WIND(frame, default_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, loc->inode,
-                             NULL, NULL, NULL, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, loc->inode, NULL, NULL,
+                        NULL, xdata);
+    return 0;
 }
 
-
 int
-ga_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+ga_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_create_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, xdata);
-
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, xdata);
 
+    return 0;
 }
 
 int
-ga_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-            loc_t *loc, mode_t umask, dict_t *xdata)
+ga_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_symlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkname, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, default_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata)
+ga_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
-                    umask, xdata);
+    STACK_WIND(frame, default_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-          dict_t *xdata)
+ga_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+         dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_rmdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
-                    &ga_loc, flag, xdata);
+    STACK_WIND(frame, default_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, &ga_loc, flag, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL,
-                             NULL, xdata);
+    STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
-           dict_t *xdata)
+ga_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+          dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    &ga_loc, xflag, xdata);
+    STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, &ga_loc, xflag, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL,
-                             NULL, xdata);
+    STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_rename (call_frame_t *frame, xlator_t *this,
-           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ga_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        int   op_errno  = ENOMEM;
-        int   ret       = 0;
-        loc_t ga_oldloc = {0, };
-        loc_t ga_newloc = {0, };
-
-        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
-        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
-
-        ret = ga_valid_inode_loc_copy (&ga_oldloc, oldloc, this);
-        if (ret < 0)
-                goto err;
-
-        ret = ga_valid_inode_loc_copy (&ga_newloc, newloc, this);
-        if (ret < 0) {
-                loc_wipe (&ga_oldloc);
-                goto err;
-        }
-
-        STACK_WIND (frame, default_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    &ga_oldloc, &ga_newloc, xdata);
-
-        loc_wipe (&ga_newloc);
-        loc_wipe (&ga_oldloc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_oldloc = {
+        0,
+    };
+    loc_t ga_newloc = {
+        0,
+    };
+
+    GFID_ACCESS_ENTRY_OP_CHECK(oldloc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(newloc, op_errno, err);
+
+    ret = ga_valid_inode_loc_copy(&ga_oldloc, oldloc, this);
+    if (ret < 0)
+        goto err;
+
+    ret = ga_valid_inode_loc_copy(&ga_newloc, newloc, this);
+    if (ret < 0) {
+        loc_wipe(&ga_oldloc);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &ga_oldloc, &ga_newloc, xdata);
+
+    loc_wipe(&ga_newloc);
+    loc_wipe(&ga_oldloc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ga_link (call_frame_t *frame, xlator_t *this,
-         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ga_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        int   op_errno  = ENOMEM;
-        int   ret       = 0;
-        loc_t ga_oldloc = {0, };
-        loc_t ga_newloc = {0, };
-
-        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
-        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
-
-        ret = ga_valid_inode_loc_copy (&ga_oldloc, oldloc, this);
-        if (ret < 0)
-                goto err;
-
-        ret = ga_valid_inode_loc_copy (&ga_newloc, newloc, this);
-        if (ret < 0) {
-                loc_wipe (&ga_oldloc);
-                goto err;
-        }
-
-        STACK_WIND (frame, default_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    &ga_oldloc, &ga_newloc, xdata);
-
-        loc_wipe (&ga_newloc);
-        loc_wipe (&ga_oldloc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_oldloc = {
+        0,
+    };
+    loc_t ga_newloc = {
+        0,
+    };
+
+    GFID_ACCESS_ENTRY_OP_CHECK(oldloc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(newloc, op_errno, err);
+
+    ret = ga_valid_inode_loc_copy(&ga_oldloc, oldloc, this);
+    if (ret < 0)
+        goto err;
+
+    ret = ga_valid_inode_loc_copy(&ga_newloc, newloc, this);
+    if (ret < 0) {
+        loc_wipe(&ga_oldloc);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, &ga_oldloc, &ga_newloc, xdata);
+
+    loc_wipe(&ga_newloc);
+    loc_wipe(&ga_oldloc);
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            fd_t *fd, dict_t *xdata)
+ga_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
 
-        /* also check if the loc->inode itself is virtual
-           inode, if yes, return with failure, mainly because we
-           can't handle all the readdirp and other things on it. */
-        if (inode_ctx_get (loc->inode, this, NULL) == 0) {
-                op_errno = ENOTSUP;
-                goto err;
-        }
+    /* also check if the loc->inode itself is virtual
+       inode, if yes, return with failure, mainly because we
+       can't handle all the readdirp and other things on it. */
+    if (inode_ctx_get(loc->inode, this, NULL) == 0) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
 
-        STACK_WIND (frame, default_opendir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+    STACK_UNWIND_STRICT(opendir, frame, -1, op_errno, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name, dict_t *xdata)
+ga_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, &ga_loc, name, xdata);
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, &ga_loc, name, xdata);
 
-        loc_wipe (&ga_loc);
+    loc_wipe(&ga_loc);
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+    STACK_UNWIND_STRICT(getxattr, frame, -1, op_errno, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-         dict_t *xdata)
+ga_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int          op_errno = ENOMEM;
-        int          ret      = -1;
-        loc_t        ga_loc   = {0, };
-        ga_private_t *priv    = NULL;
-
-        priv = this->private;
-        /* If stat is on ".gfid" itself, do not wind further,
-         * return fake stat and return success.
-         */
-        if (__is_gfid_access_dir(loc->gfid))
-                goto out;
-
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
-
-        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, &ga_loc, xdata);
-
-        loc_wipe (&ga_loc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
+    ga_private_t *priv = NULL;
+
+    priv = this->private;
+    /* If stat is on ".gfid" itself, do not wind further,
+     * return fake stat and return success.
+     */
+    if (__is_gfid_access_dir(loc->gfid))
+        goto out;
+
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
+
+    STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, &ga_loc, xdata);
+
+    loc_wipe(&ga_loc);
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+    STACK_UNWIND_STRICT(stat, frame, -1, op_errno, NULL, xdata);
 
-        return 0;
+    return 0;
 
 out:
-        STACK_UNWIND_STRICT (stat, frame, 0, 0, &priv->gfiddir_stbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, 0, 0, &priv->gfiddir_stbuf, xdata);
+    return 0;
 }
 
 int32_t
-ga_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid,
-            dict_t *xdata)
+ga_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
-
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
-
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, &ga_loc, stbuf, valid,
-                    xdata);
-
-        loc_wipe (&ga_loc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
+
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
+
+    STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, &ga_loc, stbuf, valid, xdata);
+
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(setattr, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+ga_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, &ga_loc, name,
-                    xdata);
+    STACK_WIND(frame, default_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, &ga_loc, name, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+    STACK_UNWIND_STRICT(removexattr, frame, -1, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_gfid_access_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_gfid_access_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Memory accounting"
+               " init failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        ga_private_t *priv = NULL;
-        int ret = -1;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "not configured with exactly one child. exiting");
-                goto out;
-        }
-
-        /* This can be the top of graph in certain cases */
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
-
-        /* TODO: define a mem-type structure */
-        priv = GF_CALLOC (1, sizeof (*priv), gf_gfid_access_mt_priv_t);
-        if (!priv)
-                goto out;
-
-        priv->newfile_args_pool = mem_pool_new (ga_newfile_args_t, 512);
-        if (!priv->newfile_args_pool)
-                goto out;
-
-        priv->heal_args_pool = mem_pool_new (ga_heal_args_t, 512);
-        if (!priv->heal_args_pool)
-                goto out;
-
-        this->local_pool = mem_pool_new (ga_local_t, 16);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-        this->private = priv;
-
-        ret = 0;
+    ga_private_t *priv = NULL;
+    int ret = -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "not configured with exactly one child. exiting");
+        goto out;
+    }
+
+    /* This can be the top of graph in certain cases */
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_DEBUG, "dangling volume. check volfile ");
+    }
+
+    /* TODO: define a mem-type structure */
+    priv = GF_CALLOC(1, sizeof(*priv), gf_gfid_access_mt_priv_t);
+    if (!priv)
+        goto out;
+
+    priv->newfile_args_pool = mem_pool_new(ga_newfile_args_t, 512);
+    if (!priv->newfile_args_pool)
+        goto out;
+
+    priv->heal_args_pool = mem_pool_new(ga_heal_args_t, 512);
+    if (!priv->heal_args_pool)
+        goto out;
+
+    this->local_pool = mem_pool_new(ga_local_t, 16);
+    if (!this->local_pool) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        goto out;
+    }
+
+    this->private = priv;
+
+    ret = 0;
 out:
-        if (ret && priv) {
-                if (priv->newfile_args_pool)
-                        mem_pool_destroy (priv->newfile_args_pool);
-                GF_FREE (priv);
-        }
+    if (ret && priv) {
+        if (priv->newfile_args_pool)
+            mem_pool_destroy(priv->newfile_args_pool);
+        GF_FREE(priv);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        ga_private_t *priv = NULL;
-        priv = this->private;
-        this->private = NULL;
-
-        if (priv) {
-                if (priv->newfile_args_pool)
-                        mem_pool_destroy (priv->newfile_args_pool);
-                if (priv->heal_args_pool)
-                        mem_pool_destroy (priv->heal_args_pool);
-                GF_FREE (priv);
-        }
-
-        return;
+    ga_private_t *priv = NULL;
+    priv = this->private;
+    this->private = NULL;
+
+    if (priv) {
+        if (priv->newfile_args_pool)
+            mem_pool_destroy(priv->newfile_args_pool);
+        if (priv->heal_args_pool)
+            mem_pool_destroy(priv->heal_args_pool);
+        GF_FREE(priv);
+    }
+
+    return;
 }
 
 int32_t
-ga_dump_inodectx (xlator_t *this, inode_t *inode)
+ga_dump_inodectx(xlator_t *this, inode_t *inode)
 {
-        int       ret = -1;
-        uint64_t  value = 0;
-        inode_t  *tmp_inode = NULL;
-        char      key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret == 0) {
-                tmp_inode = (void*) value;
-                gf_proc_dump_build_key (key_prefix, this->name, "inode");
-                gf_proc_dump_add_section (key_prefix);
-                gf_proc_dump_write ("real-gfid", "%s",
-                                    uuid_utoa (tmp_inode->gfid));
-        }
-
-        return 0;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *tmp_inode = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    ret = inode_ctx_get(inode, this, &value);
+    if (ret == 0) {
+        tmp_inode = (void *)(uintptr_t)value;
+        gf_proc_dump_build_key(key_prefix, this->name, "inode");
+        gf_proc_dump_add_section("%s", key_prefix);
+        gf_proc_dump_write("real-gfid", "%s", uuid_utoa(tmp_inode->gfid));
+    }
+
+    return 0;
 }
 
 struct xlator_fops fops = {
-        .lookup = ga_lookup,
-
-        /* entry fops */
-        .mkdir   = ga_mkdir,
-        .mknod   = ga_mknod,
-        .create  = ga_create,
-        .symlink = ga_symlink,
-        .link    = ga_link,
-        .unlink  = ga_unlink,
-        .rmdir   = ga_rmdir,
-        .rename  = ga_rename,
-
-        /* handle any other directory operations here */
-        .opendir  = ga_opendir,
-        .stat     = ga_stat,
-        .setattr  = ga_setattr,
-        .getxattr = ga_getxattr,
-        .removexattr = ga_removexattr,
-
-        /* special fop to handle more entry creations */
-        .setxattr = ga_setxattr,
+    .lookup = ga_lookup,
+
+    /* entry fops */
+    .mkdir = ga_mkdir,
+    .mknod = ga_mknod,
+    .create = ga_create,
+    .symlink = ga_symlink,
+    .link = ga_link,
+    .unlink = ga_unlink,
+    .rmdir = ga_rmdir,
+    .rename = ga_rename,
+
+    /* handle any other directory operations here */
+    .opendir = ga_opendir,
+    .stat = ga_stat,
+    .setattr = ga_setattr,
+    .getxattr = ga_getxattr,
+    .removexattr = ga_removexattr,
+
+    /* special fop to handle more entry creations */
+    .setxattr = ga_setxattr,
 };
 
 struct xlator_cbks cbks = {
-        .forget = ga_forget,
+    .forget = ga_forget,
 };
 
 struct xlator_dumpops dumpops = {
-        .inodectx       = ga_dump_inodectx,
+    .inodectx = ga_dump_inodectx,
 };
 
 struct volume_options options[] = {
-        /* This translator doesn't take any options, or provide any options */
-        { .key  = {NULL} },
+    /* This translator doesn't take any options, or provide any options */
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "gfid-access",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/gfid-access/src/gfid-access.h b/xlators/features/gfid-access/src/gfid-access.h
index 2b5e4fd4184..b1e255e56c0 100644
--- a/xlators/features/gfid-access/src/gfid-access.h
+++ b/xlators/features/gfid-access/src/gfid-access.h
@@ -10,97 +10,97 @@
 #ifndef __GFID_ACCESS_H__
 #define __GFID_ACCESS_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "gfid-access-mem-types.h"
 
 #define UUID_CANONICAL_FORM_LEN 36
 
 #define GF_FUSE_AUX_GFID_NEWFILE "glusterfs.gfid.newfile"
-#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal"
 
 #define GF_GFID_KEY "GLUSTERFS_GFID"
 #define GF_GFID_DIR ".gfid"
 #define GF_AUX_GFID 0xd
 
-#define GFID_ACCESS_ENTRY_OP_CHECK(loc,err,lbl)    do {                 \
-                /* need to check if the lookup is on virtual dir */     \
-                if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&  \
-                    ((loc->parent &&                                    \
-                      __is_root_gfid (loc->parent->gfid)) ||            \
-                      __is_root_gfid (loc->pargfid))) {                 \
-                        err = ENOTSUP;                                  \
-                        goto lbl;                                       \
-                }                                                       \
-                                                                        \
-                /* now, check if the lookup() is on an existing */      \
-                /* entry, but on gfid-path */                           \
-                if ((loc->parent &&                                     \
-                     __is_gfid_access_dir (loc->parent->gfid)) ||       \
-                    __is_gfid_access_dir (loc->pargfid)) {              \
-                        err = EPERM;                                    \
-                        goto lbl;                                       \
-                }                                                       \
-        } while (0)
+#define GFID_ACCESS_ENTRY_OP_CHECK(loc, err, lbl)                              \
+    do {                                                                       \
+        /* need to check if the lookup is on virtual dir */                    \
+        if ((loc->name && !strcmp(GF_GFID_DIR, loc->name)) &&                  \
+            ((loc->parent && __is_root_gfid(loc->parent->gfid)) ||             \
+             __is_root_gfid(loc->pargfid))) {                                  \
+            err = ENOTSUP;                                                     \
+            goto lbl;                                                          \
+        }                                                                      \
+                                                                               \
+        /* now, check if the lookup() is on an existing */                     \
+        /* entry, but on gfid-path */                                          \
+        if ((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||        \
+            __is_gfid_access_dir(loc->pargfid)) {                              \
+            err = EPERM;                                                       \
+            goto lbl;                                                          \
+        }                                                                      \
+    } while (0)
 
-#define GFID_ACCESS_INODE_OP_CHECK(loc,err,lbl) do {                    \
-                /*Check if it is on .gfid*/                             \
-                if (__is_gfid_access_dir(loc->gfid)) {  \
-                        err = ENOTSUP;                                  \
-                        goto lbl;                                       \
-                }                                                       \
-        } while (0)
+#define GFID_ACCESS_INODE_OP_CHECK(loc, err, lbl)                              \
+    do {                                                                       \
+        /*Check if it is on .gfid*/                                            \
+        if (__is_gfid_access_dir(loc->gfid)) {                                 \
+            err = ENOTSUP;                                                     \
+            goto lbl;                                                          \
+        }                                                                      \
+    } while (0)
 typedef struct {
-        unsigned int  uid;
-        unsigned int  gid;
-        char          gfid[UUID_CANONICAL_FORM_LEN + 1];
-        unsigned int  st_mode;
-        char         *bname;
+    unsigned int uid;
+    unsigned int gid;
+    char gfid[UUID_CANONICAL_FORM_LEN + 1];
+    unsigned int st_mode;
+    char *bname;
 
-        union {
-                struct _symlink_in {
-                        char     *linkpath;
-                } __attribute__ ((__packed__)) symlink;
+    union {
+        struct _symlink_in {
+            char *linkpath;
+        } __attribute__((__packed__)) symlink;
 
-                struct _mknod_in {
-                        unsigned int mode;
-                        unsigned int rdev;
-                        unsigned int umask;
-                } __attribute__ ((__packed__)) mknod;
+        struct _mknod_in {
+            unsigned int mode;
+            unsigned int rdev;
+            unsigned int umask;
+        } __attribute__((__packed__)) mknod;
 
-                struct _mkdir_in {
-                        unsigned int mode;
-                        unsigned int umask;
-                } __attribute__ ((__packed__)) mkdir;
-        } __attribute__ ((__packed__)) args;
+        struct _mkdir_in {
+            unsigned int mode;
+            unsigned int umask;
+        } __attribute__((__packed__)) mkdir;
+    } __attribute__((__packed__)) args;
 } __attribute__((__packed__)) ga_newfile_args_t;
 
 typedef struct {
-        char      gfid[UUID_CANONICAL_FORM_LEN + 1];
-        char     *bname; /* a null terminated basename */
+    char gfid[UUID_CANONICAL_FORM_LEN + 1];
+    char *bname; /* a null terminated basename */
 } __attribute__((__packed__)) ga_heal_args_t;
 
 struct ga_private {
-        /* root inode's stbuf */
-        struct iatt root_stbuf;
-        struct iatt gfiddir_stbuf;
-        struct mem_pool *newfile_args_pool;
-        struct mem_pool *heal_args_pool;
+    /* root inode's stbuf */
+    struct iatt root_stbuf;
+    struct iatt gfiddir_stbuf;
+    struct mem_pool *newfile_args_pool;
+    struct mem_pool *heal_args_pool;
 };
 typedef struct ga_private ga_private_t;
 
 struct __ga_local {
-        call_frame_t *orig_frame;
-        unsigned int uid;
-        unsigned int gid;
-        loc_t        loc;
-        mode_t       mode;
-        dev_t        rdev;
-        mode_t       umask;
-        dict_t      *xdata;
+    call_frame_t *orig_frame;
+    unsigned int uid;
+    unsigned int gid;
+    loc_t loc;
+    mode_t mode;
+    dev_t rdev;
+    mode_t umask;
+    dict_t *xdata;
 };
 typedef struct __ga_local ga_local_t;
 
diff --git a/xlators/features/glupy/Makefile.am b/xlators/features/glupy/Makefile.am
deleted file mode 100644
index 060429ecf0f..00000000000
--- a/xlators/features/glupy/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src examples
-
-CLEANFILES =
diff --git a/xlators/features/glupy/doc/README.md b/xlators/features/glupy/doc/README.md
deleted file mode 100644
index 4b8b863ef39..00000000000
--- a/xlators/features/glupy/doc/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-This is just the very start for a GlusterFS[1] meta-translator that will
-allow translator code to be written in Python.  It's based on the standard
-Python embedding (not extending) techniques, plus a dash of the ctypes module.
-The interface is a pretty minimal adaptation of the dispatches and callbacks
-from the C API[2] to Python, as follows:
-
-* Dispatch functions and callbacks must be defined on an "xlator" class
-  derived from gluster.Translator so that they'll be auto-registered with
-  the C translator during initialization.
-
-* For each dispatch or callback function you want to intercept, you define a
-  Python function using the xxx\_fop\_t or xxx\_cbk\_t decorator.
-
-* The arguments for each operation are different, so you'll need to refer to
-  the C API.  GlusterFS-specific types are used (though only loc\_t is fully
-  defined so far) and type correctness is enforced by ctypes.
-
-* If you do intercept a dispatch function, it is your responsibility to call
-  xxx\_wind (like STACK\_WIND in the C API but operation-specific) to pass
-  the request to the next translator.  If you do not intercept a function, it
-  will default the same way as for C (pass through to the same operation with
-  the same arguments on the first child translator).
-
-* If you intercept a callback function, it is your responsibility to call
-  xxx\_unwind (like STACK\_UNWIND\_STRICT in the C API) to pass the request back
-  to the caller.
-
-So far only the lookup and create operations are handled this way, to support
-the "negative lookup" example.  Now that the basic infrastructure is in place,
-adding more functions should be very quick, though with that much boilerplate I
-might pause to write a code generator.  I also plan to add structure
-definitions and interfaces for some of the utility functions in libglusterfs
-(especially those having to do with inode and fd context) in the fairly near
-future.  Note that you can also use ctypes to get at anything not explicitly
-exposed to Python already.
-
-_If you're coming here because of the Linux Journal article, please note that
-the code has evolved since that was written. The version that matches the
-article is here:_
-
-https://github.com/jdarcy/glupy/tree/4bbae91ba459ea46ef32f2966562492e4ca9187a
-
-[1] http://www.gluster.org
-[2] http://pl.atyp.us/hekafs.org/dist/xlator_api_2.html
diff --git a/xlators/features/glupy/doc/TESTING b/xlators/features/glupy/doc/TESTING
deleted file mode 100644
index e05f17f498f..00000000000
--- a/xlators/features/glupy/doc/TESTING
+++ /dev/null
@@ -1,9 +0,0 @@
-Loading a translator written in Python using the glupy meta translator
--------------------------------------------------------------------------------
-'test.vol' is a simple volfile with the debug-trace Python translator on top
-of a brick. The volfile can be mounted using the following command.
-
-$ glusterfs --debug -f test.vol /path/to/mntpt
-
-If then file operations are performed on the newly mounted file system, log
-output would be printed by the Python translator on the standard output.
diff --git a/xlators/features/glupy/doc/test.vol b/xlators/features/glupy/doc/test.vol
deleted file mode 100644
index 0751a488c1f..00000000000
--- a/xlators/features/glupy/doc/test.vol
+++ /dev/null
@@ -1,10 +0,0 @@
-volume vol-posix
-    type storage/posix
-    option directory /path/to/brick
-end-volume
-
-volume vol-glupy
-    type features/glupy
-    option module-name debug-trace
-    subvolumes vol-posix
-end-volume
diff --git a/xlators/features/glupy/examples/Makefile.am b/xlators/features/glupy/examples/Makefile.am
deleted file mode 100644
index c26abeaafb6..00000000000
--- a/xlators/features/glupy/examples/Makefile.am
+++ /dev/null
@@ -1,5 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-glupyexamplesdir = $(xlatordir)/glupy
-
-glupyexamples_PYTHON = negative.py helloworld.py debug-trace.py
diff --git a/xlators/features/glupy/examples/debug-trace.py b/xlators/features/glupy/examples/debug-trace.py
deleted file mode 100644
index 6eef1b58b8f..00000000000
--- a/xlators/features/glupy/examples/debug-trace.py
+++ /dev/null
@@ -1,775 +0,0 @@
-import sys
-import stat
-from uuid import UUID
-from time import strftime, localtime
-from gluster.glupy import *
-
-# This translator was written primarily to test the fop entry point definitions
-# and structure definitions in 'glupy.py'.
-
-# It is similar to the C language debug-trace translator, which logs the
-# arguments passed to the fops and their corresponding cbk functions.
-
-dl.get_id.restype = c_long
-dl.get_id.argtypes = [ POINTER(call_frame_t) ]
-
-dl.get_rootunique.restype = c_uint64
-dl.get_rootunique.argtypes = [ POINTER(call_frame_t) ]
-
-def uuid2str (gfid):
-        return str(UUID(''.join(map("{0:02x}".format, gfid))))
-
-
-def st_mode_from_ia (prot, filetype):
-        st_mode = 0
-        type_bit = 0
-        prot_bit = 0
-
-        if filetype == IA_IFREG:
-                type_bit = stat.S_IFREG
-        elif filetype == IA_IFDIR:
-                type_bit = stat.S_IFDIR
-        elif filetype == IA_IFLNK:
-                type_bit = stat.S_IFLNK
-        elif filetype == IA_IFBLK:
-                type_bit = stat.S_IFBLK
-        elif filetype == IA_IFCHR:
-                type_bit = stat.S_IFCHR
-        elif filetype == IA_IFIFO:
-                type_bit = stat.S_IFIFO
-        elif filetype == IA_IFSOCK:
-                type_bit = stat.S_IFSOCK
-        elif filetype == IA_INVAL:
-                pass
-
-
-        if prot.suid:
-                prot_bit |= stat.S_ISUID
-        if prot.sgid:
-                prot_bit |= stat.S_ISGID
-        if prot.sticky:
-                prot_bit |= stat.S_ISVTX
-
-        if prot.owner.read:
-                prot_bit |= stat.S_IRUSR
-        if prot.owner.write:
-                prot_bit |= stat.S_IWUSR
-        if prot.owner.execn:
-                prot_bit |= stat.S_IXUSR
-
-        if prot.group.read:
-                prot_bit |= stat.S_IRGRP
-        if prot.group.write:
-                prot_bit |= stat.S_IWGRP
-        if prot.group.execn:
-                prot_bit |= stat.S_IXGRP
-
-        if prot.other.read:
-                prot_bit |= stat.S_IROTH
-        if prot.other.write:
-                prot_bit |= stat.S_IWOTH
-        if prot.other.execn:
-                prot_bit |= stat.S_IXOTH
-
-        st_mode = (type_bit | prot_bit)
-
-        return st_mode
-
-
-def trace_stat2str (buf):
-        gfid = uuid2str(buf.contents.ia_gfid)
-        mode = st_mode_from_ia(buf.contents.ia_prot, buf.contents.ia_type)
-        atime_buf = strftime("[%b %d %H:%M:%S]",
-                             localtime(buf.contents.ia_atime))
-        mtime_buf = strftime("[%b %d %H:%M:%S]",
-                             localtime(buf.contents.ia_mtime))
-        ctime_buf = strftime("[%b %d %H:%M:%S]",
-                             localtime(buf.contents.ia_ctime))
-        return ("(gfid={0:s}, ino={1:d}, mode={2:o}, nlink={3:d}, uid ={4:d}, "+
-                "gid ={5:d}, size={6:d}, blocks={7:d}, atime={8:s}, mtime={9:s}, "+
-                "ctime={10:s})").format(gfid, buf.contents.ia_no, mode,
-                                        buf.contents.ia_nlink,
-                                        buf.contents.ia_uid,
-                                        buf.contents.ia_gid,
-                                        buf.contents.ia_size,
-                                        buf.contents.ia_blocks,
-                                        atime_buf, mtime_buf,
-                                        ctime_buf)
-
-class xlator(Translator):
-
-        def __init__(self, c_this):
-                Translator.__init__(self, c_this)
-                self.gfids = {}
-
-        def lookup_fop(self, frame, this, loc, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.gfid)
-                print("GLUPY TRACE LOOKUP FOP- {0:d}: gfid={1:s}; " +
-                      "path={2:s}").format(unique, gfid, loc.contents.path)
-                self.gfids[key] = gfid
-                dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
-                return 0
-
-        def lookup_cbk(self, frame, cookie, this, op_ret, op_errno,
-                       inode, buf, xdata, postparent):
-                unique =dl.get_rootunique(frame)
-                key =dl.get_id(frame)
-                if op_ret == 0:
-                        gfid = uuid2str(buf.contents.ia_gfid)
-                        statstr = trace_stat2str(buf)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE LOOKUP CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *buf={3:s}; " +
-                              "*postparent={4:s}").format(unique, gfid,
-                                                          op_ret, statstr,
-                                                          postparentstr)
-                else:
-                        gfid = self.gfids[key]
-                        print("GLUPY TRACE LOOKUP CBK - {0:d}: gfid={1:s};" +
-                              " op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                      gfid,
-                                                                      op_ret,
-                                                                      op_errno)
-                del self.gfids[key]
-                dl.unwind_lookup(frame, cookie, this, op_ret, op_errno,
-                                 inode, buf, xdata, postparent)
-                return 0
-
-        def create_fop(self, frame, this, loc, flags, mode, umask, fd,
-                       xdata):
-                unique = dl.get_rootunique(frame)
-                gfid = uuid2str(loc.contents.gfid)
-                print("GLUPY TRACE CREATE FOP- {0:d}: gfid={1:s}; path={2:s}; " +
-                      "fd={3:s}; flags=0{4:o}; mode=0{5:o}; " +
-                      "umask=0{6:o}").format(unique, gfid, loc.contents.path,
-                                             fd, flags, mode, umask)
-                dl.wind_create(frame, POINTER(xlator_t)(), loc, flags,mode,
-                               umask, fd, xdata)
-                return 0
-
-        def create_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
-                       inode, buf, preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                if op_ret >= 0:
-                        gfid = uuid2str(inode.contents.gfid)
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE CREATE CBK- {0:d}: gfid={1:s};" +
-                              " op_ret={2:d}; fd={3:s}; *stbuf={4:s}; " +
-                              "*preparent={5:s};" +
-                              " *postparent={6:s}").format(unique, gfid, op_ret,
-                                                           fd, statstr,
-                                                           preparentstr,
-                                                           postparentstr)
-                else:
-                        print ("GLUPY TRACE CREATE CBK- {0:d}: op_ret={1:d}; " +
-                              "op_errno={2:d}").format(unique, op_ret, op_errno)
-                dl.unwind_create(frame, cookie, this, op_ret, op_errno, fd,
-                                 inode, buf, preparent, postparent, xdata)
-                return 0
-
-        def open_fop(self, frame, this, loc, flags, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE OPEN FOP- {0:d}: gfid={1:s}; path={2:s}; "+
-                      "flags={3:d}; fd={4:s}").format(unique, gfid,
-                                                      loc.contents.path, flags,
-                                                      fd)
-                self.gfids[key] = gfid
-                dl.wind_open(frame, POINTER(xlator_t)(), loc, flags, fd, xdata)
-                return 0
-
-        def open_cbk(self, frame, cookie, this, op_ret, op_errno, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE OPEN CBK- {0:d}: gfid={1:s}; op_ret={2:d}; "
-                      "op_errno={3:d}; *fd={4:s}").format(unique, gfid,
-                                                          op_ret, op_errno, fd)
-                del self.gfids[key]
-                dl.unwind_open(frame, cookie, this, op_ret, op_errno, fd,
-                               xdata)
-                return 0
-
-        def readv_fop(self, frame, this, fd, size, offset, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE READV FOP- {0:d}: gfid={1:s}; "+
-                      "fd={2:s}; size ={3:d}; offset={4:d}; " +
-                      "flags=0{5:x}").format(unique, gfid, fd, size, offset,
-                                             flags)
-                self.gfids[key] = gfid
-                dl.wind_readv (frame, POINTER(xlator_t)(), fd, size, offset,
-                               flags, xdata)
-                return 0
-
-        def readv_cbk(self, frame, cookie, this, op_ret, op_errno, vector,
-                      count, buf, iobref, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret >= 0:
-                        statstr = trace_stat2str(buf)
-                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
-                              "op_ret={2:d}; *buf={3:s};").format(unique, gfid,
-                                                                  op_ret,
-                                                                  statstr)
-
-                else:
-                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_readv (frame, cookie, this, op_ret, op_errno,
-                                 vector, count, buf, iobref, xdata)
-                return 0
-
-        def writev_fop(self, frame, this, fd, vector, count, offset, flags,
-                       iobref, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE  WRITEV FOP- {0:d}: gfid={1:s}; " +
-                      "fd={2:s}; count={3:d}; offset={4:d}; " +
-                      "flags=0{5:x}").format(unique, gfid, fd, count, offset,
-                                             flags)
-                self.gfids[key] = gfid
-                dl.wind_writev(frame, POINTER(xlator_t)(), fd, vector, count,
-                               offset, flags, iobref, xdata)
-                return 0
-
-        def writev_cbk(self, frame, cookie, this, op_ret, op_errno, prebuf,
-                       postbuf, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                if op_ret >= 0:
-                        preopstr = trace_stat2str(prebuf)
-                        postopstr = trace_stat2str(postbuf)
-                        print("GLUPY TRACE WRITEV CBK- {0:d}: op_ret={1:d}; " +
-                              "*prebuf={2:s}; " +
-                              "*postbuf={3:s}").format(unique, op_ret, preopstr,
-                                                       postopstr)
-                else:
-                        gfid = self.gfids[key]
-                        print("GLUPY TRACE WRITEV CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_writev (frame, cookie, this, op_ret, op_errno,
-                                  prebuf, postbuf, xdata)
-                return 0
-
-        def opendir_fop(self, frame, this, loc, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE OPENDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
-                      "fd={3:s}").format(unique, gfid, loc.contents.path, fd)
-                self.gfids[key] = gfid
-                dl.wind_opendir(frame, POINTER(xlator_t)(), loc, fd, xdata)
-                return 0
-
-        def opendir_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
-                        xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE OPENDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
-                      " op_errno={3:d}; fd={4:s}").format(unique, gfid, op_ret,
-                                                          op_errno, fd)
-                del self.gfids[key]
-                dl.unwind_opendir(frame, cookie, this, op_ret, op_errno,
-                                  fd, xdata)
-                return 0
-
-        def readdir_fop(self, frame, this, fd, size, offset, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE READDIR FOP- {0:d}:  gfid={1:s}; fd={2:s}; " +
-                      "size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
-                                                         offset)
-                self.gfids[key] = gfid
-                dl.wind_readdir(frame, POINTER(xlator_t)(), fd, size, offset,
-                                xdata)
-                return 0
-
-        def readdir_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                        xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE READDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
-                      " op_errno={3:d}").format(unique, gfid, op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_readdir(frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata)
-                return 0
-
-        def readdirp_fop(self, frame, this, fd, size, offset, dictionary):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE READDIRP FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
-                      " size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
-                                                          offset)
-                self.gfids[key] = gfid
-                dl.wind_readdirp(frame, POINTER(xlator_t)(), fd, size, offset,
-                                 dictionary)
-                return 0
-
-        def readdirp_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                         xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE READDIRP CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                             op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_readdirp(frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata)
-                return 0
-
-        def mkdir_fop(self, frame, this, loc, mode, umask, xdata):
-                unique = dl.get_rootunique(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE MKDIR FOP- {0:d}: gfid={1:s}; path={2:s}; " +
-                      "mode={3:d}; umask=0{4:o}").format(unique, gfid,
-                                                         loc.contents.path, mode,
-                                                         umask)
-                dl.wind_mkdir(frame, POINTER(xlator_t)(), loc, mode, umask,
-                              xdata)
-                return 0
-
-        def mkdir_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                      preparent, postparent,  xdata):
-                unique = dl.get_rootunique(frame)
-                if op_ret == 0:
-                        gfid = uuid2str(inode.contents.gfid)
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE MKDIR CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *stbuf={3:s}; *prebuf={4:s}; "+
-                              "*postbuf={5:s} ").format(unique, gfid, op_ret,
-                                                        statstr,
-                                                        preparentstr,
-                                                        postparentstr)
-                else:
-                        print("GLUPY TRACE MKDIR CBK- {0:d}:  op_ret={1:d}; "+
-                              "op_errno={2:d}").format(unique, op_ret, op_errno)
-                dl.unwind_mkdir(frame, cookie, this, op_ret, op_errno, inode,
-                                buf, preparent, postparent, xdata)
-                return 0
-
-        def rmdir_fop(self, frame, this, loc, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE RMDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
-                      "flags={3:d}").format(unique, gfid, loc.contents.path,
-                                            flags)
-                self.gfids[key] = gfid
-                dl.wind_rmdir(frame, POINTER(xlator_t)(), loc, flags, xdata)
-                return 0
-
-        def rmdir_cbk(self, frame, cookie, this, op_ret, op_errno, preparent,
-                      postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *prebuf={3:s}; "+
-                              "*postbuf={4:s}").format(unique, gfid, op_ret,
-                                                       preparentstr,
-                                                       postparentstr)
-                else:
-                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_rmdir(frame, cookie, this, op_ret, op_errno,
-                                preparent, postparent, xdata)
-                return 0
-
-        def stat_fop(self, frame, this, loc, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE STAT FOP- {0:d}: gfid={1:s}; " +
-                      " path={2:s}").format(unique, gfid, loc.contents.path)
-                self.gfids[key] = gfid
-                dl.wind_stat(frame, POINTER(xlator_t)(), loc, xdata)
-                return 0
-
-        def stat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                     xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d};  *buf={3:s};").format(unique,
-                                                                   gfid,
-                                                                   op_ret,
-                                                                   statstr)
-                else:
-                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_stat(frame, cookie, this, op_ret, op_errno,
-                               buf, xdata)
-                return 0
-
-        def fstat_fop(self, frame, this, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE FSTAT FOP- {0:d}:  gfid={1:s}; " +
-                      "fd={2:s}").format(unique, gfid, fd)
-                self.gfids[key] = gfid
-                dl.wind_fstat(frame, POINTER(xlator_t)(), fd, xdata)
-                return 0
-
-        def fstat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                      xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
-                              " op_ret={2:d}; *buf={3:s}").format(unique,
-                                                                  gfid,
-                                                                  op_ret,
-                                                                  statstr)
-                else:
-                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique.
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_fstat(frame, cookie, this, op_ret, op_errno,
-                                buf, xdata)
-                return 0
-
-        def statfs_fop(self, frame, this, loc, xdata):
-                unique = dl.get_rootunique(frame)
-                if loc.contents.inode:
-                        gfid = uuid2str(loc.contents.inode.contents.gfid)
-                else:
-                        gfid = "0"
-                print("GLUPY TRACE STATFS FOP- {0:d}: gfid={1:s}; "+
-                      "path={2:s}").format(unique, gfid, loc.contents.path)
-                dl.wind_statfs(frame, POINTER(xlator_t)(), loc, xdata)
-                return 0
-
-        def statfs_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                       xdata):
-                unique = dl.get_rootunique(frame)
-                if op_ret == 0:
-                        #TBD: print buf (pointer to an iovec type object)
-                        print("GLUPY TRACE STATFS CBK {0:d}: "+
-                              "op_ret={1:d}").format(unique, op_ret)
-                else:
-                        print("GLUPY TRACE STATFS CBK-  {0:d}"+
-                              "op_ret={1:d}; op_errno={2:d}").format(unique,
-                                                                     op_ret,
-                                                                     op_errno)
-                dl.unwind_statfs(frame, cookie, this, op_ret, op_errno,
-                                 buf, xdata)
-                return 0
-
-        def getxattr_fop(self, frame, this, loc, name, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE GETXATTR FOP- {0:d}: gfid={1:s}; path={2:s};"+
-                      " name={3:s}").format(unique, gfid, loc.contents.path,
-                                            name)
-                self.gfids[key]=gfid
-                dl.wind_getxattr(frame, POINTER(xlator_t)(), loc, name, xdata)
-                return 0
-
-        def getxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
-                         dictionary, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE GETXATTR CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d}; "+
-                      " dictionary={4:s}").format(unique, gfid, op_ret, op_errno,
-                                                  dictionary)
-                del self.gfids[key]
-                dl.unwind_getxattr(frame, cookie, this, op_ret, op_errno,
-                                   dictionary, xdata)
-                return 0
-
-        def fgetxattr_fop(self, frame, this, fd, name, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE FGETXATTR FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
-                      "name={3:s}").format(unique, gfid, fd, name)
-                self.gfids[key] = gfid
-                dl.wind_fgetxattr(frame, POINTER(xlator_t)(), fd, name, xdata)
-                return 0
-
-        def fgetxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
-                          dictionary, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE FGETXATTR CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d};"+
-                      " dictionary={4:s}").format(unique, gfid, op_ret,
-                                                  op_errno, dictionary)
-                del self.gfids[key]
-                dl.unwind_fgetxattr(frame, cookie, this, op_ret, op_errno,
-                                    dictionary, xdata)
-                return 0
-
-        def setxattr_fop(self, frame, this, loc, dictionary, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE SETXATTR FOP- {0:d}:  gfid={1:s}; path={2:s};"+
-                      " flags={3:d}").format(unique, gfid, loc.contents.path,
-                                             flags)
-                self.gfids[key] = gfid
-                dl.wind_setxattr(frame, POINTER(xlator_t)(), loc, dictionary,
-                                 flags, xdata)
-                return 0
-
-        def setxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE SETXATTR CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                             op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_setxattr(frame, cookie, this, op_ret, op_errno,
-                                   xdata)
-                return 0
-
-        def fsetxattr_fop(self, frame, this, fd, dictionary, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE FSETXATTR FOP- {0:d}: gfid={1:s}; fd={2:p}; "+
-                      "flags={3:d}").format(unique, gfid, fd, flags)
-                self.gfids[key] = gfid
-                dl.wind_fsetxattr(frame, POINTER(xlator_t)(), fd, dictionary,
-                                  flags, xdata)
-                return 0
-
-        def fsetxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE FSETXATTR CBK- {0:d}: gfid={1:s};  "+
-                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                             op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_fsetxattr(frame, cookie, this, op_ret, op_errno,
-                                   xdata)
-                return 0
-
-        def removexattr_fop(self, frame, this, loc, name, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE REMOVEXATTR FOP- {0:d}:  gfid={1:s}; "+
-                      "path={2:s}; name={3:s}").format(unique, gfid,
-                                                       loc.contents.path,
-                                                       name)
-                self.gfids[key] = gfid
-                dl.wind_removexattr(frame, POINTER(xlator_t)(), loc, name,
-                                    xdata)
-                return 0
-
-        def removexattr_cbk(self, frame, cookie, this, op_ret, op_errno,
-                            xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE REMOVEXATTR CBK- {0:d}: gfid={1:s} "+
-                      " op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                              op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_removexattr(frame, cookie, this, op_ret, op_errno,
-                                      xdata)
-                return 0
-
-        def link_fop(self, frame, this, oldloc, newloc, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                if (newloc.contents.inode):
-                        newgfid = uuid2str(newloc.contents.inode.contents.gfid)
-                else:
-                        newgfid = "0"
-                oldgfid = uuid2str(oldloc.contents.inode.contents.gfid)
-                print("GLUPY TRACE LINK FOP-{0:d}: oldgfid={1:s}; oldpath={2:s};"+
-                      "newgfid={3:s};"+
-                      "newpath={4:s}").format(unique, oldgfid,
-                                              oldloc.contents.path,
-                                              newgfid,
-                                              newloc.contents.path)
-                self.gfids[key] =  oldgfid
-                dl.wind_link(frame, POINTER(xlator_t)(), oldloc, newloc,
-                             xdata)
-                return 0
-
-        def link_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                     preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE LINK CBK- {0:d}: op_ret={1:d} "+
-                              "*stbuf={2:s}; *prebuf={3:s}; "+
-                              "*postbuf={4:s} ").format(unique, op_ret, statstr,
-                                                        preparentstr,
-                                                        postparentstr)
-                else:
-                        print("GLUPY TRACE LINK CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; "+
-                              "op_errno={3:d}").format(unique, gfid,
-                                                       op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_link(frame, cookie, this, op_ret, op_errno, inode,
-                               buf, preparent, postparent, xdata)
-                return 0
-
-        def unlink_fop(self, frame, this, loc, xflag, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE UNLINK FOP- {0:d}; gfid={1:s}; path={2:s}; "+
-                      "flag={3:d}").format(unique, gfid, loc.contents.path,
-                                           xflag)
-                self.gfids[key] = gfid
-                dl.wind_unlink(frame, POINTER(xlator_t)(), loc, xflag,
-                               xdata)
-                return 0
-
-        def unlink_cbk(self, frame, cookie, this, op_ret, op_errno,
-                       preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE UNLINK CBK- {0:d}: gfid ={1:s}; "+
-                              "op_ret={2:d}; *prebuf={3:s}; "+
-                              "*postbuf={4:s} ").format(unique, gfid, op_ret,
-                                                        preparentstr,
-                                                        postparentstr)
-                else:
-                        print("GLUPY TRACE UNLINK CBK: {0:d}: gfid ={1:s}; "+
-                              "op_ret={2:d}; "+
-                              "op_errno={3:d}").format(unique, gfid, op_ret,
-                                                       op_errno)
-                del self.gfids[key]
-                dl.unwind_unlink(frame, cookie, this, op_ret, op_errno,
-                                 preparent, postparent, xdata)
-                return 0
-
-        def readlink_fop(self, frame, this, loc, size, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE READLINK FOP- {0:d}:  gfid={1:s}; path={2:s};"+
-                      " size={3:d}").format(unique, gfid, loc.contents.path,
-                                            size)
-                self.gfids[key] = gfid
-                dl.wind_readlink(frame, POINTER(xlator_t)(), loc, size,
-                               xdata)
-                return 0
-
-        def readlink_cbk(self, frame, cookie, this, op_ret, op_errno,
-                         buf, stbuf, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid  = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(stbuf)
-                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
-                              " op_ret={2:d}; op_errno={3:d}; *prebuf={4:s}; "+
-                              "*postbuf={5:s} ").format(unique, gfid,
-                                                        op_ret, op_errno,
-                                                        buf, statstr)
-                else:
-                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
-                              " op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                      gfid,
-                                                                      op_ret,
-                                                                      op_errno)
-                del self.gfids[key]
-                dl.unwind_readlink(frame, cookie, this, op_ret, op_errno, buf,
-                                   stbuf, xdata)
-                return 0
-
-        def symlink_fop(self, frame, this, linkpath, loc, umask, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE SYMLINK FOP- {0:d}: gfid={1:s}; "+
-                      "linkpath={2:s}; path={3:s};"+
-                      "umask=0{4:o}").format(unique, gfid, linkpath,
-                                             loc.contents.path, umask)
-                self.gfids[key] = gfid
-                dl.wind_symlink(frame, POINTER(xlator_t)(), linkpath, loc,
-                                umask, xdata)
-                return 0
-
-        def symlink_cbk(self, frame, cookie, this, op_ret, op_errno,
-                        inode, buf, preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *stbuf={3:s}; *preparent={4:s}; "+
-                              "*postparent={5:s}").format(unique, gfid,
-                                                          op_ret, statstr,
-                                                          preparentstr,
-                                                          postparentstr)
-                else:
-                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_symlink(frame, cookie, this, op_ret, op_errno,
-                                  inode, buf, preparent, postparent, xdata)
-                return 0
diff --git a/xlators/features/glupy/examples/helloworld.py b/xlators/features/glupy/examples/helloworld.py
deleted file mode 100644
index b565a4e5bc3..00000000000
--- a/xlators/features/glupy/examples/helloworld.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import sys
-from gluster.glupy import *
-
-class xlator (Translator):
-
-    def __init__(self, c_this):
-        Translator.__init__(self, c_this)
-
-    def lookup_fop(self, frame, this, loc, xdata):
-        print "Python xlator: Hello!"
-        dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
-        return 0
-
-    def lookup_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                   xdata, postparent):
-        print "Python xlator: Hello again!"
-        dl.unwind_lookup(frame, cookie, this, op_ret, op_errno, inode, buf,
-                         xdata, postparent)
-        return 0
diff --git a/xlators/features/glupy/examples/negative.py b/xlators/features/glupy/examples/negative.py
deleted file mode 100644
index e7a4fc07ced..00000000000
--- a/xlators/features/glupy/examples/negative.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import sys
-from uuid import UUID
-from gluster.glupy import *
-
-# Negative-lookup-caching example.  If a file wasn't there the last time we
-# looked, it's probably still not there.  This translator keeps track of
-# those failed lookups for us, and returns ENOENT without needing to pass the
-# call any further for repeated requests.
-
-# If we were doing this for real, we'd need separate caches for each xlator
-# instance.  The easiest way to do this would be to have xlator.__init__
-# "register" each instance in a module-global dict, with the key as the C
-# translator address and the value as the xlator object itself.  For testing
-# and teaching, it's sufficient just to have one cache.  The keys are parent
-# GFIDs, and the entries are lists of names within that parent that we know
-# don't exist.
-cache = {}
-
-# TBD: we need a better way of handling per-request data (frame->local in C).
-dl.get_id.restype = c_long
-dl.get_id.argtypes = [ POINTER(call_frame_t) ]
-
-def uuid2str (gfid):
-    return str(UUID(''.join(map("{0:02x}".format, gfid))))
-
-class xlator (Translator):
-
-    def __init__ (self, c_this):
-        self.requests = {}
-        Translator.__init__(self,c_this)
-
-    def lookup_fop (self, frame, this, loc, xdata):
-        pargfid = uuid2str(loc.contents.pargfid)
-        print "lookup FOP: %s:%s" % (pargfid, loc.contents.name)
-        # Check the cache.
-        if cache.has_key(pargfid):
-            if loc.contents.name in cache[pargfid]:
-                print "short-circuiting for %s:%s" % (pargfid,
-                    loc.contents.name)
-                dl.unwind_lookup(frame,0,this,-1,2,None,None,None,None)
-                return 0
-        key = dl.get_id(frame)
-        self.requests[key] = (pargfid, loc.contents.name[:])
-        # TBD: get real child xl from init, pass it here
-        dl.wind_lookup(frame,POINTER(xlator_t)(),loc,xdata)
-        return 0
-
-    def lookup_cbk (self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                    xdata, postparent):
-        print "lookup CBK: %d (%d)" % (op_ret, op_errno)
-        key = dl.get_id(frame)
-        pargfid, name = self.requests[key]
-        # Update the cache.
-        if op_ret == 0:
-            print "found %s, removing from cache" % name
-            if cache.has_key(pargfid):
-                cache[pargfid].discard(name)
-        elif op_errno == 2:    # ENOENT
-            print "failed to find %s, adding to cache" % name
-            if cache.has_key(pargfid):
-                cache[pargfid].add(name)
-            else:
-                cache[pargfid] = set([name])
-        del self.requests[key]
-        dl.unwind_lookup(frame,cookie,this,op_ret,op_errno,
-                         inode,buf,xdata,postparent)
-        return 0
-
-    def create_fop (self, frame, this, loc, flags, mode, umask, fd, xdata):
-        pargfid = uuid2str(loc.contents.pargfid)
-        print "create FOP: %s:%s" % (pargfid, loc.contents.name)
-        key = dl.get_id(frame)
-        self.requests[key] = (pargfid, loc.contents.name[:])
-        # TBD: get real child xl from init, pass it here
-        dl.wind_create(frame,POINTER(xlator_t)(),loc,flags,mode,umask,fd,xdata)
-        return 0
-
-    def create_cbk (self, frame, cookie, this, op_ret, op_errno, fd, inode,
-                    buf, preparent, postparent, xdata):
-        print "create CBK: %d (%d)" % (op_ret, op_errno)
-        key = dl.get_id(frame)
-        pargfid, name = self.requests[key]
-        # Update the cache.
-        if op_ret == 0:
-            print "created %s, removing from cache" % name
-            if cache.has_key(pargfid):
-                cache[pargfid].discard(name)
-        del self.requests[key]
-        dl.unwind_create(frame,cookie,this,op_ret,op_errno,fd,inode,buf,
-                         preparent,postparent,xdata)
-        return 0
diff --git a/xlators/features/glupy/src/Makefile.am b/xlators/features/glupy/src/Makefile.am
deleted file mode 100644
index 9b39b4687a3..00000000000
--- a/xlators/features/glupy/src/Makefile.am
+++ /dev/null
@@ -1,29 +0,0 @@
-xlator_LTLIBRARIES = glupy.la
-
-# Ensure GLUSTER_PYTHON_PATH is passed to glupy.so
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-glupydir = $(xlatordir)/glupy
-AM_CPPFLAGS = $(PYTHONDEV_CPPFLAGS) $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -isystem $(BUILD_PYTHON_INC)
-AM_CFLAGS = $(PYTHONDEV_CPPFLAGS) -Wall -fno-strict-aliasing \
-            -DGLUSTER_PYTHON_PATH=\"$(glupydir)\" \
-            -DPATH_GLUSTERFS_GLUPY_MODULE=\"${xlatordir}/glupy${shrext_cmds}\" \
-            $(GF_CFLAGS)
-
-# Flags to build glupy.so with
-glupy_la_LDFLAGS = $(PYTHONDEV_LDFLAGS) -module -avoid-version -nostartfiles -export-symbols $(top_srcdir)/xlators/features/glupy/src/glupy.sym
-glupy_la_SOURCES = glupy.c
-glupy_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	 -lpthread -l$(BUILD_PYTHON_LIB)
-
-noinst_HEADERS = glupy.h
-
-# Install __init__.py into the Python site-packages area
-pyglupydir = @BUILD_PYTHON_SITE_PACKAGES@/gluster
-pyglupy_PYTHON = __init__.py
-
-# Install glupy/__init_-.py into the Python site-packages area
-SUBDIRS = glupy
-
-CLEANFILES =
-
-EXTRA_DIST = glupy.sym
diff --git a/xlators/features/glupy/src/__init__.py.in b/xlators/features/glupy/src/__init__.py.in
deleted file mode 100644
index 3ad9513f40e..00000000000
--- a/xlators/features/glupy/src/__init__.py.in
+++ /dev/null
@@ -1,2 +0,0 @@
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
diff --git a/xlators/features/glupy/src/glupy.c b/xlators/features/glupy/src/glupy.c
deleted file mode 100644
index bca476427c8..00000000000
--- a/xlators/features/glupy/src/glupy.c
+++ /dev/null
@@ -1,2496 +0,0 @@
-/*
-   Copyright (c) 2006-2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include <ctype.h>
-#include <sys/uio.h>
-#include <Python.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-#include "glupy.h"
-
-/* UTILITY FUNCTIONS FOR FOP-SPECIFIC CODE */
-
-pthread_key_t gil_init_key;
-
-PyGILState_STATE
-glupy_enter (void)
-{
-        if (!pthread_getspecific(gil_init_key)) {
-                PyEval_ReleaseLock();
-                (void)pthread_setspecific(gil_init_key,(void *)1);
-        }
-
-        return PyGILState_Ensure();
-}
-
-void
-glupy_leave (PyGILState_STATE gstate)
-{
-        PyGILState_Release(gstate);
-}
-
-/* FOP: LOOKUP */
-
-int32_t
-glupy_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_LOOKUP]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_lookup_cbk_t)(priv->cbks[GLUPY_LOOKUP]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, xdata, postparent);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
-}
-
-int32_t
-glupy_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_LOOKUP]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_lookup_t)(priv->fops[GLUPY_LOOKUP]))(
-                frame, this, loc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
-}
-
-void
-wind_lookup (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_lookup_cbk,xl,xl->fops->lookup,loc,xdata);
-}
-
-void
-unwind_lookup (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(lookup,frame,op_ret,op_errno,
-                            inode,buf,xdata,postparent);
-}
-
-void
-set_lookup_fop (long py_this, fop_lookup_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_LOOKUP] = (long)fop;
-}
-
-void
-set_lookup_cbk (long py_this, fop_lookup_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_LOOKUP] = (long)cbk;
-}
-
-/* FOP: CREATE */
-
-int32_t
-glupy_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_CREATE]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_create_cbk_t)(priv->cbks[GLUPY_CREATE]))(
-                frame, cookie, this, op_ret, op_errno,
-                fd, inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_CREATE]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_create_t)(priv->fops[GLUPY_CREATE]))(
-                frame, this, loc, flags, mode, umask, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
-                    fd, xdata);
-        return 0;
-}
-
-void
-wind_create (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
-             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_create_cbk,xl, xl->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-}
-
-void
-unwind_create (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_create_fop (long py_this, fop_create_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_CREATE] = (long)fop;
-}
-
-void
-set_create_cbk (long py_this, fop_create_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_CREATE] = (long)cbk;
-}
-
-/* FOP: OPEN */
-
-int32_t
-glupy_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_OPEN]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_open_cbk_t)(priv->cbks[GLUPY_OPEN]))(
-                frame, cookie, this, op_ret, op_errno,
-                fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-int32_t
-glupy_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_OPEN]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_open_t)(priv->fops[GLUPY_OPEN]))(
-                frame, this, loc, flags, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-        return 0;
-}
-
-void
-wind_open (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
-           fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_open_cbk, xl, xl->fops->open, loc, flags,
-                    fd, xdata);
-}
-
-void
-unwind_open (call_frame_t *frame, long cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-}
-
-void
-set_open_fop (long py_this, fop_open_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->fops[GLUPY_OPEN] = (long)fop;
-}
-
-void
-set_open_cbk (long py_this, fop_open_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->cbks[GLUPY_OPEN] = (long)cbk;
-}
-
-/* FOP: READV */
-
-int32_t
-glupy_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                 dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READV]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readv_cbk_t)(priv->cbks[GLUPY_READV]))(
-                frame, cookie, this, op_ret, op_errno,
-                vector, count, stbuf, iobref, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
-                             count, stbuf, iobref, xdata);
-        return 0;
-}
-
-int32_t
-glupy_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READV]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readv_t)(priv->fops[GLUPY_READV]))(
-                frame, this, fd, size, offset, flags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset,
-                    flags, xdata);
-        return 0;
-}
-
-void
-wind_readv (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
-            off_t offset, uint32_t flags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_readv_cbk, xl, xl->fops->readv, fd, size,
-                    offset, flags, xdata);
-}
-
-void
-unwind_readv (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iovec *vector,
-              int32_t count, struct iatt *stbuf, struct iobref *iobref,
-              dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
-                             count, stbuf, iobref, xdata);
-}
-
-void
-set_readv_fop (long py_this, fop_readv_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->fops[GLUPY_READV] = (long)fop;
-}
-
-void
-set_readv_cbk (long py_this, fop_readv_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->cbks[GLUPY_READV] = (long)cbk;
-}
-
-/* FOP: WRITEV */
-
-int32_t
-glupy_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_WRITEV]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_writev_cbk_t)(priv->cbks[GLUPY_WRITEV]))(
-                frame, cookie, this, op_ret, op_errno,
-                prebuf, postbuf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count, off_t offset,
-              uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_WRITEV]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_writev_t)(priv->fops[GLUPY_WRITEV]))(
-                frame, this, fd, vector, count, offset, flags,
-                iobref, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    offset, flags, iobref, xdata);
-        return 0;
-}
-
-void
-wind_writev (call_frame_t *frame, xlator_t *xl, fd_t *fd, struct iovec *vector,
-             int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
-             dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_writev_cbk, xl, xl->fops->writev, fd, vector,
-                    count, offset, flags, iobref, xdata);
-}
-
-void
-unwind_writev (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-}
-
-void
-set_writev_fop (long py_this, fop_writev_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->fops[GLUPY_WRITEV] = (long)fop;
-}
-
-void
-set_writev_cbk (long py_this, fop_writev_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->cbks[GLUPY_WRITEV] = (long)cbk;
-}
-
-
-/* FOP: OPENDIR */
-
-int32_t
-glupy_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_OPENDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_opendir_cbk_t)(priv->cbks[GLUPY_OPENDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-int32_t
-glupy_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_OPENDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_opendir_t)(priv->fops[GLUPY_OPENDIR]))(
-                frame, this, loc, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-        return 0;
-}
-
-void
-wind_opendir (call_frame_t *frame, xlator_t *xl, loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_opendir_cbk,xl,xl->fops->opendir,loc,fd,xdata);
-}
-
-void
-unwind_opendir (call_frame_t *frame, long cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(opendir,frame,op_ret,op_errno,
-                            fd,xdata);
-}
-
-void
-set_opendir_fop (long py_this, fop_opendir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_OPENDIR] = (long)fop;
-}
-
-void
-set_opendir_cbk (long py_this, fop_opendir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_OPENDIR] = (long)cbk;
-}
-
-/* FOP: READDIR */
-
-int32_t
-glupy_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                   dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readdir_cbk_t)(priv->cbks[GLUPY_READDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                entries,  xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readdir_t)(priv->fops[GLUPY_READDIR]))(
-                frame, this, fd, size, offset, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,fd, size, offset, xdata);
-        return 0;
-}
-
-void
-wind_readdir(call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_readdir_cbk,xl,xl->fops->readdir,fd,size,offset,xdata);
-}
-
-void
-unwind_readdir (call_frame_t *frame, long cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(readdir,frame,op_ret,op_errno,
-                            entries, xdata);
-}
-
-void
-set_readdir_fop (long py_this, fop_readdir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_READDIR] = (long)fop;
-}
-
-void
-set_readdir_cbk (long py_this, fop_readdir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_READDIR] = (long)cbk;
-}
-
-
-/* FOP: READDIRP */
-
-int32_t
-glupy_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                    dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READDIRP]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readdirp_cbk_t)(priv->cbks[GLUPY_READDIRP]))(
-                frame, cookie, this, op_ret, op_errno,
-                entries,  xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                size_t size, off_t offset, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READDIRP]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readdirp_t)(priv->fops[GLUPY_READDIRP]))(
-                frame, this, fd, size, offset, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readdirp_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,fd, size, offset, xdata);
-        return 0;
-}
-
-void
-wind_readdirp (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
-               off_t offset, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_readdirp_cbk,xl,xl->fops->readdirp,fd,size,offset,xdata);
-}
-
-void
-unwind_readdirp (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                 dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(readdirp,frame,op_ret,op_errno,
-                            entries, xdata);
-}
-
-void
-set_readdirp_fop (long py_this, fop_readdirp_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_READDIRP] = (long)fop;
-}
-
-void
-set_readdirp_cbk (long py_this, fop_readdirp_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_READDIRP] = (long)cbk;
-}
-
-
-/* FOP:STAT */
-
-int32_t
-glupy_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_STAT]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_stat_cbk_t)(priv->cbks[GLUPY_STAT]))(
-                frame, cookie, this, op_ret, op_errno,
-                buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_STAT]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_stat_t)(priv->fops[GLUPY_STAT]))(
-                frame, this, loc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, xdata);
-        return 0;
-}
-
-void
-wind_stat (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_stat_cbk,xl,xl->fops->stat,loc,xdata);
-}
-
-void
-unwind_stat (call_frame_t *frame, long cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, struct iatt *buf,
-             dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(stat,frame,op_ret,op_errno,
-                            buf,xdata);
-}
-
-void
-set_stat_fop (long py_this, fop_stat_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_STAT] = (long)fop;
-}
-
-void
-set_stat_cbk (long py_this, fop_stat_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_STAT] = (long)cbk;
-}
-
-
-/* FOP: FSTAT */
-
-int32_t
-glupy_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FSTAT]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fstat_cbk_t)(priv->cbks[GLUPY_FSTAT]))(
-                frame, cookie, this, op_ret, op_errno,
-                buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FSTAT]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fstat_t)(priv->fops[GLUPY_FSTAT]))(
-                frame, this, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
-}
-
-void
-wind_fstat (call_frame_t *frame, xlator_t *xl, fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_fstat_cbk,xl,xl->fops->fstat,fd,xdata);
-}
-
-void
-unwind_fstat (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iatt *buf,
-              dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(fstat,frame,op_ret,op_errno,
-                            buf,xdata);
-}
-
-void
-set_fstat_fop (long py_this, fop_fstat_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FSTAT] = (long)fop;
-}
-
-void
-set_fstat_cbk (long py_this, fop_fstat_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FSTAT] = (long)cbk;
-}
-
-/* FOP:STATFS */
-
-int32_t
-glupy_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_STATFS]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_statfs_cbk_t)(priv->cbks[GLUPY_STATFS]))(
-                frame, cookie, this, op_ret, op_errno,
-                buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_STATFS]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_statfs_t)(priv->fops[GLUPY_STATFS]))(
-                frame, this, loc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_statfs_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
-        return 0;
-}
-
-void
-wind_statfs (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_statfs_cbk,xl,xl->fops->statfs,loc,xdata);
-}
-
-void
-unwind_statfs (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-               dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(statfs,frame,op_ret,op_errno,
-                            buf,xdata);
-}
-
-void
-set_statfs_fop (long py_this, fop_statfs_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_STATFS] = (long)fop;
-}
-
-void
-set_statfs_cbk (long py_this, fop_statfs_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_STATFS] = (long)cbk;
-}
-
-
-/* FOP: SETXATTR */
-
-int32_t
-glupy_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_SETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_setxattr_cbk_t)(priv->cbks[GLUPY_SETXATTR]))(
-                frame, cookie, this, op_ret, op_errno,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_SETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_setxattr_t)(priv->fops[GLUPY_SETXATTR]))(
-                frame, this, loc, dict, flags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict,
-                    flags, xdata);
-        return 0;
-}
-
-void
-wind_setxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-               dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_setxattr_cbk, xl, xl->fops->setxattr,
-                    loc, dict, flags, xdata);
-}
-
-
-void
-unwind_setxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_setxattr_fop (long py_this, fop_setxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_SETXATTR] = (long)fop;
-}
-
-void
-set_setxattr_cbk (long py_this, fop_setxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_SETXATTR] = (long)cbk;
-}
-
-/* FOP: GETXATTR */
-
-int32_t
-glupy_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict,
-                    dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_GETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_getxattr_cbk_t)(priv->cbks[GLUPY_GETXATTR]))(
-                frame, cookie, this, op_ret, op_errno, dict,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_GETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_getxattr_t)(priv->fops[GLUPY_GETXATTR]))(
-                frame, this, loc, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_getxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-               const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_getxattr_cbk, xl, xl->fops->getxattr,
-                    loc, name, xdata);
-}
-
-
-void
-unwind_getxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict,
-                 dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-
-}
-
-
-void
-set_getxattr_fop (long py_this, fop_getxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_GETXATTR] = (long)fop;
-}
-
-
-void
-set_getxattr_cbk (long py_this, fop_getxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_GETXATTR] = (long)cbk;
-}
-
-/* FOP: FSETXATTR */
-
-int32_t
-glupy_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FSETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fsetxattr_cbk_t)(priv->cbks[GLUPY_FSETXATTR]))(
-                frame, cookie, this, op_ret, op_errno,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FSETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fsetxattr_t)(priv->fops[GLUPY_FSETXATTR]))(
-                frame, this, fd, dict, flags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
-                    flags, xdata);
-        return 0;
-}
-
-void
-wind_fsetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
-                dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_fsetxattr_cbk, xl, xl->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-}
-
-
-void
-unwind_fsetxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_fsetxattr_fop (long py_this, fop_fsetxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FSETXATTR] = (long)fop;
-}
-
-void
-set_fsetxattr_cbk (long py_this, fop_fsetxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FSETXATTR] = (long)cbk;
-}
-
-/* FOP: FGETXATTR */
-
-int32_t
-glupy_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FGETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fgetxattr_cbk_t)(priv->cbks[GLUPY_FGETXATTR]))(
-                frame, cookie, this, op_ret, op_errno, dict,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FGETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fgetxattr_t)(priv->fops[GLUPY_FGETXATTR]))(
-                frame, this, fd, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_fgetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
-                const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_fgetxattr_cbk, xl, xl->fops->fgetxattr,
-                    fd, name, xdata);
-}
-
-
-void
-unwind_fgetxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict,
-                  dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-
-}
-
-
-void
-set_fgetxattr_fop (long py_this, fop_fgetxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FGETXATTR] = (long)fop;
-}
-
-
-void
-set_fgetxattr_cbk (long py_this, fop_fgetxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FGETXATTR] = (long)cbk;
-}
-
-/* FOP:REMOVEXATTR */
-
-int32_t
-glupy_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_REMOVEXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_removexattr_cbk_t)(priv->cbks[GLUPY_REMOVEXATTR]))(
-                frame, cookie, this, op_ret, op_errno, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_REMOVEXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_removexattr_t)(priv->fops[GLUPY_REMOVEXATTR]))(
-                frame, this, loc, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_removexattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-                  const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_removexattr_cbk, xl, xl->fops->removexattr,
-                    loc, name, xdata);
-}
-
-
-void
-unwind_removexattr (call_frame_t *frame, long cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_removexattr_fop (long py_this, fop_removexattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_REMOVEXATTR] = (long)fop;
-}
-
-void
-set_removexattr_cbk (long py_this, fop_removexattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_REMOVEXATTR] = (long)cbk;
-}
-
-
-/* FOP:FREMOVEXATTR */
-
-int32_t
-glupy_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FREMOVEXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fremovexattr_cbk_t)(priv->cbks[GLUPY_FREMOVEXATTR]))(
-                frame, cookie, this, op_ret, op_errno, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FREMOVEXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fremovexattr_t)(priv->fops[GLUPY_FREMOVEXATTR]))(
-                frame, this, fd, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fremovexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr, fd, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_fremovexattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_fremovexattr_cbk, xl, xl->fops->fremovexattr,
-                    fd, name, xdata);
-}
-
-
-void
-unwind_fremovexattr (call_frame_t *frame, long cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_fremovexattr_fop (long py_this, fop_fremovexattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FREMOVEXATTR] = (long)fop;
-}
-
-void
-set_fremovexattr_cbk (long py_this, fop_fremovexattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FREMOVEXATTR] = (long)cbk;
-}
-
-
-/* FOP: LINK*/
-int32_t
-glupy_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_LINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_link_cbk_t)(priv->cbks[GLUPY_LINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_LINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_link_t)(priv->fops[GLUPY_LINK]))(
-                frame, this, oldloc, newloc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc,
-                    xdata);
-        return 0;
-}
-
-void
-wind_link (call_frame_t *frame, xlator_t *xl, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_link_cbk, xl, xl->fops->link,
-                    oldloc, newloc, xdata);
-}
-
-void
-unwind_link (call_frame_t *frame, long cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, inode_t *inode,
-             struct iatt *buf, struct iatt *preparent,
-             struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_link_fop (long py_this, fop_link_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_LINK] = (long)fop;
-}
-
-void
-set_link_cbk (long py_this, fop_link_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_LINK] = (long)cbk;
-}
-
-/* FOP: SYMLINK*/
-int32_t
-glupy_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_SYMLINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_symlink_cbk_t)(priv->cbks[GLUPY_SYMLINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
-              loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_SYMLINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_symlink_t)(priv->fops[GLUPY_SYMLINK]))(
-                frame, this, linkname, loc, umask, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkname, loc,
-                    umask, xdata);
-        return 0;
-}
-
-void
-wind_symlink (call_frame_t *frame, xlator_t *xl, const char *linkname,
-              loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_symlink_cbk, xl, xl->fops->symlink,
-                    linkname, loc, umask, xdata);
-}
-
-void
-unwind_symlink (call_frame_t *frame, long cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_symlink_fop (long py_this, fop_symlink_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_SYMLINK] = (long)fop;
-}
-
-void
-set_symlink_cbk (long py_this, fop_symlink_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_SYMLINK] = (long)cbk;
-}
-
-
-/* FOP: READLINK */
-int32_t
-glupy_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, const char *path,
-                    struct iatt *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READLINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readlink_cbk_t)(priv->cbks[GLUPY_READLINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                path, buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path,
-                             buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                size_t size, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READLINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readlink_t)(priv->fops[GLUPY_READLINK]))(
-                frame, this, loc, size, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc,
-                    size, xdata);
-        return 0;
-}
-
-void
-wind_readlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-               size_t size, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_readlink_cbk, xl, xl->fops->readlink,
-                    loc, size, xdata);
-}
-
-void
-unwind_readlink (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, const char *path,
-                 struct iatt *buf, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf,
-                             xdata);
-}
-
-void
-set_readlink_fop (long py_this, fop_readlink_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_READLINK] = (long)fop;
-}
-
-void
-set_readlink_cbk (long py_this, fop_readlink_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_READLINK] = (long)cbk;
-}
-
-
-/* FOP: UNLINK */
-
-int32_t
-glupy_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_UNLINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_unlink_cbk_t)(priv->cbks[GLUPY_UNLINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int xflags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_UNLINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_unlink_t)(priv->fops[GLUPY_UNLINK]))(
-                frame, this, loc, xflags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc,
-                    xflags, xdata);
-        return 0;
-}
-
-void
-wind_unlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-             int xflags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_unlink_cbk, xl, xl->fops->unlink,
-                    loc, xflags, xdata);
-}
-
-void
-unwind_unlink (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iatt *preparent, struct iatt *postparent,
-               dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-}
-
-void
-set_unlink_fop (long py_this, fop_unlink_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_UNLINK] = (long)fop;
-}
-
-void
-set_unlink_cbk (long py_this, fop_unlink_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_UNLINK] = (long)cbk;
-}
-
-
-/* FOP: MKDIR */
-
-int32_t
-glupy_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_MKDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_mkdir_cbk_t)(priv->cbks[GLUPY_MKDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             mode_t umask, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_MKDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_mkdir_t)(priv->fops[GLUPY_MKDIR]))(
-                frame, this, loc, mode, umask, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
-                    xdata);
-        return 0;
-}
-
-void
-wind_mkdir (call_frame_t *frame, xlator_t *xl, loc_t *loc, mode_t mode,
-            mode_t umask, dict_t *xdata)
-{
-
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_mkdir_cbk, xl, xl->fops->mkdir,
-                    loc, mode, umask, xdata);
-}
-
-void
-unwind_mkdir (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, inode_t *inode,
-              struct iatt *buf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_mkdir_fop (long py_this, fop_mkdir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_MKDIR] = (long)fop;
-}
-
-void
-set_mkdir_cbk (long py_this, fop_mkdir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_MKDIR] = (long)cbk;
-}
-
-
-/* FOP: RMDIR */
-
-int32_t
-glupy_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_RMDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_rmdir_cbk_t)(priv->cbks[GLUPY_RMDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int xflags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_RMDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_rmdir_t)(priv->fops[GLUPY_RMDIR]))(
-                frame, this, loc, xflags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc,
-                    xflags, xdata);
-        return 0;
-}
-
-void
-wind_rmdir (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-            int xflags, dict_t *xdata)
-{
-
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_rmdir_cbk, xl, xl->fops->rmdir,
-                    loc, xflags, xdata);
-}
-
-void
-unwind_rmdir (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno,
-              struct iatt *preparent, struct iatt *postparent,
-              dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-}
-
-void
-set_rmdir_fop (long py_this, fop_rmdir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_RMDIR] = (long)fop;
-}
-
-void
-set_rmdir_cbk (long py_this, fop_rmdir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_RMDIR] = (long)cbk;
-}
-
-
-/* NON-FOP-SPECIFIC CODE */
-
-
-long
-get_id (call_frame_t *frame)
-{
-        return (long)(frame->local);
-}
-
-uint64_t
-get_rootunique (call_frame_t *frame)
-{
-        return frame->root->unique;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_glupy_mt_end);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       " failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t
-init (xlator_t *this)
-{
-        glupy_private_t         *priv           = NULL;
-        char                    *module_name    = NULL;
-        PyObject                *py_mod_name    = NULL;
-        PyObject                *py_init_func   = NULL;
-        PyObject                *py_args        = NULL;
-        PyObject                *syspath        = NULL;
-	PyObject                *path           = NULL;
-	PyObject                *error_type     = NULL;
-	PyObject                *error_msg      = NULL;
-	PyObject                *error_bt       = NULL;
-	static gf_boolean_t      py_inited      = _gf_false;
-        void *                   err_cleanup    = &&err_return;
-
-        if (dict_get_str(this->options,"module-name",&module_name) != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "missing module-name");
-                return -1;
-        }
-
-	priv = GF_CALLOC (1, sizeof (glupy_private_t), gf_glupy_mt_priv);
-        if (!priv) {
-                goto *err_cleanup;
-        }
-        this->private = priv;
-        err_cleanup = &&err_free_priv;
-
-	if (!py_inited) {
-	        /* 
-                 * This must be done before Py_Initialize(),
-                 * because it will duplicate the environment,
-                 * and fail to see later environment updates.
-                 */
-	        setenv("PATH_GLUSTERFS_GLUPY_MODULE",
-                       PATH_GLUSTERFS_GLUPY_MODULE, 1);
-
-                Py_Initialize();
-                PyEval_InitThreads();
-
-                (void)pthread_key_create(&gil_init_key,NULL);
-                (void)pthread_setspecific(gil_init_key,(void *)1);
-
-                /* PyEval_InitThreads takes this "for" us.  No thanks. */
-                PyEval_ReleaseLock();
-                py_inited = _gf_true;
-        }
-
-	/* Adjust python's path */
-	syspath = PySys_GetObject("path");
-	path = PyString_FromString(GLUSTER_PYTHON_PATH);
-	PyList_Append(syspath, path);
-	Py_DECREF(path);
-
-	py_mod_name = PyString_FromString(module_name);
-        if (!py_mod_name) {
-                gf_log (this->name, GF_LOG_ERROR, "could not create name");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "py_mod_name = %s", module_name);
-        priv->py_module = PyImport_Import(py_mod_name);
-        Py_DECREF(py_mod_name);
-        if (!priv->py_module) {
-                gf_log (this->name, GF_LOG_ERROR, "Python import of %s failed",
-                        module_name);
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        gf_log (this->name, GF_LOG_INFO, "Import of %s succeeded", module_name);
-        err_cleanup = &&err_deref_module;
-
-        py_init_func = PyObject_GetAttrString(priv->py_module, "xlator");
-        if (!py_init_func || !PyCallable_Check(py_init_func)) {
-                gf_log (this->name, GF_LOG_ERROR, "missing init func");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        err_cleanup = &&err_deref_init;
-
-        py_args = PyTuple_New(1);
-        if (!py_args) {
-                gf_log (this->name, GF_LOG_ERROR, "could not create args");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        PyTuple_SetItem(py_args,0,PyLong_FromLong((long)this));
-
-        /* TBD: pass in list of children */
-        priv->py_xlator = PyObject_CallObject(py_init_func, py_args);
-        Py_DECREF(py_args);
-        if (!priv->py_xlator) {
-                gf_log (this->name, GF_LOG_ERROR, "Python init failed");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "init returned %p", priv->py_xlator);
-
-        return 0;
-
-err_deref_init:
-        Py_DECREF(py_init_func);
-err_deref_module:
-        Py_DECREF(priv->py_module);
-err_free_priv:
-        GF_FREE(priv);
-err_return:
-        return -1;
-}
-
-void
-fini (xlator_t *this)
-{
-        glupy_private_t *priv = this->private;
-
-        if (!priv)
-                return;
-        Py_DECREF(priv->py_xlator);
-        Py_DECREF(priv->py_module);
-        this->private = NULL;
-        GF_FREE (priv);
-
-        return;
-}
-
-struct xlator_fops fops = {
-        .lookup       = glupy_lookup,
-        .create       = glupy_create,
-        .open         = glupy_open,
-        .readv        = glupy_readv,
-        .writev       = glupy_writev,
-        .opendir      = glupy_opendir,
-        .readdir      = glupy_readdir,
-        .stat         = glupy_stat,
-        .fstat        = glupy_fstat,
-        .setxattr     = glupy_setxattr,
-        .getxattr     = glupy_getxattr,
-        .fsetxattr    = glupy_fsetxattr,
-        .fgetxattr    = glupy_fgetxattr,
-        .removexattr  = glupy_removexattr,
-        .fremovexattr = glupy_fremovexattr,
-        .link         = glupy_link,
-        .unlink       = glupy_unlink,
-        .readlink     = glupy_readlink,
-        .symlink      = glupy_symlink,
-        .mkdir        = glupy_mkdir,
-        .rmdir        = glupy_rmdir,
-        .statfs       = glupy_statfs,
-        .readdirp     = glupy_readdirp
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/features/glupy/src/glupy.h b/xlators/features/glupy/src/glupy.h
deleted file mode 100644
index 1488c55c331..00000000000
--- a/xlators/features/glupy/src/glupy.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
-   Copyright (c) 2006-2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __GLUPY_H__
-#define __GLUPY_H__
-
-#include "mem-types.h"
-
-enum {
-        GLUPY_LOOKUP = 0,
-        GLUPY_CREATE,
-        GLUPY_OPEN,
-        GLUPY_READV,
-        GLUPY_WRITEV,
-        GLUPY_OPENDIR,
-        GLUPY_READDIR,
-        GLUPY_READDIRP,
-        GLUPY_STAT,
-        GLUPY_FSTAT,
-        GLUPY_STATFS,
-        GLUPY_SETXATTR,
-        GLUPY_GETXATTR,
-        GLUPY_FSETXATTR,
-        GLUPY_FGETXATTR,
-        GLUPY_REMOVEXATTR,
-        GLUPY_FREMOVEXATTR,
-        GLUPY_LINK,
-        GLUPY_UNLINK,
-        GLUPY_READLINK,
-        GLUPY_SYMLINK,
-        GLUPY_MKNOD,
-        GLUPY_MKDIR,
-        GLUPY_RMDIR,
-        GLUPY_N_FUNCS
-};
-
-typedef struct {
-        PyObject        *py_module;
-        PyObject        *py_xlator;
-        long            fops[GLUPY_N_FUNCS];
-        long            cbks[GLUPY_N_FUNCS];
-} glupy_private_t;
-
-enum gf_glupy_mem_types_ {
-        gf_glupy_mt_priv = gf_common_mt_end + 1,
-        gf_glupy_mt_end
-};
-
-#endif /* __GLUPY_H__ */
diff --git a/xlators/features/glupy/src/glupy.sym b/xlators/features/glupy/src/glupy.sym
deleted file mode 100644
index 55d9a300108..00000000000
--- a/xlators/features/glupy/src/glupy.sym
+++ /dev/null
@@ -1,101 +0,0 @@
-init
-fini
-fops
-cbks
-options
-notify
-mem_acct_init
-reconfigure
-dumpops
-set_lookup_fop
-set_lookup_cbk
-set_create_fop
-set_create_cbk
-set_open_fop
-set_open_cbk
-set_readv_fop
-set_readv_cbk
-set_writev_fop
-set_writev_cbk
-set_opendir_fop
-set_opendir_cbk
-set_readdir_fop
-set_readdir_cbk
-set_readdirp_fop
-set_readdirp_cbk
-set_stat_fop
-set_stat_cbk
-set_fstat_fop
-set_fstat_cbk
-set_statfs_fop
-set_statfs_cbk
-set_setxattr_fop
-set_setxattr_cbk
-set_getxattr_fop
-set_getxattr_cbk
-set_fsetxattr_fop
-set_fsetxattr_cbk
-set_fgetxattr_fop
-set_fgetxattr_cbk
-set_removexattr_fop
-set_removexattr_cbk
-set_fremovexattr_fop
-set_fremovexattr_cbk
-set_link_fop
-set_link_cbk
-set_symlink_fop
-set_symlink_cbk
-set_readlink_fop
-set_readlink_cbk
-set_unlink_fop
-set_unlink_cbk
-set_mkdir_fop
-set_mkdir_cbk
-set_rmdir_fop
-set_rmdir_cbk
-wind_lookup
-wind_create
-wind_open
-wind_readv
-wind_writev
-wind_opendir
-wind_readdir
-wind_readdirp
-wind_stat
-wind_fstat
-wind_statfs
-wind_setxattr
-wind_getxattr
-wind_fsetxattr
-wind_fgetxattr
-wind_removexattr
-wind_fremovexattr
-wind_link
-wind_symlink
-wind_readlink
-wind_unlink
-wind_mkdir
-wind_rmdir
-unwind_lookup
-unwind_create
-unwind_open
-unwind_readv
-unwind_writev
-unwind_opendir
-unwind_readdir
-unwind_readdirp
-unwind_stat
-unwind_fstat
-unwind_statfs
-unwind_setxattr
-unwind_getxattr
-unwind_fsetxattr
-unwind_fgetxattr
-unwind_removexattr
-unwind_fremovexattr
-unwind_link
-unwind_symlink
-unwind_readlink
-unwind_unlink
-unwind_mkdir
-unwind_rmdir
diff --git a/xlators/features/glupy/src/glupy/Makefile.am b/xlators/features/glupy/src/glupy/Makefile.am
deleted file mode 100644
index 573d2da12e1..00000000000
--- a/xlators/features/glupy/src/glupy/Makefile.am
+++ /dev/null
@@ -1,5 +0,0 @@
-# Install __init__.py into the Python site-packages area
-pyglupydir = @BUILD_PYTHON_SITE_PACKAGES@/gluster/glupy
-pyglupy_PYTHON = __init__.py
-
-CLEANFILES =
diff --git a/xlators/features/glupy/src/glupy/__init__.py b/xlators/features/glupy/src/glupy/__init__.py
deleted file mode 100644
index b9fc3700fa6..00000000000
--- a/xlators/features/glupy/src/glupy/__init__.py
+++ /dev/null
@@ -1,852 +0,0 @@
-##
-## Copyright (c) 2006-2014 Red Hat, Inc. <http://www.redhat.com>
-## This file is part of GlusterFS.
-##
-## This file is licensed to you under your choice of the GNU Lesser
-## General Public License, version 3 or any later version (LGPLv3 or
-## later), or the GNU General Public License, version 2 (GPLv2), in all
-## cases as published by the Free Software Foundation.
-##
-
-import sys
-import os
-from ctypes import *
-
-dl = CDLL(os.getenv("PATH_GLUSTERFS_GLUPY_MODULE", ""),RTLD_GLOBAL)
-
-
-class call_frame_t (Structure):
-        pass
-
-class dev_t (Structure):
-        pass
-
-
-class dict_t (Structure):
-        pass
-
-
-class gf_dirent_t (Structure):
-        pass
-
-
-class iobref_t (Structure):
-	pass
-
-
-class iovec_t (Structure):
-        pass
-
-
-class list_head (Structure):
-        pass
-
-list_head._fields_ = [
-                ("next", POINTER(list_head)),
-                ("prev", POINTER(list_head))
-        ]
-
-
-class rwxperm_t (Structure):
-        _fields_ = [
-                ("read", c_uint8, 1),
-                ("write", c_uint8, 1),
-                ("execn", c_uint8, 1)
-        ]
-
-
-class statvfs_t (Structure):
-        pass
-
-
-class xlator_t (Structure):
-        pass
-
-
-class ia_prot_t (Structure):
-        _fields_ = [
-                ("suid", c_uint8, 1),
-                ("sgid", c_uint8, 1),
-                ("sticky", c_uint8, 1),
-                ("owner", rwxperm_t),
-                ("group", rwxperm_t),
-                ("other", rwxperm_t)
-        ]
-
-# For checking file type.
-(IA_INVAL, IA_IFREG, IA_IFDIR, IA_IFLNK, IA_IFBLK, IA_IFCHR, IA_IFIFO,
- IA_IFSOCK) = xrange(8)
-
-
-class iatt_t (Structure):
-        _fields_ = [
-                ("ia_no", c_uint64),
-                ("ia_gfid", c_ubyte * 16),
-                ("ia_dev", c_uint64),
-                ("ia_type", c_uint),
-                ("ia_prot", ia_prot_t),
-                ("ia_nlink", c_uint32),
-                ("ia_uid", c_uint32),
-                ("ia_gid", c_uint32),
-                ("ia_rdev", c_uint64),
-                ("ia_size", c_uint64),
-                ("ia_blksize", c_uint32),
-                ("ia_blocks", c_uint64),
-                ("ia_atime", c_uint32 ),
-                ("ia_atime_nsec", c_uint32),
-                ("ia_mtime", c_uint32),
-                ("ia_mtime_nsec", c_uint32),
-                ("ia_ctime", c_uint32),
-                ("ia_ctime_nsec", c_uint32)
-        ]
-
-
-class mem_pool (Structure):
-        _fields_ = [
-                ("list", list_head),
-                ("hot_count", c_int),
-                ("cold_count", c_int),
-                ("lock", c_void_p),
-                ("padded_sizeof_type", c_ulong),
-                ("pool", c_void_p),
-                ("pool_end", c_void_p),
-                ("real_sizeof_type", c_int),
-                ("alloc_count", c_uint64),
-                ("pool_misses", c_uint64),
-                ("max_alloc", c_int),
-                ("curr_stdalloc", c_int),
-                ("max_stdalloc", c_int),
-                ("name", c_char_p),
-                ("global_list", list_head)
-        ]
-
-
-class U_ctx_key_inode (Union):
-        _fields_ = [
-                ("key", c_uint64),
-                ("xl_key", POINTER(xlator_t))
-        ]
-
-
-class U_ctx_value1 (Union):
-        _fields_ = [
-                ("value1", c_uint64),
-                ("ptr1", c_void_p)
-        ]
-
-
-class U_ctx_value2 (Union):
-        _fields_ = [
-                ("value2", c_uint64),
-                ("ptr2", c_void_p)
-        ]
-
-class inode_ctx (Structure):
-        _anonymous_ = ("u_key","u_value1","u_value2",)
-        _fields_ = [
-                ("u_key", U_ctx_key_inode),
-                ("u_value1", U_ctx_value1),
-                ("u_value2", U_ctx_value2)
-        ]
-
-class inode_t (Structure):
-        pass
-
-class inode_table_t (Structure):
-        _fields_ = [
-                ("lock", c_void_p),
-                ("hashsize", c_size_t),
-                ("name", c_char_p),
-                ("root", POINTER(inode_t)),
-                ("xl", POINTER(xlator_t)),
-                ("lru_limit", c_uint32),
-                ("inode_hash", POINTER(list_head)),
-                ("name_hash", POINTER(list_head)),
-                ("active", list_head),
-                ("active_size", c_uint32),
-                ("lru", list_head),
-                ("lru_size", c_uint32),
-                ("purge", list_head),
-                ("purge_size", c_uint32),
-                ("inode_pool", POINTER(mem_pool)),
-                ("dentry_pool", POINTER(mem_pool)),
-                ("fd_mem_pool", POINTER(mem_pool))
-        ]
-
-inode_t._fields_ = [
-                ("table", POINTER(inode_table_t)),
-                ("gfid", c_ubyte * 16),
-                ("lock", c_void_p),
-                ("nlookup", c_uint64),
-                ("fd_count", c_uint32),
-                ("ref", c_uint32),
-                ("ia_type", c_uint),
-                ("fd_list", list_head),
-                ("dentry_list", list_head),
-                ("hashv", list_head),
-                ("listv", list_head),
-                ("ctx", POINTER(inode_ctx))
-        ]
-
-
-
-class U_ctx_key_fd (Union):
-        _fields_ = [
-                ("key", c_uint64),
-                ("xl_key", c_void_p)
-        ]
-
-class fd_lk_ctx (Structure):
-        _fields_ = [
-                ("lk_list", list_head),
-                ("ref", c_int),
-                ("lock", c_void_p)
-        ]
-
-class fd_ctx (Structure):
-        _anonymous_ = ("u_key","u_value1")
-        _fields_ = [
-                ("u_key", U_ctx_key_fd),
-                ("u_value1", U_ctx_value1)
-        ]
-
-class fd_t (Structure):
-        _fields_ = [
-                ("pid", c_uint64),
-                ("flags", c_int32),
-                ("refcount", c_int32),
-                ("inode_list", list_head),
-                ("inode", POINTER(inode_t)),
-                ("lock", c_void_p),
-                ("ctx", POINTER(fd_ctx)),
-                ("xl_count", c_int),
-                ("lk_ctx", POINTER(fd_lk_ctx)),
-                ("anonymous", c_uint)
-        ]
-
-class loc_t (Structure):
-        _fields_ = [
-                ("path", c_char_p),
-                ("name", c_char_p),
-                ("inode", POINTER(inode_t)),
-                ("parent", POINTER(inode_t)),
-                ("gfid", c_ubyte * 16),
-                ("pargfid", c_ubyte * 16),
-        ]
-
-
-
-def _init_op (a_class, fop, cbk, wind, unwind):
-        # Decorators, used by translators. We could pass the signatures as
-        # parameters, but it's actually kind of nice to keep them around for
-        # inspection.
-        a_class.fop_type = apply(CFUNCTYPE,a_class.fop_sig)
-        a_class.cbk_type = apply(CFUNCTYPE,a_class.cbk_sig)
-        # Dispatch-function registration.
-        fop.restype = None
-        fop.argtypes = [ c_long, a_class.fop_type ]
-        # Callback-function registration.
-        cbk.restype = None
-        cbk.argtypes = [ c_long, a_class.cbk_type ]
-        # STACK_WIND function.
-        wind.restype = None
-        wind.argtypes = list(a_class.fop_sig[1:])
-        # STACK_UNWIND function.
-        unwind.restype = None
-        unwind.argtypes = list(a_class.cbk_sig[1:])
-
-class OpLookup:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(dict_t), POINTER(iatt_t))
-_init_op (OpLookup, dl.set_lookup_fop, dl.set_lookup_cbk,
-                    dl.wind_lookup,    dl.unwind_lookup)
-
-class OpCreate:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, c_uint, c_uint, POINTER(fd_t),
-                   POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(fd_t), POINTER(inode_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpCreate, dl.set_create_fop, dl.set_create_cbk,
-                    dl.wind_create,    dl.unwind_create)
-
-class OpOpen:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, POINTER(fd_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
-_init_op (OpOpen, dl.set_open_fop, dl.set_open_cbk,
-                  dl.wind_open,    dl.unwind_open)
-
-class OpReadv:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_size_t, c_long, c_uint32, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iovec_t), c_int, POINTER(iatt_t),
-                   POINTER(iobref_t), POINTER(dict_t))
-_init_op (OpReadv, dl.set_readv_fop, dl.set_readv_cbk,
-                   dl.wind_readv,    dl.unwind_readv)
-class OpWritev:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), POINTER(iovec_t), c_int, c_long, c_uint32,
-                   POINTER(iobref_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpWritev, dl.set_writev_fop, dl.set_writev_cbk,
-                    dl.wind_writev,    dl.unwind_writev)
-
-class OpOpendir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(fd_t) ,POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
-_init_op (OpOpendir, dl.set_opendir_fop, dl.set_opendir_cbk,
-                     dl.wind_opendir,    dl.unwind_opendir)
-
-class OpReaddir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
-_init_op (OpReaddir, dl.set_readdir_fop, dl.set_readdir_cbk,
-                     dl.wind_readdir,    dl.unwind_readdir)
-
-class OpReaddirp:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
-_init_op (OpReaddirp, dl.set_readdirp_fop, dl.set_readdirp_cbk,
-                      dl.wind_readdirp,    dl.unwind_readdirp)
-
-class OpStat:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpStat, dl.set_stat_fop, dl.set_stat_cbk,
-                  dl.wind_stat,    dl.unwind_stat)
-
-class OpFstat:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpFstat, dl.set_fstat_fop, dl.set_fstat_cbk,
-                   dl.wind_fstat,    dl.unwind_fstat)
-
-class OpStatfs:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(statvfs_t), POINTER(dict_t))
-_init_op (OpStatfs, dl.set_statfs_fop, dl.set_statfs_cbk,
-                    dl.wind_statfs,    dl.unwind_statfs)
-
-
-class OpSetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t), c_int32,
-                   POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpSetxattr, dl.set_setxattr_fop, dl.set_setxattr_cbk,
-                      dl.wind_setxattr,    dl.unwind_setxattr)
-
-class OpGetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_char_p, POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
-_init_op (OpGetxattr, dl.set_getxattr_fop, dl.set_getxattr_cbk,
-                      dl.wind_getxattr,    dl.unwind_getxattr)
-
-class OpFsetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), POINTER(dict_t), c_int32,
-                   POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpFsetxattr, dl.set_fsetxattr_fop, dl.set_fsetxattr_cbk,
-                       dl.wind_fsetxattr,    dl.unwind_fsetxattr)
-
-class OpFgetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_char_p, POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
-_init_op (OpFgetxattr, dl.set_fgetxattr_fop, dl.set_fgetxattr_cbk,
-                       dl.wind_fgetxattr,    dl.unwind_fgetxattr)
-
-class OpRemovexattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_char_p, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpRemovexattr, dl.set_removexattr_fop, dl.set_removexattr_cbk,
-                         dl.wind_removexattr,    dl.unwind_removexattr)
-
-
-class OpFremovexattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_char_p, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpFremovexattr, dl.set_fremovexattr_fop, dl.set_fremovexattr_cbk,
-                          dl.wind_fremovexattr,    dl.unwind_fremovexattr)
-
-class OpLink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpLink, dl.set_link_fop, dl.set_link_cbk,
-                  dl.wind_link,    dl.unwind_link)
-
-class OpSymlink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   c_char_p, POINTER(loc_t), c_uint, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpSymlink, dl.set_symlink_fop, dl.set_symlink_cbk,
-                     dl.wind_symlink,    dl.unwind_symlink)
-
-class OpUnlink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpUnlink, dl.set_unlink_fop, dl.set_unlink_cbk,
-                    dl.wind_unlink,    dl.unwind_unlink)
-
-class OpReadlink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_size_t, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, c_char_p, POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpReadlink, dl.set_readlink_fop, dl.set_readlink_cbk,
-                      dl.wind_readlink,    dl.unwind_readlink)
-
-class OpMkdir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_uint, c_uint, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpMkdir, dl.set_mkdir_fop, dl.set_mkdir_cbk,
-                   dl.wind_mkdir,    dl.unwind_mkdir)
-
-class OpRmdir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpRmdir, dl.set_rmdir_fop, dl.set_rmdir_cbk,
-                   dl.wind_rmdir,    dl.unwind_rmdir)
-
-
-class Translator:
-        def __init__ (self, c_this):
-                # This is only here to keep references to the stubs we create,
-                # because ctypes doesn't and glupy.so can't because it doesn't
-                # get a pointer to the actual Python object. It's a dictionary
-                # instead of a list in case we ever allow changing fops/cbks
-                # after initialization and need to look them up.
-                self.stub_refs = {}
-                funcs = dir(self.__class__)
-                if "lookup_fop" in funcs:
-                        @OpLookup.fop_type
-                        def stub (frame, this, loc, xdata, s=self):
-                                return s.lookup_fop (frame, this, loc, xdata)
-                        self.stub_refs["lookup_fop"] = stub
-                        dl.set_lookup_fop(c_this,stub)
-                if "lookup_cbk" in funcs:
-                        @OpLookup.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, inode,
-                                  buf, xdata, postparent, s=self):
-                                return s.lookup_cbk(frame, cookie, this, op_ret,
-                                                    op_errno, inode, buf, xdata,
-                                                    postparent)
-                        self.stub_refs["lookup_cbk"] = stub
-                        dl.set_lookup_cbk(c_this,stub)
-                if "create_fop" in funcs:
-                        @OpCreate.fop_type
-                        def stub (frame, this, loc, flags, mode, umask, fd,
-                                  xdata, s=self):
-                                return s.create_fop (frame, this, loc, flags,
-                                                     mode, umask, fd, xdata)
-                        self.stub_refs["create_fop"] = stub
-                        dl.set_create_fop(c_this,stub)
-                if "create_cbk" in funcs:
-                        @OpCreate.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, fd,
-                                  inode, buf, preparent, postparent, xdata,
-                                  s=self):
-                                return s.create_cbk (frame, cookie, this,
-                                                     op_ret, op_errno, fd,
-                                                     inode, buf, preparent,
-                                                     postparent, xdata)
-                        self.stub_refs["create_cbk"] = stub
-                        dl.set_create_cbk(c_this,stub)
-                if "open_fop" in funcs:
-                        @OpOpen.fop_type
-                        def stub (frame, this, loc, flags, fd,
-                                  xdata, s=self):
-                                return s.open_fop (frame, this, loc, flags,
-                                                   fd, xdata)
-                        self.stub_refs["open_fop"] = stub
-                        dl.set_open_fop(c_this,stub)
-                if "open_cbk" in funcs:
-                        @OpOpen.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, fd,
-                                  xdata, s=self):
-                                return s.open_cbk (frame, cookie, this,
-                                                   op_ret, op_errno, fd,
-                                                   xdata)
-                        self.stub_refs["open_cbk"] = stub
-                        dl.set_open_cbk(c_this,stub)
-                if "readv_fop" in funcs:
-                        @OpReadv.fop_type
-                        def stub (frame, this, fd, size, offset, flags,
-                                  xdata, s=self):
-                                return s.readv_fop (frame, this, fd, size,
-                                                    offset, flags, xdata)
-                        self.stub_refs["readv_fop"] = stub
-                        dl.set_readv_fop(c_this,stub)
-                if "readv_cbk" in funcs:
-                        @OpReadv.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  vector, count, stbuf, iobref, xdata,
-                                  s=self):
-                                return s.readv_cbk (frame, cookie, this,
-                                                    op_ret, op_errno, vector,
-                                                    count, stbuf, iobref,
-                                                    xdata)
-                        self.stub_refs["readv_cbk"] = stub
-                        dl.set_readv_cbk(c_this,stub)
-                if "writev_fop" in funcs:
-                        @OpWritev.fop_type
-                        def stub (frame, this, fd, vector, count,
-                                  offset, flags, iobref, xdata, s=self):
-                                return s.writev_fop (frame, this, fd, vector,
-                                                     count, offset, flags,
-                                                     iobref, xdata)
-                        self.stub_refs["writev_fop"] = stub
-                        dl.set_writev_fop(c_this,stub)
-                if "writev_cbk" in funcs:
-                        @OpWritev.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  prebuf, postbuf, xdata, s=self):
-                                return s.writev_cbk (frame, cookie, this,
-                                                     op_ret, op_errno, prebuf,
-                                                     postbuf, xdata)
-                        self.stub_refs["writev_cbk"] = stub
-                        dl.set_writev_cbk(c_this,stub)
-                if "opendir_fop" in funcs:
-                        @OpOpendir.fop_type
-                        def stub (frame, this, loc, fd, xdata, s=self):
-                                return s.opendir_fop (frame, this, loc, fd,
-                                                      xdata)
-                        self.stub_refs["opendir_fop"] = stub
-                        dl.set_opendir_fop(c_this,stub)
-                if "opendir_cbk" in funcs:
-                        @OpOpendir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, fd,
-                                  xdata, s=self):
-                                return s.opendir_cbk(frame, cookie, this,
-                                                     op_ret, op_errno, fd,
-                                                     xdata)
-                        self.stub_refs["opendir_cbk"] = stub
-                        dl.set_opendir_cbk(c_this,stub)
-                if "readdir_fop" in funcs:
-                        @OpReaddir.fop_type
-                        def stub (frame, this, fd, size, offset, xdata, s=self):
-                                return s.readdir_fop (frame, this, fd, size,
-                                                      offset, xdata)
-                        self.stub_refs["readdir_fop"] = stub
-                        dl.set_readdir_fop(c_this,stub)
-                if "readdir_cbk" in funcs:
-                        @OpReaddir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  entries, xdata, s=self):
-                                return s.readdir_cbk(frame, cookie, this,
-                                                     op_ret, op_errno, entries,
-                                                     xdata)
-                        self.stub_refs["readdir_cbk"] = stub
-                        dl.set_readdir_cbk(c_this,stub)
-                if "readdirp_fop" in funcs:
-                        @OpReaddirp.fop_type
-                        def stub (frame, this, fd, size, offset, xdata, s=self):
-                                return s.readdirp_fop (frame, this, fd, size,
-                                                       offset, xdata)
-                        self.stub_refs["readdirp_fop"] = stub
-                        dl.set_readdirp_fop(c_this,stub)
-                if "readdirp_cbk" in funcs:
-                        @OpReaddirp.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  entries, xdata, s=self):
-                                return s.readdirp_cbk (frame, cookie, this,
-                                                       op_ret, op_errno,
-                                                       entries, xdata)
-                        self.stub_refs["readdirp_cbk"] = stub
-                        dl.set_readdirp_cbk(c_this,stub)
-                if "stat_fop" in funcs:
-                        @OpStat.fop_type
-                        def stub (frame, this, loc, xdata, s=self):
-                                return s.stat_fop (frame, this, loc, xdata)
-                        self.stub_refs["stat_fop"] = stub
-                        dl.set_stat_fop(c_this,stub)
-                if "stat_cbk" in funcs:
-                        @OpStat.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata, s=self):
-                                return s.stat_cbk(frame, cookie, this, op_ret,
-                                                  op_errno, buf, xdata)
-                        self.stub_refs["stat_cbk"] = stub
-                        dl.set_stat_cbk(c_this,stub)
-                if "fstat_fop" in funcs:
-                        @OpFstat.fop_type
-                        def stub (frame, this, fd, xdata, s=self):
-                                return s.fstat_fop (frame, this, fd, xdata)
-                        self.stub_refs["fstat_fop"] = stub
-                        dl.set_fstat_fop(c_this,stub)
-                if "fstat_cbk" in funcs:
-                        @OpFstat.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata, s=self):
-                                return s.fstat_cbk(frame, cookie, this, op_ret,
-                                                   op_errno, buf, xdata)
-                        self.stub_refs["fstat_cbk"] = stub
-                        dl.set_fstat_cbk(c_this,stub)
-                if "statfs_fop" in funcs:
-                        @OpStatfs.fop_type
-                        def stub (frame, this, loc, xdata, s=self):
-                                return s.statfs_fop (frame, this, loc, xdata)
-                        self.stub_refs["statfs_fop"] = stub
-                        dl.set_statfs_fop(c_this,stub)
-                if "statfs_cbk" in funcs:
-                        @OpStatfs.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata, s=self):
-                                return s.statfs_cbk (frame, cookie, this,
-                                                     op_ret, op_errno, buf,
-                                                     xdata)
-                        self.stub_refs["statfs_cbk"] = stub
-                        dl.set_statfs_cbk(c_this,stub)
-                if "setxattr_fop" in funcs:
-                        @OpSetxattr.fop_type
-                        def stub (frame, this, loc, dictionary, flags, xdata,
-                                  s=self):
-                                return s.setxattr_fop (frame, this, loc,
-                                                       dictionary, flags,
-                                                       xdata)
-                        self.stub_refs["setxattr_fop"] = stub
-                        dl.set_setxattr_fop(c_this,stub)
-                if "setxattr_cbk" in funcs:
-                        @OpSetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
-                                  s=self):
-                                return s.setxattr_cbk(frame, cookie, this,
-                                                      op_ret, op_errno, xdata)
-                        self.stub_refs["setxattr_cbk"] = stub
-                        dl.set_setxattr_cbk(c_this,stub)
-                if "getxattr_fop" in funcs:
-                        @OpGetxattr.fop_type
-                        def stub (frame, this, loc, name, xdata, s=self):
-                                return s.getxattr_fop (frame, this, loc, name,
-                                                       xdata)
-                        self.stub_refs["getxattr_fop"] = stub
-                        dl.set_getxattr_fop(c_this,stub)
-                if "getxattr_cbk" in funcs:
-                        @OpGetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  dictionary, xdata, s=self):
-                                return s.getxattr_cbk(frame, cookie, this,
-                                                      op_ret, op_errno,
-                                                      dictionary, xdata)
-                        self.stub_refs["getxattr_cbk"] = stub
-                        dl.set_getxattr_cbk(c_this,stub)
-                if "fsetxattr_fop" in funcs:
-                        @OpFsetxattr.fop_type
-                        def stub (frame, this, fd, dictionary, flags, xdata,
-                                  s=self):
-                                return s.fsetxattr_fop (frame, this, fd,
-                                                        dictionary, flags,
-                                                        xdata)
-                        self.stub_refs["fsetxattr_fop"] = stub
-                        dl.set_fsetxattr_fop(c_this,stub)
-                if "fsetxattr_cbk" in funcs:
-                        @OpFsetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
-                                  s=self):
-                                return s.fsetxattr_cbk(frame, cookie, this,
-                                                       op_ret, op_errno, xdata)
-                        self.stub_refs["fsetxattr_cbk"] = stub
-                        dl.set_fsetxattr_cbk(c_this,stub)
-                if "fgetxattr_fop" in funcs:
-                        @OpFgetxattr.fop_type
-                        def stub (frame, this, fd, name, xdata, s=self):
-                                return s.fgetxattr_fop (frame, this, fd, name,
-                                                        xdata)
-                        self.stub_refs["fgetxattr_fop"] = stub
-                        dl.set_fgetxattr_fop(c_this,stub)
-                if "fgetxattr_cbk" in funcs:
-                        @OpFgetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  dictionary, xdata, s=self):
-                                return s.fgetxattr_cbk(frame, cookie, this,
-                                                       op_ret, op_errno,
-                                                       dictionary, xdata)
-                        self.stub_refs["fgetxattr_cbk"] = stub
-                        dl.set_fgetxattr_cbk(c_this,stub)
-                if "removexattr_fop" in funcs:
-                        @OpRemovexattr.fop_type
-                        def stub (frame, this, loc, name, xdata, s=self):
-                                return s.removexattr_fop (frame, this, loc,
-                                                          name, xdata)
-                        self.stub_refs["removexattr_fop"] = stub
-                        dl.set_removexattr_fop(c_this,stub)
-                if "removexattr_cbk" in funcs:
-                        @OpRemovexattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  xdata, s=self):
-                                return s.removexattr_cbk(frame, cookie, this,
-                                                         op_ret, op_errno,
-                                                         xdata)
-                        self.stub_refs["removexattr_cbk"] = stub
-                        dl.set_removexattr_cbk(c_this,stub)
-                if "fremovexattr_fop" in funcs:
-                        @OpFremovexattr.fop_type
-                        def stub (frame, this, fd, name, xdata, s=self):
-                                return s.fremovexattr_fop (frame, this, fd,
-                                                           name, xdata)
-                        self.stub_refs["fremovexattr_fop"] = stub
-                        dl.set_fremovexattr_fop(c_this,stub)
-                if "fremovexattr_cbk" in funcs:
-                        @OpFremovexattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  xdata, s=self):
-                                return s.fremovexattr_cbk(frame, cookie, this,
-                                                          op_ret, op_errno,
-                                                          xdata)
-                        self.stub_refs["fremovexattr_cbk"] = stub
-                        dl.set_fremovexattr_cbk(c_this,stub)
-                if "link_fop" in funcs:
-                        @OpLink.fop_type
-                        def stub (frame, this, oldloc, newloc,
-                                  xdata, s=self):
-                                return s.link_fop (frame, this, oldloc,
-                                                   newloc, xdata)
-                        self.stub_refs["link_fop"] = stub
-                        dl.set_link_fop(c_this,stub)
-                if "link_cbk" in funcs:
-                        @OpLink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  inode, buf, preparent, postparent, xdata,
-                                  s=self):
-                                return s.link_cbk (frame, cookie, this,
-                                                   op_ret, op_errno, inode,
-                                                   buf, preparent,
-                                                   postparent, xdata)
-                        self.stub_refs["link_cbk"] = stub
-                        dl.set_link_cbk(c_this,stub)
-                if "symlink_fop" in funcs:
-                        @OpSymlink.fop_type
-                        def stub (frame, this, linkname, loc,
-                                  umask, xdata, s=self):
-                                return s.symlink_fop (frame, this, linkname,
-                                                      loc, umask, xdata)
-                        self.stub_refs["symlink_fop"] = stub
-                        dl.set_symlink_fop(c_this,stub)
-                if "symlink_cbk" in funcs:
-                        @OpSymlink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  inode, buf, preparent, postparent, xdata,
-                                  s=self):
-                                return s.symlink_cbk (frame, cookie, this,
-                                                      op_ret, op_errno, inode,
-                                                      buf, preparent,
-                                                      postparent, xdata)
-                        self.stub_refs["symlink_cbk"] = stub
-                        dl.set_symlink_cbk(c_this,stub)
-                if "unlink_fop" in funcs:
-                        @OpUnlink.fop_type
-                        def stub (frame, this, loc, xflags,
-                                  xdata, s=self):
-                                return s.unlink_fop (frame, this, loc,
-                                                     xflags, xdata)
-                        self.stub_refs["unlink_fop"] = stub
-                        dl.set_unlink_fop(c_this,stub)
-                if "unlink_cbk" in funcs:
-                        @OpUnlink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  preparent, postparent, xdata, s=self):
-                                return s.unlink_cbk (frame, cookie, this,
-                                                     op_ret, op_errno,
-                                                     preparent, postparent,
-                                                     xdata)
-                        self.stub_refs["unlink_cbk"] = stub
-                        dl.set_unlink_cbk(c_this,stub)
-                if "readlink_fop" in funcs:
-                        @OpReadlink.fop_type
-                        def stub (frame, this, loc, size,
-                                  xdata, s=self):
-                                return s.readlink_fop (frame, this, loc,
-                                                       size, xdata)
-                        self.stub_refs["readlink_fop"] = stub
-                        dl.set_readlink_fop(c_this,stub)
-                if "readlink_cbk" in funcs:
-                        @OpReadlink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  path, buf, xdata, s=self):
-                                return s.readlink_cbk (frame, cookie, this,
-                                                       op_ret, op_errno,
-                                                       path, buf, xdata)
-                        self.stub_refs["readlink_cbk"] = stub
-                        dl.set_readlink_cbk(c_this,stub)
-                if "mkdir_fop" in funcs:
-                        @OpMkdir.fop_type
-                        def stub (frame, this, loc, mode, umask, xdata,
-                                  s=self):
-                                return s.mkdir_fop (frame, this, loc, mode,
-                                                    umask, xdata)
-                        self.stub_refs["mkdir_fop"] = stub
-                        dl.set_mkdir_fop(c_this,stub)
-                if "mkdir_cbk" in funcs:
-                        @OpMkdir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, inode,
-                                  buf, preparent, postparent, xdata, s=self):
-                                return s.mkdir_cbk (frame, cookie, this,
-                                                    op_ret, op_errno, inode,
-                                                    buf, preparent,
-                                                    postparent, xdata)
-                        self.stub_refs["mkdir_cbk"] = stub
-                        dl.set_mkdir_cbk(c_this,stub)
-                if "rmdir_fop" in funcs:
-                        @OpRmdir.fop_type
-                        def stub (frame, this, loc, xflags,
-                                  xdata, s=self):
-                                return s.rmdir_fop (frame, this, loc,
-                                                    xflags, xdata)
-                        self.stub_refs["rmdir_fop"] = stub
-                        dl.set_rmdir_fop(c_this,stub)
-                if "rmdir_cbk" in funcs:
-                        @OpRmdir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  preparent, postparent, xdata, s=self):
-                                return s.rmdir_cbk (frame, cookie, this,
-                                                    op_ret, op_errno,
-                                                    preparent, postparent,
-                                                    xdata)
-                        self.stub_refs["rmdir_cbk"] = stub
-                        dl.set_rmdir_cbk(c_this,stub)
diff --git a/xlators/features/glupy/src/setup.py.in b/xlators/features/glupy/src/setup.py.in
deleted file mode 100644
index 611e9695f76..00000000000
--- a/xlators/features/glupy/src/setup.py.in
+++ /dev/null
@@ -1,24 +0,0 @@
-from distutils.core import setup
-
-DESC = """GlusterFS is a distributed file-system capable of scaling to
-several petabytes. It aggregates various storage bricks over Infiniband
-RDMA or TCP/IP interconnect into one large parallel network file system.
-GlusterFS is one of the most sophisticated file systems in terms of
-features and extensibility.  It borrows a powerful concept called
-Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
-user space and easily manageable.
-
-This package contains Glupy, the Python translator interface for GlusterFS."""
-
-setup(
-    name='glusterfs-glupy',
-    version='@PACKAGE_VERSION@',
-    description='Glupy is the Python translator interface for GlusterFS',
-    long_description=DESC,
-    author='Gluster Community',
-    author_email='gluster-devel@gluster.org',
-    license='LGPLv3',
-    url='http://gluster.org/',
-    package_dir={'gluster':''},
-    packages=['gluster']
-)
diff --git a/xlators/features/index/src/Makefile.am b/xlators/features/index/src/Makefile.am
index 2f94a8c6273..c71c238c163 100644
--- a/xlators/features/index/src/Makefile.am
+++ b/xlators/features/index/src/Makefile.am
@@ -1,4 +1,6 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = index.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 index_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
@@ -6,10 +8,10 @@ index_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 index_la_SOURCES = index.c
 index_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = index.h index-mem-types.h
+noinst_HEADERS = index.h index-mem-types.h index-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(top_srcdir)/rpc/rpc-lib/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/features/index/src/index-mem-types.h b/xlators/features/index/src/index-mem-types.h
index 553d492dfbf..58833d0ec9b 100644
--- a/xlators/features/index/src/index-mem-types.h
+++ b/xlators/features/index/src/index-mem-types.h
@@ -8,15 +8,16 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef __QUIESCE_MEM_TYPES_H__
-#define __QUIESCE_MEM_TYPES_H__
+#ifndef __INDEX_MEM_TYPES_H__
+#define __INDEX_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_index_mem_types_ {
-        gf_index_mt_priv_t = gf_common_mt_end + 1,
-        gf_index_inode_ctx_t = gf_common_mt_end + 2,
-        gf_index_fd_ctx_t = gf_common_mt_end + 3,
-        gf_index_mt_end
+    gf_index_mt_priv_t = gf_common_mt_end + 1,
+    gf_index_inode_ctx_t,
+    gf_index_fd_ctx_t,
+    gf_index_mt_local_t,
+    gf_index_mt_end
 };
 #endif
diff --git a/xlators/features/index/src/index-messages.h b/xlators/features/index/src/index-messages.h
new file mode 100644
index 00000000000..364f17cd34e
--- /dev/null
+++ b/xlators/features/index/src/index-messages.h
@@ -0,0 +1,33 @@
+/*
+ Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _INDEX_MESSAGES_H_
+#define _INDEX_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(INDEX, INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+           INDEX_MSG_INDEX_READDIR_FAILED, INDEX_MSG_INDEX_ADD_FAILED,
+           INDEX_MSG_INDEX_DEL_FAILED, INDEX_MSG_DICT_SET_FAILED,
+           INDEX_MSG_INODE_CTX_GET_SET_FAILED, INDEX_MSG_INVALID_ARGS,
+           INDEX_MSG_FD_OP_FAILED, INDEX_MSG_WORKER_THREAD_CREATE_FAILED,
+           INDEX_MSG_INVALID_GRAPH);
+
+#endif /* !_INDEX_MESSAGES_H_ */
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index 2c8813b2959..4abb2c73ce5 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -8,2252 +8,2675 @@
    cases as published by the Free Software Foundation.
 */
 #include "index.h"
-#include "options.h"
+#include <glusterfs/options.h>
 #include "glusterfs3-xdr.h"
-#include "syscall.h"
-#include "syncop.h"
-#include "common-utils.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/common-utils.h>
+#include "index-messages.h"
+#include <ftw.h>
+#include <libgen.h> /* for dirname() */
+#include <signal.h>
 
 #define XATTROP_SUBDIR "xattrop"
 #define DIRTY_SUBDIR "dirty"
 #define ENTRY_CHANGES_SUBDIR "entry-changes"
 
 struct index_syncop_args {
-        inode_t *parent;
-        gf_dirent_t *entries;
+    inode_t *parent;
+    gf_dirent_t *entries;
+    char *path;
 };
 
 static char *index_vgfid_xattrs[XATTROP_TYPE_END] = {
-        [XATTROP] = GF_XATTROP_INDEX_GFID,
-        [DIRTY] = GF_XATTROP_DIRTY_GFID,
-        [ENTRY_CHANGES] = GF_XATTROP_ENTRY_CHANGES_GFID
-};
+    [XATTROP] = GF_XATTROP_INDEX_GFID,
+    [DIRTY] = GF_XATTROP_DIRTY_GFID,
+    [ENTRY_CHANGES] = GF_XATTROP_ENTRY_CHANGES_GFID};
 
 static char *index_subdirs[XATTROP_TYPE_END] = {
-        [XATTROP] = XATTROP_SUBDIR,
-        [DIRTY] = DIRTY_SUBDIR,
-        [ENTRY_CHANGES] = ENTRY_CHANGES_SUBDIR
-};
+    [XATTROP] = XATTROP_SUBDIR,
+    [DIRTY] = DIRTY_SUBDIR,
+    [ENTRY_CHANGES] = ENTRY_CHANGES_SUBDIR};
 
 int
-index_get_type_from_vgfid (index_priv_t *priv, uuid_t vgfid)
+index_get_type_from_vgfid(index_priv_t *priv, uuid_t vgfid)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < XATTROP_TYPE_END; i++) {
-                if (gf_uuid_compare (priv->internal_vgfid[i], vgfid) == 0)
-                        return i;
-        }
-        return -1;
+    for (i = 0; i < XATTROP_TYPE_END; i++) {
+        if (gf_uuid_compare(priv->internal_vgfid[i], vgfid) == 0)
+            return i;
+    }
+    return -1;
 }
 
 gf_boolean_t
-index_is_virtual_gfid (index_priv_t *priv, uuid_t vgfid)
+index_is_virtual_gfid(index_priv_t *priv, uuid_t vgfid)
 {
-        if (index_get_type_from_vgfid (priv, vgfid) < 0)
-                return _gf_false;
-        return _gf_true;
+    if (index_get_type_from_vgfid(priv, vgfid) < 0)
+        return _gf_false;
+    return _gf_true;
 }
 
 static int
-__index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
-{
-        int               ret = 0;
-        index_inode_ctx_t *ictx = NULL;
-        uint64_t          tmpctx = 0;
-
-        ret = __inode_ctx_get (inode, this, &tmpctx);
-        if (!ret) {
-                ictx = (index_inode_ctx_t *) (long) tmpctx;
-                goto out;
-        }
-        ictx = GF_CALLOC (1, sizeof (*ictx), gf_index_inode_ctx_t);
-        if (!ictx) {
-                ret = -1;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&ictx->callstubs);
-        ret = __inode_ctx_put (inode, this, (uint64_t)ictx);
-        if (ret) {
-                GF_FREE (ictx);
-                ictx = NULL;
-                goto out;
-        }
+__index_inode_ctx_get(inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+{
+    int ret = 0;
+    index_inode_ctx_t *ictx = NULL;
+    uint64_t tmpctx = 0;
+
+    ret = __inode_ctx_get(inode, this, &tmpctx);
+    if (!ret) {
+        ictx = (index_inode_ctx_t *)(long)tmpctx;
+        goto out;
+    }
+    ictx = GF_CALLOC(1, sizeof(*ictx), gf_index_inode_ctx_t);
+    if (!ictx) {
+        ret = -1;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&ictx->callstubs);
+    ret = __inode_ctx_put(inode, this, (uint64_t)(uintptr_t)ictx);
+    if (ret) {
+        GF_FREE(ictx);
+        ictx = NULL;
+        goto out;
+    }
 out:
-        if (ictx)
-                *ctx = ictx;
-        return ret;
+    if (ictx)
+        *ctx = ictx;
+    return ret;
 }
 
 static int
-index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+index_inode_ctx_get(inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
 {
-        int               ret = 0;
+    int ret = 0;
 
-        LOCK (&inode->lock);
-        {
-                ret = __index_inode_ctx_get (inode, this, ctx);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __index_inode_ctx_get(inode, this, ctx);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-index_is_subdir_of_entry_changes (xlator_t *this, inode_t *inode)
+index_is_subdir_of_entry_changes(xlator_t *this, inode_t *inode)
 {
-        index_priv_t      *priv = this->private;
-        index_inode_ctx_t *ctx  = NULL;
-        int               ret   = 0;
+    index_inode_ctx_t *ctx = NULL;
+    int ret = 0;
 
-        if (!inode)
-                return _gf_false;
-
-        ret = index_inode_ctx_get (inode, this, &ctx);
-        if ((ret == 0) && !gf_uuid_is_null (ctx->virtual_pargfid))
-                return _gf_true;
+    if (!inode)
         return _gf_false;
+
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if ((ret == 0) && !gf_uuid_is_null(ctx->virtual_pargfid))
+        return _gf_true;
+    return _gf_false;
 }
 
 static int
-index_get_type_from_vgfid_xattr (const char *name)
+index_get_type_from_vgfid_xattr(const char *name)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < XATTROP_TYPE_END; i++) {
-                if (strcmp (name, index_vgfid_xattrs[i]) == 0)
-                        return i;
-        }
-        return -1;
+    for (i = 0; i < XATTROP_TYPE_END; i++) {
+        if (strcmp(name, index_vgfid_xattrs[i]) == 0)
+            return i;
+    }
+    return -1;
 }
 
 gf_boolean_t
-index_is_fop_on_internal_inode (xlator_t *this, inode_t *inode, uuid_t gfid)
+index_is_fop_on_internal_inode(xlator_t *this, inode_t *inode, uuid_t gfid)
 {
-        index_priv_t *priv = this->private;
-        uuid_t       vgfid = {0};
+    index_priv_t *priv = this->private;
+    uuid_t vgfid = {0};
 
-        if (!inode)
-                return _gf_false;
+    if (!inode)
+        return _gf_false;
 
-        if (gfid && !gf_uuid_is_null (gfid))
-                gf_uuid_copy (vgfid, gfid);
-        else
-                gf_uuid_copy (vgfid, inode->gfid);
+    if (gfid && !gf_uuid_is_null(gfid))
+        gf_uuid_copy(vgfid, gfid);
+    else
+        gf_uuid_copy(vgfid, inode->gfid);
 
-        if (index_is_virtual_gfid (priv, vgfid))
-                return _gf_true;
-        if (index_is_subdir_of_entry_changes (this, inode))
-                return _gf_true;
-        return _gf_false;
+    if (index_is_virtual_gfid(priv, vgfid))
+        return _gf_true;
+    if (index_is_subdir_of_entry_changes(this, inode))
+        return _gf_true;
+    return _gf_false;
 }
 
 static gf_boolean_t
-index_is_vgfid_xattr (const char *name)
+index_is_vgfid_xattr(const char *name)
 {
-        if (index_get_type_from_vgfid_xattr (name) < 0)
-                return _gf_false;
-        return _gf_true;
+    if (index_get_type_from_vgfid_xattr(name) < 0)
+        return _gf_false;
+    return _gf_true;
 }
 
 call_stub_t *
-__index_dequeue (struct list_head *callstubs)
+__index_dequeue(struct list_head *callstubs)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (!list_empty (callstubs)) {
-                stub = list_entry (callstubs->next, call_stub_t, list);
-                list_del_init (&stub->list);
-        }
+    if (!list_empty(callstubs)) {
+        stub = list_entry(callstubs->next, call_stub_t, list);
+        list_del_init(&stub->list);
+    }
 
-        return stub;
+    return stub;
 }
 
 static void
-__index_enqueue (struct list_head *callstubs, call_stub_t *stub)
+__index_enqueue(struct list_head *callstubs, call_stub_t *stub)
 {
-        list_add_tail (&stub->list, callstubs);
+    list_add_tail(&stub->list, callstubs);
 }
 
 static void
-worker_enqueue (xlator_t *this, call_stub_t *stub)
+worker_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        index_priv_t    *priv = NULL;
+    index_priv_t *priv = NULL;
 
-        priv = this->private;
-        pthread_mutex_lock (&priv->mutex);
-        {
-                __index_enqueue (&priv->callstubs, stub);
-                pthread_cond_signal (&priv->cond);
-        }
-        pthread_mutex_unlock (&priv->mutex);
+    priv = this->private;
+    pthread_mutex_lock(&priv->mutex);
+    {
+        __index_enqueue(&priv->callstubs, stub);
+        GF_ATOMIC_INC(priv->stub_cnt);
+        pthread_cond_signal(&priv->cond);
+    }
+    pthread_mutex_unlock(&priv->mutex);
 }
 
 void *
-index_worker (void *data)
-{
-        index_priv_t     *priv = NULL;
-        xlator_t         *this = NULL;
-        call_stub_t      *stub = NULL;
-        int               ret = 0;
-
-        THIS = data;
-        this = data;
-        priv = this->private;
-
-        for (;;) {
-                pthread_mutex_lock (&priv->mutex);
-                {
-                        while (list_empty (&priv->callstubs)) {
-                                ret = pthread_cond_wait (&priv->cond,
-                                                         &priv->mutex);
-                        }
-
-                        stub = __index_dequeue (&priv->callstubs);
+index_worker(void *data)
+{
+    index_priv_t *priv = NULL;
+    xlator_t *this = NULL;
+    call_stub_t *stub = NULL;
+    gf_boolean_t bye = _gf_false;
+
+    THIS = data;
+    this = data;
+    priv = this->private;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->mutex);
+        {
+            while (list_empty(&priv->callstubs)) {
+                if (priv->down) {
+                    bye = _gf_true; /*Avoid wait*/
+                    break;
+                }
+                (void)pthread_cond_wait(&priv->cond, &priv->mutex);
+                if (priv->down) {
+                    bye = _gf_true;
+                    break;
                 }
-                pthread_mutex_unlock (&priv->mutex);
+            }
+            if (!bye)
+                stub = __index_dequeue(&priv->callstubs);
+            if (bye) {
+                priv->curr_count--;
+                if (priv->curr_count == 0)
+                    pthread_cond_broadcast(&priv->cond);
+            }
+        }
+        pthread_mutex_unlock(&priv->mutex);
 
-                if (stub) /* guard against spurious wakeups */
-                        call_resume (stub);
+        if (stub) { /* guard against spurious wakeups */
+            call_resume(stub);
+            GF_ATOMIC_DEC(priv->stub_cnt);
         }
+        stub = NULL;
+        if (bye)
+            break;
+    }
 
-        return NULL;
+    return NULL;
 }
 
 static void
-make_index_dir_path (char *base, const char *subdir,
-                     char *index_dir, size_t len)
+make_index_dir_path(char *base, const char *subdir, char *index_dir, size_t len)
 {
-        snprintf (index_dir, len, "%s/%s", base, subdir);
+    snprintf(index_dir, len, "%s/%s", base, subdir);
 }
 
 int
-index_dir_create (xlator_t *this, const char *subdir)
-{
-        int          ret = 0;
-        struct       stat st = {0};
-        char         fullpath[PATH_MAX] = {0};
-        char         path[PATH_MAX] = {0};
-        char         *dir = NULL;
-        index_priv_t *priv = NULL;
-        size_t       len = 0;
-        size_t       pathlen = 0;
-
-        priv = this->private;
-        make_index_dir_path (priv->index_basepath, subdir, fullpath,
-                             sizeof (fullpath));
-        ret = sys_stat (fullpath, &st);
-        if (!ret) {
-                if (!S_ISDIR (st.st_mode))
-                        ret = -2;
-                goto out;
-        }
-
-        pathlen = strlen (fullpath);
-        if ((pathlen > 1) && fullpath[pathlen - 1] == '/')
-                fullpath[pathlen - 1] = '\0';
-        dir = strchr (fullpath, '/');
-        while (dir) {
-                dir = strchr (dir + 1, '/');
-                if (dir)
-                        len = pathlen - strlen (dir);
-                else
-                        len = pathlen;
-                strncpy (path, fullpath, len);
-                path[len] = '\0';
-                ret = sys_mkdir (path, 0600);
-                if (ret && (errno != EEXIST))
-                        goto out;
-        }
-        ret = 0;
+index_dir_create(xlator_t *this, const char *subdir)
+{
+    int ret = 0;
+    struct stat st = {0};
+    char fullpath[PATH_MAX] = {0};
+    char path[PATH_MAX] = {0};
+    char *dir = NULL;
+    index_priv_t *priv = NULL;
+    size_t len = 0;
+    size_t pathlen = 0;
+
+    priv = this->private;
+    make_index_dir_path(priv->index_basepath, subdir, fullpath,
+                        sizeof(fullpath));
+    ret = sys_stat(fullpath, &st);
+    if (!ret) {
+        if (!S_ISDIR(st.st_mode))
+            ret = -2;
+        goto out;
+    }
+
+    pathlen = strlen(fullpath);
+    if ((pathlen > 1) && fullpath[pathlen - 1] == '/')
+        fullpath[pathlen - 1] = '\0';
+    dir = strchr(fullpath, '/');
+    while (dir) {
+        dir = strchr(dir + 1, '/');
+        if (dir)
+            len = pathlen - strlen(dir);
+        else
+            len = pathlen;
+        strncpy(path, fullpath, len);
+        path[len] = '\0';
+        ret = sys_mkdir(path, 0600);
+        if (ret && (errno != EEXIST))
+            goto out;
+    }
+    ret = 0;
 out:
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to "
-                        "create (%s)", priv->index_basepath, subdir,
-                        strerror (errno));
-        } else if (ret == -2) {
-                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to create, "
-                        "path exists, not a directory ", priv->index_basepath,
-                        subdir);
-        }
-        return ret;
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno,
+               INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+               "%s/%s: Failed to "
+               "create",
+               priv->index_basepath, subdir);
+    } else if (ret == -2) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOTDIR,
+               INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+               "%s/%s: Failed to "
+               "create, path exists, not a directory ",
+               priv->index_basepath, subdir);
+    }
+    return ret;
 }
 
 void
-index_get_index (index_priv_t *priv, uuid_t index)
+index_get_index(index_priv_t *priv, uuid_t index)
 {
-        LOCK (&priv->lock);
-        {
-                gf_uuid_copy (index, priv->index);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        gf_uuid_copy(index, priv->index);
+    }
+    UNLOCK(&priv->lock);
 }
 
 void
-index_generate_index (index_priv_t *priv, uuid_t index)
+index_generate_index(index_priv_t *priv, uuid_t index)
 {
-        LOCK (&priv->lock);
-        {
-                //To prevent duplicate generates.
-                //This method fails if number of contending threads is greater
-                //than MAX_LINK count of the fs
-                if (!gf_uuid_compare (priv->index, index))
-                        gf_uuid_generate (priv->index);
-                gf_uuid_copy (index, priv->index);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        // To prevent duplicate generates.
+        // This method fails if number of contending threads is greater
+        // than MAX_LINK count of the fs
+        if (!gf_uuid_compare(priv->index, index))
+            gf_uuid_generate(priv->index);
+        gf_uuid_copy(index, priv->index);
+    }
+    UNLOCK(&priv->lock);
 }
 
 static void
-make_index_path (char *base, const char *subdir, uuid_t index,
-                 char *index_path, size_t len)
+make_index_path(char *base, const char *subdir, uuid_t index, char *index_path,
+                size_t len)
 {
-        make_index_dir_path (base, subdir, index_path, len);
-        snprintf (index_path + strlen (index_path), len - strlen (index_path),
-                  "/%s-%s", subdir, uuid_utoa (index));
+    make_index_dir_path(base, subdir, index_path, len);
+    snprintf(index_path + strlen(index_path), len - strlen(index_path),
+             "/%s-%s", subdir, uuid_utoa(index));
 }
 
 static void
-make_gfid_path (char *base, const char *subdir, uuid_t gfid,
-                char *gfid_path, size_t len)
+make_gfid_path(char *base, const char *subdir, uuid_t gfid, char *gfid_path,
+               size_t len)
 {
-        make_index_dir_path (base, subdir, gfid_path, len);
-        snprintf (gfid_path + strlen (gfid_path), len - strlen (gfid_path),
-                  "/%s", uuid_utoa (gfid));
+    make_index_dir_path(base, subdir, gfid_path, len);
+    snprintf(gfid_path + strlen(gfid_path), len - strlen(gfid_path), "/%s",
+             uuid_utoa(gfid));
 }
 
 static void
-make_file_path (char *base, const char *subdir, const char *filename,
-                char *file_path, size_t len)
+make_file_path(char *base, const char *subdir, const char *filename,
+               char *file_path, size_t len)
 {
-        make_index_dir_path (base, subdir, file_path, len);
-        snprintf (file_path + strlen (file_path), len - strlen (file_path),
-                  "/%s", filename);
+    make_index_dir_path(base, subdir, file_path, len);
+    snprintf(file_path + strlen(file_path), len - strlen(file_path), "/%s",
+             filename);
 }
 
 static int
-is_index_file_current (char *filename, uuid_t priv_index, char *subdir)
+is_index_file_current(char *filename, uuid_t priv_index, char *subdir)
 {
-        char current_index[GF_UUID_BUF_SIZE + 16] = {0, };
+    char current_index[GF_UUID_BUF_SIZE + 16] = {
+        0,
+    };
 
-        snprintf (current_index, sizeof current_index,
-                  "%s-%s", subdir, uuid_utoa(priv_index));
-        return (!strcmp(filename, current_index));
+    snprintf(current_index, sizeof current_index, "%s-%s", subdir,
+             uuid_utoa(priv_index));
+    return (!strcmp(filename, current_index));
 }
 
 static void
-check_delete_stale_index_file (xlator_t *this, char *filename, char *subdir)
+check_delete_stale_index_file(xlator_t *this, char *filename, char *subdir)
 {
-        int             ret = 0;
-        struct stat     st = {0};
-        char            filepath[PATH_MAX] = {0};
-        index_priv_t    *priv = NULL;
+    int ret = 0;
+    struct stat st = {0};
+    char filepath[PATH_MAX] = {0};
+    index_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (is_index_file_current (filename, priv->index, subdir))
-                return;
+    if (is_index_file_current(filename, priv->index, subdir))
+        return;
 
-        make_file_path (priv->index_basepath, subdir,
-                        filename, filepath, sizeof (filepath));
-        ret = sys_stat (filepath, &st);
-        if (!ret && st.st_nlink == 1)
-                sys_unlink (filepath);
+    make_file_path(priv->index_basepath, subdir, filename, filepath,
+                   sizeof(filepath));
+    ret = sys_stat(filepath, &st);
+    if (!ret && st.st_nlink == 1)
+        sys_unlink(filepath);
 }
 
 static void
-index_set_link_count (index_priv_t *priv, int64_t count,
-                      index_xattrop_type_t type)
+index_set_link_count(index_priv_t *priv, int64_t count,
+                     index_xattrop_type_t type)
 {
-        switch (type) {
+    switch (type) {
         case XATTROP:
-                LOCK (&priv->lock);
-                {
-                        priv->pending_count = count;
-                }
-                UNLOCK (&priv->lock);
-                break;
+            LOCK(&priv->lock);
+            {
+                priv->pending_count = count;
+            }
+            UNLOCK(&priv->lock);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 static void
-index_get_link_count (index_priv_t *priv, int64_t *count,
-                      index_xattrop_type_t type)
+index_get_link_count(index_priv_t *priv, int64_t *count,
+                     index_xattrop_type_t type)
 {
-        switch (type) {
+    switch (type) {
         case XATTROP:
-                LOCK (&priv->lock);
-                {
-                        *count = priv->pending_count;
-                }
-                UNLOCK (&priv->lock);
-                break;
+            LOCK(&priv->lock);
+            {
+                *count = priv->pending_count;
+            }
+            UNLOCK(&priv->lock);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 static void
-index_dec_link_count (index_priv_t *priv, index_xattrop_type_t type)
+index_dec_link_count(index_priv_t *priv, index_xattrop_type_t type)
 {
-        switch (type) {
+    switch (type) {
         case XATTROP:
-                LOCK (&priv->lock);
-                {
-                        priv->pending_count--;
-                        if (priv->pending_count == 0)
-                                priv->pending_count--;
-                }
-                UNLOCK (&priv->lock);
-                break;
+            LOCK(&priv->lock);
+            {
+                priv->pending_count--;
+                if (priv->pending_count == 0)
+                    priv->pending_count--;
+            }
+            UNLOCK(&priv->lock);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
-char*
-index_get_subdir_from_type (index_xattrop_type_t type)
+char *
+index_get_subdir_from_type(index_xattrop_type_t type)
 {
-        if (type < XATTROP || type >= XATTROP_TYPE_END)
-                return NULL;
-        return index_subdirs[type];
+    if (type < XATTROP || type >= XATTROP_TYPE_END)
+        return NULL;
+    return index_subdirs[type];
 }
 
-char*
-index_get_subdir_from_vgfid (index_priv_t *priv, uuid_t vgfid)
+char *
+index_get_subdir_from_vgfid(index_priv_t *priv, uuid_t vgfid)
 {
-        return index_get_subdir_from_type (index_get_type_from_vgfid (priv,
-                                                                vgfid));
+    return index_get_subdir_from_type(index_get_type_from_vgfid(priv, vgfid));
 }
 
 static int
-index_fill_readdir (fd_t *fd, index_fd_ctx_t *fctx, DIR *dir, off_t off,
-                    size_t size, gf_dirent_t *entries)
-{
-        off_t     in_case = -1;
-        off_t     last_off = 0;
-        size_t    filled = 0;
-        int       count = 0;
-        char      entrybuf[sizeof(struct dirent) + 256 + 8];
-        struct dirent  *entry          = NULL;
-        int32_t              this_size      = -1;
-        gf_dirent_t          *this_entry     = NULL;
-        xlator_t             *this = NULL;
-
-        this = THIS;
-        if (!off) {
-                rewinddir (dir);
-        } else {
-                seekdir (dir, off);
+index_fill_readdir(fd_t *fd, index_fd_ctx_t *fctx, DIR *dir, off_t off,
+                   size_t size, gf_dirent_t *entries)
+{
+    off_t in_case = -1;
+    off_t last_off = 0;
+    size_t filled = 0;
+    int count = 0;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    int32_t this_size = -1;
+    gf_dirent_t *this_entry = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    if (!off) {
+        rewinddir(dir);
+    } else {
+        seekdir(dir, off);
 #ifndef GF_LINUX_HOST_OS
-                if ((u_long)telldir(dir) != off && off != fctx->dir_eof) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "seekdir(0x%llx) failed on dir=%p: "
-				"Invalid argument (offset reused from "
-				"another DIR * structure?)", off, dir);
-                        errno = EINVAL;
-                        count = -1;
-                        goto out;
-                }
-#endif /* GF_LINUX_HOST_OS */
+        if ((u_long)telldir(dir) != off && off != fctx->dir_eof) {
+            gf_msg(THIS->name, GF_LOG_ERROR, EINVAL,
+                   INDEX_MSG_INDEX_READDIR_FAILED,
+                   "seekdir(0x%llx) failed on dir=%p: "
+                   "Invalid argument (offset reused from "
+                   "another DIR * structure?)",
+                   off, dir);
+            errno = EINVAL;
+            count = -1;
+            goto out;
         }
+#endif /* GF_LINUX_HOST_OS */
+    }
 
-        while (filled <= size) {
-                in_case = (u_long)telldir (dir);
+    while (filled <= size) {
+        in_case = (u_long)telldir(dir);
 
-                if (in_case == -1) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "telldir failed on dir=%p: %s",
-                                dir, strerror (errno));
-                        goto out;
-                }
+        if (in_case == -1) {
+            gf_msg(THIS->name, GF_LOG_ERROR, errno,
+                   INDEX_MSG_INDEX_READDIR_FAILED, "telldir failed on dir=%p",
+                   dir);
+            goto out;
+        }
 
-                errno = 0;
-                entry = NULL;
-                readdir_r (dir, (struct dirent *)entrybuf, &entry);
-
-                if (!entry) {
-                        if (errno == EBADF) {
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "readdir failed on dir=%p: %s",
-                                        dir, strerror (errno));
-                                goto out;
-                        }
-                        break;
-                }
+        errno = 0;
+        entry = sys_readdir(dir, scratch);
+        if (!entry || errno != 0) {
+            if (errno == EBADF) {
+                gf_msg(THIS->name, GF_LOG_WARNING, errno,
+                       INDEX_MSG_INDEX_READDIR_FAILED,
+                       "readdir failed on dir=%p", dir);
+                goto out;
+            }
+            break;
+        }
 
-                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
-                              strlen (XATTROP_SUBDIR"-"))) {
-                        check_delete_stale_index_file (this, entry->d_name,
-                                                       XATTROP_SUBDIR);
-                        continue;
-                } else if (!strncmp (entry->d_name, DIRTY_SUBDIR"-",
-                           strlen (DIRTY_SUBDIR"-"))) {
-                        check_delete_stale_index_file (this, entry->d_name,
-                                                       DIRTY_SUBDIR);
-                        continue;
-                }
+        if (!strncmp(entry->d_name, XATTROP_SUBDIR "-",
+                     strlen(XATTROP_SUBDIR "-"))) {
+            check_delete_stale_index_file(this, entry->d_name, XATTROP_SUBDIR);
+            continue;
+        } else if (!strncmp(entry->d_name, DIRTY_SUBDIR "-",
+                            strlen(DIRTY_SUBDIR "-"))) {
+            check_delete_stale_index_file(this, entry->d_name, DIRTY_SUBDIR);
+            continue;
+        }
 
-                this_size = max (sizeof (gf_dirent_t),
-                                 sizeof (gfs3_dirplist))
-                        + strlen (entry->d_name) + 1;
+        this_size = max(sizeof(gf_dirent_t), sizeof(gfs3_dirplist)) +
+                    strlen(entry->d_name) + 1;
 
-                if (this_size + filled > size) {
-                        seekdir (dir, in_case);
+        if (this_size + filled > size) {
+            seekdir(dir, in_case);
 #ifndef GF_LINUX_HOST_OS
-                        if ((u_long)telldir(dir) != in_case &&
-                            in_case != fctx->dir_eof) {
-				gf_log (THIS->name, GF_LOG_ERROR,
-					"seekdir(0x%llx) failed on dir=%p: "
-					"Invalid argument (offset reused from "
-					"another DIR * structure?)",
-					in_case, dir);
-				errno = EINVAL;
-				count = -1;
-				goto out;
-                        }
+            if ((u_long)telldir(dir) != in_case && in_case != fctx->dir_eof) {
+                gf_msg(THIS->name, GF_LOG_ERROR, EINVAL,
+                       INDEX_MSG_INDEX_READDIR_FAILED,
+                       "seekdir(0x%llx) failed on dir=%p: "
+                       "Invalid argument (offset reused from "
+                       "another DIR * structure?)",
+                       in_case, dir);
+                errno = EINVAL;
+                count = -1;
+                goto out;
+            }
 #endif /* GF_LINUX_HOST_OS */
-                        break;
-                }
+            break;
+        }
 
-                this_entry = gf_dirent_for_name (entry->d_name);
+        this_entry = gf_dirent_for_name(entry->d_name);
 
-                if (!this_entry) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "could not create gf_dirent for entry %s: (%s)",
-                                entry->d_name, strerror (errno));
-                        goto out;
-                }
-                /*
-                 * we store the offset of next entry here, which is
-                 * probably not intended, but code using syncop_readdir()
-                 * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
-                 * for directory read resumption.
-                 */
-                last_off = (u_long)telldir(dir);
-                this_entry->d_off = last_off;
-                this_entry->d_ino = entry->d_ino;
-
-                list_add_tail (&this_entry->list, &entries->list);
-
-                filled += this_size;
-                count ++;
+        if (!this_entry) {
+            gf_msg(THIS->name, GF_LOG_ERROR, errno,
+                   INDEX_MSG_INDEX_READDIR_FAILED,
+                   "could not create gf_dirent for entry %s", entry->d_name);
+            goto out;
         }
+        /*
+         * we store the offset of next entry here, which is
+         * probably not intended, but code using syncop_readdir()
+         * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
+         * for directory read resumption.
+         */
+        last_off = (u_long)telldir(dir);
+        this_entry->d_off = last_off;
+        this_entry->d_ino = entry->d_ino;
 
-        if ((!sys_readdir (dir) && (errno == 0))) {
-                /* Indicate EOF */
-                errno = ENOENT;
-                /* Remember EOF offset for later detection */
-                fctx->dir_eof = last_off;
-        }
+        list_add_tail(&this_entry->list, &entries->list);
+
+        filled += this_size;
+        count++;
+    }
+
+    errno = 0;
+
+    if ((!sys_readdir(dir, scratch) && (errno == 0))) {
+        /* Indicate EOF */
+        errno = ENOENT;
+        /* Remember EOF offset for later detection */
+        fctx->dir_eof = last_off;
+    }
 out:
-        return count;
+    return count;
 }
 
 int
-index_link_to_base (xlator_t *this, char *base, size_t base_len,
-                    char *fpath, const char *subdir)
-{
-        int ret = 0;
-        int fd  = 0;
-        int op_errno = 0;
-        uuid_t index = {0};
-        index_priv_t *priv = this->private;
-
-        ret = sys_link (base, fpath);
-        if (!ret || (errno == EEXIST))  {
-                ret = 0;
-                goto out;
-        }
-
-        op_errno = errno;
-        if (op_errno == ENOENT) {
-                ret = index_dir_create (this, subdir);
-                if (ret) {
-                        op_errno = errno;
-                        goto out;
-                }
-        } else if (op_errno == EMLINK) {
-                index_generate_index (priv, index);
-                make_index_path (priv->index_basepath, subdir,
-                                 index, base, base_len);
-        } else {
-                goto out;
-        }
+index_link_to_base(xlator_t *this, char *fpath, const char *subdir)
+{
+    int ret = 0;
+    int fd = 0;
+    int op_errno = 0;
+    uuid_t index = {0};
+    index_priv_t *priv = this->private;
+    char base[PATH_MAX] = {0};
 
-        op_errno = 0;
-        fd = sys_creat (base, 0);
-        if ((fd < 0) && (errno != EEXIST)) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
-                        "create index (%s)", fpath, strerror (op_errno));
-                goto out;
-        }
+    index_get_index(priv, index);
+    make_index_path(priv->index_basepath, subdir, index, base, sizeof(base));
 
-        if (fd >= 0)
-                sys_close (fd);
+    ret = sys_link(base, fpath);
+    if (!ret || (errno == EEXIST)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = sys_link (base, fpath);
-        if (ret && (errno != EEXIST)) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
-                        "add to index (%s)", fpath, strerror (errno));
-                goto out;
-        }
+    op_errno = errno;
+    if (op_errno == ENOENT) {
+        ret = index_dir_create(this, subdir);
+        if (ret) {
+            op_errno = errno;
+            goto out;
+        }
+    } else if (op_errno == EMLINK) {
+        index_generate_index(priv, index);
+        make_index_path(priv->index_basepath, subdir, index, base,
+                        sizeof(base));
+    } else {
+        goto out;
+    }
+
+    op_errno = 0;
+    fd = sys_creat(base, 0);
+    if ((fd < 0) && (errno != EEXIST)) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, INDEX_MSG_INDEX_ADD_FAILED,
+               "%s: Not able to "
+               "create index",
+               fpath);
+        goto out;
+    }
+
+    if (fd >= 0)
+        sys_close(fd);
+
+    ret = sys_link(base, fpath);
+    if (ret && (errno != EEXIST)) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, errno, INDEX_MSG_INDEX_ADD_FAILED,
+               "%s: Not able to "
+               "add to index",
+               fpath);
+        goto out;
+    }
 out:
-        return -op_errno;
+    return -op_errno;
 }
 
 int
-index_add (xlator_t *this, uuid_t gfid, const char *subdir,
-           index_xattrop_type_t type)
-{
-        int32_t           op_errno = 0;
-        char              gfid_path[PATH_MAX] = {0};
-        char              index_path[PATH_MAX] = {0};
-        int               ret = -1;
-        uuid_t            index = {0};
-        index_priv_t      *priv = NULL;
-        struct stat       st = {0};
-        int               fd = 0;
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, op_errno, EINVAL);
-
-        make_gfid_path (priv->index_basepath, subdir, gfid,
-                        gfid_path, sizeof (gfid_path));
-
-        ret = sys_stat (gfid_path, &st);
-        if (!ret)
-                goto out;
-        index_get_index (priv, index);
-        make_index_path (priv->index_basepath, subdir,
-                         index, index_path, sizeof (index_path));
-        ret = index_link_to_base (this, index_path, sizeof (index_path),
-                                  gfid_path, subdir);
+index_add(xlator_t *this, uuid_t gfid, const char *subdir,
+          index_xattrop_type_t type)
+{
+    char gfid_path[PATH_MAX] = {0};
+    int ret = -1;
+    index_priv_t *priv = NULL;
+    struct stat st = {0};
+
+    priv = this->private;
+
+    if (gf_uuid_is_null(gfid)) {
+        GF_ASSERT(0);
+        goto out;
+    }
+
+    make_gfid_path(priv->index_basepath, subdir, gfid, gfid_path,
+                   sizeof(gfid_path));
+
+    ret = sys_stat(gfid_path, &st);
+    if (!ret)
+        goto out;
+    ret = index_link_to_base(this, gfid_path, subdir);
 out:
-        return ret;
+    return ret;
 }
 
 int
-index_del (xlator_t *this, uuid_t gfid, const char *subdir, int type)
-{
-        int32_t      op_errno __attribute__((unused)) = 0;
-        index_priv_t *priv = NULL;
-        int          ret = 0;
-        char         gfid_path[PATH_MAX] = {0};
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, op_errno, EINVAL);
-        make_gfid_path (priv->index_basepath, subdir, gfid,
-                        gfid_path, sizeof (gfid_path));
-        ret = sys_unlink (gfid_path);
-        if (ret && (errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to delete from index (%s)",
-                        gfid_path, strerror (errno));
-                ret = -errno;
-                goto out;
-        }
-        index_dec_link_count (priv, type);
-        ret = 0;
+index_del(xlator_t *this, uuid_t gfid, const char *subdir, int type)
+{
+    int32_t op_errno __attribute__((unused)) = 0;
+    index_priv_t *priv = NULL;
+    int ret = 0;
+    char gfid_path[PATH_MAX] = {0};
+    char rename_dst[PATH_MAX] = {
+        0,
+    };
+    uuid_t uuid;
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  op_errno, EINVAL);
+    make_gfid_path(priv->index_basepath, subdir, gfid, gfid_path,
+                   sizeof(gfid_path));
+
+    if ((strcmp(subdir, ENTRY_CHANGES_SUBDIR)) == 0) {
+        ret = sys_rmdir(gfid_path);
+        /* rmdir above could fail with ENOTEMPTY if the indices under
+         * it were created when granular-entry-heal was enabled, whereas
+         * the actual heal that happened was non-granular (or full) in
+         * nature, resulting in name indices getting left out. To
+         * clean up this directory without it affecting the IO path perf,
+         * the directory is renamed to a unique name under
+         * indices/entry-changes. Self-heal will pick up this entry
+         * during crawl and on lookup into the file system figure that
+         * the index is stale and subsequently wipe it out using rmdir().
+         */
+        if ((ret) && (errno == ENOTEMPTY)) {
+            gf_uuid_generate(uuid);
+            make_gfid_path(priv->index_basepath, subdir, uuid, rename_dst,
+                           sizeof(rename_dst));
+            ret = sys_rename(gfid_path, rename_dst);
+        }
+    } else {
+        ret = sys_unlink(gfid_path);
+    }
+
+    if (ret && (errno != ENOENT)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, INDEX_MSG_INDEX_DEL_FAILED,
+               "%s: failed to delete"
+               " from index",
+               gfid_path);
+        ret = -errno;
+        goto out;
+    }
+
+    index_dec_link_count(priv, type);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-_is_xattr_in_watchlist (dict_t *d, char *k, data_t *v, void *tmp)
+_is_xattr_in_watchlist(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        const char *data = tmp;
-
-        if (!strncmp (k, tmp, strlen (k)))
-                return _gf_true;
+    if (!strncmp(k, tmp, strlen(k)))
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-is_xattr_in_watchlist (dict_t *this, char *key, data_t *value, void *matchdata)
+is_xattr_in_watchlist(dict_t *this, char *key, data_t *value, void *matchdata)
 {
-        int    ret = -1;
+    int ret = -1;
 
-        //matchdata is a list of xattrs
-        //key is strncmp'ed with each xattr in matchdata.
-        //ret will be 0 if key pattern is not present in the matchdata
-        //else ret will be count number of xattrs the key pattern-matches with.
-        ret = dict_foreach_match (matchdata, _is_xattr_in_watchlist, key,
-                                  dict_null_foreach_fn, NULL);
+    // matchdata is a list of xattrs
+    // key is strncmp'ed with each xattr in matchdata.
+    // ret will be 0 if key pattern is not present in the matchdata
+    // else ret will be count number of xattrs the key pattern-matches with.
+    ret = dict_foreach_match(matchdata, _is_xattr_in_watchlist, key,
+                             dict_null_foreach_fn, NULL);
 
-        if (ret > 0)
-                return _gf_true;
-        return _gf_false;
+    if (ret > 0)
+        return _gf_true;
+    return _gf_false;
 }
 
 static int
-index_find_xattr_type (dict_t *d, char *k, data_t *v)
+index_find_xattr_type(dict_t *d, char *k, data_t *v)
 {
-        int             idx  = -1;
-        index_priv_t   *priv = THIS->private;
+    int idx = -1;
+    index_priv_t *priv = THIS->private;
 
-        if (priv->dirty_watchlist && is_xattr_in_watchlist (d, k, v,
-                                            priv->dirty_watchlist))
-                idx = DIRTY;
-        else if (priv->pending_watchlist && is_xattr_in_watchlist (d, k, v,
-                                                  priv->pending_watchlist))
-                idx = XATTROP;
+    if (priv->dirty_watchlist &&
+        is_xattr_in_watchlist(d, k, v, priv->dirty_watchlist))
+        idx = DIRTY;
+    else if (priv->pending_watchlist &&
+             is_xattr_in_watchlist(d, k, v, priv->pending_watchlist))
+        idx = XATTROP;
 
-        return idx;
+    return idx;
 }
 
 int
-index_fill_zero_array (dict_t *d, char *k, data_t *v, void *adata)
+index_fill_zero_array(dict_t *d, char *k, data_t *v, void *adata)
 {
-        int     idx = -1;
-        int     *zfilled = adata;
-        //zfilled array contains `state` for all types xattrs.
-        //state : whether the gfid file of this file exists in
-        //corresponding xattr directory or not.
+    int idx = -1;
+    int *zfilled = adata;
+    // zfilled array contains `state` for all types xattrs.
+    // state : whether the gfid file of this file exists in
+    // corresponding xattr directory or not.
 
-        idx = index_find_xattr_type (d, k, v);
-        if (idx == -1)
-                return 0;
-        zfilled[idx] = 0;
+    idx = index_find_xattr_type(d, k, v);
+    if (idx == -1)
         return 0;
+    zfilled[idx] = 0;
+    return 0;
 }
 
 static int
-_check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
-                           void *tmp)
+_check_key_is_zero_filled(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        int            *zfilled = tmp;
-        int             idx = -1;
-
-        idx = index_find_xattr_type (d, k, v);
-        if (idx == -1)
-                return 0;
+    int *zfilled = tmp;
+    int idx = -1;
 
-        /* Along with checking that the value of a key is zero filled
-         * the key's corresponding index should be assigned
-         * appropriate value.
-         * zfilled[idx] will be 0(false) if value not zero.
-         *              will be 1(true) if value is zero.
-         */
-        if (mem_0filled ((const char*)v->data, v->len)) {
-                zfilled[idx] = 0;
-                return 0;
-        }
+    idx = index_find_xattr_type(d, k, v);
+    if (idx == -1)
+        return 0;
 
-        /* If zfilled[idx] was previously 0, it means at least
-         * one xattr of its "kind" is non-zero. Keep its value
-         * the same.
-         */
-        if (zfilled[idx])
-                zfilled[idx] = 1;
+    /* Along with checking that the value of a key is zero filled
+     * the key's corresponding index should be assigned
+     * appropriate value.
+     * zfilled[idx] will be 0(false) if value not zero.
+     *              will be 1(true) if value is zero.
+     */
+    if (mem_0filled((const char *)v->data, v->len)) {
+        zfilled[idx] = 0;
         return 0;
+    }
+
+    /* If zfilled[idx] was previously 0, it means at least
+     * one xattr of its "kind" is non-zero. Keep its value
+     * the same.
+     */
+    if (zfilled[idx])
+        zfilled[idx] = 1;
+    return 0;
 }
 
 int
-index_entry_create (xlator_t *this, uuid_t gfid, char *filename)
-{
-        char entry_base_index_path[PATH_MAX] = {0};
-        char pgfid_path[PATH_MAX] = {0};
-        char entry_path[PATH_MAX] = {0};
-        struct stat st = {0};
-        uuid_t index = {0};
-        index_priv_t *priv = NULL;
-        int ret = -1;
-        int op_errno = 0;
-        int fd = 0;
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, op_errno, EINVAL);
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, filename, out, op_errno,
-                                       EINVAL);
-
-        priv = this->private;
-        make_gfid_path (priv->index_basepath, ENTRY_CHANGES_SUBDIR, gfid,
-                        pgfid_path, sizeof (pgfid_path));
-
-        ret = stat (pgfid_path, &st);
-        if (ret != 0) {
-                op_errno = errno;
-                if (op_errno != ENOENT)
-                        goto out;
-                ret = sys_mkdir (pgfid_path, 0600);
-                if (ret != 0 && errno != EEXIST) {
-                        op_errno = errno;
-                        goto out;
-                }
-        }
-
-        snprintf (entry_path, sizeof(entry_path), "%s/%s", pgfid_path,
-                  filename);
-        index_get_index (priv, index);
-        make_index_path (priv->index_basepath, ENTRY_CHANGES_SUBDIR, index,
-                         entry_base_index_path, sizeof(entry_base_index_path));
-        ret = index_link_to_base (this, entry_base_index_path,
-                                  sizeof (entry_base_index_path),
-                                  entry_path, ENTRY_CHANGES_SUBDIR);
+index_entry_create(xlator_t *this, inode_t *inode, char *filename)
+{
+    int ret = -1;
+    int op_errno = 0;
+    char pgfid_path[PATH_MAX] = {0};
+    char entry_path[PATH_MAX] = {0};
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ctx = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(inode->gfid),
+                                  out, op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, filename, out, op_errno, EINVAL);
+
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Not able to get inode ctx for %s", uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    make_gfid_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR, inode->gfid,
+                   pgfid_path, sizeof(pgfid_path));
+
+    if (ctx->state[ENTRY_CHANGES] != IN) {
+        ret = sys_mkdir(pgfid_path, 0600);
+        if (ret != 0 && errno != EEXIST) {
+            op_errno = errno;
+            goto out;
+        }
+        ctx->state[ENTRY_CHANGES] = IN;
+    }
+
+    if (strchr(filename, '/')) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_INDEX_ADD_FAILED,
+               "Got invalid entry (%s) for pargfid path (%s)", filename,
+               pgfid_path);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    len = snprintf(entry_path, sizeof(entry_path), "%s/%s", pgfid_path,
+                   filename);
+    if ((len < 0) || (len >= sizeof(entry_path))) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    op_errno = 0;
+
+    ret = index_link_to_base(this, entry_path, ENTRY_CHANGES_SUBDIR);
 out:
-        return ret;
+    if (op_errno)
+        ret = -op_errno;
+    return ret;
 }
 
 int
-index_entry_delete (xlator_t *this, uuid_t pgfid, char *filename)
-{
-        char entry_base_index_path[PATH_MAX] = {0};
-        char pgfid_path[PATH_MAX] = {0};
-        char entry_path[PATH_MAX] = {0};
-        index_priv_t *priv = NULL;
-        int op_errno = 0;
-        int ret = 0;
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (pgfid),
-                                       out, op_errno, EINVAL);
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, filename, out, op_errno,
-                                       EINVAL);
-        priv = this->private;
-        make_gfid_path (priv->index_basepath, ENTRY_CHANGES_SUBDIR, pgfid,
-                        pgfid_path, sizeof (pgfid_path));
-        snprintf (entry_path, sizeof(entry_path), "%s/%s", pgfid_path,
-                  filename);
-        ret = sys_unlink (entry_path);
-        if (ret && (errno != ENOENT)) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to delete from index/entry-changes (%s)",
-                        entry_path, strerror (op_errno));
-        }
+index_entry_delete(xlator_t *this, uuid_t pgfid, char *filename)
+{
+    int ret = 0;
+    int op_errno = 0;
+    char pgfid_path[PATH_MAX] = {0};
+    char entry_path[PATH_MAX] = {0};
+    index_priv_t *priv = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(pgfid), out,
+                                  op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, filename, out, op_errno, EINVAL);
+
+    make_gfid_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR, pgfid,
+                   pgfid_path, sizeof(pgfid_path));
+
+    if (strchr(filename, '/')) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_INDEX_DEL_FAILED,
+               "Got invalid entry (%s) for pargfid path (%s)", filename,
+               pgfid_path);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    len = snprintf(entry_path, sizeof(entry_path), "%s/%s", pgfid_path,
+                   filename);
+    if ((len < 0) || (len >= sizeof(entry_path))) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = sys_unlink(entry_path);
+    if (ret && (errno != ENOENT)) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, INDEX_MSG_INDEX_DEL_FAILED,
+               "%s: failed to delete from index/entry-changes", entry_path);
+    }
 
 out:
-        return -op_errno;
-
+    return -op_errno;
 }
 
 int
-index_entry_action (xlator_t *this, inode_t *inode, dict_t *xdata)
+index_entry_action(xlator_t *this, inode_t *inode, dict_t *xdata, char *key)
 {
-        char *filename = NULL;
-        char *pargfid = NULL;
-        int ret = 0;
-
-        ret = dict_get_str (xdata, GF_XATTROP_ENTRY_IN_KEY, &filename);
-        if (ret == 0) {
-                ret = index_entry_create (this, inode->gfid, filename);
-                goto out;
-        }
+    int ret = 0;
+    char *filename = NULL;
 
-        ret = dict_get_str (xdata, GF_XATTROP_ENTRY_OUT_KEY, &filename);
-        if (ret == 0) {
-                ret = index_entry_delete (this, inode->gfid, filename);
-                goto out;
-        }
+    ret = dict_get_str(xdata, key, &filename);
+    if (ret != 0) {
         ret = 0;
+        goto out;
+    }
+
+    if (strcmp(key, GF_XATTROP_ENTRY_IN_KEY) == 0)
+        ret = index_entry_create(this, inode, filename);
+    else if (strcmp(key, GF_XATTROP_ENTRY_OUT_KEY) == 0)
+        ret = index_entry_delete(this, inode->gfid, filename);
+
 out:
-        return ret;
+    return ret;
 }
 
 void
-_index_action (xlator_t *this, inode_t *inode, int *zfilled)
+_index_action(xlator_t *this, inode_t *inode, int *zfilled)
+{
+    int ret = 0;
+    int i = 0;
+    index_inode_ctx_t *ctx = NULL;
+    char *subdir = NULL;
+
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Not able to get"
+               " inode context for %s.",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    for (i = 0; i < XATTROP_TYPE_END; i++) {
+        subdir = index_get_subdir_from_type(i);
+        if (zfilled[i] == 1) {
+            if (ctx->state[i] == NOTIN)
+                continue;
+            ret = index_del(this, inode->gfid, subdir, i);
+            if (!ret)
+                ctx->state[i] = NOTIN;
+        } else if (zfilled[i] == 0) {
+            if (ctx->state[i] == IN)
+                continue;
+            ret = index_add(this, inode->gfid, subdir, i);
+            if (!ret)
+                ctx->state[i] = IN;
+        }
+    }
+out:
+    return;
+}
+
+static void
+index_init_state(xlator_t *this, inode_t *inode, index_inode_ctx_t *ctx,
+                 char *subdir)
 {
-        int               ret  = 0;
-        int               i    = 0;
-        index_inode_ctx_t *ctx = NULL;
-        char           *subdir = NULL;
+    int ret = -1;
+    char pgfid_path[PATH_MAX] = {0};
+    struct stat st = {0};
+    index_priv_t *priv = NULL;
 
-        ret = index_inode_ctx_get (inode, this, &ctx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Not able to get"
-                        " inode context for %s.", uuid_utoa (inode->gfid));
-                goto out;
-        }
+    priv = this->private;
 
-        for (i = 0; i < XATTROP_TYPE_END; i++) {
-                subdir = index_get_subdir_from_type (i);
-                if (zfilled[i] == 1) {
-                        if (ctx->state[i] == NOTIN)
-                                continue;
-                        ret = index_del (this, inode->gfid, subdir, i);
-                        if (!ret)
-                                ctx->state[i] = NOTIN;
-                } else if (zfilled[i] == 0){
-                        if (ctx->state[i] == IN)
-                                continue;
-                        ret = index_add (this, inode->gfid, subdir, i);
-                        if (!ret)
-                                ctx->state[i] = IN;
-                }
-        }
-out:
-        return;
+    make_gfid_path(priv->index_basepath, subdir, inode->gfid, pgfid_path,
+                   sizeof(pgfid_path));
+
+    ret = sys_stat(pgfid_path, &st);
+    if (ret == 0)
+        ctx->state[ENTRY_CHANGES] = IN;
+    else if (ret != 0 && errno == ENOENT)
+        ctx->state[ENTRY_CHANGES] = NOTIN;
+
+    return;
 }
 
 void
-xattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr,
-                      dict_match_t match, void *match_data)
-{
-        gf_boolean_t      zero_xattr = _gf_true;
-        int               ret = 0, i = 0;
-        int               zfilled[XATTROP_TYPE_END] = {0,};
+xattrop_index_action(xlator_t *this, index_local_t *local, dict_t *xattr,
+                     dict_match_t match, void *match_data)
+{
+    int ret = 0;
+    int zfilled[XATTROP_TYPE_END] = {
+        0,
+    };
+    int8_t value = 0;
+    char *subdir = NULL;
+    dict_t *req_xdata = NULL;
+    inode_t *inode = NULL;
+    index_inode_ctx_t *ctx = NULL;
+
+    inode = local->inode;
+    req_xdata = local->xdata;
+
+    memset(zfilled, -1, sizeof(zfilled));
+    ret = dict_foreach_match(xattr, match, match_data,
+                             _check_key_is_zero_filled, zfilled);
+    _index_action(this, inode, zfilled);
+
+    if (req_xdata) {
+        ret = index_entry_action(this, inode, req_xdata,
+                                 GF_XATTROP_ENTRY_OUT_KEY);
+
+        ret = dict_get_int8(req_xdata, GF_XATTROP_PURGE_INDEX, &value);
+        if ((ret) || (value == 0))
+            goto out;
+    }
+
+    if (zfilled[XATTROP] != 1)
+        goto out;
+
+    if (inode->ia_type != IA_IFDIR)
+        goto out;
+
+    subdir = index_get_subdir_from_type(ENTRY_CHANGES);
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if (ctx->state[ENTRY_CHANGES] == UNKNOWN)
+        index_init_state(this, inode, ctx, subdir);
+    if (ctx->state[ENTRY_CHANGES] == IN) {
+        ret = index_del(this, inode->gfid, subdir, ENTRY_CHANGES);
+        ctx->state[ENTRY_CHANGES] = NOTIN;
+    }
 
-        memset (zfilled, -1, sizeof (zfilled));
-        ret = dict_foreach_match (xattr, match, match_data,
-                                  _check_key_is_zero_filled, zfilled);
-        _index_action (this, inode, zfilled);
-        return;
+out:
+    return;
 }
 
 static gf_boolean_t
-index_xattrop_track (xlator_t *this, gf_xattrop_flags_t flags, dict_t *dict)
+index_xattrop_track(xlator_t *this, gf_xattrop_flags_t flags, dict_t *dict)
 {
-        index_priv_t *priv = this->private;
+    index_priv_t *priv = this->private;
 
-        if (flags == GF_XATTROP_ADD_ARRAY)
-                return _gf_true;
+    if (flags == GF_XATTROP_ADD_ARRAY)
+        return _gf_true;
 
-        if (flags != GF_XATTROP_ADD_ARRAY64)
-                return _gf_false;
+    if (flags != GF_XATTROP_ADD_ARRAY64)
+        return _gf_false;
 
-        if (!priv->pending_watchlist)
-                return _gf_false;
+    if (!priv->pending_watchlist)
+        return _gf_false;
 
-        if (dict_foreach_match (dict, is_xattr_in_watchlist,
-                                priv->pending_watchlist, dict_null_foreach_fn,
-                                NULL) > 0)
-                return _gf_true;
+    if (dict_foreach_match(dict, is_xattr_in_watchlist, priv->pending_watchlist,
+                           dict_null_foreach_fn, NULL) > 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 int
-index_inode_path (xlator_t *this, inode_t *inode, char *dirpath, size_t len)
-{
-        char              *subdir = NULL;
-        int               ret     = 0;
-        index_priv_t      *priv   = NULL;
-        index_inode_ctx_t *ictx   = NULL;
-
-        priv = this->private;
-        if (!index_is_fop_on_internal_inode (this, inode, NULL)) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        subdir = index_get_subdir_from_vgfid (priv, inode->gfid);
-        if (subdir) {
-                if (len <= strlen (priv->index_basepath) + 1 /*'/'*/ +
-                           strlen (subdir)) {
-                        ret = -EINVAL;
-                        goto out;
-                }
-                make_index_dir_path (priv->index_basepath, subdir,
-                                     dirpath, len);
-        } else {
-                ret = index_inode_ctx_get (inode, this, &ictx);
-                if (ret)
-                        goto out;
-                if (gf_uuid_is_null (ictx->virtual_pargfid)) {
-                        ret = -EINVAL;
-                        goto out;
-                }
-                make_index_dir_path (priv->index_basepath, ENTRY_CHANGES_SUBDIR,
-                                     dirpath, len);
-                if (len <= strlen (dirpath) + 1 /*'/'*/ + strlen (UUID0_STR)) {
-                        ret = -EINVAL;
-                        goto out;
-                }
-                strcat (dirpath, "/");
-                strcat (dirpath, uuid_utoa (ictx->virtual_pargfid));
-        }
+index_inode_path(xlator_t *this, inode_t *inode, char *dirpath, size_t len)
+{
+    char *subdir = NULL;
+    int ret = 0;
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ictx = NULL;
+
+    priv = this->private;
+    if (!index_is_fop_on_internal_inode(this, inode, NULL)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    subdir = index_get_subdir_from_vgfid(priv, inode->gfid);
+    if (subdir) {
+        if (len <= strlen(priv->index_basepath) + 1 /*'/'*/ + strlen(subdir)) {
+            ret = -EINVAL;
+            goto out;
+        }
+        make_index_dir_path(priv->index_basepath, subdir, dirpath, len);
+    } else {
+        ret = index_inode_ctx_get(inode, this, &ictx);
+        if (ret)
+            goto out;
+        if (gf_uuid_is_null(ictx->virtual_pargfid)) {
+            ret = -EINVAL;
+            goto out;
+        }
+        make_index_dir_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR, dirpath,
+                            len);
+        if (len <= strlen(dirpath) + 1 /*'/'*/ + SLEN(UUID0_STR)) {
+            ret = -EINVAL;
+            goto out;
+        }
+        strcat(dirpath, "/");
+        strcat(dirpath, uuid_utoa(ictx->virtual_pargfid));
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-__index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+__index_fd_ctx_get(fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
 {
-        int               ret = 0;
-        index_fd_ctx_t    *fctx = NULL;
-        index_inode_ctx_t *ictx = NULL;
-        uint64_t          tmpctx = 0;
-        char              dirpath[PATH_MAX] = {0};
-        index_priv_t      *priv = NULL;
-
-        priv = this->private;
-
-        ret = __fd_ctx_get (fd, this, &tmpctx);
-        if (!ret) {
-                fctx = (index_fd_ctx_t*) (long) tmpctx;
-                *ctx = fctx;
-                goto out;
-        }
-
-        ret = index_inode_path (this, fd->inode, dirpath, sizeof (dirpath));
-        if (ret)
-                goto out;
+    int ret = 0;
+    index_fd_ctx_t *fctx = NULL;
+    uint64_t tmpctx = 0;
+    char dirpath[PATH_MAX] = {0};
 
-        fctx = GF_CALLOC (1, sizeof (*fctx), gf_index_fd_ctx_t);
-        if (!fctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        fctx->dir = sys_opendir (dirpath);
-        if (!fctx->dir) {
-                ret = -errno;
-                GF_FREE (fctx);
-                fctx = NULL;
-                goto out;
-        }
-        fctx->dir_eof = -1;
-
-        ret = __fd_ctx_set (fd, this, (uint64_t)(long)fctx);
-        if (ret) {
-                sys_closedir (fctx->dir);
-                GF_FREE (fctx);
-                fctx = NULL;
-                ret = -EINVAL;
-                goto out;
-        }
+    ret = __fd_ctx_get(fd, this, &tmpctx);
+    if (!ret) {
+        fctx = (index_fd_ctx_t *)(long)tmpctx;
         *ctx = fctx;
+        goto out;
+    }
+
+    ret = index_inode_path(this, fd->inode, dirpath, sizeof(dirpath));
+    if (ret)
+        goto out;
+
+    fctx = GF_CALLOC(1, sizeof(*fctx), gf_index_fd_ctx_t);
+    if (!fctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    fctx->dir = sys_opendir(dirpath);
+    if (!fctx->dir) {
+        ret = -errno;
+        GF_FREE(fctx);
+        fctx = NULL;
+        goto out;
+    }
+    fctx->dir_eof = -1;
+
+    ret = __fd_ctx_set(fd, this, (uint64_t)(long)fctx);
+    if (ret) {
+        (void)sys_closedir(fctx->dir);
+        GF_FREE(fctx);
+        fctx = NULL;
+        ret = -EINVAL;
+        goto out;
+    }
+    *ctx = fctx;
 out:
-        return ret;
+    return ret;
 }
 
 int
-index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+index_fd_ctx_get(fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
 {
-        int     ret = 0;
-        LOCK (&fd->lock);
-        {
-                ret = __index_fd_ctx_get (fd, this, ctx);
-        }
-        UNLOCK (&fd->lock);
-        return ret;
+    int ret = 0;
+    LOCK(&fd->lock);
+    {
+        ret = __index_fd_ctx_get(fd, this, ctx);
+    }
+    UNLOCK(&fd->lock);
+    return ret;
 }
 
-//new - Not NULL means start a fop
-//new - NULL means done processing the fop
+// new - Not NULL means start a fop
+// new - NULL means done processing the fop
 void
-index_queue_process (xlator_t *this, inode_t *inode, call_stub_t *new)
+index_queue_process(xlator_t *this, inode_t *inode, call_stub_t *new)
 {
-        call_stub_t       *stub = NULL;
-        index_inode_ctx_t *ctx = NULL;
-        int               ret = 0;
-        call_frame_t      *frame = NULL;
+    call_stub_t *stub = NULL;
+    index_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ret = __index_inode_ctx_get (inode, this, &ctx);
-                if (ret)
-                        goto unlock;
-
-                if (new) {
-                        __index_enqueue (&ctx->callstubs, new);
-                        new = NULL;
-                } else {
-                        ctx->processing = _gf_false;
-                }
+    LOCK(&inode->lock);
+    {
+        ret = __index_inode_ctx_get(inode, this, &ctx);
+        if (ret)
+            goto unlock;
 
-                if (!ctx->processing) {
-                        stub = __index_dequeue (&ctx->callstubs);
-                        if (stub)
-                                ctx->processing = _gf_true;
-                        else
-                                ctx->processing = _gf_false;
-                }
+        if (new) {
+            __index_enqueue(&ctx->callstubs, new);
+            new = NULL;
+        } else {
+            ctx->processing = _gf_false;
+        }
+
+        if (!ctx->processing) {
+            stub = __index_dequeue(&ctx->callstubs);
+            if (stub)
+                ctx->processing = _gf_true;
+            else
+                ctx->processing = _gf_false;
         }
+    }
 unlock:
-        UNLOCK (&inode->lock);
-
-        if (ret && new) {
-                frame = new->frame;
-                if (new->fop == GF_FOP_XATTROP) {
-                        INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM,
-                                            NULL, NULL);
-                } else if (new->fop == GF_FOP_FXATTROP) {
-                        INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM,
-                                            NULL, NULL);
-                }
-                call_stub_destroy (new);
-        } else if (stub) {
-                call_resume (stub);
+    UNLOCK(&inode->lock);
+
+    if (ret && new) {
+        frame = new->frame;
+        if (new->fop == GF_FOP_XATTROP) {
+            INDEX_STACK_UNWIND(xattrop, frame, -1, ENOMEM, NULL, NULL);
+        } else if (new->fop == GF_FOP_FXATTROP) {
+            INDEX_STACK_UNWIND(fxattrop, frame, -1, ENOMEM, NULL, NULL);
         }
-        return;
+        call_stub_destroy(new);
+    } else if (stub) {
+        call_resume(stub);
+    }
+    return;
 }
 
 static int
-xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, dict_t *xattr,
-             dict_t *xdata, dict_match_t match, dict_t *matchdata)
+xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xattr, dict_t *xdata, dict_match_t match,
+            dict_t *matchdata)
 {
-        inode_t *inode = NULL;
+    inode_t *inode = NULL;
+    index_local_t *local = NULL;
 
-        inode = inode_ref (frame->local);
-        if (op_ret < 0)
-                goto out;
+    local = frame->local;
+    inode = inode_ref(local->inode);
+
+    if (op_ret < 0)
+        goto out;
 
-        xattrop_index_action (this, frame->local, xattr, match, matchdata);
+    xattrop_index_action(this, local, xattr, match, matchdata);
 out:
-        INDEX_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
-        index_queue_process (this, inode, NULL);
-        inode_unref (inode);
+    INDEX_STACK_UNWIND(xattrop, frame, op_ret, op_errno, xattr, xdata);
+    index_queue_process(this, inode, NULL);
+    inode_unref(inode);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-index_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                   dict_t *xdata)
+index_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                  dict_t *xdata)
 {
-        index_priv_t *priv = this->private;
+    index_priv_t *priv = this->private;
 
-        xattrop_cbk (frame, cookie, this, op_ret, op_errno,
-                     xattr, xdata, is_xattr_in_watchlist,
-                     priv->complete_watchlist);
-        return 0;
+    xattrop_cbk(frame, cookie, this, op_ret, op_errno, xattr, xdata,
+                is_xattr_in_watchlist, priv->complete_watchlist);
+    return 0;
 }
 
 int32_t
-index_xattrop64_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                     dict_t *xdata)
+index_xattrop64_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                    dict_t *xdata)
 {
-        index_priv_t *priv = this->private;
+    index_priv_t *priv = this->private;
 
-        return xattrop_cbk (frame, cookie, this, op_ret, op_errno, xattr, xdata,
-                            is_xattr_in_watchlist, priv->pending_watchlist);
+    return xattrop_cbk(frame, cookie, this, op_ret, op_errno, xattr, xdata,
+                       is_xattr_in_watchlist, priv->pending_watchlist);
 }
 
 void
-index_xattrop_do (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  fd_t *fd, gf_xattrop_flags_t optype, dict_t *xattr,
-                  dict_t *xdata)
-{
-        fop_xattrop_cbk_t x_cbk = NULL;
-        int     zfilled[XATTROP_TYPE_END] = {0,};
-        int     ret = -1, i = 0;
-        index_priv_t *priv = this->private;
-
-        if (optype == GF_XATTROP_ADD_ARRAY)
-                x_cbk = index_xattrop_cbk;
-        else
-                x_cbk = index_xattrop64_cbk;
-
-        //In wind phase bring the gfid into index. This way if the brick crashes
-        //just after posix performs xattrop before _cbk reaches index xlator
-        //we will still have the gfid in index.
-        memset (zfilled, -1, sizeof (zfilled));
-
-        /* Foreach xattr, set corresponding index of zfilled to 1
-         * zfilled[index] = 1 implies the xattr's value is zero filled
-         * and should be added in its corresponding subdir.
-         *
-         * zfilled should be set to 1 only for those index that
-         * exist in xattr variable. This is to distinguish
-         * between different types of volumes.
-         * For e.g., if the check is not made,
-         * zfilled[DIRTY] is set to 1 for EC volumes,
-         * index file will be tried to create in indices/dirty dir
-         * which doesn't exist for an EC volume.
-         */
-        ret = dict_foreach (xattr, index_fill_zero_array, zfilled);
-
-        _index_action (this, frame->local, zfilled);
-        if (xdata)
-                ret = index_entry_action (this, frame->local, xdata);
-        if (ret < 0) {
-                x_cbk (frame, NULL, this, -1, -ret, NULL, NULL);
-                return;
-        }
+index_xattrop_do(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                 gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    int ret = -1;
+    int zfilled[XATTROP_TYPE_END] = {
+        0,
+    };
+    index_local_t *local = NULL;
+    fop_xattrop_cbk_t x_cbk = NULL;
+
+    local = frame->local;
+
+    if (optype == GF_XATTROP_ADD_ARRAY)
+        x_cbk = index_xattrop_cbk;
+    else
+        x_cbk = index_xattrop64_cbk;
+
+    // In wind phase bring the gfid into index. This way if the brick crashes
+    // just after posix performs xattrop before _cbk reaches index xlator
+    // we will still have the gfid in index.
+    memset(zfilled, -1, sizeof(zfilled));
+
+    /* Foreach xattr, set corresponding index of zfilled to 1
+     * zfilled[index] = 1 implies the xattr's value is zero filled
+     * and should be added in its corresponding subdir.
+     *
+     * zfilled should be set to 1 only for those index that
+     * exist in xattr variable. This is to distinguish
+     * between different types of volumes.
+     * For e.g., if the check is not made,
+     * zfilled[DIRTY] is set to 1 for EC volumes,
+     * index file will be tried to create in indices/dirty dir
+     * which doesn't exist for an EC volume.
+     */
+    ret = dict_foreach(xattr, index_fill_zero_array, zfilled);
+
+    _index_action(this, local->inode, zfilled);
+    if (xdata)
+        ret = index_entry_action(this, local->inode, xdata,
+                                 GF_XATTROP_ENTRY_IN_KEY);
+    if (ret < 0) {
+        x_cbk(frame, NULL, this, -1, -ret, NULL, NULL);
+        return;
+    }
 
-        if (loc)
-                STACK_WIND (frame, x_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->xattrop,
-                            loc, optype, xattr, xdata);
-        else
-                STACK_WIND (frame, x_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fxattrop, fd,
-                            optype, xattr, xdata);
+    if (loc)
+        STACK_WIND(frame, x_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    else
+        STACK_WIND(frame, x_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
 }
 
 int
-index_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+index_xattrop_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        index_xattrop_do (frame, this, loc, NULL, optype, xattr, xdata);
-        return 0;
+    index_xattrop_do(frame, this, loc, NULL, optype, xattr, xdata);
+    return 0;
 }
 
 int
-index_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+index_fxattrop_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        index_xattrop_do (frame, this, NULL, fd, optype, xattr, xdata);
-        return 0;
+    index_xattrop_do(frame, this, NULL, fd, optype, xattr, xdata);
+    return 0;
 }
 
 int32_t
-index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+index_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
+    call_stub_t *stub = NULL;
+    index_local_t *local = NULL;
 
-        if (!index_xattrop_track (this, flags, dict))
-                goto out;
+    if (!index_xattrop_track(this, flags, dict))
+        goto out;
 
-        frame->local = inode_ref (loc->inode);
-        stub = fop_xattrop_stub (frame, index_xattrop_wrapper,
-                                 loc, flags, dict, xdata);
-        if (!stub) {
-                INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->inode = inode_ref(loc->inode);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+    stub = fop_xattrop_stub(frame, index_xattrop_wrapper, loc, flags, dict,
+                            xdata);
 
-        index_queue_process (this, loc->inode, stub);
+err:
+    if ((!local) || (!stub)) {
+        INDEX_STACK_UNWIND(xattrop, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    index_queue_process(this, loc->inode, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, default_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+    return 0;
 }
 
 int32_t
-index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+index_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
+    call_stub_t *stub = NULL;
+    index_local_t *local = NULL;
 
-        if (!index_xattrop_track (this, flags, dict))
-                goto out;
+    if (!index_xattrop_track(this, flags, dict))
+        goto out;
 
-        frame->local = inode_ref (fd->inode);
-        stub = fop_fxattrop_stub (frame, index_fxattrop_wrapper,
-                                  fd, flags, dict, xdata);
-        if (!stub) {
-                INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM, NULL, xdata);
-                return 0;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->inode = inode_ref(fd->inode);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+    stub = fop_fxattrop_stub(frame, index_fxattrop_wrapper, fd, flags, dict,
+                             xdata);
 
-        index_queue_process (this, fd->inode, stub);
+err:
+    if ((!local) || (!stub)) {
+        INDEX_STACK_UNWIND(fxattrop, frame, -1, ENOMEM, NULL, xdata);
         return 0;
+    }
+
+    index_queue_process(this, fd->inode, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, default_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+    return 0;
 }
 
 uint64_t
-index_entry_count (xlator_t *this, char *subdir)
+index_entry_count(xlator_t *this, char *subdir)
 {
-	index_priv_t *priv = NULL;
-	char index_dir[PATH_MAX];
-	DIR *dirp = NULL;
-	uint64_t count = 0;
-	struct dirent buf;
-	struct dirent *entry = NULL;
+    uint64_t count = 0;
+    index_priv_t *priv = NULL;
+    DIR *dirp = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char index_dir[PATH_MAX] = {
+        0,
+    };
+
+    priv = this->private;
+
+    make_index_dir_path(priv->index_basepath, subdir, index_dir,
+                        sizeof(index_dir));
+
+    dirp = sys_opendir(index_dir);
+    if (!dirp)
+        return 0;
 
-	priv = this->private;
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(dirp, scratch);
+        if (!entry || errno != 0)
+            break;
 
-	make_index_dir_path (priv->index_basepath, subdir,
-			     index_dir, sizeof (index_dir));
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
 
-	dirp = sys_opendir (index_dir);
-	if (!dirp)
-		return 0;
+        if (!strncmp(entry->d_name, subdir, strlen(subdir)))
+            continue;
 
-	while (readdir_r (dirp, &buf, &entry) == 0) {
-		if (!entry)
-			break;
-		if (!strcmp (entry->d_name, ".") ||
-		    !strcmp (entry->d_name, ".."))
-			continue;
-                if (!strncmp (entry->d_name, subdir, strlen (subdir)))
-			continue;
-		count++;
-	}
-	sys_closedir (dirp);
+        count++;
+    }
 
-	return count;
+    (void)sys_closedir(dirp);
+
+    return count;
 }
 
 int32_t
-index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, const char *name, dict_t *xdata)
-{
-        index_priv_t    *priv = NULL;
-        dict_t          *xattr = NULL;
-        int             ret = 0;
-        int             vgfid_type = 0;
-	uint64_t        count = 0;
+index_getxattr_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       const char *name, dict_t *xdata)
+{
+    index_priv_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int ret = 0;
+    int vgfid_type = 0;
+    uint64_t count = 0;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto done;
+    }
+
+    vgfid_type = index_get_type_from_vgfid_xattr(name);
+    if (vgfid_type >= 0) {
+        ret = dict_set_static_bin(xattr, (char *)name,
+                                  priv->internal_vgfid[vgfid_type],
+                                  sizeof(priv->internal_vgfid[vgfid_type]));
+        if (ret) {
+            ret = -EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, INDEX_MSG_DICT_SET_FAILED,
+                   "xattrop index "
+                   "gfid set failed");
+            goto done;
+        }
+    }
 
-        priv = this->private;
+    /* TODO: Need to check what kind of link-counts are needed for
+     * ENTRY-CHANGES before refactor of this block with array*/
+    if (strcmp(name, GF_XATTROP_INDEX_COUNT) == 0) {
+        count = index_entry_count(this, XATTROP_SUBDIR);
 
-        xattr = dict_new ();
-        if (!xattr) {
-                ret = -ENOMEM;
-                goto done;
+        ret = dict_set_uint64(xattr, (char *)name, count);
+        if (ret) {
+            ret = -EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, INDEX_MSG_DICT_SET_FAILED,
+                   "xattrop index "
+                   "count set failed");
+            goto done;
         }
+    } else if (strcmp(name, GF_XATTROP_DIRTY_COUNT) == 0) {
+        count = index_entry_count(this, DIRTY_SUBDIR);
 
-        vgfid_type = index_get_type_from_vgfid_xattr (name);
-        if (vgfid_type >= 0) {
-		ret = dict_set_static_bin (xattr, (char *)name,
-                                           priv->internal_vgfid[vgfid_type],
-				   sizeof (priv->internal_vgfid[vgfid_type]));
-		if (ret) {
-			ret = -ENOMEM;
-			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
-				"gfid set failed");
-			goto done;
-		}
+        ret = dict_set_uint64(xattr, (char *)name, count);
+        if (ret) {
+            ret = -EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, INDEX_MSG_DICT_SET_FAILED,
+                   "dirty index "
+                   "count set failed");
+            goto done;
         }
-
-        /* TODO: Need to check what kind of link-counts are needed for
-         * ENTRY-CHANGES before refactor of this block with array*/
-        if (strcmp (name, GF_XATTROP_INDEX_COUNT) == 0) {
-		count = index_entry_count (this, XATTROP_SUBDIR);
-
-		ret = dict_set_uint64 (xattr, (char *)name, count);
-		if (ret) {
-			ret = -ENOMEM;
-			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
-				"count set failed");
-			goto done;
-		}
-        } else if (strcmp (name, GF_XATTROP_DIRTY_COUNT) == 0) {
-		count = index_entry_count (this, DIRTY_SUBDIR);
-
-		ret = dict_set_uint64 (xattr, (char *)name, count);
-		if (ret) {
-			ret = -ENOMEM;
-			gf_log (this->name, GF_LOG_ERROR, "dirty index "
-				"count set failed");
-			goto done;
-		}
-	}
+    }
 done:
-        if (ret)
-                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr, NULL);
-        else
-                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, xattr, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(getxattr, frame, -1, -ret, xattr, NULL);
+    else
+        STACK_UNWIND_STRICT(getxattr, frame, 0, 0, xattr, NULL);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
+    return 0;
 }
 
 static int
-index_save_pargfid_for_entry_changes (xlator_t *this, loc_t *loc, char *path)
+index_save_pargfid_for_entry_changes(xlator_t *this, loc_t *loc, char *path)
 {
-        index_priv_t      *priv = NULL;
-        index_inode_ctx_t *ctx  = NULL;
-        int               ret   = 0;
-
-        priv = this->private;
-        if (gf_uuid_compare (loc->pargfid,
-                             priv->internal_vgfid[ENTRY_CHANGES]))
-                return 0;
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ctx = NULL;
+    int ret = 0;
 
-        ret = index_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get inode "
-                        "context for %s", path);
-                return -EINVAL;
-        }
-        ret = gf_uuid_parse (loc->name, ctx->virtual_pargfid);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to store "
-                        "virtual gfid in inode context for %s", path);
-                return -EINVAL;
-        }
+    priv = this->private;
+    if (!loc)
+        return -1;
+    if (gf_uuid_compare(loc->pargfid, priv->internal_vgfid[ENTRY_CHANGES]))
         return 0;
+
+    ret = index_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Unable to get inode context for %s", path);
+        return -EINVAL;
+    }
+    ret = gf_uuid_parse(loc->name, ctx->virtual_pargfid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Unable to store "
+               "virtual gfid in inode context for %s",
+               path);
+        return -EINVAL;
+    }
+    return 0;
 }
 
 int32_t
-index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, dict_t *xattr_req)
-{
-        index_priv_t    *priv = NULL;
-        struct stat     lstatbuf = {0};
-        int             ret = 0;
-        int32_t         op_errno = EINVAL;
-        int32_t         op_ret = -1;
-        char            path[PATH_MAX] = {0};
-        struct iatt     stbuf        = {0, };
-        struct iatt     postparent = {0,};
-        dict_t          *xattr = NULL;
-        index_inode_ctx_t *ctx = NULL;
-        gf_boolean_t    is_dir = _gf_false;
-        char            *subdir = NULL;
-        loc_t           iloc = {0};
-
-        priv = this->private;
-        loc_copy (&iloc, loc);
-
-        VALIDATE_OR_GOTO (loc, done);
-        if (index_is_fop_on_internal_inode (this, loc->parent, loc->pargfid)) {
-                subdir = index_get_subdir_from_vgfid (priv, loc->pargfid);
-                ret = index_inode_path (this, loc->parent, path, sizeof (path));
-                if (ret < 0) {
-                        op_errno = -ret;
-                        goto done;
-                }
-                strcat (path, "/");
-                strcat (path, (char *)loc->name);
-        } else if (index_is_virtual_gfid (priv, loc->gfid)) {
-                subdir = index_get_subdir_from_vgfid (priv, loc->gfid);
-                make_index_dir_path (priv->index_basepath, subdir,
-                                     path, sizeof (path));
-                is_dir = _gf_true;
-        } else {
-                if (!inode_is_linked (loc->inode)) {
-                        inode_unref (iloc.inode);
-                        iloc.inode = inode_find (loc->inode->table, loc->gfid);
-                }
-                ret = index_inode_path (this, iloc.inode, path,
-                                        sizeof (path));
-                if (ret < 0) {
-                        op_errno = -ret;
-                        goto done;
-                }
-        }
-        ret = sys_lstat (path, &lstatbuf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on %s dir "
-                        "(%s)", subdir, strerror (errno));
-                op_errno = errno;
-                goto done;
-        } else if (!S_ISDIR (lstatbuf.st_mode) && is_dir) {
-                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on %s dir, "
-                        "not a directory", subdir);
-                op_errno = ENOENT;
-                goto done;
-        }
-        xattr = dict_new ();
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto done;
+index_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xattr_req)
+{
+    index_priv_t *priv = NULL;
+    struct stat lstatbuf = {0};
+    int ret = 0;
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    uint64_t val = IA_INVAL;
+    char path[PATH_MAX] = {0};
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    gf_boolean_t is_dir = _gf_false;
+    char *subdir = NULL;
+    loc_t iloc = {0};
+
+    priv = this->private;
+    loc_copy(&iloc, loc);
+
+    VALIDATE_OR_GOTO(loc, done);
+    if (index_is_fop_on_internal_inode(this, loc->parent, loc->pargfid)) {
+        subdir = index_get_subdir_from_vgfid(priv, loc->pargfid);
+        ret = index_inode_path(this, loc->parent, path, sizeof(path));
+        if (ret < 0) {
+            op_errno = -ret;
+            goto done;
         }
+        ret = snprintf(path + strlen(path), PATH_MAX - strlen(path), "/%s",
+                       loc->name);
 
-        iatt_from_stat (&stbuf, &lstatbuf);
-        if (is_dir || inode_is_linked (iloc.inode))
-                loc_gfid (&iloc, stbuf.ia_gfid);
-        else
-                gf_uuid_generate (stbuf.ia_gfid);
+        if ((ret < 0) || (ret > (PATH_MAX - strlen(path)))) {
+            op_errno = EINVAL;
+            op_ret = -1;
+            goto done;
+        }
 
-       ret =  index_save_pargfid_for_entry_changes (this, &iloc, path);
-       if (ret) {
-               op_ret = -1;
-               op_errno = -ret;
-               goto done;
-       }
+    } else if (index_is_virtual_gfid(priv, loc->gfid)) {
+        subdir = index_get_subdir_from_vgfid(priv, loc->gfid);
+        make_index_dir_path(priv->index_basepath, subdir, path, sizeof(path));
+        is_dir = _gf_true;
 
-        stbuf.ia_ino = -1;
-        op_ret = 0;
+        if ((xattr_req) && (dict_get(xattr_req, GF_INDEX_IA_TYPE_GET_REQ))) {
+            if (0 == strcmp(subdir, index_get_subdir_from_type(ENTRY_CHANGES)))
+                val = IA_IFDIR;
+            else
+                val = IA_IFREG;
+        }
+    } else {
+        if (!inode_is_linked(loc->inode)) {
+            inode_unref(iloc.inode);
+            iloc.inode = inode_find(loc->inode->table, loc->gfid);
+        }
+        ret = index_inode_path(this, iloc.inode, path, sizeof(path));
+        if (ret < 0) {
+            op_errno = -ret;
+            goto done;
+        }
+    }
+    ret = sys_lstat(path, &lstatbuf);
+    if (ret) {
+        gf_msg_debug(this->name, errno, "Stat failed on %s dir ", path);
+        op_errno = errno;
+        goto done;
+    } else if (!S_ISDIR(lstatbuf.st_mode) && is_dir) {
+        op_errno = ENOTDIR;
+        gf_msg_debug(this->name, op_errno,
+                     "Stat failed on %s dir, "
+                     "not a directory",
+                     path);
+        goto done;
+    }
+    xattr = dict_new();
+    if (!xattr) {
+        op_errno = ENOMEM;
+        goto done;
+    }
+
+    if (val != IA_INVAL) {
+        ret = dict_set_uint64(xattr, GF_INDEX_IA_TYPE_GET_RSP, val);
+        if (ret) {
+            op_ret = -1;
+            op_errno = -ret;
+            goto done;
+        }
+    }
+
+    iatt_from_stat(&stbuf, &lstatbuf);
+    if (is_dir || inode_is_linked(iloc.inode))
+        loc_gfid(&iloc, stbuf.ia_gfid);
+    else
+        gf_uuid_generate(stbuf.ia_gfid);
+
+    ret = index_save_pargfid_for_entry_changes(this, &iloc, path);
+    if (ret) {
+        op_ret = -1;
+        op_errno = -ret;
+        goto done;
+    }
+
+    stbuf.ia_ino = -1;
+    op_ret = 0;
 done:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
-                             loc->inode, &stbuf, xattr, &postparent);
-        if (xattr)
-                dict_unref (xattr);
-        loc_wipe (&iloc);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno,
+                        loc ? loc->inode : NULL, &stbuf, xattr, &postparent);
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&iloc);
+    return 0;
 }
 
 int
-index_get_gfid_type (void *opaque)
-{
-        gf_dirent_t              *entry = NULL;
-        inode_t                  *inode = NULL;
-        xlator_t                 *this  = THIS;
-        struct index_syncop_args *args  = opaque;
-        loc_t                    loc    = {0};
-        struct iatt              iatt   = {0};
-        int                      ret    = 0;
-
-        list_for_each_entry (entry, &args->entries->list, list) {
-                if (strcmp (entry->d_name, ".") == 0 ||
-                    strcmp (entry->d_name, "..") == 0)
-                        continue;
-
-                loc_wipe (&loc);
-
-                entry->d_type = IA_INVAL;
-                if (gf_uuid_parse (entry->d_name, loc.gfid))
-                        continue;
-
-                loc.inode = inode_find (args->parent->table, loc.gfid);
-                if (loc.inode) {
-                        entry->d_type = loc.inode->ia_type;
-                        continue;
-                }
-                loc.inode = inode_new (args->parent->table);
-                if (!loc.inode)
-                        continue;
-                ret = syncop_lookup (FIRST_CHILD (this), &loc, &iatt, 0, 0, 0);
-                if (ret == 0)
-                        entry->d_type = iatt.ia_type;
+index_get_gfid_type(void *opaque)
+{
+    gf_dirent_t *entry = NULL;
+    xlator_t *this = THIS;
+    struct index_syncop_args *args = opaque;
+    loc_t loc = {0};
+    struct iatt iatt = {0};
+    int ret = 0;
+
+    list_for_each_entry(entry, &args->entries->list, list)
+    {
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
+
+        loc_wipe(&loc);
+
+        entry->d_type = gf_d_type_from_ia_type(IA_INVAL);
+        entry->d_stat.ia_type = IA_INVAL;
+        if (gf_uuid_parse(entry->d_name, loc.gfid))
+            continue;
+
+        loc.inode = inode_find(args->parent->table, loc.gfid);
+        if (loc.inode) {
+            entry->d_stat.ia_type = loc.inode->ia_type;
+            entry->d_type = gf_d_type_from_ia_type(loc.inode->ia_type);
+            continue;
+        }
+        loc.inode = inode_new(args->parent->table);
+        if (!loc.inode)
+            continue;
+        ret = syncop_lookup(FIRST_CHILD(this), &loc, &iatt, 0, 0, 0);
+        if (ret == 0) {
+            entry->d_type = gf_d_type_from_ia_type(iatt.ia_type);
+            entry->d_stat = iatt;
         }
-        loc_wipe (&loc);
+    }
+    loc_wipe(&loc);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        index_fd_ctx_t       *fctx           = NULL;
-        index_priv_t         *priv           = NULL;
-        DIR                  *dir            = NULL;
-        int                   ret            = -1;
-        int32_t               op_ret         = -1;
-        int32_t               op_errno       = 0;
-        int                   count          = 0;
-        gf_dirent_t           entries;
-        struct index_syncop_args args = {0};
-
-        priv = this->private;
-        INIT_LIST_HEAD (&entries.list);
-
-        ret = index_fd_ctx_get (fd, this, &fctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "pfd is NULL, fd=%p", fd);
-                op_errno = -ret;
-                goto done;
-        }
+index_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      size_t size, off_t off, dict_t *xdata)
+{
+    index_fd_ctx_t *fctx = NULL;
+    index_priv_t *priv = NULL;
+    DIR *dir = NULL;
+    int ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int count = 0;
+    gf_dirent_t entries;
+    struct index_syncop_args args = {0};
+
+    priv = this->private;
+    INIT_LIST_HEAD(&entries.list);
+
+    ret = index_fd_ctx_get(fd, this, &fctx);
+    if (ret < 0) {
+        op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, INDEX_MSG_FD_OP_FAILED,
+               "pfd is NULL, fd=%p", fd);
+        goto done;
+    }
+
+    dir = fctx->dir;
+    if (!dir) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               INDEX_MSG_INDEX_READDIR_FAILED, "dir is NULL for fd=%p", fd);
+        goto done;
+    }
+
+    count = index_fill_readdir(fd, fctx, dir, off, size, &entries);
+
+    /* pick ENOENT to indicate EOF */
+    op_errno = errno;
+    op_ret = count;
+    if (index_is_virtual_gfid(priv, fd->inode->gfid) && xdata &&
+        dict_get(xdata, "get-gfid-type")) {
+        args.parent = fd->inode;
+        args.entries = &entries;
+        ret = synctask_new(this->ctx->env, index_get_gfid_type, NULL, NULL,
+                           &args);
+    }
+done:
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, &entries, NULL);
+    gf_dirent_free(&entries);
+    return 0;
+}
 
-        dir = fctx->dir;
-        if (!dir) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dir is NULL for fd=%p", fd);
-                op_errno = EINVAL;
-                goto done;
-        }
+int
+deletion_handler(const char *fpath, const struct stat *sb, int typeflag,
+                 struct FTW *ftwbuf)
+{
+    ia_type_t type = IA_INVAL;
 
-        count = index_fill_readdir (fd, fctx, dir, off, size, &entries);
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFREG:
+            sys_unlink(fpath);
+            break;
 
-        /* pick ENOENT to indicate EOF */
-        op_errno = errno;
-        op_ret = count;
-        if (index_is_virtual_gfid (priv, fd->inode->gfid) &&
-            xdata && dict_get (xdata, "get-gfid-type")) {
-                args.parent = fd->inode;
-                args.entries = &entries;
-                ret = synctask_new (this->ctx->env, index_get_gfid_type,
-                                    NULL, NULL, &args);
-        }
-done:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, NULL);
-        gf_dirent_free (&entries);
-        return 0;
+        case S_IFDIR:
+            sys_rmdir(fpath);
+            break;
+        default:
+            type = ia_type_from_st_mode(sb->st_mode);
+            gf_msg(THIS->name, GF_LOG_WARNING, EINVAL, INDEX_MSG_INVALID_ARGS,
+                   "%s neither a regular file nor a directory - type:%s", fpath,
+                   gf_inode_type_to_str(type));
+            break;
+    }
+    return 0;
 }
 
-int
-index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-                      dict_t *xdata)
-{
-        index_priv_t    *priv = NULL;
-        index_inode_ctx_t *ictx = NULL;
-        int32_t         op_ret = 0;
-        int32_t         op_errno = 0;
-        int             ret = 0;
-        index_xattrop_type_t type = XATTROP_TYPE_UNSET;
-        struct  iatt    preparent = {0};
-        struct  iatt    postparent = {0};
-        char            index_dir[PATH_MAX] = {0};
-        char            filepath[PATH_MAX] = {0};
-        struct  stat    lstatbuf = {0};
-        uuid_t          gfid = {0};
-        char            *subdir = NULL;
-
-        priv = this->private;
-        type = index_get_type_from_vgfid (priv, loc->pargfid);
-        ret = index_inode_path (this, loc->parent, index_dir,
-                                sizeof (index_dir));
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = -ret;
-                goto done;
-        }
+static int
+index_wipe_index_subdir(void *opaque)
+{
+    struct index_syncop_args *args = opaque;
 
-        ret = sys_lstat (index_dir, &lstatbuf);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
-                goto done;
-        }
+    nftw(args->path, deletion_handler, 1, FTW_DEPTH | FTW_PHYS);
+    return 0;
+}
 
-        iatt_from_stat (&preparent, &lstatbuf);
-        gf_uuid_copy (preparent.ia_gfid, loc->pargfid);
-        preparent.ia_ino = -1;
+static void
+index_get_parent_iatt(struct iatt *parent, char *path, loc_t *loc,
+                      int32_t *op_ret, int32_t *op_errno)
+{
+    int ret = -1;
+    struct stat lstatbuf = {
+        0,
+    };
+
+    ret = sys_lstat(path, &lstatbuf);
+    if (ret < 0) {
+        *op_ret = -1;
+        *op_errno = errno;
+        return;
+    }
 
-        if (type <= XATTROP_TYPE_UNSET) {
-                ret = index_inode_ctx_get (loc->parent, this, &ictx);
-                if ((ret == 0) && gf_uuid_is_null (ictx->virtual_pargfid)) {
-                        ret = -EINVAL;
-                }
-                if (ret == 0) {
-                        ret = index_entry_delete (this, ictx->virtual_pargfid,
-                                                  (char *)loc->name);
-                }
-        } else if (type == ENTRY_CHANGES) {
-                make_file_path (priv->index_basepath, ENTRY_CHANGES_SUBDIR,
-                                (char *)loc->name, filepath, sizeof (filepath));
-                ret = sys_unlink (filepath);
-        } else {
-                subdir = index_get_subdir_from_type (type);
-                gf_uuid_parse (loc->name, gfid);
-                ret = index_del (this, gfid, subdir, type);
-        }
+    iatt_from_stat(parent, &lstatbuf);
+    gf_uuid_copy(parent->ia_gfid, loc->pargfid);
+    parent->ia_ino = -1;
+
+    return;
+}
+
+int
+index_rmdir_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                    dict_t *xdata)
+{
+    int ret = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    char *subdir = NULL;
+    char index_dir[PATH_MAX] = {0};
+    char index_subdir[PATH_MAX] = {0};
+    uuid_t gfid = {0};
+    struct iatt preparent = {0};
+    struct iatt postparent = {0};
+    index_priv_t *priv = NULL;
+    index_xattrop_type_t type = XATTROP_TYPE_UNSET;
+    struct index_syncop_args args = {
+        0,
+    };
+
+    priv = this->private;
+
+    type = index_get_type_from_vgfid(priv, loc->pargfid);
+    subdir = index_get_subdir_from_vgfid(priv, loc->pargfid);
+    make_index_dir_path(priv->index_basepath, subdir, index_dir,
+                        sizeof(index_dir));
+
+    index_get_parent_iatt(&preparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
+
+    gf_uuid_parse(loc->name, gfid);
+    make_gfid_path(priv->index_basepath, subdir, gfid, index_subdir,
+                   sizeof(index_subdir));
+
+    if (flag == 0) {
+        ret = index_del(this, gfid, subdir, type);
         if (ret < 0) {
-                op_ret = -1;
-                op_errno = -ret;
-                goto done;
+            op_ret = -1;
+            op_errno = -ret;
+            goto done;
         }
+    } else {
+        args.path = index_subdir;
+        ret = synctask_new(this->ctx->env, index_wipe_index_subdir, NULL, NULL,
+                           &args);
+    }
 
-        memset (&lstatbuf, 0, sizeof (lstatbuf));
-        ret = sys_lstat (index_dir, &lstatbuf);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
-                goto done;
+    index_get_parent_iatt(&postparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
+
+done:
+    INDEX_STACK_UNWIND(rmdir, frame, op_ret, op_errno, &preparent, &postparent,
+                       xdata);
+    return 0;
+}
+
+int
+index_unlink_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                     dict_t *xdata)
+{
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ictx = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    int ret = 0;
+    index_xattrop_type_t type = XATTROP_TYPE_UNSET;
+    struct iatt preparent = {0};
+    struct iatt postparent = {0};
+    char index_dir[PATH_MAX] = {0};
+    char filepath[PATH_MAX] = {0};
+    uuid_t gfid = {0};
+    char *subdir = NULL;
+
+    priv = this->private;
+    type = index_get_type_from_vgfid(priv, loc->pargfid);
+    ret = index_inode_path(this, loc->parent, index_dir, sizeof(index_dir));
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = -ret;
+        goto done;
+    }
+
+    index_get_parent_iatt(&preparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
+
+    if (type <= XATTROP_TYPE_UNSET) {
+        ret = index_inode_ctx_get(loc->parent, this, &ictx);
+        if ((ret == 0) && gf_uuid_is_null(ictx->virtual_pargfid)) {
+            ret = -EINVAL;
         }
-        iatt_from_stat (&postparent, &lstatbuf);
-        gf_uuid_copy (postparent.ia_gfid, loc->pargfid);
-        postparent.ia_ino = -1;
+        if (ret == 0) {
+            ret = index_entry_delete(this, ictx->virtual_pargfid,
+                                     (char *)loc->name);
+        }
+    } else if (type == ENTRY_CHANGES) {
+        make_file_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR,
+                       (char *)loc->name, filepath, sizeof(filepath));
+        ret = sys_unlink(filepath);
+    } else {
+        subdir = index_get_subdir_from_type(type);
+        gf_uuid_parse(loc->name, gfid);
+        ret = index_del(this, gfid, subdir, type);
+    }
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = -ret;
+        goto done;
+    }
+
+    index_get_parent_iatt(&postparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
 done:
-        INDEX_STACK_UNWIND (unlink, frame, op_ret, op_errno, &preparent,
-                            &postparent, xdata);
-        return 0;
+    INDEX_STACK_UNWIND(unlink, frame, op_ret, op_errno, &preparent, &postparent,
+                       xdata);
+    return 0;
 }
 
 int32_t
-index_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name, dict_t *xdata)
+index_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        index_priv_t    *priv = NULL;
-
-        priv = this->private;
+    call_stub_t *stub = NULL;
 
-        if (!name || (!index_is_vgfid_xattr (name) &&
-		      strcmp (GF_XATTROP_INDEX_COUNT, name) &&
-                      strcmp (GF_XATTROP_DIRTY_COUNT, name)))
-                goto out;
+    if (!name ||
+        (!index_is_vgfid_xattr(name) && strcmp(GF_XATTROP_INDEX_COUNT, name) &&
+         strcmp(GF_XATTROP_DIRTY_COUNT, name)))
+        goto out;
 
-        stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
-                                  xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-        worker_enqueue (this, stub);
+    stub = fop_getxattr_stub(frame, index_getxattr_wrapper, loc, name, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
 int64_t
-index_fetch_link_count (xlator_t *this, index_xattrop_type_t type)
-{
-	char            index_dir[PATH_MAX] = {0};
-	char            index_path[PATH_MAX] = {0};
-        index_priv_t    *priv    = this->private;
-        char            *subdir  = NULL;
-	DIR             *dirp    = NULL;
-	struct dirent   *entry   = NULL;
-        struct stat     lstatbuf = {0};
-        int             ret      = -1;
-        int64_t         count    = -1;
-        struct dirent   buf;
-
-        subdir = index_get_subdir_from_type (type);
-	make_index_dir_path (priv->index_basepath, subdir,
-			     index_dir, sizeof (index_dir));
-
-	dirp = sys_opendir (index_dir);
-	if (!dirp)
-                goto out;
-
-	while (readdir_r (dirp, &buf, &entry) == 0) {
-		if (!entry) {
-                        if (count == -1)
-                                count = 0;
-                        goto out;
-                } else if (!strcmp (entry->d_name, ".") ||
-		           !strcmp (entry->d_name, "..")) {
-			continue;
-                } else {
-                        make_file_path (priv->index_basepath, subdir,
-                                        entry->d_name, index_path,
-                                        sizeof (index_path));
-                        ret = sys_lstat (index_path, &lstatbuf);
-                        if (ret < 0) {
-                                count = -2;
-                                continue;
-                        } else {
-                                count = lstatbuf.st_nlink - 1;
-                                if (count == 0)
-                                        continue;
-                                else
-                                        break;
-                        }
-                }
-	}
+index_fetch_link_count(xlator_t *this, index_xattrop_type_t type)
+{
+    index_priv_t *priv = this->private;
+    char *subdir = NULL;
+    struct stat lstatbuf = {
+        0,
+    };
+    int ret = -1;
+    int64_t count = -1;
+    DIR *dirp = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char index_dir[PATH_MAX] = {
+        0,
+    };
+    char index_path[PATH_MAX] = {
+        0,
+    };
+
+    subdir = index_get_subdir_from_type(type);
+    make_index_dir_path(priv->index_basepath, subdir, index_dir,
+                        sizeof(index_dir));
+
+    dirp = sys_opendir(index_dir);
+    if (!dirp)
+        goto out;
+
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(dirp, scratch);
+        if (!entry || errno != 0) {
+            if (count == -1)
+                count = 0;
+            goto out;
+        }
+
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
+
+        make_file_path(priv->index_basepath, subdir, entry->d_name, index_path,
+                       sizeof(index_path));
+
+        ret = sys_lstat(index_path, &lstatbuf);
+        if (ret < 0) {
+            count = -2;
+            continue;
+        } else {
+            count = lstatbuf.st_nlink - 1;
+            if (count == 0)
+                continue;
+            else
+                break;
+        }
+    }
 out:
-        if (dirp)
-                sys_closedir (dirp);
-        return count;
+    if (dirp)
+        (void)sys_closedir(dirp);
+    return count;
 }
 
-dict_t*
-index_fill_link_count (xlator_t *this, dict_t *xdata)
+dict_t *
+index_fill_link_count(xlator_t *this, dict_t *xdata)
 {
-        int             ret       = -1;
-        index_priv_t    *priv     = NULL;
-        int64_t         count     = -1;
+    int ret = -1;
+    index_priv_t *priv = NULL;
+    int64_t count = -1;
 
-        priv = this->private;
-        xdata = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!xdata)
-                goto out;
+    priv = this->private;
+    xdata = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!xdata)
+        goto out;
 
-        index_get_link_count (priv, &count, XATTROP);
-        if (count < 0) {
-                count = index_fetch_link_count (this, XATTROP);
-                index_set_link_count (priv, count, XATTROP);
-        }
+    index_get_link_count(priv, &count, XATTROP);
+    if (count < 0) {
+        count = index_fetch_link_count(this, XATTROP);
+        index_set_link_count(priv, count, XATTROP);
+    }
 
-        if (count == 0) {
-                ret = dict_set_int8 (xdata, "link-count", 0);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set link-count");
-        } else {
-                ret = dict_set_int8 (xdata, "link-count", 1);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set link-count");
-        }
+    if (count == 0) {
+        ret = dict_set_int8(xdata, "link-count", 0);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_DICT_SET_FAILED,
+                   "Unable to set link-count");
+    } else {
+        ret = dict_set_int8(xdata, "link-count", 1);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_DICT_SET_FAILED,
+                   "Unable to set link-count");
+    }
 
 out:
-        return xdata;
+    return xdata;
 }
 
 int32_t
-index_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+index_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-
-        xdata = index_fill_link_count (this, xdata);
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
+    xdata = index_fill_link_count(this, xdata);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
 }
 
 int32_t
-index_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr_req)
-{
-        inode_t         *inode = NULL;
-        call_stub_t     *stub = NULL;
-        index_priv_t    *priv = NULL;
-        char            *flag = NULL;
-        int              ret  = -1;
-
-        priv = this->private;
-
-        if (!index_is_fop_on_internal_inode (this, loc->parent, loc->pargfid) &&
-            !index_is_fop_on_internal_inode (this, loc->inode, loc->gfid)) {
-                if (!inode_is_linked (loc->inode)) {
-                        inode = inode_find (loc->inode->table, loc->gfid);
-                        if (!index_is_fop_on_internal_inode (this, inode,
-                                                            loc->gfid)) {
-                                inode_unref (inode);
-                                goto normal;
-                        }
-                        inode_unref (inode);
-                } else {
-                        goto normal;
-                }
+index_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    inode_t *inode = NULL;
+    call_stub_t *stub = NULL;
+    char *flag = NULL;
+    int ret = -1;
+
+    if (!index_is_fop_on_internal_inode(this, loc->parent, loc->pargfid) &&
+        !index_is_fop_on_internal_inode(this, loc->inode, loc->gfid)) {
+        if (!inode_is_linked(loc->inode)) {
+            inode = inode_find(loc->inode->table, loc->gfid);
+            if (!index_is_fop_on_internal_inode(this, inode, loc->gfid)) {
+                inode_unref(inode);
+                goto normal;
+            }
+            inode_unref(inode);
+        } else {
+            goto normal;
         }
+    }
 
-        stub = fop_lookup_stub (frame, index_lookup_wrapper, loc, xattr_req);
-        if (!stub) {
-                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM, loc->inode,
-                                     NULL, NULL, NULL);
-                return 0;
-        }
-        worker_enqueue (this, stub);
+    stub = fop_lookup_stub(frame, index_lookup_wrapper, loc, xattr_req);
+    if (!stub) {
+        STACK_UNWIND_STRICT(lookup, frame, -1, ENOMEM, loc->inode, NULL, NULL,
+                            NULL);
         return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
 normal:
-        ret = dict_get_str (xattr_req, "link-count", &flag);
-        if ((ret == 0) && (strcmp (flag, GF_XATTROP_INDEX_COUNT) == 0)) {
-                STACK_WIND (frame, index_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-        } else {
-                STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-        }
+    ret = dict_get_str_sizen(xattr_req, "link-count", &flag);
+    if ((ret == 0) && (strcmp(flag, GF_XATTROP_INDEX_COUNT) == 0)) {
+        STACK_WIND(frame, index_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    } else {
+        STACK_WIND(frame, default_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-index_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                 dict_t *xdata)
-{
-        xdata = index_fill_link_count (this, xdata);
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
+index_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
+{
+    xdata = index_fill_link_count(this, xdata);
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, buf, xdata);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
 }
 
 int32_t
-index_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+index_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int  ret   = -1;
-        char *flag = NULL;
+    int ret = -1;
+    char *flag = NULL;
 
-        ret = dict_get_str (xdata, "link-count", &flag);
-        if ((ret == 0) && (strcmp (flag, GF_XATTROP_INDEX_COUNT) == 0)) {
-                STACK_WIND (frame, index_fstat_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        } else {
-                STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        }
+    ret = dict_get_str(xdata, "link-count", &flag);
+    if ((ret == 0) && (strcmp(flag, GF_XATTROP_INDEX_COUNT) == 0)) {
+        STACK_WIND(frame, index_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    } else {
+        STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-index_opendir (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, fd_t *fd, dict_t *xdata)
+index_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+              dict_t *xdata)
 {
-        index_priv_t    *priv = NULL;
+    if (!index_is_fop_on_internal_inode(this, fd->inode, NULL))
+        goto normal;
 
-        priv = this->private;
-        if (!index_is_fop_on_internal_inode (this, fd->inode, NULL))
-                goto normal;
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, 0, 0, fd, NULL);
-        return 0;
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(opendir, frame, 0, 0, fd, NULL);
+    return 0;
 
 normal:
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int32_t
-index_readdir (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t off, dict_t *xdata)
+index_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t off, dict_t *xdata)
 {
-        call_stub_t       *stub       = NULL;
-        index_priv_t      *priv       = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
-        if (!index_is_fop_on_internal_inode (this, fd->inode, NULL))
-                goto out;
+    if (!index_is_fop_on_internal_inode(this, fd->inode, NULL))
+        goto out;
 
-        stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, off,
-                                 xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-        worker_enqueue (this, stub);
+    stub = fop_readdir_stub(frame, index_readdir_wrapper, fd, size, off, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
-        return 0;
+    STACK_WIND(frame, default_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+    return 0;
 }
 
 int
-index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+index_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        index_priv_t    *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
-        if (!index_is_fop_on_internal_inode (this, loc->parent, NULL))
-                goto out;
+    if (!index_is_fop_on_internal_inode(this, loc->parent, NULL))
+        goto out;
 
-        stub = fop_unlink_stub (frame, index_unlink_wrapper, loc, xflag, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL,
-                                     NULL);
-                return 0;
-        }
-        worker_enqueue (this, stub);
+    stub = fop_unlink_stub(frame, index_unlink_wrapper, loc, xflag, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+}
+
+int
+index_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    if (!index_is_fop_on_internal_inode(this, loc->parent, NULL))
+        goto out;
+
+    stub = fop_rmdir_stub(frame, index_rmdir_wrapper, loc, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
+out:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+    return 0;
 }
 
 int
-index_make_xattrop_watchlist (xlator_t *this, index_priv_t *priv,
-                              char *watchlist, index_xattrop_type_t type)
-{
-        char   *delim         = NULL;
-        char   *dup_watchlist = NULL;
-        char   *key           = NULL;
-        char   *saveptr       = NULL;
-        dict_t *xattrs        = NULL;
-        data_t *dummy         = NULL;
-        int    ret            = 0;
-
-        if (!watchlist)
-                return 0;
-
-        dup_watchlist = gf_strdup (watchlist);
-        if (!dup_watchlist)
-                return -1;
-
-        xattrs = dict_new ();
-        if (!xattrs) {
-                ret = -1;
-                goto out;
-        }
+index_make_xattrop_watchlist(xlator_t *this, index_priv_t *priv,
+                             char *watchlist, index_xattrop_type_t type)
+{
+    char *delim = NULL;
+    char *dup_watchlist = NULL;
+    char *key = NULL;
+    char *saveptr = NULL;
+    dict_t *xattrs = NULL;
+    data_t *dummy = NULL;
+    int ret = 0;
+
+    if (!watchlist)
+        return 0;
 
-        dummy = int_to_data (1);
-        if (!dummy) {
-                ret = -1;
-                goto out;
-        }
+    dup_watchlist = gf_strdup(watchlist);
+    if (!dup_watchlist)
+        return -1;
 
-        data_ref (dummy);
+    xattrs = dict_new();
+    if (!xattrs) {
+        ret = -1;
+        goto out;
+    }
 
-        delim = ",";
-        key = strtok_r (dup_watchlist, delim, &saveptr);
-        while (key) {
-                if (strlen (key) == 0) {
-                        ret = -1;
-                        goto out;
-                }
+    dummy = int_to_data(1);
+    if (!dummy) {
+        ret = -1;
+        goto out;
+    }
 
-                ret = dict_set (xattrs, key, dummy);
-                if (ret)
-                        goto out;
+    data_ref(dummy);
 
-                key = strtok_r (NULL, delim, &saveptr);
+    delim = ",";
+    key = strtok_r(dup_watchlist, delim, &saveptr);
+    while (key) {
+        if (strlen(key) == 0) {
+            ret = -1;
+            goto out;
         }
 
-        switch (type) {
+        ret = dict_set(xattrs, key, dummy);
+        if (ret)
+            goto out;
+
+        key = strtok_r(NULL, delim, &saveptr);
+    }
+
+    switch (type) {
         case DIRTY:
-                priv->dirty_watchlist = xattrs;
-                break;
+            priv->dirty_watchlist = dict_copy_with_ref(xattrs,
+                                                       priv->dirty_watchlist);
+            if (!priv->dirty_watchlist) {
+                ret = -1;
+                goto out;
+            }
+            break;
         case XATTROP:
-                priv->pending_watchlist = xattrs;
-                break;
+            priv->pending_watchlist = dict_copy_with_ref(
+                xattrs, priv->pending_watchlist);
+            if (!priv->pending_watchlist) {
+                ret = -1;
+                goto out;
+            }
+            break;
         default:
-                break;
-        }
-        xattrs = NULL;
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (xattrs)
-                dict_unref (xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
 
-        GF_FREE (dup_watchlist);
+    GF_FREE(dup_watchlist);
 
-        if (dummy)
-                data_unref (dummy);
+    if (dummy)
+        data_unref(dummy);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_index_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_index_mt_end + 1);
 
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
-{
-        int i = 0;
-        int ret = -1;
-        int64_t count = -1;
-        index_priv_t *priv = NULL;
-        pthread_t thread;
-        pthread_attr_t  w_attr;
-        gf_boolean_t    mutex_inited = _gf_false;
-        gf_boolean_t    cond_inited  = _gf_false;
-        gf_boolean_t    attr_inited  = _gf_false;
-        char            *watchlist = NULL;
-        char            *dirtylist = NULL;
-        char            *pendinglist = NULL;
-
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"'index' not configured with exactly one child");
-                goto out;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_index_mt_priv_t);
-        if (!priv)
-                goto out;
-
-        LOCK_INIT (&priv->lock);
-        if ((ret = pthread_cond_init(&priv->cond, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_cond_init failed (%d)", ret);
-                goto out;
-        }
-        cond_inited = _gf_true;
-
-        if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_mutex_init failed (%d)", ret);
-                goto out;
-        }
-        mutex_inited = _gf_true;
-
-        if ((ret = pthread_attr_init (&w_attr)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_attr_init failed (%d)", ret);
-                goto out;
-        }
-        attr_inited = _gf_true;
-
-        ret = pthread_attr_setstacksize (&w_attr, INDEX_THREAD_STACK_SIZE);
-        if (ret == EINVAL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Using default thread stack size");
-        }
-
-        GF_OPTION_INIT ("index-base", priv->index_basepath, path, out);
-
-        GF_OPTION_INIT ("xattrop64-watchlist", watchlist, str, out);
-        ret = index_make_xattrop_watchlist (this, priv, watchlist,
-                                            XATTROP);
-        if (ret)
-                goto out;
-
-        GF_OPTION_INIT ("xattrop-dirty-watchlist", dirtylist, str, out);
-        ret = index_make_xattrop_watchlist (this, priv, dirtylist,
-                                            DIRTY);
-        if (ret)
-                goto out;
-
-        GF_OPTION_INIT ("xattrop-pending-watchlist", pendinglist, str, out);
-        ret = index_make_xattrop_watchlist (this, priv, pendinglist,
-                                            XATTROP);
-        if (ret)
-                goto out;
-
-        if (priv->dirty_watchlist)
-                priv->complete_watchlist = dict_copy_with_ref (priv->dirty_watchlist,
+init(xlator_t *this)
+{
+    int i = 0;
+    int ret = -1;
+    int64_t count = -1;
+    index_priv_t *priv = NULL;
+    pthread_attr_t w_attr;
+    gf_boolean_t mutex_inited = _gf_false;
+    gf_boolean_t cond_inited = _gf_false;
+    gf_boolean_t attr_inited = _gf_false;
+    char *watchlist = NULL;
+    char *dirtylist = NULL;
+    char *pendinglist = NULL;
+    char *index_base_parent = NULL;
+    char *tmp = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_INVALID_GRAPH,
+               "'index' not configured with exactly one child");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, INDEX_MSG_INVALID_GRAPH,
+               "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_index_mt_priv_t);
+    if (!priv)
+        goto out;
+
+    LOCK_INIT(&priv->lock);
+    if ((ret = pthread_cond_init(&priv->cond, NULL)) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, INDEX_MSG_INVALID_ARGS,
+               "pthread_cond_init failed");
+        goto out;
+    }
+    cond_inited = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, INDEX_MSG_INVALID_ARGS,
+               "pthread_mutex_init failed");
+        goto out;
+    }
+    mutex_inited = _gf_true;
+
+    if ((ret = pthread_attr_init(&w_attr)) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, INDEX_MSG_INVALID_ARGS,
+               "pthread_attr_init failed");
+        goto out;
+    }
+    attr_inited = _gf_true;
+
+    ret = pthread_attr_setstacksize(&w_attr, INDEX_THREAD_STACK_SIZE);
+    if (ret == EINVAL) {
+        gf_msg(this->name, GF_LOG_WARNING, ret, INDEX_MSG_INVALID_ARGS,
+               "Using default thread stack size");
+    }
+
+    GF_OPTION_INIT("index-base", priv->index_basepath, path, out);
+    tmp = gf_strdup(priv->index_basepath);
+    index_base_parent = dirname(tmp);
+    if (gf_lstat_dir(index_base_parent, NULL) != 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno,
+               INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+               "Failed to find parent dir (%s) of index basepath %s.",
+               index_base_parent, priv->index_basepath);
+        goto out;
+    }
+
+    GF_OPTION_INIT("xattrop64-watchlist", watchlist, str, out);
+    ret = index_make_xattrop_watchlist(this, priv, watchlist, XATTROP);
+    if (ret)
+        goto out;
+
+    GF_OPTION_INIT("xattrop-dirty-watchlist", dirtylist, str, out);
+    ret = index_make_xattrop_watchlist(this, priv, dirtylist, DIRTY);
+    if (ret)
+        goto out;
+
+    GF_OPTION_INIT("xattrop-pending-watchlist", pendinglist, str, out);
+    ret = index_make_xattrop_watchlist(this, priv, pendinglist, XATTROP);
+    if (ret)
+        goto out;
+
+    if (priv->dirty_watchlist)
+        priv->complete_watchlist = dict_copy_with_ref(priv->dirty_watchlist,
                                                       priv->complete_watchlist);
-        if (priv->pending_watchlist)
-                priv->complete_watchlist = dict_copy_with_ref (priv->pending_watchlist,
+    if (priv->pending_watchlist)
+        priv->complete_watchlist = dict_copy_with_ref(priv->pending_watchlist,
                                                       priv->complete_watchlist);
-        gf_uuid_generate (priv->index);
-        for (i = 0; i < XATTROP_TYPE_END; i++)
-                gf_uuid_generate (priv->internal_vgfid[i]);
 
-        INIT_LIST_HEAD (&priv->callstubs);
+    gf_uuid_generate(priv->index);
+    for (i = 0; i < XATTROP_TYPE_END; i++)
+        gf_uuid_generate(priv->internal_vgfid[i]);
 
-        this->private = priv;
+    INIT_LIST_HEAD(&priv->callstubs);
+    GF_ATOMIC_INIT(priv->stub_cnt, 0);
 
-        ret = index_dir_create (this, XATTROP_SUBDIR);
-        if (ret < 0)
-                goto out;
+    this->local_pool = mem_pool_new(index_local_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
 
-        if (priv->dirty_watchlist) {
-                ret = index_dir_create (this, DIRTY_SUBDIR);
-                if (ret < 0)
-                        goto out;
-        }
+    this->private = priv;
 
-        ret = index_dir_create (this, ENTRY_CHANGES_SUBDIR);
-        if (ret < 0)
-                goto out;
+    ret = index_dir_create(this, XATTROP_SUBDIR);
+    if (ret < 0)
+        goto out;
 
-        /*init indices files counts*/
-        count = index_fetch_link_count (this, XATTROP);
-        index_set_link_count (priv, count, XATTROP);
-
-        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to create "
-                        "worker thread, aborting");
-                goto out;
-        }
-
-        ret = 0;
+    if (priv->dirty_watchlist) {
+        ret = index_dir_create(this, DIRTY_SUBDIR);
+        if (ret < 0)
+            goto out;
+    }
+
+    ret = index_dir_create(this, ENTRY_CHANGES_SUBDIR);
+    if (ret < 0)
+        goto out;
+
+    /*init indices files counts*/
+    count = index_fetch_link_count(this, XATTROP);
+    index_set_link_count(priv, count, XATTROP);
+    priv->down = _gf_false;
+
+    priv->curr_count = 0;
+    ret = gf_thread_create(&priv->thread, &w_attr, index_worker, this,
+                           "idxwrker");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ret,
+               INDEX_MSG_WORKER_THREAD_CREATE_FAILED,
+               "Failed to create worker thread, aborting");
+        goto out;
+    }
+    priv->curr_count++;
+    ret = 0;
 out:
-        if (ret) {
-                if (cond_inited)
-                        pthread_cond_destroy (&priv->cond);
-                if (mutex_inited)
-                        pthread_mutex_destroy (&priv->mutex);
-                if (priv && priv->dirty_watchlist)
-                        dict_unref (priv->dirty_watchlist);
-                if (priv && priv->pending_watchlist)
-                        dict_unref (priv->pending_watchlist);
-                if (priv && priv->complete_watchlist)
-                        dict_unref (priv->complete_watchlist);
-                if (priv)
-                        GF_FREE (priv);
-                this->private = NULL;
-        }
+    GF_FREE(tmp);
+
+    if (ret) {
+        if (cond_inited)
+            pthread_cond_destroy(&priv->cond);
+        if (mutex_inited)
+            pthread_mutex_destroy(&priv->mutex);
+        if (priv && priv->dirty_watchlist)
+            dict_unref(priv->dirty_watchlist);
+        if (priv && priv->pending_watchlist)
+            dict_unref(priv->pending_watchlist);
+        if (priv && priv->complete_watchlist)
+            dict_unref(priv->complete_watchlist);
+        if (priv)
+            GF_FREE(priv);
+        this->private = NULL;
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 
-        if (attr_inited)
-                pthread_attr_destroy (&w_attr);
-        return ret;
+    if (attr_inited)
+        pthread_attr_destroy(&w_attr);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
-{
-        index_priv_t *priv = NULL;
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-        this->private = NULL;
-        LOCK_DESTROY (&priv->lock);
-        pthread_cond_destroy (&priv->cond);
-        pthread_mutex_destroy (&priv->mutex);
-        if (priv->dirty_watchlist)
-                dict_unref (priv->dirty_watchlist);
-        if (priv->pending_watchlist)
-                dict_unref (priv->pending_watchlist);
-        if (priv->complete_watchlist)
-                dict_unref (priv->complete_watchlist);
-        GF_FREE (priv);
+fini(xlator_t *this)
+{
+    index_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    priv->down = _gf_true;
+    pthread_cond_broadcast(&priv->cond);
+    if (priv->thread) {
+        gf_thread_cleanup_xint(priv->thread);
+        priv->thread = 0;
+    }
+    this->private = NULL;
+    LOCK_DESTROY(&priv->lock);
+    pthread_cond_destroy(&priv->cond);
+    pthread_mutex_destroy(&priv->mutex);
+    if (priv->dirty_watchlist)
+        dict_unref(priv->dirty_watchlist);
+    if (priv->pending_watchlist)
+        dict_unref(priv->pending_watchlist);
+    if (priv->complete_watchlist)
+        dict_unref(priv->complete_watchlist);
+    GF_FREE(priv);
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 out:
-        return;
+    return;
 }
 
 int
-index_forget (xlator_t *this, inode_t *inode)
+index_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t tmp_cache = 0;
-        if (!inode_ctx_del (inode, this, &tmp_cache))
-                GF_FREE ((index_inode_ctx_t*) (long)tmp_cache);
+    uint64_t tmp_cache = 0;
+    if (!inode_ctx_del(inode, this, &tmp_cache))
+        GF_FREE((index_inode_ctx_t *)(long)tmp_cache);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-index_releasedir (xlator_t *this, fd_t *fd)
+index_releasedir(xlator_t *this, fd_t *fd)
 {
-        index_fd_ctx_t *fctx = NULL;
-        uint64_t        ctx = 0;
-        int             ret = 0;
+    index_fd_ctx_t *fctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        ret = fd_ctx_del (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
+    ret = fd_ctx_del(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        fctx = (index_fd_ctx_t*) (long) ctx;
-        if (fctx->dir) {
-                ret = sys_closedir (fctx->dir);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "closedir error: %s", strerror (errno));
-        }
+    fctx = (index_fd_ctx_t *)(long)ctx;
+    if (fctx->dir) {
+        ret = sys_closedir(fctx->dir);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, errno, INDEX_MSG_FD_OP_FAILED,
+                   "closedir error");
+    }
 
-        GF_FREE (fctx);
+    GF_FREE(fctx);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-index_release (xlator_t *this, fd_t *fd)
+index_release(xlator_t *this, fd_t *fd)
 {
-        index_fd_ctx_t *fctx = NULL;
-        uint64_t        ctx = 0;
-        int             ret = 0;
+    index_fd_ctx_t *fctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        ret = fd_ctx_del (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
+    ret = fd_ctx_del(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        fctx = (index_fd_ctx_t*) (long) ctx;
-        GF_FREE (fctx);
+    fctx = (index_fd_ctx_t *)(long)ctx;
+    GF_FREE(fctx);
 out:
-        return 0;
+    return 0;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        int     ret = 0;
-        ret = default_notify (this, event, data);
-        return ret;
+    int ret = 0;
+    index_priv_t *priv = NULL;
+    uint64_t stub_cnt = 0;
+    xlator_t *victim = data;
+    struct timespec sleep_till = {
+        0,
+    };
+
+    if (!this)
+        return 0;
+
+    priv = this->private;
+    if (!priv)
+        return 0;
+
+    if ((event == GF_EVENT_PARENT_DOWN) && victim->cleanup_starting) {
+        stub_cnt = GF_ATOMIC_GET(priv->stub_cnt);
+        timespec_now_realtime(&sleep_till);
+        sleep_till.tv_sec += 1;
+
+        /* Wait for draining stub from queue before notify PARENT_DOWN */
+        pthread_mutex_lock(&priv->mutex);
+        {
+            while (stub_cnt) {
+                (void)pthread_cond_timedwait(&priv->cond, &priv->mutex,
+                                             &sleep_till);
+                stub_cnt = GF_ATOMIC_GET(priv->stub_cnt);
+            }
+        }
+        pthread_mutex_unlock(&priv->mutex);
+        gf_log(this->name, GF_LOG_INFO,
+               "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name);
+    }
+
+    if ((event == GF_EVENT_CHILD_DOWN) && victim->cleanup_starting) {
+        pthread_mutex_lock(&priv->mutex);
+        {
+            priv->down = _gf_true;
+            pthread_cond_broadcast(&priv->cond);
+            while (priv->curr_count)
+                pthread_cond_wait(&priv->cond, &priv->mutex);
+        }
+        pthread_mutex_unlock(&priv->mutex);
+
+        gf_log(this->name, GF_LOG_INFO,
+               "Notify GF_EVENT_CHILD_DOWN for brick %s", victim->name);
+    }
+
+    ret = default_notify(this, event, data);
+    return ret;
 }
 
 struct xlator_fops fops = {
-	.xattrop     = index_xattrop,
-	.fxattrop    = index_fxattrop,
-
-        //interface functions follow
-        .getxattr    = index_getxattr,
-        .lookup      = index_lookup,
-        .opendir     = index_opendir,
-        .readdir     = index_readdir,
-        .unlink      = index_unlink,
-        .fstat       = index_fstat,
+    .xattrop = index_xattrop,
+    .fxattrop = index_fxattrop,
+
+    // interface functions follow
+    .getxattr = index_getxattr,
+    .lookup = index_lookup,
+    .opendir = index_opendir,
+    .readdir = index_readdir,
+    .unlink = index_unlink,
+    .rmdir = index_rmdir,
+    .fstat = index_fstat,
 };
 
 struct xlator_dumpops dumpops;
 
-struct xlator_cbks cbks = {
-        .forget         = index_forget,
-        .release        = index_release,
-        .releasedir     = index_releasedir
-};
+struct xlator_cbks cbks = {.forget = index_forget,
+                           .release = index_release,
+                           .releasedir = index_releasedir};
 
 struct volume_options options[] = {
-        { .key  = {"index-base" },
-          .type = GF_OPTION_TYPE_PATH,
-          .description = "path where the index files need to be stored",
-        },
-        { .key  = {"xattrop64-watchlist" },
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Comma separated list of xattrs that are watched",
-        },
-        { .key  = {"xattrop-dirty-watchlist" },
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Comma separated list of xattrs that are watched",
-        },
-        { .key  = {"xattrop-pending-watchlist" },
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Comma separated list of xattrs that are watched",
-        },
-        { .key  = {NULL} },
+    {.key = {"index-base"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "path where the index files need to be stored",
+     .default_value = "{{ brick.path }}/.glusterfs/indices"},
+    {.key = {"xattrop64-watchlist"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are watched",
+     .default_value = "trusted.ec.dirty"},
+    {.key = {"xattrop-dirty-watchlist"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are watched",
+     .default_value = "trusted.afr.dirty"},
+    {.key = {"xattrop-pending-watchlist"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are watched",
+     .default_value = "trusted.afr.{{ volume.name }}"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "index",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
index 24fd293db70..a2b6e6e2570 100644
--- a/xlators/features/index/src/index.h
+++ b/xlators/features/index/src/index.h
@@ -11,65 +11,76 @@
 #ifndef __INDEX_H__
 #define __INDEX_H__
 
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "byte-order.h"
-#include "common-utils.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
 #include "index-mem-types.h"
 
-#define INDEX_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+#define INDEX_THREAD_STACK_SIZE ((size_t)(1024 * 1024))
 
-typedef enum {
-        UNKNOWN,
-        IN,
-        NOTIN
-} index_state_t;
+typedef enum { UNKNOWN, IN, NOTIN } index_state_t;
 
 typedef enum {
-        XATTROP_TYPE_UNSET = -1,
-        XATTROP,
-        DIRTY,
-        ENTRY_CHANGES,
-        XATTROP_TYPE_END
+    XATTROP_TYPE_UNSET = -1,
+    XATTROP,
+    DIRTY,
+    ENTRY_CHANGES,
+    XATTROP_TYPE_END
 } index_xattrop_type_t;
 
 typedef struct index_inode_ctx {
-        gf_boolean_t processing;
-        struct list_head callstubs;
-        int state[XATTROP_TYPE_END];
-        uuid_t virtual_pargfid; /* virtual gfid of dir under
-                                  .glusterfs/indices/entry-changes. */
+    gf_boolean_t processing;
+    struct list_head callstubs;
+    int state[XATTROP_TYPE_END];
+    uuid_t virtual_pargfid; /* virtual gfid of dir under
+                              .glusterfs/indices/entry-changes. */
 } index_inode_ctx_t;
 
 typedef struct index_fd_ctx {
-        DIR *dir;
-        off_t dir_eof;
+    DIR *dir;
+    off_t dir_eof;
 } index_fd_ctx_t;
 
 typedef struct index_priv {
-        char *index_basepath;
-        char *dirty_basepath;
-        uuid_t index;
-        gf_lock_t lock;
-        uuid_t internal_vgfid[XATTROP_TYPE_END];
-        struct list_head callstubs;
-        pthread_mutex_t mutex;
-        pthread_cond_t  cond;
-        dict_t  *dirty_watchlist;
-        dict_t  *pending_watchlist;
-        dict_t  *complete_watchlist;
-        int64_t  pending_count;
+    char *index_basepath;
+    char *dirty_basepath;
+    uuid_t index;
+    gf_lock_t lock;
+    uuid_t internal_vgfid[XATTROP_TYPE_END];
+    struct list_head callstubs;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    dict_t *dirty_watchlist;
+    dict_t *pending_watchlist;
+    dict_t *complete_watchlist;
+    int64_t pending_count;
+    pthread_t thread;
+    gf_boolean_t down;
+    gf_atomic_t stub_cnt;
+    int32_t curr_count;
 } index_priv_t;
 
-#define INDEX_STACK_UNWIND(fop, frame, params ...)      \
-do {                                                    \
-        if (frame) {                                    \
-                inode_t *_inode = frame->local;         \
-                frame->local = NULL;                    \
-                inode_unref (_inode);                   \
-        }                                               \
-        STACK_UNWIND_STRICT (fop, frame, params);       \
-} while (0)
+typedef struct index_local {
+    inode_t *inode;
+    dict_t *xdata;
+} index_local_t;
+
+#define INDEX_STACK_UNWIND(fop, frame, params...)                              \
+    do {                                                                       \
+        index_local_t *__local = NULL;                                         \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local) {                                                         \
+            inode_unref(__local->inode);                                       \
+            if (__local->xdata)                                                \
+                dict_unref(__local->xdata);                                    \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
 
 #endif
diff --git a/xlators/features/leases/src/Makefile.am b/xlators/features/leases/src/Makefile.am
index 343f5c82425..a1aef10e299 100644
--- a/xlators/features/leases/src/Makefile.am
+++ b/xlators/features/leases/src/Makefile.am
@@ -1,4 +1,6 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = leases.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 leases_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
@@ -10,7 +12,8 @@ leases_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 noinst_HEADERS = leases.h leases-mem-types.h leases-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-               -I$(CONTRIBDIR)/timer-wheel
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(CONTRIBDIR)/timer-wheel
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/leases/src/leases-internal.c b/xlators/features/leases/src/leases-internal.c
index 6884b581273..56dee244281 100644
--- a/xlators/features/leases/src/leases-internal.c
+++ b/xlators/features/leases/src/leases-internal.c
@@ -15,7 +15,6 @@
 
 #include "leases.h"
 
-
 /* Mutex locks used in this xlator and their order of acquisition:
  * Check lease conflict:
  *         lease_ctx lock
@@ -53,1299 +52,1361 @@
  * _gf_false - lease lock option disabled
  */
 gf_boolean_t
-is_leases_enabled (xlator_t *this)
+is_leases_enabled(xlator_t *this)
 {
-        leases_private_t    *priv       = NULL;
-        gf_boolean_t         is_enabled = _gf_false;
+    leases_private_t *priv = NULL;
+    gf_boolean_t is_enabled = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
 
-        if (this->private) {
-                priv = (leases_private_t *)this->private;
-                is_enabled = priv->leases_enabled;
-        }
+    if (this->private) {
+        priv = (leases_private_t *)this->private;
+        is_enabled = priv->leases_enabled;
+    }
 out:
-        return is_enabled;
+    return is_enabled;
 }
 
-
 /*
  * Get the recall_leaselk_timeout
  * Return Value:
  * timeout value(in seconds) set as an option to this xlator.
  * -1 error case
  */
-int32_t
-get_recall_lease_timeout (xlator_t *this)
+static int32_t
+get_recall_lease_timeout(xlator_t *this)
 {
-        leases_private_t    *priv       = NULL;
-        int32_t              timeout    = -1;
+    leases_private_t *priv = NULL;
+    int32_t timeout = -1;
 
-        GF_VALIDATE_OR_GOTO ("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
 
-        if (this->private) {
-                priv = (leases_private_t *)this->private;
-                timeout = priv->recall_lease_timeout;
-        }
+    if (this->private) {
+        priv = (leases_private_t *)this->private;
+        timeout = priv->recall_lease_timeout;
+    }
 out:
-        return timeout;
+    return timeout;
 }
 
-
 static void
-__dump_leases_info (xlator_t *this, lease_inode_ctx_t *lease_ctx)
+__dump_leases_info(xlator_t *this, lease_inode_ctx_t *lease_ctx)
 {
-        lease_id_entry_t   *lease_entry     = NULL;
-        lease_id_entry_t   *tmp             = NULL;
-
-        GF_VALIDATE_OR_GOTO ("leases", this, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-
-        gf_msg_debug (this->name, 0, "Lease held on this inode, lease_type: %d,"
-                      " lease_cnt:%"PRIu64", RD lease:%d, RW lease:%d, "
-                      "openfd cnt:%"PRIu64, lease_ctx->lease_type,
-                      lease_ctx->lease_cnt,
-                      lease_ctx->lease_type_cnt[GF_RD_LEASE],
-                      lease_ctx->lease_type_cnt[GF_RW_LEASE],
-                      lease_ctx->openfd_cnt);
-
-        list_for_each_entry_safe (lease_entry, tmp,
-                                  &lease_ctx->lease_id_list,
-                                  lease_id_list) {
-                gf_msg_debug (this->name, 0, "Leases held by client: %s, lease "
-                              "ID:%s, RD lease:%d, RW lease:%d, lease_type: %d, "
-                              "lease_cnt:%"PRIu64, lease_entry->client_uid,
-                              lease_entry->lease_id,
-                              lease_entry->lease_type_cnt[GF_RD_LEASE],
-                              lease_entry->lease_type_cnt[GF_RW_LEASE],
-                              lease_entry->lease_type, lease_entry->lease_cnt);
-        }
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    gf_msg_debug(this->name, 0,
+                 "Lease held on this inode, lease_type: %d,"
+                 " lease_cnt:%" PRIu64
+                 ", RD lease:%d, RW lease:%d, "
+                 "openfd cnt:%" PRIu64,
+                 lease_ctx->lease_type, lease_ctx->lease_cnt,
+                 lease_ctx->lease_type_cnt[GF_RD_LEASE],
+                 lease_ctx->lease_type_cnt[GF_RW_LEASE], lease_ctx->openfd_cnt);
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        gf_msg_debug(this->name, 0,
+                     "Leases held by client: %s, lease "
+                     "ID:%s, RD lease:%d, RW lease:%d, lease_type: %d, "
+                     "lease_cnt:%" PRIu64,
+                     lease_entry->client_uid, lease_entry->lease_id,
+                     lease_entry->lease_type_cnt[GF_RD_LEASE],
+                     lease_entry->lease_type_cnt[GF_RW_LEASE],
+                     lease_entry->lease_type, lease_entry->lease_cnt);
+    }
 out:
-        return;
+    return;
 }
 
-
 static int
-__lease_ctx_set (inode_t *inode, xlator_t *this)
+__lease_ctx_set(inode_t *inode, xlator_t *this)
 {
-        lease_inode_ctx_t   *inode_ctx    = NULL;
-        int                  ret          = -1;
-        uint64_t             ctx          = 0;
-
-        GF_VALIDATE_OR_GOTO ("leases", inode, out);
-        GF_VALIDATE_OR_GOTO ("leases", this, out);
-
-        ret = __inode_ctx_get (inode, this, &ctx);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, LEASE_MSG_INVAL_INODE_CTX,
-                        "inode_ctx_get failed");
-                goto out;
-        }
-
-        inode_ctx = GF_CALLOC (1, sizeof (*inode_ctx),
-                               gf_leases_mt_lease_inode_ctx_t);
-        GF_CHECK_ALLOC (inode_ctx, ret, out);
-
-        pthread_mutex_init (&inode_ctx->lock, NULL);
-        INIT_LIST_HEAD (&inode_ctx->lease_id_list);
-        INIT_LIST_HEAD (&inode_ctx->blocked_list);
-
-        inode_ctx->lease_cnt = 0;
-
-        ret = __inode_ctx_set (inode, this, (uint64_t *) inode_ctx);
-        if (ret) {
-                GF_FREE (inode_ctx);
-                gf_msg (this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_INODE_CTX,
-                        "failed to set inode ctx (%p)", inode);
-        }
+    lease_inode_ctx_t *inode_ctx = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+
+    ret = __inode_ctx_get(inode, this, &ctx);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INVAL_INODE_CTX,
+               "inode_ctx_get failed");
+        goto out;
+    }
+
+    inode_ctx = GF_CALLOC(1, sizeof(*inode_ctx),
+                          gf_leases_mt_lease_inode_ctx_t);
+    GF_CHECK_ALLOC(inode_ctx, ret, out);
+
+    pthread_mutex_init(&inode_ctx->lock, NULL);
+    INIT_LIST_HEAD(&inode_ctx->lease_id_list);
+    INIT_LIST_HEAD(&inode_ctx->blocked_list);
+
+    inode_ctx->lease_cnt = 0;
+
+    ret = __inode_ctx_set(inode, this, (uint64_t *)inode_ctx);
+    if (ret) {
+        GF_FREE(inode_ctx);
+        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_INODE_CTX,
+               "failed to set inode ctx (%p)", inode);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static lease_inode_ctx_t *
-__lease_ctx_get (inode_t *inode, xlator_t *this)
+__lease_ctx_get(inode_t *inode, xlator_t *this)
 {
-        lease_inode_ctx_t *inode_ctx    = NULL;
-        uint64_t           ctx          = 0;
-        int                ret          = 0;
+    lease_inode_ctx_t *inode_ctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO ("leases", inode, out);
-        GF_VALIDATE_OR_GOTO ("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
 
-        ret = __inode_ctx_get (inode, this, &ctx);
+    ret = __inode_ctx_get(inode, this, &ctx);
+    if (ret < 0) {
+        ret = __lease_ctx_set(inode, this);
+        if (ret < 0)
+            goto out;
+
+        ret = __inode_ctx_get(inode, this, &ctx);
         if (ret < 0) {
-                ret = __lease_ctx_set (inode, this);
-                if (ret < 0)
-                        goto out;
-
-                ret = __inode_ctx_get (inode, this, &ctx);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0, LEASE_MSG_INVAL_INODE_CTX,
-                                "failed to get inode ctx (%p)", inode);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_WARNING, 0, LEASE_MSG_INVAL_INODE_CTX,
+                   "failed to get inode ctx (%p)", inode);
+            goto out;
         }
+    }
 
-        inode_ctx = (lease_inode_ctx_t *)(long) ctx;
+    inode_ctx = (lease_inode_ctx_t *)(long)ctx;
 out:
-        return inode_ctx;
+    return inode_ctx;
 }
 
-
 lease_inode_ctx_t *
-lease_ctx_get (inode_t *inode, xlator_t *this)
+lease_ctx_get(inode_t *inode, xlator_t *this)
 {
-        lease_inode_ctx_t *inode_ctx = NULL;
+    lease_inode_ctx_t *inode_ctx = NULL;
 
-        GF_VALIDATE_OR_GOTO ("leases", inode, out);
-        GF_VALIDATE_OR_GOTO ("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
 
-        LOCK (&inode->lock);
-        {
-                inode_ctx = __lease_ctx_get (inode, this);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        inode_ctx = __lease_ctx_get(inode, this);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return inode_ctx;
+    return inode_ctx;
 }
 
-
 static lease_id_entry_t *
-new_lease_id_entry (call_frame_t *frame, const char *lease_id)
+new_lease_id_entry(call_frame_t *frame, const char *lease_id)
 {
-        lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *lease_entry = NULL;
 
-        GF_VALIDATE_OR_GOTO ("leases", frame, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_id, out);
-
-        lease_entry = GF_CALLOC (1, sizeof (*lease_entry),
-                                  gf_leases_mt_lease_id_entry_t);
-        if (!lease_entry) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
-                        "Memory allocation for lease_entry failed");
-                return NULL;
-        }
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
 
-        INIT_LIST_HEAD (&lease_entry->lease_id_list);
-        lease_entry->lease_type = NONE;
-        lease_entry->lease_cnt = 0;
-        lease_entry->recall_time =
-                        get_recall_lease_timeout (frame->this);
-        lease_entry->client_uid = gf_strdup (frame->root->client->client_uid);
-        if (!lease_entry->client_uid) {
-                gf_msg (frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
-                        "Memory allocation for client_uid failed");
-                GF_FREE (lease_entry);
-                lease_entry = NULL;
-                goto out;
-        }
-
-        memcpy (lease_entry->lease_id, lease_id, LEASE_ID_SIZE);
+    lease_entry = GF_CALLOC(1, sizeof(*lease_entry),
+                            gf_leases_mt_lease_id_entry_t);
+    if (!lease_entry) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
+               "Memory allocation for lease_entry failed");
+        return NULL;
+    }
+
+    INIT_LIST_HEAD(&lease_entry->lease_id_list);
+    lease_entry->lease_type = NONE;
+    lease_entry->lease_cnt = 0;
+    lease_entry->recall_time = get_recall_lease_timeout(frame->this);
+    lease_entry->client_uid = gf_strdup(frame->root->client->client_uid);
+    if (!lease_entry->client_uid) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
+               "Memory allocation for client_uid failed");
+        GF_FREE(lease_entry);
+        lease_entry = NULL;
+        goto out;
+    }
+
+    memcpy(lease_entry->lease_id, lease_id, LEASE_ID_SIZE);
 out:
-        return lease_entry;
+    return lease_entry;
 }
 
-
 static void
-__destroy_lease_id_entry (lease_id_entry_t *lease_entry)
+__destroy_lease_id_entry(lease_id_entry_t *lease_entry)
 {
-        GF_VALIDATE_OR_GOTO ("leases", lease_entry, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_entry, out);
 
-        list_del_init (&lease_entry->lease_id_list);
-        GF_FREE (lease_entry->client_uid);
-        GF_FREE (lease_entry);
+    list_del_init(&lease_entry->lease_id_list);
+    GF_FREE(lease_entry->client_uid);
+    GF_FREE(lease_entry);
 out:
-        return;
+    return;
 }
 
-
 static inline gf_boolean_t
-__is_same_lease_id (const char *k1, const char *k2)
+__is_same_lease_id(const char *k1, const char *k2)
 {
-        if (memcmp(k1, k2, LEASE_ID_SIZE) == 0)
-                return _gf_true;
+    if (memcmp(k1, k2, strlen(k1)) == 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
-
 /* Checks if there are any leases, other than the leases taken
  * by the given lease_id
  */
 static gf_boolean_t
-__another_lease_found (lease_inode_ctx_t *lease_ctx, const char *lease_id)
+__another_lease_found(lease_inode_ctx_t *lease_ctx, const char *lease_id)
 {
-        lease_id_entry_t   *lease_entry     = NULL;
-        lease_id_entry_t   *tmp             = NULL;
-        gf_boolean_t        found_lease     = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("leases", lease_id, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-
-        list_for_each_entry_safe (lease_entry, tmp,
-                                  &lease_ctx->lease_id_list,
-                                  lease_id_list) {
-
-                if (!__is_same_lease_id (lease_id, lease_entry->lease_id)) {
-                        if (lease_entry->lease_cnt > 0) {
-                                found_lease = _gf_true;
-                                break;
-                        }
-                }
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    gf_boolean_t found_lease = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        if (!__is_same_lease_id(lease_id, lease_entry->lease_id)) {
+            if (lease_entry->lease_cnt > 0) {
+                found_lease = _gf_true;
+                break;
+            }
         }
+    }
 out:
-        return found_lease;
+    return found_lease;
 }
 
-
 /* Returns the lease_id_entry for a given lease_id and a given inode.
  * Return values:
  * NULL - If no client entry found
  * lease_id_entry_t* - a pointer to the client entry if found
  */
 static lease_id_entry_t *
-__get_lease_id_entry (lease_inode_ctx_t *lease_ctx, const char *lease_id)
+__get_lease_id_entry(lease_inode_ctx_t *lease_ctx, const char *lease_id)
 {
-        lease_id_entry_t   *lease_entry    = NULL;
-        lease_id_entry_t   *tmp            = NULL;
-        lease_id_entry_t   *found          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("leases", lease_id, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-
-        list_for_each_entry_safe (lease_entry, tmp,
-                                  &lease_ctx->lease_id_list,
-                                  lease_id_list) {
-
-                if (__is_same_lease_id (lease_id, lease_entry->lease_id)) {
-                        found = lease_entry;
-                        gf_msg_debug ("leases", 0, "lease ID entry found "
-                                      "Client UID:%s, lease id:%s",
-                                      lease_entry->client_uid,
-                                      leaseid_utoa (lease_entry->lease_id));
-                        break;
-                }
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    lease_id_entry_t *found = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        if (__is_same_lease_id(lease_id, lease_entry->lease_id)) {
+            found = lease_entry;
+            gf_msg_debug("leases", 0,
+                         "lease ID entry found "
+                         "Client UID:%s, lease id:%s",
+                         lease_entry->client_uid,
+                         leaseid_utoa(lease_entry->lease_id));
+            break;
         }
+    }
 out:
-        return found;
+    return found;
 }
 
-
 /* Returns the lease_id_entry for a given lease_id and a given inode,
  * if none found creates one.
  * Return values:
  * lease_id_entry_t* - a pointer to the client entry
  */
 static lease_id_entry_t *
-__get_or_new_lease_entry (call_frame_t *frame, const char *lease_id,
-                           lease_inode_ctx_t *lease_ctx)
+__get_or_new_lease_entry(call_frame_t *frame, const char *lease_id,
+                         lease_inode_ctx_t *lease_ctx)
 {
-        lease_id_entry_t *lease_entry    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("leases", frame, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_id, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-
-        lease_entry = __get_lease_id_entry (lease_ctx, lease_id);
-        if (!lease_entry) { /* create one */
-                lease_entry = new_lease_id_entry (frame, lease_id);
-                if (!lease_entry)
-                        goto out;
-
-                list_add_tail (&lease_entry->lease_id_list,
-                               &lease_ctx->lease_id_list);
-
-                gf_msg_debug (frame->this->name, 0, "lease ID entry added,"
-                              " Client UID:%s, lease id:%s",
-                              lease_entry->client_uid,
-                              leaseid_utoa (lease_entry->lease_id));
-        }
+    lease_id_entry_t *lease_entry = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    lease_entry = __get_lease_id_entry(lease_ctx, lease_id);
+    if (!lease_entry) { /* create one */
+        lease_entry = new_lease_id_entry(frame, lease_id);
+        if (!lease_entry)
+            goto out;
+
+        list_add_tail(&lease_entry->lease_id_list, &lease_ctx->lease_id_list);
+
+        gf_msg_debug(frame->this->name, 0,
+                     "lease ID entry added,"
+                     " Client UID:%s, lease id:%s",
+                     lease_entry->client_uid,
+                     leaseid_utoa(lease_entry->lease_id));
+    }
 out:
-        return lease_entry;
+    return lease_entry;
 }
 
-
 static lease_inode_t *
-new_lease_inode (inode_t *inode)
+new_lease_inode(inode_t *inode)
 {
-        lease_inode_t *l_inode = NULL;
-
-        l_inode = GF_CALLOC (1, sizeof (*l_inode), gf_leases_mt_lease_inode_t);
-        if (!l_inode)
-                goto out;
+    lease_inode_t *l_inode = GF_MALLOC(sizeof(*l_inode),
+                                       gf_leases_mt_lease_inode_t);
+    if (!l_inode)
+        goto out;
 
-        INIT_LIST_HEAD (&l_inode->list);
-        l_inode->inode = inode_ref (inode);
+    INIT_LIST_HEAD(&l_inode->list);
+    l_inode->inode = inode_ref(inode);
 out:
-        return l_inode;
+    return l_inode;
 }
 
-
 static void
-__destroy_lease_inode (lease_inode_t *l_inode)
+__destroy_lease_inode(lease_inode_t *l_inode)
 {
-        list_del_init (&l_inode->list);
-        inode_unref (l_inode->inode);
-        GF_FREE (l_inode);
+    list_del_init(&l_inode->list);
+    inode_unref(l_inode->inode);
+    GF_FREE(l_inode);
 }
 
-
 static lease_client_t *
-new_lease_client (const char *client_uid)
+new_lease_client(const char *client_uid)
 {
-        lease_client_t  *clnt = NULL;
-
-        clnt = GF_CALLOC (1, sizeof (*clnt), gf_leases_mt_lease_client_t);
-        if (!clnt)
-                goto out;
-
-        INIT_LIST_HEAD (&clnt->client_list);
-        INIT_LIST_HEAD (&clnt->inode_list);
-        clnt->client_uid = gf_strdup (client_uid);
+    lease_client_t *clnt = GF_MALLOC(sizeof(*clnt),
+                                     gf_leases_mt_lease_client_t);
+    if (!clnt)
+        goto out;
+
+    INIT_LIST_HEAD(&clnt->client_list);
+    INIT_LIST_HEAD(&clnt->inode_list);
+    clnt->client_uid = gf_strdup(client_uid);
 out:
-        return clnt;
+    return clnt;
 }
 
-
 static void
-__destroy_lease_client (lease_client_t *clnt)
+__destroy_lease_client(lease_client_t *clnt)
 {
-         list_del_init (&clnt->inode_list);
-         list_del_init (&clnt->client_list);
-         GF_FREE (clnt);
+    list_del_init(&clnt->inode_list);
+    list_del_init(&clnt->client_list);
+    GF_FREE(clnt);
 
-         return;
+    return;
 }
 
-
 static lease_client_t *
-__get_lease_client (xlator_t *this, leases_private_t *priv,
-                    const char *client_uid)
+__get_lease_client(xlator_t *this, leases_private_t *priv,
+                   const char *client_uid)
 {
-        lease_client_t  *clnt  = NULL;
-        lease_client_t  *tmp   = NULL;
-        lease_client_t  *found = NULL;
-
-        list_for_each_entry_safe (clnt, tmp, &priv->client_list, client_list) {
-                if ((strcmp (clnt->client_uid, client_uid) == 0)) {
-                        found = clnt;
-                        gf_msg_debug (this->name, 0, "Client:%s already found "
-                                      "in the cleanup list", client_uid);
-                        break;
-                }
+    lease_client_t *clnt = NULL;
+    lease_client_t *tmp = NULL;
+    lease_client_t *found = NULL;
+
+    list_for_each_entry_safe(clnt, tmp, &priv->client_list, client_list)
+    {
+        if ((strcmp(clnt->client_uid, client_uid) == 0)) {
+            found = clnt;
+            gf_msg_debug(this->name, 0,
+                         "Client:%s already found "
+                         "in the cleanup list",
+                         client_uid);
+            break;
         }
-        return found;
+    }
+    return found;
 }
 
-
 static lease_client_t *
-__get_or_new_lease_client (xlator_t *this, leases_private_t *priv,
-                           const char *client_uid)
+__get_or_new_lease_client(xlator_t *this, leases_private_t *priv,
+                          const char *client_uid)
 {
-        lease_client_t  *found = NULL;
-
-        found = __get_lease_client (this, priv, client_uid);
-        if (!found) {
-                found = new_lease_client (client_uid);
-                if (!found)
-                        goto out;
-                list_add_tail (&found->client_list, &priv->client_list);
-                gf_msg_debug (this->name, 0, "Adding a new client:%s entry "
-                              "to the cleanup list", client_uid);
-        }
+    lease_client_t *found = NULL;
+
+    found = __get_lease_client(this, priv, client_uid);
+    if (!found) {
+        found = new_lease_client(client_uid);
+        if (!found)
+            goto out;
+        list_add_tail(&found->client_list, &priv->client_list);
+        gf_msg_debug(this->name, 0,
+                     "Adding a new client:%s entry "
+                     "to the cleanup list",
+                     client_uid);
+    }
 out:
-        return found;
+    return found;
 }
 
-
 static int
-add_inode_to_client_list (xlator_t *this, inode_t *inode, const char *client_uid)
+add_inode_to_client_list(xlator_t *this, inode_t *inode, const char *client_uid)
 {
-        int               ret         = 0;
-        leases_private_t *priv        = NULL;
-        lease_client_t   *clnt        = NULL;
-        lease_inode_t    *lease_inode = NULL;
-
-        priv = this->private;
-        pthread_mutex_lock (&priv->mutex);
-        {
-                clnt = __get_or_new_lease_client (this, priv, client_uid);
-                GF_CHECK_ALLOC (clnt, ret, out);
-
-                lease_inode = new_lease_inode (inode);
-                GF_CHECK_ALLOC (lease_inode, ret, out);
-
-                list_add_tail (&clnt->inode_list, &lease_inode->list);
-                gf_msg_debug (this->name, 0,
-                              "Added a new inode:%p to the client(%s) "
-                              "cleanup list, gfid(%s)", inode, client_uid,
-                              uuid_utoa (inode->gfid));
+    leases_private_t *priv = this->private;
+    lease_client_t *clnt = NULL;
+
+    lease_inode_t *lease_inode = new_lease_inode(inode);
+    if (!lease_inode)
+        return -ENOMEM;
+
+    pthread_mutex_lock(&priv->mutex);
+    {
+        clnt = __get_or_new_lease_client(this, priv, client_uid);
+        if (!clnt) {
+            pthread_mutex_unlock(&priv->mutex);
+            __destroy_lease_inode(lease_inode);
+            return -ENOMEM;
         }
-        pthread_mutex_unlock (&priv->mutex);
-out:
-        return ret;
+        list_add_tail(&clnt->inode_list, &lease_inode->list);
+    }
+    pthread_mutex_unlock(&priv->mutex);
+    gf_msg_debug(this->name, 0,
+                 "Added a new inode:%p to the client(%s) "
+                 "cleanup list, gfid(%s)",
+                 inode, client_uid, uuid_utoa(inode->gfid));
+    return 0;
 }
 
-
 /* Add lease entry to the corresponding client entry.
  * Return values:
  * 0 Success
  * -1 Failure
  */
 static int
-__add_lease (call_frame_t *frame, inode_t *inode, lease_inode_ctx_t *lease_ctx,
-             const char *client_uid, struct gf_lease *lease)
+__add_lease(call_frame_t *frame, inode_t *inode, lease_inode_ctx_t *lease_ctx,
+            const char *client_uid, struct gf_lease *lease)
 {
-        lease_id_entry_t  *lease_entry  = NULL;
-        int                ret          = -1;
-
-        GF_VALIDATE_OR_GOTO ("leases", frame, out);
-        GF_VALIDATE_OR_GOTO ("leases", client_uid, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-        GF_VALIDATE_OR_GOTO ("leases", inode, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease, out);
-
-        gf_msg_trace (frame->this->name, 0,
-                      "Granting lease lock to client %s with lease id %s"
-                      " on gfid(%s)", client_uid, leaseid_utoa (lease->lease_id),
-                      uuid_utoa (inode->gfid));
-
-        lease_entry = __get_or_new_lease_entry (frame, lease->lease_id, lease_ctx);
-        if (!lease_entry) {
-                errno = ENOMEM;
-                goto out;
-        }
-
-        lease_entry->lease_type_cnt[lease->lease_type]++;
-        lease_entry->lease_cnt++;
-        lease_entry->lease_type |= lease->lease_type;
-        /* If this is the first lease taken by the client on the file, then
-         * add this inode/file to the client disconnect cleanup list
-         */
-        if (lease_entry->lease_cnt == 1) {
-                add_inode_to_client_list (frame->this, inode, client_uid);
-        }
-
-        lease_ctx->lease_cnt++;
-        lease_ctx->lease_type_cnt[lease->lease_type]++;
-        lease_ctx->lease_type |= lease->lease_type;
-
-        /* Take a ref for the first lock taken on this inode. Corresponding
-         * unref when all the leases are unlocked or during DISCONNECT
-         * Ref is required because the inode on which lease is acquired should
-         * not be deleted when lru cleanup kicks in*/
-        if (lease_ctx->lease_cnt == 1) {
-                lease_ctx->inode = inode_ref (inode);
-        }
-
-        ret = 0;
+    lease_id_entry_t *lease_entry = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", client_uid, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+
+    gf_msg_trace(frame->this->name, 0,
+                 "Granting lease lock to client %s with lease id %s"
+                 " on gfid(%s)",
+                 client_uid, leaseid_utoa(lease->lease_id),
+                 uuid_utoa(inode->gfid));
+
+    lease_entry = __get_or_new_lease_entry(frame, lease->lease_id, lease_ctx);
+    if (!lease_entry) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    lease_entry->lease_type_cnt[lease->lease_type]++;
+    lease_entry->lease_cnt++;
+    lease_entry->lease_type |= lease->lease_type;
+    /* If this is the first lease taken by the client on the file, then
+     * add this inode/file to the client disconnect cleanup list
+     */
+    if (lease_entry->lease_cnt == 1) {
+        add_inode_to_client_list(frame->this, inode, client_uid);
+    }
+
+    lease_ctx->lease_cnt++;
+    lease_ctx->lease_type_cnt[lease->lease_type]++;
+    lease_ctx->lease_type |= lease->lease_type;
+
+    /* Take a ref for the first lock taken on this inode. Corresponding
+     * unref when all the leases are unlocked or during DISCONNECT
+     * Ref is required because the inode on which lease is acquired should
+     * not be deleted when lru cleanup kicks in*/
+    if (lease_ctx->lease_cnt == 1) {
+        lease_ctx->inode = inode_ref(inode);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 static gf_boolean_t
-__is_clnt_lease_none (const char *client_uid, lease_inode_ctx_t *lease_ctx)
+__is_clnt_lease_none(const char *client_uid, lease_inode_ctx_t *lease_ctx)
 {
-        gf_boolean_t      lease_none  = _gf_true;
-        lease_id_entry_t *lease_entry = NULL;
-        lease_id_entry_t *tmp         = NULL;
-
-        list_for_each_entry_safe (lease_entry, tmp,
-                                  &lease_ctx->lease_id_list,
-                                  lease_id_list) {
-                if ((strcmp (client_uid, lease_entry->client_uid) == 0)
-                    && (lease_entry->lease_cnt != 0)) {
-                        lease_none = _gf_false;
-                        break;
-                }
+    gf_boolean_t lease_none = _gf_true;
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        if ((strcmp(client_uid, lease_entry->client_uid) == 0) &&
+            (lease_entry->lease_cnt != 0)) {
+            lease_none = _gf_false;
+            break;
         }
+    }
 
-        return lease_none;
+    return lease_none;
 }
 
 static int
-__remove_inode_from_clnt_list (xlator_t *this, lease_client_t *clnt,
-                               inode_t *inode)
+__remove_inode_from_clnt_list(xlator_t *this, lease_client_t *clnt,
+                              inode_t *inode)
 {
-        int               ret     = -1;
-        lease_inode_t    *l_inode = NULL;
-        lease_inode_t    *tmp1    = NULL;
-
-        list_for_each_entry_safe (l_inode, tmp1,
-                                  &clnt->inode_list,
-                                  list) {
-                if (l_inode->inode == inode) {
-                        __destroy_lease_inode (l_inode);
-                        gf_msg_debug (this->name, 0,
-                                      "Removed the inode from the client cleanup list");
-                        ret = 0;
-                }
+    int ret = -1;
+    lease_inode_t *l_inode = NULL;
+    lease_inode_t *tmp1 = NULL;
+
+    list_for_each_entry_safe(l_inode, tmp1, &clnt->inode_list, list)
+    {
+        if (l_inode->inode == inode) {
+            __destroy_lease_inode(l_inode);
+            gf_msg_debug(this->name, 0,
+                         "Removed the inode from the client cleanup list");
+            ret = 0;
         }
-        /* TODO: Remove the client entry from the cleanup list */
+    }
+    /* TODO: Remove the client entry from the cleanup list */
 
-        return ret;
+    return ret;
 }
 
-
 static int
-remove_from_clnt_list (xlator_t *this, const char *client_uid, inode_t *inode)
+remove_from_clnt_list(xlator_t *this, const char *client_uid, inode_t *inode)
 {
-        leases_private_t *priv    = NULL;
-        int               ret     = -1;
-        lease_client_t   *clnt    = NULL;
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                clnt = __get_lease_client (this, priv, client_uid);
-                if (!clnt) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, LEASE_MSG_CLNT_NOTFOUND,
-                                "There is no client entry found in the cleanup list");
-                        pthread_mutex_unlock (&priv->mutex);
-                        goto out;
-                }
-                ret = __remove_inode_from_clnt_list (this, clnt, inode);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, LEASE_MSG_INODE_NOTFOUND,
-                                "There is no inode entry found in the cleanup list");
-                }
+    leases_private_t *priv = NULL;
+    int ret = -1;
+    lease_client_t *clnt = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    pthread_mutex_lock(&priv->mutex);
+    {
+        clnt = __get_lease_client(this, priv, client_uid);
+        if (!clnt) {
+            pthread_mutex_unlock(&priv->mutex);
+            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_CLNT_NOTFOUND,
+                   "There is no client entry found in the cleanup list");
+            goto out;
         }
-        pthread_mutex_unlock (&priv->mutex);
+        ret = __remove_inode_from_clnt_list(this, clnt, inode);
+        if (ret) {
+            pthread_mutex_unlock(&priv->mutex);
+            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INODE_NOTFOUND,
+                   "There is no inode entry found in the cleanup list");
+            goto out;
+        }
+    }
+    pthread_mutex_unlock(&priv->mutex);
 out:
-        return ret;
+    return ret;
 }
 
-
 /* Remove lease entry in the corresponding client entry.
  */
 static int
-__remove_lease (xlator_t *this, inode_t *inode, lease_inode_ctx_t *lease_ctx,
-                const char *client_uid, struct gf_lease *lease)
+__remove_lease(xlator_t *this, inode_t *inode, lease_inode_ctx_t *lease_ctx,
+               const char *client_uid, struct gf_lease *lease)
 {
-        lease_id_entry_t    *lease_entry     = NULL;
-        int                  ret             = 0;
-        int32_t              lease_type      = 0;
-        leases_private_t    *priv            = NULL;
-
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease, out);
-
-        priv = this->private;
-
-        gf_msg_trace (this->name, 0, "Removing lease entry for client: %s, "
-                      "lease type:%d, lease id:%s", client_uid, lease->lease_type,
-                      leaseid_utoa (lease->lease_id));
-
-        lease_entry = __get_lease_id_entry (lease_ctx, lease->lease_id);
-        if (!lease_entry) {
-                gf_msg (this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_UNLK_LEASE,
-                        "Got unlock lease request from client:%s, but has no "
-                        "corresponding lock", client_uid);
-                ret = -EINVAL;
-                errno = EINVAL;
-                goto out;
-        }
-
-        lease_type = lease->lease_type;
-        lease_entry->lease_type_cnt[lease_type]--;
-        lease_entry->lease_cnt--;
-
-        lease_ctx->lease_type_cnt[lease_type]--;
-        lease_ctx->lease_cnt--;
-
-        if (lease_entry->lease_type_cnt[lease_type] == 0)
-                lease_entry->lease_type = lease_entry->lease_type & (~lease_type);
-
-        if (lease_ctx->lease_type_cnt[lease_type] == 0)
-                lease_ctx->lease_type = lease_ctx->lease_type & (~lease_type);
-
-        if (lease_entry->lease_cnt == 0) {
-                if (__is_clnt_lease_none (client_uid, lease_ctx)) {
-                        gf_msg_debug (this->name, 0, "Client(%s) has no leases"
-                                      " on gfid (%s), hence removing the inode"
-                                      " from the client cleanup list",
-                                      client_uid, uuid_utoa (inode->gfid));
-                        remove_from_clnt_list (this, client_uid, lease_ctx->inode);
-                }
-                __destroy_lease_id_entry (lease_entry);
+    lease_id_entry_t *lease_entry = NULL;
+    int ret = 0;
+    int32_t lease_type = 0;
+    leases_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+
+    priv = this->private;
+
+    gf_msg_trace(this->name, 0,
+                 "Removing lease entry for client: %s, "
+                 "lease type:%d, lease id:%s",
+                 client_uid, lease->lease_type, leaseid_utoa(lease->lease_id));
+
+    /* There could be a race where in server recalled the lease and by the time
+     * client sends lease_unlock request, server may have revoked it. To handle
+     * such cases, if lease doesnt exist treat it as noop and return success.
+     */
+    lease_entry = __get_lease_id_entry(lease_ctx, lease->lease_id);
+    if (!lease_entry) {
+        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_UNLK_LEASE,
+               "Got unlock lease request from client:%s, but has no "
+               "corresponding lock",
+               client_uid);
+        ret = 0;
+        goto out;
+    }
+
+    if (!(lease_entry->lease_type & lease->lease_type)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_UNLK_LEASE,
+               "Got unlock lease request from client:%s for an invalid "
+               "lease_type",
+               client_uid);
+        ret = -EINVAL;
+        errno = EINVAL;
+        goto out;
+    }
+    lease_type = lease->lease_type;
+    lease_entry->lease_type_cnt[lease_type]--;
+    lease_entry->lease_cnt--;
+
+    lease_ctx->lease_type_cnt[lease_type]--;
+    lease_ctx->lease_cnt--;
+
+    if (lease_entry->lease_type_cnt[lease_type] == 0)
+        lease_entry->lease_type = lease_entry->lease_type & (~lease_type);
+
+    if (lease_ctx->lease_type_cnt[lease_type] == 0)
+        lease_ctx->lease_type = lease_ctx->lease_type & (~lease_type);
+
+    if (lease_entry->lease_cnt == 0) {
+        if (__is_clnt_lease_none(client_uid, lease_ctx)) {
+            gf_msg_trace(this->name, 0,
+                         "Client(%s) has no leases"
+                         " on gfid (%s), hence removing the inode"
+                         " from the client cleanup list",
+                         client_uid, uuid_utoa(inode->gfid));
+            remove_from_clnt_list(this, client_uid, lease_ctx->inode);
         }
+        __destroy_lease_id_entry(lease_entry);
+        lease_ctx->blocked_fops_resuming = _gf_true;
+    }
 
-        if (lease_ctx->lease_cnt == 0 && lease_ctx->timer) {
-                ret = gf_tw_del_timer (priv->timer_wheel, lease_ctx->timer);
-                lease_ctx->recall_in_progress = _gf_false;
-        }
+    if (lease_ctx->lease_cnt == 0 && lease_ctx->timer) {
+        ret = gf_tw_del_timer(priv->timer_wheel, lease_ctx->timer);
+        lease_ctx->recall_in_progress = _gf_false;
+        lease_ctx->timer = NULL;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static gf_boolean_t
-__is_lease_grantable (xlator_t *this, lease_inode_ctx_t *lease_ctx,
-                      struct gf_lease *lease, inode_t *inode)
+__is_lease_grantable(xlator_t *this, lease_inode_ctx_t *lease_ctx,
+                     struct gf_lease *lease, inode_t *inode)
 {
-        uint32_t        fd_count     = 0;
-        int32_t         flags        = 0;
-        fd_t           *iter_fd      = NULL;
-        gf_boolean_t    grant        = _gf_false;
-        int             ret          = 0;
-        lease_fd_ctx_t *fd_ctx       = NULL;
-        uint64_t        ctx          = 0;
-
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease, out);
-        GF_VALIDATE_OR_GOTO ("leases", inode, out);
-
-        if (lease_ctx->recall_in_progress) {
-                gf_msg_debug (this->name, 0, "Recall in progress, hence "
-                              "failing the lease request");
+    uint32_t fd_count = 0;
+    int32_t flags = 0;
+    fd_t *iter_fd = NULL;
+    gf_boolean_t grant = _gf_false;
+    int ret = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+    uint64_t ctx = 0;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+
+    if (lease_ctx->recall_in_progress) {
+        gf_msg_debug(this->name, 0,
+                     "Recall in progress, hence "
+                     "failing the lease request");
+        grant = _gf_false;
+        goto out;
+    }
+
+    if (lease_ctx->blocked_fops_resuming) {
+        gf_msg_debug(this->name, 0,
+                     "Previously blocked fops resuming, hence "
+                     "failing the lease request");
+        grant = _gf_false;
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(iter_fd, this, &ctx);
+            if (ret < 0) {
                 grant = _gf_false;
+                UNLOCK(&inode->lock);
+                gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INVAL_FD_CTX,
+                       "Unable to get fd ctx");
                 goto out;
+            }
+            fd_ctx = (lease_fd_ctx_t *)(long)ctx;
+
+            /* Check for open fd conflict, note that open fds from
+             * the same lease id is not checked for conflict, as it is
+             * lease id based lease.
+             */
+            if (fd_ctx->client_uid != NULL &&
+                !__is_same_lease_id(fd_ctx->lease_id, lease->lease_id)) {
+                fd_count++;
+                flags |= iter_fd->flags;
+            }
         }
+    }
+    UNLOCK(&inode->lock);
 
-        LOCK (&inode->lock);
-        {
-                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                        ret = fd_ctx_get (iter_fd, this, &ctx);
-                        if (ret < 0) {
-                                grant = _gf_false;
-                                UNLOCK (&inode->lock);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        LEASE_MSG_INVAL_FD_CTX,
-                                        "Unable to get fd ctx");
-                                goto out;
-                        }
-                        fd_ctx = (lease_fd_ctx_t *)(long) ctx;
-
-                        /* Check for open fd conflict, note that open fds from
-                         * the same lease id is not checked for conflict, as it is
-                         * lease id based lease.
-                         */
-                        if (!__is_same_lease_id (fd_ctx->lease_id, lease->lease_id)) {
-                                fd_count++;
-                                flags |= iter_fd->flags;
-                        }
-                }
-        }
-        UNLOCK (&inode->lock);
-
-        gf_msg_debug (this->name, 0, "open fd count:%d flags:%d",
-                      fd_count, flags);
+    gf_msg_debug(this->name, 0, "open fd count:%d flags:%d", fd_count, flags);
 
-        __dump_leases_info (this, lease_ctx);
+    __dump_leases_info(this, lease_ctx);
 
-        switch (lease->lease_type) {
+    switch (lease->lease_type) {
         case GF_RD_LEASE:
-                /* check open fd conflict */
-                if ((fd_count > 0) && ((flags & O_WRONLY) || (flags & O_RDWR))) {
-                        grant = _gf_false;
-                        break;
-                }
-
-                /* check for conflict with existing leases */
-                if (lease_ctx->lease_type == NONE ||
-                    lease_ctx->lease_type == GF_RD_LEASE ||
-                    !(__another_lease_found (lease_ctx, lease->lease_id)))
-                        grant = _gf_true;
-                else
-                        grant = _gf_false;
+            /* check open fd conflict */
+            if ((fd_count > 0) && ((flags & O_WRONLY) || (flags & O_RDWR))) {
+                grant = _gf_false;
                 break;
+            }
+
+            /* check for conflict with existing leases */
+            if (lease_ctx->lease_type == NONE ||
+                lease_ctx->lease_type == GF_RD_LEASE ||
+                !(__another_lease_found(lease_ctx, lease->lease_id)))
+                grant = _gf_true;
+            else
+                grant = _gf_false;
+            break;
 
         case GF_RW_LEASE:
-                /* check open fd conflict; conflict if there are any fds open
-                 * other than the client on which the lease is requested. */
-                if (fd_count > 0) {
-                        grant = _gf_false;
-                        break;
-                }
-
-                /* check existing lease conflict */
-                if (lease_ctx->lease_type == NONE ||
-                    !(__another_lease_found (lease_ctx, lease->lease_id)))
-                        grant = _gf_true;
-                else
-                        grant = _gf_false;
+            /* check open fd conflict; conflict if there are any fds open
+             * other than the client on which the lease is requested. */
+            if (fd_count > 0) {
+                grant = _gf_false;
                 break;
+            }
+
+            /* check existing lease conflict */
+            if (lease_ctx->lease_type == NONE ||
+                !(__another_lease_found(lease_ctx, lease->lease_id)))
+                grant = _gf_true;
+            else
+                grant = _gf_false;
+            break;
 
         default:
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_INVAL_LEASE_TYPE,
-                        "Invalid lease type specified");
-                break;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_INVAL_LEASE_TYPE,
+                   "Invalid lease type specified");
+            break;
+    }
 out:
-        return grant;
+    return grant;
 }
 
-
 static void
-do_blocked_fops (xlator_t *this, lease_inode_ctx_t *lease_ctx)
+do_blocked_fops(xlator_t *this, lease_inode_ctx_t *lease_ctx)
 {
-        struct list_head   wind_list;
-        fop_stub_t        *blk_fop  = NULL;
-        fop_stub_t        *tmp      = NULL;
-
-        INIT_LIST_HEAD (&wind_list);
-
-        pthread_mutex_lock (&lease_ctx->lock);
-        {
-                list_for_each_entry_safe (blk_fop, tmp,
-                                          &lease_ctx->blocked_list, list) {
-                        list_del_init (&blk_fop->list);
-                        list_add_tail (&blk_fop->list, &wind_list);
-                }
+    struct list_head wind_list;
+    fop_stub_t *blk_fop = NULL;
+    fop_stub_t *tmp = NULL;
+
+    INIT_LIST_HEAD(&wind_list);
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        if (!lease_ctx->blocked_fops_resuming) {
+            /* lease_ctx->blocked_fops_resuming will be set
+             * only when the last lease is released. That
+             * is when we need to resume blocked fops and unref
+             * the inode taken in __add_lease (when lease_cnt == 1).
+             * Return otherwise.
+             */
+            pthread_mutex_unlock(&lease_ctx->lock);
+            return;
         }
-        pthread_mutex_unlock (&lease_ctx->lock);
 
-        gf_msg_trace (this->name, 0, "Executing the blocked stubs on gfid(%s)",
-                      uuid_utoa (lease_ctx->inode->gfid));
-
-        list_for_each_entry_safe (blk_fop, tmp, &wind_list, list) {
-                list_del_init (&blk_fop->list);
-                gf_msg_trace (this->name, 0, "Executing fop:%d", blk_fop->stub->fop);
-                call_resume (blk_fop->stub);
-                GF_FREE (blk_fop);
-        }
-
-        pthread_mutex_lock (&lease_ctx->lock);
+        list_for_each_entry_safe(blk_fop, tmp, &lease_ctx->blocked_list, list)
         {
-                lease_ctx->lease_type = NONE;
-                inode_unref (lease_ctx->inode);
-                lease_ctx->inode = NULL;
+            list_del_init(&blk_fop->list);
+            list_add_tail(&blk_fop->list, &wind_list);
         }
-        pthread_mutex_unlock (&lease_ctx->lock);
-
-        return;
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+
+    gf_msg_trace(this->name, 0, "Executing the blocked stubs on gfid(%s)",
+                 uuid_utoa(lease_ctx->inode->gfid));
+    list_for_each_entry_safe(blk_fop, tmp, &wind_list, list)
+    {
+        list_del_init(&blk_fop->list);
+        gf_msg_trace(this->name, 0, "Executing fop:%d", blk_fop->stub->fop);
+        call_resume(blk_fop->stub);
+        GF_FREE(blk_fop);
+    }
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        lease_ctx->lease_type = NONE;
+        /* unref the inode taken in __add_lease
+         * (when lease_cnt == 1) */
+        lease_ctx->blocked_fops_resuming = _gf_false;
+        inode_unref(lease_ctx->inode);
+        lease_ctx->inode = NULL;
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+
+    return;
 }
 
-
 void
-recall_lease_timer_handler (struct gf_tw_timer_list *timer,
-                            void *data, unsigned long calltime)
+recall_lease_timer_handler(struct gf_tw_timer_list *timer, void *data,
+                           unsigned long calltime)
 {
-        inode_t                *inode       = NULL;
-        lease_inode_t          *lease_inode = NULL;
-        leases_private_t       *priv        = NULL;
-        lease_timer_data_t     *timer_data    = NULL;
-
-        timer_data = data;
-
-        priv = timer_data->this->private;
-        inode = timer_data->inode;
-        pthread_mutex_lock (&priv->mutex);
-        {
-                lease_inode = new_lease_inode (inode);
-                if (!lease_inode) {
-                        errno = ENOMEM;
-                        goto out;
-                }
-                list_add_tail (&lease_inode->list, &priv->recall_list);
-                pthread_cond_broadcast (&priv->cond);
-        }
+    inode_t *inode = NULL;
+    lease_inode_t *lease_inode = NULL;
+    leases_private_t *priv = NULL;
+    lease_timer_data_t *timer_data = NULL;
+
+    timer_data = data;
+
+    priv = timer_data->this->private;
+    inode = timer_data->inode;
+    lease_inode = new_lease_inode(inode);
+    if (!lease_inode) {
+        errno = ENOMEM;
+        goto out;
+    }
+    pthread_mutex_lock(&priv->mutex);
+    {
+        list_add_tail(&lease_inode->list, &priv->recall_list);
+        pthread_cond_broadcast(&priv->cond);
+    }
+    pthread_mutex_unlock(&priv->mutex);
 out:
-        pthread_mutex_unlock (&priv->mutex);
+    /* unref the inode_ref taken by timer_data in __recall_lease */
+    inode_unref(timer_data->inode);
 
-        GF_FREE (timer);
+    GF_FREE(timer);
 }
 
-
 static void
-__recall_lease (xlator_t *this, lease_inode_ctx_t *lease_ctx)
+__recall_lease(xlator_t *this, lease_inode_ctx_t *lease_ctx)
 {
-        lease_id_entry_t                  *lease_entry   = NULL;
-        lease_id_entry_t                  *tmp           = NULL;
-        struct gf_upcall                   up_req        = {0,};
-        struct gf_upcall_recall_lease      recall_req    = {0,};
-        int                                notify_ret    = -1;
-        struct gf_tw_timer_list           *timer         = NULL;
-        leases_private_t                  *priv          = NULL;
-        lease_timer_data_t                *timer_data    = NULL;
-
-        if (lease_ctx->recall_in_progress) {
-                gf_msg_debug (this->name, 0, "Lease recall is already in "
-                              "progress, hence not sending another recall");
-                goto out;
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    struct gf_upcall up_req = {
+        0,
+    };
+    struct gf_upcall_recall_lease recall_req = {
+        0,
+    };
+    int notify_ret = -1;
+    struct gf_tw_timer_list *timer = NULL;
+    leases_private_t *priv = NULL;
+    lease_timer_data_t *timer_data = NULL;
+    time_t recall_time;
+
+    if (lease_ctx->recall_in_progress) {
+        gf_msg_debug(this->name, 0,
+                     "Lease recall is already in "
+                     "progress, hence not sending another recall");
+        goto out;
+    }
+
+    priv = this->private;
+    recall_time = gf_time();
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        gf_uuid_copy(up_req.gfid, lease_ctx->inode->gfid);
+        up_req.client_uid = lease_entry->client_uid;
+        up_req.event_type = GF_UPCALL_RECALL_LEASE;
+        up_req.data = &recall_req;
+
+        notify_ret = this->notify(this, GF_EVENT_UPCALL, &up_req);
+        if (notify_ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_RECALL_FAIL,
+                   "Recall notification to client: %s failed",
+                   lease_entry->client_uid);
+            /* Do not return from here, continue registering the timer,
+               this is required mostly o keep replicas in sync*/
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Recall lease (all)"
+                         "notification sent to client %s",
+                         lease_entry->client_uid);
         }
 
-        priv = this->private;
-        list_for_each_entry_safe (lease_entry, tmp,
-                                  &lease_ctx->lease_id_list,
-                                  lease_id_list) {
-                gf_uuid_copy (up_req.gfid, lease_ctx->inode->gfid);
-                up_req.client_uid = lease_entry->client_uid;
-                up_req.event_type = GF_UPCALL_RECALL_LEASE;
-                up_req.data = &recall_req;
-
-                notify_ret = this->notify (this, GF_EVENT_UPCALL, &up_req);
-                if (notify_ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, LEASE_MSG_RECALL_FAIL,
-                                "Recall notification to client: %s failed",
-                                lease_entry->client_uid);
-                        /* Do not return from here, continue registering the timer,
-                           this is required mostly o keep replicas in sync*/
-                } else {
-                        gf_msg_debug (this->name, 0, "Recall lease (all)"
-                                      "notification sent to client %s",
-                                      lease_entry->client_uid);
-                }
-
-                lease_ctx->recall_in_progress = _gf_true;
-                lease_entry->recall_time = time (NULL);
-        }
-        timer = GF_CALLOC (1, sizeof (*timer),
-                           gf_common_mt_tw_timer_list);
-        if (!timer) {
-                goto out;
-        }
-        timer_data = GF_CALLOC (1, sizeof (*timer_data),
-                                gf_leases_mt_timer_data_t);
-        if (!timer_data) {
-                GF_FREE (timer);
-                goto out;
-        }
-
-        timer_data->inode = inode_ref (lease_ctx->inode);
-        timer_data->this = this;
-        timer->data = timer_data;
-
-        INIT_LIST_HEAD (&timer->entry);
-        timer->expires = get_recall_lease_timeout (this);
-        timer->function = recall_lease_timer_handler;
-        lease_ctx->timer = timer;
-        gf_tw_add_timer (priv->timer_wheel, timer);
-        gf_msg_trace (this->name, 0, "Registering timer " "%p, after "
-                      "sending recall", timer);
+        lease_ctx->recall_in_progress = _gf_true;
+        lease_entry->recall_time = recall_time;
+    }
+    timer = GF_MALLOC(sizeof(*timer), gf_common_mt_tw_timer_list);
+    if (!timer) {
+        goto out;
+    }
+    timer_data = GF_MALLOC(sizeof(lease_timer_data_t),
+                           gf_leases_mt_timer_data_t);
+    if (!timer_data) {
+        GF_FREE(timer);
+        goto out;
+    }
+
+    timer_data->inode = inode_ref(lease_ctx->inode);
+    timer_data->this = this;
+    timer->data = timer_data;
+
+    INIT_LIST_HEAD(&timer->entry);
+    timer->expires = get_recall_lease_timeout(this);
+    timer->function = recall_lease_timer_handler;
+    lease_ctx->timer = timer;
+    gf_tw_add_timer(priv->timer_wheel, timer);
+    gf_msg_trace(this->name, 0,
+                 "Registering timer "
+                 "%p, after "
+                 "sending recall",
+                 timer);
 out:
-        return;
+    return;
 }
 
-
 /* ret = 0; STACK_UNWIND Success
  * ret = -1; STACK_UNWIND failure
  */
 int
-process_lease_req (call_frame_t *frame, xlator_t *this,
-                   inode_t *inode, struct gf_lease *lease)
+process_lease_req(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  struct gf_lease *lease)
 {
-        int                 ret             = 0;
-        char               *client_uid      = NULL;
-        lease_inode_ctx_t  *lease_ctx       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("leases", frame, out);
-        GF_VALIDATE_OR_GOTO ("leases", this, out);
-        GF_VALIDATE_OR_GOTO ("leases", inode, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease, out);
-
-        client_uid = frame->root->client->client_uid;
-
-        if (!is_valid_lease_id (lease->lease_id)) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        LEASE_MSG_INVAL_LEASE_ID, "Invalid lease id, from"
-                        "client:%s", client_uid);
-                ret = -EINVAL;
-                errno = EINVAL;
-                goto out;
-        }
-
-        lease_ctx = lease_ctx_get (inode, this);
-        if (!lease_ctx) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        LEASE_MSG_NO_MEM, "Unable to create/get inode ctx, "
-                        "inode:%p", inode);
-                ret = -ENOMEM;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "Lease request from client: %s, "
-                      "lease type:%d, lease cmd:%d, lease ID:%s, gfid:%s",
-                      client_uid, lease->lease_type, lease->cmd,
-                      leaseid_utoa (lease->lease_id), uuid_utoa (inode->gfid));
+    int ret = 0;
+    char *client_uid = NULL;
+    lease_inode_ctx_t *lease_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+
+    client_uid = frame->root->client->client_uid;
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_INVAL_LEASE_ID,
+               "Invalid lease id, from"
+               "client:%s",
+               client_uid);
+        ret = -EINVAL;
+        errno = EINVAL;
+        goto out;
+    }
+
+    lease_ctx = lease_ctx_get(inode, this);
+    if (!lease_ctx) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "Unable to create/get inode ctx, "
+               "inode:%p",
+               inode);
+        ret = -ENOMEM;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Lease request from client: %s, "
+                 "lease type:%d, lease cmd:%d, lease ID:%s, gfid:%s",
+                 client_uid, lease->lease_type, lease->cmd,
+                 leaseid_utoa(lease->lease_id), uuid_utoa(inode->gfid));
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        switch (lease->cmd) {
+            case GF_GET_LEASE:
+                lease->lease_type = lease_ctx->lease_type;
+                gf_msg_debug(this->name, 0,
+                             "Get lease, existing lease"
+                             "type: %d",
+                             lease_ctx->lease_type);
+                /*TODO:Should it consider lease id or client_uid?*/
+                break;
 
-        pthread_mutex_lock (&lease_ctx->lock);
-        {
-                switch (lease->cmd) {
-                case GF_GET_LEASE:
-                        lease->lease_type = lease_ctx->lease_type;
-                        gf_msg_debug (this->name, 0, "Get lease, existing lease"
-                                      "type: %d", lease_ctx->lease_type);
-                        /*TODO:Should it consider lease id or client_uid?*/
-                        break;
-
-                case GF_SET_LEASE:
-                        if (__is_lease_grantable (this, lease_ctx, lease, inode)) {
-                                __add_lease (frame, inode, lease_ctx,
-                                             client_uid, lease);
-                                ret = 0;
-                        } else {
-                                gf_msg_debug (this->name, GF_LOG_DEBUG,
-                                              "Not granting the conflicting lease"
-                                              " request from %s on gfid(%s)",
-                                              client_uid, uuid_utoa (inode->gfid));
-                                __recall_lease (this, lease_ctx);
-                                ret = -1;
-                        }
-                        break;
-                case GF_UNLK_LEASE:
-                        ret = __remove_lease (this, inode, lease_ctx,
-                                              client_uid, lease);
-                        if ((ret == 0) && (lease_ctx->lease_cnt == 0)) {
-                                pthread_mutex_unlock (&lease_ctx->lock);
-                                goto unblock;
-                        }
-                        break;
-                default:
-                        ret = -EINVAL;
-                        break;
+            case GF_SET_LEASE:
+                if (__is_lease_grantable(this, lease_ctx, lease, inode)) {
+                    __add_lease(frame, inode, lease_ctx, client_uid, lease);
+                    ret = 0;
+                } else {
+                    gf_msg_debug(this->name, GF_LOG_DEBUG,
+                                 "Not granting the conflicting lease"
+                                 " request from %s on gfid(%s)",
+                                 client_uid, uuid_utoa(inode->gfid));
+                    __recall_lease(this, lease_ctx);
+                    ret = -1;
+                }
+                break;
+            case GF_UNLK_LEASE:
+                ret = __remove_lease(this, inode, lease_ctx, client_uid, lease);
+                if ((ret >= 0) && (lease_ctx->lease_cnt == 0)) {
+                    pthread_mutex_unlock(&lease_ctx->lock);
+                    goto unblock;
                 }
+                break;
+            default:
+                ret = -EINVAL;
+                break;
         }
-        pthread_mutex_unlock (&lease_ctx->lock);
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
 
-        return ret;
+    return ret;
 
 unblock:
-        do_blocked_fops (this, lease_ctx);
+    do_blocked_fops(this, lease_ctx);
 out:
-        return ret;
+    return ret;
 }
 
-
 /* ret = 1 conflict
  * ret = 0 no conflict
  */
 gf_boolean_t
-__check_lease_conflict (call_frame_t *frame, lease_inode_ctx_t *lease_ctx,
-                        const char *lease_id, gf_boolean_t is_write)
+__check_lease_conflict(call_frame_t *frame, lease_inode_ctx_t *lease_ctx,
+                       const char *lease_id, gf_boolean_t is_write)
 {
-        gf_lease_types_t   lease_type      = {0,};
-        gf_boolean_t       conflicts       = _gf_false;
-        lease_id_entry_t  *lease_entry     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("leases", frame, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_ctx, out);
-        GF_VALIDATE_OR_GOTO ("leases", lease_id, out);
-
-        lease_type = lease_ctx->lease_type;
-
-        /* If the fop is rename or unlink conflict the lease even if its
-         * from the same client??
-         */
-        if ((frame->root->op == GF_FOP_RENAME) ||
-            (frame->root->op == GF_FOP_UNLINK)) {
-                conflicts = _gf_true;
-                goto recall;
-        }
-
-        /* TODO: If lease_id is not sent, fall back to client uid conflict check?
-         * Or set conflicts = true if lease_id is 0 when there is an existing
-         * lease */
-        switch (lease_type) {
+    gf_lease_types_t lease_type = {
+        0,
+    };
+    gf_boolean_t conflicts = _gf_false;
+    lease_id_entry_t *lease_entry = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    lease_type = lease_ctx->lease_type;
+
+    /* If the fop is rename or unlink conflict the lease even if its
+     * from the same client??
+     */
+    if ((frame->root->op == GF_FOP_RENAME) ||
+        (frame->root->op == GF_FOP_UNLINK)) {
+        conflicts = _gf_true;
+        goto recall;
+    }
+
+    /* As internal fops are used to maintain data integrity but do not
+     * make modififications to the client data, no need to conflict with
+     * them.
+     *
+     * @todo: like for locks, even lease state has to be handled by
+     * rebalance or self-heal daemon process. */
+    if (frame->root->pid < 0) {
+        conflicts = _gf_false;
+        goto recall;
+    }
+
+    /* If lease_id is not sent, set conflicts = true if there is
+     * an existing lease */
+    if (!lease_id && (lease_ctx->lease_cnt > 0)) {
+        conflicts = _gf_true;
+        goto recall;
+    }
+
+    switch (lease_type) {
         case (GF_RW_LEASE | GF_RD_LEASE):
         case GF_RW_LEASE:
-                lease_entry = __get_lease_id_entry (lease_ctx, lease_id);
-                if (lease_entry && (lease_entry->lease_type & GF_RW_LEASE))
-                        conflicts = _gf_false;
-                else
-                        conflicts = _gf_true;
-                break;
+            lease_entry = __get_lease_id_entry(lease_ctx, lease_id);
+            if (lease_entry && (lease_entry->lease_type & GF_RW_LEASE))
+                conflicts = _gf_false;
+            else
+                conflicts = _gf_true;
+            break;
         case GF_RD_LEASE:
-                if (is_write && __another_lease_found(lease_ctx, lease_id))
-                        conflicts = _gf_true;
-                else
-                        conflicts = _gf_false;
-                break;
+            if (is_write && __another_lease_found(lease_ctx, lease_id))
+                conflicts = _gf_true;
+            else
+                conflicts = _gf_false;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
 recall:
-        /* If there is a conflict found and recall is not already sent to all
-         * the clients, then send recall to each of the client holding lease.
-         */
-        if (conflicts)
-                __recall_lease (frame->this, lease_ctx);
+    /* If there is a conflict found and recall is not already sent to all
+     * the clients, then send recall to each of the client holding lease.
+     */
+    if (conflicts)
+        __recall_lease(frame->this, lease_ctx);
 out:
-        return conflicts;
+    return conflicts;
 }
 
-
 /* Return values:
  * -1 : error, unwind the fop
  * WIND_FOP: No conflict, wind the fop
  * BLOCK_FOP: Found a conflicting lease, block the fop
  */
 int
-check_lease_conflict (call_frame_t *frame, inode_t *inode,
-                      const char *lease_id, uint32_t fop_flags)
+check_lease_conflict(call_frame_t *frame, inode_t *inode, const char *lease_id,
+                     uint32_t fop_flags)
 {
-        lease_inode_ctx_t       *lease_ctx  = NULL;
-        gf_boolean_t       is_blocking_fop  = _gf_false;
-        gf_boolean_t       is_write_fop     = _gf_false;
-        gf_boolean_t       conflicts        = _gf_false;
-        int                ret              = -1;
-
-        lease_ctx = lease_ctx_get (inode, frame->this);
-        if (!lease_ctx) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, ENOMEM,
-                        LEASE_MSG_NO_MEM,
-                        "Unable to create/get inode ctx");
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+    lease_inode_ctx_t *lease_ctx = NULL;
+    gf_boolean_t is_blocking_fop = _gf_false;
+    gf_boolean_t is_write_fop = _gf_false;
+    gf_boolean_t conflicts = _gf_false;
+    int ret = WIND_FOP;
+
+    lease_ctx = lease_ctx_get(inode, frame->this);
+    if (!lease_ctx) {
+        gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "Unable to create/get inode ctx");
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    is_blocking_fop = ((fop_flags & BLOCKING_FOP) != 0);
+    is_write_fop = ((fop_flags & DATA_MODIFY_FOP) != 0);
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        if (lease_ctx->lease_type == NONE) {
+            pthread_mutex_unlock(&lease_ctx->lock);
+            gf_msg_debug(frame->this->name, 0,
+                         "No leases found continuing with the"
+                         " fop:%s",
+                         gf_fop_list[frame->root->op]);
+            ret = WIND_FOP;
+            goto out;
         }
-
-        is_blocking_fop = ((fop_flags & BLOCKING_FOP) != 0);
-        is_write_fop = ((fop_flags & DATA_MODIFY_FOP) != 0);
-
-        pthread_mutex_lock (&lease_ctx->lock);
-        {
-                if (lease_ctx->lease_type == NONE) {
-                        gf_msg_debug (frame->this->name, 0,
-                                      "No leases found continuing with the"
-                                      " fop:%s", gf_fop_list[frame->root->op]);
-                        ret = WIND_FOP;
-                        goto unlock;
-                }
-                conflicts = __check_lease_conflict (frame, lease_ctx,
-                                lease_id, is_write_fop);
-                if (conflicts) {
-                        if (is_blocking_fop) {
-                                gf_msg_debug (frame->this->name, 0, "Fop: %s "
-                                              "conflicting existing "
-                                              "lease: %d, blocking the"
-                                              "fop", gf_fop_list[frame->root->op],
-                                              lease_ctx->lease_type);
-                                ret = BLOCK_FOP;
-                        } else {
-                                gf_msg_debug (frame->this->name, 0, "Fop: %s "
-                                              "conflicting existing "
-                                              "lease: %d, sending "
-                                              "EAGAIN",
-                                              gf_fop_list[frame->root->op],
-                                              lease_ctx->lease_type);
-                                errno = EAGAIN;
-                                ret = -1;
-                        }
-                }
+        conflicts = __check_lease_conflict(frame, lease_ctx, lease_id,
+                                           is_write_fop);
+        if (conflicts) {
+            if (is_blocking_fop) {
+                gf_msg_debug(frame->this->name, 0,
+                             "Fop: %s "
+                             "conflicting existing "
+                             "lease: %d, blocking the"
+                             "fop",
+                             gf_fop_list[frame->root->op],
+                             lease_ctx->lease_type);
+                ret = BLOCK_FOP;
+            } else {
+                gf_msg_debug(frame->this->name, 0,
+                             "Fop: %s "
+                             "conflicting existing "
+                             "lease: %d, sending "
+                             "EAGAIN",
+                             gf_fop_list[frame->root->op],
+                             lease_ctx->lease_type);
+                errno = EAGAIN;
+                ret = -1;
+            }
         }
-unlock:
-        pthread_mutex_unlock (&lease_ctx->lock);
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-remove_clnt_leases (const char *client_uid, inode_t *inode, xlator_t *this)
+remove_clnt_leases(const char *client_uid, inode_t *inode, xlator_t *this)
 {
-        lease_inode_ctx_t  *lease_ctx      = NULL;
-        lease_id_entry_t   *lease_entry    = NULL;
-        lease_id_entry_t   *tmp            = NULL;
-        int                 ret            = 0;
-        int                 i              = 0;
-
-        lease_ctx = lease_ctx_get (inode, this);
-        if (!lease_ctx) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        LEASE_MSG_INVAL_INODE_CTX,
-                        "Unable to create/get inode ctx");
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        pthread_mutex_lock (&lease_ctx->lock);
+    lease_inode_ctx_t *lease_ctx = NULL;
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    int ret = 0;
+    int i = 0;
+
+    lease_ctx = lease_ctx_get(inode, this);
+    if (!lease_ctx) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_INVAL_INODE_CTX,
+               "Unable to create/get inode ctx");
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                                 lease_id_list)
         {
-                list_for_each_entry_safe (lease_entry, tmp,
-                                          &lease_ctx->lease_id_list,
-                                          lease_id_list) {
-                        if (strcmp (client_uid, lease_entry->client_uid) == 0) {
-                                for (i = 0; i < GF_LEASE_MAX_TYPE; i++) {
-                                        lease_ctx->lease_type_cnt[i] -= lease_entry->lease_type_cnt[i];
-                                }
-                                lease_ctx->lease_cnt -= lease_entry->lease_cnt;
-                                __destroy_lease_id_entry (lease_entry);
-                                if (lease_ctx->lease_cnt == 0) {
-                                        pthread_mutex_unlock (&lease_ctx->lock);
-                                        goto unblock;
-                                }
-                        }
+            if (strcmp(client_uid, lease_entry->client_uid) == 0) {
+                for (i = 0; i < GF_LEASE_MAX_TYPE; i++) {
+                    lease_ctx->lease_type_cnt[i] -= lease_entry
+                                                        ->lease_type_cnt[i];
+                }
+                lease_ctx->lease_cnt -= lease_entry->lease_cnt;
+                __destroy_lease_id_entry(lease_entry);
+                if (lease_ctx->lease_cnt == 0) {
+                    lease_ctx->blocked_fops_resuming = _gf_true;
+                    pthread_mutex_unlock(&lease_ctx->lock);
+                    goto unblock;
                 }
+            }
         }
-        pthread_mutex_unlock (&lease_ctx->lock);
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
 out:
-        return ret;
+    return ret;
 
 unblock:
-        do_blocked_fops (this, lease_ctx);
-        return ret;
+    do_blocked_fops(this, lease_ctx);
+    return ret;
 }
 
-
 int
-cleanup_client_leases (xlator_t *this, const char *client_uid)
+cleanup_client_leases(xlator_t *this, const char *client_uid)
 {
-        lease_client_t    *clnt         = NULL;
-        lease_client_t    *tmp          = NULL;
-        struct list_head   cleanup_list = {0, };
-        lease_inode_t     *l_inode      = NULL;
-        lease_inode_t     *tmp1         = NULL;
-        leases_private_t  *priv         = NULL;
-        int                ret          = 0;
-
-        priv = this->private;
-        if (!priv) {
-                ret = -1;
-                errno = EINVAL;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&cleanup_list);
-        pthread_mutex_lock (&priv->mutex);
+    lease_client_t *clnt = NULL;
+    lease_client_t *tmp = NULL;
+    struct list_head cleanup_list = {
+        0,
+    };
+    lease_inode_t *l_inode = NULL;
+    lease_inode_t *tmp1 = NULL;
+    leases_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    if (!priv) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&cleanup_list);
+    pthread_mutex_lock(&priv->mutex);
+    {
+        list_for_each_entry_safe(clnt, tmp, &priv->client_list, client_list)
         {
-                list_for_each_entry_safe (clnt, tmp, &priv->client_list, client_list) {
-                        if ((strcmp (clnt->client_uid, client_uid) == 0)) {
-                                list_for_each_entry_safe (l_inode, tmp1,
-                                                          &clnt->inode_list, list) {
-                                        list_del_init (&l_inode->list);
-                                        list_add_tail (&l_inode->list, &cleanup_list);
-                                }
-                                break;
-                        }
-                        __destroy_lease_client (clnt);
+            if ((strcmp(clnt->client_uid, client_uid) == 0)) {
+                list_for_each_entry_safe(l_inode, tmp1, &clnt->inode_list, list)
+                {
+                    list_del_init(&l_inode->list);
+                    list_add_tail(&l_inode->list, &cleanup_list);
                 }
+                __destroy_lease_client(clnt);
+                break;
+            }
         }
-        pthread_mutex_unlock (&priv->mutex);
-
-        l_inode = tmp1 = NULL;
-        list_for_each_entry_safe (l_inode, tmp1, &cleanup_list, list) {
-                remove_clnt_leases (client_uid, l_inode->inode, this);
-        }
+    }
+    pthread_mutex_unlock(&priv->mutex);
+
+    l_inode = tmp1 = NULL;
+    list_for_each_entry_safe(l_inode, tmp1, &cleanup_list, list)
+    {
+        remove_clnt_leases(client_uid, l_inode->inode, this);
+        __destroy_lease_inode(l_inode);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static void
-__remove_all_leases (xlator_t *this, lease_inode_ctx_t *lease_ctx)
+__remove_all_leases(xlator_t *this, lease_inode_ctx_t *lease_ctx)
 {
-        int                 i              = 0;
-        lease_id_entry_t   *lease_entry    = NULL;
-        lease_id_entry_t   *tmp            = NULL;
-
-        __dump_leases_info (this, lease_ctx);
-
-        list_for_each_entry_safe (lease_entry, tmp,
-                                  &lease_ctx->lease_id_list,
-                                  lease_id_list) {
-                lease_entry->lease_cnt = 0;
-                remove_from_clnt_list (this, lease_entry->client_uid, lease_ctx->inode);
-                __destroy_lease_id_entry (lease_entry);
-        }
-        INIT_LIST_HEAD (&lease_ctx->lease_id_list);
-        for (i = 0; i <= GF_LEASE_MAX_TYPE; i++)
-                lease_ctx->lease_type_cnt[i] = 0;
-        lease_ctx->lease_type = 0;
-        lease_ctx->lease_cnt = 0;
-        lease_ctx->recall_in_progress = _gf_false;
-        inode_unref (lease_ctx->inode);
-        lease_ctx->timer = NULL;
+    int i = 0;
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
 
-        /* TODO:
-         * - Mark the corresponding fd bad. Could be done on client side
-         * as a result of recall
-         * - Free the lease_ctx
-         */
+    if (lease_ctx->lease_cnt == 0) {
+        /* No leases to remove. Return */
         return;
-}
+    }
+    __dump_leases_info(this, lease_ctx);
 
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        lease_entry->lease_cnt = 0;
+        remove_from_clnt_list(this, lease_entry->client_uid, lease_ctx->inode);
+        __destroy_lease_id_entry(lease_entry);
+    }
+    INIT_LIST_HEAD(&lease_ctx->lease_id_list);
+    for (i = 0; i <= GF_LEASE_MAX_TYPE; i++)
+        lease_ctx->lease_type_cnt[i] = 0;
+    lease_ctx->lease_type = 0;
+    lease_ctx->lease_cnt = 0;
+    lease_ctx->recall_in_progress = _gf_false;
+    lease_ctx->timer = NULL;
+    lease_ctx->blocked_fops_resuming = _gf_true;
+
+    /* TODO:
+     * - Mark the corresponding fd bad. Could be done on client side
+     * as a result of recall
+     * - Free the lease_ctx
+     */
+    return;
+}
 
 static int
-remove_all_leases (xlator_t *this, inode_t *inode)
+remove_all_leases(xlator_t *this, inode_t *inode)
 {
-        lease_inode_ctx_t  *lease_ctx       = NULL;
-        int                 ret             = 0;
-
-        GF_VALIDATE_OR_GOTO ("leases", inode, out);
-
-        lease_ctx = lease_ctx_get (inode, this);
-        if (!lease_ctx) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        LEASE_MSG_INVAL_INODE_CTX,
-                        "Unable to create/get inode ctx");
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
-
-        pthread_mutex_lock (&lease_ctx->lock);
-        {
-                __remove_all_leases (this, lease_ctx);
-        }
-        pthread_mutex_unlock (&lease_ctx->lock);
-
-        do_blocked_fops (this, lease_ctx);
+    lease_inode_ctx_t *lease_ctx = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+
+    lease_ctx = lease_ctx_get(inode, this);
+    if (!lease_ctx) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_INVAL_INODE_CTX,
+               "Unable to create/get inode ctx");
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        __remove_all_leases(this, lease_ctx);
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+
+    do_blocked_fops(this, lease_ctx);
 out:
-        return ret;
+    return ret;
 }
 
-
 void *
-expired_recall_cleanup (void *data)
+expired_recall_cleanup(void *data)
 {
-        struct timespec      sleep_till      = {0, };
-        struct list_head     recall_cleanup_list;
-        lease_inode_t       *recall_entry    = NULL;
-        lease_inode_t       *tmp             = NULL;
-        leases_private_t    *priv            = NULL;
-        xlator_t            *this            = NULL;
-
-        GF_VALIDATE_OR_GOTO ("leases", data, out);
-
-        this = data;
-        priv = this->private;
-
-        gf_msg_debug (this->name, 0, "Started the expired_recall_cleanup thread");
-
-        while (1) {
-                pthread_mutex_lock (&priv->mutex);
+    struct timespec sleep_till = {
+        0,
+    };
+    struct list_head recall_cleanup_list;
+    lease_inode_t *recall_entry = NULL;
+    lease_inode_t *tmp = NULL;
+    leases_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    time_t time_now;
+
+    GF_VALIDATE_OR_GOTO("leases", data, out);
+
+    this = data;
+    priv = this->private;
+
+    gf_msg_debug(this->name, 0, "Started the expired_recall_cleanup thread");
+
+    while (1) {
+        time_now = gf_time();
+        pthread_mutex_lock(&priv->mutex);
+        {
+            if (priv->fini) {
+                pthread_mutex_unlock(&priv->mutex);
+                goto out;
+            }
+            INIT_LIST_HEAD(&recall_cleanup_list);
+            if (list_empty(&priv->recall_list)) {
+                sleep_till.tv_sec = time_now + 600;
+                pthread_cond_timedwait(&priv->cond, &priv->mutex, &sleep_till);
+            }
+            if (!list_empty(&priv->recall_list)) {
+                gf_msg_debug(this->name, 0, "Found expired recalls");
+                list_for_each_entry_safe(recall_entry, tmp, &priv->recall_list,
+                                         list)
                 {
-                        if (priv->fini) {
-                                pthread_mutex_unlock (&priv->mutex);
-                                goto out;
-                        }
-                        INIT_LIST_HEAD (&recall_cleanup_list);
-                        if (list_empty (&priv->recall_list)) {
-                                sleep_till.tv_sec = time (NULL) + 600;
-                                pthread_cond_timedwait (&priv->cond, &priv->mutex,
-                                                        &sleep_till);
-                        }
-                        if (!list_empty (&priv->recall_list)) {
-                                gf_msg_debug (this->name, 0, "Found expired recalls");
-                                list_for_each_entry_safe (recall_entry, tmp,
-                                                          &priv->recall_list, list) {
-                                        list_del_init (&recall_entry->list);
-                                        list_add_tail (&recall_entry->list, &recall_cleanup_list);
-                                }
-                        }
-                }
-                pthread_mutex_unlock (&priv->mutex);
-
-                recall_entry = tmp = NULL;
-                list_for_each_entry_safe (recall_entry, tmp, &recall_cleanup_list, list) {
-                        gf_msg_debug (this->name, 0, "Recall lease was sent on"
-                                      " inode:%p, recall timer has expired"
-                                      " and clients haven't unlocked the lease"
-                                      " hence cleaning up leases on the inode",
-                                      recall_entry->inode);
-                        remove_all_leases (this, recall_entry->inode);
-                        list_del_init (&recall_entry->list);
+                    list_del_init(&recall_entry->list);
+                    list_add_tail(&recall_entry->list, &recall_cleanup_list);
                 }
+            }
         }
+        pthread_mutex_unlock(&priv->mutex);
+
+        recall_entry = tmp = NULL;
+        list_for_each_entry_safe(recall_entry, tmp, &recall_cleanup_list, list)
+        {
+            gf_msg_debug(this->name, 0,
+                         "Recall lease was sent on"
+                         " inode:%p, recall timer has expired"
+                         " and clients haven't unlocked the lease"
+                         " hence cleaning up leases on the inode",
+                         recall_entry->inode);
+            remove_all_leases(this, recall_entry->inode);
+            /* no need to take priv->mutex lock as this entry
+             * reference is removed from global recall list. */
+            __destroy_lease_inode(recall_entry);
+        }
+    }
 
 out:
-        return NULL;
+    return NULL;
 }
diff --git a/xlators/features/leases/src/leases-mem-types.h b/xlators/features/leases/src/leases-mem-types.h
index d1a59c1db2e..25664b44156 100644
--- a/xlators/features/leases/src/leases-mem-types.h
+++ b/xlators/features/leases/src/leases-mem-types.h
@@ -11,18 +11,17 @@
 #ifndef __LEASES_MEM_TYPES_H__
 #define __LEASES_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_leases_mem_types_ {
-        gf_leases_mt_conf_t = gf_common_mt_end + 1,
-        gf_leases_mt_private_t,
-        gf_leases_mt_lease_client_t,
-        gf_leases_mt_lease_inode_t,
-        gf_leases_mt_fd_ctx_t,
-        gf_leases_mt_lease_inode_ctx_t,
-        gf_leases_mt_lease_id_entry_t,
-        gf_leases_mt_fop_stub_t,
-        gf_leases_mt_timer_data_t,
-        gf_leases_mt_end
+    gf_leases_mt_private_t = gf_common_mt_end + 1,
+    gf_leases_mt_lease_client_t,
+    gf_leases_mt_lease_inode_t,
+    gf_leases_mt_fd_ctx_t,
+    gf_leases_mt_lease_inode_ctx_t,
+    gf_leases_mt_lease_id_entry_t,
+    gf_leases_mt_fop_stub_t,
+    gf_leases_mt_timer_data_t,
+    gf_leases_mt_end
 };
 #endif
diff --git a/xlators/features/leases/src/leases-messages.h b/xlators/features/leases/src/leases-messages.h
index 62df4395a59..da696b832de 100644
--- a/xlators/features/leases/src/leases-messages.h
+++ b/xlators/features/leases/src/leases-messages.h
@@ -11,119 +11,23 @@
 #ifndef _LEASES_MESSAGES_H_
 #define _LEASES_MESSAGES_H_
 
-#include "glfs-message-id.h"
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check across the code if the message ID macro in question is reused
- * anywhere. If reused then then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
- */
-
-#define LEASES_COMP_BASE        GLFS_MSGID_COMP_LEASES
-#define GLFS_NUM_MESSAGES       11
-#define GLFS_MSGID_END          (LEASES_COMP_BASE + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x LEASES_COMP_BASE, "Invalid: Start of messages"
-/*------------*/
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_NO_MEM                        (LEASES_COMP_BASE + 1)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_RECALL_FAIL                   (LEASES_COMP_BASE + 2)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_INVAL_LEASE_ID                (LEASES_COMP_BASE + 3)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_INVAL_UNLK_LEASE              (LEASES_COMP_BASE + 4)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_INVAL_INODE_CTX               (LEASES_COMP_BASE + 5)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_NOT_ENABLED                   (LEASES_COMP_BASE + 6)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_NO_TIMER_WHEEL                (LEASES_COMP_BASE + 7)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_CLNT_NOTFOUND                 (LEASES_COMP_BASE + 8)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_INODE_NOTFOUND                (LEASES_COMP_BASE + 9)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_INVAL_FD_CTX                  (LEASES_COMP_BASE + 10)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define LEASE_MSG_INVAL_LEASE_TYPE              (LEASES_COMP_BASE + 11)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- */
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(LEASES, LEASE_MSG_NO_MEM, LEASE_MSG_RECALL_FAIL,
+           LEASE_MSG_INVAL_LEASE_ID, LEASE_MSG_INVAL_UNLK_LEASE,
+           LEASE_MSG_INVAL_INODE_CTX, LEASE_MSG_NOT_ENABLED,
+           LEASE_MSG_NO_TIMER_WHEEL, LEASE_MSG_CLNT_NOTFOUND,
+           LEASE_MSG_INODE_NOTFOUND, LEASE_MSG_INVAL_FD_CTX,
+           LEASE_MSG_INVAL_LEASE_TYPE);
 
 #endif /* !_LEASES_MESSAGES_H_ */
diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c
index 3e0460000d7..04bee50ba3f 100644
--- a/xlators/features/leases/src/leases.c
+++ b/xlators/features/leases/src/leases.c
@@ -16,1153 +16,1153 @@
 #include "leases.h"
 
 int32_t
-leases_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+leases_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-leases_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-             fd_t *fd, dict_t *xdata)
+leases_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            fd_t *fd, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = 0;
-        int              ret             = 0;
-        lease_fd_ctx_t  *fd_ctx          = NULL;
-        char            *lease_id        = NULL;
-
-        EXIT_IF_LEASES_OFF (this, out);
-
-        fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx), gf_leases_mt_fd_ctx_t);
-
-        fd_ctx->client_uid = gf_strdup (frame->root->client->client_uid);
-        if (!fd_ctx->client_uid) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        GET_FLAGS (frame->root->op, flags);
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        if (lease_id != NULL)
-                memcpy (fd_ctx->lease_id, lease_id, LEASE_ID_SIZE);
-        else
-                memset (fd_ctx->lease_id, 0, LEASE_ID_SIZE);
-
-        ret = fd_ctx_set (fd, this, (uint64_t)fd_ctx);
-        if (ret) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    uint32_t fop_flags = 0;
+    int32_t op_errno = EINVAL;
+    int ret = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+    char *lease_id = NULL;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    fd_ctx = GF_CALLOC(1, sizeof(*fd_ctx), gf_leases_mt_fd_ctx_t);
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    fd_ctx->client_uid = gf_strdup(frame->root->client->client_uid);
+    if (!fd_ctx->client_uid) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    GET_FLAGS(frame->root->op, flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    if (lease_id != NULL)
+        memcpy(fd_ctx->lease_id, lease_id, LEASE_ID_SIZE);
+    else
+        memset(fd_ctx->lease_id, 0, LEASE_ID_SIZE);
+
+    ret = fd_ctx_set(fd, this, (uint64_t)(uintptr_t)fd_ctx);
+    if (ret) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, open, frame, this,
-                         loc, flags, fd, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, open, frame, this, loc, flags, fd, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_open_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, leases_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    if (fd_ctx) {
+        GF_FREE(fd_ctx->client_uid);
+        GF_FREE(fd_ctx);
+    }
+
+    STACK_UNWIND_STRICT(open, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+leases_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-leases_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iovec *vector, int count, off_t off, uint32_t flags,
-               struct iobref *iobref, dict_t *xdata)
+leases_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int count, off_t off, uint32_t flags,
+              struct iobref *iobref, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, writev, frame, this, fd, vector, count,
-                         off, flags, iobref, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, writev, frame, this, fd, vector, count, off,
+                    flags, iobref, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_writev_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, off, flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, leases_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-leases_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno,
-                  struct iovec *vector, int count, struct iatt *stbuf,
-                  struct iobref *iobref, dict_t *xdata)
+leases_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iovec *vector, int count,
+                 struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
-                             count, stbuf, iobref, xdata);
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_readv (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, size_t size, off_t offset,
-              uint32_t flags, dict_t *xdata)
+leases_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, readv, frame, this,
-                         fd, size, offset, flags, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, readv, frame, this, fd, size, offset, flags,
+                    xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_readv_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, leases_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
-                             NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(readv, frame, -1, errno, NULL, 0, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-               dict_t *xdata)
+leases_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+    STACK_UNWIND_STRICT(lk, frame, op_ret, op_errno, lock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_lk (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+leases_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+          struct gf_flock *flock, dict_t *xdata)
 {
-        int32_t         op_errno         = 0;
-        uint32_t        fop_flags        = 0;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS_LK (cmd, flock->l_type, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS_LK(cmd, flock->l_type, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, lk, frame, this,
-                         fd, cmd, flock, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, lk, frame, this, fd, cmd, flock, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, leases_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, flock, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(lk, frame, -1, errno, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_lease (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct gf_lease *lease, dict_t *xdata)
+leases_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct gf_lease *lease, dict_t *xdata)
 {
-        int32_t         op_errno         = 0;
-        int             ret              = 0;
-        struct gf_lease nullease         = {0, };
-        int32_t         op_ret           = 0;
-
-        EXIT_IF_LEASES_OFF (this, out);
-
-        ret = process_lease_req (frame, this, loc->inode, lease);
-        if (ret < 0) {
-                op_errno = -ret;
-                op_ret = -1;
-        }
-        goto unwind;
+    int32_t op_errno = 0;
+    int ret = 0;
+    struct gf_lease nullease = {
+        0,
+    };
+    int32_t op_ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    ret = process_lease_req(frame, this, loc->inode, lease);
+    if (ret < 0) {
+        op_errno = -ret;
+        op_ret = -1;
+    }
+    goto unwind;
 
 out:
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_NOT_ENABLED,
-                "\"features/leases\" translator is not enabled. "
-                "You need to enable it for proper functioning of your "
-                "application");
-        op_errno = ENOSYS;
-        op_ret = -1;
+    gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_NOT_ENABLED,
+           "\"features/leases\" translator is not enabled. "
+           "You need to enable it for proper functioning of your "
+           "application");
+    op_errno = ENOSYS;
+    op_ret = -1;
 
 unwind:
-        STACK_UNWIND_STRICT (lease, frame, op_ret, op_errno,
-                             (op_errno == ENOSYS) ? &nullease : lease, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(lease, frame, op_ret, op_errno,
+                        (op_errno == ENOSYS) ? &nullease : lease, xdata);
+    return 0;
 }
 
 int32_t
-leases_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+leases_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                 dict_t *xdata)
+leases_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, 0);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
 
-        ret = check_lease_conflict (frame, loc->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, loc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (loc->inode, truncate, frame, this,
-                         loc, offset, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(loc->inode, truncate, frame, this, loc, offset, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_truncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, leases_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, struct iatt *statpre,
-                    struct iatt *statpost, dict_t *xdata)
+leases_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno,
-                             statpre, statpost, xdata);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+leases_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, 0);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
 
-        ret = check_lease_conflict (frame, loc->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, loc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (loc->inode, setattr, frame, this,
-                         loc, stbuf, valid, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(loc->inode, setattr, frame, this, loc, stbuf, valid, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_setattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, leases_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(setattr, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent,
-                   dict_t *xdata)
+leases_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                  struct iatt *preoldparent, struct iatt *postoldparent,
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno,
-                             stbuf, preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, stbuf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_rename (call_frame_t *frame, xlator_t *this,
-               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+leases_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        /* should the lease be also checked for newloc */
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, 0);
+    /* should the lease be also checked for newloc */
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
 
-        ret = check_lease_conflict (frame, oldloc->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, oldloc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (oldloc->inode, rename, frame, this,
-                         oldloc, newloc, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(oldloc->inode, rename, frame, this, oldloc, newloc, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, leases_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(rename, frame, -1, errno, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
 int32_t
-leases_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+leases_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-               dict_t *xdata)
+leases_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, 0);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
 
-        ret = check_lease_conflict (frame, loc->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, loc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (loc->inode, unlink, frame, this,
-                         loc, xflag, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(loc->inode, unlink, frame, this, loc, xflag, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, leases_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(unlink, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+leases_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno,
-                             inode, stbuf, preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-             loc_t *newloc, dict_t *xdata)
+leases_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, 0);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
 
-        ret = check_lease_conflict (frame, oldloc->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, oldloc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (oldloc->inode, link, frame, this,
-                         oldloc, newloc, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(oldloc->inode, link, frame, this, oldloc, newloc, xdata);
+    return 0;
 out:
-        STACK_WIND (frame, leases_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, leases_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(link, frame, -1, errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, fd_t *fd, inode_t *inode,
-                   struct iatt *stbuf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+leases_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd,
-                             inode, stbuf, preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                        preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_create (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, mode_t mode,
-               mode_t umask, fd_t *fd, dict_t *xdata)
+leases_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, create, frame, this,
-                         loc, flags, mode, umask, fd, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, create, frame, this, loc, flags, mode, umask, fd,
+                    xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_create_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, leases_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, errno, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
 int32_t
-leases_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf,
-                  dict_t *xdata)
+leases_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
 int32_t
-leases_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t flags, dict_t *xdata)
+leases_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, fsync, frame, this,
-                         fd, flags, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, fsync, frame, this, fd, flags, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
-        return 0;
+    STACK_WIND(frame, leases_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsync, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf,
-                      dict_t *xdata)
+leases_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-leases_ftruncate (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, off_t offset, dict_t *xdata)
+leases_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, 0); /* TODO:fd->flags?*/
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0); /* TODO:fd->flags?*/
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, ftruncate, frame, this,
-                         fd, offset, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, ftruncate, frame, this, fd, offset, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_ftruncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, leases_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(ftruncate, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost, dict_t *xdata)
+leases_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno,
-                             statpre, statpost, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    return 0;
 }
 
 int32_t
-leases_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt  *stbuf, int32_t valid, dict_t *xdata)
+leases_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, fsetattr, frame, this,
-                         fd, stbuf, valid, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, fsetattr, frame, this, fd, stbuf, valid, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_fsetattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, leases_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetattr, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_fallocate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                      struct iatt *post, dict_t *xdata)
+leases_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                     struct iatt *post, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fallocate, frame, op_ret, op_errno, pre,
-                             post, xdata);
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, pre, post, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  int32_t mode, off_t offset, size_t len, dict_t *xdata)
+leases_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+                 off_t offset, size_t len, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, fallocate, frame, this,
-                         fd, mode, offset, len, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, fallocate, frame, this, fd, mode, offset, len,
+                    xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_fallocate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fallocate,
-                    fd, mode, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, leases_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (fallocate, frame, -1, op_errno, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fallocate, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_discard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                    struct iatt *post, dict_t *xdata)
+leases_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                   struct iatt *post, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, pre,
-                             post, xdata);
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, pre, post, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-leases_discard (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                off_t offset, size_t len, dict_t *xdata)
+leases_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, discard, frame, this,
-                         fd, offset, len, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, discard, frame, this, fd, offset, len, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_discard_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->discard,
-                    fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, leases_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (discard, frame, -1, op_errno, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(discard, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-leases_zerofill_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                     struct iatt *post, dict_t *xdata)
+leases_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                    struct iatt *post, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, pre,
-                             post, xdata);
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, pre, post, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-leases_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 off_t offset, off_t len, dict_t *xdata)
+leases_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                off_t len, dict_t *xdata)
 {
-        uint32_t         fop_flags       = 0;
-        int32_t          op_errno        = -1;
-        char            *lease_id        = NULL;
-        int              ret             = 0;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
 
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
 
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, zerofill, frame, this,
-                         fd, offset, len, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, zerofill, frame, this, fd, offset, len, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_zerofill_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->zerofill,
-                    fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, leases_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (zerofill, frame, -1, op_errno, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(zerofill, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-leases_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+leases_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-leases_flush (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, dict_t *xdata)
+leases_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t       op_errno       = -1;
-        uint32_t      fop_flags      = 0;
-        char         *lease_id       = NULL;
-        int          ret             = 0;
-
-        EXIT_IF_LEASES_OFF (this, out);
-
-        GET_LEASE_ID (xdata, lease_id, frame->root->client->client_uid);
-        GET_FLAGS (frame->root->op, fd->flags);
-
-        ret = check_lease_conflict (frame, fd->inode, lease_id, fop_flags);
-        if (ret < 0)
-                goto err;
-        else if (ret == BLOCK_FOP)
-                goto block;
-        else if (ret == WIND_FOP)
-                goto out;
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+    uint64_t ctx = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
 
 block:
-        LEASE_BLOCK_FOP (fd->inode, flush, frame, this,
-                         fd, xdata);
-        return 0;
+    LEASE_BLOCK_FOP(fd->inode, flush, frame, this, fd, xdata);
+    return 0;
 
 out:
-        STACK_WIND (frame, leases_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd, xdata);
-        return 0;
+    /* *
+     * currently release is not called after the close fop from the
+     * application. Hence lease fd ctx is reset on here.
+     * This is actually not the right way, since flush can be called
+     * not only from the close op.
+     * TODO :
+     *     - Either identify the flush is called from close call on fd from
+     *     from the application.
+     *                      OR
+     *     - Find why release is not called post the last close call
+     */
+    ret = fd_ctx_get(fd, this, &ctx);
+    if (ret == 0) {
+        fd_ctx = (lease_fd_ctx_t *)(long)ctx;
+        if (fd_ctx->client_uid) {
+            GF_FREE(fd_ctx->client_uid);
+            fd_ctx->client_uid = NULL;
+        }
+        memset(fd_ctx->lease_id, 0, LEASE_ID_SIZE);
+    }
+    STACK_WIND(frame, leases_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, errno, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_leases_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
-                        "mem account init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_leases_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "mem account init failed");
         return ret;
+    }
+
+    return ret;
 }
 
 static int
-leases_init_priv (xlator_t *this)
+leases_init_priv(xlator_t *this)
 {
-        int               ret  = 0;
-        leases_private_t *priv = NULL;
+    int ret = 0;
+    leases_private_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
+    if (!priv->timer_wheel) {
+        priv->timer_wheel = glusterfs_ctx_tw_get(this->ctx);
         if (!priv->timer_wheel) {
-                if (!glusterfs_global_timer_wheel (this)) {
-                        gf_msg_debug (this->name, 0, "Initing the global "
-                                      "timer wheel");
-                        ret = glusterfs_global_timer_wheel_init (this->ctx);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        LEASE_MSG_NO_TIMER_WHEEL,
-                                        "Initing the global timer "
-                                        "wheel failed");
-                                goto out;
-                        }
-                }
-                priv->timer_wheel = glusterfs_global_timer_wheel (this);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (!priv->inited_recall_thr) {
-                pthread_create (&priv->recall_thr, NULL,
-                                expired_recall_cleanup, this);
-                priv->inited_recall_thr = _gf_true;
-        }
+    if (!priv->inited_recall_thr) {
+        ret = gf_thread_create(&priv->recall_thr, NULL, expired_recall_cleanup,
+                               this, "leasercl");
+        if (!ret)
+            priv->inited_recall_thr = _gf_true;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        leases_private_t *priv                   = NULL;
-        int               ret                    = -1;
+    leases_private_t *priv = NULL;
+    int ret = -1;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        /* TODO: In case of reconfigure, if its enabling the leases
-         * its not an issue, but if its disabling the leases, there
-         * is more to it, like recall all the existing leases, wait
-         * for unlock of all the leases etc., hence not supporting the
-         * reconfigure for now.
+    /* TODO: In case of reconfigure, if its enabling the leases
+     * its not an issue, but if its disabling the leases, there
+     * is more to it, like recall all the existing leases, wait
+     * for unlock of all the leases etc., hence not supporting the
+     * reconfigure for now.
 
-        GF_OPTION_RECONF ("leases", priv->leases_enabled,
-                          options, bool, out);
+    GF_OPTION_RECONF ("leases", priv->leases_enabled,
+                      options, bool, out);
 
-        if (priv->leases_enabled) {
-                ret = leases_init_priv (this);
-                if (ret)
-                        goto out;
-        }
-        */
+    if (priv->leases_enabled) {
+            ret = leases_init_priv (this);
+            if (ret)
+                    goto out;
+    }
+    */
 
-        GF_OPTION_RECONF ("lease-lock-recall-timeout",
-                          priv->recall_lease_timeout,
-                          options, int32, out);
+    GF_OPTION_RECONF("lease-lock-recall-timeout", priv->recall_lease_timeout,
+                     options, int32, out);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int                       ret        = -1;
-        leases_private_t         *priv       = NULL;
-
-        priv = GF_CALLOC (1, sizeof (*priv),
-                          gf_leases_mt_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
-                        "Leases init failed");
-                goto out;
-        }
+    int ret = -1;
+    leases_private_t *priv = NULL;
 
-        GF_OPTION_INIT ("leases", priv->leases_enabled,
-                        bool, out);
-        GF_OPTION_INIT ("lease-lock-recall-timeout",
-                        priv->recall_lease_timeout, int32, out);
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->client_list);
-        INIT_LIST_HEAD (&priv->recall_list);
+    priv = GF_CALLOC(1, sizeof(*priv), gf_leases_mt_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "Leases init failed");
+        goto out;
+    }
 
-        this->private = priv;
+    GF_OPTION_INIT("leases", priv->leases_enabled, bool, out);
+    GF_OPTION_INIT("lease-lock-recall-timeout", priv->recall_lease_timeout,
+                   int32, out);
+    pthread_mutex_init(&priv->mutex, NULL);
+    INIT_LIST_HEAD(&priv->client_list);
+    INIT_LIST_HEAD(&priv->recall_list);
 
-        if (priv->leases_enabled) {
-                ret = leases_init_priv (this);
-                if (ret)
-                        goto out;
-        }
+    this->private = priv;
+
+    if (priv->leases_enabled) {
+        ret = leases_init_priv(this);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret) {
-                GF_FREE (priv);
-                this->private = NULL;
-        }
+    if (ret) {
+        GF_FREE(priv);
+        this->private = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-fini (xlator_t *this)
+void
+fini(xlator_t *this)
 {
-        leases_private_t *priv = NULL;
-
-        priv = this->private;
-        if (!priv) {
-                return 0;
-        }
-        this->private = NULL;
-
-        priv->fini = _gf_true;
-        pthread_cond_broadcast (&priv->cond);
-        pthread_join (priv->recall_thr, NULL);
-
+    leases_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv) {
+        return;
+    }
+    this->private = NULL;
+
+    priv->fini = _gf_true;
+    pthread_cond_broadcast(&priv->cond);
+    if (priv->recall_thr) {
+        gf_thread_cleanup_xint(priv->recall_thr);
+        priv->recall_thr = 0;
         priv->inited_recall_thr = _gf_false;
+    }
 
-        GF_FREE (priv);
+    if (priv->timer_wheel) {
+        glusterfs_ctx_tw_put(this->ctx);
+    }
 
-        return 0;
+    GF_FREE(priv);
+    return;
 }
 
 static int
-leases_forget (xlator_t *this, inode_t *inode)
+leases_forget(xlator_t *this, inode_t *inode)
 {
-        /* TODO:leases_cleanup_inode_ctx (this, inode); */
-        return 0;
+    /* TODO:leases_cleanup_inode_ctx (this, inode); */
+    return 0;
 }
 
 static int
-leases_release (xlator_t *this, fd_t *fd)
+leases_release(xlator_t *this, fd_t *fd)
 {
-        /* TODO:cleanup fd_ctx */
-        return 0;
+    int ret = -1;
+    uint64_t tmp = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+
+    if (fd == NULL) {
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_TRACE, "Releasing all leases with fd %p", fd);
+
+    ret = fd_ctx_del(fd, this, &tmp);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get fdctx");
+        goto out;
+    }
+
+    fd_ctx = (lease_fd_ctx_t *)(long)tmp;
+    if (fd_ctx)
+        GF_FREE(fd_ctx);
+out:
+    return ret;
 }
 
 static int
-leases_clnt_disconnect_cbk (xlator_t *this, client_t *client)
+leases_clnt_disconnect_cbk(xlator_t *this, client_t *client)
 {
-        int ret = 0;
+    int ret = 0;
 
-        EXIT_IF_LEASES_OFF (this, out);
+    EXIT_IF_LEASES_OFF(this, out);
 
-        ret = cleanup_client_leases (this, client->client_uid);
+    ret = cleanup_client_leases(this, client->client_uid);
 out:
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops = {
-        /* Metadata modifying fops */
-        .fsetattr    = leases_fsetattr,
-        .setattr     = leases_setattr,
-
-        /* File Data reading fops */
-        .open        = leases_open,
-        .readv       = leases_readv,
-
-        /* File Data modifying fops */
-        .truncate    = leases_truncate,
-        .ftruncate   = leases_ftruncate,
-        .writev      = leases_writev,
-        .zerofill    = leases_zerofill,
-        .fallocate   = leases_fallocate,
-        .discard     = leases_discard,
-        .lk          = leases_lk,
-        .fsync       = leases_fsync,
-        .flush       = leases_flush,
-        .lease       = leases_lease,
-
-        /* Directory Data modifying fops */
-        .create      = leases_create,
-        .rename      = leases_rename,
-        .unlink      = leases_unlink,
-        .link        = leases_link,
+    /* Metadata modifying fops */
+    .fsetattr = leases_fsetattr,
+    .setattr = leases_setattr,
+
+    /* File Data reading fops */
+    .open = leases_open,
+    .readv = leases_readv,
+
+    /* File Data modifying fops */
+    .truncate = leases_truncate,
+    .ftruncate = leases_ftruncate,
+    .writev = leases_writev,
+    .zerofill = leases_zerofill,
+    .fallocate = leases_fallocate,
+    .discard = leases_discard,
+    .lk = leases_lk,
+    .fsync = leases_fsync,
+    .flush = leases_flush,
+    .lease = leases_lease,
+
+    /* Directory Data modifying fops */
+    .create = leases_create,
+    .rename = leases_rename,
+    .unlink = leases_unlink,
+    .link = leases_link,
 
 #ifdef NOT_SUPPORTED
-        /* internal lk fops */
-        .inodelk     = leases_inodelk,
-        .finodelk    = leases_finodelk,
-        .entrylk     = leases_entrylk,
-        .fentrylk    = leases_fentrylk,
-
-        /* Internal special fops*/
-        .xattrop     = leases_xattrop,
-        .fxattrop    = leases_fxattrop,
+    /* internal lk fops */
+    .inodelk = leases_inodelk,
+    .finodelk = leases_finodelk,
+    .entrylk = leases_entrylk,
+    .fentrylk = leases_fentrylk,
+
+    /* Internal special fops*/
+    .xattrop = leases_xattrop,
+    .fxattrop = leases_fxattrop,
 #endif
 };
 
 struct xlator_cbks cbks = {
-        .forget            = leases_forget,
-        .release           = leases_release,
-        .client_disconnect = leases_clnt_disconnect_cbk,
+    .forget = leases_forget,
+    .release = leases_release,
+    .client_disconnect = leases_clnt_disconnect_cbk,
 };
 
 struct volume_options options[] = {
-        { .key  = {"leases"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", enables leases support"
-        },
-        { .key  = {"lease-lock-recall-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = RECALL_LEASE_LK_TIMEOUT,
-          .description = "After 'timeout' seconds since the recall_lease"
-                         " request has been sent to the client, the lease lock"
-                         " will be forcefully purged by the server."
-        },
-        { .key = {NULL} },
+    {.key = {"leases"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .description = "When \"on\", enables leases support"},
+    {.key = {"lease-lock-recall-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = RECALL_LEASE_LK_TIMEOUT,
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .description = "After 'timeout' seconds since the recall_lease"
+                    " request has been sent to the client, the lease lock"
+                    " will be forcefully purged by the server."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "leases",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/leases/src/leases.h b/xlators/features/leases/src/leases.h
index df5e8beb85c..a6e8a6824cc 100644
--- a/xlators/features/leases/src/leases.h
+++ b/xlators/features/leases/src/leases.h
@@ -16,17 +16,15 @@
 #include "config.h"
 #endif
 
-#include "common-utils.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "inode.h"
-#include "call-stub.h"
-#include "logging.h"
-#include "client_t.h"
-#include "lkowner.h"
-#include "locking.h"
-#include "upcall-utils.h"
-#include "tw.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/client_t.h>
+#include <glusterfs/lkowner.h>
+#include <glusterfs/locking.h>
+#include <glusterfs/upcall-utils.h>
 #include "timer-wheel.h"
 #include "leases-mem-types.h"
 #include "leases-messages.h"
@@ -41,24 +39,35 @@
 #define BLOCK_FOP 0x0001
 #define WIND_FOP 0x0002
 
-#define EXIT_IF_LEASES_OFF(this, label) do {                                   \
+#define EXIT_IF_LEASES_OFF(this, label)                                        \
+    do {                                                                       \
         if (!is_leases_enabled(this))                                          \
-                goto label;                                                    \
-} while (0)
+            goto label;                                                        \
+    } while (0)
 
-#define GET_LEASE_ID(xdata, lease_id, client_uid) do {                         \
-        int   ret_val = -1;                                                    \
-        ret_val = dict_get_bin (xdata, "lease-id", (void **)&lease_id);        \
+#define EXIT_IF_INTERNAL_FOP(frame, xdata, label)                              \
+    do {                                                                       \
+        if (frame->root->pid < 0)                                              \
+            goto label;                                                        \
+        if (xdata && dict_get(xdata, GLUSTERFS_INTERNAL_FOP_KEY))              \
+            goto label;                                                        \
+    } while (0)
+
+#define GET_LEASE_ID(xdata, lease_id, client_uid)                              \
+    do {                                                                       \
+        int ret_val = -1;                                                      \
+        ret_val = dict_get_bin(xdata, "lease-id", (void **)&lease_id);         \
         if (ret_val) {                                                         \
-                ret_val = 0;                                                   \
-                gf_msg_debug ("leases", 0, "Lease id is not set for client:%s", client_uid); \
+            ret_val = 0;                                                       \
+            gf_msg_debug("leases", 0, "Lease id is not set for client:%s",     \
+                         client_uid);                                          \
         }                                                                      \
-} while (0)
+    } while (0)
 
 #define GET_FLAGS(fop, fd_flags)                                               \
-do {                                                                           \
+    do {                                                                       \
         if ((fd_flags & (O_WRONLY | O_RDWR)) && fop == GF_FOP_OPEN)            \
-                fop_flags = DATA_MODIFY_FOP;                                   \
+            fop_flags = DATA_MODIFY_FOP;                                       \
                                                                                \
         if (fop == GF_FOP_UNLINK || fop == GF_FOP_RENAME ||                    \
             fop == GF_FOP_TRUNCATE || fop == GF_FOP_FTRUNCATE ||               \
@@ -67,186 +76,184 @@ do {                                                                           \
             fop == GF_FOP_DISCARD || fop == GF_FOP_ZEROFILL ||                 \
             fop == GF_FOP_SETATTR || fop == GF_FOP_FSETATTR ||                 \
             fop == GF_FOP_LINK)                                                \
-                fop_flags = DATA_MODIFY_FOP;                                   \
+            fop_flags = DATA_MODIFY_FOP;                                       \
                                                                                \
         if (!(fd_flags & (O_NONBLOCK | O_NDELAY)))                             \
-                fop_flags |= BLOCKING_FOP;                                     \
+            fop_flags |= BLOCKING_FOP;                                         \
                                                                                \
-} while (0)                                                                    \
-
+    } while (0)
 
 #define GET_FLAGS_LK(cmd, l_type, fd_flags)                                    \
-do {                                                                           \
+    do {                                                                       \
         /* TODO: handle F_RESLK_LCK and other glusterfs_lk_recovery_cmds_t */  \
-        if ((cmd == F_SETLKW || cmd == F_SETLKW64 ||                           \
-             cmd == F_SETLK || cmd == F_SETLK64) &&                            \
+        if ((cmd == F_SETLKW || cmd == F_SETLKW64 || cmd == F_SETLK ||         \
+             cmd == F_SETLK64) &&                                              \
             l_type == F_WRLCK)                                                 \
-                fop_flags = DATA_MODIFY_FOP;                                   \
+            fop_flags = DATA_MODIFY_FOP;                                       \
                                                                                \
         if (fd_flags & (O_NONBLOCK | O_NDELAY) &&                              \
             (cmd == F_SETLKW || cmd == F_SETLKW64))                            \
-                fop_flags |= BLOCKING_FOP;                                     \
+            fop_flags |= BLOCKING_FOP;                                         \
                                                                                \
-} while (0)                                                                    \
-
-#define LEASE_BLOCK_FOP(inode, fop_name, frame, this, params ...)              \
-do {                                                                           \
-        call_stub_t             *__stub     = NULL;                            \
-        fop_stub_t              *blk_fop    = NULL;                            \
-        lease_inode_ctx_t       *lease_ctx  = NULL;                            \
-        int                      __ret      = 0;                               \
+    } while (0)
+
+#define LEASE_BLOCK_FOP(inode, fop_name, frame, this, params...)               \
+    do {                                                                       \
+        call_stub_t *__stub = NULL;                                            \
+        fop_stub_t *blk_fop = NULL;                                            \
+        lease_inode_ctx_t *lease_ctx = NULL;                                   \
                                                                                \
-        __stub = fop_##fop_name##_stub (frame, default_##fop_name##_resume,    \
-                                        params);                               \
+        __stub = fop_##fop_name##_stub(frame, default_##fop_name##_resume,     \
+                                       params);                                \
         if (!__stub) {                                                         \
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,                    \
-                        LEASE_MSG_NO_MEM,                                      \
-                        "Unable to create stub");                              \
-                ret = -ENOMEM;                                                 \
-                goto __out;                                                    \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create stub");                                   \
+            ret = -ENOMEM;                                                     \
+            goto __out;                                                        \
         }                                                                      \
                                                                                \
-        blk_fop = GF_CALLOC (1, sizeof (*blk_fop),                             \
-                             gf_leases_mt_fop_stub_t);                         \
+        blk_fop = GF_CALLOC(1, sizeof(*blk_fop), gf_leases_mt_fop_stub_t);     \
         if (!blk_fop) {                                                        \
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,                    \
-                        LEASE_MSG_NO_MEM,                                      \
-                        "Unable to create lease fop stub");                    \
-                ret = -ENOMEM;                                                 \
-                goto __out;                                                    \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create lease fop stub");                         \
+            ret = -ENOMEM;                                                     \
+            goto __out;                                                        \
         }                                                                      \
                                                                                \
-        lease_ctx = lease_ctx_get (inode, this);                               \
+        lease_ctx = lease_ctx_get(inode, this);                                \
         if (!lease_ctx) {                                                      \
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,                    \
-                        LEASE_MSG_NO_MEM,                                      \
-                        "Unable to create/get inode ctx");                     \
-                op_errno = ENOMEM;                                             \
-                goto __out;                                                    \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create/get inode ctx");                          \
+            ret = -ENOMEM;                                                     \
+            goto __out;                                                        \
         }                                                                      \
                                                                                \
         blk_fop->stub = __stub;                                                \
-        pthread_mutex_lock (&lease_ctx->lock);                                 \
+        pthread_mutex_lock(&lease_ctx->lock);                                  \
         {                                                                      \
-                /*TODO: If the lease is unlocked btw check lease conflict and  \
-                 * by now, then this fop shouldn't be add to the blocked fop   \
-                 * list, can use generation number for the same?*/             \
-                list_add_tail (&blk_fop->list, &lease_ctx->blocked_list);      \
+            /*TODO: If the lease is unlocked btw check lease conflict and      \
+             * by now, then this fop shouldn't be add to the blocked fop       \
+             * list, can use generation number for the same?*/                 \
+            list_add_tail(&blk_fop->list, &lease_ctx->blocked_list);           \
         }                                                                      \
-        pthread_mutex_unlock (&lease_ctx->lock);                               \
+        pthread_mutex_unlock(&lease_ctx->lock);                                \
                                                                                \
-__out:                                                                         \
+    __out:                                                                     \
         if (ret < 0) {                                                         \
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,  \
-                        "Unable to create stub for blocking the fop:%s (%s)",  \
-                         gf_fop_list[frame->root->op], strerror(ENOMEM));      \
-                if (__stub != NULL) {                                          \
-                        call_stub_destroy (__stub);                            \
-                }                                                              \
-                GF_FREE (blk_fop);                                             \
-                goto err;                                                      \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create stub for blocking the fop:%s (%s)",       \
+                   gf_fop_list[frame->root->op], strerror(ENOMEM));            \
+            if (__stub != NULL) {                                              \
+                call_stub_destroy(__stub);                                     \
+            }                                                                  \
+            GF_FREE(blk_fop);                                                  \
+            goto err;                                                          \
         }                                                                      \
-} while (0)                                                                    \
+    } while (0)
 
 struct _leases_private {
-        gf_boolean_t      leases_enabled;
-        int32_t           recall_lease_timeout;
-        struct list_head  client_list;
-        struct list_head  recall_list;
-        struct tvec_base *timer_wheel;    /* timer wheel where the recall request
-                                             is qued and waits for unlock/expiry */
-        gf_boolean_t      fini;
-        pthread_t         recall_thr;
-        gf_boolean_t      inited_recall_thr;
-        pthread_mutex_t   mutex;
-        pthread_cond_t    cond;
+    struct list_head client_list;
+    struct list_head recall_list;
+    struct tvec_base *timer_wheel; /* timer wheel where the recall request
+                                      is qued and waits for unlock/expiry */
+    pthread_t recall_thr;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    int32_t recall_lease_timeout;
+    gf_boolean_t inited_recall_thr;
+    gf_boolean_t fini;
+    gf_boolean_t leases_enabled;
+
+    char _pad[1]; /* manual padding */
 };
 typedef struct _leases_private leases_private_t;
 
 struct _lease_client {
-        char             *client_uid;
-        struct list_head  client_list;
-        struct list_head  inode_list;
+    char *client_uid;
+    struct list_head client_list;
+    struct list_head inode_list;
 };
 typedef struct _lease_client lease_client_t;
 
 struct _lease_inode {
-        inode_t          *inode;
-        struct list_head  list;  /* This can be part of both inode_list and recall_list */
+    inode_t *inode;
+    struct list_head
+        list; /* This can be part of both inode_list and recall_list */
 };
 typedef struct _lease_inode lease_inode_t;
 
 struct _lease_fd_ctx {
-        char             *client_uid;
-        char              lease_id[LEASE_ID_SIZE];
+    char *client_uid;
+    char lease_id[LEASE_ID_SIZE];
 };
 typedef struct _lease_fd_ctx lease_fd_ctx_t;
 
 struct _lease_inode_ctx {
-        struct list_head  lease_id_list;  /* clients that have taken leases */
-        int               lease_type_cnt[GF_LEASE_MAX_TYPE+1];
-        int               lease_type;   /* Types of leases acquired */
-        uint64_t          lease_cnt;    /* Total number of leases on this inode */
-        uint64_t          openfd_cnt;   /* number of fds open */
-        gf_boolean_t      recall_in_progress;  /* if lease recall is sent on this inode */
-        struct list_head  blocked_list; /* List of fops blocked until the
-                                           lease recall is complete */
-        inode_t          *inode;        /* this represents the inode on which the
-                                           lock was taken, required mainly during
-                                           disconnect cleanup */
-        struct gf_tw_timer_list *timer;
-        pthread_mutex_t   lock;
+    struct list_head lease_id_list; /* clients that have taken leases */
+    int lease_type_cnt[GF_LEASE_MAX_TYPE + 1];
+    uint64_t lease_cnt;            /* Total number of leases on this inode */
+    uint64_t openfd_cnt;           /* number of fds open */
+    struct list_head blocked_list; /* List of fops blocked until the
+                                      lease recall is complete */
+    inode_t *inode;                /* this represents the inode on which the
+                                      lock was taken, required mainly during
+                                      disconnect cleanup */
+    struct gf_tw_timer_list *timer;
+    pthread_mutex_t lock;
+    int lease_type;                  /* Types of leases acquired */
+    gf_boolean_t recall_in_progress; /* if lease recall is sent on this inode */
+    gf_boolean_t blocked_fops_resuming; /* if blocked fops are being resumed */
+
+    char _pad[2]; /* manual padding */
 };
 typedef struct _lease_inode_ctx lease_inode_ctx_t;
 
 struct _lease_id_entry {
-        struct list_head    lease_id_list;
-        char                lease_id[LEASE_ID_SIZE];
-        char               *client_uid;  /* uid of the client that has
-                                            taken the lease */
-        int                 lease_type_cnt[GF_LEASE_MAX_TYPE+1]; /* count of each lease type */
-        int                 lease_type;  /* Union of all the leases taken
-                                            under the given lease id */
-        uint64_t            lease_cnt;   /* Number of leases taken under the
-                                            given lease id */
-        time_t              recall_time; /* time @ which recall was sent */
+    struct list_head lease_id_list;
+    char lease_id[LEASE_ID_SIZE];
+    char *client_uid;                          /* uid of the client that has
+                                                  taken the lease */
+    int lease_type_cnt[GF_LEASE_MAX_TYPE + 1]; /* count of each lease type */
+    uint64_t lease_cnt; /* Number of leases taken under the
+                           given lease id */
+    time_t recall_time; /* time @ which recall was sent */
+    int lease_type;     /* Union of all the leases taken
+                           under the given lease id */
+    char _pad[4];       /* manual padding */
 };
 typedef struct _lease_id_entry lease_id_entry_t;
 
 /* Required? as stub itself will have list */
 struct __fop_stub {
-        struct list_head      list;
-        call_stub_t          *stub;
+    struct list_head list;
+    call_stub_t *stub;
 };
 typedef struct __fop_stub fop_stub_t;
 
 struct __lease_timer_data {
-        inode_t *inode;
-        xlator_t *this;
+    inode_t *inode;
+    xlator_t *this;
 };
 typedef struct __lease_timer_data lease_timer_data_t;
 
 gf_boolean_t
-is_leases_enabled (xlator_t *this);
-
-int32_t
-get_recall_lease_timeout (xlator_t *this);
+is_leases_enabled(xlator_t *this);
 
 lease_inode_ctx_t *
-lease_ctx_get (inode_t *inode, xlator_t *this);
+lease_ctx_get(inode_t *inode, xlator_t *this);
 
 int
-process_lease_req (call_frame_t *frame, xlator_t *this,
-                   inode_t *inode, struct gf_lease *lease);
+process_lease_req(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  struct gf_lease *lease);
 
 int
-check_lease_conflict (call_frame_t *frame, inode_t *inode,
-                      const char *lease_id, uint32_t fop_flags);
+check_lease_conflict(call_frame_t *frame, inode_t *inode, const char *lease_id,
+                     uint32_t fop_flags);
 
 int
-cleanup_client_leases (xlator_t *this, const char *client_uid);
+cleanup_client_leases(xlator_t *this, const char *client_uid);
 
 void *
-expired_recall_cleanup (void *data);
+expired_recall_cleanup(void *data);
 
 #endif /* _LEASES_H */
diff --git a/xlators/features/locks/src/Makefile.am b/xlators/features/locks/src/Makefile.am
index a3b3855fb9b..0b174c19d2d 100644
--- a/xlators/features/locks/src/Makefile.am
+++ b/xlators/features/locks/src/Makefile.am
@@ -1,23 +1,29 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = locks.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 locks_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 locks_la_SOURCES = common.c posix.c entrylk.c inodelk.c reservelk.c \
-		   clear.c
+	clear.c
+
 locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h pl-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES = 
 
+if WITH_SERVER
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/posix-locks.so
 
 install-data-hook:
 	ln -sf locks.so $(DESTDIR)$(xlatordir)/posix-locks.so
+endif
diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c
index 640c6bb5553..ab1eac68a53 100644
--- a/xlators/features/locks/src/clear.c
+++ b/xlators/features/locks/src/clear.c
@@ -12,407 +12,449 @@
 #include <limits.h>
 #include <pthread.h>
 
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
 #include "locks.h"
 #include "common.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "clear.h"
 
+const char *clrlk_type_names[CLRLK_TYPE_MAX] = {
+    [CLRLK_INODE] = "inode",
+    [CLRLK_ENTRY] = "entry",
+    [CLRLK_POSIX] = "posix",
+};
+
 int
-clrlk_get_kind (char *kind)
+clrlk_get_kind(char *kind)
 {
-        char  *clrlk_kinds[CLRLK_KIND_MAX] = {"dummy", "blocked", "granted",
-                                              "all"};
-        int   ret_kind                     = CLRLK_KIND_MAX;
-        int   i                            = 0;
-
-        for (i = CLRLK_BLOCKED; i < CLRLK_KIND_MAX; i++) {
-                if (!strcmp (clrlk_kinds[i], kind)) {
-                        ret_kind = i;
-                        break;
-                }
+    char *clrlk_kinds[CLRLK_KIND_MAX] = {"dummy", "blocked", "granted", "all"};
+    int ret_kind = CLRLK_KIND_MAX;
+    int i = 0;
+
+    for (i = CLRLK_BLOCKED; i < CLRLK_KIND_MAX; i++) {
+        if (!strcmp(clrlk_kinds[i], kind)) {
+            ret_kind = i;
+            break;
         }
+    }
 
-        return ret_kind;
+    return ret_kind;
 }
 
 int
-clrlk_get_type (char *type)
+clrlk_get_type(char *type)
 {
-        char    *clrlk_types[CLRLK_TYPE_MAX]    = {"inode", "entry", "posix"};
-        int     ret_type                        = CLRLK_TYPE_MAX;
-        int     i                               = 0;
-
-        for (i = CLRLK_INODE; i < CLRLK_TYPE_MAX; i++) {
-                if (!strcmp (clrlk_types[i], type)) {
-                        ret_type = i;
-                        break;
-                }
+    char *clrlk_types[CLRLK_TYPE_MAX] = {"inode", "entry", "posix"};
+    int ret_type = CLRLK_TYPE_MAX;
+    int i = 0;
+
+    for (i = CLRLK_INODE; i < CLRLK_TYPE_MAX; i++) {
+        if (!strcmp(clrlk_types[i], type)) {
+            ret_type = i;
+            break;
         }
+    }
 
-        return ret_type;
+    return ret_type;
 }
 
 int
-clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
-                      gf_boolean_t *chk_range)
+clrlk_get_lock_range(char *range_str, struct gf_flock *ulock,
+                     gf_boolean_t *chk_range)
 {
-        int     ret     =       -1;
-
-        if (!chk_range)
-                goto out;
+    int ret = -1;
 
-        if (!range_str) {
-                ret = 0;
-                *chk_range = _gf_false;
-                goto out;
-        }
-
-        if (sscanf (range_str, "%hd,%"PRId64"-""%"PRId64, &ulock->l_whence,
-                    &ulock->l_start, &ulock->l_len) != 3) {
-                goto out;
-        }
+    if (!chk_range)
+        goto out;
 
+    if (!range_str) {
         ret = 0;
-        *chk_range = _gf_true;
+        *chk_range = _gf_false;
+        goto out;
+    }
+
+    if (sscanf(range_str,
+               "%hd,%" PRId64 "-"
+               "%" PRId64,
+               &ulock->l_whence, &ulock->l_start, &ulock->l_len) != 3) {
+        goto out;
+    }
+
+    ret = 0;
+    *chk_range = _gf_true;
 out:
-        return ret;
+    return ret;
 }
 
 int
-clrlk_parse_args (const char* cmd, clrlk_args *args)
+clrlk_parse_args(const char *cmd, clrlk_args *args)
 {
-        char            *opts           = NULL;
-        char            *cur            = NULL;
-        char            *tok            = NULL;
-        char            *sptr           = NULL;
-        char            *free_ptr       = NULL;
-        char            kw[KW_MAX]     = {[KW_TYPE]     = 't',
-                                          [KW_KIND]     = 'k',
-                                          };
-        int             ret             = -1;
-        int             i               = 0;
-
-        GF_ASSERT (cmd);
-        free_ptr = opts = GF_CALLOC (1, strlen (cmd), gf_common_mt_char);
-        if (!opts)
-                goto out;
-
-        if (sscanf (cmd, GF_XATTR_CLRLK_CMD".%s", opts) < 1) {
-                ret = -1;
-                goto out;
+    char *opts = NULL;
+    char *cur = NULL;
+    char *tok = NULL;
+    char *sptr = NULL;
+    char *free_ptr = NULL;
+    char kw[KW_MAX] = {
+        [KW_TYPE] = 't',
+        [KW_KIND] = 'k',
+    };
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(cmd);
+    free_ptr = opts = GF_CALLOC(1, strlen(cmd), gf_common_mt_char);
+    if (!opts)
+        goto out;
+
+    if (sscanf(cmd, GF_XATTR_CLRLK_CMD ".%s", opts) < 1) {
+        ret = -1;
+        goto out;
+    }
+
+    /*clr_lk_prefix.ttype.kkind.args, args - type specific*/
+    cur = opts;
+    for (i = 0; i < KW_MAX && (tok = strtok_r(cur, ".", &sptr));
+         cur = NULL, i++) {
+        if (tok[0] != kw[i]) {
+            ret = -1;
+            goto out;
         }
-
-        /*clr_lk_prefix.ttype.kkind.args, args - type specific*/
-        cur = opts;
-        for (i = 0; i < KW_MAX && (tok = strtok_r (cur, ".", &sptr));
-             cur = NULL, i++) {
-                if (tok[0] != kw[i]) {
-                        ret = -1;
-                        goto out;
-                }
-                if (i == KW_TYPE)
-                        args->type = clrlk_get_type (tok+1);
-                if (i == KW_KIND)
-                        args->kind = clrlk_get_kind (tok+1);
-        }
-
-        if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
-                goto out;
-
-        /*optional args, neither range nor basename can 'legally' contain
-         * "/" in them*/
-        tok = strtok_r (NULL, "/", &sptr);
-        if (tok)
-                args->opts = gf_strdup (tok);
-
-        ret = 0;
+        if (i == KW_TYPE)
+            args->type = clrlk_get_type(tok + 1);
+        if (i == KW_KIND)
+            args->kind = clrlk_get_kind(tok + 1);
+    }
+
+    if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
+        goto out;
+
+    /*optional args, neither range nor basename can 'legally' contain
+     * "/" in them*/
+    tok = strtok_r(NULL, "/", &sptr);
+    if (tok)
+        args->opts = gf_strdup(tok);
+
+    ret = 0;
 out:
-        GF_FREE (free_ptr);
-        return ret;
+    GF_FREE(free_ptr);
+    return ret;
 }
 
 int
-clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
-                     int *blkd, int *granted, int *op_errno)
+clrlk_clear_posixlk(xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                    int *blkd, int *granted, int *op_errno)
 {
-        posix_lock_t            *plock          = NULL;
-        posix_lock_t            *tmp            = NULL;
-        struct gf_flock         ulock           = {0, };
-        int                     ret             = -1;
-        int                     bcount          = 0;
-        int                     gcount          = 0;
-        gf_boolean_t            chk_range       = _gf_false;
-
-        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        pthread_mutex_lock (&pl_inode->mutex);
+    posix_lock_t *plock = NULL;
+    posix_lock_t *tmp = NULL;
+    struct gf_flock ulock = {
+        0,
+    };
+    int ret = -1;
+    int bcount = 0;
+    int gcount = 0;
+    gf_boolean_t chk_range = _gf_false;
+
+    if (clrlk_get_lock_range(args->opts, &ulock, &chk_range)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(plock, tmp, &pl_inode->ext_list, list)
         {
-                list_for_each_entry_safe (plock, tmp, &pl_inode->ext_list,
-                                          list) {
-                        if ((plock->blocked &&
-                             !(args->kind & CLRLK_BLOCKED)) ||
-                            (!plock->blocked &&
-                             !(args->kind & CLRLK_GRANTED)))
-                                continue;
-
-                        if (chk_range &&
-                            (plock->user_flock.l_whence != ulock.l_whence
-                            || plock->user_flock.l_start != ulock.l_start
-                            || plock->user_flock.l_len != ulock.l_len))
-                                continue;
-
-                        list_del_init (&plock->list);
-                        if (plock->blocked) {
-                                bcount++;
-                                pl_trace_out (this, plock->frame, NULL, NULL,
-                                              F_SETLKW, &plock->user_flock,
-                                              -1, EAGAIN, NULL);
-
-                                STACK_UNWIND_STRICT (lk, plock->frame, -1, EAGAIN,
-                                                     &plock->user_flock, NULL);
-
-                        } else {
-                                gcount++;
-                        }
-                        GF_FREE (plock);
-                }
+            if ((plock->blocked && !(args->kind & CLRLK_BLOCKED)) ||
+                (!plock->blocked && !(args->kind & CLRLK_GRANTED)))
+                continue;
+
+            if (chk_range && (plock->user_flock.l_whence != ulock.l_whence ||
+                              plock->user_flock.l_start != ulock.l_start ||
+                              plock->user_flock.l_len != ulock.l_len))
+                continue;
+
+            list_del_init(&plock->list);
+            if (plock->blocked) {
+                bcount++;
+                pl_trace_out(this, plock->frame, NULL, NULL, F_SETLKW,
+                             &plock->user_flock, -1, EINTR, NULL);
+
+                STACK_UNWIND_STRICT(lk, plock->frame, -1, EINTR,
+                                    &plock->user_flock, NULL);
+
+            } else {
+                gcount++;
+            }
+            __destroy_lock(plock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
-        grant_blocked_locks (this, pl_inode);
-        ret = 0;
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    grant_blocked_locks(this, pl_inode);
+    ret = 0;
 out:
-        *blkd    = bcount;
-        *granted = gcount;
-        return ret;
+    *blkd = bcount;
+    *granted = gcount;
+    return ret;
 }
 
 /* Returns 0 on success and -1 on failure */
 int
-clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+clrlk_clear_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno)
 {
-        pl_inode_lock_t         *ilock          = NULL;
-        pl_inode_lock_t         *tmp            = NULL;
-        struct gf_flock         ulock           = {0, };
-        int                     ret             = -1;
-        int                     bcount          = 0;
-        int                     gcount          = 0;
-        gf_boolean_t            chk_range       = _gf_false;
-        struct list_head        released;
-
-        INIT_LIST_HEAD (&released);
-        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        if (args->kind & CLRLK_BLOCKED)
-                goto blkd;
-
-        if (args->kind & CLRLK_GRANTED)
-                goto granted;
+    posix_locks_private_t *priv;
+    pl_inode_lock_t *ilock = NULL;
+    pl_inode_lock_t *tmp = NULL;
+    struct gf_flock ulock = {
+        0,
+    };
+    int ret = -1;
+    int bcount = 0;
+    int gcount = 0;
+    gf_boolean_t chk_range = _gf_false;
+    struct list_head *pcontend = NULL;
+    struct list_head released;
+    struct list_head contend;
+    struct timespec now = {};
+
+    INIT_LIST_HEAD(&released);
+
+    priv = this->private;
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    if (clrlk_get_lock_range(args->opts, &ulock, &chk_range)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    if (args->kind & CLRLK_BLOCKED)
+        goto blkd;
+
+    if (args->kind & CLRLK_GRANTED)
+        goto granted;
 
 blkd:
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(ilock, tmp, &dom->blocked_inodelks,
+                                 blocked_locks)
         {
-                list_for_each_entry_safe (ilock, tmp, &dom->blocked_inodelks,
-                                          blocked_locks) {
-                        if (chk_range &&
-                            (ilock->user_flock.l_whence != ulock.l_whence
-                            || ilock->user_flock.l_start != ulock.l_start
-                            || ilock->user_flock.l_len != ulock.l_len))
-                                continue;
-
-                        bcount++;
-                        list_del_init (&ilock->blocked_locks);
-                        list_add (&ilock->blocked_locks, &released);
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (ilock, tmp, &released, blocked_locks) {
-                list_del_init (&ilock->blocked_locks);
-                pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
-                              &ilock->user_flock, -1, EAGAIN,
-                              ilock->volume);
-                STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
-                                     EAGAIN, NULL);
-                //No need to take lock as the locks are only in one list
-                __pl_inodelk_unref (ilock);
+            if (chk_range && (ilock->user_flock.l_whence != ulock.l_whence ||
+                              ilock->user_flock.l_start != ulock.l_start ||
+                              ilock->user_flock.l_len != ulock.l_len))
+                continue;
+
+            bcount++;
+            list_del_init(&ilock->client_list);
+            list_del_init(&ilock->blocked_locks);
+            list_add(&ilock->blocked_locks, &released);
         }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        if (!(args->kind & CLRLK_GRANTED)) {
-                ret = 0;
-                goto out;
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(ilock, tmp, &released, blocked_locks)
+        {
+            list_del_init(&ilock->blocked_locks);
+            pl_trace_out(this, ilock->frame, NULL, NULL, F_SETLKW,
+                         &ilock->user_flock, -1, EAGAIN, ilock->volume);
+            STACK_UNWIND_STRICT(inodelk, ilock->frame, -1, EAGAIN, NULL);
+            // No need to take lock as the locks are only in one list
+            __pl_inodelk_unref(ilock);
         }
+    }
+
+    if (!(args->kind & CLRLK_GRANTED)) {
+        ret = 0;
+        goto out;
+    }
 
 granted:
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(ilock, tmp, &dom->inodelk_list, list)
         {
-                list_for_each_entry_safe (ilock, tmp, &dom->inodelk_list,
-                                          list) {
-                        if (chk_range &&
-                            (ilock->user_flock.l_whence != ulock.l_whence
-                            || ilock->user_flock.l_start != ulock.l_start
-                            || ilock->user_flock.l_len != ulock.l_len))
-                                continue;
-
-                        gcount++;
-                        list_del_init (&ilock->list);
-                        list_add (&ilock->list, &released);
-                }
+            if (chk_range && (ilock->user_flock.l_whence != ulock.l_whence ||
+                              ilock->user_flock.l_start != ulock.l_start ||
+                              ilock->user_flock.l_len != ulock.l_len))
+                continue;
+
+            gcount++;
+            list_del_init(&ilock->client_list);
+            list_del_init(&ilock->list);
+            list_add(&ilock->list, &released);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        list_for_each_entry_safe (ilock, tmp, &released, list) {
-                list_del_init (&ilock->list);
-                //No need to take lock as the locks are only in one list
-                __pl_inodelk_unref (ilock);
-        }
+    list_for_each_entry_safe(ilock, tmp, &released, list)
+    {
+        list_del_init(&ilock->list);
+        // No need to take lock as the locks are only in one list
+        __pl_inodelk_unref(ilock);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        grant_blocked_inode_locks (this, pl_inode, dom);
-        *blkd    = bcount;
-        *granted = gcount;
-        return ret;
+    grant_blocked_inode_locks(this, pl_inode, dom, &now, pcontend);
+    if (pcontend != NULL) {
+        inodelk_contention_notify(this, pcontend);
+    }
+    *blkd = bcount;
+    *granted = gcount;
+    return ret;
 }
 
 /* Returns 0 on success and -1 on failure */
 int
-clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+clrlk_clear_entrylk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno)
 {
-        pl_entry_lock_t         *elock          = NULL;
-        pl_entry_lock_t         *tmp            = NULL;
-        int                     bcount          = 0;
-        int                     gcount          = 0;
-        int                     ret             = -1;
-        struct list_head        removed;
-        struct list_head        released;
-
-        INIT_LIST_HEAD (&released);
-        if (args->kind & CLRLK_BLOCKED)
-                goto blkd;
-
-        if (args->kind & CLRLK_GRANTED)
-                goto granted;
+    posix_locks_private_t *priv;
+    pl_entry_lock_t *elock = NULL;
+    pl_entry_lock_t *tmp = NULL;
+    int bcount = 0;
+    int gcount = 0;
+    int ret = -1;
+    struct list_head *pcontend = NULL;
+    struct list_head removed;
+    struct list_head released;
+    struct list_head contend;
+    struct timespec now;
+
+    INIT_LIST_HEAD(&released);
+
+    priv = this->private;
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    if (args->kind & CLRLK_BLOCKED)
+        goto blkd;
+
+    if (args->kind & CLRLK_GRANTED)
+        goto granted;
 
 blkd:
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(elock, tmp, &dom->blocked_entrylks,
+                                 blocked_locks)
         {
-                list_for_each_entry_safe (elock, tmp, &dom->blocked_entrylks,
-                                          blocked_locks) {
-                        if (args->opts) {
-                                if (!elock->basename ||
-                                    strcmp (elock->basename, args->opts))
-                                        continue;
-                        }
-
-                        bcount++;
-
-                        list_del_init (&elock->blocked_locks);
-                        list_add_tail (&elock->blocked_locks, &released);
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+            if (args->opts) {
+                if (!elock->basename || strcmp(elock->basename, args->opts))
+                    continue;
+            }
 
-        list_for_each_entry_safe (elock, tmp, &released, blocked_locks) {
-                list_del_init (&elock->blocked_locks);
-                entrylk_trace_out (this, elock->frame, elock->volume, NULL, NULL,
-                                   elock->basename, ENTRYLK_LOCK, elock->type,
-                                   -1, EAGAIN);
-                STACK_UNWIND_STRICT (entrylk, elock->frame, -1, EAGAIN, NULL);
+            bcount++;
 
-		__pl_entrylk_unref (elock);
+            list_del_init(&elock->client_list);
+            list_del_init(&elock->blocked_locks);
+            list_add_tail(&elock->blocked_locks, &released);
         }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(elock, tmp, &released, blocked_locks)
+        {
+            list_del_init(&elock->blocked_locks);
+            entrylk_trace_out(this, elock->frame, elock->volume, NULL, NULL,
+                              elock->basename, ENTRYLK_LOCK, elock->type, -1,
+                              EAGAIN);
+            STACK_UNWIND_STRICT(entrylk, elock->frame, -1, EAGAIN, NULL);
 
-        if (!(args->kind & CLRLK_GRANTED)) {
-                ret = 0;
-                goto out;
+            __pl_entrylk_unref(elock);
         }
+    }
+
+    if (!(args->kind & CLRLK_GRANTED)) {
+        ret = 0;
+        goto out;
+    }
 
 granted:
-        INIT_LIST_HEAD (&removed);
-        pthread_mutex_lock (&pl_inode->mutex);
+    INIT_LIST_HEAD(&removed);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(elock, tmp, &dom->entrylk_list, domain_list)
         {
-                list_for_each_entry_safe (elock, tmp, &dom->entrylk_list,
-                                          domain_list) {
-                        if (args->opts) {
-                                if (!elock->basename ||
-                                    strcmp (elock->basename, args->opts))
-                                        continue;
-                        }
-
-                        gcount++;
-                        list_del_init (&elock->domain_list);
-                        list_add_tail (&elock->domain_list, &removed);
-
-			__pl_entrylk_unref (elock);
-                }
+            if (args->opts) {
+                if (!elock->basename || strcmp(elock->basename, args->opts))
+                    continue;
+            }
+
+            gcount++;
+            list_del_init(&elock->client_list);
+            list_del_init(&elock->domain_list);
+            list_add_tail(&elock->domain_list, &removed);
+
+            __pl_entrylk_unref(elock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-	grant_blocked_entry_locks (this, pl_inode, dom);
+    grant_blocked_entry_locks(this, pl_inode, dom, &now, pcontend);
+    if (pcontend != NULL) {
+        entrylk_contention_notify(this, pcontend);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        *blkd    = bcount;
-        *granted = gcount;
-        return ret;
+    *blkd = bcount;
+    *granted = gcount;
+    return ret;
 }
 
 int
-clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
-                                clrlk_args *args, int *blkd, int *granted,
-                                int *op_errno)
+clrlk_clear_lks_in_all_domains(xlator_t *this, pl_inode_t *pl_inode,
+                               clrlk_args *args, int *blkd, int *granted,
+                               int *op_errno)
 {
-        pl_dom_list_t   *dom            = NULL;
-        int             ret             = -1;
-        int             tmp_bcount      = 0;
-        int             tmp_gcount      = 0;
-
-        if (list_empty (&pl_inode->dom_list)) {
-                ret = 0;
-                goto out;
-        }
+    pl_dom_list_t *dom = NULL;
+    int ret = -1;
+    int tmp_bcount = 0;
+    int tmp_gcount = 0;
 
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                tmp_bcount = tmp_gcount = 0;
-
-                switch (args->type)
-                {
-                case CLRLK_INODE:
-                        ret = clrlk_clear_inodelk (this, pl_inode, dom, args,
-                                             &tmp_bcount, &tmp_gcount,
-                                             op_errno);
-                        if (ret)
-                                goto out;
-                        break;
-                case CLRLK_ENTRY:
-                        ret = clrlk_clear_entrylk (this, pl_inode, dom, args,
-                                             &tmp_bcount, &tmp_gcount,
-                                             op_errno);
-                        if (ret)
-                                goto out;
-                        break;
-                }
-
-                *blkd    += tmp_bcount;
-                *granted += tmp_gcount;
+    if (list_empty(&pl_inode->dom_list)) {
+        ret = 0;
+        goto out;
+    }
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        tmp_bcount = tmp_gcount = 0;
+
+        switch (args->type) {
+            case CLRLK_INODE:
+                ret = clrlk_clear_inodelk(this, pl_inode, dom, args,
+                                          &tmp_bcount, &tmp_gcount, op_errno);
+                if (ret)
+                    goto out;
+                break;
+            case CLRLK_ENTRY:
+                ret = clrlk_clear_entrylk(this, pl_inode, dom, args,
+                                          &tmp_bcount, &tmp_gcount, op_errno);
+                if (ret)
+                    goto out;
+                break;
         }
 
-        ret = 0;
+        *blkd += tmp_bcount;
+        *granted += tmp_gcount;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/locks/src/clear.h b/xlators/features/locks/src/clear.h
index 78fc5ae3398..bc118cb1b81 100644
--- a/xlators/features/locks/src/clear.h
+++ b/xlators/features/locks/src/clear.h
@@ -10,62 +10,64 @@
 #ifndef __CLEAR_H__
 #define __CLEAR_H__
 
-#include "compat-errno.h"
-#include "stack.h"
-#include "call-stub.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/call-stub.h>
 #include "locks.h"
 
 typedef enum {
-        CLRLK_INODE,
-        CLRLK_ENTRY,
-        CLRLK_POSIX,
-        CLRLK_TYPE_MAX
+    CLRLK_INODE,
+    CLRLK_ENTRY,
+    CLRLK_POSIX,
+    CLRLK_TYPE_MAX
 } clrlk_type;
 
+extern const char *clrlk_type_names[];
+
 typedef enum {
-        CLRLK_BLOCKED = 1,
-        CLRLK_GRANTED,
-        CLRLK_ALL,
-        CLRLK_KIND_MAX
+    CLRLK_BLOCKED = 1,
+    CLRLK_GRANTED,
+    CLRLK_ALL,
+    CLRLK_KIND_MAX
 } clrlk_kind;
 
 typedef enum {
-        KW_TYPE,
-        KW_KIND,
-        /*add new keywords here*/
-        KW_MAX
+    KW_TYPE,
+    KW_KIND,
+    /*add new keywords here*/
+    KW_MAX
 } clrlk_opts;
 
 struct _clrlk_args;
 typedef struct _clrlk_args clrlk_args;
 
 struct _clrlk_args {
-        int type;
-        int kind;
-        char *opts;
+    int type;
+    int kind;
+    char *opts;
 };
 
 int
-clrlk_get__kind (char *kind);
+clrlk_get__kind(char *kind);
 int
-clrlk_get_type (char *type);
+clrlk_get_type(char *type);
 int
-clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
-                      gf_boolean_t *chk_range);
+clrlk_get_lock_range(char *range_str, struct gf_flock *ulock,
+                     gf_boolean_t *chk_range);
 int
-clrlk_parse_args (const char* cmd, clrlk_args *args);
+clrlk_parse_args(const char *cmd, clrlk_args *args);
 
 int
-clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
-                     int *blkd, int *granted, int *op_errno);
+clrlk_clear_posixlk(xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                    int *blkd, int *granted, int *op_errno);
 int
-clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+clrlk_clear_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno);
 int
-clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+clrlk_clear_entrylk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno);
 int
-clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
-                                clrlk_args *args, int *blkd, int *granted,
-                                int *op_errno);
+clrlk_clear_lks_in_all_domains(xlator_t *this, pl_inode_t *pl_inode,
+                               clrlk_args *args, int *blkd, int *granted,
+                               int *op_errno);
 #endif /* __CLEAR_H__ */
diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c
index 123c1101e94..a2c6be93e03 100644
--- a/xlators/features/locks/src/common.c
+++ b/xlators/features/locks/src/common.c
@@ -12,736 +12,775 @@
 #include <limits.h>
 #include <pthread.h>
 
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syncop.h>
 
 #include "locks.h"
 #include "common.h"
 
-
 static int
-__is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock);
+__is_lock_grantable(pl_inode_t *pl_inode, posix_lock_t *lock);
 static void
-__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock);
+__insert_and_merge(pl_inode_t *pl_inode, posix_lock_t *lock);
 static int
-pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
-                        posix_lock_t *old_lock);
+pl_send_prelock_unlock(xlator_t *this, pl_inode_t *pl_inode,
+                       posix_lock_t *old_lock);
 
 static pl_dom_list_t *
-__allocate_domain (const char *volume)
+__allocate_domain(const char *volume)
 {
-        pl_dom_list_t *dom = NULL;
+    pl_dom_list_t *dom = NULL;
 
-        dom = GF_CALLOC (1, sizeof (*dom),
-                         gf_locks_mt_pl_dom_list_t);
-        if (!dom)
-                goto out;
+    dom = GF_CALLOC(1, sizeof(*dom), gf_locks_mt_pl_dom_list_t);
+    if (!dom)
+        goto out;
 
-        dom->domain = gf_strdup(volume);
-        if (!dom->domain)
-                goto out;
+    dom->domain = gf_strdup(volume);
+    if (!dom->domain)
+        goto out;
 
-        gf_log ("posix-locks", GF_LOG_TRACE,
-                "New domain allocated: %s", dom->domain);
+    gf_log("posix-locks", GF_LOG_TRACE, "New domain allocated: %s",
+           dom->domain);
 
-        INIT_LIST_HEAD (&dom->inode_list);
-        INIT_LIST_HEAD (&dom->entrylk_list);
-        INIT_LIST_HEAD (&dom->blocked_entrylks);
-        INIT_LIST_HEAD (&dom->inodelk_list);
-        INIT_LIST_HEAD (&dom->blocked_inodelks);
+    INIT_LIST_HEAD(&dom->inode_list);
+    INIT_LIST_HEAD(&dom->entrylk_list);
+    INIT_LIST_HEAD(&dom->blocked_entrylks);
+    INIT_LIST_HEAD(&dom->inodelk_list);
+    INIT_LIST_HEAD(&dom->blocked_inodelks);
 
 out:
-        if (dom && (NULL == dom->domain)) {
-                GF_FREE (dom);
-                dom = NULL;
-        }
+    if (dom && (NULL == dom->domain)) {
+        GF_FREE(dom);
+        dom = NULL;
+    }
 
-        return dom;
+    return dom;
 }
 
 /* Returns domain for the lock. If domain is not present,
  * allocates a domain and returns it
  */
 pl_dom_list_t *
-get_domain (pl_inode_t *pl_inode, const char *volume)
+get_domain(pl_inode_t *pl_inode, const char *volume)
 {
-        pl_dom_list_t *dom = NULL;
+    pl_dom_list_t *dom = NULL;
 
-        GF_VALIDATE_OR_GOTO ("posix-locks", pl_inode, out);
-        GF_VALIDATE_OR_GOTO ("posix-locks", volume, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", pl_inode, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", volume, out);
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
         {
-                list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                        if (strcmp (dom->domain, volume) == 0)
-                                goto unlock;
-                }
-
-                dom = __allocate_domain (volume);
-                if (dom)
-                        list_add (&dom->inode_list, &pl_inode->dom_list);
+            if (strcmp(dom->domain, volume) == 0)
+                goto unlock;
         }
+
+        dom = __allocate_domain(volume);
+        if (dom)
+            list_add(&dom->inode_list, &pl_inode->dom_list);
+    }
 unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
-        if (dom) {
-                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
-        } else {
-                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
-        }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    if (dom) {
+        gf_log("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
+    } else {
+        gf_log("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
+    }
 out:
-        return dom;
+    return dom;
 }
 
 unsigned long
-fd_to_fdnum (fd_t *fd)
+fd_to_fdnum(fd_t *fd)
 {
-        return ((unsigned long) fd);
+    return ((unsigned long)fd);
 }
 
 fd_t *
-fd_from_fdnum (posix_lock_t *lock)
+fd_from_fdnum(posix_lock_t *lock)
 {
-        return ((fd_t *) lock->fd_num);
+    return ((fd_t *)lock->fd_num);
 }
 
 int
-__pl_inode_is_empty (pl_inode_t *pl_inode)
+__pl_inode_is_empty(pl_inode_t *pl_inode)
 {
-        return (list_empty (&pl_inode->ext_list));
+    return (list_empty(&pl_inode->ext_list));
 }
 
 void
-pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame)
+pl_print_locker(char *str, int size, xlator_t *this, call_frame_t *frame)
 {
-        snprintf (str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
-                  (unsigned long long) frame->root->pid,
-                  lkowner_utoa (&frame->root->lk_owner),
-                  frame->root->client,
-                  (unsigned long long) frame->root->unique);
+    snprintf(str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
+             (unsigned long long)frame->root->pid,
+             lkowner_utoa(&frame->root->lk_owner), frame->root->client,
+             (unsigned long long)frame->root->unique);
 }
 
-
 void
-pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc)
+pl_print_lockee(char *str, int size, fd_t *fd, loc_t *loc)
 {
-        inode_t  *inode = NULL;
-        char     *ipath = NULL;
-        int       ret = 0;
+    inode_t *inode = NULL;
+    char *ipath = NULL;
+    int ret = 0;
 
-        if (fd)
-                inode = fd->inode;
-        if (loc)
-                inode = loc->inode;
+    if (fd)
+        inode = fd->inode;
+    if (loc)
+        inode = loc->inode;
 
-        if (!inode) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
+    if (!inode) {
+        snprintf(str, size, "<nul>");
+        return;
+    }
 
-        if (loc && loc->path) {
-                ipath = gf_strdup (loc->path);
-        } else {
-                ret = inode_path (inode, NULL, &ipath);
-                if (ret <= 0)
-                        ipath = NULL;
-        }
+    if (loc && loc->path) {
+        ipath = gf_strdup(loc->path);
+    } else {
+        ret = inode_path(inode, NULL, &ipath);
+        if (ret <= 0)
+            ipath = NULL;
+    }
 
-        snprintf (str, size, "gfid=%s, fd=%p, path=%s",
-                  uuid_utoa (inode->gfid), fd,
-                  ipath ? ipath : "<nul>");
+    snprintf(str, size, "gfid=%s, fd=%p, path=%s", uuid_utoa(inode->gfid), fd,
+             ipath ? ipath : "<nul>");
 
-        GF_FREE (ipath);
+    GF_FREE(ipath);
 }
 
-
 void
-pl_print_lock (char *str, int size, int cmd,
-               struct gf_flock *flock, gf_lkowner_t *owner)
+pl_print_lock(char *str, int size, int cmd, struct gf_flock *flock,
+              gf_lkowner_t *owner)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
 
-        switch (cmd) {
+    switch (cmd) {
 #if F_GETLK != F_GETLK64
         case F_GETLK64:
 #endif
         case F_GETLK:
-                cmd_str = "GETLK";
-                break;
+            cmd_str = "GETLK";
+            break;
 
 #if F_SETLK != F_SETLK64
         case F_SETLK64:
 #endif
         case F_SETLK:
-                cmd_str = "SETLK";
-                break;
+            cmd_str = "SETLK";
+            break;
 
 #if F_SETLKW != F_SETLKW64
         case F_SETLKW64:
 #endif
         case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
+            cmd_str = "SETLKW";
+            break;
 
         default:
-                cmd_str = "UNKNOWN";
-                break;
-        }
+            cmd_str = "UNKNOWN";
+            break;
+    }
 
-        switch (flock->l_type) {
+    switch (flock->l_type) {
         case F_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case F_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
+            type_str = "UNLOCK";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        snprintf (str, size, "lock=FCNTL, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                  cmd_str, type_str, (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid,
-                  lkowner_utoa (owner));
+            type_str = "UNKNOWN";
+            break;
+    }
+
+    snprintf(str, size,
+             "lock=FCNTL, cmd=%s, type=%s, "
+             "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
+             cmd_str, type_str, (unsigned long long)flock->l_start,
+             (unsigned long long)flock->l_len, (unsigned long long)flock->l_pid,
+             lkowner_utoa(owner));
 }
 
-
 void
-pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-             int cmd, struct gf_flock *flock, const char *domain)
+pl_trace_in(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+            struct gf_flock *flock, const char *domain)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_lock[256];
-
-        priv = this->private;
+    posix_locks_private_t *priv = this->private;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_lock[256];
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        if (domain)
-                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
-        else
-                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    if (domain)
+        pl_print_inodelk(pl_lock, 256, cmd, flock, domain);
+    else
+        pl_print_lock(pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_lock);
 }
 
-
 void
-pl_print_verdict (char *str, int size, int op_ret, int op_errno)
+pl_print_verdict(char *str, int size, int op_ret, int op_errno)
 {
-        char *verdict = NULL;
-
-        if (op_ret == 0) {
-                verdict = "GRANTED";
-        } else {
-                switch (op_errno) {
-                case EAGAIN:
-                        verdict = "TRYAGAIN";
-                        break;
-                default:
-                        verdict = strerror (op_errno);
-                }
+    char *verdict = NULL;
+
+    if (op_ret == 0) {
+        verdict = "GRANTED";
+    } else {
+        switch (op_errno) {
+            case EAGAIN:
+                verdict = "TRYAGAIN";
+                break;
+            default:
+                verdict = strerror(op_errno);
         }
+    }
 
-        snprintf (str, size, "%s", verdict);
+    snprintf(str, size, "%s", verdict);
 }
 
-
 void
-pl_trace_out (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-              int cmd, struct gf_flock *flock, int op_ret, int op_errno, const char *domain)
+pl_trace_out(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+             struct gf_flock *flock, int op_ret, int op_errno,
+             const char *domain)
 
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_lock[256];
-        char                    verdict[32];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_lock[256];
+    char verdict[32];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        if (domain)
-                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
-        else
-                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    if (domain)
+        pl_print_inodelk(pl_lock, 256, cmd, flock, domain);
+    else
+        pl_print_lock(pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        pl_print_verdict (verdict, 32, op_ret, op_errno);
+    pl_print_verdict(verdict, 32, op_ret, op_errno);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                verdict, pl_locker, pl_lockee, pl_lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}", verdict, pl_locker,
+           pl_lockee, pl_lock);
 }
 
-
 void
-pl_trace_block (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                int cmd, struct gf_flock *flock, const char *domain)
+pl_trace_block(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+               int cmd, struct gf_flock *flock, const char *domain)
 
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_lock[256];
-
-        priv = this->private;
+    posix_locks_private_t *priv = this->private;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_lock[256];
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        if (domain)
-                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
-        else
-                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    if (domain)
+        pl_print_inodelk(pl_lock, 256, cmd, flock, domain);
+    else
+        pl_print_lock(pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_lock);
 }
 
-
 void
-pl_trace_flush (xlator_t *this, call_frame_t *frame, fd_t *fd)
+pl_trace_flush(xlator_t *this, call_frame_t *frame, fd_t *fd)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        pl_inode_t             *pl_inode = NULL;
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    pl_inode_t *pl_inode = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_inode = pl_inode_get (this, fd->inode);
+    pl_inode = pl_inode_get(this, fd->inode, NULL);
 
-        if (pl_inode && __pl_inode_is_empty (pl_inode))
-                return;
+    if (pl_inode && __pl_inode_is_empty(pl_inode))
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, NULL);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, NULL);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[FLUSH] Locker = {%s} Lockee = {%s}",
-                pl_locker, pl_lockee);
+    gf_log(this->name, GF_LOG_INFO, "[FLUSH] Locker = {%s} Lockee = {%s}",
+           pl_locker, pl_lockee);
 }
 
 void
-pl_trace_release (xlator_t *this, fd_t *fd)
+pl_trace_release(xlator_t *this, fd_t *fd)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_lockee[256];
+    posix_locks_private_t *priv = NULL;
+    char pl_lockee[256];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_lockee (pl_lockee, 256, fd, NULL);
+    pl_print_lockee(pl_lockee, 256, fd, NULL);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[RELEASE] Lockee = {%s}", pl_lockee);
+    gf_log(this->name, GF_LOG_INFO, "[RELEASE] Lockee = {%s}", pl_lockee);
 }
 
-
 void
-pl_update_refkeeper (xlator_t *this, inode_t *inode)
+pl_update_refkeeper(xlator_t *this, inode_t *inode)
 {
-        pl_inode_t *pl_inode  = NULL;
-        int         is_empty  = 0;
-        int         need_unref = 0;
-        int         need_ref = 0;
+    pl_inode_t *pl_inode = NULL;
+    int is_empty = 0;
+    int need_unref = 0;
+    int need_ref = 0;
 
-        pl_inode = pl_inode_get (this, inode);
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode)
+        return;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                is_empty = __pl_inode_is_empty (pl_inode);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        is_empty = __pl_inode_is_empty(pl_inode);
 
-                if (is_empty && pl_inode->refkeeper) {
-                        need_unref = 1;
-                        pl_inode->refkeeper = NULL;
-                }
+        if (is_empty && pl_inode->refkeeper) {
+            need_unref = 1;
+            pl_inode->refkeeper = NULL;
+        }
 
-                if (!is_empty && !pl_inode->refkeeper) {
-                        need_ref = 1;
-                        pl_inode->refkeeper = inode;
-                }
+        if (!is_empty && !pl_inode->refkeeper) {
+            need_ref = 1;
+            pl_inode->refkeeper = inode;
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        if (need_unref)
-                inode_unref (inode);
+    if (need_unref)
+        inode_unref(inode);
 
-        if (need_ref)
-                inode_ref (inode);
+    if (need_ref)
+        inode_ref(inode);
 }
 
+/* Get lock enforcement info from disk */
+int
+pl_fetch_mlock_info_from_disk(xlator_t *this, pl_inode_t *pl_inode,
+                              pl_local_t *local)
+{
+    dict_t *xdata_rsp = NULL;
+    int ret = 0;
+    int op_ret = 0;
+
+    if (!local) {
+        return -1;
+    }
+
+    if (local->fd) {
+        op_ret = syncop_fgetxattr(this, local->fd, &xdata_rsp,
+                                  GF_ENFORCE_MANDATORY_LOCK, NULL, NULL);
+    } else {
+        op_ret = syncop_getxattr(this, &local->loc[0], &xdata_rsp,
+                                 GF_ENFORCE_MANDATORY_LOCK, NULL, NULL);
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (op_ret >= 0) {
+            pl_inode->mlock_enforced = _gf_true;
+            pl_inode->check_mlock_info = _gf_false;
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, -op_ret, 0,
+                   "getxattr failed with %d", op_ret);
+            pl_inode->mlock_enforced = _gf_false;
+
+            if (-op_ret == ENODATA) {
+                pl_inode->check_mlock_info = _gf_false;
+            } else {
+                pl_inode->check_mlock_info = _gf_true;
+            }
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    return ret;
+}
 
 pl_inode_t *
-pl_inode_get (xlator_t *this, inode_t *inode)
+pl_inode_get(xlator_t *this, inode_t *inode, pl_local_t *local)
 {
-        uint64_t    tmp_pl_inode = 0;
-        pl_inode_t *pl_inode = NULL;
-        int         ret = 0;
+    uint64_t tmp_pl_inode = 0;
+    pl_inode_t *pl_inode = NULL;
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &tmp_pl_inode);
+        if (ret == 0) {
+            pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+            goto unlock;
+        }
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
-                if (ret == 0) {
-                        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-                        goto unlock;
-                }
-                pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
-                                      gf_locks_mt_pl_inode_t);
-                if (!pl_inode) {
-                        goto unlock;
-                }
+        pl_inode = GF_CALLOC(1, sizeof(*pl_inode), gf_locks_mt_pl_inode_t);
+        if (!pl_inode) {
+            goto unlock;
+        }
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Allocating new pl inode");
-
-                pthread_mutex_init (&pl_inode->mutex, NULL);
-
-                INIT_LIST_HEAD (&pl_inode->dom_list);
-                INIT_LIST_HEAD (&pl_inode->ext_list);
-                INIT_LIST_HEAD (&pl_inode->rw_list);
-                INIT_LIST_HEAD (&pl_inode->reservelk_list);
-                INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
-                INIT_LIST_HEAD (&pl_inode->blocked_calls);
-                INIT_LIST_HEAD (&pl_inode->metalk_list);
-		INIT_LIST_HEAD (&pl_inode->queued_locks);
-                gf_uuid_copy (pl_inode->gfid, inode->gfid);
-
-                ret = __inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
-                if (ret) {
-                        GF_FREE (pl_inode);
-                        pl_inode = NULL;
-                        goto unlock;
-                }
+        gf_log(this->name, GF_LOG_TRACE, "Allocating new pl inode");
+
+        pthread_mutex_init(&pl_inode->mutex, NULL);
+        pthread_cond_init(&pl_inode->check_fop_wind_count, 0);
+
+        INIT_LIST_HEAD(&pl_inode->dom_list);
+        INIT_LIST_HEAD(&pl_inode->ext_list);
+        INIT_LIST_HEAD(&pl_inode->rw_list);
+        INIT_LIST_HEAD(&pl_inode->reservelk_list);
+        INIT_LIST_HEAD(&pl_inode->blocked_reservelks);
+        INIT_LIST_HEAD(&pl_inode->blocked_calls);
+        INIT_LIST_HEAD(&pl_inode->metalk_list);
+        INIT_LIST_HEAD(&pl_inode->queued_locks);
+        INIT_LIST_HEAD(&pl_inode->waiting);
+        gf_uuid_copy(pl_inode->gfid, inode->gfid);
+
+        pl_inode->check_mlock_info = _gf_true;
+        pl_inode->mlock_enforced = _gf_false;
+
+        /* -2 means never looked up. -1 means something went wrong and link
+         * tracking is disabled. */
+        pl_inode->links = -2;
+
+        ret = __inode_ctx_put(inode, this, (uint64_t)(long)(pl_inode));
+        if (ret) {
+            pthread_mutex_destroy(&pl_inode->mutex);
+            GF_FREE(pl_inode);
+            pl_inode = NULL;
+            goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-        return pl_inode;
-}
+    if ((pl_inode != NULL) && pl_is_mandatory_locking_enabled(pl_inode) &&
+        pl_inode->check_mlock_info && local) {
+        /* Note: The lock enforcement information per file can be stored in the
+           attribute flag of stat(x) in posix. With that there won't be a need
+           for doing getxattr post a reboot
+        */
+        pl_fetch_mlock_info_from_disk(this, pl_inode, local);
+    }
 
+    return pl_inode;
+}
 
 /* Create a new posix_lock_t */
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
-                gf_lkowner_t *owner, fd_t *fd, uint32_t lk_flags, int blocking)
+new_posix_lock(struct gf_flock *flock, client_t *client, pid_t client_pid,
+               gf_lkowner_t *owner, fd_t *fd, uint32_t lk_flags, int blocking,
+               int32_t *op_errno)
 {
-        posix_lock_t *lock = NULL;
+    posix_lock_t *lock = NULL;
 
-        GF_VALIDATE_OR_GOTO ("posix-locks", flock, out);
-        GF_VALIDATE_OR_GOTO ("posix-locks", client, out);
-        GF_VALIDATE_OR_GOTO ("posix-locks", fd, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", flock, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", client, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", fd, out);
 
-        lock = GF_CALLOC (1, sizeof (posix_lock_t),
-                          gf_locks_mt_posix_lock_t);
-        if (!lock) {
-                goto out;
-        }
+    if (!pl_is_lk_owner_valid(owner, client)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
 
-        lock->fl_start = flock->l_start;
-        lock->fl_type  = flock->l_type;
+    lock = GF_CALLOC(1, sizeof(posix_lock_t), gf_locks_mt_posix_lock_t);
+    if (!lock) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (flock->l_len == 0)
-                lock->fl_end = LLONG_MAX;
-        else
-                lock->fl_end = flock->l_start + flock->l_len - 1;
+    lock->fl_start = flock->l_start;
+    lock->fl_type = flock->l_type;
 
-        lock->client     = client;
+    if (flock->l_len == 0)
+        lock->fl_end = LLONG_MAX;
+    else
+        lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        lock->client_uid = gf_strdup (client->client_uid);
-        if (lock->client_uid == NULL) {
-                GF_FREE (lock);
-                goto out;
-        }
+    lock->client = client;
+
+    lock->client_uid = gf_strdup(client->client_uid);
+    if (lock->client_uid == NULL) {
+        GF_FREE(lock);
+        lock = NULL;
+        *op_errno = ENOMEM;
+        goto out;
+    }
 
-        lock->fd_num     = fd_to_fdnum (fd);
-        lock->fd         = fd;
-        lock->client_pid = client_pid;
-        lock->owner      = *owner;
-        lock->lk_flags   = lk_flags;
+    lock->fd_num = fd_to_fdnum(fd);
+    lock->fd = fd;
+    lock->client_pid = client_pid;
+    lock->owner = *owner;
+    lock->lk_flags = lk_flags;
 
-        lock->blocking  = blocking;
+    lock->blocking = blocking;
+    memcpy(&lock->user_flock, flock, sizeof(lock->user_flock));
 
-        INIT_LIST_HEAD (&lock->list);
+    INIT_LIST_HEAD(&lock->list);
 
 out:
-        return lock;
+    return lock;
 }
 
-
 /* Delete a lock from the inode's lock list */
 void
-__delete_lock (posix_lock_t *lock)
+__delete_lock(posix_lock_t *lock)
 {
-        list_del_init (&lock->list);
+    list_del_init(&lock->list);
 }
 
-
 /* Destroy a posix_lock */
 void
-__destroy_lock (posix_lock_t *lock)
+__destroy_lock(posix_lock_t *lock)
 {
-        GF_FREE (lock);
+    GF_FREE(lock->client_uid);
+    GF_FREE(lock);
 }
 
+static posix_lock_t *
+__copy_lock(posix_lock_t *src)
+{
+    posix_lock_t *dst;
+
+    dst = GF_MALLOC(sizeof(posix_lock_t), gf_locks_mt_posix_lock_t);
+    if (dst != NULL) {
+        memcpy(dst, src, sizeof(posix_lock_t));
+        dst->client_uid = gf_strdup(src->client_uid);
+        if (dst->client_uid == NULL) {
+            GF_FREE(dst);
+            dst = NULL;
+        }
+
+        if (dst != NULL)
+            INIT_LIST_HEAD(&dst->list);
+    }
+
+    return dst;
+}
 
 /* Convert a posix_lock to a struct gf_flock */
 void
-posix_lock_to_flock (posix_lock_t *lock, struct gf_flock *flock)
+posix_lock_to_flock(posix_lock_t *lock, struct gf_flock *flock)
 {
-        flock->l_pid   = lock->client_pid;
-        flock->l_type  = lock->fl_type;
-        flock->l_start = lock->fl_start;
-        flock->l_owner = lock->owner;
-
-        if (lock->fl_end == LLONG_MAX)
-                flock->l_len = 0;
-        else
-                flock->l_len = lock->fl_end - lock->fl_start + 1;
+    flock->l_pid = lock->user_flock.l_pid;
+    flock->l_type = lock->fl_type;
+    flock->l_start = lock->fl_start;
+    flock->l_owner = lock->owner;
+
+    if (lock->fl_end == LLONG_MAX)
+        flock->l_len = 0;
+    else
+        flock->l_len = lock->fl_end - lock->fl_start + 1;
 }
 
 /* Insert the lock into the inode's lock list */
 static void
-__insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
+__insert_lock(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        if (lock->blocked)
-                gettimeofday (&lock->blkd_time, NULL);
-        else
-                gettimeofday (&lock->granted_time, NULL);
-
-        list_add_tail (&lock->list, &pl_inode->ext_list);
+    if (lock->blocked)
+        lock->blkd_time = gf_time();
+    else
+        lock->granted_time = gf_time();
 
-        return;
+    list_add_tail(&lock->list, &pl_inode->ext_list);
 }
 
-
 /* Return true if the locks overlap, false otherwise */
 int
-locks_overlap (posix_lock_t *l1, posix_lock_t *l2)
+locks_overlap(posix_lock_t *l1, posix_lock_t *l2)
 {
-        /*
-           Note:
-           FUSE always gives us absolute offsets, so no need to worry
-           about SEEK_CUR or SEEK_END
-        */
+    /*
+       Note:
+       FUSE always gives us absolute offsets, so no need to worry
+       about SEEK_CUR or SEEK_END
+    */
 
-        return ((l1->fl_end >= l2->fl_start) &&
-                (l2->fl_end >= l1->fl_start));
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
 }
 
-
 /* Return true if the locks have the same owner */
 int
-same_owner (posix_lock_t *l1, posix_lock_t *l2)
+same_owner(posix_lock_t *l1, posix_lock_t *l2)
 {
-
-        return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->client == l2->client));
-
+    return (is_same_lkowner(&l1->owner, &l2->owner) &&
+            (l1->client == l2->client));
 }
 
-
 /* Delete all F_UNLCK locks */
 void
-__delete_unlck_locks (pl_inode_t *pl_inode)
+__delete_unlck_locks(pl_inode_t *pl_inode)
 {
-        posix_lock_t *l = NULL;
-        posix_lock_t *tmp = NULL;
-
-        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if (l->fl_type == F_UNLCK) {
-                        __delete_lock (l);
-                        __destroy_lock (l);
-                }
+    posix_lock_t *l = NULL;
+    posix_lock_t *tmp = NULL;
+
+    list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+    {
+        if (l->fl_type == F_UNLCK) {
+            __delete_lock(l);
+            __destroy_lock(l);
         }
+    }
 }
 
-
 /* Add two locks */
 static posix_lock_t *
-add_locks (posix_lock_t *l1, posix_lock_t *l2)
+add_locks(posix_lock_t *l1, posix_lock_t *l2, posix_lock_t *dst)
 {
-        posix_lock_t *sum = NULL;
+    posix_lock_t *sum = NULL;
 
-        sum = GF_CALLOC (1, sizeof (posix_lock_t),
-                         gf_locks_mt_posix_lock_t);
-        if (!sum)
-                return NULL;
+    sum = __copy_lock(dst);
+    if (!sum)
+        return NULL;
 
-        sum->fl_start = min (l1->fl_start, l2->fl_start);
-        sum->fl_end   = max (l1->fl_end, l2->fl_end);
+    sum->fl_start = min(l1->fl_start, l2->fl_start);
+    sum->fl_end = max(l1->fl_end, l2->fl_end);
 
-        return sum;
+    posix_lock_to_flock(sum, &sum->user_flock);
+
+    return sum;
 }
 
 /* Subtract two locks */
 struct _values {
-        posix_lock_t *locks[3];
+    posix_lock_t *locks[3];
 };
 
 /* {big} must always be contained inside {small} */
 static struct _values
-subtract_locks (posix_lock_t *big, posix_lock_t *small)
+subtract_locks(posix_lock_t *big, posix_lock_t *small)
 {
+    struct _values v = {.locks = {0, 0, 0}};
 
-        struct _values v = { .locks = {0, 0, 0} };
-
-        if ((big->fl_start == small->fl_start) &&
-            (big->fl_end   == small->fl_end)) {
-                /* both edges coincide with big */
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_type = small->fl_type;
-                goto done;
+    if ((big->fl_start == small->fl_start) && (big->fl_end == small->fl_end)) {
+        /* both edges coincide with big */
+        v.locks[0] = __copy_lock(big);
+        if (!v.locks[0]) {
+            goto out;
         }
 
-        if ((small->fl_start > big->fl_start) &&
-            (small->fl_end   < big->fl_end)) {
-                /* both edges lie inside big */
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
-
-                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                v.locks[2] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_end = small->fl_start - 1;
-
-                memcpy (v.locks[1], small, sizeof (posix_lock_t));
-
-                memcpy (v.locks[2], big, sizeof (posix_lock_t));
-                v.locks[2]->fl_start = small->fl_end + 1;
-                goto done;
-
+        v.locks[0]->fl_type = small->fl_type;
+        v.locks[0]->user_flock.l_type = small->fl_type;
+        goto done;
+    }
+
+    if ((small->fl_start > big->fl_start) && (small->fl_end < big->fl_end)) {
+        /* both edges lie inside big */
+        v.locks[0] = __copy_lock(big);
+        v.locks[1] = __copy_lock(small);
+        v.locks[2] = __copy_lock(big);
+        if ((v.locks[0] == NULL) || (v.locks[1] == NULL) ||
+            (v.locks[2] == NULL)) {
+            goto out;
         }
 
-        /* one edge coincides with big */
-        if (small->fl_start == big->fl_start) {
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
-
-                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_start = small->fl_end + 1;
-
-                memcpy (v.locks[1], small, sizeof (posix_lock_t));
-                goto done;
+        v.locks[0]->fl_end = small->fl_start - 1;
+        v.locks[2]->fl_start = small->fl_end + 1;
+        posix_lock_to_flock(v.locks[0], &v.locks[0]->user_flock);
+        posix_lock_to_flock(v.locks[2], &v.locks[2]->user_flock);
+        goto done;
+    }
+
+    /* one edge coincides with big */
+    if (small->fl_start == big->fl_start) {
+        v.locks[0] = __copy_lock(big);
+        v.locks[1] = __copy_lock(small);
+        if ((v.locks[0] == NULL) || (v.locks[1] == NULL)) {
+            goto out;
         }
 
-        if (small->fl_end  == big->fl_end) {
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
-
-                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_end = small->fl_start - 1;
+        v.locks[0]->fl_start = small->fl_end + 1;
+        posix_lock_to_flock(v.locks[0], &v.locks[0]->user_flock);
+        goto done;
+    }
 
-                memcpy (v.locks[1], small, sizeof (posix_lock_t));
-                goto done;
+    if (small->fl_end == big->fl_end) {
+        v.locks[0] = __copy_lock(big);
+        v.locks[1] = __copy_lock(small);
+        if ((v.locks[0] == NULL) || (v.locks[1] == NULL)) {
+            goto out;
         }
 
-        GF_ASSERT (0);
-        gf_log ("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
+        v.locks[0]->fl_end = small->fl_start - 1;
+        posix_lock_to_flock(v.locks[0], &v.locks[0]->user_flock);
+        goto done;
+    }
+
+    GF_ASSERT(0);
+    gf_log("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
 
 out:
-        if (v.locks[0]) {
-                GF_FREE (v.locks[0]);
-                v.locks[0] = NULL;
-        }
-        if (v.locks[1]) {
-                GF_FREE (v.locks[1]);
-                v.locks[1] = NULL;
-        }
-        if (v.locks[2]) {
-                GF_FREE (v.locks[2]);
-                v.locks[2] = NULL;
-        }
+    if (v.locks[0]) {
+        __destroy_lock(v.locks[0]);
+        v.locks[0] = NULL;
+    }
+    if (v.locks[1]) {
+        __destroy_lock(v.locks[1]);
+        v.locks[1] = NULL;
+    }
+    if (v.locks[2]) {
+        __destroy_lock(v.locks[2]);
+        v.locks[2] = NULL;
+    }
 
 done:
-        return v;
+    return v;
 }
 
 static posix_lock_t *
-first_conflicting_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+first_conflicting_overlap(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
-        posix_lock_t *conf = NULL;
+    posix_lock_t *l = NULL;
+    posix_lock_t *conf = NULL;
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
         {
-                list_for_each_entry (l, &pl_inode->ext_list, list) {
-                        if (l->blocked)
-                                continue;
-
-                        if (locks_overlap (l, lock)) {
-                                if (same_owner (l, lock))
-                                        continue;
-
-                                if ((l->fl_type == F_WRLCK) ||
-                                    (lock->fl_type == F_WRLCK)) {
-                                        conf = l;
-                                        goto unlock;
-                                }
-                        }
+            if (l->blocked)
+                continue;
+
+            if (locks_overlap(l, lock)) {
+                if (same_owner(l, lock))
+                    continue;
+
+                if ((l->fl_type == F_WRLCK) || (lock->fl_type == F_WRLCK)) {
+                    conf = l;
+                    goto unlock;
                 }
+            }
         }
+    }
 unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        return conf;
+    return conf;
 }
 
 /*
@@ -750,374 +789,803 @@ unlock:
   If {begin} is NULL, then start from the beginning of the list
 */
 static posix_lock_t *
-first_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+first_overlap(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
+    posix_lock_t *l = NULL;
 
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (l->blocked)
-                        continue;
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (l->blocked)
+            continue;
 
-                if (locks_overlap (l, lock))
-                        return l;
-        }
+        if (locks_overlap(l, lock))
+            return l;
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
-
 /* Return true if lock is grantable */
 static int
-__is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock)
+__is_lock_grantable(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
-        int           ret = 1;
-
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (!l->blocked && locks_overlap (lock, l)) {
-                        if (((l->fl_type == F_WRLCK)
-                             || (lock->fl_type == F_WRLCK))
-                            && (lock->fl_type != F_UNLCK)
-                            && !same_owner (l, lock)) {
-                                ret = 0;
-                                break;
-                        }
-                }
+    posix_lock_t *l = NULL;
+    int ret = 1;
+
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (!l->blocked && locks_overlap(lock, l)) {
+            if (((l->fl_type == F_WRLCK) || (lock->fl_type == F_WRLCK)) &&
+                (lock->fl_type != F_UNLCK) && !same_owner(l, lock)) {
+                ret = 0;
+                break;
+            }
         }
-        return ret;
+    }
+    return ret;
 }
 
-
-extern void do_blocked_rw (pl_inode_t *);
-
+extern void
+do_blocked_rw(pl_inode_t *);
 
 static void
-__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
+__insert_and_merge(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t  *conf = NULL;
-        posix_lock_t  *t = NULL;
-        posix_lock_t  *sum = NULL;
-        int            i = 0;
-        struct _values v = { .locks = {0, 0, 0} };
-        client_t      *client = NULL;
-
-        list_for_each_entry_safe (conf, t, &pl_inode->ext_list, list) {
-                if (conf->blocked)
-                        continue;
-                if (!locks_overlap (conf, lock))
-                        continue;
+    posix_lock_t *conf = NULL;
+    posix_lock_t *t = NULL;
+    posix_lock_t *sum = NULL;
+    int i = 0;
+    struct _values v = {.locks = {0, 0, 0}};
+
+    list_for_each_entry_safe(conf, t, &pl_inode->ext_list, list)
+    {
+        if (conf->blocked)
+            continue;
+        if (!locks_overlap(conf, lock))
+            continue;
+
+        if (same_owner(conf, lock)) {
+            if (conf->fl_type == lock->fl_type &&
+                conf->lk_flags == lock->lk_flags) {
+                sum = add_locks(lock, conf, lock);
+
+                __delete_lock(conf);
+                __destroy_lock(conf);
+
+                __destroy_lock(lock);
+                INIT_LIST_HEAD(&sum->list);
+                posix_lock_to_flock(sum, &sum->user_flock);
+                __insert_and_merge(pl_inode, sum);
 
-                if (same_owner (conf, lock)) {
-                        if (conf->fl_type == lock->fl_type &&
-                                        conf->lk_flags == lock->lk_flags) {
-                                sum = add_locks (lock, conf);
-
-                                sum->fl_type    = lock->fl_type;
-                                sum->client     = lock->client;
-                                client          = sum->client;
-                                sum->client_uid =
-                                         gf_strdup (client->client_uid);
-                                sum->fd_num     = lock->fd_num;
-                                sum->client_pid = lock->client_pid;
-                                sum->owner      = lock->owner;
-                                sum->lk_flags   = lock->lk_flags;
-
-                                __delete_lock (conf);
-                                __destroy_lock (conf);
-
-                                __destroy_lock (lock);
-                                INIT_LIST_HEAD (&sum->list);
-                                posix_lock_to_flock (sum, &sum->user_flock);
-                                __insert_and_merge (pl_inode, sum);
-
-                                return;
-                        } else {
-                                sum = add_locks (lock, conf);
-
-                                sum->fl_type    = conf->fl_type;
-                                sum->client     = conf->client;
-                                client          = sum->client;
-                                sum->client_uid =
-                                         gf_strdup (client->client_uid);
-
-                                sum->fd_num     = conf->fd_num;
-                                sum->client_pid = conf->client_pid;
-                                sum->owner      = conf->owner;
-                                sum->lk_flags   = conf->lk_flags;
-
-                                v = subtract_locks (sum, lock);
-
-                                __delete_lock (conf);
-                                __destroy_lock (conf);
-
-                                __delete_lock (lock);
-                                __destroy_lock (lock);
-
-                                __destroy_lock (sum);
-
-                                for (i = 0; i < 3; i++) {
-                                        if (!v.locks[i])
-                                                continue;
-
-                                        INIT_LIST_HEAD (&v.locks[i]->list);
-                                        posix_lock_to_flock (v.locks[i],
-                                                       &v.locks[i]->user_flock);
-                                        __insert_and_merge (pl_inode,
-                                                            v.locks[i]);
-                                }
-
-                                __delete_unlck_locks (pl_inode);
-                                return;
-                        }
-                }
+                return;
+            } else {
+                sum = add_locks(lock, conf, conf);
+
+                v = subtract_locks(sum, lock);
 
-                if (lock->fl_type == F_UNLCK) {
+                __delete_lock(conf);
+                __destroy_lock(conf);
+
+                __delete_lock(lock);
+                __destroy_lock(lock);
+
+                __destroy_lock(sum);
+
+                for (i = 0; i < 3; i++) {
+                    if (!v.locks[i])
                         continue;
-                }
 
-                if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
-                        __insert_lock (pl_inode, lock);
-                        return;
+                    __insert_and_merge(pl_inode, v.locks[i]);
                 }
+
+                __delete_unlck_locks(pl_inode);
+                return;
+            }
         }
 
-        /* no conflicts, so just insert */
-        if (lock->fl_type != F_UNLCK) {
-                __insert_lock (pl_inode, lock);
-        } else {
-                __destroy_lock (lock);
+        if (lock->fl_type == F_UNLCK) {
+            continue;
         }
-}
 
+        if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
+            __insert_lock(pl_inode, lock);
+            return;
+        }
+    }
+
+    /* no conflicts, so just insert */
+    if (lock->fl_type != F_UNLCK) {
+        __insert_lock(pl_inode, lock);
+    } else {
+        __destroy_lock(lock);
+    }
+}
 
 void
-__grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode, struct list_head *granted)
+__grant_blocked_locks(xlator_t *this, pl_inode_t *pl_inode,
+                      struct list_head *granted)
 {
-        struct list_head  tmp_list;
-        posix_lock_t     *l = NULL;
-        posix_lock_t     *tmp = NULL;
-        posix_lock_t     *conf = NULL;
+    struct list_head tmp_list;
+    posix_lock_t *l = NULL;
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *conf = NULL;
+
+    INIT_LIST_HEAD(&tmp_list);
+
+    list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+    {
+        if (l->blocked) {
+            conf = first_overlap(pl_inode, l);
+            if (conf)
+                continue;
+
+            l->blocked = 0;
+            list_move_tail(&l->list, &tmp_list);
+        }
+    }
 
-        INIT_LIST_HEAD (&tmp_list);
+    list_for_each_entry_safe(l, tmp, &tmp_list, list)
+    {
+        list_del_init(&l->list);
 
-        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if (l->blocked) {
-                        conf = first_overlap (pl_inode, l);
-                        if (conf)
-                                continue;
+        if (__is_lock_grantable(pl_inode, l)) {
+            conf = GF_CALLOC(1, sizeof(*conf), gf_locks_mt_posix_lock_t);
 
-                        l->blocked = 0;
-                        list_move_tail (&l->list, &tmp_list);
-                }
+            if (!conf) {
+                l->blocked = 1;
+                __insert_lock(pl_inode, l);
+                continue;
+            }
+
+            conf->frame = l->frame;
+            l->frame = NULL;
+
+            posix_lock_to_flock(l, &conf->user_flock);
+
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64
+                   " => Granted",
+                   l->fl_type == F_UNLCK ? "Unlock" : "Lock", l->client_pid,
+                   lkowner_utoa(&l->owner), l->user_flock.l_start,
+                   l->user_flock.l_len);
+
+            __insert_and_merge(pl_inode, l);
+
+            list_add(&conf->list, granted);
+        } else {
+            l->blocked = 1;
+            __insert_lock(pl_inode, l);
         }
+    }
+}
 
-        list_for_each_entry_safe (l, tmp, &tmp_list, list) {
-                list_del_init (&l->list);
+void
+grant_blocked_locks(xlator_t *this, pl_inode_t *pl_inode)
+{
+    struct list_head granted_list;
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *lock = NULL;
+    pl_local_t *local = NULL;
+    INIT_LIST_HEAD(&granted_list);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_locks(this, pl_inode, &granted_list);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted_list, list)
+    {
+        list_del_init(&lock->list);
+
+        pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                     0, 0, NULL);
+        local = lock->frame->local;
+        PL_STACK_UNWIND_AND_FREE(local, lk, lock->frame, 0, 0,
+                                 &lock->user_flock, NULL);
+        __destroy_lock(lock);
+    }
+
+    return;
+}
 
-                if (__is_lock_grantable (pl_inode, l)) {
-                        conf = GF_CALLOC (1, sizeof (*conf),
-                                          gf_locks_mt_posix_lock_t);
+static int
+pl_send_prelock_unlock(xlator_t *this, pl_inode_t *pl_inode,
+                       posix_lock_t *old_lock)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    posix_lock_t *unlock_lock = NULL;
+    int32_t op_errno = 0;
 
-                        if (!conf) {
-                                l->blocked = 1;
-                                __insert_lock (pl_inode, l);
-                                continue;
-                        }
+    struct list_head granted_list;
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *lock = NULL;
+    pl_local_t *local = NULL;
 
-                        conf->frame = l->frame;
-                        l->frame = NULL;
+    int ret = -1;
 
-                        posix_lock_to_flock (l, &conf->user_flock);
+    INIT_LIST_HEAD(&granted_list);
 
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Granted",
-                                l->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                l->client_pid, lkowner_utoa (&l->owner),
-                                l->user_flock.l_start,
-                                l->user_flock.l_len);
+    flock.l_type = F_UNLCK;
+    flock.l_whence = old_lock->user_flock.l_whence;
+    flock.l_start = old_lock->user_flock.l_start;
+    flock.l_len = old_lock->user_flock.l_len;
+    flock.l_pid = old_lock->user_flock.l_pid;
 
-                        __insert_and_merge (pl_inode, l);
+    unlock_lock = new_posix_lock(&flock, old_lock->client, old_lock->client_pid,
+                                 &old_lock->owner, old_lock->fd,
+                                 old_lock->lk_flags, 0, &op_errno);
+    GF_VALIDATE_OR_GOTO(this->name, unlock_lock, out);
+    ret = 0;
 
-                        list_add (&conf->list, granted);
-                } else {
-                        l->blocked = 1;
-                        __insert_lock (pl_inode, l);
-                }
+    __insert_and_merge(pl_inode, unlock_lock);
+
+    __grant_blocked_locks(this, pl_inode, &granted_list);
+
+    list_for_each_entry_safe(lock, tmp, &granted_list, list)
+    {
+        list_del_init(&lock->list);
+
+        pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                     0, 0, NULL);
+        local = lock->frame->local;
+        PL_STACK_UNWIND_AND_FREE(local, lk, lock->frame, 0, 0,
+                                 &lock->user_flock, NULL);
+        __destroy_lock(lock);
+    }
+
+out:
+    return ret;
+}
+
+int
+pl_setlk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+         int can_block)
+{
+    int ret = 0;
+
+    errno = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        /* Send unlock before the actual lock to
+           prevent lock upgrade / downgrade
+           problems only if:
+           - it is a blocking call
+           - it has other conflicting locks
+        */
+
+        if (can_block && !(__is_lock_grantable(pl_inode, lock))) {
+            ret = pl_send_prelock_unlock(this, pl_inode, lock);
+            if (ret)
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "Could not send pre-lock "
+                       "unlock");
+        }
+
+        if (__is_lock_grantable(pl_inode, lock)) {
+            if (pl_metalock_is_active(pl_inode)) {
+                __pl_queue_lock(pl_inode, lock);
+                pthread_mutex_unlock(&pl_inode->mutex);
+                ret = -2;
+                goto out;
+            }
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64 " => OK",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+            __insert_and_merge(pl_inode, lock);
+        } else if (can_block) {
+            if (pl_metalock_is_active(pl_inode)) {
+                __pl_queue_lock(pl_inode, lock);
+                pthread_mutex_unlock(&pl_inode->mutex);
+                ret = -2;
+                goto out;
+            }
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64
+                   " => Blocked",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+
+            pl_trace_block(this, lock->frame, NULL, NULL, F_SETLKW,
+                           &lock->user_flock, NULL);
+
+            lock->blocked = 1;
+            __insert_lock(pl_inode, lock);
+            ret = -1;
+        } else {
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64 " => NOK",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+            errno = EAGAIN;
+            ret = -1;
         }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    grant_blocked_locks(this, pl_inode);
+
+    do_blocked_rw(pl_inode);
+
+out:
+    return ret;
+}
+
+posix_lock_t *
+pl_getlk(pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+    posix_lock_t *conf = first_conflicting_overlap(pl_inode, lock);
+    if (conf == NULL) {
+        lock->fl_type = F_UNLCK;
+        return lock;
+    }
+
+    return conf;
 }
 
+gf_boolean_t
+pl_does_monkey_want_stuck_lock()
+{
+    long int monkey_unlock_rand = 0;
+    long int monkey_unlock_rand_rem = 0;
+
+    /* coverity[DC.WEAK_CRYPTO] */
+    monkey_unlock_rand = random();
+    monkey_unlock_rand_rem = monkey_unlock_rand % 100;
+    if (monkey_unlock_rand_rem == 0)
+        return _gf_true;
+    return _gf_false;
+}
 
-void
-grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode)
+int
+pl_lock_preempt(pl_inode_t *pl_inode, posix_lock_t *reqlock)
 {
-        struct list_head granted_list;
-        posix_lock_t     *tmp = NULL;
-        posix_lock_t     *lock = NULL;
+    posix_lock_t *lock = NULL;
+    posix_lock_t *i = NULL;
+    pl_rw_req_t *rw = NULL;
+    pl_rw_req_t *itr = NULL;
+    struct list_head unwind_blist = {
+        0,
+    };
+    struct list_head unwind_rw_list = {
+        0,
+    };
+    int ret = 0;
+
+    INIT_LIST_HEAD(&unwind_blist);
+    INIT_LIST_HEAD(&unwind_rw_list);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        /*
+            - go through the lock list
+            - remove all locks from different owners
+            - same owner locks will be added or substracted based on
+              the new request
+            - add the new lock
+        */
+        list_for_each_entry_safe(lock, i, &pl_inode->ext_list, list)
+        {
+            if (lock->blocked) {
+                list_del_init(&lock->list);
+                list_add(&lock->list, &unwind_blist);
+                continue;
+            }
+
+            if (locks_overlap(lock, reqlock)) {
+                if (same_owner(lock, reqlock))
+                    continue;
+
+                /* remove conflicting locks */
+                list_del_init(&lock->list);
+                __delete_lock(lock);
+                __destroy_lock(lock);
+            }
+        }
 
-        INIT_LIST_HEAD (&granted_list);
+        __insert_and_merge(pl_inode, reqlock);
 
-        pthread_mutex_lock (&pl_inode->mutex);
+        list_for_each_entry_safe(rw, itr, &pl_inode->rw_list, list)
         {
-                __grant_blocked_locks (this, pl_inode, &granted_list);
+            list_del_init(&rw->list);
+            list_add(&rw->list, &unwind_rw_list);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    /* unwind blocked locks */
+    list_for_each_entry_safe(lock, i, &unwind_blist, list)
+    {
+        PL_STACK_UNWIND_AND_FREE(((pl_local_t *)lock->frame->local), lk,
+                                 lock->frame, -1, EBUSY, &lock->user_flock,
+                                 NULL);
+        __destroy_lock(lock);
+    }
+
+    /* unwind blocked IOs */
+    list_for_each_entry_safe(rw, itr, &unwind_rw_list, list)
+    {
+        pl_clean_local(rw->stub->frame->local);
+        call_unwind_error(rw->stub, -1, EBUSY);
+    }
+
+    return ret;
+}
 
-        list_for_each_entry_safe (lock, tmp, &granted_list, list) {
-                list_del_init (&lock->list);
+/* Return true in case we need to ensure mandatory-locking
+ * semantics under different modes.
+ */
+gf_boolean_t
+pl_is_mandatory_locking_enabled(pl_inode_t *pl_inode)
+{
+    posix_locks_private_t *priv = THIS->private;
 
-                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
-                              &lock->user_flock, 0, 0, NULL);
+    if (priv->mandatory_mode == MLK_FILE_BASED && pl_inode->mandatory)
+        return _gf_true;
+    else if (priv->mandatory_mode == MLK_FORCED ||
+             priv->mandatory_mode == MLK_OPTIMAL)
+        return _gf_true;
 
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
-                                     &lock->user_flock, NULL);
+    return _gf_false;
+}
 
-                GF_FREE (lock);
+void
+pl_clean_local(pl_local_t *local)
+{
+    if (!local)
+        return;
+
+    if (local->inodelk_dom_count_req)
+        data_unref(local->inodelk_dom_count_req);
+    loc_wipe(&local->loc[0]);
+    loc_wipe(&local->loc[1]);
+    if (local->fd)
+        fd_unref(local->fd);
+    if (local->inode)
+        inode_unref(local->inode);
+    mem_put(local);
+}
+
+/*
+TODO: detach local initialization from PL_LOCAL_GET_REQUESTS and add it here
+*/
+int
+pl_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+{
+    pl_local_t *local = NULL;
+
+    if (!loc && !fd) {
+        return -1;
+    }
+
+    if (!frame->local) {
+        local = mem_get0(this->local_pool);
+        if (!local) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "mem allocation failed");
+            return -1;
         }
 
-        return;
+        local->inode = (loc ? inode_ref(loc->inode) : inode_ref(fd->inode));
+
+        frame->local = local;
+    }
+
+    return 0;
 }
 
-static int
-pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
-                        posix_lock_t *old_lock)
+gf_boolean_t
+pl_is_lk_owner_valid(gf_lkowner_t *owner, client_t *client)
+{
+    if (client && (client->opversion < GD_OP_VERSION_7_0)) {
+        return _gf_true;
+    }
+
+    if (is_lk_owner_null(owner)) {
+        return _gf_false;
+    }
+    return _gf_true;
+}
+
+static int32_t
+pl_inode_from_loc(loc_t *loc, inode_t **pinode)
 {
-        struct gf_flock  flock       = {0,};
-        posix_lock_t *unlock_lock = NULL;
+    inode_t *inode = NULL;
+    int32_t error = 0;
+
+    if (loc->inode != NULL) {
+        inode = inode_ref(loc->inode);
+        goto done;
+    }
+
+    if (loc->parent == NULL) {
+        error = EINVAL;
+        goto done;
+    }
+
+    if (!gf_uuid_is_null(loc->gfid)) {
+        inode = inode_find(loc->parent->table, loc->gfid);
+        if (inode != NULL) {
+            goto done;
+        }
+    }
 
-        struct list_head granted_list;
-        posix_lock_t     *tmp = NULL;
-        posix_lock_t     *lock = NULL;
+    if (loc->name == NULL) {
+        error = EINVAL;
+        goto done;
+    }
 
-        int ret = -1;
+    inode = inode_grep(loc->parent->table, loc->parent, loc->name);
+    if (inode == NULL) {
+        /* We haven't found any inode. This means that the file doesn't exist
+         * or that even if it exists, we don't have any knowledge about it, so
+         * we don't have locks on it either, which is fine for our purposes. */
+        goto done;
+    }
 
-        INIT_LIST_HEAD (&granted_list);
+done:
+    *pinode = inode;
 
-        flock.l_type   = F_UNLCK;
-        flock.l_whence = old_lock->user_flock.l_whence;
-        flock.l_start  = old_lock->user_flock.l_start;
-        flock.l_len    = old_lock->user_flock.l_len;
+    return error;
+}
 
+static gf_boolean_t
+pl_inode_has_owners(xlator_t *xl, client_t *client, pl_inode_t *pl_inode,
+                    struct timespec *now, struct list_head *contend)
+{
+    pl_dom_list_t *dom;
+    pl_inode_lock_t *lock;
+    gf_boolean_t has_owners = _gf_false;
 
-        unlock_lock = new_posix_lock (&flock, old_lock->client,
-                                      old_lock->client_pid, &old_lock->owner,
-                                      old_lock->fd, old_lock->lk_flags, 0);
-        GF_VALIDATE_OR_GOTO (this->name, unlock_lock, out);
-        ret = 0;
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        list_for_each_entry(lock, &dom->inodelk_list, list)
+        {
+            /* If the lock belongs to the same client, we assume it's related
+             * to the same operation, so we allow the removal to continue. */
+            if (lock->client == client) {
+                continue;
+            }
+            /* If the lock belongs to an internal process, we don't block the
+             * removal. */
+            if (lock->client_pid < 0) {
+                continue;
+            }
+            if (contend == NULL) {
+                return _gf_true;
+            }
+            has_owners = _gf_true;
+            inodelk_contention_notify_check(xl, lock, now, contend);
+        }
+    }
 
-        __insert_and_merge (pl_inode, unlock_lock);
+    return has_owners;
+}
 
-        __grant_blocked_locks (this, pl_inode, &granted_list);
+int32_t
+pl_inode_remove_prepare(xlator_t *xl, call_frame_t *frame, loc_t *loc,
+                        pl_inode_t **ppl_inode, struct list_head *contend)
+{
+    struct timespec now;
+    inode_t *inode;
+    pl_inode_t *pl_inode;
+    int32_t error;
+
+    pl_inode = NULL;
+
+    error = pl_inode_from_loc(loc, &inode);
+    if ((error != 0) || (inode == NULL)) {
+        goto done;
+    }
+
+    pl_inode = pl_inode_get(xl, inode, NULL);
+    if (pl_inode == NULL) {
+        inode_unref(inode);
+        error = ENOMEM;
+        goto done;
+    }
+
+    /* pl_inode_from_loc() already increments ref count for inode, so
+     * we only assign here our reference. */
+    pl_inode->inode = inode;
+
+    timespec_now(&now);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+
+    if (pl_inode->removed) {
+        error = ESTALE;
+        goto unlock;
+    }
+
+    if (pl_inode_has_owners(xl, frame->root->client, pl_inode, &now, contend)) {
+        error = -1;
+        /* We skip the unlock here because the caller must create a stub when
+         * we return -1 and do a call to pl_inode_remove_complete(), which
+         * assumes the lock is still acquired and will release it once
+         * everything else is prepared. */
+        goto done;
+    }
+
+    pl_inode->is_locked = _gf_true;
+    pl_inode->remove_running++;
 
-        list_for_each_entry_safe (lock, tmp, &granted_list, list) {
-                list_del_init (&lock->list);
+unlock:
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
-                              &lock->user_flock, 0, 0, NULL);
+done:
+    *ppl_inode = pl_inode;
 
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
-                                     &lock->user_flock, NULL);
+    return error;
+}
 
-                GF_FREE (lock);
+int32_t
+pl_inode_remove_complete(xlator_t *xl, pl_inode_t *pl_inode, call_stub_t *stub,
+                         struct list_head *contend)
+{
+    pl_inode_lock_t *lock;
+    int32_t error = -1;
+
+    if (stub != NULL) {
+        list_add_tail(&stub->list, &pl_inode->waiting);
+        pl_inode->is_locked = _gf_true;
+    } else {
+        error = ENOMEM;
+
+        while (!list_empty(contend)) {
+            lock = list_first_entry(contend, pl_inode_lock_t, list);
+            list_del_init(&lock->list);
+            __pl_inodelk_unref(lock);
         }
+    }
 
-out:
-        return ret;
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (error < 0) {
+        inodelk_contention_notify(xl, contend);
+    }
+
+    inode_unref(pl_inode->inode);
+
+    return error;
 }
 
-int
-pl_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-          int can_block)
+void
+pl_inode_remove_wake(struct list_head *list)
 {
-        int              ret = 0;
+    call_stub_t *stub;
 
-        errno = 0;
+    while (!list_empty(list)) {
+        stub = list_first_entry(list, call_stub_t, list);
+        list_del_init(&stub->list);
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                /* Send unlock before the actual lock to
-                   prevent lock upgrade / downgrade
-                   problems only if:
-                   - it is a blocking call
-                   - it has other conflicting locks
-                */
-
-                if (can_block &&
-                    !(__is_lock_grantable (pl_inode, lock))) {
-                        ret = pl_send_prelock_unlock (this, pl_inode,
-                                                      lock);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                "Could not send pre-lock "
-                                        "unlock");
-                }
+        call_resume(stub);
+    }
+}
 
-                if (__is_lock_grantable (pl_inode, lock)) {
-                        if (pl_metalock_is_active (pl_inode)) {
-                                __pl_queue_lock (pl_inode, lock, can_block);
-                                pthread_mutex_unlock (&pl_inode->mutex);
-                                ret = -2;
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => OK",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-                        __insert_and_merge (pl_inode, lock);
-                } else if (can_block) {
-                        if (pl_metalock_is_active (pl_inode)) {
-                                __pl_queue_lock (pl_inode, lock, can_block);
-                                pthread_mutex_unlock (&pl_inode->mutex);
-                                ret = -2;
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-                        lock->blocked = 1;
-                        __insert_lock (pl_inode, lock);
-                        ret = -1;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => NOK",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-                        errno = EAGAIN;
-                        ret = -1;
-                }
+void
+pl_inode_remove_cbk(xlator_t *xl, pl_inode_t *pl_inode, int32_t error)
+{
+    struct list_head contend, granted;
+    struct timespec now;
+    pl_dom_list_t *dom;
+
+    if (pl_inode == NULL) {
+        return;
+    }
+
+    INIT_LIST_HEAD(&contend);
+    INIT_LIST_HEAD(&granted);
+    timespec_now(&now);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+
+    if (error == 0) {
+        if (pl_inode->links >= 0) {
+            pl_inode->links--;
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+        if (pl_inode->links == 0) {
+            pl_inode->removed = _gf_true;
+        }
+    }
 
-        grant_blocked_locks (this, pl_inode);
+    pl_inode->remove_running--;
 
-        do_blocked_rw (pl_inode);
+    if ((pl_inode->remove_running == 0) && list_empty(&pl_inode->waiting)) {
+        pl_inode->is_locked = _gf_false;
 
-out:
-        return ret;
-}
+        list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+        {
+            __grant_blocked_inode_locks(xl, pl_inode, &granted, dom, &now,
+                                        &contend);
+        }
+    }
 
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-posix_lock_t *
-pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
+    unwind_granted_inodes(xl, pl_inode, &granted);
+
+    inodelk_contention_notify(xl, &contend);
+
+    inode_unref(pl_inode->inode);
+}
+
+void
+pl_inode_remove_unlocked(xlator_t *xl, pl_inode_t *pl_inode,
+                         struct list_head *list)
 {
-        posix_lock_t *conf = NULL;
+    call_stub_t *stub, *tmp;
+
+    if (!pl_inode->is_locked) {
+        return;
+    }
 
-        conf = first_conflicting_overlap (pl_inode, lock);
+    list_for_each_entry_safe(stub, tmp, &pl_inode->waiting, list)
+    {
+        if (!pl_inode_has_owners(xl, stub->frame->root->client, pl_inode, NULL,
+                                 NULL)) {
+            list_move_tail(&stub->list, list);
+        }
+    }
+}
 
-        if (conf == NULL) {
-                lock->fl_type = F_UNLCK;
-                return lock;
+/* This function determines if an inodelk attempt can be done now or it needs
+ * to wait.
+ *
+ * Possible return values:
+ *   < 0: An error occurred. Currently only -ESTALE can be returned if the
+ *        inode has been deleted previously by unlink/rmdir/rename
+ *   = 0: The lock can be attempted.
+ *   > 0: The lock needs to wait because a conflicting remove operation is
+ *        ongoing.
+ */
+int32_t
+pl_inode_remove_inodelk(pl_inode_t *pl_inode, pl_inode_lock_t *lock)
+{
+    pl_dom_list_t *dom;
+    pl_inode_lock_t *ilock;
+
+    /* If the inode has been deleted, we won't allow any lock. */
+    if (pl_inode->removed) {
+        return -ESTALE;
+    }
+
+    /* We only synchronize with locks made for regular operations coming from
+     * the user. Locks done for internal purposes are hard to control and could
+     * lead to long delays or deadlocks quite easily. */
+    if (lock->client_pid < 0) {
+        return 0;
+    }
+    if (!pl_inode->is_locked) {
+        return 0;
+    }
+    if (pl_inode->remove_running > 0) {
+        return 1;
+    }
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        list_for_each_entry(ilock, &dom->inodelk_list, list)
+        {
+            /* If a lock from the same client is already granted, we allow this
+             * one to continue. This is necessary to prevent deadlocks when
+             * multiple locks are taken for the same operation.
+             *
+             * On the other side it's unlikely that the same client sends
+             * completely unrelated locks for the same inode.
+             */
+            if (ilock->client == lock->client) {
+                return 0;
+            }
         }
+    }
 
-        return conf;
+    return 1;
 }
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index 5486f9b8314..281223bf3b8 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -10,155 +10,253 @@
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
-#include "lkowner.h"
 /*dump locks format strings */
-#define RANGE_FMT               "type=%s, whence=%hd, start=%llu, len=%llu"
-#define ENTRY_FMT               "type=%s on basename=%s"
-#define DUMP_GEN_FMT            "pid = %llu, owner=%s, client=%p"
-#define GRNTD_AT                "granted at %s"
-#define BLKD_AT                 "blocked at %s"
-#define CONN_ID                 "connection-id=%s"
-#define DUMP_BLKD_FMT           DUMP_GEN_FMT", "CONN_ID", "BLKD_AT
-#define DUMP_GRNTD_FMT          DUMP_GEN_FMT", "CONN_ID", "GRNTD_AT
-#define DUMP_BLKD_GRNTD_FMT     DUMP_GEN_FMT", "CONN_ID", "BLKD_AT", "GRNTD_AT
-
-#define ENTRY_BLKD_FMT          ENTRY_FMT", "DUMP_BLKD_FMT
-#define ENTRY_GRNTD_FMT         ENTRY_FMT", "DUMP_GRNTD_FMT
-#define ENTRY_BLKD_GRNTD_FMT    ENTRY_FMT", "DUMP_BLKD_GRNTD_FMT
-
-#define RANGE_BLKD_FMT          RANGE_FMT", "DUMP_BLKD_FMT
-#define RANGE_GRNTD_FMT         RANGE_FMT", "DUMP_GRNTD_FMT
-#define RANGE_BLKD_GRNTD_FMT    RANGE_FMT", "DUMP_BLKD_GRNTD_FMT
+#define RANGE_FMT "type=%s, whence=%hd, start=%llu, len=%llu"
+#define ENTRY_FMT "type=%s on basename=%s"
+#define DUMP_GEN_FMT "pid = %llu, owner=%s, client=%p"
+#define GRNTD_AT "granted at %s"
+#define BLKD_AT "blocked at %s"
+#define CONN_ID "connection-id=%s"
+#define DUMP_BLKD_FMT DUMP_GEN_FMT ", " CONN_ID ", " BLKD_AT
+#define DUMP_GRNTD_FMT DUMP_GEN_FMT ", " CONN_ID ", " GRNTD_AT
+#define DUMP_BLKD_GRNTD_FMT DUMP_GEN_FMT ", " CONN_ID ", " BLKD_AT ", " GRNTD_AT
+
+#define ENTRY_BLKD_FMT ENTRY_FMT ", " DUMP_BLKD_FMT
+#define ENTRY_GRNTD_FMT ENTRY_FMT ", " DUMP_GRNTD_FMT
+#define ENTRY_BLKD_GRNTD_FMT ENTRY_FMT ", " DUMP_BLKD_GRNTD_FMT
+
+#define RANGE_BLKD_FMT RANGE_FMT ", " DUMP_BLKD_FMT
+#define RANGE_GRNTD_FMT RANGE_FMT ", " DUMP_GRNTD_FMT
+#define RANGE_BLKD_GRNTD_FMT RANGE_FMT ", " DUMP_BLKD_GRNTD_FMT
 
 #define SET_FLOCK_PID(flock, lock) ((flock)->l_pid = lock->client_pid)
 
+#define PL_STACK_UNWIND_AND_FREE(__local, fop, frame, op_ret, params...)       \
+    do {                                                                       \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(fop, frame, op_ret, params);                       \
+        if (__local) {                                                         \
+            if (__local->inodelk_dom_count_req)                                \
+                data_unref(__local->inodelk_dom_count_req);                    \
+            loc_wipe(&__local->loc[0]);                                        \
+            loc_wipe(&__local->loc[1]);                                        \
+            if (__local->fd)                                                   \
+                fd_unref(__local->fd);                                         \
+            if (__local->inode)                                                \
+                inode_unref(__local->inode);                                   \
+            if (__local->xdata) {                                              \
+                dict_unref(__local->xdata);                                    \
+                __local->xdata = NULL;                                         \
+            }                                                                  \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
 
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
-                gf_lkowner_t *owner, fd_t *fd, uint32_t lk_flags,
-                int can_block);
+new_posix_lock(struct gf_flock *flock, client_t *client, pid_t client_pid,
+               gf_lkowner_t *owner, fd_t *fd, uint32_t lk_flags, int blocking,
+               int32_t *op_errno);
 
 pl_inode_t *
-pl_inode_get (xlator_t *this, inode_t *inode);
+pl_inode_get(xlator_t *this, inode_t *inode, pl_local_t *local);
 
 posix_lock_t *
-pl_getlk (pl_inode_t *inode, posix_lock_t *lock);
+pl_getlk(pl_inode_t *inode, posix_lock_t *lock);
 
 int
-pl_setlk (xlator_t *this, pl_inode_t *inode, posix_lock_t *lock,
-          int can_block);
+pl_setlk(xlator_t *this, pl_inode_t *inode, posix_lock_t *lock, int can_block);
+
+int
+pl_lock_preempt(pl_inode_t *pl_inode, posix_lock_t *reqlock);
 
 void
-grant_blocked_locks (xlator_t *this, pl_inode_t *inode);
+grant_blocked_locks(xlator_t *this, pl_inode_t *inode);
 
 void
-posix_lock_to_flock (posix_lock_t *lock, struct gf_flock *flock);
+posix_lock_to_flock(posix_lock_t *lock, struct gf_flock *flock);
 
 int
-locks_overlap (posix_lock_t *l1, posix_lock_t *l2);
+locks_overlap(posix_lock_t *l1, posix_lock_t *l2);
 
 int
-same_owner (posix_lock_t *l1, posix_lock_t *l2);
+same_owner(posix_lock_t *l1, posix_lock_t *l2);
 
-void __delete_lock (posix_lock_t *);
+void
+__delete_lock(posix_lock_t *);
 
-void __destroy_lock (posix_lock_t *);
+void
+__destroy_lock(posix_lock_t *);
 
 pl_dom_list_t *
-get_domain (pl_inode_t *pl_inode, const char *volume);
+get_domain(pl_inode_t *pl_inode, const char *volume);
+
+void
+grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend);
 
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_dom_list_t *dom);
+inodelk_contention_notify(xlator_t *this, struct list_head *contend);
 
 void
-__delete_inode_lock (pl_inode_lock_t *lock);
+__delete_inode_lock(pl_inode_lock_t *lock);
 
 void
-__pl_inodelk_unref (pl_inode_lock_t *lock);
+__pl_inodelk_unref(pl_inode_lock_t *lock);
 
 void
-grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_dom_list_t *dom);
+__grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                            struct list_head *granted, pl_dom_list_t *dom,
+                            struct timespec *now, struct list_head *contend);
 
-void pl_update_refkeeper (xlator_t *this, inode_t *inode);
+void
+unwind_granted_inodes(xlator_t *this, pl_inode_t *pl_inode,
+                      struct list_head *granted);
+
+void
+grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend);
+
+void
+entrylk_contention_notify(xlator_t *this, struct list_head *contend);
+
+void
+pl_update_refkeeper(xlator_t *this, inode_t *inode);
 
 int32_t
-__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname);
+__get_inodelk_count(xlator_t *this, pl_inode_t *pl_inode, char *domname);
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode, char *domname);
+get_inodelk_count(xlator_t *this, inode_t *inode, char *domname);
 
 int32_t
-__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode);
+__get_entrylk_count(xlator_t *this, pl_inode_t *pl_inode);
 int32_t
-get_entrylk_count (xlator_t *this, inode_t *inode);
+get_entrylk_count(xlator_t *this, inode_t *inode);
 
-void pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                  int cmd, struct gf_flock *flock, const char *domain);
+void
+pl_trace_in(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+            struct gf_flock *flock, const char *domain);
 
-void pl_trace_out (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                   int cmd, struct gf_flock *flock, int op_ret, int op_errno, const char *domain);
+void
+pl_trace_out(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+             struct gf_flock *flock, int op_ret, int op_errno,
+             const char *domain);
 
-void pl_trace_block (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                     int cmd, struct gf_flock *flock, const char *domain);
+void
+pl_trace_block(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+               int cmd, struct gf_flock *flock, const char *domain);
 
-void pl_trace_flush (xlator_t *this, call_frame_t *frame, fd_t *fd);
+void
+pl_trace_flush(xlator_t *this, call_frame_t *frame, fd_t *fd);
 
-void entrylk_trace_in (xlator_t *this, call_frame_t *frame, const char *volume,
-                       fd_t *fd, loc_t *loc, const char *basename,
-                       entrylk_cmd cmd, entrylk_type type);
+void
+entrylk_trace_in(xlator_t *this, call_frame_t *frame, const char *volume,
+                 fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type);
 
-void entrylk_trace_out (xlator_t *this, call_frame_t *frame, const char *volume,
-                        fd_t *fd, loc_t *loc, const char *basename,
-                        entrylk_cmd cmd, entrylk_type type,
-                        int op_ret, int op_errno);
+void
+entrylk_trace_out(xlator_t *this, call_frame_t *frame, const char *volume,
+                  fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, int op_ret, int op_errno);
 
-void entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
-                          fd_t *fd, loc_t *loc, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type);
+void
+entrylk_trace_block(xlator_t *this, call_frame_t *frame, const char *volume,
+                    fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                    entrylk_type type);
 
 void
-pl_print_verdict (char *str, int size, int op_ret, int op_errno);
+pl_print_verdict(char *str, int size, int op_ret, int op_errno);
 
 void
-pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc);
+pl_print_lockee(char *str, int size, fd_t *fd, loc_t *loc);
 
 void
-pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame);
+pl_print_locker(char *str, int size, xlator_t *this, call_frame_t *frame);
 
 void
-pl_print_inodelk (char *str, int size, int cmd, struct gf_flock *flock, const char *domain);
+pl_print_inodelk(char *str, int size, int cmd, struct gf_flock *flock,
+                 const char *domain);
 
 void
-pl_trace_release (xlator_t *this, fd_t *fd);
+pl_trace_release(xlator_t *this, fd_t *fd);
 
 unsigned long
-fd_to_fdnum (fd_t *fd);
+fd_to_fdnum(fd_t *fd);
 
 fd_t *
-fd_from_fdnum (posix_lock_t *lock);
+fd_from_fdnum(posix_lock_t *lock);
 
 int
-pl_reserve_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-                  int can_block);
+pl_reserve_setlk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                 int can_block);
 int
-reservelks_equal (posix_lock_t *l1, posix_lock_t *l2);
+reservelks_equal(posix_lock_t *l1, posix_lock_t *l2);
 
 int
-pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
-                     posix_lock_t *lock, int can_block);
+pl_verify_reservelk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                    int can_block);
 int
-pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+pl_reserve_unlock(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
 
 int32_t
-check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename);
+check_entrylk_on_basename(xlator_t *this, inode_t *parent, char *basename);
 
-void __pl_inodelk_unref (pl_inode_lock_t *lock);
-void __pl_entrylk_unref (pl_entry_lock_t *lock);
+void
+__pl_inodelk_unref(pl_inode_lock_t *lock);
+void
+__pl_entrylk_unref(pl_entry_lock_t *lock);
 
 int
-pl_metalock_is_active (pl_inode_t *pl_inode);
+pl_metalock_is_active(pl_inode_t *pl_inode);
+
+void
+__pl_queue_lock(pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
+void
+inodelk_contention_notify_check(xlator_t *xl, pl_inode_lock_t *lock,
+                                struct timespec *now,
+                                struct list_head *contend);
+
+void
+entrylk_contention_notify_check(xlator_t *xl, pl_entry_lock_t *lock,
+                                struct timespec *now,
+                                struct list_head *contend);
+
+gf_boolean_t
+pl_does_monkey_want_stuck_lock();
+
+gf_boolean_t
+pl_is_mandatory_locking_enabled(pl_inode_t *pl_inode);
+
+void
+pl_clean_local(pl_local_t *local);
 
 int
-__pl_queue_lock (pl_inode_t *pl_inode, posix_lock_t *reqlock, int can_block);
+pl_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd);
+
+gf_boolean_t
+pl_is_lk_owner_valid(gf_lkowner_t *owner, client_t *client);
+
+int32_t
+pl_inode_remove_prepare(xlator_t *xl, call_frame_t *frame, loc_t *loc,
+                        pl_inode_t **ppl_inode, struct list_head *contend);
+
+int32_t
+pl_inode_remove_complete(xlator_t *xl, pl_inode_t *pl_inode, call_stub_t *stub,
+                         struct list_head *contend);
+
+void
+pl_inode_remove_wake(struct list_head *list);
+
+void
+pl_inode_remove_cbk(xlator_t *xl, pl_inode_t *pl_inode, int32_t error);
+
+void
+pl_inode_remove_unlocked(xlator_t *xl, pl_inode_t *pl_inode,
+                         struct list_head *list);
+
+int32_t
+pl_inode_remove_inodelk(pl_inode_t *pl_inode, pl_inode_lock_t *lock);
+
 #endif /* __COMMON_H__ */
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 783c57e6381..fd772c850dd 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -7,72 +7,77 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/list.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "locks.h"
+#include "clear.h"
 #include "common.h"
-
+#include "pl-messages.h"
 
 void
-__pl_entrylk_unref (pl_entry_lock_t *lock)
+__pl_entrylk_unref(pl_entry_lock_t *lock)
 {
-        lock->ref--;
-        if (!lock->ref) {
-		GF_FREE ((char *)lock->basename);
-                GF_FREE (lock->connection_id);
-                GF_FREE (lock);
-        }
+    lock->ref--;
+    if (!lock->ref) {
+        GF_FREE((char *)lock->basename);
+        GF_FREE(lock->connection_id);
+        GF_FREE(lock);
+    }
 }
 
-
 static void
-__pl_entrylk_ref (pl_entry_lock_t *lock)
+__pl_entrylk_ref(pl_entry_lock_t *lock)
 {
-        lock->ref++;
+    lock->ref++;
 }
 
-
 static pl_entry_lock_t *
-new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
-		  const char *domain, call_frame_t *frame, char *conn_id)
+new_entrylk_lock(pl_inode_t *pinode, const char *basename, entrylk_type type,
+                 const char *domain, call_frame_t *frame, char *conn_id,
+                 int32_t *op_errno)
 {
-        pl_entry_lock_t *newlock = NULL;
-
-        newlock = GF_CALLOC (1, sizeof (pl_entry_lock_t),
-                             gf_locks_mt_pl_entry_lock_t);
-        if (!newlock) {
-                goto out;
-        }
-
-        newlock->basename   = basename ? gf_strdup (basename) : NULL;
-        newlock->type       = type;
-        newlock->client     = frame->root->client;
-        newlock->client_pid = frame->root->pid;
-        newlock->volume     = domain;
-        newlock->owner      = frame->root->lk_owner;
-	newlock->frame      = frame;
-        newlock->this       = frame->this;
-
-        if (conn_id) {
-                newlock->connection_id = gf_strdup (conn_id);
-        }
-
-        INIT_LIST_HEAD (&newlock->domain_list);
-        INIT_LIST_HEAD (&newlock->blocked_locks);
-	INIT_LIST_HEAD (&newlock->client_list);
-
-	__pl_entrylk_ref (newlock);
+    pl_entry_lock_t *newlock = NULL;
+
+    if (!pl_is_lk_owner_valid(&frame->root->lk_owner, frame->root->client)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    newlock = GF_CALLOC(1, sizeof(pl_entry_lock_t),
+                        gf_locks_mt_pl_entry_lock_t);
+    if (!newlock) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    newlock->basename = basename ? gf_strdup(basename) : NULL;
+    newlock->type = type;
+    newlock->client = frame->root->client;
+    newlock->client_pid = frame->root->pid;
+    newlock->volume = domain;
+    newlock->owner = frame->root->lk_owner;
+    newlock->frame = frame;
+    newlock->this = frame->this;
+
+    if (conn_id) {
+        newlock->connection_id = gf_strdup(conn_id);
+    }
+
+    INIT_LIST_HEAD(&newlock->domain_list);
+    INIT_LIST_HEAD(&newlock->blocked_locks);
+    INIT_LIST_HEAD(&newlock->client_list);
+
+    __pl_entrylk_ref(newlock);
 out:
-        return newlock;
+    return newlock;
 }
 
-
 /**
  * all_names - does a basename represent all names?
  * @basename: name to check
@@ -87,28 +92,220 @@ out:
  */
 
 static int
-names_conflict (const char *n1, const char *n2)
+names_conflict(const char *n1, const char *n2)
 {
-        return all_names (n1) || all_names (n2) || !strcmp (n1, n2);
+    return all_names(n1) || all_names(n2) || !strcmp(n1, n2);
 }
 
-
 static int
-__same_entrylk_owner (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
+__same_entrylk_owner(pl_entry_lock_t *l1, pl_entry_lock_t *l2)
 {
-        return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->client  == l2->client));
+    return (is_same_lkowner(&l1->owner, &l2->owner) &&
+            (l1->client == l2->client));
 }
 
 /* Just as in inodelk, allow conflicting name locks from same (lk_owner, conn)*/
 static int
-__conflicting_entrylks (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
+__conflicting_entrylks(pl_entry_lock_t *l1, pl_entry_lock_t *l2)
+{
+    if (names_conflict(l1->basename, l2->basename) &&
+        !__same_entrylk_owner(l1, l2))
+        return 1;
+
+    return 0;
+}
+
+/* See comments in inodelk.c for details */
+static inline gf_boolean_t
+__stale_entrylk(xlator_t *this, pl_entry_lock_t *candidate_lock,
+                pl_entry_lock_t *requested_lock, time_t *lock_age_sec)
+{
+    posix_locks_private_t *priv = NULL;
+
+    priv = this->private;
+
+    /* Question: Should we just prune them all given the
+     * chance?  Or just the locks we are attempting to acquire?
+     */
+    if (names_conflict(candidate_lock->basename, requested_lock->basename)) {
+        *lock_age_sec = gf_time() - candidate_lock->granted_time;
+        if (*lock_age_sec > priv->revocation_secs)
+            return _gf_true;
+    }
+    return _gf_false;
+}
+
+/* See comments in inodelk.c for details */
+static gf_boolean_t
+__entrylk_prune_stale(xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom,
+                      pl_entry_lock_t *lock)
+{
+    posix_locks_private_t *priv = NULL;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *lk = NULL;
+    gf_boolean_t revoke_lock = _gf_false;
+    int bcount = 0;
+    int gcount = 0;
+    int op_errno = 0;
+    clrlk_args args;
+    args.opts = NULL;
+    time_t lk_age_sec = 0;
+    uint32_t max_blocked = 0;
+    char *reason_str = NULL;
+
+    priv = this->private;
+    args.type = CLRLK_ENTRY;
+    if (priv->revocation_clear_all == _gf_true)
+        args.kind = CLRLK_ALL;
+    else
+        args.kind = CLRLK_GRANTED;
+
+    if (list_empty(&dom->entrylk_list))
+        goto out;
+
+    pthread_mutex_lock(&pinode->mutex);
+    lock->pinode = pinode;
+    list_for_each_entry_safe(lk, tmp, &dom->entrylk_list, domain_list)
+    {
+        if (__stale_entrylk(this, lk, lock, &lk_age_sec) == _gf_true) {
+            revoke_lock = _gf_true;
+            reason_str = "age";
+            break;
+        }
+    }
+    max_blocked = priv->revocation_max_blocked;
+    if (max_blocked != 0 && revoke_lock == _gf_false) {
+        list_for_each_entry_safe(lk, tmp, &dom->blocked_entrylks, blocked_locks)
+        {
+            max_blocked--;
+            if (max_blocked == 0) {
+                revoke_lock = _gf_true;
+                reason_str = "max blocked";
+                break;
+            }
+        }
+    }
+    pthread_mutex_unlock(&pinode->mutex);
+
+out:
+    if (revoke_lock == _gf_true) {
+        clrlk_clear_entrylk(this, pinode, dom, &args, &bcount, &gcount,
+                            &op_errno);
+        gf_log(this->name, GF_LOG_WARNING,
+               "Lock revocation [reason: %s; gfid: %s; domain: %s; "
+               "age: %ld sec] - Entry lock revoked:  %d granted & %d "
+               "blocked locks cleared",
+               reason_str, uuid_utoa(pinode->gfid), dom->domain, lk_age_sec,
+               gcount, bcount);
+    }
+
+    return revoke_lock;
+}
+
+void
+entrylk_contention_notify_check(xlator_t *this, pl_entry_lock_t *lock,
+                                struct timespec *now, struct list_head *contend)
+{
+    posix_locks_private_t *priv;
+    int64_t elapsed;
+
+    priv = this->private;
+
+    /* If this lock is in a list, it means that we are about to send a
+     * notification for it, so no need to do anything else. */
+    if (!list_empty(&lock->contend)) {
+        return;
+    }
+
+    elapsed = now->tv_sec;
+    elapsed -= lock->contention_time.tv_sec;
+    if (now->tv_nsec < lock->contention_time.tv_nsec) {
+        elapsed--;
+    }
+    if (elapsed < priv->notify_contention_delay) {
+        return;
+    }
+
+    /* All contention notifications will be sent outside of the locked
+     * region. This means that currently granted locks might have already
+     * been unlocked by that time. To avoid the lock or the inode to be
+     * destroyed before we process them, we take an additional reference
+     * on both. */
+    inode_ref(lock->pinode->inode);
+    __pl_entrylk_ref(lock);
+
+    lock->contention_time = *now;
+
+    list_add_tail(&lock->contend, contend);
+}
+
+void
+entrylk_contention_notify(xlator_t *this, struct list_head *contend)
 {
-        if (names_conflict (l1->basename, l2->basename)
-            && !__same_entrylk_owner (l1, l2))
-                return 1;
+    struct gf_upcall up;
+    struct gf_upcall_entrylk_contention lc;
+    pl_entry_lock_t *lock;
+    pl_inode_t *pl_inode;
+    client_t *client;
+    gf_boolean_t notify;
+
+    while (!list_empty(contend)) {
+        lock = list_first_entry(contend, pl_entry_lock_t, contend);
+
+        pl_inode = lock->pinode;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        /* If the lock has already been released, no notification is
+         * sent. We clear the notification time in this case. */
+        notify = !list_empty(&lock->domain_list);
+        if (!notify) {
+            lock->contention_time.tv_sec = 0;
+            lock->contention_time.tv_nsec = 0;
+        } else {
+            lc.type = lock->type;
+            lc.name = lock->basename;
+            lc.pid = lock->client_pid;
+            lc.domain = lock->volume;
+            lc.xdata = NULL;
+
+            gf_uuid_copy(up.gfid, lock->pinode->gfid);
+            client = (client_t *)lock->client;
+            if (client == NULL) {
+                /* A NULL client can be found if the entrylk
+                 * was issued by a server side xlator. */
+                up.client_uid = NULL;
+            } else {
+                up.client_uid = client->client_uid;
+            }
+        }
+
+        pthread_mutex_unlock(&pl_inode->mutex);
+
+        if (notify) {
+            up.event_type = GF_UPCALL_ENTRYLK_CONTENTION;
+            up.data = &lc;
+
+            if (this->notify(this, GF_EVENT_UPCALL, &up) < 0) {
+                gf_msg_debug(this->name, 0,
+                             "Entrylk contention notification "
+                             "failed");
+            } else {
+                gf_msg_debug(this->name, 0,
+                             "Entrylk contention notification "
+                             "sent");
+            }
+        }
+
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        list_del_init(&lock->contend);
+        __pl_entrylk_unref(lock);
+
+        pthread_mutex_unlock(&pl_inode->mutex);
 
-        return 0;
+        inode_unref(pl_inode->inode);
+    }
 }
 
 /**
@@ -118,184 +315,188 @@ __conflicting_entrylks (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
  * @type: type of lock
  */
 static pl_entry_lock_t *
-__entrylk_grantable (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+__entrylk_grantable(xlator_t *this, pl_dom_list_t *dom, pl_entry_lock_t *lock,
+                    struct timespec *now, struct list_head *contend)
 {
-        pl_entry_lock_t *tmp = NULL;
-
-        if (list_empty (&dom->entrylk_list))
-                return NULL;
-
-        list_for_each_entry (tmp, &dom->entrylk_list, domain_list) {
-                if (__conflicting_entrylks (tmp, lock))
-                        return tmp;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *ret = NULL;
+
+    list_for_each_entry(tmp, &dom->entrylk_list, domain_list)
+    {
+        if (__conflicting_entrylks(tmp, lock)) {
+            if (ret == NULL) {
+                ret = tmp;
+                if (contend == NULL) {
+                    break;
+                }
+            }
+            entrylk_contention_notify_check(this, tmp, now, contend);
         }
+    }
 
-        return NULL;
+    return ret;
 }
 
 static pl_entry_lock_t *
-__blocked_entrylk_conflict (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+__blocked_entrylk_conflict(pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *tmp = NULL;
 
-        if (list_empty (&dom->blocked_entrylks))
-                return NULL;
+    list_for_each_entry(tmp, &dom->blocked_entrylks, blocked_locks)
+    {
+        if (names_conflict(tmp->basename, lock->basename))
+            return lock;
+    }
 
-        list_for_each_entry (tmp, &dom->blocked_entrylks, blocked_locks) {
-                if (names_conflict (tmp->basename, lock->basename))
-                        return lock;
-        }
-
-        return NULL;
+    return NULL;
 }
 
 static int
-__owner_has_lock (pl_dom_list_t *dom, pl_entry_lock_t *newlock)
+__owner_has_lock(pl_dom_list_t *dom, pl_entry_lock_t *newlock)
 {
-        pl_entry_lock_t *lock = NULL;
+    pl_entry_lock_t *lock = NULL;
 
-        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-                if (__same_entrylk_owner (lock, newlock))
-                        return 1;
-        }
+    list_for_each_entry(lock, &dom->entrylk_list, domain_list)
+    {
+        if (__same_entrylk_owner(lock, newlock))
+            return 1;
+    }
 
-        list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-                if (__same_entrylk_owner (lock, newlock))
-                        return 1;
-        }
+    list_for_each_entry(lock, &dom->blocked_entrylks, blocked_locks)
+    {
+        if (__same_entrylk_owner(lock, newlock))
+            return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
 static int
-names_equal (const char *n1, const char *n2)
+names_equal(const char *n1, const char *n2)
 {
-        return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp (n1, n2));
+    return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp(n1, n2));
 }
 
 void
-pl_print_entrylk (char *str, int size, entrylk_cmd cmd, entrylk_type type,
-                  const char *basename, const char *domain)
+pl_print_entrylk(char *str, int size, entrylk_cmd cmd, entrylk_type type,
+                 const char *basename, const char *domain)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
 
-        switch (cmd) {
+    switch (cmd) {
         case ENTRYLK_LOCK:
-                cmd_str = "LOCK";
-                break;
+            cmd_str = "LOCK";
+            break;
 
         case ENTRYLK_LOCK_NB:
-                cmd_str = "LOCK_NB";
-                break;
+            cmd_str = "LOCK_NB";
+            break;
 
         case ENTRYLK_UNLOCK:
-                cmd_str = "UNLOCK";
-                break;
+            cmd_str = "UNLOCK";
+            break;
 
         default:
-                cmd_str = "UNKNOWN";
-                break;
-        }
+            cmd_str = "UNKNOWN";
+            break;
+    }
 
-        switch (type) {
+    switch (type) {
         case ENTRYLK_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case ENTRYLK_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
-        }
+            type_str = "UNKNOWN";
+            break;
+    }
 
-        snprintf (str, size, "lock=ENTRYLK, cmd=%s, type=%s, basename=%s, domain: %s",
-                  cmd_str, type_str, basename, domain);
+    snprintf(str, size,
+             "lock=ENTRYLK, cmd=%s, type=%s, basename=%s, domain: %s", cmd_str,
+             type_str, basename, domain);
 }
 
-
 void
-entrylk_trace_in (xlator_t *this, call_frame_t *frame, const char *domain,
-                  fd_t *fd, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type)
+entrylk_trace_in(xlator_t *this, call_frame_t *frame, const char *domain,
+                 fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_entrylk[256];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_entrylk[256];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    pl_print_entrylk(pl_entrylk, 256, cmd, type, basename, domain);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_entrylk);
+    gf_log(this->name, GF_LOG_INFO,
+           "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_entrylk);
 }
 
-
 void
-entrylk_trace_out (xlator_t *this, call_frame_t *frame, const char *domain,
-                   fd_t *fd, loc_t *loc, const char *basename,
-                   entrylk_cmd cmd, entrylk_type type, int op_ret, int op_errno)
+entrylk_trace_out(xlator_t *this, call_frame_t *frame, const char *domain,
+                  fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, int op_ret, int op_errno)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_entrylk[256];
-        char                    verdict[32];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_entrylk[256];
+    char verdict[32];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
-        pl_print_verdict (verdict, 32, op_ret, op_errno);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    pl_print_entrylk(pl_entrylk, 256, cmd, type, basename, domain);
+    pl_print_verdict(verdict, 32, op_ret, op_errno);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                verdict, pl_locker, pl_lockee, pl_entrylk);
+    gf_log(this->name, GF_LOG_INFO,
+           "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}", verdict, pl_locker,
+           pl_lockee, pl_entrylk);
 }
 
-
 void
-entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
-                     fd_t *fd, loc_t *loc, const char *basename,
-                     entrylk_cmd cmd, entrylk_type type)
+entrylk_trace_block(xlator_t *this, call_frame_t *frame, const char *volume,
+                    fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                    entrylk_type type)
 
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_entrylk[256];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_entrylk[256];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, volume);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    pl_print_entrylk(pl_entrylk, 256, cmd, type, basename, volume);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_entrylk);
+    gf_log(this->name, GF_LOG_INFO,
+           "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_entrylk);
 }
 
 /**
- * __find_most_matching_lock - find the lock struct which most matches in order of:
- *                           lock on the exact basename ||
- *                           an all_names lock
+ * __find_most_matching_lock - find the lock struct which most matches in order
+ * of: lock on the exact basename || an all_names lock
  *
  *
  * @inode: inode in which to look
@@ -303,37 +504,57 @@ entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
  */
 
 static pl_entry_lock_t *
-__find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
+__find_most_matching_lock(pl_dom_list_t *dom, const char *basename)
 {
-        pl_entry_lock_t *lock;
-        pl_entry_lock_t *all = NULL;
-        pl_entry_lock_t *exact = NULL;
+    pl_entry_lock_t *lock;
+    pl_entry_lock_t *all = NULL;
+    pl_entry_lock_t *exact = NULL;
 
-        if (list_empty (&dom->entrylk_list))
-                return NULL;
+    if (list_empty(&dom->entrylk_list))
+        return NULL;
 
-        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-                if (all_names (lock->basename))
-                        all = lock;
-                else if (names_equal (lock->basename, basename))
-                        exact = lock;
-        }
+    list_for_each_entry(lock, &dom->entrylk_list, domain_list)
+    {
+        if (all_names(lock->basename))
+            all = lock;
+        else if (names_equal(lock->basename, basename))
+            exact = lock;
+    }
+
+    return (exact ? exact : all);
+}
 
-        return (exact ? exact : all);
+static pl_entry_lock_t *
+__find_matching_lock(pl_dom_list_t *dom, pl_entry_lock_t *lock)
+{
+    pl_entry_lock_t *tmp = NULL;
+
+    list_for_each_entry(tmp, &dom->entrylk_list, domain_list)
+    {
+        if (names_equal(lock->basename, tmp->basename) &&
+            __same_entrylk_owner(lock, tmp) && (lock->type == tmp->type))
+            return tmp;
+    }
+    return NULL;
 }
 
-static pl_entry_lock_t*
-__find_matching_lock (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+static int
+__lock_blocked_add(xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom,
+                   pl_entry_lock_t *lock, int nonblock)
 {
-        pl_entry_lock_t *tmp = NULL;
+    if (nonblock)
+        goto out;
 
-        list_for_each_entry (tmp, &dom->entrylk_list, domain_list) {
-                if (names_equal (lock->basename, tmp->basename)
-                    && __same_entrylk_owner (lock, tmp)
-                    && (lock->type == tmp->type))
-                        return tmp;
-        }
-        return NULL;
+    lock->blkd_time = gf_time();
+    list_add_tail(&lock->blocked_locks, &dom->blocked_entrylks);
+
+    gf_msg_trace(this->name, 0, "Blocking lock: {pinode=%p, basename=%s}",
+                 pinode, lock->basename);
+
+    entrylk_trace_block(this, lock->frame, NULL, NULL, NULL, lock->basename,
+                        ENTRYLK_LOCK, lock->type);
+out:
+    return -EAGAIN;
 }
 
 /**
@@ -348,63 +569,49 @@ __find_matching_lock (pl_dom_list_t *dom, pl_entry_lock_t *lock)
  */
 
 int
-__lock_entrylk (xlator_t *this, pl_inode_t *pinode, pl_entry_lock_t *lock,
-		int nonblock, pl_dom_list_t *dom)
+__lock_entrylk(xlator_t *this, pl_inode_t *pinode, pl_entry_lock_t *lock,
+               int nonblock, pl_dom_list_t *dom, struct timespec *now,
+               struct list_head *contend)
 {
-        pl_entry_lock_t *conf = NULL;
-        int              ret  = -EAGAIN;
-
-        conf = __entrylk_grantable (dom, lock);
-        if (conf) {
-                ret = -EAGAIN;
-                if (nonblock)
-                        goto out;
-
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Blocking lock: {pinode=%p, basename=%s}",
-                        pinode, lock->basename);
-
-                goto out;
+    pl_entry_lock_t *conf = NULL;
+    int ret = -EAGAIN;
+
+    conf = __entrylk_grantable(this, dom, lock, now, contend);
+    if (conf) {
+        ret = __lock_blocked_add(this, pinode, dom, lock, nonblock);
+        goto out;
+    }
+
+    /* To prevent blocked locks starvation, check if there are any blocked
+     * locks thay may conflict with this lock. If there is then don't grant
+     * the lock. BUT grant the lock if the owner already has lock to allow
+     * nested locks.
+     * Example: SHD from Machine1 takes (gfid, basename=257-length-name)
+     * and is granted.
+     * SHD from machine2 takes (gfid, basename=NULL) and is blocked.
+     * When SHD from Machine1 takes (gfid, basename=NULL) it needs to be
+     * granted, without which self-heal can't progress.
+     * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
+     */
+    if (__blocked_entrylk_conflict(dom, lock) &&
+        !(__owner_has_lock(dom, lock))) {
+        if (nonblock == 0) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock is grantable, but blocking to prevent "
+                   "starvation");
         }
 
-        /* To prevent blocked locks starvation, check if there are any blocked
-         * locks thay may conflict with this lock. If there is then don't grant
-         * the lock. BUT grant the lock if the owner already has lock to allow
-         * nested locks.
-         * Example: SHD from Machine1 takes (gfid, basename=257-length-name)
-         * and is granted.
-         * SHD from machine2 takes (gfid, basename=NULL) and is blocked.
-         * When SHD from Machine1 takes (gfid, basename=NULL) it needs to be
-         * granted, without which self-heal can't progress.
-         * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
-         */
-        if (__blocked_entrylk_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
-                ret = -EAGAIN;
-                if (nonblock)
-                        goto out;
-
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock is grantable, but blocking to prevent starvation");
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Blocking lock: {pinode=%p, basename=%s}",
-                        pinode, lock->basename);
+        ret = __lock_blocked_add(this, pinode, dom, lock, nonblock);
+        goto out;
+    }
 
-                goto out;
-        }
+    __pl_entrylk_ref(lock);
+    lock->granted_time = gf_time();
+    list_add(&lock->domain_list, &dom->entrylk_list);
 
-        __pl_entrylk_ref (lock);
-        gettimeofday (&lock->granted_time, NULL);
-        list_add (&lock->domain_list, &dom->entrylk_list);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -415,291 +622,322 @@ out:
  */
 
 pl_entry_lock_t *
-__unlock_entrylk (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+__unlock_entrylk(pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *ret_lock = NULL;
+    pl_entry_lock_t *ret_lock = NULL;
 
-        ret_lock = __find_matching_lock (dom, lock);
+    ret_lock = __find_matching_lock(dom, lock);
 
-        if (ret_lock) {
-                list_del_init (&ret_lock->domain_list);
-        } else {
-                gf_log ("locks", GF_LOG_ERROR, "unlock on %s "
-                        "(type=ENTRYLK_WRLCK) attempted but no matching lock "
-                        "found", lock->basename);
-        }
+    if (ret_lock) {
+        list_del_init(&ret_lock->domain_list);
+    } else {
+        gf_log("locks", GF_LOG_ERROR,
+               "unlock on %s "
+               "(type=ENTRYLK_WRLCK) attempted but no matching lock "
+               "found",
+               lock->basename);
+    }
 
-        return ret_lock;
+    return ret_lock;
 }
 
 int32_t
-check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename)
+check_entrylk_on_basename(xlator_t *this, inode_t *parent, char *basename)
 {
-        int32_t         entrylk = 0;
-        pl_inode_t      *pinode = 0;
-        pl_dom_list_t   *dom = NULL;
-        pl_entry_lock_t *conf       = NULL;
-
-        pinode = pl_inode_get (this, parent);
-        if (!pinode)
-                goto out;
-        pthread_mutex_lock (&pinode->mutex);
+    int32_t entrylk = 0;
+    pl_dom_list_t *dom = NULL;
+    pl_entry_lock_t *conf = NULL;
+
+    pl_inode_t *pinode = pl_inode_get(this, parent, NULL);
+    if (!pinode)
+        goto out;
+    pthread_mutex_lock(&pinode->mutex);
+    {
+        list_for_each_entry(dom, &pinode->dom_list, inode_list)
         {
-                list_for_each_entry (dom, &pinode->dom_list, inode_list) {
-                        conf = __find_most_matching_lock (dom, basename);
-                        if (conf && conf->basename) {
-                                entrylk = 1;
-                                break;
-                        }
-                }
+            conf = __find_most_matching_lock(dom, basename);
+            if (conf && conf->basename) {
+                entrylk = 1;
+                break;
+            }
         }
-        pthread_mutex_unlock (&pinode->mutex);
+    }
+    pthread_mutex_unlock(&pinode->mutex);
 
 out:
-        return entrylk;
+    return entrylk;
 }
 
 void
-__grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-                             pl_dom_list_t *dom, struct list_head *granted)
+__grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode,
+                            pl_dom_list_t *dom, struct list_head *granted,
+                            struct timespec *now, struct list_head *contend)
 {
-        int              bl_ret = 0;
-        pl_entry_lock_t *bl   = NULL;
-        pl_entry_lock_t *tmp  = NULL;
+    int bl_ret = 0;
+    pl_entry_lock_t *bl = NULL;
+    pl_entry_lock_t *tmp = NULL;
 
-        struct list_head blocked_list;
+    struct list_head blocked_list;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&dom->blocked_entrylks, &blocked_list);
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&dom->blocked_entrylks, &blocked_list);
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
+    list_for_each_entry_safe(bl, tmp, &blocked_list, blocked_locks)
+    {
+        list_del_init(&bl->blocked_locks);
 
-                list_del_init (&bl->blocked_locks);
+        bl_ret = __lock_entrylk(bl->this, pl_inode, bl, 0, dom, now, contend);
 
-                bl_ret = __lock_entrylk (bl->this, pl_inode, bl, 0, dom);
-
-                if (bl_ret == 0) {
-                        list_add (&bl->blocked_locks, granted);
-                }
+        if (bl_ret == 0) {
+            list_add_tail(&bl->blocked_locks, granted);
         }
-        return;
+    }
 }
 
 /* Grants locks if possible which are blocked on a lock */
 void
-grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-			   pl_dom_list_t *dom)
+grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend)
 {
-        struct list_head  granted_list;
-        pl_entry_lock_t  *tmp = NULL;
-        pl_entry_lock_t  *lock = NULL;
-
-        INIT_LIST_HEAD (&granted_list);
-
-        pthread_mutex_lock (&pl_inode->mutex);
+    struct list_head granted_list;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *lock = NULL;
+
+    INIT_LIST_HEAD(&granted_list);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_entry_locks(this, pl_inode, dom, &granted_list, now,
+                                    contend);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted_list, blocked_locks)
+    {
+        entrylk_trace_out(this, lock->frame, NULL, NULL, NULL, lock->basename,
+                          ENTRYLK_LOCK, lock->type, 0, 0);
+
+        STACK_UNWIND_STRICT(entrylk, lock->frame, 0, 0, NULL);
+        lock->frame = NULL;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(lock, tmp, &granted_list, blocked_locks)
         {
-                __grant_blocked_entry_locks (this, pl_inode, dom,
-                                             &granted_list);
+            list_del_init(&lock->blocked_locks);
+            __pl_entrylk_unref(lock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
-                entrylk_trace_out (this, lock->frame, NULL, NULL, NULL,
-                                   lock->basename, ENTRYLK_LOCK, lock->type,
-                                   0, 0);
-
-                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
-		lock->frame = NULL;
-	}
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-		list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
-			list_del_init (&lock->blocked_locks);
-			__pl_entrylk_unref (lock);
-		}
-	}
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        return;
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 }
 
-
 /* Common entrylk code called by pl_entrylk and pl_fentrylk */
 int
-pl_common_entrylk (call_frame_t *frame, xlator_t *this,
-                   const char *volume, inode_t *inode, const char *basename,
-                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd,
-                   dict_t *xdata)
-
-{
-        int32_t          op_ret           = -1;
-        int32_t          op_errno         =  0;
-        int              ret              = -1;
-        char             unwind           =  1;
-        GF_UNUSED int    dict_ret         = -1;
-        pl_inode_t      *pinode           =  NULL;
-        pl_entry_lock_t *reqlock          =  NULL;
-        pl_entry_lock_t *unlocked         =  NULL;
-        pl_dom_list_t   *dom              =  NULL;
-        char            *conn_id          =  NULL;
-        pl_ctx_t        *ctx              =  NULL;
-	int              nonblock         =  0;
-        gf_boolean_t     need_inode_unref =  _gf_false;
-
-        if (xdata)
-                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
-
-        pinode = pl_inode_get (this, inode);
-        if (!pinode) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+pl_common_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  inode_t *inode, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, loc_t *loc, fd_t *fd, dict_t *xdata)
 
-	if (frame->root->client) {
-		ctx = pl_ctx_get (frame->root->client, this);
-		if (!ctx) {
-			op_errno = ENOMEM;
-			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
-			goto unwind;
-		}
-	}
-
-        dom = get_domain (pinode, volume);
-        if (!dom){
-                op_errno = ENOMEM;
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    char unwind = 1;
+    GF_UNUSED int dict_ret = -1;
+    pl_inode_t *pinode = NULL;
+    pl_entry_lock_t *reqlock = NULL;
+    pl_entry_lock_t *unlocked = NULL;
+    pl_dom_list_t *dom = NULL;
+    char *conn_id = NULL;
+    pl_ctx_t *ctx = NULL;
+    int nonblock = 0;
+    gf_boolean_t need_inode_unref = _gf_false;
+    posix_locks_private_t *priv = NULL;
+    struct list_head *pcontend = NULL;
+    struct list_head contend;
+    struct timespec now = {};
+
+    priv = this->private;
+
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    if (xdata)
+        dict_ret = dict_get_str(xdata, "connection-id", &conn_id);
+
+    pinode = pl_inode_get(this, inode, NULL);
+    if (!pinode) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            op_errno = ENOMEM;
+            gf_log(this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+            goto unwind;
+        }
+    }
+
+    dom = get_domain(pinode, volume);
+    if (!dom) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    entrylk_trace_in(this, frame, volume, fd, loc, basename, cmd, type);
+
+    reqlock = new_entrylk_lock(pinode, basename, type, dom->domain, frame,
+                               conn_id, &op_errno);
+    if (!reqlock) {
+        op_ret = -1;
+        goto unwind;
+    }
+
+    /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
+     * an unsuccessful blocking lock operation, the inode needs to be ref'd.
+     *
+     * But doing so might give room to a race where the lock-requesting
+     * client could send a DISCONNECT just before this thread refs the inode
+     * after the locking is done, and the epoll thread could unref the inode
+     * in cleanup which means the inode's refcount would come down to 0, and
+     * the call to pl_forget() at this point destroys @pinode. Now when
+     * the io-thread executing this function tries to access pinode,
+     * it could crash on account of illegal memory access.
+     *
+     * To get around this problem, the inode is ref'd once even before
+     * adding the lock into client_list as a precautionary measure.
+     * This way even if there are DISCONNECTs, there will always be 1 extra
+     * ref on the inode, so @pinode is still alive until after the
+     * current stack unwinds.
+     */
+    pinode->inode = inode_ref(inode);
+    if (priv->revocation_secs != 0) {
+        if (cmd != ENTRYLK_UNLOCK) {
+            __entrylk_prune_stale(this, pinode, dom, reqlock);
+        } else if (priv->monkey_unlocking == _gf_true) {
+            if (pl_does_monkey_want_stuck_lock()) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "MONKEY LOCKING (forcing stuck lock)!");
+                op_ret = 0;
+                need_inode_unref = _gf_true;
+                pthread_mutex_lock(&pinode->mutex);
+                {
+                    __pl_entrylk_unref(reqlock);
+                }
+                pthread_mutex_unlock(&pinode->mutex);
                 goto out;
+            }
         }
+    }
 
-        entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
+    switch (cmd) {
+        case ENTRYLK_LOCK_NB:
+            nonblock = 1;
+            /* fall through */
+        case ENTRYLK_LOCK:
+            if (ctx)
+                pthread_mutex_lock(&ctx->lock);
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                reqlock->pinode = pinode;
+
+                ret = __lock_entrylk(this, pinode, reqlock, nonblock, dom, &now,
+                                     pcontend);
+                if (ret == 0) {
+                    reqlock->frame = NULL;
+                    op_ret = 0;
+                } else {
+                    op_errno = -ret;
+                }
 
-        reqlock = new_entrylk_lock (pinode, basename, type, dom->domain, frame,
-				    conn_id);
-        if (!reqlock) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+                if (ctx && (!ret || !nonblock))
+                    list_add(&reqlock->client_list, &ctx->entrylk_lockers);
 
-        /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
-         * an unsuccessful blocking lock operation, the inode needs to be ref'd.
-         *
-         * But doing so might give room to a race where the lock-requesting
-         * client could send a DISCONNECT just before this thread refs the inode
-         * after the locking is done, and the epoll thread could unref the inode
-         * in cleanup which means the inode's refcount would come down to 0, and
-         * the call to pl_forget() at this point destroys @pinode. Now when
-         * the io-thread executing this function tries to access pinode,
-         * it could crash on account of illegal memory access.
-         *
-         * To get around this problem, the inode is ref'd once even before
-         * adding the lock into client_list as a precautionary measure.
-         * This way even if there are DISCONNECTs, there will always be 1 extra
-         * ref on the inode, so @pinode is still alive until after the
-         * current stack unwinds.
-         */
-        pinode->inode = inode_ref (inode);
-
-        switch (cmd) {
-        case ENTRYLK_LOCK_NB:
-		nonblock = 1;
-		/* fall through */
-	case ENTRYLK_LOCK:
-		if (ctx)
-			pthread_mutex_lock (&ctx->lock);
-                pthread_mutex_lock (&pinode->mutex);
-                {
-			reqlock->pinode = pinode;
-
-                        ret = __lock_entrylk (this, pinode, reqlock, nonblock, dom);
-			if (ret == 0) {
-				reqlock->frame = NULL;
-				op_ret = 0;
-			} else {
-				op_errno = -ret;
-			}
-
-			if (ctx && (!ret || !nonblock))
-				list_add (&reqlock->client_list,
-					  &ctx->entrylk_lockers);
-
-			if (ret == -EAGAIN && !nonblock) {
-				/* blocked */
-				unwind = 0;
-			} else {
-				__pl_entrylk_unref (reqlock);
-			}
-
-                        /* For all but the case where a non-blocking lock
-                         * attempt fails, the extra ref taken before the switch
-                         * block must be negated.
-                         */
-                        if ((ret == -EAGAIN) && (nonblock))
-                                need_inode_unref = _gf_true;
+                if (ret == -EAGAIN && !nonblock) {
+                    /* blocked */
+                    unwind = 0;
+                } else {
+                    __pl_entrylk_unref(reqlock);
                 }
-                pthread_mutex_unlock (&pinode->mutex);
-		if (ctx)
-			pthread_mutex_unlock (&ctx->lock);
-		break;
+
+                /* For all but the case where a non-blocking lock
+                 * attempt fails, the extra ref taken before the switch
+                 * block must be negated.
+                 */
+                if ((ret == -EAGAIN) && (nonblock))
+                    need_inode_unref = _gf_true;
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+            if (ctx)
+                pthread_mutex_unlock(&ctx->lock);
+            break;
 
         case ENTRYLK_UNLOCK:
-		if (ctx)
-			pthread_mutex_lock (&ctx->lock);
-                pthread_mutex_lock (&pinode->mutex);
-                {
-                        /* Irrespective of whether unlock succeeds or not,
-                         * the extra inode ref that was done before the switch
-                         * block must be negated. Towards this,
-                         * @need_inode_unref flag is set unconditionally here.
-                         */
-                        need_inode_unref = _gf_true;
-                        unlocked = __unlock_entrylk (dom, reqlock);
-			if (unlocked) {
-				list_del_init (&unlocked->client_list);
-				__pl_entrylk_unref (unlocked);
-				op_ret = 0;
-			} else {
-				op_errno = EINVAL;
-			}
-			__pl_entrylk_unref (reqlock);
+            if (ctx)
+                pthread_mutex_lock(&ctx->lock);
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                /* Irrespective of whether unlock succeeds or not,
+                 * the extra inode ref that was done before the switch
+                 * block must be negated. Towards this,
+                 * @need_inode_unref flag is set unconditionally here.
+                 */
+                need_inode_unref = _gf_true;
+                unlocked = __unlock_entrylk(dom, reqlock);
+                if (unlocked) {
+                    list_del_init(&unlocked->client_list);
+                    __pl_entrylk_unref(unlocked);
+                    op_ret = 0;
+                } else {
+                    op_errno = EINVAL;
                 }
-                pthread_mutex_unlock (&pinode->mutex);
-		if (ctx)
-			pthread_mutex_unlock (&ctx->lock);
+                __pl_entrylk_unref(reqlock);
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+            if (ctx)
+                pthread_mutex_unlock(&ctx->lock);
 
-		grant_blocked_entry_locks (this, pinode, dom);
+            grant_blocked_entry_locks(this, pinode, dom, &now, pcontend);
 
-                break;
+            break;
 
         default:
-                inode_unref (pinode->inode);
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unexpected case in entrylk (cmd=%d). Please file"
-                        "a bug report at http://bugs.gluster.com", cmd);
-                goto out;
-        }
-        if (need_inode_unref)
-                inode_unref (pinode->inode);
-
-        /* The following (extra) unref corresponds to the ref that
-         * was done at the time the lock was granted.
-         */
-        if ((cmd == ENTRYLK_UNLOCK) && (op_ret == 0))
-                inode_unref (pinode->inode);
+            need_inode_unref = _gf_true;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Unexpected case in entrylk (cmd=%d). Please file"
+                   "a bug report at http://bugs.gluster.com",
+                   cmd);
+            goto out;
+    }
+    /* The following (extra) unref corresponds to the ref that
+     * was done at the time the lock was granted.
+     */
+    if ((cmd == ENTRYLK_UNLOCK) && (op_ret == 0))
+        inode_unref(pinode->inode);
 
 out:
 
-        if (unwind) {
-                entrylk_trace_out (this, frame, volume, fd, loc, basename,
-                                   cmd, type, op_ret, op_errno);
-unwind:
-                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, NULL);
-        } else {
-                entrylk_trace_block (this, frame, volume, fd, loc, basename,
-                                     cmd, type);
-        }
+    if (need_inode_unref)
+        inode_unref(pinode->inode);
+
+    if (unwind) {
+        entrylk_trace_out(this, frame, volume, fd, loc, basename, cmd, type,
+                          op_ret, op_errno);
+    unwind:
+        STACK_UNWIND_STRICT(entrylk, frame, op_ret, op_errno, NULL);
+    }
 
-        return 0;
+    if (pcontend != NULL) {
+        entrylk_contention_notify(this, pcontend);
+    }
+
+    return 0;
 }
 
 /**
@@ -709,17 +947,16 @@ unwind:
  */
 
 int
-pl_entrylk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+pl_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
 {
-        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd,
-                           type, loc, NULL, xdata);
+    pl_common_entrylk(frame, this, volume, loc->inode, basename, cmd, type, loc,
+                      NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 /**
  * pl_fentrylk:
  *
@@ -727,176 +964,190 @@ pl_entrylk (call_frame_t *frame, xlator_t *this,
  */
 
 int
-pl_fentrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+pl_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd,
-                           type, NULL, fd, xdata);
+    pl_common_entrylk(frame, this, volume, fd->inode, basename, cmd, type, NULL,
+                      fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 static void
-pl_entrylk_log_cleanup (pl_entry_lock_t *lock)
+pl_entrylk_log_cleanup(pl_entry_lock_t *lock)
 {
-	pl_inode_t *pinode = NULL;
+    pl_inode_t *pinode = NULL;
 
-	pinode = lock->pinode;
+    pinode = lock->pinode;
 
-        gf_log (THIS->name, GF_LOG_WARNING,
-                "releasing lock on %s held by "
-                "{client=%p, pid=%"PRId64" lk-owner=%s}",
-                uuid_utoa (pinode->gfid), lock->client,
-                (uint64_t) lock->client_pid, lkowner_utoa (&lock->owner));
+    gf_log(THIS->name, GF_LOG_WARNING,
+           "releasing lock on %s held by "
+           "{client=%p, pid=%" PRId64 " lk-owner=%s}",
+           uuid_utoa(pinode->gfid), lock->client, (uint64_t)lock->client_pid,
+           lkowner_utoa(&lock->owner));
 }
 
-
 /* Release all entrylks from this client */
 int
-pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+pl_entrylk_client_cleanup(xlator_t *this, pl_ctx_t *ctx)
 {
-        pl_entry_lock_t *tmp = NULL;
-        pl_entry_lock_t *l = NULL;
-	pl_dom_list_t *dom = NULL;
-        pl_inode_t *pinode = NULL;
-
-        struct list_head released;
-        struct list_head unwind;
-
-        INIT_LIST_HEAD (&released);
-        INIT_LIST_HEAD (&unwind);
-
-	pthread_mutex_lock (&ctx->lock);
+    posix_locks_private_t *priv;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *l = NULL;
+    pl_dom_list_t *dom = NULL;
+    pl_inode_t *pinode = NULL;
+    struct list_head *pcontend = NULL;
+    struct list_head released;
+    struct list_head unwind;
+    struct list_head contend;
+    struct timespec now = {};
+
+    INIT_LIST_HEAD(&released);
+    INIT_LIST_HEAD(&unwind);
+
+    priv = this->private;
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        list_for_each_entry_safe(l, tmp, &ctx->entrylk_lockers, client_list)
         {
-                list_for_each_entry_safe (l, tmp, &ctx->entrylk_lockers,
-					  client_list) {
-                        list_del_init (&l->client_list);
-
-			pl_entrylk_log_cleanup (l);
-
-			pinode = l->pinode;
-
-			pthread_mutex_lock (&pinode->mutex);
-			{
-                        /* If the entrylk object is part of granted list but not
-                         * blocked list, then perform the following actions:
-                         * i.   delete the object from granted list;
-                         * ii.  grant other locks (from other clients) that may
-                         *      have been blocked on this entrylk; and
-                         * iii. unref the object.
-                         *
-                         * If the entrylk object (L1) is part of both granted
-                         * and blocked lists, then this means that a parallel
-                         * unlock on another entrylk (L2 say) may have 'granted'
-                         * L1 and added it to 'granted' list in
-                         * __grant_blocked_entry_locks() (although using the
-                         * 'blocked_locks' member). In that case, the cleanup
-                         * codepath must try and grant other overlapping
-                         * blocked entrylks from other clients, now that L1 is
-                         * out of their way and then unref L1 in the end, and
-                         * leave it to the other thread (the one executing
-                         * unlock codepath) to unwind L1's frame, delete it from
-                         * blocked_locks list, and perform the last unref on L1.
-                         *
-                         * If the entrylk object (L1) is part of blocked list
-                         * only, the cleanup code path must:
-                         * i.   delete it from the blocked_locks list inside
-                         *      this critical section,
-                         * ii.  unwind its frame with EAGAIN,
-                         * iii. try and grant blocked entry locks from other
-                         *      clients that were otherwise grantable, but were
-                         *      blocked to avoid leaving L1 to starve forever.
-                         * iv.  unref the object.
-                         */
-                                if (!list_empty (&l->domain_list)) {
-                                        list_del_init (&l->domain_list);
-                                        list_add_tail (&l->client_list,
-                                                       &released);
-                                } else {
-                                        list_del_init (&l->blocked_locks);
-                                        list_add_tail (&l->client_list,
-                                                       &unwind);
-                                }
-                        }
-			pthread_mutex_unlock (&pinode->mutex);
+            pl_entrylk_log_cleanup(l);
+
+            pinode = l->pinode;
+
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                /* If the entrylk object is part of granted list but not
+                 * blocked list, then perform the following actions:
+                 * i.   delete the object from granted list;
+                 * ii.  grant other locks (from other clients) that may
+                 *      have been blocked on this entrylk; and
+                 * iii. unref the object.
+                 *
+                 * If the entrylk object (L1) is part of both granted
+                 * and blocked lists, then this means that a parallel
+                 * unlock on another entrylk (L2 say) may have 'granted'
+                 * L1 and added it to 'granted' list in
+                 * __grant_blocked_entry_locks() (although using the
+                 * 'blocked_locks' member). In that case, the cleanup
+                 * codepath must try and grant other overlapping
+                 * blocked entrylks from other clients, now that L1 is
+                 * out of their way and then unref L1 in the end, and
+                 * leave it to the other thread (the one executing
+                 * unlock codepath) to unwind L1's frame, delete it from
+                 * blocked_locks list, and perform the last unref on L1.
+                 *
+                 * If the entrylk object (L1) is part of blocked list
+                 * only, the cleanup code path must:
+                 * i.   delete it from the blocked_locks list inside
+                 *      this critical section,
+                 * ii.  unwind its frame with EAGAIN,
+                 * iii. try and grant blocked entry locks from other
+                 *      clients that were otherwise grantable, but were
+                 *      blocked to avoid leaving L1 to starve forever.
+                 * iv.  unref the object.
+                 */
+                list_del_init(&l->client_list);
+
+                if (!list_empty(&l->domain_list)) {
+                    list_del_init(&l->domain_list);
+                    list_add_tail(&l->client_list, &released);
+                } else {
+                    list_del_init(&l->blocked_locks);
+                    list_add_tail(&l->client_list, &unwind);
                 }
-	}
-        pthread_mutex_unlock (&ctx->lock);
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+        }
+    }
+    pthread_mutex_unlock(&ctx->lock);
 
-        list_for_each_entry_safe (l, tmp, &unwind, client_list) {
-                list_del_init (&l->client_list);
+    if (!list_empty(&unwind)) {
+        list_for_each_entry_safe(l, tmp, &unwind, client_list)
+        {
+            list_del_init(&l->client_list);
 
-		if (l->frame)
-			STACK_UNWIND_STRICT (entrylk, l->frame, -1, EAGAIN,
-					     NULL);
-                list_add_tail (&l->client_list, &released);
+            if (l->frame)
+                STACK_UNWIND_STRICT(entrylk, l->frame, -1, EAGAIN, NULL);
+            list_add_tail(&l->client_list, &released);
         }
+    }
 
-        list_for_each_entry_safe (l, tmp, &released, client_list) {
-                list_del_init (&l->client_list);
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(l, tmp, &released, client_list)
+        {
+            list_del_init(&l->client_list);
 
-		pinode = l->pinode;
+            pinode = l->pinode;
 
-		dom = get_domain (pinode, l->volume);
+            dom = get_domain(pinode, l->volume);
 
-		grant_blocked_entry_locks (this, pinode, dom);
+            grant_blocked_entry_locks(this, pinode, dom, &now, pcontend);
 
-		pthread_mutex_lock (&pinode->mutex);
-		{
-			__pl_entrylk_unref (l);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
-                inode_unref (pinode->inode);
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                __pl_entrylk_unref(l);
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+
+            inode_unref(pinode->inode);
         }
+    }
 
-        return 0;
-}
+    if (pcontend != NULL) {
+        entrylk_contention_notify(this, pcontend);
+    }
 
+    return 0;
+}
 
 int32_t
-__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
+__get_entrylk_count(xlator_t *this, pl_inode_t *pl_inode)
 {
-        int32_t            count = 0;
-        pl_entry_lock_t   *lock  = NULL;
-        pl_dom_list_t     *dom   = NULL;
+    int32_t count = 0;
+    pl_entry_lock_t *lock = NULL;
+    pl_dom_list_t *dom = NULL;
 
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-                        count++;
-                }
-
-                list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-                        count++;
-                }
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        list_for_each_entry(lock, &dom->entrylk_list, domain_list) { count++; }
 
+        list_for_each_entry(lock, &dom->blocked_entrylks, blocked_locks)
+        {
+            count++;
         }
+    }
 
-        return count;
+    return count;
 }
 
 int32_t
-get_entrylk_count (xlator_t *this, inode_t *inode)
+get_entrylk_count(xlator_t *this, inode_t *inode)
 {
-        pl_inode_t   *pl_inode = NULL;
-        uint64_t      tmp_pl_inode = 0;
-        int           ret      = 0;
-        int32_t       count    = 0;
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int ret = 0;
+    int32_t count = 0;
 
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
-        if (ret != 0) {
-                goto out;
-        }
+    ret = inode_ctx_get(inode, this, &tmp_pl_inode);
+    if (ret != 0) {
+        goto out;
+    }
 
-        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                count = __get_entrylk_count (this, pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        count = __get_entrylk_count(this, pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
 out:
-        return count;
+    return count;
 }
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index 1564f26b8fb..d4e51d6e0a1 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -7,888 +7,1168 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "locks.h"
+#include "clear.h"
 #include "common.h"
 
 void
-__delete_inode_lock (pl_inode_lock_t *lock)
+__delete_inode_lock(pl_inode_lock_t *lock)
 {
-        list_del_init (&lock->list);
+    list_del_init(&lock->list);
 }
 
 static void
-__pl_inodelk_ref (pl_inode_lock_t *lock)
+__pl_inodelk_ref(pl_inode_lock_t *lock)
 {
-        lock->ref++;
+    lock->ref++;
 }
 
 void
-__pl_inodelk_unref (pl_inode_lock_t *lock)
+__pl_inodelk_unref(pl_inode_lock_t *lock)
 {
-        lock->ref--;
-        if (!lock->ref) {
-                GF_FREE (lock->connection_id);
-                GF_FREE (lock);
-        }
+    lock->ref--;
+    if (!lock->ref) {
+        GF_FREE(lock->connection_id);
+        GF_FREE(lock);
+    }
 }
 
-/* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't conflict */
+/* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't
+ * conflict */
 static int
-inodelk_type_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+inodelk_type_conflict(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
-                return 1;
+    if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
+        return 1;
 
-        return 0;
+    return 0;
 }
 
 void
-pl_print_inodelk (char *str, int size, int cmd, struct gf_flock *flock, const char *domain)
+pl_print_inodelk(char *str, int size, int cmd, struct gf_flock *flock,
+                 const char *domain)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
 
-        switch (cmd) {
+    switch (cmd) {
 #if F_GETLK != F_GETLK64
         case F_GETLK64:
 #endif
         case F_GETLK:
-                cmd_str = "GETLK";
-                break;
+            cmd_str = "GETLK";
+            break;
 
 #if F_SETLK != F_SETLK64
         case F_SETLK64:
 #endif
         case F_SETLK:
-                cmd_str = "SETLK";
-                break;
+            cmd_str = "SETLK";
+            break;
 
 #if F_SETLKW != F_SETLKW64
         case F_SETLKW64:
 #endif
         case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
+            cmd_str = "SETLKW";
+            break;
 
         default:
-                cmd_str = "UNKNOWN";
-                break;
-        }
+            cmd_str = "UNKNOWN";
+            break;
+    }
 
-        switch (flock->l_type) {
+    switch (flock->l_type) {
         case F_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case F_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
+            type_str = "UNLOCK";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
-                  "domain: %s, start=%llu, len=%llu, pid=%llu",
-                  cmd_str, type_str, domain,
-                  (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid);
+            type_str = "UNKNOWN";
+            break;
+    }
+
+    snprintf(str, size,
+             "lock=INODELK, cmd=%s, type=%s, "
+             "domain: %s, start=%llu, len=%llu, pid=%llu",
+             cmd_str, type_str, domain, (unsigned long long)flock->l_start,
+             (unsigned long long)flock->l_len,
+             (unsigned long long)flock->l_pid);
 }
 
 /* Determine if the two inodelks overlap reach other's lock regions */
 static int
-inodelk_overlap (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+inodelk_overlap(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        return ((l1->fl_end >= l2->fl_start) &&
-                (l2->fl_end >= l1->fl_start));
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
 }
 
 /* Returns true if the 2 inodelks have the same owner */
 static int
-same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+same_inodelk_owner(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->client == l2->client));
+    return (is_same_lkowner(&l1->owner, &l2->owner) &&
+            (l1->client == l2->client));
 }
 
 /* Returns true if the 2 inodelks conflict with each other */
 static int
-inodelk_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+inodelk_conflict(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        return (inodelk_overlap (l1, l2) &&
-                inodelk_type_conflict (l1, l2));
+    return (inodelk_overlap(l1, l2) && inodelk_type_conflict(l1, l2));
 }
 
-/* Determine if lock is grantable or not */
-static pl_inode_lock_t *
-__inodelk_grantable (pl_dom_list_t *dom, pl_inode_lock_t *lock)
+/*
+ * Check to see if the candidate lock overlaps/conflicts with the
+ * requested lock.  If so, determine how old the lock is and return
+ * true if it exceeds the configured threshold, false otherwise.
+ */
+static inline gf_boolean_t
+__stale_inodelk(xlator_t *this, pl_inode_lock_t *candidate_lock,
+                pl_inode_lock_t *requested_lock, time_t *lock_age_sec)
 {
-        pl_inode_lock_t *l = NULL;
-        pl_inode_lock_t *ret = NULL;
-        if (list_empty (&dom->inodelk_list))
-                goto out;
-        list_for_each_entry (l, &dom->inodelk_list, list){
-                if (inodelk_conflict (lock, l) &&
-                    !same_inodelk_owner (lock, l)) {
-                        ret = l;
-                        goto out;
-                }
-        }
-out:
-        return ret;
+    posix_locks_private_t *priv = NULL;
+
+    priv = this->private;
+    /* Question: Should we just prune them all given the
+     * chance?  Or just the locks we are attempting to acquire?
+     */
+    if (inodelk_conflict(candidate_lock, requested_lock)) {
+        *lock_age_sec = gf_time() - candidate_lock->granted_time;
+        if (*lock_age_sec > priv->revocation_secs)
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-static pl_inode_lock_t *
-__blocked_lock_conflict (pl_dom_list_t *dom, pl_inode_lock_t *lock)
+/* Examine any locks held on this inode and potentially revoke the lock
+ * if the age exceeds revocation_secs.  We will clear _only_ those locks
+ * which are granted, and then grant those locks which are blocked.
+ *
+ * Depending on how this patch works in the wild, we may expand this and
+ * introduce a heuristic which clears blocked locks as well if they
+ * are beyond a threshold.
+ */
+static gf_boolean_t
+__inodelk_prune_stale(xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom,
+                      pl_inode_lock_t *lock)
 {
-        pl_inode_lock_t *l   = NULL;
-        pl_inode_lock_t *ret = NULL;
-
-        if (list_empty (&dom->blocked_inodelks))
-                return NULL;
+    posix_locks_private_t *priv = NULL;
+    pl_inode_lock_t *tmp = NULL;
+    pl_inode_lock_t *lk = NULL;
+    gf_boolean_t revoke_lock = _gf_false;
+    int bcount = 0;
+    int gcount = 0;
+    int op_errno = 0;
+    clrlk_args args;
+    args.opts = NULL;
+    time_t lk_age_sec = 0;
+    uint32_t max_blocked = 0;
+    char *reason_str = NULL;
+
+    priv = this->private;
+
+    args.type = CLRLK_INODE;
+    if (priv->revocation_clear_all == _gf_true)
+        args.kind = CLRLK_ALL;
+    else
+        args.kind = CLRLK_GRANTED;
+
+    if (list_empty(&dom->inodelk_list))
+        goto out;
+
+    pthread_mutex_lock(&pinode->mutex);
+    list_for_each_entry_safe(lk, tmp, &dom->inodelk_list, list)
+    {
+        if (__stale_inodelk(this, lk, lock, &lk_age_sec) == _gf_true) {
+            revoke_lock = _gf_true;
+            reason_str = "age";
+            break;
+        }
+    }
 
-        list_for_each_entry (l, &dom->blocked_inodelks, blocked_locks) {
-                if (inodelk_conflict (lock, l)) {
-                        ret = l;
-                        goto out;
-                }
+    max_blocked = priv->revocation_max_blocked;
+    if (max_blocked != 0 && revoke_lock == _gf_false) {
+        list_for_each_entry_safe(lk, tmp, &dom->blocked_inodelks, blocked_locks)
+        {
+            max_blocked--;
+            if (max_blocked == 0) {
+                revoke_lock = _gf_true;
+                reason_str = "max blocked";
+                break;
+            }
         }
+    }
+    pthread_mutex_unlock(&pinode->mutex);
 
 out:
-        return ret;
+    if (revoke_lock == _gf_true) {
+        clrlk_clear_inodelk(this, pinode, dom, &args, &bcount, &gcount,
+                            &op_errno);
+        gf_log(this->name, GF_LOG_WARNING,
+               "Lock revocation [reason: %s; gfid: %s; domain: %s; "
+               "age: %ld sec] - Inode lock revoked:  %d granted & %d "
+               "blocked locks cleared",
+               reason_str, uuid_utoa(pinode->gfid), dom->domain, lk_age_sec,
+               gcount, bcount);
+    }
+    return revoke_lock;
 }
 
-static int
-__owner_has_lock (pl_dom_list_t *dom, pl_inode_lock_t *newlock)
+void
+inodelk_contention_notify_check(xlator_t *this, pl_inode_lock_t *lock,
+                                struct timespec *now, struct list_head *contend)
 {
-        pl_inode_lock_t *lock = NULL;
+    posix_locks_private_t *priv;
+    int64_t elapsed;
 
-        list_for_each_entry (lock, &dom->inodelk_list, list) {
-                if (same_inodelk_owner (lock, newlock))
-                        return 1;
-        }
+    priv = this->private;
 
-        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
-                if (same_inodelk_owner (lock, newlock))
-                        return 1;
-        }
+    /* If this lock is in a list, it means that we are about to send a
+     * notification for it, so no need to do anything else. */
+    if (!list_empty(&lock->contend)) {
+        return;
+    }
+
+    elapsed = now->tv_sec;
+    elapsed -= lock->contention_time.tv_sec;
+    if (now->tv_nsec < lock->contention_time.tv_nsec) {
+        elapsed--;
+    }
+    if (elapsed < priv->notify_contention_delay) {
+        return;
+    }
 
-        return 0;
-}
+    /* All contention notifications will be sent outside of the locked
+     * region. This means that currently granted locks might have already
+     * been unlocked by that time. To avoid the lock or the inode to be
+     * destroyed before we process them, we take an additional reference
+     * on both. */
+    inode_ref(lock->pl_inode->inode);
+    __pl_inodelk_ref(lock);
 
+    lock->contention_time = *now;
 
-/* Determines if lock can be granted and adds the lock. If the lock
- * is blocking, adds it to the blocked_inodelks list of the domain.
- */
-static int
-__lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
-                int can_block,  pl_dom_list_t *dom)
+    list_add_tail(&lock->contend, contend);
+}
+
+void
+inodelk_contention_notify(xlator_t *this, struct list_head *contend)
 {
-        pl_inode_lock_t *conf = NULL;
-        int ret = -EINVAL;
+    struct gf_upcall up;
+    struct gf_upcall_inodelk_contention lc;
+    pl_inode_lock_t *lock;
+    pl_inode_t *pl_inode;
+    client_t *client;
+    gf_boolean_t notify;
+
+    while (!list_empty(contend)) {
+        lock = list_first_entry(contend, pl_inode_lock_t, contend);
+
+        pl_inode = lock->pl_inode;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        /* If the lock has already been released, no notification is
+         * sent. We clear the notification time in this case. */
+        notify = !list_empty(&lock->list);
+        if (!notify) {
+            lock->contention_time.tv_sec = 0;
+            lock->contention_time.tv_nsec = 0;
+        } else {
+            memcpy(&lc.flock, &lock->user_flock, sizeof(lc.flock));
+            lc.pid = lock->client_pid;
+            lc.domain = lock->volume;
+            lc.xdata = NULL;
+
+            gf_uuid_copy(up.gfid, lock->pl_inode->gfid);
+            client = (client_t *)lock->client;
+            if (client == NULL) {
+                /* A NULL client can be found if the inodelk
+                 * was issued by a server side xlator. */
+                up.client_uid = NULL;
+            } else {
+                up.client_uid = client->client_uid;
+            }
+        }
 
-        conf = __inodelk_grantable (dom, lock);
-        if (conf) {
-                ret = -EAGAIN;
-                if (can_block == 0)
-                        goto out;
+        pthread_mutex_unlock(&pl_inode->mutex);
+
+        if (notify) {
+            up.event_type = GF_UPCALL_INODELK_CONTENTION;
+            up.data = &lc;
+
+            if (this->notify(this, GF_EVENT_UPCALL, &up) < 0) {
+                gf_msg_debug(this->name, 0,
+                             "Inodelk contention notification "
+                             "failed");
+            } else {
+                gf_msg_debug(this->name, 0,
+                             "Inodelk contention notification "
+                             "sent");
+            }
+        }
 
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
+        pthread_mutex_lock(&pl_inode->mutex);
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
+        list_del_init(&lock->contend);
+        __pl_inodelk_unref(lock);
 
+        pthread_mutex_unlock(&pl_inode->mutex);
 
-                goto out;
+        inode_unref(pl_inode->inode);
+    }
+}
+
+/* Determine if lock is grantable or not */
+static pl_inode_lock_t *
+__inodelk_grantable(xlator_t *this, pl_dom_list_t *dom, pl_inode_lock_t *lock,
+                    struct timespec *now, struct list_head *contend)
+{
+    pl_inode_lock_t *l = NULL;
+    pl_inode_lock_t *ret = NULL;
+
+    list_for_each_entry(l, &dom->inodelk_list, list)
+    {
+        if (inodelk_conflict(lock, l) && !same_inodelk_owner(lock, l)) {
+            if (ret == NULL) {
+                ret = l;
+                if (contend == NULL) {
+                    break;
+                }
+            }
+            inodelk_contention_notify_check(this, l, now, contend);
         }
+    }
 
-        /* To prevent blocked locks starvation, check if there are any blocked
-         * locks thay may conflict with this lock. If there is then don't grant
-         * the lock. BUT grant the lock if the owner already has lock to allow
-         * nested locks.
-         * Example:
-         * SHD from Machine1 takes (gfid, 0-infinity) and is granted.
-         * SHD from machine2 takes (gfid, 0-infinity) and is blocked.
-         * When SHD from Machine1 takes (gfid, 0-128KB) it
-         * needs to be granted, without which the earlier lock on 0-infinity
-         * will not be unlocked by SHD from Machine1.
-         * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
-         */
-        if (__blocked_lock_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
-                ret = -EAGAIN;
-                if (can_block == 0)
-                        goto out;
+    return ret;
+}
 
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
+static pl_inode_lock_t *
+__blocked_lock_conflict(pl_dom_list_t *dom, pl_inode_lock_t *lock)
+{
+    pl_inode_lock_t *l = NULL;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock is grantable, but blocking to prevent starvation");
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Blocked",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
+    list_for_each_entry(l, &dom->blocked_inodelks, blocked_locks)
+    {
+        if (inodelk_conflict(lock, l)) {
+            return l;
+        }
+    }
 
+    return NULL;
+}
 
-                goto out;
-        }
-        __pl_inodelk_ref (lock);
-        gettimeofday (&lock->granted_time, NULL);
-        list_add (&lock->list, &dom->inodelk_list);
+static int
+__owner_has_lock(pl_dom_list_t *dom, pl_inode_lock_t *newlock)
+{
+    pl_inode_lock_t *lock = NULL;
+
+    list_for_each_entry(lock, &dom->inodelk_list, list)
+    {
+        if (same_inodelk_owner(lock, newlock))
+            return 1;
+    }
 
-        ret = 0;
+    list_for_each_entry(lock, &dom->blocked_inodelks, blocked_locks)
+    {
+        if (same_inodelk_owner(lock, newlock))
+            return 1;
+    }
 
+    return 0;
+}
+
+static int
+__lock_blocked_add(xlator_t *this, pl_dom_list_t *dom, pl_inode_lock_t *lock,
+                   int can_block)
+{
+    if (can_block == 0) {
+        goto out;
+    }
+
+    lock->blkd_time = gf_time();
+    list_add_tail(&lock->blocked_locks, &dom->blocked_inodelks);
+
+    gf_msg_trace(this->name, 0,
+                 "%s (pid=%d) (lk-owner=%s) %" PRId64
+                 " - "
+                 "%" PRId64 " => Blocked",
+                 lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+                 lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+                 lock->user_flock.l_len);
+
+    pl_trace_block(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                   lock->volume);
 out:
+    return -EAGAIN;
+}
+
+/* Determines if lock can be granted and adds the lock. If the lock
+ * is blocking, adds it to the blocked_inodelks list of the domain.
+ */
+static int
+__lock_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
+               int can_block, pl_dom_list_t *dom, struct timespec *now,
+               struct list_head *contend)
+{
+    pl_inode_lock_t *conf = NULL;
+    int ret;
+
+    ret = pl_inode_remove_inodelk(pl_inode, lock);
+    if (ret < 0) {
         return ret;
+    }
+    if (ret == 0) {
+        conf = __inodelk_grantable(this, dom, lock, now, contend);
+    }
+    if ((ret > 0) || (conf != NULL)) {
+        return __lock_blocked_add(this, dom, lock, can_block);
+    }
+
+    /* To prevent blocked locks starvation, check if there are any blocked
+     * locks thay may conflict with this lock. If there is then don't grant
+     * the lock. BUT grant the lock if the owner already has lock to allow
+     * nested locks.
+     * Example:
+     * SHD from Machine1 takes (gfid, 0-infinity) and is granted.
+     * SHD from machine2 takes (gfid, 0-infinity) and is blocked.
+     * When SHD from Machine1 takes (gfid, 0-128KB) it
+     * needs to be granted, without which the earlier lock on 0-infinity
+     * will not be unlocked by SHD from Machine1.
+     * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
+     */
+    if (__blocked_lock_conflict(dom, lock) && !(__owner_has_lock(dom, lock))) {
+        if (can_block != 0) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock is grantable, but blocking to prevent "
+                   "starvation");
+        }
+
+        return __lock_blocked_add(this, dom, lock, can_block);
+    }
+    __pl_inodelk_ref(lock);
+    lock->granted_time = gf_time();
+    list_add(&lock->list, &dom->inodelk_list);
+
+    return 0;
 }
 
 /* Return true if the two inodelks have exactly same lock boundaries */
 static int
-inodelks_equal (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+inodelks_equal(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        if ((l1->fl_start == l2->fl_start) &&
-            (l1->fl_end == l2->fl_end))
-                return 1;
+    if ((l1->fl_start == l2->fl_start) && (l1->fl_end == l2->fl_end))
+        return 1;
 
-        return 0;
+    return 0;
 }
 
-
 static pl_inode_lock_t *
-find_matching_inodelk (pl_inode_lock_t *lock, pl_dom_list_t *dom)
+find_matching_inodelk(pl_inode_lock_t *lock, pl_dom_list_t *dom)
 {
-        pl_inode_lock_t *l = NULL;
-        list_for_each_entry (l, &dom->inodelk_list, list) {
-                if (inodelks_equal (l, lock) &&
-                    same_inodelk_owner (l, lock))
-                        return l;
-        }
-        return NULL;
+    pl_inode_lock_t *l = NULL;
+    list_for_each_entry(l, &dom->inodelk_list, list)
+    {
+        if (inodelks_equal(l, lock) && same_inodelk_owner(l, lock))
+            return l;
+    }
+    return NULL;
 }
 
 /* Set F_UNLCK removes a lock which has the exact same lock boundaries
  * as the UNLCK lock specifies. If such a lock is not found, returns invalid
  */
 static pl_inode_lock_t *
-__inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
+__inode_unlock_lock(xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
 {
-
-        pl_inode_lock_t *conf = NULL;
-
-        conf = find_matching_inodelk (lock, dom);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        " Matching lock not found for unlock %llu-%llu, by %s "
-                        "on %p", (unsigned long long)lock->fl_start,
-                        (unsigned long long)lock->fl_end,
-                        lkowner_utoa (&lock->owner), lock->client);
-                goto out;
-        }
-        __delete_inode_lock (conf);
-        gf_log (this->name, GF_LOG_DEBUG,
-                " Matching lock found for unlock %llu-%llu, by %s on %p",
-                (unsigned long long)lock->fl_start,
-                (unsigned long long)lock->fl_end, lkowner_utoa (&lock->owner),
-                lock->client);
+    pl_inode_lock_t *conf = NULL;
+    inode_t *inode = NULL;
+
+    inode = lock->pl_inode->inode;
+
+    conf = find_matching_inodelk(lock, dom);
+    if (!conf) {
+        gf_log(this->name, GF_LOG_ERROR,
+               " Matching lock not found for unlock %llu-%llu, by %s "
+               "on %p for gfid:%s",
+               (unsigned long long)lock->fl_start,
+               (unsigned long long)lock->fl_end, lkowner_utoa(&lock->owner),
+               lock->client, inode ? uuid_utoa(inode->gfid) : "UNKNOWN");
+        goto out;
+    }
+    __delete_inode_lock(conf);
+    gf_log(this->name, GF_LOG_DEBUG,
+           " Matching lock found for unlock %llu-%llu, by %s on %p for gfid:%s",
+           (unsigned long long)lock->fl_start, (unsigned long long)lock->fl_end,
+           lkowner_utoa(&lock->owner), lock->client,
+           inode ? uuid_utoa(inode->gfid) : "UNKNOWN");
 
 out:
-        return conf;
+    return conf;
 }
 
-
-static void
-__grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
-                             struct list_head *granted, pl_dom_list_t *dom)
+void
+__grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                            struct list_head *granted, pl_dom_list_t *dom,
+                            struct timespec *now, struct list_head *contend)
 {
-        int              bl_ret = 0;
-        pl_inode_lock_t *bl = NULL;
-        pl_inode_lock_t *tmp = NULL;
+    pl_inode_lock_t *bl = NULL;
+    pl_inode_lock_t *tmp = NULL;
 
-        struct list_head blocked_list;
+    struct list_head blocked_list;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&dom->blocked_inodelks, &blocked_list);
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&dom->blocked_inodelks, &blocked_list);
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
+    list_for_each_entry_safe(bl, tmp, &blocked_list, blocked_locks)
+    {
+        list_del_init(&bl->blocked_locks);
 
-                list_del_init (&bl->blocked_locks);
+        bl->status = __lock_inodelk(this, pl_inode, bl, 1, dom, now, contend);
 
-                bl_ret = __lock_inodelk (this, pl_inode, bl, 1, dom);
-
-                if (bl_ret == 0) {
-                        list_add (&bl->blocked_locks, granted);
-                }
+        if (bl->status != -EAGAIN) {
+            list_add_tail(&bl->blocked_locks, granted);
         }
-        return;
+    }
 }
 
-/* Grant all inodelks blocked on a lock */
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_dom_list_t *dom)
+unwind_granted_inodes(xlator_t *this, pl_inode_t *pl_inode,
+                      struct list_head *granted)
 {
-        struct list_head granted;
-        pl_inode_lock_t *lock;
-        pl_inode_lock_t *tmp;
+    pl_inode_lock_t *lock;
+    pl_inode_lock_t *tmp;
+    int32_t op_ret;
+    int32_t op_errno;
+
+    list_for_each_entry_safe(lock, tmp, granted, blocked_locks)
+    {
+        if (lock->status == 0) {
+            op_ret = 0;
+            op_errno = 0;
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                   " => Granted",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+        } else {
+            op_ret = -1;
+            op_errno = -lock->status;
+        }
+        pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                     op_ret, op_errno, lock->volume);
 
-        INIT_LIST_HEAD (&granted);
+        STACK_UNWIND_STRICT(inodelk, lock->frame, op_ret, op_errno, NULL);
+        lock->frame = NULL;
+    }
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(lock, tmp, granted, blocked_locks)
         {
-                __grant_blocked_inode_locks (this, pl_inode, &granted, dom);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
-
-                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
-                              &lock->user_flock, 0, 0, lock->volume);
-
-                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0, NULL);
-		lock->frame = NULL;
+            list_del_init(&lock->blocked_locks);
+            __pl_inodelk_unref(lock);
         }
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
-                        list_del_init (&lock->blocked_locks);
-                        __pl_inodelk_unref (lock);
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 }
 
+/* Grant all inodelks blocked on a lock */
+void
+grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend)
+{
+    struct list_head granted;
+
+    INIT_LIST_HEAD(&granted);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_inode_locks(this, pl_inode, &granted, dom, now,
+                                    contend);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    unwind_granted_inodes(this, pl_inode, &granted);
+}
 
 static void
-pl_inodelk_log_cleanup (pl_inode_lock_t *lock)
+pl_inodelk_log_cleanup(pl_inode_lock_t *lock)
 {
-	pl_inode_t *pl_inode = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-	pl_inode = lock->pl_inode;
+    pl_inode = lock->pl_inode;
 
-        gf_log (THIS->name, GF_LOG_WARNING, "releasing lock on %s held by "
-                "{client=%p, pid=%"PRId64" lk-owner=%s}",
-                uuid_utoa (pl_inode->gfid), lock->client,
-                (uint64_t) lock->client_pid, lkowner_utoa (&lock->owner));
+    gf_log(THIS->name, GF_LOG_WARNING,
+           "releasing lock on %s held by "
+           "{client=%p, pid=%" PRId64 " lk-owner=%s}",
+           uuid_utoa(pl_inode->gfid), lock->client, (uint64_t)lock->client_pid,
+           lkowner_utoa(&lock->owner));
 }
 
-
 /* Release all inodelks from this client */
 int
-pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+pl_inodelk_client_cleanup(xlator_t *this, pl_ctx_t *ctx)
 {
-        pl_inode_lock_t *tmp = NULL;
-        pl_inode_lock_t *l = NULL;
-	pl_dom_list_t *dom = NULL;
-        pl_inode_t *pl_inode = NULL;
-
-        struct list_head released;
-        struct list_head unwind;
-
-        INIT_LIST_HEAD (&released);
-        INIT_LIST_HEAD (&unwind);
-
-	pthread_mutex_lock (&ctx->lock);
+    posix_locks_private_t *priv;
+    pl_inode_lock_t *tmp = NULL;
+    pl_inode_lock_t *l = NULL;
+    pl_dom_list_t *dom = NULL;
+    pl_inode_t *pl_inode = NULL;
+    struct list_head *pcontend = NULL;
+    struct list_head released;
+    struct list_head unwind;
+    struct list_head contend;
+    struct timespec now = {};
+
+    priv = this->private;
+
+    INIT_LIST_HEAD(&released);
+    INIT_LIST_HEAD(&unwind);
+
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        list_for_each_entry_safe(l, tmp, &ctx->inodelk_lockers, client_list)
         {
-                list_for_each_entry_safe (l, tmp, &ctx->inodelk_lockers,
-					  client_list) {
-                        list_del_init (&l->client_list);
-
-			pl_inodelk_log_cleanup (l);
-
-			pl_inode = l->pl_inode;
-
-			pthread_mutex_lock (&pl_inode->mutex);
-			{
-                        /* If the inodelk object is part of granted list but not
-                         * blocked list, then perform the following actions:
-                         * i.   delete the object from granted list;
-                         * ii.  grant other locks (from other clients) that may
-                         *      have been blocked on this inodelk; and
-                         * iii. unref the object.
-                         *
-                         * If the inodelk object (L1) is part of both granted
-                         * and blocked lists, then this means that a parallel
-                         * unlock on another inodelk (L2 say) may have 'granted'
-                         * L1 and added it to 'granted' list in
-                         * __grant_blocked_node_locks() (although using the
-                         * 'blocked_locks' member). In that case, the cleanup
-                         * codepath must try and grant other overlapping
-                         * blocked inodelks from other clients, now that L1 is
-                         * out of their way and then unref L1 in the end, and
-                         * leave it to the other thread (the one executing
-                         * unlock codepath) to unwind L1's frame, delete it from
-                         * blocked_locks list, and perform the last unref on L1.
-                         *
-                         * If the inodelk object (L1) is part of blocked list
-                         * only, the cleanup code path must:
-                         * i.   delete it from the blocked_locks list inside
-                         *      this critical section,
-                         * ii.  unwind its frame with EAGAIN,
-                         * iii. try and grant blocked inode locks from other
-                         *      clients that were otherwise grantable, but just
-                         *      got blocked to avoid leaving L1 to starve
-                         *      forever.
-                         * iv.  unref the object.
-                         */
-                                if (!list_empty (&l->list)) {
-                                        __delete_inode_lock (l);
-                                        list_add_tail (&l->client_list,
-                                                       &released);
-                                } else {
-                                        list_del_init(&l->blocked_locks);
-                                        list_add_tail (&l->client_list,
-                                                       &unwind);
-                                }
-                        }
-			pthread_mutex_unlock (&pl_inode->mutex);
+            pl_inodelk_log_cleanup(l);
+
+            pl_inode = l->pl_inode;
+
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                /* If the inodelk object is part of granted list but not
+                 * blocked list, then perform the following actions:
+                 * i.   delete the object from granted list;
+                 * ii.  grant other locks (from other clients) that may
+                 *      have been blocked on this inodelk; and
+                 * iii. unref the object.
+                 *
+                 * If the inodelk object (L1) is part of both granted
+                 * and blocked lists, then this means that a parallel
+                 * unlock on another inodelk (L2 say) may have 'granted'
+                 * L1 and added it to 'granted' list in
+                 * __grant_blocked_inode_locks() (although using the
+                 * 'blocked_locks' member). In that case, the cleanup
+                 * codepath must try and grant other overlapping
+                 * blocked inodelks from other clients, now that L1 is
+                 * out of their way and then unref L1 in the end, and
+                 * leave it to the other thread (the one executing
+                 * unlock codepath) to unwind L1's frame, delete it from
+                 * blocked_locks list, and perform the last unref on L1.
+                 *
+                 * If the inodelk object (L1) is part of blocked list
+                 * only, the cleanup code path must:
+                 * i.   delete it from the blocked_locks list inside
+                 *      this critical section,
+                 * ii.  unwind its frame with EAGAIN,
+                 * iii. try and grant blocked inode locks from other
+                 *      clients that were otherwise grantable, but just
+                 *      got blocked to avoid leaving L1 to starve
+                 *      forever.
+                 * iv.  unref the object.
+                 */
+                list_del_init(&l->client_list);
+
+                if (!list_empty(&l->list)) {
+                    __delete_inode_lock(l);
+                    list_add_tail(&l->client_list, &released);
+                } else {
+                    list_del_init(&l->blocked_locks);
+                    list_add_tail(&l->client_list, &unwind);
                 }
-	}
-        pthread_mutex_unlock (&ctx->lock);
-
-        list_for_each_entry_safe (l, tmp, &unwind, client_list) {
-                list_del_init (&l->client_list);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+        }
+    }
+    pthread_mutex_unlock(&ctx->lock);
 
-                if (l->frame)
-			STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN,
-					     NULL);
-                list_add_tail (&l->client_list, &released);
+    if (!list_empty(&unwind)) {
+        list_for_each_entry_safe(l, tmp, &unwind, client_list)
+        {
+            list_del_init(&l->client_list);
 
+            if (l->frame)
+                STACK_UNWIND_STRICT(inodelk, l->frame, -1, EAGAIN, NULL);
+            list_add_tail(&l->client_list, &released);
         }
+    }
 
-        list_for_each_entry_safe (l, tmp, &released, client_list) {
-                list_del_init (&l->client_list);
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(l, tmp, &released, client_list)
+        {
+            list_del_init(&l->client_list);
 
-		pl_inode = l->pl_inode;
+            pl_inode = l->pl_inode;
 
-		dom = get_domain (pl_inode, l->volume);
+            dom = get_domain(pl_inode, l->volume);
 
-		grant_blocked_inode_locks (this, pl_inode, dom);
+            grant_blocked_inode_locks(this, pl_inode, dom, &now, pcontend);
 
-		pthread_mutex_lock (&pl_inode->mutex);
-		{
-			__pl_inodelk_unref (l);
-		}
-		pthread_mutex_unlock (&pl_inode->mutex);
-                inode_unref (pl_inode->inode);
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                __pl_inodelk_unref(l);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+            inode_unref(pl_inode->inode);
         }
+    }
 
-        return 0;
-}
+    if (pcontend != NULL) {
+        inodelk_contention_notify(this, pcontend);
+    }
 
+    return 0;
+}
 
 static int
-pl_inode_setlk (xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode,
-		pl_inode_lock_t *lock, int can_block, pl_dom_list_t *dom,
-                inode_t *inode)
-{
-        int               ret              = -EINVAL;
-        pl_inode_lock_t  *retlock          =  NULL;
-        gf_boolean_t      unref            =  _gf_true;
-        gf_boolean_t      need_inode_unref =  _gf_false;
-        short             fl_type;
-
-	lock->pl_inode = pl_inode;
-        fl_type = lock->fl_type;
-
-        /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
-         * an unsuccessful blocking lock operation, the inode needs to be ref'd.
-         *
-         * But doing so might give room to a race where the lock-requesting
-         * client could send a DISCONNECT just before this thread refs the inode
-         * after the locking is done, and the epoll thread could unref the inode
-         * in cleanup which means the inode's refcount would come down to 0, and
-         * the call to pl_forget() at this point destroys @pl_inode. Now when
-         * the io-thread executing this function tries to access pl_inode,
-         * it could crash on account of illegal memory access.
-         *
-         * To get around this problem, the inode is ref'd once even before
-         * adding the lock into client_list as a precautionary measure.
-         * This way even if there are DISCONNECTs, there will always be 1 extra
-         * ref on the inode, so @pl_inode is still alive until after the
-         * current stack unwinds.
-         */
-        pl_inode->inode = inode_ref (inode);
-
-	if (ctx)
-		pthread_mutex_lock (&ctx->lock);
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (lock->fl_type != F_UNLCK) {
-                        ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
-                        if (ret == 0) {
-				lock->frame = NULL;
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->fl_start,
-                                        lock->fl_end);
-                        } else if (ret == -EAGAIN) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->user_flock.l_start,
-                                        lock->user_flock.l_len);
-                                if (can_block)
-                                        unref = _gf_false;
-                                /* For all but the case where a non-blocking
-                                 * lock attempt fails, the extra ref taken at
-                                 * the start of this function must be negated.
-                                 */
-                                else
-                                        need_inode_unref = _gf_true;
-                        }
-
-			if (ctx && (!ret || can_block))
-				list_add_tail (&lock->client_list,
-					       &ctx->inodelk_lockers);
-                } else {
-                        /* Irrespective of whether unlock succeeds or not,
-                         * the extra inode ref that was done at the start of
-                         * this function must be negated. Towards this,
-                         * @need_inode_unref flag is set unconditionally here.
-                         */
-                        need_inode_unref = _gf_true;
-                        retlock = __inode_unlock_lock (this, lock, dom);
-                        if (!retlock) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Bad Unlock issued on Inode lock");
-                                ret = -EINVAL;
-                                goto out;
-                        }
-			list_del_init (&retlock->client_list);
-			__pl_inodelk_unref (retlock);
-
-                        ret = 0;
+pl_inode_setlk(xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode,
+               pl_inode_lock_t *lock, int can_block, pl_dom_list_t *dom,
+               inode_t *inode)
+{
+    posix_locks_private_t *priv = NULL;
+    int ret = -EINVAL;
+    pl_inode_lock_t *retlock = NULL;
+    gf_boolean_t unref = _gf_true;
+    gf_boolean_t need_inode_unref = _gf_false;
+    struct list_head *pcontend = NULL;
+    struct list_head contend;
+    struct list_head wake;
+    struct timespec now = {};
+    short fl_type;
+
+    lock->pl_inode = pl_inode;
+    fl_type = lock->fl_type;
+
+    priv = this->private;
+
+    /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
+     * an unsuccessful blocking lock operation, the inode needs to be ref'd.
+     *
+     * But doing so might give room to a race where the lock-requesting
+     * client could send a DISCONNECT just before this thread refs the inode
+     * after the locking is done, and the epoll thread could unref the inode
+     * in cleanup which means the inode's refcount would come down to 0, and
+     * the call to pl_forget() at this point destroys @pl_inode. Now when
+     * the io-thread executing this function tries to access pl_inode,
+     * it could crash on account of illegal memory access.
+     *
+     * To get around this problem, the inode is ref'd once even before
+     * adding the lock into client_list as a precautionary measure.
+     * This way even if there are DISCONNECTs, there will always be 1 extra
+     * ref on the inode, so @pl_inode is still alive until after the
+     * current stack unwinds.
+     */
+    pl_inode->inode = inode_ref(inode);
+
+    if (priv->revocation_secs != 0) {
+        if (lock->fl_type != F_UNLCK) {
+            __inodelk_prune_stale(this, pl_inode, dom, lock);
+        } else if (priv->monkey_unlocking == _gf_true) {
+            if (pl_does_monkey_want_stuck_lock()) {
+                pthread_mutex_lock(&pl_inode->mutex);
+                {
+                    __pl_inodelk_unref(lock);
                 }
-out:
-		if (unref)
-			__pl_inodelk_unref (lock);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-	if (ctx)
-		pthread_mutex_unlock (&ctx->lock);
-
-        if (need_inode_unref)
-                inode_unref (pl_inode->inode);
-
-        /* The following (extra) unref corresponds to the ref that
-         * was done at the time the lock was granted.
-         */
-        if ((fl_type == F_UNLCK) && (ret == 0)) {
-                inode_unref (pl_inode->inode);
-                grant_blocked_inode_locks (this, pl_inode, dom);
+                pthread_mutex_unlock(&pl_inode->mutex);
+                inode_unref(pl_inode->inode);
+                gf_log(this->name, GF_LOG_WARNING,
+                       "MONKEY LOCKING (forcing stuck lock)!");
+                return 0;
+            }
         }
+    }
+
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    INIT_LIST_HEAD(&wake);
+
+    if (ctx)
+        pthread_mutex_lock(&ctx->lock);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (lock->fl_type != F_UNLCK) {
+            ret = __lock_inodelk(this, pl_inode, lock, can_block, dom, &now,
+                                 pcontend);
+            if (ret == 0) {
+                lock->frame = NULL;
+                gf_log(this->name, GF_LOG_TRACE,
+                       "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                       " => OK",
+                       lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                       lock->client_pid, lkowner_utoa(&lock->owner),
+                       lock->fl_start, lock->fl_end);
+            } else if (ret == -EAGAIN) {
+                gf_log(this->name, GF_LOG_TRACE,
+                       "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                       " => NOK",
+                       lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                       lock->client_pid, lkowner_utoa(&lock->owner),
+                       lock->user_flock.l_start, lock->user_flock.l_len);
+                if (can_block) {
+                    unref = _gf_false;
+                }
+            }
+            /* For all but the case where a non-blocking lock attempt fails
+             * with -EAGAIN, the extra ref taken at the start of this function
+             * must be negated. */
+            need_inode_unref = (ret != 0) && ((ret != -EAGAIN) || !can_block);
+            if (ctx && !need_inode_unref) {
+                list_add_tail(&lock->client_list, &ctx->inodelk_lockers);
+            }
+        } else {
+            /* Irrespective of whether unlock succeeds or not,
+             * the extra inode ref that was done at the start of
+             * this function must be negated. Towards this,
+             * @need_inode_unref flag is set unconditionally here.
+             */
+            need_inode_unref = _gf_true;
+            retlock = __inode_unlock_lock(this, lock, dom);
+            if (!retlock) {
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "Bad Unlock issued on Inode lock");
+                ret = -EINVAL;
+                goto out;
+            }
+            list_del_init(&retlock->client_list);
+            __pl_inodelk_unref(retlock);
 
-        return ret;
+            pl_inode_remove_unlocked(this, pl_inode, &wake);
+
+            ret = 0;
+        }
+    out:
+        if (unref)
+            __pl_inodelk_unref(lock);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    if (ctx)
+        pthread_mutex_unlock(&ctx->lock);
+
+    pl_inode_remove_wake(&wake);
+
+    /* The following (extra) unref corresponds to the ref that
+     * was done at the time the lock was granted.
+     */
+    if ((fl_type == F_UNLCK) && (ret == 0)) {
+        inode_unref(pl_inode->inode);
+        grant_blocked_inode_locks(this, pl_inode, dom, &now, pcontend);
+    }
+
+    if (need_inode_unref) {
+        inode_unref(pl_inode->inode);
+    }
+
+    if (pcontend != NULL) {
+        inodelk_contention_notify(this, pcontend);
+    }
+
+    return ret;
 }
 
 /* Create a new inode_lock_t */
-pl_inode_lock_t *
-new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
-                call_frame_t *frame, xlator_t *this, const char *volume,
-                char *conn_id)
+static pl_inode_lock_t *
+new_inode_lock(struct gf_flock *flock, client_t *client, pid_t client_pid,
+               call_frame_t *frame, xlator_t *this, const char *volume,
+               char *conn_id, int32_t *op_errno)
 
 {
-        pl_inode_lock_t *lock = NULL;
-
-        lock = GF_CALLOC (1, sizeof (*lock),
-                          gf_locks_mt_pl_inode_lock_t);
-        if (!lock) {
-                return NULL;
-        }
-
-        lock->fl_start = flock->l_start;
-        lock->fl_type  = flock->l_type;
-
-        if (flock->l_len == 0)
-                lock->fl_end = LLONG_MAX;
-        else
-                lock->fl_end = flock->l_start + flock->l_len - 1;
-
-        lock->client     = client;
-        lock->client_pid = client_pid;
-        lock->volume     = volume;
-        lock->owner      = frame->root->lk_owner;
-        lock->frame      = frame;
-        lock->this       = this;
-
-        if (conn_id) {
-                lock->connection_id = gf_strdup (conn_id);
-        }
+    pl_inode_lock_t *lock = NULL;
+
+    if (!pl_is_lk_owner_valid(&frame->root->lk_owner, frame->root->client)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    lock = GF_CALLOC(1, sizeof(*lock), gf_locks_mt_pl_inode_lock_t);
+    if (!lock) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    lock->fl_start = flock->l_start;
+    lock->fl_type = flock->l_type;
+
+    if (flock->l_len == 0)
+        lock->fl_end = LLONG_MAX;
+    else
+        lock->fl_end = flock->l_start + flock->l_len - 1;
+
+    lock->client = client;
+    lock->client_pid = client_pid;
+    lock->volume = volume;
+    lock->owner = frame->root->lk_owner;
+    lock->frame = frame;
+    lock->this = this;
+
+    if (conn_id) {
+        lock->connection_id = gf_strdup(conn_id);
+    }
+
+    INIT_LIST_HEAD(&lock->list);
+    INIT_LIST_HEAD(&lock->blocked_locks);
+    INIT_LIST_HEAD(&lock->client_list);
+    INIT_LIST_HEAD(&lock->contend);
+    __pl_inodelk_ref(lock);
 
-        INIT_LIST_HEAD (&lock->list);
-        INIT_LIST_HEAD (&lock->blocked_locks);
-	INIT_LIST_HEAD (&lock->client_list);
-        __pl_inodelk_ref (lock);
-
-        return lock;
+out:
+    return lock;
 }
 
 int32_t
-_pl_convert_volume (const char *volume, char **res)
+_pl_convert_volume(const char *volume, char **res)
 {
-        char    *mdata_vol = NULL;
-        int     ret = 0;
-
-        mdata_vol = strrchr (volume, ':');
-        //if the volume already ends with :metadata don't bother
-        if (mdata_vol && (strcmp (mdata_vol, ":metadata") == 0))
-                return 0;
+    char *mdata_vol = NULL;
+    int ret = 0;
 
-        ret = gf_asprintf (res, "%s:metadata", volume);
-        if (ret <= 0)
-                return ENOMEM;
+    mdata_vol = strrchr(volume, ':');
+    // if the volume already ends with :metadata don't bother
+    if (mdata_vol && (strcmp(mdata_vol, ":metadata") == 0))
         return 0;
+
+    ret = gf_asprintf(res, "%s:metadata", volume);
+    if (ret <= 0)
+        return ENOMEM;
+    return 0;
 }
 
 int32_t
-_pl_convert_volume_for_special_range (struct gf_flock *flock,
-                                      const char *volume, char **res)
+_pl_convert_volume_for_special_range(struct gf_flock *flock, const char *volume,
+                                     char **res)
 {
-        int32_t     ret = 0;
+    int32_t ret = 0;
 
-        if ((flock->l_start == LLONG_MAX -1) &&
-            (flock->l_len == 0)) {
-                ret = _pl_convert_volume (volume, res);
-        }
+    if ((flock->l_start == LLONG_MAX - 1) && (flock->l_len == 0)) {
+        ret = _pl_convert_volume(volume, res);
+    }
 
-        return ret;
+    return ret;
 }
 
 /* Common inodelk code called from pl_inodelk and pl_finodelk */
 int
-pl_common_inodelk (call_frame_t *frame, xlator_t *this,
-                   const char *volume, inode_t *inode, int32_t cmd,
-                   struct gf_flock *flock, loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        int32_t           op_ret     = -1;
-        int32_t           op_errno   = 0;
-        int               ret        = -1;
-        GF_UNUSED int     dict_ret   = -1;
-        int               can_block  = 0;
-        pl_inode_t *      pinode     = NULL;
-        pl_inode_lock_t * reqlock    = NULL;
-        pl_dom_list_t *   dom        = NULL;
-        char             *res        = NULL;
-        char             *res1       = NULL;
-        char             *conn_id    = NULL;
-        pl_ctx_t         *ctx        = NULL;
-
-        if (xdata)
-                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (inode, unwind);
-        VALIDATE_OR_GOTO (flock, unwind);
-
-        if ((flock->l_start < 0) || (flock->l_len < 0)) {
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        op_errno = _pl_convert_volume_for_special_range (flock, volume, &res);
-        if (op_errno)
-                goto unwind;
-        if (res)
-                volume = res;
-
-        pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
-
-	if (frame->root->client) {
-		ctx = pl_ctx_get (frame->root->client, this);
-		if (!ctx) {
-			op_errno = ENOMEM;
-			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
-			goto unwind;
-		}
-	}
-
-        pinode = pl_inode_get (this, inode);
-        if (!pinode) {
-                op_errno = ENOMEM;
-                goto unwind;
+pl_common_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  inode_t *inode, int32_t cmd, struct gf_flock *flock,
+                  loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    GF_UNUSED int dict_ret = -1;
+    int can_block = 0;
+    short lock_type = 0;
+    pl_inode_t *pinode = NULL;
+    pl_inode_lock_t *reqlock = NULL;
+    pl_dom_list_t *dom = NULL;
+    char *res = NULL;
+    char *res1 = NULL;
+    char *conn_id = NULL;
+    pl_ctx_t *ctx = NULL;
+
+    if (xdata)
+        dict_ret = dict_get_str(xdata, "connection-id", &conn_id);
+
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(inode, unwind);
+    VALIDATE_OR_GOTO(flock, unwind);
+
+    if ((flock->l_start < 0) || (flock->l_len < 0)) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    op_errno = _pl_convert_volume_for_special_range(flock, volume, &res);
+    if (op_errno)
+        goto unwind;
+    if (res)
+        volume = res;
+
+    pl_trace_in(this, frame, fd, loc, cmd, flock, volume);
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            op_errno = ENOMEM;
+            gf_log(this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+            goto unwind;
         }
+    }
 
-        dom = get_domain (pinode, volume);
-        if (!dom) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    pinode = pl_inode_get(this, inode, NULL);
+    if (!pinode) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        reqlock = new_inode_lock (flock, frame->root->client, frame->root->pid,
-                                  frame, this, dom->domain, conn_id);
+    dom = get_domain(pinode, volume);
+    if (!dom) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        if (!reqlock) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    reqlock = new_inode_lock(flock, frame->root->client, frame->root->pid,
+                             frame, this, dom->domain, conn_id, &op_errno);
 
+    if (!reqlock) {
+        op_ret = -1;
+        goto unwind;
+    }
 
-        switch (cmd) {
+    switch (cmd) {
         case F_SETLKW:
-                can_block = 1;
+            can_block = 1;
 
-                /* fall through */
+            /* fall through */
 
         case F_SETLK:
-                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
-                ret = pl_inode_setlk (this, ctx, pinode, reqlock, can_block,
-				      dom, inode);
-
-                if (ret < 0) {
-                        if ((can_block) && (F_UNLCK != flock->l_type)) {
-                                pl_trace_block (this, frame, fd, loc,
-                                                cmd, flock, volume);
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_TRACE, "returning EAGAIN");
-                        op_errno = -ret;
-                        goto unwind;
+            lock_type = flock->l_type;
+            memcpy(&reqlock->user_flock, flock, sizeof(struct gf_flock));
+            ret = pl_inode_setlk(this, ctx, pinode, reqlock, can_block, dom,
+                                 inode);
+
+            if (ret < 0) {
+                if (ret == -EAGAIN) {
+                    if (can_block && (F_UNLCK != lock_type)) {
+                        goto out;
+                    }
+                    gf_log(this->name, GF_LOG_TRACE, "returning EAGAIN");
+                } else {
+                    gf_log(this->name, GF_LOG_TRACE, "returning %d", ret);
                 }
-                break;
+                op_errno = -ret;
+                goto unwind;
+            }
+            break;
 
         default:
-                op_errno = ENOTSUP;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock command F_GETLK not supported for [f]inodelk "
-                        "(cmd=%d)",
-                        cmd);
-                goto unwind;
-        }
+            op_errno = ENOTSUP;
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock command F_GETLK not supported for [f]inodelk "
+                   "(cmd=%d)",
+                   cmd);
+            goto unwind;
+    }
 
-        op_ret = 0;
+    op_ret = 0;
 
 unwind:
-        if (flock != NULL)
-                pl_trace_out (this, frame, fd, loc, cmd, flock, op_ret,
-                              op_errno, volume);
+    if (flock != NULL)
+        pl_trace_out(this, frame, fd, loc, cmd, flock, op_ret, op_errno,
+                     volume);
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(inodelk, frame, op_ret, op_errno, NULL);
 out:
-        GF_FREE (res);
-        GF_FREE (res1);
-        return 0;
+    GF_FREE(res);
+    GF_FREE(res1);
+    return 0;
 }
 
 int
-pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
-            dict_t *xdata)
+pl_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock,
-                           loc, NULL, xdata);
+    pl_common_inodelk(frame, this, volume, loc->inode, cmd, flock, loc, NULL,
+                      xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
-             dict_t *xdata)
+pl_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock,
-                           NULL, fd, xdata);
-
-        return 0;
+    pl_common_inodelk(frame, this, volume, fd->inode, cmd, flock, NULL, fd,
+                      xdata);
 
+    return 0;
 }
 
 static int32_t
-__get_inodelk_dom_count (pl_dom_list_t *dom)
+__get_inodelk_dom_count(pl_dom_list_t *dom)
 {
-        pl_inode_lock_t     *lock   = NULL;
-        int32_t             count   = 0;
-
-        list_for_each_entry (lock, &dom->inodelk_list, list) {
-                count++;
-        }
-        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
-                count++;
-        }
-        return count;
+    pl_inode_lock_t *lock = NULL;
+    int32_t count = 0;
+
+    list_for_each_entry(lock, &dom->inodelk_list, list) { count++; }
+    list_for_each_entry(lock, &dom->blocked_inodelks, blocked_locks)
+    {
+        count++;
+    }
+    return count;
 }
 
 /* Returns the no. of locks (blocked/granted) held on a given domain name
  * If @domname is NULL, returns the no. of locks in all the domains present.
  * If @domname is non-NULL and non-existent, returns 0 */
 int32_t
-__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname)
+__get_inodelk_count(xlator_t *this, pl_inode_t *pl_inode, char *domname)
 {
-        int32_t            count  = 0;
-        pl_dom_list_t     *dom    = NULL;
-
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                if (domname) {
-                        if (strcmp (domname, dom->domain) == 0) {
-                                count = __get_inodelk_dom_count (dom);
-                                goto out;
-                        }
-
-                } else {
-                    /* Counting locks from all domains */
-                        count += __get_inodelk_dom_count (dom);
+    int32_t count = 0;
+    pl_dom_list_t *dom = NULL;
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        if (domname) {
+            if (strcmp(domname, dom->domain) == 0) {
+                count = __get_inodelk_dom_count(dom);
+                goto out;
+            }
 
-                }
+        } else {
+            /* Counting locks from all domains */
+            count += __get_inodelk_dom_count(dom);
         }
+    }
 
 out:
-        return count;
+    return count;
 }
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode, char *domname)
+get_inodelk_count(xlator_t *this, inode_t *inode, char *domname)
 {
-        pl_inode_t   *pl_inode = NULL;
-        uint64_t      tmp_pl_inode = 0;
-        int           ret      = 0;
-        int32_t       count    = 0;
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int ret = 0;
+    int32_t count = 0;
 
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
-        if (ret != 0) {
-                goto out;
-        }
+    ret = inode_ctx_get(inode, this, &tmp_pl_inode);
+    if (ret != 0) {
+        goto out;
+    }
 
-        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                count = __get_inodelk_count (this, pl_inode, domname);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        count = __get_inodelk_count(this, pl_inode, domname);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
 out:
-        return count;
+    return count;
 }
diff --git a/xlators/features/locks/src/locks-mem-types.h b/xlators/features/locks/src/locks-mem-types.h
index a48b35c2044..a76605027b3 100644
--- a/xlators/features/locks/src/locks-mem-types.h
+++ b/xlators/features/locks/src/locks-mem-types.h
@@ -11,20 +11,18 @@
 #ifndef __LOCKS_MEM_TYPES_H__
 #define __LOCKS_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_locks_mem_types_ {
-        gf_locks_mt_pl_dom_list_t = gf_common_mt_end + 1,
-        gf_locks_mt_pl_inode_t,
-        gf_locks_mt_posix_lock_t,
-        gf_locks_mt_pl_entry_lock_t,
-        gf_locks_mt_pl_inode_lock_t,
-        gf_locks_mt_truncate_ops,
-        gf_locks_mt_pl_rw_req_t,
-        gf_locks_mt_posix_locks_private_t,
-        gf_locks_mt_pl_fdctx_t,
-        gf_locks_mt_pl_meta_lock_t,
-        gf_locks_mt_end
+    gf_locks_mt_pl_dom_list_t = gf_common_mt_end + 1,
+    gf_locks_mt_pl_inode_t,
+    gf_locks_mt_posix_lock_t,
+    gf_locks_mt_pl_entry_lock_t,
+    gf_locks_mt_pl_inode_lock_t,
+    gf_locks_mt_pl_rw_req_t,
+    gf_locks_mt_posix_locks_private_t,
+    gf_locks_mt_pl_fdctx_t,
+    gf_locks_mt_pl_meta_lock_t,
+    gf_locks_mt_end
 };
 #endif
-
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index e363f425b65..c868eb494a2 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -10,231 +10,283 @@
 #ifndef __POSIX_LOCKS_H__
 #define __POSIX_LOCKS_H__
 
-#include "compat-errno.h"
-#include "stack.h"
-#include "call-stub.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/call-stub.h>
 #include "locks-mem-types.h"
-#include "client_t.h"
+#include <glusterfs/client_t.h>
 
-#include "lkowner.h"
+#include <glusterfs/lkowner.h>
 
 typedef enum {
-        MLK_NONE,
-        MLK_FILE_BASED,
-        MLK_FORCED,
-        MLK_OPTIMAL
+    MLK_NONE,
+    MLK_FILE_BASED,
+    MLK_FORCED,
+    MLK_OPTIMAL
 } mlk_mode_t; /* defines different mandatory locking modes*/
 
 struct __pl_fd;
 
 struct __posix_lock {
-        struct list_head   list;
+    struct list_head list;
 
-        short              fl_type;
-        off_t              fl_start;
-        off_t              fl_end;
-        uint32_t           lk_flags;
+    off_t fl_start;
+    off_t fl_end;
+    uint32_t lk_flags;
 
-        short              blocked;    /* waiting to acquire */
-        struct gf_flock    user_flock; /* the flock supplied by the user */
-        xlator_t          *this;       /* required for blocked locks */
-        unsigned long      fd_num;
+    short fl_type;
+    short blocked;              /* waiting to acquire */
+    struct gf_flock user_flock; /* the flock supplied by the user */
+    xlator_t *this;             /* required for blocked locks */
+    unsigned long fd_num;
 
-        fd_t              *fd;
-        call_frame_t      *frame;
+    fd_t *fd;
+    call_frame_t *frame;
 
-        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
-        struct timeval     granted_time; /*time at which lock was queued into active list*/
+    time_t blkd_time;    /* time at which lock was queued into blkd list */
+    time_t granted_time; /* time at which lock was queued into active list */
 
-        /* These two together serve to uniquely identify each process
-           across nodes */
+    /* These two together serve to uniquely identify each process
+       across nodes */
 
-        void              *client;     /* to identify client node */
+    void *client; /* to identify client node */
 
-        /* This field uniquely identifies the client the lock belongs to.  As
-         * lock migration is handled by rebalance, the client_t object will be
-         * overwritten by rebalance and can't be deemed as the owner of the
-         * lock on destination. Hence, the below field is migrated from
-         * source to destination by lock_migration_info_t and updated on the
-         * destination. So that on client-server disconnection, server can
-         * cleanup the locks proper;y.  */
+    /* This field uniquely identifies the client the lock belongs to.  As
+     * lock migration is handled by rebalance, the client_t object will be
+     * overwritten by rebalance and can't be deemed as the owner of the
+     * lock on destination. Hence, the below field is migrated from
+     * source to destination by lock_migration_info_t and updated on the
+     * destination. So that on client-server disconnection, server can
+     * cleanup the locks proper;y.  */
 
-        char              *client_uid;
-        gf_lkowner_t       owner;
-        pid_t              client_pid;    /* pid of client process */
+    char *client_uid;
+    gf_lkowner_t owner;
+    pid_t client_pid; /* pid of client process */
 
-        int                blocking;
+    int blocking;
 };
 typedef struct __posix_lock posix_lock_t;
 
 struct __pl_inode_lock {
-        struct list_head   list;
-        struct list_head   blocked_locks; /* list_head pointing to blocked_inodelks */
-        int                ref;
+    struct list_head list;
+    struct list_head blocked_locks; /* list_head pointing to blocked_inodelks */
+    struct list_head contend;       /* list of contending locks */
+    int ref;
 
-        short              fl_type;
-        off_t              fl_start;
-        off_t              fl_end;
+    off_t fl_start;
+    off_t fl_end;
 
-        const char        *volume;
+    const char *volume;
 
-        struct gf_flock    user_flock; /* the flock supplied by the user */
-        xlator_t          *this;       /* required for blocked locks */
-	struct __pl_inode *pl_inode;
+    struct gf_flock user_flock; /* the flock supplied by the user */
+    xlator_t *this;             /* required for blocked locks */
+    struct __pl_inode *pl_inode;
 
-        call_frame_t      *frame;
+    call_frame_t *frame;
 
-        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
-        struct timeval     granted_time; /*time at which lock was queued into active list*/
+    time_t blkd_time;    /* time at which lock was queued into blkd list */
+    time_t granted_time; /* time at which lock was queued into active list */
 
-        /* These two together serve to uniquely identify each process
-           across nodes */
+    /*last time at which lock contention was detected and notified*/
+    struct timespec contention_time;
 
-        void              *client;     /* to identify client node */
-        gf_lkowner_t       owner;
-        pid_t              client_pid;    /* pid of client process */
+    /* These two together serve to uniquely identify each process
+       across nodes */
 
-        char              *connection_id; /* stores the client connection id */
+    void *client; /* to identify client node */
+    gf_lkowner_t owner;
+    pid_t client_pid; /* pid of client process */
 
-	struct list_head   client_list; /* list of all locks from a client */
+    char *connection_id; /* stores the client connection id */
+
+    struct list_head client_list; /* list of all locks from a client */
+    short fl_type;
+
+    int32_t status; /* Error code when we try to grant a lock in blocked
+                       state */
 };
 typedef struct __pl_inode_lock pl_inode_lock_t;
 
-struct __pl_rw_req_t {
-        struct list_head      list;
-        call_stub_t          *stub;
-        posix_lock_t          region;
+struct _pl_rw_req {
+    struct list_head list;
+    call_stub_t *stub;
+    posix_lock_t region;
 };
-typedef struct __pl_rw_req_t pl_rw_req_t;
-
-struct __pl_dom_list_t {
-        struct list_head   inode_list;       /* list_head back to pl_inode_t */
-        const char        *domain;
-        struct list_head   entrylk_list;     /* List of entry locks */
-        struct list_head   blocked_entrylks; /* List of all blocked entrylks */
-        struct list_head   inodelk_list;     /* List of inode locks */
-        struct list_head   blocked_inodelks; /* List of all blocked inodelks */
+typedef struct _pl_rw_req pl_rw_req_t;
+
+struct _pl_dom_list {
+    struct list_head inode_list; /* list_head back to pl_inode_t */
+    const char *domain;
+    struct list_head entrylk_list;     /* List of entry locks */
+    struct list_head blocked_entrylks; /* List of all blocked entrylks */
+    struct list_head inodelk_list;     /* List of inode locks */
+    struct list_head blocked_inodelks; /* List of all blocked inodelks */
 };
-typedef struct __pl_dom_list_t pl_dom_list_t;
+typedef struct _pl_dom_list pl_dom_list_t;
 
 struct __entry_lock {
-        struct list_head  domain_list;    /* list_head back to pl_dom_list_t */
-        struct list_head  blocked_locks; /* list_head back to blocked_entrylks */
-	int ref;
+    struct list_head domain_list;   /* list_head back to pl_dom_list_t */
+    struct list_head blocked_locks; /* list_head back to blocked_entrylks */
+    struct list_head contend;       /* list of contending locks */
+    int ref;
 
-        call_frame_t     *frame;
-        xlator_t         *this;
-	struct __pl_inode *pinode;
+    call_frame_t *frame;
+    xlator_t *this;
+    struct __pl_inode *pinode;
 
-        const char       *volume;
+    const char *volume;
 
-        const char       *basename;
-        entrylk_type      type;
+    const char *basename;
 
-        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
-        struct timeval     granted_time; /*time at which lock was queued into active list*/
+    time_t blkd_time;    /* time at which lock was queued into blkd list */
+    time_t granted_time; /* time at which lock was queued into active list */
 
-        void             *client;
-        gf_lkowner_t      owner;
-	pid_t             client_pid;    /* pid of client process */
+    /*last time at which lock contention was detected and notified*/
+    struct timespec contention_time;
 
-        char             *connection_id; /* stores the client connection id */
+    void *client;
+    gf_lkowner_t owner;
+    pid_t client_pid; /* pid of client process */
 
-	struct list_head   client_list; /* list of all locks from a client */
+    char *connection_id; /* stores the client connection id */
+
+    struct list_head client_list; /* list of all locks from a client */
+    entrylk_type type;
 };
 typedef struct __entry_lock pl_entry_lock_t;
 
-
 /* The "simulated" inode. This contains a list of all the locks associated
    with this file */
 
 struct __pl_inode {
-        pthread_mutex_t  mutex;
-
-        struct list_head dom_list;       /* list of domains */
-        struct list_head ext_list;       /* list of fcntl locks */
-        struct list_head rw_list;        /* list of waiting r/w requests */
-        struct list_head reservelk_list;        /* list of reservelks */
-        struct list_head blocked_reservelks;        /* list of blocked reservelks */
-        struct list_head blocked_calls;  /* List of blocked lock calls while a reserve is held*/
-        struct list_head metalk_list;    /* Meta lock list */
-         /* This is to store the incoming lock
-            requests while meta lock is enabled */
-        struct list_head queued_locks;
-        int              mandatory;      /* if mandatory locking is enabled */
-
-        inode_t          *refkeeper;     /* hold refs on an inode while locks are
-                                            held to prevent pruning */
-        uuid_t            gfid;          /* placeholder for gfid of the inode */
-        inode_t          *inode;         /* pointer to be used for ref and unref
-                                            of inode_t as long as there are
-                                            locks on it */
-        gf_boolean_t     migrated;
+    pthread_mutex_t mutex;
+
+    struct list_head dom_list;           /* list of domains */
+    struct list_head ext_list;           /* list of fcntl locks */
+    struct list_head rw_list;            /* list of waiting r/w requests */
+    struct list_head reservelk_list;     /* list of reservelks */
+    struct list_head blocked_reservelks; /* list of blocked reservelks */
+    struct list_head blocked_calls;      /* List of blocked lock calls while a
+                                            reserve is held*/
+    struct list_head metalk_list;        /* Meta lock list */
+    struct list_head queued_locks;       /* This is to store the incoming lock
+                                            requests while meta lock is enabled */
+    struct list_head waiting; /* List of pending fops waiting to unlink/rmdir
+                                 the inode. */
+    int mandatory;            /* if mandatory locking is enabled */
+
+    inode_t *refkeeper; /* hold refs on an inode while locks are
+                           held to prevent pruning */
+    uuid_t gfid;        /* placeholder for gfid of the inode */
+    inode_t *inode;     /* pointer to be used for ref and unref
+                           of inode_t as long as there are
+                           locks on it */
+    gf_boolean_t migrated;
+
+    /* Flag to indicate whether to read mlock-enforce xattr from disk */
+    gf_boolean_t check_mlock_info;
+
+    /* Mandatory_lock enforce: IO will be allowed if and only if the lkowner has
+       held the lock.
+
+       Note: An xattr is set on the file to recover this information post
+       reboot. If client does not want mandatory lock to be enforced, then it
+       should remove this xattr explicitly
+    */
+    gf_boolean_t mlock_enforced;
+    /* There are scenarios where mandatory lock is granted but there are IOs
+       pending at posix level. To avoid this before preempting the previous lock
+       owner, we wait for all the fops to be unwound.
+    */
+    int fop_wind_count;
+    pthread_cond_t check_fop_wind_count;
+
+    gf_boolean_t track_fop_wind_count;
+
+    int32_t links;           /* Number of hard links the inode has. */
+    uint32_t remove_running; /* Number of remove operations running. */
+    gf_boolean_t is_locked;  /* Regular locks will be blocked. */
+    gf_boolean_t removed;    /* The inode has been deleted. */
 };
 typedef struct __pl_inode pl_inode_t;
 
 struct __pl_metalk {
-        pthread_mutex_t mutex;
-        /* For pl_inode meta lock list */
-        struct list_head        list;
-        /* For pl_ctx_t list */
-        struct list_head        client_list;
-        char                   *client_uid;
-
-        pl_inode_t             *pl_inode;
-        int                     ref;
+    pthread_mutex_t mutex;
+    /* For pl_inode meta lock list */
+    struct list_head list;
+    /* For pl_ctx_t list */
+    struct list_head client_list;
+    char *client_uid;
+
+    pl_inode_t *pl_inode;
+    int ref;
 };
 typedef struct __pl_metalk pl_meta_lock_t;
 
 typedef struct {
-        mlk_mode_t      mandatory_mode; /* holds current mandatory locking mode */
-        gf_boolean_t    trace;          /* trace lock requests in and out */
-        char           *brickname;
+    char *brickname;
+    uint32_t revocation_secs;
+    uint32_t revocation_max_blocked;
+    uint32_t notify_contention_delay;
+    mlk_mode_t mandatory_mode; /* holds current mandatory locking mode */
+    gf_boolean_t trace;        /* trace lock requests in and out */
+    gf_boolean_t monkey_unlocking;
+    gf_boolean_t revocation_clear_all;
+    gf_boolean_t notify_contention;
+    gf_boolean_t mlock_enforced;
 } posix_locks_private_t;
 
-
 typedef struct {
-        gf_boolean_t   entrylk_count_req;
-        gf_boolean_t   inodelk_count_req;
-        gf_boolean_t   posixlk_count_req;
-        gf_boolean_t   parent_entrylk_req;
-        data_t        *inodelk_dom_count_req;
-
-        dict_t  *xdata;
-        loc_t  loc[2];
-        fd_t  *fd;
-        off_t  offset;
-        glusterfs_fop_t op;
+    data_t *inodelk_dom_count_req;
+
+    dict_t *xdata;
+    loc_t loc[2];
+    fd_t *fd;
+    inode_t *inode;
+    off_t offset;
+    glusterfs_fop_t op;
+    gf_boolean_t entrylk_count_req;
+    gf_boolean_t inodelk_count_req;
+    gf_boolean_t posixlk_count_req;
+    gf_boolean_t parent_entrylk_req;
+    gf_boolean_t multiple_dom_lk_requests;
+    int update_mlock_enforced_flag;
 } pl_local_t;
 
-
 typedef struct {
-        struct list_head locks_list;
+    struct list_head locks_list;
 } pl_fdctx_t;
 
-
 struct _locker {
-        struct list_head  lockers;
-        char             *volume;
-        inode_t          *inode;
-        gf_lkowner_t      owner;
+    struct list_head lockers;
+    char *volume;
+    inode_t *inode;
+    gf_lkowner_t owner;
 };
 
 typedef struct _locks_ctx {
-        pthread_mutex_t      lock;
-        struct list_head     inodelk_lockers;
-        struct list_head     entrylk_lockers;
-        struct list_head     metalk_list;
+    pthread_mutex_t lock;
+    struct list_head inodelk_lockers;
+    struct list_head entrylk_lockers;
+    struct list_head metalk_list;
 } pl_ctx_t;
 
+typedef struct _multi_dom_lk_data {
+    xlator_t *this;
+    inode_t *inode;
+    dict_t *xdata_rsp;
+    gf_boolean_t keep_max;
+} multi_dom_lk_data;
+
+typedef enum { DECREMENT, INCREMENT } pl_count_op_t;
 
 pl_ctx_t *
-pl_ctx_get (client_t *client, xlator_t *xlator);
+pl_ctx_get(client_t *client, xlator_t *xlator);
 
 int
-pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+pl_inodelk_client_cleanup(xlator_t *this, pl_ctx_t *ctx);
 
 int
-pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+pl_entrylk_client_cleanup(xlator_t *this, pl_ctx_t *ctx);
 
 #endif /* __POSIX_LOCKS_H__ */
diff --git a/xlators/features/locks/src/pl-messages.h b/xlators/features/locks/src/pl-messages.h
index 45c8873ecb4..e2d3d7ca974 100644
--- a/xlators/features/locks/src/pl-messages.h
+++ b/xlators/features/locks/src/pl-messages.h
@@ -11,54 +11,19 @@
 #ifndef _PL_MESSAGES_H_
 #define _PL_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs-message-id.h"
-
-/*! \file pl-messages.h
- *  \brief Locks log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_PL_COMP_BASE       GLFS_MSGID_COMP_PL
-#define GLFS_NUM_MESSAGES       1
-#define GLFS_MSGID_END          (GLFS_PL_COMP_BASE + GLFS_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x GLFS_PL_COMP_BASE, "Invalid: Start of messages"
-/*------------*/
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define PL_MSG_LOCK_NUMBER           (GLFS_PL_COMP_BASE + 1)
 
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+GLFS_MSGID(PL, PL_MSG_LOCK_NUMBER, PL_MSG_INODELK_CONTENTION_FAILED,
+           PL_MSG_ENTRYLK_CONTENTION_FAILED);
 
 #endif /* !_PL_MESSAGES_H_ */
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index dff17e70aaf..cf0ae4c57dd 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -12,362 +12,554 @@
 #include <limits.h>
 #include <pthread.h>
 
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
 
 #include "locks.h"
 #include "common.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "clear.h"
-#include "defaults.h"
-#include "syncop.h"
-#include "pl-messages.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
 
 #ifndef LLONG_MAX
 #define LLONG_MAX LONG_LONG_MAX /* compat with old gcc */
-#endif /* LLONG_MAX */
+#endif                          /* LLONG_MAX */
 
 /* Forward declarations */
 
-void do_blocked_rw (pl_inode_t *);
-static int __rw_allowable (pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
-static int format_brickname(char *);
-int pl_lockinfo_get_brickname (xlator_t *, inode_t *, int32_t *);
-static int fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
-
-#define PL_STACK_UNWIND(fop, xdata, frame, op_ret, params ...)          \
-        do {                                                            \
-                pl_local_t *__local = NULL;                             \
-                inode_t *__parent = NULL;                               \
-                inode_t *__inode = NULL;                                \
-                char    *__name = NULL;                                 \
-                dict_t  *__unref = NULL;                                \
-                int __i = 0 ;                                           \
-                __local = frame->local;                                 \
-                if (op_ret >= 0 && pl_needs_xdata_response (frame->local)) {\
-                        if (xdata)                                      \
-                                dict_ref (xdata);                       \
-                        else                                            \
-                                xdata = dict_new();                     \
-                        if (xdata) {                                    \
-                                __unref = xdata;                        \
-                                while (__local->fd || __local->loc[__i].inode) {  \
-                                        pl_get_xdata_rsp_args (__local,            \
-                                              #fop, &__parent, &__inode,           \
-                                              &__name, __i);                       \
-                                        pl_set_xdata_response (frame->this,        \
-                                               __local, __parent, __inode, __name, \
-                                               xdata, __i > 0);                    \
-                                        if (__local->fd || __i == 1)               \
-                                                break;                  \
-                                        __i++;                          \
-                                }                                       \
-                        }                                               \
-                }                                                       \
-                frame->local = NULL;                                    \
-                STACK_UNWIND_STRICT (fop, frame, op_ret, params);       \
-                if (__local) {                                          \
-                        if (__local->inodelk_dom_count_req)             \
-                                data_unref (__local->inodelk_dom_count_req);\
-                        loc_wipe (&__local->loc[0]);                    \
-                        loc_wipe (&__local->loc[1]);                    \
-                        if (__local->fd)                                \
-                                fd_unref (__local->fd);                 \
-                        mem_put (__local);                              \
-                }                                                       \
-                if (__unref)                                            \
-                        dict_unref (__unref);                           \
-        } while (0)
-
-#define PL_LOCAL_GET_REQUESTS(frame, this, xdata, __fd, __loc, __newloc)\
-        do {                                                            \
-                if (pl_has_xdata_requests (xdata)) {                    \
-                        frame->local = mem_get0 (this->local_pool);     \
-                        pl_local_t *__local = frame->local;             \
-                        if (__local) {                                  \
-                                if (__fd) {                             \
-                                        __local->fd = fd_ref (__fd);    \
-                                } else {                                \
-                                        if (__loc)                      \
-                                                loc_copy (&__local->loc[0],\
-                                                           __loc);      \
-                                        if (__newloc)                   \
-                                                 loc_copy (&__local->loc[1],\
-                                                            __newloc);  \
-                                }                                       \
-                                pl_get_xdata_requests (__local, xdata); \
-                        }                                               \
-                }                                                       \
-        } while (0)
+void
+do_blocked_rw(pl_inode_t *);
+static int
+__rw_allowable(pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
+static int
+format_brickname(char *);
+int
+pl_lockinfo_get_brickname(xlator_t *, inode_t *, int32_t *);
+static int
+fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
+
+/*
+ * The client is always requesting data, but older
+ * servers were not returning it. Newer ones are, so
+ * the client is receiving a mix of NULL and non-NULL
+ * xdata in the answers when bricks are of different
+ * versions. This triggers a bug in older clients.
+ * To prevent that, we avoid returning extra xdata to
+ * older clients (making the newer brick to behave as
+ * an old brick).
+ */
+#define PL_STACK_UNWIND_FOR_CLIENT(fop, xdata, frame, op_ret, params...)       \
+    do {                                                                       \
+        pl_local_t *__local = NULL;                                            \
+        if (frame->root->client &&                                             \
+            (frame->root->client->opversion < GD_OP_VERSION_3_10_0)) {         \
+            __local = frame->local;                                            \
+            PL_STACK_UNWIND_AND_FREE(__local, fop, frame, op_ret, params);     \
+        } else {                                                               \
+            PL_STACK_UNWIND(fop, xdata, frame, op_ret, params);                \
+        }                                                                      \
+    } while (0)
+
+#define PL_STACK_UNWIND(fop, xdata, frame, op_ret, params...)                  \
+    do {                                                                       \
+        pl_local_t *__local = NULL;                                            \
+        inode_t *__parent = NULL;                                              \
+        inode_t *__inode = NULL;                                               \
+        char *__name = NULL;                                                   \
+        dict_t *__unref = NULL;                                                \
+        int __i = 0;                                                           \
+        __local = frame->local;                                                \
+        if (op_ret >= 0 && pl_needs_xdata_response(frame->local)) {            \
+            if (xdata)                                                         \
+                dict_ref(xdata);                                               \
+            else                                                               \
+                xdata = dict_new();                                            \
+            if (xdata) {                                                       \
+                __unref = xdata;                                               \
+                while (__local->fd || __local->loc[__i].inode) {               \
+                    pl_get_xdata_rsp_args(__local, #fop, &__parent, &__inode,  \
+                                          &__name, __i);                       \
+                    pl_set_xdata_response(frame->this, __local, __parent,      \
+                                          __inode, __name, xdata, __i > 0);    \
+                    if (__local->fd || __i == 1)                               \
+                        break;                                                 \
+                    __i++;                                                     \
+                }                                                              \
+            }                                                                  \
+        }                                                                      \
+        PL_STACK_UNWIND_AND_FREE(__local, fop, frame, op_ret, params);         \
+        if (__unref)                                                           \
+            dict_unref(__unref);                                               \
+    } while (0)
+
+#define PL_LOCAL_GET_REQUESTS(frame, this, xdata, __fd, __loc, __newloc)       \
+    do {                                                                       \
+        if (pl_has_xdata_requests(xdata)) {                                    \
+            if (!frame->local)                                                 \
+                frame->local = mem_get0(this->local_pool);                     \
+            pl_local_t *__local = frame->local;                                \
+            if (__local) {                                                     \
+                if (__fd) {                                                    \
+                    __local->fd = fd_ref(__fd);                                \
+                    __local->inode = inode_ref(__fd->inode);                   \
+                } else {                                                       \
+                    if (__loc)                                                 \
+                        loc_copy(&__local->loc[0], __loc);                     \
+                    if (__newloc)                                              \
+                        loc_copy(&__local->loc[1], __newloc);                  \
+                    __local->inode = inode_ref(__local->loc[0].inode);         \
+                }                                                              \
+                pl_get_xdata_requests(__local, xdata);                         \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define PL_CHECK_LOCK_ENFORCE_KEY(frame, dict, name, this, loc, fd, priv)      \
+    do {                                                                       \
+        if ((dict && (dict_get(dict, GF_ENFORCE_MANDATORY_LOCK))) ||           \
+            (name && (strcmp(name, GF_ENFORCE_MANDATORY_LOCK) == 0))) {        \
+            inode_t *__inode = (loc ? loc->inode : fd->inode);                 \
+            pl_inode_t *__pl_inode = pl_inode_get(this, __inode, NULL);        \
+            if (__pl_inode == NULL) {                                          \
+                op_ret = -1;                                                   \
+                op_errno = ENOMEM;                                             \
+                goto unwind;                                                   \
+            }                                                                  \
+            if (!pl_is_mandatory_locking_enabled(__pl_inode) ||                \
+                !priv->mlock_enforced) {                                       \
+                op_ret = -1;                                                   \
+                gf_msg(this->name, GF_LOG_DEBUG, EINVAL, 0,                    \
+                       "option %s would need mandatory lock to be enabled "    \
+                       "and feature.enforce-mandatory-lock option to be set "  \
+                       "to on",                                                \
+                       GF_ENFORCE_MANDATORY_LOCK);                             \
+                op_errno = EINVAL;                                             \
+                goto unwind;                                                   \
+            }                                                                  \
+                                                                               \
+            op_ret = pl_local_init(frame, this, loc, fd);                      \
+            if (op_ret) {                                                      \
+                op_errno = ENOMEM;                                             \
+                goto unwind;                                                   \
+            }                                                                  \
+                                                                               \
+            ((pl_local_t *)(frame->local))->update_mlock_enforced_flag = 1;    \
+        }                                                                      \
+    } while (0)
+
+#define PL_INODE_REMOVE(_fop, _frame, _xl, _loc1, _loc2, _cont, _cbk,          \
+                        _args...)                                              \
+    ({                                                                         \
+        struct list_head contend;                                              \
+        pl_inode_t *__pl_inode;                                                \
+        call_stub_t *__stub;                                                   \
+        int32_t __error;                                                       \
+        INIT_LIST_HEAD(&contend);                                              \
+        __error = pl_inode_remove_prepare(_xl, _frame, _loc2 ? _loc2 : _loc1,  \
+                                          &__pl_inode, &contend);              \
+        if (__error < 0) {                                                     \
+            __stub = fop_##_fop##_stub(_frame, _cont, ##_args);                \
+            __error = pl_inode_remove_complete(_xl, __pl_inode, __stub,        \
+                                               &contend);                      \
+        } else if (__error == 0) {                                             \
+            PL_LOCAL_GET_REQUESTS(_frame, _xl, xdata, ((fd_t *)NULL), _loc1,   \
+                                  _loc2);                                      \
+            STACK_WIND_COOKIE(_frame, _cbk, __pl_inode, FIRST_CHILD(_xl),      \
+                              FIRST_CHILD(_xl)->fops->_fop, ##_args);          \
+        }                                                                      \
+        __error;                                                               \
+    })
 
 gf_boolean_t
-pl_has_xdata_requests (dict_t *xdata)
+pl_has_xdata_requests(dict_t *xdata)
 {
-        char *reqs[] = {GLUSTERFS_ENTRYLK_COUNT, GLUSTERFS_INODELK_COUNT,
-                        GLUSTERFS_INODELK_DOM_COUNT, GLUSTERFS_POSIXLK_COUNT,
-                        GLUSTERFS_PARENT_ENTRYLK, NULL};
-        int i = 0;
+    static char *reqs[] = {GLUSTERFS_ENTRYLK_COUNT,
+                           GLUSTERFS_INODELK_COUNT,
+                           GLUSTERFS_INODELK_DOM_COUNT,
+                           GLUSTERFS_POSIXLK_COUNT,
+                           GLUSTERFS_PARENT_ENTRYLK,
+                           GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS,
+                           NULL};
+    static int reqs_size[] = {SLEN(GLUSTERFS_ENTRYLK_COUNT),
+                              SLEN(GLUSTERFS_INODELK_COUNT),
+                              SLEN(GLUSTERFS_INODELK_DOM_COUNT),
+                              SLEN(GLUSTERFS_POSIXLK_COUNT),
+                              SLEN(GLUSTERFS_PARENT_ENTRYLK),
+                              SLEN(GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS),
+                              0};
+    int i = 0;
+
+    if (!xdata)
+        return _gf_false;
 
-        if (!xdata)
-                return _gf_false;
+    for (i = 0; reqs[i]; i++)
+        if (dict_getn(xdata, reqs[i], reqs_size[i]))
+            return _gf_true;
 
-        for (i = 0; reqs[i]; i++)
-                if (dict_get (xdata, reqs[i]))
-                        return _gf_true;
+    return _gf_false;
+}
 
-        return _gf_false;
+static int
+dict_delete_domain_key(dict_t *dict, char *key, data_t *value, void *data)
+{
+    dict_del(dict, key);
+    return 0;
 }
 
 void
-pl_get_xdata_requests (pl_local_t *local, dict_t *xdata)
+pl_get_xdata_requests(pl_local_t *local, dict_t *xdata)
 {
-        if (!local || !xdata)
-                return;
-
-        if (dict_get (xdata, GLUSTERFS_ENTRYLK_COUNT)) {
-                local->entrylk_count_req = 1;
-                dict_del (xdata, GLUSTERFS_ENTRYLK_COUNT);
-        }
-        if (dict_get (xdata, GLUSTERFS_INODELK_COUNT)) {
-                local->inodelk_count_req = 1;
-                dict_del (xdata, GLUSTERFS_INODELK_COUNT);
-        }
-
-        local->inodelk_dom_count_req = dict_get (xdata, GLUSTERFS_INODELK_DOM_COUNT);
-        if (local->inodelk_dom_count_req) {
-                data_ref (local->inodelk_dom_count_req);
-                dict_del (xdata, GLUSTERFS_INODELK_DOM_COUNT);
-        }
-
-        if (dict_get (xdata, GLUSTERFS_POSIXLK_COUNT)) {
-                local->posixlk_count_req = 1;
-                dict_del (xdata, GLUSTERFS_POSIXLK_COUNT);
-        }
+    if (!local || !xdata)
+        return;
 
-        if (dict_get (xdata, GLUSTERFS_PARENT_ENTRYLK)) {
-                local->parent_entrylk_req = 1;
-                dict_del (xdata, GLUSTERFS_PARENT_ENTRYLK);
-        }
+    GF_ASSERT(local->xdata == NULL);
+    local->xdata = dict_copy_with_ref(xdata, NULL);
+
+    if (dict_get_sizen(xdata, GLUSTERFS_ENTRYLK_COUNT)) {
+        local->entrylk_count_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_ENTRYLK_COUNT);
+    }
+    if (dict_get_sizen(xdata, GLUSTERFS_INODELK_COUNT)) {
+        local->inodelk_count_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_INODELK_COUNT);
+    }
+    if (dict_get_sizen(xdata, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS)) {
+        local->multiple_dom_lk_requests = 1;
+        dict_del_sizen(xdata, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS);
+        dict_foreach_fnmatch(xdata, GLUSTERFS_INODELK_DOM_PREFIX "*",
+                             dict_delete_domain_key, NULL);
+    }
+
+    local->inodelk_dom_count_req = dict_get_sizen(xdata,
+                                                  GLUSTERFS_INODELK_DOM_COUNT);
+    if (local->inodelk_dom_count_req) {
+        data_ref(local->inodelk_dom_count_req);
+        dict_del_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT);
+    }
+
+    if (dict_get_sizen(xdata, GLUSTERFS_POSIXLK_COUNT)) {
+        local->posixlk_count_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_POSIXLK_COUNT);
+    }
+
+    if (dict_get_sizen(xdata, GLUSTERFS_PARENT_ENTRYLK)) {
+        local->parent_entrylk_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_PARENT_ENTRYLK);
+    }
 }
 
 gf_boolean_t
-pl_needs_xdata_response (pl_local_t *local)
+pl_needs_xdata_response(pl_local_t *local)
 {
-        if (!local)
-                return _gf_false;
-
-        if (local->parent_entrylk_req)
-                return _gf_true;
-
-        if (local->entrylk_count_req)
-                return _gf_true;
-
-        if (local->inodelk_dom_count_req)
-                return _gf_true;
+    if (!local)
+        return _gf_false;
 
-        if (local->inodelk_count_req)
-                return _gf_true;
+    if (local->parent_entrylk_req || local->entrylk_count_req ||
+        local->inodelk_dom_count_req || local->inodelk_count_req ||
+        local->posixlk_count_req || local->multiple_dom_lk_requests)
+        return _gf_true;
 
-        if (local->posixlk_count_req)
-                return _gf_true;
-        return _gf_false;
+    return _gf_false;
 }
 
 void
-pl_get_xdata_rsp_args (pl_local_t *local, char *fop, inode_t **parent,
-                       inode_t **inode, char **name, int i)
+pl_get_xdata_rsp_args(pl_local_t *local, char *fop, inode_t **parent,
+                      inode_t **inode, char **name, int i)
 {
-        if (strcmp (fop, "lookup") == 0) {
-                *parent = local->loc[0].parent;
-                *inode = local->loc[0].inode;
-                *name = (char *)local->loc[0].name;
+    if (strcmp(fop, "lookup") == 0) {
+        *parent = local->loc[0].parent;
+        *inode = local->loc[0].inode;
+        *name = (char *)local->loc[0].name;
+    } else {
+        if (local->fd) {
+            *inode = local->fd->inode;
         } else {
-                if (local->fd) {
-                        *inode = local->fd->inode;
-                } else {
-                        *inode = local->loc[i].parent;
-                }
+            *inode = local->loc[i].parent;
         }
+    }
 }
 
-int32_t
-__get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
+static inline int
+pl_track_io_fop_count(pl_local_t *local, xlator_t *this, pl_count_op_t op)
 {
-        posix_lock_t *lock   = NULL;
-        int32_t       count  = 0;
+    pl_inode_t *pl_inode = NULL;
+
+    if (!local)
+        return -1;
 
-        list_for_each_entry (lock, &pl_inode->ext_list, list) {
+    pl_inode = pl_inode_get(this, local->inode, NULL);
+    if (!pl_inode)
+        return -1;
 
-                count++;
+    if (pl_inode->mlock_enforced && pl_inode->track_fop_wind_count) {
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            if (op == DECREMENT) {
+                pl_inode->fop_wind_count--;
+                /* fop_wind_count can go negative when lock enforcement is
+                 * enabled on unwind path of an IO. Hence the "<" comparision.
+                 */
+                if (pl_inode->fop_wind_count <= 0) {
+                    pthread_cond_broadcast(&pl_inode->check_fop_wind_count);
+                    pl_inode->track_fop_wind_count = _gf_false;
+                    pl_inode->fop_wind_count = 0;
+                }
+            } else {
+                pl_inode->fop_wind_count++;
+            }
         }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        return count;
+    return 0;
+}
+
+static int32_t
+__get_posixlk_count(pl_inode_t *pl_inode)
+{
+    posix_lock_t *lock = NULL;
+    int32_t count = 0;
+
+    list_for_each_entry(lock, &pl_inode->ext_list, list) { count++; }
+
+    return count;
 }
 
 int32_t
-get_posixlk_count (xlator_t *this, inode_t *inode)
+get_posixlk_count(xlator_t *this, inode_t *inode)
 {
-        pl_inode_t   *pl_inode = NULL;
-        uint64_t      tmp_pl_inode = 0;
-        int           ret      = 0;
-        int32_t       count    = 0;
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int32_t count = 0;
 
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
-        if (ret != 0) {
-                goto out;
-        }
+    int ret = inode_ctx_get(inode, this, &tmp_pl_inode);
+    if (ret != 0) {
+        goto out;
+    }
 
-        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                count = __get_posixlk_count (this, pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        count = __get_posixlk_count(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
 out:
-        return count;
+    return count;
 }
 
 void
-pl_parent_entrylk_xattr_fill (xlator_t *this, inode_t *parent,
-                              char *basename, dict_t *dict, gf_boolean_t keep_max)
+pl_parent_entrylk_xattr_fill(xlator_t *this, inode_t *parent, char *basename,
+                             dict_t *dict, gf_boolean_t keep_max)
 {
-        int32_t         entrylk  = 0;
-        int32_t         maxcount = -1;
-        int             ret      = -1;
-
-        if (!parent || !basename || !strlen (basename))
-                goto out;
-        if (keep_max) {
-                ret = dict_get_int32 (dict, GLUSTERFS_PARENT_ENTRYLK, &maxcount);
-        }
-        entrylk = check_entrylk_on_basename (this, parent, basename);
-        if (maxcount >= entrylk)
-                return;
+    int32_t entrylk = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (!parent || !basename)
+        goto out;
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_PARENT_ENTRYLK, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_PARENT_ENTRYLK);
+    }
+    entrylk = check_entrylk_on_basename(this, parent, basename);
+    if (maxcount >= entrylk)
+        return;
 out:
-        ret = dict_set_int32 (dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_PARENT_ENTRYLK);
-        }
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, " dict_set failed on key %s",
+                     GLUSTERFS_PARENT_ENTRYLK);
+    }
 }
 
 void
-pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
-                       dict_t *dict, gf_boolean_t keep_max)
+pl_entrylk_xattr_fill(xlator_t *this, inode_t *inode, dict_t *dict,
+                      gf_boolean_t keep_max)
 {
-        int32_t     count = 0;
-        int32_t     maxcount = -1;
-        int         ret   = -1;
-
-        if (keep_max) {
-                ret = dict_get_int32 (dict, GLUSTERFS_ENTRYLK_COUNT, &maxcount);
-        }
-        count = get_entrylk_count (this, inode);
-        if (maxcount >= count)
-                return;
-
-        ret = dict_set_int32 (dict, GLUSTERFS_ENTRYLK_COUNT, count);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_ENTRYLK_COUNT);
-        }
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_ENTRYLK_COUNT, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_ENTRYLK_COUNT);
+    }
+    count = get_entrylk_count(this, inode);
+    if (maxcount >= count)
+        return;
 
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_ENTRYLK_COUNT, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, " dict_set failed on key %s",
+                     GLUSTERFS_ENTRYLK_COUNT);
+    }
 }
 
 void
-pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode, dict_t *dict,
-                       char *domname, gf_boolean_t keep_max)
+pl_inodelk_xattr_fill(xlator_t *this, inode_t *inode, dict_t *dict,
+                      char *domname, gf_boolean_t keep_max)
 {
-        int32_t     count = 0;
-        int32_t     maxcount = -1;
-        int         ret   = -1;
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_INODELK_COUNT, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_INODELK_COUNT);
+    }
+    count = get_inodelk_count(this, inode, domname);
+    if (maxcount >= count)
+        return;
 
-        if (keep_max) {
-                ret = dict_get_int32 (dict, GLUSTERFS_INODELK_COUNT, &maxcount);
-        }
-        count = get_inodelk_count (this, inode, domname);
-        if (maxcount >= count)
-                return;
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_INODELK_COUNT, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set count for "
+                     "key %s",
+                     GLUSTERFS_INODELK_COUNT);
+    }
 
-        ret = dict_set_int32 (dict, GLUSTERFS_INODELK_COUNT, count);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to set count for "
-                        "key %s", GLUSTERFS_INODELK_COUNT);
-        }
+    return;
+}
 
+void
+pl_posixlk_xattr_fill(xlator_t *this, inode_t *inode, dict_t *dict,
+                      gf_boolean_t keep_max)
+{
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_POSIXLK_COUNT, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_POSIXLK_COUNT);
+    }
+    count = get_posixlk_count(this, inode);
+    if (maxcount >= count)
         return;
+
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_POSIXLK_COUNT, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, " dict_set failed on key %s",
+                     GLUSTERFS_POSIXLK_COUNT);
+    }
 }
 
 void
-pl_posixlk_xattr_fill (xlator_t *this, inode_t *inode,
-                       dict_t *dict, gf_boolean_t keep_max)
+pl_inodelk_xattr_fill_each(xlator_t *this, inode_t *inode, dict_t *dict,
+                           char *domname, gf_boolean_t keep_max, char *key)
 {
-        int32_t     count = 0;
-        int32_t     maxcount = -1;
-        int         ret   = -1;
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32(dict, key, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_INODELK_COUNT);
+    }
+    count = get_inodelk_count(this, inode, domname);
+    if (maxcount >= count)
+        return;
 
-        if (keep_max) {
-                ret = dict_get_int32 (dict, GLUSTERFS_POSIXLK_COUNT, &maxcount);
-        }
-        count = get_posixlk_count (this, inode);
-        if (maxcount >= count)
-                return;
+    ret = dict_set_int32(dict, key, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set count for "
+                     "key %s",
+                     key);
+    }
 
-        ret = dict_set_int32 (dict, GLUSTERFS_POSIXLK_COUNT, count);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_POSIXLK_COUNT);
-        }
+    return;
+}
 
+static int
+pl_inodelk_xattr_fill_multiple(dict_t *this, char *key, data_t *value,
+                               void *data)
+{
+    multi_dom_lk_data *d = data;
+    char *tmp_key = NULL;
+    char *save_ptr = NULL;
+
+    tmp_key = gf_strdup(key);
+    if (!tmp_key)
+        return -1;
+
+    strtok_r(tmp_key, ":", &save_ptr);
+    if (!*save_ptr) {
+        if (tmp_key)
+            GF_FREE(tmp_key);
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, EINVAL,
+               "Could not tokenize domain string from key %s", key);
+        return -1;
+    }
+
+    pl_inodelk_xattr_fill_each(d->this, d->inode, d->xdata_rsp, save_ptr,
+                               d->keep_max, key);
+    if (tmp_key)
+        GF_FREE(tmp_key);
+
+    return 0;
 }
 
 void
-pl_set_xdata_response (xlator_t *this, pl_local_t *local, inode_t *parent,
-                       inode_t *inode, char *name, dict_t *xdata, gf_boolean_t max_lock)
+pl_fill_multiple_dom_lk_requests(xlator_t *this, pl_local_t *local,
+                                 inode_t *inode, dict_t *dict,
+                                 gf_boolean_t keep_max)
 {
-        if (!xdata || !local)
-                return;
+    multi_dom_lk_data data;
 
-        if (local->parent_entrylk_req && parent && name && strlen (name))
-                pl_parent_entrylk_xattr_fill (this, parent, name, xdata, max_lock);
+    data.this = this;
+    data.inode = inode;
+    data.xdata_rsp = dict;
+    data.keep_max = keep_max;
+
+    dict_foreach_fnmatch(local->xdata, GLUSTERFS_INODELK_DOM_PREFIX "*",
+                         pl_inodelk_xattr_fill_multiple, &data);
+}
 
-        if (local->entrylk_count_req && inode)
-                pl_entrylk_xattr_fill (this, inode, xdata, max_lock);
+void
+pl_set_xdata_response(xlator_t *this, pl_local_t *local, inode_t *parent,
+                      inode_t *inode, char *name, dict_t *xdata,
+                      gf_boolean_t max_lock)
+{
+    if (!xdata || !local)
+        return;
 
-        if (local->inodelk_dom_count_req && inode)
-                pl_inodelk_xattr_fill (this, inode, xdata,
-                                    data_to_str (local->inodelk_dom_count_req), max_lock);
+    if (local->parent_entrylk_req && parent && name && name[0] != '\0')
+        pl_parent_entrylk_xattr_fill(this, parent, name, xdata, max_lock);
 
-        if (local->inodelk_count_req && inode)
-                pl_inodelk_xattr_fill (this, inode, xdata, NULL, max_lock);
+    if (!inode)
+        return;
 
-        if (local->posixlk_count_req && inode)
-                pl_posixlk_xattr_fill (this, inode, xdata, max_lock);
-}
+    if (local->entrylk_count_req)
+        pl_entrylk_xattr_fill(this, inode, xdata, max_lock);
 
-/* Return true in case we need to ensure mandatory-locking
- * semnatics under different modes.
- */
-gf_boolean_t
-pl_is_mandatory_locking_enabled (pl_inode_t *pl_inode)
-{
-        posix_locks_private_t *priv = NULL;
+    if (local->inodelk_dom_count_req)
+        pl_inodelk_xattr_fill(this, inode, xdata,
+                              data_to_str(local->inodelk_dom_count_req),
+                              max_lock);
 
-        priv = THIS->private;
+    if (local->inodelk_count_req)
+        pl_inodelk_xattr_fill(this, inode, xdata, NULL, max_lock);
 
-        if (priv->mandatory_mode == MLK_FILE_BASED && pl_inode->mandatory)
-                return _gf_true;
-        else if (priv->mandatory_mode == MLK_FORCED ||
-                 priv->mandatory_mode == MLK_OPTIMAL)
-                return _gf_true;
+    if (local->posixlk_count_req)
+        pl_posixlk_xattr_fill(this, inode, xdata, max_lock);
 
-        return _gf_false;
+    if (local->multiple_dom_lk_requests)
+        pl_fill_multiple_dom_lk_requests(this, local, inode, xdata, max_lock);
 }
 
 /* Checks whether the region where fop is acting upon conflicts
@@ -376,3564 +568,4528 @@ pl_is_mandatory_locking_enabled (pl_inode_t *pl_inode)
  * indicate block/fail the fop.
  */
 int
-pl_is_fop_allowed (pl_inode_t *pl_inode, posix_lock_t *region, fd_t *fd,
-                   glusterfs_fop_t op, gf_boolean_t *can_block)
+pl_is_fop_allowed(pl_inode_t *pl_inode, posix_lock_t *region, fd_t *fd,
+                  glusterfs_fop_t op, gf_boolean_t *can_block)
 {
-        int     ret     = 0;
-
-        if (!__rw_allowable (pl_inode, region, op)) {
-                if ((!fd) || (fd && (fd->flags & O_NONBLOCK))) {
-                        gf_log ("locks", GF_LOG_TRACE, "returning EAGAIN"
-                                                " because fd is O_NONBLOCK");
-                        *can_block = _gf_false;
-                } else
-                        *can_block = _gf_true;
-        } else
-                ret = 1;
+    int ret = 0;
+
+    if (!__rw_allowable(pl_inode, region, op)) {
+        if (pl_inode->mlock_enforced) {
+            *can_block = _gf_false;
+        } else if ((!fd) || (fd && (fd->flags & O_NONBLOCK))) {
+            gf_log("locks", GF_LOG_TRACE,
+                   "returning EAGAIN"
+                   " because fd is O_NONBLOCK");
+            *can_block = _gf_false;
+        } else {
+            *can_block = _gf_true;
+        }
+    } else {
+        ret = 1;
+    }
 
-        return ret;
+    return ret;
 }
 
 static pl_fdctx_t *
-pl_new_fdctx ()
+pl_new_fdctx()
 {
-        pl_fdctx_t *fdctx = NULL;
+    pl_fdctx_t *fdctx = GF_MALLOC(sizeof(*fdctx), gf_locks_mt_pl_fdctx_t);
+    GF_VALIDATE_OR_GOTO("posix-locks", fdctx, out);
 
-        fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                           gf_locks_mt_pl_fdctx_t);
-        GF_VALIDATE_OR_GOTO ("posix-locks", fdctx, out);
-
-        INIT_LIST_HEAD (&fdctx->locks_list);
+    INIT_LIST_HEAD(&fdctx->locks_list);
 
 out:
-        return fdctx;
+    return fdctx;
 }
 
 static pl_fdctx_t *
-pl_check_n_create_fdctx (xlator_t *this, fd_t *fd)
+pl_check_n_create_fdctx(xlator_t *this, fd_t *fd)
 {
-        int         ret   = 0;
-        uint64_t    tmp   = 0;
-        pl_fdctx_t *fdctx = NULL;
+    int ret = 0;
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
 
-        GF_VALIDATE_OR_GOTO ("posix-locks", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &tmp);
-                if ((ret != 0) || (tmp == 0)) {
-                        fdctx = pl_new_fdctx ();
-                        if (fdctx == NULL) {
-                                goto unlock;
-                        }
-                }
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &tmp);
+        if ((ret != 0) || (tmp == 0)) {
+            fdctx = pl_new_fdctx();
+            if (fdctx == NULL) {
+                goto unlock;
+            }
+        }
 
-                ret = __fd_ctx_set (fd, this, (uint64_t)(long)fdctx);
-                if (ret != 0) {
-                        GF_FREE (fdctx);
-                        fdctx = NULL;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set fd ctx");
-                }
+        ret = __fd_ctx_set(fd, this, (uint64_t)(long)fdctx);
+        if (ret != 0) {
+            GF_FREE(fdctx);
+            fdctx = NULL;
+            UNLOCK(&fd->lock);
+            gf_log(this->name, GF_LOG_DEBUG, "failed to set fd ctx");
+            goto out;
         }
+    }
 unlock:
-        UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 
 out:
-        return fdctx;
+    return fdctx;
 }
 
 int32_t
-pl_discard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+pl_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
+
+    PL_STACK_UNWIND(discard, xdata, frame, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+    return 0;
 }
 
 int
-pl_discard_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 size_t len, dict_t *xdata)
+pl_discard_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_discard_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
-        return 0;
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
+
+    STACK_WIND(frame, pl_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
 }
 
 int32_t
-pl_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            size_t len, dict_t *xdata)
+pl_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
 {
-        pl_inode_t            *pl_inode = NULL;
-        pl_rw_req_t           *rw = NULL;
-        posix_lock_t           region = {.list = {0, }, };
-        gf_boolean_t           enabled = _gf_false;
-        gf_boolean_t           can_block = _gf_true;
-        int                    op_ret = 0;
-        int                    op_errno = 0;
-        int                    allowed = 1;
-
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->inode = inode_ref(fd->inode);
+    local->fd = fd_ref(fd);
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + len - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_DISCARD,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-        pl_inode = pl_inode_get (this, fd->inode);
-        if (!pl_inode) {
-                op_ret   = -1;
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
                 op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        enabled = pl_is_mandatory_locking_enabled (pl_inode);
-
-        if (frame->root->pid < 0)
-                enabled = _gf_false;
-
-        if (enabled) {
-                region.fl_start   = offset;
-                region.fl_end     = offset + len - 1;
-                region.client     = frame->root->client;
-                region.fd_num     = fd_to_fdnum(fd);
-                region.client_pid = frame->root->pid;
-                region.owner      = frame->root->lk_owner;
-
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        allowed = pl_is_fop_allowed (pl_inode, &region, fd,
-                                                     GF_FOP_DISCARD,
-                                                     &can_block);
-                        if (allowed == 1)
-                                goto unlock;
-                        else if (!can_block) {
-                                op_errno = EAGAIN;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw = GF_CALLOC (1, sizeof (*rw),
-                                        gf_locks_mt_pl_rw_req_t);
-                        if (!rw) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                goto unlock;
-                        }
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        rw->stub = fop_discard_stub (frame, pl_discard_cont,
-                                                     fd, offset, len, xdata);
-                        if (!rw->stub) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                GF_FREE (rw);
-                                goto unlock;
-                        }
+            rw->stub = fop_discard_stub(frame, pl_discard_cont, fd, offset, len,
+                                        xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-                        rw->region = region;
+            rw->region = region;
 
-                        list_add_tail (&rw->list, &pl_inode->rw_list);
-                }
-        unlock:
-                pthread_mutex_unlock (&pl_inode->mutex);
+            list_add_tail(&rw->list, &pl_inode->rw_list);
         }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        if (allowed == 1)
-                STACK_WIND (frame, pl_discard_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->discard, fd, offset,
-                            len, xdata);
+    if (allowed == 1)
+        STACK_WIND(frame, pl_discard_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
 unwind:
-        if (op_ret == -1)
-                STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno,
-                                     NULL, NULL, NULL);
+    if (op_ret == -1)
+        PL_STACK_UNWIND(discard, xdata, frame, op_ret, op_errno, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-pl_zerofill_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+pl_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
+
+    PL_STACK_UNWIND(zerofill, xdata, frame, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+    return 0;
 }
 
 int
-pl_zerofill_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  off_t len, dict_t *xdata)
+pl_zerofill_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_zerofill_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
-        return 0;
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
+
+    STACK_WIND(frame, pl_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 }
 
 int32_t
-pl_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             off_t len, dict_t *xdata)
+pl_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
 {
-        pl_inode_t            *pl_inode = NULL;
-        pl_rw_req_t           *rw = NULL;
-        posix_lock_t           region = {.list = {0, }, };
-        gf_boolean_t           enabled = _gf_false;
-        gf_boolean_t           can_block = _gf_true;
-        int                    op_ret = 0;
-        int                    op_errno = 0;
-        int                    allowed = 1;
-
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->inode = inode_ref(fd->inode);
+    local->fd = fd_ref(fd);
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + len - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_ZEROFILL,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-        pl_inode = pl_inode_get (this, fd->inode);
-        if (!pl_inode) {
-                op_ret   = -1;
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
                 op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        enabled = pl_is_mandatory_locking_enabled (pl_inode);
-
-        if (frame->root->pid < 0)
-                enabled = _gf_false;
-
-        if (enabled) {
-                region.fl_start   = offset;
-                region.fl_end     = offset + len - 1;
-                region.client     = frame->root->client;
-                region.fd_num     = fd_to_fdnum(fd);
-                region.client_pid = frame->root->pid;
-                region.owner      = frame->root->lk_owner;
-
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        allowed = pl_is_fop_allowed (pl_inode, &region, fd,
-                                                     GF_FOP_ZEROFILL,
-                                                     &can_block);
-                        if (allowed == 1)
-                                goto unlock;
-                        else if (!can_block) {
-                                op_errno = EAGAIN;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw = GF_CALLOC (1, sizeof (*rw),
-                                        gf_locks_mt_pl_rw_req_t);
-                        if (!rw) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                goto unlock;
-                        }
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        rw->stub = fop_zerofill_stub (frame, pl_zerofill_cont,
-                                                      fd, offset, len, xdata);
-                        if (!rw->stub) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                GF_FREE (rw);
-                                goto unlock;
-                        }
+            rw->stub = fop_zerofill_stub(frame, pl_zerofill_cont, fd, offset,
+                                         len, xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-                        rw->region = region;
+            rw->region = region;
 
-                        list_add_tail (&rw->list, &pl_inode->rw_list);
-                }
-        unlock:
-                pthread_mutex_unlock (&pl_inode->mutex);
+            list_add_tail(&rw->list, &pl_inode->rw_list);
         }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        if (allowed == 1)
-                STACK_WIND (frame, pl_zerofill_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->zerofill, fd, offset,
-                            len, xdata);
+    if (allowed == 1)
+        STACK_WIND(frame, pl_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
 unwind:
-        if (op_ret == -1)
-                STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno,
-                                     NULL, NULL, NULL);
+    if (op_ret == -1)
+        PL_STACK_UNWIND(zerofill, xdata, frame, op_ret, op_errno, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+pl_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        pl_local_t *local = NULL;
+    pl_local_t *local = frame->local;
 
-        local = frame->local;
-
-        if (local->op == GF_FOP_TRUNCATE)
-                loc_wipe (&local->loc[0]);
-
-        if (local->xdata)
-                dict_unref (local->xdata);
-        if (local->fd)
-                fd_unref (local->fd);
+    pl_track_io_fop_count(local, this, DECREMENT);
 
-        if (local->op == GF_FOP_TRUNCATE)
-                STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                                                prebuf, postbuf, xdata);
-        else
-                STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                                                prebuf, postbuf, xdata);
-        return 0;
+    if (local->op == GF_FOP_TRUNCATE)
+        PL_STACK_UNWIND(truncate, xdata, frame, op_ret, op_errno, prebuf,
+                        postbuf, xdata);
+    else
+        PL_STACK_UNWIND(ftruncate, xdata, frame, op_ret, op_errno, prebuf,
+                        postbuf, xdata);
+    return 0;
 }
 
 int
-pl_ftruncate_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
+pl_ftruncate_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
-        STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
+
+    STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 int
-pl_truncate_cont (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
+pl_truncate_cont(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
+
+    STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
 static int
-truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   dict_t *xdata)
-{
-        pl_local_t            *local = NULL;
-        inode_t               *inode = NULL;
-        pl_inode_t            *pl_inode = NULL;
-        pl_rw_req_t           *rw = NULL;
-        posix_lock_t           region = {.list = {0, }, };
-        gf_boolean_t           enabled = _gf_false;
-        gf_boolean_t           can_block = _gf_true;
-        int                    allowed = 1;
-
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
-        local = frame->local;
-
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "got error (errno=%d, stderror=%s) from child",
-                        op_errno, strerror (op_errno));
-                goto unwind;
-        }
+truncate_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+    pl_local_t *local = frame->local;
+    inode_t *inode = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    if (op_ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "got error (errno=%d, stderror=%s) from child", op_errno,
+               strerror(op_errno));
+        goto unwind;
+    }
+
+    if (local->op == GF_FOP_TRUNCATE)
+        inode = local->loc[0].inode;
+    else
+        inode = local->fd->inode;
+
+    local->inode = inode_ref(inode);
+
+    pl_inode = pl_inode_get(this, inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = local->offset;
+        region.fl_end = LLONG_MAX;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(local->fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, local->fd, local->op,
+                                        &can_block);
 
-        if (local->op == GF_FOP_TRUNCATE)
-                inode = local->loc[0].inode;
-        else
-                inode = local->fd->inode;
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-        pl_inode = pl_inode_get (this, inode);
-        if (!pl_inode) {
-                op_ret   = -1;
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
                 op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        enabled = pl_is_mandatory_locking_enabled (pl_inode);
-
-        if (frame->root->pid < 0)
-                enabled = _gf_false;
-
-        if (enabled) {
-                region.fl_start   = local->offset;
-                region.fl_end     = LLONG_MAX;
-                region.client     = frame->root->client;
-                region.fd_num     = fd_to_fdnum(local->fd);
-                region.client_pid = frame->root->pid;
-                region.owner      = frame->root->lk_owner;
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        allowed = pl_is_fop_allowed (pl_inode, &region,
-                                                     local->fd, local->op,
-                                                     &can_block);
-
-                        if (allowed == 1)
-                                goto unlock;
-                        else if (!can_block) {
-                                op_errno = EAGAIN;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw = GF_CALLOC (1, sizeof (*rw),
-                                        gf_locks_mt_pl_rw_req_t);
-                        if (!rw) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        if (local->op == GF_FOP_TRUNCATE)
-                                rw->stub = fop_truncate_stub (frame,
-                                                pl_truncate_cont, &local->loc[0],
-                                                local->offset, local->xdata);
-                        else
-                                rw->stub = fop_ftruncate_stub (frame,
-                                                pl_ftruncate_cont, local->fd,
-                                                local->offset, local->xdata);
-                        if (!rw->stub) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                GF_FREE (rw);
-                                goto unlock;
-                        }
+                op_ret = -1;
+                goto unlock;
+            }
+
+            if (local->op == GF_FOP_TRUNCATE)
+                rw->stub = fop_truncate_stub(frame, pl_truncate_cont,
+                                             &local->loc[0], local->offset,
+                                             local->xdata);
+            else
+                rw->stub = fop_ftruncate_stub(frame, pl_ftruncate_cont,
+                                              local->fd, local->offset,
+                                              local->xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-                        rw->region = region;
+            rw->region = region;
 
-                        list_add_tail (&rw->list, &pl_inode->rw_list);
-                }
-        unlock:
-                pthread_mutex_unlock (&pl_inode->mutex);
+            list_add_tail(&rw->list, &pl_inode->rw_list);
         }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        if (allowed == 1) {
-                switch (local->op) {
-                case GF_FOP_TRUNCATE:
-                        STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
-                                    FIRST_CHILD (this)->fops->truncate,
-                                    &local->loc[0], local->offset, local->xdata);
-                        break;
-                case GF_FOP_FTRUNCATE:
-                        STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
-                                    FIRST_CHILD (this)->fops->ftruncate,
-                                    local->fd, local->offset, local->xdata);
-                        break;
-                default:
-                        break;
-                }
+    if (allowed == 1) {
+        switch (local->op) {
+            case GF_FOP_TRUNCATE:
+                STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->truncate, &local->loc[0],
+                           local->offset, local->xdata);
+                break;
+            case GF_FOP_FTRUNCATE:
+                STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->ftruncate, local->fd,
+                           local->offset, local->xdata);
+                break;
+            default:
+                break;
         }
+    }
 unwind:
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "truncate failed with "
-                        "ret: %d, error: %s", op_ret, strerror (op_errno));
-                if (local->op == GF_FOP_TRUNCATE)
-                        loc_wipe (&local->loc[0]);
-                if (local->xdata)
-                        dict_unref (local->xdata);
-                if (local->fd)
-                        fd_unref (local->fd);
-
-                switch (local->op) {
-                case GF_FOP_TRUNCATE:
-                        STACK_UNWIND_STRICT (truncate, frame, op_ret,
-                                                op_errno, buf, NULL, xdata);
-                        break;
-                case GF_FOP_FTRUNCATE:
-                        STACK_UNWIND_STRICT (ftruncate, frame, op_ret,
-                                                op_errno, buf, NULL, xdata);
-                        break;
-                default:
-                        break;
-                }
+    if (op_ret == -1) {
+        gf_log(this ? this->name : "locks", GF_LOG_ERROR,
+               "truncate failed with "
+               "ret: %d, error: %s",
+               op_ret, strerror(op_errno));
+
+        switch (local->op) {
+            case GF_FOP_TRUNCATE:
+                PL_STACK_UNWIND(truncate, xdata, frame, op_ret, op_errno, buf,
+                                NULL, xdata);
+                break;
+            case GF_FOP_FTRUNCATE:
+                PL_STACK_UNWIND(ftruncate, xdata, frame, op_ret, op_errno, buf,
+                                NULL, xdata);
+                break;
+            default:
+                break;
         }
-        return 0;
+    }
+    return 0;
 }
 
 int
-pl_truncate (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, off_t offset, dict_t *xdata)
+pl_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        pl_local_t *local = NULL;
-        int         ret   = -1;
+    pl_local_t *local = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
 
-        local = mem_get0 (this->local_pool);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
+    local = mem_get0(this->local_pool);
+    GF_VALIDATE_OR_GOTO(this->name, local, unwind);
 
-        local->op         = GF_FOP_TRUNCATE;
-        local->offset     = offset;
-        loc_copy (&local->loc[0], loc);
-        if (xdata)
-                local->xdata = dict_ref (xdata);
+    local->op = GF_FOP_TRUNCATE;
+    local->offset = offset;
+    loc_copy(&local->loc[0], loc);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
 
-        frame->local = local;
+    frame->local = local;
+
+    STACK_WIND(frame, truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, NULL);
+    ret = 0;
 
-        STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc, NULL);
-        ret = 0;
 unwind:
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "truncate on %s failed with"
-                        " ret: %d, error: %s", loc->path, -1,
-                        strerror (ENOMEM));
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL);
-        }
-        return 0;
+    if (ret == -1) {
+        gf_log(this ? this->name : "locks", GF_LOG_ERROR,
+               "truncate on %s failed with"
+               " ret: %d, error: %s",
+               loc->path, -1, strerror(ENOMEM));
+        STACK_UNWIND_STRICT(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    }
+    return 0;
 }
 
 int
-pl_ftruncate (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, off_t offset, dict_t *xdata)
+pl_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-        pl_local_t *local = NULL;
-        int         ret = -1;
+    pl_local_t *local = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
-        local = mem_get0 (this->local_pool);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+    local = mem_get0(this->local_pool);
+    GF_VALIDATE_OR_GOTO(this->name, local, unwind);
 
-        local->op         = GF_FOP_FTRUNCATE;
-        local->offset     = offset;
-        local->fd         = fd_ref (fd);
-        if (xdata)
-                local->xdata = dict_ref (xdata);
+    local->op = GF_FOP_FTRUNCATE;
+    local->offset = offset;
+    local->fd = fd_ref(fd);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        ret = 0;
+    STACK_WIND(frame, truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    ret = 0;
 unwind:
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "ftruncate failed with"
-                        " ret: %d, error: %s", -1, strerror (ENOMEM));
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL);
-        }
-        return 0;
+    if (ret == -1) {
+        gf_log(this ? this->name : "locks", GF_LOG_ERROR,
+               "ftruncate failed with"
+               " ret: %d, error: %s",
+               -1, strerror(ENOMEM));
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    }
+    return 0;
 }
 
 int
-pl_locks_by_fd (pl_inode_t *pl_inode, fd_t *fd)
+pl_locks_by_fd(pl_inode_t *pl_inode, fd_t *fd)
 {
-       posix_lock_t *l = NULL;
-       int found = 0;
-
-       pthread_mutex_lock (&pl_inode->mutex);
-       {
+    posix_lock_t *l = NULL;
+    int found = 0;
 
-               list_for_each_entry (l, &pl_inode->ext_list, list) {
-                       if (l->fd_num == fd_to_fdnum(fd)) {
-                               found = 1;
-                               break;
-                       }
-               }
-
-       }
-       pthread_mutex_unlock (&pl_inode->mutex);
-       return found;
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
+        {
+            if (l->fd_num == fd_to_fdnum(fd)) {
+                found = 1;
+                break;
+            }
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    return found;
 }
 
 static void
-delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
+delete_locks_of_fd(xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 {
-       posix_lock_t *tmp = NULL;
-       posix_lock_t *l = NULL;
-
-       struct list_head blocked_list;
-
-       INIT_LIST_HEAD (&blocked_list);
-
-       pthread_mutex_lock (&pl_inode->mutex);
-       {
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *l = NULL;
 
-               list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                       if (l->fd_num == fd_to_fdnum(fd)) {
-                               if (l->blocked) {
-                                       list_move_tail (&l->list, &blocked_list);
-                                       continue;
-                               }
-                               __delete_lock (l);
-                               __destroy_lock (l);
-                       }
-               }
+    struct list_head blocked_list;
 
-       }
-       pthread_mutex_unlock (&pl_inode->mutex);
+    INIT_LIST_HEAD(&blocked_list);
 
-       list_for_each_entry_safe (l, tmp, &blocked_list, list) {
-               list_del_init(&l->list);
-               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock,
-                                    NULL);
-               __destroy_lock (l);
-       }
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+        {
+            if (l->fd_num == fd_to_fdnum(fd)) {
+                if (l->blocked) {
+                    list_move_tail(&l->list, &blocked_list);
+                    continue;
+                }
+                __delete_lock(l);
+                __destroy_lock(l);
+            }
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        grant_blocked_locks (this, pl_inode);
+    list_for_each_entry_safe(l, tmp, &blocked_list, list)
+    {
+        list_del_init(&l->list);
+        STACK_UNWIND_STRICT(lk, l->frame, -1, EAGAIN, &l->user_flock, NULL);
+        __destroy_lock(l);
+    }
 
-        do_blocked_rw (pl_inode);
+    grant_blocked_locks(this, pl_inode);
 
+    do_blocked_rw(pl_inode);
 }
 
 static void
-__delete_locks_of_owner (pl_inode_t *pl_inode,
-                         client_t *client, gf_lkowner_t *owner)
-{
-        posix_lock_t *tmp = NULL;
-        posix_lock_t *l = NULL;
-
-        /* TODO: what if it is a blocked lock with pending l->frame */
-
-        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if (l->blocked)
-                        continue;
-                if ((l->client == client) &&
-                    is_same_lkowner (&l->owner, owner)) {
-                        gf_log ("posix-locks", GF_LOG_TRACE,
-                                " Flushing lock"
-                                "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" state: %s",
-                                l->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                l->client_pid,
-                                lkowner_utoa (&l->owner),
-                                l->user_flock.l_start,
-                                l->user_flock.l_len,
-                                l->blocked == 1 ? "Blocked" : "Active");
-
-                        __delete_lock (l);
-                        __destroy_lock (l);
-                }
-        }
-
-        return;
+__delete_locks_of_owner(pl_inode_t *pl_inode, client_t *client,
+                        gf_lkowner_t *owner)
+{
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *l = NULL;
+
+    /* TODO: what if it is a blocked lock with pending l->frame */
+
+    list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+    {
+        if (l->blocked)
+            continue;
+        if ((l->client == client) && is_same_lkowner(&l->owner, owner)) {
+            gf_log("posix-locks", GF_LOG_TRACE,
+                   " Flushing lock"
+                   "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                   " state: %s",
+                   l->fl_type == F_UNLCK ? "Unlock" : "Lock", l->client_pid,
+                   lkowner_utoa(&l->owner), l->user_flock.l_start,
+                   l->user_flock.l_len, l->blocked == 1 ? "Blocked" : "Active");
+
+            __delete_lock(l);
+            __destroy_lock(l);
+        }
+    }
+
+    return;
 }
 
-
 int32_t
-pl_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+pl_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-int32_t
-pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name, dict_t *xdata)
-{
-        int32_t                 op_errno        = EINVAL;
-        int                     op_ret          = -1;
-        int32_t                 bcount          = 0;
-        int32_t                 gcount          = 0;
-        char                    key[PATH_MAX]   = {0, };
-        char                    *lk_summary     = NULL;
-        pl_inode_t              *pl_inode       = NULL;
-        dict_t                  *dict           = NULL;
-        clrlk_args              args            = {0,};
-        char                    *brickname      = NULL;
-
-        if (!name)
-                goto usual;
-
-        if (strncmp (name, GF_XATTR_CLRLK_CMD, strlen (GF_XATTR_CLRLK_CMD)))
-                goto usual;
-
-        if (clrlk_parse_args (name, &args)) {
-                op_errno = EINVAL;
-                goto out;
+static int32_t
+pl_getxattr_clrlk(xlator_t *this, const char *name, inode_t *inode,
+                  dict_t **dict, int32_t *op_errno)
+{
+    int32_t bcount = 0;
+    int32_t gcount = 0;
+    char *key = NULL;
+    char *lk_summary = NULL;
+    pl_inode_t *pl_inode = NULL;
+    clrlk_args args = {
+        0,
+    };
+    char *brickname = NULL;
+    int32_t op_ret = -1;
+
+    *op_errno = EINVAL;
+
+    if (clrlk_parse_args(name, &args)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    *dict = dict_new();
+    if (!*dict) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    switch (args.type) {
+        case CLRLK_INODE:
+        case CLRLK_ENTRY:
+            op_ret = clrlk_clear_lks_in_all_domains(this, pl_inode, &args,
+                                                    &bcount, &gcount, op_errno);
+            break;
+        case CLRLK_POSIX:
+            op_ret = clrlk_clear_posixlk(this, pl_inode, &args, &bcount,
+                                         &gcount, op_errno);
+            break;
+        default:
+            op_ret = -1;
+            *op_errno = EINVAL;
+    }
+    if (op_ret) {
+        if (args.type >= CLRLK_TYPE_MAX) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "clear locks: invalid lock type %d", args.type);
+        } else {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "clear locks of type %s failed: %s",
+                   clrlk_type_names[args.type], strerror(*op_errno));
         }
 
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+        goto out;
+    }
 
-        pl_inode = pl_inode_get (this, loc->inode);
-        if (!pl_inode) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    op_ret = fetch_pathinfo(this, inode, op_errno, &brickname);
+    if (op_ret) {
+        gf_log(this->name, GF_LOG_WARNING, "Couldn't get brickname");
+    } else {
+        op_ret = format_brickname(brickname);
+        if (op_ret) {
+            gf_log(this->name, GF_LOG_WARNING, "Couldn't format brickname");
+            GF_FREE(brickname);
+            brickname = NULL;
+        }
+    }
+
+    if (!gcount && !bcount) {
+        if (gf_asprintf(&lk_summary, "No locks cleared.") == -1) {
+            op_ret = -1;
+            *op_errno = ENOMEM;
+            goto out;
+        }
+    } else if (gf_asprintf(&lk_summary,
+                           "%s: %s blocked locks=%d "
+                           "granted locks=%d",
+                           (brickname == NULL) ? this->name : brickname,
+                           clrlk_type_names[args.type], bcount, gcount) == -1) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    gf_log(this->name, GF_LOG_DEBUG, "%s", lk_summary);
+
+    key = gf_strdup(name);
+    if (!key) {
+        op_ret = -1;
+        goto out;
+    }
+    if (dict_set_dynstr(*dict, key, lk_summary)) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_ret = 0;
 
-        switch (args.type) {
-                case CLRLK_INODE:
-                case CLRLK_ENTRY:
-                        op_ret = clrlk_clear_lks_in_all_domains (this, pl_inode,
-                                                                 &args, &bcount,
-                                                                 &gcount,
-                                                                 &op_errno);
-                        if (op_ret)
-                                goto out;
-                        break;
-                case CLRLK_POSIX:
-                        op_ret = clrlk_clear_posixlk (this, pl_inode, &args,
-                                                      &bcount, &gcount,
-                                                      &op_errno);
-                        if (op_ret)
-                                goto out;
-                        break;
-                case CLRLK_TYPE_MAX:
-                        op_errno = EINVAL;
-                        goto out;
-        }
+out:
+    GF_FREE(brickname);
+    GF_FREE(args.opts);
+    GF_FREE(key);
+    if (op_ret) {
+        GF_FREE(lk_summary);
+    }
+
+    return op_ret;
+}
 
-        op_ret = fetch_pathinfo (this, loc->inode, &op_errno, &brickname);
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Couldn't get brickname");
-        } else {
-                op_ret = format_brickname(brickname);
-                if (op_ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Couldn't format brickname");
-                        GF_FREE(brickname);
-                        brickname = NULL;
-                }
-        }
+int32_t
+pl_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    dict_t *dict = NULL;
 
-        if (!gcount && !bcount) {
-                if (gf_asprintf (&lk_summary, "No locks cleared.") == -1) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-        } else if (gf_asprintf (&lk_summary, "%s: %s blocked locks=%d "
-                                "granted locks=%d",
-                                (brickname == NULL)? this->name : brickname,
-                                (args.type == CLRLK_INODE)? "inode":
-                                (args.type == CLRLK_ENTRY)? "entry":
-                                (args.type == CLRLK_POSIX)? "posix": " ",
-                                bcount, gcount) == -1) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (!name)
+        goto usual;
 
-        strncpy (key, name, strlen (name));
-        if (dict_set_dynstr (dict, key, lk_summary)) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)))
+        goto usual;
 
-        op_ret = 0;
-out:
-        GF_FREE(brickname);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+    op_ret = pl_getxattr_clrlk(this, name, loc->inode, &dict, &op_errno);
 
-        GF_FREE (args.opts);
-        if (op_ret && lk_summary)
-                GF_FREE (lk_summary);
-        if (dict)
-                dict_unref (dict);
-        return 0;
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    if (dict)
+        dict_unref(dict);
+    return 0;
 
 usual:
-        STACK_WIND (frame, pl_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, pl_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
 static int
 format_brickname(char *brickname)
 {
-       int   ret             = -1;
-       char *hostname        = NULL;
-       char *volume          = NULL;
-       char *saveptr         = NULL;
+    int ret = -1;
+    char *hostname = NULL;
+    char *volume = NULL;
+    char *saveptr = NULL;
 
-       if (!brickname)
-               goto out;
+    if (!brickname)
+        goto out;
 
-       strtok_r(brickname, ":", &saveptr);
-       hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
-       if (hostname == NULL)
-               goto out;
-       volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
-       if (volume == NULL)
-               goto out;
+    strtok_r(brickname, ":", &saveptr);
+    hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
+    if (hostname == NULL)
+        goto out;
+    volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
+    if (volume == NULL)
+        goto out;
 
-       sprintf(brickname, "%s:%s", hostname, volume);
+    sprintf(brickname, "%s:%s", hostname, volume);
 
-       ret = 0;
+    ret = 0;
 out:
-       GF_FREE(hostname);
-       GF_FREE(volume);
-       return ret;
+    GF_FREE(hostname);
+    GF_FREE(volume);
+    return ret;
 }
 
 static int
-fetch_pathinfo (xlator_t *this, inode_t *inode, int32_t *op_errno,
-                char **brickname)
+fetch_pathinfo(xlator_t *this, inode_t *inode, int32_t *op_errno,
+               char **brickname)
 {
-        int                    ret       = -1;
-        loc_t                  loc       = {0, };
-        dict_t                *dict      = NULL;
-
-        if (!brickname)
-                goto out;
-
-        if (!op_errno)
-                goto out;
-
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        loc.inode = inode_ref (inode);
-
-        ret = syncop_getxattr (FIRST_CHILD(this), &loc, &dict,
-                               GF_XATTR_PATHINFO_KEY, NULL, NULL);
-        if (ret < 0) {
-                *op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, brickname);
-        if (ret)
-                goto out;
-
-        *brickname = gf_strdup(*brickname);
-        if (*brickname == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    if (!brickname)
+        goto out;
+
+    if (!op_errno)
+        goto out;
+
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+
+    ret = syncop_getxattr(FIRST_CHILD(this), &loc, &dict, GF_XATTR_PATHINFO_KEY,
+                          NULL, NULL);
+    if (ret < 0) {
+        *op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, brickname);
+    if (ret)
+        goto out;
+
+    *brickname = gf_strdup(*brickname);
+    if (*brickname == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (dict != NULL) {
-                dict_unref (dict);
-        }
-        loc_wipe(&loc);
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
-
 int
-pl_lockinfo_get_brickname (xlator_t *this, inode_t *inode, int32_t *op_errno)
+pl_lockinfo_get_brickname(xlator_t *this, inode_t *inode, int32_t *op_errno)
 {
-        int                    ret       = -1;
-        posix_locks_private_t *priv      = NULL;
-        char                  *brickname = NULL;
-        char                  *end       = NULL;
-        char                  *tmp       = NULL;
-
-        priv = this->private;
-
-        ret = fetch_pathinfo (this, inode, op_errno, &brickname);
-        if (ret)
-                goto out;
-
-        end = strrchr (brickname, ':');
-        if (!end) {
-                GF_FREE(brickname);
-                ret = -1;
-                goto out;
-        }
+    posix_locks_private_t *priv = this->private;
+    char *brickname = NULL;
+    char *end = NULL;
+    char *tmp = NULL;
 
-        tmp = brickname;
-        brickname = gf_strndup (brickname, (end - brickname));
-        if (brickname == NULL) {
-                ret = -1;
-                goto out;
-        }
+    int ret = fetch_pathinfo(this, inode, op_errno, &brickname);
+    if (ret)
+        goto out;
 
-        priv->brickname = brickname;
-        ret = 0;
+    end = strrchr(brickname, ':');
+    if (!end) {
+        GF_FREE(brickname);
+        ret = -1;
+        goto out;
+    }
+
+    tmp = brickname;
+    brickname = gf_strndup(brickname, (end - brickname));
+    if (brickname == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    priv->brickname = brickname;
+    ret = 0;
 out:
-        GF_FREE(tmp);
-        return ret;
+    GF_FREE(tmp);
+    return ret;
 }
 
 char *
-pl_lockinfo_key (xlator_t *this, inode_t *inode, int32_t *op_errno)
+pl_lockinfo_key(xlator_t *this, inode_t *inode, int32_t *op_errno)
 {
-        posix_locks_private_t *priv = NULL;
-        char                  *key  = NULL;
-        int                    ret = 0;
+    posix_locks_private_t *priv = this->private;
+    char *key = NULL;
+    int ret = 0;
 
-        priv = this->private;
-
-        if (priv->brickname == NULL) {
-                ret = pl_lockinfo_get_brickname (this, inode, op_errno);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot get brickname");
-                        goto out;
-                }
+    if (priv->brickname == NULL) {
+        ret = pl_lockinfo_get_brickname(this, inode, op_errno);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_WARNING, "cannot get brickname");
+            goto out;
         }
+    }
 
-        key = priv->brickname;
+    key = priv->brickname;
 out:
-        return key;
+    return key;
 }
 
 int32_t
-pl_fgetxattr_handle_lockinfo (xlator_t *this, fd_t *fd,
-                              dict_t *dict, int32_t *op_errno)
+pl_fgetxattr_handle_lockinfo(xlator_t *this, fd_t *fd, dict_t *dict,
+                             int32_t *op_errno)
 {
-        pl_inode_t    *pl_inode = NULL;
-        char          *key      = NULL, *buf = NULL;
-        int32_t        op_ret   = 0;
-        unsigned long  fdnum    = 0;
-        int32_t        len      = 0;
-        dict_t        *tmp      = NULL;
-
-        pl_inode = pl_inode_get (this, fd->inode);
-
-        if (!pl_inode) {
-                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
-                *op_errno = EBADFD;
-                op_ret = -1;
-                goto out;
-        }
-
-        if (!pl_locks_by_fd (pl_inode, fd)) {
-                op_ret = 0;
-                goto out;
-        }
-
-        fdnum = fd_to_fdnum (fd);
-
-        key = pl_lockinfo_key (this, fd->inode, op_errno);
-        if (key == NULL) {
-                op_ret = -1;
-                goto out;
-        }
-
-        tmp = dict_new ();
-        if (tmp == NULL) {
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-
-        op_ret = dict_set_uint64 (tmp, key, fdnum);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
-                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
-                        fdnum, fd, uuid_utoa (fd->inode->gfid),
-                        strerror (*op_errno));
-                goto out;
-        }
-
-        len = dict_serialized_length (tmp);
-        if (len < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict_serialized_length failed (%s) while handling "
-                        "lockinfo for fd (ptr:%p inode-gfid:%s)",
-                        strerror (*op_errno), fd, uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        buf = GF_CALLOC (1, len, gf_common_mt_char);
-        if (buf == NULL) {
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-
-        op_ret = dict_serialize (tmp, buf);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict_serialize failed (%s) while handling lockinfo "
-                        "for fd (ptr: %p inode-gfid:%s)", strerror (*op_errno),
-                        fd, uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        op_ret = dict_set_dynptr (dict, GF_XATTR_LOCKINFO_KEY, buf, len);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
-                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
-                        fdnum, fd, uuid_utoa (fd->inode->gfid),
-                        strerror (*op_errno));
-                goto out;
-        }
-
-        buf = NULL;
+    char *key = NULL, *buf = NULL;
+    int32_t op_ret = 0;
+    unsigned long fdnum = 0;
+    int32_t len = 0;
+    dict_t *tmp = NULL;
+
+    pl_inode_t *pl_inode = pl_inode_get(this, fd->inode, NULL);
+
+    if (!pl_inode) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get inode.");
+        *op_errno = EBADFD;
+        op_ret = -1;
+        goto out;
+    }
+
+    if (!pl_locks_by_fd(pl_inode, fd)) {
+        op_ret = 0;
+        goto out;
+    }
+
+    fdnum = fd_to_fdnum(fd);
+
+    key = pl_lockinfo_key(this, fd->inode, op_errno);
+    if (key == NULL) {
+        op_ret = -1;
+        goto out;
+    }
+
+    tmp = dict_new();
+    if (tmp == NULL) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_ret = dict_set_uint64(tmp, key, fdnum);
+    if (op_ret < 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        gf_log(this->name, GF_LOG_WARNING,
+               "setting lockinfo value "
+               "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+               fdnum, fd, uuid_utoa(fd->inode->gfid), strerror(*op_errno));
+        goto out;
+    }
+
+    op_ret = dict_allocate_and_serialize(tmp, (char **)&buf,
+                                         (unsigned int *)&len);
+    if (op_ret != 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        gf_log(this->name, GF_LOG_WARNING,
+               "dict_serialized_length failed (%s) while handling "
+               "lockinfo for fd (ptr:%p inode-gfid:%s)",
+               strerror(*op_errno), fd, uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    op_ret = dict_set_dynptr(dict, GF_XATTR_LOCKINFO_KEY, buf, len);
+    if (op_ret < 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        gf_log(this->name, GF_LOG_WARNING,
+               "setting lockinfo value "
+               "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+               fdnum, fd, uuid_utoa(fd->inode->gfid), strerror(*op_errno));
+        goto out;
+    }
+
+    buf = NULL;
 out:
-        if (tmp != NULL) {
-                dict_unref (tmp);
-        }
+    if (tmp != NULL) {
+        dict_unref(tmp);
+    }
 
-        if (buf != NULL) {
-                GF_FREE (buf);
-        }
+    if (buf != NULL) {
+        GF_FREE(buf);
+    }
 
-        return op_ret;
+    return op_ret;
 }
 
-
 int32_t
-pl_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              const char *name, dict_t *xdata)
+pl_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
 {
-        int32_t  op_ret = 0, op_errno = 0;
-        dict_t  *dict   = NULL;
+    int32_t op_ret = 0, op_errno = 0;
+    dict_t *dict = NULL;
+
+    if (!name) {
+        goto usual;
+    }
 
-        if (!name) {
-                goto usual;
+    if (strcmp(name, GF_XATTR_LOCKINFO_KEY) == 0) {
+        dict = dict_new();
+        if (dict == NULL) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        if (strcmp (name, GF_XATTR_LOCKINFO_KEY) == 0) {
-                dict = dict_new ();
-                if (dict == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        op_ret = pl_fgetxattr_handle_lockinfo(this, fd, dict, &op_errno);
+        if (op_ret < 0) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
+                   "failed (%s)",
+                   fd, uuid_utoa(fd->inode->gfid), strerror(op_errno));
+        }
 
-                op_ret = pl_fgetxattr_handle_lockinfo (this, fd, dict,
-                                                       &op_errno);
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
-                                "failed (%s)", fd, uuid_utoa (fd->inode->gfid),
-                                strerror (op_errno));
-                }
+        goto unwind;
+    } else if (strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) ==
+               0) {
+        op_ret = pl_getxattr_clrlk(this, name, fd->inode, &dict, &op_errno);
 
-                goto unwind;
-        } else {
-                goto usual;
-        }
+        goto unwind;
+    } else {
+        goto usual;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
-        if (dict != NULL) {
-                dict_unref (dict);
-        }
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, NULL);
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
 
-        return 0;
+    return 0;
 
 usual:
-        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-pl_migrate_locks (call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
-                  int32_t *op_errno)
+pl_migrate_locks(call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
+                 int32_t *op_errno)
 {
-        pl_inode_t   *pl_inode  = NULL;
-        uint64_t      newfd_num = 0;
-        posix_lock_t *l         = NULL;
-        int32_t       op_ret    = 0;
-
-        newfd_num = fd_to_fdnum (newfd);
-
-        pl_inode = pl_inode_get (frame->this, newfd->inode);
-        if (pl_inode == NULL) {
-                op_ret = -1;
-                *op_errno = EBADFD;
-                goto out;
-        }
-
-        pthread_mutex_lock (&pl_inode->mutex);
+    posix_lock_t *l = NULL;
+    int32_t op_ret = 0;
+    uint64_t newfd_num = fd_to_fdnum(newfd);
+
+    pl_inode_t *pl_inode = pl_inode_get(frame->this, newfd->inode, NULL);
+    if (pl_inode == NULL) {
+        op_ret = -1;
+        *op_errno = EBADFD;
+        goto out;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
         {
-                list_for_each_entry (l, &pl_inode->ext_list, list) {
-                        if (l->fd_num == oldfd_num) {
-                                l->fd_num = newfd_num;
-                                l->client = frame->root->client;
-                        }
-                }
+            if (l->fd_num == oldfd_num) {
+                l->fd_num = newfd_num;
+                l->client = frame->root->client;
+            }
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        op_ret = 0;
+    op_ret = 0;
 out:
-        return op_ret;
+    return op_ret;
 }
 
 int32_t
-pl_fsetxattr_handle_lockinfo (call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
-                              int len, int32_t *op_errno)
+pl_fsetxattr_handle_lockinfo(call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
+                             int len, int32_t *op_errno)
 {
-        int32_t   op_ret    = -1;
-        dict_t   *lockinfo  = NULL;
-        uint64_t  oldfd_num = 0;
-        char     *key       = NULL;
-
-        lockinfo = dict_new ();
-        if (lockinfo == NULL) {
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
+    int32_t op_ret = -1;
+    uint64_t oldfd_num = 0;
+    char *key = NULL;
+
+    dict_t *lockinfo = dict_new();
+    if (lockinfo == NULL) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
+    if (op_ret < 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        goto out;
+    }
+
+    key = pl_lockinfo_key(frame->this, fd->inode, op_errno);
+    if (key == NULL) {
+        op_ret = -1;
+        goto out;
+    }
+
+    op_ret = dict_get_uint64(lockinfo, key, &oldfd_num);
+
+    if (oldfd_num == 0) {
+        op_ret = 0;
+        goto out;
+    }
+
+    op_ret = pl_migrate_locks(frame, fd, oldfd_num, op_errno);
+    if (op_ret < 0) {
+        gf_log(frame->this->name, GF_LOG_WARNING,
+               "migration of locks from oldfd (ptr:%p) to newfd "
+               "(ptr:%p) (inode-gfid:%s)",
+               (void *)(uintptr_t)oldfd_num, fd, uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
 
-        op_ret = dict_unserialize (lockinfo_buf, len, &lockinfo);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                goto out;
-        }
+out:
+    dict_unref(lockinfo);
 
-        key = pl_lockinfo_key (frame->this, fd->inode, op_errno);
-        if (key == NULL) {
-                op_ret = -1;
-                goto out;
-        }
+    return op_ret;
+}
 
-        op_ret = dict_get_uint64 (lockinfo, key, &oldfd_num);
+int32_t
+pl_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-        if (oldfd_num == 0) {
-                op_ret = 0;
-                goto out;
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        op_ret = pl_migrate_locks (frame, fd, oldfd_num, op_errno);
-        if (op_ret < 0) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "migration of locks from oldfd (ptr:%p) to newfd "
-                        "(ptr:%p) (inode-gfid:%s)", (void *)oldfd_num, fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            pl_inode->mlock_enforced = _gf_true;
+            pl_inode->check_mlock_info = _gf_false;
         }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-out:
-        dict_unref (lockinfo);
-
-        return op_ret;
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(fsetxattr, xdata, frame, op_ret, op_errno,
+                               xdata);
+    return 0;
 }
 
 int32_t
-pl_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+pl_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        int32_t  op_ret       = 0, op_errno = 0;
-        void    *lockinfo_buf = NULL;
-        int      len          = 0;
+    int32_t op_errno = 0;
+    void *lockinfo_buf = NULL;
+    int len = 0;
+    char *name = NULL;
+    posix_locks_private_t *priv = this->private;
+
+    int32_t op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                          &lockinfo_buf, &len);
+    if (lockinfo_buf == NULL) {
+        goto usual;
+    }
+
+    op_ret = pl_fsetxattr_handle_lockinfo(frame, fd, lockinfo_buf, len,
+                                          &op_errno);
+    if (op_ret < 0) {
+        goto unwind;
+    }
 
-        op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                       &lockinfo_buf, &len);
-        if (lockinfo_buf == NULL) {
-                goto usual;
-        }
+usual:
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
 
-        op_ret = pl_fsetxattr_handle_lockinfo (frame, fd, lockinfo_buf, len,
-                                               &op_errno);
-        if (op_ret < 0) {
-                goto unwind;
-        }
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, dict, name, this, ((loc_t *)NULL), fd,
+                              priv);
 
-usual:
-        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, pl_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    PL_STACK_UNWIND_FOR_CLIENT(fsetxattr, xdata, frame, op_ret, op_errno, NULL);
+
+    return 0;
 }
 
 int32_t
-pl_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+pl_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        pl_fdctx_t *fdctx = NULL;
+    pl_fdctx_t *fdctx = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        fdctx = pl_check_n_create_fdctx (this, fd);
-        if (!fdctx) {
-                op_errno = ENOMEM;
-                op_ret   = -1;
-                goto unwind;
-        }
+    fdctx = pl_check_n_create_fdctx(this, fd);
+    if (!fdctx) {
+        op_errno = ENOMEM;
+        op_ret = -1;
+        goto unwind;
+    }
 
 unwind:
-        PL_STACK_UNWIND (opendir, xdata, frame, op_ret, op_errno, fd, xdata);
+    PL_STACK_UNWIND(opendir, xdata, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-pl_opendir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, fd_t *fd, dict_t *xdata)
+pl_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
 {
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, fd, NULL, NULL);
-        STACK_WIND (frame, pl_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-        return 0;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+pl_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+    PL_STACK_UNWIND_FOR_CLIENT(flush, xdata, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-pl_flush (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, dict_t *xdata)
+pl_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        pl_inode_t *pl_inode = NULL;
-
-        pl_inode = pl_inode_get (this, fd->inode);
-
-        if (!pl_inode) {
-                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
-                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD, NULL);
-                return 0;
-        }
+    pl_inode_t *pl_inode = pl_inode_get(this, fd->inode, NULL);
+    if (!pl_inode) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get inode.");
+        STACK_UNWIND_STRICT(flush, frame, -1, EBADFD, NULL);
+        return 0;
+    }
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (pl_inode->migrated) {
-                       pthread_mutex_unlock (&pl_inode->mutex);
-                       STACK_UNWIND_STRICT (flush, frame, -1, EREMOTE,
-                                            NULL);
-                       return 0;
-                }
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (pl_inode->migrated) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            STACK_UNWIND_STRICT(flush, frame, -1, EREMOTE, NULL);
+            return 0;
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        pl_trace_flush (this, frame, fd);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        if (frame->root->lk_owner.len == 0) {
-                /* Handle special case when protocol/server sets lk-owner to zero.
-                 * This usually happens due to a client disconnection. Hence, free
-                 * all locks opened with this fd.
-                 */
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing all locks with fd %p", fd);
-                delete_locks_of_fd (this, pl_inode, fd);
-                goto wind;
+    pl_trace_flush(this, frame, fd);
 
-        }
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __delete_locks_of_owner (pl_inode, frame->root->client,
-                                         &frame->root->lk_owner);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    if (frame->root->lk_owner.len == 0) {
+        /* Handle special case when protocol/server sets lk-owner to zero.
+         * This usually happens due to a client disconnection. Hence, free
+         * all locks opened with this fd.
+         */
+        gf_log(this->name, GF_LOG_TRACE, "Releasing all locks with fd %p", fd);
+        delete_locks_of_fd(this, pl_inode, fd);
+        goto wind;
+    }
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __delete_locks_of_owner(pl_inode, frame->root->client,
+                                &frame->root->lk_owner);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        grant_blocked_locks (this, pl_inode);
+    grant_blocked_locks(this, pl_inode);
 
-        do_blocked_rw (pl_inode);
+    do_blocked_rw(pl_inode);
 
 wind:
-        STACK_WIND (frame, pl_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd, xdata);
-        return 0;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
-
 int
-pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+pl_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        pl_fdctx_t *fdctx = NULL;
+    pl_fdctx_t *fdctx = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        fdctx = pl_check_n_create_fdctx (this, fd);
-        if (!fdctx) {
-                op_errno = ENOMEM;
-                op_ret   = -1;
-                goto unwind;
-        }
+    fdctx = pl_check_n_create_fdctx(this, fd);
+    if (!fdctx) {
+        op_errno = ENOMEM;
+        op_ret = -1;
+        goto unwind;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-pl_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+pl_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        int                     op_ret         = -1;
-        int                     op_errno       = EINVAL;
-        pl_inode_t              *pl_inode      = NULL;
-        posix_lock_t            *l             = NULL;
-        posix_locks_private_t   *priv          = NULL;
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
-
-        op_ret = 0, op_errno = 0;
-        pl_inode = pl_inode_get (this, fd->inode);
-
-        /* As per design, under forced and file-based mandatory locking modes
-         * it doesn't matter whether inodes's lock list contain advisory or
-         * mandatory type locks. So we just check whether inode's lock list is
-         * empty or not to make sure that no locks are being held for the file.
-         * Whereas under optimal mandatory locking mode, we strictly fail open
-         * if and only if lock list contain mandatory locks.
-         */
-        if (((priv->mandatory_mode == MLK_FILE_BASED) && pl_inode->mandatory) ||
-                        priv->mandatory_mode == MLK_FORCED) {
-                if (fd->flags & O_TRUNC) {
-                        pthread_mutex_lock (&pl_inode->mutex);
-                        {
-                                if (!list_empty (&pl_inode->ext_list)) {
-                                        op_ret = -1;
-                                        op_errno = EAGAIN;
-                                }
-                        }
-                        pthread_mutex_unlock (&pl_inode->mutex);
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    pl_inode_t *pl_inode = NULL;
+    posix_lock_t *l = NULL;
+    posix_locks_private_t *priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    op_ret = 0, op_errno = 0;
+    pl_inode = pl_inode_get(this, fd->inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM, "Could not get inode");
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* As per design, under forced and file-based mandatory locking modes
+     * it doesn't matter whether inodes's lock list contain advisory or
+     * mandatory type locks. So we just check whether inode's lock list is
+     * empty or not to make sure that no locks are being held for the file.
+     * Whereas under optimal mandatory locking mode, we strictly fail open
+     * if and only if lock list contain mandatory locks.
+     */
+    if (((priv->mandatory_mode == MLK_FILE_BASED) && pl_inode->mandatory) ||
+        priv->mandatory_mode == MLK_FORCED) {
+        if (fd->flags & O_TRUNC) {
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                if (!list_empty(&pl_inode->ext_list)) {
+                    op_ret = -1;
+                    op_errno = EAGAIN;
                 }
-        } else if (priv->mandatory_mode == MLK_OPTIMAL) {
-                if (fd->flags & O_TRUNC) {
-                        pthread_mutex_lock (&pl_inode->mutex);
-                        {
-                                list_for_each_entry (l, &pl_inode->ext_list, list) {
-                                        if ((l->lk_flags & GF_LK_MANDATORY)) {
-                                                op_ret = -1;
-                                                op_errno = EAGAIN;
-                                                break;
-                                        }
-                                }
-                        }
-                        pthread_mutex_unlock (&pl_inode->mutex);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+        }
+    } else if (priv->mandatory_mode == MLK_OPTIMAL) {
+        if (fd->flags & O_TRUNC) {
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                list_for_each_entry(l, &pl_inode->ext_list, list)
+                {
+                    if ((l->lk_flags & GF_LK_MANDATORY)) {
+                        op_ret = -1;
+                        op_errno = EAGAIN;
+                        break;
+                    }
                 }
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
         }
+    }
 
 unwind:
-        if (op_ret == -1)
-                STACK_UNWIND_STRICT (open, frame, op_ret, op_errno,
-                                                        NULL, NULL);
-        else
-                STACK_WIND (frame, pl_open_cbk, FIRST_CHILD(this),
-                                        FIRST_CHILD(this)->fops->open,
-                                        loc, flags, fd, xdata);
-        return 0;
+    if (op_ret == -1)
+        STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, NULL, NULL);
+    else
+        STACK_WIND(frame, pl_open_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-pl_create_cbk (call_frame_t *frame, void *cookie,
-               xlator_t *this, int32_t op_ret, int32_t op_errno,
-               fd_t *fd, inode_t *inode, struct iatt *buf,
-               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+pl_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        pl_fdctx_t *fdctx = NULL;
+    pl_fdctx_t *fdctx = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        fdctx = pl_check_n_create_fdctx (this, fd);
-        if (!fdctx) {
-                op_errno = ENOMEM;
-                op_ret   = -1;
-                goto unwind;
-        }
+    fdctx = pl_check_n_create_fdctx(this, fd);
+    if (!fdctx) {
+        op_errno = ENOMEM;
+        op_ret = -1;
+        goto unwind;
+    }
 
 unwind:
-        PL_STACK_UNWIND (create, xdata, frame, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent, xdata);
+    PL_STACK_UNWIND(create, xdata, frame, op_ret, op_errno, fd, inode, buf,
+                    preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-pl_create (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-           dict_t *xdata)
+pl_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, NULL, loc, NULL);
-        STACK_WIND (frame, pl_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
-}
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
 
-int32_t
-pl_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
-{
-        PL_STACK_UNWIND (unlink, xdata, frame, op_ret, op_errno, preparent,
-                         postparent, xdata);
-        return 0;
-}
-
-int32_t
-pl_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-           dict_t *xdata)
-{
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, NULL, loc, NULL);
-        STACK_WIND (frame, pl_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, pl_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
 int
-pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno,
-              struct iovec *vector, int32_t count, struct iatt *stbuf,
-              struct iobref *iobref, dict_t *xdata)
+pl_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        PL_STACK_UNWIND (readv, xdata, frame, op_ret, op_errno,
-                         vector, count, stbuf, iobref, xdata);
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
 
-        return 0;
+    PL_STACK_UNWIND(readv, xdata, frame, op_ret, op_errno, vector, count, stbuf,
+                    iobref, xdata);
+
+    return 0;
 }
 
 int
-pl_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
+pl_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        PL_STACK_UNWIND (writev, xdata, frame, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
 
-        return 0;
+    PL_STACK_UNWIND(writev, xdata, frame, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+
+    return 0;
 }
 
 void
-do_blocked_rw (pl_inode_t *pl_inode)
+do_blocked_rw(pl_inode_t *pl_inode)
 {
-        struct list_head  wind_list;
-        pl_rw_req_t      *rw = NULL;
-        pl_rw_req_t      *tmp = NULL;
+    struct list_head wind_list;
+    pl_rw_req_t *rw = NULL;
+    pl_rw_req_t *tmp = NULL;
 
-        INIT_LIST_HEAD (&wind_list);
+    INIT_LIST_HEAD(&wind_list);
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(rw, tmp, &pl_inode->rw_list, list)
         {
-                list_for_each_entry_safe (rw, tmp, &pl_inode->rw_list, list) {
-                        if (__rw_allowable (pl_inode, &rw->region,
-                                            rw->stub->fop)) {
-                                list_del_init (&rw->list);
-                                list_add_tail (&rw->list, &wind_list);
-                        }
+            if (__rw_allowable(pl_inode, &rw->region, rw->stub->fop)) {
+                list_del_init(&rw->list);
+                list_add_tail(&rw->list, &wind_list);
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
                 }
+            }
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        list_for_each_entry_safe (rw, tmp, &wind_list, list) {
-                list_del_init (&rw->list);
-                call_resume (rw->stub);
-                GF_FREE (rw);
-        }
+    list_for_each_entry_safe(rw, tmp, &wind_list, list)
+    {
+        list_del_init(&rw->list);
+        call_resume(rw->stub);
+        GF_FREE(rw);
+    }
 
-        return;
+    return;
+}
+
+/* when mandatory lock is enforced:
+    If an IO request comes on a region which is out of the boundary of the
+    granted mandatory lock, it will be rejected.
+
+    Note: There is no IO blocking with mandatory lock enforced as it may be
+    a stale data from an old client.
+ */
+gf_boolean_t static within_range(posix_lock_t *existing, posix_lock_t *new)
+{
+    if (existing->fl_start <= new->fl_start && existing->fl_end >= new->fl_end)
+        return _gf_true;
+
+    return _gf_false;
 }
 
 static int
-__rw_allowable (pl_inode_t *pl_inode, posix_lock_t *region,
-                glusterfs_fop_t op)
-{
-        posix_lock_t *l = NULL;
-        posix_locks_private_t *priv = NULL;
-        int           ret = 1;
-
-        priv = THIS->private;
-
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (!l->blocked && locks_overlap (l, region)
-                                && !same_owner (l, region)) {
-                        if ((op == GF_FOP_READ) && (l->fl_type != F_WRLCK))
-                                continue;
-                        /* Check for mandatory lock under optimal
-                         * mandatory-locking mode */
-                        if (priv->mandatory_mode == MLK_OPTIMAL
-                                        && !(l->lk_flags & GF_LK_MANDATORY))
-                                continue;
-                        ret = 0;
-                        break;
+__rw_allowable(pl_inode_t *pl_inode, posix_lock_t *region, glusterfs_fop_t op)
+{
+    posix_lock_t *l = NULL;
+    posix_locks_private_t *priv = THIS->private;
+    int ret = 1;
+
+    if (pl_inode->mlock_enforced) {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
+        {
+            /*
+                with lock enforced (fencing) there should not be any blocking
+                lock coexisting.
+            */
+            if (same_owner(l, region)) {
+                /* Should range check be strict for same owner with fencing? */
+                if (locks_overlap(l, region)) {
+                    if (within_range(l, region)) {
+                        return 1;
+                    } else {
+                        /*
+                        Should we allow read fop if it does not fit it in the
+                        range?
+                        if (op == GF_FOP_READ && l->fl_type != F_WRLCK) {
+                            return 1;
+                        }
+                        */
+                        return 0;
+                    }
                 }
+            } else {
+                if (locks_overlap(l, region)) {
+                    /*
+                    with fencing should a read from a different owner be
+                    allowed if the mandatory lock taken is F_RDLCK?
+                    if (op == GF_FOP_READ && l->fl_type != F_WRLCK) {
+                        return 1;
+                    }
+                    */
+                    return 0;
+                }
+            }
         }
 
-        return ret;
+        /* No lock has been taken by this owner */
+        return 0;
+    }
+
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (!l->blocked && locks_overlap(l, region) && !same_owner(l, region)) {
+            if ((op == GF_FOP_READ) && (l->fl_type != F_WRLCK))
+                continue;
+            /* Check for mandatory lock under optimal
+             * mandatory-locking mode */
+            if (priv->mandatory_mode == MLK_OPTIMAL &&
+                !(l->lk_flags & GF_LK_MANDATORY))
+                continue;
+            ret = 0;
+            break;
+        }
+    }
+
+    return ret;
 }
 
 int
-pl_readv_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, uint32_t flags, dict_t *xdata)
+pl_readv_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_readv_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                    fd, size, offset, flags, xdata);
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
 
-        return 0;
+    STACK_WIND(frame, pl_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+
+    return 0;
 }
 
 int
-pl_readv (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+pl_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
 {
-        pl_inode_t            *pl_inode = NULL;
-        pl_rw_req_t           *rw = NULL;
-        posix_lock_t           region = {.list = {0, }, };
-        gf_boolean_t           enabled = _gf_false;
-        gf_boolean_t           can_block = _gf_true;
-        int                    op_ret = 0;
-        int                    op_errno = 0;
-        int                    allowed = 1;
-
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+
+    if (!frame->local) {
+        frame->local = mem_get0(this->local_pool);
+        local = frame->local;
+        local->inode = inode_ref(fd->inode);
+        local->fd = fd_ref(fd);
+    }
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + size - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_READ,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-        pl_inode = pl_inode_get (this, fd->inode);
-        if (!pl_inode) {
-                op_ret   = -1;
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
                 op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, fd, NULL, NULL);
-        enabled = pl_is_mandatory_locking_enabled (pl_inode);
-
-        if (frame->root->pid < 0)
-                enabled = _gf_false;
-
-        if (enabled) {
-                region.fl_start   = offset;
-                region.fl_end     = offset + size - 1;
-                region.client     = frame->root->client;
-                region.fd_num     = fd_to_fdnum(fd);
-                region.client_pid = frame->root->pid;
-                region.owner      = frame->root->lk_owner;
-
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        allowed = pl_is_fop_allowed (pl_inode, &region, fd,
-                                                     GF_FOP_READ, &can_block);
-                        if (allowed == 1)
-                                goto unlock;
-                        else if (!can_block) {
-                                op_errno = EAGAIN;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw = GF_CALLOC (1, sizeof (*rw),
-                                        gf_locks_mt_pl_rw_req_t);
-                        if (!rw) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                goto unlock;
-                        }
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        rw->stub = fop_readv_stub (frame, pl_readv_cont,
-                                                   fd, size, offset, flags,
-                                                   xdata);
-                        if (!rw->stub) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                GF_FREE (rw);
-                                goto unlock;
-                        }
+            rw->stub = fop_readv_stub(frame, pl_readv_cont, fd, size, offset,
+                                      flags, xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-                        rw->region = region;
+            rw->region = region;
 
-                        list_add_tail (&rw->list, &pl_inode->rw_list);
-                }
-        unlock:
-                pthread_mutex_unlock (&pl_inode->mutex);
+            list_add_tail(&rw->list, &pl_inode->rw_list);
         }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        if (allowed == 1) {
-                STACK_WIND (frame, pl_readv_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                            fd, size, offset, flags, xdata);
-        }
+    if (allowed == 1) {
+        STACK_WIND(frame, pl_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    }
 unwind:
-        if (op_ret == -1)
-                STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                                     NULL, 0, NULL, NULL, NULL);
+    if (op_ret == -1)
+        PL_STACK_UNWIND(readv, xdata, frame, op_ret, op_errno, NULL, 0, NULL,
+                        NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int count, off_t offset,
-                uint32_t flags, struct iobref *iobref, dict_t *xdata)
+pl_writev_cont(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int count, off_t offset, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_writev_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
 
-        return 0;
+    STACK_WIND(frame, pl_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+
+    return 0;
 }
 
 int
-pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           struct iovec *vector, int32_t count, off_t offset,
-           uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        pl_inode_t            *pl_inode = NULL;
-        pl_rw_req_t           *rw = NULL;
-        posix_lock_t           region = {.list = {0, }, };
-        gf_boolean_t           enabled = _gf_false;
-        gf_boolean_t           can_block = _gf_true;
-        int                    op_ret = 0;
-        int                    op_errno = 0;
-        int                    allowed = 1;
-
-        GF_VALIDATE_OR_GOTO ("locks", this, unwind);
-
-        pl_inode = pl_inode_get (this, fd->inode);
-        if (!pl_inode) {
-                op_ret   = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, fd, NULL, NULL);
-        enabled = pl_is_mandatory_locking_enabled (pl_inode);
-
-        if (frame->root->pid < 0)
-                enabled = _gf_false;
-
-        if (enabled) {
-                region.fl_start   = offset;
-                region.fl_end     = offset + iov_length (vector, count) - 1;
-                region.client     = frame->root->client;
-                region.fd_num     = fd_to_fdnum(fd);
-                region.client_pid = frame->root->pid;
-                region.owner      = frame->root->lk_owner;
+pl_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+
+    if (!frame->local) {
+        frame->local = mem_get0(this->local_pool);
+        local = frame->local;
+        local->inode = inode_ref(fd->inode);
+        local->fd = fd_ref(fd);
+    }
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + iov_length(vector, count) - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_WRITE,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                if (pl_inode->mlock_enforced) {
+                    op_errno = EBUSY;
+                } else {
+                    op_errno = EAGAIN;
+                }
 
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        allowed = pl_is_fop_allowed (pl_inode, &region, fd,
-                                                     GF_FOP_WRITE, &can_block);
-                        if (allowed == 1)
-                                goto unlock;
-                        else if (!can_block) {
-                                op_errno = EAGAIN;
-                                op_ret = -1;
-                                goto unlock;
-                        }
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        rw = GF_CALLOC (1, sizeof (*rw),
-                                        gf_locks_mt_pl_rw_req_t);
-                        if (!rw) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                goto unlock;
-                        }
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        rw->stub = fop_writev_stub (frame, pl_writev_cont,
-                                                    fd, vector, count, offset,
-                                                    flags, iobref, xdata);
-                        if (!rw->stub) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                GF_FREE (rw);
-                                goto unlock;
-                        }
+            rw->stub = fop_writev_stub(frame, pl_writev_cont, fd, vector, count,
+                                       offset, flags, iobref, xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-                        rw->region = region;
+            rw->region = region;
 
-                        list_add_tail (&rw->list, &pl_inode->rw_list);
-                }
-        unlock:
-                pthread_mutex_unlock (&pl_inode->mutex);
+            list_add_tail(&rw->list, &pl_inode->rw_list);
         }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        if (allowed == 1) {
-                STACK_WIND (frame, pl_writev_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                            fd, vector, count, offset, flags, iobref, xdata);
-        }
+    if (allowed == 1) {
+        STACK_WIND(frame, pl_writev_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+                   flags, iobref, xdata);
+    }
 unwind:
-        if (op_ret == -1)
-                STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno,
-                                     NULL, NULL, NULL);
+    if (op_ret == -1)
+        PL_STACK_UNWIND(writev, xdata, frame, op_ret, op_errno, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
 static int
-__fd_has_locks (pl_inode_t *pl_inode, fd_t *fd)
+__fd_has_locks(pl_inode_t *pl_inode, fd_t *fd)
 {
-        int           found = 0;
-        posix_lock_t *l     = NULL;
+    posix_lock_t *l = NULL;
 
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (l->fd_num == fd_to_fdnum(fd)) {
-                        found = 1;
-                        break;
-                }
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (l->fd_num == fd_to_fdnum(fd)) {
+            return 1;
         }
+    }
 
-        return found;
+    return 0;
 }
 
 static posix_lock_t *
-lock_dup (posix_lock_t *lock)
+lock_dup(posix_lock_t *lock)
 {
-        posix_lock_t *new_lock = NULL;
-
-        new_lock = new_posix_lock (&lock->user_flock, lock->client,
-                                   lock->client_pid, &lock->owner,
-                                   (fd_t *)lock->fd_num, lock->lk_flags,
-                                   lock->blocking);
-        return new_lock;
+    int32_t op_errno = 0;
+    return new_posix_lock(&lock->user_flock, lock->client, lock->client_pid,
+                          &lock->owner, (fd_t *)lock->fd_num, lock->lk_flags,
+                          lock->blocking, &op_errno);
 }
 
 static int
-__dup_locks_to_fdctx (pl_inode_t *pl_inode, fd_t *fd,
-                      pl_fdctx_t *fdctx)
-{
-        posix_lock_t *l        = NULL;
-        posix_lock_t *duplock = NULL;
-        int ret = 0;
-
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (l->fd_num == fd_to_fdnum(fd)) {
-                        duplock = lock_dup (l);
-                        if (!duplock) {
-                                ret = -1;
-                                break;
-                        }
+__dup_locks_to_fdctx(pl_inode_t *pl_inode, fd_t *fd, pl_fdctx_t *fdctx)
+{
+    posix_lock_t *l = NULL;
+    posix_lock_t *duplock = NULL;
+    int ret = 0;
+
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (l->fd_num == fd_to_fdnum(fd)) {
+            duplock = lock_dup(l);
+            if (!duplock) {
+                ret = -1;
+                break;
+            }
 
-                        list_add_tail (&duplock->list, &fdctx->locks_list);
-                }
+            list_add_tail(&duplock->list, &fdctx->locks_list);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-__copy_locks_to_fdctx (pl_inode_t *pl_inode, fd_t *fd,
-                      pl_fdctx_t *fdctx)
+__copy_locks_to_fdctx(pl_inode_t *pl_inode, fd_t *fd, pl_fdctx_t *fdctx)
 {
-        int ret = 0;
-
-        ret = __dup_locks_to_fdctx (pl_inode, fd, fdctx);
-        if (ret)
-                goto out;
-
-out:
-        return ret;
-
+    return __dup_locks_to_fdctx(pl_inode, fd, fdctx);
 }
 
 static void
-pl_mark_eol_lock (posix_lock_t *lock)
+pl_mark_eol_lock(posix_lock_t *lock)
 {
-        lock->user_flock.l_type = GF_LK_EOL;
-        return;
+    lock->user_flock.l_type = GF_LK_EOL;
+    return;
 }
 
 static posix_lock_t *
-__get_next_fdctx_lock (pl_fdctx_t *fdctx)
+__get_next_fdctx_lock(pl_fdctx_t *fdctx)
 {
-        posix_lock_t *lock = NULL;
+    posix_lock_t *lock = NULL;
 
-        GF_ASSERT (fdctx);
+    GF_ASSERT(fdctx);
 
-        if (list_empty (&fdctx->locks_list)) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "fdctx lock list empty");
-                goto out;
-        }
+    if (list_empty(&fdctx->locks_list)) {
+        gf_log(THIS->name, GF_LOG_DEBUG, "fdctx lock list empty");
+        goto out;
+    }
 
-        lock = list_entry (fdctx->locks_list.next, typeof (*lock),
-                           list);
+    lock = list_entry(fdctx->locks_list.next, typeof(*lock), list);
 
-        GF_ASSERT (lock);
+    GF_ASSERT(lock);
 
-        list_del_init (&lock->list);
+    list_del_init(&lock->list);
 
 out:
-        return lock;
+    return lock;
 }
 
 static int
-__set_next_lock_fd (pl_fdctx_t *fdctx, posix_lock_t *reqlock)
+__set_next_lock_fd(pl_fdctx_t *fdctx, posix_lock_t *reqlock)
 {
-        posix_lock_t *lock  = NULL;
-        int           ret   = 0;
+    posix_lock_t *lock = NULL;
+    int ret = 0;
 
-        GF_ASSERT (fdctx);
+    GF_ASSERT(fdctx);
 
-        lock = __get_next_fdctx_lock (fdctx);
-        if (!lock) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "marking EOL in reqlock");
-                pl_mark_eol_lock (reqlock);
-                goto out;
-        }
+    lock = __get_next_fdctx_lock(fdctx);
+    if (!lock) {
+        gf_log(THIS->name, GF_LOG_DEBUG, "marking EOL in reqlock");
+        pl_mark_eol_lock(reqlock);
+        goto out;
+    }
 
-        reqlock->user_flock  = lock->user_flock;
-        reqlock->fl_start    = lock->fl_start;
-        reqlock->fl_type     = lock->fl_type;
-        reqlock->fl_end      = lock->fl_end;
-        reqlock->owner       = lock->owner;
+    reqlock->user_flock = lock->user_flock;
+    reqlock->fl_start = lock->fl_start;
+    reqlock->fl_type = lock->fl_type;
+    reqlock->fl_end = lock->fl_end;
+    reqlock->owner = lock->owner;
 
 out:
-        if (lock)
-                __destroy_lock (lock);
+    if (lock)
+        __destroy_lock(lock);
 
-        return ret;
+    return ret;
 }
 
 static int
-pl_getlk_fd (xlator_t *this, pl_inode_t *pl_inode,
-             fd_t *fd, posix_lock_t *reqlock)
+pl_getlk_fd(xlator_t *this, pl_inode_t *pl_inode, fd_t *fd,
+            posix_lock_t *reqlock)
 {
-        uint64_t    tmp   = 0;
-        pl_fdctx_t *fdctx = NULL;
-        int         ret   = 0;
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
+    int ret = 0;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (!__fd_has_locks (pl_inode, fd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p has no active locks", fd);
-                        ret = 0;
-                        goto unlock;
-                }
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (!__fd_has_locks(pl_inode, fd)) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_log(this->name, GF_LOG_DEBUG, "fd=%p has no active locks", fd);
+            ret = 0;
+            goto out;
+        }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "There are active locks on fd");
+        gf_log(this->name, GF_LOG_DEBUG, "There are active locks on fd");
 
-                ret = fd_ctx_get (fd, this, &tmp);
-                fdctx = (pl_fdctx_t *)(long) tmp;
+        ret = fd_ctx_get(fd, this, &tmp);
+        fdctx = (pl_fdctx_t *)(long)tmp;
 
-                if (list_empty (&fdctx->locks_list)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "no fdctx -> copying all locks on fd");
+        if (list_empty(&fdctx->locks_list)) {
+            gf_log(this->name, GF_LOG_TRACE,
+                   "no fdctx -> copying all locks on fd");
 
-                        ret = __copy_locks_to_fdctx (pl_inode, fd, fdctx);
-                        if (ret) {
-                                goto unlock;
-                        }
+            ret = __copy_locks_to_fdctx(pl_inode, fd, fdctx);
+            if (ret) {
+                goto unlock;
+            }
 
-                        ret = __set_next_lock_fd (fdctx, reqlock);
+            ret = __set_next_lock_fd(fdctx, reqlock);
 
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fdctx present -> returning the next lock");
-                        ret = __set_next_lock_fd (fdctx, reqlock);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "could not get next lock of fd");
-                                goto unlock;
-                        }
-                }
+        } else {
+            gf_log(this->name, GF_LOG_TRACE,
+                   "fdctx present -> returning the next lock");
+            ret = __set_next_lock_fd(fdctx, reqlock);
+            if (ret) {
+                pthread_mutex_unlock(&pl_inode->mutex);
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "could not get next lock of fd");
+                goto out;
+            }
         }
+    }
 
 unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
-        return ret;
-
+    pthread_mutex_unlock(&pl_inode->mutex);
+out:
+    return ret;
 }
 
 int
-pl_metalock_is_active (pl_inode_t *pl_inode)
+pl_metalock_is_active(pl_inode_t *pl_inode)
 {
-        if (list_empty (&pl_inode->metalk_list))
-                return 0;
-        else
-                return 1;
+    if (list_empty(&pl_inode->metalk_list))
+        return 0;
+    else
+        return 1;
 }
 
-int
-__pl_queue_lock (pl_inode_t *pl_inode, posix_lock_t *reqlock, int can_block)
+void
+__pl_queue_lock(pl_inode_t *pl_inode, posix_lock_t *reqlock)
 {
-        list_add_tail (&reqlock->list, &pl_inode->queued_locks);
-
-        return 0;
+    list_add_tail(&reqlock->list, &pl_inode->queued_locks);
 }
 
 int
-pl_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
-{
-        pl_inode_t   *pl_inode   = NULL;
-        int           op_ret     = 0;
-        int           op_errno   = 0;
-        int           can_block  = 0;
-        posix_lock_t *reqlock    = NULL;
-        posix_lock_t *conf       = NULL;
-        int           ret        = 0;
-        uint32_t      lk_flags   = 0;
-        posix_locks_private_t *priv = NULL;
-
-        priv = this->private;
-
-        ret = dict_get_uint32 (xdata, "lkmode", &lk_flags);
-        if (ret == 0) {
-                if (priv->mandatory_mode == MLK_NONE)
-                        gf_log (this->name, GF_LOG_DEBUG, "Lock flags received "
-                                "in a non-mandatory locking environment, "
-                                "continuing");
-                else
-                        gf_log (this->name, GF_LOG_DEBUG, "Lock flags received, "
-                                "continuing");
-        }
+pl_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    pl_inode_t *pl_inode = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+    int can_block = 0;
+    posix_lock_t *reqlock = NULL;
+    posix_lock_t *conf = NULL;
+    uint32_t lk_flags = 0;
+    posix_locks_private_t *priv = this->private;
+    pl_local_t *local = NULL;
+    short lock_type = 0;
+
+    int ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_flags);
+    if (ret == 0) {
+        if (priv->mandatory_mode == MLK_NONE)
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock flags received "
+                   "in a non-mandatory locking environment, "
+                   "continuing");
+        else
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock flags received, "
+                   "continuing");
+    }
+
+    if ((flock->l_start < 0) || ((flock->l_start + flock->l_len) < 0)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    /* As per 'man 3 fcntl', the value of l_len may be
+     * negative. In such cases, lock request should be
+     * considered for the range starting at 'l_start+l_len'
+     * and ending at 'l_start-1'. Update the fields accordingly.
+     */
+    if (flock->l_len < 0) {
+        flock->l_start += flock->l_len;
+        flock->l_len = labs(flock->l_len);
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    } else {
+        frame->local = local;
+        local->fd = fd_ref(fd);
+    }
 
-        if ((flock->l_start < 0) ||
-            ((flock->l_start + flock->l_len) < 0)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
-        }
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        /* As per 'man 3 fcntl', the value of l_len may be
-         * negative. In such cases, lock request should be
-         * considered for the range starting at 'l_start+l_len'
-         * and ending at 'l_start-1'. Update the fields accordingly.
-         */
-        if (flock->l_len < 0) {
-                flock->l_start += flock->l_len;
-                flock->l_len = labs (flock->l_len);
-        }
+    reqlock = new_posix_lock(flock, frame->root->client, frame->root->pid,
+                             &frame->root->lk_owner, fd, lk_flags, can_block,
+                             &op_errno);
 
-        pl_inode = pl_inode_get (this, fd->inode);
-        if (!pl_inode) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        reqlock = new_posix_lock (flock, frame->root->client, frame->root->pid,
-                                  &frame->root->lk_owner, fd, lk_flags,
-                                  can_block);
+    if (!reqlock) {
+        op_ret = -1;
+        goto unwind;
+    }
 
-        if (!reqlock) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        pl_trace_in (this, frame, fd, NULL, cmd, flock, NULL);
-
-        switch (cmd) {
+    pl_trace_in(this, frame, fd, NULL, cmd, flock, NULL);
 
+    switch (cmd) {
         case F_RESLK_LCKW:
-                can_block = 1;
+            can_block = 1;
 
-                /* fall through */
+            /* fall through */
         case F_RESLK_LCK:
-                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
-                reqlock->frame = frame;
-                reqlock->this = this;
+            reqlock->frame = frame;
+            reqlock->this = this;
 
-                ret = pl_reserve_setlk (this, pl_inode, reqlock,
-                                        can_block);
-                if (ret < 0) {
-                        if (can_block)
-                                goto out;
+            ret = pl_reserve_setlk(this, pl_inode, reqlock, can_block);
+            if (ret < 0) {
+                if (can_block)
+                    goto out;
 
-                        op_ret = -1;
-                        op_errno = -ret;
-                        __destroy_lock (reqlock);
-                        goto unwind;
-                }
-                /* Finally a getlk and return the call */
-                conf = pl_getlk (pl_inode, reqlock);
-                if (conf)
-                        posix_lock_to_flock (conf, flock);
-                break;
+                op_ret = -1;
+                op_errno = -ret;
+                __destroy_lock(reqlock);
+                goto unwind;
+            }
+            /* Finally a getlk and return the call */
+            conf = pl_getlk(pl_inode, reqlock);
+            if (conf)
+                posix_lock_to_flock(conf, flock);
+            break;
 
         case F_RESLK_UNLCK:
-                reqlock->frame = frame;
-                reqlock->this = this;
-                ret = pl_reserve_unlock (this, pl_inode, reqlock);
-                if (ret < 0) {
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-                __destroy_lock (reqlock);
-                goto unwind;
+            reqlock->frame = frame;
+            reqlock->this = this;
+            ret = pl_reserve_unlock(this, pl_inode, reqlock);
+            if (ret < 0) {
+                op_ret = -1;
+                op_errno = -ret;
+            }
+            __destroy_lock(reqlock);
+            goto unwind;
 
-                break;
+            break;
 
         case F_GETLK_FD:
-                reqlock->frame = frame;
-                reqlock->this = this;
-                ret = pl_verify_reservelk (this, pl_inode, reqlock, can_block);
-                GF_ASSERT (ret >= 0);
-
-                ret = pl_getlk_fd (this, pl_inode, fd, reqlock);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "getting locks on fd failed");
-                        op_ret = -1;
-                        op_errno = ENOLCK;
-                        goto unwind;
-                }
+            reqlock->frame = frame;
+            reqlock->this = this;
+            ret = pl_verify_reservelk(this, pl_inode, reqlock, can_block);
+            GF_ASSERT(ret >= 0);
+
+            ret = pl_getlk_fd(this, pl_inode, fd, reqlock);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_DEBUG, "getting locks on fd failed");
+                op_ret = -1;
+                op_errno = ENOLCK;
+                goto unwind;
+            }
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Replying with a lock on fd for healing");
+            gf_log(this->name, GF_LOG_TRACE,
+                   "Replying with a lock on fd for healing");
 
-                posix_lock_to_flock (reqlock, flock);
-                __destroy_lock (reqlock);
+            posix_lock_to_flock(reqlock, flock);
+            __destroy_lock(reqlock);
 
-                break;
+            break;
 
 #if F_GETLK != F_GETLK64
         case F_GETLK64:
 #endif
         case F_GETLK:
-                conf = pl_getlk (pl_inode, reqlock);
-                posix_lock_to_flock (conf, flock);
-                __destroy_lock (reqlock);
+            conf = pl_getlk(pl_inode, reqlock);
+            posix_lock_to_flock(conf, flock);
+            __destroy_lock(reqlock);
 
-                break;
+            break;
 
 #if F_SETLKW != F_SETLKW64
         case F_SETLKW64:
 #endif
         case F_SETLKW:
-                can_block = 1;
-                reqlock->frame  = frame;
-                reqlock->this   = this;
-                reqlock->blocking = can_block;
-                /* fall through */
+            can_block = 1;
+            reqlock->frame = frame;
+            reqlock->this = this;
+            reqlock->blocking = can_block;
+            /* fall through */
 
 #if F_SETLK != F_SETLK64
         case F_SETLK64:
 #endif
         case F_SETLK:
-                reqlock->frame  = frame;
-                reqlock->this   = this;
+            reqlock->frame = frame;
+            reqlock->this = this;
+            lock_type = flock->l_type;
 
-                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                if (pl_inode->migrated) {
+                    op_errno = EREMOTE;
+                    pthread_mutex_unlock(&pl_inode->mutex);
+                    STACK_UNWIND_STRICT(lk, frame, -1, op_errno, flock, xdata);
 
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        if (pl_inode->migrated) {
-                                op_errno = EREMOTE;
-                                pthread_mutex_unlock (&pl_inode->mutex);
-                                STACK_UNWIND_STRICT (lk, frame, -1,
-                                                     op_errno, flock, xdata);
-
-                                __destroy_lock (reqlock);
-                                goto out;
-                        }
+                    __destroy_lock(reqlock);
+                    goto out;
                 }
-                pthread_mutex_unlock (&pl_inode->mutex);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
 
-                ret = pl_verify_reservelk (this, pl_inode, reqlock, can_block);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Lock blocked due to conflicting reserve lock");
-                        goto out;
-                }
+            ret = pl_verify_reservelk(this, pl_inode, reqlock, can_block);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_TRACE,
+                       "Lock blocked due to conflicting reserve lock");
+                goto out;
+            }
 
-                ret = pl_setlk (this, pl_inode, reqlock, can_block);
+            if (reqlock->fl_type != F_UNLCK && pl_inode->mlock_enforced) {
+                ret = pl_lock_preempt(pl_inode, reqlock);
                 if (ret == -1) {
-                        if ((can_block) && (F_UNLCK != flock->l_type)) {
-                                pl_trace_block (this, frame, fd, NULL, cmd, flock, NULL);
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_DEBUG, "returning EAGAIN");
-                        op_ret = -1;
-                        op_errno = EAGAIN;
-                        __destroy_lock (reqlock);
-                } else if (ret == -2) {
-                        goto out;
-                } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
-                        /* For NLM's last "unlock on fd" detection */
-                        if (pl_locks_by_fd (pl_inode, fd))
-                                flock->l_type = F_RDLCK;
-                        else
-                                flock->l_type = F_UNLCK;
+                    gf_log(this->name, GF_LOG_ERROR, "lock preempt failed");
+                    op_ret = -1;
+                    op_errno = EAGAIN;
+                    __destroy_lock(reqlock);
+                    goto out;
                 }
-        }
 
-unwind:
-        pl_trace_out (this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
-        pl_update_refkeeper (this, fd->inode);
+                pl_trace_block(this, frame, fd, NULL, cmd, flock, NULL);
+                goto unwind;
+            }
 
+            ret = pl_setlk(this, pl_inode, reqlock, can_block);
+            if (ret == -1) {
+                if ((can_block) && (F_UNLCK != lock_type)) {
+                    goto out;
+                }
+                gf_log(this->name, GF_LOG_DEBUG, "returning EAGAIN");
+                op_ret = -1;
+                op_errno = EAGAIN;
+                __destroy_lock(reqlock);
+            } else if (ret == -2) {
+                goto out;
+            } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
+                /* For NLM's last "unlock on fd" detection */
+                if (pl_locks_by_fd(pl_inode, fd))
+                    flock->l_type = F_RDLCK;
+                else
+                    flock->l_type = F_UNLCK;
+            }
+    }
+
+unwind:
+    pl_trace_out(this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
+    pl_update_refkeeper(this, fd->inode);
 
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock, xdata);
+    PL_STACK_UNWIND(lk, xdata, frame, op_ret, op_errno, flock, xdata);
 out:
-        return 0;
+    return 0;
 }
 
-
 /* TODO: this function just logs, no action required?? */
 int
-pl_forget (xlator_t *this,
-           inode_t *inode)
+pl_forget(xlator_t *this, inode_t *inode)
 {
-        pl_inode_t   *pl_inode = NULL;
-
-        posix_lock_t *ext_tmp = NULL;
-        posix_lock_t *ext_l   = NULL;
-        struct list_head posixlks_released;
+    pl_inode_t *pl_inode = NULL;
 
-        pl_inode_lock_t *ino_tmp = NULL;
-        pl_inode_lock_t *ino_l   = NULL;
-        struct list_head inodelks_released;
+    posix_lock_t *ext_tmp = NULL;
+    posix_lock_t *ext_l = NULL;
+    struct list_head posixlks_released;
 
-        pl_rw_req_t *rw_tmp = NULL;
-        pl_rw_req_t *rw_req = NULL;
+    pl_inode_lock_t *ino_tmp = NULL;
+    pl_inode_lock_t *ino_l = NULL;
+    struct list_head inodelks_released;
 
-        pl_entry_lock_t *entry_tmp = NULL;
-        pl_entry_lock_t *entry_l   = NULL;
-        struct list_head entrylks_released;
+    pl_rw_req_t *rw_tmp = NULL;
+    pl_rw_req_t *rw_req = NULL;
 
-        pl_dom_list_t *dom = NULL;
-        pl_dom_list_t *dom_tmp = NULL;
+    pl_entry_lock_t *entry_tmp = NULL;
+    pl_entry_lock_t *entry_l = NULL;
+    struct list_head entrylks_released;
 
-        INIT_LIST_HEAD (&posixlks_released);
-        INIT_LIST_HEAD (&inodelks_released);
-        INIT_LIST_HEAD (&entrylks_released);
+    pl_dom_list_t *dom = NULL;
+    pl_dom_list_t *dom_tmp = NULL;
 
-        pl_inode = pl_inode_get (this, inode);
+    INIT_LIST_HEAD(&posixlks_released);
+    INIT_LIST_HEAD(&inodelks_released);
+    INIT_LIST_HEAD(&entrylks_released);
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode)
+        return 0;
 
-                if (!list_empty (&pl_inode->rw_list)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Pending R/W requests found, releasing.");
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (!list_empty(&pl_inode->rw_list)) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Pending R/W requests found, releasing.");
+
+            list_for_each_entry_safe(rw_req, rw_tmp, &pl_inode->rw_list, list)
+            {
+                list_del(&rw_req->list);
+                call_stub_destroy(rw_req->stub);
+                GF_FREE(rw_req);
+            }
+        }
+
+        if (!list_empty(&pl_inode->ext_list)) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Pending fcntl locks found, releasing.");
+
+            list_for_each_entry_safe(ext_l, ext_tmp, &pl_inode->ext_list, list)
+            {
+                __delete_lock(ext_l);
+                if (ext_l->blocked) {
+                    list_add_tail(&ext_l->list, &posixlks_released);
+                    continue;
+                }
+                __destroy_lock(ext_l);
+            }
+        }
 
-                        list_for_each_entry_safe (rw_req, rw_tmp, &pl_inode->rw_list,
-                                                  list) {
+        list_for_each_entry_safe(dom, dom_tmp, &pl_inode->dom_list, inode_list)
+        {
+            if (!list_empty(&dom->inodelk_list)) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Pending inode locks found, releasing.");
 
-                                list_del (&rw_req->list);
-                                GF_FREE (rw_req);
-                        }
+                list_for_each_entry_safe(ino_l, ino_tmp, &dom->inodelk_list,
+                                         list)
+                {
+                    __delete_inode_lock(ino_l);
+                    __pl_inodelk_unref(ino_l);
                 }
 
-                if (!list_empty (&pl_inode->ext_list)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Pending fcntl locks found, releasing.");
+                list_splice_init(&dom->blocked_inodelks, &inodelks_released);
+            }
+            if (!list_empty(&dom->entrylk_list)) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Pending entry locks found, releasing.");
 
-                        list_for_each_entry_safe (ext_l, ext_tmp, &pl_inode->ext_list,
-                                                  list) {
+                list_for_each_entry_safe(entry_l, entry_tmp, &dom->entrylk_list,
+                                         domain_list)
+                {
+                    list_del_init(&entry_l->domain_list);
 
-                                __delete_lock (ext_l);
-                                if (ext_l->blocked) {
-                                        list_add_tail (&ext_l->list, &posixlks_released);
-                                        continue;
-                                }
-                                __destroy_lock (ext_l);
-                        }
+                    GF_FREE((char *)entry_l->basename);
+                    GF_FREE(entry_l->connection_id);
+                    GF_FREE(entry_l);
                 }
 
+                list_splice_init(&dom->blocked_entrylks, &entrylks_released);
+            }
 
-                list_for_each_entry_safe (dom, dom_tmp, &pl_inode->dom_list, inode_list) {
+            list_del(&dom->inode_list);
+            gf_log("posix-locks", GF_LOG_TRACE, " Cleaning up domain: %s",
+                   dom->domain);
+            GF_FREE((char *)(dom->domain));
+            GF_FREE(dom);
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (!list_empty(&posixlks_released)) {
+        list_for_each_entry_safe(ext_l, ext_tmp, &posixlks_released, list)
+        {
+            STACK_UNWIND_STRICT(lk, ext_l->frame, -1, 0, &ext_l->user_flock,
+                                NULL);
+            __destroy_lock(ext_l);
+        }
+    }
+
+    if (!list_empty(&inodelks_released)) {
+        list_for_each_entry_safe(ino_l, ino_tmp, &inodelks_released,
+                                 blocked_locks)
+        {
+            STACK_UNWIND_STRICT(inodelk, ino_l->frame, -1, 0, NULL);
+            __pl_inodelk_unref(ino_l);
+        }
+    }
 
-                        if (!list_empty (&dom->inodelk_list)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Pending inode locks found, releasing.");
+    if (!list_empty(&entrylks_released)) {
+        list_for_each_entry_safe(entry_l, entry_tmp, &entrylks_released,
+                                 blocked_locks)
+        {
+            STACK_UNWIND_STRICT(entrylk, entry_l->frame, -1, 0, NULL);
+            GF_FREE((char *)entry_l->basename);
+            GF_FREE(entry_l->connection_id);
+            GF_FREE(entry_l);
+        }
+    }
 
-                                list_for_each_entry_safe (ino_l, ino_tmp, &dom->inodelk_list, list) {
-                                        __delete_inode_lock (ino_l);
-                                        __pl_inodelk_unref (ino_l);
-                                }
+    pthread_mutex_destroy(&pl_inode->mutex);
 
-                                list_splice_init (&dom->blocked_inodelks, &inodelks_released);
+    GF_FREE(pl_inode);
 
+    return 0;
+}
 
-                        }
-                        if (!list_empty (&dom->entrylk_list)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Pending entry locks found, releasing.");
+int
+pl_release(xlator_t *this, fd_t *fd)
+{
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int ret = -1;
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
 
-                                list_for_each_entry_safe (entry_l, entry_tmp, &dom->entrylk_list, domain_list) {
-                                        list_del_init (&entry_l->domain_list);
+    if (fd == NULL) {
+        goto out;
+    }
 
-                                        GF_FREE ((char *)entry_l->basename);
-                                        GF_FREE (entry_l->connection_id);
-                                        GF_FREE (entry_l);
-                                }
+    ret = inode_ctx_get(fd->inode, this, &tmp_pl_inode);
+    if (ret != 0)
+        goto clean;
 
-                                list_splice_init (&dom->blocked_entrylks, &entrylks_released);
-                        }
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
 
-                        list_del (&dom->inode_list);
-                        gf_log ("posix-locks", GF_LOG_TRACE,
-                                " Cleaning up domain: %s", dom->domain);
-                        GF_FREE ((char *)(dom->domain));
-                        GF_FREE (dom);
-                }
+    pl_trace_release(this, fd);
 
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    gf_log(this->name, GF_LOG_TRACE, "Releasing all locks with fd %p", fd);
 
-        list_for_each_entry_safe (ext_l, ext_tmp, &posixlks_released, list) {
+    delete_locks_of_fd(this, pl_inode, fd);
+    pl_update_refkeeper(this, fd->inode);
 
-                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0,
-                                     &ext_l->user_flock, NULL);
-                __destroy_lock (ext_l);
-        }
+clean:
+    ret = fd_ctx_del(fd, this, &tmp);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get fdctx");
+        goto out;
+    }
 
-        list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
+    fdctx = (pl_fdctx_t *)(long)tmp;
 
-                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0, NULL);
-                __pl_inodelk_unref (ino_l);
-        }
+    GF_FREE(fdctx);
+out:
+    return ret;
+}
 
-        list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {
+int
+pl_releasedir(xlator_t *this, fd_t *fd)
+{
+    int ret = -1;
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
 
-                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0, NULL);
-                GF_FREE ((char *)entry_l->basename);
-                GF_FREE (entry_l->connection_id);
-                GF_FREE (entry_l);
+    if (fd == NULL) {
+        goto out;
+    }
 
-        }
+    ret = fd_ctx_del(fd, this, &tmp);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get fdctx");
+        goto out;
+    }
 
-        GF_FREE (pl_inode);
+    fdctx = (pl_fdctx_t *)(long)tmp;
 
-        return 0;
+    GF_FREE(fdctx);
+out:
+    return ret;
 }
 
-int
-pl_release (xlator_t *this, fd_t *fd)
+static int32_t
+pl_request_link_count(dict_t **pxdata)
 {
-        pl_inode_t *pl_inode     = NULL;
-        uint64_t    tmp_pl_inode = 0;
-        int         ret          = -1;
-        uint64_t    tmp          = 0;
-        pl_fdctx_t *fdctx        = NULL;
+    dict_t *xdata;
 
-        if (fd == NULL) {
-                goto out;
+    xdata = *pxdata;
+    if (xdata == NULL) {
+        xdata = dict_new();
+        if (xdata == NULL) {
+            return ENOMEM;
         }
+    } else {
+        dict_ref(xdata);
+    }
 
-        ret = inode_ctx_get (fd->inode, this, &tmp_pl_inode);
-        if (ret != 0)
-                goto out;
+    if (dict_set_uint32(xdata, GET_LINK_COUNT, 0) != 0) {
+        dict_unref(xdata);
+        return ENOMEM;
+    }
 
-        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+    *pxdata = xdata;
 
-        pl_trace_release (this, fd);
+    return 0;
+}
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "Releasing all locks with fd %p", fd);
+static int32_t
+pl_check_link_count(dict_t *xdata)
+{
+    int32_t count;
 
-        delete_locks_of_fd (this, pl_inode, fd);
-        pl_update_refkeeper (this, fd->inode);
+    /* In case we are unable to read the link count from xdata, we take a
+     * conservative approach and return -2, which will prevent the inode from
+     * being considered deleted. In fact it will cause link tracking for this
+     * inode to be disabled completely to avoid races. */
 
-        ret = fd_ctx_del (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Could not get fdctx");
-                goto out;
-        }
+    if (xdata == NULL) {
+        return -2;
+    }
 
-        fdctx = (pl_fdctx_t *)(long)tmp;
+    if (dict_get_int32(xdata, GET_LINK_COUNT, &count) != 0) {
+        return -2;
+    }
 
-        GF_FREE (fdctx);
-out:
-        return ret;
+    return count;
 }
 
-int
-pl_releasedir (xlator_t *this, fd_t *fd)
+int32_t
+pl_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-        int         ret          = -1;
-        uint64_t    tmp          = 0;
-        pl_fdctx_t *fdctx        = NULL;
+    pl_inode_t *pl_inode;
 
-        if (fd == NULL) {
-                goto out;
+    if (op_ret >= 0) {
+        pl_inode = pl_inode_get(this, inode, NULL);
+        if (pl_inode == NULL) {
+            PL_STACK_UNWIND(lookup, xdata, frame, -1, ENOMEM, NULL, NULL, NULL,
+                            NULL);
+            return 0;
         }
 
-        ret = fd_ctx_del (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Could not get fdctx");
-                goto out;
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        /* We only update the link count if we previously didn't know it.
+         * Doing it always can lead to races since lookup is not executed
+         * atomically most of the times. */
+        if (pl_inode->links == -2) {
+            pl_inode->links = pl_check_link_count(xdata);
+            if (buf->ia_type == IA_IFDIR) {
+                /* Directories have at least 2 links. To avoid special handling
+                 * for directories, we simply decrement the value here to make
+                 * them equivalent to regular files. */
+                pl_inode->links--;
+            }
         }
 
-        fdctx = (pl_fdctx_t *)(long)tmp;
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        GF_FREE (fdctx);
-out:
-        return ret;
+    PL_STACK_UNWIND(lookup, xdata, frame, op_ret, op_errno, inode, buf, xdata,
+                    postparent);
+    return 0;
 }
 
 int32_t
-pl_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+pl_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        PL_STACK_UNWIND (lookup, xdata, frame, op_ret, op_errno, inode, buf,
-                         xdata, postparent);
-        return 0;
+    int32_t error;
+
+    error = pl_request_link_count(&xdata);
+    if (error == 0) {
+        PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+        STACK_WIND(frame, pl_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        dict_unref(xdata);
+    } else {
+        STACK_UNWIND_STRICT(lookup, frame, -1, error, NULL, NULL, NULL, NULL);
+    }
+    return 0;
 }
 
 int32_t
-pl_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+pl_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, NULL, loc, NULL);
-        STACK_WIND (frame, pl_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
+    PL_STACK_UNWIND(fstat, xdata, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
 int32_t
-pl_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *buf, dict_t *xdata)
+pl_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        PL_STACK_UNWIND (fstat, xdata, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int32_t
-pl_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, fd, NULL, NULL);
-        STACK_WIND (frame, pl_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
 int
-pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
+pl_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-        pl_local_t *local  = NULL;
-        gf_dirent_t *entry = NULL;
+    pl_local_t *local = NULL;
+    gf_dirent_t *entry = NULL;
 
-        if (op_ret <= 0)
-                goto unwind;
+    if (op_ret <= 0)
+        goto unwind;
 
-        local = frame->local;
-        if (!local)
-                goto unwind;
+    local = frame->local;
+    if (!local)
+        goto unwind;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                pl_set_xdata_response (this, local, local->fd->inode,
-                                       entry->inode, entry->d_name,
-                                       entry->dict, 0);
-        }
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        pl_set_xdata_response(this, local, local->fd->inode, entry->inode,
+                              entry->d_name, entry->dict, 0);
+    }
 
 unwind:
-        PL_STACK_UNWIND (readdirp, xdata, frame, op_ret, op_errno, entries,
-                         xdata);
+    PL_STACK_UNWIND(readdirp, xdata, frame, op_ret, op_errno, entries, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
+pl_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, fd, NULL, NULL);
-        STACK_WIND (frame, pl_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, xdata);
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 lock_migration_info_t *
-gf_mig_info_for_lock (posix_lock_t *lock)
+gf_mig_info_for_lock(posix_lock_t *lock)
 {
-        lock_migration_info_t   *new    = NULL;
-
-        new = GF_CALLOC (1, sizeof (lock_migration_info_t),
-                         gf_common_mt_lock_mig);
-        if (new == NULL) {
-                goto out;
-        }
+    lock_migration_info_t *new = GF_MALLOC(sizeof(lock_migration_info_t),
+                                           gf_common_mt_lock_mig);
+    if (new == NULL) {
+        goto out;
+    }
 
-        INIT_LIST_HEAD (&new->list);
+    INIT_LIST_HEAD(&new->list);
 
-        posix_lock_to_flock (lock, &new->flock);
+    posix_lock_to_flock(lock, &new->flock);
 
-        new->lk_flags = lock->lk_flags;
+    new->lk_flags = lock->lk_flags;
 
-        new->client_uid = gf_strdup (lock->client_uid);
+    new->client_uid = gf_strdup(lock->client_uid);
 
 out:
-        return new;
+    return new;
 }
 
 int
-pl_fill_active_locks (pl_inode_t *pl_inode, lock_migration_info_t *lmi)
+pl_fill_active_locks(pl_inode_t *pl_inode, lock_migration_info_t *lmi)
 {
-        posix_lock_t            *temp           = NULL;
-        lock_migration_info_t   *newlock        = NULL;
-        int                      count          = 0;
+    posix_lock_t *temp = NULL;
+    lock_migration_info_t *newlock = NULL;
+    int count = 0;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (list_empty (&pl_inode->ext_list)) {
-                        count = 0;
-                        goto out;
-                }
-
-                list_for_each_entry (temp, &pl_inode->ext_list, list) {
-
-                        if (temp->blocked)
-                                continue;
-
-                        newlock = gf_mig_info_for_lock (temp);
-                        if (!newlock) {
-                                gf_msg (THIS->name, GF_LOG_ERROR, 0, 0,
-                                        "lock_dup failed");
-                                count = -1;
-                                goto out;
-                        }
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (list_empty(&pl_inode->ext_list)) {
+            count = 0;
+            goto unlock;
+        }
 
-                        list_add_tail (&newlock->list, &lmi->list);
-                        count++;
-                }
+        list_for_each_entry(temp, &pl_inode->ext_list, list)
+        {
+            if (temp->blocked)
+                continue;
+
+            newlock = gf_mig_info_for_lock(temp);
+            if (!newlock) {
+                pthread_mutex_unlock(&pl_inode->mutex);
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "lock_dup failed");
+                count = -1;
+                goto out;
+            }
 
+            list_add_tail(&newlock->list, &lmi->list);
+            count++;
         }
+    }
 
+unlock:
+    pthread_mutex_unlock(&pl_inode->mutex);
 out:
-        pthread_mutex_unlock (&pl_inode->mutex);
-        return count;
+    return count;
 }
 
 /* This function reads only active locks */
 static int
-pl_getactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+pl_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        pl_inode_t             *pl_inode        = NULL;
-        lock_migration_info_t   locks;
-        int                     op_ret          = 0;
-        int                     op_errno        = 0;
-        int                     count           = 0;
+    pl_inode_t *pl_inode = NULL;
+    lock_migration_info_t locks;
+    int op_ret = 0;
+    int op_errno = 0;
+    int count = 0;
 
-        INIT_LIST_HEAD (&locks.list);
+    INIT_LIST_HEAD(&locks.list);
 
-        pl_inode = pl_inode_get (this, loc->inode);
-        if (!pl_inode) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                        "pl_inode_get failed");
+    pl_inode = pl_inode_get(this, loc->inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_inode_get failed");
 
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        count = pl_fill_active_locks (pl_inode, &locks);
+    count = pl_fill_active_locks(pl_inode, &locks);
 
-        op_ret = count;
+    op_ret = count;
 
 out:
-        STACK_UNWIND_STRICT (getactivelk, frame, op_ret, op_errno, &locks,
-                             NULL);
+    STACK_UNWIND_STRICT(getactivelk, frame, op_ret, op_errno, &locks, NULL);
 
-        gf_free_mig_locks (&locks);
+    gf_free_mig_locks(&locks);
 
-        return 0;
+    return 0;
 }
 
 void
-pl_metalk_unref (pl_meta_lock_t *lock)
+pl_metalk_unref(pl_meta_lock_t *lock)
 {
-        lock->ref--;
-        if (!lock->ref) {
-                GF_FREE (lock->client_uid);
-                GF_FREE (lock);
-        }
+    lock->ref--;
+    if (!lock->ref) {
+        GF_FREE(lock->client_uid);
+        GF_FREE(lock);
+    }
 }
 
-
 void
-__pl_metalk_ref (pl_meta_lock_t *lock)
+__pl_metalk_ref(pl_meta_lock_t *lock)
 {
-        lock->ref++;
+    lock->ref++;
 }
 
 pl_meta_lock_t *
-new_meta_lock (call_frame_t *frame, xlator_t *this)
+new_meta_lock(call_frame_t *frame, xlator_t *this)
 {
-        pl_meta_lock_t  *lock   = NULL;
-
-        lock = GF_CALLOC (1, sizeof (*lock),
-                          gf_locks_mt_pl_meta_lock_t);
-
-        if (!lock) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, ENOMEM, "mem allocation"
-                        " failed for meta lock");
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&lock->list);
-        INIT_LIST_HEAD (&lock->client_list);
-
-        lock->client_uid = gf_strdup (frame->root->client->client_uid);
-        if (!lock->client_uid) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, ENOMEM, "mem allocation"
-                        " failed for client_uid");
-                GF_FREE (lock);
-                goto out;
-        }
-
-        __pl_metalk_ref (lock);
+    pl_meta_lock_t *lock = GF_CALLOC(1, sizeof(*lock),
+                                     gf_locks_mt_pl_meta_lock_t);
+
+    if (!lock) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "mem allocation"
+               " failed for meta lock");
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&lock->list);
+    INIT_LIST_HEAD(&lock->client_list);
+
+    lock->client_uid = gf_strdup(frame->root->client->client_uid);
+    if (!lock->client_uid) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "mem allocation"
+               " failed for client_uid");
+        GF_FREE(lock);
+        lock = NULL;
+        goto out;
+    }
+
+    __pl_metalk_ref(lock);
 out:
-        return lock;
+    return lock;
 }
 
 int
-pl_insert_metalk (pl_inode_t *pl_inode, pl_ctx_t *ctx, pl_meta_lock_t *lock)
+pl_insert_metalk(pl_inode_t *pl_inode, pl_ctx_t *ctx, pl_meta_lock_t *lock)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        if (!pl_inode || !ctx || !lock) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0, 0, "NULL parameter");
-                ret = -1;
-                goto out;
-        }
+    if (!pl_inode || !ctx || !lock) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, 0, "NULL parameter");
+        ret = -1;
+        goto out;
+    }
 
-        lock->pl_inode = pl_inode;
+    lock->pl_inode = pl_inode;
 
-        /* refer function pl_inode_setlk for more info for this ref.
-         * This should be unrefed on meta-unlock triggered by rebalance or
-         * in cleanup with client disconnect*/
-        /*TODO: unref this in  cleanup code for disconnect and meta-unlock*/
-        pl_inode->inode = inode_ref (pl_inode->inode);
+    /* refer function pl_inode_setlk for more info for this ref.
+     * This should be unrefed on meta-unlock triggered by rebalance or
+     * in cleanup with client disconnect*/
+    /*TODO: unref this in  cleanup code for disconnect and meta-unlock*/
+    pl_inode->inode = inode_ref(pl_inode->inode);
 
-        /* NOTE:In case of a client-server disconnect we need to cleanup metalk.
-         * Hence, adding the metalk to pl_ctx_t as well. The mutex lock order
-         * should always be on ctx and then on pl_inode*/
+    /* NOTE:In case of a client-server disconnect we need to cleanup metalk.
+     * Hence, adding the metalk to pl_ctx_t as well. The mutex lock order
+     * should always be on ctx and then on pl_inode*/
 
-        pthread_mutex_lock (&ctx->lock);
+    pthread_mutex_lock(&ctx->lock);
+    {
+        pthread_mutex_lock(&pl_inode->mutex);
         {
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                       list_add_tail (&lock->list, &pl_inode->metalk_list);
-                }
-                pthread_mutex_unlock (&pl_inode->mutex);
-
-                list_add_tail (&lock->client_list, &ctx->metalk_list);
+            list_add_tail(&lock->list, &pl_inode->metalk_list);
         }
-        pthread_mutex_unlock (&ctx->lock);
+        pthread_mutex_unlock(&pl_inode->mutex);
+
+        list_add_tail(&lock->client_list, &ctx->metalk_list);
+    }
+    pthread_mutex_unlock(&ctx->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-pl_metalk (call_frame_t *frame, xlator_t *this, inode_t *inode)
+pl_metalk(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        pl_inode_t      *pl_inode       = NULL;
-        int              ret            = 0;
-        pl_meta_lock_t  *reqlk          = NULL;
-        pl_ctx_t        *ctx            = NULL;
-
-        pl_inode = pl_inode_get (this, inode);
-        if (!pl_inode) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, ENOMEM,
-                        "pl_inode mem allocation failedd");
-
-                ret = -1;
-                goto out;
-        }
-
-        if (frame->root->client) {
-                ctx = pl_ctx_get (frame->root->client, this);
-                if (!ctx) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                "pl_ctx_get failed");
-
-                        ret = -1;
-                        goto out;
-
-                }
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "frame-root-client "
-                        "is NULL");
-
-                ret = -1;
-                goto out;
-        }
-
-        reqlk = new_meta_lock (frame, this);
-        if (!reqlk) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = pl_insert_metalk (pl_inode, ctx, reqlk);
-        if (ret < 0) {
-                pl_metalk_unref (reqlk);
-        }
+    pl_inode_t *pl_inode = NULL;
+    int ret = 0;
+    pl_meta_lock_t *reqlk = NULL;
+    pl_ctx_t *ctx = NULL;
+
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "pl_inode mem allocation failedd");
+
+        ret = -1;
+        goto out;
+    }
+
+    /* Non rebalance process trying to do metalock */
+    if (frame->root->pid != GF_CLIENT_PID_DEFRAG) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Note: In the current scheme of glusterfs where lock migration is
+     * experimental, (ideally) the rebalance process which is migrating
+     * the file should request for a metalock. Hence, the metalock count
+     * should not be more than one for an inode. In future, if there is a
+     * need for meta-lock from other clients, the following block can be
+     * removed.
+     *
+     * Since pl_metalk is called as part of setxattr operation, any client
+     * process(non-rebalance) residing outside trusted network can exhaust
+     * memory of the server node by issuing setxattr repetitively on the
+     * metalock key. The following code makes sure that more than
+     * one metalock cannot be granted on an inode*/
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (pl_metalock_is_active(pl_inode)) {
+            ret = -1;
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, 0,
+               "More than one meta-lock cannot be granted on"
+               " the inode");
+        goto out;
+    }
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_ctx_get failed");
+
+            ret = -1;
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "frame-root-client "
+               "is NULL");
+
+        ret = -1;
+        goto out;
+    }
+
+    reqlk = new_meta_lock(frame, this);
+    if (!reqlk) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = pl_insert_metalk(pl_inode, ctx, reqlk);
+    if (ret < 0) {
+        pl_metalk_unref(reqlk);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-void
-__unwind_queued_locks (xlator_t *this, pl_inode_t *pl_inode,
-                       struct list_head *tmp_list)
+static void
+__unwind_queued_locks(pl_inode_t *pl_inode, struct list_head *tmp_list)
 {
-        posix_lock_t    *lock   = NULL;
-        posix_lock_t    *tmp    = NULL;
-
-        if (list_empty (&pl_inode->queued_locks))
-                return;
+    if (list_empty(&pl_inode->queued_locks))
+        return;
 
-        list_splice_init (&pl_inode->queued_locks, tmp_list);
+    list_splice_init(&pl_inode->queued_locks, tmp_list);
 }
 
-void
-__unwind_blocked_locks (xlator_t *this, pl_inode_t *pl_inode,
-                        struct list_head *tmp_list)
+static void
+__unwind_blocked_locks(pl_inode_t *pl_inode, struct list_head *tmp_list)
 {
-        posix_lock_t    *lock   = NULL;
-        posix_lock_t    *tmp    = NULL;
-
-        if (list_empty (&pl_inode->ext_list))
-                return;
+    posix_lock_t *lock = NULL;
+    posix_lock_t *tmp = NULL;
 
-        list_for_each_entry_safe (lock, tmp, &pl_inode->ext_list, list) {
+    if (list_empty(&pl_inode->ext_list))
+        return;
 
-                if (!lock->blocking)
-                        continue;
+    list_for_each_entry_safe(lock, tmp, &pl_inode->ext_list, list)
+    {
+        if (!lock->blocking)
+            continue;
 
-                list_del_init (&lock->list);
-                list_add_tail (&lock->list, tmp_list);
-        }
+        list_del_init(&lock->list);
+        list_add_tail(&lock->list, tmp_list);
+    }
 }
 
 int
-pl_metaunlock (call_frame_t *frame, xlator_t *this, inode_t *inode,
-               dict_t *dict)
-{
-        pl_inode_t      *pl_inode               = NULL;
-        int              ret                    = 0;
-        pl_meta_lock_t  *meta_lock              = NULL;
-        pl_meta_lock_t  *tmp_metalk             = NULL;
-        pl_ctx_t        *ctx                    = NULL;
-        posix_lock_t    *posix_lock             = NULL;
-        posix_lock_t    *tmp_posixlk            = NULL;
-        struct list_head tmp_posixlk_list;
-
-        INIT_LIST_HEAD (&tmp_posixlk_list);
-
-        if (frame->root->client) {
-                ctx = pl_ctx_get (frame->root->client, this);
-                if (!ctx) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                "pl_ctx_get failed");
-
-                        ret = -1;
-                        goto out;
-                }
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "frame-root-client is "
-                        "NULL");
-                ret = -1;
-                goto out;
-        }
-
-        pl_inode = pl_inode_get (this, inode);
-        if (!pl_inode) {
-                ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&ctx->lock);
+pl_metaunlock(call_frame_t *frame, xlator_t *this, inode_t *inode, dict_t *dict)
+{
+    pl_inode_t *pl_inode = NULL;
+    int ret = 0;
+    pl_meta_lock_t *meta_lock = NULL;
+    pl_meta_lock_t *tmp_metalk = NULL;
+    pl_ctx_t *ctx = NULL;
+    posix_lock_t *posix_lock = NULL;
+    posix_lock_t *tmp_posixlk = NULL;
+    struct list_head tmp_posixlk_list;
+
+    INIT_LIST_HEAD(&tmp_posixlk_list);
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_ctx_get failed");
+
+            ret = -1;
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "frame-root-client is "
+               "NULL");
+        ret = -1;
+        goto out;
+    }
+
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode) {
+        ret = -1;
+        goto out;
+    }
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        pthread_mutex_lock(&pl_inode->mutex);
         {
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        /* Unwind queued locks regardless of migration status */
-                        __unwind_queued_locks (this, pl_inode,
-                                               &tmp_posixlk_list);
+            /* Unwind queued locks regardless of migration status */
+            __unwind_queued_locks(pl_inode, &tmp_posixlk_list);
 
-                        /* Unwind blocked locks only for successful migration */
-                        if (dict_get (dict, "status")) {
+            /* Unwind blocked locks only for successful migration */
+            if (dict_get_sizen(dict, "status")) {
+                /* unwind all blocked locks */
+                __unwind_blocked_locks(pl_inode, &tmp_posixlk_list);
+            }
 
-                                /* unwind all blocked locks */
-                                __unwind_blocked_locks (this, pl_inode,
-                                                        &tmp_posixlk_list);
-                        }
+            /* unlock metalk */
+            /* if this list is empty then pl_inode->metalk_list
+             * should be empty too. meta lock should in all cases
+             * be added/removed from both pl_ctx_t and pl_inode */
 
-                        /* unlock metalk */
-                        /* if this list is empty then pl_inode->metalk_list
-                         * should be empty too. meta lock should in all cases
-                         * be added/removed from both pl_ctx_t and pl_inode */
+            if (list_empty(&ctx->metalk_list))
+                goto unlock;
 
-                        if (list_empty (&ctx->metalk_list))
-                               goto unlock;
+            list_for_each_entry_safe(meta_lock, tmp_metalk, &ctx->metalk_list,
+                                     client_list)
+            {
+                list_del_init(&meta_lock->client_list);
 
-                        list_for_each_entry_safe (meta_lock, tmp_metalk,
-                                                  &ctx->metalk_list,
-                                                  client_list) {
-                                list_del_init (&meta_lock->client_list);
+                pl_inode = meta_lock->pl_inode;
 
-                                pl_inode = meta_lock->pl_inode;
+                list_del_init(&meta_lock->list);
 
-                                list_del_init (&meta_lock->list);
+                pl_metalk_unref(meta_lock);
 
-                                pl_metalk_unref (meta_lock);
+                /* The corresponding ref is taken in
+                 * pl_insert_metalk*/
+                inode_unref(pl_inode->inode);
+            }
 
-                                /* The corresponding ref is taken in
-                                 * pl_insert_metalk*/
-                                inode_unref (pl_inode->inode);
-                        }
+            if (dict_get_sizen(dict, "status"))
+                pl_inode->migrated = _gf_true;
+            else
+                pl_inode->migrated = _gf_false;
+        }
+    unlock:
 
-                        if (dict_get (dict, "status"))
-                                pl_inode->migrated = _gf_true;
-                        else
-                                pl_inode->migrated = _gf_false;
-                }
-unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&ctx->lock);
 
-               pthread_mutex_unlock (&pl_inode->mutex);
+out:
+    list_for_each_entry_safe(posix_lock, tmp_posixlk, &tmp_posixlk_list, list)
+    {
+        list_del_init(&posix_lock->list);
 
-        }
-        pthread_mutex_unlock (&ctx->lock);
+        STACK_UNWIND_STRICT(lk, posix_lock->frame, -1, EREMOTE,
+                            &posix_lock->user_flock, NULL);
 
-out:
-        list_for_each_entry_safe (posix_lock, tmp_posixlk, &tmp_posixlk_list,
-                                  list) {
-                list_del_init (&posix_lock->list);
+        __destroy_lock(posix_lock);
+    }
 
-                STACK_UNWIND_STRICT (lk, posix_lock->frame, -1, EREMOTE,
-                                     &posix_lock->user_flock, NULL);
+    return ret;
+}
 
-                GF_FREE (posix_lock->client_uid);
-                GF_FREE (posix_lock);
+int32_t
+pl_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        return ret;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            while (pl_inode->fop_wind_count > 0) {
+                gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                       "waiting for existing fops (count %d) to drain for "
+                       "gfid %s",
+                       pl_inode->fop_wind_count, uuid_utoa(pl_inode->gfid));
+                pthread_cond_wait(&pl_inode->check_fop_wind_count,
+                                  &pl_inode->mutex);
+            }
+            pl_inode->mlock_enforced = _gf_true;
+            pl_inode->check_mlock_info = _gf_false;
+        }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
+
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(setxattr, xdata, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-pl_setxattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
+pl_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int flags, dict_t *xdata)
 {
-        int             op_ret          = 0;
-        int             op_errno        = 0;
+    int op_ret = 0;
+    int op_errno = EINVAL;
+    dict_t *xdata_rsp = NULL;
+    char *name = NULL;
+    posix_locks_private_t *priv = this->private;
 
-        if (dict_get (dict, GF_META_LOCK_KEY)) {
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
 
-                op_ret = pl_metalk (frame, this, loc->inode);
+    if (dict_get_sizen(dict, GF_META_LOCK_KEY)) {
+        op_ret = pl_metalk(frame, this, loc->inode);
 
-        } else if (dict_get (dict, GF_META_UNLOCK_KEY)) {
+    } else if (dict_get_sizen(dict, GF_META_UNLOCK_KEY)) {
+        op_ret = pl_metaunlock(frame, this, loc->inode, dict);
+    } else {
+        goto usual;
+    }
 
-                op_ret = pl_metaunlock (frame, this, loc->inode, dict);
+    PL_STACK_UNWIND_FOR_CLIENT(setxattr, xdata_rsp, frame, op_ret, op_errno,
+                               xdata_rsp);
+    return 0;
 
-        } else {
-                goto usual;
-        }
+usual:
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, dict, name, this, loc, ((fd_t *)NULL),
+                              priv);
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_WIND(frame, pl_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 
-usual:
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->setxattr, loc, dict, flags,
-                         xdata);
-        return 0;
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(setxattr, xdata, frame, op_ret, op_errno, xdata);
+
+    return 0;
 }
 
 void
-pl_dump_lock (char *str, int size, struct gf_flock *flock,
-              gf_lkowner_t *owner, void *trans, char *conn_id,
-              time_t *granted_time, time_t *blkd_time, gf_boolean_t active)
-{
-        char  *type_str    = NULL;
-        char   granted[256] = {0,};
-        char   blocked[256] = {0,};
-
-        if (granted_time)
-                gf_time_fmt (granted, sizeof (granted), *granted_time,
-                             gf_timefmt_FT);
-        if (blkd_time)
-                gf_time_fmt (blocked, sizeof (blocked), *blkd_time,
-                             gf_timefmt_FT);
-        switch (flock->l_type) {
+pl_dump_lock(char *str, int size, struct gf_flock *flock, gf_lkowner_t *owner,
+             void *trans, char *conn_id, time_t *granted_time,
+             time_t *blkd_time, gf_boolean_t active)
+{
+    char *type_str = NULL;
+    char granted[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char blocked[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    if (granted_time)
+        gf_time_fmt(granted, sizeof(granted), *granted_time, gf_timefmt_FT);
+    if (blkd_time)
+        gf_time_fmt(blocked, sizeof(blocked), *blkd_time, gf_timefmt_FT);
+    switch (flock->l_type) {
         case F_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case F_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
+            type_str = "UNLOCK";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        if (active) {
-                if (blkd_time && *blkd_time == 0) {
-                        snprintf (str, size, RANGE_GRNTD_FMT,
-                                  type_str, flock->l_whence,
-                                  (unsigned long long) flock->l_start,
-                                  (unsigned long long) flock->l_len,
-                                  (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner), trans, conn_id,
-                                  granted);
-                } else {
-                        snprintf (str, size, RANGE_BLKD_GRNTD_FMT,
-                                  type_str, flock->l_whence,
-                                  (unsigned long long) flock->l_start,
-                                  (unsigned long long) flock->l_len,
-                                  (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner), trans, conn_id,
-                                  blocked, granted);
-                }
+            type_str = "UNKNOWN";
+            break;
+    }
+
+    if (active) {
+        if (blkd_time && *blkd_time == 0) {
+            snprintf(str, size, RANGE_GRNTD_FMT, type_str, flock->l_whence,
+                     (unsigned long long)flock->l_start,
+                     (unsigned long long)flock->l_len,
+                     (unsigned long long)flock->l_pid, lkowner_utoa(owner),
+                     trans, conn_id, granted);
         } else {
-                snprintf (str, size, RANGE_BLKD_FMT,
-                          type_str, flock->l_whence,
-                          (unsigned long long) flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid,
-                          lkowner_utoa (owner), trans, conn_id, blocked);
+            snprintf(str, size, RANGE_BLKD_GRNTD_FMT, type_str, flock->l_whence,
+                     (unsigned long long)flock->l_start,
+                     (unsigned long long)flock->l_len,
+                     (unsigned long long)flock->l_pid, lkowner_utoa(owner),
+                     trans, conn_id, blocked, granted);
+        }
+    } else {
+        snprintf(str, size, RANGE_BLKD_FMT, type_str, flock->l_whence,
+                 (unsigned long long)flock->l_start,
+                 (unsigned long long)flock->l_len,
+                 (unsigned long long)flock->l_pid, lkowner_utoa(owner), trans,
+                 conn_id, blocked);
+    }
+}
+
+void
+__dump_entrylks(pl_inode_t *pl_inode)
+{
+    pl_dom_list_t *dom = NULL;
+    pl_entry_lock_t *lock = NULL;
+    char blocked[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char granted[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    int count = 0;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char *k = "xlator.feature.locks.lock-dump.domain.entrylk";
+
+    char tmp[4098];
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        count = 0;
+
+        gf_proc_dump_build_key(key, "lock-dump.domain", "domain");
+        gf_proc_dump_write(key, "%s", dom->domain);
+
+        list_for_each_entry(lock, &dom->entrylk_list, domain_list)
+        {
+            gf_time_fmt(granted, sizeof(granted), lock->granted_time,
+                        gf_timefmt_FT);
+            gf_proc_dump_build_key(key, k, "entrylk[%d](ACTIVE)", count);
+            if (lock->blkd_time == 0) {
+                snprintf(tmp, sizeof(tmp), ENTRY_GRNTD_FMT,
+                         lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK"
+                                                     : "ENTRYLK_WRLCK",
+                         lock->basename, (unsigned long long)lock->client_pid,
+                         lkowner_utoa(&lock->owner), lock->client,
+                         lock->connection_id, granted);
+            } else {
+                gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time,
+                            gf_timefmt_FT);
+                snprintf(tmp, sizeof(tmp), ENTRY_BLKD_GRNTD_FMT,
+                         lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK"
+                                                     : "ENTRYLK_WRLCK",
+                         lock->basename, (unsigned long long)lock->client_pid,
+                         lkowner_utoa(&lock->owner), lock->client,
+                         lock->connection_id, blocked, granted);
+            }
+
+            gf_proc_dump_write(key, "%s", tmp);
+
+            count++;
+        }
+
+        list_for_each_entry(lock, &dom->blocked_entrylks, blocked_locks)
+        {
+            gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time,
+                        gf_timefmt_FT);
+
+            gf_proc_dump_build_key(key, k, "entrylk[%d](BLOCKED)", count);
+            snprintf(
+                tmp, sizeof(tmp), ENTRY_BLKD_FMT,
+                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
+                lock->basename, (unsigned long long)lock->client_pid,
+                lkowner_utoa(&lock->owner), lock->client, lock->connection_id,
+                blocked);
+
+            gf_proc_dump_write(key, "%s", tmp);
+
+            count++;
         }
+    }
+}
 
+void
+dump_entrylks(pl_inode_t *pl_inode)
+{
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __dump_entrylks(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 }
 
 void
-__dump_entrylks (pl_inode_t *pl_inode)
-{
-        pl_dom_list_t   *dom  = NULL;
-        pl_entry_lock_t *lock = NULL;
-        char             blocked[256] = {0,};
-        char             granted[256] = {0,};
-        int              count = 0;
-        char             key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        char            *k = "xlator.feature.locks.lock-dump.domain.entrylk";
-
-        char tmp[4098];
-
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-
-                count = 0;
-
-                gf_proc_dump_build_key(key,
-                                       "lock-dump.domain",
-                                       "domain");
-                gf_proc_dump_write(key, "%s", dom->domain);
-
-                list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-
-                        gf_time_fmt (granted, sizeof (granted),
-                                     lock->granted_time.tv_sec, gf_timefmt_FT);
-                        gf_proc_dump_build_key(key, k,
-                                               "entrylk[%d](ACTIVE)", count );
-                        if (lock->blkd_time.tv_sec == 0) {
-                                snprintf (tmp, sizeof (tmp), ENTRY_GRNTD_FMT,
-                                          lock->type == ENTRYLK_RDLCK ?
-                                          "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
-                                          lock->basename,
-                                          (unsigned long long) lock->client_pid,
-                                          lkowner_utoa (&lock->owner),
-                                          lock->client,
-                                          lock->connection_id, granted);
-                        } else {
-                                gf_time_fmt (blocked, sizeof (blocked),
-                                             lock->blkd_time.tv_sec,
-                                             gf_timefmt_FT);
-                                snprintf (tmp, sizeof (tmp),
-                                          ENTRY_BLKD_GRNTD_FMT,
-                                          lock->type == ENTRYLK_RDLCK ?
-                                          "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
-                                          lock->basename,
-                                          (unsigned long long) lock->client_pid,
-                                          lkowner_utoa (&lock->owner),
-                                          lock->client,
-                                          lock->connection_id,
-                                          blocked, granted);
-                        }
+__dump_inodelks(pl_inode_t *pl_inode)
+{
+    pl_dom_list_t *dom = NULL;
+    pl_inode_lock_t *lock = NULL;
+    int count = 0;
+    char key[GF_DUMP_MAX_BUF_LEN];
 
-                        gf_proc_dump_write(key, tmp);
+    char tmp[4098];
 
-                        count++;
-                }
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        count = 0;
 
-                list_for_each_entry (lock, &dom->blocked_entrylks,
-                                     blocked_locks) {
+        gf_proc_dump_build_key(key, "lock-dump.domain", "domain");
+        gf_proc_dump_write(key, "%s", dom->domain);
 
-                        gf_time_fmt (blocked, sizeof (blocked),
-                                     lock->blkd_time.tv_sec, gf_timefmt_FT);
+        list_for_each_entry(lock, &dom->inodelk_list, list)
+        {
+            gf_proc_dump_build_key(key, "inodelk", "inodelk[%d](ACTIVE)",
+                                   count);
 
-                        gf_proc_dump_build_key(key, k,
-                                               "entrylk[%d](BLOCKED)", count );
-                        snprintf (tmp, sizeof (tmp), ENTRY_BLKD_FMT,
-                                  lock->type == ENTRYLK_RDLCK ?
-                                  "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
-                                  lock->basename,
-                                  (unsigned long long) lock->client_pid,
-                                  lkowner_utoa (&lock->owner), lock->client,
-                                  lock->connection_id, blocked);
+            SET_FLOCK_PID(&lock->user_flock, lock);
+            pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner,
+                         lock->client, lock->connection_id, &lock->granted_time,
+                         &lock->blkd_time, _gf_true);
+            gf_proc_dump_write(key, "%s", tmp);
 
-                        gf_proc_dump_write(key, tmp);
+            count++;
+        }
 
-                        count++;
-                }
+        list_for_each_entry(lock, &dom->blocked_inodelks, blocked_locks)
+        {
+            gf_proc_dump_build_key(key, "inodelk", "inodelk[%d](BLOCKED)",
+                                   count);
+            SET_FLOCK_PID(&lock->user_flock, lock);
+            pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner,
+                         lock->client, lock->connection_id, 0, &lock->blkd_time,
+                         _gf_false);
+            gf_proc_dump_write(key, "%s", tmp);
 
+            count++;
         }
+    }
 }
 
 void
-dump_entrylks (pl_inode_t *pl_inode)
+dump_inodelks(pl_inode_t *pl_inode)
 {
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __dump_entrylks (pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __dump_inodelks(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+}
 
+void
+__dump_posixlks(pl_inode_t *pl_inode)
+{
+    posix_lock_t *lock = NULL;
+    int count = 0;
+    char key[GF_DUMP_MAX_BUF_LEN];
+
+    char tmp[4098];
+
+    list_for_each_entry(lock, &pl_inode->ext_list, list)
+    {
+        SET_FLOCK_PID(&lock->user_flock, lock);
+        gf_proc_dump_build_key(key, "posixlk", "posixlk[%d](%s)", count,
+                               lock->blocked ? "BLOCKED" : "ACTIVE");
+        pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner,
+                     lock->client, lock->client_uid, &lock->granted_time,
+                     &lock->blkd_time, (lock->blocked) ? _gf_false : _gf_true);
+        gf_proc_dump_write(key, "%s", tmp);
+
+        count++;
+    }
 }
 
 void
-__dump_inodelks (pl_inode_t *pl_inode)
+dump_posixlks(pl_inode_t *pl_inode)
 {
-        pl_dom_list_t   *dom  = NULL;
-        pl_inode_lock_t *lock = NULL;
-        int             count = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __dump_posixlks(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+}
 
-        char tmp[4098];
+int32_t
+pl_dump_inode_priv(xlator_t *this, inode_t *inode)
+{
+    int ret = -1;
+    uint64_t tmp_pl_inode = 0;
+    pl_inode_t *pl_inode = NULL;
+    char *pathname = NULL;
+    gf_boolean_t section_added = _gf_false;
+
+    int count = 0;
+
+    if (!inode) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = TRY_LOCK(&inode->lock);
+    if (ret)
+        goto out;
+    {
+        ret = __inode_ctx_get(inode, this, &tmp_pl_inode);
+        if (ret)
+            goto unlock;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+    if (ret)
+        goto out;
 
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+    if (!pl_inode) {
+        ret = -1;
+        goto out;
+    }
 
-                count = 0;
+    gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
+    section_added = _gf_true;
 
-                gf_proc_dump_build_key(key,
-                                       "lock-dump.domain",
-                                       "domain");
-                gf_proc_dump_write(key, "%s", dom->domain);
+    /*We are safe to call __inode_path since we have the
+     * inode->table->lock */
+    __inode_path(inode, NULL, &pathname);
+    if (pathname)
+        gf_proc_dump_write("path", "%s", pathname);
 
-                list_for_each_entry (lock, &dom->inodelk_list, list) {
+    gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
 
-                        gf_proc_dump_build_key(key,
-                                               "inodelk",
-                                               "inodelk[%d](ACTIVE)",count );
+    ret = pthread_mutex_trylock(&pl_inode->mutex);
+    if (ret)
+        goto out;
+    {
+        count = __get_entrylk_count(this, pl_inode);
+        if (count) {
+            gf_proc_dump_write("entrylk-count", "%d", count);
+            __dump_entrylks(pl_inode);
+        }
 
-                        SET_FLOCK_PID (&lock->user_flock, lock);
-                        pl_dump_lock (tmp, sizeof (tmp), &lock->user_flock,
-                                      &lock->owner,
-                                      lock->client, lock->connection_id,
-                                      &lock->granted_time.tv_sec,
-                                      &lock->blkd_time.tv_sec,
-                                      _gf_true);
-                        gf_proc_dump_write(key, tmp);
+        count = __get_inodelk_count(this, pl_inode, NULL);
+        if (count) {
+            gf_proc_dump_write("inodelk-count", "%d", count);
+            __dump_inodelks(pl_inode);
+        }
 
-                        count++;
-                }
+        count = __get_posixlk_count(pl_inode);
+        if (count) {
+            gf_proc_dump_write("posixlk-count", "%d", count);
+            __dump_posixlks(pl_inode);
+        }
 
-                list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+        gf_proc_dump_write("links", "%d", pl_inode->links);
+        gf_proc_dump_write("removes_pending", "%u", pl_inode->remove_running);
+        gf_proc_dump_write("removed", "%u", pl_inode->removed);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-                        gf_proc_dump_build_key(key,
-                                               "inodelk",
-                                               "inodelk[%d](BLOCKED)",count );
-                        SET_FLOCK_PID (&lock->user_flock, lock);
-                        pl_dump_lock (tmp, sizeof (tmp), &lock->user_flock,
-                                      &lock->owner,
-                                      lock->client, lock->connection_id,
-                                      0, &lock->blkd_time.tv_sec,
-                                      _gf_false);
-                        gf_proc_dump_write(key, tmp);
+out:
+    GF_FREE(pathname);
+
+    if (ret && inode) {
+        if (!section_added)
+            gf_proc_dump_add_section(
+                "xlator.features.locks.%s."
+                "inode",
+                this->name);
+        gf_proc_dump_write("Unable to print lock state",
+                           "(Lock "
+                           "acquisition failure) %s",
+                           uuid_utoa(inode->gfid));
+    }
+    return ret;
+}
 
-                        count++;
-                }
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-        }
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_locks_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
+        return ret;
+    }
+
+    return ret;
 }
 
-void
-dump_inodelks (pl_inode_t *pl_inode)
+pl_ctx_t *
+pl_ctx_get(client_t *client, xlator_t *xlator)
 {
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __dump_inodelks (pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    void *tmp = NULL;
+    pl_ctx_t *ctx = NULL;
+    pl_ctx_t *setted_ctx = NULL;
+
+    client_ctx_get(client, xlator, &tmp);
+
+    ctx = tmp;
+
+    if (ctx != NULL)
+        goto out;
+
+    ctx = GF_CALLOC(1, sizeof(pl_ctx_t), gf_locks_mt_posix_lock_t);
 
+    if (ctx == NULL)
+        goto out;
+
+    pthread_mutex_init(&ctx->lock, NULL);
+    INIT_LIST_HEAD(&ctx->inodelk_lockers);
+    INIT_LIST_HEAD(&ctx->entrylk_lockers);
+    INIT_LIST_HEAD(&ctx->metalk_list);
+
+    setted_ctx = client_ctx_set(client, xlator, ctx);
+    if (ctx != setted_ctx) {
+        pthread_mutex_destroy(&ctx->lock);
+        GF_FREE(ctx);
+        ctx = setted_ctx;
+    }
+out:
+    return ctx;
 }
 
-void
-__dump_posixlks (pl_inode_t *pl_inode)
+int
+pl_metalk_client_cleanup(xlator_t *this, pl_ctx_t *ctx)
 {
-        posix_lock_t    *lock = NULL;
-        int             count = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+    pl_meta_lock_t *meta_lock = NULL;
+    pl_meta_lock_t *tmp_metalk = NULL;
+    pl_inode_t *pl_inode = NULL;
+    posix_lock_t *posix_lock = NULL;
+    posix_lock_t *tmp_posixlk = NULL;
+    struct list_head tmp_posixlk_list;
+
+    INIT_LIST_HEAD(&tmp_posixlk_list);
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        /* if this list is empty then pl_inode->metalk_list should be
+         * empty too. meta lock should in all cases be added/removed
+         * from both pl_ctx_t and pl_inode */
+        if (list_empty(&ctx->metalk_list))
+            goto unlock;
+
+        list_for_each_entry_safe(meta_lock, tmp_metalk, &ctx->metalk_list,
+                                 client_list)
+        {
+            list_del_init(&meta_lock->client_list);
+
+            pl_inode = meta_lock->pl_inode;
+
+            pthread_mutex_lock(&pl_inode->mutex);
 
-        char tmp[4098];
+            {
+                /* Since the migration status is unknown here
+                 * unwind all queued and blocked locks to check
+                 * migration status and find the correct
+                 * destination */
+                __unwind_queued_locks(pl_inode, &tmp_posixlk_list);
 
-      list_for_each_entry (lock, &pl_inode->ext_list, list) {
+                __unwind_blocked_locks(pl_inode, &tmp_posixlk_list);
 
-              SET_FLOCK_PID (&lock->user_flock, lock);
-              gf_proc_dump_build_key(key,
-                                     "posixlk",
-                                     "posixlk[%d](%s)",
-                                     count,
-                                     lock->blocked ? "BLOCKED" : "ACTIVE");
-              pl_dump_lock (tmp, sizeof (tmp), &lock->user_flock,
-                            &lock->owner, lock->client, NULL,
-                            &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec,
-                            (lock->blocked)? _gf_false: _gf_true);
-              gf_proc_dump_write(key, tmp);
+                list_del_init(&meta_lock->list);
 
-              count++;
+                pl_metalk_unref(meta_lock);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+
+            /* The corresponding ref is taken in
+             * pl_insert_metalk*/
+            inode_unref(pl_inode->inode);
         }
+    }
+
+unlock:
+    pthread_mutex_unlock(&ctx->lock);
+
+    list_for_each_entry_safe(posix_lock, tmp_posixlk, &tmp_posixlk_list, list)
+    {
+        list_del_init(&posix_lock->list);
+
+        STACK_UNWIND_STRICT(lk, posix_lock->frame, -1, EREMOTE,
+                            &posix_lock->user_flock, NULL);
+
+        __destroy_lock(posix_lock);
+    }
+    return 0;
 }
 
-void
-dump_posixlks (pl_inode_t *pl_inode)
+static int
+pl_client_disconnect_cbk(xlator_t *this, client_t *client)
 {
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __dump_posixlks (pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pl_ctx_t *pl_ctx = pl_ctx_get(client, this);
+    if (pl_ctx) {
+        pl_inodelk_client_cleanup(this, pl_ctx);
+        pl_entrylk_client_cleanup(this, pl_ctx);
+        pl_metalk_client_cleanup(this, pl_ctx);
+    }
+
+    return 0;
+}
 
+static int
+pl_client_destroy_cbk(xlator_t *this, client_t *client)
+{
+    void *tmp = NULL;
+    pl_ctx_t *pl_ctx = NULL;
+
+    pl_client_disconnect_cbk(this, client);
+
+    client_ctx_del(client, this, &tmp);
+
+    if (tmp == NULL)
+        return 0;
+
+    pl_ctx = tmp;
+
+    GF_ASSERT(list_empty(&pl_ctx->inodelk_lockers));
+    GF_ASSERT(list_empty(&pl_ctx->entrylk_lockers));
+
+    pthread_mutex_destroy(&pl_ctx->lock);
+    GF_FREE(pl_ctx);
+
+    return 0;
 }
 
-int32_t
-pl_dump_inode_priv (xlator_t *this, inode_t *inode)
+int
+reconfigure(xlator_t *this, dict_t *options)
 {
+    posix_locks_private_t *priv = this->private;
+    int ret = -1;
+    char *tmp_str = NULL;
 
-        int             ret = -1;
-        uint64_t        tmp_pl_inode = 0;
-        pl_inode_t      *pl_inode = NULL;
-        char            *pathname = NULL;
-        gf_boolean_t    section_added = _gf_false;
+    GF_OPTION_RECONF("trace", priv->trace, options, bool, out);
 
-        int count      = 0;
+    GF_OPTION_RECONF("monkey-unlocking", priv->monkey_unlocking, options, bool,
+                     out);
 
-        if (!inode) {
-                errno = EINVAL;
-                goto out;
-        }
+    GF_OPTION_RECONF("revocation-secs", priv->revocation_secs, options, uint32,
+                     out);
 
-        ret = TRY_LOCK (&inode->lock);
-        if (ret)
-                goto out;
-        {
-                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
-                if (ret)
-                        goto unlock;
-        }
-unlock:
-        UNLOCK (&inode->lock);
-        if (ret)
-                goto out;
+    GF_OPTION_RECONF("revocation-clear-all", priv->revocation_clear_all,
+                     options, bool, out);
 
-        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-        if (!pl_inode) {
-                ret = -1;
-                goto out;
-        }
+    GF_OPTION_RECONF("revocation-max-blocked", priv->revocation_max_blocked,
+                     options, uint32, out);
 
-        gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
-        section_added = _gf_true;
+    GF_OPTION_RECONF("notify-contention", priv->notify_contention, options,
+                     bool, out);
 
-        /*We are safe to call __inode_path since we have the
-         * inode->table->lock */
-        __inode_path (inode, NULL, &pathname);
-        if (pathname)
-                gf_proc_dump_write ("path", "%s", pathname);
+    GF_OPTION_RECONF("notify-contention-delay", priv->notify_contention_delay,
+                     options, uint32, out);
 
-        gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
+    GF_OPTION_RECONF("mandatory-locking", tmp_str, options, str, out);
 
-        ret = pthread_mutex_trylock (&pl_inode->mutex);
-        if (ret)
-                goto out;
-        {
-                count = __get_entrylk_count (this, pl_inode);
-                if (count) {
-                        gf_proc_dump_write("entrylk-count", "%d", count);
-                        __dump_entrylks (pl_inode);
-                }
+    GF_OPTION_RECONF("enforce-mandatory-lock", priv->mlock_enforced, options,
+                     bool, out);
 
-                count = __get_inodelk_count (this, pl_inode, NULL);
-                if (count) {
-                        gf_proc_dump_write("inodelk-count", "%d", count);
-                        __dump_inodelks (pl_inode);
-                }
+    if (!strcmp(tmp_str, "forced"))
+        priv->mandatory_mode = MLK_FORCED;
+    else if (!strcmp(tmp_str, "file"))
+        priv->mandatory_mode = MLK_FILE_BASED;
+    else if (!strcmp(tmp_str, "optimal"))
+        priv->mandatory_mode = MLK_OPTIMAL;
+    else
+        priv->mandatory_mode = MLK_NONE;
 
-                count = __get_posixlk_count (this, pl_inode);
-                if (count) {
-                        gf_proc_dump_write("posixlk-count", "%d", count);
-                        __dump_posixlks (pl_inode);
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    ret = 0;
 
 out:
-        GF_FREE (pathname);
-
-        if (ret && inode) {
-                if (!section_added)
-                        gf_proc_dump_add_section ("xlator.features.locks.%s."
-                                                  "inode", this->name);
-                gf_proc_dump_write ("Unable to print lock state", "(Lock "
-                                    "acquisition failure) %s",
-                                    uuid_utoa (inode->gfid));
-        }
-        return ret;
+    return ret;
 }
 
-int32_t
-mem_acct_init (xlator_t *this)
+int
+init(xlator_t *this)
 {
-        int     ret = -1;
+    posix_locks_private_t *priv = NULL;
+    xlator_list_t *trav = NULL;
+    char *tmp_str = NULL;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "FATAL: posix-locks should have exactly one child");
+        goto out;
+    }
 
-        ret = xlator_mem_acct_init (this, gf_locks_mt_end + 1);
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Volume is dangling. Please check the volume file.");
+    }
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
+    trav = this->children;
+    while (trav->xlator->children)
+        trav = trav->xlator->children;
 
-        return ret;
-}
+    if (strncmp("storage/", trav->xlator->type, 8)) {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "'locks' translator is not loaded over a storage "
+               "translator");
+        goto out;
+    }
 
+    priv = GF_CALLOC(1, sizeof(*priv), gf_locks_mt_posix_locks_private_t);
 
-pl_ctx_t*
-pl_ctx_get (client_t *client, xlator_t *xlator)
-{
-        void *tmp = NULL;
-        pl_ctx_t *ctx = NULL;
+    GF_OPTION_INIT("mandatory-locking", tmp_str, str, out);
+    if (!strcmp(tmp_str, "forced"))
+        priv->mandatory_mode = MLK_FORCED;
+    else if (!strcmp(tmp_str, "file"))
+        priv->mandatory_mode = MLK_FILE_BASED;
+    else if (!strcmp(tmp_str, "optimal"))
+        priv->mandatory_mode = MLK_OPTIMAL;
+    else
+        priv->mandatory_mode = MLK_NONE;
 
-        client_ctx_get (client, xlator, &tmp);
+    tmp_str = NULL;
 
-        ctx = tmp;
+    GF_OPTION_INIT("trace", priv->trace, bool, out);
 
-        if (ctx != NULL)
-                goto out;
+    GF_OPTION_INIT("monkey-unlocking", priv->monkey_unlocking, bool, out);
 
-        ctx = GF_CALLOC (1, sizeof (pl_ctx_t), gf_locks_mt_posix_lock_t);
+    GF_OPTION_INIT("revocation-secs", priv->revocation_secs, uint32, out);
 
-        if (ctx == NULL)
-                goto out;
+    GF_OPTION_INIT("revocation-clear-all", priv->revocation_clear_all, bool,
+                   out);
 
-        pthread_mutex_init (&ctx->lock, NULL);
-        INIT_LIST_HEAD (&ctx->inodelk_lockers);
-        INIT_LIST_HEAD (&ctx->entrylk_lockers);
-        INIT_LIST_HEAD (&ctx->metalk_list);
+    GF_OPTION_INIT("revocation-max-blocked", priv->revocation_max_blocked,
+                   uint32, out);
+
+    GF_OPTION_INIT("notify-contention", priv->notify_contention, bool, out);
+
+    GF_OPTION_INIT("notify-contention-delay", priv->notify_contention_delay,
+                   uint32, out);
+
+    GF_OPTION_INIT("enforce-mandatory-lock", priv->mlock_enforced, bool, out);
+
+    this->local_pool = mem_pool_new(pl_local_t, 32);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        goto out;
+    }
+
+    this->private = priv;
+    ret = 0;
 
-        if (client_ctx_set (client, xlator, ctx) != 0) {
-                pthread_mutex_destroy (&ctx->lock);
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
 out:
-        return ctx;
+    if (ret) {
+        GF_FREE(priv);
+    }
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    posix_locks_private_t *priv = this->private;
+    if (!priv)
+        return;
+    this->private = NULL;
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    GF_FREE(priv->brickname);
+    GF_FREE(priv);
+
+    return;
 }
 
 int
-pl_metalk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+pl_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int
+pl_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int
+pl_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata);
+
+int
+pl_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
+
+int32_t
+pl_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, struct iatt *preoldparent,
+              struct iatt *postoldparent, struct iatt *prenewparent,
+              struct iatt *postnewparent, dict_t *xdata)
 {
-        pl_meta_lock_t  *meta_lock              = NULL;
-        pl_meta_lock_t  *tmp_metalk             = NULL;
-        pl_inode_t      *pl_inode               = NULL;
-        posix_lock_t    *posix_lock             = NULL;
-        posix_lock_t    *tmp_posixlk            = NULL;
-        struct list_head tmp_posixlk_list;
+    pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0);
 
-        INIT_LIST_HEAD (&tmp_posixlk_list);
+    PL_STACK_UNWIND(rename, xdata, frame, op_ret, op_errno, buf, preoldparent,
+                    postoldparent, prenewparent, postnewparent, xdata);
 
-        pthread_mutex_lock (&ctx->lock);
-        {
+    return 0;
+}
 
-                /* if this list is empty then pl_inode->metalk_list should be
-                 * empty too. meta lock should in all cases be added/removed
-                 * from both pl_ctx_t and pl_inode */
-                if (list_empty (&ctx->metalk_list))
-                       goto unlock;
+int32_t
+pl_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    int32_t error;
 
-                list_for_each_entry_safe (meta_lock, tmp_metalk,
-                                          &ctx->metalk_list, client_list) {
-                        list_del_init (&meta_lock->client_list);
+    error = PL_INODE_REMOVE(rename, frame, this, oldloc, newloc, pl_rename,
+                            pl_rename_cbk, oldloc, newloc, xdata);
+    if (error > 0) {
+        STACK_UNWIND_STRICT(rename, frame, -1, error, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+    }
 
-                        pl_inode = meta_lock->pl_inode;
+    return 0;
+}
 
-                        pthread_mutex_lock (&pl_inode->mutex);
+posix_lock_t *
+gf_lkmig_info_to_posix_lock(call_frame_t *frame, lock_migration_info_t *lmi)
+{
+    posix_lock_t *lock = GF_CALLOC(1, sizeof(posix_lock_t),
+                                   gf_locks_mt_posix_lock_t);
+    if (!lock)
+        goto out;
 
-                        {
+    lock->fl_start = lmi->flock.l_start;
+    lock->fl_type = lmi->flock.l_type;
 
-                                /* Since the migration status is unknown here
-                                 * unwind all queued and blocked locks to check
-                                 * migration status and find the correct
-                                 * destination */
-                                __unwind_queued_locks (this, pl_inode,
-                                                       &tmp_posixlk_list);
+    if (lmi->flock.l_len == 0)
+        lock->fl_end = LLONG_MAX;
+    else
+        lock->fl_end = lmi->flock.l_start + lmi->flock.l_len - 1;
 
-                                __unwind_blocked_locks (this, pl_inode,
-                                                        &tmp_posixlk_list);
+    lock->client = frame->root->client;
 
-                                list_del_init (&meta_lock->list);
+    lock->lk_flags = lmi->lk_flags;
 
-                                pl_metalk_unref (meta_lock);
+    lock->client_uid = gf_strdup(lmi->client_uid);
+    if (lock->client_uid == NULL) {
+        GF_FREE(lock);
+        lock = NULL;
+        goto out;
+    }
 
-                        }
-                        pthread_mutex_unlock (&pl_inode->mutex);
+    lock->client_pid = lmi->flock.l_pid;
+    lock->owner = lmi->flock.l_owner;
 
-                        /* The corresponding ref is taken in
-                         * pl_insert_metalk*/
-                        inode_unref (pl_inode->inode);
-                }
+    INIT_LIST_HEAD(&lock->list);
+
+out:
+    return lock;
+}
+
+/* This function is supposed to write the active locks from the source brick(in
+ * rebalance context) and write here. Hence, will add the locks directly to the
+ * pl_inode->ext_list*/
+int
+pl_write_active_locks(call_frame_t *frame, pl_inode_t *pl_inode,
+                      lock_migration_info_t *locklist)
+{
+    posix_lock_t *newlock = NULL;
+    lock_migration_info_t *temp = NULL;
+    int ret = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        /* Just making sure the activelk list is empty. Should not
+         * happen though*/
+        if (!list_empty(&pl_inode->ext_list)) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "invalid locks found");
+
+            ret = -1;
+            goto out;
         }
 
-unlock:
-        pthread_mutex_unlock (&ctx->lock);
+        /* This list also should not be empty */
+        if (list_empty(&locklist->list)) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "empty lock list");
 
-        list_for_each_entry_safe (posix_lock, tmp_posixlk, &tmp_posixlk_list,
-                                  list) {
-                list_del_init (&posix_lock->list);
+            ret = -1;
+            goto out;
+        }
 
-                STACK_UNWIND_STRICT (lk, posix_lock->frame, -1, EREMOTE,
-                                     &posix_lock->user_flock, NULL);
+        list_for_each_entry(temp, &locklist->list, list)
+        {
+            newlock = gf_lkmig_info_to_posix_lock(frame, temp);
+            if (!newlock) {
+                pthread_mutex_unlock(&pl_inode->mutex);
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, 0,
+                       "mem allocation failed for newlock");
 
-                GF_FREE (posix_lock->client_uid);
-                GF_FREE (posix_lock);
+                ret = -1;
+                goto out;
+            }
+            list_add_tail(&newlock->list, &pl_inode->ext_list);
         }
-        return 0;
+    }
+    /*TODO: What if few lock add failed with ENOMEM. Should the already
+     *      added locks be clearted */
+    pthread_mutex_unlock(&pl_inode->mutex);
+out:
+    return ret;
 }
 
 static int
-pl_client_disconnect_cbk (xlator_t *this, client_t *client)
+pl_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               lock_migration_info_t *locklist, dict_t *xdata)
 {
-        pl_ctx_t *pl_ctx = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+    int ret = 0;
 
-        pl_ctx = pl_ctx_get (client, this);
+    pl_inode_t *pl_inode = pl_inode_get(this, loc->inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_inode_get failed");
 
-        pl_inodelk_client_cleanup (this, pl_ctx);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+    ret = pl_write_active_locks(frame, pl_inode, locklist);
 
-        pl_entrylk_client_cleanup (this, pl_ctx);
+    op_ret = ret;
 
-        pl_metalk_client_cleanup (this, pl_ctx);
+out:
+    STACK_UNWIND_STRICT(setactivelk, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+pl_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0);
 
-static int
-pl_client_destroy_cbk (xlator_t *this, client_t *client)
+    PL_STACK_UNWIND(unlink, xdata, frame, op_ret, op_errno, preparent,
+                    postparent, xdata);
+
+    return 0;
+}
+
+int32_t
+pl_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+          dict_t *xdata)
 {
-        void     *tmp    = NULL;
-        pl_ctx_t *pl_ctx = NULL;
+    int32_t error;
 
-        pl_client_disconnect_cbk (this, client);
+    error = PL_INODE_REMOVE(unlink, frame, this, loc, NULL, pl_unlink,
+                            pl_unlink_cbk, loc, xflag, xdata);
+    if (error > 0) {
+        STACK_UNWIND_STRICT(unlink, frame, -1, error, NULL, NULL, NULL);
+    }
 
-        client_ctx_del (client, this, &tmp);
+    return 0;
+}
 
-        if (tmp == NULL)
-                return 0;
+int32_t
+pl_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(mkdir, xdata, frame, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata);
+    return 0;
+}
 
-        pl_ctx = tmp;
+int
+pl_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
+}
 
-        GF_ASSERT (list_empty(&pl_ctx->inodelk_lockers));
-        GF_ASSERT (list_empty(&pl_ctx->entrylk_lockers));
+int32_t
+pl_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(stat, xdata, frame, op_ret, op_errno, buf,
+                               xdata);
+    return 0;
+}
 
-        pthread_mutex_destroy (&pl_ctx->lock);
-        GF_FREE (pl_ctx);
+int
+pl_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+}
 
-        return 0;
+int32_t
+pl_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(mknod, xdata, frame, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata);
+    return 0;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+pl_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        posix_locks_private_t *priv = NULL;
-        int                   ret = -1;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
+}
 
-        priv = this->private;
+int32_t
+pl_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+             dict_t *xdata)
+{
+    pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0);
 
-        GF_OPTION_RECONF ("trace", priv->trace, options, bool, out);
+    PL_STACK_UNWIND_FOR_CLIENT(rmdir, xdata, frame, op_ret, op_errno, preparent,
+                               postparent, xdata);
 
-        ret = 0;
-out:
-        return ret;
+    return 0;
 }
 
 int
-init (xlator_t *this)
+pl_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+         dict_t *xdata)
 {
-        posix_locks_private_t *priv = NULL;
-        xlator_list_t         *trav = NULL;
-        char                  *tmp_str = NULL;
-        int                   ret = -1;
+    int32_t error;
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "FATAL: posix-locks should have exactly one child");
-                goto out;
-        }
+    error = PL_INODE_REMOVE(rmdir, frame, this, loc, NULL, pl_rmdir,
+                            pl_rmdir_cbk, loc, xflags, xdata);
+    if (error > 0) {
+        STACK_UNWIND_STRICT(rmdir, frame, -1, error, NULL, NULL, NULL);
+    }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling. Please check the volume file.");
-        }
-
-        trav = this->children;
-        while (trav->xlator->children)
-                trav = trav->xlator->children;
+    return 0;
+}
 
-        if (strncmp ("storage/", trav->xlator->type, 8)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "'locks' translator is not loaded over a storage "
-                        "translator");
-                goto out;
-        }
+int32_t
+pl_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(symlink, xdata, frame, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata);
+    return 0;
+}
 
-        priv = GF_CALLOC (1, sizeof (*priv),
-                          gf_locks_mt_posix_locks_private_t);
+int
+pl_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
+}
 
-        GF_OPTION_INIT ("mandatory-locking", tmp_str, str, out);
-        if (!strcmp (tmp_str, "forced"))
-                priv->mandatory_mode = MLK_FORCED;
-        else if (!strcmp (tmp_str, "file"))
-                priv->mandatory_mode = MLK_FILE_BASED;
-        else if (!strcmp (tmp_str, "optimal"))
-                priv->mandatory_mode = MLK_OPTIMAL;
-        else
-                priv->mandatory_mode = MLK_NONE;
-        tmp_str = NULL;
+int32_t
+pl_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *buf,
+            struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    pl_inode_t *pl_inode = (pl_inode_t *)cookie;
 
-        GF_OPTION_INIT ("trace", priv->trace, bool, out);
+    if (op_ret >= 0) {
+        pthread_mutex_lock(&pl_inode->mutex);
 
-        this->local_pool = mem_pool_new (pl_local_t, 32);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
+        /* TODO: can happen pl_inode->links == 0 ? */
+        if (pl_inode->links >= 0) {
+            pl_inode->links++;
         }
 
-        this->private = priv;
-        ret = 0;
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-out:
-        if (ret) {
-                GF_FREE (priv);
-        }
-        return ret;
+    PL_STACK_UNWIND_FOR_CLIENT(link, xdata, frame, op_ret, op_errno, inode, buf,
+                               preparent, postparent, xdata);
+    return 0;
 }
 
-
 int
-fini (xlator_t *this)
+pl_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        posix_locks_private_t *priv = NULL;
-
-        priv = this->private;
-        if (!priv)
-                return 0;
-        this->private = NULL;
-        GF_FREE (priv->brickname);
-        GF_FREE (priv);
+    pl_inode_t *pl_inode;
 
+    pl_inode = pl_inode_get(this, oldloc->inode, NULL);
+    if (pl_inode == NULL) {
+        STACK_UNWIND_STRICT(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
         return 0;
+    }
+
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), oldloc, newloc);
+    STACK_WIND_COOKIE(frame, pl_link_cbk, pl_inode, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
+int32_t
+pl_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+             dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fsync, xdata, frame, op_ret, op_errno, prebuf,
+                               postbuf, xdata);
+    return 0;
+}
 
 int
-pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
-            dict_t *xdata);
+pl_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
+}
 
-int
-pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
-             dict_t *xdata);
+int32_t
+pl_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+               dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(readdir, xdata, frame, op_ret, op_errno, entries,
+                               xdata);
+    return 0;
+}
 
 int
-pl_entrylk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
+pl_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+    return 0;
+}
+
+int32_t
+pl_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fsyncdir, xdata, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
 int
-pl_fentrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
+pl_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
+}
 
 int32_t
-pl_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf,
-               struct iatt *preoldparent, struct iatt *postoldparent,
-               struct iatt *prenewparent, struct iatt *postnewparent,
-               dict_t *xdata)
+pl_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-        PL_STACK_UNWIND (rename, xdata, frame, op_ret, op_errno,
-                         buf, preoldparent, postoldparent, prenewparent,
-                         postnewparent, xdata);
-        return 0;
+    PL_STACK_UNWIND_FOR_CLIENT(statfs, xdata, frame, op_ret, op_errno, buf,
+                               xdata);
+    return 0;
+}
+
+int
+pl_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
 int32_t
-pl_rename (call_frame_t *frame, xlator_t *this,
-           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+pl_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        PL_LOCAL_GET_REQUESTS (frame, this, xdata, NULL, oldloc,  newloc);
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-        STACK_WIND (frame, pl_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->rename, oldloc,
-                    newloc, xdata);
-        return 0;
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            pl_inode->mlock_enforced = _gf_false;
+            pl_inode->check_mlock_info = _gf_false;
+            pl_inode->track_fop_wind_count = _gf_true;
+        }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
+
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(removexattr, xdata, frame, op_ret, op_errno,
+                               xdata);
+    return 0;
 }
 
-posix_lock_t *
-gf_lkmig_info_to_posix_lock (call_frame_t *frame,
-                             lock_migration_info_t *lmi)
+int
+pl_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        posix_lock_t    *lock    = NULL;
+    int op_ret = 0;
+    int op_errno = EINVAL;
+    posix_locks_private_t *priv = this->private;
 
-        lock = GF_CALLOC (1, sizeof (posix_lock_t), gf_locks_mt_posix_lock_t);
-        if (!lock)
-                goto out;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
 
-        lock->fl_start = lmi->flock.l_start;
-        lock->fl_type  = lmi->flock.l_type;
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, ((dict_t *)NULL), name, this, loc,
+                              ((fd_t *)NULL), priv);
 
-        if (lmi->flock.l_len == 0)
-                lock->fl_end = LLONG_MAX;
-        else
-               lock->fl_end = lmi->flock.l_start + lmi->flock.l_len - 1;
+    STACK_WIND(frame, pl_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 
-        lock->client = frame->root->client;
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(removexattr, xdata, frame, op_ret, op_errno,
+                               NULL);
+
+    return 0;
+}
 
-        lock->lk_flags = lmi->lk_flags;
+int32_t
+pl_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        lock->client_uid = gf_strdup (lmi->client_uid);
-        if (lock->client_uid == NULL) {
-                GF_FREE (lock);
-               goto out;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            pl_inode->mlock_enforced = _gf_false;
+            pl_inode->check_mlock_info = _gf_false;
         }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        lock->client_pid = lmi->flock.l_pid;
-        lock->owner      = lmi->flock.l_owner;
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(fremovexattr, xdata, frame, op_ret, op_errno,
+                               xdata);
+    return 0;
+}
 
-        INIT_LIST_HEAD (&lock->list);
+int
+pl_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    posix_locks_private_t *priv = this->private;
 
-out:
-        return lock;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, ((dict_t *)NULL), name, this,
+                              ((loc_t *)NULL), fd, priv);
+
+    STACK_WIND(frame, pl_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
+
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(fremovexattr, xdata, frame, op_ret, op_errno,
+                               NULL);
+    return 0;
+}
+
+int32_t
+pl_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, uint32_t weak_cksum,
+                 uint8_t *strong_cksum, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(rchecksum, xdata, frame, op_ret, op_errno,
+                               weak_cksum, strong_cksum, xdata);
+    return 0;
 }
 
-/* This function is supposed to write the active locks from the source brick(in
- * rebalance context) and write here. Hence, will add the locks directly to the
- * pl_inode->ext_list*/
 int
-pl_write_active_locks (call_frame_t *frame, pl_inode_t *pl_inode,
-                       lock_migration_info_t *locklist)
+pl_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             int32_t len, dict_t *xdata)
 {
-        posix_lock_t            *newlock        = NULL;
-        lock_migration_info_t   *temp           = NULL;
-        int                      ret            = 0;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+    return 0;
+}
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                /* Just making sure the activelk list is empty. Should not
-                 * happen though*/
-               if (!list_empty (&pl_inode->ext_list)) {
+int32_t
+pl_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(xattrop, xdata, frame, op_ret, op_errno, dict,
+                               xdata);
+    return 0;
+}
 
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0, 0,
-                                "invalid locks found");
+int
+pl_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    return 0;
+}
 
-                        ret = -1;
-                        goto out;
-               }
+int32_t
+pl_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fxattrop, xdata, frame, op_ret, op_errno, dict,
+                               xdata);
+    return 0;
+}
 
-                /* This list also should not be empty */
-                if (list_empty (&locklist->list)) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0, 0,
-                                "empty lock list");
+int
+pl_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
+    return 0;
+}
 
-                        ret = -1;
-                        goto out;
-                }
+int32_t
+pl_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+               struct iatt *statpost, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(setattr, xdata, frame, op_ret, op_errno, statpre,
+                               statpost, xdata);
+    return 0;
+}
 
-                list_for_each_entry (temp, &locklist->list, list) {
+int
+pl_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
+}
 
-                        newlock = gf_lkmig_info_to_posix_lock (frame, temp);
-                        if (!newlock) {
-                                gf_msg (THIS->name, GF_LOG_ERROR, 0, 0,
-                                        "mem allocation failed for newlock");
+int32_t
+pl_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                struct iatt *statpost, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fsetattr, xdata, frame, op_ret, op_errno,
+                               statpre, statpost, xdata);
+    return 0;
+}
 
-                                ret = -1;
-                                goto out;
-                        }
-                        list_add_tail (&newlock->list, &pl_inode->ext_list);
-                }
-        }
+int
+pl_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
+}
 
-out:
-        /*TODO: What if few lock add failed with ENOMEM. Should the already
-         *      added locks be clearted */
-        pthread_mutex_unlock (&pl_inode->mutex);
+int32_t
+pl_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                 struct iatt *post, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fallocate, xdata, frame, op_ret, op_errno, pre,
+                               post, xdata);
+    return 0;
+}
 
-        return ret;
+int
+pl_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset, len,
+               xdata);
+    return 0;
 }
 
-static int
-pl_setactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  lock_migration_info_t *locklist, dict_t *xdata)
+int32_t
+pl_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, const char *path,
+                struct iatt *buf, dict_t *xdata)
 {
-        pl_inode_t             *pl_inode        = NULL;
-        int                     op_ret          = 0;
-        int                     op_errno        = 0;
-        int                     ret             = 0;
+    PL_STACK_UNWIND_FOR_CLIENT(readlink, xdata, frame, op_ret, op_errno, path,
+                               buf, xdata);
+    return 0;
+}
 
-        pl_inode = pl_inode_get (this, loc->inode);
-        if (!pl_inode) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                        "pl_inode_get failed");
+int
+pl_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
+}
 
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-        ret = pl_write_active_locks (frame, pl_inode, locklist);
+int32_t
+pl_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(access, xdata, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        op_ret = ret;
+int
+pl_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
+}
 
-out:
-        STACK_UNWIND_STRICT (setactivelk, frame, op_ret, op_errno, NULL);
+int32_t
+pl_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(seek, xdata, frame, op_ret, op_errno, offset,
+                               xdata);
+    return 0;
+}
 
-        return 0;
+int32_t
+pl_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+    return 0;
 }
 
 struct xlator_fops fops = {
-        .lookup      = pl_lookup,
-        .create      = pl_create,
-        .fstat       = pl_fstat,
-        .truncate    = pl_truncate,
-        .ftruncate   = pl_ftruncate,
-        .discard     = pl_discard,
-        .zerofill    = pl_zerofill,
-        .open        = pl_open,
-        .readv       = pl_readv,
-        .writev      = pl_writev,
-        .lk          = pl_lk,
-        .inodelk     = pl_inodelk,
-        .finodelk    = pl_finodelk,
-        .entrylk     = pl_entrylk,
-        .fentrylk    = pl_fentrylk,
-        .flush       = pl_flush,
-        .opendir     = pl_opendir,
-        .readdirp    = pl_readdirp,
-        .getxattr    = pl_getxattr,
-        .fgetxattr   = pl_fgetxattr,
-        .fsetxattr   = pl_fsetxattr,
-        .setxattr    = pl_setxattr,
-        .rename      = pl_rename,
-        .getactivelk = pl_getactivelk,
-        .setactivelk = pl_setactivelk,
+    .lookup = pl_lookup,
+    .create = pl_create,
+    .fstat = pl_fstat,
+    .truncate = pl_truncate,
+    .ftruncate = pl_ftruncate,
+    .discard = pl_discard,
+    .zerofill = pl_zerofill,
+    .open = pl_open,
+    .readv = pl_readv,
+    .writev = pl_writev,
+    .lk = pl_lk,
+    .inodelk = pl_inodelk,
+    .finodelk = pl_finodelk,
+    .entrylk = pl_entrylk,
+    .fentrylk = pl_fentrylk,
+    .flush = pl_flush,
+    .opendir = pl_opendir,
+    .readdirp = pl_readdirp,
+    .setxattr = pl_setxattr,
+    .fsetxattr = pl_fsetxattr,
+    .getxattr = pl_getxattr,
+    .fgetxattr = pl_fgetxattr,
+    .removexattr = pl_removexattr,
+    .fremovexattr = pl_fremovexattr,
+    .rename = pl_rename,
+    .getactivelk = pl_getactivelk,
+    .setactivelk = pl_setactivelk,
+    .unlink = pl_unlink,
+    .access = pl_access,
+    .readlink = pl_readlink,
+    .fallocate = pl_fallocate,
+    .fsetattr = pl_fsetattr,
+    .setattr = pl_setattr,
+    .fxattrop = pl_fxattrop,
+    .xattrop = pl_xattrop,
+    .rchecksum = pl_rchecksum,
+    .statfs = pl_statfs,
+    .fsyncdir = pl_fsyncdir,
+    .readdir = pl_readdir,
+    .symlink = pl_symlink,
+    .link = pl_link,
+    .rmdir = pl_rmdir,
+    .mknod = pl_mknod,
+    .stat = pl_stat,
+    .seek = pl_seek,
 };
 
 struct xlator_dumpops dumpops = {
-        .inodectx    = pl_dump_inode_priv,
+    .inodectx = pl_dump_inode_priv,
 };
 
 struct xlator_cbks cbks = {
-        .forget            = pl_forget,
-        .release           = pl_release,
-        .releasedir        = pl_releasedir,
-        .client_destroy    = pl_client_destroy_cbk,
-        .client_disconnect = pl_client_disconnect_cbk,
+    .forget = pl_forget,
+    .release = pl_release,
+    .releasedir = pl_releasedir,
+    .client_destroy = pl_client_destroy_cbk,
+    .client_disconnect = pl_client_disconnect_cbk,
 };
 
 struct volume_options options[] = {
-        { .key  = { "mandatory-locking" },
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "off",
-          .description = "Specifies the mandatory-locking mode. Valid options "
-                         "are 'file' to use linux style mandatory locks, "
-                         "'forced' to use volume striclty under mandatory lock "
-                         "semantics only and 'optimal' to treat advisory and "
-                         "mandatory locks separately on their own."
-        },
-        { .key  = { "trace" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Trace the different lock requests "
-                         "to logs."
-        },
-        { .key = {NULL} },
+    {.key = {"mandatory-locking"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"locks"},
+     .description = "Specifies the mandatory-locking mode. Valid options "
+                    "are 'file' to use linux style mandatory locks, "
+                    "'forced' to use volume strictly under mandatory lock "
+                    "semantics only and 'optimal' to treat advisory and "
+                    "mandatory locks separately on their own."},
+    {.key = {"trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"locks"},
+     .description = "Trace the different lock requests "
+                    "to logs."},
+    {.key = {"monkey-unlocking"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"locks"},
+     .description = "Ignore a random number of unlock requests.  Useful "
+                    "for testing/creating robust lock recovery mechanisms."},
+    {
+        .key = {"revocation-secs"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max = INT_MAX,
+        .default_value = "0",
+        .op_version = {GD_OP_VERSION_3_9_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"locks"},
+        .description = "Maximum time a lock can be taken out, before"
+                       "being revoked.",
+    },
+    {
+        .key = {"revocation-clear-all"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_9_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"locks"},
+        .description = "If set to true, will revoke BOTH granted and blocked "
+                       "(pending) lock requests if a revocation threshold is "
+                       "hit.",
+    },
+    {.key = {"revocation-max-blocked"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = INT_MAX,
+     .default_value = "0",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"locks"},
+     .description = "A number of blocked lock requests after which a lock "
+                    "will be revoked to allow the others to proceed.  Can "
+                    "be used in conjunction w/ revocation-clear-all."},
+    {.key = {"notify-contention"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "yes",
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .tags = {"locks", "contention"},
+     .description = "When this option is enabled and a lock request "
+                    "conflicts with a currently granted lock, an upcall "
+                    "notification will be sent to the current owner of "
+                    "the lock to request it to be released as soon as "
+                    "possible."},
+    {.key = {"notify-contention-delay"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0, /* An upcall notification is sent every time a conflict is
+                * detected. */
+     .max = 60,
+     .default_value = "5",
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .tags = {"locks", "contention", "timeout"},
+     .description = "This value determines the minimum amount of time "
+                    "(in seconds) between upcall contention notifications "
+                    "on the same inode. If multiple lock requests are "
+                    "received during this period, only one upcall will "
+                    "be sent."},
+    {.key = {"enforce-mandatory-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .flags = OPT_FLAG_SETTABLE,
+     .op_version = {GD_OP_VERSION_6_0},
+     .description = "option to enable lock enforcement"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "locks",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/locks/src/reservelk.c b/xlators/features/locks/src/reservelk.c
index 8eb08d0ef79..604691fd887 100644
--- a/xlators/features/locks/src/reservelk.c
+++ b/xlators/features/locks/src/reservelk.c
@@ -7,432 +7,376 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/list.h>
 
 #include "locks.h"
 #include "common.h"
 
-void
-__delete_reserve_lock (posix_lock_t *lock)
-{
-        list_del (&lock->list);
-}
-
-void
-__destroy_reserve_lock (posix_lock_t *lock)
-{
-        GF_FREE (lock);
-}
-
 /* Return true if the two reservelks have exactly same lock boundaries */
 int
-reservelks_equal (posix_lock_t *l1, posix_lock_t *l2)
+reservelks_equal(posix_lock_t *l1, posix_lock_t *l2)
 {
-        if ((l1->fl_start == l2->fl_start) &&
-            (l1->fl_end == l2->fl_end))
-                return 1;
+    if ((l1->fl_start == l2->fl_start) && (l1->fl_end == l2->fl_end))
+        return 1;
 
-        return 0;
+    return 0;
 }
 
 /* Determine if lock is grantable or not */
 static posix_lock_t *
-__reservelk_grantable (pl_inode_t *pl_inode, posix_lock_t *lock)
+__reservelk_grantable(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        xlator_t     *this     = NULL;
-        posix_lock_t *l        = NULL;
-        posix_lock_t *ret_lock = NULL;
-
-        this = THIS;
-
-        if (list_empty (&pl_inode->reservelk_list)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No reservelks in list");
-                goto out;
-        }
-        list_for_each_entry (l, &pl_inode->reservelk_list, list){
-                if (reservelks_equal (lock, l)) {
-                        ret_lock = l;
-                        break;
-                }
+    xlator_t *this = THIS;
+    posix_lock_t *l = NULL;
+    posix_lock_t *ret_lock = NULL;
+
+    if (list_empty(&pl_inode->reservelk_list)) {
+        gf_log(this->name, GF_LOG_TRACE, "No reservelks in list");
+        goto out;
+    }
+    list_for_each_entry(l, &pl_inode->reservelk_list, list)
+    {
+        if (reservelks_equal(lock, l)) {
+            ret_lock = l;
+            break;
         }
+    }
 out:
-        return ret_lock;
+    return ret_lock;
 }
 
 static int
-__same_owner_reservelk (posix_lock_t *l1, posix_lock_t *l2)
+__same_owner_reservelk(posix_lock_t *l1, posix_lock_t *l2)
 {
-        return (is_same_lkowner (&l1->owner, &l2->owner));
-
+    return (is_same_lkowner(&l1->owner, &l2->owner));
 }
 
 static posix_lock_t *
-__matching_reservelk (pl_inode_t *pl_inode, posix_lock_t *lock)
+__matching_reservelk(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
+    posix_lock_t *l = NULL;
 
-        if (list_empty (&pl_inode->reservelk_list)) {
-            gf_log ("posix-locks", GF_LOG_TRACE,
-                    "reservelk list empty");
-            return NULL;
-        }
+    if (list_empty(&pl_inode->reservelk_list)) {
+        gf_log("posix-locks", GF_LOG_TRACE, "reservelk list empty");
+        return NULL;
+    }
 
-        list_for_each_entry (l, &pl_inode->reservelk_list, list) {
-                if (reservelks_equal (l, lock)) {
-                        gf_log ("posix-locks", GF_LOG_TRACE,
-                                "equal reservelk found");
-                        break;
-                }
+    list_for_each_entry(l, &pl_inode->reservelk_list, list)
+    {
+        if (reservelks_equal(l, lock)) {
+            gf_log("posix-locks", GF_LOG_TRACE, "equal reservelk found");
+            break;
         }
+    }
 
-        return l;
+    return l;
 }
 
 static int
-__reservelk_conflict (xlator_t *this, pl_inode_t *pl_inode,
-                      posix_lock_t *lock)
+__reservelk_conflict(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *conf = NULL;
-        int ret = 0;
-
-        conf = __matching_reservelk (pl_inode, lock);
-        if (conf) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Matching reservelk found");
-                if (__same_owner_reservelk (lock, conf)) {
-                        list_del_init (&conf->list);
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Removing the matching reservelk for setlk to progress");
-                        GF_FREE (conf);
-                        ret = 0;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Conflicting reservelk found");
-                        ret = 1;
-                }
-
+    int ret = 0;
+
+    posix_lock_t *conf = __matching_reservelk(pl_inode, lock);
+    if (conf) {
+        gf_log(this->name, GF_LOG_TRACE, "Matching reservelk found");
+        if (__same_owner_reservelk(lock, conf)) {
+            list_del_init(&conf->list);
+            gf_log(this->name, GF_LOG_TRACE,
+                   "Removing the matching reservelk for setlk to progress");
+            __destroy_lock(conf);
+            ret = 0;
+        } else {
+            gf_log(this->name, GF_LOG_TRACE, "Conflicting reservelk found");
+            ret = 1;
         }
-        return ret;
-
+    }
+    return ret;
 }
 
 int
-pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
-                     posix_lock_t *lock, int can_block)
+pl_verify_reservelk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                    const int can_block)
 {
-        int ret = 0;
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (__reservelk_conflict (this, pl_inode, lock)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Found conflicting reservelk. Blocking until reservelk is unlocked.");
-                        lock->blocked = can_block;
-                        list_add_tail (&lock->list, &pl_inode->blocked_calls);
-                        ret = -1;
-                        goto unlock;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "no conflicting reservelk found. Call continuing");
-                ret = 0;
-
+    int ret = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (__reservelk_conflict(this, pl_inode, lock)) {
+            lock->blocked = can_block;
+            list_add_tail(&lock->list, &pl_inode->blocked_calls);
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_log(this->name, GF_LOG_TRACE,
+                   "Found conflicting reservelk. Blocking until reservelk is "
+                   "unlocked.");
+            ret = -1;
+            goto out;
         }
-unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        return ret;
-
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    gf_log(this->name, GF_LOG_TRACE,
+           "no conflicting reservelk found. Call continuing");
+    ret = 0;
+out:
+    return ret;
 }
 
-
 /* Determines if lock can be granted and adds the lock. If the lock
  * is blocking, adds it to the blocked_reservelks.
  */
 static int
-__lock_reservelk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-                  int can_block)
+__lock_reservelk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                 const int can_block)
 {
-        posix_lock_t *conf = NULL;
-        int ret = -EINVAL;
-
-        conf = __reservelk_grantable (pl_inode, lock);
-        if (conf){
-                ret = -EAGAIN;
-                if (can_block == 0)
-                        goto out;
+    int ret = -EINVAL;
 
-                list_add_tail (&lock->list, &pl_inode->blocked_reservelks);
+    posix_lock_t *conf = __reservelk_grantable(pl_inode, lock);
+    if (conf) {
+        ret = -EAGAIN;
+        if (can_block == 0)
+            goto out;
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
+        list_add_tail(&lock->list, &pl_inode->blocked_reservelks);
 
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64 " => Blocked",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+               lock->user_flock.l_len);
 
-                goto out;
-        }
+        goto out;
+    }
 
-        list_add (&lock->list, &pl_inode->reservelk_list);
+    list_add(&lock->list, &pl_inode->reservelk_list);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static posix_lock_t *
-find_matching_reservelk (posix_lock_t *lock, pl_inode_t *pl_inode)
+find_matching_reservelk(posix_lock_t *lock, pl_inode_t *pl_inode)
 {
-        posix_lock_t *l = NULL;
-        list_for_each_entry (l, &pl_inode->reservelk_list, list) {
-                if (reservelks_equal (l, lock))
-                        return l;
-        }
-        return NULL;
+    posix_lock_t *l = NULL;
+    list_for_each_entry(l, &pl_inode->reservelk_list, list)
+    {
+        if (reservelks_equal(l, lock))
+            return l;
+    }
+    return NULL;
 }
 
 /* Set F_UNLCK removes a lock which has the exact same lock boundaries
  * as the UNLCK lock specifies. If such a lock is not found, returns invalid
  */
 static posix_lock_t *
-__reserve_unlock_lock (xlator_t *this, posix_lock_t *lock, pl_inode_t *pl_inode)
+__reserve_unlock_lock(xlator_t *this, posix_lock_t *lock, pl_inode_t *pl_inode)
 {
-
-        posix_lock_t *conf = NULL;
-
-        conf = find_matching_reservelk (lock, pl_inode);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " Matching lock not found for unlock");
-                goto out;
-        }
-        __delete_reserve_lock (conf);
-        gf_log (this->name, GF_LOG_DEBUG,
-                " Matching lock found for unlock");
+    posix_lock_t *conf = find_matching_reservelk(lock, pl_inode);
+    if (!conf) {
+        gf_log(this->name, GF_LOG_DEBUG, " Matching lock not found for unlock");
+        goto out;
+    }
+    __delete_lock(conf);
+    gf_log(this->name, GF_LOG_DEBUG, " Matching lock found for unlock");
 
 out:
-        return conf;
-
-
+    return conf;
 }
 
 static void
-__grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode,
-                               struct list_head *granted)
+__grant_blocked_reserve_locks(xlator_t *this, pl_inode_t *pl_inode,
+                              struct list_head *granted)
 {
-        int              bl_ret = 0;
-        posix_lock_t *bl = NULL;
-        posix_lock_t *tmp = NULL;
-
-        struct list_head blocked_list;
+    int bl_ret = 0;
+    posix_lock_t *bl = NULL;
+    posix_lock_t *tmp = NULL;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
+    struct list_head blocked_list;
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&pl_inode->blocked_reservelks, &blocked_list);
 
-                list_del_init (&bl->list);
+    list_for_each_entry_safe(bl, tmp, &blocked_list, list)
+    {
+        list_del_init(&bl->list);
 
-                bl_ret = __lock_reservelk (this, pl_inode, bl, 1);
+        bl_ret = __lock_reservelk(this, pl_inode, bl, 1);
 
-                if (bl_ret == 0) {
-                        list_add (&bl->list, granted);
-                }
+        if (bl_ret == 0) {
+            list_add(&bl->list, granted);
         }
-        return;
+    }
+    return;
 }
 
 /* Grant all reservelks blocked on lock(s) */
 void
-grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode)
+grant_blocked_reserve_locks(xlator_t *this, pl_inode_t *pl_inode)
 {
-        struct list_head granted;
-        posix_lock_t *lock = NULL;
-        posix_lock_t *tmp = NULL;
+    struct list_head granted;
+    posix_lock_t *lock = NULL;
+    posix_lock_t *tmp = NULL;
 
-        INIT_LIST_HEAD (&granted);
-
-        if (list_empty (&pl_inode->blocked_reservelks)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No blocked locks to be granted");
-                return;
-        }
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __grant_blocked_reserve_locks (this, pl_inode, &granted);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted, list) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
-
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock,
-                                     NULL);
-        }
+    INIT_LIST_HEAD(&granted);
 
+    if (list_empty(&pl_inode->blocked_reservelks)) {
+        gf_log(this->name, GF_LOG_TRACE, "No blocked locks to be granted");
+        return;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_reserve_locks(this, pl_inode, &granted);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted, list)
+    {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 " => Granted",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+               lock->user_flock.l_len);
+
+        STACK_UNWIND_STRICT(lk, lock->frame, 0, 0, &lock->user_flock, NULL);
+    }
 }
 
 static void
-__grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode,
-                            struct list_head *granted)
+__grant_blocked_lock_calls(xlator_t *this, pl_inode_t *pl_inode,
+                           struct list_head *granted)
 {
-        int              bl_ret = 0;
-        posix_lock_t *bl = NULL;
-        posix_lock_t *tmp = NULL;
+    int bl_ret = 0;
+    posix_lock_t *bl = NULL;
+    posix_lock_t *tmp = NULL;
 
-        struct list_head blocked_list;
+    struct list_head blocked_list;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&pl_inode->blocked_reservelks, &blocked_list);
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+    list_for_each_entry_safe(bl, tmp, &blocked_list, list)
+    {
+        list_del_init(&bl->list);
 
-                list_del_init (&bl->list);
+        bl_ret = pl_verify_reservelk(this, pl_inode, bl, bl->blocked);
 
-                bl_ret = pl_verify_reservelk (this, pl_inode, bl, bl->blocked);
-
-                if (bl_ret == 0) {
-                        list_add_tail (&bl->list, granted);
-                }
+        if (bl_ret == 0) {
+            list_add_tail(&bl->list, granted);
         }
-        return;
+    }
+    return;
 }
 
 void
-grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
+grant_blocked_lock_calls(xlator_t *this, pl_inode_t *pl_inode)
 {
-        struct list_head granted;
-        posix_lock_t *lock = NULL;
-        posix_lock_t *tmp = NULL;
-        fd_t         *fd  = NULL;
-
-        int can_block = 0;
-        int32_t cmd = 0;
-        int ret = 0;
-
-        if (list_empty (&pl_inode->blocked_calls)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No blocked lock calls to be granted");
-                return;
-        }
+    struct list_head granted;
+    posix_lock_t *lock = NULL;
+    posix_lock_t *tmp = NULL;
+    fd_t *fd = NULL;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __grant_blocked_lock_calls (this, pl_inode, &granted);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted, list) {
-                fd = fd_from_fdnum (lock);
-
-                if (lock->blocked) {
-                        can_block = 1;
-                        cmd = F_SETLKW;
-                }
-                else
-                        cmd = F_SETLK;
-
-                lock->blocked = 0;
-                ret = pl_setlk (this, pl_inode, lock, can_block);
-                if (ret == -1) {
-                        if (can_block) {
-                                pl_trace_block (this, lock->frame, fd, NULL,
-                                                cmd, &lock->user_flock, NULL);
-                                continue;
-                        } else {
-                                gf_log (this->name, GF_LOG_DEBUG, "returning EAGAIN");
-                                pl_trace_out (this, lock->frame, fd, NULL, cmd,
-                                              &lock->user_flock, -1, EAGAIN, NULL);
-                                pl_update_refkeeper (this, fd->inode);
-                                STACK_UNWIND_STRICT (lk, lock->frame, -1,
-                                                     EAGAIN, &lock->user_flock,
-                                                     NULL);
-                                __destroy_lock (lock);
-                        }
-                }
+    int can_block = 0;
+    int32_t cmd = 0;
+    int ret = 0;
 
+    if (list_empty(&pl_inode->blocked_calls)) {
+        gf_log(this->name, GF_LOG_TRACE, "No blocked lock calls to be granted");
+        return;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_lock_calls(this, pl_inode, &granted);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted, list)
+    {
+        fd = fd_from_fdnum(lock);
+
+        if (lock->blocked) {
+            can_block = 1;
+            cmd = F_SETLKW;
+        } else
+            cmd = F_SETLK;
+
+        lock->blocked = 0;
+        ret = pl_setlk(this, pl_inode, lock, can_block);
+        if (ret == -1) {
+            if (can_block) {
+                continue;
+            } else {
+                gf_log(this->name, GF_LOG_DEBUG, "returning EAGAIN");
+                pl_trace_out(this, lock->frame, fd, NULL, cmd,
+                             &lock->user_flock, -1, EAGAIN, NULL);
+                pl_update_refkeeper(this, fd->inode);
+                STACK_UNWIND_STRICT(lk, lock->frame, -1, EAGAIN,
+                                    &lock->user_flock, NULL);
+                __destroy_lock(lock);
+            }
         }
-
+    }
 }
 
-
 int
-pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
+pl_reserve_unlock(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *retlock = NULL;
-        int ret = -1;
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                retlock = __reserve_unlock_lock (this, lock, pl_inode);
-                if (!retlock) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Bad Unlock issued on Inode lock");
-                        ret = -EINVAL;
-                        goto out;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Reservelk Unlock successful");
-                __destroy_reserve_lock (retlock);
-                ret = 0;
+    posix_lock_t *retlock = NULL;
+    int ret = -1;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        retlock = __reserve_unlock_lock(this, lock, pl_inode);
+        if (!retlock) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_log(this->name, GF_LOG_DEBUG, "Bad Unlock issued on Inode lock");
+            ret = -EINVAL;
+            goto out;
         }
-out:
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        grant_blocked_reserve_locks (this, pl_inode);
-        grant_blocked_lock_calls (this, pl_inode);
 
-        return ret;
+        gf_log(this->name, GF_LOG_TRACE, "Reservelk Unlock successful");
+        __destroy_lock(retlock);
+        ret = 0;
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+out:
+    grant_blocked_reserve_locks(this, pl_inode);
+    grant_blocked_lock_calls(this, pl_inode);
 
+    return ret;
 }
 
 int
-pl_reserve_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-                  int can_block)
+pl_reserve_setlk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                 int can_block)
 {
-        int ret = -EINVAL;
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-
-                        ret = __lock_reservelk (this, pl_inode, lock, can_block);
-                        if (ret < 0)
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->user_flock.l_start,
-                                        lock->user_flock.l_len);
-                        else
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->fl_start,
-                                        lock->fl_end);
-
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-        return ret;
+    int ret = -EINVAL;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        ret = __lock_reservelk(this, pl_inode, lock, can_block);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (ret < 0)
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 " => NOK",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+               lock->user_flock.l_len);
+    else
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 " => OK",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->fl_start, lock->fl_end);
+
+    return ret;
 }
diff --git a/xlators/features/locks/tests/unit-test.c b/xlators/features/locks/tests/unit-test.c
index dec2ba85909..d285b12b5aa 100644
--- a/xlators/features/locks/tests/unit-test.c
+++ b/xlators/features/locks/tests/unit-test.c
@@ -7,54 +7,71 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/list.h>
 
 #include "locks.h"
 #include "common.h"
 
-#define expect(cond) if (!(cond)) { goto out; }
+#define expect(cond)                                                           \
+    if (!(cond)) {                                                             \
+        goto out;                                                              \
+    }
 
-extern int lock_name (pl_inode_t *, const char *, entrylk_type);
-extern int unlock_name (pl_inode_t *, const char *, entrylk_type);
+extern int
+lock_name(pl_inode_t *, const char *, entrylk_type);
+extern int
+unlock_name(pl_inode_t *, const char *, entrylk_type);
 
-int main (int argc, char **argv)
+int
+main(int argc, char **argv)
 {
-	int ret = 1;
-	int r = -1;
-
-	pl_inode_t *pinode = CALLOC (sizeof (pl_inode_t), 1);
-	pthread_mutex_init (&pinode->dir_lock_mutex, NULL);
-	INIT_LIST_HEAD (&pinode->gf_dir_locks);
-
-	r = lock_name (pinode, NULL, ENTRYLK_WRLCK); expect (r == 0);
-	{
-		r = lock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == -EAGAIN);
-	}
-	r = unlock_name (pinode, NULL, ENTRYLK_WRLCK); expect (r == 0);
-
-	r = lock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-	{
-		r = lock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-		{
-			r = lock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == -EAGAIN);
-		}
-		r = unlock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-	}
-	r = unlock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-	
-	r = lock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == 0);
-	r = unlock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == 0);
-
-	r = lock_name (pinode, "baz", ENTRYLK_WRLCK); expect (r == 0);
-	r = lock_name (pinode, "baz", ENTRYLK_RDLCK); expect (r == -EAGAIN);
-
-	ret = 0;
+    int ret = 1;
+    int r = -1;
+
+    pl_inode_t *pinode = CALLOC(sizeof(pl_inode_t), 1);
+    pthread_mutex_init(&pinode->dir_lock_mutex, NULL);
+    INIT_LIST_HEAD(&pinode->gf_dir_locks);
+
+    r = lock_name(pinode, NULL, ENTRYLK_WRLCK);
+    expect(r == 0);
+    {
+        r = lock_name(pinode, "foo", ENTRYLK_WRLCK);
+        expect(r == -EAGAIN);
+    }
+    r = unlock_name(pinode, NULL, ENTRYLK_WRLCK);
+    expect(r == 0);
+
+    r = lock_name(pinode, "foo", ENTRYLK_RDLCK);
+    expect(r == 0);
+    {
+        r = lock_name(pinode, "foo", ENTRYLK_RDLCK);
+        expect(r == 0);
+        {
+            r = lock_name(pinode, "foo", ENTRYLK_WRLCK);
+            expect(r == -EAGAIN);
+        }
+        r = unlock_name(pinode, "foo", ENTRYLK_RDLCK);
+        expect(r == 0);
+    }
+    r = unlock_name(pinode, "foo", ENTRYLK_RDLCK);
+    expect(r == 0);
+
+    r = lock_name(pinode, "foo", ENTRYLK_WRLCK);
+    expect(r == 0);
+    r = unlock_name(pinode, "foo", ENTRYLK_WRLCK);
+    expect(r == 0);
+
+    r = lock_name(pinode, "baz", ENTRYLK_WRLCK);
+    expect(r == 0);
+    r = lock_name(pinode, "baz", ENTRYLK_RDLCK);
+    expect(r == -EAGAIN);
+
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
diff --git a/xlators/features/mac-compat/Makefile.am b/xlators/features/mac-compat/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/mac-compat/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/mac-compat/src/Makefile.am b/xlators/features/mac-compat/src/Makefile.am
deleted file mode 100644
index 1a312991f6b..00000000000
--- a/xlators/features/mac-compat/src/Makefile.am
+++ /dev/null
@@ -1,15 +0,0 @@
-xlator_LTLIBRARIES = mac-compat.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-mac_compat_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-mac_compat_la_SOURCES = mac-compat.c
-mac_compat_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = mac-compat.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/mac-compat/src/mac-compat.c b/xlators/features/mac-compat/src/mac-compat.c
deleted file mode 100644
index 795a387d484..00000000000
--- a/xlators/features/mac-compat/src/mac-compat.c
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
-#include "syscall.h"
-#include "mem-pool.h"
-#include "mac-compat.h"
-
-static int
-dict_key_remove_namespace(dict_t *dict, char *key, data_t *value, void *data)
-{
-  /*
-    char buffer[3*value->len+1];
-    int index = 0;
-    for (index = 0; index < value->len; index++)
-    sprintf(buffer+3*index, " %02x", value->data[index]);
-  */
-        xlator_t *this = (xlator_t *) data;
-        if (strncmp(key, "user.", 5) == 0) {
-                dict_set (dict, key + 5, value);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "remove_namespace_dict: %s -> %s ", key, key + 5);
-                dict_del (dict, key);
-        }
-        return 0;
-}
-
-int32_t
-maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict,
-                      dict_t *xdata)
-{
-        intptr_t ax = (intptr_t)this->private;
-        int i = 0;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "getxattr_cbk: dict %p private: %p xdata %p ", dict,
-                this->private, xdata);
-
-        if (dict) {
-                dict_foreach(dict, dict_key_remove_namespace, this);
-        }
-        else {
-                // TODO: we expect dict to exist here, don't know why this
-                // this is needed
-                dict = dict_new();
-        }
-        gf_log (this->name, GF_LOG_DEBUG,
-                "getxattr_cbk: dict %p ax: %ld op_ret %d op_err %d  ", dict, ax,
-                op_ret, op_errno);
-        if ((ax == GF_XATTR_ALL && op_ret >= 0) || ax != GF_XATTR_NONE) {
-                op_ret = op_errno = 0;
-                for (i = 0; i < GF_XATTR_ALL; i++) {
-                        if (dict_get (dict, apple_xattr_name[i]))
-                                continue;
-                        /* set dummy data */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "getxattr_cbk: setting dummy data %p, %s", dict,
-                                apple_xattr_name[i]);
-                        if (dict_set (dict, apple_xattr_name[i],
-                                      bin_to_data ((void *)apple_xattr_value[i],
-                                                   apple_xattr_len[i])) == -1) {
-                                op_ret = -1;
-                                op_errno = ENOATTR;
-
-                                break;
-                        }
-                }
-         }
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
-
-
-static
-int prepend_xattr_user_namespace(dict_t *dict, char *key, data_t *value, void *obj)
-{
-        xlator_t *this = (xlator_t *) obj;
-        dict_t *newdict = (dict_t *) this->private;
-        char *newkey = NULL;
-        gf_add_prefix(XATTR_USER_PREFIX, key, &newkey);
-        key = newkey;
-        dict_set(newdict, (char *)key, value);
-        if (newkey)
-                GF_FREE(newkey);
-        return 0;
-}
-
-intptr_t
-check_name(const char *name, char **newkey)
-{
-        intptr_t ax = GF_XATTR_NONE;
-        if (name) {
-                int i = 0;
-                for (i = 0; i < GF_XATTR_ALL; i++) {
-                        if (strcmp (apple_xattr_name[i], name) == 0) {
-                                ax = i;
-                                break;
-                        }
-                }
-                gf_add_prefix("user.", name, newkey);
-        } else
-                ax = GF_XATTR_ALL;
-        return ax;
-}
-
-int32_t
-maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-        this->private = (void *) check_name(name, &newkey);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "getxattr: name %s private: %p xdata %p ", name,
-                this->private, xdata);
-        STACK_WIND (frame, maccomp_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, newkey, xdata);
-        return 0;
-}
-
-
-int32_t
-maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-        this->private = (void *) check_name(name, &newkey);
-
-        STACK_WIND (frame, maccomp_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, newkey, xdata);
-        GF_FREE(newkey);
-        return 0;
-}
-
-int32_t
-maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        intptr_t ax = (intptr_t)this->private;
-
-        if (op_ret == -1 && ax != GF_XATTR_NONE)
-                op_ret = op_errno = 0;
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setxattr_cbk op_ret %d op_errno %d private: %p xdata %p ",
-                op_ret, op_errno, this->private, xdata);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-maccomp_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *iatt1,
-                     struct iatt *iattr2, dict_t *xdata)
-{
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setattr_cbk op_ret %d op_errno %d private: %p xdata %p ",
-                op_ret, op_errno, this->private, xdata);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno,
-                             iatt1, iattr2, xdata);
-        return 0;
-}
-
-int map_flags(int flags)
-{
-        /* DARWIN has different defines on XATTR_ flags.
-           There do not seem to be a POSIX standard
-           Parse any other flags over.
-           NOFOLLOW is always true on Linux and Darwin
-        */
-        int linux_flags = flags & ~(GF_XATTR_CREATE | GF_XATTR_REPLACE | XATTR_REPLACE);
-        if (XATTR_CREATE & flags)
-                linux_flags |= GF_XATTR_CREATE;
-        if (XATTR_REPLACE & flags)
-                linux_flags |= GF_XATTR_REPLACE;
-        return linux_flags;
-}
-
-int32_t
-maccomp_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-
-        this->private = (void *) check_name(name, &newkey);
-
-        STACK_WIND (frame, default_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, newkey, xdata);
-        GF_FREE(newkey);
-        return 0;
-}
-
-int32_t
-maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags, dict_t *xdata)
-{
-        intptr_t ax = GF_XATTR_NONE;
-        int i = 0;
-
-        for (i = 0; i < GF_XATTR_ALL; i++) {
-                if (dict_get (dict, apple_xattr_name[i])) {
-                        ax = i;
-
-                        break;
-                }
-        }
-        dict_t *newdict = dict_new();
-        this->private = (void *) newdict;
-        dict_foreach(dict, prepend_xattr_user_namespace, this);
-
-        this->private = (void *)ax;
-        int linux_flags = map_flags(flags);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setxattr flags: %d -> %d dict %p private: %p xdata %p ",
-                flags, linux_flags, dict, this->private, xdata);
-        STACK_WIND (frame, maccomp_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, newdict, linux_flags, xdata);
-        dict_unref(newdict);
-        return 0;
-}
-
-int32_t
-maccomp_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *iattr,
-                  int32_t flags, dict_t *xdata)
-{
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setattr iattr %p private: %p xdata %p ",
-                iattr, this->private, xdata);
-        STACK_WIND (frame, maccomp_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, iattr, flags, xdata);
-        return 0;
-}
-
-int32_t
-maccomp_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-        this->private = (void *) check_name(name, &newkey);
-
-        STACK_WIND (frame, default_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, newkey, xdata);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "removeattr name %p private: %p xdata %p ",
-                name, this->private, xdata);
-        GF_FREE(newkey);
-        return 0;
-
-}
-
-int32_t
-maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
-{
-        intptr_t ax = GF_XATTR_NONE;
-        int i = 0;
-
-        for (i = 0; i < GF_XATTR_ALL; i++) {
-                if (dict_get (dict, apple_xattr_name[i])) {
-                        ax = i;
-
-                        break;
-                }
-        }
-
-        dict_t *newdict = dict_new();
-        this->private = (void *) newdict;
-        dict_foreach(dict, prepend_xattr_user_namespace, this);
-
-        this->private = (void *)ax;
-        int linux_flags = map_flags(flags);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "fsetxattr flags: %d -> %d dict %p private: %p xdata %p ",
-                flags, linux_flags, dict, this->private, xdata);
-        STACK_WIND (frame, maccomp_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, newdict, linux_flags, xdata);
-        dict_unref(newdict);
-        return 0;
-}
-
-
-int32_t
-init (xlator_t *this)
-{
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "translator not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        return;
-}
-
-
-struct xlator_fops fops = {
-        .getxattr       = maccomp_getxattr,
-        .fgetxattr      = maccomp_fgetxattr,
-        .setxattr       = maccomp_setxattr,
-        .setattr        = maccomp_setattr,
-        .fsetxattr      = maccomp_fsetxattr,
-        .removexattr    = maccomp_removexattr,
-        .fremovexattr   = maccomp_fremovexattr,
-};
-
-struct xlator_cbks cbks;
-
-struct volume_options options[] = {
-        { .key = {NULL} },
-};
diff --git a/xlators/features/mac-compat/src/mac-compat.h b/xlators/features/mac-compat/src/mac-compat.h
deleted file mode 100644
index b033ca0e4d8..00000000000
--- a/xlators/features/mac-compat/src/mac-compat.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
-   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __MAC_COMPAT_H__
-#define __MAC_COMPAT_H__
-
-enum apple_xattr {
-        GF_FINDER_INFO_XATTR,
-        GF_RESOURCE_FORK_XATTR,
-        GF_XATTR_ALL,
-        GF_XATTR_NONE
-};
-
-static char *apple_xattr_name[] = {
-        [GF_FINDER_INFO_XATTR]   = "com.apple.FinderInfo",
-        [GF_RESOURCE_FORK_XATTR] = "com.apple.ResourceFork"
-};
-
-static const char *apple_xattr_value[] = {
-        [GF_FINDER_INFO_XATTR]   =
-        /* 1 2 3 4 5 6 7 8 */
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0",
-        [GF_RESOURCE_FORK_XATTR] = ""
-};
-
-static int32_t apple_xattr_len[] = {
-        [GF_FINDER_INFO_XATTR]   = 32,
-        [GF_RESOURCE_FORK_XATTR] = 1
-};
-
-#endif /* __MAC_COMPAT_H__ */
diff --git a/xlators/features/marker/src/Makefile.am b/xlators/features/marker/src/Makefile.am
index 0465b02012e..58056b36511 100644
--- a/xlators/features/marker/src/Makefile.am
+++ b/xlators/features/marker/src/Makefile.am
@@ -1,14 +1,21 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = marker.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 marker_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
-marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c marker-common.c
+marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c \
+	marker-common.c
+
 marker_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h marker-quota-helper.h marker-common.h $(top_builddir)/xlators/lib/src/libxlator.h
+noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h \
+	marker-quota-helper.h marker-common.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(top_srcdir)/xlators/lib/src
 
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
diff --git a/xlators/features/marker/src/marker-common.c b/xlators/features/marker/src/marker-common.c
index 6ec5e3cc8bc..9c9047005d6 100644
--- a/xlators/features/marker/src/marker-common.c
+++ b/xlators/features/marker/src/marker-common.c
@@ -11,55 +11,47 @@
 #include "marker-common.h"
 
 marker_inode_ctx_t *
-marker_inode_ctx_new ()
+marker_inode_ctx_new()
 {
-        marker_inode_ctx_t *ctx = NULL;
+    marker_inode_ctx_t *ctx = NULL;
 
-        ctx = GF_CALLOC (1, sizeof (marker_inode_ctx_t),
-                         gf_marker_mt_marker_inode_ctx_t);
-        if (ctx == NULL)
-                goto out;
+    ctx = GF_CALLOC(1, sizeof(marker_inode_ctx_t),
+                    gf_marker_mt_marker_inode_ctx_t);
+    if (ctx == NULL)
+        goto out;
 
-        ctx->quota_ctx = NULL;
+    ctx->quota_ctx = NULL;
 out:
-        return ctx;
+    return ctx;
 }
 
 int32_t
-marker_force_inode_ctx_get (inode_t *inode, xlator_t *this,
-                            marker_inode_ctx_t **ctx)
+marker_force_inode_ctx_get(inode_t *inode, xlator_t *this,
+                           marker_inode_ctx_t **ctx)
 {
-        int32_t  ret     = -1;
-        uint64_t ctx_int = 0;
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_int);
-                if (ret == 0)
-                        *ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
-                else {
-                        *ctx = marker_inode_ctx_new ();
-                        if (*ctx == NULL)
-                                goto unlock;
-
-                        ret = __inode_ctx_put (inode, this,
-                                               (uint64_t )(unsigned long) *ctx);
-                        if (ret == -1) {
-                                GF_FREE (*ctx);
-                                goto unlock;
-                        }
-                        ret = 0;
-                }
+    int32_t ret = -1;
+    uint64_t ctx_int = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
+        if (ret == 0)
+            *ctx = (marker_inode_ctx_t *)(unsigned long)ctx_int;
+        else {
+            *ctx = marker_inode_ctx_new();
+            if (*ctx == NULL)
+                goto unlock;
+
+            ret = __inode_ctx_put(inode, this, (uint64_t)(unsigned long)*ctx);
+            if (ret == -1) {
+                GF_FREE(*ctx);
+                goto unlock;
+            }
+            ret = 0;
         }
-unlock: UNLOCK (&inode->lock);
+    }
+unlock:
+    UNLOCK(&inode->lock);
 
-        return ret;
-}
-
-int
-marker_filter_quota_xattr (dict_t *dict, char *key,
-                           data_t *value, void *data)
-{
-        dict_del (dict, key);
-        return 0;
+    return ret;
 }
diff --git a/xlators/features/marker/src/marker-common.h b/xlators/features/marker/src/marker-common.h
index c6ca422dd6a..7f8cffe7d35 100644
--- a/xlators/features/marker/src/marker-common.h
+++ b/xlators/features/marker/src/marker-common.h
@@ -10,13 +10,10 @@
 #ifndef _MARKER_COMMON_H
 #define _MARKER_COMMON_H
 
-#include "inode.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "marker.h"
 
 int32_t
-marker_force_inode_ctx_get (inode_t *, xlator_t *, marker_inode_ctx_t **);
+marker_force_inode_ctx_get(inode_t *, xlator_t *, marker_inode_ctx_t **);
 
-int
-marker_filter_quota_xattr (dict_t *, char *, data_t *, void *);
 #endif
diff --git a/xlators/features/marker/src/marker-mem-types.h b/xlators/features/marker/src/marker-mem-types.h
index dc5ad16ed76..aedfdb4a1b7 100644
--- a/xlators/features/marker/src/marker-mem-types.h
+++ b/xlators/features/marker/src/marker-mem-types.h
@@ -10,18 +10,19 @@
 #ifndef __MARKER_MEM_TYPES_H__
 #define __MARKER_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_marker_mem_types_ {
-        gf_marker_mt_marker_conf_t = gf_common_mt_end + 1,
-        gf_marker_mt_loc_t,
-        gf_marker_mt_volume_mark,
-        gf_marker_mt_int64_t,
-        gf_marker_mt_quota_inode_ctx_t,
-        gf_marker_mt_marker_inode_ctx_t,
-        gf_marker_mt_inode_contribution_t,
-        gf_marker_mt_quota_meta_t,
-        gf_marker_mt_quota_synctask_t,
-        gf_marker_mt_end
+    /* Those are used by ALLOCATE_OR_GOTO macro */
+    gf_marker_mt_marker_conf_t = gf_common_mt_end + 1,
+    gf_marker_mt_loc_t,
+    gf_marker_mt_volume_mark,
+    gf_marker_mt_int64_t,
+    gf_marker_mt_quota_inode_ctx_t,
+    gf_marker_mt_marker_inode_ctx_t,
+    gf_marker_mt_inode_contribution_t,
+    gf_marker_mt_quota_meta_t,
+    gf_marker_mt_quota_synctask_t,
+    gf_marker_mt_end
 };
 #endif
diff --git a/xlators/features/marker/src/marker-quota-helper.c b/xlators/features/marker/src/marker-quota-helper.c
index 1fed9df6d6a..ecd85d67b2b 100644
--- a/xlators/features/marker/src/marker-quota-helper.c
+++ b/xlators/features/marker/src/marker-quota-helper.c
@@ -7,475 +7,374 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "locking.h"
+#include <glusterfs/locking.h>
 #include "marker-quota.h"
 #include "marker-common.h"
 #include "marker-quota-helper.h"
 #include "marker-mem-types.h"
 
 int
-mq_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+mq_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", path, out);
-        /* Not checking for parent because while filling
-         * loc of root, parent will be NULL
-         */
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", inode, out);
+    GF_VALIDATE_OR_GOTO("marker", path, out);
+    /* Not checking for parent because while filling
+     * loc of root, parent will be NULL
+     */
 
-        if (inode) {
-                loc->inode = inode_ref (inode);
-        }
+    if (inode) {
+        loc->inode = inode_ref(inode);
+    }
 
-        if (parent)
-                loc->parent = inode_ref (parent);
+    if (parent)
+        loc->parent = inode_ref(parent);
 
-        if (!gf_uuid_is_null (inode->gfid))
-                gf_uuid_copy (loc->gfid, inode->gfid);
+    if (!gf_uuid_is_null(inode->gfid))
+        gf_uuid_copy(loc->gfid, inode->gfid);
 
-        loc->path = gf_strdup (path);
-        if (!loc->path) {
-                gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
+    loc->path = gf_strdup(path);
+    if (!loc->path) {
+        gf_log("loc fill", GF_LOG_ERROR, "strdup failed");
+        goto out;
+    }
 
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-        else
-                goto out;
+    loc->name = strrchr(loc->path, '/');
+    if (loc->name)
+        loc->name++;
+    else
+        goto out;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret < 0)
-                loc_wipe (loc);
+    if (ret < 0)
+        loc_wipe(loc);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mq_inode_loc_fill (const char *parent_gfid, inode_t *inode, loc_t *loc)
+mq_inode_loc_fill(const char *parent_gfid, inode_t *inode, loc_t *loc)
 {
-        char                *resolvedpath = NULL;
-        inode_t             *parent       = NULL;
-        quota_inode_ctx_t   *ctx          = NULL;
-        xlator_t            *this         = NULL;
-        int                  ret          = -1;
-
-        this = THIS;
-
-        if (inode == NULL) {
-                gf_log_callingfn ("marker", GF_LOG_ERROR, "loc fill failed, "
-                                  "inode is NULL");
-                return ret;
-        }
+    char *resolvedpath = NULL;
+    inode_t *parent = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+
+    if (inode == NULL) {
+        gf_log_callingfn("marker", GF_LOG_ERROR,
+                         "loc fill failed, "
+                         "inode is NULL");
+        return ret;
+    }
 
-        if (loc == NULL)
-                return ret;
+    if (loc == NULL)
+        return ret;
 
-        if ((inode) && __is_root_gfid (inode->gfid)) {
-                loc->parent = NULL;
-                goto ignore_parent;
-        }
+    if ((inode) && __is_root_gfid(inode->gfid)) {
+        loc->parent = NULL;
+        goto ignore_parent;
+    }
 
-        if (parent_gfid == NULL)
-                parent = inode_parent (inode, 0, NULL);
-        else
-                parent = inode_find (inode->table,
-                                     (unsigned char *) parent_gfid);
+    if (parent_gfid == NULL)
+        parent = inode_parent(inode, 0, NULL);
+    else
+        parent = inode_find(inode->table, (unsigned char *)parent_gfid);
 
-        if (parent == NULL) {
-                gf_log ("marker", GF_LOG_ERROR, "parent is NULL for %s",
-                        uuid_utoa(inode->gfid));
-                goto err;
-        }
+    if (parent == NULL) {
+        gf_log("marker", GF_LOG_ERROR, "parent is NULL for %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
 
 ignore_parent:
-        ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0) {
-                gf_log ("marker", GF_LOG_ERROR, "failed to resolve path for %s",
-                        uuid_utoa(inode->gfid));
-                goto err;
-        }
-
-        ret = mq_loc_fill (loc, inode, parent, resolvedpath);
-        if (ret < 0)
-                goto err;
-
-        ret = mq_inode_ctx_get (inode, this, &ctx);
-        if (ret < 0 || ctx == NULL)
-                ctx = mq_inode_ctx_new (inode, this);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                        "failed for %s", uuid_utoa (inode->gfid));
-                ret = -1;
-                goto err;
-        }
-        ret = 0;
+    ret = inode_path(inode, NULL, &resolvedpath);
+    if (ret < 0) {
+        gf_log("marker", GF_LOG_ERROR, "failed to resolve path for %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
+
+    ret = mq_loc_fill(loc, inode, parent, resolvedpath);
+    if (ret < 0)
+        goto err;
+
+    ret = mq_inode_ctx_get(inode, this, &ctx);
+    if (ret < 0 || ctx == NULL)
+        ctx = mq_inode_ctx_new(inode, this);
+    if (ctx == NULL) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "mq_inode_ctx_new "
+               "failed for %s",
+               uuid_utoa(inode->gfid));
+        ret = -1;
+        goto err;
+    }
+    ret = 0;
 
 err:
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
 }
 
-
 quota_inode_ctx_t *
-mq_alloc_inode_ctx ()
+mq_alloc_inode_ctx()
 {
-        int32_t                  ret    = -1;
-        quota_inode_ctx_t       *ctx    = NULL;
-
-        QUOTA_ALLOC (ctx, quota_inode_ctx_t, ret);
-        if (ret == -1)
-                goto out;
-
-        ctx->size = 0;
-        ctx->dirty = 0;
-        ctx->updation_status = _gf_false;
-        LOCK_INIT (&ctx->lock);
-        INIT_LIST_HEAD (&ctx->contribution_head);
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+
+    QUOTA_ALLOC(ctx, quota_inode_ctx_t, ret);
+    if (ret == -1)
+        goto out;
+
+    ctx->size = 0;
+    ctx->dirty = 0;
+    ctx->updation_status = _gf_false;
+    LOCK_INIT(&ctx->lock);
+    INIT_LIST_HEAD(&ctx->contribution_head);
 out:
-        return ctx;
+    return ctx;
 }
 
-void
-mq_contri_fini (void *data)
+static void
+mq_contri_fini(inode_contribution_t *contri)
 {
-        inode_contribution_t *contri = data;
-
-        LOCK_DESTROY (&contri->lock);
-        GF_FREE (contri);
+    LOCK_DESTROY(&contri->lock);
+    GF_FREE(contri);
 }
 
-inode_contribution_t*
-mq_contri_init (inode_t *inode)
+inode_contribution_t *
+mq_contri_init(inode_t *inode)
 {
-        inode_contribution_t *contri   = NULL;
-        int32_t               ret      = 0;
+    inode_contribution_t *contri = NULL;
+    int32_t ret = 0;
 
-        QUOTA_ALLOC (contri, inode_contribution_t, ret);
-        if (ret == -1)
-                goto out;
+    QUOTA_ALLOC(contri, inode_contribution_t, ret);
+    if (ret == -1)
+        goto out;
 
-        GF_REF_INIT (contri, mq_contri_fini);
+    GF_REF_INIT(contri, mq_contri_fini);
 
-        contri->contribution = 0;
-        contri->file_count = 0;
-        contri->dir_count = 0;
-        gf_uuid_copy (contri->gfid, inode->gfid);
+    contri->contribution = 0;
+    contri->file_count = 0;
+    contri->dir_count = 0;
+    gf_uuid_copy(contri->gfid, inode->gfid);
 
-        LOCK_INIT (&contri->lock);
-        INIT_LIST_HEAD (&contri->contri_list);
+    LOCK_INIT(&contri->lock);
+    INIT_LIST_HEAD(&contri->contri_list);
 
 out:
-        return contri;
+    return contri;
 }
 
 inode_contribution_t *
-mq_get_contribution_node (inode_t *inode, quota_inode_ctx_t *ctx)
+mq_get_contribution_node(inode_t *inode, quota_inode_ctx_t *ctx)
 {
-        inode_contribution_t    *contri = NULL;
-        inode_contribution_t    *temp   = NULL;
+    inode_contribution_t *contri = NULL;
+    inode_contribution_t *temp = NULL;
+
+    if (!inode || !ctx)
+        goto out;
 
-        if (!inode || !ctx)
-                goto out;
+    LOCK(&ctx->lock);
+    {
+        if (list_empty(&ctx->contribution_head))
+            goto unlock;
 
-        LOCK (&ctx->lock);
+        list_for_each_entry(temp, &ctx->contribution_head, contri_list)
         {
-                if (list_empty (&ctx->contribution_head))
-                        goto unlock;
-
-                list_for_each_entry (temp, &ctx->contribution_head,
-                                     contri_list) {
-                        if (gf_uuid_compare (temp->gfid, inode->gfid) == 0) {
-                                contri = temp;
-                                GF_REF_GET (contri);
-                                break;
-                        }
-                }
+            if (gf_uuid_compare(temp->gfid, inode->gfid) == 0) {
+                contri = temp;
+                GF_REF_GET(contri);
+                break;
+            }
         }
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 out:
-        return contri;
+    return contri;
 }
 
 inode_contribution_t *
-__mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
-                                loc_t *loc)
+__mq_add_new_contribution_node(xlator_t *this, quota_inode_ctx_t *ctx,
+                               loc_t *loc)
 {
-        inode_contribution_t *contribution = NULL;
-
-        if (!loc->parent) {
-                if (!gf_uuid_is_null (loc->pargfid))
-                        loc->parent = inode_find (loc->inode->table,
-                                                  loc->pargfid);
-
-                if (!loc->parent)
-                        loc->parent = inode_parent (loc->inode, loc->pargfid,
-                                                    loc->name);
-                if (!loc->parent)
-                        goto out;
-        }
-
-        list_for_each_entry (contribution, &ctx->contribution_head,
-                             contri_list) {
-                if (loc->parent &&
-                    gf_uuid_compare (contribution->gfid, loc->parent->gfid) == 0) {
-                        goto out;
-                }
+    inode_contribution_t *contribution = NULL;
+
+    if (!loc->parent) {
+        if (!gf_uuid_is_null(loc->pargfid))
+            loc->parent = inode_find(loc->inode->table, loc->pargfid);
+
+        if (!loc->parent)
+            loc->parent = inode_parent(loc->inode, loc->pargfid, loc->name);
+        if (!loc->parent)
+            goto out;
+    }
+
+    list_for_each_entry(contribution, &ctx->contribution_head, contri_list)
+    {
+        if (loc->parent &&
+            gf_uuid_compare(contribution->gfid, loc->parent->gfid) == 0) {
+            goto out;
         }
+    }
 
-        contribution = mq_contri_init (loc->parent);
-        if (contribution == NULL)
-                goto out;
+    contribution = mq_contri_init(loc->parent);
+    if (contribution == NULL)
+        goto out;
 
-        list_add_tail (&contribution->contri_list, &ctx->contribution_head);
+    list_add_tail(&contribution->contri_list, &ctx->contribution_head);
 
 out:
-        return contribution;
+    return contribution;
 }
 
-
 inode_contribution_t *
-mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
-                              loc_t *loc)
+mq_add_new_contribution_node(xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
 {
-        inode_contribution_t *contribution = NULL;
+    inode_contribution_t *contribution = NULL;
 
-        if ((ctx == NULL) || (loc == NULL))
-                return NULL;
+    if ((ctx == NULL) || (loc == NULL))
+        return NULL;
 
-        if (((loc->path) && (strcmp (loc->path, "/") == 0))
-            || (!loc->path && gf_uuid_is_null (loc->pargfid)))
-                return NULL;
+    if (((loc->path) && (strcmp(loc->path, "/") == 0)) ||
+        (!loc->path && gf_uuid_is_null(loc->pargfid)))
+        return NULL;
 
-        LOCK (&ctx->lock);
-        {
-                contribution = __mq_add_new_contribution_node (this, ctx, loc);
-                if (contribution)
-                        GF_REF_GET (contribution);
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        contribution = __mq_add_new_contribution_node(this, ctx, loc);
+        if (contribution)
+            GF_REF_GET(contribution);
+    }
+    UNLOCK(&ctx->lock);
 
-        return contribution;
+    return contribution;
 }
 
-
 int32_t
-mq_dict_set_contribution (xlator_t *this, dict_t *dict, loc_t *loc,
-                          uuid_t gfid, char *contri_key)
+mq_dict_set_contribution(xlator_t *this, dict_t *dict, loc_t *loc, uuid_t gfid,
+                         char *contri_key)
 {
-        int32_t ret                  = -1;
-        char    key[QUOTA_KEY_MAX]   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", dict, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-
-        if (gfid && !gf_uuid_is_null(gfid)) {
-                GET_CONTRI_KEY (this, key, gfid, ret);
-        } else if (loc->parent) {
-                GET_CONTRI_KEY (this, key, loc->parent->gfid, ret);
-        } else {
-                /* nameless lookup, fetch contributions to all parents */
-                GET_CONTRI_KEY (this, key, NULL, ret);
+    int32_t ret = -1;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", dict, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+
+    if (gfid && !gf_uuid_is_null(gfid)) {
+        GET_CONTRI_KEY(this, key, gfid, ret);
+    } else if (loc->parent) {
+        GET_CONTRI_KEY(this, key, loc->parent->gfid, ret);
+    } else {
+        /* nameless lookup, fetch contributions to all parents */
+        GET_CONTRI_KEY(this, key, NULL, ret);
+    }
+
+    if (ret < 0)
+        goto out;
+
+    ret = dict_set_int64(dict, key, 0);
+    if (ret < 0)
+        goto out;
+
+    if (contri_key)
+        if (snprintf(contri_key, QUOTA_KEY_MAX, "%s", key) >= QUOTA_KEY_MAX) {
+            ret = -1;
+            goto out;
         }
 
-        if (ret < 0)
-                goto out;
-
-        ret = dict_set_int64 (dict, key, 0);
-        if (ret < 0)
-                goto out;
-
-        if (contri_key)
-                strncpy (contri_key, key, QUOTA_KEY_MAX);
-
 out:
-        if (ret < 0)
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "dict set failed");
+    if (ret < 0)
+        gf_log_callingfn(this ? this->name : "Marker", GF_LOG_ERROR,
+                         "dict set failed");
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mq_inode_ctx_get (inode_t *inode, xlator_t *this,
-                  quota_inode_ctx_t **ctx)
+mq_inode_ctx_get(inode_t *inode, xlator_t *this, quota_inode_ctx_t **ctx)
 {
-        int32_t             ret      = -1;
-        uint64_t            ctx_int  = 0;
-        marker_inode_ctx_t *mark_ctx = NULL;
+    int32_t ret = -1;
+    uint64_t ctx_int = 0;
+    marker_inode_ctx_t *mark_ctx = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", inode, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        ret = inode_ctx_get (inode, this, &ctx_int);
-        if (ret < 0) {
-                ret = -1;
-                *ctx = NULL;
-                goto out;
-        }
+    ret = inode_ctx_get(inode, this, &ctx_int);
+    if (ret < 0) {
+        ret = -1;
+        *ctx = NULL;
+        goto out;
+    }
 
-        mark_ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
-        if (mark_ctx->quota_ctx == NULL) {
-                ret = -1;
-                goto out;
-        }
+    mark_ctx = (marker_inode_ctx_t *)(unsigned long)ctx_int;
+    if (mark_ctx->quota_ctx == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        *ctx = mark_ctx->quota_ctx;
+    *ctx = mark_ctx->quota_ctx;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 quota_inode_ctx_t *
-__mq_inode_ctx_new (inode_t *inode, xlator_t *this)
+__mq_inode_ctx_new(inode_t *inode, xlator_t *this)
 {
-        int32_t               ret        = -1;
-        quota_inode_ctx_t    *quota_ctx  = NULL;
-        marker_inode_ctx_t   *mark_ctx   = NULL;
-
-        ret = marker_force_inode_ctx_get (inode, this, &mark_ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "marker_force_inode_ctx_get() failed");
-                goto out;
+    int32_t ret = -1;
+    quota_inode_ctx_t *quota_ctx = NULL;
+    marker_inode_ctx_t *mark_ctx = NULL;
+
+    ret = marker_force_inode_ctx_get(inode, this, &mark_ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "marker_force_inode_ctx_get() failed");
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        if (mark_ctx->quota_ctx == NULL) {
+            quota_ctx = mq_alloc_inode_ctx();
+            if (quota_ctx == NULL) {
+                ret = -1;
+                goto unlock;
+            }
+            mark_ctx->quota_ctx = quota_ctx;
+        } else {
+            quota_ctx = mark_ctx->quota_ctx;
         }
 
-        LOCK (&inode->lock);
-        {
-                if (mark_ctx->quota_ctx == NULL) {
-                        quota_ctx = mq_alloc_inode_ctx ();
-                        if (quota_ctx == NULL) {
-                                ret = -1;
-                                goto unlock;
-                        }
-                        mark_ctx->quota_ctx = quota_ctx;
-                } else {
-                        quota_ctx = mark_ctx->quota_ctx;
-                }
-
-                ret = 0;
-        }
+        ret = 0;
+    }
 unlock:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 out:
-        return quota_ctx;
+    return quota_ctx;
 }
 
-
 quota_inode_ctx_t *
-mq_inode_ctx_new (inode_t * inode, xlator_t *this)
-{
-        return __mq_inode_ctx_new (inode, this);
-}
-
-quota_local_t *
-mq_local_new ()
-{
-        quota_local_t  *local   = NULL;
-
-        local = mem_get0 (THIS->local_pool);
-        if (!local)
-                goto out;
-
-        local->ref = 1;
-        LOCK_INIT (&local->lock);
-
-        local->ctx = NULL;
-        local->contri = NULL;
-
-out:
-        return local;
-}
-
-quota_local_t *
-mq_local_ref (quota_local_t *local)
-{
-        LOCK (&local->lock);
-        {
-                local->ref ++;
-        }
-        UNLOCK (&local->lock);
-
-        return local;
-}
-
-
-int32_t
-mq_local_unref (xlator_t *this, quota_local_t *local)
-{
-        int32_t ref = 0;
-        if (local == NULL)
-                goto out;
-
-        QUOTA_SAFE_DECREMENT (&local->lock, local->ref, ref);
-
-        if (ref != 0)
-                goto out;
-
-        if (local->fd != NULL)
-                fd_unref (local->fd);
-
-        if (local->contri)
-                GF_REF_PUT (local->contri);
-
-        if (local->xdata)
-                dict_unref (local->xdata);
-
-        loc_wipe (&local->loc);
-
-        loc_wipe (&local->parent_loc);
-
-        LOCK_DESTROY (&local->lock);
-
-        mem_put (local);
-out:
-        return 0;
-}
-
-
-inode_contribution_t *
-mq_get_contribution_from_loc (xlator_t *this, loc_t *loc)
+mq_inode_ctx_new(inode_t *inode, xlator_t *this)
 {
-        int32_t               ret          = 0;
-        quota_inode_ctx_t    *ctx          = NULL;
-        inode_contribution_t *contribution = NULL;
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "cannot get marker-quota context from inode "
-                                  "(gfid:%s, path:%s)",
-                                  uuid_utoa (loc->inode->gfid), loc->path);
-                goto err;
-        }
-
-        contribution = mq_get_contribution_node (loc->parent, ctx);
-        if (contribution == NULL) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "inode (gfid:%s, path:%s) has "
-                                  "no contribution towards parent (gfid:%s)",
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, uuid_utoa (loc->parent->gfid));
-                goto err;
-        }
-
-err:
-        return contribution;
+    return __mq_inode_ctx_new(inode, this);
 }
diff --git a/xlators/features/marker/src/marker-quota-helper.h b/xlators/features/marker/src/marker-quota-helper.h
index bf417aa8241..d4091dd2180 100644
--- a/xlators/features/marker/src/marker-quota-helper.h
+++ b/xlators/features/marker/src/marker-quota-helper.h
@@ -13,69 +13,54 @@
 
 #include "marker.h"
 
-#define QUOTA_FREE_CONTRIBUTION_NODE(ctx, _contribution)             \
-        do {                                                         \
-                LOCK (&ctx->lock);                                   \
-                {                                                    \
-                        list_del_init (&_contribution->contri_list); \
-                        GF_REF_PUT (_contribution);                  \
-                }                                                    \
-                UNLOCK (&ctx->lock);                                 \
-        } while (0)
-
-#define QUOTA_SAFE_INCREMENT(lock, var)                 \
-        do {                                            \
-                LOCK (lock);                            \
-                        var ++;                         \
-                UNLOCK (lock);                          \
-        } while (0)
-
-#define QUOTA_SAFE_DECREMENT(lock, var, value)  \
-        do {                                    \
-                LOCK (lock);                    \
-                {                               \
-                      value = --var;            \
-                }                               \
-                UNLOCK (lock);                  \
-        } while (0)
+#define QUOTA_FREE_CONTRIBUTION_NODE(ctx, _contribution)                       \
+    do {                                                                       \
+        LOCK(&ctx->lock);                                                      \
+        {                                                                      \
+            list_del_init(&_contribution->contri_list);                        \
+            GF_REF_PUT(_contribution);                                         \
+        }                                                                      \
+        UNLOCK(&ctx->lock);                                                    \
+    } while (0)
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var++;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
+
+#define QUOTA_SAFE_DECREMENT(lock, var, value)                                 \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        {                                                                      \
+            value = --var;                                                     \
+        }                                                                      \
+        UNLOCK(lock);                                                          \
+    } while (0)
 
 inode_contribution_t *
-mq_add_new_contribution_node (xlator_t *, quota_inode_ctx_t *, loc_t *);
+mq_add_new_contribution_node(xlator_t *, quota_inode_ctx_t *, loc_t *);
 
 int32_t
-mq_dict_set_contribution (xlator_t *, dict_t *, loc_t *, uuid_t, char *);
+mq_dict_set_contribution(xlator_t *, dict_t *, loc_t *, uuid_t, char *);
 
 quota_inode_ctx_t *
-mq_inode_ctx_new (inode_t *, xlator_t *);
-
-int32_t
-mq_inode_ctx_get (inode_t *, xlator_t *, quota_inode_ctx_t **);
+mq_inode_ctx_new(inode_t *, xlator_t *);
 
 int32_t
-mq_delete_contribution_node (dict_t *, char *, inode_contribution_t *);
+mq_inode_ctx_get(inode_t *, xlator_t *, quota_inode_ctx_t **);
 
 int32_t
-mq_inode_loc_fill (const char *, inode_t *, loc_t *);
-
-quota_local_t *
-mq_local_new ();
-
-quota_local_t *
-mq_local_ref (quota_local_t *);
+mq_delete_contribution_node(dict_t *, char *, inode_contribution_t *);
 
 int32_t
-mq_local_unref (xlator_t *, quota_local_t *);
-
-void
-mq_contri_fini (void *data);
-
-inode_contribution_t*
-mq_contri_init (inode_t *inode);
+mq_inode_loc_fill(const char *, inode_t *, loc_t *);
 
 inode_contribution_t *
-mq_get_contribution_node (inode_t *, quota_inode_ctx_t *);
+mq_contri_init(inode_t *inode);
 
 inode_contribution_t *
-mq_get_contribution_from_loc (xlator_t *this, loc_t *loc);
+mq_get_contribution_node(inode_t *, quota_inode_ctx_t *);
 
 #endif
diff --git a/xlators/features/marker/src/marker-quota.c b/xlators/features/marker/src/marker-quota.c
index 902b8e5c272..3de2ea1c92c 100644
--- a/xlators/features/marker/src/marker-quota.c
+++ b/xlators/features/marker/src/marker-quota.c
@@ -7,266 +7,263 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "libxlator.h"
-#include "common-utils.h"
-#include "byte-order.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/byte-order.h>
 #include "marker-quota.h"
 #include "marker-quota-helper.h"
-#include "syncop.h"
-#include "quota-common-utils.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/quota-common-utils.h>
 
 int
-mq_loc_copy (loc_t *dst, loc_t *src)
+mq_loc_copy(loc_t *dst, loc_t *src)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", dst, out);
-        GF_VALIDATE_OR_GOTO ("marker", src, out);
+    GF_VALIDATE_OR_GOTO("marker", dst, out);
+    GF_VALIDATE_OR_GOTO("marker", src, out);
 
-        if (src->inode == NULL ||
-            ((src->parent == NULL) && (gf_uuid_is_null (src->pargfid))
-             && !__is_root_gfid (src->inode->gfid))) {
-                gf_log ("marker", GF_LOG_WARNING,
-                        "src loc is not valid");
-                goto out;
-        }
+    if (src->inode == NULL ||
+        ((src->parent == NULL) && (gf_uuid_is_null(src->pargfid)) &&
+         !__is_root_gfid(src->inode->gfid))) {
+        gf_log("marker", GF_LOG_WARNING, "src loc is not valid");
+        goto out;
+    }
 
-        ret = loc_copy (dst, src);
+    ret = loc_copy(dst, src);
 out:
-        return ret;
+    return ret;
 }
 
 static void
-mq_set_ctx_status (quota_inode_ctx_t *ctx, gf_boolean_t *flag,
-                   gf_boolean_t status)
+mq_set_ctx_status(quota_inode_ctx_t *ctx, gf_boolean_t *flag,
+                  gf_boolean_t status)
 {
-        LOCK (&ctx->lock);
-        {
-                *flag = status;
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        *flag = status;
+    }
+    UNLOCK(&ctx->lock);
 }
 
 static void
-mq_test_and_set_ctx_status (quota_inode_ctx_t *ctx, gf_boolean_t *flag,
-                            gf_boolean_t *status)
+mq_test_and_set_ctx_status(quota_inode_ctx_t *ctx, gf_boolean_t *flag,
+                           gf_boolean_t *status)
 {
-        gf_boolean_t    temp    = _gf_false;
-
-        LOCK (&ctx->lock);
-        {
-                temp = *status;
-                *status = *flag;
-                *flag = temp;
-        }
-        UNLOCK (&ctx->lock);
+    gf_boolean_t temp = _gf_false;
+
+    LOCK(&ctx->lock);
+    {
+        temp = *status;
+        *status = *flag;
+        *flag = temp;
+    }
+    UNLOCK(&ctx->lock);
 }
 
 static void
-mq_get_ctx_status (quota_inode_ctx_t *ctx, gf_boolean_t *flag,
-                   gf_boolean_t *status)
+mq_get_ctx_status(quota_inode_ctx_t *ctx, gf_boolean_t *flag,
+                  gf_boolean_t *status)
 {
-        LOCK (&ctx->lock);
-        {
-                *status = *flag;
-        }
-        UNLOCK (&ctx->lock);
-}
-
-int32_t
-mq_get_ctx_updation_status (quota_inode_ctx_t *ctx,
-                            gf_boolean_t *status)
-{
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", status, out);
-
-        mq_get_ctx_status (ctx, &ctx->updation_status, status);
-        return 0;
-out:
-        return -1;
+    LOCK(&ctx->lock);
+    {
+        *status = *flag;
+    }
+    UNLOCK(&ctx->lock);
 }
 
 int32_t
-mq_set_ctx_updation_status (quota_inode_ctx_t *ctx,
-                            gf_boolean_t status)
+mq_get_ctx_updation_status(quota_inode_ctx_t *ctx, gf_boolean_t *status)
 {
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", status, out);
 
-        mq_set_ctx_status (ctx, &ctx->updation_status, status);
-        return 0;
+    mq_get_ctx_status(ctx, &ctx->updation_status, status);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-mq_test_and_set_ctx_updation_status (quota_inode_ctx_t *ctx,
-                                     gf_boolean_t *status)
+mq_set_ctx_updation_status(quota_inode_ctx_t *ctx, gf_boolean_t status)
 {
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", status, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        mq_test_and_set_ctx_status (ctx, &ctx->updation_status, status);
-        return 0;
+    mq_set_ctx_status(ctx, &ctx->updation_status, status);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-mq_set_ctx_create_status (quota_inode_ctx_t *ctx,
-                          gf_boolean_t status)
+mq_test_and_set_ctx_updation_status(quota_inode_ctx_t *ctx,
+                                    gf_boolean_t *status)
 {
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", status, out);
 
-        mq_set_ctx_status (ctx, &ctx->create_status, status);
-        return 0;
+    mq_test_and_set_ctx_status(ctx, &ctx->updation_status, status);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-mq_test_and_set_ctx_create_status (quota_inode_ctx_t *ctx,
-                                   gf_boolean_t *status)
+mq_set_ctx_create_status(quota_inode_ctx_t *ctx, gf_boolean_t status)
 {
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", status, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        mq_test_and_set_ctx_status (ctx, &ctx->create_status, status);
-        return 0;
+    mq_set_ctx_status(ctx, &ctx->create_status, status);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-mq_set_ctx_dirty_status (quota_inode_ctx_t *ctx,
-                         gf_boolean_t status)
+mq_test_and_set_ctx_create_status(quota_inode_ctx_t *ctx, gf_boolean_t *status)
 {
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", status, out);
 
-        mq_set_ctx_status (ctx, &ctx->dirty_status, status);
-        return 0;
+    mq_test_and_set_ctx_status(ctx, &ctx->create_status, status);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
-int32_t
-mq_test_and_set_ctx_dirty_status (quota_inode_ctx_t *ctx,
-                                  gf_boolean_t *status)
+static void
+mq_set_ctx_dirty_status(quota_inode_ctx_t *ctx, gf_boolean_t status)
 {
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", status, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        mq_test_and_set_ctx_status (ctx, &ctx->dirty_status, status);
-        return 0;
+    mq_set_ctx_status(ctx, &ctx->dirty_status, status);
 out:
-        return -1;
+    return;
 }
 
 int
-mq_build_ancestry (xlator_t *this, loc_t *loc)
+mq_build_ancestry(xlator_t *this, loc_t *loc)
 {
-        int32_t               ret            = -1;
-        fd_t                 *fd             = NULL;
-        gf_dirent_t           entries;
-        gf_dirent_t          *entry          = NULL;
-        dict_t               *xdata          = NULL;
-        inode_t              *tmp_parent     = NULL;
-        inode_t              *tmp_inode      = NULL;
-        inode_t              *linked_inode   = NULL;
-        quota_inode_ctx_t    *ctx            = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        xdata = dict_new ();
-        if (xdata == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = dict_set_int8 (xdata, GET_ANCESTRY_DENTRY_KEY, 1);
-        if (ret < 0)
-                goto out;
-
-        fd = fd_anonymous (loc->inode);
-        if (fd == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "fd creation failed");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        fd_bind (fd);
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    dict_t *xdata = NULL;
+    inode_t *tmp_parent = NULL;
+    inode_t *tmp_inode = NULL;
+    inode_t *linked_inode = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    xdata = dict_new();
+    if (xdata == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = syncop_readdirp (this, fd, 131072, 0, &entries, xdata, NULL);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "readdirp failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+    ret = dict_set_int8(xdata, GET_ANCESTRY_DENTRY_KEY, 1);
+    if (ret < 0)
+        goto out;
 
-        if (list_empty (&entries.list)) {
-                ret = -1;
-                goto out;
-        }
+    fd = fd_anonymous(loc->inode);
+    if (fd == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "fd creation failed");
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        list_for_each_entry (entry, &entries.list, list) {
-                if (__is_root_gfid (entry->inode->gfid)) {
-                        tmp_parent = NULL;
-                } else {
-                        linked_inode = inode_link (entry->inode, tmp_parent,
-                                                   entry->d_name,
-                                                   &entry->d_stat);
-                        if (linked_inode) {
-                                tmp_inode = entry->inode;
-                                entry->inode = linked_inode;
-                                inode_unref (tmp_inode);
-                        } else {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "inode link failed");
-                                ret = -EINVAL;
-                                goto out;
-                        }
-                }
-
-                ctx = mq_inode_ctx_new (entry->inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s",
-                                uuid_utoa (entry->inode->gfid));
-                        ret = -ENOMEM;
-                        goto out;
-                }
-
-                tmp_parent = entry->inode;
-        }
+    fd_bind(fd);
 
-        if (loc->parent)
-                inode_unref (loc->parent);
+    ret = syncop_readdirp(this, fd, 131072, 0, &entries, xdata, NULL);
+    if (ret < 0) {
+        gf_log(this->name,
+               (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+               "readdirp failed "
+               "for %s: %s",
+               loc->path, strerror(-ret));
+        goto out;
+    }
 
-        loc->parent = inode_parent (loc->inode, 0, NULL);
-        if (loc->parent == NULL) {
-                ret = -1;
-                goto out;
-        }
+    if (list_empty(&entries.list)) {
+        ret = -1;
+        goto out;
+    }
+
+    list_for_each_entry(entry, &entries.list, list)
+    {
+        if (__is_root_gfid(entry->inode->gfid)) {
+            /* The list contains a sub-list for each possible path
+             * to the target inode. Each sub-list starts with the
+             * root entry of the tree and is followed by the child
+             * entries for a particular path to the target entry.
+             * The root entry is an implied sub-list delimiter,
+             * as it denotes we have started processing a new path.
+             * Reset the parent pointer and continue
+             */
+
+            tmp_parent = NULL;
+        } else {
+            linked_inode = inode_link(entry->inode, tmp_parent, entry->d_name,
+                                      &entry->d_stat);
+            if (linked_inode) {
+                tmp_inode = entry->inode;
+                entry->inode = linked_inode;
+                inode_unref(tmp_inode);
+            } else {
+                gf_log(this->name, GF_LOG_ERROR, "inode link failed");
+                ret = -EINVAL;
+                goto out;
+            }
+        }
+
+        ctx = mq_inode_ctx_new(entry->inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(entry->inode->gfid));
+            ret = -ENOMEM;
+            goto out;
+        }
+
+        /* For non-directory, posix_get_ancestry_non_directory returns
+         * all hard-links that are represented by nodes adjacent to
+         * each other in the dentry-list.
+         * (Unlike the directory case where adjacent nodes either have
+         *  a parent/child relationship or belong to different paths).
+         */
+        if (entry->inode->ia_type == IA_IFDIR)
+            tmp_parent = entry->inode;
+    }
+
+    if (loc->parent)
+        inode_unref(loc->parent);
+
+    loc->parent = inode_parent(loc->inode, 0, NULL);
+    if (loc->parent == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        if (fd)
-                fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
-
 /* This function should be used only in inspect_directory and inspect_file
  * function to heal quota xattrs.
  * Inode quota feature is introduced in 3.7.
@@ -276,1914 +273,2025 @@ out:
  * This function returns success even is inode-quota xattrs are missing and
  * hence no healing performed.
  */
-int32_t
-_quota_dict_get_meta (xlator_t *this, dict_t *dict, char *key,
-                      quota_meta_t *meta, ia_type_t ia_type,
-                      gf_boolean_t add_delta)
+static int32_t
+_quota_dict_get_meta(xlator_t *this, dict_t *dict, char *key, const int keylen,
+                     quota_meta_t *meta, ia_type_t ia_type,
+                     gf_boolean_t add_delta)
 {
-        int32_t             ret     = 0;
-        marker_conf_t      *priv    = NULL;
-
-        priv = this->private;
-
-        ret = quota_dict_get_inode_meta (dict, key, meta);
-        if (ret == -2 && (priv->feature_enabled & GF_INODE_QUOTA) == 0) {
-                /* quota_dict_get_inode_meta returns -2 if
-                 * inode quota xattrs are not present.
-                 * if inode quota self heal is turned off,
-                 * then we should skip healing inode quotas
-                 */
-
-                gf_log (this->name, GF_LOG_DEBUG, "inode quota disabled. "
-                        "inode quota self heal will not be performed");
-                ret = 0;
-                if (add_delta) {
-                        if (ia_type == IA_IFDIR)
-                                meta->dir_count = 1;
-                        else
-                                meta->file_count = 1;
-                }
+    int32_t ret = 0;
+    marker_conf_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = quota_dict_get_inode_meta(dict, key, keylen, meta);
+    if (ret == -2 && (priv->feature_enabled & GF_INODE_QUOTA) == 0) {
+        /* quota_dict_get_inode_meta returns -2 if
+         * inode quota xattrs are not present.
+         * if inode quota self heal is turned off,
+         * then we should skip healing inode quotas
+         */
+
+        gf_log(this->name, GF_LOG_DEBUG,
+               "inode quota disabled. "
+               "inode quota self heal will not be performed");
+        ret = 0;
+        if (add_delta) {
+            if (ia_type == IA_IFDIR)
+                meta->dir_count = 1;
+            else
+                meta->file_count = 1;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-quota_dict_set_size_meta (xlator_t *this, dict_t *dict,
-                          const quota_meta_t *meta)
+quota_dict_set_size_meta(xlator_t *this, dict_t *dict, const quota_meta_t *meta)
 {
-        int32_t         ret                        = -ENOMEM;
-        quota_meta_t   *value                      = NULL;
-        char            size_key[QUOTA_KEY_MAX]    = {0, };
-
-        value = GF_CALLOC (2, sizeof (quota_meta_t), gf_common_quota_meta_t);
-        if (value == NULL) {
-                goto out;
-        }
-        value[0].size = hton64 (meta->size);
-        value[0].file_count = hton64 (meta->file_count);
-        value[0].dir_count = hton64 (meta->dir_count);
+    int32_t ret = -ENOMEM;
+    quota_meta_t *value = NULL;
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    value = GF_MALLOC(2 * sizeof(quota_meta_t), gf_common_quota_meta_t);
+    if (value == NULL) {
+        goto out;
+    }
+    value[0].size = hton64(meta->size);
+    value[0].file_count = hton64(meta->file_count);
+    value[0].dir_count = hton64(meta->dir_count);
 
-        value[1].size = 0;
-        value[1].file_count = 0;
-        value[1].dir_count = hton64 (1);
+    value[1].size = 0;
+    value[1].file_count = 0;
+    value[1].dir_count = hton64(1);
 
-        GET_SIZE_KEY (this, size_key, ret);
-        if (ret < 0)
-                goto out;
-        ret = dict_set_bin (dict, size_key, value,
-                            (sizeof (quota_meta_t) * 2));
-        if (ret < 0) {
-                gf_log_callingfn ("quota", GF_LOG_ERROR, "dict set failed");
-                GF_FREE (value);
-        }
+    GET_SIZE_KEY(this, size_key, ret);
+    if (ret < 0)
+        goto out;
+    ret = dict_set_bin(dict, size_key, value, (sizeof(quota_meta_t) * 2));
+    if (ret < 0) {
+        gf_log_callingfn("quota", GF_LOG_ERROR, "dict set failed");
+        GF_FREE(value);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 void
-mq_compute_delta (quota_meta_t *delta, const quota_meta_t *op1,
-                  const quota_meta_t *op2)
+mq_compute_delta(quota_meta_t *delta, const quota_meta_t *op1,
+                 const quota_meta_t *op2)
 {
-        delta->size       = op1->size - op2->size;
-        delta->file_count = op1->file_count - op2->file_count;
-        delta->dir_count  = op1->dir_count - op2->dir_count;
+    delta->size = op1->size - op2->size;
+    delta->file_count = op1->file_count - op2->file_count;
+    delta->dir_count = op1->dir_count - op2->dir_count;
 }
 
 void
-mq_add_meta (quota_meta_t *dst, const quota_meta_t *src)
+mq_add_meta(quota_meta_t *dst, const quota_meta_t *src)
 {
-        dst->size       += src->size;
-        dst->file_count += src->file_count;
-        dst->dir_count  += src->dir_count;
+    dst->size += src->size;
+    dst->file_count += src->file_count;
+    dst->dir_count += src->dir_count;
 }
 
 void
-mq_sub_meta (quota_meta_t *dst, const quota_meta_t *src)
+mq_sub_meta(quota_meta_t *dst, const quota_meta_t *src)
 {
-        if (src == NULL) {
-                dst->size       = -dst->size;
-                dst->file_count = -dst->file_count;
-                dst->dir_count  = -dst->dir_count;
-        } else {
-                dst->size       = src->size - dst->size;
-                dst->file_count = src->file_count - dst->file_count;
-                dst->dir_count  = src->dir_count - dst->dir_count;
-        }
+    if (src == NULL) {
+        dst->size = -dst->size;
+        dst->file_count = -dst->file_count;
+        dst->dir_count = -dst->dir_count;
+    } else {
+        dst->size = src->size - dst->size;
+        dst->file_count = src->file_count - dst->file_count;
+        dst->dir_count = src->dir_count - dst->dir_count;
+    }
 }
 
 int32_t
-mq_are_xattrs_set (xlator_t *this, loc_t *loc, gf_boolean_t *contri_set,
-                   gf_boolean_t *size_set)
+mq_are_xattrs_set(xlator_t *this, loc_t *loc, gf_boolean_t *contri_set,
+                  gf_boolean_t *size_set)
 {
-        int32_t        ret                         = -1;
-        char           contri_key[QUOTA_KEY_MAX]   = {0, };
-        char           size_key[QUOTA_KEY_MAX]     = {0, };
-        quota_meta_t   meta                        = {0, };
-        struct iatt    stbuf                       = {0,};
-        dict_t        *dict                        = NULL;
-        dict_t        *rsp_dict                    = NULL;
-
-        dict = dict_new ();
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
-
-        ret = mq_req_xattr (this, loc, dict, contri_key, size_key);
-        if (ret < 0)
-                goto out;
+    int32_t ret = -1;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    quota_meta_t meta = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    dict = dict_new();
+    if (dict == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        ret = syncop_lookup (FIRST_CHILD(this), loc, &stbuf, NULL,
-                             dict, &rsp_dict);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                  ? GF_LOG_DEBUG:GF_LOG_ERROR, "lookup failed "
-                                  "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+    ret = mq_req_xattr(this, loc, dict, contri_key, size_key);
+    if (ret < 0)
+        goto out;
 
-        if (rsp_dict == NULL)
-                goto out;
+    ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL, dict, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "lookup failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
 
-        *contri_set = _gf_true;
-        *size_set = _gf_true;
-        if (loc->inode->ia_type == IA_IFDIR) {
-                ret = quota_dict_get_inode_meta (rsp_dict, size_key, &meta);
-                if (ret < 0 || meta.dir_count == 0)
-                        *size_set = _gf_false;
-        }
+    if (rsp_dict == NULL)
+        goto out;
 
-        if (!loc_is_root(loc)) {
-                ret = quota_dict_get_inode_meta (rsp_dict, contri_key, &meta);
-                if (ret < 0)
-                        *contri_set = _gf_false;
-        }
+    *contri_set = _gf_true;
+    *size_set = _gf_true;
+    if (loc->inode->ia_type == IA_IFDIR) {
+        ret = quota_dict_get_inode_meta(rsp_dict, size_key, strlen(size_key),
+                                        &meta);
+        if (ret < 0 || meta.dir_count == 0)
+            *size_set = _gf_false;
+    }
+
+    if (!loc_is_root(loc)) {
+        ret = quota_dict_get_inode_meta(rsp_dict, contri_key,
+                                        strlen(contri_key), &meta);
+        if (ret < 0)
+            *contri_set = _gf_false;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_create_size_xattrs (xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
+mq_create_size_xattrs(xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
 {
-        int32_t                ret                  = -1;
-        quota_meta_t           size                 = {0, };
-        dict_t                *dict                 = NULL;
+    int32_t ret = -1;
+    quota_meta_t size = {
+        0,
+    };
+    dict_t *dict = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        if (loc->inode->ia_type != IA_IFDIR) {
-                ret = 0;
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -1;
-                goto out;
-        }
+    if (loc->inode->ia_type != IA_IFDIR) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = quota_dict_set_size_meta (this, dict, &size);
-        if (ret < 0)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
 
-        ret = syncop_xattrop (FIRST_CHILD(this), loc,
-                              GF_XATTROP_ADD_ARRAY64_WITH_DEFAULT, dict, NULL,
-                              NULL);
+    ret = quota_dict_set_size_meta(this, dict, &size);
+    if (ret < 0)
+        goto out;
 
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                  ? GF_LOG_DEBUG:GF_LOG_ERROR, "xattrop failed "
-                                  "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+    ret = syncop_xattrop(FIRST_CHILD(this), loc,
+                         GF_XATTROP_ADD_ARRAY64_WITH_DEFAULT, dict, NULL, NULL,
+                         NULL);
+
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_lock (xlator_t *this, loc_t *loc, short l_type)
+mq_lock(xlator_t *this, loc_t *loc, short l_type)
 {
-        struct gf_flock  lock  = {0, };
-        int32_t          ret   = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        gf_log (this->name, GF_LOG_DEBUG, "set lock type %d on %s",
-                l_type, loc->path);
-
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = l_type;
-        lock.l_whence = SEEK_SET;
-
-        ret = syncop_inodelk (FIRST_CHILD(this), this->name, loc, F_SETLKW,
-                              &lock, NULL, NULL);
-        if (ret < 0)
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                  ? GF_LOG_DEBUG:GF_LOG_ERROR, "inodelk failed "
-                                  "for %s: %s", loc->path, strerror (-ret));
+    struct gf_flock lock = {
+        0,
+    };
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+
+    gf_log(this->name, GF_LOG_DEBUG, "set lock type %d on %s", l_type,
+           loc->path);
+
+    lock.l_len = 0;
+    lock.l_start = 0;
+    lock.l_type = l_type;
+    lock.l_whence = SEEK_SET;
+
+    ret = syncop_inodelk(FIRST_CHILD(this), this->name, loc, F_SETLKW, &lock,
+                         NULL, NULL);
+    if (ret < 0)
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "inodelk failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
 
 out:
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_get_dirty (xlator_t *this, loc_t *loc, int32_t *dirty)
+mq_get_dirty(xlator_t *this, loc_t *loc, int32_t *dirty)
 {
-        int32_t        ret              = -1;
-        int8_t         value            = 0;
-        dict_t        *dict             = NULL;
-        dict_t        *rsp_dict         = NULL;
-        struct iatt    stbuf            = {0,};
-
-        dict = dict_new ();
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
-
-        ret = dict_set_int64 (dict, QUOTA_DIRTY_KEY, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
+    int32_t ret = -1;
+    int8_t value = 0;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    dict = dict_new();
+    if (dict == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        ret = syncop_lookup (FIRST_CHILD(this), loc, &stbuf, NULL,
-                             dict, &rsp_dict);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                  ? GF_LOG_DEBUG:GF_LOG_ERROR, "lookup failed "
-                                  "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+    ret = dict_set_int64(dict, QUOTA_DIRTY_KEY, 0);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING, "dict set failed");
+        goto out;
+    }
+
+    ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL, dict, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "lookup failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
 
-        ret = dict_get_int8 (rsp_dict, QUOTA_DIRTY_KEY, &value);
-        if (ret < 0)
-                goto out;
+    ret = dict_get_int8(rsp_dict, QUOTA_DIRTY_KEY, &value);
+    if (ret < 0)
+        goto out;
 
-        *dirty = value;
+    *dirty = value;
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_get_set_dirty (xlator_t *this, loc_t *loc, int32_t dirty,
-                  int32_t *prev_dirty)
+mq_get_set_dirty(xlator_t *this, loc_t *loc, int32_t dirty, int32_t *prev_dirty)
 {
-        int32_t              ret              = -1;
-        int8_t               value            = 0;
-        quota_inode_ctx_t   *ctx              = NULL;
-        dict_t              *dict             = NULL;
-        dict_t              *rsp_dict         = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", prev_dirty, out);
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode ctx for "
-                        "%s", loc->path);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, dirty);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_set failed");
-                goto out;
-        }
-
-        ret = syncop_xattrop (FIRST_CHILD(this), loc, GF_XATTROP_GET_AND_SET,
-                              dict, NULL, &rsp_dict);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                          ? GF_LOG_DEBUG:GF_LOG_ERROR, "xattrop failed "
-                          "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+    int32_t ret = -1;
+    int8_t value = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", prev_dirty, out);
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to get inode ctx for "
+               "%s",
+               loc->path);
+        goto out;
+    }
 
-        *prev_dirty = 0;
-        if (rsp_dict) {
-                ret = dict_get_int8 (rsp_dict, QUOTA_DIRTY_KEY, &value);
-                if (ret == 0)
-                        *prev_dirty = value;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->dirty = dirty;
-        }
-        UNLOCK (&ctx->lock);
-        ret = 0;
+    ret = dict_set_int8(dict, QUOTA_DIRTY_KEY, dirty);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_set failed");
+        goto out;
+    }
+
+    ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_GET_AND_SET, dict,
+                         NULL, NULL, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
+
+    *prev_dirty = 0;
+    if (rsp_dict) {
+        ret = dict_get_int8(rsp_dict, QUOTA_DIRTY_KEY, &value);
+        if (ret == 0)
+            *prev_dirty = value;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->dirty = dirty;
+    }
+    UNLOCK(&ctx->lock);
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_mark_dirty (xlator_t *this, loc_t *loc, int32_t dirty)
+mq_mark_dirty(xlator_t *this, loc_t *loc, int32_t dirty)
 {
-        int32_t            ret      = -1;
-        dict_t            *dict     = NULL;
-        quota_inode_ctx_t *ctx      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode ctx for "
-                        "%s", loc->path);
-                ret = 0;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to get inode ctx for "
+               "%s",
+               loc->path);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, dirty);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_set failed");
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        ret = syncop_setxattr (FIRST_CHILD(this), loc, dict, 0, NULL, NULL);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "setxattr dirty = %d "
-                        "failed for %s: %s", dirty, loc->path, strerror (-ret));
-                goto out;
-        }
+    ret = dict_set_int8(dict, QUOTA_DIRTY_KEY, dirty);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_set failed");
+        goto out;
+    }
+
+    ret = syncop_setxattr(FIRST_CHILD(this), loc, dict, 0, NULL, NULL);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "setxattr dirty = %d "
+            "failed for %s: %s",
+            dirty, loc->path, strerror(-ret));
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->dirty = dirty;
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        ctx->dirty = dirty;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-_mq_get_metadata (xlator_t *this, loc_t *loc, quota_meta_t *contri,
-                  quota_meta_t *size, uuid_t contri_gfid)
+_mq_get_metadata(xlator_t *this, loc_t *loc, quota_meta_t *contri,
+                 quota_meta_t *size, uuid_t contri_gfid)
 {
-        int32_t            ret                         = -1;
-        quota_meta_t       meta                        = {0, };
-        char               contri_key[QUOTA_KEY_MAX]   = {0, };
-        char               size_key[QUOTA_KEY_MAX]     = {0, };
-        dict_t            *dict                        = NULL;
-        dict_t            *rsp_dict                    = NULL;
-        struct iatt        stbuf                       = {0,};
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        if (size == NULL && contri == NULL)
-                goto out;
+    int32_t ret = -1;
+    quota_meta_t meta = {
+        0,
+    };
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+
+    if (size == NULL && contri == NULL)
+        goto out;
 
-        dict = dict_new ();
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
+    dict = dict_new();
+    if (dict == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        if (size && loc->inode->ia_type == IA_IFDIR) {
-                GET_SIZE_KEY (this, size_key, ret);
-                if (ret < 0)
-                        goto out;
-                ret = dict_set_int64 (dict, size_key, 0);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "dict_set failed.");
-                        goto out;
-                }
+    if (size && loc->inode->ia_type == IA_IFDIR) {
+        GET_SIZE_KEY(this, size_key, keylen);
+        if (keylen < 0)
+            goto out;
+        ret = dict_set_int64(dict, size_key, 0);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_ERROR, "dict_set failed.");
+            goto out;
         }
+    }
 
-        if (contri && !loc_is_root(loc)) {
-                ret = mq_dict_set_contribution (this, dict, loc, contri_gfid,
-                                                contri_key);
-                if (ret < 0)
-                        goto out;
-        }
+    if (contri && !loc_is_root(loc)) {
+        ret = mq_dict_set_contribution(this, dict, loc, contri_gfid,
+                                       contri_key);
+        if (ret < 0)
+            goto out;
+    }
+
+    ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL, dict, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "lookup failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
 
-        ret = syncop_lookup (FIRST_CHILD(this), loc, &stbuf, NULL,
-                             dict, &rsp_dict);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                  ? GF_LOG_DEBUG:GF_LOG_ERROR, "lookup failed "
-                                  "for %s: %s", loc->path, strerror (-ret));
+    if (size) {
+        if (loc->inode->ia_type == IA_IFDIR) {
+            ret = quota_dict_get_meta(rsp_dict, size_key, keylen, &meta);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_ERROR, "dict_get failed.");
                 goto out;
-        }
+            }
 
-        if (size) {
-                if (loc->inode->ia_type == IA_IFDIR) {
-                        ret = quota_dict_get_meta (rsp_dict, size_key,
-                                                   &meta);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "dict_get failed.");
-                                goto out;
-                        }
-
-                        size->size = meta.size;
-                        size->file_count = meta.file_count;
-                        size->dir_count = meta.dir_count;
-                } else {
-                        size->size = stbuf.ia_blocks * 512;
-                        size->file_count = 1;
-                        size->dir_count = 0;
-                }
+            size->size = meta.size;
+            size->file_count = meta.file_count;
+            size->dir_count = meta.dir_count;
+        } else {
+            size->size = stbuf.ia_blocks * 512;
+            size->file_count = 1;
+            size->dir_count = 0;
         }
+    }
 
-        if (contri && !loc_is_root(loc)) {
-                ret = quota_dict_get_meta (rsp_dict, contri_key, &meta);
-                if (ret < 0) {
-                        contri->size = 0;
-                        contri->file_count = 0;
-                        contri->dir_count = 0;
-                } else {
-                        contri->size = meta.size;
-                        contri->file_count = meta.file_count;
-                        contri->dir_count = meta.dir_count;
-                }
+    if (contri && !loc_is_root(loc)) {
+        ret = quota_dict_get_meta(rsp_dict, contri_key, strlen(contri_key),
+                                  &meta);
+        if (ret < 0) {
+            contri->size = 0;
+            contri->file_count = 0;
+            contri->dir_count = 0;
+        } else {
+            contri->size = meta.size;
+            contri->file_count = meta.file_count;
+            contri->dir_count = meta.dir_count;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_get_metadata (xlator_t *this, loc_t *loc, quota_meta_t *contri,
-                 quota_meta_t *size, quota_inode_ctx_t *ctx,
-                 inode_contribution_t *contribution)
+mq_get_metadata(xlator_t *this, loc_t *loc, quota_meta_t *contri,
+                quota_meta_t *size, quota_inode_ctx_t *ctx,
+                inode_contribution_t *contribution)
 {
-        int32_t         ret      = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", contribution, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", contribution, out);
 
-        if (size == NULL && contri == NULL) {
-                ret = 0;
-                goto out;
-        }
+    if (size == NULL && contri == NULL) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = _mq_get_metadata (this, loc, contri, size, contribution->gfid);
-        if (ret < 0)
-                goto out;
+    ret = _mq_get_metadata(this, loc, contri, size, contribution->gfid);
+    if (ret < 0)
+        goto out;
 
-        if (size) {
-                LOCK (&ctx->lock);
-                {
-                        ctx->size = size->size;
-                        ctx->file_count = size->file_count;
-                        ctx->dir_count = size->dir_count;
-                }
-                UNLOCK  (&ctx->lock);
+    if (size) {
+        LOCK(&ctx->lock);
+        {
+            ctx->size = size->size;
+            ctx->file_count = size->file_count;
+            ctx->dir_count = size->dir_count;
         }
+        UNLOCK(&ctx->lock);
+    }
 
-        if (contri) {
-                LOCK (&contribution->lock);
-                {
-                        contribution->contribution = contri->size;
-                        contribution->file_count = contri->file_count;
-                        contribution->dir_count = contri->dir_count;
-                }
-                UNLOCK (&contribution->lock);
+    if (contri) {
+        LOCK(&contribution->lock);
+        {
+            contribution->contribution = contri->size;
+            contribution->file_count = contri->file_count;
+            contribution->dir_count = contri->dir_count;
         }
+        UNLOCK(&contribution->lock);
+    }
 
 out:
-        return ret;
-}
-
-int32_t
-mq_get_size (xlator_t *this, loc_t *loc, quota_meta_t *size)
-{
-        return _mq_get_metadata (this, loc, NULL, size, 0);
+    return ret;
 }
 
 int32_t
-mq_get_contri (xlator_t *this, loc_t *loc, quota_meta_t *contri,
-               uuid_t contri_gfid)
+mq_get_delta(xlator_t *this, loc_t *loc, quota_meta_t *delta,
+             quota_inode_ctx_t *ctx, inode_contribution_t *contribution)
 {
-        return _mq_get_metadata (this, loc, contri, NULL, contri_gfid);
-}
-
-int32_t
-mq_get_delta (xlator_t *this, loc_t *loc, quota_meta_t *delta,
-              quota_inode_ctx_t *ctx, inode_contribution_t *contribution)
-{
-        int32_t         ret      = -1;
-        quota_meta_t    size     = {0, };
-        quota_meta_t    contri   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", contribution, out);
-
-        ret = mq_get_metadata (this, loc, &contri, &size, ctx, contribution);
-        if (ret < 0)
-                goto out;
+    int32_t ret = -1;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", contribution, out);
+
+    ret = mq_get_metadata(this, loc, &contri, &size, ctx, contribution);
+    if (ret < 0)
+        goto out;
 
-        mq_compute_delta (delta, &size, &contri);
+    mq_compute_delta(delta, &size, &contri);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_remove_contri (xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx,
-                  inode_contribution_t *contri, quota_meta_t *delta,
-                  uint32_t nlink)
+mq_remove_contri(xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx,
+                 inode_contribution_t *contri, quota_meta_t *delta,
+                 uint32_t nlink)
 {
-        int32_t              ret                         = -1;
-        char                 contri_key[QUOTA_KEY_MAX]   = {0, };
+    int32_t ret = -1;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
 
-        if (nlink == 1) {
-                /*File was a last link and has been deleted */
-                ret = 0;
-                goto done;
-        }
-
-        GET_CONTRI_KEY (this, contri_key, contri->gfid, ret);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "get contri_key "
-                        "failed for %s", uuid_utoa(contri->gfid));
-                goto out;
-        }
-
-        ret = syncop_removexattr (FIRST_CHILD(this), loc, contri_key, 0, NULL);
-        if (ret < 0) {
-                if (-ret == ENOENT || -ret == ESTALE || -ret == ENODATA ||
-                    -ret == ENOATTR) {
-                        /* Remove contri in done when unlink operation is
-                         * performed, so return success on ENOENT/ESTSLE
-                         * rename operation removes xattr earlier,
-                         * so return success on ENODATA
-                         */
-                        ret = 0;
-                } else {
-                        gf_log_callingfn (this->name, GF_LOG_ERROR,
-                                          "removexattr %s failed for %s: %s",
-                                          contri_key, loc->path,
-                                          strerror (-ret));
-                        goto out;
-                }
+    if (nlink == 1) {
+        /*File was a last link and has been deleted */
+        ret = 0;
+        goto done;
+    }
+
+    GET_CONTRI_KEY(this, contri_key, contri->gfid, ret);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "get contri_key "
+               "failed for %s",
+               uuid_utoa(contri->gfid));
+        goto out;
+    }
+
+    ret = syncop_removexattr(FIRST_CHILD(this), loc, contri_key, 0, NULL);
+    if (ret < 0) {
+        if (-ret == ENOENT || -ret == ESTALE || -ret == ENODATA ||
+            -ret == ENOATTR) {
+            /* Remove contri in done when unlink operation is
+             * performed, so return success on ENOENT/ESTSLE
+             * rename operation removes xattr earlier,
+             * so return success on ENODATA
+             */
+            ret = 0;
+        } else {
+            gf_log_callingfn(this->name, GF_LOG_ERROR,
+                             "removexattr %s failed for %s: %s", contri_key,
+                             loc->path, strerror(-ret));
+            goto out;
         }
+    }
 
 done:
-        LOCK (&contri->lock);
-        {
-                contri->contribution += delta->size;
-                contri->file_count += delta->file_count;
-                contri->dir_count += delta->dir_count;
-        }
-        UNLOCK (&contri->lock);
+    LOCK(&contri->lock);
+    {
+        contri->contribution += delta->size;
+        contri->file_count += delta->file_count;
+        contri->dir_count += delta->dir_count;
+    }
+    UNLOCK(&contri->lock);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        QUOTA_FREE_CONTRIBUTION_NODE (ctx, contri);
+    QUOTA_FREE_CONTRIBUTION_NODE(ctx, contri);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_update_contri (xlator_t *this, loc_t *loc, inode_contribution_t *contri,
-                  quota_meta_t *delta)
+mq_update_contri(xlator_t *this, loc_t *loc, inode_contribution_t *contri,
+                 quota_meta_t *delta)
 {
-        int32_t              ret                         = -1;
-        char                 contri_key[QUOTA_KEY_MAX]   = {0, };
-        dict_t              *dict                        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", delta, out);
-        GF_VALIDATE_OR_GOTO ("marker", contri, out);
-
-        if (quota_meta_is_null (delta)) {
-                ret = 0;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", delta, out);
+    GF_VALIDATE_OR_GOTO("marker", contri, out);
+
+    if (quota_meta_is_null(delta)) {
+        ret = 0;
+        goto out;
+    }
 
-        GET_CONTRI_KEY (this, contri_key, contri->gfid, ret);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "get contri_key "
-                        "failed for %s", uuid_utoa(contri->gfid));
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    GET_CONTRI_KEY(this, contri_key, contri->gfid, ret);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "get contri_key "
+               "failed for %s",
+               uuid_utoa(contri->gfid));
+        goto out;
+    }
 
-        ret = quota_dict_set_meta (dict, contri_key, delta,
-                                   loc->inode->ia_type);
-        if (ret < 0)
-                goto out;
+    ret = quota_dict_set_meta(dict, contri_key, delta, loc->inode->ia_type);
+    if (ret < 0)
+        goto out;
 
-        ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_ADD_ARRAY64,
-                             dict, NULL, NULL);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                  ? GF_LOG_DEBUG:GF_LOG_ERROR, "xattrop failed "
-                                  "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+    ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_ADD_ARRAY64, dict,
+                         NULL, NULL, NULL);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
 
-        LOCK (&contri->lock);
-        {
-                contri->contribution += delta->size;
-                contri->file_count += delta->file_count;
-                contri->dir_count += delta->dir_count;
-        }
-        UNLOCK (&contri->lock);
+    LOCK(&contri->lock);
+    {
+        contri->contribution += delta->size;
+        contri->file_count += delta->file_count;
+        contri->dir_count += delta->dir_count;
+    }
+    UNLOCK(&contri->lock);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_update_size (xlator_t *this, loc_t *loc, quota_meta_t *delta)
+mq_update_size(xlator_t *this, loc_t *loc, quota_meta_t *delta)
 {
-        int32_t              ret              = -1;
-        quota_inode_ctx_t   *ctx              = NULL;
-        dict_t              *dict             = NULL;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", delta, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", delta, out);
 
-        if (quota_meta_is_null (delta)) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode ctx for "
-                        "%s", loc->path);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -1;
-                goto out;
-        }
+    if (quota_meta_is_null(delta)) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to get inode ctx for "
+               "%s",
+               loc->path);
+        goto out;
+    }
 
-        ret = quota_dict_set_size_meta (this, dict, delta);
-        if (ret < 0)
-                goto out;
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
 
-        ret = syncop_xattrop(FIRST_CHILD(this), loc,
-                             GF_XATTROP_ADD_ARRAY64_WITH_DEFAULT, dict, NULL,
-                             NULL);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                  ? GF_LOG_DEBUG:GF_LOG_ERROR, "xattrop failed "
-                                  "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+    ret = quota_dict_set_size_meta(this, dict, delta);
+    if (ret < 0)
+        goto out;
 
-        LOCK (&ctx->lock);
-        {
-                ctx->size += delta->size;
-                ctx->file_count += delta->file_count;
-                if (ctx->dir_count == 0)
-                        ctx->dir_count += delta->dir_count + 1;
-                else
-                        ctx->dir_count += delta->dir_count;
-        }
-        UNLOCK (&ctx->lock);
+    ret = syncop_xattrop(FIRST_CHILD(this), loc,
+                         GF_XATTROP_ADD_ARRAY64_WITH_DEFAULT, dict, NULL, NULL,
+                         NULL);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->size += delta->size;
+        ctx->file_count += delta->file_count;
+        if (ctx->dir_count == 0)
+            ctx->dir_count += delta->dir_count + 1;
+        else
+            ctx->dir_count += delta->dir_count;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int
-mq_synctask_cleanup (int ret, call_frame_t *frame, void *opaque)
+mq_synctask_cleanup(int ret, call_frame_t *frame, void *opaque)
 {
-        quota_synctask_t       *args         = NULL;
+    quota_synctask_t *args = NULL;
 
-        GF_ASSERT (opaque);
+    GF_ASSERT(opaque);
 
-        args = (quota_synctask_t *) opaque;
-        loc_wipe (&args->loc);
+    args = (quota_synctask_t *)opaque;
+    loc_wipe(&args->loc);
 
-        if (args->stub)
-                call_resume (args->stub);
+    if (args->stub)
+        call_resume(args->stub);
 
-        if (!args->is_static)
-                GF_FREE (args);
+    if (!args->is_static)
+        GF_FREE(args);
 
-        return 0;
+    return 0;
 }
 
 int
-mq_synctask1 (xlator_t *this, synctask_fn_t task, gf_boolean_t spawn,
-              loc_t *loc, quota_meta_t *contri, uint32_t nlink,
-              call_stub_t *stub)
+mq_synctask1(xlator_t *this, synctask_fn_t task, gf_boolean_t spawn, loc_t *loc,
+             quota_meta_t *contri, uint32_t nlink, call_stub_t *stub)
 {
-        int32_t              ret         = -1;
-        quota_synctask_t    *args        = NULL;
-        quota_synctask_t     static_args = {0, };
-
-        if (spawn) {
-                QUOTA_ALLOC_OR_GOTO (args, quota_synctask_t, ret, out);
-                args->is_static = _gf_false;
-        } else {
-                args = &static_args;
-                args->is_static = _gf_true;
-        }
-
-        args->this = this;
-        args->stub = stub;
-        loc_copy (&args->loc, loc);
-        args->ia_nlink = nlink;
-
-        if (contri) {
-                args->contri = *contri;
-        } else {
-                args->contri.size = -1;
-                args->contri.file_count = -1;
-                args->contri.dir_count = -1;
-        }
-
-        if (spawn) {
-                ret = synctask_new1 (this->ctx->env, 1024 * 16, task,
-                                      mq_synctask_cleanup, NULL, args);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to spawn "
-                                "new synctask");
-                        mq_synctask_cleanup (ret, NULL, args);
-                }
-        } else {
-                ret = task (args);
-                mq_synctask_cleanup (ret, NULL, args);
-        }
+    int32_t ret = -1;
+    quota_synctask_t *args = NULL;
+    quota_synctask_t static_args = {
+        0,
+    };
+
+    if (spawn) {
+        QUOTA_ALLOC_OR_GOTO(args, quota_synctask_t, ret, out);
+        args->is_static = _gf_false;
+    } else {
+        args = &static_args;
+        args->is_static = _gf_true;
+    }
+
+    args->this = this;
+    args->stub = stub;
+    loc_copy(&args->loc, loc);
+    args->ia_nlink = nlink;
+
+    if (contri) {
+        args->contri = *contri;
+    } else {
+        args->contri.size = -1;
+        args->contri.file_count = -1;
+        args->contri.dir_count = -1;
+    }
+
+    if (spawn) {
+        ret = synctask_new1(this->ctx->env, 1024 * 16, task,
+                            mq_synctask_cleanup, NULL, args);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to spawn "
+                   "new synctask");
+            mq_synctask_cleanup(ret, NULL, args);
+        }
+    } else {
+        ret = task(args);
+        mq_synctask_cleanup(ret, NULL, args);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-mq_synctask (xlator_t *this, synctask_fn_t task, gf_boolean_t spawn, loc_t *loc)
+mq_synctask(xlator_t *this, synctask_fn_t task, gf_boolean_t spawn, loc_t *loc)
 {
-        return mq_synctask1 (this, task, spawn, loc, NULL, -1, NULL);
+    return mq_synctask1(this, task, spawn, loc, NULL, -1, NULL);
 }
 
 int32_t
-mq_prevalidate_txn (xlator_t *this, loc_t *origin_loc, loc_t *loc,
-                    quota_inode_ctx_t **ctx, struct iatt *buf)
+mq_prevalidate_txn(xlator_t *this, loc_t *origin_loc, loc_t *loc,
+                   quota_inode_ctx_t **ctx, struct iatt *buf)
 {
-        int32_t               ret     = -1;
-        quota_inode_ctx_t    *ctxtmp  = NULL;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctxtmp = NULL;
 
-        if (buf) {
-                if (buf->ia_type == IA_IFREG && IS_DHT_LINKFILE_MODE(buf))
-                        goto out;
+    if (buf) {
+        if (buf->ia_type == IA_IFREG && IS_DHT_LINKFILE_MODE(buf))
+            goto out;
 
-                if (buf->ia_type != IA_IFREG && buf->ia_type != IA_IFLNK &&
-                    buf->ia_type != IA_IFDIR)
-                        goto out;
-        }
+        if (buf->ia_type != IA_IFREG && buf->ia_type != IA_IFLNK &&
+            buf->ia_type != IA_IFDIR)
+            goto out;
+    }
 
-        if (origin_loc == NULL || origin_loc->inode == NULL ||
-            gf_uuid_is_null(origin_loc->inode->gfid))
-                goto out;
+    if (origin_loc == NULL || origin_loc->inode == NULL ||
+        gf_uuid_is_null(origin_loc->inode->gfid))
+        goto out;
 
-        loc_copy (loc, origin_loc);
+    loc_copy(loc, origin_loc);
 
-        if (gf_uuid_is_null (loc->gfid))
-                gf_uuid_copy (loc->gfid, loc->inode->gfid);
+    if (gf_uuid_is_null(loc->gfid))
+        gf_uuid_copy(loc->gfid, loc->inode->gfid);
 
-        if (!loc_is_root(loc) && loc->parent == NULL)
-                loc->parent = inode_parent (loc->inode, 0, NULL);
+    if (!loc_is_root(loc) && loc->parent == NULL)
+        loc->parent = inode_parent(loc->inode, 0, NULL);
 
-        ret = mq_inode_ctx_get (loc->inode, this, &ctxtmp);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING, "inode ctx for "
-                                  "is NULL for %s", loc->path);
-                goto out;
-        }
-        if (ctx)
-                *ctx = ctxtmp;
+    ret = mq_inode_ctx_get(loc->inode, this, &ctxtmp);
+    if (ret < 0) {
+        gf_log_callingfn(this->name, GF_LOG_WARNING,
+                         "inode ctx for "
+                         "is NULL for %s",
+                         loc->path);
+        goto out;
+    }
+    if (ctx)
+        *ctx = ctxtmp;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-mq_create_xattrs_task (void *opaque)
+mq_create_xattrs_task(void *opaque)
 {
-        int32_t                  ret        = -1;
-        gf_boolean_t             locked     = _gf_false;
-        gf_boolean_t             contri_set = _gf_false;
-        gf_boolean_t             size_set   = _gf_false;
-        gf_boolean_t             need_txn   = _gf_false;
-        quota_synctask_t        *args       = NULL;
-        quota_inode_ctx_t       *ctx        = NULL;
-        xlator_t                *this       = NULL;
-        loc_t                   *loc        = NULL;
-        gf_boolean_t             status     = _gf_false;
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        this = args->this;
-        THIS = this;
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to"
-                        "get inode ctx, aborting quota create txn");
-                goto out;
-        }
+    int32_t ret = -1;
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t contri_set = _gf_false;
+    gf_boolean_t size_set = _gf_false;
+    gf_boolean_t need_txn = _gf_false;
+    quota_synctask_t *args = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t status = _gf_false;
+
+    GF_ASSERT(opaque);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    this = args->this;
+    THIS = this;
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Failed to"
+               "get inode ctx, aborting quota create txn");
+        goto out;
+    }
 
-        if (loc->inode->ia_type == IA_IFDIR) {
-                /* lock not required for files */
-                ret = mq_lock (this, loc, F_WRLCK);
-                if (ret < 0)
-                        goto out;
-                locked = _gf_true;
-        }
+    if (loc->inode->ia_type == IA_IFDIR) {
+        /* lock not required for files */
+        ret = mq_lock(this, loc, F_WRLCK);
+        if (ret < 0)
+            goto out;
+        locked = _gf_true;
+    }
 
-        ret = mq_are_xattrs_set (this, loc, &contri_set, &size_set);
-        if (ret < 0 || (contri_set && size_set))
-                goto out;
+    ret = mq_are_xattrs_set(this, loc, &contri_set, &size_set);
+    if (ret < 0 || (contri_set && size_set))
+        goto out;
 
-        mq_set_ctx_create_status (ctx, _gf_false);
-        status = _gf_true;
+    mq_set_ctx_create_status(ctx, _gf_false);
+    status = _gf_true;
 
-        if (loc->inode->ia_type == IA_IFDIR && size_set == _gf_false) {
-                ret = mq_create_size_xattrs (this, ctx, loc);
-                if (ret < 0)
-                        goto out;
-        }
+    if (loc->inode->ia_type == IA_IFDIR && size_set == _gf_false) {
+        ret = mq_create_size_xattrs(this, ctx, loc);
+        if (ret < 0)
+            goto out;
+    }
 
-        need_txn = _gf_true;
+    need_txn = _gf_true;
 out:
-        if (locked)
-                ret = mq_lock (this, loc, F_UNLCK);
+    if (locked)
+        ret = mq_lock(this, loc, F_UNLCK);
 
-        if (status == _gf_false)
-                mq_set_ctx_create_status (ctx, _gf_false);
+    if (status == _gf_false)
+        mq_set_ctx_create_status(ctx, _gf_false);
 
-        if (need_txn)
-                ret = mq_initiate_quota_blocking_txn (this, loc, NULL);
+    if (need_txn)
+        ret = mq_initiate_quota_blocking_txn(this, loc, NULL);
 
-        return ret;
+    return ret;
 }
 
 static int
-_mq_create_xattrs_txn (xlator_t *this, loc_t *origin_loc, struct iatt *buf,
-                       gf_boolean_t spawn)
+_mq_create_xattrs_txn(xlator_t *this, loc_t *origin_loc, struct iatt *buf,
+                      gf_boolean_t spawn)
 {
-        int32_t                  ret          = -1;
-        quota_inode_ctx_t       *ctx          = NULL;
-        gf_boolean_t             status       = _gf_true;
-        loc_t                    loc          = {0, };
-        inode_contribution_t    *contribution = NULL;
-
-        ret = mq_prevalidate_txn (this, origin_loc, &loc, &ctx, buf);
-        if (ret < 0)
-                goto out;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    gf_boolean_t status = _gf_true;
+    loc_t loc = {
+        0,
+    };
+    inode_contribution_t *contribution = NULL;
+
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, &ctx, buf);
+    if (ret < 0)
+        goto out;
 
-        ret = mq_test_and_set_ctx_create_status (ctx, &status);
-        if (ret < 0 || status == _gf_true)
-                goto out;
+    ret = mq_test_and_set_ctx_create_status(ctx, &status);
+    if (ret < 0 || status == _gf_true)
+        goto out;
 
-        if (!loc_is_root(&loc) && loc.parent) {
-                contribution = mq_add_new_contribution_node (this, ctx, &loc);
-                if (contribution == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot add a new contribution node "
-                                "(%s)", uuid_utoa (loc.gfid));
-                        ret = -1;
-                        goto out;
-                } else {
-                        GF_REF_PUT (contribution);
-                }
+    if (!loc_is_root(&loc) && loc.parent) {
+        contribution = mq_add_new_contribution_node(this, ctx, &loc);
+        if (contribution == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "cannot add a new contribution node "
+                   "(%s)",
+                   uuid_utoa(loc.gfid));
+            ret = -1;
+            goto out;
+        } else {
+            GF_REF_PUT(contribution);
         }
+    }
 
-        ret = mq_synctask (this, mq_create_xattrs_task, spawn, &loc);
+    ret = mq_synctask(this, mq_create_xattrs_task, spawn, &loc);
 out:
-        if (ret < 0 && status == _gf_false)
-                mq_set_ctx_create_status (ctx, _gf_false);
-
-        loc_wipe (&loc);
-        return ret;
-}
-
-int
-mq_create_xattrs_txn (xlator_t *this, loc_t *loc, struct iatt *buf)
-{
-        int32_t                  ret        = -1;
+    if (ret < 0 && status == _gf_false)
+        mq_set_ctx_create_status(ctx, _gf_false);
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        ret = _mq_create_xattrs_txn (this, loc, buf, _gf_true);
-out:
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-mq_create_xattrs_blocking_txn (xlator_t *this, loc_t *loc, struct iatt *buf)
+mq_create_xattrs_txn(xlator_t *this, loc_t *loc, struct iatt *buf)
 {
-        int32_t                  ret        = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        ret = _mq_create_xattrs_txn (this, loc, buf, _gf_false);
+    ret = _mq_create_xattrs_txn(this, loc, buf, _gf_true);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_reduce_parent_size_task (void *opaque)
+mq_reduce_parent_size_task(void *opaque)
 {
-        int32_t                  ret           = -1;
-        int32_t                  prev_dirty    = 0;
-        quota_inode_ctx_t       *ctx           = NULL;
-        quota_inode_ctx_t       *parent_ctx    = NULL;
-        inode_contribution_t    *contribution  = NULL;
-        quota_meta_t             delta         = {0, };
-        quota_meta_t             contri        = {0, };
-        loc_t                    parent_loc    = {0,};
-        gf_boolean_t             locked        = _gf_false;
-        gf_boolean_t             dirty         = _gf_false;
-        quota_synctask_t        *args          = NULL;
-        xlator_t                *this          = NULL;
-        loc_t                   *loc           = NULL;
-        gf_boolean_t             remove_xattr  = _gf_true;
-        uint32_t                 nlink         = 0;
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        contri = args->contri;
-        nlink = args->ia_nlink;
-        this = args->this;
-        THIS = this;
-
-        ret = mq_inode_loc_fill (NULL, loc->parent, &parent_loc);
+    int32_t ret = -1;
+    int32_t prev_dirty = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_inode_ctx_t *parent_ctx = NULL;
+    inode_contribution_t *contribution = NULL;
+    quota_meta_t delta = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t dirty = _gf_false;
+    quota_synctask_t *args = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t remove_xattr = _gf_true;
+    uint32_t nlink = 0;
+
+    GF_ASSERT(opaque);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    contri = args->contri;
+    nlink = args->ia_nlink;
+    this = args->this;
+    THIS = this;
+
+    ret = mq_inode_loc_fill(NULL, loc->parent, &parent_loc);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "parent_loc fill failed for "
+               "child inode %s: ",
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    ret = mq_lock(this, &parent_loc, F_WRLCK);
+    if (ret < 0)
+        goto out;
+    locked = _gf_true;
+
+    if (contri.size >= 0) {
+        /* contri parameter is supplied only for rename operation.
+         * remove xattr is alreday performed, we need to skip
+         * removexattr for rename operation
+         */
+        remove_xattr = _gf_false;
+        delta.size = contri.size;
+        delta.file_count = contri.file_count;
+        delta.dir_count = contri.dir_count;
+    } else {
+        remove_xattr = _gf_true;
+
+        ret = mq_inode_ctx_get(loc->inode, this, &ctx);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "parent_loc fill failed for "
-                        "child inode %s: ", uuid_utoa (loc->inode->gfid));
-                goto out;
+            gf_log_callingfn(this->name, GF_LOG_WARNING,
+                             "ctx for"
+                             " the node %s is NULL",
+                             loc->path);
+            goto out;
         }
 
-        ret = mq_lock (this, &parent_loc, F_WRLCK);
-        if (ret < 0)
-                goto out;
-        locked = _gf_true;
+        contribution = mq_get_contribution_node(loc->parent, ctx);
+        if (contribution == NULL) {
+            ret = -1;
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "contribution for the node %s is NULL", loc->path);
+            goto out;
+        }
 
-        if (contri.size >= 0) {
-                /* contri paramater is supplied only for rename operation.
-                 * remove xattr is alreday performed, we need to skip
-                 * removexattr for rename operation
-                 */
-                remove_xattr = _gf_false;
-                delta.size = contri.size;
-                delta.file_count = contri.file_count;
-                delta.dir_count = contri.dir_count;
-        } else {
-                remove_xattr = _gf_true;
-
-                ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-                if (ret < 0) {
-                        gf_log_callingfn (this->name, GF_LOG_WARNING, "ctx for"
-                                          " the node %s is NULL", loc->path);
-                        goto out;
-                }
-
-                contribution = mq_get_contribution_node (loc->parent, ctx);
-                if (contribution == NULL) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "contribution for the node %s is NULL",
-                                loc->path);
-                        goto out;
-                }
-
-                LOCK (&contribution->lock);
-                {
-                        delta.size = contribution->contribution;
-                        delta.file_count = contribution->file_count;
-                        delta.dir_count = contribution->dir_count;
-                }
-                UNLOCK (&contribution->lock);
+        LOCK(&contribution->lock);
+        {
+            delta.size = contribution->contribution;
+            delta.file_count = contribution->file_count;
+            delta.dir_count = contribution->dir_count;
         }
+        UNLOCK(&contribution->lock);
+    }
 
-        ret = mq_get_set_dirty (this, &parent_loc, 1, &prev_dirty);
-        if (ret < 0)
-                goto out;
-        dirty = _gf_true;
+    ret = mq_get_set_dirty(this, &parent_loc, 1, &prev_dirty);
+    if (ret < 0)
+        goto out;
+    dirty = _gf_true;
 
-        mq_sub_meta (&delta, NULL);
+    mq_sub_meta(&delta, NULL);
 
-        if (remove_xattr) {
-                ret = mq_remove_contri (this, loc, ctx, contribution, &delta,
-                                        nlink);
-                if (ret < 0)
-                        goto out;
-        }
+    if (remove_xattr) {
+        ret = mq_remove_contri(this, loc, ctx, contribution, &delta, nlink);
+        if (ret < 0)
+            goto out;
+    }
 
-        if (quota_meta_is_null (&delta))
-                goto out;
+    if (quota_meta_is_null(&delta))
+        goto out;
 
-        ret = mq_update_size (this, &parent_loc, &delta);
-        if (ret < 0)
-                goto out;
+    ret = mq_update_size(this, &parent_loc, &delta);
+    if (ret < 0)
+        goto out;
 
 out:
-        if (dirty) {
-                if (ret < 0 || prev_dirty) {
-                        /* On failure clear dirty status flag.
-                         * In the next lookup inspect_directory_xattr
-                         * can set the status flag and fix the
-                         * dirty directory.
-                         * Do the same if dir was dirty before
-                         * the txn
-                         */
-                        ret = mq_inode_ctx_get (parent_loc.inode, this,
-                                                &parent_ctx);
-                        if (ret == 0)
-                                mq_set_ctx_dirty_status (parent_ctx, _gf_false);
-                } else {
-                        ret = mq_mark_dirty (this, &parent_loc, 0);
-                }
+    if (dirty) {
+        if (ret < 0 || prev_dirty) {
+            /* On failure clear dirty status flag.
+             * In the next lookup inspect_directory_xattr
+             * can set the status flag and fix the
+             * dirty directory.
+             * Do the same if dir was dirty before
+             * the txn
+             */
+            ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx);
+            if (ret == 0)
+                mq_set_ctx_dirty_status(parent_ctx, _gf_false);
+        } else {
+            ret = mq_mark_dirty(this, &parent_loc, 0);
         }
+    }
 
-        if (locked)
-                ret = mq_lock (this, &parent_loc, F_UNLCK);
+    if (locked)
+        ret = mq_lock(this, &parent_loc, F_UNLCK);
 
-        if (ret >= 0)
-                ret = mq_initiate_quota_blocking_txn (this, &parent_loc, NULL);
+    if (ret >= 0)
+        ret = mq_initiate_quota_blocking_txn(this, &parent_loc, NULL);
 
-        loc_wipe (&parent_loc);
+    loc_wipe(&parent_loc);
 
-        if (contribution)
-                GF_REF_PUT (contribution);
+    if (contribution)
+        GF_REF_PUT(contribution);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_reduce_parent_size_txn (xlator_t *this, loc_t *origin_loc,
-                           quota_meta_t *contri, uint32_t nlink,
-                           call_stub_t *stub)
+mq_reduce_parent_size_txn(xlator_t *this, loc_t *origin_loc,
+                          quota_meta_t *contri, uint32_t nlink,
+                          call_stub_t *stub)
 {
-        int32_t                  ret           = -1;
-        loc_t                    loc           = {0, };
-        gf_boolean_t             resume_stub   = _gf_true;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t resume_stub = _gf_true;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", origin_loc, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", origin_loc, out);
 
-        ret = mq_prevalidate_txn (this, origin_loc, &loc, NULL, NULL);
-        if (ret < 0)
-                goto out;
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-        if (loc_is_root(&loc)) {
-                ret = 0;
-                goto out;
-        }
+    if (loc_is_root(&loc)) {
+        ret = 0;
+        goto out;
+    }
 
-        resume_stub = _gf_false;
-        ret = mq_synctask1 (this, mq_reduce_parent_size_task, _gf_true, &loc,
-                            contri, nlink, stub);
+    resume_stub = _gf_false;
+    ret = mq_synctask1(this, mq_reduce_parent_size_task, _gf_true, &loc, contri,
+                       nlink, stub);
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (resume_stub && stub)
-                call_resume (stub);
+    if (resume_stub && stub)
+        call_resume(stub);
 
-        if (ret)
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                                  "mq_reduce_parent_size_txn failed");
+    if (ret)
+        gf_log_callingfn(this ? this->name : "Marker", GF_LOG_ERROR,
+                         "mq_reduce_parent_size_txn failed");
 
-        return ret;
+    return ret;
 }
 
 int
-mq_initiate_quota_task (void *opaque)
+mq_initiate_quota_task(void *opaque)
 {
-        int32_t                ret        = -1;
-        int32_t                prev_dirty = 0;
-        loc_t                  child_loc  = {0,};
-        loc_t                  parent_loc = {0,};
-        gf_boolean_t           locked     = _gf_false;
-        gf_boolean_t           dirty      = _gf_false;
-        gf_boolean_t           status     = _gf_false;
-        quota_meta_t           delta      = {0, };
-        quota_synctask_t      *args       = NULL;
-        xlator_t              *this       = NULL;
-        loc_t                 *loc        = NULL;
-        inode_contribution_t  *contri     = NULL;
-        quota_inode_ctx_t     *ctx        = NULL;
-        quota_inode_ctx_t     *parent_ctx = NULL;
-        inode_t               *tmp_parent = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", opaque, out);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        this = args->this;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        THIS = this;
-
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = mq_loc_copy (&child_loc, loc);
+    int32_t ret = -1;
+    int32_t prev_dirty = 0;
+    loc_t child_loc = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t dirty = _gf_false;
+    gf_boolean_t status = _gf_false;
+    quota_meta_t delta = {
+        0,
+    };
+    quota_synctask_t *args = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    inode_contribution_t *contri = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_inode_ctx_t *parent_ctx = NULL;
+    inode_t *tmp_parent = NULL;
+
+    GF_VALIDATE_OR_GOTO("marker", opaque, out);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    this = args->this;
+
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    THIS = this;
+
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = mq_loc_copy(&child_loc, loc);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "loc copy failed");
+        goto out;
+    }
+
+    while (!__is_root_gfid(child_loc.gfid)) {
+        ret = mq_inode_ctx_get(child_loc.inode, this, &ctx);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "loc copy failed");
+            gf_log(this->name, GF_LOG_WARNING,
+                   "inode ctx get failed for %s, "
+                   "aborting update txn",
+                   child_loc.path);
+            goto out;
+        }
+
+        /* To improve performance, abort current transaction
+         * if one is already in progress for same inode
+         */
+        if (status == _gf_true) {
+            /* status will already set before txn start,
+             * so it should not be set in first
+             * loop iteration
+             */
+            ret = mq_test_and_set_ctx_updation_status(ctx, &status);
+            if (ret < 0 || status == _gf_true)
+                goto out;
+        }
+
+        if (child_loc.parent == NULL) {
+            ret = mq_build_ancestry(this, &child_loc);
+            if (ret < 0 || child_loc.parent == NULL) {
+                /* If application performs parallel remove
+                 * operations on same set of files/directories
+                 * then we may get ENOENT/ESTALE
+                 */
+                gf_log(this->name,
+                       (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG
+                                                          : GF_LOG_ERROR,
+                       "build ancestry failed for inode %s",
+                       uuid_utoa(child_loc.inode->gfid));
+                ret = -1;
                 goto out;
+            }
+        }
+
+        ret = mq_inode_loc_fill(NULL, child_loc.parent, &parent_loc);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "parent_loc fill "
+                   "failed for child inode %s: ",
+                   uuid_utoa(child_loc.inode->gfid));
+            goto out;
         }
 
-        while (!__is_root_gfid (child_loc.gfid)) {
+        ret = mq_lock(this, &parent_loc, F_WRLCK);
+        if (ret < 0)
+            goto out;
+        locked = _gf_true;
 
-                ret = mq_inode_ctx_get (child_loc.inode, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "inode ctx get failed for %s, "
-                                "aborting update txn", child_loc.path);
-                        goto out;
-                }
+        mq_set_ctx_updation_status(ctx, _gf_false);
+        status = _gf_true;
 
-                /* To improve performance, abort current transaction
-                 * if one is already in progress for same inode
+        /* Contribution node can be NULL in below scenarios and
+           create if needed:
+
+           Scenario 1)
+           In this case create a new contribution node
+           Suppose hard link for a file f1 present in a directory d1 is
+           created in the directory d2 (as f2). Now, since d2's
+           contribution is not there in f1's inode ctx, d2's
+           contribution xattr won't be created and will create problems
+           for quota operations.
+
+           Don't create contribution if parent has been changed after
+           taking a lock, this can happen when rename is performed
+           and writes is still in-progress for the same file
+
+           Scenario 2)
+           When a rename operation is performed, contribution node
+           for olp path will be removed.
+
+           Create contribution node only if oldparent is same as
+           newparent.
+           Consider below example
+           1) rename FOP invoked on file 'x'
+           2) write is still in progress for file 'x'
+           3) rename takes a lock on old-parent
+           4) write-update txn blocked on old-parent to acquire lock
+           5) in rename_cbk, contri xattrs are removed and contribution
+              is deleted and lock is released
+           6) now write-update txn gets the lock and updates the
+              wrong parent as it was holding lock on old parent
+              so validate parent once the lock is acquired
+
+             For more information on this problem, please see
+             doc for marker_rename in file marker.c
+        */
+        contri = mq_get_contribution_node(child_loc.parent, ctx);
+        if (contri == NULL) {
+            tmp_parent = inode_parent(child_loc.inode, 0, NULL);
+            if (tmp_parent == NULL) {
+                /* This can happen if application performs
+                 * parallel remove operations on same set
+                 * of files/directories
                  */
-                if (status == _gf_true) {
-                        /* status will already set before txn start,
-                         * so it should not be set in first
-                         * loop iteration
-                         */
-                        ret = mq_test_and_set_ctx_updation_status (ctx,
-                                                                   &status);
-                        if (ret < 0 || status == _gf_true)
-                                goto out;
-                }
-
-                if (child_loc.parent == NULL) {
-                        ret = mq_build_ancestry (this, &child_loc);
-                        if (ret < 0 || child_loc.parent == NULL) {
-                                /* If application performs parallel remove
-                                 * operations on same set of files/directories
-                                 * then we may get ENOENT/ESTALE
-                                 */
-                                gf_log (this->name,
-                                        (-ret == ENOENT || -ret == ESTALE)
-                                        ? GF_LOG_DEBUG:GF_LOG_ERROR,
-                                        "build ancestry failed for inode %s",
-                                        uuid_utoa (child_loc.inode->gfid));
-                                ret = -1;
-                                goto out;
-                        }
-                }
-
-                ret = mq_inode_loc_fill (NULL, child_loc.parent, &parent_loc);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "parent_loc fill "
-                                "failed for child inode %s: ",
-                                uuid_utoa (child_loc.inode->gfid));
-                        goto out;
-                }
-
-                ret = mq_lock (this, &parent_loc, F_WRLCK);
-                if (ret < 0)
-                        goto out;
-                locked = _gf_true;
-
-                mq_set_ctx_updation_status (ctx, _gf_false);
-                status = _gf_true;
-
-                /* Contribution node can be NULL in below scenarios and
-                   create if needed:
-
-                   Scenario 1)
-                   In this case create a new contribution node
-                   Suppose hard link for a file f1 present in a directory d1 is
-                   created in the directory d2 (as f2). Now, since d2's
-                   contribution is not there in f1's inode ctx, d2's
-                   contribution xattr wont be created and will create problems
-                   for quota operations.
-
-                   Don't create contribution if parent has been changed after
-                   taking a lock, this can happen when rename is performed
-                   and writes is still in-progress for the same file
-
-                   Scenario 2)
-                   When a rename operation is performed, contribution node
-                   for olp path will be removed.
-
-                   Create contribution node only if oldparent is same as
-                   newparent.
-                   Consider below example
-                   1) rename FOP invoked on file 'x'
-                   2) write is still in progress for file 'x'
-                   3) rename takes a lock on old-parent
-                   4) write-update txn blocked on old-parent to acquire lock
-                   5) in rename_cbk, contri xattrs are removed and contribution
-                      is deleted and lock is released
-                   6) now write-update txn gets the lock and updates the
-                      wrong parent as it was holding lock on old parent
-                      so validate parent once the lock is acquired
-
-                     For more information on this problem, please see
-                     doc for marker_rename in file marker.c
-                */
-                contri = mq_get_contribution_node (child_loc.parent, ctx);
-                if (contri == NULL) {
-                        tmp_parent = inode_parent (child_loc.inode, 0, NULL);
-                        if (tmp_parent == NULL) {
-                                /* This can happen if application performs
-                                 * parallel remove operations on same set
-                                 * of files/directories
-                                 */
-                                gf_log (this->name, GF_LOG_WARNING, "parent is "
-                                        "NULL for inode %s",
-                                        uuid_utoa (child_loc.inode->gfid));
-                                ret = -1;
-                                goto out;
-                        }
-                        if (gf_uuid_compare(tmp_parent->gfid,
-                                            parent_loc.gfid)) {
-                                /* abort txn if parent has changed */
-                                ret = 0;
-                                goto out;
-                        }
-
-                        inode_unref (tmp_parent);
-                        tmp_parent = NULL;
-
-                        contri = mq_add_new_contribution_node (this, ctx,
-                                                               &child_loc);
-                        if (contri == NULL) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "create contribution node for %s, "
-                                        "abort update txn", child_loc.path);
-                                ret = -1;
-                                goto out;
-                        }
-                }
-
-                ret = mq_get_delta (this, &child_loc, &delta, ctx, contri);
-                if (ret < 0)
-                        goto out;
-
-                if (quota_meta_is_null (&delta))
-                        goto out;
-
-                ret = mq_get_set_dirty (this, &parent_loc, 1, &prev_dirty);
-                if (ret < 0)
-                        goto out;
-                dirty = _gf_true;
-
-                ret = mq_update_contri (this, &child_loc, contri, &delta);
-                if (ret < 0)
-                        goto out;
-
-                ret = mq_update_size (this, &parent_loc, &delta);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, "rollback "
-                                "contri updation");
-                        mq_sub_meta (&delta, NULL);
-                        mq_update_contri (this, &child_loc, contri, &delta);
-                        goto out;
-                }
-
-                if (prev_dirty == 0) {
-                        ret = mq_mark_dirty (this, &parent_loc, 0);
-                } else {
-                        ret = mq_inode_ctx_get (parent_loc.inode, this,
-                                                &parent_ctx);
-                        if (ret == 0)
-                                mq_set_ctx_dirty_status (parent_ctx, _gf_false);
-                }
-                dirty = _gf_false;
-                prev_dirty = 0;
-
-                ret = mq_lock (this, &parent_loc, F_UNLCK);
-                locked = _gf_false;
-
-                if (__is_root_gfid (parent_loc.gfid))
-                        break;
-
-                /* Repeate above steps upwards till the root */
-                loc_wipe (&child_loc);
-                ret = mq_loc_copy (&child_loc, &parent_loc);
-                if (ret < 0)
-                        goto out;
-
-                loc_wipe (&parent_loc);
-                GF_REF_PUT (contri);
-                contri = NULL;
+                gf_log(this->name, GF_LOG_WARNING,
+                       "parent is "
+                       "NULL for inode %s",
+                       uuid_utoa(child_loc.inode->gfid));
+                ret = -1;
+                goto out;
+            }
+            if (gf_uuid_compare(tmp_parent->gfid, parent_loc.gfid)) {
+                /* abort txn if parent has changed */
+                ret = 0;
+                goto out;
+            }
+
+            inode_unref(tmp_parent);
+            tmp_parent = NULL;
+
+            contri = mq_add_new_contribution_node(this, ctx, &child_loc);
+            if (contri == NULL) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Failed to "
+                       "create contribution node for %s, "
+                       "abort update txn",
+                       child_loc.path);
+                ret = -1;
+                goto out;
+            }
         }
 
-out:
-        if (dirty) {
-                if (ret < 0 || prev_dirty) {
-                        /* On failure clear dirty status flag.
-                         * In the next lookup inspect_directory_xattr
-                         * can set the status flag and fix the
-                         * dirty directory.
-                         * Do the same if the dir was dirty before
-                         * txn
-                         */
-                        ret = mq_inode_ctx_get (parent_loc.inode, this,
-                                                &parent_ctx);
-                        if (ret == 0)
-                                mq_set_ctx_dirty_status (parent_ctx, _gf_false);
-                } else {
-                        ret = mq_mark_dirty (this, &parent_loc, 0);
-                }
+        ret = mq_get_delta(this, &child_loc, &delta, ctx, contri);
+        if (ret < 0)
+            goto out;
+
+        if (quota_meta_is_null(&delta))
+            goto out;
+
+        ret = mq_get_set_dirty(this, &parent_loc, 1, &prev_dirty);
+        if (ret < 0)
+            goto out;
+        dirty = _gf_true;
+
+        ret = mq_update_contri(this, &child_loc, contri, &delta);
+        if (ret < 0)
+            goto out;
+
+        ret = mq_update_size(this, &parent_loc, &delta);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "rollback "
+                   "contri updation");
+            mq_sub_meta(&delta, NULL);
+            mq_update_contri(this, &child_loc, contri, &delta);
+            goto out;
         }
 
-        if (locked)
-                ret = mq_lock (this, &parent_loc, F_UNLCK);
+        if (prev_dirty == 0) {
+            ret = mq_mark_dirty(this, &parent_loc, 0);
+        } else {
+            ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx);
+            if (ret == 0)
+                mq_set_ctx_dirty_status(parent_ctx, _gf_false);
+        }
+        dirty = _gf_false;
+        prev_dirty = 0;
 
-        if (ctx && status == _gf_false)
-                mq_set_ctx_updation_status (ctx, _gf_false);
+        ret = mq_lock(this, &parent_loc, F_UNLCK);
+        locked = _gf_false;
 
-        loc_wipe (&child_loc);
-        loc_wipe (&parent_loc);
+        if (__is_root_gfid(parent_loc.gfid))
+            break;
 
-        if (tmp_parent)
-                inode_unref (tmp_parent);
+        /* Repeate above steps upwards till the root */
+        loc_wipe(&child_loc);
+        ret = mq_loc_copy(&child_loc, &parent_loc);
+        if (ret < 0)
+            goto out;
 
-        if (contri)
-                GF_REF_PUT (contri);
+        loc_wipe(&parent_loc);
+        GF_REF_PUT(contri);
+        contri = NULL;
+    }
 
-        return 0;
+out:
+    if ((dirty) && (ret < 0)) {
+        /* On failure clear dirty status flag.
+         * In the next lookup inspect_directory_xattr
+         * can set the status flag and fix the
+         * dirty directory.
+         * Do the same if the dir was dirty before
+         * txn
+         */
+        ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx);
+        if (ret == 0)
+            mq_set_ctx_dirty_status(parent_ctx, _gf_false);
+    }
+
+    if (locked)
+        ret = mq_lock(this, &parent_loc, F_UNLCK);
+
+    if (ctx && status == _gf_false)
+        mq_set_ctx_updation_status(ctx, _gf_false);
+
+    loc_wipe(&child_loc);
+    loc_wipe(&parent_loc);
+
+    if (tmp_parent)
+        inode_unref(tmp_parent);
+
+    if (contri)
+        GF_REF_PUT(contri);
+
+    return 0;
 }
 
 int
-_mq_initiate_quota_txn (xlator_t *this, loc_t *origin_loc, struct iatt *buf,
-                        gf_boolean_t spawn)
+_mq_initiate_quota_txn(xlator_t *this, loc_t *origin_loc, struct iatt *buf,
+                       gf_boolean_t spawn)
 {
-        int32_t                 ret          = -1;
-        quota_inode_ctx_t      *ctx          = NULL;
-        gf_boolean_t            status       = _gf_true;
-        loc_t                   loc          = {0,};
-
-        ret = mq_prevalidate_txn (this, origin_loc, &loc, &ctx, buf);
-        if (ret < 0)
-                goto out;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    gf_boolean_t status = _gf_true;
+    loc_t loc = {
+        0,
+    };
+
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, &ctx, buf);
+    if (ret < 0)
+        goto out;
 
-        if (loc_is_root(&loc)) {
-                ret = 0;
-                goto out;
-        }
+    if (loc_is_root(&loc)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = mq_test_and_set_ctx_updation_status (ctx, &status);
-        if (ret < 0 || status == _gf_true)
-                goto out;
+    ret = mq_test_and_set_ctx_updation_status(ctx, &status);
+    if (ret < 0 || status == _gf_true)
+        goto out;
 
-        ret = mq_synctask (this, mq_initiate_quota_task, spawn, &loc);
+    ret = mq_synctask(this, mq_initiate_quota_task, spawn, &loc);
 
 out:
-        if (ret < 0 && status == _gf_false)
-                mq_set_ctx_updation_status (ctx, _gf_false);
+    if (ret < 0 && status == _gf_false)
+        mq_set_ctx_updation_status(ctx, _gf_false);
 
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-mq_initiate_quota_txn (xlator_t *this, loc_t *loc, struct iatt *buf)
+mq_initiate_quota_txn(xlator_t *this, loc_t *loc, struct iatt *buf)
 {
-        int32_t                 ret          = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        ret = _mq_initiate_quota_txn (this, loc, buf, _gf_true);
+    ret = _mq_initiate_quota_txn(this, loc, buf, _gf_true);
 out:
-        return ret;
+    return ret;
 }
 
 int
-mq_initiate_quota_blocking_txn (xlator_t *this, loc_t *loc, struct iatt *buf)
+mq_initiate_quota_blocking_txn(xlator_t *this, loc_t *loc, struct iatt *buf)
 {
-        int32_t                 ret          = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        ret = _mq_initiate_quota_txn (this, loc, buf, _gf_false);
+    ret = _mq_initiate_quota_txn(this, loc, buf, _gf_false);
 out:
-        return ret;
+    return ret;
 }
 
 int
-mq_update_dirty_inode_task (void *opaque)
+mq_update_dirty_inode_task(void *opaque)
 {
-        int32_t               ret                         = -1;
-        fd_t                 *fd                          = NULL;
-        off_t                 offset                      = 0;
-        gf_dirent_t           entries;
-        gf_dirent_t          *entry                       = NULL;
-        gf_boolean_t          locked                      = _gf_false;
-        gf_boolean_t          updated                     = _gf_false;
-        int32_t               dirty                       = 0;
-        quota_meta_t          contri                      = {0, };
-        quota_meta_t          size                        = {0, };
-        quota_meta_t          contri_sum                  = {0, };
-        quota_meta_t          delta                       = {0, };
-        quota_synctask_t     *args                        = NULL;
-        xlator_t             *this                        = NULL;
-        loc_t                *loc                         = NULL;
-        quota_inode_ctx_t    *ctx                         = NULL;
-        dict_t               *xdata                       = NULL;
-        char                  contri_key[QUOTA_KEY_MAX]   = {0, };
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        this = args->this;
-        THIS = this;
-        INIT_LIST_HEAD (&entries.list);
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0)
-                goto out;
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    off_t offset = 0;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t updated = _gf_false;
+    int32_t dirty = 0;
+    quota_meta_t contri = {
+        0,
+    };
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri_sum = {
+        0,
+    };
+    quota_meta_t delta = {
+        0,
+    };
+    quota_synctask_t *args = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    dict_t *xdata = NULL;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+
+    GF_ASSERT(opaque);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    this = args->this;
+    THIS = this;
+    INIT_LIST_HEAD(&entries.list);
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        GET_CONTRI_KEY (this, contri_key, loc->gfid, ret);
-        if (ret < 0)
-                goto out;
+    GET_CONTRI_KEY(this, contri_key, loc->gfid, keylen);
+    if (keylen < 0) {
+        ret = keylen;
+        goto out;
+    }
 
-        xdata = dict_new ();
-        if (xdata == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -1;
-                goto out;
-        }
+    xdata = dict_new();
+    if (xdata == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_int64 (xdata, contri_key, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_set failed");
-                goto out;
-        }
+    ret = dict_set_int64(xdata, contri_key, 0);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_set failed");
+        goto out;
+    }
 
-        ret = mq_lock (this, loc, F_WRLCK);
-        if (ret < 0)
-                goto out;
-        locked = _gf_true;
+    ret = mq_lock(this, loc, F_WRLCK);
+    if (ret < 0)
+        goto out;
+    locked = _gf_true;
 
-        ret = mq_get_dirty (this, loc, &dirty);
-        if (ret < 0 || dirty == 0) {
-                ret = 0;
-                goto out;
-        }
+    ret = mq_get_dirty(this, loc, &dirty);
+    if (ret < 0 || dirty == 0) {
+        ret = 0;
+        goto out;
+    }
 
-        fd = fd_create (loc->inode, 0);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                ret = -1;
-                goto out;
-        }
+    fd = fd_create(loc->inode, 0);
+    if (!fd) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to create fd");
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_opendir(this, loc, fd, NULL, NULL);
+    if (ret < 0) {
+        gf_log(this->name,
+               (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+               "opendir failed "
+               "for %s: %s",
+               loc->path, strerror(-ret));
+        goto out;
+    }
 
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
+    fd_bind(fd);
+    while ((ret = syncop_readdirp(this, fd, 131072, offset, &entries, xdata,
+                                  NULL)) != 0) {
         if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "opendir failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
+            gf_log(this->name,
+                   (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG
+                                                      : GF_LOG_ERROR,
+                   "readdirp failed "
+                   "for %s: %s",
+                   loc->path, strerror(-ret));
+            goto out;
         }
 
-        fd_bind (fd);
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, &entries,
-                                       xdata, NULL)) != 0) {
-                if (ret < 0) {
-                        gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                ? GF_LOG_DEBUG:GF_LOG_ERROR, "readdirp failed "
-                                "for %s: %s", loc->path, strerror (-ret));
-                        goto out;
-                }
-
-                if (list_empty (&entries.list))
-                        break;
+        if (list_empty(&entries.list))
+            break;
 
-                list_for_each_entry (entry, &entries.list, list) {
-                        offset = entry->d_off;
-
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
 
-                        memset (&contri, 0, sizeof (contri));
-                        quota_dict_get_meta (entry->dict, contri_key, &contri);
-                        if (quota_meta_is_null (&contri))
-                                continue;
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
 
-                        mq_add_meta (&contri_sum, &contri);
-                }
+            memset(&contri, 0, sizeof(contri));
+            quota_dict_get_meta(entry->dict, contri_key, keylen, &contri);
+            if (quota_meta_is_null(&contri))
+                continue;
 
-                gf_dirent_free (&entries);
+            mq_add_meta(&contri_sum, &contri);
         }
-        /* Inculde for self */
-        contri_sum.dir_count++;
 
-        ret = mq_get_size (this, loc, &size);
-        if (ret < 0)
-                goto out;
+        gf_dirent_free(&entries);
+    }
+    /* Inculde for self */
+    contri_sum.dir_count++;
 
-        mq_compute_delta (&delta, &contri_sum, &size);
+    ret = _mq_get_metadata(this, loc, NULL, &size, 0);
+    if (ret < 0)
+        goto out;
 
-        if (quota_meta_is_null (&delta))
-                goto out;
+    mq_compute_delta(&delta, &contri_sum, &size);
 
-        gf_log (this->name, GF_LOG_INFO, "calculated size = %"PRId64
-                ", original size = %"PRIu64 ", diff = %"PRIu64
-                ", path = %s ", contri_sum.size, size.size, delta.size,
-                loc->path);
+    if (quota_meta_is_null(&delta))
+        goto out;
 
-        gf_log (this->name, GF_LOG_INFO, "calculated f_count = %"PRId64
-                ", original f_count = %"PRIu64 ", diff = %"PRIu64
-                ", path = %s ", contri_sum.file_count, size.file_count,
-                delta.file_count, loc->path);
+    gf_log(this->name, GF_LOG_INFO,
+           "calculated size = %" PRId64 ", original size = %" PRIu64
+           ", diff = %" PRIu64 ", path = %s ",
+           contri_sum.size, size.size, delta.size, loc->path);
 
-        gf_log (this->name, GF_LOG_INFO, "calculated d_count = %"PRId64
-                ", original d_count = %"PRIu64 ", diff = %"PRIu64
-                ", path = %s ", contri_sum.dir_count, size.dir_count,
-                delta.dir_count, loc->path);
+    gf_log(this->name, GF_LOG_INFO,
+           "calculated f_count = %" PRId64 ", original f_count = %" PRIu64
+           ", diff = %" PRIu64 ", path = %s ",
+           contri_sum.file_count, size.file_count, delta.file_count, loc->path);
 
+    gf_log(this->name, GF_LOG_INFO,
+           "calculated d_count = %" PRId64 ", original d_count = %" PRIu64
+           ", diff = %" PRIu64 ", path = %s ",
+           contri_sum.dir_count, size.dir_count, delta.dir_count, loc->path);
 
-        ret = mq_update_size (this, loc, &delta);
-        if (ret < 0)
-                goto out;
+    ret = mq_update_size(this, loc, &delta);
+    if (ret < 0)
+        goto out;
 
-        updated = _gf_true;
+    updated = _gf_true;
 
 out:
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        if (fd)
-                fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (ret < 0) {
-                /* On failure clear dirty status flag.
-                 * In the next lookup inspect_directory_xattr
-                 * can set the status flag and fix the
-                 * dirty directory
-                 */
-                if (ctx)
-                        mq_set_ctx_dirty_status (ctx, _gf_false);
-        } else if (dirty) {
-                mq_mark_dirty (this, loc, 0);
-        }
+    if (ret < 0) {
+        /* On failure clear dirty status flag.
+         * In the next lookup inspect_directory_xattr
+         * can set the status flag and fix the
+         * dirty directory
+         */
+        if (ctx)
+            mq_set_ctx_dirty_status(ctx, _gf_false);
+    } else if (dirty) {
+        mq_mark_dirty(this, loc, 0);
+    }
 
-        if (locked)
-                mq_lock (this, loc, F_UNLCK);
+    if (locked)
+        mq_lock(this, loc, F_UNLCK);
 
-        if (updated)
-                mq_initiate_quota_blocking_txn (this, loc, NULL);
+    if (updated)
+        mq_initiate_quota_blocking_txn(this, loc, NULL);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_update_dirty_inode_txn (xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx)
+mq_update_dirty_inode_txn(xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx)
 {
-        int32_t          ret        = -1;
-        gf_boolean_t     status     = _gf_true;
+    int32_t ret = -1;
+    gf_boolean_t status = _gf_true;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        ret = mq_test_and_set_ctx_dirty_status (ctx, &status);
-        if (ret < 0 || status == _gf_true)
-                goto out;
+    mq_test_and_set_ctx_status(ctx, &ctx->dirty_status, &status);
+    if (status == _gf_true)
+        goto out;
 
-        ret = mq_synctask (this, mq_update_dirty_inode_task, _gf_true, loc);
+    ret = mq_synctask(this, mq_update_dirty_inode_task, _gf_true, loc);
 out:
-        if (ret < 0 && status == _gf_false)
-                mq_set_ctx_dirty_status (ctx, _gf_false);
+    if (ret < 0 && status == _gf_false)
+        mq_set_ctx_dirty_status(ctx, _gf_false);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_inspect_directory_xattr (xlator_t *this, quota_inode_ctx_t *ctx,
-                            inode_contribution_t *contribution, loc_t *loc,
-                            dict_t *dict, struct iatt buf)
+mq_inspect_directory_xattr(xlator_t *this, quota_inode_ctx_t *ctx,
+                           inode_contribution_t *contribution, loc_t *loc,
+                           dict_t *dict)
 {
-        int32_t               ret                          = -1;
-        int8_t                dirty                        = -1;
-        quota_meta_t          size                         = {0, };
-        quota_meta_t          contri                       = {0, };
-        quota_meta_t          delta                        = {0, };
-        char                  contri_key[QUOTA_KEY_MAX]    = {0, };
-        char                  size_key[QUOTA_KEY_MAX]      = {0, };
-        gf_boolean_t          status                       = _gf_false;
-
-        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
-        if (ret < 0) {
-                /* dirty is set only on the first file write operation
-                 * so ignore this error
-                 */
-                ret = 0;
-                dirty = 0;
-        }
+    int32_t ret = -1;
+    int8_t dirty = -1;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
+    quota_meta_t delta = {
+        0,
+    };
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    gf_boolean_t status = _gf_false;
+
+    ret = dict_get_int8(dict, QUOTA_DIRTY_KEY, &dirty);
+    if (ret < 0) {
+        /* dirty is set only on the first file write operation
+         * so ignore this error
+         */
+        ret = 0;
+        dirty = 0;
+    }
 
-        GET_SIZE_KEY (this, size_key, ret);
-        if (ret < 0)
-                goto out;
-        ret = _quota_dict_get_meta (this, dict, size_key, &size,
-                                    IA_IFDIR, _gf_false);
+    GET_SIZE_KEY(this, size_key, keylen);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+    ret = _quota_dict_get_meta(this, dict, size_key, keylen, &size, IA_IFDIR,
+                               _gf_false);
+    if (ret < 0)
+        goto create_xattr;
+
+    if (!contribution)
+        goto create_xattr;
+
+    if (!loc_is_root(loc)) {
+        GET_CONTRI_KEY(this, contri_key, contribution->gfid, keylen);
+        if (keylen < 0) {
+            ret = -1;
+            goto out;
+        }
+        ret = _quota_dict_get_meta(this, dict, contri_key, keylen, &contri,
+                                   IA_IFDIR, _gf_false);
         if (ret < 0)
-                goto create_xattr;
-
-        if (!loc_is_root(loc)) {
-                GET_CONTRI_KEY (this, contri_key, contribution->gfid, ret);
-                if (ret < 0)
-                        goto out;
-
-                ret = _quota_dict_get_meta (this, dict, contri_key, &contri,
-                                            IA_IFDIR, _gf_false);
-                if (ret < 0)
-                        goto create_xattr;
-
-                LOCK (&contribution->lock);
-                {
-                        contribution->contribution = contri.size;
-                        contribution->file_count = contri.file_count;
-                        contribution->dir_count = contri.dir_count;
-                }
-                UNLOCK (&contribution->lock);
-        }
+            goto create_xattr;
 
-        LOCK (&ctx->lock);
+        LOCK(&contribution->lock);
         {
-                ctx->size = size.size;
-                ctx->file_count = size.file_count;
-                ctx->dir_count = size.dir_count;
-                ctx->dirty = dirty;
-        }
-        UNLOCK (&ctx->lock);
-
-        ret = mq_get_ctx_updation_status (ctx, &status);
-        if (ret < 0 || status == _gf_true) {
-                /* If the update txn is in progress abort inspection */
-                ret = 0;
-                goto out;
-        }
+            contribution->contribution = contri.size;
+            contribution->file_count = contri.file_count;
+            contribution->dir_count = contri.dir_count;
+        }
+        UNLOCK(&contribution->lock);
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->size = size.size;
+        ctx->file_count = size.file_count;
+        ctx->dir_count = size.dir_count;
+        ctx->dirty = dirty;
+    }
+    UNLOCK(&ctx->lock);
+
+    ret = mq_get_ctx_updation_status(ctx, &status);
+    if (ret < 0 || status == _gf_true) {
+        /* If the update txn is in progress abort inspection */
+        ret = 0;
+        goto out;
+    }
 
-        mq_compute_delta (&delta, &size, &contri);
+    mq_compute_delta(&delta, &size, &contri);
 
-        if (dirty) {
-                ret = mq_update_dirty_inode_txn (this, loc, ctx);
-                goto out;
-        }
+    if (dirty) {
+        ret = mq_update_dirty_inode_txn(this, loc, ctx);
+        goto out;
+    }
 
-        if (!loc_is_root(loc) &&
-            !quota_meta_is_null (&delta))
-                mq_initiate_quota_txn (this, loc, NULL);
+    if (!loc_is_root(loc) && !quota_meta_is_null(&delta))
+        mq_initiate_quota_txn(this, loc, NULL);
 
-        ret = 0;
-        goto out;
+    ret = 0;
+    goto out;
 
 create_xattr:
-        if (ret < 0)
-                ret = mq_create_xattrs_txn (this, loc, NULL);
+    if (ret < 0)
+        ret = mq_create_xattrs_txn(this, loc, NULL);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_inspect_file_xattr (xlator_t *this, quota_inode_ctx_t *ctx,
-                       inode_contribution_t *contribution, loc_t *loc,
-                       dict_t *dict, struct iatt buf)
+mq_inspect_file_xattr(xlator_t *this, quota_inode_ctx_t *ctx,
+                      inode_contribution_t *contribution, loc_t *loc,
+                      dict_t *dict, struct iatt *buf)
 {
-        int32_t               ret                          = -1;
-        quota_meta_t          size                         = {0, };
-        quota_meta_t          contri                       = {0, };
-        quota_meta_t          delta                        = {0, };
-        char                  contri_key[QUOTA_KEY_MAX]    = {0, };
-        gf_boolean_t          status                       = _gf_false;
-
-        LOCK (&ctx->lock);
-        {
-                ctx->size = 512 * buf.ia_blocks;
-                ctx->file_count = 1;
-                ctx->dir_count = 0;
+    int32_t ret = -1;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
+    quota_meta_t delta = {
+        0,
+    };
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    gf_boolean_t status = _gf_false;
+
+    if (!buf || !contribution || !ctx)
+        goto out;
 
-                size.size = ctx->size;
-                size.file_count = ctx->file_count;
-                size.dir_count = ctx->dir_count;
+    LOCK(&ctx->lock);
+    {
+        ctx->size = 512 * buf->ia_blocks;
+        ctx->file_count = 1;
+        ctx->dir_count = 0;
+
+        size.size = ctx->size;
+        size.file_count = ctx->file_count;
+        size.dir_count = ctx->dir_count;
+    }
+    UNLOCK(&ctx->lock);
+
+    GET_CONTRI_KEY(this, contri_key, contribution->gfid, keylen);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = _quota_dict_get_meta(this, dict, contri_key, keylen, &contri,
+                               IA_IFREG, _gf_true);
+    if (ret < 0) {
+        ret = mq_create_xattrs_txn(this, loc, NULL);
+    } else {
+        LOCK(&contribution->lock);
+        {
+            contribution->contribution = contri.size;
+            contribution->file_count = contri.file_count;
+            contribution->dir_count = contri.dir_count;
         }
-        UNLOCK (&ctx->lock);
-
-        GET_CONTRI_KEY (this, contri_key, contribution->gfid, ret);
-        if (ret < 0)
-                goto out;
+        UNLOCK(&contribution->lock);
 
-        ret = _quota_dict_get_meta (this, dict, contri_key, &contri,
-                                    IA_IFREG, _gf_true);
-        if (ret < 0) {
-                ret = mq_create_xattrs_txn (this, loc, NULL);
-        } else {
-                LOCK (&contribution->lock);
-                {
-                        contribution->contribution = contri.size;
-                        contribution->file_count = contri.file_count;
-                        contribution->dir_count = contri.dir_count;
-                }
-                UNLOCK (&contribution->lock);
-
-                ret = mq_get_ctx_updation_status (ctx, &status);
-                if (ret < 0 || status == _gf_true) {
-                        /* If the update txn is in progress abort inspection */
-                        ret = 0;
-                        goto out;
-                }
-
-                mq_compute_delta (&delta, &size, &contri);
-                if (!quota_meta_is_null (&delta))
-                        mq_initiate_quota_txn (this, loc, NULL);
+        ret = mq_get_ctx_updation_status(ctx, &status);
+        if (ret < 0 || status == _gf_true) {
+            /* If the update txn is in progress abort inspection */
+            ret = 0;
+            goto out;
         }
-        /* TODO: revist this code when fixing hardlinks */
+
+        mq_compute_delta(&delta, &size, &contri);
+        if (!quota_meta_is_null(&delta))
+            mq_initiate_quota_txn(this, loc, NULL);
+    }
+    /* TODO: revist this code when fixing hardlinks */
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_xattr_state (xlator_t *this, loc_t *origin_loc, dict_t *dict,
-                struct iatt buf)
+mq_xattr_state(xlator_t *this, loc_t *origin_loc, dict_t *dict,
+               struct iatt *buf)
 {
-        int32_t               ret             = -1;
-        quota_inode_ctx_t    *ctx             = NULL;
-        loc_t                 loc             = {0, };
-        inode_contribution_t *contribution    = NULL;
-
-        ret = mq_prevalidate_txn (this, origin_loc, &loc, &ctx, &buf);
-        if (ret < 0 || loc.parent == NULL)
-                goto out;
-
-        if (!loc_is_root(&loc)) {
-                contribution = mq_add_new_contribution_node (this, ctx, &loc);
-                if (contribution == NULL) {
-                        if (!gf_uuid_is_null (loc.inode->gfid))
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot add a new contribution node "
-                                        "(%s)", uuid_utoa (loc.gfid));
-                        ret = -1;
-                        goto out;
-                }
-        }
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    loc_t loc = {
+        0,
+    };
+    inode_contribution_t *contribution = NULL;
+
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, &ctx, buf);
+    if (ret < 0 || loc.parent == NULL)
+        goto out;
 
-        if (buf.ia_type == IA_IFDIR)
-                mq_inspect_directory_xattr (this, ctx, contribution, &loc, dict,
-                                            buf);
+    if (!loc_is_root(&loc)) {
+        contribution = mq_add_new_contribution_node(this, ctx, &loc);
+        if (contribution == NULL) {
+            if (!gf_uuid_is_null(loc.inode->gfid))
+                gf_log(this->name, GF_LOG_WARNING,
+                       "cannot add a new contribution node "
+                       "(%s)",
+                       uuid_utoa(loc.gfid));
+            ret = -1;
+            goto out;
+        }
+        if (buf->ia_type == IA_IFDIR)
+            mq_inspect_directory_xattr(this, ctx, contribution, &loc, dict);
         else
-                mq_inspect_file_xattr (this, ctx, contribution, &loc, dict,
-                                       buf);
+            mq_inspect_file_xattr(this, ctx, contribution, &loc, dict, buf);
+    } else {
+        mq_inspect_directory_xattr(this, ctx, 0, &loc, dict);
+    }
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        if (contribution)
-                GF_REF_PUT (contribution);
+    if (contribution)
+        GF_REF_PUT(contribution);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mq_req_xattr (xlator_t *this, loc_t *loc, dict_t *dict,
-              char *contri_key, char *size_key)
+mq_req_xattr(xlator_t *this, loc_t *loc, dict_t *dict, char *contri_key,
+             char *size_key)
 {
-        int32_t               ret                = -1;
-        char                  key[QUOTA_KEY_MAX] = {0, };
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", dict, out);
-
-        if (!loc_is_root(loc)) {
-                ret = mq_dict_set_contribution (this, dict, loc, NULL,
-                                                contri_key);
-                if (ret < 0)
-                        goto out;
-        }
+    int32_t ret = -1;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
 
-        GET_SIZE_KEY (this, key, ret);
-        if (ret < 0)
-                goto out;
-        if (size_key)
-                strncpy (size_key, key, QUOTA_KEY_MAX);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", dict, out);
 
-        ret = dict_set_uint64 (dict, key, 0);
+    if (!loc_is_root(loc)) {
+        ret = mq_dict_set_contribution(this, dict, loc, NULL, contri_key);
         if (ret < 0)
-                goto out;
+            goto out;
+    }
 
-        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, 0);
+    GET_SIZE_KEY(this, key, ret);
+    if (ret < 0)
+        goto out;
+    if (size_key)
+        if (snprintf(size_key, QUOTA_KEY_MAX, "%s", key) >= QUOTA_KEY_MAX) {
+            ret = -1;
+            goto out;
+        }
 
-out:
-        if (ret < 0)
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "dict set failed");
+    ret = dict_set_uint64(dict, key, 0);
+    if (ret < 0)
+        goto out;
 
-        return ret;
-}
+    ret = dict_set_int8(dict, QUOTA_DIRTY_KEY, 0);
 
+out:
+    if (ret < 0)
+        gf_log_callingfn(this ? this->name : "Marker", GF_LOG_ERROR,
+                         "dict set failed");
+    return ret;
+}
 
 int32_t
-mq_forget (xlator_t *this, quota_inode_ctx_t *ctx)
+mq_forget(xlator_t *this, quota_inode_ctx_t *ctx)
 {
-        inode_contribution_t *contri = NULL;
-        inode_contribution_t *next   = NULL;
+    inode_contribution_t *contri = NULL;
+    inode_contribution_t *next = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        list_for_each_entry_safe (contri, next, &ctx->contribution_head,
-                                  contri_list) {
-                list_del_init (&contri->contri_list);
-                GF_REF_PUT (contri);
-        }
+    list_for_each_entry_safe(contri, next, &ctx->contribution_head, contri_list)
+    {
+        list_del_init(&contri->contri_list);
+        GF_REF_PUT(contri);
+    }
 
-        LOCK_DESTROY (&ctx->lock);
-        GF_FREE (ctx);
+    LOCK_DESTROY(&ctx->lock);
+    GF_FREE(ctx);
 out:
-        return 0;
+    return 0;
 }
diff --git a/xlators/features/marker/src/marker-quota.h b/xlators/features/marker/src/marker-quota.h
index 51e062537b8..4bbf6878b22 100644
--- a/xlators/features/marker/src/marker-quota.h
+++ b/xlators/features/marker/src/marker-quota.h
@@ -10,147 +10,131 @@
 #ifndef _MARKER_QUOTA_H
 #define _MARKER_QUOTA_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "marker-mem-types.h"
-#include "refcount.h"
-#include "quota-common-utils.h"
-#include "call-stub.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/quota-common-utils.h>
+#include <glusterfs/call-stub.h>
 
 #define QUOTA_XATTR_PREFIX "trusted.glusterfs"
 #define QUOTA_DIRTY_KEY "trusted.glusterfs.quota.dirty"
 
-#define CONTRIBUTION  "contri"
+#define CONTRIBUTION "contri"
 #define QUOTA_KEY_MAX 512
 #define READDIR_BUF 4096
 
-
-#define QUOTA_STACK_DESTROY(_frame, _this)              \
-        do {                                            \
-                quota_local_t *_local = NULL;           \
-                _local = _frame->local;                 \
-                _frame->local = NULL;                   \
-                STACK_DESTROY (_frame->root);           \
-                mq_local_unref (_this, _local);         \
-        } while (0)
-
-
-#define QUOTA_ALLOC(var, type, ret)                     \
-        do {                                            \
-                ret = 0;                                \
-                var = GF_CALLOC (sizeof (type), 1,      \
-                                 gf_marker_mt_##type);  \
-                if (!var) {                             \
-                        ret = -1;                       \
-                }                                       \
-        } while (0);
-
-#define QUOTA_ALLOC_OR_GOTO(var, type, ret, label)      \
-        do {                                            \
-                var = GF_CALLOC (sizeof (type), 1,      \
-                                 gf_marker_mt_##type);  \
-                if (!var) {                             \
-                        gf_log ("", GF_LOG_ERROR,       \
-                                "out of memory");       \
-                        ret = -1;                       \
-                        goto label;                     \
-                }                                       \
-                ret = 0;                                \
-        } while (0);
-
-#define GET_QUOTA_KEY(_this, var, key, _ret)                              \
-        do {                                                              \
-                marker_conf_t  *_priv = _this->private;                   \
-                if (_priv->version > 0)                                   \
-                        _ret = snprintf (var, QUOTA_KEY_MAX, "%s.%d",     \
-                                         key, _priv->version);            \
-                else                                                      \
-                        _ret = snprintf (var, QUOTA_KEY_MAX, "%s", key);  \
-        } while (0)
-
-#define GET_CONTRI_KEY(_this, var, _gfid, _ret)                           \
-        do {                                                              \
-                char  _tmp_var[QUOTA_KEY_MAX] = {0, };                   \
-                if (_gfid != NULL) {                                      \
-                        char _gfid_unparsed[40];                          \
-                        gf_uuid_unparse (_gfid, _gfid_unparsed);          \
-                        _ret = snprintf (_tmp_var, QUOTA_KEY_MAX,         \
-                                         QUOTA_XATTR_PREFIX               \
-                                         ".%s.%s." CONTRIBUTION,          \
-                                         "quota", _gfid_unparsed);        \
-                } else {                                                  \
-                        _ret = snprintf (_tmp_var, QUOTA_KEY_MAX,         \
-                                         QUOTA_XATTR_PREFIX               \
-                                         ".%s.." CONTRIBUTION,            \
-                                         "quota");                        \
-                }                                                         \
-                GET_QUOTA_KEY (_this, var, _tmp_var, _ret);               \
-        } while (0)
-
-#define GET_SIZE_KEY(_this, var, _ret)                                    \
-        {                                                                 \
-                GET_QUOTA_KEY (_this, var, QUOTA_SIZE_KEY, _ret);         \
-        }
-
-#define QUOTA_SAFE_INCREMENT(lock, var)         \
-        do {                                    \
-                LOCK (lock);                    \
-                var ++;                         \
-                UNLOCK (lock);                  \
-        } while (0)
+#define QUOTA_ALLOC(var, type, ret)                                            \
+    do {                                                                       \
+        ret = 0;                                                               \
+        var = GF_CALLOC(sizeof(type), 1, gf_marker_mt_##type);                 \
+        if (!var) {                                                            \
+            ret = -1;                                                          \
+        }                                                                      \
+    } while (0);
+
+#define QUOTA_ALLOC_OR_GOTO(var, type, ret, label)                             \
+    do {                                                                       \
+        var = GF_CALLOC(sizeof(type), 1, gf_marker_mt_##type);                 \
+        if (!var) {                                                            \
+            gf_log("", GF_LOG_ERROR, "out of memory");                         \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        }                                                                      \
+        ret = 0;                                                               \
+    } while (0);
+
+#define GET_QUOTA_KEY(_this, var, key, _ret)                                   \
+    do {                                                                       \
+        marker_conf_t *_priv = _this->private;                                 \
+        if (_priv->version > 0)                                                \
+            _ret = snprintf(var, QUOTA_KEY_MAX, "%s.%d", key, _priv->version); \
+        else                                                                   \
+            _ret = snprintf(var, QUOTA_KEY_MAX, "%s", key);                    \
+    } while (0)
+
+#define GET_CONTRI_KEY(_this, var, _gfid, _ret)                                \
+    do {                                                                       \
+        char _tmp_var[QUOTA_KEY_MAX] = {                                       \
+            0,                                                                 \
+        };                                                                     \
+        if (_gfid != NULL) {                                                   \
+            char _gfid_unparsed[40];                                           \
+            gf_uuid_unparse(_gfid, _gfid_unparsed);                            \
+            _ret = snprintf(_tmp_var, QUOTA_KEY_MAX,                           \
+                            QUOTA_XATTR_PREFIX ".%s.%s." CONTRIBUTION,         \
+                            "quota", _gfid_unparsed);                          \
+        } else {                                                               \
+            _ret = snprintf(_tmp_var, QUOTA_KEY_MAX,                           \
+                            QUOTA_XATTR_PREFIX ".%s.." CONTRIBUTION, "quota"); \
+        }                                                                      \
+        GET_QUOTA_KEY(_this, var, _tmp_var, _ret);                             \
+    } while (0)
+
+#define GET_SIZE_KEY(_this, var, _ret)                                         \
+    {                                                                          \
+        GET_QUOTA_KEY(_this, var, QUOTA_SIZE_KEY, _ret);                       \
+    }
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var++;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
 
 struct quota_inode_ctx {
-        int64_t                size;
-        int64_t                file_count;
-        int64_t                dir_count;
-        int8_t                 dirty;
-        gf_boolean_t           create_status;
-        gf_boolean_t           updation_status;
-        gf_boolean_t           dirty_status;
-        gf_lock_t              lock;
-        struct list_head       contribution_head;
+    int64_t size;
+    int64_t file_count;
+    int64_t dir_count;
+    int8_t dirty;
+    gf_boolean_t create_status;
+    gf_boolean_t updation_status;
+    gf_boolean_t dirty_status;
+    gf_lock_t lock;
+    struct list_head contribution_head;
 };
 typedef struct quota_inode_ctx quota_inode_ctx_t;
 
 struct quota_synctask {
-        xlator_t      *this;
-        loc_t          loc;
-        quota_meta_t   contri;
-        gf_boolean_t   is_static;
-        uint32_t       ia_nlink;
-        call_stub_t   *stub;
+    xlator_t *this;
+    loc_t loc;
+    quota_meta_t contri;
+    gf_boolean_t is_static;
+    uint32_t ia_nlink;
+    call_stub_t *stub;
 };
 typedef struct quota_synctask quota_synctask_t;
 
 struct inode_contribution {
-        struct list_head contri_list;
-        int64_t          contribution;
-        int64_t          file_count;
-        int64_t          dir_count;
-        uuid_t           gfid;
-        gf_lock_t        lock;
-        GF_REF_DECL;
+    struct list_head contri_list;
+    int64_t contribution;
+    int64_t file_count;
+    int64_t dir_count;
+    uuid_t gfid;
+    gf_lock_t lock;
+    GF_REF_DECL;
 };
 typedef struct inode_contribution inode_contribution_t;
 
 int32_t
-mq_req_xattr (xlator_t *, loc_t *, dict_t *, char *, char *);
+mq_req_xattr(xlator_t *, loc_t *, dict_t *, char *, char *);
 
 int32_t
-mq_xattr_state (xlator_t *, loc_t *, dict_t *, struct iatt);
+mq_xattr_state(xlator_t *, loc_t *, dict_t *, struct iatt *);
 
 int
-mq_initiate_quota_txn (xlator_t *, loc_t *, struct iatt *);
+mq_initiate_quota_txn(xlator_t *, loc_t *, struct iatt *);
 
 int
-mq_initiate_quota_blocking_txn (xlator_t *, loc_t *, struct iatt *);
+mq_initiate_quota_blocking_txn(xlator_t *, loc_t *, struct iatt *);
 
 int
-mq_create_xattrs_txn (xlator_t *this, loc_t *loc, struct iatt *buf);
+mq_create_xattrs_txn(xlator_t *this, loc_t *loc, struct iatt *buf);
 
 int32_t
-mq_reduce_parent_size_txn (xlator_t *, loc_t *, quota_meta_t *,
-                           uint32_t nlink, call_stub_t *stub);
+mq_reduce_parent_size_txn(xlator_t *, loc_t *, quota_meta_t *, uint32_t nlink,
+                          call_stub_t *stub);
 
 int32_t
-mq_forget (xlator_t *, quota_inode_ctx_t *);
+mq_forget(xlator_t *, quota_inode_ctx_t *);
 #endif
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index e0e7c9857e6..1375ccc498c 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -7,34 +7,34 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "libxlator.h"
 #include "marker.h"
 #include "marker-mem-types.h"
 #include "marker-quota.h"
 #include "marker-quota-helper.h"
 #include "marker-common.h"
-#include "byte-order.h"
-#include "syncop.h"
-#include "syscall.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syscall.h>
 
 #include <fnmatch.h>
 
 #define _GF_UID_GID_CHANGED 1
 
 static char *mq_ext_xattrs[] = {
-        QUOTA_SIZE_KEY,
-        QUOTA_LIMIT_KEY,
-        QUOTA_LIMIT_OBJECTS_KEY,
-        NULL,
+    QUOTA_SIZE_KEY,
+    QUOTA_LIMIT_KEY,
+    QUOTA_LIMIT_OBJECTS_KEY,
+    NULL,
 };
 
 void
-fini (xlator_t *this);
+fini(xlator_t *this);
 
 int32_t
-marker_start_setxattr (call_frame_t *, xlator_t *);
+marker_start_setxattr(call_frame_t *, xlator_t *);
 
 /* When client/quotad request for quota xattrs,
  * replace the key-name by adding the version number
@@ -46,1574 +46,1566 @@ marker_start_setxattr (call_frame_t *, xlator_t *);
  * version for the key-name
  */
 int
-marker_key_replace_with_ver (xlator_t *this, dict_t *dict)
+marker_key_replace_with_ver(xlator_t *this, dict_t *dict)
 {
-        int                ret                     = -1;
-        int                i                       = 0;
-        marker_conf_t     *priv                    = NULL;
-        char               key[QUOTA_KEY_MAX]      = {0, };
+    int ret = -1;
+    int i = 0;
+    marker_conf_t *priv = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
 
-        priv = this->private;
+    priv = this->private;
 
-        if (dict == NULL || priv->version <= 0) {
-                ret = 0;
+    if (dict == NULL || priv->version <= 0) {
+        ret = 0;
+        goto out;
+    }
+
+    for (i = 0; mq_ext_xattrs[i]; i++) {
+        if (dict_get(dict, mq_ext_xattrs[i])) {
+            GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+            if (ret < 0)
                 goto out;
-        }
 
-        for (i = 0; mq_ext_xattrs[i]; i++) {
-                if (dict_get (dict, mq_ext_xattrs[i])) {
-                        GET_QUOTA_KEY (this, key, mq_ext_xattrs[i], ret);
-                        if (ret < 0)
-                                goto out;
-
-                        ret = dict_set (dict, key,
-                                        dict_get (dict, mq_ext_xattrs[i]));
-                        if (ret < 0)
-                                goto out;
-
-                        dict_del (dict, mq_ext_xattrs[i]);
-                }
+            ret = dict_set(dict, key, dict_get(dict, mq_ext_xattrs[i]));
+            if (ret < 0)
+                goto out;
+
+            dict_del(dict, mq_ext_xattrs[i]);
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-marker_key_set_ver (xlator_t *this, dict_t *dict)
+marker_key_set_ver(xlator_t *this, dict_t *dict)
 {
-        int              ret                     = -1;
-        int              i                       = -1;
-        marker_conf_t   *priv                    = NULL;
-        char             key[QUOTA_KEY_MAX]      = {0, };
+    int ret = -1;
+    int i = -1;
+    marker_conf_t *priv = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
 
-        priv = this->private;
+    priv = this->private;
 
-        if (dict == NULL || priv->version <= 0) {
-                ret = 0;
-                goto out;
-        }
+    if (dict == NULL || priv->version <= 0) {
+        ret = 0;
+        goto out;
+    }
 
-        for (i = 0; mq_ext_xattrs[i]; i++) {
-                GET_QUOTA_KEY (this, key, mq_ext_xattrs[i], ret);
-                if (ret < 0)
-                        goto out;
+    for (i = 0; mq_ext_xattrs[i]; i++) {
+        GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+        if (ret < 0)
+            goto out;
 
-                if (dict_get (dict, key))
-                        dict_set (dict, mq_ext_xattrs[i], dict_get (dict, key));
-        }
+        if (dict_get(dict, key))
+            dict_set(dict, mq_ext_xattrs[i], dict_get(dict, key));
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 marker_local_t *
-marker_local_ref (marker_local_t *local)
+marker_local_ref(marker_local_t *local)
 {
-        GF_VALIDATE_OR_GOTO ("marker", local, err);
+    GF_VALIDATE_OR_GOTO("marker", local, err);
 
-        LOCK (&local->lock);
-        {
-                local->ref++;
-        }
-        UNLOCK (&local->lock);
+    LOCK(&local->lock);
+    {
+        local->ref++;
+    }
+    UNLOCK(&local->lock);
 
-        return local;
+    return local;
 err:
-        return NULL;
+    return NULL;
 }
 
 int
-marker_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+marker_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!loc)
-                return ret;
+    if (!loc)
+        return ret;
 
-        if (inode) {
-                loc->inode = inode_ref (inode);
-                if (gf_uuid_is_null (loc->gfid)) {
-                        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-                }
+    if (inode) {
+        loc->inode = inode_ref(inode);
+        if (gf_uuid_is_null(loc->gfid)) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
         }
+    }
 
-        if (parent)
-                loc->parent = inode_ref (parent);
-
-        if (path) {
-                loc->path = gf_strdup (path);
-                if (!loc->path) {
-                        gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
-                        goto loc_wipe;
-                }
-
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name)
-                        loc->name++;
+    if (parent)
+        loc->parent = inode_ref(parent);
+
+    if (path) {
+        loc->path = gf_strdup(path);
+        if (!loc->path) {
+            gf_log("loc fill", GF_LOG_ERROR, "strdup failed");
+            goto loc_wipe;
         }
 
-        ret = 0;
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name)
+            loc->name++;
+    }
+
+    ret = 0;
 loc_wipe:
-        if (ret < 0)
-                loc_wipe (loc);
+    if (ret < 0)
+        loc_wipe(loc);
 
-        return ret;
+    return ret;
 }
 
 int
-_marker_inode_loc_fill (inode_t *inode, inode_t *parent, char *name, loc_t *loc)
+_marker_inode_loc_fill(inode_t *inode, inode_t *parent, char *name, loc_t *loc)
 {
-        char            *resolvedpath = NULL;
-        int              ret          = -1;
-        gf_boolean_t     free_parent  = _gf_false;
+    char *resolvedpath = NULL;
+    int ret = -1;
+    gf_boolean_t free_parent = _gf_false;
 
-        if ((!inode) || (!loc))
-                return ret;
+    if ((!inode) || (!loc))
+        return ret;
 
-        if (parent && name)
-                ret = inode_path (parent, name, &resolvedpath);
-        else
-                ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0)
-                goto err;
+    if (parent && name)
+        ret = inode_path(parent, name, &resolvedpath);
+    else
+        ret = inode_path(inode, NULL, &resolvedpath);
+    if (ret < 0)
+        goto err;
 
-        if (parent == NULL) {
-	        parent = inode_parent (inode, NULL, NULL);
-                free_parent = _gf_true;
-        }
+    if (parent == NULL) {
+        parent = inode_parent(inode, NULL, NULL);
+        free_parent = _gf_true;
+    }
 
-        ret = marker_loc_fill (loc, inode, parent, resolvedpath);
-        if (ret < 0)
-                goto err;
+    ret = marker_loc_fill(loc, inode, parent, resolvedpath);
+    if (ret < 0)
+        goto err;
 
 err:
-	if (free_parent)
-		inode_unref (parent);
+    if (free_parent)
+        inode_unref(parent);
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
 }
 
 int
-marker_inode_loc_fill (inode_t *inode, loc_t *loc)
+marker_inode_loc_fill(inode_t *inode, loc_t *loc)
 {
-        return _marker_inode_loc_fill (inode, NULL, NULL, loc);
+    return _marker_inode_loc_fill(inode, NULL, NULL, loc);
 }
 
 int32_t
-marker_trav_parent (marker_local_t *local)
+marker_trav_parent(marker_local_t *local)
 {
-        int32_t ret = 0;
-        loc_t   loc = {0, };
-        inode_t *parent = NULL;
-        int8_t  need_unref = 0;
+    int32_t ret = 0;
+    loc_t loc = {
+        0,
+    };
+    inode_t *parent = NULL;
+    int8_t need_unref = 0;
 
-        if (!local->loc.parent) {
-                parent = inode_parent (local->loc.inode, NULL, NULL);
-                if (parent)
-                        need_unref = 1;
-        } else
-                parent = local->loc.parent;
+    if (!local->loc.parent) {
+        parent = inode_parent(local->loc.inode, NULL, NULL);
+        if (parent)
+            need_unref = 1;
+    } else
+        parent = local->loc.parent;
 
-        ret = marker_inode_loc_fill (parent, &loc);
+    ret = marker_inode_loc_fill(parent, &loc);
 
-        if (ret < 0) {
-                ret = -1;
-                goto out;
-        }
+    if (ret < 0) {
+        ret = -1;
+        goto out;
+    }
 
-        loc_wipe (&local->loc);
+    loc_wipe(&local->loc);
 
-        local->loc = loc;
+    local->loc = loc;
 out:
-        if (need_unref)
-                inode_unref (parent);
+    if (need_unref)
+        inode_unref(parent);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-marker_error_handler (xlator_t *this, marker_local_t *local, int32_t op_errno)
+void
+marker_error_handler(xlator_t *this, marker_local_t *local, int32_t op_errno)
 {
-         marker_conf_t *priv = NULL;
-         const char    *path = NULL;
-
-         priv = (marker_conf_t *) this->private;
-         path = local
-                 ? (local->loc.path
-                    ? local->loc.path : uuid_utoa(local->loc.gfid))
-                 : "<nul>";
-
-         gf_log (this->name, GF_LOG_CRITICAL,
-                 "Indexing gone corrupt at %s (reason: %s)."
-                 " Geo-replication slave content needs to be revalidated",
-                 path, strerror (op_errno));
-        sys_unlink (priv->timestamp_file);
+    marker_conf_t *priv = (marker_conf_t *)this->private;
+    const char *path = local ? ((local->loc.path) ? local->loc.path
+                                                  : uuid_utoa(local->loc.gfid))
+                             : "<nul>";
 
-        return 0;
+    gf_log(this->name, GF_LOG_CRITICAL,
+           "Indexing gone corrupt at %s (reason: %s)."
+           " Geo-replication slave content needs to be revalidated",
+           path, strerror(op_errno));
+    sys_unlink(priv->timestamp_file);
 }
 
 int32_t
-marker_local_unref (marker_local_t *local)
+marker_local_unref(marker_local_t *local)
 {
-        int32_t   var = 0;
-
-        if (local == NULL)
-                return -1;
+    int32_t var = 0;
 
-        LOCK (&local->lock);
-        {
-                var = --local->ref;
-        }
-        UNLOCK (&local->lock);
-
-        if (var != 0)
-                goto out;
+    if (local == NULL)
+        return -1;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->parent_loc);
-        if (local->xdata)
-                dict_unref (local->xdata);
+    LOCK(&local->lock);
+    {
+        var = --local->ref;
+    }
+    UNLOCK(&local->lock);
 
-        if (local->lk_frame) {
-                STACK_DESTROY (local->lk_frame->root);
-                local->lk_frame = NULL;
-        }
+    if (var != 0)
+        goto out;
 
-        if (local->oplocal) {
-                marker_local_unref (local->oplocal);
-                local->oplocal = NULL;
-        }
-        mem_put (local);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->parent_loc);
+    if (local->xdata)
+        dict_unref(local->xdata);
+
+    if (local->lk_frame) {
+        STACK_DESTROY(local->lk_frame->root);
+        local->lk_frame = NULL;
+    }
+
+    if (local->oplocal) {
+        marker_local_unref(local->oplocal);
+        local->oplocal = NULL;
+    }
+    mem_put(local);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-stat_stampfile (xlator_t *this, marker_conf_t *priv,
-                struct volume_mark **status)
+stat_stampfile(xlator_t *this, marker_conf_t *priv, struct volume_mark **status)
 {
-        struct stat          buf      = {0, };
-        struct volume_mark  *vol_mark = NULL;
+    struct stat buf = {
+        0,
+    };
+    struct volume_mark *vol_mark = NULL;
 
-        vol_mark = GF_CALLOC (sizeof (struct volume_mark), 1,
-                              gf_marker_mt_volume_mark);
+    vol_mark = GF_CALLOC(sizeof(struct volume_mark), 1,
+                         gf_marker_mt_volume_mark);
 
-        vol_mark->major = 1;
-        vol_mark->minor = 0;
+    vol_mark->major = 1;
+    vol_mark->minor = 0;
 
-        GF_ASSERT (sizeof (priv->volume_uuid_bin) == 16);
-        memcpy (vol_mark->uuid, priv->volume_uuid_bin, 16);
+    GF_ASSERT(sizeof(priv->volume_uuid_bin) == 16);
+    memcpy(vol_mark->uuid, priv->volume_uuid_bin, 16);
 
-        if (sys_stat (priv->timestamp_file, &buf) != -1) {
-                vol_mark->retval = 0;
-                vol_mark->sec = htonl (buf.st_mtime);
-                vol_mark->usec = htonl (ST_MTIM_NSEC (&buf)/1000);
-        } else
-                vol_mark->retval = 1;
+    if (sys_stat(priv->timestamp_file, &buf) != -1) {
+        vol_mark->retval = 0;
+        vol_mark->sec = htonl(buf.st_mtime);
+        vol_mark->usec = htonl(ST_MTIM_NSEC(&buf) / 1000);
+    } else
+        vol_mark->retval = 1;
 
-        *status = vol_mark;
+    *status = vol_mark;
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_getxattr_stampfile_cbk (call_frame_t *frame, xlator_t *this,
-                               const char *name, struct volume_mark *vol_mark,
-                               dict_t *xdata)
+marker_getxattr_stampfile_cbk(call_frame_t *frame, xlator_t *this,
+                              const char *name, struct volume_mark *vol_mark,
+                              dict_t *xdata)
 {
-        int32_t   ret  = -1;
-        dict_t   *dict = NULL;
+    int32_t ret = -1;
+    dict_t *dict = NULL;
 
-        if (vol_mark == NULL){
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+    if (vol_mark == NULL) {
+        STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
 
-                goto out;
-        }
+        goto out;
+    }
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        ret = dict_set_bin (dict, (char *)name, vol_mark,
-                            sizeof (struct volume_mark));
-        if (ret) {
-                GF_FREE (vol_mark);
-                gf_log (this->name, GF_LOG_WARNING, "failed to set key %s",
-                        name);
-        }
+    ret = dict_set_bin(dict, (char *)name, vol_mark,
+                       sizeof(struct volume_mark));
+    if (ret) {
+        GF_FREE(vol_mark);
+        gf_log(this->name, GF_LOG_WARNING, "failed to set key %s", name);
+    }
 
-        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, dict, xdata);
+    STACK_UNWIND_STRICT(getxattr, frame, 0, 0, dict, xdata);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 out:
-        return 0;
+    return 0;
 }
 
 gf_boolean_t
-call_from_special_client (call_frame_t *frame, xlator_t *this, const char *name)
+call_from_special_client(call_frame_t *frame, xlator_t *this, const char *name)
 {
-        struct volume_mark     *vol_mark   = NULL;
-        marker_conf_t          *priv       = NULL;
-        gf_boolean_t           is_true     = _gf_true;
+    struct volume_mark *vol_mark = NULL;
+    marker_conf_t *priv = NULL;
+    gf_boolean_t is_true = _gf_true;
 
-        priv = (marker_conf_t *)this->private;
+    priv = (marker_conf_t *)this->private;
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD || name == NULL ||
-            strcmp (name, MARKER_XATTR_PREFIX "." VOLUME_MARK) != 0) {
-                is_true = _gf_false;
-                goto out;
-        }
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD || name == NULL ||
+        strcmp(name, MARKER_XATTR_PREFIX "." VOLUME_MARK) != 0) {
+        is_true = _gf_false;
+        goto out;
+    }
 
-        stat_stampfile (this, priv, &vol_mark);
+    stat_stampfile(this, priv, &vol_mark);
 
-        marker_getxattr_stampfile_cbk (frame, this, name, vol_mark, NULL);
+    marker_getxattr_stampfile_cbk(frame, this, name, vol_mark, NULL);
 out:
-        return is_true;
+    return is_true;
 }
 
 static gf_boolean_t
-_is_quota_internal_xattr (dict_t *d, char *k, data_t *v, void *data)
+_is_quota_internal_xattr(dict_t *d, char *k, data_t *v, void *data)
 {
-        int     i = 0;
-        char    **external_xattrs = data;
+    int i = 0;
+    char **external_xattrs = data;
 
-        for (i = 0; external_xattrs && external_xattrs[i]; i++) {
-                if (strcmp (k, external_xattrs[i]) == 0)
-                        return _gf_false;
-        }
+    for (i = 0; external_xattrs && external_xattrs[i]; i++) {
+        if (strcmp(k, external_xattrs[i]) == 0)
+            return _gf_false;
+    }
 
-        if (fnmatch ("trusted.glusterfs.quota*", k, 0) == 0)
-                return _gf_true;
+    if (fnmatch("trusted.glusterfs.quota*", k, 0) == 0)
+        return _gf_true;
 
-        /* It would be nice if posix filters pgfid xattrs. But since marker
-         * also takes up responsibility to clean these up, adding the filtering
-         * here (Check 'quota_xattr_cleaner')
-         */
-        if (fnmatch (PGFID_XATTR_KEY_PREFIX"*", k, 0) == 0)
-                return _gf_true;
+    /* It would be nice if posix filters pgfid xattrs. But since marker
+     * also takes up responsibility to clean these up, adding the filtering
+     * here (Check 'quota_xattr_cleaner')
+     */
+    if (fnmatch(PGFID_XATTR_KEY_PREFIX "*", k, 0) == 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static void
-marker_filter_internal_xattrs (xlator_t *this, dict_t *xattrs)
+marker_filter_internal_xattrs(xlator_t *this, dict_t *xattrs)
 {
-        marker_conf_t *priv   = NULL;
-        char         **ext    = NULL;
+    marker_conf_t *priv = NULL;
+    char **ext = NULL;
 
-        priv = this->private;
-        if (priv->feature_enabled & GF_QUOTA)
-                ext = mq_ext_xattrs;
+    priv = this->private;
+    if (priv->feature_enabled & GF_QUOTA)
+        ext = mq_ext_xattrs;
 
-        dict_foreach_match (xattrs, _is_quota_internal_xattr, ext,
-                            dict_remove_foreach_fn, NULL);
+    dict_foreach_match(xattrs, _is_quota_internal_xattr, ext,
+                       dict_remove_foreach_fn, NULL);
 }
 
 static void
-marker_filter_gsyncd_xattrs (call_frame_t *frame,
-                               xlator_t *this, dict_t *xattrs)
+marker_filter_gsyncd_xattrs(call_frame_t *frame, xlator_t *this, dict_t *xattrs)
 {
-        marker_conf_t *priv   = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (frame);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(frame);
 
-        if (xattrs && frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                GF_REMOVE_INTERNAL_XATTR (GF_XATTR_XTIME_PATTERN, xattrs);
-        }
-        return;
+    if (xattrs && frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        GF_REMOVE_INTERNAL_XATTR(GF_XATTR_XTIME_PATTERN, xattrs);
+    }
+    return;
 }
 
 int32_t
-marker_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
-{
-        int32_t     ret  = -1;
-        if (op_ret < 0)
-                goto unwind;
-
-        ret = marker_key_set_ver (this, dict);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        if (cookie) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Filtering the quota extended attributes");
-
-                /* If the getxattr is from a non special client, then do not
-                   copy the quota related xattrs (except the quota limit key
-                   i.e trusted.glusterfs.quota.limit-set which has been set by
-                   glusterd on the directory on which quota limit is set.) for
-                   directories. Let the healing of xattrs happen upon lookup.
-                   NOTE: setting of trusted.glusterfs.quota.limit-set as of now
-                   happens from glusterd. It should be moved to quotad. Also
-                   trusted.glusterfs.quota.limit-set is set on directory which
-                   is permanent till quota is removed on that directory or limit
-                   is changed. So let that xattr be healed by other xlators
-                   properly whenever directory healing is done.
-                */
-                /*
-                 * Except limit-set xattr, rest of the xattrs are maintained
-                 * by quota xlator. Don't expose them to other xlators.
-                 * This filter makes sure quota xattrs are not healed as part of
-                 * metadata self-heal
-                 */
-                marker_filter_internal_xattrs (frame->this, dict);
-        }
+marker_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+    int32_t ret = -1;
+    if (op_ret < 0)
+        goto unwind;
+
+    ret = marker_key_set_ver(this, dict);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (cookie) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Filtering the quota extended attributes");
+
+        /* If the getxattr is from a non special client, then do not
+           copy the quota related xattrs (except the quota limit key
+           i.e trusted.glusterfs.quota.limit-set which has been set by
+           glusterd on the directory on which quota limit is set.) for
+           directories. Let the healing of xattrs happen upon lookup.
+           NOTE: setting of trusted.glusterfs.quota.limit-set as of now
+           happens from glusterd. It should be moved to quotad. Also
+           trusted.glusterfs.quota.limit-set is set on directory which
+           is permanent till quota is removed on that directory or limit
+           is changed. So let that xattr be healed by other xlators
+           properly whenever directory healing is done.
+        */
+        /*
+         * Except limit-set xattr, rest of the xattrs are maintained
+         * by quota xlator. Don't expose them to other xlators.
+         * This filter makes sure quota xattrs are not healed as part of
+         * metadata self-heal
+         */
+        marker_filter_internal_xattrs(frame->this, dict);
+    }
 
-        /* Filter gsyncd xtime xattr for non gsyncd clients */
-        marker_filter_gsyncd_xattrs (frame, frame->this, dict);
+    /* Filter gsyncd xtime xattr for non gsyncd clients */
+    marker_filter_gsyncd_xattrs(frame, frame->this, dict);
 
 unwind:
-        MARKER_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    MARKER_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-marker_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name, dict_t *xdata)
-{
-        gf_boolean_t     is_true  = _gf_false;
-        marker_conf_t   *priv                       = NULL;
-        unsigned long    cookie                     = 0;
-        marker_local_t  *local                      = NULL;
-        char             key[QUOTA_KEY_MAX]         = {0, };
-        int32_t          ret                        = -1;
-        int32_t          i                          = 0;
-
-        priv = this->private;
-
-        if (name) {
-                for (i = 0; mq_ext_xattrs[i]; i++) {
-                        if (strcmp (name, mq_ext_xattrs[i]))
-                                continue;
-
-                        GET_QUOTA_KEY (this, key, mq_ext_xattrs[i], ret);
-                        if (ret < 0)
-                                goto out;
-                        name = key;
-                        break;
-                }
-        }
+marker_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+    gf_boolean_t is_true = _gf_false;
+    marker_conf_t *priv = NULL;
+    unsigned long cookie = 0;
+    marker_local_t *local = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int32_t ret = -1;
+    int32_t i = 0;
+
+    priv = this->private;
+
+    if (name) {
+        for (i = 0; mq_ext_xattrs[i]; i++) {
+            if (strcmp(name, mq_ext_xattrs[i]))
+                continue;
 
-        frame->local = mem_get0 (this->local_pool);
-        local = frame->local;
-        if (local == NULL)
+            GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+            if (ret < 0)
                 goto out;
+            name = key;
+            break;
+        }
+    }
+
+    frame->local = mem_get0(this->local_pool);
+    local = frame->local;
+    if (local == NULL)
+        goto out;
+
+    MARKER_INIT_LOCAL(frame, local);
+
+    if ((loc_copy(&local->loc, loc)) < 0)
+        goto out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "USER:PID = %d", frame->root->pid);
+
+    if (priv && priv->feature_enabled & GF_XTIME)
+        is_true = call_from_special_client(frame, this, name);
 
-        MARKER_INIT_LOCAL (frame, local);
-
-        if ((loc_copy (&local->loc, loc)) < 0)
-		goto out;
-
-        gf_log (this->name, GF_LOG_DEBUG, "USER:PID = %d", frame->root->pid);
-
-        if (priv && priv->feature_enabled & GF_XTIME)
-                is_true = call_from_special_client (frame, this, name);
-
-        if (is_true == _gf_false) {
-                if (name == NULL) {
-                        /* Signifies that marker translator
-                         * has to filter the quota's xattr's,
-                         * this is to prevent afr from performing
-                         * self healing on marker-quota xattrs'
-                         */
-                        cookie = 1;
-                }
-                STACK_WIND_COOKIE (frame, marker_getxattr_cbk,
-				   (void *)cookie,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->getxattr,
-				   loc, name, xdata);
+    if (is_true == _gf_false) {
+        if (name == NULL) {
+            /* Signifies that marker translator
+             * has to filter the quota's xattr's,
+             * this is to prevent afr from performing
+             * self healing on marker-quota xattrs'
+             */
+            cookie = 1;
         }
+        STACK_WIND_COOKIE(frame, marker_getxattr_cbk, (void *)cookie,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
+                          loc, name, xdata);
+    }
 
-        return 0;
+    return 0;
 out:
-        MARKER_STACK_UNWIND (getxattr, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    MARKER_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
 int32_t
-marker_setxattr_done (call_frame_t *frame)
+marker_setxattr_done(call_frame_t *frame)
 {
-        marker_local_t *local = NULL;
+    marker_local_t *local = NULL;
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_DESTROY (frame->root);
+    STACK_DESTROY(frame->root);
 
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_specific_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_specific_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int32_t         ret   = 0;
-        int32_t         done  = 0;
-        marker_local_t *local = NULL;
+    int32_t ret = 0;
+    int32_t done = 1;
+    marker_local_t *local = NULL;
 
-        local = (marker_local_t*) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        if (op_ret == -1 && op_errno == ENOSPC) {
-                marker_error_handler (this, local, op_errno);
-                done = 1;
-                goto out;
-        }
+    if (op_ret == -1 && op_errno == ENOSPC) {
+        marker_error_handler(this, local, op_errno);
+        goto out;
+    }
 
-        if (local) {
-                if (local->loc.path && strcmp (local->loc.path, "/") == 0) {
-                        done = 1;
-                        goto out;
-                }
-                if (__is_root_gfid (local->loc.gfid)) {
-                        done = 1;
-                        goto out;
-                }
+    if (local) {
+        if (local->loc.path && strcmp(local->loc.path, "/") == 0) {
+            goto out;
         }
-
-        ret = (local) ? marker_trav_parent (local) : -1;
-
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "Error occurred "
-                        "while traversing to the parent, stopping marker");
-
-                done = 1;
-
-                goto out;
+        if (__is_root_gfid(local->loc.gfid)) {
+            goto out;
         }
+    }
 
-        marker_start_setxattr (frame, this);
+    ret = (local) ? marker_trav_parent(local) : -1;
 
+    if (ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Error occurred "
+               "while traversing to the parent, stopping marker");
+        goto out;
+    }
+
+    marker_start_setxattr(frame, this);
+    done = 0;
 out:
-        if (done) {
-                marker_setxattr_done (frame);
-        }
+    if (done) {
+        marker_setxattr_done(frame);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_start_setxattr (call_frame_t *frame, xlator_t *this)
+marker_start_setxattr(call_frame_t *frame, xlator_t *this)
 {
-        int32_t          ret   = -1;
-        dict_t          *dict  = NULL;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        local = (marker_local_t*) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict)
-                goto out;
+    if (!dict)
+        goto out;
 
-        if (local->loc.inode && gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+    if (local->loc.inode && gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, local->loc.inode->gfid);
 
-        GF_UUID_ASSERT (local->loc.gfid);
+    GF_UUID_ASSERT(local->loc.gfid);
 
-        ret = dict_set_static_bin (dict, priv->marker_xattr,
-                                   (void *)local->timebuf, 8);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set marker xattr (%s)", local->loc.path);
-                goto out;
-        }
+    ret = dict_set_static_bin(dict, priv->marker_xattr, (void *)local->timebuf,
+                              8);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING, "failed to set marker xattr (%s)",
+               local->loc.path);
+        goto out;
+    }
 
-        STACK_WIND (frame, marker_specific_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0,
-                    NULL);
+    STACK_WIND(frame, marker_specific_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0, NULL);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 void
-marker_gettimeofday (marker_local_t *local)
+marker_gettimeofday(marker_local_t *local)
 {
-        struct timeval tv = {0, };
+    struct timeval tv = {
+        0,
+    };
 
-        gettimeofday (&tv, NULL);
+    gettimeofday(&tv, NULL);
 
-        local->timebuf [0] = htonl (tv.tv_sec);
-        local->timebuf [1] = htonl (tv.tv_usec);
+    local->timebuf[0] = htonl(tv.tv_sec);
+    local->timebuf[1] = htonl(tv.tv_usec);
 
-        return;
+    return;
 }
 
 int32_t
-marker_create_frame (xlator_t *this, marker_local_t *local)
+marker_create_frame(xlator_t *this, marker_local_t *local)
 {
-        call_frame_t *frame = NULL;
+    call_frame_t *frame = NULL;
 
-        frame = create_frame (this, this->ctx->pool);
+    frame = create_frame(this, this->ctx->pool);
 
-        if (!frame)
-                return -1;
+    if (!frame)
+        return -1;
 
-        frame->local = (void *) local;
+    frame->local = (void *)local;
 
-        marker_start_setxattr (frame, this);
+    marker_start_setxattr(frame, this);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_xtime_update_marks (xlator_t *this, marker_local_t *local)
+marker_xtime_update_marks(xlator_t *this, marker_local_t *local)
 {
-        marker_conf_t *priv = NULL;
+    marker_conf_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        if ((local->pid == GF_CLIENT_PID_GSYNCD
-             && !(priv->feature_enabled & GF_XTIME_GSYNC_FORCE))
-            || (local->pid == GF_CLIENT_PID_DEFRAG))
-                goto out;
+    if ((local->pid == GF_CLIENT_PID_GSYNCD &&
+         !(priv->feature_enabled & GF_XTIME_GSYNC_FORCE)) ||
+        (local->pid == GF_CLIENT_PID_DEFRAG))
+        goto out;
 
-        marker_gettimeofday (local);
+    marker_gettimeofday(local);
 
-        marker_local_ref (local);
+    marker_local_ref(local);
 
-        marker_create_frame (this, local);
+    marker_create_frame(this, local);
 out:
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+marker_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
-        quota_inode_ctx_t  *ctx     = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "while creating directory %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "while creating directory %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
-        priv = this->private;
-
-        if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
-                ctx = mq_inode_ctx_new (inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s", uuid_utoa (inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+    frame->local = NULL;
+    priv = this->private;
+
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
+    }
 
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_create_xattrs_txn (this, &local->loc, NULL);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_create_xattrs_txn(this, &local->loc, NULL);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              mode_t umask, dict_t *xdata)
+marker_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    STACK_WIND(frame, marker_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 err:
-        MARKER_STACK_UNWIND (mkdir, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(mkdir, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+marker_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
-        quota_inode_ctx_t  *ctx     = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "while creating file %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "while creating file %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
-        priv = this->private;
-
-        if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
-                ctx = mq_inode_ctx_new (inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s", uuid_utoa (inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+    frame->local = NULL;
+    priv = this->private;
+
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
+    }
 
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_create_xattrs_txn (this, &local->loc, buf);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_create_xattrs_txn(this, &local->loc, buf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+marker_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, marker_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                             NULL, NULL);
+    MARKER_STACK_UNWIND(create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+marker_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "while write, %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "while write, %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc, postbuf);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_writev (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               struct iovec *vector,
-               int32_t count,
-               off_t offset, uint32_t flags,
-               struct iobref *iobref, dict_t *xdata)
+marker_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+              struct iobref *iobref, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
-                    flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, marker_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(writev, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+marker_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
-        call_stub_t        *stub    = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "rmdir %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "rmdir %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
-        priv = this->private;
+    frame->local = NULL;
+    priv = this->private;
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                /* If a 'rm -rf' is performed by a client, rmdir can be faster
-                   than marker background mq_reduce_parent_size_txn.
-                   In this case, as part of rmdir parent child association
-                   will be removed in the server protocol.
-                   This can lead to mq_reduce_parent_size_txn failures.
-
-                   So perform mq_reduce_parent_size_txn in foreground
-                   and unwind to server once txn is complete
-                 */
-
-                stub = fop_rmdir_cbk_stub (frame, default_rmdir_cbk, op_ret,
-                                           op_errno, preparent, postparent,
-                                           xdata);
-                mq_reduce_parent_size_txn (this, &local->loc, NULL, 1, stub);
-
-                if (stub) {
-                        marker_local_unref (local);
-                        return 0;
-                }
+    if (priv->feature_enabled & GF_QUOTA) {
+        /* If a 'rm -rf' is performed by a client, rmdir can be faster
+           than marker background mq_reduce_parent_size_txn.
+           In this case, as part of rmdir parent child association
+           will be removed in the server protocol.
+           This can lead to mq_reduce_parent_size_txn failures.
+
+           So perform mq_reduce_parent_size_txn in foreground
+           and unwind to server once txn is complete
+         */
+
+        stub = fop_rmdir_cbk_stub(frame, default_rmdir_cbk, op_ret, op_errno,
+                                  preparent, postparent, xdata);
+        mq_reduce_parent_size_txn(this, &local->loc, NULL, 1, stub);
+
+        if (stub) {
+            marker_local_unref(local);
+            return 0;
         }
+    }
 
 out:
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
 
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-              dict_t *xdata)
+marker_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
-        return 0;
+    STACK_WIND(frame, marker_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+marker_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
-        uint32_t            nlink   = -1;
-        GF_UNUSED int32_t   ret     = 0;
-        call_stub_t        *stub    = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    uint32_t nlink = -1;
+    GF_UNUSED int32_t ret = 0;
+    call_stub_t *stub = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s occurred in unlink", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred in unlink",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
-        priv = this->private;
+    frame->local = NULL;
+    priv = this->private;
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                if (local->skip_txn)
-                        goto out;
-
-                if (xdata) {
-                        ret = dict_get_uint32 (xdata,
-                                GF_RESPONSE_LINK_COUNT_XDATA, &nlink);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "dict get failed %s ",
-                                        strerror (-ret));
-                        }
-                }
-
-                /* If a 'rm -rf' is performed by a client, unlink can be faster
-                   than marker background mq_reduce_parent_size_txn.
-                   In this case, as part of unlink parent child association
-                   will be removed in the server protocol.
-                   This can lead to mq_reduce_parent_size_txn failures.
-
-                  So perform mq_reduce_parent_size_txn in foreground
-                  and unwind to server once txn is complete
-                */
-
-                stub = fop_unlink_cbk_stub (frame, default_unlink_cbk, op_ret,
-                                            op_errno, preparent, postparent,
-                                            xdata);
-                mq_reduce_parent_size_txn (this, &local->loc, NULL, nlink,
-                                           stub);
-
-                if (stub) {
-                        marker_local_unref (local);
-                        return 0;
-                }
+    if (priv->feature_enabled & GF_QUOTA) {
+        if (local->skip_txn)
+            goto out;
+
+        if (xdata) {
+            ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, &nlink);
+            if (ret) {
+                gf_log(this->name, GF_LOG_TRACE, "dict get failed %s ",
+                       strerror(-ret));
+            }
         }
 
+        /* If a 'rm -rf' is performed by a client, unlink can be faster
+           than marker background mq_reduce_parent_size_txn.
+           In this case, as part of unlink parent child association
+           will be removed in the server protocol.
+           This can lead to mq_reduce_parent_size_txn failures.
+
+          So perform mq_reduce_parent_size_txn in foreground
+          and unwind to server once txn is complete
+        */
+
+        stub = fop_unlink_cbk_stub(frame, default_unlink_cbk, op_ret, op_errno,
+                                   preparent, postparent, xdata);
+        mq_reduce_parent_size_txn(this, &local->loc, NULL, nlink, stub);
+
+        if (stub) {
+            marker_local_unref(local);
+            return 0;
+        }
+    }
+
 out:
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
 
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-               dict_t *xdata)
+marker_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
-        gf_boolean_t     dict_free = _gf_false;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    gf_boolean_t dict_free = _gf_false;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto unlink_wind;
+    if (priv->feature_enabled == 0)
+        goto unlink_wind;
 
-        local = mem_get0 (this->local_pool);
-        local->xflag = xflag;
-        if (xdata)
-                local->xdata = dict_ref (xdata);
-        MARKER_INIT_LOCAL (frame, local);
+    local = mem_get0(this->local_pool);
+    local->xflag = xflag;
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 
-        if (xdata && dict_get (xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY)) {
-                local->skip_txn = 1;
-                goto unlink_wind;
-        }
+    if (xdata && dict_get(xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY)) {
+        local->skip_txn = 1;
+        goto unlink_wind;
+    }
 
-        if (xdata == NULL) {
-                xdata = dict_new ();
-                dict_free = _gf_true;
-        }
+    if (xdata == NULL) {
+        xdata = dict_new();
+        dict_free = _gf_true;
+    }
 
-        ret = dict_set_int32 (xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
-        if (ret < 0)
-                goto err;
+    ret = dict_set_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
+    if (ret < 0)
+        goto err;
 
 unlink_wind:
-        STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        goto out;
+    STACK_WIND(frame, marker_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    goto out;
 
 err:
-        MARKER_STACK_UNWIND (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
 
 out:
-        if (dict_free)
-                dict_unref (xdata);
-        return 0;
+    if (dict_free)
+        dict_unref(xdata);
+    return 0;
 }
 
-
 int32_t
-marker_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+marker_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "linking a file ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "linking a file ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
-
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+    frame->local = NULL;
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    STACK_UNWIND_STRICT(link, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        priv = this->private;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                if (!local->skip_txn)
-                        mq_create_xattrs_txn (this, &local->loc, buf);
-        }
+    priv = this->private;
 
+    if (priv->feature_enabled & GF_QUOTA) {
+        if (!local->skip_txn)
+            mq_create_xattrs_txn(this, &local->loc, buf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-             dict_t *xdata)
+marker_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, newloc);
+    ret = loc_copy(&local->loc, newloc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 
-        if (xdata && dict_get (xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY))
-                local->skip_txn = 1;
+    if (xdata && dict_get(xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY))
+        local->skip_txn = 1;
 wind:
-        STACK_WIND (frame, marker_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, marker_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                             NULL);
+    MARKER_STACK_UNWIND(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_rename_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t *local  = NULL, *oplocal = NULL;
-        loc_t           newloc = {0, };
-        marker_conf_t  *priv   = NULL;
+    marker_local_t *local = NULL, *oplocal = NULL;
+    loc_t newloc = {
+        0,
+    };
+    marker_conf_t *priv = NULL;
 
-        local = frame->local;
-        oplocal = local->oplocal;
+    local = frame->local;
+    oplocal = local->oplocal;
 
-        priv = this->private;
+    priv = this->private;
 
-        frame->local = NULL;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "inodelk (UNLOCK) failed on path:%s (gfid:%s) (%s)",
-                        oplocal->parent_loc.path,
-                        uuid_utoa (oplocal->parent_loc.inode->gfid),
-                        strerror (op_errno));
-        }
-
-        if (local->err != 0)
-                goto err;
+    frame->local = NULL;
 
-        mq_reduce_parent_size_txn (this, &oplocal->loc, &oplocal->contribution,
-                                   -1, NULL);
-
-        if (local->loc.inode != NULL) {
-                /* If destination file exits before rename, it would have
-                 * been unlinked while renaming a file
-                 */
-                mq_reduce_parent_size_txn (this, &local->loc, NULL,
-                                           local->ia_nlink, NULL);
-        }
-
-        newloc.inode = inode_ref (oplocal->loc.inode);
-        newloc.path = gf_strdup (local->loc.path);
-        newloc.name = strrchr (newloc.path, '/');
-        if (newloc.name)
-                newloc.name++;
-        newloc.parent = inode_ref (local->loc.parent);
+    if (op_ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "inodelk (UNLOCK) failed on path:%s (gfid:%s) (%s)",
+               oplocal->parent_loc.path,
+               uuid_utoa(oplocal->parent_loc.inode->gfid), strerror(op_errno));
+    }
 
-        mq_create_xattrs_txn (this, &newloc, &local->buf);
+    if (local->err != 0)
+        goto err;
 
-        loc_wipe (&newloc);
+    mq_reduce_parent_size_txn(this, &oplocal->loc, &oplocal->contribution, -1,
+                              NULL);
 
-        if (priv->feature_enabled & GF_XTIME) {
-                //update marks on oldpath
-                gf_uuid_copy (local->loc.gfid, oplocal->loc.inode->gfid);
-                marker_xtime_update_marks (this, oplocal);
-                marker_xtime_update_marks (this, local);
-        }
+    if (local->loc.inode != NULL) {
+        /* If destination file exits before rename, it would have
+         * been unlinked while renaming a file
+         */
+        mq_reduce_parent_size_txn(this, &local->loc, NULL, local->ia_nlink,
+                                  NULL);
+    }
+
+    newloc.inode = inode_ref(oplocal->loc.inode);
+    newloc.path = gf_strdup(local->loc.path);
+    newloc.name = strrchr(newloc.path, '/');
+    if (newloc.name)
+        newloc.name++;
+    newloc.parent = inode_ref(local->loc.parent);
+
+    mq_create_xattrs_txn(this, &newloc, &local->buf);
+
+    loc_wipe(&newloc);
+
+    if (priv->feature_enabled & GF_XTIME) {
+        if (!local->loc.inode)
+            local->loc.inode = inode_ref(oplocal->loc.inode);
+        // update marks on oldpath
+        gf_uuid_copy(local->loc.gfid, oplocal->loc.inode->gfid);
+        marker_xtime_update_marks(this, oplocal);
+        marker_xtime_update_marks(this, local);
+    }
 
 err:
-        marker_local_unref (local);
-        marker_local_unref (oplocal);
+    marker_local_unref(local);
+    marker_local_unref(oplocal);
 
-        return 0;
+    return 0;
 }
 
-
 void
-marker_rename_release_oldp_lock (marker_local_t *local, xlator_t *this)
+marker_rename_release_oldp_lock(marker_local_t *local, xlator_t *this)
 {
-        marker_local_t        *oplocal  = NULL;
-        call_frame_t          *lk_frame = NULL;
-        struct gf_flock        lock     = {0, };
+    marker_local_t *oplocal = NULL;
+    call_frame_t *lk_frame = NULL;
+    struct gf_flock lock = {
+        0,
+    };
 
-        oplocal = local->oplocal;
-        lk_frame = local->lk_frame;
+    oplocal = local->oplocal;
+    lk_frame = local->lk_frame;
 
-        if (lk_frame == NULL)
-                goto err;
+    if (lk_frame == NULL)
+        goto err;
 
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
+    lock.l_type = F_UNLCK;
+    lock.l_whence = SEEK_SET;
+    lock.l_start = 0;
+    lock.l_len = 0;
+    lock.l_pid = 0;
 
-        STACK_WIND (lk_frame,
-                    marker_rename_done,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &oplocal->parent_loc, F_SETLKW, &lock, NULL);
+    STACK_WIND(lk_frame, marker_rename_done, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, this->name,
+               &oplocal->parent_loc, F_SETLKW, &lock, NULL);
 
-        return;
+    return;
 
 err:
-        marker_local_unref (local);
-        marker_local_unref (oplocal);
+    marker_local_unref(local);
+    marker_local_unref(oplocal);
 }
 
-
 int32_t
-marker_rename_unwind (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_rename_unwind(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t       *local    = NULL;
-        marker_local_t       *oplocal  = NULL;
-        quota_inode_ctx_t    *ctx      = NULL;
-        inode_contribution_t *contri   = NULL;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    inode_contribution_t *contri = NULL;
 
-        local = frame->local;
-        oplocal = local->oplocal;
-        frame->local = NULL;
+    local = frame->local;
+    oplocal = local->oplocal;
+    frame->local = NULL;
 
-        //Reset frame uid and gid if set.
-        if (cookie == (void *) _GF_UID_GID_CHANGED)
-                MARKER_RESET_UID_GID (frame, frame->root, local);
-
-        if (op_ret < 0)
-                local->err =  op_errno ? op_errno : EINVAL;
-
-        if (local->stub != NULL) {
-                /* Remove contribution node from in-memory even if
-                 * remove-xattr has failed as the rename is already performed
-                 * if local->stub is set, which means rename was sucessful
-                 */
-                mq_inode_ctx_get (oplocal->loc.inode, this, &ctx);
-                if (ctx) {
-                        contri = mq_get_contribution_node (oplocal->loc.parent,
-                                                           ctx);
-                        if (contri) {
-                                QUOTA_FREE_CONTRIBUTION_NODE (ctx, contri);
-                                GF_REF_PUT (contri);
-                        }
-                }
-
-                call_resume (local->stub);
-                local->stub = NULL;
-                local->err = 0;
-        } else if (local->err != 0) {
-                STACK_UNWIND_STRICT (rename, frame, -1, local->err, NULL, NULL,
-                                     NULL, NULL, NULL, NULL);
-        } else {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "continuation stub to unwind the call is absent, hence "
-                        "call will be hung (call-stack id = %"PRIu64")",
-                        frame->root->unique);
-        }
+    // Reset frame uid and gid if set.
+    if (cookie == (void *)_GF_UID_GID_CHANGED)
+        MARKER_RESET_UID_GID(frame, frame->root, local);
 
-        /* If there are in-progress writes on old-path when during rename
-         * operation, update txn will update the wrong path if lock
-         * is released before rename unwind.
-         * So release lock only after rename unwind
-         */
-        marker_rename_release_oldp_lock (local, this);
+    if (op_ret < 0)
+        local->err = op_errno ? op_errno : EINVAL;
 
-        return 0;
+    if (local->stub != NULL) {
+        /* Remove contribution node from in-memory even if
+         * remove-xattr has failed as the rename is already performed
+         * if local->stub is set, which means rename was successful
+         */
+        (void)mq_inode_ctx_get(oplocal->loc.inode, this, &ctx);
+        if (ctx) {
+            contri = mq_get_contribution_node(oplocal->loc.parent, ctx);
+            if (contri) {
+                QUOTA_FREE_CONTRIBUTION_NODE(ctx, contri);
+                GF_REF_PUT(contri);
+            }
+        }
+
+        call_resume(local->stub);
+        local->stub = NULL;
+        local->err = 0;
+    } else if (local->err != 0) {
+        STACK_UNWIND_STRICT(rename, frame, -1, local->err, NULL, NULL, NULL,
+                            NULL, NULL, NULL);
+    } else {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "continuation stub to unwind the call is absent, hence "
+               "call will be hung (call-stack id = %" PRIu64 ")",
+               frame->root->unique);
+    }
+
+    /* If there are in-progress writes on old-path when during rename
+     * operation, update txn will update the wrong path if lock
+     * is released before rename unwind.
+     * So release lock only after rename unwind
+     */
+    marker_rename_release_oldp_lock(local, this);
+
+    return 0;
 }
 
-
 int32_t
-marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent,
-                   dict_t *xdata)
-{
-        marker_conf_t  *priv                            = NULL;
-        marker_local_t *local                           = NULL;
-        marker_local_t *oplocal                         = NULL;
-        call_stub_t    *stub                            = NULL;
-        int32_t         ret                             = 0;
-        char            contri_key[QUOTA_KEY_MAX]       = {0, };
-        loc_t           newloc                          = {0, };
-
-        local = (marker_local_t *) frame->local;
+marker_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  struct iatt *preoldparent, struct iatt *postoldparent,
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
+{
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    call_stub_t *stub = NULL;
+    int32_t ret = 0;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+
+    local = (marker_local_t *)frame->local;
+
+    if (local != NULL) {
+        oplocal = local->oplocal;
+    }
 
+    priv = this->private;
+
+    if (op_ret < 0) {
         if (local != NULL) {
-                oplocal = local->oplocal;
+            local->err = op_errno;
         }
 
-        priv = this->private;
-
-        if (op_ret < 0) {
-                if (local != NULL) {
-                        local->err = op_errno;
-                }
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "renaming a file ",
+               strerror(op_errno));
+    }
 
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "renaming a file ", strerror (op_errno));
+    if (priv->feature_enabled & GF_QUOTA) {
+        if ((op_ret < 0) || (local == NULL)) {
+            goto quota_err;
         }
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                if ((op_ret < 0) || (local == NULL)) {
-                        goto quota_err;
-                }
-
-                local->ia_nlink = 0;
-                if (xdata)
-                        ret = dict_get_uint32 (xdata,
-                                               GF_RESPONSE_LINK_COUNT_XDATA,
-                                               &local->ia_nlink);
-
-                local->buf = *buf;
-                stub = fop_rename_cbk_stub (frame, default_rename_cbk, op_ret,
-                                            op_errno, buf, preoldparent,
-                                            postoldparent, prenewparent,
-                                            postnewparent, xdata);
-                if (stub == NULL) {
-                        local->err = ENOMEM;
-                        goto quota_err;
-                }
-
-                local->stub = stub;
-
-                GET_CONTRI_KEY (this, contri_key, oplocal->loc.parent->gfid,
-                                ret);
-                if (ret < 0) {
-                        local->err = ENOMEM;
-                        goto quota_err;
-                }
-
-                /* Removexattr requires uid and gid to be 0,
-                 * reset them in the callback.
-                 */
-                MARKER_SET_UID_GID (frame, local, frame->root);
-
-                newloc.inode = inode_ref (oplocal->loc.inode);
-                newloc.path = gf_strdup (local->loc.path);
-                newloc.name = strrchr (newloc.path, '/');
-                if (newloc.name)
-                        newloc.name++;
-                newloc.parent = inode_ref (local->loc.parent);
-                gf_uuid_copy (newloc.gfid, oplocal->loc.inode->gfid);
-
-                STACK_WIND_COOKIE (frame, marker_rename_unwind,
-                                   frame->cookie, FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->removexattr,
-                                   &newloc, contri_key, NULL);
-
-                loc_wipe (&newloc);
-        } else {
-                frame->local = NULL;
-
-                STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                                     preoldparent, postoldparent, prenewparent,
-                                     postnewparent, xdata);
-
-                if ((op_ret < 0) || (local == NULL)) {
-                        goto out;
-                }
-
-                if (priv->feature_enabled & GF_XTIME) {
-                        //update marks on oldpath
-                        gf_uuid_copy (local->loc.gfid, oplocal->loc.inode->gfid);
-                        marker_xtime_update_marks (this, oplocal);
-                        marker_xtime_update_marks (this, local);
-                }
+        local->ia_nlink = 0;
+        if (xdata)
+            ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA,
+                                  &local->ia_nlink);
+
+        local->buf = *buf;
+        stub = fop_rename_cbk_stub(frame, default_rename_cbk, op_ret, op_errno,
+                                   buf, preoldparent, postoldparent,
+                                   prenewparent, postnewparent, xdata);
+        if (stub == NULL) {
+            local->err = ENOMEM;
+            goto quota_err;
         }
 
-out:
-        if (!(priv->feature_enabled & GF_QUOTA)) {
-                marker_local_unref (local);
-                marker_local_unref (oplocal);
+        local->stub = stub;
+
+        GET_CONTRI_KEY(this, contri_key, oplocal->loc.parent->gfid, ret);
+        if (ret < 0) {
+            local->err = ENOMEM;
+            goto quota_err;
         }
 
-        return 0;
+        /* Removexattr requires uid and gid to be 0,
+         * reset them in the callback.
+         */
+        MARKER_SET_UID_GID(frame, local, frame->root);
 
-quota_err:
-        marker_rename_unwind (frame, NULL, this, 0, 0, NULL);
-        return 0;
-}
+        newloc.inode = inode_ref(oplocal->loc.inode);
+        newloc.path = gf_strdup(local->loc.path);
+        newloc.name = strrchr(newloc.path, '/');
+        if (newloc.name)
+            newloc.name++;
+        newloc.parent = inode_ref(local->loc.parent);
+        gf_uuid_copy(newloc.gfid, oplocal->loc.inode->gfid);
 
+        STACK_WIND_COOKIE(
+            frame, marker_rename_unwind, frame->cookie, FIRST_CHILD(this),
+            FIRST_CHILD(this)->fops->removexattr, &newloc, contri_key, NULL);
 
-int32_t
-marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-        marker_local_t       *local                      = NULL;
-        marker_local_t       *oplocal                    = NULL;
-        char                  contri_key[QUOTA_KEY_MAX]  = {0, };
-        int32_t               ret                        = 0;
-        quota_meta_t          contribution               = {0, };
+        loc_wipe(&newloc);
+    } else {
+        frame->local = NULL;
 
-        local = frame->local;
-        oplocal = local->oplocal;
+        STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                            postoldparent, prenewparent, postnewparent, xdata);
 
-        //Reset frame uid and gid if set.
-        if (cookie == (void *) _GF_UID_GID_CHANGED)
-                MARKER_RESET_UID_GID (frame, frame->root, local);
-
-        if ((op_ret < 0) && (op_errno != ENOATTR) && (op_errno != ENODATA)) {
-                local->err = op_errno ? op_errno : EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "fetching contribution values from %s (gfid:%s) "
-                        "failed (%s)", oplocal->loc.path,
-                        uuid_utoa (oplocal->loc.inode->gfid),
-                        strerror (op_errno));
-                goto err;
+        if ((op_ret < 0) || (local == NULL)) {
+            goto out;
         }
 
-        GET_CONTRI_KEY (this, contri_key, oplocal->loc.parent->gfid, ret);
-        if (ret < 0) {
-                local->err = errno ? errno : ENOMEM;
-                goto err;
+        if (priv->feature_enabled & GF_XTIME) {
+            // update marks on oldpath
+            if (!local->loc.inode)
+                local->loc.inode = inode_ref(oplocal->loc.inode);
+            gf_uuid_copy(local->loc.gfid, oplocal->loc.inode->gfid);
+            marker_xtime_update_marks(this, oplocal);
+            marker_xtime_update_marks(this, local);
         }
-        quota_dict_get_meta (dict, contri_key, &contribution);
-        oplocal->contribution = contribution;
+    }
 
-        STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, &oplocal->loc,
-                    &local->loc, local->xdata);
+out:
+    if (!(priv->feature_enabled & GF_QUOTA)) {
+        marker_local_unref(local);
+        marker_local_unref(oplocal);
+    }
 
-        return 0;
+    return 0;
 
-err:
-        marker_rename_unwind (frame, NULL, this, 0, 0, NULL);
-        return 0;
+quota_err:
+    marker_rename_unwind(frame, NULL, this, 0, 0, NULL);
+    return 0;
 }
 
 int32_t
-marker_get_oldpath_contribution (call_frame_t *lk_frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
-{
-        call_frame_t    *frame                      = NULL;
-        marker_local_t  *local                      = NULL;
-        marker_local_t  *oplocal                    = NULL;
-        char             contri_key[QUOTA_KEY_MAX]  = {0, };
-        int32_t          ret                        = 0;
+marker_do_rename(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    quota_meta_t contribution = {
+        0,
+    };
+
+    local = frame->local;
+    oplocal = local->oplocal;
+
+    // Reset frame uid and gid if set.
+    if (cookie == (void *)_GF_UID_GID_CHANGED)
+        MARKER_RESET_UID_GID(frame, frame->root, local);
+
+    if ((op_ret < 0) && (op_errno != ENOATTR) && (op_errno != ENODATA)) {
+        local->err = op_errno ? op_errno : EINVAL;
+        gf_log(this->name, GF_LOG_WARNING,
+               "fetching contribution values from %s (gfid:%s) "
+               "failed (%s)",
+               oplocal->loc.path, uuid_utoa(oplocal->loc.inode->gfid),
+               strerror(op_errno));
+        goto err;
+    }
+
+    GET_CONTRI_KEY(this, contri_key, oplocal->loc.parent->gfid, keylen);
+    if (keylen < 0) {
+        local->err = errno ? errno : ENOMEM;
+        goto err;
+    }
+    quota_dict_get_meta(dict, contri_key, keylen, &contribution);
+    oplocal->contribution = contribution;
+
+    STACK_WIND(frame, marker_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &oplocal->loc, &local->loc,
+               local->xdata);
+
+    return 0;
 
-        local = lk_frame->local;
-        oplocal = local->oplocal;
-        frame = local->frame;
-
-        if (op_ret < 0) {
-                local->err = op_errno ? op_errno : EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot hold inodelk on %s (gfid:%s) (%s)",
-                        oplocal->loc.path, uuid_utoa (oplocal->loc.inode->gfid),
-                        strerror (op_errno));
-                goto err;
+err:
+    marker_rename_unwind(frame, NULL, this, 0, 0, NULL);
+    return 0;
+}
 
-                STACK_DESTROY (local->lk_frame->root);
-                local->lk_frame = NULL;
+int32_t
+marker_get_oldpath_contribution(call_frame_t *lk_frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+    call_frame_t *frame = NULL;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+
+    local = lk_frame->local;
+    oplocal = local->oplocal;
+    frame = local->frame;
+
+    if (op_ret < 0) {
+        local->err = op_errno ? op_errno : EINVAL;
+        gf_log(this->name, GF_LOG_WARNING,
+               "cannot hold inodelk on %s (gfid:%s) (%s)", oplocal->loc.path,
+               uuid_utoa(oplocal->loc.inode->gfid), strerror(op_errno));
+        if (local->lk_frame) {
+            STACK_DESTROY(local->lk_frame->root);
+            local->lk_frame = NULL;
         }
+        goto err;
+    }
 
-        GET_CONTRI_KEY (this, contri_key, oplocal->loc.parent->gfid, ret);
-        if (ret < 0) {
-                local->err = errno ? errno : ENOMEM;
-                goto err;
-        }
+    GET_CONTRI_KEY(this, contri_key, oplocal->loc.parent->gfid, ret);
+    if (ret < 0) {
+        local->err = errno ? errno : ENOMEM;
+        goto err;
+    }
 
-        /* getxattr requires uid and gid to be 0,
-         * reset them in the callback.
-         */
-        MARKER_SET_UID_GID (frame, local, frame->root);
+    /* getxattr requires uid and gid to be 0,
+     * reset them in the callback.
+     */
+    MARKER_SET_UID_GID(frame, local, frame->root);
 
-        if (gf_uuid_is_null (oplocal->loc.gfid))
-                        gf_uuid_copy (oplocal->loc.gfid,
-                                   oplocal->loc.inode->gfid);
+    if (gf_uuid_is_null(oplocal->loc.gfid))
+        gf_uuid_copy(oplocal->loc.gfid, oplocal->loc.inode->gfid);
 
-        GF_UUID_ASSERT (oplocal->loc.gfid);
+    GF_UUID_ASSERT(oplocal->loc.gfid);
 
-        STACK_WIND_COOKIE (frame, marker_do_rename,
-                           frame->cookie, FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->getxattr,
-                           &oplocal->loc, contri_key, NULL);
+    STACK_WIND_COOKIE(frame, marker_do_rename, frame->cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->getxattr, &oplocal->loc,
+                      contri_key, NULL);
 
-        return 0;
+    return 0;
 err:
-        marker_rename_unwind (frame, NULL, this, 0, 0, NULL);
-        return 0;
+    marker_rename_unwind(frame, NULL, this, 0, 0, NULL);
+    return 0;
 }
 
-
 /* For a marker_rename FOP, following is the algorithm used for Quota
  * accounting. The use-case considered is:
  * 1. rename (src, dst)
@@ -1637,7 +1629,7 @@ err:
  *      b) we should subtract from src-parent exactly what we contributed to
  *         src-parent
  *    So, We hold a lock on src-parent to block any parallel transcations on
- *    src-inode (since thats the one which survives rename).
+ *    src-inode (since that's the one which survives rename).
  *
  *    If there are any parallel transactions on dst-inode they keep succeeding
  *    till the association of dst-inode with dst-parent is broken because of an
@@ -1699,628 +1691,631 @@ err:
  * 10) create contribution xattr to dst-parent on src-inode.
  */
 int32_t
-marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc, dict_t *xdata)
+marker_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        int32_t         ret              = 0;
-        marker_local_t *local            = NULL;
-        marker_local_t *oplocal          = NULL;
-        marker_conf_t  *priv             = NULL;
-        struct gf_flock lock             = {0, };
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    marker_conf_t *priv = NULL;
+    struct gf_flock lock = {
+        0,
+    };
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto rename_wind;
+    if (priv->feature_enabled == 0)
+        goto rename_wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        oplocal = mem_get0 (this->local_pool);
+    oplocal = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, oplocal);
+    MARKER_INIT_LOCAL(frame, oplocal);
 
-        frame->local = local;
+    frame->local = local;
 
-        local->oplocal = marker_local_ref (oplocal);
+    local->oplocal = marker_local_ref(oplocal);
 
-        ret = loc_copy (&local->loc, newloc);
-        if (ret < 0)
-                goto err;
+    ret = loc_copy(&local->loc, newloc);
+    if (ret < 0)
+        goto err;
 
-        ret = loc_copy (&oplocal->loc, oldloc);
-        if (ret < 0)
-                goto err;
+    ret = loc_copy(&oplocal->loc, oldloc);
+    if (ret < 0)
+        goto err;
 
-        if (!(priv->feature_enabled & GF_QUOTA)) {
-                goto rename_wind;
-        }
+    if (!(priv->feature_enabled & GF_QUOTA)) {
+        goto rename_wind;
+    }
 
-        ret = mq_inode_loc_fill (NULL, newloc->parent, &local->parent_loc);
-        if (ret < 0)
-                goto err;
+    ret = mq_inode_loc_fill(NULL, newloc->parent, &local->parent_loc);
+    if (ret < 0)
+        goto err;
 
-        ret = mq_inode_loc_fill (NULL, oldloc->parent, &oplocal->parent_loc);
-        if (ret < 0)
-                goto err;
+    ret = mq_inode_loc_fill(NULL, oldloc->parent, &oplocal->parent_loc);
+    if (ret < 0)
+        goto err;
 
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
+    lock.l_len = 0;
+    lock.l_start = 0;
+    lock.l_type = F_WRLCK;
+    lock.l_whence = SEEK_SET;
 
-        local->xdata = xdata ? dict_ref (xdata) : dict_new ();
-        ret = dict_set_int32 (local->xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
-        if (ret < 0)
-                goto err;
+    local->xdata = xdata ? dict_ref(xdata) : dict_new();
+    ret = dict_set_int32(local->xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
+    if (ret < 0)
+        goto err;
 
-        local->frame = frame;
-        local->lk_frame = create_frame (this, this->ctx->pool);
-        if (local->lk_frame == NULL)
-                goto err;
+    local->frame = frame;
+    local->lk_frame = create_frame(this, this->ctx->pool);
+    if (local->lk_frame == NULL)
+        goto err;
 
-        local->lk_frame->root->uid = 0;
-        local->lk_frame->root->gid = 0;
-        local->lk_frame->local = local;
-        set_lk_owner_from_ptr (&local->lk_frame->root->lk_owner,
-                               local->lk_frame->root);
+    local->lk_frame->root->uid = 0;
+    local->lk_frame->root->gid = 0;
+    local->lk_frame->local = local;
+    set_lk_owner_from_ptr(&local->lk_frame->root->lk_owner,
+                          local->lk_frame->root);
 
-        STACK_WIND (local->lk_frame,
-                    marker_get_oldpath_contribution,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &oplocal->parent_loc,
-                    F_SETLKW, &lock, NULL);
+    STACK_WIND(local->lk_frame, marker_get_oldpath_contribution,
+               FIRST_CHILD(this), FIRST_CHILD(this)->fops->inodelk, this->name,
+               &oplocal->parent_loc, F_SETLKW, &lock, NULL);
 
-        return 0;
+    return 0;
 
 rename_wind:
-        STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    STACK_WIND(frame, marker_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 err:
-        MARKER_STACK_UNWIND (rename, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
-        marker_local_unref (oplocal);
+    MARKER_STACK_UNWIND(rename, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    marker_local_unref(oplocal);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+marker_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "truncating a file ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "truncating a file ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        priv = this->private;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                /* DHT Rebalance process, at the end of migration will
-                 * first make the src file as a linkto file and then
-                 * truncate the file. By doing a truncate after making the
-                 * src file as linkto file, the contri which is already
-                 * accounted is left over.
-                 * So, we need to account for the linkto file when a truncate
-                 * happens, thereby updating the contri properly.
-                 * By passing NULL for postbuf, mq_prevalidate does not check
-                 * for linkto file.
-                 * Same happens with ftruncate as well.
-                 */
-                if (postbuf && IS_DHT_LINKFILE_MODE (postbuf))
-                        mq_initiate_quota_txn (this, &local->loc, NULL);
-                else
-                        mq_initiate_quota_txn (this, &local->loc, postbuf);
-        }
+    priv = this->private;
+
+    if (priv->feature_enabled & GF_QUOTA) {
+        /* DHT Rebalance process, at the end of migration will
+         * first make the src file as a linkto file and then
+         * truncate the file. By doing a truncate after making the
+         * src file as linkto file, the contri which is already
+         * accounted is left over.
+         * So, we need to account for the linkto file when a truncate
+         * happens, thereby updating the contri properly.
+         * By passing NULL for postbuf, mq_prevalidate does not check
+         * for linkto file.
+         * Same happens with ftruncate as well.
+         */
+        if (postbuf && IS_DHT_LINKFILE_MODE(postbuf))
+            mq_initiate_quota_txn(this, &local->loc, NULL);
+        else
+            mq_initiate_quota_txn(this, &local->loc, postbuf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                 dict_t *xdata)
+marker_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, marker_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+marker_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "truncating a file ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "truncating a file ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                if (postbuf && IS_DHT_LINKFILE_MODE (postbuf))
-                        mq_initiate_quota_txn (this, &local->loc, NULL);
-                else
-                        mq_initiate_quota_txn (this, &local->loc, postbuf);
-        }
+    if (priv->feature_enabled & GF_QUOTA) {
+        if (postbuf && IS_DHT_LINKFILE_MODE(postbuf))
+            mq_initiate_quota_txn(this, &local->loc, NULL);
+        else
+            mq_initiate_quota_txn(this, &local->loc, postbuf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  dict_t *xdata)
+marker_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, marker_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
-        quota_inode_ctx_t  *ctx     = NULL;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "creating symlinks ", strerror (op_errno));
-        }
+marker_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        local = (marker_local_t *) frame->local;
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "creating symlinks ",
+               strerror(op_errno));
+    }
 
-        frame->local = NULL;
-        priv = this->private;
-
-        if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
-                ctx = mq_inode_ctx_new (inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s", uuid_utoa (inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+    local = (marker_local_t *)frame->local;
+
+    frame->local = NULL;
+    priv = this->private;
+
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
+    }
 
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                mq_create_xattrs_txn (this, &local->loc, buf);
-        }
+    if (priv->feature_enabled & GF_QUOTA) {
+        mq_create_xattrs_txn(this, &local->loc, buf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, mode_t umask, dict_t *xdata)
+marker_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, marker_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (symlink, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(symlink, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+marker_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
-        quota_inode_ctx_t  *ctx     = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred with "
-                        "mknod ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred with "
+               "mknod ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
-        priv = this->private;
-
-        if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
-                ctx = mq_inode_ctx_new (inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s", uuid_utoa (inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+    frame->local = NULL;
+    priv = this->private;
+
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
+    }
 
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        if (op_ret == -1 ||  local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if ((priv->feature_enabled & GF_QUOTA) && (S_ISREG (local->mode))) {
-                mq_create_xattrs_txn (this, &local->loc, buf);
-        }
+    if ((priv->feature_enabled & GF_QUOTA) && (S_ISREG(local->mode))) {
+        mq_create_xattrs_txn(this, &local->loc, buf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *xdata)
+marker_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        local->mode = mode;
+    local->mode = mode;
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, marker_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_fallocate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+marker_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "fallocating a file ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "fallocating a file ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fallocate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc, postbuf);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		 off_t offset, size_t len, dict_t *xdata)
+                 off_t offset, size_t len, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_fallocate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		    xdata);
-        return 0;
+    STACK_WIND(frame, marker_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
 marker_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                    struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred during discard",
-                        strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred during discard",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc, postbuf);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       size_t len, dict_t *xdata)
+               size_t len, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_discard_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, marker_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (discard, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(discard, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred during zerofill",
-                        strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred during zerofill",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc, postbuf);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               off_t len, dict_t *xdata)
+                off_t len, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_zerofill_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, marker_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* when a call from the special client is received on
  * key trusted.glusterfs.volume-mark with value "RESET"
  * or if the value is 0length, update the change the
@@ -2328,1193 +2323,1246 @@ err:
  * timestamp file.
  */
 int32_t
-call_from_sp_client_to_reset_tmfile (call_frame_t *frame,
-                                     xlator_t     *this,
-                                     dict_t       *dict)
+call_from_sp_client_to_reset_tmfile(call_frame_t *frame, xlator_t *this,
+                                    dict_t *dict)
 {
-        int32_t          fd       = 0;
-        int32_t          op_ret   = 0;
-        int32_t          op_errno = 0;
-        data_t          *data     = NULL;
-        marker_conf_t   *priv     = NULL;
+    int32_t fd = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    data_t *data = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (frame == NULL || this == NULL || dict == NULL)
-                return -1;
+    if (frame == NULL || this == NULL || dict == NULL)
+        return -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        data = dict_get (dict, "trusted.glusterfs.volume-mark");
-        if (data == NULL)
-                return -1;
+    data = dict_get(dict, "trusted.glusterfs.volume-mark");
+    if (data == NULL)
+        return -1;
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                op_ret = -1;
-                op_errno = EPERM;
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        op_ret = -1;
+        op_errno = EPERM;
 
-                goto out;
+        goto out;
+    }
+
+    if (data->len == 0 ||
+        (data->len == 5 && memcmp(data->data, "RESET", 5) == 0)) {
+        fd = open(priv->timestamp_file, O_WRONLY | O_TRUNC);
+        if (fd != -1) {
+            /* TODO check  whether the O_TRUNC would update the
+             * timestamps on a zero length file on all machies.
+             */
+            sys_close(fd);
         }
 
-        if (data->len == 0 || (data->len == 5 &&
-                               memcmp (data->data, "RESET", 5) == 0)) {
-                fd = open (priv->timestamp_file, O_WRONLY|O_TRUNC);
-                if (fd != -1) {
-                        /* TODO check  whether the O_TRUNC would update the
-                         * timestamps on a zero length file on all machies.
-                         */
-                        sys_close (fd);
-                }
-
-                if (fd != -1 || errno == ENOENT) {
-                        op_ret = 0;
-                        op_errno = 0;
-                } else {
-                        op_ret = -1;
-                        op_errno = errno;
-                }
+        if (fd != -1 || errno == ENOENT) {
+            op_ret = 0;
+            op_errno = 0;
         } else {
-                op_ret = -1;
-                op_errno = EINVAL;
+            op_ret = -1;
+            op_errno = errno;
         }
+    } else {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 out:
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred in "
-                        "setxattr ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred in "
+               "setxattr ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-remove_quota_keys (dict_t *dict, char *k, data_t *v, void *data)
-{
-        call_frame_t    *frame              = data;
-        marker_local_t  *local              = frame->local;
-        xlator_t        *this               = frame->this;
-        marker_conf_t   *priv               = NULL;
-        char             ver_str[NAME_MAX]  = {0,};
-        char            *dot                = NULL;
-        int              ret                = -1;
-
-        priv = this->private;
-
-        /* If quota is enabled immediately after disable.
-         * quota healing starts creating new xattrs
-         * before completing the cleanup operation.
-         * So we should check if the xattr is the new.
-         * Do not remove xattr if its xattr
-         * version is same as current version
-         */
-        if ((priv->feature_enabled & GF_QUOTA) && priv->version > 0) {
-                snprintf (ver_str, sizeof (ver_str), ".%d", priv->version);
-                dot = strrchr (k, '.');
-                if (dot && !strcmp(dot, ver_str))
-                        return 0;
-        }
-
-	ret = syncop_removexattr (FIRST_CHILD (this), &local->loc, k, 0, NULL);
-	if (ret) {
-		gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove "
-			"extended attribute: %s", local->loc.path, k);
-                return -1;
-	}
-	return 0;
+remove_quota_keys(dict_t *dict, char *k, data_t *v, void *data)
+{
+    call_frame_t *frame = data;
+    marker_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    marker_conf_t *priv = NULL;
+    char ver_str[NAME_MAX] = {
+        0,
+    };
+    char *dot = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    /* If quota is enabled immediately after disable.
+     * quota healing starts creating new xattrs
+     * before completing the cleanup operation.
+     * So we should check if the xattr is the new.
+     * Do not remove xattr if its xattr
+     * version is same as current version
+     */
+    if ((priv->feature_enabled & GF_QUOTA) && priv->version > 0) {
+        snprintf(ver_str, sizeof(ver_str), ".%d", priv->version);
+        dot = strrchr(k, '.');
+        if (dot && !strcmp(dot, ver_str))
+            return 0;
+    }
+
+    ret = syncop_removexattr(FIRST_CHILD(this), &local->loc, k, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: Failed to remove "
+               "extended attribute: %s",
+               local->loc.path, k);
+        return -1;
+    }
+    return 0;
 }
 
 int
-quota_xattr_cleaner_cbk (int ret, call_frame_t *frame, void *args)
+quota_xattr_cleaner_cbk(int ret, call_frame_t *frame, void *args)
 {
-        dict_t *xdata = args;
-        int op_ret = -1;
-        int op_errno = 0;
+    dict_t *xdata = args;
+    int op_ret = -1;
+    int op_errno = 0;
 
-        op_ret   = (ret < 0)? -1: 0;
-        op_errno = -ret;
+    op_ret = (ret < 0) ? -1 : 0;
+    op_errno = -ret;
 
-        MARKER_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-        return ret;
+    MARKER_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return ret;
 }
 
 int
-quota_xattr_cleaner (void *args)
+quota_xattr_cleaner(void *args)
 {
-        struct synctask *task  = NULL;
-        call_frame_t    *frame = NULL;
-        xlator_t        *this  = NULL;
-        marker_local_t   *local = NULL;
-        dict_t          *xdata = NULL;
-        int             ret    = -1;
+    struct synctask *task = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    marker_local_t *local = NULL;
+    dict_t *xdata = NULL;
+    int ret = -1;
 
-        task = synctask_get ();
-        if (!task)
-                goto out;
+    task = synctask_get();
+    if (!task)
+        goto out;
 
-        frame = task->frame;
-        this  = frame->this;
-        local = frame->local;
+    frame = task->frame;
+    this = frame->this;
+    local = frame->local;
 
-        ret = syncop_listxattr (FIRST_CHILD(this), &local->loc, &xdata, NULL,
-                                NULL);
-        if (ret == -1) {
-                ret = -errno;
-                goto out;
-        }
+    ret = syncop_listxattr(FIRST_CHILD(this), &local->loc, &xdata, NULL, NULL);
+    if (ret == -1) {
+        ret = -errno;
+        goto out;
+    }
 
-	ret = dict_foreach_fnmatch (xdata, "trusted.glusterfs.quota.*",
-                                    remove_quota_keys, frame);
-        if (ret == -1) {
-                ret = -errno;
-                goto out;
-        }
-	ret = dict_foreach_fnmatch (xdata, PGFID_XATTR_KEY_PREFIX"*",
-                                    remove_quota_keys, frame);
-        if (ret == -1) {
-                ret = -errno;
-                goto out;
-        }
+    ret = dict_foreach_fnmatch(xdata, "trusted.glusterfs.quota.*",
+                               remove_quota_keys, frame);
+    if (ret == -1) {
+        ret = -errno;
+        goto out;
+    }
+    ret = dict_foreach_fnmatch(xdata, PGFID_XATTR_KEY_PREFIX "*",
+                               remove_quota_keys, frame);
+    if (ret == -1) {
+        ret = -errno;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
 int
-marker_do_xattr_cleanup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
+marker_do_xattr_cleanup(call_frame_t *frame, xlator_t *this, dict_t *xdata,
                         loc_t *loc)
 {
-        int           ret       = -1;
-        marker_local_t *local    = NULL;
+    int ret = -1;
+    marker_local_t *local = NULL;
 
-        local = mem_get0 (this->local_pool);
-	if (!local)
-		goto out;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        loc_copy (&local->loc, loc);
-        ret = synctask_new (this->ctx->env, quota_xattr_cleaner,
-			    quota_xattr_cleaner_cbk, frame, xdata);
-        if (ret) {
-		gf_log (this->name, GF_LOG_ERROR, "Failed to create synctask "
-			"for cleaning up quota extended attributes");
-                goto out;
-	}
+    loc_copy(&local->loc, loc);
+    ret = synctask_new(this->ctx->env, quota_xattr_cleaner,
+                       quota_xattr_cleaner_cbk, frame, xdata);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to create synctask "
+               "for cleaning up quota extended attributes");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret)
-                MARKER_STACK_UNWIND (setxattr, frame, -1, ENOMEM, xdata);
+    if (ret)
+        MARKER_STACK_UNWIND(setxattr, frame, -1, ENOMEM, xdata);
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-marker_xattr_cleanup_cmd (dict_t *dict)
+marker_xattr_cleanup_cmd(dict_t *dict)
 {
-        return (dict_get (dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
+    return (dict_get(dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
 }
 
 int32_t
-marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags, dict_t *xdata)
+marker_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int32_t          ret            = 0;
-        marker_local_t  *local          = NULL;
-        marker_conf_t   *priv           = NULL;
-        int              op_errno       = ENOMEM;
-
-        priv = this->private;
-
-        if (marker_xattr_cleanup_cmd (dict)) {
-                if (frame->root->uid != 0 || frame->root->gid != 0) {
-                        op_errno = EPERM;
-                        ret = -1;
-                        goto err;
-                }
-
-                /* The following function does the cleanup and then unwinds the
-                 * corresponding call*/
-                loc_path (loc, NULL);
-                marker_do_xattr_cleanup (frame, this, xdata, loc);
-                return 0;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    int op_errno = ENOMEM;
+
+    priv = this->private;
+
+    if (marker_xattr_cleanup_cmd(dict)) {
+        if (frame->root->uid != 0 || frame->root->gid != 0) {
+            op_errno = EPERM;
+            ret = -1;
+            goto err;
         }
 
-        ret = marker_key_replace_with_ver (this, dict);
-        if (ret < 0)
-                goto err;
+        /* The following function does the cleanup and then unwinds the
+         * corresponding call*/
+        loc_path(loc, NULL);
+        marker_do_xattr_cleanup(frame, this, xdata, loc);
+        return 0;
+    }
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    ret = marker_key_replace_with_ver(this, dict);
+    if (ret < 0)
+        goto err;
 
-        ret = call_from_sp_client_to_reset_tmfile (frame, this, dict);
-        if (ret == 0)
-                return 0;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    ret = call_from_sp_client_to_reset_tmfile(frame, this, dict);
+    if (ret == 0)
+        return 0;
 
-        MARKER_INIT_LOCAL (frame, local);
+    local = mem_get0(this->local_pool);
 
-        ret = loc_copy (&local->loc, loc);
+    MARKER_INIT_LOCAL(frame, local);
 
-        if (ret == -1)
-                goto err;
+    ret = loc_copy(&local->loc, loc);
+
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, marker_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+    MARKER_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred in "
-                        "fsetxattr", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred in "
+               "fsetxattr",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                  int32_t flags, dict_t *xdata)
+marker_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        ret = call_from_sp_client_to_reset_tmfile (frame, this, dict);
-        if (ret == 0)
-                return 0;
+    ret = call_from_sp_client_to_reset_tmfile(frame, this, dict);
+    if (ret == 0)
+        return 0;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, marker_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (fsetxattr, frame, -1, ENOMEM, NULL);
+    MARKER_STACK_UNWIND(fsetxattr, frame, -1, ENOMEM, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost, dict_t *xdata)
+marker_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred in "
-                        "fsetattr ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred in "
+               "fsetattr ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost, xdata);
+    STACK_UNWIND_STRICT(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+marker_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, marker_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost, dict_t *xdata)
+marker_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s occurred during setattr of %s",
-                        strerror (op_errno),
-                        (local ? local->loc.path : "<nul>"));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred during setattr of %s",
+               strerror(op_errno), (local ? local->loc.path : "<nul>"));
+    }
 
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost, xdata);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+marker_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, marker_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                                "%s occurred while "
-                                "removing extended attribute",
-                                     strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "removing extended attribute",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name, dict_t *xdata)
-{
-        int32_t          ret                  = -1;
-        int32_t          i                    = 0;
-        marker_local_t  *local                = NULL;
-        marker_conf_t   *priv                 = NULL;
-        char             key[QUOTA_KEY_MAX]   = {0, };
-
-        priv = this->private;
-
-        if (name) {
-                for (i = 0; mq_ext_xattrs[i]; i++) {
-                        if (strcmp (name, mq_ext_xattrs[i]))
-                                continue;
-
-                        GET_QUOTA_KEY (this, key, mq_ext_xattrs[i], ret);
-                        if (ret < 0)
-                                goto err;
-                        name = key;
-                        break;
-                }
+marker_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
+{
+    int32_t ret = -1;
+    int32_t i = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    priv = this->private;
+
+    if (name) {
+        for (i = 0; mq_ext_xattrs[i]; i++) {
+            if (strcmp(name, mq_ext_xattrs[i]))
+                continue;
+
+            GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+            if (ret < 0)
+                goto err;
+            name = key;
+            break;
         }
+    }
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, marker_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 err:
-        MARKER_STACK_UNWIND (removexattr, frame, -1, ENOMEM, NULL);
+    MARKER_STACK_UNWIND(removexattr, frame, -1, ENOMEM, NULL);
 
-        return 0;
+    return 0;
 }
 
 static gf_boolean_t
-__has_quota_xattrs (dict_t *xattrs)
+__has_quota_xattrs(dict_t *xattrs)
 {
-        if (dict_foreach_match (xattrs, _is_quota_internal_xattr, NULL,
-                                dict_null_foreach_fn, NULL) > 0)
-                return _gf_true;
+    if (dict_foreach_match(xattrs, _is_quota_internal_xattr, NULL,
+                           dict_null_foreach_fn, NULL) > 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 int32_t
-marker_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
-{
-        marker_conf_t      *priv                       = NULL;
-        marker_local_t     *local                      = NULL;
-        dict_t             *xattrs                     = NULL;
-        quota_inode_ctx_t  *ctx                        = NULL;
-        int32_t             ret                        = -1;
-
-        priv = this->private;
-        local = (marker_local_t *) frame->local;
-        frame->local = NULL;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "lookup failed with %s",
-                        strerror (op_errno));
-                goto unwind;
-        }
-
-        ret = marker_key_set_ver (this, dict);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        if (dict && __has_quota_xattrs (dict)) {
-                xattrs = dict_copy_with_ref (dict, NULL);
-                if (!xattrs) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                } else {
-                        marker_filter_internal_xattrs (this, xattrs);
-                }
-        } else if (dict) {
-                xattrs = dict_ref (dict);
+marker_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *dict, struct iatt *postparent)
+{
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    dict_t *xattrs = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+
+    priv = this->private;
+    local = (marker_local_t *)frame->local;
+    frame->local = NULL;
+
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "lookup failed with %s",
+               strerror(op_errno));
+        goto unwind;
+    }
+
+    ret = marker_key_set_ver(this, dict);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (dict && __has_quota_xattrs(dict)) {
+        xattrs = dict_copy_with_ref(dict, NULL);
+        if (!xattrs) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+        } else {
+            marker_filter_internal_xattrs(this, xattrs);
         }
+    } else if (dict) {
+        xattrs = dict_ref(dict);
+    }
 
-        if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
-                ctx = mq_inode_ctx_new (inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s", uuid_utoa (inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xattrs, postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xattrs,
+                        postparent);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        /* copy the gfid from the stat structure instead of inode,
-         * since if the lookup is fresh lookup, then the inode
-         * would have not yet linked to the inode table which happens
-         * in protocol/server.
-         */
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    /* copy the gfid from the stat structure instead of inode,
+     * since if the lookup is fresh lookup, then the inode
+     * would have not yet linked to the inode table which happens
+     * in protocol/server.
+     */
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-
-        if (priv->feature_enabled & GF_QUOTA) {
-                mq_xattr_state (this, &local->loc, dict, *buf);
-        }
+    if (priv->feature_enabled & GF_QUOTA) {
+        mq_xattr_state(this, &local->loc, dict, buf);
+    }
 
 out:
-        marker_local_unref (local);
-        if (xattrs)
-                dict_unref (xattrs);
+    marker_local_unref(local);
+    if (xattrs)
+        dict_unref(xattrs);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_lookup (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *xattr_req)
+marker_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr_req)
 {
-        int32_t         ret                     = 0;
-        marker_local_t *local                   = NULL;
-        marker_conf_t  *priv                    = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        xattr_req = xattr_req ? dict_ref (xattr_req) : dict_new ();
-        if (!xattr_req)
-                goto err;
+    xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+    if (!xattr_req)
+        goto err;
 
-        ret = marker_key_replace_with_ver (this, xattr_req);
-        if (ret < 0)
-                goto err;
+    ret = marker_key_replace_with_ver(this, xattr_req);
+    if (ret < 0)
+        goto err;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL)
-                goto err;
+    local = mem_get0(this->local_pool);
+    if (local == NULL)
+        goto err;
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1)
-                goto err;
+    ret = loc_copy(&local->loc, loc);
+    if (ret == -1)
+        goto err;
 
-        if ((priv->feature_enabled & GF_QUOTA))
-                mq_req_xattr (this, loc, xattr_req, NULL, NULL);
+    if ((priv->feature_enabled & GF_QUOTA))
+        mq_req_xattr(this, loc, xattr_req, NULL, NULL);
 
 wind:
-        STACK_WIND (frame, marker_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    STACK_WIND(frame, marker_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
 
-        dict_unref (xattr_req);
+    dict_unref(xattr_req);
 
-        return 0;
+    return 0;
 err:
-        MARKER_STACK_UNWIND (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        return 0;
+    return 0;
 }
 
-
 int
-marker_build_ancestry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, gf_dirent_t *entries,
-                           dict_t *xdata)
+marker_build_ancestry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, gf_dirent_t *entries,
+                          dict_t *xdata)
 {
-        gf_dirent_t        *entry  = NULL;
-        quota_inode_ctx_t  *ctx    = NULL;
-        int                 ret    = -1;
+    gf_dirent_t *entry = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        if ((op_ret <= 0) || (entries == NULL)) {
-                goto out;
-        }
+    if ((op_ret <= 0) || (entries == NULL)) {
+        goto out;
+    }
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if (entry->inode == NULL)
+            continue;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if (entry->inode == NULL)
-                        continue;
-
-                ret = marker_key_set_ver (this, entry->dict);
-                if (ret < 0) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        break;
-                }
-
-                ctx = mq_inode_ctx_new (entry->inode, this);
-                if (ctx == NULL)
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s",
-                                uuid_utoa (entry->inode->gfid));
+        ret = marker_key_set_ver(this, entry->dict);
+        if (ret < 0) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            break;
         }
 
+        ctx = mq_inode_ctx_new(entry->inode, this);
+        if (ctx == NULL)
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(entry->inode->gfid));
+    }
+
 out:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
 int
-marker_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, gf_dirent_t *entries,
-                     dict_t *xdata)
-{
-        gf_dirent_t        *entry         = NULL;
-        marker_conf_t      *priv          = NULL;
-        marker_local_t     *local         = NULL;
-        loc_t               loc           = {0, };
-        int                 ret           = -1;
-        char               *resolvedpath  = NULL;
-        quota_inode_ctx_t  *ctx           = NULL;
-
-        if (op_ret <= 0)
-                goto unwind;
-
-        priv = this->private;
-        local = frame->local;
-
-        if (!(priv->feature_enabled & GF_QUOTA) || (local == NULL)) {
-                goto unwind;
+marker_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+    gf_dirent_t *entry = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *resolvedpath = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+
+    if (op_ret <= 0)
+        goto unwind;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (!(priv->feature_enabled & GF_QUOTA) || (local == NULL)) {
+        goto unwind;
+    }
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0) || entry->inode == NULL)
+            continue;
+
+        loc.parent = inode_ref(local->loc.inode);
+        loc.inode = inode_ref(entry->inode);
+        ret = inode_path(loc.parent, entry->d_name, &resolvedpath);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to get the "
+                   "path for the entry %s",
+                   entry->d_name);
+            loc_wipe(&loc);
+            continue;
         }
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if ((strcmp (entry->d_name, ".") == 0)  ||
-                    (strcmp (entry->d_name, "..") == 0) ||
-                    entry->inode == NULL)
-                        continue;
-
-                loc.parent = inode_ref (local->loc.inode);
-                loc.inode = inode_ref (entry->inode);
-                ret = inode_path (loc.parent, entry->d_name, &resolvedpath);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "path for the entry %s", entry->d_name);
-                        loc_wipe (&loc);
-                        continue;
-                }
-
-                loc.path = resolvedpath;
-                resolvedpath = NULL;
-
-                ctx = mq_inode_ctx_new (loc.inode, this);
-                if (ctx == NULL)
-                        gf_log (this->name, GF_LOG_WARNING, "mq_inode_ctx_new "
-                                "failed for %s", uuid_utoa (loc.inode->gfid));
-
-                mq_xattr_state (this, &loc, entry->dict, entry->d_stat);
-                loc_wipe (&loc);
-
-                ret = marker_key_set_ver (this, entry->dict);
-                if (ret < 0) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        loc.path = resolvedpath;
+        resolvedpath = NULL;
+
+        ctx = mq_inode_ctx_new(loc.inode, this);
+        if (ctx == NULL)
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(loc.inode->gfid));
+
+        mq_xattr_state(this, &loc, entry->dict, &entry->d_stat);
+        loc_wipe(&loc);
+
+        ret = marker_key_set_ver(this, entry->dict);
+        if (ret < 0) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
+    }
 
 unwind:
-        MARKER_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, xdata);
+    MARKER_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset, dict_t *dict)
-{
-        marker_conf_t  *priv  = NULL;
-        loc_t           loc   = {0, };
-        marker_local_t *local = NULL;
-        int             ret   = -1;
-
-        priv = this->private;
-
-        dict = dict ? dict_ref(dict) : dict_new();
-        if (!dict)
-                goto unwind;
-
-        ret = marker_key_replace_with_ver (this, dict);
-        if (ret < 0)
-                goto unwind;
-
-        if (dict_get (dict, GET_ANCESTRY_DENTRY_KEY)) {
-                STACK_WIND (frame, marker_build_ancestry_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, offset, dict);
-        } else {
-                if (priv->feature_enabled & GF_QUOTA) {
-                        local = mem_get0 (this->local_pool);
-
-                        MARKER_INIT_LOCAL (frame, local);
+marker_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, dict_t *dict)
+{
+    marker_conf_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    marker_local_t *local = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    dict = dict ? dict_ref(dict) : dict_new();
+    if (!dict)
+        goto unwind;
+
+    ret = marker_key_replace_with_ver(this, dict);
+    if (ret < 0)
+        goto unwind;
+
+    if (dict_get(dict, GET_ANCESTRY_DENTRY_KEY)) {
+        STACK_WIND(frame, marker_build_ancestry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    } else {
+        if (priv->feature_enabled & GF_QUOTA) {
+            local = mem_get0(this->local_pool);
 
-                        loc.parent = local->loc.inode = inode_ref (fd->inode);
+            MARKER_INIT_LOCAL(frame, local);
 
-                        mq_req_xattr (this, &loc, dict, NULL, NULL);
-                }
+            loc.parent = local->loc.inode = inode_ref(fd->inode);
 
-                STACK_WIND (frame, marker_readdirp_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, offset, dict);
+            mq_req_xattr(this, &loc, dict, NULL, NULL);
         }
 
-        dict_unref (dict);
-        return 0;
+        STACK_WIND(frame, marker_readdirp_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    }
+
+    dict_unref(dict);
+    return 0;
 unwind:
-        MARKER_STACK_UNWIND (readdirp, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    MARKER_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_marker_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_marker_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int32_t
-init_xtime_priv (xlator_t *this, dict_t *options)
+init_xtime_priv(xlator_t *this, dict_t *options)
 {
-        data_t          *data    = NULL;
-        int32_t          ret     = -1;
-        marker_conf_t   *priv    = NULL;
+    int32_t ret = -1;
+    marker_conf_t *priv = NULL;
+    char *tmp_opt = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        if((data = dict_get (options, VOLUME_UUID)) != NULL) {
-                priv->volume_uuid = data->data;
+    ret = dict_get_str(options, "volume-uuid", &tmp_opt);
 
-                ret = gf_uuid_parse (priv->volume_uuid, priv->volume_uuid_bin);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid volume uuid %s", priv->volume_uuid);
-                        goto out;
-                }
+    if (ret) {
+        priv->volume_uuid = NULL;
+        tmp_opt = "";
 
-                ret = gf_asprintf (& (priv->marker_xattr), "%s.%s.%s",
-                                   MARKER_XATTR_PREFIX, priv->volume_uuid,
-                                   XTIME);
+        gf_log(this->name, GF_LOG_ERROR,
+               "please specify the volume-uuid"
+               "in the translator options");
 
-                if (ret == -1){
-                        priv->marker_xattr = NULL;
-                        goto out;
-                }
+        return -1;
+    }
+    gf_asprintf(&priv->volume_uuid, "%s", tmp_opt);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "volume-uuid = %s", priv->volume_uuid);
-        } else {
-                priv->volume_uuid = NULL;
+    ret = gf_uuid_parse(priv->volume_uuid, priv->volume_uuid_bin);
 
-                gf_log (this->name, GF_LOG_ERROR,
-                        "please specify the volume-uuid"
-                        "in the translator options");
+    if (ret == -1) {
+        gf_log(this->name, GF_LOG_ERROR, "invalid volume uuid %s",
+               priv->volume_uuid);
+        goto out;
+    }
 
-                return -1;
-        }
+    ret = gf_asprintf(&(priv->marker_xattr), "%s.%s.%s", MARKER_XATTR_PREFIX,
+                      priv->volume_uuid, XTIME);
 
-        if ((data = dict_get (options, TIMESTAMP_FILE)) != NULL) {
-                priv->timestamp_file = data->data;
+    if (ret == -1) {
+        priv->marker_xattr = NULL;
+        goto out;
+    }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "the timestamp-file is = %s",
-                        priv->timestamp_file);
+    gf_log(this->name, GF_LOG_DEBUG, "volume-uuid = %s", priv->volume_uuid);
 
-        } else {
-                priv->timestamp_file = NULL;
+    ret = dict_get_str(options, "timestamp-file", &tmp_opt);
+    if (ret) {
+        priv->timestamp_file = NULL;
+        tmp_opt = "";
 
-                gf_log (this->name, GF_LOG_ERROR,
-                        "please specify the timestamp-file"
-                        "in the translator options");
+        gf_log(this->name, GF_LOG_ERROR,
+               "please specify the timestamp-file"
+               "in the translator options");
 
-                goto out;
-        }
+        goto out;
+    }
 
-        ret = 0;
+    ret = gf_asprintf(&priv->timestamp_file, "%s", tmp_opt);
+    if (ret == -1) {
+        priv->timestamp_file = NULL;
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_DEBUG, "the timestamp-file is = %s",
+           priv->timestamp_file);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-marker_xtime_priv_cleanup (xlator_t *this)
+marker_xtime_priv_cleanup(xlator_t *this)
 {
-        marker_conf_t *priv = NULL;
+    marker_conf_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
 
-        priv = (marker_conf_t *) this->private;
+    priv = (marker_conf_t *)this->private;
 
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        GF_FREE (priv->volume_uuid);
+    GF_FREE(priv->volume_uuid);
 
-        GF_FREE (priv->timestamp_file);
+    GF_FREE(priv->timestamp_file);
 
-        GF_FREE (priv->marker_xattr);
+    GF_FREE(priv->marker_xattr);
 out:
-        return;
+    return;
 }
 
 void
-marker_priv_cleanup (xlator_t *this)
+marker_priv_cleanup(xlator_t *this)
 {
-        marker_conf_t *priv = NULL;
+    marker_conf_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("marker", this, out);
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
+    priv = (marker_conf_t *)this->private;
 
-        priv = (marker_conf_t *) this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+    marker_xtime_priv_cleanup(this);
 
-        marker_xtime_priv_cleanup (this);
+    LOCK_DESTROY(&priv->lock);
 
-        LOCK_DESTROY (&priv->lock);
+    GF_FREE(priv);
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 
-        GF_FREE (priv);
 out:
-        return;
+    return;
 }
 
 int32_t
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t         ret     = 0;
-        data_t         *data    = NULL;
-        gf_boolean_t    flag    = _gf_false;
-        marker_conf_t  *priv    = NULL;
-        int32_t         version = 0;
+    int32_t ret = 0;
+    data_t *data = NULL;
+    gf_boolean_t flag = _gf_false;
+    marker_conf_t *priv = NULL;
+    int32_t version = 0;
 
-        GF_ASSERT (this);
-        GF_ASSERT (this->private);
+    GF_ASSERT(this);
+    GF_ASSERT(this->private);
 
-        priv = this->private;
+    priv = this->private;
 
-        priv->feature_enabled = 0;
+    priv->feature_enabled = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
 
-        data = dict_get (options, "quota");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true)
-                        priv->feature_enabled |= GF_QUOTA;
-        }
+    data = dict_get(options, "quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_QUOTA;
+    }
 
-        data = dict_get (options, "inode-quota");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true)
-                        priv->feature_enabled |= GF_INODE_QUOTA;
-        }
-
-        data = dict_get (options, "quota-version");
-        if (data)
-                ret = gf_string2int32 (data->data, &version);
+    data = dict_get(options, "inode-quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_INODE_QUOTA;
+    }
 
-        if (priv->feature_enabled) {
-                if (version >= 0)
-                        priv->version = version;
-                else
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid quota "
-                                "version %d", priv->version);
-        }
+    data = dict_get(options, "quota-version");
+    if (data)
+        ret = gf_string2int32(data->data, &version);
 
-        data = dict_get (options, "xtime");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true) {
-                        marker_xtime_priv_cleanup (this);
-
-                        ret = init_xtime_priv (this, options);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to initialize xtime private, "
-                                        "xtime updation will fail");
-                        } else {
-                                priv->feature_enabled |= GF_XTIME;
-                                data = dict_get (options, "gsync-force-xtime");
-                                if (!data)
-                                        goto out;
-                                ret = gf_string2boolean (data->data, &flag);
-                                if (ret == 0 && flag)
-                                        priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
-                        }
-                }
-        }
+    if (priv->feature_enabled) {
+        if (version >= 0)
+            priv->version = version;
+        else
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Invalid quota "
+                   "version %d",
+                   priv->version);
+    }
+
+    data = dict_get(options, "xtime");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true) {
+            marker_xtime_priv_cleanup(this);
+
+            ret = init_xtime_priv(this, options);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "failed to initialize xtime private, "
+                       "xtime updation will fail");
+            } else {
+                priv->feature_enabled |= GF_XTIME;
+                data = dict_get(options, "gsync-force-xtime");
+                if (!data)
+                    goto out;
+                ret = gf_string2boolean(data->data, &flag);
+                if (ret == 0 && flag)
+                    priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
+            }
+        }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-init (xlator_t *this)
-{
-        dict_t        *options = NULL;
-        data_t        *data    = NULL;
-        int32_t        ret     = 0;
-        gf_boolean_t   flag    = _gf_false;
-        marker_conf_t *priv    = NULL;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "marker translator needs subvolume defined.");
-                return -1;
-        }
+init(xlator_t *this)
+{
+    dict_t *options = NULL;
+    data_t *data = NULL;
+    int32_t ret = 0;
+    gf_boolean_t flag = _gf_false;
+    marker_conf_t *priv = NULL;
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling.");
-                return -1;
-        }
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "marker translator needs subvolume defined.");
+        return -1;
+    }
 
-        options = this->options;
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "Volume is dangling.");
+        return -1;
+    }
 
-        ALLOCATE_OR_GOTO (this->private, marker_conf_t, err);
+    options = this->options;
 
-        priv = this->private;
+    ALLOCATE_OR_GOTO(this->private, marker_conf_t, err);
 
-        priv->feature_enabled = 0;
-        priv->version = 0;
+    priv = this->private;
 
-        LOCK_INIT (&priv->lock);
+    priv->feature_enabled = 0;
+    priv->version = 0;
 
-        data = dict_get (options, "quota");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true)
-                        priv->feature_enabled |= GF_QUOTA;
-        }
+    LOCK_INIT(&priv->lock);
 
-        data = dict_get (options, "inode-quota");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true)
-                        priv->feature_enabled |= GF_INODE_QUOTA;
-        }
+    data = dict_get(options, "quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_QUOTA;
+    }
 
-        data = dict_get (options, "quota-version");
-        if (data)
-                ret = gf_string2int32 (data->data, &priv->version);
+    data = dict_get(options, "inode-quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_INODE_QUOTA;
+    }
 
-        if (priv->feature_enabled && priv->version < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid quota version %d",
-                        priv->version);
-                goto err;
-        }
+    data = dict_get(options, "quota-version");
+    if (data)
+        ret = gf_string2int32(data->data, &priv->version);
 
-        data = dict_get (options, "xtime");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true) {
-                        ret = init_xtime_priv (this, options);
-                        if (ret < 0)
-                                goto err;
-
-                        priv->feature_enabled |= GF_XTIME;
-                        data = dict_get (options, "gsync-force-xtime");
-                        if (!data)
-                                goto cont;
-                        ret = gf_string2boolean (data->data, &flag);
-                        if (ret == 0 && flag)
-                                priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
-                }
-        }
+    if ((ret == 0) && priv->feature_enabled && priv->version < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid quota version %d",
+               priv->version);
+        goto err;
+    }
 
- cont:
-        this->local_pool = mem_pool_new (marker_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
+    data = dict_get(options, "xtime");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true) {
+            ret = init_xtime_priv(this, options);
+            if (ret < 0)
                 goto err;
-        }
 
-        return 0;
+            priv->feature_enabled |= GF_XTIME;
+            data = dict_get(options, "gsync-force-xtime");
+            if (!data)
+                goto cont;
+            ret = gf_string2boolean(data->data, &flag);
+            if (ret == 0 && flag)
+                priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
+        }
+    }
+
+cont:
+    this->local_pool = mem_pool_new(marker_local_t, 128);
+    if (!this->local_pool) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    return 0;
 err:
-        marker_priv_cleanup (this);
+    marker_priv_cleanup(this);
 
-        return -1;
+    return -1;
 }
 
 int32_t
-marker_forget (xlator_t *this, inode_t *inode)
+marker_forget(xlator_t *this, inode_t *inode)
 {
-        marker_inode_ctx_t *ctx   = NULL;
-        uint64_t            value = 0;
+    marker_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
 
-        if (inode_ctx_del (inode, this, &value) != 0)
-                goto out;
+    if (inode_ctx_del(inode, this, &value) != 0)
+        goto out;
 
-        ctx = (marker_inode_ctx_t *)(unsigned long)value;
-        if (ctx == NULL) {
-                goto out;
-        }
+    ctx = (marker_inode_ctx_t *)(unsigned long)value;
+    if (ctx == NULL) {
+        goto out;
+    }
 
-        mq_forget (this, ctx->quota_ctx);
+    mq_forget(this, ctx->quota_ctx);
 
-        GF_FREE (ctx);
+    GF_FREE(ctx);
 out:
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        marker_priv_cleanup (this);
+    marker_priv_cleanup(this);
 }
 
 struct xlator_fops fops = {
-        .lookup      = marker_lookup,
-        .create      = marker_create,
-        .mkdir       = marker_mkdir,
-        .writev      = marker_writev,
-        .truncate    = marker_truncate,
-        .ftruncate   = marker_ftruncate,
-        .symlink     = marker_symlink,
-        .link        = marker_link,
-        .unlink      = marker_unlink,
-        .rmdir       = marker_rmdir,
-        .rename      = marker_rename,
-        .mknod       = marker_mknod,
-        .setxattr    = marker_setxattr,
-        .fsetxattr   = marker_fsetxattr,
-        .setattr     = marker_setattr,
-        .fsetattr    = marker_fsetattr,
-        .removexattr = marker_removexattr,
-        .getxattr    = marker_getxattr,
-        .readdirp    = marker_readdirp,
-	.fallocate   = marker_fallocate,
-	.discard     = marker_discard,
-        .zerofill    = marker_zerofill,
+    .lookup = marker_lookup,
+    .create = marker_create,
+    .mkdir = marker_mkdir,
+    .writev = marker_writev,
+    .truncate = marker_truncate,
+    .ftruncate = marker_ftruncate,
+    .symlink = marker_symlink,
+    .link = marker_link,
+    .unlink = marker_unlink,
+    .rmdir = marker_rmdir,
+    .rename = marker_rename,
+    .mknod = marker_mknod,
+    .setxattr = marker_setxattr,
+    .fsetxattr = marker_fsetxattr,
+    .setattr = marker_setattr,
+    .fsetattr = marker_fsetattr,
+    .removexattr = marker_removexattr,
+    .getxattr = marker_getxattr,
+    .readdirp = marker_readdirp,
+    .fallocate = marker_fallocate,
+    .discard = marker_discard,
+    .zerofill = marker_zerofill,
 };
 
-struct xlator_cbks cbks = {
-        .forget = marker_forget
-};
+struct xlator_cbks cbks = {.forget = marker_forget};
 
 struct volume_options options[] = {
-        {.key = {"volume-uuid"}},
-        {.key = {"timestamp-file"}},
-        {.key = {"quota"}},
-        {.key = {"inode-quota"} },
-        {.key = {"xtime"}},
-        {.key = {"gsync-force-xtime"}},
-        {.key = {"quota-version"} },
-        {.key = {NULL}}
+    {.key = {"volume-uuid"}, .default_value = "{{ volume.id }}"},
+    {.key = {"timestamp-file"}},
+    {
+        .key = {"quota"},
+        .op_version = {1},
+        .flags = OPT_FLAG_NONE,
+        .tags = {},
+    },
+    {
+        .key = {"inode-quota"},
+        .op_version = {1},
+        .flags = OPT_FLAG_NONE,
+        .tags = {},
+    },
+    {
+        .key = {"xtime"},
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+        .tags = {},
+    },
+    {
+        .key = {"gsync-force-xtime"},
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+        .tags = {},
+    },
+    {
+        .key = {"quota-version"},
+        .flags = OPT_FLAG_NONE,
+    },
+    {.key = {NULL}}};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "marker",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/marker/src/marker.h b/xlators/features/marker/src/marker.h
index 4726880b82f..4821094c14b 100644
--- a/xlators/features/marker/src/marker.h
+++ b/xlators/features/marker/src/marker.h
@@ -11,138 +11,137 @@
 #define _MARKER_H
 
 #include "marker-quota.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-uuid.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/call-stub.h>
 
 #define MARKER_XATTR_PREFIX "trusted.glusterfs"
-#define XTIME               "xtime"
-#define VOLUME_MARK         "volume-mark"
-#define VOLUME_UUID         "volume-uuid"
-#define TIMESTAMP_FILE      "timestamp-file"
+#define XTIME "xtime"
+#define VOLUME_MARK "volume-mark"
+#define VOLUME_UUID "volume-uuid"
+#define TIMESTAMP_FILE "timestamp-file"
 
 enum {
-        GF_QUOTA             = 1,
-        GF_XTIME             = 2,
-        GF_XTIME_GSYNC_FORCE = 4,
-        GF_INODE_QUOTA       = 8,
+    GF_QUOTA = 1,
+    GF_XTIME = 2,
+    GF_XTIME_GSYNC_FORCE = 4,
+    GF_INODE_QUOTA = 8,
 };
 
 /*initialize the local variable*/
-#define MARKER_INIT_LOCAL(_frame,_local) do {                   \
-                _frame->local = _local;                         \
-                _local->pid = _frame->root->pid;                \
-                memset (&_local->loc, 0, sizeof (loc_t));       \
-                _local->ref = 1;                                \
-                _local->uid = -1;                               \
-                _local->gid = -1;                               \
-                LOCK_INIT (&_local->lock);                      \
-                _local->oplocal = NULL;                         \
-        } while (0)
+#define MARKER_INIT_LOCAL(_frame, _local)                                      \
+    do {                                                                       \
+        _frame->local = _local;                                                \
+        _local->pid = _frame->root->pid;                                       \
+        memset(&_local->loc, 0, sizeof(loc_t));                                \
+        _local->ref = 1;                                                       \
+        _local->uid = -1;                                                      \
+        _local->gid = -1;                                                      \
+        LOCK_INIT(&_local->lock);                                              \
+        _local->oplocal = NULL;                                                \
+    } while (0)
 
 /* try alloc and if it fails, goto label */
-#define ALLOCATE_OR_GOTO(var, type, label) do {                  \
-                var = GF_CALLOC (sizeof (type), 1,               \
-                                 gf_marker_mt_##type);           \
-                if (!var) {                                      \
-                        gf_log (this->name, GF_LOG_ERROR,        \
-                                "out of memory :(");             \
-                        goto label;                              \
-                }                                                \
-        } while (0)
-
-#define _MARKER_SET_UID_GID(dest, src)          \
-        do {                                    \
-                if (src->uid != -1 &&           \
-                    src->gid != -1) {           \
-                        dest->uid = src->uid;   \
-                        dest->gid = src->gid;   \
-                }                               \
-        } while (0)
-
-#define MARKER_SET_UID_GID(frame, dest, src)                    \
-        do {                                                    \
-                _MARKER_SET_UID_GID (dest, src);                \
-                frame->root->uid = 0;                           \
-                frame->root->gid = 0;                           \
-                frame->cookie = (void *) _GF_UID_GID_CHANGED;   \
-        } while (0)
-
-#define MARKER_RESET_UID_GID(frame, dest, src)                  \
-        do {                                                    \
-                _MARKER_SET_UID_GID (dest, src);                \
-                frame->cookie = NULL;                           \
-        } while (0)
-
-#define MARKER_STACK_UNWIND(fop, frame, params...)              \
-        do {                                                    \
-                quota_local_t *_local = NULL;                   \
-                if (frame) {                                    \
-                        _local = frame->local;                  \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                if (_local)                                     \
-                        marker_local_unref (_local);            \
-        } while (0)
-
-struct marker_local{
-        uint32_t        timebuf[2];
-        pid_t           pid;
-        loc_t           loc;
-        loc_t           parent_loc;
-        uid_t           uid;
-        gid_t           gid;
-        int32_t         ref;
-        uint32_t        ia_nlink;
-        struct iatt     buf;
-        gf_lock_t       lock;
-        mode_t          mode;
-        int32_t         err;
-        call_stub_t    *stub;
-        call_frame_t   *lk_frame;
-        quota_meta_t    contribution;
-        struct marker_local *oplocal;
-
-        /* marker quota specific */
-        int64_t delta;
-        int64_t d_off;
-        int64_t sum;
-        int64_t size;
-        int32_t hl_count;
-        int32_t dentry_child_count;
-
-        fd_t         *fd;
-        call_frame_t *frame;
-
-        quota_inode_ctx_t    *ctx;
-        inode_contribution_t *contri;
-
-        int xflag;
-        dict_t *xdata;
-        gf_boolean_t skip_txn;
+#define ALLOCATE_OR_GOTO(var, type, label)                                     \
+    do {                                                                       \
+        var = GF_CALLOC(sizeof(type), 1, gf_marker_mt_##type);                 \
+        if (!var) {                                                            \
+            gf_log(this->name, GF_LOG_ERROR, "out of memory :(");              \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define _MARKER_SET_UID_GID(dest, src)                                         \
+    do {                                                                       \
+        if (src->uid != -1 && src->gid != -1) {                                \
+            dest->uid = src->uid;                                              \
+            dest->gid = src->gid;                                              \
+        }                                                                      \
+    } while (0)
+
+#define MARKER_SET_UID_GID(frame, dest, src)                                   \
+    do {                                                                       \
+        _MARKER_SET_UID_GID(dest, src);                                        \
+        frame->root->uid = 0;                                                  \
+        frame->root->gid = 0;                                                  \
+        frame->cookie = (void *)_GF_UID_GID_CHANGED;                           \
+    } while (0)
+
+#define MARKER_RESET_UID_GID(frame, dest, src)                                 \
+    do {                                                                       \
+        _MARKER_SET_UID_GID(dest, src);                                        \
+        frame->cookie = NULL;                                                  \
+    } while (0)
+
+#define MARKER_STACK_UNWIND(fop, frame, params...)                             \
+    do {                                                                       \
+        quota_local_t *_local = NULL;                                          \
+        if (frame) {                                                           \
+            _local = frame->local;                                             \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (_local)                                                            \
+            marker_local_unref(_local);                                        \
+    } while (0)
+
+struct marker_local {
+    uint32_t timebuf[2];
+    pid_t pid;
+    loc_t loc;
+    loc_t parent_loc;
+    uid_t uid;
+    gid_t gid;
+    int32_t ref;
+    uint32_t ia_nlink;
+    struct iatt buf;
+    gf_lock_t lock;
+    mode_t mode;
+    int32_t err;
+    call_stub_t *stub;
+    call_frame_t *lk_frame;
+    quota_meta_t contribution;
+    struct marker_local *oplocal;
+
+    /* marker quota specific */
+    int64_t delta;
+    int64_t d_off;
+    int64_t sum;
+    int64_t size;
+    int32_t hl_count;
+    int32_t dentry_child_count;
+
+    fd_t *fd;
+    call_frame_t *frame;
+
+    quota_inode_ctx_t *ctx;
+    inode_contribution_t *contri;
+
+    int xflag;
+    dict_t *xdata;
+    gf_boolean_t skip_txn;
 };
 typedef struct marker_local marker_local_t;
 
 #define quota_local_t marker_local_t
 
 struct marker_inode_ctx {
-        struct quota_inode_ctx *quota_ctx;
+    struct quota_inode_ctx *quota_ctx;
 };
 typedef struct marker_inode_ctx marker_inode_ctx_t;
 
-struct marker_conf{
-        char         feature_enabled;
-        char        *size_key;
-        char        *dirty_key;
-        char        *volume_uuid;
-        uuid_t      volume_uuid_bin;
-        char        *timestamp_file;
-        char        *marker_xattr;
-        uint64_t     quota_lk_owner;
-        gf_lock_t    lock;
-        int32_t      version;
+struct marker_conf {
+    char feature_enabled;
+    char *size_key;
+    char *dirty_key;
+    char *volume_uuid;
+    uuid_t volume_uuid_bin;
+    char *timestamp_file;
+    char *marker_xattr;
+    uint64_t quota_lk_owner;
+    gf_lock_t lock;
+    int32_t version;
 };
 typedef struct marker_conf marker_conf_t;
 
diff --git a/xlators/experimental/posix2/common/Makefile.am b/xlators/features/metadisp/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/posix2/common/Makefile.am
+++ b/xlators/features/metadisp/Makefile.am
diff --git a/xlators/features/metadisp/src/Makefile.am b/xlators/features/metadisp/src/Makefile.am
new file mode 100644
index 00000000000..1520ad8c424
--- /dev/null
+++ b/xlators/features/metadisp/src/Makefile.am
@@ -0,0 +1,38 @@
+noinst_PYTHON = gen-fops.py
+
+EXTRA_DIST = fops-tmpl.c
+
+xlator_LTLIBRARIES = metadisp.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+nodist_metadisp_la_SOURCES = fops.c
+
+BUILT_SOURCES = fops.c
+
+metadisp_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+metadisp_la_SOURCES = metadisp.c \
+		metadisp-unlink.c \
+		metadisp-stat.c \
+		metadisp-lookup.c \
+		metadisp-readdir.c \
+		metadisp-create.c \
+		metadisp-open.c \
+		metadisp-fsync.c \
+		metadisp-setattr.c \
+		backend.c
+
+metadisp_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = metadisp.h metadisp-fops.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+fops.c: fops-tmpl.c $(top_srcdir)/libglusterfs/src/generator.py gen-fops.py
+	PYTHONPATH=$(top_srcdir)/libglusterfs/src \
+	$(PYTHON) $(srcdir)/gen-fops.py $(srcdir)/fops-tmpl.c > $@
+
+CLEANFILES = $(nodist_metadisp_la_SOURCES)
diff --git a/xlators/features/metadisp/src/backend.c b/xlators/features/metadisp/src/backend.c
new file mode 100644
index 00000000000..ee2c25bfaa7
--- /dev/null
+++ b/xlators/features/metadisp/src/backend.c
@@ -0,0 +1,45 @@
+#define GFID_STR_LEN 37
+
+#include "metadisp.h"
+
+/*
+ * backend.c
+ *
+ * functions responsible for converting user-facing paths to backend-style
+ * "/$GFID" paths.
+ */
+
+int32_t
+build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc)
+{
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    char gfid_buf[GFID_STR_LEN + 1] = {
+        0,
+    };
+    char *path = NULL;
+
+    GF_VALIDATE_OR_GOTO("metadisp", src_loc, out);
+    GF_VALIDATE_OR_GOTO("metadisp", dst_loc, out);
+
+    loc_copy(dst_loc, src_loc);
+    memcpy(dst_loc->pargfid, root, sizeof(root));
+    GF_FREE((char *)dst_loc->path);  // we are overwriting path so nuke
+                                     // whatever loc_copy gave us
+
+    uuid_utoa_r(gfid, gfid_buf);
+
+    path = GF_CALLOC(GFID_STR_LEN + 1, sizeof(char),
+                     gf_common_mt_char);  // freed via loc_wipe
+
+    path[0] = '/';
+    strncpy(path + 1, gfid_buf, GFID_STR_LEN);
+    path[GFID_STR_LEN] = 0;
+    dst_loc->path = path;
+    if (src_loc->name)
+        dst_loc->name = strrchr(dst_loc->path, '/');
+    if (dst_loc->name)
+        dst_loc->name++;
+    return 0;
+out:
+    return -1;
+}
diff --git a/xlators/features/metadisp/src/fops-tmpl.c b/xlators/features/metadisp/src/fops-tmpl.c
new file mode 100644
index 00000000000..4385b7dd5b7
--- /dev/null
+++ b/xlators/features/metadisp/src/fops-tmpl.c
@@ -0,0 +1,10 @@
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/xlator.h>
+#include "metadisp.h"
+#include "metadisp-fops.h"
+
+#pragma generate
diff --git a/xlators/features/metadisp/src/gen-fops.py b/xlators/features/metadisp/src/gen-fops.py
new file mode 100644
index 00000000000..8b5e120fdec
--- /dev/null
+++ b/xlators/features/metadisp/src/gen-fops.py
@@ -0,0 +1,160 @@
+#!/usr/bin/python
+
+import sys
+from generator import fop_subs, generate
+
+FN_METADATA_CHILD_GENERIC = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                 @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ metadata");
+  STACK_WIND (frame, default_@NAME@_cbk,
+              METADATA_CHILD(this), METADATA_CHILD(this)->fops->@NAME@,
+              @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_GENERIC_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ generic");
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          DATA_CHILD(this), DATA_CHILD(this)->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_DATAFD_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ datafd");
+  xlator_t *child = NULL;
+  child = DATA_CHILD(this);
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          child, child->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_DATALOC_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ dataloc");
+  loc_t backend_loc = {
+      0,
+  };
+  if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+      goto unwind;
+  }
+  xlator_t *child = NULL;
+  child = DATA_CHILD(this);
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          child, child->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+
+unwind:
+  STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+  return 0;
+}
+"""
+
+FOPS_LINE_TEMPLATE = "\t.@NAME@ = metadisp_@NAME@,"
+
+skipped = [
+    "readdir",
+    "readdirp",
+    "lookup",
+    "fsync",
+    "stat",
+    "open",
+    "create",
+    "unlink",
+    "setattr",
+    # TODO: implement "inodelk",
+]
+
+
+def gen_fops():
+    done = skipped
+
+    #
+    # these are fops that wind to the DATA_CHILD
+    #
+    # NOTE: re-written in order from google doc:
+    #          https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q
+    for name in [
+        "writev",
+        "readv",
+        "ftruncate",
+        "zerofill",
+        "discard",
+        "seek",
+        "fstat",
+    ]:
+        done = done + [name]
+        print(generate(FN_DATAFD_TEMPLATE, name, fop_subs))
+
+    for name in ["truncate"]:
+        done = done + [name]
+        print(generate(FN_DATALOC_TEMPLATE, name, fop_subs))
+
+    # these are fops that operate solely on dentries, folders,
+    # or extended attributes. Therefore, they must always
+    # wind to METADATA_CHILD and should never perform
+    # any path rewriting
+    #
+    # NOTE: re-written in order from google doc:
+    #          https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q
+    for name in [
+        "mkdir",
+        "symlink",
+        "link",
+        "rename",
+        "mknod",
+        "opendir",
+        # "readdir,  # special-cased
+        # "readdirp, # special-cased
+        "fsyncdir",
+        # "setattr", # special-cased
+        "readlink",
+        "fentrylk",
+        "access",
+        # TODO: these wind to both,
+        # data for backend-attributes and metadata for the rest
+        "xattrop",
+        "setxattr",
+        "getxattr",
+        "removexattr",
+        "fgetxattr",
+        "fsetxattr",
+        "fremovexattr",
+    ]:
+
+        done = done + [name]
+        print(generate(FN_METADATA_CHILD_GENERIC, name, fop_subs))
+
+    print("struct xlator_fops fops = {")
+    for name in done:
+        print(generate(FOPS_LINE_TEMPLATE, name, fop_subs))
+
+    print("};")
+
+
+for l in open(sys.argv[1], "r").readlines():
+    if l.find("#pragma generate") != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_fops()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/metadisp/src/metadisp-create.c b/xlators/features/metadisp/src/metadisp-create.c
new file mode 100644
index 00000000000..f8c9798dd59
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-create.c
@@ -0,0 +1,101 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * Create, like stat, is a two-step process. We send a create
+ * to the METADATA_CHILD, then send another create to the DATA_CHILD.
+ *
+ * We do the metadata child first to ensure that the ACLs are enforced.
+ */
+
+int32_t
+metadisp_create_dentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           inode_t *inode, struct iatt *buf,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                       dict_t *xdata)
+{
+    // create the backend data inode
+    STACK_WIND(frame, metadisp_create_dentry_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+}
+
+int32_t
+metadisp_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = cookie;
+    if (op_ret != 0) {
+        STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+    }
+
+    if (stub == NULL) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+}
+
+int32_t
+metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+
+    loc_t backend_loc = {
+        0,
+    };
+    call_stub_t *stub = NULL;
+    uuid_t *gfid_req = NULL;
+
+    RESOLVE_GFID_REQ(xdata, gfid_req, out);
+
+    if (build_backend_loc(*gfid_req, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    frame->local = loc;
+
+    stub = fop_create_stub(frame, metadisp_create_resume, &backend_loc, flags,
+                           mode, umask, fd, xdata);
+
+    STACK_WIND_COOKIE(frame, metadisp_create_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->create, loc, flags, mode,
+                      umask, fd, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+out:
+    return -1;
+}
diff --git a/xlators/features/metadisp/src/metadisp-fops.h b/xlators/features/metadisp/src/metadisp-fops.h
new file mode 100644
index 00000000000..56dd427cf34
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-fops.h
@@ -0,0 +1,51 @@
+#ifndef GF_METADISP_FOPS_H_
+#define GF_METADISP_FOPS_H_
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/glusterfs.h>
+
+#include <sys/types.h>
+
+/* fops in here are defined in their own file. Every other fop is just defined
+ * inline of fops.c */
+
+int
+metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata);
+
+int
+metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *dict);
+
+int
+metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int
+metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+int
+metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata);
+
+int
+metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int
+metadisp_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+int
+metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata);
+
+int
+metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata);
+
+int
+metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+#endif
diff --git a/xlators/features/metadisp/src/metadisp-fsync.c b/xlators/features/metadisp/src/metadisp-fsync.c
new file mode 100644
index 00000000000..2e46fa84eac
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-fsync.c
@@ -0,0 +1,54 @@
+
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+metadisp_fsync_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      int32_t flags, dict_t *xdata)
+{
+    STACK_WIND(frame, default_fsync_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    stub = fop_fsync_stub(frame, metadisp_fsync_resume, fd, flags, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_fsync_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-lookup.c b/xlators/features/metadisp/src/metadisp-lookup.c
new file mode 100644
index 00000000000..27d90c9f746
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-lookup.c
@@ -0,0 +1,90 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * Lookup, like stat, is a two-step process for grabbing the metadata details
+ * as well as the data details.
+ */
+
+int32_t
+metadisp_backend_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, dict_t *xdata,
+                            struct iatt *postparent)
+{
+    METADISP_TRACE("backend_lookup_cbk");
+    if (op_errno == ENOENT) {
+        op_errno = ENODATA;
+        op_ret = -1;
+    }
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
+}
+
+int32_t
+metadisp_backend_lookup_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               dict_t *xdata)
+{
+    METADISP_TRACE("backend_lookup_resume");
+    loc_t backend_loc = {
+        0,
+    };
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    STACK_WIND(frame, metadisp_backend_lookup_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->lookup, &backend_loc, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = NULL;
+    stub = cookie;
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!IA_ISREG(buf->ia_type)) {
+        goto unwind;
+    } else if (!stub) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    METADISP_TRACE("resuming stub");
+
+    // memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t));
+    call_resume(stub);
+    return 0;
+unwind:
+    METADISP_TRACE("unwinding %d %d", op_ret, op_errno);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    return 0;
+}
+
+int32_t
+metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    METADISP_TRACE("lookup");
+    call_stub_t *stub = NULL;
+    stub = fop_lookup_stub(frame, metadisp_backend_lookup_resume, loc, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_lookup_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-open.c b/xlators/features/metadisp/src/metadisp-open.c
new file mode 100644
index 00000000000..64814afe636
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-open.c
@@ -0,0 +1,70 @@
+#include <glusterfs/call-stub.h>
+#include "metadisp.h"
+
+int32_t
+metadisp_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    METADISP_TRACE("got open results %d %d", op_ret, op_errno);
+
+    call_stub_t *stub = NULL;
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!stub) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_open_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     int32_t flags, fd_t *fd, dict_t *xdata)
+{
+    STACK_WIND_COOKIE(frame, metadisp_open_cbk, NULL, DATA_CHILD(this),
+                      DATA_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    loc_t backend_loc = {
+        0,
+    };
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    stub = fop_open_stub(frame, metadisp_open_resume, &backend_loc, flags, fd,
+                         xdata);
+    STACK_WIND_COOKIE(frame, metadisp_open_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(open, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-readdir.c b/xlators/features/metadisp/src/metadisp-readdir.c
new file mode 100644
index 00000000000..5f840b1e88f
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-readdir.c
@@ -0,0 +1,65 @@
+#include "metadisp.h"
+
+/**
+ * With a change to the posix xlator, readdir and readdirp are shockingly
+ * simple.
+ *
+ * The issue with separating the backend data of the files
+ * with the metadata is that readdirs must now read from multiple sources
+ * to coalesce the directory entries.
+ *
+ * The way we do this is to tell the METADATA_CHILD that when it's
+ * running readdirp, each file entry should have a stat wound to
+ * 'stat-source-of-truth'.
+ *
+ * see metadisp_stat for how it handles winds _from_posix.
+ */
+
+int32_t
+metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+    /*
+     * Always use readdirp, even if the original was readdir. Why? Because NFS.
+     * There are multiple translations between Gluster, UNIX, and NFS stat
+     * structures in that path. One of them uses the type etc. from the stat
+     * structure, which is only filled in by readdirp. If we use readdir, the
+     * entries do actually go all the way back to the client and are visible in
+     * getdents, but then the readdir throws them away because of the
+     * uninitialized type.
+     */
+    GF_UNUSED int32_t ret;
+    if (!xdata) {
+        xdata = dict_new();
+    }
+
+    // ret = dict_set_int32 (xdata, "list-xattr", 1);
+
+    // I'm my own source of truth!
+    ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this);
+
+    STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this),
+               METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+    return 0;
+}
+
+int32_t
+metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+    if (!xdata) {
+        xdata = dict_new();
+    }
+    GF_UNUSED int32_t ret;
+    // ret = dict_set_int32 (xdata, "list-xattr", 1);
+
+    // I'm my own source of truth!
+    ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this);
+
+    STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this),
+               METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-setattr.c b/xlators/features/metadisp/src/metadisp-setattr.c
new file mode 100644
index 00000000000..6991cf644f3
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-setattr.c
@@ -0,0 +1,90 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+metadisp_backend_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *statpre, struct iatt *statpost,
+                             dict_t *xdata)
+
+{
+    METADISP_TRACE("backend_setattr_cbk");
+    if (op_errno == ENOENT) {
+        op_errno = ENODATA;
+        op_ret = -1;
+    }
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    return 0;
+}
+
+int32_t
+metadisp_backend_setattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                struct iatt *stbuf, int32_t valid,
+                                dict_t *xdata)
+
+{
+    METADISP_TRACE("backend_setattr_resume");
+    loc_t backend_loc = {
+        0,
+    };
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    STACK_WIND(frame, metadisp_backend_setattr_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->setattr, &backend_loc, stbuf, valid,
+               xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(setattr, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = NULL;
+    stub = cookie;
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!IA_ISREG(statpost->ia_type)) {
+        goto unwind;
+    } else if (!stub) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    METADISP_TRACE("resuming stub");
+    call_resume(stub);
+    return 0;
+unwind:
+    METADISP_TRACE("unwinding %d %d", op_ret, op_errno);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    return 0;
+}
+
+int32_t
+metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    METADISP_TRACE("setattr");
+    call_stub_t *stub = NULL;
+    stub = fop_setattr_stub(frame, metadisp_backend_setattr_resume, loc, stbuf,
+                            valid, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_setattr_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                      xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-stat.c b/xlators/features/metadisp/src/metadisp-stat.c
new file mode 100644
index 00000000000..b06d0dbcddd
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-stat.c
@@ -0,0 +1,124 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * The stat flow in METADISP is complicated because we must
+ * do ensure a few things:
+ *    1. stat, on the path within the metadata layer,
+ *       MUST get the backend FD of the data layer.
+ *        --- we wind to the metadata layer, then the data layer.
+ *
+ *    2. the metadata layer MUST be able to ask the data
+ *       layer for stat information.
+ *        --- this is 'syncop-internal-from-posix'
+ *
+ *    3. when the metadata exists BUT the data is missing,
+ *       we MUST mark the backend file as bad and heal it.
+ */
+
+int32_t
+metadisp_stat_backend_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
+{
+    METADISP_TRACE("got backend stat results %d %d", op_ret, op_errno);
+    if (op_errno == ENOENT) {
+        STACK_UNWIND_STRICT(open, frame, -1, ENODATA, NULL, NULL);
+        return 0;
+    }
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
+{
+    METADISP_TRACE("winding stat to path %s", loc->path);
+    if (gf_uuid_is_null(loc->gfid)) {
+        METADISP_TRACE("bad object, sending EUCLEAN");
+        STACK_UNWIND_STRICT(open, frame, -1, EUCLEAN, NULL, NULL);
+        return 0;
+    }
+
+    STACK_WIND(frame, metadisp_stat_backend_cbk, SECOND_CHILD(this),
+               SECOND_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    METADISP_TRACE("got stat results %d %d", op_ret, op_errno);
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    // only use the stub for the files
+    if (!IA_ISREG(buf->ia_type)) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    int32_t ret = 0;
+    loc_t backend_loc = {
+        0,
+    };
+    METADISP_FILTER_ROOT(stat, loc, xdata);
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    if (dict_get_int32(xdata, "syncop-internal-from-posix", &ret) == 0) {
+        // if we've just been sent a stat from posix, then we know
+        // that we must send down a stat for a file to the second child.
+        //
+        // that means we can skip the stat for the first child and just
+        // send to the data disk.
+        METADISP_TRACE("got syncop-internal-from-posix");
+        STACK_WIND(frame, default_stat_cbk, DATA_CHILD(this),
+                   DATA_CHILD(this)->fops->stat, &backend_loc, xdata);
+        return 0;
+    }
+
+    // we do not know if the request is for a file, folder, etc. wind
+    // to first child to find out.
+    stub = fop_stat_stub(frame, metadisp_stat_resume, &backend_loc, xdata);
+    METADISP_TRACE("winding stat to first child %s", loc->path);
+    STACK_WIND_COOKIE(frame, metadisp_stat_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(stat, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-unlink.c b/xlators/features/metadisp/src/metadisp-unlink.c
new file mode 100644
index 00000000000..1f6a8eb35ce
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-unlink.c
@@ -0,0 +1,160 @@
+
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * The unlink flow in metadisp is complicated because we must
+ * do ensure that UNLINK causes both the metadata objects
+ * to get removed and the data objects to get removed.
+ */
+
+int32_t
+metadisp_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int xflag, dict_t *xdata)
+{
+    METADISP_TRACE("winding backend unlink to path %s", loc->path);
+    STACK_WIND(frame, default_unlink_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    METADISP_TRACE(". %d %d", op_ret, op_errno);
+
+    int ret = 0;
+    call_stub_t *stub = NULL;
+    int nlink = 0;
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, &nlink);
+    if (ret != 0) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto unwind;
+    }
+    METADISP_TRACE("frontend hardlink count %d %d", ret, nlink);
+    if (nlink > 1) {
+        goto unwind;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
+}
+
+int32_t
+metadisp_unlink_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
+{
+    call_stub_t *stub = NULL;
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    // fail fast on empty gfid so we don't loop forever
+    if (gf_uuid_is_null(buf->ia_gfid)) {
+        op_ret = -1;
+        op_errno = ENODATA;
+        goto unwind;
+    }
+
+    // fill gfid since the stub is incomplete
+    memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t));
+    memcpy(stub->args.loc.pargfid, postparent->ia_gfid, sizeof(uuid_t));
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    loc_t backend_loc = {
+        0,
+    };
+
+    if (gf_uuid_is_null(loc->gfid)) {
+        METADISP_TRACE("winding lookup for unlink to path %s", loc->path);
+
+        // loop back to ourselves after a lookup
+        stub = fop_unlink_stub(frame, metadisp_unlink, loc, xflag, xdata);
+        STACK_WIND_COOKIE(frame, metadisp_unlink_lookup_cbk, stub,
+                          METADATA_CHILD(this),
+                          METADATA_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+    }
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    //
+    // ensure we get the link count on the unlink response, so we can
+    // account for hardlinks before winding to the backend.
+    // NOTE:
+    //   multiple xlators use GF_REQUEST_LINK_COUNT_XDATA. confirmation
+    //   is needed to ensure that multiple requests will work in the same
+    //   xlator stack.
+    //
+    if (!xdata) {
+        xdata = dict_new();
+    }
+    dict_set_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
+
+    METADISP_TRACE("winding frontend unlink to path %s", loc->path);
+    stub = fop_unlink_stub(frame, metadisp_unlink_resume, &backend_loc, xflag,
+                           xdata);
+
+    STACK_WIND_COOKIE(frame, metadisp_unlink_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(unlink, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp.c b/xlators/features/metadisp/src/metadisp.c
new file mode 100644
index 00000000000..3c8f150cebc
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp.c
@@ -0,0 +1,46 @@
+#include <glusterfs/call-stub.h>
+
+#include "metadisp.h"
+#include "metadisp-fops.h"
+
+int32_t
+init(xlator_t *this)
+{
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "not configured with children. exiting");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    return;
+}
+
+/* defined in fops.c */
+struct xlator_fops fops;
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .op_version = {1},
+    .identifier = "metadisp",
+    .category = GF_EXPERIMENTAL,
+};
diff --git a/xlators/features/metadisp/src/metadisp.h b/xlators/features/metadisp/src/metadisp.h
new file mode 100644
index 00000000000..c8fd7a13c04
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp.h
@@ -0,0 +1,45 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef GF_METADISP_H_
+#define GF_METADISP_H_
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+#define METADATA_CHILD(_this) FIRST_CHILD(_this)
+#define DATA_CHILD(_this) SECOND_CHILD(_this)
+
+int32_t
+build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc);
+
+#define METADISP_TRACE(_args...) gf_log("metadisp", GF_LOG_INFO, _args)
+
+#define METADISP_FILTER_ROOT(_op, _args...)                                    \
+    if (strcmp(loc->path, "/") == 0) {                                         \
+        STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this),           \
+                   METADATA_CHILD(this)->fops->_op, _args);                    \
+        return 0;                                                              \
+    }
+
+#define METADISP_FILTER_ROOT_BY_GFID(_op, _gfid, _args...)                     \
+    if (__is_root_gfid(_gfid)) {                                               \
+        STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this),           \
+                   METADATA_CHILD(this)->fops->_op, _args);                    \
+        return 0;                                                              \
+    }
+
+#define RESOLVE_GFID_REQ(_dict, _dest, _lbl)                                   \
+    VALIDATE_OR_GOTO(dict_get_ptr(_dict, "gfid-req", (void **)&_dest) == 0,    \
+                     _lbl)
+
+#endif /* __TEMPLATE_H__ */
diff --git a/xlators/experimental/posix2/ds/Makefile.am b/xlators/features/namespace/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/posix2/ds/Makefile.am
+++ b/xlators/features/namespace/Makefile.am
diff --git a/xlators/features/namespace/src/Makefile.am b/xlators/features/namespace/src/Makefile.am
new file mode 100644
index 00000000000..e355d42cf4e
--- /dev/null
+++ b/xlators/features/namespace/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = namespace.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+namespace_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+namespace_la_SOURCES = namespace.c
+namespace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = namespace.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/namespace/src/namespace.c b/xlators/features/namespace/src/namespace.c
new file mode 100644
index 00000000000..86c5ebee900
--- /dev/null
+++ b/xlators/features/namespace/src/namespace.c
@@ -0,0 +1,1344 @@
+/*
+ * Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ *
+ * xlators/features/namespace:
+ *      This translator tags each request with a namespace hash,
+ *      which then can be used in later translators to track and
+ *      throttle fops per namespace.
+ */
+
+#include <sys/types.h>
+
+#include <glusterfs/defaults.h>
+#include <glusterfs/hashfn.h>
+#include <glusterfs/logging.h>
+#include "namespace.h"
+
+/* Return codes for common path parsing functions. */
+enum _path_parse_result {
+    PATH_PARSE_RESULT_NO_PATH = 0,
+    PATH_PARSE_RESULT_FOUND = 1,
+    PATH_PARSE_RESULT_IS_GFID = 2,
+};
+
+typedef enum _path_parse_result path_parse_result_t;
+
+/* Clean up an ns_local struct. Wipe a loc (its inode is ref'd, so we're good.)
+ */
+static inline void
+ns_local_cleanup(ns_local_t *local)
+{
+    if (!local) {
+        return;
+    }
+
+    loc_wipe(&local->loc);
+    GF_FREE(local);
+}
+
+/* Create a new ns_local. We ref the inode, fake a new loc struct, and stash
+ * the stub given to us. */
+static inline ns_local_t *
+ns_local_new(call_stub_t *stub, inode_t *inode)
+{
+    ns_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    if (!stub || !inode) {
+        goto out;
+    }
+
+    local = GF_CALLOC(1, sizeof(ns_local_t), 0);
+    if (local == NULL) {
+        goto out;
+    }
+
+    /* Set up a fake loc_t struct to give to the getxattr call. */
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+
+    /* If for some reason inode_ref() fails, then just give up. */
+    if (!loc.inode) {
+        GF_FREE(local);
+        goto out;
+    }
+
+    local->stub = stub;
+    local->loc = loc;
+
+out:
+    return local;
+}
+
+/* Try parsing a path string. If the path string is a GFID, then return
+ * with PATH_PARSE_RESULT_IS_GFID. If we have no namespace (i.e. '/') then
+ * return PATH_PARSE_RESULT_NO_PATH and set the hash to 1. Otherwise, hash the
+ * namespace and store it in the info struct. */
+static path_parse_result_t
+parse_path(ns_info_t *info, const char *path)
+{
+    int len = 0;
+    const char *ns_begin = path;
+    const char *ns_end = NULL;
+
+    if (!path || strlen(path) == 0) {
+        return PATH_PARSE_RESULT_NO_PATH;
+    }
+
+    if (path[0] == '<') {
+        return PATH_PARSE_RESULT_IS_GFID;
+    }
+
+    /* Right now we only want the top-level directory, so
+     * skip the initial '/' and read until the next '/'. */
+    while (*ns_begin == '/') {
+        ns_begin++;
+    }
+
+    /* ns_end will point to the next '/' or NULL if there is no delimiting
+     * '/' (i.e. "/directory" or the top level "/") */
+    ns_end = strchr(ns_begin, '/');
+    len = ns_end ? (ns_end - ns_begin) : strlen(ns_begin);
+
+    if (len != 0) {
+        info->hash = SuperFastHash(ns_begin, len);
+    } else {
+        /* If our substring is empty, then we can hash '/' instead.
+         * '/' is used in the namespace config for the top-level
+         * namespace. */
+        info->hash = SuperFastHash("/", 1);
+    }
+
+    info->found = _gf_true;
+    return PATH_PARSE_RESULT_FOUND;
+}
+
+/* Cache namespace info stored in the stack (info) into the inode. */
+static int
+ns_inode_ctx_put(inode_t *inode, xlator_t *this, ns_info_t *info)
+{
+    ns_info_t *cached_ns_info = NULL;
+    uint64_t ns_as_64 = 0;
+    int ret = -1;
+
+    if (!inode || !this) {
+        gf_log(this ? this->name : "namespace", GF_LOG_WARNING,
+               "Need a valid inode and xlator to cache ns_info.");
+        ret = -1;
+        goto out;
+    }
+
+    cached_ns_info = GF_CALLOC(1, sizeof(ns_info_t), 0);
+
+    /* If we've run out of memory, then return ENOMEM. */
+    if (cached_ns_info == NULL) {
+        gf_log(this->name, GF_LOG_WARNING, "No memory to cache ns_info.");
+        ret = -(ENOMEM);
+        goto out;
+    }
+
+    *cached_ns_info = *info;
+    ns_as_64 = (uint64_t)(uintptr_t)cached_ns_info;
+
+    ret = inode_ctx_put(inode, this, ns_as_64);
+
+    if (ret) {
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (ret && cached_ns_info) {
+        GF_FREE(cached_ns_info);
+    }
+
+    return ret;
+}
+
+/* Retrieve namespace info cached in the inode into the stack for use in later
+ * translators. */
+static int
+ns_inode_ctx_get(inode_t *inode, xlator_t *this, ns_info_t *info)
+{
+    ns_info_t *cached_ns_info = NULL;
+    uint64_t ns_as_64 = 0;
+    int ret = -1;
+
+    if (!inode) {
+        ret = -ENOENT;
+        goto out;
+    }
+
+    ret = inode_ctx_get(inode, this, &ns_as_64);
+
+    if (!ret) {
+        cached_ns_info = (ns_info_t *)(uintptr_t)ns_as_64;
+        *info = *cached_ns_info;
+    }
+
+out:
+    return ret;
+}
+
+/* This callback is the top of the unwind path of our attempt to get the path
+ * manually from the posix translator. We'll try to parse the path returned
+ * if it exists, then cache the hash if possible. Then just return to the
+ * default stub that we provide in the local, since there's nothing else to do
+ * once we've gotten the namespace hash. */
+int32_t
+get_path_resume_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+    path_parse_result_t ret = PATH_PARSE_RESULT_NO_PATH;
+    call_frame_t *resume_frame = NULL;
+    ns_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    ns_info_t *info = NULL;
+    char *path = NULL;
+
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+    stub = local->stub;
+
+    GF_VALIDATE_OR_GOTO(this->name, stub, out);
+    /* Get the ns_info from the frame that we will eventually resume,
+     * not the frame that we're going to destroy (frame). */
+    resume_frame = stub->frame;
+
+    GF_VALIDATE_OR_GOTO(this->name, resume_frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, resume_frame->root, out);
+    info = &resume_frame->root->ns_info;
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    /* If we get a value back for the GET_ANCESTRY_PATH_KEY, then we
+     * try to access it and parse it like a path. */
+    if (!op_ret && !dict_get_str(dict, GET_ANCESTRY_PATH_KEY, &path)) {
+        gf_log(this->name, GF_LOG_DEBUG, "G>P %s retrieved path %s",
+               uuid_utoa(local->loc.gfid), path);
+        /* Now let's parse a path, finally. */
+        ret = parse_path(info, path);
+    }
+
+    if (ret == PATH_PARSE_RESULT_FOUND) {
+        /* If we finally found namespace, then stash it. */
+        ns_inode_ctx_put(local->loc.inode, this, info);
+
+        gf_log(this->name, GF_LOG_DEBUG, "G>P %s %10u namespace found %s",
+               uuid_utoa(local->loc.inode->gfid), info->hash, path);
+    } else if (ret == PATH_PARSE_RESULT_NO_PATH) {
+        gf_log(this->name, GF_LOG_WARNING, "G>P %s has no path",
+               uuid_utoa(local->loc.inode->gfid));
+    } else if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "G>P %s winding failed, still have gfid",
+               uuid_utoa(local->loc.inode->gfid));
+    }
+
+out:
+    /* Make sure to clean up local finally. */
+
+    if (frame) {
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+    }
+
+    if (local) {
+        ns_local_cleanup(local);
+    }
+
+    if (stub) {
+        call_resume(stub);
+    }
+
+    return 0;
+}
+
+/* This function tries first to set a namespace based on the information that
+ * it can retrieve from an `loc_t`. This includes first looking for a cached
+ * namespace in the inode, then trying to parse the path string in the `loc_t`
+ * struct. If this fails, then it will try to call inode_path. */
+static path_parse_result_t
+set_ns_from_loc(const char *fn, call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    path_parse_result_t ret = PATH_PARSE_RESULT_NO_PATH;
+    ns_private_t *priv = (ns_private_t *)this->private;
+    ns_info_t *info = &frame->root->ns_info;
+    char *path = NULL;
+
+    info->hash = 0;
+    info->found = _gf_false;
+
+    if (!priv->tag_namespaces) {
+        return ret;
+    }
+
+    /* This is our first pass at trying to get a path. Try getting
+     * from the inode context, then from the loc's path itself. */
+    if (!loc || !loc->path || !loc->inode) {
+        ret = PATH_PARSE_RESULT_NO_PATH;
+    } else if (!ns_inode_ctx_get(loc->inode, this, info)) {
+        ret = PATH_PARSE_RESULT_FOUND;
+    } else {
+        ret = parse_path(info, loc->path);
+        gf_log(this->name, GF_LOG_DEBUG, "%s: LOC retrieved path %s", fn,
+               loc->path);
+
+        if (ret == PATH_PARSE_RESULT_FOUND) {
+            ns_inode_ctx_put(loc->inode, this, info);
+        }
+    }
+
+    /* Keep trying by calling inode_path next, making sure to copy
+    the loc's gfid into its inode if necessary. */
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        if (gf_uuid_is_null(loc->inode->gfid)) {
+            gf_uuid_copy(loc->inode->gfid, loc->gfid);
+        }
+
+        if (inode_path(loc->inode, NULL, &path) >= 0 && path) {
+            ret = parse_path(info, loc->path);
+            gf_log(this->name, GF_LOG_DEBUG, "%s: LOC retrieved path %s", fn,
+                   path);
+
+            if (ret == PATH_PARSE_RESULT_FOUND) {
+                ns_inode_ctx_put(loc->inode, this, info);
+            }
+        }
+
+        if (path) {
+            GF_FREE(path);
+        }
+    }
+
+    /* Report our status, and if we have a GFID, we'll eventually try a
+     * GET_ANCESTRY_PATH_KEY wind when we return from this function. */
+    if (ret == PATH_PARSE_RESULT_FOUND) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "%s: LOC %s %10u namespace found for %s", fn,
+               uuid_utoa(loc->inode->gfid), info->hash, loc->path);
+    } else if (ret == PATH_PARSE_RESULT_NO_PATH) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: LOC has no path", fn);
+    } else if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        /* Make sure to copy the inode's gfid for the eventual wind. */
+        if (gf_uuid_is_null(loc->inode->gfid)) {
+            gf_uuid_copy(loc->inode->gfid, loc->gfid);
+        }
+
+        gf_log(this->name, GF_LOG_DEBUG, "%s: LOC %s winding, looking for path",
+               fn, uuid_utoa(loc->inode->gfid));
+    }
+
+    return ret;
+}
+
+/* This function tries first to set a namespace based on the information that
+ * it can retrieve from an `fd_t`. This includes first looking for a cached
+ * namespace in the inode, then trying to call inode_path manually. */
+static path_parse_result_t
+set_ns_from_fd(const char *fn, call_frame_t *frame, xlator_t *this, fd_t *fd)
+{
+    path_parse_result_t ret = PATH_PARSE_RESULT_NO_PATH;
+    ns_private_t *priv = (ns_private_t *)this->private;
+    ns_info_t *info = &frame->root->ns_info;
+    char *path = NULL;
+
+    info->hash = 0;
+    info->found = _gf_false;
+
+    if (!priv->tag_namespaces) {
+        return ret;
+    }
+
+    /* This is our first pass at trying to get a path. Try getting
+     * from the inode context, then inode_path. */
+    if (!fd || !fd->inode) {
+        ret = PATH_PARSE_RESULT_NO_PATH;
+    } else if (!ns_inode_ctx_get(fd->inode, this, info)) {
+        ret = PATH_PARSE_RESULT_FOUND;
+    } else if (inode_path(fd->inode, NULL, &path) >= 0 && path) {
+        ret = parse_path(info, path);
+        gf_log(this->name, GF_LOG_DEBUG, "%s: FD  retrieved path %s", fn, path);
+
+        if (ret == PATH_PARSE_RESULT_FOUND) {
+            ns_inode_ctx_put(fd->inode, this, info);
+        }
+    }
+
+    if (path) {
+        GF_FREE(path);
+    }
+
+    /* Report our status, and if we have a GFID, we'll eventually try a
+     * GET_ANCESTRY_PATH_KEY wind when we return from this function. */
+    if (ret == PATH_PARSE_RESULT_FOUND) {
+        gf_log(this->name, GF_LOG_DEBUG, "%s: FD  %s %10u namespace found", fn,
+               uuid_utoa(fd->inode->gfid), info->hash);
+    } else if (ret == PATH_PARSE_RESULT_NO_PATH) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: FD  has no path", fn);
+    } else if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        gf_log(this->name, GF_LOG_DEBUG, "%s: FD  %s winding, looking for path",
+               fn, uuid_utoa(fd->inode->gfid));
+    }
+
+    return ret;
+}
+
+/* This macro does the work of winding down a call of `getxattr` in the case
+ * that we have to retrieve the path manually. It assumes that there is a label
+ * called `wind` and the existence of several basic variables (frame, this),
+ * but otherwise is general enough for any fop (fd- or loc-based.) */
+#define GET_ANCESTRY_PATH_WIND(fop, inode, args...)                            \
+    do {                                                                       \
+        ns_info_t *info = &frame->root->ns_info;                               \
+        call_frame_t *new_frame = NULL;                                        \
+        ns_local_t *local = NULL;                                              \
+        call_stub_t *stub = NULL;                                              \
+                                                                               \
+        gf_log(this->name, GF_LOG_DEBUG, "    %s winding, looking for path",   \
+               uuid_utoa(inode->gfid));                                        \
+                                                                               \
+        new_frame = create_frame(this, this->ctx->pool);                       \
+        if (!new_frame) {                                                      \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "Cannot allocate new call frame.");                         \
+            goto wind;                                                         \
+        }                                                                      \
+                                                                               \
+        stub = fop_##fop##_stub(frame, default_##fop, args);                   \
+        if (!stub) {                                                           \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "Cannot allocate function stub.");                          \
+            goto wind;                                                         \
+        }                                                                      \
+                                                                               \
+        new_frame->root->uid = 0;                                              \
+        new_frame->root->gid = 0;                                              \
+        /* Put a phony "not found" NS info into this call. */                  \
+        new_frame->root->ns_info = *info;                                      \
+                                                                               \
+        local = ns_local_new(stub, inode);                                     \
+        if (!local) {                                                          \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "Cannot allocate function local.");                         \
+            goto wind;                                                         \
+        }                                                                      \
+                                                                               \
+        new_frame->local = local;                                              \
+        /* After allocating a new frame, a call stub (to                       \
+         * resume our current fop), and a local variables                      \
+         * struct (for our loc to getxattr and our resume                      \
+         * stub), call getxattr and unwind to get_path_resume_cbk.             \
+         */                                                                    \
+        STACK_WIND(new_frame, get_path_resume_cbk, FIRST_CHILD(this),          \
+                   FIRST_CHILD(this)->fops->getxattr, &local->loc,             \
+                   GET_ANCESTRY_PATH_KEY, NULL);                               \
+    } while (0)
+
+int32_t
+ns_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+         dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(rmdir, loc->inode, loc, xflags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+    return 0;
+}
+
+int32_t
+ns_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(unlink, loc->inode, loc, xflags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+    return 0;
+}
+
+int32_t
+ns_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this,
+                                              newloc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(rename, newloc->inode, oldloc, newloc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+ns_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this,
+                                              newloc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(link, newloc->inode, oldloc, newloc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+ns_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(mkdir, loc->inode, loc, mode, umask, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
+}
+
+int32_t
+ns_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(symlink, loc->inode, linkname, loc, umask,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
+}
+
+int32_t
+ns_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t dev, mode_t umask, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(mknod, loc->inode, loc, mode, dev, umask, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
+}
+
+int32_t
+ns_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(create, loc->inode, loc, flags, mode, umask, fd,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsetattr, fd->inode, fd, stbuf, valid, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+ns_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(setattr, loc->inode, loc, stbuf, valid, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+ns_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fremovexattr, fd->inode, fd, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(removexattr, loc->inode, loc, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(setxattr, loc->inode, loc, dict, flags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+ns_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsetxattr, fd->inode, fd, dict, flags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+ns_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(truncate, loc->inode, loc, offset, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
+}
+
+int32_t
+ns_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(ftruncate, fd->inode, fd, offset, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
+}
+
+int32_t
+ns_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(writev, fd->inode, fd, vector, count, offset,
+                               flags, iobref, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
+}
+
+int32_t
+ns_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(lookup, loc->inode, loc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+}
+
+int32_t
+ns_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(stat, loc->inode, loc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+}
+
+int32_t
+ns_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fstat, fd->inode, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readlink, loc->inode, loc, size, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
+}
+
+int32_t
+ns_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(access, loc->inode, loc, mask, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
+}
+
+int32_t
+ns_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(open, fd->inode, loc, flags, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readv, fd->inode, fd, size, offset, flags,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
+}
+
+int32_t
+ns_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(flush, fd->inode, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsync, fd->inode, fd, datasync, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
+}
+
+int32_t
+ns_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(opendir, loc->inode, loc, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsyncdir, fd->inode, fd, datasync, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
+}
+
+int32_t
+ns_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             int32_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(rchecksum, fd->inode, fd, offset, len, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+ns_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(statfs, loc->inode, loc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
+}
+
+int32_t
+ns_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(inodelk, loc->inode, volume, loc, cmd, flock,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(finodelk, fd->inode, volume, fd, cmd, flock,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(entrylk, loc->inode, volume, loc, basename, cmd,
+                               type, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
+}
+
+int32_t
+ns_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fentrylk, fd->inode, volume, fd, basename, cmd,
+                               type, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
+}
+
+int32_t
+ns_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fgetxattr, fd->inode, fd, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(getxattr, loc->inode, loc, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(lk, fd->inode, fd, cmd, flock, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, flock, xdata);
+    return 0;
+}
+
+int32_t
+ns_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readdir, fd->inode, fd, size, offset, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+
+    return 0;
+}
+
+int32_t
+ns_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *dict)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readdirp, fd->inode, fd, size, offset, dict);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    return 0;
+}
+
+int32_t
+ns_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(xattrop, loc->inode, loc, flags, dict, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+
+    return 0;
+}
+
+int32_t
+ns_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fxattrop, fd->inode, fd, flags, dict, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+
+    return 0;
+}
+
+int32_t
+ns_getspec(call_frame_t *frame, xlator_t *this, const char *key, int32_t flag)
+{
+    STACK_WIND(frame, default_getspec_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getspec, key, flag);
+    return 0;
+}
+
+int32_t
+ns_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fallocate, fd->inode, fd, keep_size, offset, len,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset, len,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(discard, fd->inode, fd, offset, len, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+ns_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(zerofill, fd->inode, fd, offset, len, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
+}
+
+int
+ns_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ns_as_64 = 0;
+    ns_info_t *info = NULL;
+
+    inode_ctx_del(inode, this, &ns_as_64);
+
+    if (!ns_as_64) {
+        return 0;
+    }
+
+    info = (ns_info_t *)(uintptr_t)ns_as_64;
+    GF_FREE(info);
+
+    return 0;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    int32_t ret = -1;
+    ns_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NAMESPACE, this, out);
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "translator needs a single subvolume.");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "dangling volume. please check volfile.");
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(ns_private_t), 0);
+
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "Can't allocate ns_priv structure.");
+        goto out;
+    }
+
+    GF_OPTION_INIT("tag-namespaces", priv->tag_namespaces, bool, out);
+
+    gf_log(this->name, GF_LOG_INFO, "Namespace xlator loaded");
+    this->private = priv;
+    ret = 0;
+
+out:
+    if (ret) {
+        GF_FREE(priv);
+    }
+
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    GF_FREE(this->private);
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = -1;
+    ns_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
+
+    priv = (ns_private_t *)this->private;
+
+    GF_OPTION_RECONF("tag-namespaces", priv->tag_namespaces, options, bool,
+                     out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+struct xlator_fops fops = {
+    .lookup = ns_lookup,
+    .stat = ns_stat,
+    .fstat = ns_fstat,
+    .truncate = ns_truncate,
+    .ftruncate = ns_ftruncate,
+    .access = ns_access,
+    .readlink = ns_readlink,
+    .mknod = ns_mknod,
+    .mkdir = ns_mkdir,
+    .unlink = ns_unlink,
+    .rmdir = ns_rmdir,
+    .symlink = ns_symlink,
+    .rename = ns_rename,
+    .link = ns_link,
+    .create = ns_create,
+    .open = ns_open,
+    .readv = ns_readv,
+    .writev = ns_writev,
+    .flush = ns_flush,
+    .fsync = ns_fsync,
+    .opendir = ns_opendir,
+    .readdir = ns_readdir,
+    .readdirp = ns_readdirp,
+    .fsyncdir = ns_fsyncdir,
+    .statfs = ns_statfs,
+    .setxattr = ns_setxattr,
+    .getxattr = ns_getxattr,
+    .fsetxattr = ns_fsetxattr,
+    .fgetxattr = ns_fgetxattr,
+    .removexattr = ns_removexattr,
+    .fremovexattr = ns_fremovexattr,
+    .lk = ns_lk,
+    .inodelk = ns_inodelk,
+    .finodelk = ns_finodelk,
+    .entrylk = ns_entrylk,
+    .fentrylk = ns_fentrylk,
+    .rchecksum = ns_rchecksum,
+    .xattrop = ns_xattrop,
+    .fxattrop = ns_fxattrop,
+    .setattr = ns_setattr,
+    .fsetattr = ns_fsetattr,
+    .getspec = ns_getspec,
+    .fallocate = ns_fallocate,
+    .discard = ns_discard,
+    .zerofill = ns_zerofill,
+};
+
+struct xlator_cbks cbks = {
+    .forget = ns_forget,
+};
+
+struct xlator_dumpops dumpops;
+
+struct volume_options options[] = {
+    {
+        .key = {"tag-namespaces"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "This option enables this translator's functionality "
+                       "that tags every fop with a namespace hash for later "
+                       "throttling, stats collection, logging, etc.",
+        .op_version = {GD_OP_VERSION_4_1_0},
+        .tags = {"namespace"},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "namespace",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/features/namespace/src/namespace.h b/xlators/features/namespace/src/namespace.h
new file mode 100644
index 00000000000..3a9b84d6426
--- /dev/null
+++ b/xlators/features/namespace/src/namespace.h
@@ -0,0 +1,23 @@
+#ifndef __NAMESPACE_H__
+#define __NAMESPACE_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+
+#define GF_NAMESPACE "namespace"
+
+typedef struct {
+    gf_boolean_t tag_namespaces;
+} ns_private_t;
+
+typedef struct {
+    loc_t loc;         /* We store a "fake" loc_t for the getxattr wind. */
+    call_stub_t *stub; /* A stub back to the function we're resuming. */
+} ns_local_t;
+
+#endif /* __NAMESPACE_H__ */
diff --git a/xlators/features/path-convertor/Makefile.am b/xlators/features/path-convertor/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/path-convertor/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/path-convertor/src/Makefile.am b/xlators/features/path-convertor/src/Makefile.am
deleted file mode 100644
index 7090698687b..00000000000
--- a/xlators/features/path-convertor/src/Makefile.am
+++ /dev/null
@@ -1,15 +0,0 @@
-
-xlator_LTLIBRARIES = path-converter.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
-
-path_converter_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-path_converter_la_SOURCES = path.c
-path_converter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/features/path-convertor/src/path-mem-types.h b/xlators/features/path-convertor/src/path-mem-types.h
deleted file mode 100644
index 77ada8d537a..00000000000
--- a/xlators/features/path-convertor/src/path-mem-types.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __PATH_MEM_TYPES_H__
-#define __PATH_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_path_mem_types_ {
-        gf_path_mt_path_private_t = gf_common_mt_end + 1,
-        gf_path_mt_char,
-        gf_path_mt_regex_t,
-        gf_path_mt_end
-};
-#endif
-
diff --git a/xlators/features/path-convertor/src/path.c b/xlators/features/path-convertor/src/path.c
deleted file mode 100644
index b0e5d6cc625..00000000000
--- a/xlators/features/path-convertor/src/path.c
+++ /dev/null
@@ -1,1223 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-/* TODO: add gf_log to all the cases returning errors */
-
-/**
- * xlators/features/path-translator:
- *    This translator converts the path it gets into user specified targets.
- */
-
-#include <sys/types.h>
-#include <regex.h>
-#include <time.h>
-#include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "path-mem-types.h"
-
-typedef struct path_private
-{
-	int32_t this_len;
-	int32_t start_off;
-	int32_t end_off;
-	char *this;
-	char *that;
-	char *path;
-	regex_t *preg;
-} path_private_t;
-
-static char *
-name_this_to_that (xlator_t *xl, const char *path, const char *name)
-{
-	path_private_t *priv = xl->private;
-	char priv_path[PATH_MAX] = {0,};
-	char *tmp_name = NULL;
-	int32_t path_len = strlen (path);
-	int32_t name_len = strlen (name) - ZR_FILE_CONTENT_STRLEN;
-	int32_t total_len = path_len + name_len;
-	int32_t i = 0, j = 0;
-
-	if (path_len >= priv->end_off)
-		return (char *)name;
-
-	if (priv->end_off && (total_len > priv->end_off)) {
-		j = priv->start_off;
-		tmp_name = GF_CALLOC (1, (total_len +
-                                          ZR_FILE_CONTENT_STRLEN),
-                                      gf_path_mt_char);
-		ERR_ABORT (tmp_name);
-
-		/* Get the complete path for the file first */
-		strcpy (tmp_name, path);
-		strcat (tmp_name, name + ZR_FILE_CONTENT_STRLEN);
-
-		strncpy (priv_path, tmp_name, priv->start_off);
-		for (i = priv->start_off; i < priv->end_off; i++) {
-			if (tmp_name[i] == '/')
-				continue;
-			priv_path[j++] = tmp_name[i];
-		}
-		memcpy ((priv_path + j), 
-			(tmp_name + priv->end_off), 
-			(total_len - priv->end_off));
-		priv_path[(total_len - (priv->end_off - j))] = '\0';
-
-		strcpy (tmp_name, ZR_FILE_CONTENT_STR);
-		strcat (tmp_name, priv_path);
-
-		return tmp_name;
-	}
-
-	return (char *)name;
-}
-
-/* This function should return 
- *  NULL - 
- *  converted path - if path match
- *  same path - if it doesn't match
- */
-static char *
-path_this_to_that (xlator_t *xl, const char *path)
-{
-	path_private_t *priv = xl->private;
-	char *priv_path = NULL;
-	int32_t path_len = strlen (path);
-	int32_t i = 0, j = 0;
-
-	if (priv->end_off && (path_len > priv->start_off)) {
-		priv_path = GF_CALLOC (1, path_len, gf_path_mt_char);
-		ERR_ABORT (priv_path);
-
-		if (priv->start_off && (path_len > priv->start_off))
-			memcpy (priv_path, path, priv->start_off);
-		if (path_len > priv->end_off) {
-			j = priv->start_off;
-			for (i = priv->start_off; i < priv->end_off; i++) {
-				if (path[i] == '/')
-					continue;
-				priv_path[j++] = path[i];
-			}
-			memcpy ((priv_path + j), 
-				(path + priv->end_off), 
-				(path_len - priv->end_off));
-			priv_path[(path_len - (priv->end_off - j))] = '\0';
-		}
-		return priv_path;
-	}
-	return (char *)path;
-}
-
-int32_t 
-path_create_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd,
-		 inode_t *inode,
-		 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
-}
-
-int32_t 
-path_open_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-int32_t 
-path_getdents_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   dir_entry_t *entries,
-		   int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entries, count);
-	return 0;
-}
-
-int32_t 
-path_readdir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-int32_t 
-path_readlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   const char *buf,
-                   struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf, sbuf);
-	return 0;
-}
-
-int32_t 
-path_lookup_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-		 struct iatt *buf,
-		 dict_t *xattr,
-                 struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, xattr);
-	return 0;
-}
-
-
-int32_t 
-path_symlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-
-int32_t 
-path_mknod_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-  
-
-int32_t 
-path_mkdir_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-  
-int32_t 
-path_link_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       inode_t *inode,
-               struct iatt *buf,
-               struct iatt *preparent,
-               struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-
-int32_t 
-path_opendir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-
-int32_t
-path_rename_buf_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-
-int32_t
-path_common_buf_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-path_common_dict_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t 
-path_common_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t
-path_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-int32_t
-path_common_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/* */
-int32_t 
-path_lookup (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     dict_t *xattr_req)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, path_lookup_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->lookup, 
-		    loc, xattr_req);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_stat (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_buf_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->stat, 
-		    loc);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_readlink (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       size_t size)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_readlink_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->readlink, 
-		    loc, 
-		    size);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_mknod (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    mode_t mode,
-	    dev_t dev)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_mknod_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->mknod, 
-		    loc, 
-		    mode, 
-		    dev);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_mkdir (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    mode_t mode)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_mkdir_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->mkdir, 
-		    loc, 
-		    mode);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_unlink (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_remove_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->unlink, 
-		    loc);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_rmdir (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_remove_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->rmdir, 
-		    loc);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_symlink (call_frame_t *frame,
-	      xlator_t *this,
-	      const char *linkpath,
-	      loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_symlink_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->symlink, 
-		    linkpath,
-		    loc);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_rename (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *oldloc,
-	     loc_t *newloc)
-{  
-	char *oldloc_path = (char *)oldloc->path;
-	char *tmp_oldloc_path = NULL;
-
-	char *newloc_path = (char *)newloc->path;
-	char *tmp_newloc_path = NULL;
-	
-	if (!(tmp_oldloc_path = path_this_to_that (this, oldloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	oldloc->path = tmp_oldloc_path;
-
-	if (!(tmp_newloc_path = path_this_to_that (this, newloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	newloc->path = tmp_newloc_path;
-
-	STACK_WIND (frame, 
-		    path_rename_buf_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->rename, 
-		    oldloc,
-		    newloc);
-  
-	oldloc->path = oldloc_path;	
-	if (tmp_oldloc_path != oldloc_path)
-		GF_FREE (tmp_oldloc_path);
-
-	newloc->path = newloc_path;	
-	if (tmp_newloc_path != newloc_path)
-		GF_FREE (tmp_newloc_path);
-
-	return 0;
-}
-
-int32_t 
-path_link (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *oldloc,
-	   loc_t *newloc)
-{
-	char *oldloc_path = (char *)oldloc->path;
-	char *tmp_oldloc_path = NULL;
-
-	char *newloc_path = (char *)newloc->path;
-	char *tmp_newloc_path = NULL;
-	
-	if (!(tmp_oldloc_path = path_this_to_that (this, oldloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	oldloc->path = tmp_oldloc_path;
-
-	if (!(tmp_newloc_path = path_this_to_that (this, newloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	newloc->path = tmp_newloc_path;
-
-	STACK_WIND (frame, 
-		    path_link_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->link, 
-		    oldloc, 
-		    newloc);
-
-	oldloc->path = oldloc_path;	
-	if (tmp_oldloc_path != oldloc_path)
-		GF_FREE (tmp_oldloc_path);
-
-	newloc->path = newloc_path;	
-	if (tmp_newloc_path != newloc_path)
-		GF_FREE (tmp_newloc_path);
-
-	return 0;
-}
-
-int32_t 
-path_setattr_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  struct iatt *preop,
-                  struct iatt *postop)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, preop, postop);
-	return 0;
-}
-
-int32_t
-path_setattr (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              struct iatt *stbuf,
-              int32_t valid)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr,
-		    loc,
-		    stbuf, valid);
-
-	loc->path = loc_path;
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t 
-path_truncate (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       off_t offset)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_truncate_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->truncate, 
-		    loc, 
-		    offset);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t 
-path_open (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   int32_t flags,
-	   fd_t *fd,
-           int32_t wbflags)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_open_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->open, 
-		    loc, 
-		    flags,
-		    fd,
-                    wbflags);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_create (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t flags,
-	     mode_t mode,
-	     fd_t *fd)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_create_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->create, 
-		    loc, 
-		    flags,
-		    mode,
-		    fd);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_setxattr (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       dict_t *dict,
-	       int32_t flags)
-{
-	char *tmp_name = NULL;
-	data_pair_t *trav = dict->members_list;
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	if (ZR_FILE_CONTENT_REQUEST(trav->key)) {
-		tmp_name = name_this_to_that (this, loc->path, trav->key);
-		if (tmp_name != trav->key) {
-			trav->key = tmp_name;
-		} else {
-			tmp_name = NULL;
-		}
-	}
-
-	STACK_WIND (frame, 
-		    path_common_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->setxattr, 
-		    loc, 
-		    dict,
-		    flags);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	GF_FREE (tmp_name);
-
-	return 0;
-}
-
-int32_t 
-path_getxattr (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       const char *name)
-{
-	char *tmp_name = (char *)name;
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	if (ZR_FILE_CONTENT_REQUEST(name)) {
-		tmp_name = name_this_to_that (this, loc->path, name);
-	}
-
-	STACK_WIND (frame, 
-		    path_common_dict_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, 
-		    tmp_name);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	if (tmp_name != name)
-		GF_FREE (tmp_name);
-
-	return 0;
-}
-
-int32_t 
-path_removexattr (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  const char *name)
-{
-	char *tmp_name = (char *)name;
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	if (ZR_FILE_CONTENT_REQUEST(name)) {
-		tmp_name = name_this_to_that (this, loc->path, name);
-	}
-
-	STACK_WIND (frame, 
-		    path_common_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->removexattr, 
-		    loc, 
-		    tmp_name);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	if (tmp_name != name)
-		GF_FREE (tmp_name);
-
-	return 0;
-}
-
-int32_t 
-path_opendir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      fd_t *fd)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_opendir_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->opendir, 
-		    loc, 
-		    fd);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_access (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t mask)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->access, 
-		    loc, 
-		    mask);
-
-	loc->path = loc_path;
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t
-path_checksum_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   uint8_t *fchecksum,
-		   uint8_t *dchecksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
-	return 0;
-}
-
-int32_t
-path_checksum (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       int32_t flag)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_checksum_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->checksum, 
-		    loc, 
-		    flag);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t
-path_entrylk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, loc_t *loc, const char *basename,
-	      entrylk_cmd cmd, entrylk_type type)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, path_common_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t
-path_inodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_common_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t
-path_xattrop (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      gf_xattrop_flags_t flags,
-	      dict_t *dict)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_common_dict_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->xattrop,
-		    loc,
-		    flags,
-		    dict);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_path_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t 
-init (xlator_t *this)
-{
-	dict_t *options = this->options;
-	path_private_t *priv = NULL;
-
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"path translator requires exactly one subvolume");
-		return -1;
-	}
-    
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	priv = GF_CALLOC (1, sizeof (*priv), gf_path_mt_path_private_t);
-	ERR_ABORT (priv);
-	if (dict_get (options, "start-offset")) {
-		priv->start_off = data_to_int32 (dict_get (options, 
-							   "start-offset"));
-	}
-	if (dict_get (options, "end-offset")) {
-		priv->end_off = data_to_int32 (dict_get (options, 
-							 "end-offset"));
-	}
-
-	if (dict_get (options, "regex")) {
-		int32_t ret = 0;
-		priv->preg = GF_CALLOC (1, sizeof (regex_t),
-                                        gf_path_mt_regex_t);
-		ERR_ABORT (priv->preg);
-		ret = regcomp (priv->preg, 
-			       data_to_str (dict_get (options, "regex")), 
-			       REG_EXTENDED);
-		if (ret) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"Failed to compile the 'option regex'");
-			GF_FREE (priv);
-			return -1;
-		}
-		if (dict_get (options, "replace-with")) {
-			priv->that = data_to_str (dict_get (options, 
-							    "replace-with"));
-		} else {
-			priv->that = "";
-		}
-	}
-
-	this->private = priv;
-	return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-	return;
-}
-
-struct xlator_fops fops = {
-	.stat        = path_stat,
-	.readlink    = path_readlink,
-	.mknod       = path_mknod,
-	.mkdir       = path_mkdir,
-	.unlink      = path_unlink,
-	.rmdir       = path_rmdir,
-	.symlink     = path_symlink,
-	.rename      = path_rename,
-	.link        = path_link,
-	.truncate    = path_truncate,
-	.open        = path_open,
-	.setxattr    = path_setxattr,
-	.getxattr    = path_getxattr,
-	.removexattr = path_removexattr,
-	.opendir     = path_opendir,
-	.access      = path_access,
-	.create      = path_create,
-	.lookup      = path_lookup,
-	.checksum    = path_checksum,
-	.xattrop     = path_xattrop,
-	.entrylk     = path_entrylk,
-	.inodelk     = path_inodelk,
-        .setattr     = path_setattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = { 
-	{ .key  = {"start-offset"}, 
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 0, 
-	  .max  = 4095 
-	},
-	{ .key  = {"end-offset"}, 
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 1, 
-	  .max  = 4096 
-	},
-	{ .key  = {"replace-with"}, 
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key  = {NULL} },
-};
diff --git a/xlators/features/protect/Makefile.am b/xlators/features/protect/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/protect/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/protect/src/Makefile.am b/xlators/features/protect/src/Makefile.am
deleted file mode 100644
index 98499712fab..00000000000
--- a/xlators/features/protect/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-xlator_LTLIBRARIES = prot_dht.la prot_client.la prot_server.la
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-prot_dht_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-prot_dht_la_SOURCES = prot_dht.c
-prot_dht_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-prot_client_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-prot_client_la_SOURCES = prot_client.c
-prot_client_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-prot_server_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-prot_server_la_SOURCES = prot_server.c
-prot_server_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	 -I$(CONTRIBDIR)/libexecinfo
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/protect/src/prot_client.c b/xlators/features/protect/src/prot_client.c
deleted file mode 100644
index 79636410b94..00000000000
--- a/xlators/features/protect/src/prot_client.c
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "defaults.h"
-
-#ifdef HAVE_BACKTRACE
-#include <execinfo.h>
-#else
-#include "execinfo_compat.h"
-#endif
-
-#define NUM_FRAMES 20
-
-static char PROTECT_KEY[] = "trusted.glusterfs.protect";
-
-enum {
-        PROT_ACT_NONE = 0,
-        PROT_ACT_LOG,
-        PROT_ACT_REJECT,
-};
-
-void
-pcli_print_trace (char *name, call_frame_t *frame)
-{
-        void    *frames[NUM_FRAMES];
-        char    **symbols;
-        int     size;
-        int     i;
-
-        gf_log (name, GF_LOG_INFO, "Translator stack:");
-        list_for_each_entry (frame, &frame->root->myframes, frames) {
-                gf_log (name, GF_LOG_INFO, "%s (%s)",
-                        frame->wind_from, frame->this->name);
-        }
-
-        size = backtrace (frames, NUM_FRAMES);
-        if (size <= 0) {
-                return;
-        }
-        symbols = backtrace_symbols (frames, size);
-        if (!symbols) {
-                return;
-        }
-
-        gf_log (name, GF_LOG_INFO, "Processor stack:");
-        for (i = 0; i < size; ++i) {
-                gf_log (name, GF_LOG_INFO, "%s", symbols[i]);
-        }
-        free (symbols);
-}
-
-int32_t
-pcli_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        uint64_t        value;
-
-        if (newloc->parent == oldloc->parent) {
-                gf_log (this->name, GF_LOG_DEBUG, "rename in same directory");
-                goto simple_unwind;
-        }
-        if (!oldloc->parent) {
-                goto simple_unwind;
-        }
-        if (inode_ctx_get (oldloc->parent, this, &value) != 0) {
-                goto simple_unwind;
-        }
-
-        if (value != PROT_ACT_NONE) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "got rename for protected %s", oldloc->path);
-                pcli_print_trace (this->name, frame);
-                if (value == PROT_ACT_REJECT) {
-                        STACK_UNWIND_STRICT (rename, frame, -1, EPERM,
-                                             NULL, NULL, NULL, NULL, NULL,
-                                             xdata);
-                        return 0;
-                }
-        }
-
-simple_unwind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->rename, oldloc, newloc,
-                         xdata);
-        return 0;
-}
-
-int32_t
-pcli_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-               int32_t flags, dict_t *xdata)
-{
-        data_t          *data;
-        uint64_t        value;
-
-        /*
-         * We can't use dict_get_str and strcmp here, because the value comes
-         * directly from the user and might not be NUL-terminated (it would
-         * be if we had set it ourselves.
-         */
-
-        data = dict_get(dict,PROTECT_KEY);
-        if (!data) {
-                goto simple_wind;
-        }
-
-        if (dict->count > 1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "attempted to mix %s with other keys", PROTECT_KEY);
-                goto simple_wind;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "got %s request", PROTECT_KEY);
-        if (!strncmp(data->data,"log",data->len)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "logging removals on %s", loc->path);
-                value = PROT_ACT_LOG;
-        }
-        else if (!strncmp(data->data,"reject",data->len)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "rejecting removals on %s", loc->path);
-                value = PROT_ACT_REJECT;
-        }
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "removing protection on %s", loc->path);
-                value = PROT_ACT_NONE;
-        }
-        /* Right now the value doesn't matter - just the presence. */
-        if (inode_ctx_set(loc->inode,this,&value) != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set protection status for %s", loc->path);
-        }
-        STACK_UNWIND_STRICT (setxattr, frame, 0, 0, NULL);
-        return 0;
-
-simple_wind:
-        STACK_WIND_TAIL (frame,
-                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
-                         loc, dict, flags, xdata);
-        return 0;
-}
-
-int32_t
-pcli_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-             dict_t *xdata)
-{
-        uint64_t        value;
-
-        if (!loc->parent || (inode_ctx_get(loc->parent,this,&value) != 0)) {
-                goto simple_unwind;
-        }
-
-        if (value != PROT_ACT_NONE) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "got unlink for protected %s", loc->path);
-                pcli_print_trace(this->name, frame);
-                if (value == PROT_ACT_REJECT) {
-                        STACK_UNWIND_STRICT (unlink, frame, -1, EPERM,
-                                             NULL, NULL, NULL);
-                        return 0;
-                }
-        }
-
-simple_unwind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        return 0;
-}
-
-int32_t
-init (xlator_t *this)
-{
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "translator not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        return;
-}
-
-
-struct xlator_fops fops = {
-        .rename         = pcli_rename,
-        .setxattr       = pcli_setxattr,
-        .unlink         = pcli_unlink,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-        { .key = {NULL} },
-};
diff --git a/xlators/features/protect/src/prot_dht.c b/xlators/features/protect/src/prot_dht.c
deleted file mode 100644
index 1fc8cc1ffde..00000000000
--- a/xlators/features/protect/src/prot_dht.c
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "defaults.h"
-
-enum gf_pdht_mem_types_ {
-        gf_pdht_mt_coord_t = gf_common_mt_end + 1,
-        gf_pdht_mt_end
-};
-
-typedef struct {
-        pthread_mutex_t         lock;
-        uint16_t                refs;
-        int32_t                 op_ret;
-        int32_t                 op_errno;
-        dict_t                  *xdata;
-} pdht_coord_t;
-
-static char PROTECT_KEY[] = "trusted.glusterfs.protect";
-
-void
-pdht_unref_and_unlock (call_frame_t *frame, xlator_t *this,
-                       pdht_coord_t *coord)
-{
-        gf_boolean_t    should_unwind;
-
-        should_unwind = (--(coord->refs) == 0);
-        pthread_mutex_unlock(&coord->lock);
-
-        if (should_unwind) {
-                STACK_UNWIND_STRICT (setxattr, frame, 
-                                     coord->op_ret, coord->op_errno,
-                                     coord->xdata);
-                if (coord->xdata) {
-                        dict_unref(coord->xdata);
-                }
-                GF_FREE(coord);
-        }
-}
-
-int32_t
-pdht_recurse_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        pdht_coord_t    *coord = cookie;
-
-        pthread_mutex_lock(&coord->lock);
-        if (op_ret) {
-                coord->op_ret = op_ret;
-                coord->op_errno = op_errno;
-        }
-        if (xdata) {
-                if (coord->xdata) {
-                        dict_unref(coord->xdata);
-                }
-                coord->xdata = dict_ref(xdata);
-        }
-        pdht_unref_and_unlock(frame,this,coord);
-                
-        return 0;
-}
-
-void
-pdht_recurse (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-              int32_t flags, dict_t *xdata, xlator_t *xl, pdht_coord_t *coord)
-{
-        xlator_list_t   *iter;
-
-        if (!strcmp(xl->type,"features/prot_client")) {
-                pthread_mutex_lock(&coord->lock);
-                ++(coord->refs);
-                pthread_mutex_unlock(&coord->lock);
-                STACK_WIND_COOKIE (frame, pdht_recurse_cbk, coord, xl,
-                                   xl->fops->setxattr, loc, dict, flags, xdata);
-        }
-
-        else for (iter = xl->children; iter; iter = iter->next) {
-                pdht_recurse (frame, this, loc, dict, flags, xdata,
-                              iter->xlator, coord);
-        }
-}
-
-int32_t
-pdht_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-               int32_t flags, dict_t *xdata)
-{
-        pdht_coord_t    *coord;
-
-        if (!dict_get(dict,PROTECT_KEY)) {
-                goto simple_wind;
-        }
-
-        if (dict->count > 1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "attempted to mix %s with other keys", PROTECT_KEY);
-                goto simple_wind;
-        }
-
-        coord = GF_CALLOC(1,sizeof(*coord),gf_pdht_mt_coord_t);
-        if (!coord) {
-                gf_log (this->name, GF_LOG_WARNING, "allocation failed");
-                goto simple_wind;
-        }
-
-        pthread_mutex_init(&coord->lock,NULL);
-        coord->refs = 1;
-        coord->op_ret = 0;
-        coord->xdata = NULL;
-
-        pdht_recurse(frame,this,loc,dict,flags,xdata,this,coord);
-        pthread_mutex_lock(&coord->lock);
-        pdht_unref_and_unlock(frame,this,coord);
-
-        return 0;
-
-simple_wind:
-        STACK_WIND_TAIL (frame,
-                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
-                         loc, dict, flags, xdata);
-        return 0;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	return;
-}
-
-struct xlator_fops fops = {
-        .setxattr = pdht_setxattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key = {NULL} },
-};
diff --git a/xlators/features/protect/src/prot_server.c b/xlators/features/protect/src/prot_server.c
deleted file mode 100644
index 8ebace240f3..00000000000
--- a/xlators/features/protect/src/prot_server.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "defaults.h"
-
-int32_t
-init (xlator_t *this)
-{
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	return;
-}
-
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key = {NULL} },
-};
diff --git a/xlators/features/quiesce/src/Makefile.am b/xlators/features/quiesce/src/Makefile.am
index 6468669af2a..74ea999c045 100644
--- a/xlators/features/quiesce/src/Makefile.am
+++ b/xlators/features/quiesce/src/Makefile.am
@@ -6,9 +6,10 @@ quiesce_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 quiesce_la_SOURCES = quiesce.c
 quiesce_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = quiesce.h quiesce-mem-types.h
+noinst_HEADERS = quiesce.h quiesce-mem-types.h quiesce-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/quiesce/src/quiesce-mem-types.h b/xlators/features/quiesce/src/quiesce-mem-types.h
index 6e582f424ea..416456b13af 100644
--- a/xlators/features/quiesce/src/quiesce-mem-types.h
+++ b/xlators/features/quiesce/src/quiesce-mem-types.h
@@ -11,10 +11,11 @@
 #ifndef __QUIESCE_MEM_TYPES_H__
 #define __QUIESCE_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_quiesce_mem_types_ {
-        gf_quiesce_mt_priv_t = gf_common_mt_end + 1,
-        gf_quiesce_mt_end
+    gf_quiesce_mt_priv_t = gf_common_mt_end + 1,
+    gf_quiesce_mt_failover_hosts,
+    gf_quiesce_mt_end
 };
 #endif
diff --git a/xlators/features/quiesce/src/quiesce-messages.h b/xlators/features/quiesce/src/quiesce-messages.h
new file mode 100644
index 00000000000..32ffd409807
--- /dev/null
+++ b/xlators/features/quiesce/src/quiesce-messages.h
@@ -0,0 +1,28 @@
+/*
+ *   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __QUIESCE_MESSAGES_H__
+#define __QUIESCE_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(QUIESCE, QUIESCE_MSG_INVAL_HOST, QUIESCE_MSG_FAILOVER_FAILED);
+
+#endif /* __NL_CACHE_MESSAGES_H__ */
diff --git a/xlators/features/quiesce/src/quiesce.c b/xlators/features/quiesce/src/quiesce.c
index 3a4100f796e..0e5eb60a16f 100644
--- a/xlators/features/quiesce/src/quiesce.c
+++ b/xlators/features/quiesce/src/quiesce.c
@@ -8,663 +8,814 @@
    cases as published by the Free Software Foundation.
 */
 #include "quiesce.h"
-#include "defaults.h"
-#include "call-stub.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
 
 /* TODO: */
 /* Think about 'writev/_*_lk/setattr/xattrop/' fops to do re-transmittion */
 
+void
+gf_quiesce_timeout(void *data);
 
 /* Quiesce Specific Functions */
 void
-gf_quiesce_local_wipe (xlator_t *this, quiesce_local_t *local)
+gf_quiesce_local_wipe(xlator_t *this, quiesce_local_t *local)
 {
-        if (!local || !this || !this->private)
-                return;
+    if (!local || !this || !this->private)
+        return;
 
-        if (local->loc.inode)
-                loc_wipe (&local->loc);
-        if (local->fd)
-                fd_unref (local->fd);
-        GF_FREE (local->name);
-        GF_FREE (local->volname);
-        if (local->dict)
-                dict_unref (local->dict);
-        if (local->iobref)
-                iobref_unref (local->iobref);
-        GF_FREE (local->vector);
+    if (local->loc.inode)
+        loc_wipe(&local->loc);
+    if (local->fd)
+        fd_unref(local->fd);
+    GF_FREE(local->name);
+    GF_FREE(local->volname);
+    if (local->dict)
+        dict_unref(local->dict);
+    if (local->iobref)
+        iobref_unref(local->iobref);
+    GF_FREE(local->vector);
 
-        mem_put (local);
+    mem_put(local);
 }
 
-call_stub_t *
-gf_quiesce_dequeue (xlator_t *this)
+void
+__gf_quiesce_start_timer(xlator_t *this, quiesce_priv_t *priv)
 {
-        call_stub_t  *stub = NULL;
-        quiesce_priv_t *priv = NULL;
-
-        priv = this->private;
+    struct timespec timeout = {
+        0,
+    };
 
-        if (!priv || list_empty (&priv->req))
-                return NULL;
+    if (!priv->timer) {
+        timeout.tv_sec = priv->timeout;
+        timeout.tv_nsec = 0;
 
-        LOCK (&priv->lock);
-        {
-                stub = list_entry (priv->req.next, call_stub_t, list);
-                list_del_init (&stub->list);
-                priv->queue_size--;
+        priv->timer = gf_timer_call_after(this->ctx, timeout,
+                                          gf_quiesce_timeout, (void *)this);
+        if (priv->timer == NULL) {
+            gf_log(this->name, GF_LOG_ERROR, "Cannot create timer");
         }
-        UNLOCK (&priv->lock);
+    }
+}
+
+static void
+__gf_quiesce_cleanup_failover_hosts(xlator_t *this, quiesce_priv_t *priv)
+{
+    quiesce_failover_hosts_t *tmp = NULL;
+    quiesce_failover_hosts_t *failover_host = NULL;
 
-        return stub;
+    list_for_each_entry_safe(failover_host, tmp, &priv->failover_list, list)
+    {
+        GF_FREE(failover_host->addr);
+        list_del(&failover_host->list);
+        GF_FREE(failover_host);
+    }
+    return;
 }
 
-void *
-gf_quiesce_dequeue_start (void *data)
+void
+gf_quiesce_populate_failover_hosts(xlator_t *this, quiesce_priv_t *priv,
+                                   const char *value)
+{
+    char *dup_val = NULL;
+    char *addr_tok = NULL;
+    char *save_ptr = NULL;
+    quiesce_failover_hosts_t *failover_host = NULL;
+
+    if (!value)
+        goto out;
+
+    dup_val = gf_strdup(value);
+    if (!dup_val)
+        goto out;
+
+    addr_tok = strtok_r(dup_val, ",", &save_ptr);
+    LOCK(&priv->lock);
+    {
+        if (!list_empty(&priv->failover_list))
+            __gf_quiesce_cleanup_failover_hosts(this, priv);
+
+        while (addr_tok) {
+            if (!valid_internet_address(addr_tok, _gf_true, _gf_false)) {
+                gf_msg(this->name, GF_LOG_INFO, 0, QUIESCE_MSG_INVAL_HOST,
+                       "Specified "
+                       "invalid internet address:%s",
+                       addr_tok);
+                continue;
+            }
+            failover_host = GF_CALLOC(1, sizeof(*failover_host),
+                                      gf_quiesce_mt_failover_hosts);
+            failover_host->addr = gf_strdup(addr_tok);
+            INIT_LIST_HEAD(&failover_host->list);
+            list_add(&failover_host->list, &priv->failover_list);
+            addr_tok = strtok_r(NULL, ",", &save_ptr);
+        }
+    }
+    UNLOCK(&priv->lock);
+    GF_FREE(dup_val);
+out:
+    return;
+}
+
+int32_t
+gf_quiesce_failover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        xlator_t       *this = NULL;
-        quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
 
-        this = data;
-        priv = this->private;
-        THIS = this;
+    if (op_ret < 0) {
+        /* Failure here doesn't mean the failover to another host didn't
+         * succeed, we will know if failover succeeds or not by the
+         * CHILD_UP/CHILD_DOWN event. A failure here indicates something
+         * went wrong with the submission of failover command, hence
+         * just abort the failover attempts without retrying with other
+         * hosts.
+         */
+        gf_msg(this->name, GF_LOG_INFO, op_errno, QUIESCE_MSG_FAILOVER_FAILED,
+               "Initiating failover to host:%s failed:", (char *)cookie);
+    }
 
-        while (!list_empty (&priv->req)) {
-                stub = gf_quiesce_dequeue (this);
-                if (stub) {
-                        call_resume (stub);
-                }
-        }
+    GF_FREE(cookie);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    priv = this->private;
+    __gf_quiesce_start_timer(this, priv);
+
+    return 0;
 }
 
+int
+__gf_quiesce_perform_failover(xlator_t *this)
+{
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    quiesce_priv_t *priv = NULL;
+    quiesce_failover_hosts_t *failover_host = NULL;
+    quiesce_failover_hosts_t *host = NULL;
+
+    priv = this->private;
+
+    if (priv->pass_through) {
+        gf_msg_trace(this->name, 0,
+                     "child is up, hence not "
+                     "performing any failover");
+        goto out;
+    }
+
+    list_for_each_entry(failover_host, &priv->failover_list, list)
+    {
+        if (failover_host->tried == 0) {
+            host = failover_host;
+            failover_host->tried = 1;
+            break;
+        }
+    }
+    if (!host) {
+        /*TODO: Keep trying until any of the gfproxy comes back up.
+                Currently it tries failing over once for each host,
+                if it doesn't succeed then returns error to mount point
+           list_for_each_entry (failover_host,
+                        &priv->failover_list, list) {
+                failover_host->tried = 0;
+        }*/
+        gf_msg_debug(this->name, 0,
+                     "all the failover hosts have "
+                     "been tried and looks like didn't succeed");
+        ret = -1;
+        goto out;
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_msg_debug(this->name, 0, "failed to create the frame");
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    ret = dict_set_dynstr(dict, CLIENT_CMD_CONNECT, gf_strdup(host->addr));
+
+    gf_msg_trace(this->name, 0, "Initiating failover to:%s", host->addr);
+
+    STACK_WIND_COOKIE(frame, gf_quiesce_failover_cbk, NULL, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->setxattr, NULL, dict, 0, NULL);
+out:
 
-void
-gf_quiesce_timeout (void *data)
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
+}
+
+call_stub_t *
+gf_quiesce_dequeue(xlator_t *this)
 {
-        xlator_t       *this = NULL;
-        quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_priv_t *priv = NULL;
 
-        this = data;
-        priv = this->private;
-        THIS = this;
+    priv = this->private;
 
-        LOCK (&priv->lock);
-        {
-                priv->pass_through = _gf_true;
-        }
-        UNLOCK (&priv->lock);
+    if (!priv || list_empty(&priv->req))
+        return NULL;
 
-        gf_quiesce_dequeue_start (this);
+    LOCK(&priv->lock);
+    {
+        stub = list_entry(priv->req.next, call_stub_t, list);
+        list_del_init(&stub->list);
+        priv->queue_size--;
+    }
+    UNLOCK(&priv->lock);
 
-        return;
+    return stub;
 }
 
-void
-gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
+void *
+gf_quiesce_dequeue_start(void *data)
 {
-        quiesce_priv_t *priv    = NULL;
-        struct timespec timeout = {0,};
+    xlator_t *this = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
-        if (!priv) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                                  "this->private == NULL");
-                return;
-        }
+    this = data;
+    priv = this->private;
+    THIS = this;
 
-        LOCK (&priv->lock);
-        {
-                list_add_tail (&stub->list, &priv->req);
-                priv->queue_size++;
+    while (!list_empty(&priv->req)) {
+        stub = gf_quiesce_dequeue(this);
+        if (stub) {
+            call_resume(stub);
         }
-        UNLOCK (&priv->lock);
+    }
 
-        if (!priv->timer) {
-                timeout.tv_sec = 20;
-                timeout.tv_nsec = 0;
+    return 0;
+}
+
+void
+gf_quiesce_timeout(void *data)
+{
+    xlator_t *this = NULL;
+    quiesce_priv_t *priv = NULL;
+    int ret = -1;
+
+    this = data;
+    priv = this->private;
+    THIS = this;
 
-                priv->timer = gf_timer_call_after (this->ctx,
-                                                   timeout,
-                                                   gf_quiesce_timeout,
-                                                   (void *) this);
+    LOCK(&priv->lock);
+    {
+        priv->timer = NULL;
+        if (priv->pass_through) {
+            UNLOCK(&priv->lock);
+            goto out;
         }
+        ret = __gf_quiesce_perform_failover(THIS);
+    }
+    UNLOCK(&priv->lock);
 
-        return;
+    if (ret < 0) {
+        priv->pass_through = _gf_true;
+        gf_quiesce_dequeue_start(this);
+    }
+
+out:
+    return;
 }
 
+void
+gf_quiesce_enqueue(xlator_t *this, call_stub_t *stub)
+{
+    quiesce_priv_t *priv = NULL;
 
+    priv = this->private;
+    if (!priv) {
+        gf_log_callingfn(this->name, GF_LOG_ERROR, "this->private == NULL");
+        return;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_add_tail(&stub->list, &priv->req);
+        priv->queue_size++;
+        __gf_quiesce_start_timer(this, priv);
+    }
+    UNLOCK(&priv->lock);
+
+    return;
+}
 
 /* _CBK function section */
 
 int32_t
-quiesce_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *dict, struct iatt *postparent)
+quiesce_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_lookup_stub (frame, default_lookup_resume,
-                                        &local->loc, local->dict);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM,
-                                             NULL, NULL, NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_lookup_stub(frame, default_lookup_resume, &local->loc,
+                               local->dict);
+        if (!stub) {
+            STACK_UNWIND_STRICT(lookup, frame, -1, ENOMEM, NULL, NULL, NULL,
+                                NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             dict, postparent);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, dict,
+                        postparent);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  dict_t *xdata)
+quiesce_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_stat_stub (frame, default_stat_resume,
-                                      &local->loc, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_stat_stub(frame, default_stat_resume, &local->loc, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(stat, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quiesce_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_access_stub (frame, default_access_resume,
-                                        &local->loc, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_access_stub(frame, default_access_resume, &local->loc,
+                               local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(access, frame, -1, ENOMEM, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(access, frame, op_ret, op_errno, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf, dict_t *xdata)
+quiesce_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *path,
+                     struct iatt *buf, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readlink_stub (frame, default_readlink_resume,
-                                          &local->loc, local->size, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM,
-                                             NULL, NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readlink_stub(frame, default_readlink_resume, &local->loc,
+                                 local->size, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, path, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+quiesce_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_open_stub (frame, default_open_resume,
-                                      &local->loc, local->flag, local->fd,
-                                      xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (open, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_open_stub(frame, default_open_resume, &local->loc,
+                             local->flag, local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(open, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+quiesce_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readv_stub (frame, default_readv_resume,
-                                       local->fd, local->size, local->offset,
-                                       local->io_flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                             NULL, 0, NULL, NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readv_stub(frame, default_readv_resume, local->fd,
+                              local->size, local->offset, local->io_flag,
+                              xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
+                                NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quiesce_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_flush_stub (frame, default_flush_resume,
-                                       local->fd, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_flush_stub(frame, default_flush_resume, local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(flush, frame, -1, ENOMEM, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+quiesce_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fsync_stub (frame, default_fsync_resume,
-                                       local->fd, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM,
-                                             NULL, NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fsync_stub(frame, default_fsync_resume, local->fd,
+                              local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+quiesce_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fstat_stub (frame, default_fstat_resume,
-                                       local->fd, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fstat_stub(frame, default_fstat_resume, local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fstat, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+quiesce_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_opendir_stub (frame, default_opendir_resume,
-                                         &local->loc, local->fd, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_opendir_stub(frame, default_opendir_resume, &local->loc,
+                                local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(opendir, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quiesce_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume,
-                                          local->fd, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fsyncdir_stub(frame, default_fsyncdir_resume, local->fd,
+                                 local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fsyncdir, frame, -1, ENOMEM, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fsyncdir, frame, op_ret, op_errno, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+quiesce_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_statfs_stub (frame, default_statfs_resume,
-                                        &local->loc, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_statfs_stub(frame, default_statfs_resume, &local->loc,
+                               xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(statfs, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+quiesce_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume,
-                                           local->fd, local->name, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fgetxattr_stub(frame, default_fgetxattr_resume, local->fd,
+                                  local->name, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fgetxattr, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+quiesce_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_getxattr_stub (frame, default_getxattr_resume,
-                                          &local->loc, local->name, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_getxattr_stub(frame, default_getxattr_resume, &local->loc,
+                                 local->name, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum, dict_t *xdata)
+quiesce_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                      uint8_t *strong_checksum, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                           local->fd, local->offset, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM,
-                                             0, NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_rchecksum_stub(frame, default_rchecksum_resume, local->fd,
+                                  local->offset, local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                        strong_checksum, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+quiesce_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readdir_stub (frame, default_readdir_resume,
-                                         local->fd, local->size, local->offset, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readdir_stub(frame, default_readdir_resume, local->fd,
+                                local->size, local->offset, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, entries, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+quiesce_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readdirp_stub (frame, default_readdirp_resume,
-                                          local->fd, local->size, local->offset,
-                                          local->dict);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readdirp_stub(frame, default_readdirp_resume, local->fd,
+                                 local->size, local->offset, local->dict);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 #if 0
 
 int32_t
@@ -1010,1596 +1161,1544 @@ out:
 
 #endif /* if 0 */
 
-
 /* FOP */
 
 /* No retransmittion */
 
 int32_t
-quiesce_removexattr (call_frame_t *frame,
-		     xlator_t *this,
-		     loc_t *loc,
-		     const char *name, dict_t *xdata)
+quiesce_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_removexattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->removexattr,
-                            loc,
-                            name, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_removexattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+        return 0;
+    }
 
-        stub = fop_removexattr_stub (frame, default_removexattr_resume,
-                                     loc, name, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_removexattr_stub(frame, default_removexattr_resume, loc, name,
+                                xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(removexattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_truncate (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  off_t offset, dict_t *xdata)
+quiesce_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_truncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            loc,
-                            offset, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_fremovexattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+        return 0;
+    }
 
-        stub = fop_truncate_stub (frame, default_truncate_resume, loc, offset, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_fremovexattr_stub(frame, default_fremovexattr_resume, fd, name,
+                                 xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fremovexattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fsetxattr (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t *fd,
-                   dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+quiesce_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_fsetxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsetxattr,
-                            fd,
-                            dict,
-                            flags, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_truncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        return 0;
+    }
 
-        stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume,
-                                   fd, dict, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_truncate_stub(frame, default_truncate_resume, loc, offset,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_setxattr (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  dict_t *dict,
-		  int32_t flags, dict_t *xdata)
+quiesce_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_setxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setxattr,
-                            loc,
-                            dict,
-                            flags, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+        return 0;
+    }
 
-        stub = fop_setxattr_stub (frame, default_setxattr_resume,
-                                  loc, dict, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_fsetxattr_stub(frame, default_fsetxattr_resume, fd, dict, flags,
+                              xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_create (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, int32_t flags, mode_t mode,
-                mode_t umask, fd_t *fd, dict_t *xdata)
+quiesce_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                /* Don't send O_APPEND below, as write() re-transmittions can
-                   fail with O_APPEND */
-                STACK_WIND (frame, default_create_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->create,
-                            loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_setxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+        return 0;
+    }
 
-        stub = fop_create_stub (frame, default_create_resume,
-                                loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (create, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_setxattr_stub(frame, default_setxattr_resume, loc, dict, flags,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(setxattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_link (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc, dict_t *xdata)
+quiesce_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_link_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        /* Don't send O_APPEND below, as write() re-transmittions can
+           fail with O_APPEND */
+        STACK_WIND(frame, default_create_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->create, loc, (flags & ~O_APPEND),
+                   mode, umask, fd, xdata);
+        return 0;
+    }
 
-        stub = fop_link_stub (frame, default_link_resume, oldloc, newloc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (link, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_create_stub(frame, default_create_resume, loc,
+                           (flags & ~O_APPEND), mode, umask, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_rename (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *oldloc,
-		loc_t *newloc, dict_t *xdata)
+quiesce_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_rename_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_link_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+        return 0;
+    }
 
-        stub = fop_rename_stub (frame, default_rename_resume, oldloc, newloc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_link_stub(frame, default_link_resume, oldloc, newloc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+quiesce_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+        return 0;
+    }
+
+    stub = fop_rename_stub(frame, default_rename_resume, oldloc, newloc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rename, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+        return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
 
 int
-quiesce_symlink (call_frame_t *frame, xlator_t *this,
-		 const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
+quiesce_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_symlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, umask, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_symlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                   xdata);
+        return 0;
+    }
 
-        stub = fop_symlink_stub (frame, default_symlink_resume,
-                                 linkpath, loc, umask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_symlink_stub(frame, default_symlink_resume, linkpath, loc, umask,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(symlink, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
+quiesce_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_rmdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_rmdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+        return 0;
+    }
 
-        stub = fop_rmdir_stub (frame, default_rmdir_resume, loc, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_rmdir_stub(frame, default_rmdir_resume, loc, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_unlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc, int xflag, dict_t *xdata)
+quiesce_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_unlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            loc, xflag, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+    }
 
-        stub = fop_unlink_stub (frame, default_unlink_resume, loc, xflag, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_unlink_stub(frame, default_unlink_resume, loc, xflag, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int
-quiesce_mkdir (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+quiesce_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_mkdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, umask, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+        return 0;
+    }
 
-        stub = fop_mkdir_stub (frame, default_mkdir_resume,
-                               loc, mode, umask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_mkdir_stub(frame, default_mkdir_resume, loc, mode, umask, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(mkdir, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quiesce_mknod (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+quiesce_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_mknod_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, umask, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_mknod_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                   xdata);
+        return 0;
+    }
 
-        stub = fop_mknod_stub (frame, default_mknod_resume,
-                               loc, mode, rdev, umask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_mknod_stub(frame, default_mknod_resume, loc, mode, rdev, umask,
+                          xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_ftruncate (call_frame_t *frame,
-		   xlator_t *this,
-		   fd_t *fd,
-		   off_t offset, dict_t *xdata)
+quiesce_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_ftruncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate,
-                            fd,
-                            offset, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_ftruncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        return 0;
+    }
 
-        stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_ftruncate_stub(frame, default_ftruncate_resume, fd, offset,
+                              xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 /* Re-transmittion */
 
 int32_t
-quiesce_readlink (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  size_t size, dict_t *xdata)
+quiesce_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                 dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->size = size;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readlink,
-                            loc,
-                            size, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_readlink_stub (frame, default_readlink_resume, loc, size, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->size = size;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+        return 0;
+    }
 
+    stub = fop_readlink_stub(frame, default_readlink_resume, loc, size, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
-}
+    }
+
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
 
 int32_t
-quiesce_access (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t mask, dict_t *xdata)
+quiesce_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+               dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
-
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->flag = mask;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_access_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->access,
-                            loc,
-                            mask, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_access_stub (frame, default_access_resume, loc, mask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->flag = mask;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_access_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+        return 0;
+    }
 
+    stub = fop_access_stub(frame, default_access_resume, loc, mask, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(access, frame, -1, ENOMEM, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fgetxattr (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t *fd,
-                   const char *name, dict_t *xdata)
+quiesce_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
 {
-        quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                if (name)
-                        local->name = gf_strdup (name);
-
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fgetxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fgetxattr,
-                            fd,
-                            name, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        if (name)
+            local->name = gf_strdup(name);
 
-        stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fgetxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+        return 0;
+    }
 
+    stub = fop_fgetxattr_stub(frame, default_fgetxattr_resume, fd, name, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fgetxattr, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_statfs (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc, dict_t *xdata)
+quiesce_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_statfs_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->statfs,
-                            loc, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        frame->local = local;
 
-        stub = fop_statfs_stub (frame, default_statfs_resume, loc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_statfs_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+    }
 
+    stub = fop_statfs_stub(frame, default_statfs_resume, loc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(statfs, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fsyncdir (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  int32_t flags, dict_t *xdata)
+quiesce_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                 dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->flag = flags;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fsyncdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsyncdir,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->flag = flags;
+        frame->local = local;
 
-        stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume, fd, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fsyncdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata);
+        return 0;
+    }
 
+    stub = fop_fsyncdir_stub(frame, default_fsyncdir_resume, fd, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, ENOMEM, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_opendir (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc, fd_t *fd, dict_t *xdata)
+quiesce_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->fd = fd_ref (fd);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_opendir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->opendir,
-                            loc, fd, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_opendir_stub (frame, default_opendir_resume, loc, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->fd = fd_ref(fd);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_opendir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+        return 0;
+    }
 
+    stub = fop_opendir_stub(frame, default_opendir_resume, loc, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(opendir, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fstat (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd, dict_t *xdata)
+quiesce_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fstat_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fstat,
-                            fd, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_fstat_stub (frame, default_fstat_resume, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+    }
 
+    stub = fop_fstat_stub(frame, default_fstat_resume, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fstat, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fsync (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       int32_t flags, dict_t *xdata)
+quiesce_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->flag = flags;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fsync_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsync,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->flag = flags;
+        frame->local = local;
 
-        stub = fop_fsync_stub (frame, default_fsync_resume, fd, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fsync_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+        return 0;
+    }
 
+    stub = fop_fsync_stub(frame, default_fsync_resume, fd, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_flush (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd, dict_t *xdata)
+quiesce_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_flush_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush,
-                            fd, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_flush_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->flush, fd, xdata);
+        return 0;
+    }
 
+    stub = fop_flush_stub(frame, default_flush_resume, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(flush, frame, -1, ENOMEM, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_writev (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		struct iovec *vector,
-		int32_t count,
-		off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
+quiesce_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_writev_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->writev,
-                            fd,
-                            vector,
-                            count,
-                            off, flags,
-                            iobref, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_writev_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                   flags, iobref, xdata);
+        return 0;
+    }
 
-        stub = fop_writev_stub (frame, default_writev_resume,
-                                fd, vector, count, off, flags, iobref, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_writev_stub(frame, default_writev_resume, fd, vector, count, off,
+                           flags, iobref, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_readv (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset, uint32_t flags, dict_t *xdata)
+quiesce_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
-
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->size = size;
-                local->offset = offset;
-                local->io_flag = flags;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readv_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readv,
-                            fd,
-                            size,
-                            offset, flags, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_readv_stub (frame, default_readv_resume, fd, size, offset,
-                               flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                     NULL, 0, NULL, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->size = size;
+        local->offset = offset;
+        local->io_flag = flags;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+        return 0;
+    }
 
+    stub = fop_readv_stub(frame, default_readv_resume, fd, size, offset, flags,
+                          xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
+                            NULL);
         return 0;
-}
+    }
+
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
 
 int32_t
-quiesce_open (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags, fd_t *fd,
-              dict_t *xdata)
+quiesce_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->fd = fd_ref (fd);
-
-                /* Don't send O_APPEND below, as write() re-transmittions can
-                   fail with O_APPEND */
-                local->flag = (flags & ~O_APPEND);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_open_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open,
-                            loc, (flags & ~O_APPEND), fd, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->fd = fd_ref(fd);
 
-        stub = fop_open_stub (frame, default_open_resume, loc,
-                              (flags & ~O_APPEND), fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+        /* Don't send O_APPEND below, as write() re-transmittions can
+           fail with O_APPEND */
+        local->flag = (flags & ~O_APPEND);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_open_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->open, loc, (flags & ~O_APPEND), fd,
+                   xdata);
+        return 0;
+    }
 
+    stub = fop_open_stub(frame, default_open_resume, loc, (flags & ~O_APPEND),
+                         fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(open, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_getxattr (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  const char *name, dict_t *xdata)
+quiesce_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                if (name)
-                        local->name = gf_strdup (name);
-
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_getxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->getxattr,
-                            loc,
-                            name, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        if (name)
+            local->name = gf_strdup(name);
 
-        stub = fop_getxattr_stub (frame, default_getxattr_resume, loc, name, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_getxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+    }
 
+    stub = fop_getxattr_stub(frame, default_getxattr_resume, loc, name, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
         return 0;
-}
+    }
+
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
 
 int32_t
-quiesce_xattrop (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 gf_xattrop_flags_t flags,
-		 dict_t *dict, dict_t *xdata)
+quiesce_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_xattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->xattrop,
-                            loc,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_xattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+        return 0;
+    }
 
-        stub = fop_xattrop_stub (frame, default_xattrop_resume,
-                                 loc, flags, dict, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_xattrop_stub(frame, default_xattrop_resume, loc, flags, dict,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(xattrop, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fxattrop (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  gf_xattrop_flags_t flags,
-		  dict_t *dict, dict_t *xdata)
+quiesce_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_fxattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fxattrop,
-                            fd,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fxattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+        return 0;
+    }
 
-        stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
-                                  fd, flags, dict, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_fxattrop_stub(frame, default_fxattrop_resume, fd, flags, dict,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
+
+int32_t
+quiesce_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+           struct gf_flock *lock, dict_t *xdata)
+{
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_lk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
         return 0;
+    }
+
+    stub = fop_lk_stub(frame, default_lk_resume, fd, cmd, lock, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(lk, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_lk (call_frame_t *frame,
-	    xlator_t *this,
-	    fd_t *fd,
-	    int32_t cmd,
-	    struct gf_flock *lock, dict_t *xdata)
+quiesce_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_lk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lk,
-                            fd,
-                            cmd,
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_inodelk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, lock,
+                   xdata);
+        return 0;
+    }
+
+    stub = fop_inodelk_stub(frame, default_inodelk_resume, volume, loc, cmd,
                             lock, xdata);
-                return 0;
-        }
+    if (!stub) {
+        STACK_UNWIND_STRICT(inodelk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, lock, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
+
+int32_t
+quiesce_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_quiesce_enqueue (this, stub);
+    priv = this->private;
 
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_finodelk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, lock,
+                   xdata);
         return 0;
-}
+    }
 
+    stub = fop_finodelk_stub(frame, default_finodelk_resume, volume, fd, cmd,
+                             lock, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(finodelk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
 
 int32_t
-quiesce_inodelk (call_frame_t *frame, xlator_t *this,
-		 const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock *lock, dict_t *xdata)
+quiesce_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                loc_t *loc, const char *basename, entrylk_cmd cmd,
+                entrylk_type type, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_inodelk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->inodelk,
-                            volume, loc, cmd, lock, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_entrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+                   type, xdata);
+        return 0;
+    }
 
-        stub = fop_inodelk_stub (frame, default_inodelk_resume,
-                                 volume, loc, cmd, lock, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_entrylk_stub(frame, default_entrylk_resume, volume, loc,
+                            basename, cmd, type, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(entrylk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_finodelk (call_frame_t *frame, xlator_t *this,
-		  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+quiesce_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 fd_t *fd, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_finodelk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->finodelk,
-                            volume, fd, cmd, lock, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fentrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+                   type, xdata);
+        return 0;
+    }
 
-        stub = fop_finodelk_stub (frame, default_finodelk_resume,
-                                  volume, fd, cmd, lock, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_fentrylk_stub(frame, default_fentrylk_resume, volume, fd,
+                             basename, cmd, type, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fentrylk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_entrylk (call_frame_t *frame, xlator_t *this,
-		 const char *volume, loc_t *loc, const char *basename,
-		 entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+quiesce_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  int32_t len, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame, default_entrylk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->entrylk,
-                            volume, loc, basename, cmd, type, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_entrylk_stub (frame, default_entrylk_resume,
-                                 volume, loc, basename, cmd, type, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->offset = offset;
+        local->flag = len;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_rchecksum_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+        return 0;
+    }
 
+    stub = fop_rchecksum_stub(frame, default_rchecksum_resume, fd, offset, len,
+                              xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fentrylk (call_frame_t *frame, xlator_t *this,
-		  const char *volume, fd_t *fd, const char *basename,
-		  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+quiesce_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame, default_fentrylk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fentrylk,
-                            volume, fd, basename, cmd, type, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->size = size;
+        local->offset = off;
+        frame->local = local;
 
-        stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
-                                  volume, fd, basename, cmd, type, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+        return 0;
+    }
 
+    stub = fop_readdir_stub(frame, default_readdir_resume, fd, size, off,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_rchecksum (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t *fd, off_t offset,
-                   int32_t len, dict_t *xdata)
+quiesce_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *dict)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->offset = offset;
-                local->flag = len;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_rchecksum_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rchecksum,
-                            fd, offset, len, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                   fd, offset, len, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->size = size;
+        local->offset = off;
+        local->dict = dict_ref(dict);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readdirp_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
+        return 0;
+    }
 
+    stub = fop_readdirp_stub(frame, default_readdirp_resume, fd, size, off,
+                             dict);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
         return 0;
-}
+    }
+
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
 
 int32_t
-quiesce_readdir (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 size_t size,
-		 off_t off, dict_t *xdata)
+quiesce_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->size = size;
-                local->offset = off;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+    }
 
-        stub = fop_readdir_stub (frame, default_readdir_resume, fd, size, off, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_setattr_stub(frame, default_setattr_resume, loc, stbuf, valid,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_readdirp (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  size_t size,
-		  off_t off, dict_t *dict)
+quiesce_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->size = size;
-                local->offset = off;
-                local->dict = dict_ref (dict);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readdirp_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, off, dict);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        frame->local = local;
 
-        stub = fop_readdirp_stub (frame, default_readdirp_resume, fd, size,
-                                  off, dict);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+    }
 
+    stub = fop_stat_stub(frame, default_stat_resume, loc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(stat, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_setattr (call_frame_t *frame,
-                 xlator_t *this,
-                 loc_t *loc,
-                 struct iatt *stbuf,
-                 int32_t valid, dict_t *xdata)
+quiesce_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               dict_t *xattr_req)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
-
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_setattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_setattr_stub (frame, default_setattr_resume,
-                                   loc, stbuf, valid, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->dict = dict_ref(xattr_req);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+        return 0;
+    }
 
+    stub = fop_lookup_stub(frame, default_lookup_resume, loc, xattr_req);
+    if (!stub) {
+        STACK_UNWIND_STRICT(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
         return 0;
-}
+    }
+
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
 
 int32_t
-quiesce_stat (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc, dict_t *xdata)
+quiesce_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_stat_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->stat,
-                            loc, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+    }
 
-        stub = fop_stat_stub (frame, default_stat_resume, loc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_fsetattr_stub(frame, default_fsetattr_resume, fd, stbuf, valid,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_lookup (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *xattr_req)
+quiesce_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+                  off_t offset, size_t len, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->dict = dict_ref (xattr_req);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_lookup_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup,
-                            loc, xattr_req);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fallocate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+                   xdata);
+        return 0;
+    }
+
+    stub = fop_fallocate_stub(frame, default_fallocate_resume, fd, mode, offset,
+                              len, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        stub = fop_lookup_stub (frame, default_lookup_resume, loc, xattr_req);
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
+
+int
+quiesce_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
+
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_seek_stub(frame, default_seek_resume, local->fd,
+                             local->offset, local->what, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
-                return 0;
+            STACK_UNWIND_STRICT(seek, frame, -1, ENOMEM, 0, NULL);
+            goto out;
         }
 
-        gf_quiesce_enqueue (this, stub);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
 
-        return 0;
+    STACK_UNWIND_STRICT(seek, frame, op_ret, op_errno, offset, xdata);
+out:
+    gf_quiesce_local_wipe(this, local);
+
+    return 0;
 }
 
-int32_t
-quiesce_fsetattr (call_frame_t *frame,
-                  xlator_t *this,
-                  fd_t *fd,
-                  struct iatt *stbuf,
-                  int32_t valid, dict_t *xdata)
+int
+quiesce_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             gf_seek_what_t what, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_fsetattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->offset = offset;
+        local->what = what;
 
-        stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
-                                  fd, stbuf, valid, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_seek_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+        return 0;
+    }
 
+    stub = fop_seek_stub(frame, default_seek_resume, fd, offset, what, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(seek, frame, -1, ENOMEM, 0, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_quiesce_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_quiesce_mt_end + 1);
 
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int ret = -1;
-        quiesce_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quiesce_priv_t *priv = NULL;
 
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"'quiesce' not configured with exactly one child");
-                goto out;
-	}
+    priv = this->private;
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
+    GF_OPTION_RECONF("timeout", priv->timeout, options, time, out);
+    GF_OPTION_RECONF("failover-hosts", priv->failover_hosts, options, str, out);
+    gf_quiesce_populate_failover_hosts(this, priv, priv->failover_hosts);
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_quiesce_mt_priv_t);
-        if (!priv)
-                goto out;
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+init(xlator_t *this)
+{
+    int ret = -1;
+    quiesce_priv_t *priv = NULL;
 
-        priv->local_pool =  mem_pool_new (quiesce_local_t,
-                                          GF_FOPS_EXPECTED_IN_PARALLEL);
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'quiesce' not configured with exactly one child");
+        goto out;
+    }
 
-        LOCK_INIT (&priv->lock);
-        priv->pass_through = _gf_false;
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-        INIT_LIST_HEAD (&priv->req);
+    priv = GF_CALLOC(1, sizeof(*priv), gf_quiesce_mt_priv_t);
+    if (!priv)
+        goto out;
 
-        this->private = priv;
-        ret = 0;
+    INIT_LIST_HEAD(&priv->failover_list);
+
+    GF_OPTION_INIT("timeout", priv->timeout, time, out);
+    GF_OPTION_INIT("failover-hosts", priv->failover_hosts, str, out);
+    gf_quiesce_populate_failover_hosts(this, priv, priv->failover_hosts);
+
+    priv->local_pool = mem_pool_new(quiesce_local_t,
+                                    GF_FOPS_EXPECTED_IN_PARALLEL);
+
+    LOCK_INIT(&priv->lock);
+    priv->pass_through = _gf_false;
+
+    INIT_LIST_HEAD(&priv->req);
+
+    this->private = priv;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        quiesce_priv_t *priv = NULL;
+    quiesce_priv_t *priv = NULL;
 
-        priv = this->private;
-        if (!priv)
-                goto out;
-        this->private = NULL;
+    priv = this->private;
+    if (!priv)
+        goto out;
+    this->private = NULL;
 
-        mem_pool_destroy (priv->local_pool);
-        LOCK_DESTROY (&priv->lock);
-        GF_FREE (priv);
+    mem_pool_destroy(priv->local_pool);
+    priv->local_pool = NULL;
+    LOCK_DESTROY(&priv->lock);
+    GF_FREE(priv);
 out:
-        return;
+    return;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int             ret     = 0;
-        quiesce_priv_t *priv    = NULL;
-        struct timespec timeout = {0,};
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        switch (event) {
-        case GF_EVENT_CHILD_UP:
-        {
-                ret = pthread_create (&priv->thr, NULL, gf_quiesce_dequeue_start,
-                                      this);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to create the quiesce-dequeue thread");
-                }
-
-                LOCK (&priv->lock);
-                {
-                        priv->pass_through = _gf_true;
-                }
-                UNLOCK (&priv->lock);
-                break;
+notify(xlator_t *this, int event, void *data, ...)
+{
+    int ret = 0;
+    quiesce_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    switch (event) {
+        case GF_EVENT_CHILD_UP: {
+            ret = gf_thread_create(&priv->thr, NULL, gf_quiesce_dequeue_start,
+                                   this, "quiesce");
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to create the quiesce-dequeue thread");
+            }
+
+            LOCK(&priv->lock);
+            {
+                priv->pass_through = _gf_true;
+            }
+            UNLOCK(&priv->lock);
+            break;
         }
         case GF_EVENT_CHILD_DOWN:
-                LOCK (&priv->lock);
-                {
-                        priv->pass_through = _gf_false;
-                }
-                UNLOCK (&priv->lock);
-
-                if (priv->timer)
-                        break;
-                timeout.tv_sec = 20;
-                timeout.tv_nsec = 0;
-
-                priv->timer = gf_timer_call_after (this->ctx,
-                                                   timeout,
-                                                   gf_quiesce_timeout,
-                                                   (void *) this);
-
-                if (priv->timer == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Cannot create timer");
-                }
-
-                break;
+            LOCK(&priv->lock);
+            {
+                priv->pass_through = _gf_false;
+                __gf_quiesce_start_timer(this, priv);
+            }
+            UNLOCK(&priv->lock);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = default_notify (this, event, data);
+    ret = default_notify(this, event, data);
 out:
-        return ret;
+    return ret;
 }
 
-
 struct xlator_fops fops = {
-        /* write/modifying fops */
-	.mknod       = quiesce_mknod,
-	.create      = quiesce_create,
-	.truncate    = quiesce_truncate,
-	.ftruncate   = quiesce_ftruncate,
-	.setxattr    = quiesce_setxattr,
-	.removexattr = quiesce_removexattr,
-	.symlink     = quiesce_symlink,
-	.unlink      = quiesce_unlink,
-	.link        = quiesce_link,
-	.mkdir       = quiesce_mkdir,
-	.rmdir       = quiesce_rmdir,
-	.rename      = quiesce_rename,
-
-        /* The below calls are known to change state, hence
-           re-transmittion is not advised */
-	.lk          = quiesce_lk,
-	.inodelk     = quiesce_inodelk,
-	.finodelk    = quiesce_finodelk,
-	.entrylk     = quiesce_entrylk,
-	.fentrylk    = quiesce_fentrylk,
-	.xattrop     = quiesce_xattrop,
-	.fxattrop    = quiesce_fxattrop,
-        .setattr     = quiesce_setattr,
-        .fsetattr    = quiesce_fsetattr,
-
-        /* Special case, re-transmittion is not harmful *
-         * as offset is properly sent from above layers */
-        /* TODO: not re-transmitted as of now */
-	.writev      = quiesce_writev,
-
-        /* re-transmittable fops */
-	.lookup      = quiesce_lookup,
-	.stat        = quiesce_stat,
-	.fstat       = quiesce_fstat,
-	.access      = quiesce_access,
-	.readlink    = quiesce_readlink,
-	.getxattr    = quiesce_getxattr,
-	.open        = quiesce_open,
-	.readv       = quiesce_readv,
-	.flush       = quiesce_flush,
-	.fsync       = quiesce_fsync,
-	.statfs      = quiesce_statfs,
-	.opendir     = quiesce_opendir,
-	.readdir     = quiesce_readdir,
-	.readdirp    = quiesce_readdirp,
-	.fsyncdir    = quiesce_fsyncdir,
-
+    /* write/modifying fops */
+    .mknod = quiesce_mknod,
+    .create = quiesce_create,
+    .truncate = quiesce_truncate,
+    .ftruncate = quiesce_ftruncate,
+    .setxattr = quiesce_setxattr,
+    .fsetxattr = quiesce_fsetxattr,
+    .removexattr = quiesce_removexattr,
+    .fremovexattr = quiesce_fremovexattr,
+    .symlink = quiesce_symlink,
+    .unlink = quiesce_unlink,
+    .link = quiesce_link,
+    .mkdir = quiesce_mkdir,
+    .rmdir = quiesce_rmdir,
+    .rename = quiesce_rename,
+    .fallocate = quiesce_fallocate,
+
+    /* The below calls are known to change state, hence
+       re-transmittion is not advised */
+    .lk = quiesce_lk,
+    .inodelk = quiesce_inodelk,
+    .finodelk = quiesce_finodelk,
+    .entrylk = quiesce_entrylk,
+    .fentrylk = quiesce_fentrylk,
+    .xattrop = quiesce_xattrop,
+    .fxattrop = quiesce_fxattrop,
+    .setattr = quiesce_setattr,
+    .fsetattr = quiesce_fsetattr,
+
+    /* Special case, re-transmittion is not harmful *
+     * as offset is properly sent from above layers */
+    /* TODO: not re-transmitted as of now */
+    .writev = quiesce_writev,
+
+    /* re-transmittable fops */
+    .lookup = quiesce_lookup,
+    .stat = quiesce_stat,
+    .fstat = quiesce_fstat,
+    .access = quiesce_access,
+    .readlink = quiesce_readlink,
+    .getxattr = quiesce_getxattr,
+    .fgetxattr = quiesce_fgetxattr,
+    .open = quiesce_open,
+    .readv = quiesce_readv,
+    .flush = quiesce_flush,
+    .fsync = quiesce_fsync,
+    .statfs = quiesce_statfs,
+    .opendir = quiesce_opendir,
+    .readdir = quiesce_readdir,
+    .readdirp = quiesce_readdirp,
+    .fsyncdir = quiesce_fsyncdir,
+    .seek = quiesce_seek,
 };
 
 struct xlator_dumpops dumpops;
 
-
 struct xlator_cbks cbks;
 
-
 struct volume_options options[] = {
-	{ .key  = {NULL} },
+    {
+        .key = {"timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .default_value = "45",
+        .description =
+            "After 'timeout' seconds since the time 'quiesce' "
+            "option was set to \"!pass-through\", acknowledgements to file "
+            "operations are no longer quiesced and previously "
+            "quiesced acknowledgements are sent to the application",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+    },
+    {.key = {"failover-hosts"},
+     .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .description = "It is a comma separated list of hostname/IP "
+                    "addresses. It Specifies the list of hosts where "
+                    "the gfproxy daemons are running, to which the "
+                    "the thin clients can failover to."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "quiesce",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/quiesce/src/quiesce.h b/xlators/features/quiesce/src/quiesce.h
index 878ed77e928..6ab2af40a56 100644
--- a/xlators/features/quiesce/src/quiesce.h
+++ b/xlators/features/quiesce/src/quiesce.h
@@ -12,40 +12,54 @@
 #define __QUIESCE_H__
 
 #include "quiesce-mem-types.h"
-#include "xlator.h"
-#include "timer.h"
+#include "quiesce-messages.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
 
 #define GF_FOPS_EXPECTED_IN_PARALLEL 512
 
 typedef struct {
-        gf_timer_t       *timer;
-        gf_boolean_t      pass_through;
-        gf_lock_t         lock;
-        struct list_head  req;
-        int               queue_size;
-        pthread_t         thr;
-        struct mem_pool  *local_pool;
+    struct list_head list;
+    char *addr;
+    gf_boolean_t tried; /* indicates attempted connecting */
+} quiesce_failover_hosts_t;
+
+typedef struct {
+    gf_timer_t *timer;
+    gf_boolean_t pass_through;
+    gf_lock_t lock;
+    struct list_head req;
+    int queue_size;
+    pthread_t thr;
+    struct mem_pool *local_pool;
+    uint32_t timeout;
+    char *failover_hosts;
+    struct list_head failover_list;
 } quiesce_priv_t;
 
 typedef struct {
-        fd_t               *fd;
-        char               *name;
-        char               *volname;
-        loc_t               loc;
-        off_t               size;
-        off_t               offset;
-        mode_t              mode;
-        int32_t             flag;
-        struct iatt         stbuf;
-        struct iovec       *vector;
-        struct iobref      *iobref;
-        dict_t             *dict;
-        struct gf_flock     flock;
-        entrylk_cmd         cmd;
-        entrylk_type        type;
-        gf_xattrop_flags_t  xattrop_flags;
-        int32_t             wbflags;
-        uint32_t            io_flag;
+    fd_t *fd;
+    char *name;
+    char *volname;
+    loc_t loc;
+    off_t size;
+    off_t offset;
+    mode_t mode;
+    int32_t flag;
+    struct iatt stbuf;
+    struct iovec *vector;
+    struct iobref *iobref;
+    dict_t *dict;
+    struct gf_flock flock;
+    entrylk_cmd cmd;
+    entrylk_type type;
+    gf_xattrop_flags_t xattrop_flags;
+    int32_t wbflags;
+    uint32_t io_flag;
+    /* for fallocate */
+    size_t len;
+    /* for lseek */
+    gf_seek_what_t what;
 } quiesce_local_t;
 
 #endif
diff --git a/xlators/features/quota/src/Makefile.am b/xlators/features/quota/src/Makefile.am
index a15135347ac..1c2dcef0ca3 100644
--- a/xlators/features/quota/src/Makefile.am
+++ b/xlators/features/quota/src/Makefile.am
@@ -1,25 +1,29 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = quota.la quotad.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 quota_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-quotad_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/features/quota/src/quotad.sym
+quotad_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 quota_la_SOURCES = quota.c quota-enforcer-client.c
-quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+                  $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+                  $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
 quotad_la_SOURCES = quotad.c quotad-helpers.c quotad-aggregator.c
-quotad_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+quotad_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+                   $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+                   $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
 noinst_HEADERS = quota-mem-types.h quota.h quotad-aggregator.h \
-                 quotad-helpers.h quota-messages.h
+	quotad-helpers.h quota-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/cluster/dht/src -I$(top_srcdir)/rpc/xdr/src/ \
-	-I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/xlators/cluster/dht/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
-
-EXTRA_DIST = quotad.sym
-
diff --git a/xlators/features/quota/src/quota-enforcer-client.c b/xlators/features/quota/src/quota-enforcer-client.c
index 6f36c081dbc..480d64ade27 100644
--- a/xlators/features/quota/src/quota-enforcer-client.c
+++ b/xlators/features/quota/src/quota-enforcer-client.c
@@ -32,460 +32,472 @@
 #include <malloc.h>
 #endif
 
-#ifdef HAVE_MALLOC_STATS
-#ifdef DEBUG
-#include <mcheck.h>
-#endif
-#endif
-
 #include "quota.h"
 #include "quota-messages.h"
 
 extern struct rpc_clnt_program quota_enforcer_clnt;
 
 int32_t
-quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent);
+quota_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent);
 
 int
-quota_enforcer_submit_request (void *req, call_frame_t *frame,
-                               rpc_clnt_prog_t *prog,
-                               int procnum, struct iobref *iobref,
-                               xlator_t *this, fop_cbk_fn_t cbkfn,
-                               xdrproc_t xdrproc)
+quota_enforcer_submit_request(void *req, call_frame_t *frame,
+                              rpc_clnt_prog_t *prog, int procnum,
+                              struct iobref *iobref, xlator_t *this,
+                              fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-        quota_priv_t *priv       = NULL;
-
-        GF_ASSERT (this);
-
-        priv = this->private;
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                }
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+    quota_priv_t *priv = NULL;
+
+    GF_ASSERT(this);
+
+    priv = this->private;
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        }
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
         }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (priv->rpc_clnt, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    ret = rpc_clnt_submit(priv->rpc_clnt, prog, procnum, cbkfn, &iov, count,
+                          NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return ret;
+    return ret;
 }
 
 int
-quota_enforcer_lookup_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
+quota_enforcer_lookup_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        quota_local_t    *local       = NULL;
-        call_frame_t     *frame       = NULL;
-        int               ret         = 0;
-        gfs3_lookup_rsp   rsp         = {0,};
-        struct iatt       stbuf       = {0,};
-        struct iatt       postparent  = {0,};
-        int               op_errno    = EINVAL;
-        dict_t           *xdata       = NULL;
-        inode_t          *inode       = NULL;
-        xlator_t         *this        = NULL;
-        quota_priv_t     *priv        = NULL;
-        struct timespec   retry_delay = {0,};
-        gf_timer_t       *timer       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-        inode = local->validate_loc.inode;
-        priv  = this->private;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    quota_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int op_errno = EINVAL;
+    dict_t *xdata = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    quota_priv_t *priv = NULL;
+    struct timespec retry_delay = {
+        0,
+    };
+    gf_timer_t *timer = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+    inode = local->validate_loc.inode;
+    priv = this->private;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_XDR_DECODING_FAILED,
+               "XDR decoding failed");
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        Q_MSG_XDR_DECODING_FAILED,
-                        "XDR decoding failed");
-                rsp.op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    gf_stat_to_iatt(&rsp.postparent, &postparent);
 
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        gf_stat_to_iatt (&rsp.postparent, &postparent);
+    if (rsp.op_ret == -1)
+        goto out;
 
-        if (rsp.op_ret == -1)
-                goto out;
+    rsp.op_ret = -1;
+    gf_stat_to_iatt(&rsp.stat, &stbuf);
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(frame->this, xdata, (rsp.xdata.xdata_val),
+                                 (rsp.xdata.xdata_len), rsp.op_ret, op_errno,
+                                 out);
 
+    if ((!gf_uuid_is_null(inode->gfid)) &&
+        (gf_uuid_compare(stbuf.ia_gfid, inode->gfid) != 0)) {
+        gf_msg_debug(frame->this->name, ESTALE, "gfid changed for %s",
+                     local->validate_loc.path);
         rsp.op_ret = -1;
-        gf_stat_to_iatt (&rsp.stat, &stbuf);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), rsp.op_ret,
-                                      op_errno, out);
-
-        if ((!gf_uuid_is_null (inode->gfid))
-            && (gf_uuid_compare (stbuf.ia_gfid, inode->gfid) != 0)) {
-                gf_msg_debug (frame->this->name, ESTALE,
-                              "gfid changed for %s", local->validate_loc.path);
-                rsp.op_ret = -1;
-                op_errno = ESTALE;
-                goto out;
-        }
+        op_errno = ESTALE;
+        goto out;
+    }
 
-        rsp.op_ret = 0;
+    rsp.op_ret = 0;
 
 out:
-        rsp.op_errno = op_errno;
-
-        /* We need to retry connecting to quotad on ENOTCONN error.
-         * Suppose if there are two volumes vol1 and vol2,
-         * and quota is enabled and limit is set on vol1.
-         * Now if IO is happening on vol1 and quota is enabled/disabled
-         * on vol2, quotad gets restarted and client will receive
-         * ENOTCONN in the IO path of vol1
-         */
-        if (rsp.op_ret == -1 && rsp.op_errno == ENOTCONN) {
-                if (local->quotad_conn_retry >= 12) {
-                        priv->quotad_conn_status = 1;
-                        gf_log (this->name, GF_LOG_WARNING, "failed to connect "
-                                "to quotad after retry count %d)",
-                                local->quotad_conn_retry);
-                } else {
-                        local->quotad_conn_retry++;
-                }
-
-                if (priv->quotad_conn_status == 0) {
-                        /* retry connecting after 5secs for 12 retries
-                         * (upto 60sec).
-                         */
-                        gf_log (this->name, GF_LOG_DEBUG, "retry connecting to "
-                                "quotad (retry count %d)",
-                                local->quotad_conn_retry);
-
-                        retry_delay.tv_sec = 5;
-                        retry_delay.tv_nsec = 0;
-                        timer = gf_timer_call_after (this->ctx, retry_delay,
-                                                     _quota_enforcer_lookup,
-                                                     (void *) frame);
-                        if (timer == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING, "failed to "
-                                        "set quota_enforcer_lookup with timer");
-                        } else {
-                                goto clean;
-                        }
-                }
+    rsp.op_errno = op_errno;
+
+    /* We need to retry connecting to quotad on ENOTCONN error.
+     * Suppose if there are two volumes vol1 and vol2,
+     * and quota is enabled and limit is set on vol1.
+     * Now if IO is happening on vol1 and quota is enabled/disabled
+     * on vol2, quotad gets restarted and client will receive
+     * ENOTCONN in the IO path of vol1
+     */
+    if (rsp.op_ret == -1 && rsp.op_errno == ENOTCONN) {
+        if (local->quotad_conn_retry >= 12) {
+            priv->quotad_conn_status = 1;
+            gf_log(this->name, GF_LOG_WARNING,
+                   "failed to connect "
+                   "to quotad after retry count %d)",
+                   local->quotad_conn_retry);
         } else {
-                priv->quotad_conn_status = 0;
+            local->quotad_conn_retry++;
         }
 
-        if (rsp.op_ret == -1) {
-                /* any error other than ENOENT */
-                if (rsp.op_errno != ENOENT)
-                        gf_msg (this->name, GF_LOG_WARNING, rsp.op_errno,
-                                Q_MSG_LOOKUP_FAILED,
-                                "Getting cluster-wide size of directory failed "
-                                "(path: %s gfid:%s)", local->validate_loc.path,
-                                loc_gfid_utoa (&local->validate_loc));
-                else
-                        gf_msg_trace (this->name, ENOENT,
-                                      "not found on remote node");
-
-        } else if (local->quotad_conn_retry) {
-                gf_log (this->name, GF_LOG_DEBUG, "connected to quotad after "
-                        "retry count %d", local->quotad_conn_retry);
+        if (priv->quotad_conn_status == 0) {
+            /* retry connecting after 5secs for 12 retries
+             * (up to 60sec).
+             */
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "retry connecting to "
+                   "quotad (retry count %d)",
+                   local->quotad_conn_retry);
+
+            retry_delay.tv_sec = 5;
+            retry_delay.tv_nsec = 0;
+            timer = gf_timer_call_after(this->ctx, retry_delay,
+                                        _quota_enforcer_lookup, (void *)frame);
+            if (timer == NULL) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "failed to "
+                       "set quota_enforcer_lookup with timer");
+            } else {
+                goto clean;
+            }
         }
+    } else {
+        priv->quotad_conn_status = 0;
+    }
+
+    if (rsp.op_ret == -1) {
+        /* any error other than ENOENT */
+        if (rsp.op_errno != ENOENT)
+            gf_msg(
+                this->name, GF_LOG_WARNING, rsp.op_errno, Q_MSG_LOOKUP_FAILED,
+                "Getting cluster-wide size of directory failed "
+                "(path: %s gfid:%s)",
+                local->validate_loc.path, loc_gfid_utoa(&local->validate_loc));
+        else
+            gf_msg_trace(this->name, ENOENT, "not found on remote node");
 
-        local->validate_cbk (frame, NULL, this, rsp.op_ret, rsp.op_errno, inode,
-                             &stbuf, xdata, &postparent);
+    } else if (local->quotad_conn_retry) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "connected to quotad after "
+               "retry count %d",
+               local->quotad_conn_retry);
+    }
+
+    local->validate_cbk(frame, NULL, this, rsp.op_ret, rsp.op_errno, inode,
+                        &stbuf, xdata, &postparent);
 
 clean:
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 void
-_quota_enforcer_lookup (void *data)
+_quota_enforcer_lookup(void *data)
 {
-        quota_local_t          *local      = NULL;
-        gfs3_lookup_req         req        = {{0,},};
-        int                     ret        = 0;
-        int                     op_errno   = ESTALE;
-        quota_priv_t           *priv       = NULL;
-        call_frame_t           *frame      = NULL;
-        loc_t                  *loc        = NULL;
-        xlator_t               *this       = NULL;
-        char                   *dir_path       = NULL;
-
-        frame = data;
-        local = frame->local;
-        this  = local->this;
-        loc   = &local->validate_loc;
-
-        priv = this->private;
-
-        if (!(loc && loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (loc->inode->gfid))
-                memcpy (req.gfid, loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, loc->gfid, 16);
-
-        if (local->validate_xdata) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, local->validate_xdata,
-                                            (&req.xdata.xdata_val),
-                                            req.xdata.xdata_len,
-                                            op_errno, unwind);
-        }
-
-        if (loc->name)
-                req.bname = (char *)loc->name;
-        else
-                req.bname = "";
-
-        if (loc->path)
-                dir_path = (char *)loc->path;
-        else
-                dir_path = "";
-
-        ret = quota_enforcer_submit_request (&req, frame,
-                                             priv->quota_enforcer,
-                                             GF_AGGREGATOR_LOOKUP,
-                                             NULL, this,
-                                             quota_enforcer_lookup_cbk,
-                                             (xdrproc_t)xdr_gfs3_lookup_req);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-			Q_MSG_RPC_SUBMIT_FAILED, "Couldn't send the request to "
-                        "fetch cluster wide size of directory (path:%s gfid:%s)"
-                        , dir_path, req.gfid);
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return;
+    quota_local_t *local = NULL;
+    gfs3_lookup_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    quota_priv_t *priv = NULL;
+    call_frame_t *frame = NULL;
+    loc_t *loc = NULL;
+    xlator_t *this = NULL;
+    char *dir_path = NULL;
+
+    frame = data;
+    local = frame->local;
+    this = local->this;
+    loc = &local->validate_loc;
+
+    priv = this->private;
+
+    if (!(loc && loc->inode))
+        goto unwind;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req.gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req.gfid, loc->gfid, 16);
+
+    if (local->validate_xdata) {
+        GF_PROTOCOL_DICT_SERIALIZE(this, local->validate_xdata,
+                                   (&req.xdata.xdata_val), req.xdata.xdata_len,
+                                   op_errno, unwind);
+    }
+
+    if (loc->name)
+        req.bname = (char *)loc->name;
+    else
+        req.bname = "";
+
+    if (loc->path)
+        dir_path = (char *)loc->path;
+    else
+        dir_path = "";
+
+    ret = quota_enforcer_submit_request(
+        &req, frame, priv->quota_enforcer, GF_AGGREGATOR_LOOKUP, NULL, this,
+        quota_enforcer_lookup_cbk, (xdrproc_t)xdr_gfs3_lookup_req);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_RPC_SUBMIT_FAILED,
+               "Couldn't send the request to "
+               "fetch cluster wide size of directory (path:%s gfid:%s)",
+               dir_path, req.gfid);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return;
 
 unwind:
-        local->validate_cbk (frame, NULL, this, -1, op_errno, NULL, NULL, NULL,
-                             NULL);
+    local->validate_cbk(frame, NULL, this, -1, op_errno, NULL, NULL, NULL,
+                        NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return;
+    return;
 }
 
 int
-quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
-                       fop_lookup_cbk_t validate_cbk)
+quota_enforcer_lookup(call_frame_t *frame, xlator_t *this, dict_t *xdata,
+                      fop_lookup_cbk_t validate_cbk)
 {
-        quota_local_t          *local      = NULL;
+    quota_local_t *local = NULL;
 
-        if (!frame || !this)
-                goto unwind;
+    if (!frame || !this)
+        goto unwind;
 
-        local = frame->local;
-        local->this = this;
-        local->validate_cbk = validate_cbk;
-        local->validate_xdata = dict_ref (xdata);
+    local = frame->local;
+    local->this = this;
+    local->validate_cbk = validate_cbk;
+    local->validate_xdata = dict_ref(xdata);
 
-        _quota_enforcer_lookup (frame);
+    _quota_enforcer_lookup(frame);
 
-        return 0;
+    return 0;
 
 unwind:
-        validate_cbk (frame, NULL, this, -1, ESTALE, NULL, NULL, NULL, NULL);
+    validate_cbk(frame, NULL, this, -1, ESTALE, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-quota_enforcer_notify (struct rpc_clnt *rpc, void *mydata,
-                       rpc_clnt_event_t event, void *data)
+quota_enforcer_notify(struct rpc_clnt *rpc, void *mydata,
+                      rpc_clnt_event_t event, void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
-
-        this = mydata;
-
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                gf_msg_trace (this->name, 0, "got RPC_CLNT_CONNECT");
-                break;
+    xlator_t *this = NULL;
+    int ret = 0;
+    quota_priv_t *priv = NULL;
+
+    this = mydata;
+    priv = this->private;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            pthread_mutex_lock(&priv->conn_mutex);
+            {
+                priv->conn_status = _gf_true;
+            }
+            pthread_mutex_unlock(&priv->conn_mutex);
+            gf_msg_trace(this->name, 0, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_msg_trace (this->name, 0, "got RPC_CLNT_DISCONNECT");
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            pthread_mutex_lock(&priv->conn_mutex);
+            {
+                priv->conn_status = _gf_false;
+                pthread_cond_signal(&priv->conn_cond);
+            }
+            pthread_mutex_unlock(&priv->conn_mutex);
+            gf_msg_trace(this->name, 0, "got RPC_CLNT_DISCONNECT");
+            break;
         }
 
         default:
-                gf_msg_trace (this->name, 0,
-                              "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-quota_enforcer_blocking_connect (rpc_clnt_t *rpc)
+quota_enforcer_blocking_connect(rpc_clnt_t *rpc)
 {
-        dict_t *options = NULL;
-        int     ret     = -1;
+    dict_t *options = NULL;
+    int ret = -1;
 
-        options = dict_new ();
-        if (options == NULL)
-                goto out;
+    options = dict_new();
+    if (options == NULL)
+        goto out;
 
-        ret = dict_set_str (options, "non-blocking-io", "no");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "non-blocking-io", "no");
+    if (ret)
+        goto out;
 
-        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+    rpc->conn.trans->reconfigure(rpc->conn.trans, options);
 
-        rpc_clnt_start (rpc);
+    rpc_clnt_start(rpc);
 
-        ret = dict_set_str (options, "non-blocking-io", "yes");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "non-blocking-io", "yes");
+    if (ret)
+        goto out;
 
-        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+    rpc->conn.trans->reconfigure(rpc->conn.trans, options);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (options)
-                dict_unref (options);
+    if (options)
+        dict_unref(options);
 
-        return ret;
+    return ret;
 }
 
-//Returns a started rpc_clnt. Creates a new rpc_clnt if quota_priv doesn't have
-//one already
+// Returns a started rpc_clnt. Creates a new rpc_clnt if quota_priv doesn't have
+// one already
 struct rpc_clnt *
-quota_enforcer_init (xlator_t *this, dict_t *options)
+quota_enforcer_init(xlator_t *this, dict_t *options)
 {
-        struct rpc_clnt *rpc  = NULL;
-        quota_priv_t    *priv = NULL;
-        int              ret  = -1;
+    struct rpc_clnt *rpc = NULL;
+    quota_priv_t *priv = NULL;
+    int ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        LOCK (&priv->lock);
-        {
-                if (priv->rpc_clnt) {
-                        ret = 0;
-                        rpc = priv->rpc_clnt;
-                }
+    LOCK(&priv->lock);
+    {
+        if (priv->rpc_clnt) {
+            ret = 0;
+            rpc = priv->rpc_clnt;
         }
-        UNLOCK (&priv->lock);
+    }
+    UNLOCK(&priv->lock);
 
-        if (rpc)
-                goto out;
+    if (rpc)
+        goto out;
 
-        priv->quota_enforcer = &quota_enforcer_clnt;
+    priv->quota_enforcer = &quota_enforcer_clnt;
 
-        ret = dict_set_str (options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "transport.address-family", "unix");
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "transport-type", "socket");
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "transport.socket.connect-path",
-                            "/var/run/gluster/quotad.socket");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "transport.socket.connect-path",
+                                   "/var/run/gluster/quotad.socket");
+    if (ret)
+        goto out;
 
-        rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!rpc) {
-                ret = -1;
-                goto out;
-        }
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = rpc_clnt_register_notify (rpc, quota_enforcer_notify, this);
-        if (ret) {
-                gf_msg ("quota", GF_LOG_ERROR, 0,
-                        Q_MSG_RPCCLNT_REGISTER_NOTIFY_FAILED,
-                        "failed to register notify");
-                goto out;
-        }
+    ret = rpc_clnt_register_notify(rpc, quota_enforcer_notify, this);
+    if (ret) {
+        gf_msg("quota", GF_LOG_ERROR, 0, Q_MSG_RPCCLNT_REGISTER_NOTIFY_FAILED,
+               "failed to register notify");
+        goto out;
+    }
 
-        ret = quota_enforcer_blocking_connect (rpc);
-        if (ret)
-                goto out;
+    ret = quota_enforcer_blocking_connect(rpc);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
 
-        return rpc;
-        }
+    return rpc;
+}
 
 struct rpc_clnt_procedure quota_enforcer_actors[GF_AGGREGATOR_MAXVALUE] = {
-        [GF_AGGREGATOR_NULL]     = {"NULL", NULL},
-        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", NULL},
+    [GF_AGGREGATOR_NULL] = {"NULL", NULL},
+    [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", NULL},
 };
 
 struct rpc_clnt_program quota_enforcer_clnt = {
-        .progname  = "Quota enforcer",
-        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
-        .progver   = GLUSTER_AGGREGATOR_VERSION,
-        .numproc   = GF_AGGREGATOR_MAXVALUE,
-        .proctable = quota_enforcer_actors,
+    .progname = "Quota enforcer",
+    .prognum = GLUSTER_AGGREGATOR_PROGRAM,
+    .progver = GLUSTER_AGGREGATOR_VERSION,
+    .numproc = GF_AGGREGATOR_MAXVALUE,
+    .proctable = quota_enforcer_actors,
 };
diff --git a/xlators/features/quota/src/quota-mem-types.h b/xlators/features/quota/src/quota-mem-types.h
index 97d9165681f..782a7de96bb 100644
--- a/xlators/features/quota/src/quota-mem-types.h
+++ b/xlators/features/quota/src/quota-mem-types.h
@@ -10,21 +10,21 @@
 #ifndef __QUOTA_MEM_TYPES_H__
 #define __QUOTA_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_quota_mem_types_ {
-        gf_quota_mt_quota_priv_t = gf_common_mt_end + 1,
-        gf_quota_mt_quota_inode_ctx_t,
-        gf_quota_mt_loc_t,
-        gf_quota_mt_char,
-        gf_quota_mt_int64_t,
-        gf_quota_mt_int32_t,
-        gf_quota_mt_limits_t,
-        gf_quota_mt_quota_dentry_t,
-        gf_quota_mt_quota_limits_level_t,
-        gf_quota_mt_qd_vols_conf_t,
-        gf_quota_mt_aggregator_state_t,
-        gf_quota_mt_end
+    /* Those are used by QUOTA_ALLOC_OR_GOTO macro */
+    gf_quota_mt_quota_priv_t = gf_common_mt_end + 1,
+    gf_quota_mt_quota_inode_ctx_t,
+    gf_quota_mt_loc_t,
+    gf_quota_mt_char,
+    gf_quota_mt_int64_t,
+    gf_quota_mt_int32_t,
+    gf_quota_mt_limits_t,
+    gf_quota_mt_quota_dentry_t,
+    gf_quota_mt_quota_limits_level_t,
+    gf_quota_mt_qd_vols_conf_t,
+    gf_quota_mt_aggregator_state_t,
+    gf_quota_mt_end
 };
 #endif
-
diff --git a/xlators/features/quota/src/quota-messages.h b/xlators/features/quota/src/quota-messages.h
index b01fe98e908..d434ed75e76 100644
--- a/xlators/features/quota/src/quota-messages.h
+++ b/xlators/features/quota/src/quota-messages.h
@@ -11,237 +11,29 @@
 #ifndef _QUOTA_MESSAGES_H_
 #define _QUOTA_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <glusterfs/glfs-message-id.h>
 
-#include "glfs-message-id.h"
-
-/*! \file quota-messages.h
- *  \brief Quota log-message IDs and their descriptions
+/* To add new message IDs, append new identifiers at the end of the list.
  *
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check across the code if the message ID macro in question is reused
- *    anywhere. If reused then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_QUOTA_BASE          GLFS_MSGID_COMP_QUOTA
-#define GLFS_NUM_MESSAGES        23
-#define GLFS_MSGID_END     (GLFS_QUOTA_BASE + GLFS_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x GLFS_QUOTA_BASE, "Invalid: Start of messages"
-/*------------*/
-
-/*!
- * @messageid 120001
- * @diagnosis Quota enforcement has failed.
- * @recommendedaction None
- */
-#define Q_MSG_ENFORCEMENT_FAILED        (GLFS_QUOTA_BASE + 1)
-
-
-/*!
-* @messageid 120002
-* @diagnosis system is out of memory
-* @recommendedaction None
-*/
-#define Q_MSG_ENOMEM    (GLFS_QUOTA_BASE + 2)
-
-/*!
- * @messageid 120003
- * @diagnosis Parent inode is not present in the  inode table due to the
- * inode table limits or the brick was restarted recently.
- * @recommendedaction If it is a brick restart then perform a crawl on the
- * file system or the specific directory in which the problem is observed.
- * If inode table limit has been reached,please increase the limit of
- * network.inode-lru-limit to a higher value(can be set through CLI).
- */
-#define Q_MSG_PARENT_NULL       (GLFS_QUOTA_BASE + 3)
-
-/*!
- * @messageid 120004
- * @diagnosis This is to inform the admin that the user has crossed the soft limit
- * of the quota configured on the directory and expected to cross the hard limit soon.
- * @recommendedaction You may reconfigure your quota limits.
- */
-#define Q_MSG_CROSSED_SOFT_LIMIT        (GLFS_QUOTA_BASE + 4)
-
-/*!
- * @messageid 120005
- * @diagnosis Quota translator failed to connect to quotad. This could be
- * due to one or more of the following reasons, (1) Quotad is not running.
- * (2) Brick process has run out of memory.
- * @recommendedaction If quotad is not running, consider starting quotad.
- * else check system memory consumption.
- */
-#define Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED            (GLFS_QUOTA_BASE + 5)
-
-/*!
- * @messageid 120006
- * @diagnosis Getting cluster-wide size failed
- * @recommendedaction Restart quotad. Kill quotad by searching
- * "ps ax | grep quotad" and use volume start force to restart it.
- */
-
-#define Q_MSG_REMOTE_OPERATION_FAILED   (GLFS_QUOTA_BASE + 6)
-
-/*!
- * @messageid 120007
- * @diagnosis Updation of global quota size failed. This may be due to quotad
- * is down or lost connection with quotad.
- * @recommendedaction Please restart quotad.
- */
-
-#define Q_MSG_FAILED_TO_SEND_FOP        (GLFS_QUOTA_BASE + 7)
-
-/*!
- * @messageid 120008
- * @diagnosis
- * @recommendedaction Check volfile for correctness
- */
-
-#define Q_MSG_INVALID_VOLFILE        (GLFS_QUOTA_BASE + 8)
-
-/*!
- * @messageid 120009
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_INODE_PARENT_NOT_FOUND        (GLFS_QUOTA_BASE + 9)
-
-/*!
- * @messageid 120010
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_XDR_DECODE_ERROR        (GLFS_QUOTA_BASE + 10)
-
-/*!
- * @messageid 120011
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_DICT_UNSERIALIZE_FAIL        (GLFS_QUOTA_BASE + 11)
-
-/*!
- * @messageid 120012
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_DICT_SERIALIZE_FAIL        (GLFS_QUOTA_BASE + 12)
-
-/*!
- * @messageid 120013
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_RPCSVC_INIT_FAILED        (GLFS_QUOTA_BASE + 13)
-
-/*!
- * @messageid 120014
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_RPCSVC_LISTENER_CREATION_FAILED        (GLFS_QUOTA_BASE + 14)
-
-/*!
- * @messageid 120015
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_RPCSVC_REGISTER_FAILED        (GLFS_QUOTA_BASE + 15)
-
-/*!
- * @messageid 120016
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_XDR_DECODING_FAILED        (GLFS_QUOTA_BASE + 16)
-/*!
- * @messageid 120017
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_RPCCLNT_REGISTER_NOTIFY_FAILED        (GLFS_QUOTA_BASE + 17)
-/*!
- * @messageid 120018
- * @diagnosis
- * @recommendedaction Umount and mount the corresponing volume
- */
-
-#define Q_MSG_ANCESTRY_BUILD_FAILED        (GLFS_QUOTA_BASE + 18)
-
-/*!
- * @messageid 120019
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_SIZE_KEY_MISSING        (GLFS_QUOTA_BASE + 19)
-
-/*!
- * @messageid 120020
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_INODE_CTX_GET_FAILED     (GLFS_QUOTA_BASE + 20)
-
-/*!
- * @messageid 120021
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_INODE_CTX_SET_FAILED     (GLFS_QUOTA_BASE + 21)
-
-/*!
- * @messageid 120022
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_LOOKUP_FAILED     (GLFS_QUOTA_BASE + 22)
-
-/*!
- * @messageid 120023
- * @diagnosis
- * @recommendedaction
- */
-
-#define Q_MSG_RPC_SUBMIT_FAILED     (GLFS_QUOTA_BASE + 23)
-
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(QUOTA, Q_MSG_ENFORCEMENT_FAILED, Q_MSG_ENOMEM, Q_MSG_PARENT_NULL,
+           Q_MSG_CROSSED_SOFT_LIMIT, Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
+           Q_MSG_REMOTE_OPERATION_FAILED, Q_MSG_FAILED_TO_SEND_FOP,
+           Q_MSG_INVALID_VOLFILE, Q_MSG_INODE_PARENT_NOT_FOUND,
+           Q_MSG_XDR_DECODE_ERROR, Q_MSG_DICT_UNSERIALIZE_FAIL,
+           Q_MSG_DICT_SERIALIZE_FAIL, Q_MSG_RPCSVC_INIT_FAILED,
+           Q_MSG_RPCSVC_LISTENER_CREATION_FAILED, Q_MSG_RPCSVC_REGISTER_FAILED,
+           Q_MSG_XDR_DECODING_FAILED, Q_MSG_RPCCLNT_REGISTER_NOTIFY_FAILED,
+           Q_MSG_ANCESTRY_BUILD_FAILED, Q_MSG_SIZE_KEY_MISSING,
+           Q_MSG_INODE_CTX_GET_FAILED, Q_MSG_INODE_CTX_SET_FAILED,
+           Q_MSG_LOOKUP_FAILED, Q_MSG_RPC_SUBMIT_FAILED,
+           Q_MSG_ENFORCEMENT_SKIPPED, Q_MSG_INTERNAL_FOP_KEY_MISSING);
 
 #endif /* !_QUOTA_MESSAGES_H_ */
-
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index 22d1bb9f30f..18df9ae6d19 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -7,4744 +7,4636 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include <fnmatch.h>
 
 #include "quota.h"
-#include "common-utils.h"
-#include "defaults.h"
-#include "statedump.h"
-#include "quota-common-utils.h"
+#include <glusterfs/statedump.h>
 #include "quota-messages.h"
+#include <glusterfs/events.h>
 
 struct volume_options options[];
 
 static int32_t
-__quota_init_inode_ctx (inode_t *inode, xlator_t *this,
-                        quota_inode_ctx_t **context)
+__quota_init_inode_ctx(inode_t *inode, xlator_t *this,
+                       quota_inode_ctx_t **context)
 {
-        int32_t            ret  = -1;
-        quota_inode_ctx_t *ctx  = NULL;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (inode == NULL) {
-                goto out;
-        }
+    if (inode == NULL) {
+        goto out;
+    }
 
-        QUOTA_ALLOC_OR_GOTO (ctx, quota_inode_ctx_t, out);
+    QUOTA_ALLOC_OR_GOTO(ctx, quota_inode_ctx_t, out);
 
-        LOCK_INIT(&ctx->lock);
+    LOCK_INIT(&ctx->lock);
 
-        if (context != NULL) {
-                *context = ctx;
-        }
+    if (context != NULL) {
+        *context = ctx;
+    }
 
-        INIT_LIST_HEAD (&ctx->parents);
+    INIT_LIST_HEAD(&ctx->parents);
 
-        ret = __inode_ctx_put (inode, this, (uint64_t )(long)ctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_INODE_CTX_SET_FAILED, "cannot set quota context "
-                        "in inode (gfid:%s)", uuid_utoa (inode->gfid));
-                GF_FREE (ctx);
-        }
+    ret = __inode_ctx_put(inode, this, (uint64_t)(long)ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_SET_FAILED,
+               "cannot set quota context "
+               "in inode (gfid:%s)",
+               uuid_utoa(inode->gfid));
+        GF_FREE(ctx);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static int32_t
-quota_inode_ctx_get (inode_t *inode, xlator_t *this,
-                     quota_inode_ctx_t **ctx, char create_if_absent)
+quota_inode_ctx_get(inode_t *inode, xlator_t *this, quota_inode_ctx_t **ctx,
+                    char create_if_absent)
 {
-        int32_t  ret = 0;
-        uint64_t ctx_int;
+    int32_t ret = 0;
+    uint64_t ctx_int;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_int);
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
 
-                if ((ret == 0) && (ctx != NULL)) {
-                        *ctx = (quota_inode_ctx_t *) (unsigned long)ctx_int;
-                } else if (create_if_absent) {
-                        ret = __quota_init_inode_ctx (inode, this, ctx);
-                }
+        if ((ret == 0) && (ctx != NULL)) {
+            *ctx = (quota_inode_ctx_t *)(unsigned long)ctx_int;
+        } else if (create_if_absent) {
+            ret = __quota_init_inode_ctx(inode, this, ctx);
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-quota_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+quota_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
-        int ret = -1;
+    int ret = -1;
 
-        if (!loc || (inode == NULL))
-                return ret;
+    if (!loc || (inode == NULL))
+        return ret;
 
-        if (inode) {
-                loc->inode = inode_ref (inode);
-                gf_uuid_copy (loc->gfid, inode->gfid);
-        }
+    if (inode) {
+        loc->inode = inode_ref(inode);
+        gf_uuid_copy(loc->gfid, inode->gfid);
+    }
 
-        if (parent) {
-                loc->parent = inode_ref (parent);
-        }
+    if (parent) {
+        loc->parent = inode_ref(parent);
+    }
 
-        if (path != NULL) {
-                loc->path = gf_strdup (path);
+    if (path != NULL) {
+        loc->path = gf_strdup(path);
 
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name) {
-                        loc->name++;
-                }
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name) {
+            loc->name++;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
-        return ret;
+    return ret;
 }
 
-
 int
-quota_inode_loc_fill (inode_t *inode, loc_t *loc)
+quota_inode_loc_fill(inode_t *inode, loc_t *loc)
 {
-        char            *resolvedpath = NULL;
-        inode_t         *parent       = NULL;
-        int              ret          = -1;
-        xlator_t        *this         = NULL;
+    char *resolvedpath = NULL;
+    inode_t *parent = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        if ((!inode) || (!loc)) {
-                return ret;
-        }
+    if ((!inode) || (!loc)) {
+        return ret;
+    }
 
-        this = THIS;
+    this = THIS;
 
-        if ((inode) && __is_root_gfid (inode->gfid)) {
-                loc->parent = NULL;
-                goto ignore_parent;
-        }
+    if ((inode) && __is_root_gfid(inode->gfid)) {
+        loc->parent = NULL;
+        goto ignore_parent;
+    }
 
-        parent = inode_parent (inode, 0, NULL);
-        if (!parent) {
-                gf_msg_debug (this->name, 0, "cannot find parent for "
-                              "inode (gfid:%s)", uuid_utoa (inode->gfid));
-        }
+    parent = inode_parent(inode, 0, NULL);
+    if (!parent) {
+        gf_msg_debug(this->name, 0,
+                     "cannot find parent for "
+                     "inode (gfid:%s)",
+                     uuid_utoa(inode->gfid));
+    }
 
 ignore_parent:
-        ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0) {
-                gf_msg_debug (this->name, 0, "cannot construct path for "
-                              "inode (gfid:%s)",  uuid_utoa (inode->gfid));
-        }
-
-        ret = quota_loc_fill (loc, inode, parent, resolvedpath);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
-                        "cannot fill loc");
-                goto err;
-        }
+    ret = inode_path(inode, NULL, &resolvedpath);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "cannot construct path for "
+                     "inode (gfid:%s)",
+                     uuid_utoa(inode->gfid));
+    }
+
+    ret = quota_loc_fill(loc, inode, parent, resolvedpath);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "cannot fill loc");
+        goto err;
+    }
 
 err:
-        if (parent) {
-                inode_unref (parent);
-        }
+    if (parent) {
+        inode_unref(parent);
+    }
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-quota_local_cleanup (quota_local_t *local)
+quota_local_cleanup(quota_local_t *local)
 {
-        if (local == NULL) {
-                goto out;
-        }
+    if (local == NULL) {
+        goto out;
+    }
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
-        loc_wipe (&local->oldloc);
-        loc_wipe (&local->validate_loc);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
+    loc_wipe(&local->oldloc);
+    loc_wipe(&local->validate_loc);
 
-        inode_unref (local->inode);
+    inode_unref(local->inode);
 
-        if (local->xdata)
-                dict_unref (local->xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-        if (local->validate_xdata)
-                dict_unref (local->validate_xdata);
+    if (local->validate_xdata)
+        dict_unref(local->validate_xdata);
 
-        if (local->stub)
-                call_stub_destroy (local->stub);
+    if (local->stub)
+        call_stub_destroy(local->stub);
 
-        LOCK_DESTROY (&local->lock);
+    LOCK_DESTROY(&local->lock);
 
-        mem_put (local);
+    mem_put(local);
 out:
-        return 0;
+    return 0;
 }
 
-
 static quota_local_t *
-quota_local_new ()
+quota_local_new()
 {
-        quota_local_t *local = NULL;
-        local = mem_get0 (THIS->local_pool);
-        if (local == NULL)
-                goto out;
+    quota_local_t *local = NULL;
+    local = mem_get0(THIS->local_pool);
+    if (local == NULL)
+        goto out;
 
-        LOCK_INIT (&local->lock);
-        local->space_available = -1;
+    LOCK_INIT(&local->lock);
+    local->space_available = -1;
 
 out:
-        return local;
+    return local;
 }
 
-
 quota_dentry_t *
-__quota_dentry_new (quota_inode_ctx_t *ctx, char *name, uuid_t par)
+__quota_dentry_new(quota_inode_ctx_t *ctx, char *name, uuid_t par)
 {
-        quota_dentry_t    *dentry = NULL;
-        GF_UNUSED int32_t  ret    = 0;
+    quota_dentry_t *dentry = NULL;
+    GF_UNUSED int32_t ret = 0;
 
-        QUOTA_ALLOC_OR_GOTO (dentry, quota_dentry_t, err);
+    QUOTA_ALLOC_OR_GOTO(dentry, quota_dentry_t, err);
 
-        INIT_LIST_HEAD (&dentry->next);
+    INIT_LIST_HEAD(&dentry->next);
 
-        dentry->name = gf_strdup (name);
-        if (dentry->name == NULL) {
-                GF_FREE (dentry);
-                dentry = NULL;
-                goto err;
-        }
+    dentry->name = gf_strdup(name);
+    if (dentry->name == NULL) {
+        GF_FREE(dentry);
+        dentry = NULL;
+        goto err;
+    }
 
-        gf_uuid_copy (dentry->par, par);
+    gf_uuid_copy(dentry->par, par);
 
-        if (ctx != NULL)
-                list_add_tail (&dentry->next, &ctx->parents);
+    if (ctx != NULL)
+        list_add_tail(&dentry->next, &ctx->parents);
 
 err:
-        return dentry;
+    return dentry;
 }
 
-
 void
-__quota_dentry_free (quota_dentry_t *dentry)
+__quota_dentry_free(quota_dentry_t *dentry)
 {
-        if (dentry == NULL) {
-                goto out;
-        }
+    if (dentry == NULL) {
+        goto out;
+    }
 
-        list_del_init (&dentry->next);
+    list_del_init(&dentry->next);
 
-        GF_FREE (dentry->name);
-        GF_FREE (dentry);
+    GF_FREE(dentry->name);
+    GF_FREE(dentry);
 out:
-        return;
+    return;
 }
 
 void
-__quota_dentry_del (quota_inode_ctx_t *ctx, const char *name, uuid_t par)
+__quota_dentry_del(quota_inode_ctx_t *ctx, const char *name, uuid_t par)
 {
-        quota_dentry_t    *dentry = NULL;
-        quota_dentry_t    *tmp    = NULL;
-
-        list_for_each_entry_safe (dentry, tmp, &ctx->parents, next) {
-                if ((strcmp (dentry->name, name) == 0) &&
-                    (gf_uuid_compare (dentry->par, par) == 0)) {
-                        __quota_dentry_free (dentry);
-                        break;
-                }
+    quota_dentry_t *dentry = NULL;
+    quota_dentry_t *tmp = NULL;
+
+    list_for_each_entry_safe(dentry, tmp, &ctx->parents, next)
+    {
+        if ((strcmp(dentry->name, name) == 0) &&
+            (gf_uuid_compare(dentry->par, par) == 0)) {
+            __quota_dentry_free(dentry);
+            break;
         }
+    }
 }
 
 void
-quota_dentry_del (quota_inode_ctx_t *ctx, const char *name, uuid_t par)
+quota_dentry_del(quota_inode_ctx_t *ctx, const char *name, uuid_t par)
 {
-        LOCK (&ctx->lock);
-        {
-                __quota_dentry_del (ctx, name, par);
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        __quota_dentry_del(ctx, name, par);
+    }
+    UNLOCK(&ctx->lock);
 }
 
-static inode_t*
-__quota_inode_parent (inode_t *inode, uuid_t pargfid, const char *name)
+static inode_t *
+__quota_inode_parent(inode_t *inode, uuid_t pargfid, const char *name)
 {
-        inode_t *parent            = NULL;
+    inode_t *parent = NULL;
 
-        parent = inode_parent (inode, pargfid, name);
-        inode_unref (inode);
-        return parent;
+    parent = inode_parent(inode, pargfid, name);
+    inode_unref(inode);
+    return parent;
 }
 
-static inode_t*
-quota_inode_parent (inode_t *inode, uuid_t pargfid, const char *name)
+static inode_t *
+quota_inode_parent(inode_t *inode, uuid_t pargfid, const char *name)
 {
-        inode_t *parent            = NULL;
+    inode_t *parent = NULL;
 
-        parent = __quota_inode_parent (inode, pargfid, name);
-        if (!parent)
-                gf_msg_callingfn (THIS->name, GF_LOG_ERROR, 0,
-                                  Q_MSG_PARENT_NULL,
-                                  "Failed to find "
-                                  "ancestor for inode (%s)",
-                                  uuid_utoa(inode->gfid));
+    parent = __quota_inode_parent(inode, pargfid, name);
+    if (!parent)
+        gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0, Q_MSG_PARENT_NULL,
+                         "Failed to find "
+                         "ancestor for inode (%s)",
+                         uuid_utoa(inode->gfid));
 
-        return parent;
+    return parent;
 }
 
 int32_t
-quota_inode_depth (inode_t *inode)
+quota_inode_depth(inode_t *inode)
 {
-        int      depth             = 0;
-        inode_t *cur_inode         = NULL;
-
-        cur_inode = inode_ref (inode);
-        while (cur_inode && !__is_root_gfid (cur_inode->gfid)) {
-                depth++;
-                cur_inode = quota_inode_parent (cur_inode, 0 , NULL);
-                if (!cur_inode)
-                        depth = -1;
-        }
+    int depth = 0;
+    inode_t *cur_inode = NULL;
+
+    cur_inode = inode_ref(inode);
+    while (cur_inode && !__is_root_gfid(cur_inode->gfid)) {
+        depth++;
+        cur_inode = quota_inode_parent(cur_inode, 0, NULL);
+        if (!cur_inode)
+            depth = -1;
+    }
 
-        if (cur_inode)
-                inode_unref (cur_inode);
+    if (cur_inode)
+        inode_unref(cur_inode);
 
-        return depth;
+    return depth;
 }
 
-int32_t quota_find_common_ancestor (inode_t *inode1, inode_t *inode2,
-                                    uuid_t *common_ancestor)
+int32_t
+quota_find_common_ancestor(inode_t *inode1, inode_t *inode2,
+                           uuid_t *common_ancestor)
 {
-        int32_t         depth1         = 0;
-        int32_t         depth2         = 0;
-        int32_t         ret            = -1;
-        inode_t        *cur_inode1     = NULL;
-        inode_t        *cur_inode2     = NULL;
-
-        depth1 = quota_inode_depth (inode1);
-        if (depth1 < 0)
-                goto out;
-
-        depth2 = quota_inode_depth (inode2);
-        if (depth2 < 0)
-                goto out;
-
-        cur_inode1 = inode_ref (inode1);
-        cur_inode2 = inode_ref (inode2);
-
-        while (cur_inode1 && depth1 > depth2) {
-                cur_inode1 = quota_inode_parent (cur_inode1, 0 , NULL);
-                depth1--;
-        }
-
-        while (cur_inode2 && depth2 > depth1) {
-                cur_inode2 = quota_inode_parent (cur_inode2, 0 , NULL);
-                depth2--;
-        }
-
-        while (depth1 && cur_inode1 && cur_inode2 && cur_inode1 != cur_inode2) {
-                cur_inode1 = quota_inode_parent (cur_inode1, 0 , NULL);
-                cur_inode2 = quota_inode_parent (cur_inode2, 0 , NULL);
-                depth1--;
-        }
-
-        if (cur_inode1 && cur_inode2) {
-                gf_uuid_copy (*common_ancestor, cur_inode1->gfid);
-                ret = 0;
-        }
+    int32_t depth1 = 0;
+    int32_t depth2 = 0;
+    int32_t ret = -1;
+    inode_t *cur_inode1 = NULL;
+    inode_t *cur_inode2 = NULL;
+
+    depth1 = quota_inode_depth(inode1);
+    if (depth1 < 0)
+        goto out;
+
+    depth2 = quota_inode_depth(inode2);
+    if (depth2 < 0)
+        goto out;
+
+    cur_inode1 = inode_ref(inode1);
+    cur_inode2 = inode_ref(inode2);
+
+    while (cur_inode1 && depth1 > depth2) {
+        cur_inode1 = quota_inode_parent(cur_inode1, 0, NULL);
+        depth1--;
+    }
+
+    while (cur_inode2 && depth2 > depth1) {
+        cur_inode2 = quota_inode_parent(cur_inode2, 0, NULL);
+        depth2--;
+    }
+
+    while (depth1 && cur_inode1 && cur_inode2 && cur_inode1 != cur_inode2) {
+        cur_inode1 = quota_inode_parent(cur_inode1, 0, NULL);
+        cur_inode2 = quota_inode_parent(cur_inode2, 0, NULL);
+        depth1--;
+    }
+
+    if (cur_inode1 && cur_inode2) {
+        gf_uuid_copy(*common_ancestor, cur_inode1->gfid);
+        ret = 0;
+    }
 out:
-        if (cur_inode1)
-                inode_unref (cur_inode1);
+    if (cur_inode1)
+        inode_unref(cur_inode1);
 
-        if (cur_inode2)
-                inode_unref (cur_inode2);
+    if (cur_inode2)
+        inode_unref(cur_inode2);
 
-        return ret;
- }
+    return ret;
+}
 
 void
-check_ancestory_continue (struct list_head *parents, inode_t *inode,
-                          int32_t op_ret, int32_t op_errno, void *data)
+check_ancestory_continue(struct list_head *parents, inode_t *inode,
+                         int32_t op_ret, int32_t op_errno, void *data)
 {
-        call_frame_t   *frame        = NULL;
-        quota_local_t  *local        = NULL;
-        uint32_t        link_count   = 0;
-
-        frame = data;
-        local = frame->local;
-
-        if (parents && list_empty (parents)) {
-                gf_msg (THIS->name, GF_LOG_WARNING, EIO,
-                        Q_MSG_ANCESTRY_BUILD_FAILED,
-                        "Couldn't build ancestry for inode (gfid:%s). "
-                        "Without knowing ancestors till root, quota "
-                        "cannot be enforced. "
-                        "Hence, failing fop with EIO",
-                        uuid_utoa (inode->gfid));
-                op_errno = EIO;
-                op_ret = -1;
-        }
-
-        LOCK (&local->lock);
-        {
-                link_count = --local->link_count;
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
+    call_frame_t *frame = NULL;
+    quota_local_t *local = NULL;
+    uint32_t link_count = 0;
+
+    frame = data;
+    local = frame->local;
+
+    if (parents && list_empty(parents)) {
+        gf_msg(THIS->name, GF_LOG_WARNING, EIO, Q_MSG_ANCESTRY_BUILD_FAILED,
+               "Couldn't build ancestry for inode (gfid:%s). "
+               "Without knowing ancestors till root, quota "
+               "cannot be enforced. "
+               "Hence, failing fop with EIO",
+               uuid_utoa(inode->gfid));
+        op_errno = EIO;
+        op_ret = -1;
+    }
+
+    LOCK(&local->lock);
+    {
+        link_count = --local->link_count;
+        if (op_ret < 0) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
         }
-        UNLOCK (&local->lock);
+    }
+    UNLOCK(&local->lock);
 
-        if (link_count == 0)
-                local->fop_continue_cbk (frame);
+    if (link_count == 0)
+        local->fop_continue_cbk(frame);
 }
 
 void
-check_ancestory (call_frame_t *frame, inode_t *inode)
+check_ancestory(call_frame_t *frame, inode_t *inode)
 {
-        inode_t *cur_inode = NULL;
-        inode_t *parent    = NULL;
-
-        cur_inode = inode_ref (inode);
-        while (cur_inode && !__is_root_gfid (cur_inode->gfid)) {
-                parent = inode_parent (cur_inode, 0, NULL);
-                if (!parent) {
-                        quota_build_ancestry (cur_inode,
-                                              check_ancestory_continue, frame);
-                        inode_unref (cur_inode);
-                        return;
-                }
-                inode_unref (cur_inode);
-                cur_inode = parent;
-        }
+    inode_t *cur_inode = NULL;
+    inode_t *parent = NULL;
 
-        if (cur_inode) {
-                inode_unref (cur_inode);
-                check_ancestory_continue (NULL, NULL, 0, 0, frame);
-        } else {
-                check_ancestory_continue (NULL, NULL, -1, ESTALE, frame);
-        }
+    cur_inode = inode_ref(inode);
+    while (cur_inode && !__is_root_gfid(cur_inode->gfid)) {
+        parent = inode_parent(cur_inode, 0, NULL);
+        if (!parent) {
+            quota_build_ancestry(cur_inode, check_ancestory_continue, frame);
+            inode_unref(cur_inode);
+            return;
+        }
+        inode_unref(cur_inode);
+        cur_inode = parent;
+    }
+
+    if (cur_inode) {
+        inode_unref(cur_inode);
+        check_ancestory_continue(NULL, NULL, 0, 0, frame);
+    } else {
+        check_ancestory_continue(NULL, NULL, -1, ESTALE, frame);
+    }
 }
 
 void
-check_ancestory_2_cbk (struct list_head *parents, inode_t *inode,
-                       int32_t op_ret, int32_t op_errno, void *data)
+check_ancestory_2_cbk(struct list_head *parents, inode_t *inode, int32_t op_ret,
+                      int32_t op_errno, void *data)
 {
-        inode_t            *this_inode  = NULL;
-        quota_inode_ctx_t  *ctx         = NULL;
+    inode_t *this_inode = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        this_inode = data;
+    this_inode = data;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        if (parents == NULL || list_empty (parents)) {
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        Q_MSG_ENFORCEMENT_FAILED,
-                        "Couldn't build ancestry for inode (gfid:%s). "
-                        "Without knowing ancestors till root, quota "
-                        "cannot be enforced.",
-                        uuid_utoa (this_inode->gfid));
-                goto out;
-        }
+    if (parents == NULL || list_empty(parents)) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, Q_MSG_ENFORCEMENT_FAILED,
+               "Couldn't build ancestry for inode (gfid:%s). "
+               "Without knowing ancestors till root, quota "
+               "cannot be enforced.",
+               uuid_utoa(this_inode->gfid));
+        goto out;
+    }
 
-        quota_inode_ctx_get (this_inode, THIS, &ctx, 0);
-        if (ctx)
-                ctx->ancestry_built = _gf_true;
+    quota_inode_ctx_get(this_inode, THIS, &ctx, 0);
+    if (ctx)
+        ctx->ancestry_built = _gf_true;
 
 out:
-        inode_unref (this_inode);
+    inode_unref(this_inode);
 }
 
 void
-check_ancestory_2 (xlator_t *this, quota_local_t *local, inode_t *inode)
+check_ancestory_2(xlator_t *this, quota_local_t *local, inode_t *inode)
 {
-        inode_t            *cur_inode    = NULL;
-        inode_t            *parent       = NULL;
-        quota_inode_ctx_t  *ctx          = NULL;
-        char               *name         = NULL;
-        uuid_t              pgfid        = {0};
-
-        name = (char *) local->loc.name;
-        if (local->loc.parent) {
-                gf_uuid_copy (pgfid, local->loc.parent->gfid);
-                parent = local->loc.parent;
-        }
-
-        cur_inode = inode_ref (inode);
-        while (cur_inode && !__is_root_gfid (cur_inode->gfid)) {
-                quota_inode_ctx_get (cur_inode, this, &ctx, 0);
-                /* build ancestry is required only on the first lookup,
-                 * so stop crawling when the inode_ctx is set for an inode
-                 */
-                if (ctx && ctx->ancestry_built)
-                        goto setctx;
-
-                parent = inode_parent (cur_inode, pgfid, name);
-                if (!parent) {
-                        quota_build_ancestry (cur_inode, check_ancestory_2_cbk,
-                                              inode_ref (inode));
-                        goto out;
-                }
+    inode_t *cur_inode = NULL;
+    inode_t *parent = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    char *name = NULL;
+    uuid_t pgfid = {0};
+
+    name = (char *)local->loc.name;
+    if (local->loc.parent) {
+        gf_uuid_copy(pgfid, local->loc.parent->gfid);
+    }
+
+    cur_inode = inode_ref(inode);
+    while (cur_inode && !__is_root_gfid(cur_inode->gfid)) {
+        quota_inode_ctx_get(cur_inode, this, &ctx, 0);
+        /* build ancestry is required only on the first lookup,
+         * so stop crawling when the inode_ctx is set for an inode
+         */
+        if (ctx && ctx->ancestry_built)
+            goto setctx;
 
-                if (name != NULL) {
-                        name = NULL;
-                        gf_uuid_clear (pgfid);
-                }
+        parent = inode_parent(cur_inode, pgfid, name);
+        if (!parent) {
+            quota_build_ancestry(cur_inode, check_ancestory_2_cbk,
+                                 inode_ref(inode));
+            goto out;
+        }
 
-                inode_unref (cur_inode);
-                cur_inode = parent;
+        if (name != NULL) {
+            name = NULL;
+            gf_uuid_clear(pgfid);
         }
 
+        inode_unref(cur_inode);
+        cur_inode = parent;
+    }
+
 setctx:
-        if (cur_inode && cur_inode != inode) {
-                quota_inode_ctx_get (inode, this, &ctx, 0);
-                if (ctx)
-                        ctx->ancestry_built = _gf_true;
-        }
+    if (cur_inode && cur_inode != inode) {
+        quota_inode_ctx_get(inode, this, &ctx, 0);
+        if (ctx)
+            ctx->ancestry_built = _gf_true;
+    }
 out:
-        if (cur_inode)
-                inode_unref (cur_inode);
+    if (cur_inode)
+        inode_unref(cur_inode);
 }
 
 static void
-quota_link_count_decrement (call_frame_t *frame)
+quota_link_count_decrement(call_frame_t *frame)
 {
-        call_frame_t   *tmpframe   = NULL;
-        quota_local_t  *local      = NULL;
-        call_stub_t    *stub       = NULL;
-        int             link_count = -1;
-
-        local = frame->local;
-        if (local && local->par_frame) {
-                local = local->par_frame->local;
-                tmpframe = frame;
-        }
-
-        if (local == NULL)
-                goto out;
-
-        LOCK (&local->lock);
-        {
-                link_count = --local->link_count;
-                if (link_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
-        }
-        UNLOCK (&local->lock);
-
-        if (stub != NULL) {
-                call_resume (stub);
-        }
+    call_frame_t *tmpframe = NULL;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int link_count = -1;
+
+    local = frame->local;
+    if (local && local->par_frame) {
+        local = local->par_frame->local;
+        tmpframe = frame;
+    }
+
+    if (local == NULL)
+        goto out;
+
+    LOCK(&local->lock);
+    {
+        link_count = --local->link_count;
+        if (link_count == 0) {
+            stub = local->stub;
+            local->stub = NULL;
+        }
+    }
+    UNLOCK(&local->lock);
+
+    if (stub != NULL) {
+        call_resume(stub);
+    }
 
 out:
-        if (tmpframe) {
-                local = tmpframe->local;
-                tmpframe->local = NULL;
+    if (tmpframe) {
+        local = tmpframe->local;
+        tmpframe->local = NULL;
 
-                STACK_DESTROY (frame->root);
-                if (local)
-                        quota_local_cleanup (local);
-        }
+        STACK_DESTROY(frame->root);
+        if (local)
+            quota_local_cleanup(local);
+    }
 
-        return;
+    return;
 }
 
 static void
-quota_handle_validate_error (call_frame_t *frame, int32_t op_ret,
-                             int32_t op_errno)
+quota_handle_validate_error(call_frame_t *frame, int32_t op_ret,
+                            int32_t op_errno)
 {
-        quota_local_t  *local;
+    quota_local_t *local;
 
-        local = frame->local;
-        if (local && local->par_frame)
-                local = local->par_frame->local;
+    local = frame->local;
+    if (local && local->par_frame)
+        local = local->par_frame->local;
 
-        if (local == NULL)
-                goto out;
+    if (local == NULL)
+        goto out;
 
-        LOCK (&local->lock);
+    if (op_ret < 0) {
+        LOCK(&local->lock);
         {
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
         }
-        UNLOCK (&local->lock);
-
-        /* we abort checking limits on this path to root */
-        quota_link_count_decrement (frame);
+        UNLOCK(&local->lock);
+    }
+    /* we abort checking limits on this path to root */
+    quota_link_count_decrement(frame);
 out:
-        return;
+    return;
 }
 
 int32_t
-quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+quota_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        quota_local_t     *local      = NULL;
-        int32_t            ret        = 0;
-        quota_inode_ctx_t *ctx        = NULL;
-        int64_t           *object_size = 0;
-        uint64_t           value      = 0;
-        data_t            *data       = NULL;
-        quota_meta_t       size       = {0,};
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                goto unwind;
-        }
-
-        GF_ASSERT (local);
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, unwind, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, unwind, op_errno,
-                                        EINVAL);
-
-        ret = inode_ctx_get (local->validate_loc.inode, this, &value);
-
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-			Q_MSG_INODE_CTX_GET_FAILED, "quota context is"
-			" not present in  inode (gfid:%s)",
-                        uuid_utoa (local->validate_loc.inode->gfid));
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        ret = quota_dict_get_meta (xdata, QUOTA_SIZE_KEY, &size);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-			Q_MSG_SIZE_KEY_MISSING, "quota size key not present "
-                        "in dict");
-                op_errno = EINVAL;
-        }
-
-        local->just_validated = 1; /* so that we don't go into infinite
-                                    * loop of validation and checking
-                                    * limit when timeout is zero.
-                                    */
-        LOCK (&ctx->lock);
-        {
-                ctx->size = size.size;
-                ctx->file_count = size.file_count;
-                ctx->dir_count = size.dir_count;
-                gettimeofday (&ctx->tv, NULL);
-        }
-        UNLOCK (&ctx->lock);
-
-        quota_check_limit (frame, local->validate_loc.inode, this);
-        return 0;
+    quota_local_t *local = NULL;
+    int32_t ret = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
+    quota_meta_t size = {
+        0,
+    };
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        goto unwind;
+    }
+
+    GF_ASSERT(local);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("quota", this, unwind, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, xdata, unwind, op_errno, EINVAL);
+
+    ret = inode_ctx_get(local->validate_loc.inode, this, &value);
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context is"
+               " not present in  inode (gfid:%s)",
+               uuid_utoa(local->validate_loc.inode->gfid));
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    ret = quota_dict_get_meta(xdata, QUOTA_SIZE_KEY, SLEN(QUOTA_SIZE_KEY),
+                              &size);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_SIZE_KEY_MISSING,
+               "quota size key not present "
+               "in dict");
+        op_errno = EINVAL;
+    }
+
+    local->just_validated = 1; /* so that we don't go into infinite
+                                * loop of validation and checking
+                                * limit when timeout is zero.
+                                */
+    LOCK(&ctx->lock);
+    {
+        ctx->size = size.size;
+        ctx->validate_time = gf_time();
+        ctx->file_count = size.file_count;
+        ctx->dir_count = size.dir_count;
+    }
+    UNLOCK(&ctx->lock);
+
+    quota_check_limit(frame, local->validate_loc.inode, this);
+    return 0;
 
 unwind:
-        quota_handle_validate_error (frame, op_ret, op_errno);
-        return 0;
+    quota_handle_validate_error(frame, op_ret, op_errno);
+    return 0;
 }
 
-
-static uint64_t
-quota_time_elapsed (struct timeval *now, struct timeval *then)
+static inline gf_boolean_t
+quota_timeout(time_t t, uint32_t timeout)
 {
-        return (now->tv_sec - then->tv_sec);
-}
-
-
-int32_t
-quota_timeout (struct timeval *tv, int32_t timeout)
-{
-        struct timeval now       = {0,};
-        int32_t        timed_out = 0;
-
-        gettimeofday (&now, NULL);
-
-        if (quota_time_elapsed (&now, tv) >= timeout) {
-                timed_out = 1;
-        }
-
-        return timed_out;
+    return (gf_time() - t) >= timeout;
 }
 
 /* Return: 1 if new entry added
  *         0 no entry added
+ *        -1 on errors
  */
 static int32_t
-quota_add_parent (struct list_head *list, char *name, uuid_t pgfid)
+quota_add_parent(struct list_head *list, char *name, uuid_t pgfid)
 {
-        quota_dentry_t *entry = NULL;
-        gf_boolean_t    found = _gf_false;
+    quota_dentry_t *entry = NULL;
+    gf_boolean_t found = _gf_false;
+    int ret = 0;
 
-        if (list == NULL) {
+    if (!list_empty(list)) {
+        list_for_each_entry(entry, list, next)
+        {
+            if (gf_uuid_compare(pgfid, entry->par) == 0) {
+                found = _gf_true;
                 goto out;
+            }
         }
+    }
 
-        list_for_each_entry (entry, list, next) {
-                if (gf_uuid_compare (pgfid, entry->par) == 0) {
-                        found = _gf_true;
-                        goto out;
-                }
-        }
-
-        entry = __quota_dentry_new (NULL, name, pgfid);
-        if (entry)
-                list_add_tail (&entry->next, list);
+    entry = __quota_dentry_new(NULL, name, pgfid);
+    if (entry)
+        list_add_tail(&entry->next, list);
+    else
+        ret = -1;
 
 out:
-        if (found)
-                return 0;
-        else
-                return 1;
-
+    if (found)
+        return 0;
+    else if (ret == 0)
+        return 1;
+    else
+        return -1;
 }
 
 /* This function iterates the parent list in inode
  * context and add unique parent to the list
- * Returns number of dentry added to the list
+ * Returns number of dentry added to the list, or -1 on errors
  */
 static int32_t
-quota_add_parents_from_ctx (quota_inode_ctx_t *ctx, struct list_head *list)
+quota_add_parents_from_ctx(quota_inode_ctx_t *ctx, struct list_head *list)
 {
-        int                ret     = 0;
-        quota_dentry_t    *dentry  = NULL;
-        int32_t            count   = 0;
+    int ret = 0;
+    quota_dentry_t *dentry = NULL;
+    int32_t count = 0;
 
-        if (ctx == NULL || list == NULL)
-                goto out;
+    if (ctx == NULL || list == NULL)
+        goto out;
 
-        LOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry(dentry, &ctx->parents, next)
         {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        ret = quota_add_parent (list, dentry->name,
-                                                dentry->par);
-
-                        if (ret == 1)
-                                count++;
-                }
+            ret = quota_add_parent(list, dentry->name, dentry->par);
+            if (ret == 1)
+                count++;
+            else if (ret == -1)
+                break;
         }
-        UNLOCK (&ctx->lock);
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        return count;
+    return (ret == -1) ? -1 : count;
 }
 
 int32_t
-quota_build_ancestry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          gf_dirent_t *entries, dict_t *xdata)
+quota_build_ancestry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                         dict_t *xdata)
 {
-        inode_t           *parent       = NULL;
-        inode_t           *tmp_parent   = NULL;
-        inode_t           *linked_inode = NULL;
-        inode_t           *tmp_inode    = NULL;
-        gf_dirent_t       *entry        = NULL;
-        loc_t              loc          = {0, };
-        quota_dentry_t    *dentry       = NULL;
-        quota_dentry_t    *tmp          = NULL;
-        quota_inode_ctx_t *ctx          = NULL;
-        struct list_head   parents      = {0, };
-        quota_local_t     *local        = NULL;
-
-        INIT_LIST_HEAD (&parents);
-
-        local = frame->local;
-        frame->local = NULL;
-
-        if (op_ret < 0)
-                goto err;
+    inode_t *parent = NULL;
+    inode_t *tmp_parent = NULL;
+    inode_t *linked_inode = NULL;
+    inode_t *tmp_inode = NULL;
+    gf_dirent_t *entry = NULL;
+    loc_t loc = {
+        0,
+    };
+    quota_dentry_t *dentry = NULL;
+    quota_dentry_t *tmp = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    struct list_head parents;
+    quota_local_t *local = NULL;
+    int ret;
+
+    INIT_LIST_HEAD(&parents);
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto err;
+
+    if ((op_ret > 0) && (entries != NULL)) {
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            if (__is_root_gfid(entry->inode->gfid)) {
+                /* The list contains a sub-list for each
+                 * possible path to the target inode. Each
+                 * sub-list starts with the root entry of the
+                 * tree and is followed by the child entries
+                 * for a particular path to the target entry.
+                 * The root entry is an implied sub-list
+                 * delimiter, as it denotes we have started
+                 * processing a new path. Reset the parent
+                 * pointer and continue
+                 */
 
-        if ((op_ret > 0) && (entries != NULL)) {
-                list_for_each_entry (entry, &entries->list, list) {
-                        if (__is_root_gfid (entry->inode->gfid)) {
-                                /* The list contains a sub-list for each
-                                 * possible path to the target inode. Each
-                                 * sub-list starts with the root entry of the
-                                 * tree and is followed by the child entries
-                                 * for a particular path to the target entry.
-                                 * The root entry is an implied sub-list
-                                 * delimiter, as it denotes we have started
-                                 * processing a new path. Reset the parent
-                                 * pointer and continue
-                                 */
-
-                                tmp_parent = NULL;
-                        } else {
-                                /* For a non-root entry, link this inode */
-                                linked_inode = inode_link (entry->inode,
-                                                           tmp_parent,
-                                                           entry->d_name,
-                                                           &entry->d_stat);
-                                if (linked_inode) {
-                                        tmp_inode = entry->inode;
-                                        entry->inode = linked_inode;
-                                        inode_unref (tmp_inode);
-                                } else {
-                                        gf_msg (this->name, GF_LOG_WARNING,
-                                                EINVAL, Q_MSG_PARENT_NULL,
-                                                "inode link failed");
-                                                op_errno = EINVAL;
-                                                goto err;
-                                }
-                        }
-
-                        gf_uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
-
-                        loc.inode = inode_ref (entry->inode);
-                        loc.parent = inode_ref (tmp_parent);
-                        loc.name = entry->d_name;
-
-                        quota_fill_inodectx (this, entry->inode, entry->dict,
-                                             &loc, &entry->d_stat, &op_errno);
-
-                        tmp_parent = entry->inode;
-
-                        loc_wipe (&loc);
+                tmp_parent = NULL;
+            } else {
+                /* For a non-root entry, link this inode */
+                linked_inode = inode_link(entry->inode, tmp_parent,
+                                          entry->d_name, &entry->d_stat);
+                if (linked_inode) {
+                    tmp_inode = entry->inode;
+                    entry->inode = linked_inode;
+                    inode_unref(tmp_inode);
+                } else {
+                    gf_msg(this->name, GF_LOG_WARNING, EINVAL,
+                           Q_MSG_PARENT_NULL, "inode link failed");
+                    op_errno = EINVAL;
+                    goto err;
                 }
+            }
+
+            gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid);
+
+            loc.inode = inode_ref(entry->inode);
+            loc.parent = inode_ref(tmp_parent);
+            loc.name = entry->d_name;
+
+            quota_fill_inodectx(this, entry->inode, entry->dict, &loc,
+                                &entry->d_stat, &op_errno);
+
+            /* For non-directory, posix_get_ancestry_non_directory
+             * returns all hard-links that are represented by nodes
+             * adjacent to each other in the dentry-list.
+             * (Unlike the directory case where adjacent nodes
+             * either have a parent/child relationship or belong to
+             * different paths).
+             */
+            if (entry->inode->ia_type == IA_IFDIR)
+                tmp_parent = entry->inode;
+
+            loc_wipe(&loc);
+        }
+    }
+
+    parent = inode_parent(local->loc.inode, 0, NULL);
+    if (parent == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_PARENT_NULL,
+               "parent is NULL");
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+
+    ret = quota_add_parents_from_ctx(ctx, &parents);
+    if (ret == -1) {
+        op_errno = errno;
+        goto err;
+    }
+
+    if (list_empty(&parents)) {
+        /* we built ancestry for a directory */
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            if (entry->inode == local->loc.inode)
+                break;
         }
 
-        parent = inode_parent (local->loc.inode, 0, NULL);
-        if (parent == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        Q_MSG_PARENT_NULL, "parent is NULL");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-
-        quota_add_parents_from_ctx (ctx, &parents);
+        /* Getting assertion here, need to investigate
+           comment for now
+           GF_ASSERT (&entry->list != &entries->list);
+        */
 
-        if (list_empty (&parents)) {
-                /* we built ancestry for a directory */
-                list_for_each_entry (entry, &entries->list, list) {
-                        if (entry->inode == local->loc.inode)
-                                break;
-                }
-
-                /* Getting assertion here, need to investigate
-                   comment for now
-                   GF_ASSERT (&entry->list != &entries->list);
-                */
-
-                quota_add_parent (&parents, entry->d_name, parent->gfid);
+        ret = quota_add_parent(&parents, entry->d_name, parent->gfid);
+        if (ret == -1) {
+            op_errno = errno;
+            goto err;
         }
+    }
 
-        local->ancestry_cbk (&parents, local->loc.inode, 0, 0,
-                             local->ancestry_data);
-        goto cleanup;
+    local->ancestry_cbk(&parents, local->loc.inode, 0, 0, local->ancestry_data);
+    goto cleanup;
 
 err:
-        local->ancestry_cbk (NULL, NULL, -1, op_errno, local->ancestry_data);
+    local->ancestry_cbk(NULL, NULL, -1, op_errno, local->ancestry_data);
 
 cleanup:
-        STACK_DESTROY (frame->root);
-        quota_local_cleanup (local);
+    STACK_DESTROY(frame->root);
+    quota_local_cleanup(local);
 
-        if (parent != NULL) {
-                inode_unref (parent);
-                parent = NULL;
-        }
+    if (parent != NULL) {
+        inode_unref(parent);
+        parent = NULL;
+    }
 
-        list_for_each_entry_safe (dentry, tmp, &parents, next) {
-                __quota_dentry_free (dentry);
+    if (!list_empty(&parents)) {
+        list_for_each_entry_safe(dentry, tmp, &parents, next)
+        {
+            __quota_dentry_free(dentry);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-quota_build_ancestry (inode_t *inode, quota_ancestry_built_t ancestry_cbk,
-                      void *data)
+quota_build_ancestry(inode_t *inode, quota_ancestry_built_t ancestry_cbk,
+                     void *data)
 {
-        fd_t          *fd        = NULL;
-        quota_local_t *local     = NULL;
-        call_frame_t  *new_frame = NULL;
-        int            op_errno  = ENOMEM;
-        int            op_ret    = -1;
-        xlator_t      *this      = NULL;
-        dict_t        *xdata_req = NULL;
-
-        this = THIS;
-
-        xdata_req = dict_new ();
-        if (xdata_req == NULL)
-                goto err;
-
-        fd = fd_anonymous (inode);
-        if (fd == NULL)
-                goto err;
+    fd_t *fd = NULL;
+    quota_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    int op_errno = ENOMEM;
+    int op_ret = -1;
+    xlator_t *this = NULL;
+    dict_t *xdata_req = NULL;
+
+    this = THIS;
+
+    xdata_req = dict_new();
+    if (xdata_req == NULL)
+        goto err;
+
+    fd = fd_anonymous(inode);
+    if (fd == NULL)
+        goto err;
+
+    new_frame = create_frame(this, this->ctx->pool);
+    if (new_frame == NULL)
+        goto err;
+
+    local = quota_local_new();
+    if (local == NULL)
+        goto err;
+
+    new_frame->root->uid = new_frame->root->gid = 0;
+    new_frame->local = local;
+    local->ancestry_cbk = ancestry_cbk;
+    local->ancestry_data = data;
+    local->loc.inode = inode_ref(inode);
+
+    op_ret = dict_set_int8(xdata_req, QUOTA_LIMIT_KEY, 1);
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        goto err;
+    }
+
+    op_ret = dict_set_int8(xdata_req, QUOTA_LIMIT_OBJECTS_KEY, 1);
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        goto err;
+    }
+
+    op_ret = dict_set_int8(xdata_req, GET_ANCESTRY_DENTRY_KEY, 1);
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        goto err;
+    }
+
+    /* This would ask posix layer to construct dentry chain till root
+     * We don't need to do a opendir, we can use the anonymous fd
+     * here for  the readidrp.
+     * avoiding opendir also reduces the window size where another FOP
+     * can be executed before completion of build ancestry
+     */
+    STACK_WIND(new_frame, quota_build_ancestry_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, 0, 0, xdata_req);
+
+    op_ret = 0;
 
-        new_frame = create_frame (this, this->ctx->pool);
-        if (new_frame == NULL)
-                goto err;
+err:
+    if (fd)
+        fd_unref(fd);
 
-        local = quota_local_new ();
-        if (local == NULL)
-                goto err;
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-        new_frame->root->uid = new_frame->root->gid = 0;
-        new_frame->local = local;
-        local->ancestry_cbk = ancestry_cbk;
-        local->ancestry_data = data;
-        local->loc.inode = inode_ref (inode);
+    if (op_ret < 0) {
+        ancestry_cbk(NULL, NULL, -1, op_errno, data);
 
-        op_ret = dict_set_int8 (xdata_req, QUOTA_LIMIT_KEY, 1);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto err;
+        if (new_frame) {
+            local = new_frame->local;
+            new_frame->local = NULL;
+            STACK_DESTROY(new_frame->root);
         }
 
-        op_ret = dict_set_int8 (xdata_req, QUOTA_LIMIT_OBJECTS_KEY, 1);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto err;
-        }
+        if (local)
+            quota_local_cleanup(local);
+    }
 
-        op_ret = dict_set_int8 (xdata_req, GET_ANCESTRY_DENTRY_KEY, 1);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto err;
-        }
+    return 0;
+}
 
-        /* This would ask posix layer to construct dentry chain till root
-         * We don't need to do a opendir, we can use the anonymous fd
-         * here for  the readidrp.
-         * avoiding opendir also reduces the window size where another FOP
-         * can be executed before completion of build ancestry
-         */
-        STACK_WIND (new_frame, quota_build_ancestry_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, 0, 0, xdata_req);
+int
+quota_validate(call_frame_t *frame, inode_t *inode, xlator_t *this,
+               fop_lookup_cbk_t cbk_fn)
+{
+    quota_local_t *local = NULL;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    quota_priv_t *priv = NULL;
 
-        op_ret = 0;
+    local = frame->local;
+    priv = this->private;
 
-err:
-        if (fd)
-                fd_unref (fd);
+    LOCK(&local->lock);
+    {
+        loc_wipe(&local->validate_loc);
 
-        if (xdata_req)
-                dict_unref (xdata_req);
-
-        if (op_ret < 0) {
-                ancestry_cbk (NULL, NULL, -1, op_errno, data);
+        ret = quota_inode_loc_fill(inode, &local->validate_loc);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENFORCEMENT_FAILED,
+                   "cannot fill loc for inode (gfid:%s), hence "
+                   "aborting quota-checks and continuing with fop",
+                   uuid_utoa(inode->gfid));
+        }
+    }
+    UNLOCK(&local->lock);
+
+    if (ret < 0) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    xdata = dict_new();
+    if (xdata == NULL) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, QUOTA_SIZE_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set failed");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_str(xdata, "volume-uuid", priv->volume_uuid);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set failed");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = quota_enforcer_lookup(frame, this, xdata, cbk_fn);
+    if (ret < 0) {
+        ret = -ENOTCONN;
+        goto err;
+    }
+
+    ret = 0;
+err:
+    if (xdata)
+        dict_unref(xdata);
 
-                if (new_frame) {
-                        local = new_frame->local;
-                        new_frame->local = NULL;
-                        STACK_DESTROY (new_frame->root);
-                }
+    return ret;
+}
 
-                if (local)
-                        quota_local_cleanup (local);
+void
+quota_check_limit_continuation(struct list_head *parents, inode_t *inode,
+                               int32_t op_ret, int32_t op_errno, void *data)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    quota_local_t *local = NULL;
+    quota_local_t *par_local = NULL;
+    quota_dentry_t *entry = NULL;
+    inode_t *parent = NULL;
+    int parent_count = 0;
+
+    frame = data;
+    local = frame->local;
+    this = THIS;
+
+    if (local->par_frame)
+        par_local = local->par_frame->local;
+    else
+        par_local = local;
+
+    if ((op_ret < 0) || list_empty(parents)) {
+        if (op_ret >= 0) {
+            gf_msg(this->name, GF_LOG_WARNING, EIO, Q_MSG_ANCESTRY_BUILD_FAILED,
+                   "Couldn't build ancestry for inode (gfid:%s). "
+                   "Without knowing ancestors till root, quota"
+                   "cannot be enforced. "
+                   "Hence, failing fop with EIO",
+                   uuid_utoa(inode->gfid));
+            op_errno = EIO;
+        }
+
+        quota_handle_validate_error(frame, -1, op_errno);
+        goto out;
+    }
+
+    list_for_each_entry(entry, parents, next) { parent_count++; }
+
+    LOCK(&par_local->lock);
+    {
+        par_local->link_count += (parent_count - 1);
+    }
+    UNLOCK(&par_local->lock);
+
+    if (local->par_frame) {
+        list_for_each_entry(entry, parents, next)
+        {
+            parent = inode_find(inode->table, entry->par);
+            quota_check_limit(frame, parent, this);
+            inode_unref(parent);
+        }
+    } else {
+        list_for_each_entry(entry, parents, next)
+        {
+            parent = do_quota_check_limit(frame, inode, this, entry, _gf_true);
+            if (parent)
+                inode_unref(parent);
+            else
+                quota_link_count_decrement(frame);
         }
+    }
 
-        return 0;
+out:
+    return;
 }
 
-int
-quota_validate (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                fop_lookup_cbk_t cbk_fn)
+int32_t
+quota_check_object_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                         quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                         int32_t *op_errno, int just_validated,
+                         quota_local_t *local, gf_boolean_t *skip_check)
 {
-        quota_local_t     *local = NULL;
-        int                ret   = 0;
-        dict_t            *xdata = NULL;
-        quota_priv_t      *priv  = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&local->lock);
+    int32_t ret = -1;
+    uint32_t timeout = 0;
+    char need_validate = 0;
+    gf_boolean_t hard_limit_exceeded = 0;
+    int64_t object_aggr_count = 0;
+
+    GF_ASSERT(frame);
+    GF_ASSERT(priv);
+    GF_ASSERT(_inode);
+    GF_ASSERT(this);
+    GF_ASSERT(local);
+
+    if (ctx != NULL && (ctx->object_hard_lim > 0 || ctx->object_soft_lim)) {
+        LOCK(&ctx->lock);
         {
-                loc_wipe (&local->validate_loc);
-
-                ret = quota_inode_loc_fill (inode, &local->validate_loc);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                Q_MSG_ENFORCEMENT_FAILED,
-				"cannot fill loc for inode (gfid:%s), hence "
-                                "aborting quota-checks and continuing with fop",
-                                uuid_utoa (inode->gfid));
-                }
-        }
-        UNLOCK (&local->lock);
+            timeout = priv->soft_timeout;
 
-        if (ret < 0) {
-                ret = -ENOMEM;
-                goto err;
-        }
+            object_aggr_count = ctx->file_count + ctx->dir_count + 1;
+            if (((ctx->object_soft_lim >= 0) &&
+                 (object_aggr_count) > ctx->object_soft_lim)) {
+                timeout = priv->hard_timeout;
+            }
 
-        xdata = dict_new ();
-        if (xdata == NULL) {
-                ret = -ENOMEM;
-                goto err;
+            if (!just_validated && quota_timeout(ctx->validate_time, timeout)) {
+                need_validate = 1;
+            } else if ((object_aggr_count) > ctx->object_hard_lim) {
+                hard_limit_exceeded = 1;
+            }
         }
+        UNLOCK(&ctx->lock);
 
-        ret = dict_set_int8 (xdata, QUOTA_SIZE_KEY, 1);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "dict set failed");
-                ret = -ENOMEM;
-                goto err;
+        if (need_validate && *skip_check != _gf_true) {
+            *skip_check = _gf_true;
+            ret = quota_validate(frame, _inode, this, quota_validate_cbk);
+            if (ret < 0) {
+                *op_errno = -ret;
+                *skip_check = _gf_false;
+            }
+            goto out;
         }
 
-        ret = dict_set_str (xdata, "volume-uuid", priv->volume_uuid);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "dict set failed");
-                ret = -ENOMEM;
-                goto err;
+        if (hard_limit_exceeded) {
+            local->op_ret = -1;
+            local->op_errno = EDQUOT;
+            *op_errno = EDQUOT;
+            goto out;
         }
 
-        ret = quota_enforcer_lookup (frame, this, xdata, cbk_fn);
-        if (ret < 0) {
-                ret = -ENOTCONN;
-                goto err;
-        }
+        /*We log usage only if quota limit is configured on
+           that inode
+        */
+        quota_log_usage(this, ctx, _inode, 0);
+    }
 
-        ret = 0;
-err:
-        if (xdata)
-                dict_unref (xdata);
+    ret = 0;
 
-        return ret;
+out:
+    return ret;
 }
 
-void
-quota_check_limit_continuation (struct list_head *parents, inode_t *inode,
-                                int32_t op_ret, int32_t op_errno, void *data)
+int32_t
+quota_check_size_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                       quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                       int32_t *op_errno, int just_validated, int64_t delta,
+                       quota_local_t *local, gf_boolean_t *skip_check)
 {
-        call_frame_t   *frame        = NULL;
-        xlator_t       *this         = NULL;
-        quota_local_t  *local        = NULL;
-        quota_local_t  *par_local    = NULL;
-        quota_dentry_t *entry        = NULL;
-        inode_t        *parent       = NULL;
-        int             parent_count = 0;
-
-        frame = data;
-        local = frame->local;
-        this = THIS;
-
-        if (local->par_frame)
-                par_local = local->par_frame->local;
-        else
-                par_local = local;
-
-
-        if ((op_ret < 0) || list_empty (parents)) {
-                if (op_ret >= 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, EIO,
-				Q_MSG_ANCESTRY_BUILD_FAILED,
-                                "Couldn't build ancestry for inode (gfid:%s). "
-				"Without knowing ancestors till root, quota"
-                                "cannot be enforced. "
-                                "Hence, failing fop with EIO",
-				uuid_utoa (inode->gfid));
-                        op_errno = EIO;
-                }
+    int32_t ret = -1;
+    uint32_t timeout = 0;
+    char need_validate = 0;
+    gf_boolean_t hard_limit_exceeded = 0;
+    int64_t space_available = 0;
+    int64_t wouldbe_size = 0;
+
+    GF_ASSERT(frame);
+    GF_ASSERT(priv);
+    GF_ASSERT(_inode);
+    GF_ASSERT(this);
+    GF_ASSERT(local);
+
+    if (ctx != NULL && (ctx->hard_lim > 0 || ctx->soft_lim > 0)) {
+        wouldbe_size = ctx->size + delta;
+
+        LOCK(&ctx->lock);
+        {
+            timeout = priv->soft_timeout;
 
-                quota_handle_validate_error (frame, -1, op_errno);
-                goto out;
-        }
+            if ((ctx->soft_lim >= 0) && (wouldbe_size > ctx->soft_lim)) {
+                timeout = priv->hard_timeout;
+            }
 
-        list_for_each_entry (entry, parents, next) {
-                parent_count++;
+            if (!just_validated && quota_timeout(ctx->validate_time, timeout)) {
+                need_validate = 1;
+            } else if (wouldbe_size >= ctx->hard_lim) {
+                hard_limit_exceeded = 1;
+            }
         }
+        UNLOCK(&ctx->lock);
 
-        LOCK (&par_local->lock);
-        {
-                par_local->link_count += (parent_count - 1);
+        if (need_validate && *skip_check != _gf_true) {
+            *skip_check = _gf_true;
+            ret = quota_validate(frame, _inode, this, quota_validate_cbk);
+            if (ret < 0) {
+                *op_errno = -ret;
+                *skip_check = _gf_false;
+            }
+            goto out;
         }
-        UNLOCK (&par_local->lock);
 
-        if (local->par_frame) {
-                list_for_each_entry (entry, parents, next) {
-                        parent = inode_find (inode->table, entry->par);
-                        quota_check_limit (frame, parent, this);
-                        inode_unref (parent);
-                }
-        } else {
-                list_for_each_entry (entry, parents, next) {
-                        parent = do_quota_check_limit (frame, inode, this,
-                                                       entry, _gf_true);
-                        if (parent)
-                                inode_unref (parent);
-                        else
-                                quota_link_count_decrement (frame);
-                }
-        }
+        if (hard_limit_exceeded) {
+            local->op_ret = -1;
+            local->op_errno = EDQUOT;
 
-out:
-        return;
-}
+            space_available = ctx->hard_lim - ctx->size;
 
-int32_t
-quota_check_object_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated,
-                          quota_local_t *local, gf_boolean_t *skip_check)
-{
-        int32_t         ret                     = -1;
-        uint32_t        timeout                 =  0;
-        char            need_validate           =  0;
-        gf_boolean_t    hard_limit_exceeded     =  0;
-        int64_t         object_aggr_count       =  0;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (priv);
-        GF_ASSERT (_inode);
-        GF_ASSERT (this);
-        GF_ASSERT (local);
-
-        if (ctx != NULL && (ctx->object_hard_lim > 0 ||
-                            ctx->object_soft_lim)) {
-                LOCK (&ctx->lock);
-                {
-                        timeout = priv->soft_timeout;
-
-                        object_aggr_count = ctx->file_count +
-                                            ctx->dir_count + 1;
-                        if (((ctx->object_soft_lim >= 0)
-                             && (object_aggr_count) >
-                             ctx->object_soft_lim)) {
-                                timeout = priv->hard_timeout;
-                        }
-
-                        if (!just_validated
-                            && quota_timeout (&ctx->tv, timeout)) {
-                                need_validate = 1;
-                        } else if ((object_aggr_count) >
-                                  ctx->object_hard_lim) {
-                                hard_limit_exceeded = 1;
-                        }
-                }
-                UNLOCK (&ctx->lock);
-
-                if (need_validate && *skip_check != _gf_true) {
-                        *skip_check = _gf_true;
-                        ret = quota_validate (frame, _inode, this,
-                                              quota_validate_cbk);
-                        if (ret < 0) {
-                                *op_errno = -ret;
-                                *skip_check = _gf_false;
-                        }
-                        goto out;
-                }
+            if (space_available < 0)
+                space_available = 0;
 
-                if (hard_limit_exceeded) {
-                        local->op_ret = -1;
-                        local->op_errno = EDQUOT;
-                        *op_errno = EDQUOT;
-                        goto out;
-                }
+            if ((local->space_available < 0) ||
+                (local->space_available > space_available)) {
+                local->space_available = space_available;
+            }
 
-                /*We log usage only if quota limit is configured on
-                   that inode
-                */
-                quota_log_usage (this, ctx, _inode, 0);
+            if (space_available == 0) {
+                *op_errno = EDQUOT;
+                goto out;
+            }
         }
 
-        ret = 0;
+        /* We log usage only if quota limit is configured on
+           that inode. */
+        quota_log_usage(this, ctx, _inode, delta);
+    }
 
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-quota_check_size_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated, int64_t delta,
-                          quota_local_t *local, gf_boolean_t *skip_check)
+quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this)
 {
-        int32_t         ret                     = -1;
-        uint32_t        timeout                 =  0;
-        char            need_validate           =  0;
-        gf_boolean_t    hard_limit_exceeded     =  0;
-        int64_t         space_available         =  0;
-        int64_t         wouldbe_size            =  0;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (priv);
-        GF_ASSERT (_inode);
-        GF_ASSERT (this);
-        GF_ASSERT (local);
-
-        if (ctx != NULL && (ctx->hard_lim > 0 || ctx->soft_lim > 0)) {
-                wouldbe_size = ctx->size + delta;
-
-                LOCK (&ctx->lock);
-                {
-                        timeout = priv->soft_timeout;
-
-                        if ((ctx->soft_lim >= 0)
-                            && (wouldbe_size > ctx->soft_lim)) {
-                                timeout = priv->hard_timeout;
-                        }
-
-                        if (!just_validated
-                            && quota_timeout (&ctx->tv, timeout)) {
-                                need_validate = 1;
-                        } else if (wouldbe_size >= ctx->hard_lim) {
-                                hard_limit_exceeded = 1;
-                        }
-                }
-                UNLOCK (&ctx->lock);
-
-                if (need_validate && *skip_check != _gf_true) {
-                        *skip_check = _gf_true;
-                        ret = quota_validate (frame, _inode, this,
-                                              quota_validate_cbk);
-                        if (ret < 0) {
-                                *op_errno = -ret;
-                                *skip_check = _gf_false;
-                        }
-                        goto out;
-                }
-
-                if (hard_limit_exceeded) {
-                        local->op_ret = -1;
-                        local->op_errno = EDQUOT;
-
-                        space_available = ctx->hard_lim - ctx->size;
+    int32_t ret = -1, op_errno = EINVAL;
+    inode_t *_inode = NULL, *parent = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    quota_local_t *par_local = NULL;
+    char just_validated = 0;
+    int64_t delta = 0;
+    int8_t object_delta = 0;
+    uint64_t value = 0;
+    gf_boolean_t skip_check = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("quota", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, frame, err);
+    GF_VALIDATE_OR_GOTO(this->name, inode, err);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(this->name, local, err);
+
+    if (local->par_frame) {
+        par_local = local->par_frame->local;
+        GF_VALIDATE_OR_GOTO(this->name, par_local, err);
+    } else {
+        par_local = local;
+    }
+
+    delta = par_local->delta;
+    object_delta = par_local->object_delta;
+
+    GF_VALIDATE_OR_GOTO(this->name, par_local->stub, err);
+    /* Allow all the trusted clients
+     * Don't block the gluster internal processes like rebalance, gsyncd,
+     * self heal etc from the disk quotas.
+     *
+     * Method: Allow all the clients with PID negative. This is by the
+     * assumption that any kernel assigned pid doesn't have the negative
+     * number.
+     */
+    if (0 > frame->root->pid) {
+        ret = 0;
+        quota_link_count_decrement(frame);
+        goto done;
+    }
 
-                        if (space_available < 0)
-                                space_available = 0;
+    priv = this->private;
 
-                        if ((local->space_available < 0)
-                            || (local->space_available
-                                > space_available)){
-                                local->space_available
-                                        = space_available;
+    inode_ctx_get(inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
-                        }
+    _inode = inode_ref(inode);
 
-                        if (space_available == 0) {
-                                *op_errno = EDQUOT;
-                                goto out;
-                        }
-                }
+    LOCK(&local->lock);
+    {
+        just_validated = local->just_validated;
+        local->just_validated = 0;
+    }
+    UNLOCK(&local->lock);
 
-                /* We log usage only if quota limit is configured on
-                   that inode. */
-                quota_log_usage (this, ctx, _inode, delta);
+    do {
+        /* In a rename operation, enforce should be stopped at common
+           ancestor */
+        if (!gf_uuid_is_null(par_local->common_ancestor) &&
+            !gf_uuid_compare(_inode->gfid, par_local->common_ancestor)) {
+            quota_link_count_decrement(frame);
+            break;
         }
 
-        ret = 0;
-out:
-        return ret;
-}
-
+        if (object_delta <= 0)
+            goto skip_check_object_limit;
 
-int32_t
-quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this)
-{
-        int32_t            ret                 = -1, op_errno = EINVAL;
-        inode_t           *_inode              = NULL, *parent = NULL;
-        quota_inode_ctx_t *ctx                 = NULL;
-        quota_priv_t      *priv                = NULL;
-        quota_local_t     *local               = NULL;
-        quota_local_t     *par_local           = NULL;
-        char               need_validate       = 0;
-        char               just_validated      = 0;
-        gf_boolean_t       hard_limit_exceeded = 0;
-        int64_t            delta               = 0;
-        int8_t             object_delta        = 0;
-        uint64_t           value               = 0;
-        gf_boolean_t       skip_check          = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("quota", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        GF_VALIDATE_OR_GOTO (this->name, inode, err);
-
-        local  = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-
-        if (local->par_frame) {
-                par_local = local->par_frame->local;
-                GF_VALIDATE_OR_GOTO (this->name, par_local, err);
-        } else {
-                par_local = local;
-        }
+        ret = quota_check_object_limit(frame, ctx, priv, _inode, this,
+                                       &op_errno, just_validated, par_local,
+                                       &skip_check);
+        if (skip_check == _gf_true)
+            goto done;
 
-        delta = par_local->delta;
-        object_delta = par_local->object_delta;
-
-        GF_VALIDATE_OR_GOTO (this->name, par_local->stub, err);
-        /* Allow all the trusted clients
-         * Don't block the gluster internal processes like rebalance, gsyncd,
-         * self heal etc from the disk quotas.
-         *
-         * Method: Allow all the clients with PID negative. This is by the
-         * assumption that any kernel assigned pid doesn't have the negative
-         * number.
-         */
-        if (0 > frame->root->pid) {
-                ret = 0;
-                quota_link_count_decrement (frame);
-                goto done;
+        if (ret) {
+            if (op_errno != EDQUOT)
+                gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_ENFORCEMENT_FAILED,
+                       "Failed to "
+                       "check quota object limit");
+            goto err;
         }
 
-        priv = this->private;
-
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-
-        _inode = inode_ref (inode);
+    skip_check_object_limit:
+        ret = quota_check_size_limit(frame, ctx, priv, _inode, this, &op_errno,
+                                     just_validated, delta, par_local,
+                                     &skip_check);
+        if (skip_check == _gf_true)
+            goto done;
 
-        LOCK (&local->lock);
-        {
-                just_validated = local->just_validated;
-                local->just_validated = 0;
+        if (ret) {
+            if (op_errno != EDQUOT)
+                gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_ENFORCEMENT_FAILED,
+                       "Failed to "
+                       "check quota size limit");
+            goto err;
         }
-        UNLOCK (&local->lock);
-
-        do {
-                /* In a rename operation, enforce should be stopped at common
-                   ancestor */
-                if (!gf_uuid_is_null (par_local->common_ancestor) &&
-                    !gf_uuid_compare (_inode->gfid, par_local->common_ancestor)
-                   ) {
-                        quota_link_count_decrement (frame);
-                        break;
-                }
-
-                if (object_delta <= 0)
-                        goto skip_check_object_limit;
-
-                ret = quota_check_object_limit (frame, ctx, priv, _inode, this,
-                                                &op_errno, just_validated,
-                                                par_local, &skip_check);
-                if (skip_check == _gf_true)
-                        goto done;
-
-                if (ret) {
-                        if (op_errno != EDQUOT)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        Q_MSG_ENFORCEMENT_FAILED, "Failed to "
-                                        "check quota object limit");
-                        goto err;
-                }
 
-skip_check_object_limit:
-                ret = quota_check_size_limit (frame, ctx, priv, _inode, this,
-                                              &op_errno, just_validated, delta,
-                                              par_local, &skip_check);
-                if (skip_check == _gf_true)
-                        goto done;
-
-                if (ret) {
-                        if (op_errno != EDQUOT)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        Q_MSG_ENFORCEMENT_FAILED, "Failed to "
-                                        "check quota size limit");
-                        goto err;
-                }
+        if (__is_root_gfid(_inode->gfid)) {
+            quota_link_count_decrement(frame);
+            break;
+        }
 
-                if (__is_root_gfid (_inode->gfid)) {
-                        quota_link_count_decrement (frame);
-                        break;
-                }
+        parent = inode_parent(_inode, 0, NULL);
+        if (parent == NULL) {
+            ret = quota_build_ancestry(_inode, quota_check_limit_continuation,
+                                       frame);
+            if (ret < 0) {
+                op_errno = -ret;
+                goto err;
+            }
 
-                parent = inode_parent (_inode, 0, NULL);
-                if (parent == NULL) {
-                        ret = quota_build_ancestry (_inode,
-                                                 quota_check_limit_continuation,
-                                                 frame);
-                        if (ret < 0) {
-                                op_errno = -ret;
-                                goto err;
-                        }
-
-                        break;
-                }
+            break;
+        }
 
-                inode_unref (_inode);
-                _inode = parent;
-                just_validated = 0;
+        inode_unref(_inode);
+        _inode = parent;
+        just_validated = 0;
 
-                value = 0;
-                inode_ctx_get (_inode, this, &value);
-                ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        } while (1);
+        value = 0;
+        inode_ctx_get(_inode, this, &value);
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    } while (1);
 
 done:
-        if (_inode != NULL) {
-                inode_unref (_inode);
-                _inode = NULL;
-        }
-        return 0;
+    if (_inode != NULL) {
+        inode_unref(_inode);
+        _inode = NULL;
+    }
+    return 0;
 
 err:
-        quota_handle_validate_error (frame, -1, op_errno);
+    quota_handle_validate_error(frame, -1, op_errno);
 
-        inode_unref (_inode);
-        return 0;
+    inode_unref(_inode);
+    return 0;
 }
 
 inode_t *
-do_quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                      quota_dentry_t *dentry, gf_boolean_t force)
+do_quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                     quota_dentry_t *dentry, gf_boolean_t force)
 {
-        int32_t         ret        = -1;
-        inode_t        *parent     = NULL;
-        call_frame_t   *new_frame  = NULL;
-        quota_local_t  *local      = NULL;
-        quota_local_t  *new_local  = NULL;
-
-        local = frame->local;
-
-        parent = inode_parent (inode, dentry->par, dentry->name);
-        if (parent == NULL) {
-                if (force)
-                        parent = inode_find (inode->table, dentry->par);
-                else
-                        goto out;
-        }
-        if (parent == NULL)
-                goto out;
+    int32_t ret = -1;
+    inode_t *parent = NULL;
+    call_frame_t *new_frame = NULL;
+    quota_local_t *new_local = NULL;
+
+    parent = inode_parent(inode, dentry->par, dentry->name);
+    if (parent == NULL) {
+        if (force)
+            parent = inode_find(inode->table, dentry->par);
+        else
+            goto out;
+    }
+    if (parent == NULL)
+        goto out;
 
-        new_frame = copy_frame (frame);
-        if (new_frame == NULL)
-                goto out;
+    new_frame = copy_frame(frame);
+    if (new_frame == NULL)
+        goto out;
 
-        new_local = quota_local_new ();
-        if (new_local == NULL)
-                goto out;
+    new_local = quota_local_new();
+    if (new_local == NULL)
+        goto out;
 
-        new_frame->local = new_local;
-        new_local->par_frame = frame;
+    new_frame->local = new_local;
+    new_local->par_frame = frame;
 
-        quota_check_limit (new_frame, parent, this);
+    quota_check_limit(new_frame, parent, this);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret < 0) {
-                if (parent) {
-                        /* Caller should decrement link_count, in case parent is
-                         * NULL
-                         */
-                        quota_handle_validate_error (frame, -1, ENOMEM);
-                }
-
-                if (new_frame) {
-                        new_frame->local = NULL;
-                        STACK_DESTROY (new_frame->root);
-                }
+    if (ret < 0) {
+        if (parent) {
+            /* Caller should decrement link_count, in case parent is
+             * NULL
+             */
+            quota_handle_validate_error(frame, -1, ENOMEM);
+        }
 
-                if (new_local)
-                        quota_local_cleanup (new_local);
+        if (new_frame) {
+            new_frame->local = NULL;
+            STACK_DESTROY(new_frame->root);
         }
+    }
 
-        return parent;
+    return parent;
 }
 
 static int
-quota_get_limits (xlator_t *this, dict_t *dict, int64_t *hard_lim,
-                  int64_t *soft_lim, int64_t *object_hard_limit,
-                  int64_t *object_soft_limit)
+quota_get_limits(xlator_t *this, dict_t *dict, int64_t *hard_lim,
+                 int64_t *soft_lim, int64_t *object_hard_limit,
+                 int64_t *object_soft_limit)
 {
-        quota_limits_t *limit             = NULL;
-        quota_limits_t *object_limit      = NULL;
-        quota_priv_t   *priv              = NULL;
-        int64_t         soft_lim_percent  = 0;
-        int64_t        *ptr               = NULL;
-        int             ret               = 0;
-
-        if ((this == NULL) || (dict == NULL) || (hard_lim == NULL)
-            || (soft_lim == NULL))
-                goto out;
+    quota_limits_t *limit = NULL;
+    quota_limits_t *object_limit = NULL;
+    quota_priv_t *priv = NULL;
+    int64_t soft_lim_percent = 0;
+    int64_t *ptr = NULL;
+    int ret = 0;
 
-        priv = this->private;
+    if ((this == NULL) || (dict == NULL) || (hard_lim == NULL) ||
+        (soft_lim == NULL))
+        goto out;
 
-        ret = dict_get_bin (dict, QUOTA_LIMIT_KEY, (void **) &ptr);
-        limit = (quota_limits_t *)ptr;
+    priv = this->private;
 
-        if (limit) {
-                *hard_lim = ntoh64 (limit->hl);
-                soft_lim_percent = ntoh64 (limit->sl);
-        }
+    ret = dict_get_bin(dict, QUOTA_LIMIT_KEY, (void **)&ptr);
+    limit = (quota_limits_t *)ptr;
 
-        if (soft_lim_percent < 0) {
-                soft_lim_percent = priv->default_soft_lim;
-        }
+    if (limit) {
+        *hard_lim = ntoh64(limit->hl);
+        soft_lim_percent = ntoh64(limit->sl);
+    }
 
-        if ((*hard_lim > 0) && (soft_lim_percent > 0)) {
-                *soft_lim = (soft_lim_percent * (*hard_lim))/100;
-        }
+    if (soft_lim_percent < 0) {
+        soft_lim_percent = priv->default_soft_lim;
+    }
 
-        ret = dict_get_bin (dict, QUOTA_LIMIT_OBJECTS_KEY, (void **) &ptr);
-        if (ret)
-                return 0;
-        object_limit = (quota_limits_t *)ptr;
+    if ((*hard_lim > 0) && (soft_lim_percent > 0)) {
+        *soft_lim = (soft_lim_percent * (*hard_lim)) / 100;
+    }
 
-        if (object_limit) {
-                *object_hard_limit = ntoh64 (object_limit->hl);
-                 soft_lim_percent = ntoh64 (object_limit->sl);
-        }
+    ret = dict_get_bin(dict, QUOTA_LIMIT_OBJECTS_KEY, (void **)&ptr);
+    if (ret)
+        return 0;
+    object_limit = (quota_limits_t *)ptr;
 
-        if (soft_lim_percent < 0) {
-                soft_lim_percent = priv->default_soft_lim;
-        }
+    if (object_limit) {
+        *object_hard_limit = ntoh64(object_limit->hl);
+        soft_lim_percent = ntoh64(object_limit->sl);
+    }
 
-        if ((*object_hard_limit > 0) && (soft_lim_percent > 0)) {
-                *object_soft_limit = (soft_lim_percent *
-                                     (*object_hard_limit))/100;
-        }
+    if (soft_lim_percent < 0) {
+        soft_lim_percent = priv->default_soft_lim;
+    }
+
+    if ((*object_hard_limit > 0) && (soft_lim_percent > 0)) {
+        *object_soft_limit = (soft_lim_percent * (*object_hard_limit)) / 100;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-quota_fill_inodectx (xlator_t *this, inode_t *inode, dict_t *dict,
-                     loc_t *loc, struct iatt *buf, int32_t *op_errno)
+quota_fill_inodectx(xlator_t *this, inode_t *inode, dict_t *dict, loc_t *loc,
+                    struct iatt *buf, int32_t *op_errno)
 {
-        int32_t            ret                  = -1;
-        char               found                = 0;
-        quota_inode_ctx_t *ctx                  = NULL;
-        quota_dentry_t    *dentry               = NULL;
-        uint64_t           value                = 0;
-        int64_t            hard_lim             = 0;
-        int64_t            soft_lim             = 0;
-        int64_t            object_hard_limit    = 0;
-        int64_t            object_soft_limit    = 0;
-
-        quota_get_limits (this, dict, &hard_lim, &soft_lim, &object_hard_limit,
-                          &object_soft_limit);
-
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-
-        if ((((ctx == NULL) || (ctx->hard_lim == hard_lim))
-             && (hard_lim < 0) && !QUOTA_REG_OR_LNK_FILE (buf->ia_type))) {
-                ret = 0;
-                goto out;
+    int32_t ret = -1;
+    char found = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+    uint64_t value = 0;
+    int64_t hard_lim = 0;
+    int64_t soft_lim = 0;
+    int64_t object_hard_limit = 0;
+    int64_t object_soft_limit = 0;
+
+    quota_get_limits(this, dict, &hard_lim, &soft_lim, &object_hard_limit,
+                     &object_soft_limit);
+
+    inode_ctx_get(inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+
+    if ((((ctx == NULL) || (ctx->hard_lim == hard_lim)) && (hard_lim < 0) &&
+         !QUOTA_REG_OR_LNK_FILE(buf->ia_type))) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_INODE_CTX_GET_FAILED,
+               "cannot create quota "
+               "context in inode(gfid:%s)",
+               uuid_utoa(inode->gfid));
+        ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->hard_lim = hard_lim;
+        ctx->soft_lim = soft_lim;
+        ctx->object_hard_lim = object_hard_limit;
+        ctx->object_soft_lim = object_soft_limit;
+
+        ctx->buf = *buf;
+
+        if (!QUOTA_REG_OR_LNK_FILE(buf->ia_type)) {
+            goto unlock;
+        }
+
+        /* do nothing if it is a nameless lookup */
+        if (loc->name == NULL || !loc->parent)
+            goto unlock;
+
+        list_for_each_entry(dentry, &ctx->parents, next)
+        {
+            if ((strcmp(dentry->name, loc->name) == 0) &&
+                (gf_uuid_compare(loc->parent->gfid, dentry->par) == 0)) {
+                found = 1;
+                break;
+            }
         }
 
-        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        Q_MSG_INODE_CTX_GET_FAILED, "cannot create quota "
-                        "context in inode(gfid:%s)", uuid_utoa (inode->gfid));
+        if (!found) {
+            dentry = __quota_dentry_new(ctx, (char *)loc->name,
+                                        loc->parent->gfid);
+            if (dentry == NULL) {
+                /*
+                  gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
+                          Q_MSG_ENOMEM,
+                          "cannot create a new dentry (par:%"
+-                                          PRId64", name:%s) for inode(ino:%"
+-                                          PRId64", gfid:%s)",
+-                                          uuid_utoa (local->loc.inode->gfid));
+                */
                 ret = -1;
                 *op_errno = ENOMEM;
-                goto out;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&ctx->lock);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->hard_lim = hard_lim;
-                ctx->soft_lim = soft_lim;
-                ctx->object_hard_lim = object_hard_limit;
-                ctx->object_soft_lim = object_soft_limit;
-
-                ctx->buf = *buf;
-
-                if (!QUOTA_REG_OR_LNK_FILE (buf->ia_type)) {
-                        goto unlock;
-                }
+out:
+    return ret;
+}
 
-                /* do nothing if it is a nameless lookup */
-                if (loc->name == NULL || !loc->parent)
-                        goto unlock;
-
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, loc->name) == 0) &&
-                            (gf_uuid_compare (loc->parent->gfid,
-                                           dentry->par) == 0)) {
-                                found = 1;
-                                break;
-                        }
-                }
+/*
+ * return _gf_true if enforcement is needed and _gf_false otherwise
+ */
+gf_boolean_t
+should_quota_enforce(xlator_t *this, dict_t *dict, glusterfs_fop_t fop)
+{
+    int ret = 0;
 
-                if (!found) {
-                        dentry = __quota_dentry_new (ctx,
-                                                     (char *)loc->name,
-                                                     loc->parent->gfid);
-                        if (dentry == NULL) {
-                                /*
-                                  gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-					  Q_MSG_ENOMEM,
-                                          "cannot create a new dentry (par:%"
--                                          PRId64", name:%s) for inode(ino:%"
--                                          PRId64", gfid:%s)",
--                                          uuid_utoa (local->loc.inode->gfid));
-                                */
-                                ret = -1;
-                                *op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&ctx->lock);
+    ret = dict_check_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
 
+    if (fop == GF_FOP_MKDIR && ret == DICT_FLAG_SET) {
+        return _gf_false;
+    } else if (ret == -ENOENT) {
+        gf_msg(this->name, GF_LOG_DEBUG, EINVAL, Q_MSG_INTERNAL_FOP_KEY_MISSING,
+               "No internal fop context present");
+        goto out;
+    }
 out:
-        return ret;
+    return _gf_true;
 }
 
 int32_t
-quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *dict, struct iatt *postparent)
+quota_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *dict, struct iatt *postparent)
 {
-        quota_local_t      *local        = NULL;
-        int32_t             ret          = 0;
-        inode_t            *this_inode   = NULL;
+    quota_local_t *local = NULL;
+    inode_t *this_inode = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret >= 0 && inode) {
-                this_inode = inode_ref (inode);
+    if (op_ret >= 0 && inode) {
+        this_inode = inode_ref(inode);
 
-                op_ret = quota_fill_inodectx (this, inode, dict, &local->loc,
-                                              buf, &op_errno);
-                if (op_ret < 0)
-                        op_errno = ENOMEM;
-        }
+        op_ret = quota_fill_inodectx(this, inode, dict, &local->loc, buf,
+                                     &op_errno);
+        if (op_ret < 0)
+            op_errno = ENOMEM;
+    }
 
-        QUOTA_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                            dict, postparent);
+    QUOTA_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, dict,
+                       postparent);
 
-        if (op_ret < 0 || this_inode == NULL || gf_uuid_is_null(this_inode->gfid))
-                goto out;
+    if (op_ret < 0 || this_inode == NULL || gf_uuid_is_null(this_inode->gfid))
+        goto out;
 
-        check_ancestory_2 (this, local, this_inode);
+    check_ancestory_2(this, local, this_inode);
 
 out:
-        if (this_inode)
-                inode_unref (this_inode);
+    if (this_inode)
+        inode_unref(this_inode);
 
-        quota_local_cleanup (local);
+    quota_local_cleanup(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xattr_req)
+quota_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        quota_priv_t  *priv             = NULL;
-        int32_t        ret              = -1;
-        quota_local_t *local            = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
-        if (!xattr_req)
-                goto err;
+    xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+    if (!xattr_req)
+        goto err;
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
-        loc_copy (&local->loc, loc);
+    frame->local = local;
+    loc_copy(&local->loc, loc);
 
-        ret = dict_set_int8 (xattr_req, QUOTA_LIMIT_KEY, 1);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "dict set of key for "
-                        "hard-limit failed");
-                goto err;
-        }
+    ret = dict_set_int8(xattr_req, QUOTA_LIMIT_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set of key for "
+               "hard-limit failed");
+        goto err;
+    }
 
-        ret = dict_set_int8 (xattr_req, QUOTA_LIMIT_OBJECTS_KEY, 1);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
-                        "dict set of key for quota object limit failed");
-                goto err;
-        }
+    ret = dict_set_int8(xattr_req, QUOTA_LIMIT_OBJECTS_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set of key for quota object limit failed");
+        goto err;
+    }
 
-        STACK_WIND (frame, quota_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    STACK_WIND(frame, quota_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
 
-        ret = 0;
+    ret = 0;
 
 err:
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        if (ret < 0) {
-                QUOTA_STACK_UNWIND (lookup, frame, -1, ENOMEM,
-                                    NULL, NULL, NULL, NULL);
-        }
+    if (ret < 0) {
+        QUOTA_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+    }
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
+                    loc, xattr_req);
+    return 0;
 }
 
 int32_t
-quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+quota_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t                  ret            = 0;
-        uint64_t                 ctx_int        = 0;
-        quota_inode_ctx_t       *ctx            = NULL;
-        quota_local_t           *local          = NULL;
-
-        local = frame->local;
-
-        if ((op_ret < 0) || (local == NULL) || (postbuf == NULL)) {
-                goto out;
-        }
-
-        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_INODE_CTX_GET_FAILED, "%s: failed to get the "
-			"context", local->loc.path);
-                goto out;
-        }
-
-        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
-
-        if (ctx == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-			Q_MSG_INODE_CTX_GET_FAILED,
-                        "quota context not set in %s (gfid:%s)",
-                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    int32_t ret = 0;
+    uint64_t ctx_int = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = NULL;
+
+    local = frame->local;
+
+    if ((op_ret < 0) || (local == NULL) || (postbuf == NULL)) {
+        goto out;
+    }
+
+    ret = inode_ctx_get(local->loc.inode, this, &ctx_int);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "%s: failed to get the "
+               "context",
+               local->loc.path);
+        goto out;
+    }
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)ctx_int;
+
+    if (ctx == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context not set in %s (gfid:%s)", local->loc.path,
+               uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-
-        return 0;
-}
-
-
-int32_t
-quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     struct iovec *vector, int32_t count, off_t off,
-                     uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        quota_local_t *local      = NULL;
-        int32_t        op_errno   = EINVAL;
-        quota_priv_t  *priv       = NULL;
-        struct iovec  *new_vector = NULL;
-        int32_t        new_count  = 0;
-
-        priv = this->private;
-
-        local = frame->local;
-
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
-
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-
-                if ((op_errno == EDQUOT) && (local->space_available > 0)) {
-                        new_count = iov_subset (vector, count, 0,
-                                                local->space_available, NULL);
-
-                        new_vector = GF_CALLOC (new_count,
-                                                sizeof (struct iovec),
-                                                gf_common_mt_iovec);
-                        if (new_vector == NULL) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        new_count = iov_subset (vector, count, 0,
-                                                local->space_available,
-                                                new_vector);
-
-                        vector = new_vector;
-                        count = new_count;
-                } else if (op_errno == ENOENT || op_errno == ESTALE) {
-                        /* We may get ENOENT/ESTALE in case of below scenario
-                         *     fd = open file.txt
-                         *     unlink file.txt
-                         *     write on fd
-                         * Here build_ancestry can fail as the file is removed.
-                         * For now ignore ENOENT/ESTALE with writes on active fd
-                         * We need to re-visit this code once we understand
-                         * how other file-system behave in this scenario
-                         */
-                        gf_msg_debug (this->name, 0, "quota enforcer failed "
-                                      "with ENOENT/ESTALE on %s, cannot check "
-                                      "quota limits and allowing writes",
-                                      uuid_utoa (fd->inode->gfid));
-                } else {
-                        goto unwind;
-                }
-        }
+    QUOTA_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        STACK_WIND (frame, quota_writev_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev, fd,
-                    vector, count, off, flags, iobref, xdata);
-
-        if (new_vector != NULL)
-                GF_FREE (new_vector);
-
-        return 0;
-
-unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    return 0;
 }
 
+static int gf_quota_enforcer_log;
 
 int32_t
-quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count, off_t off,
-              uint32_t flags, struct iobref *iobref, dict_t *xdata)
+quota_writev_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    struct iovec *vector, int32_t count, off_t off,
+                    uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        quota_priv_t      *priv       = NULL;
-        int32_t            ret        = -1, op_errno = EINVAL;
-        int32_t            parents    = 0;
-        int32_t            fail_count = 0;
-        uint64_t           size       = 0;
-        quota_local_t     *local      = NULL;
-        quota_inode_ctx_t *ctx        = NULL;
-        quota_dentry_t    *dentry     = NULL, *tmp = NULL;
-        call_stub_t       *stub       = NULL;
-        struct list_head   head       = {0, };
-        inode_t           *par_inode  = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+    struct iovec *new_vector = NULL;
+    int32_t new_count = 0;
 
-        priv = this->private;
+    local = frame->local;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        INIT_LIST_HEAD (&head);
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
 
-        local = quota_local_new ();
-        if (local == NULL) {
+        if ((op_errno == EDQUOT) && (local->space_available > 0)) {
+            new_count = iov_subset(vector, count, 0, local->space_available,
+                                   &new_vector, 0);
+            if (new_count < 0) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
                 goto unwind;
+            }
+
+            vector = new_vector;
+            count = new_count;
+        } else if (op_errno == ENOENT || op_errno == ESTALE) {
+            /* We may get ENOENT/ESTALE in case of below scenario
+             *     fd = open file.txt
+             *     unlink file.txt
+             *     write on fd
+             * Here build_ancestry can fail as the file is removed.
+             * For now ignore ENOENT/ESTALE with writes on active fd
+             * We need to re-visit this code once we understand
+             * how other file-system behave in this scenario
+             */
+            gf_msg_debug(this->name, 0,
+                         "quota enforcer failed "
+                         "with ENOENT/ESTALE on %s, cannot check "
+                         "quota limits and allowing writes",
+                         uuid_utoa(fd->inode->gfid));
+        } else if ((op_errno == EINVAL) &&
+                   !inode_parent(local->loc.inode, 0, NULL)) {
+            /* We may get INVAL with parent == NULL,
+             * in case of below scenario
+             *     1. enable quota
+             *     2. glusterfsd stop/start
+             *     3. nameless lookup
+             *     4. write on fd
+             * Here build_ancestry can fail as the file's pgfid
+             * is't exist.
+             * For now ignore EINVAL with writes on active fd
+             * untils the pgfid is created at name lookup
+             */
+            GF_LOG_OCCASIONALLY(gf_quota_enforcer_log, this->name,
+                                GF_LOG_CRITICAL,
+                                "Quota cannot be enforced as "
+                                "parent is not available and writes are being "
+                                "allowed without checking whether they are "
+                                "within quota limits. This can happen if Quota "
+                                "crawl is not complete. If crawl has been "
+                                "completed, please file a bug.");
+        } else {
+            goto unwind;
         }
+    }
 
-        frame->local = local;
-        local->loc.inode = inode_ref (fd->inode);
+    STACK_WIND(frame, quota_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
 
-        ret = quota_inode_ctx_get (fd->inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (fd->inode->gfid));
-        }
-
-        stub = fop_writev_stub (frame, quota_writev_helper, fd, vector, count,
-                                off, flags, iobref, xdata);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    if (new_vector != NULL)
+        GF_FREE(new_vector);
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
+    return 0;
 
-        size = iov_length (vector, count);
+unwind:
+    QUOTA_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
-        parents = quota_add_parents_from_ctx (ctx, &head);
+int32_t
+quota_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
+{
+    quota_priv_t *priv = NULL;
+    int32_t op_errno = EINVAL;
+    int32_t parents = 0;
+    int32_t fail_count = 0;
+    uint64_t size = 0;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL, *tmp = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head head;
+    inode_t *par_inode = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    INIT_LIST_HEAD(&head);
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO("quota", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->loc.inode = inode_ref(fd->inode);
+
+    (void)quota_inode_ctx_get(fd->inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(fd->inode->gfid));
+    }
+
+    stub = fop_writev_stub(frame, quota_writev_helper, fd, vector, count, off,
+                           flags, iobref, xdata);
+    if (stub == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, unwind);
+
+    parents = quota_add_parents_from_ctx(ctx, &head);
+    if (parents == -1) {
+        op_errno = errno;
+        goto unwind;
+    }
+
+    size = iov_length(vector, count);
+
+    LOCK(&local->lock);
+    {
+        local->delta = size;
+        local->object_delta = 0;
+        local->link_count = (parents != 0) ? parents : 1;
+        local->stub = stub;
+    }
+    UNLOCK(&local->lock);
 
-        LOCK (&local->lock);
+    if (parents == 0) {
+        /* nameless lookup on this inode, allow quota to reconstruct
+         * ancestry as part of check_limit.
+         */
+        quota_check_limit(frame, fd->inode, this);
+    } else {
+        list_for_each_entry_safe(dentry, tmp, &head, next)
         {
-                local->delta = size;
-                local->object_delta = 0;
-                local->link_count = (parents != 0) ? parents : 1;
-                local->stub = stub;
+            par_inode = do_quota_check_limit(frame, fd->inode, this, dentry,
+                                             _gf_false);
+            if (par_inode == NULL) {
+                if (ctx) {
+                    /* remove stale entry from inode ctx */
+                    quota_dentry_del(ctx, dentry->name, dentry->par);
+                    parents--;
+                    fail_count++;
+                }
+            } else {
+                inode_unref(par_inode);
+            }
+            __quota_dentry_free(dentry);
         }
-        UNLOCK (&local->lock);
 
         if (parents == 0) {
-                /* nameless lookup on this inode, allow quota to reconstruct
-                 * ancestry as part of check_limit.
-                 */
-                quota_check_limit (frame, fd->inode, this);
-        } else {
-                list_for_each_entry_safe (dentry, tmp, &head, next) {
-                        par_inode = do_quota_check_limit (frame, fd->inode,
-                                                          this, dentry,
-                                                          _gf_false);
-                        if (par_inode == NULL) {
-                                /* remove stale entry from inode ctx */
-                                quota_dentry_del (ctx, dentry->name,
-                                                  dentry->par);
-                                parents--;
-                                fail_count++;
-                        } else {
-                                inode_unref (par_inode);
-                        }
-                        __quota_dentry_free (dentry);
-                }
-
-                if (parents == 0) {
-                        LOCK (&local->lock);
-                        {
-                                local->link_count++;
-                        }
-                        UNLOCK (&local->lock);
-                        quota_check_limit (frame, fd->inode, this);
-                }
+            LOCK(&local->lock);
+            {
+                local->link_count++;
+            }
+            UNLOCK(&local->lock);
+            quota_check_limit(frame, fd->inode, this);
+        }
 
-                while (fail_count != 0) {
-                        quota_link_count_decrement (frame);
-                        fail_count--;
-                }
+        while (fail_count != 0) {
+            quota_link_count_decrement(frame);
+            fail_count--;
         }
+    }
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->writev, fd,
-                         vector, count, off, flags, iobref, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    fd, vector, count, off, flags, iobref, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+quota_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, mode_t umask, dict_t *xdata)
+quota_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   mode_t umask, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_mkdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc,
-                    mode, umask, xdata);
+    STACK_WIND(frame, quota_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
+    return 0;
 }
 
-
 int32_t
-quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             mode_t umask, dict_t *xdata)
+quota_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
-        quota_priv_t  *priv  = NULL;
-        int32_t        ret   = 0, op_errno = 0;
-        quota_local_t *local = NULL;
-        call_stub_t   *stub  = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto err;
-        }
-
-        stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, umask,
-                               xdata);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                local->stub = stub;
-                local->delta = 0;
-                local->object_delta = 1;
-                local->link_count = 1;
-        }
-        UNLOCK (&local->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = 0, op_errno = 0;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    if (!should_quota_enforce(this, xdata, GF_FOP_MKDIR)) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, Q_MSG_ENFORCEMENT_SKIPPED,
+               "Enforcement has been skipped(internal fop).");
+        goto off;
+    }
+
+    local = quota_local_new();
+    if (local == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    stub = fop_mkdir_stub(frame, quota_mkdir_helper, loc, mode, umask, xdata);
+    if (stub == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->stub = stub;
+        local->delta = 0;
+        local->object_delta = 1;
+        local->link_count = 1;
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, loc->parent, this);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL, NULL);
+    QUOTA_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->mkdir,
-                         loc, mode, umask, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+quota_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int32_t            ret    = -1;
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                goto unwind;
-        }
-
-        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        Q_MSG_INODE_CTX_GET_FAILED, "cannot create quota "
-                        "context in inode(gfid:%s)", uuid_utoa (inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-
-                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
-                                             local->loc.parent->gfid);
-                if (dentry == NULL) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                Q_MSG_ENOMEM, "cannot create a new dentry "
-                                "(name:%s) for inode(gfid:%s)", local->loc.name,
-                                uuid_utoa (local->loc.inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unlock;
-                }
-        }
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        goto unwind;
+    }
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_INODE_CTX_GET_FAILED,
+               "cannot create quota "
+               "context in inode(gfid:%s)",
+               uuid_utoa(inode->gfid));
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+
+        dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                    local->loc.parent->gfid);
+        if (dentry == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "cannot create a new dentry "
+                   "(name:%s) for inode(gfid:%s)",
+                   local->loc.name, uuid_utoa(local->loc.inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unlock;
+        }
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 unwind:
-        QUOTA_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                       preparent, postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-                     dict_t *xdata)
+quota_create_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                    dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        priv = this->private;
-
-
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        goto unwind;
+    }
 
-
-        STACK_WIND (frame, quota_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create, loc,
-                    flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, quota_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+quota_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        quota_priv_t  *priv     = NULL;
-        int32_t        ret      = -1;
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = 0;
-        call_stub_t   *stub     = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-        QUOTA_WIND_FOR_INTERNAL_FOP (xdata, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        stub = fop_create_stub (frame, quota_create_helper, loc, flags, mode,
-                                umask, fd, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                local->stub = stub;
-                local->delta = 0;
-                local->object_delta = 1;
-        }
-        UNLOCK (&local->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    int32_t op_errno = 0;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+    QUOTA_WIND_FOR_INTERNAL_FOP(xdata, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_create_stub(frame, quota_create_helper, loc, flags, mode, umask,
+                           fd, xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        local->stub = stub;
+        local->delta = 0;
+        local->object_delta = 1;
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, loc->parent, this);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL, NULL, NULL);
+    QUOTA_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->create, loc,
-                         flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+quota_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
-        uint64_t           value = 0;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = (quota_local_t *) frame->local;
+    local = (quota_local_t *)frame->local;
 
-        inode_ctx_get (local->loc.inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    inode_ctx_get(local->loc.inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
-        if (ctx == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-			Q_MSG_INODE_CTX_GET_FAILED,
-			"quota context not set inode (gfid:%s)",
-                        uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    if (ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, EINVAL, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context not set inode (gfid:%s)",
+               uuid_utoa(local->loc.gfid));
+        goto out;
+    }
 
-        quota_dentry_del (ctx, local->loc.name, local->loc.parent->gfid);
+    quota_dentry_del(ctx, local->loc.name, local->loc.parent->gfid);
 
 out:
-        QUOTA_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
-                            postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+quota_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t        ret = -1;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto err;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
 
-        STACK_WIND (frame, quota_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    STACK_WIND(frame, quota_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
-        ret = 0;
+    ret = 0;
 
 err:
-        if (ret == -1) {
-                QUOTA_STACK_UNWIND (unlink, frame, -1, 0, NULL, NULL, NULL);
-        }
+    if (ret == -1) {
+        QUOTA_STACK_UNWIND(unlink, frame, -1, 0, NULL, NULL, NULL);
+    }
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+quota_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        int32_t               ret          = -1;
-        quota_local_t        *local        = NULL;
-        quota_inode_ctx_t    *ctx          = NULL;
-        quota_dentry_t       *dentry       = NULL;
-        char                  found        = 0;
-
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = (quota_local_t *) frame->local;
-
-        ret = quota_inode_ctx_get (inode, this, &ctx, 0);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+    char found = 0;
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    local = (quota_local_t *)frame->local;
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 0);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry(dentry, &ctx->parents, next)
         {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, local->loc.name) == 0) &&
-                            (gf_uuid_compare (local->loc.parent->gfid,
-                                           dentry->par) == 0)) {
-                               found = 1;
-
-                               gf_msg_debug (this->name, 0, "new entry being"
-                                            " linked (name:%s) for inode "
-                                            "(gfid:%s) is already present "
-                                            "in inode-dentry-list",
-                                            dentry->name,
-                                           uuid_utoa (local->loc.inode->gfid));
-                               break;
-                        }
-                }
-
-                if (!found) {
-                        dentry = __quota_dentry_new (ctx,
-                                                     (char *)local->loc.name,
-                                                     local->loc.parent->gfid);
-                        if (dentry == NULL) {
-                                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                        Q_MSG_ENOMEM,
-					"cannot create a new dentry (name:%s)"
-					"for inode(gfid:%s)", local->loc.name,
-                                        uuid_utoa (local->loc.inode->gfid));
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-
-                ctx->buf = *buf;
+            if ((strcmp(dentry->name, local->loc.name) == 0) &&
+                (gf_uuid_compare(local->loc.parent->gfid, dentry->par) == 0)) {
+                found = 1;
+
+                gf_msg_debug(this->name, 0,
+                             "new entry being"
+                             " linked (name:%s) for inode "
+                             "(gfid:%s) is already present "
+                             "in inode-dentry-list",
+                             dentry->name, uuid_utoa(local->loc.inode->gfid));
+                break;
+            }
+        }
+
+        if (!found) {
+            dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                        local->loc.parent->gfid);
+            if (dentry == NULL) {
+                gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                       "cannot create a new dentry (name:%s)"
+                       "for inode(gfid:%s)",
+                       local->loc.name, uuid_utoa(local->loc.inode->gfid));
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+
+        ctx->buf = *buf;
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
+    QUOTA_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                   loc_t *newloc, dict_t *xdata)
+quota_link_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                  loc_t *newloc, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        priv = this->private;
+    local = frame->local;
 
-        local = frame->local;
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
+    op_errno = local->op_errno;
 
-        op_errno = local->op_errno;
-
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, oldloc,
-                    newloc, xdata);
-        return 0;
+    STACK_WIND(frame, quota_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 void
-quota_link_continue (call_frame_t *frame)
+quota_link_continue(call_frame_t *frame)
 {
-        int32_t            ret               = -1;
-        int32_t            op_errno          = EIO;
-        quota_local_t     *local             = NULL;
-        uuid_t             common_ancestor   = {0};
-        xlator_t          *this              = NULL;
-        quota_inode_ctx_t *ctx               = NULL;
-        inode_t           *src_parent        = NULL;
-        inode_t           *dst_parent        = NULL;
-
-        local = frame->local;
-        this = THIS;
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-                goto err;
-        }
-
-        if (local->xdata &&
-            dict_get (local->xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
-                /* Treat link as rename, crawl upwards only till common ancestor
-                 */
-                ret = quota_find_common_ancestor (local->oldloc.inode,
-                                                  local->newloc.parent,
-                                                  &common_ancestor);
-                if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
-                        gf_msg (this->name, GF_LOG_ERROR, ESTALE,
-                                Q_MSG_ANCESTRY_BUILD_FAILED, "failed to get "
-                                "common_ancestor for %s and %s",
-                                local->oldloc.path, local->newloc.path);
-                        op_errno = ESTALE;
-                        goto err;
-                }
-        } else {
-                /* Treat link as a new file.
-                 * TODO: Currently marker accounts twice for the links created
-                 * across directories.
-                 * This needs re-vist if marker accounts only once
-                 * for the links created across directories
-                 */
-                if (local->oldloc.parent)
-                        src_parent = inode_ref (local->oldloc.parent);
-                else
-                        src_parent = inode_parent (local->oldloc.inode, 0,
-                                                   NULL);
-                dst_parent = local->newloc.parent;
-
-                /* No need to check quota limit if src and dst parents are same
-                 */
-                if (src_parent == dst_parent ||
-                    gf_uuid_compare (src_parent->gfid, dst_parent->gfid) == 0) {
-                        inode_unref (src_parent);
-                        goto wind;
-                }
-
-                inode_unref (src_parent);
-        }
+    int32_t ret = -1;
+    int32_t op_errno = EIO;
+    quota_local_t *local = NULL;
+    uuid_t common_ancestor = {0};
+    xlator_t *this = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    inode_t *src_parent = NULL;
+    inode_t *dst_parent = NULL;
+
+    local = frame->local;
+    this = THIS;
+
+    if (local->op_ret < 0) {
+        op_errno = local->op_errno;
+        goto err;
+    }
 
-        quota_inode_ctx_get (local->oldloc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->oldloc.inode->gfid));
-        }
+    if (local->xdata && dict_get(local->xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
+        /* Treat link as rename, crawl upwards only till common ancestor
+         */
+        ret = quota_find_common_ancestor(
+            local->oldloc.inode, local->newloc.parent, &common_ancestor);
+        if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
+            gf_msg(this->name, GF_LOG_ERROR, ESTALE,
+                   Q_MSG_ANCESTRY_BUILD_FAILED,
+                   "failed to get "
+                   "common_ancestor for %s and %s",
+                   local->oldloc.path, local->newloc.path);
+            op_errno = ESTALE;
+            goto err;
+        }
+    } else {
+        /* Treat link as a new file.
+         * TODO: Currently marker accounts twice for the links created
+         * across directories.
+         * This needs re-visit if marker accounts only once
+         * for the links created across directories
+         */
+        if (local->oldloc.parent)
+            src_parent = inode_ref(local->oldloc.parent);
+        else
+            src_parent = inode_parent(local->oldloc.inode, 0, NULL);
+        dst_parent = local->newloc.parent;
 
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                local->delta = (ctx != NULL) ? ctx->buf.ia_blocks * 512 : 0;
-                local->object_delta = 1;
-                gf_uuid_copy (local->common_ancestor, common_ancestor);
-        }
-        UNLOCK (&local->lock);
+        /* No need to check quota limit if src and dst parents are same
+         */
+        if (src_parent == dst_parent ||
+            gf_uuid_compare(src_parent->gfid, dst_parent->gfid) == 0) {
+            inode_unref(src_parent);
+            goto wind;
+        }
+
+        inode_unref(src_parent);
+    }
+
+    quota_inode_ctx_get(local->oldloc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->oldloc.inode->gfid));
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        local->delta = (ctx != NULL) ? ctx->buf.ia_blocks * 512 : 0;
+        local->object_delta = 1;
+        gf_uuid_copy(local->common_ancestor, common_ancestor);
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, local->newloc.parent, this);
-        return;
+    quota_check_limit(frame, local->newloc.parent, this);
+    return;
 
 err:
-        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return;
+    QUOTA_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return;
 
 wind:
-        STACK_WIND (frame, quota_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, &(local->oldloc),
-                    &(local->newloc), local->xdata);
-        return;
+    STACK_WIND(frame, quota_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, &(local->oldloc),
+               &(local->newloc), local->xdata);
+    return;
 }
 
 int32_t
-quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-            dict_t *xdata)
+quota_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        quota_priv_t      *priv     = NULL;
-        int32_t            ret      = -1;
-        int32_t            op_errno = ENOMEM;
-        quota_local_t     *local    = NULL;
-        call_stub_t       *stub     = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
-
-        frame->local = (void *) local;
-
-        if (xdata)
-                local->xdata = dict_ref (xdata);
-
-        ret = loc_copy (&local->loc, newloc);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto err;
-        }
-
-        ret = loc_copy (&local->oldloc, oldloc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
-                        "loc_copy failed");
-                goto err;
-        }
-
-        ret = loc_copy (&local->newloc, newloc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
-                        "loc_copy failed");
-                goto err;
-        }
-
-        /* No need to check quota limit if src and dst parents are same */
-        if (oldloc->parent && newloc->parent &&
-            !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
-                gf_msg_debug (this->name, GF_LOG_DEBUG, "link %s -> %s are "
-                              "in the same directory, so skip check limit",
-                              oldloc->path, newloc->path);
-                goto wind;
-        }
-
-        stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t op_errno = ENOMEM;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
+
+    frame->local = (void *)local;
+
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+
+    ret = loc_copy(&local->loc, newloc);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    ret = loc_copy(&local->oldloc, oldloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    ret = loc_copy(&local->newloc, newloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    /* No need to check quota limit if src and dst parents are same */
+    if (oldloc->parent && newloc->parent &&
+        !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
+        gf_msg_debug(this->name, GF_LOG_DEBUG,
+                     "link %s -> %s are "
+                     "in the same directory, so skip check limit",
+                     oldloc->path, newloc->path);
+        goto wind;
+    }
+
+    stub = fop_link_stub(frame, quota_link_helper, oldloc, newloc, xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 2;
+        local->fop_continue_cbk = quota_link_continue;
+        local->stub = stub;
+    }
+    UNLOCK(&local->lock);
 
-        LOCK (&local->lock);
-        {
-                local->link_count = 2;
-                local->fop_continue_cbk = quota_link_continue;
-                local->stub = stub;
-        }
-        UNLOCK (&local->lock);
+    check_ancestory(frame, newloc->parent);
 
-        check_ancestory (frame, newloc->parent);
+    /* source parent can be NULL, so do check_ancestry on a file */
+    if (oldloc->parent)
+        check_ancestory(frame, oldloc->parent);
+    else
+        check_ancestory(frame, oldloc->inode);
 
-        /* source parent can be NULL, so do check_ancestory on a file */
-        if (oldloc->parent)
-                check_ancestory (frame, oldloc->parent);
-        else
-                check_ancestory (frame, oldloc->inode);
-
-        return 0;
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->link, oldloc,
-                         newloc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+    return 0;
 
 wind:
-        STACK_WIND (frame, quota_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc,
-                    newloc, xdata);
-        return 0;
+    STACK_WIND(frame, quota_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent,
-                  dict_t *xdata)
+quota_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 struct iatt *preoldparent, struct iatt *postoldparent,
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
 {
-        int32_t               ret              = -1;
-        int64_t               size             = 0;
-        quota_local_t        *local            = NULL;
-        quota_inode_ctx_t    *ctx              = NULL;
-        quota_dentry_t       *old_dentry       = NULL, *dentry = NULL;
-        char                  new_dentry_found = 0;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *old_dentry = NULL, *dentry = NULL;
+    char new_dentry_found = 0;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        if (QUOTA_REG_OR_LNK_FILE (local->oldloc.inode->ia_type))
-                size = buf->ia_blocks * 512;
-        else
-                goto out;
+    if (!QUOTA_REG_OR_LNK_FILE(local->oldloc.inode->ia_type))
+        goto out;
 
-        ret = quota_inode_ctx_get (local->oldloc.inode, this, &ctx, 0);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->oldloc.inode->gfid));
+    ret = quota_inode_ctx_get(local->oldloc.inode, this, &ctx, 0);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->oldloc.inode->gfid));
 
-                goto out;
-        }
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry(dentry, &ctx->parents, next)
         {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, local->oldloc.name) == 0) &&
-                            (gf_uuid_compare (local->oldloc.parent->gfid,
-                                           dentry->par) == 0)) {
-                                old_dentry = dentry;
-                        } else if ((strcmp (dentry->name,
-                                            local->newloc.name) == 0) &&
-                                   (gf_uuid_compare (local->newloc.parent->gfid,
-                                                  dentry->par) == 0)) {
-                                new_dentry_found = 1;
-                                gf_msg_debug (this->name, 0, "new entry being "
-                                       "linked (name:%s) for inode (gfid:%s) "
-                                       "is in inode-dentry-list", dentry->name,
-                                        uuid_utoa (local->oldloc.inode->gfid));
-                        }
-
-                        if (old_dentry && new_dentry_found)
-                                break;
-                }
-
-                if (old_dentry != NULL) {
-                        __quota_dentry_free (old_dentry);
-                } else {
-                        gf_msg_debug (this->name, 0, "dentry corresponding"
-                                     "the path just renamed (name:%s) is not"
-                                     " present", local->oldloc.name);
-                }
-
-                if (!new_dentry_found) {
-                        dentry = __quota_dentry_new (ctx,
-                                                     (char *)local->newloc.name,
-                                                     local->newloc.parent->gfid);
-                        if (dentry == NULL) {
-                                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                        Q_MSG_ENOMEM,
-					"cannot create a new dentry (name:%s) "
-					"for inode(gfid:%s)",
-					local->newloc.name,
-                                        uuid_utoa (local->newloc.inode->gfid));
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-
-                ctx->buf = *buf;
+            if ((strcmp(dentry->name, local->oldloc.name) == 0) &&
+                (gf_uuid_compare(local->oldloc.parent->gfid, dentry->par) ==
+                 0)) {
+                old_dentry = dentry;
+            } else if ((strcmp(dentry->name, local->newloc.name) == 0) &&
+                       (gf_uuid_compare(local->newloc.parent->gfid,
+                                        dentry->par) == 0)) {
+                new_dentry_found = 1;
+                gf_msg_debug(this->name, 0,
+                             "new entry being "
+                             "linked (name:%s) for inode (gfid:%s) "
+                             "is in inode-dentry-list",
+                             dentry->name,
+                             uuid_utoa(local->oldloc.inode->gfid));
+            }
+
+            if (old_dentry && new_dentry_found)
+                break;
+        }
+
+        if (old_dentry != NULL) {
+            __quota_dentry_free(old_dentry);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "dentry corresponding"
+                         "the path just renamed (name:%s) is not"
+                         " present",
+                         local->oldloc.name);
+        }
+
+        if (!new_dentry_found) {
+            dentry = __quota_dentry_new(ctx, (char *)local->newloc.name,
+                                        local->newloc.parent->gfid);
+            if (dentry == NULL) {
+                gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                       "cannot create a new dentry (name:%s) "
+                       "for inode(gfid:%s)",
+                       local->newloc.name,
+                       uuid_utoa(local->newloc.inode->gfid));
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+
+        ctx->buf = *buf;
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (rename, frame, op_ret, op_errno, buf, preoldparent,
-                            postoldparent, prenewparent, postnewparent, xdata);
+    QUOTA_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                       postoldparent, prenewparent, postnewparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                     loc_t *newloc, dict_t *xdata)
+quota_rename_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                    loc_t *newloc, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        priv = this->private;
+    local = frame->local;
 
-        local = frame->local;
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
+    op_errno = local->op_errno;
 
-        op_errno = local->op_errno;
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    STACK_WIND(frame, quota_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        STACK_WIND (frame, quota_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, oldloc,
-                    newloc, xdata);
-
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return 0;
 }
 
-
 static int32_t
-quota_rename_get_size_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, inode_t *inode,
-                           struct iatt *buf, dict_t *xdata,
-                           struct iatt *postparent)
+quota_rename_get_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata,
+                          struct iatt *postparent)
 {
-        quota_local_t     *local      = NULL;
-        int32_t            ret        = 0;
-        int64_t           *size       = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, out, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, out, op_errno,
-                                        EINVAL);
-        local = frame->local;
-        GF_ASSERT (local);
-        local->link_count = 1;
-
-        if (op_ret < 0)
-                goto out;
-
-
-        ret = dict_get_bin (xdata, QUOTA_SIZE_KEY, (void **) &size);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-			Q_MSG_SIZE_KEY_MISSING, "size key not present in dict");
-                op_errno = EINVAL;
-                goto out;
-        }
-        local->delta = ntoh64 (*size);
-        local->object_delta = 1;
-        quota_check_limit (frame, local->newloc.parent, this);
-        return 0;
+    quota_local_t *local = NULL;
+    int32_t ret = 0;
+    int64_t *size = 0;
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("quota", this, out, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, xdata, out, op_errno, EINVAL);
+    local = frame->local;
+    GF_ASSERT(local);
+    local->link_count = 1;
+
+    if (op_ret < 0)
+        goto out;
+
+    ret = dict_get_bin(xdata, QUOTA_SIZE_KEY, (void **)&size);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_SIZE_KEY_MISSING,
+               "size key not present in dict");
+        op_errno = EINVAL;
+        goto out;
+    }
+    local->delta = ntoh64(*size);
+    local->object_delta = 1;
+    quota_check_limit(frame, local->newloc.parent, this);
+    return 0;
 
 out:
-        quota_handle_validate_error (frame, -1, op_errno);
-        return 0;
+    quota_handle_validate_error(frame, -1, op_errno);
+    return 0;
 }
 
 void
-quota_rename_continue (call_frame_t *frame)
+quota_rename_continue(call_frame_t *frame)
 {
-        int32_t            ret               = -1;
-        int32_t            op_errno          = EIO;
-        quota_local_t     *local             = NULL;
-        uuid_t             common_ancestor   = {0};
-        xlator_t          *this              = NULL;
-        quota_inode_ctx_t *ctx               = NULL;
-
-        local = frame->local;
-        this = THIS;
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-                goto err;
-        }
-
-        ret = quota_find_common_ancestor (local->oldloc.parent,
-                                          local->newloc.parent,
-                                          &common_ancestor);
-        if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
-                gf_msg (this->name, GF_LOG_ERROR, ESTALE,
-                        Q_MSG_ANCESTRY_BUILD_FAILED, "failed to get "
-                        "common_ancestor for %s and %s",
-                        local->oldloc.path, local->newloc.path);
-                op_errno = ESTALE;
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                gf_uuid_copy (local->common_ancestor, common_ancestor);
-        }
-        UNLOCK (&local->lock);
-
-        if (QUOTA_REG_OR_LNK_FILE (local->oldloc.inode->ia_type)) {
-                ret = quota_inode_ctx_get (local->oldloc.inode, this, &ctx, 0);
-                if (ctx == NULL) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                Q_MSG_INODE_CTX_GET_FAILED,
-                                "quota context not set in inode (gfid:%s), "
-                                "considering file size as zero while enforcing "
-                                "quota on new ancestry",
-                                uuid_utoa (local->oldloc.inode->gfid));
+    int32_t ret = -1;
+    int32_t op_errno = EIO;
+    quota_local_t *local = NULL;
+    uuid_t common_ancestor = {0};
+    xlator_t *this = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
+    local = frame->local;
+    this = THIS;
 
-                        local->delta = 0;
-                        local->object_delta = 1;
-                } else {
-
-                        /* FIXME: We need to account for the size occupied by
-                         * this inode on the target directory. To avoid double
-                         * accounting, we need to modify enforcer to perform
-                         * quota_check_limit only uptil the least common
-                         * ancestor directory inode*/
+    if (local->op_ret < 0) {
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    ret = quota_find_common_ancestor(local->oldloc.parent, local->newloc.parent,
+                                     &common_ancestor);
+    if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
+        gf_msg(this->name, GF_LOG_ERROR, ESTALE, Q_MSG_ANCESTRY_BUILD_FAILED,
+               "failed to get "
+               "common_ancestor for %s and %s",
+               local->oldloc.path, local->newloc.path);
+        op_errno = ESTALE;
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        gf_uuid_copy(local->common_ancestor, common_ancestor);
+    }
+    UNLOCK(&local->lock);
 
-                        /* FIXME: The following code assumes that regular files
-                         * and linkfiles are present, in their entirety, in a
-                         * single brick. This *assumption is invalid in the
-                         * case of stripe.*/
+    if (QUOTA_REG_OR_LNK_FILE(local->oldloc.inode->ia_type)) {
+        ret = quota_inode_ctx_get(local->oldloc.inode, this, &ctx, 0);
+        if (ctx == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+                   "quota context not set in inode (gfid:%s), "
+                   "considering file size as zero while enforcing "
+                   "quota on new ancestry",
+                   uuid_utoa(local->oldloc.inode->gfid));
+
+            local->delta = 0;
+            local->object_delta = 1;
+        } else {
+            /* FIXME: We need to account for the size occupied by
+             * this inode on the target directory. To avoid double
+             * accounting, we need to modify enforcer to perform
+             * quota_check_limit only up till the least common
+             * ancestor directory inode*/
 
-                        local->delta = ctx->buf.ia_blocks * 512;
-                        local->object_delta = 1;
-                }
+            /* FIXME: The following code assumes that regular files
+             * and link files are present, in their entirety, in a
+             * single brick. This *assumption is invalid in the
+             * case of stripe.*/
 
-        } else if (IA_ISDIR (local->oldloc.inode->ia_type)) {
-                        ret = quota_validate (frame, local->oldloc.inode, this,
-                                              quota_rename_get_size_cbk);
-                        if (ret){
-                                op_errno = -ret;
-                                goto err;
-                        }
+            local->delta = ctx->buf.ia_blocks * 512;
+            local->object_delta = 1;
+        }
 
-                        return;
+    } else if (IA_ISDIR(local->oldloc.inode->ia_type)) {
+        ret = quota_validate(frame, local->oldloc.inode, this,
+                             quota_rename_get_size_cbk);
+        if (ret) {
+            op_errno = -ret;
+            goto err;
         }
 
-        quota_check_limit (frame, local->newloc.parent, this);
         return;
+    }
 
-err:
-        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                            NULL, NULL, NULL, NULL, NULL);
-        return;
+    quota_check_limit(frame, local->newloc.parent, this);
+    return;
 
+err:
+    QUOTA_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return;
 }
 
 int32_t
-quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-              loc_t *newloc, dict_t *xdata)
+quota_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        quota_priv_t      *priv              = NULL;
-        int32_t            ret               = -1;
-        int32_t            op_errno          = ENOMEM;
-        quota_local_t     *local             = NULL;
-        call_stub_t       *stub              = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->oldloc, oldloc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
-                        "loc_copy failed");
-                goto err;
-        }
-
-        ret = loc_copy (&local->newloc, newloc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
-                        "loc_copy failed");
-                goto err;
-        }
-
-        /* No need to check quota limit if src and dst parents are same */
-        if (oldloc->parent && newloc->parent &&
-            !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
-                gf_msg_debug (this->name, 0, "rename %s -> %s are "
-                              "in the same directory, so skip check limit",
-                              oldloc->path, newloc->path);
-                goto wind;
-        }
-
-        stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc,
-                                xdata);
-        if (stub == NULL) {
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                /* link_count here tell how many check_ancestory should be done
-                 * before continuing the FOP
-                 */
-                local->link_count = 2;
-                local->stub = stub;
-                local->fop_continue_cbk = quota_rename_continue;
-        }
-        UNLOCK (&local->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t op_errno = ENOMEM;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->oldloc, oldloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    ret = loc_copy(&local->newloc, newloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    /* No need to check quota limit if src and dst parents are same */
+    if (oldloc->parent && newloc->parent &&
+        !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
+        gf_msg_debug(this->name, 0,
+                     "rename %s -> %s are "
+                     "in the same directory, so skip check limit",
+                     oldloc->path, newloc->path);
+        goto wind;
+    }
+
+    stub = fop_rename_stub(frame, quota_rename_helper, oldloc, newloc, xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        /* link_count here tell how many check_ancestry should be done
+         * before continuing the FOP
+         */
+        local->link_count = 2;
+        local->stub = stub;
+        local->fop_continue_cbk = quota_rename_continue;
+    }
+    UNLOCK(&local->lock);
 
-        check_ancestory (frame, newloc->parent);
-        check_ancestory (frame, oldloc->parent);
-        return 0;
+    check_ancestory(frame, newloc->parent);
+    check_ancestory(frame, oldloc->parent);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                            NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->rename, oldloc,
-                         newloc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+    return 0;
 
 wind:
-        STACK_WIND (frame, quota_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, oldloc,
-                    newloc, xdata);
-        return 0;
+    STACK_WIND(frame, quota_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+quota_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = frame->local;
-
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->loc.inode->gfid));
-
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-
-                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
-                                             local->loc.parent->gfid);
-                if (dentry == NULL) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                Q_MSG_ENOMEM, "cannot create "
-				"a new dentry (name:%s) for inode(gfid:%s)",
-				local->loc.name,
-				uuid_utoa (local->loc.inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+    int32_t ret = -1;
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    local = frame->local;
+
+    ret = quota_inode_ctx_get(local->loc.inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+
+        dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                    local->loc.parent->gfid);
+        if (dentry == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "cannot create "
+                   "a new dentry (name:%s) for inode(gfid:%s)",
+                   local->loc.name, uuid_utoa(local->loc.inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
-        UNLOCK (&ctx->lock);
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
+    QUOTA_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                      loc_t *loc, mode_t umask, dict_t *xdata)
+quota_symlink_helper(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                     loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        priv = this->private;
-
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_symlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, quota_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
+    return 0;
 }
 
-
 int
-quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, mode_t umask, dict_t *xdata)
+quota_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        quota_priv_t  *priv     = NULL;
-        int32_t        ret      = -1;
-        int32_t        op_errno = ENOMEM;
-        quota_local_t *local    = NULL;
-        call_stub_t   *stub     = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t op_errno = ENOMEM;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto err;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
 
-        stub = fop_symlink_stub (frame, quota_symlink_helper, linkpath, loc,
-                                 umask, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
+    stub = fop_symlink_stub(frame, quota_symlink_helper, linkpath, loc, umask,
+                            xdata);
+    if (stub == NULL) {
+        goto err;
+    }
 
-        LOCK (&local->lock);
-        {
-                local->stub = stub;
-                local->delta = strlen (linkpath);
-                local->object_delta = 1;
-                local->link_count = 1;
-        }
-        UNLOCK (&local->lock);
+    LOCK(&local->lock);
+    {
+        local->stub = stub;
+        local->delta = strlen(linkpath);
+        local->object_delta = 1;
+        local->link_count = 1;
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, loc->parent, this);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL, NULL);
+    QUOTA_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->symlink,
-                         linkpath, loc, umask, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata)
+quota_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                dict_t *xdata)
+quota_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t          ret   = -1;
-        quota_local_t   *local = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret =  loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto err;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
 
-        STACK_WIND (frame, quota_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    STACK_WIND(frame, quota_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    QUOTA_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+quota_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 dict_t *xdata)
+quota_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t   *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL)
-                goto err;
+    local = quota_local_new();
+    if (local == NULL)
+        goto err;
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_ftruncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate, fd,
-                    offset, xdata);
+    STACK_WIND(frame, quota_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
 
-        return 0;
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    QUOTA_STACK_UNWIND(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->ftruncate, fd,
-                         offset, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
-int32_t
-quota_send_dir_limit_to_cli (call_frame_t *frame, xlator_t *this,
-                             inode_t *inode, const char *name)
+static int32_t
+quota_send_dir_limit_to_cli(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            const char *name, const int namelen)
 {
-        int32_t            ret               = 0;
-        char               dir_limit [1024]  = {0, };
-        dict_t            *dict              = NULL;
-        quota_inode_ctx_t *ctx               = NULL;
-        uint64_t           value             = 0;
-        quota_priv_t      *priv              = NULL;
-
-        priv = this->private;
-        if (!priv->is_quota_on) {
-                snprintf (dir_limit, 1024, "Quota is disabled please turn on");
-                goto dict_set;
-        }
-
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret < 0)
-                goto out;
-
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        snprintf (dir_limit, 1024, "%"PRId64",%"PRId64, ctx->size,
-                  ctx->hard_lim);
+    int32_t ret = 0;
+    int dir_limit_len = 0;
+    char dir_limit[64] = {
+        0,
+    };
+    dict_t *dict = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
+    quota_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv->is_quota_on) {
+        dir_limit_len = snprintf(dir_limit, sizeof(dir_limit),
+                                 "Quota is disabled please turn on");
+        goto dict_set;
+    }
+
+    ret = inode_ctx_get(inode, this, &value);
+    if (ret < 0)
+        goto out;
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    dir_limit_len = snprintf(dir_limit, sizeof(dir_limit),
+                             "%" PRId64 ",%" PRId64, ctx->size, ctx->hard_lim);
 
 dict_set:
-        dict = dict_new ();
-        if (dict == NULL) {
-                ret = -1;
-                goto out;
-        }
+    dict = dict_new();
+    if (dict == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, (char *) name, dir_limit);
-        if (ret < 0)
-                goto out;
+    ret = dict_set_nstrn(dict, (char *)name, namelen, dir_limit, dir_limit_len);
+    if (ret < 0)
+        goto out;
 
-        gf_msg_debug (this->name, 0, "str = %s", dir_limit);
+    gf_msg_debug(this->name, 0, "str = %s", dir_limit);
 
-        QUOTA_STACK_UNWIND (getxattr, frame, 0, 0, dict, NULL);
+    QUOTA_STACK_UNWIND(getxattr, frame, 0, 0, dict, NULL);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-
 int32_t
-quota_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+quota_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        int32_t ret     = 0;
+    int32_t ret = 0;
 
-        if (name && strcasecmp (name, "trusted.limit.list") == 0) {
-                ret = quota_send_dir_limit_to_cli (frame, this, fd->inode,
-                                                   name);
-                if (ret == 0) {
-                        return 0;
-                }
+    if (name && strcasecmp(name, "trusted.limit.list") == 0) {
+        ret = quota_send_dir_limit_to_cli(frame, this, fd->inode,
+                                          "trusted.limit.list",
+                                          SLEN("trusted.limit.list"));
+        if (ret == 0) {
+            return 0;
         }
+    }
 
-        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+quota_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int32_t ret     = 0;
-
-        if ((name != NULL) && strcasecmp (name, "trusted.limit.list") == 0) {
-                ret = quota_send_dir_limit_to_cli (frame, this, loc->inode,
-                                                   name);
-                if (ret == 0)
-                        return 0;
-        }
-
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    int32_t ret = 0;
+
+    if ((name != NULL) && strcasecmp(name, "trusted.limit.list") == 0) {
+        ret = quota_send_dir_limit_to_cli(frame, this, loc->inode,
+                                          "trusted.limit.list",
+                                          SLEN("trusted.limit.list"));
+        if (ret == 0)
+            return 0;
+    }
+
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                dict_t *xdata)
+quota_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = frame->local;
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (buf->ia_type)) {
-                  gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                                " (%s). If quota is not enabled recently and "
-                                "crawler has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(buf->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                if (buf)
-                        ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    if (buf) {
+        LOCK(&ctx->lock);
+        ctx->buf = *buf;
+        UNLOCK(&ctx->lock);
+    }
 
 out:
-        QUOTA_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+quota_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
-        int32_t        ret   = -1;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    int32_t ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto unwind;
-        }
+    frame->local = local;
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, quota_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(stat, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->stat, loc,
-                         xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                 dict_t *xdata)
+quota_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
-
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (buf->ia_type)) {
-                  gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                                " (%s). If quota is not enabled recently and "
-                                "crawler has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(buf->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                if (buf)
-                        ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    if (buf) {
+        LOCK(&ctx->lock);
+        ctx->buf = *buf;
+        UNLOCK(&ctx->lock);
+    }
 
 out:
-        QUOTA_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+quota_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_fstat_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat, fd,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, quota_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fstat, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fstat, fd,
-                         xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, const char *path,
-                    struct iatt *buf, dict_t *xdata)
+quota_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, const char *path,
+                   struct iatt *buf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf,
-                            xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, buf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                dict_t *xdata)
+quota_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
-        int32_t        ret   = -1;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    int32_t ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto unwind;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_readlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink, loc,
-                    size, xdata);
-        return 0;
+    STACK_WIND(frame, quota_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readlink, loc,
-                         size, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink,
+                    loc, size, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref,
-                 dict_t *xdata)
+quota_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                int32_t count, struct iatt *buf, struct iobref *iobref,
+                dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                            buf, iobref, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf,
+                       iobref, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, uint32_t flags, dict_t *xdata)
+quota_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_readv_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv, fd,
-                    size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, quota_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL,
-                            NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readv, fd,
-                         size, offset, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+quota_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+quota_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, quota_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd,
-                    flags, xdata);
-        return 0;
+    STACK_WIND(frame, quota_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fsync, fd,
-                         flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost, dict_t *xdata)
+quota_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                  struct iatt *statpost, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
-
-        if (op_ret < 0) {
-                goto out;
-        }
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        local = frame->local;
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (statpost->ia_type)) {
-                  gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                                " (%s). If quota is not enabled recently and "
-                                "crawler has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(statpost->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                if (statpost)
-                        ctx->buf = *statpost;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    if (statpost) {
+        LOCK(&ctx->lock);
+        ctx->buf = *statpost;
+        UNLOCK(&ctx->lock);
+    }
 
 out:
-        QUOTA_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
-                            statpost, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+quota_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
-        int32_t        ret   = -1;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    int32_t ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto unwind;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc,
-                    stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, quota_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->setattr, loc,
-                         stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost, dict_t *xdata)
+quota_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
-
-        if (op_ret < 0) {
-                goto out;
-        }
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        local = frame->local;
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO ("quota", local, out);
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (statpost->ia_type)) {
-                  gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                                " (%s). If quota is not enabled recently and "
-                                "crawler has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(statpost->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *statpost;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *statpost;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, statpre,
-                            statpost, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+quota_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_fsetattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr, fd,
-                    stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, quota_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->fsetattr, fd,
-                         stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+quota_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        int32_t            ret    = -1;
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                goto unwind;
-        }
-
-        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_INODE_CTX_GET_FAILED,
-                        "cannot create quota context in "
-                        "inode(gfid:%s)", uuid_utoa (inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-
-                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
-                                             local->loc.parent->gfid);
-                if (dentry == NULL) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                Q_MSG_ENOMEM, "cannot create a new dentry "
-                                "(name:%s) for inode(gfid:%s)", local->loc.name,
-				uuid_utoa (local->loc.inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unlock;
-                }
-        }
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        goto unwind;
+    }
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "cannot create quota context in "
+               "inode(gfid:%s)",
+               uuid_utoa(inode->gfid));
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+
+        dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                    local->loc.parent->gfid);
+        if (dentry == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "cannot create a new dentry "
+                   "(name:%s) for inode(gfid:%s)",
+                   local->loc.name, uuid_utoa(local->loc.inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unlock;
+        }
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 unwind:
-        QUOTA_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
 }
 
-
 int
-quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+quota_mknod_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
-
-        local = frame->local;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
+    local = frame->local;
 
-        priv = this->private;
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_mknod_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod, loc,
-                    mode, rdev, umask, xdata);
+    STACK_WIND(frame, quota_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
+    return 0;
 }
 
-
 int
-quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dev_t rdev, mode_t umask, dict_t *xdata)
+quota_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        quota_priv_t  *priv     = NULL;
-        int32_t        ret      = -1;
-        quota_local_t *local    = NULL;
-        call_stub_t   *stub     = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-        QUOTA_WIND_FOR_INTERNAL_FOP (xdata, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-			Q_MSG_ENOMEM, "loc_copy failed");
-                goto err;
-        }
-
-        stub = fop_mknod_stub (frame, quota_mknod_helper, loc, mode, rdev,
-                               umask, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+    QUOTA_WIND_FOR_INTERNAL_FOP(xdata, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    stub = fop_mknod_stub(frame, quota_mknod_helper, loc, mode, rdev, umask,
+                          xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        local->stub = stub;
+        local->delta = 0;
+        local->object_delta = 1;
+    }
+    UNLOCK(&local->lock);
 
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                local->stub = stub;
-                local->delta = 0;
-                local->object_delta = 1;
-        }
-        UNLOCK (&local->lock);
-
-        quota_check_limit (frame, loc->parent, this);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                            NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->mknod, loc,
-                         mode, rdev, umask, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, rdev, umask, xdata);
+    return 0;
 }
 
 int
-quota_setxattr_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+quota_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
-        int                ret   = 0;
-
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        ret = quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
-        if ((ret < 0) || (ctx == NULL)) {
-                op_errno = -1;
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->hard_lim = local->limit.hl;
-                ctx->soft_lim = local->limit.sl;
-                ctx->object_hard_lim = local->object_limit.hl;
-                ctx->object_soft_lim = local->object_limit.sl;
-        }
-        UNLOCK (&ctx->lock);
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    ret = quota_inode_ctx_get(local->loc.inode, this, &ctx, 1);
+    if ((ret < 0) || (ctx == NULL)) {
+        op_errno = -1;
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->hard_lim = local->limit.hl;
+        ctx->soft_lim = local->limit.sl;
+        ctx->object_hard_lim = local->object_limit.hl;
+        ctx->object_soft_lim = local->object_limit.sl;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
+quota_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int flags, dict_t *xdata)
 {
-        quota_priv_t   *priv                    = NULL;
-        int             op_errno                = EINVAL;
-        int             op_ret                  = -1;
-        int64_t         hard_lim                = -1;
-        int64_t         soft_lim                = -1;
-        int64_t         object_hard_limit       = -1;
-        int64_t         object_soft_limit       = -1;
-        quota_local_t  *local                   = NULL;
-        gf_boolean_t    internal_fop            = _gf_false;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        if (xdata && dict_get (xdata, GLUSTERFS_INTERNAL_FOP_KEY))
-                internal_fop = _gf_true;
-
-        if (frame->root->pid >= 0 && internal_fop == _gf_false) {
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                           op_errno, err);
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.pgfid*", dict, op_errno,
-                                           err);
-        }
-
-        quota_get_limits (this, dict, &hard_lim, &soft_lim, &object_hard_limit,
-                          &object_soft_limit);
-
-        if (hard_lim > 0 || object_hard_limit > 0) {
-                local = quota_local_new ();
-                if (local == NULL) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                frame->local = local;
-                loc_copy (&local->loc, loc);
-        }
-
-        if (hard_lim > 0) {
-                local->limit.hl = hard_lim;
-                local->limit.sl = soft_lim;
-        }
-
-        if (object_hard_limit > 0) {
-                local->object_limit.hl = object_hard_limit;
-                local->object_limit.sl = object_soft_limit;
+    quota_priv_t *priv = NULL;
+    int op_errno = EINVAL;
+    int op_ret = -1;
+    int64_t hard_lim = -1;
+    int64_t soft_lim = -1;
+    int64_t object_hard_limit = -1;
+    int64_t object_soft_limit = -1;
+    quota_local_t *local = NULL;
+    gf_boolean_t internal_fop = _gf_false;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    if (xdata && dict_get_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY))
+        internal_fop = _gf_true;
+
+    if (frame->root->pid >= 0 && internal_fop == _gf_false) {
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.quota*", dict, op_errno,
+                                  err);
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.pgfid*", dict, op_errno, err);
+    }
+
+    quota_get_limits(this, dict, &hard_lim, &soft_lim, &object_hard_limit,
+                     &object_soft_limit);
+
+    if (hard_lim > 0 || object_hard_limit > 0) {
+        local = quota_local_new();
+        if (local == NULL) {
+            op_errno = ENOMEM;
+            goto err;
         }
-
-        STACK_WIND (frame, quota_setxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, loc,
-                    dict, flags, xdata);
-        return 0;
+        frame->local = local;
+        loc_copy(&local->loc, loc);
+    }
+
+    if (hard_lim > 0) {
+        local->limit.hl = hard_lim;
+        local->limit.sl = soft_lim;
+    }
+
+    if (object_hard_limit > 0) {
+        local->object_limit.hl = object_hard_limit;
+        local->object_limit.sl = object_soft_limit;
+    }
+
+    STACK_WIND(frame, quota_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->setxattr, loc,
-                         dict, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
 }
 
 int
-quota_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+quota_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *xdata)
 {
-        quota_inode_ctx_t *ctx   = NULL;
-        quota_local_t     *local = NULL;
-
-        if (op_ret < 0)
-                goto out;
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        op_ret = quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
-        if ((op_ret < 0) || (ctx == NULL)) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->hard_lim = local->limit.hl;
-                ctx->soft_lim = local->limit.sl;
-                ctx->object_hard_lim = local->object_limit.hl;
-                ctx->object_soft_lim = local->object_limit.sl;
-        }
-        UNLOCK (&ctx->lock);
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = NULL;
+
+    if (op_ret < 0)
+        goto out;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_ret = quota_inode_ctx_get(local->loc.inode, this, &ctx, 1);
+    if ((op_ret < 0) || (ctx == NULL)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->hard_lim = local->limit.hl;
+        ctx->soft_lim = local->limit.sl;
+        ctx->object_hard_lim = local->object_limit.hl;
+        ctx->object_soft_lim = local->object_limit.sl;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int flags, dict_t *xdata)
+quota_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int flags, dict_t *xdata)
 {
-        quota_priv_t    *priv                   = NULL;
-        int32_t         op_ret                  = -1;
-        int32_t         op_errno                = EINVAL;
-        quota_local_t   *local                  = NULL;
-        int64_t         hard_lim                = -1;
-        int64_t         soft_lim                = -1;
-        int64_t         object_hard_limit       = -1;
-        int64_t         object_soft_limit       = -1;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        if (0 <= frame->root->pid) {
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*",
-                                            dict, op_errno, err);
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.pgfid*", dict,
-                                            op_errno, err);
-        }
-
-        quota_get_limits (this, dict, &hard_lim, &soft_lim, &object_hard_limit,
-                          &object_soft_limit);
-
-        if (hard_lim > 0 || object_hard_limit > 0) {
-                local = quota_local_new ();
-                if (local == NULL) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                frame->local = local;
-                local->loc.inode = inode_ref (fd->inode);
-        }
-
-        if (hard_lim > 0) {
-                local->limit.hl = hard_lim;
-                local->limit.sl = soft_lim;
-        }
-
-        if (object_hard_limit > 0) {
-                local->object_limit.hl = object_hard_limit;
-                local->object_limit.sl = object_soft_limit;
+    quota_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    quota_local_t *local = NULL;
+    int64_t hard_lim = -1;
+    int64_t soft_lim = -1;
+    int64_t object_hard_limit = -1;
+    int64_t object_soft_limit = -1;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    if (0 <= frame->root->pid) {
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.quota*", dict, op_errno,
+                                  err);
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.pgfid*", dict, op_errno, err);
+    }
+
+    quota_get_limits(this, dict, &hard_lim, &soft_lim, &object_hard_limit,
+                     &object_soft_limit);
+
+    if (hard_lim > 0 || object_hard_limit > 0) {
+        local = quota_local_new();
+        if (local == NULL) {
+            op_errno = ENOMEM;
+            goto err;
         }
-
-        STACK_WIND (frame, quota_fsetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr, fd,
-                    dict, flags, xdata);
-        return 0;
+        frame->local = local;
+        local->loc.inode = inode_ref(fd->inode);
+    }
+
+    if (hard_lim > 0) {
+        local->limit.hl = hard_lim;
+        local->limit.sl = soft_lim;
+    }
+
+    if (object_hard_limit > 0) {
+        local->object_limit.hl = object_hard_limit;
+        local->object_limit.sl = object_soft_limit;
+    }
+
+    STACK_WIND(frame, quota_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fsetxattr, fd,
-                         dict, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-quota_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quota_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+quota_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t         op_errno = EINVAL;
+    quota_priv_t *priv = NULL;
+    int32_t op_errno = EINVAL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        VALIDATE_OR_GOTO (this, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        /* all quota xattrs can be cleaned up by doing setxattr on special key.
-         * Hence its ok that we don't allow removexattr on quota keys here.
-         */
-        if (frame->root->pid >= 0) {
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.quota*",
-                                         name, op_errno, err);
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.pgfid*", name,
-                                         op_errno, err);
-        }
+    /* all quota xattrs can be cleaned up by doing setxattr on special key.
+     * Hence its ok that we don't allow removexattr on quota keys here.
+     */
+    if (frame->root->pid >= 0) {
+        GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.quota*", name, op_errno,
+                                err);
+        GF_IF_NATIVE_XATTR_GOTO("trusted.pgfid*", name, op_errno, err);
+    }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        STACK_WIND (frame, quota_removexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, quota_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->removexattr,
-                         loc, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-quota_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quota_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_fremovexattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name, dict_t *xdata)
+quota_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        if (frame->root->pid >= 0) {
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.quota*",
-                                         name, op_errno, err);
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.pgfid*", name,
-                                         op_errno, err);
-        }
-        STACK_WIND (frame, quota_fremovexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    quota_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    if (frame->root->pid >= 0) {
+        GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.quota*", name, op_errno,
+                                err);
+        GF_IF_NATIVE_XATTR_GOTO("trusted.pgfid*", name, op_errno, err);
+    }
+    STACK_WIND(frame, quota_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fremovexattr,
-                         fd, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                  dict_t *xdata)
+quota_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
 {
-	inode_t           *inode        = NULL;
-        uint64_t           value        = 0;
-	int64_t            usage        = -1;
-	int64_t            avail        = -1;
-        int64_t            blocks       = 0;
-        quota_inode_ctx_t *ctx          = NULL;
-        int                ret          = 0;
-
-        inode = cookie;
-
-        /* This fop will fail mostly in case of client disconnect's,
-         * which is already logged. Hence, not logging here */
-        if (op_ret == -1)
-                goto unwind;
-	/*
-	 * We should never get here unless quota_statfs (below) sent us a
-	 * cookie, and it would only do so if the value was non-NULL.  This
-	 * check is therefore just routine defensive coding.
-	 */
-
-        GF_VALIDATE_OR_GOTO ("quota", inode, unwind);
-
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        if (!ctx || ctx->hard_lim <= 0)
-                goto unwind;
-
-        { /* statfs is adjusted in this code block */
-                usage = (ctx->size) / buf->f_bsize;
-
-                blocks = ctx->hard_lim / buf->f_bsize;
-                buf->f_blocks = blocks;
-
-                avail = buf->f_blocks - usage;
-                avail = max (avail, 0);
-
-                buf->f_bfree = avail;
-                /*
-                 * We have to assume that the total assigned quota
-                 * won't cause us to dip into the reserved space,
-                 * because dealing with the overcommitted cases is
-                 * just too hairy (especially when different bricks
-                 * might be using different reserved percentages and
-                 * such).
-                 */
-                buf->f_bavail = buf->f_bfree;
-        }
+    inode_t *inode = NULL;
+    uint64_t value = 0;
+    int64_t usage = -1;
+    int64_t avail = -1;
+    int64_t blocks = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    inode = cookie;
+
+    /* This fop will fail mostly in case of client disconnect,
+     * which is already logged. Hence, not logging here */
+    if (op_ret == -1)
+        goto unwind;
+    /*
+     * We should never get here unless quota_statfs (below) sent us a
+     * cookie, and it would only do so if the value was non-NULL.  This
+     * check is therefore just routine defensive coding.
+     */
+
+    GF_VALIDATE_OR_GOTO("quota", inode, unwind);
+
+    inode_ctx_get(inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    if (!ctx || ctx->hard_lim <= 0)
+        goto unwind;
+
+    { /* statfs is adjusted in this code block */
+        usage = (ctx->size) / buf->f_bsize;
+
+        blocks = ctx->hard_lim / buf->f_bsize;
+        buf->f_blocks = blocks;
+
+        avail = buf->f_blocks - usage;
+        avail = max(avail, 0);
+
+        buf->f_bfree = avail;
+        /*
+         * We have to assume that the total assigned quota
+         * won't cause us to dip into the reserved space,
+         * because dealing with the overcommitted cases is
+         * just too hairy (especially when different bricks
+         * might be using different reserved percentages and
+         * such).
+         */
+        buf->f_bavail = buf->f_bfree;
+    }
 
-        xdata = xdata ? dict_ref(xdata) : dict_new();
-        if (!xdata)
-                goto unwind;
+    xdata = xdata ? dict_ref(xdata) : dict_new();
+    if (!xdata)
+        goto unwind;
 
-        ret = dict_set_int8 (xdata, "quota-deem-statfs", 1);
-        if (-1 == ret)
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        Q_MSG_ENOMEM, "Dict set failed, deem-statfs option may "
-                        "have no effect");
+    ret = dict_set_int8(xdata, "quota-deem-statfs", 1);
+    if (-1 == ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "Dict set failed, deem-statfs option may "
+               "have no effect");
 
 unwind:
-        QUOTA_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
+    QUOTA_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_statfs_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     dict_t *xdata)
+quota_statfs_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-        quota_local_t   *local          = frame->local;
-        int              op_errno       = EINVAL;
+    quota_local_t *local = frame->local;
+    int op_errno = EINVAL;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, err);
+    GF_VALIDATE_OR_GOTO("quota", local, err);
 
-        if (-1 == local->op_ret) {
-                op_errno = local->op_errno;
-                goto err;
-        }
+    if (-1 == local->op_ret) {
+        op_errno = local->op_errno;
+        goto err;
+    }
 
-        STACK_WIND_COOKIE (frame, quota_statfs_cbk, local->inode,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->statfs, loc, xdata);
-        return 0;
+    STACK_WIND_COOKIE(frame, quota_statfs_cbk, local->inode, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    QUOTA_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quota_statfs_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, inode_t *inode,
-                           struct iatt *buf, dict_t *xdata,
-                           struct iatt *postparent)
+quota_statfs_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata,
+                          struct iatt *postparent)
 {
-        quota_local_t     *local      = NULL;
-        int32_t            ret        = 0;
-        quota_inode_ctx_t *ctx        = NULL;
-        uint64_t           value      = 0;
-        data_t            *data       = NULL;
-        quota_meta_t       size       = {0,};
-
-        local = frame->local;
-
-        if (op_ret < 0)
-                goto resume;
-
-        GF_ASSERT (local);
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, resume, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, resume, op_errno,
-                                        EINVAL);
-
-        ret = inode_ctx_get (local->validate_loc.inode, this, &value);
-
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-			Q_MSG_INODE_CTX_GET_FAILED,
-			"quota context is not present in inode (gfid:%s)",
-                        uuid_utoa (local->validate_loc.inode->gfid));
-                op_errno = EINVAL;
-                goto resume;
-        }
-
-        ret = quota_dict_get_meta (xdata, QUOTA_SIZE_KEY, &size);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-			Q_MSG_SIZE_KEY_MISSING, "size key not present in "
-                        "dict");
-                op_errno = EINVAL;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->size = size.size;
-                ctx->file_count = size.file_count;
-                ctx->dir_count = size.dir_count;
-                gettimeofday (&ctx->tv, NULL);
-        }
-        UNLOCK (&ctx->lock);
+    quota_local_t *local = NULL;
+    int32_t ret = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
+    quota_meta_t size = {
+        0,
+    };
+
+    local = frame->local;
+
+    if (op_ret < 0)
+        goto resume;
+
+    GF_ASSERT(local);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("quota", this, resume, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, xdata, resume, op_errno, EINVAL);
+
+    ret = inode_ctx_get(local->validate_loc.inode, this, &value);
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context is not present in inode (gfid:%s)",
+               uuid_utoa(local->validate_loc.inode->gfid));
+        op_errno = EINVAL;
+        goto resume;
+    }
+
+    ret = quota_dict_get_meta(xdata, QUOTA_SIZE_KEY, SLEN(QUOTA_SIZE_KEY),
+                              &size);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_SIZE_KEY_MISSING,
+               "size key not present in "
+               "dict");
+        op_errno = EINVAL;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->size = size.size;
+        ctx->validate_time = gf_time();
+        ctx->file_count = size.file_count;
+        ctx->dir_count = size.dir_count;
+    }
+    UNLOCK(&ctx->lock);
 
 resume:
-        quota_link_count_decrement (frame);
-        return 0;
+    local->op_errno = op_errno;
+    quota_link_count_decrement(frame);
+    return 0;
 }
 
 void
-quota_get_limit_dir_continuation (struct list_head *parents, inode_t *inode,
-                                  int32_t op_ret, int32_t op_errno, void *data)
+quota_get_limit_dir_continuation(struct list_head *parents, inode_t *inode,
+                                 int32_t op_ret, int32_t op_errno, void *data)
 {
-        call_frame_t   *frame        = NULL;
-        xlator_t       *this         = NULL;
-        quota_local_t  *local        = NULL;
-        quota_dentry_t *entry        = NULL;
-        inode_t        *parent       = NULL;
-
-        frame = data;
-        local = frame->local;
-        this = THIS;
-
-        if ((op_ret < 0) || list_empty (parents)) {
-                if (op_ret >= 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, EIO,
-                                Q_MSG_ANCESTRY_BUILD_FAILED,
-                                "Couldn't build ancestry for inode (gfid:%s). "
-                                "Without knowing ancestors till root, quota "
-                                "cannot be enforced. "
-                                "Hence, failing fop with EIO",
-                                uuid_utoa (inode->gfid));
-                        op_errno = EIO;
-                }
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    quota_dentry_t *entry = NULL;
+    inode_t *parent = NULL;
 
-                quota_handle_validate_error (frame, -1, op_errno);
-                goto out;
+    frame = data;
+    this = THIS;
+
+    if ((op_ret < 0) || list_empty(parents)) {
+        if (op_ret >= 0) {
+            gf_msg(this->name, GF_LOG_WARNING, EIO, Q_MSG_ANCESTRY_BUILD_FAILED,
+                   "Couldn't build ancestry for inode (gfid:%s). "
+                   "Without knowing ancestors till root, quota "
+                   "cannot be enforced. "
+                   "Hence, failing fop with EIO",
+                   uuid_utoa(inode->gfid));
+            op_errno = EIO;
         }
 
-        entry = list_entry (parents, quota_dentry_t, next);
-        parent = inode_find (inode->table, entry->par);
+        quota_handle_validate_error(frame, -1, op_errno);
+        goto out;
+    }
+
+    entry = list_entry(parents, quota_dentry_t, next);
+    parent = inode_find(inode->table, entry->par);
 
-        quota_get_limit_dir (frame, parent, this);
+    quota_get_limit_dir(frame, parent, this);
 
-        inode_unref (parent);
+    inode_unref(parent);
 out:
-        return;
+    return;
 }
 
 void
-quota_statfs_continue (call_frame_t *frame, xlator_t *this, inode_t *inode)
+quota_statfs_continue(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        call_stub_t     *stub           = NULL;
-        quota_local_t   *local          = frame->local;
-        int              ret            = -1;
-
-        LOCK (&local->lock);
-        {
-                local->inode = inode_ref (inode);
-        }
-        UNLOCK (&local->lock);
-
-        ret = quota_validate (frame, local->inode, this,
-                              quota_statfs_validate_cbk);
-        if (0 > ret)
-                quota_handle_validate_error (frame, -1, -ret);
+    quota_local_t *local = frame->local;
+    int ret = -1;
+
+    LOCK(&local->lock);
+    {
+        local->inode = inode_ref(inode);
+    }
+    UNLOCK(&local->lock);
+
+    ret = quota_validate(frame, local->inode, this, quota_statfs_validate_cbk);
+    if (0 > ret)
+        quota_handle_validate_error(frame, -1, -ret);
 }
 
 void
-quota_get_limit_dir (call_frame_t *frame, inode_t *cur_inode, xlator_t *this)
+quota_get_limit_dir(call_frame_t *frame, inode_t *cur_inode, xlator_t *this)
 {
-        inode_t                *inode           = NULL;
-        inode_t                *parent          = NULL;
-        uint64_t                value           = 0;
-        quota_inode_ctx_t      *ctx             = NULL;
-        int                     ret             = -1;
-        quota_local_t          *local           = frame->local;
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    uint64_t value = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = frame->local;
 
-        if (!cur_inode)
-                goto out;
-
-        inode = inode_ref (cur_inode);
-        while (inode) {
-                value = 0;
-                inode_ctx_get (inode, this, &value);
+    if (!cur_inode)
+        goto out;
 
-                if (value) {
-                        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-                        if (ctx->hard_lim > 0)
-                                break;
-                }
+    inode = inode_ref(cur_inode);
+    while (inode) {
+        value = 0;
+        inode_ctx_get(inode, this, &value);
 
-                if (__is_root_gfid (inode->gfid))
-                        goto off;
+        if (value) {
+            ctx = (quota_inode_ctx_t *)(unsigned long)value;
+            if (ctx->hard_lim > 0)
+                break;
+        }
 
-                parent = inode_parent (inode, 0, NULL);
-                if (!parent) {
-                        ret = quota_build_ancestry
-                               (inode, quota_get_limit_dir_continuation,
-                                (void *)frame);
-                        goto out;
-                }
+        if (__is_root_gfid(inode->gfid))
+            goto off;
 
-                inode_unref (inode);
-                inode = parent;
+        parent = inode_parent(inode, 0, NULL);
+        if (!parent) {
+            (void)quota_build_ancestry(inode, quota_get_limit_dir_continuation,
+                                       frame);
+            goto out;
         }
 
-        quota_statfs_continue (frame, this, inode);
-        inode_unref (inode);
-        return;
+        inode_unref(inode);
+        inode = parent;
+    }
+
+    quota_statfs_continue(frame, this, inode);
+    inode_unref(inode);
+    return;
 
 off:
-        gf_msg_debug (this->name, 0,
-                      "No limit set on the inode or it's parents.");
+    gf_msg_debug(this->name, 0, "No limit set on the inode or it's parents.");
 
-        QUOTA_STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                               FIRST_CHILD(this)->fops->statfs,
-                               &local->loc, local->xdata);
+    QUOTA_STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->statfs, &local->loc,
+                          local->xdata);
 out:
-        inode_unref (inode);
+    inode_unref(inode);
 
-        return;
+    return;
 }
 
 int32_t
-quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+quota_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int              op_errno            = 0;
-        int              ret                 = -1;
-        int8_t           ignore_deem_statfs  = 0;
-        quota_priv_t    *priv                = NULL;
-        quota_local_t   *local               = NULL;
-        call_stub_t     *stub                = NULL;
+    int op_errno = 0;
+    int ret = -1;
+    int8_t ignore_deem_statfs = 0;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
-        GF_ASSERT (loc);
+    priv = this->private;
+    GF_ASSERT(loc);
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        ret = dict_get_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS,
-                             &ignore_deem_statfs);
-        ret = 0;
+    ret = dict_get_int8(xdata, GF_INTERNAL_IGNORE_DEEM_STATFS,
+                        &ignore_deem_statfs);
+    ret = 0;
 
-        if (ignore_deem_statfs)
-                goto off;
+    if (ignore_deem_statfs)
+        goto off;
 
-	if (priv->consider_statfs && loc->inode) {
-                local = quota_local_new ();
-                if (!local) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                frame->local = local;
+    if (priv->consider_statfs && loc->inode) {
+        local = quota_local_new();
+        if (!local) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        frame->local = local;
 
-                ret = loc_copy (&local->loc, loc);
-                if (-1 == ret) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+        ret = loc_copy(&local->loc, loc);
+        if (-1 == ret) {
+            op_errno = ENOMEM;
+            goto err;
+        }
 
-                if (xdata)
-                        local->xdata = dict_ref (xdata);
+        if (xdata)
+            local->xdata = dict_ref(xdata);
 
-                stub = fop_statfs_stub (frame, quota_statfs_helper,
-                                        &local->loc, local->xdata);
-                if (!stub) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+        stub = fop_statfs_stub(frame, quota_statfs_helper, &local->loc,
+                               local->xdata);
+        if (!stub) {
+            op_errno = ENOMEM;
+            goto err;
+        }
 
-                LOCK (&local->lock);
-                {
-                        local->link_count = 1;
-                        local->stub = stub;
-                }
-                UNLOCK (&local->lock);
+        LOCK(&local->lock);
+        {
+            local->link_count = 1;
+            local->stub = stub;
+        }
+        UNLOCK(&local->lock);
 
-                quota_get_limit_dir (frame, loc->inode, this);
+        quota_get_limit_dir(frame, loc->inode, this);
 
-                return 0;
-	}
+        return 0;
+    }
 
-        /*
-         * We have to make sure that we never get to quota_statfs_cbk
-         * with a cookie that points to something other than an inode,
-         * which is exactly what would happen with STACK_UNWIND using
-         * that as a callback.  Therefore, use default_statfs_cbk in
-         * this case instead.
-         *
-         * Also if the option deem-statfs is not set to "on" don't
-         * bother calculating quota limit on / in statfs_cbk.
-         */
-        if (priv->consider_statfs)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Missing inode, can't adjust for quota");
+    /*
+     * We have to make sure that we never get to quota_statfs_cbk
+     * with a cookie that points to something other than an inode,
+     * which is exactly what would happen with STACK_UNWIND using
+     * that as a callback.  Therefore, use default_statfs_cbk in
+     * this case instead.
+     *
+     * Also if the option deem-statfs is not set to "on" don't
+     * bother calculating quota limit on / in statfs_cbk.
+     */
+    if (priv->consider_statfs)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Missing inode, can't adjust for quota");
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->statfs, loc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs,
+                    loc, xdata);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    QUOTA_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-quota_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, gf_dirent_t *entries,
-                    dict_t *xdata)
+quota_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
 {
-        gf_dirent_t   *entry = NULL;
-        quota_local_t *local = NULL;
-        loc_t          loc   = {0, };
+    gf_dirent_t *entry = NULL;
+    quota_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
 
-        if (op_ret <= 0)
-                goto unwind;
+    if (op_ret <= 0)
+        goto unwind;
 
-        local = frame->local;
+    local = frame->local;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if ((strcmp (entry->d_name, ".") == 0)  ||
-                    (strcmp (entry->d_name, "..") == 0) ||
-                    entry->inode == NULL)
-                        continue;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0) || entry->inode == NULL)
+            continue;
 
-                gf_uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
-                loc.inode = inode_ref (entry->inode);
-                loc.parent = inode_ref (local->loc.inode);
-                gf_uuid_copy (loc.pargfid, loc.parent->gfid);
-                loc.name = entry->d_name;
+        gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid);
+        loc.inode = inode_ref(entry->inode);
+        loc.parent = inode_ref(local->loc.inode);
+        gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+        loc.name = entry->d_name;
 
-                quota_fill_inodectx (this, entry->inode, entry->dict,
-                                     &loc, &entry->d_stat, &op_errno);
+        quota_fill_inodectx(this, entry->inode, entry->dict, &loc,
+                            &entry->d_stat, &op_errno);
 
-                loc_wipe (&loc);
-        }
+        loc_wipe(&loc);
+    }
 
 unwind:
-        QUOTA_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, xdata);
+    QUOTA_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-quota_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset, dict_t *dict)
+quota_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *dict)
 {
-        quota_priv_t  *priv     = NULL;
-        int            ret      = 0;
-        gf_boolean_t   new_dict = _gf_false;
-        quota_local_t *local    = NULL;
+    quota_priv_t *priv = NULL;
+    int ret = 0;
+    gf_boolean_t new_dict = _gf_false;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
+    local = quota_local_new();
 
-        if (local == NULL) {
-                goto err;
-        }
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        if (dict == NULL) {
-                dict = dict_new ();
-                new_dict = _gf_true;
-        }
+    if (dict == NULL) {
+        dict = dict_new();
+        new_dict = _gf_true;
+    }
 
-        if (dict) {
-                ret = dict_set_int8 (dict, QUOTA_LIMIT_KEY, 1);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-				Q_MSG_ENOMEM,
-				"dict set of key for hard-limit");
-                        goto err;
-                }
+    if (dict) {
+        ret = dict_set_int8(dict, QUOTA_LIMIT_KEY, 1);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "dict set of key for hard-limit");
+            goto err;
         }
+    }
 
-        if (dict) {
-                ret = dict_set_int8 (dict, QUOTA_LIMIT_OBJECTS_KEY, 1);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                Q_MSG_ENOMEM, "dict set of key for hard-limit "
-                                "failed");
-                        goto err;
-                }
+    if (dict) {
+        ret = dict_set_int8(dict, QUOTA_LIMIT_OBJECTS_KEY, 1);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "dict set of key for hard-limit "
+                   "failed");
+            goto err;
         }
+    }
 
-        STACK_WIND (frame, quota_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp, fd,
-                    size, offset, dict);
+    STACK_WIND(frame, quota_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
 
-        if (new_dict) {
-                dict_unref (dict);
-        }
+    if (new_dict) {
+        dict_unref(dict);
+    }
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, EINVAL, NULL, NULL);
+    STACK_UNWIND_STRICT(readdirp, frame, -1, EINVAL, NULL, NULL);
 
-        if (new_dict) {
-                dict_unref (dict);
-        }
+    if (new_dict) {
+        dict_unref(dict);
+    }
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readdirp, fd,
-                         size, offset, dict);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
+    return 0;
 }
 
 int32_t
@@ -4752,597 +4644,693 @@ quota_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                     struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t                  ret            = 0;
-        uint64_t                 ctx_int        = 0;
-        quota_inode_ctx_t       *ctx            = NULL;
-        quota_local_t           *local          = NULL;
-
-        local = frame->local;
-
-        if ((op_ret < 0) || (local == NULL)) {
-                goto out;
-        }
-
-        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_INODE_CTX_GET_FAILED,
-			"%s: failed to get the context", local->loc.path);
-                goto out;
-        }
-
-        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
-
-        if (ctx == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-			Q_MSG_INODE_CTX_GET_FAILED,
-			"quota context not set in %s (gfid:%s)",
-                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    int32_t ret = 0;
+    uint64_t ctx_int = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = NULL;
+
+    local = frame->local;
+
+    if ((op_ret < 0) || (local == NULL)) {
+        goto out;
+    }
+
+    ret = inode_ctx_get(local->loc.inode, this, &ctx_int);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "%s: failed to get the context", local->loc.path);
+        goto out;
+    }
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)ctx_int;
+
+    if (ctx == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context not set in %s (gfid:%s)", local->loc.path,
+               uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
+    QUOTA_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_fallocate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        int32_t mode, off_t offset, size_t len, dict_t *xdata)
+quota_fallocate_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       int32_t mode, off_t offset, size_t len, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
-
-        local = frame->local;
-
-        GF_VALIDATE_OR_GOTO ("quota", local, unwind);
-
-        priv = this->private;
-
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                if (op_errno == ENOENT || op_errno == ESTALE) {
-                        /* We may get ENOENT/ESTALE in case of below scenario
-                         *     fd = open file.txt
-                         *     unlink file.txt
-                         *     fallocate on fd
-                         * Here build_ancestry can fail as the file is removed.
-                         * For now ignore ENOENT/ESTALE on active fd
-                         * We need to re-visit this code once we understand
-                         * how other file-system behave in this scenario
-                         */
-                        gf_msg_debug (this->name, 0, "quota enforcer failed "
-                                      "with ENOENT/ESTALE on %s, cannot check "
-                                      "quota limits and allowing fallocate",
-                                      uuid_utoa (fd->inode->gfid));
-                } else {
-                        goto unwind;
-                }
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
+
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        if (op_errno == ENOENT || op_errno == ESTALE) {
+            /* We may get ENOENT/ESTALE in case of below scenario
+             *     fd = open file.txt
+             *     unlink file.txt
+             *     fallocate on fd
+             * Here build_ancestry can fail as the file is removed.
+             * For now ignore ENOENT/ESTALE on active fd
+             * We need to re-visit this code once we understand
+             * how other file-system behave in this scenario
+             */
+            gf_msg_debug(this->name, 0,
+                         "quota enforcer failed "
+                         "with ENOENT/ESTALE on %s, cannot check "
+                         "quota limits and allowing fallocate",
+                         uuid_utoa(fd->inode->gfid));
+        } else {
+            goto unwind;
         }
+    }
 
-        STACK_WIND (frame, quota_fallocate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		    xdata);
-        return 0;
+    STACK_WIND(frame, quota_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
 quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		off_t offset, size_t len, dict_t *xdata)
+                off_t offset, size_t len, dict_t *xdata)
 {
-        int32_t            ret         = -1, op_errno = EINVAL;
-        int32_t            parents     = 0;
-        int32_t            fail_count  = 0;
-        quota_local_t     *local       = NULL;
-        quota_inode_ctx_t *ctx         = NULL;
-        quota_priv_t      *priv        = NULL;
-        quota_dentry_t    *dentry      = NULL;
-        quota_dentry_t    *tmp         = NULL;
-        call_stub_t       *stub        = NULL;
-        struct list_head   head        = {0, };
-        inode_t           *par_inode   = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        INIT_LIST_HEAD (&head);
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
-
-        frame->local = local;
-        local->loc.inode = inode_ref (fd->inode);
-
-        ret = quota_inode_ctx_get (fd->inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_msg_debug (this->name, 0, "quota context is NULL on inode"
-                              " (%s). If quota is not enabled recently and "
-                              "crawler has finished crawling, its an error",
-                              uuid_utoa (local->loc.inode->gfid));
-        }
-
-        stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode,
-                                  offset, len, xdata);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
+    int32_t op_errno = EINVAL;
+    int32_t parents = 0;
+    int32_t fail_count = 0;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_priv_t *priv = NULL;
+    quota_dentry_t *dentry = NULL;
+    quota_dentry_t *tmp = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head head = {
+        0,
+    };
+    inode_t *par_inode = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, unwind);
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    INIT_LIST_HEAD(&head);
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO("quota", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->loc.inode = inode_ref(fd->inode);
+
+    (void)quota_inode_ctx_get(fd->inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+    }
+
+    stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode, offset,
+                              len, xdata);
+    if (stub == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, unwind);
+
+    parents = quota_add_parents_from_ctx(ctx, &head);
+    if (parents == -1) {
+        op_errno = errno;
+        goto unwind;
+    }
+
+    /*
+     * Note that by using len as the delta we're assuming the range from
+     * offset to offset+len has not already been allocated. This can result
+     * in ENOSPC errors attempting to allocate an already allocated range.
+     */
+    local->delta = len;
+    local->object_delta = 0;
+    local->stub = stub;
+    local->link_count = parents;
+
+    if (parents == 0) {
+        local->link_count = 1;
+        quota_check_limit(frame, fd->inode, this);
+    } else {
+        list_for_each_entry_safe(dentry, tmp, &head, next)
+        {
+            par_inode = do_quota_check_limit(frame, fd->inode, this, dentry,
+                                             _gf_false);
+            if (par_inode == NULL) {
+                /* remove stale entry from inode_ctx */
+                quota_dentry_del(ctx, dentry->name, dentry->par);
+                parents--;
+                fail_count++;
+            } else {
+                inode_unref(par_inode);
+            }
+            __quota_dentry_free(dentry);
         }
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
-
-        parents = quota_add_parents_from_ctx (ctx, &head);
-
-	/*
-	 * Note that by using len as the delta we're assuming the range from
-	 * offset to offset+len has not already been allocated. This can result
-	 * in ENOSPC errors attempting to allocate an already allocated range.
-	 */
-        local->delta = len;
-        local->object_delta = 0;
-        local->stub = stub;
-        local->link_count = parents;
-
         if (parents == 0) {
-                local->link_count = 1;
-                quota_check_limit (frame, fd->inode, this);
-        } else {
-                list_for_each_entry_safe (dentry, tmp, &head, next) {
-                        par_inode = do_quota_check_limit (frame, fd->inode,
-                                                          this, dentry,
-                                                          _gf_false);
-                        if (par_inode == NULL) {
-                                /* remove stale entry from inode_ctx */
-                                quota_dentry_del (ctx, dentry->name,
-                                                  dentry->par);
-                                parents--;
-                                fail_count++;
-                        } else {
-                                inode_unref (par_inode);
-                        }
-                        __quota_dentry_free (dentry);
-                }
-
-                if (parents == 0) {
-                        LOCK (&local->lock);
-                        {
-                                local->link_count++;
-                        }
-                        UNLOCK (&local->lock);
-                        quota_check_limit (frame, fd->inode, this);
-                }
+            LOCK(&local->lock);
+            {
+                local->link_count++;
+            }
+            UNLOCK(&local->lock);
+            quota_check_limit(frame, fd->inode, this);
+        }
 
-                while (fail_count != 0) {
-                        quota_link_count_decrement (frame);
-                        fail_count--;
-                }
+        while (fail_count != 0) {
+            quota_link_count_decrement(frame);
+            fail_count--;
         }
+    }
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fallocate, fd, mode, offset,
-                         len, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+                    xdata);
+    return 0;
 }
 
 void
-quota_log_helper (char **usage_str, int64_t cur_size, inode_t *inode,
-                  char **path, struct timeval *cur_time)
+quota_log_helper(char **usage_str, int64_t cur_size, inode_t *inode,
+                 char **path, time_t *cur_time)
 {
-        xlator_t        *this   = THIS;
+    xlator_t *this = THIS;
 
-        if (!usage_str || !inode || !path || !cur_time) {
-                gf_log (this->name, GF_LOG_ERROR, "Received null argument");
-                return;
-        }
+    if (!usage_str || !inode || !path || !cur_time) {
+        gf_log(this->name, GF_LOG_ERROR, "Received null argument");
+        return;
+    }
 
-        *usage_str = gf_uint64_2human_readable (cur_size);
-        if (!(*usage_str))
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
-                        "integer to string conversion failed Reason"
-                        ":\"Cannot allocate memory\"");
+    *usage_str = gf_uint64_2human_readable(cur_size);
+    if (!(*usage_str))
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "integer to string conversion failed Reason"
+               ":\"Cannot allocate memory\"");
 
-        inode_path (inode, NULL, path);
-        if (!(*path))
-                *path = uuid_utoa (inode->gfid);
+    inode_path(inode, NULL, path);
+    if (!(*path))
+        *path = uuid_utoa(inode->gfid);
 
-        gettimeofday (cur_time, NULL);
+    *cur_time = gf_time();
 }
 
 /* Logs if
-*  i.   Usage crossed soft limit
-*  ii.  Usage above soft limit and alert-time elapsed
-*/
+ *  i.   Usage crossed soft limit
+ *  ii.  Usage above soft limit and alert-time elapsed
+ */
 void
-quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
-                 int64_t delta)
+quota_log_usage(xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                int64_t delta)
 {
-        struct timeval           cur_time       = {0,};
-        char                    *usage_str      = NULL;
-        char                    *path           = NULL;
-        int64_t                  cur_size       = 0;
-        quota_priv_t            *priv           = NULL;
+    time_t cur_time = 0;
+    char *usage_str = NULL;
+    char *path = NULL;
+    int64_t cur_size = 0;
+    quota_priv_t *priv = NULL;
 
-        priv = this->private;
-        cur_size = ctx->size + delta;
+    priv = this->private;
+    cur_size = ctx->size + delta;
 
-        if ((ctx->soft_lim <= 0) || cur_size < ctx->soft_lim)
-                return;
+    if ((ctx->soft_lim <= 0) || cur_size < ctx->soft_lim)
+        return;
 
-        /* Usage crossed/reached soft limit */
-        if (DID_REACH_LIMIT (ctx->soft_lim, ctx->size, cur_size)) {
+    /* Usage crossed/reached soft limit */
+    if (DID_REACH_LIMIT(ctx->soft_lim, ctx->size, cur_size)) {
+        quota_log_helper(&usage_str, cur_size, inode, &path, &cur_time);
 
-                quota_log_helper (&usage_str, cur_size, inode,
-                                  &path, &cur_time);
+        gf_msg(this->name, GF_LOG_ALERT, 0, Q_MSG_CROSSED_SOFT_LIMIT,
+               "Usage crossed soft limit: "
+               "%s used by %s",
+               usage_str, path);
 
-                gf_msg (this->name, GF_LOG_ALERT, 0,
-                        Q_MSG_CROSSED_SOFT_LIMIT, "Usage crossed soft limit: "
-                        "%s used by %s", usage_str, path);
-                ctx->prev_log = cur_time;
-        }
-        /* Usage is above soft limit */
-        else if (cur_size > ctx->soft_lim &&
-                           quota_timeout (&ctx->prev_log, priv->log_timeout)) {
+        gf_event(EVENT_QUOTA_CROSSED_SOFT_LIMIT,
+                 "Usage=%s;volume=%s;"
+                 "path=%s",
+                 usage_str, priv->volume_uuid, path);
 
-                quota_log_helper (&usage_str, cur_size, inode,
-                                  &path, &cur_time);
+        ctx->prev_log_time = cur_time;
 
-                gf_msg (this->name, GF_LOG_ALERT, 0, Q_MSG_CROSSED_SOFT_LIMIT,
-                        "Usage is above soft limit: %s used by %s",
-                        usage_str, path);
-                ctx->prev_log = cur_time;
-        }
+    }
+    /* Usage is above soft limit */
+    else if (cur_size > ctx->soft_lim &&
+             quota_timeout(ctx->prev_log_time, priv->log_timeout)) {
+        quota_log_helper(&usage_str, cur_size, inode, &path, &cur_time);
+
+        gf_msg(this->name, GF_LOG_ALERT, 0, Q_MSG_CROSSED_SOFT_LIMIT,
+               "Usage is above soft limit: %s used by %s", usage_str, path);
+
+        gf_event(EVENT_QUOTA_CROSSED_SOFT_LIMIT,
+                 "Usage=%s;volume=%s;"
+                 "path=%s",
+                 usage_str, priv->volume_uuid, path);
 
-        if (usage_str)
-                GF_FREE (usage_str);
+        ctx->prev_log_time = cur_time;
+    }
+
+    if (path)
+        GF_FREE(path);
+
+    if (usage_str)
+        GF_FREE(usage_str);
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_quota_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
-                        "Memory accounting init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_quota_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "Memory accounting init failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int32_t
-quota_forget (xlator_t *this, inode_t *inode)
+quota_forget(xlator_t *this, inode_t *inode)
 {
-        int32_t               ret     = 0;
-        uint64_t              ctx_int = 0;
-        quota_inode_ctx_t    *ctx     = NULL;
-        quota_dentry_t       *dentry  = NULL, *tmp;
+    int32_t ret = 0;
+    uint64_t ctx_int = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL, *tmp;
 
-        ret = inode_ctx_del (inode, this, &ctx_int);
+    ret = inode_ctx_del(inode, this, &ctx_int);
 
-        if (ret < 0) {
-                return 0;
-        }
+    if (ret < 0) {
+        return 0;
+    }
 
-        ctx = (quota_inode_ctx_t *) (long)ctx_int;
+    ctx = (quota_inode_ctx_t *)(long)ctx_int;
 
-        LOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry_safe(dentry, tmp, &ctx->parents, next)
         {
-                list_for_each_entry_safe (dentry, tmp, &ctx->parents, next) {
-                        __quota_dentry_free (dentry);
-                }
+            __quota_dentry_free(dentry);
         }
-        UNLOCK (&ctx->lock);
+    }
+    UNLOCK(&ctx->lock);
 
-        LOCK_DESTROY (&ctx->lock);
+    LOCK_DESTROY(&ctx->lock);
 
-        GF_FREE (ctx);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-init (xlator_t *this)
+int
+notify(xlator_t *this, int event, void *data, ...)
 {
-        int32_t       ret  = -1;
-        quota_priv_t *priv = NULL;
-        rpc_clnt_t   *rpc  = NULL;
-
-        if ((this->children == NULL)
-            || this->children->next) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        Q_MSG_INVALID_VOLFILE,
-                        "FATAL: quota (%s) not configured with "
-                        "exactly one child", this->name);
-                return -1;
-        }
-
-        if (this->parents == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_INVALID_VOLFILE,
-                        "dangling volume. check volfile");
-        }
-
-        QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
-
-        LOCK_INIT (&priv->lock);
-
-        this->private = priv;
-
-        GF_OPTION_INIT ("deem-statfs", priv->consider_statfs, bool, err);
-        GF_OPTION_INIT ("server-quota", priv->is_quota_on, bool, err);
-        GF_OPTION_INIT ("default-soft-limit", priv->default_soft_lim, percent,
-                        err);
-        GF_OPTION_INIT ("soft-timeout", priv->soft_timeout, time, err);
-        GF_OPTION_INIT ("hard-timeout", priv->hard_timeout, time, err);
-        GF_OPTION_INIT ("alert-time", priv->log_timeout, time, err);
-        GF_OPTION_INIT ("volume-uuid", priv->volume_uuid, str, err);
-
-        this->local_pool = mem_pool_new (quota_local_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-			Q_MSG_ENOMEM, "failed to create local_t's memory pool");
-                goto err;
+    quota_priv_t *priv = NULL;
+    int ret = 0;
+    rpc_clnt_t *rpc = NULL;
+    gf_boolean_t conn_status = _gf_true;
+    xlator_t *victim = data;
+
+    priv = this->private;
+    if (!priv || !priv->is_quota_on)
+        goto out;
+
+    if (event == GF_EVENT_PARENT_DOWN) {
+        rpc = priv->rpc_clnt;
+        if (rpc) {
+            rpc_clnt_disable(rpc);
+            pthread_mutex_lock(&priv->conn_mutex);
+            {
+                conn_status = priv->conn_status;
+                while (conn_status) {
+                    (void)pthread_cond_wait(&priv->conn_cond,
+                                            &priv->conn_mutex);
+                    conn_status = priv->conn_status;
+                }
+            }
+            pthread_mutex_unlock(&priv->conn_mutex);
+            gf_log(this->name, GF_LOG_INFO,
+                   "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name);
         }
+    }
 
-        if (priv->is_quota_on) {
-                rpc = quota_enforcer_init (this, this->options);
-                if (rpc == NULL) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-				Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
-				"quota enforcer rpc init failed");
-                        goto err;
-                }
+out:
+    ret = default_notify(this, event, data);
+    return ret;
+}
 
-                LOCK (&priv->lock);
-                {
-                        priv->rpc_clnt = rpc;
-                }
-                UNLOCK (&priv->lock);
+int32_t
+init(xlator_t *this)
+{
+    int32_t ret = -1;
+    quota_priv_t *priv = NULL;
+    rpc_clnt_t *rpc = NULL;
+
+    if ((this->children == NULL) || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_INVALID_VOLFILE,
+               "FATAL: quota (%s) not configured with "
+               "exactly one child",
+               this->name);
+        return -1;
+    }
+
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INVALID_VOLFILE,
+               "dangling volume. check volfile");
+    }
+
+    QUOTA_ALLOC_OR_GOTO(priv, quota_priv_t, err);
+
+    LOCK_INIT(&priv->lock);
+
+    this->private = priv;
+
+    GF_OPTION_INIT("deem-statfs", priv->consider_statfs, bool, err);
+    GF_OPTION_INIT("server-quota", priv->is_quota_on, bool, err);
+    GF_OPTION_INIT("default-soft-limit", priv->default_soft_lim, percent, err);
+    GF_OPTION_INIT("soft-timeout", priv->soft_timeout, time, err);
+    GF_OPTION_INIT("hard-timeout", priv->hard_timeout, time, err);
+    GF_OPTION_INIT("alert-time", priv->log_timeout, time, err);
+    GF_OPTION_INIT("volume-uuid", priv->volume_uuid, str, err);
+
+    this->local_pool = mem_pool_new(quota_local_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    pthread_mutex_init(&priv->conn_mutex, NULL);
+    pthread_cond_init(&priv->conn_cond, NULL);
+    priv->conn_status = _gf_false;
+
+    if (priv->is_quota_on) {
+        rpc = quota_enforcer_init(this, this->options);
+        if (rpc == NULL) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
+                   "quota enforcer rpc init failed");
+            goto err;
+        }
+
+        LOCK(&priv->lock);
+        {
+            priv->rpc_clnt = rpc;
         }
+        UNLOCK(&priv->lock);
+    }
 
-        ret = 0;
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t       ret      = -1;
-        quota_priv_t *priv     = NULL;
-        gf_boolean_t  quota_on = _gf_false;
-        rpc_clnt_t   *rpc      = NULL;
-
-        priv = this->private;
-
-        GF_OPTION_RECONF ("deem-statfs", priv->consider_statfs, options, bool,
-                          out);
-        GF_OPTION_RECONF ("server-quota", quota_on, options, bool,
-                          out);
-        GF_OPTION_RECONF ("default-soft-limit", priv->default_soft_lim,
-                          options, percent, out);
-        GF_OPTION_RECONF ("alert-time", priv->log_timeout, options,
-                          time, out);
-        GF_OPTION_RECONF ("soft-timeout", priv->soft_timeout, options,
-                          time, out);
-        GF_OPTION_RECONF ("hard-timeout", priv->hard_timeout, options,
-                          time, out);
-
-        if (quota_on) {
-                priv->rpc_clnt = quota_enforcer_init (this,
-                                                      this->options);
-                if (priv->rpc_clnt == NULL) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-				Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
-				"quota enforcer rpc init failed");
-                        goto out;
-                }
+    int32_t ret = -1;
+    quota_priv_t *priv = NULL;
+    gf_boolean_t quota_on = _gf_false;
+    rpc_clnt_t *rpc = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("deem-statfs", priv->consider_statfs, options, bool, out);
+    GF_OPTION_RECONF("server-quota", quota_on, options, bool, out);
+    GF_OPTION_RECONF("default-soft-limit", priv->default_soft_lim, options,
+                     percent, out);
+    GF_OPTION_RECONF("alert-time", priv->log_timeout, options, time, out);
+    GF_OPTION_RECONF("soft-timeout", priv->soft_timeout, options, time, out);
+    GF_OPTION_RECONF("hard-timeout", priv->hard_timeout, options, time, out);
+
+    if (quota_on) {
+        priv->rpc_clnt = quota_enforcer_init(this, this->options);
+        if (priv->rpc_clnt == NULL) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
+                   "quota enforcer rpc init failed");
+            goto out;
+        }
+
+    } else {
+        LOCK(&priv->lock);
+        {
+            rpc = priv->rpc_clnt;
+            priv->rpc_clnt = NULL;
+        }
+        UNLOCK(&priv->lock);
 
-        } else {
-                LOCK (&priv->lock);
-                {
-                        rpc = priv->rpc_clnt;
-                        priv->rpc_clnt = NULL;
-                }
-                UNLOCK (&priv->lock);
-
-                if (rpc != NULL) {
-                        // Quotad is shutdown when there is no started volume
-                        // which has quota enabled. So, we should disable the
-                        // enforcer client when quota is disabled on a volume,
-                        // to avoid spurious reconnect attempts to a service
-                        // (quotad), that is known to be down.
-                        rpc_clnt_unref (rpc);
-                }
+        if (rpc != NULL) {
+            // Quotad is shutdown when there is no started volume
+            // which has quota enabled. So, we should disable the
+            // enforcer client when quota is disabled on a volume,
+            // to avoid spurious reconnect attempts to a service
+            // (quotad), that is known to be down.
+            rpc_clnt_unref(rpc);
         }
+    }
 
-        priv->is_quota_on = quota_on;
+    priv->is_quota_on = quota_on;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-quota_priv_dump (xlator_t *this)
+quota_priv_dump(xlator_t *this)
 {
-        quota_priv_t *priv = NULL;
-        int32_t       ret  = -1;
-
-
-        GF_ASSERT (this);
-
-        priv = this->private;
-
-        gf_proc_dump_add_section ("xlators.features.quota.priv", this->name);
-
-        ret = TRY_LOCK (&priv->lock);
-        if (ret)
-             goto out;
-        else {
-                gf_proc_dump_write("soft-timeout", "%d", priv->soft_timeout);
-                gf_proc_dump_write("hard-timeout", "%d", priv->hard_timeout);
-                gf_proc_dump_write("alert-time", "%d", priv->log_timeout);
-                gf_proc_dump_write("quota-on", "%d", priv->is_quota_on);
-                gf_proc_dump_write("statfs", "%d", priv->consider_statfs);
-                gf_proc_dump_write("volume-uuid", "%s", priv->volume_uuid);
-                gf_proc_dump_write("validation-count", "%ld",
-                                    priv->validation_count);
-        }
-        UNLOCK (&priv->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+
+    GF_ASSERT(this);
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    gf_proc_dump_add_section("xlators.features.quota.priv");
+
+    ret = TRY_LOCK(&priv->lock);
+    if (ret)
+        goto out;
+    else {
+        gf_proc_dump_write("soft-timeout", "%u", priv->soft_timeout);
+        gf_proc_dump_write("hard-timeout", "%u", priv->hard_timeout);
+        gf_proc_dump_write("alert-time", "%u", priv->log_timeout);
+        gf_proc_dump_write("quota-on", "%d", priv->is_quota_on);
+        gf_proc_dump_write("statfs", "%d", priv->consider_statfs);
+        gf_proc_dump_write("volume-uuid", "%s", priv->volume_uuid);
+        gf_proc_dump_write("validation-count", "%" PRIu64,
+                           priv->validation_count);
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
+    quota_priv_t *priv = NULL;
+    rpc_clnt_t *rpc = NULL;
+
+    priv = this->private;
+    if (!priv)
         return;
+    rpc = priv->rpc_clnt;
+    priv->rpc_clnt = NULL;
+    if (rpc) {
+        rpc_clnt_connection_cleanup(&rpc->conn);
+        rpc_clnt_unref(rpc);
+    }
+
+    this->private = NULL;
+    LOCK_DESTROY(&priv->lock);
+    pthread_mutex_destroy(&priv->conn_mutex);
+    pthread_cond_destroy(&priv->conn_cond);
+
+    GF_FREE(priv);
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .statfs       = quota_statfs,
-        .lookup       = quota_lookup,
-        .writev       = quota_writev,
-        .create       = quota_create,
-        .mkdir        = quota_mkdir,
-        .truncate     = quota_truncate,
-        .ftruncate    = quota_ftruncate,
-        .unlink       = quota_unlink,
-        .symlink      = quota_symlink,
-        .link         = quota_link,
-        .rename       = quota_rename,
-        .getxattr     = quota_getxattr,
-        .fgetxattr    = quota_fgetxattr,
-        .stat         = quota_stat,
-        .fstat        = quota_fstat,
-        .readlink     = quota_readlink,
-        .readv        = quota_readv,
-        .fsync        = quota_fsync,
-        .setattr      = quota_setattr,
-        .fsetattr     = quota_fsetattr,
-        .mknod        = quota_mknod,
-        .setxattr     = quota_setxattr,
-        .fsetxattr    = quota_fsetxattr,
-        .removexattr  = quota_removexattr,
-        .fremovexattr = quota_fremovexattr,
-        .readdirp     = quota_readdirp,
-	.fallocate    = quota_fallocate,
+    .statfs = quota_statfs,
+    .lookup = quota_lookup,
+    .writev = quota_writev,
+    .create = quota_create,
+    .mkdir = quota_mkdir,
+    .truncate = quota_truncate,
+    .ftruncate = quota_ftruncate,
+    .unlink = quota_unlink,
+    .symlink = quota_symlink,
+    .link = quota_link,
+    .rename = quota_rename,
+    .getxattr = quota_getxattr,
+    .fgetxattr = quota_fgetxattr,
+    .stat = quota_stat,
+    .fstat = quota_fstat,
+    .readlink = quota_readlink,
+    .readv = quota_readv,
+    .fsync = quota_fsync,
+    .setattr = quota_setattr,
+    .fsetattr = quota_fsetattr,
+    .mknod = quota_mknod,
+    .setxattr = quota_setxattr,
+    .fsetxattr = quota_fsetxattr,
+    .removexattr = quota_removexattr,
+    .fremovexattr = quota_fremovexattr,
+    .readdirp = quota_readdirp,
+    .fallocate = quota_fallocate,
 };
 
-struct xlator_cbks cbks = {
-        .forget = quota_forget
-};
+struct xlator_cbks cbks = {.forget = quota_forget};
 
 struct xlator_dumpops dumpops = {
-        .priv    = quota_priv_dump,
+    .priv = quota_priv_dump,
 };
 struct volume_options options[] = {
-        {.key = {"limit-set"}},
-        {.key = {"deem-statfs"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "If set to on, it takes quota limits into"
-                        "consideration while estimating fs size. (df command)"
-                        " (Default is off)."
-        },
-        {.key = {"server-quota"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "Skip the quota enforcement if the feature is"
-                        " not turned on. This is not a user exposed option."
-        },
-        {.key = {"default-soft-limit"},
-         .type = GF_OPTION_TYPE_PERCENT,
-         .default_value = "80%",
-        },
-        {.key = {"soft-timeout"},
-         .type = GF_OPTION_TYPE_TIME,
-         .min = 0,
-         .max = 1800,
-         .default_value = "60",
-         .description = "quota caches the directory sizes on client. "
-                        "soft-timeout indicates the timeout for the validity of"
-                        " cache before soft-limit has been crossed."
-        },
-        {.key = {"hard-timeout"},
-         .type = GF_OPTION_TYPE_TIME,
-         .min = 0,
-         .max = 60,
-         .default_value = "5",
-         .description = "quota caches the directory sizes on client. "
-                        "hard-timeout indicates the timeout for the validity of"
-                        " cache after soft-limit has been crossed."
-        },
-        { .key   = {"username"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        { .key   = {"password"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        { .key   = {"transport-type"},
-          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
-                    "tcp/client", "ib-verbs/client", "rdma"},
-          .type  = GF_OPTION_TYPE_STR,
-        },
-        { .key   = {"remote-host"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS,
-        },
-        { .key   = {"remote-port"},
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key  = {"volume-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "uuid of the volume this brick is part of."
-        },
-        { .key  = {"alert-time"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min = 0,
-          .max = 7*86400,
-          .default_value = "86400",
-        },
-        {.key = {NULL}}
+    {
+        .key = {"enable"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable is the volume option that can be used "
+                       "to turn on quota.",
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .level = OPT_STATUS_BASIC,
+        .tags = {},
+    },
+    {
+        .key = {"deem-statfs"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "on",
+        .description = "If set to on, it takes quota limits into"
+                       " consideration while estimating fs size. (df command)"
+                       " (Default is on).",
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {
+        .key = {"server-quota"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "Skip the quota enforcement if the feature is"
+                       " not turned on. This is not a user exposed option.",
+        .flags = OPT_FLAG_NONE,
+    },
+    {
+        .key = {"default-soft-limit"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .default_value = "80%",
+        .op_version = {3},
+        .description = "Soft limit is expressed as a proportion of hard limit."
+                       " Default-soft-limit is the proportion used when the "
+                       " user does not supply any soft limit value.",
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {
+        .key = {"soft-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .max = 1800,
+        .default_value = "60",
+        .description = "quota caches the directory sizes on client. "
+                       "soft-timeout indicates the timeout for the validity of"
+                       " cache before soft-limit has been crossed.",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {
+        .key = {"hard-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .max = 60,
+        .default_value = "5",
+        .description = "quota caches the directory sizes on client. "
+                       "hard-timeout indicates the timeout for the validity of"
+                       " cache after soft-limit has been crossed.",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {.key = {"volume-uuid"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "{{ volume.id }}",
+     .description = "uuid of the volume this brick is part of."},
+    {
+        .key = {"alert-time"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .max = 7 * 86400,
+        .default_value = "86400",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .description = "Frequency of limit breach messages in log.",
+        .tags = {},
+    },
+    {.key = {NULL}}};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "quota",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h
index 6f74da789b6..0395d78c9ef 100644
--- a/xlators/features/quota/src/quota.h
+++ b/xlators/features/quota/src/quota.h
@@ -10,273 +10,257 @@
 #ifndef _QUOTA_H
 #define _QUOTA_H
 
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
+#include <glusterfs/call-stub.h>
 #include "quota-mem-types.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "logging.h"
-#include "dict.h"
-#include "stack.h"
-#include "event.h"
-#include "globals.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/gf-event.h>
 #include "rpcsvc.h"
 #include "rpc-clnt.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "glusterfs3-xdr.h"
 #include "glusterfs3.h"
 #include "xdr-generic.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "protocol-common.h"
-#include "quota-common-utils.h"
+#include <glusterfs/quota-common-utils.h>
 #include "quota-messages.h"
 
-#define DIRTY                   "dirty"
-#define SIZE                    "size"
-#define CONTRIBUTION            "contri"
-#define VAL_LENGTH              8
-#define READDIR_BUF             4096
+#define DIRTY "dirty"
+#define SIZE "size"
+#define CONTRIBUTION "contri"
+#define VAL_LENGTH 8
+#define READDIR_BUF 4096
 
 #ifndef UUID_CANONICAL_FORM_LEN
 #define UUID_CANONICAL_FORM_LEN 36
 #endif
 
-#define WIND_IF_QUOTAOFF(is_quota_on, label)     \
-        if (!is_quota_on)                       \
-                goto label;
-
-#define QUOTA_WIND_FOR_INTERNAL_FOP(xdata, label)                          \
-        do {                                                               \
-                if (xdata && dict_get (xdata, GLUSTERFS_INTERNAL_FOP_KEY)) \
-                goto label;                                                \
-        } while (0)
-
-#define DID_REACH_LIMIT(lim, prev_size, cur_size)               \
-        ((cur_size) >= (lim) && (prev_size) < (lim))
-
-#define QUOTA_SAFE_INCREMENT(lock, var)         \
-        do {                                    \
-                LOCK (lock);                    \
-                var ++;                         \
-                UNLOCK (lock);                  \
-        } while (0)
-
-#define QUOTA_SAFE_DECREMENT(lock, var)         \
-        do {                                    \
-                LOCK (lock);                    \
-                var --;                         \
-                UNLOCK (lock);                  \
-        } while (0)
-
-#define QUOTA_ALLOC_OR_GOTO(var, type, label)           \
-        do {                                            \
-                var = GF_CALLOC (sizeof (type), 1,      \
-                                 gf_quota_mt_##type);   \
-                if (!var) {                             \
-                        gf_msg ("", GF_LOG_ERROR,       \
-                                ENOMEM, Q_MSG_ENOMEM,   \
-				"out of memory");       \
-                        ret = -1;                       \
-                        goto label;                     \
-                }                                       \
-        } while (0);
-
-#define QUOTA_STACK_WIND_TAIL(frame, params...)                         \
-        do {                                                            \
-                quota_local_t *_local = NULL;                           \
-                xlator_t      *_this  = NULL;                           \
-                                                                        \
-                if (frame) {                                            \
-                        _local = frame->local;                          \
-                        _this  = frame->this;                           \
-                        frame->local = NULL;                            \
-                }                                                       \
-                                                                        \
-                STACK_WIND_TAIL (frame, params);                        \
-                                                                        \
-                if (_local)                                             \
-                        quota_local_cleanup (_local);                   \
-        } while (0)
-
-#define QUOTA_STACK_UNWIND(fop, frame, params...)                       \
-        do {                                                            \
-                quota_local_t *_local = NULL;                           \
-                xlator_t      *_this  = NULL;                           \
-                if (frame) {                                            \
-                        _local = frame->local;                          \
-                        _this  = frame->this;                           \
-                        frame->local = NULL;                            \
-                }                                                       \
-                STACK_UNWIND_STRICT (fop, frame, params);               \
-                quota_local_cleanup (_local);                           \
-        } while (0)
-
-#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
-        do {                                            \
-                list_del (&_contribution->contri_list); \
-                GF_FREE (_contribution);                \
-        } while (0)
-
-#define GET_CONTRI_KEY(var, _vol_name, _gfid, _ret)             \
-        do {                                                    \
-                char _gfid_unparsed[40];                        \
-                if (_gfid != NULL) {                            \
-                        gf_uuid_unparse (_gfid, _gfid_unparsed);\
-                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
-                                            "%s.%s." CONTRIBUTION,      \
-                                            _vol_name, _gfid_unparsed); \
-                } else {                                                \
-                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
-                                            "%s.." CONTRIBUTION,       \
-                                            _vol_name);             \
-                }                                                       \
-         } while (0)
-
-
-#define GET_CONTRI_KEY_OR_GOTO(var, _vol_name, _gfid, label)    \
-        do {                                                    \
-                GET_CONTRI_KEY(var, _vol_name, _gfid, ret);     \
-                if (ret == -1)                                  \
-                        goto label;                             \
-        } while (0)
-
-#define GET_DIRTY_KEY_OR_GOTO(var, _vol_name, label)            \
-        do {                                                    \
-                ret = gf_asprintf (var, QUOTA_XATTR_PREFIX      \
-                                   "%s." DIRTY, _vol_name);     \
-                if (ret == -1)                                  \
-                        goto label;                             \
-        } while (0)
-
-#define QUOTA_REG_OR_LNK_FILE(ia_type)  \
-    (IA_ISREG (ia_type) || IA_ISLNK (ia_type))
-
-
+#define WIND_IF_QUOTAOFF(is_quota_on, label)                                   \
+    if (!is_quota_on)                                                          \
+        goto label;
+
+#define QUOTA_WIND_FOR_INTERNAL_FOP(xdata, label)                              \
+    do {                                                                       \
+        if (xdata && dict_get_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY))        \
+            goto label;                                                        \
+    } while (0)
+
+#define DID_REACH_LIMIT(lim, prev_size, cur_size)                              \
+    ((cur_size) >= (lim) && (prev_size) < (lim))
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var++;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
+
+#define QUOTA_SAFE_DECREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var--;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
+
+#define QUOTA_ALLOC_OR_GOTO(var, type, label)                                  \
+    do {                                                                       \
+        var = GF_CALLOC(sizeof(type), 1, gf_quota_mt_##type);                  \
+        if (!var) {                                                            \
+            gf_msg("", GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM, "out of memory");   \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
+
+#define QUOTA_STACK_WIND_TAIL(frame, params...)                                \
+    do {                                                                       \
+        quota_local_t *_local = NULL;                                          \
+                                                                               \
+        if (frame) {                                                           \
+            _local = frame->local;                                             \
+            frame->local = NULL;                                               \
+        }                                                                      \
+                                                                               \
+        STACK_WIND_TAIL(frame, params);                                        \
+                                                                               \
+        if (_local)                                                            \
+            quota_local_cleanup(_local);                                       \
+    } while (0)
+
+#define QUOTA_STACK_UNWIND(fop, frame, params...)                              \
+    do {                                                                       \
+        quota_local_t *_local = NULL;                                          \
+        if (frame) {                                                           \
+            _local = frame->local;                                             \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        quota_local_cleanup(_local);                                           \
+    } while (0)
+
+#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)                            \
+    do {                                                                       \
+        list_del(&_contribution->contri_list);                                 \
+        GF_FREE(_contribution);                                                \
+    } while (0)
+
+#define GET_CONTRI_KEY(var, _vol_name, _gfid, _ret)                            \
+    do {                                                                       \
+        char _gfid_unparsed[40];                                               \
+        if (_gfid != NULL) {                                                   \
+            gf_uuid_unparse(_gfid, _gfid_unparsed);                            \
+            _ret = gf_asprintf(var, QUOTA_XATTR_PREFIX "%s.%s." CONTRIBUTION,  \
+                               _vol_name, _gfid_unparsed);                     \
+        } else {                                                               \
+            _ret = gf_asprintf(var, QUOTA_XATTR_PREFIX "%s.." CONTRIBUTION,    \
+                               _vol_name);                                     \
+        }                                                                      \
+    } while (0)
+
+#define GET_CONTRI_KEY_OR_GOTO(var, _vol_name, _gfid, label)                   \
+    do {                                                                       \
+        GET_CONTRI_KEY(var, _vol_name, _gfid, ret);                            \
+        if (ret == -1)                                                         \
+            goto label;                                                        \
+    } while (0)
+
+#define GET_DIRTY_KEY_OR_GOTO(var, _vol_name, label)                           \
+    do {                                                                       \
+        ret = gf_asprintf(var, QUOTA_XATTR_PREFIX "%s." DIRTY, _vol_name);     \
+        if (ret == -1)                                                         \
+            goto label;                                                        \
+    } while (0)
+
+#define QUOTA_REG_OR_LNK_FILE(ia_type) (IA_ISREG(ia_type) || IA_ISLNK(ia_type))
 
 struct quota_dentry {
-        char            *name;
-        uuid_t           par;
-        struct list_head next;
+    char *name;
+    uuid_t par;
+    struct list_head next;
 };
 typedef struct quota_dentry quota_dentry_t;
 
 struct quota_inode_ctx {
-        int64_t          size;
-        int64_t          hard_lim;
-        int64_t          soft_lim;
-        int64_t          file_count;
-        int64_t          dir_count;
-        int64_t          object_hard_lim;
-        int64_t          object_soft_lim;
-        struct iatt      buf;
-        struct list_head parents;
-        struct timeval   tv;
-        struct timeval   prev_log;
-        gf_boolean_t     ancestry_built;
-        gf_lock_t        lock;
+    int64_t size;
+    int64_t hard_lim;
+    int64_t soft_lim;
+    int64_t file_count;
+    int64_t dir_count;
+    int64_t object_hard_lim;
+    int64_t object_soft_lim;
+    struct iatt buf;
+    struct list_head parents;
+    time_t validate_time;
+    time_t prev_log_time;
+    gf_boolean_t ancestry_built;
+    gf_lock_t lock;
 };
 typedef struct quota_inode_ctx quota_inode_ctx_t;
 
-typedef void
-(*quota_ancestry_built_t) (struct list_head *parents, inode_t *inode,
-                           int32_t op_ret, int32_t op_errno, void *data);
+typedef void (*quota_ancestry_built_t)(struct list_head *parents,
+                                       inode_t *inode, int32_t op_ret,
+                                       int32_t op_errno, void *data);
 
-typedef void
-(*quota_fop_continue_t) (call_frame_t *frame);
+typedef void (*quota_fop_continue_t)(call_frame_t *frame);
 
 struct quota_local {
-        gf_lock_t               lock;
-        uint32_t                link_count;
-        loc_t                   loc;
-        loc_t                   oldloc;
-        loc_t                   newloc;
-        loc_t                   validate_loc;
-        int64_t                 delta;
-        int8_t                  object_delta;
-        int32_t                 op_ret;
-        int32_t                 op_errno;
-        int64_t                 size;
-        char                    just_validated;
-        fop_lookup_cbk_t        validate_cbk;
-        quota_fop_continue_t    fop_continue_cbk;
-        inode_t                *inode;
-        uuid_t                  common_ancestor; /* Used by quota_rename */
-        call_stub_t            *stub;
-        struct iobref          *iobref;
-        quota_limits_t          limit;
-        quota_limits_t          object_limit;
-        int64_t                 space_available;
-        quota_ancestry_built_t  ancestry_cbk;
-        void                   *ancestry_data;
-        dict_t                 *xdata;
-        dict_t                 *validate_xdata;
-        int32_t                 quotad_conn_retry;
-        xlator_t               *this;
-        call_frame_t           *par_frame;
+    gf_lock_t lock;
+    uint32_t link_count;
+    loc_t loc;
+    loc_t oldloc;
+    loc_t newloc;
+    loc_t validate_loc;
+    int64_t delta;
+    int8_t object_delta;
+    int32_t op_ret;
+    int32_t op_errno;
+    int64_t size;
+    char just_validated;
+    fop_lookup_cbk_t validate_cbk;
+    quota_fop_continue_t fop_continue_cbk;
+    inode_t *inode;
+    uuid_t common_ancestor; /* Used by quota_rename */
+    call_stub_t *stub;
+    struct iobref *iobref;
+    quota_limits_t limit;
+    quota_limits_t object_limit;
+    int64_t space_available;
+    quota_ancestry_built_t ancestry_cbk;
+    void *ancestry_data;
+    dict_t *xdata;
+    dict_t *validate_xdata;
+    int32_t quotad_conn_retry;
+    xlator_t *this;
+    call_frame_t *par_frame;
 };
-typedef struct quota_local      quota_local_t;
+typedef struct quota_local quota_local_t;
 
 struct quota_priv {
-        uint32_t                soft_timeout;
-        uint32_t                hard_timeout;
-        uint32_t               log_timeout;
-        double                 default_soft_lim;
-        gf_boolean_t           is_quota_on;
-        gf_boolean_t           consider_statfs;
-        gf_lock_t              lock;
-        rpc_clnt_prog_t       *quota_enforcer;
-        struct rpcsvc_program *quotad_aggregator;
-        struct rpc_clnt       *rpc_clnt;
-        rpcsvc_t              *rpcsvc;
-        inode_table_t         *itable;
-        char                  *volume_uuid;
-        uint64_t               validation_count;
-        int32_t                quotad_conn_status;
+    /* FIXME: consider time_t for timeouts. */
+    uint32_t soft_timeout;
+    uint32_t hard_timeout;
+    uint32_t log_timeout;
+    double default_soft_lim;
+    gf_boolean_t is_quota_on;
+    gf_boolean_t consider_statfs;
+    gf_lock_t lock;
+    rpc_clnt_prog_t *quota_enforcer;
+    struct rpcsvc_program *quotad_aggregator;
+    struct rpc_clnt *rpc_clnt;
+    rpcsvc_t *rpcsvc;
+    inode_table_t *itable;
+    char *volume_uuid;
+    uint64_t validation_count;
+    int32_t quotad_conn_status;
+    pthread_mutex_t conn_mutex;
+    pthread_cond_t conn_cond;
+    gf_boolean_t conn_status;
 };
-typedef struct quota_priv      quota_priv_t;
+typedef struct quota_priv quota_priv_t;
 
 int
-quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
-                       fop_lookup_cbk_t cbk);
+quota_enforcer_lookup(call_frame_t *frame, xlator_t *this, dict_t *xdata,
+                      fop_lookup_cbk_t cbk);
 
 void
-_quota_enforcer_lookup (void *data);
+_quota_enforcer_lookup(void *data);
 
 struct rpc_clnt *
-quota_enforcer_init (xlator_t *this, dict_t *options);
+quota_enforcer_init(xlator_t *this, dict_t *options);
 
 void
-quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
-                 int64_t delta);
+quota_log_usage(xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                int64_t delta);
 
 int
-quota_build_ancestry (inode_t *inode, quota_ancestry_built_t ancestry_cbk,
-                      void *data);
+quota_build_ancestry(inode_t *inode, quota_ancestry_built_t ancestry_cbk,
+                     void *data);
 
 void
-quota_get_limit_dir (call_frame_t *frame, inode_t *cur_inode, xlator_t *this);
+quota_get_limit_dir(call_frame_t *frame, inode_t *cur_inode, xlator_t *this);
 
 int32_t
-quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this);
+quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this);
 
 inode_t *
-do_quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                      quota_dentry_t *dentry, gf_boolean_t force);
+do_quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                     quota_dentry_t *dentry, gf_boolean_t force);
 int
-quota_fill_inodectx (xlator_t *this, inode_t *inode, dict_t *dict,
-                     loc_t *loc, struct iatt *buf, int32_t *op_errno);
+quota_fill_inodectx(xlator_t *this, inode_t *inode, dict_t *dict, loc_t *loc,
+                    struct iatt *buf, int32_t *op_errno);
 
 int32_t
-quota_check_size_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated, int64_t delta,
-                          quota_local_t *local, gf_boolean_t *skip_check);
+quota_check_size_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                       quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                       int32_t *op_errno, int just_validated, int64_t delta,
+                       quota_local_t *local, gf_boolean_t *skip_check);
 
 int32_t
-quota_check_object_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated,
-                          quota_local_t *local, gf_boolean_t *skip_check);
+quota_check_object_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                         quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                         int32_t *op_errno, int just_validated,
+                         quota_local_t *local, gf_boolean_t *skip_check);
 #endif
diff --git a/xlators/features/quota/src/quotad-aggregator.c b/xlators/features/quota/src/quotad-aggregator.c
index 8a7cfdca3f5..75d47867b5b 100644
--- a/xlators/features/quota/src/quotad-aggregator.c
+++ b/xlators/features/quota/src/quotad-aggregator.c
@@ -13,443 +13,482 @@
 #include "quotad-helpers.h"
 #include "quotad-aggregator.h"
 
-struct rpcsvc_program quotad_aggregator_prog;
+static char *qd_ext_xattrs[] = {
+    QUOTA_SIZE_KEY,
+    QUOTA_LIMIT_KEY,
+    QUOTA_LIMIT_OBJECTS_KEY,
+    NULL,
+};
+
+static struct rpcsvc_program quotad_aggregator_prog;
 
 struct iobuf *
-quotad_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
-                        xdrproc_t xdrproc)
+quotad_serialize_reply(rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
+                       xdrproc_t xdrproc)
 {
-        struct iobuf *iob      = NULL;
-        ssize_t       retlen   = 0;
-        ssize_t       xdr_size = 0;
-
-        GF_VALIDATE_OR_GOTO ("server", req, ret);
-
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
+    struct iobuf *iob = NULL;
+    ssize_t retlen = 0;
+    ssize_t xdr_size = 0;
+
+    GF_VALIDATE_OR_GOTO("server", req, ret);
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    if (arg && xdrproc) {
+        xdr_size = xdr_sizeof(xdrproc, arg);
+        iob = iobuf_get2(req->svc->ctx->iobuf_pool, xdr_size);
+        if (!iob) {
+            gf_log_callingfn(THIS->name, GF_LOG_ERROR, "Failed to get iobuf");
+            goto ret;
+        };
+
+        iobuf_to_iovec(iob, outmsg);
+        /* Use the given serializer to translate the given C structure
+         * in arg to XDR format which will be written into the buffer
+         * in outmsg.
+         */
+        /* retlen is used to received the error since size_t is unsigned and we
+         * need -1 for error notification during encoding.
          */
-        if (arg && xdrproc) {
-                xdr_size = xdr_sizeof (xdrproc, arg);
-                iob = iobuf_get2 (req->svc->ctx->iobuf_pool, xdr_size);
-                if (!iob) {
-                        gf_log_callingfn (THIS->name, GF_LOG_ERROR,
-                                          "Failed to get iobuf");
-                        goto ret;
-                };
-
-                iobuf_to_iovec (iob, outmsg);
-                /* Use the given serializer to translate the give C structure in arg
-                 * to XDR format which will be written into the buffer in outmsg.
-                 */
-                /* retlen is used to received the error since size_t is unsigned and we
-                 * need -1 for error notification during encoding.
-                 */
-
-                retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
-                if (retlen == -1) {
-                        /* Failed to Encode 'GlusterFS' msg in RPC is not exactly
-                           failure of RPC return values.. client should get
-                           notified about this, so there are no missing frames */
-                        gf_log_callingfn ("", GF_LOG_ERROR, "Failed to encode message");
-                        req->rpc_err = GARBAGE_ARGS;
-                        retlen = 0;
-                }
+
+        retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+        if (retlen == -1) {
+            /* Failed to Encode 'GlusterFS' msg in RPC is not exactly
+               failure of RPC return values.. Client should get
+               notified about this, so there are no missing frames */
+            gf_log_callingfn("", GF_LOG_ERROR, "Failed to encode message");
+            req->rpc_err = GARBAGE_ARGS;
+            retlen = 0;
         }
-        outmsg->iov_len = retlen;
+    }
+    outmsg->iov_len = retlen;
 ret:
-        return iob;
+    return iob;
 }
 
 int
-quotad_aggregator_submit_reply (call_frame_t *frame, rpcsvc_request_t *req,
-                                void *arg, struct iovec *payload,
-                                int payloadcount, struct iobref *iobref,
-                                xdrproc_t xdrproc)
+quotad_aggregator_submit_reply(call_frame_t *frame, rpcsvc_request_t *req,
+                               void *arg, struct iovec *payload,
+                               int payloadcount, struct iobref *iobref,
+                               xdrproc_t xdrproc)
 {
-        struct iobuf              *iob        = NULL;
-        int                        ret        = -1;
-        struct iovec               rsp        = {0,};
-        quotad_aggregator_state_t *state      = NULL;
-        char                       new_iobref = 0;
+    struct iobuf *iob = NULL;
+    int ret = -1;
+    struct iovec rsp = {
+        0,
+    };
+    quotad_aggregator_state_t *state = NULL;
+    char new_iobref = 0;
 
-        GF_VALIDATE_OR_GOTO ("server", req, ret);
+    GF_VALIDATE_OR_GOTO("server", req, ret);
 
-        if (frame) {
-                state = frame->root->state;
-                frame->local = NULL;
-        }
+    if (frame) {
+        state = frame->root->state;
+        frame->local = NULL;
+    }
 
+    if (!iobref) {
+        iobref = iobref_new();
         if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref) {
-                        goto ret;
-                }
-
-                new_iobref = 1;
+            goto ret;
         }
 
-        iob = quotad_serialize_reply (req, arg, &rsp, xdrproc);
-        if (!iob) {
-                gf_msg ("", GF_LOG_ERROR, 0, Q_MSG_DICT_SERIALIZE_FAIL,
-                        "Failed to serialize reply");
-                goto ret;
-        }
+        new_iobref = 1;
+    }
+
+    iob = quotad_serialize_reply(req, arg, &rsp, xdrproc);
+    if (!iob) {
+        gf_msg("", GF_LOG_ERROR, 0, Q_MSG_DICT_SERIALIZE_FAIL,
+               "Failed to serialize reply");
+        goto ret;
+    }
 
-        iobref_add (iobref, iob);
+    iobref_add(iobref, iob);
 
-        ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
-                                     iobref);
+    ret = rpcsvc_submit_generic(req, &rsp, 1, payload, payloadcount, iobref);
 
-        iobuf_unref (iob);
+    iobuf_unref(iob);
 
-        ret = 0;
+    ret = 0;
 ret:
-        if (state) {
-                quotad_aggregator_free_state (state);
-        }
+    if (state) {
+        quotad_aggregator_free_state(state);
+    }
 
-        if (frame)
-                STACK_DESTROY (frame->root);
+    if (frame)
+        STACK_DESTROY(frame->root);
 
-        if (new_iobref) {
-                iobref_unref (iobref);
-        }
+    if (new_iobref) {
+        iobref_unref(iobref);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-quotad_aggregator_getlimit_cbk (xlator_t *this, call_frame_t *frame,
-                                void *lookup_rsp)
+quotad_aggregator_getlimit_cbk(xlator_t *this, call_frame_t *frame,
+                               void *lookup_rsp)
 {
-        gfs3_lookup_rsp            *rsp    = lookup_rsp;
-        gf_cli_rsp                 cli_rsp = {0,};
-        dict_t                     *xdata  = NULL;
-        quotad_aggregator_state_t  *state  = NULL;
-        int                         ret    = -1;
-        int                         type   = 0;
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
-                                      (rsp->xdata.xdata_val),
-                                      (rsp->xdata.xdata_len), rsp->op_ret,
-                                      rsp->op_errno, out);
-
-        if (xdata) {
-                state = frame->root->state;
-                ret = dict_get_int32 (state->xdata, "type", &type);
-                if (ret < 0)
-                        goto out;
-
-                ret = dict_set_int32 (xdata, "type", type);
-                if (ret < 0)
-                        goto out;
-        }
+    gfs3_lookup_rsp *rsp = lookup_rsp;
+    gf_cli_rsp cli_rsp = {
+        0,
+    };
+    dict_t *xdata = NULL;
+    quotad_aggregator_state_t *state = NULL;
+    int ret = -1;
+    int type = 0;
+
+    if (!rsp || (rsp->op_ret == -1))
+        goto reply;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(frame->this, xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), rsp->op_ret,
+                                 rsp->op_errno, out);
+
+    if (xdata) {
+        state = frame->root->state;
+        ret = dict_get_int32n(state->req_xdata, "type", SLEN("type"), &type);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_set_int32_sizen(xdata, "type", type);
+        if (ret < 0)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        rsp->op_ret = ret;
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        Q_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize "
-                        "nameless lookup rsp");
-                goto reply;
-        }
-        cli_rsp.op_ret = rsp->op_ret;
-        cli_rsp.op_errno = rsp->op_errno;
-        cli_rsp.op_errstr = "";
-        if (xdata) {
-                GF_PROTOCOL_DICT_SERIALIZE (frame->this, xdata,
-                                            (&cli_rsp.dict.dict_val),
-                                            (cli_rsp.dict.dict_len),
-                                            cli_rsp.op_errno, reply);
-        }
+    rsp->op_ret = ret;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_DICT_UNSERIALIZE_FAIL,
+               "failed to unserialize "
+               "nameless lookup rsp");
+        goto reply;
+    }
+    cli_rsp.op_ret = rsp->op_ret;
+    cli_rsp.op_errno = rsp->op_errno;
+    cli_rsp.op_errstr = "";
+    if (xdata) {
+        GF_PROTOCOL_DICT_SERIALIZE(frame->this, xdata, (&cli_rsp.dict.dict_val),
+                                   (cli_rsp.dict.dict_len), cli_rsp.op_errno,
+                                   reply);
+    }
 
 reply:
-        quotad_aggregator_submit_reply (frame, frame->local, (void*)&cli_rsp, NULL, 0,
-                                        NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    quotad_aggregator_submit_reply(frame, (frame) ? frame->local : NULL,
+                                   (void *)&cli_rsp, NULL, 0, NULL,
+                                   (xdrproc_t)xdr_gf_cli_rsp);
 
-        dict_unref (xdata);
-        GF_FREE (cli_rsp.dict.dict_val);
-        return 0;
+    dict_unref(xdata);
+    GF_FREE(cli_rsp.dict.dict_val);
+    return 0;
 }
 
 int
-quotad_aggregator_getlimit (rpcsvc_request_t *req)
+quotad_aggregator_getlimit(rpcsvc_request_t *req)
 {
-        call_frame_t              *frame = NULL;
-        gf_cli_req                 cli_req = {{0}, };
-        gf_cli_rsp                 cli_rsp = {0};
-        gfs3_lookup_req            args  = {{0,},};
-        gfs3_lookup_rsp            rsp   = {0,};
-        quotad_aggregator_state_t *state = NULL;
-        xlator_t                  *this  = NULL;
-        dict_t                    *dict  = NULL;
-        int                        ret   = -1, op_errno = 0;
-        char                      *gfid_str = NULL;
-        uuid_t                     gfid = {0};
-
-        GF_VALIDATE_OR_GOTO ("quotad-aggregator", req, err);
-
-        this = THIS;
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0)  {
-                //failed to decode msg;
-                gf_msg ("this->name", GF_LOG_ERROR, 0, Q_MSG_XDR_DECODE_ERROR,
-                        "xdr decoding error");
-                req->rpc_err = GARBAGE_ARGS;
-                goto err;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                Q_MSG_DICT_UNSERIALIZE_FAIL,
-                                "Failed to unserialize req-buffer to "
-                                "dictionary");
-                        goto err;
-                }
-        }
-
-        ret = dict_get_str (dict, "gfid", &gfid_str);
-        if (ret) {
-                goto err;
-        }
-
-        gf_uuid_parse ((const char*)gfid_str, gfid);
-
-        frame = quotad_aggregator_get_frame_from_req (req);
-        if (frame == NULL) {
-                rsp.op_errno = ENOMEM;
-                goto err;
-        }
-        state = frame->root->state;
-        state->xdata = dict;
-
-        ret = dict_set_int32 (state->xdata, QUOTA_LIMIT_KEY, 42);
-        if (ret)
-                goto err;
-
-        ret = dict_set_int32 (state->xdata, QUOTA_LIMIT_OBJECTS_KEY, 42);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
-                        "Failed to set QUOTA_LIMIT_OBJECTS_KEY");
-                goto err;
-        }
-
-        ret = dict_set_int32 (state->xdata, QUOTA_SIZE_KEY, 42);
-        if (ret)
-                goto err;
-
-        ret = dict_set_int32 (state->xdata, GET_ANCESTRY_PATH_KEY, 42);
-        if (ret)
-                goto err;
-
-        memcpy (&args.gfid, &gfid, 16);
-
-        args.bname           = alloca (req->msg[0].iov_len);
-        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
-
-        ret = qd_nameless_lookup (this, frame, &args, state->xdata,
-                                  quotad_aggregator_getlimit_cbk);
-        if (ret) {
-                rsp.op_errno = ret;
-                goto err;
+    call_frame_t *frame = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    gf_cli_rsp cli_rsp = {0};
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    int ret = -1, op_errno = 0;
+    char *gfid_str = NULL;
+    uuid_t gfid = {0};
+    char *volume_uuid = NULL;
+
+    GF_VALIDATE_OR_GOTO("quotad-aggregator", req, err);
+
+    this = THIS;
+
+    cli_req.dict.dict_val = alloca(req->msg[0].iov_len);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("this->name", GF_LOG_ERROR, 0, Q_MSG_XDR_DECODE_ERROR,
+               "xdr decoding error");
+        req->rpc_err = GARBAGE_ARGS;
+        goto err;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to unserialize req-buffer to "
+                   "dictionary");
+            goto err;
         }
-
-        return ret;
+    }
+
+    ret = dict_get_strn(dict, "gfid", SLEN("gfid"), &gfid_str);
+    if (ret) {
+        goto err;
+    }
+
+    ret = dict_get_strn(dict, "volume-uuid", SLEN("volume-uuid"), &volume_uuid);
+    if (ret) {
+        goto err;
+    }
+
+    gf_uuid_parse((const char *)gfid_str, gfid);
+
+    frame = quotad_aggregator_get_frame_from_req(req);
+    if (frame == NULL) {
+        cli_rsp.op_errno = ENOMEM;
+        goto errx;
+    }
+    state = frame->root->state;
+    state->req_xdata = dict;
+    state->xdata = dict_new();
+    dict = NULL;
+
+    ret = dict_set_int32_sizen(state->xdata, QUOTA_LIMIT_KEY, 42);
+    if (ret)
+        goto err;
+
+    ret = dict_set_int32_sizen(state->xdata, QUOTA_LIMIT_OBJECTS_KEY, 42);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "Failed to set QUOTA_LIMIT_OBJECTS_KEY");
+        goto err;
+    }
+
+    ret = dict_set_int32_sizen(state->xdata, QUOTA_SIZE_KEY, 42);
+    if (ret)
+        goto err;
+
+    ret = dict_set_int32_sizen(state->xdata, GET_ANCESTRY_PATH_KEY, 42);
+    if (ret)
+        goto err;
+
+    ret = qd_nameless_lookup(this, frame, (char *)gfid, state->xdata,
+                             volume_uuid, quotad_aggregator_getlimit_cbk);
+    if (ret) {
+        cli_rsp.op_errno = ret;
+        goto errx;
+    }
+
+    return ret;
 
 err:
-        cli_rsp.op_ret = -1;
-        cli_rsp.op_errno = op_errno;
-        cli_rsp.op_errstr = "";
-
-        quotad_aggregator_getlimit_cbk (this, frame, &cli_rsp);
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
+    cli_rsp.op_errno = op_errno;
+errx:
+    cli_rsp.op_ret = -1;
+    cli_rsp.op_errstr = "";
+
+    quotad_aggregator_getlimit_cbk(this, frame, &cli_rsp);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int
-quotad_aggregator_lookup_cbk (xlator_t *this, call_frame_t *frame,
-                              void *rsp)
+quotad_aggregator_lookup_cbk(xlator_t *this, call_frame_t *frame, void *rsp)
 {
-        quotad_aggregator_submit_reply (frame, frame->local, rsp, NULL, 0, NULL,
-                                        (xdrproc_t)xdr_gfs3_lookup_rsp);
+    quotad_aggregator_submit_reply(frame, frame ? frame->local : NULL, rsp,
+                                   NULL, 0, NULL,
+                                   (xdrproc_t)xdr_gfs3_lookup_rsp);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quotad_aggregator_lookup (rpcsvc_request_t *req)
+quotad_aggregator_lookup(rpcsvc_request_t *req)
 {
-        call_frame_t              *frame = NULL;
-        gfs3_lookup_req            args  = {{0,},};
-        int                        ret   = -1, op_errno = 0;
-        gfs3_lookup_rsp            rsp   = {0,};
-        quotad_aggregator_state_t *state = NULL;
-        xlator_t                  *this  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("quotad-aggregator", req, err);
-
-        this = THIS;
-
-        args.bname           = alloca (req->msg[0].iov_len);
-        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gfs3_lookup_req);
-        if (ret < 0) {
-                rsp.op_errno = EINVAL;
+    call_frame_t *frame = NULL;
+    gfs3_lookup_req args = {
+        {
+            0,
+        },
+    };
+    int i = 0, ret = -1, op_errno = 0;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    char *volume_uuid = NULL;
+
+    GF_VALIDATE_OR_GOTO("quotad-aggregator", req, err);
+
+    this = THIS;
+
+    args.bname = alloca(req->msg[0].iov_len);
+    args.xdata.xdata_val = alloca(req->msg[0].iov_len);
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gfs3_lookup_req);
+    if (ret < 0) {
+        rsp.op_errno = EINVAL;
+        goto err;
+    }
+
+    frame = quotad_aggregator_get_frame_from_req(req);
+    if (frame == NULL) {
+        rsp.op_errno = ENOMEM;
+        goto err;
+    }
+
+    state = frame->root->state;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, dict, (args.xdata.xdata_val),
+                                 (args.xdata.xdata_len), ret, op_errno, err);
+
+    ret = dict_get_str(dict, "volume-uuid", &volume_uuid);
+    if (ret) {
+        goto err;
+    }
+
+    state->xdata = dict_new();
+
+    for (i = 0; qd_ext_xattrs[i]; i++) {
+        if (dict_get(dict, qd_ext_xattrs[i])) {
+            ret = dict_set_uint32(state->xdata, qd_ext_xattrs[i], 1);
+            if (ret < 0)
                 goto err;
         }
+    }
 
-        frame = quotad_aggregator_get_frame_from_req (req);
-        if (frame == NULL) {
-                rsp.op_errno = ENOMEM;
-                goto err;
-        }
-
-        state = frame->root->state;
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, state->xdata,
-                                      (args.xdata.xdata_val),
-                                      (args.xdata.xdata_len), ret,
-                                      op_errno, err);
+    ret = qd_nameless_lookup(this, frame, args.gfid, state->xdata, volume_uuid,
+                             quotad_aggregator_lookup_cbk);
+    if (ret) {
+        rsp.op_errno = ret;
+        goto err;
+    }
 
+    if (dict)
+        dict_unref(dict);
 
-        ret = qd_nameless_lookup (this, frame, &args, state->xdata,
-                                  quotad_aggregator_lookup_cbk);
-        if (ret) {
-                rsp.op_errno = ret;
-                goto err;
-        }
-
-        return ret;
+    return ret;
 
 err:
-        rsp.op_ret = -1;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = -1;
+    rsp.op_errno = op_errno;
+
+    quotad_aggregator_lookup_cbk(this, frame, &rsp);
+    if (dict)
+        dict_unref(dict);
 
-        quotad_aggregator_lookup_cbk (this, frame, &rsp);
-        return ret;
+    return ret;
 }
 
 int
-quotad_aggregator_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
-                              void *data)
+quotad_aggregator_rpc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                             void *data)
 {
-        if (!xl || !data) {
-                gf_log_callingfn ("server", GF_LOG_WARNING,
-                                  "Calling rpc_notify without initializing");
-                goto out;
-        }
+    if (!xl || !data) {
+        gf_log_callingfn("server", GF_LOG_WARNING,
+                         "Calling rpc_notify without initializing");
+        goto out;
+    }
 
-        switch (event) {
+    switch (event) {
         case RPCSVC_EVENT_ACCEPT:
-                break;
+            break;
 
         case RPCSVC_EVENT_DISCONNECT:
-                break;
+            break;
 
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-quotad_aggregator_init (xlator_t *this)
+quotad_aggregator_init(xlator_t *this)
 {
-        quota_priv_t *priv = NULL;
-        int           ret  = -1;
-
-        priv = this->private;
-
-        if (priv->rpcsvc) {
-                /* Listener already created */
-                return 0;
-        }
-
-        ret = dict_set_str (this->options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (this->options, "transport-type", "socket");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (this->options, "transport.socket.listen-path",
-                            "/var/run/gluster/quotad.socket");
-        if (ret)
-                goto out;
-
-        /* RPC related */
-        priv->rpcsvc = rpcsvc_init (this, this->ctx, this->options, 0);
-        if (priv->rpcsvc == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_RPCSVC_INIT_FAILED,
-                        "creation of rpcsvc failed");
-                ret = -1;
-                goto out;
-        }
-
-        ret = rpcsvc_create_listeners (priv->rpcsvc, this->options,
-                                       this->name);
-        if (ret < 1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_RPCSVC_LISTENER_CREATION_FAILED,
-                        "creation of listener failed");
-                ret = -1;
-                goto out;
-        }
+    quota_priv_t *priv = NULL;
+    int ret = -1;
 
-        priv->quotad_aggregator = &quotad_aggregator_prog;
-        quotad_aggregator_prog.options = this->options;
-
-        ret = rpcsvc_program_register (priv->rpcsvc, &quotad_aggregator_prog);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        Q_MSG_RPCSVC_REGISTER_FAILED,
-                        "registration of program (name:%s, prognum:%d, "
-                        "progver:%d) failed", quotad_aggregator_prog.progname,
-                        quotad_aggregator_prog.prognum,
-                        quotad_aggregator_prog.progver);
-                goto out;
-        }
+    priv = this->private;
 
-        ret = 0;
+    if (priv->rpcsvc) {
+        /* Listener already created */
+        return 0;
+    }
+
+    ret = dict_set_nstrn(this->options, "transport.address-family",
+                         SLEN("transport.address-family"), "unix",
+                         SLEN("unix"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(this->options, "transport-type",
+                         SLEN("transport-type"), "socket", SLEN("socket"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(this->options, "transport.socket.listen-path",
+                         SLEN("transport.socket.listen-path"),
+                         "/var/run/gluster/quotad.socket",
+                         SLEN("/var/run/gluster/quotad.socket"));
+    if (ret)
+        goto out;
+
+    /* RPC related */
+    priv->rpcsvc = rpcsvc_init(this, this->ctx, this->options, 0);
+    if (priv->rpcsvc == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_RPCSVC_INIT_FAILED,
+               "creation of rpcsvc failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpcsvc_create_listeners(priv->rpcsvc, this->options, this->name);
+    if (ret < 1) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               Q_MSG_RPCSVC_LISTENER_CREATION_FAILED,
+               "creation of listener failed");
+        ret = -1;
+        goto out;
+    }
+
+    priv->quotad_aggregator = &quotad_aggregator_prog;
+    quotad_aggregator_prog.options = this->options;
+
+    ret = rpcsvc_program_register(priv->rpcsvc, &quotad_aggregator_prog,
+                                  _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_RPCSVC_REGISTER_FAILED,
+               "registration of program (name:%s, prognum:%d, "
+               "progver:%d) failed",
+               quotad_aggregator_prog.progname, quotad_aggregator_prog.prognum,
+               quotad_aggregator_prog.progver);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (ret && priv->rpcsvc) {
-                GF_FREE (priv->rpcsvc);
-                priv->rpcsvc = NULL;
-        }
+    if (ret && priv->rpcsvc) {
+        GF_FREE(priv->rpcsvc);
+        priv->rpcsvc = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
-rpcsvc_actor_t quotad_aggregator_actors[GF_AGGREGATOR_MAXVALUE] = {
-        [GF_AGGREGATOR_NULL]     = {"NULL", GF_AGGREGATOR_NULL, NULL, NULL, 0,
-                                    DRC_NA},
-        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", GF_AGGREGATOR_NULL,
-                                    quotad_aggregator_lookup, NULL, 0, DRC_NA},
-        [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", GF_AGGREGATOR_GETLIMIT,
-                                   quotad_aggregator_getlimit, NULL, 0, DRC_NA},
+static rpcsvc_actor_t quotad_aggregator_actors[GF_AGGREGATOR_MAXVALUE] = {
+    [GF_AGGREGATOR_NULL] = {"NULL", NULL, NULL, GF_AGGREGATOR_NULL, DRC_NA, 0},
+    [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", quotad_aggregator_lookup, NULL,
+                              GF_AGGREGATOR_NULL, DRC_NA, 0},
+    [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", quotad_aggregator_getlimit, NULL,
+                                GF_AGGREGATOR_GETLIMIT, DRC_NA, 0},
 };
 
-
-struct rpcsvc_program quotad_aggregator_prog = {
-        .progname  = "GlusterFS 3.3",
-        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
-        .progver   = GLUSTER_AGGREGATOR_VERSION,
-        .numactors = GF_AGGREGATOR_MAXVALUE,
-        .actors    = quotad_aggregator_actors
-};
+static struct rpcsvc_program quotad_aggregator_prog = {
+    .progname = "GlusterFS 3.3",
+    .prognum = GLUSTER_AGGREGATOR_PROGRAM,
+    .progver = GLUSTER_AGGREGATOR_VERSION,
+    .numactors = GF_AGGREGATOR_MAXVALUE,
+    .actors = quotad_aggregator_actors};
diff --git a/xlators/features/quota/src/quotad-aggregator.h b/xlators/features/quota/src/quotad-aggregator.h
index 5ddea5b3c46..706592c7d50 100644
--- a/xlators/features/quota/src/quotad-aggregator.h
+++ b/xlators/features/quota/src/quotad-aggregator.h
@@ -12,26 +12,27 @@
 #define _QUOTAD_AGGREGATOR_H
 
 #include "quota.h"
-#include "stack.h"
+#include <glusterfs/stack.h>
 #include "glusterfs3-xdr.h"
-#include "inode.h"
+#include <glusterfs/inode.h>
 
 typedef struct {
-        void          *pool;
-        xlator_t      *this;
-	xlator_t      *active_subvol;
-        inode_table_t *itable;
-        loc_t          loc;
-        dict_t        *xdata;
+    void *pool;
+    xlator_t *this;
+    xlator_t *active_subvol;
+    inode_table_t *itable;
+    loc_t loc;
+    dict_t *xdata;
+    dict_t *req_xdata;
 } quotad_aggregator_state_t;
 
-typedef int (*quotad_aggregator_lookup_cbk_t) (xlator_t *this,
-                                               call_frame_t *frame,
-                                               void *rsp);
+typedef int (*quotad_aggregator_lookup_cbk_t)(xlator_t *this,
+                                              call_frame_t *frame, void *rsp);
 int
-qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req,
-                    dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk);
+qd_nameless_lookup(xlator_t *this, call_frame_t *frame, char *gfid,
+                   dict_t *xdata, char *volume_uuid,
+                   quotad_aggregator_lookup_cbk_t lookup_cbk);
 int
-quotad_aggregator_init (xlator_t *this);
+quotad_aggregator_init(xlator_t *this);
 
 #endif
diff --git a/xlators/features/quota/src/quotad-helpers.c b/xlators/features/quota/src/quotad-helpers.c
index 70298fc87f5..51ff1d7e98d 100644
--- a/xlators/features/quota/src/quotad-helpers.c
+++ b/xlators/features/quota/src/quotad-helpers.c
@@ -11,97 +11,97 @@
 #include "quotad-helpers.h"
 
 quotad_aggregator_state_t *
-get_quotad_aggregator_state (xlator_t *this, rpcsvc_request_t *req)
+get_quotad_aggregator_state(xlator_t *this, rpcsvc_request_t *req)
 {
-        quotad_aggregator_state_t *state         = NULL;
-	xlator_t                  *active_subvol = NULL;
-        quota_priv_t              *priv          = NULL;
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *active_subvol = NULL;
+    quota_priv_t *priv = NULL;
 
-        state = (void *)GF_CALLOC (1, sizeof (*state),
-                                   gf_quota_mt_aggregator_state_t);
-        if (!state)
-                return NULL;
+    state = (void *)GF_CALLOC(1, sizeof(*state),
+                              gf_quota_mt_aggregator_state_t);
+    if (!state)
+        return NULL;
 
-	state->this = THIS;
-        priv = this->private;
+    state->this = THIS;
+    priv = this->private;
 
-        LOCK (&priv->lock);
-        {
-                active_subvol = state->active_subvol = FIRST_CHILD (this);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        active_subvol = state->active_subvol = FIRST_CHILD(this);
+    }
+    UNLOCK(&priv->lock);
 
-        if (active_subvol->itable == NULL)
-                active_subvol->itable = inode_table_new (4096, active_subvol);
+    if (active_subvol->itable == NULL)
+        active_subvol->itable = inode_table_new(4096, active_subvol);
 
-	state->itable = active_subvol->itable;
+    state->itable = active_subvol->itable;
 
-        state->pool = this->ctx->pool;
+    state->pool = this->ctx->pool;
 
-        return state;
+    return state;
 }
 
 void
-quotad_aggregator_free_state (quotad_aggregator_state_t *state)
+quotad_aggregator_free_state(quotad_aggregator_state_t *state)
 {
-        if (state->xdata)
-                dict_unref (state->xdata);
+    if (state->xdata)
+        dict_unref(state->xdata);
 
-        GF_FREE (state);
+    if (state->req_xdata)
+        dict_unref(state->req_xdata);
+
+    GF_FREE(state);
 }
 
 call_frame_t *
-quotad_aggregator_alloc_frame (rpcsvc_request_t *req)
+quotad_aggregator_alloc_frame(rpcsvc_request_t *req)
 {
-        call_frame_t              *frame = NULL;
-        quotad_aggregator_state_t *state = NULL;
-        xlator_t                  *this  = NULL;
+    call_frame_t *frame = NULL;
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *this = NULL;
 
-        GF_VALIDATE_OR_GOTO ("server", req, out);
-        GF_VALIDATE_OR_GOTO ("server", req->trans, out);
-        GF_VALIDATE_OR_GOTO ("server", req->svc, out);
-        GF_VALIDATE_OR_GOTO ("server", req->svc->ctx, out);
+    GF_VALIDATE_OR_GOTO("server", req, out);
+    GF_VALIDATE_OR_GOTO("server", req->trans, out);
+    GF_VALIDATE_OR_GOTO("server", req->svc, out);
+    GF_VALIDATE_OR_GOTO("server", req->svc->ctx, out);
 
-        this = req->svc->xl;
+    this = req->svc->xl;
 
-        frame = create_frame (this, req->svc->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(this, req->svc->ctx->pool);
+    if (!frame)
+        goto out;
 
-        state = get_quotad_aggregator_state (this, req);
-        if (!state)
-                goto out;
+    state = get_quotad_aggregator_state(this, req);
+    if (!state)
+        goto out;
 
-        frame->root->state = state;
-        frame->root->unique = 0;
+    frame->root->state = state;
 
-        frame->this = this;
+    frame->this = this;
 out:
-        return frame;
+    return frame;
 }
 
 call_frame_t *
-quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req)
+quotad_aggregator_get_frame_from_req(rpcsvc_request_t *req)
 {
-        call_frame_t *frame  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", req, out);
+    call_frame_t *frame = NULL;
 
-        frame = quotad_aggregator_alloc_frame (req);
-        if (!frame)
-                goto out;
+    GF_VALIDATE_OR_GOTO("server", req, out);
 
-        frame->root->op       = req->procnum;
+    frame = quotad_aggregator_alloc_frame(req);
+    if (!frame)
+        goto out;
 
-        frame->root->unique   = req->xid;
+    frame->root->op = req->procnum;
 
-        frame->root->uid      = req->uid;
-        frame->root->gid      = req->gid;
-        frame->root->pid      = req->pid;
+    frame->root->uid = req->uid;
+    frame->root->gid = req->gid;
+    frame->root->pid = req->pid;
 
-        frame->root->lk_owner = req->lk_owner;
+    frame->root->lk_owner = req->lk_owner;
 
-        frame->local = req;
+    frame->local = req;
 out:
-        return frame;
+    return frame;
 }
diff --git a/xlators/features/quota/src/quotad-helpers.h b/xlators/features/quota/src/quotad-helpers.h
index a10fb7fa82a..bcb39fe845e 100644
--- a/xlators/features/quota/src/quotad-helpers.h
+++ b/xlators/features/quota/src/quotad-helpers.h
@@ -16,9 +16,9 @@
 #include "quotad-aggregator.h"
 
 void
-quotad_aggregator_free_state (quotad_aggregator_state_t *state);
+quotad_aggregator_free_state(quotad_aggregator_state_t *state);
 
 call_frame_t *
-quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req);
+quotad_aggregator_get_frame_from_req(rpcsvc_request_t *req);
 
 #endif
diff --git a/xlators/features/quota/src/quotad.c b/xlators/features/quota/src/quotad.c
index dc2665e9622..643f25c9c2a 100644
--- a/xlators/features/quota/src/quotad.c
+++ b/xlators/features/quota/src/quotad.c
@@ -9,234 +9,237 @@
 */
 #include "quota.h"
 #include "quotad-aggregator.h"
-#include "common-utils.h"
 
 int
-qd_notify (xlator_t *this, int32_t event, void *data, ...)
+qd_notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        switch (event) {
+    switch (event) {
         case GF_EVENT_PARENT_UP:
-                quotad_aggregator_init (this);
-        }
+            quotad_aggregator_init(this);
+    }
 
-        default_notify (this, event, data);
-        return 0;
+    default_notify(this, event, data);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_quota_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (0 != ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting "
-                        "init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_quota_mt_end + 1);
 
+    if (0 != ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Memory accounting "
+               "init failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-qd_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+qd_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-        quotad_aggregator_lookup_cbk_t  lookup_cbk = NULL;
-        gfs3_lookup_rsp                 rsp = {0, };
+    quotad_aggregator_lookup_cbk_t lookup_cbk = NULL;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
 
-        lookup_cbk = cookie;
+    lookup_cbk = cookie;
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
 
-        gf_stat_from_iatt (&rsp.postparent, postparent);
+    gf_stat_from_iatt(&rsp.postparent, postparent);
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
-                                    rsp.xdata.xdata_len, rsp.op_errno, out);
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&rsp.xdata.xdata_val),
+                               rsp.xdata.xdata_len, rsp.op_errno, out);
 
-        gf_stat_from_iatt (&rsp.stat, buf);
+    gf_stat_from_iatt(&rsp.stat, buf);
 
 out:
-        lookup_cbk (this, frame, &rsp);
+    lookup_cbk(this, frame, &rsp);
 
-        GF_FREE (rsp.xdata.xdata_val);
+    GF_FREE(rsp.xdata.xdata_val);
 
-        inode_unref (inode);
+    inode_unref(inode);
 
-        return 0;
+    return 0;
 }
 
 xlator_t *
-qd_find_subvol (xlator_t *this, char *volume_uuid)
+qd_find_subvol(xlator_t *this, char *volume_uuid)
 {
-        xlator_list_t *child  = NULL;
-        xlator_t      *subvol = NULL;
-        char           key[1024];
-        char          *optstr = NULL;
-
-        if (!this || !volume_uuid)
-                goto out;
-
-        for (child = this->children; child; child = child->next) {
-                snprintf(key, 1024, "%s.volume-id", child->xlator->name);
-                if (dict_get_str(this->options, key, &optstr) < 0)
-                        continue;
-
-                if (strcmp (optstr, volume_uuid) == 0) {
-                        subvol = child->xlator;
-                        break;
-                }
+    xlator_list_t *child = NULL;
+    xlator_t *subvol = NULL;
+    char key[1024];
+    int keylen = 0;
+    char *optstr = NULL;
+
+    if (!this || !volume_uuid)
+        goto out;
+
+    for (child = this->children; child; child = child->next) {
+        keylen = snprintf(key, sizeof(key), "%s.volume-id",
+                          child->xlator->name);
+        if (dict_get_strn(this->options, key, keylen, &optstr) < 0)
+            continue;
+
+        if (strcmp(optstr, volume_uuid) == 0) {
+            subvol = child->xlator;
+            break;
         }
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
 int
-qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req,
-                    dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk)
+qd_nameless_lookup(xlator_t *this, call_frame_t *frame, char *gfid,
+                   dict_t *xdata, char *volume_uuid,
+                   quotad_aggregator_lookup_cbk_t lookup_cbk)
 {
-        gfs3_lookup_rsp            rsp         = {0, };
-        int                        op_errno    = 0, ret = -1;
-        loc_t                      loc         = {0, };
-        quotad_aggregator_state_t *state       = NULL;
-        quota_priv_t              *priv        = NULL;
-        xlator_t                  *subvol      = NULL;
-        char                      *volume_uuid = NULL;
-
-        priv = this->private;
-        state = frame->root->state;
-
-        frame->root->op = GF_FOP_LOOKUP;
-
-        loc.inode = inode_new (state->itable);
-        if (loc.inode == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        memcpy (loc.gfid, req->gfid, 16);
-
-        ret = dict_get_str (xdata, "volume-uuid", &volume_uuid);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        ret = dict_set_int8 (xdata, QUOTA_READ_ONLY_KEY, 1);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        Q_MSG_ENOMEM, "dict set failed");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        subvol = qd_find_subvol (this, volume_uuid);
-        if (subvol == NULL) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        STACK_WIND_COOKIE (frame, qd_lookup_cbk, lookup_cbk, subvol,
-                           subvol->fops->lookup, &loc, xdata);
-        return 0;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
+    int op_errno = 0, ret = -1;
+    loc_t loc = {
+        0,
+    };
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *subvol = NULL;
+
+    state = frame->root->state;
+
+    frame->root->op = GF_FOP_LOOKUP;
+
+    loc.inode = inode_new(state->itable);
+    if (loc.inode == NULL) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    memcpy(loc.gfid, gfid, 16);
+
+    ret = dict_set_int8(xdata, QUOTA_READ_ONLY_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set failed");
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    subvol = qd_find_subvol(this, volume_uuid);
+    if (subvol == NULL) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    STACK_WIND_COOKIE(frame, qd_lookup_cbk, lookup_cbk, subvol,
+                      subvol->fops->lookup, &loc, xdata);
+    return 0;
 
 out:
-        rsp.op_ret = -1;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = -1;
+    rsp.op_errno = op_errno;
 
-        lookup_cbk (this, frame, &rsp);
+    lookup_cbk(this, frame, &rsp);
 
-        inode_unref (loc.inode);
-        return 0;
+    inode_unref(loc.inode);
+    return 0;
 }
 
 int
-qd_reconfigure (xlator_t *this, dict_t *options)
+qd_reconfigure(xlator_t *this, dict_t *options)
 {
-        /* As of now quotad is restarted upon alteration of volfile */
-        return 0;
+    /* As of now quotad is restarted upon alteration of volfile */
+    return 0;
 }
 
 void
-qd_fini (xlator_t *this)
+qd_fini(xlator_t *this)
 {
-        quota_priv_t    *priv           = NULL;
+    quota_priv_t *priv = NULL;
 
-        if (this == NULL || this->private == NULL)
-                goto out;
+    if (this == NULL || this->private == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->rpcsvc) {
-                GF_FREE (priv->rpcsvc);
-                priv->rpcsvc = NULL;
-        }
+    if (priv->rpcsvc) {
+        GF_FREE(priv->rpcsvc);
+        priv->rpcsvc = NULL;
+    }
 
-        GF_FREE (priv);
+    GF_FREE(priv);
 
 out:
-        return;
+    return;
 }
 
 int32_t
-qd_init (xlator_t *this)
+qd_init(xlator_t *this)
 {
-        int32_t          ret            = -1;
-        quota_priv_t    *priv           = NULL;
-
-        if (NULL == this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: quota (%s) not configured for min of 1 child",
-                        this->name);
-                ret = -1;
-                goto err;
-        }
+    int32_t ret = -1;
+    quota_priv_t *priv = NULL;
+
+    if (NULL == this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "FATAL: quota (%s) not configured for min of 1 child",
+               this->name);
+        ret = -1;
+        goto err;
+    }
 
-        QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
-        LOCK_INIT (&priv->lock);
+    QUOTA_ALLOC_OR_GOTO(priv, quota_priv_t, err);
+    LOCK_INIT(&priv->lock);
 
-        this->private = priv;
+    this->private = priv;
 
-        ret = 0;
+    ret = 0;
 err:
-        if (ret) {
-                GF_FREE (priv);
-        }
-        return ret;
+    if (ret) {
+        GF_FREE(priv);
+    }
+    return ret;
 }
 
-class_methods_t class_methods = {
-        .init           = qd_init,
-        .fini           = qd_fini,
-        .reconfigure    = qd_reconfigure,
-        .notify         = qd_notify
-};
+struct xlator_fops fops = {};
 
-struct xlator_fops fops = {
-};
+struct xlator_cbks cbks = {};
 
-struct xlator_cbks cbks = {
+struct volume_options options[] = {
+    {.key = {"transport-type"},
+     .value = {"rpc", "rpc-over-rdma", "tcp", "socket", "ib-verbs", "unix",
+               "ib-sdp", "tcp/server", "ib-verbs/server", "rdma",
+               "rdma*([ \t]),*([ \t])socket", "rdma*([ \t]),*([ \t])tcp",
+               "tcp*([ \t]),*([ \t])rdma", "socket*([ \t]),*([ \t])rdma"},
+     .type = GF_OPTION_TYPE_STR},
+    {
+        .key = {"transport.*"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {.key = {NULL}},
 };
 
-struct volume_options options[] = {
-        { .key   = {"transport-type"},
-          .value = {"rpc", "rpc-over-rdma", "tcp", "socket", "ib-verbs",
-                    "unix", "ib-sdp", "tcp/server", "ib-verbs/server", "rdma",
-                    "rdma*([ \t]),*([ \t])socket",
-                    "rdma*([ \t]),*([ \t])tcp",
-                    "tcp*([ \t]),*([ \t])rdma",
-                    "socket*([ \t]),*([ \t])rdma"},
-          .type  = GF_OPTION_TYPE_STR
-        },
-        { .key   = {"transport.*"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        {.key = {NULL}}
+xlator_api_t xlator_api = {
+    .init = qd_init,
+    .fini = qd_fini,
+    .reconfigure = qd_reconfigure,
+    .notify = qd_notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "quotad",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/quota/src/quotad.sym b/xlators/features/quota/src/quotad.sym
deleted file mode 100644
index 0829ffe1584..00000000000
--- a/xlators/features/quota/src/quotad.sym
+++ /dev/null
@@ -1,7 +0,0 @@
-fops
-cbks
-class_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/features/read-only/src/Makefile.am b/xlators/features/read-only/src/Makefile.am
index 3edac3f8a1d..e4a2017ef0d 100644
--- a/xlators/features/read-only/src/Makefile.am
+++ b/xlators/features/read-only/src/Makefile.am
@@ -14,7 +14,8 @@ worm_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 worm_la_SOURCES = read-only-common.c worm-helper.c worm.c
 worm_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/read-only/src/read-only-common.c b/xlators/features/read-only/src/read-only-common.c
index ad2eaaa5e26..9640e7e3eee 100644
--- a/xlators/features/read-only/src/read-only-common.c
+++ b/xlators/features/read-only/src/read-only-common.c
@@ -9,409 +9,398 @@
 */
 #include "read-only.h"
 #include "read-only-mem-types.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 
 gf_boolean_t
-is_readonly_or_worm_enabled (xlator_t *this)
+is_readonly_or_worm_enabled(call_frame_t *frame, xlator_t *this)
 {
-        read_only_priv_t  *priv                     = NULL;
-        gf_boolean_t       readonly_or_worm_enabled = _gf_false;
+    read_only_priv_t *priv = NULL;
+    gf_boolean_t readonly_or_worm_enabled = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        readonly_or_worm_enabled = priv->readonly_or_worm_enabled;
+    readonly_or_worm_enabled = priv->readonly_or_worm_enabled;
 
-        return readonly_or_worm_enabled;
+    if (frame->root->pid < GF_CLIENT_PID_MAX)
+        readonly_or_worm_enabled = _gf_false;
+
+    return readonly_or_worm_enabled;
 }
 
 static int
-_check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
-                           void *tmp)
+_check_key_is_zero_filled(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        if (mem_0filled ((const char *)v->data, v->len)) {
-                /* -1 means, no more iterations, treat as 'break' */
-                return -1;
-        }
-        return 0;
+    if (mem_0filled((const char *)v->data, v->len)) {
+        /* -1 means, no more iterations, treat as 'break' */
+        return -1;
+    }
+    return 0;
 }
 
 int32_t
-ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+ro_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        gf_boolean_t allzero = _gf_false;
-        int     ret = 0;
-
-        ret = dict_foreach (dict, _check_key_is_zero_filled, NULL);
-        if (ret == 0)
-                allzero = _gf_true;
-
-        if (is_readonly_or_worm_enabled (this) && !allzero)
-                STACK_UNWIND_STRICT (xattrop, frame, -1, EROFS, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->xattrop,
-                                 loc, flags, dict, xdata);
-        return 0;
+    gf_boolean_t allzero = _gf_false;
+    int ret = 0;
+
+    ret = dict_foreach(dict, _check_key_is_zero_filled, NULL);
+    if (ret == 0)
+        allzero = _gf_true;
+
+    if (is_readonly_or_worm_enabled(frame, this) && !allzero)
+        STACK_UNWIND_STRICT(xattrop, frame, -1, EROFS, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->xattrop, loc, flags, dict,
+                        xdata);
+    return 0;
 }
 
 int32_t
-ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+ro_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        gf_boolean_t allzero = _gf_false;
-        int     ret = 0;
+    gf_boolean_t allzero = _gf_false;
+    int ret = 0;
 
-        ret = dict_foreach (dict, _check_key_is_zero_filled, NULL);
-        if (ret == 0)
-                allzero = _gf_true;
+    ret = dict_foreach(dict, _check_key_is_zero_filled, NULL);
+    if (ret == 0)
+        allzero = _gf_true;
 
-        if (is_readonly_or_worm_enabled (this) && !allzero)
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, EROFS, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fxattrop,
-                                 fd, flags, dict, xdata);
+    if (is_readonly_or_worm_enabled(frame, this) && !allzero)
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, EROFS, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type, dict_t *xdata)
+ro_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
 {
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD(this)->fops->entrylk,
-                         volume, loc, basename, cmd, type, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type,
-             dict_t *xdata)
+ro_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD(this)->fops->fentrylk,
-                         volume, fd, basename, cmd, type, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+ro_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD(this)->fops->inodelk,
-                         volume, loc, cmd, lock, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->inodelk,
+                    volume, loc, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+ro_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD(this)->fops->finodelk,
-                         volume, fd, cmd, lock, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock, dict_t *xdata)
+ro_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+      struct gf_flock *flock, dict_t *xdata)
 {
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD(this)->fops->lk, fd, cmd, flock,
-                         xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk, fd,
+                    cmd, flock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+ro_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->setattr, loc, stbuf,
-                                 valid, xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(setattr, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                        xdata);
+
+    return 0;
+}
 
-        return 0;
+int32_t
+ro_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+ro_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fsetattr, fd, stbuf,
-                                 valid, xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(truncate, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+ro_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (truncate, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->truncate, loc, offset,
-                                 xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+
+    return 0;
 }
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+ro_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+             off_t offset, size_t len, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->ftruncate, fd, offset,
-                                 xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fallocate, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fallocate, fd, mode, offset,
+                        len, xdata);
+    return 0;
 }
 
 int
-ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata)
+ro_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (mknod, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->mknod, loc, mode,
-                                 rdev, umask, xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(mknod, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                        xdata);
+
+    return 0;
 }
 
-
 int
-ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata)
+ro_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (mkdir, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->mkdir, loc, mode,
-                                 umask, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(mkdir, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-           dict_t *xdata)
+ro_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+          dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (unlink, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->unlink, loc, xflag,
-                                 xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(unlink, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-          dict_t *xdata)
+ro_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+         dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (rmdir, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->rmdir, loc, flags,
-                                 xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(rmdir, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, mode_t umask, dict_t *xdata)
+ro_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (symlink, frame, -1, EROFS, NULL, NULL,
-                                     NULL, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->symlink, linkpath,
-                                 loc, umask, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(symlink, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                        xdata);
+
+    return 0;
 }
 
-
-
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
+ro_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (rename, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->rename, oldloc,
-                                 newloc, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(rename, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+
+    return 0;
 }
 
-
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ro_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (link, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->link, oldloc, newloc,
-                                 xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(link, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                        oldloc, newloc, xdata);
+
+    return 0;
 }
 
 int32_t
-ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+ro_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (create, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->create, loc, flags,
-                                 mode, umask, fd, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(create, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->create, loc, flags, mode,
+                        umask, fd, xdata);
+
+    return 0;
 }
 
-
 static int32_t
-ro_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd, dict_t *xdata)
+ro_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int32_t
-ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+ro_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this) &&
-            (((flags & O_ACCMODE) == O_WRONLY) ||
-              ((flags & O_ACCMODE) == O_RDWR))) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, ro_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this) &&
+        (((flags & O_ACCMODE) == O_WRONLY) ||
+         ((flags & O_ACCMODE) == O_RDWR))) {
+        STACK_UNWIND_STRICT(open, frame, -1, EROFS, NULL, xdata);
+        return 0;
+    }
+
+    STACK_WIND(frame, ro_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 int32_t
-ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+ro_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
-                                 flags, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+ro_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fsyncdir, fd, flags,
-                                 xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata);
+
+    return 0;
 }
 
 int32_t
-ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
-           dict_t *xdata)
+ro_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (writev, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->writev, fd, vector,
-                                 count, off, flags, iobref, xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(writev, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                        flags, iobref, xdata);
+
+    return 0;
 }
 
-
 int32_t
-ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+ro_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (setxattr, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->setxattr, loc, dict,
-                                 flags, xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(setxattr, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+ro_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (removexattr, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->removexattr, loc,
-                                 name, xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(removexattr, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/features/read-only/src/read-only-common.h b/xlators/features/read-only/src/read-only-common.h
index 248ca47b660..5561961ffa2 100644
--- a/xlators/features/read-only/src/read-only-common.h
+++ b/xlators/features/read-only/src/read-only-common.h
@@ -7,107 +7,115 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 gf_boolean_t
-is_readonly_or_worm_enabled (xlator_t *this);
+is_readonly_or_worm_enabled(call_frame_t *frame, xlator_t *this);
 
 int32_t
-ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+ro_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
-ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+ro_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
-ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type, dict_t *xdata);
+ro_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata);
 
 int32_t
-ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type
-             type, dict_t *xdata);
+ro_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
 
 int32_t
-ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+ro_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
-ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+ro_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
-ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock, dict_t *xdata);
+ro_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+      struct gf_flock *flock, dict_t *xdata);
 
 int32_t
-ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid, dict_t *xdata);
+ro_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata);
 
 int32_t
-ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata);
-
+ro_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata);
 
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata);
+ro_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata);
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata);
+ro_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata);
 
 int
-ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata);
+ro_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata);
 
 int
-ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata);
+ro_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata);
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-           dict_t *xdata);
-
-int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+ro_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
           dict_t *xdata);
 
+int
+ro_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+         dict_t *xdata);
 
 int
-ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, mode_t umask, dict_t *xdata);
+ro_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata);
 
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+ro_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata);
 
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+ro_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata);
 
 int32_t
-ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+ro_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
-ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata);
+ro_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata);
 
 int32_t
-ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata);
+ro_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata);
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata);
+ro_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata);
 
 int32_t
-ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata);
+ro_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata);
 
 int32_t
-ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata);
+ro_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata);
+
+int32_t
+ro_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata);
 
 int32_t
-ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata);
+ro_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+             off_t offset, size_t len, dict_t *xdata);
diff --git a/xlators/features/read-only/src/read-only-mem-types.h b/xlators/features/read-only/src/read-only-mem-types.h
index 940700a017d..c67d6c02cd0 100644
--- a/xlators/features/read-only/src/read-only-mem-types.h
+++ b/xlators/features/read-only/src/read-only-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __READONLY_MEM_TYPES_H__
 #define __READONLY_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_read_only_mem_types_ {
-        gf_read_only_mt_priv_t = gf_common_mt_end + 1,
-        gf_read_only_mt_end
+    gf_read_only_mt_priv_t = gf_common_mt_end + 1,
+    gf_read_only_mt_end
 };
 #endif
diff --git a/xlators/features/read-only/src/read-only.c b/xlators/features/read-only/src/read-only.c
index 8733a40abce..48654998e63 100644
--- a/xlators/features/read-only/src/read-only.c
+++ b/xlators/features/read-only/src/read-only.c
@@ -7,124 +7,138 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "defaults.h"
 #include "read-only-common.h"
 #include "read-only-mem-types.h"
 #include "read-only.h"
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_read_only_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
+    ret = xlator_mem_acct_init(this, gf_read_only_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
 
-        return ret;
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	int                     ret     = -1;
-        read_only_priv_t       *priv    = NULL;
+    int ret = -1;
+    read_only_priv_t *priv = NULL;
 
-        if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "translator not configured with exactly one child");
+        return -1;
+    }
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_read_only_mt_priv_t);
-        if (!priv)
-                goto out;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_read_only_mt_priv_t);
+    if (!priv)
+        goto out;
 
-        GF_OPTION_INIT ("read-only", priv->readonly_or_worm_enabled, bool, out);
+    this->private = priv;
 
-        this->private = priv;
-        ret = 0;
+    GF_OPTION_INIT("read-only", priv->readonly_or_worm_enabled, bool, out);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        read_only_priv_t  *priv             = NULL;
-        int                ret              = -1;
-        gf_boolean_t       readonly_or_worm_enabled = _gf_false;
+    read_only_priv_t *priv = NULL;
+    int ret = -1;
+    gf_boolean_t readonly_or_worm_enabled = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        GF_OPTION_RECONF ("read-only", readonly_or_worm_enabled, options, bool,
-                          out);
-        priv->readonly_or_worm_enabled = readonly_or_worm_enabled;
-        ret = 0;
+    GF_OPTION_RECONF("read-only", readonly_or_worm_enabled, options, bool, out);
+    priv->readonly_or_worm_enabled = readonly_or_worm_enabled;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_log(this->name, GF_LOG_DEBUG, "returning %d", ret);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        read_only_priv_t         *priv    = NULL;
+    read_only_priv_t *priv = NULL;
 
-        priv = this->private;
-        if (!priv)
-                return;
+    priv = this->private;
+    if (!priv)
+        return;
 
-        this->private = NULL;
-        GF_FREE (priv);
+    this->private = NULL;
+    GF_FREE(priv);
 
-        return;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .mknod       = ro_mknod,
-        .mkdir       = ro_mkdir,
-        .unlink      = ro_unlink,
-        .rmdir       = ro_rmdir,
-        .symlink     = ro_symlink,
-        .rename      = ro_rename,
-        .link        = ro_link,
-        .truncate    = ro_truncate,
-        .open        = ro_open,
-        .writev      = ro_writev,
-        .setxattr    = ro_setxattr,
-        .fsetxattr   = ro_fsetxattr,
-        .removexattr = ro_removexattr,
-        .fsyncdir    = ro_fsyncdir,
-        .ftruncate   = ro_ftruncate,
-        .create      = ro_create,
-        .setattr     = ro_setattr,
-        .fsetattr    = ro_fsetattr,
-        .xattrop     = ro_xattrop,
-        .fxattrop    = ro_fxattrop,
-        .inodelk     = ro_inodelk,
-        .finodelk    = ro_finodelk,
-        .entrylk     = ro_entrylk,
-        .fentrylk    = ro_fentrylk,
-        .lk          = ro_lk,
+    .mknod = ro_mknod,
+    .mkdir = ro_mkdir,
+    .unlink = ro_unlink,
+    .rmdir = ro_rmdir,
+    .symlink = ro_symlink,
+    .rename = ro_rename,
+    .link = ro_link,
+    .truncate = ro_truncate,
+    .open = ro_open,
+    .writev = ro_writev,
+    .setxattr = ro_setxattr,
+    .fsetxattr = ro_fsetxattr,
+    .removexattr = ro_removexattr,
+    .fsyncdir = ro_fsyncdir,
+    .ftruncate = ro_ftruncate,
+    .create = ro_create,
+    .setattr = ro_setattr,
+    .fsetattr = ro_fsetattr,
+    .xattrop = ro_xattrop,
+    .fxattrop = ro_fxattrop,
+    .inodelk = ro_inodelk,
+    .finodelk = ro_finodelk,
+    .entrylk = ro_entrylk,
+    .fentrylk = ro_fentrylk,
+    .lk = ro_lk,
+    .fallocate = ro_fallocate,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-	{ .key  = {"read-only"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", makes a volume read-only. It is turned "
-                         "\"off\" by default."
-        },
+    {.key = {"read-only"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"on\", makes a volume read-only. It is turned "
+                    "\"off\" by default."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "read-only",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/read-only/src/read-only.h b/xlators/features/read-only/src/read-only.h
index d0263e74179..aced5d3c577 100644
--- a/xlators/features/read-only/src/read-only.h
+++ b/xlators/features/read-only/src/read-only.h
@@ -11,27 +11,27 @@
 #ifndef __READONLY_H__
 #define __READONLY_H__
 
-#include "read-only-mem-types.h"
-#include "xlator.h"
-
+#include <stdint.h>               // for uint64_t, uint8_t
+#include <sys/time.h>             // for time_t
+#include "glusterfs/glusterfs.h"  // for gf_boolean_t
 
 typedef struct {
-        uint8_t worm : 1;
-        uint8_t retain : 1;
-        uint8_t legal_hold :1;
-        uint8_t ret_mode : 1;
-        uint64_t ret_period;
-        uint64_t auto_commit_period;
+    uint8_t worm : 1;
+    uint8_t retain : 1;
+    uint8_t legal_hold : 1;
+    uint8_t ret_mode : 1;
+    int64_t ret_period;
+    int64_t auto_commit_period;
 } worm_reten_state_t;
 
-
 typedef struct {
-        gf_boolean_t          readonly_or_worm_enabled;
-        gf_boolean_t          worm_file;
-        uint64_t              reten_period;
-        uint64_t              com_period;
-        char                  *reten_mode;
-        time_t                start_time;
+    gf_boolean_t readonly_or_worm_enabled;
+    gf_boolean_t worm_file;
+    gf_boolean_t worm_files_deletable;
+    int64_t reten_period;
+    int64_t com_period;
+    int reten_mode;
+    time_t start_time;
 } read_only_priv_t;
 
 #endif
diff --git a/xlators/features/read-only/src/worm-helper.c b/xlators/features/read-only/src/worm-helper.c
index 692899d9379..df45f2a940b 100644
--- a/xlators/features/read-only/src/worm-helper.c
+++ b/xlators/features/read-only/src/worm-helper.c
@@ -9,8 +9,8 @@
 */
 #include "read-only-mem-types.h"
 #include "read-only.h"
-#include "xlator.h"
-#include "syncop.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
 #include "worm-helper.h"
 
 /*Function to check whether file is read-only.
@@ -18,277 +18,259 @@
  * the write protection bits for all the users of the file.
  * Return true if all the write bits are disabled,false otherwise*/
 gf_boolean_t
-gf_worm_write_disabled (struct iatt *stbuf)
+gf_worm_write_disabled(struct iatt *stbuf)
 {
-        gf_boolean_t ret        =       _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("worm", stbuf, out);
+    GF_VALIDATE_OR_GOTO("worm", stbuf, out);
 
-        if (stbuf->ia_prot.owner.write == 0 &&
-            stbuf->ia_prot.group.write == 0 &&
-            stbuf->ia_prot.other.write == 0)
-                ret = _gf_true;
+    if (stbuf->ia_prot.owner.write == 0 && stbuf->ia_prot.group.write == 0 &&
+        stbuf->ia_prot.other.write == 0)
+        ret = _gf_true;
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-worm_init_state (xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr)
+worm_init_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr)
 {
-        int ret                 =      -1;
-        uint64_t start_time     =       0;
-        dict_t *dict            =       NULL;
-
-        GF_VALIDATE_OR_GOTO ("worm", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file_ptr, out);
-
-        start_time = time (NULL);
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "Error creating the dict");
-                goto out;
-        }
-        ret = dict_set_uint64 (dict, "trusted.start_time", start_time);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Error in setting the dict");
-                goto out;
-        }
-        if (fop_with_fd)
-                ret = syncop_fsetxattr (this, (fd_t *)file_ptr, dict, 0,
-                                        NULL, NULL);
-        else
-                ret = syncop_setxattr (this, (loc_t *)file_ptr, dict, 0, NULL,
-                                       NULL);
+    int ret = -1;
+    uint64_t start_time = 0;
+    dict_t *dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+
+    start_time = gf_time();
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "Error creating the dict");
+        goto out;
+    }
+    ret = dict_set_uint64(dict, "trusted.start_time", start_time);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error in setting the dict");
+        goto out;
+    }
+    if (fop_with_fd)
+        ret = syncop_fsetxattr(this, (fd_t *)file_ptr, dict, 0, NULL, NULL);
+    else
+        ret = syncop_setxattr(this, (loc_t *)file_ptr, dict, 0, NULL, NULL);
 out:
-        if (dict)
-                dict_destroy (dict);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-
 /*Function to set the retention state for a file.
  * It loads the WORM/Retention state into the retention_state pointer.*/
 int32_t
-worm_set_state (xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
-                worm_reten_state_t *retention_state, struct iatt *stbuf)
+worm_set_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *retention_state, struct iatt *stbuf)
 {
-        read_only_priv_t *priv   =      NULL;
-        struct iatt stpre        =      {0,};
-        int ret                  =      -1;
-
-        GF_VALIDATE_OR_GOTO ("worm", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file_ptr, out);
-        GF_VALIDATE_OR_GOTO (this->name, retention_state, out);
-        GF_VALIDATE_OR_GOTO (this->name, stbuf, out);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        retention_state->worm = 1;
-        retention_state->retain = 1;
-        retention_state->legal_hold = 0;
-        if (strcmp (priv->reten_mode, "relax") == 0)
-                retention_state->ret_mode = 0;
-        else
-                retention_state->ret_mode = 1;
-        retention_state->ret_period = priv->reten_period;
-        retention_state->auto_commit_period = priv->com_period;
-        if (fop_with_fd)
-                ret = syncop_fstat (this, (fd_t *)file_ptr, &stpre, NULL, NULL);
-        else
-                ret = syncop_stat (this, (loc_t *)file_ptr, &stpre, NULL, NULL);
-        if (ret)
-                goto out;
-        stbuf->ia_mtime = stpre.ia_mtime;
-        stbuf->ia_atime = time (NULL) + retention_state->ret_period;
-
-        if (fop_with_fd)
-                ret = syncop_fsetattr (this, (fd_t *)file_ptr, stbuf,
-                                       GF_SET_ATTR_ATIME, NULL, NULL,
-                                       NULL, NULL);
-        else
-                ret = syncop_setattr (this, (loc_t *)file_ptr, stbuf,
-                                      GF_SET_ATTR_ATIME, NULL, NULL,
-                                      NULL, NULL);
-        if (ret)
-                goto out;
-
-        ret = gf_worm_set_xattr (this, retention_state, fop_with_fd, file_ptr);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Error setting xattr");
-                goto out;
-        }
-        ret = 0;
+    read_only_priv_t *priv = NULL;
+    struct iatt stpre = {
+        0,
+    };
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+    GF_VALIDATE_OR_GOTO(this->name, retention_state, out);
+    GF_VALIDATE_OR_GOTO(this->name, stbuf, out);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    retention_state->worm = 1;
+    retention_state->retain = 1;
+    retention_state->legal_hold = 0;
+    retention_state->ret_mode = priv->reten_mode;
+    retention_state->ret_period = priv->reten_period;
+    retention_state->auto_commit_period = priv->com_period;
+    if (fop_with_fd)
+        ret = syncop_fstat(this, (fd_t *)file_ptr, &stpre, NULL, NULL);
+    else
+        ret = syncop_stat(this, (loc_t *)file_ptr, &stpre, NULL, NULL);
+    if (ret)
+        goto out;
+    stbuf->ia_mtime = stpre.ia_mtime;
+    stbuf->ia_atime = gf_time() + retention_state->ret_period;
+
+    if (fop_with_fd)
+        ret = syncop_fsetattr(this, (fd_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                              NULL, NULL, NULL, NULL);
+    else
+        ret = syncop_setattr(this, (loc_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                             NULL, NULL, NULL, NULL);
+    if (ret)
+        goto out;
+
+    ret = gf_worm_set_xattr(this, retention_state, fop_with_fd, file_ptr);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error setting xattr");
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 /*This function gets the state of the WORM/Retention xattr and loads it in the
  * dict pointer.*/
 int32_t
-worm_get_state (xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
-                worm_reten_state_t *reten_state)
+worm_get_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *reten_state)
 {
-        dict_t *dict    =       NULL;
-        char *val       =       NULL;
-        int ret         =       -1;
-
-        GF_VALIDATE_OR_GOTO ("worm", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file_ptr, out);
-        GF_VALIDATE_OR_GOTO (this->name, reten_state, out);
-
-        if (fop_with_fd)
-                ret = syncop_fgetxattr (this, (fd_t *)file_ptr, &dict,
-                                        "trusted.reten_state", NULL, NULL);
-        else
-                ret = syncop_getxattr (this, (loc_t *)file_ptr, &dict,
-                                       "trusted.reten_state", NULL, NULL);
-        if (ret < 0 || !dict) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_get_str (dict, "trusted.reten_state", &val);
-        if (ret) {
-                ret = -2;
-                gf_log (this->name, GF_LOG_ERROR, "Empty val");
-        }
-        gf_worm_deserialize_state (val, reten_state);
+    dict_t *dict = NULL;
+    char *val = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+    GF_VALIDATE_OR_GOTO(this->name, reten_state, out);
+
+    if (fop_with_fd)
+        ret = syncop_fgetxattr(this, (fd_t *)file_ptr, &dict,
+                               "trusted.reten_state", NULL, NULL);
+    else
+        ret = syncop_getxattr(this, (loc_t *)file_ptr, &dict,
+                              "trusted.reten_state", NULL, NULL);
+    if (ret < 0 || !dict) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_get_str(dict, "trusted.reten_state", &val);
+    if (ret) {
+        ret = -2;
+        gf_log(this->name, GF_LOG_ERROR, "Empty val");
+    }
+    gf_worm_deserialize_state(val, reten_state);
 out:
-        if (dict)
-                dict_unref (dict);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-
 /*Function to lookup the current state of the WORM/Retention profile.
  * Based on the retain value and the access time of the file, the transition
  * from WORM/Retention to WORM is made.*/
 void
-gf_worm_state_lookup (xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
-                      worm_reten_state_t *reten_state, struct iatt *stbuf)
+gf_worm_state_lookup(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+                     worm_reten_state_t *reten_state, struct iatt *stbuf)
 {
-        int ret                                 =       -1;
-
-        GF_VALIDATE_OR_GOTO ("worm", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file_ptr, out);
-        GF_VALIDATE_OR_GOTO (this->name, reten_state, out);
-        GF_VALIDATE_OR_GOTO (this->name, stbuf, out);
-
-        stbuf->ia_atime -= reten_state->ret_period;
-        reten_state->retain = 0;
-        reten_state->ret_period = 0;
-        reten_state->auto_commit_period = 0;
-        ret = gf_worm_set_xattr (this, reten_state, fop_with_fd, file_ptr);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Error setting xattr");
-                goto out;
-        }
-
-        if (fop_with_fd)
-                ret = syncop_fsetattr (this, (fd_t *)file_ptr, stbuf,
-                                       GF_SET_ATTR_ATIME, NULL, NULL,
-                                       NULL, NULL);
-        else
-                ret = syncop_setattr (this, (loc_t *)file_ptr, stbuf,
-                                      GF_SET_ATTR_ATIME, NULL, NULL,
-                                      NULL, NULL);
-        if (ret)
-                goto out;
-        gf_log (this->name, GF_LOG_INFO, "Retention state reset");
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+    GF_VALIDATE_OR_GOTO(this->name, reten_state, out);
+    GF_VALIDATE_OR_GOTO(this->name, stbuf, out);
+
+    stbuf->ia_atime -= reten_state->ret_period;
+    reten_state->retain = 0;
+    reten_state->ret_period = 0;
+    reten_state->auto_commit_period = 0;
+    ret = gf_worm_set_xattr(this, reten_state, fop_with_fd, file_ptr);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error setting xattr");
+        goto out;
+    }
+
+    if (fop_with_fd)
+        ret = syncop_fsetattr(this, (fd_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                              NULL, NULL, NULL, NULL);
+    else
+        ret = syncop_setattr(this, (loc_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                             NULL, NULL, NULL, NULL);
+    if (ret)
+        goto out;
+    gf_log(this->name, GF_LOG_INFO, "Retention state reset");
 out:
-        return;
+    return;
 }
 
-
 /*This function serializes and stores the WORM/Retention state of a file in an
  * uint64_t variable by setting the bits using the bitwise operations.*/
 void
-gf_worm_serialize_state (worm_reten_state_t *reten_state, char *val)
+gf_worm_serialize_state(worm_reten_state_t *reten_state, char *val)
 {
-        uint32_t state     =       0;
+    uint32_t state = 0;
 
-        GF_VALIDATE_OR_GOTO ("worm", reten_state, out);
-        GF_VALIDATE_OR_GOTO ("worm", val, out);
+    GF_VALIDATE_OR_GOTO("worm", reten_state, out);
+    GF_VALIDATE_OR_GOTO("worm", val, out);
 
-        state |= reten_state->worm << 0;
-        state |= reten_state->retain << 1;
-        state |= reten_state->legal_hold << 2;
-        state |= reten_state->ret_mode << 3;
-        sprintf (val, "%d/%ld/%ld", state, reten_state->ret_period,
-                 reten_state->auto_commit_period);
+    state |= reten_state->worm << 0;
+    state |= reten_state->retain << 1;
+    state |= reten_state->legal_hold << 2;
+    state |= reten_state->ret_mode << 3;
+    sprintf(val, "%d/%" PRIu64 "/%" PRIu64, state, reten_state->ret_period,
+            reten_state->auto_commit_period);
 
 out:
-        return;
+    return;
 }
 
-
 /*This function deserializes the data stored in the xattr of the file and loads
  * the value to the reten_state structure.*/
 void
-gf_worm_deserialize_state (char *val, worm_reten_state_t *reten_state)
+gf_worm_deserialize_state(char *val, worm_reten_state_t *reten_state)
 {
-        char *token     =       NULL;
-        uint32_t state  =       0;
-
-        GF_VALIDATE_OR_GOTO ("worm", val, out);
-        GF_VALIDATE_OR_GOTO ("worm", reten_state, out);
-
-        token = strtok (val, "/");
-        state = atoi (token);
-        reten_state->worm = (state >> 0) & 1;
-        reten_state->retain = (state >> 1) & 1;
-        reten_state->legal_hold = (state >> 2) & 1;
-        reten_state->ret_mode = (state >> 3) & 1;
-        token = strtok (NULL, "/");
-        reten_state->ret_period = atoi (token);
-        token = strtok (NULL, "/");
-        reten_state->auto_commit_period = atoi (token);
+    char *token = NULL;
+    uint32_t state = 0;
+
+    GF_VALIDATE_OR_GOTO("worm", val, out);
+    GF_VALIDATE_OR_GOTO("worm", reten_state, out);
+
+    token = strtok(val, "/");
+    state = atoi(token);
+    reten_state->worm = (state >> 0) & 1;
+    reten_state->retain = (state >> 1) & 1;
+    reten_state->legal_hold = (state >> 2) & 1;
+    reten_state->ret_mode = (state >> 3) & 1;
+    token = strtok(NULL, "/");
+    reten_state->ret_period = atoi(token);
+    token = strtok(NULL, "/");
+    reten_state->auto_commit_period = atoi(token);
 
 out:
-        return;
+    return;
 }
 
-
 /*Function to set the xattr for a file.
  * If the xattr is already present then it will replace that.*/
 int32_t
-gf_worm_set_xattr (xlator_t *this, worm_reten_state_t *reten_state,
-                   gf_boolean_t fop_with_fd, void *file_ptr)
+gf_worm_set_xattr(xlator_t *this, worm_reten_state_t *reten_state,
+                  gf_boolean_t fop_with_fd, void *file_ptr)
 {
-        char val[100]   =        "";
-        int ret         =        -1;
-        dict_t *dict    =        NULL;
-
-        GF_VALIDATE_OR_GOTO ("worm", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, reten_state, out);
-        GF_VALIDATE_OR_GOTO (this->name, file_ptr, out);
-
-        gf_worm_serialize_state (reten_state, val);
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "Error creating the dict");
-                goto out;
-        }
-        ret = dict_set_str (dict, "trusted.reten_state", val);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Error in setting the dict");
-                goto out;
-        }
-        if (fop_with_fd)
-                ret = syncop_fsetxattr (this, (fd_t *)file_ptr, dict, 0,
-                                        NULL, NULL);
-        else
-                ret = syncop_setxattr (this, (loc_t *)file_ptr, dict, 0, NULL,
-                                       NULL);
+    char val[100] = "";
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, reten_state, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+
+    gf_worm_serialize_state(reten_state, val);
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "Error creating the dict");
+        goto out;
+    }
+    ret = dict_set_str(dict, "trusted.reten_state", val);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error in setting the dict");
+        goto out;
+    }
+    if (fop_with_fd)
+        ret = syncop_fsetxattr(this, (fd_t *)file_ptr, dict, 0, NULL, NULL);
+    else
+        ret = syncop_setxattr(this, (loc_t *)file_ptr, dict, 0, NULL, NULL);
 out:
-        if (dict)
-                dict_destroy (dict);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-
-/*This function checks whether a file's timeout is happend for the state
+/*This function checks whether a file's timeout is happened for the state
  * transition and if yes, then it will do the transition from the current state
  * to the appropriate state. It also decides whether to continue or to block
  * the FOP.
@@ -299,117 +281,115 @@ out:
  * 2: Blocks the FOP if any operation fails while doing the state transition or
  *    fails to get the state of the file.*/
 int
-gf_worm_state_transition (xlator_t *this, gf_boolean_t fop_with_fd,
-                          void *file_ptr, glusterfs_fop_t op)
+gf_worm_state_transition(xlator_t *this, gf_boolean_t fop_with_fd,
+                         void *file_ptr, glusterfs_fop_t op)
 {
-        int op_errno                      =       EROFS;
-        int ret                           =       -1;
-        uint64_t com_period               =       0;
-        uint64_t ret_period               =       0;
-        uint64_t start_time               =       0;
-        dict_t *dict                      =       NULL;
-        worm_reten_state_t reten_state    =       {0,};
-        read_only_priv_t *priv            =       NULL;
-        struct iatt stbuf                 =       {0,};
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (fop_with_fd)
-                ret = syncop_fgetxattr (this, (fd_t *)file_ptr, &dict,
-                                        "trusted.start_time", NULL, NULL);
-        else
-                ret = syncop_getxattr (this, (loc_t *)file_ptr, &dict,
-                                       "trusted.start_time", NULL, NULL);
-        if (ret < 0 || !dict) {
+    int op_errno = EROFS;
+    int ret = -1;
+    time_t now = 0;
+    uint64_t com_period = 0;
+    uint64_t start_time = 0;
+    dict_t *dict = NULL;
+    worm_reten_state_t reten_state = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (fop_with_fd)
+        ret = syncop_fgetxattr(this, (fd_t *)file_ptr, &dict,
+                               "trusted.start_time", NULL, NULL);
+    else
+        ret = syncop_getxattr(this, (loc_t *)file_ptr, &dict,
+                              "trusted.start_time", NULL, NULL);
+    if (ret < 0 || !dict) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0, "Error getting xattr");
+        goto out;
+    }
+    ret = dict_get_uint64(dict, "trusted.start_time", &start_time);
+    if (ret) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0, "Error getting start time");
+        goto out;
+    }
+
+    com_period = priv->com_period;
+    if (fop_with_fd)
+        ret = syncop_fstat(this, (fd_t *)file_ptr, &stbuf, NULL, NULL);
+    else
+        ret = syncop_stat(this, (loc_t *)file_ptr, &stbuf, NULL, NULL);
+    if (ret) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0, "Error getting file stat");
+        goto out;
+    }
+
+    ret = worm_get_state(this, fop_with_fd, file_ptr, &reten_state);
+    if (ret == -2) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "Error getting worm/retention state");
+        goto out;
+    }
+
+    now = gf_time();
+
+    if (ret == -1 && (now - start_time) >= com_period) {
+        if ((now - stbuf.ia_mtime) >= com_period) {
+            ret = worm_set_state(this, fop_with_fd, file_ptr, &reten_state,
+                                 &stbuf);
+            if (ret) {
                 op_errno = ret;
-                gf_msg (this->name, GF_LOG_ERROR, -ret, 0,
-                        "Error getting xattr");
-                goto out;
-        }
-        ret = dict_get_uint64 (dict, "trusted.start_time", &start_time);
-        if (ret) {
-                op_errno = ret;
-                gf_msg (this->name, GF_LOG_ERROR, -ret, 0,
-                        "Error getting start time");
-                goto out;
-        }
-
-        com_period = priv->com_period;
-        if (fop_with_fd)
-                ret = syncop_fstat (this, (fd_t *)file_ptr, &stbuf, NULL, NULL);
-        else
-                ret = syncop_stat (this, (loc_t *)file_ptr, &stbuf, NULL, NULL);
-        if (ret) {
-                op_errno = ret;
-                gf_msg (this->name, GF_LOG_ERROR, -ret, 0,
-                        "Error getting file stat");
-                goto out;
-        }
-
-        ret = worm_get_state (this, fop_with_fd, file_ptr, &reten_state);
-        if (ret == -2) {
-                op_errno = ret;
-                gf_msg (this->name, GF_LOG_ERROR, -ret, 0,
-                        "Error getting worm/retention state");
-                goto out;
-        }
-
-        if (ret == -1 && (time (NULL) - start_time) >= com_period) {
-                ret_period = priv->reten_period;
-                if ((time (NULL) - stbuf.ia_mtime) >= ret_period) {
-                        ret = worm_set_state(this, fop_with_fd, file_ptr,
-                                             &reten_state, &stbuf);
-                        if (ret) {
-                                op_errno = ret;
-                                gf_msg (this->name, GF_LOG_ERROR, -ret, 0,
-                                        "Error setting worm/retention state");
-                                goto out;
-                        }
-                        goto out;
-                } else {
-                        op_errno = 0;
-                        goto out;
-                }
-        } else if (ret == -1 && (time (NULL) - start_time)
-                   < com_period) {
-                op_errno = 0;
-                goto out;
-        } else if (reten_state.retain &&
-                   ((time (NULL) >= stbuf.ia_atime))) {
-                gf_worm_state_lookup (this, fop_with_fd, file_ptr,
-                                      &reten_state, &stbuf);
-        }
-        if (reten_state.worm && !reten_state.retain &&
-                 op == GF_FOP_UNLINK) {
-                op_errno = 0;
+                gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+                       "Error setting worm/retention state");
                 goto out;
+            }
+            goto out;
+        } else {
+            op_errno = 0;
+            goto out;
         }
+    } else if (ret == -1 && (now - start_time) < com_period) {
+        op_errno = 0;
+        goto out;
+    } else if (reten_state.retain && ((now >= stbuf.ia_atime))) {
+        gf_worm_state_lookup(this, fop_with_fd, file_ptr, &reten_state, &stbuf);
+    }
+    if (reten_state.worm && !reten_state.retain && priv->worm_files_deletable &&
+        op == GF_FOP_UNLINK) {
+        op_errno = 0;
+        goto out;
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
-        return op_errno;
+    if (dict)
+        dict_unref(dict);
+    return op_errno;
 }
 
-
 /*Function to check whether a file is independently WORMed (i.e., file level
  * WORM is set on the file). */
 int32_t
-is_wormfile (xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr)
+is_wormfile(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr)
 {
-        int ret         =       -1;
-        dict_t *dict    =       NULL;
-
-        if (fop_with_fd)
-                ret = syncop_fgetxattr (this, (fd_t *)file_ptr, &dict,
-                                       "trusted.worm_file", NULL, NULL);
-        else
-                ret = syncop_getxattr (this, (loc_t *)file_ptr, &dict,
-                                       "trusted.worm_file", NULL, NULL);
-        if (dict) {
-                ret = 0;
-                dict_unref (dict);
-        }
-        return ret;
-}
\ No newline at end of file
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (fop_with_fd)
+        ret = syncop_fgetxattr(this, (fd_t *)file_ptr, &dict,
+                               "trusted.worm_file", NULL, NULL);
+    else
+        ret = syncop_getxattr(this, (loc_t *)file_ptr, &dict,
+                              "trusted.worm_file", NULL, NULL);
+    if (dict) {
+        ret = 0;
+        dict_unref(dict);
+    }
+    return ret;
+}
diff --git a/xlators/features/read-only/src/worm-helper.h b/xlators/features/read-only/src/worm-helper.h
index 745df8294c3..b42f8d2b40c 100644
--- a/xlators/features/read-only/src/worm-helper.h
+++ b/xlators/features/read-only/src/worm-helper.h
@@ -8,30 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-gf_boolean_t gf_worm_write_disabled (struct iatt *stbuf);
+gf_boolean_t
+gf_worm_write_disabled(struct iatt *stbuf);
 
-int32_t worm_init_state (xlator_t *this, gf_boolean_t fop_with_fd,
-                         void *file_ptr);
+int32_t
+worm_init_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr);
 
-int32_t worm_set_state (xlator_t *this, gf_boolean_t fop_with_fd,
-                        void *file_ptr, worm_reten_state_t *retention_state,
-                        struct iatt *stbuf);
+int32_t
+worm_set_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *retention_state, struct iatt *stbuf);
 
-int32_t worm_get_state (xlator_t *this, gf_boolean_t fop_with_fd,
-                        void *file_ptr, worm_reten_state_t *reten_state);
+int32_t
+worm_get_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *reten_state);
 
-void gf_worm_state_lookup (xlator_t *this, gf_boolean_t fop_with_fd,
-                           void *file_ptr, worm_reten_state_t *reten_state,
-                           struct iatt *stbuf);
+void
+gf_worm_state_lookup(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+                     worm_reten_state_t *reten_state, struct iatt *stbuf);
 
-void gf_worm_serialize_state (worm_reten_state_t *reten_state, char *val);
+void
+gf_worm_serialize_state(worm_reten_state_t *reten_state, char *val);
 
-void gf_worm_deserialize_state (char *val, worm_reten_state_t *reten_state);
+void
+gf_worm_deserialize_state(char *val, worm_reten_state_t *reten_state);
 
-int32_t gf_worm_set_xattr (xlator_t *this, worm_reten_state_t *reten_state,
-                           gf_boolean_t fop_with_fd, void *file_ptr);
+int32_t
+gf_worm_set_xattr(xlator_t *this, worm_reten_state_t *reten_state,
+                  gf_boolean_t fop_with_fd, void *file_ptr);
 
-int gf_worm_state_transition (xlator_t *this, gf_boolean_t fop_with_fd,
-                              void *file_ptr, glusterfs_fop_t op);
+int
+gf_worm_state_transition(xlator_t *this, gf_boolean_t fop_with_fd,
+                         void *file_ptr, glusterfs_fop_t op);
 
-int32_t is_wormfile (xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr);
+int32_t
+is_wormfile(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr);
diff --git a/xlators/features/read-only/src/worm.c b/xlators/features/read-only/src/worm.c
index ad06f46a4e4..1cc5526d5cd 100644
--- a/xlators/features/read-only/src/worm.c
+++ b/xlators/features/read-only/src/worm.c
@@ -7,597 +7,716 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "read-only-common.h"
 #include "read-only-mem-types.h"
 #include "read-only.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "worm-helper.h"
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_read_only_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
+    ret = xlator_mem_acct_init(this, gf_read_only_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
 
-        return ret;
+    return ret;
 }
 
-
 static int32_t
-worm_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           fd_t *fd, dict_t *xdata)
+worm_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          fd_t *fd, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this) &&
-            (flags & (O_WRONLY | O_RDWR | O_APPEND))) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, NULL);
-                return 0;
-        }
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    if (is_readonly_or_worm_enabled(frame, this) &&
+        (flags & (O_WRONLY | O_RDWR | O_APPEND | O_TRUNC))) {
+        STACK_UNWIND_STRICT(open, frame, -1, EROFS, NULL, NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+    return 0;
+}
 
 static int32_t
-worm_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
+worm_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        int op_errno                =       EROFS;
-        read_only_priv_t *priv      =       NULL;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (is_readonly_or_worm_enabled (this))
-                goto out;
-        if (!priv->worm_file) {
-                op_errno = 0;
-                goto out;
-        }
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
 
-        gf_uuid_copy (oldloc->gfid, oldloc->inode->gfid);
-        if (is_wormfile (this, _gf_false, oldloc)) {
-                op_errno = 0;
-                goto out;
-        }
-        op_errno = gf_worm_state_transition (this, _gf_false, oldloc,
-                                             GF_FOP_LINK);
+    gf_uuid_copy(oldloc->gfid, oldloc->inode->gfid);
+    if (is_wormfile(this, _gf_false, oldloc)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, oldloc, GF_FOP_LINK);
 
 out:
-        if (op_errno)
-                STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->link,
-                                 oldloc, newloc, xdata);
-        return 0;
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                        oldloc, newloc, xdata);
+    return 0;
 }
 
+static int32_t
+worm_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            dict_t *xdata)
+{
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this)) {
+        goto out;
+    }
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+
+    gf_uuid_copy(loc->gfid, loc->inode->gfid);
+    if (is_wormfile(this, _gf_false, loc)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, loc, GF_FOP_UNLINK);
+out:
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->unlink, loc, flags, xdata);
+    return 0;
+}
 
 static int32_t
-worm_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-             dict_t *xdata)
+worm_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        int op_errno                =       EROFS;
-        read_only_priv_t *priv      =       NULL;
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
 
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (is_readonly_or_worm_enabled (this)) {
-                goto out;
-        }
-        if (!priv->worm_file) {
+    gf_uuid_copy(oldloc->gfid, oldloc->inode->gfid);
+    if (is_wormfile(this, _gf_false, oldloc)) {
+        op_errno = 0;
+        goto check_newloc;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, oldloc, GF_FOP_RENAME);
+
+    if (op_errno == 0) {
+    check_newloc:
+        if (newloc->inode != NULL) {
+            gf_uuid_copy(newloc->gfid, newloc->inode->gfid);
+            if (is_wormfile(this, _gf_false, newloc)) {
                 op_errno = 0;
                 goto out;
+            }
+            op_errno = gf_worm_state_transition(this, _gf_false, newloc,
+                                                GF_FOP_RENAME);
         }
+    }
 
-        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-        if (is_wormfile (this, _gf_false, loc)) {
-                op_errno = 0;
-                goto out;
-        }
-        op_errno = gf_worm_state_transition (this, _gf_false, loc,
-                                             GF_FOP_UNLINK);
 out:
-        if (op_errno)
-                STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->unlink,
-                                 loc, flags, xdata);
-        return 0;
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 static int32_t
-worm_rename (call_frame_t *frame, xlator_t *this,
-             loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+worm_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
-        int op_errno                =       EROFS;
-        read_only_priv_t *priv      =       NULL;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (is_readonly_or_worm_enabled (this))
-                goto out;
-        if (!priv->worm_file) {
-                op_errno = 0;
-                goto out;
-        }
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
 
-        gf_uuid_copy (oldloc->gfid, oldloc->inode->gfid);
-        if (is_wormfile (this, _gf_false, oldloc)) {
-                op_errno = 0;
-                goto out;
-        }
-        op_errno = gf_worm_state_transition (this, _gf_false, oldloc,
-                                           GF_FOP_RENAME);
+    if (is_wormfile(this, _gf_false, loc)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, loc, GF_FOP_TRUNCATE);
 
 out:
-        if (op_errno)
-                STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL, NULL);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->rename,
-                                 oldloc, newloc, xdata);
-        return 0;
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 static int32_t
-worm_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+worm_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                dict_t *xdata)
 {
-        int op_errno                =       EROFS;
-        read_only_priv_t *priv      =       NULL;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (is_readonly_or_worm_enabled (this))
-                goto out;
-        if (!priv->worm_file) {
-                op_errno = 0;
-                goto out;
-        }
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
 
-        if (is_wormfile (this, _gf_false, loc)) {
-                op_errno = 0;
-                goto out;
-        }
-        op_errno = gf_worm_state_transition (this, _gf_false, loc,
-                                           GF_FOP_TRUNCATE);
+    if (is_wormfile(this, _gf_true, fd)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_true, fd, GF_FOP_FTRUNCATE);
 
 out:
-        if (op_errno)
-                STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL,
-                                     NULL);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->truncate,
-                                 loc, offset, xdata);
-        return 0;
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 static int32_t
-worm_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              struct iatt *stbuf, int32_t valid, dict_t *xdata)
+worm_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        gf_boolean_t rd_only            =       _gf_false;
-        worm_reten_state_t reten_state  =       {0,};
-        struct iatt stpre               =       {0,};
-        read_only_priv_t *priv          =       NULL;
-        int op_errno                    =       EROFS;
-        int ret                         =       -1;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (!priv->worm_file) {
-                op_errno = 0;
-                goto out;
-        }
+    gf_boolean_t rd_only = _gf_false;
+    worm_reten_state_t reten_state = {
+        0,
+    };
+    struct iatt stpre = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    int op_errno = EROFS;
+    int ret = -1;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (!priv->worm_file) {
+        op_errno = 0;
+        goto out;
+    }
 
-        if (is_wormfile (this, _gf_false, loc)) {
-                op_errno = 0;
-                goto out;
+    if (is_wormfile(this, _gf_false, loc)) {
+        op_errno = 0;
+        goto out;
+    }
+    if (valid & GF_SET_ATTR_MODE) {
+        rd_only = gf_worm_write_disabled(stbuf);
+        if (!rd_only) {
+            op_errno = 0;
+            goto out;
         }
-        if (valid & GF_SET_ATTR_MODE) {
-                rd_only = gf_worm_write_disabled (stbuf);
-                if (!rd_only) {
-                        op_errno = 0;
-                        goto out;
-                }
 
-                ret = worm_set_state (this, _gf_false, loc,
-                                           &reten_state, stbuf);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting worm state");
-                        goto out;
-                }
-        } else if (valid & GF_SET_ATTR_ATIME) {
-                ret = worm_get_state (this, _gf_false, loc, &reten_state);
-                if (ret) {
-                        op_errno = 0;
-                        goto out;
+        ret = worm_set_state(this, _gf_false, loc, &reten_state, stbuf);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error setting worm state");
+            goto out;
+        }
+    } else if (valid & GF_SET_ATTR_ATIME) {
+        ret = worm_get_state(this, _gf_false, loc, &reten_state);
+        if (ret) {
+            op_errno = 0;
+            goto out;
+        }
+        if (reten_state.retain) {
+            ret = syncop_stat(this, loc, &stpre, NULL, NULL);
+            if (ret)
+                goto out;
+            if (reten_state.ret_mode == 0) {
+                if (stbuf->ia_atime < stpre.ia_mtime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot set atime less than "
+                           "the mtime for a WORM-Retained "
+                           "file");
+                    goto out;
                 }
-                if (reten_state.retain) {
-                        ret = syncop_stat (this, loc, &stpre, NULL, NULL);
-                        if (ret)
-                                goto out;
-                        if (reten_state.ret_mode == 0) {
-                                if (stbuf->ia_atime < stpre.ia_mtime) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Cannot set atime less than "
-                                                "the mtime for a WORM-Retained "
-                                                "file");
-                                        goto out;
-                                }
-                        } else {
-                                if (stbuf->ia_atime < stpre.ia_atime) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Cannot decrease the atime of a"
-                                                " WORM-Retained file in "
-                                                "Enterprise mode");
-                                        goto out;
-                                }
-                        }
-                        stbuf->ia_mtime = stpre.ia_mtime;
+            } else {
+                if (stbuf->ia_atime < stpre.ia_atime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot decrease the atime of a"
+                           " WORM-Retained file in "
+                           "Enterprise mode");
+                    goto out;
                 }
+            }
+            reten_state.ret_period = reten_state.ret_period + stbuf->ia_atime -
+                                     stpre.ia_atime;
+            ret = gf_worm_set_xattr(this, &reten_state, _gf_false, loc);
+            if (ret) {
+                goto out;
+            }
+            stbuf->ia_mtime = stpre.ia_mtime;
         }
-        op_errno = 0;
+    }
+    op_errno = 0;
 
 out:
-        if (op_errno)
-                STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL,
-                                     NULL);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->setattr,
-                                 loc, stbuf, valid, xdata);
-        return 0;
+    if (op_errno)
+        STACK_UNWIND_STRICT(setattr, frame, -1, EROFS, NULL, NULL, NULL);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                        xdata);
+    return 0;
 }
 
-
 static int32_t
-worm_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+worm_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+              int32_t valid, dict_t *xdata)
 {
-        gf_boolean_t rd_only            =       _gf_false;
-        worm_reten_state_t reten_state  =       {0,};
-        struct iatt stpre               =       {0,};
-        read_only_priv_t *priv          =       NULL;
-        int op_errno                    =       EROFS;
-        int ret                         =       -1;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (!priv->worm_file) {
-                op_errno = 0;
-                goto out;
-        }
+    gf_boolean_t rd_only = _gf_false;
+    worm_reten_state_t reten_state = {
+        0,
+    };
+    struct iatt stpre = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    int op_errno = EROFS;
+    int ret = -1;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (!priv->worm_file) {
+        op_errno = 0;
+        goto out;
+    }
 
-        if (is_wormfile (this, _gf_true, fd)) {
-                op_errno = 0;
-                goto out;
+    if (is_wormfile(this, _gf_true, fd)) {
+        op_errno = 0;
+        goto out;
+    }
+    if (valid & GF_SET_ATTR_MODE) {
+        rd_only = gf_worm_write_disabled(stbuf);
+        if (!rd_only) {
+            op_errno = 0;
+            goto out;
         }
-        if (valid & GF_SET_ATTR_MODE) {
-                rd_only = gf_worm_write_disabled (stbuf);
-                if (!rd_only) {
-                        op_errno = 0;
-                        goto out;
-                }
 
-                ret = worm_set_state (this, _gf_true, fd,
-                                      &reten_state, stbuf);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting worm state");
-                        goto out;
-                }
-        } else if (valid & GF_SET_ATTR_ATIME) {
-                ret = worm_get_state (this, _gf_true, fd, &reten_state);
-                if (ret) {
-                        op_errno = 0;
-                        goto out;
+        ret = worm_set_state(this, _gf_true, fd, &reten_state, stbuf);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error setting worm state");
+            goto out;
+        }
+    } else if (valid & GF_SET_ATTR_ATIME) {
+        ret = worm_get_state(this, _gf_true, fd, &reten_state);
+        if (ret) {
+            op_errno = 0;
+            goto out;
+        }
+        if (reten_state.retain) {
+            ret = syncop_fstat(this, fd, &stpre, NULL, NULL);
+            if (ret)
+                goto out;
+            if (reten_state.ret_mode == 0) {
+                if (stbuf->ia_atime < stpre.ia_mtime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot set atime less than "
+                           "the mtime for a WORM-Retained "
+                           "file");
+                    goto out;
                 }
-                if (reten_state.retain) {
-                        ret = syncop_fstat (this, fd, &stpre, NULL, NULL);
-                        if (ret)
-                                goto out;
-                        if (reten_state.ret_mode == 0) {
-                                if (stbuf->ia_atime < stpre.ia_mtime) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Cannot set atime less than "
-                                                "the mtime for a WORM-Retained "
-                                                "file");
-                                        goto out;
-                                }
-                        } else {
-                                if (stbuf->ia_atime < stpre.ia_atime) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Cannot decrease the atime of a"
-                                                " WORM-Retained file in "
-                                                "Enterprise mode");
-                                        goto out;
-                                }
-                        }
-                        stbuf->ia_mtime = stpre.ia_mtime;
+            } else {
+                if (stbuf->ia_atime < stpre.ia_atime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot decrease the atime of a"
+                           " WORM-Retained file in "
+                           "Enterprise mode");
+                    goto out;
                 }
+            }
+            reten_state.ret_period = reten_state.ret_period + stbuf->ia_atime -
+                                     stpre.ia_atime;
+            ret = gf_worm_set_xattr(this, &reten_state, _gf_true, fd);
+            if (ret) {
+                goto out;
+            }
+
+            stbuf->ia_mtime = stpre.ia_mtime;
         }
-        op_errno = 0;
+    }
+    op_errno = 0;
 
 out:
-        if (op_errno)
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL,
-                                     NULL);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->fsetattr,
-                                 fd, stbuf, valid, xdata);
-        return 0;
+    if (op_errno)
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid,
+                        xdata);
+    return 0;
 }
 
-
 static int32_t
-worm_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
-             struct iobref *iobref, dict_t *xdata)
+worm_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+            int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+            dict_t *xdata)
 {
-        worm_reten_state_t reten_state    =       {0,};
-        read_only_priv_t *priv            =       NULL;
-        int op_errno                      =       EROFS;
-        int ret                           =       -1;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (!priv->worm_file) {
-                op_errno = 0;
-                goto out;
-        }
-        if (is_wormfile (this, _gf_true, fd)) {
-                op_errno = 0;
-                goto out;
-        }
-        ret = worm_get_state (this, _gf_true, fd, &reten_state);
-        if (ret)
-                goto out;
-        if (!reten_state.worm)
-                op_errno = 0;
+    read_only_priv_t *priv = NULL;
+    int op_errno = EROFS;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+    if (is_wormfile(this, _gf_true, fd)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_true, fd, GF_FOP_WRITE);
 
 out:
-        if (op_errno)
-                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL,
-                                     NULL);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->writev,
-                                 fd, vector, count, offset, flags, iobref,
-                                 xdata);
-        return 0;
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                        offset, flags, iobref, xdata);
+    return 0;
 }
 
 static int32_t
-worm_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata)
+worm_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        int ret                   =       0;
-        read_only_priv_t *priv    =       NULL;
-        dict_t *dict              =       NULL;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        if (priv->worm_file) {
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error creating the "
-                                "dict");
-                        goto out;
-                }
-                ret = dict_set_int8 (dict, "trusted.worm_file", 1);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error in setting "
-                                "the dict");
-                        goto out;
-                }
-                ret = syncop_fsetxattr (this, fd, dict, 0, NULL, NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting xattr");
-                        goto out;
-                }
-                ret = worm_init_state (this, _gf_true, fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error initializing state");
-                }
+    int ret = 0;
+    read_only_priv_t *priv = NULL;
+    // In case of an error exit because fd can be NULL and this would
+    // cause an segfault when performing fsetxattr . We explicitly
+    // unwind to avoid future problems
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (priv->worm_file) {
+        ret = fd_ctx_set(fd, this, 1);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to set the fd ctx "
+                   "for gfid:%s . Worm feature may not work for the gfid",
+                   uuid_utoa(inode->gfid));
+        }
+        ret = worm_init_state(this, _gf_true, fd);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error initializing state");
         }
+    }
 
 out:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-        if (dict)
-                dict_destroy (dict);
-        return ret;
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return ret;
 }
 
-
 static int32_t
-worm_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+worm_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, worm_create_cbk,  FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->create, loc, flags,
-                    mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, worm_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
+static void
+set_reten_mode(read_only_priv_t *priv, char *reten_mode)
+{
+    if (strcmp(reten_mode, "relax") == 0)
+        priv->reten_mode = 0;
+    else
+        priv->reten_mode = 1;
+}
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int                    ret      = -1;
-        read_only_priv_t       *priv    = NULL;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "translator not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        this->local_pool = mem_pool_new (read_only_priv_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create read_only_priv_t's memory pool");
-                goto out;
-        }
-
-        priv = mem_get0 (this->local_pool);
-        if (!priv) {
-                gf_log (this->name, GF_LOG_ERROR, "Error allocating priv");
-                goto out;
-        }
-
-        priv->reten_mode = mem_get0 (this->local_pool);
-        if (!priv->reten_mode) {
-                gf_log (this->name, GF_LOG_ERROR, "Error allocating "
-                        "reten_mode");
-                goto out;
-        }
-
-        GF_OPTION_INIT ("worm", priv->readonly_or_worm_enabled,
-                        bool, out);
-        GF_OPTION_INIT ("worm-file-level", priv->worm_file, bool, out);
-        GF_OPTION_INIT ("default-retention-period", priv->reten_period,
-                        uint64, out);
-        GF_OPTION_INIT ("auto-commit-period", priv->com_period, uint64, out);
-        GF_OPTION_INIT ("retention-mode", priv->reten_mode, str, out);
-
-        this->private = priv;
-        ret = 0;
+    int ret = -1;
+    read_only_priv_t *priv = NULL;
+    char *reten_mode = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "translator not configured with exactly one child");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    this->local_pool = mem_pool_new(read_only_priv_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create read_only_priv_t's memory pool");
+        goto out;
+    }
+
+    priv = mem_get0(this->local_pool);
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "Error allocating priv");
+        goto out;
+    }
+
+    this->private = priv;
+
+    GF_OPTION_INIT("worm", priv->readonly_or_worm_enabled, bool, out);
+    GF_OPTION_INIT("worm-file-level", priv->worm_file, bool, out);
+    GF_OPTION_INIT("default-retention-period", priv->reten_period, int64, out);
+    GF_OPTION_INIT("auto-commit-period", priv->com_period, int64, out);
+    GF_OPTION_INIT("retention-mode", reten_mode, str, out);
+    set_reten_mode(priv, reten_mode);
+    GF_OPTION_INIT("worm-files-deletable", priv->worm_files_deletable, bool,
+                   out);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        read_only_priv_t   *priv                    = NULL;
-        int                ret                      = -1;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_OPTION_RECONF ("worm", priv->readonly_or_worm_enabled,
-                          options, bool, out);
-        GF_OPTION_RECONF ("worm-file-level", priv->worm_file, options, bool,
-                          out);
-        GF_OPTION_RECONF ("default-retention-period", priv->reten_period,
-                          options, uint64, out);
-        GF_OPTION_RECONF ("retention-mode", priv->reten_mode, options, str,
-                          out);
-        GF_OPTION_RECONF ("auto-commit-period", priv->com_period, options,
-                          uint64, out);
-        ret = 0;
+    read_only_priv_t *priv = NULL;
+    char *reten_mode = NULL;
+    int ret = -1;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_OPTION_RECONF("worm", priv->readonly_or_worm_enabled, options, bool,
+                     out);
+    GF_OPTION_RECONF("worm-file-level", priv->worm_file, options, bool, out);
+    GF_OPTION_RECONF("default-retention-period", priv->reten_period, options,
+                     int64, out);
+    GF_OPTION_RECONF("retention-mode", reten_mode, options, str, out);
+    set_reten_mode(priv, reten_mode);
+    GF_OPTION_RECONF("auto-commit-period", priv->com_period, options, int64,
+                     out);
+    GF_OPTION_RECONF("worm-files-deletable", priv->worm_files_deletable,
+                     options, bool, out);
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_log(this->name, GF_LOG_DEBUG, "returning %d", ret);
+    return ret;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        read_only_priv_t         *priv    = NULL;
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-        if (priv->reten_mode != NULL) {
-                mem_put (priv->reten_mode);
-                priv->reten_mode = NULL;
-        }
-        mem_put (priv);
-        this->private = NULL;
-        mem_pool_destroy (this->local_pool);
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+    mem_put(priv);
+    this->private = NULL;
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
 out:
-        return;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .open        = worm_open,
-        .writev      = worm_writev,
-        .setattr     = worm_setattr,
-        .fsetattr    = worm_fsetattr,
-        .rename      = worm_rename,
-        .link        = worm_link,
-        .unlink      = worm_unlink,
-        .truncate    = worm_truncate,
-        .create      = worm_create,
-
-        .rmdir       = ro_rmdir,
-        .removexattr = ro_removexattr,
-        .fsyncdir    = ro_fsyncdir,
-        .xattrop     = ro_xattrop,
-        .inodelk     = ro_inodelk,
-        .finodelk    = ro_finodelk,
-        .entrylk     = ro_entrylk,
-        .fentrylk    = ro_fentrylk,
-        .lk          = ro_lk,
+    .open = worm_open,
+    .writev = worm_writev,
+    .setattr = worm_setattr,
+    .fsetattr = worm_fsetattr,
+    .rename = worm_rename,
+    .link = worm_link,
+    .unlink = worm_unlink,
+    .truncate = worm_truncate,
+    .ftruncate = worm_ftruncate,
+    .create = worm_create,
+
+    .rmdir = ro_rmdir,
+    .removexattr = ro_removexattr,
+    .fsyncdir = ro_fsyncdir,
+    .xattrop = ro_xattrop,
+    .inodelk = ro_inodelk,
+    .finodelk = ro_finodelk,
+    .entrylk = ro_entrylk,
+    .fentrylk = ro_fentrylk,
+    .lk = ro_lk,
 };
 
+int32_t
+worm_release(xlator_t *this, fd_t *fd)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    dict = dict_new();
+    uint64_t value = 0;
+    loc_t loc = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    priv = this->private;
+
+    if (priv->worm_file) {
+        if (!dict) {
+            gf_log(this->name, GF_LOG_ERROR, "Error creating the dict");
+            goto out;
+        }
+
+        ret = fd_ctx_get(fd, this, &value);
+        if (ret) {
+            gf_log(this->name, GF_LOG_DEBUG, "Failed to get the fd ctx");
+        }
+        if (!value) {
+            goto out;
+        }
+
+        ret = dict_set_int8(dict, "trusted.worm_file", 1);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Error in setting "
+                   "the dict");
+            goto out;
+        }
+
+        loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(loc.gfid, fd->inode->gfid);
+        ret = syncop_setxattr(this, &loc, dict, 0, NULL, NULL);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error setting xattr");
+            goto out;
+        }
 
-struct xlator_cbks cbks;
+        gf_worm_state_transition(this, _gf_false, &loc, GF_FOP_WRITE);
+    }
 
+out:
+    loc_wipe(&loc);
+    if (dict)
+        dict_unref(dict);
+    return 0;
+}
+
+struct xlator_cbks cbks = {
+    .release = worm_release,
+};
 
 struct volume_options options[] = {
-        { .key  = {"worm"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", makes a volume get write once read many "
-                         " feature. It is turned \"off\" by default."
-        },
-        { .key = {"worm-file-level"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", activates the file level worm. "
-                         "It is turned \"off\" by default."
-        },
-        { .key = {"default-retention-period"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = "120",
-          .description = "The default retention period for the files."
-        },
-        { .key = {"retention-mode"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "relax",
-          .description = "The mode of retention (relax/enterprise). "
-                         "It is relax by default."
-        },
-        { .key = {"auto-commit-period"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = "180",
-          .description = "Auto commit period for the files."
-        },
+    {.key = {"worm"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {2},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"on\", makes a volume get write once read many "
+                    " feature. It is turned \"off\" by default."},
+    {.key = {"worm-file-level"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"on\", activates the file level worm. "
+                    "It is turned \"off\" by default."},
+    {.key = {"worm-files-deletable"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {GD_OP_VERSION_3_13_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"off\", doesn't allow the Worm files"
+                    "to be deleted. It is turned \"on\" by default."},
+    {.key = {"default-retention-period"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = "120",
+     /*.validate_fn = validate_worm_period,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "The default retention period for the files."},
+    {.key = {"retention-mode"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "relax",
+     /*.validate_fn = validate_reten_mode,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "The mode of retention (relax/enterprise). "
+                    "It is relax by default."},
+    {.key = {"auto-commit-period"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = "180",
+     /*.validate_fn = validate_worm_period,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "Auto commit period for the files."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "worm",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/experimental/posix2/mds/Makefile.am b/xlators/features/sdfs/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/posix2/mds/Makefile.am
+++ b/xlators/features/sdfs/Makefile.am
diff --git a/xlators/features/sdfs/src/Makefile.am b/xlators/features/sdfs/src/Makefile.am
new file mode 100644
index 00000000000..6118d46ad22
--- /dev/null
+++ b/xlators/features/sdfs/src/Makefile.am
@@ -0,0 +1,19 @@
+if WITH_SERVER
+xlator_LTLIBRARIES = sdfs.la
+endif
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+sdfs_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+sdfs_la_SOURCES = sdfs.c
+sdfs_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = sdfs.h sdfs-messages.h $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+                                 -I$(top_srcdir)/xlators/lib/src \
+        -I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/sdfs/src/sdfs-messages.h b/xlators/features/sdfs/src/sdfs-messages.h
new file mode 100644
index 00000000000..3053efa8935
--- /dev/null
+++ b/xlators/features/sdfs/src/sdfs-messages.h
@@ -0,0 +1,67 @@
+/*
+ Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _DFS_MESSAGES_H_
+#define _DFS_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* file bit-rot-bitd-messages.h
+ * brief SDFS log-message IDs and their descriptions
+ */
+
+/* NOTE: Rules for message additions
+ * 1) Each instance of a message is _better_ left with a unique message ID, even
+ *    if the message format is the same. Reasoning is that, if the message
+ *    format needs to change in one instance, the other instances are not
+ *    impacted or the new change does not change the ID of the instance being
+ *    modified.
+ * 2) Addition of a message,
+ *       - Should increment the GLFS_NUM_MESSAGES
+ *       - Append to the list of messages defined, towards the end
+ *       - Retain macro naming as glfs_msg_X (for redability across developers)
+ * NOTE: Rules for message format modifications
+ * 3) Check acorss the code if the message ID macro in question is reused
+ *    anywhere. If reused then then the modifications should ensure correctness
+ *    everywhere, or needs a new message ID as (1) above was not adhered to. If
+ *    not used anywhere, proceed with the required modification.
+ * NOTE: Rules for message deletion
+ * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
+ *    anywhere, then can be deleted, but will leave a hole by design, as
+ *    addition rules specify modification to the end of the list and not filling
+ *    holes.
+ */
+
+#define GLFS_SDFS_BASE GLFS_MSGID_COMP_SDFS
+#define GLFS_SDFS_NUM_MESSAGES 2
+#define GLFS_MSGID_END (GLFS_SDFS_BASE + GLFS_SDFS_NUM_MESSAGES + 1)
+/* Messaged with message IDs */
+#define glfs_msg_start_x GLFS_DFS_BASE, "Invalid: Start of messages"
+/*------------*/
+
+#define SDFS_MSG_ENTRYLK_ERROR (GLFS_SDFS_BASE + 1)
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+
+#define SDFS_MSG_MKDIR_ERROR (GLFS_SDFS_BASE + 2)
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+/*------------*/
+
+#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#endif /* !_SDFS_MESSAGES_H_ */
diff --git a/xlators/features/sdfs/src/sdfs.c b/xlators/features/sdfs/src/sdfs.c
new file mode 100644
index 00000000000..aaf13f0852e
--- /dev/null
+++ b/xlators/features/sdfs/src/sdfs.c
@@ -0,0 +1,1479 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <libgen.h>
+#include "sdfs.h"
+
+static int
+sdfs_frame_return(call_frame_t *frame)
+{
+    sdfs_local_t *local = NULL;
+
+    if (!frame)
+        return -1;
+
+    local = frame->local;
+
+    return GF_ATOMIC_DEC(local->call_cnt);
+}
+
+static void
+sdfs_lock_free(sdfs_entry_lock_t *entrylk)
+{
+    if (entrylk == NULL)
+        goto out;
+
+    loc_wipe(&entrylk->parent_loc);
+    GF_FREE(entrylk->basename);
+
+out:
+    return;
+}
+
+static void
+sdfs_lock_array_free(sdfs_lock_t *lock)
+{
+    sdfs_entry_lock_t *entrylk = NULL;
+    int i = 0;
+
+    if (lock == NULL)
+        goto out;
+
+    for (i = 0; i < lock->lock_count; i++) {
+        entrylk = &lock->entrylk[i];
+        sdfs_lock_free(entrylk);
+    }
+
+out:
+    return;
+}
+
+static void
+sdfs_local_cleanup(sdfs_local_t *local)
+{
+    if (!local)
+        return;
+
+    loc_wipe(&local->loc);
+    loc_wipe(&local->parent_loc);
+
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
+
+    sdfs_lock_array_free(local->lock);
+    GF_FREE(local->lock);
+
+    mem_put(local);
+}
+
+static int
+sdfs_build_parent_loc(loc_t *parent, loc_t *child)
+{
+    int ret = -1;
+    char *path = NULL;
+
+    if (!child->parent) {
+        goto out;
+    }
+    parent->inode = inode_ref(child->parent);
+    path = gf_strdup(child->path);
+    if (!path) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    parent->path = dirname(path);
+    if (!parent->path) {
+        goto out;
+    }
+
+    gf_uuid_copy(parent->gfid, child->pargfid);
+    return 0;
+
+out:
+    GF_FREE(path);
+    return ret;
+}
+
+static sdfs_local_t *
+sdfs_local_init(call_frame_t *frame, xlator_t *this)
+{
+    sdfs_local_t *local = NULL;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
+
+    frame->local = local;
+out:
+    return local;
+}
+
+static int
+sdfs_get_new_frame_common(call_frame_t *frame, call_frame_t **new_frame)
+{
+    int ret = -1;
+    sdfs_local_t *local = NULL;
+    client_t *client = NULL;
+
+    *new_frame = copy_frame(frame);
+    if (!*new_frame) {
+        goto err;
+    }
+
+    client = frame->root->client;
+    gf_client_ref(client);
+    (*new_frame)->root->client = client;
+
+    local = sdfs_local_init(*new_frame, THIS);
+    if (!local) {
+        goto err;
+    }
+
+    local->main_frame = frame;
+    /*Set unique lk-owner for the fop*/
+    set_lk_owner_from_ptr(&(*new_frame)->root->lk_owner, (*new_frame)->root);
+
+    ret = 0;
+err:
+    if ((ret == -1) && (*new_frame)) {
+        SDFS_STACK_DESTROY((*new_frame));
+        *new_frame = NULL;
+    }
+
+    return ret;
+}
+
+static int
+sdfs_get_new_frame(call_frame_t *frame, loc_t *loc, call_frame_t **new_frame)
+{
+    int ret = -1;
+    sdfs_local_t *local = NULL;
+
+    ret = sdfs_get_new_frame_common(frame, new_frame);
+    if (ret < 0) {
+        goto err;
+    }
+
+    local = (*new_frame)->local;
+
+    ret = sdfs_build_parent_loc(&local->parent_loc, loc);
+    if (ret) {
+        goto err;
+    }
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret == -1) {
+        goto err;
+    }
+
+    ret = 0;
+err:
+    if (ret && (*new_frame)) {
+        SDFS_STACK_DESTROY((*new_frame));
+        *new_frame = NULL;
+        ret = -1;
+    }
+
+    return ret;
+}
+
+static int
+sdfs_get_new_frame_readdirp(call_frame_t *frame, fd_t *fd,
+                            call_frame_t **new_frame)
+{
+    int ret = -1;
+    sdfs_local_t *local = NULL;
+
+    ret = sdfs_get_new_frame_common(frame, new_frame);
+    if (ret < 0) {
+        goto err;
+    }
+
+    local = (*new_frame)->local;
+    local->parent_loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->parent_loc.gfid, fd->inode->gfid);
+
+    ret = 0;
+err:
+    return ret;
+}
+
+int
+sdfs_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    if (local->stub) {
+        stub = local->stub;
+        local->stub = NULL;
+        call_resume(stub);
+    } else {
+        if (op_ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+                   "Unlocking entry lock failed for %s", local->loc.name);
+
+        SDFS_STACK_DESTROY(frame);
+    }
+
+    return 0;
+}
+
+int
+sdfs_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *stbuf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(mkdir, local->main_frame, op_ret, op_errno, inode,
+                        stbuf, preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                  mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int op_errno = -1;
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mkdir, local->main_frame, -1, op_errno, NULL, NULL,
+                        NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_mkdir_stub(new_frame, sdfs_mkdir_helper, loc, mode, umask,
+                          xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(rmdir, local->main_frame, op_ret, op_errno, preparent,
+                        postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_rmdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                  dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(rmdir, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_rmdir_stub(new_frame, sdfs_rmdir_helper, loc, flags, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                struct iatt *stbuf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(create, local->main_frame, op_ret, op_errno, fd, inode,
+                        stbuf, preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_create_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                   dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(create, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_create_stub(new_frame, sdfs_create_helper, loc, flags, mode,
+                           umask, fd, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(unlink, local->main_frame, op_ret, op_errno, preparent,
+                        postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_unlink_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                   dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flags, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(unlink, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_unlink_stub(new_frame, sdfs_unlink_helper, loc, flags, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(link, local->main_frame, op_ret, op_errno, inode, stbuf,
+                        preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_symlink_helper(call_frame_t *frame, xlator_t *this, const char *linkname,
+                    loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(link, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+             loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_symlink_stub(new_frame, sdfs_symlink_helper, linkname, loc,
+                            umask, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_common_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int lk_index = 0;
+    sdfs_lock_t *locks = NULL;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+    locks = local->lock;
+    lk_index = (long)cookie;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    } else {
+        locks->entrylk->locked[lk_index] = _gf_true;
+    }
+
+    this_call_cnt = sdfs_frame_return(frame);
+    if (this_call_cnt > 0) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "As there are more callcnt (%d) returning without WIND",
+               this_call_cnt);
+        return 0;
+    }
+
+    if (local->stub) {
+        stub = local->stub;
+        local->stub = NULL;
+        call_resume(stub);
+    } else {
+        if (local->op_ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+                   "unlocking entry lock failed ");
+        SDFS_STACK_DESTROY(frame);
+    }
+
+    return 0;
+}
+
+int
+sdfs_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    int i = 0;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock = local->lock;
+
+    STACK_UNWIND_STRICT(link, local->main_frame, op_ret, op_errno, inode, stbuf,
+                        preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    lock_count = lock->lock_count;
+    for (i = 0; i < lock_count; i++) {
+        STACK_WIND_COOKIE(frame, sdfs_common_entrylk_cbk, (void *)(long)i,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                          this->name, &lock->entrylk[i].parent_loc,
+                          lock->entrylk[i].basename, ENTRYLK_UNLOCK,
+                          ENTRYLK_WRLCK, xdata);
+    }
+
+    return 0;
+}
+
+int
+sdfs_link_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *locks = NULL;
+    gf_boolean_t stack_destroy = _gf_true;
+    int lock_count = 0;
+    int i = 0;
+
+    local = frame->local;
+    locks = local->lock;
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed");
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(link, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    for (i = 0; i < locks->lock_count && locks->entrylk->locked[i]; i++) {
+        lock_count++;
+    }
+    GF_ATOMIC_INIT(local->call_cnt, lock_count);
+
+    for (i = 0; i < lock_count; i++) {
+        if (!locks->entrylk->locked[i]) {
+            lock_count++;
+            continue;
+        }
+
+        stack_destroy = _gf_false;
+        STACK_WIND(frame, sdfs_common_entrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->entrylk, this->name,
+                   &locks->entrylk[i].parent_loc, locks->entrylk[i].basename,
+                   ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    }
+
+    if (stack_destroy)
+        SDFS_STACK_DESTROY(frame);
+
+    return 0;
+}
+
+static int
+sdfs_init_entry_lock(sdfs_entry_lock_t *lock, loc_t *loc)
+{
+    int ret = 0;
+
+    ret = sdfs_build_parent_loc(&lock->parent_loc, loc);
+    if (ret)
+        return -1;
+
+    lock->basename = gf_strdup(loc->name);
+    if (!lock->basename)
+        return -1;
+
+    return 0;
+}
+
+int
+sdfs_entry_lock_cmp(const void *l1, const void *l2)
+{
+    const sdfs_entry_lock_t *r1 = l1;
+    const sdfs_entry_lock_t *r2 = l2;
+    int ret = 0;
+    uuid_t gfid1 = {0};
+    uuid_t gfid2 = {0};
+
+    loc_gfid((loc_t *)&r1->parent_loc, gfid1);
+    loc_gfid((loc_t *)&r2->parent_loc, gfid2);
+    ret = gf_uuid_compare(gfid1, gfid2);
+    /*Entrylks with NULL basename are the 'smallest'*/
+    if (ret == 0) {
+        if (!r1->basename)
+            return -1;
+        if (!r2->basename)
+            return 1;
+        ret = strcmp(r1->basename, r2->basename);
+    }
+
+    if (ret <= 0)
+        return -1;
+    else
+        return 1;
+}
+
+int
+sdfs_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    sdfs_lock_t *lock = NULL;
+    client_t *client = NULL;
+    int ret = 0;
+    int op_errno = ENOMEM;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /*Set unique lk-owner for the fop*/
+    set_lk_owner_from_ptr(&new_frame->root->lk_owner, new_frame->root);
+
+    gf_client_ref(client);
+    new_frame->root->client = client;
+    local = sdfs_local_init(new_frame, this);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->main_frame = frame;
+
+    lock = GF_CALLOC(1, sizeof(*lock), gf_common_mt_char);
+    if (!lock)
+        goto err;
+
+    local->lock = lock;
+
+    ret = sdfs_init_entry_lock(&lock->entrylk[0], newloc);
+    if (ret)
+        goto err;
+
+    ++lock->lock_count;
+
+    local->lock = lock;
+    GF_ATOMIC_INIT(local->call_cnt, lock->lock_count);
+
+    ret = loc_copy(&local->loc, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_link_stub(new_frame, sdfs_link_helper, oldloc, newloc, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->stub = stub;
+
+    STACK_WIND_COOKIE(new_frame, sdfs_common_entrylk_cbk, 0, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->entrylk, this->name,
+                      &lock->entrylk[0].parent_loc, lock->entrylk[0].basename,
+                      ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+
+    STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *stbuf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(mknod, local->main_frame, op_ret, op_errno, inode,
+                        stbuf, preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_mknod_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                  dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mknod, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_mknod_stub(new_frame, sdfs_mknod_helper, loc, mode, rdev, umask,
+                          xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                struct iatt *preoldparent, struct iatt *postoldparent,
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    local = frame->local;
+    lock = local->lock;
+    GF_ATOMIC_INIT(local->call_cnt, lock->lock_count);
+
+    STACK_UNWIND_STRICT(rename, local->main_frame, op_ret, op_errno, stbuf,
+                        preoldparent, postoldparent, prenewparent,
+                        postnewparent, xdata);
+
+    local->main_frame = NULL;
+    call_cnt = GF_ATOMIC_GET(local->call_cnt);
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, sdfs_common_entrylk_cbk, (void *)(long)i,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                          this->name, &lock->entrylk[i].parent_loc,
+                          lock->entrylk[i].basename, ENTRYLK_UNLOCK,
+                          ENTRYLK_WRLCK, xdata);
+    }
+
+    return 0;
+}
+
+int
+sdfs_rename_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                   loc_t *newloc, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    gf_boolean_t stack_destroy = _gf_true;
+    int lock_count = 0;
+    int i = 0;
+
+    local = frame->local;
+    lock = local->lock;
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed ");
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(rename, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    for (i = 0; i < lock->lock_count && lock->entrylk->locked[i]; i++) {
+        lock_count++;
+    }
+    GF_ATOMIC_INIT(local->call_cnt, lock_count);
+
+    for (i = 0; i < lock_count; i++) {
+        if (!lock->entrylk->locked[i]) {
+            lock_count++;
+            continue;
+        }
+        stack_destroy = _gf_false;
+        STACK_WIND(frame, sdfs_common_entrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->entrylk, this->name,
+                   &lock->entrylk[i].parent_loc, lock->entrylk[i].basename,
+                   ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    }
+
+    if (stack_destroy)
+        SDFS_STACK_DESTROY(frame);
+
+    return 0;
+}
+
+int
+sdfs_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    client_t *client = NULL;
+    int ret = 0;
+    int op_errno = ENOMEM;
+    int i = 0;
+    int call_cnt = 0;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /*Set unique lk-owner for the fop*/
+    set_lk_owner_from_ptr(&new_frame->root->lk_owner, new_frame->root);
+
+    gf_client_ref(client);
+    new_frame->root->client = client;
+    local = sdfs_local_init(new_frame, this);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->main_frame = frame;
+
+    lock = GF_CALLOC(1, sizeof(*lock), gf_common_mt_char);
+    if (!lock)
+        goto err;
+
+    local->lock = lock;
+
+    ret = sdfs_init_entry_lock(&lock->entrylk[0], oldloc);
+    if (ret)
+        goto err;
+    lock->entrylk->locked[0] = _gf_false;
+
+    ++lock->lock_count;
+
+    ret = sdfs_init_entry_lock(&lock->entrylk[1], newloc);
+    if (ret)
+        goto err;
+    lock->entrylk->locked[1] = _gf_false;
+
+    ++lock->lock_count;
+
+    qsort(lock->entrylk, lock->lock_count, sizeof(*lock->entrylk),
+          sdfs_entry_lock_cmp);
+
+    local->lock = lock;
+    GF_ATOMIC_INIT(local->call_cnt, lock->lock_count);
+
+    stub = fop_rename_stub(new_frame, sdfs_rename_helper, oldloc, newloc,
+                           xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->stub = stub;
+    call_cnt = GF_ATOMIC_GET(local->call_cnt);
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(new_frame, sdfs_common_entrylk_cbk, (void *)(long)i,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                          this->name, &lock->entrylk[i].parent_loc,
+                          lock->entrylk[i].basename, ENTRYLK_LOCK,
+                          ENTRYLK_WRLCK, xdata);
+    }
+
+    return 0;
+err:
+
+    STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *stbuf, dict_t *xdata, struct iatt *postparent)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (!local->loc.parent) {
+        sdfs_local_cleanup(local);
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf,
+                            xdata, postparent);
+        return 0;
+    }
+
+    STACK_UNWIND_STRICT(lookup, local->main_frame, op_ret, op_errno, inode,
+                        stbuf, xdata, postparent);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_RDLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_lookup_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(lookup, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL);
+    local->main_frame = NULL;
+
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (!loc->parent) {
+        local = sdfs_local_init(frame, this);
+        if (!local) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+    }
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_lookup_stub(new_frame, sdfs_lookup_helper, loc, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_RDLCK, xdata);
+
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int32_t
+sdfs_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+    STACK_UNWIND_STRICT(readdirp, local->main_frame, op_ret, op_errno, entries,
+                        xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               NULL, ENTRYLK_UNLOCK, ENTRYLK_RDLCK, xdata);
+    return 0;
+}
+
+int32_t
+sdfs_readdirp_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                     off_t off, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(fd->inode->gfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(readdirp, local->main_frame, -1, local->op_errno, NULL,
+                        NULL);
+
+    local->main_frame = NULL;
+
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int32_t
+sdfs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t off, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame_readdirp(frame, fd, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_readdirp_stub(new_frame, sdfs_readdirp_helper, fd, size, off,
+                             xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               NULL, ENTRYLK_LOCK, ENTRYLK_RDLCK, xdata);
+
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'dentry-fop-serializer' not configured with exactly one child");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    this->local_pool = mem_pool_new(sdfs_local_t, 512);
+    if (!this->local_pool) {
+        goto out;
+    }
+
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = -1;
+
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+    return;
+}
+
+struct xlator_fops fops = {
+    .mkdir = sdfs_mkdir,
+    .rmdir = sdfs_rmdir,
+    .create = sdfs_create,
+    .unlink = sdfs_unlink,
+    .symlink = sdfs_symlink,
+    .link = sdfs_link,
+    .mknod = sdfs_mknod,
+    .rename = sdfs_rename,
+    .lookup = sdfs_lookup,
+    .readdirp = sdfs_readdirp,
+};
+
+struct xlator_cbks cbks;
+
+struct volume_options options[] = {
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "true",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"sdfs"},
+     .description = "Enable/Disable dentry serialize functionality"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .op_version = {GD_OP_VERSION_4_0_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "sdfs",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/features/sdfs/src/sdfs.h b/xlators/features/sdfs/src/sdfs.h
new file mode 100644
index 00000000000..dded5a2d7fc
--- /dev/null
+++ b/xlators/features/sdfs/src/sdfs.h
@@ -0,0 +1,49 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+#include "sdfs-messages.h"
+#include <glusterfs/atomic.h>
+
+#define SDFS_LOCK_COUNT_MAX 2
+
+typedef struct {
+    loc_t parent_loc;
+    char *basename;
+    int locked[SDFS_LOCK_COUNT_MAX];
+} sdfs_entry_lock_t;
+
+typedef struct {
+    sdfs_entry_lock_t entrylk[SDFS_LOCK_COUNT_MAX];
+    int lock_count;
+} sdfs_lock_t;
+
+struct sdfs_local {
+    call_frame_t *main_frame;
+    loc_t loc;
+    loc_t parent_loc;
+    call_stub_t *stub;
+    sdfs_lock_t *lock;
+    int op_ret;
+    int op_errno;
+    gf_atomic_t call_cnt;
+};
+typedef struct sdfs_local sdfs_local_t;
+
+#define SDFS_STACK_DESTROY(frame)                                              \
+    do {                                                                       \
+        sdfs_local_t *__local = NULL;                                          \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        gf_client_unref(frame->root->client);                                  \
+        STACK_DESTROY(frame->root);                                            \
+        sdfs_local_cleanup(__local);                                           \
+    } while (0)
diff --git a/xlators/features/changetimerecorder/Makefile.am b/xlators/features/selinux/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/features/changetimerecorder/Makefile.am
+++ b/xlators/features/selinux/Makefile.am
diff --git a/xlators/features/selinux/src/Makefile.am b/xlators/features/selinux/src/Makefile.am
new file mode 100644
index 00000000000..4f1e5e149b3
--- /dev/null
+++ b/xlators/features/selinux/src/Makefile.am
@@ -0,0 +1,20 @@
+if WITH_SERVER
+xlator_LTLIBRARIES = selinux.la
+endif
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+selinux_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+selinux_la_SOURCES = selinux.c
+
+selinux_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = selinux.h selinux-messages.h selinux-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
diff --git a/xlators/features/filter/src/filter-mem-types.h b/xlators/features/selinux/src/selinux-mem-types.h
index 47a17249b8d..553e59e5a9d 100644
--- a/xlators/features/filter/src/filter-mem-types.h
+++ b/xlators/features/selinux/src/selinux-mem-types.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -7,14 +7,13 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef __FILTER_MEM_TYPES_H__
-#define __FILTER_MEM_TYPES_H__
+#ifndef __SELINUX_MEM_TYPES_H__
+#define __SELINUX_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
-enum gf_filter_mem_types_ {
-        gf_filter_mt_gf_filter = gf_common_mt_end + 1,
-        gf_filter_mt_end
+enum gf_selinux_mem_types_ {
+    gf_selinux_mt_selinux_priv_t = gf_common_mt_end + 1,
+    gf_selinux_mt_end
 };
 #endif
-
diff --git a/xlators/features/selinux/src/selinux-messages.h b/xlators/features/selinux/src/selinux-messages.h
new file mode 100644
index 00000000000..f49a54f956c
--- /dev/null
+++ b/xlators/features/selinux/src/selinux-messages.h
@@ -0,0 +1,30 @@
+/*
+  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _SELINUX_MESSAGES_H__
+#define _SELINUX_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SL, SL_MSG_INVALID_VOLFILE, SL_MSG_ENOMEM,
+           SL_MSG_MEM_ACCT_INIT_FAILED, SL_MSG_SELINUX_GLUSTER_XATTR_MISSING,
+           SL_MSG_SELINUX_XATTR_MISSING);
+
+#endif /*_SELINUX_MESSAGES_H */
diff --git a/xlators/features/selinux/src/selinux.c b/xlators/features/selinux/src/selinux.c
new file mode 100644
index 00000000000..9b1b4b55e1a
--- /dev/null
+++ b/xlators/features/selinux/src/selinux.c
@@ -0,0 +1,323 @@
+/*
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/xlator.h>
+
+#include "selinux.h"
+#include "selinux-messages.h"
+#include "selinux-mem-types.h"
+#include <glusterfs/compat-errno.h>
+
+static int
+selinux_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    char *name = cookie;
+
+    if (op_errno == 0 && dict && name &&
+        (!strcmp(name, SELINUX_GLUSTER_XATTR))) {
+        ret = dict_rename_key(dict, SELINUX_GLUSTER_XATTR, SELINUX_XATTR);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SL_MSG_SELINUX_GLUSTER_XATTR_MISSING,
+                   "getxattr failed for %s", SELINUX_XATTR);
+    }
+
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return ret;
+}
+
+static int
+selinux_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    char *xattr_name = (char *)name;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    /* name can be NULL for listxattr calls */
+    if (!priv->selinux_enabled || !name)
+        goto off;
+
+    if (strcmp(name, SELINUX_XATTR) == 0)
+        xattr_name = SELINUX_GLUSTER_XATTR;
+
+off:
+    STACK_WIND_COOKIE(frame, selinux_fgetxattr_cbk, xattr_name,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr, fd,
+                      xattr_name, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, NULL, xdata);
+
+    return 0;
+}
+
+static int
+selinux_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    char *name = cookie;
+
+    if (op_errno == 0 && dict && name &&
+        (!strcmp(name, SELINUX_GLUSTER_XATTR))) {
+        ret = dict_rename_key(dict, SELINUX_GLUSTER_XATTR, SELINUX_XATTR);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SL_MSG_SELINUX_GLUSTER_XATTR_MISSING,
+                   "getxattr failed for %s", SELINUX_XATTR);
+    }
+
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
+}
+
+static int
+selinux_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    char *xattr_name = (char *)name;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    /* name can be NULL for listxattr calls */
+    if (!priv->selinux_enabled || !name)
+        goto off;
+
+    if (strcmp(name, SELINUX_XATTR) == 0)
+        xattr_name = SELINUX_GLUSTER_XATTR;
+
+off:
+    STACK_WIND_COOKIE(frame, selinux_getxattr_cbk, xattr_name,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr, loc,
+                      xattr_name, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, NULL, xdata);
+    return 0;
+}
+
+static int
+selinux_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int
+selinux_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int flags, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int32_t ret = -1;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    if (!priv->selinux_enabled && !dict)
+        goto off;
+
+    ret = dict_rename_key(dict, SELINUX_XATTR, SELINUX_GLUSTER_XATTR);
+    if (ret < 0 && ret != -ENODATA)
+        goto err;
+
+off:
+    STACK_WIND(frame, selinux_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int
+selinux_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int
+selinux_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int flags, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int32_t ret = -1;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    if (!priv->selinux_enabled && !dict)
+        goto off;
+
+    ret = dict_rename_key(dict, SELINUX_XATTR, SELINUX_GLUSTER_XATTR);
+    if (ret < 0 && ret != -ENODATA)
+        goto err;
+
+off:
+    STACK_WIND(frame, selinux_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("selinux", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_selinux_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SL_MSG_MEM_ACCT_INIT_FAILED,
+               "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    int32_t ret = -1;
+    selinux_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("selinux", this, out);
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SL_MSG_INVALID_VOLFILE,
+               "Error: SELinux (%s) not configured with exactly one "
+               "child",
+               this->name);
+        return -1;
+    }
+
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SL_MSG_INVALID_VOLFILE,
+               "Dangling volume. Please check the volfile");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_selinux_mt_selinux_priv_t);
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        goto out;
+    }
+
+    GF_OPTION_INIT("selinux", priv->selinux_enabled, bool, out);
+
+    this->local_pool = mem_pool_new(selinux_priv_t, 64);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SL_MSG_ENOMEM,
+               "Failed to create local_t's memory pool");
+        goto out;
+    }
+
+    this->private = (void *)priv;
+    ret = 0;
+out:
+    if (ret) {
+        GF_FREE(priv);
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int32_t ret = -1;
+    selinux_priv_t *priv = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("selinux", priv->selinux_enabled, options, bool, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    selinux_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_FREE(priv);
+
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+
+    return;
+}
+
+struct xlator_fops fops = {
+    .getxattr = selinux_getxattr,
+    .fgetxattr = selinux_fgetxattr,
+    .setxattr = selinux_setxattr,
+    .fsetxattr = selinux_fsetxattr,
+};
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {
+        .key = {"selinux"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "on",
+        .description = "Enable/disable selinux translator",
+        .op_version = {GD_OP_VERSION_3_11_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"security", "linux"},
+    },
+    {
+        .key = {NULL},
+    }};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "selinux",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/features/selinux/src/selinux.h b/xlators/features/selinux/src/selinux.h
new file mode 100644
index 00000000000..1bbdad3bb36
--- /dev/null
+++ b/xlators/features/selinux/src/selinux.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __SELINUX_H__
+#define __SELINUX_H__
+
+#include <glusterfs/common-utils.h>
+
+#define SELINUX_XATTR "security.selinux"
+#define SELINUX_GLUSTER_XATTR "trusted.glusterfs.selinux"
+
+struct selinux_priv {
+    gf_boolean_t selinux_enabled;
+};
+
+typedef struct selinux_priv selinux_priv_t;
+
+#endif
diff --git a/xlators/features/shard/src/Makefile.am b/xlators/features/shard/src/Makefile.am
index 89173f1203e..bf5700d4bcc 100644
--- a/xlators/features/shard/src/Makefile.am
+++ b/xlators/features/shard/src/Makefile.am
@@ -9,7 +9,8 @@ shard_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = shard.h shard-mem-types.h shard-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/shard/src/shard-mem-types.h b/xlators/features/shard/src/shard-mem-types.h
index 77f0cee7f58..1fe7e2e2798 100644
--- a/xlators/features/shard/src/shard-mem-types.h
+++ b/xlators/features/shard/src/shard-mem-types.h
@@ -10,14 +10,15 @@
 #ifndef __SHARD_MEM_TYPES_H__
 #define __SHARD_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_shard_mem_types_ {
-        gf_shard_mt_priv_t = gf_common_mt_end + 1,
-        gf_shard_mt_inode_list,
-        gf_shard_mt_inode_ctx_t,
-        gf_shard_mt_iovec,
-        gf_shard_mt_int64_t,
-        gf_shard_mt_end
+    gf_shard_mt_priv_t = gf_common_mt_end + 1,
+    gf_shard_mt_inode_list,
+    gf_shard_mt_inode_ctx_t,
+    gf_shard_mt_iovec,
+    gf_shard_mt_int64_t,
+    gf_shard_mt_uint64_t,
+    gf_shard_mt_end
 };
 #endif
diff --git a/xlators/features/shard/src/shard-messages.h b/xlators/features/shard/src/shard-messages.h
index 588cb687d5d..2d0867eb136 100644
--- a/xlators/features/shard/src/shard-messages.h
+++ b/xlators/features/shard/src/shard-messages.h
@@ -11,174 +11,29 @@
 #ifndef _SHARD_MESSAGES_H_
 #define _SHARD_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SHARD, SHARD_MSG_BASE_FILE_LOOKUP_FAILED, SHARD_MSG_DICT_OP_FAILED,
+           SHARD_MSG_DOT_SHARD_NODIR, SHARD_MSG_FD_CTX_SET_FAILED,
+           SHARD_MSG_INODE_CTX_GET_FAILED, SHARD_MSG_INODE_CTX_SET_FAILED,
+           SHARD_MSG_INODE_PATH_FAILED, SHARD_MSG_INTERNAL_XATTR_MISSING,
+           SHARD_MSG_INVALID_VOLFILE, SHARD_MSG_LOOKUP_SHARD_FAILED,
+           SHARD_MSG_MEM_ACCT_INIT_FAILED, SHARD_MSG_NULL_THIS,
+           SHARD_MSG_SIZE_SET_FAILED, SHARD_MSG_STAT_FAILED,
+           SHARD_MSG_TRUNCATE_LAST_SHARD_FAILED,
+           SHARD_MSG_UPDATE_FILE_SIZE_FAILED, SHARD_MSG_FOP_NOT_SUPPORTED,
+           SHARD_MSG_INVALID_FOP, SHARD_MSG_MEMALLOC_FAILED,
+           SHARD_MSG_FOP_FAILED, SHARD_MSG_SHARDS_DELETION_FAILED,
+           SHARD_MSG_SHARD_DELETION_COMPLETED);
 
-/*! \file shard-messages.h
- *  \brief shard log-message IDs and their descriptions.
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check across the code if the message ID macro in question is reused
- * anywhere. If reused then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
- */
-
-#define GLFS_COMP_BASE_SHARD      GLFS_MSGID_COMP_SHARD
-#define GLFS_NUM_MESSAGES         18
-#define GLFS_MSGID_END          (GLFS_COMP_BASE_SHARD + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x GLFS_COMP_BASE_SHARD, "Invalid: Start of messages"
-
-/*!
- * @messageid 133001
- * @diagnosis
- * @recommendedaction
- */
-#define SHARD_MSG_BASE_FILE_LOOKUP_FAILED             (GLFS_COMP_BASE_SHARD + 1)
-
-
-/*!
- * @messageid 133002
- * @diagnosis
- * @recommendedaction
- */
-#define SHARD_MSG_DICT_SET_FAILED                     (GLFS_COMP_BASE_SHARD + 2)
-
-
-/*!
- * @messageid 133003
- * @diagnosis /.shard already exists and is not a directory.
- * @recommendedaction Delete the /.shard file from the backend and try again.
- */
-#define SHARD_MSG_DOT_SHARD_NODIR                     (GLFS_COMP_BASE_SHARD + 3)
-
-
-/*!
- * @messageid 133004
- * @diagnosis
- * @recommendedaction
- */
-#define SHARD_MSG_FD_CTX_SET_FAILED                   (GLFS_COMP_BASE_SHARD + 4)
-
-
-/*!
- * @messageid 133005
- * @diagnosis
- * @recommendedaction
- */
-#define SHARD_MSG_INODE_CTX_GET_FAILED                (GLFS_COMP_BASE_SHARD + 5)
-
-
-/*!
- * @messageid 133006
- * @diagnosis
- * @recommendedaction
- */
-#define SHARD_MSG_INODE_CTX_SET_FAILED                (GLFS_COMP_BASE_SHARD + 6)
-
-
-/*!
- * @messageid 133007
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_INODE_PATH_FAILED                   (GLFS_COMP_BASE_SHARD + 7)
-
-
-/*!
- * @messageid 133008
- * @diagnosis
- * @recommendedaction
- */
-#define SHARD_MSG_INTERNAL_XATTR_MISSING              (GLFS_COMP_BASE_SHARD + 8)
-
-
-/*!
- * @messageid 133009
- * @diagnosis The client process did not get launched due to incorrect volfile.
- * @recommendedaction Possibly check to see if the volfile is correct.
- */
-#define SHARD_MSG_INVALID_VOLFILE                     (GLFS_COMP_BASE_SHARD + 9)
-
-
-/*!
- * @messageid 133010
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_LOOKUP_SHARD_FAILED                (GLFS_COMP_BASE_SHARD + 10)
-
-/*!
- * @messageid 133011
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_MEM_ACCT_INIT_FAILED               (GLFS_COMP_BASE_SHARD + 11)
-
-/*!
- * @messageid 133012
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_NULL_THIS                          (GLFS_COMP_BASE_SHARD + 12)
-
-/*!
- * @messageid 133013
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_SIZE_SET_FAILED                    (GLFS_COMP_BASE_SHARD + 13)
-
-/*!
- * @messageid 133014
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_STAT_FAILED                        (GLFS_COMP_BASE_SHARD + 14)
-
-/*!
- * @messageid 133015
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_TRUNCATE_LAST_SHARD_FAILED         (GLFS_COMP_BASE_SHARD + 15)
-
-/*!
- * @messageid 133016
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_UPDATE_FILE_SIZE_FAILED            (GLFS_COMP_BASE_SHARD + 16)
-
-/*!
- * @messageid 133017
- * @diagnosis The operation invoked is not supported.
- * @recommendedaction Use other syscalls to write to the file.
-*/
-#define SHARD_MSG_FOP_NOT_SUPPORTED                  (GLFS_COMP_BASE_SHARD + 17)
-
-/*!
- * @messageid 133018
- * @diagnosis
- * @recommendedaction
-*/
-#define SHARD_MSG_INVALID_FOP                        (GLFS_COMP_BASE_SHARD + 18)
-
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* !_SHARD_MESSAGES_H_ */
diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c
index 28f1a51d330..e5f93063943 100644
--- a/xlators/features/shard/src/shard.c
+++ b/xlators/features/shard/src/shard.c
@@ -12,1910 +12,2741 @@
 
 #include "shard.h"
 #include "shard-mem-types.h"
-#include "byte-order.h"
-#include "defaults.h"
-#include "statedump.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/statedump.h>
 
 static gf_boolean_t
-__is_shard_dir (uuid_t gfid)
+__is_shard_dir(uuid_t gfid)
 {
-        shard_priv_t  *priv = THIS->private;
+    shard_priv_t *priv = THIS->private;
 
-        if (gf_uuid_compare (gfid, priv->dot_shard_gfid) == 0)
-                return _gf_true;
+    if (gf_uuid_compare(gfid, priv->dot_shard_gfid) == 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-__is_gsyncd_on_shard_dir (call_frame_t *frame, loc_t *loc)
+__is_gsyncd_on_shard_dir(call_frame_t *frame, loc_t *loc)
 {
-        if (frame->root->pid == GF_CLIENT_PID_GSYNCD &&
-            (__is_shard_dir (loc->pargfid) ||
-            (loc->parent && __is_shard_dir(loc->parent->gfid))))
-                return _gf_true;
+    if (frame->root->pid == GF_CLIENT_PID_GSYNCD &&
+        (__is_shard_dir(loc->pargfid) ||
+         (loc->parent && __is_shard_dir(loc->parent->gfid))))
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 void
-shard_make_block_bname (int block_num, uuid_t gfid, char *buf, size_t len)
+shard_make_block_bname(int block_num, uuid_t gfid, char *buf, size_t len)
 {
-        char gfid_str[GF_UUID_BUF_SIZE] = {0,};
+    char gfid_str[GF_UUID_BUF_SIZE] = {
+        0,
+    };
 
-        gf_uuid_unparse (gfid, gfid_str);
-        snprintf (buf, len, "%s.%d", gfid_str, block_num);
+    gf_uuid_unparse(gfid, gfid_str);
+    snprintf(buf, len, "%s.%d", gfid_str, block_num);
 }
 
 void
-shard_make_block_abspath (int block_num, uuid_t gfid, char *filepath,
-                          size_t len)
+shard_make_block_abspath(int block_num, uuid_t gfid, char *filepath, size_t len)
 {
-        char gfid_str[GF_UUID_BUF_SIZE] = {0,};
+    char gfid_str[GF_UUID_BUF_SIZE] = {
+        0,
+    };
 
-        gf_uuid_unparse (gfid, gfid_str);
-        snprintf (filepath, len, "/%s/%s.%d", GF_SHARD_DIR, gfid_str,
-                  block_num);
+    gf_uuid_unparse(gfid, gfid_str);
+    snprintf(filepath, len, "/%s/%s.%d", GF_SHARD_DIR, gfid_str, block_num);
 }
 
 int
-__shard_inode_ctx_get (inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)
+__shard_inode_ctx_get(inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx_p    = NULL;
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx_p = NULL;
 
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret == 0) {
-                *ctx = (shard_inode_ctx_t *) ctx_uint;
-                return ret;
-        }
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret == 0) {
+        *ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+        return ret;
+    }
 
-        ctx_p = GF_CALLOC (1, sizeof (*ctx_p), gf_shard_mt_inode_ctx_t);
-        if (!ctx_p)
-                return ret;
+    ctx_p = GF_CALLOC(1, sizeof(*ctx_p), gf_shard_mt_inode_ctx_t);
+    if (!ctx_p)
+        return ret;
 
-        INIT_LIST_HEAD (&ctx_p->ilist);
+    INIT_LIST_HEAD(&ctx_p->ilist);
+    INIT_LIST_HEAD(&ctx_p->to_fsync_list);
 
-        ret = __inode_ctx_set (inode, this, (uint64_t *)&ctx_p);
-        if (ret < 0) {
-                GF_FREE (ctx_p);
-                return ret;
-        }
+    ctx_uint = (uint64_t)(uintptr_t)ctx_p;
+    ret = __inode_ctx_set(inode, this, &ctx_uint);
+    if (ret < 0) {
+        GF_FREE(ctx_p);
+        return ret;
+    }
 
-        *ctx = ctx_p;
+    *ctx = ctx_p;
 
-        return ret;
+    return ret;
 }
 
 int
-shard_inode_ctx_get (inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)
+shard_inode_ctx_get(inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        LOCK(&inode->lock);
-        {
-                ret = __shard_inode_ctx_get (inode, this, ctx);
-        }
-        UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get(inode, this, ctx);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-__shard_inode_ctx_set (inode_t *inode, xlator_t *this, struct iatt *stbuf,
-                       uint64_t block_size, int32_t valid)
+__shard_inode_ctx_set(inode_t *inode, xlator_t *this, struct iatt *stbuf,
+                      uint64_t block_size, int32_t valid)
 {
-        int                 ret = -1;
-        shard_inode_ctx_t  *ctx = NULL;
-
-        ret = __shard_inode_ctx_get (inode, this, &ctx);
-        if (ret)
-                return ret;
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
 
-        if (valid & SHARD_MASK_BLOCK_SIZE)
-                ctx->block_size = block_size;
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
+        return ret;
 
-        if (!stbuf)
-                return 0;
+    if (valid & SHARD_MASK_BLOCK_SIZE)
+        ctx->block_size = block_size;
 
-        if (valid & SHARD_MASK_PROT)
-                ctx->stat.ia_prot = stbuf->ia_prot;
+    if (valid & SHARD_MASK_PROT)
+        ctx->stat.ia_prot = stbuf->ia_prot;
 
-        if (valid & SHARD_MASK_NLINK)
-                ctx->stat.ia_nlink = stbuf->ia_nlink;
+    if (valid & SHARD_MASK_NLINK)
+        ctx->stat.ia_nlink = stbuf->ia_nlink;
 
-        if (valid & SHARD_MASK_UID)
-                ctx->stat.ia_uid = stbuf->ia_uid;
+    if (valid & SHARD_MASK_UID)
+        ctx->stat.ia_uid = stbuf->ia_uid;
 
-        if (valid & SHARD_MASK_GID)
-                ctx->stat.ia_gid = stbuf->ia_gid;
+    if (valid & SHARD_MASK_GID)
+        ctx->stat.ia_gid = stbuf->ia_gid;
 
-        if (valid & SHARD_MASK_SIZE)
-                ctx->stat.ia_size = stbuf->ia_size;
+    if (valid & SHARD_MASK_SIZE)
+        ctx->stat.ia_size = stbuf->ia_size;
 
-        if (valid & SHARD_MASK_BLOCKS)
-                ctx->stat.ia_blocks = stbuf->ia_blocks;
+    if (valid & SHARD_MASK_BLOCKS)
+        ctx->stat.ia_blocks = stbuf->ia_blocks;
 
-        if (valid & SHARD_MASK_TIMES) {
-                SHARD_TIME_UPDATE (ctx->stat.ia_mtime, ctx->stat.ia_mtime_nsec,
-                                   stbuf->ia_mtime, stbuf->ia_mtime_nsec);
-                SHARD_TIME_UPDATE (ctx->stat.ia_ctime, ctx->stat.ia_ctime_nsec,
-                                   stbuf->ia_ctime, stbuf->ia_ctime_nsec);
-                SHARD_TIME_UPDATE (ctx->stat.ia_atime, ctx->stat.ia_atime_nsec,
-                                   stbuf->ia_atime, stbuf->ia_atime_nsec);
-        }
+    if (valid & SHARD_MASK_TIMES) {
+        SHARD_TIME_UPDATE(ctx->stat.ia_mtime, ctx->stat.ia_mtime_nsec,
+                          stbuf->ia_mtime, stbuf->ia_mtime_nsec);
+        SHARD_TIME_UPDATE(ctx->stat.ia_ctime, ctx->stat.ia_ctime_nsec,
+                          stbuf->ia_ctime, stbuf->ia_ctime_nsec);
+        SHARD_TIME_UPDATE(ctx->stat.ia_atime, ctx->stat.ia_atime_nsec,
+                          stbuf->ia_atime, stbuf->ia_atime_nsec);
+    }
 
-        if (valid & SHARD_MASK_OTHERS) {
-                ctx->stat.ia_ino = stbuf->ia_ino;
-                gf_uuid_copy (ctx->stat.ia_gfid, stbuf->ia_gfid);
-                ctx->stat.ia_dev = stbuf->ia_dev;
-                ctx->stat.ia_type = stbuf->ia_type;
-                ctx->stat.ia_rdev = stbuf->ia_rdev;
-                ctx->stat.ia_blksize = stbuf->ia_blksize;
-        }
+    if (valid & SHARD_MASK_OTHERS) {
+        ctx->stat.ia_ino = stbuf->ia_ino;
+        gf_uuid_copy(ctx->stat.ia_gfid, stbuf->ia_gfid);
+        ctx->stat.ia_dev = stbuf->ia_dev;
+        ctx->stat.ia_type = stbuf->ia_type;
+        ctx->stat.ia_rdev = stbuf->ia_rdev;
+        ctx->stat.ia_blksize = stbuf->ia_blksize;
+    }
 
-        if (valid & SHARD_MASK_REFRESH_RESET)
-                ctx->refresh = _gf_false;
+    if (valid & SHARD_MASK_REFRESH_RESET)
+        ctx->refresh = _gf_false;
 
-        return 0;
+    return 0;
 }
 
 int
-shard_inode_ctx_set (inode_t *inode, xlator_t *this, struct iatt *stbuf,
-                     uint64_t block_size, int32_t valid)
+shard_inode_ctx_set(inode_t *inode, xlator_t *this, struct iatt *stbuf,
+                    uint64_t block_size, int32_t valid)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_set (inode, this, stbuf, block_size,
-                                             valid);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_set(inode, this, stbuf, block_size, valid);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-__shard_inode_ctx_invalidate (inode_t *inode, xlator_t *this, struct iatt *stbuf)
+__shard_inode_ctx_set_refresh_flag(inode_t *inode, xlator_t *this)
 {
-        int                 ret = -1;
-        shard_inode_ctx_t  *ctx = NULL;
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
 
-        ret = __shard_inode_ctx_get (inode, this, &ctx);
-        if (ret)
-                return ret;
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
+        return ret;
 
-        if ((stbuf->ia_size != ctx->stat.ia_size) ||
-            (stbuf->ia_blocks != ctx->stat.ia_blocks))
-                ctx->refresh = _gf_true;
+    ctx->refresh = _gf_true;
 
-        return 0;
+    return 0;
 }
-
 int
-shard_inode_ctx_invalidate (inode_t *inode, xlator_t *this, struct iatt *stbuf)
+shard_inode_ctx_set_refresh_flag(inode_t *inode, xlator_t *this)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_invalidate (inode, this, stbuf);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_set_refresh_flag(inode, this);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-__shard_inode_ctx_get_block_size (inode_t *inode, xlator_t *this,
-                                  uint64_t *block_size)
+__shard_inode_ctx_mark_dir_refreshed(inode_t *inode, xlator_t *this)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
 
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret < 0)
-                return ret;
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
+        return ret;
 
-        ctx = (shard_inode_ctx_t *) ctx_uint;
+    ctx->refreshed = _gf_true;
+    return 0;
+}
 
-        *block_size = ctx->block_size;
+int
+shard_inode_ctx_mark_dir_refreshed(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
 
-        return 0;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_mark_dir_refreshed(inode, this);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
 int
-shard_inode_ctx_get_block_size (inode_t *inode, xlator_t *this,
-                                uint64_t *block_size)
+__shard_inode_ctx_add_to_fsync_list(inode_t *base_inode, xlator_t *this,
+                                    inode_t *shard_inode)
 {
-        int ret = -1;
+    int ret = -1;
+    shard_inode_ctx_t *base_ictx = NULL;
+    shard_inode_ctx_t *shard_ictx = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_get_block_size (inode, this,
-                                                        block_size);
-        }
-        UNLOCK (&inode->lock);
+    ret = __shard_inode_ctx_get(base_inode, this, &base_ictx);
+    if (ret)
+        return ret;
 
+    ret = __shard_inode_ctx_get(shard_inode, this, &shard_ictx);
+    if (ret)
         return ret;
+
+    if (shard_ictx->fsync_needed) {
+        shard_ictx->fsync_needed++;
+        return 1;
+    }
+
+    list_add_tail(&shard_ictx->to_fsync_list, &base_ictx->to_fsync_list);
+    shard_ictx->inode = shard_inode;
+    shard_ictx->fsync_needed++;
+    base_ictx->fsync_count++;
+    shard_ictx->base_inode = base_inode;
+
+    return 0;
 }
 
 int
-__shard_inode_ctx_get_all (inode_t *inode, xlator_t *this,
-                           shard_inode_ctx_t *ctx_out)
+shard_inode_ctx_add_to_fsync_list(inode_t *base_inode, xlator_t *this,
+                                  inode_t *shard_inode)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    int ret = -1;
 
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret < 0)
-                return ret;
+    /* This ref acts as a refkeepr on the base inode. We
+     * need to keep this inode alive as it holds the head
+     * of the to_fsync_list.
+     */
+    inode_ref(base_inode);
+    inode_ref(shard_inode);
 
-        ctx = (shard_inode_ctx_t *) ctx_uint;
+    LOCK(&base_inode->lock);
+    LOCK(&shard_inode->lock);
+    {
+        ret = __shard_inode_ctx_add_to_fsync_list(base_inode, this,
+                                                  shard_inode);
+    }
+    UNLOCK(&shard_inode->lock);
+    UNLOCK(&base_inode->lock);
 
-        memcpy (ctx_out, ctx, sizeof (shard_inode_ctx_t));
-        return 0;
+    /* Unref the base inode corresponding to the ref above, if the shard is
+     * found to be already part of the fsync list.
+     */
+    if (ret != 0) {
+        inode_unref(base_inode);
+        inode_unref(shard_inode);
+    }
+    return ret;
 }
 
-int
-shard_inode_ctx_get_all (inode_t *inode, xlator_t *this,
-                         shard_inode_ctx_t *ctx_out)
+gf_boolean_t
+__shard_inode_ctx_needs_lookup(inode_t *inode, xlator_t *this)
 {
-        int ret = -1;
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_get_all (inode, this, ctx_out);
-        }
-        UNLOCK (&inode->lock);
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    /* If inode ctx get fails, better to err on the side of caution and
+     * try again? Unless the failure is due to mem-allocation.
+     */
+    if (ret)
+        return _gf_true;
+
+    return !ctx->refreshed;
+}
+
+gf_boolean_t
+shard_inode_ctx_needs_lookup(inode_t *inode, xlator_t *this)
+{
+    gf_boolean_t flag = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        flag = __shard_inode_ctx_needs_lookup(inode, this);
+    }
+    UNLOCK(&inode->lock);
+
+    return flag;
+}
+int
+__shard_inode_ctx_invalidate(inode_t *inode, xlator_t *this, struct iatt *stbuf)
+{
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
 
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
         return ret;
+
+    if ((stbuf->ia_size != ctx->stat.ia_size) ||
+        (stbuf->ia_blocks != ctx->stat.ia_blocks))
+        ctx->refresh = _gf_true;
+
+    return 0;
 }
 
 int
-__shard_inode_ctx_fill_iatt_from_cache (inode_t *inode, xlator_t *this,
-                                        struct iatt *buf,
-                                        gf_boolean_t *need_refresh)
+shard_inode_ctx_invalidate(inode_t *inode, xlator_t *this, struct iatt *stbuf)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    int ret = -1;
 
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret < 0)
-                return ret;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_invalidate(inode, this, stbuf);
+    }
+    UNLOCK(&inode->lock);
 
-        ctx = (shard_inode_ctx_t *) ctx_uint;
+    return ret;
+}
 
-        if (ctx->refresh == _gf_false)
-                *buf = ctx->stat;
-        else
-                *need_refresh = _gf_true;
+int
+__shard_inode_ctx_get_block_size(inode_t *inode, xlator_t *this,
+                                 uint64_t *block_size)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-        return 0;
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
+
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    *block_size = ctx->block_size;
+
+    return 0;
 }
 
 int
-shard_inode_ctx_fill_iatt_from_cache (inode_t *inode, xlator_t *this,
-                                      struct iatt *buf,
-                                      gf_boolean_t *need_refresh)
+shard_inode_ctx_get_block_size(inode_t *inode, xlator_t *this,
+                               uint64_t *block_size)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_fill_iatt_from_cache (inode, this, buf,
-                                                              need_refresh);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get_block_size(inode, this, block_size);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
 
+int
+__shard_inode_ctx_get_fsync_count(inode_t *inode, xlator_t *this,
+                                  int *fsync_count)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
         return ret;
+
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    *fsync_count = ctx->fsync_needed;
+
+    return 0;
 }
 
-void
-shard_local_wipe (shard_local_t *local)
+int
+shard_inode_ctx_get_fsync_count(inode_t *inode, xlator_t *this,
+                                int *fsync_count)
 {
-        int  i     = 0;
-        int  count = 0;
+    int ret = -1;
 
-        count = local->num_blocks;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get_fsync_count(inode, this, fsync_count);
+    }
+    UNLOCK(&inode->lock);
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->dot_shard_loc);
-        loc_wipe (&local->loc2);
-        loc_wipe (&local->tmp_loc);
+    return ret;
+}
+int
+__shard_inode_ctx_get_all(inode_t *inode, xlator_t *this,
+                          shard_inode_ctx_t *ctx_out)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-        if (local->fd)
-                fd_unref (local->fd);
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
-        if (local->xattr_rsp)
-                dict_unref (local->xattr_rsp);
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        for (i = 0; i < count; i++) {
-                if (!local->inode_list)
-                        break;
+    memcpy(ctx_out, ctx, sizeof(shard_inode_ctx_t));
+    return 0;
+}
 
-                if (local->inode_list[i])
-                        inode_unref (local->inode_list[i]);
-        }
+int
+shard_inode_ctx_get_all(inode_t *inode, xlator_t *this,
+                        shard_inode_ctx_t *ctx_out)
+{
+    int ret = -1;
 
-        GF_FREE (local->inode_list);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get_all(inode, this, ctx_out);
+    }
+    UNLOCK(&inode->lock);
 
-        GF_FREE (local->vector);
-        if (local->iobref)
-                iobref_unref (local->iobref);
-        if (local->list_inited)
-                gf_dirent_free (&local->entries_head);
+    return ret;
 }
 
 int
-shard_modify_size_and_block_count (struct iatt *stbuf, dict_t *dict)
+__shard_inode_ctx_fill_iatt_from_cache(inode_t *inode, xlator_t *this,
+                                       struct iatt *buf,
+                                       gf_boolean_t *need_refresh)
 {
-        int                  ret       = -1;
-        void                *size_attr = NULL;
-        uint64_t             size_array[4];
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-        ret = dict_get_ptr (dict, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
-        if (ret) {
-                gf_msg_callingfn (THIS->name, GF_LOG_ERROR, 0,
-                                  SHARD_MSG_INTERNAL_XATTR_MISSING, "Failed to "
-                                  "get "GF_XATTR_SHARD_FILE_SIZE" for %s",
-                                  uuid_utoa (stbuf->ia_gfid));
-                return ret;
-        }
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
 
-        memcpy (size_array, size_attr, sizeof (size_array));
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        stbuf->ia_size = ntoh64 (size_array[0]);
-        stbuf->ia_blocks = ntoh64 (size_array[2]);
+    if (ctx->refresh == _gf_false)
+        *buf = ctx->stat;
+    else
+        *need_refresh = _gf_true;
 
-        return 0;
+    return 0;
 }
 
 int
-shard_call_count_return (call_frame_t *frame)
+shard_inode_ctx_fill_iatt_from_cache(inode_t *inode, xlator_t *this,
+                                     struct iatt *buf,
+                                     gf_boolean_t *need_refresh)
 {
-        int             call_count = 0;
-        shard_local_t  *local      = NULL;
+    int ret = -1;
 
-        local = frame->local;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_fill_iatt_from_cache(inode, this, buf,
+                                                     need_refresh);
+    }
+    UNLOCK(&inode->lock);
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        return call_count;
+    return ret;
 }
 
-static int
-shard_init_dot_shard_loc (xlator_t *this, shard_local_t *local)
-{
-        int    ret           = -1;
-        loc_t *dot_shard_loc = NULL;
-
-        if (!local)
-                return -1;
-
-        dot_shard_loc = &local->dot_shard_loc;
-        dot_shard_loc->inode = inode_new (this->itable);
-        dot_shard_loc->parent = inode_ref (this->itable->root);
-        ret = inode_path (dot_shard_loc->parent, GF_SHARD_DIR,
-                          (char **)&dot_shard_loc->path);
-        if (ret < 0 || !(dot_shard_loc->inode)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_PATH_FAILED,
-                        "Inode path failed on %s", GF_SHARD_DIR);
-                goto out;
-        }
+void
+shard_local_wipe(shard_local_t *local)
+{
+    int i = 0;
+    int count = 0;
 
-        dot_shard_loc->name = strrchr (dot_shard_loc->path, '/');
-        if (dot_shard_loc->name)
-                dot_shard_loc->name++;
+    count = local->num_blocks;
 
-        ret = 0;
-out:
+    syncbarrier_destroy(&local->barrier);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->dot_shard_loc);
+    loc_wipe(&local->dot_shard_rm_loc);
+    loc_wipe(&local->loc2);
+    loc_wipe(&local->tmp_loc);
+    loc_wipe(&local->int_inodelk.loc);
+    loc_wipe(&local->int_entrylk.loc);
+    loc_wipe(&local->newloc);
+
+    if (local->name)
+        GF_FREE(local->name);
+
+    if (local->int_entrylk.basename)
+        GF_FREE(local->int_entrylk.basename);
+    if (local->fd)
+        fd_unref(local->fd);
+
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
+
+    for (i = 0; i < count; i++) {
+        if (!local->inode_list)
+            break;
+
+        if (local->inode_list[i])
+            inode_unref(local->inode_list[i]);
+    }
+
+    GF_FREE(local->inode_list);
+
+    GF_FREE(local->vector);
+    if (local->iobref)
+        iobref_unref(local->iobref);
+    if (local->list_inited)
+        gf_dirent_free(&local->entries_head);
+    if (local->inodelk_frame)
+        SHARD_STACK_DESTROY(local->inodelk_frame);
+    if (local->entrylk_frame)
+        SHARD_STACK_DESTROY(local->entrylk_frame);
+}
+
+int
+shard_modify_size_and_block_count(struct iatt *stbuf, dict_t *dict)
+{
+    int ret = -1;
+    void *size_attr = NULL;
+    uint64_t size_array[4];
+
+    ret = dict_get_ptr(dict, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret) {
+        gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0,
+                         SHARD_MSG_INTERNAL_XATTR_MISSING,
+                         "Failed to "
+                         "get " GF_XATTR_SHARD_FILE_SIZE " for %s",
+                         uuid_utoa(stbuf->ia_gfid));
         return ret;
+    }
+
+    memcpy(size_array, size_attr, sizeof(size_array));
+
+    stbuf->ia_size = ntoh64(size_array[0]);
+    stbuf->ia_blocks = ntoh64(size_array[2]);
+
+    return 0;
 }
 
-void
-__shard_update_shards_inode_list (inode_t *linked_inode, xlator_t *this,
-                                  inode_t *base_inode, int block_num)
+int
+shard_call_count_return(call_frame_t *frame)
 {
-        char                block_bname[256] = {0,};
-        inode_t            *lru_inode        = NULL;
-        shard_priv_t       *priv             = NULL;
-        shard_inode_ctx_t  *ctx              = NULL;
-        shard_inode_ctx_t  *lru_inode_ctx    = NULL;
+    int call_count = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
 
-        priv = this->private;
+    LOCK(&frame->lock);
+    {
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        shard_inode_ctx_get (linked_inode, this, &ctx);
+    return call_count;
+}
+
+static char *
+shard_internal_dir_string(shard_internal_dir_type_t type)
+{
+    char *str = NULL;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            str = GF_SHARD_DIR;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            str = GF_SHARD_REMOVE_ME_DIR;
+            break;
+        default:
+            break;
+    }
+    return str;
+}
 
-        if (list_empty (&ctx->ilist)) {
-                if (priv->inode_count + 1 <= SHARD_MAX_INODES) {
-                /* If this inode was linked here for the first time (indicated
-                 * by empty list), and if there is still space in the priv list,
-                 * add this ctx to the tail of the list.
+static int
+shard_init_internal_dir_loc(xlator_t *this, shard_local_t *local,
+                            shard_internal_dir_type_t type)
+{
+    int ret = -1;
+    char *bname = NULL;
+    inode_t *parent = NULL;
+    loc_t *internal_dir_loc = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!local)
+        return -1;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            internal_dir_loc = &local->dot_shard_loc;
+            bname = GF_SHARD_DIR;
+            parent = inode_ref(this->itable->root);
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            internal_dir_loc = &local->dot_shard_rm_loc;
+            bname = GF_SHARD_REMOVE_ME_DIR;
+            parent = inode_ref(priv->dot_shard_inode);
+            break;
+        default:
+            break;
+    }
+
+    internal_dir_loc->inode = inode_new(this->itable);
+    internal_dir_loc->parent = parent;
+    ret = inode_path(internal_dir_loc->parent, bname,
+                     (char **)&internal_dir_loc->path);
+    if (ret < 0 || !(internal_dir_loc->inode)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path failed on %s", bname);
+        goto out;
+    }
+
+    internal_dir_loc->name = strrchr(internal_dir_loc->path, '/');
+    if (internal_dir_loc->name)
+        internal_dir_loc->name++;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+inode_t *
+__shard_update_shards_inode_list(inode_t *linked_inode, xlator_t *this,
+                                 inode_t *base_inode, int block_num,
+                                 uuid_t gfid)
+{
+    char block_bname[256] = {
+        0,
+    };
+    inode_t *lru_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *lru_inode_ctx = NULL;
+    shard_inode_ctx_t *lru_base_inode_ctx = NULL;
+    inode_t *fsync_inode = NULL;
+    inode_t *lru_base_inode = NULL;
+    gf_boolean_t do_fsync = _gf_false;
+
+    priv = this->private;
+
+    shard_inode_ctx_get(linked_inode, this, &ctx);
+
+    if (list_empty(&ctx->ilist)) {
+        if (priv->inode_count + 1 <= priv->lru_limit) {
+            /* If this inode was linked here for the first time (indicated
+             * by empty list), and if there is still space in the priv list,
+             * add this ctx to the tail of the list.
+             */
+            /* For as long as an inode is in lru list, we try to
+             * keep it alive by holding a ref on it.
+             */
+            inode_ref(linked_inode);
+            if (base_inode)
+                gf_uuid_copy(ctx->base_gfid, base_inode->gfid);
+            else
+                gf_uuid_copy(ctx->base_gfid, gfid);
+            ctx->block_num = block_num;
+            list_add_tail(&ctx->ilist, &priv->ilist_head);
+            priv->inode_count++;
+            ctx->base_inode = inode_ref(base_inode);
+        } else {
+            /*If on the other hand there is no available slot for this inode
+             * in the list, delete the lru inode from the head of the list,
+             * unlink it. And in its place add this new inode into the list.
+             */
+            lru_inode_ctx = list_first_entry(&priv->ilist_head,
+                                             shard_inode_ctx_t, ilist);
+            GF_ASSERT(lru_inode_ctx->block_num > 0);
+            lru_base_inode = lru_inode_ctx->base_inode;
+            list_del_init(&lru_inode_ctx->ilist);
+            lru_inode = inode_find(linked_inode->table,
+                                   lru_inode_ctx->stat.ia_gfid);
+            /* If the lru inode was part of the pending-fsync list,
+             * the base inode needs to be unref'd, the lru inode
+             * deleted from fsync list and fsync'd in a new frame,
+             * and then unlinked in memory and forgotten.
+             */
+            if (!lru_base_inode)
+                goto after_fsync_check;
+            LOCK(&lru_base_inode->lock);
+            LOCK(&lru_inode->lock);
+            {
+                if (!list_empty(&lru_inode_ctx->to_fsync_list)) {
+                    list_del_init(&lru_inode_ctx->to_fsync_list);
+                    lru_inode_ctx->fsync_needed = 0;
+                    do_fsync = _gf_true;
+                    __shard_inode_ctx_get(lru_base_inode, this,
+                                          &lru_base_inode_ctx);
+                    lru_base_inode_ctx->fsync_count--;
+                }
+            }
+            UNLOCK(&lru_inode->lock);
+            UNLOCK(&lru_base_inode->lock);
+
+        after_fsync_check:
+            if (!do_fsync) {
+                shard_make_block_bname(lru_inode_ctx->block_num,
+                                       lru_inode_ctx->base_gfid, block_bname,
+                                       sizeof(block_bname));
+                /* The following unref corresponds to the ref held at
+                 * the time the shard was added to the lru list.
                  */
-                        gf_uuid_copy (ctx->base_gfid, base_inode->gfid);
-                        ctx->block_num = block_num;
-                        list_add_tail (&ctx->ilist, &priv->ilist_head);
-                        priv->inode_count++;
-                } else {
-                /*If on the other hand there is no available slot for this inode
-                 * in the list, delete the lru inode from the head of the list,
-                 * unlink it. And in its place add this new inode into the list.
+                inode_unref(lru_inode);
+                inode_unlink(lru_inode, priv->dot_shard_inode, block_bname);
+                inode_forget(lru_inode, 0);
+            } else {
+                /* The following unref corresponds to the ref
+                 * held when the shard was added to fsync list.
                  */
-                        lru_inode_ctx = list_first_entry (&priv->ilist_head,
-                                                          shard_inode_ctx_t,
-                                                          ilist);
-                        GF_ASSERT (lru_inode_ctx->block_num > 0);
-                        list_del_init (&lru_inode_ctx->ilist);
-                        lru_inode = inode_find (linked_inode->table,
-                                                lru_inode_ctx->stat.ia_gfid);
-                        shard_make_block_bname (lru_inode_ctx->block_num,
-                                                lru_inode_ctx->base_gfid,
-                                                block_bname,
-                                                sizeof (block_bname));
-                        inode_unlink (lru_inode, priv->dot_shard_inode,
-                                      block_bname);
-                        /* The following unref corresponds to the ref held by
-                         * inode_find() above.
-                         */
-                        inode_forget (lru_inode, 0);
-                        inode_unref (lru_inode);
-                        gf_uuid_copy (ctx->base_gfid, base_inode->gfid);
-                        ctx->block_num = block_num;
-                        list_add_tail (&ctx->ilist, &priv->ilist_head);
-                }
-        } else {
-         /* If this is not the first time this inode is being operated on, move
+                inode_unref(lru_inode);
+                fsync_inode = lru_inode;
+                if (lru_base_inode)
+                    inode_unref(lru_base_inode);
+            }
+            /* The following unref corresponds to the ref
+             * held by inode_find() above.
+             */
+            inode_unref(lru_inode);
+
+            /* The following unref corresponds to the ref held on the base shard
+             * at the time of adding shard inode to lru list
+             */
+            if (lru_base_inode)
+                inode_unref(lru_base_inode);
+
+            /* For as long as an inode is in lru list, we try to
+             * keep it alive by holding a ref on it.
+             */
+            inode_ref(linked_inode);
+            if (base_inode)
+                gf_uuid_copy(ctx->base_gfid, base_inode->gfid);
+            else
+                gf_uuid_copy(ctx->base_gfid, gfid);
+            ctx->block_num = block_num;
+            ctx->base_inode = inode_ref(base_inode);
+            list_add_tail(&ctx->ilist, &priv->ilist_head);
+        }
+    } else {
+        /* If this is not the first time this inode is being operated on, move
          * it to the most recently used end of the list.
          */
-                list_move_tail (&ctx->ilist, &priv->ilist_head);
-        }
+        list_move_tail(&ctx->ilist, &priv->ilist_head);
+    }
+    return fsync_inode;
 }
 
 int
-shard_common_inode_write_failure_unwind (glusterfs_fop_t fop,
-                                         call_frame_t *frame, int32_t op_ret,
-                                         int32_t op_errno)
-{
-        switch (fop) {
+shard_common_failure_unwind(glusterfs_fop_t fop, call_frame_t *frame,
+                            int32_t op_ret, int32_t op_errno)
+{
+    switch (fop) {
+        case GF_FOP_LOOKUP:
+            SHARD_STACK_UNWIND(lookup, frame, op_ret, op_errno, NULL, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_STAT:
+            SHARD_STACK_UNWIND(stat, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_FSTAT:
+            SHARD_STACK_UNWIND(fstat, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_TRUNCATE:
+            SHARD_STACK_UNWIND(truncate, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_FTRUNCATE:
+            SHARD_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_MKNOD:
+            SHARD_STACK_UNWIND(mknod, frame, op_ret, op_errno, NULL, NULL, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_LINK:
+            SHARD_STACK_UNWIND(link, frame, op_ret, op_errno, NULL, NULL, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_CREATE:
+            SHARD_STACK_UNWIND(create, frame, op_ret, op_errno, NULL, NULL,
+                               NULL, NULL, NULL, NULL);
+            break;
+        case GF_FOP_UNLINK:
+            SHARD_STACK_UNWIND(unlink, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_RENAME:
+            SHARD_STACK_UNWIND(rename, frame, op_ret, op_errno, NULL, NULL,
+                               NULL, NULL, NULL, NULL);
+            break;
         case GF_FOP_WRITE:
-                SHARD_STACK_UNWIND (writev, frame, op_ret, op_errno,
-                                    NULL, NULL, NULL);
-                break;
+            SHARD_STACK_UNWIND(writev, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
         case GF_FOP_FALLOCATE:
-                SHARD_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
-                                    NULL, NULL, NULL);
-                break;
+            SHARD_STACK_UNWIND(fallocate, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
         case GF_FOP_ZEROFILL:
-                SHARD_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
-                                    NULL, NULL, NULL);
-                break;
+            SHARD_STACK_UNWIND(zerofill, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
         case GF_FOP_DISCARD:
-                SHARD_STACK_UNWIND (discard, frame, op_ret, op_errno,
-                                    NULL, NULL, NULL);
-                break;
+            SHARD_STACK_UNWIND(discard, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_READ:
+            SHARD_STACK_UNWIND(readv, frame, op_ret, op_errno, NULL, -1, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_FSYNC:
+            SHARD_STACK_UNWIND(fsync, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_REMOVEXATTR:
+            SHARD_STACK_UNWIND(removexattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_FREMOVEXATTR:
+            SHARD_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_FGETXATTR:
+            SHARD_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_GETXATTR:
+            SHARD_STACK_UNWIND(getxattr, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_FSETXATTR:
+            SHARD_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_SETXATTR:
+            SHARD_STACK_UNWIND(setxattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_SETATTR:
+            SHARD_STACK_UNWIND(setattr, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_FSETATTR:
+            SHARD_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_SEEK:
+            SHARD_STACK_UNWIND(seek, frame, op_ret, op_errno, 0, NULL);
+            break;
         default:
-                gf_msg (THIS->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
-                        "Invalid fop id = %d", fop);
-                break;
-        }
-        return 0;
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "Invalid fop id = %d", fop);
+            break;
+    }
+    return 0;
 }
 
 int
-shard_common_inode_write_success_unwind (glusterfs_fop_t fop,
-                                         call_frame_t *frame, int32_t op_ret)
+shard_common_inode_write_success_unwind(glusterfs_fop_t fop,
+                                        call_frame_t *frame, int32_t op_ret)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = frame->local;
 
-        local = frame->local;
+    /* the below 3 variables are required because, in SHARD_STACK_UNWIND()
+       macro, there is a check for local being null. So many static analyzers
+       backtrace the code with assumption of possible (local == NULL) case,
+       and complains for below lines. By handling it like below, we overcome
+       the warnings */
 
-        switch (fop) {
+    struct iatt *prebuf = ((local) ? &local->prebuf : NULL);
+    struct iatt *postbuf = ((local) ? &local->postbuf : NULL);
+    dict_t *xattr_rsp = ((local) ? local->xattr_rsp : NULL);
+
+    switch (fop) {
         case GF_FOP_WRITE:
-                SHARD_STACK_UNWIND (writev, frame, op_ret, 0, &local->prebuf,
-                                    &local->postbuf, local->xattr_rsp);
-                break;
+            SHARD_STACK_UNWIND(writev, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
         case GF_FOP_FALLOCATE:
-                SHARD_STACK_UNWIND (fallocate, frame, op_ret, 0, &local->prebuf,
-                                    &local->postbuf, local->xattr_rsp);
-                break;
+            SHARD_STACK_UNWIND(fallocate, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
         case GF_FOP_ZEROFILL:
-                SHARD_STACK_UNWIND (zerofill, frame, op_ret, 0, &local->prebuf,
-                                    &local->postbuf, local->xattr_rsp);
-                break;
+            SHARD_STACK_UNWIND(zerofill, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
         case GF_FOP_DISCARD:
-                SHARD_STACK_UNWIND (discard, frame, op_ret, 0, &local->prebuf,
-                                    &local->postbuf, local->xattr_rsp);
-                break;
+            SHARD_STACK_UNWIND(discard, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
         default:
-                gf_msg (THIS->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
-                        "Invalid fop id = %d", fop);
-                break;
-        }
-        return 0;
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "Invalid fop id = %d", fop);
+            break;
+    }
+    return 0;
 }
 
 int
-shard_common_resolve_shards (call_frame_t *frame, xlator_t *this,
-                             inode_t *res_inode,
-                             shard_post_resolve_fop_handler_t post_res_handler)
-{
-        int                   i              = -1;
-        uint32_t              shard_idx_iter = 0;
-        char                  path[PATH_MAX] = {0,};
-        inode_t              *inode          = NULL;
-        shard_priv_t         *priv           = NULL;
-        shard_local_t        *local          = NULL;
-
-        priv = this->private;
-        local = frame->local;
-        shard_idx_iter = local->first_block;
-
-        if (local->op_ret < 0)
-                goto out;
-
-        while (shard_idx_iter <= local->last_block) {
-                i++;
-                if (shard_idx_iter == 0) {
-                        local->inode_list[i] = inode_ref (res_inode);
-                        shard_idx_iter++;
-                        continue;
-                }
-
-                shard_make_block_abspath (shard_idx_iter, res_inode->gfid, path,
-                                          sizeof(path));
-
-                inode = NULL;
-                inode = inode_resolve (this->itable, path);
-                if (inode) {
-                        gf_msg_debug (this->name, 0, "Shard %d already "
-                                "present. gfid=%s. Saving inode for future.",
-                                shard_idx_iter, uuid_utoa(inode->gfid));
-                        shard_idx_iter++;
-                        local->inode_list[i] = inode;
-                        /* Let the ref on the inodes that are already present
-                         * in inode table still be held so that they don't get
-                         * forgotten by the time the fop reaches the actual
-                         * write stage.
-                         */
-                        LOCK(&priv->lock);
-                        {
-                                __shard_update_shards_inode_list (inode, this,
-                                                                  res_inode,
-                                                                shard_idx_iter);
-                        }
-                        UNLOCK(&priv->lock);
-
-                         continue;
-                } else {
-                        local->call_count++;
-                        shard_idx_iter++;
-                }
-        }
+shard_evicted_inode_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno,
+                              struct iatt *prebuf, struct iatt *postbuf,
+                              dict_t *xdata)
+{
+    char block_bname[256] = {
+        0,
+    };
+    fd_t *anon_fd = cookie;
+    inode_t *shard_inode = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+
+    if (anon_fd == NULL || op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, SHARD_MSG_MEMALLOC_FAILED,
+               "fsync failed on shard");
+        goto out;
+    }
+    shard_inode = anon_fd->inode;
+
+    LOCK(&priv->lock);
+    LOCK(&shard_inode->lock);
+    {
+        __shard_inode_ctx_get(shard_inode, this, &ctx);
+        if ((list_empty(&ctx->to_fsync_list)) && (list_empty(&ctx->ilist))) {
+            shard_make_block_bname(ctx->block_num, shard_inode->gfid,
+                                   block_bname, sizeof(block_bname));
+            inode_unlink(shard_inode, priv->dot_shard_inode, block_bname);
+            /* The following unref corresponds to the ref held by
+             * inode_link() at the time the shard was created or
+             * looked up
+             */
+            inode_unref(shard_inode);
+            inode_forget(shard_inode, 0);
+        }
+    }
+    UNLOCK(&shard_inode->lock);
+    UNLOCK(&priv->lock);
 
 out:
-        post_res_handler (frame, this);
-        return 0;
+    if (anon_fd)
+        fd_unref(anon_fd);
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 int
-shard_update_file_size_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, dict_t *dict,
-                            dict_t *xdata)
-{
-        inode_t       *inode = NULL;
-        shard_local_t *local = NULL;
-
-        local = frame->local;
-
-        if ((local->fd) && (local->fd->inode))
-                inode = local->fd->inode;
-        else if (local->loc.inode)
-                inode = local->loc.inode;
+shard_initiate_evicted_inode_fsync(xlator_t *this, inode_t *inode)
+{
+    fd_t *anon_fd = NULL;
+    call_frame_t *fsync_frame = NULL;
+
+    fsync_frame = create_frame(this, this->ctx->pool);
+    if (!fsync_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create new frame "
+               "to fsync shard");
+        return -1;
+    }
+
+    anon_fd = fd_anonymous(inode);
+    if (!anon_fd) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create anon fd to"
+               " fsync shard");
+        STACK_DESTROY(fsync_frame->root);
+        return -1;
+    }
+
+    STACK_WIND_COOKIE(fsync_frame, shard_evicted_inode_fsync_cbk, anon_fd,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                      anon_fd, 1, NULL);
+    return 0;
+}
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        SHARD_MSG_UPDATE_FILE_SIZE_FAILED, "Update to file size"
-                        " xattr failed on %s", uuid_utoa (inode->gfid));
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto err;
-        }
+int
+shard_common_inode_write_post_lookup_shards_handler(call_frame_t *frame,
+                                                    xlator_t *this);
 
-        if (shard_modify_size_and_block_count (&local->postbuf, dict)) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
+int
+shard_common_resolve_shards(call_frame_t *frame, xlator_t *this,
+                            shard_post_resolve_fop_handler_t post_res_handler)
+{
+    int i = -1;
+    uint32_t shard_idx_iter = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *res_inode = NULL;
+    inode_t *fsync_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    uint64_t resolve_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+    local->call_count = 0;
+    shard_idx_iter = local->first_block;
+    res_inode = local->resolver_base_inode;
+
+    if ((local->op_ret < 0) || (local->resolve_not))
+        goto out;
+
+    /* If this prealloc FOP is for fresh file creation, then the size of the
+     * file will be 0. Then there will be no shards associated with this file.
+     * So we can skip the lookup process for the shards which do not exists
+     * and directly issue mknod to crete shards.
+     *
+     * In case the prealloc fop is to extend the preallocated file to bigger
+     * size then just lookup and populate inodes of existing shards and
+     * update the create count
+     */
+    if (local->fop == GF_FOP_FALLOCATE) {
+        if (!local->prebuf.ia_size) {
+            local->inode_list[0] = inode_ref(res_inode);
+            local->create_count = local->last_block;
+            shard_common_inode_write_post_lookup_shards_handler(frame, this);
+            return 0;
+        }
+        if (local->prebuf.ia_size < local->total_size)
+            local->create_count = local->last_block -
+                                  ((local->prebuf.ia_size - 1) /
+                                   local->block_size);
+    }
+
+    resolve_count = local->last_block - local->create_count;
+
+    if (res_inode)
+        gf_uuid_copy(gfid, res_inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    while (shard_idx_iter <= resolve_count) {
+        i++;
+        if (shard_idx_iter == 0) {
+            local->inode_list[i] = inode_ref(res_inode);
+            shard_idx_iter++;
+            continue;
+        }
+
+        shard_make_block_abspath(shard_idx_iter, gfid, path, sizeof(path));
+
+        inode = NULL;
+        inode = inode_resolve(this->itable, path);
+        if (inode) {
+            gf_msg_debug(this->name, 0,
+                         "Shard %d already "
+                         "present. gfid=%s. Saving inode for future.",
+                         shard_idx_iter, uuid_utoa(inode->gfid));
+            local->inode_list[i] = inode;
+            /* Let the ref on the inodes that are already present
+             * in inode table still be held so that they don't get
+             * forgotten by the time the fop reaches the actual
+             * write stage.
+             */
+            LOCK(&priv->lock);
+            {
+                fsync_inode = __shard_update_shards_inode_list(
+                    inode, this, res_inode, shard_idx_iter, gfid);
+            }
+            UNLOCK(&priv->lock);
+            shard_idx_iter++;
+            if (fsync_inode)
+                shard_initiate_evicted_inode_fsync(this, fsync_inode);
+            continue;
+        } else {
+            local->call_count++;
+            shard_idx_iter++;
         }
+    }
+out:
+    post_res_handler(frame, this);
+    return 0;
+}
 
-        if (local->fop == GF_FOP_FTRUNCATE || local->fop == GF_FOP_TRUNCATE)
-                shard_inode_ctx_set (inode, this, &local->postbuf, 0,
-                                     SHARD_INODE_WRITE_MASK);
+int
+shard_update_file_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
+{
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if ((local->fd) && (local->fd->inode))
+        inode = local->fd->inode;
+    else if (local->loc.inode)
+        inode = local->loc.inode;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_UPDATE_FILE_SIZE_FAILED,
+               "Update to file size"
+               " xattr failed on %s",
+               uuid_utoa(inode->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
 
+    if (shard_modify_size_and_block_count(&local->postbuf, dict)) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
 err:
-        local->post_update_size_handler (frame, this);
-        return 0;
+    local->post_update_size_handler(frame, this);
+    return 0;
 }
 
 int
-shard_set_size_attrs (int64_t size, int64_t block_count, int64_t **size_attr_p)
+shard_set_size_attrs(int64_t size, int64_t block_count, int64_t **size_attr_p)
 {
-        int             ret       = -1;
-        int64_t       *size_attr = NULL;
+    int ret = -1;
+    int64_t *size_attr = NULL;
 
-        if (!size_attr_p)
-                goto out;
+    if (!size_attr_p)
+        goto out;
 
-        size_attr = GF_CALLOC (4, sizeof (int64_t), gf_shard_mt_int64_t);
-        if (!size_attr)
-                goto out;
+    size_attr = GF_CALLOC(4, sizeof(int64_t), gf_shard_mt_int64_t);
+    if (!size_attr)
+        goto out;
 
-        size_attr[0] = hton64 (size);
-        /* As sharding evolves, it _may_ be necessary to embed more pieces of
-         * information within the same xattr. So allocating slots for them in
-         * advance. For now, only bytes 0-63 and 128-191 which would make up the
-         * current size and block count respectively of the file are valid.
-         */
-        size_attr[2] = hton64 (block_count);
+    size_attr[0] = hton64(size);
+    /* As sharding evolves, it _may_ be necessary to embed more pieces of
+     * information within the same xattr. So allocating slots for them in
+     * advance. For now, only bytes 0-63 and 128-191 which would make up the
+     * current size and block count respectively of the file are valid.
+     */
+    size_attr[2] = hton64(block_count);
 
-        *size_attr_p = size_attr;
+    *size_attr_p = size_attr;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-shard_update_file_size (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        loc_t *loc,
-                        shard_post_update_size_fop_handler_t handler)
+shard_update_file_size(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       loc_t *loc, shard_post_update_size_fop_handler_t handler)
 {
-        int            ret       = -1;
-        int64_t       *size_attr = NULL;
-        inode_t       *inode     = NULL;
-        shard_local_t *local     = NULL;
-        dict_t        *xattr_req = NULL;
-
-        local = frame->local;
-        local->post_update_size_handler = handler;
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto out;
-        }
-
-        if (fd)
-                inode = fd->inode;
-        else
-                inode = loc->inode;
-
-        /* If both size and block count have not changed, then skip the xattrop.
-         */
-        if ((local->delta_size + local->hole_size == 0) &&
-            (local->delta_blocks == 0)) {
-                goto out;
-        }
+    int ret = -1;
+    int64_t *size_attr = NULL;
+    int64_t delta_blocks = 0;
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
+    dict_t *xattr_req = NULL;
 
-        ret = shard_set_size_attrs (local->delta_size + local->hole_size,
-                                    local->delta_blocks, &size_attr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, SHARD_MSG_SIZE_SET_FAILED,
-                        "Failed to set size attrs for %s",
-                        uuid_utoa (inode->gfid));
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto out;
-        }
+    local = frame->local;
+    local->post_update_size_handler = handler;
 
-        ret = dict_set_bin (xattr_req, GF_XATTR_SHARD_FILE_SIZE, size_attr,
-                            8 * 4);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_SET_FAILED,
-                        "Failed to set key %s into dict. gfid=%s",
-                        GF_XATTR_SHARD_FILE_SIZE, uuid_utoa (inode->gfid));
-                GF_FREE (size_attr);
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto out;
-        }
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (fd)
+        inode = fd->inode;
+    else
+        inode = loc->inode;
+
+    /* If both size and block count have not changed, then skip the xattrop.
+     */
+    delta_blocks = GF_ATOMIC_GET(local->delta_blocks);
+    if ((local->delta_size + local->hole_size == 0) && (delta_blocks == 0)) {
+        goto out;
+    }
+
+    ret = shard_set_size_attrs(local->delta_size + local->hole_size,
+                               delta_blocks, &size_attr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_SIZE_SET_FAILED,
+               "Failed to set size attrs for %s", uuid_utoa(inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_bin(xattr_req, GF_XATTR_SHARD_FILE_SIZE, size_attr, 8 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set key %s into dict. gfid=%s",
+               GF_XATTR_SHARD_FILE_SIZE, uuid_utoa(inode->gfid));
+        GF_FREE(size_attr);
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (fd)
-                STACK_WIND (frame, shard_update_file_size_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fxattrop, fd,
-                            GF_XATTROP_ADD_ARRAY64, xattr_req, NULL);
-        else
-                STACK_WIND (frame, shard_update_file_size_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->xattrop, loc,
-                            GF_XATTROP_ADD_ARRAY64, xattr_req, NULL);
+    if (fd)
+        STACK_WIND(frame, shard_update_file_size_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fd,
+                   GF_XATTROP_ADD_ARRAY64, xattr_req, NULL);
+    else
+        STACK_WIND(frame, shard_update_file_size_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, loc,
+                   GF_XATTROP_ADD_ARRAY64, xattr_req, NULL);
 
-        dict_unref (xattr_req);
-        return 0;
+    dict_unref(xattr_req);
+    return 0;
 
 out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        handler (frame, this);
-        return 0;
+    if (xattr_req)
+        dict_unref(xattr_req);
+    handler(frame, this);
+    return 0;
+}
+
+static inode_t *
+shard_link_internal_dir_inode(shard_local_t *local, inode_t *inode,
+                              struct iatt *buf, shard_internal_dir_type_t type)
+{
+    inode_t *linked_inode = NULL;
+    shard_priv_t *priv = NULL;
+    char *bname = NULL;
+    inode_t **priv_inode = NULL;
+    inode_t *parent = NULL;
+
+    priv = THIS->private;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            bname = GF_SHARD_DIR;
+            priv_inode = &priv->dot_shard_inode;
+            parent = inode->table->root;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            bname = GF_SHARD_REMOVE_ME_DIR;
+            priv_inode = &priv->dot_shard_rm_inode;
+            parent = priv->dot_shard_inode;
+            break;
+        default:
+            break;
+    }
 
+    linked_inode = inode_link(inode, parent, bname, buf);
+    inode_lookup(linked_inode);
+    *priv_inode = linked_inode;
+    return linked_inode;
 }
 
-static void
-shard_link_dot_shard_inode (shard_local_t *local, inode_t *inode,
-                            struct iatt *buf)
+int
+shard_refresh_internal_dir_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
 {
-        inode_t       *linked_inode = NULL;
-        shard_priv_t  *priv         = NULL;
+    shard_local_t *local = NULL;
+    inode_t *linked_inode = NULL;
+    shard_internal_dir_type_t type = (shard_internal_dir_type_t)cookie;
 
-        priv = THIS->private;
+    local = frame->local;
 
-        linked_inode = inode_link (inode, local->dot_shard_loc.parent,
-                                   local->dot_shard_loc.name, buf);
-        inode_lookup (linked_inode);
-        priv->dot_shard_inode = linked_inode;
+    if (op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto out;
+    }
+
+    /* To-Do: Fix refcount increment per call to
+     * shard_link_internal_dir_inode().
+     */
+    linked_inode = shard_link_internal_dir_inode(local, inode, buf, type);
+    shard_inode_ctx_mark_dir_refreshed(linked_inode, this);
+out:
+    shard_common_resolve_shards(frame, this, local->post_res_handler);
+    return 0;
 }
 
 int
-shard_lookup_dot_shard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xdata,
-                            struct iatt *postparent)
-{
-        shard_local_t *local = NULL;
+shard_refresh_internal_dir(call_frame_t *frame, xlator_t *this,
+                           shard_internal_dir_type_t type)
+{
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            gf_uuid_copy(gfid, priv->dot_shard_gfid);
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            gf_uuid_copy(gfid, priv->dot_shard_rm_gfid);
+            break;
+        default:
+            break;
+    }
 
-        local = frame->local;
+    inode = inode_find(this->itable, gfid);
 
-        if (op_ret) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
+    if (!shard_inode_ctx_needs_lookup(inode, this)) {
+        local->op_ret = 0;
+        goto out;
+    }
 
-        if (!IA_ISDIR (buf->ia_type)) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        SHARD_MSG_DOT_SHARD_NODIR, "/.shard already exists and "
-                        "is not a directory. Please remove /.shard from all "
-                        "bricks and try again");
-                local->op_ret = -1;
-                local->op_errno = EIO;
-                goto unwind;
-        }
+    /* Plain assignment because the ref is already taken above through
+     * call to inode_find()
+     */
+    loc.inode = inode;
+    gf_uuid_copy(loc.gfid, gfid);
 
-        shard_link_dot_shard_inode (local, inode, buf);
-        shard_common_resolve_shards (frame, this,
-                                     (local->fop == GF_FOP_RENAME) ?
-                                     local->loc2.inode : local->loc.inode,
-                                     local->post_res_handler);
-        return 0;
+    STACK_WIND_COOKIE(frame, shard_refresh_internal_dir_cbk, (void *)(long)type,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup, &loc,
+                      NULL);
+    loc_wipe(&loc);
 
-unwind:
-        local->post_res_handler (frame, this);
-        return 0;
+    return 0;
+
+out:
+    shard_common_resolve_shards(frame, this, local->post_res_handler);
+    return 0;
 }
 
 int
-shard_lookup_dot_shard (call_frame_t *frame, xlator_t *this,
-                        shard_post_resolve_fop_handler_t post_res_handler)
+shard_lookup_internal_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *buf, dict_t *xdata,
+                              struct iatt *postparent)
 {
-        int                 ret       = -1;
-        dict_t             *xattr_req = NULL;
-        shard_priv_t       *priv      = NULL;
-        shard_local_t      *local     = NULL;
+    inode_t *link_inode = NULL;
+    shard_local_t *local = NULL;
+    shard_internal_dir_type_t type = (shard_internal_dir_type_t)cookie;
 
-        local = frame->local;
-        priv = this->private;
-        local->post_res_handler = post_res_handler;
+    local = frame->local;
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
-        }
+    if (op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+
+    if (!IA_ISDIR(buf->ia_type)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, SHARD_MSG_DOT_SHARD_NODIR,
+               "%s already exists and "
+               "is not a directory. Please remove it from all bricks "
+               "and try again",
+               shard_internal_dir_string(type));
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        goto unwind;
+    }
+
+    link_inode = shard_link_internal_dir_inode(local, inode, buf, type);
+    if (link_inode != inode) {
+        shard_refresh_internal_dir(frame, this, type);
+    } else {
+        shard_inode_ctx_mark_dir_refreshed(link_inode, this);
+        shard_common_resolve_shards(frame, this, local->post_res_handler);
+    }
+    return 0;
 
-        ret = dict_set_static_bin (xattr_req, "gfid-req", priv->dot_shard_gfid,
-                                   16);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_SET_FAILED,
-                        "Failed to set gfid of /.shard into dict");
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
-        }
+unwind:
+    local->post_res_handler(frame, this);
+    return 0;
+}
+
+int
+shard_lookup_internal_dir(call_frame_t *frame, xlator_t *this,
+                          shard_post_resolve_fop_handler_t post_res_handler,
+                          shard_internal_dir_type_t type)
+{
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    uuid_t *gfid = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t free_gfid = _gf_true;
+
+    local = frame->local;
+    priv = this->private;
+    local->post_res_handler = post_res_handler;
+
+    gfid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!gfid)
+        goto err;
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            gf_uuid_copy(*gfid, priv->dot_shard_gfid);
+            loc = &local->dot_shard_loc;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            gf_uuid_copy(*gfid, priv->dot_shard_rm_gfid);
+            loc = &local->dot_shard_rm_loc;
+            break;
+        default:
+            bzero(*gfid, sizeof(uuid_t));
+            break;
+    }
+
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", *gfid, false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set gfid of %s into dict",
+               shard_internal_dir_string(type));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    } else {
+        free_gfid = _gf_false;
+    }
 
-        STACK_WIND (frame, shard_lookup_dot_shard_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &local->dot_shard_loc,
-                    xattr_req);
+    STACK_WIND_COOKIE(frame, shard_lookup_internal_dir_cbk, (void *)(long)type,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup, loc,
+                      xattr_req);
 
-        dict_unref (xattr_req);
-        return 0;
+    dict_unref(xattr_req);
+    return 0;
 
 err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        post_res_handler (frame, this);
-        return 0;
+    if (xattr_req)
+        dict_unref(xattr_req);
+    if (free_gfid)
+        GF_FREE(gfid);
+    post_res_handler(frame, this);
+    return 0;
 }
 
 static void
-shard_inode_ctx_update (inode_t *inode, xlator_t *this, dict_t *xdata,
-                        struct iatt *buf)
-{
-        int                ret           = 0;
-        uint64_t           size          = 0;
-        void              *bsize         = NULL;
-
-        if (shard_inode_ctx_get_block_size (inode, this, &size)) {
-                /* Fresh lookup */
-                ret = dict_get_ptr (xdata, GF_XATTR_SHARD_BLOCK_SIZE, &bsize);
-                if (!ret)
-                        size = ntoh64 (*((uint64_t *)bsize));
-                /* If the file is sharded, set its block size, otherwise just
-                 * set 0.
-                 */
-
-                shard_inode_ctx_set (inode, this, buf, size,
-                                     SHARD_MASK_BLOCK_SIZE);
-        }
-        /* If the file is sharded, also set the remaining attributes,
-         * except for ia_size and ia_blocks.
+shard_inode_ctx_update(inode_t *inode, xlator_t *this, dict_t *xdata,
+                       struct iatt *buf)
+{
+    int ret = 0;
+    uint64_t size = 0;
+    void *bsize = NULL;
+
+    if (shard_inode_ctx_get_block_size(inode, this, &size)) {
+        /* Fresh lookup */
+        ret = dict_get_ptr(xdata, GF_XATTR_SHARD_BLOCK_SIZE, &bsize);
+        if (!ret)
+            size = ntoh64(*((uint64_t *)bsize));
+        /* If the file is sharded, set its block size, otherwise just
+         * set 0.
          */
-        if (size) {
-                shard_inode_ctx_set (inode, this, buf, 0, SHARD_LOOKUP_MASK);
-                (void) shard_inode_ctx_invalidate (inode, this, buf);
-        }
+
+        shard_inode_ctx_set(inode, this, buf, size, SHARD_MASK_BLOCK_SIZE);
+    }
+    /* If the file is sharded, also set the remaining attributes,
+     * except for ia_size and ia_blocks.
+     */
+    if (size) {
+        shard_inode_ctx_set(inode, this, buf, 0, SHARD_LOOKUP_MASK);
+        (void)shard_inode_ctx_invalidate(inode, this, buf);
+    }
 }
 
 int
-shard_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        if (op_ret < 0)
-                goto unwind;
+shard_delete_shards(void *opaque);
 
-        if (IA_ISDIR (buf->ia_type))
-                goto unwind;
+int
+shard_delete_shards_cbk(int ret, call_frame_t *frame, void *data);
 
-        /* Also, if the file is sharded, get the file size and block cnt xattr,
-         * and store them in the stbuf appropriately.
-         */
+int
+shard_start_background_deletion(xlator_t *this)
+{
+    int ret = 0;
+    gf_boolean_t i_cleanup = _gf_true;
+    shard_priv_t *priv = NULL;
+    call_frame_t *cleanup_frame = NULL;
 
-        if (dict_get (xdata, GF_XATTR_SHARD_FILE_SIZE) &&
-            frame->root->pid != GF_CLIENT_PID_GSYNCD)
-                shard_modify_size_and_block_count (buf, xdata);
-
-        /* If this was a fresh lookup, there are two possibilities:
-         * 1) If the file is sharded (indicated by the presence of block size
-         *    xattr), store this block size, along with rdev and mode in its
-         *    inode ctx.
-         * 2) If the file is not sharded, store size along with rdev and mode
-         *    (which are anyway don't cares) in inode ctx. Since @ctx_tmp is
-         *    already initialised to all zeroes, nothing more needs to be done.
-         */
+    priv = this->private;
 
-        (void) shard_inode_ctx_update (inode, this, xdata, buf);
+    LOCK(&priv->lock);
+    {
+        switch (priv->bg_del_state) {
+            case SHARD_BG_DELETION_NONE:
+                i_cleanup = _gf_true;
+                priv->bg_del_state = SHARD_BG_DELETION_LAUNCHING;
+                break;
+            case SHARD_BG_DELETION_LAUNCHING:
+                i_cleanup = _gf_false;
+                break;
+            case SHARD_BG_DELETION_IN_PROGRESS:
+                priv->bg_del_state = SHARD_BG_DELETION_LAUNCHING;
+                i_cleanup = _gf_false;
+                break;
+            default:
+                break;
+        }
+    }
+    UNLOCK(&priv->lock);
+    if (!i_cleanup)
+        return 0;
+
+    cleanup_frame = create_frame(this, this->ctx->pool);
+    if (!cleanup_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create "
+               "new frame to delete shards");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    set_lk_owner_from_ptr(&cleanup_frame->root->lk_owner, cleanup_frame->root);
+
+    ret = synctask_new(this->ctx->env, shard_delete_shards,
+                       shard_delete_shards_cbk, cleanup_frame, cleanup_frame);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno,
+               SHARD_MSG_SHARDS_DELETION_FAILED,
+               "failed to create task to do background "
+               "cleanup of shards");
+        STACK_DESTROY(cleanup_frame->root);
+        goto err;
+    }
+    return 0;
 
-unwind:
-        SHARD_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                            xdata, postparent);
-        return 0;
+err:
+    LOCK(&priv->lock);
+    {
+        priv->bg_del_state = SHARD_BG_DELETION_NONE;
+    }
+    UNLOCK(&priv->lock);
+    return ret;
 }
 
 int
-shard_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xattr_req)
+shard_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        int             ret        = -1;
-        int32_t         op_errno   = ENOMEM;
-        uint64_t        block_size = 0;
-        shard_local_t  *local      = NULL;
+    int ret = -1;
+    shard_priv_t *priv = NULL;
+    gf_boolean_t i_start_cleanup = _gf_false;
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                SHARD_ENTRY_FOP_CHECK (loc, op_errno, err);
-        }
+    priv = this->private;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    if (op_ret < 0)
+        goto unwind;
 
-        frame->local = local;
+    if (IA_ISDIR(buf->ia_type))
+        goto unwind;
 
-        loc_copy (&local->loc, loc);
+    /* Also, if the file is sharded, get the file size and block cnt xattr,
+     * and store them in the stbuf appropriately.
+     */
 
-        local->xattr_req = xattr_req ? dict_ref (xattr_req) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
+    if (dict_get(xdata, GF_XATTR_SHARD_FILE_SIZE) &&
+        frame->root->pid != GF_CLIENT_PID_GSYNCD)
+        shard_modify_size_and_block_count(buf, xdata);
 
-        if (shard_inode_ctx_get_block_size (loc->inode, this, &block_size)) {
-                ret = dict_set_uint64 (local->xattr_req,
-                                       GF_XATTR_SHARD_BLOCK_SIZE, 0);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                SHARD_MSG_DICT_SET_FAILED, "Failed to set dict"
-                                " value: key:%s for path %s",
-                                GF_XATTR_SHARD_BLOCK_SIZE, loc->path);
-                        goto err;
-                }
-        }
+    /* If this was a fresh lookup, there are two possibilities:
+     * 1) If the file is sharded (indicated by the presence of block size
+     *    xattr), store this block size, along with rdev and mode in its
+     *    inode ctx.
+     * 2) If the file is not sharded, store size along with rdev and mode
+     *    (which are anyway don't cares) in inode ctx. Since @ctx_tmp is
+     *    already initialised to all zeroes, nothing more needs to be done.
+     */
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                ret = dict_set_uint64 (local->xattr_req,
-                                       GF_XATTR_SHARD_FILE_SIZE, 8 * 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                SHARD_MSG_DICT_SET_FAILED,
-                                "Failed to set dict value: key:%s for path %s.",
-                                GF_XATTR_SHARD_FILE_SIZE, loc->path);
-                        goto err;
-                }
-        }
-
-        if ((xattr_req) && (dict_get (xattr_req, GF_CONTENT_KEY)))
-                dict_del (xattr_req, GF_CONTENT_KEY);
+    (void)shard_inode_ctx_update(inode, this, xdata, buf);
 
-        STACK_WIND (frame, shard_lookup_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup, loc, local->xattr_req);
-
-        return 0;
+    LOCK(&priv->lock);
+    {
+        if (priv->first_lookup_done == _gf_false) {
+            priv->first_lookup_done = _gf_true;
+            i_start_cleanup = _gf_true;
+        }
+    }
+    UNLOCK(&priv->lock);
 
+    if (!i_start_cleanup)
+        goto unwind;
 
-err:
-        SHARD_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL,
-                             NULL, NULL);
-        return 0;
+    ret = shard_start_background_deletion(this);
+    if (ret < 0) {
+        LOCK(&priv->lock);
+        {
+            priv->first_lookup_done = _gf_false;
+        }
+        UNLOCK(&priv->lock);
+    }
 
+unwind:
+    SHARD_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                       postparent);
+    return 0;
 }
 
 int
-shard_lookup_base_file_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xdata,
-                            struct iatt *postparent)
+shard_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        int                ret    = -1;
-        int32_t            mask   = SHARD_INODE_WRITE_MASK;
-        shard_local_t     *local  = NULL;
-        shard_inode_ctx_t  ctx    = {0,};
+    int ret = -1;
+    int32_t op_errno = ENOMEM;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    this->itable = loc->inode->table;
+    if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) &&
+        (frame->root->pid != GF_CLIENT_PID_GLFS_HEAL)) {
+        SHARD_ENTRY_FOP_CHECK(loc, op_errno, err);
+    }
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        SHARD_MSG_BASE_FILE_LOOKUP_FAILED, "Lookup on base file"
-                        " failed : %s", loc_gfid_utoa (&(local->loc)));
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
 
-        local->prebuf = *buf;
-        if (shard_modify_size_and_block_count (&local->prebuf, xdata)) {
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
-        }
+    frame->local = local;
+
+    loc_copy(&local->loc, loc);
 
-        if (shard_inode_ctx_get_all (inode, this, &ctx))
-                mask = SHARD_ALL_MASK;
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+    if (!local->xattr_req)
+        goto err;
 
-        ret = shard_inode_ctx_set (inode, this, &local->prebuf, 0,
-                                   (mask | SHARD_MASK_REFRESH_RESET));
+    if (shard_inode_ctx_get_block_size(loc->inode, this, &block_size)) {
+        ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, 0);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        SHARD_MSG_INODE_CTX_SET_FAILED, 0, "Failed to set inode"
-                        " write params into inode ctx for %s",
-                        uuid_utoa (buf->ia_gfid));
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto unwind;
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+                   "Failed to set dict"
+                   " value: key:%s for path %s",
+                   GF_XATTR_SHARD_BLOCK_SIZE, loc->path);
+            goto err;
         }
+    }
+
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_FILE_SIZE,
+                              8 * 4);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+                   "Failed to set dict value: key:%s for path %s.",
+                   GF_XATTR_SHARD_FILE_SIZE, loc->path);
+            goto err;
+        }
+    }
+
+    if ((xattr_req) && (dict_get(xattr_req, GF_CONTENT_KEY)))
+        dict_del(xattr_req, GF_CONTENT_KEY);
+
+    STACK_WIND(frame, shard_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, local->xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_LOOKUP, frame, -1, op_errno);
+    return 0;
+}
+
+int
+shard_set_iattr_invoke_post_handler(call_frame_t *frame, xlator_t *this,
+                                    inode_t *inode, int32_t op_ret,
+                                    int32_t op_errno, struct iatt *buf,
+                                    dict_t *xdata)
+{
+    int ret = -1;
+    int32_t mask = SHARD_INODE_WRITE_MASK;
+    shard_local_t *local = frame->local;
+    shard_inode_ctx_t ctx = {
+        0,
+    };
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_BASE_FILE_LOOKUP_FAILED,
+               "Lookup on base file"
+               " failed : %s",
+               uuid_utoa(inode->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+
+    local->prebuf = *buf;
+    if (shard_modify_size_and_block_count(&local->prebuf, xdata)) {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    if (shard_inode_ctx_get_all(inode, this, &ctx))
+        mask = SHARD_ALL_MASK;
+
+    ret = shard_inode_ctx_set(inode, this, &local->prebuf, 0,
+                              (mask | SHARD_MASK_REFRESH_RESET));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, SHARD_MSG_INODE_CTX_SET_FAILED, 0,
+               "Failed to set inode"
+               " write params into inode ctx for %s",
+               uuid_utoa(buf->ia_gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto unwind;
+    }
 
 unwind:
-        local->handler (frame, this);
-        return 0;
+    local->handler(frame, this);
+    return 0;
 }
 
 int
-shard_lookup_base_file (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        shard_post_fop_handler_t handler)
+shard_fstat_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
 {
-        int                 ret          = -1;
-        shard_local_t      *local        = NULL;
-        dict_t             *xattr_req    = NULL;
-        gf_boolean_t        need_refresh = _gf_false;
-
-        local = frame->local;
-        local->handler = handler;
+    shard_local_t *local = frame->local;
 
-        ret = shard_inode_ctx_fill_iatt_from_cache (loc->inode, this,
-                                                        &local->prebuf,
-                                                        &need_refresh);
-        /* By this time, inode ctx should have been created either in create,
-         * mknod, readdirp or lookup. If not it is a bug!
-         */
-        if ((ret == 0) && (need_refresh == _gf_false)) {
-                gf_msg_debug (this->name, 0, "Skipping lookup on base file: %s"
-                              "Serving prebuf off the inode ctx cache",
-                              uuid_utoa (loc->gfid));
-                goto out;
-        }
+    shard_set_iattr_invoke_post_handler(frame, this, local->fd->inode, op_ret,
+                                        op_errno, buf, xdata);
+    return 0;
+}
 
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto out;
-        }
+int
+shard_lookup_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
+{
+    /* In case of op_ret < 0, inode passed to this function will be NULL
+       ex: in case of op_errno = ENOENT. So refer prefilled inode data
+       which is part of local.
+       Note: Reassigning/overriding the inode passed to this cbk with inode
+       which is part of *struct shard_local_t* won't cause any issue as
+       both inodes have same reference/address as of the inode passed */
+    inode = ((shard_local_t *)frame->local)->loc.inode;
+
+    shard_set_iattr_invoke_post_handler(frame, this, inode, op_ret, op_errno,
+                                        buf, xdata);
+    return 0;
+}
+
+/* This function decides whether to make file based lookup or
+ * fd based lookup (fstat) depending on the 3rd and 4th arg.
+ * If fd != NULL and loc == NULL then call is for fstat
+ * If fd == NULL and loc != NULL then call is for file based
+ * lookup. Please pass args based on the requirement.
+ */
+int
+shard_refresh_base_file(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        fd_t *fd, shard_post_fop_handler_t handler)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
+    dict_t *xattr_req = NULL;
+    gf_boolean_t need_refresh = _gf_false;
+
+    local = frame->local;
+    local->handler = handler;
+    inode = fd ? fd->inode : loc->inode;
+
+    ret = shard_inode_ctx_fill_iatt_from_cache(inode, this, &local->prebuf,
+                                               &need_refresh);
+    /* By this time, inode ctx should have been created either in create,
+     * mknod, readdirp or lookup. If not it is a bug!
+     */
+    if ((ret == 0) && (need_refresh == _gf_false)) {
+        gf_msg_debug(this->name, 0,
+                     "Skipping lookup on base file: %s"
+                     "Serving prebuf off the inode ctx cache",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
 
-        SHARD_MD_READ_FOP_INIT_REQ_DICT (this, xattr_req, loc->gfid,
-                                         local, out);
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, xattr_req, inode->gfid, local, out);
 
-        STACK_WIND (frame, shard_lookup_base_file_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    if (fd)
+        STACK_WIND(frame, shard_fstat_base_file_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xattr_req);
+    else
+        STACK_WIND(frame, shard_lookup_base_file_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
 
-        dict_unref (xattr_req);
-        return 0;
+    dict_unref(xattr_req);
+    return 0;
 
 out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        handler (frame, this);
-        return 0;
-
+    if (xattr_req)
+        dict_unref(xattr_req);
+    handler(frame, this);
+    return 0;
 }
 
 int
-shard_post_fstat_handler (call_frame_t *frame, xlator_t *this)
+shard_post_fstat_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret >= 0)
-                shard_inode_ctx_set (local->fd->inode, this, &local->prebuf, 0,
-                                     SHARD_LOOKUP_MASK);
+    if (local->op_ret >= 0)
+        shard_inode_ctx_set(local->fd->inode, this, &local->prebuf, 0,
+                            SHARD_LOOKUP_MASK);
 
-        SHARD_STACK_UNWIND (fstat, frame, local->op_ret, local->op_errno,
-                            &local->prebuf, local->xattr_rsp);
-        return 0;
+    SHARD_STACK_UNWIND(fstat, frame, local->op_ret, local->op_errno,
+                       &local->prebuf, local->xattr_rsp);
+    return 0;
 }
 
 int
-shard_post_stat_handler (call_frame_t *frame, xlator_t *this)
+shard_post_stat_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret >= 0)
-                shard_inode_ctx_set (local->loc.inode, this, &local->prebuf, 0,
-                                     SHARD_LOOKUP_MASK);
+    if (local->op_ret >= 0)
+        shard_inode_ctx_set(local->loc.inode, this, &local->prebuf, 0,
+                            SHARD_LOOKUP_MASK);
 
-        SHARD_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                            &local->prebuf, local->xattr_rsp);
-        return 0;
+    SHARD_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno,
+                       &local->prebuf, local->xattr_rsp);
+    return 0;
 }
 
 int
-shard_common_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                       dict_t *xdata)
+shard_common_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
-        inode_t       *inode = NULL;
-        shard_local_t *local = NULL;
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        SHARD_MSG_STAT_FAILED, "stat failed: %s",
-                        local->fd ? uuid_utoa (local->fd->inode->gfid)
-                        : uuid_utoa ((local->loc.inode)->gfid));
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_STAT_FAILED,
+               "stat failed: %s",
+               local->fd ? uuid_utoa(local->fd->inode->gfid)
+                         : uuid_utoa((local->loc.inode)->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
 
-        local->prebuf = *buf;
-        if (shard_modify_size_and_block_count (&local->prebuf, xdata)) {
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
-        }
-        local->xattr_rsp = dict_ref (xdata);
+    local->prebuf = *buf;
+    if (shard_modify_size_and_block_count(&local->prebuf, xdata)) {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+    local->xattr_rsp = dict_ref(xdata);
 
-        if (local->loc.inode)
-                inode = local->loc.inode;
-        else
-                inode = local->fd->inode;
+    if (local->loc.inode)
+        inode = local->loc.inode;
+    else
+        inode = local->fd->inode;
 
-        shard_inode_ctx_invalidate (inode, this, buf);
+    shard_inode_ctx_invalidate(inode, this, &local->prebuf);
 
 unwind:
-        local->handler (frame, this);
-        return 0;
+    local->handler(frame, this);
+    return 0;
 }
 
 int
-shard_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+shard_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int                ret        = -1;
-        uint64_t           block_size = 0;
-        shard_local_t     *local      = NULL;
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
 
-        if ((IA_ISDIR (loc->inode->ia_type)) ||
-            (IA_ISLNK (loc->inode->ia_type))) {
-                STACK_WIND (frame, default_stat_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->stat, loc, xdata);
-                return 0;
-        }
-
-        ret = shard_inode_ctx_get_block_size (loc->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size from inode ctx of %s",
-                        uuid_utoa (loc->inode->gfid));
-                goto err;
-        }
-
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_stat_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->stat, loc, xdata);
-                return 0;
-        }
+    if ((IA_ISDIR(loc->inode->ia_type)) || (IA_ISLNK(loc->inode->ia_type))) {
+        STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
 
-        frame->local = local;
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+    }
 
-        local->handler = shard_post_stat_handler;
-        loc_copy (&local->loc, loc);
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
 
-        SHARD_MD_READ_FOP_INIT_REQ_DICT (this, local->xattr_req,
-                                         local->loc.gfid, local, err);
+    frame->local = local;
 
-        STACK_WIND (frame, shard_common_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, local->xattr_req);
+    local->handler = shard_post_stat_handler;
+    loc_copy(&local->loc, loc);
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
 
-        return 0;
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, local->loc.gfid,
+                                    local, err);
 
+    STACK_WIND(frame, shard_common_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, local->xattr_req);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_STAT, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+shard_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int                ret        = -1;
-        uint64_t           block_size = 0;
-        shard_local_t     *local      = NULL;
-
-        if ((IA_ISDIR (fd->inode->ia_type)) ||
-            (IA_ISLNK (fd->inode->ia_type))) {
-                STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD (this)->fops->fstat, fd, xdata);
-                return 0;
-        }
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
 
-        ret = shard_inode_ctx_get_block_size (fd->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size from inode ctx of %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto err;
-        }
+    if ((IA_ISDIR(fd->inode->ia_type)) || (IA_ISLNK(fd->inode->ia_type))) {
+        STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+    }
 
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, fd, xdata);
-                return 0;
-        }
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
 
-        if (!this->itable)
-                this->itable = fd->inode->table;
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    if (!this->itable)
+        this->itable = fd->inode->table;
 
-        frame->local = local;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
 
-        local->handler = shard_post_fstat_handler;
-        local->fd = fd_ref (fd);
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
+    frame->local = local;
 
-        SHARD_MD_READ_FOP_INIT_REQ_DICT (this, local->xattr_req,
-                                         fd->inode->gfid, local, err);
+    local->handler = shard_post_fstat_handler;
+    local->fd = fd_ref(fd);
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
 
-        STACK_WIND (frame, shard_common_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, local->xattr_req);
-        return 0;
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, fd->inode->gfid,
+                                    local, err);
 
+    STACK_WIND(frame, shard_common_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, local->xattr_req);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_FSTAT, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_post_update_size_truncate_handler (call_frame_t *frame, xlator_t *this)
+shard_post_update_size_truncate_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->fop == GF_FOP_TRUNCATE)
-                SHARD_STACK_UNWIND (truncate, frame, local->op_ret,
-                                    local->op_errno, &local->prebuf,
-                                    &local->postbuf, NULL);
-        else
-                SHARD_STACK_UNWIND (ftruncate, frame, local->op_ret,
-                                    local->op_errno, &local->prebuf,
-                                    &local->postbuf, NULL);
-        return 0;
+    if (local->fop == GF_FOP_TRUNCATE)
+        SHARD_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, NULL);
+    else
+        SHARD_STACK_UNWIND(ftruncate, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, NULL);
+    return 0;
 }
 
 int
-shard_truncate_last_shard_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               struct iatt *prebuf, struct iatt *postbuf,
-                               dict_t *xdata)
-{
-        inode_t       *inode = NULL;
-        shard_local_t *local = NULL;
-
-        local = frame->local;
-
-        SHARD_UNSET_ROOT_FS_ID (frame, local);
-
-        inode = (local->fop == GF_FOP_TRUNCATE) ? local->loc.inode
-                                                : local->fd->inode;
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        SHARD_MSG_TRUNCATE_LAST_SHARD_FAILED, "truncate on last"
-                        " shard failed : %s", uuid_utoa (inode->gfid));
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        local->postbuf.ia_size = local->offset;
-        local->postbuf.ia_blocks -= (prebuf->ia_blocks - postbuf->ia_blocks);
-        /* Let the delta be negative. We want xattrop to do subtraction */
-        local->delta_size = local->postbuf.ia_size - local->prebuf.ia_size;
-        local->delta_blocks = postbuf->ia_blocks - prebuf->ia_blocks;
-        local->hole_size = 0;
-
-        shard_inode_ctx_set (inode, this, postbuf, 0, SHARD_MASK_TIMES);
-
-        shard_update_file_size (frame, this, NULL, &local->loc,
-                                shard_post_update_size_truncate_handler);
-        return 0;
-
+shard_truncate_last_shard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno,
+                              struct iatt *prebuf, struct iatt *postbuf,
+                              dict_t *xdata)
+{
+    inode_t *inode = NULL;
+    int64_t delta_blocks = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+
+    inode = (local->fop == GF_FOP_TRUNCATE) ? local->loc.inode
+                                            : local->fd->inode;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_TRUNCATE_LAST_SHARD_FAILED,
+               "truncate on last"
+               " shard failed : %s",
+               uuid_utoa(inode->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    local->postbuf.ia_size = local->offset;
+    /* Let the delta be negative. We want xattrop to do subtraction */
+    local->delta_size = local->postbuf.ia_size - local->prebuf.ia_size;
+    delta_blocks = GF_ATOMIC_ADD(local->delta_blocks,
+                                 postbuf->ia_blocks - prebuf->ia_blocks);
+    GF_ASSERT(delta_blocks <= 0);
+    local->postbuf.ia_blocks += delta_blocks;
+    local->hole_size = 0;
+
+    shard_inode_ctx_set(inode, this, &local->postbuf, 0, SHARD_MASK_TIMES);
+    shard_update_file_size(frame, this, NULL, &local->loc,
+                           shard_post_update_size_truncate_handler);
+    return 0;
 err:
-        if (local->fop == GF_FOP_TRUNCATE)
-                SHARD_STACK_UNWIND (truncate, frame, local->op_ret,
-                                    local->op_errno, NULL, NULL, NULL);
-        else
-                SHARD_STACK_UNWIND (ftruncate, frame, local->op_ret,
-                                    local->op_errno, NULL, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
 }
 
 int
-shard_truncate_last_shard (call_frame_t *frame, xlator_t *this, inode_t *inode)
+shard_truncate_last_shard(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        size_t          last_shard_size_after = 0;
-        loc_t           loc                   = {0,};
-        shard_local_t  *local                 = NULL;
+    size_t last_shard_size_after = 0;
+    loc_t loc = {
+        0,
+    };
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* A NULL inode could be due to the fact that the last shard which
-         * needs to be truncated does not exist due to it lying in a hole
-         * region. So the only thing left to do in that case would be an
-         * update to file size xattr.
-         */
-        if (!inode) {
-                gf_msg_debug (this->name, 0, "Last shard to be truncated absent"
-                              " in backend: %s. Directly proceeding to update "
-                              "file size", uuid_utoa (inode->gfid));
-                shard_update_file_size (frame, this, NULL, &local->loc,
-                                       shard_post_update_size_truncate_handler);
-                return 0;
-        }
+    /* A NULL inode could be due to the fact that the last shard which
+     * needs to be truncated does not exist due to it lying in a hole
+     * region. So the only thing left to do in that case would be an
+     * update to file size xattr.
+     */
+    if (!inode) {
+        gf_msg_debug(this->name, 0,
+                     "Last shard to be truncated absent in backend: %" PRIu64
+                     " of gfid %s. Directly proceeding to update file size",
+                     local->first_block, uuid_utoa(local->loc.inode->gfid));
+        shard_update_file_size(frame, this, NULL, &local->loc,
+                               shard_post_update_size_truncate_handler);
+        return 0;
+    }
 
-        SHARD_SET_ROOT_FS_ID (frame, local);
+    SHARD_SET_ROOT_FS_ID(frame, local);
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        last_shard_size_after = (local->offset % local->block_size);
+    last_shard_size_after = (local->offset % local->block_size);
 
-        STACK_WIND (frame, shard_truncate_last_shard_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, &loc,
-                    last_shard_size_after, NULL);
-        loc_wipe (&loc);
-        return 0;
+    STACK_WIND(frame, shard_truncate_last_shard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, &loc, last_shard_size_after,
+               NULL);
+    loc_wipe(&loc);
+    return 0;
 }
 
-int
-shard_unlink_shards_do_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t *xdata);
+void
+shard_unlink_block_inode(shard_local_t *local, int shard_block_num);
 
 int
-shard_truncate_htol (call_frame_t *frame, xlator_t *this, inode_t *inode)
+shard_truncate_htol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
 {
-        int i = 1;
-        int ret = -1;
-        int call_count = 0;
-        uint32_t cur_block = 0;
-        uint32_t last_block = 0;
-        char path[PATH_MAX] = {0,};
-        char *bname = NULL;
-        loc_t loc = {0,};
-        gf_boolean_t wind_failed = _gf_false;
-        shard_local_t *local = NULL;
-        shard_priv_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
+    int ret = 0;
+    int call_count = 0;
+    int shard_block_num = (long)cookie;
+    uint64_t block_count = 0;
+    shard_local_t *local = NULL;
 
-        cur_block = local->first_block + 1;
-        last_block = local->last_block;
+    local = frame->local;
 
-        /* Determine call count */
-        for (i = 1; i < local->num_blocks; i++) {
-                if (!local->inode_list[i])
-                        continue;
-                call_count++;
-        }
-
-        if (!call_count) {
-                /* Call count = 0 implies that all of the shards that need to be
-                 * unlinked do not exist. So shard xlator would now proceed to
-                 * do the final truncate + size updates.
-                 */
-                gf_msg_debug (this->name, 0, "Shards to be unlinked as part of "
-                              "truncate absent in backend: %s. Directly "
-                              "proceeding to update file size",
-                              uuid_utoa (inode->gfid));
-                local->postbuf.ia_size = local->offset;
-                local->postbuf.ia_blocks = local->prebuf.ia_blocks;
-                local->delta_size = local->postbuf.ia_size -
-                                    local->prebuf.ia_size;
-                local->delta_blocks = 0;
-                local->hole_size = 0;
-                shard_update_file_size (frame, this, local->fd, &local->loc,
-                                       shard_post_update_size_truncate_handler);
-                return 0;
-        }
-
-        local->call_count = call_count;
-        i = 1;
-
-        SHARD_SET_ROOT_FS_ID (frame, local);
-        while (cur_block <= last_block) {
-                if (!local->inode_list[i]) {
-                        cur_block++;
-                        i++;
-                        continue;
-                }
-                if (wind_failed) {
-                        shard_unlink_shards_do_cbk (frame,
-                                                    (void *)(long) cur_block,
-                                                    this, -1, ENOMEM, NULL,
-                                                    NULL, NULL);
-                        goto next;
-                }
-
-                shard_make_block_abspath (cur_block, inode->gfid, path,
-                                          sizeof (path));
-                bname = strrchr (path, '/') + 1;
-                loc.parent = inode_ref (priv->dot_shard_inode);
-                ret = inode_path (loc.parent, bname, (char **)&(loc.path));
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                SHARD_MSG_INODE_PATH_FAILED, "Inode path failed"
-                                " on %s. Base file gfid = %s", bname,
-                                uuid_utoa (inode->gfid));
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        loc_wipe (&loc);
-                        wind_failed = _gf_true;
-                        shard_unlink_shards_do_cbk (frame,
-                                                    (void *)(long) cur_block,
-                                                    this, -1, ENOMEM, NULL,
-                                                    NULL, NULL);
-                        goto next;
-                }
-                loc.name = strrchr (loc.path, '/');
-                if (loc.name)
-                        loc.name++;
-                loc.inode = inode_ref (local->inode_list[i]);
-
-                STACK_WIND_COOKIE (frame, shard_unlink_shards_do_cbk,
-                                   (void *) (long) cur_block, FIRST_CHILD(this),
-                                   FIRST_CHILD (this)->fops->unlink, &loc,
-                                   0, NULL);
-                loc_wipe (&loc);
-next:
-                i++;
-                cur_block++;
-                if (!--call_count)
-                        break;
-        }
-        return 0;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto done;
+    }
+    ret = dict_get_uint64(xdata, GF_GET_FILE_BLOCK_COUNT, &block_count);
+    if (!ret) {
+        GF_ATOMIC_SUB(local->delta_blocks, block_count);
+    } else {
+        /* dict_get failed possibly due to a heterogeneous cluster? */
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to get key %s from dict during truncate of gfid %s",
+               GF_GET_FILE_BLOCK_COUNT,
+               uuid_utoa(local->resolver_base_inode->gfid));
+    }
+
+    shard_unlink_block_inode(local, shard_block_num);
+done:
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        shard_truncate_last_shard(frame, this, local->inode_list[0]);
+    }
+    return 0;
+}
 
+int
+shard_truncate_htol(call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+    int i = 1;
+    int ret = -1;
+    int call_count = 0;
+    uint32_t cur_block = 0;
+    uint32_t last_block = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *bname = NULL;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t wind_failed = _gf_false;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    cur_block = local->first_block + 1;
+    last_block = local->last_block;
+
+    /* Determine call count */
+    for (i = 1; i < local->num_blocks; i++) {
+        if (!local->inode_list[i])
+            continue;
+        call_count++;
+    }
+
+    if (!call_count) {
+        /* Call count = 0 implies that all of the shards that need to be
+         * unlinked do not exist. So shard xlator would now proceed to
+         * do the final truncate + size updates.
+         */
+        gf_msg_debug(this->name, 0,
+                     "Shards to be unlinked as part of "
+                     "truncate absent in backend: %s. Directly "
+                     "proceeding to update file size",
+                     uuid_utoa(inode->gfid));
+        local->postbuf.ia_size = local->offset;
+        local->postbuf.ia_blocks = local->prebuf.ia_blocks;
+        local->delta_size = local->postbuf.ia_size - local->prebuf.ia_size;
+        GF_ATOMIC_INIT(local->delta_blocks, 0);
+        local->hole_size = 0;
+        shard_update_file_size(frame, this, local->fd, &local->loc,
+                               shard_post_update_size_truncate_handler);
+        return 0;
+    }
+
+    local->call_count = call_count;
+    i = 1;
+    xdata_req = dict_new();
+    if (!xdata_req) {
+        shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+        return 0;
+    }
+    ret = dict_set_uint64(xdata_req, GF_GET_FILE_BLOCK_COUNT, 8 * 8);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set key %s into dict during truncate of %s",
+               GF_GET_FILE_BLOCK_COUNT,
+               uuid_utoa(local->resolver_base_inode->gfid));
+        dict_unref(xdata_req);
+        shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+        return 0;
+    }
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+    while (cur_block <= last_block) {
+        if (!local->inode_list[i]) {
+            cur_block++;
+            i++;
+            continue;
+        }
+        if (wind_failed) {
+            shard_truncate_htol_cbk(frame, (void *)(long)cur_block, this, -1,
+                                    ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
+
+        shard_make_block_abspath(cur_block, inode->gfid, path, sizeof(path));
+        bname = strrchr(path, '/') + 1;
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on %s. Base file gfid = %s",
+                   bname, uuid_utoa(inode->gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            loc_wipe(&loc);
+            wind_failed = _gf_true;
+            shard_truncate_htol_cbk(frame, (void *)(long)cur_block, this, -1,
+                                    ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+        loc.inode = inode_ref(local->inode_list[i]);
+
+        STACK_WIND_COOKIE(frame, shard_truncate_htol_cbk,
+                          (void *)(long)cur_block, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->unlink, &loc, 0, xdata_req);
+        loc_wipe(&loc);
+    next:
+        i++;
+        cur_block++;
+        if (!--call_count)
+            break;
+    }
+    dict_unref(xdata_req);
+    return 0;
 }
 
 int
-shard_truncate_do (call_frame_t *frame, xlator_t *this)
+shard_truncate_do(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->num_blocks == 1) {
-                /* This means that there are no shards to be unlinked.
-                 * The fop boils down to truncating the last shard, updating
-                 * the size and unwinding.
-                 */
-                shard_truncate_last_shard (frame, this,
-                                                   local->inode_list[0]);
-                return 0;
-        } else {
-                shard_truncate_htol (frame, this, local->loc.inode);
-        }
+    if (local->num_blocks == 1) {
+        /* This means that there are no shards to be unlinked.
+         * The fop boils down to truncating the last shard, updating
+         * the size and unwinding.
+         */
+        shard_truncate_last_shard(frame, this, local->inode_list[0]);
         return 0;
+    } else {
+        shard_truncate_htol(frame, this, local->loc.inode);
+    }
+    return 0;
 }
 
 int
-shard_post_lookup_shards_truncate_handler (call_frame_t *frame, xlator_t *this)
+shard_post_lookup_shards_truncate_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                if (local->fop == GF_FOP_TRUNCATE)
-                        SHARD_STACK_UNWIND (truncate, frame, local->op_ret,
-                                            local->op_errno, NULL, NULL, NULL);
-                else
-                        SHARD_STACK_UNWIND (ftruncate, frame, local->op_ret,
-                                            local->op_errno, NULL, NULL, NULL);
-                return 0;
-        }
-
-        shard_truncate_do (frame, this);
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
+
+    shard_truncate_do(frame, this);
+    return 0;
 }
 
 void
-shard_link_block_inode (shard_local_t *local, int block_num, inode_t *inode,
-                        struct iatt *buf)
-{
-        int             list_index       = 0;
-        char            block_bname[256] = {0,};
-        inode_t        *linked_inode     = NULL;
-        xlator_t       *this             = NULL;
-        shard_priv_t   *priv             = NULL;
-
-        this = THIS;
-        priv = this->private;
-
-        shard_make_block_bname (block_num, (local->loc.inode)->gfid,
-                                block_bname, sizeof (block_bname));
-
-        shard_inode_ctx_set (inode, this, buf, 0, SHARD_LOOKUP_MASK);
-        linked_inode = inode_link (inode, priv->dot_shard_inode, block_bname,
-                                   buf);
-        inode_lookup (linked_inode);
-        list_index = block_num - local->first_block;
-
-        /* Defer unref'ing the inodes until write is complete. These inodes are
-         * unref'd in the event of a failure or after successful fop completion
-         * in shard_local_wipe().
-         */
-        local->inode_list[list_index] = linked_inode;
-
-        LOCK(&priv->lock);
-        {
-                __shard_update_shards_inode_list (linked_inode, this,
-                                                  local->loc.inode, block_num);
-        }
-        UNLOCK(&priv->lock);
+shard_link_block_inode(shard_local_t *local, int block_num, inode_t *inode,
+                       struct iatt *buf)
+{
+    int list_index = 0;
+    char block_bname[256] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    xlator_t *this = NULL;
+    inode_t *fsync_inode = NULL;
+    shard_priv_t *priv = NULL;
+    inode_t *base_inode = NULL;
+
+    this = THIS;
+    priv = this->private;
+    if (local->loc.inode) {
+        gf_uuid_copy(gfid, local->loc.inode->gfid);
+        base_inode = local->loc.inode;
+    } else if (local->resolver_base_inode) {
+        gf_uuid_copy(gfid, local->resolver_base_inode->gfid);
+        base_inode = local->resolver_base_inode;
+    } else {
+        gf_uuid_copy(gfid, local->base_gfid);
+    }
+
+    shard_make_block_bname(block_num, gfid, block_bname, sizeof(block_bname));
+
+    shard_inode_ctx_set(inode, this, buf, 0, SHARD_LOOKUP_MASK);
+    linked_inode = inode_link(inode, priv->dot_shard_inode, block_bname, buf);
+    inode_lookup(linked_inode);
+    list_index = block_num - local->first_block;
+    local->inode_list[list_index] = linked_inode;
+
+    LOCK(&priv->lock);
+    {
+        fsync_inode = __shard_update_shards_inode_list(
+            linked_inode, this, base_inode, block_num, gfid);
+    }
+    UNLOCK(&priv->lock);
+    if (fsync_inode)
+        shard_initiate_evicted_inode_fsync(this, fsync_inode);
 }
 
 int
-shard_common_lookup_shards_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, inode_t *inode,
-                                struct iatt *buf, dict_t *xdata,
-                                struct iatt *postparent)
-{
-        int             call_count      = 0;
-        int             shard_block_num = (long) cookie;
-        shard_local_t  *local           = NULL;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                /* Ignore absence of shards in the backend in truncate fop. */
-                if (((local->fop == GF_FOP_TRUNCATE) ||
-                    (local->fop == GF_FOP_FTRUNCATE) ||
-                    (local->fop == GF_FOP_RENAME) ||
-                    (local->fop == GF_FOP_UNLINK)) && (op_errno == ENOENT))
-                        goto done;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        SHARD_MSG_LOOKUP_SHARD_FAILED, "Lookup on shard %d "
-                        "failed. Base file gfid = %s", shard_block_num,
-                        (local->fop == GF_FOP_RENAME) ?
-                        uuid_utoa (local->loc2.inode->gfid)
-                        : uuid_utoa (local->loc.inode->gfid));
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto done;
+shard_common_lookup_shards_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
+{
+    int call_count = 0;
+    int shard_block_num = (long)cookie;
+    uuid_t gfid = {
+        0,
+    };
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    if (local->resolver_base_inode)
+        gf_uuid_copy(gfid, local->resolver_base_inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    if (op_ret < 0) {
+        /* Ignore absence of shards in the backend in truncate fop. */
+        switch (local->fop) {
+            case GF_FOP_TRUNCATE:
+            case GF_FOP_FTRUNCATE:
+            case GF_FOP_RENAME:
+            case GF_FOP_UNLINK:
+                if (op_errno == ENOENT)
+                    goto done;
+                break;
+            case GF_FOP_WRITE:
+            case GF_FOP_READ:
+            case GF_FOP_ZEROFILL:
+            case GF_FOP_DISCARD:
+            case GF_FOP_FALLOCATE:
+                if ((!local->first_lookup_done) && (op_errno == ENOENT)) {
+                    LOCK(&frame->lock);
+                    {
+                        local->create_count++;
+                    }
+                    UNLOCK(&frame->lock);
+                    goto done;
+                }
+                break;
+            default:
+                break;
         }
 
-        shard_link_block_inode (local, shard_block_num, inode, buf);
+        /* else */
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_LOOKUP_SHARD_FAILED,
+               "Lookup on shard %d "
+               "failed. Base file gfid = %s",
+               shard_block_num, uuid_utoa(gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto done;
+    }
+
+    shard_link_block_inode(local, shard_block_num, inode, buf);
 
 done:
-        call_count = shard_call_count_return (frame);
+    if (local->lookup_shards_barriered) {
+        syncbarrier_wake(&local->barrier);
+        return 0;
+    } else {
+        call_count = shard_call_count_return(frame);
         if (call_count == 0) {
-                if (local->op_ret < 0)
-                        goto unwind;
-                else
-                        local->pls_fop_handler (frame, this);
+            if (!local->first_lookup_done)
+                local->first_lookup_done = _gf_true;
+            local->pls_fop_handler(frame, this);
         }
-        return 0;
-
-unwind:
-        local->pls_fop_handler (frame, this);
-        return 0;
+    }
+    return 0;
 }
 
-dict_t*
-shard_create_gfid_dict (dict_t *dict)
+dict_t *
+shard_create_gfid_dict(dict_t *dict)
 {
-        int    ret   = 0;
-        dict_t *new  = NULL;
-        uuid_t *gfid = NULL;
+    int ret = 0;
+    dict_t *new = NULL;
+    unsigned char *gfid = NULL;
 
-        new = dict_copy_with_ref (dict, NULL);
-        if (!new)
-                return NULL;
+    new = dict_copy_with_ref(dict, NULL);
+    if (!new)
+        return NULL;
 
-        gfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!gfid) {
-                ret = -1;
-                goto out;
-        }
+    gfid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_char);
+    if (!gfid) {
+        ret = -1;
+        goto out;
+    }
 
-        gf_uuid_generate (*gfid);
+    gf_uuid_generate(gfid);
 
-        ret = dict_set_dynptr (new, "gfid-req", gfid, sizeof (uuid_t));
+    ret = dict_set_gfuuid(new, "gfid-req", gfid, false);
 
 out:
-        if (ret) {
-                dict_unref (new);
-                new = NULL;
-                GF_FREE (gfid);
-        }
-
-        return new;
-}
-
-int
-shard_common_lookup_shards (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                            shard_post_lookup_shards_fop_handler_t handler)
-{
-        int            i              = 0;
-        int            ret            = 0;
-        int            call_count     = 0;
-        int32_t        shard_idx_iter = 0;
-        int            last_block     = 0;
-        char           path[PATH_MAX] = {0,};
-        char          *bname          = NULL;
-        loc_t          loc            = {0,};
-        shard_local_t *local          = NULL;
-        shard_priv_t  *priv           = NULL;
-        gf_boolean_t   wind_failed    = _gf_false;
-        dict_t        *xattr_req      = NULL;
-
-        priv = this->private;
-        local = frame->local;
-        call_count = local->call_count;
-        shard_idx_iter = local->first_block;
-        last_block = local->last_block;
-        local->pls_fop_handler = handler;
-
-        while (shard_idx_iter <= last_block) {
-                if (local->inode_list[i]) {
-                        i++;
-                        shard_idx_iter++;
-                        continue;
-                }
-
-                if (wind_failed) {
-                        shard_common_lookup_shards_cbk (frame,
-                                                 (void *) (long) shard_idx_iter,
-                                                 this, -1, ENOMEM, NULL, NULL,
-                                                 NULL, NULL);
-                        goto next;
-                }
-
-                shard_make_block_abspath (shard_idx_iter, inode->gfid, path,
-                                          sizeof(path));
-
-                bname = strrchr (path, '/') + 1;
-                loc.inode = inode_new (this->itable);
-                loc.parent = inode_ref (priv->dot_shard_inode);
-                ret = inode_path (loc.parent, bname, (char **) &(loc.path));
-                if (ret < 0 || !(loc.inode)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                SHARD_MSG_INODE_PATH_FAILED, "Inode path failed"
-                                " on %s, base file gfid = %s", bname,
-                                uuid_utoa (inode->gfid));
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        loc_wipe (&loc);
-                        wind_failed = _gf_true;
-                        shard_common_lookup_shards_cbk (frame,
-                                                 (void *) (long) shard_idx_iter,
-                                                 this, -1, ENOMEM, NULL, NULL,
-                                                 NULL, NULL);
-                        goto next;
-                }
-
-                loc.name = strrchr (loc.path, '/');
-                if (loc.name)
-                        loc.name++;
-
-                xattr_req = shard_create_gfid_dict (local->xattr_req);
-                if (!xattr_req) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        loc_wipe (&loc);
-                        shard_common_lookup_shards_cbk (frame,
-                                                 (void *) (long) shard_idx_iter,
-                                                 this, -1, ENOMEM, NULL, NULL,
-                                                 NULL, NULL);
-                        goto next;
-                }
-
-                STACK_WIND_COOKIE (frame, shard_common_lookup_shards_cbk,
-                                   (void *) (long) shard_idx_iter,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->lookup, &loc,
-                                   xattr_req);
-                loc_wipe (&loc);
-                dict_unref (xattr_req);
-next:
-                shard_idx_iter++;
-                i++;
-
-                if (!--call_count)
-                        break;
-        }
+    if (ret) {
+        dict_unref(new);
+        new = NULL;
+        GF_FREE(gfid);
+    }
 
-        return 0;
+    return new;
 }
 
 int
-shard_post_resolve_truncate_handler (call_frame_t *frame, xlator_t *this)
-{
-        shard_local_t *local = NULL;
-
-        local = frame->local;
+shard_common_lookup_shards(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           shard_post_lookup_shards_fop_handler_t handler)
+{
+    int i = 0;
+    int ret = 0;
+    int count = 0;
+    int call_count = 0;
+    int32_t shard_idx_iter = 0;
+    int lookup_count = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *bname = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+    dict_t *xattr_req = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    count = call_count = local->call_count;
+    shard_idx_iter = local->first_block;
+    lookup_count = local->last_block - local->create_count;
+    local->pls_fop_handler = handler;
+    if (local->lookup_shards_barriered)
+        local->barrier.waitfor = local->call_count;
+
+    if (inode)
+        gf_uuid_copy(gfid, inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    while (shard_idx_iter <= lookup_count) {
+        if (local->inode_list[i]) {
+            i++;
+            shard_idx_iter++;
+            continue;
+        }
+
+        if (wind_failed) {
+            shard_common_lookup_shards_cbk(frame, (void *)(long)shard_idx_iter,
+                                           this, -1, ENOMEM, NULL, NULL, NULL,
+                                           NULL);
+            goto next;
+        }
+
+        shard_make_block_abspath(shard_idx_iter, gfid, path, sizeof(path));
+
+        bname = strrchr(path, '/') + 1;
+        loc.inode = inode_new(this->itable);
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        gf_uuid_copy(loc.pargfid, priv->dot_shard_gfid);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0 || !(loc.inode)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on %s, base file gfid = %s",
+                   bname, uuid_utoa(gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            loc_wipe(&loc);
+            wind_failed = _gf_true;
+            shard_common_lookup_shards_cbk(frame, (void *)(long)shard_idx_iter,
+                                           this, -1, ENOMEM, NULL, NULL, NULL,
+                                           NULL);
+            goto next;
+        }
+
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+
+        xattr_req = shard_create_gfid_dict(local->xattr_req);
+        if (!xattr_req) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            wind_failed = _gf_true;
+            loc_wipe(&loc);
+            shard_common_lookup_shards_cbk(frame, (void *)(long)shard_idx_iter,
+                                           this, -1, ENOMEM, NULL, NULL, NULL,
+                                           NULL);
+            goto next;
+        }
+
+        STACK_WIND_COOKIE(frame, shard_common_lookup_shards_cbk,
+                          (void *)(long)shard_idx_iter, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->lookup, &loc, xattr_req);
+        loc_wipe(&loc);
+        dict_unref(xattr_req);
+    next:
+        shard_idx_iter++;
+        i++;
+
+        if (!--call_count)
+            break;
+    }
+    if (local->lookup_shards_barriered) {
+        syncbarrier_wait(&local->barrier, count);
+        local->pls_fop_handler(frame, this);
+    }
+    return 0;
+}
 
-        if (local->op_ret < 0) {
-                if (local->op_errno == ENOENT) {
-                        /* If lookup on /.shard fails with ENOENT, it means that
-                         * the file was 0-byte in size but truncated sometime in
-                         * the past to a higher size which is reflected in the
-                         * size xattr, and now being truncated to a lower size.
-                         * In this case, the only thing that needs to be done is
-                         * to update the size xattr of the file and unwind.
-                         */
-                        local->first_block = local->last_block = 0;
-                        local->num_blocks = 1;
-                        local->call_count = 0;
-                        local->op_ret = 0;
-                        local->postbuf.ia_size = local->offset;
-                        shard_update_file_size (frame, this, local->fd,
-                                                &local->loc,
-                                       shard_post_update_size_truncate_handler);
-                        return 0;
-                } else {
-                        if (local->fop == GF_FOP_TRUNCATE)
-                                SHARD_STACK_UNWIND (truncate, frame,
-                                                    local->op_ret,
-                                                    local->op_errno, NULL, NULL,
-                                                    NULL);
-                        else
-                                SHARD_STACK_UNWIND (ftruncate, frame,
-                                                    local->op_ret,
-                                                    local->op_errno, NULL, NULL,
-                                                    NULL);
-                        return 0;
-                }
+int
+shard_post_resolve_truncate_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        if (local->op_errno == ENOENT) {
+            /* If lookup on /.shard fails with ENOENT, it means that
+             * the file was 0-byte in size but truncated sometime in
+             * the past to a higher size which is reflected in the
+             * size xattr, and now being truncated to a lower size.
+             * In this case, the only thing that needs to be done is
+             * to update the size xattr of the file and unwind.
+             */
+            local->first_block = local->last_block = 0;
+            local->num_blocks = 1;
+            local->call_count = 0;
+            local->op_ret = 0;
+            local->postbuf.ia_size = local->offset;
+            shard_update_file_size(frame, this, local->fd, &local->loc,
+                                   shard_post_update_size_truncate_handler);
+            return 0;
+        } else {
+            shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                        local->op_errno);
+            return 0;
         }
+    }
 
-        if (!local->call_count)
-                shard_truncate_do (frame, this);
-        else
-                shard_common_lookup_shards (frame, this, local->loc.inode,
-                                     shard_post_lookup_shards_truncate_handler);
+    if (!local->call_count)
+        shard_truncate_do(frame, this);
+    else
+        shard_common_lookup_shards(frame, this, local->loc.inode,
+                                   shard_post_lookup_shards_truncate_handler);
 
-        return 0;
+    return 0;
 }
 
 int
-shard_truncate_begin (call_frame_t *frame, xlator_t *this)
-{
-        int             ret    = 0;
-        shard_local_t  *local  = NULL;
-        shard_priv_t   *priv   = NULL;
-
-        priv = this->private;
-        local = frame->local;
+shard_truncate_begin(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    /* First participant block here is the lowest numbered block that would
+     * hold the last byte of the file post successful truncation.
+     * Last participant block is the block that contains the last byte in
+     * the current state of the file.
+     * If (first block == last_block):
+     *         then that means that the file only needs truncation of the
+     *         first (or last since both are same) block.
+     * Else
+     *         if (new_size % block_size == 0)
+     *                 then that means there is no truncate to be done with
+     *                 only shards from first_block + 1 through the last
+     *                 block needing to be unlinked.
+     *         else
+     *                 both truncate of the first block and unlink of the
+     *                 remaining shards until end of file is required.
+     */
+    local->first_block = (local->offset == 0)
+                             ? 0
+                             : get_lowest_block(local->offset - 1,
+                                                local->block_size);
+    local->last_block = get_highest_block(0, local->prebuf.ia_size,
+                                          local->block_size);
+
+    local->num_blocks = local->last_block - local->first_block + 1;
+    GF_ASSERT(local->num_blocks > 0);
+    local->resolver_base_inode = (local->fop == GF_FOP_TRUNCATE)
+                                     ? local->loc.inode
+                                     : local->fd->inode;
+
+    if ((local->first_block == 0) && (local->num_blocks == 1)) {
+        if (local->fop == GF_FOP_TRUNCATE)
+            STACK_WIND(frame, shard_truncate_last_shard_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->truncate, &local->loc,
+                       local->offset, local->xattr_req);
+        else
+            STACK_WIND(frame, shard_truncate_last_shard_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->ftruncate, local->fd,
+                       local->offset, local->xattr_req);
+        return 0;
+    }
 
-        /* First participant block here is the lowest numbered block that would
-         * hold the last byte of the file post successful truncation.
-         * Last participant block is the block that contains the last byte in
-         * the current state of the file.
-         * If (first block == last_block):
-         *         then that means that the file only needs truncation of the
-         *         first (or last since both are same) block.
-         * Else
-         *         if (new_size % block_size == 0)
-         *                 then that means there is no truncate to be done with
-         *                 only shards from first_block + 1 through the last
-         *                 block needing to be unlinked.
-         *         else
-         *                 both truncate of the first block and unlink of the
-         *                 remaining shards until end of file is required.
-         */
-        local->first_block = (local->offset == 0) ? 0
-                                          : get_lowest_block (local->offset - 1,
-                                                             local->block_size);
-        local->last_block = get_highest_block (0, local->prebuf.ia_size,
-                                               local->block_size);
-
-        local->num_blocks = local->last_block - local->first_block + 1;
-
-        if ((local->first_block == 0) && (local->num_blocks == 1)) {
-                if (local->fop == GF_FOP_TRUNCATE)
-                        STACK_WIND (frame, shard_truncate_last_shard_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->truncate,
-                                    &local->loc, local->offset,
-                                    local->xattr_req);
-                else
-                        STACK_WIND (frame, shard_truncate_last_shard_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->ftruncate,
-                                    local->fd, local->offset, local->xattr_req);
-                return 0;
-        }
+    local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list)
+        goto err;
 
-        local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
-                                       gf_shard_mt_inode_list);
-        if (!local->inode_list)
-                goto err;
-
-        local->dot_shard_loc.inode = inode_find (this->itable,
-                                                 priv->dot_shard_gfid);
-        if (!local->dot_shard_loc.inode) {
-                ret = shard_init_dot_shard_loc (this, local);
-                if (ret)
-                        goto err;
-                shard_lookup_dot_shard (frame, this,
-                                        shard_post_resolve_truncate_handler);
-        } else {
-                shard_common_resolve_shards (frame, this,
-                                             (local->fop == GF_FOP_TRUNCATE) ?
-                                              local->loc.inode :
-                                              local->fd->inode,
-                                           shard_post_resolve_truncate_handler);
-        }
-        return 0;
+    local->dot_shard_loc.inode = inode_find(this->itable, priv->dot_shard_gfid);
+    if (!local->dot_shard_loc.inode) {
+        ret = shard_init_internal_dir_loc(this, local,
+                                          SHARD_INTERNAL_DIR_DOT_SHARD);
+        if (ret)
+            goto err;
+        shard_lookup_internal_dir(frame, this,
+                                  shard_post_resolve_truncate_handler,
+                                  SHARD_INTERNAL_DIR_DOT_SHARD);
+    } else {
+        local->post_res_handler = shard_post_resolve_truncate_handler;
+        shard_refresh_internal_dir(frame, this, SHARD_INTERNAL_DIR_DOT_SHARD);
+    }
+    return 0;
 
 err:
-        if (local->fop == GF_FOP_TRUNCATE)
-                SHARD_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL,
-                                    NULL);
-        else
-                SHARD_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL,
-                                    NULL);
-
-       return 0;
+    shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_post_lookup_truncate_handler (call_frame_t *frame, xlator_t *this)
+shard_post_lookup_truncate_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t  *local            = NULL;
-
-        local = frame->local;
+    shard_local_t *local = NULL;
+    struct iatt tmp_stbuf = {
+        0,
+    };
 
-        if (local->op_ret < 0) {
-                if (local->fop == GF_FOP_TRUNCATE)
-                        SHARD_STACK_UNWIND (truncate, frame, local->op_ret,
-                                            local->op_errno, NULL, NULL, NULL);
-                else
-                        SHARD_STACK_UNWIND (ftruncate, frame, local->op_ret,
-                                            local->op_errno, NULL, NULL, NULL);
+    local = frame->local;
 
-                return 0;
-        }
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-        local->postbuf = local->prebuf;
+    local->postbuf = tmp_stbuf = local->prebuf;
 
-        if (local->prebuf.ia_size == local->offset) {
-                /* If the file size is same as requested size, unwind the call
-                 * immediately.
-                 */
-                if (local->fop == GF_FOP_TRUNCATE)
-                        SHARD_STACK_UNWIND (truncate, frame, 0, 0,
-                                            &local->prebuf, &local->postbuf,
-                                            NULL);
-                else
-                        SHARD_STACK_UNWIND (ftruncate, frame, 0, 0,
-                                            &local->prebuf, &local->postbuf,
-                                            NULL);
-        } else if (local->offset > local->prebuf.ia_size) {
-                /* If the truncate is from a lower to a higher size, set the
-                 * new size xattr and unwind.
-                 */
-                local->hole_size = local->offset - local->prebuf.ia_size;
-                local->delta_size = 0;
-                local->delta_blocks = 0;
-                local->postbuf.ia_size = local->offset;
-                shard_update_file_size (frame, this, NULL, &local->loc,
-                                       shard_post_update_size_truncate_handler);
-        } else {
-                /* ... else
-                 * i.   unlink all shards that need to be unlinked.
-                 * ii.  truncate the last of the shards.
-                 * iii. update the new size using setxattr.
-                 * and unwind the fop.
-                 */
-                local->hole_size = 0;
-                local->delta_size = (local->offset - local->prebuf.ia_size);
-                local->delta_blocks = 0;
-                shard_truncate_begin (frame, this);
-        }
-        return 0;
+    if (local->prebuf.ia_size == local->offset) {
+        /* If the file size is same as requested size, unwind the call
+         * immediately.
+         */
+        if (local->fop == GF_FOP_TRUNCATE)
+            SHARD_STACK_UNWIND(truncate, frame, 0, 0, &local->prebuf,
+                               &local->postbuf, NULL);
+        else
+            SHARD_STACK_UNWIND(ftruncate, frame, 0, 0, &local->prebuf,
+                               &local->postbuf, NULL);
+    } else if (local->offset > local->prebuf.ia_size) {
+        /* If the truncate is from a lower to a higher size, set the
+         * new size xattr and unwind.
+         */
+        local->hole_size = local->offset - local->prebuf.ia_size;
+        local->delta_size = 0;
+        GF_ATOMIC_INIT(local->delta_blocks, 0);
+        local->postbuf.ia_size = local->offset;
+        tmp_stbuf.ia_size = local->offset;
+        shard_inode_ctx_set(local->loc.inode, this, &tmp_stbuf, 0,
+                            SHARD_INODE_WRITE_MASK);
+        shard_update_file_size(frame, this, NULL, &local->loc,
+                               shard_post_update_size_truncate_handler);
+    } else {
+        /* ... else
+         * i.   unlink all shards that need to be unlinked.
+         * ii.  truncate the last of the shards.
+         * iii. update the new size using setxattr.
+         * and unwind the fop.
+         */
+        local->hole_size = 0;
+        local->delta_size = (local->offset - local->prebuf.ia_size);
+        GF_ATOMIC_INIT(local->delta_blocks, 0);
+        tmp_stbuf.ia_size = local->offset;
+        shard_inode_ctx_set(local->loc.inode, this, &tmp_stbuf, 0,
+                            SHARD_INODE_WRITE_MASK);
+        shard_truncate_begin(frame, this);
+    }
+    return 0;
 }
 
 /* TO-DO:
@@ -1923,3003 +2754,4629 @@ shard_post_lookup_truncate_handler (call_frame_t *frame, xlator_t *this)
  */
 
 int
-shard_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                dict_t *xdata)
-{
-        int                ret        = -1;
-        uint64_t           block_size = 0;
-        shard_local_t     *local      = NULL;
-
-        ret = shard_inode_ctx_get_block_size (loc->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size from inode ctx of %s",
-                        uuid_utoa (loc->inode->gfid));
-                goto err;
-        }
-
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->truncate, loc, offset,
-                            xdata);
-                return 0;
-        }
-
-        if (!this->itable)
-                this->itable = loc->inode->table;
-
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
-
-        frame->local = local;
-
-        loc_copy (&local->loc, loc);
-        local->offset = offset;
-        local->block_size = block_size;
-        local->fop = GF_FOP_TRUNCATE;
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
-
-        shard_lookup_base_file (frame, this, &local->loc,
-                                shard_post_lookup_truncate_handler);
-        return 0;
+shard_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_truncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        return 0;
+    }
+
+    if (!this->itable)
+        this->itable = loc->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto err;
+    loc_copy(&local->loc, loc);
+    local->offset = offset;
+    local->block_size = block_size;
+    local->fop = GF_FOP_TRUNCATE;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->resolver_base_inode = loc->inode;
+    GF_ATOMIC_INIT(local->delta_blocks, 0);
+
+    shard_refresh_base_file(frame, this, &local->loc, NULL,
+                            shard_post_lookup_truncate_handler);
+    return 0;
 
 err:
-        SHARD_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_TRUNCATE, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 dict_t *xdata)
+shard_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        int                ret        = -1;
-        uint64_t           block_size = 0;
-        shard_local_t     *local      = NULL;
-
-        ret = shard_inode_ctx_get_block_size (fd->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size from inode ctx of %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto err;
-        }
-
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->ftruncate, fd, offset,
-                            xdata);
-                return 0;
-        }
-
-        if (!this->itable)
-                this->itable = fd->inode->table;
-
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
-
-        frame->local = local;
-        local->fd = fd_ref (fd);
-        local->offset = offset;
-        local->block_size = block_size;
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
-        local->fop = GF_FOP_FTRUNCATE;
-
-        local->loc.inode = inode_ref (fd->inode);
-        gf_uuid_copy (local->loc.gfid, fd->inode->gfid);
-
-        shard_lookup_base_file (frame, this, &local->loc,
-                                shard_post_lookup_truncate_handler);
-        return 0;
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_ftruncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto err;
+    local->fd = fd_ref(fd);
+    local->offset = offset;
+    local->block_size = block_size;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->fop = GF_FOP_FTRUNCATE;
+
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+    local->resolver_base_inode = fd->inode;
+    GF_ATOMIC_INIT(local->delta_blocks, 0);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_post_lookup_truncate_handler);
+    return 0;
 err:
-
-        SHARD_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_FTRUNCATE, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+shard_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        int                ret     = -1;
-        shard_local_t     *local   = NULL;
+    int ret = -1;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto unwind;
+    if (op_ret == -1)
+        goto unwind;
 
-        ret = shard_inode_ctx_set (inode, this, buf, ntoh64 (local->block_size),
-                                   SHARD_ALL_MASK);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        SHARD_MSG_INODE_CTX_SET_FAILED, "Failed to set inode "
-                        "ctx for %s", uuid_utoa (inode->gfid));
+    ret = shard_inode_ctx_set(inode, this, buf, local->block_size,
+                              SHARD_ALL_MASK);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INODE_CTX_SET_FAILED,
+               "Failed to set inode "
+               "ctx for %s",
+               uuid_utoa(inode->gfid));
 
 unwind:
-        SHARD_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
+    SHARD_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-shard_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        shard_local_t  *local      = NULL;
-
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
-
-        frame->local = local;
-        if (!__is_gsyncd_on_shard_dir (frame, loc)) {
-                SHARD_INODE_CREATE_INIT (this, local, xdata, loc, err);
-        }
-
-        STACK_WIND (frame, shard_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
-                    xdata);
-        return 0;
-
+shard_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->block_size = priv->block_size;
+    if (!__is_gsyncd_on_shard_dir(frame, loc)) {
+        SHARD_INODE_CREATE_INIT(this, local->block_size, xdata, loc, 0, 0, err);
+    }
+
+    STACK_WIND(frame, shard_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL,
-                            NULL, NULL);
-        return 0;
-
+    shard_common_failure_unwind(GF_FOP_MKNOD, frame, -1, ENOMEM);
+    return 0;
 }
 
 int32_t
-shard_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent,
-                  dict_t *xdata)
+shard_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        if (op_ret < 0)
-                goto err;
+    shard_local_t *local = NULL;
 
-        shard_inode_ctx_set (inode, this, buf, 0,
-                             SHARD_MASK_NLINK | SHARD_MASK_TIMES);
+    local = frame->local;
+    if (op_ret < 0)
+        goto err;
 
-        SHARD_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    shard_inode_ctx_set(inode, this, buf, 0,
+                        SHARD_MASK_NLINK | SHARD_MASK_TIMES);
+    buf->ia_size = local->prebuf.ia_size;
+    buf->ia_blocks = local->prebuf.ia_blocks;
+
+    SHARD_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (link, frame, op_ret, op_errno, inode, NULL, NULL,
-                            NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_LINK, frame, op_ret, op_errno);
+    return 0;
 }
 
-int32_t
-shard_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata)
+int
+shard_post_lookup_link_handler(call_frame_t *frame, xlator_t *this)
 {
-        int                ret        = -1;
-        uint64_t           block_size = 0;
+    shard_local_t *local = NULL;
 
-        ret = shard_inode_ctx_get_block_size (oldloc->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size from inode ctx of %s",
-                        uuid_utoa (oldloc->inode->gfid));
-                goto err;
-        }
+    local = frame->local;
 
-        if (!block_size) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->link, oldloc, newloc,
-                                 xdata);
-                return 0;
-        }
-
-        STACK_WIND (frame, shard_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    if (local->op_ret < 0) {
+        SHARD_STACK_UNWIND(link, frame, local->op_ret, local->op_errno, NULL,
+                           NULL, NULL, NULL, NULL);
         return 0;
+    }
 
+    STACK_WIND(frame, shard_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, &local->loc, &local->loc2,
+               local->xattr_req);
+    return 0;
+}
+
+int32_t
+shard_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(oldloc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(oldloc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                        oldloc, newloc, xdata);
+        return 0;
+    }
+
+    if (!this->itable)
+        this->itable = oldloc->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->loc2, newloc);
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    shard_refresh_base_file(frame, this, &local->loc, NULL,
+                            shard_post_lookup_link_handler);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                            NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_LINK, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_unlink_shards_do (call_frame_t *frame, xlator_t *this, inode_t *inode);
+shard_unlink_shards_do(call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-shard_post_lookup_shards_unlink_handler (call_frame_t *frame, xlator_t *this)
+shard_post_lookup_shards_unlink_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
 
-        local = frame->local;
+    local = frame->local;
 
-        if ((local->op_ret < 0) && (local->op_errno != ENOENT)) {
-                if (local->fop == GF_FOP_UNLINK)
-                        SHARD_STACK_UNWIND (unlink, frame, local->op_ret,
-                                            local->op_errno, NULL, NULL, NULL);
-                else
-                        SHARD_STACK_UNWIND (rename, frame, local->op_ret,
-                                            local->op_errno, NULL, NULL, NULL,
-                                            NULL, NULL, NULL);
-                return 0;
-        }
-        local->op_ret = 0;
-        local->op_errno = 0;
+    if (local->resolver_base_inode)
+        gf_uuid_copy(gfid, local->resolver_base_inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
 
-        shard_unlink_shards_do (frame, this,
-                                (local->fop == GF_FOP_RENAME)
-                                             ? local->loc2.inode
-                                             : local->loc.inode);
+    if ((local->op_ret < 0) && (local->op_errno != ENOENT)) {
+        gf_msg(this->name, GF_LOG_ERROR, local->op_errno, SHARD_MSG_FOP_FAILED,
+               "failed to delete shards of %s", uuid_utoa(gfid));
         return 0;
+    }
+    local->op_ret = 0;
+    local->op_errno = 0;
+
+    shard_unlink_shards_do(frame, this, local->resolver_base_inode);
+    return 0;
 }
 
 int
-shard_rename_cbk (call_frame_t *frame, xlator_t *this);
+shard_post_resolve_unlink_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
 
-int32_t
-shard_unlink_cbk (call_frame_t *frame, xlator_t *this);
+    local = frame->local;
+    local->lookup_shards_barriered = _gf_true;
+
+    if (!local->call_count)
+        shard_unlink_shards_do(frame, this, local->resolver_base_inode);
+    else
+        shard_common_lookup_shards(frame, this, local->resolver_base_inode,
+                                   shard_post_lookup_shards_unlink_handler);
+    return 0;
+}
+
+void
+shard_unlink_block_inode(shard_local_t *local, int shard_block_num)
+{
+    char block_bname[256] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *base_inode = NULL;
+    xlator_t *this = NULL;
+    shard_priv_t *priv = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *base_ictx = NULL;
+    int unref_base_inode = 0;
+    int unref_shard_inode = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    inode = local->inode_list[shard_block_num - local->first_block];
+    shard_inode_ctx_get(inode, this, &ctx);
+    base_inode = ctx->base_inode;
+    if (base_inode)
+        gf_uuid_copy(gfid, base_inode->gfid);
+    else
+        gf_uuid_copy(gfid, ctx->base_gfid);
+    shard_make_block_bname(shard_block_num, gfid, block_bname,
+                           sizeof(block_bname));
+
+    LOCK(&priv->lock);
+    if (base_inode)
+        LOCK(&base_inode->lock);
+    LOCK(&inode->lock);
+    {
+        __shard_inode_ctx_get(inode, this, &ctx);
+        if (!list_empty(&ctx->ilist)) {
+            list_del_init(&ctx->ilist);
+            priv->inode_count--;
+            unref_base_inode++;
+            unref_shard_inode++;
+            GF_ASSERT(priv->inode_count >= 0);
+        }
+        if (ctx->fsync_needed) {
+            unref_base_inode++;
+            unref_shard_inode++;
+            list_del_init(&ctx->to_fsync_list);
+            if (base_inode) {
+                __shard_inode_ctx_get(base_inode, this, &base_ictx);
+                base_ictx->fsync_count--;
+            }
+        }
+    }
+    UNLOCK(&inode->lock);
+    if (base_inode)
+        UNLOCK(&base_inode->lock);
+
+    inode_unlink(inode, priv->dot_shard_inode, block_bname);
+    inode_ref_reduce_by_n(inode, unref_shard_inode);
+    inode_forget(inode, 0);
+
+    if (base_inode && unref_base_inode)
+        inode_ref_reduce_by_n(base_inode, unref_base_inode);
+    UNLOCK(&priv->lock);
+}
 
 int
-shard_post_resolve_unlink_handler (call_frame_t *frame, xlator_t *this)
+shard_rename_cbk(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                if (local->op_errno == ENOENT) {
-                        /* If lookup on /.shard fails with ENOENT, it probably
-                         * means that the file is being unlinked before it
-                         * could grow beyond its first block. In this case,
-                         * unlink boils down to unlinking the base file and
-                         * unwinding the call.
-                         */
-                        local->op_ret = 0;
-                        local->first_block = local->last_block = 0;
-                        local->num_blocks = 1;
-                        if (local->fop == GF_FOP_UNLINK)
-                                shard_unlink_cbk (frame, this);
-                        else
-                                shard_rename_cbk (frame, this);
-                        return 0;
-                } else {
-                        if (local->fop == GF_FOP_UNLINK)
-                                SHARD_STACK_UNWIND (unlink, frame,
-                                                    local->op_ret,
-                                                    local->op_errno, NULL, NULL,
-                                                    NULL);
-                        else
-                                shard_rename_cbk (frame, this);
-                        return 0;
-                }
-        }
+    SHARD_STACK_UNWIND(rename, frame, local->op_ret, local->op_errno,
+                       &local->prebuf, &local->preoldparent,
+                       &local->postoldparent, &local->prenewparent,
+                       &local->postnewparent, local->xattr_rsp);
+    return 0;
+}
 
-        if (!local->call_count)
-                shard_unlink_shards_do (frame, this,
-                                        (local->fop == GF_FOP_RENAME)
-                                                     ? local->loc2.inode
-                                                     : local->loc.inode);
-        else
-                shard_common_lookup_shards (frame, this,
-                                            (local->fop == GF_FOP_RENAME)
-                                                         ? local->loc2.inode
-                                                         : local->loc.inode,
-                                       shard_post_lookup_shards_unlink_handler);
-        return 0;
+int32_t
+shard_unlink_cbk(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = frame->local;
+
+    SHARD_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                       &local->preoldparent, &local->postoldparent,
+                       local->xattr_rsp);
+    return 0;
 }
 
 int
-shard_unlink_base_file_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t *xdata)
+shard_unlink_shards_do_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
 {
-        int                  ret        = 0;
-        uint32_t             link_count = 0;
-        shard_local_t       *local      = NULL;
-        shard_priv_t        *priv       = NULL;
+    int shard_block_num = (long)cookie;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                SHARD_STACK_UNWIND (unlink, frame, op_ret, op_errno, NULL, NULL,
-                                    NULL);
-                return 0;
-        }
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto done;
+    }
 
-        /* Because link() does not create links for all but the
-         * base shard, unlink() must delete these shards only when the
-         * link count is 1. We can return safely now.
-         */
-        if ((xdata) && (!dict_get_uint32 (xdata, GET_LINK_COUNT, &link_count))
-            && (link_count > 1))
-                goto unwind;
-
-        local->first_block = get_lowest_block (0, local->block_size);
-        local->last_block = get_highest_block (0, local->prebuf.ia_size,
-                                               local->block_size);
-        local->num_blocks = local->last_block - local->first_block + 1;
-
-        /* num_blocks = 1 implies that the file has not crossed its
-         * shard block size. So unlink boils down to unlinking just the
-         * base file. We can safely return now.
+    shard_unlink_block_inode(local, shard_block_num);
+done:
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+int
+shard_unlink_shards_do(call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+    int i = 0;
+    int ret = -1;
+    int count = 0;
+    uint32_t cur_block = 0;
+    uint32_t cur_block_idx = 0; /*this is idx into inode_list[] array */
+    char *bname = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t wind_failed = _gf_false;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (inode)
+        gf_uuid_copy(gfid, inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    for (i = 0; i < local->num_blocks; i++) {
+        if (!local->inode_list[i])
+            continue;
+        count++;
+    }
+
+    if (!count) {
+        /* callcount = 0 implies that all of the shards that need to be
+         * unlinked are non-existent (in other words the file is full of
+         * holes).
          */
-        if (local->num_blocks == 1)
-                goto unwind;
+        gf_msg_debug(this->name, 0,
+                     "All shards that need to be "
+                     "unlinked are non-existent: %s",
+                     uuid_utoa(gfid));
+        return 0;
+    }
 
-        local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
-                                       gf_shard_mt_inode_list);
-        if (!local->inode_list)
-                goto unwind;
+    SHARD_SET_ROOT_FS_ID(frame, local);
+    local->barrier.waitfor = count;
+    cur_block = cur_block_idx + local->first_block;
 
-        /* Save the xdata and preparent and postparent iatts now. This will be
-         * used at the time of unwinding the call to the parent xl.
-         */
-        local->preoldparent = *preparent;
-        local->postoldparent = *postparent;
-        if (xdata)
-                local->xattr_rsp = dict_ref (xdata);
+    while (cur_block_idx < local->num_blocks) {
+        if (!local->inode_list[cur_block_idx])
+            goto next;
 
-        local->dot_shard_loc.inode = inode_find (this->itable,
-                                                 priv->dot_shard_gfid);
-        if (!local->dot_shard_loc.inode) {
-                ret = shard_init_dot_shard_loc (this, local);
-                if (ret)
-                        goto unwind;
-                shard_lookup_dot_shard (frame, this,
-                                        shard_post_resolve_unlink_handler);
-        } else {
-                shard_common_resolve_shards (frame, this, local->loc.inode,
-                                             shard_post_resolve_unlink_handler);
+        if (wind_failed) {
+            shard_unlink_shards_do_cbk(frame, (void *)(long)cur_block, this, -1,
+                                       ENOMEM, NULL, NULL, NULL);
+            goto next;
         }
 
-        return 0;
+        shard_make_block_abspath(cur_block, gfid, path, sizeof(path));
+        bname = strrchr(path, '/') + 1;
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on %s, base file gfid = %s",
+                   bname, uuid_utoa(gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            loc_wipe(&loc);
+            wind_failed = _gf_true;
+            shard_unlink_shards_do_cbk(frame, (void *)(long)cur_block, this, -1,
+                                       ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
+
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+        loc.inode = inode_ref(local->inode_list[cur_block_idx]);
+
+        STACK_WIND_COOKIE(frame, shard_unlink_shards_do_cbk,
+                          (void *)(long)cur_block, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->unlink, &loc, local->xflag,
+                          local->xattr_req);
+        loc_wipe(&loc);
+    next:
+        cur_block++;
+        cur_block_idx++;
+    }
+    syncbarrier_wait(&local->barrier, count);
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+    return 0;
+}
 
-unwind:
-        SHARD_STACK_UNWIND (unlink, frame, op_ret, op_errno,  preparent,
-                            postparent, xdata);
-        return 0;
+int
+shard_regulated_shards_deletion(call_frame_t *cleanup_frame, xlator_t *this,
+                                int now, int first_block, gf_dirent_t *entry)
+{
+    int i = 0;
+    int ret = 0;
+    shard_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    local = cleanup_frame->local;
+
+    local->inode_list = GF_CALLOC(now, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list)
+        return -ENOMEM;
+
+    local->first_block = first_block;
+    local->last_block = first_block + now - 1;
+    local->num_blocks = now;
+    gf_uuid_parse(entry->d_name, gfid);
+    gf_uuid_copy(local->base_gfid, gfid);
+    local->resolver_base_inode = inode_find(this->itable, gfid);
+    local->call_count = 0;
+    ret = syncbarrier_init(&local->barrier);
+    if (ret) {
+        GF_FREE(local->inode_list);
+        local->inode_list = NULL;
+        inode_unref(local->resolver_base_inode);
+        local->resolver_base_inode = NULL;
+        return -errno;
+    }
+    shard_common_resolve_shards(cleanup_frame, this,
+                                shard_post_resolve_unlink_handler);
+
+    for (i = 0; i < local->num_blocks; i++) {
+        if (local->inode_list[i])
+            inode_unref(local->inode_list[i]);
+    }
+    GF_FREE(local->inode_list);
+    local->inode_list = NULL;
+    if (local->op_ret)
+        ret = -local->op_errno;
+    syncbarrier_destroy(&local->barrier);
+    inode_unref(local->resolver_base_inode);
+    local->resolver_base_inode = NULL;
+    STACK_RESET(cleanup_frame->root);
+    return ret;
 }
 
 int
-shard_unlink_base_file (call_frame_t *frame, xlator_t *this)
-{
-        shard_local_t *local = NULL;
+__shard_delete_shards_of_entry(call_frame_t *cleanup_frame, xlator_t *this,
+                               gf_dirent_t *entry, inode_t *inode)
+{
+    int ret = 0;
+    int shard_count = 0;
+    int first_block = 0;
+    int now = 0;
+    uint64_t size = 0;
+    uint64_t block_size = 0;
+    uint64_t size_array[4] = {
+        0,
+    };
+    void *bsize = NULL;
+    void *size_attr = NULL;
+    dict_t *xattr_rsp = NULL;
+    loc_t loc = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = cleanup_frame->local;
+    ret = dict_reset(local->xattr_req);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to reset dict");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set dict value: key:%s", GF_XATTR_SHARD_BLOCK_SIZE);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_FILE_SIZE, 8 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set dict value: key:%s", GF_XATTR_SHARD_FILE_SIZE);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    loc.inode = inode_ref(inode);
+    loc.parent = inode_ref(priv->dot_shard_rm_inode);
+    ret = inode_path(loc.parent, entry->d_name, (char **)&(loc.path));
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path  failed on %s", entry->d_name);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    loc.name = strrchr(loc.path, '/');
+    if (loc.name)
+        loc.name++;
+    ret = syncop_lookup(FIRST_CHILD(this), &loc, NULL, NULL, local->xattr_req,
+                        &xattr_rsp);
+    if (ret)
+        goto err;
+
+    ret = dict_get_ptr(xattr_rsp, GF_XATTR_SHARD_BLOCK_SIZE, &bsize);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to get dict value: key:%s", GF_XATTR_SHARD_BLOCK_SIZE);
+        goto err;
+    }
+    block_size = ntoh64(*((uint64_t *)bsize));
+
+    ret = dict_get_ptr(xattr_rsp, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to get dict value: key:%s", GF_XATTR_SHARD_FILE_SIZE);
+        goto err;
+    }
+
+    memcpy(size_array, size_attr, sizeof(size_array));
+    size = ntoh64(size_array[0]);
+
+    shard_count = (size / block_size) - 1;
+    if (shard_count < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Size of %s hasn't grown beyond "
+                     "its shard-block-size. Nothing to delete. "
+                     "Returning",
+                     entry->d_name);
+        /* File size < shard-block-size, so nothing to delete */
+        ret = 0;
+        goto delete_marker;
+    }
+    if ((size % block_size) > 0)
+        shard_count++;
+
+    if (shard_count == 0) {
+        gf_msg_debug(this->name, 0,
+                     "Size of %s is exactly equal to "
+                     "its shard-block-size. Nothing to delete. "
+                     "Returning",
+                     entry->d_name);
+        ret = 0;
+        goto delete_marker;
+    }
+    gf_msg_debug(this->name, 0,
+                 "base file = %s, "
+                 "shard-block-size=%" PRIu64 ", file-size=%" PRIu64
+                 ", "
+                 "shard_count=%d",
+                 entry->d_name, block_size, size, shard_count);
+
+    /* Perform a gfid-based lookup to see if gfid corresponding to marker
+     * file's base name exists.
+     */
+    loc_wipe(&loc);
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    gf_uuid_parse(entry->d_name, loc.gfid);
+    ret = syncop_lookup(FIRST_CHILD(this), &loc, NULL, NULL, NULL, NULL);
+    if (!ret) {
+        gf_msg_debug(this->name, 0,
+                     "Base shard corresponding to gfid "
+                     "%s is present. Skipping shard deletion. "
+                     "Returning",
+                     entry->d_name);
+        ret = 0;
+        goto delete_marker;
+    }
 
-        local = frame->local;
+    first_block = 1;
+
+    while (shard_count) {
+        if (shard_count < local->deletion_rate) {
+            now = shard_count;
+            shard_count = 0;
+        } else {
+            now = local->deletion_rate;
+            shard_count -= local->deletion_rate;
+        }
 
-        if (dict_set_uint32 (local->xattr_req, GET_LINK_COUNT, 0))
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        SHARD_MSG_DICT_SET_FAILED, "Failed to set "
-                        GET_LINK_COUNT" in dict");
+        gf_msg_debug(this->name, 0,
+                     "deleting %d shards starting from "
+                     "block %d of gfid %s",
+                     now, first_block, entry->d_name);
+        ret = shard_regulated_shards_deletion(cleanup_frame, this, now,
+                                              first_block, entry);
+        if (ret)
+            goto err;
+        first_block += now;
+    }
+
+delete_marker:
+    loc_wipe(&loc);
+    loc.inode = inode_ref(inode);
+    loc.parent = inode_ref(priv->dot_shard_rm_inode);
+    ret = inode_path(loc.parent, entry->d_name, (char **)&(loc.path));
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path  failed on %s", entry->d_name);
+        ret = -ENOMEM;
+        goto err;
+    }
+    loc.name = strrchr(loc.path, '/');
+    if (loc.name)
+        loc.name++;
+    ret = syncop_unlink(FIRST_CHILD(this), &loc, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_SHARDS_DELETION_FAILED,
+               "Failed to delete %s "
+               "from /%s",
+               entry->d_name, GF_SHARD_REMOVE_ME_DIR);
+err:
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+    loc_wipe(&loc);
+    return ret;
+}
 
-        /* To-Do: Request open-fd count on base file */
-        STACK_WIND (frame, shard_unlink_base_file_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, &local->loc, local->xflag,
-                    local->xattr_req);
-        return 0;
+int
+shard_delete_shards_of_entry(call_frame_t *cleanup_frame, xlator_t *this,
+                             gf_dirent_t *entry, inode_t *inode)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    loc.inode = inode_ref(priv->dot_shard_rm_inode);
+
+    ret = syncop_entrylk(FIRST_CHILD(this), this->name, &loc, entry->d_name,
+                         ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL, NULL);
+    if (ret < 0) {
+        if (ret == -EAGAIN) {
+            ret = 0;
+        }
+        goto out;
+    }
+    {
+        ret = __shard_delete_shards_of_entry(cleanup_frame, this, entry, inode);
+    }
+    syncop_entrylk(FIRST_CHILD(this), this->name, &loc, entry->d_name,
+                   ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL, NULL);
+out:
+    loc_wipe(&loc);
+    return ret;
 }
 
-void
-shard_unlink_block_inode (shard_local_t *local, int shard_block_num)
+int
+shard_delete_shards_cbk(int ret, call_frame_t *frame, void *data)
 {
-        char                  block_bname[256]  = {0,};
-        inode_t              *inode             = NULL;
-        xlator_t             *this              = NULL;
-        shard_priv_t         *priv              = NULL;
-        shard_inode_ctx_t    *ctx               = NULL;
+    SHARD_STACK_DESTROY(frame);
+    return 0;
+}
 
-        this = THIS;
-        priv = this->private;
+int
+shard_resolve_internal_dir(xlator_t *this, shard_local_t *local,
+                           shard_internal_dir_type_t type)
+{
+    int ret = 0;
+    char *bname = NULL;
+    loc_t *loc = NULL;
+    shard_priv_t *priv = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+
+    priv = this->private;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            loc = &local->dot_shard_loc;
+            gf_uuid_copy(gfid, priv->dot_shard_gfid);
+            bname = GF_SHARD_DIR;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            loc = &local->dot_shard_rm_loc;
+            gf_uuid_copy(gfid, priv->dot_shard_rm_gfid);
+            bname = GF_SHARD_REMOVE_ME_DIR;
+            break;
+        default:
+            break;
+    }
 
-        inode = local->inode_list[shard_block_num - local->first_block];
+    loc->inode = inode_find(this->itable, gfid);
+    if (!loc->inode) {
+        ret = shard_init_internal_dir_loc(this, local, type);
+        if (ret)
+            goto err;
+        ret = dict_reset(local->xattr_req);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+                   "Failed to reset "
+                   "dict");
+            ret = -ENOMEM;
+            goto err;
+        }
+        ret = dict_set_gfuuid(local->xattr_req, "gfid-req", gfid, true);
+        ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL,
+                            local->xattr_req, NULL);
+        if (ret < 0) {
+            if (ret != -ENOENT)
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       SHARD_MSG_SHARDS_DELETION_FAILED,
+                       "Lookup on %s failed, exiting", bname);
+            goto err;
+        } else {
+            shard_link_internal_dir_inode(local, loc->inode, &stbuf, type);
+        }
+    }
+    ret = 0;
+err:
+    return ret;
+}
 
-        shard_make_block_bname (shard_block_num, (local->loc.inode)->gfid,
-                                block_bname, sizeof (block_bname));
+int
+shard_lookup_marker_entry(xlator_t *this, shard_local_t *local,
+                          gf_dirent_t *entry)
+{
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    loc.parent = inode_ref(local->fd->inode);
+
+    ret = inode_path(loc.parent, entry->d_name, (char **)&(loc.path));
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path failed on %s", entry->d_name);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    loc.name = strrchr(loc.path, '/');
+    if (loc.name)
+        loc.name++;
+
+    ret = syncop_lookup(FIRST_CHILD(this), &loc, NULL, NULL, NULL, NULL);
+    if (ret < 0) {
+        goto err;
+    }
+    entry->inode = inode_ref(loc.inode);
+    ret = 0;
+err:
+    loc_wipe(&loc);
+    return ret;
+}
 
+int
+shard_delete_shards(void *opaque)
+{
+    int ret = 0;
+    off_t offset = 0;
+    loc_t loc = {
+        0,
+    };
+    inode_t *link_inode = NULL;
+    xlator_t *this = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    call_frame_t *cleanup_frame = NULL;
+    gf_boolean_t done = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+    INIT_LIST_HEAD(&entries.list);
+
+    cleanup_frame = opaque;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create local to "
+               "delete shards");
+        ret = -ENOMEM;
+        goto err;
+    }
+    cleanup_frame->local = local;
+    local->fop = GF_FOP_UNLINK;
+
+    local->xattr_req = dict_new();
+    if (!local->xattr_req) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    local->deletion_rate = priv->deletion_rate;
+
+    ret = shard_resolve_internal_dir(this, local, SHARD_INTERNAL_DIR_DOT_SHARD);
+    if (ret == -ENOENT) {
+        gf_msg_debug(this->name, 0,
+                     ".shard absent. Nothing to"
+                     " delete. Exiting");
+        ret = 0;
+        goto err;
+    } else if (ret < 0) {
+        goto err;
+    }
+
+    ret = shard_resolve_internal_dir(this, local,
+                                     SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME);
+    if (ret == -ENOENT) {
+        gf_msg_debug(this->name, 0,
+                     ".remove_me absent. "
+                     "Nothing to delete. Exiting");
+        ret = 0;
+        goto err;
+    } else if (ret < 0) {
+        goto err;
+    }
+
+    local->fd = fd_anonymous(local->dot_shard_rm_loc.inode);
+    if (!local->fd) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    for (;;) {
+        offset = 0;
         LOCK(&priv->lock);
         {
-                shard_inode_ctx_get (inode, this, &ctx);
-                if (!list_empty (&ctx->ilist)) {
-                        list_del_init (&ctx->ilist);
-                        priv->inode_count--;
-                }
-                GF_ASSERT (priv->inode_count >= 0);
-                inode_unlink (inode, priv->dot_shard_inode, block_bname);
-                inode_forget (inode, 0);
+            if (priv->bg_del_state == SHARD_BG_DELETION_LAUNCHING) {
+                priv->bg_del_state = SHARD_BG_DELETION_IN_PROGRESS;
+            } else if (priv->bg_del_state == SHARD_BG_DELETION_IN_PROGRESS) {
+                priv->bg_del_state = SHARD_BG_DELETION_NONE;
+                done = _gf_true;
+            }
         }
         UNLOCK(&priv->lock);
+        if (done)
+            break;
+        while (
+            (ret = syncop_readdirp(FIRST_CHILD(this), local->fd, 131072, offset,
+                                   &entries, local->xattr_req, NULL))) {
+            if (ret > 0)
+                ret = 0;
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                offset = entry->d_off;
+
+                if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                    continue;
+
+                if (!entry->inode) {
+                    ret = shard_lookup_marker_entry(this, local, entry);
+                    if (ret < 0)
+                        continue;
+                }
+                link_inode = inode_link(entry->inode, local->fd->inode,
+                                        entry->d_name, &entry->d_stat);
+
+                gf_msg_debug(this->name, 0,
+                             "Initiating deletion of "
+                             "shards of gfid %s",
+                             entry->d_name);
+                ret = shard_delete_shards_of_entry(cleanup_frame, this, entry,
+                                                   link_inode);
+                inode_unlink(link_inode, local->fd->inode, entry->d_name);
+                inode_unref(link_inode);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           SHARD_MSG_SHARDS_DELETION_FAILED,
+                           "Failed to clean up shards of gfid %s",
+                           entry->d_name);
+                    continue;
+                }
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       SHARD_MSG_SHARD_DELETION_COMPLETED,
+                       "Deleted "
+                       "shards of gfid=%s from backend",
+                       entry->d_name);
+            }
+            gf_dirent_free(&entries);
+            if (ret)
+                break;
+        }
+    }
+    ret = 0;
+    loc_wipe(&loc);
+    return ret;
 
+err:
+    LOCK(&priv->lock);
+    {
+        priv->bg_del_state = SHARD_BG_DELETION_NONE;
+    }
+    UNLOCK(&priv->lock);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-shard_rename_cbk (call_frame_t *frame, xlator_t *this);
-
-int32_t
-shard_unlink_cbk (call_frame_t *frame, xlator_t *this)
+shard_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        shard_local_t *local = frame->local;
-
-	SHARD_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-			    &local->preoldparent, &local->postoldparent,
-                            local->xattr_rsp);
-	return 0;
+    if (op_ret)
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Unlock failed. Please check brick logs for "
+               "more details");
+    SHARD_STACK_DESTROY(frame);
+    return 0;
 }
 
 int
-shard_unlink_shards_do_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t *xdata)
+shard_unlock_inodelk(call_frame_t *frame, xlator_t *this)
 {
-        int            call_count      = 0;
-        int            shard_block_num = (long) cookie;
-        shard_local_t *local           = NULL;
+    loc_t *loc = NULL;
+    call_frame_t *lk_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *lk_local = NULL;
+    shard_inodelk_t *lock = NULL;
 
-        local = frame->local;
+    local = frame->local;
+    lk_frame = local->inodelk_frame;
+    lk_local = lk_frame->local;
+    local->inodelk_frame = NULL;
+    loc = &local->int_inodelk.loc;
+    lock = &lk_local->int_inodelk;
+    lock->flock.l_type = F_UNLCK;
 
-        if (op_ret < 0) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto done;
-        }
+    STACK_WIND(lk_frame, shard_unlock_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, lock->domain, loc, F_SETLK,
+               &lock->flock, NULL);
+    local->int_inodelk.acquired_lock = _gf_false;
+    return 0;
+}
 
-        shard_unlink_block_inode (local, shard_block_num);
+int
+shard_rename_src_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata);
+int
+shard_rename_src_base_file(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    loc_t *dst_loc = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    shard_local_t *local = frame->local;
+
+    if (local->dst_block_size) {
+        tmp_loc.parent = inode_ref(local->loc2.parent);
+        ret = inode_path(tmp_loc.parent, local->loc2.name,
+                         (char **)&tmp_loc.path);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on pargfid=%s bname=%s",
+                   uuid_utoa(tmp_loc.parent->gfid), local->loc2.name);
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
+        }
+
+        tmp_loc.name = strrchr(tmp_loc.path, '/');
+        if (tmp_loc.name)
+            tmp_loc.name++;
+        dst_loc = &tmp_loc;
+    } else {
+        dst_loc = &local->loc2;
+    }
+
+    /* To-Do: Request open-fd count on dst base file */
+    STACK_WIND(frame, shard_rename_src_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &local->loc, dst_loc,
+               local->xattr_req);
+    loc_wipe(&tmp_loc);
+    return 0;
+err:
+    loc_wipe(&tmp_loc);
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
+}
 
-done:
-        call_count = shard_call_count_return (frame);
-        if (call_count == 0) {
-                SHARD_UNSET_ROOT_FS_ID (frame, local);
+int
+shard_unlink_base_file(call_frame_t *frame, xlator_t *this);
 
-                if (local->fop == GF_FOP_UNLINK)
-                        shard_unlink_cbk (frame, this);
-                else if (local->fop == GF_FOP_RENAME)
-                        shard_rename_cbk (frame, this);
-                else
-                        shard_truncate_last_shard (frame, this,
-                                                   local->inode_list[0]);
-        }
+int
+shard_set_size_attrs_on_marker_file_cbk(call_frame_t *frame, void *cookie,
+                                        xlator_t *this, int32_t op_ret,
+                                        int32_t op_errno, dict_t *dict,
+                                        dict_t *xdata)
+{
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Xattrop on marker file failed "
+               "while performing %s; entry gfid=%s",
+               gf_fop_string(local->fop), local->newloc.name);
+        goto err;
+    }
+
+    inode_unlink(local->newloc.inode, priv->dot_shard_rm_inode,
+                 local->newloc.name);
+
+    if (local->fop == GF_FOP_UNLINK)
+        shard_unlink_base_file(frame, this);
+    else if (local->fop == GF_FOP_RENAME)
+        shard_rename_src_base_file(frame, this);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, op_ret, op_errno);
+    return 0;
+}
 
-        return 0;
+int
+shard_set_size_attrs_on_marker_file(call_frame_t *frame, xlator_t *this)
+{
+    int op_errno = ENOMEM;
+    uint64_t bs = 0;
+    dict_t *xdata = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    xdata = dict_new();
+    if (!xdata)
+        goto err;
+
+    if (local->fop == GF_FOP_UNLINK)
+        bs = local->block_size;
+    else if (local->fop == GF_FOP_RENAME)
+        bs = local->dst_block_size;
+    SHARD_INODE_CREATE_INIT(this, bs, xdata, &local->newloc,
+                            local->prebuf.ia_size, 0, err);
+    STACK_WIND(frame, shard_set_size_attrs_on_marker_file_cbk,
+               FIRST_CHILD(this), FIRST_CHILD(this)->fops->xattrop,
+               &local->newloc, GF_XATTROP_GET_AND_SET, xdata, NULL);
+    dict_unref(xdata);
+    return 0;
+err:
+    if (xdata)
+        dict_unref(xdata);
+    shard_common_failure_unwind(local->fop, frame, -1, op_errno);
+    return 0;
 }
 
 int
-shard_unlink_shards_do (call_frame_t *frame, xlator_t *this, inode_t *inode)
+shard_lookup_marker_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, inode_t *inode,
+                             struct iatt *buf, dict_t *xdata,
+                             struct iatt *postparent)
+{
+    inode_t *linked_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Lookup on marker file failed "
+               "while performing %s; entry gfid=%s",
+               gf_fop_string(local->fop), local->newloc.name);
+        goto err;
+    }
+
+    linked_inode = inode_link(inode, priv->dot_shard_rm_inode,
+                              local->newloc.name, buf);
+    inode_unref(local->newloc.inode);
+    local->newloc.inode = linked_inode;
+    shard_set_size_attrs_on_marker_file(frame, this);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, op_ret, op_errno);
+    return 0;
+}
+
+int
+shard_lookup_marker_file(call_frame_t *frame, xlator_t *this)
 {
-        int               i              = 0;
-        int               ret            = -1;
-        int               count          = 0;
-        int               call_count     = 0;
-        uint32_t          last_block     = 0;
-        uint32_t          cur_block      = 0;
-        char             *bname          = NULL;
-        char              path[PATH_MAX] = {0,};
-        loc_t             loc            = {0,};
-        gf_boolean_t      wind_failed    = _gf_false;
-        shard_local_t    *local          = NULL;
-        shard_priv_t     *priv           = NULL;
+    int op_errno = ENOMEM;
+    dict_t *xattr_req = NULL;
+    shard_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    local = frame->local;
 
-        /* local->num_blocks includes the base file block. This function only
-         * deletes the shards under /.shard. So subtract num_blocks by 1.
-         */
-        local->call_count = call_count = local->num_blocks - 1;
-        last_block = local->last_block;
+    xattr_req = shard_create_gfid_dict(local->xattr_req);
+    if (!xattr_req)
+        goto err;
 
-        /* Ignore the inode associated with the base file and start counting
-         * from 1.
-         */
-        for (i = 1; i < local->num_blocks; i++) {
-                if (!local->inode_list[i])
-                        continue;
-                count++;
-        }
+    STACK_WIND(frame, shard_lookup_marker_file_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &local->newloc, xattr_req);
+    dict_unref(xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, -1, op_errno);
+    return 0;
+}
 
-        if (!count) {
-                /* callcount = 0 implies that all of the shards that need to be
-                 * unlinked are non-existent (in other words the file is full of
-                 * holes). So shard xlator can simply return the fop to its
-                 * parent now.
-                 */
-                gf_msg_debug (this->name, 0, "All shards that need to be "
-                              "unlinked are non-existent: %s",
-                              uuid_utoa (inode->gfid));
-                local->num_blocks = 1;
-                if (local->fop == GF_FOP_UNLINK) {
-                        shard_unlink_cbk (frame, this);
-                } else if (local->fop == GF_FOP_RENAME) {
-                        gf_msg_debug (this->name, 0, "Resuming rename()");
-                        shard_rename_cbk (frame, this);
-                }
-                return 0;
+int
+shard_create_marker_file_under_remove_me_cbk(
+    call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+    int32_t op_errno, inode_t *inode, struct iatt *buf, struct iatt *preparent,
+    struct iatt *postparent, dict_t *xdata)
+{
+    inode_t *linked_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+    if (op_ret < 0) {
+        if ((op_errno != EEXIST) && (op_errno != ENODATA)) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+                   "Marker file creation "
+                   "failed while performing %s; entry gfid=%s",
+                   gf_fop_string(local->fop), local->newloc.name);
+            goto err;
+        } else {
+            shard_lookup_marker_file(frame, this);
+            return 0;
         }
+    }
 
-        local->call_count = call_count = count;
-        cur_block = 1;
-        SHARD_SET_ROOT_FS_ID (frame, local);
-
-        /* Ignore the base file and start iterating from the first block shard.
-         */
-        while (cur_block <= last_block) {
-                if (!local->inode_list[cur_block]) {
-                        cur_block++;
-                        continue;
-                }
-
-                if (wind_failed) {
-                        shard_unlink_shards_do_cbk (frame,
-                                                    (void *) (long) cur_block,
-                                                    this, -1, ENOMEM, NULL,
-                                                    NULL, NULL);
-                        goto next;
-                }
+    linked_inode = inode_link(inode, priv->dot_shard_rm_inode,
+                              local->newloc.name, buf);
+    inode_unref(local->newloc.inode);
+    local->newloc.inode = linked_inode;
 
-                shard_make_block_abspath (cur_block, inode->gfid, path,
-                                          sizeof (path));
-                bname = strrchr (path, '/') + 1;
-                loc.parent = inode_ref (priv->dot_shard_inode);
-                ret = inode_path (loc.parent, bname, (char **) &(loc.path));
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                SHARD_MSG_INODE_PATH_FAILED, "Inode path failed"
-                                " on %s, base file gfid = %s", bname,
-                                uuid_utoa (inode->gfid));
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        loc_wipe (&loc);
-                        wind_failed = _gf_true;
-                        shard_unlink_shards_do_cbk (frame,
-                                                    (void *) (long) cur_block,
-                                                    this, -1, ENOMEM, NULL,
-                                                    NULL, NULL);
-                        goto next;
-                }
+    if (local->fop == GF_FOP_UNLINK)
+        shard_unlink_base_file(frame, this);
+    else if (local->fop == GF_FOP_RENAME)
+        shard_rename_src_base_file(frame, this);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
+    return 0;
+}
 
-                loc.name = strrchr (loc.path, '/');
-                if (loc.name)
-                        loc.name++;
-                loc.inode = inode_ref (local->inode_list[cur_block]);
-
-                STACK_WIND_COOKIE (frame, shard_unlink_shards_do_cbk,
-                                   (void *) (long) cur_block, FIRST_CHILD(this),
-                                   FIRST_CHILD (this)->fops->unlink, &loc,
-                                   local->xflag, local->xattr_req);
-                loc_wipe (&loc);
-
-next:
-                cur_block++;
-                if (!--call_count)
-                        break;
-        }
+int
+shard_create_marker_file_under_remove_me(call_frame_t *frame, xlator_t *this,
+                                         loc_t *loc)
+{
+    int ret = 0;
+    int op_errno = ENOMEM;
+    uint64_t bs = 0;
+    char g1[64] = {
+        0,
+    };
+    char g2[64] = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    xattr_req = shard_create_gfid_dict(local->xattr_req);
+    if (!xattr_req)
+        goto err;
+
+    local->newloc.inode = inode_new(this->itable);
+    local->newloc.parent = inode_ref(priv->dot_shard_rm_inode);
+    ret = inode_path(local->newloc.parent, uuid_utoa(loc->inode->gfid),
+                     (char **)&local->newloc.path);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path failed on "
+               "pargfid=%s bname=%s",
+               uuid_utoa_r(priv->dot_shard_rm_gfid, g1),
+               uuid_utoa_r(loc->inode->gfid, g2));
+        goto err;
+    }
+    local->newloc.name = strrchr(local->newloc.path, '/');
+    if (local->newloc.name)
+        local->newloc.name++;
+
+    if (local->fop == GF_FOP_UNLINK)
+        bs = local->block_size;
+    else if (local->fop == GF_FOP_RENAME)
+        bs = local->dst_block_size;
+
+    SHARD_INODE_CREATE_INIT(this, bs, xattr_req, &local->newloc,
+                            local->prebuf.ia_size, 0, err);
+
+    STACK_WIND(frame, shard_create_marker_file_under_remove_me_cbk,
+               FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+               &local->newloc, 0, 0, 0644, xattr_req);
+    dict_unref(xattr_req);
+    return 0;
 
-        return 0;
+err:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    shard_create_marker_file_under_remove_me_cbk(frame, 0, this, -1, op_errno,
+                                                 NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-shard_post_lookup_unlink_handler (call_frame_t *frame, xlator_t *this)
+shard_unlock_entrylk(call_frame_t *frame, xlator_t *this);
+
+int
+shard_unlink_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
 {
-        int            ret   = -1;
-        shard_priv_t  *priv  = NULL;
-        shard_local_t *local = NULL;
+    int ret = 0;
+    shard_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                SHARD_STACK_UNWIND (unlink, frame, local->op_ret,
-                                    local->op_errno, NULL, NULL, NULL);
-                return 0;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    } else {
+        shard_inode_ctx_set_refresh_flag(local->int_inodelk.loc.inode, this);
+        local->preoldparent = *preparent;
+        local->postoldparent = *postparent;
+        if (xdata)
+            local->xattr_rsp = dict_ref(xdata);
+        if (local->cleanup_required)
+            shard_start_background_deletion(this);
+    }
+
+    if (local->entrylk_frame) {
+        ret = shard_unlock_entrylk(frame, this);
+        if (ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
         }
+    }
 
-        shard_unlink_base_file (frame, this);
-        return 0;
+    ret = shard_unlock_inodelk(frame, this);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = -ret;
+    }
+
+    shard_unlink_cbk(frame, this);
+    return 0;
 }
 
 int
-shard_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+shard_unlink_base_file(call_frame_t *frame, xlator_t *this)
 {
-        int             ret        = -1;
-        uint64_t        block_size = 0;
-        shard_local_t  *local      = NULL;
+    shard_local_t *local = frame->local;
 
-        ret = shard_inode_ctx_get_block_size (loc->inode, this, &block_size);
-        if ((ret) && (!IA_ISLNK(loc->inode->ia_type))) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size from inode ctx of %s",
-                        uuid_utoa (loc->inode->gfid));
-                goto err;
-        }
+    /* To-Do: Request open-fd count on base file */
+    STACK_WIND(frame, shard_unlink_base_file_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, &local->loc, local->xflag,
+               local->xattr_req);
+    return 0;
+}
 
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-                return 0;
-        }
+int
+shard_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    if (op_ret)
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Unlock failed. Please check brick logs for "
+               "more details");
+    SHARD_STACK_DESTROY(frame);
+    return 0;
+}
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+int
+shard_unlock_entrylk(call_frame_t *frame, xlator_t *this)
+{
+    loc_t *loc = NULL;
+    call_frame_t *lk_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *lk_local = NULL;
+    shard_entrylk_t *lock = NULL;
 
-        frame->local = local;
+    local = frame->local;
+    lk_frame = local->entrylk_frame;
+    lk_local = lk_frame->local;
+    local->entrylk_frame = NULL;
+    lock = &lk_local->int_entrylk;
+    loc = &lock->loc;
 
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        local->block_size = block_size;
-        local->fop = GF_FOP_UNLINK;
-        if (!this->itable)
-                this->itable = (local->loc.inode)->table;
+    STACK_WIND(lk_frame, shard_unlock_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, loc,
+               lk_local->int_entrylk.basename, ENTRYLK_UNLOCK, ENTRYLK_WRLCK,
+               NULL);
+    local->int_entrylk.acquired_lock = _gf_false;
+    return 0;
+}
 
-        shard_lookup_base_file (frame, this, &local->loc,
-                                shard_post_lookup_unlink_handler);
-        return 0;
-err:
-        SHARD_STACK_UNWIND (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+int
+shard_post_entrylk_fop_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
 
+    switch (local->fop) {
+        case GF_FOP_UNLINK:
+        case GF_FOP_RENAME:
+            shard_create_marker_file_under_remove_me(frame, this,
+                                                     &local->int_inodelk.loc);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "post-entrylk handler not defined. This case should not"
+                   " be hit");
+            break;
+    }
+    return 0;
 }
 
 int
-shard_rename_cbk (call_frame_t *frame, xlator_t *this)
+shard_acquire_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        shard_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *main_local = NULL;
 
-        local = frame->local;
+    local = frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
 
-        SHARD_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                            &local->prebuf, &local->preoldparent,
-                            &local->postoldparent, &local->prenewparent,
-                            &local->postnewparent, local->xattr_rsp);
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(main_local->fop, main_frame, op_ret,
+                                    op_errno);
         return 0;
+    }
+    main_local->int_entrylk.acquired_lock = _gf_true;
+    shard_post_entrylk_fop_handler(main_frame, this);
+    return 0;
 }
 
 int
-shard_rename_unlink_dst_shards_do (call_frame_t *frame, xlator_t *this)
-{
-        int            ret        = -1;
-        uint32_t       link_count = 0;
-        shard_local_t *local      = NULL;
-        shard_priv_t  *priv       = NULL;
+shard_acquire_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      uuid_t gfid)
+{
+    char gfid_str[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    shard_local_t *entrylk_local = NULL;
+    shard_entrylk_t *int_entrylk = NULL;
+    call_frame_t *entrylk_frame = NULL;
+
+    local = frame->local;
+    entrylk_frame = create_frame(this, this->ctx->pool);
+    if (!entrylk_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create new frame "
+               "to lock marker file");
+        goto err;
+    }
+
+    entrylk_local = mem_get0(this->local_pool);
+    if (!entrylk_local) {
+        STACK_DESTROY(entrylk_frame->root);
+        goto err;
+    }
+
+    entrylk_frame->local = entrylk_local;
+    entrylk_local->main_frame = frame;
+    int_entrylk = &entrylk_local->int_entrylk;
+
+    int_entrylk->loc.inode = inode_ref(inode);
+    set_lk_owner_from_ptr(&entrylk_frame->root->lk_owner, entrylk_frame->root);
+    local->entrylk_frame = entrylk_frame;
+    gf_uuid_unparse(gfid, gfid_str);
+    int_entrylk->basename = gf_strdup(gfid_str);
+
+    STACK_WIND(entrylk_frame, shard_acquire_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &int_entrylk->loc,
+               int_entrylk->basename, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+    return 0;
+}
 
-        local = frame->local;
-        priv = this->private;
+int
+shard_post_lookup_base_shard_rm_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
+        return 0;
+    }
+
+    if (local->prebuf.ia_nlink > 1) {
+        gf_msg_debug(this->name, 0,
+                     "link count on %s > 1:%d, "
+                     "performing rename()/unlink()",
+                     local->int_inodelk.loc.path, local->prebuf.ia_nlink);
+        if (local->fop == GF_FOP_RENAME)
+            shard_rename_src_base_file(frame, this);
+        else if (local->fop == GF_FOP_UNLINK)
+            shard_unlink_base_file(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "link count on %s = 1, creating "
+                     "file under .remove_me",
+                     local->int_inodelk.loc.path);
+        local->cleanup_required = _gf_true;
+        shard_acquire_entrylk(frame, this, priv->dot_shard_rm_inode,
+                              local->prebuf.ia_gfid);
+    }
+    return 0;
+}
 
-        local->first_block = get_lowest_block (0, local->dst_block_size);
-        local->last_block = get_highest_block (0, local->postbuf.ia_size,
-                                               local->dst_block_size);
-        local->num_blocks = local->last_block - local->first_block + 1;
+int
+shard_post_inodelk_fop_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
 
-        if ((local->xattr_rsp) &&
-            (!dict_get_uint32 (local->xattr_rsp, GET_LINK_COUNT, &link_count))
-            && (link_count > 1)) {
-                shard_rename_cbk (frame, this);
-                return 0;
-        }
+    local = frame->local;
 
-        if (local->num_blocks == 1) {
-                shard_rename_cbk (frame, this);
-                return 0;
-        }
+    switch (local->fop) {
+        case GF_FOP_UNLINK:
+        case GF_FOP_RENAME:
+            shard_refresh_base_file(frame, this, &local->int_inodelk.loc, NULL,
+                                    shard_post_lookup_base_shard_rm_handler);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "post-inodelk handler not defined. This case should not"
+                   " be hit");
+            break;
+    }
+    return 0;
+}
 
-        local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
-                                       gf_shard_mt_inode_list);
-        if (!local->inode_list)
-                goto out;
+int
+shard_acquire_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    call_frame_t *main_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *main_local = NULL;
 
-        local->dot_shard_loc.inode = inode_find (this->itable,
-                                                 priv->dot_shard_gfid);
-        if (!local->dot_shard_loc.inode) {
-                ret = shard_init_dot_shard_loc (this, local);
-                if (ret)
-                        goto out;
-                shard_lookup_dot_shard (frame, this,
-                                        shard_post_resolve_unlink_handler);
-        } else {
-                shard_common_resolve_shards (frame, this, local->loc2.inode,
-                                             shard_post_resolve_unlink_handler);
-        }
+    local = frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
 
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(main_local->fop, main_frame, op_ret,
+                                    op_errno);
         return 0;
+    }
+    main_local->int_inodelk.acquired_lock = _gf_true;
+    shard_post_inodelk_fop_handler(main_frame, this);
+    return 0;
+}
 
-out:
-        SHARD_STACK_UNWIND (rename, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                            NULL, NULL);
-        return 0;
+int
+shard_acquire_inodelk(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    call_frame_t *lk_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *lk_local = NULL;
+    shard_inodelk_t *int_inodelk = NULL;
+
+    local = frame->local;
+    lk_frame = create_frame(this, this->ctx->pool);
+    if (!lk_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create new frame "
+               "to lock base shard");
+        goto err;
+    }
+    lk_local = mem_get0(this->local_pool);
+    if (!lk_local) {
+        STACK_DESTROY(lk_frame->root);
+        goto err;
+    }
+
+    lk_frame->local = lk_local;
+    lk_local->main_frame = frame;
+    int_inodelk = &lk_local->int_inodelk;
+
+    int_inodelk->flock.l_len = 0;
+    int_inodelk->flock.l_start = 0;
+    int_inodelk->domain = this->name;
+    int_inodelk->flock.l_type = F_WRLCK;
+    loc_copy(&local->int_inodelk.loc, loc);
+    set_lk_owner_from_ptr(&lk_frame->root->lk_owner, lk_frame->root);
+    local->inodelk_frame = lk_frame;
+
+    STACK_WIND(lk_frame, shard_acquire_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, int_inodelk->domain,
+               &local->int_inodelk.loc, F_SETLKW, &int_inodelk->flock, NULL);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_post_rename_lookup_handler (call_frame_t *frame, xlator_t *this)
+shard_post_mkdir_rm_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    loc_t *loc = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
-
-        if (local->op_ret < 0) {
-                SHARD_STACK_UNWIND (rename, frame, local->op_ret,
-                                    local->op_errno, NULL, NULL, NULL, NULL,
-                                    NULL, NULL);
-                return 0;
-        }
-
-        if (local->dst_block_size)
-                shard_rename_unlink_dst_shards_do (frame, this);
-        else
-                shard_rename_cbk (frame, this);
+    local = frame->local;
 
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
         return 0;
+    }
+    if (local->fop == GF_FOP_UNLINK)
+        loc = &local->loc;
+    else if (local->fop == GF_FOP_RENAME)
+        loc = &local->loc2;
+    shard_acquire_inodelk(frame, this, loc);
+    return 0;
 }
 
 int
-shard_rename_src_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                      struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent,
-                      dict_t *xdata)
+shard_mkdir_internal_dir(call_frame_t *frame, xlator_t *this,
+                         shard_post_resolve_fop_handler_t handler,
+                         shard_internal_dir_type_t type);
+int
+shard_pre_mkdir_rm_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
+        return 0;
+    }
+    shard_mkdir_internal_dir(frame, this, shard_post_mkdir_rm_handler,
+                             SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME);
+    return 0;
+}
 
-        local->prebuf = *buf;
-        local->preoldparent = *preoldparent;
-        local->postoldparent = *postoldparent;
-        local->prenewparent = *prenewparent;
-        local->postnewparent = *postnewparent;
-        if (xdata)
-                local->xattr_rsp = dict_ref (xdata);
+void
+shard_begin_rm_resolution(call_frame_t *frame, xlator_t *this)
+{
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
 
-        /* Now the base file is looked up to gather the ia_size and ia_blocks.*/
+    priv = this->private;
+    local = frame->local;
 
-        if (local->block_size) {
-                local->tmp_loc.inode = inode_new (this->itable);
-                gf_uuid_copy (local->tmp_loc.gfid, (local->loc.inode)->gfid);
-                shard_lookup_base_file (frame, this, &local->tmp_loc,
-                                        shard_post_rename_lookup_handler);
+    local->dot_shard_rm_loc.inode = inode_find(this->itable,
+                                               priv->dot_shard_rm_gfid);
+    if (!local->dot_shard_rm_loc.inode) {
+        local->dot_shard_loc.inode = inode_find(this->itable,
+                                                priv->dot_shard_gfid);
+        if (!local->dot_shard_loc.inode) {
+            shard_mkdir_internal_dir(frame, this, shard_pre_mkdir_rm_handler,
+                                     SHARD_INTERNAL_DIR_DOT_SHARD);
         } else {
-                shard_rename_unlink_dst_shards_do (frame, this);
+            local->post_res_handler = shard_pre_mkdir_rm_handler;
+            shard_refresh_internal_dir(frame, this,
+                                       SHARD_INTERNAL_DIR_DOT_SHARD);
         }
+    } else {
+        local->post_res_handler = shard_post_mkdir_rm_handler;
+        shard_refresh_internal_dir(frame, this,
+                                   SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME);
+    }
+}
 
-        return 0;
+int
+shard_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if ((ret) && (!IA_ISLNK(loc->inode->ia_type))) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    loc_copy(&local->loc, loc);
+    local->xflag = xflag;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    local->block_size = block_size;
+    local->resolver_base_inode = loc->inode;
+    local->fop = GF_FOP_UNLINK;
+    if (!this->itable)
+        this->itable = (local->loc.inode)->table;
+
+    local->resolve_not = _gf_true;
+    shard_begin_rm_resolution(frame, this);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL,
-                            NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_UNLINK, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_rename_src_base_file (call_frame_t *frame, xlator_t *this)
+shard_post_rename_lookup_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (dict_set_uint32 (local->xattr_req, GET_LINK_COUNT, 0))
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        SHARD_MSG_DICT_SET_FAILED, "Failed to set "
-                        GET_LINK_COUNT" in dict");
-
-        /* To-Do: Request open-fd count on dst base file */
-        STACK_WIND (frame, shard_rename_src_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, &local->loc, &local->loc2,
-                    local->xattr_req);
-        return 0;
+    shard_rename_cbk(frame, this);
+    return 0;
 }
 
 int
-shard_post_lookup_dst_base_file_handler (call_frame_t *frame, xlator_t *this)
+shard_rename_src_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
-        shard_local_t *local = NULL;
+    int ret = 0;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                SHARD_STACK_UNWIND (rename, frame, local->op_ret,
-                                    local->op_errno, NULL, NULL, NULL, NULL,
-                                    NULL, NULL);
-                return 0;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+    /* Set ctx->refresh to TRUE to force a lookup on disk when
+     * shard_lookup_base_file() is called next to refresh the hard link
+     * count in ctx. Note that this is applicable only to the case where
+     * the rename dst is already existent and sharded.
+     */
+    if ((local->dst_block_size) && (!local->cleanup_required))
+        shard_inode_ctx_set_refresh_flag(local->int_inodelk.loc.inode, this);
+
+    local->prebuf = *buf;
+    local->preoldparent = *preoldparent;
+    local->postoldparent = *postoldparent;
+    local->prenewparent = *prenewparent;
+    local->postnewparent = *postnewparent;
+    if (xdata)
+        local->xattr_rsp = dict_ref(xdata);
+
+    if (local->dst_block_size) {
+        if (local->entrylk_frame) {
+            ret = shard_unlock_entrylk(frame, this);
+            if (ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = -ret;
+            }
         }
 
-        /* Save dst base file attributes into postbuf so the information is not
-         * lost when it is overwritten after lookup on base file of src in
-         * shard_lookup_base_file_cbk().
-         */
-        local->postbuf = local->prebuf;
-        shard_rename_src_base_file (frame, this);
-        return 0;
+        ret = shard_unlock_inodelk(frame, this);
+        if (ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            goto err;
+        }
+        if (local->cleanup_required)
+            shard_start_background_deletion(this);
+    }
+
+    /* Now the base file of src, if sharded, is looked up to gather ia_size
+     * and ia_blocks.*/
+    if (local->block_size) {
+        local->tmp_loc.inode = inode_new(this->itable);
+        gf_uuid_copy(local->tmp_loc.gfid, (local->loc.inode)->gfid);
+        shard_refresh_base_file(frame, this, &local->tmp_loc, NULL,
+                                shard_post_rename_lookup_handler);
+    } else {
+        shard_rename_cbk(frame, this);
+    }
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
 }
 
 int
-shard_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata)
+shard_post_lookup_dst_base_file_handler(call_frame_t *frame, xlator_t *this)
 {
-        int             ret            = -1;
-        uint64_t        block_size     = 0;
-        uint64_t        dst_block_size = 0;
-        shard_local_t  *local          = NULL;
+    shard_local_t *local = NULL;
 
-        if (IA_ISDIR (oldloc->inode->ia_type)) {
-                STACK_WIND (frame, default_rename_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename, oldloc, newloc,
-                            xdata);
-                return 0;
-        }
+    local = frame->local;
 
-        ret = shard_inode_ctx_get_block_size (oldloc->inode, this, &block_size);
-        if ((ret) && (!IA_ISLNK (oldloc->inode->ia_type))) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size from inode ctx of %s",
-                        uuid_utoa (oldloc->inode->gfid));
-                goto err;
-        }
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-        if (newloc->inode)
-                ret = shard_inode_ctx_get_block_size (newloc->inode, this,
-                                                      &dst_block_size);
-        /* The following stack_wind covers the case where:
-         * a. the src file is not sharded and dst doesn't exist, OR
-         * b. the src and dst both exist but are not sharded.
-         */
-        if (((!block_size) && (!dst_block_size)) ||
-            frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_rename_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename, oldloc, newloc,
-                            xdata);
-                return 0;
-        }
+    /* Save dst base file attributes into postbuf so the information is not
+     * lost when it is overwritten after lookup on base file of src in
+     * shard_lookup_base_file_cbk().
+     */
+    local->postbuf = local->prebuf;
+    shard_rename_src_base_file(frame, this);
+    return 0;
+}
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
-
-        frame->local = local;
-        loc_copy (&local->loc, oldloc);
-        loc_copy (&local->loc2, newloc);
-        local->fop = GF_FOP_RENAME;
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new();
-        if (!local->xattr_req)
-                goto err;
-
-        local->block_size = block_size;
-        local->dst_block_size = dst_block_size;
-        if (!this->itable)
-                this->itable = (local->loc.inode)->table;
-
-        if (local->dst_block_size)
-                /* The if block covers the case where the dst file exists and is
-                 * sharded. So it is important to look up this inode, record its
-                 * size, before renaming src to dst, so as to NOT lose this
-                 * information.
-                 */
-                shard_lookup_base_file (frame, this, &local->loc2,
-                                       shard_post_lookup_dst_base_file_handler);
-        else
-                /* The following block covers the case where the dst either
-                 * doesn't exist or is NOT sharded. In this case, shard xlator
-                 * would go ahead and rename src to dst.
-                 */
-                shard_rename_src_base_file (frame, this);
-        return 0;
+int
+shard_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    uint64_t dst_block_size = 0;
+    shard_local_t *local = NULL;
+
+    if (IA_ISDIR(oldloc->inode->ia_type)) {
+        STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+        return 0;
+    }
+
+    ret = shard_inode_ctx_get_block_size(oldloc->inode, this, &block_size);
+    if ((ret) && (!IA_ISLNK(oldloc->inode->ia_type))) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(oldloc->inode->gfid));
+        goto err;
+    }
+
+    if (newloc->inode)
+        ret = shard_inode_ctx_get_block_size(newloc->inode, this,
+                                             &dst_block_size);
+
+    /* The following stack_wind covers the case where:
+     * a. the src file is not sharded and dst doesn't exist, OR
+     * b. the src and dst both exist but are not sharded.
+     */
+    if (((!block_size) && (!dst_block_size)) ||
+        frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+        return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->loc2, newloc);
+    local->resolver_base_inode = newloc->inode;
+    local->fop = GF_FOP_RENAME;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    local->block_size = block_size;
+    local->dst_block_size = dst_block_size;
+    if (!this->itable)
+        this->itable = (local->loc.inode)->table;
+    local->resolve_not = _gf_true;
+
+    /* The following if-block covers the case where the dst file exists
+     * and is sharded.
+     */
+    if (local->dst_block_size) {
+        shard_begin_rm_resolution(frame, this);
+    } else {
+        /* The following block covers the case where the dst either doesn't
+         * exist or is NOT sharded but the src is sharded. In this case, shard
+         * xlator would go ahead and rename src to dst. Once done, it would also
+         * lookup the base shard of src to get the ia_size and ia_blocks xattr
+         * values.
+         */
+        shard_rename_src_base_file(frame, this);
+    }
+    return 0;
 
 err:
-        SHARD_STACK_UNWIND (rename, frame, -1, ENOMEM, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
-        return 0;
-
+    shard_common_failure_unwind(GF_FOP_RENAME, frame, -1, ENOMEM);
+    return 0;
 }
 
-
 int
-shard_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                  struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+shard_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int             ret        = -1;
-        shard_local_t  *local      = NULL;
+    int ret = -1;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto unwind;
+    if (op_ret == -1)
+        goto unwind;
 
-        ret = shard_inode_ctx_set (inode, this, stbuf,
-                                   ntoh64 (local->block_size), SHARD_ALL_MASK);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        SHARD_MSG_INODE_CTX_SET_FAILED, "Failed to set inode "
-                        "ctx for %s", uuid_utoa (inode->gfid));
+    ret = shard_inode_ctx_set(inode, this, stbuf, local->block_size,
+                              SHARD_ALL_MASK);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INODE_CTX_SET_FAILED,
+               "Failed to set inode "
+               "ctx for %s",
+               uuid_utoa(inode->gfid));
 
 unwind:
-        SHARD_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf,
-                            preparent, postparent, xdata);
-        return 0;
+    SHARD_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                       preparent, postparent, xdata);
+    return 0;
 }
 
 int
-shard_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+shard_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        shard_local_t  *local      = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    priv = this->private;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
 
-        frame->local = local;
+    frame->local = local;
+    local->block_size = priv->block_size;
 
-        if (!__is_gsyncd_on_shard_dir (frame, loc)) {
-                SHARD_INODE_CREATE_INIT (this, local, xdata, loc, err);
-        }
-
-        STACK_WIND (frame, shard_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
-                    fd, xdata);
-        return 0;
+    if (!__is_gsyncd_on_shard_dir(frame, loc)) {
+        SHARD_INODE_CREATE_INIT(this, local->block_size, xdata, loc, 0, 0, err);
+    }
 
+    STACK_WIND(frame, shard_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (create, frame, -1, ENOMEM, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
-        return 0;
-
+    shard_common_failure_unwind(GF_FOP_CREATE, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+shard_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        /* To-Do: Handle open with O_TRUNC under locks */
-        SHARD_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    /* To-Do: Handle open with O_TRUNC under locks */
+    SHARD_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int
-shard_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            fd_t *fd, dict_t *xdata)
+shard_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, shard_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, shard_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 int
-shard_readv_do_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                    int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                    dict_t *xdata)
+shard_readv_do_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                   dict_t *xdata)
 {
-        int                i             = 0;
-        int                call_count    = 0;
-        void              *address       = NULL;
-        uint64_t           block_num     = 0;
-        off_t              off           = 0;
-        struct iovec       vec           = {0,};
-        shard_local_t     *local         = NULL;
-        fd_t              *anon_fd       = cookie;
+    int i = 0;
+    int call_count = 0;
+    void *address = NULL;
+    uint64_t block_num = 0;
+    off_t off = 0;
+    struct iovec vec = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    fd_t *anon_fd = cookie;
+    shard_inode_ctx_t *ctx = NULL;
+
+    local = frame->local;
+
+    /* If shard has already seen a failure here before, there is no point
+     * in aggregating subsequent reads, so just go to out.
+     */
+    if (local->op_ret < 0)
+        goto out;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto out;
+    }
 
-        local = frame->local;
+    if (local->op_ret >= 0)
+        local->op_ret += op_ret;
 
-        /* If shard has already seen a failure here before, there is no point
-         * in aggregating subsequent reads, so just go to out.
+    shard_inode_ctx_get(anon_fd->inode, this, &ctx);
+    block_num = ctx->block_num;
+
+    if (block_num == local->first_block) {
+        address = local->iobuf->ptr;
+    } else {
+        /* else
+         * address to start writing to = beginning of buffer +
+         *                    number of bytes until end of first block +
+         *                    + block_size times number of blocks
+         *                    between the current block and the first
          */
-        if (local->op_ret < 0)
-                goto out;
-
-        if (op_ret < 0) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto out;
-        }
-
-        if (local->op_ret >= 0)
-                local->op_ret += op_ret;
-
-        fd_ctx_get (anon_fd, this, &block_num);
-
-        if (block_num == local->first_block) {
-                address = local->iobuf->ptr;
-        } else {
-                /* else
-                 * address to start writing to = beginning of buffer +
-                 *                    number of bytes until end of first block +
-                 *                    + block_size times number of blocks
-                 *                    between the current block and the first
-                 */
-                address = (char *) local->iobuf->ptr + (local->block_size -
-                          (local->offset % local->block_size)) +
-                          ((block_num - local->first_block - 1) *
-                          local->block_size);
-        }
+        address = (char *)local->iobuf->ptr +
+                  (local->block_size - (local->offset % local->block_size)) +
+                  ((block_num - local->first_block - 1) * local->block_size);
+    }
 
-        for (i = 0; i < count; i++) {
-                address = (char *) address + off;
-                memcpy (address, vector[i].iov_base, vector[i].iov_len);
-                off += vector[i].iov_len;
-        }
+    for (i = 0; i < count; i++) {
+        address = (char *)address + off;
+        memcpy(address, vector[i].iov_base, vector[i].iov_len);
+        off += vector[i].iov_len;
+    }
 
 out:
-        if (anon_fd)
-                fd_unref (anon_fd);
-        call_count = shard_call_count_return (frame);
-        if (call_count == 0) {
-                SHARD_UNSET_ROOT_FS_ID (frame, local);
-                if (local->op_ret < 0) {
-                        SHARD_STACK_UNWIND (readv, frame, local->op_ret,
-                                            local->op_errno, NULL, 0, NULL,
-                                            NULL, NULL);
-                } else {
-                        if (xdata)
-                                local->xattr_rsp = dict_ref (xdata);
-                        vec.iov_base = local->iobuf->ptr;
-                        vec.iov_len = local->total_size;
-                        SHARD_STACK_UNWIND (readv, frame, local->total_size,
-                                            local->op_errno, &vec, 1,
-                                            &local->prebuf, local->iobref,
-                                            local->xattr_rsp);
-                        return 0;
-                }
+    if (anon_fd)
+        fd_unref(anon_fd);
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        if (local->op_ret < 0) {
+            shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                        local->op_errno);
+        } else {
+            if (xdata)
+                local->xattr_rsp = dict_ref(xdata);
+            vec.iov_base = local->iobuf->ptr;
+            if (local->offset + local->req_size > local->prebuf.ia_size)
+                local->total_size = local->prebuf.ia_size - local->offset;
+            vec.iov_len = local->total_size;
+            local->op_ret = local->total_size;
+            SHARD_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno,
+                               &vec, 1, &local->prebuf, local->iobref,
+                               local->xattr_rsp);
+            return 0;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-shard_readv_do (call_frame_t *frame, xlator_t *this)
-{
-        int                i                    = 0;
-        int                ret                  = 0;
-        int                call_count           = 0;
-        int                last_block           = 0;
-        int                cur_block            = 0;
-        off_t              orig_offset          = 0;
-        off_t              shard_offset         = 0;
-        size_t             read_size            = 0;
-        size_t             remaining_size       = 0;
-        fd_t              *fd                   = NULL;
-        fd_t              *anon_fd              = NULL;
-        shard_local_t     *local                = NULL;
-        gf_boolean_t       wind_failed          = _gf_false;
-
-        local = frame->local;
-        fd = local->fd;
+shard_readv_do(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int call_count = 0;
+    int last_block = 0;
+    int cur_block = 0;
+    off_t orig_offset = 0;
+    off_t shard_offset = 0;
+    size_t read_size = 0;
+    size_t remaining_size = 0;
+    fd_t *fd = NULL;
+    fd_t *anon_fd = NULL;
+    shard_local_t *local = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+
+    local = frame->local;
+    fd = local->fd;
+
+    orig_offset = local->offset;
+    cur_block = local->first_block;
+    last_block = local->last_block;
+    remaining_size = local->total_size;
+    local->call_count = call_count = local->num_blocks;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    if (fd->flags & O_DIRECT)
+        local->flags = O_DIRECT;
+
+    while (cur_block <= last_block) {
+        if (wind_failed) {
+            shard_readv_do_cbk(frame, (void *)(long)0, this, -1, ENOMEM, NULL,
+                               0, NULL, NULL, NULL);
+            goto next;
+        }
+
+        shard_offset = orig_offset % local->block_size;
+        read_size = local->block_size - shard_offset;
+        if (read_size > remaining_size)
+            read_size = remaining_size;
+
+        remaining_size -= read_size;
+
+        if (cur_block == 0) {
+            anon_fd = fd_ref(fd);
+        } else {
+            anon_fd = fd_anonymous(local->inode_list[i]);
+            if (!anon_fd) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                wind_failed = _gf_true;
+                shard_readv_do_cbk(frame, (void *)(long)anon_fd, this, -1,
+                                   ENOMEM, NULL, 0, NULL, NULL, NULL);
+                goto next;
+            }
+        }
 
-        orig_offset = local->offset;
-        cur_block = local->first_block;
-        last_block = local->last_block;
-        remaining_size = local->total_size;
-        local->call_count = call_count = local->num_blocks;
+        STACK_WIND_COOKIE(frame, shard_readv_do_cbk, anon_fd, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->readv, anon_fd, read_size,
+                          shard_offset, local->flags, local->xattr_req);
 
-        SHARD_SET_ROOT_FS_ID (frame, local);
+        orig_offset += read_size;
+    next:
+        cur_block++;
+        i++;
+        call_count--;
+    }
+    return 0;
+}
 
-        if (fd->flags & O_DIRECT)
-                local->flags = O_DIRECT;
+int
+shard_common_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
+{
+    int shard_block_num = (long)cookie;
+    int call_count = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        if (op_errno == EEXIST) {
+            LOCK(&frame->lock);
+            {
+                local->eexist_count++;
+            }
+            UNLOCK(&frame->lock);
+        } else {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+        }
+        gf_msg_debug(this->name, 0,
+                     "mknod of shard %d "
+                     "failed: %s",
+                     shard_block_num, strerror(op_errno));
+        goto done;
+    }
 
-        while (cur_block <= last_block) {
-                if (wind_failed) {
-                        shard_readv_do_cbk (frame, (void *) (long) 0,  this, -1,
-                                            ENOMEM, NULL, 0, NULL, NULL, NULL);
-                        goto next;
-                }
+    shard_link_block_inode(local, shard_block_num, inode, buf);
 
-                shard_offset = orig_offset % local->block_size;
-                read_size = local->block_size - shard_offset;
-                if (read_size > remaining_size)
-                        read_size = remaining_size;
-
-                remaining_size -= read_size;
-
-                if (cur_block == 0) {
-                        anon_fd = fd_ref (fd);
-                } else {
-                        anon_fd = fd_anonymous (local->inode_list[i]);
-                        if (!anon_fd) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                wind_failed = _gf_true;
-                                shard_readv_do_cbk (frame,
-                                                    (void *) (long) anon_fd,
-                                                    this, -1, ENOMEM, NULL, 0,
-                                                    NULL, NULL, NULL);
-                                goto next;
-                        }
-                }
-
-                ret = fd_ctx_set (anon_fd, this, cur_block);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                SHARD_MSG_FD_CTX_SET_FAILED,
-                                "Failed to set fd ctx for block %d,  gfid=%s",
-                                cur_block,
-                                uuid_utoa (local->inode_list[i]->gfid));
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        shard_readv_do_cbk (frame, (void *) (long) anon_fd,
-                                            this, -1, ENOMEM, NULL, 0, NULL,
-                                            NULL, NULL);
-                        goto next;
-                }
+done:
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        local->create_count = 0;
+        local->post_mknod_handler(frame, this);
+    }
 
-                STACK_WIND_COOKIE (frame, shard_readv_do_cbk, anon_fd,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->readv, anon_fd,
-                                   read_size, shard_offset, local->flags,
-                                   local->xattr_req);
-
-                orig_offset += read_size;
-next:
-                cur_block++;
-                i++;
-                call_count--;
-        }
-        return 0;
+    return 0;
 }
 
 int
-shard_post_lookup_shards_readv_handler (call_frame_t *frame, xlator_t *this)
-{
-        shard_local_t *local = NULL;
+shard_common_resume_mknod(call_frame_t *frame, xlator_t *this,
+                          shard_post_mknod_fop_handler_t post_mknod_handler)
+{
+    int i = 0;
+    int shard_idx_iter = 0;
+    int last_block = 0;
+    int ret = 0;
+    int call_count = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    mode_t mode = 0;
+    char *bname = NULL;
+    shard_priv_t *priv = NULL;
+    shard_inode_ctx_t ctx_tmp = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+
+    local = frame->local;
+    priv = this->private;
+    fd = local->fd;
+    shard_idx_iter = local->first_block;
+    last_block = local->last_block;
+    call_count = local->call_count = local->create_count;
+    local->post_mknod_handler = post_mknod_handler;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    ret = shard_inode_ctx_get_all(fd->inode, this, &ctx_tmp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get inode "
+               "ctx for %s",
+               uuid_utoa(fd->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    mode = st_mode_from_ia(ctx_tmp.stat.ia_prot, ctx_tmp.stat.ia_type);
 
-        local = frame->local;
+    while (shard_idx_iter <= last_block) {
+        if (local->inode_list[i]) {
+            shard_idx_iter++;
+            i++;
+            continue;
+        }
 
-        if (local->op_ret < 0) {
-                SHARD_STACK_UNWIND (readv, frame, local->op_ret,
-                                    local->op_errno, NULL, 0, NULL, NULL, NULL);
-                return 0;
+        if (wind_failed) {
+            shard_common_mknod_cbk(frame, (void *)(long)shard_idx_iter, this,
+                                   -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+            goto next;
         }
 
-        shard_readv_do (frame, this);
+        shard_make_block_abspath(shard_idx_iter, fd->inode->gfid, path,
+                                 sizeof(path));
 
-        return 0;
+        xattr_req = shard_create_gfid_dict(local->xattr_req);
+        if (!xattr_req) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            wind_failed = _gf_true;
+            shard_common_mknod_cbk(frame, (void *)(long)shard_idx_iter, this,
+                                   -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+            goto next;
+        }
+
+        bname = strrchr(path, '/') + 1;
+        loc.inode = inode_new(this->itable);
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0 || !(loc.inode)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   "on %s, base file gfid = %s",
+                   bname, uuid_utoa(fd->inode->gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            wind_failed = _gf_true;
+            loc_wipe(&loc);
+            dict_unref(xattr_req);
+            shard_common_mknod_cbk(frame, (void *)(long)shard_idx_iter, this,
+                                   -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+            goto next;
+        }
+
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+
+        STACK_WIND_COOKIE(frame, shard_common_mknod_cbk,
+                          (void *)(long)shard_idx_iter, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->mknod, &loc, mode,
+                          ctx_tmp.stat.ia_rdev, 0, xattr_req);
+        loc_wipe(&loc);
+        dict_unref(xattr_req);
+
+    next:
+        shard_idx_iter++;
+        i++;
+        if (!--call_count)
+            break;
+    }
+
+    return 0;
+err:
+    /*
+     * This block is for handling failure in shard_inode_ctx_get_all().
+     * Failures in the while-loop are handled within the loop.
+     */
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+    post_mknod_handler(frame, this);
+    return 0;
 }
 
 int
-shard_post_mknod_readv_handler (call_frame_t *frame, xlator_t *this)
-{
-        shard_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (local->op_ret < 0) {
-                SHARD_STACK_UNWIND (readv, frame, local->op_ret,
-                                    local->op_errno, NULL, 0, NULL, NULL, NULL);
-                return 0;
-        }
-
-        if (!local->eexist_count) {
-                shard_readv_do (frame, this);
-        } else {
-                local->call_count = local->eexist_count;
-                shard_common_lookup_shards (frame, this, local->loc.inode,
-                                        shard_post_lookup_shards_readv_handler);
-        }
-        return 0;
-}
+shard_post_mknod_readv_handler(call_frame_t *frame, xlator_t *this);
 
 int
-shard_common_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+shard_post_lookup_shards_readv_handler(call_frame_t *frame, xlator_t *this)
 {
-        int             shard_block_num  = (long) cookie;
-        int             call_count       = 0;
-        shard_local_t  *local            = NULL;
-
-        local = frame->local;
+    shard_local_t *local = NULL;
 
-        if (op_ret < 0) {
-                if (op_errno == EEXIST) {
-                        local->eexist_count++;
-                } else {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
-                gf_msg_debug (this->name, 0, "mknod of shard %d "
-                        "failed: %s", shard_block_num, strerror (op_errno));
-                goto done;
-        }
+    local = frame->local;
 
-        shard_link_block_inode (local, shard_block_num, inode, buf);
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-done:
-        call_count = shard_call_count_return (frame);
-        if (call_count == 0) {
-                SHARD_UNSET_ROOT_FS_ID (frame, local);
-                local->post_mknod_handler (frame, this);
-        }
+    if (local->create_count) {
+        shard_common_resume_mknod(frame, this, shard_post_mknod_readv_handler);
+    } else {
+        shard_readv_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-shard_common_resume_mknod (call_frame_t *frame, xlator_t *this,
-                           shard_post_mknod_fop_handler_t post_mknod_handler)
-{
-        int                 i              = 0;
-        int                 shard_idx_iter = 0;
-        int                 last_block     = 0;
-        int                 ret            = 0;
-        int                 call_count     = 0;
-        char                path[PATH_MAX] = {0,};
-        mode_t              mode           = 0;
-        char               *bname          = NULL;
-        shard_priv_t       *priv           = NULL;
-        shard_inode_ctx_t   ctx_tmp        = {0,};
-        shard_local_t      *local          = NULL;
-        gf_boolean_t        wind_failed    = _gf_false;
-        fd_t               *fd             = NULL;
-        loc_t               loc            = {0,};
-        dict_t             *xattr_req      = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        fd = local->fd;
-        shard_idx_iter = local->first_block;
-        last_block = local->last_block;
-        call_count = local->call_count = local->create_count;
-        local->post_mknod_handler = post_mknod_handler;
-
-        SHARD_SET_ROOT_FS_ID (frame, local);
-
-        ret = shard_inode_ctx_get_all (fd->inode, this, &ctx_tmp);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get inode "
-                        "ctx for %s", uuid_utoa (fd->inode->gfid));
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
-        }
-        mode = st_mode_from_ia (ctx_tmp.stat.ia_prot, ctx_tmp.stat.ia_type);
-
-        while (shard_idx_iter <= last_block) {
-                if (local->inode_list[i]) {
-                        shard_idx_iter++;
-                        i++;
-                        continue;
-                }
-
-                if (wind_failed) {
-                        shard_common_mknod_cbk (frame,
-                                                (void *) (long) shard_idx_iter,
-                                                this, -1, ENOMEM, NULL, NULL,
-                                                NULL, NULL, NULL);
-                        goto next;
-                }
-
-                shard_make_block_abspath (shard_idx_iter, fd->inode->gfid,
-                                          path, sizeof(path));
-
-                xattr_req = shard_create_gfid_dict (local->xattr_req);
-                if (!xattr_req) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        shard_common_mknod_cbk (frame,
-                                                (void *) (long) shard_idx_iter,
-                                                this, -1, ENOMEM, NULL, NULL,
-                                                NULL, NULL, NULL);
-                        goto next;
-                }
+shard_post_mknod_readv_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
 
-                bname = strrchr (path, '/') + 1;
-                loc.inode = inode_new (this->itable);
-                loc.parent = inode_ref (priv->dot_shard_inode);
-                ret = inode_path (loc.parent, bname,
-                                       (char **) &(loc.path));
-                if (ret < 0 || !(loc.inode)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                SHARD_MSG_INODE_PATH_FAILED, "Inode path failed"
-                                "on %s, base file gfid = %s", bname,
-                                uuid_utoa (fd->inode->gfid));
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        loc_wipe (&loc);
-                        dict_unref (xattr_req);
-                        shard_common_mknod_cbk (frame,
-                                                (void *) (long) shard_idx_iter,
-                                                this, -1, ENOMEM, NULL, NULL,
-                                                NULL, NULL, NULL);
-                        goto next;
-                }
-
-                loc.name = strrchr (loc.path, '/');
-                if (loc.name)
-                        loc.name++;
-
-                STACK_WIND_COOKIE (frame, shard_common_mknod_cbk,
-                                   (void *) (long) shard_idx_iter,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mknod, &loc,
-                                   mode, ctx_tmp.stat.ia_rdev, 0, xattr_req);
-                loc_wipe (&loc);
-                dict_unref (xattr_req);
-
-next:
-                shard_idx_iter++;
-                i++;
-                if (!--call_count)
-                        break;
-        }
+    local = frame->local;
 
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
-err:
-        /*
-         * This block is for handling failure in shard_inode_ctx_get_all().
-         * Failures in the while-loop are handled within the loop.
-         */
-        SHARD_UNSET_ROOT_FS_ID (frame, local);
-        post_mknod_handler (frame, this);
-        return 0;
+    }
+
+    if (!local->eexist_count) {
+        shard_readv_do(frame, this);
+    } else {
+        local->call_count = local->eexist_count;
+        shard_common_lookup_shards(frame, this, local->loc.inode,
+                                   shard_post_lookup_shards_readv_handler);
+    }
+    return 0;
 }
 
 int
-shard_post_resolve_readv_handler (call_frame_t *frame, xlator_t *this)
+shard_post_resolve_readv_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                if (local->op_errno != ENOENT) {
-                        SHARD_STACK_UNWIND (readv, frame, local->op_ret,
-                                            local->op_errno, NULL, 0, NULL,
-                                            NULL, NULL);
-                        return 0;
-                } else {
-                        struct iovec vec = {0,};
-
-                        vec.iov_base = local->iobuf->ptr;
-                        vec.iov_len = local->total_size;
-                        SHARD_STACK_UNWIND (readv, frame, local->total_size,
-                                            0, &vec, 1, &local->prebuf,
-                                            local->iobref, NULL);
-                        return 0;
-                }
-        }
-
-        if (local->call_count) {
-                local->create_count = local->call_count;
-                shard_common_resume_mknod (frame, this,
-                                           shard_post_mknod_readv_handler);
+    if (local->op_ret < 0) {
+        if (local->op_errno != ENOENT) {
+            shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                        local->op_errno);
+            return 0;
         } else {
-                shard_readv_do (frame, this);
+            struct iovec vec = {
+                0,
+            };
+
+            vec.iov_base = local->iobuf->ptr;
+            vec.iov_len = local->total_size;
+            local->op_ret = local->total_size;
+            SHARD_STACK_UNWIND(readv, frame, local->op_ret, 0, &vec, 1,
+                               &local->prebuf, local->iobref, NULL);
+            return 0;
         }
+    }
 
-        return 0;
+    if (local->call_count) {
+        shard_common_lookup_shards(frame, this, local->resolver_base_inode,
+                                   shard_post_lookup_shards_readv_handler);
+    } else {
+        shard_readv_do(frame, this);
+    }
+
+    return 0;
 }
 
 int
-shard_post_lookup_readv_handler (call_frame_t *frame, xlator_t *this)
+shard_post_lookup_readv_handler(call_frame_t *frame, xlator_t *this)
 {
-        int                ret         = 0;
-        struct iobuf      *iobuf       = NULL;
-        shard_local_t     *local       = NULL;
-        shard_priv_t      *priv        = NULL;
-
-        priv = this->private;
-        local = frame->local;
+    int ret = 0;
+    struct iobuf *iobuf = NULL;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
 
-        if (local->op_ret < 0) {
-                SHARD_STACK_UNWIND (readv, frame, local->op_ret,
-                                    local->op_errno, NULL, 0, NULL, NULL, NULL);
-                return 0;
-        }
+    priv = this->private;
+    local = frame->local;
 
-        if (local->offset >= local->prebuf.ia_size) {
-                /* If the read is being performed past the end of the file,
-                 * unwind the FOP with 0 bytes read as status.
-                 */
-                struct iovec      vec        = {0,};
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, local->req_size);
-                if (!iobuf)
-                        goto err;
+    if (local->offset >= local->prebuf.ia_size) {
+        /* If the read is being performed past the end of the file,
+         * unwind the FOP with 0 bytes read as status.
+         */
+        struct iovec vec = {
+            0,
+        };
 
-                vec.iov_base = iobuf->ptr;
-                vec.iov_len = 0;
-                local->iobref = iobref_new ();
-                iobref_add (local->iobref, iobuf);
-                iobuf_unref (iobuf);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, local->req_size);
+        if (!iobuf)
+            goto err;
 
-                SHARD_STACK_UNWIND (readv, frame, 0, 0, &vec, 1, &local->prebuf,
-                                    local->iobref, NULL);
-                return 0;
-        }
+        vec.iov_base = iobuf->ptr;
+        vec.iov_len = 0;
+        local->iobref = iobref_new();
+        iobref_add(local->iobref, iobuf);
+        iobuf_unref(iobuf);
 
-        local->first_block = get_lowest_block (local->offset,
-                                               local->block_size);
+        SHARD_STACK_UNWIND(readv, frame, 0, 0, &vec, 1, &local->prebuf,
+                           local->iobref, NULL);
+        return 0;
+    }
 
-        local->total_size = local->req_size;
+    local->first_block = get_lowest_block(local->offset, local->block_size);
 
-        local->last_block = get_highest_block (local->offset, local->total_size,
-                                               local->block_size);
+    local->total_size = local->req_size;
 
-        local->num_blocks = local->last_block - local->first_block + 1;
+    local->last_block = get_highest_block(local->offset, local->total_size,
+                                          local->block_size);
 
-        local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
-                                       gf_shard_mt_inode_list);
-        if (!local->inode_list)
-                goto err;
+    local->num_blocks = local->last_block - local->first_block + 1;
+    GF_ASSERT(local->num_blocks > 0);
+    local->resolver_base_inode = local->loc.inode;
 
-        iobuf = iobuf_get2 (this->ctx->iobuf_pool, local->total_size);
-        if (!iobuf)
-                goto err;
+    local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list)
+        goto err;
 
-        local->iobref = iobref_new ();
-        if (!local->iobref) {
-                iobuf_unref (iobuf);
-                goto err;
-        }
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, local->total_size);
+    if (!iobuf)
+        goto err;
 
-        if (iobref_add (local->iobref, iobuf) != 0) {
-                iobuf_unref (iobuf);
-                goto err;
-        }
+    local->iobref = iobref_new();
+    if (!local->iobref) {
+        iobuf_unref(iobuf);
+        goto err;
+    }
 
-        iobuf_unref (iobuf);
-        local->iobuf = iobuf;
-        memset (iobuf->ptr, 0, local->total_size);
+    if (iobref_add(local->iobref, iobuf) != 0) {
+        iobuf_unref(iobuf);
+        goto err;
+    }
 
-        local->dot_shard_loc.inode = inode_find (this->itable,
-                                                 priv->dot_shard_gfid);
-        if (!local->dot_shard_loc.inode) {
-                ret = shard_init_dot_shard_loc (this, local);
-                if (ret)
-                        goto err;
-                shard_lookup_dot_shard (frame, this,
-                                        shard_post_resolve_readv_handler);
-        } else {
-                shard_common_resolve_shards (frame, this, local->loc.inode,
-                                             shard_post_resolve_readv_handler);
-        }
-        return 0;
+    memset(iobuf->ptr, 0, local->total_size);
+    iobuf_unref(iobuf);
+    local->iobuf = iobuf;
 
+    local->dot_shard_loc.inode = inode_find(this->itable, priv->dot_shard_gfid);
+    if (!local->dot_shard_loc.inode) {
+        ret = shard_init_internal_dir_loc(this, local,
+                                          SHARD_INTERNAL_DIR_DOT_SHARD);
+        if (ret)
+            goto err;
+        shard_lookup_internal_dir(frame, this, shard_post_resolve_readv_handler,
+                                  SHARD_INTERNAL_DIR_DOT_SHARD);
+    } else {
+        local->post_res_handler = shard_post_resolve_readv_handler;
+        shard_refresh_internal_dir(frame, this, SHARD_INTERNAL_DIR_DOT_SHARD);
+    }
+    return 0;
 err:
-        SHARD_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
-                            NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_READ, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, uint32_t flags, dict_t *xdata)
-{
-        int             ret                     = 0;
-        uint64_t        block_size              = 0;
-        shard_local_t  *local                   = NULL;
-
-        ret = shard_inode_ctx_get_block_size (fd->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size for %s from its inode ctx",
-                        uuid_utoa (fd->inode->gfid));
-                goto err;
-        }
-
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                /* block_size = 0 means that the file was created before
-                 * sharding was enabled on the volume.
-                 */
-                STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readv, fd, size, offset,
-                            flags, xdata);
-                return 0;
-        }
-
-        if (!this->itable)
-                this->itable = fd->inode->table;
-
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
-
-        frame->local = local;
-
-        local->fd = fd_ref (fd);
-        local->block_size = block_size;
-        local->offset = offset;
-        local->req_size = size;
-        local->flags = flags;
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
-
-        local->loc.inode = inode_ref (fd->inode);
-        gf_uuid_copy (local->loc.gfid, fd->inode->gfid);
-
-        shard_lookup_base_file (frame, this, &local->loc,
-                                shard_post_lookup_readv_handler);
-
-        return 0;
-
+shard_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size for %s from its inode ctx",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        /* block_size = 0 means that the file was created before
+         * sharding was enabled on the volume.
+         */
+        STACK_WIND(frame, default_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+        return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto err;
+    local->fd = fd_ref(fd);
+    local->block_size = block_size;
+    local->offset = offset;
+    local->req_size = size;
+    local->flags = flags;
+    local->fop = GF_FOP_READ;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_post_lookup_readv_handler);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
-                            NULL);
-        return 0;
-
+    shard_common_failure_unwind(GF_FOP_READ, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_common_inode_write_post_update_size_handler (call_frame_t *frame,
-                                                   xlator_t *this)
+shard_common_inode_write_post_update_size_handler(call_frame_t *frame,
+                                                  xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                shard_common_inode_write_failure_unwind (local->fop, frame,
-                                                         local->op_ret,
-                                                         local->op_errno);
-        } else {
-                shard_common_inode_write_success_unwind (local->fop, frame,
-                                                         local->written_size);
-        }
-        return 0;
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+    } else {
+        shard_common_inode_write_success_unwind(local->fop, frame,
+                                                local->written_size);
+    }
+    return 0;
+}
+
+static gf_boolean_t
+shard_is_appending_write(shard_local_t *local)
+{
+    if (local->fop != GF_FOP_WRITE)
+        return _gf_false;
+    if (local->flags & O_APPEND)
+        return _gf_true;
+    if (local->fd->flags & O_APPEND)
+        return _gf_true;
+    return _gf_false;
 }
 
 int
-__shard_get_delta_size_from_inode_ctx (shard_local_t *local, inode_t *inode,
-                                       xlator_t *this)
+__shard_get_delta_size_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                      xlator_t *this)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret < 0)
-                return ret;
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
 
-        ctx = (shard_inode_ctx_t *) ctx_uint;
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        if (local->offset + local->total_size > ctx->stat.ia_size) {
-                local->delta_size = (local->offset + local->total_size) -
-                                    ctx->stat.ia_size;
-                ctx->stat.ia_size += (local->delta_size);
-        } else {
-                local->delta_size = 0;
-        }
-        local->postbuf = ctx->stat;
+    if (shard_is_appending_write(local)) {
+        local->delta_size = local->total_size;
+    } else if (local->offset + local->total_size > ctx->stat.ia_size) {
+        local->delta_size = (local->offset + local->total_size) -
+                            ctx->stat.ia_size;
+    } else {
+        local->delta_size = 0;
+    }
+    ctx->stat.ia_size += (local->delta_size);
+    local->postbuf = ctx->stat;
 
-        return 0;
+    return 0;
 }
 
 int
-shard_get_delta_size_from_inode_ctx (shard_local_t *local, inode_t *inode,
-                                     xlator_t *this)
+shard_get_delta_size_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                    xlator_t *this)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_get_delta_size_from_inode_ctx (local, inode,
-                                                             this);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_get_delta_size_from_inode_ctx(local, inode, this);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-shard_common_inode_write_do_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, struct iatt *pre,
-                                 struct iatt *post, dict_t *xdata)
+shard_common_inode_write_do_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, struct iatt *pre,
+                                struct iatt *post, dict_t *xdata)
 {
-        int             call_count = 0;
-        fd_t           *anon_fd    = cookie;
-        shard_local_t  *local      = NULL;
-        glusterfs_fop_t fop        = 0;
-
-        local = frame->local;
-        fop = local->fop;
+    int call_count = 0;
+    fd_t *anon_fd = cookie;
+    shard_local_t *local = NULL;
+    glusterfs_fop_t fop = 0;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                } else {
-                        local->written_size += op_ret;
-                        local->delta_blocks += (post->ia_blocks -
-                                                pre->ia_blocks);
-                        local->delta_size += (post->ia_size - pre->ia_size);
-                        shard_inode_ctx_set (local->fd->inode, this, post, 0,
-                                             SHARD_MASK_TIMES);
-                }
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
+    fop = local->fop;
 
-        if (anon_fd)
-                fd_unref (anon_fd);
-
-        call_count = shard_call_count_return (frame);
-        if (call_count == 0) {
-                SHARD_UNSET_ROOT_FS_ID (frame, local);
-                if (local->op_ret < 0) {
-                        shard_common_inode_write_failure_unwind (fop, frame,
-                                                                 local->op_ret,
-                                                               local->op_errno);
-                } else {
-                        shard_get_delta_size_from_inode_ctx (local,
-                                                             local->fd->inode,
-                                                             this);
-                        local->hole_size = 0;
-                        if (xdata)
-                                local->xattr_rsp = dict_ref (xdata);
-                        shard_update_file_size (frame, this, local->fd, NULL,
-                             shard_common_inode_write_post_update_size_handler);
-                }
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+        } else {
+            local->written_size += op_ret;
+            GF_ATOMIC_ADD(local->delta_blocks,
+                          post->ia_blocks - pre->ia_blocks);
+            local->delta_size += (post->ia_size - pre->ia_size);
+            shard_inode_ctx_set(local->fd->inode, this, post, 0,
+                                SHARD_MASK_TIMES);
+            if (local->fd->inode != anon_fd->inode)
+                shard_inode_ctx_add_to_fsync_list(local->fd->inode, this,
+                                                  anon_fd->inode);
+        }
+    }
+    UNLOCK(&frame->lock);
+
+    if (anon_fd)
+        fd_unref(anon_fd);
+
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        if (local->op_ret < 0) {
+            shard_common_failure_unwind(fop, frame, local->op_ret,
+                                        local->op_errno);
+        } else {
+            shard_get_delta_size_from_inode_ctx(local, local->fd->inode, this);
+            local->hole_size = 0;
+            if (xdata)
+                local->xattr_rsp = dict_ref(xdata);
+            shard_update_file_size(
+                frame, this, local->fd, NULL,
+                shard_common_inode_write_post_update_size_handler);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-shard_common_inode_write_wind (call_frame_t *frame, xlator_t *this,
-                               fd_t *fd, struct iovec *vec, int count,
-                               off_t shard_offset, size_t size)
+shard_common_inode_write_wind(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              struct iovec *vec, int count, off_t shard_offset,
+                              size_t size)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        switch (local->fop) {
+    switch (local->fop) {
         case GF_FOP_WRITE:
-                STACK_WIND_COOKIE (frame, shard_common_inode_write_do_cbk, fd,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->writev, fd, vec,
-                                   count, shard_offset, local->flags,
-                                   local->iobref, local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(
+                frame, shard_common_inode_write_do_cbk, fd, FIRST_CHILD(this),
+                FIRST_CHILD(this)->fops->writev, fd, vec, count, shard_offset,
+                local->flags, local->iobref, local->xattr_req);
+            break;
         case GF_FOP_FALLOCATE:
-                STACK_WIND_COOKIE (frame, shard_common_inode_write_do_cbk, fd,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->fallocate, fd,
-                                   local->flags, shard_offset, size,
-                                   local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(
+                frame, shard_common_inode_write_do_cbk, fd, FIRST_CHILD(this),
+                FIRST_CHILD(this)->fops->fallocate, fd, local->flags,
+                shard_offset, size, local->xattr_req);
+            break;
         case GF_FOP_ZEROFILL:
-                STACK_WIND_COOKIE (frame, shard_common_inode_write_do_cbk, fd,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->zerofill, fd,
-                                   shard_offset, size, local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, shard_common_inode_write_do_cbk, fd,
+                              FIRST_CHILD(this),
+                              FIRST_CHILD(this)->fops->zerofill, fd,
+                              shard_offset, size, local->xattr_req);
+            break;
         case GF_FOP_DISCARD:
-                STACK_WIND_COOKIE (frame, shard_common_inode_write_do_cbk, fd,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->discard, fd,
-                                   shard_offset, size, local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, shard_common_inode_write_do_cbk, fd,
+                              FIRST_CHILD(this),
+                              FIRST_CHILD(this)->fops->discard, fd,
+                              shard_offset, size, local->xattr_req);
+            break;
         default:
-                gf_msg (this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
-                        "Invalid fop id = %d", local->fop);
-                break;
-        }
-        return 0;
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "Invalid fop id = %d", local->fop);
+            break;
+    }
+    return 0;
 }
 
 int
-shard_common_inode_write_do (call_frame_t *frame, xlator_t *this)
-{
-        int             i                 = 0;
-        int             count             = 0;
-        int             call_count        = 0;
-        int             last_block        = 0;
-        uint32_t        cur_block         = 0;
-        fd_t           *fd                = NULL;
-        fd_t           *anon_fd           = NULL;
-        shard_local_t  *local             = NULL;
-        struct iovec   *vec               = NULL;
-        gf_boolean_t    wind_failed       = _gf_false;
-        gf_boolean_t    odirect           = _gf_false;
-        off_t           orig_offset       = 0;
-        off_t           shard_offset      = 0;
-        off_t           vec_offset        = 0;
-        size_t          remaining_size    = 0;
-        size_t          shard_write_size  = 0;
-
-        local = frame->local;
-        fd = local->fd;
-
-        orig_offset = local->offset;
-        remaining_size = local->total_size;
-        cur_block = local->first_block;
-        local->call_count = call_count = local->num_blocks;
-        last_block = local->last_block;
-
-        SHARD_SET_ROOT_FS_ID (frame, local);
-
-        if (dict_set_uint32 (local->xattr_req,
-                             GLUSTERFS_WRITE_UPDATE_ATOMIC, 4)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_SET_FAILED,
-                        "Failed to set "GLUSTERFS_WRITE_UPDATE_ATOMIC" into "
-                        "dict: %s", uuid_utoa (fd->inode->gfid));
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                local->call_count = 1;
-                shard_common_inode_write_do_cbk (frame, (void *)(long)0, this,
-                                                 -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
+shard_common_inode_write_do(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int count = 0;
+    int call_count = 0;
+    int last_block = 0;
+    uint32_t cur_block = 0;
+    fd_t *fd = NULL;
+    fd_t *anon_fd = NULL;
+    shard_local_t *local = NULL;
+    struct iovec *vec = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+    gf_boolean_t odirect = _gf_false;
+    off_t orig_offset = 0;
+    off_t shard_offset = 0;
+    off_t vec_offset = 0;
+    size_t remaining_size = 0;
+    size_t shard_write_size = 0;
+
+    local = frame->local;
+    fd = local->fd;
+
+    orig_offset = local->offset;
+    remaining_size = local->total_size;
+    cur_block = local->first_block;
+    local->call_count = call_count = local->num_blocks;
+    last_block = local->last_block;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    if (dict_set_uint32(local->xattr_req, GLUSTERFS_WRITE_UPDATE_ATOMIC, 4)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set " GLUSTERFS_WRITE_UPDATE_ATOMIC
+               " into "
+               "dict: %s",
+               uuid_utoa(fd->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        local->call_count = 1;
+        shard_common_inode_write_do_cbk(frame, (void *)(long)0, this, -1,
+                                        ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
+
+    if ((fd->flags & O_DIRECT) && (local->fop == GF_FOP_WRITE))
+        odirect = _gf_true;
+
+    while (cur_block <= last_block) {
+        if (wind_failed) {
+            shard_common_inode_write_do_cbk(frame, (void *)(long)0, this, -1,
+                                            ENOMEM, NULL, NULL, NULL);
+            goto next;
         }
 
-        if ((fd->flags & O_DIRECT) && (local->fop == GF_FOP_WRITE))
-                odirect = _gf_true;
+        shard_offset = orig_offset % local->block_size;
+        shard_write_size = local->block_size - shard_offset;
+        if (shard_write_size > remaining_size)
+            shard_write_size = remaining_size;
 
-        while (cur_block <= last_block) {
-                if (wind_failed) {
-                        shard_common_inode_write_do_cbk (frame,
-                                                         (void *) (long) 0,
-                                                         this, -1, ENOMEM, NULL,
-                                                         NULL, NULL);
-                        goto next;
-                }
+        remaining_size -= shard_write_size;
 
-                shard_offset = orig_offset % local->block_size;
-                shard_write_size = local->block_size - shard_offset;
-                if (shard_write_size > remaining_size)
-                        shard_write_size = remaining_size;
-
-                remaining_size -= shard_write_size;
-
-                if (local->fop == GF_FOP_WRITE) {
-                        count = iov_subset (local->vector, local->count,
-                                            vec_offset,
-                                            vec_offset + shard_write_size,
-                                            NULL);
-
-                        vec = GF_CALLOC (count, sizeof (struct iovec),
-                                         gf_shard_mt_iovec);
-                        if (!vec) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                wind_failed = _gf_true;
-                                GF_FREE (vec);
-                                shard_common_inode_write_do_cbk (frame,
-                                                              (void *) (long) 0,
-                                                                 this, -1,
-                                                                 ENOMEM, NULL,
-                                                                 NULL, NULL);
-                                goto next;
-                        }
-                        count = iov_subset (local->vector, local->count,
-                                            vec_offset,
-                                            vec_offset + shard_write_size, vec);
-                }
+        if (local->fop == GF_FOP_WRITE) {
+            vec = NULL;
+            count = iov_subset(local->vector, local->count, vec_offset,
+                               shard_write_size, &vec, 0);
+            if (count < 0) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                wind_failed = _gf_true;
+                shard_common_inode_write_do_cbk(frame, (void *)(long)0, this,
+                                                -1, ENOMEM, NULL, NULL, NULL);
+                goto next;
+            }
+        }
 
-                if (cur_block == 0) {
-                        anon_fd = fd_ref (fd);
-                } else {
-                        anon_fd = fd_anonymous (local->inode_list[i]);
-                        if (!anon_fd) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                wind_failed = _gf_true;
-                                GF_FREE (vec);
-                                shard_common_inode_write_do_cbk (frame,
-                                                        (void *) (long) anon_fd,
-                                                                 this, -1,
-                                                                 ENOMEM, NULL,
-                                                                 NULL, NULL);
-                                goto next;
-                        }
-
-                        if (local->fop == GF_FOP_WRITE) {
-                                if (odirect)
-                                        local->flags = O_DIRECT;
-                                else
-                                        local->flags = GF_ANON_FD_FLAGS;
-                        }
-                }
+        if (cur_block == 0) {
+            anon_fd = fd_ref(fd);
+        } else {
+            anon_fd = fd_anonymous(local->inode_list[i]);
+            if (!anon_fd) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                wind_failed = _gf_true;
+                GF_FREE(vec);
+                shard_common_inode_write_do_cbk(frame, (void *)(long)anon_fd,
+                                                this, -1, ENOMEM, NULL, NULL,
+                                                NULL);
+                goto next;
+            }
 
-                shard_common_inode_write_wind (frame, this, anon_fd,
-                                               vec, count, shard_offset,
-                                               shard_write_size);
-                if (vec)
-                        vec_offset += shard_write_size;
-                orig_offset += shard_write_size;
-                GF_FREE (vec);
-                vec = NULL;
-next:
-                cur_block++;
-                i++;
-                call_count--;
+            if (local->fop == GF_FOP_WRITE) {
+                if (odirect)
+                    local->flags = O_DIRECT;
+                else
+                    local->flags = GF_ANON_FD_FLAGS;
+            }
         }
-        return 0;
+
+        shard_common_inode_write_wind(frame, this, anon_fd, vec, count,
+                                      shard_offset, shard_write_size);
+        if (vec)
+            vec_offset += shard_write_size;
+        orig_offset += shard_write_size;
+        GF_FREE(vec);
+        vec = NULL;
+    next:
+        cur_block++;
+        i++;
+        call_count--;
+    }
+    return 0;
 }
 
 int
-shard_common_inode_write_post_lookup_shards_handler (call_frame_t *frame,
-                                                     xlator_t *this)
+shard_common_inode_write_post_mknod_handler(call_frame_t *frame,
+                                            xlator_t *this);
+
+int
+shard_common_inode_write_post_lookup_shards_handler(call_frame_t *frame,
+                                                    xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                shard_common_inode_write_failure_unwind (local->fop, frame,
-                                                         local->op_ret,
-                                                         local->op_errno);
-                return 0;
-        }
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-        shard_common_inode_write_do (frame, this);
+    if (local->create_count) {
+        shard_common_resume_mknod(frame, this,
+                                  shard_common_inode_write_post_mknod_handler);
+    } else {
+        shard_common_inode_write_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-shard_common_inode_write_post_mknod_handler (call_frame_t *frame,
-                                             xlator_t *this)
+shard_common_inode_write_post_mknod_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                shard_common_inode_write_failure_unwind (local->fop, frame,
-                                                         local->op_ret,
-                                                         local->op_errno);
-                return 0;
-        }
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-        if (!local->eexist_count) {
-                shard_common_inode_write_do (frame, this);
-        } else {
-                local->call_count = local->eexist_count;
-                shard_common_lookup_shards (frame, this, local->loc.inode,
-                           shard_common_inode_write_post_lookup_shards_handler);
-        }
+    if (!local->eexist_count) {
+        shard_common_inode_write_do(frame, this);
+    } else {
+        local->call_count = local->eexist_count;
+        shard_common_lookup_shards(
+            frame, this, local->loc.inode,
+            shard_common_inode_write_post_lookup_shards_handler);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-shard_common_inode_write_post_lookup_handler (call_frame_t *frame,
+shard_common_inode_write_post_resolve_handler(call_frame_t *frame,
                                               xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
-
-        if (local->op_ret < 0) {
-                shard_common_inode_write_failure_unwind (local->fop, frame,
-                                                         local->op_ret,
-                                                         local->op_errno);
-                return 0;
-        }
+    local = frame->local;
 
-        local->postbuf = local->prebuf;
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-        if (local->create_count)
-                shard_common_resume_mknod (frame, this,
-                                   shard_common_inode_write_post_mknod_handler);
-        else
-                shard_common_inode_write_do (frame, this);
+    if (local->call_count) {
+        shard_common_lookup_shards(
+            frame, this, local->resolver_base_inode,
+            shard_common_inode_write_post_lookup_shards_handler);
+    } else if (local->create_count) {
+        shard_common_inode_write_post_lookup_shards_handler(frame, this);
+    } else {
+        shard_common_inode_write_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-shard_common_inode_write_post_resolve_handler (call_frame_t *frame,
-                                               xlator_t *this)
+shard_common_inode_write_post_lookup_handler(call_frame_t *frame,
+                                             xlator_t *this)
 {
-        shard_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (local->op_ret < 0) {
-                shard_common_inode_write_failure_unwind (local->fop, frame,
-                                                         local->op_ret,
-                                                         local->op_errno);
-                return 0;
-        }
-
-        local->create_count = local->call_count;
-
-        shard_lookup_base_file (frame, this, &local->loc,
-                                shard_common_inode_write_post_lookup_handler);
-        return 0;
+    shard_local_t *local = frame->local;
+    shard_priv_t *priv = this->private;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
+
+    local->postbuf = local->prebuf;
+
+    /*Adjust offset to EOF so that correct shard is chosen for append*/
+    if (shard_is_appending_write(local))
+        local->offset = local->prebuf.ia_size;
+
+    local->first_block = get_lowest_block(local->offset, local->block_size);
+    local->last_block = get_highest_block(local->offset, local->total_size,
+                                          local->block_size);
+    local->num_blocks = local->last_block - local->first_block + 1;
+    GF_ASSERT(local->num_blocks > 0);
+    local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list) {
+        shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+        return 0;
+    }
+
+    gf_msg_trace(this->name, 0,
+                 "%s: gfid=%s first_block=%" PRIu64
+                 " "
+                 "last_block=%" PRIu64 " num_blocks=%" PRIu64 " offset=%" PRId64
+                 " total_size=%zu flags=%" PRId32 "",
+                 gf_fop_list[local->fop],
+                 uuid_utoa(local->resolver_base_inode->gfid),
+                 local->first_block, local->last_block, local->num_blocks,
+                 local->offset, local->total_size, local->flags);
+
+    local->dot_shard_loc.inode = inode_find(this->itable, priv->dot_shard_gfid);
+
+    if (!local->dot_shard_loc.inode) {
+        /*change handler*/
+        shard_mkdir_internal_dir(frame, this,
+                                 shard_common_inode_write_post_resolve_handler,
+                                 SHARD_INTERNAL_DIR_DOT_SHARD);
+    } else {
+        /*change handler*/
+        local->post_res_handler = shard_common_inode_write_post_resolve_handler;
+        shard_refresh_internal_dir(frame, this, SHARD_INTERNAL_DIR_DOT_SHARD);
+    }
+    return 0;
 }
 
 int
-shard_mkdir_dot_shard_cbk (call_frame_t *frame, void *cookie,
-                                       xlator_t *this, int32_t op_ret,
-                                       int32_t op_errno, inode_t *inode,
-                                       struct iatt *buf, struct iatt *preparent,
-                                       struct iatt *postparent, dict_t *xdata)
+shard_mkdir_internal_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, inode_t *inode,
+                             struct iatt *buf, struct iatt *preparent,
+                             struct iatt *postparent, dict_t *xdata)
 {
-        shard_local_t *local = NULL;
-
-        local = frame->local;
-
-        SHARD_UNSET_ROOT_FS_ID (frame, local);
+    inode_t *link_inode = NULL;
+    shard_local_t *local = NULL;
+    shard_internal_dir_type_t type = (shard_internal_dir_type_t)cookie;
 
-        if (op_ret == -1) {
-                if (op_errno != EEXIST) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                        goto unwind;
-                } else {
-                        gf_msg_debug (this->name, 0, "mkdir on /.shard failed "
-                                      "with EEXIST. Attempting lookup now");
-                        shard_lookup_dot_shard (frame, this,
-                                                local->post_res_handler);
-                        return 0;
-                }
-        }
+    local = frame->local;
 
-        shard_link_dot_shard_inode (local, inode, buf);
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
 
+    if (op_ret == -1) {
+        if (op_errno != EEXIST) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            goto unwind;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "mkdir on %s failed "
+                         "with EEXIST. Attempting lookup now",
+                         shard_internal_dir_string(type));
+            shard_lookup_internal_dir(frame, this, local->post_res_handler,
+                                      type);
+            return 0;
+        }
+    }
+
+    link_inode = shard_link_internal_dir_inode(local, inode, buf, type);
+    if (link_inode != inode) {
+        shard_refresh_internal_dir(frame, this, type);
+    } else {
+        shard_inode_ctx_mark_dir_refreshed(link_inode, this);
+        shard_common_resolve_shards(frame, this, local->post_res_handler);
+    }
+    return 0;
 unwind:
-        shard_common_resolve_shards (frame, this, local->loc.inode,
-                                     local->post_res_handler);
-        return 0;
+    shard_common_resolve_shards(frame, this, local->post_res_handler);
+    return 0;
 }
 
 int
-shard_mkdir_dot_shard (call_frame_t *frame, xlator_t *this,
-                       shard_post_resolve_fop_handler_t handler)
-{
-        int             ret           = -1;
-        shard_local_t  *local         = NULL;
-        shard_priv_t   *priv          = NULL;
-        dict_t         *xattr_req     = NULL;
+shard_mkdir_internal_dir(call_frame_t *frame, xlator_t *this,
+                         shard_post_resolve_fop_handler_t handler,
+                         shard_internal_dir_type_t type)
+{
+    int ret = -1;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+    dict_t *xattr_req = NULL;
+    uuid_t *gfid = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t free_gfid = _gf_true;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->post_res_handler = handler;
+    gfid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!gfid)
+        goto err;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            gf_uuid_copy(*gfid, priv->dot_shard_gfid);
+            loc = &local->dot_shard_loc;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            gf_uuid_copy(*gfid, priv->dot_shard_rm_gfid);
+            loc = &local->dot_shard_rm_loc;
+            break;
+        default:
+            bzero(*gfid, sizeof(uuid_t));
+            break;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto err;
+
+    ret = shard_init_internal_dir_loc(this, local, type);
+    if (ret)
+        goto err;
+
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", *gfid, false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set gfid-req for %s",
+               shard_internal_dir_string(type));
+        goto err;
+    } else {
+        free_gfid = _gf_false;
+    }
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    STACK_WIND_COOKIE(frame, shard_mkdir_internal_dir_cbk, (void *)(long)type,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc,
+                      0755, 0, xattr_req);
+    dict_unref(xattr_req);
+    return 0;
 
-        local = frame->local;
-        priv = this->private;
+err:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    local->op_ret = -1;
+    local->op_errno = ENOMEM;
+    if (free_gfid)
+        GF_FREE(gfid);
+    handler(frame, this);
+    return 0;
+}
 
-        local->post_res_handler = handler;
+int
+shard_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    /* To-Do: Wind flush on all shards of the file */
+    SHARD_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto err;
+int
+shard_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    STACK_WIND(frame, shard_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
+}
 
-        ret = shard_init_dot_shard_loc (this, local);
-        if (ret)
-                goto err;
+int
+__shard_get_timestamps_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                      xlator_t *this)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-        ret = dict_set_static_bin (xattr_req, "gfid-req", priv->dot_shard_gfid,
-                                   16);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_SET_FAILED,
-                        "Failed to set gfid-req for /.shard");
-                goto err;
-        }
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
 
-        SHARD_SET_ROOT_FS_ID (frame, local);
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        STACK_WIND (frame, shard_mkdir_dot_shard_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                    &local->dot_shard_loc, 0755, 0, xattr_req);
-        dict_unref (xattr_req);
-        return 0;
+    local->postbuf.ia_ctime = ctx->stat.ia_ctime;
+    local->postbuf.ia_ctime_nsec = ctx->stat.ia_ctime_nsec;
+    local->postbuf.ia_atime = ctx->stat.ia_atime;
+    local->postbuf.ia_atime_nsec = ctx->stat.ia_atime_nsec;
+    local->postbuf.ia_mtime = ctx->stat.ia_mtime;
+    local->postbuf.ia_mtime_nsec = ctx->stat.ia_mtime_nsec;
 
-err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        local->op_ret = -1;
-        local->op_errno = ENOMEM;
-        handler (frame, this);
-        return 0;
+    return 0;
 }
 
 int
-shard_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+shard_get_timestamps_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                    xlator_t *this)
 {
-        /* To-Do: Wind flush on all shards of the file */
-        SHARD_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __shard_get_timestamps_from_inode_ctx(local, inode, this);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
 int
-shard_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-        STACK_WIND (frame, shard_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd, xdata);
-        return 0;
+shard_fsync_shards_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    int call_count = 0;
+    uint64_t fsync_count = 0;
+    fd_t *anon_fd = cookie;
+    shard_local_t *local = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *base_ictx = NULL;
+    inode_t *base_inode = NULL;
+    gf_boolean_t unref_shard_inode = _gf_false;
+
+    local = frame->local;
+    base_inode = local->fd->inode;
+
+    if (local->op_ret < 0)
+        goto out;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            goto out;
+        }
+        shard_inode_ctx_set(local->fd->inode, this, postbuf, 0,
+                            SHARD_MASK_TIMES);
+    }
+    UNLOCK(&frame->lock);
+    fd_ctx_get(anon_fd, this, &fsync_count);
+out:
+    if (anon_fd && (base_inode != anon_fd->inode)) {
+        LOCK(&base_inode->lock);
+        LOCK(&anon_fd->inode->lock);
+        {
+            __shard_inode_ctx_get(anon_fd->inode, this, &ctx);
+            __shard_inode_ctx_get(base_inode, this, &base_ictx);
+            if (op_ret == 0)
+                ctx->fsync_needed -= fsync_count;
+            GF_ASSERT(ctx->fsync_needed >= 0);
+            if (ctx->fsync_needed != 0) {
+                list_add_tail(&ctx->to_fsync_list, &base_ictx->to_fsync_list);
+                base_ictx->fsync_count++;
+            } else {
+                unref_shard_inode = _gf_true;
+            }
+        }
+        UNLOCK(&anon_fd->inode->lock);
+        UNLOCK(&base_inode->lock);
+    }
+
+    if (unref_shard_inode)
+        inode_unref(anon_fd->inode);
+    if (anon_fd)
+        fd_unref(anon_fd);
+
+    call_count = shard_call_count_return(frame);
+    if (call_count != 0)
+        return 0;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_FSYNC, frame, local->op_ret,
+                                    local->op_errno);
+    } else {
+        shard_get_timestamps_from_inode_ctx(local, base_inode, this);
+        SHARD_STACK_UNWIND(fsync, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, local->xattr_rsp);
+    }
+    return 0;
 }
 
 int
-shard_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
-{
-        if (op_ret < 0)
-                goto out;
+shard_post_lookup_fsync_handler(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    int call_count = 0;
+    int fsync_count = 0;
+    fd_t *anon_fd = NULL;
+    inode_t *base_inode = NULL;
+    shard_local_t *local = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *iter = NULL;
+    struct list_head copy = {
+        0,
+    };
+    shard_inode_ctx_t *tmp = NULL;
+
+    local = frame->local;
+    base_inode = local->fd->inode;
+    local->postbuf = local->prebuf;
+    INIT_LIST_HEAD(&copy);
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_FSYNC, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
+
+    LOCK(&base_inode->lock);
+    {
+        __shard_inode_ctx_get(base_inode, this, &ctx);
+        list_splice_init(&ctx->to_fsync_list, &copy);
+        call_count = ctx->fsync_count;
+        ctx->fsync_count = 0;
+    }
+    UNLOCK(&base_inode->lock);
+
+    local->call_count = ++call_count;
+
+    /* Send fsync() on the base shard first */
+    anon_fd = fd_ref(local->fd);
+    STACK_WIND_COOKIE(frame, shard_fsync_shards_cbk, anon_fd, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->fsync, anon_fd, local->datasync,
+                      local->xattr_req);
+    call_count--;
+    anon_fd = NULL;
+
+    list_for_each_entry_safe(iter, tmp, &copy, to_fsync_list)
+    {
+        list_del_init(&iter->to_fsync_list);
+        fsync_count = 0;
+        shard_inode_ctx_get_fsync_count(iter->inode, this, &fsync_count);
+        GF_ASSERT(fsync_count > 0);
+        anon_fd = fd_anonymous(iter->inode);
+        if (!anon_fd) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                   SHARD_MSG_MEMALLOC_FAILED,
+                   "Failed to create "
+                   "anon fd to fsync shard");
+            shard_fsync_shards_cbk(frame, (void *)(long)anon_fd, this, -1,
+                                   ENOMEM, NULL, NULL, NULL);
+            continue;
+        }
+
+        ret = fd_ctx_set(anon_fd, this, fsync_count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_FD_CTX_SET_FAILED,
+                   "Failed to set fd "
+                   "ctx for shard inode gfid=%s",
+                   uuid_utoa(iter->inode->gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            shard_fsync_shards_cbk(frame, (void *)(long)anon_fd, this, -1,
+                                   ENOMEM, NULL, NULL, NULL);
+            continue;
+        }
+        STACK_WIND_COOKIE(frame, shard_fsync_shards_cbk, anon_fd,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                          anon_fd, local->datasync, local->xattr_req);
+        call_count--;
+    }
 
-        /* To-Do: Wind fsync on all shards of the file */
-        postbuf->ia_ctime = 0;
-out:
-        SHARD_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    return 0;
 }
 
 int
-shard_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-             dict_t *xdata)
-{
-        STACK_WIND (frame, shard_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
-        return 0;
+shard_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size for %s from its inode ctx",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_fsync_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+        return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    local->fd = fd_ref(fd);
+    local->fop = GF_FOP_FSYNC;
+    local->datasync = datasync;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_post_lookup_fsync_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_FSYNC, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_readdir_past_dot_shard_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, gf_dirent_t *orig_entries,
-                                  dict_t *xdata)
+shard_readdir_past_dot_shard_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, gf_dirent_t *orig_entries,
+                                 dict_t *xdata)
 {
-        gf_dirent_t         *entry          = NULL;
-        gf_dirent_t         *tmp            = NULL;
-        shard_local_t       *local          = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        list_for_each_entry_safe (entry, tmp, (&orig_entries->list), list) {
+    list_for_each_entry_safe(entry, tmp, (&orig_entries->list), list)
+    {
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &local->entries_head.list);
 
-                list_del_init (&entry->list);
-                list_add_tail (&entry->list, &local->entries_head.list);
+        if (!entry->dict)
+            continue;
 
-                if (!entry->dict)
-                        continue;
+        if (IA_ISDIR(entry->d_stat.ia_type))
+            continue;
 
-                if (IA_ISDIR (entry->d_stat.ia_type))
-                        continue;
+        if (dict_get(entry->dict, GF_XATTR_SHARD_FILE_SIZE))
+            shard_modify_size_and_block_count(&entry->d_stat, entry->dict);
+        if (!entry->inode)
+            continue;
 
-                if (dict_get (entry->dict, GF_XATTR_SHARD_FILE_SIZE))
-                        shard_modify_size_and_block_count (&entry->d_stat,
-                                                           entry->dict);
-                if (!entry->inode)
-                        continue;
-
-                shard_inode_ctx_update (entry->inode, this, entry->dict,
-                                        &entry->d_stat);
-        }
-        local->op_ret += op_ret;
+        shard_inode_ctx_update(entry->inode, this, entry->dict, &entry->d_stat);
+    }
+    local->op_ret += op_ret;
 
 unwind:
-        if (local->fop == GF_FOP_READDIR)
-                SHARD_STACK_UNWIND (readdir, frame, local->op_ret,
-                                    local->op_errno,
-                                    &local->entries_head, xdata);
-        else
-                SHARD_STACK_UNWIND (readdirp, frame, op_ret, op_errno,
-                                    &local->entries_head, xdata);
-        return 0;
+    if (local->fop == GF_FOP_READDIR)
+        SHARD_STACK_UNWIND(readdir, frame, local->op_ret, local->op_errno,
+                           &local->entries_head, xdata);
+    else
+        SHARD_STACK_UNWIND(readdirp, frame, op_ret, op_errno,
+                           &local->entries_head, xdata);
+    return 0;
 }
 
 int32_t
-shard_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *orig_entries,
-                   dict_t *xdata)
+shard_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *orig_entries,
+                  dict_t *xdata)
 {
-        fd_t                *fd             = NULL;
-        gf_dirent_t         *entry          = NULL;
-        gf_dirent_t         *tmp            = NULL;
-        shard_local_t       *local          = NULL;
-        gf_boolean_t         last_entry     = _gf_false;
+    fd_t *fd = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    shard_local_t *local = NULL;
+    gf_boolean_t last_entry = _gf_false;
 
-        local = frame->local;
-        fd = local->fd;
+    local = frame->local;
+    fd = local->fd;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        list_for_each_entry_safe (entry, tmp, (&orig_entries->list), list) {
-                if (last_entry)
-                        last_entry = _gf_false;
+    list_for_each_entry_safe(entry, tmp, (&orig_entries->list), list)
+    {
+        if (last_entry)
+            last_entry = _gf_false;
 
-                if (__is_root_gfid (fd->inode->gfid) &&
-                    !(strcmp (entry->d_name, GF_SHARD_DIR))) {
-                        local->offset = entry->d_off;
-                        op_ret--;
-                        last_entry = _gf_true;
-                        continue;
-                }
+        if (__is_root_gfid(fd->inode->gfid) &&
+            !(strcmp(entry->d_name, GF_SHARD_DIR))) {
+            local->offset = entry->d_off;
+            op_ret--;
+            last_entry = _gf_true;
+            continue;
+        }
 
-                list_del_init (&entry->list);
-                list_add_tail (&entry->list, &local->entries_head.list);
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &local->entries_head.list);
 
-                if (!entry->dict)
-                        continue;
+        if (!entry->dict)
+            continue;
 
-                if (IA_ISDIR (entry->d_stat.ia_type))
-                        continue;
-
-                if (dict_get (entry->dict, GF_XATTR_SHARD_FILE_SIZE) &&
-                    frame->root->pid != GF_CLIENT_PID_GSYNCD)
-                        shard_modify_size_and_block_count (&entry->d_stat,
-                                                           entry->dict);
+        if (IA_ISDIR(entry->d_stat.ia_type))
+            continue;
 
-                if (!entry->inode)
-                        continue;
+        if (dict_get(entry->dict, GF_XATTR_SHARD_FILE_SIZE) &&
+            frame->root->pid != GF_CLIENT_PID_GSYNCD)
+            shard_modify_size_and_block_count(&entry->d_stat, entry->dict);
 
-                shard_inode_ctx_update (entry->inode, this, entry->dict,
-                                        &entry->d_stat);
-        }
+        if (!entry->inode)
+            continue;
 
-        local->op_ret = op_ret;
+        shard_inode_ctx_update(entry->inode, this, entry->dict, &entry->d_stat);
+    }
 
-        if (last_entry) {
-                if (local->fop == GF_FOP_READDIR)
-                        STACK_WIND (frame, shard_readdir_past_dot_shard_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->readdir, local->fd,
-                                    local->readdir_size, local->offset,
-                                    local->xattr_req);
-                else
-                        STACK_WIND (frame, shard_readdir_past_dot_shard_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->readdirp,
-                                    local->fd, local->readdir_size,
-                                    local->offset, local->xattr_req);
-                return 0;
-        }
+    local->op_ret = op_ret;
 
-unwind:
+    if (last_entry) {
         if (local->fop == GF_FOP_READDIR)
-                SHARD_STACK_UNWIND (readdir, frame, op_ret, op_errno,
-                                    &local->entries_head, xdata);
+            STACK_WIND(frame, shard_readdir_past_dot_shard_cbk,
+                       FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdir,
+                       local->fd, local->readdir_size, local->offset,
+                       local->xattr_req);
         else
-                SHARD_STACK_UNWIND (readdirp, frame, op_ret, op_errno,
-                                    &local->entries_head, xdata);
+            STACK_WIND(frame, shard_readdir_past_dot_shard_cbk,
+                       FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                       local->fd, local->readdir_size, local->offset,
+                       local->xattr_req);
         return 0;
-}
+    }
 
+unwind:
+    if (local->fop == GF_FOP_READDIR)
+        SHARD_STACK_UNWIND(readdir, frame, op_ret, op_errno,
+                           &local->entries_head, xdata);
+    else
+        SHARD_STACK_UNWIND(readdirp, frame, op_ret, op_errno,
+                           &local->entries_head, xdata);
+    return 0;
+}
 
 int
-shard_readdir_do (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, int whichop, dict_t *xdata)
-{
-        int             ret      = 0;
-        shard_local_t  *local    = NULL;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
+shard_readdir_do(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, int whichop, dict_t *xdata)
+{
+    int ret = 0;
+    shard_local_t *local = NULL;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        goto err;
+    }
+
+    frame->local = local;
+
+    local->fd = fd_ref(fd);
+    local->fop = whichop;
+    local->readdir_size = size;
+    INIT_LIST_HEAD(&local->entries_head.list);
+    local->list_inited = _gf_true;
+
+    if (whichop == GF_FOP_READDIR) {
+        STACK_WIND(frame, shard_readdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+    } else {
+        local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+        SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, fd->inode->gfid,
+                                        local, err);
+        ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, 0);
+        if (ret) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Failed to set "
+                   "dict value: key:%s, directory gfid=%s",
+                   GF_XATTR_SHARD_BLOCK_SIZE, uuid_utoa(fd->inode->gfid));
+            goto err;
         }
 
-        frame->local = local;
-
-        local->fd = fd_ref (fd);
-        local->fop = whichop;
-        local->readdir_size = size;
-        INIT_LIST_HEAD (&local->entries_head.list);
-        local->list_inited = _gf_true;
-
-        if (whichop == GF_FOP_READDIR) {
-                STACK_WIND (frame, shard_readdir_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdir, fd, size, offset,
-                            xdata);
-        } else {
-                local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-                SHARD_MD_READ_FOP_INIT_REQ_DICT (this, local->xattr_req,
-                                                 fd->inode->gfid, local, err);
-                ret = dict_set_uint64 (local->xattr_req,
-                                       GF_XATTR_SHARD_BLOCK_SIZE, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Failed to set "
-                                "dict value: key:%s, directory gfid=%s",
-                                GF_XATTR_SHARD_BLOCK_SIZE,
-                                uuid_utoa (fd->inode->gfid));
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto err;
-                }
-
-                STACK_WIND (frame, shard_readdir_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp, fd, size, offset,
-                            local->xattr_req);
-        }
+        STACK_WIND(frame, shard_readdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, offset,
+                   local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (readdir, frame, local->op_ret, local->op_errno,
-                             NULL, NULL);
-        return 0;
-
+    STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-shard_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, dict_t *xdata)
+shard_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
-        shard_readdir_do (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
-        return 0;
+    shard_readdir_do(frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+    return 0;
 }
 
-
 int32_t
-shard_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset, dict_t *xdata)
+shard_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
 {
-        shard_readdir_do (frame, this, fd, size, offset, GF_FOP_READDIRP,
-                          xdata);
-        return 0;
+    shard_readdir_do(frame, this, fd, size, offset, GF_FOP_READDIRP, xdata);
+    return 0;
 }
 
 int32_t
-shard_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
+shard_modify_and_set_iatt_in_dict(dict_t *xdata, shard_local_t *local,
+                                  char *key)
 {
-        int op_errno = EINVAL;
-
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                GF_IF_NATIVE_XATTR_GOTO (SHARD_XATTR_PREFIX"*",
-                                         name, op_errno, out);
-        }
+    int ret = 0;
+    struct iatt *tmpbuf = NULL;
+    struct iatt *stbuf = NULL;
+    data_t *data = NULL;
 
-        if (xdata && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
-                dict_del (xdata, GF_XATTR_SHARD_BLOCK_SIZE);
-                dict_del (xdata, GF_XATTR_SHARD_FILE_SIZE);
-        }
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->removexattr, loc, name,
-                         xdata);
+    if (!xdata)
         return 0;
 
-out:
-        SHARD_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+    data = dict_get(xdata, key);
+    if (!data)
         return 0;
+
+    tmpbuf = data_to_iatt(data, key);
+    stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char);
+    if (stbuf == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    *stbuf = *tmpbuf;
+    stbuf->ia_size = local->prebuf.ia_size;
+    stbuf->ia_blocks = local->prebuf.ia_blocks;
+    ret = dict_set_iatt(xdata, key, stbuf, false);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    return 0;
+
+err:
+    GF_FREE(stbuf);
+    return -1;
 }
 
 int32_t
-shard_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
+shard_common_remove_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int op_errno = EINVAL;
+    int ret = -1;
+    shard_local_t *local = NULL;
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                GF_IF_NATIVE_XATTR_GOTO (SHARD_XATTR_PREFIX"*",
-                                         name, op_errno, out);
-        }
-
-        if (xdata && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
-                dict_del (xdata, GF_XATTR_SHARD_BLOCK_SIZE);
-                dict_del (xdata, GF_XATTR_SHARD_FILE_SIZE);
-        }
+    local = frame->local;
 
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fremovexattr, fd, name,
-                         xdata);
-        return 0;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_PRESTAT);
+    if (ret < 0)
+        goto err;
+
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_POSTSTAT);
+    if (ret < 0)
+        goto err;
+
+    if (local->fd)
+        SHARD_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    else
+        SHARD_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    return 0;
 
-out:
-        SHARD_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
-        return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
 }
 
 int32_t
-shard_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
+shard_post_lookup_remove_xattr_handler(call_frame_t *frame, xlator_t *this)
 {
-        if (op_ret < 0)
-                goto unwind;
+    shard_local_t *local = NULL;
 
-        if (dict && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
-                dict_del (dict, GF_XATTR_SHARD_BLOCK_SIZE);
-                dict_del (dict, GF_XATTR_SHARD_FILE_SIZE);
-        }
+    local = frame->local;
 
-unwind:
-        SHARD_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
+
+    if (local->fd)
+        STACK_WIND(frame, shard_common_remove_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fremovexattr, local->fd,
+                   local->name, local->xattr_req);
+    else
+        STACK_WIND(frame, shard_common_remove_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->removexattr, &local->loc,
+                   local->name, local->xattr_req);
+    return 0;
 }
 
 int32_t
-shard_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        int op_errno = EINVAL;
-
-        if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) &&
-            (name) && (!strncmp (name, SHARD_XATTR_PREFIX,
-                      strlen (SHARD_XATTR_PREFIX)))) {
-                op_errno = ENODATA;
-                goto out;
-        }
-
-        STACK_WIND (frame, shard_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+shard_common_remove_xattr(call_frame_t *frame, xlator_t *this,
+                          glusterfs_fop_t fop, loc_t *loc, fd_t *fd,
+                          const char *name, dict_t *xdata)
+{
+    int ret = -1;
+    int op_errno = ENOMEM;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+    inode_t *inode = loc ? loc->inode : fd->inode;
+
+    if ((IA_ISDIR(inode->ia_type)) || (IA_ISLNK(inode->ia_type))) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr, loc, name,
+                            xdata);
+        else
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                            xdata);
         return 0;
-
-out:
-        SHARD_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+    }
+
+    /* If shard's special xattrs are attempted to be removed,
+     * fail the fop with EPERM (except if the client is gsyncd).
+     */
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        GF_IF_NATIVE_XATTR_GOTO(SHARD_XATTR_PREFIX "*", name, op_errno, err);
+    }
+
+    /* Repeat the same check for bulk-removexattr */
+    if (xdata && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
+        dict_del(xdata, GF_XATTR_SHARD_BLOCK_SIZE);
+        dict_del(xdata, GF_XATTR_SHARD_FILE_SIZE);
+    }
+
+    ret = shard_inode_ctx_get_block_size(inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr, loc, name,
+                            xdata);
+        else
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                            xdata);
         return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->fop = fop;
+    if (loc) {
+        if (loc_copy(&local->loc, loc) != 0)
+            goto err;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+        local->loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+    }
+
+    if (name) {
+        local->name = gf_strdup(name);
+        if (!local->name)
+            goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    shard_refresh_base_file(frame, this, loc, fd,
+                            shard_post_lookup_remove_xattr_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(fop, frame, -1, op_errno);
+    return 0;
 }
 
+int32_t
+shard_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+    shard_common_remove_xattr(frame, this, GF_FOP_REMOVEXATTR, loc, NULL, name,
+                              xdata);
+    return 0;
+}
 
 int32_t
-shard_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+shard_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+    shard_common_remove_xattr(frame, this, GF_FOP_FREMOVEXATTR, NULL, fd, name,
+                              xdata);
+    return 0;
+}
+
+int32_t
+shard_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *dict,
                     dict_t *xdata)
 {
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        if (dict && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
-                dict_del (dict, GF_XATTR_SHARD_BLOCK_SIZE);
-                dict_del (dict, GF_XATTR_SHARD_FILE_SIZE);
-        }
+    if (dict && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
+        dict_del(dict, GF_XATTR_SHARD_BLOCK_SIZE);
+        dict_del(dict, GF_XATTR_SHARD_FILE_SIZE);
+    }
 
 unwind:
-        SHARD_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    SHARD_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-shard_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name, dict_t *xdata)
+shard_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        int op_errno = EINVAL;
-
-        if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) &&
-            (name) && (!strncmp (name, SHARD_XATTR_PREFIX,
-                      strlen (SHARD_XATTR_PREFIX)))) {
-                op_errno = ENODATA;
-                goto out;
-        }
+    int op_errno = EINVAL;
 
-        STACK_WIND (frame, shard_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) && (name) &&
+        (!strncmp(name, SHARD_XATTR_PREFIX, SLEN(SHARD_XATTR_PREFIX)))) {
+        op_errno = ENODATA;
+        goto out;
+    }
 
+    STACK_WIND(frame, shard_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 out:
-        SHARD_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_FGETXATTR, frame, -1, op_errno);
+    return 0;
 }
 
 int32_t
-shard_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+shard_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
-        int op_errno = EINVAL;
+    if (op_ret < 0)
+        goto unwind;
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                GF_IF_INTERNAL_XATTR_GOTO (SHARD_XATTR_PREFIX"*", dict,
-                                           op_errno, out);
-        }
+    if (dict && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
+        dict_del(dict, GF_XATTR_SHARD_BLOCK_SIZE);
+        dict_del(dict, GF_XATTR_SHARD_FILE_SIZE);
+    }
 
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
-                         xdata);
-        return 0;
-
-out:
-        SHARD_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-        return 0;
+unwind:
+    SHARD_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-shard_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags, dict_t *xdata)
+shard_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int op_errno = EINVAL;
-
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                GF_IF_INTERNAL_XATTR_GOTO (SHARD_XATTR_PREFIX"*", dict,
-                                           op_errno, out);
-        }
+    int op_errno = EINVAL;
 
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
-                         xdata);
-        return 0;
+    if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) && (name) &&
+        (!strncmp(name, SHARD_XATTR_PREFIX, sizeof(SHARD_XATTR_PREFIX) - 1))) {
+        op_errno = ENODATA;
+        goto out;
+    }
 
+    STACK_WIND(frame, shard_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 out:
-        SHARD_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_GETXATTR, frame, -1, op_errno);
+    return 0;
 }
 
-int
-shard_post_setattr_handler (call_frame_t *frame, xlator_t *this)
+int32_t
+shard_common_set_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        shard_local_t *local = NULL;
+    int ret = -1;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->fop == GF_FOP_SETATTR) {
-                if (local->op_ret >= 0)
-                        shard_inode_ctx_set (local->loc.inode, this,
-                                             &local->postbuf, 0,
-                                             SHARD_LOOKUP_MASK);
-                SHARD_STACK_UNWIND (setattr, frame, local->op_ret,
-                                    local->op_errno, &local->prebuf,
-                                    &local->postbuf, local->xattr_rsp);
-        } else if (local->fop == GF_FOP_FSETATTR) {
-                if (local->op_ret >= 0)
-                        shard_inode_ctx_set (local->fd->inode, this,
-                                             &local->postbuf, 0,
-                                             SHARD_LOOKUP_MASK);
-                SHARD_STACK_UNWIND (fsetattr, frame, local->op_ret,
-                                    local->op_errno, &local->prebuf,
-                                    &local->postbuf, local->xattr_rsp);
-        }
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_PRESTAT);
+    if (ret < 0)
+        goto err;
+
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_POSTSTAT);
+    if (ret < 0)
+        goto err;
+
+    if (local->fd)
+        SHARD_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    else
+        SHARD_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    return 0;
 
-        return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
 }
 
-int
-shard_common_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+int32_t
+shard_post_lookup_set_xattr_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
-        local->prebuf = *prebuf;
-        if (shard_modify_size_and_block_count (&local->prebuf, xdata)) {
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
-        }
-        if (xdata)
-                local->xattr_rsp = dict_ref (xdata);
-        local->postbuf = *postbuf;
-        local->postbuf.ia_size = local->prebuf.ia_size;
-        local->postbuf.ia_blocks = local->prebuf.ia_blocks;
+    if (local->fd)
+        STACK_WIND(frame, shard_common_set_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetxattr, local->fd,
+                   local->xattr_req, local->flags, local->xattr_rsp);
+    else
+        STACK_WIND(frame, shard_common_set_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setxattr, &local->loc,
+                   local->xattr_req, local->flags, local->xattr_rsp);
+    return 0;
+}
 
-unwind:
-        local->handler (frame, this);
+int32_t
+shard_common_set_xattr(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                       loc_t *loc, fd_t *fd, dict_t *dict, int32_t flags,
+                       dict_t *xdata)
+{
+    int ret = -1;
+    int op_errno = ENOMEM;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+    inode_t *inode = loc ? loc->inode : fd->inode;
+
+    if ((IA_ISDIR(inode->ia_type)) || (IA_ISLNK(inode->ia_type))) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                            xdata);
+        else
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                            xdata);
+        return 0;
+    }
+
+    /* Sharded or not, if shard's special xattrs are attempted to be set,
+     * fail the fop with EPERM (except if the client is gsyncd.
+     */
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        GF_IF_INTERNAL_XATTR_GOTO(SHARD_XATTR_PREFIX "*", dict, op_errno, err);
+    }
+
+    ret = shard_inode_ctx_get_block_size(inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                            xdata);
+        else
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                            xdata);
         return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->fop = fop;
+    if (loc) {
+        if (loc_copy(&local->loc, loc) != 0)
+            goto err;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+        local->loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+    }
+    local->flags = flags;
+    /* Reusing local->xattr_req and local->xattr_rsp to store the setxattr dict
+     * and the xdata dict
+     */
+    if (dict)
+        local->xattr_req = dict_ref(dict);
+    if (xdata)
+        local->xattr_rsp = dict_ref(xdata);
+
+    shard_refresh_base_file(frame, this, loc, fd,
+                            shard_post_lookup_set_xattr_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(fop, frame, -1, op_errno);
+    return 0;
 }
 
-int
-shard_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+int32_t
+shard_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int                ret        = -1;
-        uint64_t           block_size = 0;
-        shard_local_t     *local      = NULL;
+    shard_common_set_xattr(frame, this, GF_FOP_FSETXATTR, NULL, fd, dict, flags,
+                           xdata);
+    return 0;
+}
 
-        if ((IA_ISDIR (loc->inode->ia_type)) ||
-            (IA_ISLNK (loc->inode->ia_type))) {
-                STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr, loc, stbuf,
-                            valid, xdata);
-                return 0;
-        }
+int32_t
+shard_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+    shard_common_set_xattr(frame, this, GF_FOP_SETXATTR, loc, NULL, dict, flags,
+                           xdata);
+    return 0;
+}
 
-        ret = shard_inode_ctx_get_block_size (loc->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED,
-                        "Failed to get block size from inode ctx of %s",
-                        uuid_utoa (loc->inode->gfid));
-                goto err;
-        }
+int
+shard_post_setattr_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
 
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr, loc, stbuf,
-                            valid, xdata);
-                return 0;
-        }
+    local = frame->local;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    if (local->fop == GF_FOP_SETATTR) {
+        if (local->op_ret >= 0)
+            shard_inode_ctx_set(local->loc.inode, this, &local->postbuf, 0,
+                                SHARD_LOOKUP_MASK);
+        SHARD_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, local->xattr_rsp);
+    } else if (local->fop == GF_FOP_FSETATTR) {
+        if (local->op_ret >= 0)
+            shard_inode_ctx_set(local->fd->inode, this, &local->postbuf, 0,
+                                SHARD_LOOKUP_MASK);
+        SHARD_STACK_UNWIND(fsetattr, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, local->xattr_rsp);
+    }
 
-        frame->local = local;
+    return 0;
+}
 
-        local->handler = shard_post_setattr_handler;
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
-        local->fop = GF_FOP_SETATTR;
-        loc_copy (&local->loc, loc);
+int
+shard_common_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    shard_local_t *local = NULL;
 
-        SHARD_MD_READ_FOP_INIT_REQ_DICT (this, local->xattr_req,
-                                         local->loc.gfid, local, err);
+    local = frame->local;
 
-        STACK_WIND (frame, shard_common_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
-                    local->xattr_req);
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
 
-        return 0;
+    local->prebuf = *prebuf;
+    if (shard_modify_size_and_block_count(&local->prebuf, xdata)) {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+    if (xdata)
+        local->xattr_rsp = dict_ref(xdata);
+    local->postbuf = *postbuf;
+    local->postbuf.ia_size = local->prebuf.ia_size;
+    local->postbuf.ia_blocks = local->prebuf.ia_blocks;
 
-err:
-        SHARD_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+unwind:
+    local->handler(frame, this);
+    return 0;
 }
 
 int
-shard_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+shard_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int                ret        = -1;
-        uint64_t           block_size = 0;
-        shard_local_t     *local      = NULL;
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
 
-        if ((IA_ISDIR (fd->inode->ia_type)) ||
-            (IA_ISLNK (fd->inode->ia_type))) {
-                STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD (this)->fops->fsetattr, fd, stbuf,
-                            valid, xdata);
-                return 0;
-        }
-
-        ret = shard_inode_ctx_get_block_size (fd->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED,
-                        "Failed to get block size from inode ctx of %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto err;
-        }
-
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr, fd, stbuf,
-                            valid, xdata);
-                return 0;
-        }
+    if ((IA_ISDIR(loc->inode->ia_type)) || (IA_ISLNK(loc->inode->ia_type))) {
+        STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+    }
 
-        if (!this->itable)
-                this->itable = fd->inode->table;
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+    }
 
-        frame->local = local;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
 
-        local->handler = shard_post_setattr_handler;
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
-        local->fop = GF_FOP_FSETATTR;
-        local->fd = fd_ref (fd);
+    frame->local = local;
 
-        SHARD_MD_READ_FOP_INIT_REQ_DICT (this, local->xattr_req,
-                                         fd->inode->gfid, local, err);
+    local->handler = shard_post_setattr_handler;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->fop = GF_FOP_SETATTR;
+    loc_copy(&local->loc, loc);
 
-        STACK_WIND (frame, shard_common_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid,
-                    local->xattr_req);
-        return 0;
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, local->loc.gfid,
+                                    local, err);
 
+    STACK_WIND(frame, shard_common_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+               local->xattr_req);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_SETATTR, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_common_inode_write_begin (call_frame_t *frame, xlator_t *this,
-                                glusterfs_fop_t fop, fd_t *fd,
-                                struct iovec *vector, int32_t count,
-                                off_t offset, uint32_t flags, size_t len,
-                                struct iobref *iobref, dict_t *xdata)
+shard_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int             ret            = 0;
-        int             i              = 0;
-        uint64_t        block_size     = 0;
-        shard_local_t  *local          = NULL;
-        shard_priv_t   *priv           = NULL;
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
 
-        priv = this->private;
-
-        ret = shard_inode_ctx_get_block_size (fd->inode, this, &block_size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_INODE_CTX_GET_FAILED, "Failed to get block "
-                        "size for %s from its inode ctx",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
-                /* block_size = 0 means that the file was created before
-                 * sharding was enabled on the volume.
-                 */
-                switch (fop) {
-                case GF_FOP_WRITE:
-                        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                         FIRST_CHILD(this)->fops->writev, fd,
-                                         vector, count, offset, flags, iobref,
-                                         xdata);
-                        break;
-                case GF_FOP_FALLOCATE:
-                        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                         FIRST_CHILD(this)->fops->fallocate, fd,
-                                         flags, offset, len, xdata);
-                        break;
-                case GF_FOP_ZEROFILL:
-                        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                         FIRST_CHILD(this)->fops->zerofill,
-                                         fd, offset, len, xdata);
-                        break;
-                case GF_FOP_DISCARD:
-                        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                         FIRST_CHILD(this)->fops->discard,
-                                         fd, offset, len, xdata);
-                        break;
-                default:
-                gf_msg (this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
-                        "Invalid fop id = %d", fop);
-                        break;
-                }
-                return 0;
-        }
-
-        if (!this->itable)
-                this->itable = fd->inode->table;
-
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto out;
+    if ((IA_ISDIR(fd->inode->ia_type)) || (IA_ISLNK(fd->inode->ia_type))) {
+        STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+    }
 
-        frame->local = local;
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
 
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto out;
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+    }
 
-        if (vector) {
-                local->vector = iov_dup (vector, count);
-                if (!local->vector)
-                        goto out;
-                for (i = 0; i < count; i++)
-                        local->total_size += vector[i].iov_len;
-                local->count = count;
-        } else {
-                local->total_size = len;
-        }
+    if (!this->itable)
+        this->itable = fd->inode->table;
 
-        local->fop = fop;
-        local->offset = offset;
-        local->flags = flags;
-        if (iobref)
-                local->iobref = iobref_ref (iobref);
-        local->fd = fd_ref (fd);
-        local->block_size = block_size;
-        local->first_block = get_lowest_block (offset, local->block_size);
-        local->last_block = get_highest_block (offset, local->total_size,
-                                               local->block_size);
-        local->num_blocks = local->last_block - local->first_block + 1;
-        local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
-                                       gf_shard_mt_inode_list);
-        if (!local->inode_list)
-                goto out;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
 
-        local->loc.inode = inode_ref (fd->inode);
-        gf_uuid_copy (local->loc.gfid, fd->inode->gfid);
+    frame->local = local;
 
-        gf_msg_trace (this->name, 0, "%s: gfid=%s first_block=%"PRIu32" "
-                      "last_block=%"PRIu32" num_blocks=%"PRIu32" offset=%"PRId64""
-                      " total_size=%lu flags=%"PRId32"", gf_fop_list[fop],
-                      uuid_utoa (fd->inode->gfid), local->first_block,
-                      local->last_block, local->num_blocks, offset,
-                      local->total_size, local->flags);
+    local->handler = shard_post_setattr_handler;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->fop = GF_FOP_FSETATTR;
+    local->fd = fd_ref(fd);
 
-        local->dot_shard_loc.inode = inode_find (this->itable,
-                                                 priv->dot_shard_gfid);
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, fd->inode->gfid,
+                                    local, err);
 
-        if (!local->dot_shard_loc.inode)
-                shard_mkdir_dot_shard (frame, this,
-                                 shard_common_inode_write_post_resolve_handler);
-        else
-                shard_common_resolve_shards (frame, this, local->loc.inode,
-                                 shard_common_inode_write_post_resolve_handler);
+    STACK_WIND(frame, shard_common_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid,
+               local->xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_FSETATTR, frame, -1, ENOMEM);
+    return 0;
+}
 
+int
+shard_common_inode_write_begin(call_frame_t *frame, xlator_t *this,
+                               glusterfs_fop_t fop, fd_t *fd,
+                               struct iovec *vector, int32_t count,
+                               off_t offset, uint32_t flags, size_t len,
+                               struct iobref *iobref, dict_t *xdata)
+{
+    int ret = 0;
+    int i = 0;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size for %s from its inode ctx",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        /* block_size = 0 means that the file was created before
+         * sharding was enabled on the volume.
+         */
+        switch (fop) {
+            case GF_FOP_WRITE:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->writev, fd, vector,
+                                count, offset, flags, iobref, xdata);
+                break;
+            case GF_FOP_FALLOCATE:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->fallocate, fd, flags,
+                                offset, len, xdata);
+                break;
+            case GF_FOP_ZEROFILL:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->zerofill, fd, offset,
+                                len, xdata);
+                break;
+            case GF_FOP_DISCARD:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->discard, fd, offset,
+                                len, xdata);
+                break;
+            default:
+                gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                       "Invalid fop id = %d", fop);
+                break;
+        }
         return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
+
+    frame->local = local;
+
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto out;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto out;
+
+    if (vector) {
+        local->vector = iov_dup(vector, count);
+        if (!local->vector)
+            goto out;
+        for (i = 0; i < count; i++)
+            local->total_size += vector[i].iov_len;
+        local->count = count;
+    } else {
+        local->total_size = len;
+    }
+
+    local->fop = fop;
+    local->offset = offset;
+    local->flags = flags;
+    if (iobref)
+        local->iobref = iobref_ref(iobref);
+    local->fd = fd_ref(fd);
+    local->block_size = block_size;
+    local->resolver_base_inode = local->fd->inode;
+    GF_ATOMIC_INIT(local->delta_blocks, 0);
+
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_common_inode_write_post_lookup_handler);
+    return 0;
 out:
-        shard_common_inode_write_failure_unwind (fop, frame, -1, ENOMEM);
-        return 0;
+    shard_common_failure_unwind(fop, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
-              struct iobref *iobref, dict_t *xdata)
+shard_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
-        shard_common_inode_write_begin (frame, this, GF_FOP_WRITE, fd, vector,
-                                        count, offset, flags, 0, iobref, xdata);
-        return 0;
+    shard_common_inode_write_begin(frame, this, GF_FOP_WRITE, fd, vector, count,
+                                   offset, flags, 0, iobref, xdata);
+    return 0;
 }
 
 int
-shard_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
+shard_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
 {
-        if ((keep_size != 0) && (keep_size != FALLOC_FL_ZERO_RANGE) &&
-            (keep_size != (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)))
-                goto out;
-
-        shard_common_inode_write_begin (frame, this, GF_FOP_FALLOCATE, fd, NULL,
-                                        0, offset, keep_size, len, NULL, xdata);
-        return 0;
+    if ((keep_size != 0) && (keep_size != FALLOC_FL_ZERO_RANGE) &&
+        (keep_size != (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)))
+        goto out;
 
+    shard_common_inode_write_begin(frame, this, GF_FOP_FALLOCATE, fd, NULL, 0,
+                                   offset, keep_size, len, NULL, xdata);
+    return 0;
 out:
-        SHARD_STACK_UNWIND (fallocate, frame, -1, ENOTSUP, NULL, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_FALLOCATE, frame, -1, ENOTSUP);
+    return 0;
 }
 
 int
-shard_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                off_t len, dict_t *xdata)
+shard_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
 {
-        shard_common_inode_write_begin (frame, this, GF_FOP_ZEROFILL, fd, NULL,
-                                        0, offset, 0, len, NULL, xdata);
-        return 0;
+    shard_common_inode_write_begin(frame, this, GF_FOP_ZEROFILL, fd, NULL, 0,
+                                   offset, 0, len, NULL, xdata);
+    return 0;
 }
 
 int
-shard_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               size_t len, dict_t *xdata)
+shard_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
 {
-        shard_common_inode_write_begin (frame, this, GF_FOP_DISCARD, fd, NULL,
-                                        0, offset, 0, len, NULL, xdata);
-        return 0;
+    shard_common_inode_write_begin(frame, this, GF_FOP_DISCARD, fd, NULL, 0,
+                                   offset, 0, len, NULL, xdata);
+    return 0;
 }
 
 int32_t
-shard_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            gf_seek_what_t what, dict_t *xdata)
+shard_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
 {
-        /* TBD */
-        gf_msg (this->name, GF_LOG_INFO, ENOTSUP, SHARD_MSG_FOP_NOT_SUPPORTED,
-                "seek called on %s.", uuid_utoa (fd->inode->gfid));
-        SHARD_STACK_UNWIND (seek, frame, -1, ENOTSUP, 0, NULL);
-        return 0;
+    /* TBD */
+    gf_msg(this->name, GF_LOG_INFO, ENOTSUP, SHARD_MSG_FOP_NOT_SUPPORTED,
+           "seek called on %s.", uuid_utoa(fd->inode->gfid));
+    shard_common_failure_unwind(GF_FOP_SEEK, frame, -1, ENOTSUP);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_shard_mt_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        SHARD_MSG_MEM_ACCT_INIT_FAILED, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_shard_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_MEM_ACCT_INIT_FAILED,
+               "Memory accounting init"
+               "failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-init (xlator_t *this)
-{
-        int           ret  = -1;
-        shard_priv_t *priv = NULL;
-
-        if (!this) {
-                gf_msg ("shard", GF_LOG_ERROR, 0, SHARD_MSG_NULL_THIS,
-                        "this is NULL. init() failed");
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, SHARD_MSG_INVALID_VOLFILE,
-                        "Dangling volume. Check volfile");
-                goto out;
-        }
-
-        if (!this->children || this->children->next) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, SHARD_MSG_INVALID_VOLFILE,
-                        "shard not configured with exactly one sub-volume. "
-                        "Check volfile");
-                goto out;
-        }
-
-        priv = GF_CALLOC (1, sizeof (shard_priv_t), gf_shard_mt_priv_t);
-        if (!priv)
-                goto out;
-
-        GF_OPTION_INIT ("shard-block-size", priv->block_size, size_uint64, out);
-
-        this->local_pool = mem_pool_new (shard_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_parse (SHARD_ROOT_GFID, priv->dot_shard_gfid);
-
-        this->private = priv;
-        LOCK_INIT (&priv->lock);
-        INIT_LIST_HEAD (&priv->ilist_head);
-        ret = 0;
+init(xlator_t *this)
+{
+    int ret = -1;
+    shard_priv_t *priv = NULL;
+
+    if (!this) {
+        gf_msg("shard", GF_LOG_ERROR, 0, SHARD_MSG_NULL_THIS,
+               "this is NULL. init() failed");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INVALID_VOLFILE,
+               "Dangling volume. Check volfile");
+        goto out;
+    }
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INVALID_VOLFILE,
+               "shard not configured with exactly one sub-volume. "
+               "Check volfile");
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(shard_priv_t), gf_shard_mt_priv_t);
+    if (!priv)
+        goto out;
+
+    GF_OPTION_INIT("shard-block-size", priv->block_size, size_uint64, out);
+
+    GF_OPTION_INIT("shard-deletion-rate", priv->deletion_rate, uint32, out);
+
+    GF_OPTION_INIT("shard-lru-limit", priv->lru_limit, uint64, out);
+
+    this->local_pool = mem_pool_new(shard_local_t, 128);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_parse(SHARD_ROOT_GFID, priv->dot_shard_gfid);
+    gf_uuid_parse(DOT_SHARD_REMOVE_ME_GFID, priv->dot_shard_rm_gfid);
+
+    this->private = priv;
+    LOCK_INIT(&priv->lock);
+    INIT_LIST_HEAD(&priv->ilist_head);
+    ret = 0;
 out:
-        if  (ret) {
-                GF_FREE (priv);
-                mem_pool_destroy (this->local_pool);
-        }
-
-        return ret;
+    if (ret) {
+        GF_FREE(priv);
+        mem_pool_destroy(this->local_pool);
+    }
 
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        shard_priv_t *priv = NULL;
+    shard_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("shard", this, out);
+    GF_VALIDATE_OR_GOTO("shard", this, out);
 
-        mem_pool_destroy (this->local_pool);
-        this->local_pool = NULL;
+    /*Itable was not created by shard, hence setting to NULL.*/
+    this->itable = NULL;
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
 
-        this->private = NULL;
-        LOCK_DESTROY (&priv->lock);
-        GF_FREE (priv);
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    this->private = NULL;
+    LOCK_DESTROY(&priv->lock);
+    GF_FREE(priv);
 
 out:
-        return;
+    return;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int            ret  = -1;
-        shard_priv_t  *priv = NULL;
+    int ret = -1;
+    shard_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        GF_OPTION_RECONF ("shard-block-size", priv->block_size, options, size,
-                          out);
+    GF_OPTION_RECONF("shard-block-size", priv->block_size, options, size, out);
 
-        ret = 0;
+    GF_OPTION_RECONF("shard-deletion-rate", priv->deletion_rate, options,
+                     uint32, out);
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-shard_forget (xlator_t *this, inode_t *inode)
+shard_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_priv_t *priv = NULL;
 
-        inode_ctx_del (inode, this, &ctx_uint);
-        if (!ctx_uint)
-                return 0;
+    priv = this->private;
+    if (!priv)
+        return 0;
 
-        ctx = (shard_inode_ctx_t *)ctx_uint;
+    inode_ctx_del(inode, this, &ctx_uint);
+    if (!ctx_uint)
+        return 0;
 
-        GF_FREE (ctx);
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        return 0;
+    /* When LRU limit reaches inode will be forcefully removed from the
+     * table, inode needs to be removed from LRU of shard as well.
+     */
+    if (!list_empty(&ctx->ilist)) {
+        LOCK(&priv->lock);
+        {
+            list_del_init(&ctx->ilist);
+            priv->inode_count--;
+        }
+        UNLOCK(&priv->lock);
+    }
+    GF_FREE(ctx);
+
+    return 0;
 }
 
 int
-shard_release (xlator_t *this, fd_t *fd)
+shard_release(xlator_t *this, fd_t *fd)
 {
-        /* TBD */
-        return 0;
+    /* TBD */
+    return 0;
 }
 
 int
-shard_priv_dump (xlator_t *this)
+shard_priv_dump(xlator_t *this)
 {
-        shard_priv_t *priv                             = NULL;
-        char          key_prefix[GF_DUMP_MAX_BUF_LEN]  = {0,};
+    shard_priv_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char *str = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        snprintf (key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type,
-                  this->name);
-        gf_proc_dump_add_section (key_prefix);
-        gf_proc_dump_write ("shard-block-size", "%s",
-                            gf_uint64_2human_readable (priv->block_size));
-        gf_proc_dump_write ("inode-count", "%d", priv->inode_count);
-        gf_proc_dump_write ("ilist_head", "%p", &priv->ilist_head);
-        gf_proc_dump_write ("lru-max-limit", "%d", SHARD_MAX_INODES);
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+    str = gf_uint64_2human_readable(priv->block_size);
+    gf_proc_dump_write("shard-block-size", "%s", str);
+    gf_proc_dump_write("inode-count", "%d", priv->inode_count);
+    gf_proc_dump_write("ilist_head", "%p", &priv->ilist_head);
+    gf_proc_dump_write("lru-max-limit", "%" PRIu64, priv->lru_limit);
 
-        return 0;
+    GF_FREE(str);
+
+    return 0;
 }
 
 int
-shard_releasedir (xlator_t *this, fd_t *fd)
+shard_releasedir(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
 struct xlator_fops fops = {
-        .lookup      = shard_lookup,
-        .open        = shard_open,
-        .flush       = shard_flush,
-        .fsync       = shard_fsync,
-        .stat        = shard_stat,
-        .fstat       = shard_fstat,
-        .getxattr    = shard_getxattr,
-        .fgetxattr   = shard_fgetxattr,
-        .readv       = shard_readv,
-        .writev      = shard_writev,
-        .truncate    = shard_truncate,
-        .ftruncate   = shard_ftruncate,
-        .setxattr    = shard_setxattr,
-        .fsetxattr   = shard_fsetxattr,
-        .setattr     = shard_setattr,
-        .fsetattr    = shard_fsetattr,
-        .removexattr = shard_removexattr,
-        .fremovexattr = shard_fremovexattr,
-        .fallocate   = shard_fallocate,
-        .discard     = shard_discard,
-        .zerofill    = shard_zerofill,
-        .readdir     = shard_readdir,
-        .readdirp    = shard_readdirp,
-        .create      = shard_create,
-        .mknod       = shard_mknod,
-        .link        = shard_link,
-        .unlink      = shard_unlink,
-        .rename      = shard_rename,
-        .seek        = shard_seek,
+    .lookup = shard_lookup,
+    .open = shard_open,
+    .flush = shard_flush,
+    .fsync = shard_fsync,
+    .stat = shard_stat,
+    .fstat = shard_fstat,
+    .getxattr = shard_getxattr,
+    .fgetxattr = shard_fgetxattr,
+    .readv = shard_readv,
+    .writev = shard_writev,
+    .truncate = shard_truncate,
+    .ftruncate = shard_ftruncate,
+    .setxattr = shard_setxattr,
+    .fsetxattr = shard_fsetxattr,
+    .setattr = shard_setattr,
+    .fsetattr = shard_fsetattr,
+    .removexattr = shard_removexattr,
+    .fremovexattr = shard_fremovexattr,
+    .fallocate = shard_fallocate,
+    .discard = shard_discard,
+    .zerofill = shard_zerofill,
+    .readdir = shard_readdir,
+    .readdirp = shard_readdirp,
+    .create = shard_create,
+    .mknod = shard_mknod,
+    .link = shard_link,
+    .unlink = shard_unlink,
+    .rename = shard_rename,
+    .seek = shard_seek,
 };
 
 struct xlator_cbks cbks = {
-        .forget  = shard_forget,
-        .release = shard_release,
-        .releasedir = shard_releasedir,
+    .forget = shard_forget,
+    .release = shard_release,
+    .releasedir = shard_releasedir,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = shard_priv_dump,
+    .priv = shard_priv_dump,
 };
 
 struct volume_options options[] = {
-        {  .key = {"shard-block-size"},
-           .type = GF_OPTION_TYPE_SIZET,
-           .default_value = "4MB",
-           .min = SHARD_MIN_BLOCK_SIZE,
-           .max = SHARD_MAX_BLOCK_SIZE,
-           .description = "The size unit used to break a file into multiple "
-                          "chunks",
-        },
-        { .key = {NULL} },
+    {
+        .key = {"shard"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable shard",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {
+        .key = {"shard-block-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"shard"},
+        .default_value = "64MB",
+        .min = SHARD_MIN_BLOCK_SIZE,
+        .max = SHARD_MAX_BLOCK_SIZE,
+        .description = "The size unit used to break a file into multiple "
+                       "chunks",
+    },
+    {
+        .key = {"shard-deletion-rate"},
+        .type = GF_OPTION_TYPE_INT,
+        .op_version = {GD_OP_VERSION_5_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"shard"},
+        .default_value = "100",
+        .min = 100,
+        .max = INT_MAX,
+        .description = "The number of shards to send deletes on at a time",
+    },
+    {
+        .key = {"shard-lru-limit"},
+        .type = GF_OPTION_TYPE_INT,
+        .op_version = {GD_OP_VERSION_5_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+        .tags = {"shard"},
+        .default_value = "16384",
+        .min = 20,
+        .max = INT_MAX,
+        .description = "The number of resolved shard inodes to keep in "
+                       "memory. A higher number means shards that are "
+                       "resolved will remain in memory longer, avoiding "
+                       "frequent lookups on them when they participate in "
+                       "file operations. The option also has a bearing on "
+                       "amount of memory consumed by these inodes and their "
+                       "internal metadata",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "shard",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/shard/src/shard.h b/xlators/features/shard/src/shard.h
index 8303a2ca030..4fe181b64d5 100644
--- a/xlators/features/shard/src/shard.h
+++ b/xlators/features/shard/src/shard.h
@@ -8,265 +8,341 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __SHARD_H__
 #define __SHARD_H__
 
-#include "xlator.h"
-#include "compat-errno.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/compat-errno.h>
 #include "shard-messages.h"
+#include <glusterfs/syncop.h>
 
 #define GF_SHARD_DIR ".shard"
-#define SHARD_MIN_BLOCK_SIZE  (4 * GF_UNIT_MB)
-#define SHARD_MAX_BLOCK_SIZE  (4 * GF_UNIT_TB)
+#define GF_SHARD_REMOVE_ME_DIR ".remove_me"
+#define SHARD_MIN_BLOCK_SIZE (4 * GF_UNIT_MB)
+#define SHARD_MAX_BLOCK_SIZE (4 * GF_UNIT_TB)
 #define SHARD_XATTR_PREFIX "trusted.glusterfs.shard."
 #define GF_XATTR_SHARD_BLOCK_SIZE "trusted.glusterfs.shard.block-size"
-#define SHARD_INODE_LRU_LIMIT 4096
-#define SHARD_MAX_INODES 16384
 /**
  *  Bit masks for the valid flag, which is used while updating ctx
-**/
-#define SHARD_MASK_BLOCK_SIZE          (1 << 0)
-#define SHARD_MASK_PROT                (1 << 1)
-#define SHARD_MASK_NLINK               (1 << 2)
-#define SHARD_MASK_UID                 (1 << 3)
-#define SHARD_MASK_GID                 (1 << 4)
-#define SHARD_MASK_SIZE                (1 << 6)
-#define SHARD_MASK_BLOCKS              (1 << 7)
-#define SHARD_MASK_TIMES               (1 << 8)
-#define SHARD_MASK_OTHERS              (1 << 9)
-#define SHARD_MASK_REFRESH_RESET       (1 << 10)
-
-#define SHARD_INODE_WRITE_MASK (SHARD_MASK_SIZE | SHARD_MASK_BLOCKS         \
-                                                | SHARD_MASK_TIMES)
+ **/
+#define SHARD_MASK_BLOCK_SIZE (1 << 0)
+#define SHARD_MASK_PROT (1 << 1)
+#define SHARD_MASK_NLINK (1 << 2)
+#define SHARD_MASK_UID (1 << 3)
+#define SHARD_MASK_GID (1 << 4)
+#define SHARD_MASK_SIZE (1 << 6)
+#define SHARD_MASK_BLOCKS (1 << 7)
+#define SHARD_MASK_TIMES (1 << 8)
+#define SHARD_MASK_OTHERS (1 << 9)
+#define SHARD_MASK_REFRESH_RESET (1 << 10)
 
-#define SHARD_LOOKUP_MASK (SHARD_MASK_PROT | SHARD_MASK_NLINK | SHARD_MASK_UID \
-                           | SHARD_MASK_GID | SHARD_MASK_TIMES                 \
-                           | SHARD_MASK_OTHERS)
+#define SHARD_INODE_WRITE_MASK                                                 \
+    (SHARD_MASK_SIZE | SHARD_MASK_BLOCKS | SHARD_MASK_TIMES)
 
-#define SHARD_ALL_MASK (SHARD_MASK_BLOCK_SIZE | SHARD_MASK_PROT               \
-                        | SHARD_MASK_NLINK | SHARD_MASK_UID | SHARD_MASK_GID  \
-                        | SHARD_MASK_SIZE | SHARD_MASK_BLOCKS                 \
-                        | SHARD_MASK_TIMES | SHARD_MASK_OTHERS)
+#define SHARD_LOOKUP_MASK                                                      \
+    (SHARD_MASK_PROT | SHARD_MASK_NLINK | SHARD_MASK_UID | SHARD_MASK_GID |    \
+     SHARD_MASK_TIMES | SHARD_MASK_OTHERS)
 
+#define SHARD_ALL_MASK                                                         \
+    (SHARD_MASK_BLOCK_SIZE | SHARD_MASK_PROT | SHARD_MASK_NLINK |              \
+     SHARD_MASK_UID | SHARD_MASK_GID | SHARD_MASK_SIZE | SHARD_MASK_BLOCKS |   \
+     SHARD_MASK_TIMES | SHARD_MASK_OTHERS)
 
 #define get_lowest_block(off, shard_size) ((off) / (shard_size))
-#define get_highest_block(off, len, shard_size) \
-        (((((off)+(len)) == 0)?0:((off)+(len)-1)) / (shard_size))
+#define get_highest_block(off, len, shard_size)                                \
+    (((((off) + (len)) == 0) ? 0 : ((off) + (len)-1)) / (shard_size))
+
+int
+shard_unlock_inodelk(call_frame_t *frame, xlator_t *this);
+
+int
+shard_unlock_entrylk(call_frame_t *frame, xlator_t *this);
 
-#define SHARD_ENTRY_FOP_CHECK(loc, op_errno, label) do {               \
-        if ((loc->name && !strcmp (GF_SHARD_DIR, loc->name)) &&        \
-            (((loc->parent) &&                                          \
-            __is_root_gfid (loc->parent->gfid)) ||                     \
-            __is_root_gfid (loc->pargfid))) {                           \
-                    op_errno = EPERM;                                  \
-                    goto label;                                        \
-        }                                                              \
-                                                                       \
-        if ((loc->parent &&                                            \
-            __is_shard_dir (loc->parent->gfid)) ||                     \
-            __is_shard_dir (loc->pargfid)) {                           \
-                    op_errno = EPERM;                                  \
-                    goto label;                                        \
-        }                                                              \
-} while (0)
+#define SHARD_ENTRY_FOP_CHECK(loc, op_errno, label)                            \
+    do {                                                                       \
+        if ((loc->name && !strcmp(GF_SHARD_DIR, loc->name)) &&                 \
+            (((loc->parent) && __is_root_gfid(loc->parent->gfid)) ||           \
+             __is_root_gfid(loc->pargfid))) {                                  \
+            op_errno = EPERM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+                                                                               \
+        if ((loc->parent && __is_shard_dir(loc->parent->gfid)) ||              \
+            __is_shard_dir(loc->pargfid)) {                                    \
+            op_errno = EPERM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
-#define SHARD_INODE_OP_CHECK(gfid, err, label) do {                    \
-        if (__is_shard_dir(gfid)) {                                    \
-                err = EPERM;                                           \
-                goto label;                                            \
-        }                                                              \
-} while (0)
+#define SHARD_INODE_OP_CHECK(gfid, err, label)                                 \
+    do {                                                                       \
+        if (__is_shard_dir(gfid)) {                                            \
+            err = EPERM;                                                       \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
-#define SHARD_STACK_UNWIND(fop, frame, params ...) do {       \
-        shard_local_t *__local = NULL;                         \
-        if (frame) {                                           \
-                __local = frame->local;                        \
-                frame->local = NULL;                           \
-        }                                                      \
-        STACK_UNWIND_STRICT (fop, frame, params);              \
-        if (__local) {                                         \
-                shard_local_wipe (__local);                    \
-                mem_put (__local);                             \
-        }                                                      \
-} while (0)
+#define SHARD_STACK_UNWIND(fop, frame, params...)                              \
+    do {                                                                       \
+        shard_local_t *__local = NULL;                                         \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            if (__local && __local->int_inodelk.acquired_lock)                 \
+                shard_unlock_inodelk(frame, frame->this);                      \
+            if (__local && __local->int_entrylk.acquired_lock)                 \
+                shard_unlock_entrylk(frame, frame->this);                      \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local) {                                                         \
+            shard_local_wipe(__local);                                         \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
 
+#define SHARD_STACK_DESTROY(frame)                                             \
+    do {                                                                       \
+        shard_local_t *__local = NULL;                                         \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        if (__local) {                                                         \
+            shard_local_wipe(__local);                                         \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0);
 
-#define SHARD_INODE_CREATE_INIT(this, local, xattr_req, loc, label) do {      \
-        int            __ret       = -1;                                      \
-        int64_t       *__size_attr = NULL;                                    \
-        shard_priv_t  *__priv      = NULL;                                    \
-                                                                              \
-        __priv = this->private;                                               \
-                                                                              \
-        local->block_size = hton64 (__priv->block_size);                      \
-        __ret = dict_set_static_bin (xattr_req, GF_XATTR_SHARD_BLOCK_SIZE,    \
-                                     &local->block_size,                      \
-                                     sizeof (local->block_size));             \
-        if (__ret) {                                                          \
-                gf_msg (this->name, GF_LOG_WARNING, 0,                        \
-                        SHARD_MSG_DICT_SET_FAILED, "Failed to set key: %s "   \
-                        "on path %s", GF_XATTR_SHARD_BLOCK_SIZE, loc->path);  \
-                goto label;                                                   \
-        }                                                                     \
-                                                                              \
-        __ret = shard_set_size_attrs (0, 0, &__size_attr);                    \
-        if (__ret)                                                            \
-                goto label;                                                   \
-                                                                              \
-        __ret = dict_set_bin (xattr_req, GF_XATTR_SHARD_FILE_SIZE,            \
-                              __size_attr, 8 * 4);                            \
-        if (__ret) {                                                          \
-                gf_msg (this->name, GF_LOG_WARNING, 0,                        \
-                        SHARD_MSG_DICT_SET_FAILED, "Failed to set key: %s "   \
-                        "on path %s", GF_XATTR_SHARD_FILE_SIZE, loc->path);   \
-                GF_FREE (__size_attr);                                        \
-                goto label;                                                   \
-        }                                                                     \
-} while (0)
+#define SHARD_INODE_CREATE_INIT(this, block_size, xattr_req, loc, size,        \
+                                block_count, label)                            \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        int64_t *__size_attr = NULL;                                           \
+        uint64_t *__bs = 0;                                                    \
+                                                                               \
+        __bs = GF_MALLOC(sizeof(uint64_t), gf_shard_mt_uint64_t);              \
+        if (!__bs)                                                             \
+            goto label;                                                        \
+        *__bs = hton64(block_size);                                            \
+        __ret = dict_set_bin(xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, __bs,       \
+                             sizeof(*__bs));                                   \
+        if (__ret) {                                                           \
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,    \
+                   "Failed to set key: %s "                                    \
+                   "on path %s",                                               \
+                   GF_XATTR_SHARD_BLOCK_SIZE, (loc)->path);                    \
+            GF_FREE(__bs);                                                     \
+            goto label;                                                        \
+        }                                                                      \
+                                                                               \
+        __ret = shard_set_size_attrs(size, block_count, &__size_attr);         \
+        if (__ret)                                                             \
+            goto label;                                                        \
+                                                                               \
+        __ret = dict_set_bin(xattr_req, GF_XATTR_SHARD_FILE_SIZE, __size_attr, \
+                             8 * 4);                                           \
+        if (__ret) {                                                           \
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,    \
+                   "Failed to set key: %s "                                    \
+                   "on path %s",                                               \
+                   GF_XATTR_SHARD_FILE_SIZE, (loc)->path);                     \
+            GF_FREE(__size_attr);                                              \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
+#define SHARD_MD_READ_FOP_INIT_REQ_DICT(this, dict, gfid, local, label)        \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+                                                                               \
+        __ret = dict_set_uint64(dict, GF_XATTR_SHARD_FILE_SIZE, 8 * 4);        \
+        if (__ret) {                                                           \
+            local->op_ret = -1;                                                \
+            local->op_errno = ENOMEM;                                          \
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,    \
+                   "Failed to set dict value:"                                 \
+                   " key:%s for %s.",                                          \
+                   GF_XATTR_SHARD_FILE_SIZE, uuid_utoa(gfid));                 \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
-#define SHARD_MD_READ_FOP_INIT_REQ_DICT(this, dict, gfid, local, label)  do { \
-        int __ret = -1;                                                       \
-                                                                              \
-        __ret = dict_set_uint64 (dict, GF_XATTR_SHARD_FILE_SIZE, 8 * 4);      \
-        if (__ret) {                                                          \
-                local->op_ret = -1;                                           \
-                local->op_errno = ENOMEM;                                     \
-                gf_msg (this->name, GF_LOG_WARNING, 0,                        \
-                        SHARD_MSG_DICT_SET_FAILED, "Failed to set dict value:"\
-                        " key:%s for %s.", GF_XATTR_SHARD_FILE_SIZE,          \
-                        uuid_utoa (gfid));                                    \
-                goto label;                                                   \
-        }                                                                     \
-} while (0)
+#define SHARD_SET_ROOT_FS_ID(frame, local)                                     \
+    do {                                                                       \
+        if (!local->is_set_fsid) {                                             \
+            local->uid = frame->root->uid;                                     \
+            local->gid = frame->root->gid;                                     \
+            frame->root->uid = 0;                                              \
+            frame->root->gid = 0;                                              \
+            local->is_set_fsid = _gf_true;                                     \
+        }                                                                      \
+    } while (0)
 
-#define SHARD_SET_ROOT_FS_ID(frame, local) do {                               \
-                if (!local->is_set_fsid) {                                    \
-                        local->uid = frame->root->uid;                        \
-                        local->gid = frame->root->gid;                        \
-                        frame->root->uid = 0;                                 \
-                        frame->root->gid = 0;                                 \
-                        local->is_set_fsid = _gf_true;                        \
-                }                                                             \
-} while (0)
+#define SHARD_UNSET_ROOT_FS_ID(frame, local)                                   \
+    do {                                                                       \
+        if (local->is_set_fsid) {                                              \
+            frame->root->uid = local->uid;                                     \
+            frame->root->gid = local->gid;                                     \
+            local->is_set_fsid = _gf_false;                                    \
+        }                                                                      \
+    } while (0)
 
-#define SHARD_UNSET_ROOT_FS_ID(frame, local) do {                             \
-                if (local->is_set_fsid) {                                     \
-                        frame->root->uid = local->uid;                        \
-                        frame->root->gid = local->gid;                        \
-                        local->is_set_fsid = _gf_false;                       \
-                }                                                             \
-} while (0)
+#define SHARD_TIME_UPDATE(ctx_sec, ctx_nsec, new_sec, new_nsec)                \
+    do {                                                                       \
+        if (ctx_sec == new_sec)                                                \
+            ctx_nsec = new_nsec = max(new_nsec, ctx_nsec);                     \
+        else if (ctx_sec > new_sec) {                                          \
+            new_sec = ctx_sec;                                                 \
+            new_nsec = ctx_nsec;                                               \
+        } else {                                                               \
+            ctx_sec = new_sec;                                                 \
+            ctx_nsec = new_nsec;                                               \
+        }                                                                      \
+    } while (0)
 
-#define SHARD_TIME_UPDATE(ctx_sec, ctx_nsec, new_sec, new_nsec) do {          \
-                if (ctx_sec == new_sec)                                       \
-                        ctx_nsec = new_nsec = max (new_nsec, ctx_nsec);       \
-                else if (ctx_sec > new_sec) {                                 \
-                        new_sec = ctx_sec;                                    \
-                        new_nsec = ctx_nsec;                                  \
-                } else {                                                      \
-                        ctx_sec = new_sec;                                    \
-                        ctx_nsec = new_nsec;                                  \
-                }                                                             \
-        } while (0)
+typedef enum {
+    SHARD_BG_DELETION_NONE = 0,
+    SHARD_BG_DELETION_LAUNCHING,
+    SHARD_BG_DELETION_IN_PROGRESS,
+} shard_bg_deletion_state_t;
 
+/* rm = "remove me" */
 
 typedef struct shard_priv {
-        uint64_t block_size;
-        uuid_t dot_shard_gfid;
-        inode_t *dot_shard_inode;
-        gf_lock_t lock;
-        int inode_count;
-        struct list_head ilist_head;
+    uint64_t block_size;
+    uuid_t dot_shard_gfid;
+    uuid_t dot_shard_rm_gfid;
+    inode_t *dot_shard_inode;
+    inode_t *dot_shard_rm_inode;
+    gf_lock_t lock;
+    int inode_count;
+    struct list_head ilist_head;
+    uint32_t deletion_rate;
+    shard_bg_deletion_state_t bg_del_state;
+    gf_boolean_t first_lookup_done;
+    uint64_t lru_limit;
 } shard_priv_t;
 
 typedef struct {
-        loc_t *loc;
-        short type;
-        char *domain;
-} shard_lock_t;
+    loc_t loc;
+    char *domain;
+    struct gf_flock flock;
+    gf_boolean_t acquired_lock;
+} shard_inodelk_t;
 
-typedef int32_t (*shard_post_fop_handler_t) (call_frame_t *frame,
-                                             xlator_t *this);
-typedef int32_t (*shard_post_resolve_fop_handler_t) (call_frame_t *frame,
-                                                     xlator_t *this);
-typedef int32_t (*shard_post_lookup_shards_fop_handler_t) (call_frame_t *frame,
-                                                           xlator_t *this);
+typedef struct {
+    loc_t loc;
+    char *domain;
+    char *basename;
+    entrylk_cmd cmd;
+    entrylk_type type;
+    gf_boolean_t acquired_lock;
+} shard_entrylk_t;
+
+typedef int32_t (*shard_post_fop_handler_t)(call_frame_t *frame,
+                                            xlator_t *this);
+typedef int32_t (*shard_post_resolve_fop_handler_t)(call_frame_t *frame,
+                                                    xlator_t *this);
+typedef int32_t (*shard_post_lookup_shards_fop_handler_t)(call_frame_t *frame,
+                                                          xlator_t *this);
 
-typedef int32_t (*shard_post_mknod_fop_handler_t) (call_frame_t *frame,
-                                                   xlator_t *this);
+typedef int32_t (*shard_post_mknod_fop_handler_t)(call_frame_t *frame,
+                                                  xlator_t *this);
+
+typedef int32_t (*shard_post_update_size_fop_handler_t)(call_frame_t *frame,
+                                                        xlator_t *this);
 
-typedef int32_t (*shard_post_update_size_fop_handler_t) (call_frame_t *frame,
-                                                         xlator_t *this);
 typedef struct shard_local {
-        int op_ret;
-        int op_errno;
-        int first_block;
-        int last_block;
-        int num_blocks;
-        int call_count;
-        int eexist_count;
-        int create_count;
-        int xflag;
-        int count;
-        uint32_t flags;
-        uint32_t uid;
-        uint32_t gid;
-        uint64_t block_size;
-        uint64_t dst_block_size;
-        off_t offset;
-        size_t total_size;
-        size_t written_size;
-        size_t hole_size;
-        size_t req_size;
-        size_t readdir_size;
-        int64_t delta_size;
-        int delta_blocks;
-        loc_t loc;
-        loc_t dot_shard_loc;
-        loc_t loc2;
-        loc_t tmp_loc;
-        fd_t *fd;
-        dict_t *xattr_req;
-        dict_t *xattr_rsp;
-        inode_t **inode_list;
-        glusterfs_fop_t fop;
-        struct iatt prebuf;
-        struct iatt postbuf;
-        struct iatt preoldparent;
-        struct iatt postoldparent;
-        struct iatt prenewparent;
-        struct iatt postnewparent;
-        struct iovec *vector;
-        struct iobref *iobref;
-        struct iobuf *iobuf;
-        gf_dirent_t entries_head;
-        gf_boolean_t is_set_fsid;
-        gf_boolean_t list_inited;
-        shard_post_fop_handler_t handler;
-        shard_post_lookup_shards_fop_handler_t pls_fop_handler;
-        shard_post_resolve_fop_handler_t post_res_handler;
-        shard_post_mknod_fop_handler_t post_mknod_handler;
-        shard_post_update_size_fop_handler_t post_update_size_handler;
-        struct {
-                int lock_count;
-                fop_inodelk_cbk_t inodelk_cbk;
-                shard_lock_t *shard_lock;
-        } lock;
+    int op_ret;
+    int op_errno;
+    uint64_t first_block;
+    uint64_t last_block;
+    uint64_t num_blocks;
+    int call_count;
+    int eexist_count;
+    int create_count;
+    int xflag;
+    int count;
+    uint32_t flags;
+    uint32_t uid;
+    uint32_t gid;
+    uint64_t block_size;
+    uint64_t dst_block_size;
+    int32_t datasync;
+    off_t offset;
+    size_t total_size;
+    size_t written_size;
+    size_t hole_size;
+    size_t req_size;
+    size_t readdir_size;
+    int64_t delta_size;
+    gf_atomic_t delta_blocks;
+    loc_t loc;
+    loc_t dot_shard_loc;
+    loc_t dot_shard_rm_loc;
+    loc_t loc2;
+    loc_t tmp_loc;
+    fd_t *fd;
+    dict_t *xattr_req;
+    dict_t *xattr_rsp;
+    inode_t **inode_list;
+    glusterfs_fop_t fop;
+    struct iatt prebuf;
+    struct iatt postbuf;
+    struct iatt preoldparent;
+    struct iatt postoldparent;
+    struct iatt prenewparent;
+    struct iatt postnewparent;
+    struct iovec *vector;
+    struct iobref *iobref;
+    struct iobuf *iobuf;
+    gf_dirent_t entries_head;
+    gf_boolean_t is_set_fsid;
+    gf_boolean_t list_inited;
+    shard_post_fop_handler_t handler;
+    shard_post_lookup_shards_fop_handler_t pls_fop_handler;
+    shard_post_resolve_fop_handler_t post_res_handler;
+    shard_post_mknod_fop_handler_t post_mknod_handler;
+    shard_post_update_size_fop_handler_t post_update_size_handler;
+    shard_inodelk_t int_inodelk;
+    shard_entrylk_t int_entrylk;
+    inode_t *resolver_base_inode;
+    gf_boolean_t first_lookup_done;
+    syncbarrier_t barrier;
+    gf_boolean_t lookup_shards_barriered;
+    gf_boolean_t unlink_shards_barriered;
+    gf_boolean_t resolve_not;
+    loc_t newloc;
+    call_frame_t *main_frame;
+    call_frame_t *inodelk_frame;
+    call_frame_t *entrylk_frame;
+    uint32_t deletion_rate;
+    gf_boolean_t cleanup_required;
+    uuid_t base_gfid;
+    char *name;
 } shard_local_t;
 
 typedef struct shard_inode_ctx {
-        uint64_t block_size; /* The block size with which this inode is
-                                sharded */
-        struct iatt stat;
-        gf_boolean_t refresh;
-        /* The following members of inode ctx will be applicable only to the
-         * individual shards' ctx and never the base file ctx.
-         */
-        struct list_head ilist;
-        uuid_t base_gfid;
-        int block_num;
+    uint64_t block_size; /* The block size with which this inode is
+                            sharded */
+    struct iatt stat;
+    gf_boolean_t refresh;
+    /* The following members of inode ctx will be applicable only to the
+     * individual shards' ctx and never the base file ctx.
+     */
+    struct list_head ilist;
+    uuid_t base_gfid;
+    int block_num;
+    gf_boolean_t refreshed;
+    struct list_head to_fsync_list;
+    int fsync_needed;
+    inode_t *inode;
+    int fsync_count;
+    inode_t *base_inode;
 } shard_inode_ctx_t;
 
+typedef enum {
+    SHARD_INTERNAL_DIR_DOT_SHARD = 1,
+    SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME,
+} shard_internal_dir_type_t;
+
 #endif /* __SHARD_H__ */
diff --git a/xlators/features/snapview-client/src/Makefile.am b/xlators/features/snapview-client/src/Makefile.am
index 72d8a2a1973..fa08656c537 100644
--- a/xlators/features/snapview-client/src/Makefile.am
+++ b/xlators/features/snapview-client/src/Makefile.am
@@ -6,9 +6,10 @@ snapview_client_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 snapview_client_la_SOURCES = snapview-client.c
 snapview_client_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = snapview-client.h snapview-client-mem-types.h
+noinst_HEADERS = snapview-client.h snapview-client-mem-types.h snapview-client-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/snapview-client/src/snapview-client-mem-types.h b/xlators/features/snapview-client/src/snapview-client-mem-types.h
index 1a0158d950e..3c3ab555a55 100644
--- a/xlators/features/snapview-client/src/snapview-client-mem-types.h
+++ b/xlators/features/snapview-client/src/snapview-client-mem-types.h
@@ -11,14 +11,14 @@
 #ifndef _SVC_MEM_TYPES_H
 #define _SVC_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum svc_mem_types {
-        gf_svc_mt_svc_private_t = gf_common_mt_end + 1,
-        gf_svc_mt_svc_local_t,
-        gf_svc_mt_svc_inode_t,
-        gf_svc_mt_svc_fd_t,
-        gf_svc_mt_end
+    gf_svc_mt_svc_private_t = gf_common_mt_end + 1,
+    gf_svc_mt_svc_local_t,
+    gf_svc_mt_svc_inode_t,
+    gf_svc_mt_svc_fd_t,
+    gf_svc_mt_end
 };
 
 #endif
diff --git a/xlators/features/snapview-client/src/snapview-client-messages.h b/xlators/features/snapview-client/src/snapview-client-messages.h
new file mode 100644
index 00000000000..c02fb154930
--- /dev/null
+++ b/xlators/features/snapview-client/src/snapview-client-messages.h
@@ -0,0 +1,71 @@
+/*
+ Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _SNAPVIEW_CLIENT_MESSAGES_H_
+#define _SNAPVIEW_CLIENT_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SNAPVIEW_CLIENT, SVC_MSG_NO_MEMORY, SVC_MSG_MEM_ACNT_FAILED,
+           SVC_MSG_SET_INODE_CONTEXT_FAILED, SVC_MSG_GET_INODE_CONTEXT_FAILED,
+           SVC_MSG_DELETE_INODE_CONTEXT_FAILED, SVC_MSG_SET_FD_CONTEXT_FAILED,
+           SVC_MSG_GET_FD_CONTEXT_FAILED, SVC_MSG_DICT_SET_FAILED,
+           SVC_MSG_SUBVOLUME_NULL, SVC_MSG_NO_CHILD_FOR_XLATOR,
+           SVC_MSG_XLATOR_CHILDREN_WRONG, SVC_MSG_NORMAL_GRAPH_LOOKUP_FAIL,
+           SVC_MSG_SNAPVIEW_GRAPH_LOOKUP_FAIL, SVC_MSG_OPENDIR_SPECIAL_DIR,
+           SVC_MSG_RENAME_SNAPSHOT_ENTRY, SVC_MSG_LINK_SNAPSHOT_ENTRY,
+           SVC_MSG_COPY_ENTRY_POINT_FAILED, SVC_MSG_ENTRY_POINT_SPECIAL_DIR,
+           SVC_MSG_STR_LEN, SVC_MSG_INVALID_ENTRY_POINT, SVC_MSG_NULL_PRIV,
+           SVC_MSG_PRIV_DESTROY_FAILED, SVC_MSG_ALLOC_FD_FAILED,
+           SVC_MSG_ALLOC_INODE_FAILED, SVC_MSG_NULL_SPECIAL_DIR,
+           SVC_MSG_MEM_POOL_GET_FAILED);
+
+#define SVC_MSG_ALLOC_FD_FAILED_STR "failed to allocate new fd context"
+#define SVC_MSG_SET_FD_CONTEXT_FAILED_STR "failed to set fd context"
+#define SVC_MSG_STR_LEN_STR                                                    \
+    "destination buffer size is less than the length of entry point name"
+#define SVC_MSG_NORMAL_GRAPH_LOOKUP_FAIL_STR "lookup failed on normal graph"
+#define SVC_MSG_SNAPVIEW_GRAPH_LOOKUP_FAIL_STR "lookup failed on snapview graph"
+#define SVC_MSG_SET_INODE_CONTEXT_FAILED_STR "failed to set inode context"
+#define SVC_MSG_NO_MEMORY_STR "failed to allocate memory"
+#define SVC_MSG_COPY_ENTRY_POINT_FAILED_STR                                    \
+    "failed to copy the entry point string"
+#define SVC_MSG_GET_FD_CONTEXT_FAILED_STR "fd context not found"
+#define SVC_MSG_GET_INODE_CONTEXT_FAILED_STR "failed to get inode context"
+#define SVC_MSG_ALLOC_INODE_FAILED_STR "failed to allocate new inode"
+#define SVC_MSG_DICT_SET_FAILED_STR "failed to set dict"
+#define SVC_MSG_RENAME_SNAPSHOT_ENTRY_STR                                      \
+    "rename happening on a entry residing in snapshot"
+#define SVC_MSG_DELETE_INODE_CONTEXT_FAILED_STR "failed to delete inode context"
+#define SVC_MSG_NULL_PRIV_STR "priv NULL"
+#define SVC_MSG_INVALID_ENTRY_POINT_STR "not a valid entry point"
+#define SVC_MSG_MEM_ACNT_FAILED_STR "Memory accouting init failed"
+#define SVC_MSG_NO_CHILD_FOR_XLATOR_STR "configured without any child"
+#define SVC_MSG_XLATOR_CHILDREN_WRONG_STR                                      \
+    "snap-view-client has got wrong subvolumes. It can have only 2"
+#define SVC_MSG_ENTRY_POINT_SPECIAL_DIR_STR                                    \
+    "entry point directory cannot be part of special directory"
+#define SVC_MSG_NULL_SPECIAL_DIR_STR "null special directory"
+#define SVC_MSG_MEM_POOL_GET_FAILED_STR                                        \
+    "could not get mem pool for frame->local"
+#define SVC_MSG_PRIV_DESTROY_FAILED_STR "failed to destroy private"
+#define SVC_MSG_LINK_SNAPSHOT_ENTRY_STR                                        \
+    "link happening on a entry residin gin snapshot"
+#endif /* !_SNAPVIEW_CLIENT_MESSAGES_H_ */
diff --git a/xlators/features/snapview-client/src/snapview-client.c b/xlators/features/snapview-client/src/snapview-client.c
index 6eb7cc071c2..486c5179d5b 100644
--- a/xlators/features/snapview-client/src/snapview-client.c
+++ b/xlators/features/snapview-client/src/snapview-client.c
@@ -1,536 +1,593 @@
- /*
-   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "snapview-client.h"
-#include "inode.h"
-#include "byte-order.h"
-
+#include <glusterfs/inode.h>
+#include <glusterfs/byte-order.h>
 
 static void
-svc_local_free (svc_local_t *local)
+svc_local_free(svc_local_t *local)
 {
-        if (local) {
-                loc_wipe (&local->loc);
-                if (local->fd)
-                        fd_unref (local->fd);
-                if (local->xdata)
-                        dict_unref (local->xdata);
-                mem_put (local);
-        }
+    if (local) {
+        loc_wipe(&local->loc);
+        if (local->fd)
+            fd_unref(local->fd);
+        if (local->xdata)
+            dict_unref(local->xdata);
+        mem_put(local);
+    }
 }
 
 static xlator_t *
-svc_get_subvolume (xlator_t *this, int inode_type)
+svc_get_subvolume(xlator_t *this, int inode_type)
 {
-        xlator_t *subvolume = NULL;
+    xlator_t *subvolume = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
 
-        if (inode_type == VIRTUAL_INODE)
-                subvolume = SECOND_CHILD (this);
-        else
-                subvolume = FIRST_CHILD (this);
+    if (inode_type == VIRTUAL_INODE)
+        subvolume = SECOND_CHILD(this);
+    else
+        subvolume = FIRST_CHILD(this);
 
 out:
-        return subvolume;
+    return subvolume;
 }
 
 static int32_t
-__svc_inode_ctx_set (xlator_t *this, inode_t *inode, int inode_type)
+__svc_inode_ctx_set(xlator_t *this, inode_t *inode, int inode_type)
 {
-        uint64_t    value = 0;
-        int32_t       ret = -1;
+    uint64_t value = 0;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        value = inode_type;
+    value = inode_type;
 
-        ret = __inode_ctx_set (inode, this, &value);
+    ret = __inode_ctx_set(inode, this, &value);
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-__svc_inode_ctx_get (xlator_t *this, inode_t *inode, int *inode_type)
+__svc_inode_ctx_get(xlator_t *this, inode_t *inode, int *inode_type)
 {
-        uint64_t    value      = 0;
-        int         ret        = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = __inode_ctx_get (inode, this, &value);
-        if (ret < 0)
-                goto out;
+    ret = __inode_ctx_get(inode, this, &value);
+    if (ret < 0)
+        goto out;
 
-        *inode_type = (int)(value);
+    *inode_type = (int)(value);
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-svc_inode_ctx_get (xlator_t *this, inode_t *inode, int *inode_type)
+svc_inode_ctx_get(xlator_t *this, inode_t *inode, int *inode_type)
 {
-        int          ret        = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                ret = __svc_inode_ctx_get (this, inode, inode_type);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __svc_inode_ctx_get(this, inode, inode_type);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 static int32_t
-svc_inode_ctx_set (xlator_t *this, inode_t *inode, int inode_type)
+svc_inode_ctx_set(xlator_t *this, inode_t *inode, int inode_type)
 {
-        int32_t   ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                ret = __svc_inode_ctx_set (this, inode, inode_type);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __svc_inode_ctx_set(this, inode, inode_type);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 static svc_fd_t *
-svc_fd_new (void)
+svc_fd_new(void)
 {
-        svc_fd_t    *svc_fd = NULL;
+    svc_fd_t *svc_fd = NULL;
 
-        svc_fd = GF_CALLOC (1, sizeof (*svc_fd), gf_svc_mt_svc_fd_t);
+    svc_fd = GF_CALLOC(1, sizeof(*svc_fd), gf_svc_mt_svc_fd_t);
 
-        return svc_fd;
+    return svc_fd;
 }
 
 static svc_fd_t *
-__svc_fd_ctx_get (xlator_t *this, fd_t *fd)
+__svc_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svc_fd_t *svc_fd = NULL;
-        uint64_t  value  = 0;
-        int       ret    = -1;
+    svc_fd_t *svc_fd = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = __fd_ctx_get (fd, this, &value);
-        if (ret)
-                return NULL;
+    ret = __fd_ctx_get(fd, this, &value);
+    if (ret)
+        return NULL;
 
-        svc_fd = (svc_fd_t *) ((long) value);
+    svc_fd = (svc_fd_t *)((long)value);
 
 out:
-        return svc_fd;
+    return svc_fd;
 }
 
 static svc_fd_t *
-svc_fd_ctx_get (xlator_t *this, fd_t *fd)
+svc_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svc_fd_t *svc_fd = NULL;
+    svc_fd_t *svc_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                svc_fd = __svc_fd_ctx_get (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        svc_fd = __svc_fd_ctx_get(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return svc_fd;
+    return svc_fd;
 }
 
 static int
-__svc_fd_ctx_set (xlator_t *this, fd_t *fd, svc_fd_t *svc_fd)
+__svc_fd_ctx_set(xlator_t *this, fd_t *fd, svc_fd_t *svc_fd)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, svc_fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, svc_fd, out);
 
-        value = (uint64_t)(long) svc_fd;
+    value = (uint64_t)(long)svc_fd;
 
-        ret = __fd_ctx_set (fd, this, value);
+    ret = __fd_ctx_set(fd, this, value);
 
 out:
-        return ret;
+    return ret;
 }
 
 static svc_fd_t *
-__svc_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+__svc_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        svc_fd_t        *svc_fd    = NULL;
-        int              ret       = -1;
-        inode_t         *inode     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    svc_fd_t *svc_fd = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
 
-        inode = fd->inode;
-        svc_fd = __svc_fd_ctx_get (this, fd);
-        if (svc_fd) {
-                ret = 0;
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        svc_fd = svc_fd_new ();
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate new fd "
-                        "context for gfid %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        ret = __svc_fd_ctx_set (this, fd, svc_fd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set fd context "
-                        "for gfid %s", uuid_utoa (inode->gfid));
-                ret = -1;
-        }
+    inode = fd->inode;
+    svc_fd = __svc_fd_ctx_get(this, fd);
+    if (svc_fd) {
+        ret = 0;
+        goto out;
+    }
+
+    svc_fd = svc_fd_new();
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, SVC_MSG_ALLOC_FD_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    ret = __svc_fd_ctx_set(this, fd, svc_fd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+    }
 
 out:
-        if (ret) {
-                GF_FREE (svc_fd);
-                svc_fd = NULL;
-        }
+    if (ret) {
+        GF_FREE(svc_fd);
+        svc_fd = NULL;
+    }
 
-        return svc_fd;
+    return svc_fd;
 }
 
 static svc_fd_t *
-svc_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+svc_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        svc_fd_t  *svc_fd = NULL;
+    svc_fd_t *svc_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                svc_fd = __svc_fd_ctx_get_or_new (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        svc_fd = __svc_fd_ctx_get_or_new(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return svc_fd;
+    return svc_fd;
 }
 
-
-static int32_t
-gf_svc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+/**
+ * @this: xlator
+ * @entry_point: pointer to the buffer provided by consumer
+ *
+ * This function is mainly for copying the entry point name
+ * (stored as string in priv->path) to a buffer point to by
+ * @entry_point within the lock. It is for the consumer to
+ * allocate the memory for the buffer.
+ *
+ * This function is called by all the functions (or fops)
+ * who need to use priv->path for avoiding the race.
+ * For example, either in lookup or in any other fop,
+ * while priv->path is being accessed, a reconfigure can
+ * happen to change priv->path. This ensures that, a lock
+ * is taken before accessing priv->path.
+ **/
+int
+gf_svc_get_entry_point(xlator_t *this, char *entry_point, size_t dest_size)
 {
-        svc_local_t     *local       = NULL;
-        xlator_t        *subvolume   = NULL;
-        gf_boolean_t     do_unwind   = _gf_true;
-        int              inode_type  = -1;
-        int              ret         = -1;
-
-        local = frame->local;
-        subvolume = local->subvolume;
-        if (!subvolume) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "path: %s, "
-                                  "gfid: %s ", local->loc.path,
-                                  inode?uuid_utoa (inode->gfid):"");
-                GF_ASSERT (0);
-        }
+    int ret = -1;
+    svc_private_t *priv = NULL;
 
-        /* There is a possibility that, the client process just came online
-           and does not have the inode on which the lookup came. In that case,
-           the fresh inode created from fuse for the lookup fop, wont have
-           the inode context set without which svc cannot decide where to
-           STACK_WIND to. So by default it decides to send the fop to the
-           regular subvolume (i.e first child of the xlator). If lookup fails
-           on the regular volume, then there is a possibility that the lookup
-           is happening on a virtual inode (i.e history data residing in snaps).
-           So if lookup fails with ENOENT and the inode context is not there,
-           then send the lookup to the 2nd child of svc.
-
-           If there are any changes in volfile/client-restarted then inode-ctx
-           is lost. In this case if nameless lookup fails with ESTALE,
-           then send the lookup to the 2nd child of svc.
-        */
-        if (op_ret) {
-                if (subvolume == FIRST_CHILD (this)) {
-                        gf_log (this->name,
-                                (op_errno == ENOENT || op_errno == ESTALE)
-                                ? GF_LOG_DEBUG:GF_LOG_ERROR,
-                                "Lookup failed on normal graph with error %s",
-                                strerror (op_errno));
-                } else {
-                        gf_log (this->name,
-                                (op_errno == ENOENT || op_errno == ESTALE)
-                                ? GF_LOG_DEBUG:GF_LOG_ERROR,
-                                "Lookup failed on snapview graph with error %s",
-                                strerror (op_errno));
-                        goto out;
-                }
-
-                if ((op_errno == ENOENT || op_errno == ESTALE) &&
-                    !gf_uuid_is_null (local->loc.gfid)) {
-                        if (inode != NULL)
-                                ret = svc_inode_ctx_get (this, inode,
-                                                                &inode_type);
-
-                        if (ret < 0 || inode == NULL) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Lookup on normal graph failed. "
-                                        "Sending lookup to snapview-server");
-
-                                subvolume = SECOND_CHILD (this);
-                                local->subvolume = subvolume;
-                                STACK_WIND (frame, gf_svc_lookup_cbk,
-                                            subvolume, subvolume->fops->lookup,
-                                            &local->loc, xdata);
-                                do_unwind = _gf_false;
-                        }
-                }
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry_point, out);
 
-                goto out;
-        }
-
-        if (subvolume == FIRST_CHILD (this))
-                inode_type = NORMAL_INODE;
-        else
-                inode_type = VIRTUAL_INODE;
-
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode type"
-                        "into the context");
+    priv = this->private;
 
-out:
-        if (do_unwind) {
-                SVC_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                                  xdata, postparent);
+    LOCK(&priv->lock);
+    {
+        if (dest_size <= strlen(priv->path)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_STR_LEN,
+                    "dest-size=%zu", dest_size, "priv-path-len=%zu",
+                    strlen(priv->path), "path=%s", priv->path, NULL);
+        } else {
+            snprintf(entry_point, dest_size, "%s", priv->path);
+            ret = 0;
         }
+    }
+    UNLOCK(&priv->lock);
 
-        return 0;
+out:
+    return ret;
 }
 
 static int32_t
-gf_svc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+gf_svc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        int32_t        ret         =   -1;
-        svc_local_t   *local       = NULL;
-        xlator_t      *subvolume   = NULL;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        inode_t       *parent      = NULL;
-        svc_private_t *priv        = NULL;
-        dict_t        *new_xdata   = NULL;
-        int            inode_type  = -1;
-        int            parent_type = -1;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (!__is_root_gfid (loc->gfid)) {
-                if (loc->parent) {
-                        parent = inode_ref (loc->parent);
-                        ret = svc_inode_ctx_get (this, loc->parent,
-                                                 &parent_type);
-                } else {
-                        parent = inode_parent (loc->inode, loc->pargfid, NULL);
-                        if (parent)
-                                ret = svc_inode_ctx_get (this, parent,
-                                                         &parent_type);
-                }
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate local");
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        if (__is_root_gfid (loc->inode->gfid)) {
-                subvolume = FIRST_CHILD (this);
-                GF_ASSERT (subvolume);
+    svc_local_t *local = NULL;
+    xlator_t *subvolume = NULL;
+    gf_boolean_t do_unwind = _gf_true;
+    int inode_type = -1;
+    int ret = -1;
+
+    local = frame->local;
+    subvolume = local->subvolume;
+    if (!subvolume) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0, SVC_MSG_SUBVOLUME_NULL,
+                         "path: %s gfid: %s ", local->loc.path,
+                         inode ? uuid_utoa(inode->gfid) : "");
+        GF_ASSERT(0);
+    }
+
+    /* There is a possibility that, the client process just came online
+       and does not have the inode on which the lookup came. In that case,
+       the fresh inode created from fuse for the lookup fop, won't have
+       the inode context set without which svc cannot decide where to
+       STACK_WIND to. So by default it decides to send the fop to the
+       regular subvolume (i.e first child of the xlator). If lookup fails
+       on the regular volume, then there is a possibility that the lookup
+       is happening on a virtual inode (i.e history data residing in snaps).
+       So if lookup fails with ENOENT and the inode context is not there,
+       then send the lookup to the 2nd child of svc.
+
+       If there are any changes in volfile/client-restarted then inode-ctx
+       is lost. In this case if nameless lookup fails with ESTALE,
+       then send the lookup to the 2nd child of svc.
+    */
+    if (op_ret) {
+        if (subvolume == FIRST_CHILD(this)) {
+            gf_smsg(this->name,
+                    (op_errno == ENOENT || op_errno == ESTALE) ? GF_LOG_DEBUG
+                                                               : GF_LOG_ERROR,
+                    op_errno, SVC_MSG_NORMAL_GRAPH_LOOKUP_FAIL, "error=%s",
+                    strerror(op_errno), NULL);
+        } else {
+            gf_smsg(this->name,
+                    (op_errno == ENOENT || op_errno == ESTALE) ? GF_LOG_DEBUG
+                                                               : GF_LOG_ERROR,
+                    op_errno, SVC_MSG_SNAPVIEW_GRAPH_LOOKUP_FAIL, "error=%s",
+                    strerror(op_errno), NULL);
+            goto out;
+        }
+
+        if ((op_errno == ENOENT || op_errno == ESTALE) &&
+            !gf_uuid_is_null(local->loc.gfid)) {
+            if (inode != NULL)
+                ret = svc_inode_ctx_get(this, inode, &inode_type);
+
+            if (ret < 0 || inode == NULL) {
+                gf_msg_debug(this->name, 0,
+                             "Lookup on normal graph failed. "
+                             " Sending lookup to snapview-server");
+                subvolume = SECOND_CHILD(this);
                 local->subvolume = subvolume;
-                wind = _gf_true;
-                goto out;
+                STACK_WIND(frame, gf_svc_lookup_cbk, subvolume,
+                           subvolume->fops->lookup, &local->loc, xdata);
+                do_unwind = _gf_false;
+            }
         }
 
-        /* nfs sends nameless lookups directly using the gfid. In that case
-           loc->name will be NULL. So check if loc->name is NULL. If so, then
-           try to get the subvolume using inode context. But if the inode has
-           not been looked up yet, then send the lookup call to the first
-           subvolume.
-        */
-
-        if (!loc->name) {
-                if (gf_uuid_is_null (loc->inode->gfid)) {
-                        subvolume = FIRST_CHILD (this);
-                        local->subvolume = subvolume;
-                        wind = _gf_true;
-                        goto out;
-                } else {
-                        if (inode_type >= 0)
-                                subvolume = svc_get_subvolume (this,
-                                                               inode_type);
-                        else
-                                subvolume = FIRST_CHILD (this);
-                        local->subvolume = subvolume;
-                        wind = _gf_true;
-                        goto out;
-                }
-        }
+        goto out;
+    }
 
-        if (strcmp (loc->name, priv->path)) {
-                if (parent_type == NORMAL_INODE) {
-                        subvolume = FIRST_CHILD (this);
-                        local->subvolume = subvolume;
-                } else {
-                        subvolume = SECOND_CHILD (this);
-                        local->subvolume = subvolume;
-                }
-        } else {
-                subvolume = SECOND_CHILD (this);
-                local->subvolume = subvolume;
-                if (parent_type == NORMAL_INODE) {
-                        /* Indication of whether the lookup is happening on the
-                           entry point or not, to the snapview-server.
-                        */
-                        SVC_ENTRY_POINT_SET (this, xdata, op_ret, op_errno,
-                                             new_xdata, priv, ret, out);
-                }
-        }
+    if (subvolume == FIRST_CHILD(this))
+        inode_type = NORMAL_INODE;
+    else
+        inode_type = VIRTUAL_INODE;
 
-        wind = _gf_true;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
 
 out:
-        if (wind)
-                STACK_WIND (frame, gf_svc_lookup_cbk, subvolume,
-                            subvolume->fops->lookup, loc, xdata);
-        else
-                SVC_STACK_UNWIND (lookup, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL, NULL);
-        if (new_xdata)
-                dict_unref (new_xdata);
-
-        if (parent)
-                inode_unref (parent);
+    if (do_unwind) {
+        SVC_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                         postparent);
+    }
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+gf_svc_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        xlator_t      *subvolume  = NULL;
-        int32_t        ret        = -1;
-        int            inode_type = -1;
-        int32_t        op_ret     = -1;
-        int32_t        op_errno   = EINVAL;
-        gf_boolean_t   wind       = _gf_false;
-        svc_private_t  *priv      = NULL;
-        const char     *path      = NULL;
-        int             path_len  = -1;
-        int             snap_len  = -1;
-        loc_t           root_loc  = {0,};
-        loc_t          *temp_loc  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
-        path_len = strlen (loc->path);
-        snap_len = strlen (priv->path);
-        temp_loc = loc;
-
-        if (path_len >= snap_len && inode_type == VIRTUAL_INODE) {
-                path = &loc->path[path_len - snap_len];
-                if (!strcmp (path, priv->path)) {
-                        /*
-                         * statfs call for virtual snap directory.
-                         * Sent the fops to parent volume by removing
-                         * virtual directory from path
-                         */
-                        subvolume = FIRST_CHILD (this);
-                        root_loc.path = gf_strdup("/");
-                        gf_uuid_clear(root_loc.gfid);
-                        root_loc.gfid[15] = 1;
-                        root_loc.inode = inode_ref (loc->inode->table->root);
-                        temp_loc = &root_loc;
-                }
+    int32_t ret = -1;
+    svc_local_t *local = NULL;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    inode_t *parent = NULL;
+    dict_t *new_xdata = NULL;
+    int inode_type = -1;
+    int parent_type = -1;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (!__is_root_gfid(loc->gfid)) {
+        if (loc->parent) {
+            parent = inode_ref(loc->parent);
+            ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+        } else {
+            parent = inode_parent(loc->inode, loc->pargfid, NULL);
+            if (parent)
+                ret = svc_inode_ctx_get(this, parent, &parent_type);
+        }
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    frame->local = local;
+    loc_copy(&local->loc, loc);
+
+    if (__is_root_gfid(loc->inode->gfid)) {
+        subvolume = FIRST_CHILD(this);
+        GF_ASSERT(subvolume);
+        local->subvolume = subvolume;
+        wind = _gf_true;
+        goto out;
+    }
+
+    /* nfs sends nameless lookups directly using the gfid. In that case
+       loc->name will be NULL. So check if loc->name is NULL. If so, then
+       try to get the subvolume using inode context. But if the inode has
+       not been looked up yet, then send the lookup call to the first
+       subvolume.
+    */
+
+    if (!loc->name) {
+        if (gf_uuid_is_null(loc->inode->gfid)) {
+            subvolume = FIRST_CHILD(this);
+            local->subvolume = subvolume;
+            wind = _gf_true;
+            goto out;
+        } else {
+            if (inode_type >= 0)
+                subvolume = svc_get_subvolume(this, inode_type);
+            else
+                subvolume = FIRST_CHILD(this);
+            local->subvolume = subvolume;
+            wind = _gf_true;
+            goto out;
+        }
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point)) {
+        if (parent_type == VIRTUAL_INODE) {
+            subvolume = SECOND_CHILD(this);
+        } else {
+            /*
+             * Either parent type is normal graph, or the parent
+             * type is uncertain.
+             */
+            subvolume = FIRST_CHILD(this);
         }
+        local->subvolume = subvolume;
+    } else {
+        subvolume = SECOND_CHILD(this);
+        local->subvolume = subvolume;
+        if (parent_type == NORMAL_INODE) {
+            /* Indication of whether the lookup is happening on the
+               entry point or not, to the snapview-server.
+            */
+            SVC_ENTRY_POINT_SET(this, xdata, op_ret, op_errno, new_xdata, ret,
+                                out);
+        }
+    }
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->statfs,
-                         temp_loc, xdata);
-        if (temp_loc == &root_loc)
-                loc_wipe (temp_loc);
+    wind = _gf_true;
 
-        wind = _gf_true;
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (statfs, frame, op_ret, op_errno,
-                                  NULL, NULL);
-        return 0;
+    if (wind)
+        STACK_WIND(frame, gf_svc_lookup_cbk, subvolume, subvolume->fops->lookup,
+                   loc, xdata);
+    else
+        SVC_STACK_UNWIND(lookup, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL);
+    if (new_xdata)
+        dict_unref(new_xdata);
+
+    if (parent)
+        inode_unref(parent);
+
+    return 0;
 }
 
 static int32_t
-gf_svc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                 dict_t *xdata)
+gf_svc_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        /* Consider a testcase:
-         * #mount -t nfs host1:/vol1 /mnt
-         * #ls /mnt
-         * #ls /mnt/.snaps (As expected this fails)
-         * #gluster volume set vol1 features.uss enable
-         * Now `ls /mnt/.snaps` should work,
-         * but fails with No such file or directory.
-         * This is because NFS client caches the list of files in
-         * a directory. This cache is updated if there are any changes
-         * in the directory attributes. To solve this problem change
-         * a attribute 'ctime' when USS is enabled
-         */
-        if (op_ret == 0 && IA_ISDIR(buf->ia_type))
-                buf->ia_ctime_nsec++;
+    xlator_t *subvolume = NULL;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_private_t *priv = NULL;
+    const char *path = NULL;
+    int path_len = -1;
+    int snap_len = -1;
+    loc_t root_loc = {
+        0,
+    };
+    loc_t *temp_loc = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    priv = this->private;
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
+    path_len = strlen(loc->path);
+    snap_len = strlen(priv->path);
+    temp_loc = loc;
+
+    if (path_len >= snap_len && inode_type == VIRTUAL_INODE) {
+        path = &loc->path[path_len - snap_len];
+        if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+            goto out;
+        }
+
+        if (!strcmp(path, entry_point)) {
+            /*
+             * statfs call for virtual snap directory.
+             * Sent the fops to parent volume by removing
+             * virtual directory from path
+             */
+            subvolume = FIRST_CHILD(this);
+            root_loc.path = gf_strdup("/");
+            gf_uuid_clear(root_loc.gfid);
+            root_loc.gfid[15] = 1;
+            root_loc.inode = inode_ref(loc->inode->table->root);
+            temp_loc = &root_loc;
+        }
+    }
+
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->statfs, temp_loc, xdata);
+    if (temp_loc == &root_loc)
+        loc_wipe(temp_loc);
+
+    wind = _gf_true;
+out:
+    if (!wind)
+        SVC_STACK_UNWIND(statfs, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
+}
 
-        SVC_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+static int32_t
+gf_svc_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
+{
+    /* TODO: FIX ME
+     * Consider a testcase:
+     * #mount -t nfs host1:/vol1 /mnt
+     * #ls /mnt
+     * #ls /mnt/.snaps (As expected this fails)
+     * #gluster volume set vol1 features.uss enable
+     * Now `ls /mnt/.snaps` should work, but fails with No such file or
+     * directory. This is because NFS client (gNFS) caches the list of files
+     * in a directory. This cache is updated if there are any changes in the
+     * directory attributes. So, one way to solve this problem is to change
+     * 'ctime' attribute when USS is enabled as below.
+     *
+     * if (op_ret == 0 && IA_ISDIR(buf->ia_type))
+     *     buf->ia_ctime_nsec++;
+     *
+     * But this is not the ideal solution as applications see the unexpected
+     * ctime change causing failures.
+     */
+
+    SVC_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
 /* should all the fops be handled like lookup is supposed to be
@@ -538,122 +595,119 @@ gf_svc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
    be sent and in the call back update the contexts.
 */
 static int32_t
-gf_svc_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             dict_t *xdata)
+gf_svc_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        xlator_t    *subvolume  = NULL;
-        int32_t      op_ret     = -1;
-        int32_t      op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND (frame, gf_svc_stat_cbk, subvolume,
-                    subvolume->fops->stat, loc, xdata);
+    STACK_WIND(frame, gf_svc_stat_cbk, subvolume, subvolume->fops->stat, loc,
+               xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (stat, frame, op_ret, op_errno,
-                                  NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(stat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+gf_svc_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        xlator_t    *subvolume  = NULL;
-        int32_t      op_ret     = -1;
-        int32_t      op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->fstat, fd, xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->fstat, fd, xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (fstat, frame, op_ret, op_errno, NULL, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(fstat, frame, op_ret, op_errno, NULL, NULL);
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-gf_svc_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+gf_svc_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        svc_fd_t        *svc_fd          = NULL;
-        svc_local_t     *local           = NULL;
-        svc_private_t   *priv            = NULL;
-        gf_boolean_t     special_dir     = _gf_false;
-        char             path[PATH_MAX]  = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        if (op_ret)
-                goto out;
+    svc_fd_t *svc_fd = NULL;
+    svc_local_t *local = NULL;
+    svc_private_t *priv = NULL;
+    gf_boolean_t special_dir = _gf_false;
+    char path[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    if (op_ret)
+        goto out;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (local->subvolume == FIRST_CHILD(this) && priv->special_dir &&
+        strcmp(priv->special_dir, "")) {
+        if (!__is_root_gfid(fd->inode->gfid))
+            snprintf(path, sizeof(path), "%s/.", priv->special_dir);
+        else
+            snprintf(path, sizeof(path), "/.");
 
-        priv = this->private;
-        local = frame->local;
-
-        if (local->subvolume == FIRST_CHILD (this) && priv->special_dir
-            && strcmp (priv->special_dir, "")) {
-                if (!__is_root_gfid (fd->inode->gfid))
-                        snprintf (path, sizeof (path), "%s/.",
-                                  priv->special_dir);
-                else
-                        snprintf (path, sizeof (path), "/.");
-
-                if (!strcmp (local->loc.path, priv->special_dir) ||
-                    !strcmp (local->loc.path, path)) {
-                        gf_log_callingfn (this->name, GF_LOG_DEBUG,
-                                          "got opendir on special "
-                                          "directory %s (%s)", path,
-                                          uuid_utoa (fd->inode->gfid));
-                        special_dir = _gf_true;
-                }
+        if (!strcmp(local->loc.path, priv->special_dir) ||
+            !strcmp(local->loc.path, path)) {
+            gf_msg_debug(this->name, 0,
+                         "got opendir on special directory"
+                         " %s (gfid: %s)",
+                         path, uuid_utoa(fd->inode->gfid));
+            special_dir = _gf_true;
         }
+    }
 
-        if (special_dir) {
-                svc_fd = svc_fd_ctx_get_or_new (this, fd);
-                if (!svc_fd) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fd context not found for %s",
-                                uuid_utoa (fd->inode->gfid));
-                        goto out;
-                }
-
-                svc_fd->last_offset = -1;
-                svc_fd->special_dir = special_dir;
+    if (special_dir) {
+        svc_fd = svc_fd_ctx_get_or_new(this, fd);
+        if (!svc_fd) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto out;
         }
 
+        svc_fd->last_offset = -1;
+        svc_fd->special_dir = special_dir;
+    }
+
 out:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 /* If the inode represents a directory which is actually
    present in a snapshot, then opendir on that directory
    should be sent to the snap-view-server which opens
@@ -665,93 +719,91 @@ out:
    svc has to do things that open-behind is doing.
 */
 static int32_t
-gf_svc_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-                dict_t *xdata)
+gf_svc_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
 {
-        int32_t        ret        = -1;
-        int            inode_type = -1;
-        xlator_t      *subvolume  = NULL;
-        int            op_ret     = -1;
-        int            op_errno   = EINVAL;
-        gf_boolean_t   wind       = _gf_false;
-        svc_local_t   *local      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate memory "
-                        "for local (path: %s, gfid: %s)", loc->path,
-                        uuid_utoa (fd->inode->gfid));
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
-
-        loc_copy (&local->loc, loc);
-        local->subvolume = subvolume;
-        frame->local = local;
-
-        STACK_WIND (frame, gf_svc_opendir_cbk, subvolume,
-                    subvolume->fops->opendir, loc, fd, xdata);
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_local_t *local = NULL;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY,
+                "path=%s", loc->path, "gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto out;
+    }
+    loc_copy(&local->loc, loc);
+    frame->local = local;
+
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
+    local->subvolume = subvolume;
+
+    STACK_WIND(frame, gf_svc_opendir_cbk, subvolume, subvolume->fops->opendir,
+               loc, fd, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (opendir, frame, op_ret, op_errno, NULL, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(opendir, frame, op_ret, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+gf_svc_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s (gfid: %s)", loc->path,
-                        uuid_utoa (loc->inode->gfid));
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->setattr, loc, stbuf,
-                                 valid, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "path=%s", loc->path,
+                "gfid= %s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                        xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(setattr, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* XXX: This function is currently not used. Remove "#if 0" when required */
@@ -773,10 +825,12 @@ gf_svc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         ret = svc_inode_ctx_get (this, fd->inode, &inode_type);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
                 op_ret = -1;
                 op_errno = EINVAL;
+                gf_msg (this->name, GF_LOG_ERROR, op_errno,
+                        SVC_MSG_GET_INODE_CONTEXT_FAILED, "failed to "
+                        "get the inode context for %s",
+                        uuid_utoa (fd->inode->gfid));
                 goto out;
         }
 
@@ -801,83 +855,88 @@ out:
 #endif /* gf_svc_fsetattr() is not used */
 
 static int32_t
-gf_svc_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name, dict_t *xdata)
+gf_svc_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        int32_t          ret                    = -1;
-        int              inode_type             = -1;
-        xlator_t        *subvolume              = NULL;
-        int              op_ret                 = -1;
-        int              op_errno               = EINVAL;
-        gf_boolean_t     wind                   = _gf_false;
-        svc_private_t   *priv                   = NULL;
-        char             attrname[PATH_MAX]     = "";
-        char             attrval[64]            = "";
-        dict_t          *dict                   = NULL;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_private_t *priv = NULL;
+    char attrname[PATH_MAX] = "";
+    char attrval[64] = "";
+    dict_t *dict = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    /*
+     * Samba sends this special key for case insensitive
+     * filename check. This request comes with a parent
+     * path and with a special key GF_XATTR_GET_REAL_FILENAME_KEY.
+     * e.g. "glusterfs.get_real_filename:.snaps".
+     * If the name variable matches this key then we have
+     * to send back .snaps as the real filename.
+     */
+    if (!name)
+        goto stack_wind;
+
+    sscanf(name, "%[^:]:%[^@]", attrname, attrval);
+    strcat(attrname, ":");
+
+    if (!strcmp(attrname, GF_XATTR_GET_REAL_FILENAME_KEY)) {
+        if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+            goto out;
+        }
+
+        if (!strcasecmp(attrval, entry_point)) {
+            dict = dict_new();
+            if (NULL == dict) {
+                op_errno = ENOMEM;
+                goto out;
+            }
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        /*
-         * Samba sends this special key for case insensitive
-         * filename check. This request comes with a parent
-         * path and with a special key GF_XATTR_GET_REAL_FILENAME_KEY.
-         * e.g. "glusterfs.get_real_filename:.snaps".
-         * If the name variable matches this key then we have
-         * to send back .snaps as the real filename.
-         */
-        if (!name)
-                goto stack_wind;
-
-        sscanf (name, "%[^:]:%[^@]", attrname, attrval);
-        strcat (attrname, ":");
-
-        if (!strcmp (attrname, GF_XATTR_GET_REAL_FILENAME_KEY)) {
-                if (!strcasecmp (attrval, priv->path)) {
-                        dict = dict_new ();
-                        if (NULL == dict) {
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        ret = dict_set_dynstr_with_alloc (dict,
-                                        (char *)name,
-                                        priv->path);
-
-                        if (ret) {
-                                op_errno = ENOMEM;
-                                dict_unref (dict);
-                                goto out;
-                        }
-
-                        op_errno = 0;
-                        op_ret = strlen (priv->path) + 1;
-                        /* We should return from here */
-                        goto out;
-                }
+            ret = dict_set_dynstr_with_alloc(dict, (char *)name, entry_point);
+
+            if (ret) {
+                op_errno = ENOMEM;
+                goto out;
+            }
+
+            op_errno = 0;
+            op_ret = strlen(entry_point) + 1;
+            /* We should return from here */
+            goto out;
         }
+    }
 stack_wind:
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->getxattr, loc, name,
-                         xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->getxattr, loc, name,
+                    xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
-                                  dict, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, NULL);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 /* XXX: This function is currently not used. Mark it '#if 0' when required */
@@ -915,279 +974,286 @@ out:
 #endif /* gf_svc_fgetxattr() is not used */
 
 static int32_t
-gf_svc_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags, dict_t *xdata)
+gf_svc_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int32_t      ret         = -1;
-        int          inode_type  = -1;
-        int          op_ret      = -1;
-        int          op_errno    = EINVAL;
-        gf_boolean_t wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->setxattr, loc, dict,
-                                 flags, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "name=%s", loc->name,
+                "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                        xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (setxattr, frame, op_ret, op_errno,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(setxattr, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                  int32_t flags, dict_t *xdata)
+gf_svc_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        ret = svc_inode_ctx_get (this, fd->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->fsetxattr, fd, dict,
-                                 flags, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    ret = svc_inode_ctx_get(this, fd->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                        xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno,
-                                     NULL);
+    if (!wind)
+        STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-              dict_t *xdata)
+gf_svc_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
-        int          inode_type = -1;
-        int          ret        = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->rmdir, loc, flags,
-                                 xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int inode_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "name=%s", loc->name,
+                "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+gf_svc_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int inode_type = -1;
-        int ret        = -1;
-
-        if (op_ret < 0)
-                goto out;
+    int inode_type = -1;
+    int ret = -1;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    if (op_ret < 0)
+        goto out;
 
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode,
-                          buf, preparent, postparent, xdata);
-        return 0;
+    SVC_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
 static int32_t
-gf_svc_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              mode_t umask, dict_t *xdata)
+gf_svc_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            ret         = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, gf_svc_mkdir_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->mkdir, loc, mode,
-                            umask, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (mkdir, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(mkdir, frame, op_ret, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+gf_svc_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int inode_type = -1;
-        int ret        = -1;
+    int inode_type = -1;
+    int ret = -1;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                          buf, preparent, postparent, xdata);
-        return 0;
+    SVC_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
 static int32_t
-gf_svc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *xdata)
+gf_svc_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            ret         = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, gf_svc_mknod_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->mknod, loc, mode,
-                            rdev, umask, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_mknod_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                   xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
 /* If the flags of the open call contain O_WRONLY or O_RDWR and the inode is
@@ -1195,425 +1261,447 @@ out:
    STACK_WIND the call to the first child of svc xlator.
 */
 static int32_t
-gf_svc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-             fd_t *fd, dict_t *xdata)
+gf_svc_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            fd_t *fd, dict_t *xdata)
 {
-        xlator_t    *subvolume  = NULL;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        int          ret        = -1;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        /* Another way is to STACK_WIND to normal subvolume, if inode
-           type is not there in the context. If the file actually resides
-           in snapshots, then ENOENT would be returned. Needs more analysis.
-        */
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
-
-        if (((flags & O_ACCMODE) == O_WRONLY) ||
-            ((flags & O_ACCMODE) == O_RDWR)) {
-                if (subvolume != FIRST_CHILD (this)) {
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        goto out;
-                }
-        }
-
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->open, loc,
-                         flags, fd, xdata);
-
-        wind = _gf_true;
+    xlator_t *subvolume = NULL;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    int ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    /* Another way is to STACK_WIND to normal subvolume, if inode
+       type is not there in the context. If the file actually resides
+       in snapshots, then ENOENT would be returned. Needs more analysis.
+    */
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
+
+    if (((flags & O_ACCMODE) == O_WRONLY) || ((flags & O_ACCMODE) == O_RDWR)) {
+        if (subvolume != FIRST_CHILD(this)) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
+    }
+
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->open, loc, flags, fd,
+                    xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (open, frame, op_ret, op_errno, NULL,
-                                  NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(open, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                   struct iatt *stbuf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+gf_svc_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        int  inode_type = -1;
-        int  ret        = -1;
+    int inode_type = -1;
+    int ret = -1;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
-                          inode, stbuf, preparent, postparent, xdata);
+    SVC_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                     preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+gf_svc_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            ret         = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, gf_svc_create_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->create, loc, flags,
-                            mode, umask, fd, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_create_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+                   xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (create, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(create, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL, NULL, NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+gf_svc_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        int inode_type = -1;
-        int ret        = -1;
+    int inode_type = -1;
+    int ret = -1;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode,
-                          buf, preparent, postparent, xdata);
+    SVC_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, mode_t umask, dict_t *xdata)
+gf_svc_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        int            ret         = -1;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, gf_svc_symlink_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->symlink, linkpath, loc,
-                            umask, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    int ret = -1;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_symlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                   xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (symlink, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(symlink, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL, NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-               dict_t *xdata)
+gf_svc_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-        int            inode_type   = -1;
-        int            op_ret       = -1;
-        int            op_errno     = EINVAL;
-        int            ret          = -1;
-        gf_boolean_t   wind         = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->unlink, loc, flags,
-                                 xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    int ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->unlink, loc, flags, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (unlink, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, uint32_t flags, dict_t *xdata)
+gf_svc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int           inode_type = -1;
-        xlator_t     *subvolume  = NULL;
-        int           ret        = -1;
-        int           op_ret     = -1;
-        int           op_errno   = EINVAL;
-        gf_boolean_t  wind       = _gf_false;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->readv,
-                         fd, size, offset, flags, xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->readv, fd, size, offset,
+                    flags, xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (readv, frame, op_ret, op_errno, NULL, 0, NULL,
-                                  NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(readv, frame, op_ret, op_errno, NULL, 0, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                 dict_t *xdata)
+gf_svc_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
-        int              inode_type = -1;
-        xlator_t        *subvolume  = NULL;
-        int              ret        = -1;
-        int              op_ret     = -1;
-        int              op_errno   = EINVAL;
-        gf_boolean_t     wind       = _gf_false;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->readlink, loc, size,
-                         xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->readlink, loc, size,
+                    xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, NULL, NULL,
-                                     NULL);
-        return 0;
+    if (!wind)
+        STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, NULL, NULL,
+                            NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-               dict_t *xdata)
+gf_svc_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
-        int            ret        = -1;
-        int            inode_type = -1;
-        xlator_t      *subvolume  = NULL;
-        int            op_ret     = -1;
-        int            op_errno   = EINVAL;
-        gf_boolean_t   wind       = _gf_false;
+    int ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->access, loc, mask,
-                         xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->access, loc, mask,
+                    xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (access, frame, op_ret, op_errno, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(access, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-gf_svc_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                    dict_t *xdata)
+gf_svc_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
 {
-        gf_dirent_t   *entry      = NULL;
-        gf_dirent_t   *tmpentry  = NULL;
-        svc_local_t   *local      = NULL;
-        svc_private_t *priv       = NULL;
-
-        if (op_ret < 0)
-                goto out;
-
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        priv = this->private;
-        local = frame->local;
-
-        /* If .snaps pre-exists, then it should not be listed
-         * in the NORMAL INODE directory when USS is enabled,
-         * so filter the .snaps entry if exists.
-         * However it is OK to list .snaps in VIRTUAL world
-         */
-        if (local->subvolume != FIRST_CHILD (this))
-                goto out;
-
-        list_for_each_entry_safe (entry, tmpentry, &entries->list, list) {
-                if (strcmp(priv->path, entry->d_name) == 0)
-                        gf_dirent_entry_free (entry);
-        }
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmpentry = NULL;
+    svc_local_t *local = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto out;
+
+    local = frame->local;
+
+    /* If .snaps pre-exists, then it should not be listed
+     * in the NORMAL INODE directory when USS is enabled,
+     * so filter the .snaps entry if exists.
+     * However it is OK to list .snaps in VIRTUAL world
+     */
+    if (local->subvolume != FIRST_CHILD(this))
+        goto out;
+
+    /*
+     * Better to goto out if getting the entry point
+     * fails. We might end up sending the directory
+     * entry for the snapview entry point in the readdir
+     * response. But, the intention is to avoid the race
+     * condition where priv->path is being changed in
+     * reconfigure while this is accessing it.
+     */
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    list_for_each_entry_safe(entry, tmpentry, &entries->list, list)
+    {
+        if (strcmp(entry_point, entry->d_name) == 0)
+            gf_dirent_entry_free(entry);
+    }
 
 out:
-        SVC_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, xdata);
-        return 0;
+    SVC_STACK_UNWIND(readdir, frame, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
 static int32_t
-gf_svc_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t off, dict_t *xdata)
+gf_svc_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t off, dict_t *xdata)
 {
-        int           inode_type = -1;
-        xlator_t     *subvolume  = NULL;
-        svc_local_t  *local      = NULL;
-        int           ret        = -1;
-        int           op_ret     = -1;
-        int           op_errno   = EINVAL;
-        gf_boolean_t  wind       = _gf_false;
-        svc_fd_t     *svc_fd     = NULL;
-        gf_dirent_t   entries;
-
-        INIT_LIST_HEAD (&entries);
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        svc_fd = svc_fd_ctx_get_or_new (this,  fd);
-        if (!svc_fd)
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (fd->inode->gfid));
-        else {
-                if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
-                        op_ret = 0;
-                        op_errno = ENOENT;
-                        goto out;
-                }
-        }
-
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate local");
-                goto out;
-        }
-        local->subvolume = subvolume;
-        frame->local = local;
-
-	STACK_WIND (frame, gf_svc_readdir_cbk, subvolume,
-                    subvolume->fops->readdir, fd, size, off, xdata);
-
-        wind = _gf_true;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    svc_local_t *local = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_fd_t *svc_fd = NULL;
+    gf_dirent_t entries;
+
+    INIT_LIST_HEAD(&entries);
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    svc_fd = svc_fd_ctx_get_or_new(this, fd);
+    if (!svc_fd)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+    else {
+        if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
+            op_ret = 0;
+            op_errno = ENOENT;
+            goto out;
+        }
+    }
+
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY,
+                "inode-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+    local->subvolume = subvolume;
+    frame->local = local;
+
+    STACK_WIND(frame, gf_svc_readdir_cbk, subvolume, subvolume->fops->readdir,
+               fd, size, off, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, NULL);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -1641,814 +1729,1063 @@ out:
  */
 
 static int32_t
-gf_svc_readdirp_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xdata,
-                            struct iatt *postparent)
+gf_svc_readdirp_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
 {
-        gf_dirent_t    entries;
-        gf_dirent_t   *entry      = NULL;
-        svc_private_t *private    = NULL;
-        svc_fd_t      *svc_fd     = NULL;
-        svc_local_t   *local      = NULL;
-        int            inode_type = -1;
-        int            ret        = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    svc_fd_t *svc_fd = NULL;
+    svc_local_t *local = NULL;
+    int inode_type = -1;
+    int ret = -1;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+
+    INIT_LIST_HEAD(&entries.list);
+
+    local = frame->local;
+
+    if (op_ret) {
+        if (op_errno == ESTALE && !local->revalidate) {
+            local->revalidate = 1;
+            ret = gf_svc_special_dir_revalidate_lookup(frame, this, xdata);
+
+            if (!ret)
+                return 0;
+        }
+        op_ret = 0;
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    svc_fd = svc_fd_ctx_get(this, local->fd);
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(local->fd->inode->gfid), NULL);
+        op_ret = 0;
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_COPY_ENTRY_POINT_FAILED,
+                NULL);
+        op_ret = 0;
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    entry = gf_dirent_for_name(entry_point);
+    if (!entry) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_MEMORY,
+                "entry-point=%s", entry_point, NULL);
+        op_ret = 0;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    entry->inode = inode_ref(inode);
+    entry->d_off = svc_fd->last_offset + 22;
+    entry->d_ino = buf->ia_ino;
+    entry->d_type = DT_DIR;
+    entry->d_stat = *buf;
+    inode_type = VIRTUAL_INODE;
+    ret = svc_inode_ctx_set(this, entry->inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                "entry-name=%s", entry->d_name, NULL);
+
+    list_add_tail(&entry->list, &entries.list);
+    op_ret = 1;
+    svc_fd->last_offset = entry->d_off;
+    svc_fd->entry_point_handled = _gf_true;
 
-        private = this->private;
-        INIT_LIST_HEAD (&entries.list);
-
-        local = frame->local;
-
-        if (local->xdata != NULL)
-                dict_unref (xdata);
-
-        if (op_ret) {
-                op_ret = 0;
-                op_errno = ENOENT;
-                goto out;
-        }
-
-        svc_fd = svc_fd_ctx_get (this, local->fd);
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (local->fd->inode->gfid));
-                op_ret = 0;
-                op_errno = ENOENT;
-                goto out;
-        }
-
-        entry = gf_dirent_for_name (private->path);
-        if (!entry) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate memory "
-                        "for the entry %s", private->path);
-                op_ret = 0;
-                op_errno = ENOMEM;
-                goto out;
-        }
+out:
+    SVC_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries,
+                     local ? local->xdata : NULL);
 
-        entry->inode = inode_ref (inode);
-        entry->d_off = svc_fd->last_offset + 22;
-        entry->d_ino = buf->ia_ino;
-        entry->d_type = DT_DIR;
-        entry->d_stat = *buf;
-        inode_type = VIRTUAL_INODE;
-        ret = svc_inode_ctx_set (this, entry->inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set the inode "
-                        "context");
+    gf_dirent_free(&entries);
 
-        list_add_tail (&entry->list, &entries.list);
-        op_ret = 1;
-        svc_fd->last_offset = entry->d_off;
-        svc_fd->entry_point_handled = _gf_true;
+    return 0;
+}
 
+int
+gf_svc_special_dir_revalidate_lookup(call_frame_t *frame, xlator_t *this,
+                                     dict_t *xdata)
+{
+    svc_local_t *local = NULL;
+    loc_t *loc = NULL;
+    dict_t *tmp_xdata = NULL;
+    char *path = NULL;
+    int ret = -1;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+
+    local = frame->local;
+    loc = &local->loc;
+
+    if (local->xdata) {
+        dict_unref(local->xdata);
+        local->xdata = NULL;
+    }
+
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+
+    inode_unref(loc->inode);
+    loc->inode = inode_new(loc->parent->table);
+    if (!loc->inode) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, SVC_MSG_ALLOC_INODE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_COPY_ENTRY_POINT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(local->loc.gfid, loc->inode->gfid);
+    ret = inode_path(loc->parent, entry_point, &path);
+    if (ret < 0)
+        goto out;
+
+    if (loc->path)
+        GF_FREE((char *)loc->path);
+
+    loc->path = gf_strdup(path);
+    if (loc->path) {
+        if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+            loc->name = strrchr(loc->path, '/');
+            if (loc->name)
+                loc->name++;
+        }
+    } else
+        loc->path = NULL;
+
+    tmp_xdata = dict_new();
+    if (!tmp_xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(tmp_xdata, "entry-point", "true");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_DICT_SET_FAILED, NULL);
+        goto out;
+    }
+
+    STACK_WIND(frame, gf_svc_readdirp_lookup_cbk, SECOND_CHILD(this),
+               SECOND_CHILD(this)->fops->lookup, loc, tmp_xdata);
 out:
-        SVC_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries,
-                          local->xdata);
-
-        gf_dirent_free (&entries);
+    if (tmp_xdata)
+        dict_unref(tmp_xdata);
 
-        return 0;
+    GF_FREE(path);
+    return ret;
 }
 
 static gf_boolean_t
-gf_svc_readdir_on_special_dir (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, gf_dirent_t *entries,
-                               dict_t *xdata)
+gf_svc_readdir_on_special_dir(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno,
+                              gf_dirent_t *entries, dict_t *xdata)
 {
-        svc_local_t   *local      = NULL;
-        svc_private_t *private    = NULL;
-        inode_t       *inode      = NULL;
-        fd_t          *fd         = NULL;
-        char          *path       = NULL;
-        loc_t         *loc        = NULL;
-        dict_t        *tmp_xdata  = NULL;
-        int            ret        = -1;
-        gf_boolean_t   unwind     = _gf_true;
-        svc_fd_t      *svc_fd     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        private = this->private;
-        local = frame->local;
-
-        loc = &local->loc;
-        fd = local->fd;
-        svc_fd = svc_fd_ctx_get (this, fd);
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (fd->inode->gfid));
+    svc_local_t *local = NULL;
+    svc_private_t *private = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    char *path = NULL;
+    loc_t *loc = NULL;
+    dict_t *tmp_xdata = NULL;
+    int ret = -1;
+    gf_boolean_t unwind = _gf_true;
+    svc_fd_t *svc_fd = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+   private
+    = this->private;
+    local = frame->local;
+
+    loc = &local->loc;
+    fd = local->fd;
+    svc_fd = svc_fd_ctx_get(this, fd);
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    /*
+     * check if its end of readdir operation from posix, if special_dir
+     * option is set, if readdir is done on special directory and if
+     * readdirp is from normal regular graph.
+     */
+
+    if (!private->show_entry_point)
+        goto out;
+
+    if (op_ret == 0 && op_errno == ENOENT && private->special_dir &&
+        strcmp(private->special_dir, "") && svc_fd->special_dir &&
+        local->subvolume == FIRST_CHILD(this)) {
+        if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    SVC_MSG_GET_FD_CONTEXT_FAILED, NULL);
+            goto out;
+        }
+
+        inode = inode_grep(fd->inode->table, fd->inode, entry_point);
+        if (!inode) {
+            inode = inode_new(fd->inode->table);
+            if (!inode) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_ALLOC_INODE_FAILED,
+                        NULL);
                 goto out;
+            }
         }
 
-        /*
-         * check if its end of readdir operation from posix, if special_dir
-         * option is set, if readdir is done on special directory and if
-         * readdirp is from normal regular graph.
-         */
+        gf_uuid_copy(local->loc.pargfid, fd->inode->gfid);
+        gf_uuid_copy(local->loc.gfid, inode->gfid);
+        if (gf_uuid_is_null(inode->gfid))
+            ret = inode_path(fd->inode, entry_point, &path);
+        else
+            ret = inode_path(inode, NULL, &path);
 
-        if (!private->show_entry_point)
-                goto out;
+        if (ret < 0)
+            goto out;
+        loc->path = gf_strdup(path);
+        if (loc->path) {
+            if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+                loc->name = strrchr(loc->path, '/');
+                if (loc->name)
+                    loc->name++;
+            }
+        }
+
+        loc->inode = inode;
+        loc->parent = inode_ref(fd->inode);
+        tmp_xdata = dict_new();
+        if (!tmp_xdata)
+            goto out;
+        ret = dict_set_str(tmp_xdata, "entry-point", "true");
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_DICT_SET_FAILED, NULL);
+            goto out;
+        }
 
-        if (op_ret == 0 && op_errno == ENOENT && private->special_dir &&
-            strcmp (private->special_dir, "") && svc_fd->special_dir &&
-            local->subvolume == FIRST_CHILD (this)) {
-                inode = inode_grep (fd->inode->table, fd->inode,
-                                    private->path);
-                if (!inode) {
-                        inode = inode_new (fd->inode->table);
-                        if (!inode) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "allocate new inode");
-                                goto out;
-                        }
-                }
-
-                gf_uuid_copy (local->loc.pargfid, fd->inode->gfid);
-                gf_uuid_copy (local->loc.gfid, inode->gfid);
-                if (gf_uuid_is_null (inode->gfid))
-                        ret = inode_path (fd->inode, private->path, &path);
-                else
-                        ret = inode_path (inode, NULL, &path);
-
-                if (ret < 0)
-                        goto out;
-                loc->path = gf_strdup (path);
-                if (loc->path) {
-                        if (!loc->name ||
-                            (loc->name && !strcmp (loc->name, ""))) {
-                                loc->name = strrchr (loc->path, '/');
-                                if (loc->name)
-                                        loc->name++;
-                        }
-                }
-
-                loc->inode = inode;
-                loc->parent = inode_ref (fd->inode);
-                tmp_xdata = dict_new ();
-                if (!tmp_xdata)
-                        goto out;
-                ret = dict_set_str (tmp_xdata, "entry-point", "true");
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set dict");
-                        goto out;
-                }
-
-                local->cookie = cookie;
-                if (xdata == NULL)
-                        local->xdata = NULL;
-                else
-                        local->xdata = dict_ref (xdata);
-                STACK_WIND (frame, gf_svc_readdirp_lookup_cbk,
-                            SECOND_CHILD (this),
-                            SECOND_CHILD (this)->fops->lookup, loc, tmp_xdata);
-                unwind = _gf_false;
+        local->cookie = cookie;
+        if (local->xdata) {
+            dict_unref(local->xdata);
+            local->xdata = NULL;
         }
+        if (xdata)
+            local->xdata = dict_ref(xdata);
+
+        STACK_WIND(frame, gf_svc_readdirp_lookup_cbk, SECOND_CHILD(this),
+                   SECOND_CHILD(this)->fops->lookup, loc, tmp_xdata);
+        unwind = _gf_false;
+    }
 
 out:
-        if (tmp_xdata)
-                dict_unref (tmp_xdata);
+    if (tmp_xdata)
+        dict_unref(tmp_xdata);
 
-        GF_FREE (path);
-        return unwind;
+    GF_FREE(path);
+    return unwind;
 }
 
 static int32_t
-gf_svc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     gf_dirent_t *entries, dict_t *xdata)
+gf_svc_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-        gf_dirent_t   *entry      = NULL;
-        gf_dirent_t   *tmpentry   = NULL;
-        svc_local_t   *local      = NULL;
-        int            inode_type = -1;
-        int            ret        = -1;
-        svc_fd_t      *svc_fd     = NULL;
-        gf_boolean_t   unwind     = _gf_true;
-        svc_private_t *priv       = NULL;
-
-        if (op_ret < 0)
-                goto out;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        priv = this->private;
-        local = frame->local;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmpentry = NULL;
+    svc_local_t *local = NULL;
+    int inode_type = -1;
+    int ret = -1;
+    svc_fd_t *svc_fd = NULL;
+    gf_boolean_t unwind = _gf_true;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto out;
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+
+    local = frame->local;
+
+    svc_fd = svc_fd_ctx_get(this, local->fd);
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(local->fd->inode->gfid), NULL);
+    }
+
+    if (local->subvolume == FIRST_CHILD(this))
+        inode_type = NORMAL_INODE;
+    else
+        inode_type = VIRTUAL_INODE;
 
-        svc_fd = svc_fd_ctx_get (this, local->fd);
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to get the fd "
-                        "context for the gfid %s",
-                        uuid_utoa (local->fd->inode->gfid));
+    /*
+     * Better to goto out and return whatever is there in the
+     * readdirp response (even if the readdir response contains
+     * a directory entry for the snapshot entry point). Otherwise
+     * if we ignore the error, then there is a chance of race
+     * condition where, priv->path is changed in reconfigure
+     */
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_COPY_ENTRY_POINT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    list_for_each_entry_safe(entry, tmpentry, &entries->list, list)
+    {
+        /* If .snaps pre-exists, then it should not be listed
+         * in the NORMAL INODE directory when USS is enabled,
+         * so filter the .snaps entry if exists.
+         * However it is OK to list .snaps in VIRTUAL world
+         */
+        if (inode_type == NORMAL_INODE && !strcmp(entry_point, entry->d_name)) {
+            gf_dirent_entry_free(entry);
+            continue;
         }
 
-        if (local->subvolume == FIRST_CHILD (this))
-                inode_type = NORMAL_INODE;
-        else
-                inode_type = VIRTUAL_INODE;
-
-        list_for_each_entry_safe (entry, tmpentry, &entries->list, list) {
-                /* If .snaps pre-exists, then it should not be listed
-                 * in the NORMAL INODE directory when USS is enabled,
-                 * so filter the .snaps entry if exists.
-                 * However it is OK to list .snaps in VIRTUAL world
-                 */
-                if (inode_type == NORMAL_INODE &&
-                    !strcmp(priv->path, entry->d_name)) {
-                        gf_dirent_entry_free (entry);
-                        continue;
-                }
-
-                if (!entry->inode)
-                        continue;
-
-                ret = svc_inode_ctx_set (this, entry->inode, inode_type);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                                "context");
-                if (svc_fd)
-                        svc_fd->last_offset = entry->d_off;
-        }
+        if (!entry->inode)
+            continue;
 
-        unwind = gf_svc_readdir_on_special_dir (frame, cookie, this, op_ret,
-                                                op_errno, entries, xdata);
+        ret = svc_inode_ctx_set(this, entry->inode, inode_type);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    SVC_MSG_SET_INODE_CONTEXT_FAILED, NULL);
+        if (svc_fd)
+            svc_fd->last_offset = entry->d_off;
+    }
+
+    unwind = gf_svc_readdir_on_special_dir(frame, cookie, this, op_ret,
+                                           op_errno, entries, xdata);
 
 out:
-        if (unwind)
-                SVC_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries,
-                                  xdata);
+    if (unwind)
+        SVC_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t off, dict_t *xdata)
+gf_svc_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
 {
-        int            inode_type = -1;
-        xlator_t      *subvolume  = NULL;
-        svc_local_t   *local      = NULL;
-        int            ret        = -1;
-        int            op_ret     = -1;
-        int            op_errno   = EINVAL;
-        gf_boolean_t   wind       = _gf_false;
-        svc_fd_t      *svc_fd     = NULL;
-        gf_dirent_t    entries;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate local");
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        /*
-         * This is mainly for samba shares (or windows clients). As part of
-         * readdirp on the directory used as samba share, the entry point
-         * directory would have been added at the end. So when a new readdirp
-         * request comes, we have to check if the entry point has been handled
-         * or not in readdirp. That information and the offset used for it
-         * is remembered in fd context. If it has been handled, then simply
-         * unwind indication end of readdir operation.
-         */
-        svc_fd = svc_fd_ctx_get_or_new (this,  fd);
-        if (!svc_fd)
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (fd->inode->gfid));
-        else {
-                if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
-                        op_ret = 0;
-                        op_errno = ENOENT;
-                        goto out;
-                }
-        }
-
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
-
-        local->subvolume = subvolume;
-        local->fd = fd_ref (fd);
-        frame->local = local;
-
-        STACK_WIND (frame, gf_svc_readdirp_cbk, subvolume,
-                    subvolume->fops->readdirp, fd, size, off, xdata);
-
-        wind = _gf_true;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    svc_local_t *local = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_fd_t *svc_fd = NULL;
+    gf_dirent_t entries;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    /*
+     * This is mainly for samba shares (or windows clients). As part of
+     * readdirp on the directory used as samba share, the entry point
+     * directory would have been added at the end. So when a new readdirp
+     * request comes, we have to check if the entry point has been handled
+     * or not in readdirp. That information and the offset used for it
+     * is remembered in fd context. If it has been handled, then simply
+     * unwind indication end of readdir operation.
+     */
+    svc_fd = svc_fd_ctx_get_or_new(this, fd);
+    if (!svc_fd)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+    else {
+        if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
+            op_ret = 0;
+            op_errno = ENOENT;
+            goto out;
+        }
+    }
+
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
+
+    local->subvolume = subvolume;
+    local->fd = fd_ref(fd);
+    frame->local = local;
+
+    STACK_WIND(frame, gf_svc_readdirp_cbk, subvolume, subvolume->fops->readdirp,
+               fd, size, off, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries, NULL);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 /* Renaming the entries from or to snapshots is not allowed as the snapshots
    are read-only.
 */
 static int32_t
-gf_svc_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc, dict_t *xdata)
+gf_svc_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        int          src_inode_type  = -1;
-        int          dst_inode_type  = -1;
-        int          dst_parent_type = -1;
-        int32_t      op_ret          = -1;
-        int32_t      op_errno        = 0;
-        int32_t      ret             = -1;
-        gf_boolean_t wind            = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc, out);
-
-        ret = svc_inode_ctx_get (this, oldloc->inode, &src_inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for the inode %s",
-                        uuid_utoa (oldloc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (src_inode_type == VIRTUAL_INODE) {
-                gf_log (this->name, GF_LOG_ERROR, "rename happening on a entry"
-                        " %s residing in snapshot", oldloc->name);
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        if (newloc->inode) {
-                ret = svc_inode_ctx_get (this, newloc->inode, &dst_inode_type);
-                if (!ret && dst_inode_type == VIRTUAL_INODE) {
-                        gf_log (this->name, GF_LOG_ERROR, "rename of %s "
-                                "happening to a entry %s residing in snapshot",
-                                oldloc->name, newloc->name);
-                        op_ret = -1;
-                        op_errno = EROFS;
-                        goto out;
-                }
-        }
-
-        if (dst_inode_type < 0) {
-                ret = svc_inode_ctx_get (this, newloc->parent,
-                                         &dst_parent_type);
-                if (!ret && dst_parent_type == VIRTUAL_INODE) {
-                        gf_log (this->name, GF_LOG_ERROR, "rename of %s "
-                                "happening to a entry %s residing in snapshot",
-                                oldloc->name, newloc->name);
-                        op_ret = -1;
-                        op_errno = EROFS;
-                        goto out;
-                }
+    int src_inode_type = -1;
+    int dst_inode_type = -1;
+    int dst_parent_type = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int32_t ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, newloc, out);
+
+    ret = svc_inode_ctx_get(this, oldloc->inode, &src_inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(oldloc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (src_inode_type == VIRTUAL_INODE) {
+        op_ret = -1;
+        op_errno = EROFS;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_RENAME_SNAPSHOT_ENTRY, "name=%s", oldloc->name, NULL);
+        goto out;
+    }
+
+    if (newloc->inode) {
+        ret = svc_inode_ctx_get(this, newloc->inode, &dst_inode_type);
+        if (!ret && dst_inode_type == VIRTUAL_INODE) {
+            op_ret = -1;
+            op_errno = EROFS;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    SVC_MSG_RENAME_SNAPSHOT_ENTRY, "oldloc-name=%s",
+                    oldloc->name, "newloc-name=%s", newloc->name, NULL);
+            goto out;
+        }
+    }
+
+    if (dst_inode_type < 0) {
+        ret = svc_inode_ctx_get(this, newloc->parent, &dst_parent_type);
+        if (!ret && dst_parent_type == VIRTUAL_INODE) {
+            op_ret = -1;
+            op_errno = EROFS;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    SVC_MSG_RENAME_SNAPSHOT_ENTRY, "oldloc-name=%s",
+                    oldloc->name, "newloc-name=%s", newloc->name, NULL);
+            goto out;
         }
+    }
 
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->rename, oldloc, newloc,
-                         xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (rename, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(rename, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL, NULL, NULL);
+    return 0;
 }
 
 /* Creating hardlinks for the files from the snapshot is not allowed as it
    will be equivalent of creating hardlinks across different filesystems.
-   And so is vise versa.
+   And so is vice versa.
 */
 static int32_t
-gf_svc_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-             dict_t *xdata)
+gf_svc_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        int          src_inode_type  = -1;
-        int          dst_parent_type = -1;
-        int32_t      op_ret          = -1;
-        int32_t      op_errno        = 0;
-        int32_t      ret             = -1;
-        gf_boolean_t wind            = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc, out);
-
-        ret = svc_inode_ctx_get (this, oldloc->inode, &src_inode_type);
-        if (!ret && src_inode_type == VIRTUAL_INODE) {
-                gf_log (this->name, GF_LOG_ERROR, "rename happening on a entry"
-                        " %s residing in snapshot", oldloc->name);
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        ret = svc_inode_ctx_get (this, newloc->parent, &dst_parent_type);
-        if (!ret && dst_parent_type == VIRTUAL_INODE) {
-                gf_log (this->name, GF_LOG_ERROR, "rename of %s "
-                        "happening to a entry %s residing in snapshot",
-                        oldloc->name, newloc->name);
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->link, oldloc, newloc, xdata);
-
-        wind = _gf_true;
+    int src_inode_type = -1;
+    int dst_parent_type = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int32_t ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, newloc, out);
+
+    ret = svc_inode_ctx_get(this, oldloc->inode, &src_inode_type);
+    if (!ret && src_inode_type == VIRTUAL_INODE) {
+        op_ret = -1;
+        op_errno = EROFS;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_LINK_SNAPSHOT_ENTRY,
+                "oldloc-name=%s", oldloc->name, NULL);
+        goto out;
+    }
+
+    ret = svc_inode_ctx_get(this, newloc->parent, &dst_parent_type);
+    if (!ret && dst_parent_type == VIRTUAL_INODE) {
+        op_ret = -1;
+        op_errno = EROFS;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_LINK_SNAPSHOT_ENTRY,
+                "oldloc-name=%s", oldloc->name, "newloc-name=%s", newloc->name,
+                NULL);
+        goto out;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (link, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(link, frame, op_ret, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
 static int32_t
-gf_svc_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name, dict_t *xdata)
+gf_svc_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
 {
-        int          ret        = -1;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
+    int ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "path=%s", loc->path,
+                "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get te inode "
-                        "context for %s (gfid: %s)", loc->path,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+out:
+    if (!wind)
+        SVC_STACK_UNWIND(removexattr, frame, op_ret, op_errno, NULL);
 
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->removexattr, loc,
-                                 name, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
+    return 0;
+}
 
-        wind = _gf_true;
+static int
+gf_svc_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+             dict_t *xdata)
+{
+    int inode_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    ret = svc_inode_ctx_get(this, fd->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (removexattr, frame, op_ret, op_errno,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(fsync, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-static int
-gf_svc_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-              dict_t *xdata)
+static int32_t
+gf_svc_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int             inode_type = -1;
-        int             ret        = -1;
-        int             op_ret     = -1;
-        int             op_errno   = EINVAL;
-        gf_boolean_t    wind       = _gf_false;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
 
-        ret = svc_inode_ctx_get (this, fd->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
 
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->fsync, fd, datasync,
-                                 xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->flush, fd, xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (fsync, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(flush, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+gf_svc_releasedir(xlator_t *this, fd_t *fd)
 {
-        int32_t           op_ret     = -1;
-        int32_t           op_errno   = 0;
-        int               ret        = -1;
-        int               inode_type = -1;
-        xlator_t         *subvolume  = NULL;
-        gf_boolean_t      wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    svc_fd_t *sfd = NULL;
+    uint64_t tmp_pfd = 0;
+    int ret = 0;
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->flush, fd, xdata);
+    ret = fd_ctx_del(fd, this, &tmp_pfd);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "pfd from fd=%p is NULL", fd);
+        goto out;
+    }
 
-        wind = _gf_true;
+    GF_FREE(sfd);
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (flush, frame, op_ret, op_errno, NULL);
-
-        return 0;
+    return 0;
 }
 
 static int32_t
-gf_svc_releasedir (xlator_t *this, fd_t *fd)
+gf_svc_forget(xlator_t *this, inode_t *inode)
 {
-        svc_fd_t *sfd      = NULL;
-        uint64_t          tmp_pfd  = 0;
-        int               ret      = 0;
+    int ret = -1;
+    uint64_t value = 0;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        ret = fd_ctx_del (fd, this, &tmp_pfd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pfd from fd=%p is NULL", fd);
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        GF_FREE (sfd);
+    ret = inode_ctx_del(inode, this, &value);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                SVC_MSG_DELETE_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-static int32_t
-gf_svc_forget (xlator_t *this, inode_t *inode)
+static int
+gf_svc_priv_destroy(xlator_t *this, svc_private_t *priv)
 {
-        int            ret      = -1;
-        uint64_t       value    = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    if (!priv) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_NULL_PRIV, NULL);
+        goto out;
+    }
 
-        ret = inode_ctx_del (inode, this, &value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to delete inode "
-                        "context for %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
+    GF_FREE(priv->path);
+    GF_FREE(priv->special_dir);
+
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+
+    ret = 0;
 
 out:
-        return 0;
+    return ret;
 }
 
+/**
+ * ** NOTE **:
+ * =============
+ * The option "snapdir-entry-path" is NOT reconfigurable.
+ * That option as of now is only for the consumption of
+ * samba, where, it needs to tell glusterfs about the
+ * directory that is shared with windows client for the
+ * access. Now, in windows-explorer (GUI) interface, for
+ * the directory shared, the entry point to the snapshot
+ * world (snapshot-directory option) should be visible,
+ * atleast as a hidden entry. For that to happen, glusterfs
+ * has to send that entry in the readdir response coming on
+ * the directory used as the smb share. Therefore, samba,
+ * while initializing the gluster volume (via gfapi) sets
+ * the xlator option "snapdir-entry-path" to the directory
+ * which is to be shared with windows (check the file
+ * vfs_glusterfs.c from samba source code). So to avoid
+ * problems with smb access, not allowing snapdir-entry-path
+ * option to be configurable. That option is for those
+ * consumers who know what they are doing.
+ **/
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        svc_private_t  *priv = NULL;
-
-        priv = this->private;
+    svc_private_t *priv = NULL;
+    char *path = NULL;
+    gf_boolean_t show_entry_point = _gf_false;
+    char *tmp = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("snapshot-directory", path, options, str, out);
+    if (!path || (strlen(path) > NAME_MAX) || path[0] != '.') {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_INVALID_ENTRY_POINT,
+                "path=%s", path, NULL);
+        goto out;
+    }
+
+    GF_OPTION_RECONF("show-snapshot-directory", show_entry_point, options, bool,
+                     out);
+
+    /*
+     * The assumption now is that priv->path is an allocated memory (either
+     * in init or in a previous reconfigure).
+     * So, the intention here is to preserve the older contents of the option
+     * until the new option's value has been completely stored in the priv.
+     * So, do this.
+     *  - Store the pointer of priv->path in a temporary pointer.
+     *  - Allocate new memory for the new value of the option that is just
+     *    obtained from the above call to GF_OPTION_RECONF.
+     *  - If the above allocation fails, again set the pointer from priv
+     *    to the address stored in tmp. i.e. the previous value.
+     *  - If the allocation succeeds, then free the tmp pointer.
+     * WARNING: Before changing the allocation and freeing logic of
+     *          priv->path, always check the init function to see how
+     *          priv->path is set. Take decisions accordingly. As of now,
+     *          the assumption is that, the string elements of private
+     *          structure of snapview-client are allocated (either in
+     *          init or here in reconfugure).
+     */
+    LOCK(&priv->lock);
+    {
+        tmp = priv->path;
+        priv->path = NULL;
+        priv->path = gf_strdup(path);
+        if (!priv->path) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to reconfigure snapshot-directory option to %s",
+                   path);
+            priv->path = tmp;
+        } else {
+            GF_FREE(tmp);
+            tmp = NULL;
+        }
 
-        GF_OPTION_RECONF ("snapshot-directory", priv->path, options, str, out);
-        GF_OPTION_RECONF ("show-snapshot-directory", priv->show_entry_point,
-                          options, bool, out);
+        priv->show_entry_point = show_entry_point;
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int32_t     ret = -1;
+    int32_t ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_svc_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_svc_mt_end + 1);
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_MEM_ACNT_FAILED, NULL);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        svc_private_t *private  = NULL;
-        int            ret      = -1;
-        int            children = 0;
-        xlator_list_t *xl       = NULL;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "configured without any child");
-                goto out;
-        }
-
-        xl = this->children;
-        while (xl) {
-                children++;
-                xl = xl->next;
-        }
+    svc_private_t *private = NULL;
+    int ret = -1;
+    int children = 0;
+    xlator_list_t *xl = NULL;
+    char *path = NULL;
+    char *special_dir = NULL;
+
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_CHILD_FOR_XLATOR, NULL);
+        goto out;
+    }
+
+    xl = this->children;
+    while (xl) {
+        children++;
+        xl = xl->next;
+    }
+
+    if (children != 2) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_XLATOR_CHILDREN_WRONG,
+                "subvol-num=%d", children, NULL);
+        goto out;
+    }
+
+    /* This can be the top of graph in certain cases */
+    if (!this->parents) {
+        gf_msg_debug(this->name, 0,
+                     "dangling volume. Check "
+                     "volfile");
+    }
+
+   private
+    = GF_CALLOC(1, sizeof(*private), gf_svc_mt_svc_private_t);
+    if (!private)
+        goto out;
+
+    LOCK_INIT(&private->lock);
+
+    GF_OPTION_INIT("snapshot-directory", path, str, out);
+    if (!path || (strlen(path) > NAME_MAX) || path[0] != '.') {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_INVALID_ENTRY_POINT,
+                "path=%s", path, NULL);
+        goto out;
+    }
+
+   private
+    ->path = gf_strdup(path);
+    if (!private->path) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_MEMORY,
+                "entry-point-path=%s", path, NULL);
+        goto out;
+    }
+
+    GF_OPTION_INIT("snapdir-entry-path", special_dir, str, out);
+    if (!special_dir || strstr(special_dir, path)) {
+        if (special_dir)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    SVC_MSG_ENTRY_POINT_SPECIAL_DIR, "path=%s", path,
+                    "special-dir=%s", special_dir);
+        else
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NULL_SPECIAL_DIR,
+                    NULL);
+        goto out;
+    }
 
-        if (children != 2) {
-                gf_log (this->name, GF_LOG_ERROR, "snap-view-client has got "
-                        "%d subvolumes. It can have only 2 subvolumes.",
-                        children);
-                goto out;
-        }
+   private
+    ->special_dir = gf_strdup(special_dir);
+    if (!private->special_dir) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_MEMORY,
+                "special-directory=%s", special_dir, NULL);
+        goto out;
+    }
 
-        /* This can be the top of graph in certain cases */
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
+    GF_OPTION_INIT("show-snapshot-directory", private->show_entry_point, bool,
+                   out);
 
-        private = GF_CALLOC (1, sizeof (*private), gf_svc_mt_svc_private_t);
-        if (!private)
-                goto out;
+    this->local_pool = mem_pool_new(svc_local_t, 128);
+    if (!this->local_pool) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_MEM_POOL_GET_FAILED, NULL);
+        goto out;
+    }
 
-        GF_OPTION_INIT ("snapshot-directory", private->path, str, out);
-        GF_OPTION_INIT ("snapdir-entry-path", private->special_dir, str,
-                        out);
-        GF_OPTION_INIT ("show-snapshot-directory", private->show_entry_point,
-                        bool, out);
-
-        if (strstr (private->special_dir, private->path)) {
-                gf_log (this->name, GF_LOG_ERROR, "entry point directory "
-                        "cannot be part of the special directory");
-                GF_FREE (private->special_dir);
-                private->special_dir = NULL;
-                goto out;
-        }
-
-        this->private = private;
-        this->local_pool = mem_pool_new (svc_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR, "could not get mem pool for "
-                        "frame->local");
-                goto out;
-        }
+    this->private = private;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret)
-                GF_FREE (private);
+    if (ret)
+        (void)gf_svc_priv_destroy(this, private);
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        svc_private_t *priv = NULL;
+    svc_private_t *priv = NULL;
 
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        priv = this->private;
-        if (!priv)
-                return;
+    priv = this->private;
+    if (!priv)
+        return;
 
-        this->private = NULL;
+    /*
+     * Just log the failure and go ahead to
+     * set this->priv to NULL.
+     */
+    if (gf_svc_priv_destroy(this, priv))
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_PRIV_DESTROY_FAILED,
+                NULL);
 
-        GF_FREE (priv);
+    this->private = NULL;
 
-        return;
+    return;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        xlator_t       *subvol     = NULL;
-        int             ret        = 0;
-
-        subvol = data;
-
-        /* As there are two subvolumes in snapview-client, there is
-         * a possibility that the regular subvolume is still down and
-         * snapd subvolume come up first. So if we don't handle this situation
-         * CHILD_UP event will be propagated upwards to fuse when
-         * regular subvolume is still down.
-         * This can cause data unavailable for the application.
-         * So for now send notifications up only for regular subvolume.
-         *
-         * TODO: In future if required we may need to handle
-         * notifications from virtual subvolume
-         */
-        if (subvol != SECOND_CHILD (this))
-                ret = default_notify (this, event, data);
-
-        return ret;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+
+    subvol = data;
+
+    /* As there are two subvolumes in snapview-client, there is
+     * a possibility that the regular subvolume is still down and
+     * snapd subvolume come up first. So if we don't handle this situation
+     * CHILD_UP event will be propagated upwards to fuse when
+     * regular subvolume is still down.
+     * This can cause data unavailable for the application.
+     * So for now send notifications up only for regular subvolume.
+     *
+     * TODO: In future if required we may need to handle
+     * notifications from virtual subvolume
+     */
+    if (subvol != SECOND_CHILD(this))
+        ret = default_notify(this, event, data);
+
+    return ret;
 }
 
 struct xlator_fops fops = {
-        .lookup        = gf_svc_lookup,
-        .opendir       = gf_svc_opendir,
-        .stat          = gf_svc_stat,
-        .fstat         = gf_svc_fstat,
-        .statfs        = gf_svc_statfs,
-        .rmdir         = gf_svc_rmdir,
-        .rename        = gf_svc_rename,
-        .mkdir         = gf_svc_mkdir,
-        .open          = gf_svc_open,
-        .unlink        = gf_svc_unlink,
-        .setattr       = gf_svc_setattr,
-        .getxattr      = gf_svc_getxattr,
-        .setxattr      = gf_svc_setxattr,
-        .fsetxattr     = gf_svc_fsetxattr,
-        .readv         = gf_svc_readv,
-        .readdir       = gf_svc_readdir,
-        .readdirp      = gf_svc_readdirp,
-        .create        = gf_svc_create,
-        .readlink      = gf_svc_readlink,
-        .mknod         = gf_svc_mknod,
-        .symlink       = gf_svc_symlink,
-        .flush         = gf_svc_flush,
-        .link          = gf_svc_link,
-        .access        = gf_svc_access,
-        .removexattr   = gf_svc_removexattr,
-        .fsync         = gf_svc_fsync,
+    .lookup = gf_svc_lookup,
+    .opendir = gf_svc_opendir,
+    .stat = gf_svc_stat,
+    .fstat = gf_svc_fstat,
+    .statfs = gf_svc_statfs,
+    .rmdir = gf_svc_rmdir,
+    .rename = gf_svc_rename,
+    .mkdir = gf_svc_mkdir,
+    .open = gf_svc_open,
+    .unlink = gf_svc_unlink,
+    .setattr = gf_svc_setattr,
+    .getxattr = gf_svc_getxattr,
+    .setxattr = gf_svc_setxattr,
+    .fsetxattr = gf_svc_fsetxattr,
+    .readv = gf_svc_readv,
+    .readdir = gf_svc_readdir,
+    .readdirp = gf_svc_readdirp,
+    .create = gf_svc_create,
+    .readlink = gf_svc_readlink,
+    .mknod = gf_svc_mknod,
+    .symlink = gf_svc_symlink,
+    .flush = gf_svc_flush,
+    .link = gf_svc_link,
+    .access = gf_svc_access,
+    .removexattr = gf_svc_removexattr,
+    .fsync = gf_svc_fsync,
 };
 
 struct xlator_cbks cbks = {
-        .forget = gf_svc_forget,
-        .releasedir = gf_svc_releasedir,
+    .forget = gf_svc_forget,
+    .releasedir = gf_svc_releasedir,
 };
 
 struct volume_options options[] = {
-        { .key = {"snapshot-directory"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = ".snaps",
-        },
-        { .key = {"snapdir-entry-path"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "An option to set the path of a directory on which "
-                         "when readdir comes, dentry for the snapshot-directory"
-                         " should be created and added in the readdir response",
-          .default_value = "",
-        },
-        { .key = {"show-snapshot-directory"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "If this option is set, and the option "
-                         "\"snapdir-entry-path\" is set (which is set by samba "
-                         "vfs plugin for glusterfs, then send the entry point "
-                         "when readdir comes on the snapdir-entry-path",
-          .default_value = "off",
-        },
-        { .key  = {NULL} },
+    {
+        .key = {"snapshot-directory"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = ".snaps",
+    },
+    {
+        .key = {"snapdir-entry-path"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "An option to set the path of a directory on which "
+                       "when readdir comes, dentry for the snapshot-directory"
+                       " should be created and added in the readdir response",
+        .default_value = "",
+    },
+    {
+        .key = {"show-snapshot-directory"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .description = "If this option is set, and the option "
+                       "\"snapdir-entry-path\" is set (which is set by samba "
+                       "vfs plugin for glusterfs, then send the entry point "
+                       "when readdir comes on the snapdir-entry-path",
+        .default_value = "off",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "snapview-client",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/snapview-client/src/snapview-client.h b/xlators/features/snapview-client/src/snapview-client.h
index 5b7a862cf3f..166116a439d 100644
--- a/xlators/features/snapview-client/src/snapview-client.h
+++ b/xlators/features/snapview-client/src/snapview-client.h
@@ -1,97 +1,101 @@
- /*
-   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 #ifndef __SNAP_VIEW_CLIENT_H__
 #define __SNAP_VIEW_CLIENT_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "snapview-client-mem-types.h"
+#include "snapview-client-messages.h"
 
 struct __svc_local {
-        loc_t loc;
-        xlator_t *subvolume;
-        fd_t     *fd;
-        void *cookie;
-        dict_t *xdata;
+    loc_t loc;
+    xlator_t *subvolume;
+    fd_t *fd;
+    void *cookie;
+    dict_t *xdata;
+    uint16_t revalidate;
 };
 typedef struct __svc_local svc_local_t;
 
-#define SVC_STACK_UNWIND(fop, frame, params ...) do {           \
-                svc_local_t *__local = NULL;                    \
-                if (frame) {                                    \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                svc_local_free (__local);                       \
-        } while (0)
+#define SVC_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        svc_local_t *__local = NULL;                                           \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        svc_local_free(__local);                                               \
+    } while (0)
 
-#define SVC_ENTRY_POINT_SET(this, xdata, op_ret, op_errno, new_xdata,   \
-                            priv, ret, label)                           \
-        do {                                                            \
-                if (!xdata) {                                           \
-                        xdata = new_xdata = dict_new ();                \
-                        if (!new_xdata) {                               \
-                                gf_log (this->name, GF_LOG_ERROR,       \
-                                        "failed to allocate new dict"); \
-                                op_ret = -1;                            \
-                                op_errno = ENOMEM;                      \
-                                goto label;                             \
-                        }                                               \
-                }                                                       \
-                ret = dict_set_str (xdata, "entry-point", "true");      \
-                if (ret) {                                              \
-                        gf_log (this->name, GF_LOG_ERROR,               \
-                                "failed to set dict");                  \
-                        op_ret = -1;                                    \
-                        op_errno = ENOMEM;                              \
-                        goto label;                                     \
-                }                                                       \
-        } while (0);
+#define SVC_ENTRY_POINT_SET(this, xdata, op_ret, op_errno, new_xdata, ret,     \
+                            label)                                             \
+    do {                                                                       \
+        if (!xdata) {                                                          \
+            xdata = new_xdata = dict_new();                                    \
+            if (!new_xdata) {                                                  \
+                gf_log(this->name, GF_LOG_ERROR,                               \
+                       "failed to allocate new dict");                         \
+                op_ret = -1;                                                   \
+                op_errno = ENOMEM;                                             \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+        ret = dict_set_str(xdata, "entry-point", "true");                      \
+        if (ret) {                                                             \
+            gf_log(this->name, GF_LOG_ERROR, "failed to set dict");            \
+            op_ret = -1;                                                       \
+            op_errno = ENOMEM;                                                 \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
 
-#define SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, \
-                                inode, subvolume, label)                \
-        do {                                                            \
-                ret = svc_inode_ctx_get (this, inode, &inode_type);     \
-                if (ret < 0) {                                          \
-                        gf_log (this->name, GF_LOG_ERROR,               \
-                                "inode context not found for gfid %s",  \
-                                uuid_utoa (inode->gfid));               \
-                        op_ret = -1;                                    \
-                        op_errno = EINVAL;                              \
-                        goto label;                                     \
-                }                                                       \
-                                                                        \
-                subvolume = svc_get_subvolume (this, inode_type);       \
-        }  while (0);
+#define SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret,       \
+                                inode, subvolume, label)                       \
+    do {                                                                       \
+        ret = svc_inode_ctx_get(this, inode, &inode_type);                     \
+        if (ret < 0) {                                                         \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "inode context not found for gfid %s",                      \
+                   uuid_utoa(inode->gfid));                                    \
+            op_ret = -1;                                                       \
+            op_errno = EINVAL;                                                 \
+            goto label;                                                        \
+        }                                                                      \
+                                                                               \
+        subvolume = svc_get_subvolume(this, inode_type);                       \
+    } while (0);
 
 struct svc_private {
-        char *path;
-        char *special_dir; /* needed for samba */
-        gf_boolean_t show_entry_point;
+    char *path;
+    char *special_dir; /* needed for samba */
+    gf_boolean_t show_entry_point;
+    gf_lock_t lock; /* mainly to guard private->path */
 };
 typedef struct svc_private svc_private_t;
 
 struct svc_fd {
-        off_t last_offset;
-        gf_boolean_t entry_point_handled;
-        gf_boolean_t special_dir;
+    off_t last_offset;
+    gf_boolean_t entry_point_handled;
+    gf_boolean_t special_dir;
 };
 typedef struct svc_fd svc_fd_t;
 
-typedef enum {
-        NORMAL_INODE = 1,
-        VIRTUAL_INODE
-} inode_type_t;
+typedef enum { NORMAL_INODE = 1, VIRTUAL_INODE } inode_type_t;
+
+int
+gf_svc_special_dir_revalidate_lookup(call_frame_t *frame, xlator_t *this,
+                                     dict_t *xdata);
 
 #endif /* __SNAP_VIEW_CLIENT_H__ */
diff --git a/xlators/features/snapview-server/src/Makefile.am b/xlators/features/snapview-server/src/Makefile.am
index 6b588e5d235..2935f138a4c 100644
--- a/xlators/features/snapview-server/src/Makefile.am
+++ b/xlators/features/snapview-server/src/Makefile.am
@@ -1,21 +1,24 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = snapview-server.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 snapview_server_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
-snapview_server_la_SOURCES = snapview-server.c snapview-server-mgmt.c snapview-server-helpers.c
-snapview_server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
-                            $(top_builddir)/api/src/libgfapi.la\
-                            $(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-                            $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+snapview_server_la_SOURCES = snapview-server.c snapview-server-mgmt.c \
+	snapview-server-helpers.c
 
-noinst_HEADERS = snapview-server.h snapview-server-mem-types.h
+snapview_server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/api/src/libgfapi.la \
+	$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+
+noinst_HEADERS = snapview-server.h snapview-server-mem-types.h snapview-server-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-               -I$(top_srcdir)/api/src \
-               -I$(top_srcdir)/rpc/rpc-lib/src \
-               -I$(top_srcdir)/rpc/xdr/src \
-	       -DDATADIR=\"$(localstatedir)\"
+	-I$(top_srcdir)/api/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-DDATADIR=\"$(localstatedir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/snapview-server/src/snapview-server-helpers.c b/xlators/features/snapview-server/src/snapview-server-helpers.c
index dd2e891a849..62c1ddac49c 100644
--- a/xlators/features/snapview-server/src/snapview-server-helpers.c
+++ b/xlators/features/snapview-server/src/snapview-server-helpers.c
@@ -10,595 +10,706 @@
 #include "snapview-server.h"
 #include "snapview-server-mem-types.h"
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
 #include <pthread.h>
 
-
 int
-__svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
+__svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_inode, out);
 
-        value = (uint64_t)(long) svs_inode;
+    value = (uint64_t)(long)svs_inode;
 
-        ret = __inode_ctx_set (inode, this, &value);
+    ret = __inode_ctx_set(inode, this, &value);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_inode_t *
-__svs_inode_ctx_get (xlator_t *this, inode_t *inode)
+__svs_inode_ctx_get(xlator_t *this, inode_t *inode)
 {
-        svs_inode_t *svs_inode = NULL;
-        uint64_t     value     = 0;
-        int          ret       = -1;
+    svs_inode_t *svs_inode = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = __inode_ctx_get (inode, this, &value);
-        if (ret)
-                goto out;
+    ret = __inode_ctx_get(inode, this, &value);
+    if (ret)
+        goto out;
 
-        svs_inode = (svs_inode_t *) ((long) value);
+    svs_inode = (svs_inode_t *)((long)value);
 
 out:
-        return svs_inode;
+    return svs_inode;
 }
 
 svs_inode_t *
-svs_inode_ctx_get (xlator_t *this, inode_t *inode)
+svs_inode_ctx_get(xlator_t *this, inode_t *inode)
 {
-        svs_inode_t *svs_inode = NULL;
+    svs_inode_t *svs_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                svs_inode = __svs_inode_ctx_get (this, inode);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        svs_inode = __svs_inode_ctx_get(this, inode);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return svs_inode;
+    return svs_inode;
 }
 
 int32_t
-svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
+svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_inode, out);
 
-        LOCK (&inode->lock);
-        {
-                ret = __svs_inode_ctx_set (this, inode, svs_inode);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __svs_inode_ctx_set(this, inode, svs_inode);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_inode_t *
-svs_inode_new (void)
+svs_inode_new(void)
 {
-        svs_inode_t    *svs_inode = NULL;
+    svs_inode_t *svs_inode = NULL;
 
-        svs_inode = GF_CALLOC (1, sizeof (*svs_inode), gf_svs_mt_svs_inode_t);
+    svs_inode = GF_CALLOC(1, sizeof(*svs_inode), gf_svs_mt_svs_inode_t);
 
-        return svs_inode;
+    return svs_inode;
 }
 
 svs_inode_t *
-svs_inode_ctx_get_or_new (xlator_t *this, inode_t *inode)
+svs_inode_ctx_get_or_new(xlator_t *this, inode_t *inode)
 {
-        svs_inode_t   *svs_inode = NULL;
-        int            ret       = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        LOCK (&inode->lock);
-        {
-                svs_inode = __svs_inode_ctx_get (this, inode);
-                if (!svs_inode) {
-                        svs_inode = svs_inode_new ();
-                        if (svs_inode) {
-                                ret = __svs_inode_ctx_set (this, inode,
-                                                           svs_inode);
-                                if (ret) {
-                                        GF_FREE (svs_inode);
-                                        svs_inode = NULL;
-                                }
-                        }
+    svs_inode_t *svs_inode = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        svs_inode = __svs_inode_ctx_get(this, inode);
+        if (!svs_inode) {
+            svs_inode = svs_inode_new();
+            if (svs_inode) {
+                ret = __svs_inode_ctx_set(this, inode, svs_inode);
+                if (ret) {
+                    GF_FREE(svs_inode);
+                    svs_inode = NULL;
                 }
+            }
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return svs_inode;
+    return svs_inode;
 }
 
 svs_fd_t *
-svs_fd_new (void)
+svs_fd_new(void)
 {
-        svs_fd_t    *svs_fd = NULL;
+    svs_fd_t *svs_fd = NULL;
 
-        svs_fd = GF_CALLOC (1, sizeof (*svs_fd), gf_svs_mt_svs_fd_t);
+    svs_fd = GF_CALLOC(1, sizeof(*svs_fd), gf_svs_mt_svs_fd_t);
 
-        return svs_fd;
+    return svs_fd;
 }
 
 int
-__svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
+__svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_fd, out);
 
-        value = (uint64_t)(long) svs_fd;
+    value = (uint64_t)(long)svs_fd;
 
-        ret = __fd_ctx_set (fd, this, value);
+    ret = __fd_ctx_set(fd, this, value);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_fd_t *
-__svs_fd_ctx_get (xlator_t *this, fd_t *fd)
+__svs_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *svs_fd = NULL;
-        uint64_t  value  = 0;
-        int       ret    = -1;
+    svs_fd_t *svs_fd = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = __fd_ctx_get (fd, this, &value);
-        if (ret)
-                return NULL;
+    ret = __fd_ctx_get(fd, this, &value);
+    if (ret)
+        return NULL;
 
-        svs_fd = (svs_fd_t *) ((long) value);
+    svs_fd = (svs_fd_t *)((long)value);
 
 out:
-        return svs_fd;
+    return svs_fd;
 }
 
 svs_fd_t *
-svs_fd_ctx_get (xlator_t *this, fd_t *fd)
+svs_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *svs_fd = NULL;
+    svs_fd_t *svs_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                svs_fd = __svs_fd_ctx_get (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        svs_fd = __svs_fd_ctx_get(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return svs_fd;
+    return svs_fd;
 }
 
 int32_t
-svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
+svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_fd, out);
 
-        LOCK (&fd->lock);
-        {
-                ret = __svs_fd_ctx_set (this, fd, svs_fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __svs_fd_ctx_set(this, fd, svs_fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_fd_t *
-__svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+__svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t        *svs_fd    = NULL;
-        int              ret       = -1;
-        glfs_t          *fs        = NULL;
-        glfs_object_t   *object    = NULL;
-        svs_inode_t     *inode_ctx = NULL;
-        glfs_fd_t       *glfd      = NULL;
-        inode_t         *inode     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        inode = fd->inode;
-        svs_fd = __svs_fd_ctx_get (this, fd);
-        if (svs_fd) {
-                ret = 0;
-                goto out;
+    svs_fd_t *svs_fd = NULL;
+    int ret = -1;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_fd_t *glfd = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    inode = fd->inode;
+    svs_fd = __svs_fd_ctx_get(this, fd);
+    if (svs_fd) {
+        ret = 0;
+        goto out;
+    }
+
+    svs_fd = svs_fd_new();
+    if (!svs_fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_NEW_FD_CTX_FAILED,
+               "failed to allocate new fd "
+               "context for gfid %s",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    if (fd_is_anonymous(fd)) {
+        inode_ctx = svs_inode_ctx_get(this, inode);
+        if (!inode_ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   SVS_MSG_GET_INODE_CONTEXT_FAILED,
+                   "failed to get inode "
+                   "context for %s",
+                   uuid_utoa(inode->gfid));
+            goto out;
         }
 
-        svs_fd = svs_fd_new ();
-        if (!svs_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate new fd "
-                        "context for gfid %s", uuid_utoa (inode->gfid));
+        fs = inode_ctx->fs;
+        object = inode_ctx->object;
+
+        if (inode->ia_type == IA_IFDIR) {
+            glfd = glfs_h_opendir(fs, object);
+            if (!glfd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_OPENDIR_FAILED,
+                       "failed to "
+                       "open the directory %s",
+                       uuid_utoa(inode->gfid));
                 goto out;
+            }
         }
 
-        if (fd_is_anonymous (fd)) {
-                inode_ctx = svs_inode_ctx_get (this, inode);
-                if (!inode_ctx) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get inode "
-                                "context for %s", uuid_utoa (inode->gfid));
-                        goto out;
-                }
-
-                fs = inode_ctx->fs;
-                object = inode_ctx->object;
-
-                if (inode->ia_type == IA_IFDIR) {
-                        glfd = glfs_h_opendir (fs, object);
-                        if (!glfd) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "open the directory %s",
-                                        uuid_utoa (inode->gfid));
-                                goto out;
-                        }
-                }
-
-                if (inode->ia_type == IA_IFREG) {
-                        glfd = glfs_h_open (fs, object, O_RDONLY|O_LARGEFILE);
-                        if (!glfd) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "open the file %s",
-                                        uuid_utoa (inode->gfid));
-                                goto out;
-                        }
-                }
-
-                svs_fd->fd = glfd;
+        if (inode->ia_type == IA_IFREG) {
+            glfd = glfs_h_open(fs, object, O_RDONLY | O_LARGEFILE);
+            if (!glfd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_OPEN_FAILED,
+                       "failed to "
+                       "open the file %s",
+                       uuid_utoa(inode->gfid));
+                goto out;
+            }
         }
 
-        ret = __svs_fd_ctx_set (this, fd, svs_fd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set fd context "
-                        "for gfid %s", uuid_utoa (inode->gfid));
-                if (svs_fd->fd) {
-                        if (inode->ia_type == IA_IFDIR) {
-                                ret = glfs_closedir (svs_fd->fd);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to close the fd for %s",
-                                                uuid_utoa (inode->gfid));
-                        }
-                        if (inode->ia_type == IA_IFREG) {
-                                ret = glfs_close (svs_fd->fd);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to close the fd for %s",
-                                                uuid_utoa (inode->gfid));
-                        }
-                }
-                ret = -1;
+        svs_fd->fd = glfd;
+    }
+
+    ret = __svs_fd_ctx_set(this, fd, svs_fd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_SET_FD_CONTEXT_FAILED,
+               "failed to set fd context "
+               "for gfid %s",
+               uuid_utoa(inode->gfid));
+        if (svs_fd->fd) {
+            if (inode->ia_type == IA_IFDIR) {
+                ret = glfs_closedir(svs_fd->fd);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           SVS_MSG_CLOSEDIR_FAILED,
+                           "failed to close the fd for %s",
+                           uuid_utoa(inode->gfid));
+            }
+            if (inode->ia_type == IA_IFREG) {
+                ret = glfs_close(svs_fd->fd);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_CLOSE_FAILED,
+                           "failed to close the fd for %s",
+                           uuid_utoa(inode->gfid));
+            }
         }
+        ret = -1;
+    }
 
 out:
-        if (ret) {
-                GF_FREE (svs_fd);
-                svs_fd = NULL;
-        }
+    if (ret) {
+        GF_FREE(svs_fd);
+        svs_fd = NULL;
+    }
 
-        return svs_fd;
+    return svs_fd;
 }
 
 svs_fd_t *
-svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t  *svs_fd = NULL;
+    svs_fd_t *svs_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                svs_fd = __svs_fd_ctx_get_or_new (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        svs_fd = __svs_fd_ctx_get_or_new(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return svs_fd;
+    return svs_fd;
 }
 
-void
-svs_uuid_generate (uuid_t gfid, char *snapname, uuid_t origin_gfid)
+int
+svs_uuid_generate(xlator_t *this, uuid_t gfid, char *snapname,
+                  uuid_t origin_gfid)
 {
-        unsigned char md5_sum[MD5_DIGEST_LENGTH] = {0};
-        char          ino_string[NAME_MAX + 32]  = "";
-        int           ret                        = 0;
+    char ino_string[NAME_MAX + 32] = "";
+    uuid_t tmp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, snapname, out);
+
+    (void)snprintf(ino_string, sizeof(ino_string), "%s%s", snapname,
+                   uuid_utoa(origin_gfid));
+
+    if (gf_gfid_generate_from_xxh64(tmp, ino_string)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_GFID_GEN_FAILED,
+               "failed to generate "
+               "gfid for object with actual gfid of %s "
+               "(snapname: %s, key: %s)",
+               uuid_utoa(origin_gfid), snapname, ino_string);
+        goto out;
+    }
+
+    gf_uuid_copy(gfid, tmp);
 
-        GF_ASSERT (snapname);
+    ret = 0;
 
-        ret = snprintf (ino_string, sizeof (ino_string), "%s%s",
-                        snapname, uuid_utoa(origin_gfid));
-        MD5((unsigned char *)ino_string, strlen(ino_string), md5_sum);
-        gf_uuid_copy (gfid, md5_sum);
+    gf_msg_debug(this->name, 0, "gfid generated is %s ", uuid_utoa(gfid));
+
+out:
+    return ret;
 }
 
 void
-svs_fill_ino_from_gfid (struct iatt *buf)
+svs_fill_ino_from_gfid(struct iatt *buf)
 {
-        uint64_t  temp_ino = 0;
-        int       j        = 0;
-        int       i        = 0;
-        xlator_t *this     = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
 
-        /* consider least significant 8 bytes of value out of gfid */
-        if (gf_uuid_is_null (buf->ia_gfid)) {
-                buf->ia_ino = -1;
-                goto out;
-        }
-        for (i = 15; i > (15 - 8); i--) {
-                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
-                j += 8;
-        }
-        buf->ia_ino = temp_ino;
+    /* consider least significant 8 bytes of value out of gfid */
+    if (gf_uuid_is_null(buf->ia_gfid)) {
+        buf->ia_ino = -1;
+        goto out;
+    }
+
+    buf->ia_ino = gfid_to_ino(buf->ia_gfid);
 out:
-        return;
+    return;
 }
 
 void
-svs_iatt_fill (uuid_t gfid, struct iatt *buf)
+svs_iatt_fill(uuid_t gfid, struct iatt *buf)
 {
-        struct timeval  tv    = {0, };
-        xlator_t       *this  = NULL;
+    struct timeval tv = {
+        0,
+    };
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
 
-        buf->ia_type = IA_IFDIR;
-        buf->ia_uid  = 0;
-        buf->ia_gid  = 0;
-        buf->ia_size = 0;
-        buf->ia_nlink = 2;
-        buf->ia_blocks = 8;
-        buf->ia_size = 4096;
+    buf->ia_type = IA_IFDIR;
+    buf->ia_uid = 0;
+    buf->ia_gid = 0;
+    buf->ia_size = 0;
+    buf->ia_nlink = 2;
+    buf->ia_blocks = 8;
+    buf->ia_size = 4096;
 
-        gf_uuid_copy (buf->ia_gfid, gfid);
-        svs_fill_ino_from_gfid (buf);
+    gf_uuid_copy(buf->ia_gfid, gfid);
+    svs_fill_ino_from_gfid(buf);
 
-        buf->ia_prot = ia_prot_from_st_mode (0755);
+    buf->ia_prot = ia_prot_from_st_mode(0755);
 
-        gettimeofday (&tv, 0);
+    gettimeofday(&tv, 0);
 
-        buf->ia_mtime = buf->ia_atime = buf->ia_ctime = tv.tv_sec;
-        buf->ia_mtime_nsec = buf->ia_atime_nsec = buf->ia_ctime_nsec =
-                (tv.tv_usec * 1000);
+    buf->ia_mtime = buf->ia_atime = buf->ia_ctime = tv.tv_sec;
+    buf->ia_mtime_nsec = buf->ia_atime_nsec = buf->ia_ctime_nsec = (tv.tv_usec *
+                                                                    1000);
 
 out:
-        return;
+    return;
 }
 
 /* priv->snaplist_lock should be held before calling this function */
 snap_dirent_t *
-__svs_get_snap_dirent (xlator_t *this, const char *name)
+__svs_get_snap_dirent(xlator_t *this, const char *name)
 {
-        svs_private_t      *private     = NULL;
-        int                 i           = 0;
-        snap_dirent_t      *dirents     = NULL;
-        snap_dirent_t      *tmp_dirent  = NULL;
-        snap_dirent_t      *dirent      = NULL;
-
-        private = this->private;
-
-        dirents = private->dirents;
-        if (!dirents) {
-                goto out;
+    svs_private_t *private = NULL;
+    int i = 0;
+    snap_dirent_t *dirents = NULL;
+    snap_dirent_t *tmp_dirent = NULL;
+    snap_dirent_t *dirent = NULL;
+
+   private
+    = this->private;
+
+    dirents = private->dirents;
+    if (!dirents) {
+        goto out;
+    }
+
+    tmp_dirent = dirents;
+    for (i = 0; i < private->num_snaps; i++) {
+        if (!strcmp(tmp_dirent->name, name)) {
+            dirent = tmp_dirent;
+            break;
         }
+        tmp_dirent++;
+    }
 
-        tmp_dirent = dirents;
-        for (i = 0; i < private->num_snaps; i++) {
-                if (!strcmp (tmp_dirent->name, name)) {
-                        dirent = tmp_dirent;
-                        break;
-                }
-                tmp_dirent++;
-        }
-
- out:
-        return dirent;
+out:
+    return dirent;
 }
 
 glfs_t *
-__svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                  int32_t *op_errno)
+__svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                                 int32_t *op_errno)
 {
-        svs_private_t      *priv                = NULL;
-        int32_t             ret                 = -1;
-        int32_t             local_errno         = ESTALE;
-        snap_dirent_t      *dirent              = NULL;
-        char                volname[PATH_MAX]   = {0, };
-        glfs_t             *fs                  = NULL;
-        int                 loglevel            = GF_LOG_INFO;
-        char                logfile[PATH_MAX]   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, name, out);
-
-        priv = this->private;
-
-        dirent = __svs_get_snap_dirent (this, name);
-        if (!dirent) {
-                gf_log (this->name, GF_LOG_DEBUG, "snap entry for "
-                        "name %s not found", name);
-                local_errno = ENOENT;
-                goto out;
+    svs_private_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t local_errno = ESTALE;
+    snap_dirent_t *dirent = NULL;
+    char volname[PATH_MAX] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+    int loglevel = GF_LOG_INFO;
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char *volfile_server = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, name, out);
+
+    priv = this->private;
+
+    dirent = __svs_get_snap_dirent(this, name);
+    if (!dirent) {
+        gf_msg_debug(this->name, 0,
+                     "snap entry for "
+                     "name %s not found",
+                     name);
+        local_errno = ENOENT;
+        goto out;
+    }
+
+    if (dirent->fs) {
+        ret = 0;
+        fs = dirent->fs;
+        goto out;
+    }
+
+    snprintf(volname, sizeof(volname), "/snaps/%s/%s/%s", dirent->name,
+             dirent->snap_volname, dirent->snap_volname);
+
+    fs = glfs_new(volname);
+    if (!fs) {
+        local_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, local_errno, SVS_MSG_GLFS_NEW_FAILED,
+               "glfs instance for snap volume %s "
+               "failed",
+               dirent->name);
+        goto out;
+    }
+
+    /*
+     * Before, localhost was used as the volfile server. But, with that
+     * method, accessing snapshots started giving ENOENT error if a
+     * specific bind address is mentioned in the glusterd volume file.
+     * Check the bug https://bugzilla.redhat.com/show_bug.cgi?id=1725211.
+     * So, the new method is tried below, where, snapview-server first
+     * uses the volfile server used by the snapd (obtained from the
+     * command line arguments saved in the global context of the process).
+     * If the volfile server in global context is NULL, then localhost
+     * is tried (like before).
+     */
+    if (this->ctx->cmd_args.volfile_server) {
+        volfile_server = gf_strdup(this->ctx->cmd_args.volfile_server);
+        if (!volfile_server) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                   SVS_MSG_VOLFILE_SERVER_GET_FAIL,
+                   "failed to copy volfile server %s. ",
+                   this->ctx->cmd_args.volfile_server);
+            ret = -1;
+            goto out;
         }
-
-        if (dirent->fs) {
-                ret = 0;
-                fs = dirent->fs;
-                goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+               SVS_MSG_VOLFILE_SERVER_GET_FAIL,
+               "volfile server is NULL in cmd args. "
+               "Trying with localhost");
+        volfile_server = gf_strdup("localhost");
+        if (!volfile_server) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                   SVS_MSG_VOLFILE_SERVER_GET_FAIL,
+                   "failed to copy volfile server localhost.");
+            ret = -1;
+            goto out;
         }
+    }
+
+    ret = glfs_set_volfile_server(fs, "tcp", volfile_server, 24007);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, local_errno,
+               SVS_MSG_SET_VOLFILE_SERVR_FAILED,
+               "setting the "
+               "volfile server %s for snap volume %s "
+               "failed",
+               volfile_server, dirent->name);
+        goto out;
+    }
+
+    snprintf(logfile, sizeof(logfile),
+             DEFAULT_SVD_LOG_FILE_DIRECTORY "/snaps/%s/%s-%s.log",
+             priv->volname, name, dirent->uuid);
+
+    ret = glfs_set_logging(fs, logfile, loglevel);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, local_errno,
+               SVS_MSG_SET_LOGGING_FAILED,
+               "failed to set the "
+               "log file path");
+        goto out;
+    }
+
+    ret = glfs_init(fs);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, local_errno, SVS_MSG_GLFS_INIT_FAILED,
+               "initing the "
+               "fs for %s failed",
+               dirent->name);
+        goto out;
+    }
+
+    ret = 0;
 
-        snprintf (volname, sizeof (volname), "/snaps/%s/%s",
-                  dirent->name, dirent->snap_volname);
+out:
+    if (ret) {
+        if (op_errno)
+            *op_errno = local_errno;
 
+        if (fs)
+            glfs_fini(fs);
+        fs = NULL;
+    }
 
-        fs = glfs_new (volname);
-        if (!fs) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "glfs instance for snap volume %s "
-                        "failed", dirent->name);
-                local_errno = ENOMEM;
-                goto out;
-        }
+    if (fs) {
+        dirent->fs = fs;
+    }
 
-        ret = glfs_set_volfile_server (fs, "tcp", "localhost",
-                                       24007);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "setting the "
-                        "volfile server for snap volume %s "
-                        "failed", dirent->name);
-                goto out;
-        }
+    GF_FREE(volfile_server);
+    return fs;
+}
 
-        snprintf (logfile, sizeof (logfile),
-                  DEFAULT_SVD_LOG_FILE_DIRECTORY "/snaps/%s/%s-%s.log",
-                  priv->volname, name, dirent->uuid);
+glfs_t *
+svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                               int32_t *op_errno)
+{
+    glfs_t *fs = NULL;
+    svs_private_t *priv = NULL;
 
-        ret = glfs_set_logging(fs, logfile, loglevel);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set the "
-                        "log file path");
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, name, out);
 
-        ret = glfs_init (fs);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "initing the "
-                        "fs for %s failed", dirent->name);
-                goto out;
-        }
+    priv = this->private;
 
-        ret = 0;
+    LOCK(&priv->snaplist_lock);
+    {
+        fs = __svs_initialise_snapshot_volume(this, name, op_errno);
+    }
+    UNLOCK(&priv->snaplist_lock);
 
 out:
-        if (ret) {
-                if (op_errno)
-                        *op_errno = local_errno;
 
-                if (fs)
-                        glfs_fini (fs);
-                fs = NULL;
-        }
+    return fs;
+}
+
+snap_dirent_t *
+svs_get_latest_snap_entry(xlator_t *this)
+{
+    svs_private_t *priv = NULL;
+    snap_dirent_t *dirents = NULL;
+    snap_dirent_t *dirent = NULL;
+
+    GF_VALIDATE_OR_GOTO("svs", this, out);
 
-        if (fs) {
-                dirent->fs = fs;
+    priv = this->private;
+
+    LOCK(&priv->snaplist_lock);
+    {
+        dirents = priv->dirents;
+        if (!dirents) {
+            goto unlock;
         }
+        if (priv->num_snaps)
+            dirent = &dirents[priv->num_snaps - 1];
+    }
+unlock:
+    UNLOCK(&priv->snaplist_lock);
 
-        return fs;
+out:
+    return dirent;
 }
 
 glfs_t *
-svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                int32_t *op_errno)
+svs_get_latest_snapshot(xlator_t *this)
 {
-        glfs_t             *fs   = NULL;
-        svs_private_t      *priv = NULL;
+    glfs_t *fs = NULL;
+    snap_dirent_t *dirent = NULL;
+    svs_private_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, name, out);
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    priv = this->private;
 
-        priv = this->private;
+    dirent = svs_get_latest_snap_entry(this);
 
-        LOCK (&priv->snaplist_lock);
+    if (dirent) {
+        LOCK(&priv->snaplist_lock);
         {
-                fs = __svs_initialise_snapshot_volume (this, name, op_errno);
+            fs = dirent->fs;
         }
-        UNLOCK (&priv->snaplist_lock);
-
+        UNLOCK(&priv->snaplist_lock);
+    }
 
 out:
-
-        return fs;
+    return fs;
 }
 
-snap_dirent_t *
-svs_get_latest_snap_entry (xlator_t *this)
+glfs_t *
+svs_inode_ctx_glfs_mapping(xlator_t *this, svs_inode_t *inode_ctx)
 {
-        svs_private_t *priv       = NULL;
-        snap_dirent_t *dirents    = NULL;
-        snap_dirent_t *dirent     = NULL;
+    glfs_t *fs = NULL;
 
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode_ctx, out);
 
-        priv = this->private;
+    fs = inode_ctx->fs;
 
-        LOCK (&priv->snaplist_lock);
-        {
-                dirents = priv->dirents;
-                if (!dirents) {
-                        goto unlock;
-                }
-                if (priv->num_snaps)
-                        dirent = &dirents[priv->num_snaps - 1];
-        }
-unlock:
-        UNLOCK (&priv->snaplist_lock);
+    SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
 
 out:
-        return dirent;
+    return fs;
 }
 
 glfs_t *
-svs_get_latest_snapshot (xlator_t *this)
+svs_inode_glfs_mapping(xlator_t *this, inode_t *inode)
 {
-        glfs_t        *fs         = NULL;
-        snap_dirent_t *dirent     = NULL;
-        svs_private_t *priv       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        priv = this->private;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
 
-        dirent = svs_get_latest_snap_entry (this);
+    inode_ctx = svs_inode_ctx_get(this, inode);
+    if (!inode_ctx) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for"
+               " the inode %s",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
 
-        if (dirent) {
-                LOCK (&priv->snaplist_lock);
-                {
-                        fs = dirent->fs;
-                }
-                UNLOCK (&priv->snaplist_lock);
-        }
+    fs = svs_inode_ctx_glfs_mapping(this, inode_ctx);
 
 out:
-        return fs;
+    return fs;
 }
diff --git a/xlators/features/snapview-server/src/snapview-server-mem-types.h b/xlators/features/snapview-server/src/snapview-server-mem-types.h
index a8035165000..63456b85323 100644
--- a/xlators/features/snapview-server/src/snapview-server-mem-types.h
+++ b/xlators/features/snapview-server/src/snapview-server-mem-types.h
@@ -11,16 +11,15 @@
 #ifndef __SNAP_VIEW_MEM_TYPES_H
 #define __SNAP_VIEW_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum snapview_mem_types {
-        gf_svs_mt_priv_t = gf_common_mt_end + 1,
-        gf_svs_mt_svs_inode_t,
-        gf_svs_mt_dirents_t,
-        gf_svs_mt_svs_fd_t,
-        gf_svs_mt_snaplist_t,
-        gf_svs_mt_end
+    gf_svs_mt_priv_t = gf_common_mt_end + 1,
+    gf_svs_mt_svs_inode_t,
+    gf_svs_mt_dirents_t,
+    gf_svs_mt_svs_fd_t,
+    gf_svs_mt_snaplist_t,
+    gf_svs_mt_end
 };
 
 #endif
-
diff --git a/xlators/features/snapview-server/src/snapview-server-messages.h b/xlators/features/snapview-server/src/snapview-server-messages.h
new file mode 100644
index 00000000000..f634ab5d2b0
--- /dev/null
+++ b/xlators/features/snapview-server/src/snapview-server-messages.h
@@ -0,0 +1,54 @@
+/*
+ Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _SNAPVIEW_SERVER_MESSAGES_H_
+#define _SNAPVIEW_SERVER_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SNAPVIEW_SERVER, SVS_MSG_NO_MEMORY, SVS_MSG_MEM_ACNT_FAILED,
+           SVS_MSG_NULL_GFID, SVS_MSG_GET_LATEST_SNAP_FAILED,
+           SVS_MSG_INVALID_GLFS_CTX, SVS_MSG_LOCK_DESTROY_FAILED,
+           SVS_MSG_SNAPSHOT_LIST_CHANGED, SVS_MSG_MGMT_INIT_FAILED,
+           SVS_MSG_GET_SNAPSHOT_LIST_FAILED, SVS_MSG_GET_GLFS_H_OBJECT_FAILED,
+           SVS_MSG_PARENT_CTX_OR_NAME_NULL, SVS_MSG_SET_INODE_CONTEXT_FAILED,
+           SVS_MSG_GET_INODE_CONTEXT_FAILED, SVS_MSG_NEW_INODE_CTX_FAILED,
+           SVS_MSG_DELETE_INODE_CONTEXT_FAILED, SVS_MSG_SET_FD_CONTEXT_FAILED,
+           SVS_MSG_NEW_FD_CTX_FAILED, SVS_MSG_DELETE_FD_CTX_FAILED,
+           SVS_MSG_GETXATTR_FAILED, SVS_MSG_LISTXATTR_FAILED,
+           SVS_MSG_RELEASEDIR_FAILED, SVS_MSG_RELEASE_FAILED,
+           SVS_MSG_TELLDIR_FAILED, SVS_MSG_STAT_FAILED, SVS_MSG_STATFS_FAILED,
+           SVS_MSG_OPEN_FAILED, SVS_MSG_READ_FAILED, SVS_MSG_READLINK_FAILED,
+           SVS_MSG_ACCESS_FAILED, SVS_MSG_GET_FD_CONTEXT_FAILED,
+           SVS_MSG_DICT_SET_FAILED, SVS_MSG_OPENDIR_FAILED,
+           SVS_MSG_FS_INSTANCE_INVALID, SVS_MSG_SETFSUID_FAIL,
+           SVS_MSG_SETFSGID_FAIL, SVS_MSG_SETFSGRPS_FAIL,
+           SVS_MSG_BUILD_TRNSPRT_OPT_FAILED, SVS_MSG_RPC_INIT_FAILED,
+           SVS_MSG_REG_NOTIFY_FAILED, SVS_MSG_REG_CBK_PRGM_FAILED,
+           SVS_MSG_RPC_CLNT_START_FAILED, SVS_MSG_XDR_PAYLOAD_FAILED,
+           SVS_MSG_NULL_CTX, SVS_MSG_RPC_CALL_FAILED, SVS_MSG_XDR_DECODE_FAILED,
+           SVS_MSG_RSP_DICT_EMPTY, SVS_MSG_DICT_GET_FAILED,
+           SVS_MSG_SNAP_LIST_REFRESH_FAILED, SVS_MSG_RPC_REQ_FAILED,
+           SVS_MSG_CLOSEDIR_FAILED, SVS_MSG_CLOSE_FAILED,
+           SVS_MSG_GFID_GEN_FAILED, SVS_MSG_GLFS_NEW_FAILED,
+           SVS_MSG_SET_VOLFILE_SERVR_FAILED, SVS_MSG_SET_LOGGING_FAILED,
+           SVS_MSG_VOLFILE_SERVER_GET_FAIL, SVS_MSG_GLFS_INIT_FAILED);
+
+#endif /* !_SNAPVIEW_CLIENT_MESSAGES_H_ */
diff --git a/xlators/features/snapview-server/src/snapview-server-mgmt.c b/xlators/features/snapview-server/src/snapview-server-mgmt.c
index fc2ff2ab10d..ecf31c3b880 100644
--- a/xlators/features/snapview-server/src/snapview-server-mgmt.c
+++ b/xlators/features/snapview-server/src/snapview-server-mgmt.c
@@ -12,465 +12,513 @@
 #include <pthread.h>
 
 int
-mgmt_cbk_snap (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_snap(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        xlator_t *this = NULL;
+    xlator_t *this = NULL;
 
-        this = mydata;
-        GF_ASSERT (this);
+    this = mydata;
+    GF_ASSERT(this);
 
-        gf_log ("mgmt", GF_LOG_INFO, "list of snapshots changed");
+    gf_msg("mgmt", GF_LOG_INFO, 0, SVS_MSG_SNAPSHOT_LIST_CHANGED,
+           "list of snapshots changed");
 
-        svs_get_snapshot_list (this);
-        return 0;
+    svs_get_snapshot_list(this);
+    return 0;
 }
 
-rpcclnt_cb_actor_t svs_cbk_actors[GF_CBK_MAXVALUE] = {
-        [GF_CBK_GET_SNAPS] = {"GETSNAPS", GF_CBK_GET_SNAPS, mgmt_cbk_snap},
+static rpcclnt_cb_actor_t svs_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_GET_SNAPS] = {"GETSNAPS", mgmt_cbk_snap, GF_CBK_GET_SNAPS},
 };
 
-struct rpcclnt_cb_program svs_cbk_prog = {
-        .progname  = "GlusterFS Callback",
-        .prognum   = GLUSTER_CBK_PROGRAM,
-        .progver   = GLUSTER_CBK_VERSION,
-        .actors    = svs_cbk_actors,
-        .numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program svs_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = svs_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]         = "NULL",
-        [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
 };
 
-rpc_clnt_prog_t svs_clnt_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t svs_clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-int
-svs_mgmt_init (xlator_t *this)
+static int
+svs_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data)
 {
-        int              ret      = -1;
-        svs_private_t   *priv     = NULL;
-        dict_t          *options  = NULL;
-        int              port     = GF_DEFAULT_BASE_PORT;
-        char            *host     = NULL;
-        cmd_args_t      *cmd_args = NULL;
-        glusterfs_ctx_t *ctx      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->ctx, out);
-
-        priv = this->private;
-
-        ctx = this->ctx;
-        cmd_args = &ctx->cmd_args;
-
-        host = "localhost";
-        if (cmd_args->volfile_server)
-                host = cmd_args->volfile_server;
-
-        ret = rpc_transport_inet_options_build (&options, host, port);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to build the "
-                        "transport options");
-                goto out;
-        }
-
-        priv->rpc = rpc_clnt_new (options, this, this->name, 8);
-        if (!priv->rpc) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to initialize RPC");
-                goto out;
-        }
-
-        ret = rpcclnt_cbk_program_register (priv->rpc, &svs_cbk_prog,
-                                            this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to register callback program");
-                goto out;
-        }
-
-        ret = rpc_clnt_start (priv->rpc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to start the rpc "
-                        "client");
-                goto out;
-        }
-
-        ret = 0;
+    xlator_t *this = NULL;
+    int ret = 0;
+
+    this = mydata;
+
+    switch (event) {
+        case RPC_CLNT_CONNECT:
+            ret = svs_get_snapshot_list(this);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                       SVS_MSG_GET_SNAPSHOT_LIST_FAILED,
+                       "Error in refreshing the snaplist "
+                       "infrastructure");
+                ret = -1;
+            }
+            break;
+        default:
+            break;
+    }
+    return ret;
+}
 
-        gf_log (this->name, GF_LOG_DEBUG, "svs mgmt init successful");
+int
+svs_mgmt_init(xlator_t *this)
+{
+    int ret = -1;
+    svs_private_t *priv = NULL;
+    dict_t *options = NULL;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    cmd_args_t *cmd_args = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_cmdline_option_t *opt = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->ctx, out);
+
+    priv = this->private;
+
+    ctx = this->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    host = "localhost";
+    if (cmd_args->volfile_server)
+        host = cmd_args->volfile_server;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    opt = find_xlator_option_in_cmd_args_t("address-family", cmd_args);
+    ret = rpc_transport_inet_options_build(options, host, port,
+                                           (opt != NULL ? opt->value : NULL));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_BUILD_TRNSPRT_OPT_FAILED,
+               "failed to build the "
+               "transport options");
+        goto out;
+    }
+
+    priv->rpc = rpc_clnt_new(options, this, this->name, 8);
+    if (!priv->rpc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_RPC_INIT_FAILED,
+               "failed to initialize RPC");
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(priv->rpc, svs_rpc_notify, this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_REG_NOTIFY_FAILED,
+               "failed to register notify function");
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(priv->rpc, &svs_cbk_prog, this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_REG_CBK_PRGM_FAILED,
+               "failed to register callback program");
+        goto out;
+    }
+
+    ret = rpc_clnt_start(priv->rpc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_RPC_CLNT_START_FAILED,
+               "failed to start the rpc "
+               "client");
+        goto out;
+    }
+
+    ret = 0;
+
+    gf_msg_debug(this->name, 0, "svs mgmt init successful");
 
 out:
-        if (ret)
-                if (priv) {
-                        rpc_clnt_connection_cleanup (&priv->rpc->conn);
-                        rpc_clnt_unref (priv->rpc);
-                        priv->rpc = NULL;
-                }
+    if (options)
+        dict_unref(options);
+    if (ret)
+        if (priv) {
+            rpc_clnt_connection_cleanup(&priv->rpc->conn);
+            rpc_clnt_unref(priv->rpc);
+            priv->rpc = NULL;
+        }
 
-        return ret;
+    return ret;
 }
 
 int
-svs_mgmt_submit_request (void *req, call_frame_t *frame,
-                         glusterfs_ctx_t *ctx,
-                         rpc_clnt_prog_t *prog, int procnum,
-                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+svs_mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                        rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                        xdrproc_t xdrproc)
 {
-        int                     ret        = -1;
-        int                     count      = 0;
-        struct iovec            iov        = {0, };
-        struct iobuf            *iobuf     = NULL;
-        struct iobref           *iobref    = NULL;
-        ssize_t                 xdr_size   = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", frame, out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", req, out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", ctx, out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", prog, out);
-
-        GF_ASSERT (frame->this);
-
-        iobref = iobref_new ();
-        if (!iobref) {
-                goto out;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", frame, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", req, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", ctx, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", prog, out);
+
+    GF_ASSERT(frame->this);
+
+    iobref = iobref_new();
+    if (!iobref) {
+        gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, SVS_MSG_NO_MEMORY,
+               "failed to allocate "
+               "new iobref");
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
         }
 
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-
-                iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                }
-
-                iobref_add (iobref, iobuf);
+        iobref_add(iobref, iobuf);
 
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_pagesize (iobuf);
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
 
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "Failed to create XDR payload");
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_msg(frame->this->name, GF_LOG_WARNING, 0,
+                   SVS_MSG_XDR_PAYLOAD_FAILED, "Failed to create XDR payload");
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-        if (iobref)
-                iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
-
 int
-mgmt_get_snapinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+mgmt_get_snapinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        gf_getsnap_name_uuid_rsp        rsp             = {0,};
-        call_frame_t                    *frame          = NULL;
-        glusterfs_ctx_t                 *ctx            = NULL;
-        int                             ret             = -1;
-        dict_t                          *dict           = NULL;
-        char                            key[1024]       = {0};
-        int                             snapcount       = 0;
-        svs_private_t                   *priv           = NULL;
-        xlator_t                        *this           = NULL;
-        int                             i               = 0;
-        int                             j               = 0;
-        char                            *value          = NULL;
-        snap_dirent_t                   *dirents        = NULL;
-        snap_dirent_t                   *old_dirents    = NULL;
-        int                             oldcount        = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", req, error_out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", myframe, error_out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", iov, error_out);
-
-        frame       = myframe;
-        this        = frame->this;
-        ctx         = frame->this->ctx;
-        priv        = this->private;
-        old_dirents = priv->dirents;
-
-        if (!ctx) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "NULL context");
-                errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "RPC call is not successful");
-                errno = EINVAL;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response, rsp.op_ret = %d",
-                        rsp.op_ret);
-                goto out;
-        }
-
-        if (rsp.op_ret == -1) {
-                errno = rsp.op_errno;
-                ret = -1;
-                goto out;
+    gf_getsnap_name_uuid_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+    char key[32] = {0};
+    int len;
+    int snapcount = 0;
+    svs_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    int i = 0;
+    int j = 0;
+    char *value = NULL;
+    snap_dirent_t *dirents = NULL;
+    snap_dirent_t *old_dirents = NULL;
+    int oldcount = 0;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", req, error_out);
+    GF_VALIDATE_OR_GOTO("snapview-server", myframe, error_out);
+    GF_VALIDATE_OR_GOTO("snapview-server", iov, error_out);
+
+    frame = myframe;
+    this = frame->this;
+    ctx = frame->this->ctx;
+    priv = this->private;
+
+    if (!ctx) {
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_NULL_CTX,
+               "NULL context");
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_RPC_CALL_FAILED,
+               "RPC call is not successful");
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
+    if (ret < 0) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, SVS_MSG_XDR_DECODE_FAILED,
+               "Failed to decode xdr response, rsp.op_ret = %d", rsp.op_ret);
+        goto out;
+    }
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        ret = -1;
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_RSP_DICT_EMPTY,
+               "Response dict is not populated");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno,
+               LG_MSG_DICT_UNSERIAL_FAILED, "Failed to unserialize dictionary");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "snap-count", (int32_t *)&snapcount);
+    if (ret) {
+        errno = EINVAL;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+               "Error retrieving snapcount");
+        goto out;
+    }
+
+    if (snapcount > 0) {
+        /* first time we are fetching snap list */
+        dirents = GF_CALLOC(snapcount, sizeof(snap_dirent_t),
+                            gf_svs_mt_dirents_t);
+        if (!dirents) {
+            errno = ENOMEM;
+            ret = -1;
+            gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_NO_MEMORY,
+                   "Unable to allocate memory");
+            goto out;
         }
+    }
 
-        if (!rsp.dict.dict_len) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Response dict is not populated");
-                ret = -1;
-                errno = EINVAL;
-                goto out;
+    for (i = 0; i < snapcount; i++) {
+        len = snprintf(key, sizeof(key), "snap-volname.%d", i + 1);
+        ret = dict_get_strn(dict, key, len, &value);
+        if (ret) {
+            errno = EINVAL;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+                   "Error retrieving snap volname %d", i + 1);
+            goto out;
         }
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+        strncpy(dirents[i].snap_volname, value,
+                sizeof(dirents[i].snap_volname));
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        len = snprintf(key, sizeof(key), "snap-id.%d", i + 1);
+        ret = dict_get_strn(dict, key, len, &value);
         if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to unserialize dictionary");
-                errno = EINVAL;
-                goto out;
+            errno = EINVAL;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+                   "Error retrieving snap uuid %d", i + 1);
+            goto out;
         }
+        strncpy(dirents[i].uuid, value, sizeof(dirents[i].uuid));
 
-        ret = dict_get_int32 (dict, "snap-count", (int32_t*)&snapcount);
+        len = snprintf(key, sizeof(key), "snapname.%d", i + 1);
+        ret = dict_get_strn(dict, key, len, &value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error retrieving snapcount");
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
+            errno = EINVAL;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+                   "Error retrieving snap name %d", i + 1);
+            goto out;
         }
-
-        if (snapcount > 0) {
-                /* first time we are fetching snap list */
-                dirents = GF_CALLOC (snapcount, sizeof (snap_dirent_t),
-                                     gf_svs_mt_dirents_t);
-                if (!dirents) {
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                        errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        for (i = 0; i < snapcount; i++) {
-                snprintf (key, sizeof (key), "snap-volname.%d", i+1);
-                ret = dict_get_str (dict, key, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error retrieving snap volname %d",
-                                i+1);
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-
-                strncpy (dirents[i].snap_volname, value,
-                         sizeof (dirents[i].snap_volname));
-
-                snprintf (key, sizeof (key), "snap-id.%d", i+1);
-                ret = dict_get_str (dict, key, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error retrieving snap uuid %d", i+1);
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-                strncpy (dirents[i].uuid, value,
-                         sizeof (dirents[i].uuid));
-
-                snprintf (key, sizeof (key), "snapname.%d", i+1);
-                ret = dict_get_str (dict, key, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error retrieving snap name %d", i+1);
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-                strncpy (dirents[i].name, value,
-                         sizeof (dirents[i].name));
-        }
-
-        /*
-         * Got the new snap list populated in dirents
-         * The new snap list is either a subset or a superset of
-         * the existing snaplist old_dirents which has priv->num_snaps
-         * number of entries.
-         *
-         * If subset, then clean up the fs for entries which are
-         * no longer relevant.
-         *
-         * For other overlapping entries set the fs for new dirents
-         * entries which have a fs assigned already in old_dirents
-         *
-         * We do this as we don't want to do new glfs_init()s repeatedly
-         * as the dirents entries for snapshot volumes get repatedly
-         * cleaned up and allocated. And if we don't then that will lead
-         * to memleaks
-         */
-
-        LOCK (&priv->snaplist_lock);
-        {
-                oldcount = priv->num_snaps;
-                for (i = 0; i < priv->num_snaps; i++) {
-                        for (j = 0; j < snapcount; j++) {
-                                if ((!strcmp (old_dirents[i].name,
-                                              dirents[j].name)) &&
-                                    (!strcmp (old_dirents[i].uuid,
-                                              dirents[j].uuid)))    {
-                                        dirents[j].fs = old_dirents[i].fs;
-                                        old_dirents[i].fs = NULL;
-                                        break;
-                                }
-                        }
+        strncpy(dirents[i].name, value, sizeof(dirents[i].name));
+    }
+
+    /*
+     * Got the new snap list populated in dirents
+     * The new snap list is either a subset or a superset of
+     * the existing snaplist old_dirents which has priv->num_snaps
+     * number of entries.
+     *
+     * If subset, then clean up the fs for entries which are
+     * no longer relevant.
+     *
+     * For other overlapping entries set the fs for new dirents
+     * entries which have a fs assigned already in old_dirents
+     *
+     * We do this as we don't want to do new glfs_init()s repeatedly
+     * as the dirents entries for snapshot volumes get repatedly
+     * cleaned up and allocated. And if we don't then that will lead
+     * to memleaks
+     */
+
+    LOCK(&priv->snaplist_lock);
+    {
+        oldcount = priv->num_snaps;
+        old_dirents = priv->dirents;
+        for (i = 0; i < priv->num_snaps; i++) {
+            for (j = 0; j < snapcount; j++) {
+                if ((!strcmp(old_dirents[i].name, dirents[j].name)) &&
+                    (!strcmp(old_dirents[i].uuid, dirents[j].uuid))) {
+                    dirents[j].fs = old_dirents[i].fs;
+                    old_dirents[i].fs = NULL;
+                    break;
                 }
-
-                priv->dirents = dirents;
-                priv->num_snaps = snapcount;
+            }
         }
-        UNLOCK (&priv->snaplist_lock);
 
-        if (old_dirents) {
-                for (i = 0; i < oldcount; i++) {
-                        if (old_dirents[i].fs)
-                                glfs_fini (old_dirents[i].fs);
-                }
+        priv->dirents = dirents;
+        priv->num_snaps = snapcount;
+    }
+    UNLOCK(&priv->snaplist_lock);
+
+    if (old_dirents) {
+        for (i = 0; i < oldcount; i++) {
+            if (old_dirents[i].fs)
+                gf_msg_debug(this->name, 0,
+                             "calling glfs_fini on "
+                             "name: %s, snap_volname: %s, uuid: %s",
+                             old_dirents[i].name, old_dirents[i].snap_volname,
+                             old_dirents[i].uuid);
+            glfs_fini(old_dirents[i].fs);
         }
+    }
 
-        GF_FREE (old_dirents);
+    GF_FREE(old_dirents);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
+    if (dict) {
+        dict_unref(dict);
+    }
+    free(rsp.dict.dict_val);
+    free(rsp.op_errstr);
 
-        if (ret && dirents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Could not update dirents with refreshed snap list");
-                GF_FREE (dirents);
-        }
+    if (ret && dirents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_SNAP_LIST_REFRESH_FAILED,
+               "Could not update dirents with refreshed snap list");
+        GF_FREE(dirents);
+    }
 
-        if (myframe)
-                SVS_STACK_DESTROY (myframe);
+    if (myframe)
+        SVS_STACK_DESTROY(myframe);
 
 error_out:
-        return ret;
+    return ret;
 }
 
 int
-svs_get_snapshot_list (xlator_t *this)
+svs_get_snapshot_list(xlator_t *this)
 {
-        gf_getsnap_name_uuid_req        req             = {{0,}};
-        int                             ret             = -1;
-        dict_t                          *dict           = NULL;
-        glusterfs_ctx_t                 *ctx            = NULL;
-        call_frame_t                    *frame          = NULL;
-        svs_private_t                   *priv           = NULL;
-        gf_boolean_t                    frame_cleanup   = _gf_true;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-
-        ctx  = this->ctx;
-        if (!ctx) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "ctx is NULL");
-                goto out;
-        }
-
-        frame = create_frame (this, ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error allocating frame");
-                goto out;
-        }
-
-        priv = this->private;
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error allocating dictionary");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "volname", priv->volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting volname in dict");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                ret = -1;
-                goto out;
-        }
-
-        ret = svs_mgmt_submit_request (&req, frame, ctx,
-                                       &svs_clnt_handshake_prog,
-                                       GF_HNDSK_GET_SNAPSHOT_INFO,
-                                       mgmt_get_snapinfo_cbk,
-                                       (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error sending snapshot names RPC request");
-        }
-
-        frame_cleanup = _gf_false;
+    gf_getsnap_name_uuid_req req = {{
+        0,
+    }};
+    int ret = -1;
+    dict_t *dict = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *frame = NULL;
+    svs_private_t *priv = NULL;
+    gf_boolean_t frame_cleanup = _gf_true;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+
+    ctx = this->ctx;
+    if (!ctx) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_NULL_CTX, "ctx is NULL");
+        goto out;
+    }
+
+    frame = create_frame(this, ctx->pool);
+    if (!frame) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, LG_MSG_FRAME_ERROR,
+               "Error allocating frame");
+        goto out;
+    }
+
+    priv = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+               "Error allocating dictionary");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", priv->volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DICT_SET_FAILED,
+               "Error setting volname in dict");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, LG_MSG_DICT_UNSERIAL_FAILED,
+               "Failed to serialize dictionary");
+        ret = -1;
+        goto out;
+    }
+
+    ret = svs_mgmt_submit_request(
+        &req, frame, ctx, &svs_clnt_handshake_prog, GF_HNDSK_GET_SNAPSHOT_INFO,
+        mgmt_get_snapinfo_cbk, (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_RPC_REQ_FAILED,
+               "Error sending snapshot names RPC request");
+    }
+
+    frame_cleanup = _gf_false;
 
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
-        GF_FREE (req.dict.dict_val);
-
-        if (frame_cleanup && frame) {
-                /*
-                 * Destroy the frame if we encountered an error
-                 * Else we need to clean it up in
-                 * mgmt_get_snapinfo_cbk
-                 */
-                SVS_STACK_DESTROY (frame);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
+    GF_FREE(req.dict.dict_val);
+
+    if (frame_cleanup && frame) {
+        /*
+         * Destroy the frame if we encountered an error
+         * Else we need to clean it up in
+         * mgmt_get_snapinfo_cbk
+         */
+        SVS_STACK_DESTROY(frame);
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/snapview-server/src/snapview-server.c b/xlators/features/snapview-server/src/snapview-server.c
index 72cfc908bba..76cccae5914 100644
--- a/xlators/features/snapview-server/src/snapview-server.c
+++ b/xlators/features/snapview-server/src/snapview-server.c
@@ -9,78 +9,119 @@
 */
 #include "snapview-server.h"
 #include "snapview-server-mem-types.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include <pthread.h>
 
+#include "glfs-internal.h"
+
+int
+gf_setcredentials(uid_t *uid, gid_t *gid, uint16_t ngrps, uint32_t *groups)
+{
+    int ret = 0;
+
+    if (uid) {
+        ret = glfs_setfsuid(*uid);
+        if (ret != 0) {
+            gf_msg("snapview-server", GF_LOG_ERROR, 0, SVS_MSG_SETFSUID_FAIL,
+                   "failed to set uid "
+                   "%u in thread context",
+                   *uid);
+            return ret;
+        }
+    }
+    if (gid) {
+        ret = glfs_setfsgid(*gid);
+        if (ret != 0) {
+            gf_msg("snapview-server", GF_LOG_ERROR, 0, SVS_MSG_SETFSGID_FAIL,
+                   "failed to set gid "
+                   "%u in thread context",
+                   *gid);
+            return ret;
+        }
+    }
+
+    if (ngrps != 0 && groups) {
+        ret = glfs_setfsgroups(ngrps, groups);
+        if (ret != 0) {
+            gf_msg("snapview-server", GF_LOG_ERROR, 0, SVS_MSG_SETFSGRPS_FAIL,
+                   "failed to set "
+                   "groups in thread context");
+            return ret;
+        }
+    }
+    return 0;
+}
 
 int32_t
-svs_lookup_entry_point (xlator_t *this, loc_t *loc, inode_t *parent,
-                        struct iatt *buf, struct iatt *postparent,
-                        int32_t *op_errno)
+svs_lookup_entry_point(xlator_t *this, loc_t *loc, inode_t *parent,
+                       struct iatt *buf, struct iatt *postparent,
+                       int32_t *op_errno)
 {
-        uuid_t         gfid;
-        svs_inode_t   *inode_ctx = NULL;
-        int            op_ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-
-        if (gf_uuid_is_null (loc->inode->gfid)) {
-                gf_uuid_generate (gfid);
-                svs_iatt_fill (gfid, buf);
-
-                /* Here the inode context of the entry point directory
-                   is filled with just the type of the inode and the gfid
-                   of the parent from where the entry point was entered.
-                   The glfs object and the fs instance will be NULL.
-                */
-                if (parent)
-                        svs_iatt_fill (parent->gfid, postparent);
-                else {
-                        svs_iatt_fill (buf->ia_gfid, postparent);
-                }
+    uuid_t gfid;
+    svs_inode_t *inode_ctx = NULL;
+    int op_ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+
+    if (gf_uuid_is_null(loc->inode->gfid)) {
+        gf_uuid_generate(gfid);
+        svs_iatt_fill(gfid, buf);
+
+        /* Here the inode context of the entry point directory
+           is filled with just the type of the inode and the gfid
+           of the parent from where the entry point was entered.
+           The glfs object and the fs instance will be NULL.
+        */
+        if (parent)
+            svs_iatt_fill(parent->gfid, postparent);
+        else {
+            svs_iatt_fill(buf->ia_gfid, postparent);
+        }
 
-                inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-                if (!inode_ctx) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate inode context for entry point "
-                                "directory");
-                        op_ret = -1;
-                        *op_errno = ENOMEM;
-                        goto out;
-                }
-                gf_uuid_copy (inode_ctx->pargfid, loc->pargfid);
-                memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-                inode_ctx->type = SNAP_VIEW_ENTRY_POINT_INODE;
+        inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+        if (!inode_ctx) {
+            op_ret = -1;
+            *op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                   SVS_MSG_NEW_INODE_CTX_FAILED,
+                   "failed to "
+                   "allocate inode context for entry point "
+                   "directory");
+            goto out;
+        }
+
+        gf_uuid_copy(inode_ctx->pargfid, loc->pargfid);
+        memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+        inode_ctx->type = SNAP_VIEW_ENTRY_POINT_INODE;
+    } else {
+        inode_ctx = svs_inode_ctx_get(this, loc->inode);
+        if (inode_ctx) {
+            memcpy(buf, &inode_ctx->buf, sizeof(*buf));
+            svs_iatt_fill(inode_ctx->pargfid, postparent);
         } else {
-                if (inode_ctx) {
-                        memcpy (buf, &inode_ctx->buf, sizeof (*buf));
-                        svs_iatt_fill (inode_ctx->pargfid, postparent);
-                } else {
-                        svs_iatt_fill (loc->inode->gfid, buf);
-                        if (parent)
-                                svs_iatt_fill (parent->gfid,
-                                               postparent);
-                        else {
-                                svs_iatt_fill (loc->inode->gfid,
-                                               postparent);
-                        }
-                }
+            svs_iatt_fill(loc->inode->gfid, buf);
+            if (parent)
+                svs_iatt_fill(parent->gfid, postparent);
+            else {
+                svs_iatt_fill(loc->inode->gfid, postparent);
+            }
         }
+    }
 
-        op_ret = 0;
+    op_ret = 0;
 
 out:
-        return op_ret;
+    return op_ret;
 }
 
 /* When lookup comes from client and the protocol/server tries to resolve
@@ -103,80 +144,87 @@ out:
    snapshot is referred and a random gfid is not generated.
 */
 int32_t
-svs_lookup_gfid (xlator_t *this, loc_t *loc, struct iatt *buf,
-                 struct iatt *postparent, int32_t *op_errno)
+svs_lookup_gfid(xlator_t *this, loc_t *loc, struct iatt *buf,
+                struct iatt *postparent, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        unsigned char   handle_obj[GFAPI_HANDLE_LENGTH] = {0, };
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-        struct stat     statbuf                         = {0, };
-        svs_inode_t    *inode_ctx                       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-
-        if (gf_uuid_is_null (loc->gfid) && gf_uuid_is_null (loc->inode->gfid)) {
-                gf_log (this->name, GF_LOG_ERROR, "gfid is NULL");
-                goto out;
-        }
-
-        if (!gf_uuid_is_null (loc->inode->gfid))
-                memcpy (handle_obj, loc->inode->gfid,
-                        GFAPI_HANDLE_LENGTH);
-        else
-                memcpy (handle_obj, loc->gfid,
-                        GFAPI_HANDLE_LENGTH);
-
-        fs = svs_get_latest_snapshot (this);
-        if (!fs) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the latest "
-                        "snapshot");
-                op_ret = -1;
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-
-        object = glfs_h_create_from_handle (fs, handle_obj, GFAPI_HANDLE_LENGTH,
-                                            &statbuf);
-        if (!object) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to do lookup and get "
-                        "the handle on the snapshot %s (path: %s, gfid: %s)",
-                        loc->name, loc->path, uuid_utoa (loc->gfid));
-                op_ret = -1;
-                *op_errno = ESTALE;
-                goto out;
-        }
-
-        inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate inode "
-                        "context");
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-
-        iatt_from_stat (buf, &statbuf);
-        if (!gf_uuid_is_null (loc->gfid))
-                gf_uuid_copy (buf->ia_gfid, loc->gfid);
-        else
-                gf_uuid_copy (buf->ia_gfid, loc->inode->gfid);
-
-        inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
-        inode_ctx->fs = fs;
-        inode_ctx->object = object;
-        memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-        svs_iatt_fill (buf->ia_gfid, postparent);
-
-        op_ret = 0;
+    int32_t op_ret = -1;
+    unsigned char handle_obj[GFAPI_HANDLE_LENGTH] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat statbuf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_NULL_GFID, "gfid is NULL");
+        goto out;
+    }
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(handle_obj, loc->inode->gfid, GFAPI_HANDLE_LENGTH);
+    else
+        memcpy(handle_obj, loc->gfid, GFAPI_HANDLE_LENGTH);
+
+    fs = svs_get_latest_snapshot(this);
+    if (!fs) {
+        op_ret = -1;
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_GET_LATEST_SNAP_FAILED,
+               "failed to get the latest "
+               "snapshot");
+        goto out;
+    }
+
+    object = glfs_h_create_from_handle(fs, handle_obj, GFAPI_HANDLE_LENGTH,
+                                       &statbuf);
+    if (!object) {
+        op_ret = -1;
+        *op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_GET_GLFS_H_OBJECT_FAILED,
+               "failed to do lookup and get "
+               "the handle on the snapshot %s (path: %s, gfid: %s)",
+               loc->name, loc->path, uuid_utoa(loc->gfid));
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_NEW_INODE_CTX_FAILED,
+               "failed to allocate inode "
+               "context");
+        goto out;
+    }
+
+    iatt_from_stat(buf, &statbuf);
+    if (!gf_uuid_is_null(loc->gfid))
+        gf_uuid_copy(buf->ia_gfid, loc->gfid);
+    else
+        gf_uuid_copy(buf->ia_gfid, loc->inode->gfid);
+
+    inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
+    inode_ctx->fs = fs;
+    inode_ctx->object = object;
+    memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+    svs_iatt_fill(buf->ia_gfid, postparent);
+
+    op_ret = 0;
 
 out:
-        return op_ret;
+    return op_ret;
 }
 
 /* If the parent is an entry point inode, then create the handle for the
@@ -187,188 +235,213 @@ out:
    parent's context
 */
 int32_t
-svs_lookup_snapshot (xlator_t *this, loc_t *loc, struct iatt *buf,
-                     struct iatt *postparent, inode_t *parent,
-                     svs_inode_t *parent_ctx, int32_t *op_errno)
+svs_lookup_snapshot(xlator_t *this, loc_t *loc, struct iatt *buf,
+                    struct iatt *postparent, inode_t *parent,
+                    svs_inode_t *parent_ctx, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        unsigned char   handle_obj[GFAPI_HANDLE_LENGTH] = {0, };
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-        struct stat     statbuf                         = {0, };
-        svs_inode_t    *inode_ctx                       = NULL;
-        uuid_t          gfid;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent_ctx, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent, out);
-
-        fs = svs_initialise_snapshot_volume (this, loc->name, op_errno);
-        if (!fs) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to "
-                        "create the fs instance for snap %s",
-                        loc->name);
-                *op_errno = ENOENT;
-                op_ret = -1;
-                goto out;
-        }
-
-        memcpy (handle_obj, parent_ctx->pargfid,
-                GFAPI_HANDLE_LENGTH);
-        object = glfs_h_create_from_handle (fs, handle_obj, GFAPI_HANDLE_LENGTH,
-                                            &statbuf);
-        if (!object) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to do lookup and "
-                        "get the handle on the snapshot %s", loc->name);
-                op_ret = -1;
-                *op_errno = errno;
-                goto out;
-        }
-
-        inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "allocate inode context");
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-
-        if (gf_uuid_is_null (loc->gfid) &&
-            gf_uuid_is_null (loc->inode->gfid))
-                gf_uuid_generate (gfid);
-        else {
-                if (!gf_uuid_is_null (loc->inode->gfid))
-                        gf_uuid_copy (gfid, loc->inode->gfid);
-                else
-                        gf_uuid_copy (gfid, loc->gfid);
-        }
-        iatt_from_stat (buf, &statbuf);
-        gf_uuid_copy (buf->ia_gfid, gfid);
-        svs_fill_ino_from_gfid (buf);
-        inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
-        inode_ctx->fs = fs;
-        inode_ctx->object = object;
-        memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-        svs_iatt_fill (parent->gfid, postparent);
-
-        SVS_STRDUP (inode_ctx->snapname, loc->name);
-        if (!inode_ctx->snapname) {
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-        op_ret = 0;
+    int32_t op_ret = -1;
+    unsigned char handle_obj[GFAPI_HANDLE_LENGTH] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat statbuf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+    uuid_t gfid;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent_ctx, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+
+    fs = svs_initialise_snapshot_volume(this, loc->name, op_errno);
+    if (!fs) {
+        gf_msg_debug(this->name, 0,
+                     "failed to create "
+                     "the fs instance for snap %s",
+                     loc->name);
+        *op_errno = ENOENT;
+        op_ret = -1;
+        goto out;
+    }
+
+    memcpy(handle_obj, parent_ctx->pargfid, GFAPI_HANDLE_LENGTH);
+    object = glfs_h_create_from_handle(fs, handle_obj, GFAPI_HANDLE_LENGTH,
+                                       &statbuf);
+    if (!object) {
+        op_ret = -1;
+        *op_errno = errno;
+        /* Should this be in warning or error mode? */
+        gf_msg_debug(this->name, 0,
+                     "failed to do lookup and "
+                     "get the handle on the snapshot %s",
+                     loc->name);
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_NEW_INODE_CTX_FAILED,
+               "failed to allocate "
+               "inode context");
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid))
+        gf_uuid_generate(gfid);
+    else {
+        if (!gf_uuid_is_null(loc->inode->gfid))
+            gf_uuid_copy(gfid, loc->inode->gfid);
+        else
+            gf_uuid_copy(gfid, loc->gfid);
+    }
+    iatt_from_stat(buf, &statbuf);
+    gf_uuid_copy(buf->ia_gfid, gfid);
+    svs_fill_ino_from_gfid(buf);
+    inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
+    inode_ctx->fs = fs;
+    inode_ctx->object = object;
+    memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+    svs_iatt_fill(parent->gfid, postparent);
+
+    SVS_STRDUP(inode_ctx->snapname, loc->name);
+    if (!inode_ctx->snapname) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    op_ret = 0;
 
 out:
-        if (op_ret) {
-                if (object)
-                        glfs_h_close (object);
+    if (op_ret) {
+        if (object)
+            glfs_h_close(object);
 
-                if (inode_ctx)
-                        inode_ctx->object = NULL;
-        }
+        if (inode_ctx)
+            inode_ctx->object = NULL;
+    }
 
-        return op_ret;
+    return op_ret;
 }
 
 /* Both parent and entry are from snapshot world */
 int32_t
-svs_lookup_entry (xlator_t *this, loc_t *loc, struct iatt *buf,
-                  struct iatt *postparent, inode_t *parent,
-                  svs_inode_t *parent_ctx, int32_t *op_errno)
+svs_lookup_entry(xlator_t *this, loc_t *loc, struct iatt *buf,
+                 struct iatt *postparent, inode_t *parent,
+                 svs_inode_t *parent_ctx, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-        struct stat     statbuf                         = {0, };
-        svs_inode_t    *inode_ctx                       = NULL;
-        glfs_object_t  *parent_object                   = NULL;
-        uuid_t          gfid                            = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent_ctx, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent, out);
-
-        parent_object = parent_ctx->object;
-        fs = parent_ctx->fs;
-
-        object = glfs_h_lookupat (fs, parent_object, loc->name,
-                                  &statbuf, 0);
-        if (!object) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to do lookup and "
-                        "get the handle for entry %s (path: %s)", loc->name,
-                        loc->path);
-                op_ret = -1;
-                *op_errno = errno;
-                goto out;
-        }
-
-        if (gf_uuid_is_null(object->gfid)) {
-                gf_log (this->name, GF_LOG_DEBUG, "gfid from glfs handle is "
-                        "NULL for entry %s (path: %s)", loc->name, loc->path);
-                op_ret = -1;
-                *op_errno = errno;
-                goto out;
-        }
-
-        inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "allocate inode context");
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-
-        if (gf_uuid_is_null (loc->gfid) &&
-            gf_uuid_is_null (loc->inode->gfid))
-                svs_uuid_generate (gfid, parent_ctx->snapname, object->gfid);
-        else {
-                if (!gf_uuid_is_null (loc->inode->gfid))
-                        gf_uuid_copy (gfid, loc->inode->gfid);
-                else
-                        gf_uuid_copy (gfid, loc->gfid);
-        }
-
-        iatt_from_stat (buf, &statbuf);
-        gf_uuid_copy (buf->ia_gfid, gfid);
-        svs_fill_ino_from_gfid (buf);
-        inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
-        inode_ctx->fs = fs;
-        inode_ctx->object = object;
-        memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-        svs_iatt_fill (parent->gfid, postparent);
-
-        if (IA_ISDIR (buf->ia_type)) {
-                SVS_STRDUP (inode_ctx->snapname, parent_ctx->snapname);
-                if (!inode_ctx->snapname) {
-                        op_ret = -1;
-                        *op_errno = ENOMEM;
-                        goto out;
-                }
+    int32_t op_ret = -1;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat statbuf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+    glfs_object_t *parent_object = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent_ctx, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+
+    parent_object = parent_ctx->object;
+    fs = parent_ctx->fs;
+
+    object = glfs_h_lookupat(fs, parent_object, loc->name, &statbuf, 0);
+    if (!object) {
+        /* should this be in WARNING or ERROR mode? */
+        gf_msg_debug(this->name, 0,
+                     "failed to do lookup and "
+                     "get the handle for entry %s (path: %s)",
+                     loc->name, loc->path);
+        op_ret = -1;
+        *op_errno = errno;
+        goto out;
+    }
+
+    if (gf_uuid_is_null(object->gfid)) {
+        /* should this be in WARNING or ERROR mode? */
+        gf_msg_debug(this->name, 0,
+                     "gfid from glfs handle is "
+                     "NULL for entry %s (path: %s)",
+                     loc->name, loc->path);
+        op_ret = -1;
+        *op_errno = errno;
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_NEW_INODE_CTX_FAILED,
+               "failed to allocate "
+               "inode context");
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+        if (svs_uuid_generate(this, gfid, parent_ctx->snapname, object->gfid)) {
+            /*
+             * should op_errno be something else such as
+             * EINVAL or ESTALE?
+             */
+            op_ret = -1;
+            *op_errno = EIO;
+            goto out;
+        }
+    } else {
+        if (!gf_uuid_is_null(loc->inode->gfid))
+            gf_uuid_copy(gfid, loc->inode->gfid);
+        else
+            gf_uuid_copy(gfid, loc->gfid);
+    }
+
+    iatt_from_stat(buf, &statbuf);
+    gf_uuid_copy(buf->ia_gfid, gfid);
+    svs_fill_ino_from_gfid(buf);
+    inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
+    inode_ctx->fs = fs;
+    inode_ctx->object = object;
+    memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+    svs_iatt_fill(parent->gfid, postparent);
+
+    if (IA_ISDIR(buf->ia_type)) {
+        SVS_STRDUP(inode_ctx->snapname, parent_ctx->snapname);
+        if (!inode_ctx->snapname) {
+            op_ret = -1;
+            *op_errno = ENOMEM;
+            goto out;
         }
+    }
 
-        op_ret = 0;
+    op_ret = 0;
 
 out:
-        if (op_ret) {
-                if (object)
-                        glfs_h_close (object);
+    if (op_ret) {
+        if (object)
+            glfs_h_close(object);
 
-                if (inode_ctx)
-                        inode_ctx->object = NULL;
-        }
+        if (inode_ctx)
+            inode_ctx->object = NULL;
+    }
 
-        return op_ret;
+    return op_ret;
 }
 
 /* inode context is there means lookup has come on an object which was
@@ -397,346 +470,352 @@ out:
      world
 */
 int32_t
-svs_revalidate (xlator_t *this, loc_t *loc, inode_t *parent,
-                svs_inode_t *inode_ctx, svs_inode_t *parent_ctx,
-                struct iatt *buf, struct iatt *postparent, int32_t *op_errno)
+svs_revalidate(xlator_t *this, loc_t *loc, inode_t *parent,
+               svs_inode_t *inode_ctx, svs_inode_t *parent_ctx,
+               struct iatt *buf, struct iatt *postparent, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        int             ret                             = -1;
-        char            tmp_uuid[64]                    = {0, };
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, out);
-
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                svs_iatt_fill (loc->inode->gfid, buf);
+    int32_t op_ret = -1;
+    int ret = -1;
+    char tmp_uuid[64] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode_ctx, out);
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        svs_iatt_fill(loc->inode->gfid, buf);
+        if (parent)
+            svs_iatt_fill(parent->gfid, postparent);
+        else
+            svs_iatt_fill(loc->inode->gfid, postparent);
+        op_ret = 0;
+        goto out;
+    } else {
+        /* Though fs and object are present in the inode context, its
+         * better to check if fs is valid or not before doing anything.
+         * Its for the protection from the following operations.
+         * 1) Create a file on the glusterfs mount point
+         * 2) Create a snapshot (say "snap1")
+         * 3) Access the contents of the snapshot
+         * 4) Delete the file from the mount point
+         * 5) Delete the snapshot "snap1"
+         * 6) Create a new snapshot "snap1"
+         *
+         * Now accessing the new snapshot "snap1" gives problems.
+         * Because the inode and dentry created for snap1 would not be
+         * deleted upon the deletion of the snapshot (as deletion of
+         * snapshot is a gluster cli operation, not a fop). So next time
+         * upon creation of a new snap with same name, the previous
+         * inode and dentry itself will be used. But the inode context
+         * contains old information about the glfs_t instance and the
+         * handle in the gfapi world. Thus the glfs_t instance should
+         * be checked before accessing. If its wrong, then right
+         * instance should be obtained by doing the lookup.
+         */
+        if (inode_ctx->fs && inode_ctx->object) {
+            fs = inode_ctx->fs;
+            SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
+            if (fs) {
+                memcpy(buf, &inode_ctx->buf, sizeof(*buf));
                 if (parent)
-                        svs_iatt_fill (parent->gfid,
-                                       postparent);
+                    svs_iatt_fill(parent->gfid, postparent);
                 else
-                        svs_iatt_fill (loc->inode->gfid, postparent);
+                    svs_iatt_fill(buf->ia_gfid, postparent);
                 op_ret = 0;
                 goto out;
-        } else {
-                /* Though fs and object are present in the inode context, its
-                 * better to check if fs is valid or not before doing anything.
-                 * Its for the protection from the following operations.
-                 * 1) Create a file on the glusterfs mount point
-                 * 2) Create a snapshot (say "snap1")
-                 * 3) Access the contents of the snapshot
-                 * 4) Delete the file from the mount point
-                 * 5) Delete the snapshot "snap1"
-                 * 6) Create a new snapshot "snap1"
-                 *
-                 * Now accessing the new snapshot "snap1" gives problems.
-                 * Because the inode and dentry created for snap1 would not be
-                 * deleted upon the deletion of the snapshot (as deletion of
-                 * snapshot is a gluster cli operation, not a fop). So next time
-                 * upon creation of a new snap with same name, the previous
-                 * inode and dentry itself will be used. But the inode context
-                 * contains old information about the glfs_t instance and the
-                 * handle in the gfapi world. Thus the glfs_t instance should
-                 * be checked before accessing. If its wrong, then right
-                 * instance should be obtained by doing the lookup.
-                 */
-                if (inode_ctx->fs && inode_ctx->object) {
-                        fs = inode_ctx->fs;
-                        object = inode_ctx->object;
-                        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
-                        if (fs) {
-                                memcpy (buf, &inode_ctx->buf, sizeof (*buf));
-                                if (parent)
-                                        svs_iatt_fill (parent->gfid,
-                                                       postparent);
-                                else
-                                        svs_iatt_fill (buf->ia_gfid,
-                                                       postparent);
-                                op_ret = 0;
-                                goto out;
-                        } else {
-                                inode_ctx->fs = NULL;
-                                inode_ctx->object = NULL;
-                                ret = svs_get_handle (this, loc, inode_ctx,
-                                                      op_errno);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to get the handle for "
-                                                "%s (gfid %s)", loc->path,
-                                                uuid_utoa_r (loc->inode->gfid,
-                                                             tmp_uuid));
-                                        op_ret = -1;
-                                        goto out;
-                                }
-                        }
+            } else {
+                inode_ctx->fs = NULL;
+                inode_ctx->object = NULL;
+                ret = svs_get_handle(this, loc, inode_ctx, op_errno);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                           SVS_MSG_GET_GLFS_H_OBJECT_FAILED,
+                           "failed to get the handle for "
+                           "%s (gfid %s)",
+                           loc->path, uuid_utoa_r(loc->inode->gfid, tmp_uuid));
+                    op_ret = -1;
+                    goto out;
                 }
+            }
+        }
 
-                /* To send the lookup to gfapi world, both the name of the
-                   entry as well as the parent context is needed.
-                */
-                if (!loc->name || !parent_ctx) {
-                        *op_errno = ESTALE;
-                        gf_log (this->name, GF_LOG_ERROR, "%s is NULL",
-                                loc->name?"parent context":"loc->name");
-                        goto out;
-                }
+        /* To send the lookup to gfapi world, both the name of the
+           entry as well as the parent context is needed.
+        */
+        if (!loc->name || !parent_ctx) {
+            *op_errno = ESTALE;
+            gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                   SVS_MSG_PARENT_CTX_OR_NAME_NULL, "%s is NULL",
+                   loc->name ? "parent context" : "loc->name");
+            goto out;
+        }
 
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        op_ret = svs_lookup_snapshot (this, loc, buf,
-                                                      postparent, parent,
-                                                      parent_ctx, op_errno);
-                else
-                        op_ret = svs_lookup_entry (this, loc, buf, postparent,
-                                                   parent, parent_ctx,
-                                                   op_errno);
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            op_ret = svs_lookup_snapshot(this, loc, buf, postparent, parent,
+                                         parent_ctx, op_errno);
+        else
+            op_ret = svs_lookup_entry(this, loc, buf, postparent, parent,
+                                      parent_ctx, op_errno);
 
-                goto out;
-        }
+        goto out;
+    }
 
 out:
-        return op_ret;
+    return op_ret;
 }
 
 int32_t
-svs_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+svs_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        struct iatt    buf                            = {0, };
-        int32_t        op_ret                         = -1;
-        int32_t        op_errno                       = EINVAL;
-        struct iatt    postparent                     = {0,};
-        svs_inode_t   *inode_ctx                      = NULL;
-        svs_inode_t   *parent_ctx                     = NULL;
-        int32_t        ret                            = -1;
-        svs_private_t *private                        = NULL;
-        inode_t       *parent                         = NULL;
-        snap_dirent_t *dirent                         = NULL;
-        gf_boolean_t   entry_point_key                = _gf_false;
-        gf_boolean_t   entry_point                    = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        private = this->private;
-
-        /* For lookups sent on inodes (i.e not parent inode + basename, but
-           direct inode itself which usually is a nameless lookup or revalidate
-           on the inode), loc->name will not be there. Get it from path if
-           it is there.
-           This is the difference between nameless lookup and revalidate lookup
-           on an inode:
-           nameless lookup: loc->path contains gfid and strrchr on it fails
-           revalidate lookup: loc->path contains the entry name of the inode
-                              and strrchr gives the name of the entry from path
-        */
-        if (loc->path) {
-                if (!loc->name || (loc->name && !strcmp (loc->name, ""))) {
-                        loc->name = strrchr (loc->path, '/');
-                        if (loc->name)
-                                loc->name++;
-                }
-        }
-
-        if (loc->parent)
-                parent = inode_ref (loc->parent);
-        else {
-                parent = inode_find (loc->inode->table, loc->pargfid);
-                if (!parent)
-                        parent = inode_parent (loc->inode, NULL, NULL);
-        }
-        if (parent)
-                parent_ctx = svs_inode_ctx_get (this, parent);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-
-        /* Initialize latest snapshot, which is used for nameless lookups */
-        dirent = svs_get_latest_snap_entry (this);
-
-        if (dirent && !dirent->fs) {
-                svs_initialise_snapshot_volume (this, dirent->name, NULL);
-        }
-
-        if (xdata && !inode_ctx) {
-                ret = dict_get_str_boolean (xdata, "entry-point", _gf_false);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "failed to get the "
-                                "entry point info");
-                        entry_point_key = _gf_false;
-                } else {
-                        entry_point_key = ret;
-                }
-
-                if (loc->name && strlen (loc->name)) {
-                        /* lookup can come with the entry-point set in the dict
-                        * for the parent directory of the entry-point as well.
-                        * So consider entry_point only for named lookup
-                        */
-                        entry_point = entry_point_key;
-                }
-        }
-
-        if (inode_ctx && inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                /* entry-point may not be set in the dictonary.
-                 * This can happen if snap-view client is restarted where
-                 * inode-ctx not available and a nameless lookup has come
-                 */
-                entry_point = _gf_true;
-        }
-
-        /* lookup is on the entry point to the snapshot world */
-        if (entry_point) {
-                op_ret = svs_lookup_entry_point (this, loc, parent, &buf,
-                                                 &postparent, &op_errno);
-                goto out;
+    struct iatt buf = {
+        0,
+    };
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    struct iatt postparent = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+    svs_inode_t *parent_ctx = NULL;
+    int32_t ret = -1;
+    inode_t *parent = NULL;
+    gf_boolean_t entry_point_key = _gf_false;
+    gf_boolean_t entry_point = _gf_false;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    /* For lookups sent on inodes (i.e not parent inode + basename, but
+       direct inode itself which usually is a nameless lookup or revalidate
+       on the inode), loc->name will not be there. Get it from path if
+       it is there.
+       This is the difference between nameless lookup and revalidate lookup
+       on an inode:
+       nameless lookup: loc->path contains gfid and strrchr on it fails
+       revalidate lookup: loc->path contains the entry name of the inode
+                          and strrchr gives the name of the entry from path
+    */
+    if (loc->path) {
+        if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+            loc->name = strrchr(loc->path, '/');
+            if (loc->name)
+                loc->name++;
+        }
+    }
+
+    if (loc->parent)
+        parent = inode_ref(loc->parent);
+    else {
+        parent = inode_find(loc->inode->table, loc->pargfid);
+        if (!parent)
+            parent = inode_parent(loc->inode, NULL, NULL);
+    }
+    if (parent)
+        parent_ctx = svs_inode_ctx_get(this, parent);
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+
+    if (xdata && !inode_ctx) {
+        ret = dict_get_str_boolean(xdata, "entry-point", _gf_false);
+        if (ret == -1) {
+            gf_msg_debug(this->name, 0,
+                         "failed to get the "
+                         "entry point info");
+            entry_point_key = _gf_false;
+        } else {
+            entry_point_key = ret;
         }
 
-        /* revalidate */
-        if (inode_ctx) {
-                op_ret = svs_revalidate (this, loc, parent, inode_ctx,
-                                         parent_ctx, &buf, &postparent,
-                                         &op_errno);
-                goto out;
+        if (loc->name && strlen(loc->name)) {
+            /* lookup can come with the entry-point set in the dict
+             * for the parent directory of the entry-point as well.
+             * So consider entry_point only for named lookup
+             */
+            entry_point = entry_point_key;
         }
+    }
 
-        /* This can happen when entry point directory is entered from non-root
-           directory. (ex: if /mnt/glusterfs is the mount point, then entry
-           point (say .snaps) is entered from /mnt/glusterfs/dir/.snaps). Also
-           it can happen when client sends a nameless lookup on just a gfid and
-           the server does not have the inode in the inode table.
-        */
-        if (!inode_ctx && !parent_ctx) {
-                if (gf_uuid_is_null (loc->gfid) &&
-                    gf_uuid_is_null (loc->inode->gfid)) {
-                        gf_log (this->name, GF_LOG_DEBUG, "gfid is NULL, "
-                                "either the lookup came on missing entry or "
-                                "the entry is stale");
-                        op_ret = -1;
-                        op_errno = ESTALE;
-                        goto out;
-                }
-
-                if (!entry_point_key) {
-                        /* This can happen when there is no inode_ctx available.
-                        * snapview-server might have restarted or
-                        * graph change might have happened
-                        */
-                        op_ret = -1;
-                        op_errno = ESTALE;
-                        goto out;
-                }
-
-                /* lookup is on the parent directory of entry-point.
-                 * this would have already looked up by snap-view client
-                 * so return success
-                 */
-                if (!gf_uuid_is_null (loc->gfid))
-                        gf_uuid_copy (buf.ia_gfid, loc->gfid);
-                else
-                        gf_uuid_copy (buf.ia_gfid, loc->inode->gfid);
+    if (inode_ctx && inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        /* entry-point may not be set in the dictonary.
+         * This can happen if snap-view client is restarted where
+         * inode-ctx not available and a nameless lookup has come
+         */
+        entry_point = _gf_true;
+    }
+
+    /* lookup is on the entry point to the snapshot world */
+    if (entry_point) {
+        op_ret = svs_lookup_entry_point(this, loc, parent, &buf, &postparent,
+                                        &op_errno);
+        goto out;
+    }
+
+    /* revalidate */
+    if (inode_ctx) {
+        op_ret = svs_revalidate(this, loc, parent, inode_ctx, parent_ctx, &buf,
+                                &postparent, &op_errno);
+        goto out;
+    }
+
+    /* This can happen when entry point directory is entered from non-root
+       directory. (ex: if /mnt/glusterfs is the mount point, then entry
+       point (say .snaps) is entered from /mnt/glusterfs/dir/.snaps). Also
+       it can happen when client sends a nameless lookup on just a gfid and
+       the server does not have the inode in the inode table.
+    */
+    if (!inode_ctx && !parent_ctx) {
+        if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+            op_ret = -1;
+            op_errno = ESTALE;
+            gf_msg_debug(this->name, 0,
+                         "gfid is NULL. Either the lookup "
+                         "came on missing entry or the "
+                         "entry is stale");
+            goto out;
+        }
+
+        if (!entry_point_key) {
+            /* This can happen when there is no inode_ctx available.
+             * snapview-server might have restarted or
+             * graph change might have happened
+             */
+            op_ret = -1;
+            op_errno = ESTALE;
+            goto out;
+        }
+
+        /* lookup is on the parent directory of entry-point.
+         * this would have already looked up by snap-view client
+         * so return success
+         */
+        if (!gf_uuid_is_null(loc->gfid))
+            gf_uuid_copy(buf.ia_gfid, loc->gfid);
+        else
+            gf_uuid_copy(buf.ia_gfid, loc->inode->gfid);
 
-                svs_iatt_fill (buf.ia_gfid, &buf);
-                svs_iatt_fill (buf.ia_gfid, &postparent);
+        svs_iatt_fill(buf.ia_gfid, &buf);
+        svs_iatt_fill(buf.ia_gfid, &postparent);
 
-                op_ret = 0;
-                goto out;
-        }
+        op_ret = 0;
+        goto out;
+    }
 
-        if (parent_ctx) {
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        op_ret = svs_lookup_snapshot (this, loc, &buf,
-                                                      &postparent, parent,
-                                                      parent_ctx, &op_errno);
-                else
-                        op_ret = svs_lookup_entry (this, loc, &buf,
-                                                   &postparent, parent,
-                                                   parent_ctx, &op_errno);
-                goto out;
-        }
+    if (parent_ctx) {
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            op_ret = svs_lookup_snapshot(this, loc, &buf, &postparent, parent,
+                                         parent_ctx, &op_errno);
+        else
+            op_ret = svs_lookup_entry(this, loc, &buf, &postparent, parent,
+                                      parent_ctx, &op_errno);
+        goto out;
+    }
 
 out:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
-                             loc?loc->inode:NULL, &buf, xdata, &postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno,
+                        loc ? loc->inode : NULL, &buf, xdata, &postparent);
 
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-             dict_t *xdata)
+svs_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        svs_inode_t   *inode_ctx  = NULL;
-        int32_t        op_ret     = -1;
-        int32_t        op_errno   = EINVAL;
-        svs_fd_t      *svs_fd     = NULL;
-        glfs_fd_t     *glfd       = NULL;
-        glfs_t        *fs         = NULL;
-        glfs_object_t *object     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found "
-                        "for the inode %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = ESTALE;
-                goto out;
-        }
-
-        /* Fake success is sent if the opendir is on the entry point directory
-           or the inode is SNAP_VIEW_ENTRY_POINT_INODE
-        */
-        if  (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                op_ret = 0;
-                op_errno = 0;
-                goto out;
-        }
-        else {
+    svs_inode_t *inode_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    svs_fd_t *svs_fd = NULL;
+    glfs_fd_t *glfd = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found "
+               "for the inode %s",
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    /* Fake success is sent if the opendir is on the entry point directory
+       or the inode is SNAP_VIEW_ENTRY_POINT_INODE
+    */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_ret = 0;
+        op_errno = 0;
+        goto out;
+    } else {
+        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
+                               op_errno, out);
 
-                SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                                       op_errno, out);
-
-                glfd = glfs_h_opendir (fs, object);
-                if (!glfd) {
-                        op_ret = -1;
-                        op_errno = errno;
-                        gf_log (this->name, GF_LOG_ERROR, "opendir on %s "
-                                "failed (gfid: %s)", loc->name,
-                                uuid_utoa (loc->inode->gfid));
-                        goto out;
-                }
-                svs_fd = svs_fd_ctx_get_or_new (this, fd);
-                if (!svs_fd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to allocate "
-                                "fd context %s (gfid: %s)", loc->name,
-                                uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        glfs_closedir (glfd);
-                        goto out;
-                }
-                svs_fd->fd = glfd;
+        glfd = glfs_h_opendir(fs, object);
+        if (!glfd) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_OPENDIR_FAILED,
+                   "opendir on %s failed "
+                   "(gfid: %s)",
+                   loc->name, uuid_utoa(loc->inode->gfid));
+            goto out;
+        }
+        svs_fd = svs_fd_ctx_get_or_new(this, fd);
+        if (!svs_fd) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_NEW_FD_CTX_FAILED,
+                   "failed to allocate fd context "
+                   "for %s (gfid: %s)",
+                   loc->name, uuid_utoa(fd->inode->gfid));
+            glfs_closedir(glfd);
+            goto out;
+        }
+        svs_fd->fd = glfd;
 
-                op_ret = 0;
-                op_errno = 0;
-        }
+        op_ret = 0;
+        op_errno = 0;
+    }
 
 out:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, NULL);
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, NULL);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -749,603 +828,726 @@ out:
  * back into the dict. But to get the values for those xattrs it has to do the
  * getxattr operation on each xattr which might turn out to be a costly
  * operation. So for each of the xattrs present in the list, a 0 byte value
- * ("") is set into the dict before unwinding. This can be treated as an
+ * ("") is set into the dict before unwinding. Since ("") is also a valid xattr
+ * value(in a file system) we use an extra key in the same dictionary as an
  * indicator to other xlators which want to cache the xattrs (as of now,
  * md-cache which caches acl and selinux related xattrs) to not to cache the
  * values of the xattrs present in the dict.
  */
 int32_t
-svs_add_xattrs_to_dict (xlator_t *this, dict_t *dict, char *list, ssize_t size)
+svs_add_xattrs_to_dict(xlator_t *this, dict_t *dict, char *list, ssize_t size)
 {
-        char           keybuffer[4096]  = {0,};
-        size_t         remaining_size   = 0;
-        int32_t        list_offset      = 0;
-        int32_t        ret              = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, list, out);
-
-        remaining_size = size;
-        list_offset = 0;
-        while (remaining_size > 0) {
-                strncpy (keybuffer, list + list_offset, sizeof (keybuffer) - 1);
+    char keybuffer[4096] = {
+        0,
+    };
+    size_t remaining_size = 0;
+    int32_t list_offset = 0;
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, list, out);
+
+    remaining_size = size;
+    list_offset = 0;
+    while (remaining_size > 0) {
+        strncpy(keybuffer, list + list_offset, sizeof(keybuffer) - 1);
 #ifdef GF_DARWIN_HOST_OS
-                /* The protocol expect namespace for now */
-                char *newkey = NULL;
-                gf_add_prefix (XATTR_USER_PREFIX, keybuffer, &newkey);
-                strcpy (keybuffer, newkey);
-                GF_FREE (newkey);
+        /* The protocol expect namespace for now */
+        char *newkey = NULL;
+        gf_add_prefix(XATTR_USER_PREFIX, keybuffer, &newkey);
+        strcpy(keybuffer, newkey);
+        GF_FREE(newkey);
 #endif
-                ret = dict_set_str (dict, keybuffer, "");
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "dict set operation "
-                                "for the key %s failed.", keybuffer);
-                        goto out;
-                }
+        ret = dict_set_str(dict, keybuffer, "");
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DICT_SET_FAILED,
+                   "dict set operation "
+                   "for the key %s failed.",
+                   keybuffer);
+            goto out;
+        }
 
-                remaining_size -= strlen (keybuffer) + 1;
-                list_offset += strlen (keybuffer) + 1;
-        } /* while (remaining_size > 0) */
+        remaining_size -= strlen(keybuffer) + 1;
+        list_offset += strlen(keybuffer) + 1;
+    } /* while (remaining_size > 0) */
 
-        ret = 0;
+    /* Add an additional key to indicate that we don't need to cache these
+     * xattrs(with value "") */
+    ret = dict_set_str(dict, "glusterfs.skip-cache", "");
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DICT_SET_FAILED,
+               "dict set operation for the key glusterfs.skip-cache failed.");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-svs_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
-              dict_t *xdata)
+svs_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
 {
-        svs_inode_t   *inode_ctx        = NULL;
-        int32_t        op_ret           = -1;
-        int32_t        op_errno         = EINVAL;
-        glfs_t        *fs               = NULL;
-        glfs_object_t *object           = NULL;
-        char          *value            = 0;
-        ssize_t        size             = 0;
-        dict_t        *dict             = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", frame, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", loc, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found "
-                        "for the inode %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = ESTALE;
-                goto out;
-        }
+    svs_inode_t *inode_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    char *value = 0;
+    ssize_t size = 0;
+    dict_t *dict = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", frame, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", loc, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found "
+               "for the inode %s",
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    /* ENODATA is sent if the getxattr is on entry point directory
+       or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
+       a virtual directory on which setxattr operations are not
+       allowed. If getxattr has to be faked as success, then a value
+       for the name of the xattr has to be sent which we don't have.
+    */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_ret = -1;
+        op_errno = ENODATA;
+        goto out;
+    } else {
+        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
+                               op_errno, out);
 
-        /* ENODATA is sent if the getxattr is on entry point directory
-           or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
-           a virtual directory on which setxattr operations are not
-           allowed. If getxattr has to be faked as success, then a value
-           for the name of the xattr has to be sent which we dont have.
-        */
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                op_ret = -1;
-                op_errno = ENODATA;
+        dict = dict_new();
+        if (!dict) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            goto out;
+        }
+
+        size = glfs_h_getxattrs(fs, object, name, NULL, 0);
+        if (size == -1) {
+            op_ret = -1;
+            op_errno = errno;
+            if (errno == ENODATA) {
+                gf_msg_debug(this->name, 0,
+                             "getxattr on "
+                             "%s failed (ket: %s) with %s",
+                             loc->path, name, strerror(errno));
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_GETXATTR_FAILED,
+                       "getxattr on %s failed (key: %s) with %s", loc->path,
+                       name, strerror(errno));
+            }
+            goto out;
+        }
+        value = GF_CALLOC(size + 1, sizeof(char), gf_common_mt_char);
+        if (!value) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                   "failed to allocate memory for getxattr "
+                   "on %s (key: %s)",
+                   loc->name, name);
+            goto out;
+        }
+
+        size = glfs_h_getxattrs(fs, object, name, value, size);
+        if (size == -1) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_GETXATTR_FAILED,
+                   "failed to get the xattr %s for "
+                   "entry %s",
+                   name, loc->name);
+            goto out;
+        }
+        value[size] = '\0';
+
+        if (name) {
+            op_ret = dict_set_dynptr(dict, (char *)name, value, size);
+            if (op_ret < 0) {
+                op_errno = -op_ret;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_DICT_SET_FAILED,
+                       "dict set operation for %s for "
+                       "the key %s failed.",
+                       loc->path, name);
+                GF_FREE(value);
+                value = NULL;
                 goto out;
+            }
+        } else {
+            op_ret = svs_add_xattrs_to_dict(this, dict, value, size);
+            if (op_ret == -1) {
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to add xattrs from the list to "
+                       "dict for %s (gfid: %s)",
+                       loc->path, uuid_utoa(loc->inode->gfid));
+                goto out;
+            }
+            GF_FREE(value);
+            value = NULL;
         }
-        else {
-
-                SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                                       op_errno, out);
-
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate dict");
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                size = glfs_h_getxattrs (fs, object, name, NULL, 0);
-                if (size == -1) {
-                        gf_log (this->name,
-                                errno == ENODATA?GF_LOG_DEBUG:GF_LOG_ERROR,
-                                "getxattr on %s failed (key: %s) with %s",
-                                loc->path, name, strerror(errno));
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
-                value = GF_CALLOC (size + 1, sizeof (char),
-                                   gf_common_mt_char);
-                if (!value) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate memory for getxattr on %s "
-                                "(key: %s)", loc->name, name);
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                size = glfs_h_getxattrs (fs, object, name, value, size);
-                if (size == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "get the xattr %s for entry %s", name,
-                                loc->name);
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
-                value[size] = '\0';
-
-                if (name) {
-                        op_ret = dict_set_dynptr (dict, (char *)name, value,
-                                                  size);
-                        if (op_ret < 0) {
-                                op_errno = -op_ret;
-                                gf_log (this->name, GF_LOG_ERROR, "dict set "
-                                        "operation for %s for the key %s "
-                                        "failed.", loc->path, name);
-                                GF_FREE (value);
-                                value = NULL;
-                                goto out;
-                        }
-                } else {
-                        op_ret = svs_add_xattrs_to_dict (this, dict, value,
-                                                         size);
-                        if (op_ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "add the xattrs from the list to dict");
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-                        GF_FREE (value);
-                }
-        }
+    }
 
 out:
-        if (op_ret)
-                GF_FREE (value);
+    if (op_ret && value)
+        GF_FREE(value);
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, NULL);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, NULL);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
-               dict_t *xdata)
+svs_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
 {
-        svs_inode_t *inode_ctx  = NULL;
-        int32_t      op_ret     = -1;
-        int32_t      op_errno   = EINVAL;
-        char        *value      = 0;
-        ssize_t      size       = 0;
-        dict_t      *dict       = NULL;
-        svs_fd_t    *sfd        = NULL;
-        glfs_fd_t   *glfd       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", frame, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", fd, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", fd->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found "
-                        "for the inode %s", uuid_utoa (fd->inode->gfid));
+    svs_inode_t *inode_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    char *value = 0;
+    ssize_t size = 0;
+    dict_t *dict = NULL;
+    svs_fd_t *sfd = NULL;
+    glfs_fd_t *glfd = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", frame, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", fd, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", fd->inode, out);
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found "
+               "for the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    if (!(svs_inode_ctx_glfs_mapping(this, inode_ctx))) {
+        op_ret = -1;
+        op_errno = EBADF;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_FS_INSTANCE_INVALID,
+               "glfs instance %p to which the inode %s "
+               "belongs to does not exist. The snapshot "
+               "corresponding to the instance might have"
+               "been deleted or deactivated",
+               inode_ctx->fs, uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    sfd = svs_fd_ctx_get_or_new(this, fd);
+    if (!sfd) {
+        op_ret = -1;
+        op_errno = EBADFD;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_FD_CONTEXT_FAILED,
+               "failed to get the fd "
+               "context for %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    glfd = sfd->fd;
+    /* EINVAL is sent if the getxattr is on entry point directory
+       or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
+       a virtual directory on which setxattr operations are not
+       allowed. If getxattr has to be faked as success, then a value
+       for the name of the xattr has to be sent which we don't have.
+    */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    } else {
+        dict = dict_new();
+        if (!dict) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                   "failed to allocate dict "
+                   "(gfid: %s, key: %s)",
+                   uuid_utoa(fd->inode->gfid), name);
+            goto out;
+        }
+
+        if (name) {
+            size = glfs_fgetxattr(glfd, name, NULL, 0);
+            if (size == -1) {
                 op_ret = -1;
-                op_errno = ESTALE;
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_GETXATTR_FAILED,
+                       "getxattr on %s failed "
+                       "(key: %s)",
+                       uuid_utoa(fd->inode->gfid), name);
                 goto out;
-        }
+            }
+            value = GF_CALLOC(size + 1, sizeof(char), gf_common_mt_char);
+            if (!value) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to "
+                       "allocate memory for getxattr on %s "
+                       "(key: %s)",
+                       uuid_utoa(fd->inode->gfid), name);
+                goto out;
+            }
 
-        sfd = svs_fd_ctx_get_or_new (this, fd);
-        if (!sfd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
+            size = glfs_fgetxattr(glfd, name, value, size);
+            if (size == -1) {
                 op_ret = -1;
-                op_errno = EBADFD;
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_GETXATTR_FAILED,
+                       "failed to get the xattr %s "
+                       "for inode %s",
+                       name, uuid_utoa(fd->inode->gfid));
                 goto out;
-        }
+            }
+            value[size] = '\0';
+
+            op_ret = dict_set_dynptr(dict, (char *)name, value, size);
+            if (op_ret < 0) {
+                op_errno = -op_ret;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_DICT_SET_FAILED,
+                       "dict set operation for gfid %s "
+                       "for the key %s failed.",
+                       uuid_utoa(fd->inode->gfid), name);
+                goto out;
+            }
+        } else {
+            size = glfs_flistxattr(glfd, NULL, 0);
+            if (size == -1) {
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_LISTXATTR_FAILED, "listxattr on %s failed",
+                       uuid_utoa(fd->inode->gfid));
+                goto out;
+            }
 
-        glfd = sfd->fd;
-        /* EINVAL is sent if the getxattr is on entry point directory
-           or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
-           a virtual directory on which setxattr operations are not
-           allowed. If getxattr has to be faked as success, then a value
-           for the name of the xattr has to be sent which we dont have.
-        */
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+            value = GF_CALLOC(size + 1, sizeof(char), gf_common_mt_char);
+            if (!value) {
                 op_ret = -1;
-                op_errno = EINVAL;
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to "
+                       "allocate buffer for xattr "
+                       "list (%s)",
+                       uuid_utoa(fd->inode->gfid));
                 goto out;
-        }
-        else {
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate  dict");
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
+            }
 
-                if (name) {
-                        size = glfs_fgetxattr (glfd, name, NULL, 0);
-                        if (size == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "getxattr on "
-                                        "%s failed (key: %s)",
-                                        uuid_utoa (fd->inode->gfid), name);
-                                op_ret = -1;
-                                op_errno = errno;
-                                goto out;
-                        }
-                        value = GF_CALLOC (size + 1, sizeof (char),
-                                           gf_common_mt_char);
-                        if (!value) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "allocate memory for getxattr on %s "
-                                        "(key: %s)",
-                                        uuid_utoa (fd->inode->gfid), name);
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        size = glfs_fgetxattr (glfd, name, value, size);
-                        if (size == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "get the xattr %s for inode %s", name,
-                                        uuid_utoa (fd->inode->gfid));
-                                op_ret = -1;
-                                op_errno = errno;
-                                goto out;
-                        }
-                        value[size] = '\0';
-
-                        op_ret = dict_set_dynptr (dict, (char *)name, value,
-                                                  size);
-                        if (op_ret < 0) {
-                                op_errno = -op_ret;
-                                gf_log (this->name, GF_LOG_ERROR, "dict set "
-                                        "operation for gfid %s for the key %s "
-                                        "failed.",
-                                        uuid_utoa (fd->inode->gfid), name);
-                                GF_FREE (value);
-                                goto out;
-                        }
-                } else {
-                        size = glfs_flistxattr (glfd, NULL, 0);
-                        if (size == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "listxattr "
-                                        "on %s failed",
-                                        uuid_utoa (fd->inode->gfid));
-                                goto out;
-                        }
-
-                        value = GF_CALLOC (size + 1, sizeof (char),
-                                           gf_common_mt_char);
-                        if (!value) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "allocate buffer for xattr list (%s)",
-                                        uuid_utoa (fd->inode->gfid));
-                                goto out;
-                        }
-
-                        size = glfs_flistxattr (glfd, value, size);
-                        if (size == -1) {
-                                op_ret = -1;
-                                op_errno = errno;
-                                gf_log (this->name, GF_LOG_ERROR, "listxattr "
-                                        "on %s failed",
-                                        uuid_utoa (fd->inode->gfid));
-                                goto out;
-                        }
-
-                        op_ret = svs_add_xattrs_to_dict (this, dict, value,
-                                                         size);
-                        if (op_ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "add the xattrs from the list to dict");
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-                        GF_FREE (value);
-                }
+            size = glfs_flistxattr(glfd, value, size);
+            if (size == -1) {
+                op_ret = -1;
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_LISTXATTR_FAILED, "listxattr on %s failed",
+                       uuid_utoa(fd->inode->gfid));
+                goto out;
+            }
 
-                op_ret = 0;
-                op_errno = 0;
+            op_ret = svs_add_xattrs_to_dict(this, dict, value, size);
+            if (op_ret == -1) {
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to add xattrs from the list "
+                       "to dict (gfid: %s)",
+                       uuid_utoa(fd->inode->gfid));
+                goto out;
+            }
+            GF_FREE(value);
         }
 
+        op_ret = 0;
+        op_errno = 0;
+    }
+
 out:
-        if (op_ret)
-                GF_FREE (value);
+    if (op_ret)
+        GF_FREE(value);
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, NULL);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_releasedir (xlator_t *this, fd_t *fd)
+svs_releasedir(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *sfd      = NULL;
-        uint64_t          tmp_pfd  = 0;
-        int               ret      = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        ret = fd_ctx_del (fd, this, &tmp_pfd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pfd from fd=%p is NULL", fd);
-                goto out;
-        }
-
-        sfd = (svs_fd_t *)(long)tmp_pfd;
-        if (sfd->fd) {
-                ret = glfs_closedir (sfd->fd);
+    svs_fd_t *sfd = NULL;
+    uint64_t tmp_pfd = 0;
+    int ret = 0;
+    svs_inode_t *svs_inode = NULL;
+    glfs_t *fs = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = fd_ctx_del(fd, this, &tmp_pfd);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "pfd from fd=%p is NULL", fd);
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    svs_inode = svs_inode_ctx_get(this, inode);
+    if (svs_inode) {
+        fs = svs_inode->fs; /* should inode->lock be held for this? */
+        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
+        if (fs) {
+            sfd = (svs_fd_t *)(long)tmp_pfd;
+            if (sfd->fd) {
+                ret = glfs_closedir(sfd->fd);
                 if (ret)
-                        gf_log (this->name, GF_LOG_WARNING, "failed to close "
-                                "the glfd for directory %s",
-                                uuid_utoa (fd->inode->gfid));
+                    gf_msg(this->name, GF_LOG_WARNING, errno,
+                           SVS_MSG_RELEASEDIR_FAILED,
+                           "failed to close the glfd for "
+                           "directory %s",
+                           uuid_utoa(fd->inode->gfid));
+            }
         }
+    }
 
-        GF_FREE (sfd);
+    GF_FREE(sfd);
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_flush (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, dict_t *xdata)
+svs_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t          op_ret         = -1;
-        int32_t          op_errno       = 0;
-        int              ret            = -1;
-        uint64_t         value          = 0;
-        svs_inode_t     *inode_ctx      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " the inode %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret < 0 && inode_ctx->type != SNAP_VIEW_ENTRY_POINT_INODE) {
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "pfd is NULL on fd=%p", fd);
-                goto out;
-        }
-
-        op_ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    uint64_t value = 0;
+    svs_inode_t *inode_ctx = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for"
+               " the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    ret = fd_ctx_get(fd, this, &value);
+    if (ret < 0 && inode_ctx->type != SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               SVS_MSG_GET_FD_CONTEXT_FAILED, "pfd is NULL on fd=%p", fd);
+        goto out;
+    }
+
+    op_ret = 0;
 
 out:
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_release (xlator_t *this, fd_t *fd)
+svs_release(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *sfd      = NULL;
-        uint64_t          tmp_pfd  = 0;
-        int               ret      = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        ret = fd_ctx_del (fd, this, &tmp_pfd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pfd from fd=%p is NULL", fd);
-                goto out;
-        }
-
-        sfd = (svs_fd_t *)(long)tmp_pfd;
-        if (sfd->fd) {
-                ret = glfs_close (sfd->fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to close "
-                                "the glfd for %s",
-                                uuid_utoa (fd->inode->gfid));
-                }
+    svs_fd_t *sfd = NULL;
+    uint64_t tmp_pfd = 0;
+    int ret = 0;
+    inode_t *inode = NULL;
+    svs_inode_t *svs_inode = NULL;
+    glfs_t *fs = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = fd_ctx_del(fd, this, &tmp_pfd);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "pfd from fd=%p is NULL", fd);
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    svs_inode = svs_inode_ctx_get(this, inode);
+    if (svs_inode) {
+        fs = svs_inode->fs; /* should inode->lock be held for this? */
+        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
+        if (fs) {
+            sfd = (svs_fd_t *)(long)tmp_pfd;
+            if (sfd->fd) {
+                ret = glfs_close(sfd->fd);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           SVS_MSG_RELEASE_FAILED,
+                           "failed to close "
+                           "the glfd for %s",
+                           uuid_utoa(fd->inode->gfid));
+            }
         }
+    }
 
-        GF_FREE (sfd);
+    GF_FREE(sfd);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_forget  (xlator_t *this, inode_t *inode)
+svs_forget(xlator_t *this, inode_t *inode)
 {
-        int       ret = -1;
-        uint64_t  value = 0;
-        svs_inode_t  *inode_ctx = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = inode_ctx_del (inode, this, &value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to delte the inode "
-                        "context of %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        inode_ctx = (svs_inode_t *)value;
-        if (!inode_ctx)
-                goto out;
-
-        if (inode_ctx->snapname)
-                GF_FREE (inode_ctx->snapname);
-
-        GF_FREE (inode_ctx);
+    int ret = -1;
+    uint64_t value = 0;
+    svs_inode_t *inode_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = inode_ctx_del(inode, this, &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DELETE_INODE_CONTEXT_FAILED,
+               "failed to delete the inode "
+               "context of %s",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    inode_ctx = (svs_inode_t *)(uintptr_t)value;
+    if (!inode_ctx)
+        goto out;
+
+    if (inode_ctx->snapname)
+        GF_FREE(inode_ctx->snapname);
+
+    /*
+     * glfs_h_close leads to unref and forgetting of the
+     * underlying inode in the gfapi world. i.e. the inode
+     * which inode_ctx->object points to.
+     * As of now the only possibility is, this forget came as a
+     * result of snapdaemon's inode table reaching the lru
+     * limit and receiving forget as a result of purging of
+     * extra inodes that exceeded the limit. But, care must
+     * be taken to ensure that, the gfapi instance to which
+     * the glfs_h_object belongs to is not deleted. Otherwise
+     * this might result in access of a freed pointer.
+     * This will still be helpful in reducing the memory
+     * footprint of snapdaemon when the fs instance itself is
+     * valid (i.e. present and not destroyed due to either snap
+     * deactivate or snap delete), but the lru limit is reached.
+     * The forget due to lru limit will make the underlying inode
+     * being unrefed and forgotten.
+     */
+    if (svs_inode_ctx_glfs_mapping(this, inode_ctx)) {
+        glfs_h_close(inode_ctx->object);
+        inode_ctx->object = NULL;
+    }
+    GF_FREE(inode_ctx);
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-svs_fill_readdir (xlator_t *this, gf_dirent_t *entries, size_t size, off_t off)
+svs_fill_readdir(xlator_t *this, gf_dirent_t *entries, size_t size, off_t off)
 {
-        gf_dirent_t             *entry          = NULL;
-        svs_private_t           *priv           = NULL;
-        int                     i               = 0;
-        snap_dirent_t           *dirents        = NULL;
-        int                     this_size       = 0;
-        int                     filled_size     = 0;
-        int                     count           = 0;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", entries, out);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* create the dir entries */
-        LOCK (&priv->snaplist_lock);
-        {
-                dirents = priv->dirents;
-
-                for (i = off; i < priv->num_snaps; ) {
-                        this_size = sizeof (gf_dirent_t) +
-                                strlen (dirents[i].name) + 1;
-                        if (this_size + filled_size > size )
-                                goto unlock;
-
-                        entry = gf_dirent_for_name (dirents[i].name);
-                        if (!entry) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dentry for %s",
-                                        dirents[i].name);
-                                goto unlock;
-                        }
-
-                        entry->d_off = i + 1;
-                        /*
-                         * readdir on the entry-point directory to the snapshot
-                         * world, will return elements in the list of the
-                         * snapshots as the directory entries. Since the entries
-                         * returned are virtual entries which does not exist
-                         * physically on the disk, pseudo inode numbers are
-                         * generated.
-                         */
-                        entry->d_ino = i + 2*42;
-                        entry->d_type = DT_DIR;
-                        list_add_tail (&entry->list, &entries->list);
-                        ++i;
-                        count++;
-                        filled_size += this_size;
-                }
-        }
+    gf_dirent_t *entry = NULL;
+    svs_private_t *priv = NULL;
+    int i = 0;
+    snap_dirent_t *dirents = NULL;
+    int this_size = 0;
+    int filled_size = 0;
+    int count = 0;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", entries, out);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* create the dir entries */
+    LOCK(&priv->snaplist_lock);
+    {
+        dirents = priv->dirents;
+
+        for (i = off; i < priv->num_snaps;) {
+            this_size = sizeof(gf_dirent_t) + strlen(dirents[i].name) + 1;
+            if (this_size + filled_size > size)
+                goto unlock;
+
+            entry = gf_dirent_for_name(dirents[i].name);
+            if (!entry) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+                       "failed to allocate dentry for %s", dirents[i].name);
+                goto unlock;
+            }
+
+            entry->d_off = i + 1;
+            /*
+             * readdir on the entry-point directory to the snapshot
+             * world, will return elements in the list of the
+             * snapshots as the directory entries. Since the entries
+             * returned are virtual entries which does not exist
+             * physically on the disk, pseudo inode numbers are
+             * generated.
+             */
+            entry->d_ino = i + 2 * 42;
+            entry->d_type = DT_DIR;
+            list_add_tail(&entry->list, &entries->list);
+            ++i;
+            count++;
+            filled_size += this_size;
+        }
+    }
 unlock:
-        UNLOCK (&priv->snaplist_lock);
+    UNLOCK(&priv->snaplist_lock);
 
 out:
-        return count;
+    return count;
 }
 
 int32_t
-svs_glfs_readdir (xlator_t *this, glfs_fd_t *glfd, gf_dirent_t *entries,
-                  int32_t *op_errno, struct iatt *buf, gf_boolean_t readdirplus,
-                  size_t size)
+svs_glfs_readdir(xlator_t *this, glfs_fd_t *glfd, gf_dirent_t *entries,
+                 int32_t *op_errno, struct iatt *buf, gf_boolean_t readdirplus,
+                 size_t size)
 {
-        int              filled_size    = 0;
-        int              this_size      = 0;
-        int32_t          ret            = -1;
-        int32_t          count          = 0;
-        gf_dirent_t     *entry          = NULL;
-        struct   dirent *dirents        = NULL;
-        struct   dirent  de             = {0, };
-        struct stat      statbuf        = {0, };
-        off_t            in_case        = -1;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, glfd, out);
-        GF_VALIDATE_OR_GOTO (this->name, entries, out);
-
-        while (filled_size < size) {
-                in_case = glfs_telldir (glfd);
-                if (in_case == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "telldir failed");
-                        break;
-                }
-
-                if (readdirplus)
-                        ret = glfs_readdirplus_r (glfd, &statbuf, &de,
-                                                  &dirents);
-                else
-                        ret = glfs_readdir_r (glfd, &de, &dirents);
-
-                if (ret == 0 && dirents != NULL) {
-                        if (readdirplus)
-                                this_size = max (sizeof (gf_dirent_t),
-                                                 sizeof (gfs3_dirplist))
-                                        + strlen (de.d_name) + 1;
-                        else
-                                this_size = sizeof (gf_dirent_t)
-                                        + strlen (de.d_name) + 1;
-
-                        if (this_size + filled_size > size) {
-                                glfs_seekdir (glfd, in_case);
-                                break;
-                        }
-
-                        entry = gf_dirent_for_name (de.d_name);
-                        if (!entry) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not create gf_dirent "
-                                        "for entry %s: (%s)",
-                                        entry->d_name,
-                                        strerror (errno));
-                                break;
-                        }
-                        entry->d_off = glfs_telldir (glfd);
-                        entry->d_ino = de.d_ino;
-                        entry->d_type = de.d_type;
-                        if (readdirplus) {
-                                iatt_from_stat (buf, &statbuf);
-                                entry->d_stat = *buf;
-                        }
-                        list_add_tail (&entry->list, &entries->list);
-
-                        filled_size += this_size;
-                        count++;
-                } else if (ret == 0 && dirents == NULL) {
-                        *op_errno = ENOENT;
-                        break;
-                } else if (ret != 0) {
-                        *op_errno = errno;
-                        break;
-                }
-                dirents = NULL;
-                ret = -1;
-        }
+    int filled_size = 0;
+    int this_size = 0;
+    int32_t ret = -1;
+    int32_t count = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *dirents = NULL;
+    struct dirent de = {
+        0,
+    };
+    struct stat statbuf = {
+        0,
+    };
+    off_t in_case = -1;
+
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, glfd, out);
+    GF_VALIDATE_OR_GOTO(this->name, entries, out);
+
+    while (filled_size < size) {
+        in_case = glfs_telldir(glfd);
+        if (in_case == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_TELLDIR_FAILED,
+                   "telldir failed");
+            break;
+        }
+
+        if (readdirplus)
+            ret = glfs_readdirplus_r(glfd, &statbuf, &de, &dirents);
+        else
+            ret = glfs_readdir_r(glfd, &de, &dirents);
+
+        if (ret == 0 && dirents != NULL) {
+            if (readdirplus)
+                this_size = max(sizeof(gf_dirent_t), sizeof(gfs3_dirplist)) +
+                            strlen(de.d_name) + 1;
+            else
+                this_size = sizeof(gf_dirent_t) + strlen(de.d_name) + 1;
+
+            if (this_size + filled_size > size) {
+                glfs_seekdir(glfd, in_case);
+                break;
+            }
+
+            entry = gf_dirent_for_name(de.d_name);
+            if (!entry) {
+                /*
+                 * Since gf_dirent_for_name can return
+                 * NULL only when it fails to allocate
+                 * memory for the directory entry,
+                 * SVS_MSG_NO_MEMORY is used as the
+                 * message-id.
+                 */
+                gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_NO_MEMORY,
+                       "could not create gf_dirent "
+                       "for entry %s: (%s)",
+                       entry->d_name, strerror(errno));
+                break;
+            }
+            entry->d_off = glfs_telldir(glfd);
+            entry->d_ino = de.d_ino;
+            entry->d_type = de.d_type;
+            if (readdirplus) {
+                iatt_from_stat(buf, &statbuf);
+                entry->d_stat = *buf;
+            }
+            list_add_tail(&entry->list, &entries->list);
+
+            filled_size += this_size;
+            count++;
+        } else if (ret == 0 && dirents == NULL) {
+            *op_errno = ENOENT;
+            break;
+        } else if (ret != 0) {
+            *op_errno = errno;
+            break;
+        }
+        dirents = NULL;
+    }
 
 out:
-        return count;
+    return count;
 }
 
 /* readdirp can be of 2 types.
@@ -1360,93 +1562,97 @@ out:
       numbers will be newly generated and filled in.
 */
 void
-svs_readdirp_fill (xlator_t *this, inode_t *parent, svs_inode_t *parent_ctx,
-                   gf_dirent_t *entry)
+svs_readdirp_fill(xlator_t *this, inode_t *parent, svs_inode_t *parent_ctx,
+                  gf_dirent_t *entry)
 {
-        inode_t                *inode          = NULL;
-        uuid_t                  random_gfid    = {0,};
-        struct  iatt            buf            = {0, };
-        svs_inode_t            *inode_ctx      = NULL;
+    inode_t *inode = NULL;
+    uuid_t random_gfid = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent_ctx, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry, out);
+
+    if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+        goto out;
+
+    inode = inode_grep(parent->table, parent, entry->d_name);
+    if (inode) {
+        entry->inode = inode;
+        inode_ctx = svs_inode_ctx_get(this, inode);
+        if (!inode_ctx) {
+            gf_uuid_copy(buf.ia_gfid, inode->gfid);
+            svs_iatt_fill(inode->gfid, &buf);
+            buf.ia_type = inode->ia_type;
+        } else {
+            buf = inode_ctx->buf;
+        }
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent_ctx, out);
-        GF_VALIDATE_OR_GOTO (this->name, entry, out);
+        entry->d_ino = buf.ia_ino;
 
-        if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            entry->d_stat = buf;
+        else {
+            entry->d_stat.ia_ino = buf.ia_ino;
+            gf_uuid_copy(entry->d_stat.ia_gfid, buf.ia_gfid);
+        }
+    } else {
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+            inode = inode_new(parent->table);
+            entry->inode = inode;
+
+            /* If inode context allocation fails, then do not send
+             * the inode for that particular entry as part of
+             * readdirp response. Fuse and protocol/server will link
+             * the inodes in readdirp only if the entry contains
+             * inode in it.
+             */
+            inode_ctx = svs_inode_ctx_get_or_new(this, inode);
+            if (!inode_ctx) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+                       "failed to allocate inode "
+                       "context for %s",
+                       entry->d_name);
+                inode_unref(entry->inode);
+                entry->inode = NULL;
                 goto out;
-
-        inode = inode_grep (parent->table, parent, entry->d_name);
-        if (inode) {
-                entry->inode = inode;
-                inode_ctx = svs_inode_ctx_get (this, inode);
-                if (!inode_ctx) {
-                        gf_uuid_copy (buf.ia_gfid, inode->gfid);
-                        svs_iatt_fill (inode->gfid, &buf);
-                        buf.ia_type = inode->ia_type;
-                } else {
-                        buf = inode_ctx->buf;
-                }
-
-                entry->d_ino = buf.ia_ino;
-
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        entry->d_stat = buf;
-                else {
-                        entry->d_stat.ia_ino = buf.ia_ino;
-                        gf_uuid_copy (entry->d_stat.ia_gfid, buf.ia_gfid);
-                }
+            }
+
+            /* Generate virtual gfid for SNAPSHOT dir and
+             * update the statbuf
+             */
+            gf_uuid_generate(random_gfid);
+            gf_uuid_copy(buf.ia_gfid, random_gfid);
+            svs_fill_ino_from_gfid(&buf);
+            buf.ia_type = IA_IFDIR;
+            entry->d_ino = buf.ia_ino;
+            entry->d_stat = buf;
+            inode_ctx->buf = buf;
+            inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
         } else {
-
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                        inode = inode_new (parent->table);
-                        entry->inode = inode;
-
-                        /* If inode context allocation fails, then do not send
-                         * the inode for that particular entry as part of
-                         * readdirp response. Fuse and protocol/server will link
-                         * the inodes in readdirp only if the entry contains
-                         * inode in it.
-                         */
-                        inode_ctx = svs_inode_ctx_get_or_new (this, inode);
-                        if (!inode_ctx) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "allocate inode context for %s",
-                                        entry->d_name);
-                                inode_unref (entry->inode);
-                                entry->inode = NULL;
-                                goto out;
-                        }
-
-                        /* Generate virtual gfid for SNAPSHOT dir and
-                         * update the statbuf
-                         */
-                        gf_uuid_generate (random_gfid);
-                        gf_uuid_copy (buf.ia_gfid, random_gfid);
-                        svs_fill_ino_from_gfid (&buf);
-                        buf.ia_type = IA_IFDIR;
-                        entry->d_ino = buf.ia_ino;
-                        entry->d_stat = buf;
-                        inode_ctx->buf = buf;
-                        inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
-                } else {
-                        /* For files under snapshot world do not set
-                         * entry->inode and reset statbuf (except ia_ino),
-                         * so that FUSE/Kernel will send an explicit lookup.
-                         * entry->d_stat contains the statbuf information
-                         * of original file, so for NFS not to cache this
-                         * information and to send explicit lookup, it is
-                         * required to reset the statbuf.
-                         * Virtual gfid for these files will be generated in the
-                         * first lookup.
-                         */
-                        buf.ia_ino = entry->d_ino;
-                        entry->d_stat = buf;
-                }
-        }
+            /* For files under snapshot world do not set
+             * entry->inode and reset statbuf (except ia_ino),
+             * so that FUSE/Kernel will send an explicit lookup.
+             * entry->d_stat contains the statbuf information
+             * of original file, so for NFS not to cache this
+             * information and to send explicit lookup, it is
+             * required to reset the statbuf.
+             * Virtual gfid for these files will be generated in the
+             * first lookup.
+             */
+            buf.ia_ino = entry->d_ino;
+            entry->d_stat = buf;
+        }
+    }
 
 out:
-        return;
+    return;
 }
 
 /* In readdirp, though new inode is created along with the generation of
@@ -1458,151 +1664,176 @@ out:
    and is filled in when lookup comes on that object.
 */
 int32_t
-svs_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t off, dict_t *dict)
+svs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *dict)
 {
-        gf_dirent_t             entries;
-        gf_dirent_t            *entry                           = NULL;
-        struct  iatt            buf                             = {0, };
-        int                     count                           = 0;
-        int                     op_ret                          = -1;
-        int                     op_errno                        = EINVAL;
-        svs_inode_t            *parent_ctx                      = NULL;
-        svs_fd_t               *svs_fd                          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        INIT_LIST_HEAD (&entries.list);
-
-        parent_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!parent_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    struct iatt buf = {
+        0,
+    };
+    int count = 0;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    svs_inode_t *parent_ctx = NULL;
+    svs_fd_t *svs_fd = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    INIT_LIST_HEAD(&entries.list);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    parent_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!parent_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get the inode "
+               "context for %s",
+               uuid_utoa(fd->inode->gfid));
+        goto unwind;
+    }
+
+    if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        LOCK(&fd->lock);
+        {
+            count = svs_fill_readdir(this, &entries, size, off);
         }
+        UNLOCK(&fd->lock);
 
-        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                LOCK (&fd->lock);
-                {
-                        count = svs_fill_readdir (this, &entries, size, off);
-                }
-                UNLOCK (&fd->lock);
-
-                op_ret = count;
-
-                list_for_each_entry (entry, &entries.list, list) {
-                        svs_readdirp_fill (this, fd->inode, parent_ctx, entry);
-                }
+        op_ret = count;
 
-                goto unwind;
-        } else {
-                svs_fd = svs_fd_ctx_get_or_new (this, fd);
-                if (!svs_fd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "fd context %s", uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            svs_readdirp_fill(this, fd->inode, parent_ctx, entry);
+        }
 
-                glfs_seekdir (svs_fd->fd, off);
+        goto unwind;
+    } else {
+        svs_fd = svs_fd_ctx_get_or_new(this, fd);
+        if (!svs_fd) {
+            op_ret = -1;
+            op_errno = EBADFD;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_GET_FD_CONTEXT_FAILED,
+                   "failed to get the fd context "
+                   "for the inode %s",
+                   uuid_utoa(fd->inode->gfid));
+            goto unwind;
+        }
 
-                LOCK (&fd->lock);
-                {
-                        count = svs_glfs_readdir (this, svs_fd->fd, &entries,
-                                                  &op_errno, &buf, _gf_true,
-                                                  size);
-                }
-                UNLOCK (&fd->lock);
+        glfs_seekdir(svs_fd->fd, off);
 
-                op_ret = count;
+        LOCK(&fd->lock);
+        {
+            count = svs_glfs_readdir(this, svs_fd->fd, &entries, &op_errno,
+                                     &buf, _gf_true, size);
+        }
+        UNLOCK(&fd->lock);
 
-                list_for_each_entry (entry, &entries.list, list) {
-                        svs_readdirp_fill (this, fd->inode, parent_ctx, entry);
-                }
+        op_ret = count;
 
-                goto unwind;
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            svs_readdirp_fill(this, fd->inode, parent_ctx, entry);
         }
 
+        goto unwind;
+    }
+
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, &entries, dict);
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, &entries, dict);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off, dict_t *xdata)
+svs_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
 {
-        svs_private_t *priv      = NULL;
-        gf_dirent_t    entries   = {{{0, }, }, };
-        int            count     = 0;
-        svs_inode_t   *inode_ctx = NULL;
-        int            op_errno  = EINVAL;
-        int            op_ret    = -1;
-        svs_fd_t      *svs_fd    = NULL;
-        glfs_fd_t     *glfd      = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        GF_VALIDATE_OR_GOTO ("snap-view-server", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        priv = this->private;
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found in "
-                        "the inode %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
+    gf_dirent_t entries = {
+        {
+            {
+                0,
+            },
+        },
+    };
+    int count = 0;
+    svs_inode_t *inode_ctx = NULL;
+    int op_errno = EINVAL;
+    int op_ret = -1;
+    svs_fd_t *svs_fd = NULL;
+    glfs_fd_t *glfd = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    GF_VALIDATE_OR_GOTO("snap-view-server", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found in "
+               "the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto unwind;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        LOCK(&fd->lock);
+        {
+            count = svs_fill_readdir(this, &entries, size, off);
+        }
+        UNLOCK(&fd->lock);
+    } else {
+        svs_fd = svs_fd_ctx_get_or_new(this, fd);
+        if (!svs_fd) {
+            op_ret = -1;
+            op_errno = EBADFD;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_GET_FD_CONTEXT_FAILED,
+                   "failed to get the fd "
+                   "context for %s",
+                   uuid_utoa(fd->inode->gfid));
+            goto unwind;
         }
 
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                LOCK (&fd->lock);
-                {
-                        count = svs_fill_readdir (this, &entries, size, off);
-                }
-                UNLOCK (&fd->lock);
-        } else {
-                svs_fd = svs_fd_ctx_get_or_new (this, fd);
-                if (!svs_fd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "fd context %s", uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-
-                glfd = svs_fd->fd;
+        glfd = svs_fd->fd;
 
-                LOCK (&fd->lock);
-                {
-                        count = svs_glfs_readdir (this, glfd, &entries,
-                                                  &op_errno, NULL, _gf_false,
-                                                  size);
-                }
-                UNLOCK (&fd->lock);
+        LOCK(&fd->lock);
+        {
+            count = svs_glfs_readdir(this, glfd, &entries, &op_errno, NULL,
+                                     _gf_false, size);
         }
+        UNLOCK(&fd->lock);
+    }
 
-        op_ret = count;
+    op_ret = count;
 
 unwind:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, &entries, xdata);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -1640,11 +1871,11 @@ unwind:
  * the gfid it got from NFS client, for which it was not able to find the right
  * inode. So snapview-server was able to get the fs instance (glfs_t) of the
  * snapshot volume to which the entry belongs to, and the handle for the entry
- * from the corresponding snapshot volume and fill those informations in the
+ * from the corresponding snapshot volume and fill those information in the
  * inode context.
  *
  * But now, since NFS server is able to find the inode from the inode table for
- * the gfid it got from the NFS client, it wont send lookup. Rather it directly
+ * the gfid it got from the NFS client, it won't send lookup. Rather it directly
  * sends the fop it received from the client. Now this causes problems for
  * snapview-server. Because for each fop snapview-server assumes that lookup has
  * been performed on that entry and the entry's inode context contains the
@@ -1658,693 +1889,832 @@ unwind:
  */
 
 int32_t
-svs_get_handle (xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
-                int32_t *op_errno)
+svs_get_handle(xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
+               int32_t *op_errno)
 {
-        svs_inode_t   *parent_ctx   = NULL;
-        int            ret          = -1;
-        inode_t       *parent       = NULL;
-        struct iatt    postparent   = {0, };
-        struct iatt    buf          = {0, };
-        char           uuid1[64];
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        if (loc->path) {
-                if (!loc->name || (loc->name && !strcmp (loc->name, ""))) {
-                        loc->name = strrchr (loc->path, '/');
-                        if (loc->name)
-                                loc->name++;
-                }
-        }
-
-        if (loc->parent)
-                parent = inode_ref (loc->parent);
-        else {
-                parent = inode_find (loc->inode->table, loc->pargfid);
-                if (!parent)
-                        parent = inode_parent (loc->inode, NULL, NULL);
-        }
-
-        if (parent)
-                parent_ctx = svs_inode_ctx_get (this, parent);
-
-        if (!parent_ctx) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to get the parent "
-                        "context for %s (%s)", loc->path,
-                        uuid_utoa_r (loc->inode->gfid, uuid1));
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        if (parent_ctx) {
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        ret = svs_lookup_snapshot (this, loc, &buf,
-                                                   &postparent, parent,
-                                                   parent_ctx, op_errno);
-                else
-                        ret = svs_lookup_entry (this, loc, &buf,
-                                                &postparent, parent,
-                                                parent_ctx, op_errno);
-        }
+    svs_inode_t *parent_ctx = NULL;
+    int ret = -1;
+    inode_t *parent = NULL;
+    struct iatt postparent = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    char uuid1[64];
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    if (loc->path) {
+        if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+            loc->name = strrchr(loc->path, '/');
+            if (loc->name)
+                loc->name++;
+        }
+    }
+
+    if (loc->parent)
+        parent = inode_ref(loc->parent);
+    else {
+        parent = inode_find(loc->inode->table, loc->pargfid);
+        if (!parent)
+            parent = inode_parent(loc->inode, NULL, NULL);
+    }
+
+    if (parent)
+        parent_ctx = svs_inode_ctx_get(this, parent);
+
+    if (!parent_ctx) {
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, *op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get the parent "
+               "context for %s (%s)",
+               loc->path, uuid_utoa_r(loc->inode->gfid, uuid1));
+        goto out;
+    }
+
+    if (parent_ctx) {
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            ret = svs_lookup_snapshot(this, loc, &buf, &postparent, parent,
+                                      parent_ctx, op_errno);
+        else
+            ret = svs_lookup_entry(this, loc, &buf, &postparent, parent,
+                                   parent_ctx, op_errno);
+    }
 
 out:
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-svs_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+svs_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        svs_private_t *priv         = NULL;
-        struct iatt    buf          = {0, };
-        int32_t        op_errno     = EINVAL;
-        int32_t        op_ret       = -1;
-        svs_inode_t   *inode_ctx    = NULL;
-        glfs_t        *fs           = NULL;
-        glfs_object_t *object       = NULL;
-        struct stat    stat         = {0, };
-        int            ret          = -1;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        /* Instead of doing the check of whether it is a entry point directory
-           or not by checking the name of the entry and then deciding what
-           to do, just check the inode context and decide what to be done.
-        */
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                svs_iatt_fill (loc->inode->gfid, &buf);
-                op_ret = 0;
-        }
-        else {
-
-                SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                                       op_errno, out);
-
-                ret = glfs_h_stat (fs, object, &stat);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "glfs_h_stat on %s "
-                                "(gfid: %s) failed", loc->name,
-                                uuid_utoa (loc->inode->gfid));
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
+    struct iatt buf = {
+        0,
+    };
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat stat = {
+        0,
+    };
+    int ret = -1;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    /* Instead of doing the check of whether it is a entry point directory
+       or not by checking the name of the entry and then deciding what
+       to do, just check the inode context and decide what to be done.
+    */
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for %s", uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        svs_iatt_fill(loc->inode->gfid, &buf);
+        op_ret = 0;
+    } else {
+        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
+                               op_errno, out);
 
-                iatt_from_stat (&buf, &stat);
-                gf_uuid_copy (buf.ia_gfid, loc->inode->gfid);
-                svs_fill_ino_from_gfid (&buf);
-                op_ret = ret;
-        }
+        ret = glfs_h_stat(fs, object, &stat);
+        if (ret) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STAT_FAILED,
+                   "glfs_h_stat on %s (gfid: %s) "
+                   "failed",
+                   loc->name, uuid_utoa(loc->inode->gfid));
+            goto out;
+        } else
+            gf_msg_debug(this->name, 0, "stat on %s (%s) successful", loc->path,
+                         uuid_utoa(loc->inode->gfid));
+
+        iatt_from_stat(&buf, &stat);
+        gf_uuid_copy(buf.ia_gfid, loc->inode->gfid);
+        svs_fill_ino_from_gfid(&buf);
+        op_ret = ret;
+    }
 
 out:
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, &buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, &buf, xdata);
+    return 0;
 }
 
 int32_t
-svs_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+svs_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        svs_private_t *priv        = NULL;
-        struct iatt    buf         = {0, };
-        int32_t        op_errno    = EINVAL;
-        int32_t        op_ret      = -1;
-        svs_inode_t   *inode_ctx   = NULL;
-        struct stat    stat        = {0, };
-        int            ret         = -1;
-        glfs_fd_t     *glfd        = NULL;
-        svs_fd_t      *sfd         = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        priv = this->private;
-
-        /* Instead of doing the check of whether it is a entry point directory
-           or not by checking the name of the entry and then deciding what
-           to do, just check the inode context and decide what to be done.
-        */
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " the inode %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
+    struct iatt buf = {
+        0,
+    };
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    svs_inode_t *inode_ctx = NULL;
+    struct stat stat = {
+        0,
+    };
+    int ret = -1;
+    glfs_fd_t *glfd = NULL;
+    svs_fd_t *sfd = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    /* Instead of doing the check of whether it is a entry point directory
+       or not by checking the name of the entry and then deciding what
+       to do, just check the inode context and decide what to be done.
+    */
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for"
+               " the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        svs_iatt_fill(fd->inode->gfid, &buf);
+        op_ret = 0;
+    } else {
+        if (!(svs_inode_ctx_glfs_mapping(this, inode_ctx))) {
+            op_ret = -1;
+            op_errno = EBADF;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_FS_INSTANCE_INVALID,
+                   "glfs instance %p to which the inode %s "
+                   "belongs to does not exist. That snapshot "
+                   "corresponding to the fs instance "
+                   "might have been deleted or deactivated.",
+                   inode_ctx->fs, uuid_utoa(fd->inode->gfid));
+            goto out;
+        }
+
+        sfd = svs_fd_ctx_get_or_new(this, fd);
+        if (!sfd) {
+            op_ret = -1;
+            op_errno = EBADFD;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_GET_FD_CONTEXT_FAILED,
+                   "failed to get the fd context "
+                   "for %s",
+                   uuid_utoa(fd->inode->gfid));
+            goto out;
         }
 
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                svs_iatt_fill (fd->inode->gfid, &buf);
-                op_ret = 0;
+        glfd = sfd->fd;
+        ret = glfs_fstat(glfd, &stat);
+        if (ret) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STAT_FAILED,
+                   "glfs_fstat on gfid: %s failed", uuid_utoa(fd->inode->gfid));
+            goto out;
         }
-        else {
-                sfd = svs_fd_ctx_get_or_new (this, fd);
-                if (!sfd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "fd context for %s",
-                                uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = EBADFD;
-                        goto out;
-                }
 
-                glfd = sfd->fd;
-                ret = glfs_fstat (glfd, &stat);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "glfs_fstat on "
-                                "gfid: %s failed", uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
-
-                iatt_from_stat (&buf, &stat);
-                gf_uuid_copy (buf.ia_gfid, fd->inode->gfid);
-                svs_fill_ino_from_gfid (&buf);
-                op_ret = ret;
-        }
+        iatt_from_stat(&buf, &stat);
+        gf_uuid_copy(buf.ia_gfid, fd->inode->gfid);
+        svs_fill_ino_from_gfid(&buf);
+        op_ret = ret;
+    }
 
 out:
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, &buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, &buf, xdata);
+    return 0;
 }
 
 int32_t
-svs_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+svs_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        svs_private_t *priv         = NULL;
-        struct statvfs buf          = {0, };
-        int32_t        op_errno     = EINVAL;
-        int32_t        op_ret       = -1;
-        svs_inode_t   *inode_ctx    = NULL;
-        glfs_t        *fs           = NULL;
-        glfs_object_t *object       = NULL;
-        int            ret          = -1;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        /* Instead of doing the check of whether it is a entry point directory
-           or not by checking the name of the entry and then deciding what
-           to do, just check the inode context and decide what to be done.
-        */
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                               op_errno, out);
-
-        ret = glfs_h_statfs (fs, object, &buf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "glfs_h_statvfs on %s "
-                        "(gfid: %s) failed", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = errno;
-                goto out;
-        }
-        op_ret = ret;
+    struct statvfs buf = {
+        0,
+    };
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    int ret = -1;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    /* Instead of doing the check of whether it is a entry point directory
+       or not by checking the name of the entry and then deciding what
+       to do, just check the inode context and decide what to be done.
+    */
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for %s", uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    ret = glfs_h_statfs(fs, object, &buf);
+    if (ret) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STATFS_FAILED,
+               "glfs_h_statvfs on %s (gfid: %s) "
+               "failed",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+    op_ret = ret;
 
 out:
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, &buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, &buf, xdata);
+    return 0;
 }
 
-
 int32_t
-svs_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+svs_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
 {
-        svs_inode_t   *inode_ctx = NULL;
-        svs_fd_t      *sfd       = NULL;
-        int32_t        op_ret    = -1;
-        int32_t        op_errno  = EINVAL;
-        glfs_fd_t     *glfd      = NULL;
-        glfs_t        *fs        = NULL;
-        glfs_object_t *object    = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context for %s "
-                        "(gfid: %s) not found", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                goto out;
-        }
-
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                GF_ASSERT (0); // on entry point it should always be opendir
-
-        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                               op_errno, out);
-
-        glfd = glfs_h_open (fs, object, flags);
-        if (!glfd) {
-                gf_log (this->name, GF_LOG_ERROR, "glfs_h_open on %s failed "
-                        "(gfid: %s)", loc->name, uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = errno;
-                goto out;
-        }
-
-        sfd = svs_fd_ctx_get_or_new (this, fd);
-        if (!sfd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate fd "
-                        "context for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                glfs_close (glfd);
-                goto out;
-        }
-        sfd->fd = glfd;
-
-        op_ret = 0;
+    svs_inode_t *inode_ctx = NULL;
+    svs_fd_t *sfd = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glfs_fd_t *glfd = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context for %s (gfid: %s) "
+               "not found",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+        GF_ASSERT(0);  // on entry point it should always be opendir
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    glfd = glfs_h_open(fs, object, flags);
+    if (!glfd) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_OPEN_FAILED,
+               "glfs_h_open on %s failed (gfid: %s)", loc->name,
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    sfd = svs_fd_ctx_get_or_new(this, fd);
+    if (!sfd) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+               "failed to allocate fd context "
+               "for %s (gfid: %s)",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        glfs_close(glfd);
+        goto out;
+    }
+    sfd->fd = glfd;
+
+    op_ret = 0;
 
 out:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, NULL);
+    return 0;
 }
 
 int32_t
-svs_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+svs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int32_t                op_ret     = -1;
-        int32_t                op_errno   = 0;
-        svs_private_t         *priv       = NULL;
-        struct iobuf          *iobuf      = NULL;
-        struct iobref         *iobref     = NULL;
-        struct iovec           vec        = {0,};
-        svs_fd_t              *sfd        = NULL;
-        int                    ret        = -1;
-        struct stat            fstatbuf   = {0, };
-        glfs_fd_t             *glfd       = NULL;
-        struct iatt            stbuf      = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        priv = this->private;
-        VALIDATE_OR_GOTO (priv, out);
-
-        sfd = svs_fd_ctx_get_or_new (this, fd);
-        if (!sfd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EBADFD;
-                goto out;
-        }
-
-        glfd = sfd->fd;
-
-        iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
-        if (!iobuf) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        ret = glfs_pread (glfd, iobuf->ptr, size, offset, 0);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, "glfs_read failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        vec.iov_base = iobuf->ptr;
-        vec.iov_len  = ret;
-
-        iobref = iobref_new ();
-
-        iobref_add (iobref, iobuf);
-
-        ret = glfs_fstat (glfd, &fstatbuf);
-        if (ret) {
-                op_ret = -1;
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, "glfs_fstat failed after "
-                        "readv on %s", uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        iatt_from_stat (&stbuf, &fstatbuf);
-        gf_uuid_copy (stbuf.ia_gfid, fd->inode->gfid);
-        svs_fill_ino_from_gfid (&stbuf);
-
-        /* Hack to notify higher layers of EOF. */
-        if (!stbuf.ia_size || (offset + vec.iov_len) >= stbuf.ia_size)
-                op_errno = ENOENT;
-
-        op_ret = vec.iov_len;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    svs_private_t *priv = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec vec = {
+        0,
+    };
+    svs_fd_t *sfd = NULL;
+    int ret = -1;
+    struct glfs_stat fstatbuf = {
+        0,
+    };
+    glfs_fd_t *glfd = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    priv = this->private;
+    VALIDATE_OR_GOTO(priv, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    if (!svs_inode_glfs_mapping(this, fd->inode)) {
+        op_ret = -1;
+        op_errno = EBADF; /* should this be some other error? */
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_FS_INSTANCE_INVALID,
+               "glfs instance to which the inode "
+               "%s receiving read request belongs, "
+               "does not exist anymore",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    sfd = svs_fd_ctx_get_or_new(this, fd);
+    if (!sfd) {
+        op_ret = -1;
+        op_errno = EBADFD;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get the fd "
+               "context for %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    glfd = sfd->fd;
+
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+               "failed to "
+               "allocate iobuf while reading the "
+               "file with gfid %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    ret = glfs_pread(glfd, iobuf->ptr, size, offset, 0, &fstatbuf);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_READ_FAILED,
+               "glfs_read failed on %s (%s)", uuid_utoa(fd->inode->gfid),
+               strerror(op_errno));
+        goto out;
+    }
+
+    vec.iov_base = iobuf->ptr;
+    vec.iov_len = ret;
+
+    iobref = iobref_new();
+
+    iobref_add(iobref, iobuf);
+    glfs_iatt_from_statx(&stbuf, &fstatbuf);
+    gf_uuid_copy(stbuf.ia_gfid, fd->inode->gfid);
+    svs_fill_ino_from_gfid(&stbuf);
+
+    /* Hack to notify higher layers of EOF. */
+    if (!stbuf.ia_size || (offset + vec.iov_len) >= stbuf.ia_size)
+        op_errno = ENOENT;
+
+    op_ret = vec.iov_len;
 
 out:
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             &vec, 1, &stbuf, iobref, NULL);
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &vec, 1, &stbuf, iobref,
+                        NULL);
 
-        if (iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size, dict_t *xdata)
+svs_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        svs_inode_t     *inode_ctx = NULL;
-        glfs_t          *fs        = NULL;
-        glfs_object_t   *object    = NULL;
-        int              op_ret    = -1;
-        int              op_errno  = EINVAL;
-        char            *buf       = NULL;
-        struct iatt      stbuf     = {0, };
-        int              ret       = -1;
-        struct stat      stat      = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                               op_errno, out);
-
-        ret = glfs_h_stat (fs, object, &stat);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "glfs_h_stat on %s "
-                        "(gfid: %s) failed", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = errno;
-                goto out;
-        }
-
-        iatt_from_stat (&stbuf, &stat);
-        gf_uuid_copy (stbuf.ia_gfid, loc->inode->gfid);
-        svs_fill_ino_from_gfid (&stbuf);
-
-        buf = alloca (size + 1);
-        op_ret = glfs_h_readlink (fs, object, buf, size);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "readlink on %s failed "
-                        "(gfid: %s)", loc->name, uuid_utoa (loc->inode->gfid));
-                op_errno = errno;
-                goto out;
-        }
-
-        buf[op_ret] = 0;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    char *buf = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+    struct stat stat = {
+        0,
+    };
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get inode context "
+               "for %s (gfid: %s)",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    ret = glfs_h_stat(fs, object, &stat);
+    if (ret) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STAT_FAILED,
+               "glfs_h_stat on %s (gfid: %s) "
+               "failed",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    iatt_from_stat(&stbuf, &stat);
+    gf_uuid_copy(stbuf.ia_gfid, loc->inode->gfid);
+    svs_fill_ino_from_gfid(&stbuf);
+
+    buf = alloca(size + 1);
+    op_ret = glfs_h_readlink(fs, object, buf, size);
+    if (op_ret == -1) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_READLINK_FAILED,
+               "readlink on %s failed (gfid: %s)", loc->name,
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    buf[op_ret] = 0;
 
 out:
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, &stbuf,
-                             NULL);
+    STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, buf, &stbuf, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
-            dict_t *xdata)
+svs_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
+           dict_t *xdata)
 {
-        int             ret          = -1;
-        int32_t         op_ret       = -1;
-        int32_t         op_errno     = EINVAL;
-        svs_private_t  *priv         = NULL;
-        glfs_t         *fs           = NULL;
-        glfs_object_t  *object       = NULL;
-        svs_inode_t    *inode_ctx    = NULL;
-        gf_boolean_t    is_fuse_call = 0;
-        int             mode         = 0;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        is_fuse_call = __is_fuse_call (frame);
-
-        /*
-         * For entry-point directory, set read and execute bits. But not write
-         * permissions.
-         */
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                if (is_fuse_call) {
-                        op_ret = 0;
-                        op_errno = 0;
-                } else {
-                        op_ret = 0;
-                        mode |= POSIX_ACL_READ;
-                        mode |= POSIX_ACL_EXECUTE;
-                        op_errno = mode;
-                }
-                goto out;
-        }
-
-
-        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                               op_errno, out);
-
-        /* The actual posix_acl xlator does acl checks differently for
-           fuse and nfs. So set frame->root->pid as fspid of the syncop
-           if the call came from nfs
-        */
-        if (!is_fuse_call) {
-                syncopctx_setfspid (&frame->root->pid);
-                syncopctx_setfsuid (&frame->root->uid);
-                syncopctx_setfsgid (&frame->root->gid);
-                syncopctx_setfsgroups (frame->root->ngrps,
-                                       frame->root->groups);
-        }
-
-        ret = glfs_h_access (fs, object, mask);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to access %s "
-                        "(gfid: %s)", loc->path, uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = errno;
-                goto out;
-        }
-
-        op_ret = 0;
-        op_errno = ret;
+    int ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    svs_inode_t *inode_ctx = NULL;
+    gf_boolean_t is_fuse_call = 0;
+    int mode = 0;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for %s", uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    is_fuse_call = __is_fuse_call(frame);
+
+    /*
+     * For entry-point directory, set read and execute bits. But not write
+     * permissions.
+     */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        if (is_fuse_call) {
+            op_ret = 0;
+            op_errno = 0;
+        } else {
+            op_ret = 0;
+            mode |= POSIX_ACL_READ;
+            mode |= POSIX_ACL_EXECUTE;
+            op_errno = mode;
+        }
+        goto out;
+    }
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    /* The actual posix_acl xlator does acl checks differently for
+       fuse and nfs. So set frame->root->pid as fspid of the syncop
+       if the call came from nfs
+    */
+    if (!is_fuse_call) {
+        syncopctx_setfspid(&frame->root->pid);
+        syncopctx_setfsuid(&frame->root->uid);
+        syncopctx_setfsgid(&frame->root->gid);
+        syncopctx_setfsgroups(frame->root->ngrps, frame->root->groups);
+    }
+
+    ret = glfs_h_access(fs, object, mask);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_ACCESS_FAILED,
+               "failed to access %s (gfid: %s)", loc->path,
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    op_ret = 0;
+    op_errno = ret;
 
 out:
 
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(access, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_svs_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
-
-        return ret;
+    switch (event) {
+        case GF_EVENT_PARENT_UP: {
+            /* Tell the parent that snapview-server xlator is up */
+            default_notify(this, GF_EVENT_CHILD_UP, data);
+        } break;
+        default:
+            break;
+    }
+    return 0;
 }
 
 int32_t
-init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        svs_private_t   *priv           = NULL;
-        int             ret             = -1;
-        pthread_t       snap_thread;
-
-        /* This can be the top of graph in certain cases */
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_svs_mt_priv_t);
-        if (!priv)
-                goto out;
-
-        this->private = priv;
+    int ret = -1;
 
-        GF_OPTION_INIT ("volname", priv->volname, str, out);
-        LOCK_INIT (&priv->snaplist_lock);
+    if (!this)
+        return ret;
 
-        LOCK (&priv->snaplist_lock);
-        {
-                priv->num_snaps = 0;
-        }
-        UNLOCK (&priv->snaplist_lock);
+    ret = xlator_mem_acct_init(this, gf_svs_mt_end + 1);
 
-        /* What to do here upon failure? should init be failed or succeed? */
-        /* If succeeded, then dynamic management of snapshots will not */
-        /* happen.*/
-        ret = svs_mgmt_init (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to initiate the "
-                        "mgmt rpc callback for svs. Dymamic management of the"
-                        "snapshots will not happen");
-                goto out;
-        }
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_MEM_ACNT_FAILED,
+               "Memory accounting"
+               " init failed");
+        return ret;
+    }
 
-        /* get the list of snaps first to return to client xlator */
-        ret = svs_get_snapshot_list (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error initializing snaplist infrastructure");
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        ret = 0;
+int32_t
+init(xlator_t *this)
+{
+    svs_private_t *priv = NULL;
+    int ret = -1;
+
+    /* This can be the top of graph in certain cases */
+    if (!this->parents) {
+        gf_msg_debug(this->name, 0, "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_svs_mt_priv_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+               "failed to "
+               "allocate memory for this->private ");
+        goto out;
+    }
+
+    this->private = priv;
+
+    GF_OPTION_INIT("volname", priv->volname, str, out);
+    LOCK_INIT(&priv->snaplist_lock);
+
+    LOCK(&priv->snaplist_lock);
+    {
+        priv->num_snaps = 0;
+    }
+    UNLOCK(&priv->snaplist_lock);
+
+    /* What to do here upon failure? should init be failed or succeed? */
+    /* If succeeded, then dynamic management of snapshots will not */
+    /* happen.*/
+    ret = svs_mgmt_init(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, SVS_MSG_MGMT_INIT_FAILED,
+               "failed to initiate the "
+               "mgmt rpc callback for svs. Dymamic management of the"
+               "snapshots will not happen");
+        goto out;
+    }
+
+    /* get the list of snaps first to return to client xlator */
+    ret = svs_get_snapshot_list(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               SVS_MSG_GET_SNAPSHOT_LIST_FAILED,
+               "Error initializing snaplist infrastructure");
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        if (ret && priv) {
-                LOCK_DESTROY (&priv->snaplist_lock);
-                GF_FREE (priv->dirents);
-                GF_FREE (priv);
-        }
+    if (ret && priv) {
+        LOCK_DESTROY(&priv->snaplist_lock);
+        GF_FREE(priv->dirents);
+        GF_FREE(priv);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        svs_private_t   *priv   = NULL;
-        glusterfs_ctx_t *ctx    = NULL;
-        int             ret     = 0;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        this->private = NULL;
-        ctx = this->ctx;
-        if (!ctx)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid ctx found");
-
-        if (priv) {
-                ret = LOCK_DESTROY (&priv->snaplist_lock);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Could not destroy mutex snaplist_lock");
-                }
-
-                if (priv->dirents) {
-                        GF_FREE (priv->dirents);
-                }
+    svs_private_t *priv = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    this->private = NULL;
+    ctx = this->ctx;
+    if (!ctx)
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_INVALID_GLFS_CTX,
+               "Invalid ctx found");
+
+    if (priv) {
+        ret = LOCK_DESTROY(&priv->snaplist_lock);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_WARNING, errno,
+                   SVS_MSG_LOCK_DESTROY_FAILED,
+                   "Could not destroy mutex snaplist_lock");
+        }
 
-                if (priv->rpc) {
-                        /* cleanup the saved-frames before last unref */
-                        rpc_clnt_connection_cleanup (&priv->rpc->conn);
-                        rpc_clnt_unref (priv->rpc);
-                }
+        if (priv->dirents) {
+            GF_FREE(priv->dirents);
+        }
 
-                GF_FREE (priv);
+        if (priv->rpc) {
+            /* cleanup the saved-frames before last unref */
+            rpc_clnt_connection_cleanup(&priv->rpc->conn);
+            rpc_clnt_unref(priv->rpc);
         }
 
-        return;
+        GF_FREE(priv);
+    }
+
+    return;
 }
 
 struct xlator_fops fops = {
-        .lookup     = svs_lookup,
-        .stat       = svs_stat,
-        .statfs     = svs_statfs,
-        .opendir    = svs_opendir,
-        .readdirp   = svs_readdirp,
-        .readdir    = svs_readdir,
-        .open       = svs_open,
-        .readv      = svs_readv,
-        .flush      = svs_flush,
-        .fstat      = svs_fstat,
-        .getxattr   = svs_getxattr,
-        .access     = svs_access,
-        .readlink   = svs_readlink,
-        /* entry fops */
+    .lookup = svs_lookup,
+    .stat = svs_stat,
+    .statfs = svs_statfs,
+    .opendir = svs_opendir,
+    .readdirp = svs_readdirp,
+    .readdir = svs_readdir,
+    .open = svs_open,
+    .readv = svs_readv,
+    .flush = svs_flush,
+    .fstat = svs_fstat,
+    .getxattr = svs_getxattr,
+    .access = svs_access,
+    .readlink = svs_readlink,
+    /* entry fops */
 };
 
 struct xlator_cbks cbks = {
-        .release  = svs_release,
-        .releasedir = svs_releasedir,
-        .forget     = svs_forget,
+    .release = svs_release,
+    .releasedir = svs_releasedir,
+    .forget = svs_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"volname"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-        { .key  = {NULL} },
+    {
+        .key = {"volname"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "snapview-server",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/snapview-server/src/snapview-server.h b/xlators/features/snapview-server/src/snapview-server.h
index a12319fa9b2..6472422e715 100644
--- a/xlators/features/snapview-server/src/snapview-server.h
+++ b/xlators/features/snapview-server/src/snapview-server.h
@@ -10,231 +10,246 @@
 #ifndef __SNAP_VIEW_H__
 #define __SNAP_VIEW_H__
 
-#include "dict.h"
-#include "defaults.h"
-#include "mem-types.h"
-#include "call-stub.h"
-#include "inode.h"
-#include "byte-order.h"
-#include "iatt.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/mem-types.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/iatt.h>
 #include <ctype.h>
 #include <sys/uio.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
 #include "glfs.h"
-#include "common-utils.h"
 #include "glfs-handles.h"
 #include "glfs-internal.h"
 #include "glusterfs3-xdr.h"
-#include "glusterfs-acl.h"
-#include "syncop.h"
-#include "list.h"
-#include "timer.h"
+#include <glusterfs/glusterfs-acl.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/list.h>
+#include <glusterfs/timer.h>
 #include "rpc-clnt.h"
 #include "protocol-common.h"
 #include "xdr-generic.h"
-
+#include "snapview-server-messages.h"
 
 #define DEFAULT_SVD_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
 
-#define SNAP_VIEW_MAX_GLFS_T            256
-#define SNAP_VIEW_MAX_GLFS_FDS          1024
-#define SNAP_VIEW_MAX_GLFS_OBJ_HANDLES  1024
-
-#define SVS_STACK_DESTROY(_frame)                                   \
-        do {                                                        \
-                ((call_frame_t *)_frame)->local = NULL;             \
-                STACK_DESTROY (((call_frame_t *)_frame)->root);     \
-        } while (0)
-
-#define SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this)                       \
-        do {                                                            \
-                svs_private_t *_private = NULL;                         \
-                _private = this->private;                               \
-                int  i = 0;                                             \
-                gf_boolean_t found = _gf_false;                         \
-                LOCK (&_private->snaplist_lock);                        \
-                {                                                       \
-                        for (i = 0; i < _private->num_snaps; i++) {     \
-                                if (_private->dirents->fs && fs &&      \
-                                    _private->dirents->fs == fs) {      \
-                                        found = _gf_true;               \
-                                        break;                          \
-                                }                                       \
-                        }                                               \
-                }                                                       \
-                UNLOCK (&_private->snaplist_lock);                      \
-                                                                        \
-                if (!found)                                             \
-                        fs = NULL;                                      \
-        } while (0)
-
-#define SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, ret,   \
-                               op_errno, label)                         \
-        do {                                                            \
-                fs = inode_ctx->fs;                                     \
-                object = inode_ctx->object;                             \
-                SVS_CHECK_VALID_SNAPSHOT_HANDLE (fs, this);             \
-                if (!fs)                                                \
-                        object = NULL;                                  \
-                                                                        \
-                if (!fs || !object) {                                   \
-                        int32_t tmp = -1;                               \
-                        char    tmp_uuid[64];                           \
-                                                                        \
-                        tmp = svs_get_handle (this, loc, inode_ctx,     \
-                                              &op_errno);               \
-                        if (tmp) {                                      \
-                                gf_log (this->name, GF_LOG_ERROR,       \
-                                        "failed to get the handle for %s " \
-                                        "(gfid: %s)", loc->path,        \
-                                        uuid_utoa_r (loc->inode->gfid,  \
-                                                     tmp_uuid));        \
-                                ret = -1;                               \
-                                goto label;                             \
-                        }                                               \
-                                                                        \
-                        fs = inode_ctx->fs;                             \
-                        object = inode_ctx->object;                     \
-                }                                                       \
-        } while(0);
-
-#define SVS_STRDUP(dst, src)                                            \
-        do {                                                            \
-                if (dst && strcmp (src, dst)) {                         \
-                        GF_FREE (dst);                                  \
-                        dst = NULL;                                     \
-                }                                                       \
-                                                                        \
-                if (!dst)                                               \
-                        dst = gf_strdup (src);                          \
-        } while (0)
+#define SNAP_VIEW_MAX_GLFS_T 256
+#define SNAP_VIEW_MAX_GLFS_FDS 1024
+#define SNAP_VIEW_MAX_GLFS_OBJ_HANDLES 1024
+
+#define SVS_STACK_DESTROY(_frame)                                              \
+    do {                                                                       \
+        ((call_frame_t *)_frame)->local = NULL;                                \
+        STACK_DESTROY(((call_frame_t *)_frame)->root);                         \
+    } while (0)
+
+#define SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this)                              \
+    do {                                                                       \
+        svs_private_t *_private = NULL;                                        \
+        _private = this->private;                                              \
+        int i = 0;                                                             \
+        gf_boolean_t found = _gf_false;                                        \
+        glfs_t *tmp_fs = NULL;                                                 \
+        LOCK(&_private->snaplist_lock);                                        \
+        {                                                                      \
+            for (i = 0; i < _private->num_snaps; i++) {                        \
+                tmp_fs = _private->dirents[i].fs;                              \
+                gf_log(this->name, GF_LOG_DEBUG,                               \
+                       "snap name: %s, snap volume: %s,"                       \
+                       "dirent->fs: %p",                                       \
+                       _private->dirents[i].name,                              \
+                       _private->dirents[i].snap_volname, tmp_fs);             \
+                if (tmp_fs && fs && (tmp_fs == fs)) {                          \
+                    found = _gf_true;                                          \
+                    gf_msg_debug(this->name, 0,                                \
+                                 "found the fs "                               \
+                                 "instance");                                  \
+                    break;                                                     \
+                }                                                              \
+            }                                                                  \
+        }                                                                      \
+        UNLOCK(&_private->snaplist_lock);                                      \
+                                                                               \
+        if (!found) {                                                          \
+            gf_log(this->name, GF_LOG_WARNING,                                 \
+                   "failed to"                                                 \
+                   " find the fs instance %p",                                 \
+                   fs);                                                        \
+            fs = NULL;                                                         \
+        }                                                                      \
+    } while (0)
+
+#define SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, ret,          \
+                               op_errno, label)                                \
+    do {                                                                       \
+        fs = inode_ctx->fs;                                                    \
+        object = inode_ctx->object;                                            \
+        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);                             \
+        if (!fs)                                                               \
+            object = NULL;                                                     \
+                                                                               \
+        if (!fs || !object) {                                                  \
+            int32_t tmp = -1;                                                  \
+            char tmp_uuid[64];                                                 \
+                                                                               \
+            tmp = svs_get_handle(this, loc, inode_ctx, &op_errno);             \
+            if (tmp) {                                                         \
+                gf_log(this->name, GF_LOG_ERROR,                               \
+                       "failed to get the handle for %s "                      \
+                       "(gfid: %s)",                                           \
+                       loc->path, uuid_utoa_r(loc->inode->gfid, tmp_uuid));    \
+                ret = -1;                                                      \
+                goto label;                                                    \
+            }                                                                  \
+                                                                               \
+            fs = inode_ctx->fs;                                                \
+            object = inode_ctx->object;                                        \
+        }                                                                      \
+    } while (0);
+
+#define SVS_STRDUP(dst, src)                                                   \
+    do {                                                                       \
+        if (dst && strcmp(src, dst)) {                                         \
+            GF_FREE(dst);                                                      \
+            dst = NULL;                                                        \
+        }                                                                      \
+                                                                               \
+        if (!dst)                                                              \
+            dst = gf_strdup(src);                                              \
+    } while (0)
 
 int
-svs_mgmt_submit_request (void *req, call_frame_t *frame,
-                         glusterfs_ctx_t *ctx,
-                         rpc_clnt_prog_t *prog, int procnum,
-                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+svs_mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                        rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                        xdrproc_t xdrproc);
 
 int
-svs_get_snapshot_list (xlator_t *this);
+svs_get_snapshot_list(xlator_t *this);
 
 int
-mgmt_get_snapinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe);
+mgmt_get_snapinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe);
 
 typedef enum {
-        SNAP_VIEW_ENTRY_POINT_INODE = 0,
-        SNAP_VIEW_SNAPSHOT_INODE,
-        SNAP_VIEW_VIRTUAL_INODE
+    SNAP_VIEW_ENTRY_POINT_INODE = 0,
+    SNAP_VIEW_SNAPSHOT_INODE,
+    SNAP_VIEW_VIRTUAL_INODE
 } inode_type_t;
 
 struct svs_inode {
-        glfs_t *fs;
-        glfs_object_t *object;
-        inode_type_t type;
-
-        /* used only for entry point directory where gfid of the directory
-           from where the entry point was entered is saved.
-        */
-        uuid_t pargfid;
-
-        /* This is used to generate gfid for all sub files/dirs under this
-         * snapshot
-         */
-        char *snapname;
-        struct iatt buf;
+    glfs_t *fs;
+    glfs_object_t *object;
+    inode_type_t type;
+
+    /* used only for entry point directory where gfid of the directory
+       from where the entry point was entered is saved.
+    */
+    uuid_t pargfid;
+
+    /* This is used to generate gfid for all sub files/dirs under this
+     * snapshot
+     */
+    char *snapname;
+    struct iatt buf;
 };
 typedef struct svs_inode svs_inode_t;
 
 struct svs_fd {
-        glfs_fd_t *fd;
+    glfs_fd_t *fd;
 };
 typedef struct svs_fd svs_fd_t;
 
 struct snap_dirent {
-        char name[NAME_MAX];
-        char uuid[UUID_CANONICAL_FORM_LEN + 1];
-        char snap_volname[NAME_MAX];
-        glfs_t *fs;
+    char name[NAME_MAX];
+    char uuid[UUID_CANONICAL_FORM_LEN + 1];
+    char snap_volname[NAME_MAX];
+    glfs_t *fs;
 };
 typedef struct snap_dirent snap_dirent_t;
 
 struct svs_private {
-        snap_dirent_t           *dirents;
-        int                     num_snaps;
-        char                    *volname;
-        struct list_head        snaplist;
-        gf_lock_t               snaplist_lock;
-        struct rpc_clnt         *rpc;
+    snap_dirent_t *dirents;
+    int num_snaps;
+    char *volname;
+    struct list_head snaplist;
+    gf_lock_t snaplist_lock;
+    struct rpc_clnt *rpc;
 };
 typedef struct svs_private svs_private_t;
 
 int
-__svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
+__svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
 
 svs_inode_t *
-__svs_inode_ctx_get (xlator_t *this, inode_t *inode);
+__svs_inode_ctx_get(xlator_t *this, inode_t *inode);
 
 svs_inode_t *
-svs_inode_ctx_get (xlator_t *this, inode_t *inode);
+svs_inode_ctx_get(xlator_t *this, inode_t *inode);
 
 int32_t
-svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
+svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
 
 svs_inode_t *
-svs_inode_ctx_get_or_new (xlator_t *this, inode_t *inode);
+svs_inode_ctx_get_or_new(xlator_t *this, inode_t *inode);
 
 int
-__svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
+__svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
 
 svs_fd_t *
-__svs_fd_ctx_get (xlator_t *this, fd_t *fd);
+__svs_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 svs_fd_t *
-svs_fd_ctx_get (xlator_t *this, fd_t *fd);
+svs_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 int32_t
-svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
+svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
 
 svs_fd_t *
-__svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd);
+__svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd);
 
 svs_fd_t *
-svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd);
+svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd);
 
-void
-svs_uuid_generate (uuid_t gfid, char *snapname, uuid_t origin_gfid);
+int
+svs_uuid_generate(xlator_t *this, uuid_t gfid, char *snapname,
+                  uuid_t origin_gfid);
 
 void
-svs_fill_ino_from_gfid (struct iatt *buf);
+svs_fill_ino_from_gfid(struct iatt *buf);
 
 void
-svs_iatt_fill (uuid_t gfid, struct iatt *buf);
+svs_iatt_fill(uuid_t gfid, struct iatt *buf);
 
 snap_dirent_t *
-svs_get_latest_snap_entry (xlator_t *this);
+svs_get_latest_snap_entry(xlator_t *this);
 
 glfs_t *
-svs_get_latest_snapshot (xlator_t *this);
+svs_get_latest_snapshot(xlator_t *this);
 
 glfs_t *
-svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                int32_t *op_errno);
+svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                               int32_t *op_errno);
 
 glfs_t *
-__svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                  int32_t *op_errno);
+__svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                                 int32_t *op_errno);
 
 snap_dirent_t *
-__svs_get_snap_dirent (xlator_t *this, const char *name);
+__svs_get_snap_dirent(xlator_t *this, const char *name);
 
 int
-svs_mgmt_init (xlator_t *this);
+svs_mgmt_init(xlator_t *this);
 
 int32_t
-svs_get_handle (xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
-                int32_t *op_errno);
+svs_get_handle(xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
+               int32_t *op_errno);
+
+glfs_t *
+svs_inode_glfs_mapping(xlator_t *this, inode_t *inode);
+
+glfs_t *
+svs_inode_ctx_glfs_mapping(xlator_t *this, svs_inode_t *inode_ctx);
 
 #endif /* __SNAP_VIEW_H__ */
diff --git a/xlators/features/ganesha/Makefile.am b/xlators/features/thin-arbiter/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/features/ganesha/Makefile.am
+++ b/xlators/features/thin-arbiter/Makefile.am
diff --git a/xlators/features/thin-arbiter/src/Makefile.am b/xlators/features/thin-arbiter/src/Makefile.am
new file mode 100644
index 00000000000..a3c133e7798
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/Makefile.am
@@ -0,0 +1,22 @@
+xlator_LTLIBRARIES = thin-arbiter.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+thin_arbiter_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+thin_arbiter_la_SOURCES = thin-arbiter.c \
+    $(top_builddir)/xlators/lib/src/libxlator.c
+
+thin_arbiter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = thin-arbiter.h thin-arbiter-mem-types.h thin-arbiter-messages.h \
+    $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+    -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+    -I$(top_srcdir)/rpc/rpc-lib/src \
+    -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter-mem-types.h b/xlators/features/thin-arbiter/src/thin-arbiter-mem-types.h
new file mode 100644
index 00000000000..69562d2febc
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter-mem-types.h
@@ -0,0 +1,19 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __THIN_ARBITER_MEM_TYPES_H__
+#define __THIN_ARBITER_MEM_TYPES_H__
+#include <glusterfs/mem-types.h>
+
+typedef enum gf_ta_mem_types_ {
+    gf_ta_mt_local_t = gf_common_mt_end + 1,
+    gf_ta_mt_char,
+    gf_ta_mt_end
+} gf_ta_mem_types_t;
+#endif
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter-messages.h b/xlators/features/thin-arbiter/src/thin-arbiter-messages.h
new file mode 100644
index 00000000000..81d7491577a
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter-messages.h
@@ -0,0 +1,28 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _TA_MESSAGES_H_
+#define _TA_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(TA, TA_MSG_INVALID_FOP);
+
+#endif /* !_TA_MESSAGES_H_ */
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter.c b/xlators/features/thin-arbiter/src/thin-arbiter.c
new file mode 100644
index 00000000000..ce3008636f1
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter.c
@@ -0,0 +1,661 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "thin-arbiter.h"
+#include "thin-arbiter-messages.h"
+#include "thin-arbiter-mem-types.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
+
+int
+ta_set_incoming_values(dict_t *dict, char *key, data_t *value, void *data)
+{
+    int32_t ret = 0;
+    ta_fop_t *fop = (ta_fop_t *)data;
+    int32_t *pending = NULL;
+
+    pending = GF_CALLOC(1, value->len, gf_ta_mt_char);
+    if (!pending) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = dict_set_bin(fop->brick_xattr, key, pending, value->len);
+out:
+    return ret;
+}
+
+int
+ta_get_incoming_and_brick_values(dict_t *dict, char *key, data_t *value,
+                                 void *data)
+{
+    ta_fop_t *fop = data;
+    char *source = NULL;
+    char *in_coming = NULL;
+    int32_t len = 0, ret = 0;
+
+    source = GF_CALLOC(1, value->len, gf_ta_mt_char);
+    if (!source) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_ptr_and_len(fop->dict, key, (void **)&in_coming, &len);
+
+    if (!in_coming || value->len != len) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    if (!memcmp(value->data, source, value->len) &&
+        (!memcmp(in_coming, source, len))) {
+        fop->on_disk[fop->idx] = 0;
+    } else {
+        fop->on_disk[fop->idx] = 1;
+    }
+
+    fop->idx++;
+out:
+    GF_FREE(source);
+    return ret;
+}
+
+void
+ta_release_fop(ta_fop_t *fop)
+{
+    if (!fop) {
+        return;
+    }
+    if (fop->fd) {
+        fd_unref(fop->fd);
+    }
+    loc_wipe(&fop->loc);
+    if (fop->dict) {
+        dict_unref(fop->dict);
+    }
+    if (fop->brick_xattr) {
+        dict_unref(fop->brick_xattr);
+    }
+
+    GF_FREE(fop);
+    return;
+}
+
+int32_t
+ta_set_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+    TA_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+/*
+case 1 - If brick value is 0 and incoming value is also 0, fine
+case 2 - If brick value is 0 and incoming value is non 0, fine
+case 3 - If brick value is non 0 and incoming value is also 0, fine
+case 4 - If brick value is non 0 and incoming value is non 0, fine
+case 5 - If incoming value is non zero on both brick, it is wrong
+case 6 - If incoming value is non zero but brick value for other
+brick is also non zero, wrong
+*/
+
+int32_t
+ta_verify_on_disk_source(ta_fop_t *fop, dict_t *dict)
+{
+    int ret = 0;
+
+    if (!fop) {
+        return -EINVAL;
+    }
+
+    ret = dict_foreach(dict, ta_get_incoming_and_brick_values, (void *)fop);
+    if (ret < 0) {
+        return ret;
+    }
+    if (fop->on_disk[0] && fop->on_disk[1]) {
+        return -EINVAL;
+    }
+    return 0;
+}
+
+int32_t
+ta_get_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+    ta_fop_t *fop = NULL;
+    int ret = 0;
+
+    fop = frame->local;
+    if (op_ret) {
+        goto unwind;
+    }
+
+    ret = ta_verify_on_disk_source(fop, dict);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    if (fop->fd) {
+        STACK_WIND(frame, ta_set_xattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fop->fd,
+                   fop->xattrop_flags, fop->dict, NULL);
+    } else {
+        STACK_WIND(frame, ta_set_xattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, &fop->loc,
+                   fop->xattrop_flags, fop->dict, NULL);
+    }
+    return 0;
+
+unwind:
+
+    TA_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return -1;
+}
+
+ta_fop_t *
+ta_prepare_fop(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    ta_fop_t *fop = NULL;
+    int ret = 0;
+
+    fop = GF_CALLOC(1, sizeof(*fop), gf_ta_mt_local_t);
+    if (!fop) {
+        goto out;
+    }
+
+    if (loc) {
+        loc_copy(&fop->loc, loc);
+    }
+
+    if (fd) {
+        fop->fd = fd_ref(fd);
+    }
+
+    fop->xattrop_flags = flags;
+    fop->idx = 0;
+
+    if (dict != NULL) {
+        fop->dict = dict_ref(dict);
+    }
+    fop->brick_xattr = dict_new();
+    if (fop->brick_xattr == NULL) {
+        goto out;
+    }
+    ret = dict_foreach(dict, ta_set_incoming_values, (void *)fop);
+    if (ret < 0) {
+        goto out;
+    }
+    frame->local = fop;
+    return fop;
+
+out:
+    ta_release_fop(fop);
+    return NULL;
+}
+
+int32_t
+ta_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    ta_fop_t *fop = NULL;
+
+    fop = ta_prepare_fop(frame, this, NULL, fd, flags, dict, xdata);
+    if (!fop) {
+        ret = -ENOMEM;
+        goto unwind;
+    }
+
+    STACK_WIND(frame, ta_get_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, fop->brick_xattr,
+               xdata);
+    return 0;
+
+unwind:
+
+    TA_STACK_UNWIND(xattrop, frame, -1, -ret, NULL, NULL);
+    return 0;
+}
+
+int32_t
+ta_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    ta_fop_t *fop = NULL;
+
+    fop = ta_prepare_fop(frame, this, loc, NULL, flags, dict, xdata);
+    if (!fop) {
+        ret = -ENOMEM;
+        goto unwind;
+    }
+
+    STACK_WIND(frame, ta_get_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, fop->brick_xattr,
+               xdata);
+    return 0;
+
+unwind:
+
+    TA_STACK_UNWIND(xattrop, frame, -1, -ret, NULL, NULL);
+    return 0;
+}
+
+int32_t
+ta_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(writev, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    TA_FAILED_FOP(fsetxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    TA_FAILED_FOP(setxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    TA_FAILED_FOP(fallocate, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(access, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    TA_FAILED_FOP(discard, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
+{
+    TA_FAILED_FOP(entrylk, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(fentrylk, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    TA_FAILED_FOP(flush, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
+{
+    TA_FAILED_FOP(fsync, frame, EINVAL);
+    return 0;
+}
+int32_t
+ta_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(fsyncdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(getxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    TA_FAILED_FOP(fgetxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
+{
+    TA_FAILED_FOP(link, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    TA_FAILED_FOP(lk, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    TA_FAILED_FOP(mkdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    TA_FAILED_FOP(mknod, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
+{
+    TA_FAILED_FOP(open, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    TA_FAILED_FOP(opendir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, dict_t *xdata)
+{
+    TA_FAILED_FOP(readdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
+{
+    TA_FAILED_FOP(readdirp, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(readlink, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    TA_FAILED_FOP(readv, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    TA_FAILED_FOP(removexattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    TA_FAILED_FOP(fremovexattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(rename, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+         dict_t *xdata)
+{
+    TA_FAILED_FOP(rmdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    TA_FAILED_FOP(setattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    TA_FAILED_FOP(fsetattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    TA_FAILED_FOP(stat, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    TA_FAILED_FOP(fstat, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    TA_FAILED_FOP(statfs, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    TA_FAILED_FOP(symlink, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(truncate, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
+{
+    TA_FAILED_FOP(ftruncate, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(unlink, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    TA_FAILED_FOP(zerofill, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
+{
+    TA_FAILED_FOP(seek, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_ta_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    return 0;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'thin_arbiter' not configured with exactly one child");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_ERROR, "dangling volume. check volfile ");
+    }
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    return;
+}
+
+struct xlator_fops fops = {
+    /*Passed fop*/
+    .xattrop = ta_xattrop,
+    .fxattrop = ta_fxattrop,
+    /*Failed fop*/
+    .writev = ta_writev,
+    .stat = ta_stat,
+    .fstat = ta_fstat,
+    .truncate = ta_truncate,
+    .ftruncate = ta_ftruncate,
+    .access = ta_access,
+    .readlink = ta_readlink,
+    .mknod = ta_mknod,
+    .mkdir = ta_mkdir,
+    .unlink = ta_unlink,
+    .rmdir = ta_rmdir,
+    .symlink = ta_symlink,
+    .rename = ta_rename,
+    .link = ta_link,
+    .open = ta_open,
+    .readv = ta_readv,
+    .flush = ta_flush,
+    .fsync = ta_fsync,
+    .opendir = ta_opendir,
+    .readdir = ta_readdir,
+    .readdirp = ta_readdirp,
+    .fsyncdir = ta_fsyncdir,
+    .statfs = ta_statfs,
+    .setxattr = ta_setxattr,
+    .getxattr = ta_getxattr,
+    .fsetxattr = ta_fsetxattr,
+    .fgetxattr = ta_fgetxattr,
+    .removexattr = ta_removexattr,
+    .fremovexattr = ta_fremovexattr,
+    .lk = ta_lk,
+    .entrylk = ta_entrylk,
+    .fentrylk = ta_fentrylk,
+    .setattr = ta_setattr,
+    .fsetattr = ta_fsetattr,
+    .fallocate = ta_fallocate,
+    .discard = ta_discard,
+    .zerofill = ta_zerofill,
+    .seek = ta_seek,
+};
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_6_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "thin-arbiter",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter.h b/xlators/features/thin-arbiter/src/thin-arbiter.h
new file mode 100644
index 00000000000..e5f914b84bf
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter.h
@@ -0,0 +1,59 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _THIN_ARBITER_H
+#define _THIN_ARBITER_H
+
+#include <glusterfs/locking.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/list.h>
+
+#define THIN_ARBITER_SOURCE_XATTR "trusted.ta.source"
+#define THIN_ARBITER_SOURCE_SIZE 2
+
+#define TA_FAILED_FOP(fop, frame, op_errno)                                    \
+    do {                                                                       \
+        default_##fop##_failure_cbk(frame, op_errno);                          \
+    } while (0)
+
+#define TA_STACK_UNWIND(fop, frame, op_ret, op_errno, params...)               \
+    do {                                                                       \
+        ta_fop_t *__local = NULL;                                              \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+                                                                               \
+        __local = frame->local;                                                \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (__local) {                                                         \
+            ta_release_fop(__local);                                           \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, __op_ret, __op_errno, params);         \
+                                                                               \
+    } while (0)
+
+struct _ta_fop;
+typedef struct _ta_fop ta_fop_t;
+
+struct _ta_fop {
+    gf_xattrop_flags_t xattrop_flags;
+    loc_t loc;
+    fd_t *fd;
+    dict_t *dict;
+    dict_t *brick_xattr;
+    int32_t on_disk[2];
+    int32_t idx;
+};
+
+#endif /* _THIN_ARBITER_H */
diff --git a/xlators/features/trash/src/Makefile.am b/xlators/features/trash/src/Makefile.am
index 1304618cc68..8557e7171af 100644
--- a/xlators/features/trash/src/Makefile.am
+++ b/xlators/features/trash/src/Makefile.am
@@ -1,4 +1,6 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = trash.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 trash_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
@@ -8,7 +10,8 @@ trash_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = trash.h trash-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/trash/src/trash-mem-types.h b/xlators/features/trash/src/trash-mem-types.h
index b7cad3ce3a9..43353c8f095 100644
--- a/xlators/features/trash/src/trash-mem-types.h
+++ b/xlators/features/trash/src/trash-mem-types.h
@@ -10,14 +10,13 @@
 #ifndef __TRASH_MEM_TYPES_H__
 #define __TRASH_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_trash_mem_types_ {
-        gf_trash_mt_trash_private_t = gf_common_mt_end + 1,
-        gf_trash_mt_char,
-        gf_trash_mt_uuid,
-        gf_trash_mt_trash_elim_path,
-        gf_trash_mt_end
+    gf_trash_mt_trash_private_t = gf_common_mt_end + 1,
+    gf_trash_mt_char,
+    gf_trash_mt_uuid,
+    gf_trash_mt_trash_elim_path,
+    gf_trash_mt_end
 };
 #endif
-
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index 025d2c2efc1..7d09cba3e9c 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -9,31 +9,32 @@
 */
 #include "trash.h"
 #include "trash-mem-types.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
-#define root_gfid        (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
-#define trash_gfid       (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5}
-#define internal_op_gfid (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6}
+#define root_gfid                                                              \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }
+#define trash_gfid                                                             \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5 }
+#define internal_op_gfid                                                       \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6 }
 
 int32_t
-trash_truncate_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t *xdata);
+trash_truncate_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata);
 
 int32_t
-trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *stbuf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t *xdata);
+trash_truncate_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *stbuf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata);
 
 int32_t
-trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata);
-
+trash_unlink_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        struct iatt *preoldparent, struct iatt *postoldparent,
+                        struct iatt *prenewparent, struct iatt *postnewparent,
+                        dict_t *xdata);
 /* Common routines used in this translator */
 
 /**
@@ -42,21 +43,27 @@ trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  * the existing directory and returns the same
  */
 mode_t
-get_permission (char *path)
+get_permission(char *path)
 {
-        mode_t                  mode                    = 0755;
-        struct stat             sbuf                    = {0,};
-        struct iatt             ibuf                    = {0,};
-        int                     ret                     = 0;
-
-        ret = sys_stat (path, &sbuf);
-        if (!ret) {
-                iatt_from_stat (&ibuf, &sbuf);
-                mode = st_mode_from_ia (ibuf.ia_prot, ibuf.ia_type);
-        } else
-                gf_log ("trash", GF_LOG_DEBUG, "stat on %s failed"
-                                " using default", path);
-        return mode;
+    mode_t mode = 0755;
+    struct stat sbuf = {
+        0,
+    };
+    struct iatt ibuf = {
+        0,
+    };
+    int ret = 0;
+
+    ret = sys_stat(path, &sbuf);
+    if (!ret) {
+        iatt_from_stat(&ibuf, &sbuf);
+        mode = st_mode_from_ia(ibuf.ia_prot, ibuf.ia_type);
+    } else
+        gf_log("trash", GF_LOG_DEBUG,
+               "stat on %s failed"
+               " using default",
+               path);
+    return mode;
 }
 
 /**
@@ -65,197 +72,198 @@ get_permission (char *path)
  * striped out for additional usage.
  */
 int
-extract_trash_directory (char *priv_value, const char **trash_directory)
+extract_trash_directory(char *priv_value, const char **trash_directory)
 {
-        char                    *tmp                    = NULL;
-        int                     ret                     = 0;
-
-        GF_VALIDATE_OR_GOTO("trash", priv_value, out);
-
-        tmp = gf_strdup (priv_value + 1);
-        if (!tmp) {
-                ret = ENOMEM;
-                goto out;
-        }
-        if (tmp[strlen(tmp)-1] == '/')
-                tmp[strlen(tmp)-1] = '\0';
-        *trash_directory = gf_strdup (tmp);
-        if (!(*trash_directory)) {
-                ret = ENOMEM;
-                goto out;
-        }
+    char *tmp = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("trash", priv_value, out);
+
+    tmp = gf_strdup(priv_value + 1);
+    if (!tmp) {
+        ret = ENOMEM;
+        goto out;
+    }
+    if (tmp[strlen(tmp) - 1] == '/')
+        tmp[strlen(tmp) - 1] = '\0';
+    *trash_directory = gf_strdup(tmp);
+    if (!(*trash_directory)) {
+        ret = ENOMEM;
+        goto out;
+    }
 out:
-        if (tmp)
-                GF_FREE (tmp);
-        return ret;
+    if (tmp)
+        GF_FREE(tmp);
+    return ret;
 }
 
 /**
- * The trash directory path should be append at begining of file path for
+ * The trash directory path should be append at beginning of file path for
  * delete or truncate operations. Normal trashing moves the contents to
  * trash directory and trashing done by internal operations are moved to
  * internal_op directory inside trash.
  */
 void
-copy_trash_path (const char *priv_value, gf_boolean_t internal, char *path)
+copy_trash_path(const char *priv_value, gf_boolean_t internal, char *path,
+                size_t path_size)
 {
-        char                    trash_path[PATH_MAX]    = {0,};
-
-        strcpy (trash_path, priv_value);
-        if (internal)
-                strcat (trash_path, "internal_op/");
-
-        strcpy (path, trash_path);
+    char trash_path[PATH_MAX] = {
+        0,
+    };
+
+    strncpy(trash_path, priv_value, sizeof(trash_path));
+    trash_path[sizeof(trash_path) - 1] = 0;
+    if (internal)
+        strncat(trash_path, "internal_op/",
+                sizeof(trash_path) - strlen(trash_path) - 1);
+
+    strncpy(path, trash_path, path_size);
+    path[path_size - 1] = 0;
 }
 
 /**
  * This function performs the reverse operation of copy_trash_path(). It gives
  * out a pointer, whose starting value will be the path inside trash directory,
- * similar to orginal path.
+ * similar to original path.
  */
 void
-remove_trash_path (const char *path, gf_boolean_t internal, char **rem_path)
+remove_trash_path(const char *path, gf_boolean_t internal, char **rem_path)
 {
-        if (rem_path == NULL) {
-                return;
-        }
-
-        *rem_path =  strchr (path + 1, '/');
-        if (internal)
-                *rem_path =  strchr (*rem_path + 1, '/');
-}
-
-/**
- * Check whether the path includes trash directory or internal op directory
- * inside trash. This check is used to make sure that we avoid deletion,
- * rename and creation operations from trash directory.
- */
-int
-check_whether_trash_directory (const char *path,
-                               const char *trash_directory_path)
-{
-        char                    tmp_path[PATH_MAX]              = {0,};
-        char                    internal_op_path[PATH_MAX]      = {0,};
-        int                     ret                             = 0;
-
-        if (path[strlen(path)-1] == '/')
-                sprintf (tmp_path, "%s", path);
-        else
-                sprintf (tmp_path, "%s/", path);
-
-        copy_trash_path (trash_directory_path, _gf_true, internal_op_path);
-        ret = strcmp (tmp_path, trash_directory_path) &&
-              strcmp (tmp_path, internal_op_path);
+    if (rem_path == NULL) {
+        return;
+    }
 
-        return ret;
+    *rem_path = strchr(path + 1, '/');
+    if (internal)
+        *rem_path = strchr(*rem_path + 1, '/');
 }
 
 /**
  * Checks whether the given path reside under the specified eliminate path
  */
 int
-check_whether_eliminate_path (trash_elim_path *trav, const char *path)
+check_whether_eliminate_path(trash_elim_path *trav, const char *path)
 {
-        int                     match                           = 0;
-
-        while (trav) {
-                if (strncmp (path, trav->path, strlen(trav->path)) == 0) {
-                        match++;
-                        break;
-                }
-                trav = trav->next;
+    int match = 0;
+
+    while (trav) {
+        if (strncmp(path, trav->path, strlen(trav->path)) == 0) {
+            match++;
+            break;
         }
-        return match;
+        trav = trav->next;
+    }
+    return match;
 }
 
 /**
  * Stores the eliminate path into internal eliminate path structure
  */
 int
-store_eliminate_path (char *str, trash_elim_path **eliminate)
+store_eliminate_path(char *str, trash_elim_path **eliminate)
 {
-        trash_elim_path         *trav                   = NULL;
-        char                    *component              = NULL;
-        char                    elm_path[PATH_MAX]      = {0,};
-        int                     ret                     = 0;
-        char                    *strtokptr              = NULL;
-
-        if (eliminate == NULL) {
-                ret = EINVAL;
-                goto out;
-        }
-
-        component = strtok_r (str, ",", &strtokptr);
-        while (component) {
-                trav = GF_CALLOC (1, sizeof (*trav),
-                                 gf_trash_mt_trash_elim_path);
-                if (!trav) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                if (component[0] == '/')
-                        sprintf(elm_path, "%s", component);
-                else
-                        sprintf(elm_path, "/%s", component);
-
-                if (component[strlen(component)-1] != '/')
-                        strcat (elm_path, "/");
-
-                trav->path = gf_strdup(elm_path);
-                if (!trav->path) {
-                                ret = ENOMEM;
-                                gf_log ("trash", GF_LOG_DEBUG, "out of memory");
-                                goto out;
-                }
-                trav->next = *eliminate;
-                *eliminate = trav;
-                component = strtok_r (NULL, ",", &strtokptr);
-        }
+    trash_elim_path *trav = NULL;
+    char *component = NULL;
+    char elm_path[PATH_MAX] = {
+        0,
+    };
+    int ret = 0;
+    char *strtokptr = NULL;
+
+    if ((str == NULL) || (eliminate == NULL)) {
+        ret = EINVAL;
+        goto out;
+    }
+
+    component = strtok_r(str, ",", &strtokptr);
+    while (component) {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_trash_mt_trash_elim_path);
+        if (!trav) {
+            ret = ENOMEM;
+            goto out;
+        }
+        if (component[0] == '/')
+            sprintf(elm_path, "%s", component);
+        else
+            sprintf(elm_path, "/%s", component);
+
+        if (component[strlen(component) - 1] != '/')
+            strncat(elm_path, "/", sizeof(elm_path) - strlen(elm_path) - 1);
+
+        trav->path = gf_strdup(elm_path);
+        if (!trav->path) {
+            ret = ENOMEM;
+            gf_log("trash", GF_LOG_DEBUG, "out of memory");
+            GF_FREE(trav);
+            goto out;
+        }
+        trav->next = *eliminate;
+        *eliminate = trav;
+        component = strtok_r(NULL, ",", &strtokptr);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /**
  * Appends time stamp to given string
  */
 void
-append_time_stamp (char *name)
+append_time_stamp(char *name, size_t name_size)
 {
-        int                     i;
-        char                    timestr[64]            = {0,};
+    int i;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    gf_time_fmt(timestr, sizeof(timestr), gf_time(), gf_timefmt_F_HMS);
+
+    /* removing white spaces in timestamp */
+    for (i = 0; i < strlen(timestr); i++) {
+        if (timestr[i] == ' ')
+            timestr[i] = '_';
+    }
+    strncat(name, "_", name_size - strlen(name) - 1);
+    strncat(name, timestr, name_size - strlen(name) - 1);
+}
 
-        gf_time_fmt (timestr, sizeof(timestr), time (NULL),
-                     gf_timefmt_F_HMS);
+/* *
+ * Check whether delete/rename operation is permitted on
+ * trash directory
+ */
 
-        /* removing white spaces in timestamp */
-        for (i = 0; i < strlen (timestr); i++) {
-                if (timestr[i] == ' ')
-                        timestr[i] = '_';
-        }
-        strcat (name, "_");
-        strcat (name, timestr);
+gf_boolean_t
+check_whether_op_permitted(trash_private_t *priv, loc_t *loc)
+{
+    if ((priv->state && (gf_uuid_compare(loc->inode->gfid, trash_gfid) == 0)))
+        return _gf_false;
+    if (priv->internal &&
+        (gf_uuid_compare(loc->inode->gfid, internal_op_gfid) == 0))
+        return _gf_false;
+
+    return _gf_true;
 }
 
 /**
  * Wipe the memory used by trash location variable
  */
 void
-trash_local_wipe (trash_local_t *local)
+trash_local_wipe(trash_local_t *local)
 {
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
 
-        if (local->fd)
-                fd_unref (local->fd);
-        if (local->newfd)
-                fd_unref (local->newfd);
+    if (local->fd)
+        fd_unref(local->fd);
+    if (local->newfd)
+        fd_unref(local->newfd);
 
-        mem_put (local);
+    mem_put(local);
 out:
-        return;
+    return;
 }
 
 /**
@@ -263,308 +271,579 @@ out:
  * recursive call
  */
 void
-wipe_eliminate_path (trash_elim_path **trav)
+wipe_eliminate_path(trash_elim_path **trav)
 {
-        if (trav == NULL) {
-                return;
-        }
+    if (trav == NULL) {
+        return;
+    }
 
-        if (*trav == NULL) {
-                return;
-        }
+    if (*trav == NULL) {
+        return;
+    }
 
-        wipe_eliminate_path (&(*trav)->next);
-        GF_FREE ((*trav)->path);
-        GF_FREE (*trav);
-        *trav = NULL;
+    wipe_eliminate_path(&(*trav)->next);
+    GF_FREE((*trav)->path);
+    GF_FREE(*trav);
+    *trav = NULL;
 }
 
 /**
- * This getxattr calls returns existing trash directory path in
- * the dictionary
+ * This is the call back of rename fop initated using STACK_WIND in
+ * reconfigure/notify function which is used to rename trash directory
+ * in the brick when it is required either in volume start or set.
+ * This frame  must destroyed from this function itself since it was
+ * created by trash xlator
  */
 int32_t
-trash_notify_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict,
-                      dict_t *xdata)
+trash_dir_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
-        data_t                 *data                  = NULL;
-        trash_private_t        *priv                  = NULL;
-        int ret                                       = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        data = dict_get (dict, GET_ANCESTRY_PATH_KEY);
-        if (!data) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                                "oldtrash-directory doesnot exists");
-                priv->oldtrash_dir = gf_strdup (priv->newtrash_dir);
-                if (!priv->oldtrash_dir) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-        } else {
-                priv->oldtrash_dir = gf_strdup (data->data);
-                if (!priv->oldtrash_dir) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                         ret = ENOMEM;
-                         goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "old trash directory"
-                                                  " path is %s", data->data);
-        }
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
 
-out:
-          return ret;
-}
+    priv = this->private;
 
-/**
- * This is a nameless look up for old trash directory
- * The lookup is based on gfid, because trash directory
- * has fixed gfid.
- */
-int32_t
-trash_notify_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, dict_t *xdata,
-                         struct iatt *postparent)
-{
-        trash_private_t         *priv                  = NULL;
-        loc_t                   loc                    = {0,};
-        int                     ret                    = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
+    local = frame->local;
 
-        if (op_ret == 0) {
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "rename trash directory "
+               "failed: %s",
+               strerror(op_errno));
+        goto out;
+    }
 
-                gf_log (this->name, GF_LOG_DEBUG, "inode found with gfid %s",
-                                   uuid_utoa(buf->ia_gfid));
+    GF_FREE(priv->oldtrash_dir);
 
-                gf_uuid_copy (loc.gfid,  trash_gfid);
+    priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
+    if (!priv->oldtrash_dir) {
+        op_ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+    }
 
-                /* Find trash inode using available information */
-                priv->trash_inode = inode_link (inode, NULL, NULL, buf);
+out:
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
+}
 
-                loc.inode = inode_ref (priv->trash_inode);
+int
+rename_trash_directory(xlator_t *this)
+{
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    loc_t old_loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (frame == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "failed to create frame");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+
+    /* assign new location values to new_loc members */
+    gf_uuid_copy(loc.gfid, trash_gfid);
+    gf_uuid_copy(loc.pargfid, root_gfid);
+    ret = extract_trash_directory(priv->newtrash_dir, &loc.name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+    loc.path = gf_strdup(priv->newtrash_dir);
+    if (!loc.path) {
+        ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    /* assign old location values to old_loc members */
+    gf_uuid_copy(old_loc.gfid, trash_gfid);
+    gf_uuid_copy(old_loc.pargfid, root_gfid);
+    ret = extract_trash_directory(priv->oldtrash_dir, &old_loc.name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+    old_loc.path = gf_strdup(priv->oldtrash_dir);
+    if (!old_loc.path) {
+        ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    old_loc.inode = inode_ref(priv->trash_inode);
+    gf_uuid_copy(old_loc.inode->gfid, old_loc.gfid);
+
+    loc_copy(&local->loc, &old_loc);
+    loc_copy(&local->newloc, &loc);
+
+    STACK_WIND(frame, trash_dir_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &old_loc, &loc, NULL);
+    return 0;
 
-                /*Used to find path of old trash directory*/
-                STACK_WIND (frame, trash_notify_getxattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->getxattr, &loc,
-                            GET_ANCESTRY_PATH_KEY, xdata);
-        }
+out:
+    if (frame) {
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+    }
 
-        /* If there is no old trash directory we set its value to new one,
-         * which is the valid condition for trash directory creation
-         */
-        else {
-                priv->oldtrash_dir = gf_strdup (priv->newtrash_dir);
-                if (!priv->oldtrash_dir) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto out;
-                }
-        }
+    trash_local_wipe(local);
 
-out:
-        loc_wipe (&loc);
-        return ret;
+    return ret;
 }
 
 int32_t
-trash_internal_op_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+trash_internal_op_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
 {
-        if (op_ret != 0)
-                gf_log (this->name, GF_LOG_ERROR, "mkdir failed for "
-                        "internal op directory : %s", strerror (op_errno));
-        return op_ret;
+    trash_local_t *local = NULL;
+    local = frame->local;
+
+    if (op_ret != 0 && !(op_errno == EEXIST))
+        gf_log(this->name, GF_LOG_ERROR,
+               "mkdir failed for "
+               "internal op directory : %s",
+               strerror(op_errno));
+
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
 }
 
 /**
  * This is the call back of mkdir fop initated using STACK_WIND in
- * notify function which is used to create trash directory in the brick
- * when a volume starts.The frame of the mkdir must destroyed from
- * this function itself since it was created by trash xlator
+ * notify/reconfigure function which is used to create trash directory
+ * in the brick when "trash" is on. The frame of the mkdir must
+ * destroyed from this function itself since it was created by trash xlator
  */
+
 int32_t
-trash_notify_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+trash_dir_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        uuid_t                *gfid_ptr                         = NULL;
-        loc_t                 loc                               = {0, };
-        int                   ret                               = 0;
-        dict_t                *dict                             = NULL;
-        char                  internal_op_path[PATH_MAX]        = {0,};
-        trash_private_t       *priv       = NULL;
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
+    priv = this->private;
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-        if ((op_ret == 0) || (op_ret == -1 && op_errno == EEXIST)) {
-                gfid_ptr = GF_CALLOC (1, sizeof(uuid_t),
-                                           gf_common_mt_uuid_t);
-                if (!gfid_ptr) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                gf_uuid_copy (*gfid_ptr, internal_op_gfid);
-
-                gf_uuid_copy (loc.gfid, internal_op_gfid);
-                gf_uuid_copy (loc.pargfid, trash_gfid);
-                loc.name = gf_strdup ("internal_op");
-
-                if (!loc.name) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                         "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                sprintf (internal_op_path, "%s%s",
-                                priv->newtrash_dir, loc.name);
-
-                loc.path = gf_strdup (internal_op_path);
-
-                if (!loc.path) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                         "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                loc.inode = inode_new (priv->trash_itable);
-                loc.inode->ia_type = IA_IFDIR;
-                /* Fixed gfid is set for trash directory with
-                 * this function
-                 */
-                ret = dict_set_dynptr (dict, "gfid-req", gfid_ptr,
-                                       sizeof (uuid_t));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                 "setting key gfid-req failed");
-                        goto out;
-                }
-
-                /* The mkdir call for creating trash directory */
-                STACK_WIND (frame, trash_internal_op_mkdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir, &loc, 0755,
-                            0022, dict);
-                /* After creating we must call other notify functions */
-                default_notify (this, GF_EVENT_CHILD_UP, NULL);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "mkdir failed for trash"
-                                " directory : %s", strerror (op_errno));
+    local = frame->local;
+
+    if (op_ret == 0) {
+        priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
+        if (!priv->oldtrash_dir) {
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            op_ret = ENOMEM;
         }
+    } else if (op_ret != 0 && errno != EEXIST)
+        gf_log(this->name, GF_LOG_ERROR,
+               "mkdir failed for trash"
+               " directory : %s",
+               strerror(op_errno));
 
-        STACK_DESTROY (frame->root);
-out:
-        if (ret && gfid_ptr)
-                GF_FREE (gfid_ptr);
-        if (dict)
-                dict_unref (dict);
-        return 0;
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
 }
 
+/**
+ * This getxattr calls returns existing trash directory path in
+ * the dictionary
+ */
 int32_t
-trash_notify_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+trash_dir_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        if ((op_ret == 0) || (op_ret == -1 && op_errno == EEXIST)) {
-                /* After creating we must call other notify functions */
-                default_notify (this, GF_EVENT_CHILD_UP, NULL);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "rename failed: %s",
-                        strerror (op_errno));
+    data_t *data = NULL;
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+
+    data = dict_get(dict, GET_ANCESTRY_PATH_KEY);
+    if (!data) {
+        goto out;
+    }
+    priv->oldtrash_dir = GF_MALLOC(PATH_MAX, gf_common_mt_char);
+    if (!priv->oldtrash_dir) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    /* appending '/' if it is not present */
+    sprintf(priv->oldtrash_dir, "%s%c", data->data,
+            data->data[strlen(data->data) - 1] != '/' ? '/' : '\0');
+    gf_log(this->name, GF_LOG_DEBUG,
+           "old trash directory path "
+           "is %s",
+           priv->oldtrash_dir);
+    if (strcmp(priv->newtrash_dir, priv->oldtrash_dir) != 0) {
+        /* When user set a new name for trash directory, trash
+         * xlator will perform a rename operation on old trash
+         * directory to the new one using a STACK_WIND from here.
+         * This option can be configured only when volume is in
+         * started state
+         */
+        ret = rename_trash_directory(this);
+    }
+
+out:
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return ret;
+}
+/**
+ * This is a nameless look up for internal op directory
+ * The lookup is based on gfid, because internal op directory
+ * has fixed gfid.
+ */
+int32_t
+trash_internalop_dir_lookup_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, inode_t *inode,
+                                struct iatt *buf, dict_t *xdata,
+                                struct iatt *postparent)
+{
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    uuid_t *gfid_ptr = NULL;
+    loc_t loc = {
+        0,
+    };
+    char internal_op_path[PATH_MAX] = {
+        0,
+    };
+    dict_t *dict = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    if (op_ret != 0 && op_errno == ENOENT) {
+        loc_wipe(&local->loc);
+        gfid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!gfid_ptr) {
+            ret = ENOMEM;
+            goto out;
+        }
+
+        gf_uuid_copy(*gfid_ptr, internal_op_gfid);
+
+        dict = dict_new();
+        if (!dict) {
+            ret = ENOMEM;
+            goto out;
+        }
+        ret = dict_set_gfuuid(dict, "gfid-req", *gfid_ptr, false);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "setting key gfid-req failed");
+            goto out;
+        }
+        gf_uuid_copy(loc.gfid, internal_op_gfid);
+        gf_uuid_copy(loc.pargfid, trash_gfid);
+
+        loc.inode = inode_new(priv->trash_itable);
+
+        /* The mkdir call for creating internal op directory */
+        loc.name = gf_strdup("internal_op");
+        if (!loc.name) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        sprintf(internal_op_path, "%s%s/", priv->newtrash_dir, loc.name);
 
-        STACK_DESTROY (frame->root);
-        return op_ret;
+        loc.path = gf_strdup(internal_op_path);
+        if (!loc.path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
+        }
+
+        loc_copy(&local->loc, &loc);
+        STACK_WIND(frame, trash_internal_op_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, &loc, 0755, 0022, dict);
+        return 0;
+    }
+
+out:
+    if (ret && gfid_ptr)
+        GF_FREE(gfid_ptr);
+    if (dict)
+        dict_unref(dict);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
 }
 
 /**
- * This is the call back of rename fop initated using STACK_WIND in
- * reconfigure function which is used to rename trash directory in
- * the brick when we perform volume set.This frame  must destroyed
- * from this function itself since it was created by trash xlator
+ * This is a nameless look up for old trash directory
+ * The lookup is based on gfid, because trash directory
+ * has fixed gfid.
  */
 int32_t
-trash_reconf_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
+trash_dir_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        if (op_ret == -1 && op_errno == EEXIST) {
+    trash_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    uuid_t *gfid_ptr = NULL;
+    dict_t *dict = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+
+    loc_wipe(&local->loc);
+    if (op_ret == 0) {
+        gf_log(this->name, GF_LOG_DEBUG, "inode found with gfid %s",
+               uuid_utoa(buf->ia_gfid));
+
+        gf_uuid_copy(loc.gfid, trash_gfid);
+
+        /* Find trash inode using available information */
+        priv->trash_inode = inode_link(inode, NULL, NULL, buf);
 
-                gf_log (this->name, GF_LOG_ERROR, "rename failed: %s",
-                        strerror (op_errno));
+        loc.inode = inode_ref(priv->trash_inode);
+        loc_copy(&local->loc, &loc);
+
+        /*Used to find path of old trash directory*/
+        STACK_WIND(frame, trash_dir_getxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->getxattr, &loc,
+                   GET_ANCESTRY_PATH_KEY, xdata);
+        return 0;
+    }
+
+    /* If there is no old trash directory we set its value to new one,
+     * which is the valid condition for trash directory creation
+     */
+    else {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Creating trash "
+               "directory %s ",
+               priv->newtrash_dir);
+
+        gfid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!gfid_ptr) {
+            ret = ENOMEM;
+            goto out;
+        }
+        gf_uuid_copy(*gfid_ptr, trash_gfid);
+
+        gf_uuid_copy(loc.gfid, trash_gfid);
+        gf_uuid_copy(loc.pargfid, root_gfid);
+        ret = extract_trash_directory(priv->newtrash_dir, &loc.name);
+        if (ret) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+        loc.path = gf_strdup(priv->newtrash_dir);
+        if (!loc.path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
 
-        STACK_DESTROY (frame->root);
+        priv->trash_inode = inode_new(priv->trash_itable);
+        priv->trash_inode->ia_type = IA_IFDIR;
+        loc.inode = inode_ref(priv->trash_inode);
+        dict = dict_new();
+        if (!dict) {
+            ret = ENOMEM;
+            goto out;
+        }
+        /* Fixed gfid is set for trash directory with
+         * this function
+         */
+        ret = dict_set_gfuuid(dict, "gfid-req", *gfid_ptr, false);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "setting key gfid-req failed");
+            goto out;
+        }
+        loc_copy(&local->loc, &loc);
 
-        return op_ret;
+        /* The mkdir call for creating trash directory */
+        STACK_WIND(frame, trash_dir_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, &loc, 0755, 0022, dict);
+        return 0;
+    }
+out:
+    if (ret && gfid_ptr)
+        GF_FREE(gfid_ptr);
+    if (dict)
+        dict_unref(dict);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return ret;
+}
+
+int
+create_or_rename_trash_directory(xlator_t *this)
+{
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (frame == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "failed to create frame");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+
+    loc.inode = inode_new(priv->trash_itable);
+    gf_uuid_copy(loc.gfid, trash_gfid);
+    loc_copy(&local->loc, &loc);
+    gf_log(this->name, GF_LOG_DEBUG,
+           "nameless lookup for"
+           "old trash directory");
+    STACK_WIND(frame, trash_dir_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &loc, NULL);
+out:
+    return ret;
+}
+
+int
+create_internalop_directory(xlator_t *this)
+{
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (frame == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "failed to create frame");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+
+    gf_uuid_copy(loc.gfid, internal_op_gfid);
+    gf_uuid_copy(loc.pargfid, trash_gfid);
+    loc.inode = inode_new(priv->trash_itable);
+    loc.inode->ia_type = IA_IFDIR;
+
+    loc_copy(&local->loc, &loc);
+    STACK_WIND(frame, trash_internalop_dir_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &loc, NULL);
+out:
+
+    return ret;
 }
 
 int32_t
-trash_common_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+trash_common_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
 int32_t
-trash_common_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+trash_common_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        struct iatt *preoldparent, struct iatt *postoldparent,
+                        struct iatt *prenewparent, struct iatt *postnewparent,
+                        dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
 int32_t
-trash_common_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent,
-                   dict_t *xdata)
+trash_common_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 /**
  * move backs from trash translator to unlink call
  */
 int32_t
-trash_common_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent,
-                         dict_t *xdata)
+trash_common_unwind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
 {
-        TRASH_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
-                            postparent, xdata);
-        return 0;
+    TRASH_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
 }
 
 /**
@@ -573,178 +852,183 @@ trash_common_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  * the starting
  */
 int32_t
-trash_unlink_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *stbuf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+trash_unlink_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *stbuf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-        trash_local_t       *local              = NULL;
-        char                *tmp_str            = NULL;
-        char                *tmp_path           = NULL;
-        char                *tmp_dirname        = NULL;
-        char                *tmp_stat           = NULL;
-        char                real_path[PATH_MAX] = {0,};
-        char                *dir_name           = NULL;
-        size_t              count               = 0;
-        int32_t             loop_count          = 0;
-        int                 i                   = 0;
-        loc_t               tmp_loc             = {0,};
-        trash_private_t     *priv               = NULL;
-        int                 ret                 = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local   = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        TRASH_UNSET_PID (frame, local);
-
-        tmp_str = gf_strdup (local->newpath);
-        if (!tmp_str) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
-        }
-        loop_count = local->loop_count;
-
-        /* The directory is not present , need to create it */
-        if ((op_ret == -1) &&  (op_errno == ENOENT)) {
-                tmp_dirname = strchr (tmp_str, '/');
-                while (tmp_dirname) {
-                        count = tmp_dirname - tmp_str;
-                        if (count == 0)
-                                count = 1;
-                        i++;
-                        if (i > loop_count)
-                                break;
-                        tmp_dirname = strchr (tmp_str + count + 1, '/');
-                }
-                tmp_path = gf_memdup (local->newpath, count + 1);
-                if (!tmp_path) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                tmp_path[count] = '\0';
-
-                loc_copy (&tmp_loc, &local->loc);
-                tmp_loc.path = gf_strdup (tmp_path);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                /* Stores the the name of directory to be created */
-                tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
-                if (!tmp_loc.name) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-
-                TRASH_SET_PID (frame, local);
-
-                STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_path,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
-        }
-
-        /* Given path is created , comparing to the required path */
-        if (op_ret == 0) {
-                dir_name = dirname (tmp_str);
-                if (strcmp((char *)cookie, dir_name) == 0) {
-                        /* File path exists we can rename it*/
-                        loc_copy (&tmp_loc, &local->loc);
-                        tmp_loc.path = local->newpath;
-                        STACK_WIND (frame, trash_unlink_rename_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->rename,
-                                    &local->loc, &tmp_loc, xdata);
-                        goto out;
-                }
-        }
-
-        if ((op_ret == -1) && (op_errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_ERROR, "Directory creation failed [%s]. "
-                                "Therefore unlinking %s without moving to trash "
-                                "directory", strerror(op_errno), local->loc.name);
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, &local->loc, 0,
-                            xdata);
-                goto out;
-        }
-
-        LOCK (&frame->lock);
-        {
-                loop_count = ++local->loop_count;
-        }
-        UNLOCK (&frame->lock);
-
-        tmp_dirname = strchr (tmp_str, '/');
-
-        /* Path is not completed , need to create remaining path */
+    trash_local_t *local = NULL;
+    char *tmp_str = NULL;
+    char *tmp_path = NULL;
+    char *tmp_dirname = NULL;
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    char *dir_name = NULL;
+    size_t count = 0;
+    int32_t loop_count = 0;
+    int i = 0;
+    loc_t tmp_loc = {
+        0,
+    };
+    trash_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    TRASH_UNSET_PID(frame, local);
+
+    tmp_str = gf_strdup(local->newpath);
+    if (!tmp_str) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    loop_count = local->loop_count;
+
+    /* The directory is not present , need to create it */
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        tmp_dirname = strchr(tmp_str, '/');
         while (tmp_dirname) {
-                count = tmp_dirname - tmp_str;
-                if (count == 0)
-                        count = 1;
-                i++;
-                if (i > loop_count)
-                        break;
-                tmp_dirname = strchr (tmp_str + count + 1, '/');
-        }
-        tmp_path = gf_memdup (local->newpath, count + 1);
+            count = tmp_dirname - tmp_str;
+            if (count == 0)
+                count = 1;
+            i++;
+            if (i > loop_count)
+                break;
+            tmp_dirname = strchr(tmp_str + count + 1, '/');
+        }
+        tmp_path = gf_memdup(local->newpath, count + 1);
         if (!tmp_path) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
         tmp_path[count] = '\0';
 
-        loc_copy (&tmp_loc, &local->loc);
-        tmp_loc.path = gf_strdup (tmp_path);
+        loc_copy(&tmp_loc, &local->loc);
+        tmp_loc.path = gf_strdup(tmp_path);
         if (!tmp_loc.path) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
 
         /* Stores the the name of directory to be created */
-        tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
+        tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
         if (!tmp_loc.name) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
 
-        strcpy (real_path, priv->brick_path);
-        remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
+        remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
         if (tmp_stat)
-                strcat (real_path, tmp_stat);
-
-        TRASH_SET_PID (frame, local);
-
-        STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_path,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->mkdir, &tmp_loc,
-                           get_permission(real_path), 0022, xdata);
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        STACK_WIND_COOKIE(frame, trash_unlink_mkdir_cbk, tmp_path,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    /* Given path is created , comparing to the required path */
+    if (op_ret == 0) {
+        dir_name = dirname(tmp_str);
+        if (strcmp((char *)cookie, dir_name) == 0) {
+            /* File path exists we can rename it*/
+            loc_copy(&tmp_loc, &local->loc);
+            tmp_loc.path = local->newpath;
+            STACK_WIND(frame, trash_unlink_rename_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->rename, &local->loc, &tmp_loc,
+                       xdata);
+            goto out;
+        }
+    }
+
+    if ((op_ret == -1) && (op_errno != EEXIST)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Directory creation failed [%s]. "
+               "Therefore unlinking %s without moving to trash "
+               "directory",
+               strerror(op_errno), local->loc.name);
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    LOCK(&frame->lock);
+    {
+        loop_count = ++local->loop_count;
+    }
+    UNLOCK(&frame->lock);
+
+    tmp_dirname = strchr(tmp_str, '/');
+
+    /* Path is not completed , need to create remaining path */
+    while (tmp_dirname) {
+        count = tmp_dirname - tmp_str;
+        if (count == 0)
+            count = 1;
+        i++;
+        if (i > loop_count)
+            break;
+        tmp_dirname = strchr(tmp_str + count + 1, '/');
+    }
+    tmp_path = gf_memdup(local->newpath, count + 1);
+    if (!tmp_path) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    tmp_path[count] = '\0';
+
+    loc_copy(&tmp_loc, &local->loc);
+    tmp_loc.path = gf_strdup(tmp_path);
+    if (!tmp_loc.path) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    /* Stores the the name of directory to be created */
+    tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
+    if (!tmp_loc.name) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    strncpy(real_path, priv->brick_path, sizeof(real_path));
+    real_path[sizeof(real_path) - 1] = 0;
+
+    remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
+    if (tmp_stat)
+        strncat(real_path, tmp_stat, sizeof(real_path) - strlen(real_path) - 1);
+
+    TRASH_SET_PID(frame, local);
+
+    STACK_WIND_COOKIE(frame, trash_unlink_mkdir_cbk, tmp_path,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                      &tmp_loc, get_permission(real_path), 0022, xdata);
 
 out:
-        if (tmp_path)
-                GF_FREE (tmp_path);
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        return ret;
+    if (tmp_path)
+        GF_FREE(tmp_path);
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    return ret;
 }
 
 /**
@@ -752,252 +1036,238 @@ out:
  * from trash directory as mentioned in the mount point
  */
 int32_t
-trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
+trash_unlink_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        struct iatt *preoldparent, struct iatt *postoldparent,
+                        struct iatt *prenewparent, struct iatt *postnewparent,
+                        dict_t *xdata)
 {
-        trash_local_t      *local              = NULL;
-        trash_private_t     *priv              = NULL;
-        char               *tmp_str            = NULL;
-        char               *dir_name           = NULL;
-        char               *tmp_cookie         = NULL;
-        loc_t              tmp_loc             = {0,};
-        dict_t             *new_xdata          = NULL;
-        char               *tmp_stat           = NULL;
-        char               real_path[PATH_MAX] = {0,};
-        int                ret                 = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                /* the file path doesnot exists we want to create path
-                 * for the file
-                 */
-                tmp_str = gf_strdup (local->newpath);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                dir_name = dirname (tmp_str); /* stores directory name */
-
-                loc_copy (&tmp_loc, &local->loc);
-                tmp_loc.path = gf_strdup (dir_name);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                tmp_cookie = gf_strdup (dir_name);
-                if (!tmp_cookie) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_str, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-
-                TRASH_SET_PID (frame, local);
-
-                /* create the directory with proper permissions */
-                STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_cookie,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
+    trash_local_t *local = NULL;
+    trash_private_t *priv = NULL;
+    char *tmp_str = NULL;
+    char *dir_name = NULL;
+    char *tmp_cookie = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    dict_t *new_xdata = NULL;
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        /* the file path does not exist we want to create path
+         * for the file
+         */
+        tmp_str = gf_strdup(local->newpath);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        dir_name = dirname(tmp_str); /* stores directory name */
 
-        if ((op_ret == -1) && (op_errno == ENOTDIR)) {
-                /* if entry is already present in trash directory,
-                 * new one is not copied*/
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "target(%s) exists, cannot keep the copy, deleting",
-                        local->newpath);
-
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            &local->loc, 0, xdata);
-
-                goto out;
+        loc_copy(&tmp_loc, &local->loc);
+        tmp_loc.path = gf_strdup(dir_name);
+        if (!tmp_loc.path) {
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
 
-        if ((op_ret == -1) && (op_errno == EISDIR)) {
-
-                /* if entry is directory,we remove directly */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "target(%s) exists as directory, cannot keep copy, "
-                        "deleting", local->newpath);
-
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            &local->loc, 0, xdata);
-                goto out;
+        tmp_cookie = gf_strdup(dir_name);
+        if (!tmp_cookie) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
-
-        /**********************************************************************
-         *
-         * CTR Xlator message handling done here!
-         *
-         **********************************************************************/
-        /**
-         * If unlink is handled by trash translator, it should inform the
-         * CTR Xlator. And trash translator only handles the unlink for
-         * the last hardlink.
-         *
-         * Check if there is a GF_REQUEST_LINK_COUNT_XDATA from CTR Xlator
-         *
-         */
-
-        if (local->ctr_link_count_req) {
-
-                /* Sending back inode link count to ctr_unlink
-                 * (changetimerecoder xlator) via
-                 * "GF_RESPONSE_LINK_COUNT_XDATA" key using xdata.
-                 * */
-                if (xdata) {
-                        ret = dict_set_uint32 (xdata,
-                                               GF_RESPONSE_LINK_COUNT_XDATA,
-                                               1);
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Failed to set"
-                                        " GF_RESPONSE_LINK_COUNT_XDATA");
-                        }
-                } else {
-                        new_xdata = dict_new ();
-                        if (!new_xdata) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Memory allocation failure while "
-                                        "creating new_xdata");
-                                goto ctr_out;
-                        }
-                        ret = dict_set_uint32 (new_xdata,
-                                               GF_RESPONSE_LINK_COUNT_XDATA,
-                                               1);
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Failed to set"
-                                        " GF_RESPONSE_LINK_COUNT_XDATA");
-                        }
-ctr_out:
-                        TRASH_STACK_UNWIND (unlink, frame, 0, op_errno,
-                                            preoldparent, postoldparent,
-                                            new_xdata);
-                        goto out;
-                }
-         }
-        /* All other cases, unlink should return success */
-        TRASH_STACK_UNWIND (unlink, frame, 0, op_errno, preoldparent,
-                            postoldparent, xdata);
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
+        remove_trash_path(tmp_str, (frame->root->pid < 0), &tmp_stat);
+        if (tmp_stat)
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        /* create the directory with proper permissions */
+        STACK_WIND_COOKIE(frame, trash_unlink_mkdir_cbk, tmp_cookie,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    if ((op_ret == -1) && (op_errno == ENOTDIR)) {
+        /* if entry is already present in trash directory,
+         * new one is not copied*/
+        gf_log(this->name, GF_LOG_DEBUG,
+               "target(%s) exists, cannot keep the copy, deleting",
+               local->newpath);
+
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+
+        goto out;
+    }
+
+    if ((op_ret == -1) && (op_errno == EISDIR)) {
+        /* if entry is directory,we remove directly */
+        gf_log(this->name, GF_LOG_DEBUG,
+               "target(%s) exists as directory, cannot keep copy, "
+               "deleting",
+               local->newpath);
+
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    /**********************************************************************
+     *
+     * CTR Xlator message handling done here!
+     *
+     **********************************************************************/
+    /**
+     * If unlink is handled by trash translator, it should inform the
+     * CTR Xlator. And trash translator only handles the unlink for
+     * the last hardlink.
+     *
+     * Check if there is a GF_REQUEST_LINK_COUNT_XDATA from CTR Xlator
+     *
+     */
+
+    if (local->ctr_link_count_req) {
+        /* Sending back inode link count to ctr_unlink
+         * (changetimerecoder xlator) via
+         * "GF_RESPONSE_LINK_COUNT_XDATA" key using xdata.
+         * */
+        if (xdata) {
+            ret = dict_set_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, 1);
+            if (ret == -1) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set"
+                       " GF_RESPONSE_LINK_COUNT_XDATA");
+            }
+        } else {
+            new_xdata = dict_new();
+            if (!new_xdata) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Memory allocation failure while "
+                       "creating new_xdata");
+                goto ctr_out;
+            }
+            ret = dict_set_uint32(new_xdata, GF_RESPONSE_LINK_COUNT_XDATA, 1);
+            if (ret == -1) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set"
+                       " GF_RESPONSE_LINK_COUNT_XDATA");
+            }
+        ctr_out:
+            TRASH_STACK_UNWIND(unlink, frame, 0, op_errno, preoldparent,
+                               postoldparent, new_xdata);
+            goto out;
+        }
+    }
+    /* All other cases, unlink should return success */
+    TRASH_STACK_UNWIND(unlink, frame, 0, op_errno, preoldparent, postoldparent,
+                       xdata);
 out:
 
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (tmp_cookie)
-                GF_FREE (tmp_cookie);
-        if (new_xdata)
-                dict_unref (new_xdata);
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (tmp_cookie)
+        GF_FREE(tmp_cookie);
+    if (new_xdata)
+        dict_unref(new_xdata);
 
-        return ret;
+    return ret;
 }
 
 /**
  * move backs from trash translator to truncate call
  */
 int32_t
-trash_common_unwind_buf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *prebuf, struct iatt *postbuf,
-                             dict_t *xdata)
+trash_common_unwind_buf_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *prebuf, struct iatt *postbuf,
+                            dict_t *xdata)
 {
-        TRASH_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
+    TRASH_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
 }
 
-
-
 int32_t
-trash_unlink_stat_cbk (call_frame_t *frame,  void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                       dict_t *xdata)
+trash_unlink_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
-        trash_private_t     *priv        = NULL;
-        trash_local_t       *local       = NULL;
-        loc_t               new_loc      = {0,};
-        int                 ret          = 0;
-
-        priv  = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: %s",
-                        local->loc.path, strerror (op_errno));
-                TRASH_STACK_UNWIND (unlink, frame, op_ret, op_errno, buf,
-                            NULL, xdata);
-                ret = -1;
-                goto out;
-        }
-
-        /* Only last hardlink will be moved to trash directory */
-        if (buf->ia_nlink > 1) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, &local->loc,
-                            0, xdata);
-                goto out;
-        }
-
-        /* if the file is too big  just unlink it */
-        if (buf->ia_size > (priv->max_trash_file_size)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                                "%s: file size too big (%"PRId64") to "
-                                "move into trash directory",
-                                local->loc.path, buf->ia_size);
-
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, &local->loc,
-                            0, xdata);
-                goto out;
-        }
-
-        /* Copies new path for renaming */
-        loc_copy (&new_loc, &local->loc);
-        new_loc.path = gf_strdup (local->newpath);
-        if (!new_loc.path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-
-
-        STACK_WIND (frame, trash_unlink_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    &local->loc, &new_loc, xdata);
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
+    loc_t new_loc = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG, "%s: %s", local->loc.path,
+               strerror(op_errno));
+        TRASH_STACK_UNWIND(unlink, frame, op_ret, op_errno, buf, NULL, xdata);
+        ret = -1;
+        goto out;
+    }
+
+    /* Only last hardlink will be moved to trash directory */
+    if (buf->ia_nlink > 1) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    /* if the file is too big  just unlink it */
+    if (buf->ia_size > (priv->max_trash_file_size)) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "%s: file size too big (%" PRId64
+               ") to "
+               "move into trash directory",
+               local->loc.path, buf->ia_size);
+
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    /* Copies new path for renaming */
+    loc_copy(&new_loc, &local->loc);
+    new_loc.path = gf_strdup(local->newpath);
+    if (!new_loc.path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    STACK_WIND(frame, trash_unlink_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &local->loc, &new_loc, xdata);
 
 out:
-        loc_wipe (&new_loc);
-
-        return ret;
+    loc_wipe(&new_loc);
 
+    return ret;
 }
 
 /**
@@ -1005,273 +1275,248 @@ out:
  * by internal operations of gluster such as self-heal
  */
 int32_t
-trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
-              dict_t *xdata)
+trash_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+             dict_t *xdata)
 {
-        trash_private_t         *priv           = NULL;
-        trash_local_t           *local          = NULL;/* files inside trash */
-        int32_t                 match           = 0;
-        int32_t                 ctr_link_req    = 0;
-        char                    *pathbuf        = NULL;
-        int                     ret             = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        /* If trash is not active or not enabled through cli, then
-         *  we bypass and wind back
-         */
-        if (!priv->state) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
-
-        /* The files removed by gluster internal operations such as self-heal,
-         * should moved to trash directory , but files by client should not
-         * moved
-         */
-        if ((frame->root->pid < 0) && !priv->internal) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
-        /* loc need some gfid which will be present in inode */
-        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-
-        /* Checking for valid location */
-        if (gf_uuid_is_null (loc->gfid) && gf_uuid_is_null (loc->inode->gfid)) {
-                gf_log (this->name, GF_LOG_DEBUG, "Bad address");
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                ret = EFAULT;
-                goto out;
-        }
-
-        /* This will be more accurate */
-        inode_path (loc->inode, NULL, &pathbuf);
-        /* Check whether the file is present under eliminate paths or
-         * inside trash directory. In both cases we don't need to move the
-         * file to trash directory. Instead delete it permanently
-         */
-        match = check_whether_eliminate_path (priv->eliminate, pathbuf);
-        if ((strncmp (pathbuf, priv->newtrash_dir,
-                      strlen (priv->newtrash_dir)) == 0) || (match)) {
-                if (match) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s is a file comes under an eliminate path, "
-                                "so it is not moved to trash", loc->name);
-                }
-
-                /* Trying to unlink from the trash-dir. So do the
-                 * actual unlink without moving to trash-dir.
-                 */
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                TRASH_STACK_UNWIND (unlink, frame, -1, ENOMEM, NULL, NULL,
-                                    xdata);
-                ret = ENOMEM;
-                goto out;
-        }
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        /* rename new location of file as starting from trash directory */
-        copy_trash_path (priv->newtrash_dir, (frame->root->pid < 0),
-                                                        local->newpath);
-        strcat (local->newpath, pathbuf);
-
-        /* append timestamp to file name so that we can avoid
-         * name collisions inside trash
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL; /* files inside trash */
+    int32_t match = 0;
+    int32_t ctr_link_req = 0;
+    char *pathbuf = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    /* If trash is not active or not enabled through cli, then
+     *  we bypass and wind back
+     */
+    if (!priv->state) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+
+    /* The files removed by gluster internal operations such as self-heal,
+     * should moved to trash directory , but files by client should not
+     * moved
+     */
+    if ((frame->root->pid < 0) && !priv->internal) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+    /* loc need some gfid which will be present in inode */
+    gf_uuid_copy(loc->gfid, loc->inode->gfid);
+
+    /* Checking for valid location */
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+        gf_log(this->name, GF_LOG_DEBUG, "Bad address");
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        ret = EFAULT;
+        goto out;
+    }
+
+    /* This will be more accurate */
+    inode_path(loc->inode, NULL, &pathbuf);
+    /* Check whether the file is present under eliminate paths or
+     * inside trash directory. In both cases we don't need to move the
+     * file to trash directory. Instead delete it permanently
+     */
+    match = check_whether_eliminate_path(priv->eliminate, pathbuf);
+    if ((strncmp(pathbuf, priv->newtrash_dir, strlen(priv->newtrash_dir)) ==
+         0) ||
+        (match)) {
+        if (match) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "%s is a file comes under an eliminate path, "
+                   "so it is not moved to trash",
+                   loc->name);
+        }
+
+        /* Trying to unlink from the trash-dir. So do the
+         * actual unlink without moving to trash-dir.
          */
-        append_time_stamp (local->newpath);
-        if (strlen (local->newpath) > PATH_MAX) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
-
-        /* To know whether CTR xlator requested for the link count */
-        ret = dict_get_int32 (xdata, GF_REQUEST_LINK_COUNT_XDATA,
-                              &ctr_link_req);
-        if (ret) {
-                local->ctr_link_count_req = _gf_false;
-                ret = 0;
-        } else
-                local->ctr_link_count_req = _gf_true;
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        TRASH_STACK_UNWIND(unlink, frame, -1, ENOMEM, NULL, NULL, xdata);
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+    loc_copy(&local->loc, loc);
+
+    /* rename new location of file as starting from trash directory */
+    copy_trash_path(priv->newtrash_dir, (frame->root->pid < 0), local->newpath,
+                    sizeof(local->newpath));
+    strncat(local->newpath, pathbuf,
+            sizeof(local->newpath) - strlen(local->newpath) - 1);
+
+    /* append timestamp to file name so that we can avoid
+     * name collisions inside trash
+     */
+    append_time_stamp(local->newpath, sizeof(local->newpath));
+    if (strlen(local->newpath) > PATH_MAX) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+
+    /* To know whether CTR xlator requested for the link count */
+    ret = dict_get_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, &ctr_link_req);
+    if (ret) {
+        local->ctr_link_count_req = _gf_false;
+        ret = 0;
+    } else
+        local->ctr_link_count_req = _gf_true;
 
-        LOCK_INIT (&frame->lock);
+    LOCK_INIT(&frame->lock);
 
-        STACK_WIND (frame, trash_unlink_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+    STACK_WIND(frame, trash_unlink_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 out:
-        return ret;
+    return ret;
 }
 
 /**
  * Use this when a failure occurs, and delete the newly created file
  */
 int32_t
-trash_truncate_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t *xdata)
+trash_truncate_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata)
 {
-        trash_local_t     *local  = NULL;
+    trash_local_t *local = NULL;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "deleting the newly created file: %s",
-                        strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG, "deleting the newly created file: %s",
+               strerror(op_errno));
+    }
 
-        STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    &local->loc, local->fop_offset, xdata);
+    STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, &local->loc,
+               local->fop_offset, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /**
  * Read from source file
  */
 int32_t
-trash_truncate_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iovec *vector, int32_t count,
-                          struct iatt *stbuf, struct iobref *iobuf,
-                          dict_t *xdata)
+trash_truncate_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                         int32_t count, struct iatt *stbuf,
+                         struct iobref *iobuf, dict_t *xdata)
 {
+    trash_local_t *local = NULL;
 
-        trash_local_t    *local  = NULL;
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "readv on the existing file failed: %s",
-                        strerror (op_errno));
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "readv on the existing file failed: %s", strerror(op_errno));
 
-                STACK_WIND (frame, trash_truncate_unlink_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                            &local->newloc, 0, xdata);
-                goto out;
-        }
+        STACK_WIND(frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->newloc, 0, xdata);
+        goto out;
+    }
 
-        local->fsize = stbuf->ia_size;
-        STACK_WIND (frame, trash_truncate_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    local->newfd, vector, count, local->cur_offset, 0, iobuf,
-                    xdata);
+    local->fsize = stbuf->ia_size;
+    STACK_WIND(frame, trash_truncate_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, local->newfd, vector, count,
+               local->cur_offset, 0, iobuf, xdata);
 
 out:
-        return 0;
-
+    return 0;
 }
 
 /**
  * Write to file created in trash directory
  */
 int32_t
-trash_truncate_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t *xdata)
+trash_truncate_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata)
 {
-        trash_local_t    *local  = NULL;
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        if (op_ret == -1) {
-                /* Let truncate work, but previous copy is not preserved. */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "writev on the existing file failed: %s",
-                        strerror (op_errno));
-
-                STACK_WIND (frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, &local->newloc, 0,
-                            xdata);
-                goto out;
-        }
-
-        if (local->cur_offset < local->fsize) {
-                local->cur_offset += GF_BLOCK_READV_SIZE;
-                /* Loop back and Read the contents again. */
-                STACK_WIND (frame, trash_truncate_readv_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
-                            local->fd, (size_t)GF_BLOCK_READV_SIZE,
-                            local->cur_offset, 0, xdata);
-                goto out;
-        }
-
-
-        /* OOFH.....Finally calling Truncate. */
-        STACK_WIND (frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, &local->loc,
-                    local->fop_offset, xdata);
+    trash_local_t *local = NULL;
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    if (op_ret == -1) {
+        /* Let truncate work, but previous copy is not preserved. */
+        gf_log(this->name, GF_LOG_DEBUG,
+               "writev on the existing file failed: %s", strerror(op_errno));
+
+        STACK_WIND(frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->newloc, 0, xdata);
+        goto out;
+    }
+
+    if (local->cur_offset < local->fsize) {
+        local->cur_offset += GF_BLOCK_READV_SIZE;
+        /* Loop back and Read the contents again. */
+        STACK_WIND(frame, trash_truncate_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, local->fd,
+                   (size_t)GF_BLOCK_READV_SIZE, local->cur_offset, 0, xdata);
+        goto out;
+    }
+
+    /* OOFH.....Finally calling Truncate. */
+    STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, &local->loc,
+               local->fop_offset, xdata);
 
 out:
-        return 0;
+    return 0;
 }
 
 /**
  * The source file is opened for reading and writing
  */
 int32_t
-trash_truncate_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, fd_t *fd,
-                         dict_t *xdata)
+trash_truncate_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata)
 {
-        trash_local_t    *local  = NULL;
+    trash_local_t *local = NULL;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
 
-        if (op_ret == -1) {
-                /* Let truncate work, but previous copy is not preserved. */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "open on the existing file failed: %s",
-                        strerror (op_errno));
+    if (op_ret == -1) {
+        /* Let truncate work, but previous copy is not preserved. */
+        gf_log(this->name, GF_LOG_DEBUG, "open on the existing file failed: %s",
+               strerror(op_errno));
 
-                STACK_WIND (frame, trash_truncate_unlink_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                            &local->newloc, 0, xdata);
-                goto out;
-        }
+        STACK_WIND(frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->newloc, 0, xdata);
+        goto out;
+    }
 
-        fd_bind (fd);
+    fd_bind(fd);
 
-        local->cur_offset = 0;
+    local->cur_offset = 0;
 
-        STACK_WIND (frame, trash_truncate_readv_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                    local->fd, (size_t)GF_BLOCK_READV_SIZE, local->cur_offset,
-                    0, xdata);
+    STACK_WIND(frame, trash_truncate_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, local->fd,
+               (size_t)GF_BLOCK_READV_SIZE, local->cur_offset, 0, xdata);
 
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -1279,99 +1524,104 @@ out:
  * if the path is present in trash directory
  */
 int32_t
-trash_truncate_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, fd_t *fd,
-                           inode_t *inode, struct iatt *buf,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t *xdata)
+trash_truncate_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, fd_t *fd,
+                          inode_t *inode, struct iatt *buf,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata)
 {
-        trash_local_t        *local                  = NULL;
-        char                 *tmp_str                = NULL;
-        char                 *dir_name               = NULL;
-        char                 *tmp_path               = NULL;
-        int32_t              flags                   = 0;
-        loc_t                tmp_loc                 = {0,};
-        char                 *tmp_stat               = NULL;
-        char                 real_path[PATH_MAX]     = {0,};
-        trash_private_t     *priv               = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        /* Checks whether path is present in trash directory or not */
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                /* Creating the directory structure here. */
-                tmp_str = gf_strdup (local->newpath);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-                dir_name = dirname (tmp_str);
-
-                tmp_path = gf_strdup (dir_name);
-                if (!tmp_path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-                loc_copy (&tmp_loc, &local->newloc);
-                tmp_loc.path = gf_strdup (tmp_path);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-
-                TRASH_SET_PID (frame, local);
-
-                /* create the directory with proper permissions */
-                STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk,
-                                   tmp_path, FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
+    trash_local_t *local = NULL;
+    char *tmp_str = NULL;
+    char *dir_name = NULL;
+    char *tmp_path = NULL;
+    int32_t flags = 0;
+    loc_t tmp_loc = {
+        0,
+    };
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    TRASH_UNSET_PID(frame, local);
+
+    /* Checks whether path is present in trash directory or not */
+
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        /* Creating the directory structure here. */
+        tmp_str = gf_strdup(local->newpath);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
         }
+        dir_name = dirname(tmp_str);
 
-        if (op_ret == -1) {
-                /* Let truncate work, but previous copy is not preserved.
-                 * Deleting the newly created copy.
-                 */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "creation of new file in trash-dir failed, "
-                        "when truncate was called: %s", strerror (op_errno));
-
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, &local->loc,
-                            local->fop_offset, xdata);
-                goto out;
+        tmp_path = gf_strdup(dir_name);
+        if (!tmp_path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
         }
+        loc_copy(&tmp_loc, &local->newloc);
+        tmp_loc.path = gf_strdup(tmp_path);
+        if (!tmp_loc.path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
+        remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
+        if (tmp_stat)
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        /* create the directory with proper permissions */
+        STACK_WIND_COOKIE(frame, trash_truncate_mkdir_cbk, tmp_path,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    if (op_ret == -1) {
+        /* Let truncate work, but previous copy is not preserved.
+         * Deleting the newly created copy.
+         */
+        gf_log(this->name, GF_LOG_DEBUG,
+               "creation of new file in trash-dir failed, "
+               "when truncate was called: %s",
+               strerror(op_errno));
 
-        fd_bind (fd);
-        flags = O_RDONLY;
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
 
-        /* fd which represents source file for reading and writing from it */
+    fd_bind(fd);
+    flags = O_RDONLY;
 
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
+    /* fd which represents source file for reading and writing from it */
 
-        STACK_WIND (frame, trash_truncate_open_cbk,  FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, &local->loc, flags,
-                    local->fd, 0);
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+
+    STACK_WIND(frame, trash_truncate_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, &local->loc, flags, local->fd, 0);
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (tmp_path)
-                GF_FREE (tmp_path);
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (tmp_path)
+        GF_FREE(tmp_path);
 
-        return 0;
+    return 0;
 }
 
 /**
@@ -1380,313 +1630,316 @@ out:
  * beginning
  */
 int32_t
-trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *stbuf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t *xdata)
+trash_truncate_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *stbuf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-        trash_local_t        *local              = NULL;
-        trash_private_t      *priv               = NULL;
-        char                 *tmp_str            = NULL;
-        char                 *tmp_path           = NULL;
-        char                 *tmp_dirname        = NULL;
-        char                 *dir_name           = NULL;
-        char                 *tmp_stat           = NULL;
-        char                 real_path[PATH_MAX] = {0,};
-        size_t               count               = 0;
-        int32_t              flags               = 0;
-        int32_t              loop_count          = 0;
-        int                  i                   = 0;
-        loc_t                tmp_loc             = {0,};
-        int                  ret                 = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        loop_count = local->loop_count;
-
-        TRASH_UNSET_PID (frame, local);
-
-        tmp_str = gf_strdup (local->newpath);
-        if (!tmp_str) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                tmp_dirname = strchr (tmp_str, '/');
-                while (tmp_dirname) {
-                        count = tmp_dirname - tmp_str;
-                        if (count == 0)
-                                count = 1;
-                        i++;
-                        if (i > loop_count)
-                                break;
-                        tmp_dirname = strchr (tmp_str + count + 1, '/');
-                }
-                tmp_path = gf_memdup (local->newpath, count + 1);
-                if (!tmp_path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                tmp_path[count] = '\0';
-
-                loc_copy (&tmp_loc, &local->newloc);
-                tmp_loc.path = gf_strdup (tmp_path);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                /* Stores the the name of directory to be created */
-                tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
-                if (!tmp_loc.name) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-
-                TRASH_SET_PID (frame, local);
-
-                STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk,
-                                   tmp_path, FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
-        }
-
-        if (op_ret == 0) {
-                dir_name = dirname (tmp_str);
-                if (strcmp ((char*)cookie, dir_name) == 0) {
-                        flags = O_CREAT|O_EXCL|O_WRONLY;
-                        strcpy (real_path, priv->brick_path);
-                        strcat (real_path, local->origpath);
-                        /* Call create again once directory structure
-                           is created. */
-                        STACK_WIND (frame, trash_truncate_create_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->create,
-                                    &local->newloc, flags,
-                                    get_permission (real_path),
-                                    0022, local->newfd, xdata);
-                        goto out;
-                }
-        }
-
-        if ((op_ret == -1) && (op_errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_ERROR, "Directory creation failed [%s]. "
-                                "Therefore truncating %s without moving the "
-                                "original copy to trash directory",
-                                strerror(op_errno), local->loc.name);
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, &local->loc,
-                            local->fop_offset, xdata);
-                goto out;
-        }
-
-        LOCK (&frame->lock);
-        {
-                loop_count = ++local->loop_count;
-        }
-        UNLOCK (&frame->lock);
-
-        tmp_dirname = strchr (tmp_str, '/');
+    trash_local_t *local = NULL;
+    trash_private_t *priv = NULL;
+    char *tmp_str = NULL;
+    char *tmp_path = NULL;
+    char *tmp_dirname = NULL;
+    char *dir_name = NULL;
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    size_t count = 0;
+    int32_t flags = 0;
+    int32_t loop_count = 0;
+    int i = 0;
+    loc_t tmp_loc = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    loop_count = local->loop_count;
+
+    TRASH_UNSET_PID(frame, local);
+
+    tmp_str = gf_strdup(local->newpath);
+    if (!tmp_str) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        tmp_dirname = strchr(tmp_str, '/');
         while (tmp_dirname) {
-                count = tmp_dirname - tmp_str;
-                if (count == 0)
-                        count = 1;
-                i++;
-                if (i > loop_count)
-                        break;
-                tmp_dirname = strchr (tmp_str + count + 1, '/');
-        }
-        tmp_path = gf_memdup (local->newpath, count + 1);
+            count = tmp_dirname - tmp_str;
+            if (count == 0)
+                count = 1;
+            i++;
+            if (i > loop_count)
+                break;
+            tmp_dirname = strchr(tmp_str + count + 1, '/');
+        }
+        tmp_path = gf_memdup(local->newpath, count + 1);
         if (!tmp_path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
         tmp_path[count] = '\0';
 
-        loc_copy (&tmp_loc, &local->newloc);
-        tmp_loc.path = gf_strdup (tmp_path);
+        loc_copy(&tmp_loc, &local->newloc);
+        tmp_loc.path = gf_strdup(tmp_path);
         if (!tmp_loc.path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
 
         /* Stores the the name of directory to be created */
-        tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
+        tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
         if (!tmp_loc.name) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
-
-        strcpy (real_path, priv->brick_path);
-        remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
+        remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
         if (tmp_stat)
-                strcat (real_path, tmp_stat);
-
-        TRASH_SET_PID (frame, local);
-
-        STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk, tmp_path,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->mkdir, &tmp_loc,
-                           get_permission(real_path),
-                           0022, xdata);
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        STACK_WIND_COOKIE(frame, trash_truncate_mkdir_cbk, tmp_path,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    if (op_ret == 0) {
+        dir_name = dirname(tmp_str);
+        if (strcmp((char *)cookie, dir_name) == 0) {
+            flags = O_CREAT | O_EXCL | O_WRONLY;
+            strncpy(real_path, priv->brick_path, sizeof(real_path));
+            real_path[sizeof(real_path) - 1] = 0;
+            strncat(real_path, local->origpath,
+                    sizeof(real_path) - strlen(real_path) - 1);
+            /* Call create again once directory structure
+               is created. */
+
+            TRASH_SET_PID(frame, local);
+
+            STACK_WIND(frame, trash_truncate_create_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->create, &local->newloc, flags,
+                       get_permission(real_path), 0022, local->newfd, xdata);
+            goto out;
+        }
+    }
+
+    if ((op_ret == -1) && (op_errno != EEXIST)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Directory creation failed [%s]. "
+               "Therefore truncating %s without moving the "
+               "original copy to trash directory",
+               strerror(op_errno), local->loc.name);
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    LOCK(&frame->lock);
+    {
+        loop_count = ++local->loop_count;
+    }
+    UNLOCK(&frame->lock);
+
+    tmp_dirname = strchr(tmp_str, '/');
+    while (tmp_dirname) {
+        count = tmp_dirname - tmp_str;
+        if (count == 0)
+            count = 1;
+        i++;
+        if (i > loop_count)
+            break;
+        tmp_dirname = strchr(tmp_str + count + 1, '/');
+    }
+    tmp_path = gf_memdup(local->newpath, count + 1);
+    if (!tmp_path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    tmp_path[count] = '\0';
+
+    loc_copy(&tmp_loc, &local->newloc);
+    tmp_loc.path = gf_strdup(tmp_path);
+    if (!tmp_loc.path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    /* Stores the the name of directory to be created */
+    tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
+    if (!tmp_loc.name) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    strncpy(real_path, priv->brick_path, sizeof(real_path));
+    real_path[sizeof(real_path) - 1] = 0;
+    remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
+    if (tmp_stat)
+        strncat(real_path, tmp_stat, sizeof(real_path) - strlen(real_path) - 1);
+
+    TRASH_SET_PID(frame, local);
+
+    STACK_WIND_COOKIE(frame, trash_truncate_mkdir_cbk, tmp_path,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                      &tmp_loc, get_permission(real_path), 0022, xdata);
 
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (tmp_path)
-                GF_FREE (tmp_path);
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (tmp_path)
+        GF_FREE(tmp_path);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-trash_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         dict_t *xdata)
+trash_truncate_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        dict_t *xdata)
 {
-        trash_private_t       *priv                   = NULL;
-        trash_local_t         *local                  = NULL;
-        char                  loc_newname[PATH_MAX]   = {0,};
-        int32_t               flags                   = 0;
-        dentry_t              *dir_entry              = NULL;
-        inode_table_t         *table                  = NULL;
-        int                   ret                     = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        table = local->loc.inode->table;
-
-        pthread_mutex_lock (&table->lock);
-        {
-                dir_entry = __dentry_search_arbit (local->loc.inode);
-        }
-        pthread_mutex_unlock (&table->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "fstat on the file failed: %s",
-                        strerror (op_errno));
-
-                TRASH_STACK_UNWIND (truncate, frame, op_ret, op_errno, buf,
-                                NULL, xdata);
-                goto out;
-        }
-
-        /* Only last hardlink will be moved to trash directory */
-        if (buf->ia_nlink > 1) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            &local->loc, local->fop_offset, xdata);
-                goto out;
-        }
-
-        /**
-         * If the file is too big or if it is extended truncate,
-         * just don't move it to trash directory.
-         */
-        if (buf->ia_size > (priv->max_trash_file_size) ||
-                                buf->ia_size <= local->fop_offset) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: not moving to trash , "
-                           "having inappropiate file size", local->loc.path);
-
-                STACK_WIND (frame,  trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            &local->loc, local->fop_offset, xdata);
-                goto out;
-        }
-
-        /* Retrives the name of file from path */
-        local->loc.name = gf_strdup (strrchr (local->loc.path, '/'));
-        if (!local->loc.name) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                goto out;
-        }
-
-        /* Stores new path for source file */
-        copy_trash_path (priv->newtrash_dir, (frame->root->pid < 0),
-                                                        local->newpath);
-        strcat (local->newpath, local->loc.path);
-
-        /* append timestamp to file name so that we can avoid
-           name collisions inside trash */
-        append_time_stamp (local->newpath);
-        if (strlen (local->newpath) > PATH_MAX) {
-                STACK_WIND (frame,  trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            &local->loc, local->fop_offset, xdata);
-                goto out;
-        }
-
-        strcpy (loc_newname, local->loc.name);
-        append_time_stamp (loc_newname);
-        /* local->newloc represents old file(file inside trash),
-           where as local->loc represents truncated file. We need
-           to create new inode and fd for new file*/
-        local->newloc.name = gf_strdup (loc_newname);
-        if (!local->newloc.name) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-        local->newloc.path = gf_strdup (local->newpath);
-        if (!local->newloc.path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-        local->newloc.inode = inode_new (local->loc.inode->table);
-        local->newfd = fd_create (local->newloc.inode, frame->root->pid);
-
-        /* Creating vaild parent and pargfids for both files */
-
-        if (dir_entry == NULL) {
-                ret = EINVAL;
-                goto out;
-        }
-        local->loc.parent = inode_ref (dir_entry->parent);
-        gf_uuid_copy (local->loc.pargfid, dir_entry->parent->gfid);
-
-        local->newloc.parent = inode_ref (dir_entry->parent);
-        gf_uuid_copy (local->newloc.pargfid, dir_entry->parent->gfid);
-
-        flags = O_CREAT|O_EXCL|O_WRONLY;
-
-        STACK_WIND (frame, trash_truncate_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    &local->newloc, flags,
-                    st_mode_from_ia (buf->ia_prot, local->loc.inode->ia_type),
-                    0022, local->newfd, xdata);
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
+    char loc_newname[PATH_MAX] = {
+        0,
+    };
+    int32_t flags = 0;
+    dentry_t *dir_entry = NULL;
+    inode_table_t *table = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    table = local->loc.inode->table;
+
+    pthread_mutex_lock(&table->lock);
+    {
+        dir_entry = __dentry_search_arbit(local->loc.inode);
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG, "fstat on the file failed: %s",
+               strerror(op_errno));
+
+        TRASH_STACK_UNWIND(truncate, frame, op_ret, op_errno, buf, NULL, xdata);
+        goto out;
+    }
+
+    /* Only last hardlink will be moved to trash directory */
+    if (buf->ia_nlink > 1) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    /**
+     * If the file is too big or if it is extended truncate,
+     * just don't move it to trash directory.
+     */
+    if (buf->ia_size > (priv->max_trash_file_size) ||
+        buf->ia_size <= local->fop_offset) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "%s: file is too large to move to trash", local->loc.path);
+
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    /* Retrieves the name of file from path */
+    local->loc.name = gf_strdup(strrchr(local->loc.path, '/'));
+    if (!local->loc.name) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    /* Stores new path for source file */
+    copy_trash_path(priv->newtrash_dir, (frame->root->pid < 0), local->newpath,
+                    sizeof(local->newpath));
+    strncat(local->newpath, local->loc.path,
+            sizeof(local->newpath) - strlen(local->newpath) - 1);
+
+    /* append timestamp to file name so that we can avoid
+       name collisions inside trash */
+    append_time_stamp(local->newpath, sizeof(local->newpath));
+    if (strlen(local->newpath) > PATH_MAX) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    strncpy(loc_newname, local->loc.name, sizeof(loc_newname));
+    loc_newname[sizeof(loc_newname) - 1] = 0;
+    append_time_stamp(loc_newname, sizeof(loc_newname));
+    /* local->newloc represents old file(file inside trash),
+       where as local->loc represents truncated file. We need
+       to create new inode and fd for new file*/
+    local->newloc.name = gf_strdup(loc_newname);
+    if (!local->newloc.name) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    local->newloc.path = gf_strdup(local->newpath);
+    if (!local->newloc.path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    local->newloc.inode = inode_new(local->loc.inode->table);
+    local->newfd = fd_create(local->newloc.inode, frame->root->pid);
+
+    /* Creating valid parent and pargfids for both files */
+
+    if (dir_entry == NULL) {
+        ret = EINVAL;
+        goto out;
+    }
+    local->loc.parent = inode_ref(dir_entry->parent);
+    gf_uuid_copy(local->loc.pargfid, dir_entry->parent->gfid);
+
+    local->newloc.parent = inode_ref(dir_entry->parent);
+    gf_uuid_copy(local->newloc.pargfid, dir_entry->parent->gfid);
+
+    flags = O_CREAT | O_EXCL | O_WRONLY;
+
+    TRASH_SET_PID(frame, local);
+
+    STACK_WIND(frame, trash_truncate_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, &local->newloc, flags,
+               st_mode_from_ia(buf->ia_prot, local->loc.inode->ia_type), 0022,
+               local->newfd, xdata);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1694,91 +1947,85 @@ out:
  * like text editors etc..
  */
 int32_t
-trash_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset, dict_t *xdata)
+trash_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        trash_private_t        *priv           = NULL;
-        trash_local_t          *local          = NULL;
-        int32_t                match           = 0;
-        char                   *pathbuf        = NULL;
-        int                    ret             = 0;
-
-        priv  = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-        /* If trash is not active or not enabled through cli, then
-         * we bypass and wind back
-         */
-        if (!priv->state) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc,
-                            offset, xdata);
-                goto out;
-        }
-
-        /* The files removed by gluster operations such as self-heal,
-           should moved to trash directory, but files by client should
-           not moved */
-        if ((frame->root->pid < 0) && !priv->internal) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc,
-                            offset, xdata);
-                goto out;
-        }
-        /* This will be more accurate */
-        inode_path(loc->inode, NULL, &pathbuf);
-
-        /* Checks whether file is in trash directory or eliminate path.
-         * In all such cases it does not move to trash directory,
-         * truncate will be performed
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
+    int32_t match = 0;
+    char *pathbuf = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+    /* If trash is not active or not enabled through cli, then
+     * we bypass and wind back
+     */
+    if (!priv->state) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        goto out;
+    }
+
+    /* The files removed by gluster operations such as self-heal,
+       should moved to trash directory, but files by client should
+       not moved */
+    if ((frame->root->pid < 0) && !priv->internal) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        goto out;
+    }
+    /* This will be more accurate */
+    inode_path(loc->inode, NULL, &pathbuf);
+
+    /* Checks whether file is in trash directory or eliminate path.
+     * In all such cases it does not move to trash directory,
+     * truncate will be performed
+     */
+    match = check_whether_eliminate_path(priv->eliminate, pathbuf);
+
+    if ((strncmp(pathbuf, priv->newtrash_dir, strlen(priv->newtrash_dir)) ==
+         0) ||
+        (match)) {
+        if (match) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "%s: file not moved to trash as per option "
+                   "'eliminate path'",
+                   loc->path);
+        }
+
+        /* Trying to truncate from the trash-dir. So do the
+         * actual truncate without moving to trash-dir.
          */
-        match = check_whether_eliminate_path (priv->eliminate, pathbuf);
-
-        if ((strncmp (pathbuf, priv->newtrash_dir,
-                      strlen (priv->newtrash_dir)) == 0) || (match)) {
-                if (match) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s: file not moved to trash as per option "
-                                "'eliminate path'", loc->path);
-                }
-
-                /* Trying to truncate from the trash-dir. So do the
-                 * actual truncate without moving to trash-dir.
-                 */
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc, offset,
-                            xdata);
-                goto out;
-        }
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        goto out;
+    }
 
-        LOCK_INIT (&frame->lock);
+    LOCK_INIT(&frame->lock);
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                TRASH_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL,
-                                    xdata);
-                ret = ENOMEM;
-                goto out;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        TRASH_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, xdata);
+        ret = ENOMEM;
+        goto out;
+    }
 
-        strcpy (local->origpath, pathbuf);
+    strncpy(local->origpath, pathbuf, sizeof(local->origpath));
+    local->origpath[sizeof(local->origpath) - 1] = 0;
 
-        loc_copy (&local->loc, loc);
-        local->loc.path = pathbuf;
-        local->fop_offset = offset;
+    loc_copy(&local->loc, loc);
+    local->loc.path = pathbuf;
+    local->fop_offset = offset;
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, trash_truncate_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc,
-                    xdata);
+    STACK_WIND(frame, trash_truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1787,97 +2034,91 @@ out:
  * other than that it also called by Rebalance operation
  */
 int32_t
-trash_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 dict_t *xdata)
+trash_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        trash_private_t      *priv           = NULL;
-        trash_local_t        *local          = NULL;/* file inside trash */
-        char                 *pathbuf        = NULL;/* path of file from fd */
-        int32_t              retval          = 0;
-        int32_t              match           = 0;
-        int                  ret             = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-        /* If trash is not active or not enabled through cli, then
-         * we bypass and wind back
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL; /* file inside trash */
+    char *pathbuf = NULL;        /* path of file from fd */
+    int32_t retval = 0;
+    int32_t match = 0;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+    /* If trash is not active or not enabled through cli, then
+     * we bypass and wind back
+     */
+    if (!priv->state) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        goto out;
+    }
+
+    /* The files removed by gluster operations such as self-heal,
+     * should moved to trash directory, but files by client
+     * should not moved
+     */
+    if ((frame->root->pid < 0) && !priv->internal) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        goto out;
+    }
+    /* This will be more accurate */
+    retval = inode_path(fd->inode, NULL, &pathbuf);
+
+    /* Checking  the eliminate path */
+
+    /* Checks whether file is trash directory or eliminate path or
+     * invalid fd. In all such cases it does not move to trash directory,
+     * ftruncate will be performed
+     */
+    match = check_whether_eliminate_path(priv->eliminate, pathbuf);
+    if ((strncmp(pathbuf, priv->newtrash_dir, strlen(priv->newtrash_dir)) ==
+         0) ||
+        match || !retval) {
+        if (match) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "%s: file matches eliminate path, "
+                   "not moved to trash",
+                   pathbuf);
+        }
+
+        /* Trying to ftruncate from the trash-dir. So do the
+         * actual ftruncate without moving to trash-dir
          */
-        if (!priv->state) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd,
-                            offset, xdata);
-                goto out;
-        }
-
-        /* The files removed by gluster operations such as self-heal,
-         * should moved to trash directory, but files by client
-         * should not moved
-         */
-        if ((frame->root->pid < 0) && !priv->internal) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd,
-                            offset, xdata);
-                goto out;
-        }
-        /* This will be more accurate */
-        retval = inode_path (fd->inode, NULL, &pathbuf);
-
-        /* Checking  the eliminate path */
-
-        /* Checks whether file is trash directory or eliminate path or
-         * invalid fd. In all such cases it does not move to trash directory,
-         * ftruncate will be performed
-         */
-        match = check_whether_eliminate_path (priv->eliminate, pathbuf);
-        if ((strncmp (pathbuf, priv->newtrash_dir,
-                      strlen (priv->newtrash_dir)) == 0) || match ||
-                      !retval) {
-
-                if (match) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s: file matches eliminate path, "
-                                "not moved to trash", pathbuf);
-                }
-
-                /* Trying to ftruncate from the trash-dir. So do the
-                 * actual ftruncate without moving to trash-dir
-                 */
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate,
-                            fd, offset, xdata);
-                goto out;
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                TRASH_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL,
-                                    NULL, xdata);
-                ret     = -1;
-                goto out;
-        }
-
-        strcpy (local->origpath, pathbuf);
-
-        /* To convert fd to location */
-        frame->local=local;
-
-        local->loc.path  = pathbuf;
-        local->loc.inode = inode_ref (fd->inode);
-        gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-
-        local->fop_offset = offset;
-
-        /* Else remains same to truncate code, so from here flow goes
-         * to truncate_stat
-         */
-        STACK_WIND (frame, trash_truncate_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        TRASH_STACK_UNWIND(ftruncate, frame, -1, ENOMEM, NULL, NULL, xdata);
+        ret = -1;
+        goto out;
+    }
+
+    strncpy(local->origpath, pathbuf, sizeof(local->origpath));
+    local->origpath[sizeof(local->origpath) - 1] = 0;
+
+    /* To convert fd to location */
+    frame->local = local;
+
+    local->loc.path = pathbuf;
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, local->loc.inode->gfid);
+
+    local->fop_offset = offset;
+
+    /* Else remains same to truncate code, so from here flow goes
+     * to truncate_stat
+     */
+    STACK_WIND(frame, trash_truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1885,31 +2126,32 @@ out:
  * trash directory in the mount by the user
  */
 int32_t
-trash_mkdir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+trash_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
-        int32_t               op_ret      = 0;
-        int32_t               op_errno    = 0;
-        trash_private_t       *priv       = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        if (!check_whether_trash_directory (loc->path, priv->newtrash_dir)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "mkdir issued on %s, which is not permitted",
-                        priv->newtrash_dir);
-                op_errno = EPERM;
-                op_ret = -1;
-
-                STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
-                             NULL, NULL, NULL, NULL, xdata);
-        } else {
-                STACK_WIND (frame, trash_common_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
-        }
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    if (!check_whether_op_permitted(priv, loc)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "mkdir issued on %s, which is not permitted",
+               priv->newtrash_dir);
+        op_errno = EPERM;
+        op_ret = -1;
+
+        STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, NULL, NULL, NULL,
+                            NULL, xdata);
+    } else {
+        STACK_WIND(frame, trash_common_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    }
+
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -1917,31 +2159,32 @@ out:
  * of trash directory in the mount by the user
  */
 int
-trash_rename (call_frame_t *frame, xlator_t *this,
-              loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+trash_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        int32_t               op_ret      = 0;
-        int32_t               op_errno    = 0;
-        trash_private_t       *priv       = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        if (!check_whether_trash_directory (oldloc->path, priv->newtrash_dir)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "rename issued on %s, which is not permitted",
-                        priv->newtrash_dir);
-                op_errno = EPERM;
-                op_ret = -1;
-
-                STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL, xdata);
-        } else {
-                STACK_WIND (frame, trash_common_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
-        }
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    if (!check_whether_op_permitted(priv, oldloc)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "rename issued on %s, which is not permitted",
+               priv->newtrash_dir);
+        op_errno = EPERM;
+        op_ret = -1;
+
+        STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, NULL, NULL, NULL,
+                            NULL, NULL, xdata);
+    } else {
+        STACK_WIND(frame, trash_common_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    }
+
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -1949,192 +2192,128 @@ out:
  * trash directory in the mount by the user
  */
 int32_t
-trash_rmdir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int flags, dict_t *xdata)
+trash_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            dict_t *xdata)
 {
-        int32_t               op_ret      = 0;
-        int32_t               op_errno    = 0;
-        trash_private_t       *priv       = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        if (!check_whether_trash_directory (loc->path, priv->newtrash_dir)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "rmdir issued on %s, which is not permitted",
-                        priv->newtrash_dir);
-                op_errno = EPERM;
-                op_ret = -1;
-
-                STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             NULL, NULL, xdata);
-        } else {
-                STACK_WIND (frame, trash_common_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
-        }
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    if (!check_whether_op_permitted(priv, loc)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "rmdir issued on %s, which is not permitted",
+               priv->newtrash_dir);
+        op_errno = EPERM;
+        op_ret = -1;
+
+        STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, NULL, NULL, xdata);
+    } else {
+        STACK_WIND(frame, trash_common_rmdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    }
+
 out:
-        return 0;
+    return 0;
 }
 
 /**
- * Volume set option is handled by the reconfigure funtion.
+ * Volume set option is handled by the reconfigure function.
  * Here we checks whether each option is set or not ,if it
  * sets then corresponding modifciations will be made
  */
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        uint64_t              max_fsize           = 0;
-        int                   ret                 = 0;
-        char                  *tmp                = NULL;
-        char                  *tmp_str            = NULL;
-        trash_private_t       *priv               = NULL;
-        loc_t                 old_loc             = {0, };
-        loc_t                 new_loc             = {0, };
-        call_frame_t          *frame              = NULL;
-        char                  trash_dir[PATH_MAX] = {0,};
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        GF_OPTION_RECONF ("trash", priv->state, options, bool, out);
-
-        GF_OPTION_RECONF ("trash-dir", tmp, options, str, out);
-        if (tmp) {
-                sprintf(trash_dir, "/%s/", tmp);
-                if (strcmp(priv->newtrash_dir, trash_dir) != 0) {
-
-                        /* When user set a new name for trash directory, trash
-                        * xlator will perform a rename operation on old trash
-                        * directory to the new one using a STACK_WIND from here.
-                        * This option can be configured only when volume is in
-                        * started state
-                        */
-
-                        GF_FREE (priv->newtrash_dir);
-
-                        priv->newtrash_dir = gf_strdup (trash_dir);
-                        if (!priv->newtrash_dir) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Renaming %s -> %s from reconfigure",
-                                priv->oldtrash_dir, priv->newtrash_dir);
-
-                        if (!priv->newtrash_dir) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        frame = create_frame (this, this->ctx->pool);
-                        if (frame == NULL) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to create frame");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-
-                        /* assign new location values to new_loc members */
-                        gf_uuid_copy (new_loc.gfid, trash_gfid);
-                        gf_uuid_copy (new_loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->newtrash_dir,
-                                                                &new_loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        new_loc.path = gf_strdup (priv->newtrash_dir);
-                        if (!new_loc.path) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-
-                        /* assign old location values to old_loc members */
-                        gf_uuid_copy (old_loc.gfid, trash_gfid);
-                        gf_uuid_copy (old_loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->oldtrash_dir,
-                                                                &old_loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        old_loc.path = gf_strdup (priv->oldtrash_dir);
-                        if (!old_loc.path) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-
-                        old_loc.inode = inode_ref (priv->trash_inode);
-                        gf_uuid_copy(old_loc.inode->gfid, old_loc.gfid);
-
-                        STACK_WIND (frame, trash_reconf_rename_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->rename,
-                                    &old_loc, &new_loc, options);
-                        GF_FREE (priv->oldtrash_dir);
-
-                        priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
-                        if (!priv->oldtrash_dir) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                }
-        }
-        tmp = NULL;
-
-        GF_OPTION_RECONF ("trash-internal-op", priv->internal, options,
-                                               bool, out);
-
-        GF_OPTION_RECONF ("trash-max-filesize", max_fsize, options,
-                                                size_uint64, out);
-        if (max_fsize) {
-                if (max_fsize > GF_ALLOWED_MAX_FILE_SIZE) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Size specified for max-size(in MB) is too "
-                                "large so using 1GB as max-size (NOT IDEAL)");
-                        priv->max_trash_file_size = GF_ALLOWED_MAX_FILE_SIZE;
-                } else
-                        priv->max_trash_file_size = max_fsize;
-                gf_log (this->name, GF_LOG_DEBUG, "%"GF_PRI_SIZET" max-size",
-                        priv->max_trash_file_size);
-        }
-        GF_OPTION_RECONF ("trash-eliminate-path", tmp, options, str, out);
-        if (!tmp) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no option specified for 'eliminate', using NULL");
-        } else {
-                if (priv->eliminate)
-                        wipe_eliminate_path (&priv->eliminate);
+    uint64_t max_fsize = 0;
+    int ret = 0;
+    char *tmp = NULL;
+    char *tmp_str = NULL;
+    trash_private_t *priv = NULL;
+    char trash_dir[PATH_MAX] = {
+        0,
+    };
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    GF_OPTION_RECONF("trash-internal-op", priv->internal, options, bool, out);
+    GF_OPTION_RECONF("trash-dir", tmp, options, str, out);
+
+    GF_OPTION_RECONF("trash", priv->state, options, bool, out);
 
-                tmp_str = gf_strdup (tmp);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                ret = store_eliminate_path (tmp_str, &priv->eliminate);
+    if (priv->state) {
+        ret = create_or_rename_trash_directory(this);
 
+        if (tmp)
+            sprintf(trash_dir, "/%s/", tmp);
+        else
+            sprintf(trash_dir, "%s", priv->oldtrash_dir);
+
+        if (strcmp(priv->newtrash_dir, trash_dir) != 0) {
+            /* When user set a new name for trash directory, trash
+             * xlator will perform a rename operation on old trash
+             * directory to the new one using a STACK_WIND from here.
+             * This option can be configured only when volume is in
+             * started state
+             */
+
+            GF_FREE(priv->newtrash_dir);
+
+            priv->newtrash_dir = gf_strdup(trash_dir);
+            if (!priv->newtrash_dir) {
+                ret = ENOMEM;
+                gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+                goto out;
+            }
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Renaming %s -> %s from reconfigure", priv->oldtrash_dir,
+                   priv->newtrash_dir);
+
+            if (!priv->newtrash_dir) {
+                gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+                ret = ENOMEM;
+                goto out;
+            }
+            ret = rename_trash_directory(this);
+        }
+
+        if (priv->internal) {
+            ret = create_internalop_directory(this);
+        }
+    }
+    tmp = NULL;
+
+    GF_OPTION_RECONF("trash-max-filesize", max_fsize, options, size_uint64,
+                     out);
+    if (max_fsize) {
+        priv->max_trash_file_size = max_fsize;
+        gf_log(this->name, GF_LOG_DEBUG, "%" GF_PRI_SIZET " max-size",
+               priv->max_trash_file_size);
+    }
+    GF_OPTION_RECONF("trash-eliminate-path", tmp, options, str, out);
+    if (!tmp) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "no option specified for 'eliminate', using NULL");
+    } else {
+        if (priv->eliminate)
+            wipe_eliminate_path(&priv->eliminate);
+
+        tmp_str = gf_strdup(tmp);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        ret = store_eliminate_path(tmp_str, &priv->eliminate);
+    }
 
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        loc_wipe (&new_loc);
-        loc_wipe (&old_loc);
 
-        return ret;
+    return ret;
 }
 
 /**
@@ -2142,424 +2321,333 @@ out:
  * using STACK_WIND only when posix xlator is up
  */
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        trash_private_t       *priv      = NULL;
-        dict_t                *dict      = NULL;
-        int                   ret        = 0;
-        uuid_t                *tgfid_ptr = NULL;
-        loc_t                 loc        = {0, };
-        loc_t                 old_loc    = {0, };
-        call_frame_t          *frame     = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        /* Check whether posix is up not */
-        if (event == GF_EVENT_CHILD_UP) {
-                frame = create_frame(this, this->ctx->pool);
-                if (frame == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to create frame");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                dict = dict_new ();
-                if (!dict) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                priv->trash_itable = inode_table_new (0, this);
-
-                /* Here there is two possiblities ,if trash directory already
-                 * exist ,then we need to perform a rename operation on the
-                 * old one. Otherwise, we need to create the trash directory
-                 * For both, we need to pass location variable, gfid of parent
-                 * and a frame for calling STACK_WIND.The location variable
-                 * requires name,path,gfid and inode
-                 */
-                if (!priv->oldtrash_dir) {
-                        loc.inode = inode_new (priv->trash_itable);
-                        gf_uuid_copy (loc.gfid, trash_gfid);
-
-                        gf_log (this->name, GF_LOG_DEBUG, "nameless lookup for"
-                                           "old trash directory");
-                        STACK_WIND (frame, trash_notify_lookup_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup,
-                                    &loc, dict);
-                        gf_log (this->name, GF_LOG_DEBUG, "old_trash_dir %s",
-                                        priv->oldtrash_dir);
-                        loc_wipe (&loc);
-                }
-
-                if (priv->oldtrash_dir == NULL) {
-                        ret = EINVAL;
-                        goto out;
-                }
-                if (strcmp (priv->oldtrash_dir, priv->newtrash_dir) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Creating trash "
-                                           "directory %s from notify",
-                                           priv->newtrash_dir);
-
-                        tgfid_ptr = GF_CALLOC (1, sizeof(uuid_t),
-                                                  gf_common_mt_uuid_t);
-                        if (!tgfid_ptr) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        gf_uuid_copy (*tgfid_ptr, trash_gfid);
-
-                        gf_uuid_copy (loc.gfid, trash_gfid);
-                        gf_uuid_copy (loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->newtrash_dir,
-                                                                &loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        loc.path = gf_strdup (priv->newtrash_dir);
-                        if (!loc.path) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-
-                        priv->trash_inode = inode_new (priv->trash_itable);
-                        priv->trash_inode->ia_type = IA_IFDIR;
-                        loc.inode = inode_ref (priv->trash_inode);
-
-                        /* Fixed gfid is set for trash directory with
-                         * this function
-                         */
-                        ret = dict_set_dynptr (dict, "gfid-req", tgfid_ptr,
-                                              sizeof (uuid_t));
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "setting key gfid-req failed");
-                                goto out;
-                        }
-
-                        /* The mkdir call for creating trash directory */
-                        STACK_WIND (frame, trash_notify_mkdir_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->mkdir, &loc, 0755,
-                                    0022, dict);
-                } else {
-                        /* assign new location values to new_loc members */
-                        gf_log (this->name, GF_LOG_DEBUG, "Renaming %s -> %s"
-                                        " from notify", priv->oldtrash_dir,
-                                        priv->newtrash_dir);
-                        gf_uuid_copy (loc.gfid, trash_gfid);
-                        gf_uuid_copy (loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->newtrash_dir,
-                                                                &loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        loc.path = gf_strdup (priv->newtrash_dir);
-                        if (!loc.path) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        /* assign old location values to old_loc members */
-                        gf_uuid_copy (old_loc.gfid, trash_gfid);
-                        gf_uuid_copy (old_loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->oldtrash_dir,
-                                                                &old_loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        old_loc.path = gf_strdup (priv->oldtrash_dir);
-                        if (!old_loc.path) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-
-                        old_loc.inode = inode_ref (priv->trash_inode);
-                        gf_uuid_copy(old_loc.inode->gfid, old_loc.gfid);
-
-                        STACK_WIND (frame, trash_notify_rename_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->rename,
-                                    &old_loc, &loc, dict);
-                        GF_FREE (priv->oldtrash_dir);
-
-                        priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
-                        if (!priv->oldtrash_dir) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                }
-        } else {
-                ret = default_notify (this, event, data);
-                if (ret)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "default notify event failed");
+    trash_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    /* Check whether posix is up not */
+    if (event == GF_EVENT_CHILD_UP) {
+        if (!priv->state) {
+            gf_log(this->name, GF_LOG_DEBUG, "trash xlator is off");
+            goto out;
         }
 
-out:
-        if (ret && tgfid_ptr)
-                GF_FREE (tgfid_ptr);
-        if (dict)
-                dict_unref (dict);
-        loc_wipe (&loc);
-        loc_wipe (&old_loc);
+        /* Here there is two possibilities ,if trash directory already
+         * exist ,then we need to perform a rename operation on the
+         * old one. Otherwise, we need to create the trash directory
+         * For both, we need to pass location variable, gfid of parent
+         * and a frame for calling STACK_WIND.The location variable
+         * requires name,path,gfid and inode
+         */
+        if (!priv->oldtrash_dir)
+            ret = create_or_rename_trash_directory(this);
+        else if (strcmp(priv->newtrash_dir, priv->oldtrash_dir) != 0)
+            ret = rename_trash_directory(this);
+        if (ret)
+            goto out;
 
-        return ret;
+        if (priv->internal)
+            (void)create_internalop_directory(this);
+    }
+
+out:
+    ret = default_notify(this, event, data);
+    if (ret)
+        gf_log(this->name, GF_LOG_INFO, "default notify event failed");
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int         ret          = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("trash", this, out);
+    GF_VALIDATE_OR_GOTO("trash", this, out);
 
-        ret = xlator_mem_acct_init (this, gf_trash_mt_end + 1);
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
-out:
+    ret = xlator_mem_acct_init(this, gf_trash_mt_end + 1);
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
         return ret;
+    }
+out:
+    return ret;
 }
 
 /**
  * trash_init
  */
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        trash_private_t        *priv                   = NULL;
-        int                    ret                     = -1;
-        char                   *tmp                    = NULL;
-        char                   *tmp_str                = NULL;
-        char                   trash_dir[PATH_MAX]     = {0,};
-        uint64_t               max_trash_file_size64   = 0;
-        data_t                *data                    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("trash", this, out);
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "not configured with exactly one child. exiting");
-                ret = -1;
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_trash_mt_trash_private_t);
-        if (!priv) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-
-        /* Trash priv data members are initialized through the following
-         * set of statements
-         */
-        GF_OPTION_INIT ("trash", priv->state, bool, out);
-
-        GF_OPTION_INIT ("trash-dir", tmp, str, out);
-
-        /* We store trash dir value as path for easier manipulation*/
-        if (!tmp) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no option specified for 'trash-dir', "
-                        "using \"/.trashcan/\"");
-                priv->newtrash_dir = gf_strdup ("/.trashcan/");
-                if (!priv->newtrash_dir) {
-                        ret = ENOMEM;
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-        } else {
-                sprintf(trash_dir, "/%s/", tmp);
-                priv->newtrash_dir = gf_strdup (trash_dir);
-                if (!priv->newtrash_dir) {
-                        ret = ENOMEM;
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-        }
-        tmp = NULL;
-
-        GF_OPTION_INIT ("trash-eliminate-path", tmp, str, out);
-        if (!tmp) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no option specified for 'eliminate', using NULL");
-        } else {
-                tmp_str = gf_strdup (tmp);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                        "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                ret = store_eliminate_path (tmp_str, &priv->eliminate);
-
-        }
-        tmp = NULL;
-
-        GF_OPTION_INIT ("trash-max-filesize", max_trash_file_size64,
-                        size_uint64, out);
-        if (!max_trash_file_size64) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "no option specified for 'max-trashable-file-size', "
-                        "using default = %lld MB",
-                        GF_DEFAULT_MAX_FILE_SIZE / GF_UNIT_MB);
-                priv->max_trash_file_size = GF_DEFAULT_MAX_FILE_SIZE;
-        } else {
-                if( max_trash_file_size64 > GF_ALLOWED_MAX_FILE_SIZE ) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Size specified for max-size(in MB) is too "
-                                "large so using 1GB as max-size (NOT IDEAL)");
-                        priv->max_trash_file_size = GF_ALLOWED_MAX_FILE_SIZE;
-                } else
-                        priv->max_trash_file_size = max_trash_file_size64;
-                gf_log (this->name, GF_LOG_DEBUG, "%"GF_PRI_SIZET" max-size",
-                        priv->max_trash_file_size);
-        }
-
-        GF_OPTION_INIT ("trash-internal-op", priv->internal, bool, out);
-
-        this->local_pool = mem_pool_new (trash_local_t, 64);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                ret = ENOMEM;
-                goto out;
-        }
-
-        /* For creating directories inside trash with proper permissions,
-         * we need to perform stat on that directories, for this we use
-         * brick path
-         */
-        data = dict_get (this->options, "brick-path");
-        if (!data) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "no option specified for 'brick-path'");
-                ret = ENOMEM;
-                goto out;
-        }
-        priv->brick_path = gf_strdup (data->data);
-        if (!priv->brick_path) {
-                ret = ENOMEM;
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "brick path is%s", priv->brick_path);
-
-        this->private = (void *)priv;
-        ret = 0;
+    trash_private_t *priv = NULL;
+    int ret = -1;
+    char *tmp = NULL;
+    char *tmp_str = NULL;
+    char trash_dir[PATH_MAX] = {
+        0,
+    };
+    uint64_t max_trash_file_size64 = 0;
+    data_t *data = NULL;
+
+    GF_VALIDATE_OR_GOTO("trash", this, out);
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "not configured with exactly one child. exiting");
+        ret = -1;
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_trash_mt_trash_private_t);
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    /* Trash priv data members are initialized through the following
+     * set of statements
+     */
+    GF_OPTION_INIT("trash", priv->state, bool, out);
+
+    GF_OPTION_INIT("trash-dir", tmp, str, out);
+
+    /* We store trash dir value as path for easier manipulation*/
+    if (!tmp) {
+        gf_log(this->name, GF_LOG_INFO,
+               "no option specified for 'trash-dir', "
+               "using \"/.trashcan/\"");
+        priv->newtrash_dir = gf_strdup("/.trashcan/");
+        if (!priv->newtrash_dir) {
+            ret = ENOMEM;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+    } else {
+        sprintf(trash_dir, "/%s/", tmp);
+        priv->newtrash_dir = gf_strdup(trash_dir);
+        if (!priv->newtrash_dir) {
+            ret = ENOMEM;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+    }
+    tmp = NULL;
+
+    GF_OPTION_INIT("trash-eliminate-path", tmp, str, out);
+    if (!tmp) {
+        gf_log(this->name, GF_LOG_INFO,
+               "no option specified for 'eliminate', using NULL");
+    } else {
+        tmp_str = gf_strdup(tmp);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
+        }
+        ret = store_eliminate_path(tmp_str, &priv->eliminate);
+    }
+    tmp = NULL;
+
+    GF_OPTION_INIT("trash-max-filesize", max_trash_file_size64, size_uint64,
+                   out);
+    if (!max_trash_file_size64) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "no option specified for 'max-trashable-file-size', "
+               "using default = %lld MB",
+               GF_DEFAULT_MAX_FILE_SIZE / GF_UNIT_MB);
+        priv->max_trash_file_size = GF_DEFAULT_MAX_FILE_SIZE;
+    } else {
+        priv->max_trash_file_size = max_trash_file_size64;
+        gf_log(this->name, GF_LOG_DEBUG, "%" GF_PRI_SIZET " max-size",
+               priv->max_trash_file_size);
+    }
+
+    GF_OPTION_INIT("trash-internal-op", priv->internal, bool, out);
+
+    this->local_pool = mem_pool_new(trash_local_t, 64);
+    if (!this->local_pool) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    /* For creating directories inside trash with proper permissions,
+     * we need to perform stat on that directories, for this we use
+     * brick path
+     */
+    data = dict_get(this->options, "brick-path");
+    if (!data) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "no option specified for 'brick-path'");
+        ret = ENOMEM;
+        goto out;
+    }
+    priv->brick_path = gf_strdup(data->data);
+    if (!priv->brick_path) {
+        ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    priv->trash_itable = inode_table_new(0, this);
+    gf_log(this->name, GF_LOG_DEBUG, "brick path is%s", priv->brick_path);
+
+    this->private = (void *)priv;
+    ret = 0;
 
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (ret) {
-                if (priv) {
-                        if (priv->newtrash_dir)
-                                GF_FREE (priv->newtrash_dir);
-                        if (priv->oldtrash_dir)
-                                GF_FREE (priv->oldtrash_dir);
-                        if (priv->brick_path)
-                                GF_FREE (priv->brick_path);
-                        if (priv->eliminate)
-                                wipe_eliminate_path (&priv->eliminate);
-                        GF_FREE (priv);
-                }
-                mem_pool_destroy (this->local_pool);
-        }
-        return ret;
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (ret) {
+        if (priv) {
+            if (priv->newtrash_dir)
+                GF_FREE(priv->newtrash_dir);
+            if (priv->oldtrash_dir)
+                GF_FREE(priv->oldtrash_dir);
+            if (priv->brick_path)
+                GF_FREE(priv->brick_path);
+            if (priv->eliminate)
+                wipe_eliminate_path(&priv->eliminate);
+            GF_FREE(priv);
+        }
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    return ret;
 }
 
 /**
  * trash_fini
  */
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        trash_private_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("trash", this, out);
-        priv = this->private;
-
-        if (priv) {
-                if (priv->newtrash_dir)
-                        GF_FREE (priv->newtrash_dir);
-                if (priv->oldtrash_dir)
-                        GF_FREE (priv->oldtrash_dir);
-                if (priv->brick_path)
-                        GF_FREE (priv->brick_path);
-                if (priv->eliminate)
-                        wipe_eliminate_path (&priv->eliminate);
-                GF_FREE (priv);
-        }
-        mem_pool_destroy (this->local_pool);
-        this->private = NULL;
+    trash_private_t *priv = NULL;
+    inode_table_t *inode_table = NULL;
+
+    GF_VALIDATE_OR_GOTO("trash", this, out);
+    priv = this->private;
+    if (priv) {
+        inode_table = priv->trash_itable;
+        if (priv->newtrash_dir) {
+            GF_FREE(priv->newtrash_dir);
+            priv->newtrash_dir = NULL;
+        }
+        if (priv->oldtrash_dir) {
+            GF_FREE(priv->oldtrash_dir);
+            priv->oldtrash_dir = NULL;
+        }
+        if (priv->brick_path) {
+            GF_FREE(priv->brick_path);
+            priv->brick_path = NULL;
+        }
+        if (priv->eliminate) {
+            wipe_eliminate_path(&priv->eliminate);
+            priv->eliminate = NULL;
+        }
+        if (inode_table) {
+            inode_table_destroy(inode_table);
+            priv->trash_itable = NULL;
+        }
+        GF_FREE(priv);
+    }
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    this->private = NULL;
 out:
-        return;
+    return;
 }
 
 struct xlator_fops fops = {
-        .unlink          = trash_unlink,
-        .truncate        = trash_truncate,
-        .ftruncate       = trash_ftruncate,
-        .rmdir           = trash_rmdir,
-        .mkdir           = trash_mkdir,
-        .rename          = trash_rename,
+    .unlink = trash_unlink,
+    .truncate = trash_truncate,
+    .ftruncate = trash_ftruncate,
+    .rmdir = trash_rmdir,
+    .mkdir = trash_mkdir,
+    .rename = trash_rename,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-        { .key           = { "trash" },
-          .type          = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description   = "Enable/disable trash translator",
-        },
-        { .key           = { "trash-dir" },
-          .type          = GF_OPTION_TYPE_STR,
-          .default_value = ".trashcan",
-          .description   = "Directory for trash files",
-        },
-        { .key           = { "trash-eliminate-path" },
-          .type          = GF_OPTION_TYPE_STR,
-          .description   = "Eliminate paths to be excluded "
-                           "from trashing",
-        },
-        { .key           = { "trash-max-filesize" },
-          .type          = GF_OPTION_TYPE_SIZET,
-          .default_value = "5MB",
-          .description   = "Maximum size of file that can be "
-                           "moved to trash",
-        },
-        { .key           = { "trash-internal-op" },
-          .type          = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description   = "Enable/disable trash translator for "
-                           "internal operations",
-        },
-        { .key           = {NULL} },
+    {
+        .key = {"trash"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "Enable/disable trash translator",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-dir"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = ".trashcan",
+        .description = "Directory for trash files",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-eliminate-path"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Eliminate paths to be excluded "
+                       "from trashing",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-max-filesize"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .default_value = "5MB",
+        .description = "Maximum size of file that can be "
+                       "moved to trash",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-internal-op"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "Enable/disable trash translator for "
+                       "internal operations",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {.key = {"brick-path"},
+     .type = GF_OPTION_TYPE_PATH,
+     .default_value = "{{ brick.path }}"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "trash",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/trash/src/trash.h b/xlators/features/trash/src/trash.h
index 088c1b9a286..6671617c2c6 100644
--- a/xlators/features/trash/src/trash.h
+++ b/xlators/features/trash/src/trash.h
@@ -10,89 +10,88 @@
 #ifndef __TRASH_H__
 #define __TRASH_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "inode.c"
 #include "fnmatch.h"
 
 #include <libgen.h>
 
 #ifndef GF_BLOCK_READV_SIZE
-#define GF_BLOCK_READV_SIZE      (128 * GF_UNIT_KB)
+#define GF_BLOCK_READV_SIZE (128 * GF_UNIT_KB)
 #endif
 
 #ifndef GF_DEFAULT_MAX_FILE_SIZE
 #define GF_DEFAULT_MAX_FILE_SIZE (200 * GF_UNIT_MB)
 #endif
 
-#ifndef GF_ALLOWED_MAX_FILE_SIZE
-#define GF_ALLOWED_MAX_FILE_SIZE (1 * GF_UNIT_GB)
-#endif
-
 struct trash_struct {
-        fd_t    *fd;         /* for the fd of existing file */
-        fd_t    *newfd;      /* for the newly created file */
-        loc_t    loc;        /* to store the location of the existing file */
-        loc_t    newloc;     /* to store the location for the new file */
-        size_t   fsize;      /* for keeping the size of existing file */
-        off_t    cur_offset; /* current offset for read and write ops */
-        off_t    fop_offset; /* original offset received with the fop */
-        pid_t    pid;
-        char     origpath[PATH_MAX];
-        char     newpath[PATH_MAX];
-        int32_t  loop_count;
-        gf_boolean_t is_set_pid;
-        struct iatt preparent;
-        struct iatt postparent;
-        gf_boolean_t ctr_link_count_req;
+    fd_t *fd;         /* for the fd of existing file */
+    fd_t *newfd;      /* for the newly created file */
+    loc_t loc;        /* to store the location of the existing file */
+    loc_t newloc;     /* to store the location for the new file */
+    size_t fsize;     /* for keeping the size of existing file */
+    off_t cur_offset; /* current offset for read and write ops */
+    off_t fop_offset; /* original offset received with the fop */
+    pid_t pid;
+    char origpath[PATH_MAX];
+    char newpath[PATH_MAX];
+    int32_t loop_count;
+    gf_boolean_t is_set_pid;
+    struct iatt preparent;
+    struct iatt postparent;
+    gf_boolean_t ctr_link_count_req;
 };
 typedef struct trash_struct trash_local_t;
 
 struct _trash_elim_path {
-        struct _trash_elim_path    *next;
-        char                       *path;
+    struct _trash_elim_path *next;
+    char *path;
 };
 typedef struct _trash_elim_path trash_elim_path;
 
 struct trash_priv {
-        char                 *oldtrash_dir;
-        char                 *newtrash_dir;
-        char                 *brick_path;
-        trash_elim_path      *eliminate;
-        size_t                max_trash_file_size;
-        gf_boolean_t          state;
-        gf_boolean_t          internal;
-        inode_t              *trash_inode;
-        inode_table_t        *trash_itable;
+    char *oldtrash_dir;
+    char *newtrash_dir;
+    char *brick_path;
+    trash_elim_path *eliminate;
+    size_t max_trash_file_size;
+    gf_boolean_t state;
+    gf_boolean_t internal;
+    inode_t *trash_inode;
+    inode_table_t *trash_itable;
 };
 typedef struct trash_priv trash_private_t;
 
-#define TRASH_SET_PID(frame, local) do {                        \
-                GF_ASSERT (!local->is_set_pid);                 \
-                if (!local->is_set_pid) {                       \
-                        local->pid = frame->root->pid;          \
-                        frame->root->pid = GF_SERVER_PID_TRASH; \
-                        local->is_set_pid = _gf_true;           \
-                }                                               \
-} while (0)
+#define TRASH_SET_PID(frame, local)                                            \
+    do {                                                                       \
+        GF_ASSERT(!local->is_set_pid);                                         \
+        if (!local->is_set_pid) {                                              \
+            local->pid = frame->root->pid;                                     \
+            frame->root->pid = GF_SERVER_PID_TRASH;                            \
+            local->is_set_pid = _gf_true;                                      \
+        }                                                                      \
+    } while (0)
 
-#define TRASH_UNSET_PID(frame, local) do {              \
-                GF_ASSERT (local->is_set_pid);          \
-                if (local->is_set_pid) {                \
-                        frame->root->pid = local->pid;  \
-                        local->is_set_pid = _gf_false;  \
-                }                                       \
-} while (0)
+#define TRASH_UNSET_PID(frame, local)                                          \
+    do {                                                                       \
+        GF_ASSERT(local->is_set_pid);                                          \
+        if (local->is_set_pid) {                                               \
+            frame->root->pid = local->pid;                                     \
+            local->is_set_pid = _gf_false;                                     \
+        }                                                                      \
+    } while (0)
 
-#define TRASH_STACK_UNWIND(op, frame, params ...) do {    \
-                trash_local_t *__local = NULL;            \
-                __local = frame->local;                   \
-                frame->local = NULL;                      \
-                STACK_UNWIND_STRICT (op, frame, params);  \
-                trash_local_wipe (__local);               \
-        } while (0)
+#define TRASH_STACK_UNWIND(op, frame, params...)                               \
+    do {                                                                       \
+        trash_local_t *__local = NULL;                                         \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(op, frame, params);                                \
+        trash_local_wipe(__local);                                             \
+    } while (0)
 
 #endif /* __TRASH_H__ */
diff --git a/xlators/features/upcall/src/Makefile.am b/xlators/features/upcall/src/Makefile.am
index 7f63e792281..72b7f55ae0a 100644
--- a/xlators/features/upcall/src/Makefile.am
+++ b/xlators/features/upcall/src/Makefile.am
@@ -1,4 +1,6 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = upcall.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 upcall_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
@@ -6,15 +8,15 @@ upcall_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 upcall_la_SOURCES = upcall.c upcall-internal.c
 
 upcall_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-        $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-        $(top_builddir)/rpc/xdr/src/libgfxdr.la
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
 noinst_HEADERS = upcall.h upcall-mem-types.h upcall-messages.h \
 	upcall-cache-invalidation.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-        -I$(top_srcdir)/rpc/rpc-lib/src \
-        -I$(top_srcdir)/rpc/xdr/src
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
diff --git a/xlators/features/upcall/src/upcall-cache-invalidation.h b/xlators/features/upcall/src/upcall-cache-invalidation.h
index 62b458fa295..db649b2c9a6 100644
--- a/xlators/features/upcall/src/upcall-cache-invalidation.h
+++ b/xlators/features/upcall/src/upcall-cache-invalidation.h
@@ -15,8 +15,4 @@
  * events post its last access */
 #define CACHE_INVALIDATION_TIMEOUT "60"
 
-/* xlator options */
-gf_boolean_t is_cache_invalidation_enabled(xlator_t *this);
-int32_t get_cache_invalidation_timeout(xlator_t *this);
-
 #endif /* __UPCALL_CACHE_INVALIDATION_H__ */
diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c
index f3c81aff15c..c641bd6f432 100644
--- a/xlators/features/upcall/src/upcall-internal.c
+++ b/xlators/features/upcall/src/upcall-internal.c
@@ -12,355 +12,287 @@
 #include <fcntl.h>
 #include <limits.h>
 
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
-#include "statedump.h"
-#include "syncop.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syncop.h>
 
 #include "upcall.h"
 #include "upcall-mem-types.h"
 #include "glusterfs3-xdr.h"
 #include "protocol-common.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 
 /*
  * Check if any of the upcall options are enabled:
  *     - cache_invalidation
  */
 gf_boolean_t
-is_upcall_enabled(xlator_t *this) {
-        upcall_private_t *priv      = NULL;
-        gf_boolean_t     is_enabled = _gf_false;
-
-        if (this->private) {
-                priv = (upcall_private_t *)this->private;
+is_upcall_enabled(xlator_t *this)
+{
+    upcall_private_t *priv = NULL;
 
-                if (priv->cache_invalidation_enabled) {
-                        is_enabled = _gf_true;
-                }
-        }
+    if (this->private) {
+        priv = (upcall_private_t *)this->private;
+        return priv->cache_invalidation_enabled;
+    }
 
-        return is_enabled;
+    return _gf_false;
 }
 
 /*
  * Get the cache_invalidation_timeout
  */
-int32_t
-get_cache_invalidation_timeout(xlator_t *this) {
-        upcall_private_t *priv      = NULL;
-        int32_t          timeout    = 0;
-
-        if (this->private) {
-                priv = (upcall_private_t *)this->private;
-                timeout = priv->cache_invalidation_timeout;
-        }
-
-        return timeout;
-}
-
-/*
- * Allocate and add a new client entry to the given upcall entry
- */
-upcall_client_t*
-add_upcall_client (call_frame_t *frame, client_t *client,
-                   upcall_inode_ctx_t *up_inode_ctx)
+static int32_t
+get_cache_invalidation_timeout(xlator_t *this)
 {
-        upcall_client_t *up_client_entry = NULL;
+    upcall_private_t *priv = NULL;
 
-        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
-        {
-                up_client_entry = __add_upcall_client (frame,
-                                                       client,
-                                                       up_inode_ctx);
-        }
-        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+    if (this->private) {
+        priv = (upcall_private_t *)this->private;
+        return priv->cache_invalidation_timeout;
+    }
 
-        return up_client_entry;
+    return 0;
 }
 
-upcall_client_t*
-__add_upcall_client (call_frame_t *frame, client_t *client,
-                     upcall_inode_ctx_t *up_inode_ctx)
+static upcall_client_t *
+__add_upcall_client(call_frame_t *frame, client_t *client,
+                    upcall_inode_ctx_t *up_inode_ctx, time_t now)
 {
-        upcall_client_t *up_client_entry = NULL;
-
-        up_client_entry = GF_CALLOC (1, sizeof(*up_client_entry),
-                                     gf_upcall_mt_upcall_client_entry_t);
-        if (!up_client_entry) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_NO_MEMORY,
-                        "Memory allocation failed");
-                return NULL;
-        }
-        INIT_LIST_HEAD (&up_client_entry->client_list);
-        up_client_entry->client_uid = gf_strdup(client->client_uid);
-        up_client_entry->access_time = time(NULL);
-        up_client_entry->expire_time_attr =
-                        get_cache_invalidation_timeout(frame->this);
+    upcall_client_t *up_client_entry = GF_MALLOC(
+        sizeof(*up_client_entry), gf_upcall_mt_upcall_client_entry_t);
+    if (!up_client_entry) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return NULL;
+    }
+    INIT_LIST_HEAD(&up_client_entry->client_list);
+    up_client_entry->client_uid = gf_strdup(client->client_uid);
+    up_client_entry->access_time = now;
+    up_client_entry->expire_time_attr = get_cache_invalidation_timeout(
+        frame->this);
 
-        list_add_tail (&up_client_entry->client_list,
-                       &up_inode_ctx->client_list);
+    list_add_tail(&up_client_entry->client_list, &up_inode_ctx->client_list);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "upcall_entry_t client added - %s",
-                up_client_entry->client_uid);
+    gf_log(THIS->name, GF_LOG_DEBUG, "upcall_entry_t client added - %s",
+           up_client_entry->client_uid);
 
-        return up_client_entry;
+    return up_client_entry;
 }
 
-/*
- * Given client->uid, retrieve the corresponding upcall client entry.
- * If none found, create a new entry.
- */
-upcall_client_t*
-__get_upcall_client (call_frame_t *frame, client_t *client,
-                     upcall_inode_ctx_t *up_inode_ctx)
+static int
+__upcall_inode_ctx_set(inode_t *inode, xlator_t *this)
 {
-        upcall_client_t *up_client_entry = NULL;
-        upcall_client_t *up_client       = NULL;
-        upcall_client_t *tmp             = NULL;
-        gf_boolean_t    found_client     = _gf_false;
-
-        list_for_each_entry_safe (up_client_entry, tmp,
-                                  &up_inode_ctx->client_list,
-                                  client_list) {
-                if (strcmp(client->client_uid,
-                           up_client_entry->client_uid) == 0) {
-                        /* found client entry. Update the access_time */
-                        up_client_entry->access_time = time(NULL);
-                        found_client = _gf_true;
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "upcall_entry_t client found - %s",
-                                up_client_entry->client_uid);
-                        break;
-                }
-        }
-
-        if (!found_client) { /* create one */
-                up_client_entry = __add_upcall_client (frame, client,
-                                                       up_inode_ctx);
-        }
-
-        return up_client_entry;
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    upcall_private_t *priv = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = __inode_ctx_get(inode, this, &ctx);
+
+    if (!ret)
+        goto out;
+
+    inode_ctx = GF_MALLOC(sizeof(upcall_inode_ctx_t),
+                          gf_upcall_mt_upcall_inode_ctx_t);
+
+    if (!inode_ctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    pthread_mutex_init(&inode_ctx->client_list_lock, NULL);
+    INIT_LIST_HEAD(&inode_ctx->inode_ctx_list);
+    INIT_LIST_HEAD(&inode_ctx->client_list);
+    inode_ctx->destroy = 0;
+    gf_uuid_copy(inode_ctx->gfid, inode->gfid);
+
+    ctx = (long)inode_ctx;
+    ret = __inode_ctx_set(inode, this, &ctx);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "failed to set inode ctx (%p)", inode);
+        GF_FREE(inode_ctx);
+        goto out;
+    }
+
+    /* add this inode_ctx to the global list */
+    LOCK(&priv->inode_ctx_lk);
+    {
+        list_add_tail(&inode_ctx->inode_ctx_list, &priv->inode_ctx_list);
+    }
+    UNLOCK(&priv->inode_ctx_lk);
+out:
+    return ret;
 }
 
-int
-__upcall_inode_ctx_set (inode_t *inode, xlator_t *this)
+static upcall_inode_ctx_t *
+__upcall_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-        upcall_inode_ctx_t *inode_ctx   = NULL;
-        upcall_private_t   *priv        = NULL;
-        int                ret          = -1;
-        uint64_t           ctx          = 0;
-
-        priv = this->private;
-        GF_ASSERT(priv);
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        ret = __inode_ctx_get (inode, this, &ctx);
+    ret = __inode_ctx_get(inode, this, &ctx);
 
-        if (!ret)
-                goto out;
+    if (ret < 0) {
+        ret = __upcall_inode_ctx_set(inode, this);
+        if (ret < 0)
+            goto out;
 
-        inode_ctx = GF_CALLOC (1, sizeof (upcall_inode_ctx_t),
-                               gf_upcall_mt_upcall_inode_ctx_t);
+        ret = __inode_ctx_get(inode, this, &ctx);
+        if (ret < 0)
+            goto out;
+    }
 
-        if (!inode_ctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    inode_ctx = (upcall_inode_ctx_t *)(long)(ctx);
 
-        pthread_mutex_init (&inode_ctx->client_list_lock, NULL);
-        INIT_LIST_HEAD (&inode_ctx->inode_ctx_list);
-        INIT_LIST_HEAD (&inode_ctx->client_list);
-        inode_ctx->destroy = 0;
-        gf_uuid_copy (inode_ctx->gfid, inode->gfid);
-
-        ctx = (long) inode_ctx;
-        ret = __inode_ctx_set (inode, this, &ctx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set inode ctx (%p)", inode);
-                goto out;
-        }
-
-        /* add this inode_ctx to the global list */
-        LOCK (&priv->inode_ctx_lk);
-        {
-                list_add_tail (&inode_ctx->inode_ctx_list,
-                               &priv->inode_ctx_list);
-        }
-        UNLOCK (&priv->inode_ctx_lk);
 out:
-        return ret;
+    return inode_ctx;
 }
 
 upcall_inode_ctx_t *
-__upcall_inode_ctx_get (inode_t *inode, xlator_t *this)
+upcall_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-        upcall_inode_ctx_t *inode_ctx   = NULL;
-        uint64_t           ctx          = 0;
-        int                ret          = 0;
+    upcall_inode_ctx_t *inode_ctx = NULL;
 
-        ret = __inode_ctx_get (inode, this, &ctx);
+    LOCK(&inode->lock);
+    {
+        inode_ctx = __upcall_inode_ctx_get(inode, this);
+    }
+    UNLOCK(&inode->lock);
 
-        if (ret < 0) {
-                ret = __upcall_inode_ctx_set (inode, this);
-                if (ret < 0)
-                        goto out;
-
-                ret = __inode_ctx_get (inode, this, &ctx);
-                if (ret < 0)
-                        goto out;
-        }
-
-        inode_ctx = (upcall_inode_ctx_t *) (long) (ctx);
-
-out:
-        return inode_ctx;
+    return inode_ctx;
 }
 
-upcall_inode_ctx_t *
-upcall_inode_ctx_get (inode_t *inode, xlator_t *this)
+static int
+__upcall_cleanup_client_entry(upcall_client_t *up_client)
 {
-        upcall_inode_ctx_t *inode_ctx = NULL;
+    list_del_init(&up_client->client_list);
 
-        LOCK (&inode->lock);
-        {
-                inode_ctx = __upcall_inode_ctx_get (inode, this);
-        }
-        UNLOCK (&inode->lock);
+    GF_FREE(up_client->client_uid);
+    GF_FREE(up_client);
 
-        return inode_ctx;
+    return 0;
 }
 
-int
-upcall_cleanup_expired_clients (xlator_t *this,
-                                upcall_inode_ctx_t *up_inode_ctx) {
+static int
+upcall_cleanup_expired_clients(xlator_t *this, upcall_inode_ctx_t *up_inode_ctx,
+                               time_t now)
+{
+    upcall_client_t *up_client = NULL;
+    upcall_client_t *tmp = NULL;
+    int ret = -1;
+    time_t timeout = 0;
+    time_t t_expired = 0;
+
+    timeout = get_cache_invalidation_timeout(this);
+
+    pthread_mutex_lock(&up_inode_ctx->client_list_lock);
+    {
+        list_for_each_entry_safe(up_client, tmp, &up_inode_ctx->client_list,
+                                 client_list)
+        {
+            t_expired = now - up_client->access_time;
 
-        upcall_client_t *up_client      = NULL;
-        upcall_client_t *tmp            = NULL;
-        int             ret             = -1;
-        time_t          timeout         = 0;
-        time_t          t_expired       = 0;
+            if (t_expired > (2 * timeout)) {
+                gf_log(THIS->name, GF_LOG_TRACE, "Cleaning up client_entry(%s)",
+                       up_client->client_uid);
 
-        timeout = get_cache_invalidation_timeout(this);
+                ret = __upcall_cleanup_client_entry(up_client);
 
-        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
-        {
-                list_for_each_entry_safe (up_client,
-                                          tmp,
-                                          &up_inode_ctx->client_list,
-                                          client_list) {
-                        t_expired = time(NULL) -
-                                         up_client->access_time;
-
-                        if (t_expired > (2*timeout)) {
-                                ret =
-                                  __upcall_cleanup_client_entry (up_client);
-
-                                if (ret) {
-                                        gf_msg ("upcall", GF_LOG_WARNING, 0,
-                                                UPCALL_MSG_INTERNAL_ERROR,
-                                                "Client entry cleanup failed (%p)",
-                                                up_client);
-                                        goto out;
-                                }
-                                gf_log (THIS->name, GF_LOG_TRACE,
-                                        "Cleaned up client_entry(%s)",
-                                        up_client->client_uid);
-                        }
+                if (ret) {
+                    gf_msg("upcall", GF_LOG_WARNING, 0,
+                           UPCALL_MSG_INTERNAL_ERROR,
+                           "Client entry cleanup failed (%p)", up_client);
+                    goto out;
                 }
+            }
         }
-        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+    }
+    pthread_mutex_unlock(&up_inode_ctx->client_list_lock);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
-}
-
-int
-__upcall_cleanup_client_entry (upcall_client_t *up_client)
-{
-        list_del_init (&up_client->client_list);
-
-        GF_FREE (up_client->client_uid);
-        GF_FREE (up_client);
-
-        return 0;
+    return ret;
 }
 
 /*
  * Free Upcall inode_ctx client list
  */
 int
-__upcall_cleanup_inode_ctx_client_list (upcall_inode_ctx_t *inode_ctx)
+__upcall_cleanup_inode_ctx_client_list(upcall_inode_ctx_t *inode_ctx)
 {
-        upcall_client_t *up_client        = NULL;
-        upcall_client_t *tmp              = NULL;
+    upcall_client_t *up_client = NULL;
+    upcall_client_t *tmp = NULL;
 
-        list_for_each_entry_safe (up_client, tmp,
-                                  &inode_ctx->client_list,
-                                  client_list) {
-                __upcall_cleanup_client_entry (up_client);
-        }
+    list_for_each_entry_safe(up_client, tmp, &inode_ctx->client_list,
+                             client_list)
+    {
+        __upcall_cleanup_client_entry(up_client);
+    }
 
-        return 0;
+    return 0;
 }
 
+static void
+upcall_cache_forget(xlator_t *this, inode_t *inode,
+                    upcall_inode_ctx_t *up_inode_ctx);
+
 /*
  * Free upcall_inode_ctx
  */
 int
-upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode)
+upcall_cleanup_inode_ctx(xlator_t *this, inode_t *inode)
 {
-        uint64_t           ctx          = 0;
-        upcall_inode_ctx_t *inode_ctx   = NULL;
-        int                ret          = 0;
-        upcall_private_t   *priv        = NULL;
-
-        priv = this->private;
-        GF_ASSERT(priv);
-
-        ret = inode_ctx_del (inode, this, &ctx);
-
-        if (ret < 0) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_INTERNAL_ERROR,
-                        "Failed to del upcall_inode_ctx (%p)",
-                        inode);
-                goto out;
-        }
+    uint64_t ctx = 0;
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    int ret = 0;
+    upcall_private_t *priv = NULL;
 
-        inode_ctx = (upcall_inode_ctx_t *)(long) ctx;
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (inode_ctx) {
+    ret = inode_ctx_del(inode, this, &ctx);
 
-                /* Invalidate all the upcall cache entries */
-                upcall_cache_forget (this, inode, inode_ctx);
+    if (ret < 0) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+               "Failed to del upcall_inode_ctx (%p)", inode);
+        goto out;
+    }
 
-                /* do we really need lock? yes now reaper thread
-                 * may also be trying to cleanup the client entries.
-                 */
-                pthread_mutex_lock (&inode_ctx->client_list_lock);
-                {
-                        if (!list_empty (&inode_ctx->client_list)) {
-                                __upcall_cleanup_inode_ctx_client_list (inode_ctx);
-                        }
-                }
-                pthread_mutex_unlock (&inode_ctx->client_list_lock);
+    inode_ctx = (upcall_inode_ctx_t *)(long)ctx;
 
-                /* Mark the inode_ctx to be destroyed */
-                inode_ctx->destroy = 1;
-                gf_msg_debug ("upcall", 0, "set upcall_inode_ctx (%p) to destroy mode",
-                              inode_ctx);
+    if (inode_ctx) {
+        /* Invalidate all the upcall cache entries */
+        upcall_cache_forget(this, inode, inode_ctx);
+
+        /* do we really need lock? yes now reaper thread
+         * may also be trying to cleanup the client entries.
+         */
+        pthread_mutex_lock(&inode_ctx->client_list_lock);
+        {
+            if (!list_empty(&inode_ctx->client_list)) {
+                __upcall_cleanup_inode_ctx_client_list(inode_ctx);
+            }
         }
+        pthread_mutex_unlock(&inode_ctx->client_list_lock);
+
+        /* Mark the inode_ctx to be destroyed */
+        inode_ctx->destroy = 1;
+        gf_msg_debug("upcall", 0, "set upcall_inode_ctx (%p) to destroy mode",
+                     inode_ctx);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -369,80 +301,161 @@ out:
  * which is no longer valid and has destroy bit set.
  */
 void *
-upcall_reaper_thread (void *data)
+upcall_reaper_thread(void *data)
 {
-        upcall_private_t *priv          = NULL;
-        upcall_inode_ctx_t *inode_ctx   = NULL;
-        upcall_inode_ctx_t *tmp         = NULL;
-        xlator_t           *this        = NULL;
-        time_t              timeout     = 0;
-
-        this = (xlator_t *)data;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-
-        while (!priv->fini) {
-                list_for_each_entry_safe (inode_ctx, tmp,
-                                          &priv->inode_ctx_list,
-                                          inode_ctx_list) {
-
-                        /* cleanup expired clients */
-                        upcall_cleanup_expired_clients (this, inode_ctx);
-
-                        if (!inode_ctx->destroy) {
-                                continue;
-                        }
-
-                        LOCK (&priv->inode_ctx_lk);
-                        {
-                                /* client list would have been cleaned up*/
-                                gf_msg_debug ("upcall", 0, "Freeing upcall_inode_ctx (%p)",
-                                              inode_ctx);
-                                list_del_init (&inode_ctx->inode_ctx_list);
-                                pthread_mutex_destroy (&inode_ctx->client_list_lock);
-                                GF_FREE (inode_ctx);
-                                inode_ctx = NULL;
-                        }
-                        UNLOCK (&priv->inode_ctx_lk);
-                }
-
-                /* don't do a very busy loop */
-                timeout = get_cache_invalidation_timeout (this);
-                sleep (timeout / 2);
+    upcall_private_t *priv = NULL;
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    upcall_inode_ctx_t *tmp = NULL;
+    xlator_t *this = NULL;
+    time_t timeout = 0;
+    time_t time_now;
+
+    this = (xlator_t *)data;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    time_now = gf_time();
+    while (!priv->fini) {
+        list_for_each_entry_safe(inode_ctx, tmp, &priv->inode_ctx_list,
+                                 inode_ctx_list)
+        {
+            /* cleanup expired clients */
+            upcall_cleanup_expired_clients(this, inode_ctx, time_now);
+
+            if (!inode_ctx->destroy) {
+                continue;
+            }
+
+            /* client list would have been cleaned up*/
+            gf_msg_debug("upcall", 0, "Freeing upcall_inode_ctx (%p)",
+                         inode_ctx);
+            LOCK(&priv->inode_ctx_lk);
+            {
+                list_del_init(&inode_ctx->inode_ctx_list);
+                pthread_mutex_destroy(&inode_ctx->client_list_lock);
+            }
+            UNLOCK(&priv->inode_ctx_lk);
+            GF_FREE(inode_ctx);
+            inode_ctx = NULL;
         }
 
-        return NULL;
+        /* don't do a very busy loop */
+        timeout = get_cache_invalidation_timeout(this);
+        sleep(timeout / 2);
+        time_now = gf_time();
+    }
+
+    return NULL;
 }
 
 /*
  * Initialize upcall reaper thread.
  */
 int
-upcall_reaper_thread_init (xlator_t *this)
+upcall_reaper_thread_init(xlator_t *this)
 {
-        upcall_private_t *priv  = NULL;
-        int              ret    = -1;
+    upcall_private_t *priv = NULL;
+    int ret = -1;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        ret = pthread_create (&priv->reaper_thr, NULL,
-                              upcall_reaper_thread, this);
+    ret = gf_thread_create(&priv->reaper_thr, NULL, upcall_reaper_thread, this,
+                           "upreaper");
 
-        return ret;
+    return ret;
 }
 
 int
-up_filter_virtual_xattr (dict_t *d, char *k, data_t *v, void *tmp)
+up_compare_afr_xattr(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        if (is_virtual_xattr (k) == _gf_true) {
-                dict_del (d, k);
-        }
+    dict_t *dict = tmp;
+
+    if (!strncmp(k, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)) &&
+        (!is_data_equal(v, dict_get(dict, k))))
+        return -1;
+
+    return 0;
+}
+
+static void
+up_filter_afr_xattr(dict_t *xattrs, char *xattr, data_t *v)
+{
+    /* Filter the afr pending xattrs, with value 0. Ideally this should
+     * be executed only in case of xattrop and not in set and removexattr,
+     * butset and remove xattr fops do not come with keys AFR_XATTR_PREFIX
+     */
+    if (!strncmp(xattr, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)) &&
+        (mem_0filled(v->data, v->len) == 0)) {
+        dict_del(xattrs, xattr);
+    }
+    return;
+}
+
+static gf_boolean_t
+up_key_is_regd_xattr(dict_t *regd_xattrs, char *regd_xattr, data_t *v,
+                     void *xattr)
+{
+    int ret = _gf_false;
+    char *key = xattr;
+
+    if (fnmatch(regd_xattr, key, 0) == 0)
+        ret = _gf_true;
 
-        return 0;
+    return ret;
+}
+
+int
+up_filter_unregd_xattr(dict_t *xattrs, char *xattr, data_t *v,
+                       void *regd_xattrs)
+{
+    int ret = 0;
+
+    ret = dict_foreach_match(regd_xattrs, up_key_is_regd_xattr, xattr,
+                             dict_null_foreach_fn, NULL);
+    if (ret == 0) {
+        /* xattr was not found in the registered xattr, hence do not
+         * send notification for its change
+         */
+        dict_del(xattrs, xattr);
+        goto out;
+    }
+    up_filter_afr_xattr(xattrs, xattr, v);
+out:
+    return 0;
+}
+
+int
+up_filter_xattr(dict_t *xattr, dict_t *regd_xattrs)
+{
+    int ret = 0;
+
+    ret = dict_foreach(xattr, up_filter_unregd_xattr, regd_xattrs);
+
+    return ret;
+}
+
+static void
+upcall_client_cache_invalidate(xlator_t *this, uuid_t gfid,
+                               upcall_client_t *up_client_entry, uint32_t flags,
+                               struct iatt *stbuf, struct iatt *p_stbuf,
+                               struct iatt *oldp_stbuf, dict_t *xattr,
+                               time_t now);
+
+gf_boolean_t
+up_invalidate_needed(dict_t *xattrs)
+{
+    if (dict_key_count(xattrs) == 0) {
+        gf_msg_trace("upcall", 0,
+                     "None of xattrs requested for"
+                     " invalidation, were changed. Nothing to "
+                     "invalidate");
+        return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 /*
@@ -456,207 +469,221 @@ up_filter_virtual_xattr (dict_t *d, char *k, data_t *v, void *tmp)
  * any errors during the process are logged and ignored.
  */
 void
-upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
-                         inode_t *inode, uint32_t flags, struct iatt *stbuf,
-                         struct iatt *p_stbuf, struct iatt *oldp_stbuf,
-                         dict_t *xattr)
+upcall_cache_invalidate(call_frame_t *frame, xlator_t *this, client_t *client,
+                        inode_t *inode, uint32_t flags, struct iatt *stbuf,
+                        struct iatt *p_stbuf, struct iatt *oldp_stbuf,
+                        dict_t *xattr)
 {
-        upcall_client_t *up_client       = NULL;
-        upcall_client_t *up_client_entry = NULL;
-        upcall_client_t *tmp             = NULL;
-        upcall_inode_ctx_t *up_inode_ctx = NULL;
-        gf_boolean_t     found           = _gf_false;
-
-        if (!is_upcall_enabled(this))
-                return;
+    upcall_client_t *up_client_entry = NULL;
+    upcall_client_t *tmp = NULL;
+    upcall_inode_ctx_t *up_inode_ctx = NULL;
+    gf_boolean_t found = _gf_false;
+    time_t time_now;
+    inode_t *linked_inode = NULL;
+
+    if (!is_upcall_enabled(this))
+        return;
 
-        /* server-side generated fops like quota/marker will not have any
-         * client associated with them. Ignore such fops.
-         */
-        if (!client) {
-                gf_msg_debug ("upcall", 0, "Internal fop - client NULL");
-                return;
+    /* server-side generated fops like quota/marker will not have any
+     * client associated with them. Ignore such fops.
+     */
+    if (!client) {
+        gf_msg_debug("upcall", 0, "Internal fop - client NULL");
+        return;
+    }
+
+    /* For nameless LOOKUPs, inode created shall always be
+     * invalid. Hence check if there is any already linked inode.
+     * If yes, update the inode_ctx of that valid inode
+     */
+    if (inode && (inode->ia_type == IA_INVAL) && stbuf) {
+        linked_inode = inode_find(inode->table, stbuf->ia_gfid);
+        if (linked_inode) {
+            gf_log("upcall", GF_LOG_DEBUG,
+                   "upcall_inode_ctx_get of linked inode (%p)", inode);
+            up_inode_ctx = upcall_inode_ctx_get(linked_inode, this);
         }
+    }
 
-        up_inode_ctx = ((upcall_local_t *)frame->local)->upcall_inode_ctx;
-
-        if (!up_inode_ctx)
-                up_inode_ctx = upcall_inode_ctx_get (inode, this);
-
-        if (!up_inode_ctx) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_INTERNAL_ERROR,
-                        "upcall_inode_ctx_get failed (%p)",
-                        inode);
-                return;
-        }
+    if (inode && !up_inode_ctx)
+        up_inode_ctx = upcall_inode_ctx_get(inode, this);
 
-        /* In case of LOOKUP, if first time, inode created shall be
-         * invalid till it gets linked to inode table. Read gfid from
-         * the stat returned in such cases.
+    if (!up_inode_ctx) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+               "upcall_inode_ctx_get failed (%p)", inode);
+        return;
+    }
+
+    /* In case of LOOKUP, if first time, inode created shall be
+     * invalid till it gets linked to inode table. Read gfid from
+     * the stat returned in such cases.
+     */
+    if (gf_uuid_is_null(up_inode_ctx->gfid) && stbuf) {
+        /* That means inode must have been invalid when this inode_ctx
+         * is created. Copy the gfid value from stbuf instead.
          */
-        if (gf_uuid_is_null (up_inode_ctx->gfid)) {
-                /* That means inode must have been invalid when this inode_ctx
-                 * is created. Copy the gfid value from stbuf instead.
-                 */
-                gf_uuid_copy (up_inode_ctx->gfid, stbuf->ia_gfid);
-        }
-
-        GF_VALIDATE_OR_GOTO ("upcall_cache_invalidate",
-                             !(gf_uuid_is_null (up_inode_ctx->gfid)), out);
-        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
+        gf_uuid_copy(up_inode_ctx->gfid, stbuf->ia_gfid);
+    }
+
+    if (gf_uuid_is_null(up_inode_ctx->gfid)) {
+        gf_msg_debug(this->name, 0,
+                     "up_inode_ctx->gfid and "
+                     "stbuf->ia_gfid is NULL, fop:%s",
+                     gf_fop_list[frame->root->op]);
+        goto out;
+    }
+
+    time_now = gf_time();
+    pthread_mutex_lock(&up_inode_ctx->client_list_lock);
+    {
+        list_for_each_entry_safe(up_client_entry, tmp,
+                                 &up_inode_ctx->client_list, client_list)
         {
-                list_for_each_entry_safe (up_client_entry, tmp,
-                                          &up_inode_ctx->client_list,
-                                          client_list) {
-
-                        /* Do not send UPCALL event if same client. */
-                        if (!strcmp(client->client_uid,
-                                   up_client_entry->client_uid)) {
-                                up_client_entry->access_time = time(NULL);
-                                found = _gf_true;
-                                continue;
-                        }
-
-                        /*
-                         * Ignore sending notifications in case of only UP_ATIME
-                         */
-                        if (!(flags & ~(UP_ATIME))) {
-                                if (found)
-                                        break;
-                                else /* we still need to find current client entry*/
-                                        continue;
-                        }
-
-                        /* any other client */
-
-                        /* XXX: Send notifications asynchrounously
-                         * instead of in the I/O path - BZ 1200264
-                         *  Also if the file is frequently accessed, set
-                         *  expire_time_attr to 0.
-                         */
-                        upcall_client_cache_invalidate (this,
-                                                       up_inode_ctx->gfid,
-                                                       up_client_entry,
-                                                       flags, stbuf,
-                                                       p_stbuf, oldp_stbuf,
-                                                       xattr);
-                }
+            /* Do not send UPCALL event if same client. */
+            if (!strcmp(client->client_uid, up_client_entry->client_uid)) {
+                up_client_entry->access_time = time_now;
+                found = _gf_true;
+                continue;
+            }
+
+            /*
+             * Ignore sending notifications in case of only UP_ATIME
+             */
+            if (!(flags & ~(UP_ATIME))) {
+                if (found)
+                    break;
+                else /* we still need to find current client entry*/
+                    continue;
+            }
+
+            /* any other client */
+
+            /* XXX: Send notifications asynchrounously
+             * instead of in the I/O path - BZ 1200264
+             *  Also if the file is frequently accessed, set
+             *  expire_time_attr to 0.
+             */
+            upcall_client_cache_invalidate(
+                this, up_inode_ctx->gfid, up_client_entry, flags, stbuf,
+                p_stbuf, oldp_stbuf, xattr, time_now);
+        }
 
-                if (!found) {
-                        up_client_entry = __add_upcall_client (frame,
-                                                               client,
-                                                               up_inode_ctx);
-                }
+        if (!found) {
+            up_client_entry = __add_upcall_client(frame, client, up_inode_ctx,
+                                                  time_now);
         }
-        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+    }
+    pthread_mutex_unlock(&up_inode_ctx->client_list_lock);
 out:
-        return;
+    /* release the ref from inode_find */
+    if (linked_inode)
+        inode_unref(linked_inode);
+    return;
 }
 
 /*
  * If the upcall_client_t has recently accessed the file (i.e, within
  * priv->cache_invalidation_timeout), send a upcall notification.
  */
-void
-upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
-                                upcall_client_t *up_client_entry,
-                                uint32_t flags, struct iatt *stbuf,
-                                struct iatt *p_stbuf,
-                                struct iatt *oldp_stbuf, dict_t *xattr)
+static void
+upcall_client_cache_invalidate(xlator_t *this, uuid_t gfid,
+                               upcall_client_t *up_client_entry, uint32_t flags,
+                               struct iatt *stbuf, struct iatt *p_stbuf,
+                               struct iatt *oldp_stbuf, dict_t *xattr,
+                               time_t now)
 {
-        struct gf_upcall                    up_req  = {0,};
-        struct gf_upcall_cache_invalidation ca_req  = {0,};
-        time_t                              timeout = 0;
-        int                                 ret     = -1;
-        time_t t_expired = time(NULL) - up_client_entry->access_time;
-
-        GF_VALIDATE_OR_GOTO ("upcall_client_cache_invalidate",
-                             !(gf_uuid_is_null (gfid)), out);
-        timeout = get_cache_invalidation_timeout(this);
+    struct gf_upcall up_req = {
+        0,
+    };
+    struct gf_upcall_cache_invalidation ca_req = {
+        0,
+    };
+    time_t timeout = 0;
+    int ret = -1;
+    time_t t_expired = now - up_client_entry->access_time;
+
+    GF_VALIDATE_OR_GOTO("upcall_client_cache_invalidate",
+                        !(gf_uuid_is_null(gfid)), out);
+    timeout = get_cache_invalidation_timeout(this);
+
+    if (t_expired < timeout) {
+        /* Send notify call */
+        up_req.client_uid = up_client_entry->client_uid;
+        gf_uuid_copy(up_req.gfid, gfid);
+
+        ca_req.flags = flags;
+        ca_req.expire_time_attr = up_client_entry->expire_time_attr;
+        if (stbuf)
+            ca_req.stat = *stbuf;
+        if (p_stbuf)
+            ca_req.p_stat = *p_stbuf;
+        if (oldp_stbuf)
+            ca_req.oldp_stat = *oldp_stbuf;
+        ca_req.dict = xattr;
+
+        up_req.data = &ca_req;
+        up_req.event_type = GF_UPCALL_CACHE_INVALIDATION;
+
+        gf_log(THIS->name, GF_LOG_TRACE,
+               "Cache invalidation notification sent to %s",
+               up_client_entry->client_uid);
+
+        /* Need to send inode flags */
+        ret = this->notify(this, GF_EVENT_UPCALL, &up_req);
+
+        /*
+         * notify may fail as the client could have been
+         * dis(re)connected. Cleanup the client entry.
+         */
+        if (ret < 0)
+            __upcall_cleanup_client_entry(up_client_entry);
 
-        if (t_expired < timeout) {
-                /* Send notify call */
-                up_req.client_uid = up_client_entry->client_uid;
-                gf_uuid_copy (up_req.gfid, gfid);
-
-                ca_req.flags = flags;
-                ca_req.expire_time_attr =
-                                 up_client_entry->expire_time_attr;
-                if (stbuf)
-                        ca_req.stat = *stbuf;
-                if (p_stbuf)
-                        ca_req.p_stat = *p_stbuf;
-                if (oldp_stbuf)
-                        ca_req.oldp_stat = *oldp_stbuf;
-                ca_req.dict = xattr;
-
-                up_req.data = &ca_req;
-                up_req.event_type = GF_UPCALL_CACHE_INVALIDATION;
-
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "Cache invalidation notification sent to %s",
-                        up_client_entry->client_uid);
-
-                /* Need to send inode flags */
-                ret = this->notify (this, GF_EVENT_UPCALL, &up_req);
-
-                /*
-                 * notify may fail as the client could have been
-                 * dis(re)connected. Cleanup the client entry.
-                 */
-                if (ret < 0)
-                        __upcall_cleanup_client_entry (up_client_entry);
-
-        } else {
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "Cache invalidation notification NOT sent to %s",
-                        up_client_entry->client_uid);
-
-                if (t_expired > (2*timeout)) {
-                        /* Cleanup the entry */
-                        __upcall_cleanup_client_entry (up_client_entry);
-                }
+    } else {
+        gf_log(THIS->name, GF_LOG_TRACE,
+               "Cache invalidation notification NOT sent to %s",
+               up_client_entry->client_uid);
+
+        if (t_expired > (2 * timeout)) {
+            /* Cleanup the entry */
+            __upcall_cleanup_client_entry(up_client_entry);
         }
+    }
 out:
-        return;
+    return;
 }
 
 /*
- * This is called during upcall_inode_ctx cleanup incase of 'inode_forget'.
+ * This is called during upcall_inode_ctx cleanup in case of 'inode_forget'.
  * Send "UP_FORGET" to all the clients so that they invalidate their cache
  * entry and do a fresh lookup next time when any I/O comes in.
  */
-void
-upcall_cache_forget (xlator_t *this, inode_t *inode, upcall_inode_ctx_t *up_inode_ctx)
+static void
+upcall_cache_forget(xlator_t *this, inode_t *inode,
+                    upcall_inode_ctx_t *up_inode_ctx)
 {
-        upcall_client_t *up_client       = NULL;
-        upcall_client_t *up_client_entry = NULL;
-        upcall_client_t *tmp             = NULL;
-        uint32_t        flags            = 0;
+    upcall_client_t *up_client_entry = NULL;
+    upcall_client_t *tmp = NULL;
+    uint32_t flags = UP_FORGET;
+    time_t time_now;
 
-        if (!up_inode_ctx) {
-                return;
-        }
+    if (!up_inode_ctx) {
+        return;
+    }
 
-        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
+    time_now = gf_time();
+    pthread_mutex_lock(&up_inode_ctx->client_list_lock);
+    {
+        list_for_each_entry_safe(up_client_entry, tmp,
+                                 &up_inode_ctx->client_list, client_list)
         {
-                list_for_each_entry_safe (up_client_entry, tmp,
-                                          &up_inode_ctx->client_list,
-                                          client_list) {
-                        flags = UP_FORGET;
-
-                        /* Set the access time to time(NULL)
-                         * to send notify */
-                        up_client_entry->access_time = time(NULL);
-
-                        upcall_client_cache_invalidate(this,
-                                                       up_inode_ctx->gfid,
-                                                       up_client_entry,
-                                                       flags, NULL,
-                                                       NULL, NULL, NULL);
-                }
+            /* Set the access time to gf_time()
+             * to send notify */
+            up_client_entry->access_time = time_now;
 
+            upcall_client_cache_invalidate(this, up_inode_ctx->gfid,
+                                           up_client_entry, flags, NULL, NULL,
+                                           NULL, NULL, time_now);
         }
-        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+    }
+    pthread_mutex_unlock(&up_inode_ctx->client_list_lock);
 }
diff --git a/xlators/features/upcall/src/upcall-mem-types.h b/xlators/features/upcall/src/upcall-mem-types.h
index 55793ec65ca..f9883d9d72c 100644
--- a/xlators/features/upcall/src/upcall-mem-types.h
+++ b/xlators/features/upcall/src/upcall-mem-types.h
@@ -11,14 +11,13 @@
 #ifndef __UPCALL_MEM_TYPES_H__
 #define __UPCALL_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_upcall_mem_types_ {
-        gf_upcall_mt_conf_t = gf_common_mt_end + 1,
-        gf_upcall_mt_private_t,
-        gf_upcall_mt_upcall_inode_ctx_t,
-        gf_upcall_mt_upcall_client_entry_t,
-        gf_upcall_mt_end
+    gf_upcall_mt_conf_t = gf_common_mt_end + 1,
+    gf_upcall_mt_private_t,
+    gf_upcall_mt_upcall_inode_ctx_t,
+    gf_upcall_mt_upcall_client_entry_t,
+    gf_upcall_mt_end
 };
 #endif
-
diff --git a/xlators/features/upcall/src/upcall-messages.h b/xlators/features/upcall/src/upcall-messages.h
index 0cfdfd68b77..4095a34c200 100644
--- a/xlators/features/upcall/src/upcall-messages.h
+++ b/xlators/features/upcall/src/upcall-messages.h
@@ -11,49 +11,19 @@
 #ifndef _UPCALL_MESSAGES_H_
 #define _UPCALL_MESSAGES_H_
 
-#include "glfs-message-id.h"
-
-/*! \file upcall-messages.h
- *  \brief UPCALL log-message IDs and their descriptions.
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check across the code if the message ID macro in question is reused
- * anywhere. If reused then then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
- */
-
-#define GLFS_COMP_BASE_UPCALL GLFS_MSGID_COMP_UPCALL
-#define GLFS_NUM_MESSAGES 1
-#define GLFS_MSGID_END (GLFS_COMP_BASE_UPCALL + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x GLFS_COMP_BASE_UPCALL, "Invalid: Start of messages"
-
-/*!
- * @messageid 110001
- * @diagnosis Out of Memory
- * @recommendedaction None
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define UPCALL_MSG_NO_MEMORY             (GLFS_COMP_BASE_UPCALL + 1)
-#define UPCALL_MSG_INTERNAL_ERROR        (GLFS_COMP_BASE_UPCALL + 2)
-#define UPCALL_MSG_NOTIFY_FAILED         (GLFS_COMP_BASE_UPCALL + 3)
 
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+GLFS_MSGID(UPCALL, UPCALL_MSG_NO_MEMORY, UPCALL_MSG_INTERNAL_ERROR,
+           UPCALL_MSG_NOTIFY_FAILED);
 
 #endif /* !_UPCALL_MESSAGES_H_ */
diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c
index e4cb50cd42b..0795f58059d 100644
--- a/xlators/features/upcall/src/upcall.c
+++ b/xlators/features/upcall/src/upcall.c
@@ -13,2264 +13,2493 @@
 #include <limits.h>
 #include <pthread.h>
 
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
-#include "statedump.h"
-#include "syncop.h"
+#include <glusterfs/statedump.h>
 
 #include "upcall.h"
 #include "upcall-mem-types.h"
 #include "glusterfs3-xdr.h"
 #include "protocol-common.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 
-int32_t
-up_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+static int32_t
+up_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+    UPCALL_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+static int32_t
+up_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_open_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
+    STACK_WIND(frame, up_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
+static int32_t
+up_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 postbuf, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, postbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (writev, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
+    UPCALL_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           struct iovec *vector, int count, off_t off, uint32_t flags,
-           struct iobref *iobref, dict_t *xdata)
+static int32_t
+up_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_writev_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, off, flags, iobref, xdata);
+    STACK_WIND(frame, up_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              struct iovec *vector, int count, struct iatt *stbuf,
-              struct iobref *iobref, dict_t *xdata)
+static int32_t
+up_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+             struct iobref *iobref, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 stbuf, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (readv, frame, op_ret, op_errno, vector,
-                             count, stbuf, iobref, xdata);
+    UPCALL_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_readv (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, size_t size, off_t offset,
-          uint32_t flags, dict_t *xdata)
+static int32_t
+up_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_readv_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
+    STACK_WIND(frame, up_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0,
-                             NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-           int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-           dict_t *xdata)
+static int32_t
+up_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
+    UPCALL_STACK_UNWIND(lk, frame, op_ret, op_errno, lock, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+static int32_t
+up_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, up_lk_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk,
+               fd, cmd, flock, xdata);
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+static int32_t
+up_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 postbuf, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, postbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
+    UPCALL_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-             dict_t *xdata)
+static int32_t
+up_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_truncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
+    STACK_WIND(frame, up_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *statpre,
-                struct iatt *statpost, dict_t *xdata)
+static int32_t
+up_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *statpre, struct iatt *statpost,
+               dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        /* XXX: setattr -> UP_SIZE or UP_OWN or UP_MODE or UP_TIMES
-         * or INODE_UPDATE (or UP_PERM esp incase of ACLs -> INODE_INVALIDATE)
-         * Need to check what attr is changed and accordingly pass UP_FLAGS.
-         * Bug1200271.
-         */
-        flags = UP_ATTR_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 statpost, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    /* XXX: setattr -> UP_SIZE or UP_OWN or UP_MODE or UP_TIMES
+     * or INODE_UPDATE (or UP_PERM esp in case of ACLs -> INODE_INVALIDATE)
+     * Need to check what attr is changed and accordingly pass UP_FLAGS.
+     * Bug1200271.
+     */
+    flags = UP_ATTR_FLAGS;
+    /* If mode bits have changed invalidate the xattrs, as posix-acl and
+     * others store permission related information in xattrs. With changing
+     * of permissions/mode, we need to make clients to forget all the
+     * xattrs related to permissions.
+     * TODO: Invalidate the xattr system.posix_acl_access alone.
+     */
+    if (is_same_mode(statpre->ia_prot, statpost->ia_prot) != 0)
+        flags |= UP_XATTR;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, statpost,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                             statpre, statpost, xdata);
+    UPCALL_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+static int32_t
+up_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_setattr_cbk,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->setattr,
-                            loc, stbuf, valid, xdata);
+    STACK_WIND(frame, up_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-               struct iatt *preoldparent, struct iatt *postoldparent,
-               struct iatt *prenewparent, struct iatt *postnewparent,
-               dict_t *xdata)
+static int32_t
+up_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *stbuf, struct iatt *preoldparent,
+              struct iatt *postoldparent, struct iatt *prenewparent,
+              struct iatt *postnewparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = (UP_RENAME_FLAGS | UP_PARENT_DENTRY_FLAGS);
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 stbuf, postnewparent, postoldparent, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = (UP_RENAME_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            postnewparent, postoldparent, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->rename_oldloc.parent,
+                            flags, postoldparent, NULL, NULL, NULL);
+
+    if (local->rename_oldloc.parent == local->loc.parent)
+        goto out;
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postnewparent, NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (rename, frame, op_ret, op_errno,
-                             stbuf, preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
+    UPCALL_STACK_UNWIND(rename, frame, op_ret, op_errno, stbuf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_rename (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+static int32_t
+up_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, oldloc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, newloc, NULL, oldloc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
-        /* copy oldloc */
-        loc_copy (&local->rename_oldloc, oldloc);
+    /* copy oldloc */
+    loc_copy(&local->rename_oldloc, oldloc);
 out:
-        STACK_WIND (frame, up_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
+    STACK_WIND(frame, up_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, postparent, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            postparent, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postparent, NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-           dict_t *xdata)
+static int32_t
+up_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+          dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
+    STACK_WIND(frame, up_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, inode_t *inode, struct iatt *stbuf,
+            struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 stbuf, postparent, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            postparent, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postparent, NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (link, frame, op_ret, op_errno,
-                             inode, stbuf, preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-         loc_t *newloc, dict_t *xdata)
+static int32_t
+up_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, oldloc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, newloc, NULL, oldloc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
+    STACK_WIND(frame, up_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (link, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iatt *preparent, struct iatt *postparent,
+             dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            postparent, NULL, NULL);
 
-        flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, postparent, NULL, NULL);
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postparent, NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-          dict_t *xdata)
+static int32_t
+up_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+         dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_rmdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
+    STACK_WIND(frame, up_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, inode_t *inode,
-              struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, inode_t *inode, struct iatt *stbuf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* invalidate parent's entry too */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
 
-        /* invalidate parent's entry too */
-        flags = UP_TIMES;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 postparent, NULL, NULL, NULL);
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, stbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                             inode, stbuf, preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_mkdir (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
+static int32_t
+up_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *params)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->parent, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_mkdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, params);
+    STACK_WIND(frame, up_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, fd_t *fd, inode_t *inode,
-               struct iatt *stbuf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* As its a new file create, no need of sending notification
+     * However invalidate parent's entry and update that fact that the
+     * client has accessed the newly created entry */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
 
-        /* As its a new file create, no need of sending notification */
-        /* However invalidate parent's entry */
-        flags = UP_TIMES;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 postparent, NULL, NULL, NULL);
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, stbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
-                             inode, stbuf, preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                        preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_create (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int32_t flags, mode_t mode,
-           mode_t umask, fd_t *fd, dict_t *params)
+static int32_t
+up_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->parent, NULL);
-
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_create_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, params);
+    STACK_WIND(frame, up_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               params);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (create, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-               struct iatt *postparent)
+static int32_t
+up_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+              struct iatt *postparent)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 stbuf, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf,
-                             xattr, postparent);
+    UPCALL_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                        postparent);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_lookup (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, dict_t *xattr_req)
+static int32_t
+up_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_lookup_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
+    STACK_WIND(frame, up_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (lookup, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_stat_cbk (call_frame_t *frame, void *cookie,
-             xlator_t *this, int32_t op_ret, int32_t op_errno,
-             struct iatt *buf, dict_t *xdata)
+static int32_t
+up_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 buf, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, buf, NULL,
+                            NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (stat, frame, op_ret, op_errno, buf,
-                             xdata);
+    UPCALL_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+static int32_t
+up_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_stat_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
+    STACK_WIND(frame, up_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_fstat (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, dict_t *xdata)
+static int32_t
+up_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_stat_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
+    STACK_WIND(frame, up_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_ftruncate (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, off_t offset, dict_t *xdata)
+static int32_t
+up_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_truncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
+    STACK_WIND(frame, up_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL,
-                             NULL, NULL);
+    UPCALL_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, dict_t *xdata)
+static int32_t
+up_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+    UPCALL_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_access (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int32_t mask, dict_t *xdata)
+static int32_t
+up_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_access_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->access,
-                    loc, mask, xdata);
+    STACK_WIND(frame, up_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    UPCALL_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, const char *path,
-                 struct iatt *stbuf, dict_t *xdata)
+static int32_t
+up_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, const char *path, struct iatt *stbuf,
+                dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 stbuf, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf,
-                             xdata);
+    UPCALL_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_readlink (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, size_t size, dict_t *xdata)
+static int32_t
+up_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_readlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
+    STACK_WIND(frame, up_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (readlink, frame, -1, op_errno, NULL,
-                             NULL, NULL);
+    UPCALL_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, inode_t *inode,
-              struct iatt *buf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* invalidate parent's entry too */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
 
-        /* invalidate parent's entry too */
-        flags = UP_TIMES;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 postparent, NULL, NULL, NULL);
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, buf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-          mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+static int32_t
+up_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->parent, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_mknod_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
+    STACK_WIND(frame, up_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (mknod, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* invalidate parent's entry too */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
 
-        /* invalidate parent's entry too */
-        flags = UP_TIMES;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 postparent, NULL, NULL, NULL);
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, buf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_symlink (call_frame_t   *frame, xlator_t *this,
-            const char *linkpath, loc_t *loc, mode_t umask,
-            dict_t *xdata)
+static int32_t
+up_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->parent, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_symlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
+    STACK_WIND(frame, up_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (symlink, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd,
-                dict_t *xdata)
+static int32_t
+up_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
+    UPCALL_STACK_UNWIND(opendir, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_opendir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, fd_t *fd, dict_t *xdata)
+static int32_t
+up_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_opendir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
+    STACK_WIND(frame, up_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-               dict_t *xdata)
+static int32_t
+up_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
+    UPCALL_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_statfs (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, dict_t *xdata)
+static int32_t
+up_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_statfs_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
+    STACK_WIND(frame, up_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                dict_t *xdata)
+static int32_t
+up_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+               dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, xdata);
+    UPCALL_STACK_UNWIND(readdir, frame, op_ret, op_errno, entries, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_readdir (call_frame_t  *frame, xlator_t *this,
-            fd_t *fd, size_t size, off_t off, dict_t *xdata)
+static int32_t
+up_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t off, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_readdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdir,
-                    fd, size, off, xdata);
+    STACK_WIND(frame, up_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_readdirp (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, size_t size, off_t off, dict_t *dict)
+static int32_t
+up_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    gf_dirent_t *entry = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    client = frame->root->client;
+    local = frame->local;
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if (entry->inode == NULL) {
+            continue;
         }
+        upcall_cache_invalidate(frame, this, client, entry->inode, flags,
+                                &entry->d_stat, NULL, NULL, NULL);
+    }
 
 out:
-        STACK_WIND (frame, up_readdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, off, dict);
+    UPCALL_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *dict)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        return 0;
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
+
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (readdirp, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(readdirp, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt  *stbuf, int32_t valid, dict_t *xdata)
+static int32_t
+up_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_setattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
+    STACK_WIND(frame, up_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL,
-                             NULL, NULL);
+    UPCALL_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
+static int32_t
 up_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *pre,
                  struct iatt *post, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 post, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, post,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (fallocate, frame, op_ret, op_errno, pre,
-                             post, xdata);
+    UPCALL_STACK_UNWIND(fallocate, frame, op_ret, op_errno, pre, post, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
-             int32_t mode, off_t offset, size_t len, dict_t *xdata)
+static int32_t
+up_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+             off_t offset, size_t len, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_fallocate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fallocate,
-                    fd, mode, offset, len, xdata);
+    STACK_WIND(frame, up_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL,
-                             NULL, NULL);
+    UPCALL_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
+static int32_t
 up_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *pre,
                struct iatt *post, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 post, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, post,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (discard, frame, op_ret, op_errno, pre,
-                             post, xdata);
+    UPCALL_STACK_UNWIND(discard, frame, op_ret, op_errno, pre, post, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
-           off_t offset, size_t len, dict_t *xdata)
+static int32_t
+up_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_discard_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->discard,
-                    fd, offset, len, xdata);
+    STACK_WIND(frame, up_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (discard, frame, -1, op_errno, NULL,
-                             NULL, NULL);
+    UPCALL_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
+static int32_t
 up_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, struct iatt *pre,
                 struct iatt *post, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_WRITE_FLAGS;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 post, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, post,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (zerofill, frame, op_ret, op_errno, pre,
-                             post, xdata);
+    UPCALL_STACK_UNWIND(zerofill, frame, op_ret, op_errno, pre, post, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-up_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
-            off_t offset, off_t len, dict_t *xdata)
+static int
+up_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_zerofill_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->zerofill,
-                    fd, offset, len, xdata);
+    STACK_WIND(frame, up_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL,
-                             NULL, NULL);
+    UPCALL_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_seek_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-             int op_errno, off_t offset, dict_t *xdata)
+static int32_t
+up_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, off_t offset, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (seek, frame, op_ret, op_errno, offset, xdata);
+    UPCALL_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-         gf_seek_what_t what, dict_t *xdata)
+static int32_t
+up_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_seek_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+    STACK_WIND(frame, up_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (seek, frame, -1, op_errno, 0, NULL);
+    UPCALL_STACK_UNWIND(seek, frame, -1, op_errno, 0, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+static int32_t
+up_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
-        int              ret            = 0;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    struct iatt stbuf = {
+        0,
+    };
+    upcall_private_t *priv = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    client = frame->root->client;
+    local = frame->local;
 
-        flags = UP_XATTR;
-        /* Remove the virtual xattrs from the dict */
-        ret = dict_foreach (local->xattr, up_filter_virtual_xattr, NULL);
-        if (ret < 0) {
-                op_ret = ret;
-                goto out;
-        }
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, local->xattr);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = UP_XATTR;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
 
 out:
-        UPCALL_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+    UPCALL_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+static int32_t
+up_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
-        dict_t           *xattr          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        xattr = dict_copy_with_ref (dict, NULL);
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local = upcall_local_init (frame, this, loc->inode, xattr);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, dict);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
-                    xdata);
+    STACK_WIND(frame, up_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+    UPCALL_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+up_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
-        int              ret            = 0;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    struct iatt stbuf = {
+        0,
+    };
+    upcall_private_t *priv = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    client = frame->root->client;
+    local = frame->local;
 
-        flags = UP_XATTR;
-        /* Remove the virtual xattrs from the dict */
-        ret = dict_foreach (local->xattr, up_filter_virtual_xattr, NULL);
-        if (ret < 0) {
-                op_ret = ret;
-                goto out;
-        }
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, local->xattr);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = UP_XATTR;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
 
 out:
-        UPCALL_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+    UPCALL_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+static int32_t
+up_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
-        dict_t           *xattr          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        xattr = dict_copy_with_ref (dict, NULL);
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local = upcall_local_init (frame, this, fd->inode, xattr);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, fd, fd->inode, dict);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_fsetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
+    STACK_WIND(frame, up_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+    UPCALL_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-up_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+up_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = 0;
+    upcall_private_t *priv = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_XATTR_RM;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, local->xattr);
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_XATTR_RM;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
 
 out:
-        UPCALL_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno,
-                             xdata);
-        return 0;
+    UPCALL_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
-int32_t
-up_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+static int32_t
+up_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
-        dict_t           *xattr          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+    dict_t *xattr = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        xattr = dict_for_key_value (name, "", 1);
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xattr = dict_for_key_value(name, "", 1, _gf_true);
+    if (!xattr) {
+        goto err;
+    }
 
-        local = upcall_local_init (frame, this, fd->inode, xattr);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, fd, fd->inode, xattr);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_fremovexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    if (xattr)
+        dict_unref(xattr);
+
+    STACK_WIND(frame, up_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
-}
+    UPCALL_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
+    return 0;
+}
 
-int32_t
-up_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+up_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = 0;
+    upcall_private_t *priv = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_XATTR_RM;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, local->xattr);
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_XATTR_RM;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
 
 out:
-        UPCALL_STACK_UNWIND (removexattr, frame, op_ret, op_errno,
-                             xdata);
-        return 0;
+    UPCALL_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
-int32_t
-up_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+static int32_t
+up_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
-        dict_t           *xattr          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+    dict_t *xattr = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        xattr = dict_for_key_value (name, "", 1);
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xattr = dict_for_key_value(name, "", 1, _gf_true);
+    if (!xattr) {
+        goto err;
+    }
 
-        local = upcall_local_init (frame, this, loc->inode, xattr);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, xattr);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_removexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    if (xattr)
+        dict_unref(xattr);
+
+    STACK_WIND(frame, up_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 
 err:
-        UPCALL_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
-}
+    UPCALL_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
 
+    return 0;
+}
 
-int32_t
-up_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict,
-                  dict_t *xdata)
+static int32_t
+up_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
 
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno,
-                             dict, xdata);
-        return 0;
+    UPCALL_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
-int32_t
-up_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              const char *name, dict_t *xdata)
+static int32_t
+up_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_fgetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, up_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 err:
-        UPCALL_STACK_UNWIND (fgetxattr, frame, -1, op_errno,
-                             NULL, NULL);
-        return 0;
+    UPCALL_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
-int32_t
-up_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict,
-                 dict_t *xdata)
+static int32_t
+up_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
 
-        flags = UP_UPDATE_CLIENT;
-        upcall_cache_invalidate (frame, this, client, local->inode, flags,
-                                 NULL, NULL, NULL, NULL);
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
-                             dict, xdata);
-        return 0;
+    UPCALL_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-int32_t
-up_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name, dict_t *xdata)
+static int32_t
+up_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode, NULL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_getxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, up_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 err:
-        UPCALL_STACK_UNWIND (getxattr, frame, -1, op_errno,
-                             NULL, NULL);
-        return 0;
+    UPCALL_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+/* The xattrops here mainly tracks changes in afr pending xattr.
+ *    1. xattrop doesn't carry info saying post op/pre op.
+ *    2. Pre xattrop will have 0 value for all pending xattrs,
+ *       the cbk of pre xattrop carries the on-disk xattr value.
+ *       Non zero on-disk xattr indicates pending healing.
+ *    3. Post xattrop will either have 0 or 1 as value of pending xattrs,
+ *       0 on success, 1 on failure. But the post xattrop cbk will have
+ *       0 or 1 or any higher value.
+ *       0 - if no healing required*
+ *       1 - if this is the first time pending xattr is being set.
+ *       n - if there is already a pending xattr set, it will increment
+ *       the on-disk value and send that in cbk.
+ * Our aim is to send an invalidation, only the first time a pending
+ * xattr was set on a file. Below are some of the exceptions in handling
+ * xattrop:
+ * - Do not filter unregistered xattrs in the cbk, but in the call path.
+ *   Else, we will be invalidating on every preop, if the file already has
+ *   pending xattr set. Filtering unregistered xattrs on the fop path
+ *   ensures we invalidate only in postop, every time a postop comes with
+ *   pending xattr value 1.
+ * - Consider a brick is down, and the postop sets pending xattrs as long
+ *   as the other brick is down. But we do not want to invalidate every time
+ *   a pending xattr is set, but we want to invalidate only the first time
+ *   a pending xattr is set on any file. Hence, to identify if its the first
+ *   time a pending xattr is set, we compare the value of pending xattrs that
+ *   came in postop and postop cbk, if its same then its the first time.
+ */
+static int32_t
+up_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    client_t *client = NULL;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    if (up_invalidate_needed(local->xattr)) {
+        if (dict_foreach(local->xattr, up_compare_afr_xattr, dict) < 0)
+            goto out;
+
+        upcall_cache_invalidate(frame, this, client, local->inode, UP_XATTR,
+                                NULL, NULL, NULL, local->xattr);
+    }
+out:
+    if (frame->root->op == GF_FOP_FXATTROP) {
+        UPCALL_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    } else {
+        UPCALL_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    }
+    return 0;
+}
+
+static int32_t
+up_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    int32_t op_errno = EINVAL;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    upcall_private_t *priv = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, xattr);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    return 0;
+err:
+    UPCALL_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
+static int32_t
+up_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    int32_t op_errno = EINVAL;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    upcall_private_t *priv = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    local = upcall_local_init(frame, this, NULL, fd, fd->inode, xattr);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(fxattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_upcall_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_NO_MEMORY,
-                        "Memory allocation failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_upcall_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_NO_MEMORY,
+               "Memory allocation failed");
         return ret;
+    }
+
+    return ret;
 }
 
 void
-upcall_local_wipe (xlator_t *this, upcall_local_t *local)
-{
-        if (local) {
-                inode_unref (local->inode);
-                if (local->xattr) {
-                        /* There will be 2 refs at this point, hence dict_destroy:
-                         * 1. taken by dict_copy_with_ref
-                         * 2. taken by upcall_local_init ()
-                         */
-                        dict_destroy (local->xattr);
-                }
-                loc_wipe (&local->rename_oldloc);
-                mem_put (local);
-        }
+upcall_local_wipe(xlator_t *this, upcall_local_t *local)
+{
+    if (local) {
+        inode_unref(local->inode);
+        if (local->xattr)
+            dict_unref(local->xattr);
+        loc_wipe(&local->rename_oldloc);
+        loc_wipe(&local->loc);
+        if (local->fd)
+            fd_unref(local->fd);
+        mem_put(local);
+    }
 }
 
 upcall_local_t *
-upcall_local_init (call_frame_t *frame, xlator_t *this, inode_t *inode, dict_t *xattr)
+upcall_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                  inode_t *inode, dict_t *xattr)
 {
-        upcall_local_t *local = NULL;
+    upcall_local_t *local = NULL;
 
-        local = mem_get0 (THIS->local_pool);
+    GF_VALIDATE_OR_GOTO("upcall", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        if (!local)
-                goto out;
+    local = mem_get0(THIS->local_pool);
+
+    if (!local)
+        goto out;
+
+    local->inode = inode_ref(inode);
+    if (xattr)
+        local->xattr = dict_copy_with_ref(xattr, NULL);
+
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd)
+        local->fd = fd_ref(fd);
+
+    frame->local = local;
+
+out:
+    return local;
+}
+
+static int32_t
+update_xattrs(dict_t *dict, char *key, data_t *value, void *data)
+{
+    dict_t *xattrs = data;
+    int ret = 0;
+
+    ret = dict_set_int8(xattrs, key, 0);
+    return ret;
+}
+
+int32_t
+up_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    upcall_private_t *priv = NULL;
+    int ret = 0;
 
-        local->inode = inode_ref (inode);
-        if (xattr)
-                local->xattr = dict_ref (xattr);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        /* Shall we get inode_ctx and store it here itself? */
-        local->upcall_inode_ctx = upcall_inode_ctx_get (inode, this);
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind;
 
-        frame->local = local;
+    /* TODO: Bz-1371622 Along with the xattrs also store list of clients
+     * that are interested in notifications, so that the notification
+     * can be sent to the clients that have registered.
+     * Once this implemented there can be unregister of xattrs for
+     * notifications. Until then there is no unregister of xattrs*/
+    if (xdata && priv->xattrs) {
+        ret = dict_foreach(xdata, update_xattrs, priv->xattrs);
+    }
 
 out:
-        return local;
+    STACK_UNWIND_STRICT(ipc, frame, ret, 0, NULL);
+    return 0;
+
+wind:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        upcall_private_t *priv                   = NULL;
-        int              ret                    = -1;
+    upcall_private_t *priv = NULL;
+    int ret = -1;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        GF_OPTION_RECONF ("cache-invalidation", priv->cache_invalidation_enabled,
-                          options, bool, out);
-        GF_OPTION_RECONF ("cache-invalidation-timeout", priv->cache_invalidation_timeout,
-                          options, int32, out);
+    GF_OPTION_RECONF("cache-invalidation", priv->cache_invalidation_enabled,
+                     options, bool, out);
+    GF_OPTION_RECONF("cache-invalidation-timeout",
+                     priv->cache_invalidation_timeout, options, int32, out);
 
-        ret = 0;
+    ret = 0;
 
-        if (priv->cache_invalidation_enabled &&
-            !priv->reaper_init_done) {
-                ret = upcall_reaper_thread_init (this);
+    if (priv->cache_invalidation_enabled && !priv->reaper_init_done) {
+        ret = upcall_reaper_thread_init(this);
 
-                if (ret) {
-                        gf_msg ("upcall", GF_LOG_WARNING, 0,
-                                UPCALL_MSG_INTERNAL_ERROR,
-                                "reaper_thread creation failed (%s)."
-                                " Disabling cache_invalidation",
-                                strerror(errno));
-                }
-                priv->reaper_init_done = 1;
+        if (ret) {
+            gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+                   "reaper_thread creation failed (%s)."
+                   " Disabling cache_invalidation",
+                   strerror(errno));
         }
+        priv->reaper_init_done = _gf_true;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int                       ret        = -1;
-        upcall_private_t         *priv       = NULL;
+    int ret = -1;
+    upcall_private_t *priv = NULL;
 
-        priv = GF_CALLOC (1, sizeof (*priv),
-                          gf_upcall_mt_private_t);
-        if (!priv) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_NO_MEMORY,
-                        "Memory allocation failed");
-                goto out;
-        }
+    priv = GF_CALLOC(1, sizeof(*priv), gf_upcall_mt_private_t);
+    if (!priv)
+        goto out;
 
-        GF_OPTION_INIT ("cache-invalidation", priv->cache_invalidation_enabled,
-                        bool, out);
-        GF_OPTION_INIT ("cache-invalidation-timeout",
-                        priv->cache_invalidation_timeout, int32, out);
+    priv->xattrs = dict_new();
+    if (!priv->xattrs)
+        goto out;
 
-        LOCK_INIT (&priv->inode_ctx_lk);
-        INIT_LIST_HEAD (&priv->inode_ctx_list);
+    GF_OPTION_INIT("cache-invalidation", priv->cache_invalidation_enabled, bool,
+                   out);
+    GF_OPTION_INIT("cache-invalidation-timeout",
+                   priv->cache_invalidation_timeout, int32, out);
 
-        this->private = priv;
-        priv->fini = 0;
-        priv->reaper_init_done = 0;
+    LOCK_INIT(&priv->inode_ctx_lk);
+    INIT_LIST_HEAD(&priv->inode_ctx_list);
 
-        this->local_pool = mem_pool_new (upcall_local_t, 512);
-        ret = 0;
+    priv->fini = 0;
+    priv->reaper_init_done = _gf_false;
 
-        if (priv->cache_invalidation_enabled) {
-                ret = upcall_reaper_thread_init (this);
+    this->private = priv;
+    this->local_pool = mem_pool_new(upcall_local_t, 512);
+    ret = 0;
+
+    if (priv->cache_invalidation_enabled) {
+        ret = upcall_reaper_thread_init(this);
 
-                if (ret) {
-                        gf_msg ("upcall", GF_LOG_WARNING, 0,
-                                UPCALL_MSG_INTERNAL_ERROR,
-                                "reaper_thread creation failed (%s)."
-                                " Disabling cache_invalidation",
-                                strerror(errno));
-                }
-                priv->reaper_init_done = 1;
-        }
-out:
         if (ret) {
-                GF_FREE (priv);
+            gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+                   "reaper_thread creation failed (%s)."
+                   " Disabling cache_invalidation",
+                   strerror(errno));
         }
+        priv->reaper_init_done = _gf_true;
+    }
+out:
+    if (ret && priv) {
+        if (priv->xattrs)
+            dict_unref(priv->xattrs);
 
-        return ret;
+        GF_FREE(priv);
+    }
+
+    return ret;
 }
 
-int
-fini (xlator_t *this)
+void
+fini(xlator_t *this)
 {
-        upcall_private_t *priv = NULL;
+    upcall_private_t *priv = NULL;
 
-        priv = this->private;
-        if (!priv) {
-                return 0;
-        }
-        this->private = NULL;
+    priv = this->private;
+    if (!priv) {
+        return;
+    }
+    this->private = NULL;
 
-        priv->fini = 1;
+    priv->fini = 1;
 
-        pthread_join (priv->reaper_thr, NULL);
+    if (priv->reaper_thr) {
+        gf_thread_cleanup_xint(priv->reaper_thr);
+        priv->reaper_thr = 0;
+        priv->reaper_init_done = _gf_false;
+    }
 
-        LOCK_DESTROY (&priv->inode_ctx_lk);
+    dict_unref(priv->xattrs);
+    LOCK_DESTROY(&priv->inode_ctx_lk);
 
-        /* Do we need to cleanup the inode_ctxs? IMO not required
-         * as inode_forget would have been done on all the inodes
-         * before calling xlator_fini */
-        GF_FREE (priv);
+    /* Do we need to cleanup the inode_ctxs? IMO not required
+     * as inode_forget would have been done on all the inodes
+     * before calling xlator_fini */
+    GF_FREE(priv);
 
-        return 0;
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+
+    return;
 }
 
 int
-upcall_forget (xlator_t *this, inode_t *inode)
+upcall_forget(xlator_t *this, inode_t *inode)
 {
-        upcall_cleanup_inode_ctx (this, inode);
-        return 0;
+    upcall_private_t *priv = this->private;
+
+    if (!priv)
+        goto out;
+
+    upcall_cleanup_inode_ctx(this, inode);
+out:
+    return 0;
 }
 
 int
-upcall_release (xlator_t *this, fd_t *fd)
+upcall_release(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int                 ret              = -1;
-        int32_t             val              = 0;
-        struct gf_upcall    *up_req          = NULL;
+    int ret = -1;
+    struct gf_upcall *up_req = NULL;
 
-        switch (event) {
-        case GF_EVENT_UPCALL:
-        {
-                gf_log (this->name, GF_LOG_DEBUG, "Upcall Notify event = %d",
-                        event);
+    switch (event) {
+        case GF_EVENT_UPCALL: {
+            gf_log(this->name, GF_LOG_DEBUG, "Upcall Notify event = %d", event);
 
-                up_req = (struct gf_upcall *) data;
+            up_req = (struct gf_upcall *)data;
 
-                GF_VALIDATE_OR_GOTO(this->name, up_req, out);
+            GF_VALIDATE_OR_GOTO(this->name, up_req, out);
 
-                ret = default_notify (this, event, up_req);
+            ret = default_notify(this, event, up_req);
 
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                UPCALL_MSG_NOTIFY_FAILED,
-                                "Failed to notify cache invalidation"
-                                " to client(%s)",
-                                up_req->client_uid);
-                        goto out;
-                }
-        }
-        break;
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, UPCALL_MSG_NOTIFY_FAILED,
+                       "Failed to notify cache invalidation"
+                       " to client(%s)",
+                       up_req->client_uid);
+                goto out;
+            }
+        } break;
         default:
-                default_notify (this, event, data);
-        break;
-        }
-        ret = 0;
+            default_notify(this, event, data);
+            break;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops = {
-        /* fops which change only "ATIME" do not result
-         * in any cache invalidation. Hence upcall
-         * notifications are not sent in this case.
-         * But however, we need to store/update the
-         * client info in the upcall state to be able
-         * to notify them incase of any changes done
-         * to the data.
-         *
-         * Below such fops do not trigger upcall
-         * notifications but will add/update
-         * clients info in the upcall inode ctx.*/
-        .lookup      = up_lookup,
-        .open        = up_open,
-        .statfs      = up_statfs,
-        .opendir     = up_opendir,
-        .readdir     = up_readdir,
-        .readdirp    = up_readdirp,
-        .stat        = up_stat,
-        .fstat       = up_fstat,
-        .access      = up_access,
-        .readlink    = up_readlink,
-        .readv       = up_readv,
-        .lk          = up_lk,
-        .seek        = up_seek,
-
-        /* fops doing  write */
-        .truncate    = up_truncate,
-        .ftruncate   = up_ftruncate,
-        .writev      = up_writev,
-        .zerofill    = up_zerofill,
-        .fallocate   = up_fallocate,
-        .discard     = up_discard,
-
-        /* fops changing attributes */
-        .fsetattr    = up_fsetattr,
-        .setattr     = up_setattr,
-
-        /* fops affecting parent dirent */
-        .mknod       = up_mknod,
-        .create      = up_create,
-        .symlink     = up_symlink,
-        .mkdir       = up_mkdir,
-
-        /* fops affecting both file and parent
-         * cache entries */
-        .unlink      = up_unlink,
-        .link        = up_link,
-        .rmdir       = up_rmdir,
-        .rename      = up_rename,
-
-        .setxattr    = up_setxattr,
-        .fsetxattr   = up_fsetxattr,
-        .getxattr    = up_getxattr,
-        .fgetxattr   = up_fgetxattr,
-        .fremovexattr = up_fremovexattr,
-        .removexattr = up_removexattr,
+    .ipc = up_ipc,
+    /* fops which change only "ATIME" do not result
+     * in any cache invalidation. Hence upcall
+     * notifications are not sent in this case.
+     * But however, we need to store/update the
+     * client info in the upcall state to be able
+     * to notify them in case of any changes done
+     * to the data.
+     *
+     * Below such fops do not trigger upcall
+     * notifications but will add/update
+     * clients info in the upcall inode ctx.*/
+    .lookup = up_lookup,
+    .open = up_open,
+    .statfs = up_statfs,
+    .opendir = up_opendir,
+    .readdir = up_readdir,
+    .readdirp = up_readdirp,
+    .stat = up_stat,
+    .fstat = up_fstat,
+    .access = up_access,
+    .readlink = up_readlink,
+    .readv = up_readv,
+    .lk = up_lk,
+    .seek = up_seek,
+
+    /* fops doing  write */
+    .truncate = up_truncate,
+    .ftruncate = up_ftruncate,
+    .writev = up_writev,
+    .zerofill = up_zerofill,
+    .fallocate = up_fallocate,
+    .discard = up_discard,
+
+    /* fops changing attributes */
+    .fsetattr = up_fsetattr,
+    .setattr = up_setattr,
+
+    /* fops affecting parent dirent */
+    .mknod = up_mknod,
+    .create = up_create,
+    .symlink = up_symlink,
+    .mkdir = up_mkdir,
+
+    /* fops affecting both file and parent
+     * cache entries */
+    .unlink = up_unlink,
+    .link = up_link,
+    .rmdir = up_rmdir,
+    .rename = up_rename,
+
+    .setxattr = up_setxattr,
+    .fsetxattr = up_fsetxattr,
+    .getxattr = up_getxattr,
+    .fgetxattr = up_fgetxattr,
+    .fremovexattr = up_fremovexattr,
+    .removexattr = up_removexattr,
+    .xattrop = up_xattrop,
+    .fxattrop = up_fxattrop,
 
 #ifdef NOT_SUPPORTED
-        /* internal lk fops */
-        .inodelk     = up_inodelk,
-        .finodelk    = up_finodelk,
-        .entrylk     = up_entrylk,
-        .fentrylk    = up_fentrylk,
-
-        /* Below fops follow 'WRITE' which
-         * would have already sent upcall
-         * notifications */
-        .flush       = up_flush,
-        .fsync       = up_fsync,
-        .fsyncdir    = up_fsyncdir,
-
-        .xattrop     = up_xattrop,
-        .fxattrop    = up_fxattrop,
+    /* internal lk fops */
+    .inodelk = up_inodelk,
+    .finodelk = up_finodelk,
+    .entrylk = up_entrylk,
+    .fentrylk = up_fentrylk,
+
+    /* Below fops follow 'WRITE' which
+     * would have already sent upcall
+     * notifications */
+    .flush = up_flush,
+    .fsync = up_fsync,
+    .fsyncdir = up_fsyncdir,
 #endif
 };
 
 struct xlator_cbks cbks = {
-        .forget  = upcall_forget,
-        .release = upcall_release,
+    .forget = upcall_forget,
+    .release = upcall_release,
 };
 
 struct volume_options options[] = {
-        { .key  = {"cache-invalidation"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", sends cache-invalidation"
-                         " notifications."
-        },
-        { .key  = {"cache-invalidation-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = CACHE_INVALIDATION_TIMEOUT,
-          .description = "After 'timeout' seconds since the time"
-                         " client accessed any file, cache-invalidation"
-                         " notifications are no longer sent to that client."
-        },
-        { .key = {NULL} },
+    {
+        .key = {"cache-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "When \"on\", sends cache-invalidation"
+                       " notifications.",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"cache", "cacheconsistency", "upcall"},
+    },
+    {.key = {"cache-invalidation-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = CACHE_INVALIDATION_TIMEOUT,
+     .description = "After 'timeout' seconds since the time"
+                    " client accessed any file, cache-invalidation"
+                    " notifications are no longer sent to that client.",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"cache", "cachetimeout", "upcall"}},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "upcall",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/upcall/src/upcall.h b/xlators/features/upcall/src/upcall.h
index 5ad502f4a89..aa535088ad7 100644
--- a/xlators/features/upcall/src/upcall.h
+++ b/xlators/features/upcall/src/upcall.h
@@ -10,123 +10,122 @@
 #ifndef __UPCALL_H__
 #define __UPCALL_H__
 
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "upcall-mem-types.h"
-#include "client_t.h"
+#include <glusterfs/client_t.h>
 #include "upcall-messages.h"
 #include "upcall-cache-invalidation.h"
-#include "upcall-utils.h"
-
-#define EXIT_IF_UPCALL_OFF(this, label) do {                       \
-        if (!is_upcall_enabled(this))                              \
-                goto label;                                        \
-} while (0)
-
-#define UPCALL_STACK_UNWIND(fop, frame, params ...) do {        \
-        upcall_local_t *__local = NULL;                         \
-        xlator_t *__xl          = NULL;                         \
-        if (frame) {                                            \
-                        __xl         = frame->this;             \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-        }                                                       \
-        STACK_UNWIND_STRICT (fop, frame, params);               \
-        upcall_local_wipe (__xl, __local);                      \
-} while (0)
-
-#define UPCALL_STACK_DESTROY(frame) do {                   \
-                upcall_local_t *__local = NULL;            \
-                xlator_t    *__xl    = NULL;               \
-                __xl                 = frame->this;        \
-                __local              = frame->local;       \
-                frame->local         = NULL;               \
-                STACK_DESTROY (frame->root);               \
-                upcall_local_wipe (__xl, __local);         \
-} while (0)
-
-struct _upcall_private_t {
-        gf_boolean_t     cache_invalidation_enabled;
-        int32_t          cache_invalidation_timeout;
-        struct list_head inode_ctx_list;
-        gf_lock_t        inode_ctx_lk;
-        int32_t          reaper_init_done;
-        pthread_t        reaper_thr;
-        int32_t          fini;
+#include <glusterfs/upcall-utils.h>
+
+#define EXIT_IF_UPCALL_OFF(this, label)                                        \
+    do {                                                                       \
+        if (!is_upcall_enabled(this))                                          \
+            goto label;                                                        \
+    } while (0)
+
+#define UPCALL_STACK_UNWIND(fop, frame, params...)                             \
+    do {                                                                       \
+        upcall_local_t *__local = NULL;                                        \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        upcall_local_wipe(__xl, __local);                                      \
+    } while (0)
+
+#define UPCALL_STACK_DESTROY(frame)                                            \
+    do {                                                                       \
+        upcall_local_t *__local = NULL;                                        \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        upcall_local_wipe(__xl, __local);                                      \
+    } while (0)
+
+struct _upcall_private {
+    gf_boolean_t cache_invalidation_enabled;
+    int32_t cache_invalidation_timeout;
+    struct list_head inode_ctx_list;
+    gf_lock_t inode_ctx_lk;
+    gf_boolean_t reaper_init_done;
+    pthread_t reaper_thr;
+    int32_t fini;
+    dict_t *xattrs; /* list of xattrs registered by clients
+                       for receiving invalidation */
 };
-typedef struct _upcall_private_t upcall_private_t;
-
-struct _upcall_client_t {
-        struct list_head client_list;
-        /* strdup to store client_uid, strdup. Free it explicitly */
-        char *client_uid;
-        time_t access_time; /* time last accessed */
-        /* the amount of time which client can cache this entry */
-        uint32_t expire_time_attr;
+typedef struct _upcall_private upcall_private_t;
+
+struct _upcall_client {
+    struct list_head client_list;
+    /* strdup to store client_uid, strdup. Free it explicitly */
+    char *client_uid;
+    time_t access_time; /* time last accessed */
+    /* the amount of time which client can cache this entry */
+    uint32_t expire_time_attr;
 };
-typedef struct _upcall_client_t upcall_client_t;
+typedef struct _upcall_client upcall_client_t;
 
 /* Upcall entries are maintained in inode_ctx */
-struct _upcall_inode_ctx_t {
-        struct list_head inode_ctx_list;
-        struct list_head client_list;
-        pthread_mutex_t client_list_lock; /* mutex for clients list
-                                             of this upcall entry */
-        int destroy;
-        uuid_t   gfid; /* gfid of the entry */
+struct _upcall_inode_ctx {
+    struct list_head inode_ctx_list;
+    struct list_head client_list;
+    pthread_mutex_t client_list_lock; /* mutex for clients list
+                                         of this upcall entry */
+    int destroy;
+    uuid_t gfid; /* gfid of the entry */
 };
-typedef struct _upcall_inode_ctx_t upcall_inode_ctx_t;
+typedef struct _upcall_inode_ctx upcall_inode_ctx_t;
 
 struct upcall_local {
-        /* XXX: need to check if we can store
-         * pointers in 'local' which may get freed
-         * in future by other thread
-         */
-        upcall_inode_ctx_t *upcall_inode_ctx;
-        inode_t   *inode;
-        loc_t     rename_oldloc;
-        dict_t    *xattr;
+    /* XXX: need to check if we can store
+     * pointers in 'local' which may get freed
+     * in future by other thread
+     */
+    inode_t *inode;
+    loc_t rename_oldloc;
+    loc_t loc; /* required for stat in *xattr_cbk */
+    fd_t *fd;  /* required for fstat in *xattr_cbk */
+    dict_t *xattr;
 };
 typedef struct upcall_local upcall_local_t;
 
-void upcall_local_wipe (xlator_t *this, upcall_local_t *local);
-upcall_local_t *upcall_local_init (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode, dict_t *xattr);
-
-upcall_client_t *add_upcall_client (call_frame_t *frame, client_t *client,
-                                    upcall_inode_ctx_t *up_inode_ctx);
-upcall_client_t *__add_upcall_client (call_frame_t *frame, client_t *client,
-                                      upcall_inode_ctx_t *up_inode_ctx);
-upcall_client_t *__get_upcall_client (call_frame_t *frame, client_t *client,
-                                      upcall_inode_ctx_t *up_inode_ctx);
-int __upcall_cleanup_client_entry (upcall_client_t *up_client);
-int upcall_cleanup_expired_clients (xlator_t *this,
-                                    upcall_inode_ctx_t *up_inode_ctx);
-
-int __upcall_inode_ctx_set (inode_t *inode, xlator_t *this);
-upcall_inode_ctx_t *__upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
-upcall_inode_ctx_t *upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
-int upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode);
-void upcall_cache_forget (xlator_t *this, inode_t *inode,
-                          upcall_inode_ctx_t *up_inode_ctx);
-
-void *upcall_reaper_thread (void *data);
-int upcall_reaper_thread_init (xlator_t *this);
+void
+upcall_local_wipe(xlator_t *this, upcall_local_t *local);
+upcall_local_t *
+upcall_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                  inode_t *inode, dict_t *xattr);
+
+upcall_inode_ctx_t *
+upcall_inode_ctx_get(inode_t *inode, xlator_t *this);
+int
+upcall_cleanup_inode_ctx(xlator_t *this, inode_t *inode);
+
+void *
+upcall_reaper_thread(void *data);
+int
+upcall_reaper_thread_init(xlator_t *this);
 
 /* Xlator options */
-gf_boolean_t is_upcall_enabled (xlator_t *this);
+gf_boolean_t
+is_upcall_enabled(xlator_t *this);
 
 /* Cache invalidation specific */
-void upcall_cache_invalidate (call_frame_t *frame, xlator_t *this,
-                              client_t *client, inode_t *inode,
-                              uint32_t flags, struct iatt *stbuf,
-                              struct iatt *p_stbuf,
-                              struct iatt *oldp_stbuf, dict_t *xattr);
-void upcall_client_cache_invalidate (xlator_t *xl, uuid_t gfid,
-                                     upcall_client_t *up_client_entry,
-                                     uint32_t flags, struct iatt *stbuf,
-                                     struct iatt *p_stbuf,
-                                     struct iatt *oldp_stbuf, dict_t *xattr);
-
-int up_filter_virtual_xattr (dict_t *d, char *k, data_t *v, void *tmp);
-
+void
+upcall_cache_invalidate(call_frame_t *frame, xlator_t *this, client_t *client,
+                        inode_t *inode, uint32_t flags, struct iatt *stbuf,
+                        struct iatt *p_stbuf, struct iatt *oldp_stbuf,
+                        dict_t *xattr);
+int
+up_filter_xattr(dict_t *xattr, dict_t *regd_xattrs);
+
+int
+up_compare_afr_xattr(dict_t *d, char *k, data_t *v, void *tmp);
+
+gf_boolean_t
+up_invalidate_needed(dict_t *xattrs);
 #endif /* __UPCALL_H__ */
diff --git a/xlators/storage/bd/Makefile.am b/xlators/features/utime/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/storage/bd/Makefile.am
+++ b/xlators/features/utime/Makefile.am
diff --git a/xlators/features/utime/src/Makefile.am b/xlators/features/utime/src/Makefile.am
new file mode 100644
index 00000000000..7c3adbc2195
--- /dev/null
+++ b/xlators/features/utime/src/Makefile.am
@@ -0,0 +1,41 @@
+xlator_LTLIBRARIES = utime.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+UTIME_SRC = $(top_srcdir)/xlators/features/utime/src
+
+utime_sources  = $(UTIME_SRC)/utime-helpers.c
+utime_sources += $(UTIME_SRC)/utime.c
+
+utime_la_SOURCES = $(utime_sources)
+nodist_utime_la_SOURCES = utime-autogen-fops.c utime-autogen-fops.h
+BUILT_SOURCES = utime-autogen-fops.h
+
+utime_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+utime_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS_utime  = $(UTIME_SRC)/utime-helpers.h
+noinst_HEADERS_utime += $(UTIME_SRC)/utime.h
+noinst_HEADERS_utime += $(UTIME_SRC)/utime-messages.h
+noinst_HEADERS_utime += $(UTIME_SRC)/utime-mem-types.h
+noinst_HEADERS = $(top_srcdir)/xlators/lib/src/libxlator.h
+noinst_HEADERS += $(noinst_HEADERS_utime)
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+noinst_PYTHON	= utime-gen-fops-c.py utime-gen-fops-h.py
+EXTRA_DIST	= utime-autogen-fops-tmpl.c utime-autogen-fops-tmpl.h
+
+utime-autogen-fops.c: utime-gen-fops-c.py utime-autogen-fops-tmpl.c
+	$(PYTHON) $(UTIME_SRC)/utime-gen-fops-c.py $(UTIME_SRC)/utime-autogen-fops-tmpl.c > $@
+
+utime-autogen-fops.h: utime-gen-fops-h.py utime-autogen-fops-tmpl.h
+	$(PYTHON) $(UTIME_SRC)/utime-gen-fops-h.py $(UTIME_SRC)/utime-autogen-fops-tmpl.h > $@
+
+CLEANFILES = $(nodist_utime_la_SOURCES)
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/utime.so
diff --git a/xlators/features/utime/src/utime-autogen-fops-tmpl.c b/xlators/features/utime/src/utime-autogen-fops-tmpl.c
new file mode 100644
index 00000000000..f2f35322926
--- /dev/null
+++ b/xlators/features/utime/src/utime-autogen-fops-tmpl.c
@@ -0,0 +1,28 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* File: utime-autogen-fops-tmpl.c
+ * This file contains the utime autogenerated FOPs. This is run through
+ * the code generator, generator.py to generate the required FOPs.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/statedump.h>
+#include "utime-helpers.h"
+#include <glusterfs/timespec.h>
+
+#pragma generate
diff --git a/xlators/features/utime/src/utime-autogen-fops-tmpl.h b/xlators/features/utime/src/utime-autogen-fops-tmpl.h
new file mode 100644
index 00000000000..4e102ffed6c
--- /dev/null
+++ b/xlators/features/utime/src/utime-autogen-fops-tmpl.h
@@ -0,0 +1,22 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* File: utime-autogen-fops-tmpl.h
+ * This file contains the utime autogenerated FOPs declarations.
+ */
+
+#ifndef _UTIME_AUTOGEN_FOPS_H
+#define _UTIME_AUTOGEN_FOPS_H
+
+#include <glusterfs/xlator.h>
+
+#pragma generate
+
+#endif /* _UTIME_AUTOGEN_FOPS_H */
diff --git a/xlators/features/utime/src/utime-gen-fops-c.py b/xlators/features/utime/src/utime-gen-fops-c.py
new file mode 100755
index 00000000000..9fb3e1b8b1a
--- /dev/null
+++ b/xlators/features/utime/src/utime-gen-fops-c.py
@@ -0,0 +1,147 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, cbk_subs, generate
+
+FOPS_COMMON_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_@UPNAME@);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_CBK_COMMON_TEMPLATE = """
+int32_t
+gf_utime_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                    @LONG_ARGS@)
+{
+        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_READ_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_READ);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_WRITE_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_WRITE);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_COPY_FILE_RANGE_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_COPY_FILE_RANGE);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_SETATTR_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+             @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        if (!valid) {
+                frame->root->flags |= MDATA_CTIME;
+        }
+
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+                frame->root->flags |= MDATA_CTIME;
+        }
+
+        if (valid & GF_SET_ATTR_MODE) {
+                frame->root->flags |= MDATA_CTIME;
+        }
+
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            if (valid & GF_ATTR_ATIME_NOW) {
+                frame->root->ctime.tv_sec = stbuf->ia_atime;
+                frame->root->ctime.tv_nsec = stbuf->ia_atime_nsec;
+            } else if (valid & GF_ATTR_MTIME_NOW) {
+                frame->root->ctime.tv_sec = stbuf->ia_mtime;
+                frame->root->ctime.tv_nsec = stbuf->ia_mtime_nsec;
+            }
+        }
+
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+utime_ops = ['fallocate', 'zerofill', 'opendir', 'mknod', 'mkdir',
+             'unlink', 'rmdir', 'symlink', 'rename', 'link', 'truncate',
+             'ftruncate', 'create', 'open', 'removexattr', 'fremovexattr']
+
+utime_read_op = ['readv']
+utime_write_op = ['writev']
+utime_setattr_ops = ['setattr', 'fsetattr']
+utime_copy_file_range_ops = ['copy_file_range']
+
+def gen_defaults():
+    for name in ops:
+        if name in utime_ops:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_COMMON_TEMPLATE, name, fop_subs))
+        if name in utime_read_op:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_READ_TEMPLATE, name, fop_subs))
+        if name in utime_write_op:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_WRITE_TEMPLATE, name, fop_subs))
+        if name in utime_setattr_ops:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_SETATTR_TEMPLATE, name, fop_subs))
+        if name in utime_copy_file_range_ops:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_COPY_FILE_RANGE_TEMPLATE, name, fop_subs))
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/utime/src/utime-gen-fops-h.py b/xlators/features/utime/src/utime-gen-fops-h.py
new file mode 100755
index 00000000000..e96274c229a
--- /dev/null
+++ b/xlators/features/utime/src/utime-gen-fops-h.py
@@ -0,0 +1,35 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, generate
+
+OP_FOP_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@);
+"""
+
+utime_ops = ['fallocate', 'zerofill', 'opendir', 'mknod', 'mkdir',
+             'unlink', 'rmdir', 'symlink', 'rename', 'link', 'truncate',
+             'ftruncate', 'create', 'open', 'removexattr', 'fremovexattr',
+             'readv', 'writev', 'setattr', 'fsetattr', 'copy_file_range']
+
+def gen_defaults():
+    for name, value in ops.items():
+        if name in utime_ops:
+            print(generate(OP_FOP_TEMPLATE, name, fop_subs))
+
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/utime/src/utime-helpers.c b/xlators/features/utime/src/utime-helpers.c
new file mode 100644
index 00000000000..29d9ad93561
--- /dev/null
+++ b/xlators/features/utime/src/utime-helpers.c
@@ -0,0 +1,110 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "utime-helpers.h"
+#include "utime.h"
+
+void
+gl_timespec_get(struct timespec *ts)
+{
+#ifdef TIME_UTC
+    timespec_get(ts, TIME_UTC);
+#else
+    timespec_now_realtime(ts);
+#endif
+}
+
+void
+utime_update_attribute_flags(call_frame_t *frame, xlator_t *this,
+                             glusterfs_fop_t fop)
+{
+    utime_priv_t *utime_priv = NULL;
+
+    if (!frame || !this) {
+        goto out;
+    }
+
+    utime_priv = this->private;
+
+    switch (fop) {
+        case GF_FOP_SETXATTR:
+        case GF_FOP_FSETXATTR:
+            frame->root->flags |= MDATA_CTIME;
+            break;
+
+        case GF_FOP_FALLOCATE:
+        case GF_FOP_ZEROFILL:
+            frame->root->flags |= MDATA_MTIME;
+            frame->root->flags |= MDATA_ATIME;
+            break;
+
+        case GF_FOP_OPENDIR:
+        case GF_FOP_OPEN:
+        case GF_FOP_READ:
+            if (!utime_priv->noatime) {
+                frame->root->flags |= MDATA_ATIME;
+            }
+            break;
+        case GF_FOP_MKNOD:
+        case GF_FOP_MKDIR:
+        case GF_FOP_SYMLINK:
+        case GF_FOP_CREATE:
+            frame->root->flags |= MDATA_ATIME;
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_MTIME;
+            frame->root->flags |= MDATA_PAR_CTIME;
+            frame->root->flags |= MDATA_PAR_MTIME;
+            break;
+
+        case GF_FOP_UNLINK:
+        case GF_FOP_RMDIR:
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_PAR_CTIME;
+            frame->root->flags |= MDATA_PAR_MTIME;
+            break;
+
+        case GF_FOP_WRITE:
+            frame->root->flags |= MDATA_MTIME;
+            frame->root->flags |= MDATA_CTIME;
+            break;
+
+        case GF_FOP_LINK:
+        case GF_FOP_RENAME:
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_PAR_CTIME;
+            frame->root->flags |= MDATA_PAR_MTIME;
+            break;
+
+        case GF_FOP_TRUNCATE:
+        case GF_FOP_FTRUNCATE:
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_MTIME;
+            break;
+
+        case GF_FOP_REMOVEXATTR:
+        case GF_FOP_FREMOVEXATTR:
+            frame->root->flags |= MDATA_CTIME;
+            break;
+
+        case GF_FOP_COPY_FILE_RANGE:
+            /* Below 2 are for destination fd */
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_MTIME;
+            /* Below flag is for the source fd */
+            if (!utime_priv->noatime) {
+                frame->root->flags |= MDATA_ATIME;
+            }
+            break;
+        default:
+            frame->root->flags = 0;
+    }
+out:
+    return;
+}
diff --git a/xlators/features/utime/src/utime-helpers.h b/xlators/features/utime/src/utime-helpers.h
new file mode 100644
index 00000000000..2e32d4bece6
--- /dev/null
+++ b/xlators/features/utime/src/utime-helpers.h
@@ -0,0 +1,25 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _UTIME_HELPERS_H
+#define _UTIME_HELPERS_H
+
+#include <glusterfs/stack.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/timespec.h>
+#include <time.h>
+
+void
+gl_timespec_get(struct timespec *ts);
+void
+utime_update_attribute_flags(call_frame_t *frame, xlator_t *this,
+                             glusterfs_fop_t fop);
+
+#endif /* _UTIME_HELPERS_H */
diff --git a/xlators/features/utime/src/utime-mem-types.h b/xlators/features/utime/src/utime-mem-types.h
new file mode 100644
index 00000000000..ad1255f85f3
--- /dev/null
+++ b/xlators/features/utime/src/utime-mem-types.h
@@ -0,0 +1,21 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UTIME_MEM_TYPES_H__
+#define __UTIME_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_utime_mem_types_ {
+    utime_mt_utime_t = gf_common_mt_end + 1,
+    utime_mt_end
+};
+
+#endif /* __UTIME_MEM_TYPES_H__ */
diff --git a/xlators/features/utime/src/utime-messages.h b/xlators/features/utime/src/utime-messages.h
new file mode 100644
index 00000000000..bd40265abaf
--- /dev/null
+++ b/xlators/features/utime/src/utime-messages.h
@@ -0,0 +1,29 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UTIME_MESSAGES_H__
+#define __UTIME_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(UTIME, UTIME_MSG_NO_MEMORY, UTIME_MSG_SET_MDATA_FAILED,
+           UTIME_MSG_DICT_SET_FAILED);
+
+#endif /* __UTIME_MESSAGES_H__ */
diff --git a/xlators/features/utime/src/utime.c b/xlators/features/utime/src/utime.c
new file mode 100644
index 00000000000..2acc63e6a05
--- /dev/null
+++ b/xlators/features/utime/src/utime.c
@@ -0,0 +1,392 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "utime.h"
+#include "utime-helpers.h"
+#include "utime-messages.h"
+#include "utime-mem-types.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+gf_utime_invalidate(xlator_t *this, inode_t *inode)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_forget(xlator_t *this, inode_t *inode)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_client_destroy(xlator_t *this, client_t *client)
+{
+    return 0;
+}
+
+void
+gf_utime_ictxmerge(xlator_t *this, fd_t *fd, inode_t *inode,
+                   inode_t *linked_inode)
+{
+    return;
+}
+
+int32_t
+gf_utime_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_client_disconnect(xlator_t *this, client_t *client)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fdctx_to_dict(xlator_t *this, fd_t *fd, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inode(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inode_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_history(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fd(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fd_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fdctx(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inodectx(xlator_t *this, inode_t *ino)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inodectx_to_dict(xlator_t *this, inode_t *ino, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_priv(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    if (xlator_mem_acct_init(this, utime_mt_end + 1) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, UTIME_MSG_NO_MEMORY,
+               "Memory accounting initialization failed.");
+        return -1;
+    }
+    return 0;
+}
+
+int32_t
+gf_utime_set_mdata_setxattr_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int op_ret, int op_errno,
+                                dict_t *xdata)
+{
+    call_stub_t *stub = frame->local;
+    /* Don't fail lookup if mdata setxattr fails */
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, UTIME_MSG_SET_MDATA_FAILED,
+               "dict set of key for set-ctime-mdata failed");
+    }
+    frame->local = NULL;
+    call_resume(stub);
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+
+int32_t
+gf_utime_set_mdata_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *stbuf, dict_t *xdata,
+                              struct iatt *postparent)
+{
+    dict_t *dict = NULL;
+    struct mdata_iatt *mdata = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+
+    if (!op_ret && dict_get(xdata, GF_XATTR_MDATA_KEY) == NULL) {
+        dict = dict_new();
+        if (!dict) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        mdata = GF_MALLOC(sizeof(struct mdata_iatt), gf_common_mt_char);
+        if (mdata == NULL) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        iatt_to_mdata(mdata, stbuf);
+        ret = dict_set_mdata(dict, CTIME_MDATA_XDATA_KEY, mdata, _gf_false);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, UTIME_MSG_NO_MEMORY,
+                   "dict set of key for set-ctime-mdata failed");
+            goto err;
+        }
+        new_frame = copy_frame(frame);
+        if (!new_frame) {
+            op_errno = ENOMEM;
+            goto stub_err;
+        }
+
+        new_frame->local = fop_lookup_cbk_stub(frame, default_lookup_cbk,
+                                               op_ret, op_errno, inode, stbuf,
+                                               xdata, postparent);
+        if (!new_frame->local) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, UTIME_MSG_NO_MEMORY,
+                   "lookup_cbk stub allocation failed");
+            op_errno = ENOMEM;
+            STACK_DESTROY(new_frame->root);
+            goto stub_err;
+        }
+
+        loc.inode = inode_ref(inode);
+        gf_uuid_copy(loc.gfid, stbuf->ia_gfid);
+
+        new_frame->root->uid = 0;
+        new_frame->root->gid = 0;
+        new_frame->root->pid = GF_CLIENT_PID_SET_UTIME;
+        STACK_WIND(new_frame, gf_utime_set_mdata_setxattr_cbk,
+                   FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, &loc,
+                   dict, 0, NULL);
+
+        dict_unref(dict);
+        inode_unref(loc.inode);
+        return 0;
+    }
+
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf, xdata,
+                        postparent);
+    return 0;
+
+err:
+    if (mdata) {
+        GF_FREE(mdata);
+    }
+stub_err:
+    if (dict) {
+        dict_unref(dict);
+    }
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int
+gf_utime_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int op_errno = EINVAL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    xdata = xdata ? dict_ref(xdata) : dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, GF_XATTR_MDATA_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, UTIME_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GF_XATTR_MDATA_KEY);
+        op_errno = -ret;
+        goto free_dict;
+    }
+
+    STACK_WIND(frame, gf_utime_set_mdata_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    dict_unref(xdata);
+    return 0;
+
+free_dict:
+    dict_unref(xdata);
+err:
+    STACK_UNWIND_STRICT(lookup, frame, ret, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    utime_priv_t *utime = NULL;
+
+    utime = GF_MALLOC(sizeof(*utime), utime_mt_utime_t);
+    if (utime == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, UTIME_MSG_NO_MEMORY,
+               "Failed to allocate private memory.");
+        return -1;
+    }
+    memset(utime, 0, sizeof(*utime));
+
+    this->private = utime;
+    GF_OPTION_INIT("noatime", utime->noatime, bool, err);
+
+    return 0;
+err:
+    return -1;
+}
+
+void
+fini(xlator_t *this)
+{
+    utime_priv_t *utime = NULL;
+
+    utime = this->private;
+    GF_FREE(utime);
+    return;
+}
+
+int32_t
+reconfigure(xlator_t *this, dict_t *options)
+{
+    utime_priv_t *utime = this->private;
+
+    GF_OPTION_RECONF("noatime", utime->noatime, options, bool, err);
+
+    return 0;
+err:
+    return -1;
+}
+
+int
+notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+struct xlator_fops fops = {
+    .rename = gf_utime_rename,
+    .mknod = gf_utime_mknod,
+    .readv = gf_utime_readv,
+    .fremovexattr = gf_utime_fremovexattr,
+    .open = gf_utime_open,
+    .create = gf_utime_create,
+    .mkdir = gf_utime_mkdir,
+    .writev = gf_utime_writev,
+    .rmdir = gf_utime_rmdir,
+    .fallocate = gf_utime_fallocate,
+    .truncate = gf_utime_truncate,
+    .symlink = gf_utime_symlink,
+    .zerofill = gf_utime_zerofill,
+    .link = gf_utime_link,
+    .ftruncate = gf_utime_ftruncate,
+    .unlink = gf_utime_unlink,
+    .setattr = gf_utime_setattr,
+    .fsetattr = gf_utime_fsetattr,
+    .opendir = gf_utime_opendir,
+    .removexattr = gf_utime_removexattr,
+    .lookup = gf_utime_lookup,
+};
+struct xlator_cbks cbks = {
+    .invalidate = gf_utime_invalidate,
+    .forget = gf_utime_forget,
+    .client_destroy = gf_utime_client_destroy,
+    .ictxmerge = gf_utime_ictxmerge,
+    .release = gf_utime_release,
+    .releasedir = gf_utime_releasedir,
+    .client_disconnect = gf_utime_client_disconnect,
+};
+struct xlator_dumpops dumpops = {
+    .fdctx_to_dict = gf_utime_fdctx_to_dict,
+    .inode = gf_utime_inode,
+    .inode_to_dict = gf_utime_inode_to_dict,
+    .history = gf_utime_history,
+    .fd = gf_utime_fd,
+    .fd_to_dict = gf_utime_fd_to_dict,
+    .fdctx = gf_utime_fdctx,
+    .inodectx = gf_utime_inodectx,
+    .inodectx_to_dict = gf_utime_inodectx_to_dict,
+    .priv_to_dict = gf_utime_priv_to_dict,
+    .priv = gf_utime_priv,
+};
+
+struct volume_options options[] = {
+    {.key = {"noatime"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_5_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"ctime"},
+     .description = "Enable/Disable atime updation when ctime feature is "
+                    "enabled. When noatime is on, atime is not updated with "
+                    "ctime feature enabled and vice versa."},
+    {.key = {NULL}}};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_5_0},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "utime",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/features/utime/src/utime.h b/xlators/features/utime/src/utime.h
new file mode 100644
index 00000000000..ba55eec00de
--- /dev/null
+++ b/xlators/features/utime/src/utime.h
@@ -0,0 +1,23 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UTIME_H__
+#define __UTIME_H__
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include "utime-autogen-fops.h"
+
+typedef struct utime_priv {
+    gf_boolean_t noatime;
+} utime_priv_t;
+
+#endif /* __UTIME_H__ */
diff --git a/xlators/lib/src/libxlator.c b/xlators/lib/src/libxlator.c
index 627d74070e6..8075fa0c29f 100644
--- a/xlators/lib/src/libxlator.c
+++ b/xlators/lib/src/libxlator.c
@@ -7,506 +7,484 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "mem-types.h"
-#include "libxlator.h"
 
+#include "libxlator.h"
 
 int marker_xtime_default_gauge[] = {
-        [MCNT_FOUND]    =  1,
-        [MCNT_NOTFOUND] = -1,
-        [MCNT_ENODATA]  = -1,
-        [MCNT_ENOTCONN] = -1,
-        [MCNT_ENOENT]   = -1,
-        [MCNT_EOTHER]   = -1,
+    [MCNT_FOUND] = 1,     [MCNT_NOTFOUND] = -1, [MCNT_ENODATA] = -1,
+    [MCNT_ENOTCONN] = -1, [MCNT_ENOENT] = -1,   [MCNT_EOTHER] = -1,
 };
 
 int marker_uuid_default_gauge[] = {
-        [MCNT_FOUND]    =  1,
-        [MCNT_NOTFOUND] =  0,
-        [MCNT_ENODATA]  =  0,
-        [MCNT_ENOTCONN] =  0,
-        [MCNT_ENOENT]   =  0,
-        [MCNT_EOTHER]   =  0,
+    [MCNT_FOUND] = 1,    [MCNT_NOTFOUND] = 0, [MCNT_ENODATA] = 0,
+    [MCNT_ENOTCONN] = 0, [MCNT_ENOENT] = 0,   [MCNT_EOTHER] = 0,
 };
 
 static int marker_idx_errno_map[] = {
-        [MCNT_FOUND]    = EINVAL,
-        [MCNT_NOTFOUND] = EINVAL,
-        [MCNT_ENOENT]   = ENOENT,
-        [MCNT_ENOTCONN] = ENOTCONN,
-        [MCNT_ENODATA]  = ENODATA,
-        [MCNT_EOTHER]   = EINVAL,
-        [MCNT_MAX]      = 0,
+    [MCNT_FOUND] = EINVAL,    [MCNT_NOTFOUND] = EINVAL,
+    [MCNT_ENOENT] = ENOENT,   [MCNT_ENOTCONN] = ENOTCONN,
+    [MCNT_ENODATA] = ENODATA, [MCNT_EOTHER] = EINVAL,
+    [MCNT_MAX] = 0,
 };
 
 /*Copy the contents of oldtimebuf to newtimbuf*/
 static void
-update_timebuf (uint32_t *oldtimbuf, uint32_t *newtimebuf)
+update_timebuf(uint32_t *oldtimbuf, uint32_t *newtimebuf)
 {
-        newtimebuf[0] =  (oldtimbuf[0]);
-        newtimebuf[1] =  (oldtimbuf[1]);
+    newtimebuf[0] = (oldtimbuf[0]);
+    newtimebuf[1] = (oldtimbuf[1]);
 }
 
 /* Convert Timebuf in network order to host order */
 static void
-get_hosttime (uint32_t *oldtimbuf, uint32_t *newtimebuf)
+get_hosttime(uint32_t *oldtimbuf, uint32_t *newtimebuf)
 {
-        newtimebuf[0] = ntohl (oldtimbuf[0]);
-        newtimebuf[1] = ntohl (oldtimbuf[1]);
+    newtimebuf[0] = ntohl(oldtimbuf[0]);
+    newtimebuf[1] = ntohl(oldtimbuf[1]);
 }
 
-
-
 /* Match the Incoming trusted.glusterfs.<uuid>.xtime against volume uuid */
 int
-match_uuid_local (const char *name, char *uuid)
+match_uuid_local(const char *name, char *uuid)
 {
-        if (!uuid || !*uuid)
-                return -1;
+    if (!uuid || !*uuid)
+        return -1;
 
-        name = strtail ((char *)name, MARKER_XATTR_PREFIX);
-        if (!name || name++[0] != '.')
-                return -1;
+    name = strtail((char *)name, MARKER_XATTR_PREFIX);
+    if (!name || name++ [0] != '.')
+        return -1;
 
-        name = strtail ((char *)name, uuid);
-        if (!name || strcmp (name, ".xtime") != 0)
-                return -1;
+    name = strtail((char *)name, uuid);
+    if (!name || strcmp(name, ".xtime") != 0)
+        return -1;
 
-        return 0;
+    return 0;
 }
 
 static void
-marker_local_incr_errcount (xl_marker_local_t *local, int op_errno)
+marker_local_incr_errcount(xl_marker_local_t *local, int op_errno)
 {
-        marker_result_idx_t i = -1;
-
-        if (!local)
-                return;
-
-        switch (op_errno) {
-                case ENODATA:
-                        i = MCNT_ENODATA;
-                        break;
-                case ENOENT:
-                        i = MCNT_ENOENT;
-                        break;
-                case ENOTCONN:
-                        i = MCNT_ENOTCONN;
-                        break;
-                default:
-                        i = MCNT_EOTHER;
-                        break;
-        }
-
-        local->count[i]++;
+    marker_result_idx_t i = -1;
+
+    if (!local)
+        return;
+
+    switch (op_errno) {
+        case ENODATA:
+            i = MCNT_ENODATA;
+            break;
+        case ENOENT:
+            i = MCNT_ENOENT;
+            break;
+        case ENOTCONN:
+            i = MCNT_ENOTCONN;
+            break;
+        default:
+            i = MCNT_EOTHER;
+            break;
+    }
+
+    local->count[i]++;
 }
 
 static int
-evaluate_marker_results (int *gauge, int *count)
+evaluate_marker_results(int *gauge, int *count)
 {
-        int i = 0;
-        int op_errno = 0;
-        gf_boolean_t sane = _gf_true;
-
-        /* check if the policy of the gauge is violated;
-         * if yes, try to get the best errno, ie. look
-         * for the first position where there is a more
-         * specific kind of vioilation than the generic EINVAL
-         */
-        for (i = 0; i < MCNT_MAX; i++) {
-                if (sane) {
-                        if ((gauge[i] > 0 && count[i] < gauge[i]) ||
-                            (gauge[i] < 0 && count[i] >= -gauge[i])) {
-                                sane = _gf_false;
-                                /* generic action: adopt corresponding errno */
-                                op_errno = marker_idx_errno_map[i];
-                        }
-                } else {
-                        /* already insane; trying to get a more informative
-                         * errno by checking subsequent counters
-                         */
-                        if (count[i] > 0)
-                                op_errno = marker_idx_errno_map[i];
-                }
-                if (op_errno && op_errno != EINVAL)
-                        break;
+    int i = 0;
+    int op_errno = 0;
+    gf_boolean_t sane = _gf_true;
+
+    /* check if the policy of the gauge is violated;
+     * if yes, try to get the best errno, ie. look
+     * for the first position where there is a more
+     * specific kind of vioilation than the generic EINVAL
+     */
+    for (i = 0; i < MCNT_MAX; i++) {
+        if (sane) {
+            if ((gauge[i] > 0 && count[i] < gauge[i]) ||
+                (gauge[i] < 0 && count[i] >= -gauge[i])) {
+                sane = _gf_false;
+                /* generic action: adopt corresponding errno */
+                op_errno = marker_idx_errno_map[i];
+            }
+        } else {
+            /* already insane; trying to get a more informative
+             * errno by checking subsequent counters
+             */
+            if (count[i] > 0)
+                op_errno = marker_idx_errno_map[i];
         }
+        if (op_errno && op_errno != EINVAL)
+            break;
+    }
 
-        return op_errno;
+    return op_errno;
 }
 
 static void
-cluster_marker_unwind (call_frame_t *frame, char *key, void *value, size_t size,
-                       dict_t *dict)
+cluster_marker_unwind(call_frame_t *frame, char *key, void *value, size_t size,
+                      dict_t *dict)
 {
-        xl_marker_local_t *local   = frame->local;
-        int               ret      = 0;
-        int32_t           op_ret   = 0;
-        int32_t           op_errno = 0;
-        gf_boolean_t      unref    = _gf_false;
-
-        frame->local = local->xl_local;
-
-        if (local->count[MCNT_FOUND]) {
-                if (!dict) {
-                        dict = dict_new();
-                        if (dict) {
-                                unref = _gf_true;
-                        } else {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-                }
-
-                ret = dict_set_static_bin (dict, key, value, size);
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-        }
-
-        op_errno = evaluate_marker_results (local->gauge, local->count);
-        if (op_errno)
+    xl_marker_local_t *local = frame->local;
+    int ret = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    gf_boolean_t unref = _gf_false;
+
+    frame->local = local->xl_local;
+
+    if (local->count[MCNT_FOUND]) {
+        if (!dict) {
+            dict = dict_new();
+            if (dict) {
+                unref = _gf_true;
+            } else {
                 op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+            }
+        }
 
-out:
-        if (local->xl_specf_unwind) {
-                local->xl_specf_unwind (frame, op_ret,
-                                        op_errno, dict, NULL);
-        } else {
-                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
-                                     dict, NULL);
+        ret = dict_set_static_bin(dict, key, value, size);
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
 
-        GF_FREE (local);
-        if (unref)
-                dict_unref (dict);
+    op_errno = evaluate_marker_results(local->gauge, local->count);
+    if (op_errno)
+        op_ret = -1;
 
+out:
+    if (local->xl_specf_unwind) {
+        local->xl_specf_unwind(frame, op_ret, op_errno, dict, NULL);
+    } else {
+        STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, NULL);
+    }
+
+    GF_FREE(local);
+    if (unref)
+        dict_unref(dict);
 }
 
 /* Aggregate all the <volid>.xtime attrs of the cluster and send the max*/
 int32_t
-cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+cluster_markerxtime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
+    int32_t callcnt = 0;
+    uint32_t *net_timebuf = NULL;
+    uint32_t host_timebuf[2] = {
+        0,
+    };
+    char marker_xattr[128] = {0};
+    xl_marker_local_t *local = NULL;
+
+    local = frame->local;
+
+    snprintf(marker_xattr, sizeof(marker_xattr), "%s.%s.%s",
+             MARKER_XATTR_PREFIX, local->vol_uuid, XTIME);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret) {
+            marker_local_incr_errcount(local, op_errno);
+            goto unlock;
+        }
 
-        int32_t           callcnt         = 0;
-        uint32_t          *net_timebuf    = NULL;
-        uint32_t          host_timebuf[2] = {0,};
-        char              marker_xattr[128]   = {0};
-        xl_marker_local_t *local          = NULL;
-
-        local = frame->local;
-
-        snprintf (marker_xattr, sizeof (marker_xattr), "%s.%s.%s",
-                  MARKER_XATTR_PREFIX, local->vol_uuid, XTIME);
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret) {
-                        marker_local_incr_errcount (local, op_errno);
-                        goto unlock;
-                }
-
-                if (dict_get_ptr (dict, marker_xattr, (void **)&net_timebuf)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to get <uuid>.xtime attr");
-                        local->count[MCNT_NOTFOUND]++;
-                        goto unlock;
-                }
-
-                if (local->count[MCNT_FOUND]) {
-                        get_hosttime (net_timebuf, host_timebuf);
-                        if ( (host_timebuf[0]>local->host_timebuf[0]) ||
-                                (host_timebuf[0] == local->host_timebuf[0] &&
-                                 host_timebuf[1] >= local->host_timebuf[1])) {
-                                update_timebuf (net_timebuf, local->net_timebuf);
-                                update_timebuf (host_timebuf, local->host_timebuf);
-                        }
-
-                } else {
-                        get_hosttime (net_timebuf, local->host_timebuf);
-                        update_timebuf (net_timebuf, local->net_timebuf);
-                        local->count[MCNT_FOUND]++;
-                }
-
+        if (dict_get_ptr(dict, marker_xattr, (void **)&net_timebuf)) {
+            local->count[MCNT_NOTFOUND]++;
+            UNLOCK(&frame->lock);
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Unable to get <uuid>.xtime attr");
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        if (callcnt == 0)
-                cluster_marker_unwind (frame, marker_xattr, local->net_timebuf,
-                                       8, dict);
+        if (local->count[MCNT_FOUND]) {
+            get_hosttime(net_timebuf, host_timebuf);
+            if ((host_timebuf[0] > local->host_timebuf[0]) ||
+                (host_timebuf[0] == local->host_timebuf[0] &&
+                 host_timebuf[1] >= local->host_timebuf[1])) {
+                update_timebuf(net_timebuf, local->net_timebuf);
+                update_timebuf(host_timebuf, local->host_timebuf);
+            }
 
-        return 0;
+        } else {
+            get_hosttime(net_timebuf, local->host_timebuf);
+            update_timebuf(net_timebuf, local->net_timebuf);
+            local->count[MCNT_FOUND]++;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (callcnt == 0)
+        cluster_marker_unwind(frame, marker_xattr, local->net_timebuf, 8, dict);
 
+    return 0;
 }
 
 int32_t
-cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+cluster_markeruuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        int32_t             callcnt     = 0;
-        struct volume_mark  *volmark    = NULL;
-        xl_marker_local_t   *local      = NULL;
-        int32_t             ret         = -1;
-        char                *vol_uuid   = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                vol_uuid = local->vol_uuid;
-
-                if (op_ret) {
-                        marker_local_incr_errcount (local, op_errno);
-                        goto unlock;
-                }
-
-                ret = dict_get_bin (dict, GF_XATTR_MARKER_KEY,
-                                    (void *)&volmark);
-                if (ret)
-                        goto unlock;
-
-                if (local->count[MCNT_FOUND]) {
-                        if ((local->volmark->major != volmark->major) ||
-                            (local->volmark->minor != volmark->minor)) {
-                                op_ret = -1;
-                                op_errno = EINVAL;
-                                goto unlock;
-                        }
-
-                        if (local->retval) {
-                                goto unlock;
-                        } else if (volmark->retval) {
-                                GF_FREE (local->volmark);
-                                local->volmark =
-                                        memdup (volmark, sizeof (*volmark));
-                                local->retval = volmark->retval;
-                        } else if ((volmark->sec > local->volmark->sec) ||
-                                   ((volmark->sec == local->volmark->sec) &&
-                                    (volmark->usec >= local->volmark->usec))) {
-                                GF_FREE (local->volmark);
-                                local->volmark =
-                                      memdup (volmark, sizeof (*volmark));
-                        }
-
-                } else {
-                        local->volmark = memdup (volmark, sizeof (*volmark));
-                        VALIDATE_OR_GOTO (local->volmark, unlock);
-                        gf_uuid_unparse (volmark->uuid, vol_uuid);
-                        if (volmark->retval)
-                                local->retval = volmark->retval;
-                        local->count[MCNT_FOUND]++;
-                }
+    int32_t callcnt = 0;
+    struct volume_mark *volmark = NULL;
+    xl_marker_local_t *local = NULL;
+    int32_t ret = -1;
+    char *vol_uuid = NULL;
+
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        vol_uuid = local->vol_uuid;
+
+        if (op_ret) {
+            marker_local_incr_errcount(local, op_errno);
+            goto unlock;
         }
+
+        ret = dict_get_bin(dict, GF_XATTR_MARKER_KEY, (void *)&volmark);
+        if (ret)
+            goto unlock;
+
+        if (local->count[MCNT_FOUND]) {
+            if ((local->volmark->major != volmark->major) ||
+                (local->volmark->minor != volmark->minor)) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto unlock;
+            }
+
+            if (local->retval) {
+                goto unlock;
+            } else if (volmark->retval) {
+                GF_FREE(local->volmark);
+                local->volmark = gf_memdup(volmark, sizeof(*volmark));
+                local->retval = volmark->retval;
+            } else if ((volmark->sec > local->volmark->sec) ||
+                       ((volmark->sec == local->volmark->sec) &&
+                        (volmark->usec >= local->volmark->usec))) {
+                GF_FREE(local->volmark);
+                local->volmark = gf_memdup(volmark, sizeof(*volmark));
+            }
+
+        } else {
+            local->volmark = gf_memdup(volmark, sizeof(*volmark));
+            VALIDATE_OR_GOTO(local->volmark, unlock);
+            gf_uuid_unparse(volmark->uuid, vol_uuid);
+            if (volmark->retval)
+                local->retval = volmark->retval;
+            local->count[MCNT_FOUND]++;
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (callcnt == 0)
-                cluster_marker_unwind (frame, GF_XATTR_MARKER_KEY,
-                                       local->volmark, sizeof (*local->volmark),
-                                       dict);
+    if (callcnt == 0)
+        cluster_marker_unwind(frame, GF_XATTR_MARKER_KEY, local->volmark,
+                              sizeof(*local->volmark), dict);
 
-        return 0;
+    return 0;
 }
 
 int
-gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+gf_get_min_stime(xlator_t *this, dict_t *dst, char *key, data_t *value)
 {
-        int ret = -1;
-        uint32_t *net_timebuf = NULL;
-        uint32_t *value_timebuf = NULL;
-        uint32_t host_timebuf[2] = {0,};
-        uint32_t host_value_timebuf[2] = {0,};
-
-        /* stime should be minimum of all the other nodes */
-        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+    int ret = -1;
+    uint32_t *net_timebuf = NULL;
+    uint32_t *value_timebuf = NULL;
+    uint32_t host_timebuf[2] = {
+        0,
+    };
+    uint32_t host_value_timebuf[2] = {
+        0,
+    };
+
+    /* stime should be minimum of all the other nodes */
+    ret = dict_get_bin(dst, key, (void **)&net_timebuf);
+    if (ret < 0) {
+        net_timebuf = GF_CALLOC(1, sizeof(int64_t), gf_common_mt_char);
+        if (!net_timebuf)
+            goto out;
+
+        ret = dict_set_bin(dst, key, net_timebuf, sizeof(int64_t));
         if (ret < 0) {
-                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
-                                           gf_common_mt_char);
-                if (!net_timebuf)
-                        goto out;
-
-                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "key=%s: dict set failed", key);
-                        goto error;
-                }
+            gf_log(this->name, GF_LOG_WARNING, "key=%s: dict set failed", key);
+            goto error;
         }
-
-        value_timebuf = data_to_bin (value);
-        if (!value_timebuf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "key=%s: getting value of stime failed", key);
-                ret = -1;
-                goto out;
-        }
-
-        get_hosttime (value_timebuf, host_value_timebuf);
-        get_hosttime (net_timebuf, host_timebuf);
-
-        /* can't use 'min()' macro here as we need to compare two fields
-           in the array, selectively */
-        if ((host_value_timebuf[0] < host_timebuf[0]) ||
-            ((host_value_timebuf[0] == host_timebuf[0]) &&
-             (host_value_timebuf[1] < host_timebuf[1]))) {
-                update_timebuf (value_timebuf, net_timebuf);
-        }
-
-        ret = 0;
+    }
+
+    value_timebuf = data_to_bin(value);
+    if (!value_timebuf) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "key=%s: getting value of stime failed", key);
+        ret = -1;
+        goto out;
+    }
+
+    get_hosttime(value_timebuf, host_value_timebuf);
+    get_hosttime(net_timebuf, host_timebuf);
+
+    /* can't use 'min()' macro here as we need to compare two fields
+       in the array, selectively */
+    if ((host_value_timebuf[0] < host_timebuf[0]) ||
+        ((host_value_timebuf[0] == host_timebuf[0]) &&
+         (host_value_timebuf[1] < host_timebuf[1]))) {
+        update_timebuf(value_timebuf, net_timebuf);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 error:
-        /* To be used only when net_timebuf is not set in the dict */
-        if (net_timebuf)
-                GF_FREE (net_timebuf);
+    /* To be used only when net_timebuf is not set in the dict */
+    if (net_timebuf)
+        GF_FREE(net_timebuf);
 
-        return ret;
+    return ret;
 }
 
 int
-gf_get_max_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+gf_get_max_stime(xlator_t *this, dict_t *dst, char *key, data_t *value)
 {
-        int ret = -ENOMEM;
-        uint32_t *net_timebuf = NULL;
-        uint32_t *value_timebuf = NULL;
-        uint32_t host_timebuf[2] = {0,};
-        uint32_t host_value_timebuf[2] = {0,};
-
-        /* stime should be maximum of all the other nodes */
-        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+    int ret = -ENOMEM;
+    uint32_t *net_timebuf = NULL;
+    uint32_t *value_timebuf = NULL;
+    uint32_t host_timebuf[2] = {
+        0,
+    };
+    uint32_t host_value_timebuf[2] = {
+        0,
+    };
+
+    /* stime should be maximum of all the other nodes */
+    ret = dict_get_bin(dst, key, (void **)&net_timebuf);
+    if (ret < 0) {
+        net_timebuf = GF_CALLOC(1, sizeof(int64_t), gf_common_mt_char);
+        if (!net_timebuf)
+            goto out;
+
+        ret = dict_set_bin(dst, key, net_timebuf, sizeof(int64_t));
         if (ret < 0) {
-                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
-                                           gf_common_mt_char);
-                if (!net_timebuf)
-                        goto out;
-
-                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "key=%s: dict set failed", key);
-                        goto error;
-                }
+            gf_log(this->name, GF_LOG_WARNING, "key=%s: dict set failed", key);
+            goto error;
         }
-
-        value_timebuf = data_to_bin (value);
-        if (!value_timebuf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "key=%s: getting value of stime failed", key);
-                ret = -EINVAL;
-                goto out;
-        }
-
-        get_hosttime (value_timebuf, host_value_timebuf);
-        get_hosttime (net_timebuf, host_timebuf);
-
-        /* can't use 'max()' macro here as we need to compare two fields
-           in the array, selectively */
-        if ((host_value_timebuf[0] > host_timebuf[0]) ||
-            ((host_value_timebuf[0] == host_timebuf[0]) &&
-             (host_value_timebuf[1] > host_timebuf[1]))) {
-                update_timebuf (value_timebuf, net_timebuf);
-        }
-
-        ret = 0;
+    }
+
+    value_timebuf = data_to_bin(value);
+    if (!value_timebuf) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "key=%s: getting value of stime failed", key);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    get_hosttime(value_timebuf, host_value_timebuf);
+    get_hosttime(net_timebuf, host_timebuf);
+
+    /* can't use 'max()' macro here as we need to compare two fields
+       in the array, selectively */
+    if ((host_value_timebuf[0] > host_timebuf[0]) ||
+        ((host_value_timebuf[0] == host_timebuf[0]) &&
+         (host_value_timebuf[1] > host_timebuf[1]))) {
+        update_timebuf(value_timebuf, net_timebuf);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 error:
-        /* To be used only when net_timebuf is not set in the dict */
-        if (net_timebuf)
-                GF_FREE (net_timebuf);
+    /* To be used only when net_timebuf is not set in the dict */
+    if (net_timebuf)
+        GF_FREE(net_timebuf);
 
-        return ret;
+    return ret;
 }
 
 static int
-_get_children_count (xlator_t *xl)
+_get_children_count(xlator_t *xl)
 {
-        int i = 0;
-        xlator_list_t *trav = NULL;
-        for (i = 0, trav = xl->children; trav ; trav = trav->next, i++) {
-                /*'i' will have the value */
-        }
+    int i = 0;
+    xlator_list_t *trav = NULL;
+    for (i = 0, trav = xl->children; trav; trav = trav->next, i++) {
+        /*'i' will have the value */
+    }
 
-        return i;
+    return i;
 }
 
 int
-cluster_handle_marker_getxattr (call_frame_t *frame, loc_t *loc,
-                                const char *name, char *vol_uuid,
-                                xlator_specf_unwind_t unwind,
-                                int (*populate_args) (call_frame_t *frame,
-                                                      int type, int *gauge,
-                                                      xlator_t **subvols))
+cluster_handle_marker_getxattr(call_frame_t *frame, loc_t *loc,
+                               const char *name, char *vol_uuid,
+                               xlator_specf_unwind_t unwind,
+                               int (*populate_args)(call_frame_t *frame,
+                                                    int type, int *gauge,
+                                                    xlator_t **subvols))
 {
-        xlator_t                *this = frame->this;
-        xlator_t                **subvols = NULL;
-        int                     num_subvols = 0;
-        int                     type = 0;
-        int                i     = 0;
-        int                     gauge[MCNT_MAX] = {0};
-        xl_marker_local_t  *local = NULL;
-
-        if (GF_CLIENT_PID_GSYNCD != frame->root->pid)
-                return -EINVAL;
-
-        if (name == NULL)
-                return -EINVAL;
-
-        if (strcmp (GF_XATTR_MARKER_KEY, name) == 0) {
-                type = MARKER_UUID_TYPE;
-                memcpy (gauge, marker_uuid_default_gauge, sizeof (gauge));
-        } else if (match_uuid_local (name, vol_uuid) == 0) {
-                type = MARKER_XTIME_TYPE;
-                memcpy (gauge, marker_xtime_default_gauge, sizeof (gauge));
-        } else {
-                return -EINVAL;
-        }
-
-        num_subvols = _get_children_count (this);
-        subvols = alloca (num_subvols * sizeof (*subvols));
-        num_subvols = populate_args (frame, type, gauge, subvols);
-
-        local = GF_CALLOC (sizeof (struct marker_str), 1,
-                            gf_common_mt_libxl_marker_local);
-
-        if (!local)
-                goto fail;
-
-        local->xl_local = frame->local;
-        local->call_count = num_subvols;
-        local->xl_specf_unwind = unwind;
-        local->vol_uuid = vol_uuid;
-        memcpy (local->gauge, gauge, sizeof (local->gauge));
-
-        frame->local = local;
-
-        for (i = 0; i < num_subvols; i++) {
-                if (MARKER_UUID_TYPE == type)
-                        STACK_WIND (frame, cluster_markeruuid_cbk,
-                                    subvols[i],
-                                    subvols[i]->fops->getxattr,
-                                    loc, name, NULL);
-                else if (MARKER_XTIME_TYPE == type)
-                        STACK_WIND (frame, cluster_markerxtime_cbk,
-                                    subvols[i],
-                                    subvols[i]->fops->getxattr,
-                                    loc, name, NULL);
-        }
-
-        return 0;
+    xlator_t *this = frame->this;
+    xlator_t **subvols = NULL;
+    int num_subvols = 0;
+    int type = 0;
+    int i = 0;
+    int gauge[MCNT_MAX] = {0};
+    xl_marker_local_t *local = NULL;
+
+    if (GF_CLIENT_PID_GSYNCD != frame->root->pid)
+        return -EINVAL;
+
+    if (name == NULL)
+        return -EINVAL;
+
+    if (strcmp(GF_XATTR_MARKER_KEY, name) == 0) {
+        type = MARKER_UUID_TYPE;
+        memcpy(gauge, marker_uuid_default_gauge, sizeof(gauge));
+    } else if (match_uuid_local(name, vol_uuid) == 0) {
+        type = MARKER_XTIME_TYPE;
+        memcpy(gauge, marker_xtime_default_gauge, sizeof(gauge));
+    } else {
+        return -EINVAL;
+    }
+
+    num_subvols = _get_children_count(this);
+    subvols = alloca(num_subvols * sizeof(*subvols));
+    num_subvols = populate_args(frame, type, gauge, subvols);
+
+    local = GF_CALLOC(sizeof(struct marker_str), 1,
+                      gf_common_mt_libxl_marker_local);
+
+    if (!local)
+        goto fail;
+
+    local->xl_local = frame->local;
+    local->call_count = num_subvols;
+    local->xl_specf_unwind = unwind;
+    local->vol_uuid = vol_uuid;
+    memcpy(local->gauge, gauge, sizeof(local->gauge));
+
+    frame->local = local;
+
+    for (i = 0; i < num_subvols; i++) {
+        if (MARKER_UUID_TYPE == type)
+            STACK_WIND(frame, cluster_markeruuid_cbk, subvols[i],
+                       subvols[i]->fops->getxattr, loc, name, NULL);
+        else if (MARKER_XTIME_TYPE == type)
+            STACK_WIND(frame, cluster_markerxtime_cbk, subvols[i],
+                       subvols[i]->fops->getxattr, loc, name, NULL);
+    }
+
+    return 0;
 fail:
-        if (unwind)
-                unwind (frame, -1, ENOMEM, NULL, NULL);
-        else
-                default_getxattr_failure_cbk (frame, ENOMEM);
-        return 0;
+    if (unwind)
+        unwind(frame, -1, ENOMEM, NULL, NULL);
+    else
+        default_getxattr_failure_cbk(frame, ENOMEM);
+    return 0;
 }
diff --git a/xlators/lib/src/libxlator.h b/xlators/lib/src/libxlator.h
index 53ea404cd73..81da4060d55 100644
--- a/xlators/lib/src/libxlator.h
+++ b/xlators/lib/src/libxlator.h
@@ -10,37 +10,35 @@
 #ifndef _LIBXLATOR_H
 #define _LIBXLATOR_H
 
+#include <glusterfs/defaults.h>
 
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat.h"
-#include "compat-errno.h"
-
+#include <stdint.h>             // for int32_t
+#include "glusterfs/dict.h"     // for dict_t, data_t
+#include "glusterfs/globals.h"  // for xlator_t, loc_t
+#include "glusterfs/stack.h"    // for call_frame_t
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
 
 #define MARKER_XATTR_PREFIX "trusted.glusterfs"
-#define XTIME               "xtime"
-#define VOLUME_MARK         "volume-mark"
+#define XTIME "xtime"
+#define VOLUME_MARK "volume-mark"
 #define GF_XATTR_MARKER_KEY MARKER_XATTR_PREFIX "." VOLUME_MARK
 #define UUID_SIZE 36
-#define MARKER_UUID_TYPE    1
-#define MARKER_XTIME_TYPE   2
-
-typedef int32_t (*xlator_specf_unwind_t) (call_frame_t *frame,
-                                          int op_ret, int op_errno,
-                                          dict_t *dict, dict_t *xdata);
+#define MARKER_UUID_TYPE 1
+#define MARKER_XTIME_TYPE 2
 
+typedef int32_t (*xlator_specf_unwind_t)(call_frame_t *frame, int op_ret,
+                                         int op_errno, dict_t *dict,
+                                         dict_t *xdata);
 
 struct volume_mark {
-        uint8_t major;
-        uint8_t minor;
-        uint8_t uuid[16];
-        uint8_t retval;
-        uint32_t sec;
-        uint32_t usec;
-}__attribute__ ((__packed__));
-
+    uint8_t major;
+    uint8_t minor;
+    uint8_t uuid[16];
+    uint8_t retval;
+    uint32_t sec;
+    uint32_t usec;
+} __attribute__((__packed__));
 
 /*
  * The enumerated type here
@@ -84,7 +82,7 @@ struct volume_mark {
  * Cf. evaluate_marker_results() and marker_idx_errno_map[]
  * in libxlator.c
 
- * We provide two default gauges, one inteded for xtime
+ * We provide two default gauges, one intended for xtime
  * aggregation, other for volume mark aggregation. The
  * policies they represent agree with the hard-coded
  * one prior to gauges. Cf. marker_xtime_default_gauge
@@ -92,58 +90,58 @@ struct volume_mark {
  */
 
 typedef enum {
-        MCNT_FOUND,
-        MCNT_NOTFOUND,
-        MCNT_ENODATA,
-        MCNT_ENOTCONN,
-        MCNT_ENOENT,
-        MCNT_EOTHER,
-        MCNT_MAX
+    MCNT_FOUND,
+    MCNT_NOTFOUND,
+    MCNT_ENODATA,
+    MCNT_ENOTCONN,
+    MCNT_ENOENT,
+    MCNT_EOTHER,
+    MCNT_MAX
 } marker_result_idx_t;
 
 extern int marker_xtime_default_gauge[];
 extern int marker_uuid_default_gauge[];
 
 struct marker_str {
-        struct volume_mark    *volmark;
-        data_t                *data;
-
-        uint32_t               host_timebuf[2];
-        uint32_t               net_timebuf[2];
-        int32_t                call_count;
-        int                    gauge[MCNT_MAX];
-        int                    count[MCNT_MAX];
-
-        xlator_specf_unwind_t  xl_specf_unwind;
-        void                  *xl_local;
-        char                  *vol_uuid;
-        uint8_t                retval;
+    struct volume_mark *volmark;
+    data_t *data;
+
+    uint32_t host_timebuf[2];
+    uint32_t net_timebuf[2];
+    int32_t call_count;
+    int gauge[MCNT_MAX];
+    int count[MCNT_MAX];
+
+    xlator_specf_unwind_t xl_specf_unwind;
+    void *xl_local;
+    char *vol_uuid;
+    uint8_t retval;
 };
 
 typedef struct marker_str xl_marker_local_t;
 
 int32_t
-cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
+cluster_markerxtime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
-cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
+cluster_markeruuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int
-cluster_handle_marker_getxattr (call_frame_t *frame, loc_t *loc,
-                                const char *name, char *vol_uuid,
-                                xlator_specf_unwind_t unwind,
-                                int (*populate_args) (call_frame_t *frame,
-                                                      int type, int *gauge,
-                                                      xlator_t **subvols));
+cluster_handle_marker_getxattr(call_frame_t *frame, loc_t *loc,
+                               const char *name, char *vol_uuid,
+                               xlator_specf_unwind_t unwind,
+                               int (*populate_args)(call_frame_t *frame,
+                                                    int type, int *gauge,
+                                                    xlator_t **subvols));
 int
-match_uuid_local (const char *name, char *uuid);
+match_uuid_local(const char *name, char *uuid);
 
 int
-gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
+gf_get_min_stime(xlator_t *this, dict_t *dst, char *key, data_t *value);
 
 int
-gf_get_max_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
+gf_get_max_stime(xlator_t *this, dict_t *dst, char *key, data_t *value);
 
 #endif /* !_LIBXLATOR_H */
diff --git a/xlators/meta/Makefile.am b/xlators/meta/Makefile.am
index e1c45f3051c..af437a64d6d 100644
--- a/xlators/meta/Makefile.am
+++ b/xlators/meta/Makefile.am
@@ -1 +1 @@
-SUBDIRS=src
\ No newline at end of file
+SUBDIRS = src
diff --git a/xlators/meta/src/Makefile.am b/xlators/meta/src/Makefile.am
index df06760b409..29b871d7984 100644
--- a/xlators/meta/src/Makefile.am
+++ b/xlators/meta/src/Makefile.am
@@ -36,7 +36,8 @@ meta_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = meta.h meta-hooks.h meta-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/meta/src/active-link.c b/xlators/meta/src/active-link.c
index dfa26b695b3..7ee780d89e9 100644
--- a/xlators/meta/src/active-link.c
+++ b/xlators/meta/src/active-link.c
@@ -8,32 +8,27 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-active_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+active_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	strprintf (strfd, "%s", this->ctx->active->graph_uuid);
+    strprintf(strfd, "%s", this->ctx->active->graph_uuid);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops active_link_ops = {
-	.link_fill = active_link_fill
-};
-
+struct meta_ops active_link_ops = {.link_fill = active_link_fill};
 
 int
-meta_active_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *xdata)
+meta_active_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &active_link_ops);
+    meta_ops_set(loc->inode, this, &active_link_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/cmdline-file.c b/xlators/meta/src/cmdline-file.c
index 941b8073f4f..eb24e985af9 100644
--- a/xlators/meta/src/cmdline-file.c
+++ b/xlators/meta/src/cmdline-file.c
@@ -8,36 +8,32 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-cmdline_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+cmdline_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-        if (this->ctx->cmdlinestr)
-                strprintf (strfd, "{ \n  \"Cmdlinestr\": \"%s\"\n}",
-                           this->ctx->cmdlinestr);
-        return strfd->size;
+    if (this->ctx->cmdlinestr)
+        strprintf(strfd, "{ \n  \"Cmdlinestr\": \"%s\"\n}",
+                  this->ctx->cmdlinestr);
+    return strfd->size;
 }
 
-
 static struct meta_ops cmdline_file_ops = {
-        .file_fill = cmdline_file_fill,
+    .file_fill = cmdline_file_fill,
 };
 
-
 int
-meta_cmdline_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        dict_t *xdata)
+meta_cmdline_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-        meta_ops_set (loc->inode, this, &cmdline_file_ops);
+    meta_ops_set(loc->inode, this, &cmdline_file_ops);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/frames-file.c b/xlators/meta/src/frames-file.c
index ebac3d9cbaa..9a13db9a934 100644
--- a/xlators/meta/src/frames-file.c
+++ b/xlators/meta/src/frames-file.c
@@ -8,110 +8,100 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-frames_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+frames_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-        struct call_pool *pool = NULL;
-        call_stack_t *stack = NULL;
-        call_frame_t *frame = NULL;
-        int i = 0;
-        int j = 1;
+    struct call_pool *pool = NULL;
+    call_stack_t *stack = NULL;
+    call_frame_t *frame = NULL;
+    int i = 0;
+    int j = 1;
 
-        if (!this || !file || !strfd)
-                return -1;
+    if (!this || !file || !strfd)
+        return -1;
 
-        pool = this->ctx->pool;
+    pool = this->ctx->pool;
 
-        LOCK (&pool->lock);
+    strprintf(strfd, "{ \n\t\"Stack\": [\n");
+
+    LOCK(&pool->lock);
+    {
+        list_for_each_entry(stack, &pool->all_frames, all_frames)
         {
-                strprintf (strfd, "{ \n\t\"Stack\": [\n");
-                list_for_each_entry (stack, &pool->all_frames, all_frames) {
-                        strprintf (strfd, "\t   {\n");
-                        strprintf (strfd, "\t\t\"Number\": %d,\n", ++i);
-                        strprintf (strfd, "\t\t\"Frame\": [\n");
-                        j = 1;
-                        list_for_each_entry (frame, &stack->myframes, frames) {
-                                strprintf (strfd, "\t\t   {\n");
-                                strprintf (strfd, "\t\t\t\"Number\": %d,\n",
-                                                j++);
-                                strprintf (strfd,
-                                                "\t\t\t\"Xlator\": \"%s\",\n",
-                                                frame->this->name);
-                                if (frame->begin.tv_sec)
-                                        strprintf (strfd,
-                                                        "\t\t\t\"Creation_time\": %d.%d,\n",
-                                                        (int)frame->begin.tv_sec,
-                                                        (int)frame->begin.tv_usec);
-                                strprintf (strfd, " \t\t\t\"Refcount\": %d,\n",
-                                                frame->ref_count);
-                                if (frame->parent)
-                                        strprintf (strfd, "\t\t\t\"Parent\": \"%s\",\n",
-                                                        frame->parent->this->name);
-                                if (frame->wind_from)
-                                        strprintf (strfd, "\t\t\t\"Wind_from\": \"%s\",\n",
-                                                        frame->wind_from);
-                                if (frame->wind_to)
-                                        strprintf (strfd, "\t\t\t\"Wind_to\": \"%s\",\n",
-                                                        frame->wind_to);
-                                if (frame->unwind_from)
-                                        strprintf (strfd, "\t\t\t\"Unwind_from\": \"%s\",\n",
-                                                        frame->unwind_from);
-                                if (frame->unwind_to)
-                                        strprintf (strfd, "\t\t\t\"Unwind_to\": \"%s\",\n",
-                                                        frame->unwind_to);
-                                strprintf (strfd, "\t\t\t\"Complete\": %d\n",
-                                                frame->complete);
-                                if (list_is_last (&frame->frames,
-                                                  &stack->myframes))
-                                        strprintf (strfd, "\t\t   }\n");
-                                else
-                                        strprintf (strfd, "\t\t   },\n");
-                        }
-                        strprintf (strfd, "\t\t],\n");
-                        strprintf (strfd, "\t\t\"Unique\": %"PRId64",\n",
-                                        stack->unique);
-                        strprintf (strfd, "\t\t\"Type\": \"%s\",\n",
-                                        gf_fop_list[stack->op]);
-                        strprintf (strfd, "\t\t\"UID\": %d,\n",
-                                        stack->uid);
-                        strprintf (strfd, "\t\t\"GID\": %d,\n",
-                                        stack->gid);
-                        strprintf (strfd, "\t\t\"LK_owner\": \"%s\"\n",
-                                        lkowner_utoa (&stack->lk_owner));
-                        if (i == (int)pool->cnt)
-                                strprintf (strfd, "\t   }\n");
-                        else
-                                strprintf (strfd, "\t   },\n");
-                }
-                strprintf (strfd, "\t],\n");
-                strprintf (strfd, "\t\"Call_Count\": %d\n",
-                                (int)pool->cnt);
-                strprintf (strfd, "}");
+            strprintf(strfd, "\t   {\n");
+            strprintf(strfd, "\t\t\"Number\": %d,\n", ++i);
+            strprintf(strfd, "\t\t\"Frame\": [\n");
+            j = 1;
+            list_for_each_entry(frame, &stack->myframes, frames)
+            {
+                strprintf(strfd, "\t\t   {\n");
+                strprintf(strfd, "\t\t\t\"Number\": %d,\n", j++);
+                strprintf(strfd, "\t\t\t\"Xlator\": \"%s\",\n",
+                          frame->this->name);
+                if (frame->begin.tv_sec)
+                    strprintf(strfd, "\t\t\t\"Creation_time\": %d.%09d,\n",
+                              (int)frame->begin.tv_sec,
+                              (int)frame->begin.tv_nsec);
+                strprintf(strfd, " \t\t\t\"Refcount\": %d,\n",
+                          frame->ref_count);
+                if (frame->parent)
+                    strprintf(strfd, "\t\t\t\"Parent\": \"%s\",\n",
+                              frame->parent->this->name);
+                if (frame->wind_from)
+                    strprintf(strfd, "\t\t\t\"Wind_from\": \"%s\",\n",
+                              frame->wind_from);
+                if (frame->wind_to)
+                    strprintf(strfd, "\t\t\t\"Wind_to\": \"%s\",\n",
+                              frame->wind_to);
+                if (frame->unwind_from)
+                    strprintf(strfd, "\t\t\t\"Unwind_from\": \"%s\",\n",
+                              frame->unwind_from);
+                if (frame->unwind_to)
+                    strprintf(strfd, "\t\t\t\"Unwind_to\": \"%s\",\n",
+                              frame->unwind_to);
+                strprintf(strfd, "\t\t\t\"Complete\": %d\n", frame->complete);
+                if (list_is_last(&frame->frames, &stack->myframes))
+                    strprintf(strfd, "\t\t   }\n");
+                else
+                    strprintf(strfd, "\t\t   },\n");
+            }
+            strprintf(strfd, "\t\t],\n");
+            strprintf(strfd, "\t\t\"Unique\": %" PRId64 ",\n", stack->unique);
+            strprintf(strfd, "\t\t\"Type\": \"%s\",\n", gf_fop_list[stack->op]);
+            strprintf(strfd, "\t\t\"UID\": %d,\n", stack->uid);
+            strprintf(strfd, "\t\t\"GID\": %d,\n", stack->gid);
+            strprintf(strfd, "\t\t\"LK_owner\": \"%s\"\n",
+                      lkowner_utoa(&stack->lk_owner));
+            if (i == (int)pool->cnt)
+                strprintf(strfd, "\t   }\n");
+            else
+                strprintf(strfd, "\t   },\n");
         }
-        UNLOCK (&pool->lock);
+        strprintf(strfd, "\t],\n");
+        strprintf(strfd, "\t\"Call_Count\": %d\n", (int)pool->cnt);
+        strprintf(strfd, "}");
+    }
+    UNLOCK(&pool->lock);
 
-        return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops frames_file_ops = {
-        .file_fill = frames_file_fill,
+    .file_fill = frames_file_fill,
 };
 
-
 int
-meta_frames_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       dict_t *xdata)
+meta_frames_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-        meta_ops_set (loc->inode, this, &frames_file_ops);
-        return 0;
+    meta_ops_set(loc->inode, this, &frames_file_ops);
+    return 0;
 }
diff --git a/xlators/meta/src/graph-dir.c b/xlators/meta/src/graph-dir.c
index 541e806ddb5..a8f4787880d 100644
--- a/xlators/meta/src/graph-dir.c
+++ b/xlators/meta/src/graph-dir.c
@@ -8,94 +8,91 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent graph_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "top",
-	  .type = IA_IFLNK,
-	  .hook = meta_top_link_hook,
-	},
-	{ .name = "volfile",
-	  .type = IA_IFREG,
-	  .hook = meta_volfile_file_hook,
-	},
-	{ .name = NULL }
-};
-
+    DOT_DOTDOT,
+
+    {
+        .name = "top",
+        .type = IA_IFLNK,
+        .hook = meta_top_link_hook,
+    },
+    {
+        .name = "volfile",
+        .type = IA_IFREG,
+        .hook = meta_volfile_file_hook,
+    },
+    {.name = NULL}};
 
 static int
-graph_dir_fill (xlator_t *this, inode_t *inode, struct meta_dirent **dp)
+graph_dir_fill(xlator_t *this, inode_t *inode, struct meta_dirent **dp)
 {
-	struct meta_dirent *dirents = NULL;
-	glusterfs_graph_t *graph = NULL;
-	int i = 0;
-	int count = 0;
-	xlator_t *xl = NULL;
-
-	graph = meta_ctx_get (inode, this);
-
-	for (xl = graph->first; xl; xl = xl->next)
-		count++;
-
-	dirents = GF_CALLOC (sizeof (*dirents), count, gf_meta_mt_dirents_t);
-	if (!dirents)
-		return -1;
-
-	i = 0;
-	for (xl = graph->first; xl; xl = xl->next) {
-		dirents[i].name = gf_strdup (xl->name);
-		dirents[i].type = IA_IFDIR;
-		dirents[i].hook = meta_xlator_dir_hook;
-		i++;
-	}
-
-	*dp = dirents;
-	return i;
+    struct meta_dirent *dirents = NULL;
+    glusterfs_graph_t *graph = NULL;
+    int i = 0;
+    int count = 0;
+    xlator_t *xl = NULL;
+
+    graph = meta_ctx_get(inode, this);
+
+    for (xl = graph->first; xl; xl = xl->next)
+        count++;
+
+    dirents = GF_MALLOC(sizeof(*dirents) * count, gf_meta_mt_dirents_t);
+    if (!dirents)
+        return -1;
+
+    i = 0;
+    for (xl = graph->first; xl; xl = xl->next) {
+        dirents[i].name = gf_strdup(xl->name);
+        dirents[i].type = IA_IFDIR;
+        dirents[i].hook = meta_xlator_dir_hook;
+        i++;
+    }
+
+    *dp = dirents;
+    return i;
 }
 
-
 struct meta_ops graph_dir_ops = {
-	.fixed_dirents = graph_dir_dirents,
-	.dir_fill = graph_dir_fill,
+    .fixed_dirents = graph_dir_dirents,
+    .dir_fill = graph_dir_fill,
 };
 
-
 static glusterfs_graph_t *
-glusterfs_graph_lookup (xlator_t *this, const char *graph_uuid)
+glusterfs_graph_lookup(xlator_t *this, const char *graph_uuid)
 {
-	glusterfs_graph_t *graph = NULL;
-	glusterfs_graph_t *tmp = NULL;
-
-	list_for_each_entry (tmp, &this->ctx->graphs, list) {
-		if (strcmp (graph_uuid, tmp->graph_uuid) == 0) {
-			graph = tmp;
-			break;
-		}
-	}
-
-	return graph;
+    glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    list_for_each_entry(tmp, &this->ctx->graphs, list)
+    {
+        if (strcmp(graph_uuid, tmp->graph_uuid) == 0) {
+            graph = tmp;
+            break;
+        }
+    }
+
+    return graph;
 }
 
-
 int
-meta_graph_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_graph_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *graph = NULL;
 
-	graph = glusterfs_graph_lookup (this, loc->name);
+    graph = glusterfs_graph_lookup(this, loc->name);
 
-	meta_ops_set (loc->inode, this, &graph_dir_ops);
+    meta_ops_set(loc->inode, this, &graph_dir_ops);
 
-	meta_ctx_set (loc->inode, this, (void *) graph);
+    meta_ctx_set(loc->inode, this, (void *)graph);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/graphs-dir.c b/xlators/meta/src/graphs-dir.c
index e5f1319ec26..a1ffbca7d5a 100644
--- a/xlators/meta/src/graphs-dir.c
+++ b/xlators/meta/src/graphs-dir.c
@@ -8,67 +8,60 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent graphs_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "active",
-	  .type = IA_IFLNK,
-	  .hook = meta_active_link_hook,
-	},
-	{ .name = NULL }
-};
+    DOT_DOTDOT,
 
+    {
+        .name = "active",
+        .type = IA_IFLNK,
+        .hook = meta_active_link_hook,
+    },
+    {.name = NULL}};
 
 static int
-graphs_dir_fill (xlator_t *this, inode_t *dir, struct meta_dirent **dp)
+graphs_dir_fill(xlator_t *this, inode_t *dir, struct meta_dirent **dp)
 {
-	glusterfs_graph_t *graph = NULL;
-	int graphs_count = 0;
-	int i = 0;
-	struct meta_dirent *dirents = NULL;
-
-	list_for_each_entry (graph, &this->ctx->graphs, list) {
-		graphs_count++;
-	}
-
-	dirents = GF_CALLOC (sizeof (*dirents), graphs_count + 3,
-			     gf_meta_mt_dirents_t);
-	if (!dirents)
-		return -1;
-
-	i = 0;
-	list_for_each_entry (graph, &this->ctx->graphs, list) {
-		dirents[i].name = gf_strdup (graph->graph_uuid);
-		dirents[i].type = IA_IFDIR;
-		dirents[i].hook = meta_graph_dir_hook;
-		i++;
-	}
-
-	*dp = dirents;
-
-	return i;
+    glusterfs_graph_t *graph = NULL;
+    int graphs_count = 0;
+    int i = 0;
+    struct meta_dirent *dirents = NULL;
+
+    list_for_each_entry(graph, &this->ctx->graphs, list) { graphs_count++; }
+
+    dirents = GF_CALLOC(sizeof(*dirents), graphs_count + 3,
+                        gf_meta_mt_dirents_t);
+    if (!dirents)
+        return -1;
+
+    i = 0;
+    list_for_each_entry(graph, &this->ctx->graphs, list)
+    {
+        dirents[i].name = gf_strdup(graph->graph_uuid);
+        dirents[i].type = IA_IFDIR;
+        dirents[i].hook = meta_graph_dir_hook;
+        i++;
+    }
+
+    *dp = dirents;
+
+    return i;
 }
 
-
-struct meta_ops graphs_dir_ops = {
-	.fixed_dirents = graphs_dir_dirents,
-	.dir_fill = graphs_dir_fill
-};
-
+struct meta_ops graphs_dir_ops = {.fixed_dirents = graphs_dir_dirents,
+                                  .dir_fill = graphs_dir_fill};
 
 int
-meta_graphs_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      dict_t *xdata)
+meta_graphs_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &graphs_dir_ops);
+    meta_ops_set(loc->inode, this, &graphs_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/history-file.c b/xlators/meta/src/history-file.c
index eadc9821f83..7742a635fed 100644
--- a/xlators/meta/src/history-file.c
+++ b/xlators/meta/src/history-file.c
@@ -8,40 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-history_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+history_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_history (xl, strfd);
+    gf_proc_dump_xlator_history(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops history_file_ops = {
-	.file_fill = history_file_fill,
+    .file_fill = history_file_fill,
 };
 
-
 int
-meta_history_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_history_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &history_file_ops);
+    meta_ops_set(loc->inode, this, &history_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/logfile-link.c b/xlators/meta/src/logfile-link.c
index d7b16b92eae..616a54518c0 100644
--- a/xlators/meta/src/logfile-link.c
+++ b/xlators/meta/src/logfile-link.c
@@ -8,32 +8,27 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-logfile_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+logfile_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	strprintf (strfd, "%s", this->ctx->log.filename);
+    strprintf(strfd, "%s", this->ctx->log.filename);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops logfile_link_ops = {
-	.link_fill = logfile_link_fill
-};
-
+struct meta_ops logfile_link_ops = {.link_fill = logfile_link_fill};
 
 int
-meta_logfile_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_logfile_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &logfile_link_ops);
+    meta_ops_set(loc->inode, this, &logfile_link_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/logging-dir.c b/xlators/meta/src/logging-dir.c
index cfd0c123308..46e6f9e95dd 100644
--- a/xlators/meta/src/logging-dir.c
+++ b/xlators/meta/src/logging-dir.c
@@ -8,39 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent logging_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "logfile",
-	  .type = IA_IFLNK,
-	  .hook = meta_logfile_link_hook,
-	},
-	{ .name = "loglevel",
-	  .type = IA_IFREG,
-	  .hook = meta_loglevel_file_hook,
-	},
-	{ .name = NULL }
-};
-
+    DOT_DOTDOT,
+
+    {
+        .name = "logfile",
+        .type = IA_IFLNK,
+        .hook = meta_logfile_link_hook,
+    },
+    {
+        .name = "loglevel",
+        .type = IA_IFREG,
+        .hook = meta_loglevel_file_hook,
+    },
+    {.name = NULL}};
 
 struct meta_ops logging_dir_ops = {
-	.fixed_dirents = logging_dir_dirents,
+    .fixed_dirents = logging_dir_dirents,
 };
 
-
 int
-meta_logging_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_logging_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &logging_dir_ops);
+    meta_ops_set(loc->inode, this, &logging_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/loglevel-file.c b/xlators/meta/src/loglevel-file.c
index f9c5a993d73..eeeeeaa5907 100644
--- a/xlators/meta/src/loglevel-file.c
+++ b/xlators/meta/src/loglevel-file.c
@@ -8,47 +8,43 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-
+#include <glusterfs/strfd.h>
 
 static int
-loglevel_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+loglevel_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	strprintf (strfd, "%d\n", this->ctx->log.loglevel);
+    strprintf(strfd, "%d\n", this->ctx->log.loglevel);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static int
-loglevel_file_write (xlator_t *this, fd_t *fd, struct iovec *iov, int count)
+loglevel_file_write(xlator_t *this, fd_t *fd, struct iovec *iov, int count)
 {
-	long int level = -1;
+    long int level = -1;
 
-	level = strtol (iov[0].iov_base, NULL, 0);
-	if (level >= GF_LOG_NONE && level <= GF_LOG_TRACE)
-		gf_log_set_loglevel (level);
+    level = strtol(iov[0].iov_base, NULL, 0);
+    if (level >= GF_LOG_NONE && level <= GF_LOG_TRACE)
+        gf_log_set_loglevel(this->ctx, level);
 
-	return iov_length (iov, count);
+    return iov_length(iov, count);
 }
 
-
 static struct meta_ops loglevel_file_ops = {
-	.file_fill = loglevel_file_fill,
-	.file_write = loglevel_file_write,
+    .file_fill = loglevel_file_fill,
+    .file_write = loglevel_file_write,
 };
 
-
 int
-meta_loglevel_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			 dict_t *xdata)
+meta_loglevel_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &loglevel_file_ops);
+    meta_ops_set(loc->inode, this, &loglevel_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/mallinfo-file.c b/xlators/meta/src/mallinfo-file.c
index a1aec25e3a6..b4396d72189 100644
--- a/xlators/meta/src/mallinfo-file.c
+++ b/xlators/meta/src/mallinfo-file.c
@@ -8,32 +8,29 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "statedump.h"
-
+#include <glusterfs/statedump.h>
 
 static int
-mallinfo_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+mallinfo_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	gf_proc_dump_mallinfo (strfd);
-	return strfd->size;
+    gf_proc_dump_mallinfo(strfd);
+    return strfd->size;
 }
 
-
 static struct meta_ops mallinfo_file_ops = {
-	.file_fill = mallinfo_file_fill,
+    .file_fill = mallinfo_file_fill,
 };
 
-
 int
-meta_mallinfo_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_mallinfo_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &mallinfo_file_ops);
+    meta_ops_set(loc->inode, this, &mallinfo_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/measure-file.c b/xlators/meta/src/measure-file.c
index 7fe9ff390c0..52e92e48590 100644
--- a/xlators/meta/src/measure-file.c
+++ b/xlators/meta/src/measure-file.c
@@ -8,45 +8,42 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-
+#include <glusterfs/strfd.h>
 
 static int
-measure_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+measure_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	strprintf (strfd, "%d\n", this->ctx->measure_latency);
+    strprintf(strfd, "%d\n", this->ctx->measure_latency);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static int
-measure_file_write (xlator_t *this, fd_t *fd, struct iovec *iov, int count)
+measure_file_write(xlator_t *this, fd_t *fd, struct iovec *iov, int count)
 {
-	long int num = -1;
+    long int num = -1;
 
-	num = strtol (iov[0].iov_base, NULL, 0);
-	this->ctx->measure_latency = !!num;
+    num = strtol(iov[0].iov_base, NULL, 0);
+    this->ctx->measure_latency = !!num;
 
-	return iov_length (iov, count);
+    return iov_length(iov, count);
 }
 
 static struct meta_ops measure_file_ops = {
-	.file_fill = measure_file_fill,
-	.file_write = measure_file_write,
+    .file_fill = measure_file_fill,
+    .file_write = measure_file_write,
 };
 
-
 int
-meta_measure_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_measure_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &measure_file_ops);
+    meta_ops_set(loc->inode, this, &measure_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/meminfo-file.c b/xlators/meta/src/meminfo-file.c
index 900976ada3b..d889dfb2ae8 100644
--- a/xlators/meta/src/meminfo-file.c
+++ b/xlators/meta/src/meminfo-file.c
@@ -8,40 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-meminfo_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+meminfo_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_meminfo (xl, strfd);
+    gf_proc_dump_xlator_meminfo(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops meminfo_file_ops = {
-	.file_fill = meminfo_file_fill,
+    .file_fill = meminfo_file_fill,
 };
 
-
 int
-meta_meminfo_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_meminfo_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &meminfo_file_ops);
+    meta_ops_set(loc->inode, this, &meminfo_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/meta-defaults.c b/xlators/meta/src/meta-defaults.c
index 5a8558291ba..91c328473f8 100644
--- a/xlators/meta/src/meta-defaults.c
+++ b/xlators/meta/src/meta-defaults.c
@@ -8,629 +8,648 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 
 int
-meta_default_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			const char *name, dict_t *xdata)
+meta_default_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-        return default_fgetxattr_failure_cbk (frame, EPERM);
+    return default_fgetxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			dict_t *dict, int32_t flags, dict_t *xdata)
+meta_default_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        return default_fsetxattr_failure_cbk (frame, EPERM);
+    return default_fsetxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *dict, int32_t flags, dict_t *xdata)
+meta_default_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        return default_setxattr_failure_cbk (frame, EPERM);
+    return default_setxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_default_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-        return default_statfs_failure_cbk (frame, EPERM);
+    return default_statfs_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       int32_t flags, dict_t *xdata)
+meta_default_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      int32_t flags, dict_t *xdata)
 {
-        return default_fsyncdir_failure_cbk (frame, EPERM);
+    return default_fsyncdir_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      fd_t *fd, dict_t *xdata)
+meta_default_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                     dict_t *xdata)
 {
-	META_STACK_UNWIND (opendir, frame, 0, 0, fd, xdata);
-	return 0;
+    META_STACK_UNWIND(opendir, frame, 0, 0, fd, xdata);
+    return 0;
 }
 
 int
-meta_default_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    dict_t *xdata)
+meta_default_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, fd->inode, fd->inode->ia_type);
+    meta_iatt_fill(&iatt, fd->inode, fd->inode->ia_type);
 
-	META_STACK_UNWIND (fstat, frame, 0, 0, &iatt, xdata);
+    META_STACK_UNWIND(fstat, frame, 0, 0, &iatt, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    int32_t flags, dict_t *xdata)
+meta_default_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                   dict_t *xdata)
 {
-        return default_fsync_failure_cbk (frame, EPERM);
+    return default_fsync_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_flush (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    dict_t *xdata)
+meta_default_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	META_STACK_UNWIND (flush, frame, 0, 0, xdata);
-	return 0;
+    META_STACK_UNWIND(flush, frame, 0, 0, xdata);
+    return 0;
 }
 
 int
-meta_default_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     struct iovec *vector, int32_t count, off_t off,
-		     uint32_t flags, struct iobref *iobref, dict_t *xdata)
+meta_default_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    struct iovec *vector, int32_t count, off_t off,
+                    uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-	struct meta_ops *ops = NULL;
-	int ret = 0;
-	struct iatt dummy = { };
+    struct meta_ops *ops = NULL;
+    int ret = 0;
+    struct iatt dummy = {};
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops)
-		goto err;
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops)
+        goto err;
 
-	if (!ops->file_write)
-		goto err;
+    if (!ops->file_write)
+        goto err;
 
-	ret = ops->file_write (this, fd, vector, count);
+    ret = ops->file_write(this, fd, vector, count);
 
-	META_STACK_UNWIND (writev, frame, (ret >= 0 ? ret : -1), (ret < 0 ? -ret : 0),
-			   &dummy, &dummy, xdata);
-	return 0;
+    META_STACK_UNWIND(writev, frame, (ret >= 0 ? ret : -1),
+                      (ret < 0 ? -ret : 0), &dummy, &dummy, xdata);
+    return 0;
 err:
-        return default_writev_failure_cbk (frame, EPERM);
+    return default_writev_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-		    off_t offset, uint32_t flags, dict_t *xdata)
+meta_default_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                   off_t offset, uint32_t flags, dict_t *xdata)
 {
-	meta_fd_t *meta_fd = NULL;
-	struct iovec iov = {};
-	struct iobuf *iobuf = NULL;
-	struct iobref *iobref = NULL;
-	off_t copy_offset = 0;
-	size_t copy_size = 0;
-	struct iatt iatt = {};
+    meta_fd_t *meta_fd = NULL;
+    struct iovec iov = {};
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    off_t copy_offset = 0;
+    int copy_size = 0;
+    struct iatt iatt = {};
 
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        return default_readv_failure_cbk(frame, ENODATA);
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		return default_readv_failure_cbk (frame, ENODATA);
+    if (!meta_fd->size)
+        meta_file_fill(this, fd);
 
-	if (!meta_fd->size)
-		meta_file_fill (this, fd);
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, size);
+    if (!iobuf)
+        return default_readv_failure_cbk(frame, ENOMEM);
 
-	iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
-	if (!iobuf)
-		return default_readv_failure_cbk (frame, ENOMEM);
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        return default_readv_failure_cbk(frame, ENOMEM);
+    }
 
-	iobref = iobref_new ();
-	if (!iobref) {
-		iobuf_unref (iobuf);
-		return default_readv_failure_cbk (frame, ENOMEM);
-	}
+    if (iobref_add(iobref, iobuf) != 0) {
+        iobref_unref(iobref);
+        iobuf_unref(iobuf);
+        return default_readv_failure_cbk(frame, ENOMEM);
+    }
 
-	if (iobref_add (iobref, iobuf) != 0) {
-		iobref_unref (iobref);
-		iobuf_unref (iobuf);
-		return default_readv_failure_cbk (frame, ENOMEM);
-	}
+    iov.iov_base = iobuf_ptr(iobuf);
 
-	iov.iov_base = iobuf_ptr (iobuf);
+    /* iobref would have taken a ref */
+    iobuf_unref(iobuf);
 
-	copy_offset = min (meta_fd->size, offset);
-	copy_size = min (size, (meta_fd->size - copy_offset));
+    copy_offset = min(meta_fd->size, offset);
+    copy_size = min(size, (meta_fd->size - copy_offset));
 
-	if (copy_size)
-		memcpy (iov.iov_base, meta_fd->data + copy_offset, copy_size);
-	iov.iov_len = copy_size;
+    if (copy_size)
+        memcpy(iov.iov_base, meta_fd->data + copy_offset, copy_size);
+    iov.iov_len = copy_size;
 
-	META_STACK_UNWIND (readv, frame, copy_size, 0, &iov, 1, &iatt, iobref, 0);
+    META_STACK_UNWIND(readv, frame, copy_size, 0, &iov, 1, &iatt, iobref, 0);
 
-	return 0;
-}
+    iobref_unref(iobref);
 
+    return 0;
+}
 
 int
-meta_default_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   int32_t flags, fd_t *fd, dict_t *xdata)
+meta_default_open(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  int32_t flags, fd_t *fd, dict_t *xdata)
 {
-	dict_t *xdata_rsp = NULL;
+    dict_t *xdata_rsp = NULL;
 
-	xdata_rsp = meta_direct_io_mode (xdata, frame);
+    xdata_rsp = meta_direct_io_mode(xdata, frame);
 
-	META_STACK_UNWIND (open, frame, 0, 0, fd, xdata_rsp);
+    META_STACK_UNWIND(open, frame, 0, 0, fd, xdata_rsp);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-		     dict_t *xdata)
+meta_default_create(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                    dict_t *xdata)
 {
-        return default_create_failure_cbk (frame, EPERM);
+    return default_create_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-		   loc_t *newloc, dict_t *xdata)
+meta_default_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                  loc_t *newloc, dict_t *xdata)
 {
-        return default_link_failure_cbk (frame, EPERM);
+    return default_link_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-		     loc_t *newloc, dict_t *xdata)
+meta_default_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                    loc_t *newloc, dict_t *xdata)
 {
-        return default_rename_failure_cbk (frame, EPERM);
+    return default_rename_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		      loc_t *loc, mode_t umask, dict_t *xdata)
+meta_default_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                     loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        return default_symlink_failure_cbk (frame, EPERM);
+    return default_symlink_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-		    dict_t *xdata)
+meta_default_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                   dict_t *xdata)
 {
-        return default_rmdir_failure_cbk (frame, EPERM);
+    return default_rmdir_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-		     dict_t *xdata)
+meta_default_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                    dict_t *xdata)
 {
-        return default_unlink_failure_cbk (frame, EPERM);
+    return default_unlink_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    mode_t mode, mode_t umask, dict_t *xdata)
+meta_default_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   mode_t umask, dict_t *xdata)
 {
-        return default_mkdir_failure_cbk (frame, EPERM);
+    return default_mkdir_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+meta_default_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        return default_mknod_failure_cbk (frame, EPERM);
+    return default_mknod_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       size_t size, dict_t *xdata)
+meta_default_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      size_t size, dict_t *xdata)
 {
-	struct meta_ops *ops = NULL;
-	strfd_t *strfd = NULL;
-	struct iatt iatt = { };
+    struct meta_ops *ops = NULL;
+    strfd_t *strfd = NULL;
+    struct iatt iatt = {};
+    int len = -1;
 
-	ops = meta_ops_get (loc->inode, this);
-	if (!ops->link_fill) {
-		META_STACK_UNWIND (readlink, frame, -1, EPERM, 0, 0, 0);
-		return 0;
-	}
+    ops = meta_ops_get(loc->inode, this);
+    if (!ops || !ops->link_fill) {
+        META_STACK_UNWIND(readlink, frame, -1, EPERM, 0, 0, 0);
+        return 0;
+    }
 
-	strfd = strfd_open ();
-	if (!strfd) {
-		META_STACK_UNWIND (readlink, frame, -1, ENOMEM, 0, 0, 0);
-		return 0;
-	}
+    strfd = strfd_open();
+    if (!strfd) {
+        META_STACK_UNWIND(readlink, frame, -1, ENOMEM, 0, 0, 0);
+        return 0;
+    }
 
-	ops->link_fill (this, loc->inode, strfd);
+    ops->link_fill(this, loc->inode, strfd);
 
-	meta_iatt_fill (&iatt, loc->inode, IA_IFLNK);
+    meta_iatt_fill(&iatt, loc->inode, IA_IFLNK);
 
-	if (strfd->data)
-		META_STACK_UNWIND (readlink, frame, strlen (strfd->data), 0,
-				   strfd->data, &iatt, xdata);
-	else
-		META_STACK_UNWIND (readlink, frame, -1, ENODATA, 0, 0, 0);
+    if (strfd->data) {
+        len = strlen(strfd->data);
+        META_STACK_UNWIND(readlink, frame, len, 0, strfd->data, &iatt, xdata);
+    } else
+        META_STACK_UNWIND(readlink, frame, -1, ENODATA, 0, 0, 0);
 
-	strfd_close (strfd);
+    strfd_close(strfd);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     int32_t mask, dict_t *xdata)
+meta_default_access(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t mask, dict_t *xdata)
 {
-        return default_access_failure_cbk (frame, EPERM);
+    return default_access_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			off_t offset, dict_t *xdata)
+meta_default_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       off_t offset, dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, fd->inode, IA_IFREG);
+    meta_iatt_fill(&iatt, fd->inode, IA_IFREG);
 
-	META_STACK_UNWIND (ftruncate, frame, 0, 0, &iatt, &iatt, xdata);
+    META_STACK_UNWIND(ftruncate, frame, 0, 0, &iatt, &iatt, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-meta_default_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name, dict_t *xdata)
+meta_default_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-        return default_getxattr_failure_cbk (frame, EPERM);
+    return default_getxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+meta_default_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        return default_xattrop_failure_cbk (frame, EPERM);
+    return default_xattrop_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+meta_default_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        return default_fxattrop_failure_cbk (frame, EPERM);
+    return default_fxattrop_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			  const char *name, dict_t *xdata)
+meta_default_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         const char *name, dict_t *xdata)
 {
-        return default_removexattr_failure_cbk (frame, EPERM);
+    return default_removexattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			   const char *name, dict_t *xdata)
+meta_default_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                          const char *name, dict_t *xdata)
 {
-        return default_fremovexattr_failure_cbk (frame, EPERM);
+    return default_fremovexattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		 int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+meta_default_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+                struct gf_flock *lock, dict_t *xdata)
 {
-        return default_lk_failure_cbk (frame, EPERM);
+    return default_lk_failure_cbk(frame, EPERM);
 }
 
-
 int
-meta_default_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-		      loc_t *loc, int32_t cmd, struct gf_flock *lock,
-		      dict_t *xdata)
+meta_default_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                     loc_t *loc, int32_t cmd, struct gf_flock *lock,
+                     dict_t *xdata)
 {
-        return default_inodelk_failure_cbk (frame, EPERM);
+    return default_inodelk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-		       fd_t *fd, int32_t cmd, struct gf_flock *lock,
-		       dict_t *xdata)
+meta_default_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                      fd_t *fd, int32_t cmd, struct gf_flock *lock,
+                      dict_t *xdata)
 {
-        return default_finodelk_failure_cbk (frame, EPERM);
+    return default_finodelk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-		      loc_t *loc, const char *basename, entrylk_cmd cmd,
-		      entrylk_type type, dict_t *xdata)
+meta_default_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                     loc_t *loc, const char *basename, entrylk_cmd cmd,
+                     entrylk_type type, dict_t *xdata)
 {
-        return default_entrylk_failure_cbk (frame, EPERM);
+    return default_entrylk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-		       fd_t *fd, const char *basename, entrylk_cmd cmd,
-		       entrylk_type type, dict_t *xdata)
+meta_default_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                      fd_t *fd, const char *basename, entrylk_cmd cmd,
+                      entrylk_type type, dict_t *xdata)
 {
-        return default_fentrylk_failure_cbk (frame, EPERM);
+    return default_fentrylk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			off_t offset, int32_t len, dict_t *xdata)
+meta_default_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       off_t offset, int32_t len, dict_t *xdata)
 {
-        return default_rchecksum_failure_cbk (frame, EPERM);
+    return default_rchecksum_failure_cbk(frame, EPERM);
 }
 
-
 int
-meta_default_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      size_t size, off_t off, dict_t *xdata)
+meta_default_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                     off_t off, dict_t *xdata)
 {
-	meta_fd_t *meta_fd = NULL;
-	int i = 0;
-	gf_dirent_t head;
-	gf_dirent_t *list = NULL;
-	int ret = 0;
-	int this_size = 0;
-	int filled_size = 0;
-	int fixed_size = 0;
-	int dyn_size = 0;
-	struct meta_dirent *fixed_dirents = NULL;
-	struct meta_dirent *dyn_dirents = NULL;
-	struct meta_dirent *dirents = NULL;
-	struct meta_dirent *end = NULL;
-	struct meta_ops *ops = NULL;
-
-	INIT_LIST_HEAD (&head.list);
+    meta_fd_t *meta_fd = NULL;
+    int i = 0;
+    gf_dirent_t head;
+    gf_dirent_t *list = NULL;
+    int ret = 0;
+    int this_size = 0;
+    int filled_size = 0;
+    int fixed_size = 0;
+    int dyn_size = 0;
+    struct meta_dirent *fixed_dirents = NULL;
+    struct meta_dirent *dyn_dirents = NULL;
+    struct meta_dirent *dirents = NULL;
+    struct meta_dirent *end = NULL;
+    struct meta_ops *ops = NULL;
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops)
-		goto err;
+    INIT_LIST_HEAD(&head.list);
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		goto err;
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops)
+        goto err;
 
-	meta_dir_fill (this, fd);
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        goto err;
 
-	fixed_dirents = ops->fixed_dirents;
-	fixed_size = fixed_dirents_len (fixed_dirents);
+    meta_dir_fill(this, fd);
 
-	dyn_dirents = meta_fd->dirents;
-	dyn_size = meta_fd->size;
+    fixed_dirents = ops->fixed_dirents;
+    fixed_size = fixed_dirents_len(fixed_dirents);
 
-	for (i = off; i < (fixed_size + dyn_size);) {
-		if (i >= fixed_size) {
-			dirents = dyn_dirents + (i - fixed_size);
-			end = dyn_dirents + dyn_size;
-		} else {
-			dirents = fixed_dirents + i;
-			end = fixed_dirents + fixed_size;
-		}
+    dyn_dirents = meta_fd->dirents;
+    dyn_size = meta_fd->size;
 
-		while (dirents < end) {
-			this_size = sizeof (gf_dirent_t) +
-				strlen (dirents->name) + 1;
-			if (this_size + filled_size > size)
-				goto unwind;
+    for (i = off; i < (fixed_size + dyn_size);) {
+        if (i >= fixed_size) {
+            dirents = dyn_dirents + (i - fixed_size);
+            end = dyn_dirents + dyn_size;
+        } else {
+            dirents = fixed_dirents + i;
+            end = fixed_dirents + fixed_size;
+        }
 
-			list = gf_dirent_for_name (dirents->name);
-			if (!list)
-				break;
+        while (dirents < end) {
+            this_size = sizeof(gf_dirent_t) + strlen(dirents->name) + 1;
+            if (this_size + filled_size > size)
+                goto unwind;
 
-			list->d_off = i + 1;
-			list->d_ino = i + 42;
-			switch (dirents->type) {
-			case IA_IFDIR: list->d_type = DT_DIR; break;
-			case IA_IFCHR: list->d_type = DT_CHR; break;
-			case IA_IFBLK: list->d_type = DT_BLK; break;
-			case IA_IFIFO: list->d_type = DT_FIFO; break;
-			case IA_IFLNK: list->d_type = DT_LNK; break;
-			case IA_IFREG: list->d_type = DT_REG; break;
-			case IA_IFSOCK: list->d_type = DT_SOCK; break;
-			case IA_INVAL: list->d_type = DT_UNKNOWN; break;
-			}
+            list = gf_dirent_for_name(dirents->name);
+            if (!list)
+                break;
 
-			list_add_tail (&list->list, &head.list);
-			ret++; i++; dirents++;
-			filled_size += this_size;
-		}
-	}
+            list->d_off = i + 1;
+            list->d_ino = i + 42;
+            switch (dirents->type) {
+                case IA_IFDIR:
+                    list->d_type = DT_DIR;
+                    break;
+                case IA_IFCHR:
+                    list->d_type = DT_CHR;
+                    break;
+                case IA_IFBLK:
+                    list->d_type = DT_BLK;
+                    break;
+                case IA_IFIFO:
+                    list->d_type = DT_FIFO;
+                    break;
+                case IA_IFLNK:
+                    list->d_type = DT_LNK;
+                    break;
+                case IA_IFREG:
+                    list->d_type = DT_REG;
+                    break;
+                case IA_IFSOCK:
+                    list->d_type = DT_SOCK;
+                    break;
+                case IA_INVAL:
+                    list->d_type = DT_UNKNOWN;
+                    break;
+            }
+
+            list_add_tail(&list->list, &head.list);
+            ret++;
+            i++;
+            dirents++;
+            filled_size += this_size;
+        }
+    }
 
 unwind:
-	META_STACK_UNWIND (readdir, frame, ret, 0, &head, xdata);
+    META_STACK_UNWIND(readdir, frame, ret, 0, &head, xdata);
 
-	gf_dirent_free (&head);
+    gf_dirent_free(&head);
 
-	return 0;
+    return 0;
 err:
-	META_STACK_UNWIND (readdir, frame, -1, ENOMEM, 0, 0);
-	return 0;
+    META_STACK_UNWIND(readdir, frame, -1, ENOMEM, 0, 0);
+    return 0;
 }
 
-
 int
-meta_default_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       size_t size, off_t off, dict_t *xdata)
+meta_default_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      size_t size, off_t off, dict_t *xdata)
 {
-        return meta_default_readdir (frame, this, fd, size, off, xdata);
+    return meta_default_readdir(frame, this, fd, size, off, xdata);
 }
 
 int
-meta_default_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      struct iatt *stbuf, int32_t valid,
-		      dict_t *xdata)
+meta_default_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        return default_setattr_failure_cbk (frame, EPERM);
+    return default_setattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       off_t offset, dict_t *xdata)
+meta_default_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      off_t offset, dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, loc->inode, IA_IFREG);
+    meta_iatt_fill(&iatt, loc->inode, IA_IFREG);
 
-	META_STACK_UNWIND (truncate, frame, 0, 0, &iatt, &iatt, xdata);
+    META_STACK_UNWIND(truncate, frame, 0, 0, &iatt, &iatt, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-meta_default_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   dict_t *xdata)
+meta_default_stat(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, loc->inode, loc->inode->ia_type);
+    meta_iatt_fill(&iatt, loc->inode, loc->inode->ia_type);
 
-	META_STACK_UNWIND (stat, frame, 0, 0, &iatt, xdata);
+    META_STACK_UNWIND(stat, frame, 0, 0, &iatt, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_default_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	struct meta_ops *ops = NULL;
-	struct meta_dirent *dirent = NULL;
-	struct meta_dirent *dp = NULL;
-	int i = 0;
-	int ret = 0;
+    struct meta_ops *ops = NULL;
+    struct meta_dirent *dirent = NULL;
+    struct meta_dirent *dp = NULL;
+    int i = 0;
+    int ret = 0;
 
-	if (!loc->name)
-		return meta_inode_discover (frame, this, loc, xdata);
+    if (!loc->name)
+        return meta_inode_discover(frame, this, loc, xdata);
 
-	ops = meta_ops_get (loc->parent, this);
-	if (!ops)
-		return default_lookup_failure_cbk (frame, EPERM);
+    ops = meta_ops_get(loc->parent, this);
+    if (!ops)
+        return default_lookup_failure_cbk(frame, EPERM);
 
-	for (dirent = ops->fixed_dirents; dirent && dirent->name; dirent++) {
-		if (strcmp (dirent->name, loc->name) == 0)
-			goto hook;
-	}
+    for (dirent = ops->fixed_dirents; dirent && dirent->name; dirent++) {
+        if (strcmp(dirent->name, loc->name) == 0)
+            goto hook;
+    }
 
-	dirent = NULL;
-	if (ops->dir_fill)
-		ret = ops->dir_fill (this, loc->parent, &dp);
+    dirent = NULL;
+    if (ops->dir_fill)
+        ret = ops->dir_fill(this, loc->parent, &dp);
 
-	for (i = 0; i < ret; i++) {
-		if (strcmp (dp[i].name, loc->name) == 0) {
-			dirent = &dp[i];
-			goto hook;
-		}
-	}
+    for (i = 0; i < ret; i++) {
+        if (strcmp(dp[i].name, loc->name) == 0) {
+            dirent = &dp[i];
+            goto hook;
+        }
+    }
 hook:
-	if (dirent && dirent->hook) {
-		struct iatt parent = { };
-		struct iatt iatt = { };
+    if (dirent && dirent->hook) {
+        struct iatt parent = {};
+        struct iatt iatt = {};
 
-		dirent->hook (frame, this, loc, xdata);
+        dirent->hook(frame, this, loc, xdata);
 
-		meta_iatt_fill (&iatt, loc->inode, dirent->type);
+        meta_iatt_fill(&iatt, loc->inode, dirent->type);
 
-		META_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &iatt,
-				   xdata, &parent);
-	} else {
-		META_STACK_UNWIND (lookup, frame, -1, ENOENT, 0, 0, 0, 0);
-	}
+        META_STACK_UNWIND(lookup, frame, 0, 0, loc->inode, &iatt, xdata,
+                          &parent);
+    } else {
+        META_STACK_UNWIND(lookup, frame, -1, ENOENT, 0, 0, 0, 0);
+    }
 
-	for (i = 0; i < ret; i++)
-		GF_FREE ((void *)dp[i].name);
-	GF_FREE (dp);
+    for (i = 0; i < ret; i++)
+        GF_FREE((void *)dp[i].name);
+    GF_FREE(dp);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       struct iatt *stbuf, int32_t valid, dict_t *xdata)
+meta_default_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        return default_fsetattr_failure_cbk (frame, EPERM);
+    return default_fsetattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			int32_t keep_size, off_t offset, size_t len,
-			dict_t *xdata)
+meta_default_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       int32_t keep_size, off_t offset, size_t len,
+                       dict_t *xdata)
 {
-        return default_fallocate_failure_cbk (frame, EPERM);
+    return default_fallocate_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_discard (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      off_t offset, size_t len, dict_t *xdata)
+meta_default_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     off_t offset, size_t len, dict_t *xdata)
 {
-        return default_discard_failure_cbk (frame, EPERM);
+    return default_discard_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       off_t offset, off_t len, dict_t *xdata)
+meta_default_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      off_t offset, off_t len, dict_t *xdata)
 {
-        return default_zerofill_failure_cbk (frame, EPERM);
+    return default_zerofill_failure_cbk(frame, EPERM);
 }
 
-#define SET_META_DEFAULT_FOP(f,name) do { if (!f->name) f->name = meta_default_##name ; } while (0)
+#define SET_META_DEFAULT_FOP(f, name)                                          \
+    do {                                                                       \
+        if (!f->name)                                                          \
+            f->name = meta_default_##name;                                     \
+    } while (0)
 
 struct xlator_fops *
-meta_defaults_init (struct xlator_fops *fops)
-{
-        SET_META_DEFAULT_FOP (fops,create);
-        SET_META_DEFAULT_FOP (fops,open);
-        SET_META_DEFAULT_FOP (fops,stat);
-        SET_META_DEFAULT_FOP (fops,readlink);
-        SET_META_DEFAULT_FOP (fops,mknod);
-        SET_META_DEFAULT_FOP (fops,mkdir);
-        SET_META_DEFAULT_FOP (fops,unlink);
-        SET_META_DEFAULT_FOP (fops,rmdir);
-        SET_META_DEFAULT_FOP (fops,symlink);
-        SET_META_DEFAULT_FOP (fops,rename);
-        SET_META_DEFAULT_FOP (fops,link);
-        SET_META_DEFAULT_FOP (fops,truncate);
-        SET_META_DEFAULT_FOP (fops,readv);
-        SET_META_DEFAULT_FOP (fops,writev);
-        SET_META_DEFAULT_FOP (fops,statfs);
-        SET_META_DEFAULT_FOP (fops,flush);
-        SET_META_DEFAULT_FOP (fops,fsync);
-        SET_META_DEFAULT_FOP (fops,setxattr);
-        SET_META_DEFAULT_FOP (fops,getxattr);
-        SET_META_DEFAULT_FOP (fops,fsetxattr);
-        SET_META_DEFAULT_FOP (fops,fgetxattr);
-        SET_META_DEFAULT_FOP (fops,removexattr);
-        SET_META_DEFAULT_FOP (fops,fremovexattr);
-        SET_META_DEFAULT_FOP (fops,opendir);
-        SET_META_DEFAULT_FOP (fops,readdir);
-        SET_META_DEFAULT_FOP (fops,readdirp);
-        SET_META_DEFAULT_FOP (fops,fsyncdir);
-        SET_META_DEFAULT_FOP (fops,access);
-        SET_META_DEFAULT_FOP (fops,ftruncate);
-        SET_META_DEFAULT_FOP (fops,fstat);
-        SET_META_DEFAULT_FOP (fops,lk);
-        SET_META_DEFAULT_FOP (fops,inodelk);
-        SET_META_DEFAULT_FOP (fops,finodelk);
-        SET_META_DEFAULT_FOP (fops,entrylk);
-        SET_META_DEFAULT_FOP (fops,fentrylk);
-        SET_META_DEFAULT_FOP (fops,lookup);
-        SET_META_DEFAULT_FOP (fops,rchecksum);
-        SET_META_DEFAULT_FOP (fops,xattrop);
-        SET_META_DEFAULT_FOP (fops,fxattrop);
-        SET_META_DEFAULT_FOP (fops,setattr);
-        SET_META_DEFAULT_FOP (fops,fsetattr);
-	SET_META_DEFAULT_FOP (fops,fallocate);
-	SET_META_DEFAULT_FOP (fops,discard);
-        SET_META_DEFAULT_FOP (fops,zerofill);
-
-	return fops;
+meta_defaults_init(struct xlator_fops *fops)
+{
+    SET_META_DEFAULT_FOP(fops, create);
+    SET_META_DEFAULT_FOP(fops, open);
+    SET_META_DEFAULT_FOP(fops, stat);
+    SET_META_DEFAULT_FOP(fops, readlink);
+    SET_META_DEFAULT_FOP(fops, mknod);
+    SET_META_DEFAULT_FOP(fops, mkdir);
+    SET_META_DEFAULT_FOP(fops, unlink);
+    SET_META_DEFAULT_FOP(fops, rmdir);
+    SET_META_DEFAULT_FOP(fops, symlink);
+    SET_META_DEFAULT_FOP(fops, rename);
+    SET_META_DEFAULT_FOP(fops, link);
+    SET_META_DEFAULT_FOP(fops, truncate);
+    SET_META_DEFAULT_FOP(fops, readv);
+    SET_META_DEFAULT_FOP(fops, writev);
+    SET_META_DEFAULT_FOP(fops, statfs);
+    SET_META_DEFAULT_FOP(fops, flush);
+    SET_META_DEFAULT_FOP(fops, fsync);
+    SET_META_DEFAULT_FOP(fops, setxattr);
+    SET_META_DEFAULT_FOP(fops, getxattr);
+    SET_META_DEFAULT_FOP(fops, fsetxattr);
+    SET_META_DEFAULT_FOP(fops, fgetxattr);
+    SET_META_DEFAULT_FOP(fops, removexattr);
+    SET_META_DEFAULT_FOP(fops, fremovexattr);
+    SET_META_DEFAULT_FOP(fops, opendir);
+    SET_META_DEFAULT_FOP(fops, readdir);
+    SET_META_DEFAULT_FOP(fops, readdirp);
+    SET_META_DEFAULT_FOP(fops, fsyncdir);
+    SET_META_DEFAULT_FOP(fops, access);
+    SET_META_DEFAULT_FOP(fops, ftruncate);
+    SET_META_DEFAULT_FOP(fops, fstat);
+    SET_META_DEFAULT_FOP(fops, lk);
+    SET_META_DEFAULT_FOP(fops, inodelk);
+    SET_META_DEFAULT_FOP(fops, finodelk);
+    SET_META_DEFAULT_FOP(fops, entrylk);
+    SET_META_DEFAULT_FOP(fops, fentrylk);
+    SET_META_DEFAULT_FOP(fops, lookup);
+    SET_META_DEFAULT_FOP(fops, rchecksum);
+    SET_META_DEFAULT_FOP(fops, xattrop);
+    SET_META_DEFAULT_FOP(fops, fxattrop);
+    SET_META_DEFAULT_FOP(fops, setattr);
+    SET_META_DEFAULT_FOP(fops, fsetattr);
+    SET_META_DEFAULT_FOP(fops, fallocate);
+    SET_META_DEFAULT_FOP(fops, discard);
+    SET_META_DEFAULT_FOP(fops, zerofill);
+
+    return fops;
 }
diff --git a/xlators/meta/src/meta-helpers.c b/xlators/meta/src/meta-helpers.c
index d01fcb3c679..cb54f547468 100644
--- a/xlators/meta/src/meta-helpers.c
+++ b/xlators/meta/src/meta-helpers.c
@@ -8,358 +8,325 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 meta_fd_t *
-meta_fd_get (fd_t *fd, xlator_t *this)
+meta_fd_get(fd_t *fd, xlator_t *this)
 {
-	uint64_t value = 0;
-	meta_fd_t *meta_fd = NULL;
-
-	LOCK (&fd->lock);
-	{
-		__fd_ctx_get (fd, this, &value);
-		if (!value) {
-			meta_fd = GF_CALLOC (1, sizeof (*meta_fd),
-					     gf_meta_mt_fd_t);
-			if (!meta_fd)
-				goto unlock;
-
-			value = (long) meta_fd;
-			__fd_ctx_set (fd, this, value);
-		}
-
-		meta_fd = (void *) value;
-	}
+    uint64_t value = 0;
+    meta_fd_t *meta_fd = NULL;
+
+    LOCK(&fd->lock);
+    {
+        if (__fd_ctx_get(fd, this, &value) < 0) {
+            if (!value) {
+                meta_fd = GF_CALLOC(1, sizeof(*meta_fd), gf_meta_mt_fd_t);
+                if (!meta_fd)
+                    goto unlock;
+                value = (long)meta_fd;
+                __fd_ctx_set(fd, this, value);
+            }
+        } else {
+            meta_fd = (void *)(uintptr_t)value;
+        }
+    }
 unlock:
-	UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 
-	return meta_fd;
+    return meta_fd;
 }
 
-
 int
-meta_fd_release (fd_t *fd, xlator_t *this)
+meta_fd_release(fd_t *fd, xlator_t *this)
 {
-	uint64_t value = 0;
-	meta_fd_t *meta_fd = NULL;
-	int i = 0;
-
-	fd_ctx_get (fd, this, &value);
-	meta_fd = (void *) value;
-
-	if (meta_fd->dirents) {
-		for (i = 0; i < meta_fd->size; i++)
-			GF_FREE ((void *)meta_fd->dirents[i].name);
-		GF_FREE (meta_fd->dirents);
-	}
-
-	if (meta_fd) {
-		GF_FREE (meta_fd->data);
-		GF_FREE (meta_fd);
-	}
-	return 0;
+    uint64_t value = 0;
+    meta_fd_t *meta_fd = NULL;
+    int i = 0;
+
+    fd_ctx_get(fd, this, &value);
+    meta_fd = (void *)(uintptr_t)value;
+
+    if (meta_fd && meta_fd->dirents) {
+        for (i = 0; i < meta_fd->size; i++)
+            GF_FREE((void *)meta_fd->dirents[i].name);
+        GF_FREE(meta_fd->dirents);
+    }
+
+    if (meta_fd) {
+        GF_FREE(meta_fd->data);
+        GF_FREE(meta_fd);
+    }
+    return 0;
 }
 
-
 struct meta_ops *
-meta_ops_get (inode_t *inode, xlator_t *this)
+meta_ops_get(inode_t *inode, xlator_t *this)
 {
-	struct meta_ops *ops = NULL;
-	uint64_t value = 0;
+    struct meta_ops *ops = NULL;
+    uint64_t value = 0;
 
-	inode_ctx_get2 (inode, this, NULL, &value);
+    inode_ctx_get2(inode, this, NULL, &value);
 
-	ops = (void *) value;
+    ops = (void *)(uintptr_t)value;
 
-	return ops;
+    return ops;
 }
 
-
 struct xlator_fops *
-meta_fops_get (inode_t *inode, xlator_t *this)
+meta_fops_get(inode_t *inode, xlator_t *this)
 {
-	struct meta_ops *ops = NULL;
+    struct meta_ops *ops = NULL;
 
-	ops = meta_ops_get (inode, this);
-	if (!ops)
-		return default_fops;
+    ops = meta_ops_get(inode, this);
+    if (!ops)
+        return default_fops;
 
-	return &ops->fops;
+    return &ops->fops;
 }
 
-
 int
-meta_ops_set (inode_t *inode, xlator_t *this, struct meta_ops *ops)
+meta_ops_set(inode_t *inode, xlator_t *this, struct meta_ops *ops)
 {
-	uint64_t value = 0;
-	int ret = 0;
+    uint64_t value = 0;
+    int ret = 0;
 
-	meta_defaults_init (&ops->fops);
+    meta_defaults_init(&ops->fops);
 
-	value = (long) ops;
+    value = (long)ops;
 
-	ret = inode_ctx_set2 (inode, this, NULL, &value);
+    ret = inode_ctx_set2(inode, this, NULL, &value);
 
-	return ret;
+    return ret;
 }
 
 void *
-meta_ctx_get (inode_t *inode, xlator_t *this)
+meta_ctx_get(inode_t *inode, xlator_t *this)
 {
-	void *ctx = NULL;
-	uint64_t value = 0;
+    void *ctx = NULL;
+    uint64_t value = 0;
 
-	inode_ctx_get2 (inode, this, &value, 0);
+    inode_ctx_get2(inode, this, &value, 0);
 
-	ctx = (void *) value;
+    ctx = (void *)(uintptr_t)value;
 
-	return ctx;
+    return ctx;
 }
 
-
 int
-meta_ctx_set (inode_t *inode, xlator_t *this, void *ctx)
+meta_ctx_set(inode_t *inode, xlator_t *this, void *ctx)
 {
-	uint64_t value = 0;
-	int ret = 0;
+    uint64_t value = 0;
+    int ret = 0;
 
-	value = (long) ctx;
+    value = (long)ctx;
 
-	ret = inode_ctx_set2 (inode, this, &value, 0);
+    ret = inode_ctx_set2(inode, this, &value, 0);
 
-	return ret;
+    return ret;
 }
 
-
 void
-meta_local_cleanup (meta_local_t *local, xlator_t *this)
+meta_local_cleanup(meta_local_t *local, xlator_t *this)
 {
-	if (!local)
-		return;
+    if (!local)
+        return;
 
-	if (local->xdata)
-		dict_unref (local->xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-	GF_FREE (local);
-	return;
+    GF_FREE(local);
+    return;
 }
 
-
 meta_local_t *
-meta_local (call_frame_t *frame)
+meta_local(call_frame_t *frame)
 {
-	meta_local_t *local = NULL;
+    meta_local_t *local = NULL;
 
-	local = frame->local;
-	if (!local)
-		local = frame->local = GF_CALLOC (1, sizeof(*local),
-						  gf_meta_mt_local_t);
-	return local;
+    local = frame->local;
+    if (!local)
+        local = frame->local = GF_CALLOC(1, sizeof(*local), gf_meta_mt_local_t);
+    return local;
 }
 
-
 dict_t *
-meta_direct_io_mode (dict_t *xdata, call_frame_t *frame)
+meta_direct_io_mode(dict_t *xdata, call_frame_t *frame)
 {
-	meta_local_t *local = NULL;
-
-	if (!xdata) {
-		local = meta_local (frame);
-		if (!local)
-			return NULL;
-		xdata = local->xdata = dict_new();
-		if (!xdata)
-			return NULL;
-	}
-
-	if (dict_set_int8 (xdata, "direct-io-mode", 1) != 0)
-		return NULL;
+    meta_local_t *local = NULL;
 
-	return xdata;
-}
-
-
-static uint64_t
-gfid_to_ino (uuid_t gfid)
-{
-        uint64_t  ino = 0;
-        int       i = 0, j = 0;
+    if (!xdata) {
+        local = meta_local(frame);
+        if (!local)
+            return NULL;
+        xdata = local->xdata = dict_new();
+        if (!xdata)
+            return NULL;
+    }
 
-        for (i = 15; i > (15 - 8); i--) {
-                ino += (uint64_t)(gfid[i]) << j;
-                j += 8;
-        }
+    if (dict_set_int8(xdata, "direct-io-mode", 1) != 0)
+        return NULL;
 
-        return ino;
+    return xdata;
 }
 
-
 static void
-meta_uuid_copy (uuid_t dst, uuid_t src)
+meta_uuid_copy(uuid_t dst, uuid_t src)
 {
-	gf_uuid_copy (dst, src);
-	if (gf_uuid_is_null (dst))
-		gf_uuid_generate (dst);
+    gf_uuid_copy(dst, src);
+    if (gf_uuid_is_null(dst))
+        gf_uuid_generate(dst);
 }
 
-
 static void
-default_meta_iatt_fill (struct iatt *iatt, inode_t *inode, ia_type_t type)
+default_meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type,
+                       gf_boolean_t is_tunable)
 {
-	struct timeval tv = { };
-
-	iatt->ia_type = type;
-	switch (type)
-	{
-	case IA_IFDIR:
-		iatt->ia_prot = ia_prot_from_st_mode (0755);
-		iatt->ia_nlink = 2;
-		break;
-	case IA_IFLNK:
-		iatt->ia_prot = ia_prot_from_st_mode (0777);
-		iatt->ia_nlink = 1;
-		break;
-	default:
-		iatt->ia_prot = ia_prot_from_st_mode (0644);
-		iatt->ia_nlink = 1;
-		break;
-	}
-	iatt->ia_uid = 0;
-	iatt->ia_gid = 0;
-	iatt->ia_size = 0;
-
-	meta_uuid_copy (iatt->ia_gfid, inode->gfid);
-	iatt->ia_ino = gfid_to_ino (iatt->ia_gfid);
-
-	gettimeofday (&tv, 0);
-	iatt->ia_mtime = iatt->ia_ctime = iatt->ia_atime = tv.tv_sec;
-	iatt->ia_mtime_nsec = iatt->ia_ctime_nsec = iatt->ia_atime_nsec =
-		(tv.tv_usec * 1000);
-	return;
+    struct timeval tv = {};
+
+    iatt->ia_type = type;
+    switch (type) {
+        case IA_IFDIR:
+            iatt->ia_prot = ia_prot_from_st_mode(0555);
+            iatt->ia_nlink = 2;
+            break;
+        case IA_IFLNK:
+            iatt->ia_prot = ia_prot_from_st_mode(0777);
+            iatt->ia_nlink = 1;
+            break;
+        default:
+            iatt->ia_prot = ia_prot_from_st_mode(is_tunable ? 0644 : 0444);
+            iatt->ia_nlink = 1;
+            break;
+    }
+    iatt->ia_uid = 0;
+    iatt->ia_gid = 0;
+    iatt->ia_size = 0;
+
+    meta_uuid_copy(iatt->ia_gfid, inode->gfid);
+    iatt->ia_ino = gfid_to_ino(iatt->ia_gfid);
+
+    gettimeofday(&tv, 0);
+    iatt->ia_mtime = iatt->ia_ctime = iatt->ia_atime = tv.tv_sec;
+    iatt->ia_mtime_nsec = iatt->ia_ctime_nsec = iatt->ia_atime_nsec =
+        (tv.tv_usec * 1000);
+    return;
 }
 
-
 void
-meta_iatt_fill (struct iatt *iatt, inode_t *inode, ia_type_t type)
+meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type)
 {
-	struct meta_ops *ops = NULL;
+    struct meta_ops *ops = NULL;
 
-	ops = meta_ops_get (inode, THIS);
-	if (!ops)
-		return;
+    ops = meta_ops_get(inode, THIS);
+    if (!ops)
+        return;
 
-	if (!ops->iatt_fill)
-		default_meta_iatt_fill (iatt, inode, type);
-	else
-		ops->iatt_fill (THIS, inode, iatt);
-	return;
+    if (!ops->iatt_fill)
+        default_meta_iatt_fill(iatt, inode, type, !!ops->file_write);
+    else
+        ops->iatt_fill(THIS, inode, iatt);
+    return;
 }
 
-
 int
-meta_inode_discover (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_inode_discover(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	struct iatt iatt = { };
-	struct iatt postparent = { };
+    struct iatt iatt = {};
+    struct iatt postparent = {};
 
-	meta_iatt_fill (&iatt, loc->inode, loc->inode->ia_type);
+    meta_iatt_fill(&iatt, loc->inode, loc->inode->ia_type);
 
-	META_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &iatt, xdata,
-			   &postparent);
-	return 0;
+    META_STACK_UNWIND(lookup, frame, 0, 0, loc->inode, &iatt, xdata,
+                      &postparent);
+    return 0;
 }
 
-
 int
-meta_file_fill (xlator_t *this, fd_t *fd)
+meta_file_fill(xlator_t *this, fd_t *fd)
 {
-	meta_fd_t *meta_fd = NULL;
-	strfd_t *strfd = NULL;
-	struct meta_ops *ops = NULL;
-	int ret = 0;
+    meta_fd_t *meta_fd = NULL;
+    strfd_t *strfd = NULL;
+    struct meta_ops *ops = NULL;
+    int ret = 0;
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		return -1;
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        return -1;
 
-	if (meta_fd->data)
-		return meta_fd->size;
+    if (meta_fd->data)
+        return meta_fd->size;
 
-	strfd = strfd_open ();
-	if (!strfd)
-		return -1;
+    strfd = strfd_open();
+    if (!strfd)
+        return -1;
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops) {
-		strfd_close (strfd);
-		return -1;
-	}
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops) {
+        strfd_close(strfd);
+        return -1;
+    }
 
-	if (ops->file_fill)
-		ret = ops->file_fill (this, fd->inode, strfd);
+    if (ops->file_fill)
+        ret = ops->file_fill(this, fd->inode, strfd);
 
-	if (ret >= 0) {
-		meta_fd->data = strfd->data;
-		meta_fd->size = strfd->size;
+    if (ret >= 0) {
+        meta_fd->data = strfd->data;
+        meta_fd->size = strfd->size;
 
-		strfd->data = NULL;
-	}
+        strfd->data = NULL;
+    }
 
-	strfd_close (strfd);
+    strfd_close(strfd);
 
-	return meta_fd->size;
+    return meta_fd->size;
 }
 
-
 int
-meta_dir_fill (xlator_t *this, fd_t *fd)
+meta_dir_fill(xlator_t *this, fd_t *fd)
 {
-	meta_fd_t *meta_fd = NULL;
-	struct meta_ops *ops = NULL;
-	struct meta_dirent *dp = NULL;
-	int ret = 0;
+    meta_fd_t *meta_fd = NULL;
+    struct meta_ops *ops = NULL;
+    struct meta_dirent *dp = NULL;
+    int ret = 0;
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		return -1;
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        return -1;
 
-	if (meta_fd->dirents)
-		return meta_fd->size;
+    if (meta_fd->dirents)
+        return meta_fd->size;
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops)
-		return -1;
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops)
+        return -1;
 
-	if (ops->dir_fill)
-		ret = ops->dir_fill (this, fd->inode, &dp);
+    if (ops->dir_fill)
+        ret = ops->dir_fill(this, fd->inode, &dp);
 
-	if (dp) {
-		meta_fd->dirents = dp;
-		meta_fd->size = ret;
-	}
+    if (dp) {
+        meta_fd->dirents = dp;
+        meta_fd->size = ret;
+    }
 
-	return meta_fd->size;
+    return meta_fd->size;
 }
 
-
 int
-fixed_dirents_len (struct meta_dirent *dirents)
+fixed_dirents_len(struct meta_dirent *dirents)
 {
-	int i = 0;
-	struct meta_dirent *dirent = NULL;
+    int i = 0;
+    struct meta_dirent *dirent = NULL;
 
-	if (!dirents)
-		return 0;
+    if (!dirents)
+        return 0;
 
-	for (dirent = dirents; dirent->name; dirent++)
-		i++;
+    for (dirent = dirents; dirent->name; dirent++)
+        i++;
 
-	return i;
+    return i;
 }
diff --git a/xlators/meta/src/meta-hooks.h b/xlators/meta/src/meta-hooks.h
index bcf3643d223..7208641398a 100644
--- a/xlators/meta/src/meta-hooks.h
+++ b/xlators/meta/src/meta-hooks.h
@@ -10,9 +10,11 @@
 
 #ifndef __META_HOOKS_H
 #define __META_HOOKS_H
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
-#define DECLARE_HOOK(name) int meta_##name##_hook (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+#define DECLARE_HOOK(name)                                                     \
+    int meta_##name##_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,    \
+                           dict_t *xdata)
 
 DECLARE_HOOK(root_dir);
 DECLARE_HOOK(graphs_dir);
diff --git a/xlators/meta/src/meta-mem-types.h b/xlators/meta/src/meta-mem-types.h
index e8a31856e71..033c306682f 100644
--- a/xlators/meta/src/meta-mem-types.h
+++ b/xlators/meta/src/meta-mem-types.h
@@ -11,16 +11,15 @@
 #ifndef __META_MEM_TYPES_H__
 #define __META_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_meta_mem_types_ {
-        gf_meta_mt_priv_t = gf_common_mt_end + 1,
-	gf_meta_mt_fd_t,
-	gf_meta_mt_fd_data_t,
-	gf_meta_mt_strfd_t,
-	gf_meta_mt_dirents_t,
-	gf_meta_mt_local_t,
-        gf_meta_mt_end
+    gf_meta_mt_priv_t = gf_common_mt_end + 1,
+    gf_meta_mt_fd_t,
+    gf_meta_mt_fd_data_t,
+    gf_meta_mt_strfd_t,
+    gf_meta_mt_dirents_t,
+    gf_meta_mt_local_t,
+    gf_meta_mt_end
 };
 #endif
-
diff --git a/xlators/meta/src/meta.c b/xlators/meta/src/meta.c
index 25720136714..e1b9a2b6581 100644
--- a/xlators/meta/src/meta.c
+++ b/xlators/meta/src/meta.c
@@ -8,276 +8,269 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
 #include "meta-hooks.h"
 
-
 int
-meta_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+meta_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
 
-	if (META_HOOK (loc) || IS_META_ROOT_GFID (loc->gfid)) {
-		struct iatt iatt = { };
-		struct iatt parent = { };
+    if (META_HOOK(loc) || IS_META_ROOT_GFID(loc->gfid)) {
+        struct iatt iatt = {};
+        struct iatt parent = {};
 
-		meta_root_dir_hook (frame, this, loc, xdata);
+        meta_root_dir_hook(frame, this, loc, xdata);
 
-		meta_iatt_fill (&iatt, loc->inode, IA_IFDIR);
-		gf_uuid_parse (META_ROOT_GFID, iatt.ia_gfid);
+        meta_iatt_fill(&iatt, loc->inode, IA_IFDIR);
+        gf_uuid_parse(META_ROOT_GFID, iatt.ia_gfid);
 
-		META_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &iatt,
-				   xdata, &parent);
-		return 0;
-	}
+        META_STACK_UNWIND(lookup, frame, 0, 0, loc->inode, &iatt, xdata,
+                          &parent);
+        return 0;
+    }
 
-	if (loc->parent)
-		inode = loc->parent;
-	else
-		inode = loc->inode;
+    if (loc->parent)
+        inode = loc->parent;
+    else
+        inode = loc->inode;
 
-	META_FOP (inode, lookup, frame, this, loc, xdata);
+    META_FOP(inode, lookup, frame, this, loc, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-	      dict_t *xdata)
+meta_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
-	META_FOP (fd->inode, opendir, frame, this, loc, fd, xdata);
+    META_FOP(fd->inode, opendir, frame, this, loc, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
-	   dict_t *xdata)
+meta_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+          dict_t *xdata)
 {
-	META_FOP (fd->inode, open, frame, this, loc, flags, fd, xdata);
+    META_FOP(fd->inode, open, frame, this, loc, flags, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	    off_t offset, uint32_t flags, dict_t *xdata)
+meta_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, uint32_t flags, dict_t *xdata)
 {
-	META_FOP (fd->inode, readv, frame, this, fd, size, offset, flags, xdata);
+    META_FOP(fd->inode, readv, frame, this, fd, size, offset, flags, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+meta_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	META_FOP (fd->inode, flush, frame, this, fd, xdata);
+    META_FOP(fd->inode, flush, frame, this, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+meta_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	META_FOP (loc->inode, stat, frame, this, loc, xdata);
+    META_FOP(loc->inode, stat, frame, this, loc, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+meta_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	META_FOP (fd->inode, fstat, frame, this, fd, xdata);
+    META_FOP(fd->inode, fstat, frame, this, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	      off_t offset, dict_t *xdata)
+meta_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
 {
-	META_FOP (fd->inode, readdir, frame, this, fd, size, offset, xdata);
+    META_FOP(fd->inode, readdir, frame, this, fd, size, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	       off_t offset, dict_t *xdata)
+meta_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
-	META_FOP (fd->inode, readdirp, frame, this, fd, size, offset, xdata);
+    META_FOP(fd->inode, readdirp, frame, this, fd, size, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-	       dict_t *xdata)
+meta_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+              dict_t *xdata)
 {
-	META_FOP (loc->inode, readlink, frame, this, loc, size, xdata);
+    META_FOP(loc->inode, readlink, frame, this, loc, size, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
-	     int count, off_t offset, uint32_t flags, struct iobref *iobref,
-	     dict_t *xdata)
+meta_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+            int count, off_t offset, uint32_t flags, struct iobref *iobref,
+            dict_t *xdata)
 {
-	META_FOP (fd->inode, writev, frame, this, fd, iov, count, offset, flags,
-		  iobref, xdata);
-	return 0;
+    META_FOP(fd->inode, writev, frame, this, fd, iov, count, offset, flags,
+             iobref, xdata);
+    return 0;
 }
 
-
 int
-meta_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-	       dict_t *xdata)
+meta_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
-	META_FOP (loc->inode, truncate, frame, this, loc, offset, xdata);
+    META_FOP(loc->inode, truncate, frame, this, loc, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-		dict_t *xdata)
+meta_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
-	META_FOP (fd->inode, ftruncate, frame, this, fd, offset, xdata);
+    META_FOP(fd->inode, ftruncate, frame, this, fd, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
 int32_t
-meta_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-            dict_t *xdata)
+meta_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+           dict_t *xdata)
 {
-	META_FOP (fd->inode, fsync, frame, this, fd, flags, xdata);
+    META_FOP(fd->inode, fsync, frame, this, fd, flags, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-meta_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-               dict_t *xdata)
+meta_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
 {
-	META_FOP (fd->inode, fsyncdir, frame, this, fd, flags, xdata);
+    META_FOP(fd->inode, fsyncdir, frame, this, fd, flags, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-meta_forget (xlator_t *this, inode_t *inode)
+meta_forget(xlator_t *this, inode_t *inode)
 {
-	return 0;
+    return 0;
 }
 
-
 int
-meta_release (xlator_t *this, fd_t *fd)
+meta_release(xlator_t *this, fd_t *fd)
 {
-	return meta_fd_release (fd, this);
+    return meta_fd_release(fd, this);
 }
 
-
 int
-meta_releasedir (xlator_t *this, fd_t *fd)
+meta_releasedir(xlator_t *this, fd_t *fd)
 {
-	return meta_fd_release (fd, this);
+    return meta_fd_release(fd, this);
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_meta_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Memory accounting init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_meta_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR, "Memory accounting init failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	meta_priv_t *priv = NULL;
+    meta_priv_t *priv = NULL;
+    int ret = -1;
 
-	priv = GF_CALLOC (sizeof(*priv), 1, gf_meta_mt_priv_t);
-	if (!priv)
-		return -1;
+    priv = GF_CALLOC(sizeof(*priv), 1, gf_meta_mt_priv_t);
+    if (!priv)
+        return ret;
 
-	GF_OPTION_INIT ("meta-dir-name", priv->meta_dir_name, str, out);
+    GF_OPTION_INIT("meta-dir-name", priv->meta_dir_name, str, out);
 
-	this->private = priv;
+    this->private = priv;
+    ret = 0;
 out:
-	return 0;
-}
+    if (ret)
+        GF_FREE(priv);
 
+    return ret;
+}
 
-int
-fini (xlator_t *this)
+void
+fini(xlator_t *this)
 {
-	return 0;
+    GF_FREE(this->private);
+    return;
 }
 
-
-struct xlator_fops fops = {
-	.lookup    = meta_lookup,
-	.opendir   = meta_opendir,
-	.open      = meta_open,
-	.readv     = meta_readv,
-	.flush     = meta_flush,
-	.stat      = meta_stat,
-	.fstat     = meta_fstat,
-	.readdir   = meta_readdir,
-	.readdirp  = meta_readdirp,
-	.readlink  = meta_readlink,
-	.writev    = meta_writev,
-	.truncate  = meta_truncate,
-	.ftruncate = meta_ftruncate,
-        .fsync     = meta_fsync,
-        .fsyncdir  = meta_fsyncdir
-};
-
+struct xlator_fops fops = {.lookup = meta_lookup,
+                           .opendir = meta_opendir,
+                           .open = meta_open,
+                           .readv = meta_readv,
+                           .flush = meta_flush,
+                           .stat = meta_stat,
+                           .fstat = meta_fstat,
+                           .readdir = meta_readdir,
+                           .readdirp = meta_readdirp,
+                           .readlink = meta_readlink,
+                           .writev = meta_writev,
+                           .truncate = meta_truncate,
+                           .ftruncate = meta_ftruncate,
+                           .fsync = meta_fsync,
+                           .fsyncdir = meta_fsyncdir};
 
 struct xlator_cbks cbks = {
-	.forget = meta_forget,
-	.release = meta_release,
-	.releasedir = meta_releasedir,
+    .forget = meta_forget,
+    .release = meta_release,
+    .releasedir = meta_releasedir,
 };
 
-
 struct volume_options options[] = {
-	{ .key = {"meta-dir-name"},
-	  .type = GF_OPTION_TYPE_STR,
-	  .default_value = DEFAULT_META_DIR_NAME,
-	  .description = "Name of default meta directory."
-	},
-	{ .key = {NULL} },
+    {.key = {"meta-dir-name"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = DEFAULT_META_DIR_NAME,
+     .description = "Name of default meta directory."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "meta",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/meta/src/meta.h b/xlators/meta/src/meta.h
index d9c56c656ad..7f0cf28808a 100644
--- a/xlators/meta/src/meta.h
+++ b/xlators/meta/src/meta.h
@@ -10,112 +10,131 @@
 #ifndef __META_H__
 #define __META_H__
 
-#include "strfd.h"
+#include <glusterfs/strfd.h>
 
 #define DEFAULT_META_DIR_NAME ".meta"
 
 #define META_ROOT_GFID "ba926388-bb9c-4eec-ad60-79dba4cc083a"
 
-#define IS_META_ROOT_GFID(g) (strcmp (uuid_utoa(g), META_ROOT_GFID) == 0)
+#define IS_META_ROOT_GFID(g) (strcmp(uuid_utoa(g), META_ROOT_GFID) == 0)
 
-typedef int (*meta_hook_t) (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+typedef int (*meta_hook_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           dict_t *xdata);
 
 typedef struct {
-	dict_t *xdata;
+    dict_t *xdata;
 } meta_local_t;
 
 typedef struct {
-	char *meta_dir_name;
+    char *meta_dir_name;
 } meta_priv_t;
 
 struct meta_dirent {
-	const char *name;
-	ia_type_t type;
-	meta_hook_t hook;
+    const char *name;
+    ia_type_t type;
+    meta_hook_t hook;
 };
 
-#define DOT_DOTDOT { .name = ".", .type = IA_IFDIR }, { .name = "..", .type = IA_IFDIR }
+#define DOT_DOTDOT                                                             \
+    {.name = ".", .type = IA_IFDIR}, { .name = "..", .type = IA_IFDIR }
 
 struct meta_ops {
-	struct meta_dirent *fixed_dirents;
-	int (*dir_fill) (xlator_t *this, inode_t *dir, struct meta_dirent **entries);
-	int (*file_fill) (xlator_t *this, inode_t *file, strfd_t *strfd);
-	int (*iatt_fill) (xlator_t *this, inode_t *inode, struct iatt *iatt);
-	int (*link_fill) (xlator_t *this, inode_t *inode, strfd_t *strfd);
-	int (*file_write) (xlator_t *this, fd_t *fd, struct iovec *iov, int count);
-	struct xlator_fops fops;
-	struct xlator_cbks cbks;
+    struct meta_dirent *fixed_dirents;
+    int (*dir_fill)(xlator_t *this, inode_t *dir, struct meta_dirent **entries);
+    int (*file_fill)(xlator_t *this, inode_t *file, strfd_t *strfd);
+    int (*iatt_fill)(xlator_t *this, inode_t *inode, struct iatt *iatt);
+    int (*link_fill)(xlator_t *this, inode_t *inode, strfd_t *strfd);
+    int (*file_write)(xlator_t *this, fd_t *fd, struct iovec *iov, int count);
+    struct xlator_fops fops;
+    struct xlator_cbks cbks;
 };
 
 typedef struct {
-	char *data;
-	struct meta_dirent *dirents;
-	size_t size;
+    char *data;
+    struct meta_dirent *dirents;
+    size_t size;
 } meta_fd_t;
 
+#define COUNT(arr) (sizeof(arr) / sizeof(arr[0]))
 
-#define COUNT(arr) (sizeof(arr)/sizeof(arr[0]))
-
-#define META_HOOK(loc) (__is_root_gfid (loc->pargfid) && !strcmp (loc->name, META_PRIV(THIS)->meta_dir_name))
+#define META_HOOK(loc)                                                         \
+    (__is_root_gfid(loc->pargfid) &&                                           \
+     !strcmp(loc->name, META_PRIV(THIS)->meta_dir_name))
 
 #define META_PRIV(t) ((meta_priv_t *)(t->private))
 
-#define META_STACK_UNWIND(fop, frame, params ...)		\
-        do {                                                    \
-                meta_local_t *__local = NULL;			\
-                xlator_t    *__this = NULL;                     \
-                if (frame) {                                    \
-                        __local = frame->local;                 \
-                        __this = frame->this;                   \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                if (__local) {                                  \
-                        meta_local_cleanup (__local, __this);	\
-                }                                               \
-        } while (0)
-
-
-#define META_FOP(i, fop, fr, t, params ...) {	\
-	struct xlator_fops *_fops = NULL;	\
-						\
-	_fops = meta_fops_get (i, t);		\
-						\
-	_fops->fop (fr, t, params);		\
-	} while (0)
-
-
-void meta_iatt_fill (struct iatt *iatt, inode_t *inode, ia_type_t type);
-
-int meta_inode_discover (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			 dict_t *xdata);
-
-int meta_ops_set (inode_t *inode, xlator_t *this, struct meta_ops *ops);
-
-struct xlator_fops *meta_fops_get (inode_t *inode, xlator_t *this);
-struct xlator_cbks *meta_cbks_get (inode_t *inode, xlator_t *this);
-struct meta_ops *meta_ops_get (inode_t *inode, xlator_t *this);
-
-int meta_ctx_set (inode_t *inode, xlator_t *this, void *ctx);
-
-void *meta_ctx_get (inode_t *inode, xlator_t *this);
-
-
-void meta_local_cleanup (meta_local_t *local, xlator_t *this);
-
-struct xlator_fops *meta_defaults_init (struct xlator_fops *fops);
-
-meta_fd_t *meta_fd_get (fd_t *fd, xlator_t *this);
-
-int meta_fd_release (fd_t *fd, xlator_t *this);
-
-dict_t *meta_direct_io_mode (dict_t *xdata, call_frame_t *frame);
-
-meta_local_t *meta_local (call_frame_t *frame);
-
-int meta_file_fill (xlator_t *this, fd_t *fd);
-
-int meta_dir_fill (xlator_t *this, fd_t *fd);
-
-int fixed_dirents_len (struct meta_dirent *dirents);
+#define META_STACK_UNWIND(fop, frame, params...)                               \
+    do {                                                                       \
+        meta_local_t *__local = NULL;                                          \
+        xlator_t *__this = NULL;                                               \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __this = frame->this;                                              \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local) {                                                         \
+            meta_local_cleanup(__local, __this);                               \
+        }                                                                      \
+    } while (0)
+
+#define META_FOP(i, fop, fr, t, params...)                                     \
+    {                                                                          \
+        struct xlator_fops *_fops = NULL;                                      \
+                                                                               \
+        _fops = meta_fops_get(i, t);                                           \
+                                                                               \
+        _fops->fop(fr, t, params);                                             \
+    }                                                                          \
+    while (0)
+
+void
+meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type);
+
+int
+meta_inode_discover(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata);
+
+int
+meta_ops_set(inode_t *inode, xlator_t *this, struct meta_ops *ops);
+
+struct xlator_fops *
+meta_fops_get(inode_t *inode, xlator_t *this);
+struct xlator_cbks *
+meta_cbks_get(inode_t *inode, xlator_t *this);
+struct meta_ops *
+meta_ops_get(inode_t *inode, xlator_t *this);
+
+int
+meta_ctx_set(inode_t *inode, xlator_t *this, void *ctx);
+
+void *
+meta_ctx_get(inode_t *inode, xlator_t *this);
+
+void
+meta_local_cleanup(meta_local_t *local, xlator_t *this);
+
+struct xlator_fops *
+meta_defaults_init(struct xlator_fops *fops);
+
+meta_fd_t *
+meta_fd_get(fd_t *fd, xlator_t *this);
+
+int
+meta_fd_release(fd_t *fd, xlator_t *this);
+
+dict_t *
+meta_direct_io_mode(dict_t *xdata, call_frame_t *frame);
+
+meta_local_t *
+meta_local(call_frame_t *frame);
+
+int
+meta_file_fill(xlator_t *this, fd_t *fd);
+
+int
+meta_dir_fill(xlator_t *this, fd_t *fd);
+
+int
+fixed_dirents_len(struct meta_dirent *dirents);
 #endif /* __META_H__ */
diff --git a/xlators/meta/src/name-file.c b/xlators/meta/src/name-file.c
index 44c359ef5c8..5874a24d78a 100644
--- a/xlators/meta/src/name-file.c
+++ b/xlators/meta/src/name-file.c
@@ -8,41 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-name_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+name_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	strprintf (strfd, "%s\n", xl->name);
+    strprintf(strfd, "%s\n", xl->name);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops name_file_ops = {
-	.file_fill = name_file_fill,
+    .file_fill = name_file_fill,
 };
 
-
 int
-meta_name_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_name_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &name_file_ops);
+    meta_ops_set(loc->inode, this, &name_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/option-file.c b/xlators/meta/src/option-file.c
index 5a8465c5d8b..ff55eca592f 100644
--- a/xlators/meta/src/option-file.c
+++ b/xlators/meta/src/option-file.c
@@ -8,44 +8,38 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static int
-option_file_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+option_file_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	data_t *data = NULL;
+    data_t *data = NULL;
 
-	data = meta_ctx_get (inode, this);
+    data = meta_ctx_get(inode, this);
 
-	strprintf (strfd, "%s\n", data_to_str (data));
+    strprintf(strfd, "%s\n", data_to_str(data));
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
-static struct meta_ops option_file_ops = {
-	.file_fill = option_file_fill
-};
-
+static struct meta_ops option_file_ops = {.file_fill = option_file_fill};
 
 int
-meta_option_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *xdata)
+meta_option_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (loc->parent, this);
+    xl = meta_ctx_get(loc->parent, this);
 
-	meta_ctx_set (loc->inode, this,
-		      dict_get (xl->options, (char *) loc->name));
+    meta_ctx_set(loc->inode, this, dict_get(xl->options, (char *)loc->name));
 
-	meta_ops_set (loc->inode, this, &option_file_ops);
+    meta_ops_set(loc->inode, this, &option_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/options-dir.c b/xlators/meta/src/options-dir.c
index e637afb1f73..d68a7eeaffc 100644
--- a/xlators/meta/src/options-dir.c
+++ b/xlators/meta/src/options-dir.c
@@ -8,64 +8,58 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static int
-dict_key_add (dict_t *dict, char *key, data_t *value, void *data)
+dict_key_add(dict_t *dict, char *key, data_t *value, void *data)
 {
-	struct meta_dirent **direntp = data;
+    struct meta_dirent **direntp = data;
 
-	(*direntp)->name = gf_strdup (key);
-	(*direntp)->type = IA_IFREG;
-	(*direntp)->hook = meta_option_file_hook;
+    (*direntp)->name = gf_strdup(key);
+    (*direntp)->type = IA_IFREG;
+    (*direntp)->hook = meta_option_file_hook;
 
-	(*direntp)++;
-	return 0;
+    (*direntp)++;
+    return 0;
 }
 
-
 static int
-options_dir_fill (xlator_t *this, inode_t *inode, struct meta_dirent **dp)
+options_dir_fill(xlator_t *this, inode_t *inode, struct meta_dirent **dp)
 {
-	struct meta_dirent *dirent = NULL;
-	struct meta_dirent *direntp = NULL;
-	xlator_t *xl = NULL;
+    struct meta_dirent *dirent = NULL;
+    struct meta_dirent *direntp = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (inode, this);
+    xl = meta_ctx_get(inode, this);
 
-	dirent = GF_CALLOC (sizeof (*dirent), xl->options->count,
-			    gf_meta_mt_dirents_t);
-	if (!dirent)
-		return -1;
+    dirent = GF_CALLOC(sizeof(*dirent), xl->options->count,
+                       gf_meta_mt_dirents_t);
+    if (!dirent)
+        return -1;
 
-	direntp = dirent;
+    direntp = dirent;
 
-	dict_foreach (xl->options, dict_key_add, &direntp);
+    dict_foreach(xl->options, dict_key_add, &direntp);
 
-	*dp = dirent;
+    *dp = dirent;
 
-	return xl->options->count;
+    return xl->options->count;
 }
 
-
-static struct meta_ops options_dir_ops = {
-	.dir_fill = options_dir_fill
-};
-
+static struct meta_ops options_dir_ops = {.dir_fill = options_dir_fill};
 
 int
-meta_options_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *xdata)
+meta_options_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	meta_ops_set (loc->inode, this, &options_dir_ops);
+    meta_ops_set(loc->inode, this, &options_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/private-file.c b/xlators/meta/src/private-file.c
index 8d12b467d75..23ec319456b 100644
--- a/xlators/meta/src/private-file.c
+++ b/xlators/meta/src/private-file.c
@@ -8,40 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-private_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+private_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_private (xl, strfd);
+    gf_proc_dump_xlator_private(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops private_file_ops = {
-	.file_fill = private_file_fill,
+    .file_fill = private_file_fill,
 };
 
-
 int
-meta_private_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_private_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &private_file_ops);
+    meta_ops_set(loc->inode, this, &private_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/process_uuid-file.c b/xlators/meta/src/process_uuid-file.c
index 3210de1d484..a24c1b57ab3 100644
--- a/xlators/meta/src/process_uuid-file.c
+++ b/xlators/meta/src/process_uuid-file.c
@@ -8,34 +8,30 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-process_uuid_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+process_uuid_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	strprintf (strfd, "%s\n", this->ctx->process_uuid);
-	return strfd->size;
+    strprintf(strfd, "%s\n", this->ctx->process_uuid);
+    return strfd->size;
 }
 
-
 static struct meta_ops process_uuid_file_ops = {
-	.file_fill = process_uuid_file_fill,
+    .file_fill = process_uuid_file_fill,
 };
 
-
 int
-meta_process_uuid_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			     dict_t *xdata)
+meta_process_uuid_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                            dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &process_uuid_file_ops);
+    meta_ops_set(loc->inode, this, &process_uuid_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/profile-file.c b/xlators/meta/src/profile-file.c
index 7a8d0bf810e..829dcb77451 100644
--- a/xlators/meta/src/profile-file.c
+++ b/xlators/meta/src/profile-file.c
@@ -8,40 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-profile_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+profile_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_profile (xl, strfd);
+    gf_proc_dump_xlator_profile(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops profile_file_ops = {
-	.file_fill = profile_file_fill,
+    .file_fill = profile_file_fill,
 };
 
-
 int
-meta_profile_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_profile_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &profile_file_ops);
+    meta_ops_set(loc->inode, this, &profile_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/root-dir.c b/xlators/meta/src/root-dir.c
index b57313fd9ad..80292bd3dda 100644
--- a/xlators/meta/src/root-dir.c
+++ b/xlators/meta/src/root-dir.c
@@ -8,67 +8,70 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent root_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "graphs",
-	  .type = IA_IFDIR,
-	  .hook = meta_graphs_dir_hook,
-	},
-	{ .name = "frames",
-	  .type = IA_IFREG,
-	  .hook = meta_frames_file_hook,
-	},
-	{ .name = "logging",
-	  .type = IA_IFDIR,
-	  .hook = meta_logging_dir_hook,
-	},
-	{ .name = "process_uuid",
-	  .type = IA_IFREG,
-	  .hook = meta_process_uuid_file_hook,
-	},
-	{ .name = "version",
-	  .type = IA_IFREG,
-	  .hook = meta_version_file_hook,
-	},
-	{ .name = "cmdline",
-	  .type = IA_IFREG,
-	  .hook = meta_cmdline_file_hook,
-	},
-	{ .name = "mallinfo",
-	  .type = IA_IFREG,
-	  .hook = meta_mallinfo_file_hook,
-	},
-	{ .name = "master",
-	  .type = IA_IFDIR,
-	  .hook = meta_master_dir_hook,
-	},
-	{ .name = "measure_latency",
-	  .type = IA_IFREG,
-	  .hook = meta_measure_file_hook,
-	},
-	{ .name = NULL }
-};
-
+    DOT_DOTDOT,
 
-static struct meta_ops meta_root_dir_ops = {
-	.fixed_dirents = root_dir_dirents
-};
+    {
+        .name = "graphs",
+        .type = IA_IFDIR,
+        .hook = meta_graphs_dir_hook,
+    },
+    {
+        .name = "frames",
+        .type = IA_IFREG,
+        .hook = meta_frames_file_hook,
+    },
+    {
+        .name = "logging",
+        .type = IA_IFDIR,
+        .hook = meta_logging_dir_hook,
+    },
+    {
+        .name = "process_uuid",
+        .type = IA_IFREG,
+        .hook = meta_process_uuid_file_hook,
+    },
+    {
+        .name = "version",
+        .type = IA_IFREG,
+        .hook = meta_version_file_hook,
+    },
+    {
+        .name = "cmdline",
+        .type = IA_IFREG,
+        .hook = meta_cmdline_file_hook,
+    },
+    {
+        .name = "mallinfo",
+        .type = IA_IFREG,
+        .hook = meta_mallinfo_file_hook,
+    },
+    {
+        .name = "master",
+        .type = IA_IFDIR,
+        .hook = meta_master_dir_hook,
+    },
+    {
+        .name = "measure_latency",
+        .type = IA_IFREG,
+        .hook = meta_measure_file_hook,
+    },
+    {.name = NULL}};
 
+static struct meta_ops meta_root_dir_ops = {.fixed_dirents = root_dir_dirents};
 
 int
-meta_root_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *xdata)
+meta_root_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &meta_root_dir_ops);
+    meta_ops_set(loc->inode, this, &meta_root_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/subvolume-link.c b/xlators/meta/src/subvolume-link.c
index 018d42a53a1..5b1f752efd0 100644
--- a/xlators/meta/src/subvolume-link.c
+++ b/xlators/meta/src/subvolume-link.c
@@ -8,54 +8,49 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-subvolume_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+subvolume_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (inode, this);
+    xl = meta_ctx_get(inode, this);
 
-	strprintf (strfd, "../../%s", xl->name);
+    strprintf(strfd, "../../%s", xl->name);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops subvolume_link_ops = {
-	.link_fill = subvolume_link_fill
-};
-
+struct meta_ops subvolume_link_ops = {.link_fill = subvolume_link_fill};
 
 int
-meta_subvolume_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			  dict_t *xdata)
+meta_subvolume_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         dict_t *xdata)
 {
-	int count = 0;
-	int i = 0;
-	xlator_t *xl = NULL;
-	xlator_list_t *subv = NULL;
-	xlator_t *subvol = NULL;
-
-	count = strtol (loc->name, 0, 0);
-	xl = meta_ctx_get (loc->parent, this);
-
-	for (subv = xl->children; subv; subv = subv->next) {
-		if (i == count) {
-			subvol = subv->xlator;
-			break;
-		}
-		i++;
-	}
-
-	meta_ctx_set (loc->inode, this, subvol);
-
-	meta_ops_set (loc->inode, this, &subvolume_link_ops);
-	return 0;
+    int count = 0;
+    int i = 0;
+    xlator_t *xl = NULL;
+    xlator_list_t *subv = NULL;
+    xlator_t *subvol = NULL;
+
+    count = strtol(loc->name, 0, 0);
+    xl = meta_ctx_get(loc->parent, this);
+
+    for (subv = xl->children; subv; subv = subv->next) {
+        if (i == count) {
+            subvol = subv->xlator;
+            break;
+        }
+        i++;
+    }
+
+    meta_ctx_set(loc->inode, this, subvol);
+
+    meta_ops_set(loc->inode, this, &subvolume_link_ops);
+    return 0;
 }
diff --git a/xlators/meta/src/subvolumes-dir.c b/xlators/meta/src/subvolumes-dir.c
index 00218b1a8ec..3cb170ea1f4 100644
--- a/xlators/meta/src/subvolumes-dir.c
+++ b/xlators/meta/src/subvolumes-dir.c
@@ -8,60 +8,55 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static int
-subvolumes_dir_fill (xlator_t *this, inode_t *dir, struct meta_dirent **dp)
+subvolumes_dir_fill(xlator_t *this, inode_t *dir, struct meta_dirent **dp)
 {
-	struct meta_dirent *dirents = NULL;
-	xlator_t *xl = NULL;
-	xlator_list_t *subv = NULL;
-	int i = 0;
-	int count = 0;
+    struct meta_dirent *dirents = NULL;
+    xlator_t *xl = NULL;
+    xlator_list_t *subv = NULL;
+    int i = 0;
+    int count = 0;
 
-	xl = meta_ctx_get (dir, this);
+    xl = meta_ctx_get(dir, this);
 
-	for (subv = xl->children; subv; subv = subv->next)
-		count++;
+    for (subv = xl->children; subv; subv = subv->next)
+        count++;
 
-	dirents = GF_CALLOC (sizeof (*dirents), count, gf_meta_mt_dirents_t);
-	if (!dirents)
-		return -1;
+    dirents = GF_MALLOC(sizeof(*dirents) * count, gf_meta_mt_dirents_t);
+    if (!dirents)
+        return -1;
 
-	for (subv = xl->children; subv; subv = subv->next) {
-		char num[16] = { };
-		snprintf (num, 16, "%d", i);
+    for (subv = xl->children; subv; subv = subv->next) {
+        char num[16] = {};
+        snprintf(num, 16, "%d", i);
 
-		dirents[i].name = gf_strdup (num);
-		dirents[i].type = IA_IFLNK;
-		dirents[i].hook = meta_subvolume_link_hook;
-		i++;
-	}
+        dirents[i].name = gf_strdup(num);
+        dirents[i].type = IA_IFLNK;
+        dirents[i].hook = meta_subvolume_link_hook;
+        i++;
+    }
 
-	*dp = dirents;
+    *dp = dirents;
 
-	return count;
+    return count;
 }
 
-
-static struct meta_ops subvolumes_dir_ops = {
-	.dir_fill = subvolumes_dir_fill
-};
-
+static struct meta_ops subvolumes_dir_ops = {.dir_fill = subvolumes_dir_fill};
 
 int
-meta_subvolumes_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			  dict_t *xdata)
+meta_subvolumes_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	meta_ops_set (loc->inode, this, &subvolumes_dir_ops);
+    meta_ops_set(loc->inode, this, &subvolumes_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/top-link.c b/xlators/meta/src/top-link.c
index 97cec0a2b62..33f0d407411 100644
--- a/xlators/meta/src/top-link.c
+++ b/xlators/meta/src/top-link.c
@@ -8,38 +8,33 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-top_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+top_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *graph = NULL;
 
-	graph = meta_ctx_get (inode, this);
+    graph = meta_ctx_get(inode, this);
 
-	strprintf (strfd, "%s", ((xlator_t *)graph->top)->name);
+    strprintf(strfd, "%s", ((xlator_t *)graph->top)->name);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops top_link_ops = {
-	.link_fill = top_link_fill
-};
-
+struct meta_ops top_link_ops = {.link_fill = top_link_fill};
 
 int
-meta_top_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *xdata)
+meta_top_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &top_link_ops);
+    meta_ops_set(loc->inode, this, &top_link_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/type-file.c b/xlators/meta/src/type-file.c
index f27e4b0a777..ece342a0b2a 100644
--- a/xlators/meta/src/type-file.c
+++ b/xlators/meta/src/type-file.c
@@ -8,41 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-type_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+type_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	strprintf (strfd, "%s\n", xl->type);
+    strprintf(strfd, "%s\n", xl->type);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops type_file_ops = {
-	.file_fill = type_file_fill,
+    .file_fill = type_file_fill,
 };
 
-
 int
-meta_type_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_type_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &type_file_ops);
+    meta_ops_set(loc->inode, this, &type_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/version-file.c b/xlators/meta/src/version-file.c
index ace419ea439..36276fb810a 100644
--- a/xlators/meta/src/version-file.c
+++ b/xlators/meta/src/version-file.c
@@ -8,35 +8,30 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-version_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+version_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-        strprintf (strfd, "{ \n  \"Package Version\": \"%s\"\n}",
-                   PACKAGE_VERSION);
-        return strfd->size;
+    strprintf(strfd, "{ \n  \"Package Version\": \"%s\"\n}", PACKAGE_VERSION);
+    return strfd->size;
 }
 
-
 static struct meta_ops version_file_ops = {
-        .file_fill = version_file_fill,
+    .file_fill = version_file_fill,
 };
 
-
 int
-meta_version_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        dict_t *xdata)
+meta_version_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-        meta_ops_set (loc->inode, this, &version_file_ops);
+    meta_ops_set(loc->inode, this, &version_file_ops);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/view-dir.c b/xlators/meta/src/view-dir.c
index dc208cb5b65..30931061567 100644
--- a/xlators/meta/src/view-dir.c
+++ b/xlators/meta/src/view-dir.c
@@ -8,33 +8,26 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
+static struct meta_dirent view_dir_dirents[] = {DOT_DOTDOT,
 
-static struct meta_dirent view_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = NULL }
-};
-
-
-static struct meta_ops view_dir_ops = {
-	.fixed_dirents = view_dir_dirents
-};
+                                                {.name = NULL}};
 
+static struct meta_ops view_dir_ops = {.fixed_dirents = view_dir_dirents};
 
 int
-meta_view_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *xdata)
+meta_view_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	meta_ops_set (loc->inode, this, &view_dir_ops);
+    meta_ops_set(loc->inode, this, &view_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/volfile-file.c b/xlators/meta/src/volfile-file.c
index c6027658fee..b2e2562ab8b 100644
--- a/xlators/meta/src/volfile-file.c
+++ b/xlators/meta/src/volfile-file.c
@@ -8,79 +8,72 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-
-
+#include <glusterfs/strfd.h>
 
 static int
-xldump_options (dict_t *this, char *key, data_t *value,	void *strfd)
+xldump_options(dict_t *this, char *key, data_t *value, void *strfd)
 {
-	strprintf (strfd, "    option %s %s\n", key, value->data);
-	return 0;
+    strprintf(strfd, "    option %s %s\n", key, value->data);
+    return 0;
 }
 
-
 static void
-xldump_subvolumes (xlator_t *this, void *strfd)
+xldump_subvolumes(xlator_t *this, void *strfd)
 {
-	xlator_list_t *subv = NULL;
+    xlator_list_t *subv = NULL;
 
-	if (!this->children)
-		return;
+    if (!this->children)
+        return;
 
-	strprintf (strfd, "    subvolumes");
+    strprintf(strfd, "    subvolumes");
 
-	for (subv = this->children; subv; subv= subv->next)
-		strprintf (strfd, " %s", subv->xlator->name);
+    for (subv = this->children; subv; subv = subv->next)
+        strprintf(strfd, " %s", subv->xlator->name);
 
-	strprintf (strfd, "\n");
+    strprintf(strfd, "\n");
 }
 
-
 static void
-xldump (xlator_t *each, void *strfd)
+xldump(xlator_t *each, void *strfd)
 {
-	strprintf (strfd, "volume %s\n", each->name);
-	strprintf (strfd, "    type %s\n", each->type);
-	dict_foreach (each->options, xldump_options, strfd);
+    strprintf(strfd, "volume %s\n", each->name);
+    strprintf(strfd, "    type %s\n", each->type);
+    dict_foreach(each->options, xldump_options, strfd);
 
-	xldump_subvolumes (each, strfd);
+    xldump_subvolumes(each, strfd);
 
-	strprintf (strfd, "end-volume\n");
-	strprintf (strfd, "\n");
+    strprintf(strfd, "end-volume\n");
+    strprintf(strfd, "\n");
 }
 
-
 static int
-volfile_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+volfile_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *graph = NULL;
 
-	graph = meta_ctx_get (file, this);
+    graph = meta_ctx_get(file, this);
 
-	xlator_foreach_depth_first (graph->top, xldump, strfd);
+    xlator_foreach_depth_first(graph->top, xldump, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops volfile_file_ops = {
-	.file_fill = volfile_file_fill,
+    .file_fill = volfile_file_fill,
 };
 
-
 int
-meta_volfile_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_volfile_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &volfile_file_ops);
+    meta_ops_set(loc->inode, this, &volfile_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/xlator-dir.c b/xlators/meta/src/xlator-dir.c
index 910e82b3871..86189715790 100644
--- a/xlators/meta/src/xlator-dir.c
+++ b/xlators/meta/src/xlator-dir.c
@@ -8,88 +8,90 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent xlator_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "view",
-	  .type = IA_IFDIR,
-	  .hook = meta_view_dir_hook,
-	},
-	{ .name = "type",
-	  .type = IA_IFREG,
-	  .hook = meta_type_file_hook,
-	},
-	{ .name = "name",
-	  .type = IA_IFREG,
-	  .hook = meta_name_file_hook,
-	},
-	{ .name = "subvolumes",
-	  .type = IA_IFDIR,
-	  .hook = meta_subvolumes_dir_hook,
-	},
-	{ .name = "options",
-	  .type = IA_IFDIR,
-	  .hook = meta_options_dir_hook,
-	},
-	{ .name = "private",
-	  .type = IA_IFREG,
-	  .hook = meta_private_file_hook,
-	},
-	{ .name = "history",
-	  .type = IA_IFREG,
-	  .hook = meta_history_file_hook,
-	},
-	{ .name = "meminfo",
-	  .type = IA_IFREG,
-	  .hook = meta_meminfo_file_hook,
-	},
-	{ .name = "profile",
-	  .type = IA_IFREG,
-	  .hook = meta_profile_file_hook,
-	},
-	{ .name = NULL }
-};
-
-
-static struct meta_ops xlator_dir_ops = {
-	.fixed_dirents = xlator_dir_dirents
-};
-
+    DOT_DOTDOT,
+
+    {
+        .name = "view",
+        .type = IA_IFDIR,
+        .hook = meta_view_dir_hook,
+    },
+    {
+        .name = "type",
+        .type = IA_IFREG,
+        .hook = meta_type_file_hook,
+    },
+    {
+        .name = "name",
+        .type = IA_IFREG,
+        .hook = meta_name_file_hook,
+    },
+    {
+        .name = "subvolumes",
+        .type = IA_IFDIR,
+        .hook = meta_subvolumes_dir_hook,
+    },
+    {
+        .name = "options",
+        .type = IA_IFDIR,
+        .hook = meta_options_dir_hook,
+    },
+    {
+        .name = "private",
+        .type = IA_IFREG,
+        .hook = meta_private_file_hook,
+    },
+    {
+        .name = "history",
+        .type = IA_IFREG,
+        .hook = meta_history_file_hook,
+    },
+    {
+        .name = "meminfo",
+        .type = IA_IFREG,
+        .hook = meta_meminfo_file_hook,
+    },
+    {
+        .name = "profile",
+        .type = IA_IFREG,
+        .hook = meta_profile_file_hook,
+    },
+    {.name = NULL}};
+
+static struct meta_ops xlator_dir_ops = {.fixed_dirents = xlator_dir_dirents};
 
 int
-meta_xlator_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      dict_t *xdata)
+meta_xlator_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-	glusterfs_graph_t *graph = NULL;
-	xlator_t *xl = NULL;
+    glusterfs_graph_t *graph = NULL;
+    xlator_t *xl = NULL;
 
-	graph = meta_ctx_get (loc->parent, this);
+    graph = meta_ctx_get(loc->parent, this);
 
-	xl = xlator_search_by_name (graph->first, loc->name);
+    xl = xlator_search_by_name(graph->first, loc->name);
 
-	meta_ctx_set (loc->inode, this, xl);
+    meta_ctx_set(loc->inode, this, xl);
 
-	meta_ops_set (loc->inode, this, &xlator_dir_ops);
+    meta_ops_set(loc->inode, this, &xlator_dir_ops);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_master_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      dict_t *xdata)
+meta_master_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, this->ctx->master);
+    meta_ctx_set(loc->inode, this, this->ctx->master);
 
-	meta_ops_set (loc->inode, this, &xlator_dir_ops);
+    meta_ops_set(loc->inode, this, &xlator_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index f3381e34930..685beb42d27 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -1,67 +1,79 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = glusterd.la
+endif
+
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mgmt
-glusterd_la_CPPFLAGS = $(AM_CPPFLAGS) "-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\""
+glusterd_la_CPPFLAGS = $(AM_CPPFLAGS) \
+	-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\" \
+	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-I$(top_srcdir)/libglusterd/src/
+
 glusterd_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \
 	glusterd-op-sm.c glusterd-utils.c glusterd-rpc-ops.c \
 	glusterd-store.c glusterd-handshake.c glusterd-pmap.c \
-	glusterd-volgen.c glusterd-rebalance.c glusterd-ganesha.c \
+	glusterd-volgen.c glusterd-rebalance.c \
 	glusterd-quota.c glusterd-bitrot.c glusterd-geo-rep.c \
 	glusterd-replace-brick.c glusterd-log-ops.c \
 	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c \
 	glusterd-syncop.c glusterd-hooks.c glusterd-volume-set.c \
 	glusterd-locks.c glusterd-snapshot.c glusterd-mgmt-handler.c \
 	glusterd-mgmt.c glusterd-peer-utils.c glusterd-statedump.c \
-        glusterd-snapshot-utils.c glusterd-conn-mgmt.c \
-	glusterd-proc-mgmt.c glusterd-svc-mgmt.c glusterd-shd-svc.c \
+	glusterd-snapshot-utils.c glusterd-conn-mgmt.c \
+	glusterd-proc-mgmt.c glusterd-svc-mgmt.c \
 	glusterd-nfs-svc.c glusterd-quotad-svc.c glusterd-svc-helper.c \
 	glusterd-conn-helper.c glusterd-snapd-svc.c glusterd-snapd-svc-helper.c \
-	glusterd-bitd-svc.c glusterd-scrub-svc.c glusterd-server-quorum.c
-
+	glusterd-bitd-svc.c glusterd-scrub-svc.c glusterd-server-quorum.c \
+	glusterd-reset-brick.c glusterd-shd-svc.c glusterd-shd-svc-helper.c \
+        glusterd-gfproxyd-svc.c glusterd-gfproxyd-svc-helper.c glusterd-ganesha.c \
+	$(CONTRIBDIR)/mount/mntent.c
 
 glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-		     $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		     $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-		     $(XML_LIBS) -lcrypto $(URCU_LIBS) $(URCU_CDS_LIBS)
-if ENABLE_BD_XLATOR
-glusterd_la_LIBADD += -llvm2app
-endif
+	$(top_builddir)/libglusterd/src/libglusterd.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(XML_LIBS) -lcrypto $(URCU_LIBS) $(URCU_CDS_LIBS) $(LIB_DL) $(GF_XLATOR_MGNT_LIBADD)
 
 noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \
 	glusterd-sm.h glusterd-store.h glusterd-mem-types.h \
 	glusterd-pmap.h glusterd-volgen.h glusterd-mountbroker.h \
-	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h \
+	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h glusterd-quota.h \
 	glusterd-mgmt.h glusterd-messages.h glusterd-peer-utils.h \
 	glusterd-statedump.h glusterd-snapshot-utils.h glusterd-geo-rep.h \
 	glusterd-conn-mgmt.h glusterd-conn-helper.h glusterd-proc-mgmt.h \
-	glusterd-svc-mgmt.h glusterd-shd-svc.h glusterd-nfs-svc.h \
+	glusterd-svc-mgmt.h glusterd-nfs-svc.h \
 	glusterd-quotad-svc.h glusterd-svc-helper.h glusterd-snapd-svc.h \
 	glusterd-snapd-svc-helper.h glusterd-rcu.h glusterd-bitd-svc.h \
 	glusterd-scrub-svc.h glusterd-server-quorum.h glusterd-errno.h \
-	$(CONTRIBDIR)/userspace-rcu/rculist-extra.h
+        glusterd-shd-svc.h glusterd-shd-svc-helper.h \
+        glusterd-gfproxyd-svc.h glusterd-gfproxyd-svc-helper.h \
+	$(CONTRIBDIR)/userspace-rcu/rculist-extra.h \
+	$(CONTRIBDIR)/mount/mntent_compat.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(CONTRIBDIR)/rbtree \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
-	-I$(CONTRIBDIR)/mount \
-	-I$(CONTRIBDIR)/userspace-rcu \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(CONTRIBDIR)/rbtree -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(CONTRIBDIR)/mount -I$(CONTRIBDIR)/userspace-rcu \
 	-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\" \
-	-DCONFDIR=\"$(sysconfdir)/ganesha\" \
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\" \
+	-DCONFDIR=\"$(localstatedir)/run/gluster/shared_storage/nfs-ganesha\" \
 	-DGANESHA_PREFIX=\"$(libexecdir)/ganesha\" \
-	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) \
+	-I$(top_srcdir)/libglusterd/src/
 
 
-AM_CFLAGS = -Wall $(GF_CFLAGS) $(URCU_CFLAGS) $(URCU_CDS_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(URCU_CFLAGS) $(URCU_CDS_CFLAGS) $(XML_CFLAGS)
 
 AM_LDFLAGS = -L$(xlatordir) $(URCU_LIBS) $(URCU_CDS_LIBS)
 
 CLEANFILES =
 
 install-data-hook:
+if WITH_SERVER
 if GF_INSTALL_GLUSTERD_WORKDIR
 	$(mkdir_p) $(DESTDIR)$(GLUSTERD_WORKDIR)
 	(stat $(DESTDIR)$(sysconfdir)/glusterd && \
 	    mv $(DESTDIR)$(sysconfdir)/glusterd $(DESTDIR)$(GLUSTERD_WORKDIR)) || true;
 	(ln -sf $(DESTDIR)$(GLUSTERD_WORKDIR) $(sysconfdir)/glusterd) || true;
 endif
+endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
index ee96ccbff80..6adb799b18f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
@@ -8,8 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
@@ -17,191 +17,190 @@
 #include "glusterd-svc-helper.h"
 
 void
-glusterd_bitdsvc_build (glusterd_svc_t *svc)
+glusterd_bitdsvc_build(glusterd_svc_t *svc)
 {
-        svc->manager = glusterd_bitdsvc_manager;
-        svc->start = glusterd_bitdsvc_start;
-        svc->stop = glusterd_bitdsvc_stop;
+    svc->manager = glusterd_bitdsvc_manager;
+    svc->start = glusterd_bitdsvc_start;
+    svc->stop = glusterd_bitdsvc_stop;
 }
 
 int
-glusterd_bitdsvc_init (glusterd_svc_t *svc)
+glusterd_bitdsvc_init(glusterd_svc_t *svc)
 {
-        return glusterd_svc_init (svc, bitd_svc_name);
+    return glusterd_svc_init(svc, bitd_svc_name);
 }
 
 static int
-glusterd_bitdsvc_create_volfile ()
+glusterd_bitdsvc_create_volfile()
 {
-        char              filepath[PATH_MAX] = {0,};
-        int               ret                = -1;
-        glusterd_conf_t   *conf              = NULL;
-        xlator_t          *this              = NULL;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-
-        glusterd_svc_build_volfile_path (bitd_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
-
-        ret = glusterd_create_global_volfile (build_bitd_graph,
-                                              filepath, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "Failed to create volfile");
-                goto out;
-        }
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    glusterd_svc_build_volfile_path(bitd_svc_name, conf->workdir, filepath,
+                                    sizeof(filepath));
+
+    ret = glusterd_create_global_volfile(build_bitd_graph, filepath, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_bitdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_bitdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                      ret           = 0;
-        xlator_t                *this          = NULL;
-        glusterd_brickinfo_t    *brickinfo     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!svc->inited) {
-                ret = glusterd_bitdsvc_init (svc);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BITD_INIT_FAIL, "Failed to init "
-                                "bitd service");
-                        goto out;
-                } else {
-                        svc->inited = _gf_true;
-                        gf_msg_debug (this->name, 0, "BitD service "
-                                      "initialized");
-                }
-        }
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (glusterd_should_i_stop_bitd ()) {
-                ret = svc->stop (svc, SIGTERM);
+    if (!svc->inited) {
+        ret = glusterd_bitdsvc_init(svc);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITD_INIT_FAIL,
+                   "Failed to init "
+                   "bitd service");
+            goto out;
         } else {
-                ret = glusterd_bitdsvc_create_volfile ();
-                if (ret)
-                        goto out;
+            svc->inited = _gf_true;
+            gf_msg_debug(this->name, 0,
+                         "BitD service "
+                         "initialized");
+        }
+    }
+
+    if (glusterd_should_i_stop_bitd()) {
+        ret = svc->stop(svc, SIGTERM);
+    } else {
+        ret = glusterd_bitdsvc_create_volfile();
+        if (ret)
+            goto out;
 
-                ret = svc->stop (svc, SIGKILL);
-                if (ret)
-                        goto out;
+        ret = svc->stop(svc, SIGKILL);
+        if (ret)
+            goto out;
 
-                ret = svc->start (svc, flags);
-                if (ret)
-                        goto out;
+        ret = svc->start(svc, flags);
+        if (ret)
+            goto out;
 
-                ret = glusterd_conn_connect (&(svc->conn));
-                if (ret)
-                        goto out;
-        }
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_bitdsvc_start (glusterd_svc_t *svc, int flags)
+glusterd_bitdsvc_start(glusterd_svc_t *svc, int flags)
 {
-        int ret = -1;
-        dict_t *cmdict = NULL;
+    int ret = -1;
+    dict_t *cmdict = NULL;
 
-        cmdict = dict_new ();
-        if (!cmdict)
-                goto error_return;
+    cmdict = dict_new();
+    if (!cmdict)
+        goto error_return;
 
-        ret = dict_set_str (cmdict, "cmdarg0", "--global-timer-wheel");
-        if (ret)
-                goto dealloc_dict;
+    ret = dict_set_str(cmdict, "cmdarg0", "--global-timer-wheel");
+    if (ret)
+        goto dealloc_dict;
 
-        ret = glusterd_svc_start (svc, flags, cmdict);
+    ret = glusterd_svc_start(svc, flags, cmdict);
 
- dealloc_dict:
-        dict_unref (cmdict);
- error_return:
-        return ret;
+dealloc_dict:
+    dict_unref(cmdict);
+error_return:
+    return ret;
 }
 
 int
-glusterd_bitdsvc_stop (glusterd_svc_t *svc, int sig)
+glusterd_bitdsvc_stop(glusterd_svc_t *svc, int sig)
 {
-        return glusterd_svc_stop (svc, sig);
+    return glusterd_svc_stop(svc, sig);
 }
 
 int
-glusterd_bitdsvc_reconfigure ()
+glusterd_bitdsvc_reconfigure()
 {
-        int              ret             = -1;
-        xlator_t        *this            = NULL;
-        glusterd_conf_t *priv            = NULL;
-        gf_boolean_t     identical       = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        if (glusterd_should_i_stop_bitd ())
-                goto manager;
-        /*
-         * Check both OLD and NEW volfiles, if they are SAME by size
-         * and cksum i.e. "character-by-character". If YES, then
-         * NOTHING has been changed, just return.
-         */
-        ret = glusterd_svc_check_volfile_identical (priv->bitd_svc.name,
-                                                    build_bitd_graph,
-                                                    &identical);
-        if (ret)
-                goto out;
-        if (identical) {
-                ret = 0;
-                goto out;
-        }
-
-        /*
-         * They are not identical. Find out if the topology is changed
-         * OR just the volume options. If just the options which got
-         * changed, then inform the xlator to reconfigure the options.
-         */
-        identical = _gf_false; /* RESET the FLAG */
-        ret = glusterd_svc_check_topology_identical (priv->bitd_svc.name,
-                                                     build_bitd_graph,
-                                                     &identical);
-        if (ret)
-                goto out; /*not able to compare due to some corruption */
-
-        /* Topology is not changed, but just the options. But write the
-         * options to bitd volfile, so that bitd will be reconfigured.
-         */
-        if (identical) {
-                ret = glusterd_bitdsvc_create_volfile ();
-                if (ret == 0) {/* Only if above PASSES */
-                        ret = glusterd_fetchspec_notify (THIS);
-                }
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (glusterd_should_i_stop_bitd())
+        goto manager;
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_volfile_identical(priv->bitd_svc.name,
+                                               build_bitd_graph, &identical);
+    if (ret)
+        goto out;
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->bitd_svc.name,
+                                                build_bitd_graph, &identical);
+    if (ret)
+        goto out; /*not able to compare due to some corruption */
+
+    /* Topology is not changed, but just the options. But write the
+     * options to bitd volfile, so that bitd will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_bitdsvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
         }
+        goto out;
+    }
 
 manager:
-        /*
-         * bitd volfile's topology has been changed. bitd server needs
-         * to be RESTARTED to ACT on the changed volfile.
-         */
-        ret = priv->bitd_svc.manager (&(priv->bitd_svc), NULL,
-                                     PROC_START_NO_WAIT);
+    /*
+     * bitd volfile's topology has been changed. bitd server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL, PROC_START_NO_WAIT);
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h
index 7f276fb0b5c..1bff084a9a8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h
@@ -16,25 +16,25 @@
 #define bitd_svc_name "bitd"
 
 void
-glusterd_bitdsvc_build (glusterd_svc_t *svc);
+glusterd_bitdsvc_build(glusterd_svc_t *svc);
 
 int
-glusterd_bitdsvc_init (glusterd_svc_t *svc);
+glusterd_bitdsvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_bitdsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_bitdsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_bitdsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_bitdsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_bitdsvc_stop (glusterd_svc_t *svc, int sig);
+glusterd_bitdsvc_stop(glusterd_svc_t *svc, int sig);
 
 int
-glusterd_bitdsvc_reconfigure ();
+glusterd_bitdsvc_reconfigure();
 
 void
-glusterd_bitdsvc_build_volfile_path (char *server, char *workdir,
-                                     char *volfile, size_t len);
+glusterd_bitdsvc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                    size_t len);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitrot.c b/xlators/mgmt/glusterd/src/glusterd-bitrot.c
index 6e91106c8e5..37429fe9214 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitrot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-bitrot.c
@@ -8,7 +8,7 @@
    cases as published by the Free Software Foundation.
  */
 
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -16,10 +16,10 @@
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
-#include "run.h"
-#include "syscall.h"
-#include "byte-order.h"
-#include "compat-errno.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/compat-errno.h>
 #include "glusterd-scrub-svc.h"
 #include "glusterd-messages.h"
 
@@ -27,683 +27,796 @@
 #include <dlfcn.h>
 
 const char *gd_bitrot_op_list[GF_BITROT_OPTION_TYPE_MAX] = {
-        [GF_BITROT_OPTION_TYPE_NONE]            = "none",
-        [GF_BITROT_OPTION_TYPE_ENABLE]          = "enable",
-        [GF_BITROT_OPTION_TYPE_DISABLE]         = "disable",
-        [GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE]  = "scrub-throttle",
-        [GF_BITROT_OPTION_TYPE_SCRUB_FREQ]      = "scrub-frequency",
-        [GF_BITROT_OPTION_TYPE_SCRUB]           = "scrub",
-        [GF_BITROT_OPTION_TYPE_EXPIRY_TIME]     = "expiry-time",
+    [GF_BITROT_OPTION_TYPE_NONE] = "none",
+    [GF_BITROT_OPTION_TYPE_ENABLE] = "enable",
+    [GF_BITROT_OPTION_TYPE_DISABLE] = "disable",
+    [GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE] = "scrub-throttle",
+    [GF_BITROT_OPTION_TYPE_SCRUB_FREQ] = "scrub-frequency",
+    [GF_BITROT_OPTION_TYPE_SCRUB] = "scrub",
+    [GF_BITROT_OPTION_TYPE_EXPIRY_TIME] = "expiry-time",
+    [GF_BITROT_OPTION_TYPE_SIGNER_THREADS] = "signer-threads",
 };
 
 int
-__glusterd_handle_bitrot (rpcsvc_request_t *req)
+__glusterd_handle_bitrot(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gf_cli_req                      cli_req   = { {0,} };
-        dict_t                         *dict      = NULL;
-        glusterd_op_t                   cli_op    = GD_OP_BITROT;
-        char                           *volname   = NULL;
-        char                           *scrub     = NULL;
-        int32_t                         type      = 0;
-        char                            msg[2048] = {0,};
-        xlator_t                       *this      = NULL;
-        glusterd_conf_t                *conf      = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_BITROT;
+    char *volname = NULL;
+    char *scrub = NULL;
+    int32_t type = 0;
+    char msg[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name, "
+               "while handling bitrot command");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get type of command");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type of cmd, "
+               "while handling bitrot command");
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_3_7_0) {
+        snprintf(msg, sizeof(msg),
+                 "Cannot execute command. The "
+                 "cluster is operating at version %d. Bitrot command "
+                 "%s is unavailable in this version",
+                 conf->op_version, gd_bitrot_op_list[type]);
+        ret = -1;
+        goto out;
+    }
+
+    if (type == GF_BITROT_CMD_SCRUB_STATUS) {
+        /* Backward compatibility handling for scrub status command*/
+        if (conf->op_version < GD_OP_VERSION_3_7_7) {
+            snprintf(msg, sizeof(msg),
+                     "Cannot execute command. "
+                     "The cluster is operating at version %d. "
+                     "Bitrot scrub status command unavailable in "
+                     "this version",
+                     conf->op_version);
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_get_str(dict, "scrub-value", &scrub);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get scrub value.");
+            ret = -1;
+            goto out;
         }
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+        if (!strncmp(scrub, "status", SLEN("status"))) {
+            ret = glusterd_op_begin_synctask(req, GD_OP_SCRUB_STATUS, dict);
+            goto out;
         }
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get volume name");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name, "
-                        "while handling bitrot command");
-                goto out;
+    if (type == GF_BITROT_CMD_SCRUB_ONDEMAND) {
+        /* Backward compatibility handling for scrub status command*/
+        if (conf->op_version < GD_OP_VERSION_3_9_0) {
+            snprintf(msg, sizeof(msg),
+                     "Cannot execute command. "
+                     "The cluster is operating at version %d. "
+                     "Bitrot scrub ondemand command unavailable in "
+                     "this version",
+                     conf->op_version);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "type", &type);
+        ret = dict_get_str(dict, "scrub-value", &scrub);
         if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get type of command");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get type of cmd, "
-                        "while handling bitrot command");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get scrub value.");
+            ret = -1;
+            goto out;
         }
 
-        if (conf->op_version < GD_OP_VERSION_3_7_0) {
-                snprintf (msg, sizeof (msg), "Cannot execute command. The "
-                          "cluster is operating at version %d. Bitrot command "
-                          "%s is unavailable in this version", conf->op_version,
-                          gd_bitrot_op_list[type]);
-                ret = -1;
-                goto out;
+        if (!strncmp(scrub, "ondemand", SLEN("ondemand"))) {
+            ret = glusterd_op_begin_synctask(req, GD_OP_SCRUB_ONDEMAND, dict);
+            goto out;
         }
+    }
 
-        if (type == GF_BITROT_CMD_SCRUB_STATUS) {
-                /* Backward compatibility handling for scrub status command*/
-                if (conf->op_version < GD_OP_VERSION_3_7_7) {
-                        snprintf (msg, sizeof (msg), "Cannot execute command. "
-                                  "The cluster is operating at version %d. "
-                                  "Bitrot scrub status command unavailable in "
-                                  "this version", conf->op_version);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_get_str (dict, "scrub-value", &scrub);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Failed to get scrub value.");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!strncmp (scrub, "status", strlen ("status"))) {
-                        ret = glusterd_op_begin_synctask (req,
-                                                          GD_OP_SCRUB_STATUS,
-                                                          dict);
-                        goto out;
-                }
-        }
-
-        ret = glusterd_op_begin_synctask (req, GD_OP_BITROT, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_BITROT, dict);
 
 out:
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Bitrot operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, msg);
-        }
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Bitrot operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, msg);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_bitrot (rpcsvc_request_t *req)
+glusterd_handle_bitrot(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_bitrot);
+    return glusterd_big_locked_handler(req, __glusterd_handle_bitrot);
 }
 
 static int
-glusterd_bitrot_scrub_throttle (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                char *key, char **op_errstr)
+glusterd_bitrot_scrub_throttle(glusterd_volinfo_t *volinfo, dict_t *dict,
+                               char *key, char **op_errstr)
 {
-        int32_t        ret                  = -1;
-        char           *scrub_throttle      = NULL;
-        char           *option              = NULL;
-        xlator_t       *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "scrub-throttle-value", &scrub_throttle);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch scrub-"
-                        "throttle value");
-                goto out;
-        }
-
-        option = gf_strdup (scrub_throttle);
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set option %s",
-                        key);
-                goto out;
-        }
-
-        ret = glusterd_scrubsvc_reconfigure ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SCRUBSVC_RECONF_FAIL,
-                        "Failed to reconfigure scrub "
-                        "services");
-                goto out;
-        }
+    int32_t ret = -1;
+    char *scrub_throttle = NULL;
+    char *option = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_str(dict, "scrub-throttle-value", &scrub_throttle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch scrub-"
+               "throttle value");
+        goto out;
+    }
+
+    option = gf_strdup(scrub_throttle);
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_scrubsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
+               "Failed to reconfigure scrub "
+               "services");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_scrub_freq (glusterd_volinfo_t *volinfo, dict_t *dict,
-                            char *key, char **op_errstr)
+glusterd_bitrot_scrub_freq(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                           char **op_errstr)
 {
-        int32_t        ret                  = -1;
-        char           *scrub_freq          = NULL;
-        xlator_t       *this                = NULL;
-        char           *option              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "scrub-frequency-value", &scrub_freq);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch scrub-"
-                        "freq value");
-                goto out;
-        }
-
-        option = gf_strdup (scrub_freq);
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set option %s",
-                        key);
-                goto out;
-        }
-
-        ret = glusterd_scrubsvc_reconfigure ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SCRUBSVC_RECONF_FAIL,
-                        "Failed to reconfigure scrub "
-                        "services");
-                goto out;
-        }
+    int32_t ret = -1;
+    char *scrub_freq = NULL;
+    xlator_t *this = NULL;
+    char *option = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_str(dict, "scrub-frequency-value", &scrub_freq);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch scrub-"
+               "freq value");
+        goto out;
+    }
+
+    option = gf_strdup(scrub_freq);
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_scrubsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
+               "Failed to reconfigure scrub "
+               "services");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_scrub (glusterd_volinfo_t *volinfo, dict_t *dict,
-                       char *key, char **op_errstr)
+glusterd_bitrot_scrub(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                      char **op_errstr)
 {
-        int32_t        ret                  = -1;
-        char           *scrub_value         = NULL;
-        xlator_t       *this                = NULL;
-        char           *option              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "scrub-value", &scrub_value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch scrub"
-                        "value");
-                goto out;
-        }
-
-        if (!strcmp (scrub_value, "resume")) {
-                option = gf_strdup ("Active");
-        } else {
-                option = gf_strdup (scrub_value);
-        }
-
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set option %s",
-                        key);
-                goto out;
-        }
-
-        ret = glusterd_scrubsvc_reconfigure ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SCRUBSVC_RECONF_FAIL,
-                        "Failed to reconfigure scrub "
-                        "services");
-                goto out;
-        }
+    int32_t ret = -1;
+    char *scrub_value = NULL;
+    xlator_t *this = NULL;
+    char *option = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_str(dict, "scrub-value", &scrub_value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch scrub"
+               "value");
+        goto out;
+    }
+
+    if (!strcmp(scrub_value, "resume")) {
+        option = gf_strdup("Active");
+    } else {
+        option = gf_strdup(scrub_value);
+    }
+
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_scrubsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
+               "Failed to reconfigure scrub "
+               "services");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_expiry_time (glusterd_volinfo_t *volinfo, dict_t *dict,
-                             char *key, char **op_errstr)
+glusterd_bitrot_expiry_time(glusterd_volinfo_t *volinfo, dict_t *dict,
+                            char *key, char **op_errstr)
 {
-        int32_t        ret                  = -1;
-        uint32_t       expiry_time          = 0;
-        xlator_t       *this                = NULL;
-        char           dkey[1024]           = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_uint32 (dict, "expiry-time", &expiry_time);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get bitrot expiry"
-                        " timer value.");
-                goto out;
-        }
-
-        snprintf (dkey, sizeof (dkey), "%d", expiry_time);
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, key, dkey);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set option %s",
-                        key);
-                goto out;
-        }
+    int32_t ret = -1;
+    uint32_t expiry_time = 0;
+    xlator_t *this = NULL;
+    char dkey[32] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_uint32(dict, "expiry-time", &expiry_time);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bitrot expiry"
+               " timer value.");
+        goto out;
+    }
+
+    snprintf(dkey, sizeof(dkey), "%d", expiry_time);
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, key, dkey);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_bitdsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITDSVC_RECONF_FAIL,
+               "Failed to reconfigure bitrot"
+               "services");
+        goto out;
+    }
+out:
+    return ret;
+}
 
-        ret = glusterd_bitdsvc_reconfigure ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BITDSVC_RECONF_FAIL,
-                        "Failed to reconfigure bitrot"
-                         "services");
-                goto out;
-        }
+static gf_boolean_t
+is_bitd_configure_noop(xlator_t *this, glusterd_volinfo_t *volinfo)
+{
+    gf_boolean_t noop = _gf_true;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    if (!glusterd_is_bitrot_enabled(volinfo))
+        goto out;
+    else if (volinfo->status != GLUSTERD_STATUS_STARTED)
+        goto out;
+    else {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+                continue;
+            noop = _gf_false;
+            return noop;
+        }
+    }
 out:
-        return ret;
+    return noop;
 }
 
 static int
-glusterd_bitrot_enable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_bitrot_signer_threads(glusterd_volinfo_t *volinfo, dict_t *dict,
+                               char *key, char **op_errstr)
 {
-        int32_t         ret             = -1;
-        xlator_t        *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        if (glusterd_is_volume_started (volinfo) == 0) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "to enable bitrot.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_is_bitrot_enabled (volinfo);
-        if (ret) {
-                *op_errstr = gf_strdup ("Bitrot is already enabled");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, VKEY_FEATURES_BITROT,
-                                          "on");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "dict set failed");
-                goto out;
-        }
-
-        /*Once bitrot is enable scrubber should be in Active state*/
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, "features.scrub",
-                                          "Active");
+    int32_t ret = -1;
+    uint32_t signer_th_count = 0;
+    uint32_t existing_th_count = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char dkey[32] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    ret = dict_get_uint32(dict, "signer-threads", &signer_th_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bitrot signer thread count.");
+        goto out;
+    }
+
+    ret = dict_get_uint32(volinfo->dict, key, &existing_th_count);
+    if (ret == 0 && signer_th_count == existing_th_count) {
+        goto out;
+    }
+
+    snprintf(dkey, sizeof(dkey), "%d", signer_th_count);
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, key, dkey);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    if (!is_bitd_configure_noop(this, volinfo)) {
+        ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL,
+                                     PROC_START_NO_WAIT);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set option "
-                        "features.scrub value");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITDSVC_RECONF_FAIL,
+                   "Failed to reconfigure bitrot services");
+            goto out;
         }
-
-        ret = 0;
+    }
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Enabling bitrot on volume %s has been "
-                             "unsuccessful", volinfo->volname);
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_bitrot_enable(glusterd_volinfo_t *volinfo, char **op_errstr)
 {
-        int32_t           ret            = -1;
-        xlator_t          *this          = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, VKEY_FEATURES_BITROT,
-                                          "off");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "dict set failed");
-                goto out;
-        }
-
-        /*Once bitrot disabled scrubber should be Inactive state*/
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, "features.scrub",
-                                          "Inactive");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "features.scrub value");
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    if (glusterd_is_volume_started(volinfo) == 0) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "to enable bitrot.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_is_bitrot_enabled(volinfo);
+    if (ret) {
+        *op_errstr = gf_strdup("Bitrot is already enabled");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_BITROT, "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    /*Once bitrot is enable scrubber should be in Active state*/
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, "features.scrub", "Active");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option "
+               "features.scrub value");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Enabling bitrot on volume %s has been "
+                    "unsuccessful",
+                    volinfo->volname);
+    return ret;
+}
 
-        ret = 0;
+static int
+glusterd_bitrot_disable(glusterd_volinfo_t *volinfo, char **op_errstr)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_BITROT,
+                                     "off");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    /*Once bitrot disabled scrubber should be Inactive state*/
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, "features.scrub",
+                                     "Inactive");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "features.scrub value");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Disabling bitrot on volume %s has "
-                             "been unsuccessful", volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Disabling bitrot on volume %s has "
+                    "been unsuccessful",
+                    volinfo->volname);
+    return ret;
 }
 
 gf_boolean_t
-glusterd_should_i_stop_bitd ()
+glusterd_should_i_stop_bitd()
 {
-        glusterd_conf_t       *conf       = THIS->private;
-        glusterd_volinfo_t    *volinfo    = NULL;
-        gf_boolean_t           stopped    = _gf_true;
-        glusterd_brickinfo_t  *brickinfo  = NULL;
-        xlator_t              *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                if (!glusterd_is_bitrot_enabled (volinfo))
-                        continue;
-                else if (volinfo->status != GLUSTERD_STATUS_STARTED)
-                        continue;
-                else {
-                        cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                                 brick_list) {
-                                if (!glusterd_is_local_brick (this, volinfo,
-                                                              brickinfo))
-                                        continue;
-                                stopped = _gf_false;
-                                return stopped;
-                        }
-
-                        /* Before stoping bitrot/scrubber daemon check
-                         * other volume also whether respective volume
-                         * host a brick from this node or not.*/
-                        continue;
-                }
-        }
-
-        return stopped;
+    glusterd_conf_t *conf = THIS->private;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t stopped = _gf_true;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        if (!glusterd_is_bitrot_enabled(volinfo))
+            continue;
+        else if (volinfo->status != GLUSTERD_STATUS_STARTED)
+            continue;
+        else {
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+                    continue;
+                stopped = _gf_false;
+                return stopped;
+            }
+
+            /* Before stopping bitrot/scrubber daemon check
+             * other volume also whether respective volume
+             * host a brick from this node or not.*/
+            continue;
+        }
+    }
+
+    return stopped;
 }
 
 static int
-glusterd_manage_bitrot (int opcode)
+glusterd_manage_bitrot(int opcode)
 {
-        int              ret   = -1;
-        xlator_t         *this = NULL;
-        glusterd_conf_t  *priv = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        switch (opcode) {
+    switch (opcode) {
         case GF_BITROT_OPTION_TYPE_ENABLE:
         case GF_BITROT_OPTION_TYPE_DISABLE:
-                ret = priv->bitd_svc.manager (&(priv->bitd_svc),
-                                                NULL, PROC_START_NO_WAIT);
-                if (ret)
-                        break;
-                ret = priv->scrub_svc.manager (&(priv->scrub_svc), NULL,
-                                               PROC_START_NO_WAIT);
+            ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL,
+                                         PROC_START_NO_WAIT);
+            if (ret)
                 break;
+            ret = priv->scrub_svc.manager(&(priv->scrub_svc), NULL,
+                                          PROC_START_NO_WAIT);
+            break;
         default:
-                ret = 0;
-                break;
-        }
-
-        return ret;
+            ret = 0;
+            break;
+    }
 
+    return ret;
 }
 
 int
-glusterd_op_bitrot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        glusterd_volinfo_t     *volinfo      = NULL;
-        int32_t                 ret          = -1;
-        char                   *volname      = NULL;
-        int                     type         = -1;
-        glusterd_conf_t        *priv         = NULL;
-        xlator_t               *this         = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get type from "
-                        "dict");
-                goto out;
-        }
-
-        switch (type) {
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t ret = -1;
+    char *volname = NULL;
+    int type = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type from "
+               "dict");
+        goto out;
+    }
+
+    switch (type) {
         case GF_BITROT_OPTION_TYPE_ENABLE:
-                ret = glusterd_bitrot_enable (volinfo, op_errstr);
-                if (ret < 0)
-                        goto out;
-                break;
+            ret = glusterd_bitrot_enable(volinfo, op_errstr);
+            if (ret < 0)
+                goto out;
+            break;
 
         case GF_BITROT_OPTION_TYPE_DISABLE:
-                ret = glusterd_bitrot_disable (volinfo, op_errstr);
-                if (ret < 0)
-                        goto out;
+            ret = glusterd_bitrot_disable(volinfo, op_errstr);
+            if (ret < 0)
+                goto out;
 
-                break;
+            break;
 
         case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
-                ret = glusterd_bitrot_scrub_throttle (volinfo, dict,
-                                                      "features.scrub-throttle",
-                                                      op_errstr);
-                if (ret)
-                        goto out;
-                break;
+            ret = glusterd_bitrot_scrub_throttle(
+                volinfo, dict, "features.scrub-throttle", op_errstr);
+            if (ret)
+                goto out;
+            break;
 
         case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
-                ret = glusterd_bitrot_scrub_freq (volinfo, dict,
-                                                  "features.scrub-freq",
-                                                  op_errstr);
-                if (ret)
-                        goto out;
-                break;
+            ret = glusterd_bitrot_scrub_freq(volinfo, dict,
+                                             "features.scrub-freq", op_errstr);
+            if (ret)
+                goto out;
+            break;
 
         case GF_BITROT_OPTION_TYPE_SCRUB:
-                ret = glusterd_bitrot_scrub (volinfo, dict, "features.scrub",
-                                             op_errstr);
-                if (ret)
-                        goto out;
-                break;
+            ret = glusterd_bitrot_scrub(volinfo, dict, "features.scrub",
+                                        op_errstr);
+            if (ret)
+                goto out;
+            break;
 
         case GF_BITROT_OPTION_TYPE_EXPIRY_TIME:
-                ret = glusterd_bitrot_expiry_time (volinfo, dict,
-                                                   "features.expiry-time",
-                                                   op_errstr);
-                if (ret)
-                        goto out;
-        case GF_BITROT_CMD_SCRUB_STATUS:
-                break;
-
-        default:
-                gf_asprintf (op_errstr, "Bitrot command failed. Invalid "
-                             "opcode");
-                ret = -1;
+            ret = glusterd_bitrot_expiry_time(
+                volinfo, dict, "features.expiry-time", op_errstr);
+            if (ret)
                 goto out;
-        }
+            break;
 
-        ret = glusterd_manage_bitrot (type);
-        if (ret)
+        case GF_BITROT_OPTION_TYPE_SIGNER_THREADS:
+            ret = glusterd_bitrot_signer_threads(
+                volinfo, dict, "features.signer-threads", op_errstr);
+            if (ret)
                 goto out;
+            break;
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "Unable to re-create "
-                        "volfiles");
-                ret = -1;
-                goto out;
-        }
+        case GF_BITROT_CMD_SCRUB_STATUS:
+        case GF_BITROT_CMD_SCRUB_ONDEMAND:
+            break;
 
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to store volinfo for "
-                        "bitrot");
-                goto out;
-        }
+        default:
+            gf_asprintf(op_errstr,
+                        "Bitrot command failed. Invalid "
+                        "opcode");
+            ret = -1;
+            goto out;
+    }
+
+    ret = glusterd_manage_bitrot(type);
+    if (ret)
+        goto out;
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to re-create "
+               "volfiles");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to store volinfo for "
+                     "bitrot");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_bitrot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                  ret                      = 0;
-        char                *volname                  = NULL;
-        char                *scrub_cmd                = NULL;
-        char                *scrub_cmd_from_dict      = NULL;
-        char                 msg[2048]                = {0,};
-        int                  type                     = 0;
-        xlator_t            *this                     = NULL;
-        glusterd_conf_t     *priv                     = NULL;
-        glusterd_volinfo_t  *volinfo                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-
-        if (!glusterd_is_volume_started (volinfo)) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "before executing bit rot command.");
+    int ret = 0;
+    char *volname = NULL;
+    char *scrub_cmd = NULL;
+    char *scrub_cmd_from_dict = NULL;
+    char msg[2048] = {
+        0,
+    };
+    int type = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(volinfo)) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "before executing bit rot command.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type for "
+               "operation");
+
+        *op_errstr = gf_strdup(
+            "Staging stage failed for bitrot "
+            "operation.");
+        goto out;
+    }
+
+    if ((GF_BITROT_OPTION_TYPE_ENABLE != type) &&
+        (glusterd_is_bitrot_enabled(volinfo) == 0)) {
+        ret = -1;
+        gf_asprintf(op_errstr, "Bitrot is not enabled on volume %s", volname);
+        goto out;
+    }
+
+    if ((GF_BITROT_OPTION_TYPE_SCRUB == type)) {
+        ret = dict_get_str(volinfo->dict, "features.scrub",
+                           &scrub_cmd_from_dict);
+        if (!ret) {
+            ret = dict_get_str(dict, "scrub-value", &scrub_cmd);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                       "Unable to "
+                       "get scrub-value");
+                *op_errstr = gf_strdup(
+                    "Staging failed for "
+                    "bitrot operation. "
+                    "Please check log file"
+                    " for more details.");
+                goto out;
+            }
+            /* If scrubber is resume then value of scrubber will be
+             * "Active" in the dictionary. */
+            if (!strcmp(scrub_cmd_from_dict, scrub_cmd) ||
+                (!strncmp("Active", scrub_cmd_from_dict, SLEN("Active")) &&
+                 !strncmp("resume", scrub_cmd, SLEN("resume")))) {
+                snprintf(msg, sizeof(msg),
+                         "Scrub is already"
+                         " %sd for volume %s",
+                         scrub_cmd, volinfo->volname);
+                *op_errstr = gf_strdup(msg);
                 ret = -1;
                 goto out;
+            }
         }
+        ret = 0;
+    }
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get type for "
-                        "operation");
-
-                *op_errstr = gf_strdup ("Staging stage failed for bitrot "
-                                        "operation.");
-                goto out;
-        }
-
-
-        if ((GF_BITROT_OPTION_TYPE_ENABLE != type) &&
-            (glusterd_is_bitrot_enabled (volinfo) == 0)) {
-                ret = -1;
-                gf_asprintf (op_errstr, "Bitrot is not enabled on volume %s",
-                             volname);
-                goto out;
-        }
-
-        if ((GF_BITROT_OPTION_TYPE_SCRUB == type)) {
-                ret = dict_get_str (volinfo->dict, "features.scrub",
-                                    &scrub_cmd_from_dict);
-                if (!ret) {
-                        ret = dict_get_str (dict, "scrub-value", &scrub_cmd);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_DICT_GET_FAILED, "Unable to "
-                                        "get scrub-value");
-                                *op_errstr = gf_strdup ("Staging failed for "
-                                                        "bitrot operation. "
-                                                        "Please check log file"
-                                                        " for more details.");
-                                goto out;
-                        }
-                        /* If scrubber is resume then value of scrubber will be
-                         * "Active" in the dictionary. */
-                        if (!strcmp (scrub_cmd_from_dict, scrub_cmd) ||
-                            (!strncmp ("Active", scrub_cmd_from_dict,
-                                       strlen("Active")) && !strncmp ("resume",
-                                       scrub_cmd, strlen("resume")))) {
-                                snprintf (msg, sizeof (msg), "Scrub is already"
-                                          " %sd for volume %s", scrub_cmd,
-                                          volinfo->volname);
-                                *op_errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                }
-                ret = 0;
-        }
-
- out:
-        if (ret && op_errstr && *op_errstr)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_STAGE_BITROT_FAIL, "%s", *op_errstr);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+out:
+    if (ret && op_errstr && *op_errstr)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_BITROT_FAIL, "%s",
+               *op_errstr);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index a90114ab2b3..e56cd0e6c74 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -7,7 +7,7 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -20,2297 +20,2075 @@
 #include "glusterd-svc-helper.h"
 #include "glusterd-messages.h"
 #include "glusterd-server-quorum.h"
-#include "run.h"
-#include "glusterd-volgen.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 #include <sys/signal.h>
 
 /* misc */
 
-gf_boolean_t
-glusterd_is_tiering_supported (char *op_errstr)
-{
-        xlator_t           *this        = NULL;
-        glusterd_conf_t    *conf        = NULL;
-        gf_boolean_t        supported   = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        if (conf->op_version < GD_OP_VERSION_3_7_0)
-                goto out;
-
-        supported = _gf_true;
-
-out:
-        if (!supported && op_errstr != NULL && conf)
-                sprintf (op_errstr, "Tier operation failed. The cluster is "
-                         "operating at version %d. Tiering"
-                         " is unavailable in this version.",
-                         conf->op_version);
-
-        return supported;
-}
-
 /* In this function, we decide, based on the 'count' of the brick,
    where to add it in the current volume. 'count' tells us already
    how many of the given bricks are added. other argument are self-
    descriptive. */
 int
-add_brick_at_right_order (glusterd_brickinfo_t *brickinfo,
-                          glusterd_volinfo_t *volinfo, int count,
-                          int32_t stripe_cnt, int32_t replica_cnt)
+add_brick_at_right_order(glusterd_brickinfo_t *brickinfo,
+                         glusterd_volinfo_t *volinfo, int count,
+                         int32_t stripe_cnt, int32_t replica_cnt)
 {
-        int                   idx     = 0;
-        int                   i       = 0;
-        int                   sub_cnt = 0;
-        glusterd_brickinfo_t *brick   = NULL;
-
-        /* The complexity of the function is in deciding at which index
-           to add new brick. Even though it can be defined with a complex
-           single formula for all volume, it is separated out to make it
-           more readable */
-        if (stripe_cnt) {
-                /* common formula when 'stripe_count' is set */
-                /* idx = ((count / ((stripe_cnt * volinfo->replica_count) -
-                   volinfo->dist_leaf_count)) * volinfo->dist_leaf_count) +
-                   (count + volinfo->dist_leaf_count);
-                */
-
-                sub_cnt = volinfo->dist_leaf_count;
-
-                idx = ((count / ((stripe_cnt * volinfo->replica_count) -
-                                 sub_cnt)) * sub_cnt) +
-                        (count + sub_cnt);
-
-                goto insert_brick;
-        }
-
-        /* replica count is set */
-        /* common formula when 'replica_count' is set */
-        /* idx = ((count / (replica_cnt - existing_replica_count)) *
-           existing_replica_count) +
-           (count + existing_replica_count);
+    int idx = 0;
+    int i = 0;
+    int sub_cnt = 0;
+    glusterd_brickinfo_t *brick = NULL;
+
+    /* The complexity of the function is in deciding at which index
+       to add new brick. Even though it can be defined with a complex
+       single formula for all volume, it is separated out to make it
+       more readable */
+    if (stripe_cnt) {
+        /* common formula when 'stripe_count' is set */
+        /* idx = ((count / ((stripe_cnt * volinfo->replica_count) -
+           volinfo->dist_leaf_count)) * volinfo->dist_leaf_count) +
+           (count + volinfo->dist_leaf_count);
         */
 
-        sub_cnt = volinfo->replica_count;
-        idx = (count / (replica_cnt - sub_cnt) * sub_cnt) +
-                (count + sub_cnt);
+        sub_cnt = volinfo->dist_leaf_count;
 
-insert_brick:
-        i = 0;
-        cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
-                i++;
-                if (i < idx)
-                        continue;
-                gf_msg_debug (THIS->name, 0, "brick:%s index=%d, count=%d",
-                        brick->path, idx, count);
-
-                cds_list_add (&brickinfo->brick_list, &brick->brick_list);
-                break;
-        }
+        idx = ((count / ((stripe_cnt * volinfo->replica_count) - sub_cnt)) *
+               sub_cnt) +
+              (count + sub_cnt);
 
-        return 0;
-}
+        goto insert_brick;
+    }
 
+    /* replica count is set */
+    /* common formula when 'replica_count' is set */
+    /* idx = ((count / (replica_cnt - existing_replica_count)) *
+       existing_replica_count) +
+       (count + existing_replica_count);
+    */
 
-static int
-gd_addbr_validate_stripe_count (glusterd_volinfo_t *volinfo, int stripe_count,
-                                int total_bricks, int *type, char *err_str,
-                                size_t err_len)
-{
-        int ret = -1;
+    sub_cnt = volinfo->replica_count;
+    idx = (count / (replica_cnt - sub_cnt) * sub_cnt) + (count + sub_cnt);
 
-        switch (volinfo->type) {
-        case GF_CLUSTER_TYPE_NONE:
-                if ((volinfo->brick_count * stripe_count) == total_bricks) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_STRIPE;
-                        gf_msg (THIS->name, GF_LOG_INFO, 0,
-                                GD_MSG_VOL_TYPE_CHANGING_INFO,
-                                "Changing the type of volume %s from "
-                                "'distribute' to 'stripe'", volinfo->volname);
-                        ret = 0;
-                        goto out;
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for stripe count (%d).",
-                                  (total_bricks - volinfo->brick_count),
-                                  stripe_count);
-                        gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        goto out;
-                }
-                break;
-        case GF_CLUSTER_TYPE_REPLICATE:
-                if (!(total_bricks % (volinfo->replica_count * stripe_count))) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                        gf_msg (THIS->name, GF_LOG_INFO, 0,
-                                GD_MSG_VOL_TYPE_CHANGING_INFO,
-                                "Changing the type of volume %s from "
-                                "'replicate' to 'replicate-stripe'",
-                                volinfo->volname);
-                        ret = 0;
-                        goto out;
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for changing volume's "
-                                  "stripe count to %d, need at least %d bricks",
-                                  (total_bricks - volinfo->brick_count),
-                                  stripe_count,
-                                  (volinfo->replica_count * stripe_count));
-                        gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        goto out;
-                }
-                break;
-        case GF_CLUSTER_TYPE_STRIPE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                if (stripe_count < volinfo->stripe_count) {
-                        snprintf (err_str, err_len,
-                                  "Incorrect stripe count (%d) supplied. "
-                                  "Volume already has stripe count (%d)",
-                                  stripe_count, volinfo->stripe_count);
-                        gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        goto out;
-                }
-                if (stripe_count == volinfo->stripe_count) {
-                        if (!(total_bricks % volinfo->dist_leaf_count)) {
-                                /* its same as the one which exists */
-                                ret = 1;
-                                goto out;
-                        }
-                }
-                if (stripe_count > volinfo->stripe_count) {
-                        /* We have to make sure before and after 'add-brick',
-                           the number or subvolumes for distribute will remain
-                           same, when stripe count is given */
-                        if ((volinfo->brick_count * (stripe_count *
-                                                     volinfo->replica_count)) ==
-                            (total_bricks * volinfo->dist_leaf_count)) {
-                                /* Change the dist_leaf_count */
-                                gf_msg (THIS->name, GF_LOG_INFO, 0,
-                                        GD_MSG_STRIPE_COUNT_CHANGE_INFO,
-                                        "Changing the stripe count of "
-                                        "volume %s from %d to %d",
-                                        volinfo->volname,
-                                        volinfo->stripe_count, stripe_count);
-                                ret = 0;
-                                goto out;
-                        }
-                }
-                break;
-        case GF_CLUSTER_TYPE_DISPERSE:
-                snprintf (err_str, err_len, "Volume %s cannot be converted "
-                                            "from dispersed to striped-"
-                                            "dispersed", volinfo->volname);
-                gf_msg(THIS->name, GF_LOG_ERROR, EPERM,
-                       GD_MSG_OP_NOT_PERMITTED, "%s", err_str);
-                goto out;
-        }
-
-out:
-        return ret;
+insert_brick:
+    i = 0;
+    cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+    {
+        i++;
+        if (i < idx)
+            continue;
+        gf_msg_debug(THIS->name, 0, "brick:%s index=%d, count=%d", brick->path,
+                     idx, count);
+
+        cds_list_add(&brickinfo->brick_list, &brick->brick_list);
+        break;
+    }
+
+    return 0;
 }
 
 static int
-gd_addbr_validate_replica_count (glusterd_volinfo_t *volinfo, int replica_count,
-                                 int arbiter_count, int total_bricks, int *type,
-                                 char *err_str, int err_len)
+gd_addbr_validate_replica_count(glusterd_volinfo_t *volinfo, int replica_count,
+                                int arbiter_count, int total_bricks, int *type,
+                                char *err_str, int err_len)
 {
-        int ret = -1;
+    int ret = -1;
 
-        /* replica count is set */
-        switch (volinfo->type) {
+    /* replica count is set */
+    switch (volinfo->type) {
         case GF_CLUSTER_TYPE_NONE:
-                if ((volinfo->brick_count * replica_count) == total_bricks) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_REPLICATE;
-                        gf_msg (THIS->name, GF_LOG_INFO, 0,
-                                GD_MSG_VOL_TYPE_CHANGING_INFO,
-                                "Changing the type of volume %s from "
-                                "'distribute' to 'replica'", volinfo->volname);
-                        ret = 0;
-                        goto out;
-
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for replica count (%d).",
-                                  (total_bricks - volinfo->brick_count),
-                                  replica_count);
-                        gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        goto out;
-                }
-                break;
-        case GF_CLUSTER_TYPE_STRIPE:
-                if (!(total_bricks % (volinfo->dist_leaf_count * replica_count))) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                        gf_msg (THIS->name, GF_LOG_INFO, 0,
-                                GD_MSG_VOL_TYPE_CHANGING_INFO,
-                                "Changing the type of volume %s from "
-                                "'stripe' to 'replicate-stripe'",
-                                volinfo->volname);
-                        ret = 0;
-                        goto out;
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for changing volume's "
-                                  "replica count to %d, need at least %d "
-                                  "bricks",
-                                  (total_bricks - volinfo->brick_count),
-                                  replica_count, (volinfo->dist_leaf_count *
-                                                  replica_count));
-                        gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        goto out;
-                }
-                break;
+            if ((volinfo->brick_count * replica_count) == total_bricks) {
+                /* Change the volume type */
+                *type = GF_CLUSTER_TYPE_REPLICATE;
+                gf_msg(THIS->name, GF_LOG_INFO, 0,
+                       GD_MSG_VOL_TYPE_CHANGING_INFO,
+                       "Changing the type of volume %s from "
+                       "'distribute' to 'replica'",
+                       volinfo->volname);
+                ret = 0;
+                goto out;
+
+            } else {
+                snprintf(err_str, err_len,
+                         "Incorrect number of "
+                         "bricks (%d) supplied for replica count (%d).",
+                         (total_bricks - volinfo->brick_count), replica_count);
+                gf_msg(THIS->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "%s", err_str);
+                goto out;
+            }
+            break;
         case GF_CLUSTER_TYPE_REPLICATE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                if (replica_count < volinfo->replica_count) {
-                        snprintf (err_str, err_len,
-                                  "Incorrect replica count (%d) supplied. "
-                                  "Volume already has (%d)",
-                                  replica_count, volinfo->replica_count);
-                        gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        goto out;
-                }
-                if (replica_count == volinfo->replica_count) {
-                        if (arbiter_count && !volinfo->arbiter_count) {
-                                snprintf (err_str, err_len,
-                                          "Cannot convert replica 3 volume "
-                                          "to arbiter volume.");
-                                gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_INVALID_ENTRY, "%s", err_str);
-                                goto out;
-                        }
-                        if (!(total_bricks % volinfo->dist_leaf_count)) {
-                                ret = 1;
-                                goto out;
-                        }
-                }
-                if (replica_count > volinfo->replica_count) {
-                        /* We have to make sure before and after 'add-brick',
-                           the number or subvolumes for distribute will remain
-                           same, when replica count is given */
-                        if ((total_bricks * volinfo->dist_leaf_count) ==
-                            (volinfo->brick_count * (replica_count *
-                                                     volinfo->stripe_count))) {
-                                /* Change the dist_leaf_count */
-                                gf_msg (THIS->name, GF_LOG_INFO, 0,
-                                        GD_MSG_REPLICA_COUNT_CHANGE_INFO,
-                                        "Changing the replica count of "
-                                        "volume %s from %d to %d",
-                                        volinfo->volname, volinfo->replica_count,
-                                        replica_count);
-                                ret = 0;
-                                goto out;
-                        }
-                }
-                break;
+            if (replica_count < volinfo->replica_count) {
+                snprintf(err_str, err_len,
+                         "Incorrect replica count (%d) supplied. "
+                         "Volume already has (%d)",
+                         replica_count, volinfo->replica_count);
+                gf_msg(THIS->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "%s", err_str);
+                goto out;
+            }
+            if (replica_count == volinfo->replica_count) {
+                if (arbiter_count && !volinfo->arbiter_count) {
+                    snprintf(err_str, err_len,
+                             "Cannot convert replica 3 volume "
+                             "to arbiter volume.");
+                    gf_msg(THIS->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INVALID_ENTRY, "%s", err_str);
+                    goto out;
+                }
+                if (!(total_bricks % volinfo->dist_leaf_count)) {
+                    ret = 1;
+                    goto out;
+                }
+            }
+            if (replica_count > volinfo->replica_count) {
+                /* We have to make sure before and after 'add-brick',
+                   the number or subvolumes for distribute will remain
+                   same, when replica count is given */
+                if ((total_bricks * volinfo->dist_leaf_count) ==
+                    (volinfo->brick_count *
+                     (replica_count * volinfo->stripe_count))) {
+                    /* Change the dist_leaf_count */
+                    gf_msg(THIS->name, GF_LOG_INFO, 0,
+                           GD_MSG_REPLICA_COUNT_CHANGE_INFO,
+                           "Changing the replica count of "
+                           "volume %s from %d to %d",
+                           volinfo->volname, volinfo->replica_count,
+                           replica_count);
+                    ret = 0;
+                    goto out;
+                }
+            }
+            break;
         case GF_CLUSTER_TYPE_DISPERSE:
-                snprintf (err_str, err_len, "Volume %s cannot be converted "
-                                            "from dispersed to replicated-"
-                                            "dispersed", volinfo->volname);
-                gf_msg(THIS->name, GF_LOG_ERROR, EPERM,
-                       GD_MSG_OP_NOT_PERMITTED, "%s", err_str);
-                goto out;
-        }
+            snprintf(err_str, err_len,
+                     "Volume %s cannot be converted "
+                     "from dispersed to replicated-"
+                     "dispersed",
+                     volinfo->volname);
+            gf_msg(THIS->name, GF_LOG_ERROR, EPERM, GD_MSG_OP_NOT_PERMITTED,
+                   "%s", err_str);
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo,
-                                int32_t replica_count,
-                                int32_t brick_count, char *err_str,
-                                size_t err_len)
+gd_rmbr_validate_replica_count(glusterd_volinfo_t *volinfo,
+                               int32_t replica_count, int32_t brick_count,
+                               char *err_str, size_t err_len)
 {
-        int ret = -1;
-        int replica_nodes = 0;
-
-        switch (volinfo->type) {
-        case GF_CLUSTER_TYPE_TIER:
-                ret = 1;
-                goto out;
+    int ret = -1;
+    int replica_nodes = 0;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
 
+    switch (volinfo->type) {
         case GF_CLUSTER_TYPE_NONE:
-        case GF_CLUSTER_TYPE_STRIPE:
         case GF_CLUSTER_TYPE_DISPERSE:
-                snprintf (err_str, err_len,
-                          "replica count (%d) option given for non replicate "
-                          "volume %s", replica_count, volinfo->volname);
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOL_NOT_REPLICA, "%s", err_str);
-                goto out;
+            snprintf(err_str, err_len,
+                     "replica count (%d) option given for non replicate "
+                     "volume %s",
+                     replica_count, volinfo->volname);
+            gf_smsg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ARGUMENT,
+                    err_str, NULL);
+            goto out;
 
         case GF_CLUSTER_TYPE_REPLICATE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                /* in remove brick, you can only reduce the replica count */
-                if (replica_count > volinfo->replica_count) {
-                        snprintf (err_str, err_len,
-                                  "given replica count (%d) option is more "
-                                  "than volume %s's replica count (%d)",
-                                  replica_count, volinfo->volname,
-                                  volinfo->replica_count);
-                        gf_msg (THIS->name, GF_LOG_WARNING, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        goto out;
-                }
-                if (replica_count == volinfo->replica_count) {
-                        /* This means the 'replica N' option on CLI was
-                           redundant. Check if the total number of bricks given
-                           for removal is same as 'dist_leaf_count' */
-                        if (brick_count % volinfo->dist_leaf_count) {
-                                snprintf (err_str, err_len,
-                                          "number of bricks provided (%d) is "
-                                          "not valid. need at least %d "
-                                          "(or %dxN)", brick_count,
-                                          volinfo->dist_leaf_count,
-                                          volinfo->dist_leaf_count);
-                                gf_msg (THIS->name, GF_LOG_WARNING, EINVAL,
-                                        GD_MSG_INVALID_ENTRY, "%s",
-                                        err_str);
-                                goto out;
-                        }
-                        ret = 1;
-                        goto out;
+            /* in remove brick, you can only reduce the replica count */
+            if (replica_count > volinfo->replica_count) {
+                snprintf(err_str, err_len,
+                         "given replica count (%d) option is more "
+                         "than volume %s's replica count (%d)",
+                         replica_count, volinfo->volname,
+                         volinfo->replica_count);
+                gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                        GD_MSG_INVALID_ARGUMENT, err_str, NULL);
+                goto out;
+            }
+            if (replica_count == volinfo->replica_count) {
+                /* This means the 'replica N' option on CLI was
+                   redundant. Check if the total number of bricks given
+                   for removal is same as 'dist_leaf_count' */
+                if (brick_count % volinfo->dist_leaf_count) {
+                    snprintf(err_str, err_len,
+                             "number of bricks provided (%d) is "
+                             "not valid. need at least %d "
+                             "(or %dxN)",
+                             brick_count, volinfo->dist_leaf_count,
+                             volinfo->dist_leaf_count);
+                    gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                            GD_MSG_INVALID_ARGUMENT, err_str, NULL);
+                    goto out;
                 }
+                ret = 1;
+                goto out;
+            }
 
-                replica_nodes = ((volinfo->brick_count /
-                                  volinfo->replica_count) *
-                                 (volinfo->replica_count - replica_count));
+            replica_nodes = ((volinfo->brick_count / volinfo->replica_count) *
+                             (volinfo->replica_count - replica_count));
 
-                if (brick_count % replica_nodes) {
-                        snprintf (err_str, err_len,
-                                  "need %d(xN) bricks for reducing replica "
-                                  "count of the volume from %d to %d",
-                                  replica_nodes, volinfo->replica_count,
-                                  replica_count);
-                        goto out;
-                }
-                break;
-        }
+            if (brick_count % replica_nodes) {
+                snprintf(err_str, err_len,
+                         "need %d(xN) bricks for reducing replica "
+                         "count of the volume from %d to %d",
+                         replica_nodes, volinfo->replica_count, replica_count);
+                gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                        GD_MSG_INVALID_ARGUMENT, err_str, NULL);
+                goto out;
+            }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Handler functions */
 int
-__glusterd_handle_add_brick (rpcsvc_request_t *req)
+__glusterd_handle_add_brick(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        char                            *bricks = NULL;
-        char                            *volname = NULL;
-        int                             brick_count = 0;
-        void                            *cli_rsp = NULL;
-        char                            err_str[2048] = {0,};
-        gf_cli_rsp                      rsp = {0,};
-        glusterd_volinfo_t              *volinfo = NULL;
-        xlator_t                        *this = NULL;
-        int                             total_bricks = 0;
-        int32_t                         replica_count = 0;
-        int32_t                         arbiter_count = 0;
-        int32_t                         stripe_count = 0;
-        int                             type = 0;
-        glusterd_conf_t                 *conf = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        GF_ASSERT (req);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char *bricks = NULL;
+    char *volname = NULL;
+    int brick_count = 0;
+    void *cli_rsp = NULL;
+    char err_str[2048] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int total_bricks = 0;
+    int32_t replica_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t stripe_count = 0;
+    int type = 0;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_ADD_BRICK_REQ_RECVD,
+           "Received add brick req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                snprintf (err_str, sizeof (err_str), "Garbage args received");
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_ADD_BRICK_REQ_RECVD, "Received add brick req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
-
-        if (!(ret = glusterd_check_volume_exists (volname))) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Volume %s does not exist",
-                          volname);
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "brick count");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, errno,
-                        GD_MSG_DICT_GET_SUCCESS, "replica-count is %d",
-                        replica_count);
-        }
-
-        ret = dict_get_int32 (dict, "arbiter-count", &arbiter_count);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, errno,
-                        GD_MSG_DICT_GET_SUCCESS, "arbiter-count is %d",
-                        arbiter_count);
-        }
-
-        ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, errno,
-                        GD_MSG_DICT_GET_SUCCESS, "stripe-count is %d",
-                        stripe_count);
-        }
-
-        if (!dict_get (dict, "force")) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get flag");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volinfo "
-                          "for volume name %s", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_GET_FAIL, "%s", err_str);
-                goto out;
-
-        }
-
-        total_bricks = volinfo->brick_count + brick_count;
-
-        if (dict_get (dict, "attach-tier")) {
-                if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume %s is already a tier.", volname);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_ALREADY_TIER, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (glusterd_is_tiering_supported(err_str) == _gf_false) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VERSION_UNSUPPORTED,
-                                "Tiering not supported at this version");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, "hot-type", &type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_GET_FAILED,
-                                "failed to get type from dictionary");
-                        goto out;
-                }
-
-                goto brick_val;
-        }
-
-        ret = glusterd_disallow_op_for_tier (volinfo, GD_OP_ADD_BRICK, -1);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Add-brick operation is "
-                          "not supported on a tiered volume %s", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_UNSUPPORTED, "%s", err_str);
-                goto out;
-        }
-
-        if (!stripe_count && !replica_count) {
-                if (volinfo->type == GF_CLUSTER_TYPE_NONE)
-                        goto brick_val;
-
-                if ((volinfo->brick_count < volinfo->dist_leaf_count) &&
-                    (total_bricks <= volinfo->dist_leaf_count))
-                        goto brick_val;
-
-                if ((brick_count % volinfo->dist_leaf_count) != 0) {
-                        snprintf (err_str, sizeof (err_str), "Incorrect number "
-                                  "of bricks supplied %d with count %d",
-                                 brick_count, volinfo->dist_leaf_count);
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_REPLICA, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-                goto brick_val;
-                /* done with validation.. below section is if stripe|replica
-                   count is given */
-        }
-
-        /* These bricks needs to be added one per a replica or stripe volume */
-        if (stripe_count) {
-                ret = gd_addbr_validate_stripe_count (volinfo, stripe_count,
-                                                      total_bricks, &type,
-                                                      err_str,
-                                                      sizeof (err_str));
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_COUNT_VALIDATE_FAILED, "%s", err_str);
-                        goto out;
-                }
-
-                /* if stripe count is same as earlier, set it back to 0 */
-                if (ret == 1)
-                        stripe_count = 0;
-
-                ret = dict_set_int32 (dict, "stripe-count", stripe_count);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_GET_FAILED,
-                                "failed to set the stripe-count in dict");
-                        goto out;
-                }
-                goto brick_val;
-        }
-
-        ret = gd_addbr_validate_replica_count (volinfo, replica_count,
-                                               arbiter_count, total_bricks,
-                                               &type, err_str,
-                                               sizeof (err_str));
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COUNT_VALIDATE_FAILED, "%s", err_str);
-                goto out;
-        }
-
-        /* if replica count is same as earlier, set it back to 0 */
-        if (ret == 1)
-                replica_count = 0;
-
-        ret = dict_set_int32 (dict, "replica-count", replica_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED,
-                        "failed to set the replica-count in dict");
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volinfo "
+                 "for volume name %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &brick_count);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "brick count");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+               "replica-count is %d", replica_count);
+    }
+
+    ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"),
+                          &arbiter_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+               "arbiter-count is %d", arbiter_count);
+    }
+
+    ret = dict_get_int32n(dict, "stripe-count", SLEN("stripe-count"),
+                          &stripe_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+               "stripe-count is %d", stripe_count);
+    }
+
+    if (!dict_getn(dict, "force", SLEN("force"))) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Failed to get flag");
+        goto out;
+    }
+
+    total_bricks = volinfo->brick_count + brick_count;
+
+    if (!stripe_count && !replica_count) {
+        if (volinfo->type == GF_CLUSTER_TYPE_NONE)
+            goto brick_val;
+
+        if ((volinfo->brick_count < volinfo->dist_leaf_count) &&
+            (total_bricks <= volinfo->dist_leaf_count))
+            goto brick_val;
+
+        if ((brick_count % volinfo->dist_leaf_count) != 0) {
+            snprintf(err_str, sizeof(err_str),
+                     "Incorrect number "
+                     "of bricks supplied %d with count %d",
+                     brick_count, volinfo->dist_leaf_count);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_REPLICA,
+                   "%s", err_str);
+            ret = -1;
+            goto out;
+        }
+        goto brick_val;
+        /* done with validation.. below section is if stripe|replica
+           count is given */
+    }
+
+    ret = gd_addbr_validate_replica_count(volinfo, replica_count, arbiter_count,
+                                          total_bricks, &type, err_str,
+                                          sizeof(err_str));
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COUNT_VALIDATE_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    /* if replica count is same as earlier, set it back to 0 */
+    if (ret == 1)
+        replica_count = 0;
+
+    ret = dict_set_int32n(dict, "replica-count", SLEN("replica-count"),
+                          replica_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "failed to set the replica-count in dict");
+        goto out;
+    }
 
 brick_val:
-        ret = dict_get_str (dict, "bricks", &bricks);
+    ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "bricks");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (type != volinfo->type) {
+        ret = dict_set_int32n(dict, "type", SLEN("type"), type);
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "bricks");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
-
-        if (type != volinfo->type) {
-                ret = dict_set_int32 (dict, "type", type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_SET_FAILED,
-                                "failed to set the new type in dict");
-                        goto out;
-                }
-        }
-
-        if (conf->op_version <= GD_OP_VERSION_3_7_5) {
-                gf_msg_debug (this->name, 0, "The cluster is operating at "
-                          "version less than or equal to %d. Falling back "
-                          "to syncop framework.",
-                          GD_OP_VERSION_3_7_5);
-                ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
-        } else {
-                ret = glusterd_mgmt_v3_initiate_all_phases (req,
-                                                            GD_OP_ADD_BRICK,
-                                                            dict);
-        }
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "failed to set the new type in dict");
+            goto out;
+        }
+    }
+
+    if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than or equal to %d. Falling back "
+                     "to syncop framework.",
+                     GD_OP_VERSION_3_7_5);
+        ret = glusterd_op_begin_synctask(req, GD_OP_ADD_BRICK, dict);
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_ADD_BRICK, dict);
+    }
 
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                rsp.op_errno = 0;
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str), "Operation failed");
-                rsp.op_errstr = err_str;
-                cli_rsp = &rsp;
-                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
-                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
-                ret = 0; //sent error to cli, prevent second reply
-        }
-
-        free (cli_req.dict.dict_val); //its malloced by xdr
-
-        return ret;
+    if (ret) {
+        rsp.op_ret = -1;
+        rsp.op_errno = 0;
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        rsp.op_errstr = err_str;
+        cli_rsp = &rsp;
+        glusterd_to_cli(req, cli_rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp,
+                        dict);
+        ret = 0;  // sent error to cli, prevent second reply
+    }
+
+    free(cli_req.dict.dict_val);  // its malloced by xdr
+
+    return ret;
 }
 
 int
-glusterd_handle_add_brick (rpcsvc_request_t *req)
+glusterd_handle_add_brick(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+    return glusterd_big_locked_handler(req, __glusterd_handle_add_brick);
 }
 
 static int
-subvol_matcher_init (int **subvols, int count)
+subvol_matcher_init(int **subvols, int count)
 {
-        int ret = -1;
+    int ret = -1;
 
-        *subvols = GF_CALLOC (count, sizeof(int), gf_gld_mt_int);
-        if (*subvols)
-                ret = 0;
+    *subvols = GF_CALLOC(count, sizeof(int), gf_gld_mt_int);
+    if (*subvols)
+        ret = 0;
 
-        return ret;
+    return ret;
 }
 
 static void
-subvol_matcher_update (int *subvols, glusterd_volinfo_t *volinfo,
-                       glusterd_brickinfo_t *brickinfo)
+subvol_matcher_update(int *subvols, glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo)
 {
-        glusterd_brickinfo_t *tmp        = NULL;
-        int32_t               sub_volume = 0;
-        int                   pos        = 0;
-
-        cds_list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-
-                if (strcmp (tmp->hostname, brickinfo->hostname) ||
-                    strcmp (tmp->path, brickinfo->path)) {
-                        pos++;
-                        continue;
-                }
-                gf_msg_debug (THIS->name, 0, LOGSTR_FOUND_BRICK,
-                                        brickinfo->hostname, brickinfo->path,
-                                        volinfo->volname);
-                sub_volume = (pos / volinfo->dist_leaf_count);
-                subvols[sub_volume]++;
-                break;
-        }
-
+    glusterd_brickinfo_t *tmp = NULL;
+    int32_t sub_volume = 0;
+    int pos = 0;
+    if (subvols) {
+        cds_list_for_each_entry(tmp, &volinfo->bricks, brick_list)
+        {
+            if (strcmp(tmp->hostname, brickinfo->hostname) ||
+                strcmp(tmp->path, brickinfo->path)) {
+                pos++;
+                continue;
+            }
+            gf_msg_debug(THIS->name, 0, LOGSTR_FOUND_BRICK, brickinfo->hostname,
+                         brickinfo->path, volinfo->volname);
+            sub_volume = (pos / volinfo->dist_leaf_count);
+            subvols[sub_volume]++;
+            break;
+        }
+    }
 }
 
 static int
-subvol_matcher_verify (int *subvols, glusterd_volinfo_t *volinfo, char *err_str,
-                       size_t err_len, char *vol_type, int replica_count)
+subvol_matcher_verify(int *subvols, glusterd_volinfo_t *volinfo, char *err_str,
+                      size_t err_len, char *vol_type, int replica_count)
 {
-        int i = 0;
-        int ret = 0;
-        int count = volinfo->replica_count-replica_count;
-
-        if (replica_count) {
-                for (i = 0; i < volinfo->subvol_count; i++) {
-                        if (subvols[i] != count) {
-                                ret = -1;
-                                snprintf (err_str, err_len, "Remove exactly %d"
-                                " brick(s) from each subvolume.", count);
-                                break;
-                        }
-                }
-                return ret;
+    int i = 0;
+    int ret = 0;
+    int count = volinfo->replica_count - replica_count;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (replica_count && subvols) {
+        for (i = 0; i < volinfo->subvol_count; i++) {
+            if (subvols[i] != count) {
+                ret = -1;
+                snprintf(err_str, err_len,
+                         "Remove exactly %d"
+                         " brick(s) from each subvolume.",
+                         count);
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, err_str, NULL);
+                break;
+            }
         }
+        return ret;
+    }
 
-       do {
-
-                if (subvols[i] % volinfo->dist_leaf_count == 0) {
-                        continue;
-                } else {
-                        ret = -1;
-                        snprintf (err_str, err_len,
-                                "Bricks not from same subvol for %s", vol_type);
-                        break;
-                }
-        } while (++i < volinfo->subvol_count);
+    do {
+        if (subvols && (subvols[i] % volinfo->dist_leaf_count == 0)) {
+            continue;
+        } else {
+            ret = -1;
+            snprintf(err_str, err_len, "Bricks not from same subvol for %s",
+                     vol_type);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, err_str, NULL);
+            break;
+        }
+    } while (++i < volinfo->subvol_count);
 
-        return ret;
+    return ret;
 }
 
 static void
-subvol_matcher_destroy (int *subvols)
+subvol_matcher_destroy(int *subvols)
 {
-        GF_FREE (subvols);
+    GF_FREE(subvols);
 }
 
 static int
-glusterd_set_detach_bricks(dict_t *dict, glusterd_volinfo_t *volinfo)
+glusterd_remove_brick_validate_arbiters(glusterd_volinfo_t *volinfo,
+                                        int32_t count, int32_t replica_count,
+                                        glusterd_brickinfo_t **brickinfo_list,
+                                        char *err_str, size_t err_len)
 {
-        char key[256] = {0,};
-        char value[256] = {0,};
-        int brick_num = 0;
-        int hot_brick_num = 0;
-        glusterd_brickinfo_t *brickinfo;
-        int ret = 0;
-
-        /* cold tier bricks at tail of list so use reverse iteration */
-        cds_list_for_each_entry_reverse (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                brick_num++;
-                if (brick_num > volinfo->tier_info.cold_brick_count) {
-                        hot_brick_num++;
-                        sprintf (key, "brick%d", hot_brick_num);
-                        snprintf (value, 256, "%s:%s",
-                                  brickinfo->hostname,
-                                  brickinfo->path);
-
-                        ret = dict_set_str (dict, key, strdup(value));
-                        if (ret)
-                                break;
-                }
-        }
-
-        ret = dict_set_int32(dict, "count", hot_brick_num);
-        if (ret)
-                return -1;
-
-        return hot_brick_num;
-}
-
-static int
-glusterd_remove_brick_validate_arbiters (glusterd_volinfo_t *volinfo,
-                                         int32_t count, int32_t replica_count,
-                                         glusterd_brickinfo_t **brickinfo_list,
-                                         char *err_str, size_t err_len)
-{
-        int i = 0;
-        int ret = 0;
-        glusterd_brickinfo_t *brickinfo = NULL;
-        glusterd_brickinfo_t *last = NULL;
-        char *arbiter_array = NULL;
-
-        if ((volinfo->type != GF_CLUSTER_TYPE_REPLICATE) &&
-            (volinfo->type != GF_CLUSTER_TYPE_STRIPE_REPLICATE))
-                goto out;
-
-        if (!replica_count || !volinfo->arbiter_count)
+    int i = 0;
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *last = NULL;
+    char *arbiter_array = NULL;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (volinfo->type != GF_CLUSTER_TYPE_REPLICATE)
+        goto out;
+
+    if (!replica_count || !volinfo->arbiter_count)
+        goto out;
+
+    if (replica_count == 2) {
+        /* If it is an arbiter to replica 2 conversion, only permit
+         *  removal of the arbiter brick.*/
+        for (i = 0; i < count; i++) {
+            brickinfo = brickinfo_list[i];
+            last = get_last_brick_of_brick_group(volinfo, brickinfo);
+            if (last != brickinfo) {
+                snprintf(err_str, err_len,
+                         "Remove arbiter "
+                         "brick(s) only when converting from "
+                         "arbiter to replica 2 subvolume.");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_REMOVE_ARBITER_BRICK, err_str, NULL);
+                ret = -1;
                 goto out;
-
-        if (replica_count == 2) {
-                /* If it is an arbiter to replica 2 conversion, only permit
-                *  removal of the arbiter brick.*/
-                for (i = 0; i < count; i++) {
-                        brickinfo = brickinfo_list[i];
-                        last = get_last_brick_of_brick_group (volinfo,
-                                                              brickinfo);
-                        if (last != brickinfo) {
-                                snprintf (err_str, err_len, "Remove arbiter "
-                                          "brick(s) only when converting from "
-                                           "arbiter to replica 2 subvolume.");
-                                ret = -1;
-                                goto out;
-                        }
-                }
-        } else if (replica_count == 1) {
-                /* If it is an arbiter to plain distribute conversion, in every
-                 * replica subvol, the arbiter has to be one of the bricks that
-                 * are removed. */
-                arbiter_array = GF_CALLOC (volinfo->subvol_count,
-                                           sizeof (*arbiter_array),
-                                           gf_common_mt_char);
-                if (!arbiter_array)
-                        return -1;
-                for (i = 0; i < count; i++) {
-                        brickinfo = brickinfo_list[i];
-                        last = get_last_brick_of_brick_group (volinfo,
-                                                              brickinfo);
-                        if (last == brickinfo)
-                                arbiter_array[brickinfo->group] = 1;
-                }
-                for (i = 0; i < volinfo->subvol_count; i++)
-                        if (!arbiter_array[i]) {
-                                snprintf (err_str, err_len, "Removed bricks "
-                                          "must contain arbiter when converting"
-                                           " to plain distrubute.");
-                                ret = -1;
-                                break;
-                        }
-                GF_FREE (arbiter_array);
-        }
+            }
+        }
+    } else if (replica_count == 1) {
+        /* If it is an arbiter to plain distribute conversion, in every
+         * replica subvol, the arbiter has to be one of the bricks that
+         * are removed. */
+        arbiter_array = GF_CALLOC(volinfo->subvol_count, sizeof(*arbiter_array),
+                                  gf_common_mt_char);
+        if (!arbiter_array)
+            return -1;
+        for (i = 0; i < count; i++) {
+            brickinfo = brickinfo_list[i];
+            last = get_last_brick_of_brick_group(volinfo, brickinfo);
+            if (last == brickinfo)
+                arbiter_array[brickinfo->group] = 1;
+        }
+        for (i = 0; i < volinfo->subvol_count; i++)
+            if (!arbiter_array[i]) {
+                snprintf(err_str, err_len,
+                         "Removed bricks "
+                         "must contain arbiter when converting"
+                         " to plain distribute.");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_REMOVE_ARBITER_BRICK, err_str, NULL);
+                ret = -1;
+                break;
+            }
+        GF_FREE(arbiter_array);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_handle_remove_brick (rpcsvc_request_t *req)
+__glusterd_handle_remove_brick(rpcsvc_request_t *req)
 {
-        int32_t                   ret              = -1;
-        gf_cli_req                cli_req          = {{0,}};
-        dict_t                   *dict             = NULL;
-        int32_t                   count            = 0;
-        char                     *brick            = NULL;
-        char                      key[256]         = {0,};
-        int                       i                = 1;
-        glusterd_volinfo_t       *volinfo          = NULL;
-        glusterd_brickinfo_t     *brickinfo        = NULL;
-        glusterd_brickinfo_t     **brickinfo_list  = NULL;
-        int                      *subvols          = NULL;
-        char                      err_str[2048]    = {0};
-        gf_cli_rsp                rsp              = {0,};
-        void                     *cli_rsp          = NULL;
-        char                      vol_type[256]    = {0,};
-        int32_t                   replica_count    = 0;
-        char                     *volname          = 0;
-        xlator_t                 *this             = NULL;
-        int                       cmd              = -1;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    int32_t count = 0;
+    char *brick = NULL;
+    char key[64] = "";
+    int keylen;
+    int i = 1;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t **brickinfo_list = NULL;
+    int *subvols = NULL;
+    char err_str[2048] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    void *cli_rsp = NULL;
+    char vol_type[256] = "";
+    int32_t replica_count = 0;
+    char *volname = 0;
+    xlator_t *this = NULL;
+    int cmd = -1;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Received garbage args");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REM_BRICK_REQ_RECVD,
+           "Received rem brick req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                snprintf (err_str, sizeof (err_str), "Received garbage args");
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_REM_BRICK_REQ_RECVD,
-                "Received rem brick req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get brick "
+                 "count");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Volume %s does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &cmd);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get cmd "
+                 "ccommand");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+               "request to change replica-count to %d", replica_count);
+        ret = gd_rmbr_validate_replica_count(volinfo, replica_count, count,
+                                             err_str, sizeof(err_str));
+        if (ret < 0) {
+            /* logging and error msg are done in above function
+               itself */
+            goto out;
         }
-
-        ret = dict_get_str (dict, "volname", &volname);
+        dict_deln(dict, "replica-count", SLEN("replica-count"));
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
+            replica_count = 0;
+        } else {
+            ret = dict_set_int32n(dict, "replica-count", SLEN("replica-count"),
+                                  replica_count);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, errno,
+                       GD_MSG_DICT_SET_FAILED,
+                       "failed to set the replica_count "
+                       "in dict");
+                goto out;
+            }
+        }
+    }
+
+    /* 'vol_type' is used for giving the meaning full error msg for user */
+    if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+        strcpy(vol_type, "replica");
+    } else if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        strcpy(vol_type, "disperse");
+    } else {
+        strcpy(vol_type, "distribute");
+    }
+
+    if (!replica_count && (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+        (volinfo->brick_count == volinfo->dist_leaf_count)) {
+        snprintf(err_str, sizeof(err_str),
+                 "Removing bricks from replicate configuration "
+                 "is not allowed without reducing replica count "
+                 "explicitly.");
+        gf_msg(this->name, GF_LOG_ERROR, EPERM, GD_MSG_OP_NOT_PERMITTED_AC_REQD,
+               "%s", err_str);
+        ret = -1;
+        goto out;
+    }
+
+    /* Do not allow remove-brick if the bricks given is less than
+       the replica count or stripe count */
+    if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE)) {
+        if (volinfo->dist_leaf_count && (count % volinfo->dist_leaf_count)) {
+            snprintf(err_str, sizeof(err_str),
+                     "Remove brick "
+                     "incorrect brick count of %d for %s %d",
+                     count, vol_type, volinfo->dist_leaf_count);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+                   err_str);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+        (volinfo->subvol_count > 1)) {
+        ret = subvol_matcher_init(&subvols, volinfo->subvol_count);
+        if (ret)
+            goto out;
+    }
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get brick "
-                          "count");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
+    brickinfo_list = GF_CALLOC(count, sizeof(*brickinfo_list),
+                               gf_common_mt_pointer);
+    if (!brickinfo_list) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_strn(dict, key, keylen, &brick);
         if (ret) {
-                 snprintf (err_str, sizeof (err_str),"Volume %s does not exist",
-                           volname);
-                 gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                         GD_MSG_VOL_NOT_FOUND, "%s", err_str);
-                 goto out;
+            snprintf(err_str, sizeof(err_str), "Unable to get %s", key);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "%s", err_str);
+            goto out;
         }
+        gf_msg_debug(this->name, 0,
+                     "Remove brick count %d brick:"
+                     " %s",
+                     i, brick);
 
-        if ((volinfo->type == GF_CLUSTER_TYPE_TIER) &&
-            (glusterd_is_tiering_supported(err_str) == _gf_false)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VERSION_UNSUPPORTED,
-                        "Tiering not supported at this version");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "command", &cmd);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get cmd "
-                          "ccommand");
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
 
-        ret = glusterd_disallow_op_for_tier (volinfo, GD_OP_REMOVE_BRICK, cmd);
         if (ret) {
-                snprintf (err_str, sizeof (err_str),
-                          "Removing brick from a Tier volume is not allowed");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_UNSUPPORTED, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "request to change replica-count to %d", replica_count);
-                ret = gd_rmbr_validate_replica_count (volinfo, replica_count,
-                                                      count, err_str,
-                                                      sizeof (err_str));
-                if (ret < 0) {
-                        /* logging and error msg are done in above function
-                           itself */
-                        goto out;
-                }
-                dict_del (dict, "replica-count");
-                if (ret) {
-                        replica_count = 0;
-                } else {
-                        ret = dict_set_int32 (dict, "replica-count",
-                                              replica_count);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, errno,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "failed to set the replica_count "
-                                        "in dict");
-                                goto out;
-                        }
-                }
-        }
-
-        /* 'vol_type' is used for giving the meaning full error msg for user */
-        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
-                strcpy (vol_type, "replica");
-        } else if (volinfo->type == GF_CLUSTER_TYPE_STRIPE) {
-                strcpy (vol_type, "stripe");
-        } else if (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
-                strcpy (vol_type, "stripe-replicate");
-        } else if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                strcpy (vol_type, "disperse");
-        } else {
-                strcpy (vol_type, "distribute");
-        }
-
-	/* Do not allow remove-brick if the volume is a stripe volume*/
-	if ((volinfo->type == GF_CLUSTER_TYPE_STRIPE) &&
-            (volinfo->brick_count == volinfo->stripe_count)) {
-                snprintf (err_str, sizeof (err_str),
-                          "Removing brick from a stripe volume is not allowed");
-                gf_msg (this->name, GF_LOG_ERROR, EPERM,
-                        GD_MSG_OP_NOT_PERMITTED, "%s", err_str);
-                ret = -1;
-                goto out;
-	}
-
-	if (!replica_count &&
-            (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) &&
-            (volinfo->brick_count == volinfo->dist_leaf_count)) {
-                snprintf (err_str, sizeof(err_str),
-                          "Removing bricks from stripe-replicate"
-                          " configuration is not allowed without reducing "
-                          "replica or stripe count explicitly.");
-                gf_msg (this->name, GF_LOG_ERROR, EPERM,
-                        GD_MSG_OP_NOT_PERMITTED_AC_REQD, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-	if (!replica_count &&
-            (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
-            (volinfo->brick_count == volinfo->dist_leaf_count)) {
-                snprintf (err_str, sizeof (err_str),
-                          "Removing bricks from replicate configuration "
-                          "is not allowed without reducing replica count "
-                          "explicitly.");
-                gf_msg (this->name, GF_LOG_ERROR, EPERM,
-                        GD_MSG_OP_NOT_PERMITTED_AC_REQD, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-	/* Do not allow remove-brick if the bricks given is less than
-           the replica count or stripe count */
-        if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-            (volinfo->type != GF_CLUSTER_TYPE_TIER))  {
-                if (volinfo->dist_leaf_count &&
-                    (count % volinfo->dist_leaf_count)) {
-                        snprintf (err_str, sizeof (err_str), "Remove brick "
-                                  "incorrect brick count of %d for %s %d",
-                                  count, vol_type, volinfo->dist_leaf_count);
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        /* subvol match is not required for tiered volume*/
-        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-             (volinfo->type != GF_CLUSTER_TYPE_TIER) &&
-             (volinfo->subvol_count > 1)) {
-                ret = subvol_matcher_init (&subvols, volinfo->subvol_count);
-                if (ret)
-                        goto out;
-        }
-
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                count = glusterd_set_detach_bricks(dict, volinfo);
-
-        brickinfo_list = GF_CALLOC (count, sizeof (*brickinfo_list),
-                                    gf_common_mt_pointer);
-        if (!brickinfo_list) {
-                ret = -1;
-                goto out;
-        }
-
-        while ( i <= count) {
-                snprintf (key, sizeof (key), "brick%d", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Unable to get %s",
-                                  key);
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                        goto out;
-                }
-                gf_msg_debug (this->name, 0, "Remove brick count %d brick:"
-                        " %s", i, brick);
-
-                ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
-                                                             &brickinfo,
-                                                             _gf_false);
-
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Incorrect brick "
-                                  "%s for volume %s", brick, volname);
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_BRICK_NOT_FOUND, "%s", err_str);
-                        goto out;
-                }
-                brickinfo_list[i-1] = brickinfo;
-
-                i++;
-                if ((volinfo->type == GF_CLUSTER_TYPE_NONE) ||
-                    (volinfo->brick_count <= volinfo->dist_leaf_count))
-                        continue;
-
-                /* Find which subvolume the brick belongs to.
-                 * subvol match is not required for tiered volume
-                 *
-                 */
-                if (volinfo->type != GF_CLUSTER_TYPE_TIER)
-                        subvol_matcher_update (subvols, volinfo, brickinfo);
-        }
-
-        /* Check if the bricks belong to the same subvolumes.*/
-        /* subvol match is not required for tiered volume*/
-         if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-             (volinfo->type != GF_CLUSTER_TYPE_TIER) &&
-             (volinfo->subvol_count > 1)) {
-                ret = subvol_matcher_verify (subvols, volinfo,
-                                             err_str, sizeof(err_str),
-                                             vol_type, replica_count);
-                if (ret)
-                        goto out;
-        }
-
-        ret = glusterd_remove_brick_validate_arbiters (volinfo, count,
-                                                       replica_count,
-                                                       brickinfo_list,
-                                                       err_str,
-                                                       sizeof (err_str));
+            snprintf(err_str, sizeof(err_str),
+                     "Incorrect brick "
+                     "%s for volume %s",
+                     brick, volname);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+                   "%s", err_str);
+            goto out;
+        }
+        brickinfo_list[i - 1] = brickinfo;
+
+        i++;
+        if ((volinfo->type == GF_CLUSTER_TYPE_NONE) ||
+            (volinfo->brick_count <= volinfo->dist_leaf_count))
+            continue;
+
+        subvol_matcher_update(subvols, volinfo, brickinfo);
+    }
+
+    if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+        (volinfo->subvol_count > 1)) {
+        ret = subvol_matcher_verify(subvols, volinfo, err_str, sizeof(err_str),
+                                    vol_type, replica_count);
         if (ret)
-                goto out;
-
-        ret = glusterd_op_begin_synctask (req, GD_OP_REMOVE_BRICK, dict);
+            goto out;
+    }
+
+    ret = glusterd_remove_brick_validate_arbiters(volinfo, count, replica_count,
+                                                  brickinfo_list, err_str,
+                                                  sizeof(err_str));
+    if (ret)
+        goto out;
+
+    if (conf->op_version < GD_OP_VERSION_8_0) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than %d. remove-brick operation"
+                     "falling back to syncop framework.",
+                     GD_OP_VERSION_8_0);
+        ret = glusterd_op_begin_synctask(req, GD_OP_REMOVE_BRICK, dict);
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_REMOVE_BRICK,
+                                                   dict);
+    }
 
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                rsp.op_errno = 0;
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_OP_FAILED, "%s", err_str);
-                rsp.op_errstr = err_str;
-                cli_rsp = &rsp;
-                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
-                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
-
-                ret = 0; //sent error to cli, prevent second reply
-
-        }
-
-        if (brickinfo_list)
-                GF_FREE (brickinfo_list);
-        subvol_matcher_destroy (subvols);
-        free (cli_req.dict.dict_val); //its malloced by xdr
-
-        return ret;
+    if (ret) {
+        rsp.op_ret = -1;
+        rsp.op_errno = 0;
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_OP_FAILED, "%s",
+               err_str);
+        rsp.op_errstr = err_str;
+        cli_rsp = &rsp;
+        glusterd_to_cli(req, cli_rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp,
+                        dict);
+
+        ret = 0;  // sent error to cli, prevent second reply
+    }
+
+    if (brickinfo_list)
+        GF_FREE(brickinfo_list);
+    subvol_matcher_destroy(subvols);
+    free(cli_req.dict.dict_val);  // its malloced by xdr
+
+    return ret;
 }
 
 int
-glusterd_handle_remove_brick (rpcsvc_request_t *req)
+glusterd_handle_remove_brick(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_remove_brick);
+    return glusterd_big_locked_handler(req, __glusterd_handle_remove_brick);
 }
 
 static int
-_glusterd_restart_gsync_session (dict_t *this, char *key,
-                                 data_t *value, void *data)
+_glusterd_restart_gsync_session(dict_t *this, char *key, data_t *value,
+                                void *data)
 {
-        char                          *slave      = NULL;
-        char                          *slave_buf  = NULL;
-        char                          *path_list  = NULL;
-        char                          *slave_vol  = NULL;
-        char                          *slave_host = NULL;
-        char                          *slave_url  = NULL;
-        char                          *conf_path  = NULL;
-        char                         **errmsg     = NULL;
-        int                            ret        = -1;
-        glusterd_gsync_status_temp_t  *param      = NULL;
-        gf_boolean_t                   is_running = _gf_false;
-
-        param = (glusterd_gsync_status_temp_t *)data;
-
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
-
-        slave = strchr(value->data, ':');
-        if (slave) {
-                slave++;
-                slave_buf = gf_strdup (slave);
-                if (!slave_buf) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY,
-                                "Failed to gf_strdup");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        else
-                return 0;
-
-        ret = dict_set_dynstr (param->rsp_dict, "slave", slave_buf);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to store slave");
-                if (slave_buf)
-                        GF_FREE(slave_buf);
-                goto out;
-        }
-
-        ret = glusterd_get_slave_details_confpath (param->volinfo,
-                                                   param->rsp_dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, errmsg);
-        if (ret) {
-                if (*errmsg)
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL,
-                                "%s", *errmsg);
-                else
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL,
-                                "Unable to fetch slave or confpath details.");
-                goto out;
-        }
-
-        /* In cases that gsyncd is not running, we will not invoke it
-         * because of add-brick. */
-        ret = glusterd_check_gsync_running_local (param->volinfo->volname,
-                                                  slave, conf_path,
-                                                  &is_running);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_GSYNC_VALIDATION_FAIL, "gsync running validation failed.");
-                goto out;
-        }
-        if (_gf_false == is_running) {
-                gf_msg_debug ("glusterd", 0, "gsync session for %s and %s is"
-                        " not running on this node. Hence not restarting.",
-                        param->volinfo->volname, slave);
-                ret = 0;
-                goto out;
-        }
+    char *slave = NULL;
+    char *slave_buf = NULL;
+    char *path_list = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *slave_url = NULL;
+    char *conf_path = NULL;
+    char **errmsg = NULL;
+    int ret = -1;
+    glusterd_gsync_status_temp_t *param = NULL;
+    gf_boolean_t is_running = _gf_false;
+
+    param = (glusterd_gsync_status_temp_t *)data;
+
+    GF_ASSERT(param);
+    GF_ASSERT(param->volinfo);
+
+    slave = strchr(value->data, ':');
+    if (slave) {
+        slave++;
+        slave_buf = gf_strdup(slave);
+        if (!slave_buf) {
+            gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Failed to gf_strdup");
+            ret = -1;
+            goto out;
+        }
+    } else
+        return 0;
 
-        ret = glusterd_get_local_brickpaths (param->volinfo, &path_list);
-        if (!path_list) {
-                gf_msg_debug ("glusterd", 0, "This node not being part of"
-                        " volume should not be running gsyncd. Hence"
-                        " no gsyncd process to restart.");
-                ret = 0;
-                goto out;
-        }
+    ret = dict_set_dynstrn(param->rsp_dict, "slave", SLEN("slave"), slave_buf);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave");
+        if (slave_buf)
+            GF_FREE(slave_buf);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(param->volinfo, param->rsp_dict,
+                                              &slave_url, &slave_host,
+                                              &slave_vol, &conf_path, errmsg);
+    if (ret) {
+        if (errmsg && *errmsg)
+            gf_msg("glusterd", GF_LOG_ERROR, 0,
+                   GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL, "%s", *errmsg);
+        else
+            gf_msg("glusterd", GF_LOG_ERROR, 0,
+                   GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL,
+                   "Unable to fetch slave or confpath details.");
+        goto out;
+    }
+
+    /* In cases that gsyncd is not running, we will not invoke it
+     * because of add-brick. */
+    ret = glusterd_check_gsync_running_local(param->volinfo->volname, slave,
+                                             conf_path, &is_running);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_GSYNC_VALIDATION_FAIL,
+               "gsync running validation failed.");
+        goto out;
+    }
+    if (_gf_false == is_running) {
+        gf_msg_debug("glusterd", 0,
+                     "gsync session for %s and %s is"
+                     " not running on this node. Hence not restarting.",
+                     param->volinfo->volname, slave);
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_get_local_brickpaths(param->volinfo, &path_list);
+    if (!path_list) {
+        gf_msg_debug("glusterd", 0,
+                     "This node not being part of"
+                     " volume should not be running gsyncd. Hence"
+                     " no gsyncd process to restart.");
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_check_restart_gsync_session (param->volinfo, slave,
-                                                    param->rsp_dict, path_list,
-                                                    conf_path, 0);
-        if (ret)
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_GSYNC_RESTART_FAIL,
-                        "Unable to restart gsync session.");
+    ret = glusterd_check_restart_gsync_session(
+        param->volinfo, slave, param->rsp_dict, path_list, conf_path, 0);
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_GSYNC_RESTART_FAIL,
+               "Unable to restart gsync session.");
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d.", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d.", ret);
+    return ret;
 }
 
 /* op-sm */
 
 int
-glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
-                                char  *bricks, dict_t *dict)
+glusterd_op_perform_add_bricks(glusterd_volinfo_t *volinfo, int32_t count,
+                               char *bricks, dict_t *dict)
 {
-        char                         *brick          = NULL;
-        int32_t                       i              = 1;
-        char                         *brick_list     = NULL;
-        char                         *free_ptr1      = NULL;
-        char                         *free_ptr2      = NULL;
-        char                         *saveptr        = NULL;
-        int32_t                       ret            = -1;
-        int32_t                       stripe_count   = 0;
-        int32_t                       replica_count  = 0;
-        int32_t                       arbiter_count  = 0;
-        int32_t                       type           = 0;
-        glusterd_brickinfo_t         *brickinfo      = NULL;
-        glusterd_gsync_status_temp_t  param          = {0, };
-        gf_boolean_t                  restart_needed = 0;
-        char                          msg[1024] __attribute__((unused)) = {0, };
-        int                           caps           = 0;
-        int                           brickid        = 0;
-        char                          key[PATH_MAX]  = "";
-        char                         *brick_mount_dir  = NULL;
-        xlator_t                     *this           = NULL;
-        glusterd_conf_t              *conf           = NULL;
-        gf_boolean_t                  is_valid_add_brick = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                free_ptr1 = brick_list;
-        }
-
-        if (count)
-                brick = strtok_r (brick_list+1, " \n", &saveptr);
-
-        if (dict) {
-                ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
-                if (!ret)
-                        gf_msg (THIS->name, GF_LOG_INFO, errno,
-                                GD_MSG_DICT_GET_SUCCESS,
-                                "stripe-count is set %d", stripe_count);
-
-                ret = dict_get_int32 (dict, "replica-count", &replica_count);
-                if (!ret)
-                        gf_msg (THIS->name, GF_LOG_INFO, errno,
-                                GD_MSG_DICT_GET_SUCCESS,
-                                "replica-count is set %d", replica_count);
-                ret = dict_get_int32 (dict, "arbiter-count", &arbiter_count);
-                if (!ret)
-                        gf_msg (THIS->name, GF_LOG_INFO, errno,
-                                GD_MSG_DICT_GET_SUCCESS,
-                                "arbiter-count is set %d", arbiter_count);
-                ret = dict_get_int32 (dict, "type", &type);
-                if (!ret)
-                        gf_msg (THIS->name, GF_LOG_INFO, errno,
-                                GD_MSG_DICT_GET_SUCCESS,
-                                "type is set %d, need to change it", type);
-        }
-
-        brickid = glusterd_get_next_available_brickid (volinfo);
-        if (brickid < 0)
-                goto out;
-        while ( i <= count) {
-                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo,
-                                                         _gf_true, NULL);
-                if (ret)
-                        goto out;
-
-                GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
-                                                      brickid++);
-
-                /* A bricks mount dir is required only by snapshots which were
-                 * introduced in gluster-3.6.0
-                 */
-                if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                        brick_mount_dir = NULL;
-
-                        snprintf (key, sizeof(key), "brick%d.mount_dir", i);
-                        ret = dict_get_str (dict, key, &brick_mount_dir);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "%s not present", key);
-                                goto out;
-                        }
-                        strncpy (brickinfo->mount_dir, brick_mount_dir,
-                                 sizeof(brickinfo->mount_dir));
-                }
+    char *brick = NULL;
+    int32_t i = 1;
+    char *brick_list = NULL;
+    char *free_ptr1 = NULL;
+    char *free_ptr2 = NULL;
+    char *saveptr = NULL;
+    int32_t ret = -1;
+    int32_t stripe_count = 0;
+    int32_t replica_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t type = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_gsync_status_temp_t param = {
+        0,
+    };
+    gf_boolean_t restart_needed = 0;
+    int brickid = 0;
+    char key[64] = "";
+    char *brick_mount_dir = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t is_valid_add_brick = _gf_false;
+    gf_boolean_t restart_shd = _gf_false;
+    struct statvfs brickstat = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (bricks) {
+        brick_list = gf_strdup(bricks);
+        free_ptr1 = brick_list;
+    }
+
+    if (count)
+        brick = strtok_r(brick_list + 1, " \n", &saveptr);
+
+    if (dict) {
+        ret = dict_get_int32n(dict, "stripe-count", SLEN("stripe-count"),
+                              &stripe_count);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "stripe-count is set %d", stripe_count);
 
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret)
-                        goto out;
-
-                /* hot tier bricks are added to head of brick list */
-                if (dict_get (dict, "attach-tier")) {
-                        cds_list_add (&brickinfo->brick_list, &volinfo->bricks);
-                } else if (stripe_count || replica_count) {
-                        add_brick_at_right_order (brickinfo, volinfo, (i - 1),
-                                                  stripe_count, replica_count);
-                } else {
-                        cds_list_add_tail (&brickinfo->brick_list,
-                                           &volinfo->bricks);
-                }
-                brick = strtok_r (NULL, " \n", &saveptr);
-                i++;
-                volinfo->brick_count++;
+        ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                              &replica_count);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "replica-count is set %d", replica_count);
+        ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"),
+                              &arbiter_count);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "arbiter-count is set %d", arbiter_count);
+        ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "type is set %d, need to change it", type);
+    }
+
+    brickid = glusterd_get_next_available_brickid(volinfo);
+    if (brickid < 0)
+        goto out;
+    while (i <= count) {
+        ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_true,
+                                                NULL);
+        if (ret)
+            goto out;
 
-        }
+        GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid++);
 
-        /* Gets changed only if the options are given in add-brick cli */
-        if (type)
-                volinfo->type = type;
+        /* A bricks mount dir is required only by snapshots which were
+         * introduced in gluster-3.6.0
+         */
+        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+            brick_mount_dir = NULL;
 
-        if (replica_count) {
-                volinfo->replica_count = replica_count;
-        }
-        if (arbiter_count) {
-                volinfo->arbiter_count = arbiter_count;
-        }
-        if (stripe_count) {
-                volinfo->stripe_count = stripe_count;
+            snprintf(key, sizeof(key), "brick%d.mount_dir", i);
+            ret = dict_get_str(dict, key, &brick_mount_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                       "%s not present", key);
+                goto out;
+            }
+            strncpy(brickinfo->mount_dir, brick_mount_dir,
+                    SLEN(brickinfo->mount_dir));
         }
-        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
-
-        /* backward compatibility */
-        volinfo->sub_count = ((volinfo->dist_leaf_count == 1) ? 0:
-                              volinfo->dist_leaf_count);
-
-        volinfo->subvol_count = (volinfo->brick_count /
-                                 volinfo->dist_leaf_count);
 
-        ret = 0;
-        if (GLUSTERD_STATUS_STARTED != volinfo->status)
-                goto generate_volfiles;
-
-        ret = generate_brick_volfiles (volinfo);
+        ret = glusterd_resolve_brick(brickinfo);
         if (ret)
-                goto out;
+            goto out;
 
-        brick_list = gf_strdup (bricks);
-        free_ptr2 = brick_list;
-        i = 1;
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            ret = sys_statvfs(brickinfo->path, &brickstat);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_STATVFS_FAILED,
+                       "Failed to fetch disk utilization "
+                       "from the brick (%s:%s). Please check the health of "
+                       "the brick. Error code was %s",
+                       brickinfo->hostname, brickinfo->path, strerror(errno));
 
-        if (count)
-                brick = strtok_r (brick_list+1, " \n", &saveptr);
-#ifdef HAVE_BD_XLATOR
-        if (brickinfo->vg[0])
-                caps = CAPS_BD | CAPS_THIN |
-                        CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
-#endif
-
-        /* This check needs to be added to distinguish between
-         * attach-tier commands and add-brick commands.
-         * When a tier is attached, adding is done via add-brick
-         * and setting of pending xattrs shouldn't be done for
-         * attach-tiers as they are virtually new volumes.
-         */
-        if (glusterd_is_volume_replicate (volinfo)) {
-                if (replica_count &&
-                    !dict_get (dict, "attach-tier") &&
-                    conf->op_version >= GD_OP_VERSION_3_7_10) {
-                        is_valid_add_brick = _gf_true;
-                        ret = generate_dummy_client_volfiles (volinfo);
-                        if (ret) {
-                                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOLFILE_CREATE_FAIL,
-                                        "Failed to create volfile.");
-                                goto out;
-                                }
-                        }
+                goto out;
+            }
+            brickinfo->statfs_fsid = brickstat.f_fsid;
         }
-
-        while (i <= count) {
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                             _gf_true);
-                if (ret)
-                        goto out;
-#ifdef HAVE_BD_XLATOR
-                /* Check for VG/thin pool if its BD volume */
-                if (brickinfo->vg[0]) {
-                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
-                        if (ret) {
-                                gf_msg (THIS->name, GF_LOG_CRITICAL, 0,
-                                        GD_MSG_INVALID_VG, "%s", msg);
-                                goto out;
-                        }
-                        /* if anyone of the brick does not have thin support,
-                           disable it for entire volume */
-                        caps &= brickinfo->caps;
-                } else
-                        caps = 0;
-#endif
-
-                if (gf_uuid_is_null (brickinfo->uuid)) {
-                        ret = glusterd_resolve_brick (brickinfo);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_RESOLVE_BRICK_FAIL, FMTSTR_RESOLVE_BRICK,
-                                        brickinfo->hostname, brickinfo->path);
-                                goto out;
-                        }
+        if (stripe_count || replica_count) {
+            add_brick_at_right_order(brickinfo, volinfo, (i - 1), stripe_count,
+                                     replica_count);
+        } else {
+            cds_list_add_tail(&brickinfo->brick_list, &volinfo->bricks);
+        }
+        brick = strtok_r(NULL, " \n", &saveptr);
+        i++;
+        volinfo->brick_count++;
+    }
+
+    /* Gets changed only if the options are given in add-brick cli */
+    if (type)
+        volinfo->type = type;
+    /* performance.client-io-threads is turned on by default,
+     * however this has adverse effects on replicate volumes due to
+     * replication design issues, till that get addressed
+     * performance.client-io-threads option is turned off for all
+     * replicate volumes if not already explicitly enabled.
+     */
+    if (type && glusterd_is_volume_replicate(volinfo) &&
+        conf->op_version >= GD_OP_VERSION_3_12_2) {
+        ret = dict_set_nstrn(volinfo->dict, "performance.client-io-threads",
+                             SLEN("performance.client-io-threads"), "off",
+                             SLEN("off"));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "performance.client-io-threads to off");
+            goto out;
+        }
+    }
+
+    if (replica_count) {
+        volinfo->replica_count = replica_count;
+    }
+    if (arbiter_count) {
+        volinfo->arbiter_count = arbiter_count;
+    }
+    if (stripe_count) {
+        volinfo->stripe_count = stripe_count;
+    }
+    volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
+
+    /* backward compatibility */
+    volinfo->sub_count = ((volinfo->dist_leaf_count == 1)
+                              ? 0
+                              : volinfo->dist_leaf_count);
+
+    volinfo->subvol_count = (volinfo->brick_count / volinfo->dist_leaf_count);
+
+    ret = 0;
+    if (GLUSTERD_STATUS_STARTED != volinfo->status)
+        goto generate_volfiles;
+
+    ret = generate_brick_volfiles(volinfo);
+    if (ret)
+        goto out;
+
+    brick_list = gf_strdup(bricks);
+    free_ptr2 = brick_list;
+    i = 1;
+
+    if (count)
+        brick = strtok_r(brick_list + 1, " \n", &saveptr);
+
+    if (glusterd_is_volume_replicate(volinfo)) {
+        if (replica_count && conf->op_version >= GD_OP_VERSION_3_7_10) {
+            is_valid_add_brick = _gf_true;
+            if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+                ret = volinfo->shd.svc.stop(&(volinfo->shd.svc), SIGTERM);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0,
+                           GD_MSG_GLUSTER_SERVICES_STOP_FAIL,
+                           "Failed to stop shd for %s.", volinfo->volname);
                 }
+                restart_shd = _gf_true;
+            }
+            ret = generate_dummy_client_volfiles(volinfo);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                       "Failed to create volfile.");
+                goto out;
+            }
+        }
+    }
 
-                /* if the volume is a replicate volume, do: */
-                if (is_valid_add_brick) {
-                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                ret = glusterd_handle_replicate_brick_ops (
-                                                           volinfo, brickinfo,
-                                                           GD_OP_ADD_BRICK);
-                                if (ret < 0)
-                                        goto out;
-                        }
-                }
-                ret = glusterd_brick_start (volinfo, brickinfo,
-                                            _gf_true);
-                if (ret)
-                        goto out;
-                i++;
-                brick = strtok_r (NULL, " \n", &saveptr);
-
-                /* Check if the brick is added in this node, and set
-                 * the restart_needed flag. */
-                if ((!gf_uuid_compare (brickinfo->uuid, MY_UUID)) &&
-                    !restart_needed) {
-                        restart_needed = 1;
-                        gf_msg_debug ("glusterd", 0,
-                                "Restart gsyncd session, if it's already "
-                                "running.");
-                }
+    while (i <= count) {
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_true);
+        if (ret)
+            goto out;
+
+        if (gf_uuid_is_null(brickinfo->uuid)) {
+            ret = glusterd_resolve_brick(brickinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                       FMTSTR_RESOLVE_BRICK, brickinfo->hostname,
+                       brickinfo->path);
+                goto out;
+            }
         }
 
-        /* If the restart_needed flag is set, restart gsyncd sessions for that
-         * particular master with all the slaves. */
-        if (restart_needed) {
-                param.rsp_dict = dict;
-                param.volinfo = volinfo;
-                dict_foreach (volinfo->gsync_slaves,
-                              _glusterd_restart_gsync_session, &param);
+        /* if the volume is a replicate volume, do: */
+        if (is_valid_add_brick) {
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                ret = glusterd_handle_replicate_brick_ops(volinfo, brickinfo,
+                                                          GD_OP_ADD_BRICK);
+                if (ret < 0)
+                    goto out;
+            }
         }
-        volinfo->caps = caps;
+        ret = glusterd_brick_start(volinfo, brickinfo, _gf_true, _gf_false);
+        if (ret)
+            goto out;
+        i++;
+        brick = strtok_r(NULL, " \n", &saveptr);
+
+        /* Check if the brick is added in this node, and set
+         * the restart_needed flag. */
+        if ((!gf_uuid_compare(brickinfo->uuid, MY_UUID)) && !restart_needed) {
+            restart_needed = 1;
+            gf_msg_debug("glusterd", 0,
+                         "Restart gsyncd session, if it's already "
+                         "running.");
+        }
+    }
+
+    /* If the restart_needed flag is set, restart gsyncd sessions for that
+     * particular master with all the slaves. */
+    if (restart_needed) {
+        param.rsp_dict = dict;
+        param.volinfo = volinfo;
+        dict_foreach(volinfo->gsync_slaves, _glusterd_restart_gsync_session,
+                     &param);
+    }
 
 generate_volfiles:
-        if (conf->op_version <= GD_OP_VERSION_3_7_5) {
-               ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        } else {
-                /*
-                 * The cluster is operating at version greater than
-                 * gluster-3.7.5. So no need to sent volfile fetch
-                 * request in commit phase, the same will be done
-                 * in post validate phase with v3 framework.
-                 */
-        }
+    if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+        ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    } else {
+        /*
+         * The cluster is operating at version greater than
+         * gluster-3.7.5. So no need to sent volfile fetch
+         * request in commit phase, the same will be done
+         * in post validate phase with v3 framework.
+         */
+    }
 
 out:
-        GF_FREE (free_ptr1);
-        GF_FREE (free_ptr2);
-
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(free_ptr1);
+    GF_FREE(free_ptr2);
+    if (restart_shd) {
+        if (volinfo->shd.svc.manager(&(volinfo->shd.svc), volinfo,
+                                     PROC_START_NO_WAIT)) {
+            gf_msg("glusterd", GF_LOG_CRITICAL, 0,
+                   GD_MSG_GLUSTER_SERVICE_START_FAIL,
+                   "Failed to start shd for %s.", volinfo->volname);
+        }
+    }
+
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int
-glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
-                                  int force, int *need_migrate)
+glusterd_op_perform_remove_brick(glusterd_volinfo_t *volinfo, char *brick,
+                                 int force, int *need_migrate)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int32_t                  ret = -1;
-        glusterd_conf_t         *priv = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brick);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brick);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                      &brickinfo,
-                                                      _gf_false);
-        if (ret)
-                goto out;
+    ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                 _gf_false);
+    if (ret)
+        goto out;
 
-        ret = glusterd_resolve_brick (brickinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_resolve_brick(brickinfo);
+    if (ret)
+        goto out;
 
-        glusterd_volinfo_reset_defrag_stats (volinfo);
+    glusterd_volinfo_reset_defrag_stats(volinfo);
 
-        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                /* Only if the brick is in this glusterd, do the rebalance */
-                if (need_migrate)
-                        *need_migrate = 1;
-        }
+    if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+        /* Only if the brick is in this glusterd, do the rebalance */
+        if (need_migrate)
+            *need_migrate = 1;
+    }
 
-        if (force) {
-                ret = glusterd_brick_stop (volinfo, brickinfo,
-                                           _gf_true);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_STOP_FAIL, "Unable to stop "
-                                "glusterfs, ret: %d", ret);
-                }
-                goto out;
+    if (force) {
+        ret = glusterd_brick_stop(volinfo, brickinfo, _gf_true);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                   "Unable to stop "
+                   "glusterfs, ret: %d",
+                   ret);
         }
+        goto out;
+    }
 
-        brickinfo->decommissioned = 1;
-        ret = 0;
+    brickinfo->decommissioned = 1;
+    ret = 0;
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        int                                     count = 0;
-        int                                     replica_count = 0;
-        int                                     arbiter_count = 0;
-        int                                     i = 0;
-        int32_t                                 local_brick_count = 0;
-        char                                    *bricks    = NULL;
-        char                                    *brick_list = NULL;
-        char                                    *saveptr = NULL;
-        char                                    *free_ptr = NULL;
-        char                                    *brick = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    msg[2048] = {0,};
-        char                                    key[PATH_MAX] = "";
-        gf_boolean_t                            brick_alloc = _gf_false;
-        char                                    *all_bricks = NULL;
-        char                                    *str_ret = NULL;
-        gf_boolean_t                            is_force = _gf_false;
-        glusterd_conf_t                         *conf = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = dict_get_str (dict, "volname", &volname);
+    int ret = 0;
+    char *volname = NULL;
+    int count = 0;
+    int replica_count = 0;
+    int arbiter_count = 0;
+    int i = 0;
+    int32_t local_brick_count = 0;
+    char *bricks = NULL;
+    char *brick_list = NULL;
+    char *saveptr = NULL;
+    char *free_ptr = NULL;
+    char *brick = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char msg[4096] = "";
+    char key[64] = "";
+    gf_boolean_t brick_alloc = _gf_false;
+    char *all_bricks = NULL;
+    char *str_ret = NULL;
+    gf_boolean_t is_force = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find volume: %s", volname);
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get replica count");
+    }
+
+    if (replica_count > 0) {
+        ret = op_version_check(this, GD_OP_VER_PERSISTENT_AFR_XATTRS, msg,
+                               sizeof(msg));
         if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get volume name");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+                   "%s", msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
         }
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND,
-                        "Unable to find volume: %s", volname);
-                goto out;
-        }
+    glusterd_add_peers_to_auth_list(volname);
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (ret) {
-                gf_msg_debug (THIS->name, 0,
-                        "Unable to get replica count");
+    if (replica_count && glusterd_is_volume_replicate(volinfo)) {
+        /* Do not allow add-brick for stopped volumes when replica-count
+         * is being increased.
+         */
+        if (GLUSTERD_STATUS_STOPPED == volinfo->status &&
+            conf->op_version >= GD_OP_VERSION_3_7_10) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     " Volume must not be in"
+                     " stopped state when replica-count needs to "
+                     " be increased.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+        /* op-version check for replica 2 to arbiter conversion. If we
+         * don't have this check, an older peer added as arbiter brick
+         * will not have the  arbiter xlator in its volfile. */
+        if ((replica_count == 3) && (conf->op_version < GD_OP_VERSION_3_8_0)) {
+            ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"),
+                                  &arbiter_count);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "No arbiter count present in the dict");
+            } else if (arbiter_count == 1) {
+                ret = -1;
+                snprintf(msg, sizeof(msg),
+                         "Cluster op-version must "
+                         "be >= 30800 to add arbiter brick to a "
+                         "replica 2 volume.");
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                       msg);
+                *op_errstr = gf_strdup(msg);
+                goto out;
+            }
+        }
+        /* Do not allow increasing replica count for arbiter volumes. */
+        if (volinfo->arbiter_count) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Increasing replica count "
+                     "for arbiter volumes is not supported.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    /* Check brick order if the volume type is replicate or disperse. If
+     * force at the end of command not given then check brick order.
+     * doing this check at the originator node is sufficient.
+     */
+
+    if (!is_force && is_origin_glusterd(dict)) {
+        ret = 0;
+        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+            gf_msg_debug(this->name, 0,
+                         "Replicate cluster type "
+                         "found. Checking brick order.");
+            if (replica_count)
+                ret = glusterd_check_brick_order(dict, msg, volinfo->type,
+                                                 &volname, &bricks, &count,
+                                                 replica_count);
+            else
+                ret = glusterd_check_brick_order(dict, msg, volinfo->type,
+                                                 &volname, &bricks, &count,
+                                                 volinfo->replica_count);
+        } else if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+            gf_msg_debug(this->name, 0,
+                         "Disperse cluster type"
+                         " found. Checking brick order.");
+            ret = glusterd_check_brick_order(dict, msg, volinfo->type, &volname,
+                                             &bricks, &count,
+                                             volinfo->disperse_count);
         }
-
-        ret = dict_get_int32 (dict, "arbiter-count", &arbiter_count);
         if (ret) {
-                gf_msg_debug (THIS->name, 0,
-                        "No arbiter count present in the dict");
-        }
-
-        if (replica_count > 0) {
-                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
-                                        msg, sizeof(msg));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OP_VERSION_MISMATCH, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-        }
-
-        if (glusterd_is_volume_replicate (volinfo)) {
-                /* Do not allow add-brick for stopped volumes when replica-count
-                 * is being increased.
-                 */
-                if (conf->op_version >= GD_OP_VERSION_3_7_10 &&
-                    !dict_get (dict, "attach-tier") &&
-                    replica_count &&
-                    GLUSTERD_STATUS_STOPPED == volinfo->status) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), " Volume must not be in"
-                                  " stopped state when replica-count needs to "
-                                  " be increased.");
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_ADD_FAIL, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-                /* op-version check for replica 2 to arbiter conversion. If we
-                * dont have this check, an older peer added as arbiter brick
-                * will not have the  arbiter xlator in its volfile. */
-                if ((conf->op_version < GD_OP_VERSION_3_8_0) &&
-                    (arbiter_count == 1) && (replica_count == 3)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Cluster op-version must "
-                                  "be >= 30800 to add arbiter brick to a "
-                                  "replica 2 volume.");
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_ADD_FAIL, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-        }
-
-        if (conf->op_version > GD_OP_VERSION_3_7_5 &&
-            is_origin_glusterd (dict)) {
-                ret = glusterd_validate_quorum (this, GD_OP_ADD_BRICK, dict,
-                                                op_errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                GD_MSG_SERVER_QUORUM_NOT_MET,
-                                "Server quorum not met. Rejecting operation.");
-                        goto out;
-                }
-        } else {
-                /* Case 1: conf->op_version <= GD_OP_VERSION_3_7_5
-                 *         in this case the add-brick is running
-                 *         syncop framework that will do a quorum
-                 *         check by default
-                 * Case 2: We don't need to do quorum check on every
-                 *         node, only originator glusterd need to
-                 *         check for quorum
-                 * So nothing need to be done in else
-                 */
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER,
+                   "Not adding brick because of "
+                   "bad brick order. %s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
         }
+    }
 
-        if (glusterd_is_defrag_on(volinfo)) {
-                snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
-                          "progress. Please retry after completion", volname);
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OIP_RETRY_LATER, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+    if (volinfo->replica_count < replica_count && !is_force) {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
+            if (brickinfo->status == GF_BRICK_STOPPED) {
                 ret = -1;
-                goto out;
-        }
-
-        if (dict_get(dict, "attach-tier")) {
-
-                /*
-                 * This check is needed because of add/remove brick
-                 * is not supported on a tiered volume. So once a tier
-                 * is attached we cannot commit or stop the remove-brick
-                 * task. Please change this comment once we start supporting
-                 * add/remove brick on a tiered volume.
-                 */
-                if (!gd_is_remove_brick_committed (volinfo)) {
-
-                        snprintf (msg, sizeof (msg), "An earlier remove-brick "
-                                  "task exists for volume %s. Either commit it"
-                                  " or stop it before attaching a tier.",
-                                  volinfo->volname);
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OLD_REMOVE_BRICK_EXISTS, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
+                len = snprintf(msg, sizeof(msg),
+                               "Brick %s "
+                               "is down, changing replica "
+                               "count needs all the bricks "
+                               "to be up to avoid data loss",
+                               brickinfo->path);
+                if (len < 0) {
+                    strcpy(msg, "<error>");
                 }
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                       msg);
+                *op_errstr = gf_strdup(msg);
+                goto out;
+            }
         }
+    }
 
-        ret = dict_get_int32 (dict, "count", &count);
+    if (conf->op_version > GD_OP_VERSION_3_7_5 && is_origin_glusterd(dict)) {
+        ret = glusterd_validate_quorum(this, GD_OP_ADD_BRICK, dict, op_errstr);
         if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get count");
-                goto out;
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Server quorum not met. Rejecting operation.");
+            goto out;
+        }
+    } else {
+        /* Case 1: conf->op_version <= GD_OP_VERSION_3_7_5
+         *         in this case the add-brick is running
+         *         syncop framework that will do a quorum
+         *         check by default
+         * Case 2: We don't need to do quorum check on every
+         *         node, only originator glusterd need to
+         *         check for quorum
+         * So nothing need to be done in else
+         */
+    }
+
+    if (glusterd_is_defrag_on(volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume name %s rebalance is in "
+                 "progress. Please retry after completion",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OIP_RETRY_LATER, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
+
+    if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s  has %" PRIu64
+                 " snapshots. "
+                 "Changing the volume configuration will not effect snapshots."
+                 "But the snapshot brick mount should be intact to "
+                 "make them function.",
+                 volname, volinfo->snap_count);
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", msg);
+        msg[0] = '\0';
+    }
+
+    if (!count) {
+        ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get count");
+            goto out;
         }
+    }
 
-        ret = dict_get_str (dict, "bricks", &bricks);
+    if (!bricks) {
+        ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
         if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get bricks");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get bricks");
+            goto out;
+        }
+    }
+
+    if (bricks) {
+        brick_list = gf_strdup(bricks);
+        all_bricks = gf_strdup(bricks);
+        free_ptr = brick_list;
+    }
+
+    if (count)
+        brick = strtok_r(brick_list + 1, " \n", &saveptr);
+
+    while (i < count) {
+        if (!glusterd_store_is_valid_brickpath(volname, brick) ||
+            !glusterd_is_valid_volfpath(volname, brick)) {
+            snprintf(msg, sizeof(msg),
+                     "brick path %s is "
+                     "too long",
+                     brick);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRKPATH_TOO_LONG, "%s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_true,
+                                                NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND,
+                   "Add-brick: Unable"
+                   " to get brickinfo");
+            goto out;
         }
+        brick_alloc = _gf_true;
 
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                all_bricks = gf_strdup (bricks);
-                free_ptr = brick_list;
+        ret = glusterd_new_brick_validate(brick, brickinfo, msg, sizeof(msg),
+                                          NULL);
+        if (ret) {
+            *op_errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
         }
 
-        if (count)
-                brick = strtok_r (brick_list+1, " \n", &saveptr);
-
-
-        while ( i < count) {
-                if (!glusterd_store_is_valid_brickpath (volname, brick) ||
-                    !glusterd_is_valid_volfpath (volname, brick)) {
-                        snprintf (msg, sizeof (msg), "brick path %s is "
-                                  "too long", brick);
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRKPATH_TOO_LONG, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-
-                        ret = -1;
-                        goto out;
-
-                }
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            ret = glusterd_validate_and_create_brickpath(
+                brickinfo, volinfo->volume_id, volinfo->volname, op_errstr,
+                is_force, _gf_false);
+            if (ret)
+                goto out;
 
-                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo,
-                                                         _gf_true, NULL);
+            /* A bricks mount dir is required only by snapshots which were
+             * introduced in gluster-3.6.0
+             */
+            if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+                ret = glusterd_get_brick_mount_dir(
+                    brickinfo->path, brickinfo->hostname, brickinfo->mount_dir);
                 if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_NOT_FOUND,
-                                "Add-brick: Unable"
-                                " to get brickinfo");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                           "Failed to get brick mount_dir");
+                    goto out;
                 }
-                brick_alloc = _gf_true;
 
-                ret = glusterd_new_brick_validate (brick, brickinfo, msg,
-                                                   sizeof (msg));
+                snprintf(key, sizeof(key), "brick%d.mount_dir", i + 1);
+                ret = dict_set_dynstr_with_alloc(rsp_dict, key,
+                                                 brickinfo->mount_dir);
                 if (ret) {
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-#ifdef HAVE_BD_XLATOR
-                        if (brickinfo->vg[0]) {
-                                ret = glusterd_is_valid_vg (brickinfo, 1, msg);
-                                if (ret) {
-                                        gf_msg (THIS->name, GF_LOG_ERROR, EINVAL,
-                                               GD_MSG_INVALID_VG, "%s",
-                                                msg);
-                                        *op_errstr = gf_strdup (msg);
-                                        goto out;
-                                }
-                        }
-#endif
-
-                        ret = glusterd_validate_and_create_brickpath (brickinfo,
-                                                          volinfo->volume_id,
-                                                          op_errstr, is_force);
-                        if (ret)
-                                goto out;
-
-                        /* A bricks mount dir is required only by snapshots which were
-                         * introduced in gluster-3.6.0
-                         */
-                        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                                ret = glusterd_get_brick_mount_dir
-                                        (brickinfo->path, brickinfo->hostname,
-                                         brickinfo->mount_dir);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
-                                                "Failed to get brick mount_dir");
-                                        goto out;
-                                }
-
-                                snprintf (key, sizeof(key), "brick%d.mount_dir",
-                                          i + 1);
-                                ret = dict_set_dynstr_with_alloc
-                                        (rsp_dict, key, brickinfo->mount_dir);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                                GD_MSG_DICT_SET_FAILED,
-                                                "Failed to set %s", key);
-                                        goto out;
-                                }
-                        }
-
-                        local_brick_count = i + 1;
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_DICT_SET_FAILED, "Failed to set %s", key);
+                    goto out;
                 }
+            }
 
-                glusterd_brickinfo_delete (brickinfo);
-                brick_alloc = _gf_false;
-                brickinfo = NULL;
-                brick = strtok_r (NULL, " \n", &saveptr);
-                i++;
-        }
-
-        ret = dict_set_int32 (rsp_dict, "brick_count",
-                              local_brick_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set local_brick_count");
-                goto out;
+            local_brick_count = i + 1;
         }
 
-out:
-        GF_FREE (free_ptr);
-        if (brick_alloc && brickinfo)
-                glusterd_brickinfo_delete (brickinfo);
-        GF_FREE (str_ret);
-        GF_FREE (all_bricks);
-
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-
-        return ret;
-}
-
-static int
-glusterd_remove_brick_validate_bricks (gf1_op_commands cmd, int32_t brick_count,
-                                       dict_t *dict,
-                                       glusterd_volinfo_t *volinfo,
-                                       char **errstr)
-{
-        char                   *brick       = NULL;
-        char                    msg[2048]   = {0,};
-        char                    key[256]    = {0,};
-        glusterd_brickinfo_t   *brickinfo   = NULL;
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-        int                     i           = 0;
-        int                     ret         = -1;
-
-        /* Check whether all the nodes of the bricks to be removed are
-        * up, if not fail the operation */
-        for (i = 1; i <= brick_count; i++) {
-                snprintf (key, sizeof (key), "brick%d", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        snprintf (msg, sizeof (msg),
-                                  "Unable to get %s", key);
-                        *errstr = gf_strdup (msg);
-                        goto out;
-                }
-
-                ret =
-                glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
-                                                       &brickinfo,
-                                                       _gf_false);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Incorrect brick "
-                                  "%s for volume %s", brick, volinfo->volname);
-                        *errstr = gf_strdup (msg);
-                        goto out;
-                }
-                /* Do not allow commit if the bricks are not decommissioned
-                 * if its a remove brick commit or detach-tier commit
-                 */
-                if (!brickinfo->decommissioned) {
-                        if (cmd == GF_OP_CMD_COMMIT) {
-                                snprintf (msg, sizeof (msg), "Brick %s "
-                                          "is not decommissioned. "
-                                          "Use start or force option", brick);
-                                *errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
+        glusterd_brickinfo_delete(brickinfo);
+        brick_alloc = _gf_false;
+        brickinfo = NULL;
+        brick = strtok_r(NULL, " \n", &saveptr);
+        i++;
+    }
 
-                        if (cmd == GF_OP_CMD_DETACH_COMMIT) {
-                                snprintf (msg, sizeof (msg), "Brick's in Hot "
-                                          "tier is not decommissioned yet. Use "
-                                          "gluster volume detach-tier <VOLNAME>"
-                                          " <start | commit | force>"
-                                          " command instead");
-                                *errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                }
+    ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"),
+                          local_brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set local_brick_count");
+        goto out;
+    }
 
-                if (glusterd_is_local_brick (THIS, volinfo, brickinfo)) {
-                        if (((cmd == GF_OP_CMD_START) ||
-                            (cmd == GF_OP_CMD_DETACH_START))  &&
-                            brickinfo->status != GF_BRICK_STARTED) {
-                                snprintf (msg, sizeof (msg), "Found stopped "
-                                          "brick %s", brick);
-                                *errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                        continue;
-                }
+out:
+    GF_FREE(free_ptr);
+    if (brick_alloc && brickinfo)
+        glusterd_brickinfo_delete(brickinfo);
+    GF_FREE(str_ret);
+    GF_FREE(all_bricks);
 
-                rcu_read_lock ();
-                peerinfo = glusterd_peerinfo_find_by_uuid
-                                                (brickinfo->uuid);
-                if (!peerinfo) {
-                        snprintf (msg, sizeof(msg), "Host node of the "
-                                  "brick %s is not in cluster", brick);
-                        *errstr = gf_strdup (msg);
-                        ret = -1;
-                        rcu_read_unlock ();
-                        goto out;
-                }
-                if (!peerinfo->connected) {
-                        snprintf (msg, sizeof(msg), "Host node of the "
-                                  "brick %s is down", brick);
-                        *errstr = gf_strdup (msg);
-                        ret = -1;
-                        rcu_read_unlock ();
-                        goto out;
-                }
-                rcu_read_unlock ();
-        }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
+glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count,
+                                      dict_t *dict, glusterd_volinfo_t *volinfo,
+                                      char **errstr,
+                                      gf_cli_defrag_type cmd_defrag)
 {
-        int                     ret         = -1;
-        char                   *volname     = NULL;
-        glusterd_volinfo_t     *volinfo     = NULL;
-        char                   *errstr      = NULL;
-        int32_t                 brick_count = 0;
-        char                    msg[2048]   = {0,};
-        int32_t                 flag        = 0;
-        gf1_op_commands         cmd         = GF_OP_CMD_NONE;
-        char                   *task_id_str = NULL;
-        xlator_t               *this        = NULL;
-        int                     i           = 1;
-        char                    key[256]    = {0,};
-        char                   *brick       = NULL;
-        glusterd_brickinfo_t   *brickinfo   = NULL;
-        gsync_status_param_t    param       = {0,};
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
-                                msg, sizeof(msg));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_VERSION_MISMATCH, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "Volume %s does not exist", volname);
-                goto out;
-        }
-
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "command", &flag);
+    char *brick = NULL;
+    char msg[2048] = "";
+    char key[64] = "";
+    int keylen;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int i = 0;
+    int ret = -1;
+    char pidfile[PATH_MAX + 1] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+    int pid = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    /* Check whether all the nodes of the bricks to be removed are
+     * up, if not fail the operation */
+    for (i = 1; i <= brick_count; i++) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_strn(dict, key, keylen, &brick);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get brick command");
-                goto out;
+            snprintf(msg, sizeof(msg), "Unable to get %s", key);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                    "key=%s", key, NULL);
+            *errstr = gf_strdup(msg);
+            goto out;
         }
-        cmd = flag;
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get brick count");
-                goto out;
-        }
-
-        ret = 0;
-        if (volinfo->brick_count == brick_count) {
-                errstr = gf_strdup ("Deleting all the bricks of the "
-                                    "volume is not allowed");
+            snprintf(msg, sizeof(msg),
+                     "Incorrect brick "
+                     "%s for volume %s",
+                     brick, volinfo->volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INCORRECT_BRICK,
+                    "Brick=%s, Volume=%s", brick, volinfo->volname, NULL);
+            *errstr = gf_strdup(msg);
+            goto out;
+        }
+        /* Do not allow commit if the bricks are not decommissioned
+         * if its a remove brick commit
+         */
+        if (!brickinfo->decommissioned && cmd == GF_OP_CMD_COMMIT) {
+            snprintf(msg, sizeof(msg),
+                     "Brick %s "
+                     "is not decommissioned. "
+                     "Use start or force option",
+                     brick);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_NOT_DECOM,
+                    "Use 'start' or 'force' option, Brick=%s", brick, NULL);
+            *errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
+
+        if (glusterd_is_local_brick(THIS, volinfo, brickinfo)) {
+            switch (cmd) {
+                case GF_OP_CMD_START:
+                    goto check;
+                case GF_OP_CMD_NONE:
+                default:
+                    break;
+            }
+
+            switch (cmd_defrag) {
+                case GF_DEFRAG_CMD_NONE:
+                default:
+                    continue;
+            }
+        check:
+            if (brickinfo->status != GF_BRICK_STARTED) {
+                snprintf(msg, sizeof(msg),
+                         "Found stopped "
+                         "brick %s. Use force option to "
+                         "remove the offline brick",
+                         brick);
+                gf_smsg(
+                    this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_STOPPED,
+                    "Use 'force' option to remove the offline brick, Brick=%s",
+                    brick, NULL);
+                *errstr = gf_strdup(msg);
                 ret = -1;
                 goto out;
-        }
-
-        ret = -1;
-        switch (cmd) {
-        case GF_OP_CMD_NONE:
-                errstr = gf_strdup ("no remove-brick command issued");
+            }
+            GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, priv);
+            if (!gf_is_service_running(pidfile, &pid)) {
+                snprintf(msg, sizeof(msg),
+                         "Found dead "
+                         "brick %s",
+                         brick);
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_DEAD,
+                        "Brick=%s", brick, NULL);
+                *errstr = gf_strdup(msg);
+                ret = -1;
                 goto out;
-
-        case GF_OP_CMD_STATUS:
+            } else {
                 ret = 0;
-                goto out;
+            }
+            continue;
+        }
+
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find_by_uuid(brickinfo->uuid);
+        if (!peerinfo) {
+            RCU_READ_UNLOCK;
+            snprintf(msg, sizeof(msg),
+                     "Host node of the "
+                     "brick %s is not in cluster",
+                     brick);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_BRICK_HOST_NOT_FOUND, "Brick=%s", brick, NULL);
+            *errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
+        if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            snprintf(msg, sizeof(msg),
+                     "Host node of the "
+                     "brick %s is down",
+                     brick);
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_HOST_DOWN,
+                    "Brick=%s", brick, NULL);
+            *errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
+        RCU_READ_UNLOCK;
+    }
 
-        case GF_OP_CMD_DETACH_START:
-                if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                        snprintf (msg, sizeof(msg), "volume %s is not a tier "
-                                  "volume", volinfo->volname);
-                        errstr = gf_strdup (msg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_TIER, "%s", errstr);
-                        goto out;
-                }
+out:
+    return ret;
+}
 
-        case GF_OP_CMD_START:
-        {
-                if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
-                    dict_get (dict, "replica-count")) {
-                        snprintf (msg, sizeof(msg), "Migration of data is not "
-                                  "needed when reducing replica count. Use the"
-                                  " 'force' option");
-                        errstr = gf_strdup (msg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_USE_THE_FORCE, "%s", errstr);
-                        goto out;
-                }
+int
+glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr)
+{
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *errstr = NULL;
+    int32_t brick_count = 0;
+    char msg[2048] = "";
+    int32_t flag = 0;
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    gsync_status_param_t param = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = op_version_check(this, GD_OP_VER_PERSISTENT_AFR_XATTRS, msg,
+                           sizeof(msg));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH, "%s",
+               msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume %s does not exist", volname);
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &flag);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get brick command");
+        goto out;
+    }
+    cmd = flag;
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get brick count");
+        goto out;
+    }
+
+    ret = 0;
+    if (volinfo->brick_count == brick_count) {
+        errstr = gf_strdup(
+            "Deleting all the bricks of the "
+            "volume is not allowed");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DELETE, NULL);
+        ret = -1;
+        goto out;
+    }
 
-                if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                                snprintf (msg, sizeof (msg), "Volume %s needs "
-                                          "to be started before detach-tier "
-                                          "(you can use 'force' or 'commit' "
-                                          "to override this behavior)",
-                                          volinfo->volname);
-                        } else {
-                                snprintf (msg, sizeof (msg), "Volume %s needs "
-                                          "to be started before remove-brick "
-                                          "(you can use 'force' or 'commit' "
-                                          "to override this behavior)",
-                                          volinfo->volname);
-                        }
-                        errstr = gf_strdup (msg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_STARTED, "%s", errstr);
-                        goto out;
-                }
-                if (!gd_is_remove_brick_committed (volinfo)) {
-                        snprintf (msg, sizeof (msg), "An earlier remove-brick "
-                                  "task exists for volume %s. Either commit it"
-                                  " or stop it before starting a new task.",
-                                  volinfo->volname);
-                        errstr = gf_strdup (msg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OLD_REMOVE_BRICK_EXISTS, "Earlier remove-brick"
-                                " task exists for volume %s.",
-                                volinfo->volname);
-                        goto out;
-                }
-                if (glusterd_is_defrag_on(volinfo)) {
-                        errstr = gf_strdup("Rebalance is in progress. Please "
-                                           "retry after completion");
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OIP_RETRY_LATER, "%s", errstr);
-                        goto out;
-                }
+    ret = -1;
+    switch (cmd) {
+        case GF_OP_CMD_NONE:
+            errstr = gf_strdup("no remove-brick command issued");
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NO_REMOVE_CMD,
+                    NULL);
+            goto out;
 
-                /* Check if the connected clients are all of version
-                 * glusterfs-3.6 and higher. This is needed to prevent some data
-                 * loss issues that could occur when older clients are connected
-                 * when rebalance is run.
-                 */
-                ret = glusterd_check_client_op_version_support
-                        (volname, GD_OP_VERSION_3_6_0, NULL);
+        case GF_OP_CMD_STATUS:
+            ret = 0;
+            goto out;
+        case GF_OP_CMD_START: {
+            if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+                dict_getn(dict, "replica-count", SLEN("replica-count"))) {
+                snprintf(msg, sizeof(msg),
+                         "Migration of data is not "
+                         "needed when reducing replica count. Use the"
+                         " 'force' option");
+                errstr = gf_strdup(msg);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_USE_THE_FORCE, "%s",
+                       errstr);
+                goto out;
+            }
+
+            if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+                snprintf(msg, sizeof(msg),
+                         "Volume %s needs "
+                         "to be started before remove-brick "
+                         "(you can use 'force' or 'commit' "
+                         "to override this behavior)",
+                         volinfo->volname);
+                errstr = gf_strdup(msg);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED,
+                       "%s", errstr);
+                goto out;
+            }
+            if (!gd_is_remove_brick_committed(volinfo)) {
+                snprintf(msg, sizeof(msg),
+                         "An earlier remove-brick "
+                         "task exists for volume %s. Either commit it"
+                         " or stop it before starting a new task.",
+                         volinfo->volname);
+                errstr = gf_strdup(msg);
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_OLD_REMOVE_BRICK_EXISTS,
+                       "Earlier remove-brick"
+                       " task exists for volume %s.",
+                       volinfo->volname);
+                goto out;
+            }
+            if (glusterd_is_defrag_on(volinfo)) {
+                errstr = gf_strdup(
+                    "Rebalance is in progress. Please "
+                    "retry after completion");
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OIP_RETRY_LATER,
+                       "%s", errstr);
+                goto out;
+            }
+
+            /* Check if the connected clients are all of version
+             * glusterfs-3.6 and higher. This is needed to prevent some data
+             * loss issues that could occur when older clients are connected
+             * when rebalance is run.
+             */
+            ret = glusterd_check_client_op_version_support(
+                volname, GD_OP_VERSION_3_6_0, NULL);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume %s has one or "
+                                  "more connected clients of a version"
+                                  " lower than GlusterFS-v3.6.0. "
+                                  "Starting remove-brick in this state "
+                                  "could lead to data loss.\nPlease "
+                                  "disconnect those clients before "
+                                  "attempting this command again.",
+                                  volname);
+                goto out;
+            }
+
+            if (volinfo->snap_count > 0 ||
+                !cds_list_empty(&volinfo->snap_volumes)) {
+                snprintf(msg, sizeof(msg),
+                         "Volume %s  has %" PRIu64
+                         " snapshots. "
+                         "Changing the volume configuration will not effect "
+                         "snapshots."
+                         "But the snapshot brick mount should be intact to "
+                         "make them function.",
+                         volname, volinfo->snap_count);
+                gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s",
+                       msg);
+                msg[0] = '\0';
+            }
+
+            ret = glusterd_remove_brick_validate_bricks(
+                cmd, brick_count, dict, volinfo, &errstr, GF_DEFRAG_CMD_NONE);
+            if (ret)
+                goto out;
+
+            if (is_origin_glusterd(dict)) {
+                ret = glusterd_generate_and_set_task_id(
+                    dict, GF_REMOVE_BRICK_TID_KEY,
+                    SLEN(GF_REMOVE_BRICK_TID_KEY));
                 if (ret) {
-                        ret = gf_asprintf (op_errstr, "Volume %s has one or "
-                                           "more connected clients of a version"
-                                           " lower than GlusterFS-v3.6.0. "
-                                           "Starting remove-brick in this state "
-                                           "could lead to data loss.\nPlease "
-                                           "disconnect those clients before "
-                                           "attempting this command again.",
-                                           volname);
-                        goto out;
-                }
-
-                ret = glusterd_remove_brick_validate_bricks (cmd, brick_count,
-                                                             dict, volinfo,
-                                                             &errstr);
-                if (ret)
-                        goto out;
-
-                if (is_origin_glusterd (dict)) {
-                        ret = glusterd_generate_and_set_task_id
-                                (dict, GF_REMOVE_BRICK_TID_KEY);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_TASKID_GEN_FAIL,
-                                        "Failed to generate task-id");
-                                goto out;
-                        }
-                } else {
-                        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, errno,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Missing remove-brick-id");
-                                ret = 0;
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                           "Failed to generate task-id");
+                    goto out;
+                }
+            } else {
+                ret = dict_get_strn(dict, GF_REMOVE_BRICK_TID_KEY,
+                                    SLEN(GF_REMOVE_BRICK_TID_KEY),
+                                    &task_id_str);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_WARNING, errno,
+                           GD_MSG_DICT_GET_FAILED, "Missing remove-brick-id");
+                    ret = 0;
                 }
-                break;
+            }
+            break;
         }
 
         case GF_OP_CMD_STOP:
-        case GF_OP_CMD_STOP_DETACH_TIER:
-                ret = 0;
-                break;
-
-        case GF_OP_CMD_DETACH_COMMIT:
-                if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                        snprintf (msg, sizeof(msg), "volume %s is not a tier "
-                                  "volume", volinfo->volname);
-                        errstr = gf_strdup (msg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_TIER, "%s", errstr);
-                        goto out;
-                }
-                ret = glusterd_remove_brick_validate_bricks (cmd, brick_count,
-                                                             dict, volinfo,
-                                                             &errstr);
-                if (ret)
-                        goto out;
-
-                /* If geo-rep is configured, for this volume, it should be
-                 * stopped.
-                 */
-                param.volinfo = volinfo;
-                ret = glusterd_check_geo_rep_running (&param, op_errstr);
-                if (ret || param.is_active) {
-                        ret = -1;
-                        goto out;
-                }
-
-                if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_STARTED) {
-                        ret = -1;
-                        errstr = gf_strdup("Detach is in progress. Please "
-                                           "retry after completion");
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OIP_RETRY_LATER, "%s", errstr);
-                        goto out;
-                }
-                break;
+            ret = 0;
+            break;
 
         case GF_OP_CMD_COMMIT:
-                if (volinfo->decommission_in_progress) {
-                        errstr = gf_strdup ("use 'force' option as migration "
-                                            "is in progress");
-                        goto out;
-                }
-
-                if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_FAILED) {
-                        errstr = gf_strdup ("use 'force' option as migration "
-                                            "has failed");
-                        goto out;
-                }
-
-                ret = glusterd_remove_brick_validate_bricks (cmd, brick_count,
-                                                             dict, volinfo,
-                                                             &errstr);
-                if (ret)
-                        goto out;
-
-                /* If geo-rep is configured, for this volume, it should be
-                 * stopped.
-                 */
-                param.volinfo = volinfo;
-                ret = glusterd_check_geo_rep_running (&param, op_errstr);
-                if (ret || param.is_active) {
-                        ret = -1;
-                        goto out;
-                }
+            if (volinfo->decommission_in_progress) {
+                errstr = gf_strdup(
+                    "use 'force' option as migration "
+                    "is in progress");
+                gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_MIGRATION_PROG,
+                        "Use 'force' option", NULL);
+                goto out;
+            }
+
+            if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_FAILED) {
+                errstr = gf_strdup(
+                    "use 'force' option as migration "
+                    "has failed");
+                gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_MIGRATION_FAIL,
+                        "Use 'force' option", NULL);
+                goto out;
+            }
+
+            if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_COMPLETE) {
+                if (volinfo->rebal.rebalance_failures > 0 ||
+                    volinfo->rebal.skipped_files > 0) {
+                    errstr = gf_strdup(
+                        "use 'force' option as migration "
+                        "of some files might have been skipped or "
+                        "has failed");
+                    gf_smsg(this->name, GF_LOG_WARNING, 0,
+                            GD_MSG_MIGRATION_FAIL,
+                            "Use 'force' option, some files might have been "
+                            "skipped",
+                            NULL);
+                    goto out;
+                }
+            }
+
+            ret = glusterd_remove_brick_validate_bricks(
+                cmd, brick_count, dict, volinfo, &errstr, GF_DEFRAG_CMD_NONE);
+            if (ret)
+                goto out;
+
+            /* If geo-rep is configured, for this volume, it should be
+             * stopped.
+             */
+            param.volinfo = volinfo;
+            ret = glusterd_check_geo_rep_running(&param, op_errstr);
+            if (ret || param.is_active) {
+                ret = -1;
+                goto out;
+            }
 
-                break;
+            break;
 
-        case GF_OP_CMD_DETACH_COMMIT_FORCE:
-                if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                        snprintf (msg, sizeof(msg), "volume %s is not a tier "
-                                  "volume", volinfo->volname);
-                        errstr = gf_strdup (msg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_TIER, "%s", errstr);
-                        goto out;
-                }
         case GF_OP_CMD_COMMIT_FORCE:
-                break;
-        }
-        ret = 0;
+        case GF_OP_CMD_DETACH_START:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
+        case GF_OP_CMD_STOP_DETACH_TIER:
+            break;
+    }
+    ret = 0;
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        if (ret && errstr) {
-                if (op_errstr)
-                        *op_errstr = errstr;
-        }
-
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    if (ret && errstr) {
+        if (op_errstr)
+            *op_errstr = errstr;
+    }
+    if (!op_errstr && errstr)
+        GF_FREE(errstr);
+    return ret;
 }
 
 int
-glusterd_remove_brick_migrate_cbk (glusterd_volinfo_t *volinfo,
-                                   gf_defrag_status_t status)
+glusterd_remove_brick_migrate_cbk(glusterd_volinfo_t *volinfo,
+                                  gf_defrag_status_t status)
 {
-        int                   ret = 0;
+    int ret = 0;
 
-#if 0  /* TODO: enable this behavior once cluster-wide awareness comes for
-          defrag cbk function */
+#if 0 /* TODO: enable this behavior once cluster-wide awareness comes for      \
+         defrag cbk function */
         glusterd_brickinfo_t *brickinfo = NULL;
         glusterd_brickinfo_t *tmp = NULL;
 
@@ -2377,648 +2155,642 @@ glusterd_remove_brick_migrate_cbk (glusterd_volinfo_t *volinfo,
 
 #endif
 
-        volinfo->decommission_in_progress = 0;
-        return ret;
+    volinfo->decommission_in_progress = 0;
+    return ret;
 }
 
-static int
-glusterd_op_perform_attach_tier (dict_t *dict,
-                                 glusterd_volinfo_t *volinfo,
-                                 int count,
-                                 char *bricks)
+int
+glusterd_op_add_brick(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = 0;
-        int                                     replica_count = 0;
-        int                                     type = 0;
-
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char *bricks = NULL;
+    int32_t count = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get count");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bricks");
+        goto out;
+    }
+
+    ret = glusterd_op_perform_add_bricks(volinfo, count, bricks, dict);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL,
+               "Unable to add bricks");
+        goto out;
+    }
+    if (priv->op_version <= GD_OP_VERSION_3_7_5) {
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            goto out;
+    } else {
         /*
-         * Store the new (cold) tier's structure until the graph is generated.
-         * If there is a failure before the graph is generated the
-         * structure will revert to its original state.
+         * The cluster is operating at version greater than
+         * gluster-3.7.5. So no need to store volfiles
+         * in commit phase, the same will be done
+         * in post validate phase with v3 framework.
          */
-        volinfo->tier_info.cold_dist_leaf_count = volinfo->dist_leaf_count;
-        volinfo->tier_info.cold_type           = volinfo->type;
-        volinfo->tier_info.cold_brick_count    = volinfo->brick_count;
-        volinfo->tier_info.cold_replica_count  = volinfo->replica_count;
-        volinfo->tier_info.cold_disperse_count = volinfo->disperse_count;
-        volinfo->tier_info.cold_redundancy_count = volinfo->redundancy_count;
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret)
-                volinfo->tier_info.hot_replica_count  = replica_count;
-        else
-                volinfo->tier_info.hot_replica_count  = 1;
-        volinfo->tier_info.hot_brick_count     = count;
-        ret = dict_get_int32 (dict, "hot-type", &type);
-        volinfo->tier_info.hot_type      = type;
-        ret = dict_set_int32 (dict, "type", GF_CLUSTER_TYPE_TIER);
+    }
 
-        if (!ret)
-                ret = dict_set_str (volinfo->dict, "features.ctr-enabled", "on");
-
-        if (!ret)
-                ret = dict_set_str (volinfo->dict, "cluster.tier-mode", "cache");
+    if (GLUSTERD_STATUS_STARTED == volinfo->status)
+        ret = glusterd_svcs_manager(volinfo);
 
-        return ret;
+out:
+    return ret;
 }
 
 int
-glusterd_op_add_brick (dict_t *dict, char **op_errstr)
+glusterd_post_commit_add_brick(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    *bricks = NULL;
-        int32_t                                 count = 0;
-        int32_t                                 replica_count = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
+    int ret = 0;
+    char *volname = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get count");
-                goto out;
-        }
-
-
-        ret = dict_get_str (dict, "bricks", &bricks);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get bricks");
-                goto out;
-        }
-
-        if (dict_get(dict, "attach-tier")) {
-                gf_msg_debug (THIS->name, 0, "Adding tier");
-                glusterd_op_perform_attach_tier (dict, volinfo, count, bricks);
-        }
-
-        ret = glusterd_op_perform_add_bricks (volinfo, count, bricks, dict);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_ADD_FAIL, "Unable to add bricks");
-                goto out;
-        }
-        if (priv->op_version <= GD_OP_VERSION_3_7_5) {
-               ret = glusterd_store_volinfo (volinfo,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret)
-                        goto out;
-        } else {
-                 /*
-                 * The cluster is operating at version greater than
-                 * gluster-3.7.5. So no need to store volfiles
-                 * in commit phase, the same will be done
-                 * in post validate phase with v3 framework.
-                 */
-        }
-
-        if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                ret = glusterd_svcs_manager (volinfo);
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+    ret = glusterd_replace_old_auth_allow_list(volname);
 out:
-        return ret;
+    return ret;
 }
 
-static void
-glusterd_op_perform_detach_tier (glusterd_volinfo_t *volinfo)
+int
+glusterd_post_commit_replace_brick(dict_t *dict, char **op_errstr)
 {
-        volinfo->type             = volinfo->tier_info.cold_type;
-        volinfo->replica_count    = volinfo->tier_info.cold_replica_count;
-        volinfo->disperse_count   = volinfo->tier_info.cold_disperse_count;
-        volinfo->redundancy_count = volinfo->tier_info.cold_redundancy_count;
-        volinfo->dist_leaf_count  = volinfo->tier_info.cold_dist_leaf_count;
+    int ret = 0;
+    char *volname = NULL;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+    ret = glusterd_replace_old_auth_allow_list(volname);
+out:
+    return ret;
 }
 
 int
-glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
+glusterd_set_rebalance_id_for_remove_brick(dict_t *req_dict, dict_t *rsp_dict)
 {
-        int                     ret            = -1;
-        char                    *volname       = NULL;
-        glusterd_volinfo_t      *volinfo       = NULL;
-        char                    *brick         = NULL;
-        int32_t                 count          = 0;
-        int32_t                 i              = 1;
-        char                    key[256]       = {0,};
-        int32_t                 flag           = 0;
-        char                    err_str[4096]  = {0,};
-        int                     need_rebalance = 0;
-        int                     force          = 0;
-        gf1_op_commands         cmd            = 0;
-        int32_t                 replica_count  = 0;
-        glusterd_brickinfo_t    *brickinfo     = NULL;
-        glusterd_brickinfo_t    *tmp           = NULL;
-        char                    *task_id_str   = NULL;
-        xlator_t                *this          = NULL;
-        dict_t                  *bricks_dict   = NULL;
-        char                    *brick_tmpstr  = NULL;
-        int                      start_remove  = 0;
-        uint32_t                 commit_hash   = 0;
-        int                      defrag_cmd    = 0;
-        int                      detach_commit = 0;
-        void                    *tier_info     = NULL;
-        char                    *cold_shd_key  = NULL;
-        char                    *hot_shd_key   = NULL;
-        int                      delete_key    = 1;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_ADD_FAIL, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    int32_t cmd = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(req_dict);
+
+    ret = dict_get_strn(rsp_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(rsp_dict, "command", SLEN("command"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command");
+        goto out;
+    }
+
+    /* remove brick task id is generted in glusterd_op_stage_remove_brick(),
+     * but rsp_dict is unavailable there. So copying it to rsp_dict from
+     * req_dict here. */
+
+    if (is_origin_glusterd(rsp_dict)) {
+        ret = dict_get_strn(req_dict, GF_REMOVE_BRICK_TID_KEY,
+                            SLEN(GF_REMOVE_BRICK_TID_KEY), &task_id_str);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "command", &flag);
+            snprintf(msg, sizeof(msg), "Missing rebalance id for remove-brick");
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_REBALANCE_ID_MISSING,
+                   "%s", msg);
+            ret = 0;
+        } else {
+            gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+
+            ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                             rsp_dict, GF_REMOVE_BRICK_TID_KEY,
+                                             SLEN(GF_REMOVE_BRICK_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_REMOVE_BRICK_ID_SET_FAIL,
+                       "Failed to set remove-brick-id");
+                goto out;
+            }
+        }
+    }
+    if (!gf_uuid_is_null(volinfo->rebal.rebalance_id) &&
+        GD_OP_REMOVE_BRICK == volinfo->rebal.op) {
+        ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id, rsp_dict,
+                                         GF_REMOVE_BRICK_TID_KEY,
+                                         SLEN(GF_REMOVE_BRICK_TID_KEY));
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get command");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set task-id for volume %s", volname);
+            goto out;
         }
-        cmd = flag;
-
-        if ((GF_OP_CMD_START == cmd) ||
-            (GF_OP_CMD_DETACH_START == cmd))
-                start_remove = 1;
-
-        /* Set task-id, if available, in ctx dict for operations other than
-         * start
-         */
-
-        if (is_origin_glusterd (dict) && (!start_remove)) {
-                if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
-                        ret = glusterd_copy_uuid_to_dict
-                                (volinfo->rebal.rebalance_id, dict,
-                                 GF_REMOVE_BRICK_TID_KEY);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_REMOVE_BRICK_ID_SET_FAIL,
-                                        "Failed to set remove-brick-id");
-                                goto out;
-                        }
-                }
-        }
-
-        /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
-         * remove-brick */
-        if ((!start_remove) && (cmd != GF_OP_CMD_STATUS)) {
-                gf_uuid_clear (volinfo->rebal.rebalance_id);
-                volinfo->rebal.op = GD_OP_NONE;
-                dict_unref (volinfo->rebal.dict);
-                volinfo->rebal.dict = NULL;
-        }
-
-        ret = -1;
-        switch (cmd) {
+    }
+out:
+    return ret;
+}
+int
+glusterd_op_remove_brick(dict_t *dict, char **op_errstr)
+{
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *brick = NULL;
+    int32_t count = 0;
+    int32_t i = 1;
+    char key[64] = "";
+    int keylen;
+    int32_t flag = 0;
+    int need_rebalance = 0;
+    int force = 0;
+    gf1_op_commands cmd = 0;
+    int32_t replica_count = 0;
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    dict_t *bricks_dict = NULL;
+    char *brick_tmpstr = NULL;
+    int start_remove = 0;
+    uint32_t commit_hash = 0;
+    int defrag_cmd = 0;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &flag);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command");
+        goto out;
+    }
+    cmd = flag;
+
+    if (GF_OP_CMD_START == cmd)
+        start_remove = 1;
+
+    /* Set task-id, if available, in ctx dict for operations other than
+     * start
+     */
+
+    if (is_origin_glusterd(dict) && (!start_remove)) {
+        if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+            ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id, dict,
+                                             GF_REMOVE_BRICK_TID_KEY,
+                                             SLEN(GF_REMOVE_BRICK_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_REMOVE_BRICK_ID_SET_FAIL,
+                       "Failed to set remove-brick-id");
+                goto out;
+            }
+        }
+    }
+
+    /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
+     * remove-brick */
+    if ((!start_remove) && (cmd != GF_OP_CMD_STATUS)) {
+        gf_uuid_clear(volinfo->rebal.rebalance_id);
+        volinfo->rebal.op = GD_OP_NONE;
+        dict_unref(volinfo->rebal.dict);
+        volinfo->rebal.dict = NULL;
+    }
+
+    ret = -1;
+    switch (cmd) {
         case GF_OP_CMD_NONE:
-                goto out;
+            goto out;
 
         case GF_OP_CMD_STATUS:
-                ret = 0;
-                goto out;
+            ret = 0;
+            goto out;
 
         case GF_OP_CMD_STOP:
-        case GF_OP_CMD_STOP_DETACH_TIER:
-        {
-                /* Fall back to the old volume file */
-                cds_list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
-                                              brick_list) {
-                        if (!brickinfo->decommissioned)
-                                continue;
-                        brickinfo->decommissioned = 0;
-                }
-                ret = glusterd_create_volfiles_and_notify_services (volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOLFILE_CREATE_FAIL,
-                                "failed to create volfiles");
-                        goto out;
-                }
-
-                ret = glusterd_store_volinfo (volinfo,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOLINFO_SET_FAIL,
-                                "failed to store volinfo");
-                        goto out;
-                }
-
-                ret = 0;
-                goto out;
-        }
-
-        case GF_OP_CMD_DETACH_START:
         case GF_OP_CMD_START:
-                /* Reset defrag status to 'NOT STARTED' whenever a
-                 * remove-brick/rebalance command is issued to remove
-                 * stale information from previous run.
-                 */
-                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
-                ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_msg_debug (this->name, errno,
-                                "Missing remove-brick-id");
-                        ret = 0;
-                } else {
-                        gf_uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
-                        volinfo->rebal.op = GD_OP_REMOVE_BRICK;
-                }
-                force = 0;
-                break;
+            /* Reset defrag status to 'NOT STARTED' whenever a
+             * remove-brick/rebalance command is issued to remove
+             * stale information from previous run.
+             * Update defrag_cmd as well or it will only be done
+             * for nodes on which the brick to be removed exists.
+             */
+            /* coverity[MIXED_ENUMS] */
+            volinfo->rebal.defrag_cmd = cmd;
+            volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
+            ret = dict_get_strn(dict, GF_REMOVE_BRICK_TID_KEY,
+                                SLEN(GF_REMOVE_BRICK_TID_KEY), &task_id_str);
+            if (ret) {
+                gf_msg_debug(this->name, errno, "Missing remove-brick-id");
+                ret = 0;
+            } else {
+                gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_REMOVE_BRICK;
+            }
+            force = 0;
+            break;
 
         case GF_OP_CMD_COMMIT:
-                force = 1;
-                break;
-
-        case GF_OP_CMD_DETACH_COMMIT:
-        case GF_OP_CMD_DETACH_COMMIT_FORCE:
-                glusterd_op_perform_detach_tier (volinfo);
-                detach_commit = 1;
-
-                /* Disabling ctr when detaching a tier, since
-                 * currently tier is the only consumer of ctr.
-                 * Revisit this code when this constraint no
-                 * longer exist.
-                 */
-                dict_del (volinfo->dict, "features.ctr-enabled");
-                dict_del (volinfo->dict, "cluster.tier-mode");
-
-                hot_shd_key = gd_get_shd_key (volinfo->tier_info.hot_type);
-                cold_shd_key = gd_get_shd_key (volinfo->tier_info.cold_type);
-                if (hot_shd_key) {
-                        /*
-                         * Since post detach, shd graph will not contain hot
-                         * tier. So we need to clear option set for hot tier.
-                         * For a tiered volume there can be different key
-                         * for both hot and cold. If hot tier is shd compatible
-                         * then we need to remove the configured value when
-                         * detaching a tier, only if the key's are different or
-                         * cold key is NULL. So we will set delete_key first,
-                         * and if cold key is not null and they are equal then
-                         * we will clear the flag. Otherwise we will delete the
-                         * key.
-                         */
-                        if (cold_shd_key)
-                                delete_key = strcmp (hot_shd_key, cold_shd_key);
-                        if (delete_key)
-                                dict_del (volinfo->dict, hot_shd_key);
-               }
-                /* fall through */
+            force = 1;
+            break;
 
         case GF_OP_CMD_COMMIT_FORCE:
 
-                if (volinfo->decommission_in_progress) {
-                        if (volinfo->rebal.defrag) {
-                                LOCK (&volinfo->rebal.defrag->lock);
-                                /* Fake 'rebalance-complete' so the graph change
-                                   happens right away */
-                                volinfo->rebal.defrag_status =
-                                                GF_DEFRAG_STATUS_COMPLETE;
+            if (volinfo->decommission_in_progress) {
+                if (volinfo->rebal.defrag) {
+                    LOCK(&volinfo->rebal.defrag->lock);
+                    /* Fake 'rebalance-complete' so the graph change
+                       happens right away */
+                    volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_COMPLETE;
 
-                                UNLOCK (&volinfo->rebal.defrag->lock);
-                        }
-                        /* Graph change happens in rebalance _cbk function,
-                           no need to do anything here */
-                        /* TODO: '_cbk' function is not doing anything for now */
+                    UNLOCK(&volinfo->rebal.defrag->lock);
                 }
+                /* Graph change happens in rebalance _cbk function,
+                   no need to do anything here */
+                /* TODO: '_cbk' function is not doing anything for now */
+            }
 
-                ret = 0;
-                force = 1;
-                break;
+            ret = 0;
+            force = 1;
+            break;
+        case GF_OP_CMD_DETACH_START:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_STOP_DETACH_TIER:
+            break;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get count");
+        goto out;
+    }
+    /* Save the list of bricks for later usage only on starting a
+     * remove-brick. Right now this is required for displaying the task
+     * parameters with task status in volume status.
+     */
+
+    if (start_remove) {
+        bricks_dict = dict_new();
+        if (!bricks_dict) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32n(bricks_dict, "count", SLEN("count"), count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "Failed to save remove-brick count");
+            goto out;
         }
+    }
 
-        ret = dict_get_int32 (dict, "count", &count);
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_strn(dict, key, keylen, &brick);
         if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get count");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s", key);
+            goto out;
         }
 
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                count = glusterd_set_detach_bricks(dict, volinfo);
-
-        /* Save the list of bricks for later usage only on starting a
-         * remove-brick. Right now this is required for displaying the task
-         * parameters with task status in volume status.
-         */
-
         if (start_remove) {
-                bricks_dict = dict_new ();
-                if (!bricks_dict) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (bricks_dict, "count", count);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to save remove-brick count");
-                        goto out;
-                }
-        }
-
-        while ( i <= count) {
-                snprintf (key, 256, "brick%d", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_GET_FAILED, "Unable to get %s",
-                                key);
-                        goto out;
-                }
-
-                if (start_remove) {
-                        brick_tmpstr = gf_strdup (brick);
-                        if (!brick_tmpstr) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        GD_MSG_NO_MEMORY,
-                                        "Failed to duplicate brick name");
-                                goto out;
-                        }
-                        ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to add brick to dict");
-                                goto out;
-                        }
-                        brick_tmpstr = NULL;
-                }
-
-                ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
-                                                        &need_rebalance);
-                if (ret)
-                        goto out;
-                i++;
-        }
-
-        if (detach_commit) {
-                 /* Clear related information from volinfo */
-                tier_info = ((void *)(&volinfo->tier_info));
-                memset (tier_info, 0, sizeof (volinfo->tier_info));
+            brick_tmpstr = gf_strdup(brick);
+            if (!brick_tmpstr) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Failed to duplicate brick name");
+                goto out;
+            }
+            ret = dict_set_dynstrn(bricks_dict, key, keylen, brick_tmpstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                       "Failed to add brick to dict");
+                goto out;
+            }
+            brick_tmpstr = NULL;
         }
 
-        if (start_remove)
-                volinfo->rebal.dict = dict_ref (bricks_dict);
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "changing replica count %d to %d on volume %s",
-                        volinfo->replica_count, replica_count,
-                        volinfo->volname);
-                volinfo->replica_count = replica_count;
-                /* A reduction in replica count implies an arbiter volume
-                 * earlier is now no longer one. */
-                if (volinfo->arbiter_count)
-                        volinfo->arbiter_count = 0;
-                volinfo->sub_count = replica_count;
-                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
-
-                /*
-                 * volinfo->type and sub_count have already been set for
-                 * volumes undergoing a detach operation, they should not
-                 * be modified here.
-                 */
-                if ((replica_count == 1) && (cmd != GF_OP_CMD_DETACH_COMMIT) &&
-                    (cmd != GF_OP_CMD_DETACH_COMMIT_FORCE)) {
-                        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
-                                volinfo->type = GF_CLUSTER_TYPE_NONE;
-                                /* backward compatibility */
-                                volinfo->sub_count = 0;
-                        } else {
-                                volinfo->type = GF_CLUSTER_TYPE_STRIPE;
-                                /* backward compatibility */
-                                volinfo->sub_count = volinfo->dist_leaf_count;
-                        }
-                }
-        }
-        volinfo->subvol_count = (volinfo->brick_count /
-                                 volinfo->dist_leaf_count);
+        ret = glusterd_op_perform_remove_brick(volinfo, brick, force,
+                                               &need_rebalance);
+        if (ret)
+            goto out;
+        i++;
+    }
+
+    if (start_remove)
+        volinfo->rebal.dict = dict_ref(bricks_dict);
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+               "changing replica count %d to %d on volume %s",
+               volinfo->replica_count, replica_count, volinfo->volname);
+        volinfo->replica_count = replica_count;
+        /* A reduction in replica count implies an arbiter volume
+         * earlier is now no longer one. */
+        if (volinfo->arbiter_count)
+            volinfo->arbiter_count = 0;
+        volinfo->sub_count = replica_count;
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        /*
+         * volinfo->type and sub_count have already been set for
+         * volumes undergoing a detach operation, they should not
+         * be modified here.
+         */
+        if (replica_count == 1) {
+            if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+                volinfo->type = GF_CLUSTER_TYPE_NONE;
+                /* backward compatibility */
+                volinfo->sub_count = 0;
+            }
+        }
+    }
+    volinfo->subvol_count = (volinfo->brick_count / volinfo->dist_leaf_count);
+
+    if (!glusterd_is_volume_replicate(volinfo) &&
+        conf->op_version >= GD_OP_VERSION_3_12_2) {
+        ret = dict_set_nstrn(volinfo->dict, "performance.client-io-threads",
+                             SLEN("performance.client-io-threads"), "on",
+                             SLEN("on"));
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "failed to create volfiles");
-                goto out;
-        }
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "performance.client-io-threads to on");
+            goto out;
+        }
+    }
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "failed to create volfiles");
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_STORE_FAIL,
+               "failed to store volinfo");
+        goto out;
+    }
+
+    if (start_remove && volinfo->status == GLUSTERD_STATUS_STARTED) {
+        ret = glusterd_svcs_reconfigure(volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOLINFO_STORE_FAIL, "failed to store volinfo");
-                goto out;
-        }
-
-        if (start_remove &&
-            volinfo->status == GLUSTERD_STATUS_STARTED) {
-                ret = glusterd_svcs_reconfigure ();
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_NFS_RECONF_FAIL,
-                               "Unable to reconfigure NFS-Server");
-                goto out;
-                }
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_RECONF_FAIL,
+                   "Unable to reconfigure NFS-Server");
+            goto out;
         }
+    }
 
-        /* Need to reset the defrag/rebalance status accordingly */
-        switch (volinfo->rebal.defrag_status) {
+    /* Need to reset the defrag/rebalance status accordingly */
+    switch (volinfo->rebal.defrag_status) {
         case GF_DEFRAG_STATUS_FAILED:
         case GF_DEFRAG_STATUS_COMPLETE:
-                volinfo->rebal.defrag_status = 0;
+            volinfo->rebal.defrag_status = 0;
+        /* FALLTHROUGH */
         default:
-                break;
-        }
-        if (!force && need_rebalance) {
-                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
-                        volinfo->rebal.commit_hash = commit_hash;
-                }
-                /* perform the rebalance operations */
-                defrag_cmd = GF_DEFRAG_CMD_START_FORCE;
-                if (cmd == GF_OP_CMD_DETACH_START)
-                        defrag_cmd = GF_DEFRAG_CMD_START_DETACH_TIER;
-                ret = glusterd_handle_defrag_start
-                        (volinfo, err_str, sizeof (err_str),
-                         defrag_cmd,
-                         glusterd_remove_brick_migrate_cbk, GD_OP_REMOVE_BRICK);
-
-                if (!ret)
-                        volinfo->decommission_in_progress = 1;
+            break;
+    }
+    if (!force && need_rebalance) {
+        if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+            volinfo->rebal.commit_hash = commit_hash;
+        }
+        /* perform the rebalance operations */
+        defrag_cmd = GF_DEFRAG_CMD_START_FORCE;
+        /*
+         * We need to set this *before* we issue commands to the
+         * bricks, or else we might end up setting it after the bricks
+         * have responded.  If we fail to send the request(s) we'll
+         * clear it ourselves because nobody else will.
+         */
+        volinfo->decommission_in_progress = 1;
+        char err_str[4096] = "";
+        ret = glusterd_handle_defrag_start(
+            volinfo, err_str, sizeof(err_str), defrag_cmd,
+            glusterd_remove_brick_migrate_cbk, GD_OP_REMOVE_BRICK);
 
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_REBALANCE_START_FAIL,
-                                "failed to start the rebalance");
-                }
-        } else {
-                if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                        ret = glusterd_svcs_manager (volinfo);
-        }
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REBALANCE_START_FAIL,
+                   "failed to start the rebalance");
+            /* TBD: shouldn't we do more than print a message? */
+            volinfo->decommission_in_progress = 0;
+            if (op_errstr)
+                *op_errstr = gf_strdup(err_str);
+        }
+    } else {
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+            ret = glusterd_svcs_manager(volinfo);
+    }
 out:
-       if (ret && err_str[0] && op_errstr)
-                *op_errstr = gf_strdup (err_str);
-
-        GF_FREE (brick_tmpstr);
-        if (bricks_dict)
-                dict_unref (bricks_dict);
-
-        return ret;
+    GF_FREE(brick_tmpstr);
+    if (bricks_dict)
+        dict_unref(bricks_dict);
+    gf_msg_debug(this->name, 0, "returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_barrier (dict_t *dict, char **op_errstr)
+glusterd_op_stage_barrier(dict_t *dict, char **op_errstr)
 {
-        int                  ret         = -1;
-        xlator_t             *this       = NULL;
-        char                 *volname    = NULL;
-        glusterd_volinfo_t   *vol        = NULL;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Volname not present in "
-                        "dict");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &vol);
-        if (ret) {
-                gf_asprintf (op_errstr, "Volume %s does not exist", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "%s", *op_errstr);
-                goto out;
-        }
-
-        if (!glusterd_is_volume_started (vol)) {
-                gf_asprintf (op_errstr, "Volume %s is not started", volname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str_boolean (dict, "barrier", -1);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "Barrier op for volume %s not present "
-                             "in dict", volname);
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", *op_errstr);
-                goto out;
-        }
-        ret = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *barrier_op = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &vol);
+    if (ret) {
+        gf_asprintf(op_errstr, "Volume %s does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(vol)) {
+        gf_asprintf(op_errstr, "Volume %s is not started", volname);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "barrier", SLEN("barrier"), &barrier_op);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "Barrier op for volume %s not present "
+                    "in dict",
+                    volname);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               *op_errstr);
+        goto out;
+    }
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_barrier (dict_t *dict, char **op_errstr)
+glusterd_op_barrier(dict_t *dict, char **op_errstr)
 {
-        int                  ret         = -1;
-        xlator_t             *this       = NULL;
-        char                 *volname    = NULL;
-        glusterd_volinfo_t   *vol        = NULL;
-        char                 *barrier_op = NULL;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "Volname not present in "
-                        "dict");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &vol);
-        if (ret) {
-                gf_asprintf (op_errstr, "Volume %s does not exist", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "%s", *op_errstr);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "barrier", &barrier_op);
-        if (ret) {
-                gf_asprintf (op_errstr, "Barrier op for volume %s not present "
-                             "in dict", volname);
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED, "%s", *op_errstr);
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (vol->dict, "features.barrier",
-                                          barrier_op);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set barrier op in"
-                        " volume option dict");
-                goto out;
-        }
-
-        gd_update_volume_op_versions (vol);
-        ret = glusterd_create_volfiles (vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "Failed to create volfiles");
-                goto out;
-        }
-        ret = glusterd_store_volinfo (vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    int ret = -1;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *barrier_op = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &vol);
+    if (ret) {
+        gf_asprintf(op_errstr, "Volume %s does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "barrier", SLEN("barrier"), &barrier_op);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Barrier op for volume %s not present "
+                    "in dict",
+                    volname);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(vol->dict, "features.barrier", barrier_op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set barrier op in"
+               " volume option dict");
+        goto out;
+    }
+
+    gd_update_volume_op_versions(vol);
+    ret = glusterd_create_volfiles(vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfiles");
+        goto out;
+    }
+    ret = glusterd_store_volinfo(vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
+
+int
+glusterd_handle_add_tier_brick(rpcsvc_request_t *req)
+{
+    return 0;
 }
 
 int
-glusterd_handle_attach_tier (rpcsvc_request_t *req)
+glusterd_handle_attach_tier(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+    return 0;
 }
 
 int
-glusterd_handle_detach_tier (rpcsvc_request_t *req)
+glusterd_handle_detach_tier(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_remove_brick);
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-helper.c b/xlators/mgmt/glusterd/src/glusterd-conn-helper.c
index bfa9d02aa1b..a7f54ec24b7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-helper.c
@@ -15,7 +15,7 @@
 #include <urcu/rculist.h>
 
 glusterd_svc_t *
-glusterd_conn_get_svc_object (glusterd_conn_t *conn)
+glusterd_conn_get_svc_object(glusterd_conn_t *conn)
 {
-        return cds_list_entry (conn, glusterd_svc_t, conn);
+    return cds_list_entry(conn, glusterd_svc_t, conn);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-helper.h b/xlators/mgmt/glusterd/src/glusterd-conn-helper.h
index 80468d6de75..6f500309175 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-helper.h
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-helper.h
@@ -16,6 +16,6 @@
 #include "glusterd-conn-mgmt.h"
 
 glusterd_svc_t *
-glusterd_conn_get_svc_object (glusterd_conn_t *conn);
+glusterd_conn_get_svc_object(glusterd_conn_t *conn);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c
index 607a0655432..5c01f0c70b6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c
@@ -8,7 +8,7 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 #include "glusterd.h"
 #include "glusterd-conn-mgmt.h"
@@ -17,120 +17,175 @@
 #include "glusterd-messages.h"
 
 int
-glusterd_conn_init (glusterd_conn_t *conn, char *sockpath,
-                    int frame_timeout, glusterd_conn_notify_t notify)
+glusterd_conn_init(glusterd_conn_t *conn, char *sockpath, int frame_timeout,
+                   glusterd_conn_notify_t notify)
 {
-        int                  ret      = -1;
-        dict_t              *options  = NULL;
-        struct rpc_clnt     *rpc      = NULL;
-        xlator_t            *this     = THIS;
-        glusterd_svc_t      *svc      = NULL;
-
-        if (!this)
-                goto out;
-
-        svc = glusterd_conn_get_svc_object (conn);
-        if (!svc) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SVC_GET_FAIL, "Failed to get the service");
-                goto out;
-        }
-
-        ret = rpc_transport_unix_options_build (&options, sockpath,
-                                                frame_timeout);
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport.socket.ignore-enoent", "on");
-        if (ret)
-                goto out;
-
-        /* @options is free'd by rpc_transport when destroyed */
-        rpc = rpc_clnt_new (options, this, (char *)svc->name, 16);
-        if (!rpc) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = rpc_clnt_register_notify (rpc, glusterd_conn_common_notify,
-                                        conn);
-        if (ret)
-                goto out;
-
-        ret = snprintf (conn->sockpath, sizeof (conn->sockpath), "%s",
-                        sockpath);
-        if (ret < 0)
-                goto out;
-        else
-                ret = 0;
-
-        conn->frame_timeout = frame_timeout;
-        conn->rpc = rpc;
-        conn->notify = notify;
+    int ret = -1;
+    dict_t *options = NULL;
+    struct rpc_clnt *rpc = NULL;
+    xlator_t *this = THIS;
+    glusterd_svc_t *svc = NULL;
+
+    if (!this) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_XLATOR_NOT_DEFINED,
+                NULL);
+        goto out;
+    }
+
+    options = dict_new();
+    if (!options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    svc = glusterd_conn_get_svc_object(conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        goto out;
+    }
+
+    ret = rpc_transport_unix_options_build(options, sockpath, frame_timeout);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32n(options, "transport.socket.ignore-enoent",
+                          SLEN("transport.socket.ignore-enoent"), 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=transport.socket.ignore-enoent", NULL);
+        goto out;
+    }
+
+    /* @options is free'd by rpc_transport when destroyed */
+    rpc = rpc_clnt_new(options, this, (char *)svc->name, 16);
+    if (!rpc) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, glusterd_conn_common_notify, conn);
+    if (ret)
+        goto out;
+
+    ret = snprintf(conn->sockpath, sizeof(conn->sockpath), "%s", sockpath);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    } else
+        ret = 0;
+
+    conn->frame_timeout = frame_timeout;
+    conn->rpc = rpc;
+    conn->notify = notify;
 out:
-        if (ret) {
-                if (rpc) {
-                        rpc_clnt_unref (rpc);
-                        rpc = NULL;
-                }
+    if (options)
+        dict_unref(options);
+    if (ret) {
+        if (rpc) {
+            rpc_clnt_unref(rpc);
+            rpc = NULL;
         }
-        return ret;
+    }
+    return ret;
 }
 
 int
-glusterd_conn_term (glusterd_conn_t *conn)
+glusterd_conn_term(glusterd_conn_t *conn)
 {
-        rpc_clnt_unref (conn->rpc);
-        return 0;
+    rpc_clnt_unref(conn->rpc);
+    return 0;
 }
 
 int
-glusterd_conn_connect (glusterd_conn_t *conn)
+glusterd_conn_connect(glusterd_conn_t *conn)
 {
-        return rpc_clnt_start (conn->rpc);
+    return rpc_clnt_start(conn->rpc);
 }
 
 int
-glusterd_conn_disconnect (glusterd_conn_t *conn)
+glusterd_conn_disconnect(glusterd_conn_t *conn)
 {
-        rpc_clnt_disconnect (conn->rpc);
+    rpc_clnt_disable(conn->rpc);
 
-        return 0;
+    return 0;
 }
 
-
 int
-__glusterd_conn_common_notify (struct rpc_clnt *rpc, void *mydata,
-                               rpc_clnt_event_t event, void *data)
+__glusterd_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                              rpc_clnt_event_t event, void *data)
 {
-        glusterd_conn_t *conn = mydata;
+    glusterd_conn_t *conn = mydata;
 
-        /* Silently ignoring this error, exactly like the current
-         * implementation */
-        if (!conn)
-                return 0;
+    /* Silently ignoring this error, exactly like the current
+     * implementation */
+    if (!conn)
+        return 0;
 
-        return conn->notify (conn, event);
+    return conn->notify(conn, event);
 }
 
 int
-glusterd_conn_common_notify (struct rpc_clnt *rpc, void *mydata,
-                             rpc_clnt_event_t event, void *data)
+glusterd_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify
-                                (rpc, mydata, event, data,
-                                 __glusterd_conn_common_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_conn_common_notify);
 }
 
 int32_t
-glusterd_conn_build_socket_filepath (char *rundir, uuid_t uuid,
-                                     char *socketpath, int len)
+glusterd_conn_build_socket_filepath(char *rundir, uuid_t uuid, char *socketpath,
+                                    int len)
 {
-        char sockfilepath[PATH_MAX] = {0,};
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+
+    snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+             uuid_utoa(uuid));
 
-        snprintf (sockfilepath, sizeof (sockfilepath), "%s/run-%s",
-                  rundir, uuid_utoa (uuid));
+    glusterd_set_socket_filepath(sockfilepath, socketpath, len);
+    return 0;
+}
+
+int
+__glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                                     rpc_clnt_event_t event, void *data)
+{
+    glusterd_conf_t *conf = THIS->private;
+    glusterd_svc_proc_t *mux_proc = mydata;
+    int ret = -1;
 
-        glusterd_set_socket_filepath (sockfilepath, socketpath, len);
+    /* Silently ignoring this error, exactly like the current
+     * implementation */
+    if (!mux_proc)
         return 0;
+
+    if (event == RPC_CLNT_DESTROY) {
+        /*RPC_CLNT_DESTROY will only called after mux_proc detached from the
+         * list. So it is safe to call without lock. Processing
+         * RPC_CLNT_DESTROY under a lock will lead to deadlock.
+         */
+        if (mux_proc->data) {
+            glusterd_volinfo_unref(mux_proc->data);
+            mux_proc->data = NULL;
+        }
+        GF_FREE(mux_proc);
+        ret = 0;
+    } else {
+        pthread_mutex_lock(&conf->attach_lock);
+        {
+            ret = mux_proc->notify(mux_proc, event);
+        }
+        pthread_mutex_unlock(&conf->attach_lock);
+    }
+    return ret;
+}
+
+int
+glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                                   rpc_clnt_event_t event, void *data)
+{
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_muxsvc_conn_common_notify);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h
index 5820419dbf5..1b225621ab1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h
@@ -15,37 +15,39 @@
 
 typedef struct glusterd_conn_ glusterd_conn_t;
 
-typedef int (*glusterd_conn_notify_t)
-                (glusterd_conn_t *conn, rpc_clnt_event_t event);
+typedef int (*glusterd_conn_notify_t)(glusterd_conn_t *conn,
+                                      rpc_clnt_event_t event);
 
 struct glusterd_conn_ {
-        struct rpc_clnt *rpc;
-        char sockpath[PATH_MAX];
-        int frame_timeout;
-        /* Existing daemons tend to specialize their respective
-         * notify implementations, so ... */
-        glusterd_conn_notify_t notify;
+    struct rpc_clnt *rpc;
+    /* Existing daemons tend to specialize their respective
+     * notify implementations, so ... */
+    glusterd_conn_notify_t notify;
+    int frame_timeout;
+    char sockpath[PATH_MAX];
 };
 
 int
-glusterd_conn_init (glusterd_conn_t *conn, char *sockpath,
-                    int frame_timeout, glusterd_conn_notify_t notify);
+glusterd_conn_init(glusterd_conn_t *conn, char *sockpath, int frame_timeout,
+                   glusterd_conn_notify_t notify);
 
 int
-glusterd_conn_term (glusterd_conn_t *conn);
+glusterd_conn_term(glusterd_conn_t *conn);
 
 int
-glusterd_conn_connect (glusterd_conn_t *conn);
+glusterd_conn_connect(glusterd_conn_t *conn);
 
 int
-glusterd_conn_disconnect (glusterd_conn_t *conn);
+glusterd_conn_disconnect(glusterd_conn_t *conn);
 
 int
-glusterd_conn_common_notify (struct rpc_clnt *rpc, void *mydata,
-                             rpc_clnt_event_t event, void *data);
+glusterd_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data);
+int
+glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                                   rpc_clnt_event_t event, void *data);
 
 int32_t
-glusterd_conn_build_socket_filepath (char *rundir, uuid_t uuid,
-                                     char *socketpath, int len);
-
+glusterd_conn_build_socket_filepath(char *rundir, uuid_t uuid, char *socketpath,
+                                    int len);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-errno.h b/xlators/mgmt/glusterd/src/glusterd-errno.h
index 55d44a5c6a1..c74070e0e8d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-errno.h
+++ b/xlators/mgmt/glusterd/src/glusterd-errno.h
@@ -11,22 +11,23 @@
 #define _GLUSTERD_ERRNO_H
 
 enum glusterd_op_errno {
-        EG_INTRNL      = 30800,          /* Internal Error                    */
-        EG_OPNOTSUP    = 30801,          /* Gluster Op Not Supported          */
-        EG_ANOTRANS    = 30802,          /* Another Transaction in Progress   */
-        EG_BRCKDWN     = 30803,          /* One or more brick is down         */
-        EG_NODEDWN     = 30804,          /* One or more node is down          */
-        EG_HRDLMT      = 30805,          /* Hard Limit is reached             */
-        EG_NOVOL       = 30806,          /* Volume does not exist             */
-        EG_NOSNAP      = 30807,          /* Snap does not exist               */
-        EG_RBALRUN     = 30808,          /* Rebalance is running              */
-        EG_VOLRUN      = 30809,          /* Volume is running                 */
-        EG_VOLSTP      = 30810,          /* Volume is not running             */
-        EG_VOLEXST     = 30811,          /* Volume exists                     */
-        EG_SNAPEXST    = 30812,          /* Snapshot exists                   */
-        EG_ISSNAP      = 30813,          /* Volume is a snap volume           */
-        EG_GEOREPRUN   = 30814,          /* Geo-Replication is running        */
-        EG_NOTTHINP    = 30815,          /* Bricks are not thinly provisioned */
+    EG_INTRNL = 30800,    /* Internal Error                    */
+    EG_OPNOTSUP = 30801,  /* Gluster Op Not Supported          */
+    EG_ANOTRANS = 30802,  /* Another Transaction in Progress   */
+    EG_BRCKDWN = 30803,   /* One or more brick is down         */
+    EG_NODEDWN = 30804,   /* One or more node is down          */
+    EG_HRDLMT = 30805,    /* Hard Limit is reached             */
+    EG_NOVOL = 30806,     /* Volume does not exist             */
+    EG_NOSNAP = 30807,    /* Snap does not exist               */
+    EG_RBALRUN = 30808,   /* Rebalance is running              */
+    EG_VOLRUN = 30809,    /* Volume is running                 */
+    EG_VOLSTP = 30810,    /* Volume is not running             */
+    EG_VOLEXST = 30811,   /* Volume exists                     */
+    EG_SNAPEXST = 30812,  /* Snapshot exists                   */
+    EG_ISSNAP = 30813,    /* Volume is a snap volume           */
+    EG_GEOREPRUN = 30814, /* Geo-Replication is running        */
+    EG_NOTTHINP = 30815,  /* Bricks are not thinly provisioned */
+    EG_NOGANESHA = 30816, /* Global ganesha is not enabled   */
 };
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
index 8d26216d908..f08bd6cebee 100644
--- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c
+++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
@@ -8,852 +8,920 @@
    cases as published by the Free Software Foundation.
 */
 
-
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
 #include <ctype.h>
 
-#define SHARED_STORAGE_MNT "/var/run/gluster/shared_storage/nfs-ganesha"
-
-int start_ganesha (char **op_errstr);
-
+int
+start_ganesha(char **op_errstr);
 
 typedef struct service_command {
-        char *binary;
-        char *service;
-        int (*action) (struct service_command *, char *);
+    char *binary;
+    char *service;
+    int (*action)(struct service_command *, char *);
 } service_command;
 
 /* parsing_ganesha_ha_conf will allocate the returned string
  * to be freed (GF_FREE) by the caller
  * return NULL if error or not found */
-static char*
-parsing_ganesha_ha_conf(const char *key) {
+static char *
+parsing_ganesha_ha_conf(const char *key)
+{
 #define MAX_LINE 1024
-        char scratch[MAX_LINE * 2] = {0,};
-        char *value = NULL, *pointer = NULL, *end_pointer = NULL;
-        FILE *fp;
-        struct stat st = {0,};
-
-        fp = fopen (GANESHA_HA_CONF, "r");
-        if (fp == NULL) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED, "couldn't open the file %s",
-                        GANESHA_HA_CONF);
-                goto end_ret;
-        }
-        while ((pointer = fgets (scratch, MAX_LINE, fp)) != NULL) {
-                /* Read config file until we get matching "^[[:space:]]*key" */
-                if (*pointer == '#') {
-                        continue;
-                }
-                while (isblank(*pointer)) {
-                        pointer++;
-                }
-                if (strncmp (pointer, key, strlen (key))) {
-                        continue;
-                }
-                pointer += strlen (key);
-                /* key found : if we fail to parse, we'll return an error
-                 * rather than trying next one
-                 * - supposition : conf file is bash compatible : no space
-                 *   around the '=' */
-                if (*pointer != '=') {
-                        gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                                GD_MSG_GET_CONFIG_INFO_FAILED,
-                                "Parsing %s failed at key %s",
-                                GANESHA_HA_CONF, key);
-                        goto end_close;
-                }
-                pointer++;  /* jump the '=' */
-
-                if (*pointer == '"' || *pointer == '\'') {
-                        /* dont get the quote */
-                        pointer++;
-                }
-                end_pointer = pointer;
-                /* stop at the next closing quote or  blank/newline */
-                do {
-                        end_pointer++;
-                } while (!(*end_pointer == '\'' || *end_pointer == '"' ||
-                                isspace(*end_pointer) || *end_pointer == '\0'));
-                *end_pointer = '\0';
-
-                /* got it. copy it and return */
-                value = gf_strdup (pointer);
-                break;
-        }
+    char scratch[MAX_LINE * 2] = {
+        0,
+    };
+    char *value = NULL, *pointer = NULL, *end_pointer = NULL;
+    FILE *fp;
+
+    fp = fopen(GANESHA_HA_CONF, "r");
+    if (fp == NULL) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "couldn't open the file %s", GANESHA_HA_CONF);
+        goto end_ret;
+    }
+    while ((pointer = fgets(scratch, MAX_LINE, fp)) != NULL) {
+        /* Read config file until we get matching "^[[:space:]]*key" */
+        if (*pointer == '#') {
+            continue;
+        }
+        while (isblank(*pointer)) {
+            pointer++;
+        }
+        if (strncmp(pointer, key, strlen(key))) {
+            continue;
+        }
+        pointer += strlen(key);
+        /* key found : if we fail to parse, we'll return an error
+         * rather than trying next one
+         * - supposition : conf file is bash compatible : no space
+         *   around the '=' */
+        if (*pointer != '=') {
+            gf_msg(THIS->name, GF_LOG_ERROR, errno,
+                   GD_MSG_GET_CONFIG_INFO_FAILED, "Parsing %s failed at key %s",
+                   GANESHA_HA_CONF, key);
+            goto end_close;
+        }
+        pointer++; /* jump the '=' */
+
+        if (*pointer == '"' || *pointer == '\'') {
+            /* dont get the quote */
+            pointer++;
+        }
+        end_pointer = pointer;
+        /* stop at the next closing quote or  blank/newline */
+        do {
+            end_pointer++;
+        } while (!(*end_pointer == '\'' || *end_pointer == '"' ||
+                   isspace(*end_pointer) || *end_pointer == '\0'));
+        *end_pointer = '\0';
+
+        /* got it. copy it and return */
+        value = gf_strdup(pointer);
+        break;
+    }
 
 end_close:
-        fclose(fp);
+    fclose(fp);
 end_ret:
-        return value;
+    return value;
 }
 
 static int
-sc_systemctl_action (struct service_command *sc, char *command)
+sc_systemctl_action(struct service_command *sc, char *command)
 {
-        runner_t        runner   = {0,};
+    runner_t runner = {
+        0,
+    };
 
-        runinit (&runner);
-        runner_add_args (&runner, sc->binary, command, sc->service, NULL);
-        return runner_run (&runner);
+    runinit(&runner);
+    runner_add_args(&runner, sc->binary, command, sc->service, NULL);
+    return runner_run(&runner);
 }
 
 static int
-sc_service_action (struct service_command *sc, char *command)
+sc_service_action(struct service_command *sc, char *command)
 {
-        runner_t      runner = {0,};
+    runner_t runner = {
+        0,
+    };
 
-        runinit (&runner);
-        runner_add_args (&runner, sc->binary, sc->service, command, NULL);
-        return runner_run (&runner);
+    runinit(&runner);
+    runner_add_args(&runner, sc->binary, sc->service, command, NULL);
+    return runner_run(&runner);
 }
 
 static int
-manage_service (char *action)
+manage_service(char *action)
 {
-        struct stat stbuf       = {0,};
-        int     i               = 0;
-        int     ret             = 0;
-        struct service_command sc_list[] = {
-                { .binary  = "/usr/bin/systemctl",
-                  .service = "nfs-ganesha",
-                  .action  = sc_systemctl_action
-                },
-                { .binary  = "/sbin/invoke-rc.d",
-                  .service = "nfs-ganesha",
-                  .action  = sc_service_action
-                },
-                { .binary  = "/sbin/service",
-                  .service = "nfs-ganesha",
-                  .action  = sc_service_action
-                },
-                { .binary  = NULL
-                }
-        };
-
-        while (sc_list[i].binary != NULL) {
-                ret = sys_stat (sc_list[i].binary, &stbuf);
-                if (ret == 0) {
-                        gf_msg_debug (THIS->name, 0,
-                                "%s found.", sc_list[i].binary);
-                        if (strcmp (sc_list[i].binary, "/usr/bin/systemctl") == 0)
-                                ret = sc_systemctl_action (&sc_list[i], action);
-                        else
-                                ret = sc_service_action (&sc_list[i], action);
-
-                        return ret;
-                }
-                i++;
-        }
-        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                GD_MSG_UNRECOGNIZED_SVC_MNGR,
-                "Could not %s NFS-Ganesha.Service manager for distro"
-                " not recognized.", action);
-        return ret;
+    int i = 0;
+    int ret = 0;
+    struct service_command sc_list[] = {{.binary = "/bin/systemctl",
+                                         .service = "nfs-ganesha",
+                                         .action = sc_systemctl_action},
+                                        {.binary = "/sbin/invoke-rc.d",
+                                         .service = "nfs-ganesha",
+                                         .action = sc_service_action},
+                                        {.binary = "/sbin/service",
+                                         .service = "nfs-ganesha",
+                                         .action = sc_service_action},
+                                        {.binary = NULL}};
+
+    while (sc_list[i].binary != NULL) {
+        ret = sys_access(sc_list[i].binary, X_OK);
+        if (ret == 0) {
+            gf_msg_debug(THIS->name, 0, "%s found.", sc_list[i].binary);
+            return sc_list[i].action(&sc_list[i], action);
+        }
+        i++;
+    }
+    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNRECOGNIZED_SVC_MNGR,
+           "Could not %s NFS-Ganesha.Service manager for distro"
+           " not recognized.",
+           action);
+    return ret;
+}
+
+/*
+ * Check if the cluster is a ganesha cluster or not *
+ */
+gf_boolean_t
+glusterd_is_ganesha_cluster()
+{
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t ret_bool = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("ganesha", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
+                               _gf_false);
+    if (ret == _gf_true) {
+        ret_bool = _gf_true;
+        gf_msg_debug(this->name, 0, "nfs-ganesha is enabled for the cluster");
+    } else
+        gf_msg_debug(this->name, 0, "nfs-ganesha is disabled for the cluster");
+
+out:
+    return ret_bool;
 }
+
 /* Check if ganesha.enable is set to 'on', that checks if
  * a  particular volume is exported via NFS-Ganesha */
 gf_boolean_t
-glusterd_check_ganesha_export (glusterd_volinfo_t *volinfo) {
-
-        char *value              = NULL;
-        gf_boolean_t is_exported = _gf_false;
-        int ret                 = 0;
-
-        ret = glusterd_volinfo_get (volinfo, "ganesha.enable", &value);
-        if ((ret == 0) && value) {
-                if (strcmp (value, "on") == 0) {
-                        gf_msg_debug (THIS->name, 0, "ganesha.enable set"
-                                " to %s", value);
-                        is_exported = _gf_true;
-                }
-        }
-        return is_exported;
+glusterd_check_ganesha_export(glusterd_volinfo_t *volinfo)
+{
+    char *value = NULL;
+    gf_boolean_t is_exported = _gf_false;
+    int ret = 0;
+
+    ret = glusterd_volinfo_get(volinfo, "ganesha.enable", &value);
+    if ((ret == 0) && value) {
+        if (strcmp(value, "on") == 0) {
+            gf_msg_debug(THIS->name, 0,
+                         "ganesha.enable set"
+                         " to %s",
+                         value);
+            is_exported = _gf_true;
+        }
+    }
+    return is_exported;
 }
 
-
+/* *
+ * The below function is called as part of commit phase for volume set option
+ * "ganesha.enable". If the value is "on", it creates export configuration file
+ * and then export the volume via dbus command. Incase of "off", the volume
+ * will be already unexported during stage phase, so it will remove the conf
+ * file from shared storage
+ */
 int
-glusterd_check_ganesha_cmd (char *key, char *value, char **errstr, dict_t *dict)
+glusterd_check_ganesha_cmd(char *key, char *value, char **errstr, dict_t *dict)
 {
-        int                ret = 0;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
-
-        if ((strcmp (key, "ganesha.enable") == 0)) {
-                if ((strcmp (value, "on")) && (strcmp (value, "off"))) {
-                        gf_asprintf (errstr, "Invalid value"
-                                " for volume set command. Use on/off only.");
-                        ret = -1;
-                        goto out;
-                }
-                ret = glusterd_handle_ganesha_op (dict, errstr, key, value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_NFS_GNS_OP_HANDLE_FAIL,
-                                "Handling NFS-Ganesha"
-                                " op failed.");
-                }
+    int ret = 0;
+    char *volname = NULL;
+
+    GF_ASSERT(key);
+    GF_ASSERT(value);
+    GF_ASSERT(dict);
+
+    if ((strcmp(key, "ganesha.enable") == 0)) {
+        if ((strcmp(value, "on")) && (strcmp(value, "off"))) {
+            gf_asprintf(errstr,
+                        "Invalid value"
+                        " for volume set command. Use on/off only.");
+            ret = -1;
+            goto out;
+        }
+        if (strcmp(value, "on") == 0) {
+            ret = glusterd_handle_ganesha_op(dict, errstr, key, value);
+
+        } else if (is_origin_glusterd(dict)) {
+            ret = dict_get_str(dict, "volname", &volname);
+            if (ret) {
+                gf_msg("glusterd-ganesha", GF_LOG_ERROR, errno,
+                       GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
+                goto out;
+            }
+            ret = manage_export_config(volname, "off", errstr);
         }
+    }
 out:
-        return ret;
+    if (ret) {
+        gf_msg("glusterd-ganesha", GF_LOG_ERROR, 0,
+               GD_MSG_NFS_GNS_OP_HANDLE_FAIL,
+               "Handling NFS-Ganesha"
+               " op failed.");
+    }
+    return ret;
 }
 
 int
-glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr)
+glusterd_op_stage_set_ganesha(dict_t *dict, char **op_errstr)
 {
-        int                             ret                     = -1;
-        char                            *volname                = NULL;
-        int                             exists                  = 0;
-        int                             value                   = -1;
-        gf_boolean_t                    option                  = _gf_false;
-        char                            *str                    = NULL;
-        int                             dict_count              = 0;
-        int                             flags                   = 0;
-        glusterd_volinfo_t              *volinfo                = NULL;
-        glusterd_conf_t                 *priv                   = NULL;
-        xlator_t                        *this                   = NULL;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        value = dict_get_str_boolean (dict, "value", _gf_false);
-        if (value == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "value not present.");
-                goto out;
-        }
-        /* This dict_get will fail if the user had never set the key before */
-        /*Ignoring the ret value and proceeding */
-        ret = dict_get_str (priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, &str);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        GD_MSG_DICT_GET_FAILED, "Global dict not present.");
-                ret = 0;
-                goto out;
-        }
-        /* Validity of the value is already checked */
-        ret = gf_string2boolean (str, &option);
-        /* Check if the feature is already enabled, fail in that case */
-        if (value == option) {
-                gf_asprintf (op_errstr, "nfs-ganesha is already %sd.", str);
-                ret = -1;
-                goto out;
-        }
-
-        if (value) {
-                ret =  start_ganesha (op_errstr);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_NFS_GNS_START_FAIL,
-                                "Could not start NFS-Ganesha");
-
-                }
+    int ret = -1;
+    char *value = NULL;
+    char *str = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "value", &value);
+    if (value == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "value not present.");
+        goto out;
+    }
+    /* This dict_get will fail if the user had never set the key before */
+    /*Ignoring the ret value and proceeding */
+    ret = dict_get_str(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, &str);
+    if (str ? strcmp(value, str) == 0 : strcmp(value, "disable") == 0) {
+        gf_asprintf(op_errstr, "nfs-ganesha is already %sd.", value);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(value, "enable") == 0) {
+        ret = start_ganesha(op_errstr);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_NFS_GNS_START_FAIL,
+                   "Could not start NFS-Ganesha");
         }
+    } else {
+        ret = stop_ganesha(op_errstr);
+        if (ret)
+            gf_msg_debug(THIS->name, 0,
+                         "Could not stop "
+                         "NFS-Ganesha.");
+    }
 
 out:
 
-        if (ret) {
-                if (!(*op_errstr)) {
-                        *op_errstr = gf_strdup ("Error, Validation Failed");
-                        gf_msg_debug (this->name, 0,
-                                "Error, Cannot Validate option :%s",
-                                GLUSTERD_STORE_KEY_GANESHA_GLOBAL);
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                "Error, Cannot Validate option");
-                }
+    if (ret) {
+        if (!(*op_errstr)) {
+            *op_errstr = gf_strdup("Error, Validation Failed");
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option :%s",
+                         GLUSTERD_STORE_KEY_GANESHA_GLOBAL);
+        } else {
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option");
         }
-        return ret;
+    }
+    return ret;
 }
 
 int
-glusterd_op_set_ganesha (dict_t *dict, char **errstr)
+glusterd_op_set_ganesha(dict_t *dict, char **errstr)
 {
-        int                                      ret = 0;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        char                                    *key = NULL;
-        char                                    *value = NULL;
-        dict_t                                  *vol_opts = NULL;
-        char                                    *next_version =  NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Couldn't get key in global option set");
-                goto out;
-       }
-
-        ret = dict_get_str (dict, "value", &value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Couldn't get value in global option set");
-                goto out;
-        }
-
-        ret = glusterd_handle_ganesha_op (dict, errstr, key, value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NFS_GNS_SETUP_FAIL,
-                        "Initial NFS-Ganesha set up failed");
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_dynstr_with_alloc (priv->opts,
-                                   GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
-                                   value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set"
-                        " nfs-ganesha in dict.");
-                goto out;
-        }
-        ret = glusterd_get_next_global_opt_version_str (priv->opts,
-                                                        &next_version);
-        if (ret) {
-                gf_msg_debug (THIS->name, 0, "Could not fetch "
-                        " global op version");
-                goto out;
-        }
-        ret = dict_set_str (priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                            next_version);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_options (this, priv->opts);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_FAIL, "Failed to store options");
-                goto out;
-        }
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *next_version = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "key", &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get key in global option set");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "value", &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get value in global option set");
+        goto out;
+    }
+
+    ret = glusterd_handle_ganesha_op(dict, errstr, key, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NFS_GNS_SETUP_FAIL,
+               "Initial NFS-Ganesha set up failed");
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_dynstr_with_alloc(priv->opts,
+                                     GLUSTERD_STORE_KEY_GANESHA_GLOBAL, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set"
+               " nfs-ganesha in dict.");
+        goto out;
+    }
+    ret = glusterd_get_next_global_opt_version_str(priv->opts, &next_version);
+    if (ret) {
+        gf_msg_debug(THIS->name, 0,
+                     "Could not fetch "
+                     " global op version");
+        goto out;
+    }
+    ret = dict_set_str(priv->opts, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_options(this, priv->opts);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_FAIL,
+               "Failed to store options");
+        goto out;
+    }
 
 out:
-       gf_msg_debug (this->name, 0, "returning %d", ret);
-       return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
-/* Following 2 functions parse GANESHA_HA_CONF
+/* Following function parse GANESHA_HA_CONF
  * The sample file looks like below,
  * HA_NAME="ganesha-ha-360"
  * HA_VOL_NAME="ha-state"
- * HA_VOL_MNT="/mount-point"
- * HA_VOL_SERVER="server1"
  * HA_CLUSTER_NODES="server1,server2"
  * VIP_rhs_1="10.x.x.x"
  * VIP_rhs_2="10.x.x.x." */
 
-gf_boolean_t
-is_ganesha_host (void)
-{
-        char    *host_from_file          = NULL;
-        gf_boolean_t    ret              = _gf_false;
-        xlator_t        *this            = NULL;
-
-        this = THIS;
-
-        host_from_file = parsing_ganesha_ha_conf ("HA_VOL_SERVER");
-        if (host_from_file == NULL) {
-                gf_msg (this->name, GF_LOG_INFO, errno,
-                        GD_MSG_GET_CONFIG_INFO_FAILED,
-                        "couldn't get HA_VOL_SERVER from file %s",
-                        GANESHA_HA_CONF);
-                return _gf_false;
-        }
-
-        ret = gf_is_local_addr (host_from_file);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_NFS_GNS_HOST_FOUND,
-                        "ganesha host found "
-                        "Hostname is %s", host_from_file);
-        }
-
-        GF_FREE (host_from_file);
-        return ret;
-}
-
 /* Check if the localhost is listed as one of nfs-ganesha nodes */
 gf_boolean_t
-check_host_list (void)
+check_host_list(void)
 {
-
-        glusterd_conf_t     *priv        = NULL;
-        char    *hostname, *hostlist;
-        int     ret                      = _gf_false;
-        xlator_t        *this            = NULL;
-
-        this = THIS;
-        priv =  THIS->private;
-        GF_ASSERT (priv);
-
-        hostlist = parsing_ganesha_ha_conf ("HA_CLUSTER_NODES");
-        if (hostlist == NULL) {
-                gf_msg (this->name, GF_LOG_INFO, errno,
-                        GD_MSG_GET_CONFIG_INFO_FAILED,
-                        "couldn't get HA_CLUSTER_NODES from file %s",
-                        GANESHA_HA_CONF);
-                return _gf_false;
-        }
-
-        /* Hostlist is a comma separated list now */
-        hostname = strtok (hostlist, ",");
-        while (hostname != NULL) {
-                ret = gf_is_local_addr (hostname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_NFS_GNS_HOST_FOUND,
-                                "ganesha host found "
-                                "Hostname is %s", hostname);
-                        break;
-                }
-                hostname = strtok (NULL, ",");
+    glusterd_conf_t *priv = NULL;
+    char *hostname, *hostlist;
+    gf_boolean_t ret = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    hostlist = parsing_ganesha_ha_conf("HA_CLUSTER_NODES");
+    if (hostlist == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "couldn't get HA_CLUSTER_NODES from file %s", GANESHA_HA_CONF);
+        return _gf_false;
+    }
+
+    /* Hostlist is a comma separated list now */
+    hostname = strtok(hostlist, ",");
+    while (hostname != NULL) {
+        ret = gf_is_local_addr(hostname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NFS_GNS_HOST_FOUND,
+                   "ganesha host found "
+                   "Hostname is %s",
+                   hostname);
+            break;
         }
+        hostname = strtok(NULL, ",");
+    }
 
-        GF_FREE (hostlist);
-        return ret;
-
+    GF_FREE(hostlist);
+    return ret;
 }
 
 int
-create_export_config (char *volname, char **op_errstr)
+gd_ganesha_send_dbus(char *volname, char *value)
 {
-        runner_t                runner                     = {0,};
-        int                     ret                        = -1;
-
-        GF_ASSERT(volname);
-        runinit (&runner);
-        runner_add_args (&runner, "sh",
-                        GANESHA_PREFIX"/create-export-ganesha.sh",
-                        CONFDIR, volname, NULL);
-        ret = runner_run(&runner);
-
-        if (ret && op_errstr)
-                gf_asprintf (op_errstr, "Failed to create"
-                            " NFS-Ganesha export config file.");
-
-        return ret;
+    runner_t runner = {
+        0,
+    };
+    int ret = -1;
+    runinit(&runner);
+
+    GF_VALIDATE_OR_GOTO("glusterd-ganesha", volname, out);
+    GF_VALIDATE_OR_GOTO("glusterd-ganesha", value, out);
+
+    ret = 0;
+    if (check_host_list()) {
+        /* Check whether ganesha is running on this node */
+        if (manage_service("status")) {
+            gf_msg("glusterd-ganesha", GF_LOG_WARNING, 0,
+                   GD_MSG_GANESHA_NOT_RUNNING,
+                   "Export failed, NFS-Ganesha is not running");
+        } else {
+            runner_add_args(&runner, GANESHA_PREFIX "/dbus-send.sh", CONFDIR,
+                            value, volname, NULL);
+            ret = runner_run(&runner);
+        }
+    }
+out:
+    return ret;
 }
 
 int
-copy_export_config (char *volname, char **op_errstr)
+manage_export_config(char *volname, char *value, char **op_errstr)
 {
-        runner_t                runner                     = {0,};
-        int                     ret                        = -1;
-
-        GF_ASSERT(volname);
-        runinit (&runner);
-        runner_add_args (&runner, "sh",
-                        GANESHA_PREFIX"/copy-export-ganesha.sh",
-                        CONFDIR, volname, NULL);
-        ret = runner_run(&runner);
-
-        if (ret && op_errstr)
-                gf_asprintf (op_errstr, "Failed to copy"
-                            " NFS-Ganesha export config file.");
-
-        return ret;
+    runner_t runner = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(volname);
+    runinit(&runner);
+    runner_add_args(&runner, GANESHA_PREFIX "/create-export-ganesha.sh",
+                    CONFDIR, value, volname, NULL);
+    ret = runner_run(&runner);
+
+    if (ret && op_errstr)
+        gf_asprintf(op_errstr,
+                    "Failed to create"
+                    " NFS-Ganesha export config file.");
+
+    return ret;
 }
+
 /* Exports and unexports a particular volume via NFS-Ganesha */
 int
-ganesha_manage_export (char *volname, char *value, char **op_errstr,
-                       gf_boolean_t reboot)
+ganesha_manage_export(dict_t *dict, char *value,
+                      gf_boolean_t update_cache_invalidation, char **op_errstr)
 {
-        runner_t                 runner = {0,};
-        int                      ret = -1;
-        char                     str[1024];
-        glusterd_volinfo_t      *volinfo = NULL;
-        dict_t                  *vol_opts = NULL;
-        xlator_t                *this    = NULL;
-        glusterd_conf_t         *priv    = NULL;
-        gf_boolean_t             option  = _gf_false;
-        int                     i = 1;
-
-        runinit (&runner);
-        this =  THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (value);
-        GF_ASSERT (priv);
-        GF_VALIDATE_OR_GOTO (this->name, volname, out);
-
-
-        ret = gf_string2boolean (value, &option);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "invalid value.");
-                goto out;
-        }
-
-        /* *
-         * Incase of reboot, following checks are already made before calling
-         * ganesha_manage_export. So it will be reductant do it again
-         */
-        if (!reboot) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_FOUND,
-                                FMTSTR_CHECK_VOL_EXISTS, volname);
-                        goto out;
-                }
-
-                ret = glusterd_check_ganesha_export (volinfo);
-                if (ret && option) {
-                        if (op_errstr)
-                                gf_asprintf (op_errstr, "ganesha.enable "
-                                                     "is already 'on'.");
-                        ret = -1;
-                        goto out;
-
-                }  else if (!option && !ret) {
-                        if (op_errstr)
-                                gf_asprintf (op_errstr, "ganesha.enable "
-                                                    "is already 'off'.");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        ret = 0;
-
-        /* *
-         * Incase of restart, there is chance that global option turned off
-         * with volume set command. Still we may need to clean up the
-         * configuration files.
-         * Otherwise check if global option is enabled, only then proceed
-         * */
-        if (!(reboot && !option)) {
-                ret = dict_get_str_boolean (priv->opts,
-                            GLUSTERD_STORE_KEY_GANESHA_GLOBAL, _gf_false);
-                if (ret == -1) {
-                        gf_msg_debug (this->name, 0, "Failed to get "
-                                                "global option dict.");
-                        if (op_errstr)
-                                gf_asprintf (op_errstr, "The option "
-                                             "nfs-ganesha should be "
-                                             "enabled before setting "
-                                             "ganesha.enable.");
-                        goto out;
-                }
-                if (!ret) {
-                        if (op_errstr)
-                                gf_asprintf (op_errstr, "The option "
-                                             "nfs-ganesha should be "
-                                             "enabled before setting "
-                                             "ganesha.enable.");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        /* Create the export file only when ganesha.enable "on" is executed */
-         if (option) {
-                if (reboot)
-                       ret  =  copy_export_config (volname, op_errstr);
-                else
-                       ret  =  create_export_config (volname, op_errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_EXPORT_FILE_CREATE_FAIL,
-                                "Failed to create/copy "
-                                "export file for NFS-Ganesha\n");
-                        goto out;
-                }
-        }
-
-        if (check_host_list()) {
-                runner_add_args (&runner, "sh", GANESHA_PREFIX"/dbus-send.sh",
-                         CONFDIR, value, volname, NULL);
-                ret = runner_run (&runner);
-                if (ret) {
-                        if (op_errstr)
-                                gf_asprintf(op_errstr, "Dynamic export"
-                                            " addition/deletion failed."
-                                            " Please see log file for details");
-                        /* *
-                         * Incase of reboot scenarios, we cannot guarantee
-                         * nfs-ganesha to be running on that node, so that
-                         * dynamic export may fail
-                         */
-                        if (reboot)
-                                ret = 0;
-                        else
-                                goto out;
-                }
-        }
-
-
-        /* *
-         * cache-invalidation should be on when a volume is exported
-         * and off when a volume is unexported. It is not required
-         * for reboot scenarios, already it will be copied.
-         * */
-        if (!reboot) {
-                vol_opts = volinfo->dict;
-                ret = dict_set_dynstr_with_alloc (vol_opts,
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    dict_t *vol_opts = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t option = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(value);
+    GF_ASSERT(dict);
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+    ret = gf_string2boolean(value, &option);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "invalid value.");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    ret = glusterd_check_ganesha_export(volinfo);
+    if (ret && option) {
+        gf_asprintf(op_errstr,
+                    "ganesha.enable "
+                    "is already 'on'.");
+        ret = -1;
+        goto out;
+
+    } else if (!option && !ret) {
+        gf_asprintf(op_errstr,
+                    "ganesha.enable "
+                    "is already 'off'.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Check if global option is enabled, proceed only then */
+    ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
+                               _gf_false);
+    if (ret == -1) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to get "
+                     "global option dict.");
+        gf_asprintf(op_errstr,
+                    "The option "
+                    "nfs-ganesha should be "
+                    "enabled before setting ganesha.enable.");
+        goto out;
+    }
+    if (!ret) {
+        gf_asprintf(op_errstr,
+                    "The option "
+                    "nfs-ganesha should be "
+                    "enabled before setting ganesha.enable.");
+        ret = -1;
+        goto out;
+    }
+
+    /* *
+     * Create the export file from the node where ganesha.enable "on"
+     * is executed
+     * */
+    if (option && is_origin_glusterd(dict)) {
+        ret = manage_export_config(volname, "on", op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                   "Failed to create"
+                   "export file for NFS-Ganesha\n");
+            goto out;
+        }
+    }
+    ret = gd_ganesha_send_dbus(volname, value);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Dynamic export addition/deletion failed."
+                    " Please see log file for details");
+        goto out;
+    }
+    if (update_cache_invalidation) {
+        vol_opts = volinfo->dict;
+        ret = dict_set_dynstr_with_alloc(vol_opts,
                                          "features.cache-invalidation", value);
-                if (ret && op_errstr)
-                        gf_asprintf (op_errstr, "Cache-invalidation could not"
-                                                " be set to %s.", value);
-        }
+        if (ret)
+            gf_asprintf(op_errstr,
+                        "Cache-invalidation could not"
+                        " be set to %s.",
+                        value);
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            gf_asprintf(op_errstr, "failed to store volinfo for %s",
+                        volinfo->volname);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-tear_down_cluster(void)
+tear_down_cluster(gf_boolean_t run_teardown)
 {
-        int     ret     =       0;
-        runner_t runner =       {0,};
-
-        if (is_ganesha_host()) {
-                runinit (&runner);
-                runner_add_args (&runner, "sh",
-                                GANESHA_PREFIX"/ganesha-ha.sh", "teardown",
-                                CONFDIR, NULL);
-                ret = runner_run(&runner);
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+    struct stat st = {
+        0,
+    };
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+
+    if (run_teardown) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "teardown",
+                        CONFDIR, NULL);
+        ret = runner_run(&runner);
+        /* *
+         * Remove all the entries in CONFDIR expect ganesha.conf and
+         * ganesha-ha.conf
+         */
+        dir = sys_opendir(CONFDIR);
+        if (!dir) {
+            gf_msg_debug(THIS->name, 0,
+                         "Failed to open directory %s. "
+                         "Reason : %s",
+                         CONFDIR, strerror(errno));
+            ret = 0;
+            goto out;
+        }
+
+        while ((entry = sys_readdir(dir, scratch))) {
+            if (gf_irrelevant_entry(entry))
+                continue;
+            snprintf(path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name);
+            ret = sys_lstat(path, &st);
+            if (ret == -1) {
+                gf_msg_debug(THIS->name, 0,
+                             "Failed to stat entry %s :"
+                             " %s",
+                             path, strerror(errno));
+                goto out;
+            }
+
+            if (strcmp(entry->d_name, "ganesha.conf") == 0 ||
+                strcmp(entry->d_name, "ganesha-ha.conf") == 0)
+                gf_msg_debug(THIS->name, 0,
+                             " %s is not required"
+                             " to remove",
+                             path);
+            else if (S_ISDIR(st.st_mode))
+                ret = recursive_rmdir(path);
+            else
+                ret = sys_unlink(path);
+
+            if (ret) {
+                gf_msg_debug(THIS->name, 0,
+                             " Failed to remove %s. "
+                             "Reason : %s",
+                             path, strerror(errno));
+            }
+
+            gf_msg_debug(THIS->name, 0, "%s %s",
+                         ret ? "Failed to remove" : "Removed", entry->d_name);
+        }
+
+        ret = sys_closedir(dir);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0,
+                         "Failed to close dir %s. Reason :"
+                         " %s",
+                         CONFDIR, strerror(errno));
         }
-        return ret;
-}
+        goto exit;
+    }
 
+out:
+    if (dir && sys_closedir(dir)) {
+        gf_msg_debug(THIS->name, 0,
+                     "Failed to close dir %s. Reason :"
+                     " %s",
+                     CONFDIR, strerror(errno));
+    }
+exit:
+    return ret;
+}
 
 int
-setup_cluster(void)
+setup_cluster(gf_boolean_t run_setup)
 {
-        int ret         = 0;
-        runner_t runner = {0,};
-
-        if (is_ganesha_host()) {
-                runinit (&runner);
-                runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
-                                 "setup", CONFDIR,  NULL);
-                ret =  runner_run (&runner);
-        }
-        return ret;
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+
+    if (run_setup) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "setup",
+                        CONFDIR, NULL);
+        ret = runner_run(&runner);
+    }
+    return ret;
 }
 
-
 static int
-teardown (char **op_errstr)
+teardown(gf_boolean_t run_teardown, char **op_errstr)
 {
-        runner_t                runner                     = {0,};
-        int                     ret                        = 1;
-        glusterd_volinfo_t      *volinfo                   = NULL;
-        glusterd_conf_t         *priv                      = NULL;
-        dict_t                  *vol_opts                  = NULL;
-
-        priv = THIS->private;
-
-        ret = tear_down_cluster();
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "Cleanup of NFS-Ganesha"
-                             " HA config failed.");
-                goto out;
-        }
-        ret =  stop_ganesha (op_errstr);
-        if (ret) {
-                gf_asprintf (op_errstr, "Could not stop NFS-Ganesha.");
-                goto out;
-        }
+    runner_t runner = {
+        0,
+    };
+    int ret = 1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *vol_opts = NULL;
+
+    priv = THIS->private;
+
+    ret = tear_down_cluster(run_teardown);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "Cleanup of NFS-Ganesha"
+                    " HA config failed.");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "cleanup",
+                    CONFDIR, NULL);
+    ret = runner_run(&runner);
+    if (ret)
+        gf_msg_debug(THIS->name, 0,
+                     "Could not clean up"
+                     " NFS-Ganesha related config");
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        vol_opts = volinfo->dict;
+        /* All the volumes exported via NFS-Ganesha will be
+        unexported, hence setting the appropriate keys */
+        ret = dict_set_str(vol_opts, "features.cache-invalidation", "off");
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED,
+                   "Could not set features.cache-invalidation "
+                   "to off for %s",
+                   volinfo->volname);
 
-        runinit (&runner);
-        runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
-                                 "cleanup", CONFDIR,  NULL);
-        ret = runner_run (&runner);
+        ret = dict_set_str(vol_opts, "ganesha.enable", "off");
         if (ret)
-                gf_msg_debug (THIS->name, 0, "Could not clean up"
-                        " NFS-Ganesha related config");
-
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                vol_opts = volinfo->dict;
-                /* All the volumes exported via NFS-Ganesha will be
-                unexported, hence setting the appropriate key */
-                ret = dict_set_str (vol_opts, "ganesha.enable", "off");
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Could not set ganesha.enable to off");
-        }
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED,
+                   "Could not set ganesha.enable to off for %s",
+                   volinfo->volname);
+
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
+                   "failed to store volinfo for %s", volinfo->volname);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-stop_ganesha (char **op_errstr) {
-
-        int ret = 0;
-
-        if (check_host_list ()) {
-                ret = manage_service ("stop");
-                if (ret)
-                        gf_asprintf (op_errstr, "NFS-Ganesha service could not"
-                                     "be stopped.");
+stop_ganesha(char **op_errstr)
+{
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+
+    if (check_host_list()) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh",
+                        "--setup-ganesha-conf-files", CONFDIR, "no", NULL);
+        ret = runner_run(&runner);
+        if (ret) {
+            gf_asprintf(op_errstr,
+                        "removal of symlink ganesha.conf "
+                        "in /etc/ganesha failed");
         }
-        return ret;
-
+        ret = manage_service("stop");
+        if (ret)
+            gf_asprintf(op_errstr,
+                        "NFS-Ganesha service could not"
+                        "be stopped.");
+    }
+    return ret;
 }
 
 int
-start_ganesha (char **op_errstr)
+start_ganesha(char **op_errstr)
 {
-        int                     ret                        = -1;
-        char                    *hostname                  = NULL;
-        dict_t *vol_opts                                   = NULL;
-        glusterd_volinfo_t *volinfo                        = NULL;
-        int count                                          = 0;
-        char *volname                                      = NULL;
-        glusterd_conf_t *priv                              = NULL;
-
-        priv =  THIS->private;
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                vol_opts = volinfo->dict;
-                /* Gluster-nfs has to be disabled across the trusted pool */
-                /* before attempting to start nfs-ganesha */
-                ret = dict_set_str (vol_opts, NFS_DISABLE_MAP_KEY, "on");
-                if (ret)
-                        goto out;
-
-                ret = glusterd_store_volinfo (volinfo,
-                                GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        *op_errstr = gf_strdup ("Failed to store the "
-                                                "Volume information");
-                        goto out;
-                }
-        }
-
-        /* If the nfs svc is not initialized it means that the service is not
-         * running, hence we can skip the process of stopping gluster-nfs
-         * service
-         */
-        if (priv->nfs_svc.inited) {
-                ret = priv->nfs_svc.stop (&(priv->nfs_svc), SIGKILL);
-                if (ret) {
-                        ret = -1;
-                        gf_asprintf (op_errstr, "Gluster-NFS service could"
-                                     "not be stopped, exiting.");
-                        goto out;
-                }
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+#ifdef BUILD_GNFS
+        /* Gluster-nfs has to be disabled across the trusted pool */
+        /* before attempting to start nfs-ganesha */
+        ret = dict_set_str_sizen(volinfo->dict, NFS_DISABLE_MAP_KEY, "on");
+        if (ret)
+            goto out;
+#endif
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+            *op_errstr = gf_strdup(
+                "Failed to store the "
+                "Volume information");
+            goto out;
+        }
+    }
+
+    /* If the nfs svc is not initialized it means that the service is not
+     * running, hence we can skip the process of stopping gluster-nfs
+     * service
+     */
+#ifdef BUILD_GNFS
+    if (priv->nfs_svc.inited) {
+        ret = priv->nfs_svc.stop(&(priv->nfs_svc), SIGKILL);
+        if (ret) {
+            ret = -1;
+            gf_asprintf(op_errstr,
+                        "Gluster-NFS service could"
+                        "not be stopped, exiting.");
+            goto out;
+        }
+    }
+#endif
+
+    if (check_host_list()) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh",
+                        "--setup-ganesha-conf-files", CONFDIR, "yes", NULL);
+        ret = runner_run(&runner);
+        if (ret) {
+            gf_asprintf(op_errstr,
+                        "creation of symlink ganesha.conf "
+                        "in /etc/ganesha failed");
+            goto out;
         }
-        if (check_host_list()) {
-                ret = manage_service ("start");
-                if (ret)
-                        gf_asprintf (op_errstr, "NFS-Ganesha failed to start."
+        ret = manage_service("start");
+        if (ret)
+            gf_asprintf(op_errstr,
+                        "NFS-Ganesha failed to start."
                         "Please see log file for details");
-        }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-pre_setup (char **op_errstr)
+pre_setup(gf_boolean_t run_setup, char **op_errstr)
 {
-        int    ret = 0;
-
-        ret = sys_mkdir (SHARED_STORAGE_MNT, 0775);
-
-        if ((-1 == ret) && (EEXIST != errno)) {
-                gf_msg ("THIS->name", GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "mkdir() failed on path %s,",
-                        SHARED_STORAGE_MNT);
-                goto out;
-        }
-
-        ret = check_host_list();
-
-        if (ret) {
-                ret = setup_cluster();
-                if (ret == -1)
-                        gf_asprintf (op_errstr, "Failed to set up HA "
-                                     "config for NFS-Ganesha. "
-                                     "Please check the log file for details");
-        }
-
-out:
-        return ret;
+    int ret = 0;
+    if (run_setup) {
+        if (!check_host_list()) {
+            gf_asprintf(op_errstr,
+                        "Running nfs-ganesha setup command "
+                        "from node which is not part of ganesha cluster");
+            return -1;
+        }
+    }
+    ret = setup_cluster(run_setup);
+    if (ret == -1)
+        gf_asprintf(op_errstr,
+                    "Failed to set up HA "
+                    "config for NFS-Ganesha. "
+                    "Please check the log file for details");
+    return ret;
 }
 
 int
-glusterd_handle_ganesha_op (dict_t *dict, char **op_errstr,
-                            char *key, char *value)
+glusterd_handle_ganesha_op(dict_t *dict, char **op_errstr, char *key,
+                           char *value)
 {
-
-        int32_t                ret           = -1;
-        char                   *volname      = NULL;
-        gf_boolean_t           option        = _gf_false;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
-
-
-        if (strcmp (key, "ganesha.enable") == 0) {
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get volume name");
-                        goto out;
-                }
-                ret =  ganesha_manage_export (volname, value, op_errstr,
-                                              _gf_false);
-                if (ret < 0)
-                        goto out;
-        }
-
-        /* It is possible that the key might not be set */
-        ret =  gf_string2boolean (value, &option);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "Invalid value in key-value pair.");
+    int32_t ret = -1;
+    gf_boolean_t option = _gf_false;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(key);
+    GF_ASSERT(value);
+
+    if (strcmp(key, "ganesha.enable") == 0) {
+        ret = ganesha_manage_export(dict, value, _gf_true, op_errstr);
+        if (ret < 0)
+            goto out;
+    }
+
+    /* It is possible that the key might not be set */
+    ret = gf_string2boolean(value, &option);
+    if (ret == -1) {
+        gf_asprintf(op_errstr, "Invalid value in key-value pair.");
+        goto out;
+    }
+
+    if (strcmp(key, GLUSTERD_STORE_KEY_GANESHA_GLOBAL) == 0) {
+        /* *
+         * The set up/teardown of pcs cluster should be performed only
+         * once. This will done on the node in which the cli command
+         * 'gluster nfs-ganesha <enable/disable>' got executed. So that
+         * node should part of ganesha HA cluster
+         */
+        if (option) {
+            ret = pre_setup(is_origin_glusterd(dict), op_errstr);
+            if (ret < 0)
+                goto out;
+        } else {
+            ret = teardown(is_origin_glusterd(dict), op_errstr);
+            if (ret < 0)
                 goto out;
         }
-
-        if (strcmp (key, GLUSTERD_STORE_KEY_GANESHA_GLOBAL) == 0) {
-                if (option) {
-                        ret = pre_setup (op_errstr);
-                        if (ret < 0)
-                                goto out;
-                } else {
-                        ret = teardown (op_errstr);
-                        if (ret < 0)
-                                goto out;
-                }
-        }
+    }
 
 out:
-        return ret;
+    return ret;
 }
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
index 341e2f91b8d..bf062c87060 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -7,7 +7,7 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -17,362 +17,397 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-svc-helper.h"
-#include "run.h"
-#include "syscall.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 #include "glusterd-messages.h"
 
 #include <signal.h>
 
 static int
-dict_get_param (dict_t *dict, char *key, char **param);
+dict_get_param(dict_t *dict, char *key, char **param);
 
 struct gsync_config_opt_vals_ gsync_confopt_vals[] = {
-        {.op_name        = "change_detector",
-         .no_of_pos_vals = 2,
-         .case_sensitive = _gf_true,
-         .values         = {"xsync", "changelog"},
-        },
-        {.op_name        = "special_sync_mode",
-         .no_of_pos_vals = 2,
-         .case_sensitive = _gf_true,
-         .values         = {"partial", "recover"}
-        },
-        {.op_name        = "log-level",
-         .no_of_pos_vals = 5,
-         .case_sensitive = _gf_false,
-         .values         = {"critical", "error", "warning", "info", "debug"}
-        },
-        {.op_name        = "use-tarssh",
-         .no_of_pos_vals = 6,
-         .case_sensitive = _gf_false,
-         .values         = {"true", "false", "0", "1", "yes", "no"}
-        },
-        {.op_name        = "ignore_deletes",
-         .no_of_pos_vals = 6,
-         .case_sensitive = _gf_false,
-         .values         = {"true", "false", "0", "1", "yes", "no"}
-        },
-        {.op_name        = "use_meta_volume",
-         .no_of_pos_vals = 6,
-         .case_sensitive = _gf_false,
-         .values         = {"true", "false", "0", "1", "yes", "no"}
-        },
-        {.op_name        = "use-meta-volume",
-         .no_of_pos_vals = 6,
-         .case_sensitive = _gf_false,
-         .values         = {"true", "false", "0", "1", "yes", "no"}
-        },
-        {.op_name = NULL,
-        },
+    {
+        .op_name = "change_detector",
+        .no_of_pos_vals = 2,
+        .case_sensitive = _gf_true,
+        .values = {"xsync", "changelog"},
+    },
+    {.op_name = "special_sync_mode",
+     .no_of_pos_vals = 2,
+     .case_sensitive = _gf_true,
+     .values = {"partial", "recover"}},
+    {.op_name = "log-level",
+     .no_of_pos_vals = 5,
+     .case_sensitive = _gf_false,
+     .values = {"critical", "error", "warning", "info", "debug"}},
+    {.op_name = "use-tarssh",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {.op_name = "ignore_deletes",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {.op_name = "use_meta_volume",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {.op_name = "use-meta-volume",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {
+        .op_name = NULL,
+    },
 };
 
-static char *gsync_reserved_opts[] = {
-        "gluster-command-dir",
-        "pid-file",
-        "state-file",
-        "session-owner",
-        "state-socket-unencoded",
-        "socketdir",
-        "local-id",
-        "local-path",
-        "slave-id",
-        NULL
-};
+static char *gsync_reserved_opts[] = {"gluster-command",
+                                      "pid-file",
+                                      "state-file",
+                                      "session-owner",
+                                      "state-socket-unencoded",
+                                      "socketdir",
+                                      "local-id",
+                                      "local-path",
+                                      "slave-id",
+                                      NULL};
 
-static char *gsync_no_restart_opts[] = {
-        "checkpoint",
-        NULL
-};
+static char *gsync_no_restart_opts[] = {"checkpoint", "log_rsync_performance",
+                                        "log-rsync-performance", NULL};
 
-int
-__glusterd_handle_sys_exec (rpcsvc_request_t *req)
+void
+set_gsyncd_inet6_arg(runner_t *runner)
 {
-        int32_t                 ret     = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req              cli_req = {{0},};
-        glusterd_op_t           cli_op = GD_OP_SYS_EXEC;
-        glusterd_conf_t         *priv   = NULL;
-        char                    *host_uuid = NULL;
-        char                    err_str[2048] = {0,};
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    xlator_t *this = NULL;
+    char *af;
+    int ret;
+
+    this = THIS;
+    ret = dict_get_str(this->options, "transport.address-family", &af);
+    if (ret == 0)
+        runner_argprintf(runner, "--%s", af);
+}
 
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+int
+__glusterd_handle_sys_exec(rpcsvc_request_t *req)
+{
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_SYS_EXEC;
+    glusterd_conf_t *priv = NULL;
+    char *host_uuid = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
 
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-               } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret)
-                        goto out;
+        ret = dict_set_dynstr(dict, "host-uuid", host_uuid);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=host-uuid", NULL);
+            goto out;
         }
+    }
 
-        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+    ret = glusterd_op_begin_synctask(req, cli_op, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    return ret;
 }
 
 int
-__glusterd_handle_copy_file (rpcsvc_request_t *req)
+__glusterd_handle_copy_file(rpcsvc_request_t *req)
 {
-        int32_t                 ret     = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req              cli_req = {{0},};
-        glusterd_op_t           cli_op = GD_OP_COPY_FILE;
-        glusterd_conf_t         *priv   = NULL;
-        char                    *host_uuid = NULL;
-        char                    err_str[2048] = {0,};
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_COPY_FILE;
+    glusterd_conf_t *priv = NULL;
+    char *host_uuid = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to"
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
 
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to"
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-               } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret)
-                        goto out;
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
 
-        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+        ret = dict_set_dynstr(dict, "host-uuid", host_uuid);
+        if (ret)
+            goto out;
+    }
+
+    ret = glusterd_op_begin_synctask(req, cli_op, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    return ret;
 }
 
 int
-__glusterd_handle_gsync_set (rpcsvc_request_t *req)
+__glusterd_handle_gsync_set(rpcsvc_request_t *req)
 {
-        int32_t                 ret     = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req              cli_req = {{0},};
-        glusterd_op_t           cli_op = GD_OP_GSYNC_SET;
-        char                    *master = NULL;
-        char                    *slave = NULL;
-        char                    operation[256] = {0,};
-        int                     type = 0;
-        glusterd_conf_t         *priv   = NULL;
-        char                    *host_uuid = NULL;
-        char                    err_str[2048] = {0,};
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (req);
-
-       this = THIS;
-       GF_ASSERT (this);
-       priv = this->private;
-       GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_GSYNC_SET;
+    char *master = NULL;
+    char *slave = NULL;
+    char operation[64] = {
+        0,
+    };
+    int type = 0;
+    glusterd_conf_t *priv = NULL;
+    char *host_uuid = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret)
-                        goto out;
-
-        }
-
-        ret = dict_get_str (dict, "master", &master);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
-                        "master not found, while handling "GEOREP" options");
-                master = "(No Master)";
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
 
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
-                         "slave not found, while handling "GEOREP" options");
-                slave = "(No Slave)";
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Command type not found "
-                          "while handling "GEOREP" options");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "%s", err_str);
-                goto out;
-        }
-
-        switch (type) {
+        ret = dict_set_dynstr(dict, "host-uuid", host_uuid);
+        if (ret)
+            goto out;
+    }
+
+    ret = dict_get_str(dict, "master", &master);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "master not found, while handling " GEOREP " options");
+        master = "(No Master)";
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "slave not found, while handling " GEOREP " options");
+        slave = "(No Slave)";
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Command type not found "
+                 "while handling " GEOREP " options");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    switch (type) {
         case GF_GSYNC_OPTION_TYPE_CREATE:
-                strncpy (operation, "create", sizeof (operation));
-                cli_op = GD_OP_GSYNC_CREATE;
-                break;
+            snprintf(operation, sizeof(operation), "create");
+            cli_op = GD_OP_GSYNC_CREATE;
+            break;
 
         case GF_GSYNC_OPTION_TYPE_START:
-                strncpy (operation, "start", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "start");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_STOP:
-                strncpy (operation, "stop", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "stop");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_PAUSE:
-                strncpy (operation, "pause", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "pause");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_RESUME:
-                strncpy (operation, "resume", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "resume");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_CONFIG:
-                strncpy (operation, "config", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "config");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_STATUS:
-                strncpy (operation, "status", sizeof (operation));
-                break;
-        }
+            snprintf(operation, sizeof(operation), "status");
+            break;
+    }
 
-        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+    ret = glusterd_op_begin_synctask(req, cli_op, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    return ret;
 }
 
 int
-glusterd_handle_sys_exec (rpcsvc_request_t *req)
+glusterd_handle_sys_exec(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_sys_exec);
+    return glusterd_big_locked_handler(req, __glusterd_handle_sys_exec);
 }
 
 int
-glusterd_handle_copy_file (rpcsvc_request_t *req)
+glusterd_handle_copy_file(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_copy_file);
+    return glusterd_big_locked_handler(req, __glusterd_handle_copy_file);
 }
 
 int
-glusterd_handle_gsync_set (rpcsvc_request_t *req)
+glusterd_handle_gsync_set(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_gsync_set);
+    return glusterd_big_locked_handler(req, __glusterd_handle_gsync_set);
 }
 
 /*****
@@ -382,1523 +417,1597 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
  *****/
 
 static void
-glusterd_urltransform_init (runner_t *runner, const char *transname)
+glusterd_urltransform_init(runner_t *runner, const char *transname)
 {
-        runinit (runner);
-        runner_add_arg (runner, GSYNCD_PREFIX"/gsyncd");
-        runner_argprintf (runner, "--%s-url", transname);
+    runinit(runner);
+    runner_add_arg(runner, GSYNCD_PREFIX "/gsyncd");
+    set_gsyncd_inet6_arg(runner);
+    runner_argprintf(runner, "--%s-url", transname);
 }
 
 static void
-glusterd_urltransform_add (runner_t *runner, const char *url)
+glusterd_urltransform_add(runner_t *runner, const char *url)
 {
-        runner_add_arg (runner, url);
+    runner_add_arg(runner, url);
 }
 
 /* Helper routine to terminate just before slave_voluuid */
 static int32_t
-parse_slave_url (char *slv_url, char **slave)
+parse_slave_url(char *slv_url, char **slave)
 {
-        char      *tmp  = NULL;
-        xlator_t  *this = NULL;
-        int32_t    ret  = -1;
-
-        this = THIS;
-
-        /* slave format:
-         * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
-        *slave = strchr (slv_url, ':');
-        if (!(*slave)) {
-                goto out;
-        }
-        (*slave)++;
-
-        /* To terminate at : before slave volume uuid */
-        tmp = strstr (*slave, "::");
-        if (!tmp) {
-                goto out;
-        }
-        tmp += 2;
-        tmp = strchr (tmp, ':');
-        if (!tmp)
-                gf_msg_debug (this->name, 0, "old slave: %s!", *slave);
-        else
-                *tmp = '\0';
-
-        ret = 0;
-        gf_msg_debug (this->name, 0, "parsed slave: %s!", *slave);
+    char *tmp = NULL;
+    xlator_t *this = NULL;
+    int32_t ret = -1;
+
+    this = THIS;
+
+    /* slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    *slave = strchr(slv_url, ':');
+    if (!(*slave)) {
+        goto out;
+    }
+    (*slave)++;
+
+    /* To terminate at : before slave volume uuid */
+    tmp = strstr(*slave, "::");
+    if (!tmp) {
+        goto out;
+    }
+    tmp += 2;
+    tmp = strchr(tmp, ':');
+    if (!tmp)
+        gf_msg_debug(this->name, 0, "old slave: %s!", *slave);
+    else
+        *tmp = '\0';
+
+    ret = 0;
+    gf_msg_debug(this->name, 0, "parsed slave: %s!", *slave);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-_glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *data)
+_glusterd_urltransform_add_iter(dict_t *dict, char *key, data_t *value,
+                                void *data)
 {
-        runner_t *runner = (runner_t *)data;
-        char slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
-        char *slave          = NULL;
-        xlator_t  *this      = NULL;
-        int32_t  ret         = -1;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        gf_msg_debug (this->name, 0, "value->data %s", value->data);
-
-        strncpy (slv_url, value->data, sizeof(slv_url));
-        ret = parse_slave_url (slv_url, &slave);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVE_VOL_PARSE_FAIL,
-                        "Error in parsing slave: %s!", value->data);
-                goto out;
-        }
-
-        runner_add_arg (runner, slave);
-        ret = 0;
+    runner_t *runner = (runner_t *)data;
+    char slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
+    char *slave = NULL;
+    xlator_t *this = NULL;
+    int32_t ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    gf_msg_debug(this->name, 0, "value->data %s", value->data);
+
+    if (snprintf(slv_url, sizeof(slv_url), "%s", value->data) >=
+        sizeof(slv_url)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in copying slave: %s!", value->data);
+        goto out;
+    }
+
+    ret = parse_slave_url(slv_url, &slave);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in parsing slave: %s!", value->data);
+        goto out;
+    }
+
+    runner_add_arg(runner, slave);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static void
-glusterd_urltransform_free (char **linearr, unsigned n)
+glusterd_urltransform_free(char **linearr, unsigned n)
 {
-        int i = 0;
+    int i = 0;
 
-        for (; i < n; i++)
-                GF_FREE (linearr[i]);
+    for (; i < n; i++)
+        GF_FREE(linearr[i]);
 
-        GF_FREE (linearr);
+    GF_FREE(linearr);
 }
 
 static int
-glusterd_urltransform (runner_t *runner, char ***linearrp)
+glusterd_urltransform(runner_t *runner, char ***linearrp)
 {
-        char **linearr = NULL;
-        char *line = NULL;
-        unsigned arr_len = 32;
-        unsigned arr_idx = 0;
-        gf_boolean_t error = _gf_false;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        linearr = GF_CALLOC (arr_len, sizeof (char *), gf_gld_mt_linearr);
-        if (!linearr) {
-                error = _gf_true;
-                goto out;
-        }
-
-        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
-        if (runner_start (runner) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SPAWNING_CHILD_FAILED,
-                        "spawning child failed");
-
+    char **linearr = NULL;
+    char *line = NULL;
+    unsigned arr_len = 32;
+    unsigned arr_idx = 0;
+    gf_boolean_t error = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    linearr = GF_CALLOC(arr_len, sizeof(char *), gf_gld_mt_linearr);
+    if (!linearr) {
+        error = _gf_true;
+        goto out;
+    }
+
+    runner_redir(runner, STDOUT_FILENO, RUN_PIPE);
+    if (runner_start(runner) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SPAWNING_CHILD_FAILED,
+               "spawning child failed");
+
+        error = _gf_true;
+        goto out;
+    }
+
+    arr_idx = 0;
+    for (;;) {
+        size_t len;
+        line = GF_MALLOC(1024, gf_gld_mt_linebuf);
+        if (!line) {
+            error = _gf_true;
+            goto out;
+        }
+
+        if (fgets(line, 1024, runner_chio(runner, STDOUT_FILENO)) == NULL) {
+            GF_FREE(line);
+            break;
+        }
+
+        len = strlen(line);
+        if (len == 0 || line[len - 1] != '\n') {
+            GF_FREE(line);
+            error = _gf_true;
+            goto out;
+        }
+        line[len - 1] = '\0';
+
+        if (arr_idx == arr_len) {
+            void *p = linearr;
+            arr_len <<= 1;
+            p = GF_REALLOC(linearr, arr_len);
+            if (!p) {
+                GF_FREE(line);
                 error = _gf_true;
                 goto out;
+            }
+            linearr = p;
         }
+        linearr[arr_idx] = line;
 
-        arr_idx = 0;
-        for (;;) {
-                size_t len;
-                line = GF_MALLOC (1024, gf_gld_mt_linebuf);
-                if (!line) {
-                        error = _gf_true;
-                        goto out;
-                }
-
-                if (fgets (line, 1024, runner_chio (runner, STDOUT_FILENO)) ==
-                    NULL)
-                        break;
-
-                len = strlen (line);
-                if (len == 0 || line[len - 1] != '\n') {
-                        GF_FREE (line);
-                        error = _gf_true;
-                        goto out;
-                }
-                line[len - 1] = '\0';
-
-                if (arr_idx == arr_len) {
-                        void *p = linearr;
-                        arr_len <<= 1;
-                        p = GF_REALLOC (linearr, arr_len);
-                        if (!p) {
-                                GF_FREE (line);
-                                error = _gf_true;
-                                goto out;
-                        }
-                        linearr = p;
-                }
-                linearr[arr_idx] = line;
-
-                arr_idx++;
-        }
-
- out:
+        arr_idx++;
+    }
 
-        /* XXX chpid field is not exported by run API
-         * but runner_end() does not abort the invoked
-         * process (ie. it might block in waitpid(2))
-         * so we resort to a manual kill a the private field
-         */
-        if (error && runner->chpid > 0)
-                kill (runner->chpid, SIGKILL);
-
-        if (runner_end (runner) != 0)
-                error = _gf_true;
+out:
 
-        if (error) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_READ_CHILD_DATA_FAILED,
-                        "reading data from child failed");
-                glusterd_urltransform_free (linearr, arr_idx);
-                return -1;
-        }
+    /* XXX chpid field is not exported by run API
+     * but runner_end() does not abort the invoked
+     * process (ie. it might block in waitpid(2))
+     * so we resort to a manual kill a the private field
+     */
+    if (error && runner->chpid > 0)
+        kill(runner->chpid, SIGKILL);
+
+    if (runner_end(runner) != 0)
+        error = _gf_true;
+
+    if (error) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_CHILD_DATA_FAILED,
+               "reading data from child failed");
+        glusterd_urltransform_free(linearr, arr_idx);
+        return -1;
+    }
 
-        *linearrp = linearr;
-        return arr_idx;
+    *linearrp = linearr;
+    return arr_idx;
 }
 
 static int
-glusterd_urltransform_single (const char *url, const char *transname,
-                              char ***linearrp)
+glusterd_urltransform_single(const char *url, const char *transname,
+                             char ***linearrp)
 {
-        runner_t runner = {0,};
+    runner_t runner = {
+        0,
+    };
 
-        glusterd_urltransform_init (&runner, transname);
-        glusterd_urltransform_add (&runner, url);
-        return glusterd_urltransform (&runner, linearrp);
+    glusterd_urltransform_init(&runner, transname);
+    glusterd_urltransform_add(&runner, url);
+    return glusterd_urltransform(&runner, linearrp);
 }
 
-
 struct dictidxmark {
-        unsigned isrch;
-        unsigned ithis;
-        char *ikey;
+    unsigned isrch;
+    unsigned ithis;
+    char *ikey;
 };
 
-
 struct slave_vol_config {
-       char      old_slvhost[_POSIX_HOST_NAME_MAX+1];
-       unsigned  old_slvidx;
-       char      slave_voluuid[GF_UUID_BUF_SIZE];
+    char old_slvhost[_POSIX_HOST_NAME_MAX + 1];
+    char old_slvuser[LOGIN_NAME_MAX];
+    unsigned old_slvidx;
+    char slave_voluuid[UUID_CANONICAL_FORM_LEN + 1];
 };
 
 static int
-_dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
+_dict_mark_atindex(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct dictidxmark *dim = data;
+    struct dictidxmark *dim = data;
 
-        if (dim->isrch == dim->ithis)
-                dim->ikey = key;
+    if (dim->isrch == dim->ithis)
+        dim->ikey = key;
 
-        dim->ithis++;
-        return 0;
+    dim->ithis++;
+    return 0;
 }
 
 static char *
-dict_get_by_index (dict_t *dict, unsigned i)
+dict_get_by_index(dict_t *dict, unsigned i)
 {
-        struct dictidxmark dim = {0,};
+    struct dictidxmark dim = {
+        0,
+    };
 
-        dim.isrch = i;
-        dict_foreach (dict, _dict_mark_atindex, &dim);
+    dim.isrch = i;
+    dict_foreach(dict, _dict_mark_atindex, &dim);
 
-        return dim.ikey;
+    return dim.ikey;
 }
 
 static int
-glusterd_get_slave (glusterd_volinfo_t *vol, const char *slaveurl, char **slavekey)
+glusterd_get_slave(glusterd_volinfo_t *vol, const char *slaveurl,
+                   char **slavekey)
 {
-        runner_t runner = {0,};
-        int n = 0;
-        int i = 0;
-        char **linearr = NULL;
-        int32_t ret = 0;
-
-        glusterd_urltransform_init (&runner, "canonicalize");
-        ret = dict_foreach (vol->gsync_slaves, _glusterd_urltransform_add_iter,
-                                               &runner);
-        if (ret < 0)
-                return -2;
-
-        glusterd_urltransform_add (&runner, slaveurl);
-
-        n = glusterd_urltransform (&runner, &linearr);
-        if (n == -1)
-                return -2;
-
-        for (i = 0; i < n - 1; i++) {
-                if (strcmp (linearr[i], linearr[n - 1]) == 0)
-                        break;
-        }
-        glusterd_urltransform_free (linearr, i);
-
-        if (i < n - 1)
-                *slavekey = dict_get_by_index (vol->gsync_slaves, i);
-        else
-                i = -1;
-
-        return i;
+    runner_t runner = {
+        0,
+    };
+    int n = 0;
+    int i = 0;
+    char **linearr = NULL;
+    int32_t ret = 0;
+
+    glusterd_urltransform_init(&runner, "canonicalize");
+    ret = dict_foreach(vol->gsync_slaves, _glusterd_urltransform_add_iter,
+                       &runner);
+    if (ret < 0)
+        return -2;
+
+    glusterd_urltransform_add(&runner, slaveurl);
+
+    n = glusterd_urltransform(&runner, &linearr);
+    if (n == -1)
+        return -2;
+
+    for (i = 0; i < n - 1; i++) {
+        if (strcmp(linearr[i], linearr[n - 1]) == 0)
+            break;
+    }
+    glusterd_urltransform_free(linearr, n);
+
+    if (i < n - 1)
+        *slavekey = dict_get_by_index(vol->gsync_slaves, i);
+    else
+        i = -1;
+
+    return i;
 }
 
 static int
-glusterd_query_extutil_generic (char *resbuf, size_t blen, runner_t *runner, void *data,
-                                int (*fcbk)(char *resbuf, size_t blen, FILE *fp, void *data))
+glusterd_query_extutil_generic(char *resbuf, size_t blen, runner_t *runner,
+                               void *data,
+                               int (*fcbk)(char *resbuf, size_t blen, FILE *fp,
+                                           void *data))
 {
-        int                 ret = 0;
-        xlator_t            *this = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
-        if (runner_start (runner) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SPAWNING_CHILD_FAILED,
-                        "spawning child failed");
+    runner_redir(runner, STDOUT_FILENO, RUN_PIPE);
+    if (runner_start(runner) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SPAWNING_CHILD_FAILED,
+               "spawning child failed");
 
-                return -1;
-        }
+        return -1;
+    }
 
-        ret = fcbk (resbuf, blen, runner_chio (runner, STDOUT_FILENO), data);
+    ret = fcbk(resbuf, blen, runner_chio(runner, STDOUT_FILENO), data);
 
-        ret |= runner_end (runner);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_READ_CHILD_DATA_FAILED,
-                        "reading data from child failed");
+    ret |= runner_end(runner);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_CHILD_DATA_FAILED,
+               "reading data from child failed");
 
-        return ret ? -1 : 0;
+    return ret ? -1 : 0;
 }
 
 static int
 _fcbk_singleline(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        char *ptr = NULL;
+    char *ptr = NULL;
 
-        errno = 0;
-        ptr = fgets (resbuf, blen, fp);
-        if (ptr) {
-                size_t len = strlen(resbuf);
-                if (len && resbuf[len-1] == '\n')
-                        resbuf[len-1] = '\0'; //strip off \n
-        }
+    errno = 0;
+    ptr = fgets(resbuf, blen, fp);
+    if (ptr) {
+        size_t len = strlen(resbuf);
+        if (len && resbuf[len - 1] == '\n')
+            resbuf[len - 1] = '\0';  // strip off \n
+    }
 
-        return errno ? -1 : 0;
+    return errno ? -1 : 0;
 }
 
 static int
-glusterd_query_extutil (char *resbuf, runner_t *runner)
+glusterd_query_extutil(char *resbuf, runner_t *runner)
 {
-        return glusterd_query_extutil_generic (resbuf, PATH_MAX, runner, NULL,
-                                               _fcbk_singleline);
+    return glusterd_query_extutil_generic(resbuf, PATH_MAX, runner, NULL,
+                                          _fcbk_singleline);
 }
 
 static int
-glusterd_get_slave_voluuid (char *slave_host, char *slave_vol, char *vol_uuid)
+glusterd_get_slave_voluuid(char *slave_host, char *slave_vol, char *vol_uuid)
 {
-        runner_t          runner = {0,};
-        glusterd_conf_t   *priv  = NULL;
-        xlator_t          *this  = NULL;
-        int ret                  = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
 
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        runinit (&runner);
-        runner_add_arg  (&runner, GSYNCD_PREFIX"/gsyncd");
-        runner_add_arg  (&runner, "--slavevoluuid-get");
-        runner_argprintf (&runner, "%s::%s", slave_host, slave_vol);
+    runinit(&runner);
+    runner_add_arg(&runner, GSYNCD_PREFIX "/gsyncd");
+    set_gsyncd_inet6_arg(&runner);
+    runner_add_arg(&runner, "--slavevoluuid-get");
+    runner_argprintf(&runner, "%s::%s", slave_host, slave_vol);
 
-        synclock_unlock (&priv->big_lock);
-        ret = glusterd_query_extutil (vol_uuid, &runner);
-        synclock_lock (&priv->big_lock);
+    synclock_unlock(&priv->big_lock);
+    ret = glusterd_query_extutil(vol_uuid, &runner);
+    synclock_lock(&priv->big_lock);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-_fcbk_conftodict (char *resbuf, size_t blen, FILE *fp, void *data)
+_fcbk_conftodict(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        char   *ptr = NULL;
-        dict_t *dict = data;
-        char   *v = NULL;
-
-        for (;;) {
-                errno = 0;
-                ptr = fgets (resbuf, blen, fp);
-                if (!ptr)
-                        break;
-                v = resbuf + strlen(resbuf) - 1;
-                while (isspace (*v))
-                        /* strip trailing space */
-                        *v-- = '\0';
-                if (v == resbuf)
-                        /* skip empty line */
-                        continue;
-                v = strchr (resbuf, ':');
-                if (!v)
-                        return -1;
-                *v++ = '\0';
-                while (isspace (*v))
-                        v++;
-                v = gf_strdup (v);
-                if (!v)
-                        return -1;
-                if (dict_set_dynstr (dict, resbuf, v) != 0) {
-                        GF_FREE (v);
-                        return -1;
-                }
-        }
+    char *ptr = NULL;
+    dict_t *dict = data;
+    char *v = NULL;
 
-        return errno ? -1 : 0;
+    for (;;) {
+        errno = 0;
+        ptr = fgets(resbuf, blen - 2, fp);
+        if (!ptr)
+            break;
+        v = resbuf + strlen(resbuf) - 1;
+        while (isspace(*v))
+            /* strip trailing space */
+            *v-- = '\0';
+        if (v == resbuf)
+            /* skip empty line */
+            continue;
+        v = strchr(resbuf, ':');
+        if (!v)
+            return -1;
+        *v++ = '\0';
+        while (isspace(*v))
+            v++;
+        v = gf_strdup(v);
+        if (!v)
+            return -1;
+        if (dict_set_dynstr(dict, resbuf, v) != 0) {
+            GF_FREE(v);
+            return -1;
+        }
+    }
+
+    return errno ? -1 : 0;
 }
 
 static int
-glusterd_gsync_get_config (char *master, char *slave, char *conf_path, dict_t *dict)
+glusterd_gsync_get_config(char *master, char *slave, char *conf_path,
+                          dict_t *dict)
 {
-        /* key + value, where value must be able to accommodate a path */
-        char resbuf[256 + PATH_MAX] = {0,};
-        runner_t             runner = {0,};
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get-all", NULL);
-
-        return glusterd_query_extutil_generic (resbuf, sizeof (resbuf),
-                                               &runner, dict, _fcbk_conftodict);
+    /* key + value, where value must be able to accommodate a path */
+    char resbuf[256 + PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    set_gsyncd_inet6_arg(&runner);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, "--config-get-all", NULL);
+
+    return glusterd_query_extutil_generic(resbuf, sizeof(resbuf), &runner, dict,
+                                          _fcbk_conftodict);
 }
 
 static int
-_fcbk_statustostruct (char *resbuf, size_t blen, FILE *fp,
-                      void *data)
+_fcbk_statustostruct(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        char                  *ptr = NULL;
-        char                    *v = NULL;
-        char                    *k = NULL;
-        gf_gsync_status_t *sts_val = NULL;
-
-        sts_val = (gf_gsync_status_t *)data;
-
-        for (;;) {
-                errno = 0;
-                ptr = fgets (resbuf, blen, fp);
-                if (!ptr)
-                        break;
-                v = resbuf + strlen(resbuf) - 1;
-                while (isspace (*v))
-                        /* strip trailing space */
-                        *v-- = '\0';
-                if (v == resbuf)
-                        /* skip empty line */
-                        continue;
-                v = strchr (resbuf, ':');
-                if (!v)
-                        return -1;
-                *v++ = '\0';
-                while (isspace (*v))
-                        v++;
-                v = gf_strdup (v);
-                if (!v)
-                        return -1;
-
-                k = gf_strdup (resbuf);
-                if (!k)
-                        return -1;
-
-                if (strcmp (k, "worker_status") == 0) {
-                        memcpy (sts_val->worker_status, v,
-                                strlen(v));
-                        sts_val->worker_status[strlen(v)] = '\0';
-                } else if (strcmp (k, "slave_node") == 0) {
-                        memcpy (sts_val->slave_node, v,
-                                strlen(v));
-                        sts_val->slave_node[strlen(v)] = '\0';
-                } else if (strcmp (k, "crawl_status") == 0) {
-                        memcpy (sts_val->crawl_status, v,
-                                strlen(v));
-                        sts_val->crawl_status[strlen(v)] = '\0';
-                } else if (strcmp (k, "last_synced") == 0) {
-                        memcpy (sts_val->last_synced, v,
-                                strlen(v));
-                        sts_val->last_synced[strlen(v)] = '\0';
-                } else if (strcmp (k, "last_synced_utc") == 0) {
-                        memcpy (sts_val->last_synced_utc, v,
-                                strlen(v));
-                        sts_val->last_synced_utc[strlen(v)] = '\0';
-                } else if (strcmp (k, "entry") == 0) {
-                        memcpy (sts_val->entry, v,
-                                strlen(v));
-                        sts_val->entry[strlen(v)] = '\0';
-                } else if (strcmp (k, "data") == 0) {
-                        memcpy (sts_val->data, v,
-                                strlen(v));
-                        sts_val->data[strlen(v)] = '\0';
-                } else if (strcmp (k, "meta") == 0) {
-                        memcpy (sts_val->meta, v,
-                                strlen(v));
-                        sts_val->meta[strlen(v)] = '\0';
-                } else if (strcmp (k, "failures") == 0) {
-                        memcpy (sts_val->failures, v,
-                                strlen(v));
-                        sts_val->failures[strlen(v)] = '\0';
-                } else if (strcmp (k, "checkpoint_time") == 0) {
-                        memcpy (sts_val->checkpoint_time, v,
-                                strlen(v));
-                        sts_val->checkpoint_time[strlen(v)] = '\0';
-                } else if (strcmp (k, "checkpoint_time_utc") == 0) {
-                        memcpy (sts_val->checkpoint_time_utc, v,
-                                strlen(v));
-                        sts_val->checkpoint_time_utc[strlen(v)] = '\0';
-                } else if (strcmp (k, "checkpoint_completed") == 0) {
-                        memcpy (sts_val->checkpoint_completed, v,
-                                strlen(v));
-                        sts_val->checkpoint_completed[strlen(v)] = '\0';
-                } else if (strcmp (k, "checkpoint_completion_time") == 0) {
-                        memcpy (sts_val->checkpoint_completion_time, v,
-                                strlen(v));
-                        sts_val->checkpoint_completion_time[strlen(v)] = '\0';
-                } else if (strcmp (k, "checkpoint_completion_time_utc") == 0) {
-                        memcpy (sts_val->checkpoint_completion_time_utc, v,
-                                strlen(v));
-                        sts_val->checkpoint_completion_time_utc[strlen(v)] =
-                                '\0';
-                }
-        }
+    char *ptr = NULL;
+    char *v = NULL;
+    char *k = NULL;
+    gf_gsync_status_t *sts_val = NULL;
+    size_t len = 0;
 
-        return errno ? -1 : 0;
-}
+    sts_val = (gf_gsync_status_t *)data;
 
+    for (;;) {
+        errno = 0;
+        ptr = fgets(resbuf, blen - 2, fp);
+        if (!ptr)
+            break;
+
+        v = resbuf + strlen(resbuf) - 1;
+        while (isspace(*v))
+            /* strip trailing space */
+            *v-- = '\0';
+        if (v == resbuf)
+            /* skip empty line */
+            continue;
+        v = strchr(resbuf, ':');
+        if (!v)
+            return -1;
+        *v++ = '\0';
+        while (isspace(*v))
+            v++;
+        v = gf_strdup(v);
+        if (!v)
+            return -1;
+
+        k = gf_strdup(resbuf);
+        if (!k) {
+            GF_FREE(v);
+            return -1;
+        }
+
+        if (strcmp(k, "worker_status") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->worker_status) - 1));
+            memcpy(sts_val->worker_status, v, len);
+            sts_val->worker_status[len] = '\0';
+        } else if (strcmp(k, "slave_node") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->slave_node) - 1));
+            memcpy(sts_val->slave_node, v, len);
+            sts_val->slave_node[len] = '\0';
+        } else if (strcmp(k, "crawl_status") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->crawl_status) - 1));
+            memcpy(sts_val->crawl_status, v, len);
+            sts_val->crawl_status[len] = '\0';
+        } else if (strcmp(k, "last_synced") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->last_synced) - 1));
+            memcpy(sts_val->last_synced, v, len);
+            sts_val->last_synced[len] = '\0';
+        } else if (strcmp(k, "last_synced_utc") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->last_synced_utc) - 1));
+            memcpy(sts_val->last_synced_utc, v, len);
+            sts_val->last_synced_utc[len] = '\0';
+        } else if (strcmp(k, "entry") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->entry) - 1));
+            memcpy(sts_val->entry, v, len);
+            sts_val->entry[len] = '\0';
+        } else if (strcmp(k, "data") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->data) - 1));
+            memcpy(sts_val->data, v, len);
+            sts_val->data[len] = '\0';
+        } else if (strcmp(k, "meta") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->meta) - 1));
+            memcpy(sts_val->meta, v, len);
+            sts_val->meta[len] = '\0';
+        } else if (strcmp(k, "failures") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->failures) - 1));
+            memcpy(sts_val->failures, v, len);
+            sts_val->failures[len] = '\0';
+        } else if (strcmp(k, "checkpoint_time") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->checkpoint_time) - 1));
+            memcpy(sts_val->checkpoint_time, v, len);
+            sts_val->checkpoint_time[len] = '\0';
+        } else if (strcmp(k, "checkpoint_time_utc") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->checkpoint_time_utc) - 1));
+            memcpy(sts_val->checkpoint_time_utc, v, len);
+            sts_val->checkpoint_time_utc[len] = '\0';
+        } else if (strcmp(k, "checkpoint_completed") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->checkpoint_completed) - 1));
+            memcpy(sts_val->checkpoint_completed, v, len);
+            sts_val->checkpoint_completed[len] = '\0';
+        } else if (strcmp(k, "checkpoint_completion_time") == 0) {
+            len = min(strlen(v),
+                      (sizeof(sts_val->checkpoint_completion_time) - 1));
+            memcpy(sts_val->checkpoint_completion_time, v, len);
+            sts_val->checkpoint_completion_time[len] = '\0';
+        } else if (strcmp(k, "checkpoint_completion_time_utc") == 0) {
+            len = min(strlen(v),
+                      (sizeof(sts_val->checkpoint_completion_time_utc) - 1));
+            memcpy(sts_val->checkpoint_completion_time_utc, v, len);
+            sts_val->checkpoint_completion_time_utc[len] = '\0';
+        }
+        GF_FREE(v);
+        GF_FREE(k);
+    }
+
+    return errno ? -1 : 0;
+}
 
 static int
-glusterd_gsync_get_status (char *master, char *slave, char *conf_path,
-                           char *brick_path, gf_gsync_status_t *sts_val)
+glusterd_gsync_get_status(char *master, char *slave, char *conf_path,
+                          char *brick_path, gf_gsync_status_t *sts_val)
 {
-        /* key + value, where value must be able to accommodate a path */
-        char resbuf[256 + PATH_MAX] = {0,};
-        runner_t             runner = {0,};
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--status-get", NULL);
-        runner_add_args  (&runner, "--path", brick_path, NULL);
-
-        return glusterd_query_extutil_generic (resbuf, sizeof (resbuf),
-                                               &runner, sts_val,
-                                               _fcbk_statustostruct);
+    /* key + value, where value must be able to accommodate a path */
+    char resbuf[256 + PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    set_gsyncd_inet6_arg(&runner);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, "--status-get", NULL);
+    runner_add_args(&runner, "--path", brick_path, NULL);
+
+    return glusterd_query_extutil_generic(resbuf, sizeof(resbuf), &runner,
+                                          sts_val, _fcbk_statustostruct);
 }
 
 static int
-glusterd_gsync_get_param_file (char *prmfile, const char *param, char *master,
-                               char *slave, char *conf_path)
+glusterd_gsync_get_param_file(char *prmfile, const char *param, char *master,
+                              char *slave, char *conf_path)
 {
-        runner_t            runner = {0,};
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get", NULL);
-        runner_argprintf (&runner, "%s-file", param);
-
-        return glusterd_query_extutil (prmfile, &runner);
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    set_gsyncd_inet6_arg(&runner);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, "--config-get", NULL);
+    runner_argprintf(&runner, "%s-file", param);
+
+    return glusterd_query_extutil(prmfile, &runner);
 }
 
 static int
-gsyncd_getpidfile (char *master, char *slave, char *pidfile,
-                   char *conf_path, gf_boolean_t *is_template_in_use)
+gsyncd_getpidfile(char *master, char *slave, char *pidfile, char *conf_path,
+                  gf_boolean_t *is_template_in_use)
 {
-        char                temp_conf_path[PATH_MAX] = "";
-        char               *working_conf_path        = NULL;
-        glusterd_conf_t    *priv                     = NULL;
-        int                 ret                      = -1;
-        struct stat         stbuf                    = {0,};
-        xlator_t            *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (this->private);
-        GF_ASSERT (conf_path);
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO ("gsync", master, out);
-        GF_VALIDATE_OR_GOTO ("gsync", slave, out);
-
-        snprintf (temp_conf_path, sizeof(temp_conf_path) - 1,
-                  "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
-
-        ret = sys_lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_msg_debug (this->name, 0, "Using passed config template(%s).",
-                        conf_path);
-                working_conf_path = conf_path;
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, ENOENT,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Config file (%s) missing. Looking for template "
-                        "config file (%s)", conf_path, temp_conf_path);
-                ret = sys_lstat (temp_conf_path, &stbuf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Template config file (%s) missing.",
-                                temp_conf_path);
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
-                        "Using default config template(%s).",
-                        temp_conf_path);
-                working_conf_path = temp_conf_path;
-                *is_template_in_use = _gf_true;
+    char temp_conf_path[PATH_MAX] = "";
+    char *working_conf_path = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+    GF_ASSERT(conf_path);
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("gsync", master, out);
+    GF_VALIDATE_OR_GOTO("gsync", slave, out);
+
+    len = snprintf(temp_conf_path, sizeof(temp_conf_path),
+                   "%s/" GSYNC_CONF_TEMPLATE, priv->workdir);
+    if ((len < 0) || (len >= sizeof(temp_conf_path))) {
+        goto out;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "Using passed config template(%s).",
+                     conf_path);
+        working_conf_path = conf_path;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Config file (%s) missing. Looking for template "
+               "config file (%s)",
+               conf_path, temp_conf_path);
+        ret = sys_lstat(temp_conf_path, &stbuf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "Template config file (%s) missing.", temp_conf_path);
+            goto out;
         }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
+               "Using default config template(%s).", temp_conf_path);
+        working_conf_path = temp_conf_path;
+        *is_template_in_use = _gf_true;
+    }
 
 fetch_data:
 
-        ret = glusterd_gsync_get_param_file (pidfile, "pid", master,
-                                             slave, working_conf_path);
-        if ((ret == -1) || strlen(pidfile) == 0) {
-                if (*is_template_in_use == _gf_false) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_PIDFILE_CREATE_FAILED,
-                                "failed to create the pidfile string. "
-                                "Trying default config template");
-                        working_conf_path = temp_conf_path;
-                        *is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        ret = -2;
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_PIDFILE_CREATE_FAILED, "failed to "
-                                "create the pidfile string from template "
-                                "config");
-                        goto out;
-                }
+    ret = glusterd_gsync_get_param_file(pidfile, "pid", master, slave,
+                                        working_conf_path);
+    if ((ret == -1) || strlen(pidfile) == 0) {
+        if (*is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PIDFILE_CREATE_FAILED,
+                   "failed to create the pidfile string. "
+                   "Trying default config template");
+            working_conf_path = temp_conf_path;
+            *is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            ret = -2;
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PIDFILE_CREATE_FAILED,
+                   "failed to "
+                   "create the pidfile string from template "
+                   "config");
+            goto out;
         }
+    }
 
-        gf_msg_debug (this->name, 0, "pidfile = %s", pidfile);
+    gf_msg_debug(this->name, 0, "pidfile = %s", pidfile);
 
-        ret = open (pidfile, O_RDWR);
- out:
-        return ret;
+    ret = open(pidfile, O_RDWR);
+out:
+    return ret;
 }
 
 static int
-gsync_status_byfd (int fd)
+gsync_status_byfd(int fd)
 {
-        GF_ASSERT (fd >= -1);
+    GF_ASSERT(fd >= -1);
 
-        if (lockf (fd, F_TEST, 0) == -1 &&
-            (errno == EAGAIN || errno == EACCES))
-                /* gsyncd keeps the pidfile locked */
-                return 0;
+    if (lockf(fd, F_TEST, 0) == -1 && (errno == EAGAIN || errno == EACCES))
+        /* gsyncd keeps the pidfile locked */
+        return 0;
 
-        return -1;
+    return -1;
 }
 
 /* status: return 0 when gsync is running
  * return -1 when not running
  */
 int
-gsync_status (char *master, char *slave, char *conf_path,
-              int *status, gf_boolean_t *is_template_in_use)
+gsync_status(char *master, char *slave, char *conf_path, int *status,
+             gf_boolean_t *is_template_in_use)
 {
-        char pidfile[PATH_MAX] = {0,};
-        int  fd                = -1;
-
-        fd = gsyncd_getpidfile (master, slave, pidfile,
-                                conf_path, is_template_in_use);
-        if (fd == -2)
-                return -1;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    int fd = -1;
+
+    fd = gsyncd_getpidfile(master, slave, pidfile, conf_path,
+                           is_template_in_use);
+    if (fd == -2)
+        return -1;
 
-        *status = gsync_status_byfd (fd);
+    *status = gsync_status_byfd(fd);
 
-        sys_close (fd);
+    sys_close(fd);
 
-        return 0;
+    return 0;
 }
 
-
 static int32_t
-glusterd_gsync_volinfo_dict_set (glusterd_volinfo_t *volinfo,
-                                 char *key, char *value)
+glusterd_gsync_volinfo_dict_set(glusterd_volinfo_t *volinfo, char *key,
+                                char *value)
 {
-        int32_t  ret            = -1;
-        char    *gsync_status   = NULL;
-        xlator_t *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        gsync_status = gf_strdup (value);
-        if (!gsync_status) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
-                        "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = dict_set_dynstr (volinfo->dict, key, gsync_status);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Unable to set dict");
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *gsync_status = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    gsync_status = gf_strdup(value);
+    if (!gsync_status) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_set_dynstr(volinfo->dict, key, gsync_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set dict");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return 0;
+    return ret;
 }
 
 static int
-glusterd_verify_gsyncd_spawn (char *master, char *slave)
+glusterd_verify_gsyncd_spawn(char *master, char *slave)
 {
-        int                 ret = 0;
-        runner_t            runner = {0,};
-        xlator_t            *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd",
-                         "--verify", "spawning", NULL);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SPAWNING_CHILD_FAILED,
-                        "spawning child failed");
-                ret = -1;
-                goto out;
-        }
-
-        if (runner_end (&runner) != 0)
-                ret = -1;
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--verify", "spawning",
+                    NULL);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SPAWNING_CHILD_FAILED,
+               "spawning child failed");
+        ret = -1;
+        goto out;
+    }
+
+    if (runner_end(&runner) != 0)
+        ret = -1;
 
 out:
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 static int
-gsync_verify_config_options (dict_t *dict, char **op_errstr, char *volname)
+gsync_verify_config_options(dict_t *dict, char **op_errstr, char *volname)
 {
-        char                         **resopt           = NULL;
-        int                            i                = 0;
-        int                            ret              = -1;
-        char                          *subop            = NULL;
-        char                          *slave            = NULL;
-        char                          *op_name          = NULL;
-        char                          *op_value         = NULL;
-        char                          *t                = NULL;
-        char                           errmsg[PATH_MAX] = "";
-        gf_boolean_t                   banned           = _gf_true;
-        gf_boolean_t                   op_match         = _gf_true;
-        gf_boolean_t                   val_match        = _gf_true;
-        struct gsync_config_opt_vals_ *conf_vals        = NULL;
-        xlator_t                       *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (dict_get_str (dict, "subop", &subop) != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
-                        "missing subop");
-                *op_errstr = gf_strdup ("Invalid config request");
-                return -1;
-        }
-
-        if (dict_get_str (dict, "slave", &slave) != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
-                        GEOREP" CONFIG: no slave given");
-                *op_errstr = gf_strdup ("Slave required");
-                return -1;
-        }
-
-        if (strcmp (subop, "get-all") == 0)
-                return 0;
+    char **resopt = NULL;
+    int i = 0;
+    int ret = -1;
+    char *subop = NULL;
+    char *slave = NULL;
+    char *op_name = NULL;
+    char *op_value = NULL;
+    char *t = NULL;
+    char errmsg[PATH_MAX] = "";
+    gf_boolean_t banned = _gf_true;
+    gf_boolean_t op_match = _gf_true;
+    gf_boolean_t val_match = _gf_true;
+    struct gsync_config_opt_vals_ *conf_vals = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (dict_get_str(dict, "subop", &subop) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "missing subop");
+        *op_errstr = gf_strdup("Invalid config request");
+        return -1;
+    }
 
-        if (dict_get_str (dict, "op_name", &op_name) != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
-                        "option name missing");
-                *op_errstr = gf_strdup ("Option name missing");
-                return -1;
-        }
+    if (dict_get_str(dict, "slave", &slave) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               GEOREP " CONFIG: no slave given");
+        *op_errstr = gf_strdup("Slave required");
+        return -1;
+    }
 
-        if (runcmd (GSYNCD_PREFIX"/gsyncd", "--config-check", op_name, NULL)) {
-                ret = glusterd_verify_gsyncd_spawn (volname, slave);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GSYNCD_SPAWN_FAILED, "Unable to spawn "
-                                "gsyncd");
-                        return 0;
-                }
+    if (strcmp(subop, "get-all") == 0)
+        return 0;
 
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        GD_MSG_INVALID_ENTRY,
-                        "Invalid option %s", op_name);
-                *op_errstr = gf_strdup ("Invalid option");
+    if (dict_get_str(dict, "op_name", &op_name) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "option name missing");
+        *op_errstr = gf_strdup("Option name missing");
+        return -1;
+    }
 
-                return -1;
+    if (runcmd(GSYNCD_PREFIX "/gsyncd", "--config-check", op_name, NULL)) {
+        ret = glusterd_verify_gsyncd_spawn(volname, slave);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED,
+                   "Unable to spawn "
+                   "gsyncd");
+            return 0;
         }
 
-        if (strcmp (subop, "get") == 0)
-                return 0;
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid option %s", op_name);
+        *op_errstr = gf_strdup("Invalid option");
 
-        t = strtail (subop, "set");
-        if (!t)
-                t = strtail (subop, "del");
-        if (!t || (t[0] && strcmp (t, "-glob") != 0)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_SUBOP_NOT_FOUND,
-                        "unknown subop %s", subop);
-                *op_errstr = gf_strdup ("Invalid config request");
-                return -1;
-        }
+        return -1;
+    }
 
-        if (strtail (subop, "set") &&
-            dict_get_str (dict, "op_value", &op_value) != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
-                        "missing value for set");
-                *op_errstr = gf_strdup ("missing value");
-        }
+    if (strcmp(subop, "get") == 0)
+        return 0;
 
-        /* match option name against reserved options, modulo -/_
-         * difference
-         */
-        for (resopt = gsync_reserved_opts; *resopt; resopt++) {
-                banned = _gf_true;
-                for (i = 0; (*resopt)[i] && op_name[i]; i++) {
-                        if ((*resopt)[i] == op_name[i] ||
-                            ((*resopt)[i] == '-' && op_name[i] == '_'))
-                                continue;
-                        banned = _gf_false;
-                }
-                if (banned) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_RESERVED_OPTION,
-                                "Reserved option %s", op_name);
-                        *op_errstr = gf_strdup ("Reserved option");
-
-                        return -1;
-                        break;
+    t = strtail(subop, "set");
+    if (!t)
+        t = strtail(subop, "del");
+    if (!t || (t[0] && strcmp(t, "-glob") != 0)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SUBOP_NOT_FOUND,
+               "unknown subop %s", subop);
+        *op_errstr = gf_strdup("Invalid config request");
+        return -1;
+    }
+
+    if (strtail(subop, "set") &&
+        dict_get_str(dict, "op_value", &op_value) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "missing value for set");
+        *op_errstr = gf_strdup("missing value");
+    }
+
+    /* match option name against reserved options, modulo -/_
+     * difference
+     */
+    for (resopt = gsync_reserved_opts; *resopt; resopt++) {
+        banned = _gf_true;
+        for (i = 0; (*resopt)[i] && op_name[i]; i++) {
+            if ((*resopt)[i] == op_name[i] ||
+                ((*resopt)[i] == '-' && op_name[i] == '_'))
+                continue;
+            banned = _gf_false;
+        }
+
+        if (op_name[i] != '\0')
+            banned = _gf_false;
+
+        if (banned) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_RESERVED_OPTION,
+                   "Reserved option %s", op_name);
+            *op_errstr = gf_strdup("Reserved option");
+
+            return -1;
+            break;
+        }
+    }
+
+    /* Check options in gsync_confopt_vals for invalid values */
+    for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
+        op_match = _gf_true;
+        for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
+            if (conf_vals->op_name[i] == op_name[i] ||
+                (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
+                continue;
+            op_match = _gf_false;
+        }
+
+        if (op_match) {
+            if (!op_value)
+                goto out;
+            val_match = _gf_false;
+            for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
+                if (conf_vals->case_sensitive) {
+                    if (!strcmp(conf_vals->values[i], op_value))
+                        val_match = _gf_true;
+                } else {
+                    if (!strcasecmp(conf_vals->values[i], op_value))
+                        val_match = _gf_true;
                 }
-        }
+            }
 
-        /* Check options in gsync_confopt_vals for invalid values */
-        for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
-                op_match = _gf_true;
-                for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
-                        if (conf_vals->op_name[i] == op_name[i] ||
-                            (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
-                                continue;
-                        op_match = _gf_false;
-                }
+            if (!val_match) {
+                ret = snprintf(errmsg, sizeof(errmsg) - 1,
+                               "Invalid value(%s) for"
+                               " option %s",
+                               op_value, op_name);
+                errmsg[ret] = '\0';
 
-                if (op_match) {
-                        if (!op_value)
-                                goto out;
-                        val_match = _gf_false;
-                        for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
-                                if(conf_vals->case_sensitive){
-                                        if (!strcmp (conf_vals->values[i], op_value))
-                                                val_match = _gf_true;
-                                } else {
-                                        if (!strcasecmp (conf_vals->values[i], op_value))
-                                                val_match = _gf_true;
-                                }
-                        }
-
-                        if (!val_match) {
-                                ret = snprintf (errmsg, sizeof(errmsg) - 1,
-                                                "Invalid value(%s) for"
-                                                " option %s", op_value,
-                                                op_name);
-                                errmsg[ret] = '\0';
-
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_INVALID_ENTRY, "%s", errmsg);
-                                *op_errstr = gf_strdup (errmsg);
-                                return -1;
-                        }
-                }
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "%s", errmsg);
+                *op_errstr = gf_strdup(errmsg);
+                return -1;
+            }
         }
+    }
 out:
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
-                                   char *slave, char *conf_path,
-                                   dict_t *rsp_dict, char *node);
+glusterd_get_gsync_status_mst_slv(glusterd_volinfo_t *volinfo, char *slave,
+                                  char *conf_path, dict_t *rsp_dict,
+                                  char *node);
 
 static int
-_get_status_mst_slv (dict_t *dict, char *key, data_t *value, void *data)
+_get_status_mst_slv(dict_t *dict, char *key, data_t *value, void *data)
 {
-        glusterd_gsync_status_temp_t *param                = NULL;
-        char                         *slave                = NULL;
-        char                         *slave_buf            = NULL;
-        char                         *slave_url            = NULL;
-        char                         *slave_vol            = NULL;
-        char                         *slave_host           = NULL;
-        char                         *errmsg               = NULL;
-        char                           conf_path[PATH_MAX] = "";
-        int                           ret                  = -1;
-        glusterd_conf_t              *priv                 = NULL;
-        xlator_t                     *this                 = NULL;
-        char                         slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        param = (glusterd_gsync_status_temp_t *)data;
-
-        GF_VALIDATE_OR_GOTO (this->name, param, out);
-        GF_VALIDATE_OR_GOTO (this->name, param->volinfo, out);
-
-        if (this)
-                priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        strncpy (slv_url, value->data, sizeof(slv_url));
-        ret = parse_slave_url (slv_url, &slave);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVE_VOL_PARSE_FAIL,
-                        "Error in parsing slave: %s!", value->data);
-                goto out;
-        }
+    glusterd_gsync_status_temp_t *param = NULL;
+    char *slave = NULL;
+    char *slave_buf = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *errmsg = NULL;
+    char conf_path[PATH_MAX] = "";
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    param = (glusterd_gsync_status_temp_t *)data;
+
+    GF_VALIDATE_OR_GOTO(this->name, param, out);
+    GF_VALIDATE_OR_GOTO(this->name, param->volinfo, out);
+
+    if (this)
+        priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (snprintf(slv_url, sizeof(slv_url), "%s", value->data) >=
+        sizeof(slv_url)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in copying slave: %s!", value->data);
+        goto out;
+    }
+
+    ret = parse_slave_url(slv_url, &slave);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in parsing slave: %s!", value->data);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_info(slave, &slave_url, &slave_host, &slave_vol,
+                                  &errmsg);
+    if (ret) {
+        if (errmsg)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch slave details. Error: %s", errmsg);
+        else
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch slave details.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                   "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", priv->workdir,
+                   param->volinfo->volname, slave_host, slave_vol);
+    conf_path[ret] = '\0';
+
+    ret = glusterd_get_gsync_status_mst_slv(param->volinfo, slave, conf_path,
+                                            param->rsp_dict, param->node);
+out:
 
-        ret = glusterd_get_slave_info (slave, &slave_url,
-                                       &slave_host, &slave_vol, &errmsg);
-        if (ret) {
-                if (errmsg)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVEINFO_FETCH_ERROR,
-                                "Unable to fetch slave details. Error: %s",
-                                errmsg);
-                else
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVEINFO_FETCH_ERROR,
-                                "Unable to fetch slave details.");
-                ret = -1;
-                goto out;
-        }
+    if (errmsg)
+        GF_FREE(errmsg);
 
-        ret = snprintf (conf_path, sizeof(conf_path) - 1,
-                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                        priv->workdir, param->volinfo->volname,
-                        slave_host, slave_vol);
-        conf_path[ret] = '\0';
+    if (slave_buf)
+        GF_FREE(slave_buf);
 
-        ret = glusterd_get_gsync_status_mst_slv(param->volinfo,
-                                                slave, conf_path,
-                                                param->rsp_dict,
-                                                param->node);
-out:
+    if (slave_vol)
+        GF_FREE(slave_vol);
 
-        if (errmsg)
-                GF_FREE (errmsg);
+    if (slave_url)
+        GF_FREE(slave_url);
 
-        if (slave_buf)
-                GF_FREE(slave_buf);
+    if (slave_host)
+        GF_FREE(slave_host);
 
-        gf_msg_debug (this->name, 0, "Returning %d.", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d.", ret);
+    return ret;
 }
 
-
 static int
-_get_max_gsync_slave_num (dict_t *dict, char *key, data_t *value, void *data)
+_get_max_gsync_slave_num(dict_t *dict, char *key, data_t *value, void *data)
 {
-        int  tmp_slvnum = 0;
-        int *slvnum = (int *)data;
+    int tmp_slvnum = 0;
+    int *slvnum = (int *)data;
 
-        sscanf (key, "slave%d", &tmp_slvnum);
-        if (tmp_slvnum > *slvnum)
-                *slvnum = tmp_slvnum;
+    sscanf(key, "slave%d", &tmp_slvnum);
+    if (tmp_slvnum > *slvnum)
+        *slvnum = tmp_slvnum;
 
-        return 0;
+    return 0;
 }
 
 static int
-_get_slave_idx_slave_voluuid (dict_t *dict, char *key, data_t *value,
-                              void *data)
+_get_slave_idx_slave_voluuid(dict_t *dict, char *key, data_t *value, void *data)
 {
-        char  *slave_voluuid                 = NULL;
-        char  *slave_info                    = NULL;
-        xlator_t  *this                      = NULL;
-        struct slave_vol_config *slave_cfg   = NULL;
-        int  i                               = 0;
-        int  ret                             = -1;
-        unsigned  tmp_slvnum                 = 0;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        slave_cfg = data;
-
-        if (value)
-                slave_info = value->data;
-
-        if (!(slave_info) || strlen (slave_info) == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_SLAVE,
-                        "Invalid slave in dict");
-                ret = -2;
-                goto out;
-        }
+    char *slave_info = NULL;
+    xlator_t *this = NULL;
+    struct slave_vol_config *slave_cfg = NULL;
+    int i = 0;
+    int ret = -1;
+    unsigned tmp_slvnum = 0;
 
-        /* slave format:
-         * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
-        while (i++ < 5) {
-                slave_info = strchr (slave_info, ':');
-                if (slave_info)
-                        slave_info++;
-                else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVE_VOL_PARSE_FAIL,
-                                "slave_info becomes NULL!");
-                        ret = -2;
-                        goto out;
-                }
-        }
-        if (strcmp (slave_info, slave_cfg->slave_voluuid) == 0) {
-                gf_msg_debug (this->name, 0, "Same slave volume "
-                              "already present %s",
-                              slave_cfg->slave_voluuid);
-                ret = -1;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-                sscanf (key, "slave%d", &tmp_slvnum);
-                slave_cfg->old_slvidx = tmp_slvnum;
+    slave_cfg = data;
 
-                gf_msg_debug (this->name, 0, "and "
-                              "its index is: %d", tmp_slvnum);
-                goto out;
-        }
+    if (value)
+        slave_info = value->data;
 
-        ret = 0;
+    if (!(slave_info) || strlen(slave_info) == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_SLAVE,
+               "Invalid slave in dict");
+        ret = -2;
+        goto out;
+    }
+
+    /* slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    while (i++ < 5) {
+        slave_info = strchr(slave_info, ':');
+        if (slave_info)
+            slave_info++;
+        else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                   "slave_info becomes NULL!");
+            ret = -2;
+            goto out;
+        }
+    }
+    if (strcmp(slave_info, slave_cfg->slave_voluuid) == 0) {
+        gf_msg_debug(this->name, 0,
+                     "Same slave volume "
+                     "already present %s",
+                     slave_cfg->slave_voluuid);
+        ret = -1;
+
+        sscanf(key, "slave%d", &tmp_slvnum);
+        slave_cfg->old_slvidx = tmp_slvnum;
+
+        gf_msg_debug(this->name, 0,
+                     "and "
+                     "its index is: %d",
+                     tmp_slvnum);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_remove_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
-                               char **op_errstr)
+glusterd_remove_slave_in_info(glusterd_volinfo_t *volinfo, char *slave,
+                              char **op_errstr)
 {
-        int   zero_slave_entries = _gf_true;
-        int   ret = 0;
-        char *slavekey = NULL;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-
-        do {
-                ret = glusterd_get_slave (volinfo, slave, &slavekey);
-                if (ret < 0 && zero_slave_entries) {
-                        ret++;
-                        goto out;
-                }
-                zero_slave_entries = _gf_false;
-                dict_del (volinfo->gsync_slaves, slavekey);
-        } while (ret >= 0);
-
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                 *op_errstr = gf_strdup ("Failed to store the Volume"
-                                         "information");
-                goto out;
-        }
- out:
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        return ret;
-
+    int zero_slave_entries = _gf_true;
+    int ret = 0;
+    char *slavekey = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+
+    do {
+        ret = glusterd_get_slave(volinfo, slave, &slavekey);
+        if (ret < 0 && zero_slave_entries) {
+            ret++;
+            goto out;
+        }
+        zero_slave_entries = _gf_false;
+        dict_del(volinfo->gsync_slaves, slavekey);
+    } while (ret >= 0);
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        *op_errstr = gf_strdup(
+            "Failed to store the Volume"
+            "information");
+        goto out;
+    }
+out:
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_gsync_get_uuid (char *slave, glusterd_volinfo_t *vol,
-                         uuid_t uuid)
+glusterd_gsync_get_uuid(char *slave, glusterd_volinfo_t *vol, uuid_t uuid)
 {
-        int   ret = 0;
-        char *slavekey = NULL;
-        char *slaveentry = NULL;
-        char *t = NULL;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (vol);
-        GF_ASSERT (slave);
-
-        ret = glusterd_get_slave (vol, slave, &slavekey);
-        if (ret < 0) {
-                /* XXX colliding cases of failure and non-extant
-                 * slave... now just doing this as callers of this
-                 * function can make sense only of -1 and 0 as retvals;
-                 * getting at the proper semanticals will involve
-                 * fixing callers as well.
-                 */
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    char *slavekey = NULL;
+    char *slaveentry = NULL;
+    char *t = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(vol);
+    GF_ASSERT(slave);
+
+    ret = glusterd_get_slave(vol, slave, &slavekey);
+    if (ret < 0) {
+        /* XXX colliding cases of failure and non-extant
+         * slave... now just doing this as callers of this
+         * function can make sense only of -1 and 0 as retvals;
+         * getting at the proper semanticals will involve
+         * fixing callers as well.
+         */
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (vol->gsync_slaves, slavekey, &slaveentry);
-        GF_ASSERT (ret == 0);
+    ret = dict_get_str(vol->gsync_slaves, slavekey, &slaveentry);
+    GF_ASSERT(ret == 0);
 
-        t = strchr (slaveentry, ':');
-        GF_ASSERT (t);
-        *t = '\0';
-        ret = gf_uuid_parse (slaveentry, uuid);
-        *t = ':';
+    t = strchr(slaveentry, ':');
+    GF_ASSERT(t);
+    *t = '\0';
+    ret = gf_uuid_parse(slaveentry, uuid);
+    *t = ':';
 
- out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-update_slave_voluuid (dict_t *dict, char *key, data_t *value, void *data)
+update_slave_voluuid(dict_t *dict, char *key, data_t *value, void *data)
 {
-        char                         *slave                = NULL;
-        char                         *slave_url            = NULL;
-        char                         *slave_vol            = NULL;
-        char                         *slave_host           = NULL;
-        char                         *errmsg               = NULL;
-        xlator_t                     *this                 = NULL;
-        int                          ret                   = -1;
-        char                         slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
-        char                         slave_voluuid[GF_UUID_BUF_SIZE] = {0};
-        char                         *slave_info           = NULL;
-        char                         *new_value            = NULL;
-        char                         *same_key             = NULL;
-        int                          cnt                   = 0;
-        gf_boolean_t                 *voluuid_updated      = NULL;
-
-        this = THIS;
-
-        voluuid_updated = data;
-        slave_info = value->data;
-        gf_msg_debug (this->name, 0, "slave_info: %s!", slave_info);
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *errmsg = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    char slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
+    char slave_voluuid[GF_UUID_BUF_SIZE] = {0};
+    char *slave_info = NULL;
+    char *new_value = NULL;
+    char *same_key = NULL;
+    int cnt = 0;
+    gf_boolean_t *voluuid_updated = NULL;
+
+    this = THIS;
+
+    voluuid_updated = data;
+    slave_info = value->data;
+    gf_msg_debug(this->name, 0, "slave_info: %s!", slave_info);
+
+    /* old slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol
+     * New slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    while (slave_info) {
+        slave_info = strchr(slave_info, ':');
+        if (slave_info)
+            cnt++;
+        else
+            break;
 
-        /* old slave format:
-         * master_node_uuid:ssh://slave_host::slave_vol
-         * New slave format:
-         * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
-        while (slave_info) {
-                slave_info = strchr (slave_info, ':');
-                if (slave_info)
-                        cnt++;
-                else
-                        break;
+        slave_info++;
+    }
 
-                slave_info++;
+    gf_msg_debug(this->name, 0, "cnt: %d", cnt);
+    /* check whether old slave format and update vol uuid if old format.
+     * With volume uuid, number of ':' is 5 and is 4 without.
+     */
+    if (cnt == 4) {
+        if (snprintf(slv_url, sizeof(slv_url), "%s", value->data) >=
+            sizeof(slv_url)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                   "Error in copying slave: %s!", value->data);
+            goto out;
         }
 
-        gf_msg_debug (this->name, 0, "cnt: %d", cnt);
-        /* check whether old slave format and update vol uuid if old format.
-         * With volume uuid, number of ':' is 5 and is 4 without.
-         */
-        if (cnt == 4) {
-                strncpy (slv_url, value->data, sizeof(slv_url));
-
-                ret = parse_slave_url (slv_url, &slave);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVE_VOL_PARSE_FAIL,
-                                "Error in parsing slave: %s!", value->data);
-                        goto out;
-                }
+        ret = parse_slave_url(slv_url, &slave);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                   "Error in parsing slave: %s!", value->data);
+            goto out;
+        }
 
-                ret = glusterd_get_slave_info (slave, &slave_url,
-                                       &slave_host, &slave_vol, &errmsg);
-                if (ret) {
-                        if (errmsg)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SLAVEINFO_FETCH_ERROR,
-                                        "Unable to fetch slave details. Error: %s",
-                                        errmsg);
-                        else
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SLAVEINFO_FETCH_ERROR,
-                                        "Unable to fetch slave details.");
-                        ret = -1;
-                        goto out;
-                }
+        ret = glusterd_get_slave_info(slave, &slave_url, &slave_host,
+                                      &slave_vol, &errmsg);
+        if (ret) {
+            if (errmsg)
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SLAVEINFO_FETCH_ERROR,
+                       "Unable to fetch slave details. Error: %s", errmsg);
+            else
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SLAVEINFO_FETCH_ERROR,
+                       "Unable to fetch slave details.");
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_get_slave_voluuid(slave_host, slave_vol, slave_voluuid);
+        if ((ret) || (strlen(slave_voluuid) == 0)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+                   "Unable to get remote volume uuid"
+                   "slavehost:%s slavevol:%s",
+                   slave_host, slave_vol);
+            /* Avoiding failure due to remote vol uuid fetch */
+            ret = 0;
+            goto out;
+        }
+        ret = gf_asprintf(&new_value, "%s:%s", value->data, slave_voluuid);
+        ret = gf_asprintf(&same_key, "%s", key);
+
+        /* delete old key and add new value */
+        dict_del(dict, key);
+
+        /* set new value for the same key*/
+        ret = dict_set_dynstr(dict, same_key, new_value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+                   "Error in setting dict value"
+                   "new_value :%s",
+                   new_value);
+            goto out;
+        }
+        *voluuid_updated = _gf_true;
+    }
 
-                ret = glusterd_get_slave_voluuid (slave_host, slave_vol,
-                                                  slave_voluuid);
-                if ((ret) || (strlen(slave_voluuid) == 0)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_REMOTE_VOL_UUID_FAIL,
-                                "Unable to get remote volume uuid"
-                                "slavehost:%s slavevol:%s",
-                                 slave_host, slave_vol);
-                        /* Avoiding failure due to remote vol uuid fetch */
-                        ret = 0;
-                        goto out;
-                }
-                ret = gf_asprintf (&new_value,  "%s:%s",
-                                    value->data, slave_voluuid);
-                ret = gf_asprintf (&same_key, "%s", key);
+    ret = 0;
+out:
+    if (errmsg)
+        GF_FREE(errmsg);
 
-                /* delete old key and add new value */
-                dict_del (dict, key);
+    if (slave_url)
+        GF_FREE(slave_url);
 
-                /* set new value for the same key*/
-                ret = dict_set_dynstr (dict, same_key, new_value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_REMOTE_VOL_UUID_FAIL,
-                                "Error in setting dict value"
-                                "new_value :%s", new_value);
-                        goto out;
-                }
-                *voluuid_updated = _gf_true;
-        }
+    if (slave_vol)
+        GF_FREE(slave_vol);
 
-        ret = 0;
-out:
-        if (errmsg)
-                GF_FREE (errmsg);
+    if (slave_host)
+        GF_FREE(slave_host);
 
-        gf_msg_debug (this->name, 0, "Returning %d.", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d.", ret);
+    return ret;
 }
 
 static int
-glusterd_update_slave_voluuid_slaveinfo (glusterd_volinfo_t *volinfo)
+glusterd_update_slave_voluuid_slaveinfo(glusterd_volinfo_t *volinfo)
 {
-        int   ret = -1;
-        xlator_t *this = NULL;
-        gf_boolean_t voluuid_updated = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-
-        ret = dict_foreach (volinfo->gsync_slaves, update_slave_voluuid,
-                            &voluuid_updated);
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_boolean_t voluuid_updated = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+
+    ret = dict_foreach(volinfo->gsync_slaves, update_slave_voluuid,
+                       &voluuid_updated);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+               "Error in updating"
+               "volinfo");
+        goto out;
+    }
+
+    if (_gf_true == voluuid_updated) {
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REMOTE_VOL_UUID_FAIL, "Error in updating"
-                        "volinfo");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                   "Error in storing"
+                   "volinfo");
+            goto out;
         }
+    }
 
-        if (_gf_true == voluuid_updated) {
-                ret = glusterd_store_volinfo (volinfo,
-                                          GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_STORE_FAIL, "Error in storing"
-                                "volinfo");
-                        goto out;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug((this ? this->name : "glusterd"), 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_check_gsync_running_local (char *master, char *slave,
-                                    char *conf_path,
-                                    gf_boolean_t *is_run)
+glusterd_check_gsync_running_local(char *master, char *slave, char *conf_path,
+                                   gf_boolean_t *is_run)
 {
-        int                 ret    = -1;
-        int                 ret_status = 0;
-        gf_boolean_t        is_template_in_use = _gf_false;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (master);
-        GF_ASSERT (slave);
-        GF_ASSERT (is_run);
-
-        *is_run = _gf_false;
-        ret = gsync_status (master, slave, conf_path,
-                            &ret_status, &is_template_in_use);
-        if (ret == 0 && ret_status == 0)
-                *is_run = _gf_true;
-        else if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_VALIDATE_FAILED,
-                        GEOREP" validation failed");
-                goto out;
-        }
-        ret = 0;
- out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
-
+    int ret = -1;
+    int ret_status = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(master);
+    GF_ASSERT(slave);
+    GF_ASSERT(is_run);
+
+    *is_run = _gf_false;
+    ret = gsync_status(master, slave, conf_path, &ret_status,
+                       &is_template_in_use);
+    if (ret == 0 && ret_status == 0)
+        *is_run = _gf_true;
+    else if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VALIDATE_FAILED,
+               GEOREP " validation failed");
+        goto out;
+    }
+    ret = 0;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
-                              char *host_uuid, char *slave_voluuid,
-                              char **op_errstr, gf_boolean_t is_force)
+glusterd_store_slave_in_info(glusterd_volinfo_t *volinfo, char *slave,
+                             char *host_uuid, char *slave_voluuid,
+                             char **op_errstr, gf_boolean_t is_force)
 {
-        int    ret = 0;
-        int    maxslv = 0;
-        char **linearr = NULL;
-        char  *value = NULL;
-        char  *slavekey = NULL;
-        char  *slaveentry = NULL;
-        char   key[512] = {0, };
-        char  *t = NULL;
-        xlator_t *this = NULL;
-        struct slave_vol_config slave1  = {{0},};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (host_uuid);
-        GF_VALIDATE_OR_GOTO (this->name, slave_voluuid, out);
-
-        ret = glusterd_get_slave (volinfo, slave, &slavekey);
-        switch (ret) {
+    int ret = 0;
+    int maxslv = 0;
+    char **linearr = NULL;
+    char *value = NULL;
+    char *slavekey = NULL;
+    char *slaveentry = NULL;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *t = NULL;
+    xlator_t *this = NULL;
+    struct slave_vol_config slave1 = {
+        {0},
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(host_uuid);
+    GF_VALIDATE_OR_GOTO(this->name, slave_voluuid, out);
+
+    ret = glusterd_get_slave(volinfo, slave, &slavekey);
+    switch (ret) {
         case -2:
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         case -1:
-                                break;
+            break;
         default:
-                if (!is_force)
-                        GF_ASSERT (ret > 0);
-                ret = dict_get_str (volinfo->gsync_slaves, slavekey, &slaveentry);
-                GF_ASSERT (ret == 0);
-
-                /* same-name + same-uuid slave entries should have been filtered
-                 * out in glusterd_op_verify_gsync_start_options(), so we can
-                 * assert an uuid mismatch
-                 */
-                t = strtail (slaveentry, host_uuid);
-                if (!is_force)
-                        GF_ASSERT (!t || *t != ':');
-
-                if (is_force) {
-                        gf_msg_debug (this->name, 0, GEOREP" has already "
-                                      "been invoked for the %s (master) and "
-                                      "%s (slave). Allowing without saving "
-                                      "info again due to force command.",
-                                      volinfo->volname, slave);
-                        ret = 0;
-                        goto out;
-                }
-
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_INVOKE_ERROR,
-                        GEOREP" has already been invoked for "
-                         "the %s (master) and %s (slave) from a different "
-                         "machine", volinfo->volname, slave);
-                *op_errstr = gf_strdup (GEOREP" already running in "
-                                        "another machine");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_urltransform_single (slave, "normalize", &linearr);
-        if (ret == -1)
-                goto out;
-
-        ret = gf_asprintf (&value,  "%s:%s:%s", host_uuid,
-                                    linearr[0], slave_voluuid);
-
-        glusterd_urltransform_free (linearr, 1);
-        if (ret  == -1)
+            if (!is_force)
+                GF_ASSERT(ret > 0);
+            ret = dict_get_str(volinfo->gsync_slaves, slavekey, &slaveentry);
+            GF_ASSERT(ret == 0);
+
+            /* same-name + same-uuid slave entries should have been filtered
+             * out in glusterd_op_verify_gsync_start_options(), so we can
+             * assert an uuid mismatch
+             */
+            t = strtail(slaveentry, host_uuid);
+            if (!is_force)
+                GF_ASSERT(!t || *t != ':');
+
+            if (is_force) {
+                gf_msg_debug(this->name, 0,
+                             GEOREP
+                             " has already "
+                             "been invoked for the %s (master) and "
+                             "%s (slave). Allowing without saving "
+                             "info again due to force command.",
+                             volinfo->volname, slave);
+                ret = 0;
                 goto out;
+            }
 
-        /* Given the slave volume uuid, check and get any existing slave */
-        strncpy (slave1.slave_voluuid, slave_voluuid, GF_UUID_BUF_SIZE);
-        ret = dict_foreach (volinfo->gsync_slaves,
-                            _get_slave_idx_slave_voluuid, &slave1);
-
-        if (ret == 0) { /* New slave */
-                dict_foreach (volinfo->gsync_slaves, _get_max_gsync_slave_num,
-                              &maxslv);
-                snprintf (key, 512, "slave%d", maxslv + 1);
-
-                ret = dict_set_dynstr (volinfo->gsync_slaves, key, value);
-                if (ret)
-                        goto out;
-        } else if (ret == -1) { /* Existing slave */
-                snprintf (key, 512, "slave%d", slave1.old_slvidx);
-
-                /* Delete present slave info(with old hostname) */
-                dict_del (volinfo->gsync_slaves, key);
-
-                gf_msg_debug (this->name, 0, "Replacing key:%s with new value"
-                              ":%s", key, value);
-
-                /* Add new slave's value, with the same slave index */
-                ret = dict_set_dynstr (volinfo->gsync_slaves, key, value);
-                if (ret)
-                        goto out;
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REMOTE_VOL_UUID_FAIL,
-                        "_get_slave_idx_slave_voluuid failed!");
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVOKE_ERROR,
+                   GEOREP
+                   " has already been invoked for "
+                   "the %s (master) and %s (slave) from a different "
+                   "machine",
+                   volinfo->volname, slave);
+            *op_errstr = gf_strdup(GEOREP
+                                   " already running in "
+                                   "another machine");
+            ret = -1;
+            goto out;
+    }
+
+    ret = glusterd_urltransform_single(slave, "normalize", &linearr);
+    if (ret == -1)
+        goto out;
+
+    ret = gf_asprintf(&value, "%s:%s:%s", host_uuid, linearr[0], slave_voluuid);
+
+    glusterd_urltransform_free(linearr, 1);
+    if (ret == -1)
+        goto out;
+
+    /* Given the slave volume uuid, check and get any existing slave */
+    memcpy(slave1.slave_voluuid, slave_voluuid, UUID_CANONICAL_FORM_LEN);
+    ret = dict_foreach(volinfo->gsync_slaves, _get_slave_idx_slave_voluuid,
+                       &slave1);
+
+    if (ret == 0) { /* New slave */
+        dict_foreach(volinfo->gsync_slaves, _get_max_gsync_slave_num, &maxslv);
+        keylen = snprintf(key, sizeof(key), "slave%d", maxslv + 1);
+
+        ret = dict_set_dynstrn(volinfo->gsync_slaves, key, keylen, value);
+        if (ret) {
+            GF_FREE(value);
+            goto out;
         }
+    } else if (ret == -1) { /* Existing slave */
+        keylen = snprintf(key, sizeof(key), "slave%d", slave1.old_slvidx);
+
+        gf_msg_debug(this->name, 0,
+                     "Replacing key:%s with new value"
+                     ":%s",
+                     key, value);
 
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        /* Add new slave's value, with the same slave index */
+        ret = dict_set_dynstrn(volinfo->gsync_slaves, key, keylen, value);
         if (ret) {
-                *op_errstr = gf_strdup ("Failed to store the Volume "
-                                        "information");
-                goto out;
-        }
-        ret = 0;
- out:
-        return ret;
+            GF_FREE(value);
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+               "_get_slave_idx_slave_voluuid failed!");
+        GF_FREE(value);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        *op_errstr = gf_strdup(
+            "Failed to store the Volume "
+            "information");
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
 }
 
 static int
-glusterd_op_verify_gsync_start_options (glusterd_volinfo_t *volinfo,
-                                        char *slave, char *conf_path,
-                                        char *statefile, char **op_errstr,
-                                        gf_boolean_t is_force)
+glusterd_op_verify_gsync_start_options(glusterd_volinfo_t *volinfo, char *slave,
+                                       char *conf_path, char *statefile,
+                                       char **op_errstr, gf_boolean_t is_force)
 {
-        int                     ret = -1;
-        int                     ret_status = 0;
-        gf_boolean_t            is_template_in_use = _gf_false;
-        char                    msg[2048] = {0};
-        uuid_t                  uuid = {0};
-        glusterd_conf_t        *priv = NULL;
-        xlator_t               *this = NULL;
-        struct stat             stbuf = {0,};
-        char                    statefiledir[PATH_MAX] = {0,};
-        char                    *statedir = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (conf_path);
-        GF_ASSERT (this && this->private);
-
-        priv  = this->private;
-
-        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                snprintf (msg, sizeof (msg), "Volume %s needs to be started "
-                          "before "GEOREP" start", volinfo->volname);
-                goto out;
-        }
-
-        /* check session directory as statefile may not present
-         * during upgrade */
-        strncpy (statefiledir, statefile, sizeof(statefiledir));
-        statedir = dirname (statefiledir);
-
-        ret = sys_lstat (statedir, &stbuf);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Session between %s and %s has"
-                          " not been created. Please create session and retry.",
-                          volinfo->volname, slave);
-                gf_msg (this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
-                        "%s statefile: %s", msg, statefile);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        /* Check if the gsync slave info is stored. If not
-         * session has not been created */
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Session between %s and %s has"
-                          " not been created. Please create session and retry.",
-                          volinfo->volname, slave);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SESSION_CREATE_ERROR,
-                        "%s", msg);
-                goto out;
-        }
-
-        /*Check if the gsync is already started in cmd. inited host
-         * If so initiate add it into the glusterd's priv*/
-        ret = gsync_status (volinfo->volname, slave, conf_path,
-                            &ret_status, &is_template_in_use);
-        if (ret == 0) {
-                if ((ret_status == 0) && !is_force) {
-                        snprintf (msg, sizeof (msg), GEOREP " session between"
-                                  " %s & %s already started", volinfo->volname,
-                                  slave);
-                        ret = -1;
-                        goto out;
-                }
-        } else if (ret == -1) {
-                snprintf (msg, sizeof (msg), GEOREP" start option "
-                          "validation failed ");
-                goto out;
-        }
-
-        if (is_template_in_use == _gf_true) {
-                snprintf (msg, sizeof (msg), GEOREP" start "
-                          "failed : pid-file entry missing "
-                          "in config file.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
-        if (ret && !is_force) {
-                snprintf (msg, sizeof (msg), "Unable to spawn gsyncd");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED,
-                        "%s", msg);
-        }
+    int ret = -1;
+    int ret_status = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    char msg[2048] = {0};
+    uuid_t uuid = {0};
+    xlator_t *this = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    char statefiledir[PATH_MAX] = {
+        0,
+    };
+    char *statedir = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(conf_path);
+    GF_ASSERT(this && this->private);
+
+    if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s needs to be started "
+                 "before " GEOREP " start",
+                 volinfo->volname);
+        goto out;
+    }
+
+    /* check session directory as statefile may not present
+     * during upgrade */
+    if (snprintf(statefiledir, sizeof(statefiledir), "%s", statefile) >=
+        sizeof(statefiledir)) {
+        snprintf(msg, sizeof(msg), "statefiledir truncated");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, "%s",
+               msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+    statedir = dirname(statefiledir);
+
+    ret = sys_lstat(statedir, &stbuf);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Session between %s and %s has"
+                 " not been created. Please create session and retry.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "%s statefile: %s", msg, statefile);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    /* Check if the gsync slave info is stored. If not
+     * session has not been created */
+    ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Session between %s and %s has"
+                 " not been created. Please create session and retry.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_CREATE_ERROR, "%s",
+               msg);
+        goto out;
+    }
+
+    /*Check if the gsync is already started in cmd. inited host
+     * If so initiate add it into the glusterd's priv*/
+    ret = gsync_status(volinfo->volname, slave, conf_path, &ret_status,
+                       &is_template_in_use);
+    if (ret == 0) {
+        if ((ret_status == 0) && !is_force) {
+            snprintf(msg, sizeof(msg),
+                     GEOREP
+                     " session between"
+                     " %s & %s already started",
+                     volinfo->volname, slave);
+            ret = -1;
+            goto out;
+        }
+    } else if (ret == -1) {
+        snprintf(msg, sizeof(msg),
+                 GEOREP
+                 " start option "
+                 "validation failed ");
+        goto out;
+    }
+
+    if (is_template_in_use == _gf_true) {
+        snprintf(msg, sizeof(msg),
+                 GEOREP
+                 " start "
+                 "failed : pid-file entry missing "
+                 "in config file.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_verify_gsyncd_spawn(volinfo->volname, slave);
+    if (ret && !is_force) {
+        snprintf(msg, sizeof(msg), "Unable to spawn gsyncd");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED, "%s",
+               msg);
+    }
 out:
-        if (ret && (msg[0] != '\0')) {
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && (msg[0] != '\0')) {
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 void
-glusterd_check_geo_rep_configured (glusterd_volinfo_t *volinfo,
-                                   gf_boolean_t *flag)
+glusterd_check_geo_rep_configured(glusterd_volinfo_t *volinfo,
+                                  gf_boolean_t *flag)
 {
+    GF_ASSERT(volinfo);
+    GF_ASSERT(flag);
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (flag);
-
-        if (volinfo->gsync_slaves->count)
-                *flag = _gf_true;
-        else
-                *flag = _gf_false;
+    if (volinfo->gsync_slaves->count)
+        *flag = _gf_true;
+    else
+        *flag = _gf_false;
 
-        return;
+    return;
 }
 
 /*
@@ -1912,70 +2021,70 @@ glusterd_check_geo_rep_configured (glusterd_volinfo_t *volinfo,
  */
 
 static int
-is_geo_rep_active (glusterd_volinfo_t *volinfo, char *slave,
-                   char *conf_path, int *is_active)
+is_geo_rep_active(glusterd_volinfo_t *volinfo, char *slave, char *conf_path,
+                  int *is_active)
 {
-        dict_t                 *confd                      = NULL;
-        char                   *statefile                  = NULL;
-        char                   *master                     = NULL;
-        char                    monitor_status[PATH_MAX]   = "";
-        int                     ret                        = -1;
-        xlator_t               *this                       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        master = volinfo->volname;
-
-        confd = dict_new ();
-        if (!confd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
-                        "Not able to create dict.");
-                goto out;
-        }
-
-        ret = glusterd_gsync_get_config (master, slave, conf_path,
-                                         confd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GET_CONFIG_INFO_FAILED,
-                        "Unable to get configuration data "
-                        "for %s(master), %s(slave)", master, slave);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_param (confd, "state_file", &statefile);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to get state_file's name "
-                        "for %s(master), %s(slave). Please check gsync "
-                        "config file.", master, slave);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                              sizeof (monitor_status));
-        if (ret <= 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STAT_FILE_READ_FAILED,
-                        "Unable to read the status file for %s(master), "
-                        "%s(slave)", master, slave);
-                strncpy (monitor_status, "defunct", sizeof (monitor_status));
-        }
-
-        if ((!strcmp(monitor_status, "Stopped")) ||
-            (!strcmp(monitor_status, "Created"))) {
-                *is_active = 0;
-        } else {
-                *is_active = 1;
-        }
-        ret = 0;
+    dict_t *confd = NULL;
+    char *statefile = NULL;
+    char *master = NULL;
+    char monitor_status[PATH_MAX] = "";
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    master = volinfo->volname;
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Not able to create dict.");
+        goto out;
+    }
+
+    ret = glusterd_gsync_get_config(master, slave, conf_path, confd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "Unable to get configuration data "
+               "for %s(master), %s(slave)",
+               master, slave);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_param(confd, "state_file", &statefile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get state_file's name "
+               "for %s(master), %s(slave). Please check gsync "
+               "config file.",
+               master, slave);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAT_FILE_READ_FAILED,
+               "Unable to read the status file for %s(master), "
+               "%s(slave)",
+               master, slave);
+        snprintf(monitor_status, sizeof(monitor_status), "defunct");
+    }
+
+    if ((!strcmp(monitor_status, "Stopped")) ||
+        (!strcmp(monitor_status, "Created"))) {
+        *is_active = 0;
+    } else {
+        *is_active = 1;
+    }
+    ret = 0;
 out:
-        if (confd)
-                dict_destroy (confd);
-        return ret;
+    if (confd)
+        dict_unref(confd);
+    return ret;
 }
 
 /*
@@ -1991,81 +2100,86 @@ out:
  */
 
 int
-_get_slave_status (dict_t *dict, char *key, data_t *value, void *data)
+_get_slave_status(dict_t *dict, char *key, data_t *value, void *data)
 {
-        gsync_status_param_t *param                        = NULL;
-        char                 *slave                        = NULL;
-        char                 *slave_url                    = NULL;
-        char                 *slave_vol                    = NULL;
-        char                 *slave_host                   = NULL;
-        char                 *errmsg                       = NULL;
-        char                           conf_path[PATH_MAX] = "";
-        int                   ret                          = -1;
-        glusterd_conf_t      *priv                         = NULL;
-        xlator_t             *this                         = NULL;
-
-        param = (gsync_status_param_t *)data;
-
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
+    gsync_status_param_t *param = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *errmsg = NULL;
+    char conf_path[PATH_MAX] = "";
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    param = (gsync_status_param_t *)data;
+
+    GF_ASSERT(param);
+    GF_ASSERT(param->volinfo);
+    if (param->is_active) {
+        ret = 0;
+        goto out;
+    }
 
-        if (param->is_active) {
-                ret = 0;
-                goto out;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        this = THIS;
-        GF_ASSERT (this);
+    priv = this->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        goto out;
+    }
 
-        if (this)
-                priv = this->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                goto out;
-        }
+    slave = strchr(value->data, ':');
+    if (!slave) {
+        ret = 0;
+        goto out;
+    }
+    slave++;
 
-        slave = strchr(value->data, ':');
-        if (!slave) {
-                ret = 0;
-                goto out;
-        }
-        slave++;
+    ret = glusterd_get_slave_info(slave, &slave_url, &slave_host, &slave_vol,
+                                  &errmsg);
+    if (ret) {
+        if (errmsg)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch"
+                   " slave details. Error: %s",
+                   errmsg);
+        else
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch slave details.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                   "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", priv->workdir,
+                   param->volinfo->volname, slave_host, slave_vol);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CONF_PATH_ASSIGN_FAILED,
+               "Unable to assign conf_path.");
+        ret = -1;
+        goto out;
+    }
+    conf_path[ret] = '\0';
+
+    ret = is_geo_rep_active(param->volinfo, slave, conf_path,
+                            &param->is_active);
+out:
+    if (errmsg)
+        GF_FREE(errmsg);
 
-        ret = glusterd_get_slave_info (slave, &slave_url,
-                                       &slave_host, &slave_vol, &errmsg);
-        if (ret) {
-                if (errmsg)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVEINFO_FETCH_ERROR, "Unable to fetch"
-                                " slave details. Error: %s", errmsg);
-                else
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVEINFO_FETCH_ERROR,
-                                "Unable to fetch slave details.");
-                ret = -1;
-                goto out;
-        }
+    if (slave_vol)
+        GF_FREE(slave_vol);
 
-        ret = snprintf (conf_path, sizeof(conf_path) - 1,
-                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                        priv->workdir, param->volinfo->volname,
-                        slave_host, slave_vol);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CONF_PATH_ASSIGN_FAILED,
-                        "Unable to assign conf_path.");
-                ret = -1;
-                goto out;
-        }
-        conf_path[ret] = '\0';
+    if (slave_url)
+        GF_FREE(slave_url);
+    if (slave_host)
+        GF_FREE(slave_host);
 
-        ret = is_geo_rep_active (param->volinfo,slave, conf_path,
-                                 &param->is_active);
-out:
-        GF_FREE(errmsg);
-        return ret;
+    return ret;
 }
 
 /* glusterd_check_geo_rep_running:
@@ -2079,829 +2193,946 @@ out:
  */
 
 int
-glusterd_check_geo_rep_running (gsync_status_param_t *param, char **op_errstr)
+glusterd_check_geo_rep_running(gsync_status_param_t *param, char **op_errstr)
 {
-        char                    msg[2048]   = {0,};
-        gf_boolean_t            enabled     = _gf_false;
-        int                     ret         = 0;
-        xlator_t               *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
-        GF_ASSERT (op_errstr);
-
-        glusterd_check_geo_rep_configured (param->volinfo, &enabled);
-
-        if (enabled) {
-                ret = dict_foreach (param->volinfo->gsync_slaves,
-                                    _get_slave_status, param);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVEINFO_FETCH_ERROR,
-                                "_get_slave_satus failed");
-                        snprintf (msg, sizeof(msg), GEOREP" Unable to"
-                                  " get the status of active "GEOREP""
-                                  " session for the volume '%s'.\n"
-                                  " Please check the log file for"
-                                  " more info.", param->volinfo->volname);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
+    char msg[2048] = {
+        0,
+    };
+    gf_boolean_t enabled = _gf_false;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(param);
+    GF_ASSERT(param->volinfo);
+    GF_ASSERT(op_errstr);
+
+    glusterd_check_geo_rep_configured(param->volinfo, &enabled);
+
+    if (enabled) {
+        ret = dict_foreach(param->volinfo->gsync_slaves, _get_slave_status,
+                           param);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "_get_slave_satus failed");
+            snprintf(msg, sizeof(msg),
+                     GEOREP
+                     " Unable to"
+                     " get the status of active " GEOREP
+                     ""
+                     " session for the volume '%s'.\n"
+                     " Please check the log file for"
+                     " more info.",
+                     param->volinfo->volname);
+            *op_errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
 
-                if (param->is_active) {
-                        snprintf (msg, sizeof(msg), GEOREP" sessions"
-                                  " are active for the volume %s.\nStop"
-                                  " "GEOREP " sessions involved in this"
-                                  " volume. Use 'volume "GEOREP
-                                  " status' command for more info.",
-                                  param->volinfo->volname);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                 }
-         }
- out:
-        return ret;
+        if (param->is_active) {
+            snprintf(msg, sizeof(msg),
+                     GEOREP
+                     " sessions"
+                     " are active for the volume %s.\nStop"
+                     " " GEOREP
+                     " sessions involved in this"
+                     " volume. Use 'volume " GEOREP
+                     " status' command for more info.",
+                     param->volinfo->volname);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+    }
+out:
+    return ret;
 }
 
 static int
-glusterd_op_verify_gsync_running (glusterd_volinfo_t *volinfo,
-                                  char *slave, char *conf_path,
-                                  char **op_errstr)
+glusterd_op_verify_gsync_running(glusterd_volinfo_t *volinfo, char *slave,
+                                 char *conf_path, char **op_errstr)
 {
-        int        pfd               = -1;
-        int        ret               = -1;
-        char       msg[2048]         = {0};
-        char       pidfile[PATH_MAX] = {0,};
-        gf_boolean_t is_template_in_use = _gf_false;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (THIS && THIS->private);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (conf_path);
-        GF_ASSERT (op_errstr);
-
-        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                snprintf (msg, sizeof (msg), "Volume %s needs to be started "
-                          "before "GEOREP" start", volinfo->volname);
-
-                goto out;
-        }
-
-        pfd = gsyncd_getpidfile (volinfo->volname, slave, pidfile,
-                                 conf_path, &is_template_in_use);
-        if (pfd == -2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
-                        GEOREP" stop validation failed for %s & %s",
-                        volinfo->volname, slave);
-                ret = -1;
-                goto out;
-        }
-        if (gsync_status_byfd (pfd) == -1) {
-                snprintf (msg, sizeof (msg), GEOREP" session b/w %s & %s is "
-                          "not running on this node.", volinfo->volname,
-                          slave);
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_INACTIVE,
-                        "%s", msg);
-                ret = -1;
-                /* monitor gsyncd already dead */
-                goto out;
-        }
-
-        if (is_template_in_use) {
-                snprintf (msg, sizeof (msg), "pid-file entry missing in "
-                          "the config file(%s).", conf_path);
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND,
-                        "%s", msg);
-                ret = -1;
-                goto out;
-        }
-
-        if (pfd < 0)
-                goto out;
-
-        ret = 0;
+    int pfd = -1;
+    int ret = -1;
+    char msg[2048] = {0};
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    gf_boolean_t is_template_in_use = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(THIS && THIS->private);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(conf_path);
+    GF_ASSERT(op_errstr);
+
+    if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s needs to be started "
+                 "before " GEOREP " start",
+                 volinfo->volname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_GEO_REP_START_FAILED,
+                "Volume is not in a started state, Volname=%s",
+                volinfo->volname, NULL);
+
+        goto out;
+    }
+
+    pfd = gsyncd_getpidfile(volinfo->volname, slave, pidfile, conf_path,
+                            &is_template_in_use);
+    if (pfd == -2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               GEOREP " stop validation failed for %s & %s", volinfo->volname,
+               slave);
+        ret = -1;
+        goto out;
+    }
+    if (gsync_status_byfd(pfd) == -1) {
+        snprintf(msg, sizeof(msg),
+                 GEOREP
+                 " session b/w %s & %s is "
+                 "not running on this node.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_INACTIVE, "%s", msg);
+        ret = -1;
+        /* monitor gsyncd already dead */
+        goto out;
+    }
+
+    if (is_template_in_use) {
+        snprintf(msg, sizeof(msg),
+                 "pid-file entry missing in "
+                 "the config file(%s).",
+                 conf_path);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND, "%s",
+               msg);
+        ret = -1;
+        goto out;
+    }
+
+    if (pfd < 0)
+        goto out;
+
+    ret = 0;
 out:
-        if (ret && (msg[0] != '\0')) {
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && (msg[0] != '\0')) {
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_verify_gsync_status_opts (dict_t *dict, char **op_errstr)
+glusterd_verify_gsync_status_opts(dict_t *dict, char **op_errstr)
 {
-        char               *slave           = NULL;
-        char               *volname         = NULL;
-        char               errmsg[PATH_MAX] = {0, };
-        gf_boolean_t        exists          = _gf_false;
-        glusterd_volinfo_t *volinfo         = NULL;
-        int                 ret             = 0;
-        char               *conf_path       = NULL;
-        char               *slave_url       = NULL;
-        char               *slave_host      = NULL;
-        char               *slave_vol       = NULL;
-        glusterd_conf_t    *priv            = NULL;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "master", &volname);
-        if (ret < 0) {
-                ret = 0;
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
-                        "volume name does not exist");
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVEINFO_FETCH_ERROR,
-                        "Unable to fetch slave  or confpath details.");
-                ret = -1;
-                goto out;
-        }
+    char *slave = NULL;
+    char *volname = NULL;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    char *conf_path = NULL;
+    char *slave_url = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "master", &volname);
+    if (ret < 0) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume name does not exist");
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        *op_errstr = gf_strdup(errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(volinfo, dict, &slave_url,
+                                              &slave_host, &slave_vol,
+                                              &conf_path, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave  or confpath details.");
+        ret = -1;
+        goto out;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int
-glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
-                            char **master, char **slave, char **host_uuid)
+glusterd_op_gsync_args_get(dict_t *dict, char **op_errstr, char **master,
+                           char **slave, char **host_uuid)
 {
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        int             ret = -1;
-        xlator_t        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
 
-        if (master) {
-                ret = dict_get_str (dict, "master", master);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_DICT_GET_FAILED, "master not found");
-                        *op_errstr = gf_strdup ("master not found");
-                        goto out;
-                }
+    if (master) {
+        ret = dict_get_str(dict, "master", master);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "master not found");
+            *op_errstr = gf_strdup("master not found");
+            goto out;
         }
+    }
 
-        if (slave) {
-                ret = dict_get_str (dict, "slave", slave);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_DICT_GET_FAILED, "slave not found");
-                        *op_errstr = gf_strdup ("slave not found");
-                        goto out;
-                }
+    if (slave) {
+        ret = dict_get_str(dict, "slave", slave);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "slave not found");
+            *op_errstr = gf_strdup("slave not found");
+            goto out;
         }
+    }
 
-        if (host_uuid) {
-                ret = dict_get_str (dict, "host-uuid", host_uuid);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_DICT_GET_FAILED, "host_uuid not found");
-                        *op_errstr = gf_strdup ("host_uuid not found");
-                        goto out;
-                }
+    if (host_uuid) {
+        ret = dict_get_str(dict, "host-uuid", host_uuid);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "host_uuid not found");
+            *op_errstr = gf_strdup("host_uuid not found");
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr)
+glusterd_op_stage_sys_exec(dict_t *dict, char **op_errstr)
 {
-        char             errmsg[PATH_MAX]          = "";
-        char            *command                   = NULL;
-        char             command_path[PATH_MAX]    = "";
-        struct stat      st                        = {0,};
-        int              ret                       = -1;
-        glusterd_conf_t *conf                      = NULL;
-        xlator_t        *this                      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (conf->op_version < 2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
-                        "Op Version not supported.");
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "command", &command);
-        if (ret) {
-                strcpy (errmsg, "internal error");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to get command from dict");
-                goto out;
-        }
-
-        /* enforce local occurrence of the command */
-        if (strchr (command, '/')) {
-                strcpy (errmsg, "invalid command name");
-                ret = -1;
-                goto out;
-        }
-
-        sprintf (command_path, GSYNCD_PREFIX"/peer_%s", command);
-        /* check if it's executable */
-        ret = sys_access (command_path, X_OK);
-        if (!ret)
-                /* check if it's a regular file */
-                ret = sys_stat (command_path, &st);
-        if (!ret && !S_ISREG (st.st_mode))
-                ret = -1;
+    char errmsg[PATH_MAX] = "";
+    char *command = NULL;
+    char command_path[PATH_MAX] = "";
+    struct stat st = {
+        0,
+    };
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (conf->op_version < 2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "Op Version not supported.");
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "command", &command);
+    if (ret) {
+        strcpy(errmsg, "internal error");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command from dict");
+        goto out;
+    }
+
+    /* enforce local occurrence of the command */
+    if (strchr(command, '/')) {
+        strcpy(errmsg, "invalid command name");
+        ret = -1;
+        goto out;
+    }
+
+    sprintf(command_path, GSYNCD_PREFIX "/peer_%s", command);
+    /* check if it's executable */
+    ret = sys_access(command_path, X_OK);
+    if (!ret)
+        /* check if it's a regular file */
+        ret = sys_stat(command_path, &st);
+    if (!ret && !S_ISREG(st.st_mode))
+        ret = -1;
 
 out:
-        if (ret) {
-                if (errmsg[0] == '\0') {
-                        if (command)
-                                snprintf (errmsg, sizeof (errmsg),
-                                          "gsync peer_%s command not found.",
-                                          command);
-                        else
-                                snprintf (errmsg, sizeof (errmsg), "%s",
-                                          "gsync peer command was not "
-                                          "specified");
-                }
-                *op_errstr = gf_strdup (errmsg);
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_CMD_ERROR,
-                        "%s", errmsg);
-        }
-
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret) {
+        if (errmsg[0] == '\0') {
+            if (command)
+                snprintf(errmsg, sizeof(errmsg),
+                         "gsync peer_%s command not found.", command);
+            else
+                snprintf(errmsg, sizeof(errmsg), "%s",
+                         "gsync peer command was not "
+                         "specified");
+        }
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_CMD_ERROR, "%s",
+               errmsg);
+    }
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr)
+glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr)
 {
-        char             abs_filename[PATH_MAX] = "";
-        char             errmsg[PATH_MAX]       = "";
-        char            *filename               = NULL;
-        char            *host_uuid              = NULL;
-        char             uuid_str [64]          = {0};
-        int              ret                    = -1;
-        glusterd_conf_t *priv                   = NULL;
-        struct stat      stbuf                  = {0,};
-        xlator_t         *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
-        }
-
-        if (priv->op_version < 2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
-                        "Op Version not supported.");
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "host-uuid", &host_uuid);
+    char abs_filename[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    char *filename = NULL;
+    char *host_uuid = NULL;
+    char uuid_str[64] = {0};
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    char realpath_filename[PATH_MAX] = {
+        0,
+    };
+    char realpath_workdir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    if (priv->op_version < 2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "Op Version not supported.");
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "host-uuid", &host_uuid);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch host-uuid from dict.");
+        goto out;
+    }
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = dict_get_str(dict, "source", &filename);
         if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch host-uuid from dict.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch filename from dict.");
+            *op_errstr = gf_strdup("command unsuccessful");
+            goto out;
+        }
+        len = snprintf(abs_filename, sizeof(abs_filename), "%s/%s",
+                       priv->workdir, filename);
+        if ((len < 0) || (len >= sizeof(abs_filename))) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+
+        if (!realpath(priv->workdir, realpath_workdir)) {
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Failed to "
+                           "get realpath of %s: %s",
+                           priv->workdir, strerror(errno));
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_REALPATH_GET_FAIL,
+                    "Realpath=%s, Reason=%s", priv->workdir, strerror(errno),
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
+        }
+
+        if (!realpath(abs_filename, realpath_filename)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Failed to get "
+                     "realpath of %s: %s",
+                     filename, strerror(errno));
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_REALPATH_GET_FAIL,
+                    "Filename=%s, Reason=%s", filename, strerror(errno), NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
+        }
+
+        /* Add Trailing slash to workdir, without slash strncmp
+           will succeed for /var/lib/glusterd_bad */
+        len = snprintf(workdir, sizeof(workdir), "%s/", realpath_workdir);
+        if ((len < 0) || (len >= sizeof(workdir))) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+
+        /* Protect against file copy outside $workdir */
+        if (strncmp(workdir, realpath_filename, strlen(workdir))) {
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Source file"
+                           " is outside of %s directory",
+                           priv->workdir);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg,
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
         }
 
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = dict_get_str (dict, "source", &filename);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to fetch filename from dict.");
-                        *op_errstr = gf_strdup ("command unsuccessful");
-                        goto out;
-                }
-                snprintf (abs_filename, sizeof(abs_filename),
-                          "%s/%s", priv->workdir, filename);
-
-                ret = sys_lstat (abs_filename, &stbuf);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Source file"
-                                  " does not exist in %s", priv->workdir);
-                        *op_errstr = gf_strdup (errmsg);
-                        goto out;
-                }
+        ret = sys_lstat(abs_filename, &stbuf);
+        if (ret) {
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Source file"
+                           " does not exist in %s",
+                           priv->workdir);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg,
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            goto out;
+        }
 
-                if (!S_ISREG(stbuf.st_mode)) {
-                        snprintf (errmsg, sizeof (errmsg), "Source file"
-                                 " is not a regular file.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SRC_FILE_ERROR,
-                                "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
+        if (!S_ISREG(stbuf.st_mode)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Source file"
+                     " is not a regular file.");
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg,
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
-                             char *conf_path, char **statefile,
-                             gf_boolean_t *is_template_in_use)
+glusterd_get_statefile_name(glusterd_volinfo_t *volinfo, char *slave,
+                            char *conf_path, char **statefile,
+                            gf_boolean_t *is_template_in_use)
 {
-        char            *master                   = NULL;
-        char            *buf                      = NULL;
-        char            *working_conf_path        = NULL;
-        char             temp_conf_path[PATH_MAX] = "";
-        dict_t          *confd                    = NULL;
-        glusterd_conf_t *priv                     = NULL;
-        int              ret                      = -1;
-        struct stat      stbuf                    = {0,};
-        xlator_t        *this                     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (this->private);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (conf_path);
-        GF_ASSERT (is_template_in_use);
-
-        master = volinfo->volname;
-
-        confd = dict_new ();
-        if (!confd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
-                        "Unable to create new dict");
-                goto out;
-        }
-
-        priv = THIS->private;
-
-        snprintf (temp_conf_path, sizeof(temp_conf_path) - 1,
-                  "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
-
-        ret = sys_lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_CONFIG_INFO,
-                        "Using passed config template(%s).",
-                        conf_path);
-                working_conf_path = conf_path;
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, ENOENT,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Config file (%s) missing. Looking for template config"
-                        " file (%s)", conf_path, temp_conf_path);
-                ret = sys_lstat (temp_conf_path, &stbuf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                                GD_MSG_FILE_OP_FAILED, "Template "
-                                "config file (%s) missing.", temp_conf_path);
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
-                        "Using default config template(%s).", temp_conf_path);
-                working_conf_path = temp_conf_path;
-                *is_template_in_use = _gf_true;
-        }
-
-fetch_data:
-        ret = glusterd_gsync_get_config (master, slave, working_conf_path,
-                                         confd);
+    char *master = NULL;
+    char *buf = NULL;
+    char *working_conf_path = NULL;
+    char temp_conf_path[PATH_MAX] = "";
+    dict_t *confd = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(conf_path);
+    GF_ASSERT(is_template_in_use);
+
+    master = volinfo->volname;
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create new dict");
+        goto out;
+    }
+
+    priv = THIS->private;
+
+    len = snprintf(temp_conf_path, sizeof(temp_conf_path),
+                   "%s/" GSYNC_CONF_TEMPLATE, priv->workdir);
+    if ((len < 0) || (len >= sizeof(temp_conf_path))) {
+        goto out;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONFIG_INFO,
+               "Using passed config template(%s).", conf_path);
+        working_conf_path = conf_path;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Config file (%s) missing. Looking for template config"
+               " file (%s)",
+               conf_path, temp_conf_path);
+        ret = sys_lstat(temp_conf_path, &stbuf);
         if (ret) {
-                if (*is_template_in_use == _gf_false) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GET_CONFIG_INFO_FAILED,
-                                "Unable to get configuration data "
-                                "for %s(master), %s(slave). "
-                                "Trying template config.",
-                                master, slave);
-                        working_conf_path = temp_conf_path;
-                        *is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GET_CONFIG_INFO_FAILED,
-                                "Unable to get configuration data "
-                                "for %s(master), %s(slave) from "
-                                "template config",
-                                master, slave);
-                        goto out;
-                }
-        }
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "Template "
+                   "config file (%s) missing.",
+                   temp_conf_path);
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
+               "Using default config template(%s).", temp_conf_path);
+        working_conf_path = temp_conf_path;
+        *is_template_in_use = _gf_true;
+    }
 
-        ret = dict_get_param (confd, "state_file", &buf);
-        if (ret) {
-                if (*is_template_in_use == _gf_false) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get state_file's name. "
-                                "Trying template config.");
-                        working_conf_path = temp_conf_path;
-                        *is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GET_STATEFILE_NAME_FAILED,
-                                "Unable to get state_file's "
-                                "name from template.");
-                        goto out;
-                }
+fetch_data:
+    ret = glusterd_gsync_get_config(master, slave, working_conf_path, confd);
+    if (ret) {
+        if (*is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+                   "Unable to get configuration data "
+                   "for %s(master), %s(slave). "
+                   "Trying template config.",
+                   master, slave);
+            working_conf_path = temp_conf_path;
+            *is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+                   "Unable to get configuration data "
+                   "for %s(master), %s(slave) from "
+                   "template config",
+                   master, slave);
+            goto out;
+        }
+    }
+
+    ret = dict_get_param(confd, "state_file", &buf);
+    if (ret) {
+        if (*is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get state_file's name. "
+                   "Trying template config.");
+            working_conf_path = temp_conf_path;
+            *is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GET_STATEFILE_NAME_FAILED,
+                   "Unable to get state_file's "
+                   "name from template.");
+            goto out;
         }
+    }
 
-        ret = 0;
- out:
-        if (buf) {
-                *statefile = gf_strdup(buf);
-                if (!*statefile)
-                        ret = -1;
-        }
+    ret = 0;
+out:
+    if (buf) {
+        *statefile = gf_strdup(buf);
+        if (!*statefile)
+            ret = -1;
+    }
 
-        if (confd)
-                dict_destroy (confd);
+    if (confd)
+        dict_unref(confd);
 
-        gf_msg_debug (this->name, 0, "Returning %d ", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_create_status_file (char *master, char *slave, char *slave_host,
-                             char *slave_vol, char *status)
+glusterd_create_status_file(char *master, char *slave, char *slave_host,
+                            char *slave_vol, char *status)
 {
-        int                ret    = -1;
-        runner_t           runner = {0,};
-        glusterd_conf_t   *priv   = NULL;
-        xlator_t          *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                goto out;
-        }
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        if (!status) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_STATUS_NULL,
-                        "Status Empty");
-                goto out;
-        }
-        gf_msg_debug (this->name, 0, "slave = %s", slave);
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--create",
-                         status, "-c", NULL);
-        runner_argprintf (&runner, "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                          priv->workdir, master, slave_host, slave_vol);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, NULL);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STATUSFILE_CREATE_FAILED,
-                        "Creating status file failed.");
-                ret = -1;
-                goto out;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        ret = 0;
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        goto out;
+    }
+
+    if (!status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATUS_NULL, "Status Empty");
+        goto out;
+    }
+    gf_msg_debug(this->name, 0, "slave = %s", slave);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--create", status, "-c",
+                    NULL);
+    runner_argprintf(&runner, "%s/" GEOREP "/%s_%s_%s/gsyncd.conf",
+                     priv->workdir, master, slave_host, slave_vol);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, NULL);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATUSFILE_CREATE_FAILED,
+               "Creating status file failed.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_verify_slave (char *volname, char *slave_url, char *slave_vol,
-                       int ssh_port, char **op_errstr,
-                       gf_boolean_t *is_force_blocker)
+glusterd_verify_slave(char *volname, char *slave_url, char *slave_vol,
+                      int ssh_port, char **op_errstr,
+                      gf_boolean_t *is_force_blocker)
 {
-        int32_t          ret                     = -1;
-        runner_t         runner                  = {0,};
-        char             log_file_path[PATH_MAX] = "";
-        char             buf[PATH_MAX]           = "";
-        char            *tmp                     = NULL;
-        char            *slave_url_buf           = NULL;
-        char            *save_ptr                = NULL;
-        char            *slave_user              = NULL;
-        char            *slave_ip                = NULL;
-        glusterd_conf_t *priv                    = NULL;
-        xlator_t        *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volname);
-        GF_ASSERT (slave_url);
-        GF_ASSERT (slave_vol);
-
-        /* Fetch the slave_user and slave_ip from the slave_url.
-         * If the slave_user is not present. Use "root"
-         */
-        if (strstr(slave_url, "@")) {
-                slave_url_buf = gf_strdup (slave_url);
-                if (!slave_url_buf)
-                        goto out;
-
-                slave_user = strtok_r (slave_url_buf, "@", &save_ptr);
-                slave_ip = strtok_r (NULL, "@", &save_ptr);
-        } else {
-                slave_user = "root";
-                slave_ip = slave_url;
-        }
-
-        if (!slave_user || !slave_ip) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
-                        "Invalid slave url.");
-                goto out;
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    char log_file_path[PATH_MAX] = "";
+    char buf[PATH_MAX] = "";
+    char *tmp = NULL;
+    char *slave_url_buf = NULL;
+    char *save_ptr = NULL;
+    char *slave_user = NULL;
+    char *slave_ip = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char *af = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volname);
+    GF_ASSERT(slave_url);
+    GF_ASSERT(slave_vol);
+
+    /* Fetch the slave_user and slave_ip from the slave_url.
+     * If the slave_user is not present. Use "root"
+     */
+    if (strstr(slave_url, "@")) {
+        slave_url_buf = gf_strdup(slave_url);
+        if (!slave_url_buf) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                    "Slave_url=%s", slave_url, NULL);
+            goto out;
+        }
+
+        slave_user = strtok_r(slave_url_buf, "@", &save_ptr);
+        slave_ip = strtok_r(NULL, "@", &save_ptr);
+    } else {
+        slave_user = "root";
+        slave_ip = slave_url;
+    }
+
+    if (!slave_user || !slave_ip) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+               "Invalid slave url.");
+        goto out;
+    }
+
+    snprintf(log_file_path, sizeof(log_file_path), "%s/create_verify_log",
+             priv->logdir);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gverify.sh", NULL);
+    runner_argprintf(&runner, "%s", volname);
+    runner_argprintf(&runner, "%s", slave_user);
+    runner_argprintf(&runner, "%s", slave_ip);
+    runner_argprintf(&runner, "%s", slave_vol);
+    runner_argprintf(&runner, "%d", ssh_port);
+    runner_argprintf(&runner, "%s", log_file_path);
+    ret = dict_get_str(this->options, "transport.address-family", &af);
+    if (ret)
+        af = "-";
+
+    runner_argprintf(&runner, "%s", af);
+
+    gf_msg_debug(this->name, 0, "gverify Args = %s %s %s %s %s %s %s %s",
+                 runner.argv[0], runner.argv[1], runner.argv[2], runner.argv[3],
+                 runner.argv[4], runner.argv[5], runner.argv[6],
+                 runner.argv[7]);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_SLAVE,
+               "Not a valid slave");
+        ret = glusterd_gsync_read_frm_status(log_file_path, buf, sizeof(buf));
+        if (ret <= 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_ERROR,
+                   "Unable to read from %s", log_file_path);
+            goto out;
         }
 
-        snprintf (log_file_path, sizeof(log_file_path),
-                  DEFAULT_LOG_FILE_DIRECTORY"/create_verify_log");
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gverify.sh", NULL);
-        runner_argprintf (&runner, "%s", volname);
-        runner_argprintf (&runner, "%s", slave_user);
-        runner_argprintf (&runner, "%s", slave_ip);
-        runner_argprintf (&runner, "%s", slave_vol);
-        runner_argprintf (&runner, "%d", ssh_port);
-        runner_argprintf (&runner, "%s", log_file_path);
-        gf_msg_debug (this->name, 0, "gverify Args = %s %s %s %s %s %s %s",
-                      runner.argv[0], runner.argv[1], runner.argv[2],
-                      runner.argv[3], runner.argv[4], runner.argv[5],
-                      runner.argv[6]);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_SLAVE,
-                        "Not a valid slave");
-                ret = glusterd_gsync_read_frm_status (log_file_path,
-                                                      buf, sizeof(buf));
-                if (ret <= 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_READ_ERROR,
-                                "Unable to read from %s", log_file_path);
-                        goto out;
-                }
-
-                /* Tokenize the error message from gverify.sh to figure out
-                 * if the error is a force blocker or not. */
-                tmp = strtok_r (buf, "|", &save_ptr);
-                if (!strcmp (tmp, "FORCE_BLOCKER"))
-                        *is_force_blocker = 1;
-                else {
-                        /* No FORCE_BLOCKER flag present so all that is
-                         * present is the error message. */
-                        *is_force_blocker = 0;
-                        if (tmp)
-                                *op_errstr = gf_strdup (tmp);
-                        ret = -1;
-                        goto out;
-                }
-
-                /* Copy rest of the error message to op_errstr */
-                tmp = strtok_r (NULL, "|", &save_ptr);
-                if (tmp)
-                        *op_errstr = gf_strdup (tmp);
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+        /* Tokenize the error message from gverify.sh to figure out
+         * if the error is a force blocker or not. */
+        tmp = strtok_r(buf, "|", &save_ptr);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(tmp, "FORCE_BLOCKER"))
+            *is_force_blocker = 1;
+        else {
+            /* No FORCE_BLOCKER flag present so all that is
+             * present is the error message. */
+            *is_force_blocker = 0;
+            *op_errstr = gf_strdup(tmp);
+            ret = -1;
+            goto out;
+        }
+
+        /* Copy rest of the error message to op_errstr */
+        tmp = strtok_r(NULL, "|", &save_ptr);
+        if (tmp)
+            *op_errstr = gf_strdup(tmp);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        GF_FREE (slave_url_buf);
-        sys_unlink (log_file_path);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(slave_url_buf);
+    sys_unlink(log_file_path);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /** @slave_ip remains unmodified */
 int
-glusterd_geo_rep_parse_slave (char *slave_url,
-                              char **hostname, char **op_errstr)
+glusterd_geo_rep_parse_slave(char *slave_url, char **hostname, char **op_errstr)
 {
-        int   ret             = -1;
-        char *tmp             = NULL;
-        char *save_ptr        = NULL;
-        char *host            = NULL;
-        char errmsg[PATH_MAX] = "";
-        char *saved_url       = NULL;
-        xlator_t *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (slave_url);
-        GF_ASSERT (*slave_url);
-
-        saved_url = gf_strdup (slave_url);
-        if (!saved_url)
-                goto out;
+    int ret = -1;
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char *host = NULL;
+    char errmsg[PATH_MAX] = "";
+    char *saved_url = NULL;
+    xlator_t *this = NULL;
 
-        /* Checking if hostname has user specified */
-        host = strstr (saved_url, "@");
-        if (!host) { /* no user specified */
-                if (hostname) {
-                        *hostname = gf_strdup (saved_url);
-                        if (!*hostname)
-                                goto out;
-                }
+    this = THIS;
+    GF_ASSERT(this);
 
-                ret = 0;
-                goto out;
-        } else {
-                /* Moving the host past the '@' and checking if the
-                 * actual hostname also has '@' */
-                host++;
-                if (strstr (host, "@")) {
-                        gf_msg_debug (this->name, 0, "host = %s", host);
-                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
-                                        "Invalid Hostname (%s).", host);
-                        errmsg[ret] = '\0';
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "%s", errmsg);
-                        ret = -1;
-                        if (op_errstr)
-                                *op_errstr = gf_strdup (errmsg);
-                        goto out;
-                }
+    GF_ASSERT(slave_url);
+    GF_ASSERT(*slave_url);
 
-                ret = -1;
+    saved_url = gf_strdup(slave_url);
+    if (!saved_url)
+        goto out;
 
-                /**
-                 * preliminary check for valid slave format.
-                 */
-                tmp = strtok_r (saved_url, "@", &save_ptr);
-                tmp = strtok_r (NULL, "@", &save_ptr);
-                if (!tmp)
-                        goto out;
-                if (hostname) {
-                        *hostname = gf_strdup (tmp);
-                        if (!*hostname)
-                                goto out;
-                }
+    /* Checking if hostname has user specified */
+    host = strstr(saved_url, "@");
+    if (!host) { /* no user specified */
+        if (hostname) {
+            *hostname = gf_strdup(saved_url);
+            if (!*hostname)
+                goto out;
         }
 
         ret = 0;
+        goto out;
+    } else {
+        /* Moving the host past the '@' and checking if the
+         * actual hostname also has '@' */
+        host++;
+        if (strstr(host, "@")) {
+            gf_msg_debug(this->name, 0, "host = %s", host);
+            ret = snprintf(errmsg, sizeof(errmsg) - 1, "Invalid Hostname (%s).",
+                           host);
+            errmsg[ret] = '\0';
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+                   errmsg);
+            ret = -1;
+            if (op_errstr)
+                *op_errstr = gf_strdup(errmsg);
+            goto out;
+        }
+
+        ret = -1;
+
+        /**
+         * preliminary check for valid slave format.
+         */
+        tmp = strtok_r(saved_url, "@", &save_ptr);
+        tmp = strtok_r(NULL, "@", &save_ptr);
+        if (!tmp)
+            goto out;
+        if (hostname) {
+            *hostname = gf_strdup(tmp);
+            if (!*hostname)
+                goto out;
+        }
+    }
+
+    ret = 0;
 out:
-        GF_FREE (saved_url);
-        if (ret)
-                if (hostname)
-                        GF_FREE (*hostname);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(saved_url);
+    if (ret)
+        if (hostname)
+            GF_FREE(*hostname);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Return -1 only if there is a match in volume uuid */
 static int
-get_slavehost_from_voluuid (dict_t *dict, char *key, data_t *value, void *data)
+get_slavehost_from_voluuid(dict_t *dict, char *key, data_t *value, void *data)
 {
-        char *slave_voluuid                 = NULL;
-        char *slave_info                    = NULL;
-        char *tmp                           = NULL;
-        char tmp_char                       = 0;
-        char *slave_host                    = NULL;
-        xlator_t  *this                     = NULL;
-        struct slave_vol_config *slave_vol  = NULL;
-        int i                               = 0;
-        int ret                             = -1;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        slave_vol = data;
-        slave_info = value->data;
+    char *slave_info = NULL;
+    char *tmp = NULL;
+    char *slave_host = NULL;
+    xlator_t *this = NULL;
+    struct slave_vol_config *slave_vol = NULL;
+    int i = 0;
+    int ret = -1;
 
-        gf_msg_debug (this->name, 0, "slave_info:%s !", slave_info);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        if (!(slave_info) || strlen (slave_info) == 0) {
-                /* no slaves present, peace  */
-                ret = 0;
-                goto out;
-        }
+    slave_vol = data;
+    slave_info = value->data;
 
-        /* slave format:
-         * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
-        while (i++ < 5) {
-                slave_info = strchr (slave_info, ':');
-                if (slave_info)
-                        slave_info++;
-                else
-                        break;
-        }
+    gf_msg_debug(this->name, 0, "slave_info:%s !", slave_info);
 
-        if (!(slave_info) || strlen(slave_info) == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVE_VOL_PARSE_FAIL,
-                        "slave_info format is wrong!");
+    if (!(slave_info) || strlen(slave_info) == 0) {
+        /* no slaves present, peace  */
+        ret = 0;
+        goto out;
+    }
+
+    /* slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    while (i++ < 5) {
+        slave_info = strchr(slave_info, ':');
+        if (slave_info)
+            slave_info++;
+        else
+            break;
+    }
+
+    if (!(slave_info) || strlen(slave_info) == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "slave_info format is wrong!");
+        ret = -2;
+        goto out;
+    } else {
+        if (strcmp(slave_info, slave_vol->slave_voluuid) == 0) {
+            ret = -1;
+
+            /* get corresponding slave host for reference*/
+            slave_host = value->data;
+            slave_host = strstr(slave_host, "://");
+            if (slave_host) {
+                slave_host += 3;
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                       "Invalid slave_host format!");
                 ret = -2;
                 goto out;
-        } else {
-                if (strcmp (slave_info, slave_vol->slave_voluuid) == 0) {
-                        ret = -1;
-
-                        /* get corresponding slave host for reference*/
-                        slave_host = value->data;
-                        slave_host = strstr (slave_host, "://");
-                        if (slave_host)
-                                slave_host += 3;
-
-                        /* To go past username in non-root geo-rep session */
-                        tmp = strchr (slave_host, '@');
-                        if (tmp)
-                                slave_host = tmp + 1;
-
-                        tmp = strchr (slave_host, ':');
-                        if (!tmp) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SLAVE_VOL_PARSE_FAIL,
-                                        "Invalid slave_host!");
-                                ret = -2;
-                                goto out;
-                        }
-
-                        strncpy (slave_vol->old_slvhost, slave_host,
-                                                 (tmp - slave_host));
-                        slave_vol->old_slvhost[(tmp - slave_host)+1] = '\0';
+            }
+            /* To go past username in non-root geo-rep session */
+            tmp = strchr(slave_host, '@');
+            if (tmp) {
+                if ((tmp - slave_host) >= LOGIN_NAME_MAX) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                           "Invalid slave user length in %s", slave_host);
+                    ret = -2;
+                    goto out;
+                }
+                strncpy(slave_vol->old_slvuser, slave_host, (tmp - slave_host));
+                slave_vol->old_slvuser[(tmp - slave_host) + 1] = '\0';
+                slave_host = tmp + 1;
+            } else
+                strcpy(slave_vol->old_slvuser, "root");
+
+            tmp = strchr(slave_host, ':');
+            if (!tmp) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                       "Invalid slave_host!");
+                ret = -2;
+                goto out;
+            }
 
-                        goto out;
-                }
+            strncpy(slave_vol->old_slvhost, slave_host, (tmp - slave_host));
+            slave_vol->old_slvhost[(tmp - slave_host) + 1] = '\0';
+
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Given slave host and slave volume, check whether slave volume uuid
@@ -2909,392 +3140,444 @@ out:
  * If slave volume uuid is present, get corresponding slave host
  * for reference */
 static int
-glusterd_get_slavehost_from_voluuid (glusterd_volinfo_t *volinfo,
-                      char *slave_host, char *slave_vol,
-                      struct slave_vol_config *slave1)
+glusterd_get_slavehost_from_voluuid(glusterd_volinfo_t *volinfo,
+                                    char *slave_host, char *slave_vol,
+                                    struct slave_vol_config *slave1)
 {
-        int                ret   = -1;
-        xlator_t           *this = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
 
-        ret = dict_foreach (volinfo->gsync_slaves, get_slavehost_from_voluuid,
-                                                   slave1);
+    ret = dict_foreach(volinfo->gsync_slaves, get_slavehost_from_voluuid,
+                       slave1);
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr)
+glusterd_op_stage_gsync_create(dict_t *dict, char **op_errstr)
 {
-        char               *down_peerstr              = NULL;
-        char               *slave                     = NULL;
-        char               *volname                   = NULL;
-        char               *host_uuid                 = NULL;
-        char               *statefile                 = NULL;
-        char               *slave_url                 = NULL;
-        char               *slave_host                = NULL;
-        char               *slave_vol                 = NULL;
-        char               *conf_path                 = NULL;
-        char                errmsg[PATH_MAX]          = "";
-        char                common_pem_file[PATH_MAX] = "";
-        char                hook_script[PATH_MAX]     = "";
-        char                uuid_str [64]             = "";
-        int                 ret                       = -1;
-        int                 is_pem_push               = -1;
-        int                 ssh_port                  = 22;
-        gf_boolean_t        is_force                  = -1;
-        gf_boolean_t        is_no_verify              = -1;
-        gf_boolean_t        is_force_blocker          = -1;
-        gf_boolean_t        exists                    = _gf_false;
-        gf_boolean_t        is_template_in_use        = _gf_false;
-        glusterd_conf_t    *conf                      = NULL;
-        glusterd_volinfo_t *volinfo                   = NULL;
-        struct stat         stbuf                     = {0,};
-        xlator_t           *this                      = NULL;
-        char                     *georep_session_wrkng_dir  = NULL;
-        struct slave_vol_config  slave1                     = {{0},};
-        int                      type                       = 0;
-        char                     monitor_status[NAME_MAX]   = {0,};
-        char                     old_slave_url_info[SLAVE_URL_INFO_MAX] = {0};
-        char                     *old_slave_url             = NULL;
-        char                     old_confpath[PATH_MAX]     = {0};
-        gf_boolean_t             is_running                 = _gf_false;
-        int                      ret_status                 = 0;
-        char                     *statedir                  = NULL;
-        char                     statefiledir[PATH_MAX]     = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = glusterd_op_gsync_args_get (dict, op_errstr, &volname,
-                                          &slave, &host_uuid);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_ARG_FETCH_ERROR,
-                        "Unable to fetch arguments");
-                gf_msg_debug (this->name, 0, "Returning %d", ret);
-                return -1;
+    char *down_peerstr = NULL;
+    char *slave = NULL;
+    char *volname = NULL;
+    char *host_uuid = NULL;
+    char *statefile = NULL;
+    char *slave_url = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    char *conf_path = NULL;
+    char errmsg[PATH_MAX] = "";
+    char common_pem_file[PATH_MAX] = "";
+    char hook_script[PATH_MAX] = "";
+    char uuid_str[64] = "";
+    int ret = -1;
+    int is_pem_push = -1;
+    int ssh_port = 22;
+    gf_boolean_t is_force = -1;
+    gf_boolean_t is_no_verify = -1;
+    gf_boolean_t is_force_blocker = -1;
+    gf_boolean_t is_template_in_use = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    struct slave_vol_config slave1 = {
+        {0},
+    };
+    char old_slave_url[SLAVE_URL_INFO_MAX] = {0};
+    char old_confpath[PATH_MAX] = {0};
+    gf_boolean_t is_running = _gf_false;
+    char *statedir = NULL;
+    char statefiledir[PATH_MAX] = {
+        0,
+    };
+    gf_boolean_t is_different_slavehost = _gf_false;
+    gf_boolean_t is_different_username = _gf_false;
+    char *slave_user = NULL;
+    char *save_ptr = NULL;
+    char *slave_url_buf = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, &slave,
+                                     &host_uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ARG_FETCH_ERROR,
+               "Unable to fetch arguments");
+        gf_msg_debug(this->name, 0, "Returning %d", ret);
+        return -1;
+    }
+
+    if (conf->op_version < 2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "Op Version not supported.");
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume name does not exist");
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(volinfo, dict, &slave_url,
+                                              &slave_host, &slave_vol,
+                                              &conf_path, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave or confpath details.");
+        ret = -1;
+        goto out;
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = glusterd_are_vol_all_peers_up(volinfo, &conf->peers,
+                                            &down_peerstr);
+        if ((ret == _gf_false) && !is_force) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Peer %s,"
+                     " which is a part of %s volume, is"
+                     " down. Please bring up the peer and"
+                     " retry.",
+                     down_peerstr, volinfo->volname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_DISCONNECTED, "%s",
+                   errmsg);
+            *op_errstr = gf_strdup(errmsg);
+            GF_FREE(down_peerstr);
+            down_peerstr = NULL;
+            gf_msg_debug(this->name, 0, "Returning %d", ret);
+            return -1;
+        } else if (ret == _gf_false) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_PEER_DISCONNECTED,
+                   "Peer %s, which is a part of %s volume, is"
+                   " down. Force creating geo-rep session."
+                   " On bringing up the peer, re-run"
+                   " \"gluster system:: execute"
+                   " gsec_create\" and \"gluster volume"
+                   " geo-replication %s %s create push-pem"
+                   " force\"",
+                   down_peerstr, volinfo->volname, volinfo->volname, slave);
+            GF_FREE(down_peerstr);
+            down_peerstr = NULL;
+        }
+
+        ret = dict_get_int32(dict, "ssh_port", &ssh_port);
+        if (ret < 0 && ret != -ENOENT) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Fetching ssh_port failed while "
+                     "handling " GEOREP " options");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+
+        is_no_verify = dict_get_str_boolean(dict, "no_verify", _gf_false);
+
+        if (!is_no_verify) {
+            /* Checking if slave host is pingable, has proper passwordless
+             * ssh login setup, slave volume is created, slave vol is empty,
+             * and if it has enough memory and bypass in case of force if
+             * the error is not a force blocker */
+            ret = glusterd_verify_slave(volname, slave_url, slave_vol, ssh_port,
+                                        op_errstr, &is_force_blocker);
+            if (ret) {
+                if (is_force && !is_force_blocker) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_INVALID_SLAVE,
+                           "%s is not a valid slave "
+                           "volume. Error: %s. Force "
+                           "creating geo-rep"
+                           " session.",
+                           slave, *op_errstr);
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_SLAVE,
+                           "%s is not a valid slave "
+                           "volume. Error: %s",
+                           slave, *op_errstr);
+                    ret = -1;
+
+                    goto out;
+                }
+            }
         }
 
-        if (conf->op_version < 2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
-                        "Op Version not supported.");
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                *op_errstr = gf_strdup (errmsg);
+        ret = dict_get_int32(dict, "push_pem", &is_pem_push);
+        if (!ret && is_pem_push) {
+            ret = snprintf(common_pem_file, sizeof(common_pem_file),
+                           "%s" GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+            if ((ret < 0) || (ret >= sizeof(common_pem_file))) {
                 ret = -1;
                 goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
-                        "volume name does not exist");
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
-                *op_errstr = gf_strdup (errmsg);
-                gf_msg_debug (this->name, 0, "Returning %d", ret);
-                return -1;
-        }
+            }
 
-        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVEINFO_FETCH_ERROR,
-                        "Unable to fetch slave or confpath details.");
+            ret = snprintf(hook_script, sizeof(hook_script),
+                           "%s" GLUSTERD_CREATE_HOOK_SCRIPT, conf->workdir);
+            if ((ret < 0) || (ret >= sizeof(hook_script))) {
                 ret = -1;
                 goto out;
-        }
-
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = glusterd_are_vol_all_peers_up (volinfo,
-                                                     &conf->peers,
-                                                     &down_peerstr);
-                if ((ret == _gf_false) && !is_force) {
-                        snprintf (errmsg, sizeof (errmsg), "Peer %s,"
-                                  " which is a part of %s volume, is"
-                                  " down. Please bring up the peer and"
-                                  " retry.", down_peerstr,
-                                  volinfo->volname);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_PEER_DISCONNECTED,
-                                "%s", errmsg);
-                        *op_errstr = gf_strdup (errmsg);
-                        GF_FREE (down_peerstr);
-                        down_peerstr = NULL;
-                        gf_msg_debug (this->name, 0, "Returning %d", ret);
-                        return -1;
-                } else if (ret == _gf_false) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_PEER_DISCONNECTED,
-                                "Peer %s, which is a part of %s volume, is"
-                                " down. Force creating geo-rep session."
-                                " On bringing up the peer, re-run"
-                                " \"gluster system:: execute"
-                                " gsec_create\" and \"gluster volume"
-                                " geo-replication %s %s create push-pem"
-                                " force\"", down_peerstr, volinfo->volname,
-                                volinfo->volname, slave);
-                         GF_FREE (down_peerstr);
-                         down_peerstr = NULL;
-                }
-
-                ret = dict_get_int32 (dict, "ssh_port", &ssh_port);
-                if (ret < 0 && ret != -ENOENT) {
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "Fetching ssh_port failed while "
-                                  "handling "GEOREP" options");
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "%s", errmsg);
-                        goto out;
-                }
-
-                is_no_verify = dict_get_str_boolean (dict, "no_verify", _gf_false);
-
-                if (!is_no_verify) {
-                        /* Checking if slave host is pingable, has proper passwordless
-                        * ssh login setup, slave volume is created, slave vol is empty,
-                        * and if it has enough memory and bypass in case of force if
-                        * the error is not a force blocker */
-                        ret = glusterd_verify_slave (volname, slave_url, slave_vol,
-                                                     ssh_port, op_errstr,
-                                                     &is_force_blocker);
-                        if (ret) {
-                                if (is_force && !is_force_blocker) {
-                                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                                GD_MSG_INVALID_SLAVE,
-                                                "%s is not a valid slave "
-                                                "volume. Error: %s. Force "
-                                                "creating geo-rep"
-                                                " session.", slave,
-                                                *op_errstr);
-                                } else {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_INVALID_SLAVE,
-                                                "%s is not a valid slave "
-                                                "volume. Error: %s",
-                                                slave, *op_errstr);
-                                        ret = -1;
-
-                                        goto out;
-                                }
-                        }
-                }
+            }
 
-                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
-                if (!ret && is_pem_push) {
-                        ret = snprintf (common_pem_file,
-                                        sizeof(common_pem_file) - 1,
-                                        "%s"GLUSTERD_COMMON_PEM_PUB_FILE,
-                                        conf->workdir);
-                        common_pem_file[ret] = '\0';
-
-                        ret = snprintf (hook_script, sizeof(hook_script) - 1,
-                                        "%s"GLUSTERD_CREATE_HOOK_SCRIPT,
-                                        conf->workdir);
-                        hook_script[ret] = '\0';
-
-                        ret = sys_lstat (common_pem_file, &stbuf);
-                        if (ret) {
-                                snprintf (errmsg, sizeof (errmsg), "%s"
-                                          " required for push-pem is"
-                                          " not present. Please run"
-                                          " \"gluster system:: execute"
-                                          " gsec_create\"", common_pem_file);
-                                gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                                        GD_MSG_FILE_OP_FAILED,
-                                        "%s", errmsg);
-                                *op_errstr = gf_strdup (errmsg);
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = sys_lstat (hook_script, &stbuf);
-                        if (ret) {
-                                snprintf (errmsg, sizeof (errmsg),
-                                          "The hook-script (%s) required "
-                                          "for push-pem is not present. "
-                                          "Please install the hook-script "
-                                          "and retry", hook_script);
-                                gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                                        GD_MSG_FILE_OP_FAILED, "%s", errmsg);
-                                *op_errstr = gf_strdup (errmsg);
-                                ret = -1;
-                                goto out;
-                        }
-
-                        if (!S_ISREG(stbuf.st_mode)) {
-                                snprintf (errmsg, sizeof (errmsg), "%s"
-                                          " required for push-pem is"
-                                          " not a regular file. Please run"
-                                          " \"gluster system:: execute"
-                                          " gsec_create\"", common_pem_file);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_REG_FILE_MISSING, "%s", errmsg);
-                                ret = -1;
-                                goto out;
-                        }
-                }
-        }
+            ret = sys_lstat(common_pem_file, &stbuf);
+            if (ret) {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "%s"
+                               " required for push-pem is"
+                               " not present. Please run"
+                               " \"gluster system:: execute"
+                               " gsec_create\"",
+                               common_pem_file);
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
+                }
+                gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                       "%s", errmsg);
+                *op_errstr = gf_strdup(errmsg);
+                ret = -1;
+                goto out;
+            }
 
-        ret = glusterd_get_statefile_name (volinfo, slave,
-                                           conf_path, &statefile,
-                                           &is_template_in_use);
-        if (ret) {
-                if (!strstr(slave, "::"))
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "%s is not a valid slave url.", slave);
-                else
-                        snprintf (errmsg, sizeof (errmsg), "Please check gsync "
-                                  "config file. Unable to get statefile's name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STATEFILE_NAME_NOT_FOUND,
-                        "%s", errmsg);
+            ret = sys_lstat(hook_script, &stbuf);
+            if (ret) {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "The hook-script (%s) "
+                               "required for push-pem is not "
+                               "present. Please install the "
+                               "hook-script and retry",
+                               hook_script);
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
+                }
+                gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                       "%s", errmsg);
+                *op_errstr = gf_strdup(errmsg);
                 ret = -1;
                 goto out;
-        }
+            }
 
-        ret = dict_set_str (dict, "statefile", statefile);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Unable to store statefile path");
+            if (!S_ISREG(stbuf.st_mode)) {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "%s"
+                               " required for push-pem is"
+                               " not a regular file. Please"
+                               " run \"gluster system:: "
+                               "execute gsec_create\"",
+                               common_pem_file);
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
+                }
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REG_FILE_MISSING,
+                       "%s", errmsg);
+                ret = -1;
                 goto out;
+            }
         }
+    }
 
-        strncpy (statefiledir, statefile, sizeof(statefiledir));
-        statedir = dirname (statefiledir);
+    ret = glusterd_get_statefile_name(volinfo, slave, conf_path, &statefile,
+                                      &is_template_in_use);
+    if (ret) {
+        if (!strstr(slave, "::"))
+            snprintf(errmsg, sizeof(errmsg), "%s is not a valid slave url.",
+                     slave);
+        else
+            snprintf(errmsg, sizeof(errmsg),
+                     "Please check gsync "
+                     "config file. Unable to get statefile's name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATEFILE_NAME_NOT_FOUND,
+               "%s", errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "statefile", statefile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store statefile path");
+        goto out;
+    }
+
+    if (snprintf(statefiledir, sizeof(statefiledir), "%s", statefile) >=
+        sizeof(statefiledir)) {
+        snprintf(errmsg, sizeof(errmsg), "Failed copying statefiledir");
+        goto out;
+    }
+    statedir = dirname(statefiledir);
+
+    ret = sys_lstat(statedir, &stbuf);
+    if (!ret && !is_force) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Session between %s"
+                 " and %s is already created.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_ALREADY_EXIST, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    } else if (!ret)
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+               "Session between %s and %s is already created. Force"
+               " creating again.",
+               volinfo->volname, slave);
+
+    ret = glusterd_get_slave_voluuid(slave_host, slave_vol,
+                                     slave1.slave_voluuid);
+    if ((ret) || (strlen(slave1.slave_voluuid) == 0)) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to get remote volume uuid.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "slave_voluuid",
+                                     slave1.slave_voluuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set slave volume uuid in the dict");
+        goto out;
+    }
+
+    /* Check whether session is already created using slave volume uuid */
+    ret = glusterd_get_slavehost_from_voluuid(volinfo, slave_host, slave_vol,
+                                              &slave1);
+    if (ret == -1) {
+        if (!is_force) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Session between %s"
+                     " and %s:%s is already created! Cannot create "
+                     "with new slave:%s again!",
+                     volinfo->volname, slave1.old_slvhost, slave_vol,
+                     slave_host);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FORCE_CREATE_SESSION,
+                   "Session between"
+                   " %s and %s:%s is already created! "
+                   "Cannot create with new slave:%s again!",
+                   volinfo->volname, slave1.old_slvhost, slave_vol, slave_host);
+            goto out;
+        }
+
+        /* There is a remote possibility that slave_host can be NULL when
+           control reaches here. Add a check so we wouldn't crash in next
+           line */
+        if (!slave_host)
+            goto out;
+
+        /* Now, check whether session is already started.If so, warn!*/
+        is_different_slavehost = (strcmp(slave_host, slave1.old_slvhost) != 0)
+                                     ? _gf_true
+                                     : _gf_false;
 
-        ret = sys_lstat (statedir, &stbuf);
-        if (!ret && !is_force) {
-                snprintf (errmsg, sizeof (errmsg), "Session between %s"
-                          " and %s is already created.",
-                          volinfo->volname, slave);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SESSION_ALREADY_EXIST,
-                        "%s", errmsg);
+        if (strstr(slave_url, "@")) {
+            slave_url_buf = gf_strdup(slave_url);
+            if (!slave_url_buf) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Unable to allocate memory");
                 ret = -1;
                 goto out;
-        } else if (!ret)
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
-                        "Session between %s and %s is already created. Force"
-                        " creating again.", volinfo->volname, slave);
-
-        ret = glusterd_get_slave_voluuid (slave_host, slave_vol,
-                                          slave1.slave_voluuid);
-        if ((ret) || (strlen(slave1.slave_voluuid) == 0)) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to get remote volume uuid.");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REMOTE_VOL_UUID_FAIL, "%s", errmsg);
+            }
+            slave_user = strtok_r(slave_url_buf, "@", &save_ptr);
+        } else
+            slave_user = "root";
+        is_different_username = (strcmp(slave_user, slave1.old_slvuser) != 0)
+                                    ? _gf_true
+                                    : _gf_false;
+
+        /* Do the check, only if different slave host/slave user */
+        if (is_different_slavehost || is_different_username) {
+            len = snprintf(old_confpath, sizeof(old_confpath),
+                           "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", conf->workdir,
+                           volinfo->volname, slave1.old_slvhost, slave_vol);
+            if ((len < 0) || (len >= sizeof(old_confpath))) {
                 ret = -1;
                 goto out;
-        }
+            }
 
-        ret = dict_set_dynstr_with_alloc (dict, "slave_voluuid",
-                                          slave1.slave_voluuid);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Unable to set slave volume uuid in the dict");
+            /* construct old slave url with (old) slave host */
+            len = snprintf(old_slave_url, sizeof(old_slave_url), "%s::%s",
+                           slave1.old_slvhost, slave_vol);
+            if ((len < 0) || (len >= sizeof(old_slave_url))) {
+                ret = -1;
                 goto out;
-        }
-
-        /* Check whether session is already created using slave volume uuid */
-        ret = glusterd_get_slavehost_from_voluuid (volinfo, slave_host,
-                                              slave_vol, &slave1);
-        if (ret == -1) {
-                if (!is_force) {
-                        snprintf (errmsg, sizeof (errmsg), "Session between %s"
-                                  " and %s:%s is already created! Cannot create "
-                                  "with new slave:%s again!",
-                                  volinfo->volname, slave1.old_slvhost,
-                                  slave_vol, slave_host);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                 GD_MSG_FORCE_CREATE_SESSION, "Session between"
-                                  " %s and %s:%s is already created! "
-                                  "Cannot create with new slave:%s again!",
-                                  volinfo->volname, slave1.old_slvhost,
-                                  slave_vol, slave_host);
-                        goto out;
-                }
-                /* Now, check whether session is already started, if so, warn!*/
-                ret = snprintf (old_confpath, sizeof(old_confpath) - 1,
-                                "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                                conf->workdir, volinfo->volname,
-                                slave1.old_slvhost, slave_vol);
-
-                /* construct old slave url with (old) slave host */
-                old_slave_url = old_slave_url_info;
-                strncpy (old_slave_url, slave1.old_slvhost,
-                                        sizeof(old_slave_url_info));
-                old_slave_url = strcat (old_slave_url, "::");
-                old_slave_url = strncat (old_slave_url, slave_vol,
-                                                  sizeof(old_slave_url_info));
-
-                ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                          old_slave_url,
-                                                          old_confpath,
-                                                          &is_running);
-                if (_gf_true == is_running) {
-                        snprintf (errmsg, sizeof(errmsg), "Geo"
-                                  " -replication session between %s and %s"
-                                  " is still active. Please stop the "
-                                  "session and retry.",
-                                  volinfo->volname, old_slave_url);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (dict, "old_slavehost",
-                                                  slave1.old_slvhost);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set old_slavehost in the dict");
-                        goto out;
-                }
+            }
 
-                ret = dict_set_int32 (dict, "existing_session", _gf_true);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set existing_session in the dict");
-                        goto out;
-                }
-        } else if (ret == -2) {
-                snprintf (errmsg, sizeof (errmsg), "get_slavehost_from_voluuid"
-                          " failed %s %s!!", slave_host, slave_vol);
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
-                        "get_slavehost_from_voluuid failed %s %s!!",
-                         slave_host, slave_vol);
+            ret = glusterd_check_gsync_running_local(
+                volinfo->volname, old_slave_url, old_confpath, &is_running);
+            if (_gf_true == is_running) {
+                (void)snprintf(errmsg, sizeof(errmsg),
+                               "Geo"
+                               "-replication session between %s and %s"
+                               " is still active. Please stop the "
+                               "session and retry.",
+                               volinfo->volname, old_slave_url);
+                ret = -1;
                 goto out;
+            }
         }
 
-        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        ret = dict_set_dynstr_with_alloc(dict, "old_slavehost",
+                                         slave1.old_slvhost);
         if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to spawn gsyncd.");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED,
-                        "%s", errmsg);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set old_slavehost in the dict");
+            goto out;
         }
 
-        ret = 0;
+        ret = dict_set_int32(dict, "existing_session", _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set existing_session in the dict");
+            goto out;
+        }
+    } else if (ret == -2) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "get_slavehost_from_voluuid"
+                 " failed for %s::%s. Please check the glusterd logs.",
+                 slave_host, slave_vol);
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+               "get_slavehost_from_voluuid failed %s %s!!", slave_host,
+               slave_vol);
+        goto out;
+    }
+
+    ret = glusterd_verify_gsyncd_spawn(volinfo->volname, slave);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to spawn gsyncd.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        if (ret && errmsg[0] != '\0')
-                *op_errstr = gf_strdup (errmsg);
+    if (ret && errmsg[0] != '\0')
+        *op_errstr = gf_strdup(errmsg);
+
+    if (slave_url_buf)
+        GF_FREE(slave_url_buf);
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 /* pre-condition check for geo-rep pause/resume.
@@ -3302,625 +3585,649 @@ out:
  *        -1 on any check failed.
  */
 static int
-gd_pause_resume_validation (int type, glusterd_volinfo_t *volinfo,
-                            char *slave, char *statefile, char **op_errstr)
+gd_pause_resume_validation(int type, glusterd_volinfo_t *volinfo, char *slave,
+                           char *statefile, char **op_errstr)
 {
-        int        ret                        = 0;
-        char       errmsg[PATH_MAX]           = {0,};
-        char       monitor_status[NAME_MAX]   = {0,};
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (statefile);
-        GF_ASSERT (op_errstr);
-
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                               sizeof (monitor_status));
-        if (ret <= 0) {
-                snprintf (errmsg, sizeof(errmsg), "Pause check Failed:"
-                          " Geo-rep session is not setup");
-                ret = -1;
-                goto out;
-        }
-
-        if ( type == GF_GSYNC_OPTION_TYPE_PAUSE &&
-             strstr (monitor_status, "Paused")) {
-                snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                          " session between %s and %s already Paused.",
-                          volinfo->volname, slave);
-                ret = -1;
-                goto out;
-        }
-        if ( type == GF_GSYNC_OPTION_TYPE_RESUME &&
-             !strstr (monitor_status, "Paused")) {
-                snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                          " session between %s and %s is not Paused.",
-                          volinfo->volname, slave);
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+    int ret = 0;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    char monitor_status[NAME_MAX] = {
+        0,
+    };
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(statefile);
+    GF_ASSERT(op_errstr);
+
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Pause check Failed:"
+                 " Geo-rep session is not setup");
+        ret = -1;
+        goto out;
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_PAUSE &&
+        strstr(monitor_status, "Paused")) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Geo-replication"
+                 " session between %s and %s already Paused.",
+                 volinfo->volname, slave);
+        ret = -1;
+        goto out;
+    }
+    if (type == GF_GSYNC_OPTION_TYPE_RESUME &&
+        !strstr(monitor_status, "Paused")) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Geo-replication"
+                 " session between %s and %s is not Paused.",
+                 volinfo->volname, slave);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        if (ret && (errmsg[0] != '\0')) {
-                *op_errstr = gf_strdup (errmsg);
-        }
-        return ret;
+    if (ret && (errmsg[0] != '\0')) {
+        *op_errstr = gf_strdup(errmsg);
+    }
+    return ret;
 }
 
 int
-glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
+glusterd_op_stage_gsync_set(dict_t *dict, char **op_errstr)
 {
-        int                 ret                   = 0;
-        int                 type                  = 0;
-        int                 pfd                   = -1;
-        char               *volname               = NULL;
-        char               *slave                 = NULL;
-        char               *slave_url             = NULL;
-        char               *slave_host            = NULL;
-        char               *slave_vol             = NULL;
-        char               *down_peerstr          = NULL;
-        char               *statefile             = NULL;
-        char               statefiledir[PATH_MAX] = {0,};
-        char               *statedir              = NULL;
-        char               *path_list             = NULL;
-        char               *conf_path             = NULL;
-        gf_boolean_t        exists                = _gf_false;
-        glusterd_volinfo_t *volinfo               = NULL;
-        char                    errmsg[PATH_MAX]  = {0,};
-        char                    pidfile[PATH_MAX] = {0,};
-        dict_t             *ctx                   = NULL;
-        gf_boolean_t        is_force              = 0;
-        gf_boolean_t        is_running            = _gf_false;
-        gf_boolean_t        is_template_in_use    = _gf_false;
-        uuid_t              uuid                  = {0};
-        char                    uuid_str [64]     = {0};
-        char               *host_uuid             = NULL;
-        xlator_t           *this                  = NULL;
-        glusterd_conf_t    *conf                  = NULL;
-        struct stat         stbuf                 = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
-                        "command type not found");
-                *op_errstr = gf_strdup ("command unsuccessful");
+    int ret = 0;
+    int type = 0;
+    char *volname = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    char *down_peerstr = NULL;
+    char *statefile = NULL;
+    char statefiledir[PATH_MAX] = {
+        0,
+    };
+    char *statedir = NULL;
+    char *path_list = NULL;
+    char *conf_path = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    dict_t *ctx = NULL;
+    gf_boolean_t is_force = 0;
+    gf_boolean_t is_running = _gf_false;
+    gf_boolean_t is_template_in_use = _gf_false;
+    uuid_t uuid = {0};
+    char uuid_str[64] = {0};
+    char *host_uuid = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "command type not found");
+        *op_errstr = gf_strdup("command unsuccessful");
+        goto out;
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
+        ret = glusterd_verify_gsync_status_opts(dict, op_errstr);
+        goto out;
+    }
+
+    ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, &slave,
+                                     &host_uuid);
+    if (ret)
+        goto out;
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+
+    if (conf->op_version < 2) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(volinfo, dict, &slave_url,
+                                              &slave_host, &slave_vol,
+                                              &conf_path, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave or confpath details.");
+        ret = -1;
+        goto out;
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    ret = glusterd_get_statefile_name(volinfo, slave, conf_path, &statefile,
+                                      &is_template_in_use);
+    if (ret) {
+        if (!strstr(slave, "::")) {
+            snprintf(errmsg, sizeof(errmsg), "%s is not a valid slave url.",
+                     slave);
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+                   "state_file entry missing in config file (%s)", conf_path);
+
+            if ((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_STOP_FORCE,
+                       "Allowing stop "
+                       "force to bypass missing statefile "
+                       "entry in config file (%s), and "
+                       "template file",
+                       conf_path);
+                ret = 0;
+            } else
                 goto out;
         }
-
-        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
-                ret = glusterd_verify_gsync_status_opts (dict, op_errstr);
-                goto out;
+    } else {
+        ret = dict_set_str(dict, "statefile", statefile);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to store statefile path");
+            goto out;
         }
+    }
 
-        ret = glusterd_op_gsync_args_get (dict, op_errstr,
-                                          &volname, &slave, &host_uuid);
-        if (ret)
-                goto out;
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-
-        if (conf->op_version < 2) {
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                ret = -1;
-                goto out;
+    /* Allowing stop force to bypass the statefile check
+     * as this command acts as a fail safe method to stop geo-rep
+     * session. */
+    if (!((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force)) {
+        /* check session directory as statefile may not present
+         * during upgrade */
+        if (snprintf(statefiledir, sizeof(statefiledir), "%s", statefile) >=
+            sizeof(statefiledir)) {
+            snprintf(errmsg, sizeof(errmsg), "Failed copying statefiledir");
+            ret = -1;
+            goto out;
         }
+        statedir = dirname(statefiledir);
 
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
+        ret = sys_lstat(statedir, &stbuf);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Geo-replication"
+                     " session between %s and %s does not exist.",
+                     volinfo->volname, slave);
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "%s. statefile = %s", errmsg, statefile);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* Check if all peers that are a part of the volume are up or not */
+    if ((type == GF_GSYNC_OPTION_TYPE_DELETE) ||
+        ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force) ||
+        (type == GF_GSYNC_OPTION_TYPE_PAUSE) ||
+        (type == GF_GSYNC_OPTION_TYPE_RESUME)) {
+        if (!strcmp(uuid_str, host_uuid)) {
+            ret = glusterd_are_vol_all_peers_up(volinfo, &conf->peers,
+                                                &down_peerstr);
+            if (ret == _gf_false) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "Peer %s,"
+                         " which is a part of %s volume, is"
+                         " down. Please bring up the peer and"
+                         " retry.",
+                         down_peerstr, volinfo->volname);
                 ret = -1;
+                GF_FREE(down_peerstr);
+                down_peerstr = NULL;
                 goto out;
+            }
         }
+    }
 
-        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVEINFO_FETCH_ERROR,
-                        "Unable to fetch slave or confpath details.");
+    switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry "
+                         "missing in the config file(%s).",
+                         conf_path);
                 ret = -1;
                 goto out;
-        }
-
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        ret = glusterd_get_statefile_name (volinfo, slave,
-                                           conf_path, &statefile,
-                                           &is_template_in_use);
-        if (ret) {
-                if (!strstr(slave, "::")) {
-                        snprintf (errmsg, sizeof(errmsg),
-                                  "%s is not a valid slave url.", slave);
-                        ret = -1;
-                        goto out;
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVE_URL_INVALID,
-                                "state_file entry missing in config file (%s)",
-                                conf_path);
-
-                        if ((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_STOP_FORCE, "Allowing stop "
-                                        "force to bypass missing statefile "
-                                        "entry in config file (%s), and "
-                                        "template file", conf_path);
-                                ret = 0;
-                        } else
-                                goto out;
-                }
-        } else {
-                ret = dict_set_str (dict, "statefile", statefile);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to store statefile path");
-                        goto out;
-                }
-        }
-
-        /* Allowing stop force to bypass the statefile check
-         * as this command acts as a fail safe method to stop geo-rep
-         * session. */
-        if (!((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force)) {
-
-                /* check session directory as statefile may not present
-                 * during upgrade */
-                strncpy (statefiledir, statefile, sizeof(statefiledir));
-                statedir = dirname (statefiledir);
-
-                ret = sys_lstat (statedir, &stbuf);
-                if (ret) {
-                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                                  " session between %s and %s does not exist.",
-                                  volinfo->volname, slave);
-                        gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                                GD_MSG_FILE_OP_FAILED,
-                                "%s. statefile = %s", errmsg, statefile);
-                        ret = -1;
-                        goto out;
-                }
-        }
+            }
 
-        /* Check if all peers that are a part of the volume are up or not */
-        if ((type == GF_GSYNC_OPTION_TYPE_DELETE) ||
-            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force) ||
-            (type == GF_GSYNC_OPTION_TYPE_PAUSE) ||
-            (type == GF_GSYNC_OPTION_TYPE_RESUME)) {
-                if (!strcmp (uuid_str, host_uuid)) {
-                        ret = glusterd_are_vol_all_peers_up (volinfo,
-                                                             &conf->peers,
-                                                             &down_peerstr);
-                        if (ret == _gf_false) {
-                                snprintf (errmsg, sizeof (errmsg), "Peer %s,"
-                                          " which is a part of %s volume, is"
-                                          " down. Please bring up the peer and"
-                                          " retry.", down_peerstr,
-                                          volinfo->volname);
-                                ret = -1;
-                                GF_FREE (down_peerstr);
-                                down_peerstr = NULL;
-                                goto out;
-                        }
+            ret = glusterd_op_verify_gsync_start_options(
+                volinfo, slave, conf_path, statefile, op_errstr, is_force);
+            if (ret)
+                goto out;
+            ctx = glusterd_op_get_ctx();
+            if (ctx) {
+                /* gsyncd does a fuse mount to start
+                 * the geo-rep session */
+                if (!glusterd_is_fuse_available()) {
+                    gf_msg("glusterd", GF_LOG_ERROR, errno,
+                           GD_MSG_GEO_REP_START_FAILED,
+                           "Unable "
+                           "to open /dev/fuse (%s), "
+                           "geo-replication start failed",
+                           strerror(errno));
+                    snprintf(errmsg, sizeof(errmsg), "fuse unavailable");
+                    ret = -1;
+                    goto out;
                 }
-        }
+            }
+            break;
 
-        switch (type) {
-        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+            if (!is_force) {
                 if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "state-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
-                        goto out;
+                    snprintf(errmsg, sizeof(errmsg),
+                             "state-file entry missing in "
+                             "the config file(%s).",
+                             conf_path);
+                    ret = -1;
+                    goto out;
                 }
 
-                ret = glusterd_op_verify_gsync_start_options (volinfo, slave,
-                                                              conf_path,
-                                                              statefile,
-                                                              op_errstr, is_force);
-                if (ret)
+                ret = glusterd_op_verify_gsync_running(volinfo, slave,
+                                                       conf_path, op_errstr);
+                if (ret) {
+                    ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+                    if (!path_list && ret == -1)
                         goto out;
-                ctx = glusterd_op_get_ctx();
-                if (ctx) {
-                        /* gsyncd does a fuse mount to start
-                         * the geo-rep session */
-                        if (!glusterd_is_fuse_available ()) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                                        GD_MSG_GEO_REP_START_FAILED, "Unable "
-                                        "to open /dev/fuse (%s), "
-                                        "geo-replication start failed",
-                                        strerror (errno));
-                                snprintf (errmsg, sizeof(errmsg),
-                                          "fuse unvailable");
-                                ret = -1;
-                                goto out;
-                        }
                 }
-                break;
 
-        case GF_GSYNC_OPTION_TYPE_STOP:
-                if (!is_force) {
-                        if (is_template_in_use) {
-                                snprintf (errmsg, sizeof(errmsg),
-                                          "state-file entry missing in "
-                                          "the config file(%s).", conf_path);
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = glusterd_op_verify_gsync_running (volinfo, slave,
-                                                                conf_path,
-                                                                op_errstr);
-                        if (ret) {
-                                ret = glusterd_get_local_brickpaths (volinfo,
-                                                                     &path_list);
-                                if (path_list)
-                                        ret = -1;
-                        }
+                /* Check for geo-rep session is active or not for
+                 * configured user.*/
+                ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+                if (ret) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             "Geo-replication session between %s "
+                             "and %s does not exist.",
+                             volinfo->volname, slave);
+                    ret = -1;
+                    goto out;
                 }
-                break;
+            }
+            break;
 
         case GF_GSYNC_OPTION_TYPE_PAUSE:
         case GF_GSYNC_OPTION_TYPE_RESUME:
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg),
-                                  "state-file entry missing in "
-                                  "the config file(%s).", conf_path);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_op_verify_gsync_running (volinfo, slave,
-                                                        conf_path, op_errstr);
-                if (ret) {
-                        ret = glusterd_get_local_brickpaths (volinfo,
-                                                             &path_list);
-                        if (path_list) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry missing in "
+                         "the config file(%s).",
+                         conf_path);
+                ret = -1;
+                goto out;
+            }
 
-                if (!is_force) {
-                        ret = gd_pause_resume_validation (type, volinfo, slave,
-                                                          statefile, op_errstr);
-                        if (ret) {
-                                ret = glusterd_get_local_brickpaths (volinfo,
-                                                                    &path_list);
-                                if (path_list) {
-                                        ret = -1;
-                                        goto out;
-                                }
-                        }
-                }
-                break;
+            ret = glusterd_op_verify_gsync_running(volinfo, slave, conf_path,
+                                                   op_errstr);
+            if (ret) {
+                ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+                if (!path_list && ret == -1)
+                    goto out;
+            }
 
-        case GF_GSYNC_OPTION_TYPE_CONFIG:
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "state-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
-                        goto out;
-                }
+            /* Check for geo-rep session is active or not
+             * for configured user.*/
+            ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+            if (ret) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "Geo-replication"
+                         " session between %s and %s does not exist.",
+                         volinfo->volname, slave);
+                ret = -1;
+                goto out;
+            }
 
-                pfd = gsyncd_getpidfile (volname, slave, pidfile,
-                                         conf_path, &is_template_in_use);
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "pid-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
+            if (!is_force) {
+                ret = gd_pause_resume_validation(type, volinfo, slave,
+                                                 statefile, op_errstr);
+                if (ret) {
+                    ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+                    if (!path_list && ret == -1)
                         goto out;
                 }
+            }
+            break;
 
-                ret = gsync_verify_config_options (dict, op_errstr, volname);
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry "
+                         "missing in the config file(%s).",
+                         conf_path);
+                ret = -1;
                 goto out;
-                break;
+            }
+
+            ret = gsync_verify_config_options(dict, op_errstr, volname);
+            goto out;
+            break;
 
         case GF_GSYNC_OPTION_TYPE_DELETE:
-                /* Check if the gsync session is still running
-                 * If so ask the user to stop geo-replication first.*/
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "state-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
-                        goto out;
-                }
+            /* Check if the gsync session is still running
+             * If so ask the user to stop geo-replication first.*/
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry "
+                         "missing in the config file(%s).",
+                         conf_path);
+                ret = -1;
+                goto out;
+            }
 
-                ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-                if (ret) {
-                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                                  " session between %s and %s does not exist.",
-                                  volinfo->volname, slave);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                                  slave, conf_path,
-                                                                  &is_running);
-                        if (_gf_true == is_running) {
-                                snprintf (errmsg, sizeof (errmsg), GEOREP
-                                          " session between %s & %s is "
-                                          "still active. Please stop the "
-                                          "session and retry.",
-                                          volinfo->volname, slave);
-                                ret = -1;
-                                goto out;
-                        }
+            ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+            if (ret) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "Geo-replication"
+                         " session between %s and %s does not exist.",
+                         volinfo->volname, slave);
+                ret = -1;
+                goto out;
+            } else {
+                ret = glusterd_check_gsync_running_local(
+                    volinfo->volname, slave, conf_path, &is_running);
+                if (_gf_true == is_running) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             GEOREP
+                             " session between %s & %s is "
+                             "still active. Please stop the "
+                             "session and retry.",
+                             volinfo->volname, slave);
+                    ret = -1;
+                    goto out;
                 }
+            }
 
-                ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "Unable to spawn gsyncd");
-                }
+            ret = glusterd_verify_gsyncd_spawn(volinfo->volname, slave);
+            if (ret) {
+                snprintf(errmsg, sizeof(errmsg), "Unable to spawn gsyncd");
+            }
 
-                break;
-        }
+            break;
+    }
 
 out:
-        if (path_list)
-                GF_FREE (path_list);
 
-        if (ret && errmsg[0] != '\0') {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
-                        "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-        }
+    if (path_list)
+        GF_FREE(path_list);
+
+    if (ret && errmsg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR, "%s", errmsg);
+        *op_errstr = gf_strdup(errmsg);
+    }
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-gd_pause_or_resume_gsync (dict_t *dict, char *master, char *slave,
-                          char *slave_host, char *slave_vol, char *conf_path,
-                          char **op_errstr, gf_boolean_t is_pause)
+gd_pause_or_resume_gsync(dict_t *dict, char *master, char *slave,
+                         char *slave_host, char *slave_vol, char *conf_path,
+                         char **op_errstr, gf_boolean_t is_pause)
 {
-        int32_t         ret                      = 0;
-        int             pfd                      = -1;
-        pid_t           pid                      = 0;
-        char            pidfile[PATH_MAX]        = {0,};
-        char            errmsg[PATH_MAX]         = "";
-        char            buf [1024]               = {0,};
-        int             i                        = 0;
-        gf_boolean_t    is_template_in_use       = _gf_false;
-        char            monitor_status[NAME_MAX] = {0,};
-        char            *statefile               = NULL;
-        xlator_t        *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (master);
-        GF_ASSERT (slave);
-        GF_ASSERT (slave_host);
-        GF_ASSERT (slave_vol);
-        GF_ASSERT (conf_path);
-
-        pfd = gsyncd_getpidfile (master, slave, pidfile,
-                                 conf_path, &is_template_in_use);
-        if (pfd == -2) {
-                snprintf (errmsg, sizeof(errmsg),
-                          "pid-file entry mising in config file and "
-                          "template config file.");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND,
-                        "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        if (gsync_status_byfd (pfd) == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
-                        "gsyncd b/w %s & %s is not running", master, slave);
-                /* monitor gsyncd already dead */
-                goto out;
-        }
-
-        if (pfd < 0)
+    int32_t ret = 0;
+    int pfd = -1;
+    long pid = 0;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char errmsg[PATH_MAX] = "";
+    char buf[4096] = {
+        0,
+    };
+    gf_boolean_t is_template_in_use = _gf_false;
+    char monitor_status[NAME_MAX] = {
+        0,
+    };
+    char *statefile = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(master);
+    GF_ASSERT(slave);
+    GF_ASSERT(slave_host);
+    GF_ASSERT(slave_vol);
+    GF_ASSERT(conf_path);
+
+    pfd = gsyncd_getpidfile(master, slave, pidfile, conf_path,
+                            &is_template_in_use);
+    if (pfd == -2) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "pid-file entry mising in config file and "
+                 "template config file.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND, "%s",
+               errmsg);
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    if (gsync_status_byfd(pfd) == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "gsyncd b/w %s & %s is not running", master, slave);
+        /* monitor gsyncd already dead */
+        goto out;
+    }
+
+    if (pfd < 0)
+        goto out;
+
+    /* Prepare to update status file*/
+    ret = dict_get_str(dict, "statefile", &statefile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Pause/Resume Failed: Unable to fetch statefile path");
+        goto out;
+    }
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAT_FILE_READ_FAILED,
+               "Pause/Resume Failed: "
+               "Unable to read status file for %s(master)"
+               " %s(slave)",
+               master, slave);
+        goto out;
+    }
+
+    ret = sys_read(pfd, buf, sizeof(buf) - 1);
+    if (ret > 0) {
+        buf[ret] = '\0';
+        pid = strtol(buf, NULL, 10);
+        if (is_pause) {
+            ret = kill(-pid, SIGSTOP);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                       "Failed"
+                       " to pause gsyncd. Error: %s",
+                       strerror(errno));
                 goto out;
-
-        /* Prepare to update status file*/
-        ret = dict_get_str (dict, "statefile", &statefile);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Pause/Resume Failed: Unable to fetch statefile path");
+            }
+            /*On pause force, if status is already paused
+              do not update status again*/
+            if (strstr(monitor_status, "Paused"))
+                goto out;
+
+            ret = glusterd_create_status_file(master, slave, slave_host,
+                                              slave_vol, "Paused");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_UPDATE_STATEFILE_FAILED,
+                       "Unable  to update state_file."
+                       " Error : %s",
+                       strerror(errno));
+                /* If status cannot be updated resume back */
+                if (kill(-pid, SIGCONT)) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             "Pause successful but could "
+                             "not update status file. "
+                             "Please use 'resume force' to"
+                             " resume back and retry pause"
+                             " to reflect in status");
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_PID_KILL_FAIL,
+                           "Resume back Failed. Error:"
+                           "%s",
+                           strerror(errno));
+                    *op_errstr = gf_strdup(errmsg);
+                }
                 goto out;
-        }
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                              sizeof (monitor_status));
-        if (ret <= 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STAT_FILE_READ_FAILED, "Pause/Resume Failed: "
-                        "Unable to read status file for %s(master)"
-                        " %s(slave)", master, slave);
+            }
+        } else {
+            ret = glusterd_create_status_file(master, slave, slave_host,
+                                              slave_vol, "Started");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_UPDATE_STATEFILE_FAILED,
+                       "Resume Failed: Unable to update "
+                       "state_file. Error : %s",
+                       strerror(errno));
                 goto out;
-        }
-
-        ret = sys_read (pfd, buf, 1024);
-        if (ret > 0) {
-                pid = strtol (buf, NULL, 10);
-                if (is_pause) {
-                        ret = kill (-pid, SIGSTOP);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_PID_KILL_FAIL, "Failed"
-                                        " to pause gsyncd. Error: %s",
-                                        strerror (errno));
-                                goto out;
-                        }
-                        /*On pause force, if status is already paused
-                          do not update status again*/
-                        if (strstr (monitor_status, "Paused"))
-                                goto out;
-
-                        ret = glusterd_create_status_file ( master, slave,
-                                                     slave_host, slave_vol,
-                                                     "Paused");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_UPDATE_STATEFILE_FAILED,
-                                        "Unable  to update state_file."
-                                        " Error : %s", strerror (errno));
-                                /* If status cannot be updated resume back */
-                                if (kill (-pid, SIGCONT)) {
-                                        snprintf (errmsg, sizeof(errmsg),
-                                                  "Pause successful but could "
-                                                  "not update status file. "
-                                                  "Please use 'resume force' to"
-                                                  " resume back and retry pause"
-                                                  " to reflect in status");
-                                        gf_msg (this->name, GF_LOG_ERROR,
-                                                errno,
-                                                GD_MSG_PID_KILL_FAIL,
-                                                "Resume back Failed. Error:"
-                                                "%s", strerror (errno));
-                                        *op_errstr = gf_strdup (errmsg);
-                                }
-                                goto out;
-                        }
-                } else {
-                        ret = glusterd_create_status_file (master, slave,
-                                                           slave_host,
-                                                           slave_vol,
-                                                           "Started");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_UPDATE_STATEFILE_FAILED,
-                                        "Resume Failed: Unable to update "
-                                        "state_file. Error : %s",
-                                        strerror (errno));
-                                goto out;
-                        }
-                        ret = kill (-pid, SIGCONT);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_PID_KILL_FAIL,
-                                        "Resumed Failed: Unable to send"
-                                        " SIGCONT. Error: %s",
-                                         strerror (errno));
-                                /* Process can't be resumed, update status
-                                 * back to paused. */
-                                ret = glusterd_create_status_file (master,
-                                                                   slave,
-                                                                   slave_host,
-                                                                   slave_vol,
-                                                                monitor_status);
-                                if (ret) {
-                                        snprintf (errmsg, sizeof(errmsg),
-                                                  "Resume failed!!! Status "
-                                                  "inconsistent. Please use "
-                                                  "'resume force' to resume and"
-                                                  " reach consistent state");
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_STATUS_UPDATE_FAILED,
-                                                "Updating status back to paused"
-                                                " Failed. Error: %s",
-                                                strerror (errno));
-                                        *op_errstr = gf_strdup (errmsg);
-                                }
-                                goto out;
-                        }
+            }
+            ret = kill(-pid, SIGCONT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                       "Resumed Failed: Unable to send"
+                       " SIGCONT. Error: %s",
+                       strerror(errno));
+                /* Process can't be resumed, update status
+                 * back to paused. */
+                ret = glusterd_create_status_file(master, slave, slave_host,
+                                                  slave_vol, monitor_status);
+                if (ret) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             "Resume failed!!! Status "
+                             "inconsistent. Please use "
+                             "'resume force' to resume and"
+                             " reach consistent state");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_STATUS_UPDATE_FAILED,
+                           "Updating status back to paused"
+                           " Failed. Error: %s",
+                           strerror(errno));
+                    *op_errstr = gf_strdup(errmsg);
                 }
+                goto out;
+            }
         }
-        ret = 0;
+    }
+    ret = 0;
 
 out:
-        sys_close (pfd);
-        return ret;
+    sys_close(pfd);
+    /* coverity[INTEGER_OVERFLOW] */
+    return ret;
 }
 
 static int
-stop_gsync (char *master, char *slave, char **msg,
-            char *conf_path, char **op_errstr,
-            gf_boolean_t is_force)
+stop_gsync(char *master, char *slave, char **msg, char *conf_path,
+           char **op_errstr, gf_boolean_t is_force)
 {
-        int32_t         ret     = 0;
-        int             pfd     = -1;
-        pid_t           pid     = 0;
-        char            pidfile[PATH_MAX] = {0,};
-        char            errmsg[PATH_MAX] = "";
-        char            buf[1024] = {0,};
-        int             i       = 0;
-        gf_boolean_t    is_template_in_use = _gf_false;
-        xlator_t        *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (this->private);
-
-        pfd = gsyncd_getpidfile (master, slave, pidfile,
-                                 conf_path, &is_template_in_use);
-        if (pfd == -2) {
-                snprintf (errmsg, sizeof(errmsg) - 1,
-                          "pid-file entry mising in config file and "
-                          "template config file.");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND,
-                        "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-        if (gsync_status_byfd (pfd) == -1 && !is_force) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
-                        "gsyncd b/w %s & %s is not running", master,
-                        slave);
-                /* monitor gsyncd already dead */
-                goto out;
-        }
-
-        if (pfd < 0)
-                goto out;
-
-        ret = sys_read (pfd, buf, 1024);
-        if (ret > 0) {
-                pid = strtol (buf, NULL, 10);
-                ret = kill (-pid, SIGTERM);
-                if (ret && !is_force) {
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                GD_MSG_PID_KILL_FAIL,
-                                "failed to kill gsyncd");
-                        goto out;
-                }
-                for (i = 0; i < 20; i++) {
-                        if (gsync_status_byfd (pfd) == -1) {
-                                /* monitor gsyncd is dead but worker may
-                                 * still be alive, give some more time
-                                 * before SIGKILL (hack)
-                                 */
-                                usleep (50000);
-                                break;
-                        }
-                        usleep (50000);
-                }
-                kill (-pid, SIGKILL);
-                sys_unlink (pidfile);
+    int32_t ret = 0;
+    int pfd = -1;
+    long pid = 0;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char errmsg[PATH_MAX] = "";
+    char buf[4096] = {
+        0,
+    };
+    int i = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+
+    pfd = gsyncd_getpidfile(master, slave, pidfile, conf_path,
+                            &is_template_in_use);
+    if (pfd == -2) {
+        snprintf(errmsg, sizeof(errmsg) - 1,
+                 "pid-file entry mising in config file and "
+                 "template config file.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND, "%s",
+               errmsg);
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+    if (gsync_status_byfd(pfd) == -1 && !is_force) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "gsyncd b/w %s & %s is not running", master, slave);
+        /* monitor gsyncd already dead */
+        goto out;
+    }
+
+    if (pfd < 0)
+        goto out;
+
+    ret = sys_read(pfd, buf, sizeof(buf) - 1);
+    if (ret > 0) {
+        buf[ret] = '\0';
+        pid = strtol(buf, NULL, 10);
+        ret = kill(-pid, SIGTERM);
+        if (ret && !is_force) {
+            gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_PID_KILL_FAIL,
+                   "failed to kill gsyncd");
+            goto out;
+        }
+        for (i = 0; i < 20; i++) {
+            if (gsync_status_byfd(pfd) == -1) {
+                /* monitor gsyncd is dead but worker may
+                 * still be alive, give some more time
+                 * before SIGKILL (hack)
+                 */
+                gf_nanosleep(50000 * GF_US_IN_NS);
+                break;
+            }
+            gf_nanosleep(50000 * GF_US_IN_NS);
         }
-        ret = 0;
+        kill(-pid, SIGKILL);
+        sys_unlink(pidfile);
+    }
+    ret = 0;
 
 out:
-        sys_close (pfd);
-
-        return ret;
+    sys_close(pfd);
+    /* coverity[INTEGER_OVERFLOW] */
+    return ret;
 }
 
 /*
  * glusterd_gsync_op_already_set:
- *      This funcion checks whether the op_value is same as in the
+ *      This function checks whether the op_value is same as in the
  *      gsyncd.conf file.
  *
  * RETURN VALUE:
@@ -3931,2509 +4238,2545 @@ out:
  */
 
 int
-glusterd_gsync_op_already_set (char* master, char* slave, char* conf_path,
-                                             char* op_name, char* op_value)
+glusterd_gsync_op_already_set(char *master, char *slave, char *conf_path,
+                              char *op_name, char *op_value)
 {
-        dict_t                 *confd                       = NULL;
-        char                   *op_val_buf                  = NULL;
-        int32_t                 op_val_conf                 = 0;
-        int32_t                 op_val_cli                  = 0;
-        int32_t                 ret                         = -1;
-        gf_boolean_t            is_bool                     = _gf_true;
-        xlator_t                *this                       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        confd = dict_new ();
-        if (!confd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
-                        "Not able to create dict.");
-                return -1;
-        }
-
-        ret = glusterd_gsync_get_config (master, slave, conf_path,
-                                         confd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GET_CONFIG_INFO_FAILED,
-                        "Unable to get configuration data for %s(master), "
-                        "%s(slave)", master, slave);
-                goto out;
-        }
-
-        ret = dict_get_param (confd, op_name, &op_val_buf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to get op_value for %s(master), %s(slave). "
-                        "Please check gsync config file.", master, slave);
-                ret = 1;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "val_cli:%s  val_conf:%s", op_value,
-                      op_val_buf);
-
-        if (!strcmp(op_val_buf,"true") || !strcmp(op_val_buf,"1")
-            || !strcmp(op_val_buf,"yes")) {
-                op_val_conf = 1;
-        } else if(!strcmp(op_val_buf,"false") || !strcmp(op_val_buf,"0")
-                || !strcmp(op_val_buf,"no")) {
-                op_val_conf = 0;
+    dict_t *confd = NULL;
+    char *op_val_buf = NULL;
+    int32_t op_val_conf = 0;
+    int32_t op_val_cli = 0;
+    int32_t ret = -1;
+    gf_boolean_t is_bool = _gf_true;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Not able to create dict.");
+        return -1;
+    }
+
+    ret = glusterd_gsync_get_config(master, slave, conf_path, confd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "Unable to get configuration data for %s(master), "
+               "%s(slave)",
+               master, slave);
+        goto out;
+    }
+
+    ret = dict_get_param(confd, op_name, &op_val_buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get op_value for %s(master), %s(slave). "
+               "Please check gsync config file.",
+               master, slave);
+        ret = 1;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "val_cli:%s  val_conf:%s", op_value,
+                 op_val_buf);
+
+    if (!strcmp(op_val_buf, "true") || !strcmp(op_val_buf, "1") ||
+        !strcmp(op_val_buf, "yes")) {
+        op_val_conf = 1;
+    } else if (!strcmp(op_val_buf, "false") || !strcmp(op_val_buf, "0") ||
+               !strcmp(op_val_buf, "no")) {
+        op_val_conf = 0;
+    } else {
+        is_bool = _gf_false;
+    }
+
+    if (is_bool) {
+        if (op_value && (!strcmp(op_value, "true") || !strcmp(op_value, "1") ||
+                         !strcmp(op_value, "yes"))) {
+            op_val_cli = 1;
         } else {
-                is_bool = _gf_false;
+            op_val_cli = 0;
         }
 
-        if (is_bool) {
-                if (!strcmp(op_value,"true") || !strcmp(op_value,"1")
-                    || !strcmp(op_value,"yes")) {
-                        op_val_cli = 1;
-                } else {
-                        op_val_cli = 0;
-                }
-
-                if ( op_val_cli == op_val_conf ) {
-                        ret = 0;
-                        goto out;
-                }
-        } else {
-                if (!strcmp(op_val_buf,op_value)) {
-                        ret = 0;
-                        goto out;
-                }
+        if (op_val_cli == op_val_conf) {
+            ret = 0;
+            goto out;
+        }
+    } else {
+        if (op_value && !strcmp(op_val_buf, op_value)) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        ret = 1;
+    ret = 1;
 
 out:
-        dict_unref(confd);
-        return ret;
+    dict_unref(confd);
+    return ret;
 }
 
 static int
-glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
-                          char *path_list, dict_t *dict,
-                          dict_t *resp_dict, char **op_errstr)
+glusterd_gsync_configure(glusterd_volinfo_t *volinfo, char *slave,
+                         char *path_list, dict_t *dict, dict_t *resp_dict,
+                         char **op_errstr)
 {
-        int32_t         ret     = -1;
-        char            *op_name = NULL;
-        char            *op_value = NULL;
-        runner_t        runner    = {0,};
-        glusterd_conf_t *priv   = NULL;
-        char            *subop  = NULL;
-        char            *master = NULL;
-        char            *conf_path = NULL;
-        char            *slave_host = NULL;
-        char            *slave_vol = NULL;
-        struct stat      stbuf     = {0, };
-        gf_boolean_t     restart_required = _gf_true;
-        char           **resopt    = NULL;
-        gf_boolean_t     op_already_set = _gf_false;
-        xlator_t        *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (slave);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-        GF_ASSERT (resp_dict);
-
-        ret = dict_get_str (dict, "subop", &subop);
-        if (ret != 0)
-                goto out;
+    int32_t ret = -1;
+    char *op_name = NULL;
+    char *op_value = NULL;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char *subop = NULL;
+    char *master = NULL;
+    char *conf_path = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    gf_boolean_t restart_required = _gf_true;
+    char **resopt = NULL;
+    gf_boolean_t op_already_set = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(slave);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+    GF_ASSERT(resp_dict);
+
+    ret = dict_get_str(dict, "subop", &subop);
+    if (ret != 0)
+        goto out;
+
+    if (strcmp(subop, "get") == 0 || strcmp(subop, "get-all") == 0) {
+        /* deferred to cli */
+        gf_msg_debug(this->name, 0, "Returning 0");
+        return 0;
+    }
 
-        if (strcmp (subop, "get") == 0 || strcmp (subop, "get-all") == 0) {
-                /* deferred to cli */
-                gf_msg_debug (this->name, 0, "Returning 0");
-                return 0;
-        }
+    ret = dict_get_str(dict, "op_name", &op_name);
+    if (ret != 0)
+        goto out;
 
-        ret = dict_get_str (dict, "op_name", &op_name);
+    if (strtail(subop, "set")) {
+        ret = dict_get_str(dict, "op_value", &op_value);
         if (ret != 0)
-                goto out;
-
-        if (strtail (subop, "set")) {
-                ret = dict_get_str (dict, "op_value", &op_value);
-                if (ret != 0)
-                        goto out;
-        }
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
+            goto out;
+    }
+
+    priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    master = "";
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    if (volinfo) {
+        master = volinfo->volname;
+        runner_argprintf(&runner, ":%s", master);
+    }
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
+    runner_add_arg(&runner, op_name);
+    if (op_value) {
+        runner_argprintf(&runner, "--value=%s", op_value);
+    }
+
+    if (strcmp(op_name, "checkpoint") != 0 && strtail(subop, "set")) {
+        ret = glusterd_gsync_op_already_set(master, slave, conf_path, op_name,
+                                            op_value);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GSYNCD_OP_SET_FAILED,
+                   "glusterd_gsync_op_already_set failed.");
+            gf_asprintf(op_errstr,
+                        GEOREP
+                        " config-%s failed for "
+                        "%s %s",
+                        subop, master, slave);
+            goto out;
         }
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch conf file path.");
-                goto out;
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0, "op_value is already set");
+            op_already_set = _gf_true;
+            goto out;
         }
+    }
 
-        master = "";
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        if (volinfo) {
-                master = volinfo->volname;
-                runner_argprintf (&runner, ":%s", master);
-        }
-        runner_add_arg (&runner, slave);
-        runner_argprintf (&runner, "--config-%s", subop);
-        runner_add_arg (&runner, op_name);
-        if (op_value)
-                runner_add_arg (&runner, op_value);
-
-        if ( strcmp(op_name,"checkpoint") != 0 && strtail (subop, "set")) {
-                ret = glusterd_gsync_op_already_set(master,slave,conf_path,
-                                                           op_name,op_value);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_GSYNCD_OP_SET_FAILED,
-                                "glusterd_gsync_op_already_set failed.");
-                        gf_asprintf (op_errstr, GEOREP" config-%s failed for "
-                                     "%s %s", subop, master, slave);
-                        goto out;
-                }
-                if (ret == 0) {
-                        gf_msg_debug (this->name, 0, "op_value is already set");
-                        op_already_set = _gf_true;
-                        goto out;
-                }
-        }
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GSYNCD_ERROR,
+               "gsyncd failed to %s %s option for "
+               "%s %s peers",
+               subop, op_name, master, slave);
 
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_GSYNCD_ERROR,
-                        "gsyncd failed to %s %s option for "
-                        "%s %s peers", subop, op_name, master,
-                        slave);
+        gf_asprintf(op_errstr, GEOREP " config-%s failed for %s %s", subop,
+                    master, slave);
 
-                gf_asprintf (op_errstr, GEOREP" config-%s failed for %s %s",
-                             subop, master, slave);
+        goto out;
+    }
 
+    if ((!strcmp(op_name, "state_file")) && (op_value)) {
+        ret = sys_lstat(op_value, &stbuf);
+        if (ret) {
+            ret = dict_get_str(dict, "slave_host", &slave_host);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch slave host.");
                 goto out;
-        }
+            }
 
-        if ((!strcmp (op_name, "state_file")) && (op_value)) {
+            ret = dict_get_str(dict, "slave_vol", &slave_vol);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch slave volume name.");
+                goto out;
+            }
 
-                ret = sys_lstat (op_value, &stbuf);
-                if (ret) {
-                        ret = dict_get_str (dict, "slave_host", &slave_host);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to fetch slave host.");
-                                goto out;
-                        }
-
-                        ret = dict_get_str (dict, "slave_vol", &slave_vol);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to fetch slave volume name.");
-                                goto out;
-                        }
-
-                        ret = glusterd_create_status_file (volinfo->volname,
-                                                           slave, slave_host,
-                                                           slave_vol,
-                                                           "Switching Status "
-                                                           "File");
-                        if (ret || sys_lstat (op_value, &stbuf)) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_FILE_OP_FAILED, "Unable to "
-                                        "create %s. Error : %s", op_value,
-                                        strerror (errno));
-                                ret = -1;
-                                goto out;
-                        }
-                }
+            ret = glusterd_create_status_file(volinfo->volname, slave,
+                                              slave_host, slave_vol,
+                                              "Switching Status "
+                                              "File");
+            if (ret || sys_lstat(op_value, &stbuf)) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                       "Unable to "
+                       "create %s. Error : %s",
+                       op_value, strerror(errno));
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = 0;
-        gf_asprintf (op_errstr, "config-%s successful", subop);
+    ret = 0;
+    gf_asprintf(op_errstr, "config-%s successful", subop);
 
 out:
-        if (!ret && volinfo && !op_already_set) {
-            for (resopt = gsync_no_restart_opts; *resopt; resopt++) {
-                restart_required = _gf_true;
-                if (!strcmp ((*resopt), op_name)){
-                    restart_required = _gf_false;
-                    break;
-                }
+    if (!ret && volinfo && !op_already_set) {
+        for (resopt = gsync_no_restart_opts; *resopt; resopt++) {
+            restart_required = _gf_true;
+            if (!strcmp((*resopt), op_name)) {
+                restart_required = _gf_false;
+                break;
             }
+        }
 
-            if (restart_required) {
-                ret = glusterd_check_restart_gsync_session (volinfo, slave,
-                                                            resp_dict, path_list,
-                                                            conf_path, 0);
-                if (ret)
-                    *op_errstr = gf_strdup ("internal error");
-            }
+        if (restart_required) {
+            ret = glusterd_check_restart_gsync_session(
+                volinfo, slave, resp_dict, path_list, conf_path, 0);
+            if (ret)
+                *op_errstr = gf_strdup("internal error");
         }
+    }
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen)
+glusterd_gsync_read_frm_status(char *path, char *buf, size_t blen)
 {
-        int                 ret = 0;
-        int           status_fd = -1;
-        xlator_t          *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (path);
-        GF_ASSERT (buf);
-        status_fd = open (path, O_RDONLY);
-        if (status_fd == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
-                        "Unable to read gsyncd status file %s", path);
-                return -1;
-        }
-        ret = sys_read (status_fd, buf, blen - 1);
-        if (ret > 0) {
-                size_t len = strnlen (buf, ret);
-                /* Ensure there is a NUL byte and that it's not the first.  */
-                if (len == 0 || len == blen - 1) {
-                        ret = -1;
-                } else {
-                        char *p = buf + len - 1;
-                        while (isspace (*p))
-                                *p-- = '\0';
-                }
-        } else if (ret < 0)
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
-                        "Status file of gsyncd is corrupt");
-
-        sys_close (status_fd);
-        return ret;
+    int ret = 0;
+    int status_fd = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(path);
+    GF_ASSERT(buf);
+    status_fd = open(path, O_RDONLY);
+    if (status_fd == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+               "Unable to read gsyncd status file %s", path);
+        return -1;
+    }
+    ret = sys_read(status_fd, buf, blen - 1);
+    if (ret > 0) {
+        size_t len = strnlen(buf, ret);
+        /* Ensure there is a NUL byte and that it's not the first.  */
+        if (len == 0 || len == blen - 1) {
+            ret = -1;
+        } else {
+            char *p = buf + len - 1;
+            while (isspace(*p))
+                *p-- = '\0';
+        }
+    } else if (ret == 0)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "Status file of gsyncd is empty");
+    else /* ret < 0 */
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "Status file of gsyncd is corrupt");
+
+    sys_close(status_fd);
+    return ret;
 }
 
 static int
-dict_get_param (dict_t *dict, char *key, char **param)
+dict_get_param(dict_t *dict, char *key, char **param)
 {
-        char  *dk = NULL;
-        char   *s = NULL;
-        char    x = '\0';
-        int   ret = 0;
+    char *dk = NULL;
+    char *s = NULL;
+    char x = '\0';
+    int ret = 0;
 
-        if (dict_get_str (dict, key, param) == 0)
-                return 0;
+    if (dict_get_str(dict, key, param) == 0)
+        return 0;
 
-        dk = gf_strdup (key);
-        if (!key)
-                return -1;
+    dk = gf_strdup(key);
+    if (!dk)
+        return -1;
 
-        s = strpbrk (dk, "-_");
-        if (!s) {
-                ret = -1;
-                goto out;
-        }
-        x = (*s == '-') ? '_' : '-';
+    s = strpbrk(dk, "-_");
+    if (!s) {
+        ret = -1;
+        goto out;
+    }
+    x = (*s == '-') ? '_' : '-';
+    *s++ = x;
+    while ((s = strpbrk(s, "-_")))
         *s++ = x;
-        while ((s = strpbrk (s, "-_")))
-                *s++ = x;
 
-        ret = dict_get_str (dict, dk, param);
+    ret = dict_get_str(dict, dk, param);
 out:
-        GF_FREE (dk);
-        return ret;
+    GF_FREE(dk);
+    return ret;
 }
 
 int
-glusterd_fetch_values_from_config (char *master, char *slave,
-                                   char *confpath, dict_t *confd,
-                                   char **statefile,
-                                   char **georep_session_wrkng_dir,
-                                   char **socketfile)
+glusterd_fetch_values_from_config(char *master, char *slave, char *confpath,
+                                  dict_t *confd, char **statefile,
+                                  char **georep_session_wrkng_dir,
+                                  char **socketfile)
 {
-        int                     ret                        = 0;
-        xlator_t                *this                      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_gsync_get_config (master, slave, confpath,
-                                         confd);
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_gsync_get_config(master, slave, confpath, confd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "Unable to get configuration data for %s(master), "
+               "%s(slave)",
+               master, slave);
+        goto out;
+    }
+
+    if (statefile) {
+        ret = dict_get_param(confd, "state_file", statefile);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GET_CONFIG_INFO_FAILED,
-                        "Unable to get configuration data for %s(master), "
-                        "%s(slave)", master, slave);
-                goto out;
-        }
-
-        if (statefile) {
-                ret = dict_get_param (confd, "state_file", statefile);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get state_file's name "
-                                "for %s(master), %s(slave). "
-                                "Please check gsync config file.",
-                                master, slave);
-                        goto out;
-                }
-        }
-
-        if (georep_session_wrkng_dir) {
-                ret = dict_get_param (confd, "georep_session_working_dir",
-                                      georep_session_wrkng_dir);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get geo-rep session's "
-                                "working directory name for %s(master), "
-                                "%s(slave). Please check gsync config file.",
-                                master, slave);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get state_file's name "
+                   "for %s(master), %s(slave). "
+                   "Please check gsync config file.",
+                   master, slave);
+            goto out;
+        }
+    }
+
+    if (georep_session_wrkng_dir) {
+        ret = dict_get_param(confd, "georep_session_working_dir",
+                             georep_session_wrkng_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get geo-rep session's "
+                   "working directory name for %s(master), "
+                   "%s(slave). Please check gsync config file.",
+                   master, slave);
+            goto out;
         }
+    }
 
-        if (socketfile) {
-                ret = dict_get_param (confd, "state_socket_unencoded",
-                                      socketfile);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get socket file's name "
-                                "for %s(master), %s(slave). "
-                                "Please check gsync config file.",
-                                master, slave);
-                        goto out;
-                }
+    if (socketfile) {
+        ret = dict_get_param(confd, "state_socket_unencoded", socketfile);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get socket file's name "
+                   "for %s(master), %s(slave). "
+                   "Please check gsync config file.",
+                   master, slave);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_read_status_file (glusterd_volinfo_t *volinfo, char *slave,
-                           char *conf_path, dict_t *dict, char *node)
+glusterd_read_status_file(glusterd_volinfo_t *volinfo, char *slave,
+                          char *conf_path, dict_t *dict, char *node)
 {
-        char                    brick_state_file[PATH_MAX] = "";
-        char                    brick_path[PATH_MAX]       = "";
-        char                    temp_conf_path[PATH_MAX]   = "";
-        char                   *working_conf_path          = NULL;
-        char                   *georep_session_wrkng_dir   = NULL;
-        char                   *master                     = NULL;
-        char                    tmp[1024]                  = "";
-        char                    sts_val_name[1024]         = "";
-        char                    monitor_status[NAME_MAX]   = "";
-        char                   *statefile                  = NULL;
-        char                   *socketfile                 = NULL;
-        dict_t                 *confd                      = NULL;
-        char                   *slavekey                   = NULL;
-        char                   *slaveentry                 = NULL;
-        char                   *slaveuser                  = NULL;
-        char                   *saveptr                    = NULL;
-        char                   *temp                       = NULL;
-        char                   *temp_inp                   = NULL;
-        char                   *brick_host_uuid            = NULL;
-        int                     brick_host_uuid_length     = 0;
-        int                     gsync_count                = 0;
-        int                     i                          = 0;
-        int                     ret                        = 0;
-        glusterd_brickinfo_t   *brickinfo                  = NULL;
-        gf_gsync_status_t      *sts_val                    = NULL;
-        gf_boolean_t            is_template_in_use         = _gf_false;
-        glusterd_conf_t        *priv                       = NULL;
-        struct stat             stbuf                      = {0,};
-        dict_t                 *statusd                    = NULL;
-        xlator_t               *this                       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (this->private);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (conf_path);
-
-        master = volinfo->volname;
-
-        confd = dict_new ();
-        if (!confd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
-                        "Not able to create dict.");
-                return -1;
-        }
-
-        priv = THIS->private;
+    char temp_conf_path[PATH_MAX] = "";
+    char *working_conf_path = NULL;
+    char *georep_session_wrkng_dir = NULL;
+    char *master = NULL;
+    char sts_val_name[1024] = "";
+    char monitor_status[NAME_MAX] = "";
+    char *statefile = NULL;
+    char *socketfile = NULL;
+    dict_t *confd = NULL;
+    char *slavekey = NULL;
+    char *slaveentry = NULL;
+    char *slaveuser = NULL;
+    char *saveptr = NULL;
+    char *temp = NULL;
+    char *temp_inp = NULL;
+    char *brick_host_uuid = NULL;
+    int brick_host_uuid_length = 0;
+    int gsync_count = 0;
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gf_gsync_status_t *sts_val = NULL;
+    gf_boolean_t is_template_in_use = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(conf_path);
+
+    master = volinfo->volname;
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Not able to create dict.");
+        return -1;
+    }
 
-        snprintf (temp_conf_path, sizeof(temp_conf_path) - 1,
-                  "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+    priv = THIS->private;
 
-        ret = sys_lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_CONFIG_INFO,
-                        "Using passed config template(%s).",
-                        conf_path);
-                working_conf_path = conf_path;
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, ENOENT,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Config file (%s) missing. Looking for template "
-                        "config file (%s)", conf_path, temp_conf_path);
-                ret = sys_lstat (temp_conf_path, &stbuf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                                GD_MSG_FILE_OP_FAILED, "Template "
-                                "config file (%s) missing.", temp_conf_path);
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
-                        "Using default config template(%s).", temp_conf_path);
-                working_conf_path = temp_conf_path;
-                is_template_in_use = _gf_true;
-        }
-
-fetch_data:
-        ret = glusterd_fetch_values_from_config (master, slave,
-                                                 working_conf_path,
-                                                 confd,
-                                                 &statefile,
-                                                 &georep_session_wrkng_dir,
-                                                 &socketfile);
+    len = snprintf(temp_conf_path, sizeof(temp_conf_path),
+                   "%s/" GSYNC_CONF_TEMPLATE, priv->workdir);
+    if ((len < 0) || (len >= sizeof(temp_conf_path))) {
+        return -1;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONFIG_INFO,
+               "Using passed config template(%s).", conf_path);
+        working_conf_path = conf_path;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Config file (%s) missing. Looking for template "
+               "config file (%s)",
+               conf_path, temp_conf_path);
+        ret = sys_lstat(temp_conf_path, &stbuf);
         if (ret) {
-                if (is_template_in_use == _gf_false) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FETCH_CONFIG_VAL_FAILED,
-                                "Unable to fetch config values "
-                                "for %s(master), %s(slave). "
-                                "Trying default config template",
-                                master, slave);
-                        working_conf_path = temp_conf_path;
-                        is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FETCH_CONFIG_VAL_FAILED, "Unable to "
-                                "fetch config values for %s(master), "
-                                "%s(slave)", master, slave);
-                        goto out;
-                }
-        }
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "Template "
+                   "config file (%s) missing.",
+                   temp_conf_path);
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
+               "Using default config template(%s).", temp_conf_path);
+        working_conf_path = temp_conf_path;
+        is_template_in_use = _gf_true;
+    }
 
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                              sizeof (monitor_status));
-        if (ret <= 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STAT_FILE_READ_FAILED,
-                        "Unable to read the status file for %s(master), "
-                        "%s(slave) statefile: %s", master, slave,
-                         statefile);
-                strncpy (monitor_status, "defunct", sizeof (monitor_status));
+fetch_data:
+    ret = glusterd_fetch_values_from_config(
+        master, slave, working_conf_path, confd, &statefile,
+        &georep_session_wrkng_dir, &socketfile);
+    if (ret) {
+        if (is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FETCH_CONFIG_VAL_FAILED,
+                   "Unable to fetch config values "
+                   "for %s(master), %s(slave). "
+                   "Trying default config template",
+                   master, slave);
+            working_conf_path = temp_conf_path;
+            is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FETCH_CONFIG_VAL_FAILED,
+                   "Unable to "
+                   "fetch config values for %s(master), "
+                   "%s(slave)",
+                   master, slave);
+            goto out;
+        }
+    }
+
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAT_FILE_READ_FAILED,
+               "Unable to read the status file for %s(master), "
+               "%s(slave) statefile: %s",
+               master, slave, statefile);
+        snprintf(monitor_status, sizeof(monitor_status), "defunct");
+    }
+
+    ret = dict_get_int32(dict, "gsync-count", &gsync_count);
+    if (ret)
+        gsync_count = 0;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        sts_val = GF_CALLOC(1, sizeof(gf_gsync_status_t),
+                            gf_common_mt_gsync_status_t);
+        if (!sts_val) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Out Of Memory");
+            goto out;
+        }
+
+        /* Slave Key */
+        ret = glusterd_get_slave(volinfo, slave, &slavekey);
+        if (ret < 0) {
+            GF_FREE(sts_val);
+            goto out;
         }
+        memcpy(sts_val->slavekey, slavekey, strlen(slavekey));
+        sts_val->slavekey[strlen(slavekey)] = '\0';
 
-        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
-        if (ret)
-                gsync_count = 0;
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
-
-                sts_val = GF_CALLOC (1, sizeof(gf_gsync_status_t),
-                                     gf_common_mt_gsync_status_t);
-                if (!sts_val) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY,
-                                "Out Of Memory");
-                        goto out;
-                }
-
-                /* Slave Key */
-                ret = glusterd_get_slave (volinfo, slave, &slavekey);
-                if (ret < 0) {
-                        GF_FREE (sts_val);
-                        goto out;
-                }
-                memcpy (sts_val->slavekey, slavekey, strlen(slavekey));
-                sts_val->slavekey[strlen(slavekey)] = '\0';
-
-                /* Master Volume */
-                memcpy (sts_val->master, master, strlen(master));
-                sts_val->master[strlen(master)] = '\0';
+        /* Master Volume */
+        memcpy(sts_val->master, master, strlen(master));
+        sts_val->master[strlen(master)] = '\0';
 
-                /* Master Brick Node */
-                memcpy (sts_val->node, brickinfo->hostname,
-                        strlen(brickinfo->hostname));
-                sts_val->node[strlen(brickinfo->hostname)] = '\0';
+        /* Master Brick Node */
+        memcpy(sts_val->node, brickinfo->hostname, strlen(brickinfo->hostname));
+        sts_val->node[strlen(brickinfo->hostname)] = '\0';
 
-                /* Master Brick Path */
-                memcpy (sts_val->brick, brickinfo->path,
-                        strlen(brickinfo->path));
-                sts_val->brick[strlen(brickinfo->path)] = '\0';
+        /* Master Brick Path */
+        memcpy(sts_val->brick, brickinfo->path, strlen(brickinfo->path));
+        sts_val->brick[strlen(brickinfo->path)] = '\0';
 
-                /* Brick Host UUID */
-                brick_host_uuid = uuid_utoa(brickinfo->uuid);
-                brick_host_uuid_length = strlen (brick_host_uuid);
-                memcpy (sts_val->brick_host_uuid, brick_host_uuid,
-                        brick_host_uuid_length);
-                sts_val->brick_host_uuid[brick_host_uuid_length] = '\0';
+        /* Brick Host UUID */
+        brick_host_uuid = uuid_utoa(brickinfo->uuid);
+        brick_host_uuid_length = strlen(brick_host_uuid);
+        memcpy(sts_val->brick_host_uuid, brick_host_uuid,
+               brick_host_uuid_length);
+        sts_val->brick_host_uuid[brick_host_uuid_length] = '\0';
 
-                /* Slave */
-                memcpy (sts_val->slave, slave, strlen(slave));
-                sts_val->slave[strlen(slave)] = '\0';
+        /* Slave */
+        memcpy(sts_val->slave, slave, strlen(slave));
+        sts_val->slave[strlen(slave)] = '\0';
 
-                snprintf (sts_val->slave_node,
-                          sizeof(sts_val->slave_node), "N/A");
+        snprintf(sts_val->slave_node, sizeof(sts_val->slave_node), "N/A");
 
-                snprintf (sts_val->worker_status,
-                          sizeof(sts_val->worker_status), "N/A");
+        snprintf(sts_val->worker_status, sizeof(sts_val->worker_status), "N/A");
 
-                snprintf (sts_val->crawl_status,
-                          sizeof(sts_val->crawl_status), "N/A");
+        snprintf(sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
 
-                snprintf (sts_val->last_synced,
-                          sizeof(sts_val->last_synced), "N/A");
+        snprintf(sts_val->last_synced, sizeof(sts_val->last_synced), "N/A");
 
-                snprintf (sts_val->last_synced_utc,
-                          sizeof(sts_val->last_synced_utc), "N/A");
+        snprintf(sts_val->last_synced_utc, sizeof(sts_val->last_synced_utc),
+                 "N/A");
 
-                snprintf (sts_val->entry, sizeof(sts_val->entry), "N/A");
+        snprintf(sts_val->entry, sizeof(sts_val->entry), "N/A");
 
-                snprintf (sts_val->data, sizeof(sts_val->data), "N/A");
+        snprintf(sts_val->data, sizeof(sts_val->data), "N/A");
 
-                snprintf (sts_val->meta, sizeof(sts_val->meta), "N/A");
+        snprintf(sts_val->meta, sizeof(sts_val->meta), "N/A");
 
-                snprintf (sts_val->failures, sizeof(sts_val->failures), "N/A");
+        snprintf(sts_val->failures, sizeof(sts_val->failures), "N/A");
 
-                snprintf (sts_val->checkpoint_time,
-                          sizeof(sts_val->checkpoint_time), "N/A");
+        snprintf(sts_val->checkpoint_time, sizeof(sts_val->checkpoint_time),
+                 "N/A");
 
-                snprintf (sts_val->checkpoint_time_utc,
-                          sizeof(sts_val->checkpoint_time_utc), "N/A");
+        snprintf(sts_val->checkpoint_time_utc,
+                 sizeof(sts_val->checkpoint_time_utc), "N/A");
 
-                snprintf (sts_val->checkpoint_completed,
-                          sizeof(sts_val->checkpoint_completed), "N/A");
+        snprintf(sts_val->checkpoint_completed,
+                 sizeof(sts_val->checkpoint_completed), "N/A");
 
-                snprintf (sts_val->checkpoint_completion_time,
-                          sizeof(sts_val->checkpoint_completion_time),
-                          "N/A");
+        snprintf(sts_val->checkpoint_completion_time,
+                 sizeof(sts_val->checkpoint_completion_time), "N/A");
 
-                snprintf (sts_val->checkpoint_completion_time_utc,
-                          sizeof(sts_val->checkpoint_completion_time_utc),
-                          "N/A");
+        snprintf(sts_val->checkpoint_completion_time_utc,
+                 sizeof(sts_val->checkpoint_completion_time_utc), "N/A");
 
-                /* Get all the other values from Gsyncd */
-                ret = glusterd_gsync_get_status (master, slave, conf_path,
-                                                 brickinfo->path, sts_val);
+        /* Get all the other values from Gsyncd */
+        ret = glusterd_gsync_get_status(master, slave, conf_path,
+                                        brickinfo->path, sts_val);
 
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GET_STATUS_DATA_FAIL,
-                                "Unable to get status data "
-                                "for %s(master), %s(slave), %s(brick)",
-                                master, slave, brickinfo->path);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (is_template_in_use) {
-                        snprintf (sts_val->worker_status,
-                                  sizeof(sts_val->worker_status),
-                                  "Config Corrupted");
-                }
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_STATUS_DATA_FAIL,
+                   "Unable to get status data "
+                   "for %s(master), %s(slave), %s(brick)",
+                   master, slave, brickinfo->path);
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_get_str (volinfo->gsync_slaves, slavekey,
-                                    &slaveentry);
-                if (ret < 0) {
-                        GF_FREE (sts_val);
-                        goto out;
-                }
+        if (is_template_in_use) {
+            snprintf(sts_val->worker_status, sizeof(sts_val->worker_status),
+                     "Config Corrupted");
+        }
 
+        ret = dict_get_str(volinfo->gsync_slaves, slavekey, &slaveentry);
+        if (ret < 0) {
+            GF_FREE(sts_val);
+            goto out;
+        }
 
-                memcpy (sts_val->session_slave, slaveentry,
-                        strlen(slaveentry));
-                sts_val->session_slave[strlen(slaveentry)] = '\0';
+        memcpy(sts_val->session_slave, slaveentry, strlen(slaveentry));
+        sts_val->session_slave[strlen(slaveentry)] = '\0';
 
-                temp_inp = gf_strdup(slaveentry);
-                if (!temp_inp)
-                        goto out;
+        temp_inp = gf_strdup(slaveentry);
+        if (!temp_inp)
+            goto out;
 
-                if (strstr(temp_inp, "@") == NULL) {
-                        slaveuser = "root";
-                } else {
-                        temp = strtok_r(temp_inp, "//", &saveptr);
-                        temp = strtok_r(NULL, "/", &saveptr);
-                        slaveuser = strtok_r(temp, "@", &saveptr);
-                }
-                memcpy (sts_val->slave_user, slaveuser,
-                        strlen(slaveuser));
-                sts_val->slave_user[strlen(slaveuser)] = '\0';
-
-                snprintf (sts_val_name, sizeof (sts_val_name),
-                          "status_value%d", gsync_count);
-                ret = dict_set_bin (dict, sts_val_name, sts_val,
-                                    sizeof(gf_gsync_status_t));
-                if (ret) {
-                        GF_FREE (sts_val);
-                        goto out;
-                }
+        if (strstr(temp_inp, "@") == NULL) {
+            slaveuser = "root";
+        } else {
+            temp = strtok_r(temp_inp, "//", &saveptr);
+            temp = strtok_r(NULL, "/", &saveptr);
+            slaveuser = strtok_r(temp, "@", &saveptr);
+        }
+        memcpy(sts_val->slave_user, slaveuser, strlen(slaveuser));
+        sts_val->slave_user[strlen(slaveuser)] = '\0';
 
-                gsync_count++;
-                sts_val = NULL;
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d",
+                 gsync_count);
+        ret = dict_set_bin(dict, sts_val_name, sts_val,
+                           sizeof(gf_gsync_status_t));
+        if (ret) {
+            GF_FREE(sts_val);
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "gsync-count", gsync_count);
-        if (ret)
-                goto out;
+        gsync_count++;
+        sts_val = NULL;
+    }
+
+    ret = dict_set_int32(dict, "gsync-count", gsync_count);
+    if (ret)
+        goto out;
 
 out:
-        GF_FREE (temp_inp);
-        dict_unref (confd);
+    GF_FREE(temp_inp);
+    dict_unref(confd);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
-                                      dict_t *resp_dict, char *path_list,
-                                      char *conf_path, gf_boolean_t is_force)
+glusterd_check_restart_gsync_session(glusterd_volinfo_t *volinfo, char *slave,
+                                     dict_t *resp_dict, char *path_list,
+                                     char *conf_path, gf_boolean_t is_force)
 {
-
-        int                    ret              = 0;
-        glusterd_conf_t        *priv            = NULL;
-        char                   *status_msg      = NULL;
-        gf_boolean_t           is_running       = _gf_false;
-        char                   *op_errstr       = NULL;
-        char                   *key             = NULL;
-        xlator_t               *this            = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        key  = slave;
-
-        ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                  slave, conf_path,
-                                                  &is_running);
-        if (!ret && (_gf_true != is_running))
-                /* gsynd not running, nothing to do */
+    int ret = 0;
+    glusterd_conf_t *priv = NULL;
+    char *status_msg = NULL;
+    gf_boolean_t is_running = _gf_false;
+    char *op_errstr = NULL;
+    char *key = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    key = slave;
+
+    ret = glusterd_check_gsync_running_local(volinfo->volname, slave, conf_path,
+                                             &is_running);
+    if (!ret && (_gf_true != is_running))
+        /* gsynd not running, nothing to do */
+        goto out;
+
+    ret = stop_gsync(volinfo->volname, slave, &status_msg, conf_path,
+                     &op_errstr, is_force);
+    if (ret == 0 && status_msg)
+        ret = dict_set_str(resp_dict, "gsync-status", status_msg);
+    if (ret == 0) {
+        dict_del(volinfo->gsync_active_slaves, key);
+        ret = glusterd_start_gsync(volinfo, slave, path_list, conf_path,
+                                   uuid_utoa(MY_UUID), NULL, _gf_false);
+        if (!ret) {
+            /* Add slave to the dict indicating geo-rep session is
+             * running.*/
+            ret = dict_set_dynstr_with_alloc(volinfo->gsync_active_slaves, key,
+                                             "running");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set"
+                       " key:%s value:running in dict. But "
+                       "the config succeeded.",
+                       key);
                 goto out;
-
-        ret = stop_gsync (volinfo->volname, slave, &status_msg,
-                          conf_path, &op_errstr,
-                          is_force);
-        if (ret == 0 && status_msg)
-                ret = dict_set_str (resp_dict, "gsync-status",
-                                    status_msg);
-        if (ret == 0) {
-                dict_del (volinfo->gsync_active_slaves, key);
-                ret = glusterd_start_gsync (volinfo, slave, path_list,
-                                            conf_path, uuid_utoa(MY_UUID),
-                                            NULL, _gf_false);
-                if (!ret) {
-                        /* Add slave to the dict indicating geo-rep session is
-                         * running.*/
-                        ret = dict_set_dynstr_with_alloc (
-                                              volinfo->gsync_active_slaves,
-                                              key, "running");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "Unable to set"
-                                        " key:%s value:running in dict. But "
-                                        "the config succeeded.", key);
-                              goto out;
-                        }
-                }
+            }
         }
+    }
 
- out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    if (op_errstr)
+        GF_FREE(op_errstr);
+    return ret;
 }
 
 static int32_t
-glusterd_marker_changelog_create_volfile (glusterd_volinfo_t *volinfo)
+glusterd_marker_changelog_create_volfile(glusterd_volinfo_t *volinfo)
 {
-        int32_t          ret     = 0;
-        xlator_t         *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
-                        "Unable to create volfile for setting of marker "
-                        "while '"GEOREP" start'");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                goto out;
-
-        if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                ret = glusterd_svcs_manager (volinfo);
-        ret = 0;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to create volfile for setting of marker "
+               "while '" GEOREP " start'");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        goto out;
+
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_svcs_manager(volinfo);
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_set_gsync_knob (glusterd_volinfo_t *volinfo, char *key, int *vc)
+glusterd_set_gsync_knob(glusterd_volinfo_t *volinfo, char *key, int *vc)
 {
-        int   ret          = -1;
-        int   conf_enabled = _gf_false;
-        xlator_t *this     = NULL;
+    int ret = -1;
+    int conf_enabled = _gf_false;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        GF_ASSERT (this->private);
+    GF_ASSERT(this->private);
 
-        conf_enabled = glusterd_volinfo_get_boolean (volinfo, key);
-        if (conf_enabled == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GET_KEY_FAILED,
-                        "failed to get key %s from volinfo", key);
-                goto out;
-        }
+    conf_enabled = glusterd_volinfo_get_boolean(volinfo, key);
+    if (conf_enabled == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_KEY_FAILED,
+               "failed to get key %s from volinfo", key);
+        goto out;
+    }
 
-        ret = 0;
-        if (conf_enabled == _gf_false) {
-                *vc = 1;
-                ret = glusterd_gsync_volinfo_dict_set (volinfo,
-                                                       key, "on");
-        }
+    ret = 0;
+    if (conf_enabled == _gf_false) {
+        *vc = 1;
+        ret = glusterd_gsync_volinfo_dict_set(volinfo, key, "on");
+    }
 
- out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_set_gsync_confs (glusterd_volinfo_t *volinfo)
+glusterd_set_gsync_confs(glusterd_volinfo_t *volinfo)
 {
-        int ret             = -1;
-        int volfile_changed = 0;
+    int ret = -1;
+    int volfile_changed = 0;
 
-        ret = glusterd_set_gsync_knob (volinfo,
-                                       VKEY_MARKER_XTIME, &volfile_changed);
-        if (ret)
-                goto out;
+    ret = glusterd_set_gsync_knob(volinfo, VKEY_MARKER_XTIME, &volfile_changed);
+    if (ret)
+        goto out;
 
-        /**
-         * enable ignore-pid-check blindly as it could be needed for
-         * cascading setups.
-         */
-        ret = glusterd_set_gsync_knob (volinfo, VKEY_MARKER_XTIME_FORCE,
-                                       &volfile_changed);
-        if (ret)
-                goto out;
+    /**
+     * enable ignore-pid-check blindly as it could be needed for
+     * cascading setups.
+     */
+    ret = glusterd_set_gsync_knob(volinfo, VKEY_MARKER_XTIME_FORCE,
+                                  &volfile_changed);
+    if (ret)
+        goto out;
 
-        ret = glusterd_set_gsync_knob (volinfo,
-                                       VKEY_CHANGELOG, &volfile_changed);
-        if (ret)
-                goto out;
+    ret = glusterd_set_gsync_knob(volinfo, VKEY_CHANGELOG, &volfile_changed);
+    if (ret)
+        goto out;
 
-        if (volfile_changed)
-                ret = glusterd_marker_changelog_create_volfile (volinfo);
+    if (volfile_changed)
+        ret = glusterd_marker_changelog_create_volfile(volinfo);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int
-glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
-                                   char *slave, char *conf_path,
-                                   dict_t *rsp_dict, char *node)
+glusterd_get_gsync_status_mst_slv(glusterd_volinfo_t *volinfo, char *slave,
+                                  char *conf_path, dict_t *rsp_dict, char *node)
 {
-        char              *statefile = NULL;
-        uuid_t             uuid = {0, };
-        glusterd_conf_t    *priv = NULL;
-        int                ret = 0;
-        gf_boolean_t       is_template_in_use = _gf_false;
-        struct stat        stbuf = {0, };
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (this->private);
-
-        priv = this->private;
-
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+    char *statefile = NULL;
+    uuid_t uuid = {
+        0,
+    };
+    int ret = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(this->private);
+
+    ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SESSION_INACTIVE,
+               "geo-replication status %s %s : session is not "
+               "active",
+               volinfo->volname, slave);
+
+        ret = glusterd_get_statefile_name(volinfo, slave, conf_path, &statefile,
+                                          &is_template_in_use);
         if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_SESSION_INACTIVE,
-                        "geo-replication status %s %s : session is not "
-                        "active", volinfo->volname, slave);
-
-                ret = glusterd_get_statefile_name (volinfo, slave,
-                                                   conf_path, &statefile,
-                                                   &is_template_in_use);
-                if (ret) {
-                        if (!strstr(slave, "::"))
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        GD_MSG_SLAVE_URL_INVALID,
-                                        "%s is not a valid slave url.", slave);
-                        else
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        GD_MSG_GET_STATEFILE_NAME_FAILED,
-                                        "Unable to get statefile's name");
-                        ret = 0;
-                        goto out;
-                }
-
-                ret = sys_lstat (statefile, &stbuf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, ENOENT,
-                                GD_MSG_FILE_OP_FAILED,
-                                "%s statefile not present.", statefile);
-                        ret = 0;
-                        goto out;
-                }
+            if (!strstr(slave, "::"))
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SLAVE_URL_INVALID,
+                       "%s is not a valid slave url.", slave);
+            else
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       GD_MSG_GET_STATEFILE_NAME_FAILED,
+                       "Unable to get statefile's name");
+            ret = 0;
+            goto out;
+        }
+
+        ret = sys_lstat(statefile, &stbuf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_INFO, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "%s statefile not present.", statefile);
+            ret = 0;
+            goto out;
         }
+    }
 
-        ret = glusterd_read_status_file (volinfo, slave, conf_path,
-                                         rsp_dict, node);
+    ret = glusterd_read_status_file(volinfo, slave, conf_path, rsp_dict, node);
 out:
-        if (statefile)
-                GF_FREE (statefile);
+    if (statefile)
+        GF_FREE(statefile);
 
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
-static int
-glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
-                               char *node)
+int
+glusterd_get_gsync_status_mst(glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
+                              char *node)
 {
-        glusterd_gsync_status_temp_t  param = {0, };
+    glusterd_gsync_status_temp_t param = {
+        0,
+    };
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        param.rsp_dict = rsp_dict;
-        param.volinfo = volinfo;
-        param.node = node;
-        dict_foreach (volinfo->gsync_slaves, _get_status_mst_slv, &param);
+    param.rsp_dict = rsp_dict;
+    param.volinfo = volinfo;
+    param.node = node;
+    dict_foreach(volinfo->gsync_slaves, _get_status_mst_slv, &param);
 
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_get_gsync_status_all (dict_t *rsp_dict, char *node)
+glusterd_get_gsync_status_all(dict_t *rsp_dict, char *node)
 {
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
 
-        int32_t                 ret = 0;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
+    priv = this->private;
 
-        GF_ASSERT (priv);
+    GF_ASSERT(priv);
 
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict, node);
-                if (ret)
-                        goto out;
-        }
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        ret = glusterd_get_gsync_status_mst(volinfo, rsp_dict, node);
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-        return ret;
-
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_get_gsync_status (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_get_gsync_status(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char               *slave  = NULL;
-        char               *volname = NULL;
-        char               *conf_path = NULL;
-        char               errmsg[PATH_MAX] = {0, };
-        gf_boolean_t       exists = _gf_false;
-        glusterd_volinfo_t *volinfo = NULL;
-        int                ret = 0;
-        char my_hostname[256] = {0,};
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = gethostname(my_hostname, 256);
-        if (ret) {
-                /* stick to N/A */
-                (void) strcpy (my_hostname, "N/A");
-        }
-
-        ret = dict_get_str (dict, "master", &volname);
-        if (ret < 0){
-                ret = glusterd_get_gsync_status_all (rsp_dict, my_hostname);
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
-                        "volume name does not exist");
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
+    char *slave = NULL;
+    char *volname = NULL;
+    char *conf_path = NULL;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    char my_hostname[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = gethostname(my_hostname, 256);
+    if (ret) {
+        /* stick to N/A */
+        (void)strcpy(my_hostname, "N/A");
+    }
+
+    ret = dict_get_str(dict, "master", &volname);
+    if (ret < 0) {
+        ret = glusterd_get_gsync_status_all(rsp_dict, my_hostname);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume name does not exist");
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        *op_errstr = gf_strdup(errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0) {
+        ret = glusterd_get_gsync_status_mst(volinfo, rsp_dict, my_hostname);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    ret = glusterd_get_gsync_status_mst_slv(volinfo, slave, conf_path, rsp_dict,
+                                            my_hostname);
 
-
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0) {
-                ret = glusterd_get_gsync_status_mst (volinfo,
-                                                     rsp_dict, my_hostname);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch conf file path.");
-                goto out;
-        }
-
-        ret = glusterd_get_gsync_status_mst_slv (volinfo, slave, conf_path,
-                                                 rsp_dict, my_hostname);
-
- out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_gsync_delete (glusterd_volinfo_t *volinfo, char *slave,
-                       char *slave_host, char *slave_vol, char *path_list,
-                       dict_t *dict, dict_t *resp_dict, char **op_errstr)
+glusterd_gsync_delete(glusterd_volinfo_t *volinfo, char *slave,
+                      char *slave_host, char *slave_vol, char *path_list,
+                      dict_t *dict, dict_t *resp_dict, char **op_errstr)
 {
-        int32_t         ret     = -1;
-        runner_t        runner    = {0,};
-        glusterd_conf_t *priv   = NULL;
-        char            *master = NULL;
-        char            *gl_workdir = NULL;
-        char             geo_rep_dir[PATH_MAX] = "";
-        char            *conf_path = NULL;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (slave);
-        GF_ASSERT (slave_host);
-        GF_ASSERT (slave_vol);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-        GF_ASSERT (resp_dict);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch conf file path.");
-                goto out;
-        }
-
-        gl_workdir = priv->workdir;
-        master = "";
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
-                          "--delete", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-
-        if (volinfo) {
-                master = volinfo->volname;
-                runner_argprintf (&runner, ":%s", master);
-        }
-        runner_add_arg (&runner, slave);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_DEL_FAILED,
-                        "gsyncd failed to delete session info for %s and "
-                        "%s peers", master, slave);
-
-                gf_asprintf (op_errstr, "gsyncd failed to "
-                             "delete session info for %s and %s peers",
-                             master, slave);
-
-                goto out;
-        }
-
-        ret = snprintf (geo_rep_dir, sizeof(geo_rep_dir) - 1,
-                        "%s/"GEOREP"/%s_%s_%s", gl_workdir,
-                        volinfo->volname, slave_host, slave_vol);
-        geo_rep_dir[ret] = '\0';
-
-        ret = sys_rmdir (geo_rep_dir);
-        if (ret) {
-                if (errno == ENOENT)
-                        gf_msg_debug (this->name, 0, "Geo Rep Dir(%s) Not Present.",
-                                      geo_rep_dir);
-                else {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DIR_OP_FAILED,
-                                "Unable to delete Geo Rep Dir(%s). Error: %s",
-                                geo_rep_dir, strerror (errno));
-                        goto out;
-                }
-        }
-
-        ret = 0;
-
-        gf_asprintf (op_errstr, "delete successful");
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char *master = NULL;
+    char *gl_workdir = NULL;
+    char geo_rep_dir[PATH_MAX] = "";
+    char *conf_path = NULL;
+    xlator_t *this = NULL;
+    uint32_t reset_sync_time = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(slave);
+    GF_ASSERT(slave_host);
+    GF_ASSERT(slave_vol);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+    GF_ASSERT(resp_dict);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    gl_workdir = priv->workdir;
+    master = "";
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--delete", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+
+    runner_argprintf(&runner, "--path-list=%s", path_list);
+
+    ret = dict_get_uint32(dict, "reset-sync-time", &reset_sync_time);
+    if (!ret && reset_sync_time) {
+        runner_add_args(&runner, "--reset-sync-time", NULL);
+    }
+
+    if (volinfo) {
+        master = volinfo->volname;
+        runner_argprintf(&runner, ":%s", master);
+    }
+    runner_add_arg(&runner, slave);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_DEL_FAILED,
+               "gsyncd failed to delete session info for %s and "
+               "%s peers",
+               master, slave);
+
+        gf_asprintf(op_errstr,
+                    "gsyncd failed to "
+                    "delete session info for %s and %s peers",
+                    master, slave);
+
+        goto out;
+    }
+
+    ret = snprintf(geo_rep_dir, sizeof(geo_rep_dir) - 1,
+                   "%s/" GEOREP "/%s_%s_%s", gl_workdir, volinfo->volname,
+                   slave_host, slave_vol);
+    geo_rep_dir[ret] = '\0';
+
+    ret = sys_rmdir(geo_rep_dir);
+    if (ret) {
+        if (errno == ENOENT)
+            gf_msg_debug(this->name, 0, "Geo Rep Dir(%s) Not Present.",
+                         geo_rep_dir);
+        else {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Unable to delete Geo Rep Dir(%s). Error: %s", geo_rep_dir,
+                   strerror(errno));
+            goto out;
+        }
+    }
+
+    ret = 0;
+
+    gf_asprintf(op_errstr, "delete successful");
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_sys_exec(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char                   buf[PATH_MAX]          = "";
-        char                   cmd_arg_name[PATH_MAX] = "";
-        char                   output_name[PATH_MAX]  = "";
-        char                   errmsg[PATH_MAX]       = "";
-        char                  *ptr                    = NULL;
-        char                  *bufp                   = NULL;
-        char                  *command                = NULL;
-        char                 **cmd_args               = NULL;
-        int                    ret                    = -1;
-        int                    i                      = -1;
-        int                    cmd_args_count         = 0;
-        int                    output_count           = 0;
-        glusterd_conf_t       *priv                   = NULL;
-        runner_t               runner                 = {0,};
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
+    char buf[PATH_MAX] = "";
+    char cmd_arg_name[PATH_MAX] = "";
+    char output_name[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    char *ptr = NULL;
+    char *bufp = NULL;
+    char *command = NULL;
+    char **cmd_args = NULL;
+    int ret = -1;
+    int i = -1;
+    int cmd_args_count = 0;
+    int output_count = 0;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "command", &command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command from dict");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "cmd_args_count", &cmd_args_count);
+    if (ret)
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "No cmd_args_count");
+
+    if (cmd_args_count) {
+        cmd_args = GF_CALLOC(cmd_args_count, sizeof(char *), gf_common_mt_char);
+        if (!cmd_args) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Unable to calloc. Errno = %s", strerror(errno));
+            goto out;
+        }
+
+        for (i = 1; i <= cmd_args_count; i++) {
+            snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d", i);
+            ret = dict_get_str(dict, cmd_arg_name, &cmd_args[i - 1]);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " %s in dict",
+                       cmd_arg_name);
                 goto out;
+            }
         }
+    }
+
+    runinit(&runner);
+    runner_argprintf(&runner, GSYNCD_PREFIX "/peer_%s", command);
+    for (i = 0; i < cmd_args_count; i++)
+        runner_add_arg(&runner, cmd_args[i]);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to "
+                 "execute command. Error : %s",
+                 strerror(errno));
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CMD_EXEC_FAIL, "%s", errmsg);
+        ret = -1;
+        synclock_lock(&priv->big_lock);
+        goto out;
+    }
+
+    do {
+        ptr = fgets(buf, sizeof(buf), runner_chio(&runner, STDOUT_FILENO));
+        if (ptr) {
+            ret = dict_get_int32(rsp_dict, "output_count", &output_count);
+            if (ret)
+                output_count = 1;
+            else
+                output_count++;
+            snprintf(output_name, sizeof(output_name), "output_%d",
+                     output_count);
+            if (buf[strlen(buf) - 1] == '\n')
+                buf[strlen(buf) - 1] = '\0';
+            bufp = gf_strdup(buf);
+            if (!bufp)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                       "gf_strdup failed.");
+            ret = dict_set_dynstr(rsp_dict, output_name, bufp);
+            if (ret) {
+                GF_FREE(bufp);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "output set "
+                       "failed.");
+            }
+            ret = dict_set_int32(rsp_dict, "output_count", output_count);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "output_count "
+                       "set failed.");
+        }
+    } while (ptr);
+
+    ret = runner_end(&runner);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to "
+                 "end. Error : %s",
+                 strerror(errno));
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNABLE_TO_END, "%s", errmsg);
+        ret = -1;
+        synclock_lock(&priv->big_lock);
+        goto out;
+    }
+    synclock_lock(&priv->big_lock);
+
+    ret = 0;
+out:
+    if (cmd_args) {
+        GF_FREE(cmd_args);
+        cmd_args = NULL;
+    }
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        ret = dict_get_str (dict, "command", &command);
+int
+glusterd_op_copy_file(dict_t *dict, char **op_errstr)
+{
+    char abs_filename[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    char *filename = NULL;
+    char *host_uuid = NULL;
+    char uuid_str[64] = {0};
+    char *contents = NULL;
+    char buf[4096] = "";
+    int ret = -1;
+    int fd = -1;
+    int bytes_writen = 0;
+    int bytes_read = 0;
+    int contents_size = -1;
+    int file_mode = -1;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    gf_boolean_t free_contents = _gf_true;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "host-uuid", &host_uuid);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, "source", &filename);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch filename from dict.");
+        *op_errstr = gf_strdup("command unsuccessful");
+        goto out;
+    }
+    len = snprintf(abs_filename, sizeof(abs_filename), "%s/%s", priv->workdir,
+                   filename);
+    if ((len < 0) || (len >= sizeof(abs_filename))) {
+        ret = -1;
+        goto out;
+    }
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = sys_lstat(abs_filename, &stbuf);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to get command from dict");
-                goto out;
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Source file "
+                           "does not exist in %s",
+                           priv->workdir);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "%s", errmsg);
+            goto out;
+        }
+
+        contents = GF_CALLOC(1, stbuf.st_size + 1, gf_common_mt_char);
+        if (!contents) {
+            snprintf(errmsg, sizeof(errmsg), "Unable to allocate memory");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
+        }
+
+        fd = open(abs_filename, O_RDONLY);
+        if (fd < 0) {
+            len = snprintf(errmsg, sizeof(errmsg), "Unable to open %s",
+                           abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "cmd_args_count", &cmd_args_count);
-        if (ret)
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
-                        "No cmd_args_count");
-
-        if (cmd_args_count) {
-                cmd_args = GF_CALLOC (cmd_args_count, sizeof (char*),
-                                      gf_common_mt_char);
-                if (!cmd_args) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY,
-                                "Unable to calloc. Errno = %s",
-                                strerror(errno));
-                        goto out;
-                }
+        do {
+            ret = sys_read(fd, buf, sizeof(buf) - 1);
+            if (ret > 0) {
+                buf[ret] = '\0';
+                memcpy(contents + bytes_read, buf, ret);
+                bytes_read += ret;
+            }
+        } while (ret > 0);
 
-                for (i=1; i <= cmd_args_count; i++) {
-                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                                  "cmd_arg_%d", i);
-                        ret = dict_get_str (dict, cmd_arg_name, &cmd_args[i-1]);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "Unable to get"
-                                        " %s in dict", cmd_arg_name);
-                                goto out;
-                        }
-                }
+        if (bytes_read != stbuf.st_size) {
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Unable to read all the data from %s", abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_ERROR, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        runinit (&runner);
-        runner_argprintf (&runner, GSYNCD_PREFIX"/peer_%s", command);
-        for (i=0; i < cmd_args_count; i++)
-                runner_add_arg (&runner, cmd_args[i]);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to "
-                          "execute command. Error : %s",
-                          strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_CMD_EXEC_FAIL, "%s",
-                        errmsg);
-                ret = -1;
-                synclock_lock (&priv->big_lock);
-                goto out;
+        ret = dict_set_int32(dict, "contents_size", stbuf.st_size);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " contents size in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+                   errmsg);
+            goto out;
         }
 
-        do {
-                ptr = fgets(buf, sizeof(buf), runner_chio (&runner, STDOUT_FILENO));
-                if (ptr) {
-                        ret = dict_get_int32 (rsp_dict, "output_count", &output_count);
-                        if (ret)
-                                output_count = 1;
-                        else
-                                output_count++;
-                        memset (output_name, '\0', sizeof (output_name));
-                        snprintf (output_name, sizeof (output_name),
-                                  "output_%d", output_count);
-                        if (buf[strlen(buf) - 1] == '\n')
-                                buf[strlen(buf) - 1] = '\0';
-                        bufp = gf_strdup (buf);
-                        if (!bufp)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_STRDUP_FAILED,
-                                        "gf_strdup failed.");
-                        ret = dict_set_dynstr (rsp_dict, output_name, bufp);
-                        if (ret) {
-                                GF_FREE (bufp);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "output set "
-                                        "failed.");
-                        }
-                        ret = dict_set_int32 (rsp_dict, "output_count", output_count);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "output_count "
-                                        "set failed.");
-                }
-        } while (ptr);
-
-        ret = runner_end (&runner);
+        ret = dict_set_int32(dict, "file_mode", (int32_t)stbuf.st_mode);
         if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to "
-                          "end. Error : %s",
-                          strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_UNABLE_TO_END, "%s",
-                        errmsg);
-                ret = -1;
-                synclock_lock (&priv->big_lock);
-                goto out;
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " file mode in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+                   errmsg);
+            goto out;
         }
-        synclock_lock (&priv->big_lock);
 
-        ret = 0;
-out:
-        if (cmd_args) {
-                GF_FREE (cmd_args);
-                cmd_args = NULL;
+        ret = dict_set_bin(dict, "common_pem_contents", contents,
+                           stbuf.st_size);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " pem contents in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+        free_contents = _gf_false;
+    } else {
+        free_contents = _gf_false;
+        ret = dict_get_bin(dict, "common_pem_contents", (void **)&contents);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to get"
+                     " pem contents in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+        ret = dict_get_int32(dict, "contents_size", &contents_size);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " contents size in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
         }
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
-}
-
-int
-glusterd_op_copy_file (dict_t *dict, char **op_errstr)
-{
-        char             abs_filename[PATH_MAX] = "";
-        char             errmsg[PATH_MAX]       = "";
-        char            *filename               = NULL;
-        char            *host_uuid              = NULL;
-        char             uuid_str [64]          = {0};
-        char            *contents               = NULL;
-        char             buf[1024]              = "";
-        int              ret                    = -1;
-        int              fd                     = -1;
-        int              bytes_writen           = 0;
-        int              bytes_read             = 0;
-        int              contents_size          = -1;
-        int              file_mode              = -1;
-        glusterd_conf_t *priv                   = NULL;
-        struct stat      stbuf                  = {0,};
-        gf_boolean_t     free_contents          = _gf_true;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
-                        "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
+        ret = dict_get_int32(dict, "file_mode", &file_mode);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to get"
+                     " file mode in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+
+        fd = open(abs_filename, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+        if (fd < 0) {
+            len = snprintf(errmsg, sizeof(errmsg), "Unable to open %s",
+                           abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "host-uuid", &host_uuid);
-        if (ret < 0)
-                goto out;
+        bytes_writen = sys_write(fd, contents, contents_size);
 
-        ret = dict_get_str (dict, "source", &filename);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch filename from dict.");
-               *op_errstr = gf_strdup ("command unsuccessful");
-               goto out;
+        if (bytes_writen != contents_size) {
+            len = snprintf(errmsg, sizeof(errmsg), "Failed to write to %s",
+                           abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
-        snprintf (abs_filename, sizeof(abs_filename),
-                  "%s/%s", priv->workdir, filename);
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = sys_lstat (abs_filename, &stbuf);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Source file"
-                                 " does not exist in %s", priv->workdir);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                                GD_MSG_FILE_OP_FAILED, "%s", errmsg);
-                        goto out;
-                }
 
-                contents = GF_CALLOC(1, stbuf.st_size+1, gf_common_mt_char);
-                if (!contents) {
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "Unable to allocate memory");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY,
-                                "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                fd = open (abs_filename, O_RDONLY);
-                if (fd < 0) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
-                                  abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FILE_OP_FAILED,
-                                "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
+        sys_fchmod(fd, file_mode);
+    }
 
-                do {
-                        ret = sys_read (fd, buf, sizeof(buf));
-                        if (ret > 0) {
-                                memcpy (contents+bytes_read, buf, ret);
-                                bytes_read += ret;
-                                memset (buf, '\0', sizeof(buf));
-                        }
-                } while (ret > 0);
-
-                if (bytes_read != stbuf.st_size) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to read all "
-                                  "the data from %s", abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_READ_ERROR,
-                                "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_int32 (dict, "contents_size", stbuf.st_size);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " contents size in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "%s", errmsg);
-                        goto out;
-                }
-
-                ret = dict_set_int32 (dict, "file_mode",
-                                      (int32_t)stbuf.st_mode);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " file mode in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "%s", errmsg);
-                        goto out;
-                }
-
-                ret = dict_set_bin (dict, "common_pem_contents",
-                                    contents, stbuf.st_size);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " pem contents in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "%s", errmsg);
-                        goto out;
-                }
-                free_contents = _gf_false;
-        } else {
-                free_contents = _gf_false;
-                ret = dict_get_bin (dict, "common_pem_contents",
-                                    (void **) &contents);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
-                                  " pem contents in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "%s", errmsg);
-                        goto out;
-                }
-                ret = dict_get_int32 (dict, "contents_size", &contents_size);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " contents size in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "%s", errmsg);
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, "file_mode", &file_mode);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
-                                  " file mode in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "%s", errmsg);
-                        goto out;
-                }
-
-                fd = open (abs_filename, O_WRONLY | O_TRUNC | O_CREAT, 0600);
-                if (fd < 0) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
-                                  abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FILE_OP_FAILED, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                bytes_writen = sys_write (fd, contents, contents_size);
-
-                if (bytes_writen != contents_size) {
-                        snprintf (errmsg, sizeof (errmsg), "Failed to write"
-                                  " to %s", abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FILE_OP_FAILED, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                sys_fchmod (fd, file_mode);
-        }
-
-        ret = 0;
+    ret = 0;
 out:
-        if (fd != -1)
-                sys_close (fd);
+    if (fd != -1)
+        sys_close(fd);
 
-        if (free_contents)
-                GF_FREE(contents);
+    if (free_contents)
+        GF_FREE(contents);
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_gsync_set(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t             ret                 = -1;
-        int32_t             type                = -1;
-        char               *host_uuid           = NULL;
-        char               *slave               = NULL;
-        char               *slave_url           = NULL;
-        char               *slave_vol           = NULL;
-        char               *slave_host          = NULL;
-        char               *volname             = NULL;
-        char               *path_list           = NULL;
-        glusterd_volinfo_t *volinfo             = NULL;
-        glusterd_conf_t    *priv                = NULL;
-        gf_boolean_t        is_force            = _gf_false;
-        char               *status_msg          = NULL;
-        gf_boolean_t        is_running          = _gf_false;
-        char               *conf_path           = NULL;
-        char                errmsg[PATH_MAX]    = "";
-        char               *key                 = NULL;
-        xlator_t           *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret < 0)
-                goto out;
+    int32_t ret = -1;
+    int32_t type = -1;
+    char *host_uuid = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *volname = NULL;
+    char *path_list = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t is_force = _gf_false;
+    char *status_msg = NULL;
+    gf_boolean_t is_running = _gf_false;
+    char *conf_path = NULL;
+    char *key = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, "host-uuid", &host_uuid);
+    if (ret < 0)
+        goto out;
+
+    if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
+        ret = glusterd_get_gsync_status(dict, op_errstr, rsp_dict);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0)
+        goto out;
+
+    key = slave;
+
+    ret = dict_get_str(dict, "slave_url", &slave_url);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave url.");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_host", &slave_host);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave hostname.");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_vol", &slave_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave volume name.");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    if (dict_get_str(dict, "master", &volname) == 0) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "Volinfo for"
+                   " %s (master) not found",
+                   volname);
+            goto out;
+        }
 
-        ret = dict_get_str (dict, "host-uuid", &host_uuid);
-        if (ret < 0)
-                goto out;
+        ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+        if (!path_list && ret == -1)
+            goto out;
+    }
 
-        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
-                ret = glusterd_get_gsync_status (dict, op_errstr, rsp_dict);
+    if (type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+        ret = glusterd_gsync_configure(volinfo, slave, path_list, dict,
+                                       rsp_dict, op_errstr);
+        if (!ret) {
+            ret = dict_set_str(rsp_dict, "conf_path", conf_path);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to store conf_file_path.");
                 goto out;
+            }
         }
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0)
-                goto out;
+    if (type == GF_GSYNC_OPTION_TYPE_DELETE) {
+        ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
+        if (ret && !is_force && path_list)
+            goto out;
 
-        key = slave;
+        ret = glusterd_gsync_delete(volinfo, slave, slave_host, slave_vol,
+                                    path_list, dict, rsp_dict, op_errstr);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "slave_url", &slave_url);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch slave url.");
-                goto out;
-        }
+    if (!volinfo) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "slave_host", &slave_host);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch slave hostname.");
-                goto out;
-        }
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
 
-        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+    if (type == GF_GSYNC_OPTION_TYPE_START) {
+        /* Add slave to the dict indicating geo-rep session is running*/
+        ret = dict_set_dynstr_with_alloc(volinfo->gsync_active_slaves, key,
+                                         "running");
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch slave volume name.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set key:%s"
+                   " value:running in the dict",
+                   key);
+            goto out;
         }
 
-        ret = dict_get_str (dict, "conf_path", &conf_path);
+        /* If slave volume uuid is not present in gsync_slaves
+         * update it*/
+        ret = glusterd_update_slave_voluuid_slaveinfo(volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch conf file path.");
-                goto out;
-        }
-
-        if (dict_get_str (dict, "master", &volname) == 0) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_DICT_GET_FAILED, "Volinfo for"
-                                " %s (master) not found", volname);
-                        goto out;
-                }
-
-                ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+                   "Error in updating"
+                   " slave volume uuid for old slave info");
+            goto out;
         }
 
-        if (type == GF_GSYNC_OPTION_TYPE_CONFIG) {
-                ret = glusterd_gsync_configure (volinfo, slave, path_list,
-                                                dict, rsp_dict, op_errstr);
-                if (!ret) {
-                        ret = dict_set_str (rsp_dict, "conf_path", conf_path);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to store conf_file_path.");
-                                goto out;
-                        }
-                }
-                goto out;
-        }
-
-        if (type == GF_GSYNC_OPTION_TYPE_DELETE) {
-                ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
-                if (ret && !is_force && path_list)
-                        goto out;
-
-                ret = glusterd_gsync_delete (volinfo, slave, slave_host,
-                                             slave_vol, path_list, dict,
-                                             rsp_dict, op_errstr);
-                goto out;
-        }
+        ret = glusterd_start_gsync(volinfo, slave, path_list, conf_path,
+                                   host_uuid, op_errstr, _gf_false);
 
-        if (!volinfo) {
-                ret = -1;
+        /* Delete added slave in the dict if start fails*/
+        if (ret)
+            dict_del(volinfo->gsync_active_slaves, key);
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_STOP ||
+        type == GF_GSYNC_OPTION_TYPE_PAUSE ||
+        type == GF_GSYNC_OPTION_TYPE_RESUME) {
+        ret = glusterd_check_gsync_running_local(volinfo->volname, slave,
+                                                 conf_path, &is_running);
+        if (!ret && !is_force && path_list && (_gf_true != is_running)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GSYNCD_OP_SET_FAILED,
+                   GEOREP
+                   " is not "
+                   "set up for %s(master) and %s(slave)",
+                   volname, slave);
+            *op_errstr = gf_strdup(GEOREP " is not set up");
+            goto out;
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_PAUSE) {
+            ret = gd_pause_or_resume_gsync(dict, volname, slave, slave_host,
+                                           slave_vol, conf_path, op_errstr,
+                                           _gf_true);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PAUSE_FAILED,
+                       GEOREP " Pause Failed");
+            else
+                dict_del(volinfo->gsync_active_slaves, key);
+
+        } else if (type == GF_GSYNC_OPTION_TYPE_RESUME) {
+            /* Add slave to the dict indicating geo-rep session is
+             * running*/
+            ret = dict_set_dynstr_with_alloc(volinfo->gsync_active_slaves, key,
+                                             "running");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set "
+                       "key:%s value:running in dict",
+                       key);
                 goto out;
-        }
-
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        if (type == GF_GSYNC_OPTION_TYPE_START) {
-                /* Add slave to the dict indicating geo-rep session is running*/
-                ret = dict_set_dynstr_with_alloc (volinfo->gsync_active_slaves,
-                                                  key, "running");
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Unable to set key:%s"
-                                " value:running in the dict", key);
-                        goto out;
-                }
+            }
 
-                /* If slave volume uuid is not present in gsync_slaves
-                 * update it*/
-                ret = glusterd_update_slave_voluuid_slaveinfo (volinfo);
+            ret = gd_pause_or_resume_gsync(dict, volname, slave, slave_host,
+                                           slave_vol, conf_path, op_errstr,
+                                           _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESUME_FAILED,
+                       GEOREP " Resume Failed");
+                dict_del(volinfo->gsync_active_slaves, key);
+            }
+        } else {
+            ret = stop_gsync(volname, slave, &status_msg, conf_path, op_errstr,
+                             is_force);
+
+            if (ret == 0 && status_msg)
+                ret = dict_set_str(rsp_dict, "gsync-status", status_msg);
+            if (!ret) {
+                ret = glusterd_create_status_file(
+                    volinfo->volname, slave, slave_host, slave_vol, "Stopped");
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_REMOTE_VOL_UUID_FAIL, "Error in updating"
-                                " slave volume uuid for old slave info");
-                        goto out;
-                }
-
-                ret = glusterd_start_gsync (volinfo, slave, path_list,
-                                            conf_path, host_uuid, op_errstr,
-                                            _gf_false);
-
-                /* Delete added slave in the dict if start fails*/
-                if (ret)
-                        dict_del (volinfo->gsync_active_slaves, key);
-       }
-
-        if (type == GF_GSYNC_OPTION_TYPE_STOP ||
-            type == GF_GSYNC_OPTION_TYPE_PAUSE ||
-            type == GF_GSYNC_OPTION_TYPE_RESUME) {
-                ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                          slave, conf_path,
-                                                          &is_running);
-                if (!ret && !is_force && path_list &&
-                    (_gf_true != is_running)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_GSYNCD_OP_SET_FAILED, GEOREP" is not "
-                                "set up for %s(master) and %s(slave)",
-                                volname, slave);
-                        *op_errstr = strdup (GEOREP" is not set up");
-                        goto out;
-                }
-
-                if (type == GF_GSYNC_OPTION_TYPE_PAUSE) {
-                        ret = gd_pause_or_resume_gsync (dict, volname, slave,
-                                                        slave_host, slave_vol,
-                                                        conf_path, op_errstr,
-                                                        _gf_true);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_PAUSE_FAILED,
-                                        GEOREP" Pause Failed");
-                        else
-                                dict_del (volinfo->gsync_active_slaves, key);
-
-                } else if (type == GF_GSYNC_OPTION_TYPE_RESUME) {
-
-                        /* Add slave to the dict indicating geo-rep session is
-                         * running*/
-                        ret = dict_set_dynstr_with_alloc (
-                                              volinfo->gsync_active_slaves,
-                                              key, "running");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "Unable to set "
-                                        "key:%s value:running in dict", key);
-                              goto out;
-                        }
-
-                        ret = gd_pause_or_resume_gsync (dict, volname, slave,
-                                                        slave_host, slave_vol,
-                                                        conf_path, op_errstr,
-                                                        _gf_false);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_RESUME_FAILED,
-                                        GEOREP" Resume Failed");
-                                dict_del (volinfo->gsync_active_slaves, key);
-                        }
-                } else {
-
-                        ret = stop_gsync (volname, slave, &status_msg,
-                                          conf_path, op_errstr, is_force);
-
-                        if (ret == 0 && status_msg)
-                                ret = dict_set_str (rsp_dict, "gsync-status",
-                                                    status_msg);
-                        if (!ret) {
-                                ret = glusterd_create_status_file (
-                                                           volinfo->volname,
-                                                           slave, slave_host,
-                                                           slave_vol,"Stopped");
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_UPDATE_STATEFILE_FAILED,
-                                                "Unable to update state_file. "
-                                                "Error : %s", strerror (errno));
-                                }
-                                dict_del (volinfo->gsync_active_slaves, key);
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_UPDATE_STATEFILE_FAILED,
+                           "Unable to update state_file. "
+                           "Error : %s",
+                           strerror(errno));
                 }
+                dict_del(volinfo->gsync_active_slaves, key);
+            }
         }
+    }
 
 out:
-        if (path_list) {
-                GF_FREE (path_list);
-                path_list = NULL;
-        }
+    if (path_list) {
+        GF_FREE(path_list);
+        path_list = NULL;
+    }
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo,
-                                     dict_t *dict, char **slave_url,
-                                     char **slave_host, char **slave_vol,
-                                     char **conf_path, char **op_errstr)
+glusterd_get_slave_details_confpath(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                    char **slave_url, char **slave_host,
+                                    char **slave_vol, char **conf_path,
+                                    char **op_errstr)
 {
-        int              ret                  = -1;
-        char               confpath[PATH_MAX] = "";
-        glusterd_conf_t *priv                 = NULL;
-        char            *slave                = NULL;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret || !slave) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch slave from dict");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_get_slave_info (slave, slave_url,
-                                       slave_host, slave_vol, op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVEINFO_FETCH_ERROR,
-                        "Unable to fetch slave details.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "slave_url", *slave_url);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Unable to store slave IP.");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "slave_host", *slave_host);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Unable to store slave hostname");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "slave_vol", *slave_vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Unable to store slave volume name.");
-                goto out;
-        }
-
-        ret = snprintf (confpath, sizeof(confpath) - 1,
-                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                        priv->workdir, volinfo->volname,
-                        *slave_host, *slave_vol);
-        confpath[ret] = '\0';
-        *conf_path = gf_strdup (confpath);
-        if (!(*conf_path)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
-                        "Unable to gf_strdup. Error: %s", strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "conf_path", *conf_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Unable to store conf_path");
-                goto out;
-        }
+    int ret = -1;
+    char confpath[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    char *slave = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret || !slave) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave from dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_get_slave_info(slave, slave_url, slave_host, slave_vol,
+                                  op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave details.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "slave_url", *slave_url);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave IP.");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "slave_host", *slave_host);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave hostname");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "slave_vol", *slave_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave volume name.");
+        goto out;
+    }
+
+    ret = snprintf(confpath, sizeof(confpath) - 1,
+                   "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", priv->workdir,
+                   volinfo->volname, *slave_host, *slave_vol);
+    confpath[ret] = '\0';
+    *conf_path = gf_strdup(confpath);
+    if (!(*conf_path)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+               "Unable to gf_strdup. Error: %s", strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "conf_path", *conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store conf_path");
+        goto out;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
-
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_slave_info (char *slave,
-                         char **slave_url, char **hostname,
-                         char **slave_vol, char **op_errstr)
+glusterd_get_slave_info(char *slave, char **slave_url, char **hostname,
+                        char **slave_vol, char **op_errstr)
 {
-        char     *tmp              = NULL;
-        char     *save_ptr         = NULL;
-        char    **linearr          = NULL;
-        int32_t   ret              = -1;
-        char      errmsg[PATH_MAX] = "";
-        xlator_t *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_urltransform_single (slave, "normalize",
-                                            &linearr);
-        if (ret == -1) {
-                ret = snprintf (errmsg, sizeof(errmsg) - 1,
-                                "Invalid Url: %s", slave);
-                errmsg[ret] = '\0';
-                *op_errstr = gf_strdup (errmsg);
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_NORMALIZE_URL_FAIL,
-                        "Failed to normalize url");
-                goto out;
-        }
-
-        tmp = strtok_r (linearr[0], "/", &save_ptr);
-        tmp = strtok_r (NULL, "/", &save_ptr);
-        slave = strtok_r (tmp, ":", &save_ptr);
-        if (slave) {
-                ret = glusterd_geo_rep_parse_slave (slave, hostname, op_errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SLAVE_URL_INVALID,
-                                "Invalid slave url: %s", *op_errstr);
-                        goto out;
-                }
-                gf_msg_debug (this->name, 0, "Hostname : %s", *hostname);
-
-                *slave_url = gf_strdup (slave);
-                if (!*slave_url) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_STRDUP_FAILED,
-                                "Failed to gf_strdup");
-                        ret = -1;
-                        goto out;
-                }
-                gf_msg_debug (this->name, 0, "Slave URL : %s", *slave_url);
-                ret = 0;
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid slave name");
-                goto out;
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char **linearr = NULL;
+    int32_t ret = -1;
+    char errmsg[PATH_MAX] = "";
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_urltransform_single(slave, "normalize", &linearr);
+    if ((ret == -1) || (linearr[0] == NULL)) {
+        ret = snprintf(errmsg, sizeof(errmsg) - 1, "Invalid Url: %s", slave);
+        errmsg[ret] = '\0';
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NORMALIZE_URL_FAIL,
+               "Failed to normalize url");
+        goto out;
+    }
+
+    tmp = strtok_r(linearr[0], "/", &save_ptr);
+    tmp = strtok_r(NULL, "/", &save_ptr);
+    slave = NULL;
+    if (tmp != NULL) {
+        slave = strtok_r(tmp, ":", &save_ptr);
+    }
+    if (slave) {
+        ret = glusterd_geo_rep_parse_slave(slave, hostname, op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+                   "Invalid slave url: %s", *op_errstr);
+            goto out;
         }
+        gf_msg_debug(this->name, 0, "Hostname : %s", *hostname);
 
-        slave = strtok_r (NULL, ":", &save_ptr);
-        if (slave) {
-                *slave_vol = gf_strdup (slave);
-                if (!*slave_vol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_STRDUP_FAILED,
-                                "Failed to gf_strdup");
-                        ret = -1;
-                        GF_FREE (*slave_url);
-                        goto out;
-                }
-                gf_msg_debug (this->name, 0, "Slave Vol : %s", *slave_vol);
-                ret = 0;
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid slave name");
-                goto out;
+        *slave_url = gf_strdup(slave);
+        if (!*slave_url) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                   "Failed to gf_strdup");
+            ret = -1;
+            goto out;
         }
+        gf_msg_debug(this->name, 0, "Slave URL : %s", *slave_url);
+        ret = 0;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid slave name");
+        goto out;
+    }
+
+    slave = strtok_r(NULL, ":", &save_ptr);
+    if (slave) {
+        *slave_vol = gf_strdup(slave);
+        if (!*slave_vol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                   "Failed to gf_strdup");
+            ret = -1;
+            GF_FREE(*slave_url);
+            goto out;
+        }
+        gf_msg_debug(this->name, 0, "Slave Vol : %s", *slave_vol);
+        ret = 0;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid slave name");
+        goto out;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (linearr)
+        glusterd_urltransform_free(linearr, 1);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static void
-runinit_gsyncd_setrx (runner_t *runner, char *conf_path)
+runinit_gsyncd_setrx(runner_t *runner, char *conf_path)
 {
-        runinit (runner);
-        runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (runner, "%s", conf_path);
-        runner_add_arg (runner, "--config-set-rx");
+    runinit(runner);
+    runner_add_args(runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(runner, "%s", conf_path);
+    runner_add_arg(runner, "--config-set-rx");
 }
 
 static int
-glusterd_check_gsync_present (int *valid_state)
+glusterd_check_gsync_present(int *valid_state)
 {
-        char                buff[PATH_MAX] = {0, };
-        runner_t            runner = {0,};
-        char               *ptr = NULL;
-        int                 ret = 0;
-        xlator_t           *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                if (errno == ENOENT) {
-                        gf_msg ("glusterd", GF_LOG_INFO, ENOENT,
-                                GD_MSG_MODULE_NOT_INSTALLED, GEOREP" module "
-                                "not installed in the system");
-                        *valid_state = 0;
-                }
-                else {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_MODULE_ERROR,
-                                GEOREP" module not working as desired");
-                        *valid_state = -1;
-                }
-                goto out;
-        }
-
-        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
-        if (ptr) {
-                if (!strstr (buff, "gsyncd")) {
-                        ret = -1;
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_MODULE_ERROR,
-                                 GEOREP" module not working as desired");
-                        *valid_state = -1;
-                        goto out;
-                }
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        if (errno == ENOENT) {
+            gf_msg("glusterd", GF_LOG_INFO, ENOENT, GD_MSG_MODULE_NOT_INSTALLED,
+                   GEOREP
+                   " module "
+                   "not installed in the system");
+            *valid_state = 0;
         } else {
-                ret = -1;
-                gf_msg ("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_ERROR,
-                        GEOREP" module not working as desired");
-                *valid_state = -1;
-                goto out;
-        }
-
-        ret = 0;
- out:
-
-        runner_end (&runner);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_ERROR,
+                   GEOREP " module not working as desired");
+            *valid_state = -1;
+        }
+        goto out;
+    }
+
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_ERROR,
+                   GEOREP " module not working as desired");
+            *valid_state = -1;
+            goto out;
+        }
+    } else {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_ERROR,
+               GEOREP " module not working as desired");
+        *valid_state = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
 
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    runner_end(&runner);
 
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-create_conf_file (glusterd_conf_t *conf, char *conf_path)
-#define RUN_GSYNCD_CMD do {                                                          \
-        ret = runner_run_reuse (&runner);                                            \
-        if (ret == -1) {                                                             \
-                runner_log (&runner, "glusterd", GF_LOG_ERROR, "command failed");    \
-                runner_end (&runner);                                                \
-                goto out;                                                            \
-        }                                                                            \
-        runner_end (&runner);                                                        \
-} while (0)
+create_conf_file(glusterd_conf_t *conf, char *conf_path)
+#define RUN_GSYNCD_CMD                                                         \
+    do {                                                                       \
+        ret = runner_run_reuse(&runner);                                       \
+        if (ret == -1) {                                                       \
+            runner_log(&runner, "glusterd", GF_LOG_ERROR, "command failed");   \
+            runner_end(&runner);                                               \
+            goto out;                                                          \
+        }                                                                      \
+        runner_end(&runner);                                                   \
+    } while (0)
 {
-        int ret = 0;
-        runner_t runner = {0,};
-        char georepdir[PATH_MAX] = {0,};
-        int valid_state = 0;
-
-        valid_state = -1;
-        ret = glusterd_check_gsync_present (&valid_state);
-        if (-1 == ret) {
-                ret = valid_state;
-                goto out;
-        }
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+    char georepdir[PATH_MAX] = {
+        0,
+    };
+    int valid_state = 0;
+
+    valid_state = -1;
+    ret = glusterd_check_gsync_present(&valid_state);
+    if (-1 == ret) {
+        ret = valid_state;
+        goto out;
+    }
+
+    ret = snprintf(georepdir, sizeof(georepdir) - 1, "%s/" GEOREP,
+                   conf->workdir);
+    georepdir[ret] = '\0';
+
+    /************
+     * master pre-configuration
+     ************/
+
+    /* remote-gsyncd */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "remote-gsyncd", GSYNCD_PREFIX "/gsyncd", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "remote-gsyncd", "/nonexistent/gsyncd", ".",
+                    "^ssh:", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-command-dir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "gluster-command-dir", SBIN_DIR "/", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-params */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ssh-command */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "ssh-command");
+    runner_argprintf(&runner,
+                     "ssh -oPasswordAuthentication=no "
+                     "-oStrictHostKeyChecking=no "
+                     "-i %s/secret.pem",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ssh-command tar */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "ssh-command-tar");
+    runner_argprintf(&runner,
+                     "ssh -oPasswordAuthentication=no "
+                     "-oStrictHostKeyChecking=no "
+                     "-i %s/tar_ssh.pem",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* pid-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "pid-file");
+    runner_argprintf(&runner,
+                     "%s/${mastervol}_${remotehost}_${slavevol}/monitor.pid",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* geo-rep-working-dir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "georep-session-working-dir");
+    runner_argprintf(&runner, "%s/${mastervol}_${remotehost}_${slavevol}/",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "state-file");
+    runner_argprintf(&runner,
+                     "%s/${mastervol}_${remotehost}_${slavevol}/monitor.status",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-detail-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "state-detail-file");
+    runner_argprintf(
+        &runner,
+        "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+        georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-socket */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "state-socket-unencoded");
+    runner_argprintf(
+        &runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.socket",
+        georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* socketdir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "log-file");
+    runner_argprintf(&runner, "%s/%s/${mastervol}/${eSlave}.log", conf->logdir,
+                     GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* changelog-log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "changelog-log-file");
+    runner_argprintf(&runner,
+                     "%s/%s/${mastervol}/${eSlave}${local_id}-changes.log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "gluster-log-file");
+    runner_argprintf(&runner,
+                     "%s/%s/${mastervol}/${eSlave}${local_id}.gluster.log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ignore-deletes */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "ignore-deletes", "false", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* special-sync-mode */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "special-sync-mode", "partial", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* change-detector == changelog */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "working-dir");
+    runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                     DEFAULT_GLUSTERFSD_MISC_DIRETORY);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /************
+     * slave pre-configuration
+     ************/
+
+    /* slave-gluster-command-dir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "slave-gluster-command-dir", SBIN_DIR "/", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-params */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "log-file");
+    runner_argprintf(&runner,
+                     "%s/%s-slaves/"
+                     "${session_owner}:${local_node}${local_id}.${slavevol}."
+                     "log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* MountBroker log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "log-file-mbr");
+    runner_argprintf(&runner,
+                     "%s/%s-slaves/mbr/"
+                     "${session_owner}:${local_node}${local_id}.${slavevol}."
+                     "log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "gluster-log-file");
+    runner_argprintf(&runner,
+                     "%s/%s-slaves/"
+                     "${session_owner}:${local_node}${local_id}.${slavevol}."
+                     "gluster.log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
 
-        ret = snprintf (georepdir, sizeof(georepdir) - 1, "%s/"GEOREP,
-                        conf->workdir);
-        georepdir[ret] = '\0';
-
-        /************
-         * master pre-configuration
-         ************/
-
-        /* remote-gsyncd */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "remote-gsyncd", GSYNCD_PREFIX"/gsyncd", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
-                         ".", "^ssh:", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-command-dir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-params */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-params",
-                         "aux-gfid-mount acl",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ssh-command */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "ssh-command");
-        runner_argprintf (&runner,
-                          "ssh -oPasswordAuthentication=no "
-                           "-oStrictHostKeyChecking=no "
-                           "-i %s/secret.pem", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ssh-command tar */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "ssh-command-tar");
-        runner_argprintf (&runner,
-                          "ssh -oPasswordAuthentication=no "
-                           "-oStrictHostKeyChecking=no "
-                           "-i %s/tar_ssh.pem", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* pid-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "pid-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/monitor.pid", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* geo-rep-working-dir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "georep-session-working-dir");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "state-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/monitor.status", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-detail-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "state-detail-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-socket */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "state-socket-unencoded");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.socket", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* socketdir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* changelog-log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "changelog-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}-changes.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ignore-deletes */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "ignore-deletes", "false", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* special-sync-mode */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* change-detector == changelog */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg(&runner, "working-dir");
-        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
-                         DEFAULT_GLUSTERFSD_MISC_DIRETORY);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /************
-         * slave pre-configuration
-         ************/
-
-        /* gluster-command-dir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-params */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-params",
-                         "aux-gfid-mount acl",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* MountBroker log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "log-file-mbr",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
- out:
-        return ret ? -1 : 0;
+out:
+    return ret ? -1 : 0;
 }
 
 static int
-glusterd_create_essential_dir_files (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                     char *slave, char *slave_host,
-                                     char *slave_vol, char **op_errstr)
+glusterd_create_essential_dir_files(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                    char *slave, char *slave_host,
+                                    char *slave_vol, char **op_errstr)
 {
-        int                ret              = -1;
-        char              *conf_path        = NULL;
-        char              *statefile        = NULL;
-        char               buf[PATH_MAX]    = "";
-        char               errmsg[PATH_MAX] = "";
-        glusterd_conf_t   *conf             = NULL;
-        struct stat        stbuf            = {0,};
-        xlator_t          *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        conf = this->private;
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch conf file path.");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "%s", errmsg);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "statefile", &statefile);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch statefile path.");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "%s", errmsg);
-                goto out;
-        }
-
-        ret = snprintf (buf, sizeof(buf) - 1, "%s/"GEOREP"/%s_%s_%s",
-                        conf->workdir, volinfo->volname, slave_host, slave_vol);
-        buf[ret] = '\0';
-        ret = mkdir_p (buf, 0777, _gf_true);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
-                          ". Error : %s", buf, strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_msg (this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
-                        "%s", errmsg);
-                goto out;
-        }
-
-        ret = snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
-                        volinfo->volname);
-        buf[ret] = '\0';
-        ret = mkdir_p (buf, 0777, _gf_true);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
-                          ". Error : %s", buf, strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_msg (this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
-                        "%s", errmsg);
-                goto out;
-        }
-
-        ret = sys_lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_msg_debug (this->name, 0, "Session already running."
-                              " Not creating config file again.");
-        } else {
-                ret = create_conf_file (conf, conf_path);
-                if (ret || sys_lstat (conf_path, &stbuf)) {
-                        snprintf (errmsg, sizeof (errmsg), "Failed to create"
-                                  " config file(%s).", conf_path);
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED, "%s", errmsg);
-                        goto out;
-                }
-        }
-
-        ret = sys_lstat (statefile, &stbuf);
-        if (!ret) {
-                gf_msg_debug (this->name, 0, "Session already running."
-                              " Not creating status file again.");
-                goto out;
-        } else {
-                ret = glusterd_create_status_file (volinfo->volname, slave,
-                                                   slave_host, slave_vol,
-                                                   "Created");
-                if (ret || sys_lstat (statefile, &stbuf)) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
-                                  ". Error : %s", statefile, strerror (errno));
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-        }
+    int ret = -1;
+    char *conf_path = NULL;
+    char *statefile = NULL;
+    char buf[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch conf file path.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "statefile", &statefile);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch statefile path.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "%s/" GEOREP "/%s_%s_%s", conf->workdir,
+                   volinfo->volname, slave_host, slave_vol);
+    if ((ret < 0) || (ret >= sizeof(buf))) {
+        ret = -1;
+        goto out;
+    }
+    ret = mkdir_p(buf, 0755, _gf_true);
+    if (ret) {
+        len = snprintf(errmsg, sizeof(errmsg),
+                       "Unable to create %s"
+                       ". Error : %s",
+                       buf, strerror(errno));
+        if (len < 0) {
+            strcpy(errmsg, "<error>");
+        }
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = snprintf(buf, PATH_MAX, "%s/" GEOREP "/%s", conf->logdir,
+                   volinfo->volname);
+    if ((ret < 0) || (ret >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+    ret = mkdir_p(buf, 0755, _gf_true);
+    if (ret) {
+        len = snprintf(errmsg, sizeof(errmsg),
+                       "Unable to create %s"
+                       ". Error : %s",
+                       buf, strerror(errno));
+        if (len < 0) {
+            strcpy(errmsg, "<error>");
+        }
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg_debug(this->name, 0,
+                     "Session already running."
+                     " Not creating config file again.");
+    } else {
+        ret = create_conf_file(conf, conf_path);
+        if (ret || sys_lstat(conf_path, &stbuf)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Failed to create"
+                     " config file(%s).",
+                     conf_path);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+    }
+
+    ret = sys_lstat(statefile, &stbuf);
+    if (!ret) {
+        gf_msg_debug(this->name, 0,
+                     "Session already running."
+                     " Not creating status file again.");
+        goto out;
+    } else {
+        ret = glusterd_create_status_file(volinfo->volname, slave, slave_host,
+                                          slave_vol, "Created");
+        if (ret || sys_lstat(statefile, &stbuf)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to create %s"
+                     ". Error : %s",
+                     statefile, strerror(errno));
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
+        }
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_gsync_create(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char                common_pem_file[PATH_MAX] = "";
-        char                errmsg[PATH_MAX]  = {0,};
-        char                hooks_args[PATH_MAX]      = "";
-        char                uuid_str [64]             = "";
-        char               *host_uuid                 = NULL;
-        char               *slave_url                 = NULL;
-        char               *slave_url_buf             = NULL;
-        char               *slave_user                = NULL;
-        char               *slave_ip                  = NULL;
-        char               *save_ptr                  = NULL;
-        char               *slave_host                = NULL;
-        char               *slave_vol                 = NULL;
-        char               *arg_buf                   = NULL;
-        char               *volname                   = NULL;
-        char               *slave                     = NULL;
-        int32_t             ret                       = -1;
-        int32_t             is_pem_push               = -1;
-        int32_t             ssh_port                  = 22;
-        gf_boolean_t        is_force                  = -1;
-        glusterd_conf_t    *conf                      = NULL;
-        glusterd_volinfo_t *volinfo                   = NULL;
-        xlator_t           *this                      = NULL;
-        char                old_working_dir[PATH_MAX] = {0};
-        char                new_working_dir[PATH_MAX] = {0};
-        char               *slave_info                = NULL;
-        char                slave_url_info[SLAVE_URL_INFO_MAX] = {0};
-        char               *slave_voluuid             = NULL;
-        char               *old_slavehost             = NULL;
-        gf_boolean_t        is_existing_session       = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        ret = glusterd_op_gsync_args_get (dict, op_errstr,
-                                          &volname, &slave, &host_uuid);
-        if (ret)
-                goto out;
-
-        snprintf (common_pem_file, sizeof(common_pem_file),
-                  "%s"GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
-                        "Volinfo for %s (master) not found", volname);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "slave_vol", &slave_vol);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch slave volume name.");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "%s", errmsg);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "slave_url", &slave_url);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch slave IP.");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "%s", errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        /* Fetch the slave_user and slave_ip from the slave_url.
-         * If the slave_user is not present. Use "root"
-         */
-        if (strstr(slave_url, "@")) {
-                slave_url_buf = gf_strdup (slave_url);
-                if (!slave_url_buf) {
-                        ret = -1;
-                        goto out;
-                }
-                slave_user = strtok_r (slave_url, "@", &save_ptr);
-                slave_ip = strtok_r (NULL, "@", &save_ptr);
-        } else {
-                slave_user = "root";
-                slave_ip = slave_url;
-        }
-
-        if (!slave_user || !slave_ip) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
-                        "Invalid slave url.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "slave_host", &slave_host);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch slave host");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "%s", errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "ssh_port", &ssh_port);
-        if (ret < 0 && ret != -ENOENT) {
-                snprintf (errmsg, sizeof (errmsg), "Fetching ssh_port failed");
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "%s", errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
-                if (!ret && is_pem_push) {
-                        gf_msg_debug (this->name, 0, "Trying to setup"
-                                      " pem files in slave");
-                        is_pem_push = 1;
-                } else
-                        is_pem_push = 0;
-
-                snprintf(hooks_args, sizeof(hooks_args),
-                         "is_push_pem=%d,pub_file=%s,slave_user=%s,slave_ip=%s,"
-                         "slave_vol=%s,ssh_port=%d", is_pem_push,
-                         common_pem_file, slave_user, slave_ip, slave_vol,
-                         ssh_port);
+    char common_pem_file[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    char hooks_args[PATH_MAX] = "";
+    char uuid_str[64] = "";
+    char *host_uuid = NULL;
+    char *slave_url = NULL;
+    char *slave_url_buf = NULL;
+    char *slave_user = NULL;
+    char *slave_ip = NULL;
+    char *save_ptr = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    char *arg_buf = NULL;
+    char *volname = NULL;
+    char *slave = NULL;
+    int32_t ret = -1;
+    int32_t is_pem_push = -1;
+    int32_t ssh_port = 22;
+    gf_boolean_t is_force = -1;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char old_working_dir[PATH_MAX] = {0};
+    char new_working_dir[PATH_MAX] = {0};
+    char *slave_voluuid = NULL;
+    char *old_slavehost = NULL;
+    gf_boolean_t is_existing_session = _gf_false;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, &slave,
+                                     &host_uuid);
+    if (ret)
+        goto out;
+
+    len = snprintf(common_pem_file, sizeof(common_pem_file),
+                   "%s" GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+    if ((len < 0) || (len >= sizeof(common_pem_file))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volinfo for %s (master) not found", volname);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_vol", &slave_vol);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch slave volume name.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_url", &slave_url);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch slave IP.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    /* Fetch the slave_user and slave_ip from the slave_url.
+     * If the slave_user is not present. Use "root"
+     */
+    if (strstr(slave_url, "@")) {
+        slave_url_buf = gf_strdup(slave_url);
+        if (!slave_url_buf) {
+            ret = -1;
+            goto out;
+        }
+        slave_user = strtok_r(slave_url, "@", &save_ptr);
+        slave_ip = strtok_r(NULL, "@", &save_ptr);
+    } else {
+        slave_user = "root";
+        slave_ip = slave_url;
+    }
+
+    if (!slave_user || !slave_ip) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+               "Invalid slave url.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_host", &slave_host);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch slave host");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "ssh_port", &ssh_port);
+    if (ret < 0 && ret != -ENOENT) {
+        snprintf(errmsg, sizeof(errmsg), "Fetching ssh_port failed");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = dict_get_int32(dict, "push_pem", &is_pem_push);
+        if (!ret && is_pem_push) {
+            gf_msg_debug(this->name, 0,
+                         "Trying to setup"
+                         " pem files in slave");
+            is_pem_push = 1;
         } else
-                snprintf(hooks_args, sizeof(hooks_args),
-                         "This argument will stop the hooks script");
-
-        arg_buf = gf_strdup (hooks_args);
-        if (!arg_buf) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
-                        "Failed to gf_strdup");
-                if (is_force) {
-                        ret = 0;
-                        goto create_essentials;
-                }
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "hooks_args", arg_buf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "Failed to set hooks_args in dict.");
-                if (is_force) {
-                        ret = 0;
-                        goto create_essentials;
-                }
-                goto out;
-        }
+            is_pem_push = 0;
+
+        len = snprintf(hooks_args, sizeof(hooks_args),
+                       "is_push_pem=%d,pub_file=%s,slave_user=%s,"
+                       "slave_ip=%s,slave_vol=%s,ssh_port=%d",
+                       is_pem_push, common_pem_file, slave_user, slave_ip,
+                       slave_vol, ssh_port);
+        if ((len < 0) || (len >= sizeof(hooks_args))) {
+            ret = -1;
+            goto out;
+        }
+    } else
+        snprintf(hooks_args, sizeof(hooks_args),
+                 "This argument will stop the hooks script");
+
+    arg_buf = gf_strdup(hooks_args);
+    if (!arg_buf) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+               "Failed to gf_strdup");
+        if (is_force) {
+            ret = 0;
+            goto create_essentials;
+        }
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "hooks_args", arg_buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set hooks_args in dict.");
+        if (is_force) {
+            ret = 0;
+            goto create_essentials;
+        }
+        goto out;
+    }
 
 create_essentials:
-        /* Fetch slave volume uuid, to get stored in volume info. */
-        ret = dict_get_str (dict, "slave_voluuid", &slave_voluuid);
+    /* Fetch slave volume uuid, to get stored in volume info. */
+    ret = dict_get_str(dict, "slave_voluuid", &slave_voluuid);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to fetch slave volume uuid from dict");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    is_existing_session = dict_get_str_boolean(dict, "existing_session",
+                                               _gf_false);
+    if (is_existing_session) {
+        ret = dict_get_str(dict, "old_slavehost", &old_slavehost);
         if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch slave volume uuid from dict");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                         GD_MSG_DICT_GET_FAILED,
-                         "%s", errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        is_existing_session = dict_get_str_boolean (dict, "existing_session",
-                                                    _gf_false);
-        if (is_existing_session) {
-                ret = dict_get_str (dict, "old_slavehost", &old_slavehost);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "Unable to fetch old_slavehost");
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                 GD_MSG_DICT_GET_FAILED,
-                                 "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                /* Rename existing geo-rep session with new Slave Host */
-                ret = snprintf (old_working_dir,
-                         sizeof (old_working_dir) - 1,
-                         "%s/"GEOREP"/%s_%s_%s", conf->workdir,
-                         volinfo->volname, old_slavehost,
-                         slave_vol);
-
-                ret = snprintf (new_working_dir,
-                         sizeof (new_working_dir) - 1,
-                         "%s/"GEOREP"/%s_%s_%s", conf->workdir,
-                         volinfo->volname, slave_host, slave_vol);
-
-                ret = sys_rename (old_working_dir, new_working_dir);
-                if (!ret) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_FORCE_CREATE_SESSION,
-                                "rename of old working dir %s to "
-                                "new working dir %s is done! ",
-                                 old_working_dir, new_working_dir);
-                } else {
-                        if (errno == ENOENT) {
-                                /* log error, but proceed with directory
-                                 * creation below */
-                                gf_msg_debug (this->name, 0,
-                                              "old_working_dir(%s) "
-                                               "not present.",
-                                               old_working_dir);
-                        } else {
-                                snprintf (errmsg, sizeof (errmsg),
-                                          "rename of old working dir %s to "
-                                          "new working dir %s failed! Error: %s",
-                                          old_working_dir, new_working_dir,
-                                          strerror (errno));
-                                gf_msg (this->name,  GF_LOG_INFO, 0,
-                                        GD_MSG_FORCE_CREATE_SESSION,
-                                        "rename of old working dir %s to "
-                                        "new working dir %s failed! Error: %s!",
-                                        old_working_dir, new_working_dir,
-                                        strerror (errno));
-
-                                ret = -1;
-                                goto out;
-                        }
-                }
+            snprintf(errmsg, sizeof(errmsg), "Unable to fetch old_slavehost");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        ret = glusterd_create_essential_dir_files (volinfo, dict, slave,
-                                                   slave_host, slave_vol,
-                                                   op_errstr);
-        if (ret)
-                goto out;
+        /* Rename existing geo-rep session with new Slave Host */
+        ret = snprintf(old_working_dir, sizeof(old_working_dir) - 1,
+                       "%s/" GEOREP "/%s_%s_%s", conf->workdir,
+                       volinfo->volname, old_slavehost, slave_vol);
 
-        ret = glusterd_store_slave_in_info (volinfo, slave,
-                                            host_uuid, slave_voluuid,
-                                            op_errstr, is_force);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to store"
-                          " slave info.");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SLAVEINFO_STORE_ERROR,
-                        "%s", errmsg);
-                goto out;
-        }
+        ret = snprintf(new_working_dir, sizeof(new_working_dir) - 1,
+                       "%s/" GEOREP "/%s_%s_%s", conf->workdir,
+                       volinfo->volname, slave_host, slave_vol);
 
-        /* Enable marker and changelog */
-        ret = glusterd_set_gsync_confs (volinfo);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_MARKER_START_FAIL, "marker/changelog"
-                        " start failed");
-                snprintf (errmsg, sizeof (errmsg),
-                          "Index initialization failed");
+        ret = sys_rename(old_working_dir, new_working_dir);
+        if (!ret) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+                   "rename of old working dir %s to "
+                   "new working dir %s is done! ",
+                   old_working_dir, new_working_dir);
+        } else {
+            if (errno == ENOENT) {
+                /* log error, but proceed with directory
+                 * creation below */
+                gf_msg_debug(this->name, 0,
+                             "old_working_dir(%s) "
+                             "not present.",
+                             old_working_dir);
+            } else {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "rename of old working dir %s "
+                               "to new working dir %s "
+                               "failed! Error: %s",
+                               old_working_dir, new_working_dir,
+                               strerror(errno));
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
+                }
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+                       "rename of old working dir %s to "
+                       "new working dir %s failed! Error: %s!",
+                       old_working_dir, new_working_dir, strerror(errno));
 
                 ret = -1;
                 goto out;
+            }
         }
+    }
+
+    ret = glusterd_create_essential_dir_files(volinfo, dict, slave, slave_host,
+                                              slave_vol, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_slave_in_info(volinfo, slave, host_uuid, slave_voluuid,
+                                       op_errstr, is_force);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to store"
+                 " slave info.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_STORE_ERROR, "%s",
+               errmsg);
+        goto out;
+    }
+
+    /* Enable marker and changelog */
+    ret = glusterd_set_gsync_confs(volinfo);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_MARKER_START_FAIL,
+               "marker/changelog"
+               " start failed");
+        snprintf(errmsg, sizeof(errmsg), "Index initialization failed");
+
+        ret = -1;
+        goto out;
+    }
 
 out:
-        if (ret && errmsg[0] != '\0') {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
-                        "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-        }
-
-        GF_FREE (slave_url_buf);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && errmsg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR, "%s", errmsg);
+        *op_errstr = gf_strdup(errmsg);
+    }
+
+    GF_FREE(slave_url_buf);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.h b/xlators/mgmt/glusterd/src/glusterd-geo-rep.h
index 0524ec48fca..7d1318f522c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.h
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.h
@@ -11,7 +11,7 @@
 #define _GLUSTERD_GEO_REP_H_
 
 #ifndef GSYNC_CONF_TEMPLATE
-#define GSYNC_CONF_TEMPLATE GEOREP"/gsyncd_template.conf"
+#define GSYNC_CONF_TEMPLATE GEOREP "/gsyncd_template.conf"
 #endif
 
 /* <slave host>::<slave volume> */
@@ -20,30 +20,33 @@
 /* slave info format:
  * <master host uuid>:ssh://{<slave_user>@}<slave host>::<slave volume> \
  * :<slave volume uuid> */
-#define VOLINFO_SLAVE_URL_MAX (_POSIX_LOGIN_NAME_MAX + (2*GF_UUID_BUF_SIZE) \
-                                 + SLAVE_URL_INFO_MAX + 10)
+#define VOLINFO_SLAVE_URL_MAX                                                  \
+    (LOGIN_NAME_MAX + (2 * GF_UUID_BUF_SIZE) + SLAVE_URL_INFO_MAX + 10)
 
 typedef struct glusterd_gsync_status_temp {
-        dict_t *rsp_dict;
-        glusterd_volinfo_t *volinfo;
-        char *node;
+    dict_t *rsp_dict;
+    glusterd_volinfo_t *volinfo;
+    char *node;
 } glusterd_gsync_status_temp_t;
 
 typedef struct gsync_status_param {
-        int is_active;
-        glusterd_volinfo_t *volinfo;
+    glusterd_volinfo_t *volinfo;
+    int is_active;
 } gsync_status_param_t;
 
 int
-gsync_status (char *master, char *slave, char *conf_path,
-              int *status, gf_boolean_t *is_template_in_use);
+gsync_status(char *master, char *slave, char *conf_path, int *status,
+             gf_boolean_t *is_template_in_use);
 
 void
-glusterd_check_geo_rep_configured (glusterd_volinfo_t *volinfo,
-                                   gf_boolean_t *flag);
+glusterd_check_geo_rep_configured(glusterd_volinfo_t *volinfo,
+                                  gf_boolean_t *flag);
 int
-_get_slave_status (dict_t *dict, char *key, data_t *value, void *data);
+_get_slave_status(dict_t *dict, char *key, data_t *value, void *data);
 int
-glusterd_check_geo_rep_running (gsync_status_param_t *param, char **op_errstr);
-#endif
+glusterd_check_geo_rep_running(gsync_status_param_t *param, char **op_errstr);
 
+int
+glusterd_get_gsync_status_mst(glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
+                              char *node);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.c
new file mode 100644
index 00000000000..319bfa140f3
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.c
@@ -0,0 +1,235 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+#include "glusterd-messages.h"
+#include <glusterfs/syscall.h>
+#include "glusterd-volgen.h"
+
+void
+glusterd_svc_build_gfproxyd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                   int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    GLUSTERD_GET_VOLUME_PID_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s", workdir);
+}
+
+void
+glusterd_svc_build_gfproxyd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                            char *path, int path_len)
+{
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    glusterd_svc_build_gfproxyd_rundir(volinfo, rundir, sizeof(rundir));
+    len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+                   uuid_utoa(MY_UUID));
+    if ((len < 0) || (len >= sizeof(sockfilepath))) {
+        sockfilepath[0] = 0;
+    }
+
+    glusterd_set_socket_filepath(sockfilepath, path, path_len);
+}
+
+void
+glusterd_svc_build_gfproxyd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len)
+{
+    char rundir[PATH_MAX] = {
+        0,
+    };
+
+    glusterd_svc_build_gfproxyd_rundir(volinfo, rundir, sizeof(rundir));
+
+    snprintf(path, path_len, "%s/%s.gfproxyd.pid", rundir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_gfproxyd_volfile_path(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/%s.gfproxyd.vol", workdir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_gfproxyd_logdir(char *logdir, char *volname, size_t len)
+{
+    glusterd_conf_t *conf = THIS->private;
+    snprintf(logdir, len, "%s/gfproxy/%s", conf->logdir, volname);
+}
+
+void
+glusterd_svc_build_gfproxyd_logfile(char *logfile, char *logdir, size_t len)
+{
+    snprintf(logfile, len, "%s/gfproxyd.log", logdir);
+}
+
+int
+glusterd_is_gfproxyd_enabled(glusterd_volinfo_t *volinfo)
+{
+    return glusterd_volinfo_get_boolean(volinfo, VKEY_CONFIG_GFPROXY);
+}
+
+static int
+glusterd_svc_get_gfproxyd_volfile(glusterd_volinfo_t *volinfo, char *svc_name,
+                                  char *orgvol, char **tmpvol, int path_len)
+{
+    int tmp_fd = -1;
+    int ret = -1;
+    int need_unlink = 0;
+
+    glusterd_svc_build_gfproxyd_volfile_path(volinfo, orgvol, path_len);
+
+    ret = gf_asprintf(tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(*tmpvol);
+    if (tmp_fd < 0) {
+        gf_msg("glusterd", GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               *tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    need_unlink = 1;
+    ret = glusterd_build_gfproxyd_volfile(volinfo, *tmpvol);
+out:
+    if (need_unlink && ret < 0)
+        sys_unlink(*tmpvol);
+
+    if ((ret < 0) && (*tmpvol != NULL)) {
+        GF_FREE(*tmpvol);
+        *tmpvol = NULL;
+    }
+
+    if (tmp_fd >= 0)
+        sys_close(tmp_fd);
+
+    return ret;
+}
+
+int
+glusterd_svc_check_gfproxyd_volfile_identical(char *svc_name,
+                                              glusterd_volinfo_t *volinfo,
+                                              gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    int ret = -1;
+    int need_unlink = 0;
+
+    GF_VALIDATE_OR_GOTO("glusterd", identical, out);
+
+    ret = glusterd_svc_get_gfproxyd_volfile(volinfo, svc_name, orgvol, &tmpvol,
+                                            PATH_MAX);
+    if (ret)
+        goto out;
+
+    need_unlink = 1;
+    ret = glusterd_check_files_identical(orgvol, tmpvol, identical);
+    if (ret)
+        goto out;
+
+out:
+    if (need_unlink)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    return ret;
+}
+
+int
+glusterd_svc_check_gfproxyd_topology_identical(char *svc_name,
+                                               glusterd_volinfo_t *volinfo,
+                                               gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    int ret = -1;
+    int tmpclean = 0;
+
+    GF_VALIDATE_OR_GOTO("glusterd", identical, out);
+
+    ret = glusterd_svc_get_gfproxyd_volfile(volinfo, svc_name, orgvol, &tmpvol,
+                                            PATH_MAX);
+    if (ret)
+        goto out;
+
+    tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+    /* Compare the topology of volfiles */
+    ret = glusterd_check_topology_identical(orgvol, tmpvol, identical);
+out:
+    if (tmpclean)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    return ret;
+}
+
+glusterd_volinfo_t *
+glusterd_gfproxyd_volinfo_from_svc(glusterd_svc_t *svc)
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_gfproxydsvc_t *gfproxyd = NULL;
+
+    /* Get volinfo->gfproxyd from svc object */
+    gfproxyd = cds_list_entry(svc, glusterd_gfproxydsvc_t, svc);
+    if (!gfproxyd) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
+               "Failed to get gfproxyd "
+               "object from gfproxyd service");
+        goto out;
+    }
+
+    /* Get volinfo from gfproxyd */
+    volinfo = cds_list_entry(gfproxyd, glusterd_volinfo_t, gfproxyd);
+    if (!volinfo) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from gfproxyd");
+        goto out;
+    }
+out:
+    return volinfo;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.h
new file mode 100644
index 00000000000..3aca218a65d
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.h
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERD_GFPROXYD_SVC_HELPER_H_
+#define _GLUSTERD_GFPROXYD_SVC_HELPER_H_
+
+#include "glusterd.h"
+
+void
+glusterd_svc_build_gfproxyd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                   int path_len);
+
+void
+glusterd_svc_build_gfproxyd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                            char *path, int path_len);
+
+void
+glusterd_svc_build_gfproxyd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len);
+
+void
+glusterd_svc_build_gfproxyd_volfile_path(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len);
+
+void
+glusterd_svc_build_gfproxyd_logdir(char *logdir, char *volname, size_t len);
+
+void
+glusterd_svc_build_gfproxyd_logfile(char *logfile, char *logdir, size_t len);
+
+int
+glusterd_svc_check_gfproxyd_volfile_identical(char *svc_name,
+                                              glusterd_volinfo_t *volinfo,
+                                              gf_boolean_t *identical);
+int
+glusterd_svc_check_gfproxyd_topology_identical(char *svc_name,
+                                               glusterd_volinfo_t *volinfo,
+                                               gf_boolean_t *identical);
+int
+glusterd_is_gfproxyd_enabled(glusterd_volinfo_t *volinfo);
+
+glusterd_volinfo_t *
+glusterd_gfproxyd_volinfo_from_svc(glusterd_svc_t *svc);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c
new file mode 100644
index 00000000000..a0bfea41f0f
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c
@@ -0,0 +1,478 @@
+/*
+   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-gfproxyd-svc.h"
+#include "glusterd-messages.h"
+#include "glusterd-svc-helper.h"
+#include "glusterd-svc-mgmt.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+#include <glusterfs/syscall.h>
+
+void
+glusterd_gfproxydsvc_build(glusterd_svc_t *svc)
+{
+    svc->manager = glusterd_gfproxydsvc_manager;
+    svc->start = glusterd_gfproxydsvc_start;
+    svc->stop = glusterd_gfproxydsvc_stop;
+    svc->reconfigure = glusterd_gfproxydsvc_reconfigure;
+}
+
+int
+glusterd_gfproxydsvc_stop(glusterd_svc_t *svc, int sig)
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+
+    ret = glusterd_svc_stop(svc, sig);
+    if (ret)
+        goto out;
+
+    volinfo = glusterd_gfproxyd_volinfo_from_svc(svc);
+    volinfo->gfproxyd.port = 0;
+
+out:
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_init(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    char sockpath[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char logdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    glusterd_svc_t *svc = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_conn_notify_t notify = NULL;
+    xlator_t *this = NULL;
+    char *volfileserver = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    svc = &(volinfo->gfproxyd.svc);
+
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", gfproxyd_svc_name);
+    if (ret < 0)
+        goto out;
+
+    notify = glusterd_svc_common_rpc_notify;
+
+    glusterd_svc_build_gfproxyd_rundir(volinfo, rundir, sizeof(rundir));
+    glusterd_svc_create_rundir(rundir);
+
+    /* Initialize the connection mgmt */
+    glusterd_svc_build_gfproxyd_socket_filepath(volinfo, sockpath,
+                                                sizeof(sockpath));
+    ret = glusterd_conn_init(&(svc->conn), sockpath, 600, notify);
+    if (ret)
+        goto out;
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_gfproxyd_pidfile(volinfo, pidfile, sizeof(pidfile));
+    glusterd_svc_build_gfproxyd_volfile_path(volinfo, volfile, sizeof(volfile));
+    glusterd_svc_build_gfproxyd_logdir(logdir, volinfo->volname,
+                                       sizeof(logdir));
+    ret = mkdir_p(logdir, 0755, _gf_true);
+    if ((ret == -1) && (EEXIST != errno)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create logdir %s", logdir);
+        goto out;
+    }
+    glusterd_svc_build_gfproxyd_logfile(logfile, logdir, sizeof(logfile));
+    len = snprintf(volfileid, sizeof(volfileid), "gfproxyd/%s",
+                   volinfo->volname);
+    if ((len < 0) || (len >= sizeof(volfileid))) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+    ret = glusterd_proc_init(&(svc->proc), gfproxyd_svc_name, pidfile, logdir,
+                             logfile, volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
+
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
+}
+
+static int
+glusterd_gfproxydsvc_create_volfile(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    ret = glusterd_generate_gfproxyd_volfile(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
+
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_manager(glusterd_svc_t *svc, void *data, int flags)
+{
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    volinfo = data;
+    GF_VALIDATE_OR_GOTO(this->name, data, out);
+
+    if (!svc->inited) {
+        ret = glusterd_gfproxydsvc_init(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_QUOTASVC,
+                   "Failed to init "
+                   "gfproxyd service");
+            goto out;
+        } else {
+            svc->inited = _gf_true;
+            gf_msg_debug(this->name, 0,
+                         "gfproxyd service "
+                         "initialized");
+        }
+    }
+
+    ret = glusterd_is_gfproxyd_enabled(volinfo);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to read volume "
+               "options");
+        goto out;
+    }
+
+    if (ret) {
+        if (!glusterd_is_volume_started(volinfo)) {
+            if (glusterd_proc_is_running(&svc->proc)) {
+                ret = svc->stop(svc, SIGTERM);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                           "Couldn't stop gfproxyd for "
+                           "volume: %s",
+                           volinfo->volname);
+            } else {
+                /* Since gfproxyd is not running set ret to 0 */
+                ret = 0;
+            }
+            goto out;
+        }
+
+        ret = glusterd_gfproxydsvc_create_volfile(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_CREATE_FAIL,
+                   "Couldn't create "
+                   "gfroxyd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+        ret = svc->stop(svc, SIGTERM);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                   "Couldn't stop "
+                   "gfproxyd for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+
+        ret = svc->start(svc, flags);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                   "Couldn't start "
+                   "gfproxyd for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+
+        glusterd_volinfo_ref(volinfo);
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret) {
+            glusterd_volinfo_unref(volinfo);
+            volinfo = NULL;
+            goto out;
+        }
+
+    } else if (glusterd_proc_is_running(&svc->proc)) {
+        ret = svc->stop(svc, SIGTERM);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                   "Couldn't stop gfproxyd for volume: %s", volinfo->volname);
+            goto out;
+        }
+    }
+
+out:
+    if (ret) {
+        if (volinfo) {
+            gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                     volinfo->volname, svc->name);
+        }
+    }
+
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_start(glusterd_svc_t *svc, int flags)
+{
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char valgrind_logfile[PATH_MAX] = {0};
+    int gfproxyd_port = 0;
+    char msg[1024] = {
+        0,
+    };
+    char gfproxyd_id[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    char *localtime_logging = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    volinfo = glusterd_gfproxyd_volinfo_from_svc(svc);
+    if (!volinfo)
+        goto out;
+
+    ret = sys_access(svc->proc.volfile, F_OK);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "gfproxyd Volfile %s is not present", svc->proc.volfile);
+        ret = glusterd_gfproxydsvc_create_volfile(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Couldn't create "
+                   "gfproxyd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+    }
+    runinit(&runner);
+
+    if (this->ctx->cmd_args.vgtool != _gf_none) {
+        len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s",
+                       svc->proc.logdir, svc->proc.logfile);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
+        }
+
+        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+            runner_add_args(&runner, "valgrind", "--leak-check=full",
+                            "--trace-children=yes", "--track-origins=yes",
+                            NULL);
+        else
+            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
+
+        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
+    }
+
+    snprintf(gfproxyd_id, sizeof(gfproxyd_id), "gfproxyd-%s", volinfo->volname);
+    runner_add_args(&runner, SBIN_DIR "/glusterfsd", "-s",
+                    svc->proc.volfileserver, "--volfile-id",
+                    svc->proc.volfileid, "-p", svc->proc.pidfile, "-l",
+                    svc->proc.logfile, "--brick-name", gfproxyd_id, "-S",
+                    svc->conn.sockpath, NULL);
+
+    if (volinfo->memory_accounting)
+        runner_add_arg(&runner, "--mem-accounting");
+    if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                      SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
+                      &localtime_logging) == 0) {
+        if (strcmp(localtime_logging, "enable") == 0)
+            runner_add_arg(&runner, "--localtime-logging");
+    }
+
+    gfproxyd_port = pmap_assign_port(this, volinfo->gfproxyd.port, gfproxyd_id);
+    volinfo->gfproxyd.port = gfproxyd_port;
+
+    runner_add_arg(&runner, "--brick-port");
+    runner_argprintf(&runner, "%d", gfproxyd_port);
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "%s-server.listen-port=%d", volinfo->volname,
+                     gfproxyd_port);
+
+    snprintf(msg, sizeof(msg), "Starting the gfproxyd service for volume %s",
+             volinfo->volname);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    if (flags == PROC_START_NO_WAIT) {
+        ret = runner_run_nowait(&runner);
+    } else {
+        synclock_unlock(&priv->big_lock);
+        {
+            ret = runner_run(&runner);
+        }
+        synclock_lock(&priv->big_lock);
+    }
+
+out:
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_restart()
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list)
+    {
+        /* Start per volume gfproxyd svc */
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            svc = &(volinfo->gfproxyd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                       "Couldn't resolve gfproxyd for "
+                       "vol: %s on restart",
+                       volinfo->volname);
+                gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                goto out;
+            }
+        }
+    }
+out:
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_reconfigure(void *data)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_boolean_t identical = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    volinfo = data;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    if (!volinfo->gfproxyd.svc.inited)
+        goto manager;
+
+    if (!glusterd_is_gfproxyd_enabled(volinfo))
+        goto manager;
+    else if (!glusterd_proc_is_running(&volinfo->gfproxyd.svc.proc))
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_gfproxyd_volfile_identical(
+        volinfo->gfproxyd.svc.name, volinfo, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_gfproxyd_topology_identical(
+        volinfo->gfproxyd.svc.name, volinfo, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to gfproxyd volfile, so that gfproxyd will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_gfproxydsvc_create_volfile(volinfo);
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(this);
+        }
+        goto out;
+    }
+manager:
+    /*
+     * gfproxyd volfile's topology has been changed. gfproxyd server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = volinfo->gfproxyd.svc.manager(&(volinfo->gfproxyd.svc), volinfo,
+                                        PROC_START_NO_WAIT);
+
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.h b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.h
new file mode 100644
index 00000000000..d396b4015f3
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.h
@@ -0,0 +1,47 @@
+/*
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERD_GFPROXYD_SVC_H_
+#define _GLUSTERD_GFPROXYD_SVC_H_
+
+#include "glusterd-svc-mgmt.h"
+
+#define gfproxyd_svc_name "gfproxyd"
+
+struct glusterd_gfproxydsvc_ {
+    glusterd_svc_t svc;
+    gf_store_handle_t *handle;
+    int port;
+};
+
+typedef struct glusterd_gfproxydsvc_ glusterd_gfproxydsvc_t;
+
+void
+glusterd_gfproxydsvc_build(glusterd_svc_t *svc);
+
+int
+glusterd_gfproxydsvc_manager(glusterd_svc_t *svc, void *data, int flags);
+
+int
+glusterd_gfproxydsvc_start(glusterd_svc_t *svc, int flags);
+
+int
+glusterd_gfproxydsvc_stop(glusterd_svc_t *svc, int sig);
+
+int
+glusterd_gfproxydsvc_reconfigure();
+
+void
+glusterd_gfproxydsvc_build_volfile_path(char *server, char *workdir,
+                                        char *volfile, size_t len);
+
+int
+glusterd_gfproxydsvc_restart();
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index c7ffd818a44..1b21c40596d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -9,29 +9,30 @@
 */
 #include <inttypes.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "dict.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "syscall.h"
-#include "timer.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "run.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/run.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-server-quorum.h"
 #include "glusterd-store.h"
 #include "glusterd-locks.h"
 #include "glusterd-snapshot-utils.h"
+#include "glusterd-geo-rep.h"
 
 #include "glusterd1-xdr.h"
 #include "cli1-xdr.h"
@@ -40,242 +41,241 @@
 #include "glusterd-volgen.h"
 #include "glusterd-mountbroker.h"
 #include "glusterd-messages.h"
+#include "glusterd-errno.h"
 
 #include <sys/resource.h>
 #include <inttypes.h>
 
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 
-#include "globals.h"
 #include "glusterd-syncop.h"
 #include "glusterd-messages.h"
 
-#ifdef HAVE_BD_XLATOR
-#include <lvm2app.h>
-#endif
-
 extern glusterd_op_info_t opinfo;
+static int volcount;
 
-int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
-                                rpc_clnt_event_t event,
-                                void *data, rpc_clnt_notify_t notify_fn)
+int
+glusterd_big_locked_notify(struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data,
+                           rpc_clnt_notify_t notify_fn)
 {
-        glusterd_conf_t *priv = THIS->private;
-        int              ret   = -1;
+    glusterd_conf_t *priv = THIS->private;
+    int ret = -1;
 
-        synclock_lock (&priv->big_lock);
-        ret = notify_fn (rpc, mydata, event, data);
-        synclock_unlock (&priv->big_lock);
+    synclock_lock(&priv->big_lock);
+    ret = notify_fn(rpc, mydata, event, data);
+    synclock_unlock(&priv->big_lock);
 
-        return ret;
+    return ret;
 }
 
-int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn)
+int
+glusterd_big_locked_handler(rpcsvc_request_t *req, rpcsvc_actor actor_fn)
 {
-        glusterd_conf_t *priv = THIS->private;
-        int             ret   = -1;
+    glusterd_conf_t *priv = THIS->private;
+    int ret = -1;
 
-        synclock_lock (&priv->big_lock);
-        ret = actor_fn (req);
-        synclock_unlock (&priv->big_lock);
+    synclock_lock(&priv->big_lock);
+    ret = actor_fn(req);
+    synclock_unlock(&priv->big_lock);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
-                            char *hostname, int port,
-                            gd1_mgmt_friend_req *friend_req)
+glusterd_handle_friend_req(rpcsvc_request_t *req, uuid_t uuid, char *hostname,
+                           int port, gd1_mgmt_friend_req *friend_req)
 {
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_friend_req_ctx_t       *ctx = NULL;
-        char                            rhost[UNIX_PATH_MAX + 1] = {0};
-        uuid_t                          friend_uuid = {0};
-        dict_t                          *dict = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_req_ctx_t *ctx = NULL;
+    char rhost[UNIX_PATH_MAX + 1] = {0};
+    dict_t *dict = NULL;
 
-        gf_uuid_parse (uuid_utoa (uuid), friend_uuid);
-        if (!port)
-                port = GF_DEFAULT_BASE_PORT;
+    if (!port)
+        port = GF_DEFAULT_BASE_PORT;
 
-        ret = glusterd_remote_hostname_get (req, rhost, sizeof (rhost));
+    ret = glusterd_remote_hostname_get(req, rhost, sizeof(rhost));
 
-        rcu_read_lock ();
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_friend_req_ctx_t);
+    dict = dict_new();
 
-        peerinfo = glusterd_peerinfo_find (uuid, rhost);
+    RCU_READ_LOCK;
 
-        if (peerinfo == NULL) {
-                ret = glusterd_xfer_friend_add_resp (req, hostname, rhost, port,
-                                                     -1, GF_PROBE_UNKNOWN_PEER);
-                if (friend_req->vols.vols_val) {
-                        free (friend_req->vols.vols_val);
-                        friend_req->vols.vols_val = NULL;
-                }
-                goto out;
-        }
-
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_RCVD_FRIEND_REQ, &event);
+    peerinfo = glusterd_peerinfo_find(uuid, rhost);
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_NEW_GET_FAIL,
-                        "event generation failed: %d", ret);
-                goto out;
+    if (peerinfo == NULL) {
+        gf_event(EVENT_PEER_REJECT, "peer=%s", hostname);
+        ret = glusterd_xfer_friend_add_resp(req, hostname, rhost, port, -1,
+                                            GF_PROBE_UNKNOWN_PEER);
+        if (friend_req->vols.vols_val) {
+            free(friend_req->vols.vols_val);
+            friend_req->vols.vols_val = NULL;
         }
+        goto out;
+    }
 
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_friend_req_ctx_t);
-
-        if (!ctx) {
-                gf_msg ("glusterd", GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Unable to allocate memory");
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (ctx->uuid, uuid);
-        if (hostname)
-                ctx->hostname = gf_strdup (hostname);
-        ctx->req = req;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_RCVD_FRIEND_REQ, &event);
 
-        ret = dict_unserialize (friend_req->vols.vols_val,
-                                friend_req->vols.vols_len,
-                                &dict);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "event generation failed: %d", ret);
+        goto out;
+    }
 
-        if (ret)
-                goto out;
-        else
-                dict->extra_stdfree = friend_req->vols.vols_val;
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-        ctx->vols = dict;
-        event->ctx = ctx;
+    if (!ctx) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_friend_sm_inject_event (event);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL,
-                        "Unable to inject event %d, "
-                        "ret = %d", event->event, ret);
-                goto out;
-        }
+    gf_uuid_copy(ctx->uuid, uuid);
+    if (hostname)
+        ctx->hostname = gf_strdup(hostname);
+    ctx->req = req;
 
-        ret = 0;
-        if (peerinfo && (0 == peerinfo->connected))
-                ret = GLUSTERD_CONNECTION_AWAITED;
+    if (!dict) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(friend_req->vols.vols_val, friend_req->vols.vols_len,
+                           &dict);
+
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    } else
+        dict->extra_stdfree = friend_req->vols.vols_val;
+
+    ctx->vols = dict;
+    event->ctx = ctx;
+
+    ret = glusterd_friend_sm_inject_event(event);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject event %d, "
+               "ret = %d",
+               event->event, ret);
+        goto out;
+    }
+
+    ret = 0;
+    if (peerinfo && (0 == peerinfo->connected))
+        ret = GLUSTERD_CONNECTION_AWAITED;
 
 out:
-        rcu_read_unlock ();
-
-        if (ret && (ret != GLUSTERD_CONNECTION_AWAITED)) {
-                if (ctx && ctx->hostname)
-                        GF_FREE (ctx->hostname);
-                GF_FREE (ctx);
-                if (dict) {
-                        if ((!dict->extra_stdfree) &&
-                            friend_req->vols.vols_val)
-                                free (friend_req->vols.vols_val);
-                        dict_unref (dict);
-                } else {
-                    free (friend_req->vols.vols_val);
-                }
-                if (event)
-                        GF_FREE (event->peername);
-                GF_FREE (event);
-        }
+    RCU_READ_UNLOCK;
 
+    if (ret && (ret != GLUSTERD_CONNECTION_AWAITED)) {
+        if (ctx && ctx->hostname)
+            GF_FREE(ctx->hostname);
+        GF_FREE(ctx);
+        if (dict) {
+            if ((!dict->extra_stdfree) && friend_req->vols.vols_val)
+                free(friend_req->vols.vols_val);
+            dict_unref(dict);
+        } else {
+            free(friend_req->vols.vols_val);
+        }
+        if (event)
+            GF_FREE(event->peername);
+        GF_FREE(event);
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_handle_unfriend_req (rpcsvc_request_t *req, uuid_t  uuid,
-                              char *hostname, int port)
+glusterd_handle_unfriend_req(rpcsvc_request_t *req, uuid_t uuid, char *hostname,
+                             int port)
 {
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_friend_req_ctx_t       *ctx = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_req_ctx_t *ctx = NULL;
 
-        if (!port)
-                port = GF_DEFAULT_BASE_PORT;
+    if (!port)
+        port = GF_DEFAULT_BASE_PORT;
 
-        rcu_read_lock ();
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_friend_req_ctx_t);
 
-        peerinfo = glusterd_peerinfo_find (uuid, hostname);
+    RCU_READ_LOCK;
 
-        if (peerinfo == NULL) {
-                gf_msg ("glusterd", GF_LOG_CRITICAL, 0,
-                        GD_MSG_REQ_FROM_UNKNOWN_PEER,
-                        "Received remove-friend from unknown peer %s",
-                        hostname);
-                ret = glusterd_xfer_friend_remove_resp (req, hostname,
-                                                        port);
-                goto out;
-        }
+    peerinfo = glusterd_peerinfo_find(uuid, hostname);
 
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND, &event);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_REQ_FROM_UNKNOWN_PEER,
+               "Received remove-friend from unknown peer %s", hostname);
+        ret = glusterd_xfer_friend_remove_resp(req, hostname, port);
+        goto out;
+    }
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_NEW_GET_FAIL,
-                        "event generation failed: %d", ret);
-                goto out;
-        }
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
+                                       &event);
 
-        event->peername = gf_strdup (hostname);
-        gf_uuid_copy (event->peerid, uuid);
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "event generation failed: %d", ret);
+        goto out;
+    }
 
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_friend_req_ctx_t);
+    if (hostname)
+        event->peername = gf_strdup(hostname);
 
-        if (!ctx) {
-                gf_msg ("glusterd", GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Unable to allocate memory");
-                ret = -1;
-                goto out;
-        }
+    gf_uuid_copy(event->peerid, uuid);
 
-        gf_uuid_copy (ctx->uuid, uuid);
-        if (hostname)
-                ctx->hostname = gf_strdup (hostname);
-        ctx->req = req;
+    if (!ctx) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
 
-        event->ctx = ctx;
+    gf_uuid_copy(ctx->uuid, uuid);
+    if (hostname)
+        ctx->hostname = gf_strdup(hostname);
+    ctx->req = req;
 
-        ret = glusterd_friend_sm_inject_event (event);
+    event->ctx = ctx;
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL, "Unable to inject event %d, "
-                        "ret = %d", event->event, ret);
-                goto out;
-        }
+    ret = glusterd_friend_sm_inject_event(event);
 
-        ret = 0;
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject event %d, "
+               "ret = %d",
+               event->event, ret);
+        goto out;
+    }
+
+    RCU_READ_UNLOCK;
+
+    return 0;
 
 out:
-        rcu_read_unlock ();
-
-        if (0 != ret) {
-                if (ctx && ctx->hostname)
-                        GF_FREE (ctx->hostname);
-                GF_FREE (ctx);
-                if (event)
-                        GF_FREE (event->peername);
-                GF_FREE (event);
-        }
 
-        return ret;
+    if (0 != ret) {
+        if (ctx && ctx->hostname)
+            GF_FREE(ctx->hostname);
+        GF_FREE(ctx);
+        if (event)
+            GF_FREE(event->peername);
+        GF_FREE(event);
+    }
+
+    return ret;
 }
 
 struct args_pack {
@@ -285,5112 +285,6429 @@ struct args_pack {
 };
 
 static int
-_build_option_key (dict_t *d, char *k, data_t *v, void *tmp)
+_build_option_key(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        char                    reconfig_key[256] = {0, };
-        struct args_pack        *pack             = NULL;
-        int                     ret               = -1;
-        xlator_t                *this             = NULL;
-        glusterd_conf_t         *priv             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        pack = tmp;
-        if (strcmp (k, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
-                return 0;
-
-        if (priv->op_version > GD_OP_VERSION_MIN) {
-                if ((strcmp (k, "features.limit-usage") == 0) ||
-                    (strcmp (k, "features.soft-limit") == 0))
-                        return 0;
-        }
-
-        /* snap-max-hard-limit and snap-max-soft-limit are system   *
-         * options set and managed by snapshot config option. Hence *
-         * they should not be displayed in gluster volume info.     *
-         */
-        if ((strcmp (k, "snap-max-hard-limit") == 0) ||
-            (strcmp (k, "snap-max-soft-limit") == 0))
-                return 0;
-
-        snprintf (reconfig_key, 256, "volume%d.option.%s",
-                  pack->vol_count, k);
-        ret = dict_set_str (pack->dict, reconfig_key, v->data);
-        if (0 == ret)
-                pack->opt_count++;
-
+    char reconfig_key[256] = {
+        0,
+    };
+    int keylen;
+    struct args_pack *pack = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    pack = tmp;
+    if (strcmp(k, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
         return 0;
-}
 
-int
-glusterd_add_tier_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                    dict_t  *dict, int count)
-{
-        int            ret            = -1;
-        char           key[256]      = {0,};
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (dict);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_type", count);
-        ret = dict_set_int32 (dict, key, volinfo->tier_info.cold_type);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_brick_count", count);
-        ret = dict_set_int32 (dict, key, volinfo->tier_info.cold_brick_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_dist_count", count);
-        ret = dict_set_int32 (dict, key,
-                              volinfo->tier_info.cold_dist_leaf_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_replica_count", count);
-        ret = dict_set_int32 (dict, key,
-                              volinfo->tier_info.cold_replica_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_arbiter_count", count);
-        ret = dict_set_int32 (dict, key, volinfo->arbiter_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_disperse_count", count);
-        ret = dict_set_int32 (dict, key,
-                              volinfo->tier_info.cold_disperse_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.cold_redundancy_count", count);
-        ret = dict_set_int32 (dict, key,
-                              volinfo->tier_info.cold_redundancy_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_type", count);
-        ret = dict_set_int32 (dict, key, volinfo->tier_info.hot_type);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_brick_count", count);
-        ret = dict_set_int32 (dict, key, volinfo->tier_info.hot_brick_count);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "volume%d.hot_replica_count", count);
-        ret = dict_set_int32 (dict, key, volinfo->tier_info.hot_replica_count);
-        if (ret)
-                goto out;
+    if (priv->op_version > GD_OP_VERSION_MIN) {
+        if ((strcmp(k, "features.limit-usage") == 0) ||
+            (strcmp(k, "features.soft-limit") == 0))
+            return 0;
+    }
+
+    /* snap-max-hard-limit and snap-max-soft-limit are system   *
+     * options set and managed by snapshot config option. Hence *
+     * they should not be displayed in gluster volume info.     *
+     */
+    if ((strcmp(k, "snap-max-hard-limit") == 0) ||
+        (strcmp(k, "snap-max-soft-limit") == 0))
+        return 0;
 
-out:
-        return ret;
+    keylen = snprintf(reconfig_key, sizeof(reconfig_key), "volume%d.option.%s",
+                      pack->vol_count, k);
+    ret = dict_set_strn(pack->dict, reconfig_key, keylen, v->data);
+    if (0 == ret)
+        pack->opt_count++;
 
+    return 0;
 }
 
 int
-glusterd_add_arbiter_info_to_bricks (glusterd_volinfo_t *volinfo,
-                                     dict_t *volumes, int count)
+glusterd_add_arbiter_info_to_bricks(glusterd_volinfo_t *volinfo,
+                                    dict_t *volumes, int count)
 {
-        char                    key[256]    = {0, };
-        int                     i           = 0;
-        int                     start_index = 0;
-        int                     ret         = 0;
-
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                /*TODO: Add info for hot tier once attach tier of arbiter
-                 * volumes is supported. */
-
-                /* cold tier */
-                if (volinfo->tier_info.cold_replica_count == 1 ||
-                    volinfo->arbiter_count != 1)
-                        return 0;
-
-                i = start_index = volinfo->tier_info.hot_brick_count + 1;
-                for (; i <= volinfo->brick_count; i++) {
-                        if ((i - start_index + 1) %
-                            volinfo->tier_info.cold_replica_count != 0)
-                                continue;
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, 256, "volume%d.brick%d.isArbiter",
-                                  count, i);
-                        ret = dict_set_int32 (volumes, key, 1);
-                        if (ret)
-                                return ret;
-                }
-        } else {
-                if (volinfo->replica_count == 1 || volinfo->arbiter_count != 1)
-                        return 0;
-                for (i = 1; i <= volinfo->brick_count; i++) {
-                        if (i % volinfo->replica_count != 0)
-                                continue;
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, 256, "volume%d.brick%d.isArbiter",
-                                  count, i);
-                        ret = dict_set_int32 (volumes, key, 1);
-                        if (ret)
-                                return ret;
-                }
-        }
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int i = 0;
+    int ret = 0;
+
+    if (volinfo->replica_count == 1 || volinfo->arbiter_count != 1)
         return 0;
+    for (i = 1; i <= volinfo->brick_count; i++) {
+        if (i % volinfo->replica_count != 0)
+            continue;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter", count,
+                          i);
+        ret = dict_set_int32n(volumes, key, keylen, 1);
+        if (ret)
+            return ret;
+    }
+    return 0;
 }
 
 int
-glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                    dict_t  *volumes, int count)
+glusterd_add_volume_detail_to_dict(glusterd_volinfo_t *volinfo, dict_t *volumes,
+                                   int count)
 {
+    int ret = -1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    char *buf = NULL;
+    int i = 1;
+    dict_t *dict = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *volume_id_str = NULL;
+    struct args_pack pack = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    char ta_brick[4096] = {
+        0,
+    };
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volumes);
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    keylen = snprintf(key, sizeof(key), "volume%d.name", count);
+    ret = dict_set_strn(volumes, key, keylen, volinfo->volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.type", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.status", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->status);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.brick_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->brick_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.dist_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->dist_leaf_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.stripe_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->stripe_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.replica_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->replica_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.disperse_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->disperse_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.redundancy_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->redundancy_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.arbiter_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->arbiter_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.transport", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->transport_type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->thin_arbiter_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id));
+    if (!volume_id_str) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.volume_id", count);
+    ret = dict_set_dynstrn(volumes, key, keylen, volume_id_str);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.rebalance", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->rebal.defrag_cmd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.snap_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->snap_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        char brick[1024] = {
+            0,
+        };
+        char brick_uuid[64] = {
+            0,
+        };
+        len = snprintf(brick, sizeof(brick), "%s:%s", brickinfo->hostname,
+                       brickinfo->path);
+        if ((len < 0) || (len >= sizeof(brick))) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        buf = gf_strdup(brick);
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d", count, i);
+        ret = dict_set_dynstrn(volumes, key, keylen, buf);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.uuid", count, i);
+        snprintf(brick_uuid, sizeof(brick_uuid), "%s",
+                 uuid_utoa(brickinfo->uuid));
+        buf = gf_strdup(brick_uuid);
+        if (!buf) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    "brick_uuid=%s", brick_uuid, NULL);
+            goto out;
+        }
+        ret = dict_set_dynstrn(volumes, key, keylen, buf);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+
+        i++;
+    }
+    if (volinfo->thin_arbiter_count == 1) {
+        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
+                                        glusterd_brickinfo_t, brick_list);
+        len = snprintf(ta_brick, sizeof(ta_brick), "%s:%s",
+                       ta_brickinfo->hostname, ta_brickinfo->path);
+        if ((len < 0) || (len >= sizeof(ta_brick))) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        buf = gf_strdup(ta_brick);
+        keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_brick",
+                          count);
+        ret = dict_set_dynstrn(volumes, key, keylen, buf);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterd_add_arbiter_info_to_bricks(volinfo, volumes, count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ARBITER_BRICK_SET_INFO_FAIL, NULL);
+        goto out;
+    }
+
+    dict = volinfo->dict;
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        int                     ret = -1;
-        char                    key[256] = {0, };
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        char                    *buf = NULL;
-        int                     i = 1;
-        dict_t                  *dict = NULL;
-        glusterd_conf_t         *priv = NULL;
-        char                    *volume_id_str  = NULL;
-        struct args_pack        pack = {0,};
-        xlator_t                *this = NULL;
-        GF_UNUSED int           caps = 0;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volumes);
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        snprintf (key, 256, "volume%d.name", count);
-        ret = dict_set_str (volumes, key, volinfo->volname);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.type", count);
-        ret = dict_set_int32 (volumes, key, volinfo->type);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.status", count);
-        ret = dict_set_int32 (volumes, key, volinfo->status);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.brick_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->brick_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.hot_brick_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->tier_info.hot_brick_count);
-        if (ret)
-                goto out;
-
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                ret = glusterd_add_tier_volume_detail_to_dict (volinfo,
-                                                       volumes, count);
-                if (ret)
-                        goto out;
-        }
-
-        snprintf (key, 256, "volume%d.dist_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->dist_leaf_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.stripe_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->stripe_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.replica_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->replica_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.disperse_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->disperse_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.redundancy_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->redundancy_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, sizeof (key), "volume%d.arbiter_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->arbiter_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.transport", count);
-        ret = dict_set_int32 (volumes, key, volinfo->transport_type);
-        if (ret)
-                goto out;
-
-        volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
-        if (!volume_id_str)
-                goto out;
-
-        snprintf (key, sizeof (key), "volume%d.volume_id", count);
-        ret = dict_set_dynstr (volumes, key, volume_id_str);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.rebalance", count);
-        ret = dict_set_int32 (volumes, key, volinfo->rebal.defrag_cmd);
-        if (ret)
-                goto out;
-
-#ifdef HAVE_BD_XLATOR
-        if (volinfo->caps) {
-                caps = 0;
-                snprintf (key, 256, "volume%d.xlator0", count);
-                buf = GF_MALLOC (256, gf_common_mt_char);
-                if (!buf) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                if (volinfo->caps & CAPS_BD)
-                        snprintf (buf, 256, "BD");
-                ret = dict_set_dynstr (volumes, key, buf);
-                if (ret) {
-                        GF_FREE (buf);
-                        goto out;
-                }
-
-                if (volinfo->caps & CAPS_THIN) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "thin");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret) {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-                if (volinfo->caps & CAPS_OFFLOAD_COPY) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "offload_copy");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret) {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-                if (volinfo->caps & CAPS_OFFLOAD_SNAPSHOT) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "offload_snapshot");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret)  {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-                if (volinfo->caps & CAPS_OFFLOAD_ZERO) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "offload_zerofill");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret)  {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-        }
-#endif
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                char    brick[1024] = {0,};
-                char    brick_uuid[64] = {0,};
-                snprintf (key, 256, "volume%d.brick%d", count, i);
-                snprintf (brick, 1024, "%s:%s", brickinfo->hostname,
-                          brickinfo->path);
-                buf = gf_strdup (brick);
-                ret = dict_set_dynstr (volumes, key, buf);
-                if (ret)
-                        goto out;
-                snprintf (key, 256, "volume%d.brick%d.uuid", count, i);
-                snprintf (brick_uuid, 64, "%s", uuid_utoa (brickinfo->uuid));
-                buf = gf_strdup (brick_uuid);
-                if (!buf)
-                        goto out;
-                ret = dict_set_dynstr (volumes, key, buf);
-                if (ret)
-                        goto out;
-
-#ifdef HAVE_BD_XLATOR
-                if (volinfo->caps & CAPS_BD) {
-                        snprintf (key, 256, "volume%d.vg%d", count, i);
-                        snprintf (brick, 1024, "%s", brickinfo->vg);
-                        buf = gf_strdup (brick);
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret)
-                                goto out;
-                }
-#endif
-                i++;
-        }
-        ret = glusterd_add_arbiter_info_to_bricks (volinfo, volumes, count);
-        if (ret)
-                goto out;
-
-        dict = volinfo->dict;
-        if (!dict) {
-                ret = 0;
-                goto out;
-        }
-
-        pack.dict = volumes;
-        pack.vol_count = count;
-        pack.opt_count = 0;
-        dict_foreach (dict, _build_option_key, (void *) &pack);
-        dict_foreach (priv->opts, _build_option_key, &pack);
+    pack.dict = volumes;
+    pack.vol_count = count;
+    pack.opt_count = 0;
+    dict_foreach(dict, _build_option_key, (void *)&pack);
+    dict_foreach(priv->opts, _build_option_key, &pack);
 
-        snprintf (key, 256, "volume%d.opt_count", pack.vol_count);
-        ret = dict_set_int32 (volumes, key, pack.opt_count);
+    keylen = snprintf(key, sizeof(key), "volume%d.opt_count", pack.vol_count);
+    ret = dict_set_int32n(volumes, key, keylen, pack.opt_count);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
-                       char *err_str, size_t err_len)
+glusterd_op_txn_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                      char *err_str, size_t err_len)
 {
-        int32_t                     ret             = -1;
-        int                         npeers          = 0;
-        dict_t                     *dict            = NULL;
-        xlator_t                   *this            = NULL;
-        glusterd_conf_t            *priv            = NULL;
-        int32_t                     locked          = 0;
-        char                       *tmp             = NULL;
-        char                       *volname         = NULL;
-        uuid_t                     *txn_id          = NULL;
-        glusterd_op_info_t          txn_op_info     = {{0},};
-        glusterd_op_sm_event_type_t event_type      = GD_OP_EVENT_NONE;
-        uint32_t                    op_errno        = 0;
-
-        GF_ASSERT (req);
-        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
-        GF_ASSERT (NULL != ctx);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        dict = ctx;
-
-        /* Generate a transaction-id for this operation and
-         * save it in the dict. This transaction id distinguishes
-         * each transaction, and helps separate opinfos in the
-         * op state machine. */
-        ret = glusterd_generate_txn_id (dict, &txn_id);
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t locked = 0;
+    char *tmp = NULL;
+    char *volname = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    uint32_t op_errno = 0;
+    uint32_t timeout = 0;
+
+    GF_ASSERT(req);
+    GF_ASSERT((op > GD_OP_NONE) && (op < GD_OP_MAX));
+    GF_ASSERT(NULL != ctx);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    dict = ctx;
+
+    /* Generate a transaction-id for this operation and
+     * save it in the dict. This transaction id distinguishes
+     * each transaction, and helps separate opinfos in the
+     * op state machine. */
+    ret = glusterd_generate_txn_id(dict, &txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_IDGEN_FAIL,
+               "Failed to generate transaction id");
+        goto out;
+    }
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    ret = glusterd_set_originator_uuid(dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_SET_FAIL,
+               "Failed to set originator_uuid.");
+        goto out;
+    }
+
+    /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+    if (priv->op_version < GD_OP_VERSION_3_6_0) {
+        ret = glusterd_lock(MY_UUID);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_IDGEN_FAIL,
-                        "Failed to generate transaction id");
-                goto out;
-        }
-
-        /* Save the MY_UUID as the originator_uuid. This originator_uuid
-         * will be used by is_origin_glusterd() to determine if a node
-         * is the originator node for a command. */
-        ret = glusterd_set_originator_uuid (dict);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_LOCK_FAIL,
+                   "Unable to acquire lock on localhost, ret: %d", ret);
+            snprintf(err_str, err_len,
+                     "Another transaction is in progress. "
+                     "Please try again after some time.");
+            goto out;
+        }
+    } else {
+        /* If no volname is given as a part of the command, locks will
+         * not be held */
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &tmp);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UUID_SET_FAIL,
-                        "Failed to set originator_uuid.");
+            gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+                   "No Volume name present. "
+                   "Locks not being held.");
+            goto local_locking_done;
+        } else {
+            /* Use a copy of volname, as cli response will be
+             * sent before the unlock, and the volname in the
+             * dict, might be removed */
+            volname = gf_strdup(tmp);
+            if (!volname)
                 goto out;
         }
 
-        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
-        if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                ret = glusterd_lock (MY_UUID);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GLUSTERD_LOCK_FAIL,
-                                "Unable to acquire lock on localhost, ret: %d",
-                                ret);
-                        snprintf (err_str, err_len,
-                                  "Another transaction is in progress. "
-                                  "Please try again after sometime.");
-                        goto out;
-                }
-        } else {
-                /* If no volname is given as a part of the command, locks will
-                 * not be held */
-                ret = dict_get_str (dict, "volname", &tmp);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, errno,
-                                GD_MSG_DICT_GET_FAILED,
-                                "No Volume name present. "
-                                "Locks not being held.");
-                        goto local_locking_done;
-                } else {
-                        /* Use a copy of volname, as cli response will be
-                         * sent before the unlock, and the volname in the
-                         * dict, might be removed */
-                        volname = gf_strdup (tmp);
-                        if (!volname)
-                                goto out;
-                }
+        /* Cli will add timeout key to dict if the default timeout is
+         * other than 2 minutes. Here we use this value to check whether
+         * mgmt_v3_lock_timeout should be set to default value or we
+         * need to change the value according to timeout value
+         * i.e, timeout + 120 seconds. */
+        ret = dict_get_uint32(dict, "timeout", &timeout);
+        if (!ret)
+            priv->mgmt_v3_lock_timeout = timeout + 120;
 
-                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, &op_errno,
-                                             "vol");
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                                "Unable to acquire lock for %s", volname);
-                        snprintf (err_str, err_len,
-                                  "Another transaction is in progress for %s. "
-                                  "Please try again after sometime.", volname);
-                        goto out;
-                }
+        ret = glusterd_mgmt_v3_lock(volname, MY_UUID, &op_errno, "vol");
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Unable to acquire lock for %s", volname);
+            snprintf(err_str, err_len,
+                     "Another transaction is in progress for %s. "
+                     "Please try again after some time.",
+                     volname);
+            goto out;
         }
+    }
 
-        locked = 1;
-        gf_msg_debug (this->name, 0, "Acquired lock on localhost");
+    locked = 1;
+    gf_msg_debug(this->name, 0, "Acquired lock on localhost");
 
 local_locking_done:
-        /* If no volname is given as a part of the command, locks will
-         * not be held, hence sending stage event. */
-        if (volname || (priv->op_version < GD_OP_VERSION_3_6_0))
-                event_type = GD_OP_EVENT_START_LOCK;
-        else {
-                txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
-                event_type = GD_OP_EVENT_ALL_ACC;
-        }
-
-        /* Save opinfo for this transaction with the transaction id */
-        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, ctx, req);
-
-        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_OPINFO_SET_FAIL,
-                        "Unable to set transaction's opinfo");
-                if (ctx)
-                        dict_unref (ctx);
-                goto out;
-        }
-
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, ctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL, "Failed to acquire cluster"
-                        " lock.");
-                goto out;
-        }
+    /* If no volname is given as a part of the command, locks will
+     * not be held, hence sending stage event. */
+    if (volname || (priv->op_version < GD_OP_VERSION_3_6_0))
+        event_type = GD_OP_EVENT_START_LOCK;
+    else {
+        txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
+        event_type = GD_OP_EVENT_ALL_ACC;
+    }
+
+    /* Save opinfo for this transaction with the transaction id */
+    glusterd_txn_opinfo_init(&txn_op_info, NULL, &op, ctx, req);
+
+    ret = glusterd_set_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set transaction's opinfo");
+        if (ctx)
+            dict_unref(ctx);
+        goto out;
+    }
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Failed to acquire cluster"
+               " lock.");
+        goto out;
+    }
 
 out:
-        if (locked && ret) {
-                /* Based on the op-version, we release the
-                 * cluster or mgmt_v3 lock */
-                if (priv->op_version < GD_OP_VERSION_3_6_0)
-                        glusterd_unlock (MY_UUID);
-                else {
-                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
-                                                       "vol");
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                        "Unable to release lock for %s",
-                                        volname);
-                        ret = -1;
-                }
+    if (locked && ret) {
+        /* Based on the op-version, we release the
+         * cluster or mgmt_v3 lock */
+        if (priv->op_version < GD_OP_VERSION_3_6_0)
+            glusterd_unlock(MY_UUID);
+        else {
+            ret = glusterd_mgmt_v3_unlock(volname, MY_UUID, "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", volname);
+            ret = -1;
         }
+    }
 
-        if (volname)
-                GF_FREE (volname);
+    if (volname)
+        GF_FREE(volname);
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-__glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_lock(rpcsvc_request_t *req)
 {
-        dict_t                         *op_ctx      = NULL;
-        int32_t                         ret         = -1;
-        gd1_mgmt_cluster_lock_req       lock_req    = {{0},};
-        glusterd_op_lock_ctx_t         *ctx         = NULL;
-        glusterd_op_sm_event_type_t     op          = GD_OP_EVENT_LOCK;
-        glusterd_op_info_t              txn_op_info = {{0},};
-        glusterd_conf_t                *priv        = NULL;
-        uuid_t                         *txn_id      = NULL;
-        xlator_t                       *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &lock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode lock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "Received LOCK from uuid: %s",
-                uuid_utoa (lock_req.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (lock_req.uuid) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (lock_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+    dict_t *op_ctx = NULL;
+    int32_t ret = -1;
+    gd1_mgmt_cluster_lock_req lock_req = {
+        {0},
+    };
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    glusterd_op_sm_event_type_t op = GD_OP_EVENT_LOCK;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_conf_t *priv = NULL;
+    uuid_t *txn_id = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &lock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode lock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Received LOCK from uuid: %s",
+                 uuid_utoa(lock_req.uuid));
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(lock_req.uuid) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(lock_req.uuid));
+        ret = -1;
+        goto out;
+    }
 
-        if (!ctx) {
-                //respond here
-                return -1;
-        }
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
 
-        gf_uuid_copy (ctx->uuid, lock_req.uuid);
-        ctx->req = req;
-        ctx->dict = NULL;
+    if (!ctx) {
+        // respond here
+        return -1;
+    }
 
-        op_ctx =  dict_new ();
-        if (!op_ctx) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Unable to set new dict");
-                goto out;
-        }
-
-        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, op_ctx, req);
-
-        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_OPINFO_SET_FAIL,
-                        "Unable to set transaction's opinfo");
-                dict_unref (txn_op_info.op_ctx);
-                goto out;
-        }
+    gf_uuid_copy(ctx->uuid, lock_req.uuid);
+    ctx->req = req;
+    ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, txn_id, ctx);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL,
-                        "Failed to inject event GD_OP_EVENT_LOCK");
+    op_ctx = dict_new();
+    if (!op_ctx) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to set new dict");
+        goto out;
+    }
+
+    glusterd_txn_opinfo_init(&txn_op_info, NULL, &op, op_ctx, req);
+
+    ret = glusterd_set_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set transaction's opinfo");
+        dict_unref(txn_op_info.op_ctx);
+        goto out;
+    }
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_LOCK, txn_id, ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Failed to inject event GD_OP_EVENT_LOCK");
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    if (ret)
+        GF_FREE(ctx);
+
+    return ret;
 }
 
 int
-glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+glusterd_handle_cluster_lock(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cluster_lock);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cluster_lock);
 }
 
 static int
-glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
-                         int op, uuid_t uuid,
-                         char *buf_val, size_t buf_len,
-                         gf_gld_mem_types_t mem_type,
-                         glusterd_req_ctx_t **req_ctx_out)
+glusterd_req_ctx_create(rpcsvc_request_t *rpc_req, int op, uuid_t uuid,
+                        char *buf_val, size_t buf_len,
+                        gf_gld_mem_types_t mem_type,
+                        glusterd_req_ctx_t **req_ctx_out)
 {
-        int                 ret     = -1;
-        char                str[50] = {0,};
-        glusterd_req_ctx_t *req_ctx = NULL;
-        dict_t             *dict    = NULL;
-        xlator_t           *this    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        gf_uuid_unparse (uuid, str);
-        gf_msg_debug (this->name, 0, "Received op from uuid %s", str);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        req_ctx = GF_CALLOC (1, sizeof (*req_ctx), mem_type);
-        if (!req_ctx) {
-                goto out;
-        }
-
-        gf_uuid_copy (req_ctx->uuid, uuid);
-        req_ctx->op = op;
-        ret = dict_unserialize (buf_val, buf_len, &dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        req_ctx->dict = dict;
-        req_ctx->req = rpc_req;
-        *req_ctx_out = req_ctx;
-        ret = 0;
+    int ret = -1;
+    char str[50] = {
+        0,
+    };
+    glusterd_req_ctx_t *req_ctx = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_uuid_unparse(uuid, str);
+    gf_msg_debug(this->name, 0, "Received op from uuid %s", str);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    req_ctx = GF_CALLOC(1, sizeof(*req_ctx), mem_type);
+    if (!req_ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req_ctx->uuid, uuid);
+    req_ctx->op = op;
+    ret = dict_unserialize(buf_val, buf_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    req_ctx->dict = dict;
+    req_ctx->req = rpc_req;
+    *req_ctx_out = req_ctx;
+    ret = 0;
 out:
-        if (ret) {
-                if (dict)
-                        dict_unref (dict);
-                GF_FREE (req_ctx);
-        }
-        return ret;
+    if (ret) {
+        if (dict)
+            dict_unref(dict);
+        GF_FREE(req_ctx);
+    }
+    return ret;
 }
 
 int
-__glusterd_handle_stage_op (rpcsvc_request_t *req)
+__glusterd_handle_stage_op(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        gd1_mgmt_stage_op_req           op_req = {{0},};
-        xlator_t                        *this = NULL;
-        uuid_t                          *txn_id = NULL;
-        glusterd_op_info_t              txn_op_info = {{0},};
-        glusterd_op_sm_state_info_t     state = {0,};
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode stage "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        ret = glusterd_req_ctx_create (req, op_req.op, op_req.uuid,
-                                       op_req.buf.buf_val, op_req.buf.buf_len,
-                                       gf_gld_mt_op_stage_ctx_t, &req_ctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_CTX_CREATE_FAIL, "Failed to create req_ctx");
-                goto out;
-        }
-
-        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
-        gf_msg_debug (this->name, 0, "transaction ID = %s",
-                uuid_utoa (*txn_id));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL);
-        rcu_read_unlock ();
+    int32_t ret = -1;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    gd1_mgmt_stage_op_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_op_sm_state_info_t state = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode stage "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ret = glusterd_req_ctx_create(req, op_req.op, op_req.uuid,
+                                  op_req.buf.buf_val, op_req.buf.buf_len,
+                                  gf_gld_mt_op_stage_ctx_t, &req_ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_CTX_CREATE_FAIL,
+               "Failed to create req_ctx");
+        goto out;
+    }
+
+    ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+    gf_msg_debug(this->name, 0, "transaction ID = %s", uuid_utoa(*txn_id));
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    /* In cases where there is no volname, the receivers won't have a
+     * transaction opinfo created, as for those operations, the locking
+     * phase where the transaction opinfos are created, won't be called.
+     * skip_locking will be true for all such transaction and we clear
+     * the txn_opinfo after the staging phase, except for geo-replication
+     * operations where we need to access txn_opinfo in the later phases also.
+     */
+    ret = glusterd_get_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No transaction's opinfo set");
+
+        state.state = GD_OP_STATE_LOCKED;
+        glusterd_txn_opinfo_init(&txn_op_info, &state, &op_req.op,
+                                 req_ctx->dict, req);
+
+        if (req_ctx->op != GD_OP_GSYNC_SET)
+            txn_op_info.skip_locking = _gf_true;
+        ret = glusterd_set_txn_opinfo(txn_id, &txn_op_info);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+                   "Unable to set transaction's opinfo");
+            dict_unref(req_ctx->dict);
+            goto out;
         }
+    }
 
-        /* In cases where there is no volname, the receivers won't have a
-         * transaction opinfo created, as for those operations, the locking
-         * phase where the transaction opinfos are created, won't be called. */
-        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "No transaction's opinfo set");
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_STAGE_OP, txn_id, req_ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Failed to inject event GD_OP_EVENT_STAGE_OP");
 
-                state.state = GD_OP_STATE_LOCKED;
-                glusterd_txn_opinfo_init (&txn_op_info, &state, &op_req.op,
-                                          req_ctx->dict, req);
-
-                ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set transaction's opinfo");
-                        dict_unref (req_ctx->dict);
-                        goto out;
-                }
-        }
-
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP,
-                                           txn_id, req_ctx);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL,
-                        "Failed to inject event GD_OP_EVENT_STAGE_OP");
-
- out:
-        free (op_req.buf.buf_val);//malloced by xdr
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-        return ret;
+out:
+    free(op_req.buf.buf_val);  // malloced by xdr
+    glusterd_friend_sm();
+    glusterd_op_sm();
+    return ret;
 }
 
 int
-glusterd_handle_stage_op (rpcsvc_request_t *req)
+glusterd_handle_stage_op(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_stage_op);
+    return glusterd_big_locked_handler(req, __glusterd_handle_stage_op);
 }
 
-
 int
-__glusterd_handle_commit_op (rpcsvc_request_t *req)
+__glusterd_handle_commit_op(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        gd1_mgmt_commit_op_req          op_req = {{0},};
-        xlator_t                        *this = NULL;
-        uuid_t                          *txn_id = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode commit "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    gd1_mgmt_commit_op_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode commit "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
 
-        //the structures should always be equal
-        GF_ASSERT (sizeof (gd1_mgmt_commit_op_req) == sizeof (gd1_mgmt_stage_op_req));
-        ret = glusterd_req_ctx_create (req, op_req.op, op_req.uuid,
-                                       op_req.buf.buf_val, op_req.buf.buf_len,
-                                       gf_gld_mt_op_commit_ctx_t, &req_ctx);
-        if (ret)
-                goto out;
+    // the structures should always be equal
+    GF_ASSERT(sizeof(gd1_mgmt_commit_op_req) == sizeof(gd1_mgmt_stage_op_req));
+    ret = glusterd_req_ctx_create(req, op_req.op, op_req.uuid,
+                                  op_req.buf.buf_val, op_req.buf.buf_len,
+                                  gf_gld_mt_op_commit_ctx_t, &req_ctx);
+    if (ret)
+        goto out;
 
-        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
-        gf_msg_debug (this->name, 0, "transaction ID = %s",
-                uuid_utoa (*txn_id));
+    ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+    gf_msg_debug(this->name, 0, "transaction ID = %s", uuid_utoa(*txn_id));
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP,
-                                           txn_id, req_ctx);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_COMMIT_OP, txn_id, req_ctx);
 
 out:
-        free (op_req.buf.buf_val);//malloced by xdr
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-        return ret;
+    free(op_req.buf.buf_val);  // malloced by xdr
+    glusterd_friend_sm();
+    glusterd_op_sm();
+    return ret;
 }
 
 int
-glusterd_handle_commit_op (rpcsvc_request_t *req)
+glusterd_handle_commit_op(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_commit_op);
+    return glusterd_big_locked_handler(req, __glusterd_handle_commit_op);
 }
 
 int
-__glusterd_handle_cli_probe (rpcsvc_request_t *req)
+__glusterd_handle_cli_probe(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                  cli_req = {{0,},};
-        glusterd_peerinfo_t       *peerinfo = NULL;
-        gf_boolean_t                run_fsm = _gf_true;
-        xlator_t                      *this = NULL;
-        char                     *bind_name = NULL;
-        dict_t                        *dict = NULL;
-        char                      *hostname = NULL;
-        int                            port = 0;
-        int                        op_errno = 0;
-
-        GF_ASSERT (req);
-        this = THIS;
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0)  {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "xdr decoding error");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "Failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
-                        "Failed to get hostname");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PORT_NOTFOUND_IN_DICT, "Failed to get port");
-                goto out;
-        }
-
-        if (glusterd_is_any_volume_in_server_quorum (this) &&
-            !does_gd_meet_server_quorum (this)) {
-                glusterd_xfer_cli_probe_resp (req, -1, GF_PROBE_QUORUM_NOT_MET,
-                                              NULL, hostname, port, dict);
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET,
-                        "Server quorum not met. Rejecting operation.");
-                ret = 0;
-                goto out;
-        }
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_CLI_REQ_RECVD,
-                "Received CLI probe req %s %d",
-                hostname, port);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {
+        {
+            0,
+        },
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gf_boolean_t run_fsm = _gf_true;
+    xlator_t *this = NULL;
+    char *bind_name = NULL;
+    dict_t *dict = NULL;
+    char *hostname = NULL;
+    int port = 0;
+    int op_errno = 0;
+
+    GF_ASSERT(req);
+    this = THIS;
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "xdr decoding error");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
 
-        if (dict_get_str(this->options,"transport.socket.bind-address",
-                         &bind_name) == 0) {
-                gf_msg_debug ("glusterd", 0,
-                        "only checking probe address vs. bind address");
-                ret = gf_is_same_address (bind_name, hostname);
-        }
-        else {
-                ret = gf_is_local_addr (hostname);
-        }
-        if (ret) {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST,
-                                              NULL, hostname, port, dict);
-                ret = 0;
-                goto out;
-        }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
+               "Failed to get hostname");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "port", SLEN("port"), &port);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORT_NOTFOUND_IN_DICT,
+               "Failed to get port");
+        goto out;
+    }
+
+    if (glusterd_is_any_volume_in_server_quorum(this) &&
+        !does_gd_meet_server_quorum(this)) {
+        glusterd_xfer_cli_probe_resp(req, -1, GF_PROBE_QUORUM_NOT_MET, NULL,
+                                     hostname, port, dict);
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        ret = 0;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_CLI_REQ_RECVD,
+           "Received CLI probe req %s %d", hostname, port);
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"), &bind_name) == 0) {
+        gf_msg_debug("glusterd", 0,
+                     "only checking probe address vs. bind address");
+        ret = gf_is_same_address(bind_name, hostname);
+    } else {
+        ret = gf_is_local_addr(hostname);
+    }
+    if (ret) {
+        glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_LOCALHOST, NULL, hostname,
+                                     port, dict);
+        ret = 0;
+        goto out;
+    }
 
-        rcu_read_lock ();
+    RCU_READ_LOCK;
 
-        peerinfo = glusterd_peerinfo_find_by_hostname (hostname);
-        ret = (peerinfo && gd_peer_has_address (peerinfo, hostname));
+    peerinfo = glusterd_peerinfo_find_by_hostname(hostname);
+    ret = (peerinfo && gd_peer_has_address(peerinfo, hostname));
 
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        if (ret) {
-                gf_msg_debug ("glusterd", 0, "Probe host %s port %d "
-                        "already a peer", hostname, port);
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
-                                              hostname, port, dict);
-                ret = 0;
-                goto out;
-        }
+    if (ret) {
+        gf_msg_debug("glusterd", 0,
+                     "Probe host %s port %d "
+                     "already a peer",
+                     hostname, port);
+        glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_FRIEND, NULL, hostname,
+                                     port, dict);
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_probe_begin (req, hostname, port, dict, &op_errno);
+    ret = glusterd_probe_begin(req, hostname, port, dict, &op_errno);
 
-        if (ret == GLUSTERD_CONNECTION_AWAITED) {
-                //fsm should be run after connection establishes
-                run_fsm = _gf_false;
-                ret = 0;
+    if (ret == GLUSTERD_CONNECTION_AWAITED) {
+        // fsm should be run after connection establishes
+        run_fsm = _gf_false;
+        ret = 0;
 
-        } else if (ret == -1) {
-                glusterd_xfer_cli_probe_resp (req, -1, op_errno,
-                                              NULL, hostname, port, dict);
-                goto out;
-        }
+    } else if (ret == -1) {
+        glusterd_xfer_cli_probe_resp(req, -1, op_errno, NULL, hostname, port,
+                                     dict);
+        goto out;
+    }
 
 out:
-        free (cli_req.dict.dict_val);
+    free(cli_req.dict.dict_val);
 
-        if (run_fsm) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (run_fsm) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_probe (rpcsvc_request_t *req)
+glusterd_handle_cli_probe(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_cli_probe);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_probe);
 }
 
 int
-__glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+__glusterd_handle_cli_deprobe(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                  cli_req = {{0,},};
-        uuid_t                         uuid = {0};
-        int                        op_errno = 0;
-        xlator_t                      *this = NULL;
-        glusterd_conf_t               *priv = NULL;
-        dict_t                        *dict = NULL;
-        char                      *hostname = NULL;
-        int                            port = 0;
-        int                           flags = 0;
-        glusterd_volinfo_t         *volinfo = NULL;
-        glusterd_volinfo_t             *tmp = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "Failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_CLI_REQ_RECVD,
-                "Received CLI deprobe req");
-
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
-                        "Failed to get hostname");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PORT_NOTFOUND_IN_DICT, "Failed to get port");
-                goto out;
-        }
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_FLAGS_NOTFOUND_IN_DICT, "Failed to get flags");
-                goto out;
-        }
-
-        ret = glusterd_hostname_to_uuid (hostname, uuid);
-        if (ret) {
-                op_errno = GF_DEPROBE_NOT_FRIEND;
-                goto out;
-        }
-
-        if (!gf_uuid_compare (uuid, MY_UUID)) {
-                op_errno = GF_DEPROBE_LOCALHOST;
-                ret = -1;
-                goto out;
-        }
-
-        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                /* Check if peers are connected, except peer being
-                * detached*/
-                if (!glusterd_chk_peers_connected_befriended (uuid)) {
-                        ret = -1;
-                        op_errno = GF_DEPROBE_FRIEND_DOWN;
-                        goto out;
-                }
-        }
-
-        /* Check for if volumes exist with some bricks on the peer being
-        * detached. It's not a problem if a volume contains none or all
-        * of its bricks on the peer being detached
-        */
-        cds_list_for_each_entry_safe (volinfo, tmp, &priv->volumes,
-                                      vol_list) {
-                ret = glusterd_friend_contains_vol_bricks (volinfo,
-                                                           uuid);
-                if (ret == 1) {
-                        op_errno = GF_DEPROBE_BRICK_EXIST;
-                        goto out;
-                }
-        }
-
-        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                if (glusterd_is_any_volume_in_server_quorum (this) &&
-                    !does_gd_meet_server_quorum (this)) {
-                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                GD_MSG_SERVER_QUORUM_NOT_MET,
-                                "Server quorum not met. Rejecting operation.");
-                        ret = -1;
-                        op_errno = GF_DEPROBE_QUORUM_NOT_MET;
-                        goto out;
-                }
-        }
+    int32_t ret = -1;
+    gf_cli_req cli_req = {
+        {
+            0,
+        },
+    };
+    uuid_t uuid = {0};
+    int op_errno = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    char *hostname = NULL;
+    int port = 0;
+    int flags = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    glusterd_snap_t *snapinfo = NULL;
+    glusterd_snap_t *tmpsnap = NULL;
+    gf_boolean_t need_free = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
 
-        if (!gf_uuid_is_null (uuid)) {
-                ret = glusterd_deprobe_begin (req, hostname, port, uuid, dict,
-                                              &op_errno);
+        if (dict) {
+            need_free = _gf_true;
         } else {
-                ret = glusterd_deprobe_begin (req, hostname, port, NULL, dict,
-                                              &op_errno);
+            ret = -1;
+            goto out;
         }
 
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        }
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_CLI_REQ_RECVD,
+           "Received CLI deprobe req");
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
+               "Failed to get hostname");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "port", SLEN("port"), &port);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORT_NOTFOUND_IN_DICT,
+               "Failed to get port");
+        goto out;
+    }
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+               "Failed to get flags");
+        goto out;
+    }
+
+    ret = glusterd_hostname_to_uuid(hostname, uuid);
+    if (ret) {
+        op_errno = GF_DEPROBE_NOT_FRIEND;
+        goto out;
+    }
+
+    if (!gf_uuid_compare(uuid, MY_UUID)) {
+        op_errno = GF_DEPROBE_LOCALHOST;
+        ret = -1;
+        goto out;
+    }
+
+    if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+        /* Check if peers are connected, except peer being
+         * detached*/
+        if (!glusterd_chk_peers_connected_befriended(uuid)) {
+            ret = -1;
+            op_errno = GF_DEPROBE_FRIEND_DOWN;
+            goto out;
+        }
+    }
+
+    /* Check for if volumes exist with some bricks on the peer being
+     * detached. It's not a problem if a volume contains none or all
+     * of its bricks on the peer being detached
+     */
+    cds_list_for_each_entry_safe(volinfo, tmp, &priv->volumes, vol_list)
+    {
+        ret = glusterd_friend_contains_vol_bricks(volinfo, uuid);
+        if (ret == 1) {
+            op_errno = GF_DEPROBE_BRICK_EXIST;
+            goto out;
+        }
+    }
+
+    cds_list_for_each_entry_safe(snapinfo, tmpsnap, &priv->snapshots, snap_list)
+    {
+        ret = glusterd_friend_contains_snap_bricks(snapinfo, uuid);
+        if (ret == 1) {
+            op_errno = GF_DEPROBE_SNAP_BRICK_EXIST;
+            goto out;
+        }
+    }
+    if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+        if (glusterd_is_any_volume_in_server_quorum(this) &&
+            !does_gd_meet_server_quorum(this)) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Server quorum not met. Rejecting operation.");
+            ret = -1;
+            op_errno = GF_DEPROBE_QUORUM_NOT_MET;
+            goto out;
+        }
+    }
+
+    if (!gf_uuid_is_null(uuid)) {
+        ret = glusterd_deprobe_begin(req, hostname, port, uuid, dict,
+                                     &op_errno);
+    } else {
+        ret = glusterd_deprobe_begin(req, hostname, port, NULL, dict,
+                                     &op_errno);
+    }
+
+    need_free = _gf_false;
+
 out:
-        free (cli_req.dict.dict_val);
+    free(cli_req.dict.dict_val);
 
-        if (ret) {
-                ret = glusterd_xfer_cli_deprobe_resp (req, ret, op_errno, NULL,
-                                                      hostname, dict);
+    if (ret) {
+        ret = glusterd_xfer_cli_deprobe_resp(req, ret, op_errno, NULL, hostname,
+                                             dict);
+        if (need_free) {
+            dict_unref(dict);
         }
+    }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+glusterd_handle_cli_deprobe(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_cli_deprobe);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_deprobe);
 }
 
 int
-__glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+__glusterd_handle_cli_list_friends(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf1_cli_peer_list_req           cli_req = {0,};
-        dict_t                          *dict = NULL;
-
-        GF_ASSERT (req);
+    int32_t ret = -1;
+    gf1_cli_peer_list_req cli_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req,
+                         (xdrproc_t)xdr_gf1_cli_peer_list_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_CLI_REQ_RECVD,
+           "Received cli list req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf1_cli_peer_list_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_CLI_REQ_RECVD,
-                "Received cli list req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = glusterd_list_friends (req, dict, cli_req.flags);
+    ret = glusterd_list_friends(req, dict, cli_req.flags);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+glusterd_handle_cli_list_friends(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_list_friends);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_list_friends);
 }
 
 static int
-__glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_get_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        int32_t                         flags = 0;
-        dict_t                         *dict = NULL;
-        xlator_t                       *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    int32_t flags = 0;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_GET_VOL_REQ_RCVD,
+           "Received get vol req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_GET_VOL_REQ_RCVD,
-                "Received get vol req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                         GD_MSG_FLAGS_NOTFOUND_IN_DICT, "failed to get flags");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = glusterd_get_volumes (req, dict, flags);
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+               "failed to get flags");
+        goto out;
+    }
+    ret = glusterd_get_volumes(req, dict, flags);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_get_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_get_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_get_volume);
 }
 
 int
-__glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+__glusterd_handle_cli_uuid_reset(rpcsvc_request_t *req)
 {
-        int                     ret     = -1;
-        dict_t                  *dict   = NULL;
-        xlator_t                *this   = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        uuid_t                  uuid    = {0};
-        gf_cli_rsp              rsp     = {0,};
-        gf_cli_req              cli_req = {{0,}};
-        char                    msg_str[2048] = {0,};
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg_debug ("glusterd", 0, "Received uuid reset req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
-                                  "the buffer");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
+    int ret = -1;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uuid_t uuid = {0};
+    gf_cli_rsp rsp = {
+        0,
+    };
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    char msg_str[128] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug("glusterd", 0, "Received uuid reset req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        /* In the above section if dict_unserialize is successful, ret is set
-         * to zero.
-         */
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg_str, sizeof(msg_str),
+                     "Unable to decode "
+                     "the buffer");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    /* In the above section if dict_unserialize is successful, ret is set
+     * to zero.
+     */
+    ret = -1;
+    // Do not allow peer reset if there are any volumes in the cluster
+    if (!cds_list_empty(&priv->volumes)) {
+        snprintf(msg_str, sizeof(msg_str),
+                 "volumes are already "
+                 "present in the cluster. Resetting uuid is not "
+                 "allowed");
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLS_ALREADY_PRESENT, "%s",
+               msg_str);
+        goto out;
+    }
+
+    // Do not allow peer reset if trusted storage pool is already formed
+    if (!cds_list_empty(&priv->peers)) {
+        snprintf(msg_str, sizeof(msg_str),
+                 "trusted storage pool "
+                 "has been already formed. Please detach this peer "
+                 "from the pool and reset its uuid.");
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_TSP_ALREADY_FORMED, "%s",
+               msg_str);
+        goto out;
+    }
+
+    gf_uuid_copy(uuid, priv->uuid);
+    ret = glusterd_uuid_generate_save();
+
+    if (!gf_uuid_compare(uuid, MY_UUID)) {
+        snprintf(msg_str, sizeof(msg_str),
+                 "old uuid and the new uuid"
+                 " are same. Try gluster peer reset again");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUIDS_SAME_RETRY, "%s",
+               msg_str);
         ret = -1;
-        // Do not allow peer reset if there are any volumes in the cluster
-        if (!cds_list_empty (&priv->volumes)) {
-                snprintf (msg_str, sizeof (msg_str), "volumes are already "
-                          "present in the cluster. Resetting uuid is not "
-                          "allowed");
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOLS_ALREADY_PRESENT, "%s", msg_str);
-                goto out;
-        }
-
-        // Do not allow peer reset if trusted storage pool is already formed
-        if (!cds_list_empty (&priv->peers)) {
-                snprintf (msg_str, sizeof (msg_str),"trusted storage pool "
-                          "has been already formed. Please detach this peer "
-                          "from the pool and reset its uuid.");
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_TSP_ALREADY_FORMED, "%s", msg_str);
-                goto out;
-        }
-
-        gf_uuid_copy (uuid, priv->uuid);
-        ret = glusterd_uuid_generate_save ();
-
-        if (!gf_uuid_compare (uuid, MY_UUID)) {
-                snprintf (msg_str, sizeof (msg_str), "old uuid and the new uuid"
-                          " are same. Try gluster peer reset again");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UUIDS_SAME_RETRY, "%s", msg_str);
-                ret = -1;
-                goto out;
-        }
+        goto out;
+    }
 
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                if (msg_str[0] == '\0')
-                        snprintf (msg_str, sizeof (msg_str), "Operation "
-                                  "failed");
-                rsp.op_errstr = msg_str;
-                ret = 0;
-        } else {
-                rsp.op_errstr = "";
-        }
+    if (ret) {
+        rsp.op_ret = -1;
+        if (msg_str[0] == '\0')
+            snprintf(msg_str, sizeof(msg_str),
+                     "Operation "
+                     "failed");
+        rsp.op_errstr = msg_str;
+        ret = 0;
+    } else {
+        rsp.op_errstr = "";
+    }
 
-        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
-                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+    glusterd_to_cli(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp, dict);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+glusterd_handle_cli_uuid_reset(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_uuid_reset);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_uuid_reset);
 }
 
 int
-__glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+__glusterd_handle_cli_uuid_get(rpcsvc_request_t *req)
 {
-        int                     ret         = -1;
-        dict_t                  *dict       = NULL;
-        dict_t                  *rsp_dict   = NULL;
-        xlator_t                *this       = NULL;
-        glusterd_conf_t         *priv       = NULL;
-        gf_cli_rsp              rsp         = {0,};
-        gf_cli_req              cli_req     = {{0,}};
-        char                    msg_str[2048] = {0,};
-        char                    uuid_str[64] = {0,};
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg_debug ("glusterd", 0, "Received uuid get req");
-
-        if (cli_req.dict.dict_len) {
-                dict  = dict_new ();
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
-                                  "the buffer");
-                        goto out;
-
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-
-                }
+    int ret = -1;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    char err_str[64] = {
+        0,
+    };
+    char uuid_str[64] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug("glusterd", 0, "Received uuid get req");
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
 
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                ret = -1;
-                goto out;
-        }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the buffer");
+            goto out;
 
-        uuid_utoa_r (MY_UUID, uuid_str);
-        ret = dict_set_str (rsp_dict, "uuid", uuid_str);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set uuid in "
-                        "dictionary.");
-                goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "Failed to serialize "
-                        "dictionary.");
-                goto out;
-        }
-        ret = 0;
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    ret = dict_set_strn(rsp_dict, "uuid", SLEN("uuid"), uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set uuid in "
+               "dictionary.");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    ret = 0;
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                if (msg_str[0] == '\0')
-                        snprintf (msg_str, sizeof (msg_str), "Operation "
-                                  "failed");
-                rsp.op_errstr = msg_str;
-
-        } else {
-                rsp.op_errstr = "";
+    if (ret) {
+        rsp.op_ret = -1;
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str),
+                     "Operation "
+                     "failed");
+        rsp.op_errstr = err_str;
+
+    } else {
+        rsp.op_errstr = "";
+    }
 
-        }
+    glusterd_to_cli(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp, dict);
 
-        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
-                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    GF_FREE(rsp.dict.dict_val);
 
-        return 0;
+    return 0;
 }
 int
-glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+glusterd_handle_cli_uuid_get(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_uuid_get);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_uuid_get);
 }
 
 int
-__glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_list_volume(rpcsvc_request_t *req)
 {
-        int                     ret = -1;
-        dict_t                  *dict = NULL;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        int                     count = 0;
-        char                    key[1024] = {0,};
-        gf_cli_rsp              rsp = {0,};
-
-        GF_ASSERT (req);
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d", count);
-                ret = dict_set_str (dict, key, volinfo->volname);
-                if (ret)
-                        goto out;
-                count++;
-        }
-
-        ret = dict_set_int32 (dict, "count", count);
+    int ret = -1;
+    dict_t *dict = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int count = 0;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    gf_cli_rsp rsp = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        keylen = snprintf(key, sizeof(key), "volume%d", count);
+        ret = dict_set_strn(dict, key, keylen, volinfo->volname);
         if (ret)
-                goto out;
+            goto out;
+        count++;
+    }
 
-        ret = dict_allocate_and_serialize (dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret)
-                goto out;
+    ret = dict_set_int32n(dict, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
 
-        ret = 0;
+    ret = dict_allocate_and_serialize(dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret)
+        goto out;
+
+    ret = 0;
 
 out:
-        rsp.op_ret = ret;
-        if (ret)
-                rsp.op_errstr = "Error listing volumes";
-        else
-                rsp.op_errstr = "";
+    rsp.op_ret = ret;
+    if (ret)
+        rsp.op_errstr = "Error listing volumes";
+    else
+        rsp.op_errstr = "";
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    ret = 0;
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    GF_FREE(rsp.dict.dict_val);
 
-        return ret;
+    glusterd_friend_sm();
+    glusterd_op_sm();
+
+    return ret;
 }
 
 int
-glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_list_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_list_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_list_volume);
 }
 
 int32_t
-glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
-                   char *err_str, size_t err_len)
+glusterd_op_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                  char *err_str, size_t err_len)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        ret = glusterd_op_txn_begin (req, op, ctx, err_str, err_len);
+    ret = glusterd_op_txn_begin(req, op, ctx, err_str, err_len);
 
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_handle_ganesha_cmd (rpcsvc_request_t *req)
+__glusterd_handle_ganesha_cmd(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = { {0,} } ;
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_GANESHA;
-        char                            *volname = NULL;
-        char                            *op_errstr = NULL;
-        gf_boolean_t                    help = _gf_false;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode "
-                          "request received from cli");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_GANESHA;
+    char *op_errstr = NULL;
+    char err_str[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        gf_msg_trace (this->name, 0, "Received global option request");
+    gf_msg_trace(this->name, 0, "Received global option request");
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_GANESHA, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_GANESHA, dict);
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        if (op_errstr)
-                GF_FREE (op_errstr);
-        if (dict)
-                dict_unref(dict);
-
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    if (op_errstr)
+        GF_FREE(op_errstr);
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
-
 int
-glusterd_handle_ganesha_cmd (rpcsvc_request_t *req)
+glusterd_handle_ganesha_cmd(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_ganesha_cmd);
+    return glusterd_big_locked_handler(req, __glusterd_handle_ganesha_cmd);
 }
 
 static int
-__glusterd_handle_reset_volume (rpcsvc_request_t *req)
+__glusterd_handle_reset_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_RESET_VOLUME;
-        char                            *volname = NULL;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                "Received reset vol req");
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode request "
-                          "received from cli");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLNAME_NOTFOUND_IN_DICT, "%s", err_str);
-                goto out;
-        }
-        gf_msg_debug (this->name, 0, "Received volume reset request for "
-                "volume %s", volname);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_RESET_VOLUME;
+    char *volname = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, 0, "Received reset vol req");
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode request "
+                 "received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_RESET_VOLUME, dict);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Received volume reset request for "
+                 "volume %s",
+                 volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_RESET_VOLUME, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_reset_volume (rpcsvc_request_t *req)
+glusterd_handle_reset_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_reset_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_reset_volume);
 }
 
 int
-__glusterd_handle_set_volume (rpcsvc_request_t *req)
+__glusterd_handle_set_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_SET_VOLUME;
-        char                            *key = NULL;
-        char                            *value = NULL;
-        char                            *volname = NULL;
-        char                            *op_errstr = NULL;
-        gf_boolean_t                    help = _gf_false;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode "
-                          "request received from cli");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name while handling volume set command");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
-
-        if (strcmp (volname, "help") == 0 ||
-            strcmp (volname, "help-xml") == 0) {
-                ret = glusterd_volset_help (dict, &op_errstr);
-                help = _gf_true;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "key1", &key);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get key while"
-                          " handling volume set for %s", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "value1", &value);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get value while"
-                          " handling volume set for %s", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
-        gf_msg_debug (this->name, 0, "Received volume set request for "
-                "volume %s", volname);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_SET_VOLUME;
+    char *key = NULL;
+    char *value = NULL;
+    char *volname = NULL;
+    char *op_errstr = NULL;
+    gf_boolean_t help = _gf_false;
+    char err_str[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_SET_VOLUME, dict);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name while handling volume set command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (strcmp(volname, "help") == 0 || strcmp(volname, "help-xml") == 0) {
+        ret = glusterd_volset_help(dict, &op_errstr);
+        help = _gf_true;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "key1", SLEN("key1"), &key);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get key while"
+                 " handling volume set for %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "value1", SLEN("value1"), &value);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get value while"
+                 " handling volume set for %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Received volume set request for "
+                 "volume %s",
+                 volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_SET_VOLUME, dict);
 
 out:
-        if (help)
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req, dict,
-                                                     (op_errstr)? op_errstr:"");
-        else if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        if (op_errstr)
-                GF_FREE (op_errstr);
-
-        return ret;
+    if (help)
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict,
+                                            (op_errstr) ? op_errstr : "");
+    else if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    if (op_errstr)
+        GF_FREE(op_errstr);
+
+    return ret;
 }
 
 int
-glusterd_handle_set_volume (rpcsvc_request_t *req)
+glusterd_handle_set_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_set_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_set_volume);
 }
 
 int
-__glusterd_handle_sync_volume (rpcsvc_request_t *req)
+__glusterd_handle_sync_volume(rpcsvc_request_t *req)
 {
-        int32_t                          ret     = -1;
-        gf_cli_req                       cli_req = {{0,}};
-        dict_t                           *dict = NULL;
-        gf_cli_rsp                       cli_rsp = {0.};
-        char                             msg[2048] = {0,};
-        char                             *volname = NULL;
-        gf1_cli_sync_volume              flags = 0;
-        char                             *hostname = NULL;
-        xlator_t                         *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    gf_cli_rsp cli_rsp = {0.};
+    char msg[2048] = {
+        0,
+    };
+    char *volname = NULL;
+    gf1_cli_sync_volume flags = 0;
+    char *hostname = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get hostname");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HOSTNAME_NOTFOUND_IN_DICT, "%s", msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get hostname");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
+               "%s", msg);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        ret = dict_get_int32n(dict, "flags", SLEN("flags"), (int32_t *)&flags);
         if (ret) {
-                ret = dict_get_int32 (dict, "flags", (int32_t*)&flags);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Failed to get volume name"
-                                  " or flags");
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FLAGS_NOTFOUND_IN_DICT, "%s", msg);
-                        goto out;
-                }
+            snprintf(msg, sizeof(msg), "Failed to get volume name or flags");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+                   "%s", msg);
+            goto out;
         }
+    }
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_VOL_SYNC_REQ_RCVD, "Received volume sync req "
-                "for volume %s", (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_SYNC_REQ_RCVD,
+           "Received volume sync req "
+           "for volume %s",
+           (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
 
-        if (gf_is_local_addr (hostname)) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "sync from localhost"
-                          " not allowed");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SYNC_FROM_LOCALHOST_UNALLOWED, "%s", msg);
-                goto out;
-        }
+    if (gf_is_local_addr(hostname)) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "sync from localhost"
+                 " not allowed");
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_SYNC_FROM_LOCALHOST_UNALLOWED, "%s", msg);
+        goto out;
+    }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_SYNC_VOLUME, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_SYNC_VOLUME, dict);
 
 out:
-        if (ret) {
-                cli_rsp.op_ret = -1;
-                cli_rsp.op_errstr = msg;
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                glusterd_to_cli (req, &cli_rsp, NULL, 0, NULL,
-                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
-
-                ret = 0; //sent error to cli, prevent second reply
-        }
-
-        return ret;
+    if (ret) {
+        cli_rsp.op_ret = -1;
+        cli_rsp.op_errstr = msg;
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        glusterd_to_cli(req, &cli_rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp,
+                        dict);
+
+        ret = 0;  // sent error to cli, prevent second reply
+    }
+
+    return ret;
 }
 
 int
-glusterd_handle_sync_volume (rpcsvc_request_t *req)
+glusterd_handle_sync_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_sync_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_sync_volume);
 }
 
 int
-glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
-                            char *op_errstr, dict_t *dict)
+glusterd_fsm_log_send_resp(rpcsvc_request_t *req, int op_ret, char *op_errstr,
+                           dict_t *dict)
 {
+    int ret = -1;
+    gf1_cli_fsm_log_rsp rsp = {0};
 
-        int                             ret = -1;
-        gf1_cli_fsm_log_rsp             rsp = {0};
-
-        GF_ASSERT (req);
-        GF_ASSERT (op_errstr);
+    GF_ASSERT(req);
+    GF_ASSERT(op_errstr);
 
-        rsp.op_ret = op_ret;
-        rsp.op_errstr = op_errstr;
-        if (rsp.op_ret == 0)
-                ret = dict_allocate_and_serialize (dict, &rsp.fsm_log.fsm_log_val,
-                                                &rsp.fsm_log.fsm_log_len);
+    rsp.op_ret = op_ret;
+    rsp.op_errstr = op_errstr;
+    if (rsp.op_ret == 0) {
+        ret = dict_allocate_and_serialize(dict, &rsp.fsm_log.fsm_log_val,
+                                          &rsp.fsm_log.fsm_log_len);
+        if (ret < 0) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+            return ret;
+        }
+    }
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
-        GF_FREE (rsp.fsm_log.fsm_log_val);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
+    GF_FREE(rsp.fsm_log.fsm_log_val);
 
-        gf_msg_debug ("glusterd", 0, "Responded, ret: %d", ret);
+    gf_msg_debug("glusterd", 0, "Responded, ret: %d", ret);
 
-        return 0;
+    return 0;
 }
 
 int
-__glusterd_handle_fsm_log (rpcsvc_request_t *req)
+__glusterd_handle_fsm_log(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf1_cli_fsm_log_req             cli_req = {0,};
-        dict_t                          *dict = NULL;
-        glusterd_sm_tr_log_t            *log = NULL;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        char                            msg[2048] = {0};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("xlator", (this != NULL), out);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf1_cli_fsm_log_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from client.");
-                req->rpc_err = GARBAGE_ARGS;
-                snprintf (msg, sizeof (msg), "Garbage request");
-                goto out;
-        }
+    int32_t ret = -1;
+    gf1_cli_fsm_log_req cli_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char msg[2048] = {0};
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("xlator", (this != NULL), out);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req,
+                         (xdrproc_t)xdr_gf1_cli_fsm_log_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from client.");
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(msg, sizeof(msg), "Garbage request");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (strcmp("", cli_req.name) == 0) {
+        conf = this->private;
+        ret = glusterd_sm_tr_log_add_to_dict(dict, &conf->op_sm_log);
+    } else {
+        RCU_READ_LOCK;
 
-        if (strcmp ("", cli_req.name) == 0) {
-                conf = this->private;
-                ret = glusterd_sm_tr_log_add_to_dict (dict, &conf->op_sm_log);
+        peerinfo = glusterd_peerinfo_find_by_hostname(cli_req.name);
+        if (!peerinfo) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s is not a peer", cli_req.name);
         } else {
-                rcu_read_lock ();
-
-                peerinfo = glusterd_peerinfo_find_by_hostname (cli_req.name);
-                if (!peerinfo) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "%s is not a peer",
-                                  cli_req.name);
-                } else {
-                        ret = glusterd_sm_tr_log_add_to_dict
-                                (dict, &peerinfo->sm_log);
-                }
-
-                rcu_read_unlock ();
+            ret = glusterd_sm_tr_log_add_to_dict(dict, &peerinfo->sm_log);
+            RCU_READ_UNLOCK;
         }
+    }
 
 out:
-        (void)glusterd_fsm_log_send_resp (req, ret, msg, dict);
-        free (cli_req.name);//malloced by xdr
-        if (dict)
-                dict_unref (dict);
+    (void)glusterd_fsm_log_send_resp(req, ret, msg, dict);
+    free(cli_req.name);  // malloced by xdr
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return 0;//send 0 to avoid double reply
+    return 0;  // send 0 to avoid double reply
 }
 
 int
-glusterd_handle_fsm_log (rpcsvc_request_t *req)
+glusterd_handle_fsm_log(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_fsm_log);
+    return glusterd_big_locked_handler(req, __glusterd_handle_fsm_log);
 }
 
 int
-glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
+glusterd_op_lock_send_resp(rpcsvc_request_t *req, int32_t status)
 {
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_cluster_lock_rsp       rsp = {{0},};
-        int                             ret = -1;
-
-        GF_ASSERT (req);
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op_ret = status;
+    GF_ASSERT(req);
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op_ret = status;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
 
-        gf_msg_debug (THIS->name, 0, "Responded to lock, ret: %d", ret);
+    gf_msg_debug(THIS->name, 0, "Responded to lock, ret: %d", ret);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+glusterd_op_unlock_send_resp(rpcsvc_request_t *req, int32_t status)
 {
+    gd1_mgmt_cluster_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_cluster_unlock_rsp     rsp = {{0},};
-        int                             ret = -1;
+    GF_ASSERT(req);
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
 
-        GF_ASSERT (req);
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+    gf_msg_debug(THIS->name, 0, "Responded to unlock, ret: %d", ret);
 
-        gf_msg_debug (THIS->name, 0, "Responded to unlock, ret: %d", ret);
-
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+glusterd_op_mgmt_v3_lock_send_resp(rpcsvc_request_t *req, uuid_t *txn_id,
                                    int32_t status)
 {
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_v3_lock_rsp    rsp = {{0},};
-        int                     ret = -1;
-
-        GF_ASSERT (req);
-        GF_ASSERT (txn_id);
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-               rsp.op_errno = errno;
-        gf_uuid_copy (rsp.txn_id, *txn_id);
+    GF_ASSERT(req);
+    GF_ASSERT(txn_id);
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = errno;
+    gf_uuid_copy(rsp.txn_id, *txn_id);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
 
-        gf_msg_debug (THIS->name, 0, "Responded to mgmt_v3 lock, ret: %d",
-                ret);
+    gf_msg_debug(THIS->name, 0, "Responded to mgmt_v3 lock, ret: %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+glusterd_op_mgmt_v3_unlock_send_resp(rpcsvc_request_t *req, uuid_t *txn_id,
                                      int32_t status)
 {
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_v3_unlock_rsp      rsp = {{0},};
-        int                             ret = -1;
-
-        GF_ASSERT (req);
-        GF_ASSERT (txn_id);
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-               rsp.op_errno = errno;
-        glusterd_get_uuid (&rsp.uuid);
-        gf_uuid_copy (rsp.txn_id, *txn_id);
+    GF_ASSERT(req);
+    GF_ASSERT(txn_id);
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = errno;
+    glusterd_get_uuid(&rsp.uuid);
+    gf_uuid_copy(rsp.txn_id, *txn_id);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
 
-        gf_msg_debug (THIS->name, 0, "Responded to mgmt_v3 unlock, ret: %d",
-                ret);
+    gf_msg_debug(THIS->name, 0, "Responded to mgmt_v3 unlock, ret: %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_unlock(rpcsvc_request_t *req)
 {
-        gd1_mgmt_cluster_unlock_req     unlock_req = {{0}, };
-        int32_t                         ret = -1;
-        glusterd_op_lock_ctx_t          *ctx = NULL;
-        xlator_t                        *this = NULL;
-        uuid_t                          *txn_id = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &unlock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode unlock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-
-        gf_msg_debug (this->name, 0,
-                "Received UNLOCK from uuid: %s", uuid_utoa (unlock_req.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (unlock_req.uuid) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (unlock_req.uuid));
-                ret = -1;
-                goto out;
-        }
+    gd1_mgmt_cluster_unlock_req unlock_req = {
+        {0},
+    };
+    int32_t ret = -1;
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &unlock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode unlock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Received UNLOCK from uuid: %s",
+                 uuid_utoa(unlock_req.uuid));
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(unlock_req.uuid) == NULL);
+    RCU_READ_LOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(unlock_req.uuid));
+        ret = -1;
+        goto out;
+    }
 
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
 
-        if (!ctx) {
-                //respond here
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "No memory.");
-                return -1;
-        }
-        gf_uuid_copy (ctx->uuid, unlock_req.uuid);
-        ctx->req = req;
-        ctx->dict = NULL;
+    if (!ctx) {
+        // respond here
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "No memory.");
+        return -1;
+    }
+    gf_uuid_copy(ctx->uuid, unlock_req.uuid);
+    ctx->req = req;
+    ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, txn_id, ctx);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_UNLOCK, txn_id, ctx);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+glusterd_handle_cluster_unlock(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cluster_unlock);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cluster_unlock);
 }
 
 int
-glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
-                             int32_t op, int32_t status,
-                             char *op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status,
+                            char *op_errstr, dict_t *rsp_dict)
 {
-        gd1_mgmt_stage_op_rsp           rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "failed to get serialized length of dict");
-                return ret;
-        }
+    gd1_mgmt_stage_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        return ret;
+    }
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
 
-        gf_msg_debug (this->name, 0, "Responded to stage, ret: %d", ret);
-        GF_FREE (rsp.dict.dict_val);
+    gf_msg_debug(this->name, 0, "Responded to stage, ret: %d", ret);
+    GF_FREE(rsp.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_commit_send_resp (rpcsvc_request_t *req,
-                               int32_t op, int32_t status, char *op_errstr,
-                               dict_t *rsp_dict)
+glusterd_op_commit_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status,
+                             char *op_errstr, dict_t *rsp_dict)
 {
-        gd1_mgmt_commit_op_rsp          rsp      = {{0}, };
-        int                             ret      = -1;
-        xlator_t                        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        if (rsp_dict) {
-                ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                                   &rsp.dict.dict_len);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                                "failed to get serialized length of dict");
-                        goto out;
-                }
-        }
+    gd1_mgmt_commit_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
 
+    if (rsp_dict) {
+        ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                          &rsp.dict.dict_len);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+            goto out;
+        }
+    }
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
 
-        gf_msg_debug (this->name, 0, "Responded to commit, ret: %d", ret);
+    gf_msg_debug(this->name, 0, "Responded to commit, ret: %d", ret);
 
 out:
-        GF_FREE (rsp.dict.dict_val);
-        return ret;
+    GF_FREE(rsp.dict.dict_val);
+    return ret;
 }
 
 int
-__glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+__glusterd_handle_incoming_friend_req(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gd1_mgmt_friend_req     friend_req = {{0},};
-        gf_boolean_t            run_fsm = _gf_true;
-
-        GF_ASSERT (req);
-        ret = xdr_to_generic (req->msg[0], &friend_req,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from friend");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_PROBE_RCVD,
-                "Received probe from uuid: %s", uuid_utoa (friend_req.uuid));
-        ret = glusterd_handle_friend_req (req, friend_req.uuid,
-                                          friend_req.hostname, friend_req.port,
-                                          &friend_req);
-
-        if (ret == GLUSTERD_CONNECTION_AWAITED) {
-                //fsm should be run after connection establishes
-                run_fsm = _gf_false;
-                ret = 0;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_friend_req friend_req = {
+        {0},
+    };
+    gf_boolean_t run_fsm = _gf_true;
+
+    GF_ASSERT(req);
+    ret = xdr_to_generic(req->msg[0], &friend_req,
+                         (xdrproc_t)xdr_gd1_mgmt_friend_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from friend");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PROBE_RCVD,
+           "Received probe from uuid: %s", uuid_utoa(friend_req.uuid));
+    ret = glusterd_handle_friend_req(req, friend_req.uuid, friend_req.hostname,
+                                     friend_req.port, &friend_req);
+
+    if (ret == GLUSTERD_CONNECTION_AWAITED) {
+        // fsm should be run after connection establishes
+        run_fsm = _gf_false;
+        ret = 0;
+    }
 
 out:
-        free (friend_req.hostname);//malloced by xdr
+    free(friend_req.hostname);  // malloced by xdr
 
-        if (run_fsm) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (run_fsm) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+glusterd_handle_incoming_friend_req(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_incoming_friend_req);
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_incoming_friend_req);
 }
 
 int
-__glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+__glusterd_handle_incoming_unfriend_req(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gd1_mgmt_friend_req     friend_req = {{0},};
-        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
-
-        GF_ASSERT (req);
-        ret = xdr_to_generic (req->msg[0], &friend_req,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received.");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_UNFRIEND_REQ_RCVD,
-                "Received unfriend from uuid: %s", uuid_utoa (friend_req.uuid));
-
-        ret = glusterd_remote_hostname_get (req, remote_hostname,
-                                            sizeof (remote_hostname));
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_HOSTNAME_RESOLVE_FAIL,
-                        "Unable to get the remote hostname");
-                goto out;
-        }
-        ret = glusterd_handle_unfriend_req (req, friend_req.uuid,
-                                            remote_hostname, friend_req.port);
+    int32_t ret = -1;
+    gd1_mgmt_friend_req friend_req = {
+        {0},
+    };
+    char remote_hostname[UNIX_PATH_MAX + 1] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+    ret = xdr_to_generic(req->msg[0], &friend_req,
+                         (xdrproc_t)xdr_gd1_mgmt_friend_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received.");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_UNFRIEND_REQ_RCVD,
+           "Received unfriend from uuid: %s", uuid_utoa(friend_req.uuid));
+
+    ret = glusterd_remote_hostname_get(req, remote_hostname,
+                                       sizeof(remote_hostname));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_RESOLVE_FAIL,
+               "Unable to get the remote hostname");
+        goto out;
+    }
+    ret = glusterd_handle_unfriend_req(req, friend_req.uuid, remote_hostname,
+                                       friend_req.port);
 
 out:
-        free (friend_req.hostname);//malloced by xdr
-        free (friend_req.vols.vols_val);//malloced by xdr
+    free(friend_req.hostname);       // malloced by xdr
+    free(friend_req.vols.vols_val);  // malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+glusterd_handle_incoming_unfriend_req(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_incoming_unfriend_req);
-
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_incoming_unfriend_req);
 }
 
 int
-glusterd_handle_friend_update_delete (dict_t *dict)
+glusterd_handle_friend_update_delete(dict_t *dict)
 {
-        char                    *hostname = NULL;
-        int32_t                 ret = -1;
+    char *hostname = NULL;
+    int32_t ret = -1;
 
-        GF_ASSERT (dict);
+    GF_ASSERT(dict);
 
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret)
+        goto out;
 
-        ret = glusterd_friend_remove (NULL, hostname);
+    ret = glusterd_friend_remove(NULL, hostname);
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_peer_hostname_update (glusterd_peerinfo_t *peerinfo,
-                               const char *hostname, gf_boolean_t store_update)
+glusterd_peer_hostname_update(glusterd_peerinfo_t *peerinfo,
+                              const char *hostname, gf_boolean_t store_update)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (hostname);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(hostname);
 
-        ret = gd_add_address_to_peer (peerinfo, hostname);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
-                        "Couldn't add address to the peer info");
-                goto out;
-        }
+    ret = gd_add_address_to_peer(peerinfo, hostname);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0,
+               GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
+               "Couldn't add address to the peer info");
+        goto out;
+    }
 
-        if (store_update)
-                ret = glusterd_store_peerinfo (peerinfo);
+    if (store_update)
+        ret = glusterd_store_peerinfo(peerinfo);
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-__glusterd_handle_friend_update (rpcsvc_request_t *req)
+__glusterd_handle_friend_update(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gd1_mgmt_friend_update     friend_req = {{0},};
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        gd1_mgmt_friend_update_rsp rsp = {{0},};
-        dict_t                  *dict = NULL;
-        char                    key[100] = {0,};
-        char                    *uuid_buf = NULL;
-        int                     i = 1;
-        int                     count = 0;
-        uuid_t                  uuid = {0,};
-        glusterd_peerctx_args_t args = {0};
-        int32_t                 op = 0;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &friend_req,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_update);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        ret = 0;
-        rcu_read_lock ();
-        if (glusterd_peerinfo_find (friend_req.uuid, NULL) == NULL) {
-                ret = -1;
-        }
-        rcu_read_unlock ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_REQ_FROM_UNKNOWN_PEER,
-                        "Received friend update request "
-                        "from unknown peer %s", uuid_utoa (friend_req.uuid));
-                goto out;
-        }
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_FRIEND_UPDATE_RCVD,
-                "Received friend update from uuid: %s", uuid_utoa (friend_req.uuid));
-
-        if (friend_req.friends.friends_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (friend_req.friends.friends_val,
-                                        friend_req.friends.friends_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = friend_req.friends.friends_val;
-                }
-        }
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
+    int32_t ret = -1;
+    gd1_mgmt_friend_update friend_req = {
+        {0},
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    gd1_mgmt_friend_update_rsp rsp = {
+        {0},
+    };
+    dict_t *dict = NULL;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *uuid_buf = NULL;
+    int i = 1;
+    int count = 0;
+    uuid_t uuid = {
+        0,
+    };
+    glusterd_peerctx_args_t args = {0};
+    int32_t op = 0;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &friend_req,
+                         (xdrproc_t)xdr_gd1_mgmt_friend_update);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ret = 0;
+    RCU_READ_LOCK;
+    if (glusterd_peerinfo_find(friend_req.uuid, NULL) == NULL) {
+        ret = -1;
+    }
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_REQ_FROM_UNKNOWN_PEER,
+               "Received friend update request "
+               "from unknown peer %s",
+               uuid_utoa(friend_req.uuid));
+        gf_event(EVENT_UNKNOWN_PEER, "peer=%s", uuid_utoa(friend_req.uuid));
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_FRIEND_UPDATE_RCVD,
+           "Received friend update from uuid: %s", uuid_utoa(friend_req.uuid));
+
+    if (friend_req.friends.friends_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = dict_get_int32 (dict, "op", &op);
+        ret = dict_unserialize(friend_req.friends.friends_val,
+                               friend_req.friends.friends_len, &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        } else {
+            dict->extra_stdfree = friend_req.friends.friends_val;
+        }
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=op", NULL);
+        goto out;
+    }
+
+    if (GD_FRIEND_UPDATE_DEL == op) {
+        (void)glusterd_handle_friend_update_delete(dict);
+        goto out;
+    }
+
+    args.mode = GD_MODE_ON;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
         if (ret)
-                goto out;
+            goto out;
+        gf_uuid_parse(uuid_buf, uuid);
 
-        if (GD_FRIEND_UPDATE_DEL == op) {
-                ret = glusterd_handle_friend_update_delete (dict);
-                goto out;
+        if (!gf_uuid_compare(uuid, MY_UUID)) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_UUID_RECEIVED,
+                   "Received my uuid as Friend");
+            i++;
+            continue;
         }
 
-        args.mode = GD_MODE_ON;
-        while ( i <= count) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
-                gf_uuid_parse (uuid_buf, uuid);
-
-                if (!gf_uuid_compare (uuid, MY_UUID)) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_UUID_RECEIVED,
-                                "Received my uuid as Friend");
-                        i++;
-                        continue;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d", i);
-
-                rcu_read_lock ();
-                peerinfo = glusterd_peerinfo_find (uuid, NULL);
-                if (peerinfo == NULL) {
-                        /* Create a new peer and add it to the list as there is
-                         * no existing peer with the uuid
-                         */
-                        peerinfo = gd_peerinfo_from_dict (dict, key);
-                        if (peerinfo == NULL) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_PEERINFO_CREATE_FAIL,
-                                        "Could not create peerinfo from dict "
-                                        "for prefix %s", key);
-                                goto unlock;
-                        }
-
-                        /* As this is a new peer, it should be added as a
-                         * friend.  The friend state machine will take care of
-                         * correcting the state as required
-                         */
-                        peerinfo->state.state = GD_FRIEND_STATE_BEFRIENDED;
-
-                        ret = glusterd_friend_add_from_peerinfo (peerinfo, 0,
-                                                                 &args);
-                } else {
-                        /* As an existing peer was found, update it with the new
-                         * information
-                         */
-                        ret = gd_update_peerinfo_from_dict (peerinfo, dict,
-                                                            key);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_PEER_INFO_UPDATE_FAIL,
-                                        "Failed to "
-                                        "update peer %s", peerinfo->hostname);
-                                goto unlock;
-                        }
-                        ret = glusterd_store_peerinfo (peerinfo);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_PEERINFO_CREATE_FAIL,
-                                        "Failed to store peerinfo");
-                }
-unlock:
-                rcu_read_unlock ();
-                if (ret)
-                        break;
-
-                peerinfo = NULL;
-                i++;
-        }
+        snprintf(key, sizeof(key), "friend%d", i);
 
-out:
-        gf_uuid_copy (rsp.uuid, MY_UUID);
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
-        if (dict) {
-                if (!dict->extra_stdfree && friend_req.friends.friends_val)
-                        free (friend_req.friends.friends_val);//malloced by xdr
-                dict_unref (dict);
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find(uuid, NULL);
+        if (peerinfo == NULL) {
+            /* Create a new peer and add it to the list as there is
+             * no existing peer with the uuid
+             */
+            peerinfo = gd_peerinfo_from_dict(dict, key);
+            if (peerinfo == NULL) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                       "Could not create peerinfo from dict "
+                       "for prefix %s",
+                       key);
+                goto unlock;
+            }
+
+            /* As this is a new peer, it should be added as a
+             * friend.  The friend state machine will take care of
+             * correcting the state as required
+             */
+            peerinfo->state.state = GD_FRIEND_STATE_BEFRIENDED;
+
+            ret = glusterd_friend_add_from_peerinfo(peerinfo, 0, &args);
         } else {
-                free (friend_req.friends.friends_val);//malloced by xdr
-        }
-
-        if (peerinfo)
-                glusterd_peerinfo_cleanup (peerinfo);
+            /* As an existing peer was found, update it with the new
+             * information
+             */
+            ret = gd_update_peerinfo_from_dict(peerinfo, dict, key);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_PEER_INFO_UPDATE_FAIL,
+                       "Failed to "
+                       "update peer %s",
+                       peerinfo->hostname);
+                goto unlock;
+            }
+            ret = glusterd_store_peerinfo(peerinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                       "Failed to store peerinfo");
+                gf_event(EVENT_PEER_STORE_FAILURE, "peer=%s",
+                         peerinfo->hostname);
+            }
+        }
+    unlock:
+        RCU_READ_UNLOCK;
+        if (ret)
+            break;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        peerinfo = NULL;
+        i++;
+    }
 
-        return ret;
+out:
+    gf_uuid_copy(rsp.uuid, MY_UUID);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+    if (dict) {
+        if (!dict->extra_stdfree && friend_req.friends.friends_val)
+            free(friend_req.friends.friends_val);  // malloced by xdr
+        dict_unref(dict);
+    } else {
+        free(friend_req.friends.friends_val);  // malloced by xdr
+    }
+
+    if (peerinfo)
+        glusterd_peerinfo_cleanup(peerinfo);
+
+    glusterd_friend_sm();
+    glusterd_op_sm();
+
+    return ret;
 }
 
 int
-glusterd_handle_friend_update (rpcsvc_request_t *req)
+glusterd_handle_friend_update(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_friend_update);
+    return glusterd_big_locked_handler(req, __glusterd_handle_friend_update);
 }
 
 int
-__glusterd_handle_probe_query (rpcsvc_request_t *req)
+__glusterd_handle_probe_query(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        gd1_mgmt_probe_req              probe_req = {{0},};
-        gd1_mgmt_probe_rsp              rsp = {{0},};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_peerctx_args_t         args = {0};
-        int                             port = 0;
-        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("xlator", (this != NULL), out);
-
-        ret = xdr_to_generic (req->msg[0], &probe_req,
-                              (xdrproc_t)xdr_gd1_mgmt_probe_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode probe "
-                        "request");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        conf = this->private;
-        if (probe_req.port)
-                port = probe_req.port;
-        else
-                port = GF_DEFAULT_BASE_PORT;
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_PROBE_RCVD,
-                "Received probe from uuid: %s", uuid_utoa (probe_req.uuid));
-
-        /* Check for uuid collision and handle it in a user friendly way by
-         * sending the error.
-         */
-        if (!gf_uuid_compare (probe_req.uuid, MY_UUID)) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UUIDS_SAME_RETRY, "Peer uuid %s is same as "
-                        "local uuid. Please check the uuid of both the peers "
-                        "from %s/%s", uuid_utoa (probe_req.uuid),
-                        GLUSTERD_DEFAULT_WORKDIR, GLUSTERD_INFO_FILE);
-                rsp.op_ret = -1;
-                rsp.op_errno = GF_PROBE_SAME_UUID;
-                rsp.port = port;
-                goto respond;
-        }
-
-        ret = glusterd_remote_hostname_get (req, remote_hostname,
-                                            sizeof (remote_hostname));
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gd1_mgmt_probe_req probe_req = {
+        {0},
+    };
+    gd1_mgmt_probe_rsp rsp = {
+        {0},
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_args_t args = {0};
+    int port = 0;
+    char remote_hostname[UNIX_PATH_MAX + 1] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("xlator", (this != NULL), out);
+
+    ret = xdr_to_generic(req->msg[0], &probe_req,
+                         (xdrproc_t)xdr_gd1_mgmt_probe_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode probe "
+               "request");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    conf = this->private;
+    if (probe_req.port)
+        port = probe_req.port;
+    else
+        port = GF_DEFAULT_BASE_PORT;
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PROBE_RCVD,
+           "Received probe from uuid: %s", uuid_utoa(probe_req.uuid));
+
+    /* Check for uuid collision and handle it in a user friendly way by
+     * sending the error.
+     */
+    if (!gf_uuid_compare(probe_req.uuid, MY_UUID)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UUIDS_SAME_RETRY,
+               "Peer uuid %s is same as "
+               "local uuid. Please check the uuid of both the peers "
+               "from %s/%s",
+               uuid_utoa(probe_req.uuid), GLUSTERD_DEFAULT_WORKDIR,
+               GLUSTERD_INFO_FILE);
+        rsp.op_ret = -1;
+        rsp.op_errno = GF_PROBE_SAME_UUID;
+        rsp.port = port;
+        goto respond;
+    }
+
+    ret = glusterd_remote_hostname_get(req, remote_hostname,
+                                       sizeof(remote_hostname));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_RESOLVE_FAIL,
+               "Unable to get the remote hostname");
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(probe_req.uuid, remote_hostname);
+    if ((peerinfo == NULL) && (!cds_list_empty(&conf->peers))) {
+        rsp.op_ret = -1;
+        rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER;
+    } else if (peerinfo == NULL) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PEER_NOT_FOUND,
+               "Unable to find peerinfo"
+               " for host: %s (%d)",
+               remote_hostname, port);
+        args.mode = GD_MODE_ON;
+        ret = glusterd_friend_add(remote_hostname, port,
+                                  GD_FRIEND_STATE_PROBE_RCVD, NULL, &peerinfo,
+                                  0, &args);
         if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_HOSTNAME_RESOLVE_FAIL,
-                        "Unable to get the remote hostname");
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PEER_ADD_FAIL,
+                   "Failed to add peer %s", remote_hostname);
+            rsp.op_errno = GF_PROBE_ADD_FAILED;
         }
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (probe_req.uuid, remote_hostname);
-        if ((peerinfo == NULL) && (!cds_list_empty (&conf->peers))) {
-                rsp.op_ret = -1;
-                rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER;
-        } else if (peerinfo == NULL) {
-                gf_msg ("glusterd", GF_LOG_INFO, 0,
-                        GD_MSG_PEER_NOT_FOUND,
-                        "Unable to find peerinfo"
-                        " for host: %s (%d)", remote_hostname, port);
-                args.mode = GD_MODE_ON;
-                ret = glusterd_friend_add (remote_hostname, port,
-                                           GD_FRIEND_STATE_PROBE_RCVD,
-                                           NULL, &peerinfo, 0, &args);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_PEER_ADD_FAIL,
-                                "Failed to add peer %s",
-                                remote_hostname);
-                        rsp.op_errno = GF_PROBE_ADD_FAILED;
-                }
-        }
-        rcu_read_unlock ();
+    }
+    RCU_READ_UNLOCK;
 
 respond:
-        gf_uuid_copy (rsp.uuid, MY_UUID);
+    gf_uuid_copy(rsp.uuid, MY_UUID);
 
-        rsp.hostname = probe_req.hostname;
-        rsp.op_errstr = "";
+    rsp.hostname = probe_req.hostname;
+    rsp.op_errstr = "";
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+    ret = 0;
 
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_RESPONSE_INFO, "Responded to %s, op_ret: %d, "
-                "op_errno: %d, ret: %d", remote_hostname,
-                rsp.op_ret, rsp.op_errno, ret);
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Responded to %s, op_ret: %d, "
+           "op_errno: %d, ret: %d",
+           remote_hostname, rsp.op_ret, rsp.op_errno, ret);
 
 out:
-        free (probe_req.hostname);//malloced by xdr
+    free(probe_req.hostname);  // malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
-int glusterd_handle_probe_query (rpcsvc_request_t *req)
+int
+glusterd_handle_probe_query(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_probe_query);
+    return glusterd_big_locked_handler(req, __glusterd_handle_probe_query);
 }
 
 int
-__glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_profile_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret     = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_PROFILE_VOLUME;
-        char                            *volname = NULL;
-        int32_t                         op = 0;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len > 0) {
-                dict = dict_new();
-                if (!dict)
-                        goto out;
-                dict_unserialize (cli_req.dict.dict_val,
-                                  cli_req.dict.dict_len, &dict);
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLNAME_NOTFOUND_IN_DICT, "%s", err_str);
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_VOL_PROFILE_REQ_RCVD,
-                "Received volume profile req "
-                "for volume %s", volname);
-        ret = dict_get_int32 (dict, "op", &op);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get operation");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
-        }
-
-        ret = glusterd_op_begin (req, cli_op, dict, err_str, sizeof (err_str));
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_PROFILE_VOLUME;
+    char *volname = NULL;
+    int32_t op = 0;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len > 0) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+        dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict);
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_PROFILE_REQ_RCVD,
+           "Received volume profile req "
+           "for volume %s",
+           volname);
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &op);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Unable to get operation");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_6_0) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than %d. Falling back "
+                     "to op-sm framework.",
+                     GD_OP_VERSION_6_0);
+        ret = glusterd_op_begin(req, cli_op, dict, err_str, sizeof(err_str));
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(
+            req, cli_op, dict);
+    }
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        free (cli_req.dict.dict_val);
+    free(cli_req.dict.dict_val);
 
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_profile_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_profile_volume);
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_cli_profile_volume);
 }
 
 int
-__glusterd_handle_getwd (rpcsvc_request_t *req)
+__glusterd_handle_getwd(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gf1_cli_getwd_rsp     rsp = {0,};
-        glusterd_conf_t         *priv = NULL;
+    int32_t ret = -1;
+    gf1_cli_getwd_rsp rsp = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (req);
+    GF_ASSERT(req);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_GETWD_REQ_RCVD, "Received getwd req");
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_GETWD_REQ_RCVD,
+           "Received getwd req");
 
-        rsp.wd = priv->workdir;
+    rsp.wd = priv->workdir;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_getwd_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+    ret = 0;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_getwd (rpcsvc_request_t *req)
+glusterd_handle_getwd(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_getwd);
+    return glusterd_big_locked_handler(req, __glusterd_handle_getwd);
 }
 
 int
-__glusterd_handle_mount (rpcsvc_request_t *req)
+__glusterd_handle_mount(rpcsvc_request_t *req)
 {
-        gf1_cli_mount_req mnt_req = {0,};
-        gf1_cli_mount_rsp rsp     = {0,};
-        dict_t *dict              = NULL;
-        int ret                   = 0;
-        glusterd_conf_t     *priv   = NULL;
-
-        GF_ASSERT (req);
-	priv = THIS->private;
+    gf1_cli_mount_req mnt_req = {
+        0,
+    };
+    gf1_cli_mount_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int ret = 0;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(req);
+    priv = THIS->private;
+
+    ret = xdr_to_generic(req->msg[0], &mnt_req,
+                         (xdrproc_t)xdr_gf1_cli_mount_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode mount "
+               "request received");
+        req->rpc_err = GARBAGE_ARGS;
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_MOUNT_REQ_RCVD,
+           "Received mount req");
+
+    if (mnt_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &mnt_req,
-                              (xdrproc_t)xdr_gf1_cli_mount_req);
+        ret = dict_unserialize(mnt_req.dict.dict_val, mnt_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode mount "
-                        "request received");
-                req->rpc_err = GARBAGE_ARGS;
-                rsp.op_ret = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            rsp.op_ret = -1;
+            rsp.op_errno = -EINVAL;
+            goto out;
+        } else {
+            dict->extra_stdfree = mnt_req.dict.dict_val;
         }
+    }
 
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_MOUNT_REQ_RCVD,
-                "Received mount req");
-
-        if (mnt_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (mnt_req.dict.dict_val,
-                                        mnt_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        rsp.op_ret = -1;
-                        rsp.op_errno = -EINVAL;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = mnt_req.dict.dict_val;
-                }
-        }
+    synclock_unlock(&priv->big_lock);
+    rsp.op_ret = glusterd_do_mount(mnt_req.label, dict, &rsp.path,
+                                   &rsp.op_errno);
+    synclock_lock(&priv->big_lock);
 
-	synclock_unlock (&priv->big_lock);
-        rsp.op_ret = glusterd_do_mount (mnt_req.label, dict,
-                                        &rsp.path, &rsp.op_errno);
-	synclock_lock (&priv->big_lock);
+out:
+    if (!rsp.path)
+        rsp.path = gf_strdup("");
 
- out:
-        if (!rsp.path)
-                rsp.path = "";
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_mount_rsp);
+    ret = 0;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_mount_rsp);
-        ret = 0;
+    if (dict)
+        dict_unref(dict);
 
-        if (dict)
-                dict_unref (dict);
-        if (*rsp.path)
-                GF_FREE (rsp.path);
+    GF_FREE(rsp.path);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_mount (rpcsvc_request_t *req)
+glusterd_handle_mount(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_mount);
+    return glusterd_big_locked_handler(req, __glusterd_handle_mount);
 }
 
 int
-__glusterd_handle_umount (rpcsvc_request_t *req)
+__glusterd_handle_umount(rpcsvc_request_t *req)
 {
-        gf1_cli_umount_req umnt_req = {0,};
-        gf1_cli_umount_rsp rsp      = {0,};
-        char *mountbroker_root      = NULL;
-        char mntp[PATH_MAX]         = {0,};
-        char *path                  = NULL;
-        runner_t runner             = {0,};
-        int ret                     = 0;
-        xlator_t *this              = THIS;
-        gf_boolean_t dir_ok         = _gf_false;
-        char *pdir                  = NULL;
-        char *t                     = NULL;
-        glusterd_conf_t     *priv   = NULL;
-
-        GF_ASSERT (req);
-        GF_ASSERT (this);
-        priv = this->private;
-
-        ret = xdr_to_generic (req->msg[0], &umnt_req,
-                              (xdrproc_t)xdr_gf1_cli_umount_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode umount"
-                        "request");
-                req->rpc_err = GARBAGE_ARGS;
-                rsp.op_ret = -1;
-                goto out;
-        }
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_UMOUNT_REQ_RCVD,
-                "Received umount req");
-
-        if (dict_get_str (this->options, "mountbroker-root",
-                          &mountbroker_root) != 0) {
-                rsp.op_errno = ENOENT;
-                goto out;
-        }
-
-        /* check if it is allowed to umount path */
-        path = gf_strdup (umnt_req.path);
-        if (!path) {
-                rsp.op_errno = ENOMEM;
-                goto out;
-        }
-        dir_ok = _gf_false;
-        pdir = dirname (path);
-        t = strtail (pdir, mountbroker_root);
-        if (t && *t == '/') {
-                t = strtail(++t, MB_HIVE);
-                if (t && !*t)
-                        dir_ok = _gf_true;
-        }
-        GF_FREE (path);
-        if (!dir_ok) {
-                rsp.op_errno = EACCES;
-                goto out;
-        }
-
-        synclock_unlock (&priv->big_lock);
-
-        if (umnt_req.lazy) {
-                rsp.op_ret = gf_umount_lazy (this->name, umnt_req.path, 0);
-        } else {
-                runinit (&runner);
-                runner_add_args (&runner, _PATH_UMOUNT, umnt_req.path, NULL);
-                rsp.op_ret = runner_run (&runner);
+    gf1_cli_umount_req umnt_req = {
+        0,
+    };
+    gf1_cli_umount_rsp rsp = {
+        0,
+    };
+    char *mountbroker_root = NULL;
+    char mntp[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    runner_t runner = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = THIS;
+    gf_boolean_t dir_ok = _gf_false;
+    char *pdir = NULL;
+    char *t = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(req);
+    GF_ASSERT(this);
+    priv = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &umnt_req,
+                         (xdrproc_t)xdr_gf1_cli_umount_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode umount"
+               "request");
+        req->rpc_err = GARBAGE_ARGS;
+        rsp.op_ret = -1;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_UMOUNT_REQ_RCVD,
+           "Received umount req");
+
+    if (dict_get_strn(this->options, "mountbroker-root",
+                      SLEN("mountbroker-root"), &mountbroker_root) != 0) {
+        rsp.op_errno = ENOENT;
+        goto out;
+    }
+
+    /* check if it is allowed to umount path */
+    path = gf_strdup(umnt_req.path);
+    if (!path) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, NULL);
+        rsp.op_errno = ENOMEM;
+        goto out;
+    }
+    dir_ok = _gf_false;
+    pdir = dirname(path);
+    t = strtail(pdir, mountbroker_root);
+    if (t && *t == '/') {
+        t = strtail(++t, MB_HIVE);
+        if (t && !*t)
+            dir_ok = _gf_true;
+    }
+    GF_FREE(path);
+    if (!dir_ok) {
+        rsp.op_errno = EACCES;
+        goto out;
+    }
+
+    synclock_unlock(&priv->big_lock);
+
+    if (umnt_req.lazy) {
+        rsp.op_ret = gf_umount_lazy(this->name, umnt_req.path, 0);
+    } else {
+        runinit(&runner);
+        runner_add_args(&runner, _PATH_UMOUNT, umnt_req.path, NULL);
+        rsp.op_ret = runner_run(&runner);
+    }
+
+    synclock_lock(&priv->big_lock);
+    if (rsp.op_ret == 0) {
+        if (realpath(umnt_req.path, mntp))
+            sys_rmdir(mntp);
+        else {
+            rsp.op_ret = -1;
+            rsp.op_errno = errno;
         }
-
-        synclock_lock (&priv->big_lock);
-        if (rsp.op_ret == 0) {
-                if (realpath (umnt_req.path, mntp))
-                        sys_rmdir (mntp);
-                else {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = errno;
-                }
-                if (sys_unlink (umnt_req.path) != 0) {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = errno;
-                }
+        if (sys_unlink(umnt_req.path) != 0) {
+            rsp.op_ret = -1;
+            rsp.op_errno = errno;
         }
+    }
 
- out:
-        if (rsp.op_errno)
-                rsp.op_ret = -1;
+out:
+    if (rsp.op_errno)
+        rsp.op_ret = -1;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_umount_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_umount_rsp);
+    ret = 0;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_umount (rpcsvc_request_t *req)
+glusterd_handle_umount(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_umount);
+    return glusterd_big_locked_handler(req, __glusterd_handle_umount);
 }
 
 int
-glusterd_friend_remove (uuid_t uuid, char *hostname)
+glusterd_friend_remove(uuid_t uuid, char *hostname)
 {
-        int                           ret = -1;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (uuid, hostname);
-        if (peerinfo == NULL) {
-                rcu_read_unlock ();
-                goto out;
-        }
-
-        ret = glusterd_friend_remove_cleanup_vols (peerinfo->uuid);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOL_CLEANUP_FAIL, "Volumes cleanup failed");
-        rcu_read_unlock ();
-        /* Giving up the critical section here as glusterd_peerinfo_cleanup must
-         * be called from outside a critical section
-         */
-        ret = glusterd_peerinfo_cleanup (peerinfo);
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(uuid, hostname);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK;
+        goto out;
+    }
+
+    ret = glusterd_friend_remove_cleanup_vols(peerinfo->uuid);
+    RCU_READ_UNLOCK;
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_CLEANUP_FAIL,
+               "Volumes cleanup failed");
+    /* Giving up the critical section here as glusterd_peerinfo_cleanup must
+     * be called from outside a critical section
+     */
+    ret = glusterd_peerinfo_cleanup(peerinfo);
 out:
-        gf_msg_debug (THIS->name, 0, "returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "returning %d", ret);
+    /* coverity[LOCK] */
+    return ret;
 }
 
 int
-glusterd_rpc_create (struct rpc_clnt **rpc,
-                     dict_t *options,
-                     rpc_clnt_notify_t notify_fn,
-                     void *notify_data)
+glusterd_rpc_create(struct rpc_clnt **rpc, dict_t *options,
+                    rpc_clnt_notify_t notify_fn, void *notify_data,
+                    gf_boolean_t force)
 {
-        struct rpc_clnt         *new_rpc = NULL;
-        int                     ret = -1;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (options);
-
-        /* TODO: is 32 enough? or more ? */
-        new_rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!new_rpc)
-                goto out;
-
-        ret = rpc_clnt_register_notify (new_rpc, notify_fn, notify_data);
-        *rpc = new_rpc;
-        if (ret)
-                goto out;
-        ret = rpc_clnt_start (new_rpc);
+    struct rpc_clnt *new_rpc = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(options);
+    GF_VALIDATE_OR_GOTO(this->name, rpc, out);
+
+    if (force && rpc && *rpc) {
+        (void)rpc_clnt_unref(*rpc);
+        *rpc = NULL;
+    }
+
+    /* TODO: is 32 enough? or more ? */
+    new_rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!new_rpc)
+        goto out;
+
+    ret = rpc_clnt_register_notify(new_rpc, notify_fn, notify_data);
+    if (ret)
+        goto out;
+    ret = rpc_clnt_start(new_rpc);
 out:
-        if (ret) {
-                if (new_rpc) {
-                        (void) rpc_clnt_unref (new_rpc);
-                }
+    if (ret) {
+        if (new_rpc) {
+            (void)rpc_clnt_unref(new_rpc);
         }
+    } else {
+        *rpc = new_rpc;
+    }
 
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_transport_keepalive_options_get (int *interval, int *time,
-                                          int *timeout)
+glusterd_transport_inet_options_build(dict_t *dict, const char *hostname,
+                                      int port, char *af)
 {
-        int     ret = 0;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_int32 (this->options,
-                              "transport.socket.keepalive-interval",
-                              interval);
-        ret = dict_get_int32 (this->options,
-                              "transport.socket.keepalive-time",
-                              time);
-        ret = dict_get_int32 (this->options,
-                              "transport.tcp-user-timeout",
-                              timeout);
-        return 0;
+    xlator_t *this = NULL;
+    int32_t interval = -1;
+    int32_t time = -1;
+    int32_t timeout = -1;
+    int ret = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(hostname);
+
+    if (!port)
+        port = GLUSTERD_DEFAULT_PORT;
+
+    /* Build default transport options */
+    ret = rpc_transport_inet_options_build(dict, hostname, port, af);
+    if (ret)
+        goto out;
+
+    /* Set frame-timeout to 10mins. Default timeout of 30 mins is too long
+     * when compared to 2 mins for cli timeout. This ensures users don't
+     * wait too long after cli timesout before being able to resume normal
+     * operations
+     */
+    ret = dict_set_int32n(dict, "frame-timeout", SLEN("frame-timeout"), 600);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set frame-timeout");
+        goto out;
+    }
+
+    /* Set keepalive options */
+    ret = dict_get_int32n(this->options, "transport.socket.keepalive-interval",
+                          SLEN("transport.socket.keepalive-interval"),
+                          &interval);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get socket keepalive-interval");
+    }
+    ret = dict_get_int32n(this->options, "transport.socket.keepalive-time",
+                          SLEN("transport.socket.keepalive-time"), &time);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get socket keepalive-time");
+    }
+    ret = dict_get_int32n(this->options, "transport.tcp-user-timeout",
+                          SLEN("transport.tcp-user-timeout"), &timeout);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get tcp-user-timeout");
+    }
+
+    if ((interval > 0) || (time > 0))
+        ret = rpc_transport_keepalive_options_set(dict, interval, time,
+                                                  timeout);
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
-                                       int port)
+glusterd_friend_rpc_create(xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                           glusterd_peerctx_args_t *args)
 {
-        dict_t  *dict = NULL;
-        int32_t interval = -1;
-        int32_t time     = -1;
-        int32_t timeout  = -1;
-        int     ret = 0;
-
-        GF_ASSERT (options);
-        GF_ASSERT (hostname);
-
-        if (!port)
-                port = GLUSTERD_DEFAULT_PORT;
+    dict_t *options = NULL;
+    int ret = -1;
+    glusterd_peerctx_t *peerctx = NULL;
+    data_t *data = NULL;
+    char *af = NULL;
+
+    peerctx = GF_CALLOC(1, sizeof(*peerctx), gf_gld_mt_peerctx_t);
+    if (!peerctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    options = dict_new();
+    if (!options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    if (args)
+        peerctx->args = *args;
+
+    gf_uuid_copy(peerctx->peerid, peerinfo->uuid);
+    peerctx->peername = gf_strdup(peerinfo->hostname);
+    peerctx->peerinfo_gen = peerinfo->generation; /* A peerinfos generation
+                                                     number can be used to
+                                                     uniquely identify a
+                                                     peerinfo */
+
+    ret = dict_get_str(this->options, "transport.address-family", &af);
+    if (ret)
+        gf_log(this->name, GF_LOG_TRACE,
+               "option transport.address-family is not set in xlator options");
+    ret = glusterd_transport_inet_options_build(options, peerinfo->hostname,
+                                                peerinfo->port, af);
+    if (ret)
+        goto out;
+
+    /*
+     * For simulated multi-node testing, we need to make sure that we
+     * create our RPC endpoint with the same address that the peer would
+     * use to reach us.
+     */
+
+    if (this->options) {
+        data = dict_getn(this->options, "transport.socket.bind-address",
+                         SLEN("transport.socket.bind-address"));
+        if (data) {
+            ret = dict_set_sizen(options, "transport.socket.source-addr", data);
+        }
+        data = dict_getn(this->options, "ping-timeout", SLEN("ping-timeout"));
+        if (data) {
+            ret = dict_set_sizen(options, "ping-timeout", data);
+        }
+    }
+
+    /* Enable encryption for the client connection if management encryption
+     * is enabled
+     */
+    if (this->ctx->secure_mgmt) {
+        ret = dict_set_nstrn(options, "transport.socket.ssl-enabled",
+                             SLEN("transport.socket.ssl-enabled"), "on",
+                             SLEN("on"));
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set ssl-enabled in dict");
+            goto out;
+        }
+
+        this->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    ret = glusterd_rpc_create(&peerinfo->rpc, options, glusterd_peer_rpc_notify,
+                              peerctx, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
+               "failed to create rpc for"
+               " peer %s",
+               peerinfo->hostname);
+        gf_event(EVENT_PEER_RPC_CREATE_FAILED, "peer=%s", peerinfo->hostname);
+        goto out;
+    }
+    peerctx = NULL;
+    ret = 0;
+out:
+    if (options)
+        dict_unref(options);
 
-        /* Build default transport options */
-        ret = rpc_transport_inet_options_build (&dict, hostname, port);
-        if (ret)
-                goto out;
+    GF_FREE(peerctx);
+    return ret;
+}
 
-        /* Set frame-timeout to 10mins. Default timeout of 30 mins is too long
-         * when compared to 2 mins for cli timeout. This ensures users don't
-         * wait too long after cli timesout before being able to resume normal
-         * operations
-         */
-        ret = dict_set_int32 (dict, "frame-timeout", 600);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set frame-timeout");
-                goto out;
+int
+glusterd_friend_add(const char *hoststr, int port,
+                    glusterd_friend_sm_state_t state, uuid_t *uuid,
+                    glusterd_peerinfo_t **friend, gf_boolean_t restore,
+                    glusterd_peerctx_args_t *args)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(hoststr);
+    GF_ASSERT(friend);
+
+    *friend = glusterd_peerinfo_new(state, uuid, hoststr, port);
+    if (*friend == NULL) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADD_FAIL, NULL);
+        goto out;
+    }
+
+    /*
+     * We can't add to the list after calling glusterd_friend_rpc_create,
+     * even if it succeeds, because by then the callback to take it back
+     * off and free might have happened already (notably in the case of an
+     * invalid peer name).  That would mean we're adding something that had
+     * just been free, and we're likely to crash later.
+     */
+    cds_list_add_tail_rcu(&(*friend)->uuid_list, &conf->peers);
+
+    // restore needs to first create the list of peers, then create rpcs
+    // to keep track of quorum in race-free manner. In restore for each peer
+    // rpc-create calls rpc_notify when the friend-list is partially
+    // constructed, leading to wrong quorum calculations.
+    if (!restore) {
+        ret = glusterd_store_peerinfo(*friend);
+        if (ret == 0) {
+            ret = glusterd_friend_rpc_create(this, *friend, args);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                   "Failed to store peerinfo");
+            gf_event(EVENT_PEER_STORE_FAILURE, "peer=%s", (*friend)->hostname);
         }
+    }
 
-        /* Set keepalive options */
-        glusterd_transport_keepalive_options_get (&interval, &time, &timeout);
+    if (ret) {
+        (void)glusterd_peerinfo_cleanup(*friend);
+        *friend = NULL;
+    }
 
-        if ((interval > 0) || (time > 0))
-                ret = rpc_transport_keepalive_options_set (dict, interval,
-                                                           time, timeout);
-        *options = dict;
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONNECT_RETURNED,
+           "connect returned %d", ret);
+    return ret;
 }
 
+/* glusterd_friend_add_from_peerinfo() adds a new peer into the local friends
+ * list from a pre created @peerinfo object. It otherwise works similarly to
+ * glusterd_friend_add()
+ */
 int
-glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
-                            glusterd_peerctx_args_t *args)
+glusterd_friend_add_from_peerinfo(glusterd_peerinfo_t *friend,
+                                  gf_boolean_t restore,
+                                  glusterd_peerctx_args_t *args)
 {
-        dict_t                   *options = NULL;
-        int                       ret     = -1;
-        glusterd_peerctx_t       *peerctx = NULL;
-        data_t                   *data    = NULL;
-
-        peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
-        if (!peerctx)
-                goto out;
-
-        if (args)
-                peerctx->args = *args;
-
-        gf_uuid_copy (peerctx->peerid, peerinfo->uuid);
-        peerctx->peername = gf_strdup (peerinfo->hostname);
-        peerctx->peerinfo_gen = peerinfo->generation; /* A peerinfos generation
-                                                         number can be used to
-                                                         uniquely identify a
-                                                         peerinfo */
-
-        ret = glusterd_transport_inet_options_build (&options,
-                                                     peerinfo->hostname,
-                                                     peerinfo->port);
-        if (ret)
-                goto out;
-
-        /*
-         * For simulated multi-node testing, we need to make sure that we
-         * create our RPC endpoint with the same address that the peer would
-         * use to reach us.
-         */
-        if (this->options) {
-                data = dict_get(this->options,"transport.socket.bind-address");
-                if (data) {
-                        ret = dict_set(options,
-                                       "transport.socket.source-addr",data);
-                }
-                data = dict_get(this->options,"ping-timeout");
-                if (data) {
-                        ret = dict_set(options,
-                                       "ping-timeout",data);
-                }
-        }
-
-        /* Enable encryption for the client connection if management encryption
-         * is enabled
-         */
-        if (this->ctx->secure_mgmt) {
-                ret = dict_set_str (options, "transport.socket.ssl-enabled",
-                                    "on");
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "failed to set ssl-enabled in dict");
-                        goto out;
-                }
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_VALIDATE_OR_GOTO(this->name, (friend != NULL), out);
+
+    /*
+     * We can't add to the list after calling glusterd_friend_rpc_create,
+     * even if it succeeds, because by then the callback to take it back
+     * off and free might have happened already (notably in the case of an
+     * invalid peer name).  That would mean we're adding something that had
+     * just been free, and we're likely to crash later.
+     */
+    cds_list_add_tail_rcu(&friend->uuid_list, &conf->peers);
+
+    // restore needs to first create the list of peers, then create rpcs
+    // to keep track of quorum in race-free manner. In restore for each peer
+    // rpc-create calls rpc_notify when the friend-list is partially
+    // constructed, leading to wrong quorum calculations.
+    if (!restore) {
+        ret = glusterd_store_peerinfo(friend);
+        if (ret == 0) {
+            ret = glusterd_friend_rpc_create(this, friend, args);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                   "Failed to store peerinfo");
+            gf_event(EVENT_PEER_STORE_FAILURE, "peer=%s", friend->hostname);
         }
+    }
 
-        ret = glusterd_rpc_create (&peerinfo->rpc, options,
-                                   glusterd_peer_rpc_notify, peerctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RPC_CREATE_FAIL,
-                        "failed to create rpc for"
-                        " peer %s", peerinfo->hostname);
-                goto out;
-        }
-        peerctx = NULL;
-        ret = 0;
 out:
-        GF_FREE (peerctx);
-        return ret;
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONNECT_RETURNED,
+           "connect returned %d", ret);
+    return ret;
 }
 
 int
-glusterd_friend_add (const char *hoststr, int port,
-                     glusterd_friend_sm_state_t state,
-                     uuid_t *uuid,
-                     glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore,
-                     glusterd_peerctx_args_t *args)
+glusterd_probe_begin(rpcsvc_request_t *req, const char *hoststr, int port,
+                     dict_t *dict, int *op_errno)
 {
-        int                     ret = 0;
-        xlator_t               *this = NULL;
-        glusterd_conf_t        *conf = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_args_t args = {0};
+    glusterd_friend_sm_event_t *event = NULL;
 
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-        GF_ASSERT (hoststr);
-        GF_ASSERT (friend);
+    GF_ASSERT(hoststr);
 
-        *friend = glusterd_peerinfo_new (state, uuid, hoststr, port);
-        if (*friend == NULL) {
-                ret = -1;
-                goto out;
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(NULL, hoststr);
+
+    if (peerinfo == NULL) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PEER_NOT_FOUND,
+               "Unable to find peerinfo"
+               " for host: %s (%d)",
+               hoststr, port);
+        args.mode = GD_MODE_ON;
+        args.req = req;
+        args.dict = dict;
+        ret = glusterd_friend_add(hoststr, port, GD_FRIEND_STATE_DEFAULT, NULL,
+                                  &peerinfo, 0, &args);
+        if ((!ret) && (!peerinfo->connected)) {
+            ret = GLUSTERD_CONNECTION_AWAITED;
         }
 
-        /*
-         * We can't add to the list after calling glusterd_friend_rpc_create,
-         * even if it succeeds, because by then the callback to take it back
-         * off and free might have happened already (notably in the case of an
-         * invalid peer name).  That would mean we're adding something that had
-         * just been free, and we're likely to crash later.
-         */
-        cds_list_add_tail_rcu (&(*friend)->uuid_list, &conf->peers);
-
-        //restore needs to first create the list of peers, then create rpcs
-        //to keep track of quorum in race-free manner. In restore for each peer
-        //rpc-create calls rpc_notify when the friend-list is partially
-        //constructed, leading to wrong quorum calculations.
-        if (!restore) {
-                ret = glusterd_store_peerinfo (*friend);
-                if (ret == 0) {
-                        ret = glusterd_friend_rpc_create (this, *friend, args);
-                }
-                else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_PEERINFO_CREATE_FAIL,
-                                "Failed to store peerinfo");
-                }
+    } else if (peerinfo->connected &&
+               (GD_FRIEND_STATE_BEFRIENDED == peerinfo->state.state)) {
+        if (peerinfo->detaching) {
+            ret = -1;
+            if (op_errno)
+                *op_errno = GF_PROBE_FRIEND_DETACHING;
+            goto out;
         }
+        ret = glusterd_peer_hostname_update(peerinfo, hoststr, _gf_false);
+        if (ret)
+            goto out;
+        // Injecting a NEW_NAME event to update cluster
+        ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_NEW_NAME, &event);
+        if (!ret) {
+            event->peername = gf_strdup(peerinfo->hostname);
+            gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-        if (ret) {
-                (void) glusterd_peerinfo_cleanup (*friend);
-                *friend = NULL;
+            ret = glusterd_friend_sm_inject_event(event);
+            glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_SUCCESS, NULL,
+                                         (char *)hoststr, port, dict);
         }
+    } else {
+        glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_FRIEND, NULL,
+                                     (char *)hoststr, port, dict);
+        ret = 0;
+    }
 
 out:
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_CONNECT_RETURNED, "connect returned %d", ret);
-        return ret;
+    RCU_READ_UNLOCK;
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
-/* glusterd_friend_add_from_peerinfo() adds a new peer into the local friends
- * list from a pre created @peerinfo object. It otherwise works similarly to
- * glusterd_friend_add()
- */
 int
-glusterd_friend_add_from_peerinfo (glusterd_peerinfo_t *friend,
-                                   gf_boolean_t restore,
-                                   glusterd_peerctx_args_t *args)
+glusterd_deprobe_begin(rpcsvc_request_t *req, const char *hoststr, int port,
+                       uuid_t uuid, dict_t *dict, int *op_errno)
 {
-        int                     ret = 0;
-        xlator_t               *this = NULL;
-        glusterd_conf_t        *conf = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_probe_ctx_t *ctx = NULL;
 
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
+    GF_ASSERT(hoststr);
+    GF_ASSERT(req);
 
-        GF_VALIDATE_OR_GOTO (this->name, (friend != NULL), out);
+    RCU_READ_LOCK;
 
-        /*
-         * We can't add to the list after calling glusterd_friend_rpc_create,
-         * even if it succeeds, because by then the callback to take it back
-         * off and free might have happened already (notably in the case of an
-         * invalid peer name).  That would mean we're adding something that had
-         * just been free, and we're likely to crash later.
-         */
-        cds_list_add_tail_rcu (&friend->uuid_list, &conf->peers);
-
-        //restore needs to first create the list of peers, then create rpcs
-        //to keep track of quorum in race-free manner. In restore for each peer
-        //rpc-create calls rpc_notify when the friend-list is partially
-        //constructed, leading to wrong quorum calculations.
-        if (!restore) {
-                ret = glusterd_store_peerinfo (friend);
-                if (ret == 0) {
-                        ret = glusterd_friend_rpc_create (this, friend, args);
-                }
-                else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_PEERINFO_CREATE_FAIL,
-                                "Failed to store peerinfo");
-                }
-        }
+    peerinfo = glusterd_peerinfo_find(uuid, hoststr);
+    if (peerinfo == NULL) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PEER_NOT_FOUND,
+               "Unable to find peerinfo"
+               " for host: %s %d",
+               hoststr, port);
+        goto out;
+    }
+
+    if (!peerinfo->rpc) {
+        // handle this case
+        goto out;
+    }
+
+    if (peerinfo->detaching) {
+        ret = -1;
+        if (op_errno)
+            *op_errno = GF_DEPROBE_FRIEND_DETACHING;
+        goto out;
+    }
 
-out:
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_CONNECT_RETURNED,
-                "connect returned %d", ret);
-        return ret;
-}
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
+                                       &event);
 
-int
-glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                      dict_t *dict, int *op_errno)
-{
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_peerctx_args_t         args = {0};
-        glusterd_friend_sm_event_t      *event = NULL;
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get new event");
+        goto out;
+    }
 
-        GF_ASSERT (hoststr);
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (NULL, hoststr);
+    if (!ctx) {
+        goto out;
+    }
 
-        if (peerinfo == NULL) {
-                gf_msg ("glusterd", GF_LOG_INFO, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Unable to find peerinfo"
-                        " for host: %s (%d)", hoststr, port);
-                args.mode = GD_MODE_ON;
-                args.req  = req;
-                args.dict = dict;
-                ret = glusterd_friend_add (hoststr, port,
-                                           GD_FRIEND_STATE_DEFAULT,
-                                           NULL, &peerinfo, 0, &args);
-                if ((!ret) && (!peerinfo->connected)) {
-                        ret = GLUSTERD_CONNECTION_AWAITED;
-                }
+    ctx->hostname = gf_strdup(hoststr);
+    ctx->port = port;
+    ctx->req = req;
+    ctx->dict = dict;
 
-        } else if (peerinfo->connected &&
-                   (GD_FRIEND_STATE_BEFRIENDED == peerinfo->state.state)) {
-                if (peerinfo->detaching) {
-                        ret = -1;
-                        if (op_errno)
-                                *op_errno = GF_PROBE_FRIEND_DETACHING;
-                        goto out;
-                }
-                ret = glusterd_peer_hostname_update (peerinfo, hoststr,
-                                                     _gf_false);
-                if (ret)
-                        goto out;
-                // Injecting a NEW_NAME event to update cluster
-                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_NEW_NAME,
-                                                    &event);
-                if (!ret) {
-                        event->peername = gf_strdup (peerinfo->hostname);
-                        gf_uuid_copy (event->peerid, peerinfo->uuid);
+    event->ctx = ctx;
 
-                        ret = glusterd_friend_sm_inject_event (event);
-                        glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_SUCCESS,
-                                                      NULL, (char*)hoststr,
-                                                      port, dict);
-                }
-        } else {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
-                                              (char*)hoststr, port, dict);
-                ret = 0;
-        }
+    event->peername = gf_strdup(hoststr);
+    gf_uuid_copy(event->peerid, uuid);
+
+    ret = glusterd_friend_sm_inject_event(event);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject event %d, "
+               "ret = %d",
+               event->event, ret);
+        goto out;
+    }
+    peerinfo->detaching = _gf_true;
 
 out:
-        rcu_read_unlock ();
-        gf_msg_debug ("glusterd", 0, "returning %d", ret);
-        return ret;
+    RCU_READ_UNLOCK;
+    return ret;
 }
 
 int
-glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                        uuid_t uuid, dict_t *dict, int *op_errno)
+glusterd_xfer_friend_remove_resp(rpcsvc_request_t *req, char *hostname,
+                                 int port)
 {
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_probe_ctx_t            *ctx = NULL;
-
-        GF_ASSERT (hoststr);
-        GF_ASSERT (req);
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(hostname);
+
+    rsp.op_ret = 0;
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_uuid_copy(rsp.uuid, MY_UUID);
+    rsp.hostname = hostname;
+    rsp.port = port;
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Responded to %s (%d), ret: %d", hostname, port, ret);
+    return ret;
+}
 
-        rcu_read_lock ();
+int
+glusterd_xfer_friend_add_resp(rpcsvc_request_t *req, char *myhostname,
+                              char *remote_hostname, int port, int32_t op_ret,
+                              int32_t op_errno)
+{
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(myhostname);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_uuid_copy(rsp.uuid, MY_UUID);
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+    rsp.hostname = gf_strdup(myhostname);
+    rsp.port = port;
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Responded to %s (%d), ret: %d, op_ret: %d", remote_hostname, port,
+           ret, op_ret);
+    GF_FREE(rsp.hostname);
+    return ret;
+}
 
-        peerinfo = glusterd_peerinfo_find (uuid, hoststr);
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_msg ("glusterd", GF_LOG_INFO, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Unable to find peerinfo"
-                        " for host: %s %d", hoststr, port);
-                goto out;
-        }
+static void
+set_probe_error_str(int op_ret, int op_errno, char *op_errstr, char *errstr,
+                    size_t len, char *hostname, int port)
+{
+    if ((op_errstr) && (strcmp(op_errstr, ""))) {
+        snprintf(errstr, len, "%s", op_errstr);
+        return;
+    }
+
+    if (!op_ret) {
+        switch (op_errno) {
+            case GF_PROBE_LOCALHOST:
+                snprintf(errstr, len,
+                         "Probe on localhost not "
+                         "needed");
+                break;
 
-        if (!peerinfo->rpc) {
-                //handle this case
-                goto out;
-        }
+            case GF_PROBE_FRIEND:
+                snprintf(errstr, len,
+                         "Host %s port %d already"
+                         " in peer list",
+                         hostname, port);
+                break;
 
-        if (peerinfo->detaching) {
-                ret = -1;
-                if (op_errno)
-                        *op_errno = GF_DEPROBE_FRIEND_DETACHING;
-                goto out;
+            case GF_PROBE_FRIEND_DETACHING:
+                snprintf(errstr, len,
+                         "Peer is already being "
+                         "detached from cluster.\n"
+                         "Check peer status by running "
+                         "gluster peer status");
+                break;
+            default:
+                if (op_errno != 0)
+                    snprintf(errstr, len,
+                             "Probe returned "
+                             "with %s",
+                             strerror(op_errno));
+                break;
         }
+    } else {
+        switch (op_errno) {
+            case GF_PROBE_ANOTHER_CLUSTER:
+                snprintf(errstr, len,
+                         "%s is either already "
+                         "part of another cluster or having "
+                         "volumes configured",
+                         hostname);
+                break;
 
-        ret = glusterd_friend_sm_new_event
-                (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND, &event);
-
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_NEW_GET_FAIL,
-                        "Unable to get new event");
-                goto out;
-        }
+            case GF_PROBE_VOLUME_CONFLICT:
+                snprintf(errstr, len,
+                         "At least one volume on "
+                         "%s conflicts with existing volumes "
+                         "in the cluster",
+                         hostname);
+                break;
 
-        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+            case GF_PROBE_UNKNOWN_PEER:
+                snprintf(errstr, len,
+                         "%s responded with "
+                         "'unknown peer' error, this could "
+                         "happen if %s doesn't have localhost "
+                         "in its peer database",
+                         hostname, hostname);
+                break;
 
-        if (!ctx) {
-                goto out;
-        }
+            case GF_PROBE_ADD_FAILED:
+                snprintf(errstr, len,
+                         "Failed to add peer "
+                         "information on %s",
+                         hostname);
+                break;
 
-        ctx->hostname = gf_strdup (hoststr);
-        ctx->port = port;
-        ctx->req = req;
-        ctx->dict = dict;
+            case GF_PROBE_SAME_UUID:
+                snprintf(errstr, len,
+                         "Peer uuid (host %s) is "
+                         "same as local uuid",
+                         hostname);
+                break;
 
-        event->ctx = ctx;
+            case GF_PROBE_QUORUM_NOT_MET:
+                snprintf(errstr, len,
+                         "Cluster quorum is not "
+                         "met. Changing peers is not allowed "
+                         "in this state");
+                break;
 
-        event->peername = gf_strdup (hoststr);
-        gf_uuid_copy (event->peerid, uuid);
+            case GF_PROBE_MISSED_SNAP_CONFLICT:
+                snprintf(errstr, len,
+                         "Failed to update "
+                         "list of missed snapshots from "
+                         "peer %s",
+                         hostname);
+                break;
 
-        ret = glusterd_friend_sm_inject_event (event);
+            case GF_PROBE_SNAP_CONFLICT:
+                snprintf(errstr, len,
+                         "Conflict in comparing "
+                         "list of snapshots from "
+                         "peer %s",
+                         hostname);
+                break;
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL, "Unable to inject event %d, "
-                        "ret = %d", event->event, ret);
-                goto out;
+            default:
+                snprintf(errstr, len,
+                         "Probe returned with "
+                         "%s",
+                         strerror(op_errno));
+                break;
         }
-        peerinfo->detaching = _gf_true;
-
-out:
-        rcu_read_unlock ();
-        return ret;
+    }
 }
 
-
 int
-glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int port)
+glusterd_xfer_cli_probe_resp(rpcsvc_request_t *req, int32_t op_ret,
+                             int32_t op_errno, char *op_errstr, char *hostname,
+                             int port, dict_t *dict)
 {
-        gd1_mgmt_friend_rsp  rsp = {{0}, };
-        int32_t              ret = -1;
-        xlator_t             *this = NULL;
-        glusterd_conf_t      *conf = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int32_t ret = -1;
+    char errstr[2048] = {
+        0,
+    };
+    char *cmd_str = NULL;
+    xlator_t *this = THIS;
+
+    GF_ASSERT(req);
+    GF_ASSERT(this);
+
+    (void)set_probe_error_str(op_ret, op_errno, op_errstr, errstr,
+                              sizeof(errstr), hostname, port);
+
+    if (dict) {
+        ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CMDSTR_NOTFOUND_IN_DICT,
+                   "Failed to get "
+                   "command string");
+    }
 
-        GF_ASSERT (hostname);
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+    rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
 
-        rsp.op_ret = 0;
-        this = THIS;
-        GF_ASSERT (this);
+    gf_cmd_log("", "%s : %s %s %s", cmd_str, (op_ret) ? "FAILED" : "SUCCESS",
+               (errstr[0] != '\0') ? ":" : " ",
+               (errstr[0] != '\0') ? errstr : " ");
 
-        conf = this->private;
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gf_cli_rsp);
 
-        gf_uuid_copy (rsp.uuid, MY_UUID);
-        rsp.hostname = hostname;
-        rsp.port = port;
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+    if (dict)
+        dict_unref(dict);
+    gf_msg_debug(this->name, 0, "Responded to CLI, ret: %d", ret);
 
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_RESPONSE_INFO,
-                "Responded to %s (%d), ret: %d", hostname, port, ret);
-        return ret;
+    return ret;
 }
 
-
-int
-glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
-                               char *remote_hostname, int port, int32_t op_ret,
-                               int32_t op_errno)
+static void
+set_deprobe_error_str(int op_ret, int op_errno, char *op_errstr, char *errstr,
+                      size_t len, char *hostname)
 {
-        gd1_mgmt_friend_rsp  rsp = {{0}, };
-        int32_t              ret = -1;
-        xlator_t             *this = NULL;
-        glusterd_conf_t      *conf = NULL;
-
-        GF_ASSERT (myhostname);
-
-        this = THIS;
-        GF_ASSERT (this);
+    if ((op_errstr) && (strcmp(op_errstr, ""))) {
+        snprintf(errstr, len, "%s", op_errstr);
+        return;
+    }
+
+    if (op_ret) {
+        switch (op_errno) {
+            case GF_DEPROBE_LOCALHOST:
+                snprintf(errstr, len, "%s is localhost", hostname);
+                break;
 
-        conf = this->private;
+            case GF_DEPROBE_NOT_FRIEND:
+                snprintf(errstr, len,
+                         "%s is not part of "
+                         "cluster",
+                         hostname);
+                break;
 
-        gf_uuid_copy (rsp.uuid, MY_UUID);
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
-        rsp.hostname = gf_strdup (myhostname);
-        rsp.port = port;
+            case GF_DEPROBE_BRICK_EXIST:
+                snprintf(errstr, len,
+                         "Peer %s hosts one or more bricks. If the peer is in "
+                         "not recoverable state then use either replace-brick "
+                         "or remove-brick command with force to remove all "
+                         "bricks from the peer and attempt the peer detach "
+                         "again.",
+                         hostname);
+                break;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+            case GF_DEPROBE_SNAP_BRICK_EXIST:
+                snprintf(errstr, len,
+                         "%s is part of existing "
+                         "snapshot. Remove those snapshots "
+                         "before proceeding ",
+                         hostname);
+                break;
 
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_RESPONSE_INFO,
-                "Responded to %s (%d), ret: %d, op_ret: %d", remote_hostname,
-                port, ret, op_ret);
-        GF_FREE (rsp.hostname);
-        return ret;
-}
+            case GF_DEPROBE_FRIEND_DOWN:
+                snprintf(errstr, len,
+                         "One of the peers is "
+                         "probably down. Check with "
+                         "'peer status'");
+                break;
 
-static void
-set_probe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
-                     size_t len, char *hostname, int port)
-{
-        if ((op_errstr) && (strcmp (op_errstr, ""))) {
-                snprintf (errstr, len, "%s", op_errstr);
-                return;
-        }
+            case GF_DEPROBE_QUORUM_NOT_MET:
+                snprintf(errstr, len,
+                         "Cluster quorum is not "
+                         "met. Changing peers is not allowed "
+                         "in this state");
+                break;
 
-        if (!op_ret) {
-                switch (op_errno) {
-                        case GF_PROBE_LOCALHOST:
-                                snprintf (errstr, len, "Probe on localhost not "
-                                          "needed");
-                                break;
-
-                        case GF_PROBE_FRIEND:
-                                snprintf (errstr, len, "Host %s port %d already"
-                                          " in peer list", hostname, port);
-                                break;
-
-                        case GF_PROBE_FRIEND_DETACHING:
-                                snprintf (errstr, len, "Peer is already being "
-                                          "detached from cluster.\n"
-                                          "Check peer status by running "
-                                          "gluster peer status");
-                                break;
-                        default:
-                                if (op_errno != 0)
-                                        snprintf (errstr, len, "Probe returned "
-                                                  "with %s",
-                                                  strerror (op_errno));
-                                break;
-                }
-        } else {
-                switch (op_errno) {
-                        case GF_PROBE_ANOTHER_CLUSTER:
-                                snprintf (errstr, len, "%s is either already "
-                                          "part of another cluster or having "
-                                          "volumes configured", hostname);
-                                break;
-
-                        case GF_PROBE_VOLUME_CONFLICT:
-                                snprintf (errstr, len, "Atleast one volume on "
-                                          "%s conflicts with existing volumes "
-                                          "in the cluster", hostname);
-                                break;
-
-                        case GF_PROBE_UNKNOWN_PEER:
-                                snprintf (errstr, len, "%s responded with "
-                                          "'unknown peer' error, this could "
-                                          "happen if %s doesn't have localhost "
-                                          "in its peer database", hostname,
-                                          hostname);
-                                break;
-
-                        case GF_PROBE_ADD_FAILED:
-                                snprintf (errstr, len, "Failed to add peer "
-                                          "information on %s", hostname);
-                                break;
-
-                        case GF_PROBE_SAME_UUID:
-                                snprintf (errstr, len, "Peer uuid (host %s) is "
-                                          "same as local uuid", hostname);
-                                break;
-
-                        case GF_PROBE_QUORUM_NOT_MET:
-                                snprintf (errstr, len, "Cluster quorum is not "
-                                          "met. Changing peers is not allowed "
-                                          "in this state");
-                                break;
-
-                        case GF_PROBE_MISSED_SNAP_CONFLICT:
-                                snprintf (errstr, len, "Failed to update "
-                                          "list of missed snapshots from "
-                                          "peer %s", hostname);
-                                break;
-
-                        case GF_PROBE_SNAP_CONFLICT:
-                                snprintf (errstr, len, "Conflict in comparing "
-                                          "list of snapshots from "
-                                          "peer %s", hostname);
-                                break;
-
-                        default:
-                                snprintf (errstr, len, "Probe returned with "
-                                          "%s", strerror (op_errno));
-                                break;
-                }
+            case GF_DEPROBE_FRIEND_DETACHING:
+                snprintf(errstr, len,
+                         "Peer is already being "
+                         "detached from cluster.\n"
+                         "Check peer status by running "
+                         "gluster peer status");
+                break;
+            default:
+                snprintf(errstr, len,
+                         "Detach returned with "
+                         "%s",
+                         strerror(op_errno));
+                break;
         }
+    }
 }
 
 int
-glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *op_errstr, char *hostname,
-                              int port, dict_t *dict)
+glusterd_xfer_cli_deprobe_resp(rpcsvc_request_t *req, int32_t op_ret,
+                               int32_t op_errno, char *op_errstr,
+                               char *hostname, dict_t *dict)
 {
-        gf_cli_rsp           rsp = {0,};
-        int32_t              ret = -1;
-        char        errstr[2048] = {0,};
-        char            *cmd_str = NULL;
-        xlator_t           *this = THIS;
-
-        GF_ASSERT (req);
-        GF_ASSERT (this);
-
-        (void) set_probe_error_str (op_ret, op_errno, op_errstr, errstr,
-                                    sizeof (errstr), hostname, port);
-
-        if (dict) {
-                ret = dict_get_str (dict, "cmd-str", &cmd_str);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_CMDSTR_NOTFOUND_IN_DICT, "Failed to get "
-                                "command string");
-        }
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int32_t ret = -1;
+    char *cmd_str = NULL;
+    char errstr[2048] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    (void)set_deprobe_error_str(op_ret, op_errno, op_errstr, errstr,
+                                sizeof(errstr), hostname);
+
+    if (dict) {
+        ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_CMDSTR_NOTFOUND_IN_DICT,
+                   "Failed to get "
+                   "command string");
+    }
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
-        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+    rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
 
-        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
-                    (op_ret) ? "FAILED" : "SUCCESS",
-                    (errstr[0] != '\0') ? ":" : " ",
-                    (errstr[0] != '\0') ? errstr : " ");
+    gf_cmd_log("", "%s : %s %s %s", cmd_str, (op_ret) ? "FAILED" : "SUCCESS",
+               (errstr[0] != '\0') ? ":" : " ",
+               (errstr[0] != '\0') ? errstr : " ");
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gf_cli_rsp);
 
-        if (dict)
-                dict_unref (dict);
-        gf_msg_debug (this->name, 0, "Responded to CLI, ret: %d", ret);
+    gf_msg_debug(THIS->name, 0, "Responded to CLI, ret: %d", ret);
 
-        return ret;
+    return ret;
 }
 
-static void
-set_deprobe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
-                       size_t len, char *hostname)
+int32_t
+glusterd_list_friends(rpcsvc_request_t *req, dict_t *dict, int32_t flags)
 {
-        if ((op_errstr) && (strcmp (op_errstr, ""))) {
-                snprintf (errstr, len, "%s", op_errstr);
-                return;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
+    int32_t count = 0;
+    dict_t *friends = NULL;
+    gf1_cli_peer_list_rsp rsp = {
+        0,
+    };
+    char my_uuid_str[64] = {
+        0,
+    };
+    char key[64] = {
+        0,
+    };
+    int keylen;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    friends = dict_new();
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    /* Reset ret to 0, needed to prevent failure in case no peers exist */
+    ret = 0;
+    RCU_READ_LOCK;
+    if (!cds_list_empty(&priv->peers)) {
+        cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+        {
+            count++;
+            ret = gd_add_peer_detail_to_dict(entry, friends, count);
+            if (ret)
+                goto unlock;
         }
-
-        if (op_ret) {
-                switch (op_errno) {
-                        case GF_DEPROBE_LOCALHOST:
-                                snprintf (errstr, len, "%s is localhost",
-                                hostname);
-                                break;
-
-                        case GF_DEPROBE_NOT_FRIEND:
-                                snprintf (errstr, len, "%s is not part of "
-                                          "cluster", hostname);
-                                break;
-
-                        case GF_DEPROBE_BRICK_EXIST:
-                                snprintf (errstr, len, "Brick(s) with the peer "
-                                          "%s exist in cluster", hostname);
-                                break;
-
-                        case GF_DEPROBE_FRIEND_DOWN:
-                                snprintf (errstr, len, "One of the peers is "
-                                          "probably down. Check with "
-                                          "'peer status'");
-                                break;
-
-                        case GF_DEPROBE_QUORUM_NOT_MET:
-                                snprintf (errstr, len, "Cluster quorum is not "
-                                          "met. Changing peers is not allowed "
-                                          "in this state");
-                                break;
-
-                        case GF_DEPROBE_FRIEND_DETACHING:
-                                snprintf (errstr, len, "Peer is already being "
-                                          "detached from cluster.\n"
-                                          "Check peer status by running "
-                                          "gluster peer status");
-                                break;
-                        default:
-                                snprintf (errstr, len, "Detach returned with "
-                                          "%s", strerror (op_errno));
-                                break;
-
-                }
+    }
+unlock:
+    RCU_READ_UNLOCK;
+    if (ret)
+        goto out;
+
+    if (flags == GF_CLI_LIST_POOL_NODES) {
+        count++;
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", count);
+        uuid_utoa_r(MY_UUID, my_uuid_str);
+        ret = dict_set_strn(friends, key, keylen, my_uuid_str);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
-}
 
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", count);
+        ret = dict_set_nstrn(friends, key, keylen, "localhost",
+                             SLEN("localhost"));
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
 
-int
-glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                                int32_t op_errno, char *op_errstr,
-                                char *hostname, dict_t *dict)
-{
-        gf_cli_rsp             rsp = {0,};
-        int32_t                ret = -1;
-        char              *cmd_str = NULL;
-        char          errstr[2048] = {0,};
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", count);
+        ret = dict_set_int32n(friends, key, keylen, 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+    }
 
-        GF_ASSERT (req);
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
 
-        (void) set_deprobe_error_str (op_ret, op_errno, op_errstr, errstr,
-                                      sizeof (errstr), hostname);
+    ret = dict_allocate_and_serialize(friends, &rsp.friends.friends_val,
+                                      &rsp.friends.friends_len);
 
-        if (dict) {
-                ret = dict_get_str (dict, "cmd-str", &cmd_str);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_CMDSTR_NOTFOUND_IN_DICT, "Failed to get "
-                                "command string");
-        }
+    if (ret)
+        goto out;
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
-        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+    ret = 0;
+out:
 
-        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
-                    (op_ret) ? "FAILED" : "SUCCESS",
-                    (errstr[0] != '\0') ? ":" : " ",
-                    (errstr[0] != '\0') ? errstr : " ");
+    if (friends)
+        dict_unref(friends);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
+    rsp.op_ret = ret;
 
-        gf_msg_debug (THIS->name, 0, "Responded to CLI, ret: %d", ret);
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+    ret = 0;
+    GF_FREE(rsp.friends.friends_val);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_list_friends (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
+glusterd_get_volumes(rpcsvc_request_t *req, dict_t *dict, int32_t flags)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *entry = NULL;
-        int32_t                 count = 0;
-        dict_t                  *friends = NULL;
-        gf1_cli_peer_list_rsp   rsp = {0,};
-        char                    my_uuid_str[64] = {0,};
-        char                    key[256] = {0,};
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        friends = dict_new ();
-        if (!friends) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out of Memory");
-                goto out;
-        }
-
-        /* Reset ret to 0, needed to prevent failure incase no peers exist */
+    int32_t ret = -1;
+    int32_t ret_bkp = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *entry = NULL;
+    int32_t count = 0;
+    dict_t *volumes = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    char *volname = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    volumes = dict_new();
+    if (!volumes) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of Memory");
+        goto out;
+    }
+
+    if (cds_list_empty(&priv->volumes)) {
+        if (flags == GF_CLI_GET_VOLUME)
+            ret_bkp = -1;
         ret = 0;
-        rcu_read_lock ();
-        if (!cds_list_empty (&priv->peers)) {
-                cds_list_for_each_entry_rcu (entry, &priv->peers, uuid_list) {
-                        count++;
-                        ret = gd_add_peer_detail_to_dict (entry,
-                                                                friends, count);
-                        if (ret)
-                                goto unlock;
-                }
+        goto respond;
+    }
+    if (flags == GF_CLI_GET_VOLUME_ALL) {
+        cds_list_for_each_entry(entry, &priv->volumes, vol_list)
+        {
+            ret = glusterd_add_volume_detail_to_dict(entry, volumes, count);
+            if (ret)
+                goto respond;
+
+            count++;
         }
-unlock:
-        rcu_read_unlock ();
-        if (ret)
-                goto out;
 
-        if (flags == GF_CLI_LIST_POOL_NODES) {
-                count++;
-                snprintf (key, 256, "friend%d.uuid", count);
-                uuid_utoa_r (MY_UUID, my_uuid_str);
-                ret = dict_set_str (friends, key, my_uuid_str);
-                if (ret)
-                        goto out;
+    } else if (flags == GF_CLI_GET_NEXT_VOLUME) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-                snprintf (key, 256, "friend%d.hostname", count);
-                ret = dict_set_str (friends, key, "localhost");
-                if (ret)
-                        goto out;
+        if (ret) {
+            if (priv->volumes.next) {
+                entry = cds_list_entry(priv->volumes.next, typeof(*entry),
+                                       vol_list);
+            }
+        } else {
+            ret = glusterd_volinfo_find(volname, &entry);
+            if (ret)
+                goto respond;
+            entry = cds_list_entry(entry->vol_list.next, typeof(*entry),
+                                   vol_list);
+        }
 
-                snprintf (key, 256, "friend%d.connected", count);
-                ret = dict_set_int32 (friends, key, 1);
-                if (ret)
-                        goto out;
+        if (&entry->vol_list == &priv->volumes) {
+            goto respond;
+        } else {
+            ret = glusterd_add_volume_detail_to_dict(entry, volumes, count);
+            if (ret)
+                goto respond;
+
+            count++;
         }
+    } else if (flags == GF_CLI_GET_VOLUME) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-        ret = dict_set_int32 (friends, "count", count);
         if (ret)
-                goto out;
+            goto respond;
 
-        ret = dict_allocate_and_serialize (friends, &rsp.friends.friends_val,
-                                           &rsp.friends.friends_len);
+        ret = glusterd_volinfo_find(volname, &entry);
+        if (ret) {
+            ret_bkp = ret;
+            goto respond;
+        }
 
+        ret = glusterd_add_volume_detail_to_dict(entry, volumes, count);
         if (ret)
-                goto out;
+            goto respond;
 
-        ret = 0;
-out:
+        count++;
+    }
+
+respond:
+    ret = dict_set_int32n(volumes, "count", SLEN("count"), count);
+    if (ret)
+        goto out;
+    ret = dict_allocate_and_serialize(volumes, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
 
-        if (friends)
-                dict_unref (friends);
+    if (ret)
+        goto out;
 
+    ret = 0;
+out:
+    if (ret_bkp == -1) {
+        rsp.op_ret = ret_bkp;
+        rsp.op_errstr = "Volume does not exist";
+        rsp.op_errno = EG_NOVOL;
+    } else {
         rsp.op_ret = ret;
+        rsp.op_errstr = "";
+    }
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    ret = 0;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
-        ret = 0;
-        GF_FREE (rsp.friends.friends_val);
+    if (volumes)
+        dict_unref(volumes);
 
-        return ret;
+    GF_FREE(rsp.dict.dict_val);
+    return ret;
 }
 
-int32_t
-glusterd_get_volumes (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
+int
+__glusterd_handle_status_volume(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *entry = NULL;
-        int32_t                 count = 0;
-        dict_t                  *volumes = NULL;
-        gf_cli_rsp              rsp = {0,};
-        char                    *volname = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        volumes = dict_new ();
-        if (!volumes) {
-                gf_msg ("glusterd", GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out of Memory");
-                goto out;
+    int32_t ret = -1;
+    uint32_t cmd = 0;
+    dict_t *dict = NULL;
+    char *volname = 0;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    glusterd_op_t cli_op = GD_OP_STATUS_VOLUME;
+    char err_str[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len > 0) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
         }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize buffer");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if (!(cmd & GF_CLI_STATUS_ALL)) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to get "
+                     "volume name");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s",
+                   err_str);
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_STATUS_VOL_REQ_RCVD,
+               "Received status volume req for volume %s", volname);
+    }
+    if ((cmd & GF_CLI_STATUS_CLIENT_LIST) &&
+        (conf->op_version < GD_OP_VERSION_3_13_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at version less than %d. Getting the client-list "
+                 "is not allowed in this state.",
+                 GD_OP_VERSION_3_13_0);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_QUOTAD) &&
+        (conf->op_version == GD_OP_VERSION_MIN)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at version 1. Getting the status of quotad is not "
+                 "allowed in this state.");
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_SNAPD) &&
+        (conf->op_version < GD_OP_VERSION_3_6_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at a lesser version than %d. Getting the status of "
+                 "snapd is not allowed in this state",
+                 GD_OP_VERSION_3_6_0);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_BITD) &&
+        (conf->op_version < GD_OP_VERSION_3_7_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at a lesser version than %d. Getting the status of "
+                 "bitd is not allowed in this state",
+                 GD_OP_VERSION_3_7_0);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_SCRUB) &&
+        (conf->op_version < GD_OP_VERSION_3_7_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at a lesser version than %d. Getting the status of "
+                 "scrub is not allowed in this state",
+                 GD_OP_VERSION_3_7_0);
+        ret = -1;
+        goto out;
+    }
 
-        if (cds_list_empty (&priv->volumes)) {
-                ret = 0;
-                goto respond;
-        }
+    ret = glusterd_op_begin_synctask(req, GD_OP_STATUS_VOLUME, dict);
 
-        if (flags == GF_CLI_GET_VOLUME_ALL) {
-                cds_list_for_each_entry (entry, &priv->volumes, vol_list) {
-                        ret = glusterd_add_volume_detail_to_dict (entry,
-                                                        volumes, count);
-                        if (ret)
-                                goto respond;
+out:
 
-                        count++;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    free(cli_req.dict.dict_val);
 
-                }
+    return ret;
+}
 
-        } else if (flags == GF_CLI_GET_NEXT_VOLUME) {
-                ret = dict_get_str (dict, "volname", &volname);
+int
+glusterd_handle_status_volume(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req, __glusterd_handle_status_volume);
+}
 
-                if (ret) {
-                        if (priv->volumes.next) {
-                                entry = cds_list_entry (priv->volumes.next,
-                                                        typeof (*entry),
-                                                        vol_list);
-                        }
-                } else {
-                        ret = glusterd_volinfo_find (volname, &entry);
-                        if (ret)
-                                goto respond;
-                        entry = cds_list_entry (entry->vol_list.next,
-                                                typeof (*entry),
-                                                vol_list);
-                }
+int
+__glusterd_handle_cli_clearlocks_volume(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    glusterd_op_t cli_op = GD_OP_CLEARLOCKS_VOLUME;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = -1;
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
 
-                if (&entry->vol_list == &priv->volumes) {
-                       goto respond;
-                } else {
-                        ret = glusterd_add_volume_detail_to_dict (entry,
-                                                         volumes, count);
-                        if (ret)
-                                goto respond;
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to unserialize req-buffer to"
+                   " dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "unable to decode "
+                     "the command");
+            goto out;
+        }
 
-                        count++;
-                }
-        } else if (flags == GF_CLI_GET_VOLUME) {
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto respond;
+    } else {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLI_REQ_EMPTY,
+               "Empty cli request.");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CLRCLK_VOL_REQ_RCVD,
+           "Received clear-locks volume req "
+           "for volume %s",
+           volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_CLEARLOCKS_VOLUME, dict);
 
-                ret = glusterd_volinfo_find (volname, &entry);
-                if (ret)
-                        goto respond;
+out:
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    free(cli_req.dict.dict_val);
+
+    return ret;
+}
 
-                ret = glusterd_add_volume_detail_to_dict (entry,
-                                                 volumes, count);
-                if (ret)
-                        goto respond;
+int
+glusterd_handle_cli_clearlocks_volume(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_cli_clearlocks_volume);
+}
 
-                count++;
+static int
+get_volinfo_from_brickid(char *brickid, glusterd_volinfo_t **volinfo)
+{
+    int ret = -1;
+    char *volid_str = NULL;
+    char *brick = NULL;
+    char *brickid_dup = NULL;
+    uuid_t volid = {0};
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickid);
+
+    brickid_dup = gf_strdup(brickid);
+    if (!brickid_dup)
+        goto out;
+
+    volid_str = brickid_dup;
+    brick = strchr(brickid_dup, ':');
+    if (!brick) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND,
+               "Invalid brickid");
+        goto out;
+    }
+
+    *brick = '\0';
+    brick++;
+    gf_uuid_parse(volid_str, volid);
+    ret = glusterd_volinfo_find_by_volume_id(volid, volinfo);
+    if (ret) {
+        /* Check if it is a snapshot volume */
+        ret = glusterd_snap_volinfo_find_by_volume_id(volid, volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to find volinfo");
+            goto out;
         }
+    }
 
-respond:
-        ret = dict_set_int32 (volumes, "count", count);
-        if (ret)
-                goto out;
-        ret = dict_allocate_and_serialize (volumes, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-
-        if (ret)
-                goto out;
-
-        ret = 0;
+    ret = 0;
 out:
-        rsp.op_ret = ret;
+    GF_FREE(brickid_dup);
+    return ret;
+}
 
-        rsp.op_errstr = "";
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_cli_rsp);
-        ret = 0;
+static int
+__glusterd_handle_barrier(rpcsvc_request_t *req)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char *volname = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (!cli_req.dict.dict_len) {
+        ret = -1;
+        goto out;
+    }
 
-        if (volumes)
-                dict_unref (volumes);
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+               "Failed to unserialize "
+               "request dictionary.");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_BARRIER_VOL_REQ_RCVD,
+           "Received barrier volume request for "
+           "volume %s",
+           volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_BARRIER, dict);
 
-        GF_FREE (rsp.dict.dict_val);
-        return ret;
+out:
+    if (ret) {
+        ret = glusterd_op_send_cli_response(GD_OP_BARRIER, ret, 0, req, dict,
+                                            "Operation failed");
+    }
+    free(cli_req.dict.dict_val);
+    return ret;
 }
 
 int
-__glusterd_handle_status_volume (rpcsvc_request_t *req)
+glusterd_handle_barrier(rpcsvc_request_t *req)
 {
-        int32_t                         ret     = -1;
-        uint32_t                        cmd     = 0;
-        dict_t                         *dict    = NULL;
-        char                           *volname = 0;
-        gf_cli_req                      cli_req = {{0,}};
-        glusterd_op_t                   cli_op  = GD_OP_STATUS_VOLUME;
-        char                            err_str[2048] = {0,};
-        xlator_t                       *this = NULL;
-        glusterd_conf_t                *conf = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+    return glusterd_big_locked_handler(req, __glusterd_handle_barrier);
+}
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+static gf_boolean_t
+gd_is_global_option(char *opt_key)
+{
+    GF_VALIDATE_OR_GOTO(THIS->name, opt_key, out);
 
-        if (cli_req.dict.dict_len > 0) {
-                dict = dict_new();
-                if (!dict)
-                        goto out;
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                "unserialize buffer");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
+    return (strcmp(opt_key, GLUSTERD_SHARED_STORAGE_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_QUORUM_RATIO_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_GLOBAL_OP_VERSION_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_BRICK_MULTIPLEX_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_LOCALTIME_LOGGING_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_DAEMON_LOG_LEVEL_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_MAX_OP_VERSION_KEY) == 0);
 
-        }
+out:
+    return _gf_false;
+}
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
+int32_t
+glusterd_get_volume_opts(rpcsvc_request_t *req, dict_t *dict)
+{
+    int32_t ret = -1;
+    int32_t count = 1;
+    int exists = 0;
+    char *key = NULL;
+    char *orig_key = NULL;
+    char *key_fixed = NULL;
+    char *volname = NULL;
+    char *value = NULL;
+    char err_str[2048] = {
+        0,
+    };
+    char dict_key[50] = {
+        0,
+    };
+    int keylen;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    char op_version_buff[10] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name while handling get volume option command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") == 0) {
+        ret = glusterd_get_global_options_for_all_vols(req, dict,
+                                                       &rsp.op_errstr);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get key "
+                 "while handling get volume option for %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Received get volume opt request for "
+                 "volume %s",
+                 volname);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+    if (strcmp(key, "all")) {
+        if (fnmatch(GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
+            keylen = sprintf(dict_key, "key%d", count);
+            ret = dict_set_strn(dict, dict_key, keylen, key);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to "
+                       "set %s in dictionary",
+                       key);
+                goto out;
+            }
+            ret = dict_get_str(volinfo->dict, key, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to "
+                       "get %s in dictionary",
+                       key);
+                goto out;
+            }
+            keylen = sprintf(dict_key, "value%d", count);
+            ret = dict_set_strn(dict, dict_key, keylen, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to "
+                       "set %s in dictionary",
+                       key);
+                goto out;
+            }
+        } else {
+            exists = glusterd_check_option_exists(key, &key_fixed);
+            if (!exists) {
+                snprintf(err_str, sizeof(err_str),
+                         "Option "
+                         "with name: %s does not exist",
+                         key);
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNKNOWN_KEY,
+                       "%s", err_str);
+                if (key_fixed)
+                    snprintf(err_str + ret, sizeof(err_str) - ret,
+                             "Did you mean %s?", key_fixed);
+                ret = -1;
                 goto out;
-
-        if (!(cmd & GF_CLI_STATUS_ALL)) {
-                ret = dict_get_str (dict, "volname", &volname);
+            }
+            if (key_fixed) {
+                orig_key = key;
+                key = key_fixed;
+            }
+
+            if (gd_is_global_option(key)) {
+                char warn_str[] =
+                    "Warning: support to get \
+                                        global option value using volume get \
+                                        <volname>` will be deprecated from \
+                                        next release. Consider using `volume \
+                                        get all` instead for global options";
+
+                ret = dict_set_strn(dict, "warning", SLEN("warning"), warn_str);
                 if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Unable to get "
-                                  "volume name");
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_FOUND, "%s", err_str);
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set warning "
+                           "message in dictionary");
+                    goto out;
                 }
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_STATUS_VOL_REQ_RCVD,
-                        "Received status volume req for volume %s", volname);
+            }
 
-        }
-        if ((cmd & GF_CLI_STATUS_QUOTAD) &&
-            (conf->op_version == GD_OP_VERSION_MIN)) {
-                snprintf (err_str, sizeof (err_str), "The cluster is operating "
-                          "at version 1. Getting the status of quotad is not "
-                          "allowed in this state.");
-                ret = -1;
-                goto out;
-        }
+            if (strcmp(key, GLUSTERD_MAX_OP_VERSION_KEY) == 0) {
+                ret = glusterd_get_global_max_op_version(req, dict, 1);
+                if (ret)
+                    goto out;
+            } else if (strcmp(key, GLUSTERD_GLOBAL_OP_VERSION_KEY) == 0) {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           "to set %s in dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+                sprintf(op_version_buff, "%d", priv->op_version);
+                ret = dict_set_strn(dict, dict_key, keylen, op_version_buff);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           " to set value for key %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+            } else if (strcmp(key, "config.memory-accounting") == 0) {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           " to set %s in dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+
+                if (volinfo->memory_accounting)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "Enabled",
+                                         SLEN("Enabled"));
+                else
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "Disabled",
+                                         SLEN("Disabled"));
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           " to set value for key %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+            } else if (strcmp(key, "config.transport") == 0) {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+
+                if (volinfo->transport_type == GF_TRANSPORT_RDMA)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "rdma",
+                                         SLEN("rdma"));
+                else if (volinfo->transport_type == GF_TRANSPORT_TCP)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "tcp",
+                                         SLEN("tcp"));
+                else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "tcp,rdma",
+                                         SLEN("tcp,rdma"));
+                else
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "none",
+                                         SLEN("none"));
 
-        if ((cmd & GF_CLI_STATUS_SNAPD) &&
-            (conf->op_version < GD_OP_VERSION_3_6_0)) {
-                snprintf (err_str, sizeof (err_str), "The cluster is operating "
-                          "at a lesser version than %d. Getting the status of "
-                          "snapd is not allowed in this state",
-                          GD_OP_VERSION_3_6_0);
-                ret = -1;
-                goto out;
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set value for key "
+                           "%s in dictionary",
+                           key);
+                    goto out;
+                }
+            } else {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+                ret = dict_get_str(priv->opts, key, &value);
+                if (!ret) {
+                    ret = dict_set_strn(dict, dict_key, keylen, value);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_DICT_SET_FAILED,
+                               "Failed to set %s in "
+                               " dictionary",
+                               key);
+                        goto out;
+                    }
+                } else {
+                    ret = glusterd_get_default_val_for_volopt(
+                        dict, _gf_false, key, orig_key, volinfo,
+                        &rsp.op_errstr);
+                    if (ret && !rsp.op_errstr) {
+                        snprintf(err_str, sizeof(err_str),
+                                 "Failed to fetch the "
+                                 "value of %s, check "
+                                 "log file for more"
+                                 " details",
+                                 key);
+                    }
+                }
+            }
         }
+        /* Request is for a single option, explicitly set count to 1
+         * in the dictionary.
+         */
+        ret = dict_set_int32n(dict, "count", SLEN("count"), 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set count "
+                   "value in the dictionary");
+            goto out;
+        }
+    } else {
+        /* Handle the "all" volume option request */
+        ret = glusterd_get_default_val_for_volopt(dict, _gf_true, NULL, NULL,
+                                                  volinfo, &rsp.op_errstr);
+        if (ret && !rsp.op_errstr) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to fetch the value of all volume "
+                     "options, check log file for more details");
+        }
+    }
 
-        if ((cmd & GF_CLI_STATUS_BITD) &&
-            (conf->op_version < GD_OP_VERSION_3_7_0)) {
-                snprintf (err_str, sizeof (err_str), "The cluster is operating "
-                          "at a lesser version than %d. Getting the status of "
-                          "bitd is not allowed in this state",
-                          GD_OP_VERSION_3_7_0);
-                ret = -1;
-                goto out;
-        }
+out:
+    if (ret) {
+        if (!rsp.op_errstr)
+            rsp.op_errstr = err_str;
+        rsp.op_ret = ret;
+    } else {
+        rsp.op_errstr = "";
+        rsp.op_ret = 0;
+    }
 
-        if ((cmd & GF_CLI_STATUS_SCRUB) &&
-            (conf->op_version < GD_OP_VERSION_3_7_0)) {
-                snprintf (err_str, sizeof (err_str), "The cluster is operating "
-                          "at a lesser version than %d. Getting the status of "
-                          "scrub is not allowed in this state",
-                          GD_OP_VERSION_3_7_0);
-                ret = -1;
-                goto out;
-        }
+    ret = dict_allocate_and_serialize(dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_STATUS_VOLUME, dict);
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    GF_FREE(rsp.dict.dict_val);
+    GF_FREE(key_fixed);
+    return ret;
+}
 
-out:
+int
+__glusterd_handle_get_vol_opt(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
-        free (cli_req.dict.dict_val);
+    }
+    ret = glusterd_get_volume_opts(req, dict);
 
-        return ret;
+out:
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 int
-glusterd_handle_status_volume (rpcsvc_request_t *req)
+glusterd_handle_get_vol_opt(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_status_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_get_vol_opt);
 }
 
-int
-__glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
-{
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        glusterd_op_t                   cli_op = GD_OP_CLEARLOCKS_VOLUME;
-        char                            *volname = NULL;
-        dict_t                          *dict = NULL;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
+extern struct rpc_clnt_program gd_brick_prog;
 
-        ret = -1;
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+static int
+glusterd_print_global_options(dict_t *opts, char *key, data_t *val, void *data)
+{
+    FILE *fp = NULL;
 
-        if (cli_req.dict.dict_len) {
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to unserialize req-buffer to"
-                                " dictionary");
-                        snprintf (err_str, sizeof (err_str), "unable to decode "
-                                  "the command");
-                        goto out;
-                }
+    GF_VALIDATE_OR_GOTO(THIS->name, key, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, val, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
-        } else {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLI_REQ_EMPTY, "Empty cli request.");
-                goto out;
-        }
+    if (strcmp(key, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
+        goto out;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLNAME_NOTFOUND_IN_DICT, "%s", err_str);
-                goto out;
-        }
+    fp = (FILE *)data;
+    fprintf(fp, "%s: %s\n", key, val->data);
+out:
+    return 0;
+}
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_CLRCLK_VOL_REQ_RCVD, "Received clear-locks volume req "
-                "for volume %s", volname);
+static int
+glusterd_print_volume_options(dict_t *opts, char *key, data_t *val, void *data)
+{
+    FILE *fp = NULL;
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_CLEARLOCKS_VOLUME, dict);
+    GF_VALIDATE_OR_GOTO(THIS->name, key, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, val, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
+    fp = (FILE *)data;
+    fprintf(fp, "Volume%d.options.%s: %s\n", volcount, key, val->data);
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        free (cli_req.dict.dict_val);
+    return 0;
+}
 
-        return ret;
+static int
+glusterd_print_gsync_status(FILE *fp, dict_t *gsync_dict)
+{
+    int ret = -1;
+    int gsync_count = 0;
+    int i = 0;
+    gf_gsync_status_t *status_vals = NULL;
+    char status_val_name[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, fp, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, gsync_dict, out);
+
+    ret = dict_get_int32n(gsync_dict, "gsync-count", SLEN("gsync-count"),
+                          &gsync_count);
+
+    fprintf(fp, "Volume%d.gsync_count: %d\n", volcount, gsync_count);
+
+    if (gsync_count == 0) {
+        ret = 0;
+        goto out;
+    }
+
+    for (i = 0; i < gsync_count; i++) {
+        snprintf(status_val_name, sizeof(status_val_name), "status_value%d", i);
+
+        ret = dict_get_bin(gsync_dict, status_val_name,
+                           (void **)&(status_vals));
+        if (ret)
+            goto out;
+
+        fprintf(fp, "Volume%d.pair%d.session_slave: %s\n", volcount, i + 1,
+                get_struct_variable(21, status_vals));
+        fprintf(fp, "Volume%d.pair%d.master_node: %s\n", volcount, i + 1,
+                get_struct_variable(0, status_vals));
+        fprintf(fp, "Volume%d.pair%d.master_volume: %s\n", volcount, i + 1,
+                get_struct_variable(1, status_vals));
+        fprintf(fp, "Volume%d.pair%d.master_brick: %s\n", volcount, i + 1,
+                get_struct_variable(2, status_vals));
+        fprintf(fp, "Volume%d.pair%d.slave_user: %s\n", volcount, i + 1,
+                get_struct_variable(3, status_vals));
+        fprintf(fp, "Volume%d.pair%d.slave: %s\n", volcount, i + 1,
+                get_struct_variable(4, status_vals));
+        fprintf(fp, "Volume%d.pair%d.slave_node: %s\n", volcount, i + 1,
+                get_struct_variable(5, status_vals));
+        fprintf(fp, "Volume%d.pair%d.status: %s\n", volcount, i + 1,
+                get_struct_variable(6, status_vals));
+        fprintf(fp, "Volume%d.pair%d.crawl_status: %s\n", volcount, i + 1,
+                get_struct_variable(7, status_vals));
+        fprintf(fp, "Volume%d.pair%d.last_synced: %s\n", volcount, i + 1,
+                get_struct_variable(8, status_vals));
+        fprintf(fp, "Volume%d.pair%d.entry: %s\n", volcount, i + 1,
+                get_struct_variable(9, status_vals));
+        fprintf(fp, "Volume%d.pair%d.data: %s\n", volcount, i + 1,
+                get_struct_variable(10, status_vals));
+        fprintf(fp, "Volume%d.pair%d.meta: %s\n", volcount, i + 1,
+                get_struct_variable(11, status_vals));
+        fprintf(fp, "Volume%d.pair%d.failures: %s\n", volcount, i + 1,
+                get_struct_variable(12, status_vals));
+        fprintf(fp, "Volume%d.pair%d.checkpoint_time: %s\n", volcount, i + 1,
+                get_struct_variable(13, status_vals));
+        fprintf(fp, "Volume%d.pair%d.checkpoint_completed: %s\n", volcount,
+                i + 1, get_struct_variable(14, status_vals));
+        fprintf(fp, "Volume%d.pair%d.checkpoint_completion_time: %s\n",
+                volcount, i + 1, get_struct_variable(15, status_vals));
+    }
+out:
+    return ret;
 }
 
-int
-glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+static int
+glusterd_print_gsync_status_by_vol(FILE *fp, glusterd_volinfo_t *volinfo)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_clearlocks_volume);
+    int ret = -1;
+    dict_t *gsync_rsp_dict = NULL;
+    char my_hostname[256] = {
+        0,
+    };
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, fp, out);
+
+    gsync_rsp_dict = dict_new();
+    if (!gsync_rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gethostname(my_hostname, sizeof(my_hostname));
+    if (ret) {
+        /* stick to N/A */
+        (void)strcpy(my_hostname, "N/A");
+    }
+
+    ret = glusterd_get_gsync_status_mst(volinfo, gsync_rsp_dict, my_hostname);
+    /* Ignoring ret as above function always returns ret = 0 */
+
+    ret = glusterd_print_gsync_status(fp, gsync_rsp_dict);
+out:
+    if (gsync_rsp_dict)
+        dict_unref(gsync_rsp_dict);
+    return ret;
 }
 
 static int
-get_volinfo_from_brickid (char *brickid, glusterd_volinfo_t **volinfo)
+glusterd_print_snapinfo_by_vol(FILE *fp, glusterd_volinfo_t *volinfo,
+                               int volcount)
 {
-        int             ret         = -1;
-        char           *volid_str  = NULL;
-        char           *brick      = NULL;
-        char           *brickid_dup = NULL;
-        uuid_t          volid       = {0};
-        xlator_t       *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickid);
-
-        brickid_dup = gf_strdup (brickid);
-        if (!brickid_dup)
-                goto out;
+    int ret = -1;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    glusterd_snap_t *snapinfo = NULL;
+    int snapcount = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char snap_status_str[STATUS_STRLEN] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, fp, out);
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        snapcount++;
+        snapinfo = snap_vol->snapshot;
+
+        ret = glusterd_get_snap_status_str(snapinfo, snap_status_str);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get status for snapshot: %s", snapinfo->snapname);
 
-        volid_str = brickid_dup;
-        brick = strchr (brickid_dup, ':');
-        if (!brick) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_NOT_FOUND,
-                        "Invalid brickid");
-                goto out;
+            goto out;
         }
+        gf_time_fmt(timestr, sizeof timestr, snapinfo->time_stamp,
+                    gf_timefmt_FT);
 
-        *brick = '\0';
-        brick++;
-        gf_uuid_parse (volid_str, volid);
-        ret = glusterd_volinfo_find_by_volume_id (volid, volinfo);
-        if (ret) {
-                /* Check if it is a snapshot volume */
-                ret = glusterd_snap_volinfo_find_by_volume_id (volid, volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOLINFO_GET_FAIL,
-                                "Failed to find volinfo");
-                        goto out;
-                }
-        }
+        fprintf(fp, "Volume%d.snapshot%d.name: %s\n", volcount, snapcount,
+                snapinfo->snapname);
+        fprintf(fp, "Volume%d.snapshot%d.id: %s\n", volcount, snapcount,
+                uuid_utoa(snapinfo->snap_id));
+        fprintf(fp, "Volume%d.snapshot%d.time: %s\n", volcount, snapcount,
+                timestr);
 
-        ret = 0;
+        if (snapinfo->description)
+            fprintf(fp, "Volume%d.snapshot%d.description: %s\n", volcount,
+                    snapcount, snapinfo->description);
+        fprintf(fp, "Volume%d.snapshot%d.status: %s\n", volcount, snapcount,
+                snap_status_str);
+    }
+
+    ret = 0;
 out:
-        GF_FREE (brickid_dup);
-        return ret;
+    return ret;
 }
 
 static int
-__glusterd_handle_barrier (rpcsvc_request_t *req)
+glusterd_print_client_details(FILE *fp, dict_t *dict,
+                              glusterd_volinfo_t *volinfo, int volcount,
+                              glusterd_brickinfo_t *brickinfo, int brickcount)
 {
-        int          ret     = -1;
-        xlator_t     *this   = NULL;
-        gf_cli_req   cli_req = {{0,}};
-        dict_t       *dict   = NULL;
-        char *volname = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    int brick_index = -1;
+    int client_count = 0;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *clientname = NULL;
+    uint64_t bytesread = 0;
+    uint64_t byteswrite = 0;
+    uint32_t opversion = 0;
+
+    glusterd_pending_node_t *pending_node = NULL;
+    rpc_clnt_t *rpc = NULL;
+    struct syncargs args = {
+        0,
+    };
+    gd1_mgmt_brick_op_req *brick_req = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+        !glusterd_is_brick_started(brickinfo)) {
+        ret = 0;
+        goto out;
+    }
 
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT(this);
+    brick_index++;
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    pending_node->node = brickinfo;
+    pending_node->type = GD_NODE_BRICK;
+    pending_node->index = brick_index;
 
-        if (!cli_req.dict.dict_len) {
-                ret = -1;
-                goto out;
-        }
+    rpc = glusterd_pending_node_get_rpc(pending_node);
+    if (!rpc) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE,
+               "Failed to retrieve rpc object");
+        goto out;
+    }
 
-        dict = dict_new();
-        if (!dict) {
-                ret = -1;
-                goto out;
+    brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t);
+    if (!brick_req) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    brick_req->op = GLUSTERD_BRICK_STATUS;
+    brick_req->name = "";
+    brick_req->dict.dict_val = NULL;
+    brick_req->dict.dict_len = 0;
+
+    ret = dict_set_strn(dict, "brick-name", SLEN("brick-name"),
+                        brickinfo->path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=brick-name", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(dict, "cmd", SLEN("cmd"), GF_CLI_STATUS_CLIENTS);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cmd", NULL);
+        goto out;
+    }
+
+    ret = dict_set_strn(dict, "volname", SLEN("volname"), volinfo->volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val,
+                                      &brick_req->input.input_len);
+    if (ret)
+        goto out;
+
+    GD_SYNCOP(rpc, (&args), NULL, gd_syncop_brick_op_cbk, brick_req,
+              &gd_brick_prog, brick_req->op, xdr_gd1_mgmt_brick_op_req);
+
+    if (args.op_ret)
+        goto out;
+
+    ret = dict_get_int32n(args.dict, "clientcount", SLEN("clientcount"),
+                          &client_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get client count");
+        goto out;
+    }
+
+    fprintf(fp, "Volume%d.Brick%d.client_count: %d\n", volcount, brickcount,
+            client_count);
+
+    if (client_count == 0) {
+        ret = 0;
+        goto out;
+    }
+
+    int i;
+    for (i = 1; i <= client_count; i++) {
+        keylen = snprintf(key, sizeof(key), "client%d.hostname", i - 1);
+        ret = dict_get_strn(args.dict, key, keylen, &clientname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get client hostname");
+            goto out;
         }
-        ret = dict_unserialize (cli_req.dict.dict_val, cli_req.dict.dict_len,
-                                &dict);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL, "Failed to unserialize "
-                        "request dictionary.");
-                goto out;
+
+        snprintf(key, sizeof(key), "Client%d.hostname", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %s\n", volcount, brickcount, key,
+                clientname);
+
+        snprintf(key, sizeof(key), "client%d.bytesread", i - 1);
+        ret = dict_get_uint64(args.dict, key, &bytesread);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get bytesread from client");
+            goto out;
         }
 
-        ret = dict_get_str (dict, "volname", &volname);
+        snprintf(key, sizeof(key), "Client%d.bytesread", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %" PRIu64 "\n", volcount, brickcount,
+                key, bytesread);
+
+        snprintf(key, sizeof(key), "client%d.byteswrite", i - 1);
+        ret = dict_get_uint64(args.dict, key, &byteswrite);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
-                        "Volname not present in "
-                        "dict");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get byteswrite from client");
+            goto out;
         }
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_BARRIER_VOL_REQ_RCVD,
-                "Received barrier volume request for "
-                "volume %s", volname);
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_BARRIER, dict);
+        snprintf(key, sizeof(key), "Client%d.byteswrite", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %" PRIu64 "\n", volcount, brickcount,
+                key, byteswrite);
 
-out:
+        snprintf(key, sizeof(key), "client%d.opversion", i - 1);
+        ret = dict_get_uint32(args.dict, key, &opversion);
         if (ret) {
-                ret = glusterd_op_send_cli_response (GD_OP_BARRIER, ret, 0, req,
-                                                     dict, "Operation failed");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get client opversion");
+            goto out;
         }
-        free (cli_req.dict.dict_val);
-        return ret;
-}
 
-int
-glusterd_handle_barrier (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __glusterd_handle_barrier);
+        snprintf(key, sizeof(key), "Client%d.opversion", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %" PRIu32 "\n", volcount, brickcount,
+                key, opversion);
+    }
+
+out:
+    if (pending_node)
+        GF_FREE(pending_node);
+
+    if (brick_req) {
+        if (brick_req->input.input_val)
+            GF_FREE(brick_req->input.input_val);
+        GF_FREE(brick_req);
+    }
+    if (args.dict)
+        dict_unref(args.dict);
+    if (args.errstr)
+        GF_FREE(args.errstr);
+
+    return ret;
 }
 
-int32_t
-glusterd_get_volume_opts (rpcsvc_request_t *req, dict_t *dict)
+static int
+glusterd_get_state(rpcsvc_request_t *req, dict_t *dict)
 {
-        int32_t                   ret = -1;
-        int32_t                   count = 1;
-        int                       exists = 0;
-        char                      *key = NULL;
-        char                      *orig_key = NULL;
-        char                      *key_fixed = NULL;
-        char                      *volname = NULL;
-        char                      *value = NULL;
-        char                      err_str[2048] = {0,};
-        char                      dict_key[50] = {0,};
-        xlator_t                  *this = NULL;
-        glusterd_conf_t           *priv = NULL;
-        glusterd_volinfo_t        *volinfo = NULL;
-        gf_cli_rsp                rsp = {0,};
-        char                      op_version_buff[10] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name while handling get volume option command");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLNAME_NOTFOUND_IN_DICT, "%s", err_str);
-                goto out;
+    int32_t ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    FILE *fp = NULL;
+    DIR *dp = NULL;
+    char err_str[2048] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peer_hostname_t *peer_hostname_info = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    dict_t *vol_all_opts = NULL;
+    struct statvfs brickstat = {0};
+    char *odir = NULL;
+    char *filename = NULL;
+    char *ofilepath = NULL;
+    char *tmp_str = NULL;
+    int count = 0;
+    int count_bkp = 0;
+    int odirlen = 0;
+    time_t now = 0;
+    char timestamp[16] = {
+        0,
+    };
+    uint32_t get_state_cmd = 0;
+    uint64_t memtotal = 0;
+    uint64_t memfree = 0;
+    char id_str[64] = {
+        0,
+    };
+
+    char *vol_type_str = NULL;
+
+    char transport_type_str[STATUS_STRLEN] = {
+        0,
+    };
+    char quorum_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    char rebal_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    char vol_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    char brick_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    priv = THIS->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    ret = dict_get_strn(dict, "odir", SLEN("odir"), &tmp_str);
+    if (ret) {
+        odirlen = gf_asprintf(&odir, "%s", "/var/run/gluster/");
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "Default output directory: %s", odir);
+    } else {
+        odirlen = gf_asprintf(&odir, "%s", tmp_str);
+    }
+
+    dp = sys_opendir(odir);
+    if (dp) {
+        sys_closedir(dp);
+    } else {
+        if (errno == ENOENT) {
+            snprintf(err_str, sizeof(err_str),
+                     "Output directory %s does not exist.", odir);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   err_str);
+        } else if (errno == ENOTDIR) {
+            snprintf(err_str, sizeof(err_str),
+                     "Output directory "
+                     "does not exist. %s points to a file.",
+                     odir);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   err_str);
+        }
+
+        GF_FREE(odir);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "filename", SLEN("filename"), &tmp_str);
+    if (ret) {
+        now = gf_time();
+        strftime(timestamp, sizeof(timestamp), "%Y%m%d_%H%M%S",
+                 localtime(&now));
+        gf_asprintf(&filename, "%s_%s", "glusterd_state", timestamp);
+
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "Default filename: %s", filename);
+    } else {
+        gf_asprintf(&filename, "%s", tmp_str);
+    }
+
+    ret = gf_asprintf(&ofilepath, "%s%s%s", odir,
+                      ((odir[odirlen - 1] != '/') ? "/" : ""), filename);
+
+    if (ret < 0) {
+        GF_FREE(odir);
+        GF_FREE(filename);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to get the output path");
+        ret = -1;
+        goto out;
+    }
+    GF_FREE(odir);
+    GF_FREE(filename);
+
+    ret = dict_set_dynstrn(dict, "ofilepath", SLEN("ofilepath"), ofilepath);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set output path");
+        goto out;
+    }
+
+    fp = fopen(ofilepath, "w");
+    if (!fp) {
+        snprintf(err_str, sizeof(err_str), "Failed to open file at %s",
+                 ofilepath);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_uint32(dict, "getstate-cmd", &get_state_cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "get-state command type not set");
+        ret = 0;
+    }
+
+    if (get_state_cmd == GF_CLI_GET_STATE_VOLOPTS) {
+        fprintf(fp, "[Volume Options]\n");
+        cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+        {
+            fprintf(fp, "Volume%d.name: %s\n", ++count, volinfo->volname);
+
+            volcount = count;
+            vol_all_opts = dict_new();
+
+            ret = glusterd_get_default_val_for_volopt(
+                vol_all_opts, _gf_true, NULL, NULL, volinfo, &rsp.op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OPTS_IMPORT_FAIL,
+                       "Failed to "
+                       "fetch the value of all volume options "
+                       "for volume %s",
+                       volinfo->volname);
+                if (vol_all_opts)
+                    dict_unref(vol_all_opts);
+                continue;
+            }
+
+            dict_foreach(vol_all_opts, glusterd_print_volume_options, fp);
+
+            if (vol_all_opts)
+                dict_unref(vol_all_opts);
         }
+        ret = 0;
+        goto out;
+    }
+
+    fprintf(fp, "[Global]\n");
+
+    uuid_utoa_r(priv->uuid, id_str);
+    fprintf(fp, "MYUUID: %s\n", id_str);
+
+    fprintf(fp, "op-version: %d\n", priv->op_version);
+
+    fprintf(fp, "\n[Global options]\n");
+
+    if (priv->opts)
+        dict_foreach(priv->opts, glusterd_print_global_options, fp);
+
+    fprintf(fp, "\n[Peers]\n");
+    RCU_READ_LOCK;
+
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        fprintf(fp, "Peer%d.primary_hostname: %s\n", ++count,
+                peerinfo->hostname);
+        fprintf(fp, "Peer%d.uuid: %s\n", count, gd_peer_uuid_str(peerinfo));
+        fprintf(fp, "Peer%d.state: %s\n", count,
+                glusterd_friend_sm_state_name_get(peerinfo->state.state));
+        fprintf(fp, "Peer%d.connected: %s\n", count,
+                peerinfo->connected ? "Connected" : "Disconnected");
 
-        ret = dict_get_str (dict, "key", &key);
+        fprintf(fp, "Peer%d.othernames: ", count);
+        count_bkp = 0;
+        cds_list_for_each_entry(peer_hostname_info, &peerinfo->hostnames,
+                                hostname_list)
+        {
+            if (strcmp(peerinfo->hostname, peer_hostname_info->hostname) == 0)
+                continue;
+
+            if (count_bkp > 0)
+                fprintf(fp, ",");
+
+            fprintf(fp, "%s", peer_hostname_info->hostname);
+            count_bkp++;
+        }
+        count_bkp = 0;
+        fprintf(fp, "\n");
+    }
+    RCU_READ_UNLOCK;
+
+    count = 0;
+    fprintf(fp, "\n[Volumes]\n");
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        ret = glusterd_volume_get_type_str(volinfo, &vol_type_str);
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get key "
-                          "while handling get volume option for %s", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", err_str);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get type for volume: %s", volinfo->volname);
+            goto out;
         }
-        gf_msg_debug (this->name, 0, "Received get volume opt request for "
-                "volume %s", volname);
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+        ret = glusterd_volume_get_status_str(volinfo, vol_status_str);
         if (ret) {
-                snprintf (err_str, sizeof(err_str),
-                          FMTSTR_CHECK_VOL_EXISTS, volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get status for volume: %s", volinfo->volname);
+            goto out;
         }
-        if (strcmp(key, "all")) {
-                if (fnmatch (GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
-                        sprintf (dict_key, "key%d", count);
-                        ret = dict_set_str(dict, dict_key, key);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "Failed to "
-                                        "set %s in dictionary", key);
-                                goto out;
-                        }
-                        sprintf (dict_key, "value%d", count);
-                        ret = dict_get_str (volinfo->dict, key, &value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "Failed to "
-                                        "get %s in dictionary", key);
-                                goto out;
-                        }
-                        ret = dict_set_str(dict, dict_key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "Failed to "
-                                        "set %s in dictionary", key);
-                                goto out;
-                        }
-                } else {
-                        exists = glusterd_check_option_exists (key, &key_fixed);
-                        if (!exists) {
-                                snprintf (err_str, sizeof (err_str), "Option "
-                                          "with name: %s does not exist", key);
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_UNKNOWN_KEY, "%s",
-                                        err_str);
-                                if (key_fixed)
-                                        snprintf (err_str + ret,
-                                                  sizeof (err_str) - ret,
-                                                  "Did you mean %s?",
-                                                  key_fixed);
-                                ret = -1;
-                                goto out;
-                        }
-                        if (key_fixed) {
-                                orig_key = key;
-                                key = key_fixed;
-                        }
-                        if (strcmp (key, "cluster.op-version") == 0) {
-                                sprintf (dict_key, "key%d", count);
-                                ret = dict_set_str(dict, dict_key, key);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED, "Failed"
-                                                "to set %s in dictionary", key);
-                                        goto out;
-                                }
-                                sprintf (dict_key, "value%d", count);
-                                sprintf (op_version_buff, "%d",
-                                         priv->op_version);
-                                ret = dict_set_str (dict, dict_key,
-                                                    op_version_buff);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED, "Failed"
-                                                " to set value for key %s in "
-                                                "dictionary", key);
-                                        goto out;
-                                }
-                        } else if (strcmp (key,
-                                         "config.memory-accounting") == 0) {
-                                sprintf (dict_key, "key%d", count);
-                                ret = dict_set_str(dict, dict_key, key);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED, "Failed"
-                                                " to set %s in dictionary",
-                                                key);
-                                        goto out;
-                                }
-                                sprintf (dict_key, "value%d", count);
-
-                                if (volinfo->memory_accounting)
-                                        ret = dict_set_str(dict, dict_key,
-                                                           "Enabled");
-                                else
-                                        ret = dict_set_str(dict, dict_key,
-                                                           "Disabled");
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED, "Failed"
-                                                " to set value for key %s in "
-                                                "dictionary", key);
-                                        goto out;
-                                }
-                        } else if (strcmp (key, "config.transport") == 0) {
-                                sprintf (dict_key, "key%d", count);
-                                ret = dict_set_str(dict, dict_key, key);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED, "Failed"
-                                                "to set %s in dictionary", key);
-                                        goto out;
-                                }
-                                sprintf (dict_key, "value%d", count);
-
-                                if (volinfo->transport_type
-                                                == GF_TRANSPORT_RDMA)
-                                        ret = dict_set_str(dict, dict_key,
-                                                           "rdma");
-                                else if (volinfo->transport_type
-                                                == GF_TRANSPORT_TCP)
-                                        ret = dict_set_str(dict, dict_key,
-                                                           "tcp");
-                                else if (volinfo->transport_type ==
-                                         GF_TRANSPORT_BOTH_TCP_RDMA)
-                                        ret = dict_set_str(dict, dict_key,
-                                                           "tcp,rdma");
-                                else
-                                        ret = dict_set_str(dict, dict_key,
-                                                           "none");
-
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED, "Failed"
-                                                " to set value for key %s in "
-                                                "dictionary", key);
-                                        goto out;
-                                }
-                        } else {
-                                sprintf (dict_key, "key%d", count);
-                                ret = dict_set_str(dict, dict_key, key);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED, "Failed"
-                                                " to set %s in dictionary",
-                                                key);
-                                        goto out;
-                                }
-                                sprintf (dict_key, "value%d", count);
-                                ret = dict_get_str (priv->opts, key, &value);
-                                if (!ret) {
-                                        ret = dict_set_str(dict, dict_key,
-                                                           value);
-                                        if (ret) {
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        GD_MSG_DICT_SET_FAILED,
-                                                        "Failed to set %s in "
-                                                        " dictionary", key);
-                                                goto out;
-                                        }
-                                } else {
-                                        ret = glusterd_get_default_val_for_volopt
-                                                               (dict,
-                                                                _gf_false,
-                                                                key, orig_key,
-                                                                volinfo->dict,
-                                                                &rsp.op_errstr);
-                                        if (ret && !rsp.op_errstr) {
-                                                snprintf (err_str,
-                                                          sizeof(err_str),
-                                                          "Failed to fetch the "
-                                                          "value of %s, check "
-                                                          "log file for more"
-                                                          " details", key);
-                                        }
-                                }
-                        }
-                }
-                /* Request is for a single option, explicitly set count to 1
-                 * in the dictionary.
-                 */
-                ret = dict_set_int32 (dict, "count", 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set count "
-                                "value in the dictionary");
-                        goto out;
-                }
-        } else {
-                /* Handle the "all" volume option request */
-                ret = glusterd_get_default_val_for_volopt (dict, _gf_true, NULL,
-                                                           NULL, volinfo->dict,
-                                                           &rsp.op_errstr);
-                if (ret && !rsp.op_errstr) {
-                        snprintf (err_str, sizeof(err_str),
-                                  "Failed to fetch the value of all volume "
-                                  "options, check log file for more details");
-                }
 
+        ret = glusterd_volume_get_transport_type_str(volinfo,
+                                                     transport_type_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get transport type for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
 
-out:
+        ret = glusterd_volume_get_quorum_status_str(volinfo, quorum_status_str);
         if (ret) {
-                if (!rsp.op_errstr)
-                        rsp.op_errstr = err_str;
-                rsp.op_ret =  ret;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get quorum status for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
-        else {
-                rsp.op_errstr = "";
-                rsp.op_ret = 0;
+
+        ret = glusterd_volume_get_rebalance_status_str(volinfo,
+                                                       rebal_status_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get rebalance status for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
 
-        ret = dict_allocate_and_serialize (dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
+        fprintf(fp, "Volume%d.name: %s\n", ++count, volinfo->volname);
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_cli_rsp);
-        return ret;
-}
+        uuid_utoa_r(volinfo->volume_id, id_str);
+        fprintf(fp, "Volume%d.id: %s\n", count, id_str);
 
-int
-__glusterd_handle_get_vol_opt (rpcsvc_request_t *req)
-{
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
+        fprintf(fp, "Volume%d.type: %s\n", count, vol_type_str);
+        fprintf(fp, "Volume%d.transport_type: %s\n", count, transport_type_str);
+        fprintf(fp, "Volume%d.status: %s\n", count, vol_status_str);
+        fprintf(fp, "Volume%d.profile_enabled: %d\n", count,
+                glusterd_is_profile_on(volinfo));
+        fprintf(fp, "Volume%d.brickcount: %d\n", count, volinfo->brick_count);
+
+        count_bkp = count;
+        count = 0;
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            fprintf(fp, "Volume%d.Brick%d.path: %s:%s\n", count_bkp, ++count,
+                    brickinfo->hostname, brickinfo->path);
+            fprintf(fp, "Volume%d.Brick%d.hostname: %s\n", count_bkp, count,
+                    brickinfo->hostname);
+            /* Determine which one is the arbiter brick */
+            if (volinfo->arbiter_count == 1) {
+                if (count % volinfo->replica_count == 0) {
+                    fprintf(fp,
+                            "Volume%d.Brick%d."
+                            "is_arbiter: 1\n",
+                            count_bkp, count);
+                }
+            }
+            /* Add following information only for bricks
+             *  local to current node */
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
+            fprintf(fp, "Volume%d.Brick%d.port: %d\n", count_bkp, count,
+                    brickinfo->port);
+            fprintf(fp, "Volume%d.Brick%d.rdma_port: %d\n", count_bkp, count,
+                    brickinfo->rdma_port);
+            fprintf(fp, "Volume%d.Brick%d.port_registered: %d\n", count_bkp,
+                    count, brickinfo->port_registered);
+            glusterd_brick_get_status_str(brickinfo, brick_status_str);
+            fprintf(fp, "Volume%d.Brick%d.status: %s\n", count_bkp, count,
+                    brick_status_str);
+
+            ret = sys_statvfs(brickinfo->path, &brickstat);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                       "statfs error: %s ", strerror(errno));
+                memfree = 0;
+                memtotal = 0;
+            } else {
+                memfree = brickstat.f_bfree * brickstat.f_bsize;
+                memtotal = brickstat.f_blocks * brickstat.f_bsize;
+            }
+
+            fprintf(fp, "Volume%d.Brick%d.spacefree: %" PRIu64 "Bytes\n",
+                    count_bkp, count, memfree);
+            fprintf(fp, "Volume%d.Brick%d.spacetotal: %" PRIu64 "Bytes\n",
+                    count_bkp, count, memtotal);
+
+            if (get_state_cmd != GF_CLI_GET_STATE_DETAIL)
+                continue;
+
+            ret = glusterd_print_client_details(fp, dict, volinfo, count_bkp,
+                                                brickinfo, count);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_CLIENTS_GET_STATE_FAILED,
+                       "Failed to get client details");
+                goto out;
+            }
+        }
+
+        count = count_bkp;
+
+        ret = glusterd_print_snapinfo_by_vol(fp, volinfo, count);
+        if (ret)
+            goto out;
+
+        fprintf(fp, "Volume%d.snap_count: %" PRIu64 "\n", count,
+                volinfo->snap_count);
+        fprintf(fp, "Volume%d.stripe_count: %d\n", count,
+                volinfo->stripe_count);
+        fprintf(fp, "Volume%d.replica_count: %d\n", count,
+                volinfo->replica_count);
+        fprintf(fp, "Volume%d.subvol_count: %d\n", count,
+                volinfo->subvol_count);
+        fprintf(fp, "Volume%d.arbiter_count: %d\n", count,
+                volinfo->arbiter_count);
+        fprintf(fp, "Volume%d.disperse_count: %d\n", count,
+                volinfo->disperse_count);
+        fprintf(fp, "Volume%d.redundancy_count: %d\n", count,
+                volinfo->redundancy_count);
+        fprintf(fp, "Volume%d.quorum_status: %s\n", count, quorum_status_str);
+
+        fprintf(fp, "Volume%d.snapd_svc.online_status: %s\n", count,
+                volinfo->snapd.svc.online ? "Online" : "Offline");
+        fprintf(fp, "Volume%d.snapd_svc.inited: %s\n", count,
+                volinfo->snapd.svc.inited ? "True" : "False");
+
+        uuid_utoa_r(volinfo->rebal.rebalance_id, id_str);
+        char *rebal_data = gf_uint64_2human_readable(
+            volinfo->rebal.rebalance_data);
+
+        fprintf(fp, "Volume%d.rebalance.id: %s\n", count, id_str);
+        fprintf(fp, "Volume%d.rebalance.status: %s\n", count, rebal_status_str);
+        fprintf(fp, "Volume%d.rebalance.failures: %" PRIu64 "\n", count,
+                volinfo->rebal.rebalance_failures);
+        fprintf(fp, "Volume%d.rebalance.skipped: %" PRIu64 "\n", count,
+                volinfo->rebal.skipped_files);
+        fprintf(fp, "Volume%d.rebalance.lookedup: %" PRIu64 "\n", count,
+                volinfo->rebal.lookedup_files);
+        fprintf(fp, "Volume%d.rebalance.files: %" PRIu64 "\n", count,
+                volinfo->rebal.rebalance_files);
+        fprintf(fp, "Volume%d.rebalance.data: %s\n", count, rebal_data);
+        fprintf(fp, "Volume%d.time_left: %" PRIu64 "\n", count,
+                volinfo->rebal.time_left);
+
+        GF_FREE(rebal_data);
+
+        fprintf(fp, "Volume%d.shd_svc.online_status: %s\n", count,
+                volinfo->shd.svc.online ? "Online" : "Offline");
+        fprintf(fp, "Volume%d.shd_svc.inited: %s\n", count,
+                volinfo->shd.svc.inited ? "True" : "False");
+
+        if (volinfo->rep_brick.src_brick && volinfo->rep_brick.dst_brick) {
+            fprintf(fp, "Volume%d.replace_brick.src: %s:%s\n", count,
+                    volinfo->rep_brick.src_brick->hostname,
+                    volinfo->rep_brick.src_brick->path);
+            fprintf(fp, "Volume%d.replace_brick.dest: %s:%s\n", count,
+                    volinfo->rep_brick.dst_brick->hostname,
+                    volinfo->rep_brick.dst_brick->path);
+        }
+
+        volcount = count;
+        ret = glusterd_print_gsync_status_by_vol(fp, volinfo);
+        if (ret)
+            goto out;
+
+        if (volinfo->dict)
+            dict_foreach(volinfo->dict, glusterd_print_volume_options, fp);
 
-        this = THIS;
-        GF_ASSERT (this);
+        fprintf(fp, "\n");
+    }
 
-        GF_ASSERT (req);
+    count = 0;
+
+    fprintf(fp, "\n[Services]\n");
+#ifdef BUILD_GNFS
+    if (priv->nfs_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->nfs_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->nfs_svc.online ? "Online" : "Offline");
+    }
+#endif
+    if (priv->bitd_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->bitd_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->bitd_svc.online ? "Online" : "Offline");
+    }
+
+    if (priv->scrub_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->scrub_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->scrub_svc.online ? "Online" : "Offline");
+    }
+
+    if (priv->quotad_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->quotad_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->quotad_svc.online ? "Online" : "Offline");
+    }
+
+    fprintf(fp, "\n[Misc]\n");
+    if (priv->pmap) {
+        fprintf(fp, "Base port: %d\n", priv->pmap->base_port);
+        fprintf(fp, "Last allocated port: %d\n", priv->pmap->last_alloc);
+    }
+out:
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (fp)
+        fclose(fp);
+
+    rsp.op_ret = ret;
+    if (rsp.op_errstr == NULL)
+        rsp.op_errstr = err_str;
+
+    ret = dict_allocate_and_serialize(dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    glusterd_to_cli(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp, dict);
+    GF_FREE(rsp.dict.dict_val);
+
+    return ret;
+}
+
+static int
+__glusterd_handle_get_state(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gf_cli_req cli_req = {
+        {
+            0,
+        },
+    };
+    dict_t *dict = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DAEMON_STATE_REQ_RCVD,
+           "Received request to get state for glusterd");
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode "
-                          "request received from cli");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode"
+                     " the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-        ret = glusterd_get_volume_opts (req, dict);
+    ret = glusterd_get_state(req, dict);
 
 out:
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
+    if (dict && ret) {
+        /*
+         * When glusterd_to_cli (called from glusterd_get_state)
+         * succeeds, it frees the dict for us, so this would be a
+         * double free, but in other cases it's our responsibility.
+         */
+        dict_unref(dict);
+    }
+    return ret;
 }
 
 int
-glusterd_handle_get_vol_opt (rpcsvc_request_t *req)
+glusterd_handle_get_state(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_get_vol_opt);
+    return glusterd_big_locked_handler(req, __glusterd_handle_get_state);
 }
+
 static int
-get_brickinfo_from_brickid (char *brickid, glusterd_brickinfo_t **brickinfo)
+get_brickinfo_from_brickid(char *brickid, glusterd_brickinfo_t **brickinfo)
 {
-        glusterd_volinfo_t      *volinfo    = NULL;
-        char                    *volid_str  = NULL;
-        char                    *brick      = NULL;
-        char                    *brickid_dup = NULL;
-        uuid_t                  volid       = {0};
-        int                     ret         = -1;
-
-        brickid_dup = gf_strdup (brickid);
-        if (!brickid_dup)
-                goto out;
-
-        volid_str = brickid_dup;
-        brick = strchr (brickid_dup, ':');
-        if (!volid_str || !brick)
-                goto out;
-
-        *brick = '\0';
-        brick++;
-        gf_uuid_parse (volid_str, volid);
-        ret = glusterd_volinfo_find_by_volume_id (volid, &volinfo);
-        if (ret) {
-                /* Check if it a snapshot volume */
-                ret = glusterd_snap_volinfo_find_by_volume_id (volid, &volinfo);
-                if (ret)
-                        goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                      brickinfo,
-                                                      _gf_false);
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volid_str = NULL;
+    char *brick = NULL;
+    char *brickid_dup = NULL;
+    uuid_t volid = {0};
+    int ret = -1;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    brickid_dup = gf_strdup(brickid);
+    if (!brickid_dup) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "brick_id=%s", brickid, NULL);
+        goto out;
+    }
+
+    volid_str = brickid_dup;
+    brick = strchr(brickid_dup, ':');
+    if (!volid_str) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+        goto out;
+    }
+
+    if (!brick) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+        goto out;
+    }
+
+    *brick = '\0';
+    brick++;
+    gf_uuid_parse(volid_str, volid);
+    ret = glusterd_volinfo_find_by_volume_id(volid, &volinfo);
+    if (ret) {
+        /* Check if it a snapshot volume */
+        ret = glusterd_snap_volinfo_find_by_volume_id(volid, &volinfo);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        ret = 0;
+    ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, brickinfo,
+                                                 _gf_false);
+    if (ret)
+        goto out;
+
+    ret = 0;
 out:
-        GF_FREE (brickid_dup);
-        return ret;
+    GF_FREE(brickid_dup);
+    return ret;
 }
 
+static int gd_stale_rpc_disconnect_log;
+
 int
-__glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                             rpc_clnt_event_t event, void *data)
+__glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data)
 {
-        char                    *brickid           = NULL;
-        int                      ret               = 0;
-        glusterd_conf_t         *conf              = NULL;
-        glusterd_brickinfo_t    *brickinfo         = NULL;
-        glusterd_volinfo_t      *volinfo           = NULL;
-        xlator_t                *this              = NULL;
-
-        brickid = mydata;
-        if (!brickid)
-                return 0;
-
-        ret = get_brickinfo_from_brickid (brickid, &brickinfo);
-        if (ret)
-                return 0;
+    char *brickid = NULL;
+    int ret = 0;
+    glusterd_conf_t *conf = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int32_t pid = -1;
+    glusterd_brickinfo_t *brickinfo_tmp = NULL;
+    glusterd_brick_proc_t *brick_proc = NULL;
+    char pidfile[PATH_MAX] = {0};
+    char *brickpath = NULL;
+    gf_boolean_t is_service_running = _gf_true;
+
+    brickid = mydata;
+    if (!brickid)
+        return 0;
 
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+    ret = get_brickinfo_from_brickid(brickid, &brickinfo);
+    if (ret)
+        return 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        switch (event) {
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                /* If a node on coming back up, already starts a brick
-                 * before the handshake, and the notification comes after
-                 * the handshake is done, then we need to check if this
-                 * is a restored brick with a snapshot pending. If so, we
-                 * need to stop the brick
-                 */
-                if (brickinfo->snap_status == -1) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_SNAPSHOT_PENDING,
-                                "Snapshot is pending on %s:%s. "
-                                "Hence not starting the brick",
-                                brickinfo->hostname,
-                                brickinfo->path);
-                        ret = get_volinfo_from_brickid (brickid, &volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOLINFO_GET_FAIL,
-                                        "Failed to get volinfo from "
-                                        "brickid(%s)", brickid);
-                                goto out;
-                        }
-
-                        ret = glusterd_brick_stop (volinfo, brickinfo,
-                                                   _gf_false);
-                        if (ret) {
-                                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_BRICK_STOP_FAIL,
-                                        "Unable to stop %s:%s",
-                                        brickinfo->hostname, brickinfo->path);
-                                goto out;
-                        }
-
-                        break;
+            ret = get_volinfo_from_brickid(brickid, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                       "Failed to get volinfo from "
+                       "brickid(%s)",
+                       brickid);
+                goto out;
+            }
+            /* If a node on coming back up, already starts a brick
+             * before the handshake, and the notification comes after
+             * the handshake is done, then we need to check if this
+             * is a restored brick with a snapshot pending. If so, we
+             * need to stop the brick
+             */
+            if (brickinfo->snap_status == -1) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SNAPSHOT_PENDING,
+                       "Snapshot is pending on %s:%s. "
+                       "Hence not starting the brick",
+                       brickinfo->hostname, brickinfo->path);
+                ret = glusterd_brick_stop(volinfo, brickinfo, _gf_false);
+                if (ret) {
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                           "Unable to stop %s:%s", brickinfo->hostname,
+                           brickinfo->path);
+                    goto out;
                 }
-                rpc_clnt_set_connected (&rpc->conn);
-                gf_msg_debug (this->name, 0, "Connected to %s:%s",
-                        brickinfo->hostname, brickinfo->path);
-                glusterd_set_brick_status (brickinfo, GF_BRICK_STARTED);
-                ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
+            }
+            gf_msg_debug(this->name, 0, "Connected to %s:%s",
+                         brickinfo->hostname, brickinfo->path);
+
+            glusterd_set_brick_status(brickinfo, GF_BRICK_STARTED);
+
+            gf_event(EVENT_BRICK_CONNECTED, "peer=%s;volume=%s;brick=%s",
+                     brickinfo->hostname, volinfo->volname, brickinfo->path);
+
+            ret = default_notify(this, GF_EVENT_CHILD_UP, NULL);
+
+            break;
 
         case RPC_CLNT_DISCONNECT:
-                rpc_clnt_unset_connected (&rpc->conn);
-                if (glusterd_is_brick_started (brickinfo))
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_BRICK_DISCONNECTED,
-                                "Brick %s:%s has disconnected from glusterd.",
-                                brickinfo->hostname, brickinfo->path);
-
-                glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
+            if (rpc != brickinfo->rpc) {
+                /*
+                 * There used to be a bunch of races in the volume
+                 * start/stop code that could result in us getting here
+                 * and setting the brick status incorrectly.  Many of
+                 * those have been fixed or avoided, but just in case
+                 * any are still left it doesn't hurt to keep the extra
+                 * check and avoid further damage.
+                 */
+                GF_LOG_OCCASIONALLY(gd_stale_rpc_disconnect_log, this->name,
+                                    GF_LOG_WARNING,
+                                    "got disconnect from stale rpc on "
+                                    "%s",
+                                    brickinfo->path);
                 break;
+            }
+            if (glusterd_is_brick_started(brickinfo)) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "Brick %s:%s has disconnected from glusterd.",
+                       brickinfo->hostname, brickinfo->path);
+
+                ret = get_volinfo_from_brickid(brickid, &volinfo);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                           "Failed to get volinfo from "
+                           "brickid(%s)",
+                           brickid);
+                    goto out;
+                }
+                gf_event(EVENT_BRICK_DISCONNECTED, "peer=%s;volume=%s;brick=%s",
+                         brickinfo->hostname, volinfo->volname,
+                         brickinfo->path);
+                /* In case of an abrupt shutdown of a brick PMAP_SIGNOUT
+                 * event is not received by glusterd which can lead to a
+                 * stale port entry in glusterd, so forcibly clean up
+                 * the same if the process is not running sometime
+                 * gf_is_service_running true so to ensure about brick instance
+                 * call search_brick_path_from_proc
+                 */
+                GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+                is_service_running = gf_is_service_running(pidfile, &pid);
+                if (pid > 0)
+                    brickpath = search_brick_path_from_proc(pid,
+                                                            brickinfo->path);
+                if (!is_service_running || !brickpath) {
+                    ret = pmap_registry_remove(
+                        THIS, brickinfo->port, brickinfo->path,
+                        GF_PMAP_PORT_BRICKSERVER, NULL, _gf_true);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_WARNING,
+                               GD_MSG_PMAP_REGISTRY_REMOVE_FAIL, 0,
+                               "Failed to remove pmap "
+                               "registry for port %d for "
+                               "brick %s",
+                               brickinfo->port, brickinfo->path);
+                        ret = 0;
+                    }
+                }
+            }
+
+            if (brickpath)
+                GF_FREE(brickpath);
+
+            if (is_brick_mx_enabled() && glusterd_is_brick_started(brickinfo)) {
+                brick_proc = brickinfo->brick_proc;
+                if (!brick_proc)
+                    break;
+                cds_list_for_each_entry(brickinfo_tmp, &brick_proc->bricks,
+                                        mux_bricks)
+                {
+                    glusterd_set_brick_status(brickinfo_tmp, GF_BRICK_STOPPED);
+                    brickinfo_tmp->start_triggered = _gf_false;
+                    /* When bricks are stopped, ports also need to
+                     * be cleaned up
+                     */
+                    pmap_registry_remove(
+                        THIS, brickinfo_tmp->port, brickinfo_tmp->path,
+                        GF_PMAP_PORT_BRICKSERVER, NULL, _gf_true);
+                }
+            } else {
+                glusterd_set_brick_status(brickinfo, GF_BRICK_STOPPED);
+                brickinfo->start_triggered = _gf_false;
+            }
+            break;
 
         case RPC_CLNT_DESTROY:
-                GF_FREE (mydata);
-                mydata = NULL;
-                break;
+            GF_FREE(mydata);
+            mydata = NULL;
+            break;
         default:
-                gf_msg_trace (this->name, 0,
-                        "got some other RPC event %d", event);
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                           rpc_clnt_event_t event, void *data)
+glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify (rpc, mydata, event, data,
-                                           __glusterd_brick_rpc_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_brick_rpc_notify);
 }
 
 int
-glusterd_friend_remove_notify (glusterd_peerctx_t *peerctx, int32_t op_errno)
+glusterd_friend_remove_notify(glusterd_peerctx_t *peerctx, int32_t op_errno)
 {
-        int                             ret = -1;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        rpcsvc_request_t                *req = NULL;
-        char                            *errstr = NULL;
-        dict_t                          *dict = NULL;
+    int ret = -1;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpcsvc_request_t *req = NULL;
+    char *errstr = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(peerctx);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        gf_msg_debug(THIS->name, 0,
+                     "Could not find peer %s(%s). "
+                     "Peer could have been deleted.",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        ret = 0;
+        goto out;
+    }
 
-        GF_ASSERT (peerctx);
+    req = peerctx->args.req;
+    dict = peerctx->args.dict;
+    errstr = peerctx->errstr;
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                gf_msg_debug (THIS->name, 0, "Could not find peer %s(%s). "
-                        "Peer could have been deleted.", peerctx->peername,
-                        uuid_utoa (peerctx->peerid));
-                ret = 0;
-                goto out;
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_REMOVE_FRIEND,
+                                       &new_event);
+    if (!ret) {
+        if (!req) {
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+                   "Unable to find the request for responding "
+                   "to User (%s)",
+                   peerinfo->hostname);
+            goto out;
         }
 
-        req = peerctx->args.req;
-        dict = peerctx->args.dict;
-        errstr = peerctx->errstr;
-
-        ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
-                                            &new_event);
-        if (!ret) {
-                if (!req) {
-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                GD_MSG_EVENT_NEW_GET_FAIL,
-                                "Unable to find the request for responding "
-                                "to User (%s)", peerinfo->hostname);
-                        goto out;
-                }
+        glusterd_xfer_cli_probe_resp(req, -1, op_errno, errstr,
+                                     peerinfo->hostname, peerinfo->port, dict);
 
-                glusterd_xfer_cli_probe_resp (req, -1, op_errno, errstr,
-                                              peerinfo->hostname,
-                                              peerinfo->port, dict);
+        new_event->peername = gf_strdup(peerinfo->hostname);
+        gf_uuid_copy(new_event->peerid, peerinfo->uuid);
+        ret = glusterd_friend_sm_inject_event(new_event);
 
-                new_event->peername = gf_strdup (peerinfo->hostname);
-                gf_uuid_copy (new_event->peerid, peerinfo->uuid);
-                ret = glusterd_friend_sm_inject_event (new_event);
-
-        } else {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL,
-                        "Unable to create event for removing peer %s",
-                        peerinfo->hostname);
-        }
+    } else {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to create event for removing peer %s",
+               peerinfo->hostname);
+    }
 
 out:
-        rcu_read_unlock ();
-        return ret;
+    RCU_READ_UNLOCK;
+    return ret;
 }
 
 int
-__glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                            rpc_clnt_event_t event, void *data)
+__glusterd_peer_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data)
 {
-        xlator_t             *this        = NULL;
-        glusterd_conf_t      *conf        = NULL;
-        int                   ret         = 0;
-        int32_t               op_errno    = ENOTCONN;
-        glusterd_peerinfo_t  *peerinfo    = NULL;
-        glusterd_peerctx_t   *peerctx     = NULL;
-        gf_boolean_t         quorum_action = _gf_false;
-        glusterd_volinfo_t   *volinfo     = NULL;
-        uuid_t               uuid;
-
-        peerctx = mydata;
-        if (!peerctx)
-                return 0;
-
-        this = THIS;
-        conf = this->private;
-
-        if (RPC_CLNT_DESTROY == event) {
-                GF_FREE (peerctx->errstr);
-                GF_FREE (peerctx->peername);
-                GF_FREE (peerctx);
-                return 0;
-        }
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    int32_t op_errno = ENOTCONN;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    uuid_t uuid;
+
+    peerctx = mydata;
+    if (!peerctx)
+        return 0;
 
-        rcu_read_lock ();
+    this = THIS;
+    conf = this->private;
 
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                /* Peerinfo should be available at this point if its a connect
-                 * event. Not finding it means that something terrible has
-                 * happened. For non-connect event we might end up having a null
-                 * peerinfo, so log at debug level.
-                 */
-                gf_msg (THIS->name, (RPC_CLNT_CONNECT == event) ?
-                        GF_LOG_CRITICAL : GF_LOG_DEBUG, ENOENT,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer "
-                        "%s(%s)", peerctx->peername,
-                        uuid_utoa (peerctx->peerid));
+    switch (event) {
+        case RPC_CLNT_DESTROY:
+            GF_FREE(peerctx->errstr);
+            GF_FREE(peerctx->peername);
+            GF_FREE(peerctx);
+            return 0;
+        case RPC_CLNT_PING:
+            return 0;
+        default:
+            break;
+    }
+    ctx = this->ctx;
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
+    if (ctx->cleanup_started) {
+        gf_log(this->name, GF_LOG_INFO,
+               "glusterd already received a SIGTERM, "
+               "dropping the event %d for peer %s",
+               event, peerctx->peername);
+        return 0;
+    }
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        /* Peerinfo should be available at this point if its a connect
+         * event. Not finding it means that something terrible has
+         * happened. For non-connect event we might end up having a null
+         * peerinfo, so log at debug level.
+         */
+        gf_msg(THIS->name,
+               (RPC_CLNT_CONNECT == event) ? GF_LOG_CRITICAL : GF_LOG_DEBUG,
+               ENOENT, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer "
+               "%s(%s)",
+               peerctx->peername, uuid_utoa(peerctx->peerid));
 
-                ret = -1;
-                goto out;
+        if (RPC_CLNT_CONNECT == event) {
+            gf_event(EVENT_PEER_NOT_FOUND, "peer=%s;uuid=%s", peerctx->peername,
+                     uuid_utoa(peerctx->peerid));
         }
-
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                rpc_clnt_set_connected (&rpc->conn);
-                gf_msg_debug (this->name, 0, "got RPC_CLNT_CONNECT");
-                peerinfo->connected = 1;
-                peerinfo->quorum_action = _gf_true;
-                peerinfo->generation = uatomic_add_return
-                                                   (&conf->generation, 1);
-                peerctx->peerinfo_gen = peerinfo->generation;
-
-                ret = glusterd_peer_dump_version (this, rpc, peerctx);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_HANDSHAKE_FAILED,
-                                "glusterd handshake failed");
+        ret = -1;
+        goto out;
+    }
+
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            gf_msg_debug(this->name, 0, "got RPC_CLNT_CONNECT");
+            peerinfo->connected = 1;
+            peerinfo->quorum_action = _gf_true;
+            peerinfo->generation = uatomic_add_return(&conf->generation, 1);
+            peerctx->peerinfo_gen = peerinfo->generation;
+            /* EVENT_PEER_CONNECT will only be sent if peerctx->uuid is not
+             * NULL, otherwise it indicates this RPC_CLNT_CONNECT is from a
+             * peer probe trigger and given we already generate an event for
+             * peer probe this would be unnecessary.
+             */
+            if (!gf_uuid_is_null(peerinfo->uuid)) {
+                gf_event(EVENT_PEER_CONNECT, "host=%s;uuid=%s",
+                         peerinfo->hostname, uuid_utoa(peerinfo->uuid));
+            }
+            ret = glusterd_peer_dump_version(this, rpc, peerctx);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDSHAKE_FAILED,
+                       "glusterd handshake failed");
+            break;
+        }
+
+        case RPC_CLNT_DISCONNECT: {
+            /* If DISCONNECT event is already processed, skip the further
+             * ones
+             */
+            if (is_rpc_clnt_disconnected(&rpc->conn))
                 break;
-        }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                /* If DISCONNECT event is already processed, skip the further
-                 * ones
-                 */
-                if (is_rpc_clnt_disconnected (&rpc->conn))
-                        break;
-
-                rpc_clnt_unset_connected (&rpc->conn);
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_PEER_DISCONNECTED,
-                        "Peer <%s> (<%s>), in state <%s>, has disconnected "
-                        "from glusterd.",
-                        peerinfo->hostname, uuid_utoa (peerinfo->uuid),
-                        glusterd_friend_sm_state_name_get (peerinfo->state.state));
-
-                if (peerinfo->connected) {
-                        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                                glusterd_get_lock_owner (&uuid);
-                                if (!gf_uuid_is_null (uuid) &&
-                                    !gf_uuid_compare (peerinfo->uuid, uuid))
-                                        glusterd_unlock (peerinfo->uuid);
-                        } else {
-                                cds_list_for_each_entry (volinfo,
-                                                         &conf->volumes,
-                                                         vol_list) {
-                                        ret = glusterd_mgmt_v3_unlock
-                                                    (volinfo->volname,
-                                                     peerinfo->uuid,
-                                                     "vol");
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                        GF_LOG_WARNING, 0,
-                                                        GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                                        "Lock not released "
-                                                        "for %s",
-                                                         volinfo->volname);
-                                }
-                        }
-
-                        op_errno = GF_PROBE_ANOTHER_CLUSTER;
-                        ret = 0;
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_PEER_DISCONNECTED,
+                   "Peer <%s> (<%s>), in state <%s>, has disconnected "
+                   "from glusterd.",
+                   peerinfo->hostname, uuid_utoa(peerinfo->uuid),
+                   glusterd_friend_sm_state_name_get(peerinfo->state.state));
+            gf_event(EVENT_PEER_DISCONNECT, "peer=%s;uuid=%s;state=%s",
+                     peerinfo->hostname, uuid_utoa(peerinfo->uuid),
+                     glusterd_friend_sm_state_name_get(peerinfo->state.state));
+
+            if (peerinfo->connected) {
+                if (conf->op_version < GD_OP_VERSION_3_6_0) {
+                    glusterd_get_lock_owner(&uuid);
+                    if (!gf_uuid_is_null(uuid) &&
+                        !gf_uuid_compare(peerinfo->uuid, uuid))
+                        glusterd_unlock(peerinfo->uuid);
+                } else {
+                    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+                    {
+                        ret = glusterd_mgmt_v3_unlock(volinfo->volname,
+                                                      peerinfo->uuid, "vol");
+                        if (ret)
+                            gf_msg(this->name, GF_LOG_WARNING, 0,
+                                   GD_MSG_MGMTV3_UNLOCK_FAIL,
+                                   "Lock not released "
+                                   "for %s",
+                                   volinfo->volname);
+                    }
                 }
 
-                if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
-                    (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
-                        peerinfo->quorum_contrib = QUORUM_DOWN;
-                        quorum_action = _gf_true;
-                        peerinfo->quorum_action = _gf_false;
-                }
+                op_errno = GF_PROBE_ANOTHER_CLUSTER;
+                ret = 0;
+            }
 
-                /* Remove peer if it is not a friend and connection/handshake
-                *  fails, and notify cli. Happens only during probe.
-                */
-                if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT) {
-                        glusterd_friend_remove_notify (peerctx, op_errno);
-                        goto out;
-                }
+            if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
+                (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
+                peerinfo->quorum_contrib = QUORUM_DOWN;
+                quorum_action = _gf_true;
+                peerinfo->quorum_action = _gf_false;
+            }
 
-                peerinfo->connected = 0;
-                break;
+            /* Remove peer if it is not a friend and connection/handshake
+             *  fails, and notify cli. Happens only during probe.
+             */
+            if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT) {
+                glusterd_friend_remove_notify(peerctx, op_errno);
+                goto out;
+            }
+
+            peerinfo->connected = 0;
+            break;
         }
 
         default:
-                gf_msg_trace (this->name, 0,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            ret = 0;
+            break;
+    }
 
 out:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        return ret;
+    glusterd_friend_sm();
+    glusterd_op_sm();
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    return ret;
 }
 
 int
-glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event, void *data)
+glusterd_peer_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                         rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify (rpc, mydata, event, data,
-                                           __glusterd_peer_rpc_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_peer_rpc_notify);
 }
 
 int
-glusterd_null (rpcsvc_request_t *req)
+glusterd_null(rpcsvc_request_t *req)
 {
-
-        return 0;
+    return 0;
 }
 
-rpcsvc_actor_t gd_svc_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
-        [GLUSTERD_MGMT_NULL]           = { "NULL",           GLUSTERD_MGMT_NULL,           glusterd_null,                  NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = { "CLUSTER_LOCK",   GLUSTERD_MGMT_CLUSTER_LOCK,   glusterd_handle_cluster_lock,   NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = { "CLUSTER_UNLOCK", GLUSTERD_MGMT_CLUSTER_UNLOCK, glusterd_handle_cluster_unlock, NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_STAGE_OP]       = { "STAGE_OP",       GLUSTERD_MGMT_STAGE_OP,       glusterd_handle_stage_op,       NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_COMMIT_OP]      = { "COMMIT_OP",      GLUSTERD_MGMT_COMMIT_OP,      glusterd_handle_commit_op,      NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
+    [GLUSTERD_MGMT_NULL] = {"NULL", glusterd_null, NULL, GLUSTERD_MGMT_NULL,
+                            DRC_NA, 0},
+    [GLUSTERD_MGMT_CLUSTER_LOCK] = {"CLUSTER_LOCK",
+                                    glusterd_handle_cluster_lock, NULL,
+                                    GLUSTERD_MGMT_CLUSTER_LOCK, DRC_NA, 0},
+    [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK",
+                                      glusterd_handle_cluster_unlock, NULL,
+                                      GLUSTERD_MGMT_CLUSTER_UNLOCK, DRC_NA, 0},
+    [GLUSTERD_MGMT_STAGE_OP] = {"STAGE_OP", glusterd_handle_stage_op, NULL,
+                                GLUSTERD_MGMT_STAGE_OP, DRC_NA, 0},
+    [GLUSTERD_MGMT_COMMIT_OP] =
+        {
+            "COMMIT_OP",
+            glusterd_handle_commit_op,
+            NULL,
+            GLUSTERD_MGMT_COMMIT_OP,
+            DRC_NA,
+            0,
+        },
 };
 
 struct rpcsvc_program gd_svc_mgmt_prog = {
-        .progname  = "GlusterD svc mgmt",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_VERSION,
-        .numactors = GLUSTERD_MGMT_MAXVALUE,
-        .actors    = gd_svc_mgmt_actors,
-	.synctask  = _gf_true,
+    .progname = "GlusterD svc mgmt",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_VERSION,
+    .numactors = GLUSTERD_MGMT_MAXVALUE,
+    .actors = gd_svc_mgmt_actors,
+    .synctask = _gf_true,
 };
 
-rpcsvc_actor_t gd_svc_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
-        [GLUSTERD_FRIEND_NULL]    = { "NULL",          GLUSTERD_MGMT_NULL,     glusterd_null,                         NULL, 0, DRC_NA},
-        [GLUSTERD_PROBE_QUERY]    = { "PROBE_QUERY",   GLUSTERD_PROBE_QUERY,   glusterd_handle_probe_query,           NULL, 0, DRC_NA},
-        [GLUSTERD_FRIEND_ADD]     = { "FRIEND_ADD",    GLUSTERD_FRIEND_ADD,    glusterd_handle_incoming_friend_req,   NULL, 0, DRC_NA},
-        [GLUSTERD_FRIEND_REMOVE]  = { "FRIEND_REMOVE", GLUSTERD_FRIEND_REMOVE, glusterd_handle_incoming_unfriend_req, NULL, 0, DRC_NA},
-        [GLUSTERD_FRIEND_UPDATE]  = { "FRIEND_UPDATE", GLUSTERD_FRIEND_UPDATE, glusterd_handle_friend_update,         NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
+    [GLUSTERD_FRIEND_NULL] = {"NULL", glusterd_null, NULL, GLUSTERD_MGMT_NULL,
+                              DRC_NA, 0},
+    [GLUSTERD_PROBE_QUERY] = {"PROBE_QUERY", glusterd_handle_probe_query, NULL,
+                              GLUSTERD_PROBE_QUERY, DRC_NA, 0},
+    [GLUSTERD_FRIEND_ADD] = {"FRIEND_ADD", glusterd_handle_incoming_friend_req,
+                             NULL, GLUSTERD_FRIEND_ADD, DRC_NA, 0},
+    [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE",
+                                glusterd_handle_incoming_unfriend_req, NULL,
+                                GLUSTERD_FRIEND_REMOVE, DRC_NA, 0},
+    [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_handle_friend_update,
+                                NULL, GLUSTERD_FRIEND_UPDATE, DRC_NA, 0},
 };
 
 struct rpcsvc_program gd_svc_peer_prog = {
-        .progname  = "GlusterD svc peer",
-        .prognum   = GD_FRIEND_PROGRAM,
-        .progver   = GD_FRIEND_VERSION,
-        .numactors = GLUSTERD_FRIEND_MAXVALUE,
-        .actors    = gd_svc_peer_actors,
-	.synctask  = _gf_false,
+    .progname = "GlusterD svc peer",
+    .prognum = GD_FRIEND_PROGRAM,
+    .progver = GD_FRIEND_VERSION,
+    .numactors = GLUSTERD_FRIEND_MAXVALUE,
+    .actors = gd_svc_peer_actors,
+    .synctask = _gf_false,
 };
 
-
-
-rpcsvc_actor_t gd_svc_cli_actors[GLUSTER_CLI_MAXVALUE] = {
-        [GLUSTER_CLI_PROBE]              = { "CLI_PROBE",         GLUSTER_CLI_PROBE,            glusterd_handle_cli_probe,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_CREATE_VOLUME]      = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME,    glusterd_handle_create_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DEFRAG_VOLUME]      = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME,    glusterd_handle_defrag_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DEPROBE]            = { "FRIEND_REMOVE",     GLUSTER_CLI_DEPROBE,          glusterd_handle_cli_deprobe,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
-        [GLUSTER_CLI_UUID_RESET]         = { "UUID_RESET",        GLUSTER_CLI_UUID_RESET,       glusterd_handle_cli_uuid_reset,        NULL, 0, DRC_NA},
-        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_START_VOLUME]       = { "START_VOLUME",      GLUSTER_CLI_START_VOLUME,     glusterd_handle_cli_start_volume,      NULL, 0, DRC_NA},
-        [GLUSTER_CLI_STOP_VOLUME]        = { "STOP_VOLUME",       GLUSTER_CLI_STOP_VOLUME,      glusterd_handle_cli_stop_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DELETE_VOLUME]      = { "DELETE_VOLUME",     GLUSTER_CLI_DELETE_VOLUME,    glusterd_handle_cli_delete_volume,     NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
-        [GLUSTER_CLI_ADD_BRICK]          = { "ADD_BRICK",         GLUSTER_CLI_ADD_BRICK,        glusterd_handle_add_brick,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_ATTACH_TIER]        = { "ATTACH_TIER",       GLUSTER_CLI_ATTACH_TIER,      glusterd_handle_attach_tier,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DETACH_TIER]        = { "DETACH_TIER",       GLUSTER_CLI_DETACH_TIER,      glusterd_handle_detach_tier,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_REPLACE_BRICK]      = { "REPLACE_BRICK",     GLUSTER_CLI_REPLACE_BRICK,    glusterd_handle_replace_brick,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_REMOVE_BRICK]       = { "REMOVE_BRICK",      GLUSTER_CLI_REMOVE_BRICK,     glusterd_handle_remove_brick,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LOG_ROTATE]         = { "LOG FILENAME",      GLUSTER_CLI_LOG_ROTATE,       glusterd_handle_log_rotate,            NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SET_VOLUME]         = { "SET_VOLUME",        GLUSTER_CLI_SET_VOLUME,       glusterd_handle_set_volume,            NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SYNC_VOLUME]        = { "SYNC_VOLUME",       GLUSTER_CLI_SYNC_VOLUME,      glusterd_handle_sync_volume,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_RESET_VOLUME]       = { "RESET_VOLUME",      GLUSTER_CLI_RESET_VOLUME,     glusterd_handle_reset_volume,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_FSM_LOG]            = { "FSM_LOG",           GLUSTER_CLI_FSM_LOG,          glusterd_handle_fsm_log,               NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GSYNC_SET]          = { "GSYNC_SET",         GLUSTER_CLI_GSYNC_SET,        glusterd_handle_gsync_set,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_PROFILE_VOLUME]     = { "STATS_VOLUME",      GLUSTER_CLI_PROFILE_VOLUME,   glusterd_handle_cli_profile_volume,    NULL, 0, DRC_NA},
-        [GLUSTER_CLI_QUOTA]              = { "QUOTA",             GLUSTER_CLI_QUOTA,            glusterd_handle_quota,                 NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_MOUNT]              = { "MOUNT",             GLUSTER_CLI_MOUNT,            glusterd_handle_mount,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_UMOUNT]             = { "UMOUNT",            GLUSTER_CLI_UMOUNT,           glusterd_handle_umount,                NULL, 1, DRC_NA},
-        [GLUSTER_CLI_HEAL_VOLUME]        = { "HEAL_VOLUME",       GLUSTER_CLI_HEAL_VOLUME,      glusterd_handle_cli_heal_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_STATEDUMP_VOLUME]   = {"STATEDUMP_VOLUME",   GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume,  NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_CLRLOCKS_VOLUME]    = {"CLEARLOCKS_VOLUME",  GLUSTER_CLI_CLRLOCKS_VOLUME,  glusterd_handle_cli_clearlocks_volume, NULL, 0, DRC_NA},
-        [GLUSTER_CLI_COPY_FILE]          = {"COPY_FILE",          GLUSTER_CLI_COPY_FILE,        glusterd_handle_copy_file,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SYS_EXEC]           = {"SYS_EXEC",           GLUSTER_CLI_SYS_EXEC,         glusterd_handle_sys_exec,              NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SNAP]               = {"SNAP",               GLUSTER_CLI_SNAP,             glusterd_handle_snapshot,              NULL, 0, DRC_NA},
-        [GLUSTER_CLI_BARRIER_VOLUME]     = {"BARRIER_VOLUME",     GLUSTER_CLI_BARRIER_VOLUME,   glusterd_handle_barrier,               NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GANESHA]            = { "GANESHA"  ,         GLUSTER_CLI_GANESHA,          glusterd_handle_ganesha_cmd,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GET_VOL_OPT]        = {"GET_VOL_OPT",        GLUSTER_CLI_GET_VOL_OPT,      glusterd_handle_get_vol_opt,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_BITROT]             = {"BITROT",             GLUSTER_CLI_BITROT,           glusterd_handle_bitrot,                NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_cli_actors[GLUSTER_CLI_MAXVALUE] = {
+    [GLUSTER_CLI_PROBE] = {"CLI_PROBE", glusterd_handle_cli_probe, NULL,
+                           GLUSTER_CLI_PROBE, DRC_NA, 0},
+    [GLUSTER_CLI_CREATE_VOLUME] = {"CLI_CREATE_VOLUME",
+                                   glusterd_handle_create_volume, NULL,
+                                   GLUSTER_CLI_CREATE_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_DEFRAG_VOLUME] = {"CLI_DEFRAG_VOLUME",
+                                   glusterd_handle_defrag_volume, NULL,
+                                   GLUSTER_CLI_DEFRAG_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_DEPROBE] = {"FRIEND_REMOVE", glusterd_handle_cli_deprobe, NULL,
+                             GLUSTER_CLI_DEPROBE, DRC_NA, 0},
+    [GLUSTER_CLI_LIST_FRIENDS] = {"LIST_FRIENDS",
+                                  glusterd_handle_cli_list_friends, NULL,
+                                  GLUSTER_CLI_LIST_FRIENDS, DRC_NA, 0},
+    [GLUSTER_CLI_UUID_RESET] = {"UUID_RESET", glusterd_handle_cli_uuid_reset,
+                                NULL, GLUSTER_CLI_UUID_RESET, DRC_NA, 0},
+    [GLUSTER_CLI_UUID_GET] = {"UUID_GET", glusterd_handle_cli_uuid_get, NULL,
+                              GLUSTER_CLI_UUID_GET, DRC_NA, 0},
+    [GLUSTER_CLI_START_VOLUME] = {"START_VOLUME",
+                                  glusterd_handle_cli_start_volume, NULL,
+                                  GLUSTER_CLI_START_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_STOP_VOLUME] = {"STOP_VOLUME", glusterd_handle_cli_stop_volume,
+                                 NULL, GLUSTER_CLI_STOP_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_DELETE_VOLUME] = {"DELETE_VOLUME",
+                                   glusterd_handle_cli_delete_volume, NULL,
+                                   GLUSTER_CLI_DELETE_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_GET_VOLUME] = {"GET_VOLUME", glusterd_handle_cli_get_volume,
+                                NULL, GLUSTER_CLI_GET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_ADD_BRICK] = {"ADD_BRICK", glusterd_handle_add_brick, NULL,
+                               GLUSTER_CLI_ADD_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_ATTACH_TIER] = {"ATTACH_TIER", glusterd_handle_attach_tier,
+                                 NULL, GLUSTER_CLI_ATTACH_TIER, DRC_NA, 0},
+    [GLUSTER_CLI_REPLACE_BRICK] = {"REPLACE_BRICK",
+                                   glusterd_handle_replace_brick, NULL,
+                                   GLUSTER_CLI_REPLACE_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_REMOVE_BRICK] = {"REMOVE_BRICK", glusterd_handle_remove_brick,
+                                  NULL, GLUSTER_CLI_REMOVE_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_LOG_ROTATE] = {"LOG FILENAME", glusterd_handle_log_rotate,
+                                NULL, GLUSTER_CLI_LOG_ROTATE, DRC_NA, 0},
+    [GLUSTER_CLI_SET_VOLUME] = {"SET_VOLUME", glusterd_handle_set_volume, NULL,
+                                GLUSTER_CLI_SET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_SYNC_VOLUME] = {"SYNC_VOLUME", glusterd_handle_sync_volume,
+                                 NULL, GLUSTER_CLI_SYNC_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_RESET_VOLUME] = {"RESET_VOLUME", glusterd_handle_reset_volume,
+                                  NULL, GLUSTER_CLI_RESET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_FSM_LOG] = {"FSM_LOG", glusterd_handle_fsm_log, NULL,
+                             GLUSTER_CLI_FSM_LOG, DRC_NA, 0},
+    [GLUSTER_CLI_GSYNC_SET] = {"GSYNC_SET", glusterd_handle_gsync_set, NULL,
+                               GLUSTER_CLI_GSYNC_SET, DRC_NA, 0},
+    [GLUSTER_CLI_PROFILE_VOLUME] = {"STATS_VOLUME",
+                                    glusterd_handle_cli_profile_volume, NULL,
+                                    GLUSTER_CLI_PROFILE_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_QUOTA] = {"QUOTA", glusterd_handle_quota, NULL,
+                           GLUSTER_CLI_QUOTA, DRC_NA, 0},
+    [GLUSTER_CLI_GETWD] = {"GETWD", glusterd_handle_getwd, NULL,
+                           GLUSTER_CLI_GETWD, DRC_NA, 1},
+    [GLUSTER_CLI_STATUS_VOLUME] = {"STATUS_VOLUME",
+                                   glusterd_handle_status_volume, NULL,
+                                   GLUSTER_CLI_STATUS_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_MOUNT] = {"MOUNT", glusterd_handle_mount, NULL,
+                           GLUSTER_CLI_MOUNT, DRC_NA, 1},
+    [GLUSTER_CLI_UMOUNT] = {"UMOUNT", glusterd_handle_umount, NULL,
+                            GLUSTER_CLI_UMOUNT, DRC_NA, 1},
+    [GLUSTER_CLI_HEAL_VOLUME] = {"HEAL_VOLUME", glusterd_handle_cli_heal_volume,
+                                 NULL, GLUSTER_CLI_HEAL_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME",
+                                      glusterd_handle_cli_statedump_volume,
+                                      NULL, GLUSTER_CLI_STATEDUMP_VOLUME,
+                                      DRC_NA, 0},
+    [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", glusterd_handle_cli_list_volume,
+                                 NULL, GLUSTER_CLI_LIST_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_CLRLOCKS_VOLUME] = {"CLEARLOCKS_VOLUME",
+                                     glusterd_handle_cli_clearlocks_volume,
+                                     NULL, GLUSTER_CLI_CLRLOCKS_VOLUME, DRC_NA,
+                                     0},
+    [GLUSTER_CLI_COPY_FILE] = {"COPY_FILE", glusterd_handle_copy_file, NULL,
+                               GLUSTER_CLI_COPY_FILE, DRC_NA, 0},
+    [GLUSTER_CLI_SYS_EXEC] = {"SYS_EXEC", glusterd_handle_sys_exec, NULL,
+                              GLUSTER_CLI_SYS_EXEC, DRC_NA, 0},
+    [GLUSTER_CLI_SNAP] = {"SNAP", glusterd_handle_snapshot, NULL,
+                          GLUSTER_CLI_SNAP, DRC_NA, 0},
+    [GLUSTER_CLI_BARRIER_VOLUME] = {"BARRIER_VOLUME", glusterd_handle_barrier,
+                                    NULL, GLUSTER_CLI_BARRIER_VOLUME, DRC_NA,
+                                    0},
+    [GLUSTER_CLI_GANESHA] = {"GANESHA", glusterd_handle_ganesha_cmd, NULL,
+                             GLUSTER_CLI_GANESHA, DRC_NA, 0},
+    [GLUSTER_CLI_GET_VOL_OPT] = {"GET_VOL_OPT", glusterd_handle_get_vol_opt,
+                                 NULL, DRC_NA, 0},
+    [GLUSTER_CLI_BITROT] = {"BITROT", glusterd_handle_bitrot, NULL,
+                            GLUSTER_CLI_BITROT, DRC_NA, 0},
+    [GLUSTER_CLI_GET_STATE] = {"GET_STATE", glusterd_handle_get_state, NULL,
+                               GLUSTER_CLI_GET_STATE, DRC_NA, 0},
+    [GLUSTER_CLI_RESET_BRICK] = {"RESET_BRICK", glusterd_handle_reset_brick,
+                                 NULL, GLUSTER_CLI_RESET_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_TIER] = {"TIER", glusterd_handle_tier, NULL, GLUSTER_CLI_TIER,
+                          DRC_NA, 0},
+    [GLUSTER_CLI_REMOVE_TIER_BRICK] = {"REMOVE_TIER_BRICK",
+                                       glusterd_handle_tier, NULL,
+                                       GLUSTER_CLI_REMOVE_TIER_BRICK, DRC_NA,
+                                       0},
+    [GLUSTER_CLI_ADD_TIER_BRICK] = {"ADD_TIER_BRICK",
+                                    glusterd_handle_add_tier_brick, NULL,
+                                    GLUSTER_CLI_ADD_TIER_BRICK, DRC_NA, 0},
 };
 
 struct rpcsvc_program gd_svc_cli_prog = {
-        .progname  = "GlusterD svc cli",
-        .prognum   = GLUSTER_CLI_PROGRAM,
-        .progver   = GLUSTER_CLI_VERSION,
-        .numactors = GLUSTER_CLI_MAXVALUE,
-        .actors    = gd_svc_cli_actors,
-	.synctask  = _gf_true,
+    .progname = "GlusterD svc cli",
+    .prognum = GLUSTER_CLI_PROGRAM,
+    .progver = GLUSTER_CLI_VERSION,
+    .numactors = GLUSTER_CLI_MAXVALUE,
+    .actors = gd_svc_cli_actors,
+    .synctask = _gf_true,
 };
 
 /**
  * This set of RPC progs are deemed to be trusted. Most of the actors support
  * read only queries, the only exception being MOUNT/UMOUNT which is required
- * by geo-replication to supprt unprivileged master -> slave sessions.
+ * by geo-replication to support unprivileged master -> slave sessions.
  */
-rpcsvc_actor_t gd_svc_cli_trusted_actors[GLUSTER_CLI_MAXVALUE] = {
-        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
-        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_MOUNT]              = { "MOUNT",             GLUSTER_CLI_MOUNT,            glusterd_handle_mount,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_UMOUNT]             = { "UMOUNT",            GLUSTER_CLI_UMOUNT,           glusterd_handle_umount,                NULL, 1, DRC_NA},
+static rpcsvc_actor_t gd_svc_cli_trusted_actors[GLUSTER_CLI_MAXVALUE] = {
+    [GLUSTER_CLI_LIST_FRIENDS] = {"LIST_FRIENDS",
+                                  glusterd_handle_cli_list_friends, NULL,
+                                  GLUSTER_CLI_LIST_FRIENDS, DRC_NA, 0},
+    [GLUSTER_CLI_UUID_GET] = {"UUID_GET", glusterd_handle_cli_uuid_get, NULL,
+                              GLUSTER_CLI_UUID_GET, DRC_NA, 0},
+    [GLUSTER_CLI_GET_VOLUME] = {"GET_VOLUME", glusterd_handle_cli_get_volume,
+                                NULL, GLUSTER_CLI_GET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_GETWD] = {"GETWD", glusterd_handle_getwd, NULL,
+                           GLUSTER_CLI_GETWD, DRC_NA, 1},
+    [GLUSTER_CLI_STATUS_VOLUME] = {"STATUS_VOLUME",
+                                   glusterd_handle_status_volume, NULL,
+                                   GLUSTER_CLI_STATUS_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", glusterd_handle_cli_list_volume,
+                                 NULL, GLUSTER_CLI_LIST_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_MOUNT] = {"MOUNT", glusterd_handle_mount, NULL,
+                           GLUSTER_CLI_MOUNT, DRC_NA, 1},
+    [GLUSTER_CLI_UMOUNT] = {"UMOUNT", glusterd_handle_umount, NULL,
+                            GLUSTER_CLI_UMOUNT, DRC_NA, 1},
 };
 
 struct rpcsvc_program gd_svc_cli_trusted_progs = {
-        .progname  = "GlusterD svc cli read-only",
-        .prognum   = GLUSTER_CLI_PROGRAM,
-        .progver   = GLUSTER_CLI_VERSION,
-        .numactors = GLUSTER_CLI_MAXVALUE,
-        .actors    = gd_svc_cli_trusted_actors,
-	.synctask  = _gf_true,
+    .progname = "GlusterD svc cli read-only",
+    .prognum = GLUSTER_CLI_PROGRAM,
+    .progver = GLUSTER_CLI_VERSION,
+    .numactors = GLUSTER_CLI_MAXVALUE,
+    .actors = gd_svc_cli_trusted_actors,
+    .synctask = _gf_true,
 };
+
+/* As we cant remove the handlers, I'm moving the tier based
+ * handlers to this file as we no longer have gluster-tier.c
+ * and other tier.c files
+ */
+
+int
+glusterd_handle_tier(rpcsvc_request_t *req)
+{
+    return 0;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index 0ea66a027bf..d96e35503dd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -8,11 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "glusterfs.h"
-#include "syscall.h"
-#include "compat-errno.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat-errno.h>
 
 #include "glusterd.h"
 #include "glusterd-utils.h"
@@ -21,388 +21,504 @@
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-snapd-svc-helper.h"
+#include "glusterd-volgen.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-messages.h"
-
 #include "glusterfs3.h"
 #include "protocol-common.h"
 #include "rpcsvc.h"
 #include "rpc-common-xdr.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+#include "glusterd-shd-svc-helper.h"
 
 extern struct rpc_clnt_program gd_peer_prog;
 extern struct rpc_clnt_program gd_mgmt_prog;
 extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
+#define TRUSTED_PREFIX "trusted-"
+#define GD_PEER_ID_KEY "peer-id"
 
-#define TRUSTED_PREFIX         "trusted-"
-#define GD_PEER_ID_KEY         "peer-id"
-
-typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data);
+typedef ssize_t (*gfs_serialize_t)(struct iovec outmsg, void *data);
 
 static int
-get_snap_volname_and_volinfo (const char *volpath, char **volname,
-                              glusterd_volinfo_t **volinfo)
+get_snap_volname_and_volinfo(const char *volpath, char **volname,
+                             glusterd_volinfo_t **volinfo)
 {
-        int              ret            = -1;
-        char            *save_ptr       = NULL;
-        char            *str_token      = NULL;
-        char            *snapname       = NULL;
-        char            *volname_token  = NULL;
-        char            *vol            = NULL;
-        glusterd_snap_t *snap           = NULL;
-        xlator_t        *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volpath);
-        GF_ASSERT (volinfo);
-
-        str_token = gf_strdup (volpath);
-        if (NULL == str_token) {
-                goto out;
-        }
-
-        /* Input volname will have below formats:
-         * /snaps/<snapname>/<volname>.<hostname>
-         * or
-         * /snaps/<snapname>/<parent-volname>
-         * We need to extract snapname and parent_volname */
-
-        /*split string by "/" */
-        strtok_r (str_token, "/",  &save_ptr);
-        snapname  = strtok_r(NULL, "/", &save_ptr);
-        if (!snapname) {
-                gf_msg(this->name, GF_LOG_ERROR, EINVAL,
-                       GD_MSG_INVALID_ENTRY,
-                       "Invalid path: %s", volpath);
-                goto out;
-        }
-
-        volname_token = strtok_r(NULL, "/", &save_ptr);
-        if (!volname_token) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        EINVAL, GD_MSG_INVALID_ENTRY,
-                        "Invalid path: %s", volpath);
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_msg(this->name, GF_LOG_ERROR, EINVAL,
-                       GD_MSG_SNAP_NOT_FOUND, "Failed to "
-                       "fetch snap %s", snapname);
-                goto out;
-        }
-
-        /* Find if its a parent volume name or snap volume
-         * name. This function will succeed if volname_token
-         * is a parent volname
+    int ret = -1;
+    char *save_ptr = NULL;
+    char *str_token = NULL;
+    char *snapname = NULL;
+    char *volname_token = NULL;
+    char *vol = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    char *tmp_str_token = NULL;
+    char *volfile_token = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volpath);
+    GF_ASSERT(volinfo);
+
+    str_token = gf_strdup(volpath);
+    if (NULL == str_token) {
+        goto out;
+    }
+
+    tmp_str_token = str_token;
+
+    /* Input volname will have below formats:
+     * /snaps/<snapname>/<volname>.<hostname>
+     * or
+     * /snaps/<snapname>/<parent-volname>
+     * We need to extract snapname and parent_volname */
+
+    /*split string by "/" */
+    strtok_r(str_token, "/", &save_ptr);
+    snapname = strtok_r(NULL, "/", &save_ptr);
+    if (!snapname) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid path: %s", volpath);
+        goto out;
+    }
+
+    volname_token = strtok_r(NULL, "/", &save_ptr);
+    if (!volname_token) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid path: %s", volpath);
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Failed to "
+               "fetch snap %s",
+               snapname);
+        goto out;
+    }
+
+    /* Find if its a parent volume name or snap volume
+     * name. This function will succeed if volname_token
+     * is a parent volname
+     */
+    ret = glusterd_volinfo_find(volname_token, volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "failed to get the volinfo for the volume %s", volname_token);
+
+        /* Get the actual volfile name. */
+        volfile_token = strtok_r(NULL, "/", &save_ptr);
+        *volname = gf_strdup(volfile_token);
+        if (NULL == *volname) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    "Volname=%s", volfile_token, NULL);
+            ret = -1;
+            goto out;
+        }
+
+        /*
+         * Ideally, this should succeed as volname_token now contains
+         * the name of the snap volume (i.e. name of the volume that
+         * represents the snapshot). But, if for some reason, volinfo
+         * for the snap volume is not found, then try to get from the
+         * name of the volfile. Name of the volfile is like this.
+         * <snap volume name>.<hostname>.<brick path>.vol
          */
-        ret = glusterd_volinfo_find (volname_token, volinfo);
+        ret = glusterd_snap_volinfo_find(volname_token, snap, volinfo);
         if (ret) {
-                *volname = gf_strdup (volname_token);
-                if (NULL == *volname) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_snap_volinfo_find (volname_token, snap,
-                                                  volinfo);
-                if (ret) {
-                        /* Split the volume name */
-                        vol = strtok_r (volname_token, ".", &save_ptr);
-                        if (!vol) {
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_INVALID_ENTRY, "Invalid "
-                                                "volname (%s)", volname_token);
-                                goto out;
-                        }
-
-                        ret = glusterd_snap_volinfo_find (vol, snap, volinfo);
-                        if (ret) {
-                                gf_msg(this->name, GF_LOG_ERROR, 0,
-                                       GD_MSG_SNAP_INFO_FAIL, "Failed to "
-                                       "fetch snap volume from volname (%s)",
-                                       vol);
-                                goto out;
-                        }
-                }
-        } else {
-                /*volname_token is parent volname*/
-                ret = glusterd_snap_volinfo_find_from_parent_volname (
-                                volname_token, snap, volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_INFO_FAIL, "Failed to "
-                                "fetch snap volume from parent "
-                                "volname (%s)", volname_token);
-                        goto out;
-                }
+            /* Split the volume name */
+            vol = strtok_r(volfile_token, ".", &save_ptr);
+            if (!vol) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid "
+                       "volname (%s)",
+                       volfile_token);
+                goto out;
+            }
+
+            ret = glusterd_snap_volinfo_find(vol, snap, volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INFO_FAIL,
+                       "Failed to "
+                       "fetch snap volume from volname (%s)",
+                       vol);
+                goto out;
+            }
+        }
+    } else {
+        /*volname_token is parent volname*/
+        ret = glusterd_snap_volinfo_find_from_parent_volname(volname_token,
+                                                             snap, volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INFO_FAIL,
+                   "Failed to "
+                   "fetch snap volume from parent "
+                   "volname (%s)",
+                   volname_token);
+            goto out;
+        }
 
-                /* Since volname_token is a parent volname we should
-                 * get the snap volname here*/
-                *volname = gf_strdup ((*volinfo)->volname);
-                if (NULL == *volname) {
-                        ret = -1;
-                        goto out;
-                }
+        /* Since volname_token is a parent volname we should
+         * get the snap volname here*/
+        *volname = gf_strdup((*volinfo)->volname);
+        if (NULL == *volname) {
+            ret = -1;
+            goto out;
         }
+    }
 
 out:
-        if (ret && NULL != *volname) {
-                GF_FREE (*volname);
-                *volname = NULL;
-        }
-        return ret;
+    if (ret && NULL != *volname) {
+        GF_FREE(*volname);
+        *volname = NULL;
+    }
+
+    if (tmp_str_token)
+        GF_FREE(tmp_str_token);
+    return ret;
 }
 
 int32_t
-glusterd_get_client_per_brick_volfile (glusterd_volinfo_t *volinfo,
-                                       char *filename, char *path, int path_len)
+glusterd_get_client_per_brick_volfile(glusterd_volinfo_t *volinfo,
+                                      char *filename, char *path, int path_len)
 {
-        char                    workdir[PATH_MAX]      = {0,};
-        glusterd_conf_t        *priv                   = NULL;
-        int32_t                 ret                    = -1;
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("glusterd", THIS, out);
-        priv = THIS->private;
-        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+    GF_VALIDATE_OR_GOTO("glusterd", THIS, out);
+    priv = THIS->private;
+    GF_VALIDATE_OR_GOTO(THIS->name, priv, out);
 
-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
 
-        snprintf (path, path_len, "%s/%s", workdir, filename);
+    snprintf(path, path_len, "%s/%s", workdir, filename);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-static size_t
-build_volfile_path (char *volume_id, char *path,
-                    size_t path_len, char *trusted_str)
+size_t
+build_volfile_path(char *volume_id, char *path, size_t path_len,
+                   char *trusted_str, dict_t *dict)
 {
-        struct stat              stbuf                  = {0,};
-        int32_t                  ret                    = -1;
-        char                    *vol                    = NULL;
-        char                    *dup_volname            = NULL;
-        char                    *save_ptr               = NULL;
-        char                    *free_ptr               = NULL;
-        char                    *volname                = NULL;
-        char                    *volid_ptr              = NULL;
-        char                     dup_volid[PATH_MAX]    = {0,};
-        char                     path_prefix[PATH_MAX]  = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_volinfo_t      *volinfo                = NULL;
-        glusterd_conf_t         *priv                   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volume_id);
-        GF_ASSERT (path);
-
-        volid_ptr = strstr (volume_id, "snapd/");
-        if (volid_ptr) {
-                volid_ptr = strchr (volid_ptr, '/');
-                if (!volid_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volid_ptr++;
-
-                ret = glusterd_volinfo_find (volid_ptr, &volinfo);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL,
-                                "Couldn't find volinfo");
-                        goto out;
-                }
-                glusterd_svc_build_snapd_volfile (volinfo, path, path_len);
-                ret = 0;
-                goto out;
+    struct stat stbuf = {
+        0,
+    };
+    int32_t ret = -1;
+    char *vol = NULL;
+    char *dup_volname = NULL;
+    char *save_ptr = NULL;
+    char *free_ptr = NULL;
+    char *volname = NULL;
+    char *volid_ptr = NULL;
+    char dup_volid[PATH_MAX] = {
+        0,
+    };
+    char path_prefix[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volume_id);
+    GF_ASSERT(path);
+
+    volid_ptr = strstr(volume_id, "snapd/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        volid_ptr++;
+
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo");
+            goto out;
+        }
+        glusterd_svc_build_snapd_volfile(volinfo, path, path_len);
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "gluster/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        volid_ptr = strstr (volume_id, "gluster/");
-        if (volid_ptr) {
-                volid_ptr = strchr (volid_ptr, '/');
-                if (!volid_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volid_ptr++;
-
-                glusterd_svc_build_volfile_path (volid_ptr,
-                                                 priv->workdir,
-                                                 path, path_len);
-                ret = 0;
-                goto out;
+        glusterd_svc_build_volfile_path(volid_ptr, priv->workdir, path,
+                                        path_len);
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "gfproxy-client/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        volid_ptr = strstr (volume_id, "/snaps/");
-        if (volid_ptr) {
-                ret = get_snap_volname_and_volinfo (volid_ptr, &volname,
-                                                    &volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_INFO_FAIL, "Failed to get snap"
-                                " volinfo from path (%s)", volume_id);
-                        ret = -1;
-                        goto out;
-                }
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_log(this->name, GF_LOG_ERROR, "Couldn't find volinfo");
+            goto out;
+        }
 
-                snprintf (path_prefix, sizeof (path_prefix), "%s/snaps/%s",
-                          priv->workdir, volinfo->snapshot->snapname);
+        glusterd_get_gfproxy_client_volfile(volinfo, path, path_len);
 
-                volid_ptr = volname;
-                /* this is to ensure that volname recvd from
-                   get_snap_volname_and_volinfo is free'd */
-                free_ptr = volname;
-                goto gotvolinfo;
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "gfproxyd/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        volid_ptr = strstr (volume_id, "rebalance/");
-        if (volid_ptr) {
-                volid_ptr = strchr (volid_ptr, '/');
-                if (!volid_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volid_ptr++;
-
-                ret = glusterd_volinfo_find (volid_ptr, &volinfo);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL,
-                                "Couldn't find volinfo");
-                        goto out;
-                }
-                glusterd_get_rebalance_volfile (volinfo, path, path_len);
-                ret = 0;
-                goto out;
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_log(this->name, GF_LOG_ERROR, "Couldn't find volinfo");
+            goto out;
         }
 
-        volid_ptr = strstr (volume_id, "client_per_brick/");
-        if (volid_ptr) {
-                volid_ptr = strchr (volid_ptr, '/');
-                if (!volid_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volid_ptr++;
-
-                dup_volname = gf_strdup (volid_ptr);
-                if (!dup_volname) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY,
-                                "strdup failed");
-                        ret = -1;
-                        goto out;
-                }
-
-                /* Split the volume name */
-                vol = strtok_r (dup_volname, ".", &save_ptr);
-                if (!vol) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = glusterd_volinfo_find (vol, &volinfo);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL,
-                                "Couldn't find volinfo");
-                        goto out;
-                }
-                ret = glusterd_get_client_per_brick_volfile (volinfo, volid_ptr,
-                                                             path, path_len);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_NO_MEMORY, "failed to get volinfo path");
-                        goto out;
-                }
+        glusterd_svc_build_gfproxyd_volfile_path(volinfo, path, path_len);
+        ret = 0;
+        goto out;
+    }
 
-                ret = sys_access (path, F_OK);
-                goto out;
+    volid_ptr = strstr(volume_id, "shd/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        if (volume_id[0] == '/') {
-                /* Normal behavior */
-                volid_ptr = volume_id;
-                volid_ptr++;
-
-        } else {
-                /* Bringing in NFS like behavior for mount command, */
-                /* With this, one can mount a volume with below cmd */
-                /* bash# mount -t glusterfs server:/volume /mnt/pnt */
-                volid_ptr = volume_id;
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo for volid=%s", volid_ptr);
+            goto out;
         }
 
-        snprintf (path_prefix, sizeof (path_prefix), "%s/vols",
-                  priv->workdir);
+        glusterd_svc_build_shd_volfile_path(volinfo, path, path_len);
 
-        ret = glusterd_volinfo_find (volid_ptr, &volinfo);
+        ret = glusterd_svc_set_shd_pidfile(volinfo, dict);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set pidfile in dict for volid=%s", volid_ptr);
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "/snaps/");
+    if (volid_ptr) {
+        ret = get_snap_volname_and_volinfo(volid_ptr, &volname, &volinfo);
         if (ret) {
-                dup_volname = gf_strdup (volid_ptr);
-                if (!dup_volname) {
-                        ret = -1;
-                        goto out;
-                }
-                /* Split the volume name */
-                vol = strtok_r (dup_volname, ".", &save_ptr);
-                if (!vol) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = glusterd_volinfo_find (vol, &volinfo);
-                if (ret)
-                        goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INFO_FAIL,
+                   "Failed to get snap"
+                   " volinfo from path (%s)",
+                   volume_id);
+            ret = -1;
+            goto out;
+        }
+
+        len = snprintf(path_prefix, sizeof(path_prefix), "%s/snaps/%s",
+                       priv->workdir, volinfo->snapshot->snapname);
+        volid_ptr = volname;
+        /* this is to ensure that volname recvd from
+           get_snap_volname_and_volinfo is free'd */
+        free_ptr = volname;
+        if ((len < 0) || (len >= sizeof(path_prefix))) {
+            ret = -1;
+            goto out;
+        }
+
+        goto gotvolinfo;
+    }
+
+    volid_ptr = strstr(volume_id, "rebalance/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        volid_ptr++;
+
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo");
+            goto out;
+        }
+        glusterd_get_rebalance_volfile(volinfo, path, path_len);
+        ret = 0;
+        goto out;
+    }
+
+    volid_ptr = strstr(volume_id, "client_per_brick/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        volid_ptr++;
+
+        dup_volname = gf_strdup(volid_ptr);
+        if (!dup_volname) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "strdup failed");
+            ret = -1;
+            goto out;
+        }
+
+        /* Split the volume name */
+        vol = strtok_r(dup_volname, ".", &save_ptr);
+        if (!vol) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SPLIT_FAIL,
+                    "Volume name=%s", dup_volname, NULL);
+            ret = -1;
+            goto out;
+        }
+        ret = glusterd_volinfo_find(vol, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo");
+            goto out;
+        }
+        ret = glusterd_get_client_per_brick_volfile(volinfo, volid_ptr, path,
+                                                    path_len);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY,
+                   "failed to get volinfo path");
+            goto out;
         }
 
-gotvolinfo:
-        if (!glusterd_auth_get_username (volinfo))
-                trusted_str = NULL;
+        ret = sys_access(path, F_OK);
+        goto out;
+    }
 
-        ret = snprintf (path, path_len, "%s/%s/%s.vol", path_prefix,
-                        volinfo->volname, volid_ptr);
-        if (ret == -1)
-                goto out;
+    if (volume_id[0] == '/') {
+        /* Normal behavior */
+        volid_ptr = volume_id;
+        volid_ptr++;
 
-        ret = sys_stat (path, &stbuf);
-
-        if ((ret == -1) && (errno == ENOENT)) {
-                strncpy (dup_volid, volid_ptr, (PATH_MAX - 1));
-                if (!strchr (dup_volid, '.')) {
-                        switch (volinfo->transport_type) {
-                        case GF_TRANSPORT_TCP:
-                                strcat (dup_volid, ".tcp");
-                                break;
-                        case GF_TRANSPORT_RDMA:
-                                strcat (dup_volid, ".rdma");
-                                break;
-                        case GF_TRANSPORT_BOTH_TCP_RDMA:
-                                strcat (dup_volid, ".tcp");
-                                break;
-                        default:
-                                ret = -1;
-                                break;
-                        }
-                }
-                snprintf (path, path_len, "%s/%s/%s%s-fuse.vol",
-                          path_prefix, volinfo->volname,
-                          (trusted_str ? trusted_str : ""),
-                          dup_volid);
-                ret = sys_stat (path, &stbuf);
+    } else {
+        /* Bringing in NFS like behavior for mount command, */
+        /* With this, one can mount a volume with below cmd */
+        /* bash# mount -t glusterfs server:/volume /mnt/pnt */
+        volid_ptr = volume_id;
+    }
+
+    len = snprintf(path_prefix, sizeof(path_prefix), "%s/vols", priv->workdir);
+    if ((len < 0) || (len >= sizeof(path_prefix))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+
+    if (ret) {
+        dup_volname = gf_strdup(volid_ptr);
+        if (!dup_volname) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    "Volume name=%s", volid_ptr, NULL);
+            ret = -1;
+            goto out;
+        }
+        /* Split the volume name */
+        vol = strtok_r(dup_volname, ".", &save_ptr);
+        if (!vol) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SPLIT_FAIL,
+                    "Volume name=%s", dup_volname, NULL);
+            ret = -1;
+            goto out;
+        }
+        ret = glusterd_volinfo_find(vol, &volinfo);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL,
+                    NULL);
+            goto out;
         }
+    }
+
+gotvolinfo:
+    if (!glusterd_auth_get_username(volinfo))
+        trusted_str = NULL;
+
+    ret = snprintf(path, path_len, "%s/%s/%s.vol", path_prefix,
+                   volinfo->volname, volid_ptr);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = sys_stat(path, &stbuf);
+
+    if ((ret == -1) && (errno == ENOENT)) {
+        if (snprintf(dup_volid, PATH_MAX, "%s", volid_ptr) >= PATH_MAX)
+            goto out;
+        if (!strchr(dup_volid, '.')) {
+            switch (volinfo->transport_type) {
+                case GF_TRANSPORT_TCP:
+                    strcat(dup_volid, ".tcp");
+                    break;
+                case GF_TRANSPORT_RDMA:
+                    strcat(dup_volid, ".rdma");
+                    break;
+                case GF_TRANSPORT_BOTH_TCP_RDMA:
+                    strcat(dup_volid, ".tcp");
+                    break;
+                default:
+                    break;
+            }
+        }
+        snprintf(path, path_len, "%s/%s/%s%s-fuse.vol", path_prefix,
+                 volinfo->volname, (trusted_str ? trusted_str : ""), dup_volid);
+        ret = sys_stat(path, &stbuf);
+    }
 out:
-        if (dup_volname)
-                GF_FREE (dup_volname);
-        if (free_ptr)
-                GF_FREE (free_ptr);
-        return ret;
+    if (dup_volname)
+        GF_FREE(dup_volname);
+    if (free_ptr)
+        GF_FREE(free_ptr);
+    return ret;
 }
 
 /* Get and store op-versions of the clients sending the getspec request
@@ -410,642 +526,756 @@ out:
  * defaulted to 1. Also fetch brick_name.
  */
 int32_t
-glusterd_get_args_from_dict (gf_getspec_req *args, peer_info_t *peerinfo,
-                             char **brick_name)
+glusterd_get_args_from_dict(gf_getspec_req *args, peer_info_t *peerinfo,
+                            char **brick_name)
 {
-        dict_t    *dict                  = NULL;
-        int        client_max_op_version = 1;
-        int        client_min_op_version = 1;
-        int32_t    ret                   = -1;
-        xlator_t  *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (args);
-        GF_ASSERT (peerinfo);
-
-        if (!args->xdata.xdata_len) {
-                ret = 0;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (args->xdata.xdata_val,
-                                args->xdata.xdata_len, &dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "Failed to unserialize request dictionary");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "min-op-version",
-                              &client_min_op_version);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to get client-min-op-version");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "max-op-version",
-                              &client_max_op_version);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to get client-max-op-version");
-                goto out;
-        }
+    dict_t *dict = NULL;
+    int client_max_op_version = 1;
+    int client_min_op_version = 1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char *name = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(args);
+    GF_ASSERT(peerinfo);
+
+    if (!args->xdata.xdata_len) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "brick_name",
-                            brick_name);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "No brick name present");
-                ret = 0;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(args->xdata.xdata_val, args->xdata.xdata_len, &dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+               "Failed to unserialize request dictionary");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "min-op-version", &client_min_op_version);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get client-min-op-version");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "max-op-version", &client_max_op_version);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get client-max-op-version");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick_name", &name);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No brick name present");
+        ret = 0;
+        goto out;
+    }
+    *brick_name = gf_strdup(name);
+    if (*brick_name == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "Brick_name=%s", name, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        gf_msg_debug (this->name, 0, "brick_name = %s", *brick_name);
+    gf_msg_debug(this->name, 0, "brick_name = %s", *brick_name);
 out:
-        peerinfo->max_op_version = client_max_op_version;
-        peerinfo->min_op_version = client_min_op_version;
+    peerinfo->max_op_version = client_max_op_version;
+    peerinfo->min_op_version = client_min_op_version;
 
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 /* Given the missed_snapinfo and snap_opinfo take the
  * missed lvm snapshot
  */
 int32_t
-glusterd_create_missed_snap (glusterd_missed_snap_info *missed_snapinfo,
-                             glusterd_snap_op_t *snap_opinfo)
+glusterd_create_missed_snap(glusterd_missed_snap_info *missed_snapinfo,
+                            glusterd_snap_op_t *snap_opinfo)
 {
-        char                        *device           = NULL;
-        glusterd_conf_t             *priv             = NULL;
-        glusterd_snap_t             *snap             = NULL;
-        glusterd_volinfo_t          *snap_vol         = NULL;
-        glusterd_volinfo_t          *volinfo          = NULL;
-        glusterd_brickinfo_t        *brickinfo        = NULL;
-        int32_t                      ret              = -1;
-        int32_t                      i                = 0;
-        uuid_t                       snap_uuid        = {0,};
-        xlator_t                    *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (missed_snapinfo);
-        GF_ASSERT (snap_opinfo);
-
-        gf_uuid_parse (missed_snapinfo->snap_uuid, snap_uuid);
-
-        /* Find the snap-object */
-        snap = glusterd_find_snap_by_id (snap_uuid);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_NOT_FOUND,
-                        "Unable to find the snap with snap_uuid %s",
-                        missed_snapinfo->snap_uuid);
-                ret = -1;
-                goto out;
-        }
-
-        /* Find the snap_vol */
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                if (!strcmp (volinfo->volname,
-                             snap_opinfo->snap_vol_id)) {
-                        snap_vol = volinfo;
-                        break;
-                }
-        }
-
-        if (!snap_vol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND,
-                        "Unable to find the snap_vol(%s) "
-                        "for snap(%s)", snap_opinfo->snap_vol_id,
-                        snap->snapname);
-                ret = -1;
-                goto out;
-        }
-
-        /* Find the missed brick in the snap volume */
-        cds_list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
-                i++;
-                if (i == snap_opinfo->brick_num)
-                        break;
-        }
-
-        if (brickinfo->snap_status != -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_STATUS_NOT_PENDING,
-                        "The snap status of the missed "
-                        "brick(%s) is not pending", brickinfo->path);
-                goto out;
-        }
-
-        /* Fetch the device path */
-        device = glusterd_get_brick_mount_device (snap_opinfo->brick_path);
-        if (!device) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_GET_INFO_FAIL,
-                        "Getting device name for the"
-                        "brick %s:%s failed", brickinfo->hostname,
-                        snap_opinfo->brick_path);
-                ret = -1;
-                goto out;
-        }
-
-        device = glusterd_build_snap_device_path (device, snap_vol->volname,
-                                                  snap_opinfo->brick_num - 1);
-        if (!device) {
-                gf_msg (this->name, GF_LOG_ERROR, ENXIO,
-                        GD_MSG_SNAP_DEVICE_NAME_GET_FAIL,
-                        "cannot copy the snapshot "
-                        "device name (volname: %s, snapname: %s)",
-                         snap_vol->volname, snap->snapname);
-                ret = -1;
-                goto out;
-        }
-        strncpy (brickinfo->device_path, device,
-                 sizeof(brickinfo->device_path));
-
-        /* Update the backend file-system type of snap brick in
-         * snap volinfo. */
-        ret = glusterd_update_mntopts (snap_opinfo->brick_path, brickinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRK_MOUNTOPTS_FAIL, "Failed to update "
-                        "mount options for %s brick", brickinfo->path);
-                /* We should not fail snapshot operation if we fail to get
-                 * the file-system type */
-        }
-
-        ret = glusterd_take_lvm_snapshot (brickinfo, snap_opinfo->brick_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED,
-                        "Failed to take snapshot of %s",
-                        snap_opinfo->brick_path);
-                goto out;
-        }
-
-        /* After the snapshot both the origin brick (LVM brick) and
-         * the snapshot brick will have the same file-system label. This
-         * will cause lot of problems at mount time. Therefore we must
-         * generate a new label for the snapshot brick
+    char *device = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    int32_t i = 0;
+    uuid_t snap_uuid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char *mnt_device = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(missed_snapinfo);
+    GF_ASSERT(snap_opinfo);
+
+    gf_uuid_parse(missed_snapinfo->snap_uuid, snap_uuid);
+
+    /* Find the snap-object */
+    snap = glusterd_find_snap_by_id(snap_uuid);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Unable to find the snap with snap_uuid %s",
+               missed_snapinfo->snap_uuid);
+        ret = -1;
+        goto out;
+    }
+
+    /* Find the snap_vol */
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        if (!strcmp(volinfo->volname, snap_opinfo->snap_vol_id)) {
+            snap_vol = volinfo;
+            break;
+        }
+    }
+
+    if (!snap_vol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find the snap_vol(%s) "
+               "for snap(%s)",
+               snap_opinfo->snap_vol_id, snap->snapname);
+        ret = -1;
+        goto out;
+    }
+
+    /* Find the missed brick in the snap volume */
+    cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+    {
+        i++;
+        if (i == snap_opinfo->brick_num)
+            break;
+    }
+
+    if (brickinfo->snap_status != -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_NOT_PENDING,
+               "The snap status of the missed "
+               "brick(%s) is not pending",
+               brickinfo->path);
+        goto out;
+    }
+
+    /* Fetch the device path */
+    mnt_device = glusterd_get_brick_mount_device(snap_opinfo->brick_path);
+    if (!mnt_device) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "Getting device name for the"
+               "brick %s:%s failed",
+               brickinfo->hostname, snap_opinfo->brick_path);
+        ret = -1;
+        goto out;
+    }
+
+    device = glusterd_build_snap_device_path(mnt_device, snap_vol->volname,
+                                             snap_opinfo->brick_num - 1);
+    if (!device) {
+        gf_msg(this->name, GF_LOG_ERROR, ENXIO,
+               GD_MSG_SNAP_DEVICE_NAME_GET_FAIL,
+               "cannot copy the snapshot "
+               "device name (volname: %s, snapname: %s)",
+               snap_vol->volname, snap->snapname);
+        ret = -1;
+        goto out;
+    }
+    if (snprintf(brickinfo->device_path, sizeof(brickinfo->device_path), "%s",
+                 device) >= sizeof(brickinfo->device_path)) {
+        gf_msg(this->name, GF_LOG_ERROR, ENXIO,
+               GD_MSG_SNAP_DEVICE_NAME_GET_FAIL,
+               "cannot copy the device_path "
+               "(device_path: %s)",
+               brickinfo->device_path);
+        ret = -1;
+        goto out;
+    }
+
+    /* Update the backend file-system type of snap brick in
+     * snap volinfo. */
+    ret = glusterd_update_mntopts(snap_opinfo->brick_path, brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MOUNTOPTS_FAIL,
+               "Failed to update "
+               "mount options for %s brick",
+               brickinfo->path);
+        /* We should not fail snapshot operation if we fail to get
+         * the file-system type */
+    }
+
+    ret = glusterd_take_lvm_snapshot(brickinfo, snap_opinfo->brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+               "Failed to take snapshot of %s", snap_opinfo->brick_path);
+        goto out;
+    }
+
+    /* After the snapshot both the origin brick (LVM brick) and
+     * the snapshot brick will have the same file-system label. This
+     * will cause lot of problems at mount time. Therefore we must
+     * generate a new label for the snapshot brick
+     */
+    ret = glusterd_update_fs_label(brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SET_INFO_FAIL,
+               "Failed to update "
+               "file-system label for %s brick",
+               brickinfo->path);
+        /* Failing to update label should not cause snapshot failure.
+         * Currently label is updated only for XFS and ext2/ext3/ext4
+         * file-system.
          */
-        ret = glusterd_update_fs_label (brickinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_SET_INFO_FAIL, "Failed to update "
-                        "file-system label for %s brick", brickinfo->path);
-                /* Failing to update label should not cause snapshot failure.
-                 * Currently label is updated only for XFS and ext2/ext3/ext4
-                 * file-system.
-                 */
-        }
-
-        /* Create and mount the snap brick */
-        ret = glusterd_snap_brick_create (snap_vol, brickinfo,
-                                          snap_opinfo->brick_num - 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_CREATION_FAIL, "Failed to "
-                        " create and mount the brick(%s) for the snap %s",
-                        snap_opinfo->brick_path,
-                        snap_vol->snapshot->snapname);
-                goto out;
-        }
+    }
+
+    /* Create and mount the snap brick */
+    ret = glusterd_snap_brick_create(snap_vol, brickinfo,
+                                     snap_opinfo->brick_num - 1, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_CREATION_FAIL,
+               "Failed to "
+               " create and mount the brick(%s) for the snap %s",
+               snap_opinfo->brick_path, snap_vol->snapshot->snapname);
+        goto out;
+    }
+
+    brickinfo->snap_status = 0;
+    ret = glusterd_brick_start(snap_vol, brickinfo, _gf_false, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_DISCONNECTED,
+               "starting the "
+               "brick %s:%s for the snap %s failed",
+               brickinfo->hostname, brickinfo->path, snap->snapname);
+        goto out;
+    }
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+               "Failed to store snapshot "
+               "volinfo (%s) for snap %s",
+               snap_vol->volname, snap->snapname);
+        goto out;
+    }
 
-        brickinfo->snap_status = 0;
-        ret = glusterd_store_volinfo (snap_vol,
-                                      GLUSTERD_VOLINFO_VER_AC_NONE);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_STORE_FAIL, "Failed to store snapshot "
-                        "volinfo (%s) for snap %s", snap_vol->volname,
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = glusterd_brick_start (snap_vol, brickinfo, _gf_false);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_BRICK_DISCONNECTED, "starting the "
-                        "brick %s:%s for the snap %s failed",
-                        brickinfo->hostname, brickinfo->path,
-                        snap->snapname);
-                goto out;
-        }
 out:
-        if (device)
-                GF_FREE (device);
+    if (mnt_device)
+        GF_FREE(mnt_device);
+    if (device)
+        GF_FREE(device);
 
-        return ret;
+    return ret;
 }
 
 /* Look into missed_snap_list, to see it the given brick_name,
  * has any missed snap creates for the local node */
 int32_t
-glusterd_take_missing_brick_snapshots (char *brick_name)
+glusterd_take_missing_brick_snapshots(char *brick_name)
 {
-        char                        *my_node_uuid     = NULL;
-        glusterd_conf_t             *priv             = NULL;
-        glusterd_missed_snap_info   *missed_snapinfo  = NULL;
-        glusterd_snap_op_t          *snap_opinfo      = NULL;
-        int32_t                      ret              = -1;
-        gf_boolean_t                 update_list      = _gf_false;
-        xlator_t                    *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (brick_name);
-
-        my_node_uuid = uuid_utoa (MY_UUID);
-
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                /* If the missed snap op is not for the local node
-                 * then continue
+    char *my_node_uuid = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    int32_t ret = -1;
+    gf_boolean_t update_list = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(brick_name);
+
+    my_node_uuid = uuid_utoa(MY_UUID);
+
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        /* If the missed snap op is not for the local node
+         * then continue
+         */
+        if (strcmp(my_node_uuid, missed_snapinfo->node_uuid))
+            continue;
+
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            /* Check if the missed snap's op is a create for
+             * the brick name in question
+             */
+            if ((snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
+                (!strcmp(brick_name, snap_opinfo->brick_path))) {
+                /* Perform a snap create if the
+                 * op is still pending
                  */
-                if (strcmp (my_node_uuid, missed_snapinfo->node_uuid))
-                        continue;
-
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        /* Check if the missed snap's op is a create for
-                         * the brick name in question
+                if (snap_opinfo->status == GD_MISSED_SNAP_PENDING) {
+                    ret = glusterd_create_missed_snap(missed_snapinfo,
+                                                      snap_opinfo);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                               "Failed to create "
+                               "missed snap for %s",
+                               brick_name);
+                        /* At this stage, we will mark
+                         * the entry as done. Because
+                         * of the failure other
+                         * snapshots will not be
+                         * affected, and neither the
+                         * brick. Only the current snap
+                         * brick will always remain as
+                         * pending.
                          */
-                        if ((snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
-                            (!strcmp (brick_name, snap_opinfo->brick_path))) {
-                                /* Perform a snap create if the
-                                 * op is still pending
-                                 */
-                                if (snap_opinfo->status ==
-                                                 GD_MISSED_SNAP_PENDING) {
-                                        ret = glusterd_create_missed_snap
-                                                              (missed_snapinfo,
-                                                               snap_opinfo);
-                                        if (ret) {
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                                                        "Failed to create "
-                                                        "missed snap for %s",
-                                                        brick_name);
-                                                /* At this stage, we will mark
-                                                 * the entry as done. Because
-                                                 * of the failure other
-                                                 * snapshots will not be
-                                                 * affected, and neither the
-                                                 * brick. Only the current snap
-                                                 * brick will always remain as
-                                                 * pending.
-                                                 */
-                                        }
-                                        snap_opinfo->status =
-                                                 GD_MISSED_SNAP_DONE;
-                                        update_list = _gf_true;
-                                }
-                                /* One snap-id won't have more than one missed
-                                 * create for the same brick path. Hence
-                                 * breaking in search of another missed create
-                                 * for the same brick path in the local node
-                                 */
-                                break;
-                        }
+                    }
+                    snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                    update_list = _gf_true;
                 }
+                /* One snap-id won't have more than one missed
+                 * create for the same brick path. Hence
+                 * breaking in search of another missed create
+                 * for the same brick path in the local node
+                 */
+                break;
+            }
         }
+    }
 
-        if (update_list == _gf_true) {
-                ret = glusterd_store_update_missed_snaps ();
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
-                                "Failed to update missed_snaps_list");
-                        goto out;
-                }
+    if (update_list == _gf_true) {
+        ret = glusterd_store_update_missed_snaps();
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Failed to update missed_snaps_list");
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Checks if the client supports the volume, ie. client can understand all the
  * options in the volfile
  */
 static gf_boolean_t
-_client_supports_volume (peer_info_t *peerinfo, int32_t *op_errno)
+_client_supports_volume(peer_info_t *peerinfo, int32_t *op_errno)
 {
-        gf_boolean_t       ret       = _gf_true;
-        glusterd_volinfo_t *volinfo  = NULL;
-
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (op_errno);
-
-
-        /* Only check when the volfile being requested is a volume. Not finding
-         * a volinfo implies that the volfile requested for is not of a gluster
-         * volume. A non volume volfile is requested by the local gluster
-         * services like shd and nfs-server. These need not be checked as they
-         * will be running at the same op-version as glusterd and will be able
-         * to support all the features
-         */
-        if ((glusterd_volinfo_find (peerinfo->volname, &volinfo) == 0) &&
-            ((peerinfo->min_op_version > volinfo->client_op_version) ||
-             (peerinfo->max_op_version < volinfo->client_op_version))) {
-                ret = _gf_false;
-                *op_errno = ENOTSUP;
-                gf_msg ("glusterd", GF_LOG_INFO, ENOTSUP,
-                        GD_MSG_UNSUPPORTED_VERSION,
-                        "Client %s (%d -> %d) doesn't support required "
-                        "op-version (%d). Rejecting volfile request.",
-                        peerinfo->identifier, peerinfo->min_op_version,
-                        peerinfo->max_op_version, volinfo->client_op_version);
-        }
-
-        return ret;
+    gf_boolean_t ret = _gf_true;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(op_errno);
+
+    /* Only check when the volfile being requested is a volume. Not finding
+     * a volinfo implies that the volfile requested for is not of a gluster
+     * volume. A non volume volfile is requested by the local gluster
+     * services like shd and nfs-server. These need not be checked as they
+     * will be running at the same op-version as glusterd and will be able
+     * to support all the features
+     */
+    if ((glusterd_volinfo_find(peerinfo->volname, &volinfo) == 0) &&
+        ((peerinfo->min_op_version > volinfo->client_op_version) ||
+         (peerinfo->max_op_version < volinfo->client_op_version))) {
+        ret = _gf_false;
+        *op_errno = ENOTSUP;
+        gf_msg("glusterd", GF_LOG_INFO, ENOTSUP, GD_MSG_UNSUPPORTED_VERSION,
+               "Client %s (%d -> %d) doesn't support required "
+               "op-version (%d). Rejecting volfile request.",
+               peerinfo->identifier, peerinfo->min_op_version,
+               peerinfo->max_op_version, volinfo->client_op_version);
+    }
+
+    return ret;
 }
 
 int
-__server_getspec (rpcsvc_request_t *req)
+__server_getspec(rpcsvc_request_t *req)
 {
-        int32_t               ret                    = -1;
-        int32_t               op_ret                 = -1;
-        int32_t               op_errno               = 0;
-        int32_t               spec_fd                = -1;
-        size_t                file_len               = 0;
-        char                  filename[PATH_MAX]  = {0,};
-        struct stat           stbuf                  = {0,};
-        char                 *brick_name             = NULL;
-        char                 *volume                 = NULL;
-        char                 *tmp                    = NULL;
-        int                   cookie                 = 0;
-        rpc_transport_t      *trans                  = NULL;
-        gf_getspec_req        args                   = {0,};
-        gf_getspec_rsp        rsp                    = {0,};
-        char                  addrstr[RPCSVC_PEER_STRLEN] = {0};
-        peer_info_t          *peerinfo               = NULL;
-        xlator_t             *this                   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_getspec_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        peerinfo = &req->trans->peerinfo;
-
-        volume = args.key;
-        /* Need to strip leading '/' from volnames. This was introduced to
-         * support nfs style mount parameters for native gluster mount
-         */
-        if (volume[0] == '/')
-                strncpy (peerinfo->volname, &volume[1], strlen(&volume[1]));
-        else
-                strncpy (peerinfo->volname, volume, strlen(volume));
-
-        ret = glusterd_get_args_from_dict (&args, peerinfo, &brick_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to get args from dict");
-                goto fail;
-        }
-
-        if (!_client_supports_volume (peerinfo, &op_errno)) {
-                ret = -1;
-                goto fail;
-        }
-
-        trans = req->trans;
-        /* addrstr will be empty for cli socket connections */
-        ret = rpcsvc_transport_peername (trans, (char *)&addrstr,
-                                         sizeof (addrstr));
-        if (ret)
-                goto fail;
-
-        tmp  = strrchr (addrstr, ':');
-        if (tmp)
-                *tmp = '\0';
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int32_t spec_fd = -1;
+    size_t file_len = 0;
+    char filename[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    char *brick_name = NULL;
+    char *volume = NULL;
+    char *tmp = NULL;
+    rpc_transport_t *trans = NULL;
+    gf_getspec_req args = {
+        0,
+    };
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    char addrstr[RPCSVC_PEER_STRLEN] = {0};
+    peer_info_t *peerinfo = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    glusterd_peerinfo_t *peer = NULL;
+    glusterd_conf_t *conf = NULL;
+    int peer_cnt = 0;
+    char *peer_hosts = NULL;
+    char *tmp_str = NULL;
+    char portstr[10] = {
+        0,
+    };
+    int len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_getspec_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode the message");
+        goto fail;
+    }
+
+    peerinfo = &req->trans->peerinfo;
+
+    volume = args.key;
+
+    if (strlen(volume) >= (NAME_MAX)) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_NAME_TOO_LONG,
+               "volume name too long (%s)", volume);
+        goto fail;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MOUNT_REQ_RCVD,
+           "Received mount request for volume %s", volume);
+
+    /* Need to strip leading '/' from volnames. This was introduced to
+     * support nfs style mount parameters for native gluster mount
+     */
+    if (volume[0] == '/')
+        ret = snprintf(peerinfo->volname, sizeof(peerinfo->volname), "%s",
+                       &volume[1]);
+    else
+        ret = snprintf(peerinfo->volname, sizeof(peerinfo->volname), "%s",
+                       volume);
+    if (ret < 0 || ret >= sizeof(peerinfo->volname)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "peerinfo->volname %s truncated or error occurred: "
+               "(ret: %d)",
+               peerinfo->volname, ret);
+        ret = -1;
+        goto fail;
+    }
 
-        /* The trusted volfiles are given to the glusterd owned process like NFS
-         * server, self-heal daemon etc., so that they are not inadvertently
-         * blocked by a auth.{allow,reject} setting. The trusted volfile is not
-         * meant for external users.
-         * For unix domain socket, address will be empty.
-         */
-        if (strlen (addrstr) == 0 || gf_is_local_addr (addrstr)) {
+    ret = glusterd_get_args_from_dict(&args, peerinfo, &brick_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get args from dict");
+        goto fail;
+    }
 
-                ret = build_volfile_path (volume, filename,
-                                          sizeof (filename),
-                                          TRUSTED_PREFIX);
+    if (!_client_supports_volume(peerinfo, &op_errno)) {
+        ret = -1;
+        goto fail;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    trans = req->trans;
+    /* addrstr will be empty for cli socket connections */
+    ret = rpcsvc_transport_peername(trans, (char *)&addrstr, sizeof(addrstr));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_RPC_TRANSPORT_GET_PEERNAME_FAIL,
+               "Failed to get the peername");
+        goto fail;
+    }
+
+    tmp = strrchr(addrstr, ':');
+    if (tmp)
+        *tmp = '\0';
+
+    /* The trusted volfiles are given to the glusterd owned process like NFS
+     * server, self-heal daemon etc., so that they are not inadvertently
+     * blocked by a auth.{allow,reject} setting. The trusted volfile is not
+     * meant for external users.
+     * For unix domain socket, address will be empty.
+     */
+    if (strlen(addrstr) == 0 || gf_is_local_addr(addrstr)) {
+        ret = build_volfile_path(volume, filename, sizeof(filename),
+                                 TRUSTED_PREFIX, dict);
+    } else {
+        ret = build_volfile_path(volume, filename, sizeof(filename), NULL,
+                                 dict);
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &conf->peers, uuid_list)
+    {
+        if (!peer->connected)
+            continue;
+        if (!peer_hosts) {
+            if (peer->port) {
+                snprintf(portstr, sizeof(portstr), "%d", peer->port);
+            } else {
+                snprintf(portstr, sizeof(portstr), "%d", GLUSTERD_DEFAULT_PORT);
+            }
+            len = strlen(peer->hostname) + strlen(portstr) + 3;
+            tmp_str = GF_CALLOC(1, len, gf_gld_mt_char);
+            snprintf(tmp_str, len, "%s%s%s%s", peer->hostname, ":", portstr,
+                     " ");
+            peer_hosts = tmp_str;
         } else {
-                ret = build_volfile_path (volume, filename,
-                                          sizeof (filename), NULL);
+            len = strlen(peer_hosts) + strlen(peer->hostname) +
+                  strlen(portstr) + 3;
+            tmp_str = GF_CALLOC(1, len, gf_gld_mt_char);
+            snprintf(tmp_str, len, "%s%s%s%s%s", peer_hosts, peer->hostname,
+                     ":", portstr, " ");
+            GF_FREE(peer_hosts);
+            peer_hosts = tmp_str;
+        }
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+    if (peer_cnt) {
+        op_ret = dict_set_str(dict, GLUSTERD_BRICK_SERVERS, peer_hosts);
+        if (op_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peer_host in dict");
+            ret = op_ret;
+            goto fail;
+        }
+    }
+
+    if (ret == 0) {
+        if (dict->count > 0) {
+            ret = dict_allocate_and_serialize(dict, &rsp.xdata.xdata_val,
+                                              &rsp.xdata.xdata_len);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+                goto fail;
+            }
         }
 
-        if (ret == 0) {
-                /* to allocate the proper buffer to hold the file data */
-                ret = sys_stat (filename, &stbuf);
-                if (ret < 0){
-                        gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Unable to stat %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
+        /* to allocate the proper buffer to hold the file data */
+        ret = sys_stat(filename, &stbuf);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to stat %s (%s)", filename, strerror(errno));
+            goto fail;
+        }
+
+        spec_fd = open(filename, O_RDONLY);
+        if (spec_fd < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to open %s (%s)", filename, strerror(errno));
+            goto fail;
+        }
+        ret = file_len = stbuf.st_size;
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
+        op_errno = ENOENT;
+        goto fail;
+    }
+
+    if (file_len) {
+        rsp.spec = CALLOC(file_len + 1, sizeof(char));
+        if (!rsp.spec) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+            ret = -1;
+            op_errno = ENOMEM;
+            goto fail;
+        }
+        ret = sys_read(spec_fd, rsp.spec, file_len);
+    }
+
+    if (brick_name) {
+        gf_msg_debug(this->name, 0, "Look for missing snap creates for %s",
+                     brick_name);
+        op_ret = glusterd_take_missing_brick_snapshots(brick_name);
+        if (op_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                   "Failed to take missing brick snapshots");
+            ret = -1;
+            goto fail;
+        }
+    }
+    /* convert to XDR */
+fail:
+    if (spec_fd >= 0)
+        sys_close(spec_fd);
 
-                spec_fd = open (filename, O_RDONLY);
-                if (spec_fd < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Unable to open %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
-                ret = file_len = stbuf.st_size;
-        } else {
-                op_errno = ENOENT;
-                goto fail;
-        }
+    GF_FREE(brick_name);
 
-        if (file_len) {
-                rsp.spec = CALLOC (file_len+1, sizeof (char));
-                if (!rsp.spec) {
-                        ret = -1;
-                        op_errno = ENOMEM;
-                        goto fail;
-                }
-                ret = sys_read (spec_fd, rsp.spec, file_len);
-        }
+    rsp.op_ret = ret;
+    if (rsp.op_ret < 0)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MOUNT_REQ_FAIL,
+               "Failed to mount the volume");
 
-        if (brick_name) {
-                gf_msg_debug (this->name, 0,
-                        "Look for missing snap creates for %s", brick_name);
-                op_ret = glusterd_take_missing_brick_snapshots (brick_name);
-                if (op_ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                                "Failed to take missing brick snapshots");
-                        ret = -1;
-                        goto fail;
-                }
-        }
+    if (op_errno)
+        rsp.op_errno = gf_errno_to_error(op_errno);
 
-        /* convert to XDR */
-fail:
-        if (spec_fd > 0)
-                sys_close (spec_fd);
+    if (!rsp.spec)
+        rsp.spec = strdup("");
 
-        rsp.op_ret   = ret;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_getspec_rsp);
+    free(args.key);  // malloced by xdr
+    free(rsp.spec);
 
-        if (op_errno)
-                rsp.op_errno = gf_errno_to_error (op_errno);
-        if (cookie)
-                rsp.op_errno = cookie;
+    if (peer_hosts)
+        GF_FREE(peer_hosts);
+    if (dict)
+        dict_unref(dict);
 
-        if (!rsp.spec)
-                rsp.spec = strdup ("");
+    if (args.xdata.xdata_val)
+        free(args.xdata.xdata_val);
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_getspec_rsp);
-        free (args.key);//malloced by xdr
-        free (rsp.spec);
+    if (rsp.xdata.xdata_val)
+        GF_FREE(rsp.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int
-server_getspec (rpcsvc_request_t *req)
+server_getspec(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __server_getspec);
+    return glusterd_big_locked_handler(req, __server_getspec);
 }
 
 int32_t
-__server_event_notify (rpcsvc_request_t *req)
+__server_event_notify(rpcsvc_request_t *req)
 {
-        int32_t                 ret             = -1;
-        int32_t                 op_errno        =  0;
-        gf_event_notify_req     args            = {0,};
-        gf_event_notify_rsp     rsp             = {0,};
-        dict_t                 *dict            = NULL;
-        gf_boolean_t            need_rsp        = _gf_true;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_event_notify_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
+    int32_t ret = -1;
+    gf_event_notify_req args = {
+        0,
+    };
+    gf_event_notify_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    gf_boolean_t need_rsp = _gf_true;
+
+    ret = xdr_to_generic(req->msg[0], &args,
+                         (xdrproc_t)xdr_gf_event_notify_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    if (args.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            return ret;
         }
-
-        if (args.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        return ret;
-                ret = dict_unserialize (args.dict.dict_val,
-                                        args.dict.dict_len, &dict);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "Failed to unserialize req");
-                        goto fail;
-                }
+        ret = dict_unserialize(args.dict.dict_val, args.dict.dict_len, &dict);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to unserialize req");
+            goto fail;
         }
+    }
 
-        switch (args.op) {
+    switch (args.op) {
         case GF_EN_DEFRAG_STATUS:
-                gf_msg ("glusterd", GF_LOG_INFO, 0,
-                        GD_MSG_DEFRAG_STATUS_UPDATED,
-                        "received defrag status updated");
-                if (dict) {
-                        glusterd_defrag_event_notify_handle (dict);
-                        need_rsp = _gf_false;
-                }
-                break;
+            gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEFRAG_STATUS_UPDATED,
+                   "received defrag status updated");
+            if (dict) {
+                glusterd_defrag_event_notify_handle(dict);
+                need_rsp = _gf_false;
+            }
+            break;
         default:
-                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                        GD_MSG_OP_UNSUPPORTED, "Unknown op received in event "
-                        "notify");
-                ret = -1;
-                break;
-        }
+            gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_OP_UNSUPPORTED,
+                   "Unknown op received in event "
+                   "notify");
+            gf_event(EVENT_NOTIFY_UNKNOWN_OP, "op=%d", args.op);
+            ret = -1;
+            break;
+    }
 
 fail:
-        rsp.op_ret   = ret;
-
-        if (op_errno)
-                rsp.op_errno = gf_errno_to_error (op_errno);
+    rsp.op_ret = ret;
 
-        if (need_rsp)
-                glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                       (xdrproc_t)xdr_gf_event_notify_rsp);
-        if (dict)
-                dict_unref (dict);
-        free (args.dict.dict_val);//malloced by xdr
+    if (need_rsp)
+        glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                              (xdrproc_t)xdr_gf_event_notify_rsp);
+    if (dict)
+        dict_unref(dict);
+    free(args.dict.dict_val);  // malloced by xdr
 
-        return 0;
+    return 0;
 }
 
 int32_t
-server_event_notify (rpcsvc_request_t *req)
+server_event_notify(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __server_event_notify);
+    return glusterd_big_locked_handler(req, __server_event_notify);
 }
 
 int
-gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version,
-                                char *peerid)
+gd_validate_cluster_op_version(xlator_t *this, int cluster_op_version,
+                               char *peerid)
 {
-        int              ret  = -1;
-        glusterd_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        if (cluster_op_version > GD_OP_VERSION_MAX) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_VERSION_MISMATCH,
-                        "operating version %d is more than the maximum "
-                        "supported (%d) on the machine (as per peer request "
-                        "from %s)", cluster_op_version, GD_OP_VERSION_MAX,
-                        peerid);
-                goto out;
-        }
-
-        /* The peer can only reduce its op-version when it doesn't have any
-         * volumes. Reducing op-version when it already contains volumes can
-         * lead to inconsistencies in the cluster
-         */
-        if ((cluster_op_version < conf->op_version) &&
-            !cds_list_empty (&conf->volumes)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_VERS_ADJUST_FAIL,
-                        "cannot reduce operating version to %d from current "
-                        "version %d as volumes exist (as per peer request from "
-                        "%s)", cluster_op_version, conf->op_version, peerid);
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (cluster_op_version > GD_OP_VERSION_MAX) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+               "operating version %d is more than the maximum "
+               "supported (%d) on the machine (as per peer request "
+               "from %s)",
+               cluster_op_version, GD_OP_VERSION_MAX, peerid);
+        goto out;
+    }
+
+    /* The peer can only reduce its op-version when it doesn't have any
+     * volumes. Reducing op-version when it already contains volumes can
+     * lead to inconsistencies in the cluster
+     */
+    if ((cluster_op_version < conf->op_version) &&
+        !cds_list_empty(&conf->volumes)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_ADJUST_FAIL,
+               "cannot reduce operating version to %d from current "
+               "version %d as volumes exist (as per peer request from "
+               "%s)",
+               cluster_op_version, conf->op_version, peerid);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Validate if glusterd can serve the management handshake request
@@ -1059,1231 +1289,1292 @@ out:
  *  - the incoming request address is matched with the peer list
  */
 gf_boolean_t
-gd_validate_mgmt_hndsk_req (rpcsvc_request_t *req, dict_t *dict)
+gd_validate_mgmt_hndsk_req(rpcsvc_request_t *req, dict_t *dict)
 {
-        int                  ret                         = -1;
-        char                 hostname[UNIX_PATH_MAX + 1] = {0,};
-        glusterd_peerinfo_t *peer                        = NULL;
-        xlator_t            *this                        = NULL;
-        char                *uuid_str                    = NULL;
-        uuid_t               peer_uuid                   = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!glusterd_have_peers () && !glusterd_have_volumes ())
-                return _gf_true;
-
-        ret = dict_get_str (dict, GD_PEER_ID_KEY, &uuid_str);
-        /* Try to match uuid only if available, don't fail as older peers will
-         * not send a uuid
-         */
-        if (!ret) {
-                gf_uuid_parse (uuid_str, peer_uuid);
-                rcu_read_lock ();
-                ret = (glusterd_peerinfo_find (peer_uuid, NULL) != NULL);
-                rcu_read_unlock ();
-                if (ret)
-                        return _gf_true;
-        }
+    int ret = -1;
+    char hostname[UNIX_PATH_MAX + 1] = {
+        0,
+    };
+    glusterd_peerinfo_t *peer = NULL;
+    xlator_t *this = NULL;
+    char *uuid_str = NULL;
+    uuid_t peer_uuid = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!glusterd_have_peers() && !glusterd_have_volumes())
+        return _gf_true;
 
-        /* If you cannot get the hostname, you cannot authenticate */
-        ret = glusterd_remote_hostname_get (req, hostname, sizeof (hostname));
+    ret = dict_get_str(dict, GD_PEER_ID_KEY, &uuid_str);
+    /* Try to match uuid only if available, don't fail as older peers will
+     * not send a uuid
+     */
+    if (!ret) {
+        gf_uuid_parse(uuid_str, peer_uuid);
+        RCU_READ_LOCK;
+        ret = (glusterd_peerinfo_find(peer_uuid, NULL) != NULL);
+        RCU_READ_UNLOCK;
         if (ret)
-                return _gf_false;
-
-        /* If peer object is not found it indicates that request is from an
-         * unknown peer, if its found, validate whether its uuid is also
-         * available in the peerinfo list. There could be a case where hostname
-         * is available in the peerinfo list but the uuid has changed of the
-         * node due to a reinstall, in that case the validation should fail!
-         */
-        rcu_read_lock ();
-        peer = glusterd_peerinfo_find (NULL, hostname);
+            return _gf_true;
+    }
+
+    /* If you cannot get the hostname, you cannot authenticate */
+    ret = glusterd_remote_hostname_get(req, hostname, sizeof(hostname));
+    if (ret)
+        return _gf_false;
+
+    /* If peer object is not found it indicates that request is from an
+     * unknown peer, if its found, validate whether its uuid is also
+     * available in the peerinfo list. There could be a case where hostname
+     * is available in the peerinfo list but the uuid has changed of the
+     * node due to a reinstall, in that case the validation should fail!
+     */
+    RCU_READ_LOCK;
+    if (!uuid_str) {
+        ret = (glusterd_peerinfo_find(NULL, hostname) == NULL);
+    } else {
+        peer = glusterd_peerinfo_find(NULL, hostname);
         if (!peer) {
-                ret = -1;
-        } else if (peer && glusterd_peerinfo_find (peer_uuid, NULL) != NULL) {
-                ret = 0;
+            ret = -1;
+        } else if (peer && glusterd_peerinfo_find(peer_uuid, NULL) != NULL) {
+            ret = 0;
         } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HANDSHAKE_REQ_REJECTED, "Request from peer %s "
-                        "has an entry in peerinfo, but uuid does not match",
-                        req->trans->peerinfo.identifier);
-                ret = -1;
-        }
-        rcu_read_unlock ();
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HANDSHAKE_REQ_REJECTED, "Rejecting management "
-                        "handshake request from unknown peer %s",
-                        req->trans->peerinfo.identifier);
-                return _gf_false;
-        }
-
-        return _gf_true;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDSHAKE_REQ_REJECTED,
+                   "Request from "
+                   "peer %s has an entry in peerinfo, but uuid "
+                   "does not match",
+                   req->trans->peerinfo.identifier);
+            ret = -1;
+        }
+    }
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDSHAKE_REQ_REJECTED,
+               "Rejecting management "
+               "handshake request from unknown peer %s",
+               req->trans->peerinfo.identifier);
+        gf_event(EVENT_PEER_REJECT, "peer=%s", req->trans->peerinfo.identifier);
+        return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 int
-__glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+__glusterd_mgmt_hndsk_versions(rpcsvc_request_t *req)
 {
-        dict_t            *dict            = NULL;
-        xlator_t          *this            = NULL;
-        glusterd_conf_t   *conf            = NULL;
-        int                ret             = -1;
-        int                op_errno        = EINVAL;
-        gf_mgmt_hndsk_req  args            = {{0,},};
-        gf_mgmt_hndsk_rsp  rsp             = {0,};
-        dict_t            *args_dict       = NULL;
-
-        this = THIS;
-        conf = this->private;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, args_dict, args.hndsk.hndsk_val,
-                                      (args.hndsk.hndsk_len), ret, op_errno,
-                                      out);
-
-        /* Check if we can service the request */
-        if (!gd_validate_mgmt_hndsk_req (req, args_dict)) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_int32 (dict, GD_OP_VERSION_KEY, conf->op_version);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "failed to set operating version");
-                rsp.op_ret = ret;
-                goto out;
-        }
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+    gf_mgmt_hndsk_req args = {
+        {
+            0,
+        },
+    };
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+    dict_t *args_dict = NULL;
+
+    this = THIS;
+    conf = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, args_dict, args.hndsk.hndsk_val,
+                                 (args.hndsk.hndsk_len), ret, op_errno, out);
+
+    /* Check if we can service the request */
+    if (!gd_validate_mgmt_hndsk_req(req, args_dict)) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, GD_OP_VERSION_KEY, conf->op_version);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set operating version");
+        rsp.op_ret = ret;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "failed to set %s", GD_MIN_OP_VERSION_KEY);
-                rsp.op_ret = ret;
-                goto out;
-        }
+    ret = dict_set_int32(dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set %s", GD_MIN_OP_VERSION_KEY);
+        rsp.op_ret = ret;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "failed to set %s", GD_MAX_OP_VERSION_KEY);
-                rsp.op_ret = ret;
-                goto out;
-        }
+    ret = dict_set_int32(dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set %s", GD_MAX_OP_VERSION_KEY);
+        rsp.op_ret = ret;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, dict, (&rsp.hndsk.hndsk_val),
-                                    rsp.hndsk.hndsk_len, op_errno, out);
+    GF_PROTOCOL_DICT_SERIALIZE(this, dict, (&rsp.hndsk.hndsk_val),
+                               rsp.hndsk.hndsk_len, op_errno, out);
 out:
 
-        rsp.op_ret = ret;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = ret;
+    rsp.op_errno = op_errno;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
 
-        ret = 0;
+    ret = 0;
+
+    if (dict)
+        dict_unref(dict);
 
-        if (dict)
-                dict_unref (dict);
+    if (args.hndsk.hndsk_val)
+        free(args.hndsk.hndsk_val);
 
-        if (args.hndsk.hndsk_val)
-                free (args.hndsk.hndsk_val);
+    if (rsp.hndsk.hndsk_val)
+        GF_FREE(rsp.hndsk.hndsk_val);
 
-        if (rsp.hndsk.hndsk_val)
-                GF_FREE (rsp.hndsk.hndsk_val);
+    if (args_dict)
+        dict_unref(args_dict);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+glusterd_mgmt_hndsk_versions(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_mgmt_hndsk_versions);
+    return glusterd_big_locked_handler(req, __glusterd_mgmt_hndsk_versions);
 }
 
 int
-__glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+__glusterd_mgmt_hndsk_versions_ack(rpcsvc_request_t *req)
 {
-        dict_t            *clnt_dict       = NULL;
-        xlator_t          *this            = NULL;
-        glusterd_conf_t   *conf            = NULL;
-        int                ret             = -1;
-        int                op_errno        = EINVAL;
-        int                peer_op_version = 0;
-        gf_mgmt_hndsk_req  args            = {{0,},};
-        gf_mgmt_hndsk_rsp  rsp             = {0,};
-
-        this = THIS;
-        conf = this->private;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, clnt_dict, args.hndsk.hndsk_val,
-                                      (args.hndsk.hndsk_len), ret, op_errno,
-                                      out);
-
-        ret = dict_get_int32 (clnt_dict, GD_OP_VERSION_KEY, &peer_op_version);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "failed to get the op-version key peer=%s",
-                        req->trans->peerinfo.identifier);
-                goto out;
-        }
-
-        ret = gd_validate_cluster_op_version (this, peer_op_version,
-                                              req->trans->peerinfo.identifier);
-        if (ret)
-                goto out;
-
-
-        /* As this is ACK from the Cluster for the versions supported,
-           can set the op-version of 'this' glusterd to the one
-           received. */
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_VERS_INFO, "using the op-version %d",
-                peer_op_version);
-        conf->op_version = peer_op_version;
-        ret = glusterd_store_global_info (this);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLOBAL_OP_VERSION_SET_FAIL,
-                        "Failed to store op-version");
+    dict_t *clnt_dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+    int peer_op_version = 0;
+    gf_mgmt_hndsk_req args = {
+        {
+            0,
+        },
+    };
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+
+    this = THIS;
+    conf = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, clnt_dict, args.hndsk.hndsk_val,
+                                 (args.hndsk.hndsk_len), ret, op_errno, out);
+
+    ret = dict_get_int32(clnt_dict, GD_OP_VERSION_KEY, &peer_op_version);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the op-version key peer=%s",
+               req->trans->peerinfo.identifier);
+        goto out;
+    }
+
+    ret = gd_validate_cluster_op_version(this, peer_op_version,
+                                         req->trans->peerinfo.identifier);
+    if (ret)
+        goto out;
+
+    /* As this is ACK from the Cluster for the versions supported,
+       can set the op-version of 'this' glusterd to the one
+       received. */
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+           "using the op-version %d", peer_op_version);
+    conf->op_version = peer_op_version;
+    ret = glusterd_store_global_info(this);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLOBAL_OP_VERSION_SET_FAIL,
+               "Failed to store op-version");
 
 out:
-        rsp.op_ret = ret;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = ret;
+    rsp.op_errno = op_errno;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
 
-        ret = 0;
+    ret = 0;
 
-        if (clnt_dict)
-                dict_unref (clnt_dict);
+    if (clnt_dict)
+        dict_unref(clnt_dict);
 
-        if (args.hndsk.hndsk_val)
-                free (args.hndsk.hndsk_val);
+    if (args.hndsk.hndsk_val)
+        free(args.hndsk.hndsk_val);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+glusterd_mgmt_hndsk_versions_ack(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_mgmt_hndsk_versions_ack);
+    return glusterd_big_locked_handler(req, __glusterd_mgmt_hndsk_versions_ack);
 }
 
 int
-__server_get_volume_info (rpcsvc_request_t *req)
+__server_get_volume_info(rpcsvc_request_t *req)
 {
-        int                     ret             = -1;
-        int32_t                 op_errno        = ENOENT;
-        gf_get_volume_info_req  vol_info_req    = {{0,}};
-        gf_get_volume_info_rsp  vol_info_rsp    = {0,};
-        char                    *volname        = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        dict_t                  *dict           = NULL;
-        dict_t                  *dict_rsp       = NULL;
-        char                    *volume_id_str  = NULL;
-        int32_t                 flags           = 0;
-
-        ret = xdr_to_generic (req->msg[0], &vol_info_req,
-                             (xdrproc_t)xdr_gf_get_volume_info_req);
-        if (ret < 0) {
-                /* failed to decode msg */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_VOL_INFO_REQ_RECVD, "Received get volume info req");
-
-        if (vol_info_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-                if (!dict) {
-                        gf_msg ("glusterd", GF_LOG_WARNING, ENOMEM,
-                                GD_MSG_NO_MEMORY, "Out of Memory");
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (vol_info_req.dict.dict_val,
-                                        vol_info_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = vol_info_req.dict.dict_val;
-                }
+    int ret = -1;
+    int32_t op_errno = ENOENT;
+    gf_get_volume_info_req vol_info_req = {{
+        0,
+    }};
+    gf_get_volume_info_rsp vol_info_rsp = {
+        0,
+    };
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    dict_t *dict = NULL;
+    dict_t *dict_rsp = NULL;
+    char *volume_id_str = NULL;
+    int32_t flags = 0;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &vol_info_req,
+                         (xdrproc_t)xdr_gf_get_volume_info_req);
+    if (ret < 0) {
+        /* failed to decode msg */
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+    gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_INFO_REQ_RECVD, NULL);
+
+    if (vol_info_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            op_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, -ret,
-                        GD_MSG_DICT_GET_FAILED, "failed to get flags");
-                op_errno = -ret;
-                ret = -1;
-                goto out;
+        ret = dict_unserialize(vol_info_req.dict.dict_val,
+                               vol_info_req.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                    NULL);
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        } else {
+            dict->extra_stdfree = vol_info_req.dict.dict_val;
         }
+    }
 
-        if (!flags) {
-                /* Nothing to query about. Just return success */
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_FLAG_SET, "No flags set");
-                ret = 0;
-                goto out;
-        }
+    ret = dict_get_int32(dict, "flags", &flags);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=flags", NULL);
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                op_errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
+    if (!flags) {
+        /* Nothing to query about. Just return success */
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_FLAG_SET, NULL);
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        op_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL,
+                "Volname=%s", volname, NULL);
+        op_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                op_errno = EINVAL;
-                ret = -1;
-                goto out;
+    if (flags & (int32_t)GF_GET_VOLUME_UUID) {
+        volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id));
+        if (!volume_id_str) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    NULL);
+            op_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        if (flags | (int32_t)GF_GET_VOLUME_UUID) {
-                volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
-                if (!volume_id_str) {
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                dict_rsp = dict_new ();
-                if (!dict_rsp) {
-                        gf_msg ("glusterd", GF_LOG_WARNING, ENOMEM,
-                                GD_MSG_NO_MEMORY, "Out of Memory");
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict_rsp, "volume_id", volume_id_str);
-                if (ret) {
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = dict_allocate_and_serialize (dict_rsp, &vol_info_rsp.dict.dict_val,
-                                           &vol_info_rsp.dict.dict_len);
+        dict_rsp = dict_new();
+        if (!dict_rsp) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            op_errno = ENOMEM;
+            GF_FREE(volume_id_str);
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstr(dict_rsp, "volume_id", volume_id_str);
         if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=volume_id", NULL);
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = dict_allocate_and_serialize(dict_rsp, &vol_info_rsp.dict.dict_val,
+                                      &vol_info_rsp.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
 out:
-        vol_info_rsp.op_ret = ret;
-        vol_info_rsp.op_errno = op_errno;
-        vol_info_rsp.op_errstr = "";
-        glusterd_submit_reply (req, &vol_info_rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_get_volume_info_rsp);
-        ret = 0;
-
-        if (dict) {
-                dict_unref (dict);
-        }
-
-        if (dict_rsp) {
-                dict_unref (dict_rsp);
-        }
-
-        if (vol_info_rsp.dict.dict_val) {
-                GF_FREE (vol_info_rsp.dict.dict_val);
-        }
-        return ret;
+    vol_info_rsp.op_ret = ret;
+    vol_info_rsp.op_errno = op_errno;
+    vol_info_rsp.op_errstr = "";
+    glusterd_submit_reply(req, &vol_info_rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_get_volume_info_rsp);
+    ret = 0;
+
+    if (dict) {
+        dict_unref(dict);
+    }
+
+    if (dict_rsp) {
+        dict_unref(dict_rsp);
+    }
+
+    if (vol_info_rsp.dict.dict_val) {
+        GF_FREE(vol_info_rsp.dict.dict_val);
+    }
+    return ret;
 }
 
 int
-server_get_volume_info (rpcsvc_request_t *req)
+server_get_volume_info(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __server_get_volume_info);
+    return glusterd_big_locked_handler(req, __server_get_volume_info);
 }
 
-
 /*
  * glusterd function to get the list of snapshot names and uuids
  */
 int
-__server_get_snap_info (rpcsvc_request_t *req)
+__server_get_snap_info(rpcsvc_request_t *req)
 {
-        int                             ret             = -1;
-        int                             op_errno        = ENOENT;
-        gf_getsnap_name_uuid_req        snap_info_req   = {{0,}};
-        gf_getsnap_name_uuid_rsp        snap_info_rsp   = {0,};
-        dict_t                          *dict           = NULL;
-        dict_t                          *dict_rsp       = NULL;
-        glusterd_volinfo_t              *volinfo        = NULL;
-        char                            *volname        = NULL;
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &snap_info_req,
-                              (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL,
-                        "Failed to decode management handshake response");
-                goto out;
-        }
-
-        if (snap_info_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict) {
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (snap_info_req.dict.dict_val,
-                                        snap_info_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "Failed to unserialize dictionary");
-                        op_errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = snap_info_req.dict.dict_val;
-                }
+    int ret = -1;
+    int op_errno = ENOENT;
+    gf_getsnap_name_uuid_req snap_info_req = {{
+        0,
+    }};
+    gf_getsnap_name_uuid_rsp snap_info_rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    dict_t *dict_rsp = NULL;
+    char *volname = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &snap_info_req,
+                         (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode management handshake response");
+        goto out;
+    }
+
+    if (snap_info_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            op_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                op_errno = EINVAL;
-                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to retrieve volname");
-                ret = -1;
-                goto out;
+        ret = dict_unserialize(snap_info_req.dict.dict_val,
+                               snap_info_req.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, EINVAL,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to unserialize dictionary");
+            op_errno = EINVAL;
+            ret = -1;
+            goto out;
+        } else {
+            dict->extra_stdfree = snap_info_req.dict.dict_val;
         }
+    }
 
-        dict_rsp = dict_new ();
-        if (!dict_rsp) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_DICT_GET_FAILED,
+               "Failed to retrieve volname");
+        ret = -1;
+        goto out;
+    }
+
+    dict_rsp = dict_new();
+    if (!dict_rsp) {
+        gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_snapshot_get_volnames_uuids (dict_rsp, volname,
-                                                    &snap_info_rsp);
+    ret = glusterd_snapshot_get_volnames_uuids(dict_rsp, volname,
+                                               &snap_info_rsp);
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND,
-                        "Error getting snapshot volume names and uuids : %s",
-                        volname);
-                op_errno = EINVAL;
-        }
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Error getting snapshot volume names and uuids : %s", volname);
+        op_errno = EINVAL;
+    }
 
 out:
-        snap_info_rsp.op_ret = ret;
-        snap_info_rsp.op_errno = op_errno;
-        snap_info_rsp.op_errstr = "";
-        glusterd_submit_reply (req, &snap_info_rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
-
-        if (dict) {
-                dict_unref (dict);
-        }
+    snap_info_rsp.op_ret = ret;
+    snap_info_rsp.op_errno = op_errno;
+    snap_info_rsp.op_errstr = "";
+    glusterd_submit_reply(req, &snap_info_rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
 
-        if (dict_rsp) {
-                dict_unref (dict_rsp);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        if (snap_info_rsp.dict.dict_val) {
-                GF_FREE (snap_info_rsp.dict.dict_val);
-        }
+    if (dict_rsp) {
+        dict_unref(dict_rsp);
+    }
+
+    if (snap_info_rsp.dict.dict_val) {
+        GF_FREE(snap_info_rsp.dict.dict_val);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-server_get_snap_info (rpcsvc_request_t *req)
+server_get_snap_info(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __server_get_snap_info);
+    return glusterd_big_locked_handler(req, __server_get_snap_info);
 }
 
-rpcsvc_actor_t gluster_handshake_actors[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]         = {"NULL",        GF_HNDSK_NULL,         NULL,                NULL, 0, DRC_NA},
-        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
-        [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_HNDSK_EVENT_NOTIFY, server_event_notify, NULL, 0, DRC_NA},
-        [GF_HNDSK_GET_VOLUME_INFO] = {"GETVOLUMEINFO", GF_HNDSK_GET_VOLUME_INFO, server_get_volume_info, NULL, 0, DRC_NA},
-        [GF_HNDSK_GET_SNAPSHOT_INFO] = {"GETSNAPINFO", GF_HNDSK_GET_SNAPSHOT_INFO, server_get_snap_info, NULL, 0, DRC_NA},
+static rpcsvc_actor_t gluster_handshake_actors[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = {"NULL", NULL, NULL, GF_HNDSK_NULL, DRC_NA, 0},
+    [GF_HNDSK_GETSPEC] = {"GETSPEC", server_getspec, NULL, GF_HNDSK_GETSPEC,
+                          DRC_NA, 0},
+    [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", server_event_notify, NULL,
+                               GF_HNDSK_EVENT_NOTIFY, DRC_NA, 0},
+    [GF_HNDSK_GET_VOLUME_INFO] = {"GETVOLUMEINFO", server_get_volume_info, NULL,
+                                  GF_HNDSK_GET_VOLUME_INFO, DRC_NA, 0},
+    [GF_HNDSK_GET_SNAPSHOT_INFO] = {"GETSNAPINFO", server_get_snap_info, NULL,
+                                    GF_HNDSK_GET_SNAPSHOT_INFO, DRC_NA, 0},
 };
 
-
 struct rpcsvc_program gluster_handshake_prog = {
-        .progname  = "Gluster Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .actors    = gluster_handshake_actors,
-        .numactors = GF_HNDSK_MAXVALUE,
+    .progname = "Gluster Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .actors = gluster_handshake_actors,
+    .numactors = GF_HNDSK_MAXVALUE,
 };
 
 /* A minimal RPC program just for the cli getspec command */
-rpcsvc_actor_t gluster_cli_getspec_actors[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+static rpcsvc_actor_t gluster_cli_getspec_actors[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_GETSPEC] = {"GETSPEC", server_getspec, NULL, GF_HNDSK_GETSPEC,
+                          DRC_NA, 0},
 };
 
 struct rpcsvc_program gluster_cli_getspec_prog = {
-        .progname  = "Gluster Handshake (CLI Getspec)",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .actors    = gluster_cli_getspec_actors,
-        .numactors = GF_HNDSK_MAXVALUE,
+    .progname = "Gluster Handshake (CLI Getspec)",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .actors = gluster_cli_getspec_actors,
+    .numactors = GF_HNDSK_MAXVALUE,
 };
 
-
-char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
-        [GF_DUMP_NULL] = "NULL",
-        [GF_DUMP_DUMP] = "DUMP",
-        [GF_DUMP_PING] = "PING",
+static char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
+    [GF_DUMP_NULL] = "NULL",
+    [GF_DUMP_DUMP] = "DUMP",
+    [GF_DUMP_PING] = "PING",
 };
 
-rpc_clnt_prog_t glusterd_dump_prog = {
-        .progname       = "GLUSTERD-DUMP",
-        .prognum        = GLUSTER_DUMP_PROGRAM,
-        .progver        = GLUSTER_DUMP_VERSION,
-        .procnames      = glusterd_dump_proc,
+static rpc_clnt_prog_t glusterd_dump_prog = {
+    .progname = "GLUSTERD-DUMP",
+    .prognum = GLUSTER_DUMP_PROGRAM,
+    .progver = GLUSTER_DUMP_VERSION,
+    .procnames = glusterd_dump_proc,
 };
 
-
-rpcsvc_actor_t glusterd_mgmt_hndsk_actors[GD_MGMT_HNDSK_MAXVALUE] = {
-        [GD_MGMT_HNDSK_NULL]            = {"NULL", GD_MGMT_HNDSK_NULL, NULL,
-                                           NULL, 0, DRC_NA},
-        [GD_MGMT_HNDSK_VERSIONS]        = {"MGMT-VERS", GD_MGMT_HNDSK_VERSIONS,
-                                           glusterd_mgmt_hndsk_versions, NULL,
-                                           0, DRC_NA},
-        [GD_MGMT_HNDSK_VERSIONS_ACK]    = {"MGMT-VERS-ACK",
-                                           GD_MGMT_HNDSK_VERSIONS_ACK,
-                                           glusterd_mgmt_hndsk_versions_ack,
-                                           NULL, 0, DRC_NA},
+static rpcsvc_actor_t glusterd_mgmt_hndsk_actors[GD_MGMT_HNDSK_MAXVALUE] = {
+    [GD_MGMT_HNDSK_NULL] = {"NULL", NULL, NULL, GD_MGMT_HNDSK_NULL, DRC_NA, 0},
+    [GD_MGMT_HNDSK_VERSIONS] = {"MGMT-VERS", glusterd_mgmt_hndsk_versions, NULL,
+                                GD_MGMT_HNDSK_VERSIONS, DRC_NA, 0},
+    [GD_MGMT_HNDSK_VERSIONS_ACK] = {"MGMT-VERS-ACK",
+                                    glusterd_mgmt_hndsk_versions_ack, NULL,
+                                    GD_MGMT_HNDSK_VERSIONS_ACK, DRC_NA, 0},
 };
 
 struct rpcsvc_program glusterd_mgmt_hndsk_prog = {
-        .progname       = "Gluster MGMT Handshake",
-        .prognum        = GD_MGMT_HNDSK_PROGRAM,
-        .progver        = GD_MGMT_HNDSK_VERSION,
-        .actors         = glusterd_mgmt_hndsk_actors,
-        .numactors      = GD_MGMT_HNDSK_MAXVALUE,
+    .progname = "Gluster MGMT Handshake",
+    .prognum = GD_MGMT_HNDSK_PROGRAM,
+    .progver = GD_MGMT_HNDSK_VERSION,
+    .actors = glusterd_mgmt_hndsk_actors,
+    .numactors = GD_MGMT_HNDSK_MAXVALUE,
 };
 
-char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = {
-        [GD_MGMT_HNDSK_NULL]          = "NULL",
-        [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS",
-        [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK",
+static char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = {
+    [GD_MGMT_HNDSK_NULL] = "NULL",
+    [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS",
+    [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK",
 };
 
-rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = {
-        .progname  = "Gluster MGMT Handshake",
-        .prognum   = GD_MGMT_HNDSK_PROGRAM,
-        .progver   = GD_MGMT_HNDSK_VERSION,
-        .procnames = glusterd_mgmt_hndsk_proc,
+static rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = {
+    .progname = "Gluster MGMT Handshake",
+    .prognum = GD_MGMT_HNDSK_PROGRAM,
+    .progver = GD_MGMT_HNDSK_VERSION,
+    .procnames = glusterd_mgmt_hndsk_proc,
 };
 
-
 static int
-glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
+glusterd_event_connected_inject(glusterd_peerctx_t *peerctx)
 {
-        GF_ASSERT (peerctx);
+    GF_ASSERT(peerctx);
 
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_probe_ctx_t            *ctx = NULL;
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_probe_ctx_t *ctx = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
 
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_CONNECTED, &event);
 
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_CONNECTED, &event);
-
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_NEW_GET_FAIL, "Unable to get new event");
-                goto out;
-        }
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get new event");
+        goto out;
+    }
 
-        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
 
-        if (!ctx) {
-                ret = -1;
-                gf_msg ("glusterd", GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Memory not available");
-                goto out;
-        }
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                ret = -1;
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto unlock;
-        }
-        ctx->hostname = gf_strdup (peerinfo->hostname);
-        ctx->port = peerinfo->port;
-        ctx->req = peerctx->args.req;
-        ctx->dict = peerctx->args.dict;
-
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
-        event->ctx = ctx;
+    if (!ctx) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Memory not available");
+        goto out;
+    }
 
-        ret = glusterd_friend_sm_inject_event (event);
+    RCU_READ_LOCK;
 
-        if (ret)
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL, "Unable to inject "
-                        "EVENT_CONNECTED ret = %d", ret);
-unlock:
-        rcu_read_unlock ();
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", peerctx->peername,
+               uuid_utoa(peerctx->peerid));
+        GF_FREE(ctx);
+        goto out;
+    }
+    ctx->hostname = gf_strdup(peerinfo->hostname);
+    ctx->port = peerinfo->port;
+    ctx->req = peerctx->args.req;
+    ctx->dict = peerctx->args.dict;
+
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
+    event->ctx = ctx;
+
+    ret = glusterd_friend_sm_inject_event(event);
+
+    RCU_READ_UNLOCK;
+
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject "
+               "EVENT_CONNECTED ret = %d",
+               ret);
 
 out:
-        gf_msg_debug ("glusterd", 0, "returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
-
 int
-gd_validate_peer_op_version (xlator_t *this, glusterd_peerinfo_t *peerinfo,
-                             dict_t *dict, char **errstr)
+gd_validate_peer_op_version(xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            dict_t *dict, char **errstr)
 {
-        int              ret  = -1;
-        glusterd_conf_t *conf = NULL;
-        int32_t          peer_op_version = 0;
-        int32_t          peer_min_op_version = 0;
-        int32_t          peer_max_op_version = 0;
-
-        if (!dict && !this && !peerinfo)
-                goto out;
-
-        conf = this->private;
-
-        ret = dict_get_int32 (dict, GD_OP_VERSION_KEY, &peer_op_version);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, GD_MAX_OP_VERSION_KEY,
-                              &peer_max_op_version);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, GD_MIN_OP_VERSION_KEY,
-                              &peer_min_op_version);
-        if (ret)
-                goto out;
-
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    int32_t peer_op_version = 0;
+    int32_t peer_min_op_version = 0;
+    int32_t peer_max_op_version = 0;
+
+    if (!dict) {
+        gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    if (!this) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_XLATOR_NOT_DEFINED,
+                NULL);
+        goto out;
+    }
+
+    if (!peerinfo) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    conf = this->private;
+
+    ret = dict_get_int32(dict, GD_OP_VERSION_KEY, &peer_op_version);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GD_OP_VERSION_KEY, NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, GD_MAX_OP_VERSION_KEY, &peer_max_op_version);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GD_MAX_OP_VERSION_KEY, NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, GD_MIN_OP_VERSION_KEY, &peer_min_op_version);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GD_MIN_OP_VERSION_KEY, NULL);
+        goto out;
+    }
+
+    ret = -1;
+    /* Check if peer can support our op_version */
+    if ((peer_max_op_version < conf->op_version) ||
+        (peer_min_op_version > conf->op_version)) {
+        ret = gf_asprintf(errstr,
+                          "Peer %s does not support required "
+                          "op-version",
+                          peerinfo->hostname);
         ret = -1;
-        /* Check if peer can support our op_version */
-        if ((peer_max_op_version < conf->op_version) ||
-            (peer_min_op_version > conf->op_version)) {
-                ret = gf_asprintf (errstr, "Peer %s does not support required "
-                                   "op-version", peerinfo->hostname);
-                ret = -1;
-                goto out;
-        }
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name , 0, "Peer %s %s", peerinfo->hostname,
-                ((ret < 0) ? "rejected" : "accepted"));
-        return ret;
+    if (peerinfo)
+        gf_msg_debug((this ? this->name : "glusterd"), 0, "Peer %s %s",
+                     peerinfo->hostname, ((ret < 0) ? "rejected" : "accepted"));
+    return ret;
 }
 
 int
-__glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
-                                     int count, void *myframe)
+__glusterd_mgmt_hndsk_version_ack_cbk(struct rpc_req *req, struct iovec *iov,
+                                      int count, void *myframe)
 {
-        int                  ret      = -1;
-        int                  op_errno = EINVAL;
-        gf_mgmt_hndsk_rsp    rsp      = {0,};
-        xlator_t            *this     = NULL;
-        call_frame_t        *frame    = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-        glusterd_peerctx_t  *peerctx  = NULL;
-        char                msg[1024] = {0,};
-
-        this = THIS;
-        frame = myframe;
-        peerctx = frame->local;
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                gf_msg_debug (this->name, 0, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                ret = -1;
-                goto out;
-        }
-
-        if (-1 == req->rpc_status) {
-                snprintf (msg, sizeof (msg),
-                          "Error through RPC layer, retry again later");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RPC_LAYER_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
-        if (ret < 0) {
-                snprintf (msg, sizeof (msg), "Failed to decode XDR");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        op_errno = rsp.op_errno;
-        if (-1 == rsp.op_ret) {
-                ret = -1;
-                snprintf (msg, sizeof (msg),
-                          "Failed to get handshake ack from remote server");
-                gf_msg (frame->this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_HANDSHAKE_ACK, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        /* TODO: this is hardcoded as of now, but I don't forsee any problems
-         * with this as long as we are properly handshaking operating versions
-         */
-        peerinfo->mgmt = &gd_mgmt_prog;
-        peerinfo->peer = &gd_peer_prog;
-        peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
-
-        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
-
-        if (GD_MODE_ON == peerctx->args.mode) {
-                ret = glusterd_event_connected_inject (peerctx);
-                peerctx->args.req = NULL;
-        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
-                peerctx->args.mode = GD_MODE_ON;
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_UNKNOWN_MODE, "unknown mode %d",
-                        peerctx->args.mode);
-        }
-
-        ret = 0;
+    int ret = -1;
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    char msg[64] = {
+        0,
+    };
+
+    this = THIS;
+    frame = myframe;
+    peerctx = frame->local;
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        gf_msg_debug(this->name, 0, "Could not find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        snprintf(msg, sizeof(msg),
+                 "Error through RPC layer, retry again later");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LAYER_ERROR, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    if (ret < 0) {
+        snprintf(msg, sizeof(msg), "Failed to decode XDR");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "Failed to get handshake ack from remote server");
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_NO_HANDSHAKE_ACK,
+               "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    /* TODO: this is hardcoded as of now, but I don't forsee any problems
+     * with this as long as we are properly handshaking operating versions
+     */
+    peerinfo->mgmt = &gd_mgmt_prog;
+    peerinfo->peer = &gd_peer_prog;
+    peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
+
+    ret = default_notify(this, GF_EVENT_CHILD_UP, NULL);
+
+    if (GD_MODE_ON == peerctx->args.mode) {
+        (void)glusterd_event_connected_inject(peerctx);
+        peerctx->args.req = NULL;
+    } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+        peerctx->args.mode = GD_MODE_ON;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UNKNOWN_MODE,
+               "unknown mode %d", peerctx->args.mode);
+    }
+
+    ret = 0;
 out:
 
-        if (ret != 0 && peerinfo)
-                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+    if (ret != 0 && peerinfo)
+        rpc_transport_disconnect(peerinfo->rpc->conn.trans, _gf_false);
 
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        if (rsp.hndsk.hndsk_val)
-                free (rsp.hndsk.hndsk_val);
+    if (rsp.hndsk.hndsk_val)
+        free(rsp.hndsk.hndsk_val);
 
-        glusterd_friend_sm ();
+    glusterd_friend_sm();
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
-                                     int count, void *myframe)
+glusterd_mgmt_hndsk_version_ack_cbk(struct rpc_req *req, struct iovec *iov,
+                                    int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_mgmt_hndsk_version_ack_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_mgmt_hndsk_version_ack_cbk);
 }
 
 int
-__glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+__glusterd_mgmt_hndsk_version_cbk(struct rpc_req *req, struct iovec *iov,
+                                  int count, void *myframe)
 {
-        int                  ret       = -1;
-        int                  op_errno  = EINVAL;
-        gf_mgmt_hndsk_rsp    rsp       = {0,};
-        gf_mgmt_hndsk_req    arg       = {{0,}};
-        xlator_t            *this      = NULL;
-        call_frame_t        *frame     = NULL;
-        glusterd_peerinfo_t *peerinfo  = NULL;
-        glusterd_peerctx_t  *peerctx   = NULL;
-        dict_t              *dict      = NULL;
-        dict_t              *rsp_dict  = NULL;
-        glusterd_conf_t     *conf      = NULL;
-        char                 msg[1024] = {0,};
-
-        this = THIS;
-        conf = this->private;
-        frame = myframe;
-        peerctx = frame->local;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                ret = -1;
-                gf_msg_debug (this->name, 0, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto out;
-        }
-
-        if (-1 == req->rpc_status) {
-                ret = -1;
-                snprintf (msg, sizeof (msg),
-                          "Error through RPC layer, retry again later");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RPC_LAYER_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
-        if (ret < 0) {
-                snprintf (msg, sizeof (msg), "Failed to decode management "
-                          "handshake response");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, dict, rsp.hndsk.hndsk_val,
-                                      rsp.hndsk.hndsk_len, ret, op_errno,
-                                      out);
-
-        op_errno = rsp.op_errno;
-        if (-1 == rsp.op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        GD_MSG_VERS_GET_FAIL,
-                        "failed to get the 'versions' from peer (%s)",
-                        req->conn->trans->peerinfo.identifier);
-                goto out;
-        }
-
-        /* Check if peer can be part of cluster */
-        ret = gd_validate_peer_op_version (this, peerinfo, dict,
-                                           &peerctx->errstr);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_VERSION_MISMATCH,
-                        "failed to validate the operating version of peer (%s)",
-                        peerinfo->hostname);
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict)
-                goto out;
-
-        ret = dict_set_int32 (rsp_dict, GD_OP_VERSION_KEY, conf->op_version);
-        if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_DICT_SET_FAILED,
-                       "failed to set operating version in dict");
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, rsp_dict, (&arg.hndsk.hndsk_val),
-                                    arg.hndsk.hndsk_len, op_errno, out);
+    int ret = -1;
+    int op_errno = EINVAL;
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+    gf_mgmt_hndsk_req arg = {{
+        0,
+    }};
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char msg[64] = {
+        0,
+    };
+
+    this = THIS;
+    conf = this->private;
+    frame = myframe;
+    peerctx = frame->local;
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        ret = -1;
+        gf_msg_debug(this->name, 0, "Could not find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
 
-        ret = glusterd_submit_request (peerinfo->rpc, &arg, frame,
-                                       &gd_clnt_mgmt_hndsk_prog,
-                                       GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this,
-                                       glusterd_mgmt_hndsk_version_ack_cbk,
-                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "Error through RPC layer, retry again later");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LAYER_ERROR, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    if (ret < 0) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to decode management "
+                 "handshake response");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, dict, rsp.hndsk.hndsk_val,
+                                 rsp.hndsk.hndsk_len, ret, op_errno, out);
+
+    op_errno = rsp.op_errno;
+    if (-1 == rsp.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, GD_MSG_VERS_GET_FAIL,
+               "failed to get the 'versions' from peer (%s)",
+               req->conn->trans->peerinfo.identifier);
+        goto out;
+    }
+
+    /* Check if peer can be part of cluster */
+    ret = gd_validate_peer_op_version(this, peerinfo, dict, &peerctx->errstr);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+               "failed to validate the operating version of peer (%s)",
+               peerinfo->hostname);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict)
+        goto out;
+
+    ret = dict_set_int32(rsp_dict, GD_OP_VERSION_KEY, conf->op_version);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set operating version in dict");
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, rsp_dict, (&arg.hndsk.hndsk_val),
+                               arg.hndsk.hndsk_len, op_errno, out);
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &arg, frame, &gd_clnt_mgmt_hndsk_prog,
+        GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this,
+        glusterd_mgmt_hndsk_version_ack_cbk, (xdrproc_t)xdr_gf_mgmt_hndsk_req);
 
 out:
-        if (ret) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-                if (peerinfo)
-                        rpc_transport_disconnect (peerinfo->rpc->conn.trans);
-        }
+    if (ret) {
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+        if (peerinfo)
+            rpc_transport_disconnect(peerinfo->rpc->conn.trans, _gf_false);
+    }
 
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        if (rsp.hndsk.hndsk_val)
-                free (rsp.hndsk.hndsk_val);
+    if (rsp.hndsk.hndsk_val)
+        free(rsp.hndsk.hndsk_val);
 
-        if (arg.hndsk.hndsk_val)
-                GF_FREE (arg.hndsk.hndsk_val);
+    if (arg.hndsk.hndsk_val)
+        GF_FREE(arg.hndsk.hndsk_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                     int count, void *myframe)
+glusterd_mgmt_hndsk_version_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_mgmt_hndsk_version_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_mgmt_hndsk_version_cbk);
 }
 
 int
-glusterd_mgmt_handshake (xlator_t *this, glusterd_peerctx_t *peerctx)
+glusterd_mgmt_handshake(xlator_t *this, glusterd_peerctx_t *peerctx)
 {
-        call_frame_t        *frame    = NULL;
-        gf_mgmt_hndsk_req    req      = {{0,},};
-        glusterd_peerinfo_t *peerinfo = NULL;
-        dict_t              *req_dict = NULL;
-        int                  ret      = -1;
-        int                  op_errno = EINVAL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    call_frame_t *frame = NULL;
+    gf_mgmt_hndsk_req req = {
+        {
+            0,
+        },
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *req_dict = NULL;
+    int ret = -1;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg("glusterd", GF_LOG_WARNING, errno, GD_MSG_FRAME_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    frame->local = peerctx;
+
+    req_dict = dict_new();
+    if (!req_dict) {
+        gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    ret = dict_set_dynstr(req_dict, GD_PEER_ID_KEY,
+                          gf_strdup(uuid_utoa(MY_UUID)));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "failed to set peer ID in dict");
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, req_dict, (&req.hndsk.hndsk_val),
+                               req.hndsk.hndsk_len, ret, out);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg_debug(THIS->name, 0, "Could not find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, &gd_clnt_mgmt_hndsk_prog,
+        GD_MGMT_HNDSK_VERSIONS, NULL, this, glusterd_mgmt_hndsk_version_cbk,
+        (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+
+    RCU_READ_UNLOCK;
+
+    ret = 0;
 
-        frame->local = peerctx;
-
-        req_dict = dict_new ();
-        if (!req_dict)
-                goto out;
-
-        ret = dict_set_dynstr (req_dict, GD_PEER_ID_KEY,
-                               gf_strdup (uuid_utoa (MY_UUID)));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED,
-                       "failed to set peer ID in dict");
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, req_dict, (&req.hndsk.hndsk_val),
-                                    req.hndsk.hndsk_len, op_errno, out);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                gf_msg_debug (THIS->name, 0, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto unlock;
-        }
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       &gd_clnt_mgmt_hndsk_prog,
-                                       GD_MGMT_HNDSK_VERSIONS, NULL, this,
-                                       glusterd_mgmt_hndsk_version_cbk,
-                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
-        ret = 0;
-unlock:
-        rcu_read_unlock ();
 out:
-        if (ret && frame)
-                STACK_DESTROY (frame->root);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        return ret;
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
+
+    return ret;
 }
 
 int
-glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
-                                gf_prog_detail *prog)
+glusterd_set_clnt_mgmt_program(glusterd_peerinfo_t *peerinfo,
+                               gf_prog_detail *prog)
 {
-        gf_prog_detail  *trav   = NULL;
-        int             ret     = -1;
+    gf_prog_detail *trav = NULL;
+    int ret = -1;
 
-        if (!peerinfo || !prog)
-                goto out;
+    if (!peerinfo || !prog)
+        goto out;
 
-        trav = prog;
+    trav = prog;
 
-        while (trav) {
-                ret = -1;
-                if ((gd_mgmt_prog.prognum == trav->prognum) &&
-                    (gd_mgmt_prog.progver == trav->progver)) {
-                        peerinfo->mgmt = &gd_mgmt_prog;
-                        ret = 0;
-                }
-
-                if ((gd_peer_prog.prognum == trav->prognum) &&
-                    (gd_peer_prog.progver == trav->progver)) {
-                        peerinfo->peer = &gd_peer_prog;
-                        ret = 0;
-                }
-
-                if (ret) {
-                        gf_msg_debug ("glusterd", 0,
-                                "%s (%"PRId64":%"PRId64") not supported",
-                                trav->progname, trav->prognum,
-                                trav->progver);
-                }
-
-                trav = trav->next;
-        }
-
-        if (peerinfo->mgmt) {
-                 gf_msg ("glusterd", GF_LOG_INFO, 0,
-                         GD_MSG_VERS_INFO,
-                         "Using Program %s, Num (%d), Version (%d)",
-                         peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
-                         peerinfo->mgmt->progver);
-        }
-
-        if (peerinfo->peer) {
-                 gf_msg ("glusterd", GF_LOG_INFO, 0,
-                         GD_MSG_VERS_INFO,
-                         "Using Program %s, Num (%d), Version (%d)",
-                         peerinfo->peer->progname, peerinfo->peer->prognum,
-                         peerinfo->peer->progver);
+    while (trav) {
+        ret = -1;
+        if ((gd_mgmt_prog.prognum == trav->prognum) &&
+            (gd_mgmt_prog.progver == trav->progver)) {
+            peerinfo->mgmt = &gd_mgmt_prog;
+            ret = 0;
         }
 
-        if (peerinfo->mgmt_v3) {
-                 gf_msg ("glusterd", GF_LOG_INFO, 0,
-                         GD_MSG_VERS_INFO,
-                         "Using Program %s, Num (%d), Version (%d)",
-                         peerinfo->mgmt_v3->progname,
-                         peerinfo->mgmt_v3->prognum,
-                         peerinfo->mgmt_v3->progver);
+        if ((gd_peer_prog.prognum == trav->prognum) &&
+            (gd_peer_prog.progver == trav->progver)) {
+            peerinfo->peer = &gd_peer_prog;
+            ret = 0;
         }
 
-        ret = 0;
+        if (ret) {
+            gf_msg_debug("glusterd", 0,
+                         "%s (%" PRId64 ":%" PRId64 ") not supported",
+                         trav->progname, trav->prognum, trav->progver);
+        }
+
+        trav = trav->next;
+    }
+
+    if (peerinfo->mgmt) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+               "Using Program %s, Num (%d), Version (%d)",
+               peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
+               peerinfo->mgmt->progver);
+    }
+
+    if (peerinfo->peer) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+               "Using Program %s, Num (%d), Version (%d)",
+               peerinfo->peer->progname, peerinfo->peer->prognum,
+               peerinfo->peer->progver);
+    }
+
+    if (peerinfo->mgmt_v3) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+               "Using Program %s, Num (%d), Version (%d)",
+               peerinfo->mgmt_v3->progname, peerinfo->mgmt_v3->prognum,
+               peerinfo->mgmt_v3->progver);
+    }
+
+    ret = 0;
 out:
-        return ret;
-
+    return ret;
 }
 
 static gf_boolean_t
-_mgmt_hndsk_prog_present (gf_prog_detail *prog) {
-        gf_boolean_t    ret = _gf_false;
-        gf_prog_detail  *trav = NULL;
+_mgmt_hndsk_prog_present(gf_prog_detail *prog)
+{
+    gf_boolean_t ret = _gf_false;
+    gf_prog_detail *trav = NULL;
 
-        GF_ASSERT (prog);
+    GF_ASSERT(prog);
 
-        trav = prog;
+    trav = prog;
 
-        while (trav) {
-                if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) &&
-                    (trav->progver == GD_MGMT_HNDSK_VERSION)) {
-                        ret = _gf_true;
-                        goto out;
-                }
-                trav = trav->next;
+    while (trav) {
+        if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) &&
+            (trav->progver == GD_MGMT_HNDSK_VERSION)) {
+            ret = _gf_true;
+            goto out;
         }
+        trav = trav->next;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+__glusterd_peer_dump_version_cbk(struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
 {
-        int                  ret      = -1;
-        gf_dump_rsp          rsp      = {0,};
-        xlator_t            *this     = NULL;
-        gf_prog_detail      *trav     = NULL;
-        gf_prog_detail      *next     = NULL;
-        call_frame_t        *frame    = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-        glusterd_peerctx_t  *peerctx  = NULL;
-        glusterd_conf_t     *conf     = NULL;
-        char                msg[1024] = {0,};
-
-        this = THIS;
-        conf = this->private;
-        frame = myframe;
-        peerctx = frame->local;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                gf_msg_debug (this->name, 0, "Couldn't find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto out;
-        }
-
-        if (-1 == req->rpc_status) {
-                snprintf (msg, sizeof (msg),
-                          "Error through RPC layer, retry again later");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RPC_LAYER_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
-        if (ret < 0) {
-                snprintf (msg, sizeof (msg), "Failed to decode XDR");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-        if (-1 == rsp.op_ret) {
-                snprintf (msg, sizeof (msg),
-                          "Failed to get the 'versions' from remote server");
-                gf_msg (frame->this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VERS_GET_FAIL, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (_mgmt_hndsk_prog_present (rsp.prog)) {
-                gf_msg_debug (this->name, 0,
-                        "Proceeding to op-version handshake with peer %s",
-                        peerinfo->hostname);
-                ret = glusterd_mgmt_handshake (this, peerctx);
-                goto out;
-        } else if (conf->op_version > 1) {
-                ret = -1;
-                snprintf (msg, sizeof (msg),
-                          "Peer %s does not support required op-version",
-                          peerinfo->hostname);
-                peerctx->errstr = gf_strdup (msg);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VERSION_UNSUPPORTED, "%s", msg);
-                goto out;
-        }
-
-        /* Make sure we assign the proper program to peer */
-        ret = glusterd_set_clnt_mgmt_program (peerinfo, rsp.prog);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_MGMT_PGM_SET_FAIL,
-                        "failed to set the mgmt program");
-                goto out;
-        }
+    int ret = -1;
+    gf_dump_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_prog_detail *trav = NULL;
+    gf_prog_detail *next = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    glusterd_conf_t *conf = NULL;
+    char msg[1024] = {
+        0,
+    };
+
+    this = THIS;
+    conf = this->private;
+    frame = myframe;
+    peerctx = frame->local;
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        gf_msg_debug(this->name, 0, "Couldn't find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        snprintf(msg, sizeof(msg),
+                 "Error through RPC layer, retry again later");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LAYER_ERROR, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
+    if (ret < 0) {
+        snprintf(msg, sizeof(msg), "Failed to decode XDR");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+    if (-1 == rsp.op_ret) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to get the 'versions' from remote server");
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_VERS_GET_FAIL, "%s",
+               msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (_mgmt_hndsk_prog_present(rsp.prog)) {
+        gf_msg_debug(this->name, 0,
+                     "Proceeding to op-version handshake with peer %s",
+                     peerinfo->hostname);
+        ret = glusterd_mgmt_handshake(this, peerctx);
+        goto out;
+    } else if (conf->op_version > 1) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "Peer %s does not support required op-version",
+                 peerinfo->hostname);
+        peerctx->errstr = gf_strdup(msg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERSION_UNSUPPORTED, "%s",
+               msg);
+        goto out;
+    }
+
+    /* Make sure we assign the proper program to peer */
+    ret = glusterd_set_clnt_mgmt_program(peerinfo, rsp.prog);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_MGMT_PGM_SET_FAIL,
+               "failed to set the mgmt program");
+        goto out;
+    }
+
+    ret = default_notify(this, GF_EVENT_CHILD_UP, NULL);
+
+    if (GD_MODE_ON == peerctx->args.mode) {
+        (void)glusterd_event_connected_inject(peerctx);
+        peerctx->args.req = NULL;
+    } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+        peerctx->args.mode = GD_MODE_ON;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UNKNOWN_MODE,
+               "unknown mode %d", peerctx->args.mode);
+    }
+
+    ret = 0;
 
-        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+out:
+    if (ret != 0 && peerinfo)
+        rpc_transport_disconnect(peerinfo->rpc->conn.trans, _gf_false);
 
-        if (GD_MODE_ON == peerctx->args.mode) {
-                ret = glusterd_event_connected_inject (peerctx);
-                peerctx->args.req = NULL;
-        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
-                peerctx->args.mode = GD_MODE_ON;
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_UNKNOWN_MODE, "unknown mode %d",
-                        peerctx->args.mode);
-        }
+    RCU_READ_UNLOCK;
 
-        ret = 0;
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-out:
-        if (ret != 0 && peerinfo)
-                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
-
-        rcu_read_unlock ();
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        /* don't use GF_FREE, buffer was allocated by libc */
-        if (rsp.prog) {
-                trav = rsp.prog;
-                while (trav) {
-                        next = trav->next;
-                        free (trav->progname);
-                        free (trav);
-                        trav = next;
-                }
+    /* don't use GF_FREE, buffer was allocated by libc */
+    if (rsp.prog) {
+        trav = rsp.prog;
+        while (trav) {
+            next = trav->next;
+            free(trav->progname);
+            free(trav);
+            trav = next;
         }
+    }
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
-
 int
-glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+glusterd_peer_dump_version_cbk(struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_peer_dump_version_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_peer_dump_version_cbk);
 }
 
 int
-glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
-                            glusterd_peerctx_t *peerctx)
+glusterd_peer_dump_version(xlator_t *this, struct rpc_clnt *rpc,
+                           glusterd_peerctx_t *peerctx)
 {
-        call_frame_t        *frame    = NULL;
-        gf_dump_req          req      = {0,};
-        glusterd_peerinfo_t *peerinfo = NULL;
-        int                  ret      = -1;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        frame->local = peerctx;
-        if (!peerctx)
-                goto out;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find_by_generation (peerctx->peerinfo_gen);
-        if (!peerinfo) {
-                gf_msg_debug (this->name, 0, "Couldn't find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto unlock;
-        }
-
-        req.gfs_id = 0xcafe;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       &glusterd_dump_prog, GF_DUMP_DUMP,
-                                       NULL, this,
-                                       glusterd_peer_dump_version_cbk,
-                                       (xdrproc_t)xdr_gf_dump_req);
-unlock:
-        rcu_read_unlock ();
+    call_frame_t *frame = NULL;
+    gf_dump_req req = {
+        0,
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int ret = -1;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_WARNING, errno, GD_MSG_FRAME_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    frame->local = peerctx;
+    if (!peerctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg_debug(this->name, 0, "Couldn't find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
+
+    req.gfs_id = 0xcafe;
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, &glusterd_dump_prog, GF_DUMP_DUMP, NULL,
+        this, glusterd_peer_dump_version_cbk, (xdrproc_t)xdr_gf_dump_req);
+
+    RCU_READ_UNLOCK;
 out:
-        if (ret && frame)
-                STACK_DESTROY (frame->root);
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c
index 45a5912a1eb..61c0f1c946f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-hooks.c
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c
@@ -8,16 +8,15 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "logging.h"
-#include "run.h"
-#include "defaults.h"
-#include "syscall.h"
-#include "compat.h"
-#include "compat-errno.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/run.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
@@ -29,569 +28,614 @@
 #include <fnmatch.h>
 
 #define EMPTY ""
-char glusterd_hook_dirnames[GD_OP_MAX][256] =
-{
-        [GD_OP_NONE]                    = EMPTY,
-        [GD_OP_CREATE_VOLUME]           = "create",
-        [GD_OP_START_BRICK]             = EMPTY,
-        [GD_OP_STOP_BRICK]              = EMPTY,
-        [GD_OP_DELETE_VOLUME]           = "delete",
-        [GD_OP_START_VOLUME]            = "start",
-        [GD_OP_STOP_VOLUME]             = "stop",
-        [GD_OP_DEFRAG_VOLUME]           = EMPTY,
-        [GD_OP_ADD_BRICK]               = "add-brick",
-        [GD_OP_REMOVE_BRICK]            = "remove-brick",
-        [GD_OP_REPLACE_BRICK]           = EMPTY,
-        [GD_OP_SET_VOLUME]              = "set",
-        [GD_OP_RESET_VOLUME]            = "reset",
-        [GD_OP_SYNC_VOLUME]             = EMPTY,
-        [GD_OP_LOG_ROTATE]              = EMPTY,
-        [GD_OP_GSYNC_CREATE]            = "gsync-create",
-        [GD_OP_GSYNC_SET]               = EMPTY,
-        [GD_OP_PROFILE_VOLUME]          = EMPTY,
-        [GD_OP_QUOTA]                   = EMPTY,
-        [GD_OP_STATUS_VOLUME]           = EMPTY,
-        [GD_OP_REBALANCE]               = EMPTY,
-        [GD_OP_HEAL_VOLUME]             = EMPTY,
-        [GD_OP_STATEDUMP_VOLUME]        = EMPTY,
-        [GD_OP_LIST_VOLUME]             = EMPTY,
-        [GD_OP_CLEARLOCKS_VOLUME]       = EMPTY,
-        [GD_OP_DEFRAG_BRICK_VOLUME]     = EMPTY,
+char glusterd_hook_dirnames[GD_OP_MAX][256] = {
+    [GD_OP_NONE] = EMPTY,
+    [GD_OP_CREATE_VOLUME] = "create",
+    [GD_OP_START_BRICK] = EMPTY,
+    [GD_OP_STOP_BRICK] = EMPTY,
+    [GD_OP_DELETE_VOLUME] = "delete",
+    [GD_OP_START_VOLUME] = "start",
+    [GD_OP_STOP_VOLUME] = "stop",
+    [GD_OP_DEFRAG_VOLUME] = EMPTY,
+    [GD_OP_ADD_BRICK] = "add-brick",
+    [GD_OP_REMOVE_BRICK] = "remove-brick",
+    [GD_OP_REPLACE_BRICK] = EMPTY,
+    [GD_OP_SET_VOLUME] = "set",
+    [GD_OP_RESET_VOLUME] = "reset",
+    [GD_OP_SYNC_VOLUME] = EMPTY,
+    [GD_OP_LOG_ROTATE] = EMPTY,
+    [GD_OP_GSYNC_CREATE] = "gsync-create",
+    [GD_OP_GSYNC_SET] = EMPTY,
+    [GD_OP_PROFILE_VOLUME] = EMPTY,
+    [GD_OP_QUOTA] = EMPTY,
+    [GD_OP_STATUS_VOLUME] = EMPTY,
+    [GD_OP_REBALANCE] = EMPTY,
+    [GD_OP_HEAL_VOLUME] = EMPTY,
+    [GD_OP_STATEDUMP_VOLUME] = EMPTY,
+    [GD_OP_LIST_VOLUME] = EMPTY,
+    [GD_OP_CLEARLOCKS_VOLUME] = EMPTY,
+    [GD_OP_DEFRAG_BRICK_VOLUME] = EMPTY,
+    [GD_OP_RESET_BRICK] = EMPTY,
 };
 #undef EMPTY
 
 static gf_boolean_t
-glusterd_is_hook_enabled (char *script)
+glusterd_is_hook_enabled(char *script)
 {
-        return (script[0] == 'S' && (fnmatch ("*.rpmsave", script, 0) != 0)
-                                 && (fnmatch ("*.rpmnew", script, 0) != 0));
+    return (script[0] == 'S' && (fnmatch("*.rpmsave", script, 0) != 0) &&
+            (fnmatch("*.rpmnew", script, 0) != 0));
 }
 
 int
-glusterd_hooks_create_hooks_directory (char *basedir)
+glusterd_hooks_create_hooks_directory(char *basedir)
 {
-        int  ret                                        = -1;
-        int  op                                         = GD_OP_NONE;
-        int  type                                       = GD_COMMIT_HOOK_NONE;
-        char version_dir[PATH_MAX]                      = {0, };
-        char path[PATH_MAX]                             = {0, };
-        char *cmd_subdir                                = NULL;
-        char type_subdir[GD_COMMIT_HOOK_MAX][256]       = {{0, },
-                                                           "pre",
-                                                           "post"};
-        glusterd_conf_t *priv                           = NULL;
-
-        priv = THIS->private;
-
-        snprintf (path, sizeof (path), "%s/hooks", basedir);
-        ret = mkdir_p (path, 0777, _gf_true);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_CRITICAL, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Unable to create %s",
-                        path);
-                goto out;
+    int ret = -1;
+    int op = GD_OP_NONE;
+    int type = GD_COMMIT_HOOK_NONE;
+    char version_dir[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *cmd_subdir = NULL;
+    char type_subdir[GD_COMMIT_HOOK_MAX][256] = {{
+                                                     0,
+                                                 },
+                                                 "pre",
+                                                 "post"};
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
+
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+
+    snprintf(path, sizeof(path), "%s/hooks", basedir);
+    ret = mkdir_p(path, 0755, _gf_true);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+                "Path=%s", path, NULL);
+        goto out;
+    }
+
+    GLUSTERD_GET_HOOKS_DIR(version_dir, GLUSTERD_HOOK_VER, priv);
+    ret = mkdir_p(version_dir, 0755, _gf_true);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+                "Directory=%s", version_dir, NULL);
+        goto out;
+    }
+
+    for (op = GD_OP_NONE + 1; op < GD_OP_MAX; op++) {
+        cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir(op);
+        if (strlen(cmd_subdir) == 0)
+            continue;
+
+        len = snprintf(path, sizeof(path), "%s/%s", version_dir, cmd_subdir);
+        if ((len < 0) || (len >= sizeof(path))) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
-
-        GLUSTERD_GET_HOOKS_DIR (version_dir, GLUSTERD_HOOK_VER, priv);
-        ret = mkdir_p (version_dir, 0777, _gf_true);
+        ret = mkdir_p(path, 0755, _gf_true);
         if (ret) {
-                gf_msg (THIS->name, GF_LOG_CRITICAL, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Unable to create %s",
-                        version_dir);
-                goto out;
+            gf_smsg(this->name, GF_LOG_CRITICAL, errno,
+                    GD_MSG_CREATE_DIR_FAILED, "Path=%s", path, NULL);
+            goto out;
         }
 
-        for (op = GD_OP_NONE+1; op < GD_OP_MAX; op++) {
-                cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
-                if (strlen (cmd_subdir) == 0)
-                        continue;
-
-                snprintf (path, sizeof (path), "%s/%s", version_dir,
-                          cmd_subdir);
-                ret = mkdir_p (path, 0777, _gf_true);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_CRITICAL, errno,
-                                GD_MSG_CREATE_DIR_FAILED,
-                                "Unable to create %s",
-                                path);
-                        goto out;
-                }
-
-                for (type = GD_COMMIT_HOOK_PRE; type < GD_COMMIT_HOOK_MAX;
-                     type++) {
-                        snprintf (path, sizeof (path), "%s/%s/%s",
-                                  version_dir, cmd_subdir, type_subdir[type]);
-                        ret = mkdir_p (path, 0777, _gf_true);
-                        if (ret) {
-                                gf_msg (THIS->name, GF_LOG_CRITICAL, errno,
-                                        GD_MSG_CREATE_DIR_FAILED,
-                                        "Unable to create %s",
-                                        path);
-                                goto out;
-                        }
-                }
+        for (type = GD_COMMIT_HOOK_PRE; type < GD_COMMIT_HOOK_MAX; type++) {
+            len = snprintf(path, sizeof(path), "%s/%s/%s", version_dir,
+                           cmd_subdir, type_subdir[type]);
+            if ((len < 0) || (len >= sizeof(path))) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                ret = -1;
+                goto out;
+            }
+            ret = mkdir_p(path, 0755, _gf_true);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_CRITICAL, errno,
+                        GD_MSG_CREATE_DIR_FAILED, "Path=%s", path, NULL);
+                goto out;
+            }
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-char*
-glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op)
+char *
+glusterd_hooks_get_hooks_cmd_subdir(glusterd_op_t op)
 {
-        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
+    GF_ASSERT((op > GD_OP_NONE) && (op < GD_OP_MAX));
 
-        return glusterd_hook_dirnames[op];
+    return glusterd_hook_dirnames[op];
 }
 
 void
-glusterd_hooks_add_working_dir (runner_t *runner, glusterd_conf_t *priv)
+glusterd_hooks_add_working_dir(runner_t *runner, glusterd_conf_t *priv)
 {
-        runner_argprintf (runner, "--gd-workdir=%s", priv->workdir);
+    runner_argprintf(runner, "--gd-workdir=%s", priv->workdir);
 }
 
 void
-glusterd_hooks_add_op (runner_t *runner, char *op)
+glusterd_hooks_add_op(runner_t *runner, char *op)
 {
-        runner_argprintf (runner, "--volume-op=%s", op);
+    runner_argprintf(runner, "--volume-op=%s", op);
 }
 
 void
-glusterd_hooks_add_hooks_version (runner_t* runner)
+glusterd_hooks_add_hooks_version(runner_t *runner)
 {
-        runner_argprintf (runner, "--version=%d", GLUSTERD_HOOK_VER);
+    runner_argprintf(runner, "--version=%d", GLUSTERD_HOOK_VER);
 }
 
 static void
-glusterd_hooks_add_custom_args (dict_t *dict, runner_t *runner)
+glusterd_hooks_add_custom_args(dict_t *dict, runner_t *runner)
 {
-        char      *hooks_args     = NULL;
-        int32_t    ret            = -1;
-        xlator_t  *this           = NULL;
+    char *hooks_args = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, runner, out);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, runner, out);
 
-        ret = dict_get_str (dict, "hooks_args", &hooks_args);
-        if (ret)
-                gf_msg_debug (this->name, 0,
-                        "No Hooks Arguments.");
-        else
-                gf_msg_debug (this->name, 0,
-                        "Hooks Args = %s", hooks_args);
+    ret = dict_get_str(dict, "hooks_args", &hooks_args);
+    if (ret)
+        gf_msg_debug(this->name, 0, "No Hooks Arguments.");
+    else
+        gf_msg_debug(this->name, 0, "Hooks Args = %s", hooks_args);
 
-        if (hooks_args)
-                runner_argprintf (runner, "%s", hooks_args);
+    if (hooks_args)
+        runner_argprintf(runner, "%s", hooks_args);
 
 out:
-        return;
+    return;
 }
 
-
 int
-glusterd_hooks_set_volume_args (dict_t *dict, runner_t *runner)
+glusterd_hooks_set_volume_args(dict_t *dict, runner_t *runner)
 {
-        int     i               = 0;
-        int     count           = 0;
-        int     ret             = -1;
-        char    query[1024]     = {0,};
-        char    *key            = NULL;
-        char    *value          = NULL;
-
-        ret = dict_get_int32 (dict, "count", &count);
+    int i = 0;
+    int count = 0;
+    int ret = -1;
+    int flag = 0;
+    char query[1024] = {
+        0,
+    };
+    char *key = NULL;
+    char *value = NULL;
+    char *inet_family = NULL;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    /* This will not happen unless op_ctx
+     * is corrupted*/
+    if (!count) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, "count",
+                NULL);
+        goto out;
+    }
+
+    runner_add_arg(runner, "-o");
+    for (i = 1; ret == 0; i++) {
+        snprintf(query, sizeof(query), "key%d", i);
+        ret = dict_get_str(dict, query, &key);
         if (ret)
-                goto out;
+            continue;
 
-        /* This will not happen unless op_ctx
-         * is corrupted*/
-        if (!count)
-                goto out;
-
-        runner_add_arg (runner, "-o");
-        for (i = 1; ret == 0; i++) {
-                snprintf (query, sizeof (query), "key%d", i);
-                ret = dict_get_str (dict, query, &key);
-                if (ret)
-                       continue;
-
-                snprintf (query, sizeof (query), "value%d", i);
-                ret = dict_get_str (dict, query, &value);
-                if (ret)
-                       continue;
-
-                runner_argprintf (runner, "%s=%s", key, value);
+        snprintf(query, sizeof(query), "value%d", i);
+        ret = dict_get_str(dict, query, &value);
+        if (ret)
+            continue;
+
+        runner_argprintf(runner, "%s=%s", key, value);
+        if ((strncmp(key, "cluster.enable-shared-storage",
+                     SLEN("cluster.enable-shared-storage")) == 0 ||
+             strncmp(key, "enable-shared-storage",
+                     SLEN("enable-shared-storage")) == 0) &&
+            strncmp(value, "enable", SLEN("enable")) == 0)
+            flag = 1;
+    }
+
+    glusterd_hooks_add_custom_args(dict, runner);
+    if (flag == 1) {
+        ret = dict_get_str_sizen(this->options, "transport.address-family",
+                                 &inet_family);
+        if (!ret) {
+            runner_argprintf(runner, "transport.address-family=%s",
+                             inet_family);
         }
+    }
 
-        glusterd_hooks_add_custom_args (dict, runner);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_hooks_add_op_args (runner_t *runner, glusterd_op_t op,
-                            dict_t *op_ctx, glusterd_commit_hook_type_t type)
+glusterd_hooks_add_op_args(runner_t *runner, glusterd_op_t op, dict_t *op_ctx,
+                           glusterd_commit_hook_type_t type)
 {
-        char                   *hooks_args       = NULL;
-        int                     vol_count        = 0;
-        gf_boolean_t            truth            = _gf_false;
-        glusterd_volinfo_t      *voliter         = NULL;
-        glusterd_conf_t         *priv            = NULL;
-        int                     ret              = -1;
-
-        priv = THIS->private;
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (glusterd_is_volume_started (voliter))
-                        vol_count++;
-        }
-
-        ret = 0;
-        switch (op) {
-                case GD_OP_START_VOLUME:
-                        if (type == GD_COMMIT_HOOK_PRE &&
-                            vol_count == 0)
-                                truth = _gf_true;
-
-                        else if (type == GD_COMMIT_HOOK_POST &&
-                                 vol_count == 1)
-                                truth = _gf_true;
-
-                        else
-                                truth = _gf_false;
-
-                        runner_argprintf (runner, "--first=%s",
-                                          truth? "yes":"no");
-
-                        glusterd_hooks_add_hooks_version (runner);
-                        glusterd_hooks_add_op (runner, "start");
-                        glusterd_hooks_add_working_dir (runner, priv);
-
-                        break;
-
-                case GD_OP_STOP_VOLUME:
-                        if (type == GD_COMMIT_HOOK_PRE &&
-                            vol_count == 1)
-                                truth = _gf_true;
-
-                        else if (type == GD_COMMIT_HOOK_POST &&
-                                 vol_count == 0)
-                                truth = _gf_true;
-
-                        else
-                                truth = _gf_false;
-
-                        runner_argprintf (runner, "--last=%s",
-                                          truth? "yes":"no");
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        ret = glusterd_hooks_set_volume_args (op_ctx, runner);
-                        glusterd_hooks_add_working_dir (runner, priv);
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                        glusterd_hooks_add_custom_args (op_ctx, runner);
-                        break;
-
-                case GD_OP_ADD_BRICK:
-                        glusterd_hooks_add_hooks_version (runner);
-                        glusterd_hooks_add_op (runner, "add-brick");
-                        glusterd_hooks_add_working_dir (runner, priv);
-                        break;
-
-               case GD_OP_RESET_VOLUME:
-                        glusterd_hooks_add_hooks_version (runner);
-                        glusterd_hooks_add_op (runner, "reset");
-                        glusterd_hooks_add_working_dir (runner, priv);
-                        break;
-
-                default:
-                        break;
-
-        }
-
-        return ret;
+    int vol_count = 0;
+    gf_boolean_t truth = _gf_false;
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+
+    priv = THIS->private;
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (glusterd_is_volume_started(voliter))
+            vol_count++;
+    }
+
+    ret = 0;
+    switch (op) {
+        case GD_OP_START_VOLUME:
+            if (type == GD_COMMIT_HOOK_PRE && vol_count == 0)
+                truth = _gf_true;
+
+            else if (type == GD_COMMIT_HOOK_POST && vol_count == 1)
+                truth = _gf_true;
+
+            else
+                truth = _gf_false;
+
+            runner_argprintf(runner, "--first=%s", truth ? "yes" : "no");
+
+            glusterd_hooks_add_hooks_version(runner);
+            glusterd_hooks_add_op(runner, "start");
+            glusterd_hooks_add_working_dir(runner, priv);
+
+            break;
+
+        case GD_OP_STOP_VOLUME:
+            if (type == GD_COMMIT_HOOK_PRE && vol_count == 1)
+                truth = _gf_true;
+
+            else if (type == GD_COMMIT_HOOK_POST && vol_count == 0)
+                truth = _gf_true;
+
+            else
+                truth = _gf_false;
+
+            runner_argprintf(runner, "--last=%s", truth ? "yes" : "no");
+            break;
+
+        case GD_OP_SET_VOLUME:
+            ret = glusterd_hooks_set_volume_args(op_ctx, runner);
+            glusterd_hooks_add_working_dir(runner, priv);
+            break;
+
+        case GD_OP_GSYNC_CREATE:
+            glusterd_hooks_add_custom_args(op_ctx, runner);
+            break;
+
+        case GD_OP_ADD_BRICK:
+            glusterd_hooks_add_hooks_version(runner);
+            glusterd_hooks_add_op(runner, "add-brick");
+            glusterd_hooks_add_working_dir(runner, priv);
+            break;
+
+        case GD_OP_RESET_VOLUME:
+            glusterd_hooks_add_hooks_version(runner);
+            glusterd_hooks_add_op(runner, "reset");
+            glusterd_hooks_add_working_dir(runner, priv);
+            break;
+
+        default:
+            break;
+    }
+
+    return ret;
 }
 
 int
-glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
-                          glusterd_commit_hook_type_t type)
+glusterd_hooks_run_hooks(char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type)
 {
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        runner_t        runner                  = {0, };
-        struct dirent   *entry                  = NULL;
-        DIR             *hookdir                = NULL;
-        char            *volname                = NULL;
-        char            **lines                 = NULL;
-        int             N                       = 8; /*arbitrary*/
-        int             lineno                  = 0;
-        int             line_count              = 0;
-        int             ret                     = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        ret = dict_get_str (op_ctx, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get volname "
-                        "from operation context");
+    xlator_t *this = NULL;
+    runner_t runner = {
+        0,
+    };
+    DIR *hookdir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char *volname = NULL;
+    char **lines = NULL;
+    int N = 8; /*arbitrary*/
+    int lineno = 0;
+    int line_count = 0;
+    int ret = -1;
+
+    this = THIS;
+
+    ret = dict_get_str(op_ctx, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volname "
+               "from operation context");
+        goto out;
+    }
+
+    hookdir = sys_opendir(hooks_path);
+    if (!hookdir) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to open dir %s", hooks_path);
+        goto out;
+    }
+
+    lines = GF_CALLOC(1, N * sizeof(*lines), gf_gld_mt_charptr);
+    if (!lines) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+    line_count = 0;
+
+    while ((entry = sys_readdir(hookdir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (line_count == N - 1) {
+            N *= 2;
+            lines = GF_REALLOC(lines, N * sizeof(char *));
+            if (!lines) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
                 goto out;
+            }
         }
 
-        hookdir = sys_opendir (hooks_path);
-        if (!hookdir) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "Failed to open dir %s",
-                        hooks_path);
-                goto out;
+        if (glusterd_is_hook_enabled(entry->d_name)) {
+            lines[line_count] = gf_strdup(entry->d_name);
+            line_count++;
         }
+    }
 
-        lines = GF_CALLOC (1, N * sizeof (*lines), gf_gld_mt_charptr);
-        if (!lines) {
-                ret = -1;
-                goto out;
-        }
+    lines[line_count] = NULL;
+    lines = GF_REALLOC(lines, (line_count + 1) * sizeof(char *));
+    if (!lines)
+        goto out;
 
-        ret = -1;
-        line_count = 0;
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, hookdir);
-        while (entry) {
-                if (line_count == N-1) {
-                        N *= 2;
-                        lines = GF_REALLOC (lines, N * sizeof (char *));
-                        if (!lines)
-                                goto out;
-                }
-
-                if (glusterd_is_hook_enabled (entry->d_name)) {
-                        lines[line_count] = gf_strdup (entry->d_name);
-                        line_count++;
-                }
-
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, hookdir);
-        }
+    qsort(lines, line_count, sizeof(*lines), glusterd_compare_lines);
 
-        lines[line_count] = NULL;
-        lines = GF_REALLOC (lines, (line_count + 1) * sizeof (char *));
-        if (!lines)
-                goto out;
+    for (lineno = 0; lineno < line_count; lineno++) {
+        runinit(&runner);
+        runner_argprintf(&runner, "%s/%s", hooks_path, lines[lineno]);
+        /*Add future command line arguments to hook scripts below*/
+        runner_argprintf(&runner, "--volname=%s", volname);
+        ret = glusterd_hooks_add_op_args(&runner, op, op_ctx, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ADD_OP_ARGS_FAIL,
+                   "Failed to add "
+                   "command specific arguments");
+            goto out;
+        }
 
-        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
-
-        for (lineno = 0; lineno < line_count; lineno++) {
-
-                runinit (&runner);
-                runner_argprintf (&runner, "%s/%s", hooks_path, lines[lineno]);
-                /*Add future command line arguments to hook scripts below*/
-                runner_argprintf (&runner, "--volname=%s", volname);
-                ret = glusterd_hooks_add_op_args (&runner, op, op_ctx, type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_ADD_OP_ARGS_FAIL, "Failed to add "
-                                "command specific arguments");
-                        goto out;
-                }
-
-                ret = runner_run_reuse (&runner);
-                if (ret) {
-                        runner_log (&runner, this->name, GF_LOG_ERROR,
-                                    "Failed to execute script");
-                } else {
-                        runner_log (&runner, this->name, GF_LOG_INFO,
-                                    "Ran script");
-                }
-                runner_end (&runner);
+        ret = runner_run_reuse(&runner);
+        if (ret) {
+            runner_log(&runner, this->name, GF_LOG_ERROR,
+                       "Failed to execute script");
+        } else {
+            runner_log(&runner, this->name, GF_LOG_INFO, "Ran script");
         }
+        runner_end(&runner);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (lines) {
-                for (lineno = 0; lineno < line_count+1; lineno++)
-                        GF_FREE (lines[lineno]);
+    if (lines) {
+        for (lineno = 0; lineno < line_count + 1; lineno++)
+            GF_FREE(lines[lineno]);
 
-                GF_FREE (lines);
-        }
+        GF_FREE(lines);
+    }
 
-        if (hookdir)
-                sys_closedir (hookdir);
+    if (hookdir)
+        sys_closedir(hookdir);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
-                                  dict_t *op_ctx)
+glusterd_hooks_post_stub_enqueue(char *scriptdir, glusterd_op_t op,
+                                 dict_t *op_ctx)
 {
-        int                     ret     = -1;
-        glusterd_hooks_stub_t   *stub   = NULL;
-        glusterd_hooks_private_t *hooks_priv = NULL;
-        glusterd_conf_t         *conf   = NULL;
-
-        conf = THIS->private;
-        hooks_priv = conf->hooks_priv;
-
-        ret = glusterd_hooks_stub_init (&stub, scriptdir, op, op_ctx);
-        if (ret)
-                goto out;
-
-        pthread_mutex_lock (&hooks_priv->mutex);
-        {
-                hooks_priv->waitcount++;
-                cds_list_add_tail (&stub->all_hooks, &hooks_priv->list);
-                pthread_cond_signal (&hooks_priv->cond);
-        }
-        pthread_mutex_unlock (&hooks_priv->mutex);
-
-        ret = 0;
+    int ret = -1;
+    glusterd_hooks_stub_t *stub = NULL;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    conf = THIS->private;
+    hooks_priv = conf->hooks_priv;
+
+    ret = glusterd_hooks_stub_init(&stub, scriptdir, op, op_ctx);
+    if (ret)
+        goto out;
+
+    pthread_mutex_lock(&hooks_priv->mutex);
+    {
+        hooks_priv->waitcount++;
+        cds_list_add_tail(&stub->all_hooks, &hooks_priv->list);
+        pthread_cond_signal(&hooks_priv->cond);
+    }
+    pthread_mutex_unlock(&hooks_priv->mutex);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
-                          glusterd_op_t op, dict_t *op_ctx)
+glusterd_hooks_stub_init(glusterd_hooks_stub_t **stub, char *scriptdir,
+                         glusterd_op_t op, dict_t *op_ctx)
 {
-        int                     ret             = -1;
-        glusterd_hooks_stub_t   *hooks_stub     = NULL;
-
-        GF_ASSERT (stub);
-        if (!stub)
-                goto out;
-
-        hooks_stub = GF_CALLOC (1, sizeof (*hooks_stub),
-                                gf_gld_mt_hooks_stub_t);
-        if (!hooks_stub)
-                goto out;
-
-        CDS_INIT_LIST_HEAD (&hooks_stub->all_hooks);
-        hooks_stub->op = op;
-        hooks_stub->scriptdir = gf_strdup (scriptdir);
-        if (!hooks_stub->scriptdir)
-                goto out;
-
-        hooks_stub->op_ctx = dict_copy_with_ref (op_ctx, hooks_stub->op_ctx);
-        if (!hooks_stub->op_ctx)
-                goto out;
-
-        *stub = hooks_stub;
-        ret = 0;
+    int ret = -1;
+    glusterd_hooks_stub_t *hooks_stub = NULL;
+
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(stub);
+    if (!stub)
+        goto out;
+
+    hooks_stub = GF_CALLOC(1, sizeof(*hooks_stub), gf_gld_mt_hooks_stub_t);
+    if (!hooks_stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    CDS_INIT_LIST_HEAD(&hooks_stub->all_hooks);
+    hooks_stub->op = op;
+    hooks_stub->scriptdir = gf_strdup(scriptdir);
+    if (!hooks_stub->scriptdir) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "scriptdir=%s", scriptdir, NULL);
+        goto out;
+    }
+
+    hooks_stub->op_ctx = dict_copy_with_ref(op_ctx, hooks_stub->op_ctx);
+    if (!hooks_stub->op_ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    *stub = hooks_stub;
+    ret = 0;
 out:
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_POST_HOOK_STUB_INIT_FAIL, "Failed to initialize "
-                        "post hooks stub");
-                glusterd_hooks_stub_cleanup (hooks_stub);
-        }
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_HOOK_STUB_INIT_FAIL,
+                NULL);
+        glusterd_hooks_stub_cleanup(hooks_stub);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub)
+glusterd_hooks_stub_cleanup(glusterd_hooks_stub_t *stub)
 {
-        if (!stub) {
-                gf_msg_callingfn (THIS->name, GF_LOG_WARNING, 0,
-                                  GD_MSG_HOOK_STUB_NULL,
-                                  "hooks_stub is NULL");
-                return;
-        }
+    if (!stub) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, GD_MSG_HOOK_STUB_NULL,
+                         "hooks_stub is NULL");
+        return;
+    }
 
-        if (stub->op_ctx)
-                dict_unref (stub->op_ctx);
+    if (stub->op_ctx)
+        dict_unref(stub->op_ctx);
 
-        GF_FREE (stub->scriptdir);
+    GF_FREE(stub->scriptdir);
 
-        GF_FREE (stub);
+    GF_FREE(stub);
 }
 
-static void*
-hooks_worker (void *args)
+static void *
+hooks_worker(void *args)
 {
-        glusterd_conf_t *conf = NULL;
-        glusterd_hooks_private_t *hooks_priv = NULL;
-        glusterd_hooks_stub_t *stub = NULL;
-
-        THIS = args;
-        conf = THIS->private;
-        hooks_priv = conf->hooks_priv;
-
-        for (;;) {
-                pthread_mutex_lock (&hooks_priv->mutex);
-                {
-                        while (cds_list_empty (&hooks_priv->list)) {
-                                pthread_cond_wait (&hooks_priv->cond,
-                                                   &hooks_priv->mutex);
-                        }
-                        stub = cds_list_entry (hooks_priv->list.next,
-                                               glusterd_hooks_stub_t,
-                                               all_hooks);
-                        cds_list_del_init (&stub->all_hooks);
-                        hooks_priv->waitcount--;
-
-                }
-                pthread_mutex_unlock (&hooks_priv->mutex);
-
-                glusterd_hooks_run_hooks (stub->scriptdir, stub->op,
-                                          stub->op_ctx, GD_COMMIT_HOOK_POST);
-                glusterd_hooks_stub_cleanup (stub);
+    glusterd_conf_t *conf = NULL;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+    glusterd_hooks_stub_t *stub = NULL;
+
+    THIS = args;
+    conf = THIS->private;
+    hooks_priv = conf->hooks_priv;
+
+    for (;;) {
+        pthread_mutex_lock(&hooks_priv->mutex);
+        {
+            while (cds_list_empty(&hooks_priv->list)) {
+                pthread_cond_wait(&hooks_priv->cond, &hooks_priv->mutex);
+            }
+            stub = cds_list_entry(hooks_priv->list.next, glusterd_hooks_stub_t,
+                                  all_hooks);
+            cds_list_del_init(&stub->all_hooks);
+            hooks_priv->waitcount--;
         }
+        pthread_mutex_unlock(&hooks_priv->mutex);
 
-        return NULL;
+        glusterd_hooks_run_hooks(stub->scriptdir, stub->op, stub->op_ctx,
+                                 GD_COMMIT_HOOK_POST);
+        glusterd_hooks_stub_cleanup(stub);
+    }
+
+    return NULL;
 }
 
 int
-glusterd_hooks_priv_init (glusterd_hooks_private_t **new)
+glusterd_hooks_priv_init(glusterd_hooks_private_t **new)
 {
-        int                      ret            = -1;
-        glusterd_hooks_private_t *hooks_priv    = NULL;
-
-        if (!new)
-                goto out;
-
-        hooks_priv = GF_CALLOC (1, sizeof (*hooks_priv),
-                                gf_gld_mt_hooks_priv_t);
-        if (!hooks_priv)
-                goto out;
-
-        pthread_mutex_init (&hooks_priv->mutex, NULL);
-        pthread_cond_init (&hooks_priv->cond, NULL);
-        CDS_INIT_LIST_HEAD (&hooks_priv->list);
-        hooks_priv->waitcount = 0;
-
-        *new = hooks_priv;
-        ret = 0;
+    int ret = -1;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!new) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    hooks_priv = GF_CALLOC(1, sizeof(*hooks_priv), gf_gld_mt_hooks_priv_t);
+    if (!hooks_priv) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    pthread_mutex_init(&hooks_priv->mutex, NULL);
+    pthread_cond_init(&hooks_priv->cond, NULL);
+    CDS_INIT_LIST_HEAD(&hooks_priv->list);
+    hooks_priv->waitcount = 0;
+
+    *new = hooks_priv;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_hooks_spawn_worker (xlator_t *this)
+glusterd_hooks_spawn_worker(xlator_t *this)
 {
-        int                       ret          = -1;
-        glusterd_conf_t           *conf        = NULL;
-        glusterd_hooks_private_t  *hooks_priv  = NULL;
-
-
-        ret = glusterd_hooks_priv_init (&hooks_priv);
-        if (ret)
-                goto out;
-
-        conf = this->private;
-        conf->hooks_priv = hooks_priv;
-        ret = pthread_create (&hooks_priv->worker, NULL, hooks_worker,
-                              (void *)this);
-        if (ret)
-                gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                        GD_MSG_SPAWN_THREADS_FAIL, "Failed to spawn post "
-                        "hooks worker thread");
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+
+    ret = glusterd_hooks_priv_init(&hooks_priv);
+    if (ret)
+        goto out;
+
+    conf = this->private;
+    conf->hooks_priv = hooks_priv;
+    ret = gf_thread_create(&hooks_priv->worker, NULL, hooks_worker,
+                           (void *)this, "gdhooks");
+    if (ret)
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_SPAWN_THREADS_FAIL,
+               "Failed to spawn post "
+               "hooks worker thread");
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.h b/xlators/mgmt/glusterd/src/glusterd-hooks.h
index 7bab6adb626..f8b887b9bd7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-hooks.h
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.h
@@ -12,73 +12,77 @@
 
 #include <fnmatch.h>
 
-#define GLUSTERD_GET_HOOKS_DIR(path, version, priv) \
-        snprintf (path, PATH_MAX, "%s/hooks/%d", priv->workdir,\
-                  version);
+#define GLUSTERD_GET_HOOKS_DIR(path, version, priv)                            \
+    do {                                                                       \
+        int32_t len;                                                           \
+        len = snprintf(path, PATH_MAX, "%s/hooks/%d", priv->workdir, version); \
+        if (len < 0) {                                                         \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
 
-#define GLUSTERD_HOOK_VER       1
+#define GLUSTERD_HOOK_VER 1
 
-#define GD_HOOKS_SPECIFIC_KEY   "user.*"
+#define GD_HOOKS_SPECIFIC_KEY "user.*"
 
 typedef enum glusterd_commit_hook_type {
-        GD_COMMIT_HOOK_NONE = 0,
-        GD_COMMIT_HOOK_PRE,
-        GD_COMMIT_HOOK_POST,
-        GD_COMMIT_HOOK_MAX
+    GD_COMMIT_HOOK_NONE = 0,
+    GD_COMMIT_HOOK_PRE,
+    GD_COMMIT_HOOK_POST,
+    GD_COMMIT_HOOK_MAX
 } glusterd_commit_hook_type_t;
 
 typedef struct hooks_private {
-        struct cds_list_head    list;
-        int                     waitcount; //debug purposes
-        pthread_mutex_t         mutex;
-        pthread_cond_t          cond;
-        pthread_t               worker;
+    struct cds_list_head list;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t worker;
+    int waitcount;  // debug purposes
 } glusterd_hooks_private_t;
 
 typedef struct hooks_stub {
-        struct cds_list_head    all_hooks;
-        char                    *scriptdir;
-        glusterd_op_t           op;
-        dict_t                  *op_ctx;
+    struct cds_list_head all_hooks;
+    char *scriptdir;
+    dict_t *op_ctx;
+    glusterd_op_t op;
 
 } glusterd_hooks_stub_t;
 
-
 static inline gf_boolean_t
-is_key_glusterd_hooks_friendly (char *key)
+is_key_glusterd_hooks_friendly(char *key)
 {
-        gf_boolean_t is_friendly = _gf_false;
+    gf_boolean_t is_friendly = _gf_false;
 
-        /* This is very specific to hooks friendly behavior */
-        if (fnmatch (GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
-                gf_msg_debug (THIS->name, 0, "user namespace key %s", key);
-                is_friendly = _gf_true;
-        }
+    /* This is very specific to hooks friendly behavior */
+    if (fnmatch(GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
+        gf_msg_debug(THIS->name, 0, "user namespace key %s", key);
+        is_friendly = _gf_true;
+    }
 
-        return is_friendly;
+    return is_friendly;
 }
 
 int
-glusterd_hooks_create_hooks_directory (char *basedir);
+glusterd_hooks_create_hooks_directory(char *basedir);
 
 char *
-glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op);
+glusterd_hooks_get_hooks_cmd_subdir(glusterd_op_t op);
 
 int
-glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
-                          glusterd_commit_hook_type_t type);
+glusterd_hooks_run_hooks(char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type);
 int
-glusterd_hooks_spawn_worker (xlator_t *this);
+glusterd_hooks_spawn_worker(xlator_t *this);
 
 int
-glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
-                          glusterd_op_t op, dict_t *op_ctx);
+glusterd_hooks_stub_init(glusterd_hooks_stub_t **stub, char *scriptdir,
+                         glusterd_op_t op, dict_t *op_ctx);
 void
-glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub);
+glusterd_hooks_stub_cleanup(glusterd_hooks_stub_t *stub);
 
 int
-glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
-                                      dict_t *op_ctx);
+glusterd_hooks_post_stub_enqueue(char *scriptdir, glusterd_op_t op,
+                                 dict_t *op_ctx);
 int
-glusterd_hooks_priv_init (glusterd_hooks_private_t **new);
+glusterd_hooks_priv_init(glusterd_hooks_private_t **new);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c
index 146092db79b..11523f2854b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-locks.c
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c
@@ -7,7 +7,7 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -17,8 +17,8 @@
 #include "glusterd-volgen.h"
 #include "glusterd-locks.h"
 #include "glusterd-errno.h"
-#include "run.h"
-#include "syscall.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 #include "glusterd-messages.h"
 
 #include <signal.h>
@@ -28,161 +28,194 @@
 /* Valid entities that the mgmt_v3 lock can hold locks upon    *
  * To add newer entities to be locked, we can just add more    *
  * entries to this table along with the type and default value */
-glusterd_valid_entities   valid_types[] = {
-        { "vol",  _gf_true  },
-        { "snap", _gf_false },
-        { "global", _gf_false},
-        { NULL              },
+glusterd_valid_entities valid_types[] = {
+    {"vol", _gf_true},
+    {"snap", _gf_false},
+    {"global", _gf_false},
+    {NULL},
 };
 
 /* Checks if the lock request is for a valid entity */
-gf_boolean_t
-glusterd_mgmt_v3_is_type_valid (char *type)
+static gf_boolean_t
+glusterd_mgmt_v3_is_type_valid(char *type)
 {
-        int32_t         i   = 0;
-        gf_boolean_t    ret = _gf_false;
+    int i = 0;
 
-        GF_ASSERT (type);
+    GF_ASSERT(type);
 
-        for (i = 0; valid_types[i].type; i++) {
-                if (!strcmp (type, valid_types[i].type)) {
-                        ret = _gf_true;
-                        break;
-                }
+    for (i = 0; valid_types[i].type; i++) {
+        if (!strcmp(type, valid_types[i].type)) {
+            return _gf_true;
         }
+    }
 
-        return ret;
+    return _gf_false;
 }
 
 /* Initialize the global mgmt_v3 lock list(dict) when
  * glusterd is spawned */
 int32_t
-glusterd_mgmt_v3_lock_init ()
+glusterd_mgmt_v3_lock_init()
 {
-        int32_t             ret = -1;
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        priv->mgmt_v3_lock = dict_new ();
-        if (!priv->mgmt_v3_lock)
-                goto out;
+    priv->mgmt_v3_lock = dict_new();
+    if (!priv->mgmt_v3_lock)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Destroy the global mgmt_v3 lock list(dict) when
  * glusterd cleanup is performed */
 void
-glusterd_mgmt_v3_lock_fini ()
+glusterd_mgmt_v3_lock_fini()
 {
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->mgmt_v3_lock)
-                dict_unref (priv->mgmt_v3_lock);
+    if (priv->mgmt_v3_lock)
+        dict_unref(priv->mgmt_v3_lock);
 }
 
+/* Initialize the global mgmt_v3_timer lock list(dict) when
+ * glusterd is spawned */
 int32_t
-glusterd_get_mgmt_v3_lock_owner (char *key, uuid_t *uuid)
+glusterd_mgmt_v3_lock_timer_init()
 {
-        int32_t                         ret      = -1;
-        glusterd_mgmt_v3_lock_obj      *lock_obj = NULL;
-        glusterd_conf_t                *priv     = NULL;
-        uuid_t                          no_owner = {0,};
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!key || !uuid) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "key or uuid is null.");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        ret = dict_get_bin (priv->mgmt_v3_lock, key, (void **) &lock_obj);
-        if (!ret)
-                gf_uuid_copy (*uuid, lock_obj->lock_owner);
-        else
-                gf_uuid_copy (*uuid, no_owner);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        ret = 0;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    priv->mgmt_v3_lock_timer = dict_new();
+    if (!priv->mgmt_v3_lock_timer)
+        goto out;
+
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    return ret;
+}
+
+/* Destroy the global mgmt_v3_timer lock list(dict) when
+ * glusterd cleanup is performed */
+void
+glusterd_mgmt_v3_lock_timer_fini()
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (priv->mgmt_v3_lock_timer)
+        dict_unref(priv->mgmt_v3_lock_timer);
+out:
+    return;
+}
+
+static int32_t
+glusterd_get_mgmt_v3_lock_owner(char *key, uuid_t *uuid)
+{
+    int32_t ret = -1;
+    glusterd_mgmt_v3_lock_obj *lock_obj = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!key || !uuid) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "key or uuid is null.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(priv->mgmt_v3_lock, key, (void **)&lock_obj);
+    if (!ret)
+        gf_uuid_copy(*uuid, lock_obj->lock_owner);
+
+    ret = 0;
+out:
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function is called with the locked_count and type, to   *
  * release all the acquired locks. */
 static int32_t
-glusterd_release_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
-                                            int32_t locked_count,
-                                            char *type)
+glusterd_release_multiple_locks_per_entity(dict_t *dict, uuid_t uuid,
+                                           int32_t locked_count, char *type)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        i                     = -1;
-        int32_t        op_ret                = 0;
-        int32_t        ret                   = -1;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        if (locked_count == 0) {
-                gf_msg_debug (this->name, 0,
-                        "No %s locked as part of this transaction",
-                        type);
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t i = -1;
+    int32_t op_ret = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    if (locked_count == 0) {
+        gf_msg_debug(this->name, 0, "No %s locked as part of this transaction",
+                     type);
+        goto out;
+    }
+
+    /* Release all the locks held */
+    for (i = 0; i < locked_count; i++) {
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname%d", type, i + 1);
+
+        /* Looking for volname1, volname2 or snapname1, *
+         * as key in the dict snapname2 */
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s locked_count = %d", name_buf,
+                   locked_count);
+            op_ret = ret;
+            continue;
         }
 
-        /* Release all the locks held */
-        for (i = 0; i < locked_count; i++) {
-                snprintf (name_buf, sizeof(name_buf),
-                          "%sname%d", type, i+1);
-
-                /* Looking for volname1, volname2 or snapname1, *
-                 * as key in the dict snapname2 */
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get %s locked_count = %d",
-                                name_buf, locked_count);
-                        op_ret = ret;
-                        continue;
-                }
-
-                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                "Failed to release lock for %s.",
-                                name);
-                        op_ret = ret;
-                }
+        ret = glusterd_mgmt_v3_unlock(name, uuid, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release lock for %s.", name);
+            op_ret = ret;
         }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", op_ret);
-        return op_ret;
+    gf_msg_trace(this->name, 0, "Returning %d", op_ret);
+    return op_ret;
 }
 
 /* Given the count and type of the entity this function acquires     *
@@ -190,71 +223,64 @@ out:
  * If type is "vol" this function tries to acquire locks on multiple *
  * volumes */
 static int32_t
-glusterd_acquire_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
-                                            uint32_t *op_errno,
-                                            int32_t count, char *type)
+glusterd_acquire_multiple_locks_per_entity(dict_t *dict, uuid_t uuid,
+                                           uint32_t *op_errno, int32_t count,
+                                           char *type)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        i                     = -1;
-        int32_t        ret                   = -1;
-        int32_t        locked_count          = 0;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        /* Locking one element after other */
-        for (i = 0; i < count; i++) {
-                snprintf (name_buf, sizeof(name_buf),
-                          "%sname%d", type, i+1);
-
-                /* Looking for volname1, volname2 or snapname1, *
-                 * as key in the dict snapname2 */
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get %s count = %d",
-                                name_buf, count);
-                        break;
-                }
-
-                ret = glusterd_mgmt_v3_lock (name, uuid, op_errno, type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                                "Failed to acquire lock for %s %s "
-                                "on behalf of %s. Reversing "
-                                "this transaction", type, name,
-                                uuid_utoa(uuid));
-                        break;
-                }
-                locked_count++;
-        }
-
-        if (count == locked_count) {
-                /* If all locking ops went successfuly, return as success */
-                ret = 0;
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t locked_count = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    /* Locking one element after other */
+    for (i = 0; i < count; i++) {
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname%d", type, i + 1);
+
+        /* Looking for volname1, volname2 or snapname1, *
+         * as key in the dict snapname2 */
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s count = %d", name_buf, count);
+            break;
         }
 
-        /* If we failed to lock one element, unlock others and return failure */
-        ret = glusterd_release_multiple_locks_per_entity (dict, uuid,
-                                                          locked_count,
-                                                          type);
+        ret = glusterd_mgmt_v3_lock(name, uuid, op_errno, type);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
-                        "Failed to release multiple %s locks",
-                        type);
-        }
-        ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire lock for %s %s "
+                   "on behalf of %s. Reversing "
+                   "this transaction",
+                   type, name, uuid_utoa(uuid));
+            break;
+        }
+        locked_count++;
+    }
+
+    if (count == locked_count) {
+        /* If all locking ops went successfully, return as success */
+        ret = 0;
+        goto out;
+    }
+
+    /* If we failed to lock one element, unlock others and return failure */
+    ret = glusterd_release_multiple_locks_per_entity(dict, uuid, locked_count,
+                                                     type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
+               "Failed to release multiple %s locks", type);
+    }
+    ret = -1;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Given the type of entity, this function figures out if it should unlock a   *
@@ -262,74 +288,69 @@ out:
  * if the type is "vol", this function will accordingly unlock a single volume *
  * or multiple volumes */
 static int32_t
-glusterd_mgmt_v3_unlock_entity (dict_t *dict, uuid_t uuid, char *type,
-                                gf_boolean_t default_value)
+glusterd_mgmt_v3_unlock_entity(dict_t *dict, uuid_t uuid, char *type,
+                               gf_boolean_t default_value)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        count                 = -1;
-        int32_t        ret                   = -1;
-        gf_boolean_t   hold_locks            = _gf_false;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
-        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
-
-        if (hold_locks == _gf_false) {
-                /* Locks were not held for this particular entity *
-                 * Hence nothing to release */
-                ret = 0;
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t count = -1;
+    int32_t ret = -1;
+    gf_boolean_t hold_locks = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    snprintf(name_buf, sizeof(name_buf), "hold_%s_locks", type);
+    hold_locks = dict_get_str_boolean(dict, name_buf, default_value);
+
+    if (hold_locks == _gf_false) {
+        /* Locks were not held for this particular entity *
+         * Hence nothing to release */
+        ret = 0;
+        goto out;
+    }
+
+    /* Looking for volcount or snapcount in the dict */
+    ret = snprintf(name_buf, sizeof(name_buf), "%scount", type);
+    ret = dict_get_int32n(dict, name_buf, ret, &count);
+    if (ret) {
+        /* count is not present. Only one *
+         * element name needs to be unlocked */
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname", type);
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %sname", type);
+            goto out;
         }
 
-        /* Looking for volcount or snapcount in the dict */
-        snprintf (name_buf, sizeof(name_buf), "%scount", type);
-        ret = dict_get_int32 (dict, name_buf, &count);
+        ret = glusterd_mgmt_v3_unlock(name, uuid, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release lock for %s %s "
+                   "on behalf of %s.",
+                   type, name, uuid_utoa(uuid));
+            goto out;
+        }
+    } else {
+        /* Unlocking one element name after another */
+        ret = glusterd_release_multiple_locks_per_entity(dict, uuid, count,
+                                                         type);
         if (ret) {
-                /* count is not present. Only one *
-                 * element name needs to be unlocked */
-                snprintf (name_buf, sizeof(name_buf), "%sname",
-                          type);
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to fetch %sname", type);
-                        goto out;
-                }
-
-                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                "Failed to release lock for %s %s "
-                                "on behalf of %s.", type, name,
-                                uuid_utoa(uuid));
-                        goto out;
-                }
-        } else {
-                /* Unlocking one element name after another */
-                ret = glusterd_release_multiple_locks_per_entity (dict,
-                                                                  uuid,
-                                                                  count,
-                                                                  type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
-                                "Failed to release all %s locks", type);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
+                   "Failed to release all %s locks", type);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Given the type of entity, this function figures out if it should lock a   *
@@ -337,378 +358,513 @@ out:
  * if the type is "vol", this function will accordingly lock a single volume *
  * or multiple volumes */
 static int32_t
-glusterd_mgmt_v3_lock_entity (dict_t *dict, uuid_t uuid, uint32_t *op_errno,
-                              char *type, gf_boolean_t default_value)
+glusterd_mgmt_v3_lock_entity(dict_t *dict, uuid_t uuid, uint32_t *op_errno,
+                             char *type, gf_boolean_t default_value)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        count                 = -1;
-        int32_t        ret                   = -1;
-        gf_boolean_t   hold_locks            = _gf_false;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
-        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
-
-        if (hold_locks == _gf_false) {
-                /* Not holding locks for this particular entity */
-                ret = 0;
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t count = -1;
+    int32_t ret = -1;
+    gf_boolean_t hold_locks = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    snprintf(name_buf, sizeof(name_buf), "hold_%s_locks", type);
+    hold_locks = dict_get_str_boolean(dict, name_buf, default_value);
+
+    if (hold_locks == _gf_false) {
+        /* Not holding locks for this particular entity */
+        ret = 0;
+        goto out;
+    }
+
+    /* Looking for volcount or snapcount in the dict */
+    ret = snprintf(name_buf, sizeof(name_buf), "%scount", type);
+    ret = dict_get_int32n(dict, name_buf, ret, &count);
+    if (ret) {
+        /* count is not present. Only one *
+         * element name needs to be locked */
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname", type);
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %sname", type);
+            goto out;
         }
 
-        /* Looking for volcount or snapcount in the dict */
-        snprintf (name_buf, sizeof(name_buf), "%scount", type);
-        ret = dict_get_int32 (dict, name_buf, &count);
+        ret = glusterd_mgmt_v3_lock(name, uuid, op_errno, type);
         if (ret) {
-                /* count is not present. Only one *
-                 * element name needs to be locked */
-                snprintf (name_buf, sizeof(name_buf), "%sname",
-                          type);
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to fetch %sname", type);
-                        goto out;
-                }
-
-                ret = glusterd_mgmt_v3_lock (name, uuid, op_errno, type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                                "Failed to acquire lock for %s %s "
-                                "on behalf of %s.", type, name,
-                                uuid_utoa(uuid));
-                        goto out;
-                }
-        } else {
-                /* Locking one element name after another */
-                ret = glusterd_acquire_multiple_locks_per_entity (dict,
-                                                                  uuid,
-                                                                  op_errno,
-                                                                  count,
-                                                                  type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL,
-                                "Failed to acquire all %s locks", type);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire lock for %s %s "
+                   "on behalf of %s.",
+                   type, name, uuid_utoa(uuid));
+            goto out;
+        }
+    } else {
+        /* Locking one element name after another */
+        ret = glusterd_acquire_multiple_locks_per_entity(dict, uuid, op_errno,
+                                                         count, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL,
+                   "Failed to acquire all %s locks", type);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Try to release locks of multiple entities like *
  * volume, snaps etc. */
 int32_t
-glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid)
+glusterd_multiple_mgmt_v3_unlock(dict_t *dict, uuid_t uuid)
 {
-        int32_t        i                     = -1;
-        int32_t        ret                   = -1;
-        int32_t        op_ret                = 0;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "dict is null.");
-                ret = -1;
-                goto out;
-        }
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t op_ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is null.");
+        ret = -1;
+        goto out;
+    }
 
-        for (i = 0; valid_types[i].type; i++) {
-                ret = glusterd_mgmt_v3_unlock_entity
-                                            (dict, uuid,
-                                             valid_types[i].type,
+    for (i = 0; valid_types[i].type; i++) {
+        ret = glusterd_mgmt_v3_unlock_entity(dict, uuid, valid_types[i].type,
                                              valid_types[i].default_value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
-                                "Unable to unlock all %s",
-                                valid_types[i].type);
-                        op_ret = ret;
-                }
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL, "Unable to unlock all %s",
+                   valid_types[i].type);
+            op_ret = ret;
         }
+    }
 
-        ret = op_ret;
+    ret = op_ret;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Try to acquire locks on multiple entities like *
  * volume, snaps etc. */
 int32_t
-glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid, uint32_t *op_errno)
+glusterd_multiple_mgmt_v3_lock(dict_t *dict, uuid_t uuid, uint32_t *op_errno)
 {
-        int32_t        i                     = -1;
-        int32_t        ret                   = -1;
-        int32_t        locked_count          = 0;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "dict is null.");
-                ret = -1;
-                goto out;
-        }
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t locked_count = 0;
+    xlator_t *this = NULL;
 
-        /* Locking one entity after other */
-        for (i = 0; valid_types[i].type; i++) {
-                ret = glusterd_mgmt_v3_lock_entity
-                                            (dict, uuid, op_errno,
-                                             valid_types[i].type,
-                                             valid_types[i].default_value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL,
-                                "Unable to lock all %s",
-                                valid_types[i].type);
-                        break;
-                }
-                locked_count++;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (locked_count == GF_MAX_LOCKING_ENTITIES) {
-                /* If all locking ops went successfuly, return as success */
-                ret = 0;
-                goto out;
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is null.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Locking one entity after other */
+    for (i = 0; valid_types[i].type; i++) {
+        ret = glusterd_mgmt_v3_lock_entity(dict, uuid, op_errno,
+                                           valid_types[i].type,
+                                           valid_types[i].default_value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL, "Unable to lock all %s",
+                   valid_types[i].type);
+            break;
         }
+        locked_count++;
+    }
 
-        /* If we failed to lock one entity, unlock others and return failure */
-        for (i = 0; i < locked_count; i++) {
-                ret = glusterd_mgmt_v3_unlock_entity
-                                              (dict, uuid,
-                                               valid_types[i].type,
-                                               valid_types[i].default_value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
-                                "Unable to unlock all %s",
-                                valid_types[i].type);
-                }
+    if (locked_count == GF_MAX_LOCKING_ENTITIES) {
+        /* If all locking ops went successfully, return as success */
+        ret = 0;
+        goto out;
+    }
+
+    /* If we failed to lock one entity, unlock others and return failure */
+    for (i = 0; i < locked_count; i++) {
+        ret = glusterd_mgmt_v3_unlock_entity(dict, uuid, valid_types[i].type,
+                                             valid_types[i].default_value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL, "Unable to unlock all %s",
+                   valid_types[i].type);
         }
-        ret = -1;
+    }
+    ret = -1;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, uint32_t *op_errno,
-                       char *type)
+glusterd_mgmt_v3_lock(const char *name, uuid_t uuid, uint32_t *op_errno,
+                      char *type)
 {
-        char                            key[PATH_MAX]   = "";
-        int32_t                         ret             = -1;
-        glusterd_mgmt_v3_lock_obj      *lock_obj        = NULL;
-        glusterd_conf_t                *priv            = NULL;
-        gf_boolean_t                    is_valid        = _gf_true;
-        uuid_t                          owner           = {0};
-        xlator_t                       *this            = NULL;
-        char                           *bt              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!name || !type) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "name or type is null.");
-                ret = -1;
-                goto out;
-        }
-
-        is_valid = glusterd_mgmt_v3_is_type_valid (type);
-        if (is_valid != _gf_true) {
-                gf_msg_callingfn (this->name, GF_LOG_ERROR,
-                        EINVAL, GD_MSG_INVALID_ENTRY,
-                        "Invalid entity. Cannot perform locking "
-                        "operation on %s types", type);
-                ret = -1;
-                goto out;
-        }
+    char key[PATH_MAX] = "";
+    int32_t ret = -1;
+    glusterd_mgmt_v3_lock_obj *lock_obj = NULL;
+    glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t is_valid = _gf_true;
+    uuid_t owner = {0};
+    xlator_t *this = NULL;
+    char *bt = NULL;
+    struct timespec delay = {0};
+    char *key_dup = NULL;
+    glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL;
+    xlator_t *mgmt_lock_timer_xl = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!name || !type) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "name or type is null.");
+        ret = -1;
+        goto out;
+    }
+
+    is_valid = glusterd_mgmt_v3_is_type_valid(type);
+    if (is_valid != _gf_true) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                         "Invalid entity. Cannot perform locking "
+                         "operation on %s types",
+                         type);
+        ret = -1;
+        goto out;
+    }
 
-        ret = snprintf (key, sizeof(key), "%s_%s", name, type);
-        if (ret != strlen(name) + 1 + strlen(type)) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CREATE_KEY_FAIL, "Unable to create key");
-                goto out;
-        }
+    ret = snprintf(key, sizeof(key), "%s_%s", name, type);
+    if (ret != strlen(name) + 1 + strlen(type)) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create key");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Trying to acquire lock of %s for %s", key,
+                 uuid_utoa(uuid));
+
+    ret = glusterd_get_mgmt_v3_lock_owner(key, &owner);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get mgmt_v3 lock owner");
+        goto out;
+    }
+
+    /* If the lock has already been held for the given volume
+     * we fail */
+    if (!gf_uuid_is_null(owner)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         GD_MSG_LOCK_ALREADY_HELD, "Lock for %s held by %s",
+                         name, uuid_utoa(owner));
+        ret = -1;
+        *op_errno = EG_ANOTRANS;
+        goto out;
+    }
 
-        gf_msg_debug (this->name, 0,
-                "Trying to acquire lock of %s %s for %s as %s",
-                type, name, uuid_utoa (uuid), key);
+    lock_obj = GF_MALLOC(sizeof(glusterd_mgmt_v3_lock_obj),
+                         gf_common_mt_mgmt_v3_lock_obj_t);
+    if (!lock_obj) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "Unable to get mgmt_v3 lock owner");
-                goto out;
-        }
+    gf_uuid_copy(lock_obj->lock_owner, uuid);
 
-        /* If the lock has already been held for the given volume
-         * we fail */
-        if (!gf_uuid_is_null (owner)) {
-                gf_msg_callingfn (this->name, GF_LOG_WARNING,
-                                  0, GD_MSG_LOCK_ALREADY_HELD,
-                                  "Lock for %s held by %s",
-                                  name, uuid_utoa (owner));
-                ret = -1;
-                *op_errno = EG_ANOTRANS;
-                goto out;
-        }
+    ret = dict_set_bin(priv->mgmt_v3_lock, key, lock_obj,
+                       sizeof(glusterd_mgmt_v3_lock_obj));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set lock owner in mgmt_v3 lock");
+        GF_FREE(lock_obj);
+        goto out;
+    }
 
-        lock_obj = GF_CALLOC (1, sizeof(glusterd_mgmt_v3_lock_obj),
-                              gf_common_mt_mgmt_v3_lock_obj_t);
-        if (!lock_obj) {
-                ret = -1;
-                goto out;
-        }
+    mgmt_lock_timer = GF_CALLOC(1, sizeof(glusterd_mgmt_v3_lock_timer),
+                                gf_common_mt_mgmt_v3_lock_timer_t);
 
-        gf_uuid_copy (lock_obj->lock_owner, uuid);
+    if (!mgmt_lock_timer) {
+        ret = -1;
+        goto out;
+    }
+
+    mgmt_lock_timer->xl = THIS;
+    /*changing to default timeout value*/
+    priv->mgmt_v3_lock_timeout = GF_LOCK_TIMER;
+
+    ret = -1;
+    mgmt_lock_timer_xl = mgmt_lock_timer->xl;
+    if (!mgmt_lock_timer_xl) {
+        GF_FREE(mgmt_lock_timer);
+        goto out;
+    }
+
+    mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;
+    if (!mgmt_lock_timer_ctx) {
+        GF_FREE(mgmt_lock_timer);
+        goto out;
+    }
+
+    key_dup = gf_strdup(key);
+    delay.tv_sec = priv->mgmt_v3_lock_timeout;
+    delay.tv_nsec = 0;
+
+    mgmt_lock_timer->timer = gf_timer_call_after(
+        mgmt_lock_timer_ctx, delay, gd_mgmt_v3_unlock_timer_cbk, key_dup);
+
+    ret = dict_set_bin(priv->mgmt_v3_lock_timer, key, mgmt_lock_timer,
+                       sizeof(glusterd_mgmt_v3_lock_timer));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set timer in mgmt_v3 lock");
+        GF_FREE(key_dup);
+        GF_FREE(mgmt_lock_timer);
+        goto out;
+    }
+
+    /* Saving the backtrace into the pre-allocated buffer, ctx->btbuf*/
+    if ((bt = gf_backtrace_save(NULL))) {
+        snprintf(key, sizeof(key), "debug.last-success-bt-%s", key_dup);
+        ret = dict_set_dynstr_with_alloc(priv->mgmt_v3_lock, key, bt);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to save "
+                   "the back trace for lock %s granted to %s",
+                   key_dup, uuid_utoa(uuid));
+        ret = 0;
+    }
 
-        ret = dict_set_bin (priv->mgmt_v3_lock, key, lock_obj,
-                            sizeof(glusterd_mgmt_v3_lock_obj));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set lock owner in mgmt_v3 lock");
-                GF_FREE (lock_obj);
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "Lock for %s successfully held by %s", key_dup,
+                 uuid_utoa(uuid));
 
-        /* Saving the backtrace into the pre-allocated buffer, ctx->btbuf*/
-        if ((bt = gf_backtrace_save (NULL))) {
-                snprintf (key, sizeof (key), "debug.last-success-bt-%s-%s",
-                          name, type);
-                ret = dict_set_dynstr_with_alloc (priv->mgmt_v3_lock, key, bt);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to save "
-                                "the back trace for lock %s-%s granted to %s",
-                                name, type, uuid_utoa (uuid));
-                ret = 0;
-        }
+    ret = 0;
+out:
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        gf_msg_debug (this->name, 0,
-                "Lock for %s %s successfully held by %s",
-                type, name, uuid_utoa (uuid));
+/*
+ * This call back will ensure to unlock the lock_obj, in case we hit a situation
+ * where unlocking failed and stale lock exist*/
+void
+gd_mgmt_v3_unlock_timer_cbk(void *data)
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL;
+    char *key = NULL;
+    int keylen;
+    char bt_key[PATH_MAX] = "";
+    int bt_key_len = 0;
+    int32_t ret = -1;
+    glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL;
+    xlator_t *mgmt_lock_timer_xl = NULL;
+    gf_timer_t *timer = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    GF_ASSERT(NULL != data);
+    key = (char *)data;
+
+    keylen = strlen(key);
+    dict_deln(conf->mgmt_v3_lock, key, keylen);
+
+    bt_key_len = snprintf(bt_key, PATH_MAX, "debug.last-success-bt-%s", key);
+    if (bt_key_len != SLEN("debug.last-success-bt-") + keylen) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create backtrace "
+               "key");
+        goto out;
+    }
+
+    dict_deln(conf->mgmt_v3_lock, bt_key, bt_key_len);
+
+    ret = dict_get_bin(conf->mgmt_v3_lock_timer, key,
+                       (void **)&mgmt_lock_timer);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to get lock owner in mgmt_v3 lock");
+    }
 
-        ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (mgmt_lock_timer && mgmt_lock_timer->timer) {
+        mgmt_lock_timer_xl = mgmt_lock_timer->xl;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_xl, ret_function);
+
+        mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_ctx, ret_function);
+
+        timer = mgmt_lock_timer->timer;
+        GF_FREE(timer->data);
+        gf_timer_call_cancel(mgmt_lock_timer_ctx, mgmt_lock_timer->timer);
+        dict_deln(conf->mgmt_v3_lock_timer, bt_key, bt_key_len);
+        mgmt_lock_timer->timer = NULL;
+        gf_log(this->name, GF_LOG_INFO,
+               "unlock timer is cancelled for volume_type"
+               " %s",
+               key);
+    }
+
+ret_function:
+
+    return;
 }
 
 int32_t
-glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)
+glusterd_mgmt_v3_unlock(const char *name, uuid_t uuid, char *type)
 {
-        char                    key[PATH_MAX]   = "";
-        int32_t                 ret             = -1;
-        gf_boolean_t            is_valid        = _gf_true;
-        glusterd_conf_t        *priv            = NULL;
-        uuid_t                  owner           = {0};
-        xlator_t               *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!name || !type) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "name is null.");
-                ret = -1;
-                goto out;
-        }
-
-        is_valid = glusterd_mgmt_v3_is_type_valid (type);
-        if (is_valid != _gf_true) {
-                gf_msg_callingfn (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY,
-                        "Invalid entity. Cannot perform unlocking "
-                        "operation on %s types", type);
-                ret = -1;
-                goto out;
-        }
-
-        ret = snprintf (key, sizeof(key), "%s_%s",
-                        name, type);
-        if (ret != strlen(name) + 1 + strlen(type)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CREATE_KEY_FAIL, "Unable to create key");
-                ret = -1;
-                goto out;
-        }
+    char key[PATH_MAX] = "";
+    char key_dup[PATH_MAX] = "";
+    int keylen;
+    int32_t ret = -1;
+    gf_boolean_t is_valid = _gf_true;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL;
+    uuid_t owner = {0};
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL;
+    xlator_t *mgmt_lock_timer_xl = NULL;
+    gf_timer_t *timer = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!name || !type) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "name is null.");
+        ret = -1;
+        goto out;
+    }
+
+    is_valid = glusterd_mgmt_v3_is_type_valid(type);
+    if (is_valid != _gf_true) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                         "Invalid entity. Cannot perform unlocking "
+                         "operation on %s types",
+                         type);
+        ret = -1;
+        goto out;
+    }
 
-        gf_msg_debug (this->name, 0,
-                "Trying to release lock of %s %s for %s as %s",
-                type, name, uuid_utoa (uuid), key);
+    keylen = snprintf(key, sizeof(key), "%s_%s", name, type);
+    if (keylen != strlen(name) + 1 + strlen(type)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create key");
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "Unable to get mgmt_v3 lock owner");
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "Trying to release lock of %s %s for %s as %s",
+                 type, name, uuid_utoa(uuid), key);
 
-        if (gf_uuid_is_null (owner)) {
-                gf_msg_callingfn (this->name, GF_LOG_WARNING,
-                        0, GD_MSG_LOCK_NOT_HELD,
-                        "Lock for %s %s not held", type, name);
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_get_mgmt_v3_lock_owner(key, &owner);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get mgmt_v3 lock owner");
+        goto out;
+    }
 
-        ret = gf_uuid_compare (uuid, owner);
-        if (ret) {
-                gf_msg_callingfn (this->name, GF_LOG_WARNING,
-                                  0, GD_MSG_LOCK_OWNER_MISMATCH,
-                                  "Lock owner mismatch. "
-                                  "Lock for %s %s held by %s",
-                                  type, name, uuid_utoa (owner));
-                goto out;
-        }
+    if (gf_uuid_is_null(owner)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0, GD_MSG_LOCK_NOT_HELD,
+                         "Lock for %s %s not held", type, name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_uuid_compare(uuid, owner);
+    if (ret) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         GD_MSG_LOCK_OWNER_MISMATCH,
+                         "Lock owner mismatch. "
+                         "Lock for %s %s held by %s",
+                         type, name, uuid_utoa(owner));
+        goto out;
+    }
+
+    /* Removing the mgmt_v3 lock from the global list */
+    dict_deln(priv->mgmt_v3_lock, key, keylen);
+
+    ret = dict_get_bin(priv->mgmt_v3_lock_timer, key,
+                       (void **)&mgmt_lock_timer);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to get mgmt lock key in mgmt_v3 lock");
+        goto out;
+    }
+
+    (void)snprintf(key_dup, sizeof(key_dup), "%s", key);
+
+    /* Remove the backtrace key as well */
+    ret = snprintf(key, sizeof(key), "debug.last-success-bt-%s", key_dup);
+    if (ret != SLEN("debug.last-success-bt-") + keylen) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create backtrace "
+               "key");
+        ret = -1;
+        goto out;
+    }
+    dict_deln(priv->mgmt_v3_lock, key, ret);
 
-        /* Removing the mgmt_v3 lock from the global list */
-        dict_del (priv->mgmt_v3_lock, key);
-
-        /* Remove the backtrace key as well */
-        ret = snprintf (key, sizeof(key), "debug.last-success-bt-%s-%s", name,
-                        type);
-        if (ret != strlen ("debug.last-success-bt-") + strlen (name) +
-                   strlen (type) + 1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CREATE_KEY_FAIL, "Unable to create backtrace "
-                        "key");
-                ret = -1;
-                goto out;
-        }
-        dict_del (priv->mgmt_v3_lock, key);
+    gf_msg_debug(this->name, 0, "Lock for %s %s successfully released", type,
+                 name);
 
-        gf_msg_debug (this->name, 0,
-                "Lock for %s %s successfully released",
-                type, name);
+    /* Release owner reference which was held during lock */
+    if (mgmt_lock_timer && mgmt_lock_timer->timer) {
+        ret = -1;
+        mgmt_lock_timer_xl = mgmt_lock_timer->xl;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_xl, out);
 
+        mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_ctx, out);
         ret = 0;
+
+        timer = mgmt_lock_timer->timer;
+        GF_FREE(timer->data);
+        gf_timer_call_cancel(mgmt_lock_timer_ctx, mgmt_lock_timer->timer);
+        dict_deln(priv->mgmt_v3_lock_timer, key_dup, keylen);
+    }
+    ret = glusterd_volinfo_find(name, &volinfo);
+    if (volinfo && volinfo->stage_deleted) {
+        /* this indicates a volume still exists and the volume delete
+         * operation has failed in some of the phases, need to ensure
+         * stage_deleted flag is set back to false
+         */
+        volinfo->stage_deleted = _gf_false;
+        gf_log(this->name, GF_LOG_INFO,
+               "Volume %s still exist, setting "
+               "stage deleted flag to false for the volume",
+               volinfo->volname);
+    }
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.h b/xlators/mgmt/glusterd/src/glusterd-locks.h
index 437053d9f38..44667cebd3d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-locks.h
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.h
@@ -11,37 +11,47 @@
 #define _GLUSTERD_LOCKS_H_
 
 typedef struct glusterd_mgmt_v3_lock_object_ {
-        uuid_t              lock_owner;
+    uuid_t lock_owner;
 } glusterd_mgmt_v3_lock_obj;
 
+typedef struct glusterd_mgmt_v3_lock_timer_ {
+    gf_timer_t *timer;
+    xlator_t *xl;
+} glusterd_mgmt_v3_lock_timer;
+
 typedef struct glusterd_mgmt_v3_lock_valid_entities {
-        char          *type;          /* Entity type like vol, snap */
-        gf_boolean_t   default_value; /* The default value that  *
-                                       * determines if the locks *
-                                       * should be held for that *
-                                       * entity */
+    char *type;                 /* Entity type like vol, snap */
+    gf_boolean_t default_value; /* The default value that  *
+                                 * determines if the locks *
+                                 * should be held for that *
+                                 * entity */
 } glusterd_valid_entities;
 
 int32_t
-glusterd_mgmt_v3_lock_init ();
+glusterd_mgmt_v3_lock_init();
 
 void
-glusterd_mgmt_v3_lock_fini ();
+glusterd_mgmt_v3_lock_fini();
 
 int32_t
-glusterd_get_mgmt_v3_lock_owner (char *volname, uuid_t *uuid);
+glusterd_mgmt_v3_lock_timer_init();
+
+void
+glusterd_mgmt_v3_lock_timer_fini();
 
 int32_t
-glusterd_mgmt_v3_lock (const char *key, uuid_t uuid, uint32_t *op_errno,
-                       char *type);
+glusterd_mgmt_v3_lock(const char *key, uuid_t uuid, uint32_t *op_errno,
+                      char *type);
 
 int32_t
-glusterd_mgmt_v3_unlock (const char *key, uuid_t uuid, char *type);
+glusterd_mgmt_v3_unlock(const char *key, uuid_t uuid, char *type);
 
 int32_t
-glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid, uint32_t *op_errno);
+glusterd_multiple_mgmt_v3_lock(dict_t *dict, uuid_t uuid, uint32_t *op_errno);
 
 int32_t
-glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid);
+glusterd_multiple_mgmt_v3_unlock(dict_t *dict, uuid_t uuid);
 
+void
+gd_mgmt_v3_unlock_timer_cbk(void *data);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-log-ops.c b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
index d04492af7cc..34abf35cb00 100644
--- a/xlators/mgmt/glusterd/src/glusterd-log-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
@@ -7,7 +7,7 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -16,270 +16,275 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
 #include <signal.h>
 
 int
-__glusterd_handle_log_rotate (rpcsvc_request_t *req)
+__glusterd_handle_log_rotate(rpcsvc_request_t *req)
 {
-        int32_t                 ret     = -1;
-        gf_cli_req              cli_req = {{0,}};
-        dict_t                 *dict    = NULL;
-        glusterd_op_t           cli_op = GD_OP_LOG_ROTATE;
-        char                   *volname = NULL;
-        char                   msg[2048] = {0,};
-        xlator_t               *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_LOG_ROTATE;
+    char *volname = NULL;
+    char msg[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
         }
+    }
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get volume name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", msg);
-                goto out;
-        }
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_LOG_ROTATE_REQ_RECVD,
-                "Received log rotate req "
-                "for volume %s", volname);
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_LOG_ROTATE_REQ_RECVD,
+           "Received log rotate req "
+           "for volume %s",
+           volname);
 
-        ret = dict_set_uint64 (dict, "rotate-key", (uint64_t)time (NULL));
-        if (ret)
-                goto out;
+    ret = dict_set_uint64(dict, "rotate-key", (uint64_t)gf_time());
+    if (ret)
+        goto out;
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_LOG_ROTATE, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_LOG_ROTATE, dict);
 
 out:
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, msg);
-        }
-
-        free (cli_req.dict.dict_val);
-        return ret;
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, msg);
+    }
+
+    free(cli_req.dict.dict_val);
+    return ret;
 }
 
 int
-glusterd_handle_log_rotate (rpcsvc_request_t *req)
+glusterd_handle_log_rotate(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_log_rotate);
+    return glusterd_big_locked_handler(req, __glusterd_handle_log_rotate);
 }
 
 /* op-sm */
 int
-glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
+glusterd_op_stage_log_rotate(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        char                                    msg[2048] = {0};
-        char                                    *brick = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
-        if (_gf_false == glusterd_is_volume_started (volinfo)) {
-                snprintf (msg, sizeof (msg), "Volume %s needs to be started before"
-                          " log rotate.", volname);
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_STARTED, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "brick", &brick);
-        /* If no brick is specified, do log-rotate for
-           all the bricks in the volume */
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo, NULL,
-                                                      _gf_false);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Incorrect brick %s "
-                          "for volume %s", brick, volname);
-                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    char *brick = NULL;
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (_gf_false == glusterd_is_volume_started(volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s needs to be started before"
+                 " log rotate.",
+                 volname);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick", &brick);
+    /* If no brick is specified, do log-rotate for
+       all the bricks in the volume */
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=brick", NULL);
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, NULL,
+                                                 _gf_false);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Incorrect brick %s "
+                 "for volume %s",
+                 brick, volname);
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
-
 int
-glusterd_op_log_rotate (dict_t *dict)
+glusterd_op_log_rotate(dict_t *dict)
 {
-        int                   ret                = -1;
-        glusterd_conf_t      *priv               = NULL;
-        glusterd_volinfo_t   *volinfo            = NULL;
-        glusterd_brickinfo_t *brickinfo          = NULL;
-        xlator_t             *this               = NULL;
-        char                 *volname            = NULL;
-        char                 *brick              = NULL;
-        char                  logfile[PATH_MAX]  = {0,};
-        char                  pidfile[PATH_MAX]  = {0,};
-        FILE                 *file               = NULL;
-        pid_t                 pid                = 0;
-        uint64_t              key                = 0;
-        int                   valid_brick        = 0;
-        glusterd_brickinfo_t *tmpbrkinfo         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "volname not found");
-                goto out;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    char *brick = NULL;
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    FILE *file = NULL;
+    pid_t pid = 0;
+    uint64_t key = 0;
+    int valid_brick = 0;
+    glusterd_brickinfo_t *tmpbrkinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_uint64(dict, "rotate-key", &key);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "rotate key not found");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick", &brick);
+    /* If no brick is specified, do log-rotate for
+       all the bricks in the volume */
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=brick", NULL);
+        goto cont;
+    }
+
+    ret = glusterd_brickinfo_new_from_brick(brick, &tmpbrkinfo, _gf_false,
+                                            NULL);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND,
+               "cannot get brickinfo from brick");
+        goto out;
+    }
+
+cont:
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret)
+        goto out;
+
+    ret = -1;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        if (tmpbrkinfo && brick &&
+            (strcmp(tmpbrkinfo->hostname, brickinfo->hostname) ||
+             strcmp(tmpbrkinfo->path, brickinfo->path)))
+            continue;
+
+        valid_brick = 1;
+
+        GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, priv);
+        file = fopen(pidfile, "r+");
+        if (!file) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to open pidfile: %s", pidfile);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_uint64 (dict, "rotate-key", &key);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "rotate key not found");
-                goto out;
+        ret = fscanf(file, "%d", &pid);
+        if (ret <= 0) {
+            fclose(file);
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to read pidfile: %s", pidfile);
+            ret = -1;
+            goto out;
         }
+        fclose(file);
+        file = NULL;
 
-        ret = dict_get_str (dict, "brick", &brick);
-        /* If no brick is specified, do log-rotate for
-           all the bricks in the volume */
+        snprintf(logfile, PATH_MAX, "%s.%" PRIu64, brickinfo->logfile, key);
+
+        ret = sys_rename(brickinfo->logfile, logfile);
         if (ret)
-                goto cont;
+            gf_msg("glusterd", GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+                   "rename failed");
 
-        ret = glusterd_brickinfo_new_from_brick (brick, &tmpbrkinfo,
-                                                 _gf_false, NULL);
+        ret = kill(pid, SIGHUP);
         if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_NOT_FOUND,
-                        "cannot get brickinfo from brick");
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                   "Unable to SIGHUP to %d", pid);
+            goto out;
         }
+        ret = 0;
 
-cont:
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret)
-                goto out;
-
-        ret = -1;
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
-
-                if (brick &&
-                    (strcmp (tmpbrkinfo->hostname, brickinfo->hostname) ||
-                     strcmp (tmpbrkinfo->path,brickinfo->path)))
-                        continue;
-
-                valid_brick = 1;
-
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
-                file = fopen (pidfile, "r+");
-                if (!file) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED, "Unable to open pidfile: %s",
-                                pidfile);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = fscanf (file, "%d", &pid);
-                if (ret <= 0) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED, "Unable to read pidfile: %s",
-                                pidfile);
-                        ret = -1;
-                        goto out;
-                }
-                fclose (file);
-                file = NULL;
-
-                snprintf (logfile, PATH_MAX, "%s.%"PRIu64,
-                          brickinfo->logfile, key);
-
-                ret = sys_rename (brickinfo->logfile, logfile);
-                if (ret)
-                        gf_msg ("glusterd", GF_LOG_WARNING, errno,
-                                GD_MSG_FILE_OP_FAILED, "rename failed");
-
-                ret = kill (pid, SIGHUP);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                                GD_MSG_PID_KILL_FAIL, "Unable to SIGHUP to %d", pid);
-                        goto out;
-                }
-                ret = 0;
-
-                /* If request was for brick, only one iteration is enough */
-                if (brick)
-                        break;
-        }
+        /* If request was for brick, only one iteration is enough */
+        if (brick)
+            break;
+    }
 
-        if (ret && !valid_brick)
-                ret = 0;
+    if (ret && !valid_brick)
+        ret = 0;
 
 out:
-        if (tmpbrkinfo)
-                glusterd_brickinfo_delete (tmpbrkinfo);
+    if (tmpbrkinfo)
+        glusterd_brickinfo_delete(tmpbrkinfo);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index ed171b69b66..d7257e1a7b5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -11,67 +11,48 @@
 #ifndef __GLUSTERD_MEM_TYPES_H__
 #define __GLUSTERD_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 typedef enum gf_gld_mem_types_ {
-        gf_gld_mt_dir_entry_t                   = gf_common_mt_end + 1,
-        gf_gld_mt_volfile_ctx                   = gf_common_mt_end + 2,
-        gf_gld_mt_glusterd_state_t              = gf_common_mt_end + 3,
-        gf_gld_mt_glusterd_conf_t               = gf_common_mt_end + 4,
-        gf_gld_mt_locker                        = gf_common_mt_end + 5,
-        gf_gld_mt_string                        = gf_common_mt_end + 6,
-        gf_gld_mt_lock_table                    = gf_common_mt_end + 7,
-        gf_gld_mt_char                          = gf_common_mt_end + 8,
-        gf_gld_mt_glusterd_connection_t         = gf_common_mt_end + 9,
-        gf_gld_mt_resolve_comp                  = gf_common_mt_end + 10,
-        gf_gld_mt_peerinfo_t                    = gf_common_mt_end + 11,
-        gf_gld_mt_friend_sm_event_t             = gf_common_mt_end + 12,
-        gf_gld_mt_friend_req_ctx_t              = gf_common_mt_end + 13,
-        gf_gld_mt_friend_update_ctx_t           = gf_common_mt_end + 14,
-        gf_gld_mt_op_sm_event_t                 = gf_common_mt_end + 15,
-        gf_gld_mt_op_lock_ctx_t                 = gf_common_mt_end + 16,
-        gf_gld_mt_op_stage_ctx_t                = gf_common_mt_end + 17,
-        gf_gld_mt_op_commit_ctx_t               = gf_common_mt_end + 18,
-        gf_gld_mt_mop_stage_req_t               = gf_common_mt_end + 19,
-        gf_gld_mt_probe_ctx_t                   = gf_common_mt_end + 20,
-        gf_gld_mt_create_volume_ctx_t           = gf_common_mt_end + 21,
-        gf_gld_mt_start_volume_ctx_t            = gf_common_mt_end + 22,
-        gf_gld_mt_stop_volume_ctx_t             = gf_common_mt_end + 23,
-        gf_gld_mt_delete_volume_ctx_t           = gf_common_mt_end + 24,
-        gf_gld_mt_glusterd_volinfo_t            = gf_common_mt_end + 25,
-        gf_gld_mt_glusterd_brickinfo_t          = gf_common_mt_end + 26,
-        gf_gld_mt_peer_hostname_t               = gf_common_mt_end + 27,
-        gf_gld_mt_ifreq                         = gf_common_mt_end + 28,
-        gf_gld_mt_store_handle_t                = gf_common_mt_end + 29,
-        gf_gld_mt_store_iter_t                  = gf_common_mt_end + 30,
-        gf_gld_mt_defrag_info                   = gf_common_mt_end + 31,
-        gf_gld_mt_log_filename_ctx_t            = gf_common_mt_end + 32,
-        gf_gld_mt_log_locate_ctx_t              = gf_common_mt_end + 33,
-        gf_gld_mt_log_rotate_ctx_t              = gf_common_mt_end + 34,
-        gf_gld_mt_peerctx_t                     = gf_common_mt_end + 35,
-        gf_gld_mt_sm_tr_log_t                   = gf_common_mt_end + 36,
-        gf_gld_mt_pending_node_t                = gf_common_mt_end + 37,
-        gf_gld_mt_brick_rsp_ctx_t               = gf_common_mt_end + 38,
-        gf_gld_mt_mop_brick_req_t               = gf_common_mt_end + 39,
-        gf_gld_mt_op_allack_ctx_t               = gf_common_mt_end + 40,
-        gf_gld_mt_linearr                       = gf_common_mt_end + 41,
-        gf_gld_mt_linebuf                       = gf_common_mt_end + 42,
-        gf_gld_mt_mount_pattern                 = gf_common_mt_end + 43,
-        gf_gld_mt_mount_comp_container          = gf_common_mt_end + 44,
-        gf_gld_mt_mount_component               = gf_common_mt_end + 45,
-        gf_gld_mt_mount_spec                    = gf_common_mt_end + 46,
-        gf_gld_mt_georep_meet_spec              = gf_common_mt_end + 47,
-        gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 48,
-        gf_gld_mt_charptr                       = gf_common_mt_end + 49,
-        gf_gld_mt_hooks_stub_t                  = gf_common_mt_end + 50,
-        gf_gld_mt_hooks_priv_t                  = gf_common_mt_end + 51,
-        gf_gld_mt_mop_commit_req_t              = gf_common_mt_end + 52,
-        gf_gld_mt_int                           = gf_common_mt_end + 53,
-        gf_gld_mt_snap_t                        = gf_common_mt_end + 54,
-        gf_gld_mt_missed_snapinfo_t             = gf_common_mt_end + 55,
-        gf_gld_mt_snap_create_args_t            = gf_common_mt_end + 56,
-        gf_gld_mt_local_peers_t                 = gf_common_mt_end + 57,
-        gf_gld_mt_end                           = gf_common_mt_end + 58,
+    gf_gld_mt_glusterd_conf_t = gf_common_mt_end + 1,
+    gf_gld_mt_char,
+    gf_gld_mt_peerinfo_t,
+    gf_gld_mt_friend_sm_event_t,
+    gf_gld_mt_friend_req_ctx_t,
+    gf_gld_mt_friend_update_ctx_t,
+    gf_gld_mt_op_sm_event_t,
+    gf_gld_mt_op_lock_ctx_t,
+    gf_gld_mt_op_stage_ctx_t,
+    gf_gld_mt_op_commit_ctx_t,
+    gf_gld_mt_mop_stage_req_t,
+    gf_gld_mt_probe_ctx_t,
+    gf_gld_mt_glusterd_volinfo_t,
+    gf_gld_mt_volinfo_dict_data_t,
+    gf_gld_mt_glusterd_brickinfo_t,
+    gf_gld_mt_peer_hostname_t,
+    gf_gld_mt_defrag_info,
+    gf_gld_mt_peerctx_t,
+    gf_gld_mt_sm_tr_log_t,
+    gf_gld_mt_pending_node_t,
+    gf_gld_mt_brick_rsp_ctx_t,
+    gf_gld_mt_mop_brick_req_t,
+    gf_gld_mt_op_allack_ctx_t,
+    gf_gld_mt_linearr,
+    gf_gld_mt_linebuf,
+    gf_gld_mt_mount_pattern,
+    gf_gld_mt_mount_comp_container,
+    gf_gld_mt_mount_spec,
+    gf_gld_mt_georep_meet_spec,
+    gf_gld_mt_charptr,
+    gf_gld_mt_hooks_stub_t,
+    gf_gld_mt_hooks_priv_t,
+    gf_gld_mt_mop_commit_req_t,
+    gf_gld_mt_int,
+    gf_gld_mt_snap_t,
+    gf_gld_mt_missed_snapinfo_t,
+    gf_gld_mt_snap_create_args_t,
+    gf_gld_mt_glusterd_brick_proc_t,
+    gf_gld_mt_glusterd_svc_proc_t,
+    gf_gld_mt_end,
 } gf_gld_mem_types_t;
 #endif
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index ba40b8f7628..3a1e600fb03 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -11,4669 +11,441 @@
 #ifndef _GLUSTERD_MESSAGES_H_
 #define _GLUSTERD_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    GLUSTERD, GD_MSG_SERVER_QUORUM_NOT_MET,
+    GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
+    GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS, GD_MSG_PEER_DISCONNECTED,
+    GD_MSG_BRICK_DISCONNECTED, GD_MSG_NODE_DISCONNECTED,
+    GD_MSG_REBALANCE_DISCONNECTED, GD_MSG_VOL_CLEANUP_FAIL,
+    GD_MSG_VOL_VERS_MISMATCH, GD_MSG_CKSUM_VERS_MISMATCH,
+    GD_MSG_QUOTA_CONFIG_VERS_MISMATCH, GD_MSG_QUOTA_CONFIG_CKSUM_MISMATCH,
+    GD_MSG_BRICK_STOP_FAIL, GD_MSG_SVC_KILL_FAIL, GD_MSG_PID_KILL_FAIL,
+    GD_MSG_REBAL_NO_SOCK_FILE, GD_MSG_UNIX_OP_BUILD_FAIL,
+    GD_MSG_RPC_CREATE_FAIL, GD_MSG_FAIL_DEFAULT_OPT_SET,
+    GD_MSG_CLUSTER_UNLOCK_FAILED, GD_MSG_NO_MEMORY, GD_MSG_UNSUPPORTED_VERSION,
+    GD_MSG_COMMAND_NOT_FOUND, GD_MSG_SNAPSHOT_OP_FAILED, GD_MSG_INVALID_ENTRY,
+    GD_MSG_VOL_NOT_FOUND, GD_MSG_REG_COMPILE_FAILED, GD_MSG_FILE_OP_FAILED,
+    GD_MSG_SNAP_CREATION_FAIL, GD_MSG_VOL_OP_FAILED, GD_MSG_CREATE_DIR_FAILED,
+    GD_MSG_DIR_OP_FAILED, GD_MSG_VOL_STOP_FAILED, GD_MSG_NO_CLI_RESP,
+    GD_MSG_LOCK_INIT_FAILED, GD_MSG_SNAP_LIST_GET_FAIL, GD_MSG_UNOUNT_FAILED,
+    GD_MSG_LOCK_DESTROY_FAILED, GD_MSG_SNAP_CLEANUP_FAIL,
+    GD_MSG_SNAP_ACTIVATE_FAIL, GD_MSG_SNAP_DEACTIVATE_FAIL,
+    GD_MSG_SNAP_RESTORE_FAIL, GD_MSG_SNAP_REMOVE_FAIL, GD_MSG_SNAP_CONFIG_FAIL,
+    GD_MSG_SNAP_STATUS_FAIL, GD_MSG_SNAP_INIT_FAIL, GD_MSG_VOLINFO_SET_FAIL,
+    GD_MSG_VOLINFO_GET_FAIL, GD_MSG_BRICK_CREATION_FAIL,
+    GD_MSG_BRICK_GET_INFO_FAIL, GD_MSG_BRICK_NEW_INFO_FAIL, GD_MSG_LVS_FAIL,
+    GD_MSG_SET_XATTR_FAIL, GD_MSG_UMOUNTING_SNAP_BRICK, GD_MSG_OP_UNSUPPORTED,
+    GD_MSG_SNAP_NOT_FOUND, GD_MSG_FS_LABEL_UPDATE_FAIL, GD_MSG_LVM_MOUNT_FAILED,
+    GD_MSG_DICT_SET_FAILED, GD_MSG_CANONICALIZE_FAIL, GD_MSG_DICT_GET_FAILED,
+    GD_MSG_SNAP_INFO_FAIL, GD_MSG_SNAP_VOL_CONFIG_FAIL,
+    GD_MSG_SNAP_OBJECT_STORE_FAIL, GD_MSG_DICT_UNSERIALIZE_FAIL,
+    GD_MSG_SNAP_RESTORE_REVERT_FAIL, GD_MSG_SNAP_LIST_SET_FAIL,
+    GD_MSG_VOLFILE_CREATE_FAIL, GD_MSG_VOLINFO_REMOVE_FAIL,
+    GD_MSG_VOL_DELETE_FAIL, GD_MSG_SNAPSHOT_PENDING,
+    GD_MSG_BRICK_PATH_UNMOUNTED, GD_MSG_BRICK_ADD_FAIL,
+    GD_MSG_BRICK_SET_INFO_FAIL, GD_MSG_LVCREATE_FAIL, GD_MSG_VG_GET_FAIL,
+    GD_MSG_TPOOL_GET_FAIL, GD_MSG_LVM_REMOVE_FAILED,
+    GD_MSG_MISSEDSNAP_INFO_SET_FAIL, GD_MSG_BRK_MOUNTOPTS_FAIL,
+    GD_MSG_MISSED_SNAP_LIST_STORE_FAIL, GD_MSG_INVALID_MISSED_SNAP_ENTRY,
+    GD_MSG_MISSED_SNAP_GET_FAIL, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+    GD_MSG_DUP_ENTRY, GD_MSG_MISSED_SNAP_STATUS_DONE, GD_MSG_NO_EXEC_PERMS,
+    GD_MSG_GLOBAL_OP_VERSION_SET_FAIL, GD_MSG_HARD_LIMIT_SET_FAIL,
+    GD_MSG_OP_SUCCESS, GD_MSG_STORE_FAIL, GD_MSG_GLOBAL_OP_VERSION_GET_FAIL,
+    GD_MSG_GEOREP_GET_FAILED, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+    GD_MSG_QUORUM_CHECK_FAIL, GD_MSG_QUORUM_COUNT_IGNORED,
+    GD_MSG_SNAP_MOUNT_FAIL, GD_MSG_RSP_DICT_USE_FAIL, GD_MSG_SNAP_IMPORT_FAIL,
+    GD_MSG_SNAP_CONFLICT, GD_MSG_MISSED_SNAP_DELETE,
+    GD_MSG_QUOTA_CONFIG_IMPORT_FAIL, GD_MSG_SNAPDIR_CREATE_FAIL,
+    GD_MSG_MISSED_SNAP_PRESENT, GD_MSG_UUID_NULL, GD_MSG_TSTAMP_SET_FAIL,
+    GD_MSG_RESP_AGGR_FAIL, GD_MSG_DICT_EMPTY, GD_MSG_DICT_CREATE_FAIL,
+    GD_MSG_SNAPD_STOP_FAIL, GD_MSG_SOFT_LIMIT_REACHED, GD_MSG_SNAPD_START_FAIL,
+    GD_MSG_SNAPD_CREATE_FAIL, GD_MSG_SNAPD_INIT_FAIL, GD_MSG_MGMTV3_OP_FAIL,
+    GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, GD_MSG_MGMTV3_UNLOCK_FAIL,
+    GD_MSG_MGMTV3_LOCK_GET_FAIL, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+    GD_MSG_POST_VALIDATION_FAIL, GD_MSG_PRE_VALIDATION_FAIL,
+    GD_MSG_COMMIT_OP_FAIL, GD_MSG_PEER_LIST_CREATE_FAIL, GD_MSG_BRICK_OP_FAIL,
+    GD_MSG_OPINFO_SET_FAIL, GD_MSG_OP_EVENT_UNLOCK_FAIL,
+    GD_MSG_MGMTV3_OP_RESP_FAIL, GD_MSG_PEER_NOT_FOUND, GD_MSG_REQ_DECODE_FAIL,
+    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, GD_MSG_ALREADY_STOPPED,
+    GD_MSG_PRE_VALD_RESP_FAIL, GD_MSG_SVC_GET_FAIL, GD_MSG_VOLFILE_NOT_FOUND,
+    GD_MSG_OP_EVENT_LOCK_FAIL, GD_MSG_NON_STRIPE_VOL, GD_MSG_SNAPD_OBJ_GET_FAIL,
+    GD_MSG_QUOTA_DISABLED, GD_MSG_CACHE_MINMAX_SIZE_INVALID,
+    GD_MSG_QUOTA_GET_STAT_FAIL, GD_MSG_SUBVOLUMES_EXCEED, GD_MSG_BRICK_ADD,
+    GD_MSG_BRICK_REMOVE, GD_MSG_CREATE_KEY_FAIL,
+    GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL, GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
+    GD_MSG_RESP_FROM_UNKNOWN_PEER, GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL,
+    GD_MSG_GFID_VALIDATE_SET_FAIL, GD_MSG_PEER_LOCK_FAIL,
+    GD_MSG_PEER_UNLOCK_FAIL, GD_MSG_MGMT_OP_FAIL,
+    GD_MSG_TRANS_OPINFO_CLEAR_FAIL, GD_MSG_GLUSTERD_LOCK_FAIL,
+    GD_MSG_TRANS_OPINFO_SET_FAIL, GD_MSG_TRANS_IDGEN_FAIL, GD_MSG_RPC_FAILURE,
+    GD_MSG_OP_VERS_ADJUST_FAIL, GD_MSG_SNAP_DEVICE_NAME_GET_FAIL,
+    GD_MSG_SNAP_STATUS_NOT_PENDING, GD_MSG_MGMT_PGM_SET_FAIL,
+    GD_MSG_EVENT_INJECT_FAIL, GD_MSG_VERS_INFO, GD_MSG_VOL_INFO_REQ_RECVD,
+    GD_MSG_VERS_GET_FAIL, GD_MSG_EVENT_NEW_GET_FAIL, GD_MSG_RPC_LAYER_ERROR,
+    GD_MSG_NO_HANDSHAKE_ACK, GD_MSG_OP_VERSION_MISMATCH,
+    GD_MSG_HANDSHAKE_REQ_REJECTED, GD_MSG_UNKNOWN_MODE,
+    GD_MSG_DEFRAG_STATUS_UPDATED, GD_MSG_NO_FLAG_SET,
+    GD_MSG_VERSION_UNSUPPORTED, GD_MSG_UUID_SET_FAIL, GD_MSG_MOUNT_REQ_FAIL,
+    GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL, GD_MSG_OP_VERS_STORE_FAIL,
+    GD_MSG_SNAP_AUTOMIC_UPDATE_FAIL, GD_MSG_SNAPINFO_WRITE_FAIL,
+    GD_MSG_SNAPINFO_CREATE_FAIL, GD_MSG_SNAPD_INFO_STORE_FAIL,
+    GD_MSG_BRK_MNTPATH_MOUNT_FAIL, GD_MSG_BRK_MNTPATH_GET_FAIL,
+    GD_MSG_SNAP_BRK_MNT_RECREATE_FAIL, GD_MSG_SNAP_RESOLVE_BRICK_FAIL,
+    GD_MSG_RESOLVE_BRICK_FAIL, GD_MSG_BRK_MNT_RECREATE_FAIL,
+    GD_MSG_TMP_FILE_UNLINK_FAIL, GD_MSG_VOL_VALS_WRITE_FAIL,
+    GD_MSG_STORE_HANDLE_GET_FAIL, GD_MSG_STORE_HANDLE_WRITE_FAIL,
+    GD_MSG_MISSED_SNAP_LIST_STORE_HANDLE_GET_FAIL,
+    GD_MSG_MISSED_SNAP_LIST_EMPTY, GD_MSG_SNAP_VOL_RETRIEVE_FAIL,
+    GD_MSG_SNAPSHOT_UPDATE_FAIL, GD_MSG_SNAPD_PORT_STORE_FAIL,
+    GD_MSG_CKSUM_STORE_FAIL, GD_MSG_STORE_HANDLE_CREATE_FAIL,
+    GD_MSG_HANDLE_NULL, GD_MSG_VOL_RESTORE_FAIL, GD_MSG_NAME_TOO_LONG,
+    GD_MSG_UUID_PARSE_FAIL, GD_MSG_UNKNOWN_KEY, GD_MSG_STORE_ITER_DESTROY_FAIL,
+    GD_MSG_STORE_ITER_GET_FAIL, GD_MSG_VOLINFO_UPDATE_FAIL,
+    GD_MSG_PARSE_BRICKINFO_FAIL, GD_MSG_VERS_STORE_FAIL, GD_MSG_HEADER_ADD_FAIL,
+    GD_MSG_QUOTA_CONF_WRITE_FAIL, GD_MSG_QUOTA_CONF_CORRUPT, GD_MSG_FORK_FAIL,
+    GD_MSG_CKSUM_COMPUTE_FAIL, GD_MSG_VERS_CKSUM_STORE_FAIL,
+    GD_MSG_GET_XATTR_FAIL, GD_MSG_CONVERSION_FAILED, GD_MSG_VOL_NOT_DISTRIBUTE,
+    GD_MSG_VOL_STOPPED, GD_MSG_OPCTX_GET_FAIL, GD_MSG_TASKID_GEN_FAIL,
+    GD_MSG_REBALANCE_ID_MISSING, GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME,
+    GD_MSG_DEFRAG_STATUS_UPDATE_FAIL, GD_MSG_UUID_GEN_STORE_FAIL,
+    GD_MSG_UUID_STORE_FAIL, GD_MSG_NO_INIT, GD_MSG_MODULE_NOT_INSTALLED,
+    GD_MSG_MODULE_NOT_WORKING, GD_MSG_WRITE_ACCESS_GRANT_FAIL,
+    GD_MSG_DIRPATH_TOO_LONG, GD_MSG_LOGGROUP_INVALID, GD_MSG_DIR_PERM_LIBERAL,
+    GD_MSG_DIR_PERM_STRICT, GD_MSG_MOUNT_SPEC_INSTALL_FAIL,
+    GD_MSG_GLUSTERD_SOCK_LISTENER_START_FAIL, GD_MSG_DIR_NOT_FOUND,
+    GD_MSG_FAILED_INIT_SHDSVC, GD_MSG_FAILED_INIT_NFSSVC,
+    GD_MSG_FAILED_INIT_QUOTASVC, GD_MSG_RPC_INIT_FAIL,
+    GD_MSG_RPCSVC_REG_NOTIFY_RETURNED, GD_MSG_RPC_TRANSPORT_COUNT_GET_FAIL,
+    GD_MSG_RPC_LISTENER_CREATE_FAIL, GD_MSG_OP_VERS_RESTORE_FAIL,
+    GD_MSG_SELF_HEALD_DISABLED, GD_MSG_PRIV_NULL, GD_MSG_GSYNC_VALIDATION_FAIL,
+    GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL, GD_MSG_OP_NOT_PERMITTED_AC_REQD,
+    GD_MSG_OP_NOT_PERMITTED, GD_MSG_REBALANCE_START_FAIL,
+    GD_MSG_NFS_RECONF_FAIL, GD_MSG_REMOVE_BRICK_ID_SET_FAIL,
+    GD_MSG_BRICK_MOUNTDIR_GET_FAIL, GD_MSG_BRICK_NOT_FOUND,
+    GD_MSG_BRKPATH_TOO_LONG, GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL,
+    GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL, GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL,
+    GD_MSG_BRK_PORT_NUM_GET_FAIL, GD_MSG_BRK_STATEDUMP_FAIL,
+    GD_MSG_VOL_GRAPH_CHANGE_NOTIFY_FAIL, GD_MSG_INVALID_VG,
+    GD_MSG_GLUSTERD_OP_FAILED, GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
+    GD_MSG_STALE_PEERINFO_REMOVE_FAIL, GD_MSG_TRANS_ID_GET_FAIL,
+    GD_MSG_RES_DECODE_FAIL, GD_MSG_VOL_ALREADY_EXIST, GD_MSG_BAD_BRKORDER,
+    GD_MSG_BAD_BRKORDER_CHECK_FAIL, GD_MSG_BRICK_SELECT_FAIL,
+    GD_MSG_NO_LOCK_RESP_FROM_PEER, GD_MSG_MGMTV3_LOCK_FROM_UUID_REJCT,
+    GD_MSG_STAGE_FROM_UUID_REJCT, GD_MSG_UNLOCK_FROM_UUID_REJCT,
+    GD_MSG_MGMTV3_UNLOCK_FROM_UUID_REJCT, GD_MSG_COMMIT_FROM_UUID_REJCT,
+    GD_MSG_VOL_NOT_STARTED, GD_MSG_VOL_NOT_REPLICA, GD_MSG_VOL_NOT_DISPERSE,
+    GD_MSG_OLD_REMOVE_BRICK_EXISTS, GD_MSG_USE_THE_FORCE, GD_MSG_OIP,
+    GD_MSG_OIP_RETRY_LATER, GD_MSG_GSYNC_RESTART_FAIL,
+    GD_MSG_LOCK_FROM_UUID_REJCT, GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+    GD_MSG_HOSTNAME_RESOLVE_FAIL, GD_MSG_COUNT_VALIDATE_FAILED,
+    GD_MSG_SPAWNING_CHILD_FAILED, GD_MSG_READ_CHILD_DATA_FAILED,
+    GD_MSG_DEFAULT_TEMP_CONFIG, GD_MSG_PIDFILE_CREATE_FAILED,
+    GD_MSG_GSYNCD_SPAWN_FAILED, GD_MSG_SUBOP_NOT_FOUND, GD_MSG_RESERVED_OPTION,
+    GD_MSG_GLUSTERD_PRIV_NOT_FOUND, GD_MSG_SLAVEINFO_FETCH_ERROR,
+    GD_MSG_VALIDATE_FAILED, GD_MSG_INVOKE_ERROR, GD_MSG_SESSION_CREATE_ERROR,
+    GD_MSG_STOP_FORCE, GD_MSG_GET_CONFIG_INFO_FAILED,
+    GD_MSG_STAT_FILE_READ_FAILED, GD_MSG_CONF_PATH_ASSIGN_FAILED,
+    GD_MSG_SESSION_INACTIVE, GD_MSG_PIDFILE_NOT_FOUND, GD_MSG_PEER_CMD_ERROR,
+    GD_MSG_SRC_FILE_ERROR, GD_MSG_GET_STATEFILE_NAME_FAILED, GD_MSG_STATUS_NULL,
+    GD_MSG_STATUSFILE_CREATE_FAILED, GD_MSG_SLAVE_URL_INVALID,
+    GD_MSG_INVALID_SLAVE, GD_MSG_READ_ERROR, GD_MSG_ARG_FETCH_ERROR,
+    GD_MSG_REG_FILE_MISSING, GD_MSG_STATEFILE_NAME_NOT_FOUND,
+    GD_MSG_GEO_REP_START_FAILED, GD_MSG_GSYNCD_ERROR,
+    GD_MSG_UPDATE_STATEFILE_FAILED, GD_MSG_STATUS_UPDATE_FAILED,
+    GD_MSG_GSYNCD_OP_SET_FAILED, GD_MSG_BUFFER_EMPTY, GD_MSG_CONFIG_INFO,
+    GD_MSG_FETCH_CONFIG_VAL_FAILED, GD_MSG_GSYNCD_PARSE_ERROR,
+    GD_MSG_SESSION_ALREADY_EXIST, GD_MSG_FORCE_CREATE_SESSION,
+    GD_MSG_GET_KEY_FAILED, GD_MSG_SESSION_DEL_FAILED, GD_MSG_CMD_EXEC_FAIL,
+    GD_MSG_STRDUP_FAILED, GD_MSG_UNABLE_TO_END, GD_MSG_PAUSE_FAILED,
+    GD_MSG_NORMALIZE_URL_FAIL, GD_MSG_MODULE_ERROR,
+    GD_MSG_SLAVEINFO_STORE_ERROR, GD_MSG_MARKER_START_FAIL,
+    GD_MSG_RESUME_FAILED, GD_MSG_GLUSTERFS_START_FAIL,
+    GD_MSG_GLUSTERFS_STOP_FAIL, GD_MSG_RBOP_STATE_STORE_FAIL,
+    GD_MSG_PUMP_XLATOR_DISABLED, GD_MSG_ABORT_OP_FAIL, GD_MSG_PAUSE_OP_FAIL,
+    GD_MSG_GLUSTER_SERVICE_START_FAIL, GD_MSG_HANDSHAKE_FAILED,
+    GD_MSG_CLI_REQ_EMPTY, GD_MSG_PEER_ADD_FAIL,
+    GD_MSG_SYNC_FROM_LOCALHOST_UNALLOWED, GD_MSG_UUIDS_SAME_RETRY,
+    GD_MSG_TSP_ALREADY_FORMED, GD_MSG_VOLS_ALREADY_PRESENT,
+    GD_MSG_REQ_CTX_CREATE_FAIL, GD_MSG_PEER_INFO_UPDATE_FAIL,
+    GD_MSG_PEERINFO_CREATE_FAIL, GD_MSG_REQ_FROM_UNKNOWN_PEER,
+    GD_MSG_STATUS_REPLY_STRING_CREATE_FAIL, GD_MSG_TOKENIZE_FAIL,
+    GD_MSG_LAZY_UMOUNT_FAIL, GD_MSG_NFS_SERVER_START_FAIL,
+    GD_MSG_GLUSTER_SERVICES_STOP_FAIL, GD_MSG_BRK_CLEANUP_FAIL,
+    GD_MSG_RB_ALREADY_STARTED, GD_MSG_RB_BRICKINFO_GET_FAIL, GD_MSG_BAD_FORMAT,
+    GD_MSG_RB_CMD_FAIL, GD_MSG_RB_NOT_STARTED_OR_PAUSED, GD_MSG_RB_NOT_STARTED,
+    GD_MSG_RB_PAUSED_ALREADY, GD_MSG_NO_FREE_PORTS,
+    GD_MSG_EVENT_STATE_TRANSITION_FAIL, GD_MSG_HANDLER_RETURNED,
+    GD_MSG_SNAP_COMPARE_CONFLICT, GD_MSG_PEER_DETACH_CLEANUP_FAIL,
+    GD_MSG_STALE_VOL_REMOVE_FAIL, GD_MSG_AC_ERROR, GD_MSG_LOCK_FAIL,
+    GD_MSG_MGMTV3_LOCK_REQ_SEND_FAIL, GD_MSG_GLUSTERD_UNLOCK_FAIL,
+    GD_MSG_RBOP_START_FAIL, GD_MSG_UNKNOWN_RESPONSE,
+    GD_MSG_COMMIT_REQ_SEND_FAIL, GD_MSG_OPCTX_UPDATE_FAIL, GD_MSG_OPCTX_NULL,
+    GD_MSG_DICT_COPY_FAIL, GD_MSG_SHD_STATUS_SET_FAIL,
+    GD_MSG_REPLICA_INDEX_GET_FAIL, GD_MSG_NFS_SERVER_NOT_RUNNING,
+    GD_MSG_STAGE_REQ_SEND_FAIL, GD_MSG_LOCK_REQ_SEND_FAIL,
+    GD_MSG_VOLNAMES_GET_FAIL, GD_MSG_NO_TASK_ID, GD_MSG_ADD_REMOVE_BRICK_FAIL,
+    GD_MSG_SVC_RESTART_FAIL, GD_MSG_VOL_SET_FAIL, GD_MSG_QUOTAD_NOT_RUNNING,
+    GD_MSG_XLATOR_COUNT_GET_FAIL, GD_MSG_TRANS_OPINFO_GET_FAIL,
+    GD_MSG_TRANS_ID_INVALID, GD_MSG_NO_OPTIONS_GIVEN, GD_MSG_SNAPD_NOT_RUNNING,
+    GD_MSG_ADD_ADDRESS_TO_PEER_FAIL, GD_MSG_PEER_ADDRESS_GET_FAIL,
+    GD_MSG_GETADDRINFO_FAIL, GD_MSG_PEERINFO_DELETE_FAIL, GD_MSG_KEY_NULL,
+    GD_MSG_SPAWN_SVCS_FAIL, GD_MSG_DICT_ITER_FAIL,
+    GD_MSG_TASK_STATUS_UPDATE_FAIL, GD_MSG_VOL_ID_MISMATCH,
+    GD_MSG_STR_TO_BOOL_FAIL, GD_MSG_RB_MNT_BRICKS_MISMATCH,
+    GD_MSG_RB_SRC_BRICKS_MISMATCH, GD_MSG_MNTENTRY_GET_FAIL,
+    GD_MSG_INODE_SIZE_GET_FAIL, GD_MSG_NO_STATEFILE_ENTRY,
+    GD_MSG_PMAP_UNSET_FAIL, GD_MSG_GLOBAL_OPT_IMPORT_FAIL,
+    GD_MSD_BRICK_DISCONNECT_FAIL, GD_MSG_SNAP_DETAILS_IMPORT_FAIL,
+    GD_MSG_BRICKINFO_CREATE_FAIL, GD_MSG_QUOTA_CKSUM_VER_STORE_FAIL,
+    GD_MSG_CKSUM_GET_FAIL, GD_MSG_BRICKPATH_ROOT_GET_FAIL,
+    GD_MSG_HOSTNAME_TO_UUID_FAIL, GD_MSG_REPLY_SUBMIT_FAIL,
+    GD_MSG_SERIALIZE_MSG_FAIL, GD_MSG_ENCODE_FAIL,
+    GD_MSG_RB_DST_BRICKS_MISMATCH, GD_MSG_XLATOR_VOLOPT_DYNLOAD_ERROR,
+    GD_MSG_VOLNAME_NOTFOUND_IN_DICT, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+    GD_MSG_HOSTNAME_NOTFOUND_IN_DICT, GD_MSG_PORT_NOTFOUND_IN_DICT,
+    GD_MSG_CMDSTR_NOTFOUND_IN_DICT, GD_MSG_SNAP_OBJ_NEW_FAIL,
+    GD_MSG_SNAP_BACKEND_MAKE_FAIL, GD_MSG_SNAP_CLONE_FAILED,
+    GD_MSG_SNAP_CLONE_PREVAL_FAILED, GD_MSG_SNAP_CLONE_POSTVAL_FAILED,
+    GD_MSG_VOLINFO_STORE_FAIL, GD_MSG_NEW_FRIEND_SM_EVENT_GET_FAIL,
+    GD_MSG_VOL_TYPE_CHANGING_INFO, GD_MSG_BRKPATH_MNTPNT_MISMATCH,
+    GD_MSG_TASKS_COUNT_MISMATCH, GD_MSG_WRONG_OPTS_SETTING,
+    GD_MSG_PATH_ALREADY_PART_OF_VOL, GD_MSG_BRICK_VALIDATE_FAIL,
+    GD_MSG_READIN_FILE_FAILED, GD_MSG_IMPORT_PRDICT_DICT,
+    GD_MSG_VOL_OPTS_IMPORT_FAIL, GD_MSG_BRICK_IMPORT_FAIL,
+    GD_MSG_VOLINFO_IMPORT_FAIL, GD_MSG_BRICK_ID_GEN_FAILED,
+    GD_MSG_GET_STATUS_DATA_FAIL, GD_MSG_BITROT_NOT_RUNNING,
+    GD_MSG_SCRUBBER_NOT_RUNNING, GD_MSG_SRC_BRICK_PORT_UNAVAIL,
+    GD_MSG_BITD_INIT_FAIL, GD_MSG_SCRUB_INIT_FAIL, GD_MSG_VAR_RUN_DIR_INIT_FAIL,
+    GD_MSG_VAR_RUN_DIR_FIND_FAIL, GD_MSG_SCRUBSVC_RECONF_FAIL,
+    GD_MSG_BITDSVC_RECONF_FAIL, GD_MSG_NFS_GNS_START_FAIL,
+    GD_MSG_NFS_GNS_SETUP_FAIL, GD_MSG_UNRECOGNIZED_SVC_MNGR,
+    GD_MSG_NFS_GNS_OP_HANDLE_FAIL, GD_MSG_EXPORT_FILE_CREATE_FAIL,
+    GD_MSG_NFS_GNS_HOST_FOUND, GD_MSG_REBALANCE_CMD_IN_TIER_VOL,
+    GD_MSG_INCOMPATIBLE_VALUE, GD_MSG_GENERATED_UUID,
+    GD_MSG_FILE_DESC_LIMIT_SET, GD_MSG_CURR_WORK_DIR_INFO,
+    GD_MSG_STRIPE_COUNT_CHANGE_INFO, GD_MSG_REPLICA_COUNT_CHANGE_INFO,
+    GD_MSG_ADD_BRICK_REQ_RECVD, GD_MSG_VOL_ALREADY_TIER,
+    GD_MSG_REM_BRICK_REQ_RECVD, GD_MSG_VOL_NOT_TIER,
+    GD_MSG_LOG_ROTATE_REQ_RECVD, GD_MSG_CLI_REQ_RECVD, GD_MSG_GET_VOL_REQ_RCVD,
+    GD_MSG_VOL_SYNC_REQ_RCVD, GD_MSG_PROBE_RCVD, GD_MSG_UNFRIEND_REQ_RCVD,
+    GD_MSG_FRIEND_UPDATE_RCVD, GD_MSG_RESPONSE_INFO,
+    GD_MSG_VOL_PROFILE_REQ_RCVD, GD_MSG_GETWD_REQ_RCVD, GD_MSG_MOUNT_REQ_RCVD,
+    GD_MSG_UMOUNT_REQ_RCVD, GD_MSG_CONNECT_RETURNED, GD_MSG_STATUS_VOL_REQ_RCVD,
+    GD_MSG_CLRCLK_VOL_REQ_RCVD, GD_MSG_BARRIER_VOL_REQ_RCVD,
+    GD_MSG_UUID_RECEIVED, GD_MSG_REPLACE_BRK_COMMIT_FORCE_REQ_RCVD,
+    GD_MSG_BRK_PORT_NO_ADD_INDO, GD_MSG_REPLACE_BRK_REQ_RCVD,
+    GD_MSG_ADD_OP_ARGS_FAIL, GD_MSG_POST_HOOK_STUB_INIT_FAIL,
+    GD_MSG_HOOK_STUB_NULL, GD_MSG_SPAWN_THREADS_FAIL,
+    GD_MSG_STALE_VOL_DELETE_INFO, GD_MSG_PROBE_REQ_RESP_RCVD,
+    GD_MSG_HOST_PRESENT_ALREADY, GD_MSG_OP_VERS_INFO, GD_MSG_OP_VERS_SET_INFO,
+    GD_MSG_NEW_NODE_STATE_CREATION, GD_MSG_ALREADY_MOUNTED,
+    GD_MSG_SHARED_STRG_VOL_OPT_VALIDATE_FAIL, GD_MSG_NFS_GNS_STOP_FAIL,
+    GD_MSG_NFS_GNS_RESET_FAIL, GD_MSG_SHARED_STRG_SET_FAIL,
+    GD_MSG_VOL_TRANSPORT_TYPE_CHANGE, GD_MSG_PEER_COUNT_GET_FAIL,
+    GD_MSG_INSUFFICIENT_UP_NODES, GD_MSG_OP_STAGE_STATS_VOL_FAIL,
+    GD_MSG_VOL_ID_SET_FAIL, GD_MSG_OP_STAGE_RESET_VOL_FAIL,
+    GD_MSG_OP_STAGE_BITROT_FAIL, GD_MSG_OP_STAGE_QUOTA_FAIL,
+    GD_MSG_OP_STAGE_DELETE_VOL_FAIL, GD_MSG_HANDLE_HEAL_CMD_FAIL,
+    GD_MSG_CLRCLK_SND_CMD_FAIL, GD_MSG_DISPERSE_CLUSTER_FOUND,
+    GD_MSG_HEAL_VOL_REQ_RCVD, GD_MSG_STATEDUMP_VOL_REQ_RCVD,
+    GD_MSG_THINPOOLS_FOR_THINLVS, GD_MSG_OP_STAGE_CREATE_VOL_FAIL,
+    GD_MSG_OP_STAGE_START_VOL_FAIL, GD_MSG_NFS_GNS_UNEXPRT_VOL_FAIL,
+    GD_MSG_TASK_ID_INFO, GD_MSG_DEREGISTER_SUCCESS, GD_MSG_STATEDUMP_OPTS_RCVD,
+    GD_MSG_STATEDUMP_INFO, GD_MSG_RECOVERING_CORRUPT_CONF,
+    GD_MSG_RETRIEVED_UUID, GD_MSG_XLATOR_CREATE_FAIL,
+    GD_MSG_GRAPH_ENTRY_ADD_FAIL, GD_MSG_ERROR_ENCOUNTERED,
+    GD_MSG_FILTER_RUN_FAILED, GD_MSG_DEFAULT_OPT_INFO,
+    GD_MSG_MARKER_STATUS_GET_FAIL, GD_MSG_MARKER_DISABLE_FAIL,
+    GD_MSG_GRAPH_FEATURE_ADD_FAIL, GD_MSG_XLATOR_SET_OPT_FAIL,
+    GD_MSG_BUILD_GRAPH_FAILED, GD_MSG_XML_TEXT_WRITE_FAIL,
+    GD_MSG_XML_DOC_START_FAIL, GD_MSG_XML_ELE_CREATE_FAIL,
+    GD_MSG_VOLUME_INCONSISTENCY, GD_MSG_XLATOR_LINK_FAIL,
+    GD_MSG_REMOTE_HOST_GET_FAIL, GD_MSG_GRAPH_SET_OPT_FAIL,
+    GD_MSG_ROOT_SQUASH_ENABLED, GD_MSG_ROOT_SQUASH_FAILED,
+    GD_MSG_LOCK_OWNER_MISMATCH, GD_MSG_LOCK_NOT_HELD, GD_MSG_LOCK_ALREADY_HELD,
+    GD_MSG_SVC_START_SUCCESS, GD_MSG_SVC_STOP_SUCCESS, GD_MSG_PARAM_NULL,
+    GD_MSG_SVC_STOP_FAIL, GD_MSG_SHARED_STORAGE_DOES_NOT_EXIST,
+    GD_MSG_SNAP_PAUSE_TIER_FAIL, GD_MSG_SNAP_RESUME_TIER_FAIL,
+    GD_MSG_FILE_NOT_FOUND, GD_MSG_RETRY_WITH_NEW_PORT,
+    GD_MSG_REMOTE_VOL_UUID_FAIL, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+    GD_MSG_DICT_GET_SUCCESS, GD_MSG_PMAP_REGISTRY_REMOVE_FAIL,
+    GD_MSG_MNTBROKER_LABEL_NULL, GD_MSG_MNTBROKER_LABEL_MISS,
+    GD_MSG_MNTBROKER_SPEC_MISMATCH, GD_MSG_SYSCALL_FAIL,
+    GD_MSG_DAEMON_STATE_REQ_RCVD, GD_MSG_BRICK_CLEANUP_SUCCESS,
+    GD_MSG_STATE_STR_GET_FAILED, GD_MSG_RESET_BRICK_COMMIT_FORCE_REQ_RCVD,
+    GD_MSG_RESET_BRICK_CMD_FAIL, GD_MSG_TIERD_STOP_FAIL,
+    GD_MSG_TIERD_CREATE_FAIL, GD_MSG_TIERD_START_FAIL,
+    GD_MSG_TIERD_OBJ_GET_FAIL, GD_MSG_TIERD_NOT_RUNNING, GD_MSG_TIERD_INIT_FAIL,
+    GD_MSG_BRICK_MX_SET_FAIL, GD_MSG_NO_SIG_TO_PID_ZERO,
+    GD_MSG_TIER_WATERMARK_RESET_FAIL, GD_MSG_CLIENTS_GET_STATE_FAILED,
+    GD_MSG_GNFS_XLATOR_NOT_INSTALLED, GD_MSG_PIDFILE_UNLINKING,
+    GD_MSG_VOL_SET_VALIDATION_INFO, GD_MSG_NO_MUX_LIMIT,
+    GD_MSG_BRICKPROC_REM_BRICK_FAILED, GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
+    GD_MSG_BRICKPROC_NEW_FAILED, GD_MSG_STATVFS_FAILED, GD_MSG_GARBAGE_ARGS,
+    GD_MSG_LOCALTIME_LOGGING_VOL_OPT_VALIDATE_FAIL,
+    GD_MSG_LOCALTIME_LOGGING_ENABLE, GD_MSG_LOCALTIME_LOGGING_DISABLE,
+    GD_MSG_PORTS_EXHAUSTED, GD_MSG_CHANGELOG_GET_FAIL,
+    GD_MSG_MANAGER_FUNCTION_FAILED,
+    GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL, GD_MSG_SHD_START_FAIL,
+    GD_MSG_SHD_OBJ_GET_FAIL, GD_MSG_SVC_ATTACH_FAIL, GD_MSG_ATTACH_INFO,
+    GD_MSG_DETACH_INFO, GD_MSG_SVC_DETACH_FAIL,
+    GD_MSG_RPC_TRANSPORT_GET_PEERNAME_FAIL, GD_MSG_CLUSTER_RC_ENABLE,
+    GD_MSG_NFS_GANESHA_DISABLED, GD_MSG_GANESHA_NOT_RUNNING, GD_MSG_SNAP_WARN,
+    GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, GD_MSG_REMOVE_ARBITER_BRICK,
+    GD_MSG_BRICK_NOT_DECOM, GD_MSG_BRICK_STOPPED, GD_MSG_BRICK_DEAD,
+    GD_MSG_BRICK_HOST_NOT_FOUND, GD_MSG_BRICK_HOST_DOWN, GD_MSG_BRICK_DELETE,
+    GD_MSG_BRICK_NO_REMOVE_CMD, GD_MSG_MIGRATION_PROG, GD_MSG_MIGRATION_FAIL,
+    GD_MSG_COPY_FAIL, GD_MSG_REALPATH_GET_FAIL,
+    GD_MSG_ARBITER_BRICK_SET_INFO_FAIL, GD_MSG_STRCHR_FAIL, GD_MSG_SPLIT_FAIL,
+    GD_MSG_ALLOC_AND_COPY_UUID_FAIL, GD_MSG_VOL_SHD_NOT_COMP,
+    GD_MSG_BITROT_NOT_ENABLED, GD_MSG_CREATE_BRICK_DIR_FAILED,
+    GD_MSG_CREATE_GLUSTER_DIR_FAILED, GD_MSG_BRICK_CREATE_MNTPNT,
+    GD_MSG_BRICK_CREATE_ROOT, GD_MSG_SET_XATTR_BRICK_FAIL,
+    GD_MSG_REMOVE_XATTR_FAIL, GD_MSG_XLATOR_NOT_DEFINED,
+    GD_MSG_BRICK_NOT_RUNNING, GD_MSG_INCORRECT_BRICK, GD_MSG_UUID_GET_FAIL,
+    GD_MSG_INVALID_ARGUMENT, GD_MSG_FRAME_CREATE_FAIL,
+    GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED, GD_MSG_VOL_STOP_ARGS_GET_FAILED,
+    GD_MSG_LSTAT_FAIL, GD_MSG_VOLUME_NOT_IMPORTED,
+    GD_MSG_ADD_BRICK_MNT_INFO_FAIL, GD_MSG_GET_MNT_ENTRY_INFO_FAIL,
+    GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL, GD_MSG_POST_COMMIT_OP_FAIL,
+    GD_MSG_POST_COMMIT_FROM_UUID_REJCT, GD_MSG_POST_COMMIT_REQ_SEND_FAIL);
+
+#define GD_MSG_INVALID_ENTRY_STR "Invalid data entry"
+#define GD_MSG_INVALID_ARGUMENT_STR                                            \
+    "Invalid arguments have been given to function"
+#define GD_MSG_GARBAGE_ARGS_STR "Garbage args received"
+#define GD_MSG_BRICK_SUBVOL_VERIFY_FAIL_STR "Brick's subvol verification fail"
+#define GD_MSG_REMOVE_ARBITER_BRICK_STR "Failed to remove arbiter bricks"
+#define GD_MSG_DICT_GET_FAILED_STR "Dict get failed"
+#define GD_MSG_DICT_SET_FAILED_STR "Dict set failed"
+#define GD_MSG_BRICK_NOT_FOUND_STR "Brick not found in volume"
+#define GD_MSG_BRICK_NOT_DECOM_STR "Brick is not decommissoned"
+#define GD_MSG_BRICK_STOPPED_STR "Found stopped brick"
+#define GD_MSG_BRICK_DEAD_STR "Found dead brick"
+#define GD_MSG_BRICK_HOST_NOT_FOUND_STR                                        \
+    "Host node of the brick is not a part of cluster"
+#define GD_MSG_BRICK_HOST_DOWN_STR "Host node of the brick is down"
+#define GD_MSG_BRICK_DELETE_STR                                                \
+    "Deleting all the bricks of the volume is not allowed"
+#define GD_MSG_BRICK_NO_REMOVE_CMD_STR "No remove-brick command issued"
+#define GD_MSG_INCORRECT_BRICK_STR "Incorrect brick for volume"
+#define GD_MSG_MIGRATION_PROG_STR "Migration is in progress"
+#define GD_MSG_MIGRATION_FAIL_STR "Migration has failed"
+#define GD_MSG_XLATOR_NOT_DEFINED_STR "Xlator not defined"
+#define GD_MSG_DICT_CREATE_FAIL_STR "Failed to create dictionary"
+#define GD_MSG_COPY_FAIL_STR "Failed to copy"
+#define GD_MSG_UUID_GET_FAIL_STR "Failed to get the uuid of local glusterd"
+#define GD_MSG_GEO_REP_START_FAILED_STR "Georep start failed for volume"
+#define GD_MSG_REALPATH_GET_FAIL_STR "Failed to get realpath"
+#define GD_MSG_FILE_NOT_FOUND_STR "File not found in directory"
+#define GD_MSG_SRC_FILE_ERROR_STR "Error in source file"
+#define GD_MSG_DICT_UNSERIALIZE_FAIL_STR "Failed to unserialize dict"
+#define GD_MSG_VOL_ID_SET_FAIL_STR "Failed to set volume id"
+#define GD_MSG_ARBITER_BRICK_SET_INFO_FAIL_STR                                 \
+    "Failed to add arbiter info to brick"
+#define GD_MSG_NO_MEMORY_STR "Out of memory"
+#define GD_MSG_GLUSTERD_UMOUNT_FAIL_STR "Failed to unmount path"
+#define GD_MSG_PEER_ADD_FAIL_STR "Failed to add new peer"
+#define GD_MSG_BRICK_GET_INFO_FAIL_STR "Failed to get brick info"
+#define GD_MSG_STRCHR_FAIL_STR "Failed to get the character"
+#define GD_MSG_SPLIT_FAIL_STR "Failed to split"
+#define GD_MSG_VOLINFO_GET_FAIL_STR "Failed to get volinfo"
+#define GD_MSG_PEER_NOT_FOUND_STR "Failed to find peer info"
+#define GD_MSG_DICT_COPY_FAIL_STR "Failed to copy values from dictionary"
+#define GD_MSG_ALLOC_AND_COPY_UUID_FAIL_STR                                    \
+    "Failed to allocate memory or copy uuid"
+#define GD_MSG_VOL_NOT_FOUND_STR "Volume not found"
+#define GD_MSG_PEER_DISCONNECTED_STR "Peer is disconnected"
+#define GD_MSG_QUOTA_GET_STAT_FAIL_STR "Failed to get quota status"
+#define GD_MSG_SNAP_STATUS_FAIL_STR "Failed to get status of snapd"
+#define GD_MSG_VALIDATE_FAILED_STR "Failed to validate volume"
+#define GD_MSG_VOL_NOT_STARTED_STR "Volume is not started"
+#define GD_MSG_VOL_SHD_NOT_COMP_STR "Volume is not Self-heal compatible"
+#define GD_MSG_SELF_HEALD_DISABLED_STR "Self-heal daemon is disabled"
+#define GD_MSG_NFS_GANESHA_DISABLED_STR "NFS server is disabled"
+#define GD_MSG_QUOTA_DISABLED_STR "Quota is disabled"
+#define GD_MSG_BITROT_NOT_RUNNING_STR "Bitrot is not enabled"
+#define GD_MSG_BITROT_NOT_ENABLED_STR "Volume does not have bitrot enabled"
+#define GD_MSG_SNAPD_NOT_RUNNING_STR "Snapd is not enabled"
+#define GD_MSG_STRDUP_FAILED_STR "Strdup operation failed"
+#define GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL_STR                               \
+    "Failed to get quorum cluster counts"
+#define GD_MSG_GLUSTER_SERVICE_START_FAIL_STR "Failed to start glusterd service"
+#define GD_MSG_PEER_ADDRESS_GET_FAIL_STR "Failed to get the address of peer"
+#define GD_MSG_INVALID_SLAVE_STR "Volume is not a slave volume"
+#define GD_MSG_BRICK_NOT_RUNNING_STR "One or more bricks are not running"
+#define GD_MSG_BRK_MNTPATH_GET_FAIL_STR "Failed to get brick mount device"
+#define GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED_STR                               \
+    "Snapshot is supported only for thin provisioned LV."
+#define GD_MSG_SNAP_DEVICE_NAME_GET_FAIL_STR                                   \
+    "Failed to copy snapshot device name"
+#define GD_MSG_SNAP_NOT_FOUND_STR "Snapshot does not exist"
+#define GD_MSG_CREATE_BRICK_DIR_FAILED_STR "Failed to create brick directory"
+#define GD_MSG_LSTAT_FAIL_STR "Lstat operation failed"
+#define GD_MSG_DIR_OP_FAILED_STR                                               \
+    "The provided path is already present. It is not a directory"
+#define GD_MSG_BRICK_CREATION_FAIL_STR                                         \
+    "Brick isn't allowed to be created inside glusterd's working directory."
+#define GD_MSG_BRICK_CREATE_ROOT_STR                                           \
+    "The brick is being created in the root partition. It is recommended "     \
+    "that you don't use the system's root partition for storage backend."
+#define GD_MSG_BRICK_CREATE_MNTPNT_STR                                         \
+    "The brick is a mount point. Please create a sub-directory under the "     \
+    "mount point and use that as the brick directory."
+#define GD_MSG_CREATE_GLUSTER_DIR_FAILED_STR                                   \
+    "Failed to create glusterfs directory"
+#define GD_MSG_VOLINFO_IMPORT_FAIL_STR "Volume is not yet imported"
+#define GD_MSG_BRICK_SET_INFO_FAIL_STR                                         \
+    "Failed to add brick mount details to dict"
+#define GD_MSG_SET_XATTR_BRICK_FAIL_STR                                        \
+    "Glusterfs is not supported on brick. Setting extended attribute failed"
+#define GD_MSG_SET_XATTR_FAIL_STR "Failed to set extended attribute"
+#define GD_MSG_REMOVE_XATTR_FAIL_STR "Failed to remove extended attribute"
+#define GD_MSG_XLATOR_SET_OPT_FAIL_STR "Failed to set xlator type"
+#define GD_MSG_XLATOR_LINK_FAIL_STR                                            \
+    "Failed to do the link of xlator with children"
+#define GD_MSG_READ_ERROR_STR "Failed to read directory"
+#define GD_MSG_INCOMPATIBLE_VALUE_STR "Incompatible transport type"
+#define GD_MSG_VOL_STOP_ARGS_GET_FAILED_STR "Failed to get volume stop args"
+#define GD_MSG_FRAME_CREATE_FAIL_STR "Failed to create frame"
+#define GD_MSG_VOLUME_NOT_IMPORTED_STR "Volume has not been imported"
+#define GD_MSG_ADD_BRICK_MNT_INFO_FAIL_STR                                     \
+    "Failed to add brick mount details to dict"
+#define GD_MSG_GET_MNT_ENTRY_INFO_FAIL_STR "Failed to get mount entry details"
+#define GD_MSG_BRICKPATH_ROOT_GET_FAIL_STR "failed to get brick root details"
+#define GD_MSG_VOL_INFO_REQ_RECVD_STR "Received get volume info req"
+#define GD_MSG_NO_FLAG_SET_STR "No flags set"
+#define GD_MSG_CREATE_DIR_FAILED_STR "Failed to create directory"
+#define GD_MSG_POST_HOOK_STUB_INIT_FAIL_STR                                    \
+    "Failed to initialize post hooks stub"
+#define GD_MSG_FILE_OP_FAILED_STR "File operation failed"
+#define GD_MSG_INODE_SIZE_GET_FAIL_STR "Failed to get inode size"
+#define GD_MSG_CMD_EXEC_FAIL_STR "Command execution failed"
+#define GD_MSG_XLATOR_CREATE_FAIL_STR "Failed to create xlator"
+#define GD_MSG_CLRCLK_VOL_REQ_RCVD_STR "Received clear-locks request for volume"
+#define GD_MSG_BRK_PORT_NUM_GET_FAIL_STR                                       \
+    "Couldn't get port number of local bricks"
+#define GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL_STR                               \
+    "Creating mount directory for clear-locks failed"
+#define GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL_STR                                    \
+    "Failed to mount clear-locks maintenance client"
+#define GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL_STR                                   \
+    "Failed to unmount clear-locks mount point"
+#define GD_MSG_CLRCLK_SND_CMD_FAIL_STR "Failed to send command for clear-locks"
+#define GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL_STR                         \
+    "Failed to allocate memory or get serialized length of dict"
+#define GD_MSG_GET_XATTR_FAIL_STR "Failed to get extended attribute"
 
-/*! \file glusterd-messages.h
- *  \brief Glusterd log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD
-
-#define GLFS_NUM_MESSAGES       578
-
-#define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x GLFS_COMP_BASE, "Invalid: Start of messages"
-/*------------*/
-
-/*!
- * @messageid 106001
- * @diagnosis Operation could not be performed because the server quorum was not
- *            met
- * @recommendedaction Ensure that other peer nodes are online and reachable from
- *                    the local peer node
- */
-#define GD_MSG_SERVER_QUORUM_NOT_MET (GLUSTERD_COMP_BASE + 1)
-
-/*!
- * @messageid 106002
- * @diagnosis The local bricks belonging to the volume were killed because
- *            the server-quorum was not met
- * @recommendedaction Ensure that other peer nodes are online and reachable from
- *                    the local peer node
- */
-#define GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS (GLUSTERD_COMP_BASE + 2)
-
-/*!
- * @messageid 106003
- * @diagnosis The local bricks belonging to the named volume were (re)started
- *            because the server-quorum was met
- * @recommendedaction  None
- */
-#define GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS (GLUSTERD_COMP_BASE + 3)
-
-/*!
- * @messageid 106004
- * @diagnosis Glusterd on the peer might be down or unreachable
- * @recommendedaction  Check if glusterd is running on the peer node or if
- *                     the firewall rules are not blocking port 24007
- */
-#define GD_MSG_PEER_DISCONNECTED (GLUSTERD_COMP_BASE + 4)
-
-/*!
- * @messageid 106005
- * @diagnosis Brick process might be down
- * @recommendedaction Check brick log files to get more information on the cause
- *                    for the brick's offline status. To bring the brick back
- *                    online,run gluster volume start <VOLNAME> force
- */
-#define GD_MSG_BRICK_DISCONNECTED (GLUSTERD_COMP_BASE + 5)
-
-/*!
- * @messageid 106006
- * @diagnosis NFS Server or Self-heal daemon might be down
- * @recommendedaction Check nfs or self-heal daemon log files to get more
- *                    information on the cause for the brick's offline status.
- *                    To bring the brick back online, run gluster volume
- *                    start <VOLNAME> force
- */
-#define GD_MSG_NODE_DISCONNECTED (GLUSTERD_COMP_BASE + 6)
-
-/*!
- * @messageid 106007
- * @diagnosis Rebalance process might be down
- * @recommendedaction None
- */
-#define GD_MSG_REBALANCE_DISCONNECTED (GLUSTERD_COMP_BASE + 7)
-
-/*!
- * @messageid 106008
- * @diagnosis Volume cleanup failed
- * @recommendedaction None
- */
-#define GD_MSG_VOL_CLEANUP_FAIL (GLUSTERD_COMP_BASE + 8)
-
-/*!
- * @messageid 106009
- * @diagnosis Volume version mismatch while adding a peer
- * @recommendedaction None
- */
-#define GD_MSG_VOL_VERS_MISMATCH (GLUSTERD_COMP_BASE + 9)
-
-/*!
- * @messageid 106010
- * @diagnosis Volume checksum mismatch while adding a peer
- * @recommendedaction Check for which node the checksum mismatch happens
- *                    and delete the volume configuration files from it andi
- *                    restart glusterd
- */
-#define GD_MSG_CKSUM_VERS_MISMATCH (GLUSTERD_COMP_BASE + 10)
-
-/*!
- * @messageid 106011
- * @diagnosis A volume quota-conf version mismatch occurred while adding a peer
- * @recommendedaction None
- */
-#define GD_MSG_QUOTA_CONFIG_VERS_MISMATCH (GLUSTERD_COMP_BASE + 11)
-
-/*!
- * @messageid 106012
- * @diagnosis A quota-conf checksum mismatch occurred while adding a peer
- * @recommendedaction Check for which node the checksum mismatch happens
- *                    and delete the volume configuration files from it and
- *                    restart glusterd
- */
-#define GD_MSG_QUOTA_CONFIG_CKSUM_MISMATCH (GLUSTERD_COMP_BASE + 12)
-
-/*!
- * @messageid 106013
- * @diagnosis Brick process could not be terminated
- * @recommendedaction Find the pid of the brick process from the log file and
- *                    manually kill it
- */
-#define GD_MSG_BRICK_STOP_FAIL (GLUSTERD_COMP_BASE + 13)
-
-/*!
- * @messageid 106014
- * @diagnosis One of the listed services:NFS Server, Quota Daemon, Self Heal
- *            Daemon, or brick process could not be brought offline
- * @recommendedaction Find the pid of the process from the log file and
- *                    manually kill it
- */
-#define GD_MSG_SVC_KILL_FAIL (GLUSTERD_COMP_BASE + 14)
-
-/*!
- * @messageid 106015
- * @diagnosis The process could not be killed with the specified PID
- * @recommendedaction None
- */
-#define GD_MSG_PID_KILL_FAIL (GLUSTERD_COMP_BASE + 15)
-
-/*!
- * @messageid 106016
- * @diagnosis Rebalance socket file is not found
- * @recommendedaction Rebalance failed as the socket file for rebalance is
- *                    missing. Restart the rebalance process
- */
-#define GD_MSG_REBAL_NO_SOCK_FILE (GLUSTERD_COMP_BASE + 16)
-
-/*!
- * @messageid 106017
- * @diagnosis Unix options could not be set
- * @recommendedaction Server is out of memory and needs a restart
- */
-#define GD_MSG_UNIX_OP_BUILD_FAIL (GLUSTERD_COMP_BASE + 17)
-
-/*!
- * @messageid 106018
- * @diagnosis RPC creation failed
- * @recommendedaction Rebalance failed as glusterd could not establish an RPC
- *                    connection. Check the log file for the exact reason of the
- *                    failure and then restart the rebalance process
- */
-#define GD_MSG_RPC_CREATE_FAIL (GLUSTERD_COMP_BASE + 18)
-
-/*!
- * @messageid 106019
- * @diagnosis The default options on volume could not be set with the volume
- *            create and volume reset commands
- * @recommendedaction Check glusterd log files to see the exact reason for
- *                    failure to set default options
- */
-#define GD_MSG_FAIL_DEFAULT_OPT_SET (GLUSTERD_COMP_BASE + 19)
-
-/*!
- * @messageid 106020
- * @diagnosis Failed to release cluster wide lock for one of the peer
- * @recommendedaction Restart the glusterd service on the node where the command
- * was issued
- */
-#define GD_MSG_CLUSTER_UNLOCK_FAILED (GLUSTERD_COMP_BASE + 20)
-
-/*!
- * @messageid 106021
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_MEMORY    (GLUSTERD_COMP_BASE + 21)
-
-/*!
- * @messageid 106022
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNSUPPORTED_VERSION    (GLUSTERD_COMP_BASE + 22)
-
-/*!
- * @messageid 106023
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_COMMAND_NOT_FOUND    (GLUSTERD_COMP_BASE + 23)
-
-/*!
- * @messageid 106024
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPSHOT_OP_FAILED    (GLUSTERD_COMP_BASE + 24)
-
-/*!
- * @messageid 106025
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INVALID_ENTRY    (GLUSTERD_COMP_BASE + 25)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_NOT_FOUND    (GLUSTERD_COMP_BASE + 27)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REG_COMPILE_FAILED    (GLUSTERD_COMP_BASE + 28)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FILE_OP_FAILED             (GLUSTERD_COMP_BASE + 29)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_CREATION_FAIL        (GLUSTERD_COMP_BASE + 30)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_OP_FAILED             (GLUSTERD_COMP_BASE + 31)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CREATE_DIR_FAILED        (GLUSTERD_COMP_BASE + 32)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DIR_OP_FAILED            (GLUSTERD_COMP_BASE + 33)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_STOP_FAILED          (GLUSTERD_COMP_BASE + 34)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_CLI_RESP                (GLUSTERD_COMP_BASE + 35)
-
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_INIT_FAILED          (GLUSTERD_COMP_BASE + 36)
-
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_LIST_GET_FAIL        (GLUSTERD_COMP_BASE + 37)
-
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNOUNT_FAILED              (GLUSTERD_COMP_BASE + 38)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_DESTROY_FAILED       (GLUSTERD_COMP_BASE + 39)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_CLEANUP_FAIL         (GLUSTERD_COMP_BASE + 40)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_ACTIVATE_FAIL        (GLUSTERD_COMP_BASE + 41)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_DEACTIVATE_FAIL     (GLUSTERD_COMP_BASE + 42)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_RESTORE_FAIL        (GLUSTERD_COMP_BASE + 43)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_REMOVE_FAIL         (GLUSTERD_COMP_BASE + 44)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_CONFIG_FAIL         (GLUSTERD_COMP_BASE + 45)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_STATUS_FAIL         (GLUSTERD_COMP_BASE + 46)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_INIT_FAIL           (GLUSTERD_COMP_BASE + 47)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLINFO_SET_FAIL         (GLUSTERD_COMP_BASE + 48)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLINFO_GET_FAIL         (GLUSTERD_COMP_BASE + 49)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_CREATION_FAIL      (GLUSTERD_COMP_BASE + 50)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_GET_INFO_FAIL      (GLUSTERD_COMP_BASE + 51)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_NEW_INFO_FAIL      (GLUSTERD_COMP_BASE + 52)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LVS_FAIL                 (GLUSTERD_COMP_BASE + 53)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SETXATTR_FAIL            (GLUSTERD_COMP_BASE + 54)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UMOUNTING_SNAP_BRICK     (GLUSTERD_COMP_BASE + 55)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_UNSUPPORTED           (GLUSTERD_COMP_BASE + 56)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_NOT_FOUND           (GLUSTERD_COMP_BASE + 57)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FS_LABEL_UPDATE_FAIL     (GLUSTERD_COMP_BASE + 58)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LVM_MOUNT_FAILED         (GLUSTERD_COMP_BASE + 59)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_SET_FAILED          (GLUSTERD_COMP_BASE + 60)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CANONICALIZE_FAIL        (GLUSTERD_COMP_BASE + 61)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_GET_FAILED          (GLUSTERD_COMP_BASE + 62)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_INFO_FAIL   (GLUSTERD_COMP_BASE + 63)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_VOL_CONFIG_FAIL     (GLUSTERD_COMP_BASE + 64)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_OBJECT_STORE_FAIL   (GLUSTERD_COMP_BASE + 65)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_UNSERIALIZE_FAIL    (GLUSTERD_COMP_BASE + 66)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_RESTORE_REVERT_FAIL        (GLUSTERD_COMP_BASE + 67)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_LIST_SET_FAIL       (GLUSTERD_COMP_BASE + 68)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLFILE_CREATE_FAIL      (GLUSTERD_COMP_BASE + 69)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLINFO_REMOVE_FAIL      (GLUSTERD_COMP_BASE + 70)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_DELETE_FAIL          (GLUSTERD_COMP_BASE + 71)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPSHOT_PENDING         (GLUSTERD_COMP_BASE + 72)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_PATH_UNMOUNTED         (GLUSTERD_COMP_BASE + 73)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_ADD_FAIL           (GLUSTERD_COMP_BASE + 74)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_SET_INFO_FAIL      (GLUSTERD_COMP_BASE + 75)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LVCREATE_FAIL            (GLUSTERD_COMP_BASE + 76)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VG_GET_FAIL      (GLUSTERD_COMP_BASE + 77)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TPOOL_GET_FAIL          (GLUSTERD_COMP_BASE + 78)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LVM_REMOVE_FAILED        (GLUSTERD_COMP_BASE + 79)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSEDSNAP_INFO_SET_FAIL         (GLUSTERD_COMP_BASE + 80)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_MOUNTOPTS_FAIL      (GLUSTERD_COMP_BASE + 81)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_LIST_STORE_FAIL        (GLUSTERD_COMP_BASE + 82)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INVALID_MISSED_SNAP_ENTRY        (GLUSTERD_COMP_BASE + 83)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_GET_FAIL         (GLUSTERD_COMP_BASE + 84)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_CREATE_FAIL          (GLUSTERD_COMP_BASE + 85)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DUP_ENTRY         (GLUSTERD_COMP_BASE + 86)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_STATUS_DONE          (GLUSTERD_COMP_BASE + 87)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_EXEC_PERMS            (GLUSTERD_COMP_BASE + 88)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLOBAL_OP_VERSION_SET_FAIL       (GLUSTERD_COMP_BASE + 89)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HARD_LIMIT_SET_FAIL      (GLUSTERD_COMP_BASE + 90)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_SUCCESS      (GLUSTERD_COMP_BASE + 91)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STORE_FAIL       (GLUSTERD_COMP_BASE + 92)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLOBAL_OP_VERSION_GET_FAIL       (GLUSTERD_COMP_BASE + 93)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GEOREP_GET_FAILED        (GLUSTERD_COMP_BASE + 94)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERD_UMOUNT_FAIL     (GLUSTERD_COMP_BASE + 95)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUORUM_CHECK_FAIL        (GLUSTERD_COMP_BASE + 96)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUORUM_COUNT_IGNORED     (GLUSTERD_COMP_BASE + 97)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_MOUNT_FAIL          (GLUSTERD_COMP_BASE + 98)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RSP_DICT_USE_FAIL       (GLUSTERD_COMP_BASE + 99)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_IMPORT_FAIL         (GLUSTERD_COMP_BASE + 100)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_CONFLICT            (GLUSTERD_COMP_BASE + 101)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_DELETE       (GLUSTERD_COMP_BASE + 102)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUOTA_CONFIG_IMPORT_FAIL         (GLUSTERD_COMP_BASE + 103)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPDIR_CREATE_FAIL      (GLUSTERD_COMP_BASE + 104)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_PRESENT      (GLUSTERD_COMP_BASE + 105)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UUID_NULL                (GLUSTERD_COMP_BASE + 106)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TSTAMP_SET_FAIL          (GLUSTERD_COMP_BASE + 107)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RESP_AGGR_FAIL           (GLUSTERD_COMP_BASE + 108)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_EMPTY               (GLUSTERD_COMP_BASE + 109)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_CREATE_FAIL         (GLUSTERD_COMP_BASE + 110)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_STOP_FAIL          (GLUSTERD_COMP_BASE + 111)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SOFT_LIMIT_REACHED       (GLUSTERD_COMP_BASE + 112)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_START_FAIL        (GLUSTERD_COMP_BASE + 113)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_CREATE_FAIL        (GLUSTERD_COMP_BASE + 114)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_INIT_FAIL          (GLUSTERD_COMP_BASE + 115)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_OP_FAIL           (GLUSTERD_COMP_BASE + 116)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL        (GLUSTERD_COMP_BASE + 117)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_UNLOCK_FAIL   (GLUSTERD_COMP_BASE + 118)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_LOCK_GET_FAIL     (GLUSTERD_COMP_BASE + 119)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_LOCKDOWN_FAIL     (GLUSTERD_COMP_BASE + 120)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_POST_VALIDATION_FAIL     (GLUSTERD_COMP_BASE + 121)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PRE_VALIDATION_FAIL      (GLUSTERD_COMP_BASE + 122)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_COMMIT_OP_FAIL           (GLUSTERD_COMP_BASE + 123)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_LIST_CREATE_FAIL    (GLUSTERD_COMP_BASE + 124)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_OP_FAIL            (GLUSTERD_COMP_BASE + 125)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OPINFO_SET_FAIL          (GLUSTERD_COMP_BASE + 126)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_EVENT_UNLOCK_FAIL    (GLUSTERD_COMP_BASE + 127)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_OP_RESP_FAIL      (GLUSTERD_COMP_BASE + 128)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_NOT_FOUND           (GLUSTERD_COMP_BASE + 129)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REQ_DECODE_FAIL          (GLUSTERD_COMP_BASE + 130)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_SERL_LENGTH_GET_FAIL        (GLUSTERD_COMP_BASE + 131)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ALREADY_STOPPED          (GLUSTERD_COMP_BASE + 132)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PRE_VALD_RESP_FAIL       (GLUSTERD_COMP_BASE + 133)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SVC_GET_FAIL             (GLUSTERD_COMP_BASE + 134)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLFILE_NOT_FOUND        (GLUSTERD_COMP_BASE + 135)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_EVENT_LOCK_FAIL    (GLUSTERD_COMP_BASE + 136)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NON_STRIPE_VOL           (GLUSTERD_COMP_BASE + 137)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_OBJ_GET_FAIL      (GLUSTERD_COMP_BASE + 138)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUOTA_DISABLED           (GLUSTERD_COMP_BASE + 139)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CACHE_MINMAX_SIZE_INVALID    (GLUSTERD_COMP_BASE + 140)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUOTA_GET_STAT_FAIL          (GLUSTERD_COMP_BASE + 141)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SUBVOLUMES_EXCEED          (GLUSTERD_COMP_BASE + 142)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_ADD                 (GLUSTERD_COMP_BASE + 143)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_REMOVE                 (GLUSTERD_COMP_BASE + 144)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CREATE_KEY_FAIL          (GLUSTERD_COMP_BASE + 145)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL       (GLUSTERD_COMP_BASE + 146)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL       (GLUSTERD_COMP_BASE + 147)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RESP_FROM_UNKNOWN_PEER           (GLUSTERD_COMP_BASE + 148)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL         (GLUSTERD_COMP_BASE + 149)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GFID_VALIDATE_SET_FAIL           (GLUSTERD_COMP_BASE + 150)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_LOCK_FAIL                   (GLUSTERD_COMP_BASE + 151)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_UNLOCK_FAIL                 (GLUSTERD_COMP_BASE + 152)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMT_OP_FAIL                     (GLUSTERD_COMP_BASE + 153)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TRANS_OPINFO_CLEAR_FAIL          (GLUSTERD_COMP_BASE + 154)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERD_LOCK_FAIL               (GLUSTERD_COMP_BASE + 155)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TRANS_OPINFO_SET_FAIL            (GLUSTERD_COMP_BASE + 156)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TRANS_IDGEN_FAIL                 (GLUSTERD_COMP_BASE + 157)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RPC_FAILURE                      (GLUSTERD_COMP_BASE + 158)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_VERS_ADJUST_FAIL              (GLUSTERD_COMP_BASE + 159)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_DEVICE_NAME_GET_FAIL        (GLUSTERD_COMP_BASE + 160)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_STATUS_NOT_PENDING          (GLUSTERD_COMP_BASE + 161)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMT_PGM_SET_FAIL                (GLUSTERD_COMP_BASE + 161)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_EVENT_INJECT_FAIL                (GLUSTERD_COMP_BASE + 162)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VERS_INFO                        (GLUSTERD_COMP_BASE + 163)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_INFO_REQ_RECVD               (GLUSTERD_COMP_BASE + 164)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VERS_GET_FAIL                    (GLUSTERD_COMP_BASE + 165)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_EVENT_NEW_GET_FAIL               (GLUSTERD_COMP_BASE + 166)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RPC_LAYER_ERROR                  (GLUSTERD_COMP_BASE + 167)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_HANDSHAKE_ACK                 (GLUSTERD_COMP_BASE + 168)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_VERSION_MISMATCH              (GLUSTERD_COMP_BASE + 169)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HANDSHAKE_REQ_REJECTED           (GLUSTERD_COMP_BASE + 170)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNKNOWN_MODE                     (GLUSTERD_COMP_BASE + 171)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DEFRAG_STATUS_UPDATED            (GLUSTERD_COMP_BASE + 172)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_FLAG_SET                      (GLUSTERD_COMP_BASE + 173)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VERSION_UNSUPPORTED              (GLUSTERD_COMP_BASE + 174)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UUID_SET_FAIL                    (GLUSTERD_COMP_BASE + 175)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MOUNT_REQ_FAIL                    (GLUSTERD_COMP_BASE + 176)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL   (GLUSTERD_COMP_BASE + 177)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_VERS_STORE_FAIL               (GLUSTERD_COMP_BASE + 178)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_AUTOMIC_UPDATE_FAIL         (GLUSTERD_COMP_BASE + 179)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPINFO_WRITE_FAIL              (GLUSTERD_COMP_BASE + 180)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPINFO_CREATE_FAIL             (GLUSTERD_COMP_BASE + 181)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_INFO_STORE_FAIL            (GLUSTERD_COMP_BASE + 182)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_MNTPATH_MOUNT_FAIL           (GLUSTERD_COMP_BASE + 183)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_MNTPATH_GET_FAIL             (GLUSTERD_COMP_BASE + 184)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_BRK_MNT_RECREATE_FAIL       (GLUSTERD_COMP_BASE + 185)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_RESOLVE_BRICK_FAIL          (GLUSTERD_COMP_BASE + 186)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RESOLVE_BRICK_FAIL               (GLUSTERD_COMP_BASE + 187)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_MNT_RECREATE_FAIL            (GLUSTERD_COMP_BASE + 188)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TMP_FILE_UNLINK_FAIL             (GLUSTERD_COMP_BASE + 189)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_VALS_WRITE_FAIL              (GLUSTERD_COMP_BASE + 190)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STORE_HANDLE_GET_FAIL            (GLUSTERD_COMP_BASE + 191)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STORE_HANDLE_WRITE_FAIL          (GLUSTERD_COMP_BASE + 192)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_LIST_STORE_HANDLE_GET_FAIL    \
-                                                (GLUSTERD_COMP_BASE + 193)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MISSED_SNAP_LIST_EMPTY           (GLUSTERD_COMP_BASE + 194)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_VOL_RETRIEVE_FAIL           (GLUSTERD_COMP_BASE + 195)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPSHOT_UPDATE_FAIL             (GLUSTERD_COMP_BASE + 196)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_PORT_STORE_FAIL            (GLUSTERD_COMP_BASE + 197)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CKSUM_STORE_FAIL                 (GLUSTERD_COMP_BASE + 198)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STORE_HANDLE_CREATE_FAIL         (GLUSTERD_COMP_BASE + 199)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HANDLE_NULL                      (GLUSTERD_COMP_BASE + 200)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_RESTORE_FAIL                 (GLUSTERD_COMP_BASE + 201)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NAME_TOO_LONG                    (GLUSTERD_COMP_BASE + 202)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UUID_PARSE_FAIL                  (GLUSTERD_COMP_BASE + 203)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNKNOWN_KEY                      (GLUSTERD_COMP_BASE + 204)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STORE_ITER_DESTROY_FAIL          (GLUSTERD_COMP_BASE + 205)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STORE_ITER_GET_FAIL              (GLUSTERD_COMP_BASE + 206)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLINFO_UPDATE_FAIL              (GLUSTERD_COMP_BASE + 207)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PARSE_BRICKINFO_FAIL             (GLUSTERD_COMP_BASE + 208)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VERS_STORE_FAIL                  (GLUSTERD_COMP_BASE + 209)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HEADER_ADD_FAIL                  (GLUSTERD_COMP_BASE + 210)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUOTA_CONF_WRITE_FAIL            (GLUSTERD_COMP_BASE + 211)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUOTA_CONF_CORRUPT               (GLUSTERD_COMP_BASE + 212)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FORK_FAIL                        (GLUSTERD_COMP_BASE + 213)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CKSUM_COMPUTE_FAIL               (GLUSTERD_COMP_BASE + 214)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VERS_CKSUM_STORE_FAIL            (GLUSTERD_COMP_BASE + 215)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GETXATTR_FAIL                    (GLUSTERD_COMP_BASE + 216)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CONVERSION_FAILED                (GLUSTERD_COMP_BASE + 217)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_NOT_DISTRIBUTE               (GLUSTERD_COMP_BASE + 218)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_STOPPED                      (GLUSTERD_COMP_BASE + 219)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OPCTX_GET_FAIL                   (GLUSTERD_COMP_BASE + 220)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TASKID_GEN_FAIL                  (GLUSTERD_COMP_BASE + 221)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REBALANCE_ID_MISSING             (GLUSTERD_COMP_BASE + 222)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME      (GLUSTERD_COMP_BASE + 223)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DEFRAG_STATUS_UPDATE_FAIL        (GLUSTERD_COMP_BASE + 224)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UUID_GEN_STORE_FAIL              (GLUSTERD_COMP_BASE + 225)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UUID_STORE_FAIL                  (GLUSTERD_COMP_BASE + 226)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_INIT                          (GLUSTERD_COMP_BASE + 227)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MODULE_NOT_INSTALLED             (GLUSTERD_COMP_BASE + 228)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MODULE_NOT_WORKING               (GLUSTERD_COMP_BASE + 229)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_WRITE_ACCESS_GRANT_FAIL          (GLUSTERD_COMP_BASE + 230)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DIRPATH_TOO_LONG                 (GLUSTERD_COMP_BASE + 231)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOGGROUP_INVALID                 (GLUSTERD_COMP_BASE + 232)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DIR_PERM_LIBERAL                 (GLUSTERD_COMP_BASE + 233)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DIR_PERM_STRICT                  (GLUSTERD_COMP_BASE + 234)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MOUNT_SPEC_INSTALL_FAIL          (GLUSTERD_COMP_BASE + 234)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERD_SOCK_LISTENER_START_FAIL      (GLUSTERD_COMP_BASE + 235)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DIR_NOT_FOUND                    (GLUSTERD_COMP_BASE + 236)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FAILED_INIT_SHDSVC               (GLUSTERD_COMP_BASE + 237)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FAILED_INIT_NFSSVC               (GLUSTERD_COMP_BASE + 238)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FAILED_INIT_QUOTASVC             (GLUSTERD_COMP_BASE + 239)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RPC_INIT_FAIL                    (GLUSTERD_COMP_BASE + 240)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RPCSVC_REG_NOTIFY_RETURNED       (GLUSTERD_COMP_BASE + 241)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RPC_TRANSPORT_COUNT_GET_FAIL     (GLUSTERD_COMP_BASE + 242)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RPC_LISTENER_CREATE_FAIL         (GLUSTERD_COMP_BASE + 243)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_VERS_RESTORE_FAIL               (GLUSTERD_COMP_BASE + 244)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SELF_HEALD_DISABLED              (GLUSTERD_COMP_BASE + 245)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PRIV_NULL                        (GLUSTERD_COMP_BASE + 246)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GSYNC_VALIDATION_FAIL            (GLUSTERD_COMP_BASE + 247)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL     (GLUSTERD_COMP_BASE + 248)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_NOT_PERMITTED_AC_REQD         (GLUSTERD_COMP_BASE + 250)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_NOT_PERMITTED                 (GLUSTERD_COMP_BASE + 251)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REBALANCE_START_FAIL             (GLUSTERD_COMP_BASE + 252)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_RECONF_FAIL                  (GLUSTERD_COMP_BASE + 253)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REMOVE_BRICK_ID_SET_FAIL         (GLUSTERD_COMP_BASE + 254)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_MOUNTDIR_GET_FAIL          (GLUSTERD_COMP_BASE + 255)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_NOT_FOUND                  (GLUSTERD_COMP_BASE + 256)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRKPATH_TOO_LONG                 (GLUSTERD_COMP_BASE + 257)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL        (GLUSTERD_COMP_BASE + 258)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL         (GLUSTERD_COMP_BASE + 259)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL    (GLUSTERD_COMP_BASE + 260)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_PORT_NUM_GET_FAIL            (GLUSTERD_COMP_BASE + 261)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_STATEDUMP_FAIL               (GLUSTERD_COMP_BASE + 262)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_GRAPH_CHANGE_NOTIFY_FAIL     (GLUSTERD_COMP_BASE + 263)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INVALID_VG                       (GLUSTERD_COMP_BASE + 264)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERD_OP_FAILED               (GLUSTERD_COMP_BASE + 265)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL    (GLUSTERD_COMP_BASE + 266)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STALE_PEERINFO_REMOVE_FAIL       (GLUSTERD_COMP_BASE + 267)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TRANS_ID_GET_FAIL                (GLUSTERD_COMP_BASE + 268)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RES_DECODE_FAIL                  (GLUSTERD_COMP_BASE + 269)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_ALREADY_EXIST                (GLUSTERD_COMP_BASE + 270)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BAD_BRKORDER                     (GLUSTERD_COMP_BASE + 271)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BAD_BRKORDER_CHECK_FAIL          (GLUSTERD_COMP_BASE + 272)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_SELECT_FAIL                (GLUSTERD_COMP_BASE + 273)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_LOCK_RESP_FROM_PEER           (GLUSTERD_COMP_BASE + 274)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_LOCK_FROM_UUID_REJCT      (GLUSTERD_COMP_BASE + 275)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STAGE_FROM_UUID_REJCT            (GLUSTERD_COMP_BASE + 276)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNLOCK_FROM_UUID_REJCT           (GLUSTERD_COMP_BASE + 277)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_UNLOCK_FROM_UUID_REJCT    (GLUSTERD_COMP_BASE + 278)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_COMMIT_FROM_UUID_REJCT           (GLUSTERD_COMP_BASE + 279)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_NOT_STARTED                  (GLUSTERD_COMP_BASE + 280)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_NOT_REPLICA                  (GLUSTERD_COMP_BASE + 281)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OLD_REMOVE_BRICK_EXISTS          (GLUSTERD_COMP_BASE + 283)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_USE_THE_FORCE                    (GLUSTERD_COMP_BASE + 284)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OIP                              (GLUSTERD_COMP_BASE + 285)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OIP_RETRY_LATER                  (GLUSTERD_COMP_BASE + 286)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GSYNC_RESTART_FAIL               (GLUSTERD_COMP_BASE + 287)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_FROM_UUID_REJCT             (GLUSTERD_COMP_BASE + 288)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL      (GLUSTERD_COMP_BASE + 289)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HOSTNAME_RESOLVE_FAIL            (GLUSTERD_COMP_BASE + 290)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_COUNT_VALIDATE_FAILED            (GLUSTERD_COMP_BASE + 291)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SPAWNING_CHILD_FAILED            (GLUSTERD_COMP_BASE + 292)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_READ_CHILD_DATA_FAILED           (GLUSTERD_COMP_BASE + 293)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DEFAULT_TEMP_CONFIG              (GLUSTERD_COMP_BASE + 294)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PIDFILE_CREATE_FAILED            (GLUSTERD_COMP_BASE + 295)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GSYNCD_SPAWN_FAILED              (GLUSTERD_COMP_BASE + 296)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SUBOP_NOT_FOUND                  (GLUSTERD_COMP_BASE + 297)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RESERVED_OPTION                  (GLUSTERD_COMP_BASE + 298)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERD_PRIV_NOT_FOUND          (GLUSTERD_COMP_BASE + 299)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SLAVEINFO_FETCH_ERROR            (GLUSTERD_COMP_BASE + 300)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VALIDATE_FAILED                  (GLUSTERD_COMP_BASE + 301)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INVOKE_ERROR                     (GLUSTERD_COMP_BASE + 302)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SESSION_CREATE_ERROR             (GLUSTERD_COMP_BASE + 303)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STOP_FORCE                        (GLUSTERD_COMP_BASE + 304)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GET_CONFIG_INFO_FAILED           (GLUSTERD_COMP_BASE + 305)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STAT_FILE_READ_FAILED            (GLUSTERD_COMP_BASE + 306)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CONF_PATH_ASSIGN_FAILED          (GLUSTERD_COMP_BASE + 307)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SESSION_INACTIVE                 (GLUSTERD_COMP_BASE + 308)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PIDFILE_NOT_FOUND                (GLUSTERD_COMP_BASE + 309)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_CMD_ERROR                   (GLUSTERD_COMP_BASE + 310)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SRC_FILE_ERROR                   (GLUSTERD_COMP_BASE + 311)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GET_STATEFILE_NAME_FAILED        (GLUSTERD_COMP_BASE + 312)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATUS_NULL                      (GLUSTERD_COMP_BASE + 313)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATUSFILE_CREATE_FAILED         (GLUSTERD_COMP_BASE + 314)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SLAVE_URL_INVALID                (GLUSTERD_COMP_BASE + 315)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INVALID_SLAVE                    (GLUSTERD_COMP_BASE + 316)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_READ_ERROR                       (GLUSTERD_COMP_BASE + 317)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ARG_FETCH_ERROR                  (GLUSTERD_COMP_BASE + 318)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REG_FILE_MISSING                 (GLUSTERD_COMP_BASE + 319)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATEFILE_NAME_NOT_FOUND         (GLUSTERD_COMP_BASE + 320)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GEO_REP_START_FAILED             (GLUSTERD_COMP_BASE + 321)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GSYNCD_ERROR                     (GLUSTERD_COMP_BASE + 322)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UPDATE_STATEFILE_FAILED          (GLUSTERD_COMP_BASE + 323)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATUS_UPDATE_FAILED             (GLUSTERD_COMP_BASE + 324)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GSYNCD_OP_SET_FAILED             (GLUSTERD_COMP_BASE + 325)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BUFFER_EMPTY                     (GLUSTERD_COMP_BASE + 326)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CONFIG_INFO                      (GLUSTERD_COMP_BASE + 327)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FETCH_CONFIG_VAL_FAILED          (GLUSTERD_COMP_BASE + 328)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GSYNCD_PARSE_ERROR               (GLUSTERD_COMP_BASE + 329)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SESSION_ALREADY_EXIST            (GLUSTERD_COMP_BASE + 330)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FORCE_CREATE_SESSION             (GLUSTERD_COMP_BASE + 331)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GET_KEY_FAILED                   (GLUSTERD_COMP_BASE + 332)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SESSION_DEL_FAILED               (GLUSTERD_COMP_BASE + 333)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CMD_EXEC_FAIL                (GLUSTERD_COMP_BASE + 334)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STRDUP_FAILED                    (GLUSTERD_COMP_BASE + 335)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNABLE_TO_END                    (GLUSTERD_COMP_BASE + 336)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PAUSE_FAILED                     (GLUSTERD_COMP_BASE + 337)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NORMALIZE_URL_FAIL               (GLUSTERD_COMP_BASE + 338)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MODULE_ERROR                     (GLUSTERD_COMP_BASE + 339)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SLAVEINFO_STORE_ERROR            (GLUSTERD_COMP_BASE + 340)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MARKER_START_FAIL                (GLUSTERD_COMP_BASE + 341)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RESUME_FAILED                    (GLUSTERD_COMP_BASE + 342)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERFS_START_FAIL             (GLUSTERD_COMP_BASE + 343)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERFS_STOP_FAIL              (GLUSTERD_COMP_BASE + 344)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RBOP_STATE_STORE_FAIL            (GLUSTERD_COMP_BASE + 345)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PUMP_XLATOR_DISABLED             (GLUSTERD_COMP_BASE + 346)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ABORT_OP_FAIL                     (GLUSTERD_COMP_BASE + 347)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PAUSE_OP_FAIL                    (GLUSTERD_COMP_BASE + 348)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_VOL_FILE_GEN_FAIL             (GLUSTERD_COMP_BASE + 349)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HANDSHAKE_FAILED                   (GLUSTERD_COMP_BASE + 350)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CLI_REQ_EMPTY                      (GLUSTERD_COMP_BASE + 351)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_ADD_FAIL                       (GLUSTERD_COMP_BASE + 352)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SYNC_FROM_LOCALHOST_UNALLOWED       (GLUSTERD_COMP_BASE + 353)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UUIDS_SAME_RETRY                    (GLUSTERD_COMP_BASE + 354)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TSP_ALREADY_FORMED                  (GLUSTERD_COMP_BASE + 355)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLS_ALREADY_PRESENT                (GLUSTERD_COMP_BASE + 356)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REQ_CTX_CREATE_FAIL                 (GLUSTERD_COMP_BASE + 357)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_INFO_UPDATE_FAIL               (GLUSTERD_COMP_BASE + 358)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEERINFO_CREATE_FAIL                (GLUSTERD_COMP_BASE + 359)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REQ_FROM_UNKNOWN_PEER               (GLUSTERD_COMP_BASE + 360)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATUS_REPLY_STRING_CREATE_FAIL      (GLUSTERD_COMP_BASE + 361)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TOKENIZE_FAIL                        (GLUSTERD_COMP_BASE + 362)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LAZY_UMOUNT_FAIL                      (GLUSTERD_COMP_BASE + 363)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_SERVER_START_FAIL                 (GLUSTERD_COMP_BASE + 364)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_SERVER_STOP_FAIL                  (GLUSTERD_COMP_BASE + 365)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_CLEANUP_FAIL                      (GLUSTERD_COMP_BASE + 366)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_ALREADY_STARTED                    (GLUSTERD_COMP_BASE + 367)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_BRICKINFO_GET_FAIL                 (GLUSTERD_COMP_BASE + 368)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BAD_FORMAT                            (GLUSTERD_COMP_BASE + 369)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_CMD_FAIL                          (GLUSTERD_COMP_BASE + 370)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_NOT_STARTED_OR_PAUSED             (GLUSTERD_COMP_BASE + 371)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_NOT_STARTED                       (GLUSTERD_COMP_BASE + 372)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_PAUSED_ALREADY                    (GLUSTERD_COMP_BASE + 373)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_FREE_PORTS                        (GLUSTERD_COMP_BASE + 374)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_EVENT_STATE_TRANSITION_FAIL          (GLUSTERD_COMP_BASE + 375)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HANDLER_RETURNED                     (GLUSTERD_COMP_BASE + 376)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_COMPARE_CONFLICT                (GLUSTERD_COMP_BASE + 377)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_DETACH_CLEANUP_FAIL             (GLUSTERD_COMP_BASE + 378)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STALE_VOL_REMOVE_FAIL               (GLUSTERD_COMP_BASE + 379)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_AC_ERROR                            (GLUSTERD_COMP_BASE + 380)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_FAIL                           (GLUSTERD_COMP_BASE + 381)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MGMTV3_LOCK_REQ_SEND_FAIL           (GLUSTERD_COMP_BASE + 382)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLUSTERD_UNLOCK_FAIL                (GLUSTERD_COMP_BASE + 383)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RBOP_START_FAIL                     (GLUSTERD_COMP_BASE + 384)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNKNOWN_RESPONSE                    (GLUSTERD_COMP_BASE + 385)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_COMMIT_REQ_SEND_FAIL                (GLUSTERD_COMP_BASE + 386)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OPCTX_UPDATE_FAIL                   (GLUSTERD_COMP_BASE + 387)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OPCTX_NULL                          (GLUSTERD_COMP_BASE + 388)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_COPY_FAIL                       (GLUSTERD_COMP_BASE + 389)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SHD_STATUS_SET_FAIL                  (GLUSTERD_COMP_BASE + 390)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REPLICA_INDEX_GET_FAIL               (GLUSTERD_COMP_BASE + 391)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_SERVER_NOT_RUNNING               (GLUSTERD_COMP_BASE + 392)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STAGE_REQ_SEND_FAIL                  (GLUSTERD_COMP_BASE + 393)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_REQ_SEND_FAIL                    (GLUSTERD_COMP_BASE + 394)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLNAMES_GET_FAIL                     (GLUSTERD_COMP_BASE + 395)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_TASK_ID                            (GLUSTERD_COMP_BASE + 396)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ADD_REMOVE_BRICK_FAIL                 (GLUSTERD_COMP_BASE + 397)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SVC_RESTART_FAIL                      (GLUSTERD_COMP_BASE + 398)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_SET_FAIL                          (GLUSTERD_COMP_BASE + 399)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUOTAD_NOT_RUNNING                    (GLUSTERD_COMP_BASE + 400)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XLATOR_COUNT_GET_FAIL                (GLUSTERD_COMP_BASE + 401)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TRANS_OPINFO_GET_FAIL                 (GLUSTERD_COMP_BASE + 402)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TRANS_ID_INVALID                      (GLUSTERD_COMP_BASE + 403)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_OPTIONS_GIVEN                      (GLUSTERD_COMP_BASE + 404)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAPD_NOT_RUNNING                     (GLUSTERD_COMP_BASE + 405)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ADD_ADDRESS_TO_PEER_FAIL              (GLUSTERD_COMP_BASE + 406)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_ADDRESS_GET_FAIL                 (GLUSTERD_COMP_BASE + 407)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GETADDRINFO_FAIL                      (GLUSTERD_COMP_BASE + 408)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEERINFO_DELETE_FAIL                 (GLUSTERD_COMP_BASE + 409)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_KEY_NULL                            (GLUSTERD_COMP_BASE + 410)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SPAWN_SVCS_FAIL                     (GLUSTERD_COMP_BASE + 411)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_ITER_FAIL                     (GLUSTERD_COMP_BASE + 412)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TASK_STATUS_UPDATE_FAIL             (GLUSTERD_COMP_BASE + 413)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_ID_MISMATCH                     (GLUSTERD_COMP_BASE + 414)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STR_TO_BOOL_FAIL                     (GLUSTERD_COMP_BASE + 415)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_MNT_BRICKS_MISMATCH                (GLUSTERD_COMP_BASE + 416)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_SRC_BRICKS_MISMATCH                 (GLUSTERD_COMP_BASE + 417)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MNTENTRY_GET_FAIL                     (GLUSTERD_COMP_BASE + 418)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INODE_SIZE_GET_FAIL                   (GLUSTERD_COMP_BASE + 419)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NO_STATEFILE_ENTRY                     (GLUSTERD_COMP_BASE + 420)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PMAP_UNSET_FAIL                        (GLUSTERD_COMP_BASE + 421)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GLOBAL_OPT_IMPORT_FAIL                 (GLUSTERD_COMP_BASE + 422)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSD_BRICK_DISCONNECT_FAIL                  (GLUSTERD_COMP_BASE + 423)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_DETAILS_IMPORT_FAIL               (GLUSTERD_COMP_BASE + 424)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICKINFO_CREATE_FAIL                  (GLUSTERD_COMP_BASE + 425)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_QUOTA_CKSUM_VER_STORE_FAIL             (GLUSTERD_COMP_BASE + 426)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CKSUM_GET_FAIL                         (GLUSTERD_COMP_BASE + 427)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICKPATH_ROOT_GET_FAIL                (GLUSTERD_COMP_BASE + 428)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HOSTNAME_TO_UUID_FAIL                  (GLUSTERD_COMP_BASE + 429)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REPLY_SUBMIT_FAIL                      (GLUSTERD_COMP_BASE + 430)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SERIALIZE_MSG_FAIL                    (GLUSTERD_COMP_BASE + 431)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ENCODE_FAIL                           (GLUSTERD_COMP_BASE + 432)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RB_DST_BRICKS_MISMATCH                (GLUSTERD_COMP_BASE + 433)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XLATOR_VOLOPT_DYNLOAD_ERROR           (GLUSTERD_COMP_BASE + 434)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLNAME_NOTFOUND_IN_DICT              (GLUSTERD_COMP_BASE + 435)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FLAGS_NOTFOUND_IN_DICT                (GLUSTERD_COMP_BASE + 436)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedactio
- *
- */
-#define GD_MSG_HOSTNAME_NOTFOUND_IN_DICT             (GLUSTERD_COMP_BASE + 437)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PORT_NOTFOUND_IN_DICT                  (GLUSTERD_COMP_BASE + 438)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CMDSTR_NOTFOUND_IN_DICT                (GLUSTERD_COMP_BASE + 439)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_OBJ_NEW_FAIL                    (GLUSTERD_COMP_BASE + 440)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_BACKEND_MAKE_FAIL               (GLUSTERD_COMP_BASE + 441)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_CLONE_FAILED                     (GLUSTERD_COMP_BASE + 442)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_CLONE_PREVAL_FAILED              (GLUSTERD_COMP_BASE + 443)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_CLONE_POSTVAL_FAILED             (GLUSTERD_COMP_BASE + 444)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLINFO_STORE_FAIL                   (GLUSTERD_COMP_BASE + 445)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NEW_FRIEND_SM_EVENT_GET_FAIL         (GLUSTERD_COMP_BASE + 446)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_TYPE_CHANGING_INFO               (GLUSTERD_COMP_BASE + 447)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRKPATH_MNTPNT_MISMATCH               (GLUSTERD_COMP_BASE + 448)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TASKS_COUNT_MISMATCH                  (GLUSTERD_COMP_BASE + 449)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_WRONG_OPTS_SETTING                    (GLUSTERD_COMP_BASE + 450)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PATH_ALREADY_PART_OF_VOL               (GLUSTERD_COMP_BASE + 451)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_VALIDATE_FAIL                    (GLUSTERD_COMP_BASE + 452)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_READIN_FILE_FAILED                     (GLUSTERD_COMP_BASE + 453)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_IMPORT_PRDICT_DICT                     (GLUSTERD_COMP_BASE + 454)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_OPTS_IMPORT_FAIL                   (GLUSTERD_COMP_BASE + 455)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_IMPORT_FAIL                      (GLUSTERD_COMP_BASE + 456)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLINFO_IMPORT_FAIL                    (GLUSTERD_COMP_BASE + 457)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRICK_ID_GEN_FAILED                    (GLUSTERD_COMP_BASE + 458)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GET_STATUS_DATA_FAIL                   (GLUSTERD_COMP_BASE + 459)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BITROT_NOT_RUNNING                     (GLUSTERD_COMP_BASE + 460)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SCRUBBER_NOT_RUNNING                   (GLUSTERD_COMP_BASE + 461)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SRC_BRICK_PORT_UNAVAIL                 (GLUSTERD_COMP_BASE + 462)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BITD_INIT_FAIL                         (GLUSTERD_COMP_BASE + 463)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SCRUB_INIT_FAIL                        (GLUSTERD_COMP_BASE + 464)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VAR_RUN_DIR_INIT_FAIL                  (GLUSTERD_COMP_BASE + 465)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VAR_RUN_DIR_FIND_FAIL                  (GLUSTERD_COMP_BASE + 466)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SCRUBSVC_RECONF_FAIL                  (GLUSTERD_COMP_BASE + 467)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BITDSVC_RECONF_FAIL                   (GLUSTERD_COMP_BASE + 468)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_GNS_START_FAIL                    (GLUSTERD_COMP_BASE + 469)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_GNS_SETUP_FAIL                    (GLUSTERD_COMP_BASE + 470)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNRECOGNIZED_SVC_MNGR                 (GLUSTERD_COMP_BASE + 471)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_GNS_OP_HANDLE_FAIL                 (GLUSTERD_COMP_BASE + 472)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_EXPORT_FILE_CREATE_FAIL                (GLUSTERD_COMP_BASE + 473)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_GNS_HOST_FOUND                     (GLUSTERD_COMP_BASE + 474)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REBALANCE_CMD_IN_TIER_VOL              (GLUSTERD_COMP_BASE + 475)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INCOMPATIBLE_VALUE                     (GLUSTERD_COMP_BASE + 476)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GENERATED_UUID                        (GLUSTERD_COMP_BASE + 477)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FILE_DESC_LIMIT_SET                   (GLUSTERD_COMP_BASE + 478)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CURR_WORK_DIR_INFO                   (GLUSTERD_COMP_BASE + 479)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STRIPE_COUNT_CHANGE_INFO              (GLUSTERD_COMP_BASE + 480)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REPLICA_COUNT_CHANGE_INFO             (GLUSTERD_COMP_BASE + 481)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ADD_BRICK_REQ_RECVD                   (GLUSTERD_COMP_BASE + 482)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_ALREADY_TIER                      (GLUSTERD_COMP_BASE + 483)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REM_BRICK_REQ_RECVD                   (GLUSTERD_COMP_BASE + 484)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_NOT_TIER                         (GLUSTERD_COMP_BASE + 485)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOG_ROTATE_REQ_RECVD                 (GLUSTERD_COMP_BASE + 486)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CLI_REQ_RECVD                        (GLUSTERD_COMP_BASE + 487)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GET_VOL_REQ_RCVD                     (GLUSTERD_COMP_BASE + 488)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_SYNC_REQ_RCVD                     (GLUSTERD_COMP_BASE + 489)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PROBE_RCVD                            (GLUSTERD_COMP_BASE + 490)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UNFRIEND_REQ_RCVD                     (GLUSTERD_COMP_BASE + 491)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FRIEND_UPDATE_RCVD                   (GLUSTERD_COMP_BASE + 492)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RESPONSE_INFO                         (GLUSTERD_COMP_BASE + 493)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_PROFILE_REQ_RCVD                  (GLUSTERD_COMP_BASE + 494)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GETWD_REQ_RCVD                       (GLUSTERD_COMP_BASE + 495)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MOUNT_REQ_RCVD                       (GLUSTERD_COMP_BASE + 496)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UMOUNT_REQ_RCVD                       (GLUSTERD_COMP_BASE + 497)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CONNECT_RETURNED                       (GLUSTERD_COMP_BASE + 498)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATUS_VOL_REQ_RCVD                    (GLUSTERD_COMP_BASE + 499)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CLRCLK_VOL_REQ_RCVD                    (GLUSTERD_COMP_BASE + 500)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BARRIER_VOL_REQ_RCVD                   (GLUSTERD_COMP_BASE + 501)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_UUID_RECEIVED                          (GLUSTERD_COMP_BASE + 502)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REPLACE_BRK_COMMIT_FORCE_REQ_RCVD      (GLUSTERD_COMP_BASE + 503)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BRK_PORT_NO_ADD_INDO                   (GLUSTERD_COMP_BASE + 504)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REPLACE_BRK_REQ_RCVD                   (GLUSTERD_COMP_BASE + 505)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ADD_OP_ARGS_FAIL                      (GLUSTERD_COMP_BASE + 506)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_POST_HOOK_STUB_INIT_FAIL              (GLUSTERD_COMP_BASE + 507)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HOOK_STUB_NULL                       (GLUSTERD_COMP_BASE + 508)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SPAWN_THREADS_FAIL                   (GLUSTERD_COMP_BASE + 509)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STALE_VOL_DELETE_INFO                (GLUSTERD_COMP_BASE + 510)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PROBE_REQ_RESP_RCVD                  (GLUSTERD_COMP_BASE + 511)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HOST_PRESENT_ALREADY                  (GLUSTERD_COMP_BASE + 512)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_VERS_INFO                          (GLUSTERD_COMP_BASE + 513)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_VERS_SET_INFO                      (GLUSTERD_COMP_BASE + 514)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NEW_NODE_STATE_CREATION               (GLUSTERD_COMP_BASE + 515)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ALREADY_MOUNTED                      (GLUSTERD_COMP_BASE + 516)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SHARED_STRG_VOL_OPT_VALIDATE_FAIL     (GLUSTERD_COMP_BASE + 517)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_GNS_STOP_FAIL                     (GLUSTERD_COMP_BASE + 518)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_GNS_RESET_FAIL                     (GLUSTERD_COMP_BASE + 519)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SHARED_STRG_SET_FAIL                   (GLUSTERD_COMP_BASE + 520)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_TRANSPORT_TYPE_CHANGE             (GLUSTERD_COMP_BASE + 521)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PEER_COUNT_GET_FAIL                   (GLUSTERD_COMP_BASE + 522)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_INSUFFICIENT_UP_NODES                (GLUSTERD_COMP_BASE + 523)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_STAGE_STATS_VOL_FAIL              (GLUSTERD_COMP_BASE + 524)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOL_ID_SET_FAIL                     (GLUSTERD_COMP_BASE + 525)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_STAGE_RESET_VOL_FAIL                (GLUSTERD_COMP_BASE + 526)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_STAGE_BITROT_FAIL                  (GLUSTERD_COMP_BASE + 527)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_STAGE_QUOTA_FAIL                    (GLUSTERD_COMP_BASE + 528)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_STAGE_DELETE_VOL_FAIL               (GLUSTERD_COMP_BASE + 529)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HANDLE_HEAL_CMD_FAIL                   (GLUSTERD_COMP_BASE + 530)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_CLRCLK_SND_CMD_FAIL                   (GLUSTERD_COMP_BASE + 531)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DISPERSE_CLUSTER_FOUND               (GLUSTERD_COMP_BASE + 532)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_HEAL_VOL_REQ_RCVD                     (GLUSTERD_COMP_BASE + 533)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATEDUMP_VOL_REQ_RCVD               (GLUSTERD_COMP_BASE + 534)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_THINPOOLS_FOR_THINLVS                 (GLUSTERD_COMP_BASE + 535)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_STAGE_CREATE_VOL_FAIL              (GLUSTERD_COMP_BASE + 536)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_OP_STAGE_START_VOL_FAIL              (GLUSTERD_COMP_BASE + 537)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_NFS_GNS_UNEXPRT_VOL_FAIL              (GLUSTERD_COMP_BASE + 538)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_TASK_ID_INFO                          (GLUSTERD_COMP_BASE + 539)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DEREGISTER_SUCCESS                    (GLUSTERD_COMP_BASE + 540)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATEDUMP_OPTS_RCVD                    (GLUSTERD_COMP_BASE + 541)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_STATEDUMP_INFO                         (GLUSTERD_COMP_BASE + 542)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RECOVERING_CORRUPT_CONF                (GLUSTERD_COMP_BASE + 543)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_RETRIEVED_UUID                        (GLUSTERD_COMP_BASE + 544)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XLATOR_CREATE_FAIL                     (GLUSTERD_COMP_BASE + 545)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GRAPH_ENTRY_ADD_FAIL                   (GLUSTERD_COMP_BASE + 546)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ERROR_ENCOUNTERED                      (GLUSTERD_COMP_BASE + 547)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FILTER_RUN_FAILED                      (GLUSTERD_COMP_BASE + 548)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DEFAULT_OPT_INFO                       (GLUSTERD_COMP_BASE + 549)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MARKER_STATUS_GET_FAIL                 (GLUSTERD_COMP_BASE + 550)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_MARKER_DISABLE_FAIL                    (GLUSTERD_COMP_BASE + 551)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GRAPH_FEATURE_ADD_FAIL                 (GLUSTERD_COMP_BASE + 552)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XLATOR_SET_OPT_FAIL                   (GLUSTERD_COMP_BASE + 553)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_BUILD_GRAPH_FAILED                   (GLUSTERD_COMP_BASE + 554)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XML_TEXT_WRITE_FAIL                  (GLUSTERD_COMP_BASE + 555)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XML_DOC_START_FAIL                   (GLUSTERD_COMP_BASE + 556)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XML_ELE_CREATE_FAIL                  (GLUSTERD_COMP_BASE + 557)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_VOLUME_INCONSISTENCY                  (GLUSTERD_COMP_BASE + 558)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_XLATOR_LINK_FAIL                     (GLUSTERD_COMP_BASE + 559)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REMOTE_HOST_GET_FAIL                 (GLUSTERD_COMP_BASE + 560)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_GRAPH_SET_OPT_FAIL                    (GLUSTERD_COMP_BASE + 561)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ROOT_SQUASH_ENABLED                    (GLUSTERD_COMP_BASE + 562)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_ROOT_SQUASH_FAILED                     (GLUSTERD_COMP_BASE + 563)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_OWNER_MISMATCH                    (GLUSTERD_COMP_BASE + 564)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_NOT_HELD                          (GLUSTERD_COMP_BASE + 565)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_LOCK_ALREADY_HELD                      (GLUSTERD_COMP_BASE + 566)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SVC_START_SUCCESS                     (GLUSTERD_COMP_BASE + 567)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SVC_STOP_SUCCESS                     (GLUSTERD_COMP_BASE + 568)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_PARAM_NULL                           (GLUSTERD_COMP_BASE + 569)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SVC_STOP_FAIL                        (GLUSTERD_COMP_BASE + 570)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define GD_MSG_SHARED_STORAGE_DOES_NOT_EXIST        (GLUSTERD_COMP_BASE + 571)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define GD_MSG_SNAP_PAUSE_TIER_FAIL                 (GLUSTERD_COMP_BASE + 572)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SNAP_RESUME_TIER_FAIL                (GLUSTERD_COMP_BASE + 573)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_FILE_NOT_FOUND                       (GLUSTERD_COMP_BASE + 574)
-
-/*!
- * @messageid 106575
- * @diagnosis  Brick failed to start with given port, hence it gets a fresh port
- *             on its own and try to restart the brick with a new port
- * @recommendedaction  Ensure the new port is not blocked by firewall
- */
-#define GD_MSG_RETRY_WITH_NEW_PORT                   (GLUSTERD_COMP_BASE + 575)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_REMOTE_VOL_UUID_FAIL                (GLUSTERD_COMP_BASE + 576)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_SLAVE_VOL_PARSE_FAIL               (GLUSTERD_COMP_BASE + 577)
-
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define GD_MSG_DICT_GET_SUCCESS                   (GLUSTERD_COMP_BASE + 578)
-
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* !_GLUSTERD_MESSAGES_H_ */
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
index 5b7f0fa3c25..1069688a89d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
@@ -22,994 +22,1123 @@
 #include "glusterd-messages.h"
 
 static int
-glusterd_mgmt_v3_null (rpcsvc_request_t *req)
+glusterd_mgmt_v3_null(rpcsvc_request_t *req)
 {
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, int32_t status,
-                                 uint32_t op_errno)
+glusterd_mgmt_v3_lock_send_resp(rpcsvc_request_t *req, int32_t status,
+                                uint32_t op_errno)
 {
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = op_errno;
 
-        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
-        int                           ret   = -1;
-        xlator_t                     *this  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-                rsp.op_errno = op_errno;
-
-        glusterd_get_uuid (&rsp.uuid);
+    glusterd_get_uuid(&rsp.uuid);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
 
-        gf_msg_debug (this->name, 0,
-                "Responded to mgmt_v3 lock, ret: %d", ret);
+    gf_msg_debug(this->name, 0, "Responded to mgmt_v3 lock, ret: %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_synctasked_mgmt_v3_lock (rpcsvc_request_t *req,
-                                gd1_mgmt_v3_lock_req *lock_req,
-                                glusterd_op_lock_ctx_t *ctx)
+glusterd_synctasked_mgmt_v3_lock(rpcsvc_request_t *req,
+                                 gd1_mgmt_v3_lock_req *lock_req,
+                                 glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                         ret         = -1;
-        xlator_t                       *this        = NULL;
-        uint32_t                        op_errno    = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (ctx);
-        GF_ASSERT (ctx->dict);
-
-        /* Trying to acquire multiple mgmt_v3 locks */
-        ret = glusterd_multiple_mgmt_v3_lock (ctx->dict, ctx->uuid, &op_errno);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                        "Failed to acquire mgmt_v3 locks for %s",
-                         uuid_utoa (ctx->uuid));
-
-        ret = glusterd_mgmt_v3_lock_send_resp (req, ret, op_errno);
-
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(ctx);
+    GF_ASSERT(ctx->dict);
+
+    /* Trying to acquire multiple mgmt_v3 locks */
+    ret = glusterd_multiple_mgmt_v3_lock(ctx->dict, ctx->uuid, &op_errno);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+               "Failed to acquire mgmt_v3 locks for %s", uuid_utoa(ctx->uuid));
+
+    ret = glusterd_mgmt_v3_lock_send_resp(req, ret, op_errno);
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_state_machine_mgmt_v3_lock (rpcsvc_request_t *req,
+glusterd_op_state_machine_mgmt_v3_lock(rpcsvc_request_t *req,
                                        gd1_mgmt_v3_lock_req *lock_req,
                                        glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                         ret         = -1;
-        xlator_t                       *this        = NULL;
-        glusterd_op_info_t              txn_op_info = {{0},};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        glusterd_txn_opinfo_init (&txn_op_info, NULL, &lock_req->op, ctx->dict,
-                                  req);
-
-        ret = glusterd_set_txn_opinfo (&lock_req->txn_id, &txn_op_info);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OPINFO_SET_FAIL,
-                        "Unable to set transaction's opinfo");
-                goto out;
-        }
-
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK,
-                                           &lock_req->txn_id, ctx);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_EVENT_LOCK_FAIL,
-                        "Failed to inject event GD_OP_EVENT_LOCK");
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    glusterd_txn_opinfo_init(&txn_op_info, NULL, &lock_req->op, ctx->dict, req);
+
+    ret = glusterd_set_txn_opinfo(&lock_req->txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPINFO_SET_FAIL,
+               "Unable to set transaction's opinfo");
+        goto out;
+    }
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_LOCK, &lock_req->txn_id, ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_EVENT_LOCK_FAIL,
+               "Failed to inject event GD_OP_EVENT_LOCK");
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_mgmt_v3_lock_fn (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_lock_fn(rpcsvc_request_t *req)
 {
-        gd1_mgmt_v3_lock_req    lock_req        = {{0},};
-        int32_t                 ret             = -1;
-        glusterd_op_lock_ctx_t *ctx             = NULL;
-        xlator_t               *this            = NULL;
-        gf_boolean_t            is_synctasked   = _gf_false;
-        gf_boolean_t            free_ctx        = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &lock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode lock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "Received mgmt_v3 lock req "
-                "from uuid: %s", uuid_utoa (lock_req.uuid));
-
-        if (glusterd_peerinfo_find_by_uuid (lock_req.uuid) == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (lock_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
-        if (!ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (ctx->uuid, lock_req.uuid);
-        ctx->req = req;
-
-        ctx->dict = dict_new ();
-        if (!ctx->dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (lock_req.dict.dict_val,
-                                lock_req.dict.dict_len, &ctx->dict);
+    gd1_mgmt_v3_lock_req lock_req = {
+        {0},
+    };
+    int32_t ret = -1;
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_synctasked = _gf_false;
+    gf_boolean_t free_ctx = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    uint32_t timeout = 0;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &lock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode lock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Received mgmt_v3 lock req "
+                 "from uuid: %s",
+                 uuid_utoa(lock_req.uuid));
+
+    if (glusterd_peerinfo_find_by_uuid(lock_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(lock_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
+    if (!ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(ctx->uuid, lock_req.uuid);
+    ctx->req = req;
+
+    ctx->dict = dict_new();
+    if (!ctx->dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(lock_req.dict.dict_val, lock_req.dict.dict_len,
+                           &ctx->dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    /* Cli will add timeout key to dict if the default timeout is
+     * other than 2 minutes. Here we use this value to check whether
+     * mgmt_v3_lock_timeout should be set to default value or we
+     * need to change the value according to timeout value
+     * i.e, timeout + 120 seconds. */
+    ret = dict_get_uint32(ctx->dict, "timeout", &timeout);
+    if (!ret)
+        conf->mgmt_v3_lock_timeout = timeout + 120;
+
+    is_synctasked = dict_get_str_boolean(ctx->dict, "is_synctasked", _gf_false);
+    if (is_synctasked) {
+        ret = glusterd_synctasked_mgmt_v3_lock(req, &lock_req, ctx);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize the dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire mgmt_v3_locks");
+            /* Ignore the return code, as it shouldn't be propagated
+             * from the handler function so as to avoid double
+             * deletion of the req
+             */
+            ret = 0;
         }
 
-        is_synctasked = dict_get_str_boolean (ctx->dict,
-                                              "is_synctasked", _gf_false);
-        if (is_synctasked) {
-                ret = glusterd_synctasked_mgmt_v3_lock (req, &lock_req, ctx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                                "Failed to acquire mgmt_v3_locks");
-                        /* Ignore the return code, as it shouldn't be propagated
-                         * from the handler function so as to avoid double
-                         * deletion of the req
-                         */
-                        ret = 0;
-                }
-
-                /* The above function does not take ownership of ctx.
-                 * Therefore we need to free the ctx explicitly. */
-                free_ctx = _gf_true;
-        }
-        else {
-                /* Shouldn't ignore the return code here, and it should
-                 * be propagated from the handler function as in failure
-                 * case it doesn't delete the req object
-                 */
-                ret = glusterd_op_state_machine_mgmt_v3_lock (req, &lock_req,
-                                                              ctx);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                                "Failed to acquire mgmt_v3_locks");
-        }
+        /* The above function does not take ownership of ctx.
+         * Therefore we need to free the ctx explicitly. */
+        free_ctx = _gf_true;
+    } else {
+        /* Shouldn't ignore the return code here, and it should
+         * be propagated from the handler function as in failure
+         * case it doesn't delete the req object
+         */
+        ret = glusterd_op_state_machine_mgmt_v3_lock(req, &lock_req, ctx);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire mgmt_v3_locks");
+    }
 
 out:
 
-        if (ctx && (ret || free_ctx)) {
-                if (ctx->dict)
-                        dict_unref (ctx->dict);
+    if (ctx && (ret || free_ctx)) {
+        if (ctx->dict)
+            dict_unref(ctx->dict);
 
-                GF_FREE (ctx);
-        }
+        GF_FREE(ctx);
+    }
 
-        free (lock_req.dict.dict_val);
+    free(lock_req.dict.dict_val);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_mgmt_v3_pre_validate_send_resp (rpcsvc_request_t   *req,
-                                         int32_t op, int32_t status,
-                                         char *op_errstr, dict_t *rsp_dict,
-                                         uint32_t op_errno)
+glusterd_mgmt_v3_pre_validate_send_resp(rpcsvc_request_t *req, int32_t op,
+                                        int32_t status, char *op_errstr,
+                                        dict_t *rsp_dict, uint32_t op_errno)
 {
-        gd1_mgmt_v3_pre_val_rsp          rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        rsp.op_errno = op_errno;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_pre_val_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    rsp.op_errno = op_errno;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_msg_debug (this->name, 0,
-                "Responded to pre validation, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to pre validation, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_pre_validate_fn (rpcsvc_request_t *req)
+glusterd_handle_pre_validate_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_pre_val_req         op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-        uint32_t                        op_errno  = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL,
-                        "Failed to decode pre validation "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        ret = gd_mgmt_v3_pre_validate_fn (op_req.op, dict, &op_errstr,
-                                          rsp_dict, &op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL,
-                        "Pre Validation failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
-
-        ret = glusterd_mgmt_v3_pre_validate_send_resp (req, op_req.op,
-                                                       ret, op_errstr,
-                                                       rsp_dict, op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_OP_RESP_FAIL,
-                        "Failed to send Pre Validation "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_pre_val_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode pre validation "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_pre_validate_fn(op_req.op, dict, &op_errstr, rsp_dict,
+                                     &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_pre_validate_send_resp(
+        req, op_req.op, ret, op_errstr, rsp_dict, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send Pre Validation "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_brick_op_send_resp (rpcsvc_request_t   *req,
-                                    int32_t op, int32_t status,
-                                    char *op_errstr, dict_t *rsp_dict)
+glusterd_mgmt_v3_brick_op_send_resp(rpcsvc_request_t *req, int32_t op,
+                                    int32_t status, char *op_errstr,
+                                    dict_t *rsp_dict)
 {
-        gd1_mgmt_v3_brick_op_rsp         rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_brick_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_msg_debug (this->name, 0,
-                "Responded to brick op, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to brick op, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_brick_op_fn (rpcsvc_request_t *req)
+glusterd_handle_brick_op_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_brick_op_req         op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode brick op "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        ret = gd_mgmt_v3_brick_op_fn (op_req.op, dict, &op_errstr,
-                                     rsp_dict);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_OP_FAIL,
-                        "Brick Op failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
-
-        ret = glusterd_mgmt_v3_brick_op_send_resp (req, op_req.op,
-                                                  ret, op_errstr,
-                                                  rsp_dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALD_RESP_FAIL,
-                        "Failed to send brick op "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_brick_op_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode brick op "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_brick_op_fn(op_req.op, dict, &op_errstr, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick Op failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_brick_op_send_resp(req, op_req.op, ret, op_errstr,
+                                              rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALD_RESP_FAIL,
+               "Failed to send brick op "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_commit_send_resp (rpcsvc_request_t   *req,
-                                   int32_t op, int32_t status,
-                                   char *op_errstr, uint32_t op_errno,
-                                   dict_t *rsp_dict)
+glusterd_mgmt_v3_commit_send_resp(rpcsvc_request_t *req, int32_t op,
+                                  int32_t status, char *op_errstr,
+                                  uint32_t op_errno, dict_t *rsp_dict)
 {
-        gd1_mgmt_v3_commit_rsp           rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        rsp.op_errno = op_errno;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_commit_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    rsp.op_errno = op_errno;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_msg_debug (this->name, 0, "Responded to commit, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to commit, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_commit_fn (rpcsvc_request_t *req)
+glusterd_handle_commit_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_commit_req           op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-        uint32_t                        op_errno  = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode commit "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_commit_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode commit "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_commit_fn(op_req.op, dict, &op_errstr, &op_errno,
+                               rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "commit failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_commit_send_resp(req, op_req.op, ret, op_errstr,
+                                            op_errno, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send commit "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_msg  (this->name, GF_LOG_WARNING, 0,
-                         GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
+out:
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    free(op_req.dict.dict_val);
 
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
+    if (dict)
+        dict_unref(dict);
 
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to get new dictionary");
-                return -1;
-        }
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        ret = gd_mgmt_v3_commit_fn (op_req.op, dict, &op_errstr,
-                                    &op_errno, rsp_dict);
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
+}
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL,
-                        "commit failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
+static int
+glusterd_mgmt_v3_post_commit_send_resp(rpcsvc_request_t *req, int32_t op,
+                                       int32_t status, char *op_errstr,
+                                       uint32_t op_errno, dict_t *rsp_dict)
+{
+    gd1_mgmt_v3_post_commit_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    rsp.op_errno = op_errno;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
+out:
+    gf_msg_debug(this->name, 0, "Responded to post commit, ret: %d", ret);
+    return ret;
+}
 
-        ret = glusterd_mgmt_v3_commit_send_resp (req, op_req.op,
-                                                 ret, op_errstr,
+static int
+glusterd_handle_post_commit_fn(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_commit_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode post commit "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_post_commit_fn(op_req.op, dict, &op_errstr, &op_errno,
+                                    rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "post commit failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_post_commit_send_resp(req, op_req.op, ret, op_errstr,
                                                  op_errno, rsp_dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_OP_RESP_FAIL,
-                        "Failed to send commit "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send post commit "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_post_validate_send_resp (rpcsvc_request_t   *req,
-                                         int32_t op, int32_t status,
-                                         char *op_errstr, dict_t *rsp_dict)
+glusterd_mgmt_v3_post_validate_send_resp(rpcsvc_request_t *req, int32_t op,
+                                         int32_t status, char *op_errstr,
+                                         dict_t *rsp_dict)
 {
-        gd1_mgmt_v3_post_val_rsp         rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_post_val_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_msg_debug (this->name, 0,
-                "Responded to post validation, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to post validation, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_post_validate_fn (rpcsvc_request_t *req)
+glusterd_handle_post_validate_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_post_val_req         op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL,
-                        "Failed to decode post validation "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        ret = gd_mgmt_v3_post_validate_fn (op_req.op, op_req.op_ret, dict,
-                                           &op_errstr, rsp_dict);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_POST_VALIDATION_FAIL,
-                        "Post Validation failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
-
-        ret = glusterd_mgmt_v3_post_validate_send_resp (req, op_req.op,
-                                                       ret, op_errstr,
-                                                       rsp_dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_OP_RESP_FAIL,
-                        "Failed to send Post Validation "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_val_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode post validation "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_post_validate_fn(op_req.op, op_req.op_ret, dict,
+                                      &op_errstr, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_post_validate_send_resp(req, op_req.op, ret,
+                                                   op_errstr, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send Post Validation "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+glusterd_mgmt_v3_unlock_send_resp(rpcsvc_request_t *req, int32_t status)
 {
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        gd1_mgmt_v3_unlock_rsp          rsp   = {{0},};
-        int                             ret   = -1;
-        xlator_t                       *this  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
 
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-                rsp.op_errno = errno;
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = errno;
 
-        glusterd_get_uuid (&rsp.uuid);
+    glusterd_get_uuid(&rsp.uuid);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
 
-        gf_msg_debug (this->name, 0,
-                "Responded to mgmt_v3 unlock, ret: %d", ret);
+    gf_msg_debug(this->name, 0, "Responded to mgmt_v3 unlock, ret: %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_syctasked_mgmt_v3_unlock (rpcsvc_request_t *req,
+glusterd_syctasked_mgmt_v3_unlock(rpcsvc_request_t *req,
                                   gd1_mgmt_v3_unlock_req *unlock_req,
                                   glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                         ret         = -1;
-        xlator_t                       *this        = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (ctx);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(ctx);
 
-        /* Trying to release multiple mgmt_v3 locks */
-        ret = glusterd_multiple_mgmt_v3_unlock (ctx->dict, ctx->uuid);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_UNLOCK_FAIL,
-                        "Failed to release mgmt_v3 locks for %s",
-                        uuid_utoa(ctx->uuid));
-        }
+    /* Trying to release multiple mgmt_v3 locks */
+    ret = glusterd_multiple_mgmt_v3_unlock(ctx->dict, ctx->uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+               "Failed to release mgmt_v3 locks for %s", uuid_utoa(ctx->uuid));
+    }
 
-        ret = glusterd_mgmt_v3_unlock_send_resp (req, ret);
+    ret = glusterd_mgmt_v3_unlock_send_resp(req, ret);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 static int
-glusterd_op_state_machine_mgmt_v3_unlock (rpcsvc_request_t *req,
+glusterd_op_state_machine_mgmt_v3_unlock(rpcsvc_request_t *req,
                                          gd1_mgmt_v3_unlock_req *lock_req,
                                          glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                           ret      = -1;
-        xlator_t                         *this     = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK,
-                                           &lock_req->txn_id, ctx);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_EVENT_UNLOCK_FAIL,
-                        "Failed to inject event GD_OP_EVENT_UNLOCK");
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_UNLOCK, &lock_req->txn_id,
+                                      ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_EVENT_UNLOCK_FAIL,
+               "Failed to inject event GD_OP_EVENT_UNLOCK");
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_mgmt_v3_unlock_fn (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_unlock_fn(rpcsvc_request_t *req)
 {
-        gd1_mgmt_v3_unlock_req  lock_req        = {{0},};
-        int32_t                 ret             = -1;
-        glusterd_op_lock_ctx_t *ctx             = NULL;
-        xlator_t               *this            = NULL;
-        gf_boolean_t            is_synctasked   = _gf_false;
-        gf_boolean_t            free_ctx        = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &lock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode unlock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "Received volume unlock req "
-                "from uuid: %s", uuid_utoa (lock_req.uuid));
-
-        if (glusterd_peerinfo_find_by_uuid (lock_req.uuid) == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_PEER_NOT_FOUND, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (lock_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
-        if (!ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (ctx->uuid, lock_req.uuid);
-        ctx->req = req;
-
-        ctx->dict = dict_new ();
-        if (!ctx->dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (lock_req.dict.dict_val,
-                                lock_req.dict.dict_len, &ctx->dict);
+    gd1_mgmt_v3_unlock_req lock_req = {
+        {0},
+    };
+    int32_t ret = -1;
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_synctasked = _gf_false;
+    gf_boolean_t free_ctx = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &lock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode unlock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Received volume unlock req "
+                 "from uuid: %s",
+                 uuid_utoa(lock_req.uuid));
+
+    if (glusterd_peerinfo_find_by_uuid(lock_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(lock_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
+    if (!ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(ctx->uuid, lock_req.uuid);
+    ctx->req = req;
+
+    ctx->dict = dict_new();
+    if (!ctx->dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(lock_req.dict.dict_val, lock_req.dict.dict_len,
+                           &ctx->dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    is_synctasked = dict_get_str_boolean(ctx->dict, "is_synctasked", _gf_false);
+    if (is_synctasked) {
+        ret = glusterd_syctasked_mgmt_v3_unlock(req, &lock_req, ctx);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_UNSERIALIZE_FAIL,
-                        "failed to unserialize the dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3_locks");
+            /* Ignore the return code, as it shouldn't be propagated
+             * from the handler function so as to avoid double
+             * deletion of the req
+             */
+            ret = 0;
         }
 
-        is_synctasked = dict_get_str_boolean (ctx->dict,
-                                              "is_synctasked", _gf_false);
-        if (is_synctasked) {
-                ret = glusterd_syctasked_mgmt_v3_unlock (req, &lock_req, ctx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                "Failed to release mgmt_v3_locks");
-                        /* Ignore the return code, as it shouldn't be propagated
-                         * from the handler function so as to avoid double
-                         * deletion of the req
-                         */
-                        ret = 0;
-                }
-
-                /* The above function does not take ownership of ctx.
-                 * Therefore we need to free the ctx explicitly. */
-                free_ctx = _gf_true;
-        }
-        else {
-                /* Shouldn't ignore the return code here, and it should
-                 * be propagated from the handler function as in failure
-                 * case it doesn't delete the req object
-                 */
-                ret = glusterd_op_state_machine_mgmt_v3_unlock (req, &lock_req,
-                                                                ctx);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                "Failed to release mgmt_v3_locks");
-        }
+        /* The above function does not take ownership of ctx.
+         * Therefore we need to free the ctx explicitly. */
+        free_ctx = _gf_true;
+    } else {
+        /* Shouldn't ignore the return code here, and it should
+         * be propagated from the handler function as in failure
+         * case it doesn't delete the req object
+         */
+        ret = glusterd_op_state_machine_mgmt_v3_unlock(req, &lock_req, ctx);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3_locks");
+    }
 
 out:
 
-        if (ctx && (ret || free_ctx)) {
-                if (ctx->dict)
-                        dict_unref (ctx->dict);
+    if (ctx && (ret || free_ctx)) {
+        if (ctx->dict)
+            dict_unref(ctx->dict);
 
-                GF_FREE (ctx);
-        }
+        GF_FREE(ctx);
+    }
 
-        free (lock_req.dict.dict_val);
+    free(lock_req.dict.dict_val);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_handle_mgmt_v3_lock (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_lock(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req, glusterd_handle_mgmt_v3_lock_fn);
+}
+
+static int
+glusterd_handle_pre_validate(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_mgmt_v3_lock_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_pre_validate_fn);
 }
 
 static int
-glusterd_handle_pre_validate (rpcsvc_request_t *req)
+glusterd_handle_brick_op(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_pre_validate_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_brick_op_fn);
 }
 
 static int
-glusterd_handle_brick_op (rpcsvc_request_t *req)
+glusterd_handle_commit(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_brick_op_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_commit_fn);
 }
 
 static int
-glusterd_handle_commit (rpcsvc_request_t *req)
+glusterd_handle_post_commit(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_commit_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_post_commit_fn);
 }
 
 static int
-glusterd_handle_post_validate (rpcsvc_request_t *req)
+glusterd_handle_post_validate(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_post_validate_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_post_validate_fn);
 }
 
 int
-glusterd_handle_mgmt_v3_unlock (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_unlock(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_mgmt_v3_unlock_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_mgmt_v3_unlock_fn);
 }
 
-rpcsvc_actor_t gd_svc_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
-        [GLUSTERD_MGMT_V3_NULL]          = { "NULL",           GLUSTERD_MGMT_V3_NULL,          glusterd_mgmt_v3_null,         NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_LOCK]          = { "MGMT_V3_LOCK",   GLUSTERD_MGMT_V3_LOCK,          glusterd_handle_mgmt_v3_lock,   NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_PRE_VALIDATE]  = { "PRE_VAL",        GLUSTERD_MGMT_V3_PRE_VALIDATE,  glusterd_handle_pre_validate,  NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_BRICK_OP]      = { "BRCK_OP",        GLUSTERD_MGMT_V3_BRICK_OP,      glusterd_handle_brick_op,      NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_COMMIT]        = { "COMMIT",         GLUSTERD_MGMT_V3_COMMIT,        glusterd_handle_commit,        NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_POST_VALIDATE] = { "POST_VAL",       GLUSTERD_MGMT_V3_POST_VALIDATE, glusterd_handle_post_validate, NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_UNLOCK]        = { "MGMT_V3_UNLOCK", GLUSTERD_MGMT_V3_UNLOCK,        glusterd_handle_mgmt_v3_unlock, NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
+    [GLUSTERD_MGMT_V3_NULL] = {"NULL", glusterd_mgmt_v3_null, NULL,
+                               GLUSTERD_MGMT_V3_NULL, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_LOCK] = {"MGMT_V3_LOCK", glusterd_handle_mgmt_v3_lock,
+                               NULL, GLUSTERD_MGMT_V3_LOCK, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_PRE_VALIDATE] = {"PRE_VAL", glusterd_handle_pre_validate,
+                                       NULL, GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                       DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_BRICK_OP] = {"BRCK_OP", glusterd_handle_brick_op, NULL,
+                                   GLUSTERD_MGMT_V3_BRICK_OP, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_COMMIT] = {"COMMIT", glusterd_handle_commit, NULL,
+                                 GLUSTERD_MGMT_V3_COMMIT, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_POST_COMMIT] = {"POST_COMMIT",
+                                      glusterd_handle_post_commit, NULL,
+                                      GLUSTERD_MGMT_V3_POST_COMMIT, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_POST_VALIDATE] = {"POST_VAL",
+                                        glusterd_handle_post_validate, NULL,
+                                        GLUSTERD_MGMT_V3_POST_VALIDATE, DRC_NA,
+                                        0},
+    [GLUSTERD_MGMT_V3_UNLOCK] = {"MGMT_V3_UNLOCK",
+                                 glusterd_handle_mgmt_v3_unlock, NULL,
+                                 GLUSTERD_MGMT_V3_UNLOCK, DRC_NA, 0},
 };
 
 struct rpcsvc_program gd_svc_mgmt_v3_prog = {
-        .progname  = "GlusterD svc mgmt v3",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_V3_VERSION,
-        .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
-        .actors    = gd_svc_mgmt_v3_actors,
-        .synctask  = _gf_true,
+    .progname = "GlusterD svc mgmt v3",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_V3_VERSION,
+    .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
+    .actors = gd_svc_mgmt_v3_actors,
+    .synctask = _gf_true,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
index 092283a7daf..bca7221062b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -19,6 +19,7 @@
 #include "glusterd-locks.h"
 #include "glusterd-mgmt.h"
 #include "glusterd-op-sm.h"
+#include "glusterd-server-quorum.h"
 #include "glusterd-volgen.h"
 #include "glusterd-store.h"
 #include "glusterd-snapshot-utils.h"
@@ -28,2384 +29,3086 @@
 
 extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
-
 void
-gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
-                           char *op_errstr, int op_code, uuid_t peerid,
-                           u_char *uuid)
+gd_mgmt_v3_collate_errors(struct syncargs *args, int op_ret, int op_errno,
+                          char *op_errstr, int op_code, uuid_t peerid,
+                          u_char *uuid)
 {
-        char      *peer_str          = NULL;
-        char       err_str[PATH_MAX] = "Please check log file for details.";
-        char       op_err[PATH_MAX]  = "";
-        int32_t    len               = -1;
-        xlator_t  *this              = NULL;
-        int        is_operrstr_blk   = 0;
-        char       *err_string       = NULL;
-        char       *cli_err_str      = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (args);
-        GF_ASSERT (uuid);
-
-        if (op_ret) {
-                args->op_ret = op_ret;
-                args->op_errno = op_errno;
-
-                rcu_read_lock ();
-                peerinfo = glusterd_peerinfo_find (peerid, NULL);
-                if (peerinfo)
-                        peer_str = gf_strdup (peerinfo->hostname);
-                else
-                        peer_str = gf_strdup (uuid_utoa (uuid));
-
-                rcu_read_unlock ();
-
-                is_operrstr_blk = (op_errstr && strcmp (op_errstr, ""));
-                err_string     = (is_operrstr_blk) ? op_errstr : err_str;
-
-                switch (op_code) {
-                case GLUSTERD_MGMT_V3_LOCK:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Locking failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_PRE_VALIDATE:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Pre Validation failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_BRICK_OP:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Brick ops failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_COMMIT:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Commit failed"
-                                                " on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_POST_VALIDATE:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Post Validation failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_UNLOCK:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Unlocking failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                default :
-                        len = snprintf (op_err, sizeof(op_err),
-                                       "Unknown error! "
-                                       "on %s. %s", peer_str,
-                                        err_string);
-                }
-
-                if (args->errstr) {
-                        len = snprintf (err_str, sizeof(err_str),
-                                      "%s\n%s", args->errstr,
-                                      op_err);
-                        GF_FREE (args->errstr);
-                        args->errstr = NULL;
-                } else
-                        len = snprintf (err_str, sizeof(err_str),
-                                "%s", op_err);
-
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_OP_FAIL, "%s", op_err);
-                args->errstr = gf_strdup (err_str);
-        }
-
-        GF_FREE (peer_str);
-
-        return;
+    char *peer_str = NULL;
+    char err_str[PATH_MAX] = "Please check log file for details.";
+    char op_err[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    int is_operrstr_blk = 0;
+    char *err_string = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(args);
+    GF_ASSERT(uuid);
+
+    if (op_ret) {
+        args->op_ret = op_ret;
+        args->op_errno = op_errno;
+
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find(peerid, NULL);
+        if (peerinfo)
+            peer_str = gf_strdup(peerinfo->hostname);
+        else
+            peer_str = gf_strdup(uuid_utoa(uuid));
+
+        RCU_READ_UNLOCK;
+
+        is_operrstr_blk = (op_errstr && strcmp(op_errstr, ""));
+        err_string = (is_operrstr_blk) ? op_errstr : err_str;
+
+        switch (op_code) {
+            case GLUSTERD_MGMT_V3_LOCK: {
+                snprintf(op_err, sizeof(op_err), "Locking failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_PRE_VALIDATE: {
+                snprintf(op_err, sizeof(op_err),
+                         "Pre Validation failed on %s. %s", peer_str,
+                         err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_BRICK_OP: {
+                snprintf(op_err, sizeof(op_err), "Brick ops failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_COMMIT: {
+                snprintf(op_err, sizeof(op_err), "Commit failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_POST_COMMIT: {
+                snprintf(op_err, sizeof(op_err), "Post commit failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_POST_VALIDATE: {
+                snprintf(op_err, sizeof(op_err),
+                         "Post Validation failed on %s. %s", peer_str,
+                         err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_UNLOCK: {
+                snprintf(op_err, sizeof(op_err), "Unlocking failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            default:
+                snprintf(op_err, sizeof(op_err), "Unknown error! on %s. %s",
+                         peer_str, err_string);
+        }
+
+        if (args->errstr) {
+            len = snprintf(err_str, sizeof(err_str), "%s\n%s", args->errstr,
+                           op_err);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            GF_FREE(args->errstr);
+            args->errstr = NULL;
+        } else
+            snprintf(err_str, sizeof(err_str), "%s", op_err);
+
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_FAIL, "%s",
+               op_err);
+        args->errstr = gf_strdup(err_str);
+    }
+
+    GF_FREE(peer_str);
+
+    return;
 }
 
 int32_t
-gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
-                            char **op_errstr, dict_t *rsp_dict,
-                            uint32_t *op_errno)
+gd_mgmt_v3_pre_validate_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict, uint32_t *op_errno)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_SNAP:
-                ret = glusterd_snapshot_prevalidate (dict, op_errstr,
-                                                     rsp_dict, op_errno);
+            ret = glusterd_snapshot_prevalidate(dict, op_errstr, rsp_dict,
+                                                op_errno);
 
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_PRE_VALIDATION_FAIL,
-                                "Snapshot Prevalidate Failed");
-                        goto out;
-                }
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Snapshot Prevalidate Failed");
+                goto out;
+            }
 
-                break;
+            break;
 
         case GD_OP_REPLACE_BRICK:
-                ret = glusterd_op_stage_replace_brick (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_PRE_VALIDATION_FAIL,
-                                "Replace-brick prevalidation failed.");
-                        goto out;
-                }
-                break;
+            ret = glusterd_op_stage_replace_brick(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Replace-brick prevalidation failed.");
+                goto out;
+            }
+            break;
         case GD_OP_ADD_BRICK:
-                ret = glusterd_op_stage_add_brick (dict, op_errstr, rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_PRE_VALIDATION_FAIL,
-                                "ADD-brick prevalidation failed.");
-                        goto out;
-                }
-                break;
+            ret = glusterd_op_stage_add_brick(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "ADD-brick prevalidation failed.");
+                goto out;
+            }
+            break;
         case GD_OP_START_VOLUME:
-                ret = glusterd_op_stage_start_volume (dict, op_errstr,
-                                                      rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_PRE_VALIDATION_FAIL,
-                                "Volume start prevalidation failed.");
-                        goto out;
-                }
-                break;
+            ret = glusterd_op_stage_start_volume(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Volume start prevalidation failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_STOP_VOLUME:
+            ret = glusterd_op_stage_stop_volume(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Volume stop prevalidation failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_REMOVE_BRICK:
+            ret = glusterd_op_stage_remove_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Remove brick prevalidation failed.");
+                goto out;
+            }
+            break;
+
+        case GD_OP_RESET_BRICK:
+            ret = glusterd_reset_brick_prevalidate(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Reset brick prevalidation failed.");
+                goto out;
+            }
+            break;
+
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_op_stage_stats_volume(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "prevalidation failed for profile operation.");
+                goto out;
+            }
+            break;
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_mgmt_v3_op_stage_rebalance(dict, op_errstr);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Rebalance Prevalidate Failed");
+                goto out;
+            }
+            break;
+
+        case GD_OP_MAX_OPVERSION:
+            ret = 0;
+            break;
 
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
-                        char **op_errstr, dict_t *rsp_dict)
+gd_mgmt_v3_brick_op_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                       dict_t *rsp_dict)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        switch (op) {
-        case GD_OP_SNAP:
-        {
-                ret = glusterd_snapshot_brickop (dict, op_errstr, rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_BRICK_OP_FAIL,
-                                "snapshot brickop failed");
-                        goto out;
-                }
-                break;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    switch (op) {
+        case GD_OP_SNAP: {
+            ret = glusterd_snapshot_brickop(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_OP_FAIL,
+                       "snapshot brickop failed");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_PROFILE_VOLUME:
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME: {
+            ret = gd_brick_op_phase(op, rsp_dict, dict, op_errstr);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "%s brickop "
+                       "failed",
+                       gd_op_list[op]);
+                goto out;
+            }
+            break;
         }
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
-                      char **op_errstr, uint32_t *op_errno,
-                      dict_t *rsp_dict)
+gd_mgmt_v3_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                     uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-        GF_ASSERT (rsp_dict);
-
-        glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_PRE);
-        switch (op) {
-               case GD_OP_SNAP:
-               {
-                       ret = glusterd_snapshot (dict, op_errstr,
-                                                op_errno, rsp_dict);
-                       if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                               GD_MSG_COMMIT_OP_FAIL,
-                                               "Snapshot Commit Failed");
-                                goto out;
-                       }
-                       break;
-               }
-                case GD_OP_REPLACE_BRICK:
-                {
-                        ret = glusterd_op_replace_brick (dict, rsp_dict);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_COMMIT_OP_FAIL,
-                                        "Replace-brick commit failed.");
-                                goto out;
-                        }
-                        break;
-                }
-                case GD_OP_ADD_BRICK:
-                {
-                        ret = glusterd_op_add_brick (dict, op_errstr);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_COMMIT_OP_FAIL,
-                                        "Add-brick commit failed.");
-                                goto out;
-                        }
-                        break;
-
-                }
-                case GD_OP_START_VOLUME:
-                {
-                        ret = glusterd_op_start_volume (dict, op_errstr);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_COMMIT_OP_FAIL,
-                                        "Volume start commit failed.");
-                                goto out;
-                        }
-                        break;
-
-                }
-
-               default:
-                       break;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
+
+    glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_PRE);
+    switch (op) {
+        case GD_OP_SNAP: {
+            ret = glusterd_snapshot(dict, op_errstr, op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Snapshot Commit Failed");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_REPLACE_BRICK: {
+            ret = glusterd_op_replace_brick(dict, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Replace-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_ADD_BRICK: {
+            ret = glusterd_op_add_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Add-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_START_VOLUME: {
+            ret = glusterd_op_start_volume(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Volume start commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_STOP_VOLUME: {
+            ret = glusterd_op_stop_volume(dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Volume stop commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_REMOVE_BRICK: {
+            ret = glusterd_op_remove_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Remove-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_RESET_BRICK: {
+            ret = glusterd_op_reset_brick(dict, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Reset-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_MAX_OPVERSION: {
+            ret = glusterd_op_get_max_opversion(op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_PROFILE_VOLUME: {
+            ret = glusterd_op_stats_volume(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "commit failed for volume profile operation.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME: {
+            ret = glusterd_mgmt_v3_op_rebalance(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Rebalance Commit Failed");
+                goto out;
+            }
+            break;
         }
 
-        ret = 0;
+        default:
+            break;
+    }
+
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
-                             char **op_errstr, dict_t *rsp_dict)
+gd_mgmt_v3_post_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int32_t                  ret       = -1;
-        xlator_t                *this      = NULL;
-        char                    *volname   = NULL;
-        glusterd_volinfo_t      *volinfo   = NULL;
-
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        if (op_ret == 0)
-            glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_POST);
-
-        switch (op) {
-               case GD_OP_SNAP:
-               {
-                       ret = glusterd_snapshot_postvalidate (dict, op_ret,
-                                                             op_errstr,
-                                                             rsp_dict);
-                       if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                               GD_MSG_POST_VALIDATION_FAIL,
-                                               "postvalidate operation failed");
-                                goto out;
-                       }
-                       break;
-               }
-               case GD_OP_ADD_BRICK:
-               {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "Unable to get"
-                                        " volume name");
-                                goto out;
-                        }
-
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_VOL_NOT_FOUND, "Unable to "
-                                        "allocate memory");
-                                goto out;
-                        }
-                        ret = glusterd_create_volfiles_and_notify_services (
-                                                                     volinfo);
-                        if (ret)
-                                goto out;
-                        ret = glusterd_store_volinfo (volinfo,
-                                            GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                        if (ret)
-                                goto out;
-                        break;
-
-               }
-               case GD_OP_START_VOLUME:
-               {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "Unable to get"
-                                        " volume name");
-                                goto out;
-                        }
-
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_VOL_NOT_FOUND, "Unable to "
-                                        "allocate memory");
-                                goto out;
-                        }
-
-                        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                                if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
-                                        glusterd_defrag_info_set (volinfo, dict,
-                                                  GF_DEFRAG_CMD_START_TIER,
-                                                  GF_DEFRAG_CMD_START,
-                                                  GD_OP_REBALANCE);
-                                }
-                                glusterd_restart_rebalance_for_volume (volinfo);
-                        }
-                        break;
-               }
-
-               default:
-                        break;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        ret = 0;
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
 
+    switch (op) {
+        case GD_OP_ADD_BRICK:
+            ret = glusterd_post_commit_add_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+                       "Add-brick post commit failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_REPLACE_BRICK:
+            ret = glusterd_post_commit_replace_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+                       "Replace-brick post commit failed.");
+                goto out;
+            }
+            break;
+        default:
+            break;
+    }
+
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+gd_mgmt_v3_post_validate_fn(glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_lock_rsp        rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-        uuid_t                     *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        /* Even though the lock command has failed, while collating the errors
-           (gd_mgmt_v3_collate_errors), args->op_ret and args->op_errno will be
-           used. @args is obtained from frame->local. So before checking the
-           status of the request and going out if its a failure, args should be
-           set to frame->local. Otherwise, while collating args will be NULL.
-           This applies to other phases such as prevalidate, brickop, commit and
-           postvalidate also.
-        */
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    if (op_ret == 0)
+        glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_POST);
+
+    switch (op) {
+        case GD_OP_SNAP: {
+            ret = glusterd_snapshot_postvalidate(dict, op_ret, op_errstr,
+                                                 rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_POST_VALIDATION_FAIL,
+                       "postvalidate operation failed");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_ADD_BRICK: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " volume name");
+                goto out;
+            }
+
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Unable to "
+                       "allocate memory");
+                goto out;
+            }
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret)
+                goto out;
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret)
+                goto out;
+            break;
+        }
+        case GD_OP_START_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " volume name");
+                goto out;
+            }
+
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Unable to "
+                       "allocate memory");
+                goto out;
+            }
+
+            break;
+        }
+        case GD_OP_STOP_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " volume name");
+                goto out;
+            }
+
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Unable to "
+                       "allocate memory");
+                goto out;
+            }
+            break;
         }
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
+        default:
+            break;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
-        if (ret < 0)
-                goto out;
+    ret = 0;
 
-        gf_uuid_copy (args->uuid, rsp.uuid);
+out:
+    gf_msg_trace(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
+}
 
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+int32_t
+gd_mgmt_v3_lock_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    /* Even though the lock command has failed, while collating the errors
+       (gd_mgmt_v3_collate_errors), args->op_ret and args->op_errno will be
+       used. @args is obtained from frame->local. So before checking the
+       status of the request and going out if its a failure, args should be
+       set to frame->local. Otherwise, while collating args will be NULL.
+       This applies to other phases such as prevalidate, brickop, commit and
+       postvalidate also.
+    */
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                   GLUSTERD_MGMT_V3_LOCK, *peerid, rsp.uuid);
-        GF_FREE (peerid);
-
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_LOCK, *peerid, rsp.uuid);
+    GF_FREE(peerid);
+
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+gd_mgmt_v3_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_lock_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_lock_cbk_fn);
 }
 
 int
-gd_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
-                 glusterd_peerinfo_t *peerinfo,
-                 struct syncargs *args, uuid_t my_uuid,
-                 uuid_t recv_uuid)
+gd_mgmt_v3_lock(glusterd_op_t op, dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                struct syncargs *args, uuid_t my_uuid, uuid_t recv_uuid)
 {
-        gd1_mgmt_v3_lock_req     req  = {{0},};
-        int32_t                  ret  = -1;
-        xlator_t                *this = NULL;
-        uuid_t                  *peerid = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_LOCK,
-                                        gd_mgmt_v3_lock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
+    gd1_mgmt_v3_lock_req req = {
+        {0},
+    };
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_LOCK,
+                                   gd_mgmt_v3_lock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_initiate_lockdown (glusterd_op_t op, dict_t *dict,
-                                    char **op_errstr, uint32_t *op_errno,
-                                    gf_boolean_t  *is_acquired,
-                                    uint32_t txn_generation)
+glusterd_mgmt_v3_initiate_lockdown(glusterd_op_t op, dict_t *dict,
+                                   char **op_errstr, uint32_t *op_errno,
+                                   gf_boolean_t *is_acquired,
+                                   uint32_t txn_generation)
 {
-        char                *volname    = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-        glusterd_conf_t     *conf       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (is_acquired);
-
-        /* Trying to acquire multiple mgmt_v3 locks on local node */
-        ret = glusterd_multiple_mgmt_v3_lock (dict, MY_UUID, op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                        "Failed to acquire mgmt_v3 locks on localhost");
-                goto out;
-        }
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    uint32_t timeout = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(is_acquired);
+
+    /* Cli will add timeout key to dict if the default timeout is
+     * other than 2 minutes. Here we use this value to check whether
+     * mgmt_v3_lock_timeout should be set to default value or we
+     * need to change the value according to timeout value
+     * i.e, timeout + 120 seconds. */
+    ret = dict_get_uint32(dict, "timeout", &timeout);
+    if (!ret)
+        conf->mgmt_v3_lock_timeout = timeout + 120;
+
+    /* Trying to acquire multiple mgmt_v3 locks on local node */
+    ret = glusterd_multiple_mgmt_v3_lock(dict, MY_UUID, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+               "Failed to acquire mgmt_v3 locks on localhost");
+        goto out;
+    }
+
+    *is_acquired = _gf_true;
+
+    /* Sending mgmt_v3 lock req to other nodes in the cluster */
+    gd_syncargs_init(&args, NULL);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
 
-        *is_acquired = _gf_true;
-
-        /* Sending mgmt_v3 lock req to other nodes in the cluster */
-        gd_syncargs_init (&args, NULL);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                gd_mgmt_v3_lock (op, dict, peerinfo, &args,
-                                 MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
 
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
+        gd_mgmt_v3_lock(op, dict, peerinfo, &args, MY_UUID, peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        if (args.errstr)
-                *op_errstr = gf_strdup (args.errstr);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        ret = args.op_ret;
-        *op_errno = args.op_errno;
+    if (args.errstr)
+        *op_errstr = gf_strdup(args.errstr);
 
-        gf_msg_debug (this->name, 0, "Sent lock op req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
+
+    gf_msg_debug(this->name, 0,
+                 "Sent lock op req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        if (ret) {
-                if (*op_errstr)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_LOCK_GET_FAIL, "%s",
-                                *op_errstr);
-
-                if (volname)
-                        ret = gf_asprintf (op_errstr,
-                                           "Another transaction is in progress "
-                                           "for %s. Please try again after "
-                                           "sometime.", volname);
-                else
-                        ret = gf_asprintf (op_errstr,
-                                           "Another transaction is in progress "
-                                           "Please try again after sometime.");
-
-                if (ret == -1)
-                        *op_errstr = NULL;
+    if (ret) {
+        if (*op_errstr)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "%s", *op_errstr);
 
-                ret = -1;
-        }
+        ret = gf_asprintf(op_errstr,
+                          "Another transaction is in progress. "
+                          "Please try again after some time.");
+
+        if (ret == -1)
+            *op_errstr = NULL;
+
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op,
-                                     dict_t *aggr, dict_t *rsp)
+glusterd_pre_validate_aggr_rsp_dict(glusterd_op_t op, dict_t *aggr, dict_t *rsp)
 {
-        int32_t              ret      = 0;
-        xlator_t            *this     = NULL;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (aggr);
-        GF_ASSERT (rsp);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(aggr);
+    GF_ASSERT(rsp);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_SNAP:
-                ret = glusterd_snap_pre_validate_use_rsp_dict (aggr, rsp);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_PRE_VALIDATION_FAIL,
-                                "Failed to aggregate prevalidate "
-                                "response dictionaries.");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_pre_validate_use_rsp_dict(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                       "Failed to aggregate prevalidate "
+                       "response dictionaries.");
+                goto out;
+            }
+            break;
         case GD_OP_REPLACE_BRICK:
-                ret = glusterd_rb_use_rsp_dict (aggr, rsp);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_PRE_VALIDATION_FAIL,
-                                "Failed to aggregate prevalidate "
-                                "response dictionaries.");
-                        goto out;
-                }
-                break;
+            ret = glusterd_rb_use_rsp_dict(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                       "Failed to aggregate prevalidate "
+                       "response dictionaries.");
+                goto out;
+            }
+            break;
         case GD_OP_START_VOLUME:
         case GD_OP_ADD_BRICK:
-                ret = glusterd_aggr_brick_mount_dirs (aggr, rsp);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL, "Failed to "
-                                "aggregate brick mount dirs");
-                        goto out;
-                }
-                break;
+            ret = glusterd_aggr_brick_mount_dirs(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL,
+                       "Failed to "
+                       "aggregate brick mount dirs");
+                goto out;
+            }
+            break;
+        case GD_OP_RESET_BRICK:
+            ret = glusterd_rb_use_rsp_dict(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                       "Failed to aggregate prevalidate "
+                       "response dictionaries.");
+                goto out;
+            }
+        case GD_OP_STOP_VOLUME:
+        case GD_OP_REMOVE_BRICK:
+        case GD_OP_PROFILE_VOLUME:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_REBALANCE:
+            break;
+        case GD_OP_MAX_OPVERSION:
+            break;
         default:
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid op (%s)",
-                        gd_op_list[op]);
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid op (%s)", gd_op_list[op]);
 
-                break;
-        }
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_pre_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+gd_mgmt_v3_pre_validate_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_pre_val_rsp     rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        dict_t                     *rsp_dict      = NULL;
-        xlator_t                   *this          = NULL;
-        uuid_t                     *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        free (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        pthread_mutex_lock (&args->lock_dict);
-        {
-                ret = glusterd_pre_validate_aggr_rsp_dict (rsp.op, args->dict,
-                                                           rsp_dict);
-        }
-        pthread_mutex_unlock (&args->lock_dict);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RESP_AGGR_FAIL, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                if (!rsp.op_ret)
-                        op_ret = ret;
-                else {
-                        op_ret = rsp.op_ret;
-                        op_errno = rsp.op_errno;
-                }
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_pre_val_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            free(rsp.dict.dict_val);
+            goto out;
         } else {
-                op_ret = rsp.op_ret;
-                op_errno = rsp.op_errno;
-        }
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        ret = glusterd_pre_validate_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+    }
 
 out:
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                  GLUSTERD_MGMT_V3_PRE_VALIDATE,
-                                  *peerid, rsp.uuid);
-
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_PRE_VALIDATE, *peerid, rsp.uuid);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_pre_validate_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+gd_mgmt_v3_pre_validate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_pre_validate_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_pre_validate_cbk_fn);
 }
 
 int
-gd_mgmt_v3_pre_validate_req (glusterd_op_t op, dict_t *op_ctx,
-                             glusterd_peerinfo_t *peerinfo,
-                             struct syncargs *args, uuid_t my_uuid,
-                             uuid_t recv_uuid)
+gd_mgmt_v3_pre_validate_req(glusterd_op_t op, dict_t *op_ctx,
+                            glusterd_peerinfo_t *peerinfo,
+                            struct syncargs *args, uuid_t my_uuid,
+                            uuid_t recv_uuid)
 {
-        int32_t                  ret   = -1;
-        gd1_mgmt_v3_pre_val_req  req   = {{0},};
-        xlator_t                *this  = NULL;
-        uuid_t                  *peerid = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_PRE_VALIDATE,
-                                        gd_mgmt_v3_pre_validate_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_pre_val_req);
+    int32_t ret = -1;
+    gd1_mgmt_v3_pre_val_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog,
+        GLUSTERD_MGMT_V3_PRE_VALIDATE, gd_mgmt_v3_pre_validate_cbk,
+        (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_pre_validate (glusterd_op_t op, dict_t *req_dict,
-                               char **op_errstr, uint32_t *op_errno,
-                               uint32_t txn_generation)
+glusterd_mgmt_v3_pre_validate(glusterd_op_t op, dict_t *req_dict,
+                              char **op_errstr, uint32_t *op_errno,
+                              uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-        glusterd_conf_t     *conf       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (req_dict);
-        GF_ASSERT (op_errstr);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to create response dictionary");
-                goto out;
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    if (op == GD_OP_PROFILE_VOLUME || op == GD_OP_STOP_VOLUME ||
+        op == GD_OP_REBALANCE || op == GD_OP_REMOVE_BRICK) {
+        ret = glusterd_validate_quorum(this, op, req_dict, op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Server quorum not met. Rejecting operation.");
+            goto out;
         }
+    }
 
-        /* Pre Validation on local node */
-        ret = gd_mgmt_v3_pre_validate_fn (op, req_dict, op_errstr,
-                                          rsp_dict, op_errno);
+    /* Pre Validation on local node */
+    ret = gd_mgmt_v3_pre_validate_fn(op, req_dict, op_errstr, rsp_dict,
+                                     op_errno);
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL,
-                        "Pre Validation failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Pre-validation failed "
-                                           "on localhost. Please "
-                                           "check log file for details");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Pre-validation failed "
+                              "on localhost. Please "
+                              "check log file for details");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
         }
+        goto out;
+    }
 
-        ret = glusterd_pre_validate_aggr_rsp_dict (op, req_dict,
-                                                   rsp_dict);
+    if (op != GD_OP_MAX_OPVERSION) {
+        ret = glusterd_pre_validate_aggr_rsp_dict(op, req_dict, rsp_dict);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                   "%s",
+                   "Failed to aggregate response from "
+                   " node/brick");
+            goto out;
         }
 
-        dict_unref (rsp_dict);
+        dict_unref(rsp_dict);
         rsp_dict = NULL;
+    }
 
-        /* Sending Pre Validation req to other nodes in the cluster */
-        gd_syncargs_init (&args, req_dict);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                gd_mgmt_v3_pre_validate_req (op, req_dict, peerinfo, &args,
-                                             MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
+    /* Sending Pre Validation req to other nodes in the cluster */
+    gd_syncargs_init(&args, req_dict);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
 
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
+    peer_cnt = 0;
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_pre_validate_req(op, req_dict, peerinfo, &args, MY_UUID,
+                                    peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        if (args.op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL,
-                        "Pre Validation failed on peers");
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation failed on peers");
 
-        ret = args.op_ret;
-        *op_errno = args.op_errno;
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
+
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
 
-        gf_msg_debug (this->name, 0, "Sent pre valaidation req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent pre valaidation req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
-                                glusterd_op_t op)
+glusterd_mgmt_v3_build_payload(dict_t **req, char **op_errstr, dict_t *dict,
+                               glusterd_op_t op)
 {
-        int32_t                 ret      = -1;
-        dict_t                 *req_dict = NULL;
-        xlator_t               *this     = NULL;
-        char                   *volname  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-
-        req_dict = dict_new ();
-        if (!req_dict)
+    int32_t ret = -1;
+    dict_t *req_dict = NULL;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+
+    req_dict = dict_new();
+    if (!req_dict)
+        goto out;
+
+    switch (op) {
+        case GD_OP_MAX_OPVERSION:
+        case GD_OP_SNAP:
+            dict_copy(dict, req_dict);
+            break;
+        case GD_OP_START_VOLUME:
+        case GD_OP_STOP_VOLUME:
+        case GD_OP_ADD_BRICK:
+        case GD_OP_REMOVE_BRICK:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_REPLACE_BRICK:
+        case GD_OP_RESET_BRICK:
+        case GD_OP_PROFILE_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                       GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
+                goto out;
+            }
+
+            if (strcasecmp(volname, "all")) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
+
+        case GD_OP_REBALANCE: {
+            if (gd_set_commit_hash(dict) != 0) {
+                ret = -1;
                 goto out;
+            }
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                       GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
+                goto out;
+            }
 
-        switch (op) {
-                case GD_OP_SNAP:
-                        dict_copy (dict, req_dict);
-                        break;
-                case GD_OP_START_VOLUME:
-                case GD_OP_ADD_BRICK:
-                case GD_OP_REPLACE_BRICK:
-                {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "volname is not present in "
-                                        "operation ctx");
-                                goto out;
-                        }
-
-                        if (strcasecmp (volname, "all")) {
-                                ret = glusterd_dict_set_volid (dict,
-                                                               volname,
-                                                             op_errstr);
-                                if (ret)
-                                        goto out;
-                        }
-                        dict_copy (dict, req_dict);
-                }
-                        break;
-                default:
-                        break;
-        }
+            if (strcasecmp(volname, "all")) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
 
-        *req = req_dict;
-        ret = 0;
+        default:
+            break;
+    }
+
+    *req = req_dict;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_brick_op_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
+gd_mgmt_v3_brick_op_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_brick_op_rsp     rsp          = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-        uuid_t                     *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        /* If the operation failed, then iov can be NULL. So better check the
-           status of the operation and then worry about iov (if the status of
-           the command is success)
-        */
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_brick_op_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    /* If the operation failed, then iov can be NULL. So better check the
+       status of the operation and then worry about iov (if the status of
+       the command is success)
+    */
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        if (rsp.op == GD_OP_DEFRAG_BRICK_VOLUME ||
+            rsp.op == GD_OP_PROFILE_VOLUME)
+            ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+    }
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
+out:
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
-        if (ret < 0)
-                goto out;
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_BRICK_OP, *peerid, rsp.uuid);
 
-        gf_uuid_copy (args->uuid, rsp.uuid);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
 
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                   GLUSTERD_MGMT_V3_BRICK_OP, *peerid,
-                                   rsp.uuid);
-
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+gd_mgmt_v3_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_brick_op_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_brick_op_cbk_fn);
 }
 
 int
-gd_mgmt_v3_brick_op_req (glusterd_op_t op, dict_t *op_ctx,
-                         glusterd_peerinfo_t *peerinfo,
-                         struct syncargs *args, uuid_t my_uuid,
-                         uuid_t recv_uuid)
+gd_mgmt_v3_brick_op_req(glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                        uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int32_t                   ret  = -1;
-        gd1_mgmt_v3_brick_op_req  req  = {{0},};
-        xlator_t                 *this = NULL;
-        uuid_t                   *peerid = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_BRICK_OP,
-                                        gd_mgmt_v3_brick_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_brick_op_req);
+    int32_t ret = -1;
+    gd1_mgmt_v3_brick_op_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_BRICK_OP,
+                                   gd_mgmt_v3_brick_op_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict, char **op_errstr,
-                           uint32_t txn_generation)
+glusterd_mgmt_v3_brick_op(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                          char **op_errstr, uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-        glusterd_conf_t     *conf       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (req_dict);
-        GF_ASSERT (op_errstr);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to create response dictionary");
-                goto out;
-        }
-
-        /* Perform brick op on local node */
-        ret = gd_mgmt_v3_brick_op_fn (op, req_dict, op_errstr,
-                                     rsp_dict);
-
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Perform brick op on local node */
+    ret = gd_mgmt_v3_brick_op_fn(op, req_dict, op_errstr, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick ops failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Brick ops failed "
+                              "on localhost. Please "
+                              "check log file for details");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+    if (op == GD_OP_DEFRAG_BRICK_VOLUME || op == GD_OP_PROFILE_VOLUME) {
+        ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_OP_FAIL,
-                        "Brick ops failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Brick ops failed "
-                                           "on localhost. Please "
-                                           "check log file for details");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "%s",
+                   "Failed to aggregate response from "
+                   " node/brick");
+            goto out;
         }
+    }
 
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
 
-        /* Sending brick op req to other nodes in the cluster */
-        gd_syncargs_init (&args, NULL);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                gd_mgmt_v3_brick_op_req (op, req_dict, peerinfo, &args,
-                                         MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
+    /* Sending brick op req to other nodes in the cluster */
+    gd_syncargs_init(&args, op_ctx);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
 
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
+    peer_cnt = 0;
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_brick_op_req(op, req_dict, peerinfo, &args, MY_UUID,
+                                peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        if (args.op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_OP_FAIL,
-                        "Brick ops failed on peers");
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick ops failed on peers");
 
-        ret = args.op_ret;
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        gf_msg_debug (this->name, 0, "Sent brick op req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    ret = args.op_ret;
+
+    gf_msg_debug(this->name, 0,
+                 "Sent brick op req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_commit_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+gd_mgmt_v3_commit_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_commit_rsp       rsp          = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        dict_t                     *rsp_dict      = NULL;
-        xlator_t                   *this          = NULL;
-        uuid_t                     *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        free (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        pthread_mutex_lock (&args->lock_dict);
-        {
-                ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
-                                                     rsp_dict);
-        }
-        pthread_mutex_unlock (&args->lock_dict);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RESP_AGGR_FAIL, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                if (!rsp.op_ret)
-                        op_ret = ret;
-                else {
-                        op_ret = rsp.op_ret;
-                        op_errno = rsp.op_errno;
-                }
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_commit_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            free(rsp.dict.dict_val);
+            goto out;
         } else {
-                op_ret = rsp.op_ret;
-                op_errno = rsp.op_errno;
-        }
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+    }
 
 out:
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                  GLUSTERD_MGMT_V3_COMMIT, *peerid, rsp.uuid);
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_COMMIT, *peerid, rsp.uuid);
+    GF_FREE(peerid);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_commit_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+gd_mgmt_v3_commit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_commit_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_commit_cbk_fn);
 }
 
 int
-gd_mgmt_v3_commit_req (glusterd_op_t op, dict_t *op_ctx,
-                       glusterd_peerinfo_t *peerinfo,
-                       struct syncargs *args, uuid_t my_uuid,
-                       uuid_t recv_uuid)
+gd_mgmt_v3_commit_req(glusterd_op_t op, dict_t *op_ctx,
+                      glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                      uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int32_t                  ret  = -1;
-        gd1_mgmt_v3_commit_req   req  = {{0},};
-        xlator_t                *this = NULL;
-        uuid_t                  *peerid = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_COMMIT,
-                                        gd_mgmt_v3_commit_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_commit_req);
+    int32_t ret = -1;
+    gd1_mgmt_v3_commit_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_COMMIT,
+                                   gd_mgmt_v3_commit_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_commit (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
-                         char **op_errstr, uint32_t *op_errno,
-                         uint32_t txn_generation)
+glusterd_mgmt_v3_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                        char **op_errstr, uint32_t *op_errno,
+                        uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-        glusterd_conf_t     *conf       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (req_dict);
-        GF_ASSERT (op_errstr);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to create response dictionary");
-                goto out;
-        }
-
-        /* Commit on local node */
-        ret = gd_mgmt_v3_commit_fn (op, req_dict, op_errstr,
-                                    op_errno, rsp_dict);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL,
-                        "Commit failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Commit failed "
-                                           "on localhost. Please "
-                                           "check log file for details.");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx,
-                                             rsp_dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RESP_AGGR_FAIL, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                goto out;
-        }
-
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
-
-        /* Sending commit req to other nodes in the cluster */
-        gd_syncargs_init (&args, op_ctx);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                gd_mgmt_v3_commit_req (op, req_dict, peerinfo, &args,
-                                       MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
-
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    switch (op) {
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+
+            ret = glusterd_set_rebalance_id_in_rsp_dict(req_dict, op_ctx);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set rebalance id in dict.");
+            }
+            break;
+        case GD_OP_REMOVE_BRICK:
+            ret = glusterd_set_rebalance_id_for_remove_brick(req_dict, op_ctx);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set rebalance id for remove-brick in dict.");
+            }
+            break;
+        default:
+            break;
+    }
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Commit on local node */
+    ret = gd_mgmt_v3_commit_fn(op, req_dict, op_errstr, op_errno, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Commit failed "
+                              "on localhost. Please "
+                              "check log file for details.");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+
+    ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        goto out;
+    }
+
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
+
+    /* Sending commit req to other nodes in the cluster */
+    gd_syncargs_init(&args, op_ctx);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+        if (!peerinfo->connected)
+            continue;
+
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_commit_req(op, req_dict, peerinfo, &args, MY_UUID,
+                              peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (args.op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL,
-                        "Commit failed on peers");
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit failed on peers");
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        ret = args.op_ret;
-        *op_errno = args.op_errno;
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
 
-        gf_msg_debug (this->name, 0, "Sent commit req for %s to %d "
-                "peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent commit req for %s to %d "
+                 "peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    glusterd_op_modify_op_ctx(op, op_ctx);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_post_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+gd_mgmt_v3_post_commit_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                              void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_post_val_rsp    rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-        uuid_t                     *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_post_commit_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            free(rsp.dict.dict_val);
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
         op_ret = rsp.op_ret;
         op_errno = rsp.op_errno;
+    }
 
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                  GLUSTERD_MGMT_V3_POST_VALIDATE, *peerid,
-                                  rsp.uuid);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_POST_COMMIT, *peerid, rsp.uuid);
+    GF_FREE(peerid);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_post_validate_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+gd_mgmt_v3_post_commit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_post_validate_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_post_commit_cbk_fn);
 }
 
 int
-gd_mgmt_v3_post_validate_req (glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
-                              glusterd_peerinfo_t *peerinfo,
-                              struct syncargs *args, uuid_t my_uuid,
-                              uuid_t recv_uuid)
+gd_mgmt_v3_post_commit_req(glusterd_op_t op, dict_t *op_ctx,
+                           glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                           uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int32_t                   ret  = -1;
-        gd1_mgmt_v3_post_val_req  req  = {{0},};
-        xlator_t                 *this = NULL;
-        uuid_t                   *peerid = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-        req.op_ret = op_ret;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_POST_VALIDATE,
-                                        gd_mgmt_v3_post_validate_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_post_val_req);
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_commit_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog,
+        GLUSTERD_MGMT_V3_POST_COMMIT, gd_mgmt_v3_post_commit_cbk,
+        (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_post_validate (glusterd_op_t op, int32_t op_ret, dict_t *dict,
-                                dict_t *req_dict, char **op_errstr,
-                                uint32_t txn_generation)
+glusterd_mgmt_v3_post_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                             char **op_errstr, uint32_t *op_errno,
+                             uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-        glusterd_conf_t     *conf       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (dict);
-        GF_VALIDATE_OR_GOTO (this->name, req_dict, out);
-        GF_ASSERT (op_errstr);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to create response dictionary");
-                goto out;
-        }
-
-        /* Copy the contents of dict like missed snaps info to req_dict */
-        dict_copy (dict, req_dict);
-
-        /* Post Validation on local node */
-        ret = gd_mgmt_v3_post_validate_fn (op, op_ret, req_dict, op_errstr,
-                                           rsp_dict);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_POST_VALIDATION_FAIL,
-                        "Post Validation failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Post-validation failed "
-                                           "on localhost. Please check "
-                                           "log file for details");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
-
-        /* Sending Post Validation req to other nodes in the cluster */
-        gd_syncargs_init (&args, req_dict);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                gd_mgmt_v3_post_validate_req (op, op_ret, req_dict, peerinfo,
-                                              &args, MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
-
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Post commit on local node */
+    ret = gd_mgmt_v3_post_commit_fn(op, req_dict, op_errstr, op_errno,
+                                    rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "Post commit failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Post commit failed "
+                              "on localhost. Please "
+                              "check log file for details.");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+
+    ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        goto out;
+    }
+
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
+
+    /* Sending post commit req to other nodes in the cluster */
+    gd_syncargs_init(&args, op_ctx);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+        if (!peerinfo->connected)
+            continue;
+
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_post_commit_req(op, req_dict, peerinfo, &args, MY_UUID,
+                                   peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (args.op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_POST_VALIDATION_FAIL,
-                        "Post Validation failed on peers");
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "Post commit failed on peers");
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        ret = args.op_ret;
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
 
-        gf_msg_debug (this->name, 0, "Sent post valaidation req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent post commit req for %s to %d "
+                 "peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    glusterd_op_modify_op_ctx(op, op_ctx);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+gd_mgmt_v3_post_validate_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_unlock_rsp      rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-        uuid_t                     *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_post_val_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                  GLUSTERD_MGMT_V3_UNLOCK, *peerid, rsp.uuid);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_POST_VALIDATE, *peerid,
+                              rsp.uuid);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+gd_mgmt_v3_post_validate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_unlock_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_post_validate_cbk_fn);
 }
 
 int
-gd_mgmt_v3_unlock (glusterd_op_t op, dict_t *op_ctx,
-                   glusterd_peerinfo_t *peerinfo,
-                   struct syncargs *args, uuid_t my_uuid,
-                   uuid_t recv_uuid)
+gd_mgmt_v3_post_validate_req(glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
+                             glusterd_peerinfo_t *peerinfo,
+                             struct syncargs *args, uuid_t my_uuid,
+                             uuid_t recv_uuid)
 {
-        int32_t                  ret  = -1;
-        gd1_mgmt_v3_unlock_req   req  = {{0},};
-        xlator_t                *this = NULL;
-        uuid_t                  *peerid = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_UNLOCK,
-                                        gd_mgmt_v3_unlock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_val_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+    req.op_ret = op_ret;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog,
+        GLUSTERD_MGMT_V3_POST_VALIDATE, gd_mgmt_v3_post_validate_cbk,
+        (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_release_peer_locks (glusterd_op_t op, dict_t *dict,
-                                     int32_t op_ret, char **op_errstr,
-                                     gf_boolean_t  is_acquired,
-                                     uint32_t txn_generation)
+glusterd_mgmt_v3_post_validate(glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                               dict_t *req_dict, char **op_errstr,
+                               uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        glusterd_conf_t     *conf       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        /* If the lock has not been held during this
-         * transaction, do not send unlock requests */
-        if (!is_acquired)
-                goto out;
-
-        /* Sending mgmt_v3 unlock req to other nodes in the cluster */
-        gd_syncargs_init (&args, NULL);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                gd_mgmt_v3_unlock (op, dict, peerinfo, &args,
-                                   MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
-
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
-
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_VALIDATE_OR_GOTO(this->name, req_dict, out);
+    GF_ASSERT(op_errstr);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Post Validation on local node */
+    ret = gd_mgmt_v3_post_validate_fn(op, op_ret, req_dict, op_errstr,
+                                      rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Post-validation failed "
+                              "on localhost. Please check "
+                              "log file for details");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
+
+    /* Sending Post Validation req to other nodes in the cluster */
+    gd_syncargs_init(&args, req_dict);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_post_validate_req(op, op_ret, req_dict, peerinfo, &args,
+                                     MY_UUID, peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        if (args.op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_UNLOCK_FAIL,
-                        "Unlock failed on peers");
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-                if (!op_ret && args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation failed on peers");
 
-        ret = args.op_ret;
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        gf_msg_debug (this->name, 0, "Sent unlock op req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    ret = args.op_ret;
 
+    gf_msg_debug(this->name, 0,
+                 "Sent post valaidation req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
-                                      dict_t *dict)
+gd_mgmt_v3_unlock_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int32_t                     ret              = -1;
-        int32_t                     op_ret           = -1;
-        dict_t                      *req_dict        = NULL;
-        dict_t                      *tmp_dict        = NULL;
-        glusterd_conf_t             *conf            = NULL;
-        char                        *op_errstr       = NULL;
-        xlator_t                    *this            = NULL;
-        gf_boolean_t                is_acquired      = _gf_false;
-        uuid_t                      *originator_uuid = NULL;
-        uint32_t                    txn_generation   = 0;
-        uint32_t                    op_errno         = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        /* Save the peer list generation */
-        txn_generation = conf->generation;
-        cmm_smp_rmb ();
-        /* This read memory barrier makes sure that this assignment happens here
-         * only and is not reordered and optimized by either the compiler or the
-         * processor.
-         */
-
-        /* Save the MY_UUID as the originator_uuid. This originator_uuid
-         * will be used by is_origin_glusterd() to determine if a node
-         * is the originator node for a command. */
-        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
-                                     gf_common_mt_uuid_t);
-        if (!originator_uuid) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (*originator_uuid, MY_UUID);
-        ret = dict_set_bin (dict, "originator_uuid",
-                            originator_uuid, sizeof (uuid_t));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set originator_uuid.");
-                GF_FREE (originator_uuid);
-                goto out;
-        }
-
-        /* Marking the operation as complete synctasked */
-        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set synctasked flag.");
-                goto out;
-        }
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
-        /* Use a copy at local unlock as cli response will be sent before
-         * the unlock and the volname in the dict might be removed */
-        tmp_dict = dict_new();
-        if (!tmp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL, "Unable to create dict");
-                goto out;
-        }
-        dict_copy (dict, tmp_dict);
+out:
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_UNLOCK, *peerid, rsp.uuid);
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
+}
 
-        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
-        ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr,
-                                                  &op_errno, &is_acquired,
-                                                  txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_LOCKDOWN_FAIL,
-                        "mgmt_v3 lockdown failed.");
-                goto out;
-        }
+int32_t
+gd_mgmt_v3_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_unlock_cbk_fn);
+}
 
-        /* BUILD PAYLOAD */
-        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                goto out;
-        }
+int
+gd_mgmt_v3_unlock(glusterd_op_t op, dict_t *op_ctx,
+                  glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                  uuid_t my_uuid, uuid_t recv_uuid)
+{
+    int32_t ret = -1;
+    gd1_mgmt_v3_unlock_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_UNLOCK,
+                                   gd_mgmt_v3_unlock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        /* PRE-COMMIT VALIDATE PHASE */
-        ret = glusterd_mgmt_v3_pre_validate (op, req_dict, &op_errstr,
-                                             &op_errno, txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL, "Pre Validation Failed");
-                goto out;
-        }
+int
+glusterd_mgmt_v3_release_peer_locks(glusterd_op_t op, dict_t *dict,
+                                    int32_t op_ret, char **op_errstr,
+                                    gf_boolean_t is_acquired,
+                                    uint32_t txn_generation)
+{
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    /* If the lock has not been held during this
+     * transaction, do not send unlock requests */
+    if (!is_acquired)
+        goto out;
+
+    /* Sending mgmt_v3 unlock req to other nodes in the cluster */
+    gd_syncargs_init(&args, NULL);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    peer_cnt = 0;
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
 
-        /* COMMIT OP PHASE */
-        ret = glusterd_mgmt_v3_commit (op, dict, req_dict, &op_errstr,
-                                       &op_errno, txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL, "Commit Op Failed");
-                goto out;
-        }
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
 
-        /* POST-COMMIT VALIDATE PHASE */
-        /* As of now, post_validate is not trying to cleanup any failed
-           commands. So as of now, I am sending 0 (op_ret as 0).
-        */
-        ret = glusterd_mgmt_v3_post_validate (op, 0, dict, req_dict, &op_errstr,
-                                              txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_POST_VALIDATION_FAIL, "Post Validation Failed");
-                goto out;
-        }
+        gd_mgmt_v3_unlock(op, dict, peerinfo, &args, MY_UUID, peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
 
+    if (0 == peer_cnt) {
         ret = 0;
-out:
-        op_ret = ret;
-        /* UNLOCK PHASE FOR PEERS*/
-        (void) glusterd_mgmt_v3_release_peer_locks (op, dict, op_ret,
-                                                    &op_errstr, is_acquired,
-                                                    txn_generation);
-
-        /* LOCAL VOLUME(S) UNLOCK */
-        if (is_acquired) {
-                /* Trying to release multiple mgmt_v3 locks */
-                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                "Failed to release mgmt_v3 locks on localhost");
-                        op_ret = ret;
-                }
-        }
+        goto out;
+    }
 
-        if (op_ret && (op_errno == 0))
-                op_errno = EG_INTRNL;
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        /* SEND CLI RESPONSE */
-        glusterd_op_send_cli_response (op, op_ret, op_errno, req,
-                                       dict, op_errstr);
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+               "Unlock failed on peers");
 
-        if (req_dict)
-                dict_unref (req_dict);
+        if (!op_ret && args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        if (tmp_dict)
-                dict_unref (tmp_dict);
+    ret = args.op_ret;
 
-        if (op_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-        }
+    gf_msg_debug(this->name, 0,
+                 "Sent unlock op req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 
-        return 0;
+out:
+    return ret;
 }
 
 int32_t
-glusterd_set_barrier_value (dict_t *dict, char *option)
+glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(rpcsvc_request_t *req,
+                                                        glusterd_op_t op,
+                                                        dict_t *dict)
 {
-        int32_t                 ret             = -1;
-        xlator_t                *this           = NULL;
-        glusterd_volinfo_t     *vol             = NULL;
-        char                    *volname        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (option);
-
-        /* TODO : Change this when we support multiple volume.
-         * As of now only snapshot of single volume is supported,
-         * Hence volname1 is directly fetched
-         */
-        ret = dict_get_str (dict, "volname1", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Volname not present in "
-                        "dict");
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *op_errstr = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+    uuid_t *originator_uuid = NULL;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Save the peer list generation */
+    txn_generation = conf->generation;
+    cmm_smp_rmb();
+    /* This read memory barrier makes sure that this assignment happens here
+     * only and is not reordered and optimized by either the compiler or the
+     * processor.
+     */
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    /* Marking the operation as complete synctasked */
+    ret = dict_set_int32(dict, "is_synctasked", _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag.");
+        goto out;
+    }
+
+    /* Use a copy at local unlock as cli response will be sent before
+     * the unlock and the volname in the dict might be removed */
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    /* BUILD PAYLOAD */
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    /* PRE-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    /* BRICK-OPS */
+    ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr,
+                                    txn_generation);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Brick Op Failed");
+        goto out;
+    }
+
+    /* COMMIT OP PHASE */
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        goto out;
+    }
+
+    /* POST-COMMIT VALIDATE PHASE */
+    /* As of now, post_validate is not trying to cleanup any failed
+       commands. So as of now, I am sending 0 (op_ret as 0).
+    */
+    ret = glusterd_mgmt_v3_post_validate(op, 0, dict, req_dict, &op_errstr,
+                                         txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation Failed");
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &vol);
+    ret = 0;
+out:
+    op_ret = ret;
+    /* UNLOCK PHASE FOR PEERS*/
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
+
+    /* LOCAL VOLUME(S) UNLOCK */
+    if (is_acquired) {
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "Volume %s not found ",
-                        volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on localhost");
+            op_ret = ret;
         }
+    }
 
-        ret = dict_set_dynstr_with_alloc (dict, "barrier", option);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set barrier op "
-                        "in request dictionary");
-                goto out;
-        }
+    if (op_ret && (op_errno == 0))
+        op_errno = EG_INTRNL;
 
-        ret = dict_set_dynstr_with_alloc (vol->dict, "features.barrier",
-                                          option);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set barrier op "
-                        "in volume option dict");
-                goto out;
-        }
+    if (op != GD_OP_MAX_OPVERSION) {
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict,
+                                      op_errstr);
+    }
 
-        gd_update_volume_op_versions (vol);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        ret = glusterd_create_volfiles (vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "Failed to create volfiles");
-                goto out;
-        }
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        ret = glusterd_store_volinfo (vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    return 0;
 }
 
 int32_t
-glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
-                                       dict_t *dict)
+glusterd_mgmt_v3_initiate_all_phases(rpcsvc_request_t *req, glusterd_op_t op,
+                                     dict_t *dict)
 {
-        int32_t                     ret              = -1;
-        int32_t                     op_ret           = -1;
-        dict_t                      *req_dict        = NULL;
-        dict_t                      *tmp_dict        = NULL;
-        glusterd_conf_t             *conf            = NULL;
-        char                        *op_errstr       = NULL;
-        xlator_t                    *this            = NULL;
-        gf_boolean_t                is_acquired      = _gf_false;
-        uuid_t                      *originator_uuid = NULL;
-        gf_boolean_t                success          = _gf_false;
-        char                        *cli_errstr      = NULL;
-        uint32_t                    txn_generation   = 0;
-        uint32_t                    op_errno         = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        /* Save the peer list generation */
-        txn_generation = conf->generation;
-        cmm_smp_rmb ();
-        /* This read memory barrier makes sure that this assignment happens here
-         * only and is not reordered and optimized by either the compiler or the
-         * processor.
-         */
-
-        /* Save the MY_UUID as the originator_uuid. This originator_uuid
-         * will be used by is_origin_glusterd() to determine if a node
-         * is the originator node for a command. */
-        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
-                                     gf_common_mt_uuid_t);
-        if (!originator_uuid) {
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *op_errstr = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+    uuid_t *originator_uuid = NULL;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Save the peer list generation */
+    txn_generation = conf->generation;
+    cmm_smp_rmb();
+    /* This read memory barrier makes sure that this assignment happens here
+     * only and is not reordered and optimized by either the compiler or the
+     * processor.
+     */
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    /* Marking the operation as complete synctasked */
+    ret = dict_set_int32(dict, "is_synctasked", _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag.");
+        goto out;
+    }
+
+    /* Use a copy at local unlock as cli response will be sent before
+     * the unlock and the volname in the dict might be removed */
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    /* BUILD PAYLOAD */
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    /* PRE-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    /* COMMIT OP PHASE */
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        goto out;
+    }
+
+    /* POST COMMIT OP PHASE */
+    ret = glusterd_mgmt_v3_post_commit(op, dict, req_dict, &op_errstr,
+                                       &op_errno, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "Post commit Op Failed");
+        goto out;
+    }
+
+    /* POST-COMMIT VALIDATE PHASE */
+    /* As of now, post_validate is not trying to cleanup any failed
+       commands. So as of now, I am sending 0 (op_ret as 0).
+    */
+    ret = glusterd_mgmt_v3_post_validate(op, 0, dict, req_dict, &op_errstr,
+                                         txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation Failed");
+        goto out;
+    }
 
-        gf_uuid_copy (*originator_uuid, MY_UUID);
-        ret = dict_set_bin (dict, "originator_uuid",
-                            originator_uuid, sizeof (uuid_t));
+    ret = 0;
+out:
+    op_ret = ret;
+    /* UNLOCK PHASE FOR PEERS*/
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
+
+    /* LOCAL VOLUME(S) UNLOCK */
+    if (is_acquired) {
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set originator_uuid.");
-                GF_FREE (originator_uuid);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on localhost");
+            op_ret = ret;
         }
+    }
 
-        /* Marking the operation as complete synctasked */
-        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set synctasked flag.");
-                goto out;
-        }
+    if (op_ret && (op_errno == 0))
+        op_errno = EG_INTRNL;
 
-        /* Use a copy at local unlock as cli response will be sent before
-         * the unlock and the volname in the dict might be removed */
-        tmp_dict = dict_new();
-        if (!tmp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL, "Unable to create dict");
-                goto out;
-        }
-        dict_copy (dict, tmp_dict);
+    if (op != GD_OP_MAX_OPVERSION) {
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict,
+                                      op_errstr);
+    }
 
-        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
-        ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr,
-                                                  &op_errno, &is_acquired,
-                                                  txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_LOCKDOWN_FAIL,
-                        "mgmt_v3 lockdown failed.");
-                goto out;
-        }
+    if (req_dict)
+        dict_unref(req_dict);
 
-        /* BUILD PAYLOAD */
-        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                goto out;
-        }
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        /* PRE-COMMIT VALIDATE PHASE */
-        ret = glusterd_mgmt_v3_pre_validate (op, req_dict, &op_errstr,
-                                             &op_errno, txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL, "Pre Validation Failed");
-                goto out;
-        }
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-        /* quorum check of the volume is done here */
-        ret = glusterd_snap_quorum_check (req_dict, _gf_false, &op_errstr,
-                                          &op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_QUORUM_CHECK_FAIL, "Volume quorum check failed");
-                goto out;
-        }
+    return 0;
+}
 
-        /* Set the operation type as pre, so that differentiation can be
-         * made whether the brickop is sent during pre-commit or post-commit
-         */
-        ret = dict_set_dynstr_with_alloc (req_dict, "operation-type", "pre");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "operation-type in dictionary");
-                goto out;
-        }
+int32_t
+glusterd_set_barrier_value(dict_t *dict, char *option)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(option);
+
+    /* TODO : Change this when we support multiple volume.
+     * As of now only snapshot of single volume is supported,
+     * Hence volname1 is directly fetched
+     */
+    ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume %s not found ", volname);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "barrier", option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set barrier op "
+               "in request dictionary");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(vol->dict, "features.barrier", option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set barrier op "
+               "in volume option dict");
+        goto out;
+    }
+
+    gd_update_volume_op_versions(vol);
+
+    ret = glusterd_create_volfiles(vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfiles");
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
 
-        ret = glusterd_mgmt_v3_brick_op (op, req_dict, &op_errstr,
-                                         txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_OP_FAIL, "Brick Ops Failed");
-                goto unbarrier;
-        }
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        /* COMMIT OP PHASE */
-        /* TODO: As of now, the plan is to do quorum check before sending the
-           commit fop and if the quorum succeeds, then commit is sent to all
-           the other glusterds.
-           snap create functionality now creates the in memory and on disk
-           objects for the snapshot (marking them as incomplete), takes the lvm
-           snapshot and then updates the status of the in memory and on disk
-           snap objects as complete. Suppose one of the glusterds goes down
-           after taking the lvm snapshot, but before updating the snap object,
-           then treat it as a snapshot create failure and trigger cleanup.
-           i.e the number of commit responses received by the originator
-           glusterd shold be the same as the number of peers it has sent the
-           request to (i.e npeers variable). If not, then originator glusterd
-           will initiate cleanup in post-validate fop.
-           Question: What if one of the other glusterds goes down as explained
-           above and along with it the originator glusterd also goes down?
-           Who will initiate the cleanup?
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *op_errstr = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+    uuid_t *originator_uuid = NULL;
+    gf_boolean_t success = _gf_false;
+    char *cli_errstr = NULL;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Save the peer list generation */
+    txn_generation = conf->generation;
+    cmm_smp_rmb();
+    /* This read memory barrier makes sure that this assignment happens here
+     * only and is not reordered and optimized by either the compiler or the
+     * processor.
+     */
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    /* Marking the operation as complete synctasked */
+    ret = dict_set_int32n(dict, "is_synctasked", SLEN("is_synctasked"),
+                          _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag.");
+        goto out;
+    }
+
+    /* Use a copy at local unlock as cli response will be sent before
+     * the unlock and the volname in the dict might be removed */
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    /* BUILD PAYLOAD */
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    /* PRE-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    /* quorum check of the volume is done here */
+    ret = glusterd_snap_quorum_check(req_dict, _gf_false, &op_errstr,
+                                     &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+               "Volume quorum check failed");
+        goto out;
+    }
+
+    /* Set the operation type as pre, so that differentiation can be
+     * made whether the brickop is sent during pre-commit or post-commit
+     */
+    ret = dict_set_dynstr_with_alloc(req_dict, "operation-type", "pre");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "operation-type in dictionary");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr,
+                                    txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick Ops Failed");
+        goto unbarrier;
+    }
+
+    /* COMMIT OP PHASE */
+    /* TODO: As of now, the plan is to do quorum check before sending the
+       commit fop and if the quorum succeeds, then commit is sent to all
+       the other glusterds.
+       snap create functionality now creates the in memory and on disk
+       objects for the snapshot (marking them as incomplete), takes the lvm
+       snapshot and then updates the status of the in memory and on disk
+       snap objects as complete. Suppose one of the glusterds goes down
+       after taking the lvm snapshot, but before updating the snap object,
+       then treat it as a snapshot create failure and trigger cleanup.
+       i.e the number of commit responses received by the originator
+       glusterd shold be the same as the number of peers it has sent the
+       request to (i.e npeers variable). If not, then originator glusterd
+       will initiate cleanup in post-validate fop.
+       Question: What if one of the other glusterds goes down as explained
+       above and along with it the originator glusterd also goes down?
+       Who will initiate the cleanup?
+    */
+    ret = dict_set_int32n(req_dict, "cleanup", SLEN("cleanup"), 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set dict");
+        goto unbarrier;
+    }
+
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        /* If the main op fails, we should save the error string.
+           Because, op_errstr will be used for unbarrier and
+           unlock ops also. We might lose the actual error that
+           caused the failure.
         */
-        ret = dict_set_int32 (req_dict, "cleanup", 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "failed to set dict");
-                goto unbarrier;
-        }
+        cli_errstr = op_errstr;
+        op_errstr = NULL;
+        goto unbarrier;
+    }
 
-        ret = glusterd_mgmt_v3_commit (op, dict, req_dict, &op_errstr,
-                                       &op_errno, txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL,  "Commit Op Failed");
-                /* If the main op fails, we should save the error string.
-                   Because, op_errstr will be used for unbarrier and
-                   unlock ops also. We might lose the actual error that
-                   caused the failure.
-                */
-                cli_errstr = op_errstr;
-                op_errstr = NULL;
-                goto unbarrier;
-        }
-
-        success = _gf_true;
+    success = _gf_true;
 unbarrier:
-        /* Set the operation type as post, so that differentiation can be
-         * made whether the brickop is sent during pre-commit or post-commit
-         */
-        ret = dict_set_dynstr_with_alloc (req_dict, "operation-type", "post");
+    /* Set the operation type as post, so that differentiation can be
+     * made whether the brickop is sent during pre-commit or post-commit
+     */
+    ret = dict_set_dynstr_with_alloc(req_dict, "operation-type", "post");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "operation-type in dictionary");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr,
+                                    txn_generation);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick Ops Failed");
+        goto out;
+    }
+
+    /*Do a quorum check if the commit phase is successful*/
+    if (success) {
+        // quorum check of the snapshot volume
+        ret = glusterd_snap_quorum_check(dict, _gf_true, &op_errstr, &op_errno);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "operation-type in dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+                   "Snapshot Volume quorum check failed");
+            goto out;
         }
+    }
 
-        ret = glusterd_mgmt_v3_brick_op (op, req_dict, &op_errstr,
-                                         txn_generation);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_OP_FAIL, "Brick Ops Failed");
-                goto out;
-        }
-
-        /*Do a quorum check if the commit phase is successful*/
-        if (success) {
-                //quorum check of the snapshot volume
-                ret = glusterd_snap_quorum_check (dict, _gf_true, &op_errstr,
-                                                  &op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_QUORUM_CHECK_FAIL,
-                                "Snapshot Volume quorum check failed");
-                        goto out;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        op_ret = ret;
+    op_ret = ret;
 
-        if (success == _gf_false)
-                op_ret = -1;
+    if (success == _gf_false)
+        op_ret = -1;
 
-        /* POST-COMMIT VALIDATE PHASE */
-        ret = glusterd_mgmt_v3_post_validate (op, op_ret, dict, req_dict,
-                                              &op_errstr, txn_generation);
+    /* POST-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_post_validate(op, op_ret, dict, req_dict, &op_errstr,
+                                         txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation Failed");
+        op_ret = -1;
+    }
+
+    /* UNLOCK PHASE FOR PEERS*/
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
+
+    /* If the commit op (snapshot taking) failed, then the error is stored
+       in cli_errstr and unbarrier is called. Suppose, if unbarrier also
+       fails, then the error happened in unbarrier is logged and freed.
+       The error happened in commit op, which is stored in cli_errstr
+       is sent to cli.
+    */
+    if (cli_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+        op_errstr = cli_errstr;
+    }
+
+    /* LOCAL VOLUME(S) UNLOCK */
+    if (is_acquired) {
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL, "Post Validation Failed");
-                op_ret = -1;
-        }
-
-        /* UNLOCK PHASE FOR PEERS*/
-        (void) glusterd_mgmt_v3_release_peer_locks (op, dict, op_ret,
-                                                    &op_errstr, is_acquired,
-                                                    txn_generation);
-
-        /* If the commit op (snapshot taking) failed, then the error is stored
-           in cli_errstr and unbarrier is called. Suppose, if unbarrier also
-           fails, then the error happened in unbarrier is logged and freed.
-           The error happened in commit op, which is stored in cli_errstr
-           is sent to cli.
-        */
-        if (cli_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-                op_errstr = cli_errstr;
-        }
-
-        /* LOCAL VOLUME(S) UNLOCK */
-        if (is_acquired) {
-                /* Trying to release multiple mgmt_v3 locks */
-                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                "Failed to release mgmt_v3 locks on localhost");
-                        op_ret = ret;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on localhost");
+            op_ret = ret;
         }
+    }
 
-        if (op_ret && (op_errno == 0))
-                op_errno = EG_INTRNL;
+    if (op_ret && (op_errno == 0))
+        op_errno = EG_INTRNL;
 
-        /* SEND CLI RESPONSE */
-        glusterd_op_send_cli_response (op, op_ret, op_errno, req,
-                                       dict, op_errstr);
+    /* SEND CLI RESPONSE */
+    glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict, op_errstr);
 
-        if (req_dict)
-                dict_unref (req_dict);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        if (tmp_dict)
-                dict_unref (tmp_dict);
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        if (op_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-        }
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
index bf87ec710f1..27dd1849519 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
@@ -10,68 +10,88 @@
 #ifndef _GLUSTERD_MGMT_H_
 #define _GLUSTERD_MGMT_H_
 
-void gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
-                                char *op_errstr, int op_code, uuid_t peerid,
-                                u_char *uuid);
+void
+gd_mgmt_v3_collate_errors(struct syncargs *args, int op_ret, int op_errno,
+                          char *op_errstr, int op_code, uuid_t peerid,
+                          u_char *uuid);
 
 int32_t
-gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
-                            char **op_errstr, dict_t *rsp_dict,
-                            uint32_t *op_errno);
+gd_mgmt_v3_pre_validate_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict, uint32_t *op_errno);
 
 int32_t
-gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
-                       char **op_errstr, dict_t *rsp_dict);
+gd_mgmt_v3_brick_op_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                       dict_t *rsp_dict);
 
 int32_t
-gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
-                      char **op_errstr, uint32_t *op_errno,
-                      dict_t *rsp_dict);
+gd_mgmt_v3_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                     uint32_t *op_errno, dict_t *rsp_dict);
 
 int32_t
-gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+gd_mgmt_v3_post_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          uint32_t *op_errno, dict_t *rsp_dict);
+
+int32_t
+gd_mgmt_v3_post_validate_fn(glusterd_op_t op, int32_t op_ret, dict_t *dict,
                             char **op_errstr, dict_t *rsp_dict);
 
 int32_t
-glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+glusterd_mgmt_v3_initiate_all_phases(rpcsvc_request_t *req, glusterd_op_t op,
                                      dict_t *dict);
 
 int32_t
-glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(rpcsvc_request_t *req,
+                                                        glusterd_op_t op,
+                                                        dict_t *dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op,
                                       dict_t *dict);
 
 int
-glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src);
+glusterd_snap_pre_validate_use_rsp_dict(dict_t *dst, dict_t *src);
 
 int32_t
-glusterd_set_barrier_value (dict_t *dict, char *option);
+glusterd_set_barrier_value(dict_t *dict, char *option);
 int
 
-glusterd_mgmt_v3_initiate_lockdown (glusterd_op_t op, dict_t *dict,
-                                    char **op_errstr, uint32_t *op_errno,
-                                    gf_boolean_t  *is_acquired,
-                                    uint32_t txn_generation);
+glusterd_mgmt_v3_initiate_lockdown(glusterd_op_t op, dict_t *dict,
+                                   char **op_errstr, uint32_t *op_errno,
+                                   gf_boolean_t *is_acquired,
+                                   uint32_t txn_generation);
 
 int
-glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
-                                glusterd_op_t op);
+glusterd_mgmt_v3_build_payload(dict_t **req, char **op_errstr, dict_t *dict,
+                               glusterd_op_t op);
 
 int
-glusterd_mgmt_v3_pre_validate (glusterd_op_t op, dict_t *req_dict,
-                               char **op_errstr, uint32_t *op_errno,
-                               uint32_t txn_generation);
+glusterd_mgmt_v3_pre_validate(glusterd_op_t op, dict_t *req_dict,
+                              char **op_errstr, uint32_t *op_errno,
+                              uint32_t txn_generation);
 
 int
-glusterd_mgmt_v3_commit (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
-                         char **op_errstr, uint32_t *op_errno,
-                         uint32_t txn_generation);
+glusterd_mgmt_v3_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                        char **op_errstr, uint32_t *op_errno,
+                        uint32_t txn_generation);
 
 int
-glusterd_mgmt_v3_release_peer_locks (glusterd_op_t op, dict_t *dict,
-                                     int32_t op_ret, char **op_errstr,
-                                     gf_boolean_t  is_acquired,
-                                     uint32_t txn_generation);
+glusterd_mgmt_v3_release_peer_locks(glusterd_op_t op, dict_t *dict,
+                                    int32_t op_ret, char **op_errstr,
+                                    gf_boolean_t is_acquired,
+                                    uint32_t txn_generation);
 
 int32_t
-glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid);
+glusterd_multiple_mgmt_v3_unlock(dict_t *dict, uuid_t uuid);
+
+int
+glusterd_reset_brick_prevalidate(dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict);
+int
+glusterd_op_reset_brick(dict_t *dict, dict_t *rsp_dict);
+
+int
+glusterd_post_commit_add_brick(dict_t *dict, char **op_errstr);
+
+int
+glusterd_post_commit_replace_brick(dict_t *dict, char **op_errstr);
 #endif /* _GLUSTERD_MGMT_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
index 7c069ced984..645d845ee76 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
@@ -11,688 +11,711 @@
 #include <fnmatch.h>
 #include <pwd.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "dict.h"
-#include "list.h"
-#include "logging.h"
-#include "syscall.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/run.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-utils.h"
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd-mountbroker.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-messages.h"
 
 static int
-seq_dict_foreach (dict_t *dict,
-                  int (*fn)(char *str, void *data),
-                  void *data)
+seq_dict_foreach(dict_t *dict, int (*fn)(char *str, void *data), void *data)
 {
-        char index[] = "4294967296"; // 1<<32
-        int        i = 0;
-        char    *val = NULL;
-        int      ret = 0;
-
-        for (;;i++) {
-                snprintf(index, sizeof(index), "%d", i);
-                ret = dict_get_str (dict, index, &val);
-                if (ret != 0)
-                        return ret == -ENOENT ? 0 : ret;
-                ret = fn (val, data);
-                if (ret != 0)
-                        return ret;
-        }
+    char index[] = "4294967296";  // 1<<32
+    int i = 0;
+    char *val = NULL;
+    int ret = 0;
+
+    for (;; i++) {
+        snprintf(index, sizeof(index), "%d", i);
+        ret = dict_get_str(dict, index, &val);
+        if (ret != 0)
+            return ret == -ENOENT ? 0 : ret;
+        ret = fn(val, data);
+        if (ret != 0)
+            return ret;
+    }
 }
 
 int
-parse_mount_pattern_desc (gf_mount_spec_t *mspec, char *pdesc)
+parse_mount_pattern_desc(gf_mount_spec_t *mspec, char *pdesc)
 #define SYNTAX_ERR -2
 {
-        char *curs              = NULL;
-        char *c2                = NULL;
-        char sc                 = '\0';
-        char **cc               = NULL;
-        gf_mount_pattern_t *pat = NULL;
-        int pnum                = 0;
-        int ret                 = 0;
-        int lastsup             = -1;
-        int incl                = -1;
-        char **pcc              = NULL;
-        int pnc                 = 0;
-
-        skipwhite (&pdesc);
-
-        /* a bow to theory */
-        if (!*pdesc)
-                return 0;
-
-        /* count number of components, separated by '&' */
-        mspec->len = 0;
-        for (curs = pdesc; *curs; curs++) {
-                if (*curs == ')')
-                        mspec->len++;
+    char *curs = NULL;
+    char *c2 = NULL;
+    char sc = '\0';
+    char **cc = NULL;
+    gf_mount_pattern_t *pat = NULL;
+    int pnum = 0;
+    int ret = 0;
+    int lastsup = -1;
+    int incl = -1;
+    char **pcc = NULL;
+    int pnc = 0;
+
+    skipwhite(&pdesc);
+
+    /* a bow to theory */
+    if (!*pdesc)
+        return 0;
+
+    /* count number of components, separated by '&' */
+    mspec->len = 0;
+    for (curs = pdesc; *curs; curs++) {
+        if (*curs == ')')
+            mspec->len++;
+    }
+
+    mspec->patterns = GF_CALLOC(mspec->len, sizeof(*mspec->patterns),
+                                gf_gld_mt_mount_pattern);
+    if (!mspec->patterns) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    pat = mspec->patterns;
+    curs = pdesc;
+    skipwhite(&curs);
+    for (;;) {
+        incl = -1;
+
+        /* check for pattern signedness modifier */
+        if (*curs == '-') {
+            pat->negative = _gf_true;
+            curs++;
         }
 
-        mspec->patterns = GF_CALLOC (mspec->len, sizeof (*mspec->patterns),
-                                     gf_gld_mt_mount_pattern);
-        if (!mspec->patterns) {
-                ret = -1;
-                goto out;
+        /* now should come condition specifier,
+         * then opening paren
+         */
+        c2 = nwstrtail(curs, "SUB(");
+        if (c2) {
+            pat->condition = SET_SUB;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "SUP(");
+        if (c2) {
+            pat->condition = SET_SUPER;
+            lastsup = pat - mspec->patterns;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "EQL(");
+        if (c2) {
+            pat->condition = SET_EQUAL;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "MEET(");
+        if (c2) {
+            pat->condition = SET_INTERSECT;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "SUB+(");
+        if (c2) {
+            pat->condition = SET_SUB;
+            incl = lastsup;
+            goto got_cond;
         }
 
-        pat = mspec->patterns;
-        curs = pdesc;
-        skipwhite (&curs);
-        for (;;) {
-                incl = -1;
-
-                /* check for pattern signedness modifier */
-                if (*curs == '-') {
-                        pat->negative = _gf_true;
-                        curs++;
-                }
-
-                /* now should come condition specifier,
-                 * then opening paren
-                 */
-                c2 = nwstrtail (curs, "SUB(");
-                if (c2) {
-                        pat->condition = SET_SUB;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "SUP(");
-                if (c2) {
-                        pat->condition = SET_SUPER;
-                        lastsup = pat - mspec->patterns;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "EQL(");
-                if (c2) {
-                        pat->condition = SET_EQUAL;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "MEET(");
-                if (c2) {
-                        pat->condition = SET_INTERSECT;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "SUB+(");
-                if (c2) {
-                        pat->condition = SET_SUB;
-                        incl = lastsup;
-                        goto got_cond;
-                }
+        ret = SYNTAX_ERR;
+        goto out;
 
+    got_cond:
+        curs = c2;
+        skipwhite(&curs);
+        /* count the number of components for pattern */
+        pnum = *curs == ')' ? 0 : 1;
+        for (c2 = curs; *c2 != ')';) {
+            if (strchr("&|", *c2)) {
                 ret = SYNTAX_ERR;
                 goto out;
-
- got_cond:
-                curs = c2;
-                skipwhite (&curs);
-                /* count the number of components for pattern */
-                pnum = *curs == ')' ? 0 : 1;
-                for (c2 = curs ;*c2 != ')';) {
-                        if (strchr ("&|", *c2)) {
-                                ret = SYNTAX_ERR;
-                                goto out;
-                        }
-                        while (!strchr ("|&)", *c2) && !isspace (*c2))
-                                c2++;
-                        skipwhite (&c2);
-                        switch (*c2) {
-                        case ')':
-                                break;
-                        case '\0':
-                        case '&':
-                                ret = SYNTAX_ERR;
-                                goto out;
-                        case '|':
-                                *c2 = ' ';
-                                skipwhite (&c2);
-                                /* fall through */
-                        default:
-                                pnum++;
-                        }
-                }
-                if (incl >= 0) {
-                        pnc = 0;
-                        for (pcc = mspec->patterns[incl].components; *pcc; pcc++)
-                                pnc++;
-                        pnum += pnc;
-                }
-                pat->components = GF_CALLOC (pnum + 1, sizeof (*pat->components),
-                                             gf_gld_mt_mount_comp_container);
-                if (!pat->components) {
-                        ret = -1;
-                        goto out;
-                }
-
-                cc = pat->components;
-                /* copy over included component set */
-                if (incl >= 0) {
-                        memcpy (pat->components,
-                                mspec->patterns[incl].components,
-                                pnc * sizeof (*pat->components));
-                        cc += pnc;
-                }
-                /* parse and add components */
-                c2 = ""; /* reset c2 */
-                while (*c2 != ')') {
-                        c2 = curs;
-                        while (!isspace (*c2) && *c2 != ')')
-                                c2++;
-                        sc = *c2;
-                        *c2 = '\0';;
-                        *cc = gf_strdup (curs);
-                        if (!*cc) {
-                                ret = -1;
-                                goto out;
-                        }
-                        *c2 = sc;
-                        skipwhite (&c2);
-                        curs = c2;
-                        cc++;
-                }
-
-                curs++;
-                skipwhite (&curs);
-                if (*curs == '&') {
-                        curs++;
-                        skipwhite (&curs);
-                }
-
-                if (!*curs)
-                        break;
-                pat++;
+            }
+            while (!strchr("|&)", *c2) && !isspace(*c2))
+                c2++;
+            skipwhite(&c2);
+            switch (*c2) {
+                case ')':
+                    break;
+                case '\0':
+                case '&':
+                    ret = SYNTAX_ERR;
+                    goto out;
+                case '|':
+                    *c2 = ' ';
+                    skipwhite(&c2);
+                    /* fall through */
+                default:
+                    pnum++;
+            }
         }
-
- out:
-        if (ret == SYNTAX_ERR) {
-                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "cannot parse mount patterns %s",
-                        pdesc);
+        if (incl >= 0) {
+            pnc = 0;
+            for (pcc = mspec->patterns[incl].components; *pcc; pcc++)
+                pnc++;
+            pnum += pnc;
         }
-
-        /* We've allocted a lotta stuff here but don't bother with freeing
-         * on error, in that case we'll terminate anyway
-         */
-        return ret ? -1 : 0;
-}
-#undef SYNTAX_ERR
-
-
-const char *georep_mnt_desc_template =
-        "SUP("
-                "aux-gfid-mount "
-                "acl "
-                "volfile-server=localhost "
-                "client-pid=%d "
-                "user-map-root=%s "
-        ")"
-        "SUB+("
-                "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"*/* "
-                "log-level=* "
-                "volfile-id=* "
-        ")"
-        "MEET("
-                "%s"
-        ")";
-
-const char *hadoop_mnt_desc_template =
-        "SUP("
-                "volfile-server=%s "
-                "client-pid=%d "
-                "volfile-id=%s "
-                "user-map-root=%s "
-        ")"
-        "SUB+("
-                "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GHADOOP"*/* "
-                "log-level=* "
-        ")";
-
-int
-make_georep_mountspec (gf_mount_spec_t *mspec, const char *volnames,
-                       char *user)
-{
-        char *georep_mnt_desc = NULL;
-        char *meetspec        = NULL;
-        char *vols            = NULL;
-        char *vol             = NULL;
-        char *p               = NULL;
-        char *savetok         = NULL;
-        char *fa[3]           = {0,};
-        size_t siz            = 0;
-        int vc                = 0;
-        int i                 = 0;
-        int ret               = 0;
-
-        vols = gf_strdup ((char *)volnames);
-        if (!vols)
-                goto out;
-
-        for (vc = 1, p = vols; *p; p++) {
-                if (*p == ',')
-                        vc++;
+        pat->components = GF_CALLOC(pnum + 1, sizeof(*pat->components),
+                                    gf_gld_mt_mount_comp_container);
+        if (!pat->components) {
+            ret = -1;
+            goto out;
         }
-        siz = strlen (volnames) + vc * strlen("volfile-id=");
-        meetspec = GF_CALLOC (1, siz + 1, gf_gld_mt_georep_meet_spec);
-        if (!meetspec)
-                goto out;
 
-        for (p = vols;;) {
-                vol = strtok_r (p, ",", &savetok);
-                if (!vol) {
-                        GF_ASSERT (vc == 0);
-                        break;
-                }
-                p = NULL;
-                strcat (meetspec, "volfile-id=");
-                strcat (meetspec, vol);
-                if (--vc > 0)
-                        strcat (meetspec, " ");
+        cc = pat->components;
+        /* copy over included component set */
+        if (incl >= 0) {
+            memcpy(pat->components, mspec->patterns[incl].components,
+                   pnc * sizeof(*pat->components));
+            cc += pnc;
         }
-
-        ret = gf_asprintf (&georep_mnt_desc, georep_mnt_desc_template,
-                           GF_CLIENT_PID_GSYNCD, user, meetspec);
-        if (ret == -1) {
-                georep_mnt_desc = NULL;
+        /* parse and add components */
+        c2 = ""; /* reset c2 */
+        while (*c2 != ')') {
+            c2 = curs;
+            while (!isspace(*c2) && *c2 != ')')
+                c2++;
+            sc = *c2;
+            *c2 = '\0';
+            ;
+            *cc = gf_strdup(curs);
+            if (!*cc) {
+                ret = -1;
                 goto out;
+            }
+            *c2 = sc;
+            skipwhite(&c2);
+            curs = c2;
+            cc++;
         }
 
-        ret = parse_mount_pattern_desc (mspec, georep_mnt_desc);
-
- out:
-        fa[0] = meetspec;
-        fa[1] = vols;
-        fa[2] = georep_mnt_desc;
-
-        for (i = 0; i < 3; i++) {
-                if (fa[i] == NULL)
-                        ret = -1;
-                else
-                        GF_FREE (fa[i]);
+        curs++;
+        skipwhite(&curs);
+        if (*curs == '&') {
+            curs++;
+            skipwhite(&curs);
         }
 
-        return ret;
+        if (!*curs)
+            break;
+        pat++;
+    }
+
+out:
+    if (ret == SYNTAX_ERR) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "cannot parse mount patterns %s", pdesc);
+    }
+
+    /* We've allocted a lotta stuff here but don't bother with freeing
+     * on error, in that case we'll terminate anyway
+     */
+    return ret ? -1 : 0;
 }
+#undef SYNTAX_ERR
+
+const char *georep_mnt_desc_template =
+    "SUP("
+    "aux-gfid-mount "
+    "acl "
+    "volfile-server=localhost "
+    "client-pid=%d "
+    "user-map-root=%s "
+    ")"
+    "SUB+("
+    "log-file=%s/" GEOREP
+    "*/* "
+    "log-level=* "
+    "volfile-id=* "
+    ")"
+    "MEET("
+    "%s"
+    ")";
 
 int
-make_ghadoop_mountspec (gf_mount_spec_t *mspec, const char *volname,
-                        char *user, char *server)
+make_georep_mountspec(gf_mount_spec_t *mspec, const char *volnames, char *user,
+                      char *logdir)
 {
-        char *hadoop_mnt_desc = NULL;
-        int   ret             = 0;
-
-        ret = gf_asprintf (&hadoop_mnt_desc, hadoop_mnt_desc_template,
-                           server, GF_CLIENT_PID_HADOOP, volname, user);
-        if (ret == -1)
-                return ret;
-
-        return parse_mount_pattern_desc (mspec, hadoop_mnt_desc);
+    char *georep_mnt_desc = NULL;
+    char *meetspec = NULL;
+    char *vols = NULL;
+    char *vol = NULL;
+    char *p = NULL;
+    char *savetok = NULL;
+    char *fa[3] = {
+        0,
+    };
+    size_t siz = 0;
+    int vc = 0;
+    int i = 0;
+    int ret = 0;
+
+    vols = gf_strdup((char *)volnames);
+    if (!vols) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "Volume name=%s", volnames, NULL);
+        goto out;
+    }
+
+    for (vc = 1, p = vols; *p; p++) {
+        if (*p == ',')
+            vc++;
+    }
+    siz = strlen(volnames) + vc * SLEN("volfile-id=");
+    meetspec = GF_CALLOC(1, siz + 1, gf_gld_mt_georep_meet_spec);
+    if (!meetspec) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    for (p = vols;;) {
+        vol = strtok_r(p, ",", &savetok);
+        if (!vol) {
+            GF_ASSERT(vc == 0);
+            break;
+        }
+        p = NULL;
+        strcat(meetspec, "volfile-id=");
+        strcat(meetspec, vol);
+        if (--vc > 0)
+            strcat(meetspec, " ");
+    }
+
+    ret = gf_asprintf(&georep_mnt_desc, georep_mnt_desc_template,
+                      GF_CLIENT_PID_GSYNCD, user, logdir, meetspec);
+    if (ret == -1) {
+        georep_mnt_desc = NULL;
+        goto out;
+    }
+
+    ret = parse_mount_pattern_desc(mspec, georep_mnt_desc);
+
+out:
+    fa[0] = meetspec;
+    fa[1] = vols;
+    fa[2] = georep_mnt_desc;
+
+    for (i = 0; i < 3; i++) {
+        if (fa[i] == NULL)
+            ret = -1;
+        else
+            GF_FREE(fa[i]);
+    }
+
+    return ret;
 }
 
 static gf_boolean_t
-match_comp (char *str, char *patcomp)
+match_comp(char *str, char *patcomp)
 {
-        char *c1 = patcomp;
-        char *c2 = str;
-
-        GF_ASSERT (c1);
-        GF_ASSERT (c2);
-
-        while (*c1 == *c2) {
-                if (!*c1)
-                        return _gf_true;
-                c1++;
-                c2++;
-                if (c1[-1] == '=')
-                        break;
-        }
-
-        return fnmatch (c1, c2, 0) == 0 ? _gf_true : _gf_false;
+    char *c1 = patcomp;
+    char *c2 = str;
+
+    GF_ASSERT(c1);
+    GF_ASSERT(c2);
+
+    while (*c1 == *c2) {
+        if (!*c1)
+            return _gf_true;
+        c1++;
+        c2++;
+        if (c1[-1] == '=')
+            break;
+    }
+
+    return fnmatch(c1, c2, 0) == 0 ? _gf_true : _gf_false;
 }
 
 struct gf_set_descriptor {
-        gf_boolean_t priv[2];
-        gf_boolean_t common;
+    gf_boolean_t priv[2];
+    gf_boolean_t common;
 };
 
 static int
-_gf_set_dict_iter1 (char *val, void *data)
+_gf_set_dict_iter1(char *val, void *data)
 {
-        void **dataa                 = data;
-        struct gf_set_descriptor *sd = dataa[0];
-        char **curs                  = dataa[1];
-        gf_boolean_t priv            = _gf_true;
-
-        while (*curs) {
-                if (match_comp (val, *curs)) {
-                        priv = _gf_false;
-                        sd->common = _gf_true;
-                }
-                curs++;
+    void **dataa = data;
+    struct gf_set_descriptor *sd = dataa[0];
+    char **curs = dataa[1];
+    gf_boolean_t priv = _gf_true;
+
+    while (*curs) {
+        if (match_comp(val, *curs)) {
+            priv = _gf_false;
+            sd->common = _gf_true;
         }
+        curs++;
+    }
 
-        if (priv)
-                sd->priv[0] = _gf_true;
+    if (priv)
+        sd->priv[0] = _gf_true;
 
-        return 0;
+    return 0;
 }
 
 static int
-_gf_set_dict_iter2 (char *val, void *data)
+_gf_set_dict_iter2(char *val, void *data)
 {
-        void **dataa      = data;
-        gf_boolean_t *boo = dataa[0];
-        char *comp        = dataa[1];
+    void **dataa = data;
+    gf_boolean_t *boo = dataa[0];
+    char *comp = dataa[1];
 
-        if (match_comp (val, comp))
-                *boo = _gf_true;
+    if (match_comp(val, comp))
+        *boo = _gf_true;
 
-        return 0;
+    return 0;
 }
 
 static void
-relate_sets (struct gf_set_descriptor *sd, dict_t *argdict, char **complist)
+relate_sets(struct gf_set_descriptor *sd, dict_t *argdict, char **complist)
 {
-        void *dataa[] = {NULL, NULL};
-        gf_boolean_t boo = _gf_false;
+    void *dataa[] = {NULL, NULL};
+    gf_boolean_t boo = _gf_false;
 
-        memset (sd, 0, sizeof (*sd));
+    memset(sd, 0, sizeof(*sd));
 
-        dataa[0] = sd;
-        dataa[1] = complist;
-        seq_dict_foreach (argdict, _gf_set_dict_iter1, dataa);
+    dataa[0] = sd;
+    dataa[1] = complist;
+    seq_dict_foreach(argdict, _gf_set_dict_iter1, dataa);
 
-        while (*complist) {
-                boo = _gf_false;
-                dataa[0] = &boo;
-                dataa[1] = *complist;
-                seq_dict_foreach (argdict, _gf_set_dict_iter2, dataa);
+    while (*complist) {
+        boo = _gf_false;
+        dataa[0] = &boo;
+        dataa[1] = *complist;
+        seq_dict_foreach(argdict, _gf_set_dict_iter2, dataa);
 
-                if (boo)
-                        sd->common = _gf_true;
-                else
-                        sd->priv[1] = _gf_true;
+        if (boo)
+            sd->common = _gf_true;
+        else
+            sd->priv[1] = _gf_true;
 
-                complist++;
-        }
+        complist++;
+    }
 }
 
 static int
-_arg_parse_uid (char *val, void *data)
+_arg_parse_uid(char *val, void *data)
 {
-        char *user        = strtail (val, "user-map-root=");
-        struct passwd *pw = NULL;
+    char *user = strtail(val, "user-map-root=");
+    struct passwd *pw = NULL;
 
-        if (!user)
-                return 0;
-        pw = getpwnam (user);
-        if (!pw)
-                return -EINVAL;
+    if (!user)
+        return 0;
+    pw = getpwnam(user);
+    if (!pw)
+        return -EINVAL;
 
-        if (*(int *)data >= 0)
-                /* uid ambiguity, already found */
-                return -EINVAL;
+    if (*(int *)data >= 0)
+        /* uid ambiguity, already found */
+        return -EINVAL;
 
-        *(int *)data = pw->pw_uid;
-        return 0;
+    *(int *)data = pw->pw_uid;
+    return 0;
 }
 
 static int
-evaluate_mount_request (gf_mount_spec_t *mspec, dict_t *argdict)
+evaluate_mount_request(xlator_t *this, gf_mount_spec_t *mspec, dict_t *argdict)
 {
-        struct gf_set_descriptor sd = {{0,},};
-        int i                       = 0;
-        int uid                     = -1;
-        int ret                     = 0;
-        gf_boolean_t match          = _gf_false;
-
-        for (i = 0; i < mspec->len; i++) {
-                relate_sets (&sd, argdict, mspec->patterns[i].components);
-                switch (mspec->patterns[i].condition) {
-                case SET_SUB:
-                        match = !sd.priv[0];
-                        break;
-                case SET_SUPER:
-                        match = !sd.priv[1];
-                        break;
-                case SET_EQUAL:
-                        match = (!sd.priv[0] && !sd.priv[1]);
-                        break;
-                case SET_INTERSECT:
-                        match = sd.common;
-                        break;
-                default:
-                        GF_ASSERT(!"unreached");
-                }
-                if (mspec->patterns[i].negative)
-                        match = !match;
-
-                if (!match)
-                        return -EPERM;
+    struct gf_set_descriptor sd = {
+        {
+            0,
+        },
+    };
+    int i = 0;
+    int uid = -1;
+    int ret = 0;
+    gf_boolean_t match = _gf_false;
+
+    for (i = 0; i < mspec->len; i++) {
+        relate_sets(&sd, argdict, mspec->patterns[i].components);
+        switch (mspec->patterns[i].condition) {
+            case SET_SUB:
+                match = !sd.priv[0];
+                break;
+            case SET_SUPER:
+                match = !sd.priv[1];
+                break;
+            case SET_EQUAL:
+                match = (!sd.priv[0] && !sd.priv[1]);
+                break;
+            case SET_INTERSECT:
+                match = sd.common;
+                break;
+            default:
+                GF_ASSERT(!"unreached");
         }
+        if (mspec->patterns[i].negative)
+            match = !match;
+
+        if (!match) {
+            gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                   GD_MSG_MNTBROKER_SPEC_MISMATCH,
+                   "Mountbroker spec mismatch!!! SET: %d "
+                   "COMPONENT: %d. Review the mount args passed",
+                   mspec->patterns[i].condition, i);
+            return -EPERM;
+        }
+    }
 
-        ret = seq_dict_foreach (argdict, _arg_parse_uid, &uid);
-        if (ret != 0)
-                return ret;
+    ret = seq_dict_foreach(argdict, _arg_parse_uid, &uid);
+    if (ret != 0)
+        return ret;
 
-        return uid;
+    return uid;
 }
 
 static int
-_volname_get (char *val, void *data)
+_volname_get(char *val, void *data)
 {
-        char **volname = data;
+    char **volname = data;
 
-        *volname = strtail (val, "volfile-id=");
+    *volname = strtail(val, "volfile-id=");
 
-        return *volname ? 1 : 0;
+    return *volname ? 1 : 0;
 }
 
 static int
-_runner_add (char *val, void *data)
+_runner_add(char *val, void *data)
 {
-        runner_t *runner = data;
+    runner_t *runner = data;
 
-        runner_argprintf (runner, "--%s", val);
+    runner_argprintf(runner, "--%s", val);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno)
+glusterd_do_mount(char *label, dict_t *argdict, char **path, int *op_errno)
 {
-        glusterd_conf_t *priv      = NULL;
-        char *mountbroker_root     = NULL;
-        gf_mount_spec_t *mspec     = NULL;
-        int uid                    = -ENOENT;
-        char *volname              = NULL;
-        glusterd_volinfo_t *vol    = NULL;
-        char *mtptemp              = NULL;
-        char *mntlink              = NULL;
-        char *cookieswitch         = NULL;
-        char *cookie               = NULL;
-        char *sla                  = NULL;
-        struct stat st             = {0,};
-        runner_t runner            = {0,};
-        int ret                    = 0;
-        xlator_t *this             = THIS;
-        mode_t orig_umask          = 0;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (op_errno);
-        *op_errno = 0;
-
-        if (dict_get_str (this->options, "mountbroker-root",
-                          &mountbroker_root) != 0) {
-                *op_errno = ENOENT;
-                goto out;
-        }
-
-        GF_ASSERT (label);
-        if (!*label) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        /* look up spec for label */
-        cds_list_for_each_entry (mspec, &priv->mount_specs,
-                                 speclist) {
-                if (strcmp (mspec->label, label) != 0)
-                        continue;
-                uid = evaluate_mount_request (mspec, argdict);
-                break;
-        }
-        if (uid < 0) {
-                *op_errno = -uid;
-                goto out;
-        }
-
-        /* some sanity check on arguments */
-        seq_dict_foreach (argdict, _volname_get, &volname);
-        if (!volname) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-        if (glusterd_volinfo_find (volname, &vol) != 0 ||
-            !glusterd_is_volume_started (vol)) {
-                *op_errno = ENOENT;
-                goto out;
-        }
-
-        /* go do mount */
-
-        /** create actual mount dir */
-
-        /*** "overload" string name to be possible to used for cookie
-             creation, see below */
-        ret = gf_asprintf (&mtptemp, "%s/user%d/mtpt-%s-XXXXXX/cookie",
-                           mountbroker_root, uid, label);
-        if (ret == -1) {
-                mtptemp = NULL;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-        /*** hide cookie part */
-        cookieswitch = strrchr (mtptemp, '/');
-        *cookieswitch = '\0';
-
-        sla = strrchr (mtptemp, '/');
-        *sla = '\0';
-        ret = sys_mkdir (mtptemp, 0700);
-        if (ret == 0)
-                ret = sys_chown (mtptemp, uid, 0);
-        else if (errno == EEXIST)
-                ret = 0;
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
-        }
-        ret = sys_lstat (mtptemp, &st);
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
-        }
-        if (!(S_ISDIR (st.st_mode) && (st.st_mode & ~S_IFMT) == 0700 &&
-              st.st_uid == uid && st.st_gid == 0)) {
-                *op_errno = EACCES;
-                goto out;
-        }
-        *sla = '/';
-
-        if (!mkdtemp (mtptemp)) {
-                *op_errno = errno;
-                goto out;
-        }
-
-        /** create private "cookie" symlink */
-
-        /*** occupy an entry in the hive dir via mkstemp */
-        ret = gf_asprintf (&cookie, "%s/"MB_HIVE"/mntXXXXXX",
-                           mountbroker_root);
-        if (ret == -1) {
-                cookie = NULL;
-                *op_errno = ENOMEM;
-                goto out;
+    glusterd_conf_t *priv = NULL;
+    char *mountbroker_root = NULL;
+    gf_mount_spec_t *mspec = NULL;
+    int uid = -ENOENT;
+    char *volname = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *mtptemp = NULL;
+    char *mntlink = NULL;
+    char *cookieswitch = NULL;
+    char *cookie = NULL;
+    char *sla = NULL;
+    struct stat st = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = THIS;
+    mode_t orig_umask = 0;
+    gf_boolean_t found_label = _gf_false;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(op_errno);
+    *op_errno = 0;
+
+    if (dict_get_strn(this->options, "mountbroker-root",
+                      SLEN("mountbroker-root"), &mountbroker_root) != 0) {
+        *op_errno = ENOENT;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "'option mountbroker-root' "
+               "missing in glusterd vol file");
+        goto out;
+    }
+
+    GF_ASSERT(label);
+    if (!*label) {
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_MNTBROKER_LABEL_NULL,
+               "label is NULL (%s)", strerror(*op_errno));
+        goto out;
+    }
+
+    /* look up spec for label */
+    cds_list_for_each_entry(mspec, &priv->mount_specs, speclist)
+    {
+        if (strcmp(mspec->label, label) != 0)
+            continue;
+
+        found_label = _gf_true;
+        uid = evaluate_mount_request(this, mspec, argdict);
+        break;
+    }
+    if (uid < 0) {
+        *op_errno = -uid;
+        if (!found_label) {
+            gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                   GD_MSG_MNTBROKER_LABEL_MISS,
+                   "Missing mspec: Check the corresponding option "
+                   "in glusterd vol file for mountbroker user: %s",
+                   label);
         }
-        orig_umask = umask(S_IRWXG | S_IRWXO);
-        ret = mkstemp (cookie);
-        umask(orig_umask);
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
+        goto out;
+    }
+
+    /* some sanity check on arguments */
+    seq_dict_foreach(argdict, _volname_get, &volname);
+    if (!volname) {
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_DICT_GET_FAILED,
+               "Dict get failed for the key 'volname'");
+        goto out;
+    }
+    if (glusterd_volinfo_find(volname, &vol) != 0 ||
+        !glusterd_is_volume_started(vol)) {
+        *op_errno = ENOENT;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_MOUNT_REQ_FAIL,
+               "Either volume is not started or volinfo not found");
+        goto out;
+    }
+
+    /* go do mount */
+
+    /** create actual mount dir */
+
+    /*** "overload" string name to be possible to used for cookie
+         creation, see below */
+    ret = gf_asprintf(&mtptemp, "%s/user%d/mtpt-%s-XXXXXX/cookie",
+                      mountbroker_root, uid, label);
+    if (ret == -1) {
+        mtptemp = NULL;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    /*** hide cookie part */
+    cookieswitch = strrchr(mtptemp, '/');
+    *cookieswitch = '\0';
+
+    sla = strrchr(mtptemp, '/');
+    *sla = '\0';
+    ret = sys_mkdir(mtptemp, 0700);
+    if (ret == 0)
+        ret = sys_chown(mtptemp, uid, 0);
+    else if (errno == EEXIST)
+        ret = 0;
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "Mountbroker User directory creation failed");
+        goto out;
+    }
+    ret = sys_lstat(mtptemp, &st);
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "stat on mountbroker user directory failed");
+        goto out;
+    }
+    if (!(S_ISDIR(st.st_mode) && (st.st_mode & ~S_IFMT) == 0700 &&
+          st.st_uid == uid && st.st_gid == 0)) {
+        *op_errno = EACCES;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_MOUNT_REQ_FAIL,
+               "Incorrect mountbroker user directory attributes");
+        goto out;
+    }
+    *sla = '/';
+
+    if (!mkdtemp(mtptemp)) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "Mountbroker mount directory creation failed");
+        goto out;
+    }
+
+    /** create private "cookie" symlink */
+
+    /*** occupy an entry in the hive dir via mkstemp */
+    ret = gf_asprintf(&cookie, "%s/" MB_HIVE "/mntXXXXXX", mountbroker_root);
+    if (ret == -1) {
+        cookie = NULL;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    orig_umask = umask(S_IRWXG | S_IRWXO);
+    ret = mkstemp(cookie);
+    umask(orig_umask);
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "Mountbroker cookie file creation failed");
+        goto out;
+    }
+    sys_close(ret);
+
+    /*** assembly the path from cookie to mountpoint */
+    sla = strchr(sla - 1, '/');
+    GF_ASSERT(sla);
+    ret = gf_asprintf(&mntlink, "../user%d%s", uid, sla);
+    if (ret == -1) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    /*** create cookie link in (to-be) mountpoint,
+         move it over to the final place */
+    *cookieswitch = '/';
+    ret = sys_symlink(mntlink, mtptemp);
+    if (ret != -1)
+        ret = sys_rename(mtptemp, cookie);
+    *cookieswitch = '\0';
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "symlink or rename failed");
+        goto out;
+    }
+
+    /** invoke glusterfs on the mountpoint */
+
+    runinit(&runner);
+    runner_add_arg(&runner, SBIN_DIR "/glusterfs");
+    seq_dict_foreach(argdict, _runner_add, &runner);
+    runner_add_arg(&runner, mtptemp);
+    ret = runner_run_reuse(&runner);
+    if (ret == -1) {
+        *op_errno = EIO; /* XXX hacky fake */
+        runner_log(&runner, "", GF_LOG_ERROR, "command failed");
+    }
+    runner_end(&runner);
+
+out:
+
+    if (*op_errno) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *op_errno, GD_MSG_MOUNT_REQ_FAIL,
+               "unsuccessful mount request");
+        if (mtptemp) {
+            *cookieswitch = '/';
+            sys_unlink(mtptemp);
+            *cookieswitch = '\0';
+            sys_rmdir(mtptemp);
         }
-        sys_close (ret);
-
-        /*** assembly the path from cookie to mountpoint */
-        sla = strchr (sla - 1, '/');
-        GF_ASSERT (sla);
-        ret = gf_asprintf (&mntlink, "../user%d%s", uid, sla);
-        if (ret == -1) {
-                *op_errno = ENOMEM;
-                goto out;
+        if (cookie) {
+            sys_unlink(cookie);
+            GF_FREE(cookie);
         }
 
-        /*** create cookie link in (to-be) mountpoint,
-             move it over to the final place */
-        *cookieswitch = '/';
-        ret = sys_symlink (mntlink, mtptemp);
-        if (ret != -1)
-                ret = sys_rename (mtptemp, cookie);
-        *cookieswitch = '\0';
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
-        }
-
-        /** invoke glusterfs on the mountpoint */
+    } else {
+        ret = 0;
+        *path = cookie;
+    }
 
-        runinit (&runner);
-        runner_add_arg (&runner, SBIN_DIR"/glusterfs");
-        seq_dict_foreach (argdict, _runner_add, &runner);
-        runner_add_arg (&runner, mtptemp);
-        ret = runner_run_reuse (&runner);
-        if (ret == -1) {
-                *op_errno = EIO; /* XXX hacky fake */
-                runner_log (&runner, "", GF_LOG_ERROR, "command failed");
-        }
-        runner_end (&runner);
+    if (mtptemp)
+        GF_FREE(mtptemp);
+    if (mntlink)
+        GF_FREE(mntlink);
 
- out:
-
-        if (*op_errno) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_WARNING, *op_errno,
-                        GD_MSG_MOUNT_REQ_FAIL,
-                        "unsuccessful mount request (%s)",
-                        strerror (*op_errno));
-                if (mtptemp) {
-                        *cookieswitch = '/';
-                        sys_unlink (mtptemp);
-                        *cookieswitch = '\0';
-                        sys_rmdir (mtptemp);
-                }
-                if (cookie) {
-                        sys_unlink (cookie);
-                        GF_FREE (cookie);
-                }
-
-        } else {
-                ret = 0;
-                *path = cookie;
-        }
-
-        GF_FREE (mtptemp);
-
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
index 83267c203ca..20c1347f52f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
@@ -9,34 +9,29 @@
 */
 #define MB_HIVE "mb_hive"
 
-typedef enum {
-        SET_SUB = 1,
-        SET_SUPER,
-        SET_EQUAL,
-        SET_INTERSECT
-} gf_setrel_t;
+typedef enum { SET_SUB = 1, SET_SUPER, SET_EQUAL, SET_INTERSECT } gf_setrel_t;
 
 struct gf_mount_pattern {
-        char **components;
-        gf_setrel_t condition;
-        gf_boolean_t negative;
+    char **components;
+    gf_setrel_t condition;
+    gf_boolean_t negative;
 };
 typedef struct gf_mount_pattern gf_mount_pattern_t;
 
 struct gf_mount_spec {
-        struct cds_list_head speclist;
-        char *label;
-        gf_mount_pattern_t *patterns;
-        size_t len;
+    struct cds_list_head speclist;
+    char *label;
+    gf_mount_pattern_t *patterns;
+    size_t len;
 };
 typedef struct gf_mount_spec gf_mount_spec_t;
 
+int
+parse_mount_pattern_desc(gf_mount_spec_t *mspec, char *pdesc);
 
-int parse_mount_pattern_desc (gf_mount_spec_t *mspec, char *pdesc);
+int
+make_georep_mountspec(gf_mount_spec_t *mspec, const char *volname, char *user,
+                      char *logdir);
 
-int make_georep_mountspec (gf_mount_spec_t *mspec, const char *volname,
-                           char *user);
-int make_ghadoop_mountspec (gf_mount_spec_t *mspec, const char *volname,
-                            char *user, char *server);
-
-int glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno);
+int
+glusterd_do_mount(char *label, dict_t *argdict, char **path, int *op_errno);
diff --git a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c
index 60b792ffac2..4908dbbc213 100644
--- a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c
@@ -8,8 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#ifdef BUILD_GNFS
+
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
@@ -17,185 +20,209 @@
 #include "glusterd-messages.h"
 #include "glusterd-svc-helper.h"
 
-static char *nfs_svc_name = "nfs";
-
 static gf_boolean_t
-glusterd_nfssvc_need_start ()
+glusterd_nfssvc_need_start()
 {
-        glusterd_conf_t    *priv    = NULL;
-        gf_boolean_t       start   = _gf_false;
-        glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t start = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
 
-        priv = THIS->private;
+    priv = THIS->private;
 
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                if (!glusterd_is_volume_started (volinfo))
-                        continue;
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        if (!glusterd_is_volume_started(volinfo))
+            continue;
 
-                if (dict_get_str_boolean (volinfo->dict, NFS_DISABLE_MAP_KEY, 1))
-                        continue;
-                start = _gf_true;
-                break;
-        }
+        if (dict_get_str_boolean(volinfo->dict, NFS_DISABLE_MAP_KEY, 1))
+            continue;
+        start = _gf_true;
+        break;
+    }
 
-        return start;
-}
-
-int
-glusterd_nfssvc_init (glusterd_svc_t *svc)
-{
-        return glusterd_svc_init (svc, nfs_svc_name);
+    return start;
 }
 
 static int
-glusterd_nfssvc_create_volfile ()
+glusterd_nfssvc_create_volfile()
 {
-        char            filepath[PATH_MAX] = {0,};
-        glusterd_conf_t *conf = THIS->private;
-
-        glusterd_svc_build_volfile_path (nfs_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
-        return glusterd_create_global_volfile (build_nfs_graph,
-                                               filepath, NULL);
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *conf = THIS->private;
+
+    glusterd_svc_build_volfile_path(conf->nfs_svc.name, conf->workdir, filepath,
+                                    sizeof(filepath));
+    return glusterd_create_global_volfile(build_nfs_graph, filepath, NULL);
 }
 
 static int
-glusterd_nfssvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_nfssvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                 ret     = -1;
-
-        if (!svc->inited) {
-                ret = glusterd_nfssvc_init (svc);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FAILED_INIT_NFSSVC, "Failed to init nfs "
-                                "service");
-                        goto out;
-                } else {
-                        svc->inited = _gf_true;
-                        gf_msg_debug (THIS->name, 0, "nfs service initialized");
-                }
+    int ret = -1;
+
+    if (!svc->inited) {
+        ret = glusterd_svc_init(svc, "nfs");
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_NFSSVC,
+                   "Failed to init nfs service");
+            goto out;
+        } else {
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0, "nfs service initialized");
         }
+    }
 
-        ret = svc->stop (svc, SIGKILL);
-        if (ret)
-                goto out;
+    ret = svc->stop(svc, SIGKILL);
+    if (ret)
+        goto out;
 
-        ret = glusterd_nfssvc_create_volfile ();
-        if (ret)
-                goto out;
+    /* not an error, or a (very) soft error at best */
+    if (sys_access(XLATORDIR "/nfs/server.so", R_OK) != 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_GNFS_XLATOR_NOT_INSTALLED,
+               "nfs/server.so xlator is not installed");
+        goto out;
+    }
 
-        if (glusterd_nfssvc_need_start ()) {
-                ret = svc->start (svc, flags);
-                if (ret)
-                        goto out;
+    ret = glusterd_nfssvc_create_volfile();
+    if (ret)
+        goto out;
 
-                ret = glusterd_conn_connect (&(svc->conn));
-                if (ret)
-                        goto out;
-        }
+    if (glusterd_nfssvc_need_start()) {
+        ret = svc->start(svc, flags);
+        if (ret)
+            goto out;
+
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret)
+            goto out;
+    }
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
 
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
 static int
-glusterd_nfssvc_start (glusterd_svc_t *svc, int flags)
+glusterd_nfssvc_start(glusterd_svc_t *svc, int flags)
 {
-        return glusterd_svc_start (svc, flags, NULL);
+    return glusterd_svc_start(svc, flags, NULL);
 }
 
 static int
-glusterd_nfssvc_stop (glusterd_svc_t *svc, int sig)
+glusterd_nfssvc_stop(glusterd_svc_t *svc, int sig)
 {
-        int                    ret        = -1;
-        gf_boolean_t           deregister = _gf_false;
+    int ret = -1;
+    gf_boolean_t deregister = _gf_false;
 
-        if (glusterd_proc_is_running (&(svc->proc)))
-                deregister = _gf_true;
+    if (glusterd_proc_is_running(&(svc->proc)))
+        deregister = _gf_true;
 
-        ret = glusterd_svc_stop (svc, sig);
-        if (ret)
-                goto out;
-        if (deregister)
-                glusterd_nfs_pmap_deregister ();
+    ret = glusterd_svc_stop(svc, sig);
+    if (ret)
+        goto out;
+    if (deregister)
+        glusterd_nfs_pmap_deregister();
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 void
-glusterd_nfssvc_build (glusterd_svc_t *svc)
+glusterd_nfssvc_build(glusterd_svc_t *svc)
 {
-        svc->manager = glusterd_nfssvc_manager;
-        svc->start = glusterd_nfssvc_start;
-        svc->stop = glusterd_nfssvc_stop;
+    svc->manager = glusterd_nfssvc_manager;
+    svc->start = glusterd_nfssvc_start;
+    svc->stop = glusterd_nfssvc_stop;
 }
 
 int
-glusterd_nfssvc_reconfigure ()
+glusterd_nfssvc_reconfigure()
 {
-        int              ret             = -1;
-        xlator_t        *this            = NULL;
-        glusterd_conf_t *priv            = NULL;
-        gf_boolean_t     identical       = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        /*
-         * Check both OLD and NEW volfiles, if they are SAME by size
-         * and cksum i.e. "character-by-character". If YES, then
-         * NOTHING has been changed, just return.
-         */
-        ret = glusterd_svc_check_volfile_identical (priv->nfs_svc.name,
-                                                    build_nfs_graph,
-                                                    &identical);
-        if (ret)
-                goto out;
-
-        if (identical) {
-                ret = 0;
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+    gf_boolean_t vol_started = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    /* not an error, or a (very) soft error at best */
+    if (sys_access(XLATORDIR "/nfs/server.so", R_OK) != 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_GNFS_XLATOR_NOT_INSTALLED,
+               "nfs/server.so xlator is not installed");
+        ret = 0;
+        goto out;
+    }
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+            vol_started = _gf_true;
+            break;
         }
-
-        /*
-         * They are not identical. Find out if the topology is changed
-         * OR just the volume options. If just the options which got
-         * changed, then inform the xlator to reconfigure the options.
-         */
-        identical = _gf_false; /* RESET the FLAG */
-        ret = glusterd_svc_check_topology_identical (priv->nfs_svc.name,
-                                                     build_nfs_graph,
-                                                     &identical);
-        if (ret)
-                goto out;
-
-        /* Topology is not changed, but just the options. But write the
-         * options to NFS volfile, so that NFS will be reconfigured.
-         */
-        if (identical) {
-                ret = glusterd_nfssvc_create_volfile();
-                if (ret == 0) {/* Only if above PASSES */
-                        ret = glusterd_fetchspec_notify (THIS);
-                }
-                goto out;
+    }
+    if (!vol_started) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+
+    ret = glusterd_svc_check_volfile_identical(priv->nfs_svc.name,
+                                               build_nfs_graph, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->nfs_svc.name,
+                                                build_nfs_graph, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to NFS volfile, so that NFS will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_nfssvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
         }
+        goto out;
+    }
 
-        /*
-         * NFS volfile's topology has been changed. NFS server needs
-         * to be RESTARTED to ACT on the changed volfile.
-         */
-        ret = priv->nfs_svc.manager (&(priv->nfs_svc), NULL,
-                                     PROC_START_NO_WAIT);
+    /*
+     * NFS volfile's topology has been changed. NFS server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->nfs_svc.manager(&(priv->nfs_svc), NULL, PROC_START_NO_WAIT);
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
-
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h
index 6330b71ba7d..6bfdde95749 100644
--- a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h
@@ -13,13 +13,15 @@
 
 #include "glusterd-svc-mgmt.h"
 
+#ifdef BUILD_GNFS
 void
-glusterd_nfssvc_build (glusterd_svc_t *svc);
+glusterd_nfssvc_build(glusterd_svc_t *svc);
 
 int
-glusterd_nfssvc_init (glusterd_svc_t *svc);
+glusterd_nfssvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_nfssvc_reconfigure ();
+glusterd_nfssvc_reconfigure();
 
+#endif /* BUILD_GNFS */
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index e9f261c2fb3..c537fc33a85 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -14,4404 +14,4696 @@
 #include <sys/mount.h>
 
 #include <libgen.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "fnmatch.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "protocol-common.h"
 #include "glusterd.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "list.h"
-#include "dict.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "glusterd-sm.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/list.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
-#include "glusterd-hooks.h"
-#include "glusterd-volgen.h"
 #include "glusterd-locks.h"
-#include "glusterd-messages.h"
-#include "glusterd-utils.h"
-#include "syscall.h"
+#include "glusterd-quota.h"
+#include <glusterfs/syscall.h>
 #include "cli1-xdr.h"
-#include "common-utils.h"
-#include "run.h"
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-svc-helper.h"
+#include "glusterd-shd-svc-helper.h"
 #include "glusterd-shd-svc.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-server-quorum.h"
-#include "glusterd-volgen.h"
 #include <sys/types.h>
 #include <signal.h>
 #include <sys/wait.h>
+#include "glusterd-gfproxyd-svc-helper.h"
+
+#define len_strcmp(key, len, str)                                              \
+    ((len == SLEN(str)) && (strcmp(key, str) == 0))
 
 extern char local_node_hostname[PATH_MAX];
 static int
-glusterd_set_shared_storage (dict_t *dict, char *key, char *value,
-                             char **op_errstr);
+glusterd_set_shared_storage(dict_t *dict, char *key, char *value,
+                            char **op_errstr);
 
-/* Valid options for all volumes to be listed in the *
- * valid_all_vol_opts table. To add newer options to *
- * all volumes, we can just add more entries to this *
- * table                                             *
+/*
+ * Valid options for all volumes to be listed in the valid_all_vol_opts table.
+ * To add newer options to all volumes, we can just add more entries to this
+ * table.
+ *
+ * It's important that every value have a default, or have a special handler
+ * in glusterd_get_global_options_for_all_vols, or else we might crash there.
  */
-glusterd_all_vol_opts   valid_all_vol_opts[] = {
-        { GLUSTERD_QUORUM_RATIO_KEY },
-        { GLUSTERD_SHARED_STORAGE_KEY },
-        { NULL },
+const glusterd_all_vol_opts valid_all_vol_opts[] = {
+    {GLUSTERD_QUORUM_RATIO_KEY, "51"},
+    {GLUSTERD_SHARED_STORAGE_KEY, "disable"},
+    /* This one actually gets filled in dynamically. */
+    {GLUSTERD_GLOBAL_OP_VERSION_KEY, "BUG_NO_OP_VERSION"},
+    /*
+     * This one should be filled in dynamically, but it didn't used to be
+     * (before the defaults were added here) so the value is unclear.
+     *
+     * TBD: add a dynamic handler to set the appropriate value
+     */
+    {GLUSTERD_MAX_OP_VERSION_KEY, "BUG_NO_MAX_OP_VERSION"},
+    {GLUSTERD_BRICK_MULTIPLEX_KEY, "disable"},
+    /* Set this value to 0 by default implying brick-multiplexing
+     * behaviour with no limit set on the number of brick instances that
+     * can be attached per process.
+     * TBD: Discuss the default value for this. Maybe this should be a
+     * dynamic value depending on the memory specifications per node */
+    {GLUSTERD_BRICKMUX_LIMIT_KEY, GLUSTERD_BRICKMUX_LIMIT_DFLT_VALUE},
+    {GLUSTERD_VOL_CNT_PER_THRD, GLUSTERD_VOL_CNT_PER_THRD_DEFAULT_VALUE},
+    {GLUSTERD_LOCALTIME_LOGGING_KEY, "disable"},
+    {GLUSTERD_DAEMON_LOG_LEVEL_KEY, "INFO"},
+    {NULL},
 };
 
-#define ALL_VOLUME_OPTION_CHECK(volname, key, ret, op_errstr, label)           \
-        do {                                                                   \
-                gf_boolean_t    _all   = !strcmp ("all", volname);             \
-                gf_boolean_t    _ratio = _gf_false;                            \
-                int32_t         i      = 0;                                    \
-                                                                               \
-                for (i = 0; valid_all_vol_opts[i].option; i++) {               \
-                        if (!strcmp (key, valid_all_vol_opts[i].option)) {     \
-                                _ratio = _gf_true;                             \
-                                break;                                         \
-                        }                                                      \
-                }                                                              \
-                                                                               \
-                if (_all && !_ratio) {                                         \
-                        ret = -1;                                              \
-                        *op_errstr = gf_strdup ("Not a valid option for all "  \
-                                                "volumes");                    \
-                        goto label;                                            \
-                } else if (!_all && _ratio) {                                  \
-                        ret = -1;                                              \
-                        *op_errstr = gf_strdup ("Not a valid option for "      \
-                                                "single volume");              \
-                        goto label;                                            \
-                }                                                              \
-         } while (0)
-
 static struct cds_list_head gd_op_sm_queue;
 synclock_t gd_op_sm_lock;
-glusterd_op_info_t    opinfo = {{0},};
-
-int
-glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
-                                         struct cds_list_head *selected);
-
+glusterd_op_info_t opinfo = {
+    {0},
+};
 
 int32_t
-glusterd_txn_opinfo_dict_init ()
+glusterd_txn_opinfo_dict_init()
 {
-        int32_t             ret    = -1;
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        priv->glusterd_txn_opinfo = dict_new ();
-        if (!priv->glusterd_txn_opinfo) {
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    priv->glusterd_txn_opinfo = dict_new();
+    if (!priv->glusterd_txn_opinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        memset (priv->global_txn_id, '\0', sizeof(uuid_t));
+    memset(priv->global_txn_id, '\0', sizeof(uuid_t));
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-glusterd_txn_opinfo_dict_fini ()
+glusterd_txn_opinfo_dict_fini()
 {
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->glusterd_txn_opinfo)
-                dict_unref (priv->glusterd_txn_opinfo);
+    if (priv->glusterd_txn_opinfo)
+        dict_unref(priv->glusterd_txn_opinfo);
 }
 
 void
-glusterd_txn_opinfo_init (glusterd_op_info_t  *opinfo,
-                          glusterd_op_sm_state_info_t *state, int *op,
-                          dict_t *op_ctx, rpcsvc_request_t *req)
+glusterd_txn_opinfo_init(glusterd_op_info_t *opinfo,
+                         glusterd_op_sm_state_info_t *state, int *op,
+                         dict_t *op_ctx, rpcsvc_request_t *req)
 {
-        glusterd_conf_t *conf = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        GF_ASSERT (opinfo);
+    GF_ASSERT(opinfo);
 
-        conf = THIS->private;
-        GF_ASSERT (conf);
+    conf = THIS->private;
+    GF_ASSERT(conf);
 
-        if (state)
-                opinfo->state = *state;
+    if (state)
+        opinfo->state = *state;
 
-        if (op)
-                opinfo->op = *op;
+    if (op)
+        opinfo->op = *op;
 
-        if (op_ctx)
-                opinfo->op_ctx = dict_ref(op_ctx);
-        else
-                opinfo->op_ctx = NULL;
+    if (op_ctx)
+        opinfo->op_ctx = dict_ref(op_ctx);
+    else
+        opinfo->op_ctx = NULL;
 
-        if (req)
-                opinfo->req = req;
+    if (req)
+        opinfo->req = req;
 
-        opinfo->txn_generation = conf->generation;
-        cmm_smp_rmb ();
+    opinfo->txn_generation = conf->generation;
+    cmm_smp_rmb();
 
-        return;
+    return;
 }
 
 int32_t
-glusterd_generate_txn_id (dict_t *dict, uuid_t **txn_id)
+glusterd_generate_txn_id(dict_t *dict, uuid_t **txn_id)
 {
-        int32_t             ret      = -1;
-        glusterd_conf_t    *priv     = NULL;
-        xlator_t           *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-
-        *txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!*txn_id)
-                goto out;
-
-        if (priv->op_version < GD_OP_VERSION_3_6_0)
-                gf_uuid_copy (**txn_id, priv->global_txn_id);
-        else
-                gf_uuid_generate (**txn_id);
-
-        ret = dict_set_bin (dict, "transaction_id",
-                            *txn_id, sizeof (**txn_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                       "Failed to set transaction id.");
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                "Transaction_id = %s", uuid_utoa (**txn_id));
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+
+    *txn_id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!*txn_id) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    if (priv->op_version < GD_OP_VERSION_3_6_0)
+        gf_uuid_copy(**txn_id, priv->global_txn_id);
+    else
+        gf_uuid_generate(**txn_id);
+
+    ret = dict_set_bin(dict, "transaction_id", *txn_id, sizeof(**txn_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set transaction id.");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(**txn_id));
 out:
-        if (ret && *txn_id) {
-                GF_FREE (*txn_id);
-                *txn_id = NULL;
-        }
+    if (ret && *txn_id) {
+        GF_FREE(*txn_id);
+        *txn_id = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+glusterd_get_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo)
 {
-        int32_t                   ret         = -1;
-        glusterd_txn_opinfo_obj  *opinfo_obj  = NULL;
-        glusterd_conf_t          *priv        = NULL;
-        xlator_t                 *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!txn_id || !opinfo) {
-                gf_msg_callingfn (this->name, GF_LOG_ERROR, 0,
-                                  GD_MSG_TRANS_ID_GET_FAIL,
-                                  "Empty transaction id or opinfo received.");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_txn_opinfo_obj *opinfo_obj = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!txn_id || !opinfo) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+                         "Empty transaction id or opinfo received.");
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_bin(priv->glusterd_txn_opinfo,
-                           uuid_utoa (*txn_id),
-                           (void **) &opinfo_obj);
-        if (ret)
-                goto out;
+    ret = dict_get_bin(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id),
+                       (void **)&opinfo_obj);
+    if (ret)
+        goto out;
 
-        (*opinfo) = opinfo_obj->opinfo;
+    (*opinfo) = opinfo_obj->opinfo;
 
-        gf_msg_debug (this->name, 0,
-                "Successfully got opinfo for transaction ID : %s",
-                uuid_utoa (*txn_id));
+    gf_msg_debug(this->name, 0,
+                 "Successfully got opinfo for transaction ID : %s",
+                 uuid_utoa(*txn_id));
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+glusterd_set_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo)
 {
-        int32_t                   ret        = -1;
-        glusterd_txn_opinfo_obj  *opinfo_obj = NULL;
-        glusterd_conf_t          *priv        = NULL;
-        xlator_t                 *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!txn_id) {
-                gf_msg_callingfn (this->name, GF_LOG_ERROR, 0,
-                                  GD_MSG_TRANS_ID_GET_FAIL,
-                                  "Empty transaction id received.");
-                ret = -1;
-                goto out;
+    int32_t ret = -1;
+    glusterd_txn_opinfo_obj *opinfo_obj = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!txn_id) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+                         "Empty transaction id received.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id),
+                       (void **)&opinfo_obj);
+    if (ret) {
+        opinfo_obj = GF_CALLOC(1, sizeof(glusterd_txn_opinfo_obj),
+                               gf_common_mt_txn_opinfo_obj_t);
+        if (!opinfo_obj) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_bin(priv->glusterd_txn_opinfo,
-                           uuid_utoa (*txn_id),
-                           (void **) &opinfo_obj);
+        ret = dict_set_bin(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id),
+                           opinfo_obj, sizeof(glusterd_txn_opinfo_obj));
         if (ret) {
-                opinfo_obj = GF_CALLOC (1, sizeof(glusterd_txn_opinfo_obj),
-                                        gf_common_mt_txn_opinfo_obj_t);
-                if (!opinfo_obj) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_bin(priv->glusterd_txn_opinfo,
-                                   uuid_utoa (*txn_id), opinfo_obj,
-                                   sizeof(glusterd_txn_opinfo_obj));
-                if (ret) {
-                        gf_msg_callingfn (this->name, GF_LOG_ERROR, errno,
-                                          GD_MSG_DICT_SET_FAILED,
-                                          "Unable to set opinfo for transaction"
-                                          " ID : %s", uuid_utoa (*txn_id));
-                        goto out;
-                }
+            gf_msg_callingfn(this->name, GF_LOG_ERROR, errno,
+                             GD_MSG_DICT_SET_FAILED,
+                             "Unable to set opinfo for transaction"
+                             " ID : %s",
+                             uuid_utoa(*txn_id));
+            goto out;
         }
+    }
 
-        opinfo_obj->opinfo = (*opinfo);
+    opinfo_obj->opinfo = (*opinfo);
 
-        gf_msg_debug (this->name, 0,
-                "Successfully set opinfo for transaction ID : %s",
-                uuid_utoa (*txn_id));
-        ret = 0;
+    gf_msg_debug(this->name, 0,
+                 "Successfully set opinfo for transaction ID : %s",
+                 uuid_utoa(*txn_id));
+    ret = 0;
 out:
-        if (ret)
-                if (opinfo_obj)
-                        GF_FREE (opinfo_obj);
+    if (ret)
+        if (opinfo_obj)
+            GF_FREE(opinfo_obj);
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_clear_txn_opinfo (uuid_t *txn_id)
+glusterd_clear_txn_opinfo(uuid_t *txn_id)
 {
-        int32_t               ret         = -1;
-        glusterd_op_info_t    txn_op_info = {{0},};
-        glusterd_conf_t      *priv        = NULL;
-        xlator_t             *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!txn_id) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_ID_GET_FAIL,
-                        "Empty transaction id received.");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!txn_id) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Empty transaction id received.");
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_msg_callingfn (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_OPINFO_GET_FAIL,
-                        "Unable to get transaction opinfo "
-                        "for transaction ID : %s",
-                        uuid_utoa (*txn_id));
-                goto out;
-        }
+    ret = glusterd_get_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0,
+                         GD_MSG_TRANS_OPINFO_GET_FAIL,
+                         "Unable to get transaction opinfo "
+                         "for transaction ID : %s",
+                         uuid_utoa(*txn_id));
+        goto out;
+    }
 
-        if (txn_op_info.op_ctx)
-                dict_unref (txn_op_info.op_ctx);
+    if (txn_op_info.op_ctx)
+        dict_unref(txn_op_info.op_ctx);
 
-        dict_del(priv->glusterd_txn_opinfo, uuid_utoa (*txn_id));
+    dict_del(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id));
 
-        gf_msg_debug (this->name, 0,
-                "Successfully cleared opinfo for transaction ID : %s",
-                uuid_utoa (*txn_id));
+    gf_msg_debug(this->name, 0,
+                 "Successfully cleared opinfo for transaction ID : %s",
+                 uuid_utoa(*txn_id));
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int glusterfs_port = GLUSTERD_DEFAULT_PORT;
 static char *glusterd_op_sm_state_names[] = {
-        "Default",
-        "Lock sent",
-        "Locked",
-        "Stage op sent",
-        "Staged",
-        "Commit op sent",
-        "Committed",
-        "Unlock sent",
-        "Stage op failed",
-        "Commit op failed",
-        "Brick op sent",
-        "Brick op failed",
-        "Brick op Committed",
-        "Brick op Commit failed",
-        "Ack drain",
-        "Invalid",
+    "Default",
+    "Lock sent",
+    "Locked",
+    "Stage op sent",
+    "Staged",
+    "Commit op sent",
+    "Committed",
+    "Unlock sent",
+    "Stage op failed",
+    "Commit op failed",
+    "Brick op sent",
+    "Brick op failed",
+    "Brick op Committed",
+    "Brick op Commit failed",
+    "Ack drain",
+    "Invalid",
 };
 
 static char *glusterd_op_sm_event_names[] = {
-        "GD_OP_EVENT_NONE",
-        "GD_OP_EVENT_START_LOCK",
-        "GD_OP_EVENT_LOCK",
-        "GD_OP_EVENT_RCVD_ACC",
-        "GD_OP_EVENT_ALL_ACC",
-        "GD_OP_EVENT_STAGE_ACC",
-        "GD_OP_EVENT_COMMIT_ACC",
-        "GD_OP_EVENT_RCVD_RJT",
-        "GD_OP_EVENT_STAGE_OP",
-        "GD_OP_EVENT_COMMIT_OP",
-        "GD_OP_EVENT_UNLOCK",
-        "GD_OP_EVENT_START_UNLOCK",
-        "GD_OP_EVENT_ALL_ACK",
-        "GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP",
-        "GD_OP_EVENT_INVALID"
-};
-
-char*
-glusterd_op_sm_state_name_get (int state)
+    "GD_OP_EVENT_NONE",       "GD_OP_EVENT_START_LOCK",
+    "GD_OP_EVENT_LOCK",       "GD_OP_EVENT_RCVD_ACC",
+    "GD_OP_EVENT_ALL_ACC",    "GD_OP_EVENT_STAGE_ACC",
+    "GD_OP_EVENT_COMMIT_ACC", "GD_OP_EVENT_RCVD_RJT",
+    "GD_OP_EVENT_STAGE_OP",   "GD_OP_EVENT_COMMIT_OP",
+    "GD_OP_EVENT_UNLOCK",     "GD_OP_EVENT_START_UNLOCK",
+    "GD_OP_EVENT_ALL_ACK",    "GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP",
+    "GD_OP_EVENT_INVALID"};
+
+char *
+glusterd_op_sm_state_name_get(int state)
 {
-        if (state < 0 || state >= GD_OP_STATE_MAX)
-                return glusterd_op_sm_state_names[GD_OP_STATE_MAX];
-        return glusterd_op_sm_state_names[state];
+    if (state < 0 || state >= GD_OP_STATE_MAX)
+        return glusterd_op_sm_state_names[GD_OP_STATE_MAX];
+    return glusterd_op_sm_state_names[state];
 }
 
-char*
-glusterd_op_sm_event_name_get (int event)
+char *
+glusterd_op_sm_event_name_get(int event)
 {
-        if (event < 0 || event >= GD_OP_EVENT_MAX)
-                return glusterd_op_sm_event_names[GD_OP_EVENT_MAX];
-        return glusterd_op_sm_event_names[event];
+    if (event < 0 || event >= GD_OP_EVENT_MAX)
+        return glusterd_op_sm_event_names[GD_OP_EVENT_MAX];
+    return glusterd_op_sm_event_names[event];
 }
 
-void
-glusterd_destroy_lock_ctx (glusterd_op_lock_ctx_t *ctx)
+static void
+glusterd_destroy_lock_ctx(glusterd_op_lock_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
-        GF_FREE (ctx);
+    if (!ctx)
+        return;
+    GF_FREE(ctx);
 }
 
 void
-glusterd_set_volume_status (glusterd_volinfo_t  *volinfo,
-                            glusterd_volume_status status)
+glusterd_set_volume_status(glusterd_volinfo_t *volinfo,
+                           glusterd_volume_status status)
 {
-        GF_ASSERT (volinfo);
-        volinfo->status = status;
+    GF_ASSERT(volinfo);
+    volinfo->status = status;
 }
 
 static int
-glusterd_op_sm_inject_all_acc (uuid_t *txn_id)
+glusterd_op_sm_inject_all_acc(uuid_t *txn_id)
 {
-        int32_t                 ret = -1;
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, txn_id, NULL);
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    int ret = -1;
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACC, txn_id, NULL);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_check_bitrot_cmd (char *key, char *value, char *errstr, size_t size)
+glusterd_check_bitrot_cmd(char *key, const int keylen, char *errstr,
+                          const size_t size)
 {
-        int     ret = -1;
-
-        if ((!strncmp (key, "bitrot", strlen ("bitrot"))) ||
-            (!strncmp (key, "features.bitrot", strlen ("features.bitrot")))) {
-                snprintf (errstr, size, " 'gluster volume set <VOLNAME> %s' "
-                          "is invalid command. Use 'gluster volume bitrot "
-                          "<VOLNAME> {enable|disable}' instead.", key);
-                ret = -1;
-                goto out;
-        } else if ((!strncmp (key, "scrub-freq", strlen ("scrub-freq"))) ||
-                   (!strncmp (key, "features.scrub-freq",
-                    strlen ("features.scrub-freq")))) {
-                snprintf (errstr, size, " 'gluster volume "
-                          "set <VOLNAME> %s' is invalid command. Use 'gluster "
-                          "volume bitrot <VOLNAME> scrub-frequency"
-                          " {hourly|daily|weekly|biweekly|monthly}' instead.",
-                          key);
-                ret = -1;
-                goto out;
-        } else if ((!strncmp (key, "scrub", strlen ("scrub"))) ||
-                  (!strncmp (key, "features.scrub",
-                   strlen ("features.scrub")))) {
-                snprintf (errstr, size, " 'gluster volume set <VOLNAME> %s' is "
-                          "invalid command. Use 'gluster volume bitrot "
-                          "<VOLNAME> scrub {pause|resume}' instead.", key);
-                ret = -1;
-                goto out;
-        } else if ((!strncmp (key, "scrub-throttle",
-                     strlen ("scrub-throttle"))) ||
-                   (!strncmp (key, "features.scrub-throttle",
-                     strlen ("features.scrub-throttle")))) {
-                snprintf (errstr, size, " 'gluster volume set <VOLNAME> %s' is "
-                          "invalid command. Use 'gluster volume bitrot "
-                          "<VOLNAME> scrub-throttle {lazy|normal|aggressive}' "
-                          "instead.",
-                          key);
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+
+    if (len_strcmp(key, keylen, "bitrot") ||
+        len_strcmp(key, keylen, "features.bitrot")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> {enable|disable}'"
+                 " instead.",
+                 key);
+        goto out;
+    } else if (len_strcmp(key, keylen, "scrub-freq") ||
+               len_strcmp(key, keylen, "features.scrub-freq")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> scrub-frequency"
+                 " {hourly|daily|weekly|biweekly|monthly}' instead.",
+                 key);
+        goto out;
+    } else if (len_strcmp(key, keylen, "scrub") ||
+               len_strcmp(key, keylen, "features.scrub")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> scrub {pause|resume}'"
+                 " instead.",
+                 key);
+        goto out;
+    } else if (len_strcmp(key, keylen, "scrub-throttle") ||
+               len_strcmp(key, keylen, "features.scrub-throttle")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> scrub-throttle "
+                 " {lazy|normal|aggressive}' instead.",
+                 key);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_check_quota_cmd (char *key, char *value, char *errstr, size_t size)
+glusterd_check_quota_cmd(char *key, const int keylen, char *value, char *errstr,
+                         size_t size)
 {
-        int                ret = -1;
-        gf_boolean_t       b   = _gf_false;
+    int ret = -1;
+    gf_boolean_t b = _gf_false;
 
-        if ((strcmp (key, "quota") == 0) ||
-           (strcmp (key, "features.quota") == 0)) {
-                ret = gf_string2boolean (value, &b);
-                if (ret)
-                        goto out;
-                if (b) {
-                        snprintf (errstr, size, " 'gluster "
-                                  "volume set <VOLNAME> %s %s' is "
-                                  "deprecated. Use 'gluster volume "
-                                  "quota <VOLNAME> enable' instead.",
-                                   key, value);
-                        ret = -1;
-                        goto out;
-                } else {
-                        snprintf (errstr, size, " 'gluster "
-                                  "volume set <VOLNAME> %s %s' is "
-                                  "deprecated. Use 'gluster volume "
-                                  "quota <VOLNAME> disable' instead.",
-                                   key, value);
-                        ret = -1;
-                        goto out;
-                }
-        } else if ((strcmp (key, "inode-quota") == 0) ||
-                   (strcmp (key, "features.inode-quota") == 0)) {
-                ret = gf_string2boolean (value, &b);
-                if (ret)
-                        goto out;
-                if (b) {
-                        snprintf (errstr, size, " 'gluster "
-                                  "volume set <VOLNAME> %s %s' is "
-                                  "deprecated. Use 'gluster volume "
-                                  "inode-quota <VOLNAME> enable' instead.",
-                                  key, value);
-                        ret = -1;
-                        goto out;
-                } else {
-                        /* inode-quota disable not supported,
-                         * use quota disable
-                         */
-                        snprintf (errstr, size, " 'gluster "
-                                  "volume set <VOLNAME> %s %s' is "
-                                  "deprecated. Use 'gluster volume "
-                                  "quota <VOLNAME> disable' instead.",
-                                   key, value);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        ret = 0;
+    if (len_strcmp(key, keylen, "quota") ||
+        len_strcmp(key, keylen, "features.quota")) {
+        ret = gf_string2boolean(value, &b);
+        if (ret)
+            goto out;
+        ret = -1;
+        if (b) {
+            snprintf(errstr, size,
+                     " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                     " Use 'gluster volume quota <VOLNAME> enable' instead.",
+                     key, value);
+        } else {
+            snprintf(errstr, size,
+                     " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                     " Use 'gluster volume quota <VOLNAME> disable' instead.",
+                     key, value);
+        }
+        goto out;
+    } else if (len_strcmp(key, keylen, "inode-quota") ||
+               len_strcmp(key, keylen, "features.inode-quota")) {
+        ret = gf_string2boolean(value, &b);
+        if (ret)
+            goto out;
+        ret = -1;
+        if (b) {
+            snprintf(
+                errstr, size,
+                " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                " Use 'gluster volume inode-quota <VOLNAME> enable' instead.",
+                key, value);
+        } else {
+            /* inode-quota disable not supported,
+             * use quota disable
+             */
+            snprintf(errstr, size,
+                     " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                     " Use 'gluster volume quota <VOLNAME> disable' instead.",
+                     key, value);
+        }
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
-                                 gd1_mgmt_brick_op_req **req, dict_t *dict)
+glusterd_brick_op_build_payload(glusterd_op_t op,
+                                glusterd_brickinfo_t *brickinfo,
+                                gd1_mgmt_brick_op_req **req, dict_t *dict)
 {
-        int                     ret = -1;
-        gd1_mgmt_brick_op_req   *brick_req = NULL;
-        char                    *volname = NULL;
-        char                    name[1024] = {0,};
-        gf_xl_afr_op_t          heal_op = GF_SHD_OP_INVALID;
-        xlator_t                *this = NULL;
-        glusterd_volinfo_t        *volinfo      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (op < GD_OP_MAX);
-        GF_ASSERT (op > GD_OP_NONE);
-        GF_ASSERT (req);
-
-
-        switch (op) {
+    int ret = -1;
+    gd1_mgmt_brick_op_req *brick_req = NULL;
+    char *volname = NULL;
+    char name[1024] = {
+        0,
+    };
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(op < GD_OP_MAX);
+    GF_ASSERT(op > GD_OP_NONE);
+    GF_ASSERT(req);
+
+    switch (op) {
         case GD_OP_REMOVE_BRICK:
         case GD_OP_STOP_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-                brick_req->op = GLUSTERD_BRICK_TERMINATE;
-                brick_req->name = "";
-        break;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+            brick_req->op = GLUSTERD_BRICK_TERMINATE;
+            brick_req->name = brickinfo->path;
+            glusterd_set_brick_status(brickinfo, GF_BRICK_STOPPING);
+            break;
         case GD_OP_PROFILE_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_XLATOR_INFO;
-                brick_req->name = brickinfo->path;
-
-                break;
-        case GD_OP_HEAL_VOLUME:
-        {
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_XLATOR_OP;
-                brick_req->name = "";
-                ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
-                if (ret)
-                        goto out;
-                ret = dict_set_int32 (dict, "xl-op", heal_op);
-        }
-                break;
-        case GD_OP_STATUS_VOLUME:
-        {
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-                brick_req->op = GLUSTERD_BRICK_STATUS;
-                brick_req->name = "";
-        }
-                break;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+
+            brick_req->op = GLUSTERD_BRICK_XLATOR_INFO;
+            brick_req->name = brickinfo->path;
+
+            break;
+        case GD_OP_HEAL_VOLUME: {
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+
+            brick_req->op = GLUSTERD_BRICK_XLATOR_OP;
+            brick_req->name = "";
+            ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"),
+                                  (int32_t *)&heal_op);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                        "Key=heal-op", NULL);
+                goto out;
+            }
+            ret = dict_set_int32n(dict, "xl-op", SLEN("xl-op"), heal_op);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                        "Key=xl-op", NULL);
+                goto out;
+            }
+        } break;
+        case GD_OP_STATUS_VOLUME: {
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+            brick_req->op = GLUSTERD_BRICK_STATUS;
+            brick_req->name = "";
+            ret = dict_set_strn(dict, "brick-name", SLEN("brick-name"),
+                                brickinfo->path);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                        "Key=brick-name", NULL);
+                goto out;
+            }
+        } break;
         case GD_OP_REBALANCE:
         case GD_OP_DEFRAG_BRICK_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_XLATOR_DEFRAG;
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                        snprintf (name, 1024, "%s-tier-dht", volname);
-                else
-                        snprintf (name, 1024, "%s-dht", volname);
-                brick_req->name = gf_strdup (name);
-
-                break;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+
+            brick_req->op = GLUSTERD_BRICK_XLATOR_DEFRAG;
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                        "Key=volname", NULL);
+                goto out;
+            }
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_VOLINFO_GET_FAIL, "Volume=%s", volname, NULL);
+                goto out;
+            }
+            snprintf(name, sizeof(name), "%s-dht", volname);
+            brick_req->name = gf_strdup(name);
+
+            break;
         case GD_OP_SNAP:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_BARRIER;
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                brick_req->name = gf_strdup (volname);
-
-                break;
         case GD_OP_BARRIER:
-                brick_req = GF_CALLOC (1, sizeof(*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-                brick_req->op = GLUSTERD_BRICK_BARRIER;
-                ret = dict_get_str(dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                brick_req->name = gf_strdup (volname);
-                break;
-
-        default:
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
                 goto out;
-        break;
-        }
+            }
+            brick_req->op = GLUSTERD_BRICK_BARRIER;
+            brick_req->name = brickinfo->path;
+            break;
 
-        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
-                                           &brick_req->input.input_len);
-        if (ret)
-                goto out;
-        *req = brick_req;
-        ret = 0;
+        default:
+            goto out;
+            break;
+    }
+
+    brick_req->dict.dict_len = 0;
+    brick_req->dict.dict_val = NULL;
+    ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val,
+                                      &brick_req->input.input_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    *req = brick_req;
+    ret = 0;
 
 out:
-        if (ret && brick_req)
-                GF_FREE (brick_req);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && brick_req)
+        GF_FREE(brick_req);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
-                                dict_t *dict)
+glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                               dict_t *dict)
 {
-        int                     ret = -1;
-        gd1_mgmt_brick_op_req   *brick_req = NULL;
-        char                    xlname[1024] = {0,};
-        char                    *volname = NULL;
-
-        GF_ASSERT (op < GD_OP_MAX);
-        GF_ASSERT (op > GD_OP_NONE);
-        GF_ASSERT (req);
-
-        switch (op) {
+    int ret = -1;
+    gd1_mgmt_brick_op_req *brick_req = NULL;
+    char *volname = NULL;
+
+    GF_ASSERT(op < GD_OP_MAX);
+    GF_ASSERT(op > GD_OP_NONE);
+    GF_ASSERT(req);
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    switch (op) {
         case GD_OP_PROFILE_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
 
-                brick_req->op = GLUSTERD_NODE_PROFILE;
-                brick_req->name = "";
+            brick_req->op = GLUSTERD_NODE_PROFILE;
+            brick_req->name = "";
 
-                break;
+            break;
 
         case GD_OP_STATUS_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
 
-                brick_req->op = GLUSTERD_NODE_STATUS;
-                brick_req->name = "";
+            brick_req->op = GLUSTERD_NODE_STATUS;
+            brick_req->name = "";
 
-                break;
+            break;
 
         case GD_OP_SCRUB_STATUS:
-                brick_req = GF_CALLOC (1, sizeof(*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
+        case GD_OP_SCRUB_ONDEMAND:
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
 
-                brick_req->op = GLUSTERD_NODE_BITROT;
+            brick_req->op = GLUSTERD_NODE_BITROT;
 
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                        "Key=volname", NULL);
+                goto out;
+            }
 
-                brick_req->name = gf_strdup (volname);
-                break;
+            brick_req->name = gf_strdup(volname);
+            break;
         default:
-                goto out;
-        }
+            goto out;
+    }
 
-        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
-                                           &brick_req->input.input_len);
+    brick_req->dict.dict_len = 0;
+    brick_req->dict.dict_val = NULL;
+    ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val,
+                                      &brick_req->input.input_len);
 
-        if (ret)
-                goto out;
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
 
-        *req = brick_req;
-        ret = 0;
+    *req = brick_req;
+    ret = 0;
 
 out:
-        if (ret && brick_req)
-                GF_FREE (brick_req);
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && brick_req)
+        GF_FREE(brick_req);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_validate_quorum_options (xlator_t *this, char *fullkey, char *value,
-                                  char **op_errstr)
+glusterd_validate_quorum_options(xlator_t *this, char *fullkey, char *value,
+                                 char **op_errstr)
 {
-        int             ret = 0;
-        char            *key = NULL;
-        volume_option_t *opt = NULL;
-
-        if (!glusterd_is_quorum_option (fullkey))
-                goto out;
-        key = strchr (fullkey, '.');
-        if (key == NULL) {
-                ret = -1;
-                goto out;
-        }
-        key++;
-        opt = xlator_volume_option_get (this, key);
-        ret = xlator_option_validate (this, key, value, opt, op_errstr);
+    int ret = 0;
+    char *key = NULL;
+    volume_option_t *opt = NULL;
+
+    if (!glusterd_is_quorum_option(fullkey))
+        goto out;
+    key = strchr(fullkey, '.');
+    if (key == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    key++;
+    opt = xlator_volume_option_get(this, key);
+    if (!opt) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = xlator_option_validate(this, key, value, opt, op_errstr);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_validate_shared_storage (char *key, char *value, char *errstr)
+glusterd_validate_brick_mx_options(xlator_t *this, char *fullkey, char *value,
+                                   char **op_errstr)
 {
-        int32_t            ret      = -1;
-        int32_t            exists   = -1;
-        int32_t            count    = -1;
-        char              *op       = NULL;
-        xlator_t          *this     = NULL;
-        glusterd_conf_t   *conf     = NULL;
+    int ret = 0;
 
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+    // Placeholder function for now
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    return ret;
+}
 
-        GF_VALIDATE_OR_GOTO (this->name, key, out);
-        GF_VALIDATE_OR_GOTO (this->name, value, out);
-        GF_VALIDATE_OR_GOTO (this->name, errstr, out);
+static int
+glusterd_validate_shared_storage(char *value, char *errstr)
+{
+    int32_t ret = -1;
+    int32_t count = -1;
+    char *op = NULL;
+    char hook_script[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int32_t len = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+    GF_VALIDATE_OR_GOTO(this->name, errstr, out);
+
+    if ((strcmp(value, "enable")) && (strcmp(value, "disable"))) {
+        snprintf(errstr, PATH_MAX,
+                 "Invalid option(%s). Valid options "
+                 "are 'enable' and 'disable'",
+                 value);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    len = snprintf(hook_script, sizeof(hook_script),
+                   "%s" GLUSTERD_SHRD_STRG_HOOK_SCRIPT, conf->workdir);
+    if ((len < 0) || (len >= sizeof(hook_script))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_access(hook_script, R_OK | X_OK);
+    if (ret) {
+        len = snprintf(errstr, PATH_MAX,
+                       "The hook-script (%s) required "
+                       "for this operation is not present. "
+                       "Please install the hook-script "
+                       "and retry",
+                       hook_script);
+        if (len < 0) {
+            strncpy(errstr, "<error>", PATH_MAX);
+        }
+        gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED, "%s",
+               errstr);
+        goto out;
+    }
+
+    if (!strncmp(value, "disable", SLEN("disable"))) {
+        ret = dict_get_strn(conf->opts, GLUSTERD_SHARED_STORAGE_KEY,
+                            SLEN(GLUSTERD_SHARED_STORAGE_KEY), &op);
+        if (ret || !strncmp(op, "disable", SLEN("disable"))) {
+            snprintf(errstr, PATH_MAX,
+                     "Shared storage volume "
+                     "does not exist. Please enable shared storage"
+                     " for creating shared storage volume.");
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_SHARED_STORAGE_DOES_NOT_EXIST, "%s", errstr);
+            ret = -1;
+            goto out;
+        }
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(GLUSTER_SHARED_STORAGE, &volinfo);
+    if (!ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Shared storage volume(" GLUSTER_SHARED_STORAGE
+                 ") already exists.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_ALREADY_EXIST, "%s",
+               errstr);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_count_connected_peers(&count);
+    if (ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Failed to calculate number of connected peers.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_COUNT_GET_FAIL, "%s",
+               errstr);
+        goto out;
+    }
+
+    if (count <= 1) {
+        snprintf(errstr, PATH_MAX,
+                 "More than one node should "
+                 "be up/present in the cluster to enable this option");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INSUFFICIENT_UP_NODES, "%s",
+               errstr);
+        ret = -1;
+        goto out;
+    }
 
-        if (strcmp (key, GLUSTERD_SHARED_STORAGE_KEY)) {
-                goto out;
-        }
+out:
+    return ret;
+}
 
-        if ((strcmp (value, "enable")) &&
-            (strcmp (value, "disable"))) {
-                snprintf (errstr, PATH_MAX,
-                          "Invalid option(%s). Valid options "
-                          "are 'enable' and 'disable'", value);
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "%s", errstr);
-                ret = -1;
-                goto out;
-        }
+static int
+glusterd_validate_localtime_logging(char *value, char *errstr)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int already_enabled = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+
+    already_enabled = gf_log_get_localtime();
+
+    ret = 0;
+    if (strcmp(value, "enable") == 0) {
+        gf_log_set_localtime(1);
+        if (!already_enabled)
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_LOCALTIME_LOGGING_ENABLE,
+                   "localtime logging enable");
+    } else if (strcmp(value, "disable") == 0) {
+        gf_log_set_localtime(0);
+        if (already_enabled)
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_LOCALTIME_LOGGING_DISABLE,
+                   "localtime logging disable");
+    } else {
+        ret = -1;
+        GF_VALIDATE_OR_GOTO(this->name, errstr, out);
+        snprintf(errstr, PATH_MAX,
+                 "Invalid option(%s). Valid options "
+                 "are 'enable' and 'disable'",
+                 value);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+    }
 
-        if (!strncmp (value, "disable", strlen ("disable"))) {
-                ret = dict_get_str (conf->opts, GLUSTERD_SHARED_STORAGE_KEY,
-                                                                           &op);
-                if (ret || !strncmp (op, "disable", strlen ("disable"))) {
-                        snprintf (errstr, PATH_MAX, "Shared storage volume "
-                                  "does not exist. Please enable shared storage"
-                                  " for creating shared storage volume.");
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SHARED_STORAGE_DOES_NOT_EXIST, "%s",
-                                errstr);
-                        ret = -1;
-                        goto out;
-                }
-                goto out;
-        }
+out:
+    return ret;
+}
 
-        exists = glusterd_check_volume_exists (GLUSTER_SHARED_STORAGE);
-        if (exists) {
-                snprintf (errstr, PATH_MAX,
-                          "Shared storage volume("GLUSTER_SHARED_STORAGE
-                          ") already exists.");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_ALREADY_EXIST, "%s", errstr);
-                ret = -1;
-                goto out;
-        }
+static int
+glusterd_validate_daemon_log_level(char *value, char *errstr)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        ret = glusterd_count_connected_peers (&count);
-        if (ret) {
-                snprintf (errstr, PATH_MAX,
-                          "Failed to calculate number of connected peers.");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_COUNT_GET_FAIL, "%s", errstr);
-                goto out;
-        }
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        if (count <= 1) {
-                snprintf (errstr, PATH_MAX,
-                          "More than one node should "
-                          "be up/present in the cluster to enable this option");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INSUFFICIENT_UP_NODES, "%s", errstr);
-                ret = -1;
-                goto out;
-        }
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+
+    ret = 0;
+
+    if ((strcmp(value, "INFO")) && (strcmp(value, "WARNING")) &&
+        (strcmp(value, "DEBUG")) && (strcmp(value, "TRACE")) &&
+        (strcmp(value, "ERROR"))) {
+        ret = -1;
+        GF_VALIDATE_OR_GOTO(this->name, errstr, out);
+        snprintf(errstr, PATH_MAX,
+                 "Invalid option(%s). Valid options "
+                 "are 'INFO' or 'WARNING' or 'ERROR' or 'DEBUG' or "
+                 " 'TRACE'",
+                 value);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_set_volume(dict_t *dict, char **op_errstr)
 {
-        int                   ret                         = -1;
-        char                 *volname                     = NULL;
-        int                   exists                      = 0;
-        char                 *key                         = NULL;
-        char                 *key_fixed                   = NULL;
-        char                 *value                       = NULL;
-        char                 *val_dup                     = NULL;
-        char                  str[100]                    = {0, };
-        char                 *trash_path                  = NULL;
-        int                   trash_path_len              = 0;
-        int                   count                       = 0;
-        int                   dict_count                  = 0;
-        char                  errstr[PATH_MAX]            = {0, };
-        glusterd_volinfo_t   *volinfo                     = NULL;
-        glusterd_brickinfo_t *brickinfo                   = NULL;
-        dict_t               *val_dict                    = NULL;
-        gf_boolean_t          global_opt                  = _gf_false;
-        glusterd_volinfo_t   *voliter                     = NULL;
-        glusterd_conf_t      *priv                        = NULL;
-        xlator_t             *this                        = NULL;
-        uint32_t              new_op_version              = GD_OP_VERSION_MIN;
-        uint32_t              local_new_op_version        = GD_OP_VERSION_MIN;
-        uint32_t              local_new_client_op_version = GD_OP_VERSION_MIN;
-        uint32_t              key_op_version              = GD_OP_VERSION_MIN;
-        uint32_t              local_key_op_version        = GD_OP_VERSION_MIN;
-        gf_boolean_t          origin_glusterd             = _gf_true;
-        gf_boolean_t          check_op_version            = _gf_true;
-        gf_boolean_t          trash_enabled               = _gf_false;
-        gf_boolean_t          all_vol                     = _gf_false;
-        struct stat           stbuf                       = {0, };
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        val_dict = dict_new();
-        if (!val_dict)
-                goto out;
+    int ret = -1;
+    char *volname = NULL;
+    int exists = 0;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    char *value = NULL;
+    char *val_dup = NULL;
+    char keystr[100] = {
+        0,
+    };
+    int keystr_len;
+    int keylen;
+    char *trash_path = NULL;
+    int trash_path_len = 0;
+    int count = 0;
+    int dict_count = 0;
+    char errstr[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    dict_t *val_dict = NULL;
+    gf_boolean_t global_opt = _gf_false;
+    gf_boolean_t key_matched = _gf_false; /* if a key was processed or not*/
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    uint32_t new_op_version = GD_OP_VERSION_MIN;
+    uint32_t local_new_op_version = GD_OP_VERSION_MIN;
+    uint32_t local_new_client_op_version = GD_OP_VERSION_MIN;
+    uint32_t key_op_version = GD_OP_VERSION_MIN;
+    uint32_t local_key_op_version = GD_OP_VERSION_MIN;
+    gf_boolean_t origin_glusterd = _gf_true;
+    gf_boolean_t check_op_version = _gf_true;
+    gf_boolean_t trash_enabled = _gf_false;
+    gf_boolean_t all_vol = _gf_false;
+    struct volopt_map_entry *vmep = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Check if we can support the required op-version
+     * This check is not done on the originator glusterd. The originator
+     * glusterd sets this value.
+     */
+    origin_glusterd = is_origin_glusterd(dict);
+
+    if (!origin_glusterd) {
+        /* Check for v3.3.x origin glusterd */
+        check_op_version = dict_get_str_boolean(dict, "check-op-version",
+                                                _gf_false);
 
-        /* Check if we can support the required op-version
-         * This check is not done on the originator glusterd. The originator
-         * glusterd sets this value.
-         */
-        origin_glusterd = is_origin_glusterd (dict);
-
-        if (!origin_glusterd) {
-                /* Check for v3.3.x origin glusterd */
-                check_op_version = dict_get_str_boolean (dict,
-                                                         "check-op-version",
-                                                         _gf_false);
-
-                if (check_op_version) {
-                        ret = dict_get_uint32 (dict, "new-op-version",
-                                               &new_op_version);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get new_op_version");
-                                goto out;
-                        }
-
-                        if ((new_op_version > GD_OP_VERSION_MAX) ||
-                            (new_op_version < GD_OP_VERSION_MIN)) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr),
-                                          "Required op_version (%d) is not "
-                                          "supported", new_op_version);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_UNSUPPORTED_VERSION, "%s",
-                                        errstr);
-                                goto out;
-                        }
-                }
-        }
+        if (check_op_version) {
+            ret = dict_get_uint32(dict, "new-op-version", &new_op_version);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                        "Key=new-op-version", NULL);
+                goto out;
+            }
 
-        ret = dict_get_int32 (dict, "count", &dict_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Count(dict),not set in Volume-Set");
+            if ((new_op_version > GD_OP_VERSION_MAX) ||
+                (new_op_version < GD_OP_VERSION_MIN)) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Required op_version (%d) is not supported."
+                         " Max supported op version is %d",
+                         new_op_version, priv->op_version);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+                       "%s", errstr);
                 goto out;
+            }
         }
+    }
 
-        if (dict_count == 0) {
+    ret = dict_get_int32_sizen(dict, "count", &dict_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Count(dict),not set in Volume-Set");
+        goto out;
+    }
+
+    if (dict_count == 0) {
         /*No options would be specified of volume set help */
-                if (dict_get (dict, "help" ))  {
-                        ret = 0;
-                        goto out;
-                }
+        if (dict_get_sizen(dict, "help")) {
+            ret = 0;
+            goto out;
+        }
 
-                if (dict_get (dict, "help-xml" )) {
+        if (dict_get_sizen(dict, "help-xml")) {
 #if (HAVE_LIB_XML)
-                        ret = 0;
-                        goto out;
+            ret = 0;
+            goto out;
 #else
-                        ret  = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MODULE_NOT_INSTALLED,
-                                "libxml not present in the system");
-                        *op_errstr = gf_strdup ("Error: xml libraries not "
-                                                "present to produce xml-output");
-                        goto out;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_INSTALLED,
+                   "libxml not present in the system");
+            *op_errstr = gf_strdup(
+                "Error: xml libraries not present to produce xml-output");
+            goto out;
 #endif
-                }
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_OPTIONS_GIVEN, "No options received ");
-                *op_errstr = gf_strdup ("Options not specified");
-                ret = -1;
-                goto out;
         }
-
-        ret = dict_get_str (dict, "volname", &volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_OPTIONS_GIVEN,
+               "No options received ");
+        *op_errstr = gf_strdup("Options not specified");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") != 0) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get volume name");
-                goto out;
+            snprintf(errstr, sizeof(errstr), FMTSTR_CHECK_VOL_EXISTS, volname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                   FMTSTR_CHECK_VOL_EXISTS, volname);
+            goto out;
         }
 
-        if (strcasecmp (volname, "all") != 0) {
-                exists = glusterd_check_volume_exists (volname);
-                if (!exists) {
-                        snprintf (errstr, sizeof (errstr),
-                                  FMTSTR_CHECK_VOL_EXISTS, volname);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_FOUND, "%s", errstr);
-                        ret = -1;
-                        goto out;
-                }
+        ret = glusterd_validate_volume_id(dict, volinfo);
+        if (ret)
+            goto out;
 
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_FOUND,
-                                FMTSTR_CHECK_VOL_EXISTS, volname);
-                        goto out;
-                }
+        local_new_op_version = volinfo->op_version;
+        local_new_client_op_version = volinfo->client_op_version;
 
-                ret = glusterd_validate_volume_id (dict, volinfo);
-                if (ret)
-                        goto out;
+    } else {
+        all_vol = _gf_true;
+    }
 
-                local_new_op_version = volinfo->op_version;
-                local_new_client_op_version = volinfo->client_op_version;
+    val_dict = dict_new();
+    if (!val_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
 
-        } else {
-                all_vol = _gf_true;
+    for (count = 1; ret != 1; count++) {
+        keystr_len = sprintf(keystr, "key%d", count);
+        ret = dict_get_strn(dict, keystr, keystr_len, &key);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                    "Key=%s", keystr, NULL);
+            break;
         }
 
-        for ( count = 1; ret != 1 ; count++ ) {
-                global_opt = _gf_false;
-                sprintf (str, "key%d", count);
-                ret = dict_get_str (dict, str, &key);
+        keystr_len = sprintf(keystr, "value%d", count);
+        ret = dict_get_strn(dict, keystr, keystr_len, &value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "invalid key,value pair in 'volume set'");
+            ret = -1;
+            goto out;
+        }
+
+        key_matched = _gf_false;
+        keylen = strlen(key);
+        if (len_strcmp(key, keylen, "config.memory-accounting")) {
+            key_matched = _gf_true;
+            gf_msg_debug(this->name, 0,
+                         "enabling memory accounting for volume %s", volname);
+            ret = 0;
+        } else if (len_strcmp(key, keylen, "config.transport")) {
+            key_matched = _gf_true;
+            gf_msg_debug(this->name, 0, "changing transport-type for volume %s",
+                         volname);
+            ret = 0;
+            /* if value is none of 'tcp/rdma/tcp,rdma' error out */
+            if (!((strcasecmp(value, "rdma") == 0) ||
+                  (strcasecmp(value, "tcp") == 0) ||
+                  (strcasecmp(value, "tcp,rdma") == 0) ||
+                  (strcasecmp(value, "rdma,tcp") == 0))) {
+                ret = snprintf(errstr, sizeof(errstr),
+                               "transport-type %s does not exist", value);
+                /* lets not bother about above return value,
+                   its a failure anyways */
+                ret = -1;
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, "ganesha.enable")) {
+            key_matched = _gf_true;
+            if (!strcmp(value, "off") == 0) {
+                ret = ganesha_manage_export(dict, "off", _gf_true, op_errstr);
                 if (ret)
-                        break;
-
-                sprintf (str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "invalid key,value pair in 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
+                    goto out;
+            }
+        }
 
-                if (strcmp (key, "config.memory-accounting") == 0) {
-                        gf_msg_debug (this->name, 0,
-                                "enabling memory accounting for volume %s",
-                                volname);
-                        ret = 0;
-                }
+        if (!key_matched) {
+            ret = glusterd_check_bitrot_cmd(key, keylen, errstr,
+                                            sizeof(errstr));
+            if (ret)
+                goto out;
+            ret = glusterd_check_quota_cmd(key, keylen, value, errstr,
+                                           sizeof(errstr));
+            if (ret)
+                goto out;
+        }
 
-                if (strcmp (key, "config.transport") == 0) {
-                        gf_msg_debug (this->name, 0,
-                                "changing transport-type for volume %s",
-                                volname);
-                        ret = 0;
-                        /* if value is none of 'tcp/rdma/tcp,rdma' error out */
-                        if (!((strcasecmp (value, "rdma") == 0) ||
-                              (strcasecmp (value, "tcp") == 0) ||
-                              (strcasecmp (value, "tcp,rdma") == 0) ||
-                              (strcasecmp (value, "rdma,tcp") == 0))) {
-                                ret = snprintf (errstr, sizeof (errstr),
-                                                "transport-type %s does "
-                                                "not exist", value);
-                                /* lets not bother about above return value,
-                                   its a failure anyways */
-                                ret = -1;
-                                goto out;
-                        }
-                }
+        if (is_key_glusterd_hooks_friendly(key))
+            continue;
 
-                ret = glusterd_check_bitrot_cmd (key, value, errstr,
-                                                 sizeof (errstr));
-                if (ret)
-                        goto out;
+        ret = glusterd_volopt_validate(volinfo, dict, key, value, op_errstr);
+        if (ret)
+            goto out;
+
+        exists = glusterd_check_option_exists(key, &key_fixed);
+        if (exists == -1) {
+            ret = -1;
+            goto out;
+        }
+
+        if (!exists) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Option with name: %s does not exist", key);
+            ret = snprintf(errstr, sizeof(errstr), "option : %s does not exist",
+                           key);
+            if (key_fixed)
+                snprintf(errstr + ret, sizeof(errstr) - ret,
+                         "\nDid you mean %s?", key_fixed);
+            ret = -1;
+            goto out;
+        }
+
+        if (key_fixed) {
+            key = key_fixed;
+            keylen = strlen(key_fixed);
+        }
+
+        if (len_strcmp(key, keylen, "cluster.granular-entry-heal")) {
+            /* For granular entry-heal, if the set command was
+             * invoked through volume-set CLI, then allow the
+             * command only if the volume is still in 'Created'
+             * state
+             */
+            if (volinfo && volinfo->status != GLUSTERD_STATUS_NONE &&
+                (dict_get_sizen(dict, "is-special-key") == NULL)) {
+                snprintf(errstr, sizeof(errstr),
+                         " 'gluster volume set <VOLNAME> %s {enable, disable}'"
+                         " is not supported."
+                         " Use 'gluster volume heal <VOLNAME> "
+                         "granular-entry-heal {enable, disable}' instead.",
+                         key);
+                ret = -1;
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, GLUSTERD_GLOBAL_OP_VERSION_KEY)) {
+            /* Check if the key is cluster.op-version and set
+             * local_new_op_version to the value given if possible.
+             */
+            if (!all_vol) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Option \"%s\" is not valid for a single volume", key);
+                goto out;
+            }
+            /* Check if cluster.op-version is the only option being
+             * set
+             */
+            if (count != 1) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Option \"%s\" cannot be set along with other options",
+                         key);
+                goto out;
+            }
+            /* Just reusing the variable, but I'm using it for
+             * storing the op-version from value
+             */
+            ret = gf_string2uint(value, &local_key_op_version);
+            if (ret) {
+                snprintf(errstr, sizeof(errstr),
+                         "invalid number format \"%s\" in option \"%s\"", value,
+                         key);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+                       errstr);
+                goto out;
+            }
+
+            if (local_key_op_version > GD_OP_VERSION_MAX ||
+                local_key_op_version < GD_OP_VERSION_MIN) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Required op_version (%d) is not supported."
+                         " Max supported op version is %d",
+                         local_key_op_version, priv->op_version);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERSION_UNSUPPORTED,
+                       "%s", errstr);
+                goto out;
+            }
+            if (local_key_op_version > priv->op_version) {
+                local_new_op_version = local_key_op_version;
+            } else {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Required op-version (%d) should"
+                         " not be equal or lower than current"
+                         " cluster op-version (%d).",
+                         local_key_op_version, priv->op_version);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERSION_UNSUPPORTED,
+                       "%s", errstr);
+                goto out;
+            }
 
-                ret = glusterd_check_quota_cmd (key, value, errstr, sizeof (errstr));
-                if (ret)
-                        goto out;
+            goto cont;
+        }
 
-                if (is_key_glusterd_hooks_friendly (key))
-                        continue;
+        ALL_VOLUME_OPTION_CHECK(volname, _gf_false, key, ret, op_errstr, out);
+        ret = glusterd_validate_quorum_options(this, key, value, op_errstr);
+        if (ret)
+            goto out;
 
-                ret = glusterd_volopt_validate (volinfo, dict, key, value,
-                                                op_errstr);
-                if (ret)
-                        goto out;
+        ret = glusterd_validate_brick_mx_options(this, key, value, op_errstr);
+        if (ret)
+            goto out;
 
-                exists = glusterd_check_option_exists (key, &key_fixed);
-                if (exists == -1) {
-                        ret = -1;
-                        goto out;
-                }
+        vmep = gd_get_vmep(key);
+        local_key_op_version = glusterd_get_op_version_from_vmep(vmep);
+        if (local_key_op_version > local_new_op_version)
+            local_new_op_version = local_key_op_version;
+        if (gd_is_client_option(vmep) &&
+            (local_key_op_version > local_new_client_op_version))
+            local_new_client_op_version = local_key_op_version;
 
-                if (!exists) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_INVALID_ENTRY,
-                                "Option with name: %s does not exist", key);
-                        ret = snprintf (errstr, sizeof (errstr),
-                                       "option : %s does not exist",
-                                       key);
-                        if (key_fixed)
-                                snprintf (errstr + ret, sizeof (errstr) - ret,
-                                          "\nDid you mean %s?", key_fixed);
+        sprintf(keystr, "op-version%d", count);
+        if (origin_glusterd) {
+            ret = dict_set_uint32(dict, keystr, local_key_op_version);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set key-op-version in dict");
+                goto out;
+            }
+        } else if (check_op_version) {
+            ret = dict_get_uint32(dict, keystr, &key_op_version);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get key-op-version from dict");
+                goto out;
+            }
+            if (local_key_op_version != key_op_version) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "option: %s op-version mismatch", key);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+                       "%s, required op-version = %" PRIu32
+                       ", available op-version = %" PRIu32,
+                       errstr, key_op_version, local_key_op_version);
+                goto out;
+            }
+        }
+
+        global_opt = glusterd_check_globaloption(key);
+
+        if (len_strcmp(key, keylen, GLUSTERD_SHARED_STORAGE_KEY)) {
+            ret = glusterd_validate_shared_storage(value, errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SHARED_STRG_VOL_OPT_VALIDATE_FAIL,
+                       "Failed to validate shared storage volume options");
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, GLUSTERD_LOCALTIME_LOGGING_KEY)) {
+            ret = glusterd_validate_localtime_logging(value, errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_LOCALTIME_LOGGING_VOL_OPT_VALIDATE_FAIL,
+                       "Failed to validate localtime logging volume options");
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, GLUSTERD_DAEMON_LOG_LEVEL_KEY)) {
+            ret = glusterd_validate_daemon_log_level(value, errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL,
+                       "Failed to validate daemon-log-level volume options");
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, "features.trash-dir")) {
+            if (volinfo) {
+                ret = glusterd_volinfo_get(volinfo, VKEY_FEATURES_TRASH,
+                                           &val_dup);
+                if (!ret && val_dup) {
+                    ret = gf_string2boolean(val_dup, &trash_enabled);
+                    if (ret)
+                        goto out;
+                }
+            }
+            if (!trash_enabled) {
+                snprintf(errstr, sizeof(errstr),
+                         "Trash translator is not enabled. "
+                         "Use volume set %s trash on",
+                         volname);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                       "Unable to set the options in 'volume set': %s", errstr);
+                ret = -1;
+                goto out;
+            }
+            if (strchr(value, '/')) {
+                snprintf(errstr, sizeof(errstr),
+                         "Path is not allowed as option");
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                       "Unable to set the options in 'volume set': %s", errstr);
+                ret = -1;
+                goto out;
+            }
+
+            list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                /* Check for local brick */
+                if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                    trash_path_len = strlen(value) + strlen(brickinfo->path) +
+                                     2;
+                    trash_path = GF_MALLOC(trash_path_len, gf_common_mt_char);
+                    snprintf(trash_path, trash_path_len, "%s/%s",
+                             brickinfo->path, value);
+
+                    /* Checks whether a directory with
+                       given option exists or not */
+                    if (!sys_access(trash_path, R_OK)) {
+                        snprintf(errstr, sizeof(errstr), "Path %s exists",
+                                 value);
+                        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                               "Unable to set the options in 'volume set': %s",
+                               errstr);
                         ret = -1;
                         goto out;
-                }
-
-                if (key_fixed)
-                        key = key_fixed;
-
-                /* Check if the key is cluster.op-version and set
-                 * local_new_op_version to the value given if possible.
-                 */
-                if (strcmp (key, "cluster.op-version") == 0) {
-                        if (!all_vol) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr), "Option \""
-                                          "%s\" is not valid for a single "
-                                          "volume", key);
-                                goto out;
-                        }
-                        /* Check if cluster.op-version is the only option being
-                         * set
-                         */
-                        if (count != 1) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr), "Option \""
-                                          "%s\" cannot be set along with other "
-                                          "options", key);
-                                goto out;
-                        }
-                        /* Just reusing the variable, but I'm using it for
-                         * storing the op-version from value
-                         */
-                        ret = gf_string2uint (value, &local_key_op_version);
-                        if (ret) {
-                                snprintf (errstr, sizeof (errstr), "invalid "
-                                          "number format \"%s\" in option "
-                                          "\"%s\"", value, key);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_INVALID_ENTRY, "%s", errstr);
-                                goto out;
-                        }
-
-                        if (local_key_op_version > GD_OP_VERSION_MAX ||
-                            local_key_op_version < GD_OP_VERSION_MIN) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr),
-                                          "Required op_version (%d) is not "
-                                          "supported", local_key_op_version);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VERSION_UNSUPPORTED,
-                                        "%s", errstr);
-                                goto out;
-                        }
-                        if (local_key_op_version > priv->op_version) {
-                                local_new_op_version = local_key_op_version;
-                        } else {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr),
-                                          "Required op-version (%d) should"
-                                          " not be equal or lower than current"
-                                          " cluster op-version (%d).",
-                                          local_key_op_version,
-                                          priv->op_version);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VERSION_UNSUPPORTED,
-                                        "%s", errstr);
-                                goto out;
-                        }
-
-                        goto cont;
-                }
-
-                ALL_VOLUME_OPTION_CHECK (volname, key, ret, op_errstr, out);
-                ret = glusterd_validate_quorum_options (this, key, value,
-                                                        op_errstr);
-                if (ret)
-                        goto out;
-
-                local_key_op_version = glusterd_get_op_version_for_key (key);
-                if (local_key_op_version > local_new_op_version)
-                        local_new_op_version = local_key_op_version;
-                if (gd_is_client_option (key) &&
-                    (local_key_op_version > local_new_client_op_version))
-                        local_new_client_op_version = local_key_op_version;
-
-                sprintf (str, "op-version%d", count);
-                if (origin_glusterd) {
-                        ret = dict_set_uint32 (dict, str, local_key_op_version);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set key-op-version in dict");
-                                goto out;
-                        }
-                } else if (check_op_version) {
-                        ret = dict_get_uint32 (dict, str, &key_op_version);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get key-op-version from"
-                                        " dict");
-                                goto out;
-                        }
-                        if (local_key_op_version != key_op_version) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr),
-                                          "option: %s op-version mismatch",
-                                          key);
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_OP_VERSION_MISMATCH,
-                                        "%s, required op-version = %"PRIu32", "
-                                        "available op-version = %"PRIu32,
-                                        errstr, key_op_version,
-                                        local_key_op_version);
-                                goto out;
-                        }
-                }
-
-                if (glusterd_check_globaloption (key))
-                        global_opt = _gf_true;
-
-                if (volinfo) {
-                        ret = glusterd_volinfo_get (volinfo,
-                                                VKEY_FEATURES_TRASH, &val_dup);
-                        if (val_dup) {
-                                ret = gf_string2boolean (val_dup,
-                                                        &trash_enabled);
-                                if (ret)
-                                        goto out;
-                        }
-                }
-
-                ret = glusterd_validate_shared_storage (key, value, errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SHARED_STRG_VOL_OPT_VALIDATE_FAIL,
-                                "Failed to validate shared "
-                                "storage volume options");
-                        goto out;
-                }
-
-                if (!strcmp(key, "features.trash-dir") && trash_enabled) {
-                        if (strchr (value, '/')) {
-                                snprintf (errstr, sizeof (errstr),
-                                          "Path is not allowed as option");
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOL_SET_FAIL,
-                                        "Unable to set the options in 'volume "
-                                        "set': %s", errstr);
-                                ret = -1;
-                                goto out;
-                        }
-
-                        list_for_each_entry (brickinfo, &volinfo->bricks,
-                                             brick_list) {
-                                /* Check for local brick */
-                                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                        trash_path_len = strlen (value) +
-                                                   strlen (brickinfo->path) + 2;
-                                        trash_path = GF_CALLOC (1,
-                                                     trash_path_len,
-                                                     gf_common_mt_char);
-                                        snprintf (trash_path, trash_path_len,
-                                                  "%s/%s", brickinfo->path,
-                                                  value);
-
-                                        /* Checks whether a directory with
-                                           given option exists or not */
-                                        if (!sys_stat (trash_path, &stbuf)) {
-                                                snprintf (errstr,
-                                                          sizeof (errstr),
-                                                          "Path %s exists",
-                                                          value);
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR,
-                                                        0, GD_MSG_VOL_SET_FAIL,
-                                                        "Unable to set the "
-                                                        "options in "
-                                                        "'volume set': %s",
-                                                        errstr);
-                                                ret = -1;
-                                                goto out;
-                                        } else {
-                                                gf_msg_debug (this->name, 0,
-                                                        "Directory with given "
-                                                        "name does not exists,"
-                                                        " continuing");
-                                        }
-
-                                        if (volinfo->status == GLUSTERD_STATUS_STARTED
-                                                && brickinfo->status != GF_BRICK_STARTED) {
-                                                /* If volume is in started state , checks
-                                                   whether bricks are online */
-                                                snprintf (errstr, sizeof (errstr),
-                                                          "One or more bricks are down");
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        GD_MSG_VOL_SET_FAIL,
-                                                        "Unable to set the "
-                                                        "options in "
-                                                        "'volume set': %s",
-                                                        errstr);
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                }
-                                if (trash_path) {
-                                         GF_FREE (trash_path);
-                                         trash_path = NULL;
-                                         trash_path_len = 0;
-                                }
-                        }
-                } else if (!strcmp(key, "features.trash-dir") && !trash_enabled) {
-                        snprintf (errstr, sizeof (errstr),
-                                        "Trash translator is not enabled. Use "
-                                        "volume set %s trash on", volname);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_SET_FAIL,
-                                "Unable to set the options in 'volume "
-                                "set': %s", errstr);
+                    } else {
+                        gf_msg_debug(this->name, 0,
+                                     "Directory with given name does not exist,"
+                                     " continuing");
+                    }
+
+                    if (volinfo->status == GLUSTERD_STATUS_STARTED &&
+                        brickinfo->status != GF_BRICK_STARTED) {
+                        /* If volume is in started state , checks
+                           whether bricks are online */
+                        snprintf(errstr, sizeof(errstr),
+                                 "One or more bricks are down");
+                        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                               "Unable to set the options in 'volume set': %s",
+                               errstr);
                         ret = -1;
                         goto out;
+                    }
                 }
-                ret = dict_set_str (val_dict, key, value);
-
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set the options in 'volume set'");
-                        ret = -1;
-                        goto out;
+                if (trash_path) {
+                    GF_FREE(trash_path);
+                    trash_path = NULL;
                 }
+            }
+        }
 
-                *op_errstr = NULL;
-                if (!global_opt && !all_vol)
-                        ret = glusterd_validate_reconfopts (volinfo, val_dict, op_errstr);
-                else if (!all_vol) {
-                        voliter = NULL;
-                        cds_list_for_each_entry (voliter, &priv->volumes,
-                                                 vol_list) {
-                                ret = glusterd_validate_globalopts (voliter,
-                                                                    val_dict,
-                                                                    op_errstr);
-                                if (ret)
-                                        break;
-                        }
-                }
+        ret = dict_set_strn(val_dict, key, keylen, value);
 
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLFILE_CREATE_FAIL,
-                                "Could not create "
-                                "temp volfile, some option failed: %s",
-                                *op_errstr);
-                        goto out;
-                }
-                dict_del (val_dict, key);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set the options in 'volume set'");
+            ret = -1;
+            goto out;
+        }
+
+        *op_errstr = NULL;
+        if (!global_opt && !all_vol)
+            ret = glusterd_validate_reconfopts(volinfo, val_dict, op_errstr);
+        else if (!all_vol) {
+            voliter = NULL;
+            cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+            {
+                ret = glusterd_validate_globalopts(voliter, val_dict,
+                                                   op_errstr);
+                if (ret)
+                    break;
+            }
+        }
 
-                if (key_fixed) {
-                        GF_FREE (key_fixed);
-                        key_fixed = NULL;
-                }
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Could not create temp volfile, some option failed: %s",
+                   *op_errstr);
+            goto out;
         }
+        dict_deln(val_dict, key, keylen);
 
-        /* Check if all the connected clients support the new client-op-version
-         */
-        ret = glusterd_check_client_op_version_support
-                (volname, local_new_client_op_version, op_errstr);
-        if (ret)
-                goto out;
+        if (key_fixed) {
+            GF_FREE(key_fixed);
+            key_fixed = NULL;
+        }
+    }
 
+    /* Check if all the connected clients support the new client-op-version
+     */
+    ret = glusterd_check_client_op_version_support(
+        volname, local_new_client_op_version, op_errstr);
+    if (ret)
+        goto out;
 cont:
-        if (origin_glusterd) {
-                ret = dict_set_uint32 (dict, "new-op-version",
-                                       local_new_op_version);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set new-op-version in dict");
-                        goto out;
-                }
-                /* Set this value in dict so other peers know to check for
-                 * op-version. This is a hack for 3.3.x compatibility
-                 *
-                 * TODO: Remove this and the other places this is referred once
-                 * 3.3.x compatibility is not required
-                 */
-                ret = dict_set_uint32 (dict, "check-op-version",
-                                       _gf_true);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set check-op-version in dict");
-                        goto out;
-                }
+    if (origin_glusterd) {
+        ret = dict_set_uint32(dict, "new-op-version", local_new_op_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set new-op-version in dict");
+            goto out;
         }
+        /* Set this value in dict so other peers know to check for
+         * op-version. This is a hack for 3.3.x compatibility
+         *
+         * TODO: Remove this and the other places this is referred once
+         * 3.3.x compatibility is not required
+         */
+        ret = dict_set_int32_sizen(dict, "check-op-version", 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set check-op-version in dict");
+            goto out;
+        }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (val_dict)
-                dict_unref (val_dict);
+    if (val_dict)
+        dict_unref(val_dict);
 
-        if (trash_path)
-                GF_FREE (trash_path);
+    if (trash_path)
+        GF_FREE(trash_path);
 
-        GF_FREE (key_fixed);
-        if (errstr[0] != '\0')
-                *op_errstr = gf_strdup (errstr);
+    GF_FREE(key_fixed);
+    if (errstr[0] != '\0')
+        *op_errstr = gf_strdup(errstr);
 
-        if (ret) {
-                if (!(*op_errstr)) {
-                        *op_errstr = gf_strdup ("Error, Validation Failed");
-                        gf_msg_debug (this->name, 0,
-                                "Error, Cannot Validate option :%s",
-                                *op_errstr);
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                "Error, Cannot Validate option");
-                }
+    if (ret) {
+        if (!(*op_errstr)) {
+            *op_errstr = gf_strdup("Error, Validation Failed");
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option :%s",
+                         *op_errstr);
+        } else {
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option");
         }
-        return ret;
+    }
+    return ret;
 }
 
 static int
-glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_reset_volume(dict_t *dict, char **op_errstr)
 {
-        int                                      ret           = 0;
-        char                                    *volname       = NULL;
-        int                                     exists         = 0;
-        char                                    msg[2048]      = {0};
-        char                                    *key = NULL;
-        char                                    *key_fixed = NULL;
-        glusterd_volinfo_t                      *volinfo       = NULL;
-        xlator_t                                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-
+    int ret = 0;
+    char *volname = NULL;
+    int exists = 0;
+    char msg[2048] = {0};
+    char *key = NULL;
+    char *key_fixed = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") != 0) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
+            snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+            goto out;
         }
 
-        if (strcasecmp (volname, "all") != 0) {
-                exists = glusterd_check_volume_exists (volname);
-                if (!exists) {
-                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
-                                  volname);
-                        ret = -1;
-                        goto out;
-                }
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
-                                  volname);
-                        goto out;
-                }
-
-                ret = glusterd_validate_volume_id (dict, volinfo);
-                if (ret)
-                        goto out;
-        }
+        ret = glusterd_validate_volume_id(dict, volinfo);
+        if (ret)
+            goto out;
+    }
+
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get option key");
+        goto out;
+    }
+
+    /* *
+     * If key ganesha.enable is set, then volume should be unexported from
+     * ganesha server. Also it is a volume-level option, perform only when
+     * volume name not equal to "all"(in other words if volinfo != NULL)
+     */
+    if (volinfo && (!strcmp(key, "all") || !strcmp(key, "ganesha.enable"))) {
+        if (glusterd_check_ganesha_export(volinfo)) {
+            ret = ganesha_manage_export(dict, "off", _gf_true, op_errstr);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_GNS_RESET_FAIL,
+                       "Could not reset ganesha.enable key");
+        }
+    }
+
+    if (strcmp(key, "all")) {
+        exists = glusterd_check_option_exists(key, &key_fixed);
+        if (exists == -1) {
+            ret = -1;
+            goto out;
+        }
+
+        if (!exists) {
+            ret = snprintf(msg, sizeof(msg), "Option %s does not exist", key);
+            if (key_fixed)
+                snprintf(msg + ret, sizeof(msg) - ret, "\nDid you mean %s?",
+                         key_fixed);
+            ret = -1;
+            goto out;
+        } else if (exists > 0) {
+            if (key_fixed)
+                key = key_fixed;
 
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get option key");
+            /* 'gluster volume set/reset <VOLNAME>
+             * features.quota/features.inode-quota' should
+             * not be allowed as it is deprecated.
+             * Setting and resetting quota/inode-quota features
+             * should be allowed only through 'gluster volume quota
+             * <VOLNAME> enable/disable'.
+             * But, 'gluster volume set features.quota-deem-statfs'
+             * can be turned on/off when quota is enabled.
+             */
+
+            if (strcmp(VKEY_FEATURES_INODE_QUOTA, key) == 0 ||
+                strcmp(VKEY_FEATURES_QUOTA, key) == 0) {
+                snprintf(msg, sizeof(msg),
+                         "'gluster volume "
+                         "reset <VOLNAME> %s' is deprecated. "
+                         "Use 'gluster volume quota <VOLNAME> "
+                         "disable' instead.",
+                         key);
+                ret = -1;
                 goto out;
+            }
+            ALL_VOLUME_OPTION_CHECK(volname, _gf_false, key, ret, op_errstr,
+                                    out);
         }
-        if (strcmp(key, "all")) {
-                exists = glusterd_check_option_exists (key, &key_fixed);
-                if (exists == -1) {
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!exists) {
-                        ret = snprintf (msg, sizeof (msg),
-                                        "Option %s does not exist", key);
-                        if (key_fixed)
-                                snprintf (msg + ret, sizeof (msg) - ret,
-                                          "\nDid you mean %s?", key_fixed);
-                        ret = -1;
-                        goto out;
-                } else if (exists > 0) {
-                        if (key_fixed)
-                                key = key_fixed;
-
-                        /* 'gluster volume set/reset <VOLNAME>
-                         * features.quota/features.inode-quota' should
-                         * not be allowed as it is deprecated.
-                         * Setting and resetting quota/inode-quota features
-                         * should be allowed only through 'gluster volume quota
-                         * <VOLNAME> enable/disable'.
-                         * But, 'gluster volume set features.quota-deem-statfs'
-                         * can be turned on/off when quota is enabled.
-                         */
-
-                        if (strcmp (VKEY_FEATURES_INODE_QUOTA, key) == 0 ||
-                            strcmp (VKEY_FEATURES_QUOTA, key) == 0) {
-                                snprintf (msg, sizeof (msg), "'gluster volume "
-                                          "reset <VOLNAME> %s' is deprecated. "
-                                          "Use 'gluster volume quota <VOLNAME> "
-                                          "disable' instead.", key);
-                                ret = -1;
-                                goto out;
-                        }
-                        ALL_VOLUME_OPTION_CHECK (volname, key, ret,
-                                                 op_errstr, out);
-                }
-        }
+    }
 
 out:
-        GF_FREE (key_fixed);
+    GF_FREE(key_fixed);
 
-        if (msg[0] != '\0') {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_STAGE_RESET_VOL_FAIL, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
+    if (msg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_RESET_VOL_FAIL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
-
-
 static int
-glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_sync_volume(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    *hostname = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        glusterd_peerinfo_t                     *peerinfo = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_volinfo_t                      *volinfo  = NULL;
-
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "hostname couldn't be "
-                          "retrieved from msg");
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (gf_is_local_addr (hostname)) {
-                //volname is not present in case of sync all
-                ret = dict_get_str (dict, "volname", &volname);
-                if (!ret) {
-                        exists = glusterd_check_volume_exists (volname);
-                        if (!exists) {
-                                snprintf (msg, sizeof (msg), "Volume %s "
-                                         "does not exist", volname);
-                                *op_errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret)
-                                goto out;
-
-                } else {
-                        ret = 0;
-                }
-         } else {
-                 rcu_read_lock ();
-
-                 peerinfo = glusterd_peerinfo_find (NULL, hostname);
-                 if (peerinfo == NULL) {
-                         ret = -1;
-                         snprintf (msg, sizeof (msg), "%s, is not a friend",
-                                         hostname);
-                         *op_errstr = gf_strdup (msg);
-
-                 } else if (!peerinfo->connected) {
-                         snprintf (msg, sizeof (msg), "%s, is not connected at "
-                                         "the moment", hostname);
-                         *op_errstr = gf_strdup (msg);
-                         ret = -1;
-                 }
-
-                 rcu_read_unlock ();
-        }
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "hostname couldn't be "
+                 "retrieved from msg");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=hostname", NULL);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (gf_is_local_addr(hostname)) {
+        // volname is not present in case of sync all
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+        if (!ret) {
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                snprintf(msg, sizeof(msg),
+                         "Volume %s "
+                         "does not exist",
+                         volname);
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_NOT_FOUND,
+                        "Volume=%s", volname, NULL);
+                *op_errstr = gf_strdup(msg);
+                goto out;
+            }
+        }
+    } else {
+        RCU_READ_LOCK;
+
+        peerinfo = glusterd_peerinfo_find(NULL, hostname);
+        if (peerinfo == NULL) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s, is not a friend", hostname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND,
+                    "Peer_name=%s", hostname, NULL);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not connected at "
+                     "the moment",
+                     hostname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_DISCONNECTED,
+                    "Peer_name=%s", hostname, NULL);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+
+        RCU_READ_UNLOCK;
+    }
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr)
 {
-        int                    ret            = -1;
-        uint32_t               cmd            = 0;
-        char                   msg[2048]      = {0,};
-        char                  *volname        = NULL;
-        char                  *brick          = NULL;
-        xlator_t              *this           = NULL;
-        glusterd_conf_t       *priv           = NULL;
-        glusterd_brickinfo_t  *brickinfo      = NULL;
-        glusterd_volinfo_t    *volinfo        = NULL;
-        dict_t                *vol_opts       = NULL;
-        gf_boolean_t           nfs_disabled   = _gf_false;
-        gf_boolean_t           shd_enabled    = _gf_false;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT(priv);
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
-
-        if (cmd & GF_CLI_STATUS_ALL)
-                goto out;
-
-        if ((cmd & GF_CLI_STATUS_QUOTAD) &&
-            (priv->op_version == GD_OP_VERSION_MIN)) {
-                snprintf (msg, sizeof (msg), "The cluster is operating at "
-                          "version 1. Getting the status of quotad is not "
-                          "allowed in this state.");
-                ret = -1;
-                goto out;
-        }
+    int ret = -1;
+    uint32_t cmd = 0;
+    char msg[2048] = {
+        0,
+    };
+    char *volname = NULL;
+    char *brick = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    dict_t *vol_opts = NULL;
+#ifdef BUILD_GNFS
+    gf_boolean_t nfs_disabled = _gf_false;
+#endif
+    gf_boolean_t shd_enabled = _gf_false;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=cmd", NULL);
+        goto out;
+    }
+
+    if (cmd & GF_CLI_STATUS_ALL)
+        goto out;
+
+    if ((cmd & GF_CLI_STATUS_QUOTAD) &&
+        (priv->op_version == GD_OP_VERSION_MIN)) {
+        snprintf(msg, sizeof(msg),
+                 "The cluster is operating at "
+                 "version 1. Getting the status of quotad is not "
+                 "allowed in this state.");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_QUOTA_GET_STAT_FAIL,
+                msg, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_SNAPD) &&
+        (priv->op_version < GD_OP_VERSION_3_6_0)) {
+        snprintf(msg, sizeof(msg),
+                 "The cluster is operating at "
+                 "version less than %d. Getting the "
+                 "status of snapd is not allowed in this state.",
+                 GD_OP_VERSION_3_6_0);
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAP_STATUS_FAIL, msg,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL,
+                "Volume=%s", volname, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VALIDATE_FAILED, NULL);
+        goto out;
+    }
+
+    ret = glusterd_is_volume_started(volinfo);
+    if (!ret) {
+        snprintf(msg, sizeof(msg), "Volume %s is not started", volname);
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_NOT_STARTED,
+                "Volume=%s", volname, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if ((cmd & GF_CLI_STATUS_SNAPD) &&
-            (priv->op_version < GD_OP_VERSION_3_6_0)) {
-                snprintf (msg, sizeof (msg), "The cluster is operating at "
-                          "version less than %d. Getting the "
-                          "status of snapd is not allowed in this state.",
-                          GD_OP_VERSION_3_6_0);
-                ret = -1;
-                goto out;
-        }
+    vol_opts = volinfo->dict;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
+    if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+        if (glusterd_is_shd_compatible_volume(volinfo)) {
+            shd_enabled = gd_is_self_heal_enabled(volinfo, vol_opts);
+        } else {
+            ret = -1;
+            snprintf(msg, sizeof(msg), "Volume %s is not Self-heal compatible",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_SHD_NOT_COMP,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+        if (!shd_enabled) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Self-heal Daemon is disabled for volume %s", volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SELF_HEALD_DISABLED,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+#ifdef BUILD_GNFS
+    } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+        nfs_disabled = dict_get_str_boolean(vol_opts, NFS_DISABLE_MAP_KEY,
+                                            _gf_false);
+        if (nfs_disabled) {
+            ret = -1;
+            snprintf(msg, sizeof(msg), "NFS server is disabled for volume %s",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_NFS_GANESHA_DISABLED, "Volume=%s", volname, NULL);
+            goto out;
         }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
+#endif
+    } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+        if (!glusterd_is_volume_quota_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "quota enabled",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_QUOTA_DISABLED,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
+        if (!glusterd_is_bitrot_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "bitrot enabled",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BITROT_NOT_ENABLED,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
+        if (!glusterd_is_bitrot_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "bitrot enabled. Scrubber will be enabled "
+                     "automatically if bitrot is enabled",
+                     volname);
+            gf_smsg(
+                this->name, GF_LOG_ERROR, errno, GD_MSG_BITROT_NOT_ENABLED,
+                "Scrubber will be enabled automatically if bitrot is enabled",
+                "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
+        if (!glusterd_is_snapd_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "uss enabled",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAPD_NOT_RUNNING,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick);
         if (ret) {
-                snprintf (msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                ret = -1;
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                    "Key=brick", NULL);
+            goto out;
         }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_is_volume_started (volinfo);
-        if (!ret) {
-                snprintf (msg, sizeof (msg), "Volume %s is not started",
-                          volname);
-                ret = -1;
-                goto out;
-        }
-
-        vol_opts = volinfo->dict;
-
-        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                nfs_disabled = dict_get_str_boolean (vol_opts,
-                                                     NFS_DISABLE_MAP_KEY,
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
                                                      _gf_false);
-                if (nfs_disabled) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg),
-                                  "NFS server is disabled for volume %s",
-                                  volname);
-                        goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                if (glusterd_is_shd_compatible_volume (volinfo)) {
-                        shd_enabled = gd_is_self_heal_enabled (volinfo,
-                                                               vol_opts);
-                } else {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg),
-                              "Volume %s is not Self-heal compatible",
-                              volname);
-                        goto out;
-                }
-                if (!shd_enabled) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg),
-                                  "Self-heal Daemon is disabled for volume %s",
-                                  volname);
-                        goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
-                if (!glusterd_is_volume_quota_enabled (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s does not have "
-                                  "quota enabled", volname);
-                        goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
-                if (!glusterd_is_bitrot_enabled (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s does not have "
-                                  "bitrot enabled", volname);
-                        goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
-                if (!glusterd_is_bitrot_enabled (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s does not have "
-                                  "bitrot enabled. Scrubber will be enabled "
-                                  "automatically if bitrot is enabled",
-                                  volname);
-                        goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
-                if (!glusterd_is_snapd_enabled (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s does not have "
-                                  "uss enabled", volname);
-                        goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brick);
-                if (ret)
-                        goto out;
-
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              _gf_false);
-                if (ret) {
-                        snprintf (msg, sizeof(msg), "No brick %s in"
-                                  " volume %s", brick, volname);
-                        ret = -1;
-                        goto out;
-                }
+        if (ret) {
+            snprintf(msg, sizeof(msg),
+                     "No brick %s in"
+                     " volume %s",
+                     brick, volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_NOT_FOUND,
+                    "Brick=%s, Volume=%s", brick, volname, NULL);
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
- out:
-        if (ret) {
-                if (msg[0] != '\0')
-                        *op_errstr = gf_strdup (msg);
-                else
-                        *op_errstr = gf_strdup ("Validation Failed for Status");
-        }
+out:
+    if (ret) {
+        if (msg[0] != '\0')
+            *op_errstr = gf_strdup(msg);
+        else
+            *op_errstr = gf_strdup("Validation Failed for Status");
+    }
 
-        gf_msg_debug (this->name, 0, "Returning: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning: %d", ret);
+    return ret;
 }
 
-
-static gf_boolean_t
-glusterd_is_profile_on (glusterd_volinfo_t *volinfo)
+int
+glusterd_op_stage_stats_volume(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = -1;
-        gf_boolean_t                            is_latency_on = _gf_false;
-        gf_boolean_t                            is_fd_stats_on = _gf_false;
-
-        GF_ASSERT (volinfo);
-
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_DIAG_CNT_FOP_HITS);
-        if (ret != -1)
-                is_fd_stats_on = ret;
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_DIAG_LAT_MEASUREMENT);
-        if (ret != -1)
-                is_latency_on = ret;
-        if ((_gf_true == is_latency_on) &&
-            (_gf_true == is_fd_stats_on))
-                return _gf_true;
-        return _gf_false;
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    int32_t stats_op = GF_CLI_STATS_NONE;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s, "
+                 "doesn't exist",
+                 volname);
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &stats_op);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume profile op get failed");
+        goto out;
+    }
+
+    if (GF_CLI_STATS_START == stats_op) {
+        if (_gf_true == glusterd_is_profile_on(volinfo)) {
+            snprintf(msg, sizeof(msg),
+                     "Profile on Volume %s is"
+                     " already started",
+                     volinfo->volname);
+            ret = -1;
+            goto out;
+        }
+    } else if ((GF_CLI_STATS_STOP == stats_op) ||
+               (GF_CLI_STATS_INFO == stats_op)) {
+        if (_gf_false == glusterd_is_profile_on(volinfo)) {
+            snprintf(msg, sizeof(msg),
+                     "Profile on Volume %s is"
+                     " not started",
+                     volinfo->volname);
+            ret = -1;
+
+            goto out;
+        }
+    }
+    if ((GF_CLI_STATS_TOP == stats_op) || (GF_CLI_STATS_INFO == stats_op)) {
+        if (_gf_false == glusterd_is_volume_started(volinfo)) {
+            snprintf(msg, sizeof(msg), "Volume %s is not started.",
+                     volinfo->volname);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED, "%s",
+                   msg);
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = 0;
+out:
+    if (msg[0] != '\0') {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_STATS_VOL_FAIL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr)
+_delete_reconfig_opt(dict_t *this, char *key, data_t *value, void *data)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        char                                    msg[2048] = {0,};
-        int32_t                                 stats_op = GF_CLI_STATS_NONE;
-        glusterd_volinfo_t                      *volinfo = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume name get failed");
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((!exists) || (ret < 0)) {
-                snprintf (msg, sizeof (msg), "Volume %s, "
-                         "doesn't exist", volname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, "op", &stats_op);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume profile op get failed");
-                goto out;
-        }
+    int32_t *is_force = 0;
+
+    GF_ASSERT(data);
+    is_force = (int32_t *)data;
+
+    /* Keys which has the flag VOLOPT_FLAG_NEVER_RESET
+     * should not be deleted
+     */
+
+    if (_gf_true ==
+        glusterd_check_voloption_flags(key, VOLOPT_FLAG_NEVER_RESET)) {
+        if (*is_force != 1)
+            *is_force = *is_force | GD_OP_PROTECTED;
+        goto out;
+    }
+
+    if (*is_force != 1) {
+        if (_gf_true ==
+            glusterd_check_voloption_flags(key, VOLOPT_FLAG_FORCE)) {
+            /* indicate to caller that we don't set the option
+             * due to being protected
+             */
+            *is_force = *is_force | GD_OP_PROTECTED;
+            goto out;
+        } else {
+            *is_force = *is_force | GD_OP_UNPROTECTED;
+        }
+    }
+
+    gf_msg_debug("glusterd", 0, "deleting dict with key=%s,value=%s", key,
+                 value->data);
+    dict_del(this, key);
+    /**Delete scrubber (pause/resume) option from the dictionary if bitrot
+     * option is going to be reset
+     * */
+    if (!strncmp(key, VKEY_FEATURES_BITROT, strlen(VKEY_FEATURES_BITROT))) {
+        dict_del_sizen(this, VKEY_FEATURES_SCRUB);
+    }
+out:
+    return 0;
+}
 
-        if (GF_CLI_STATS_START == stats_op) {
-                if (_gf_true == glusterd_is_profile_on (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Profile on Volume %s is"
-                                  " already started", volinfo->volname);
-                        ret = -1;
-                        goto out;
-                }
+static int
+_delete_reconfig_global_opt(dict_t *this, char *key, data_t *value, void *data)
+{
+    GF_ASSERT(data);
 
-        }
-        if ((GF_CLI_STATS_STOP == stats_op) ||
-            (GF_CLI_STATS_INFO == stats_op)) {
-                if (_gf_false == glusterd_is_profile_on (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Profile on Volume %s is"
-                                  " not started", volinfo->volname);
-                        ret = -1;
+    if (strcmp(GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
+        goto out;
 
-                        goto out;
-                }
-        }
-        if ((GF_CLI_STATS_TOP == stats_op) ||
-            (GF_CLI_STATS_INFO == stats_op)) {
-                if (_gf_false == glusterd_is_volume_started (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Volume %s is not started.",
-                                  volinfo->volname);
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_STARTED, "%s", msg);
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = 0;
+    _delete_reconfig_opt(this, key, value, data);
 out:
-        if (msg[0] != '\0') {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_OP_STAGE_STATS_VOL_FAIL, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    return 0;
 }
 
-
 static int
-_delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
+glusterd_options_reset(glusterd_volinfo_t *volinfo, char *key,
+                       int32_t *is_force)
 {
-        int32_t        *is_force = 0;
+    int ret = 0;
+    data_t *value = NULL;
+    char *key_fixed = NULL;
+    xlator_t *this = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo->dict);
+    GF_ASSERT(key);
+
+    if (!strncmp(key, "all", 3)) {
+        dict_foreach(volinfo->dict, _delete_reconfig_opt, is_force);
+        ret = glusterd_enable_default_options(volinfo, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FAIL_DEFAULT_OPT_SET,
+                   "Failed to set "
+                   "default options on reset for volume %s",
+                   volinfo->volname);
+            goto out;
+        }
+    } else {
+        value = dict_get(volinfo->dict, key);
+        if (!value) {
+            gf_msg_debug(this->name, 0, "no value set for option %s", key);
+            goto out;
+        }
+        _delete_reconfig_opt(volinfo->dict, key, value, is_force);
+        ret = glusterd_enable_default_options(volinfo, key);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FAIL_DEFAULT_OPT_SET,
+                   "Failed to set "
+                   "default value for option '%s' on reset for "
+                   "volume %s",
+                   key, volinfo->volname);
+            goto out;
+        }
+    }
+
+    gd_update_volume_op_versions(volinfo);
+    if (!volinfo->is_snap_volume) {
+        svc = &(volinfo->snapd.svc);
+        ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+        if (ret)
+            goto out;
+    }
+    svc = &(volinfo->gfproxyd.svc);
+    ret = svc->reconfigure(volinfo);
+    if (ret)
+        goto out;
+
+    svc = &(volinfo->shd.svc);
+    ret = svc->reconfigure(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to create volfile for"
+               " 'volume reset'");
+        ret = -1;
+        goto out;
+    }
 
-        GF_ASSERT (data);
-        is_force = (int32_t*)data;
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        goto out;
 
-        /* Keys which has the flag OPT_FLAG_NEVER_RESET
-         * should not be deleted
-         */
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_svcs_reconfigure(volinfo);
+        if (ret)
+            goto out;
+    }
 
-        if (_gf_true == glusterd_check_voloption_flags (key,
-                                                OPT_FLAG_NEVER_RESET)) {
-                if (*is_force != 1)
-                        *is_force = *is_force | GD_OP_PROTECTED;
-                goto out;
-        }
+    ret = 0;
 
-        if (*is_force != 1) {
-                if (_gf_true == glusterd_check_voloption_flags (key,
-                                                         OPT_FLAG_FORCE)) {
-                /* indicate to caller that we don't set the option
-                 * due to being protected
-                 */
-                        *is_force = *is_force | GD_OP_PROTECTED;
-                        goto out;
-                } else {
-                        *is_force = *is_force | GD_OP_UNPROTECTED;
-                }
-        }
-
-        gf_msg_debug ("glusterd", 0, "deleting dict with key=%s,value=%s",
-                key, value->data);
-        dict_del (this, key);
-        /**Delete scrubber (pause/resume) option from the dictionary if bitrot
-         * option is going to be reset
-         * */
-        if (!strncmp (key, VKEY_FEATURES_BITROT,
-             strlen (VKEY_FEATURES_BITROT))) {
-                dict_del (this, VKEY_FEATURES_SCRUB);
-        }
 out:
-        return 0;
+    GF_FREE(key_fixed);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-_delete_reconfig_global_opt (dict_t *this, char *key, data_t *value, void *data)
+glusterd_op_reset_all_volume_options(xlator_t *this, dict_t *dict)
 {
-        int32_t        *is_force = 0;
-
-        GF_ASSERT (data);
-        is_force = (int32_t*)data;
-
-        if (strcmp (GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
-                goto out;
-
-        _delete_reconfig_opt (this, key, value, data);
+    char *key = NULL;
+    char *key_fixed = NULL;
+    int ret = -1;
+    int32_t is_force = 0;
+    glusterd_conf_t *conf = NULL;
+    dict_t *dup_opt = NULL;
+    gf_boolean_t all = _gf_false;
+    char *next_version = NULL;
+    gf_boolean_t quorum_action = _gf_false;
+
+    conf = this->private;
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get key");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+    if (ret)
+        is_force = 0;
+
+    if (strcmp(key, "all")) {
+        ret = glusterd_check_option_exists(key, &key_fixed);
+        if (ret <= 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Option %s does not "
+                   "exist",
+                   key);
+            ret = -1;
+            goto out;
+        }
+    } else {
+        all = _gf_true;
+    }
+
+    if (key_fixed)
+        key = key_fixed;
+
+    ret = -1;
+    dup_opt = dict_new();
+    if (!dup_opt) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+    if (!all) {
+        dict_copy(conf->opts, dup_opt);
+        dict_del(dup_opt, key);
+    }
+    ret = glusterd_get_next_global_opt_version_str(conf->opts, &next_version);
+    if (ret)
+        goto out;
+
+    ret = dict_set_strn(dup_opt, GLUSTERD_GLOBAL_OPT_VERSION,
+                        SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    }
+
+    ret = glusterd_store_options(this, dup_opt);
+    if (ret)
+        goto out;
+
+    if (glusterd_is_quorum_changed(conf->opts, key, NULL))
+        quorum_action = _gf_true;
+
+    ret = dict_set_dynstrn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                           SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    } else
+        next_version = NULL;
+
+    if (!all) {
+        dict_del(conf->opts, key);
+    } else {
+        dict_foreach(conf->opts, _delete_reconfig_global_opt, &is_force);
+    }
 out:
-        return 0;
+    GF_FREE(key_fixed);
+    if (dup_opt)
+        dict_unref(dup_opt);
+
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    GF_FREE(next_version);
+    return ret;
 }
 
 static int
-glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
-                        int32_t *is_force)
+glusterd_op_reset_volume(dict_t *dict, char **op_rspstr)
 {
-        int                      ret = 0;
-        data_t                  *value = NULL;
-        char                    *key_fixed = NULL;
-        xlator_t                *this = NULL;
-        glusterd_svc_t          *svc  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo->dict);
-        GF_ASSERT (key);
-
-        if (!strncmp(key, "all", 3)) {
-                dict_foreach (volinfo->dict, _delete_reconfig_opt, is_force);
-                ret = glusterd_enable_default_options (volinfo, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FAIL_DEFAULT_OPT_SET, "Failed to set "
-                                "default options on reset for volume %s",
-                                volinfo->volname);
-                        goto out;
-                }
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    int32_t is_force = 0;
+    gf_boolean_t quorum_action = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") == 0) {
+        ret = glusterd_op_reset_all_volume_options(this, dict);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+    if (ret)
+        is_force = 0;
+
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get option key");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (strcmp(key, "all") &&
+        glusterd_check_option_exists(key, &key_fixed) != 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "volinfo dict inconsistency: option %s not found", key);
+        ret = -1;
+        goto out;
+    }
+    if (key_fixed)
+        key = key_fixed;
+
+    if (glusterd_is_quorum_changed(volinfo->dict, key, NULL))
+        quorum_action = _gf_true;
+
+    ret = glusterd_options_reset(volinfo, key, &is_force);
+    if (ret == -1) {
+        gf_asprintf(op_rspstr, "Volume reset : failed");
+    } else if (is_force & GD_OP_PROTECTED) {
+        if (is_force & GD_OP_UNPROTECTED) {
+            gf_asprintf(op_rspstr,
+                        "All unprotected fields were"
+                        " reset. To reset the protected fields,"
+                        " use 'force'.");
         } else {
-                value = dict_get (volinfo->dict, key);
-                if (!value) {
-                        gf_msg_debug (this->name, 0,
-                                "no value set for option %s", key);
-                        goto out;
-                }
-                _delete_reconfig_opt (volinfo->dict, key, value, is_force);
-                ret = glusterd_enable_default_options (volinfo, key);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FAIL_DEFAULT_OPT_SET, "Failed to set "
-                                "default value for option '%s' on reset for "
-                                "volume %s", key, volinfo->volname);
-                        goto out;
-                }
+            ret = -1;
+            gf_asprintf(op_rspstr,
+                        "'%s' is protected. To reset"
+                        " use 'force'.",
+                        key);
         }
+    }
 
-        gd_update_volume_op_versions (volinfo);
-        if (!volinfo->is_snap_volume) {
-                svc = &(volinfo->snapd.svc);
-                ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
-                if (ret)
-                        goto out;
+    if (!strcmp(key, "ganesha.enable") || !strcmp(key, "all")) {
+        if (glusterd_check_ganesha_export(volinfo) &&
+            is_origin_glusterd(dict)) {
+            ret = manage_export_config(volname, "off", op_rspstr);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_GNS_RESET_FAIL,
+                       "Could not reset ganesha.enable key");
         }
+    }
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "Unable to create volfile for"
-                        " 'volume reset'");
-                ret = -1;
-                goto out;
-        }
+out:
+    GF_FREE(key_fixed);
+    if (quorum_action)
+        glusterd_do_quorum_action();
 
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                goto out;
+    gf_msg_debug(this->name, 0, "'volume reset' returning %d", ret);
+    return ret;
+}
 
-        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                ret = glusterd_svcs_reconfigure ();
-                if (ret)
-                        goto out;
+int
+glusterd_stop_bricks(glusterd_volinfo_t *volinfo)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        /*TODO: Need to change @del_brick in brick_stop to _gf_true
+         * once we enable synctask in peer rpc prog */
+        if (glusterd_brick_stop(volinfo, brickinfo, _gf_false)) {
+            gf_event(EVENT_BRICK_STOP_FAILED, "peer=%s;volume=%s;brick=%s",
+                     brickinfo->hostname, volinfo->volname, brickinfo->path);
+            return -1;
         }
+    }
 
-        ret = 0;
+    return 0;
+}
+
+int
+glusterd_start_bricks(glusterd_volinfo_t *volinfo)
 
+{
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!brickinfo->start_triggered) {
+            pthread_mutex_lock(&brickinfo->restart_mutex);
+            {
+                /* coverity[SLEEP] */
+                ret = glusterd_brick_start(volinfo, brickinfo, _gf_false,
+                                           _gf_false);
+            }
+            pthread_mutex_unlock(&brickinfo->restart_mutex);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "Failed to start %s:%s for %s", brickinfo->hostname,
+                       brickinfo->path, volinfo->volname);
+                gf_event(EVENT_BRICK_START_FAILED, "peer=%s;volume=%s;brick=%s",
+                         brickinfo->hostname, volinfo->volname,
+                         brickinfo->path);
+                goto out;
+            }
+        }
+    }
+    ret = 0;
 out:
-        GF_FREE (key_fixed);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
+glusterd_update_volumes_dict(glusterd_volinfo_t *volinfo)
 {
-        char            *key            = NULL;
-        char            *key_fixed      = NULL;
-        int             ret             = -1;
-        int32_t         is_force        = 0;
-        glusterd_conf_t *conf           = NULL;
-        dict_t          *dup_opt        = NULL;
-        gf_boolean_t    all             = _gf_false;
-        char            *next_version   = NULL;
-        gf_boolean_t    quorum_action   = _gf_false;
-        gf_boolean_t     option         = _gf_false;
-        char            *op_errstr      = NULL;
-
-        conf = this->private;
-        ret = dict_get_str (dict, "key", &key);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *address_family_str = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* 3.9.0 onwards gNFS will be disabled by default. In case of an upgrade
+     * from anything below than 3.9.0 to 3.9.x the volume's dictionary will
+     * not have 'nfs.disable' key set which means the same will not be set
+     * to on until explicitly done. setnfs.disable to 'on' at op-version
+     * bump up flow is the ideal way here. The same is also applicable for
+     * transport.address-family where if the transport type is set to tcp
+     * then transport.address-family is defaulted to 'inet'.
+     */
+    if (conf->op_version >= GD_OP_VERSION_3_9_0) {
+        if (dict_get_str_boolean(volinfo->dict, NFS_DISABLE_MAP_KEY, 1)) {
+            ret = dict_set_dynstr_with_alloc(volinfo->dict, NFS_DISABLE_MAP_KEY,
+                                             "on");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set "
+                       "option ' NFS_DISABLE_MAP_KEY ' on "
+                       "volume %s",
+                       volinfo->volname);
+                goto out;
+            }
+        }
+        ret = dict_get_strn(volinfo->dict, "transport.address-family",
+                            SLEN("transport.address-family"),
+                            &address_family_str);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get key");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "force", &is_force);
-        if (ret)
-                is_force = 0;
-
-        if (strcmp (key, "all")) {
-                ret = glusterd_check_option_exists (key, &key_fixed);
-                if (ret <= 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_INVALID_ENTRY, "Option %s does not "
-                                "exist", key);
-                        ret = -1;
-                        goto out;
+            if (volinfo->transport_type == GF_TRANSPORT_TCP) {
+                ret = dict_set_dynstr_with_alloc(
+                    volinfo->dict, "transport.address-family", "inet");
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_DICT_SET_FAILED,
+                           "failed to set transport."
+                           "address-family on %s",
+                           volinfo->volname);
+                    goto out;
                 }
-        } else {
-                all = _gf_true;
+            }
         }
+    }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
 
-        if (key_fixed)
-                key = key_fixed;
-        option = dict_get_str_boolean (conf->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
-                            _gf_false);
-        if (option) {
-                ret = tear_down_cluster();
-                if (ret == -1)
-                        gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Could not tear down NFS-Ganesha cluster");
-                ret =  stop_ganesha (&op_errstr);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                GD_MSG_NFS_GNS_STOP_FAIL,
-                                "Could not stop NFS-Ganesha service");
-        }
+out:
+    return ret;
+}
 
-        ret = -1;
-        dup_opt = dict_new ();
-        if (!dup_opt)
-                goto out;
-        if (!all) {
-                dict_copy (conf->opts, dup_opt);
-                dict_del (dup_opt, key);
-        }
-        ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                        &next_version);
-        if (ret)
-                goto out;
+static int
+glusterd_set_brick_mx_opts(dict_t *dict, char *key, char *value,
+                           char **op_errstr)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
-        if (ret)
-                goto out;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, key, out);
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
 
-        ret = glusterd_store_options (this, dup_opt);
-        if (ret)
-                goto out;
+    ret = 0;
 
-        if (glusterd_is_quorum_changed (conf->opts, key, NULL))
-                quorum_action = _gf_true;
+    priv = this->private;
 
-        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                               next_version);
-        if (ret)
-                goto out;
-        else
-                next_version = NULL;
+    if (!strcmp(key, GLUSTERD_BRICK_MULTIPLEX_KEY)) {
+        ret = dict_set_dynstrn(priv->opts, GLUSTERD_BRICK_MULTIPLEX_KEY,
+                               SLEN(GLUSTERD_BRICK_MULTIPLEX_KEY),
+                               gf_strdup(value));
+    }
 
-        if (!all) {
-                dict_del (conf->opts, key);
-        } else {
-                dict_foreach (conf->opts, _delete_reconfig_global_opt,
-                              &is_force);
-        }
 out:
-        GF_FREE (key_fixed);
-        if (dup_opt)
-                dict_unref (dup_opt);
-
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        GF_FREE (next_version);
-        return ret;
+    return ret;
 }
 
+/* This is a hack to prevent client-io-threads from being loaded in the graph
+ * when the cluster-op-version is bumped up from 3.8.x to 3.13.x. The key is
+ * deleted subsequently in glusterd_create_volfiles(). */
 static int
-glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
+glusterd_dict_set_skip_cliot_key(glusterd_volinfo_t *volinfo)
 {
-        glusterd_volinfo_t      *volinfo    = NULL;
-        int                     ret         = -1;
-        char                    *volname    = NULL;
-        char                    *key        = NULL;
-        char                    *key_fixed  = NULL;
-        int32_t                 is_force    = 0;
-        gf_boolean_t            quorum_action = _gf_false;
-        xlator_t                *this         = NULL;
-
-        this = THIS;
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
+    return dict_set_int32n(volinfo->dict, "skip-CLIOT", SLEN("skip-CLIOT"), 1);
+}
 
-        if (strcasecmp (volname, "all") == 0) {
-                ret = glusterd_op_reset_all_volume_options (this, dict);
-                goto out;
-        }
+static int
+glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict,
+                                   char **op_errstr)
+{
+    char *key = NULL;
+    char *key_fixed = NULL;
+    char *value = NULL;
+    char *dup_value = NULL;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    dict_t *dup_opt = NULL;
+    char *next_version = NULL;
+    gf_boolean_t quorum_action = _gf_false;
+    uint32_t op_version = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_svc_t *svc = NULL;
+    gf_boolean_t svcs_reconfigure = _gf_false;
+
+    conf = this->private;
+    ret = dict_get_strn(dict, "key1", SLEN("key1"), &key);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=key1", NULL);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "value1", SLEN("value1"), &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "invalid key,value pair in 'volume set'");
+        goto out;
+    }
+
+    ret = glusterd_check_option_exists(key, &key_fixed);
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+               "Invalid key %s", key);
+        ret = -1;
+        goto out;
+    }
+
+    if (key_fixed)
+        key = key_fixed;
+
+    ret = glusterd_set_shared_storage(dict, key, value, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SHARED_STRG_SET_FAIL,
+               "Failed to set shared storage option");
+        goto out;
+    }
+
+    ret = glusterd_set_brick_mx_opts(dict, key, value, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_MX_SET_FAIL,
+               "Failed to set brick multiplexing option");
+        goto out;
+    }
+
+    /* If the key is cluster.op-version, set conf->op_version to the value
+     * if needed and save it.
+     */
+    if (strcmp(key, GLUSTERD_GLOBAL_OP_VERSION_KEY) == 0) {
+        ret = 0;
 
-        ret = dict_get_int32 (dict, "force", &is_force);
+        ret = gf_string2uint(value, &op_version);
         if (ret)
-                is_force = 0;
+            goto out;
+
+        if (op_version >= conf->op_version) {
+            conf->op_version = op_version;
+
+            /* When a bump up happens, update the quota.conf file
+             * as well. This is because, till 3.7 we had a quota
+             * conf version v1.1 in quota.conf. When inode-quota
+             * feature is introduced, this needs to be changed to
+             * v1.2 in quota.conf and 16 bytes uuid in quota.conf
+             * needs to be changed to 17 bytes. Look
+             * glusterd_store_quota_config for more details.
+             */
+            cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+            {
+                ret = glusterd_store_quota_config(
+                    volinfo, NULL, NULL, GF_QUOTA_OPTION_TYPE_UPGRADE, NULL);
+                if (ret)
+                    goto out;
+                ret = glusterd_update_volumes_dict(volinfo);
+                if (ret)
+                    goto out;
 
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get option key");
-                goto out;
-        }
+                if (glusterd_dict_set_skip_cliot_key(volinfo))
+                    goto out;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
-        }
+                if (!volinfo->is_snap_volume) {
+                    svc = &(volinfo->snapd.svc);
+                    ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+                    if (ret)
+                        goto out;
+                }
 
-        if (strcmp (key, "all") &&
-            glusterd_check_option_exists (key, &key_fixed) != 1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY,
-                        "volinfo dict inconsistency: option %s not found",
-                        key);
-                ret = -1;
-                goto out;
-        }
-        if (key_fixed)
-                key = key_fixed;
+                svc = &(volinfo->gfproxyd.svc);
+                ret = svc->reconfigure(volinfo);
+                if (ret)
+                    goto out;
 
-        if (glusterd_is_quorum_changed (volinfo->dict, key, NULL))
-                quorum_action = _gf_true;
-        ret = glusterd_check_ganesha_export (volinfo);
-        if (ret) {
-                ret = ganesha_manage_export (volname, "off", op_rspstr, _gf_false);
+                svc = &(volinfo->shd.svc);
+                ret = svc->reconfigure(volinfo);
+                if (ret)
+                    goto out;
+
+                ret = glusterd_create_volfiles_and_notify_services(volinfo);
                 if (ret) {
-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                GD_MSG_NFS_GNS_RESET_FAIL,
-                                "Could not reset ganesha.enable key");
-                        ret = 0;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_VOLFILE_CREATE_FAIL,
+                           "Unable to create volfile for"
+                           " 'volume set'");
+                    goto out;
                 }
-        }
-
-        ret = glusterd_options_reset (volinfo, key, &is_force);
-        if (ret == -1) {
-                gf_asprintf(op_rspstr, "Volume reset : failed");
-        } else if (is_force & GD_OP_PROTECTED) {
-                if (is_force & GD_OP_UNPROTECTED) {
-                        gf_asprintf (op_rspstr, "All unprotected fields were"
-                                     " reset. To reset the protected fields,"
-                                     " use 'force'.");
-                } else {
-                        ret = -1;
-                        gf_asprintf (op_rspstr, "'%s' is protected. To reset"
-                                     " use 'force'.", key);
+                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                    svcs_reconfigure = _gf_true;
                 }
-        }
-
-out:
-        GF_FREE (key_fixed);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-
-        gf_msg_debug (this->name, 0, "'volume reset' returning %d", ret);
-        return ret;
-}
-
-int
-glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
-{
-        glusterd_brickinfo_t                    *brickinfo = NULL;
+            }
+            if (svcs_reconfigure) {
+                ret = glusterd_svcs_reconfigure(NULL);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL,
+                           "Unable to restart "
+                           "services");
+                    goto out;
+                }
+            }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-            /*TODO: Need to change @del_brick in brick_stop to _gf_true
-             * once we enable synctask in peer rpc prog */
-                if (glusterd_brick_stop (volinfo, brickinfo, _gf_false))
-                        return -1;
+            ret = glusterd_store_global_info(this);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_STORE_FAIL,
+                       "Failed to store op-version.");
+            }
         }
+        /* No need to save cluster.op-version in conf->opts
+         */
+        goto out;
+    }
+    ret = -1;
+    dup_opt = dict_new();
+    if (!dup_opt) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+    dict_copy(conf->opts, dup_opt);
+    ret = dict_set_str(dup_opt, key, value);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    ret = glusterd_get_next_global_opt_version_str(conf->opts, &next_version);
+    if (ret)
+        goto out;
+
+    ret = dict_set_strn(dup_opt, GLUSTERD_GLOBAL_OPT_VERSION,
+                        SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    }
+
+    ret = glusterd_store_options(this, dup_opt);
+    if (ret)
+        goto out;
+
+    if (glusterd_is_quorum_changed(conf->opts, key, value))
+        quorum_action = _gf_true;
+
+    ret = dict_set_dynstrn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                           SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    } else
+        next_version = NULL;
+
+    dup_value = gf_strdup(value);
+    if (!dup_value)
+        goto out;
+
+    ret = dict_set_dynstr(conf->opts, key, dup_value);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    } else
+        dup_value = NULL; /* Protect the allocation from GF_FREE */
 
-        return 0;
+out:
+    GF_FREE(dup_value);
+    GF_FREE(key_fixed);
+    if (dup_opt)
+        dict_unref(dup_opt);
+
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    GF_FREE(next_version);
+    return ret;
 }
 
 int
-glusterd_start_bricks (glusterd_volinfo_t *volinfo)
+glusterd_op_get_max_opversion(char **op_errstr, dict_t *rsp_dict)
 {
-        int                      ret            = -1;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
+    int ret = -1;
 
-        GF_ASSERT (volinfo);
+    GF_VALIDATE_OR_GOTO(THIS->name, rsp_dict, out);
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_start (volinfo, brickinfo, _gf_false);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_DISCONNECTED,
-                                "Failed to start %s:%s for %s",
-                                brickinfo->hostname, brickinfo->path,
-                                volinfo->volname);
-                        goto out;
-                }
-        }
+    ret = dict_set_int32n(rsp_dict, "max-opversion", SLEN("max-opversion"),
+                          GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Setting value for max-opversion to dict failed");
+        goto out;
+    }
 
-        ret = 0;
 out:
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_set_all_volume_options (xlator_t *this, dict_t *dict,
-                                    char **op_errstr)
+glusterd_set_shared_storage(dict_t *dict, char *key, char *value,
+                            char **op_errstr)
 {
-        char            *key            = NULL;
-        char            *key_fixed      = NULL;
-        char            *value          = NULL;
-        char            *dup_value      = NULL;
-        int             ret             = -1;
-        glusterd_conf_t *conf           = NULL;
-        dict_t          *dup_opt        = NULL;
-        char            *next_version   = NULL;
-        gf_boolean_t    quorum_action   = _gf_false;
-        uint32_t        op_version      = 0;
-        glusterd_volinfo_t  *volinfo    = NULL;
-
-        conf = this->private;
-        ret = dict_get_str (dict, "key1", &key);
-        if (ret)
-                goto out;
-
-        ret = dict_get_str (dict, "value1", &value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "invalid key,value pair in 'volume set'");
-                goto out;
-        }
-
-        ret = glusterd_check_option_exists (key, &key_fixed);
-        if (ret <= 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UNKNOWN_KEY, "Invalid key %s", key);
-                ret = -1;
-                goto out;
-        }
-
-        if (key_fixed)
-                key = key_fixed;
-
-        ret = glusterd_set_shared_storage (dict, key, value, op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SHARED_STRG_SET_FAIL,
-                        "Failed to set shared storage option");
-                goto out;
-        }
-
-        /* If the key is cluster.op-version, set conf->op_version to the value
-         * if needed and save it.
-         */
-        if (strcmp(key, "cluster.op-version") == 0) {
-                ret = 0;
-
-                ret = gf_string2uint (value, &op_version);
-                if (ret)
-                        goto out;
-
-                if (op_version >= conf->op_version) {
-                        conf->op_version = op_version;
-                        ret = glusterd_store_global_info (this);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_OP_VERS_STORE_FAIL,
-                                        "Failed to store op-version.");
-                        }
-                }
-                cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                        ret = glusterd_store_volinfo
-                                (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                        if (ret)
-                                goto out;
-                }
-                /* No need to save cluster.op-version in conf->opts
-                 */
-                goto out;
-        }
+    int32_t ret = -1;
+    char hooks_args[PATH_MAX] = {
+        0,
+    };
+    char errstr[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, key, out);
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = 0;
+
+    if (strcmp(key, GLUSTERD_SHARED_STORAGE_KEY)) {
+        goto out;
+    }
+
+    /* Re-create the brick path so as to be *
+     * able to re-use it                    *
+     */
+    ret = recursive_rmdir(GLUSTER_SHARED_STORAGE_BRICK_DIR);
+    if (ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Failed to remove shared "
+                 "storage brick(%s). "
+                 "Reason: %s",
+                 GLUSTER_SHARED_STORAGE_BRICK_DIR, strerror(errno));
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, "%s",
+               errstr);
         ret = -1;
-        dup_opt = dict_new ();
-        if (!dup_opt)
-                goto out;
-        dict_copy (conf->opts, dup_opt);
-        ret = dict_set_str (dup_opt, key, value);
-        if (ret)
-                goto out;
-
-        ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                        &next_version);
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_options (this, dup_opt);
-        if (ret)
-                goto out;
-
-        if (glusterd_is_quorum_changed (conf->opts, key, value))
-                quorum_action = _gf_true;
-
-        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                               next_version);
-        if (ret)
-                goto out;
-        else
-                next_version = NULL;
+        goto out;
+    }
+
+    ret = mkdir_p(GLUSTER_SHARED_STORAGE_BRICK_DIR, 0755, _gf_true);
+    if (-1 == ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Failed to create shared "
+                 "storage brick(%s). "
+                 "Reason: %s",
+                 GLUSTER_SHARED_STORAGE_BRICK_DIR, strerror(errno));
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED, "%s",
+               errstr);
+        goto out;
+    }
+
+    if (is_origin_glusterd(dict)) {
+        len = snprintf(hooks_args, sizeof(hooks_args),
+                       "is_originator=1,local_node_hostname=%s",
+                       local_node_hostname);
+    } else {
+        len = snprintf(hooks_args, sizeof(hooks_args),
+                       "is_originator=0,local_node_hostname=%s",
+                       local_node_hostname);
+    }
+    if ((len < 0) || (len >= sizeof(hooks_args))) {
+        ret = -1;
+        goto out;
+    }
 
-        dup_value = gf_strdup (value);
-        if (!dup_value)
-                goto out;
-
-        ret = dict_set_dynstr (conf->opts, key, dup_value);
-        if (ret)
-                goto out;
-        else
-                dup_value = NULL; /* Protect the allocation from GF_FREE */
+    ret = dict_set_dynstr_with_alloc(dict, "hooks_args", hooks_args);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set"
+               " hooks_args in dict.");
+        goto out;
+    }
 
 out:
-        GF_FREE (dup_value);
-        GF_FREE (key_fixed);
-        if (dup_opt)
-                dict_unref (dup_opt);
-
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        GF_FREE (next_version);
-        return ret;
+    if (ret && strlen(errstr)) {
+        *op_errstr = gf_strdup(errstr);
+    }
+
+    return ret;
 }
 
 static int
-glusterd_set_shared_storage (dict_t *dict, char *key, char *value,
-                             char **op_errstr)
+glusterd_op_set_volume(dict_t *dict, char **errstr)
 {
-        int32_t       ret                  = -1;
-        int32_t       exists               = -1;
-        int32_t       count                = -1;
-        char          hooks_args[PATH_MAX] = {0, };
-        char          errstr[PATH_MAX]     = {0, };
-        xlator_t     *this                 = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, key, out);
-        GF_VALIDATE_OR_GOTO (this->name, value, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = 0;
-
-        if (strcmp (key, GLUSTERD_SHARED_STORAGE_KEY)) {
-                goto out;
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int count = 1;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    char *value = NULL;
+    char keystr[50] = {
+        0,
+    };
+    int keylen;
+    gf_boolean_t global_opt = _gf_false;
+    gf_boolean_t global_opts_set = _gf_false;
+    glusterd_volinfo_t *voliter = NULL;
+    int32_t dict_count = 0;
+    gf_boolean_t check_op_version = _gf_false;
+    uint32_t new_op_version = 0;
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &dict_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Count(dict),not set in Volume-Set");
+        goto out;
+    }
+
+    if (dict_count == 0) {
+        ret = glusterd_volset_help(NULL, errstr);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") == 0) {
+        ret = glusterd_op_set_all_volume_options(this, dict, errstr);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    /* TODO: Remove this once v3.3 compatibility is not required */
+    check_op_version = dict_get_str_boolean(dict, "check-op-version",
+                                            _gf_false);
+
+    if (check_op_version) {
+        ret = dict_get_uint32(dict, "new-op-version", &new_op_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get new op-version from dict");
+            goto out;
         }
+    }
 
-        /* Re-create the brick path so as to be *
-         * able to re-use it                    *
-         */
-        ret = recursive_rmdir (GLUSTER_SHARED_STORAGE_BRICK_DIR);
+    for (count = 1; ret != -1; count++) {
+        keylen = snprintf(keystr, sizeof(keystr), "key%d", count);
+        ret = dict_get_strn(dict, keystr, keylen, &key);
+        if (ret)
+            break;
+
+        keylen = snprintf(keystr, sizeof(keystr), "value%d", count);
+        ret = dict_get_strn(dict, keystr, keylen, &value);
         if (ret) {
-                snprintf (errstr, PATH_MAX,
-                          "Failed to remove shared "
-                          "storage brick(%s). "
-                          "Reason: %s", GLUSTER_SHARED_STORAGE_BRICK_DIR,
-                          strerror (errno));
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "%s", errstr);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "invalid key,value pair in 'volume set'");
+            ret = -1;
+            goto out;
+        }
+
+        if (strcmp(key, "config.memory-accounting") == 0) {
+            ret = gf_string2boolean(value, &volinfo->memory_accounting);
+            if (ret == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid value in key-value pair.");
+                goto out;
+            }
+        }
+
+        if (strcmp(key, "config.transport") == 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_TRANSPORT_TYPE_CHANGE,
+                   "changing transport-type for volume %s to %s", volname,
+                   value);
+            ret = 0;
+            if (strcasecmp(value, "rdma") == 0) {
+                volinfo->transport_type = GF_TRANSPORT_RDMA;
+            } else if (strcasecmp(value, "tcp") == 0) {
+                volinfo->transport_type = GF_TRANSPORT_TCP;
+            } else if ((strcasecmp(value, "tcp,rdma") == 0) ||
+                       (strcasecmp(value, "rdma,tcp") == 0)) {
+                volinfo->transport_type = GF_TRANSPORT_BOTH_TCP_RDMA;
+            } else {
                 ret = -1;
                 goto out;
+            }
         }
 
-        ret = mkdir_p (GLUSTER_SHARED_STORAGE_BRICK_DIR, 0777, _gf_true);
-        if (-1 == ret) {
-                snprintf (errstr, PATH_MAX,
-                          "Failed to create shared "
-                          "storage brick(%s). "
-                          "Reason: %s", GLUSTER_SHARED_STORAGE_BRICK_DIR,
-                          strerror (errno));
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "%s", errstr);
+        ret = glusterd_check_ganesha_cmd(key, value, errstr, dict);
+        if (ret == -1)
+            goto out;
+
+        if (!is_key_glusterd_hooks_friendly(key)) {
+            ret = glusterd_check_option_exists(key, &key_fixed);
+            GF_ASSERT(ret);
+            if (ret <= 0) {
+                key_fixed = NULL;
                 goto out;
+            }
         }
 
-        if (is_origin_glusterd (dict)) {
-                snprintf(hooks_args, sizeof(hooks_args),
-                         "is_originator=1,local_node_hostname=%s",
-                         local_node_hostname);
-        } else {
-                snprintf(hooks_args, sizeof(hooks_args),
-                         "is_originator=0,local_node_hostname=%s",
-                         local_node_hostname);
+        global_opt = _gf_false;
+        if (glusterd_check_globaloption(key)) {
+            global_opt = _gf_true;
+            global_opts_set = _gf_true;
         }
 
-        ret = dict_set_dynstr_with_alloc (dict, "hooks_args", hooks_args);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set"
-                        " hooks_args in dict.");
-                goto out;
-        }
+        if (!global_opt)
+            value = gf_strdup(value);
 
-out:
-        if (ret && strlen(errstr)) {
-                *op_errstr = gf_strdup (errstr);
+        if (!value) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                   "Unable to set the options in 'volume set'");
+            ret = -1;
+            goto out;
         }
 
-        return ret;
-}
+        if (key_fixed)
+            key = key_fixed;
 
+        if (glusterd_is_quorum_changed(volinfo->dict, key, value))
+            quorum_action = _gf_true;
 
-static int
-glusterd_op_set_volume (dict_t *dict, char **errstr)
-{
-        int                                      ret = 0;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        char                                    *volname = NULL;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        int                                      count = 1;
-        char                                    *key = NULL;
-        char                                    *key_fixed = NULL;
-        char                                    *value = NULL;
-        char                                     str[50] = {0, };
-        char                                    *op_errstr = NULL;
-        gf_boolean_t                             global_opt    = _gf_false;
-        gf_boolean_t                             global_opts_set = _gf_false;
-        glusterd_volinfo_t                      *voliter = NULL;
-        int32_t                                  dict_count = 0;
-        gf_boolean_t                             check_op_version = _gf_false;
-        uint32_t                                 new_op_version = 0;
-        gf_boolean_t                             quorum_action  = _gf_false;
-        glusterd_svc_t                          *svc            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int32 (dict, "count", &dict_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Count(dict),not set in Volume-Set");
+        if (global_opt) {
+            cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+            {
+                value = gf_strdup(value);
+                ret = dict_set_dynstr(voliter->dict, key, value);
+                if (ret)
+                    goto out;
+            }
+        } else {
+            ret = dict_set_dynstr(volinfo->dict, key, value);
+            if (ret)
                 goto out;
         }
 
-        if (dict_count == 0) {
-                ret = glusterd_volset_help (NULL, &op_errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOL_SET_FAIL, "%s",
-                                       (op_errstr)? op_errstr:
-                                       "Volume set help internal error");
-                }
-
-                GF_FREE(op_errstr);
-                goto out;
-         }
+        if (key_fixed) {
+            GF_FREE(key_fixed);
+            key_fixed = NULL;
+        }
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
+    if (count == 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_OPTIONS_GIVEN,
+               "No options received ");
+        ret = -1;
+        goto out;
+    }
+
+    /* Update the cluster op-version before regenerating volfiles so that
+     * correct volfiles are generated
+     */
+    if (new_op_version > priv->op_version) {
+        priv->op_version = new_op_version;
+        ret = glusterd_store_global_info(this);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_STORE_FAIL,
+                   "Failed to store op-version");
+            goto out;
         }
+    }
+    if (!global_opts_set) {
+        gd_update_volume_op_versions(volinfo);
 
-        if (strcasecmp (volname, "all") == 0) {
-                ret = glusterd_op_set_all_volume_options (this, dict,
-                                                          &op_errstr);
+        if (!volinfo->is_snap_volume) {
+            svc = &(volinfo->snapd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret)
                 goto out;
         }
+        svc = &(volinfo->gfproxyd.svc);
+        ret = svc->reconfigure(volinfo);
+        if (ret)
+            goto out;
+
+        svc = &(volinfo->shd.svc);
+        ret = svc->reconfigure(volinfo);
+        if (ret)
+            goto out;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+        ret = glusterd_create_volfiles_and_notify_services(volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Unable to create volfile for"
+                   " 'volume set'");
+            ret = -1;
+            goto out;
         }
 
-        /* TODO: Remove this once v3.3 compatibility is not required */
-        check_op_version = dict_get_str_boolean (dict, "check-op-version",
-                                                 _gf_false);
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            goto out;
 
-        if (check_op_version) {
-                ret = dict_get_uint32 (dict, "new-op-version", &new_op_version);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get new op-version from dict");
-                        goto out;
-                }
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+            ret = glusterd_svcs_reconfigure(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL,
+                       "Unable to restart services");
+                goto out;
+            }
         }
 
-        for (count = 1; ret != -1 ; count++) {
+    } else {
+        cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+        {
+            volinfo = voliter;
+            gd_update_volume_op_versions(volinfo);
 
-                snprintf (str, sizeof str, "key%d", count);
-                ret = dict_get_str (dict, str, &key);
+            if (!volinfo->is_snap_volume) {
+                svc = &(volinfo->snapd.svc);
+                ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
                 if (ret)
-                        break;
+                    goto out;
+            }
 
-                snprintf (str, sizeof str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "invalid key,value pair in 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (strcmp (key, "config.memory-accounting") == 0) {
-                        ret = gf_string2boolean (value,
-                                                 &volinfo->memory_accounting);
-                }
-
-                if (strcmp (key, "config.transport") == 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_VOL_TRANSPORT_TYPE_CHANGE,
-                                "changing transport-type for volume %s to %s",
-                                volname, value);
-                        ret = 0;
-                        if (strcasecmp (value, "rdma") == 0) {
-                                volinfo->transport_type = GF_TRANSPORT_RDMA;
-                        } else if (strcasecmp (value, "tcp") == 0) {
-                                volinfo->transport_type = GF_TRANSPORT_TCP;
-                        } else if ((strcasecmp (value, "tcp,rdma") == 0) ||
-                                   (strcasecmp (value, "rdma,tcp") == 0)) {
-                                volinfo->transport_type =
-                                        GF_TRANSPORT_BOTH_TCP_RDMA;
-                        } else {
-                                ret = -1;
-                                goto out;
-                        }
-                }
-
-                ret =  glusterd_check_ganesha_cmd (key, value, errstr, dict);
-                if (ret == -1)
-                        goto out;
-                if (!is_key_glusterd_hooks_friendly (key)) {
-                        ret = glusterd_check_option_exists (key, &key_fixed);
-                        GF_ASSERT (ret);
-                        if (ret <= 0) {
-                                key_fixed = NULL;
-                                goto out;
-                        }
-                }
-
-                global_opt = _gf_false;
-                if (glusterd_check_globaloption (key)) {
-                        global_opt = _gf_true;
-                        global_opts_set = _gf_true;
-                }
-
-                if (!global_opt)
-                        value = gf_strdup (value);
-
-                if (!value) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_SET_FAIL,
-                                "Unable to set the options in 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (key_fixed)
-                        key = key_fixed;
-
-                if (glusterd_is_quorum_changed (volinfo->dict, key, value))
-                        quorum_action = _gf_true;
-
-                if (global_opt) {
-                        cds_list_for_each_entry (voliter, &priv->volumes,
-                                                 vol_list) {
-                                value = gf_strdup (value);
-                                ret = dict_set_dynstr (voliter->dict, key,
-                                                       value);
-                                if (ret)
-                                        goto out;
-                        }
-                } else {
-                        ret = dict_set_dynstr (volinfo->dict, key, value);
-                        if (ret)
-                                goto out;
-                }
+            svc = &(volinfo->gfproxyd.svc);
+            ret = svc->reconfigure(volinfo);
+            if (ret)
+                goto out;
 
-                if (key_fixed) {
-                        GF_FREE (key_fixed);
-                        key_fixed = NULL;
-                }
-        }
+            svc = &(volinfo->shd.svc);
+            ret = svc->reconfigure(volinfo);
+            if (ret)
+                goto out;
 
-        if (count == 1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_OPTIONS_GIVEN, "No options received ");
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                       "Unable to create volfile for"
+                       " 'volume set'");
                 ret = -1;
                 goto out;
-        }
+            }
 
-        /* Update the cluster op-version before regenerating volfiles so that
-         * correct volfiles are generated
-         */
-        if (new_op_version > priv->op_version) {
-                priv->op_version = new_op_version;
-                ret = glusterd_store_global_info (this);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OP_VERS_STORE_FAIL,
-                                "Failed to store op-version");
-                        goto out;
-                }
-        }
-        if (!global_opts_set) {
-                gd_update_volume_op_versions (volinfo);
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret)
+                goto out;
 
-                if (!volinfo->is_snap_volume) {
-                        svc = &(volinfo->snapd.svc);
-                        ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
-                        if (ret)
-                                goto out;
-                }
-                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+            if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                ret = glusterd_svcs_reconfigure(volinfo);
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLFILE_CREATE_FAIL,
-                                "Unable to create volfile for"
-                                " 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret)
-                        goto out;
-
-                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                        ret = glusterd_svcs_reconfigure ();
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SVC_RESTART_FAIL,
-                                         "Unable to restart services");
-                                goto out;
-                        }
-                }
-
-        } else {
-                cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                        volinfo = voliter;
-                        gd_update_volume_op_versions (volinfo);
-
-                        if (!volinfo->is_snap_volume) {
-                                svc = &(volinfo->snapd.svc);
-                                ret = svc->manager (svc, volinfo,
-                                                    PROC_START_NO_WAIT);
-                                if (ret)
-                                        goto out;
-                        }
-
-                        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOLFILE_CREATE_FAIL,
-                                        "Unable to create volfile for"
-                                        " 'volume set'");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                        if (ret)
-                                goto out;
-
-                        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                                ret = glusterd_svcs_reconfigure ();
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                GD_MSG_NFS_SERVER_START_FAIL,
-                                                "Unable to restart NFS-Server");
-                                        goto out;
-                                }
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL,
+                           "Unable to restart services");
+                    goto out;
                 }
+            }
         }
+    }
 
- out:
-        GF_FREE (key_fixed);
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        return ret;
+out:
+    GF_FREE(key_fixed);
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    return ret;
 }
 
-
 static int
-glusterd_op_sync_volume (dict_t *dict, char **op_errstr,
-                         dict_t *rsp_dict)
+glusterd_op_sync_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    *hostname = NULL;
-        char                                    msg[2048] = {0,};
-        int                                     count = 1;
-        int                                     vol_count = 0;
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "hostname couldn't be "
-                          "retrieved from msg");
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    int count = 1;
+    int vol_count = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "hostname couldn't be "
+                 "retrieved from msg");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=hostname", NULL);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (!gf_is_local_addr(hostname)) {
+        ret = 0;
+        goto out;
+    }
 
-        if (!gf_is_local_addr (hostname)) {
-                ret = 0;
-                goto out;
+    // volname is not present in case of sync all
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (!ret) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                   "Volume with name: %s "
+                   "not exists",
+                   volname);
+            goto out;
         }
+    }
 
-        //volname is not present in case of sync all
-        ret = dict_get_str (dict, "volname", &volname);
-        if (!ret) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_NOT_FOUND, "Volume with name: %s "
-                                "not exists", volname);
-                        goto out;
-                }
-        }
+    if (!rsp_dict) {
+        // this should happen only on source
+        gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        if (!rsp_dict) {
-                //this should happen only on source
-                ret = 0;
+    if (volname) {
+        ret = glusterd_add_volume_to_dict(volinfo, rsp_dict, 1, "volume");
+        if (ret)
+            goto out;
+        vol_count = 1;
+    } else {
+        cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+        {
+            ret = glusterd_add_volume_to_dict(volinfo, rsp_dict, count,
+                                              "volume");
+            if (ret)
                 goto out;
-        }
 
-        if (volname) {
-                ret = glusterd_add_volume_to_dict (volinfo, rsp_dict,
-                                                   1, "volume");
-                vol_count = 1;
-        } else {
-                cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                        ret = glusterd_add_volume_to_dict (volinfo, rsp_dict,
-                                                           count, "volume");
-                        if (ret)
-                                goto out;
-
-                        vol_count = count++;
-                }
+            vol_count = count++;
         }
-        ret = dict_set_int32 (rsp_dict, "count", vol_count);
+    }
+    ret = dict_set_int32n(rsp_dict, "count", SLEN("count"), vol_count);
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_add_profile_volume_options (glusterd_volinfo_t *volinfo)
+glusterd_add_profile_volume_options(glusterd_volinfo_t *volinfo)
 {
-        int                                     ret = -1;
-        char                                    *latency_key = NULL;
-        char                                    *fd_stats_key = NULL;
-
-        GF_ASSERT (volinfo);
-
-        latency_key = VKEY_DIAG_LAT_MEASUREMENT;
-        fd_stats_key = VKEY_DIAG_CNT_FOP_HITS;
-
-        ret = dict_set_str (volinfo->dict, latency_key, "on");
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "failed to set the volume %s "
-                        "option %s value %s",
-                        volinfo->volname, latency_key, "on");
-                goto out;
-        }
-
-        ret = dict_set_str (volinfo->dict, fd_stats_key, "on");
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "failed to set the volume %s "
-                        "option %s value %s",
-                        volinfo->volname, fd_stats_key, "on");
-                goto out;
-        }
+    int ret = -1;
+
+    GF_ASSERT(volinfo);
+
+    ret = dict_set_nstrn(volinfo->dict, VKEY_DIAG_LAT_MEASUREMENT,
+                         SLEN(VKEY_DIAG_LAT_MEASUREMENT), "on", SLEN("on"));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set the volume %s "
+               "option %s value %s",
+               volinfo->volname, VKEY_DIAG_LAT_MEASUREMENT, "on");
+        goto out;
+    }
+
+    ret = dict_set_nstrn(volinfo->dict, VKEY_DIAG_CNT_FOP_HITS,
+                         SLEN(VKEY_DIAG_CNT_FOP_HITS), "on", SLEN("on"));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set the volume %s "
+               "option %s value %s",
+               volinfo->volname, VKEY_DIAG_CNT_FOP_HITS, "on");
+        goto out;
+    }
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static void
-glusterd_remove_profile_volume_options (glusterd_volinfo_t *volinfo)
+glusterd_remove_profile_volume_options(glusterd_volinfo_t *volinfo)
 {
-        char                                    *latency_key = NULL;
-        char                                    *fd_stats_key = NULL;
+    GF_ASSERT(volinfo);
 
-        GF_ASSERT (volinfo);
-
-        latency_key = VKEY_DIAG_LAT_MEASUREMENT;
-        fd_stats_key = VKEY_DIAG_CNT_FOP_HITS;
-        dict_del (volinfo->dict, latency_key);
-        dict_del (volinfo->dict, fd_stats_key);
+    dict_del_sizen(volinfo->dict, VKEY_DIAG_LAT_MEASUREMENT);
+    dict_del_sizen(volinfo->dict, VKEY_DIAG_CNT_FOP_HITS);
 }
 
-static int
-glusterd_op_stats_volume (dict_t *dict, char **op_errstr,
-                          dict_t *rsp_dict)
+int
+glusterd_op_stats_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_volinfo_t                      *volinfo = NULL;
-        int32_t                                 stats_op = GF_CLI_STATS_NONE;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "volume name get failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exists",
-                          volname);
-
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "%s", msg);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "op", &stats_op);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "volume profile op get failed");
-                goto out;
-        }
-
-        switch (stats_op) {
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t stats_op = GF_CLI_STATS_NONE;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exists", volname);
+
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &stats_op);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume profile op get failed");
+        goto out;
+    }
+
+    switch (stats_op) {
         case GF_CLI_STATS_START:
-                ret = glusterd_add_profile_volume_options (volinfo);
-                if (ret)
-                        goto out;
-                break;
+            ret = glusterd_add_profile_volume_options(volinfo);
+            if (ret)
+                goto out;
+            break;
         case GF_CLI_STATS_STOP:
-                glusterd_remove_profile_volume_options (volinfo);
-                break;
+            glusterd_remove_profile_volume_options(volinfo);
+            break;
         case GF_CLI_STATS_INFO:
         case GF_CLI_STATS_TOP:
-                //info is already collected in brick op.
-                //just goto out;
-                ret = 0;
-                goto out;
-                break;
+            // info is already collected in brick op.
+            // just goto out;
+            ret = 0;
+            goto out;
+            break;
         default:
-                GF_ASSERT (0);
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "Invalid profile op: %d",
-                        stats_op);
-                ret = -1;
-                goto out;
-                break;
-        }
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+            GF_ASSERT(0);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Invalid profile op: %d", stats_op);
+            ret = -1;
+            goto out;
+            break;
+    }
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to create volfile for"
+               " 'volume set'");
+        ret = -1;
+        goto out;
+    }
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "Unable to create volfile for"
-                        " 'volume set'");
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_svcs_reconfigure(volinfo);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                ret = glusterd_svcs_reconfigure ();
-
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-_add_remove_bricks_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
-                            char *prefix)
+_add_remove_bricks_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo,
+                           char *prefix)
 {
-        int             ret = -1;
-        int             count = 0;
-        int             i = 0;
-        char            brick_key[1024] = {0,};
-        char            dict_key[1024] ={0,};
-        char            *brick = NULL;
-        xlator_t        *this = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (prefix);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
+    int ret = -1;
+    int count = 0;
+    int i = 0;
+    char brick_key[16] = {
+        0,
+    };
+    char dict_key[64] = {
+        /* dict_key is small as prefix is up to 32 chars */
+        0,
+    };
+    int keylen;
+    char *brick = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(prefix);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32n(volinfo->rebal.dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get brick count");
+        goto out;
+    }
+
+    keylen = snprintf(dict_key, sizeof(dict_key), "%s.count", prefix);
+    ret = dict_set_int32n(dict, dict_key, keylen, count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set brick count in dict");
+        goto out;
+    }
+
+    for (i = 1; i <= count; i++) {
+        keylen = snprintf(brick_key, sizeof(brick_key), "brick%d", i);
+
+        ret = dict_get_strn(volinfo->rebal.dict, brick_key, keylen, &brick);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to get brick count");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s", brick_key);
+            goto out;
         }
 
-        snprintf (dict_key, sizeof (dict_key), "%s.count", prefix);
-        ret = dict_set_int32 (dict, dict_key, count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set brick count in dict");
-                goto out;
-        }
-
-        for (i = 1; i <= count; i++) {
-                memset (brick_key, 0, sizeof (brick_key));
-                snprintf (brick_key, sizeof (brick_key), "brick%d", i);
-
-                ret = dict_get_str (volinfo->rebal.dict, brick_key, &brick);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get %s", brick_key);
-                        goto out;
-                }
-
-                memset (dict_key, 0, sizeof (dict_key));
-                snprintf (dict_key, sizeof (dict_key), "%s.%s", prefix,
+        keylen = snprintf(dict_key, sizeof(dict_key), "%s.%s", prefix,
                           brick_key);
-                ret = dict_set_str (dict, dict_key, brick);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to add brick to dict");
-                        goto out;
-                }
-                brick = NULL;
+        if ((keylen < 0) || (keylen >= sizeof(dict_key))) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_strn(dict, dict_key, keylen, brick);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add brick to dict");
+            goto out;
         }
+        brick = NULL;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This adds the respective task-id and all available parameters of a task into
  * a dictionary
  */
 static int
-_add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
+_add_task_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
 {
-
-        int             ret = -1;
-        char            key[128] = {0,};
-        char            *uuid_str = NULL;
-        int             status = 0;
-        xlator_t        *this = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        switch (op) {
-        case GD_OP_DETACH_TIER:
+    int ret = -1;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *uuid_str = NULL;
+    int status = 0;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    switch (op) {
         case GD_OP_REMOVE_BRICK:
-                snprintf (key, sizeof (key), "task%d", index);
-                ret = _add_remove_bricks_to_dict (dict, volinfo, key);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_ADD_REMOVE_BRICK_FAIL,
-                                "Failed to add remove bricks to dict");
-                        goto out;
-                }
-        case GD_OP_TIER_MIGRATE:
+            snprintf(key, sizeof(key), "task%d", index);
+            ret = _add_remove_bricks_to_dict(dict, volinfo, key);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_ADD_REMOVE_BRICK_FAIL,
+                       "Failed to add remove bricks to dict");
+                goto out;
+            }
         case GD_OP_REBALANCE:
-                uuid_str = gf_strdup (uuid_utoa (volinfo->rebal.rebalance_id));
-                status = volinfo->rebal.defrag_status;
-                break;
+            uuid_str = gf_strdup(uuid_utoa(volinfo->rebal.rebalance_id));
+            status = volinfo->rebal.defrag_status;
+            break;
 
         default:
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_TASK_ID, "%s operation doesn't have a"
-                        " task_id", gd_op_list[op]);
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "task%d.type", index);
-        ret = dict_set_str (dict, key, (char *)gd_op_list[op]);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Error setting task type in dict");
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "task%d.id", index);
-
-        if (!uuid_str)
-                goto out;
-        ret = dict_set_dynstr (dict, key, uuid_str);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Error setting task id in dict");
-                goto out;
-        }
-        uuid_str = NULL;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "task%d.status", index);
-        ret = dict_set_int32 (dict, key, status);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Error setting task status in dict");
-                goto out;
-        }
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_TASK_ID,
+                   "%s operation doesn't have a"
+                   " task_id",
+                   gd_op_list[op]);
+            goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "task%d.type", index);
+    ret = dict_set_strn(dict, key, keylen, (char *)gd_op_list[op]);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting task type in dict");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "task%d.id", index);
+
+    if (!uuid_str)
+        goto out;
+    ret = dict_set_dynstrn(dict, key, keylen, uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting task id in dict");
+        goto out;
+    }
+    uuid_str = NULL;
+
+    keylen = snprintf(key, sizeof(key), "task%d.status", index);
+    ret = dict_set_int32n(dict, key, keylen, status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting task status in dict");
+        goto out;
+    }
 
 out:
-        if (uuid_str)
-                GF_FREE (uuid_str);
-        return ret;
+    if (uuid_str)
+        GF_FREE(uuid_str);
+    return ret;
 }
 
 static int
-glusterd_aggregate_task_status (dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
+glusterd_aggregate_task_status(dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
 {
-        int        ret   = -1;
-        int        tasks = 0;
-        xlator_t  *this  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
-                if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                        if (volinfo->rebal.op == GD_OP_REMOVE_BRICK)
-                                ret = _add_task_to_dict (rsp_dict, volinfo,
-                                                         GD_OP_DETACH_TIER,
-                                                         tasks);
-                        else if (volinfo->rebal.op == GD_OP_REBALANCE)
-                                ret = _add_task_to_dict (rsp_dict, volinfo,
-                                                         GD_OP_TIER_MIGRATE,
-                                                         tasks);
-                } else
-                ret = _add_task_to_dict (rsp_dict, volinfo,
-                                         volinfo->rebal.op, tasks);
+    int ret = -1;
+    int tasks = 0;
+    xlator_t *this = NULL;
 
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to add task details to dict");
-                        goto out;
-                }
-                tasks++;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        ret = dict_set_int32 (rsp_dict, "tasks", tasks);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Error setting tasks count in dict");
-                goto out;
-        }
-        ret = 0;
+    if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+        ret = _add_task_to_dict(rsp_dict, volinfo, volinfo->rebal.op, tasks);
 
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add task details to dict");
+            goto out;
+        }
+        tasks++;
+    }
+    ret = dict_set_int32n(rsp_dict, "tasks", SLEN("tasks"), tasks);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting tasks count in dict");
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_status_volume (dict_t *dict, char **op_errstr,
-                           dict_t *rsp_dict)
+glusterd_op_status_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                     ret             = -1;
-        int                     node_count      = 0;
-        int                     brick_index     = -1;
-        int                     other_count     = 0;
-        int                     hot_brick_count = -1;
-        int                     other_index     = 0;
-        uint32_t                cmd             = 0;
-        char                   *volname         = NULL;
-        char                   *brick           = NULL;
-        xlator_t               *this            = NULL;
-        glusterd_volinfo_t     *volinfo         = NULL;
-        glusterd_brickinfo_t   *brickinfo       = NULL;
-        glusterd_conf_t        *priv            = NULL;
-        dict_t                 *vol_opts        = NULL;
-        gf_boolean_t            nfs_disabled    = _gf_false;
-        gf_boolean_t            shd_enabled     = _gf_false;
-        gf_boolean_t            origin_glusterd = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-
-        origin_glusterd = is_origin_glusterd (dict);
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    int ret = -1;
+    int node_count = 0;
+    int brick_index = -1;
+    int other_count = 0;
+    int other_index = 0;
+    uint32_t cmd = 0;
+    char *volname = NULL;
+    char *brick = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *vol_opts = NULL;
+#ifdef BUILD_GNFS
+    gf_boolean_t nfs_disabled = _gf_false;
+#endif
+    gf_boolean_t shd_enabled = _gf_false;
+    gf_boolean_t origin_glusterd = _gf_false;
+    int snapd_enabled, bitrot_enabled, volume_quota_enabled;
 
-        if (origin_glusterd) {
-                ret = 0;
-                if ((cmd & GF_CLI_STATUS_ALL)) {
-                        ret = glusterd_get_all_volnames (rsp_dict);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOLNAMES_GET_FAIL,
-                                        "failed to get all volume "
-                                        "names for status");
-                }
-        }
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
 
-        ret = dict_set_uint32 (rsp_dict, "cmd", cmd);
-        if (ret)
-                goto out;
+    GF_ASSERT(priv);
 
-        if (cmd & GF_CLI_STATUS_ALL)
-                goto out;
+    GF_ASSERT(dict);
+
+    origin_glusterd = is_origin_glusterd(dict);
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if (origin_glusterd) {
+        ret = 0;
+        if ((cmd & GF_CLI_STATUS_ALL)) {
+            ret = glusterd_get_all_volnames(rsp_dict);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAMES_GET_FAIL,
+                       "failed to get all volume "
+                       "names for status");
+        }
+    }
+
+    ret = dict_set_uint32(rsp_dict, "cmd", cmd);
+    if (ret)
+        goto out;
+
+    if (cmd & GF_CLI_STATUS_ALL)
+        goto out;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret)
+        goto out;
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume with name: %s "
+               "does not exist",
+               volname);
+        goto out;
+    }
+    vol_opts = volinfo->dict;
+
+    if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+        ret = glusterd_add_node_to_dict(priv->quotad_svc.name, rsp_dict, 0,
+                                        vol_opts);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+#ifdef BUILD_GNFS
+    } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+        ret = glusterd_add_node_to_dict(priv->nfs_svc.name, rsp_dict, 0,
+                                        vol_opts);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+#endif
+    } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
+        ret = glusterd_add_node_to_dict(priv->bitd_svc.name, rsp_dict, 0,
+                                        vol_opts);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
+        ret = glusterd_add_node_to_dict(priv->scrub_svc.name, rsp_dict, 0,
+                                        vol_opts);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
+        ret = glusterd_add_snapd_to_dict(volinfo, rsp_dict, other_index);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+        ret = glusterd_add_shd_to_dict(volinfo, rsp_dict, other_index);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick);
+        if (ret)
+            goto out;
 
-        ret = dict_get_str (dict, "volname", &volname);
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
         if (ret)
-                goto out;
+            goto out;
+
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            goto out;
+
+        glusterd_add_brick_to_dict(volinfo, brickinfo, rsp_dict, ++brick_index);
+        if (cmd & GF_CLI_STATUS_DETAIL)
+            glusterd_add_brick_detail_to_dict(volinfo, brickinfo, rsp_dict,
+                                              brick_index);
+        node_count++;
+
+    } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+        ret = glusterd_aggregate_task_status(rsp_dict, volinfo);
+        goto out;
+
+    } else {
+        snapd_enabled = glusterd_is_snapd_enabled(volinfo);
+        shd_enabled = gd_is_self_heal_enabled(volinfo, vol_opts);
+#ifdef BUILD_GNFS
+        nfs_disabled = dict_get_str_boolean(vol_opts, NFS_DISABLE_MAP_KEY,
+                                            _gf_false);
+#endif
+        volume_quota_enabled = glusterd_is_volume_quota_enabled(volinfo);
+        bitrot_enabled = glusterd_is_bitrot_enabled(volinfo);
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "Volume with name: %s "
-                        "does not exist", volname);
-                goto out;
-        }
-        vol_opts = volinfo->dict;
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            brick_index++;
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
 
-        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                ret = glusterd_add_node_to_dict (priv->nfs_svc.name, rsp_dict,
-                                                 0, vol_opts);
-                if (ret)
-                        goto out;
-                other_count++;
-                node_count++;
+            glusterd_add_brick_to_dict(volinfo, brickinfo, rsp_dict,
+                                       brick_index);
 
-        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                ret = glusterd_add_node_to_dict (priv->shd_svc.name, rsp_dict,
-                                                 0, vol_opts);
-                if (ret)
-                        goto out;
-                other_count++;
-                node_count++;
+            if (cmd & GF_CLI_STATUS_DETAIL) {
+                glusterd_add_brick_detail_to_dict(volinfo, brickinfo, rsp_dict,
+                                                  brick_index);
+            }
+            node_count++;
+        }
 
-        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
-                ret = glusterd_add_node_to_dict (priv->quotad_svc.name,
-                                                 rsp_dict, 0, vol_opts);
+        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
+            other_index = brick_index + 1;
+            if (snapd_enabled) {
+                ret = glusterd_add_snapd_to_dict(volinfo, rsp_dict,
+                                                 other_index);
                 if (ret)
-                        goto out;
+                    goto out;
                 other_count++;
+                other_index++;
                 node_count++;
-        } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
-                ret = glusterd_add_node_to_dict (priv->bitd_svc.name,
-                                                 rsp_dict, 0, vol_opts);
+            }
+
+            if (glusterd_is_shd_compatible_volume(volinfo)) {
+                if (shd_enabled) {
+                    ret = glusterd_add_shd_to_dict(volinfo, rsp_dict,
+                                                   other_index);
+                    if (ret)
+                        goto out;
+                    other_count++;
+                    other_index++;
+                    node_count++;
+                }
+            }
+#ifdef BUILD_GNFS
+            if (!nfs_disabled) {
+                ret = glusterd_add_node_to_dict(priv->nfs_svc.name, rsp_dict,
+                                                other_index, vol_opts);
                 if (ret)
-                        goto out;
+                    goto out;
+                other_index++;
                 other_count++;
                 node_count++;
-        } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
-                ret = glusterd_add_node_to_dict (priv->scrub_svc.name,
-                                                 rsp_dict, 0, vol_opts);
+            }
+#endif
+            if (volume_quota_enabled) {
+                ret = glusterd_add_node_to_dict(priv->quotad_svc.name, rsp_dict,
+                                                other_index, vol_opts);
                 if (ret)
-                        goto out;
+                    goto out;
                 other_count++;
                 node_count++;
-        } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
-                ret = glusterd_add_snapd_to_dict (volinfo, rsp_dict,
-                                                  other_index);
+                other_index++;
+            }
+
+            if (bitrot_enabled) {
+                ret = glusterd_add_node_to_dict(priv->bitd_svc.name, rsp_dict,
+                                                other_index, vol_opts);
                 if (ret)
-                        goto out;
+                    goto out;
                 other_count++;
                 node_count++;
-        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brick);
+                other_index++;
+                /* For handling scrub status. Scrub daemon will be
+                 * running automatically when bitrot is enable */
+                ret = glusterd_add_node_to_dict(priv->scrub_svc.name, rsp_dict,
+                                                other_index, vol_opts);
                 if (ret)
-                        goto out;
-
-                ret = glusterd_volume_brickinfo_get_by_brick (brick,
-                                                              volinfo,
-                                                              &brickinfo,
-                                                              _gf_false);
-                if (ret)
-                        goto out;
-
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        goto out;
-
-                glusterd_add_brick_to_dict (volinfo, brickinfo, rsp_dict,
-                                            ++brick_index);
-                if (cmd & GF_CLI_STATUS_DETAIL)
-                        glusterd_add_brick_detail_to_dict (volinfo, brickinfo,
-                                                           rsp_dict,
-                                                           brick_index);
+                    goto out;
+                other_count++;
                 node_count++;
-
-        } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
-                ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
-                goto out;
-
-        } else {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        brick_index++;
-                        if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                                continue;
-
-                        glusterd_add_brick_to_dict (volinfo, brickinfo,
-                                                    rsp_dict, brick_index);
-
-                        if (cmd & GF_CLI_STATUS_DETAIL) {
-                                glusterd_add_brick_detail_to_dict (volinfo,
-                                                                   brickinfo,
-                                                                   rsp_dict,
-                                                                   brick_index);
-                        }
-                        node_count++;
-                }
-
-                if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
-                        other_index = brick_index + 1;
-                        if (glusterd_is_snapd_enabled (volinfo)) {
-                                ret = glusterd_add_snapd_to_dict (volinfo,
-                                                                  rsp_dict,
-                                                                  other_index);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                other_index++;
-                                node_count++;
-                        }
-
-                        nfs_disabled = dict_get_str_boolean (vol_opts,
-                                                             NFS_DISABLE_MAP_KEY,
-                                                             _gf_false);
-                        if (!nfs_disabled) {
-                                ret = glusterd_add_node_to_dict
-                                                           (priv->nfs_svc.name,
-                                                            rsp_dict,
-                                                            other_index,
-                                                            vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_index++;
-                                other_count++;
-                                node_count++;
-                        }
-
-                        if (glusterd_is_shd_compatible_volume (volinfo))
-                                shd_enabled = gd_is_self_heal_enabled
-                                                         (volinfo, vol_opts);
-                        if (shd_enabled) {
-                                ret = glusterd_add_node_to_dict
-                                        (priv->shd_svc.name, rsp_dict,
-                                         other_index, vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                node_count++;
-                                other_index++;
-                        }
-
-                        if (glusterd_is_volume_quota_enabled (volinfo)) {
-                                ret = glusterd_add_node_to_dict
-                                                        (priv->quotad_svc.name,
-                                                         rsp_dict,
-                                                         other_index,
-                                                         vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                node_count++;
-                                other_index++;
-                        }
-
-                        if (glusterd_is_bitrot_enabled (volinfo)) {
-                                ret = glusterd_add_node_to_dict
-                                                        (priv->bitd_svc.name,
-                                                         rsp_dict,
-                                                         other_index,
-                                                         vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                node_count++;
-                                other_index++;
-                        }
-
-                        /* For handling scrub status. Scrub daemon will be
-                         * running automatically when bitrot is enable*/
-                        if (glusterd_is_bitrot_enabled (volinfo)) {
-                                ret = glusterd_add_node_to_dict
-                                                        (priv->scrub_svc.name,
-                                                         rsp_dict,
-                                                         other_index,
-                                                         vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                node_count++;
-                        }
-                }
-        }
-
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                hot_brick_count = volinfo->tier_info.hot_brick_count;
-        ret = dict_set_int32 (rsp_dict, "hot_brick_count", hot_brick_count);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (rsp_dict, "type", volinfo->type);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (rsp_dict, "brick-index-max", brick_index);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Error setting brick-index-max to dict");
-                goto out;
-        }
-        ret = dict_set_int32 (rsp_dict, "other-count", other_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Error setting other-count to dict");
-                goto out;
-        }
-        ret = dict_set_int32 (rsp_dict, "count", node_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Error setting node count to dict");
-                goto out;
-        }
-
-        /* Active tasks */
-        /* Tasks are added only for normal volume status request for either a
-         * single volume or all volumes
-         */
-        if (!glusterd_status_has_tasks (cmd))
-                goto out;
-
-        ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
-        if (ret)
-                goto out;
-        ret = 0;
+            }
+        }
+    }
+
+    ret = dict_set_int32n(rsp_dict, "type", SLEN("type"), volinfo->type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=type", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(rsp_dict, "brick-index-max", SLEN("brick-index-max"),
+                          brick_index);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=brick-index-max", NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(rsp_dict, "other-count", SLEN("other-count"),
+                          other_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=other-count", NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(rsp_dict, "count", SLEN("count"), node_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    /* Active tasks */
+    /* Tasks are added only for normal volume status request for either a
+     * single volume or all volumes
+     */
+    if (!glusterd_status_has_tasks(cmd))
+        goto out;
+
+    ret = glusterd_aggregate_task_status(rsp_dict, volinfo);
+    if (ret)
+        goto out;
+    ret = 0;
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_none (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_none(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_sm_locking_failed (uuid_t *txn_id)
+glusterd_op_sm_locking_failed(uuid_t *txn_id)
 {
-        int ret = -1;
+    int ret = -1;
 
-        opinfo.op_ret = -1;
-        opinfo.op_errstr = gf_strdup ("locking failed for one of the peer.");
+    opinfo.op_ret = -1;
+    opinfo.op_errstr = gf_strdup("locking failed for one of the peer.");
 
-        ret = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-        /* Inject a reject event such that unlocking gets triggered right away*/
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, txn_id, NULL);
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+    /* Inject a reject event such that unlocking gets triggered right away*/
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, txn_id, NULL);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_lock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                   ret      = 0;
-        rpc_clnt_procedure_t *proc     = NULL;
-        glusterd_conf_t      *priv     = NULL;
-        xlator_t             *this     = NULL;
-        glusterd_peerinfo_t  *peerinfo = NULL;
-        uint32_t             pending_count = 0;
-        dict_t               *dict     = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > opinfo.txn_generation)
-                        continue;
-
-                if (!peerinfo->connected || !peerinfo->mgmt)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
-
-                /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
-                if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                        proc = &peerinfo->mgmt->proctable
-                                          [GLUSTERD_MGMT_CLUSTER_LOCK];
-                        if (proc->fn) {
-                                ret = proc->fn (NULL, this, peerinfo);
-                                if (ret) {
-                                        rcu_read_unlock ();
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                GD_MSG_LOCK_REQ_SEND_FAIL,
-                                                "Failed to send lock request "
-                                                "for operation 'Volume %s' to "
-                                                "peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        goto out;
-                                }
-                                /* Mark the peer as locked*/
-                                peerinfo->locked = _gf_true;
-                                pending_count++;
-                        }
-                } else {
-                        dict = glusterd_op_get_ctx ();
-                        dict_ref (dict);
-
-                        proc = &peerinfo->mgmt_v3->proctable
-                                          [GLUSTERD_MGMT_V3_LOCK];
-                        if (proc->fn) {
-                                ret = dict_set_static_ptr (dict, "peerinfo",
-                                                           peerinfo);
-                                if (ret) {
-                                        rcu_read_unlock ();
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED,
-                                                "failed to set peerinfo");
-                                        dict_unref (dict);
-                                        goto out;
-                                }
-
-                                ret = proc->fn (NULL, this, dict);
-                                if (ret) {
-                                        rcu_read_unlock ();
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                               GD_MSG_MGMTV3_LOCK_REQ_SEND_FAIL,
-                                                "Failed to send mgmt_v3 lock "
-                                                "request for operation "
-                                                "'Volume %s' to peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        dict_unref (dict);
-                                        goto out;
-                                }
-                                /* Mark the peer as locked*/
-                                peerinfo->locked = _gf_true;
-                                pending_count++;
-                        }
-                }
-        }
-        rcu_read_unlock ();
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    uint32_t pending_count = 0;
+    dict_t *dict = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
 
-        opinfo.pending_count = pending_count;
+        if (!peerinfo->connected || !peerinfo->mgmt)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
 
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (priv->op_version < GD_OP_VERSION_3_6_0) {
+            proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
+            if (proc->fn) {
+                ret = proc->fn(NULL, this, peerinfo);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_LOCK_REQ_SEND_FAIL,
+                           "Failed to send lock request "
+                           "for operation 'Volume %s' to "
+                           "peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    goto out;
+                }
+                /* Mark the peer as locked*/
+                peerinfo->locked = _gf_true;
+                pending_count++;
+            }
+        } else {
+            dict = glusterd_op_get_ctx();
+            dict_ref(dict);
 
+            proc = &peerinfo->mgmt_v3->proctable[GLUSTERD_MGMT_V3_LOCK];
+            if (proc->fn) {
+                ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "failed to set peerinfo");
+                    dict_unref(dict);
+                    goto out;
+                }
 
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+                ret = proc->fn(NULL, this, dict);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_MGMTV3_LOCK_REQ_SEND_FAIL,
+                           "Failed to send mgmt_v3 lock "
+                           "request for operation "
+                           "'Volume %s' to peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    dict_unref(dict);
+                    goto out;
+                }
+                /* Mark the peer as locked*/
+                peerinfo->locked = _gf_true;
+                pending_count++;
+            }
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
 
 out:
-        if (ret)
-                ret = glusterd_op_sm_locking_failed (&event->txn_id);
+    if (ret)
+        ret = glusterd_op_sm_locking_failed(&event->txn_id);
 
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_unlock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                   ret           = 0;
-        rpc_clnt_procedure_t *proc          = NULL;
-        glusterd_conf_t      *priv          = NULL;
-        xlator_t             *this          = NULL;
-        glusterd_peerinfo_t  *peerinfo      = NULL;
-        uint32_t              pending_count = 0;
-        dict_t               *dict          = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > opinfo.txn_generation)
-                        continue;
-
-                if (!peerinfo->connected || !peerinfo->mgmt ||
-                    !peerinfo->locked)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
-                /* Based on the op_version,
-                 * release the cluster or mgmt_v3 lock */
-                if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                        proc = &peerinfo->mgmt->proctable
-                                          [GLUSTERD_MGMT_CLUSTER_UNLOCK];
-                        if (proc->fn) {
-                                ret = proc->fn (NULL, this, peerinfo);
-                                if (ret) {
-                                        opinfo.op_errstr = gf_strdup
-                                               ("Unlocking failed for one of "
-                                                "the peer.");
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_CLUSTER_UNLOCK_FAILED,
-                                                "Unlocking failed for operation"
-                                                " volume %s on peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        continue;
-                                }
-                                pending_count++;
-                                peerinfo->locked = _gf_false;
-                        }
-                } else {
-                        dict = glusterd_op_get_ctx ();
-                        dict_ref (dict);
-
-                        proc = &peerinfo->mgmt_v3->proctable
-                                          [GLUSTERD_MGMT_V3_UNLOCK];
-                        if (proc->fn) {
-                                ret = dict_set_static_ptr (dict, "peerinfo",
-                                                           peerinfo);
-                                if (ret) {
-                                        opinfo.op_errstr = gf_strdup
-                                          ("Unlocking failed for one of the "
-                                           "peer.");
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_CLUSTER_UNLOCK_FAILED,
-                                                "Unlocking failed for operation"
-                                                " volume %s on peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        dict_unref (dict);
-                                        continue;
-                                }
-
-                                ret = proc->fn (NULL, this, dict);
-                                if (ret) {
-                                        opinfo.op_errstr = gf_strdup
-                                          ("Unlocking failed for one of the "
-                                           "peer.");
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_CLUSTER_UNLOCK_FAILED,
-                                                "Unlocking failed for operation"
-                                                " volume %s on peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        dict_unref (dict);
-                                        continue;
-                                }
-                                pending_count++;
-                                peerinfo->locked = _gf_false;
-                        }
-                }
-        }
-        rcu_read_unlock ();
-
-        opinfo.pending_count = pending_count;
-
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    uint32_t pending_count = 0;
+    dict_t *dict = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
+
+        if (!peerinfo->connected || !peerinfo->mgmt || !peerinfo->locked)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
+        /* Based on the op_version,
+         * release the cluster or mgmt_v3 lock */
+        if (priv->op_version < GD_OP_VERSION_3_6_0) {
+            proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
+            if (proc->fn) {
+                ret = proc->fn(NULL, this, peerinfo);
+                if (ret) {
+                    opinfo.op_errstr = gf_strdup(
+                        "Unlocking failed for one of "
+                        "the peer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_CLUSTER_UNLOCK_FAILED,
+                           "Unlocking failed for operation"
+                           " volume %s on peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    continue;
+                }
+                pending_count++;
+                peerinfo->locked = _gf_false;
+            }
+        } else {
+            dict = glusterd_op_get_ctx();
+            dict_ref(dict);
 
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-        return ret;
+            proc = &peerinfo->mgmt_v3->proctable[GLUSTERD_MGMT_V3_UNLOCK];
+            if (proc->fn) {
+                ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+                if (ret) {
+                    opinfo.op_errstr = gf_strdup(
+                        "Unlocking failed for one of the "
+                        "peer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_CLUSTER_UNLOCK_FAILED,
+                           "Unlocking failed for operation"
+                           " volume %s on peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    dict_unref(dict);
+                    continue;
+                }
+
+                ret = proc->fn(NULL, this, dict);
+                if (ret) {
+                    opinfo.op_errstr = gf_strdup(
+                        "Unlocking failed for one of the "
+                        "peer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_CLUSTER_UNLOCK_FAILED,
+                           "Unlocking failed for operation"
+                           " volume %s on peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    dict_unref(dict);
+                    continue;
+                }
+                pending_count++;
+                peerinfo->locked = _gf_false;
+            }
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
+
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_ack_drain (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_ack_drain(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                          NULL);
 
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                                   &event->txn_id, NULL);
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
 
-        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);
-
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_unlock_drain (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_unlock_drain(glusterd_op_sm_event_t *event, void *ctx)
 {
-        return glusterd_op_ac_ack_drain (event, ctx);
+    return glusterd_op_ac_ack_drain(event, ctx);
 }
 
 static int
-glusterd_op_ac_lock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_lock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int32_t                         ret             = 0;
-        int32_t                         err             = 0;
-        char                           *volname         = NULL;
-        char                           *globalname      = NULL;
-        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
-        glusterd_conf_t                *priv            = NULL;
-        xlator_t                       *this            = NULL;
-        uint32_t                        op_errno        = 0;
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-
-        this = THIS;
-        priv = this->private;
-
-        lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
-
-        /* If the req came from a node running on older op_version
-         * the dict won't be present. Based on it acquiring a cluster
-         * or mgmt_v3 lock */
-        if (lock_ctx->dict == NULL) {
-                ret = glusterd_lock (lock_ctx->uuid);
-                glusterd_op_lock_send_resp (lock_ctx->req, ret);
-        } else {
-                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to acquire volname");
-                else {
-                        ret = glusterd_mgmt_v3_lock (volname, lock_ctx->uuid,
-                                                     &op_errno, "vol");
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                                        "Unable to acquire lock for %s",
-                                        volname);
-                        goto out;
-                }
-                ret = dict_get_str (lock_ctx->dict, "globalname", &globalname);
-                if (!ret) {
-                        ret = glusterd_mgmt_v3_lock (globalname, lock_ctx->uuid,
-                                                     &op_errno, "global");
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MGMTV3_LOCK_GET_FAIL,
-                                        "Unable to acquire lock for %s",
-                                        globalname);
-
-                }
-out:
-                glusterd_op_mgmt_v3_lock_send_resp (lock_ctx->req,
-                                                   &event->txn_id, ret);
-
-                dict_unref (lock_ctx->dict);
+    int32_t ret = 0;
+    char *volname = NULL;
+    char *globalname = NULL;
+    glusterd_op_lock_ctx_t *lock_ctx = NULL;
+    xlator_t *this = NULL;
+    uint32_t op_errno = 0;
+    glusterd_conf_t *conf = NULL;
+    uint32_t timeout = 0;
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
+
+    /* If the req came from a node running on older op_version
+     * the dict won't be present. Based on it acquiring a cluster
+     * or mgmt_v3 lock */
+    if (lock_ctx->dict == NULL) {
+        ret = glusterd_lock(lock_ctx->uuid);
+        glusterd_op_lock_send_resp(lock_ctx->req, ret);
+    } else {
+        /* Cli will add timeout key to dict if the default timeout is
+         * other than 2 minutes. Here we use this value to check whether
+         * mgmt_v3_lock_timeout should be set to default value or we
+         * need to change the value according to timeout value
+         * i.e, timeout + 120 seconds. */
+        ret = dict_get_uint32(lock_ctx->dict, "timeout", &timeout);
+        if (!ret)
+            conf->mgmt_v3_lock_timeout = timeout + 120;
+
+        ret = dict_get_strn(lock_ctx->dict, "volname", SLEN("volname"),
+                            &volname);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to acquire volname");
+        else {
+            ret = glusterd_mgmt_v3_lock(volname, lock_ctx->uuid, &op_errno,
+                                        "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                       "Unable to acquire lock for %s", volname);
+            goto out;
+        }
+        ret = dict_get_strn(lock_ctx->dict, "globalname", SLEN("globalname"),
+                            &globalname);
+        if (!ret) {
+            ret = glusterd_mgmt_v3_lock(globalname, lock_ctx->uuid, &op_errno,
+                                        "global");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                       "Unable to acquire lock for %s", globalname);
         }
+    out:
+        glusterd_op_mgmt_v3_lock_send_resp(lock_ctx->req, &event->txn_id, ret);
 
-        gf_msg_debug (THIS->name, 0, "Lock Returned %d", ret);
-        return ret;
+        dict_unref(lock_ctx->dict);
+    }
+
+    gf_msg_debug(THIS->name, 0, "Lock Returned %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_unlock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_unlock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int32_t                         ret             = 0;
-        char                           *volname         = NULL;
-        char                           *globalname      = NULL;
-        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
-        glusterd_conf_t                *priv            = NULL;
-        xlator_t                       *this            = NULL;
-
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-
-        this = THIS;
-        priv = this->private;
-
-        lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
-
-        /* If the req came from a node running on older op_version
-         * the dict won't be present. Based on it releasing the cluster
-         * or mgmt_v3 lock */
-        if (lock_ctx->dict == NULL) {
-                ret = glusterd_unlock (lock_ctx->uuid);
-                glusterd_op_unlock_send_resp (lock_ctx->req, ret);
-        } else {
-                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to acquire volname");
-                else {
-                        ret = glusterd_mgmt_v3_unlock (volname, lock_ctx->uuid,
-                                                       "vol");
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                        "Unable to release lock for %s",
-                                        volname);
-                        goto out;
-                }
-
-                ret = dict_get_str (lock_ctx->dict, "globalname", &globalname);
-                if (!ret) {
-                        ret = glusterd_mgmt_v3_unlock (globalname, lock_ctx->uuid,
-                                                      "global");
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                        "Unable to release lock for %s",
-                                        globalname);
-
-                }
-out:
-                glusterd_op_mgmt_v3_unlock_send_resp (lock_ctx->req,
-                                                     &event->txn_id, ret);
+    int32_t ret = 0;
+    char *volname = NULL;
+    char *globalname = NULL;
+    glusterd_op_lock_ctx_t *lock_ctx = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+
+    this = THIS;
+    priv = this->private;
+
+    lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
+
+    /* If the req came from a node running on older op_version
+     * the dict won't be present. Based on it releasing the cluster
+     * or mgmt_v3 lock */
+    if (lock_ctx->dict == NULL) {
+        ret = glusterd_unlock(lock_ctx->uuid);
+        glusterd_op_unlock_send_resp(lock_ctx->req, ret);
+    } else {
+        ret = dict_get_strn(lock_ctx->dict, "volname", SLEN("volname"),
+                            &volname);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to acquire volname");
+        else {
+            ret = glusterd_mgmt_v3_unlock(volname, lock_ctx->uuid, "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", volname);
+            goto out;
+        }
 
-                dict_unref (lock_ctx->dict);
+        ret = dict_get_strn(lock_ctx->dict, "globalname", SLEN("globalname"),
+                            &globalname);
+        if (!ret) {
+            ret = glusterd_mgmt_v3_unlock(globalname, lock_ctx->uuid, "global");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", globalname);
         }
+    out:
+        glusterd_op_mgmt_v3_unlock_send_resp(lock_ctx->req, &event->txn_id,
+                                             ret);
 
-        gf_msg_debug (this->name, 0, "Unlock Returned %d", ret);
+        dict_unref(lock_ctx->dict);
+    }
 
-        if (priv->pending_quorum_action)
-                glusterd_do_quorum_action ();
-        return ret;
+    gf_msg_debug(this->name, 0, "Unlock Returned %d", ret);
+
+    if (priv->pending_quorum_action)
+        glusterd_do_quorum_action();
+    return ret;
 }
 
 static int
-glusterd_op_ac_local_unlock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_local_unlock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int              ret          = 0;
-        uuid_t          *originator   = NULL;
+    int ret = 0;
+    uuid_t *originator = NULL;
 
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
 
-        originator = (uuid_t *) ctx;
+    originator = (uuid_t *)ctx;
 
-        ret = glusterd_unlock (*originator);
+    ret = glusterd_unlock(*originator);
 
-        gf_msg_debug (THIS->name, 0, "Unlock Returned %d", ret);
+    gf_msg_debug(THIS->name, 0, "Unlock Returned %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_lock_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_lock_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
-
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    if (opinfo.pending_count > 0)
+        goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
-                                           &event->txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACC, &event->txn_id,
+                                      NULL);
 
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr)
+glusterd_dict_set_volid(dict_t *dict, char *volname, char **op_errstr)
 {
-        int                     ret = -1;
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    *volid = NULL;
-        char                    msg[1024] = {0,};
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dict || !volname)
-                goto out;
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-        volid = gf_strdup (uuid_utoa (volinfo->volume_id));
-        if (!volid) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_dynstr (dict, "vol-id", volid);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to set volume id of volume"
-                          " %s", volname);
-                goto out;
-        }
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volid = NULL;
+    char msg[1024] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dict || !volname) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+    volid = gf_strdup(uuid_utoa(volinfo->volume_id));
+    if (!volid) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_dynstrn(dict, "vol-id", SLEN("vol-id"), volid);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to set volume id of volume"
+                 " %s",
+                 volname);
+        GF_FREE(volid);
+        goto out;
+    }
 out:
-        if (msg[0] != '\0') {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_ID_SET_FAIL, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
-        return ret;
+    if (msg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_ID_SET_FAIL, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
+    return ret;
 }
 
 int
-gd_set_commit_hash (dict_t *dict)
+gd_set_commit_hash(dict_t *dict)
 {
-        struct timeval          tv;
-        uint32_t                hash;
-
-        /*
-         * We need a commit hash that won't conflict with others we might have
-         * set, or zero which is the implicit value if we never have.  Using
-         * seconds<<3 like this ensures that we'll only get a collision if two
-         * consecutive rebalances are separated by exactly 2^29 seconds - about
-         * 17 years - and even then there's only a 1/8 chance of a collision in
-         * the low order bits.  It's far more likely that this code will have
-         * changed completely by then.  If not, call me in 2031.
-         *
-         * P.S. Time zone changes?  Yeah, right.
-         */
-        gettimeofday (&tv, NULL);
-        hash = tv.tv_sec << 3;
-
-        /*
-         * Make sure at least one of those low-order bits is set.  The extra
-         * shifting is because not all machines have sub-millisecond time
-         * resolution.
-         */
-        hash |= 1 << ((tv.tv_usec >> 10) % 3);
-
-        return dict_set_uint32 (dict, "commit-hash", hash);
+    struct timeval tv;
+    uint32_t hash;
+
+    /*
+     * We need a commit hash that won't conflict with others we might have
+     * set, or zero which is the implicit value if we never have.  Using
+     * seconds<<3 like this ensures that we'll only get a collision if two
+     * consecutive rebalances are separated by exactly 2^29 seconds - about
+     * 17 years - and even then there's only a 1/8 chance of a collision in
+     * the low order bits.  It's far more likely that this code will have
+     * changed completely by then.  If not, call me in 2031.
+     *
+     * P.S. Time zone changes?  Yeah, right.
+     */
+    gettimeofday(&tv, NULL);
+    hash = tv.tv_sec << 3;
+
+    /*
+     * Make sure at least one of those low-order bits is set.  The extra
+     * shifting is because not all machines have sub-millisecond time
+     * resolution.
+     */
+    hash |= 1 << ((tv.tv_usec >> 10) % 3);
+
+    return dict_set_uint32(dict, "commit-hash", hash);
 }
 
 int
-glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
+glusterd_op_build_payload(dict_t **req, char **op_errstr, dict_t *op_ctx)
 {
-        int                     ret = -1;
-        void                    *ctx = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *req_dict = NULL;
-        glusterd_op_t           op = GD_OP_NONE;
-        char                    *volname = NULL;
-        uint32_t                status_cmd = GF_CLI_STATUS_NONE;
-        char                    *errstr = NULL;
-        xlator_t                *this = NULL;
-        gf_boolean_t            do_common = _gf_false;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        req_dict = dict_new ();
-        if (!req_dict)
-                goto out;
-
-        if (!op_ctx) {
-                op  = glusterd_op_get_op ();
-                ctx = (void*)glusterd_op_get_ctx ();
-                if (!ctx) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_NO_OPTIONS_GIVEN, "Null Context for "
-                                "op %d", op);
-                        ret = -1;
-                        goto out;
-                }
-
-        } else {
+    int ret = -1;
+    void *ctx = NULL;
+    dict_t *dict = NULL;
+    dict_t *req_dict = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    char *volname = NULL;
+    uint32_t status_cmd = GF_CLI_STATUS_NONE;
+    xlator_t *this = NULL;
+    gf_boolean_t do_common = _gf_false;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    req_dict = dict_new();
+    if (!req_dict)
+        goto out;
+
+    if (!op_ctx) {
+        op = glusterd_op_get_op();
+        ctx = (void *)glusterd_op_get_ctx();
+        if (!ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_OPTIONS_GIVEN,
+                   "Null Context for "
+                   "op %d",
+                   op);
+            ret = -1;
+            goto out;
+        }
+
+    } else {
 #define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
-                ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, (int32_t*)&op);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to get volume"
-                                " operation");
-                        goto out;
-                }
-                ctx = op_ctx;
-#undef GD_SYNC_OPCODE_KEY
+        ret = dict_get_int32(op_ctx, GD_SYNC_OPCODE_KEY, (int32_t *)&op);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get volume"
+                   " operation");
+            goto out;
         }
+        ctx = op_ctx;
+#undef GD_SYNC_OPCODE_KEY
+    }
+
+    dict = ctx;
+    switch (op) {
+        case GD_OP_CREATE_VOLUME: {
+            ++glusterfs_port;
+            ret = dict_set_int32n(dict, "port", SLEN("port"), glusterfs_port);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set port in "
+                       "dictionary");
+                goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
+
+        case GD_OP_GSYNC_CREATE:
+        case GD_OP_GSYNC_SET: {
+            ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, NULL,
+                                             NULL);
+            if (ret == 0) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
+
+        case GD_OP_SET_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
+                goto out;
+            }
+            if (strcmp(volname, "help") && strcmp(volname, "help-xml") &&
+                strcasecmp(volname, "all")) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_unref(req_dict);
+            req_dict = dict_ref(dict);
+        } break;
 
-        dict = ctx;
-        switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                        {
-                                ++glusterfs_port;
-                                ret = dict_set_int32 (dict, "port",
-                                                      glusterfs_port);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED,
-                                                "Failed to set port in "
-                                                "dictionary");
-                                        goto out;
-                                }
-                                dict_copy (dict, req_dict);
-                        }
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                case GD_OP_GSYNC_SET:
-                        {
-                                ret = glusterd_op_gsync_args_get (dict,
-                                                                  &errstr,
-                                                                  &volname,
-                                                                  NULL, NULL);
-                                if (ret == 0) {
-                                        ret = glusterd_dict_set_volid
-                                                (dict, volname, op_errstr);
-                                        if (ret)
-                                                goto out;
-                                }
-                                dict_copy (dict, req_dict);
-                        }
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        {
-                                ret = dict_get_str (dict, "volname", &volname);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                                GD_MSG_DICT_GET_FAILED,
-                                                "volname is not present in "
-                                                "operation ctx");
-                                        goto out;
-                                }
-                                if (strcmp (volname, "help") &&
-                                    strcmp (volname, "help-xml") &&
-                                    strcasecmp (volname, "all")) {
-                                        ret = glusterd_dict_set_volid
-                                                 (dict, volname, op_errstr);
-                                        if (ret)
-                                                goto out;
-                                }
-                                dict_destroy (req_dict);
-                                req_dict = dict_ref (dict);
-                        }
-                        break;
-
-                case GD_OP_REMOVE_BRICK:
-                        {
-                                dict_t *dict = ctx;
-                                ret = dict_get_str (dict, "volname", &volname);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                                GD_MSG_DICT_GET_FAILED,
-                                                "volname is not present in "
-                                                "operation ctx");
-                                        goto out;
-                                }
-
-                                ret = glusterd_dict_set_volid (dict, volname,
-                                                               op_errstr);
-                                if (ret)
-                                        goto out;
-
-                                if (gd_set_commit_hash(dict) != 0) {
-                                        goto out;
-                                }
-
-                                dict_destroy (req_dict);
-                                req_dict = dict_ref (dict);
-                        }
-                        break;
-
-                case GD_OP_STATUS_VOLUME:
-                        {
-                                ret = dict_get_uint32 (dict, "cmd",
-                                                       &status_cmd);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_GET_FAILED,
-                                                "Status command not present "
-                                                "in op ctx");
-                                        goto out;
-                                }
-                                if (GF_CLI_STATUS_ALL & status_cmd) {
-                                        dict_copy (dict, req_dict);
-                                        break;
-                                }
-                                do_common = _gf_true;
-                        }
-                        break;
-
-                case GD_OP_DELETE_VOLUME:
-                case GD_OP_START_VOLUME:
-                case GD_OP_STOP_VOLUME:
-                case GD_OP_ADD_BRICK:
-                case GD_OP_REPLACE_BRICK:
-                case GD_OP_RESET_VOLUME:
-                case GD_OP_LOG_ROTATE:
-                case GD_OP_QUOTA:
-                case GD_OP_PROFILE_VOLUME:
-                case GD_OP_HEAL_VOLUME:
-                case GD_OP_STATEDUMP_VOLUME:
-                case GD_OP_CLEARLOCKS_VOLUME:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                case GD_OP_BARRIER:
-                case GD_OP_BITROT:
-                case GD_OP_SCRUB_STATUS:
-                        {
-                                do_common = _gf_true;
-                        }
-                        break;
-
-                case GD_OP_REBALANCE:
-                        {
-                                if (gd_set_commit_hash(dict) != 0) {
-                                        goto out;
-                                }
-                                do_common = _gf_true;
-                        }
-                        break;
-
-                case GD_OP_SYNC_VOLUME:
-                case GD_OP_COPY_FILE:
-                case GD_OP_SYS_EXEC:
-                        {
-                                dict_copy (dict, req_dict);
-                        }
-                        break;
-
-                case GD_OP_GANESHA:
-                        {
-                                dict_copy (dict, req_dict);
-                        }
-                        break;
-
-                default:
-                        break;
-        }
+        case GD_OP_REMOVE_BRICK: {
+            dict_t *dict = ctx;
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
+                goto out;
+            }
 
-        /*
-         * This has been moved out of the switch so that multiple ops with
-         * other special needs can all "fall through" to it.
-         */
-        if (do_common) {
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_CRITICAL, -ret,
-                                GD_MSG_DICT_GET_FAILED,
-                                "volname is not present in "
-                                "operation ctx");
-                        goto out;
-                }
+            ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+            if (ret)
+                goto out;
 
-                if (strcasecmp (volname, "all")) {
-                        ret = glusterd_dict_set_volid (dict,
-                                                       volname,
-                                                     op_errstr);
-                        if (ret)
-                                goto out;
-                }
-                dict_copy (dict, req_dict);
-        }
+            if (gd_set_commit_hash(dict) != 0) {
+                goto out;
+            }
 
-       *req = req_dict;
-       ret = 0;
+            dict_unref(req_dict);
+            req_dict = dict_ref(dict);
+        } break;
 
-out:
-        return ret;
-}
+        case GD_OP_STATUS_VOLUME: {
+            ret = dict_get_uint32(dict, "cmd", &status_cmd);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Status command not present "
+                       "in op ctx");
+                goto out;
+            }
+            if (GF_CLI_STATUS_ALL & status_cmd) {
+                dict_copy(dict, req_dict);
+                break;
+            }
+            do_common = _gf_true;
+        } break;
 
-static int
-glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
-{
-        int                     ret = 0;
-	int                     ret1 = 0;
-        rpc_clnt_procedure_t    *proc = NULL;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
-        char                    *op_errstr  = NULL;
-        glusterd_op_t           op = GD_OP_NONE;
-        uint32_t                pending_count = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        op = glusterd_op_get_op ();
+        case GD_OP_DELETE_VOLUME:
+        case GD_OP_START_VOLUME:
+        case GD_OP_STOP_VOLUME:
+        case GD_OP_ADD_BRICK:
+        case GD_OP_REPLACE_BRICK:
+        case GD_OP_RESET_VOLUME:
+        case GD_OP_LOG_ROTATE:
+        case GD_OP_QUOTA:
+        case GD_OP_PROFILE_VOLUME:
+        case GD_OP_HEAL_VOLUME:
+        case GD_OP_STATEDUMP_VOLUME:
+        case GD_OP_CLEARLOCKS_VOLUME:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_BARRIER:
+        case GD_OP_BITROT:
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+        case GD_OP_RESET_BRICK: {
+            do_common = _gf_true;
+        } break;
 
-        rsp_dict = dict_new();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to create rsp_dict");
-                ret = -1;
+        case GD_OP_REBALANCE: {
+            if (gd_set_commit_hash(dict) != 0) {
                 goto out;
-        }
+            }
+            do_common = _gf_true;
+        } break;
 
-        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
-                        LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                opinfo.op_errstr = op_errstr;
-                goto out;
-        }
+        case GD_OP_SYNC_VOLUME:
+        case GD_OP_COPY_FILE:
+        case GD_OP_SYS_EXEC:
+        case GD_OP_GANESHA: {
+            dict_copy(dict, req_dict);
+        } break;
 
-        ret = glusterd_validate_quorum (this, op, dict, &op_errstr);
+        default:
+            break;
+    }
+
+    /*
+     * This has been moved out of the switch so that multiple ops with
+     * other special needs can all "fall through" to it.
+     */
+    if (do_common) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET,
-                        "Server quorum not met. Rejecting operation.");
-                opinfo.op_errstr = op_errstr;
-                goto out;
+            gf_msg(this->name, GF_LOG_CRITICAL, -ret, GD_MSG_DICT_GET_FAILED,
+                   "volname is not present in "
+                   "operation ctx");
+            goto out;
         }
 
-        ret = glusterd_op_stage_validate (op, dict, &op_errstr, rsp_dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VALIDATE_FAILED, LOGSTR_STAGE_FAIL,
-                        gd_op_list[op], "localhost",
-                        (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_STAGE_FAIL,
-                                     "localhost");
-                opinfo.op_errstr = op_errstr;
+        if (strcasecmp(volname, "all")) {
+            ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+            if (ret)
                 goto out;
         }
+        dict_copy(dict, req_dict);
+    }
 
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > opinfo.txn_generation)
-                        continue;
+    *req = req_dict;
+    ret = 0;
 
-                if (!peerinfo->connected || !peerinfo->mgmt)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
+out:
+    return ret;
+}
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_STAGE_OP];
-                GF_ASSERT (proc);
-                if (proc->fn) {
-                        ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
-                        if (ret) {
-                                rcu_read_unlock ();
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "failed to "
-                                        "set peerinfo");
-                                goto out;
-                        }
+static int
+glusterd_op_ac_send_stage_op(glusterd_op_sm_event_t *event, void *ctx)
+{
+    int ret = 0;
+    int ret1 = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    char *op_errstr = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    uint32_t pending_count = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    op = glusterd_op_get_op();
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create rsp_dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_op_build_payload(&dict, &op_errstr, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    ret = glusterd_validate_quorum(this, op, dict, &op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    ret = glusterd_op_stage_validate(op, dict, &op_errstr, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               LOGSTR_STAGE_FAIL, gd_op_list[op], "localhost",
+               (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_STAGE_FAIL, "localhost");
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
 
-                        ret = proc->fn (NULL, this, dict);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_STAGE_REQ_SEND_FAIL, "Failed to "
-                                        "send stage request for operation "
-                                        "'Volume %s' to peer %s",
-                                        gd_op_list[op], peerinfo->hostname);
-                                continue;
-                        }
-                        pending_count++;
-                }
-        }
-        rcu_read_unlock ();
+        if (!peerinfo->connected || !peerinfo->mgmt)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
 
-        opinfo.pending_count = pending_count;
+        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_STAGE_OP];
+        GF_ASSERT(proc);
+        if (proc->fn) {
+            ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+            if (ret) {
+                RCU_READ_UNLOCK;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to "
+                       "set peerinfo");
+                goto out;
+            }
+
+            ret = proc->fn(NULL, this, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_STAGE_REQ_SEND_FAIL,
+                       "Failed to "
+                       "send stage request for operation "
+                       "'Volume %s' to peer %s",
+                       gd_op_list[op], peerinfo->hostname);
+                continue;
+            }
+            pending_count++;
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
 out:
-        if (ret)
-                opinfo.op_ret = ret;
+    if (ret)
+        opinfo.op_ret = ret;
 
-        ret1 = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret1)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    ret1 = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret1)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (dict)
+        dict_unref(dict);
+    if (ret) {
+        glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, &event->txn_id, NULL);
+        opinfo.op_ret = ret;
+    }
 
-        if (dict)
-                dict_unref (dict);
-        if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                             &event->txn_id, NULL);
-                opinfo.op_ret = ret;
-        }
+    gf_msg_debug(this->name, 0,
+                 "Sent stage op request for "
+                 "'Volume %s' to %d peers",
+                 gd_op_list[op], opinfo.pending_count);
 
-        gf_msg_debug (this->name, 0, "Sent stage op request for "
-                "'Volume %s' to %d peers", gd_op_list[op],
-                opinfo.pending_count);
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
 
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
-
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
+    return ret;
 }
 
 /* This function takes a dict and converts the uuid values of key specified
  * into hostnames
  */
 static int
-glusterd_op_volume_dict_uuid_to_hostname (dict_t *dict, const char *key_fmt,
-                                          int idx_min, int idx_max)
+glusterd_op_volume_dict_uuid_to_hostname(dict_t *dict, const char *key_fmt,
+                                         int idx_min, int idx_max)
 {
-        int             ret = -1;
-        int             i = 0;
-        char            key[1024];
-        char            *uuid_str = NULL;
-        uuid_t          uuid = {0,};
-        char            *hostname = NULL;
-        xlator_t        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (key_fmt);
-
-        for (i = idx_min; i < idx_max; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), key_fmt, i);
-                ret = dict_get_str (dict, key, &uuid_str);
-                if (ret)
-                        continue;
+    int ret = -1;
+    int i = 0;
+    char key[128];
+    int keylen;
+    char *uuid_str = NULL;
+    uuid_t uuid = {
+        0,
+    };
+    char *hostname = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(key_fmt);
+
+    for (i = idx_min; i < idx_max; i++) {
+        keylen = snprintf(key, sizeof(key), key_fmt, i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_str);
+        if (ret) {
+            ret = 0;
+            continue;
+        }
 
-                gf_msg_debug (this->name, 0, "Got uuid %s",
-                        uuid_str);
+        gf_msg_debug(this->name, 0, "Got uuid %s", uuid_str);
 
-                ret = gf_uuid_parse (uuid_str, uuid);
-                /* if parsing fails don't error out
-                 * let the original value be retained
-                 */
-                if (ret)
-                        continue;
+        ret = gf_uuid_parse(uuid_str, uuid);
+        /* if parsing fails don't error out
+         * let the original value be retained
+         */
+        if (ret) {
+            ret = 0;
+            continue;
+        }
 
-                hostname = glusterd_uuid_to_hostname (uuid);
-                if (hostname) {
-                        gf_msg_debug (this->name, 0, "%s -> %s",
-                                uuid_str, hostname);
-                        ret = dict_set_dynstr (dict, key, hostname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Error setting hostname %s to dict",
-                                        hostname);
-                                GF_FREE (hostname);
-                                goto out;
-                        }
-                }
+        hostname = glusterd_uuid_to_hostname(uuid);
+        if (hostname) {
+            gf_msg_debug(this->name, 0, "%s -> %s", uuid_str, hostname);
+            ret = dict_set_dynstrn(dict, key, keylen, hostname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Error setting hostname %s to dict", hostname);
+                GF_FREE(hostname);
+                goto out;
+            }
         }
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
+reassign_defrag_status(dict_t *dict, char *key, int keylen,
+                       gf_defrag_status_t *status)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!*status)
-                return ret;
+    if (!*status)
+        return ret;
 
-        switch (*status) {
+    switch (*status) {
         case GF_DEFRAG_STATUS_STARTED:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
+            break;
 
         case GF_DEFRAG_STATUS_STOPPED:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
+            break;
 
         case GF_DEFRAG_STATUS_COMPLETE:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+            break;
 
         case GF_DEFRAG_STATUS_FAILED:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
+            break;
         default:
-                break;
-         }
+            break;
+    }
 
-        ret = dict_set_int32(dict, key, *status);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "failed to reset defrag %s in dict", key);
+    ret = dict_set_int32n(dict, key, keylen, *status);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to reset defrag %s in dict", key);
 
-        return ret;
+    return ret;
 }
 
 /* Check and reassign the defrag_status enum got from the rebalance process
@@ -4419,57 +4711,56 @@ reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
  * full-rebalance or just a fix-layout was carried out.
  */
 static int
-glusterd_op_check_peer_defrag_status (dict_t *dict, int count)
+glusterd_op_check_peer_defrag_status(dict_t *dict, int count)
 {
-        glusterd_volinfo_t *volinfo  = NULL;
-        gf_defrag_status_t status    = GF_DEFRAG_STATUS_NOT_STARTED;
-        char               key[256]  = {0,};
-        char               *volname  = NULL;
-        int                ret       = -1;
-        int                i         = 1;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_defrag_status_t status = GF_DEFRAG_STATUS_NOT_STARTED;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *volname = NULL;
+    int ret = -1;
+    int i = 1;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        /* Fix layout was not issued; we don't need to reassign
+           the status */
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    do {
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status);
         if (ret) {
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOL_NOT_FOUND, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get defrag %s", key);
+            goto out;
         }
+        ret = reassign_defrag_status(dict, key, keylen, &status);
+        if (ret)
+            goto out;
+        i++;
+    } while (i <= count);
 
-        if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
-                /* Fix layout was not issued; we don't need to reassign
-                   the status */
-                ret = 0;
-                goto out;
-        }
-
-        do {
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-                ret = dict_get_int32 (dict, key, (int32_t *)&status);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "failed to get defrag %s", key);
-                        goto out;
-                }
-                ret = reassign_defrag_status (dict, key, &status);
-                if (ret)
-                        goto out;
-                i++;
-        } while (i <= count);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
-
+    return ret;
 }
 
 /* This function is used to verify if op_ctx indeed
@@ -4493,52 +4784,49 @@ out:
 */
 
 static gf_boolean_t
-glusterd_is_volume_status_modify_op_ctx (uint32_t cmd)
+glusterd_is_volume_status_modify_op_ctx(uint32_t cmd)
 {
-        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
-                if (cmd & GF_CLI_STATUS_BRICK)
-                        return _gf_false;
-                if (cmd & GF_CLI_STATUS_ALL)
-                        return _gf_false;
-                return _gf_true;
-        }
-        return _gf_false;
+    if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
+        if (cmd & GF_CLI_STATUS_BRICK)
+            return _gf_false;
+        if (cmd & GF_CLI_STATUS_ALL)
+            return _gf_false;
+        return _gf_true;
+    }
+    return _gf_false;
 }
 
 int
-glusterd_op_modify_port_key (dict_t *op_ctx, int brick_index_max)
+glusterd_op_modify_port_key(dict_t *op_ctx, int brick_index_max)
 {
-        char      *port         = NULL;
-        int       i             = 0;
-        int       ret = -1;
-        char      key[1024]     = {0};
-        char      old_key[1024] = {0};
-
-        for (i = 0; i <= brick_index_max; i++) {
-
-               memset (key, 0, sizeof (key));
-               snprintf (key, sizeof (key), "brick%d.rdma_port", i);
-               ret = dict_get_str (op_ctx, key, &port);
-
-               if (ret) {
-
-                       memset (old_key, 0, sizeof (old_key));
-                       snprintf (old_key, sizeof (old_key),
-                                           "brick%d.port", i);
-                       ret = dict_get_str (op_ctx, old_key, &port);
-                       if (ret)
-                               goto out;
-
-                       ret = dict_set_str (op_ctx, key, port);
-                       if (ret)
-                               goto out;
-                       ret = dict_set_str (op_ctx, old_key, "\0");
-                       if (ret)
-                               goto out;
-               }
+    char *port = NULL;
+    int i = 0;
+    int ret = -1;
+    char key[64] = {0};
+    int keylen;
+    char old_key[64] = {0};
+    int old_keylen;
+
+    for (i = 0; i <= brick_index_max; i++) {
+        keylen = snprintf(key, sizeof(key), "brick%d.rdma_port", i);
+        ret = dict_get_strn(op_ctx, key, keylen, &port);
+
+        if (ret) {
+            old_keylen = snprintf(old_key, sizeof(old_key), "brick%d.port", i);
+            ret = dict_get_strn(op_ctx, old_key, old_keylen, &port);
+            if (ret)
+                goto out;
+
+            ret = dict_set_strn(op_ctx, key, keylen, port);
+            if (ret)
+                goto out;
+            ret = dict_set_nstrn(op_ctx, old_key, old_keylen, "\0", SLEN("\0"));
+            if (ret)
+                goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is used to modify the op_ctx dict before sending it back
@@ -4546,2236 +4834,2223 @@ out:
  * hostnames etc.
  */
 void
-glusterd_op_modify_op_ctx (glusterd_op_t op, void *ctx)
+glusterd_op_modify_op_ctx(glusterd_op_t op, void *ctx)
 {
-        int             ret = -1;
-        dict_t          *op_ctx = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             count = 0;
-        uint32_t        cmd = GF_CLI_STATUS_NONE;
-        xlator_t        *this = NULL;
-        glusterd_conf_t    *conf    = NULL;
-        char               *volname = NULL;
-        glusterd_volinfo_t *volinfo = NULL;
-        char            *port = 0;
-        int             i = 0;
-        char            key[1024] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-
-        if (ctx)
-                op_ctx = ctx;
-        else
-                op_ctx = glusterd_op_get_ctx();
+    int ret = -1;
+    dict_t *op_ctx = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int count = 0;
+    uint32_t cmd = GF_CLI_STATUS_NONE;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *port = 0;
+    int i = 0;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    if (ctx)
+        op_ctx = ctx;
+    else
+        op_ctx = glusterd_op_get_ctx();
+
+    if (!op_ctx) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_OPCTX_NULL,
+               "Operation context is not present.");
+        goto out;
+    }
+
+    switch (op) {
+        case GD_OP_STATUS_VOLUME:
+            ret = dict_get_uint32(op_ctx, "cmd", &cmd);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get status cmd");
+                goto out;
+            }
 
-        if (!op_ctx) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_OPCTX_NULL,
-                        "Operation context is not present.");
+            if (!glusterd_is_volume_status_modify_op_ctx(cmd)) {
+                gf_msg_debug(this->name, 0,
+                             "op_ctx modification not required for status "
+                             "operation being performed");
                 goto out;
-        }
+            }
 
-        switch (op) {
-        case GD_OP_STATUS_VOLUME:
-                ret = dict_get_uint32 (op_ctx, "cmd", &cmd);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Failed to get status cmd");
-                        goto out;
-                }
+            ret = dict_get_int32n(op_ctx, "brick-index-max",
+                                  SLEN("brick-index-max"), &brick_index_max);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get brick-index-max");
+                goto out;
+            }
 
-                if (!glusterd_is_volume_status_modify_op_ctx (cmd)) {
-                        gf_msg_debug (this->name, 0,
-                                "op_ctx modification not required for status "
-                                "operation being performed");
-                        goto out;
-                }
+            ret = dict_get_int32n(op_ctx, "other-count", SLEN("other-count"),
+                                  &other_count);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get other-count");
+                goto out;
+            }
 
-                ret = dict_get_int32 (op_ctx, "brick-index-max",
-                                      &brick_index_max);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Failed to get brick-index-max");
-                        goto out;
-                }
+            count = brick_index_max + other_count + 1;
+
+            /*
+             * a glusterd lesser than version 3.7 will be sending the
+             * rdma port in older key. Changing that value from here
+             * to support backward compatibility
+             */
+            ret = dict_get_strn(op_ctx, "volname", SLEN("volname"), &volname);
+            if (ret)
+                goto out;
 
-                ret = dict_get_int32 (op_ctx, "other-count", &other_count);
+            for (i = 0; i <= brick_index_max; i++) {
+                keylen = snprintf(key, sizeof(key), "brick%d.rdma_port", i);
+                ret = dict_get_strn(op_ctx, key, keylen, &port);
                 if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Failed to get other-count");
+                    ret = dict_set_nstrn(op_ctx, key, keylen, "\0", SLEN("\0"));
+                    if (ret)
                         goto out;
                 }
-
-                count = brick_index_max + other_count + 1;
-
-                /*
-                 * a glusterd lesser than version 3.7 will be sending the
-                 * rdma port in older key. Changing that value from here
-                 * to support backward compatibility
-                 */
-                 ret = dict_get_str (op_ctx, "volname", &volname);
-                 if (ret)
-                        goto out;
-
-                 for (i = 0; i <= brick_index_max; i++) {
-                         memset (key, 0, sizeof (key));
-                         snprintf (key, sizeof (key), "brick%d.rdma_port", i);
-                         ret = dict_get_str (op_ctx, key, &port);
-                         if (ret) {
-                                 ret = dict_set_str (op_ctx, key, "\0");
-                                 if (ret)
-                                         goto out;
-                         }
-                 }
-                 glusterd_volinfo_find (volname, &volinfo);
-                 if (conf->op_version < GD_OP_VERSION_3_7_0 &&
-                     volinfo->transport_type == GF_TRANSPORT_RDMA) {
-                         ret = glusterd_op_modify_port_key (op_ctx,
-                                                            brick_index_max);
-                         if (ret)
-                                 goto out;
-                 }
-                /* add 'brick%d.peerid' into op_ctx with value of 'brick%d.path'.
-                   nfs/sshd like services have this additional uuid */
-                {
-                        char  key[1024];
-                        char *uuid_str = NULL;
-                        char *uuid = NULL;
-                        int   i;
-
-                        for (i = brick_index_max + 1; i < count; i++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "brick%d.path", i);
-                                ret = dict_get_str (op_ctx, key, &uuid_str);
-                                if (!ret) {
-                                        memset (key, 0, sizeof (key));
-                                        snprintf (key, sizeof (key),
-                                                  "brick%d.peerid", i);
-                                        uuid = gf_strdup (uuid_str);
-                                        if (!uuid) {
-                                                gf_msg_debug (this->name, 0,
-                                                        "unable to create dup of"
-                                                        " uuid_str");
-                                                continue;
-                                        }
-                                        ret = dict_set_dynstr (op_ctx, key,
-                                                               uuid);
-                                        if (ret != 0) {
-                                                GF_FREE (uuid);
-                                        }
-                                }
+            }
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret)
+                goto out;
+            if (conf->op_version < GD_OP_VERSION_3_7_0 &&
+                volinfo->transport_type == GF_TRANSPORT_RDMA) {
+                ret = glusterd_op_modify_port_key(op_ctx, brick_index_max);
+                if (ret)
+                    goto out;
+            }
+            /* add 'brick%d.peerid' into op_ctx with value of 'brick%d.path'.
+               nfs/sshd like services have this additional uuid */
+            {
+                char *uuid_str = NULL;
+                char *uuid = NULL;
+                int i;
+
+                for (i = brick_index_max + 1; i < count; i++) {
+                    keylen = snprintf(key, sizeof(key), "brick%d.path", i);
+                    ret = dict_get_strn(op_ctx, key, keylen, &uuid_str);
+                    if (!ret) {
+                        keylen = snprintf(key, sizeof(key), "brick%d.peerid",
+                                          i);
+                        uuid = gf_strdup(uuid_str);
+                        if (!uuid) {
+                            gf_msg_debug(this->name, 0,
+                                         "unable to create dup of"
+                                         " uuid_str");
+                            continue;
+                        }
+                        ret = dict_set_dynstrn(op_ctx, key, keylen, uuid);
+                        if (ret != 0) {
+                            GF_FREE(uuid);
                         }
+                    }
                 }
+            }
 
-                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
-                                                                "brick%d.path",
-                                                                0, count);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_CONVERSION_FAILED,
-                                "Failed uuid to hostname conversion");
+            ret = glusterd_op_volume_dict_uuid_to_hostname(
+                op_ctx, "brick%d.path", 0, count);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_CONVERSION_FAILED,
+                       "Failed uuid to hostname conversion");
 
-                break;
+            break;
 
         case GD_OP_PROFILE_VOLUME:
-                ret = dict_get_str_boolean (op_ctx, "nfs", _gf_false);
-                if (!ret)
-                        goto out;
+            ret = dict_get_str_boolean(op_ctx, "nfs", _gf_false);
+            if (!ret)
+                goto out;
 
-                ret = dict_get_int32 (op_ctx, "count", &count);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Failed to get brick count");
-                        goto out;
-                }
+            ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &count);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get brick count");
+                goto out;
+            }
 
-                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
-                                                                "%d-brick",
-                                                                1, (count + 1));
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_CONVERSION_FAILED,
-                                "Failed uuid to hostname conversion");
+            ret = glusterd_op_volume_dict_uuid_to_hostname(op_ctx, "%d-brick",
+                                                           1, (count + 1));
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_CONVERSION_FAILED,
+                       "Failed uuid to hostname conversion");
 
-                break;
+            break;
 
         /* For both rebalance and remove-brick status, the glusterd op is the
          * same
          */
         case GD_OP_DEFRAG_BRICK_VOLUME:
         case GD_OP_SCRUB_STATUS:
-                ret = dict_get_int32 (op_ctx, "count", &count);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Failed to get count");
-                        goto out;
-                }
-
-                /* add 'node-name-%d' into op_ctx with value uuid_str.
-                   this will be used to convert to hostname later */
-                {
-                        char  key[1024];
-                        char *uuid_str = NULL;
-                        char *uuid = NULL;
-                        int   i;
-
-                        for (i = 1; i <= count; i++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "node-uuid-%d", i);
-                                ret = dict_get_str (op_ctx, key, &uuid_str);
-                                if (!ret) {
-                                        memset (key, 0, sizeof (key));
-                                        snprintf (key, sizeof (key),
-                                                  "node-name-%d", i);
-                                        uuid = gf_strdup (uuid_str);
-                                        if (!uuid) {
-                                                gf_msg_debug (this->name, 0,
-                                                        "unable to create dup of"
-                                                        " uuid_str");
-                                                continue;
-                                        }
-                                        ret = dict_set_dynstr (op_ctx, key,
-                                                               uuid);
-                                        if (ret != 0) {
-                                                GF_FREE (uuid);
-                                        }
-                                }
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &count);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get count");
+                goto out;
+            }
+
+            /* add 'node-name-%d' into op_ctx with value uuid_str.
+               this will be used to convert to hostname later */
+            {
+                char *uuid_str = NULL;
+                char *uuid = NULL;
+                int i;
+
+                for (i = 1; i <= count; i++) {
+                    keylen = snprintf(key, sizeof(key), "node-uuid-%d", i);
+                    ret = dict_get_strn(op_ctx, key, keylen, &uuid_str);
+                    if (!ret) {
+                        keylen = snprintf(key, sizeof(key), "node-name-%d", i);
+                        uuid = gf_strdup(uuid_str);
+                        if (!uuid) {
+                            gf_msg_debug(this->name, 0,
+                                         "unable to create dup of"
+                                         " uuid_str");
+                            continue;
                         }
+                        ret = dict_set_dynstrn(op_ctx, key, keylen, uuid);
+                        if (ret != 0) {
+                            GF_FREE(uuid);
+                        }
+                    }
                 }
+            }
 
-                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
-                                                                "node-name-%d",
-                                                                1, (count + 1));
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_CONVERSION_FAILED,
-                                "Failed uuid to hostname conversion");
-
-                /* Since Both rebalance and bitrot scrub status are going to
-                 * use same code path till here, we should break in case
-                 * of scrub status */
-                if (op == GD_OP_SCRUB_STATUS) {
-                        break;
-                }
+            ret = glusterd_op_volume_dict_uuid_to_hostname(
+                op_ctx, "node-name-%d", 1, (count + 1));
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_CONVERSION_FAILED,
+                       "Failed uuid to hostname conversion");
 
-                ret = glusterd_op_check_peer_defrag_status (op_ctx, count);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DEFRAG_STATUS_UPDATE_FAIL,
-                                "Failed to reset defrag status for fix-layout");
+            /* Since Both rebalance and bitrot scrub status/ondemand
+             * are going to use same code path till here, we should
+             * break in case of scrub status.
+             */
+            if (op == GD_OP_SCRUB_STATUS || op == GD_OP_SCRUB_ONDEMAND) {
                 break;
+            }
 
-        default:
-                ret = 0;
-                gf_msg_debug (this->name, 0,
-                        "op_ctx modification not required");
-                break;
+            ret = glusterd_op_check_peer_defrag_status(op_ctx, count);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_DEFRAG_STATUS_UPDATE_FAIL,
+                       "Failed to reset defrag status for fix-layout");
+            break;
 
-        }
+        default:
+            ret = 0;
+            gf_msg_debug(this->name, 0, "op_ctx modification not required");
+            break;
+    }
 
 out:
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_OPCTX_UPDATE_FAIL,
-                        "op_ctx modification failed");
-        return;
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_OPCTX_UPDATE_FAIL,
+               "op_ctx modification failed");
+    return;
 }
 
 int
-glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
-                         glusterd_commit_hook_type_t type)
+glusterd_op_commit_hook(glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_commit_hook_type_t type)
 {
-        glusterd_conf_t *priv                   = NULL;
-        char            hookdir[PATH_MAX]       = {0, };
-        char            scriptdir[PATH_MAX]     = {0, };
-        char            type_subdir[256]        = {0, };
-        char            *cmd_subdir             = NULL;
-        int             ret                     = -1;
-
-        priv = THIS->private;
-        switch (type) {
-                case GD_COMMIT_HOOK_NONE:
-                case GD_COMMIT_HOOK_MAX:
-                        /*Won't be called*/
-                        break;
-
-                case GD_COMMIT_HOOK_PRE:
-                        strcpy (type_subdir, "pre");
-                        break;
-                case GD_COMMIT_HOOK_POST:
-                        strcpy (type_subdir, "post");
-                        break;
-        }
-
-        cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
-        if (strlen (cmd_subdir) == 0)
-                return -1;
-
-        GLUSTERD_GET_HOOKS_DIR (hookdir, GLUSTERD_HOOK_VER, priv);
-        snprintf (scriptdir, sizeof (scriptdir), "%s/%s/%s",
-                  hookdir, cmd_subdir, type_subdir);
-
-        switch (type) {
-                case GD_COMMIT_HOOK_NONE:
-                case GD_COMMIT_HOOK_MAX:
-                        /*Won't be called*/
-                        break;
-
-                case GD_COMMIT_HOOK_PRE:
-                        ret = glusterd_hooks_run_hooks (scriptdir, op, op_ctx,
-                                                        type);
-                        break;
-                case GD_COMMIT_HOOK_POST:
-                        ret = glusterd_hooks_post_stub_enqueue (scriptdir, op,
-                                                                op_ctx);
-                        break;
-        }
-
-        return ret;
+    glusterd_conf_t *priv = NULL;
+    char hookdir[PATH_MAX] = {
+        0,
+    };
+    char scriptdir[PATH_MAX] = {
+        0,
+    };
+    char *type_subdir = "";
+    char *cmd_subdir = NULL;
+    int ret = -1;
+    int32_t len = 0;
+
+    priv = THIS->private;
+    switch (type) {
+        case GD_COMMIT_HOOK_NONE:
+        case GD_COMMIT_HOOK_MAX:
+            /*Won't be called*/
+            break;
+
+        case GD_COMMIT_HOOK_PRE:
+            type_subdir = "pre";
+            break;
+        case GD_COMMIT_HOOK_POST:
+            type_subdir = "post";
+            break;
+    }
+
+    cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir(op);
+    if (strlen(cmd_subdir) == 0)
+        return -1;
+
+    GLUSTERD_GET_HOOKS_DIR(hookdir, GLUSTERD_HOOK_VER, priv);
+    len = snprintf(scriptdir, sizeof(scriptdir), "%s/%s/%s", hookdir,
+                   cmd_subdir, type_subdir);
+    if ((len < 0) || (len >= sizeof(scriptdir))) {
+        return -1;
+    }
+
+    switch (type) {
+        case GD_COMMIT_HOOK_NONE:
+        case GD_COMMIT_HOOK_MAX:
+            /*Won't be called*/
+            break;
+
+        case GD_COMMIT_HOOK_PRE:
+            ret = glusterd_hooks_run_hooks(scriptdir, op, op_ctx, type);
+            break;
+        case GD_COMMIT_HOOK_POST:
+            ret = glusterd_hooks_post_stub_enqueue(scriptdir, op, op_ctx);
+            break;
+    }
+
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_commit_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret           = 0;
-        int                     ret1          = 0;
-        rpc_clnt_procedure_t    *proc         = NULL;
-        glusterd_conf_t         *priv         = NULL;
-        xlator_t                *this         = NULL;
-        dict_t                  *dict         = NULL;
-        glusterd_peerinfo_t     *peerinfo     = NULL;
-        char                    *op_errstr    = NULL;
-        glusterd_op_t           op            = GD_OP_NONE;
-        uint32_t                pending_count = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        op      = glusterd_op_get_op ();
-
-        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
-                        LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                opinfo.op_errstr = op_errstr;
-                goto out;
-        }
-
-        ret = glusterd_op_commit_perform (op, dict, &op_errstr, NULL); //rsp_dict invalid for source
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL, LOGSTR_COMMIT_FAIL,
-                        gd_op_list[op], "localhost", (op_errstr) ? ":" : " ",
-                        (op_errstr) ? op_errstr : " ");
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_COMMIT_FAIL,
-                                     "localhost");
-                opinfo.op_errstr = op_errstr;
-                goto out;
-        }
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > opinfo.txn_generation)
-                        continue;
-
-                if (!peerinfo->connected || !peerinfo->mgmt)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
+    int ret = 0;
+    int ret1 = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    char *op_errstr = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    uint32_t pending_count = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    op = glusterd_op_get_op();
+
+    ret = glusterd_op_build_payload(&dict, &op_errstr, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    ret = glusterd_op_commit_perform(op, dict, &op_errstr,
+                                     NULL);  // rsp_dict invalid for source
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               LOGSTR_COMMIT_FAIL, gd_op_list[op], "localhost",
+               (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_COMMIT_FAIL, "localhost");
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_COMMIT_OP];
-                GF_ASSERT (proc);
-                if (proc->fn) {
-                        ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
-                        if (ret) {
-                                rcu_read_unlock ();
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "failed to set peerinfo");
-                                goto out;
-                        }
-                        ret = proc->fn (NULL, this, dict);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_COMMIT_REQ_SEND_FAIL,
-                                        "Failed to "
-                                        "send commit request for operation "
-                                        "'Volume %s' to peer %s",
-                                        gd_op_list[op], peerinfo->hostname);
-                                continue;
-                        }
-                        pending_count++;
-                }
-        }
-        rcu_read_unlock ();
+        if (!peerinfo->connected || !peerinfo->mgmt)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
 
-        opinfo.pending_count = pending_count;
-        gf_msg_debug (this->name, 0, "Sent commit op req for 'Volume %s' "
-                "to %d peers", gd_op_list[op], opinfo.pending_count);
+        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_COMMIT_OP];
+        GF_ASSERT(proc);
+        if (proc->fn) {
+            ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+            if (ret) {
+                RCU_READ_UNLOCK;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to set peerinfo");
+                goto out;
+            }
+            ret = proc->fn(NULL, this, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_COMMIT_REQ_SEND_FAIL,
+                       "Failed to "
+                       "send commit request for operation "
+                       "'Volume %s' to peer %s",
+                       gd_op_list[op], peerinfo->hostname);
+                continue;
+            }
+            pending_count++;
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
+    gf_msg_debug(this->name, 0,
+                 "Sent commit op req for 'Volume %s' "
+                 "to %d peers",
+                 gd_op_list[op], opinfo.pending_count);
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (ret)
-                opinfo.op_ret = ret;
+    if (ret)
+        opinfo.op_ret = ret;
 
-        ret1 = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret1)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    ret1 = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret1)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                             &event->txn_id, NULL);
-                opinfo.op_ret = ret;
-        }
+    if (ret) {
+        glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, &event->txn_id, NULL);
+        opinfo.op_ret = ret;
+    }
 
-        if (!opinfo.pending_count) {
-                if (op == GD_OP_REPLACE_BRICK) {
-                        ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
-                } else {
-                        glusterd_op_modify_op_ctx (op, NULL);
-                        ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
-                }
-                goto err;
+    if (!opinfo.pending_count) {
+        if (op == GD_OP_REPLACE_BRICK) {
+            ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
+        } else {
+            glusterd_op_modify_op_ctx(op, NULL);
+            ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
         }
+        goto err;
+    }
 
 err:
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_stage_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_stage_op_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-
-        GF_ASSERT (event);
+    int ret = 0;
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    GF_ASSERT(event);
 
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
+    if (opinfo.pending_count > 0)
+        goto out;
 
-
-        if (opinfo.pending_count > 0)
-                goto out;
-
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC,
-                                           &event->txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_STAGE_ACC, &event->txn_id,
+                                      NULL);
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_stage_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_stage_op_failed(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-
-        GF_ASSERT (event);
+    int ret = 0;
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    GF_ASSERT(event);
 
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
+    if (opinfo.pending_count > 0)
+        goto out;
 
-        if (opinfo.pending_count > 0)
-                goto out;
-
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                           &event->txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      NULL);
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_commit_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_commit_op_failed(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-
-        GF_ASSERT (event);
-
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
-
+    int ret = 0;
 
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    GF_ASSERT(event);
 
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    if (opinfo.pending_count > 0)
+        goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                           &event->txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      NULL);
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_brick_op_failed(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-        glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
-        gf_boolean_t                free_errstr = _gf_false;
-        xlator_t                    *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
-
-        ret = glusterd_remove_pending_entry (&opinfo.pending_bricks, ev_ctx->pending_node->node);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UNKNOWN_RESPONSE, "unknown response received ");
-                ret = -1;
-                free_errstr = _gf_true;
-                goto out;
-        }
-        if (opinfo.brick_pending_count > 0)
-                opinfo.brick_pending_count--;
-        if (opinfo.op_ret == 0)
-                opinfo.op_ret = ev_ctx->op_ret;
-
-        if (opinfo.op_errstr == NULL)
-                opinfo.op_errstr = ev_ctx->op_errstr;
-        else
-                free_errstr = _gf_true;
-
-
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
-
-        if (opinfo.brick_pending_count > 0)
-                goto out;
-
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                           &event->txn_id, ev_ctx->commit_ctx);
+    int ret = 0;
+    glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+    gf_boolean_t free_errstr = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+    ev_ctx = ctx;
+
+    ret = glusterd_remove_pending_entry(&opinfo.pending_bricks,
+                                        ev_ctx->pending_node->node);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_RESPONSE,
+               "unknown response received ");
+        ret = -1;
+        free_errstr = _gf_true;
+        goto out;
+    }
+    if (opinfo.brick_pending_count > 0)
+        opinfo.brick_pending_count--;
+    if (opinfo.op_ret == 0)
+        opinfo.op_ret = ev_ctx->op_ret;
+
+    if (opinfo.op_errstr == NULL)
+        opinfo.op_errstr = ev_ctx->op_errstr;
+    else
+        free_errstr = _gf_true;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (opinfo.brick_pending_count > 0)
+        goto out;
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      ev_ctx->commit_ctx);
 
 out:
-        if (ev_ctx->rsp_dict)
-                dict_unref (ev_ctx->rsp_dict);
-        if (free_errstr && ev_ctx->op_errstr)
-                GF_FREE (ev_ctx->op_errstr);
-        GF_FREE (ctx);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-
-        return ret;
+    if (ev_ctx->rsp_dict)
+        dict_unref(ev_ctx->rsp_dict);
+    if (free_errstr && ev_ctx->op_errstr)
+        GF_FREE(ev_ctx->op_errstr);
+    GF_FREE(ctx);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_commit_op_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret               = 0;
-        gf_boolean_t            commit_ack_inject = _gf_true;
-        glusterd_op_t           op                = GD_OP_NONE;
-        xlator_t               *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        op = glusterd_op_get_op ();
-        GF_ASSERT (event);
-
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
-
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
-
-        if (opinfo.pending_count > 0)
-                goto out;
-
-        if (op == GD_OP_REPLACE_BRICK) {
-                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_RBOP_START_FAIL, "Couldn't start "
-                                "replace-brick operation.");
-                        goto out;
-                }
-
-                commit_ack_inject = _gf_false;
-                goto out;
+    int ret = 0;
+    gf_boolean_t commit_ack_inject = _gf_true;
+    glusterd_op_t op = GD_OP_NONE;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    op = glusterd_op_get_op();
+    GF_ASSERT(event);
+
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (opinfo.pending_count > 0)
+        goto out;
+
+    if (op == GD_OP_REPLACE_BRICK) {
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RBOP_START_FAIL,
+                   "Couldn't start "
+                   "replace-brick operation.");
+            goto out;
         }
 
+        commit_ack_inject = _gf_false;
+        goto out;
+    }
 
 out:
-        if (commit_ack_inject) {
-                if (ret)
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                                           &event->txn_id,
-                                                           NULL);
-                else if (!opinfo.pending_count) {
-                        glusterd_op_modify_op_ctx (op, NULL);
-                        ret = glusterd_op_sm_inject_event
-                                                  (GD_OP_EVENT_COMMIT_ACC,
-                                                   &event->txn_id, NULL);
-                }
-                /*else do nothing*/
+    if (commit_ack_inject) {
+        if (ret)
+            ret = glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT,
+                                              &event->txn_id, NULL);
+        else if (!opinfo.pending_count) {
+            glusterd_op_modify_op_ctx(op, NULL);
+            ret = glusterd_op_sm_inject_event(GD_OP_EVENT_COMMIT_ACC,
+                                              &event->txn_id, NULL);
         }
+        /*else do nothing*/
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_unlock_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_unlock_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
-
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    if (opinfo.pending_count > 0)
+        goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
-                                           &event->txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACC, &event->txn_id,
+                                      NULL);
 
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_op_clear_errstr() {
-        opinfo.op_errstr = NULL;
-        return 0;
+glusterd_op_clear_errstr()
+{
+    opinfo.op_errstr = NULL;
+    return 0;
 }
 
 int32_t
-glusterd_op_set_ctx (void *ctx)
+glusterd_op_set_ctx(void *ctx)
 {
+    opinfo.op_ctx = ctx;
 
-        opinfo.op_ctx = ctx;
-
-        return 0;
-
+    return 0;
 }
 
 int32_t
-glusterd_op_reset_ctx ()
+glusterd_op_reset_ctx()
 {
+    glusterd_op_set_ctx(NULL);
 
-        glusterd_op_set_ctx (NULL);
-
-        return 0;
+    return 0;
 }
 
 int32_t
-glusterd_op_txn_complete (uuid_t *txn_id)
+glusterd_op_txn_complete(uuid_t *txn_id)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        int32_t                 op = -1;
-        int32_t                 op_ret = 0;
-        int32_t                 op_errno = 0;
-        rpcsvc_request_t        *req = NULL;
-        void                    *ctx = NULL;
-        char                    *op_errstr = NULL;
-        char                    *volname = NULL;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        op  = glusterd_op_get_op ();
-        ctx = glusterd_op_get_ctx ();
-        op_ret = opinfo.op_ret;
-        op_errno = opinfo.op_errno;
-        req = opinfo.req;
-        if (opinfo.op_errstr)
-                op_errstr = opinfo.op_errstr;
-
-        opinfo.op_ret = 0;
-        opinfo.op_errno = 0;
-        glusterd_op_clear_op ();
-        glusterd_op_reset_ctx ();
-        glusterd_op_clear_errstr ();
-
-        /* Based on the op-version, we release the cluster or mgmt_v3 lock */
-        if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                ret = glusterd_unlock (MY_UUID);
-                /* unlock cant/shouldnt fail here!! */
-                if (ret)
-                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                GD_MSG_GLUSTERD_UNLOCK_FAIL,
-                                "Unable to clear local lock, ret: %d", ret);
-                else
-                        gf_msg_debug (this->name, 0, "Cleared local lock");
-        } else {
-                ret = dict_get_str (ctx, "volname", &volname);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "No Volume name present. "
-                                "Locks have not been held.");
-
-                if (volname) {
-                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
-                                                       "vol");
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                        "Unable to release lock for %s",
-                                        volname);
-                }
-        }
-
-        ret = glusterd_op_send_cli_response (op, op_ret,
-                                             op_errno, req, ctx, op_errstr);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_CLI_RESP,
-                        "Responding to cli failed, "
-                        "ret: %d", ret);
-                //Ignore this error, else state machine blocks
-                ret = 0;
-        }
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    int32_t op = -1;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    rpcsvc_request_t *req = NULL;
+    void *ctx = NULL;
+    char *op_errstr = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    op = glusterd_op_get_op();
+    ctx = glusterd_op_get_ctx();
+    op_ret = opinfo.op_ret;
+    op_errno = opinfo.op_errno;
+    req = opinfo.req;
+    if (opinfo.op_errstr)
+        op_errstr = opinfo.op_errstr;
+
+    opinfo.op_ret = 0;
+    opinfo.op_errno = 0;
+    glusterd_op_clear_op();
+    glusterd_op_reset_ctx();
+    glusterd_op_clear_errstr();
+
+    /* Based on the op-version, we release the cluster or mgmt_v3 lock */
+    if (priv->op_version < GD_OP_VERSION_3_6_0) {
+        ret = glusterd_unlock(MY_UUID);
+        /* unlock can't/shouldn't fail here!! */
+        if (ret)
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_GLUSTERD_UNLOCK_FAIL,
+                   "Unable to clear local lock, ret: %d", ret);
+        else
+            gf_msg_debug(this->name, 0, "Cleared local lock");
+    } else {
+        ret = dict_get_strn(ctx, "volname", SLEN("volname"), &volname);
+        if (ret)
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                   "No Volume name present. "
+                   "Locks have not been held.");
 
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+        if (volname) {
+            ret = glusterd_mgmt_v3_unlock(volname, MY_UUID, "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", volname);
+        }
+    }
+
+    ret = glusterd_op_send_cli_response(op, op_ret, op_errno, req, ctx,
+                                        op_errstr);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+               "Responding to cli failed, "
+               "ret: %d",
+               ret);
+        // Ignore this error, else state machine blocks
+        ret = 0;
+    }
 
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        if (priv->pending_quorum_action)
-                glusterd_do_quorum_action ();
+    if (priv->pending_quorum_action)
+        glusterd_do_quorum_action();
 
-        /* Clearing the transaction opinfo */
-        ret = glusterd_clear_txn_opinfo (txn_id);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_OPINFO_CLEAR_FAIL,
-                        "Unable to clear transaction's opinfo");
+    /* Clearing the transaction opinfo */
+    ret = glusterd_clear_txn_opinfo(txn_id);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_CLEAR_FAIL,
+               "Unable to clear transaction's opinfo");
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_unlocked_all (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_unlocked_all(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        ret = glusterd_op_txn_complete (&event->txn_id);
+    ret = glusterd_op_txn_complete(&event->txn_id);
 
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_stage_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = -1;
-        glusterd_req_ctx_t      *req_ctx = NULL;
-        int32_t                 status = 0;
-        dict_t                  *rsp_dict  = NULL;
-        char                    *op_errstr = NULL;
-        dict_t                  *dict = NULL;
-        xlator_t                *this = NULL;
-        uuid_t                  *txn_id = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (ctx);
-
-        req_ctx = ctx;
-
-        dict = req_ctx->dict;
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        status = glusterd_op_stage_validate (req_ctx->op, dict, &op_errstr,
-                                             rsp_dict);
-
-        if (status) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VALIDATE_FAILED, "Stage failed on operation"
-                        " 'Volume %s', Status : %d", gd_op_list[req_ctx->op],
-                        status);
-        }
-
-        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-
-        if (txn_id)
-                gf_uuid_copy (*txn_id, event->txn_id);
-        else {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_bin (rsp_dict, "transaction_id",
-                            txn_id, sizeof(*txn_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                       "Failed to set transaction id.");
-                GF_FREE (txn_id);
-                goto out;
-        }
-
-        ret = glusterd_op_stage_send_resp (req_ctx->req, req_ctx->op,
-                                           status, op_errstr, rsp_dict);
+    int ret = -1;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    int32_t status = 0;
+    dict_t *rsp_dict = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(ctx);
+
+    req_ctx = ctx;
+
+    dict = req_ctx->dict;
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to get new dictionary");
+        return -1;
+    }
+
+    status = glusterd_op_stage_validate(req_ctx->op, dict, &op_errstr,
+                                        rsp_dict);
+
+    if (status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               "Stage failed on operation"
+               " 'Volume %s', Status : %d",
+               gd_op_list[req_ctx->op], status);
+    }
+
+    txn_id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+
+    if (txn_id)
+        gf_uuid_copy(*txn_id, event->txn_id);
+    else {
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_get_txn_opinfo(&event->txn_id, &txn_op_info);
+
+    ret = dict_set_bin(rsp_dict, "transaction_id", txn_id, sizeof(*txn_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set transaction id.");
+        GF_FREE(txn_id);
+        txn_id = NULL;
+        goto out;
+    }
+
+    ret = glusterd_op_stage_send_resp(req_ctx->req, req_ctx->op, status,
+                                      op_errstr, rsp_dict);
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    /* for no volname transactions, the txn_opinfo needs to be cleaned up
+     * as there's no unlock event triggered. However if the originator node of
+     * this transaction is still running with a version lower than 60000,
+     * txn_opinfo can't be cleared as that'll lead to a race of referring op_ctx
+     * after it's being freed.
+     */
+    if (txn_op_info.skip_locking && priv->op_version >= GD_OP_VERSION_6_0 &&
+        txn_id)
+        ret = glusterd_clear_txn_opinfo(txn_id);
 
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
 }
 
 static gf_boolean_t
-glusterd_need_brick_op (glusterd_op_t op)
+glusterd_need_brick_op(glusterd_op_t op)
 {
-        gf_boolean_t ret        = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
+    GF_ASSERT(GD_OP_NONE < op && op < GD_OP_MAX);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_PROFILE_VOLUME:
         case GD_OP_STATUS_VOLUME:
         case GD_OP_DEFRAG_BRICK_VOLUME:
         case GD_OP_HEAL_VOLUME:
         case GD_OP_SCRUB_STATUS:
-                ret = _gf_true;
-                break;
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = _gf_true;
+            break;
         default:
-                ret = _gf_false;
-        }
+            ret = _gf_false;
+    }
 
-        return ret;
+    return ret;
 }
 
-dict_t*
-glusterd_op_init_commit_rsp_dict (glusterd_op_t op)
+dict_t *
+glusterd_op_init_commit_rsp_dict(glusterd_op_t op)
 {
-        dict_t                  *rsp_dict = NULL;
-        dict_t                  *op_ctx   = NULL;
+    dict_t *rsp_dict = NULL;
+    dict_t *op_ctx = NULL;
 
-        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
+    GF_ASSERT(GD_OP_NONE < op && op < GD_OP_MAX);
 
-        if (glusterd_need_brick_op (op)) {
-                op_ctx = glusterd_op_get_ctx ();
-                GF_ASSERT (op_ctx);
-                rsp_dict = dict_ref (op_ctx);
-        } else {
-                rsp_dict = dict_new ();
-        }
+    if (glusterd_need_brick_op(op)) {
+        op_ctx = glusterd_op_get_ctx();
+        GF_ASSERT(op_ctx);
+        rsp_dict = dict_ref(op_ctx);
+    } else {
+        rsp_dict = dict_new();
+    }
 
-        return rsp_dict;
+    return rsp_dict;
 }
 
 static int
-glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_commit_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                       ret        = 0;
-        glusterd_req_ctx_t       *req_ctx    = NULL;
-        int32_t                   status     = 0;
-        char                     *op_errstr  = NULL;
-        dict_t                   *dict       = NULL;
-        dict_t                   *rsp_dict   = NULL;
-        xlator_t                 *this       = NULL;
-        uuid_t                   *txn_id     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (ctx);
-
-        req_ctx = ctx;
+    int ret = 0;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    int32_t status = 0;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    gf_boolean_t need_cleanup = _gf_true;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(ctx);
+
+    req_ctx = ctx;
+
+    dict = req_ctx->dict;
+
+    rsp_dict = glusterd_op_init_commit_rsp_dict(req_ctx->op);
+    if (NULL == rsp_dict)
+        return -1;
+
+    if (GD_OP_CLEARLOCKS_VOLUME == req_ctx->op) {
+        /*clear locks should be run only on
+         * originator glusterd*/
+        status = 0;
+
+    } else {
+        status = glusterd_op_commit_perform(req_ctx->op, dict, &op_errstr,
+                                            rsp_dict);
+    }
+
+    if (status)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit of operation "
+               "'Volume %s' failed: %d",
+               gd_op_list[req_ctx->op], status);
+
+    txn_id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+
+    if (txn_id)
+        gf_uuid_copy(*txn_id, event->txn_id);
+    else {
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_get_txn_opinfo(&event->txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0,
+                         GD_MSG_TRANS_OPINFO_GET_FAIL,
+                         "Unable to get transaction opinfo "
+                         "for transaction ID : %s",
+                         uuid_utoa(event->txn_id));
+        goto out;
+    }
+
+    ret = dict_set_bin(rsp_dict, "transaction_id", txn_id, sizeof(*txn_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set transaction id.");
+        if (txn_op_info.skip_locking)
+            ret = glusterd_clear_txn_opinfo(txn_id);
+        need_cleanup = _gf_false;
+        GF_FREE(txn_id);
+        goto out;
+    }
+
+    ret = glusterd_op_commit_send_resp(req_ctx->req, req_ctx->op, status,
+                                       op_errstr, rsp_dict);
 
-        dict = req_ctx->dict;
+out:
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    /* for no volname transactions, the txn_opinfo needs to be cleaned up
+     * as there's no unlock event triggered
+     */
+    if (need_cleanup && txn_id && txn_op_info.skip_locking)
+        ret = glusterd_clear_txn_opinfo(txn_id);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+
+    return ret;
+}
 
-        rsp_dict = glusterd_op_init_commit_rsp_dict (req_ctx->op);
-        if (NULL == rsp_dict)
-                return -1;
+static int
+glusterd_op_ac_send_commit_failed(glusterd_op_sm_event_t *event, void *ctx)
+{
+    int ret = 0;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    dict_t *op_ctx = NULL;
 
+    GF_ASSERT(ctx);
 
-        if (GD_OP_CLEARLOCKS_VOLUME == req_ctx->op) {
-                /*clear locks should be run only on
-                 * originator glusterd*/
-                status = 0;
+    req_ctx = ctx;
 
-        } else {
-                status = glusterd_op_commit_perform (req_ctx->op, dict,
-                                                     &op_errstr, rsp_dict);
-        }
+    op_ctx = glusterd_op_get_ctx();
 
-        if (status)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL, "Commit of operation "
-                        "'Volume %s' failed: %d", gd_op_list[req_ctx->op],
-                        status);
+    ret = glusterd_op_commit_send_resp(req_ctx->req, req_ctx->op, opinfo.op_ret,
+                                       opinfo.op_errstr, op_ctx);
 
-        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (opinfo.op_errstr && (strcmp(opinfo.op_errstr, ""))) {
+        GF_FREE(opinfo.op_errstr);
+        opinfo.op_errstr = NULL;
+    }
 
-        if (txn_id)
-                gf_uuid_copy (*txn_id, event->txn_id);
-        else {
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = dict_set_bin (rsp_dict, "transaction_id",
-                            txn_id, sizeof(*txn_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                       "Failed to set transaction id.");
-                GF_FREE (txn_id);
-                goto out;
-        }
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
+}
 
-        ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
-                                            status, op_errstr, rsp_dict);
+static int
+glusterd_op_sm_transition_state(glusterd_op_info_t *opinfo,
+                                glusterd_op_sm_t *state,
+                                glusterd_op_sm_event_type_t event_type)
+{
+    glusterd_conf_t *conf = NULL;
 
-out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    GF_ASSERT(state);
+    GF_ASSERT(opinfo);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    conf = THIS->private;
+    GF_ASSERT(conf);
 
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+    (void)glusterd_sm_tr_log_transition_add(
+        &conf->op_sm_log, opinfo->state.state, state[event_type].next_state,
+        event_type);
 
-        return ret;
+    opinfo->state.state = state[event_type].next_state;
+    return 0;
 }
 
-static int
-glusterd_op_ac_send_commit_failed (glusterd_op_sm_event_t *event, void *ctx)
+int32_t
+glusterd_op_stage_validate(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict)
 {
-        int                             ret = 0;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        dict_t                          *op_ctx = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
 
-        GF_ASSERT (ctx);
+    switch (op) {
+        case GD_OP_CREATE_VOLUME:
+            ret = glusterd_op_stage_create_volume(dict, op_errstr, rsp_dict);
+            break;
 
-        req_ctx = ctx;
+        case GD_OP_START_VOLUME:
+            ret = glusterd_op_stage_start_volume(dict, op_errstr, rsp_dict);
+            break;
 
-        op_ctx = glusterd_op_get_ctx ();
+        case GD_OP_STOP_VOLUME:
+            ret = glusterd_op_stage_stop_volume(dict, op_errstr);
+            break;
 
-        ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
-                                            opinfo.op_ret, opinfo.op_errstr,
-                                            op_ctx);
+        case GD_OP_DELETE_VOLUME:
+            ret = glusterd_op_stage_delete_volume(dict, op_errstr);
+            break;
 
-        if (opinfo.op_errstr && (strcmp (opinfo.op_errstr, ""))) {
-                GF_FREE (opinfo.op_errstr);
-                opinfo.op_errstr = NULL;
-        }
+        case GD_OP_ADD_BRICK:
+            ret = glusterd_op_stage_add_brick(dict, op_errstr, rsp_dict);
+            break;
 
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+        case GD_OP_REPLACE_BRICK:
+            ret = glusterd_op_stage_replace_brick(dict, op_errstr, rsp_dict);
+            break;
 
+        case GD_OP_SET_VOLUME:
+            ret = glusterd_op_stage_set_volume(dict, op_errstr);
+            break;
 
-        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);
-        return ret;
-}
+        case GD_OP_GANESHA:
+            ret = glusterd_op_stage_set_ganesha(dict, op_errstr);
+            break;
 
-static int
-glusterd_op_sm_transition_state (glusterd_op_info_t *opinfo,
-                                 glusterd_op_sm_t *state,
-                                 glusterd_op_sm_event_type_t event_type)
-{
-        glusterd_conf_t         *conf = NULL;
+        case GD_OP_RESET_VOLUME:
+            ret = glusterd_op_stage_reset_volume(dict, op_errstr);
+            break;
+        case GD_OP_REMOVE_BRICK:
+            ret = glusterd_op_stage_remove_brick(dict, op_errstr);
+            break;
 
-        GF_ASSERT (state);
-        GF_ASSERT (opinfo);
+        case GD_OP_LOG_ROTATE:
+            ret = glusterd_op_stage_log_rotate(dict, op_errstr);
+            break;
 
-        conf = THIS->private;
-        GF_ASSERT (conf);
+        case GD_OP_SYNC_VOLUME:
+            ret = glusterd_op_stage_sync_volume(dict, op_errstr);
+            break;
 
-        (void) glusterd_sm_tr_log_transition_add (&conf->op_sm_log,
-                                           opinfo->state.state,
-                                           state[event_type].next_state,
-                                           event_type);
+        case GD_OP_GSYNC_CREATE:
+            ret = glusterd_op_stage_gsync_create(dict, op_errstr);
+            break;
 
-        opinfo->state.state = state[event_type].next_state;
-        return 0;
-}
+        case GD_OP_GSYNC_SET:
+            ret = glusterd_op_stage_gsync_set(dict, op_errstr);
+            break;
 
-int32_t
-glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                            dict_t *rsp_dict)
-{
-        int ret = -1;
-        xlator_t *this = THIS;
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_op_stage_stats_volume(dict, op_errstr);
+            break;
 
-        switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                        ret = glusterd_op_stage_create_volume (dict, op_errstr,
-                                                               rsp_dict);
-                        break;
-
-                case GD_OP_START_VOLUME:
-                        ret = glusterd_op_stage_start_volume (dict, op_errstr,
-                                                              rsp_dict);
-                        break;
-
-                case GD_OP_STOP_VOLUME:
-                        ret = glusterd_op_stage_stop_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_DELETE_VOLUME:
-                        ret = glusterd_op_stage_delete_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_ADD_BRICK:
-                        ret = glusterd_op_stage_add_brick (dict, op_errstr,
-                                                           rsp_dict);
-                        break;
-
-                case GD_OP_REPLACE_BRICK:
-                        ret = glusterd_op_stage_replace_brick (dict, op_errstr,
-                                                               rsp_dict);
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        ret = glusterd_op_stage_set_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_GANESHA:
-                        ret = glusterd_op_stage_set_ganesha (dict, op_errstr);
-                        break;
-
-                case GD_OP_RESET_VOLUME:
-                        ret = glusterd_op_stage_reset_volume (dict, op_errstr);
-                        break;
-                case GD_OP_REMOVE_BRICK:
-                        ret = glusterd_op_stage_remove_brick (dict, op_errstr);
-                        break;
-
-                case GD_OP_LOG_ROTATE:
-                        ret = glusterd_op_stage_log_rotate (dict, op_errstr);
-                        break;
-
-                case GD_OP_SYNC_VOLUME:
-                        ret = glusterd_op_stage_sync_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                        ret = glusterd_op_stage_gsync_create (dict, op_errstr);
-                        break;
-
-                case GD_OP_GSYNC_SET:
-                        ret = glusterd_op_stage_gsync_set (dict, op_errstr);
-                        break;
-
-                case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_op_stage_stats_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_QUOTA:
-                        ret = glusterd_op_stage_quota (dict, op_errstr,
-                                                       rsp_dict);
-                        break;
-
-                case GD_OP_STATUS_VOLUME:
-                        ret = glusterd_op_stage_status_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_REBALANCE:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        ret = glusterd_op_stage_rebalance (dict, op_errstr);
-                        break;
-
-                case GD_OP_HEAL_VOLUME:
-                        ret = glusterd_op_stage_heal_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_STATEDUMP_VOLUME:
-                        ret = glusterd_op_stage_statedump_volume (dict,
-                                                                  op_errstr);
-                        break;
-                case GD_OP_CLEARLOCKS_VOLUME:
-                        ret = glusterd_op_stage_clearlocks_volume (dict,
-                                                                   op_errstr);
-                        break;
-
-                case GD_OP_COPY_FILE:
-                        ret = glusterd_op_stage_copy_file (dict, op_errstr);
-                        break;
-
-                case GD_OP_SYS_EXEC:
-                        ret = glusterd_op_stage_sys_exec (dict, op_errstr);
-                        break;
-
-                case GD_OP_BARRIER:
-                        ret = glusterd_op_stage_barrier (dict, op_errstr);
-                        break;
-
-                case GD_OP_BITROT:
-                case GD_OP_SCRUB_STATUS:
-                        ret = glusterd_op_stage_bitrot (dict, op_errstr,
-                                                        rsp_dict);
-                        break;
-
-                default:
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_INVALID_ENTRY, "Unknown op %s",
-                                gd_op_list[op]);
-        }
-
-        gf_msg_debug (this->name, 0, "OP = %d. Returning %d", op, ret);
-        return ret;
+        case GD_OP_QUOTA:
+            ret = glusterd_op_stage_quota(dict, op_errstr, rsp_dict);
+            break;
+
+        case GD_OP_STATUS_VOLUME:
+            ret = glusterd_op_stage_status_volume(dict, op_errstr);
+            break;
+
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_op_stage_rebalance(dict, op_errstr);
+            break;
+
+        case GD_OP_HEAL_VOLUME:
+            ret = glusterd_op_stage_heal_volume(dict, op_errstr);
+            break;
+
+        case GD_OP_STATEDUMP_VOLUME:
+            ret = glusterd_op_stage_statedump_volume(dict, op_errstr);
+            break;
+        case GD_OP_CLEARLOCKS_VOLUME:
+            ret = glusterd_op_stage_clearlocks_volume(dict, op_errstr);
+            break;
+
+        case GD_OP_COPY_FILE:
+            ret = glusterd_op_stage_copy_file(dict, op_errstr);
+            break;
+
+        case GD_OP_SYS_EXEC:
+            ret = glusterd_op_stage_sys_exec(dict, op_errstr);
+            break;
+
+        case GD_OP_BARRIER:
+            ret = glusterd_op_stage_barrier(dict, op_errstr);
+            break;
+
+        case GD_OP_BITROT:
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = glusterd_op_stage_bitrot(dict, op_errstr, rsp_dict);
+            break;
+
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Unknown op %s", gd_op_list[op]);
+    }
+
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
+static void
+glusterd_wait_for_blockers(glusterd_conf_t *priv)
+{
+    while (GF_ATOMIC_GET(priv->blockers)) {
+        synccond_wait(&priv->cond_blockers, &priv->big_lock);
+    }
+}
 
 int32_t
-glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                            dict_t *rsp_dict)
+glusterd_op_commit_perform(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict)
 {
-        int ret = -1;
-        xlator_t *this = THIS;
+    int ret = -1;
+    xlator_t *this = THIS;
 
-        glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_PRE);
-        switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                        ret = glusterd_op_create_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_START_VOLUME:
-                        ret = glusterd_op_start_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_STOP_VOLUME:
-                        ret = glusterd_op_stop_volume (dict);
-                        break;
-
-                case GD_OP_DELETE_VOLUME:
-                        ret = glusterd_op_delete_volume (dict);
-                        break;
-
-                case GD_OP_ADD_BRICK:
-                        ret = glusterd_op_add_brick (dict, op_errstr);
-                        break;
-
-                case GD_OP_REPLACE_BRICK:
-                        ret = glusterd_op_replace_brick (dict, rsp_dict);
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        ret = glusterd_op_set_volume (dict, op_errstr);
-                        break;
-                case GD_OP_GANESHA:
-                        ret = glusterd_op_set_ganesha (dict, op_errstr);
-                        break;
-
-                case GD_OP_RESET_VOLUME:
-                        ret = glusterd_op_reset_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_REMOVE_BRICK:
-                        ret = glusterd_op_remove_brick (dict, op_errstr);
-                        break;
-
-                case GD_OP_LOG_ROTATE:
-                        ret = glusterd_op_log_rotate (dict);
-                        break;
-
-                case GD_OP_SYNC_VOLUME:
-                        ret = glusterd_op_sync_volume (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                        ret = glusterd_op_gsync_create (dict, op_errstr,
-                                                        rsp_dict);
-                        break;
-
-                case GD_OP_GSYNC_SET:
-                        ret = glusterd_op_gsync_set (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_op_stats_volume (dict, op_errstr,
-                                                        rsp_dict);
-                        break;
-
-                case GD_OP_QUOTA:
-                        ret = glusterd_op_quota (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_STATUS_VOLUME:
-                        ret = glusterd_op_status_volume (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_REBALANCE:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        ret = glusterd_op_rebalance (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_HEAL_VOLUME:
-                        ret = glusterd_op_heal_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_STATEDUMP_VOLUME:
-                        ret = glusterd_op_statedump_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_CLEARLOCKS_VOLUME:
-                        ret = glusterd_op_clearlocks_volume (dict, op_errstr,
-                                                             rsp_dict);
-                        break;
-
-                case GD_OP_COPY_FILE:
-                        ret = glusterd_op_copy_file (dict, op_errstr);
-                        break;
-
-                case GD_OP_SYS_EXEC:
-                        ret = glusterd_op_sys_exec (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_BARRIER:
-                        ret = glusterd_op_barrier (dict, op_errstr);
-                        break;
-
-                case GD_OP_BITROT:
-                case GD_OP_SCRUB_STATUS:
-                        ret = glusterd_op_bitrot (dict, op_errstr, rsp_dict);
-                        break;
-
-                default:
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_INVALID_ENTRY, "Unknown op %s",
-                                gd_op_list[op]);
-                        break;
-        }
-
-        if (ret == 0)
-            glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_POST);
-
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
-}
+    glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_PRE);
+    switch (op) {
+        case GD_OP_CREATE_VOLUME:
+            ret = glusterd_op_create_volume(dict, op_errstr);
+            break;
 
+        case GD_OP_START_VOLUME:
+            ret = glusterd_op_start_volume(dict, op_errstr);
+            break;
 
-static int
-glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
-                                    struct cds_list_head *selected)
-{
-        int                                     ret = 0;
-        int                                     flags = 0;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        glusterd_pending_node_t                 *pending_node = NULL;
-
-        ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
-        if (ret)
-                goto out;
+        case GD_OP_STOP_VOLUME:
+            ret = glusterd_op_stop_volume(dict);
+            break;
+
+        case GD_OP_DELETE_VOLUME:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_delete_volume(dict);
+            break;
+
+        case GD_OP_ADD_BRICK:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_add_brick(dict, op_errstr);
+            break;
+
+        case GD_OP_REPLACE_BRICK:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_replace_brick(dict, rsp_dict);
+            break;
+
+        case GD_OP_SET_VOLUME:
+            ret = glusterd_op_set_volume(dict, op_errstr);
+            break;
+        case GD_OP_GANESHA:
+            ret = glusterd_op_set_ganesha(dict, op_errstr);
+            break;
+        case GD_OP_RESET_VOLUME:
+            ret = glusterd_op_reset_volume(dict, op_errstr);
+            break;
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
+        case GD_OP_REMOVE_BRICK:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_remove_brick(dict, op_errstr);
+            break;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_is_brick_started (brickinfo)) {
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        } else {
-                                pending_node->node = brickinfo;
-                                pending_node->type = GD_NODE_BRICK;
-                                cds_list_add_tail (&pending_node->list,
-                                                   selected);
-                                pending_node = NULL;
-                        }
-                }
-        }
+        case GD_OP_LOG_ROTATE:
+            ret = glusterd_op_log_rotate(dict);
+            break;
 
-out:
-        return ret;
-}
+        case GD_OP_SYNC_VOLUME:
+            ret = glusterd_op_sync_volume(dict, op_errstr, rsp_dict);
+            break;
 
-static int
-glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
-                                     struct cds_list_head *selected)
-{
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        char                                    *brick = NULL;
-        int32_t                                 count = 0;
-        int32_t                                 i = 1;
-        char                                    key[256] = {0,};
-        glusterd_pending_node_t                 *pending_node = NULL;
-        int32_t                                 command = 0;
-        int32_t                                 force = 0;
+        case GD_OP_GSYNC_CREATE:
+            ret = glusterd_op_gsync_create(dict, op_errstr, rsp_dict);
+            break;
 
+        case GD_OP_GSYNC_SET:
+            ret = glusterd_op_gsync_set(dict, op_errstr, rsp_dict);
+            break;
 
-        ret = dict_get_str (dict, "volname", &volname);
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_op_stats_volume(dict, op_errstr, rsp_dict);
+            break;
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
+        case GD_OP_QUOTA:
+            ret = glusterd_op_quota(dict, op_errstr, rsp_dict);
+            break;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+        case GD_OP_STATUS_VOLUME:
+            ret = glusterd_op_status_volume(dict, op_errstr, rsp_dict);
+            break;
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "Unable to allocate memory");
-                goto out;
-        }
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_op_rebalance(dict, op_errstr, rsp_dict);
+            break;
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, -ret,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get count");
-                goto out;
-        }
+        case GD_OP_HEAL_VOLUME:
+            ret = glusterd_op_heal_volume(dict, op_errstr);
+            break;
 
-        ret = dict_get_int32 (dict, "command", &command);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, -ret,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get command");
-                goto out;
-        }
+        case GD_OP_STATEDUMP_VOLUME:
+            ret = glusterd_op_statedump_volume(dict, op_errstr);
+            break;
 
-        if (command == GF_OP_CMD_DETACH_START)
-                return glusterd_bricks_select_rebalance_volume(dict, op_errstr, selected);
+        case GD_OP_CLEARLOCKS_VOLUME:
+            ret = glusterd_op_clearlocks_volume(dict, op_errstr, rsp_dict);
+            break;
 
-        ret = dict_get_int32 (dict, "force", &force);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0,
-                        GD_MSG_DICT_GET_FAILED, "force flag is not set");
-                ret = 0;
-                goto out;
-        }
+        case GD_OP_COPY_FILE:
+            ret = glusterd_op_copy_file(dict, op_errstr);
+            break;
 
-        while ( i <= count) {
-                snprintf (key, 256, "brick%d", i);
+        case GD_OP_SYS_EXEC:
+            ret = glusterd_op_sys_exec(dict, op_errstr, rsp_dict);
+            break;
 
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Unable to get brick");
-                        goto out;
-                }
+        case GD_OP_BARRIER:
+            ret = glusterd_op_barrier(dict, op_errstr);
+            break;
 
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo,
-                                                              _gf_false);
+        case GD_OP_BITROT:
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = glusterd_op_bitrot(dict, op_errstr, rsp_dict);
+            break;
 
-                if (ret)
-                        goto out;
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Unknown op %s", gd_op_list[op]);
+            break;
+    }
 
-                if (glusterd_is_brick_started (brickinfo)) {
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        } else {
-                                pending_node->node = brickinfo;
-                                pending_node->type = GD_NODE_BRICK;
-                                cds_list_add_tail (&pending_node->list,
-                                                   selected);
-                                pending_node = NULL;
-                        }
-                }
-                i++;
-        }
+    if (ret == 0)
+        glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_POST);
 
-out:
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr,
-                                       struct cds_list_head *selected)
+glusterd_bricks_select_stop_volume(dict_t *dict, char **op_errstr,
+                                   struct cds_list_head *selected)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        int32_t                                 stats_op = GF_CLI_STATS_NONE;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        glusterd_pending_node_t                 *pending_node = NULL;
-        char                                    *brick = NULL;
-
+    int ret = 0;
+    int flags = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+
+    ret = glusterd_op_stop_volume_args_get(dict, &volname, &flags);
+    if (ret)
+        goto out;
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (glusterd_is_brick_started(brickinfo)) {
+            pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                     gf_gld_mt_pending_node_t);
+            if (!pending_node) {
+                ret = -1;
+                goto out;
+            } else {
+                pending_node->node = brickinfo;
+                pending_node->type = GD_NODE_BRICK;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
+            }
+            /*
+             * This is not really the right place to do it, but
+             * it's the most convenient.
+             * TBD: move this to *after* the RPC
+             */
+            brickinfo->status = GF_BRICK_STOPPED;
+        }
+    }
 
+out:
+    return ret;
+}
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+static int
+glusterd_bricks_select_remove_brick(dict_t *dict, char **op_errstr,
+                                    struct cds_list_head *selected)
+{
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char *brick = NULL;
+    int32_t count = 0;
+    int32_t i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    glusterd_pending_node_t *pending_node = NULL;
+    int32_t command = 0;
+    int32_t force = 0;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, -ret, GD_MSG_DICT_GET_FAILED,
+               "Unable to get count");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &command);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, -ret, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "force", SLEN("force"), &force);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "force flag is not set");
+        ret = 0;
+        goto out;
+    }
 
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
 
-        ret = dict_get_str (dict, "volname", &volname);
+        ret = dict_get_strn(dict, key, keylen, &brick);
         if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "volume name get failed");
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get brick");
+            goto out;
         }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exists",
-                          volname);
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
 
-                *op_errstr = gf_strdup (msg);
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "%s", msg);
-                goto out;
-        }
+        if (ret)
+            goto out;
 
-        ret = dict_get_int32 (dict, "op", &stats_op);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "volume profile op get failed");
+        if (glusterd_is_brick_started(brickinfo)) {
+            pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                     gf_gld_mt_pending_node_t);
+            if (!pending_node) {
+                ret = -1;
                 goto out;
+            } else {
+                pending_node->node = brickinfo;
+                pending_node->type = GD_NODE_BRICK;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
+            }
+            /*
+             * This is not really the right place to do it, but
+             * it's the most convenient.
+             * TBD: move this to *after* the RPC
+             */
+            brickinfo->status = GF_BRICK_STOPPED;
         }
+        i++;
+    }
+
+out:
+    return ret;
+}
 
-        switch (stats_op) {
+static int
+glusterd_bricks_select_profile_volume(dict_t *dict, char **op_errstr,
+                                      struct cds_list_head *selected)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int32_t stats_op = GF_CLI_STATS_NONE;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+    char *brick = NULL;
+    int32_t pid = -1;
+    char pidfile[PATH_MAX] = {0};
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exists", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &stats_op);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume profile op get failed");
+        goto out;
+    }
+
+    switch (stats_op) {
         case GF_CLI_STATS_START:
         case GF_CLI_STATS_STOP:
-                goto out;
-                break;
+            goto out;
+            break;
         case GF_CLI_STATS_INFO:
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
-                if (ret) {
-                        if (!priv->nfs_svc.online) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_NFS_SERVER_NOT_RUNNING,
-                                        "NFS server"
-                                        " is not running");
-                                goto out;
-                        }
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        }
-                        pending_node->node = &(priv->nfs_svc);
-                        pending_node->type = GD_NODE_NFS;
-                        cds_list_add_tail (&pending_node->list, selected);
-                        pending_node = NULL;
+#ifdef BUILD_GNFS
+            ret = dict_get_str_boolean(dict, "nfs", _gf_false);
+            if (ret) {
+                if (!priv->nfs_svc.online) {
+                    ret = -1;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_NFS_SERVER_NOT_RUNNING,
+                           "NFS server"
+                           " is not running");
+                    goto out;
+                }
+                pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                         gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                    ret = -1;
+                    goto out;
+                }
+                pending_node->node = &(priv->nfs_svc);
+                pending_node->type = GD_NODE_NFS;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
 
-                        ret = 0;
+                ret = 0;
+                goto out;
+            }
+#endif
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (glusterd_is_brick_started(brickinfo)) {
+                    /*
+                     * In normal use, glusterd_is_brick_started
+                     * will give us the answer we need.  However,
+                     * in our tests the brick gets detached behind
+                     * our back, so we need to double-check this
+                     * way.
+                     */
+                    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo,
+                                               priv);
+                    if (!gf_is_service_running(pidfile, &pid)) {
+                        continue;
+                    }
+                    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                             gf_gld_mt_pending_node_t);
+                    if (!pending_node) {
+                        ret = -1;
                         goto out;
-
-                }
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (glusterd_is_brick_started (brickinfo)) {
-                                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                          gf_gld_mt_pending_node_t);
-                                if (!pending_node) {
-                                        ret = -1;
-                                        goto out;
-                                } else {
-                                        pending_node->node = brickinfo;
-                                        pending_node->type = GD_NODE_BRICK;
-                                        cds_list_add_tail (&pending_node->list,
-                                                           selected);
-                                        pending_node = NULL;
-                                }
-                        }
+                    } else {
+                        pending_node->node = brickinfo;
+                        pending_node->type = GD_NODE_BRICK;
+                        cds_list_add_tail(&pending_node->list, selected);
+                        pending_node = NULL;
+                    }
                 }
-                break;
+            }
+            break;
 
         case GF_CLI_STATS_TOP:
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
-                if (ret) {
-                        if (!priv->nfs_svc.online) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_NFS_SERVER_NOT_RUNNING,
-                                        "NFS server"
-                                        " is not running");
-                                goto out;
-                        }
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        }
-                        pending_node->node = &(priv->nfs_svc);
-                        pending_node->type = GD_NODE_NFS;
-                        cds_list_add_tail (&pending_node->list, selected);
-                        pending_node = NULL;
-
-                        ret = 0;
-                        goto out;
-
-                }
-                ret = dict_get_str (dict, "brick", &brick);
-                if (!ret) {
-                        ret = glusterd_volume_brickinfo_get_by_brick
-                                                (brick, volinfo, &brickinfo,
-                                                 _gf_true);
-                        if (ret)
-                                goto out;
-
-                        if (!glusterd_is_brick_started (brickinfo))
-                                goto out;
-
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        } else {
-                                pending_node->node = brickinfo;
-                                pending_node->type = GD_NODE_BRICK;
-                                cds_list_add_tail (&pending_node->list,
-                                                   selected);
-                                pending_node = NULL;
-                                goto out;
-                        }
+#ifdef BUILD_GNFS
+            ret = dict_get_str_boolean(dict, "nfs", _gf_false);
+            if (ret) {
+                if (!priv->nfs_svc.online) {
+                    ret = -1;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_NFS_SERVER_NOT_RUNNING,
+                           "NFS server"
+                           " is not running");
+                    goto out;
+                }
+                pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                         gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                    ret = -1;
+                    goto out;
                 }
+                pending_node->node = &(priv->nfs_svc);
+                pending_node->type = GD_NODE_NFS;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
+
                 ret = 0;
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (glusterd_is_brick_started (brickinfo)) {
-                                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                          gf_gld_mt_pending_node_t);
-                                if (!pending_node) {
-                                        ret = -1;
-                                        goto out;
-                                } else {
-                                        pending_node->node = brickinfo;
-                                        pending_node->type = GD_NODE_BRICK;
-                                        cds_list_add_tail (&pending_node->list,
-                                                           selected);
-                                        pending_node = NULL;
-                                }
-                        }
+                goto out;
+            }
+#endif
+            ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick);
+            if (!ret) {
+                ret = glusterd_volume_brickinfo_get_by_brick(
+                    brick, volinfo, &brickinfo, _gf_true);
+                if (ret)
+                    goto out;
+
+                if (!glusterd_is_brick_started(brickinfo))
+                    goto out;
+
+                pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                         gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                    ret = -1;
+                    goto out;
+                } else {
+                    pending_node->node = brickinfo;
+                    pending_node->type = GD_NODE_BRICK;
+                    cds_list_add_tail(&pending_node->list, selected);
+                    pending_node = NULL;
+                    goto out;
+                }
+            }
+            ret = 0;
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (glusterd_is_brick_started(brickinfo)) {
+                    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                             gf_gld_mt_pending_node_t);
+                    if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                    } else {
+                        pending_node->node = brickinfo;
+                        pending_node->type = GD_NODE_BRICK;
+                        cds_list_add_tail(&pending_node->list, selected);
+                        pending_node = NULL;
+                    }
                 }
-                break;
+            }
+            break;
 
         default:
-                GF_ASSERT (0);
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "Invalid profile op: %d",
-                        stats_op);
-                ret = -1;
-                goto out;
-                break;
-        }
-
+            GF_ASSERT(0);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Invalid profile op: %d", stats_op);
+            ret = -1;
+            goto out;
+            break;
+    }
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-_get_hxl_children_count (glusterd_volinfo_t *volinfo)
+_get_hxl_children_count(glusterd_volinfo_t *volinfo)
 {
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                return volinfo->disperse_count;
-        } else {
-                return volinfo->replica_count;
-        }
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        return volinfo->disperse_count;
+    } else {
+        return volinfo->replica_count;
+    }
 }
 
 static int
-_add_hxlator_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int index,
-                      int count)
+_add_hxlator_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo, int index,
+                     int count)
 {
-        int     ret             = -1;
-        char    key[128]        = {0,};
-        char    *xname          = NULL;
-        char    *xl_type        = 0;
-
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                xl_type = "disperse";
-        } else {
-                xl_type = "replicate";
-        }
-        snprintf (key, sizeof (key), "xl-%d", count);
-        ret = gf_asprintf (&xname, "%s-%s-%d", volinfo->volname, xl_type,
-                           index);
-        if (ret == -1)
-                goto out;
-
-        ret = dict_set_dynstr (dict, key, xname);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (dict, xname, index);
+    int ret = -1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *xname = NULL;
+    char *xl_type = 0;
+
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        xl_type = "disperse";
+    } else {
+        xl_type = "replicate";
+    }
+    keylen = snprintf(key, sizeof(key), "xl-%d", count);
+    ret = gf_asprintf(&xname, "%s-%s-%d", volinfo->volname, xl_type, index);
+    if (ret == -1)
+        goto out;
+
+    ret = dict_set_dynstrn(dict, key, keylen, xname);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, xname, index);
 out:
-        return ret;
+    return ret;
 }
 
 int
-get_replica_index_for_per_replica_cmd (glusterd_volinfo_t *volinfo,
-                                       dict_t *dict)
+get_replica_index_for_per_replica_cmd(glusterd_volinfo_t *volinfo, dict_t *dict)
 {
-        int                     ret = 0;
-        char                    *hostname = NULL;
-        char                    *path = NULL;
-        int                     index = 0;
-        glusterd_brickinfo_t   *brickinfo = NULL;
-        int                     cmd_replica_index = -1;
-        int                     replica_count = -1;
-
-
-        if (!dict)  {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "per-replica-cmd-hostname", &hostname);
-        if (ret)
-                goto out;
-        ret = dict_get_str (dict, "per-replica-cmd-path", &path);
-        if (ret)
-                goto out;
-
-        replica_count = volinfo->replica_count;
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
-                if (!strcmp (brickinfo->path,  path) &&
-                    !strcmp (brickinfo->hostname, hostname)) {
-                        cmd_replica_index = index/(replica_count);
-                        goto out;
-                }
-                index++;
-        }
-
+    int ret = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int index = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int cmd_replica_index = -1;
+    int replica_count = -1;
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "per-replica-cmd-hostname",
+                        SLEN("per-replica-cmd-hostname"), &hostname);
+    if (ret)
+        goto out;
+    ret = dict_get_strn(dict, "per-replica-cmd-path",
+                        SLEN("per-replica-cmd-path"), &path);
+    if (ret)
+        goto out;
+
+    replica_count = volinfo->replica_count;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+        if (!strcmp(brickinfo->path, path) &&
+            !strcmp(brickinfo->hostname, hostname)) {
+            cmd_replica_index = index / (replica_count);
+            goto out;
+        }
+        index++;
+    }
 
 out:
-        if (ret)
-                cmd_replica_index = -1;
+    if (ret)
+        cmd_replica_index = -1;
 
-        return cmd_replica_index;
+    return cmd_replica_index;
 }
 
 int
-_select_hxlator_with_matching_brick (xlator_t *this,
-                                     glusterd_volinfo_t *volinfo, dict_t *dict,
-                                     int *index)
+_select_hxlator_with_matching_brick(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                    dict_t *dict, int *index)
 {
-        char                    *hostname = NULL;
-        char                    *path = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int                     hxl_children = 0;
-
-        priv = this->private;
-        if (!dict ||
-            dict_get_str (dict, "per-replica-cmd-hostname", &hostname) ||
-            dict_get_str (dict, "per-replica-cmd-path", &path))
-                return -1;
-
-        hxl_children = _get_hxl_children_count (volinfo);
-        if ((*index) == 0)
-                (*index)++;
+    char *path = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int hxl_children = 0;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
+    if (!dict || dict_get_strn(dict, "per-replica-cmd-path",
+                               SLEN("per-replica-cmd-path"), &path))
+        return -1;
 
-                if (!gf_uuid_compare (MY_UUID, brickinfo->uuid)) {
-                        _add_hxlator_to_dict (dict, volinfo,
-                                              ((*index) - 1)/hxl_children, 0);
-                        return 1;
-                }
-                (*index)++;
+    hxl_children = _get_hxl_children_count(volinfo);
+    if ((*index) == 0)
+        (*index)++;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+
+        if ((!gf_uuid_compare(MY_UUID, brickinfo->uuid)) &&
+            (!strncmp(brickinfo->path, path, strlen(path)))) {
+            _add_hxlator_to_dict(dict, volinfo, ((*index) - 1) / hxl_children,
+                                 0);
+            return 1;
         }
+        (*index)++;
+    }
 
-        return 0;
+    return 0;
 }
 void
-_select_hxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
-                                    dict_t *dict, int *index,
-                                    int *hxlator_count)
+_select_hxlators_with_local_bricks(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                   dict_t *dict, int *index, int *hxlator_count)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int                     hxl_children = 0;
-        gf_boolean_t            add     = _gf_false;
-        int                     cmd_replica_index = -1;
-
-        priv = this->private;
-        hxl_children = _get_hxl_children_count (volinfo);
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int hxl_children = 0;
+    gf_boolean_t add = _gf_false;
 
-        if ((*index) == 0)
-                (*index)++;
+    hxl_children = _get_hxl_children_count(volinfo);
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
+    if ((*index) == 0)
+        (*index)++;
 
-                if (!gf_uuid_compare (MY_UUID, brickinfo->uuid))
-                        add = _gf_true;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
 
-                if ((*index) % hxl_children == 0) {
-                        if (add) {
-                              _add_hxlator_to_dict (dict, volinfo,
-                                                    ((*index) - 1)/hxl_children,
-                                                    (*hxlator_count));
-                                (*hxlator_count)++;
-                        }
-                        add = _gf_false;
-                }
+        if (!gf_uuid_compare(MY_UUID, brickinfo->uuid))
+            add = _gf_true;
 
-                (*index)++;
+        if ((*index) % hxl_children == 0) {
+            if (add) {
+                _add_hxlator_to_dict(dict, volinfo,
+                                     ((*index) - 1) / hxl_children,
+                                     (*hxlator_count));
+                (*hxlator_count)++;
+            }
+            add = _gf_false;
         }
 
+        (*index)++;
+    }
 }
 
 int
-_select_hxlators_for_full_self_heal (xlator_t *this,
-                                     glusterd_volinfo_t *volinfo,
-                                     dict_t *dict, int *index,
-                                     int *hxlator_count)
+_select_hxlators_for_full_self_heal(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                    dict_t *dict, int *index,
+                                    int *hxlator_count)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int                     hxl_children = 0;
-        uuid_t                  candidate = {0};
-
-        priv = this->private;
-        if ((*index) == 0)
-                (*index)++;
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                hxl_children = volinfo->disperse_count;
-        } else {
-                hxl_children = volinfo->replica_count;
-        }
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
-
-                if (gf_uuid_compare (brickinfo->uuid, candidate) > 0)
-                        gf_uuid_copy (candidate, brickinfo->uuid);
-
-                if ((*index) % hxl_children == 0) {
-                        if (!gf_uuid_compare (MY_UUID, candidate)) {
-                                _add_hxlator_to_dict (dict, volinfo,
-                                                      ((*index)-1)/hxl_children,
-                                                      (*hxlator_count));
-                                (*hxlator_count)++;
-                        }
-                        gf_uuid_clear (candidate);
-                }
-
-                (*index)++;
-        }
-        return *hxlator_count;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int hxl_children = 0;
+    uuid_t candidate = {0};
+    int brick_index = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int delta = 0;
+    uuid_t candidate_max = {0};
+
+    if ((*index) == 0)
+        (*index)++;
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        hxl_children = volinfo->disperse_count;
+    } else {
+        hxl_children = volinfo->replica_count;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, candidate_max) > 0) {
+            if (!gf_uuid_compare(MY_UUID, brickinfo->uuid)) {
+                gf_uuid_copy(candidate_max, brickinfo->uuid);
+            } else {
+                peerinfo = glusterd_peerinfo_find(brickinfo->uuid, NULL);
+                if (peerinfo && peerinfo->connected) {
+                    gf_uuid_copy(candidate_max, brickinfo->uuid);
+                }
+            }
+        }
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+
+        delta %= hxl_children;
+        if ((*index + delta) == (brick_index + hxl_children)) {
+            if (!gf_uuid_compare(MY_UUID, brickinfo->uuid)) {
+                gf_uuid_copy(candidate, brickinfo->uuid);
+            } else {
+                peerinfo = glusterd_peerinfo_find(brickinfo->uuid, NULL);
+                if (peerinfo && peerinfo->connected) {
+                    gf_uuid_copy(candidate, brickinfo->uuid);
+                } else if (peerinfo &&
+                           (!gf_uuid_compare(candidate_max, MY_UUID))) {
+                    _add_hxlator_to_dict(dict, volinfo,
+                                         ((*index) - 1) / hxl_children,
+                                         (*hxlator_count));
+                    (*hxlator_count)++;
+                }
+            }
+
+            if (!gf_uuid_compare(MY_UUID, candidate)) {
+                _add_hxlator_to_dict(dict, volinfo,
+                                     ((*index) - 1) / hxl_children,
+                                     (*hxlator_count));
+                (*hxlator_count)++;
+            }
+            gf_uuid_clear(candidate);
+            brick_index += hxl_children;
+            delta++;
+        }
+
+        (*index)++;
+    }
+    return *hxlator_count;
 }
 
-
 static int
-glusterd_bricks_select_snap (dict_t *dict, char **op_errstr,
-                             struct cds_list_head *selected)
+glusterd_bricks_select_snap(dict_t *dict, char **op_errstr,
+                            struct cds_list_head *selected)
 {
-        int                        ret           = -1;
-        glusterd_conf_t            *priv         = NULL;
-        xlator_t                   *this         = NULL;
-        glusterd_pending_node_t    *pending_node = NULL;
-        glusterd_volinfo_t         *volinfo      = NULL;
-        char                       *volname      = NULL;
-        glusterd_brickinfo_t       *brickinfo    = NULL;
-        int                         brick_index  = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get"
-                        " volname");
-                goto out;
-        }
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret)
-                goto out;
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                brick_index++;
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID) ||
-                    !glusterd_is_brick_started (brickinfo)) {
-                        continue;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = brickinfo;
-                pending_node->type = GD_NODE_BRICK;
-                pending_node->index = brick_index;
-                cds_list_add_tail (&pending_node->list, selected);
-                pending_node = NULL;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int brick_index = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get"
+               " volname");
+        goto out;
+    }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret)
+        goto out;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        brick_index++;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+            !glusterd_is_brick_started(brickinfo)) {
+            continue;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
         }
+        pending_node->node = brickinfo;
+        pending_node->type = GD_NODE_BRICK;
+        pending_node->index = brick_index;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning ret %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning ret %d", ret);
+    return ret;
 }
 
 static int
-fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo,
-                                  cli_cmd_type type, int *index,
-                                  dict_t *req_dict)
+fill_shd_status_for_local_bricks(dict_t *dict, glusterd_volinfo_t *volinfo,
+                                 cli_cmd_type type, int *index,
+                                 dict_t *req_dict)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        char                    msg[1024] = {0,};
-        char                    key[1024]  = {0,};
-        char                    value[1024] = {0,};
-        int                     ret = 0;
-        xlator_t               *this = NULL;
-        int                     cmd_replica_index = -1;
-
-        this = THIS;
-        snprintf (msg, sizeof (msg), "self-heal-daemon is not running on");
-
-        if (type == PER_HEAL_XL) {
-                cmd_replica_index = get_replica_index_for_per_replica_cmd
-                                    (volinfo, req_dict);
-                if (cmd_replica_index == -1) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_REPLICA_INDEX_GET_FAIL,
-                                "Could not find the "
-                                "replica index for per replica type command");
-                        ret = -1;
-                        goto out;
-                }
+    glusterd_brickinfo_t *brickinfo = NULL;
+    static char *msg = "self-heal-daemon is not running on";
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char value[128] = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    int cmd_replica_index = -1;
+
+    this = THIS;
+
+    if (type == PER_HEAL_XL) {
+        cmd_replica_index = get_replica_index_for_per_replica_cmd(volinfo,
+                                                                  req_dict);
+        if (cmd_replica_index == -1) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_REPLICA_INDEX_GET_FAIL,
+                   "Could not find the "
+                   "replica index for per replica type command");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+
+        if (gf_uuid_compare(MY_UUID, brickinfo->uuid)) {
+            (*index)++;
+            continue;
         }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
-
-                if (gf_uuid_compare (MY_UUID, brickinfo->uuid)) {
-                        (*index)++;
-                        continue;
-                }
-
-                if (type == PER_HEAL_XL) {
-                   if (cmd_replica_index != ((*index)/volinfo->replica_count)) {
-                              (*index)++;
-                              continue;
-                   }
-
-                }
-                snprintf (key, sizeof (key), "%d-status", (*index));
-                snprintf (value, sizeof (value), "%s %s",msg,
-                          uuid_utoa(MY_UUID));
-                ret = dict_set_dynstr (dict, key, gf_strdup(value));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Unable to"
-                                "set the dictionary for shd status msg");
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-shd-status", (*index));
-                ret = dict_set_str (dict, key, "off");
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Unable to"
-                                " set dictionary for shd status msg");
-                        goto out;
-                }
-
+        if (type == PER_HEAL_XL) {
+            if (cmd_replica_index != ((*index) / volinfo->replica_count)) {
                 (*index)++;
+                continue;
+            }
+        }
+        keylen = snprintf(key, sizeof(key), "%d-status", (*index));
+        snprintf(value, sizeof(value), "%s %s", msg, uuid_utoa(MY_UUID));
+        ret = dict_set_dynstrn(dict, key, keylen, gf_strdup(value));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to"
+                   "set the dictionary for shd status msg");
+            goto out;
+        }
+        keylen = snprintf(key, sizeof(key), "%d-shd-status", (*index));
+        ret = dict_set_nstrn(dict, key, keylen, "off", SLEN("off"));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to"
+                   " set dictionary for shd status msg");
+            goto out;
         }
 
-out:
-        return ret;
+        (*index)++;
+    }
 
+out:
+    return ret;
 }
 int
-glusterd_shd_select_brick_xlator (dict_t *dict, gf_xl_afr_op_t heal_op,
-                                  glusterd_volinfo_t *volinfo, int *index,
-                                  int *hxlator_count, dict_t *rsp_dict)
+glusterd_shd_select_brick_xlator(dict_t *dict, gf_xl_afr_op_t heal_op,
+                                 glusterd_volinfo_t *volinfo, int *index,
+                                 int *hxlator_count, dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        glusterd_conf_t                         *priv = NULL;
-        xlator_t                                *this = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_pending_node_t                 *pending_node = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-
-        switch (heal_op) {
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    svc = &(volinfo->shd.svc);
+
+    switch (heal_op) {
         case GF_SHD_OP_INDEX_SUMMARY:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
-        if (!priv->shd_svc.online) {
+            if (!svc->online) {
                 if (!rsp_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OPCTX_NULL, "Received "
-                                "empty ctx.");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_NULL,
+                           "Received "
+                           "empty ctx.");
+                    goto out;
                 }
 
-                ret = fill_shd_status_for_local_bricks (rsp_dict,
-                                                        volinfo,
-                                                        ALL_HEAL_XL,
-                                                        index,
-                                                        dict);
+                ret = fill_shd_status_for_local_bricks(
+                    rsp_dict, volinfo, ALL_HEAL_XL, index, dict);
                 if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SHD_STATUS_SET_FAIL, "Unable to "
-                                "fill the shd status for the local "
-                                "bricks");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SHD_STATUS_SET_FAIL,
+                           "Unable to "
+                           "fill the shd status for the local "
+                           "bricks");
                 goto out;
-        }
-        break;
+            }
+            break;
 
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-        if (!priv->shd_svc.online) {
+            if (!svc->online) {
                 if (!rsp_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OPCTX_NULL, "Received "
-                                "empty ctx.");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_NULL,
+                           "Received "
+                           "empty ctx.");
+                    goto out;
                 }
-                ret = fill_shd_status_for_local_bricks (rsp_dict,
-                                                        volinfo,
-                                                        PER_HEAL_XL,
-                                                        index,
-                                                        dict);
+                ret = fill_shd_status_for_local_bricks(
+                    rsp_dict, volinfo, PER_HEAL_XL, index, dict);
                 if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SHD_STATUS_SET_FAIL, "Unable to "
-                                "fill the shd status for the local"
-                                " bricks.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SHD_STATUS_SET_FAIL,
+                           "Unable to "
+                           "fill the shd status for the local"
+                           " bricks.");
                 goto out;
-
-        }
-        break;
+            }
+            break;
 
         default:
-                break;
-        }
-
+            break;
+    }
 
-        switch (heal_op) {
+    switch (heal_op) {
         case GF_SHD_OP_HEAL_FULL:
-                _select_hxlators_for_full_self_heal (this, volinfo, dict,
-                                                     index, hxlator_count);
-                break;
+            _select_hxlators_for_full_self_heal(this, volinfo, dict, index,
+                                                hxlator_count);
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                 (*hxlator_count) += _select_hxlator_with_matching_brick (this,
-                                                                       volinfo,
-                                                                       dict,
-                                                                       index);
-                break;
+            (*hxlator_count) += _select_hxlator_with_matching_brick(
+                this, volinfo, dict, index);
+            break;
         default:
-                 _select_hxlators_with_local_bricks (this, volinfo, dict,
-                                                      index, hxlator_count);
-                break;
-        }
-        ret = (*hxlator_count);
+            _select_hxlators_with_local_bricks(this, volinfo, dict, index,
+                                               hxlator_count);
+            break;
+    }
+    ret = (*hxlator_count);
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr,
-                                    struct cds_list_head *selected,
-                                    dict_t *rsp_dict)
+glusterd_bricks_select_heal_volume(dict_t *dict, char **op_errstr,
+                                   struct cds_list_head *selected,
+                                   dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_volinfo_t                      *dup_volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_pending_node_t                 *pending_node = NULL;
-        gf_xl_afr_op_t                          heal_op = GF_SHD_OP_INVALID;
-        int                                     hxlator_count = 0;
-        int                                     index = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "volume name get failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-
-                *op_errstr = gf_strdup (msg);
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "%s", msg);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "heal-op", (int32_t *)&heal_op);
-        if (ret || (heal_op == GF_SHD_OP_INVALID)) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "heal op invalid");
-                goto out;
-        }
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                ret = glusterd_create_sub_tier_volinfo (volinfo, &dup_volinfo,
-                                                         _gf_false, volname);
-                if (ret < 0)
-                        goto out;
-
-                ret = glusterd_shd_select_brick_xlator (dict, heal_op,
-                                                        dup_volinfo,
-                                                        &index, &hxlator_count,
-                                                        rsp_dict);
-                glusterd_volinfo_delete (dup_volinfo);
-                if (ret < 0)
-                        goto out;
-                ret = glusterd_create_sub_tier_volinfo (volinfo, &dup_volinfo,
-                                                        _gf_true, volname);
-                if (ret < 0)
-                        goto out;
-                ret = glusterd_shd_select_brick_xlator (dict, heal_op,
-                                                        dup_volinfo,
-                                                        &index, &hxlator_count,
-                                                        rsp_dict);
-                glusterd_volinfo_delete (dup_volinfo);
-                if (ret < 0)
-                        goto out;
-        } else {
-                ret = glusterd_shd_select_brick_xlator (dict, heal_op,
-                                                        volinfo,
-                                                        &index, &hxlator_count,
-                                                        rsp_dict);
-                if (ret < 0)
-                        goto out;
-        }
-
-        if (!hxlator_count)
-                goto out;
-        if (hxlator_count == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_XLATOR_COUNT_GET_FAIL, "Could not determine the"
-                        "translator count");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_int32 (dict, "count", hxlator_count);
-        if (ret)
-                goto out;
-        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                  gf_gld_mt_pending_node_t);
-        if (!pending_node) {
-                ret = -1;
-                goto out;
-        } else {
-                pending_node->node = &(priv->shd_svc);
-                pending_node->type = GD_NODE_SHD;
-                cds_list_add_tail (&pending_node->list, selected);
-                pending_node = NULL;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_pending_node_t *pending_node = NULL;
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    int hxlator_count = 0;
+    int index = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"),
+                          (int32_t *)&heal_op);
+    if (ret || (heal_op == GF_SHD_OP_INVALID)) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "heal op invalid");
+        goto out;
+    }
+    ret = glusterd_shd_select_brick_xlator(dict, heal_op, volinfo, &index,
+                                           &hxlator_count, rsp_dict);
+    if (ret < 0) {
+        goto out;
+    }
+
+    if (!hxlator_count)
+        goto out;
+    if (hxlator_count == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_XLATOR_COUNT_GET_FAIL,
+               "Could not determine the"
+               "translator count");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32n(dict, "count", SLEN("count"), hxlator_count);
+    if (ret)
+        goto out;
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        goto out;
+    } else {
+        pending_node->node = &(volinfo->shd.svc);
+        pending_node->type = GD_NODE_SHD;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning ret %d", ret);
-        return ret;
-
+    gf_msg_debug(THIS->name, 0, "Returning ret %d", ret);
+    return ret;
 }
 
-int
-glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
-                                         struct cds_list_head *selected)
+static int
+glusterd_bricks_select_rebalance_volume(dict_t *dict, char **op_errstr,
+                                        struct cds_list_head *selected)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_pending_node_t                 *pending_node = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "volume name get failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-
-                *op_errstr = gf_strdup (msg);
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "%s", msg);
-                goto out;
-        }
-        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                  gf_gld_mt_pending_node_t);
-        if (!pending_node) {
-                ret = -1;
-                goto out;
-        } else {
-                pending_node->node = volinfo;
-                pending_node->type = GD_NODE_REBALANCE;
-                cds_list_add_tail (&pending_node->list, selected);
-                pending_node = NULL;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_pending_node_t *pending_node = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        goto out;
+    } else {
+        pending_node->node = volinfo;
+        pending_node->type = GD_NODE_REBALANCE;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr,
-                                      struct cds_list_head *selected)
+glusterd_bricks_select_status_volume(dict_t *dict, char **op_errstr,
+                                     struct cds_list_head *selected)
 {
-        int                     ret = -1;
-        int                     cmd = 0;
-        int                     brick_index = -1;
-        char                    *volname = NULL;
-        char                    *brickname = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_pending_node_t *pending_node = NULL;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_snapdsvc_t     *snapd = NULL;
-
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int32 (dict, "cmd", &cmd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get status type");
-                goto out;
-        }
-
-        if (cmd & GF_CLI_STATUS_ALL)
-                goto out;
-
-        switch (cmd & GF_CLI_STATUS_MASK) {
+    int ret = -1;
+    int cmd = 0;
+    int brick_index = -1;
+    char *volname = NULL;
+    char *brickname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_ASSERT(dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int32n(dict, "cmd", SLEN("cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get status type");
+        goto out;
+    }
+
+    if (cmd & GF_CLI_STATUS_ALL)
+        goto out;
+
+    switch (cmd & GF_CLI_STATUS_MASK) {
         case GF_CLI_STATUS_MEM:
         case GF_CLI_STATUS_CLIENTS:
         case GF_CLI_STATUS_INODE:
@@ -6787,1091 +7062,1103 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr,
         case GF_CLI_STATUS_SNAPD:
         case GF_CLI_STATUS_BITD:
         case GF_CLI_STATUS_SCRUB:
-                break;
+        case GF_CLI_STATUS_CLIENT_LIST:
+            break;
         default:
-                goto out;
-        }
-        ret = dict_get_str (dict, "volname", &volname);
+            goto out;
+    }
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volname");
+        goto out;
+    }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        ret = dict_get_strn(dict, "brick", SLEN("brick"), &brickname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volname");
-                goto out;
-        }
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get brick");
+            goto out;
         }
+        ret = glusterd_volume_brickinfo_get_by_brick(brickname, volinfo,
+                                                     &brickinfo, _gf_false);
+        if (ret)
+            goto out;
 
-        if ( (cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brickname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get brick");
-                        goto out;
-                }
-                ret = glusterd_volume_brickinfo_get_by_brick (brickname,
-                                                              volinfo,
-                                                              &brickinfo,
-                                                              _gf_false);
-                if (ret)
-                        goto out;
-
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID)||
-                    !glusterd_is_brick_started (brickinfo))
-                        goto out;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+            !glusterd_is_brick_started(brickinfo))
+            goto out;
 
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = brickinfo;
-                pending_node->type = GD_NODE_BRICK;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = brickinfo;
+        pending_node->type = GD_NODE_BRICK;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                if (!priv->nfs_svc.online) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_NFS_SERVER_NOT_RUNNING,
-                                "NFS server is not running");
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->nfs_svc);
-                pending_node->type = GD_NODE_NFS;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+#ifdef BUILD_GNFS
+    } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+        if (!priv->nfs_svc.online) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NFS_SERVER_NOT_RUNNING,
+                   "NFS server is not running");
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->nfs_svc);
+        pending_node->type = GD_NODE_NFS;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                if (!priv->shd_svc.online) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SELF_HEALD_DISABLED,
-                                "Self-heal daemon is not running");
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->shd_svc);
-                pending_node->type = GD_NODE_SHD;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+#endif
+    } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+        svc = &(volinfo->shd.svc);
+        if (!svc->online) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SELF_HEALD_DISABLED,
+                   "Self-heal daemon is not running");
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = svc;
+        pending_node->type = GD_NODE_SHD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
-                if (!priv->quotad_svc.online) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_QUOTAD_NOT_RUNNING, "Quotad is not "
-                                "running");
-                        ret = -1;
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->quotad_svc);
-                pending_node->type = GD_NODE_QUOTAD;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+        if (!priv->quotad_svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTAD_NOT_RUNNING,
+                   "Quotad is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->quotad_svc);
+        pending_node->type = GD_NODE_QUOTAD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
-                if (!priv->bitd_svc.online) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BITROT_NOT_RUNNING, "Bitrot is not "
-                                "running");
-                        ret = -1;
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->bitd_svc);
-                pending_node->type = GD_NODE_BITD;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
+        if (!priv->bitd_svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITROT_NOT_RUNNING,
+                   "Bitrot is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->bitd_svc);
+        pending_node->type = GD_NODE_BITD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
-                if (!priv->scrub_svc.online) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SCRUBBER_NOT_RUNNING, "Scrubber is not "
-                                "running");
-                        ret = -1;
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->scrub_svc);
-                pending_node->type = GD_NODE_SCRUB;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
+        if (!priv->scrub_svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBBER_NOT_RUNNING,
+                   "Scrubber is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->scrub_svc);
+        pending_node->type = GD_NODE_SCRUB;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
-                if (!volinfo->snapd.svc.online) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPD_NOT_RUNNING, "snapd is not "
-                                "running");
-                        ret = -1;
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY, "failed to allocate "
-                                "memory for pending node");
-                        ret = -1;
-                        goto out;
-                }
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
+        if (!volinfo->snapd.svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_NOT_RUNNING,
+                   "snapd is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "failed to allocate "
+                   "memory for pending node");
+            ret = -1;
+            goto out;
+        }
 
-                pending_node->node = (void *)(&volinfo->snapd);
-                pending_node->type = GD_NODE_SNAPD;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        pending_node->node = (void *)(&volinfo->snapd);
+        pending_node->type = GD_NODE_SNAPD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        brick_index++;
-                        if (gf_uuid_compare (brickinfo->uuid, MY_UUID) ||
-                            !glusterd_is_brick_started (brickinfo)) {
-                                continue;
-                        }
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
-                                        GD_MSG_NO_MEMORY,
-                                        "Unable to allocate memory");
-                                goto out;
-                        }
-                        pending_node->node = brickinfo;
-                        pending_node->type = GD_NODE_BRICK;
-                        pending_node->index = brick_index;
-                        cds_list_add_tail (&pending_node->list, selected);
-                        pending_node = NULL;
-                }
+        ret = 0;
+    } else {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            brick_index++;
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+                !glusterd_is_brick_started(brickinfo)) {
+                continue;
+            }
+            pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                     gf_gld_mt_pending_node_t);
+            if (!pending_node) {
+                ret = -1;
+                gf_msg(THIS->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Unable to allocate memory");
+                goto out;
+            }
+            pending_node->node = brickinfo;
+            pending_node->type = GD_NODE_BRICK;
+            pending_node->index = brick_index;
+            cds_list_add_tail(&pending_node->list, selected);
+            pending_node = NULL;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bricks_select_scrub (dict_t *dict, char **op_errstr,
-                              struct cds_list_head *selected)
+glusterd_bricks_select_scrub(dict_t *dict, char **op_errstr,
+                             struct cds_list_head *selected)
 {
-        int                       ret           = -1;
-        char                      *volname      = NULL;
-        char                      msg[2048]     = {0,};
-        xlator_t                  *this         = NULL;
-        glusterd_conf_t           *priv         = NULL;
-        glusterd_volinfo_t        *volinfo      = NULL;
-        glusterd_brickinfo_t      *brickinfo    = NULL;
-        glusterd_pending_node_t   *pending_node = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (this);
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get"
-                        " volname");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-
-                *op_errstr = gf_strdup (msg);
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "%s", msg);
-                goto out;
-        }
-
-        if (!priv->scrub_svc.online) {
-                ret = 0;
-                snprintf (msg, sizeof (msg), "Scrubber daemon is not running");
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(this);
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get"
+               " volname");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+               msg);
+        goto out;
+    }
+
+    if (!priv->scrub_svc.online) {
+        ret = 0;
+        snprintf(msg, sizeof(msg), "Scrubber daemon is not running");
 
-                gf_msg_debug (this->name, 0, "%s", msg);
-                goto out;
-        }
+        gf_msg_debug(this->name, 0, "%s", msg);
+        goto out;
+    }
 
-        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                  gf_gld_mt_pending_node_t);
-        if (!pending_node) {
-                ret = -1;
-                goto out;
-        }
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        goto out;
+    }
 
-        pending_node->node = &(priv->scrub_svc);
-        pending_node->type = GD_NODE_SCRUB;
-        cds_list_add_tail (&pending_node->list, selected);
-        pending_node = NULL;
+    pending_node->node = &(priv->scrub_svc);
+    pending_node->type = GD_NODE_SCRUB;
+    cds_list_add_tail(&pending_node->list, selected);
+    pending_node = NULL;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 /* Select the bricks to send the barrier request to.
  * This selects the bricks of the given volume which are present on this peer
  * and are running
  */
 static int
-glusterd_bricks_select_barrier (dict_t *dict, struct cds_list_head *selected)
+glusterd_bricks_select_barrier(dict_t *dict, struct cds_list_head *selected)
 {
-        int                       ret           = -1;
-        char                      *volname      = NULL;
-        glusterd_volinfo_t        *volinfo      = NULL;
-        glusterd_brickinfo_t      *brickinfo    = NULL;
-        glusterd_pending_node_t   *pending_node = NULL;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get volname");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "Failed to find volume %s",
-                        volname);
-                goto out;
-        }
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID) ||
-                    !glusterd_is_brick_started (brickinfo)) {
-                        continue;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = brickinfo;
-                pending_node->type = GD_NODE_BRICK;
-                cds_list_add_tail (&pending_node->list, selected);
-                pending_node = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volname");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Failed to find volume %s", volname);
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+            !glusterd_is_brick_started(brickinfo)) {
+            continue;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
         }
+        pending_node->node = brickinfo;
+        pending_node->type = GD_NODE_BRICK;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_brick_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                             ret = 0;
-        rpc_clnt_procedure_t            *proc = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        xlator_t                        *this = NULL;
-        glusterd_op_t                   op = GD_OP_NONE;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        char                            *op_errstr = NULL;
-
-        this = THIS;
-        priv = this->private;
-
-        if (ctx) {
-                req_ctx = ctx;
-        } else {
-                req_ctx = GF_CALLOC (1, sizeof (*req_ctx),
-                                     gf_gld_mt_op_allack_ctx_t);
-                op = glusterd_op_get_op ();
-                req_ctx->op = op;
-                gf_uuid_copy (req_ctx->uuid, MY_UUID);
-                ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr,
-                                                 NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
-                                LOGSTR_BUILD_PAYLOAD,
-                                gd_op_list[op]);
-                        if (op_errstr == NULL)
-                                gf_asprintf (&op_errstr,
-                                             OPERRSTR_BUILD_PAYLOAD);
-                        opinfo.op_errstr = op_errstr;
-                        goto out;
-                }
-        }
-
-        proc = &priv->gfs_mgmt->proctable[GLUSTERD_BRICK_OP];
-        if (proc->fn) {
-                ret = proc->fn (NULL, this, req_ctx);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    char *op_errstr = NULL;
+    gf_boolean_t free_req_ctx = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+
+    if (ctx) {
+        req_ctx = ctx;
+    } else {
+        req_ctx = GF_CALLOC(1, sizeof(*req_ctx), gf_gld_mt_op_allack_ctx_t);
+        if (!req_ctx)
+            goto out;
+        free_req_ctx = _gf_true;
+        op = glusterd_op_get_op();
+        req_ctx->op = op;
+        gf_uuid_copy(req_ctx->uuid, MY_UUID);
+        ret = glusterd_op_build_payload(&req_ctx->dict, &op_errstr, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL, LOGSTR_BUILD_PAYLOAD,
+                   gd_op_list[op]);
+            if (op_errstr == NULL)
+                gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+            opinfo.op_errstr = op_errstr;
+            goto out;
+        }
+    }
+
+    proc = &priv->gfs_mgmt->proctable[GLUSTERD_BRICK_OP];
+    if (proc->fn) {
+        ret = proc->fn(NULL, this, req_ctx);
+        if (ret)
+            goto out;
+    }
 
-        if (!opinfo.pending_count && !opinfo.brick_pending_count) {
-                glusterd_clear_pending_nodes (&opinfo.pending_bricks);
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                                   &event->txn_id, req_ctx);
-        }
+    if (!opinfo.pending_count && !opinfo.brick_pending_count) {
+        glusterd_clear_pending_nodes(&opinfo.pending_bricks);
+        ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                          req_ctx);
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+    if (ret && free_req_ctx)
+        GF_FREE(req_ctx);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_brick_op_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-        glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
-        char                        *op_errstr = NULL;
-        glusterd_op_t               op = GD_OP_NONE;
-        gd_node_type                type = GD_NODE_NONE;
-        dict_t                      *op_ctx = NULL;
-        glusterd_req_ctx_t          *req_ctx = NULL;
-        void                        *pending_entry = NULL;
-        xlator_t                    *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
-
-        req_ctx = ev_ctx->commit_ctx;
-        GF_ASSERT (req_ctx);
-
-        op = req_ctx->op;
-        op_ctx = glusterd_op_get_ctx ();
-        pending_entry = ev_ctx->pending_node->node;
-        type = ev_ctx->pending_node->type;
-
-        ret = glusterd_remove_pending_entry (&opinfo.pending_bricks,
-                                             pending_entry);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UNKNOWN_RESPONSE, "unknown response received ");
-                ret = -1;
-                goto out;
-        }
-
-        if (opinfo.brick_pending_count > 0)
-                opinfo.brick_pending_count--;
+    int ret = -1;
+    glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+    char *op_errstr = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    gd_node_type type = GD_NODE_NONE;
+    dict_t *op_ctx = NULL;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    void *pending_entry = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, event, out);
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
+    ev_ctx = ctx;
+    GF_VALIDATE_OR_GOTO(this->name, ev_ctx, out);
+
+    req_ctx = ev_ctx->commit_ctx;
+    GF_VALIDATE_OR_GOTO(this->name, req_ctx, out);
+
+    op = req_ctx->op;
+    op_ctx = glusterd_op_get_ctx();
+    pending_entry = ev_ctx->pending_node->node;
+    type = ev_ctx->pending_node->type;
+
+    ret = glusterd_remove_pending_entry(&opinfo.pending_bricks, pending_entry);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_RESPONSE,
+               "unknown response received ");
+        ret = -1;
+        goto out;
+    }
 
+    if (opinfo.brick_pending_count > 0)
+        opinfo.brick_pending_count--;
 
-        ret = glusterd_set_txn_opinfo (&event->txn_id, &opinfo);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
+    glusterd_handle_node_rsp(req_ctx->dict, pending_entry, op, ev_ctx->rsp_dict,
+                             op_ctx, &op_errstr, type);
 
-        glusterd_handle_node_rsp (req_ctx->dict, pending_entry, op, ev_ctx->rsp_dict,
-                                  op_ctx, &op_errstr, type);
-
-        if (opinfo.brick_pending_count > 0)
-                goto out;
+    if (opinfo.brick_pending_count > 0)
+        goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, &event->txn_id,
-                                           ev_ctx->commit_ctx);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      ev_ctx->commit_ctx);
 
 out:
-        if (ev_ctx->rsp_dict)
-                dict_unref (ev_ctx->rsp_dict);
-        GF_FREE (ev_ctx);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-
-        return ret;
+    if (ev_ctx && ev_ctx->rsp_dict)
+        dict_unref(ev_ctx->rsp_dict);
+    GF_FREE(ev_ctx);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                           struct cds_list_head *selected, dict_t *rsp_dict)
+glusterd_op_bricks_select(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          struct cds_list_head *selected, dict_t *rsp_dict)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (op > GD_OP_NONE);
-        GF_ASSERT (op < GD_OP_MAX);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(op > GD_OP_NONE);
+    GF_ASSERT(op < GD_OP_MAX);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_STOP_VOLUME:
-                ret = glusterd_bricks_select_stop_volume (dict, op_errstr,
-                                                          selected);
-                break;
+            ret = glusterd_bricks_select_stop_volume(dict, op_errstr, selected);
+            break;
         case GD_OP_REMOVE_BRICK:
-                ret = glusterd_bricks_select_remove_brick (dict, op_errstr,
-                                                           selected);
-                break;
+            ret = glusterd_bricks_select_remove_brick(dict, op_errstr,
+                                                      selected);
+            break;
 
         case GD_OP_PROFILE_VOLUME:
-                ret = glusterd_bricks_select_profile_volume (dict, op_errstr,
-                                                             selected);
-                break;
+            ret = glusterd_bricks_select_profile_volume(dict, op_errstr,
+                                                        selected);
+            break;
 
         case GD_OP_HEAL_VOLUME:
-                ret = glusterd_bricks_select_heal_volume (dict, op_errstr,
-                                                          selected, rsp_dict);
-                break;
+            ret = glusterd_bricks_select_heal_volume(dict, op_errstr, selected,
+                                                     rsp_dict);
+            break;
 
         case GD_OP_STATUS_VOLUME:
-                ret = glusterd_bricks_select_status_volume (dict, op_errstr,
-                                                            selected);
-                break;
-
+            ret = glusterd_bricks_select_status_volume(dict, op_errstr,
+                                                       selected);
+            break;
         case GD_OP_DEFRAG_BRICK_VOLUME:
-                ret = glusterd_bricks_select_rebalance_volume (dict, op_errstr,
-                                                               selected);
-                break;
+            ret = glusterd_bricks_select_rebalance_volume(dict, op_errstr,
+                                                          selected);
+            break;
 
         case GD_OP_BARRIER:
-                ret = glusterd_bricks_select_barrier (dict, selected);
-                break;
+            ret = glusterd_bricks_select_barrier(dict, selected);
+            break;
         case GD_OP_SNAP:
-                ret = glusterd_bricks_select_snap (dict, op_errstr, selected);
-                break;
+            ret = glusterd_bricks_select_snap(dict, op_errstr, selected);
+            break;
         case GD_OP_SCRUB_STATUS:
-                ret = glusterd_bricks_select_scrub (dict, op_errstr, selected);
-                break;
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = glusterd_bricks_select_scrub(dict, op_errstr, selected);
+            break;
         default:
-                break;
-         }
+            break;
+    }
 
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
-glusterd_op_sm_t glusterd_op_state_default [] = {
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_send_lock},//EVENT_START_LOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_default[] = {
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_NONE
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_send_lock},  // EVENT_START_LOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_lock},          // EVENT_LOCK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_RCVD_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_ALL_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_STAGE_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_COMMIT_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_RCVD_RJT
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_STAGE_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},       // EVENT_UNLOCK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_START_UNLOCK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_lock_sent [] = {
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_rcvd_lock_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_send_stage_op}, //EVENT_ALL_ACC
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_lock_sent[] = {
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},           // EVENT_NONE
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},           // EVENT_START_LOCK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_lock},           // EVENT_LOCK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_rcvd_lock_acc},  // EVENT_RCVD_ACC
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_send_stage_op},  // EVENT_ALL_ACC
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_ACK_DRAIN,
+     glusterd_op_ac_send_unlock_drain},            // EVENT_RCVD_RJT
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_locked [] = {
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGED, glusterd_op_ac_stage_op}, //EVENT_STAGE_OP
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_locked[] = {
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_NONE
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_START_LOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_lock},      // EVENT_LOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_RCVD_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_ALL_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_STAGE_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_COMMIT_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGED, glusterd_op_ac_stage_op},  // EVENT_STAGE_OP
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},   // EVENT_UNLOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_START_UNLOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},              // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_stage_op_sent [] = {
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_rcvd_stage_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED,   glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_stage_op_sent[] = {
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_STAGE_OP_SENT,
+     glusterd_op_ac_rcvd_stage_op_acc},  // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_OP_SENT,
+     glusterd_op_ac_send_brick_op},                    // EVENT_STAGE_ACC
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_stage_op_failed},                  // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},      // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},      // EVENT_START_UNLOCK
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_STAGE_OP_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_stage_op_failed [] = {
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_stage_op_failed[] = {
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_stage_op_failed},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_stage_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},        // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},        // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_staged [] = {
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_send_brick_op}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_staged[] = {
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_RCVD_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_BRICK_COMMITTED,
+     glusterd_op_ac_send_brick_op},                // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},     // EVENT_START_UNLOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},     // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},              // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_op_sent [] = {
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_rcvd_brick_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED,   glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_BRICK_OP
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_send_commit_op}, //EVENT_ALL_ACK
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_op_sent[] = {
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_OP_SENT,
+     glusterd_op_ac_rcvd_brick_op_acc},                // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_brick_op_failed},                  // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_BRICK_OP
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},      // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},      // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMIT_OP_SENT,
+     glusterd_op_ac_send_commit_op},  // EVENT_ALL_ACK
+    {GD_OP_STATE_BRICK_OP_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_op_failed [] = {
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_BRICK_OP
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_op_failed[] = {
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_brick_op_failed},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_brick_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_BRICK_OP
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},        // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},        // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_committed [] = {
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_rcvd_brick_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_commit_op}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_committed[] = {
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_COMMITTED,
+     glusterd_op_ac_rcvd_brick_op_acc},                  // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_brick_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},        // EVENT_UNLOCK
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_commit_op},    // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_commit_failed [] = {
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_send_commit_failed}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_commit_failed[] = {
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_brick_op_failed},                        // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_brick_op_failed},                        // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},            // EVENT_UNLOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_send_commit_failed},  // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_commit_op_failed [] = {
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_commit_op_failed[] = {
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_commit_op_failed},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_commit_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},         // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},         // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_commit_op_sent [] = {
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_rcvd_commit_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_UNLOCK_SENT,    glusterd_op_ac_send_unlock}, //EVENT_ALL_ACC
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_UNLOCK_SENT,    glusterd_op_ac_send_unlock}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT,        glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_commit_op_sent[] = {
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_COMMIT_OP_SENT,
+     glusterd_op_ac_rcvd_commit_op_acc},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACC
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},      // EVENT_STAGE_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_commit_op_failed},                  // EVENT_RCVD_RJT
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},       // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},       // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_COMMIT_OP_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_committed [] = {
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_committed[] = {
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_NONE
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_START_LOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_lock},   // EVENT_LOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_RCVD_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_ALL_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_STAGE_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_COMMIT_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_RCVD_RJT
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_STAGE_OP
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},                // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_unlock_sent [] = {
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_rcvd_unlock_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlocked_all}, //EVENT_ALL_ACC
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_rcvd_unlock_acc}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_unlock_sent[] = {
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_UNLOCK_SENT,
+     glusterd_op_ac_rcvd_unlock_acc},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlocked_all},  // EVENT_ALL_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},      // EVENT_STAGE_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},      // EVENT_COMMIT_ACC
+    {GD_OP_STATE_UNLOCK_SENT,
+     glusterd_op_ac_rcvd_unlock_acc},                // EVENT_RCVD_RJT
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},    // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},    // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_UNLOCK_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_ack_drain [] = {
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_ack_drain[] = {
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_ACK_DRAIN,
+     glusterd_op_ac_send_unlock_drain},            // EVENT_RCVD_ACC
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_ACK_DRAIN,
+     glusterd_op_ac_send_unlock_drain},            // EVENT_RCVD_RJT
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t *glusterd_op_state_table [] = {
-        glusterd_op_state_default,
-        glusterd_op_state_lock_sent,
-        glusterd_op_state_locked,
-        glusterd_op_state_stage_op_sent,
-        glusterd_op_state_staged,
-        glusterd_op_state_commit_op_sent,
-        glusterd_op_state_committed,
-        glusterd_op_state_unlock_sent,
-        glusterd_op_state_stage_op_failed,
-        glusterd_op_state_commit_op_failed,
-        glusterd_op_state_brick_op_sent,
-        glusterd_op_state_brick_op_failed,
-        glusterd_op_state_brick_committed,
-        glusterd_op_state_brick_commit_failed,
-        glusterd_op_state_ack_drain
-};
+glusterd_op_sm_t *glusterd_op_state_table[] = {
+    glusterd_op_state_default,         glusterd_op_state_lock_sent,
+    glusterd_op_state_locked,          glusterd_op_state_stage_op_sent,
+    glusterd_op_state_staged,          glusterd_op_state_commit_op_sent,
+    glusterd_op_state_committed,       glusterd_op_state_unlock_sent,
+    glusterd_op_state_stage_op_failed, glusterd_op_state_commit_op_failed,
+    glusterd_op_state_brick_op_sent,   glusterd_op_state_brick_op_failed,
+    glusterd_op_state_brick_committed, glusterd_op_state_brick_commit_failed,
+    glusterd_op_state_ack_drain};
 
 int
-glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
-                          glusterd_op_sm_event_t **new_event)
+glusterd_op_sm_new_event(glusterd_op_sm_event_type_t event_type,
+                         glusterd_op_sm_event_t **new_event)
 {
-        glusterd_op_sm_event_t      *event = NULL;
+    glusterd_op_sm_event_t *event = NULL;
 
-        GF_ASSERT (new_event);
-        GF_ASSERT (GD_OP_EVENT_NONE <= event_type &&
-                        GD_OP_EVENT_MAX > event_type);
+    GF_ASSERT(new_event);
+    GF_ASSERT(GD_OP_EVENT_NONE <= event_type && GD_OP_EVENT_MAX > event_type);
 
-        event = GF_CALLOC (1, sizeof (*event), gf_gld_mt_op_sm_event_t);
+    event = GF_CALLOC(1, sizeof(*event), gf_gld_mt_op_sm_event_t);
 
-        if (!event)
-                return -1;
+    if (!event)
+        return -1;
 
-        *new_event = event;
-        event->event = event_type;
-        CDS_INIT_LIST_HEAD (&event->list);
+    *new_event = event;
+    event->event = event_type;
+    CDS_INIT_LIST_HEAD(&event->list);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             uuid_t *txn_id, void *ctx)
+glusterd_op_sm_inject_event(glusterd_op_sm_event_type_t event_type,
+                            uuid_t *txn_id, void *ctx)
 {
-        int32_t                 ret = -1;
-        glusterd_op_sm_event_t  *event = NULL;
+    int32_t ret = -1;
+    glusterd_op_sm_event_t *event = NULL;
 
-        GF_ASSERT (event_type < GD_OP_EVENT_MAX &&
-                        event_type >= GD_OP_EVENT_NONE);
+    GF_ASSERT(event_type < GD_OP_EVENT_MAX && event_type >= GD_OP_EVENT_NONE);
 
-        ret = glusterd_op_sm_new_event (event_type, &event);
+    ret = glusterd_op_sm_new_event(event_type, &event);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        event->ctx = ctx;
+    event->ctx = ctx;
 
-        if (txn_id)
-                gf_uuid_copy (event->txn_id, *txn_id);
+    if (txn_id)
+        gf_uuid_copy(event->txn_id, *txn_id);
 
-        gf_msg_debug (THIS->name, 0, "Enqueue event: '%s'",
-                glusterd_op_sm_event_name_get (event->event));
-        cds_list_add_tail (&event->list, &gd_op_sm_queue);
+    gf_msg_debug(THIS->name, 0, "Enqueue event: '%s'",
+                 glusterd_op_sm_event_name_get(event->event));
+    cds_list_add_tail(&event->list, &gd_op_sm_queue);
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-glusterd_destroy_req_ctx (glusterd_req_ctx_t *ctx)
+glusterd_destroy_req_ctx(glusterd_req_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
-        if (ctx->dict)
-                dict_unref (ctx->dict);
-        GF_FREE (ctx);
+    if (!ctx)
+        return;
+    if (ctx->dict)
+        dict_unref(ctx->dict);
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_local_unlock_ctx (uuid_t *ctx)
+glusterd_destroy_local_unlock_ctx(uuid_t *ctx)
 {
-        if (!ctx)
-                return;
-        GF_FREE (ctx);
+    if (!ctx)
+        return;
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_op_event_ctx (glusterd_op_sm_event_t *event)
+glusterd_destroy_op_event_ctx(glusterd_op_sm_event_t *event)
 {
-        if (!event)
-                return;
+    if (!event)
+        return;
 
-        switch (event->event) {
+    switch (event->event) {
         case GD_OP_EVENT_LOCK:
         case GD_OP_EVENT_UNLOCK:
-                glusterd_destroy_lock_ctx (event->ctx);
-                break;
+            glusterd_destroy_lock_ctx(event->ctx);
+            break;
         case GD_OP_EVENT_STAGE_OP:
         case GD_OP_EVENT_ALL_ACK:
-                glusterd_destroy_req_ctx (event->ctx);
-                break;
+            glusterd_destroy_req_ctx(event->ctx);
+            break;
         case GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP:
-                glusterd_destroy_local_unlock_ctx (event->ctx);
-                break;
+            glusterd_destroy_local_unlock_ctx(event->ctx);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 int
-glusterd_op_sm ()
+glusterd_op_sm()
 {
-        glusterd_op_sm_event_t          *event = NULL;
-        glusterd_op_sm_event_t          *tmp = NULL;
-        int                             ret = -1;
-        int                             lock_err = 0;
-        glusterd_op_sm_ac_fn            handler = NULL;
-        glusterd_op_sm_t                *state = NULL;
-        glusterd_op_sm_event_type_t     event_type = GD_OP_EVENT_NONE;
-        xlator_t                        *this = NULL;
-        glusterd_op_info_t              txn_op_info;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = synclock_trylock (&gd_op_sm_lock);
-        if (ret) {
-                lock_err = errno;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_LOCK_FAIL, "lock failed due to %s",
-                        strerror (lock_err));
-                goto lock_failed;
-        }
-
-        while (!cds_list_empty (&gd_op_sm_queue)) {
-
-                cds_list_for_each_entry_safe (event, tmp, &gd_op_sm_queue,
-                                              list) {
-
-                        cds_list_del_init (&event->list);
-                        event_type = event->event;
-                        gf_msg_debug (this->name, 0, "Dequeued event of "
-                                "type: '%s'",
-                                glusterd_op_sm_event_name_get(event_type));
-
-                        gf_msg_debug (this->name, 0, "transaction ID = %s",
-                                uuid_utoa (event->txn_id));
-
-                        ret = glusterd_get_txn_opinfo (&event->txn_id,
-                                                       &txn_op_info);
-                        if (ret) {
-                                gf_msg_callingfn (this->name, GF_LOG_ERROR, 0,
-                                                  GD_MSG_TRANS_OPINFO_GET_FAIL,
-                                                  "Unable to get transaction "
-                                                  "opinfo for transaction ID :"
-                                                  "%s",
-                                                  uuid_utoa (event->txn_id));
-                                glusterd_destroy_op_event_ctx (event);
-                                GF_FREE (event);
-                                continue;
-                        } else
-                                opinfo = txn_op_info;
-
-                        state = glusterd_op_state_table[opinfo.state.state];
-
-                        GF_ASSERT (state);
-
-                        handler = state[event_type].handler;
-                        GF_ASSERT (handler);
-
-                        ret = handler (event, event->ctx);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_HANDLER_RETURNED,
-                                        "handler returned: %d", ret);
-                                glusterd_destroy_op_event_ctx (event);
-                                GF_FREE (event);
-                                continue;
-                        }
-
-                        ret = glusterd_op_sm_transition_state (&opinfo, state,
-                                                                event_type);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_EVENT_STATE_TRANSITION_FAIL,
-                                        "Unable to transition"
-                                        "state from '%s' to '%s'",
-                         glusterd_op_sm_state_name_get(opinfo.state.state),
-                         glusterd_op_sm_state_name_get(state[event_type].next_state));
-                                (void) synclock_unlock (&gd_op_sm_lock);
-                                return ret;
-                        }
-
-                        if ((state[event_type].next_state ==
-                             GD_OP_STATE_DEFAULT) &&
-                           (event_type == GD_OP_EVENT_UNLOCK)) {
-                                /* Clearing the transaction opinfo */
-                                ret = glusterd_clear_txn_opinfo(&event->txn_id);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_TRANS_OPINFO_CLEAR_FAIL,
-                                                "Unable to clear "
-                                                "transaction's opinfo");
-                        } else {
-                                ret = glusterd_set_txn_opinfo (&event->txn_id,
-                                                               &opinfo);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                                "Unable to set "
-                                                "transaction's opinfo");
-                        }
-
-                        glusterd_destroy_op_event_ctx (event);
-                        GF_FREE (event);
-
-                }
-        }
-
+    glusterd_op_sm_event_t *event = NULL;
+    glusterd_op_sm_event_t *tmp = NULL;
+    int ret = -1;
+    int lock_err = 0;
+    glusterd_op_sm_ac_fn handler = NULL;
+    glusterd_op_sm_t *state = NULL;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    glusterd_op_info_t txn_op_info;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = synclock_trylock(&gd_op_sm_lock);
+    if (ret) {
+        lock_err = errno;
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_LOCK_FAIL,
+               "lock failed due to %s", strerror(lock_err));
+        goto lock_failed;
+    }
+
+    while (!cds_list_empty(&gd_op_sm_queue)) {
+        cds_list_for_each_entry_safe(event, tmp, &gd_op_sm_queue, list)
+        {
+            cds_list_del_init(&event->list);
+            event_type = event->event;
+            gf_msg_debug(this->name, 0,
+                         "Dequeued event of "
+                         "type: '%s'",
+                         glusterd_op_sm_event_name_get(event_type));
+
+            gf_msg_debug(this->name, 0, "transaction ID = %s",
+                         uuid_utoa(event->txn_id));
+
+            ret = glusterd_get_txn_opinfo(&event->txn_id, &txn_op_info);
+            if (ret) {
+                gf_msg_callingfn(this->name, GF_LOG_ERROR, 0,
+                                 GD_MSG_TRANS_OPINFO_GET_FAIL,
+                                 "Unable to get transaction "
+                                 "opinfo for transaction ID :"
+                                 "%s",
+                                 uuid_utoa(event->txn_id));
+                glusterd_destroy_op_event_ctx(event);
+                GF_FREE(event);
+                continue;
+            } else
+                opinfo = txn_op_info;
+
+            state = glusterd_op_state_table[opinfo.state.state];
+
+            GF_ASSERT(state);
+
+            handler = state[event_type].handler;
+            GF_ASSERT(handler);
+
+            ret = handler(event, event->ctx);
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLER_RETURNED,
+                       "handler returned: %d", ret);
+                glusterd_destroy_op_event_ctx(event);
+                GF_FREE(event);
+                continue;
+            }
+
+            ret = glusterd_op_sm_transition_state(&opinfo, state, event_type);
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_EVENT_STATE_TRANSITION_FAIL,
+                       "Unable to transition"
+                       "state from '%s' to '%s'",
+                       glusterd_op_sm_state_name_get(opinfo.state.state),
+                       glusterd_op_sm_state_name_get(
+                           state[event_type].next_state));
+                (void)synclock_unlock(&gd_op_sm_lock);
+                return ret;
+            }
 
-        (void) synclock_unlock (&gd_op_sm_lock);
-        ret = 0;
+            if ((state[event_type].next_state == GD_OP_STATE_DEFAULT) &&
+                (event_type == GD_OP_EVENT_UNLOCK)) {
+                /* Clearing the transaction opinfo */
+                ret = glusterd_clear_txn_opinfo(&event->txn_id);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_TRANS_OPINFO_CLEAR_FAIL,
+                           "Unable to clear "
+                           "transaction's opinfo");
+            } else {
+                if ((priv->op_version < GD_OP_VERSION_6_0) ||
+                    !(event_type == GD_OP_EVENT_STAGE_OP &&
+                      opinfo.state.state == GD_OP_STATE_STAGED &&
+                      opinfo.skip_locking)) {
+                    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_TRANS_OPINFO_SET_FAIL,
+                               "Unable to set "
+                               "transaction's opinfo");
+                }
+            }
+
+            glusterd_destroy_op_event_ctx(event);
+            GF_FREE(event);
+        }
+    }
+
+    (void)synclock_unlock(&gd_op_sm_lock);
+    ret = 0;
 
 lock_failed:
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_op_set_op (glusterd_op_t op)
+glusterd_op_set_op(glusterd_op_t op)
 {
+    GF_ASSERT(op < GD_OP_MAX);
+    GF_ASSERT(op > GD_OP_NONE);
 
-        GF_ASSERT (op < GD_OP_MAX);
-        GF_ASSERT (op > GD_OP_NONE);
-
-        opinfo.op = op;
-
-        return 0;
+    opinfo.op = op;
 
+    return 0;
 }
 
 int32_t
-glusterd_op_get_op ()
+glusterd_op_get_op()
 {
-
-        return opinfo.op;
-
+    return opinfo.op;
 }
 
 int32_t
-glusterd_op_set_req (rpcsvc_request_t *req)
+glusterd_op_set_req(rpcsvc_request_t *req)
 {
-
-        GF_ASSERT (req);
-        opinfo.req = req;
-        return 0;
+    GF_ASSERT(req);
+    opinfo.req = req;
+    return 0;
 }
 
 int32_t
-glusterd_op_clear_op (glusterd_op_t op)
+glusterd_op_clear_op(glusterd_op_t op)
 {
+    opinfo.op = GD_OP_NONE;
 
-        opinfo.op = GD_OP_NONE;
-
-        return 0;
-
+    return 0;
 }
 
 int32_t
-glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
+glusterd_op_free_ctx(glusterd_op_t op, void *ctx)
 {
-
-        if (ctx) {
-                switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                case GD_OP_DELETE_VOLUME:
-                case GD_OP_STOP_VOLUME:
-                case GD_OP_ADD_BRICK:
-                case GD_OP_REMOVE_BRICK:
-                case GD_OP_REPLACE_BRICK:
-                case GD_OP_LOG_ROTATE:
-                case GD_OP_SYNC_VOLUME:
-                case GD_OP_SET_VOLUME:
-                case GD_OP_START_VOLUME:
-                case GD_OP_RESET_VOLUME:
-                case GD_OP_GSYNC_SET:
-                case GD_OP_QUOTA:
-                case GD_OP_PROFILE_VOLUME:
-                case GD_OP_STATUS_VOLUME:
-                case GD_OP_REBALANCE:
-                case GD_OP_HEAL_VOLUME:
-                case GD_OP_STATEDUMP_VOLUME:
-                case GD_OP_CLEARLOCKS_VOLUME:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        dict_unref (ctx);
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
+    if (ctx) {
+        switch (op) {
+            case GD_OP_CREATE_VOLUME:
+            case GD_OP_DELETE_VOLUME:
+            case GD_OP_STOP_VOLUME:
+            case GD_OP_ADD_BRICK:
+            case GD_OP_REMOVE_BRICK:
+            case GD_OP_REPLACE_BRICK:
+            case GD_OP_LOG_ROTATE:
+            case GD_OP_SYNC_VOLUME:
+            case GD_OP_SET_VOLUME:
+            case GD_OP_START_VOLUME:
+            case GD_OP_RESET_VOLUME:
+            case GD_OP_GSYNC_SET:
+            case GD_OP_QUOTA:
+            case GD_OP_PROFILE_VOLUME:
+            case GD_OP_STATUS_VOLUME:
+            case GD_OP_REBALANCE:
+            case GD_OP_HEAL_VOLUME:
+            case GD_OP_STATEDUMP_VOLUME:
+            case GD_OP_CLEARLOCKS_VOLUME:
+            case GD_OP_DEFRAG_BRICK_VOLUME:
+            case GD_OP_MAX_OPVERSION:
+                dict_unref(ctx);
+                break;
+            default:
+                GF_ASSERT(0);
+                break;
         }
+    }
 
-        glusterd_op_reset_ctx ();
-        return 0;
-
+    glusterd_op_reset_ctx();
+    return 0;
 }
 
 void *
-glusterd_op_get_ctx ()
+glusterd_op_get_ctx()
 {
-
-        return opinfo.op_ctx;
-
+    return opinfo.op_ctx;
 }
 
 int
-glusterd_op_sm_init ()
+glusterd_op_sm_init()
 {
-        CDS_INIT_LIST_HEAD (&gd_op_sm_queue);
-        synclock_init (&gd_op_sm_lock, SYNC_LOCK_DEFAULT);
-        return 0;
+    CDS_INIT_LIST_HEAD(&gd_op_sm_queue);
+    synclock_init(&gd_op_sm_lock, SYNC_LOCK_DEFAULT);
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index 19b1bd97e04..8a24b16612a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -10,293 +10,304 @@
 #ifndef _GLUSTERD_OP_SM_H_
 #define _GLUSTERD_OP_SM_H_
 
-
 #include <pthread.h>
-#include "compat-uuid.h"
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
+#include <glusterfs/compat-uuid.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "glusterd.h"
 #include "protocol-common.h"
 #include "glusterd-hooks.h"
 
-#define GD_OP_PROTECTED    (0x02)
-#define GD_OP_UNPROTECTED  (0x04)
+#define GD_OP_PROTECTED (0x02)
+#define GD_OP_UNPROTECTED (0x04)
 
 typedef enum glusterd_op_sm_state_ {
-        GD_OP_STATE_DEFAULT = 0,
-        GD_OP_STATE_LOCK_SENT,
-        GD_OP_STATE_LOCKED,
-        GD_OP_STATE_STAGE_OP_SENT,
-        GD_OP_STATE_STAGED,
-        GD_OP_STATE_COMMIT_OP_SENT,
-        GD_OP_STATE_COMMITED,
-        GD_OP_STATE_UNLOCK_SENT,
-        GD_OP_STATE_STAGE_OP_FAILED,
-        GD_OP_STATE_COMMIT_OP_FAILED,
-        GD_OP_STATE_BRICK_OP_SENT,
-        GD_OP_STATE_BRICK_OP_FAILED,
-        GD_OP_STATE_BRICK_COMMITTED,
-        GD_OP_STATE_BRICK_COMMIT_FAILED,
-        GD_OP_STATE_ACK_DRAIN,
-        GD_OP_STATE_MAX,
+    GD_OP_STATE_DEFAULT = 0,
+    GD_OP_STATE_LOCK_SENT,
+    GD_OP_STATE_LOCKED,
+    GD_OP_STATE_STAGE_OP_SENT,
+    GD_OP_STATE_STAGED,
+    GD_OP_STATE_COMMIT_OP_SENT,
+    GD_OP_STATE_COMMITED,
+    GD_OP_STATE_UNLOCK_SENT,
+    GD_OP_STATE_STAGE_OP_FAILED,
+    GD_OP_STATE_COMMIT_OP_FAILED,
+    GD_OP_STATE_BRICK_OP_SENT,
+    GD_OP_STATE_BRICK_OP_FAILED,
+    GD_OP_STATE_BRICK_COMMITTED,
+    GD_OP_STATE_BRICK_COMMIT_FAILED,
+    GD_OP_STATE_ACK_DRAIN,
+    GD_OP_STATE_MAX,
 } glusterd_op_sm_state_t;
 
 typedef enum glusterd_op_sm_event_type_ {
-        GD_OP_EVENT_NONE = 0,
-        GD_OP_EVENT_START_LOCK,
-        GD_OP_EVENT_LOCK,
-        GD_OP_EVENT_RCVD_ACC,
-        GD_OP_EVENT_ALL_ACC,
-        GD_OP_EVENT_STAGE_ACC,
-        GD_OP_EVENT_COMMIT_ACC,
-        GD_OP_EVENT_RCVD_RJT,
-        GD_OP_EVENT_STAGE_OP,
-        GD_OP_EVENT_COMMIT_OP,
-        GD_OP_EVENT_UNLOCK,
-        GD_OP_EVENT_START_UNLOCK,
-        GD_OP_EVENT_ALL_ACK,
-        GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP,
-        GD_OP_EVENT_MAX
+    GD_OP_EVENT_NONE = 0,
+    GD_OP_EVENT_START_LOCK,
+    GD_OP_EVENT_LOCK,
+    GD_OP_EVENT_RCVD_ACC,
+    GD_OP_EVENT_ALL_ACC,
+    GD_OP_EVENT_STAGE_ACC,
+    GD_OP_EVENT_COMMIT_ACC,
+    GD_OP_EVENT_RCVD_RJT,
+    GD_OP_EVENT_STAGE_OP,
+    GD_OP_EVENT_COMMIT_OP,
+    GD_OP_EVENT_UNLOCK,
+    GD_OP_EVENT_START_UNLOCK,
+    GD_OP_EVENT_ALL_ACK,
+    GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP,
+    GD_OP_EVENT_MAX
 } glusterd_op_sm_event_type_t;
 
-
 struct glusterd_op_sm_event_ {
-        struct cds_list_head            list;
-        void                            *ctx;
-        glusterd_op_sm_event_type_t     event;
-        uuid_t                          txn_id;
+    struct cds_list_head list;
+    void *ctx;
+    glusterd_op_sm_event_type_t event;
+    uuid_t txn_id;
 };
 
 typedef struct glusterd_op_sm_event_ glusterd_op_sm_event_t;
 
-typedef int (*glusterd_op_sm_ac_fn) (glusterd_op_sm_event_t *, void *);
+typedef int (*glusterd_op_sm_ac_fn)(glusterd_op_sm_event_t *, void *);
 
 typedef struct glusterd_op_sm_ {
-        glusterd_op_sm_state_t      next_state;
-        glusterd_op_sm_ac_fn        handler;
+    glusterd_op_sm_state_t next_state;
+    glusterd_op_sm_ac_fn handler;
 } glusterd_op_sm_t;
 
 typedef struct glusterd_op_sm_state_info_ {
-        glusterd_op_sm_state_t          state;
-        struct timeval                  time;
+    glusterd_op_sm_state_t state;
+    struct timeval time;
 } glusterd_op_sm_state_info_t;
 
 struct glusterd_op_info_ {
-        glusterd_op_sm_state_info_t     state;
-        int32_t                         pending_count;
-        int32_t                         brick_pending_count;
-        int32_t                         op_count;
-        /* op is an enum, glusterd_op_t or glusterd_op_sm_state_info_t */
-        int                             op;
-        struct cds_list_head            op_peers;
-        void                            *op_ctx;
-        rpcsvc_request_t                *req;
-        int32_t                         op_ret;
-        int32_t                         op_errno;
-        char                            *op_errstr;
-        struct  cds_list_head           pending_bricks;
-        uint32_t                        txn_generation;
+    glusterd_op_sm_state_info_t state;
+    int32_t pending_count;
+    int32_t brick_pending_count;
+    int32_t op_count;
+    /* op is an enum, glusterd_op_t or glusterd_op_sm_state_info_t */
+    int op;
+    struct cds_list_head op_peers;
+    void *op_ctx;
+    rpcsvc_request_t *req;
+    int32_t op_ret;
+    int32_t op_errno;
+    char *op_errstr;
+    struct cds_list_head pending_bricks;
+    uint32_t txn_generation;
+    gf_boolean_t skip_locking;
 };
 
 typedef struct glusterd_op_info_ glusterd_op_info_t;
 
 struct glusterd_op_log_filename_ctx_ {
-        char                    volume_name[GD_VOLUME_NAME_MAX];
-        char                    brick[GD_VOLUME_NAME_MAX];
-        char                    path[PATH_MAX];
+    char volume_name[GD_VOLUME_NAME_MAX];
+    char brick[GD_VOLUME_NAME_MAX];
+    char path[PATH_MAX];
 };
 typedef struct glusterd_op_log_filename_ctx_ glusterd_op_log_filename_ctx_t;
 
 struct glusterd_op_lock_ctx_ {
-        uuid_t                  uuid;
-        dict_t                 *dict;
-        rpcsvc_request_t        *req;
+    uuid_t uuid;
+    dict_t *dict;
+    rpcsvc_request_t *req;
 };
 
 typedef struct glusterd_op_lock_ctx_ glusterd_op_lock_ctx_t;
 
 struct glusterd_req_ctx_ {
-        rpcsvc_request_t *req;
-	u_char            uuid[16];
-	int               op;
-        dict_t           *dict;
+    rpcsvc_request_t *req;
+    u_char uuid[16];
+    int op;
+    dict_t *dict;
 };
 
 typedef struct glusterd_req_ctx_ glusterd_req_ctx_t;
 
 typedef struct glusterd_op_brick_rsp_ctx_ {
-        int  op_ret;
-        char *op_errstr;
-        dict_t *rsp_dict;
-        glusterd_req_ctx_t *commit_ctx;
-        glusterd_pending_node_t *pending_node;
+    int op_ret;
+    char *op_errstr;
+    dict_t *rsp_dict;
+    glusterd_req_ctx_t *commit_ctx;
+    glusterd_pending_node_t *pending_node;
 } glusterd_op_brick_rsp_ctx_t;
 
 typedef struct glusterd_pr_brick_rsp_conv_t {
-        int count;
-        dict_t *dict;
+    int count;
+    dict_t *dict;
 } glusterd_pr_brick_rsp_conv_t;
 
 typedef struct glusterd_heal_rsp_conv_ {
-        dict_t *dict;
-        glusterd_volinfo_t *volinfo;
-        xlator_t *this;
+    dict_t *dict;
+    glusterd_volinfo_t *volinfo;
+    xlator_t *this;
 } glusterd_heal_rsp_conv_t;
 
 typedef struct glusterd_status_rsp_conv_ {
-        int count;
-        int brick_index_max;
-        int other_count;
-        dict_t *dict;
+    int count;
+    int brick_index_max;
+    int other_count;
+    dict_t *dict;
 } glusterd_status_rsp_conv_t;
 
-
 typedef struct glusterd_txn_opinfo_object_ {
-        glusterd_op_info_t    opinfo;
+    glusterd_op_info_t opinfo;
 } glusterd_txn_opinfo_obj;
 
 typedef enum cli_cmd_type_ {
-        PER_HEAL_XL,
-        ALL_HEAL_XL,
- } cli_cmd_type;
+    PER_HEAL_XL,
+    ALL_HEAL_XL,
+} cli_cmd_type;
 
 typedef struct glusterd_all_volume_options {
-        char          *option;
+    char *option;
+    char *dflt_val;
 } glusterd_all_vol_opts;
 
 int
-glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
-                         glusterd_commit_hook_type_t type);
+glusterd_op_commit_hook(glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_commit_hook_type_t type);
 
 int
-glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
-                          glusterd_op_sm_event_t **new_event);
+glusterd_op_sm_new_event(glusterd_op_sm_event_type_t event_type,
+                         glusterd_op_sm_event_t **new_event);
 int
-glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             uuid_t *txn_id, void *ctx);
+glusterd_op_sm_inject_event(glusterd_op_sm_event_type_t event_type,
+                            uuid_t *txn_id, void *ctx);
 
 int
-glusterd_op_sm_init ();
+glusterd_op_sm_init();
 
 int
-glusterd_op_sm ();
+glusterd_op_sm();
 
 int32_t
-glusterd_op_set_ctx (void *ctx);
+glusterd_op_set_ctx(void *ctx);
 
 int32_t
-glusterd_op_set_op (glusterd_op_t op);
+glusterd_op_set_op(glusterd_op_t op);
 
 int
-glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx);
+glusterd_op_build_payload(dict_t **req, char **op_errstr, dict_t *op_ctx);
 
 int32_t
-glusterd_op_stage_validate (glusterd_op_t op, dict_t *req, char **op_errstr,
-                            dict_t *rsp_dict);
+glusterd_op_stage_validate(glusterd_op_t op, dict_t *req, char **op_errstr,
+                           dict_t *rsp_dict);
 
 int32_t
-glusterd_op_commit_perform (glusterd_op_t op, dict_t *req, char **op_errstr,
-                            dict_t* dict);
+glusterd_op_commit_perform(glusterd_op_t op, dict_t *req, char **op_errstr,
+                           dict_t *dict);
 
 int32_t
-glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
-                       char *err_str, size_t err_len);
+glusterd_op_txn_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                      char *err_str, size_t err_len);
 
 int32_t
-glusterd_op_txn_complete ();
+glusterd_op_txn_complete();
 
 void *
-glusterd_op_get_ctx ();
+glusterd_op_get_ctx();
 
 int32_t
-glusterd_op_set_req (rpcsvc_request_t *req);
+glusterd_op_set_req(rpcsvc_request_t *req);
 
 int32_t
-glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
-                               int32_t op_errno, rpcsvc_request_t *req,
-                               void *ctx, char *op_errstr);
+glusterd_op_send_cli_response(glusterd_op_t op, int32_t op_ret,
+                              int32_t op_errno, rpcsvc_request_t *req,
+                              void *ctx, char *op_errstr);
 int32_t
-glusterd_op_get_op ();
+glusterd_op_get_op();
 
 int32_t
-glusterd_op_clear_op ();
+glusterd_op_clear_op();
 
 int32_t
-glusterd_op_free_ctx (glusterd_op_t op, void *ctx);
+glusterd_op_free_ctx(glusterd_op_t op, void *ctx);
 
 int
 glusterd_check_option_exists(char *optstring, char **completion);
 
 int
-set_xlator_option (dict_t *dict, char *key, char *value);
+set_xlator_option(dict_t *dict, char *key, char *value);
 
-char*
-glusterd_op_sm_state_name_get (int state);
+char *
+glusterd_op_sm_state_name_get(int state);
 
-char*
-glusterd_op_sm_event_name_get (int event);
+char *
+glusterd_op_sm_event_name_get(int event);
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                           struct cds_list_head *selected, dict_t *rsp_dict);
+glusterd_op_bricks_select(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          struct cds_list_head *selected, dict_t *rsp_dict);
 int
-glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
-                                 gd1_mgmt_brick_op_req **req, dict_t *dict);
+glusterd_brick_op_build_payload(glusterd_op_t op,
+                                glusterd_brickinfo_t *brickinfo,
+                                gd1_mgmt_brick_op_req **req, dict_t *dict);
 int
-glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req,
                                dict_t *dict);
 int32_t
-glusterd_handle_brick_rsp (void *pending_entry, glusterd_op_t op,
-                           dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
-                           gd_node_type type);
+glusterd_handle_brick_rsp(void *pending_entry, glusterd_op_t op,
+                          dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
+                          gd_node_type type);
 
-dict_t*
-glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
+dict_t *
+glusterd_op_init_commit_rsp_dict(glusterd_op_t op);
 
 void
-glusterd_op_modify_op_ctx (glusterd_op_t op, void *op_ctx);
+glusterd_op_modify_op_ctx(glusterd_op_t op, void *op_ctx);
+
+int
+glusterd_set_detach_bricks(dict_t *dict, glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_volume_stats_read_perf (char *brick_path, int32_t blk_size,
-                int32_t blk_count, double *throughput, double *time);
+glusterd_volume_stats_read_perf(char *brick_path, int32_t blk_size,
+                                int32_t blk_count, double *throughput,
+                                double *time);
 int32_t
-glusterd_volume_stats_write_perf (char *brick_path, int32_t blk_size,
-                int32_t blk_count, double *throughput, double *time);
+glusterd_volume_stats_write_perf(char *brick_path, int32_t blk_size,
+                                 int32_t blk_count, double *throughput,
+                                 double *time);
 gf_boolean_t
-glusterd_is_volume_started (glusterd_volinfo_t  *volinfo);
+glusterd_is_volume_started(glusterd_volinfo_t *volinfo);
+
 int
-glusterd_start_bricks (glusterd_volinfo_t *volinfo);
+glusterd_start_bricks(glusterd_volinfo_t *volinfo);
+
 gf_boolean_t
-glusterd_are_all_volumes_stopped ();
+glusterd_are_all_volumes_stopped();
 int
-glusterd_stop_bricks (glusterd_volinfo_t *volinfo);
+glusterd_stop_bricks(glusterd_volinfo_t *volinfo);
 int
-glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
-                                 dict_t *op_ctx);
-#ifdef HAVE_BD_XLATOR
-int
-glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg);
-#endif
+glusterd_defrag_volume_node_rsp(dict_t *req_dict, dict_t *rsp_dict,
+                                dict_t *op_ctx);
 
 int32_t
-glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+glusterd_get_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo);
 
 int32_t
-glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+glusterd_set_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo);
 
 int32_t
-glusterd_clear_txn_opinfo (uuid_t *txn_id);
+glusterd_clear_txn_opinfo(uuid_t *txn_id);
 
 int32_t
-glusterd_generate_txn_id (dict_t *dict, uuid_t **txn_id);
+glusterd_generate_txn_id(dict_t *dict, uuid_t **txn_id);
 
 void
-glusterd_set_opinfo (char *errstr, int32_t op_errno, int32_t op_ret);
+glusterd_set_opinfo(char *errstr, int32_t op_errno, int32_t op_ret);
+
+int
+glusterd_dict_set_volid(dict_t *dict, char *volname, char **op_errstr);
+
+int
+glusterd_op_stats_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_stats_volume(dict_t *dict, char **op_errstr);
 
 int
-glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr);
+gd_set_commit_hash(dict_t *dict);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-peer-utils.c b/xlators/mgmt/glusterd/src/glusterd-peer-utils.c
index 607ad3d38be..18d355cb186 100644
--- a/xlators/mgmt/glusterd/src/glusterd-peer-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-peer-utils.c
@@ -12,77 +12,180 @@
 #include "glusterd-store.h"
 #include "glusterd-server-quorum.h"
 #include "glusterd-messages.h"
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
+#include "glusterd-utils.h"
 
 void
-glusterd_peerinfo_destroy (struct rcu_head *head)
+glusterd_peerinfo_destroy(struct rcu_head *head)
 {
-        int32_t                   ret      = -1;
-        glusterd_peerinfo_t      *peerinfo = NULL;
-        glusterd_peer_hostname_t *hostname = NULL;
-        glusterd_peer_hostname_t *tmp      = NULL;
+    int32_t ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peer_hostname_t *hostname = NULL;
+    glusterd_peer_hostname_t *tmp = NULL;
 
-        /* This works as rcu_head is the first member of gd_rcu_head */
-        peerinfo = caa_container_of ((gd_rcu_head *)head, glusterd_peerinfo_t,
-                                      rcu_head);
+    /* This works as rcu_head is the first member of gd_rcu_head */
+    peerinfo = caa_container_of((gd_rcu_head *)head, glusterd_peerinfo_t,
+                                rcu_head);
 
-        /* Set THIS to the saved this. Needed by some functions below */
-        THIS = peerinfo->rcu_head.this;
+    /* Set THIS to the saved this. Needed by some functions below */
+    THIS = peerinfo->rcu_head.this;
 
-        CDS_INIT_LIST_HEAD (&peerinfo->uuid_list);
+    CDS_INIT_LIST_HEAD(&peerinfo->uuid_list);
 
-        ret = glusterd_store_delete_peerinfo (peerinfo);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_PEERINFO_DELETE_FAIL,
-                        "Deleting peer info failed");
-        }
+    ret = glusterd_store_delete_peerinfo(peerinfo);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_PEERINFO_DELETE_FAIL,
+               "Deleting peer info failed");
+    }
 
-        GF_FREE (peerinfo->hostname);
-        peerinfo->hostname = NULL;
+    GF_FREE(peerinfo->hostname);
+    peerinfo->hostname = NULL;
 
-        cds_list_for_each_entry_safe (hostname, tmp, &peerinfo->hostnames,
-                                      hostname_list) {
-                glusterd_peer_hostname_free (hostname);
-        }
+    cds_list_for_each_entry_safe(hostname, tmp, &peerinfo->hostnames,
+                                 hostname_list)
+    {
+        glusterd_peer_hostname_free(hostname);
+    }
 
-        glusterd_sm_tr_log_delete (&peerinfo->sm_log);
-        pthread_mutex_destroy (&peerinfo->delete_lock);
-        GF_FREE (peerinfo);
+    glusterd_sm_tr_log_delete(&peerinfo->sm_log);
+    pthread_mutex_unlock(&peerinfo->delete_lock);
+    pthread_mutex_destroy(&peerinfo->delete_lock);
+    GF_FREE(peerinfo);
 
-        peerinfo = NULL;
+    peerinfo = NULL;
 
-        return;
+    return;
 }
 
 int32_t
-glusterd_peerinfo_cleanup (glusterd_peerinfo_t *peerinfo)
+glusterd_peerinfo_cleanup(glusterd_peerinfo_t *peerinfo)
 {
-        GF_ASSERT (peerinfo);
-        glusterd_peerctx_t      *peerctx = NULL;
-        gf_boolean_t            quorum_action = _gf_false;
-        glusterd_conf_t         *priv = THIS->private;
-
-        if (pthread_mutex_trylock (&peerinfo->delete_lock)) {
-                /* Someone else is already deleting the peer, so give up */
-                return 0;
-        }
+    GF_ASSERT(peerinfo);
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_conf_t *priv = THIS->private;
+
+    if (pthread_mutex_trylock(&peerinfo->delete_lock)) {
+        /* Someone else is already deleting the peer, so give up */
+        return 0;
+    }
+
+    if (peerinfo->quorum_contrib != QUORUM_NONE)
+        quorum_action = _gf_true;
+    if (peerinfo->rpc) {
+        peerinfo->rpc = glusterd_rpc_clnt_unref(priv, peerinfo->rpc);
+        peerinfo->rpc = NULL;
+    }
+
+    cds_list_del_rcu(&peerinfo->uuid_list);
+    /* Saving THIS, as it is needed by the callback function */
+    peerinfo->rcu_head.this = THIS;
+    call_rcu(&peerinfo->rcu_head.head, glusterd_peerinfo_destroy);
+
+    if (quorum_action)
+        /* coverity[SLEEP] */
+        glusterd_do_quorum_action();
+    return 0;
+}
 
-        if (peerinfo->quorum_contrib != QUORUM_NONE)
-                quorum_action = _gf_true;
-        if (peerinfo->rpc) {
-                peerinfo->rpc = glusterd_rpc_clnt_unref (priv, peerinfo->rpc);
-                peerinfo->rpc = NULL;
+/* gd_peerinfo_find_from_hostname iterates over all the addresses saved for each
+ * peer and matches it to @hoststr.
+ * Returns the matched peer if found else returns NULL
+ */
+static glusterd_peerinfo_t *
+gd_peerinfo_find_from_hostname(const char *hoststr)
+{
+    xlator_t *this = THIS;
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *peer = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    glusterd_peer_hostname_t *tmphost = NULL;
+
+    GF_ASSERT(this != NULL);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (priv != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (hoststr != NULL), out);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &priv->peers, uuid_list)
+    {
+        cds_list_for_each_entry_rcu(tmphost, &peer->hostnames, hostname_list)
+        {
+            if (!strncasecmp(tmphost->hostname, hoststr, 1024)) {
+                gf_msg_debug(this->name, 0, "Friend %s found.. state: %d",
+                             tmphost->hostname, peer->state.state);
+                found = peer; /* Probably needs to be
+                                 dereferenced*/
+                goto unlock;
+            }
         }
+    }
+unlock:
+    RCU_READ_UNLOCK;
+out:
+    return found;
+}
 
-        cds_list_del_rcu (&peerinfo->uuid_list);
-        /* Saving THIS, as it is needed by the callback function */
-        peerinfo->rcu_head.this = THIS;
-        call_rcu (&peerinfo->rcu_head.head, glusterd_peerinfo_destroy);
+/* gd_peerinfo_find_from_addrinfo iterates over all the addresses saved for each
+ * peer, resolves them and compares them to @addr.
+ *
+ *
+ * NOTE: As getaddrinfo is a blocking call and is being performed multiple times
+ * in this function, it could lead to the calling thread to be blocked for
+ * significant amounts of time.
+ *
+ * Returns the matched peer if found else returns NULL
+ */
+static glusterd_peerinfo_t *
+gd_peerinfo_find_from_addrinfo(const struct addrinfo *addr)
+{
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *peer = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    glusterd_peer_hostname_t *address = NULL;
+    int ret = 0;
+    struct addrinfo *paddr = NULL;
+    struct addrinfo *tmp = NULL;
+
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &conf->peers, uuid_list)
+    {
+        cds_list_for_each_entry_rcu(address, &peer->hostnames, hostname_list)
+        {
+            /* TODO: Cache the resolved addrinfos to improve
+             * performance
+             */
+            ret = getaddrinfo(address->hostname, NULL, NULL, &paddr);
+            if (ret) {
+                /* Don't fail if getaddrinfo fails, continue
+                 * onto the next address
+                 */
+                gf_msg_trace(this->name, 0, "getaddrinfo for %s failed (%s)",
+                             address->hostname, gai_strerror(ret));
+                continue;
+            }
+
+            for (tmp = paddr; tmp != NULL; tmp = tmp->ai_next) {
+                if (gf_compare_sockaddr(addr->ai_addr, tmp->ai_addr)) {
+                    found = peer; /* (de)referenced? */
+                    break;
+                }
+            }
 
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        return 0;
+            freeaddrinfo(paddr);
+            if (found)
+                goto unlock;
+        }
+    }
+unlock:
+    RCU_READ_UNLOCK;
+out:
+    return found;
 }
 
 /* glusterd_peerinfo_find_by_hostname searches for a peer which matches the
@@ -94,79 +197,73 @@ glusterd_peerinfo_cleanup (glusterd_peerinfo_t *peerinfo)
  * the resolved addrinfos.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_hostname (const char *hoststr)
+glusterd_peerinfo_find_by_hostname(const char *hoststr)
 {
-        int                  ret      = -1;
-        struct addrinfo     *addr     = NULL;
-        struct addrinfo     *p        = NULL;
-        xlator_t            *this     = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-
-
-        this = THIS;
-        GF_ASSERT (hoststr);
-
-        peerinfo = NULL;
-
-        peerinfo = gd_peerinfo_find_from_hostname (hoststr);
-        if (peerinfo)
-                return peerinfo;
-
-        ret = getaddrinfo (hoststr, NULL, NULL, &addr);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, ret,
-                        GD_MSG_GETADDRINFO_FAIL,
-                        "error in getaddrinfo: %s\n",
-                        gai_strerror(ret));
-                goto out;
-        }
-
-        for (p = addr; p != NULL; p = p->ai_next) {
-                peerinfo = gd_peerinfo_find_from_addrinfo (p);
-                if (peerinfo) {
-                        freeaddrinfo (addr);
-                        return peerinfo;
-                }
+    int ret = -1;
+    struct addrinfo *addr = NULL;
+    struct addrinfo *p = NULL;
+    xlator_t *this = THIS;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    GF_ASSERT(hoststr);
+
+    peerinfo = gd_peerinfo_find_from_hostname(hoststr);
+    if (peerinfo)
+        return peerinfo;
+
+    ret = getaddrinfo(hoststr, NULL, NULL, &addr);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, GD_MSG_GETADDRINFO_FAIL,
+               "error in getaddrinfo: %s\n", gai_strerror(ret));
+        goto out;
+    }
+
+    for (p = addr; p != NULL; p = p->ai_next) {
+        peerinfo = gd_peerinfo_find_from_addrinfo(p);
+        if (peerinfo) {
+            freeaddrinfo(addr);
+            return peerinfo;
         }
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Unable to find friend: %s", hoststr);
-        if (addr)
-                freeaddrinfo (addr);
-        return NULL;
+    gf_msg_debug(this->name, 0, "Unable to find friend: %s", hoststr);
+    if (addr)
+        freeaddrinfo(addr);
+    return NULL;
 }
 
 int
-glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
+glusterd_hostname_to_uuid(char *hostname, uuid_t uuid)
 {
-        GF_ASSERT (hostname);
-        GF_ASSERT (uuid);
+    GF_ASSERT(hostname);
+    GF_ASSERT(uuid);
 
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-        int                     ret = -1;
-        xlator_t                *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        peerinfo = glusterd_peerinfo_find_by_hostname (hostname);
-        if (peerinfo) {
-                ret = 0;
-                gf_uuid_copy (uuid, peerinfo->uuid);
+    peerinfo = glusterd_peerinfo_find_by_hostname(hostname);
+    if (peerinfo) {
+        ret = 0;
+        gf_uuid_copy(uuid, peerinfo->uuid);
+    } else {
+        if (gf_is_local_addr(hostname)) {
+            gf_uuid_copy(uuid, MY_UUID);
+            ret = 0;
         } else {
-                if (gf_is_local_addr (hostname)) {
-                        gf_uuid_copy (uuid, MY_UUID);
-                        ret = 0;
-                } else {
-                        ret = -1;
-                }
+            ret = -1;
         }
+    }
 
-        gf_msg_debug (this->name, 0, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 /* glusterd_peerinfo_find_by_uuid searches for a peer which matches the
@@ -174,40 +271,41 @@ glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
  * Returns NULL otherwise.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_uuid (uuid_t uuid)
+glusterd_peerinfo_find_by_uuid(uuid_t uuid)
 {
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *entry = NULL;
-        glusterd_peerinfo_t     *found = NULL;
-        xlator_t                *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    xlator_t *this = THIS;
+    glusterd_friend_sm_state_t state;
 
-        this = THIS;
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        priv    = this->private;
-
-        GF_ASSERT (priv);
+    if (gf_uuid_is_null(uuid))
+        return NULL;
 
-        if (gf_uuid_is_null (uuid))
-                return NULL;
+    priv = this->private;
 
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (entry, &priv->peers, uuid_list) {
-                if (!gf_uuid_compare (entry->uuid, uuid)) {
+    GF_ASSERT(priv);
 
-                        gf_msg_debug (this->name, 0,
-                                 "Friend found... state: %s",
-                        glusterd_friend_sm_state_name_get (entry->state.state));
-                        found = entry; /* Probably should be rcu_dereferenced */
-                        break;
-                }
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+    {
+        if (!gf_uuid_compare(entry->uuid, uuid)) {
+            found = entry; /* Probably should be rcu_dereferenced */
+            state = found->state.state;
+            break;
         }
-        rcu_read_unlock ();
-
-        if (!found)
-                gf_msg_debug (this->name, 0,
-                        "Friend with uuid: %s, not found", uuid_utoa (uuid));
-        return found;
+    }
+    RCU_READ_UNLOCK;
+
+    if (found)
+        gf_msg_debug(this->name, 0, "Friend found... state: %s",
+                     glusterd_friend_sm_state_name_get(state));
+    else
+        gf_msg_debug(this->name, 0, "Friend with uuid: %s, not found",
+                     uuid_utoa(uuid));
+    return found;
 }
 
 /* glusterd_peerinfo_find will search for a peer matching either @uuid or
@@ -215,39 +313,35 @@ glusterd_peerinfo_find_by_uuid (uuid_t uuid)
  * Returns NULL otherwise.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_find (uuid_t uuid, const char *hostname)
+glusterd_peerinfo_find(uuid_t uuid, const char *hostname)
 {
-        glusterd_peerinfo_t *peerinfo = NULL;
-        xlator_t            *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
+    glusterd_peerinfo_t *peerinfo = NULL;
+    xlator_t *this = THIS;
 
-        if (uuid) {
-                peerinfo = glusterd_peerinfo_find_by_uuid (uuid);
+    GF_ASSERT(this);
 
-                if (peerinfo) {
-                        return peerinfo;
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                 "Unable to find peer by uuid: %s",
-                                 uuid_utoa (uuid));
-                }
+    if (uuid) {
+        peerinfo = glusterd_peerinfo_find_by_uuid(uuid);
 
+        if (peerinfo) {
+            return peerinfo;
+        } else {
+            gf_msg_debug(this->name, 0, "Unable to find peer by uuid: %s",
+                         uuid_utoa(uuid));
         }
+    }
 
-        if (hostname) {
-                peerinfo = glusterd_peerinfo_find_by_hostname (hostname);
+    if (hostname) {
+        peerinfo = glusterd_peerinfo_find_by_hostname(hostname);
 
-                if (peerinfo) {
-                        return peerinfo;
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                "Unable to find hostname: %s", hostname);
-                }
+        if (peerinfo) {
+            return peerinfo;
+        } else {
+            gf_msg_debug(this->name, 0, "Unable to find hostname: %s",
+                         hostname);
         }
-        return NULL;
+    }
+    return NULL;
 }
 
 /* glusterd_peerinfo_new will create a new peerinfo object and set it's members
@@ -259,267 +353,299 @@ glusterd_peerinfo_find (uuid_t uuid, const char *hostname)
  * object.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_new (glusterd_friend_sm_state_t state, uuid_t *uuid,
-                       const char *hostname, int port)
+glusterd_peerinfo_new(glusterd_friend_sm_state_t state, uuid_t *uuid,
+                      const char *hostname, int port)
 {
-        glusterd_peerinfo_t *new_peer = NULL;
-        int                  ret      = -1;
-        xlator_t            *this     = NULL;
-        glusterd_conf_t     *conf     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        new_peer = GF_CALLOC (1, sizeof (*new_peer), gf_gld_mt_peerinfo_t);
-        if (!new_peer)
-                goto out;
+    glusterd_peerinfo_t *new_peer = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        CDS_INIT_LIST_HEAD (&new_peer->uuid_list);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        new_peer->state.state = state;
+    new_peer = GF_CALLOC(1, sizeof(*new_peer), gf_gld_mt_peerinfo_t);
+    if (!new_peer) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        CDS_INIT_LIST_HEAD (&new_peer->hostnames);
-        if (hostname) {
-                ret = gd_add_address_to_peer (new_peer, hostname);
-                if (ret)
-                        goto out;
-                /* Also set it to peerinfo->hostname. Doing this as we use
-                 * peerinfo->hostname in a lot of places and is really hard to
-                 * get everything right
-                 */
-                new_peer->hostname = gf_strdup (hostname);
-        }
+    CDS_INIT_LIST_HEAD(&new_peer->uuid_list);
 
-        if (uuid) {
-                gf_uuid_copy (new_peer->uuid, *uuid);
-        }
+    new_peer->state.state = state;
 
-        ret = glusterd_sm_tr_log_init (&new_peer->sm_log,
-                                       glusterd_friend_sm_state_name_get,
-                                       glusterd_friend_sm_event_name_get,
-                                       GLUSTERD_TR_LOG_SIZE);
+    CDS_INIT_LIST_HEAD(&new_peer->hostnames);
+    if (hostname) {
+        ret = gd_add_address_to_peer(new_peer, hostname);
         if (ret)
-                goto out;
+            goto out;
+        /* Also set it to peerinfo->hostname. Doing this as we use
+         * peerinfo->hostname in a lot of places and is really hard to
+         * get everything right
+         */
+        new_peer->hostname = gf_strdup(hostname);
+    }
+
+    if (uuid) {
+        gf_uuid_copy(new_peer->uuid, *uuid);
+    }
 
-        if (new_peer->state.state == GD_FRIEND_STATE_BEFRIENDED)
-                new_peer->quorum_contrib = QUORUM_WAITING;
-        new_peer->port = port;
+    ret = glusterd_sm_tr_log_init(
+        &new_peer->sm_log, glusterd_friend_sm_state_name_get,
+        glusterd_friend_sm_event_name_get, GLUSTERD_TR_LOG_SIZE);
+    if (ret)
+        goto out;
 
-        pthread_mutex_init (&new_peer->delete_lock, NULL);
+    if (new_peer->state.state == GD_FRIEND_STATE_BEFRIENDED)
+        new_peer->quorum_contrib = QUORUM_WAITING;
+    new_peer->port = port;
 
-        new_peer->generation = uatomic_add_return (&conf->generation, 1);
+    pthread_mutex_init(&new_peer->delete_lock, NULL);
+
+    new_peer->generation = uatomic_add_return(&conf->generation, 1);
 out:
-        if (ret && new_peer) {
-                glusterd_peerinfo_cleanup (new_peer);
-                new_peer = NULL;
-        }
-        return new_peer;
+    if (ret && new_peer) {
+        glusterd_peerinfo_cleanup(new_peer);
+        new_peer = NULL;
+    }
+    return new_peer;
 }
 
 /* Check if the all peers are connected and befriended, except the peer
  * specified (the peer being detached)
  */
 gf_boolean_t
-glusterd_chk_peers_connected_befriended (uuid_t skip_uuid)
+glusterd_chk_peers_connected_befriended(uuid_t skip_uuid)
 {
-        gf_boolean_t            ret = _gf_true;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-
-        priv= THIS->private;
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-
-                if (!gf_uuid_is_null (skip_uuid) && !gf_uuid_compare (skip_uuid,
-                                                           peerinfo->uuid))
-                        continue;
-
-                if ((GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state)
-                    || !(peerinfo->connected)) {
-                        ret = _gf_false;
-                        break;
-                }
+    gf_boolean_t ret = _gf_true;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!gf_uuid_is_null(skip_uuid) &&
+            !gf_uuid_compare(skip_uuid, peerinfo->uuid))
+            continue;
+
+        if ((GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state) ||
+            !(peerinfo->connected)) {
+            ret = _gf_false;
+            break;
         }
-        rcu_read_unlock ();
+    }
+    RCU_READ_UNLOCK;
 
-        gf_msg_debug (THIS->name, 0, "Returning %s",
-                (ret?"TRUE":"FALSE"));
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %s", (ret ? "TRUE" : "FALSE"));
+    return ret;
 }
 
 /* Return hostname for given uuid if it exists
  * else return NULL
  */
 char *
-glusterd_uuid_to_hostname (uuid_t uuid)
+glusterd_uuid_to_hostname(uuid_t uuid)
 {
-        char                    *hostname = NULL;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *entry = NULL;
+    char *hostname = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        if (!gf_uuid_compare (MY_UUID, uuid)) {
-                hostname = gf_strdup ("localhost");
-        }
-        rcu_read_lock ();
-        if (!cds_list_empty (&priv->peers)) {
-                cds_list_for_each_entry_rcu (entry, &priv->peers, uuid_list) {
-                        if (!gf_uuid_compare (entry->uuid, uuid)) {
-                                hostname = gf_strdup (entry->hostname);
-                                break;
-                        }
-                }
+    if (!gf_uuid_compare(MY_UUID, uuid)) {
+        hostname = gf_strdup("localhost");
+        return hostname;
+    }
+    RCU_READ_LOCK;
+    if (!cds_list_empty(&priv->peers)) {
+        cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+        {
+            if (!gf_uuid_compare(entry->uuid, uuid)) {
+                hostname = gf_strdup(entry->hostname);
+                break;
+            }
         }
-        rcu_read_unlock ();
+    }
+    RCU_READ_UNLOCK;
 
-        return hostname;
+    return hostname;
 }
 
-char*
-gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo)
+char *
+gd_peer_uuid_str(glusterd_peerinfo_t *peerinfo)
 {
-        if ((peerinfo == NULL) || gf_uuid_is_null (peerinfo->uuid))
-                return NULL;
+    if ((peerinfo == NULL) || gf_uuid_is_null(peerinfo->uuid))
+        return NULL;
 
-        if (peerinfo->uuid_str[0] == '\0')
-                uuid_utoa_r (peerinfo->uuid, peerinfo->uuid_str);
+    if (peerinfo->uuid_str[0] == '\0')
+        uuid_utoa_r(peerinfo->uuid, peerinfo->uuid_str);
 
-        return peerinfo->uuid_str;
+    return peerinfo->uuid_str;
 }
 
 gf_boolean_t
-glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
-                               struct cds_list_head *peers,
-                               char **down_peerstr)
+glusterd_are_all_peers_up()
 {
-        glusterd_peerinfo_t   *peerinfo  = NULL;
-        glusterd_brickinfo_t  *brickinfo = NULL;
-        gf_boolean_t           ret       = _gf_false;
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
-
-                rcu_read_lock ();
-                cds_list_for_each_entry_rcu (peerinfo, peers, uuid_list) {
-                        if (gf_uuid_compare (peerinfo->uuid, brickinfo->uuid))
-                                continue;
-
-                        /*Found peer who owns the brick, return false
-                         * if peer is not connected or not friend */
-                        if (!(peerinfo->connected) ||
-                           (peerinfo->state.state !=
-                             GD_FRIEND_STATE_BEFRIENDED)) {
-                                *down_peerstr = gf_strdup (peerinfo->hostname);
-                                gf_msg_debug (THIS->name, 0, "Peer %s is down. ",
-                                        peerinfo->hostname);
-                                rcu_read_unlock ();
-                                goto out;
-                        }
-                }
-                rcu_read_unlock ();
+    glusterd_peerinfo_t *peerinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t peers_up = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            goto out;
         }
+    }
+    RCU_READ_UNLOCK;
+
+    peers_up = _gf_true;
 
-        ret = _gf_true;
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    return peers_up;
+}
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up(glusterd_volinfo_t *volinfo,
+                              struct cds_list_head *peers, char **down_peerstr)
+{
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gf_boolean_t ret = _gf_false;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        RCU_READ_LOCK;
+        cds_list_for_each_entry_rcu(peerinfo, peers, uuid_list)
+        {
+            if (gf_uuid_compare(peerinfo->uuid, brickinfo->uuid))
+                continue;
+
+            /*Found peer who owns the brick, return false
+             * if peer is not connected or not friend */
+            if (!(peerinfo->connected) ||
+                (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)) {
+                *down_peerstr = gf_strdup(peerinfo->hostname);
+                RCU_READ_UNLOCK;
+                gf_msg_debug(THIS->name, 0, "Peer %s is down. ", *down_peerstr);
+                goto out;
+            }
+        }
+        RCU_READ_UNLOCK;
+    }
+
+    ret = _gf_true;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_peer_hostname_new (const char *hostname,
-                            glusterd_peer_hostname_t **name)
+glusterd_peer_hostname_new(const char *hostname,
+                           glusterd_peer_hostname_t **name)
 {
-        glusterd_peer_hostname_t        *peer_hostname = NULL;
-        int32_t                         ret = -1;
+    glusterd_peer_hostname_t *peer_hostname = NULL;
+    int32_t ret = -1;
 
-        GF_ASSERT (hostname);
-        GF_ASSERT (name);
+    GF_ASSERT(hostname);
+    GF_ASSERT(name);
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        peer_hostname = GF_CALLOC (1, sizeof (*peer_hostname),
-                                   gf_gld_mt_peer_hostname_t);
+    peer_hostname = GF_CALLOC(1, sizeof(*peer_hostname),
+                              gf_gld_mt_peer_hostname_t);
 
-        if (!peer_hostname)
-                goto out;
+    if (!peer_hostname) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        peer_hostname->hostname = gf_strdup (hostname);
-        CDS_INIT_LIST_HEAD (&peer_hostname->hostname_list);
+    peer_hostname->hostname = gf_strdup(hostname);
+    CDS_INIT_LIST_HEAD(&peer_hostname->hostname_list);
 
-        *name = peer_hostname;
-        ret = 0;
+    *name = peer_hostname;
+    ret = 0;
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 void
-glusterd_peer_hostname_free (glusterd_peer_hostname_t *name)
+glusterd_peer_hostname_free(glusterd_peer_hostname_t *name)
 {
-        if (!name)
-                return;
+    if (!name)
+        return;
 
-        cds_list_del_init (&name->hostname_list);
+    cds_list_del_init(&name->hostname_list);
 
-        GF_FREE (name->hostname);
-        name->hostname = NULL;
+    GF_FREE(name->hostname);
+    name->hostname = NULL;
 
-        GF_FREE (name);
+    GF_FREE(name);
 
-        return;
+    return;
 }
 
 gf_boolean_t
-gd_peer_has_address (glusterd_peerinfo_t *peerinfo, const char *address)
+gd_peer_has_address(glusterd_peerinfo_t *peerinfo, const char *address)
 {
-        gf_boolean_t ret = _gf_false;
-        glusterd_peer_hostname_t *hostname = NULL;
+    glusterd_peer_hostname_t *hostname = NULL;
 
-        GF_VALIDATE_OR_GOTO ("glusterd", (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO ("glusterd", (address != NULL), out);
+    GF_VALIDATE_OR_GOTO("glusterd", (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO("glusterd", (address != NULL), out);
 
-        cds_list_for_each_entry (hostname, &peerinfo->hostnames,
-                                 hostname_list) {
-                if (strcmp (hostname->hostname, address) == 0) {
-                        ret = _gf_true;
-                        break;
-                }
+    cds_list_for_each_entry(hostname, &peerinfo->hostnames, hostname_list)
+    {
+        if (strcmp(hostname->hostname, address) == 0) {
+            return _gf_true;
         }
+    }
 
 out:
-        return ret;
+    return _gf_false;
 }
 
 int
-gd_add_address_to_peer (glusterd_peerinfo_t *peerinfo, const char *address)
+gd_add_address_to_peer(glusterd_peerinfo_t *peerinfo, const char *address)
 {
+    int ret = -1;
+    glusterd_peer_hostname_t *hostname = NULL;
 
-        int ret = -1;
-        glusterd_peer_hostname_t *hostname = NULL;
+    GF_VALIDATE_OR_GOTO("glusterd", (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO("glusterd", (address != NULL), out);
 
-        GF_VALIDATE_OR_GOTO ("glusterd", (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO ("glusterd", (address != NULL), out);
+    if (gd_peer_has_address(peerinfo, address)) {
+        ret = 0;
+        goto out;
+    }
 
-        if (gd_peer_has_address (peerinfo, address)) {
-                ret = 0;
-                goto out;
-        }
+    ret = glusterd_peer_hostname_new(address, &hostname);
+    if (ret)
+        goto out;
 
-        ret = glusterd_peer_hostname_new (address, &hostname);
-        if (ret)
-                goto out;
+    cds_list_add_tail_rcu(&hostname->hostname_list, &peerinfo->hostnames);
 
-        cds_list_add_tail_rcu (&hostname->hostname_list, &peerinfo->hostnames);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* gd_add_friend_to_dict() adds details of @friend into @dict with the given
@@ -529,199 +655,78 @@ out:
  * is >= GD_OP_VERSION_3_6_0
  */
 int
-gd_add_friend_to_dict (glusterd_peerinfo_t *friend, dict_t *dict,
-                       const char *prefix)
+gd_add_friend_to_dict(glusterd_peerinfo_t *friend, dict_t *dict,
+                      const char *prefix)
 {
-        int                       ret      = -1;
-        xlator_t                 *this     = NULL;
-        glusterd_conf_t          *conf     = NULL;
-        char                      key[100] = {0,};
-        glusterd_peer_hostname_t *address  = NULL;
-        int                       count    = 0;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", (this != NULL), out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (friend != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[100] = {
+        0,
+    };
+    glusterd_peer_hostname_t *address = NULL;
+    int count = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", (this != NULL), out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (friend != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    snprintf(key, sizeof(key), "%s.uuid", prefix);
+    ret = dict_set_dynstr_with_alloc(dict, key, uuid_utoa(friend->uuid));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set key %s in dict", key);
+        goto out;
+    }
+
+    /* Setting the first hostname from the list with this key for backward
+     * compatibility
+     */
+    snprintf(key, sizeof(key), "%s.hostname", prefix);
+    address = cds_list_entry(&friend->hostnames, glusterd_peer_hostname_t,
+                             hostname_list);
+    ret = dict_set_dynstr_with_alloc(dict, key, address->hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set key %s in dict", key);
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
 
-        snprintf (key, sizeof (key), "%s.uuid", prefix);
-        ret = dict_set_dynstr_with_alloc (dict, key, uuid_utoa (friend->uuid));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set key %s in dict", key);
-                goto out;
-        }
+    address = NULL;
+    count = 0;
+    cds_list_for_each_entry(address, &friend->hostnames, hostname_list)
+    {
+        GF_VALIDATE_OR_GOTO(this->name, (address != NULL), out);
 
-        /* Setting the first hostname from the list with this key for backward
-         * compatibility
-         */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.hostname", prefix);
-        address = cds_list_entry (&friend->hostnames, glusterd_peer_hostname_t,
-                                  hostname_list);
-        if (!address) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_ADDRESS_GET_FAIL,
-                        "Could not retrieve first "
-                        "address for peer");
-                goto out;
-        }
-        ret = dict_set_dynstr_with_alloc (dict, key, address->hostname);
+        snprintf(key, sizeof(key), "%s.hostname%d", prefix, count);
+        ret = dict_set_dynstr_with_alloc(dict, key, address->hostname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set key %s in dict", key);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set key %s in dict", key);
+            goto out;
         }
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        address = NULL;
-        count = 0;
-        cds_list_for_each_entry (address, &friend->hostnames, hostname_list) {
-                GF_VALIDATE_OR_GOTO (this->name, (address != NULL), out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, count);
-                ret = dict_set_dynstr_with_alloc (dict, key, address->hostname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set key %s in dict", key);
-                        goto out;
-                }
-                count++;
-        }
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.address-count", prefix);
-        ret = dict_set_int32 (dict, key, count);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set key %s in dict", key);
+        count++;
+    }
+    ret = snprintf(key, sizeof(key), "%s.address-count", prefix);
+    ret = dict_set_int32n(dict, key, ret, count);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set key %s in dict", key);
 
 out:
-        gf_msg_debug (this ? this->name : "glusterd", 0, "Returning %d",
-                ret);
-        return ret;
-}
-
-/* gd_peerinfo_find_from_hostname iterates over all the addresses saved for each
- * peer and matches it to @hoststr.
- * Returns the matched peer if found else returns NULL
- */
-glusterd_peerinfo_t *
-gd_peerinfo_find_from_hostname (const char *hoststr)
-{
-        xlator_t                 *this    = NULL;
-        glusterd_conf_t          *priv    = NULL;
-        glusterd_peerinfo_t      *peer    = NULL;
-        glusterd_peerinfo_t      *found   = NULL;
-        glusterd_peer_hostname_t *tmphost = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (priv != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (hoststr != NULL), out);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peer, &priv->peers, uuid_list) {
-                cds_list_for_each_entry_rcu (tmphost, &peer->hostnames,
-                                             hostname_list) {
-                        if (!strncasecmp (tmphost->hostname, hoststr, 1024)) {
-                                gf_msg_debug (this->name, 0,
-                                        "Friend %s found.. state: %d",
-                                        tmphost->hostname, peer->state.state);
-                                found = peer; /* Probably needs to be
-                                                 dereferenced*/
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        rcu_read_unlock ();
-out:
-        return found;
-}
-
-/* gd_peerinfo_find_from_addrinfo iterates over all the addresses saved for each
- * peer, resolves them and compares them to @addr.
- *
- *
- * NOTE: As getaddrinfo is a blocking call and is being performed multiple times
- * in this function, it could lead to the calling thread to be blocked for
- * significant amounts of time.
- *
- * Returns the matched peer if found else returns NULL
- */
-glusterd_peerinfo_t *
-gd_peerinfo_find_from_addrinfo (const struct addrinfo *addr)
-{
-        xlator_t                 *this    = NULL;
-        glusterd_conf_t          *conf    = NULL;
-        glusterd_peerinfo_t      *peer    = NULL;
-        glusterd_peerinfo_t      *found   = NULL;
-        glusterd_peer_hostname_t *address = NULL;
-        int                       ret     = 0;
-        struct addrinfo          *paddr   = NULL;
-        struct addrinfo          *tmp     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (addr != NULL), out);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peer, &conf->peers, uuid_list) {
-                cds_list_for_each_entry_rcu (address, &peer->hostnames,
-                                             hostname_list) {
-                        /* TODO: Cache the resolved addrinfos to improve
-                         * performance
-                         */
-                        ret = getaddrinfo (address->hostname, NULL, NULL,
-                                           &paddr);
-                        if (ret) {
-                                /* Don't fail if getaddrinfo fails, continue
-                                 * onto the next address
-                                 */
-                                gf_msg_trace (this->name, 0,
-                                        "getaddrinfo for %s failed (%s)",
-                                        address->hostname, gai_strerror (ret));
-                                ret = 0;
-                                continue;
-                        }
-
-                        for (tmp = paddr; tmp != NULL; tmp = tmp->ai_next) {
-                                if (gf_compare_sockaddr (addr->ai_addr,
-                                                         tmp->ai_addr)) {
-                                        found = peer; /* (de)referenced? */
-                                        break;
-                                }
-                        }
-
-                        freeaddrinfo (paddr);
-                        if (found)
-                                goto unlock;
-                }
-        }
-unlock:
-        rcu_read_unlock ();
-out:
-        return found;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 /* gd_update_peerinfo_from_dict will update the hostnames for @peerinfo from
@@ -729,87 +734,87 @@ out:
  * Returns 0 on success and -1 on failure.
  */
 int
-gd_update_peerinfo_from_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                              const char *prefix)
+gd_update_peerinfo_from_dict(glusterd_peerinfo_t *peerinfo, dict_t *dict,
+                             const char *prefix)
 {
-        int                  ret      = -1;
-        xlator_t            *this     = NULL;
-        glusterd_conf_t     *conf     = NULL;
-        char                 key[100] = {0,};
-        char                *hostname = NULL;
-        int                  count    = 0;
-        int                  i        = 0;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.hostname", prefix);
-        ret = dict_get_str (dict, key, &hostname);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[100] = {
+        0,
+    };
+    char *hostname = NULL;
+    int count = 0;
+    int i = 0;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    ret = snprintf(key, sizeof(key), "%s.hostname", prefix);
+    ret = dict_get_strn(dict, key, ret, &hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Key %s not present in "
+               "dictionary",
+               key);
+        goto out;
+    }
+    ret = gd_add_address_to_peer(peerinfo, hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
+               "Could not add address to peer");
+        goto out;
+    }
+    /* Also set peerinfo->hostname to the first address */
+    if (peerinfo->hostname != NULL)
+        GF_FREE(peerinfo->hostname);
+    peerinfo->hostname = gf_strdup(hostname);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "%s.address-count", prefix);
+    ret = dict_get_int32n(dict, key, ret, &count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Key %s not present in "
+               "dictionary",
+               key);
+        goto out;
+    }
+    hostname = NULL;
+    for (i = 0; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        ret = dict_get_strn(dict, key, ret, &hostname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Key %s not present in "
-                        "dictionary", key);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Key %s not present "
+                   "in dictionary",
+                   key);
+            goto out;
         }
-        ret = gd_add_address_to_peer (peerinfo, hostname);
+        ret = gd_add_address_to_peer(peerinfo, hostname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
-                        "Could not add address to peer");
-                goto out;
-        }
-        /* Also set peerinfo->hostname to the first address */
-        if (peerinfo->hostname != NULL)
-                GF_FREE (peerinfo->hostname);
-        peerinfo->hostname = gf_strdup (hostname);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
+                   "Could not add address to peer");
+            goto out;
         }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.address-count", prefix);
-        ret = dict_get_int32 (dict, key, &count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Key %s not present in "
-                        "dictionary", key);
-                goto out;
-        }
         hostname = NULL;
-        for (i = 0; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d",prefix, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Key %s not present "
-                                "in dictionary", key);
-                        goto out;
-                }
-                ret = gd_add_address_to_peer (peerinfo, hostname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
-                                "Could not add address to peer");
-                        goto out;
-                }
-
-                hostname = NULL;
-        }
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* gd_peerinfo_from_dict creates a peerinfo object from details of peer with
@@ -818,154 +823,176 @@ out:
  * failure.
  */
 glusterd_peerinfo_t *
-gd_peerinfo_from_dict (dict_t *dict, const char *prefix)
+gd_peerinfo_from_dict(dict_t *dict, const char *prefix)
 {
-        int                  ret      = -1;
-        xlator_t            *this     = NULL;
-        glusterd_conf_t     *conf     = NULL;
-        glusterd_peerinfo_t *new_peer = NULL;
-        char                 key[100] = {0,};
-        char                *uuid_str = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", (this != NULL), out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        new_peer = glusterd_peerinfo_new (GD_FRIEND_STATE_DEFAULT, NULL, NULL,
-                                          0);
-        if (new_peer == NULL) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEERINFO_CREATE_FAIL,
-                        "Could not create peerinfo "
-                        "object");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.uuid", prefix);
-        ret = dict_get_str (dict, key, &uuid_str);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Key %s not present in "
-                        "dictionary", key);
-                goto out;
-        }
-        gf_uuid_parse (uuid_str, new_peer->uuid);
-
-        ret = gd_update_peerinfo_from_dict (new_peer, dict, prefix);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *new_peer = NULL;
+    char key[64] = {
+        0,
+    };
+    char *uuid_str = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", (this != NULL), out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    new_peer = glusterd_peerinfo_new(GD_FRIEND_STATE_DEFAULT, NULL, NULL, 0);
+    if (new_peer == NULL) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+               "Could not create peerinfo "
+               "object");
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "%s.uuid", prefix);
+    ret = dict_get_strn(dict, key, ret, &uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Key %s not present in "
+               "dictionary",
+               key);
+        goto out;
+    }
+    gf_uuid_parse(uuid_str, new_peer->uuid);
+
+    ret = gd_update_peerinfo_from_dict(new_peer, dict, prefix);
 
 out:
-        if ((ret != 0) && (new_peer != NULL)) {
-                glusterd_peerinfo_cleanup (new_peer);
-                new_peer = NULL;
-        }
+    if ((ret != 0) && (new_peer != NULL)) {
+        glusterd_peerinfo_cleanup(new_peer);
+        new_peer = NULL;
+    }
 
-        return new_peer;
+    return new_peer;
 }
 
-int
-gd_add_peer_hostnames_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                               const char *prefix)
+static int
+gd_add_peer_hostnames_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *dict,
+                              const char *prefix)
 {
-        int                       ret      = -1;
-        xlator_t                 *this     = NULL;
-        glusterd_conf_t          *conf     = NULL;
-        char                      key[256] = {0,};
-        glusterd_peer_hostname_t *addr     = NULL;
-        int                       count    = 0;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[64] = {
+        0,
+    };
+    glusterd_peer_hostname_t *addr = NULL;
+    int count = 0;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
 
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO (this->name, (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        cds_list_for_each_entry (addr, &peerinfo->hostnames, hostname_list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, count);
-                ret = dict_set_dynstr_with_alloc (dict, key, addr->hostname);
-                if (ret)
-                        goto out;
-                count++;
+    cds_list_for_each_entry(addr, &peerinfo->hostnames, hostname_list)
+    {
+        snprintf(key, sizeof(key), "%s.hostname%d", prefix, count);
+        ret = dict_set_dynstr_with_alloc(dict, key, addr->hostname);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
+        count++;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.hostname_count", prefix);
-        ret = dict_set_int32 (dict, key, count);
+    ret = snprintf(key, sizeof(key), "%s.hostname_count", prefix);
+    ret = dict_set_int32n(dict, key, ret, count);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-gd_add_peer_detail_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *friends,
-                            int count)
+gd_add_peer_detail_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *friends,
+                           int count)
 {
-
-        int             ret = -1;
-        char            key[256] = {0, };
-        char           *peer_uuid_str = NULL;
-
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (friends);
-
-        snprintf (key, sizeof (key), "friend%d.uuid", count);
-        peer_uuid_str = gd_peer_uuid_str (peerinfo);
-        ret = dict_set_str (friends, key, peer_uuid_str);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.hostname", count);
-        ret = dict_set_str (friends, key, peerinfo->hostname);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.port", count);
-        ret = dict_set_int32 (friends, key, peerinfo->port);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.stateId", count);
-        ret = dict_set_int32 (friends, key, peerinfo->state.state);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.state", count);
-        ret = dict_set_str (friends, key,
-                    glusterd_friend_sm_state_name_get(peerinfo->state.state));
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.connected", count);
-        ret = dict_set_int32 (friends, key, (int32_t)peerinfo->connected);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d", count);
-        ret = gd_add_peer_hostnames_to_dict (peerinfo, friends, key);
+    int ret = -1;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *peer_uuid_str = NULL;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(friends);
+
+    peer_uuid_str = gd_peer_uuid_str(peerinfo);
+    keylen = snprintf(key, sizeof(key), "friend%d.uuid", count);
+    ret = dict_set_strn(friends, key, keylen, peer_uuid_str);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.hostname", count);
+    ret = dict_set_strn(friends, key, keylen, peerinfo->hostname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.port", count);
+    ret = dict_set_int32n(friends, key, keylen, peerinfo->port);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.stateId", count);
+    ret = dict_set_int32n(friends, key, keylen, peerinfo->state.state);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=%s in dict", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.state", count);
+    ret = dict_set_strn(
+        friends, key, keylen,
+        glusterd_friend_sm_state_name_get(peerinfo->state.state));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.connected", count);
+    ret = dict_set_int32n(friends, key, keylen, (int32_t)peerinfo->connected);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "friend%d", count);
+    ret = gd_add_peer_hostnames_to_dict(peerinfo, friends, key);
 
 out:
-        return ret;
+    return ret;
 }
 
 /* glusterd_peerinfo_find_by_generation searches for a peer which has the
@@ -973,35 +1000,59 @@ out:
  * object. Returns NULL otherwise.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_generation (uint32_t generation) {
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *entry = NULL;
-        glusterd_peerinfo_t     *found = NULL;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+glusterd_peerinfo_find_by_generation(uint32_t generation)
+{
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    xlator_t *this = THIS;
+    glusterd_friend_sm_state_t state;
 
-        priv    = this->private;
+    GF_ASSERT(this);
 
-        GF_ASSERT (priv);
+    priv = this->private;
 
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (entry, &priv->peers, uuid_list) {
-                if (entry->generation == generation) {
+    GF_ASSERT(priv);
 
-                        gf_msg_debug (this->name, 0,
-                                 "Friend found... state: %s",
-                        glusterd_friend_sm_state_name_get (entry->state.state));
-                        found = entry; /* Probably should be rcu_dereferenced */
-                        break;
-                }
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+    {
+        if (entry->generation == generation) {
+            found = entry; /* Probably should be rcu_dereferenced */
+            state = found->state.state;
+            break;
         }
-        rcu_read_unlock ();
+    }
+    RCU_READ_UNLOCK;
+
+    if (found)
+        gf_msg_debug(this->name, 0, "Friend found... state: %s",
+                     glusterd_friend_sm_state_name_get(state));
+    else
+        gf_msg_debug(this->name, 0,
+                     "Friend with generation: %" PRIu32 ", not found",
+                     generation);
+    return found;
+}
 
-        if (!found)
-                gf_msg_debug (this->name, 0,
-                        "Friend with generation: %"PRIu32", not found",
-                        generation);
-        return found;
+int
+glusterd_get_peers_count()
+{
+    int count = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *peer = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &conf->peers, uuid_list) count++;
+    RCU_READ_UNLOCK;
+
+out:
+    return count;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-peer-utils.h b/xlators/mgmt/glusterd/src/glusterd-peer-utils.h
index bd30e335f69..fd254d57391 100644
--- a/xlators/mgmt/glusterd/src/glusterd-peer-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-peer-utils.h
@@ -12,76 +12,71 @@
 #define _GLUSTERD_PEER_UTILS_H
 
 #include "glusterd.h"
-#include "glusterd-utils.h"
 
 int32_t
-glusterd_peerinfo_cleanup (glusterd_peerinfo_t *peerinfo);
+glusterd_peerinfo_cleanup(glusterd_peerinfo_t *peerinfo);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_hostname (const char *hoststr);
+glusterd_peerinfo_find_by_hostname(const char *hoststr);
 
 int
-glusterd_hostname_to_uuid (char *hostname, uuid_t uuid);
+glusterd_hostname_to_uuid(char *hostname, uuid_t uuid);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_uuid (uuid_t uuid);
+glusterd_peerinfo_find_by_uuid(uuid_t uuid);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_find (uuid_t uuid, const char *hostname);
+glusterd_peerinfo_find(uuid_t uuid, const char *hostname);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_new (glusterd_friend_sm_state_t state, uuid_t *uuid,
-                       const char *hostname, int port);
+glusterd_peerinfo_new(glusterd_friend_sm_state_t state, uuid_t *uuid,
+                      const char *hostname, int port);
 
 gf_boolean_t
-glusterd_chk_peers_connected_befriended (uuid_t skip_uuid);
+glusterd_chk_peers_connected_befriended(uuid_t skip_uuid);
 
 char *
-glusterd_uuid_to_hostname (uuid_t uuid);
+glusterd_uuid_to_hostname(uuid_t uuid);
 
-char*
-gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo);
+char *
+gd_peer_uuid_str(glusterd_peerinfo_t *peerinfo);
+
+gf_boolean_t
+glusterd_are_all_peers_up();
 
 gf_boolean_t
-glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
-                               struct cds_list_head *peers,
-                               char **down_peerstr);
+glusterd_are_vol_all_peers_up(glusterd_volinfo_t *volinfo,
+                              struct cds_list_head *peers, char **down_peerstr);
 
 int32_t
-glusterd_peer_hostname_new (const char *hostname,
-                            glusterd_peer_hostname_t **name);
+glusterd_peer_hostname_new(const char *hostname,
+                           glusterd_peer_hostname_t **name);
 void
-glusterd_peer_hostname_free (glusterd_peer_hostname_t *name);
+glusterd_peer_hostname_free(glusterd_peer_hostname_t *name);
 
 gf_boolean_t
-gd_peer_has_address (glusterd_peerinfo_t *peerinfo, const char *address);
+gd_peer_has_address(glusterd_peerinfo_t *peerinfo, const char *address);
 
 int
-gd_add_address_to_peer (glusterd_peerinfo_t *peerinfo, const char *address);
+gd_add_address_to_peer(glusterd_peerinfo_t *peerinfo, const char *address);
 
 int
-gd_add_friend_to_dict (glusterd_peerinfo_t *friend, dict_t *dict,
-                       const char *prefix);
+gd_add_friend_to_dict(glusterd_peerinfo_t *friend, dict_t *dict,
+                      const char *prefix);
 
-glusterd_peerinfo_t *
-gd_peerinfo_find_from_hostname (const char *hoststr);
+int
+gd_update_peerinfo_from_dict(glusterd_peerinfo_t *peerinfo, dict_t *dict,
+                             const char *prefix);
 
 glusterd_peerinfo_t *
-gd_peerinfo_find_from_addrinfo (const struct addrinfo *addr);
+gd_peerinfo_from_dict(dict_t *dict, const char *prefix);
 
 int
-gd_update_peerinfo_from_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                              const char *prefix);
-
+gd_add_peer_detail_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *friends,
+                           int count);
 glusterd_peerinfo_t *
-gd_peerinfo_from_dict (dict_t *dict, const char *prefix);
+glusterd_peerinfo_find_by_generation(uint32_t generation);
 
 int
-gd_add_peer_hostnames_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                               const char *prefix);
-int
-gd_add_peer_detail_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *friends,
-                            int count);
-glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_generation (uint32_t generation);
+glusterd_get_peers_count();
 #endif /* _GLUSTERD_PEER_UTILS_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index cbc800595ab..16ac628ab82 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -8,10 +8,10 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "glusterfs.h"
-#include "syscall.h"
-#include "compat-errno.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat-errno.h>
 
 #include "glusterd.h"
 #include "glusterd-utils.h"
@@ -26,477 +26,641 @@
 #include <sys/types.h>
 #include <netinet/in.h>
 
-
-int
-pmap_port_isfree (int port)
+static int
+pmap_port_isfree(int port)
 {
-        struct sockaddr_in sin;
-        int                sock = -1;
-        int                ret = 0;
+    struct sockaddr_in sin;
+    int sock = -1;
+    int ret = 0;
 
-        memset (&sin, 0, sizeof (sin));
-        sin.sin_family = PF_INET;
-        sin.sin_port = hton16 (port);
+    memset(&sin, 0, sizeof(sin));
+    sin.sin_family = PF_INET;
+    sin.sin_port = hton16(port);
 
-        sock = socket (PF_INET, SOCK_STREAM, 0);
-        if (sock == -1)
-                return -1;
+    sock = socket(PF_INET, SOCK_STREAM, 0);
+    if (sock == -1)
+        return -1;
 
-        ret = bind (sock, (struct sockaddr *)&sin, sizeof (sin));
-        sys_close (sock);
+    ret = bind(sock, (struct sockaddr *)&sin, sizeof(sin));
+    sys_close(sock);
 
-        return (ret == 0) ? 1 : 0;
+    return (ret == 0) ? 1 : 0;
 }
 
-
 static struct pmap_registry *
-pmap_registry_new (xlator_t *this)
+pmap_registry_new(xlator_t *this)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   i = 0;
-
-        pmap = CALLOC (sizeof (*pmap), 1);
-        if (!pmap)
-                return NULL;
-
-        pmap->base_port = pmap->last_alloc =
-                ((glusterd_conf_t *)(this->private))->base_port;
-
-        for (i = pmap->base_port; i <= GF_PORT_MAX; i++) {
-                if (pmap_port_isfree (i))
-                        pmap->ports[i].type = GF_PMAP_PORT_FREE;
-                else
-                        pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
-        }
-
-        return pmap;
+    struct pmap_registry *pmap = NULL;
+    int i = 0;
+
+    pmap = CALLOC(sizeof(*pmap), 1);
+    if (!pmap)
+        return NULL;
+
+    pmap->base_port = pmap->last_alloc = ((glusterd_conf_t *)(this->private))
+                                             ->base_port;
+    pmap->max_port = ((glusterd_conf_t *)(this->private))->max_port;
+    for (i = pmap->base_port; i <= pmap->max_port; i++) {
+        if (pmap_port_isfree(i))
+            pmap->ports[i].type = GF_PMAP_PORT_FREE;
+        else
+            pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
+    }
+
+    return pmap;
 }
 
-
 struct pmap_registry *
-pmap_registry_get (xlator_t *this)
+pmap_registry_get(xlator_t *this)
 {
-        glusterd_conf_t      *priv = NULL;
-        struct pmap_registry *pmap = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct pmap_registry *pmap = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        pmap = priv->pmap;
-        if (!pmap) {
-                pmap = pmap_registry_new (this);
-                if (!pmap)
-                        return NULL;
-                priv->pmap = pmap;
-        }
-
-        return pmap;
-}
-
-
-static char*
-nextword (char *str)
-{
-        while (*str && !isspace (*str))
-                str++;
-        while (*str && isspace (*str))
-                str++;
+    pmap = priv->pmap;
+    if (!pmap) {
+        pmap = pmap_registry_new(this);
+        if (!pmap)
+            return NULL;
+        priv->pmap = pmap;
+    }
 
-        return str;
+    return pmap;
 }
 
+/*
+ * The "destroy" argument avoids a double search in pmap_registry_remove - one
+ * to find the entry in the table, and the other to find the particular
+ * brickname within that entry (which might cover multiple bricks).  We do the
+ * actual deletion here by "whiting out" the brick name with spaces.  It's up
+ * to pmap_registry_remove to figure out what to do from there.
+ */
 int
-pmap_registry_search (xlator_t *this, const char *brickname,
-                      gf_pmap_port_type_t type)
+pmap_registry_search(xlator_t *this, const char *brickname,
+                     gf_pmap_port_type_t type, gf_boolean_t destroy)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-        char                 *brck = NULL;
-        char                 *nbrck = NULL;
-
-        pmap = pmap_registry_get (this);
-
-        for (p = pmap->base_port; p <= pmap->last_alloc; p++) {
-                if (!pmap->ports[p].brickname || pmap->ports[p].type != type)
-                        continue;
-
-                for (brck = pmap->ports[p].brickname;;) {
-                        nbrck = strtail (brck, brickname);
-                        if (nbrck && (!*nbrck || isspace (*nbrck)))
-                                return p;
-                        brck = nextword (brck);
-                        if (!*brck)
-                                break;
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    char *brck = NULL;
+    size_t i;
+
+    pmap = pmap_registry_get(this);
+
+    for (p = pmap->last_alloc; p >= pmap->base_port; p--) {
+        if (!pmap->ports[p].brickname || pmap->ports[p].type != type)
+            continue;
+
+        brck = pmap->ports[p].brickname;
+        for (;;) {
+            for (i = 0; brck[i] && !isspace(brck[i]); ++i)
+                ;
+            if (i == 0 && brck[i] == '\0')
+                break;
+
+            if (strncmp(brck, brickname, i) == 0) {
+                /*
+                 * Without this check, we'd break when brck
+                 * is merely a substring of brickname.
+                 */
+                if (brickname[i] == '\0') {
+                    if (destroy)
+                        do {
+                            *(brck++) = ' ';
+                        } while (--i);
+                    return p;
                 }
+            }
+
+            brck += i;
+
+            /*
+             * Skip over *any* amount of whitespace, including
+             * none (if we're already at the end of the string).
+             */
+            while (isspace(*brck))
+                ++brck;
+            /*
+             * We're either at the end of the string (which will be
+             * handled above strncmp on the next iteration) or at
+             * the next non-whitespace substring (which will be
+             * handled by strncmp itself).
+             */
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-pmap_registry_search_by_xprt (xlator_t *this, void *xprt,
-                              gf_pmap_port_type_t type)
+static int
+pmap_registry_search_by_xprt(xlator_t *this, void *xprt,
+                             gf_pmap_port_type_t type)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p    = 0;
-        int                   port = 0;
-
-        pmap = pmap_registry_get (this);
-
-        for (p = pmap->base_port; p <= pmap->last_alloc; p++) {
-                if (!pmap->ports[p].xprt)
-                        continue;
-                if (pmap->ports[p].xprt == xprt &&
-                    pmap->ports[p].type == type) {
-                        port = p;
-                        break;
-                }
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    int port = 0;
+
+    pmap = pmap_registry_get(this);
+
+    for (p = pmap->last_alloc; p >= pmap->base_port; p--) {
+        if (!pmap->ports[p].xprt)
+            continue;
+        if (pmap->ports[p].xprt == xprt) {
+            if (pmap->ports[p].type == type || type == GF_PMAP_PORT_ANY) {
+                port = p;
+                break;
+            }
         }
+    }
 
-        return port;
+    return port;
 }
 
-
-char *
-pmap_registry_search_by_port (xlator_t *this, int port)
+static char *
+pmap_registry_search_by_port(xlator_t *this, int port)
 {
-        struct pmap_registry *pmap = NULL;
-        char *brickname = NULL;
+    struct pmap_registry *pmap = NULL;
+    char *brickname = NULL;
+    int max_port = 0;
 
-        if (port > GF_PORT_MAX)
-                goto out;
+    max_port = ((glusterd_conf_t *)(this->private))->max_port;
+    if (port > max_port)
+        goto out;
 
-        pmap = pmap_registry_get (this);
+    pmap = pmap_registry_get(this);
 
-        if (pmap->ports[port].type == GF_PMAP_PORT_BRICKSERVER)
-                brickname = pmap->ports[port].brickname;
+    if (pmap->ports[port].type == GF_PMAP_PORT_BRICKSERVER)
+        brickname = pmap->ports[port].brickname;
 
 out:
-        return brickname;
+    return brickname;
 }
 
-
 int
-pmap_registry_alloc (xlator_t *this)
+pmap_registry_alloc(xlator_t *this)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-        int                   port = 0;
-
-        pmap = pmap_registry_get (this);
-
-        for (p = pmap->last_alloc; p <= GF_PORT_MAX; p++) {
-                /* GF_PMAP_PORT_FOREIGN may be freed up ? */
-                if ((pmap->ports[p].type == GF_PMAP_PORT_FREE) ||
-                    (pmap->ports[p].type == GF_PMAP_PORT_FOREIGN)) {
-
-                        if (pmap_port_isfree (p)) {
-                                pmap->ports[p].type = GF_PMAP_PORT_LEASED;
-                                port = p;
-                                break;
-                        }
-                }
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    int port = 0;
+
+    pmap = pmap_registry_get(this);
+
+    for (p = pmap->base_port; p <= pmap->max_port; p++) {
+        /* GF_PMAP_PORT_FOREIGN may be freed up ? */
+        if ((pmap->ports[p].type == GF_PMAP_PORT_FREE) ||
+            (pmap->ports[p].type == GF_PMAP_PORT_FOREIGN)) {
+            if (pmap_port_isfree(p)) {
+                pmap->ports[p].type = GF_PMAP_PORT_LEASED;
+                port = p;
+                break;
+            }
         }
+    }
 
-        if (port)
-                pmap->last_alloc = port;
+    if (port > pmap->last_alloc)
+        pmap->last_alloc = port;
 
-        return port;
+    return port;
 }
 
+/* pmap_assign_port does a pmap_registry_remove followed by pmap_registry_alloc,
+ * the reason for the former is to ensure we don't end up with stale ports
+ */
 int
-pmap_registry_bind (xlator_t *this, int port, const char *brickname,
-                    gf_pmap_port_type_t type, void *xprt)
+pmap_assign_port(xlator_t *this, int old_port, const char *path)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-
-        pmap = pmap_registry_get (this);
+    int ret = -1;
+    int new_port = 0;
+
+    if (old_port) {
+        ret = pmap_registry_remove(this, 0, path, GF_PMAP_PORT_BRICKSERVER,
+                                   NULL, _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, GD_MSG_PMAP_REGISTRY_REMOVE_FAIL,
+                   0,
+                   "Failed to"
+                   "remove pmap registry for older signin for path"
+                   " %s",
+                   path);
+        }
+    }
+    new_port = pmap_registry_alloc(this);
+    return new_port;
+}
 
-        if (port > GF_PORT_MAX)
-                goto out;
+int
+pmap_registry_bind(xlator_t *this, int port, const char *brickname,
+                   gf_pmap_port_type_t type, void *xprt)
+{
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
 
-        p = port;
-        pmap->ports[p].type = type;
-        free (pmap->ports[p].brickname);
-        pmap->ports[p].brickname = strdup (brickname);
-        pmap->ports[p].type = type;
-        pmap->ports[p].xprt = xprt;
+    pmap = pmap_registry_get(this);
 
-        gf_msg ("pmap", GF_LOG_INFO, 0,
-                GD_MSG_BRICK_ADD, "adding brick %s on port %d",
-                brickname, port);
+    if (port > pmap->max_port)
+        goto out;
 
-        if (pmap->last_alloc < p)
-                pmap->last_alloc = p;
+    p = port;
+    if (pmap->ports[p].type == GF_PMAP_PORT_FREE) {
+        /* Because of some crazy race in volume start code path because
+         * of friend handshaking with volumes with quorum enabled we
+         * might end up into a situation where glusterd would start a
+         * brick and get a disconnect and then immediately try to start
+         * the same brick instance based on another friend update
+         * request. And then if for the very first brick even if the
+         * process doesn't come up at the end sign in event gets sent
+         * and we end up having two duplicate portmap entries for the
+         * same brick. Since in brick start we mark the previous port as
+         * free, its better to consider a sign in request as no op if
+         * the corresponding port type is marked as free
+         */
+        goto out;
+    }
+    if (pmap->ports[p].brickname) {
+        char *tmp = pmap->ports[p].brickname;
+        asprintf(&pmap->ports[p].brickname, "%s %s", tmp, brickname);
+        free(tmp);
+    } else {
+        pmap->ports[p].brickname = strdup(brickname);
+    }
+    pmap->ports[p].type = type;
+    pmap->ports[p].xprt = xprt;
+
+    gf_msg("pmap", GF_LOG_INFO, 0, GD_MSG_BRICK_ADD,
+           "adding brick %s on port %d", brickname, port);
+
+    if (pmap->last_alloc < p)
+        pmap->last_alloc = p;
 out:
-        return 0;
+    return 0;
 }
 
 int
-pmap_registry_remove (xlator_t *this, int port, const char *brickname,
-                      gf_pmap_port_type_t type, void *xprt)
+pmap_registry_extend(xlator_t *this, int port, const char *brickname)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-        glusterd_conf_t      *priv = NULL;
-
-        priv = this->private;
-        pmap = priv->pmap;
-        if (!pmap)
-                goto out;
-
-        if (port) {
-                if (port > GF_PORT_MAX)
-                        goto out;
-
-                p = port;
-                goto remove;
+    struct pmap_registry *pmap = NULL;
+    char *old_bn;
+    char *new_bn;
+    size_t bn_len;
+    char *entry;
+    int found = 0;
+
+    pmap = pmap_registry_get(this);
+
+    if (port > pmap->max_port) {
+        return -1;
+    }
+
+    switch (pmap->ports[port].type) {
+        case GF_PMAP_PORT_LEASED:
+        case GF_PMAP_PORT_BRICKSERVER:
+            break;
+        default:
+            return -1;
+    }
+
+    old_bn = pmap->ports[port].brickname;
+    if (old_bn) {
+        bn_len = strlen(brickname);
+        entry = strstr(old_bn, brickname);
+        while (entry) {
+            found = 1;
+            if ((entry != old_bn) && (entry[-1] != ' ')) {
+                found = 0;
+            }
+            if ((entry[bn_len] != ' ') && (entry[bn_len] != '\0')) {
+                found = 0;
+            }
+            if (found) {
+                return 0;
+            }
+            entry = strstr(entry + bn_len, brickname);
         }
+        asprintf(&new_bn, "%s %s", old_bn, brickname);
+    } else {
+        new_bn = strdup(brickname);
+    }
 
-        if (brickname && strchr (brickname, '/')) {
-                p = pmap_registry_search (this, brickname, type);
-                if (p)
-                        goto remove;
-        }
-
-        if (xprt) {
-                p = pmap_registry_search_by_xprt (this, xprt, type);
-                if (p)
-                        goto remove;
-        }
+    if (!new_bn) {
+        return -1;
+    }
 
-        goto out;
-remove:
-        gf_msg ("pmap", GF_LOG_INFO, 0,
-                GD_MSG_BRICK_REMOVE, "removing brick %s on port %d",
-                pmap->ports[p].brickname, p);
-
-        free (pmap->ports[p].brickname);
-
-        pmap->ports[p].type = GF_PMAP_PORT_FREE;
-        pmap->ports[p].brickname = NULL;
-        pmap->ports[p].xprt = NULL;
+    pmap->ports[port].brickname = new_bn;
+    free(old_bn);
 
-out:
-        return 0;
+    return 0;
 }
 
 int
-__gluster_pmap_portbybrick (rpcsvc_request_t *req)
+pmap_registry_remove(xlator_t *this, int port, const char *brickname,
+                     gf_pmap_port_type_t type, void *xprt,
+                     gf_boolean_t brick_disconnect)
 {
-        pmap_port_by_brick_req    args = {0,};
-        pmap_port_by_brick_rsp    rsp  = {0,};
-        char                     *brick = NULL;
-        int                       port = 0;
-        int                       ret = -1;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_port_by_brick_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        brick = args.brick;
-
-        port = pmap_registry_search (THIS, brick, GF_PMAP_PORT_BRICKSERVER);
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    glusterd_conf_t *priv = NULL;
+    char *brick_str;
+
+    priv = this->private;
+    pmap = priv->pmap;
+    if (!pmap)
+        goto out;
 
-        if (!port)
-                rsp.op_ret = -1;
+    if (port) {
+        if (port > pmap->max_port)
+            goto out;
+    }
 
-        rsp.port = port;
+    if (brickname) {
+        p = pmap_registry_search(this, brickname, type, _gf_true);
+        if (p)
+            goto remove;
+    }
 
-fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        free (args.brick);//malloced by xdr
+    if (xprt) {
+        p = pmap_registry_search_by_xprt(this, xprt, type);
+        if (p)
+            goto remove;
+    }
 
-        return 0;
-}
+    goto out;
+remove:
+    gf_msg("pmap", GF_LOG_INFO, 0, GD_MSG_BRICK_REMOVE,
+           "removing brick %s on port %d", brickname, p);
 
+    if (xprt && (xprt == pmap->ports[p].xprt)) {
+        pmap->ports[p].xprt = NULL;
+    }
+
+    /*
+     * This is where we garbage-collect.  If all of the brick names have
+     * been "whited out" by pmap_registry_search(...,destroy=_gf_true) and
+     * there's no xprt either, then we have nothing left worth saving and
+     * can delete the entire entry.
+     */
+    if (brick_disconnect || !pmap->ports[p].xprt) {
+        /* If the signout call is being triggered by brick disconnect
+         * then clean up all the bricks (in case of brick mux)
+         */
+        if (!brick_disconnect) {
+            brick_str = pmap->ports[p].brickname;
+            if (brick_str) {
+                while (*brick_str != '\0') {
+                    if (*(brick_str++) != ' ') {
+                        goto out;
+                    }
+                }
+            }
+        }
+        free(pmap->ports[p].brickname);
+        pmap->ports[p].brickname = NULL;
+        pmap->ports[p].type = GF_PMAP_PORT_FREE;
+    }
 
-int
-gluster_pmap_portbybrick (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __gluster_pmap_portbybrick);
+out:
+    return 0;
 }
 
-
 int
-__gluster_pmap_brickbyport (rpcsvc_request_t *req)
+__gluster_pmap_portbybrick(rpcsvc_request_t *req)
 {
-        pmap_brick_by_port_req    args = {0,};
-        pmap_brick_by_port_rsp    rsp  = {0,};
-        int                       ret = -1;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_brick_by_port_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
+    pmap_port_by_brick_req args = {
+        0,
+    };
+    pmap_port_by_brick_rsp rsp = {
+        0,
+    };
+    char *brick = NULL;
+    int port = 0;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &args,
+                         (xdrproc_t)xdr_pmap_port_by_brick_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    brick = args.brick;
+
+    port = pmap_registry_search(this, brick, GF_PMAP_PORT_BRICKSERVER,
+                                _gf_false);
+
+    if (!port)
+        rsp.op_ret = -1;
+
+    rsp.port = port;
 
-        rsp.brick = pmap_registry_search_by_port (THIS, args.port);
-        if (!rsp.brick) {
-                rsp.op_ret = -1;
-                rsp.brick = "";
-        }
 fail:
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+    free(args.brick);  // malloced by xdr
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_brick_by_port_rsp);
-
-        return 0;
+    return 0;
 }
 
-
 int
-gluster_pmap_brickbyport (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __gluster_pmap_brickbyport);
-}
-
-
-static int
-glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
-                              gf_boolean_t value)
+gluster_pmap_portbybrick(rpcsvc_request_t *req)
 {
-        brickinfo->signed_in = value;
-
-        return 0;
+    return glusterd_big_locked_handler(req, __gluster_pmap_portbybrick);
 }
 
 int
-__gluster_pmap_signup (rpcsvc_request_t *req)
+__gluster_pmap_brickbyport(rpcsvc_request_t *req)
 {
-        pmap_signup_req    args = {0,};
-        pmap_signup_rsp    rsp  = {0,};
-        int                ret = -1;
-
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_signup_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick,
-                                         GF_PMAP_PORT_BRICKSERVER, req->trans);
-
+    pmap_brick_by_port_req args = {
+        0,
+    };
+    pmap_brick_by_port_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &args,
+                         (xdrproc_t)xdr_pmap_brick_by_port_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    rsp.brick = pmap_registry_search_by_port(THIS, args.port);
+    if (!rsp.brick) {
+        rsp.op_ret = -1;
+        rsp.brick = "";
+    }
 fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_signup_rsp);
-        free (args.brick);//malloced by xdr
 
-        return 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_brick_by_port_rsp);
+
+    return 0;
 }
 
 int
-gluster_pmap_signup (rpcsvc_request_t *req)
+gluster_pmap_brickbyport(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __gluster_pmap_signup);
+    return glusterd_big_locked_handler(req, __gluster_pmap_brickbyport);
 }
 
 int
-__gluster_pmap_signin (rpcsvc_request_t *req)
+__gluster_pmap_signin(rpcsvc_request_t *req)
 {
-        pmap_signin_req    args = {0,};
-        pmap_signin_rsp    rsp  = {0,};
-        glusterd_brickinfo_t *brickinfo = NULL;
-        int                ret = -1;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_signin_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick,
-                                         GF_PMAP_PORT_BRICKSERVER, req->trans);
+    pmap_signin_req args = {
+        0,
+    };
+    pmap_signin_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_pmap_signin_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    rsp.op_ret = pmap_registry_bind(THIS, args.port, args.brick,
+                                    GF_PMAP_PORT_BRICKSERVER, req->trans);
+
+    ret = glusterd_get_brickinfo(THIS, args.brick, args.port, &brickinfo);
+    /* Update portmap status in brickinfo */
+    if (brickinfo)
+        brickinfo->port_registered = _gf_true;
 
-        ret = glusterd_get_brickinfo (THIS, args.brick, args.port, _gf_true,
-                                      &brickinfo);
 fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_signin_rsp);
-        free (args.brick);//malloced by xdr
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_signin_rsp);
+    free(args.brick);  // malloced by xdr
 
-        if (!ret)
-                glusterd_brick_update_signin (brickinfo, _gf_true);
-
-        return 0;
+    return 0;
 }
 
-
 int
-gluster_pmap_signin (rpcsvc_request_t *req)
+gluster_pmap_signin(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __gluster_pmap_signin);
+    return glusterd_big_locked_handler(req, __gluster_pmap_signin);
 }
 
-
 int
-__gluster_pmap_signout (rpcsvc_request_t *req)
+__gluster_pmap_signout(rpcsvc_request_t *req)
 {
-        pmap_signout_req    args                 = {0,};
-        pmap_signout_rsp    rsp                  = {0,};
-        int                 ret                  = -1;
-        char                brick_path[PATH_MAX] = {0,};
-        glusterd_brickinfo_t *brickinfo = NULL;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_signout_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
+    pmap_signout_req args = {
+        0,
+    };
+    pmap_signout_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char pidfile[PATH_MAX] = {0};
+    char brick_path[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, fail);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, fail);
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_pmap_signout_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+    rsp.op_ret = pmap_registry_remove(THIS, args.port, args.brick,
+                                      GF_PMAP_PORT_BRICKSERVER, req->trans,
+                                      _gf_false);
+
+    ret = glusterd_get_brickinfo(THIS, args.brick, args.port, &brickinfo);
+    if (args.rdma_port) {
+        snprintf(brick_path, PATH_MAX, "%s.rdma", args.brick);
+        rsp.op_ret = pmap_registry_remove(THIS, args.rdma_port, brick_path,
+                                          GF_PMAP_PORT_BRICKSERVER, req->trans,
+                                          _gf_false);
+    }
+    /* Update portmap status on brickinfo */
+    if (brickinfo)
+        brickinfo->port_registered = _gf_false;
+
+    /* Clean up the pidfile for this brick given glusterfsd doesn't clean it
+     * any more. This is required to ensure we don't end up with having
+     * stale pid files in case a brick is killed from the backend
+     */
+    ret = glusterd_get_volinfo_from_brick(args.brick, &volinfo);
+    if (!ret) {
+        if (volinfo && brickinfo) {
+            GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+            sys_unlink(pidfile);
+
+            /* Setting the brick status to GF_BRICK_STOPPED to
+             * ensure correct brick status is maintained on the
+             * glusterd end when a brick is killed from the
+             * backend */
+            brickinfo->status = GF_BRICK_STOPPED;
+
+            /* Remove brick from brick process if not already
+             * removed in the brick op phase. This situation would
+             * arise when the brick is killed explicitly from the
+             * backend */
+            ret = glusterd_brick_process_remove_brick(brickinfo, NULL);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "Couldn't remove "
+                             "brick %s:%s from brick process",
+                             brickinfo->hostname, brickinfo->path);
+                /* Ignore 'ret' here since the brick might
+                 * have already been deleted in brick op phase
+                 */
+                ret = 0;
+            }
         }
-
-        rsp.op_ret = pmap_registry_remove (THIS, args.port, args.brick,
-                                           GF_PMAP_PORT_BRICKSERVER, req->trans);
-
-        ret = glusterd_get_brickinfo (THIS, args.brick, args.port, _gf_true,
-                                      &brickinfo);
-        if (args.rdma_port) {
-                snprintf(brick_path, PATH_MAX, "%s.rdma", args.brick);
-                rsp.op_ret = pmap_registry_remove (THIS, args.rdma_port,
-                                brick_path, GF_PMAP_PORT_BRICKSERVER,
-                                req->trans);
-        }
-
-        if (!ret)
-                glusterd_brick_update_signin (brickinfo, _gf_false);
+    }
 
 fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_signout_rsp);
-        free (args.brick);//malloced by xdr
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_signout_rsp);
+    free(args.brick);  // malloced by xdr
 
-        return 0;
+    return 0;
 }
 
 int
-gluster_pmap_signout (rpcsvc_request_t *req)
+gluster_pmap_signout(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __gluster_pmap_signout);
+    return glusterd_big_locked_handler(req, __gluster_pmap_signout);
 }
 
-rpcsvc_actor_t gluster_pmap_actors[GF_PMAP_MAXVALUE] = {
-        [GF_PMAP_NULL]        = {"NULL",        GF_PMAP_NULL,        NULL,                     NULL, 0, DRC_NA},
-        [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK, gluster_pmap_portbybrick, NULL, 0, DRC_NA},
-        [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT, gluster_pmap_brickbyport, NULL, 0, DRC_NA},
-        [GF_PMAP_SIGNUP]      = {"SIGNUP",      GF_PMAP_SIGNUP,      gluster_pmap_signup,      NULL, 0, DRC_NA},
-        [GF_PMAP_SIGNIN]      = {"SIGNIN",      GF_PMAP_SIGNIN,      gluster_pmap_signin,      NULL, 0, DRC_NA},
-        [GF_PMAP_SIGNOUT]     = {"SIGNOUT",     GF_PMAP_SIGNOUT,     gluster_pmap_signout,     NULL, 0, DRC_NA},
+static rpcsvc_actor_t gluster_pmap_actors[GF_PMAP_MAXVALUE] = {
+    [GF_PMAP_NULL] = {"NULL", NULL, NULL, GF_PMAP_NULL, DRC_NA, 0},
+    [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", gluster_pmap_portbybrick, NULL,
+                             GF_PMAP_PORTBYBRICK, DRC_NA, 0},
+    [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", gluster_pmap_brickbyport, NULL,
+                             GF_PMAP_BRICKBYPORT, DRC_NA, 0},
+    [GF_PMAP_SIGNIN] = {"SIGNIN", gluster_pmap_signin, NULL, GF_PMAP_SIGNIN,
+                        DRC_NA, 0},
+    [GF_PMAP_SIGNOUT] = {"SIGNOUT", gluster_pmap_signout, NULL, GF_PMAP_SIGNOUT,
+                         DRC_NA, 0},
 };
 
-
 struct rpcsvc_program gluster_pmap_prog = {
-        .progname  = "Gluster Portmap",
-        .prognum   = GLUSTER_PMAP_PROGRAM,
-        .progver   = GLUSTER_PMAP_VERSION,
-        .actors    = gluster_pmap_actors,
-        .numactors = GF_PMAP_MAXVALUE,
+    .progname = "Gluster Portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
+    .actors = gluster_pmap_actors,
+    .numactors = GF_PMAP_MAXVALUE,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
index 95ded04208d..51d75361431 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.h
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -11,37 +11,47 @@
 #define _GLUSTERD_PMAP_H_
 
 #include <pthread.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
-#include "glusterd.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "rpcsvc.h"
 
-
 struct pmap_port_status {
-        gf_pmap_port_type_t type;
-        char  *brickname;
-        void  *xprt;
+    char *brickname;
+    void *xprt;
+    gf_pmap_port_type_t type;
 };
 
 struct pmap_registry {
-        int     base_port;
-        int     last_alloc;
-        struct  pmap_port_status ports[65536];
+    struct pmap_port_status ports[GF_PORT_MAX + 1];
+    int base_port;
+    int max_port;
+    int last_alloc;
 };
 
-int pmap_registry_alloc (xlator_t *this);
-int pmap_registry_bind (xlator_t *this, int port, const char *brickname,
-                        gf_pmap_port_type_t type, void *xprt);
-int pmap_registry_remove (xlator_t *this, int port, const char *brickname,
-                          gf_pmap_port_type_t type, void *xprt);
-int pmap_registry_search (xlator_t *this, const char *brickname,
-                          gf_pmap_port_type_t type);
-struct pmap_registry *pmap_registry_get (xlator_t *this);
+int
+pmap_assign_port(xlator_t *this, int port, const char *path);
+int
+pmap_mark_port_leased(xlator_t *this, int port);
+int
+pmap_registry_alloc(xlator_t *this);
+int
+pmap_registry_bind(xlator_t *this, int port, const char *brickname,
+                   gf_pmap_port_type_t type, void *xprt);
+int
+pmap_registry_extend(xlator_t *this, int port, const char *brickname);
+int
+pmap_registry_remove(xlator_t *this, int port, const char *brickname,
+                     gf_pmap_port_type_t type, void *xprt,
+                     gf_boolean_t brick_disconnect);
+int
+pmap_registry_search(xlator_t *this, const char *brickname,
+                     gf_pmap_port_type_t type, gf_boolean_t destroy);
+struct pmap_registry *
+pmap_registry_get(xlator_t *this);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
index 9f934629330..a05c90d7b10 100644
--- a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
@@ -12,124 +12,141 @@
 #include <limits.h>
 #include <signal.h>
 
-#include "common-utils.h"
-#include "xlator.h"
-#include "logging.h"
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
 #include "glusterd-messages.h"
 #include "glusterd-proc-mgmt.h"
 
 int
-glusterd_proc_init (glusterd_proc_t *proc, char *name, char *pidfile,
-                    char *logdir, char *logfile, char *volfile, char *volfileid,
-                    char *volfileserver)
+glusterd_proc_init(glusterd_proc_t *proc, char *name, char *pidfile,
+                   char *logdir, char *logfile, char *volfile, char *volfileid,
+                   char *volfileserver)
 {
-        int ret = -1;
+    int ret = -1;
 
-        ret = snprintf (proc->name, sizeof (proc->name), "%s", name);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->name, sizeof(proc->name), "%s", name);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->pidfile, sizeof (proc->pidfile), "%s", pidfile);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->pidfile, sizeof(proc->pidfile), "%s", pidfile);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->logdir, sizeof (proc->logdir), "%s", logdir);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->logdir, sizeof(proc->logdir), "%s", logdir);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->logfile, sizeof (proc->logfile), "%s", logfile);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->logfile, sizeof(proc->logfile), "%s", logfile);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->volfile, sizeof (proc->volfile), "%s", volfile);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->volfile, sizeof(proc->volfile), "%s", volfile);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->volfileid, sizeof (proc->volfileid), "%s",
-                        volfileid);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->volfileid, sizeof(proc->volfileid), "%s", volfileid);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->volfileserver, sizeof (proc->volfileserver), "%s",
-                        volfileserver);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->volfileserver, sizeof(proc->volfileserver), "%s",
+                   volfileserver);
+    if (ret < 0)
+        goto out;
 
 out:
-        if (ret > 0)
-                ret = 0;
+    if (ret > 0)
+        ret = 0;
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_proc_stop (glusterd_proc_t *proc, int sig, int flags)
+glusterd_proc_stop(glusterd_proc_t *proc, int sig, int flags)
 {
+    /* NB: Copy-paste code from glusterd_service_stop, the source may be
+     * removed once all daemon management use proc */
 
-        /* NB: Copy-paste code from glusterd_service_stop, the source may be
-         * removed once all daemon management use proc */
+    int32_t ret = -1;
+    pid_t pid = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        int32_t  ret    = -1;
-        pid_t    pid    = -1;
-        xlator_t *this  = NULL;
+    this = THIS;
+    GF_ASSERT(this);
 
-        this = THIS;
-        GF_ASSERT (this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        if (!gf_is_service_running (proc->pidfile, &pid)) {
+    if (!gf_is_service_running(proc->pidfile, &pid)) {
+        ret = 0;
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_ALREADY_STOPPED,
+               "%s already stopped", proc->name);
+        goto out;
+    }
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
+           "Stopping %s daemon running in pid: "
+           "%d",
+           proc->name, pid);
+
+    ret = kill(pid, sig);
+    if (ret) {
+        switch (errno) {
+            case ESRCH:
+                gf_msg_debug(this->name, 0,
+                             "%s is already "
+                             "stopped",
+                             proc->name);
                 ret = 0;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_ALREADY_STOPPED, "%s already stopped",
-                        proc->name);
                 goto out;
+            default:
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SVC_KILL_FAIL,
+                       "Unable to kill %s "
+                       "service, reason:%s",
+                       proc->name, strerror(errno));
         }
-        gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
-                "Stopping %s daemon running in pid: " "%d", proc->name, pid);
-
-        ret = kill (pid, sig);
+    } else {
+        (void)glusterd_unlink_file(proc->pidfile);
+    }
+    if (flags != PROC_STOP_FORCE)
+        goto out;
+
+    synclock_unlock(&conf->big_lock);
+    synctask_sleep(1);
+    synclock_lock(&conf->big_lock);
+    if (gf_is_service_running(proc->pidfile, &pid)) {
+        ret = kill(pid, SIGKILL);
         if (ret) {
-                switch (errno) {
-                case ESRCH:
-                        gf_msg_debug (this->name, 0, "%s is already "
-                                "stopped", proc->name);
-                        ret = 0;
-                        goto out;
-                default:
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_SVC_KILL_FAIL, "Unable to kill %s "
-                                "service, reason:%s", proc->name,
-                                strerror (errno));
-                }
-        }
-        if (flags != PROC_STOP_FORCE)
-                goto out;
-
-        sleep (1);
-        if (gf_is_service_running (proc->pidfile, NULL)) {
-                ret = kill (pid, SIGKILL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_PID_KILL_FAIL, "Unable to kill pid:%d, "
-                                "reason:%s", pid, strerror(errno));
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                   "Unable to kill pid:%d, "
+                   "reason:%s",
+                   pid, strerror(errno));
+            goto out;
         }
+        ret = glusterd_unlink_file(proc->pidfile);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_proc_get_pid (glusterd_proc_t *proc)
+glusterd_proc_get_pid(glusterd_proc_t *proc)
 {
-        int pid = -1;
-        (void) gf_is_service_running (proc->pidfile, &pid);
-        return pid;
+    int pid = -1;
+    (void)gf_is_service_running(proc->pidfile, &pid);
+    return pid;
 }
 
 int
-glusterd_proc_is_running (glusterd_proc_t *proc)
+glusterd_proc_is_running(glusterd_proc_t *proc)
 {
-        return gf_is_service_running (proc->pidfile, NULL);
+    int pid = -1;
+
+    return gf_is_service_running(proc->pidfile, &pid);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h
index f5235171816..e8e9ffc5082 100644
--- a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h
@@ -14,31 +14,31 @@
 typedef struct glusterd_proc_ glusterd_proc_t;
 
 enum proc_flags {
-        PROC_NONE = 0,
-        PROC_START,
-        PROC_START_NO_WAIT,
-        PROC_STOP,
-        PROC_STOP_FORCE
+    PROC_NONE = 0,
+    PROC_START,
+    PROC_START_NO_WAIT,
+    PROC_STOP,
+    PROC_STOP_FORCE
 };
 
 struct glusterd_proc_ {
-        char name[PATH_MAX];
-        char pidfile[PATH_MAX];
-        char logdir[PATH_MAX];
-        char logfile[PATH_MAX];
-        char volfile[PATH_MAX];
-        char volfileserver[PATH_MAX];
-        char volfileid[256];
+    char name[NAME_MAX];
+    char pidfile[PATH_MAX];
+    char logdir[PATH_MAX];
+    char logfile[PATH_MAX];
+    char volfile[PATH_MAX];
+    char volfileserver[PATH_MAX];
+    char volfileid[256];
 };
 
 int
-glusterd_proc_init (glusterd_proc_t *proc, char *name, char *pidfile,
-                    char *logdir, char *logfile, char *volfile, char *volfileid,
-                    char *volfileserver);
+glusterd_proc_init(glusterd_proc_t *proc, char *name, char *pidfile,
+                   char *logdir, char *logfile, char *volfile, char *volfileid,
+                   char *volfileserver);
 
 int
-glusterd_proc_stop (glusterd_proc_t *proc, int sig, int flags);
+glusterd_proc_stop(glusterd_proc_t *proc, int sig, int flags);
 
 int
-glusterd_proc_is_running (glusterd_proc_t *proc);
+glusterd_proc_is_running(glusterd_proc_t *proc);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index 55699cc57a9..8370c174ce3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -7,800 +7,886 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
-#include "run.h"
-#include "syscall.h"
-#include "byte-order.h"
-#include "compat-errno.h"
-#include "quota-common-utils.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/quota-common-utils.h>
+#include "glusterd-quota.h"
 
 #include <sys/wait.h>
 #include <dlfcn.h>
 
 #ifndef _PATH_SETFATTR
-# ifdef GF_LINUX_HOST_OS
-#  define _PATH_SETFATTR "/usr/bin/setfattr"
-# endif
-# ifdef __NetBSD__
-#  define _PATH_SETFATTR "/usr/pkg/bin/setfattr"
-# endif
+#ifdef GF_LINUX_HOST_OS
+#define _PATH_SETFATTR "setfattr"
+#endif
+#ifdef __NetBSD__
+#define _PATH_SETFATTR "/usr/pkg/bin/setfattr"
+#endif
 #endif
 
 /* Any negative pid to make it special client */
 #define QUOTA_CRAWL_PID "-100"
 
-const char *gd_quota_op_list[GF_QUOTA_OPTION_TYPE_MAX + 1] = {
-        [GF_QUOTA_OPTION_TYPE_NONE]               = "none",
-        [GF_QUOTA_OPTION_TYPE_ENABLE]             = "enable",
-        [GF_QUOTA_OPTION_TYPE_DISABLE]            = "disable",
-        [GF_QUOTA_OPTION_TYPE_LIMIT_USAGE]        = "limit-usage",
-        [GF_QUOTA_OPTION_TYPE_REMOVE]             = "remove",
-        [GF_QUOTA_OPTION_TYPE_LIST]               = "list",
-        [GF_QUOTA_OPTION_TYPE_VERSION]            = "version",
-        [GF_QUOTA_OPTION_TYPE_ALERT_TIME]         = "alert-time",
-        [GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT]       = "soft-timeout",
-        [GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT]       = "hard-timeout",
-        [GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT] = "default-soft-limit",
-        [GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS]      = "limit-objects",
-        [GF_QUOTA_OPTION_TYPE_LIST_OBJECTS]       = "list-objects",
-        [GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS]     = "remove-objects",
-        [GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS]     = "enable-objects",
-        [GF_QUOTA_OPTION_TYPE_MAX]                = NULL
-};
+#define GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE(pidfile, volname)              \
+    {                                                                          \
+        snprintf(pidfile, PATH_MAX - 1,                                        \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_limit.pid", volname);    \
+    }
+
+#define GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE(pidfile, volname)               \
+    {                                                                          \
+        snprintf(pidfile, PATH_MAX - 1,                                        \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_list.pid", volname);     \
+    }
+
+#define GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(piddir, volinfo, type)                 \
+    do {                                                                       \
+        char _volpath[PATH_MAX] = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        int32_t _crawl_pid_len;                                                \
+        GLUSTERD_GET_VOLUME_DIR(_volpath, volinfo, priv);                      \
+        if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||                             \
+            type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)                       \
+            _crawl_pid_len = snprintf(piddir, PATH_MAX, "%s/run/quota/enable", \
+                                      _volpath);                               \
+        else                                                                   \
+            _crawl_pid_len = snprintf(piddir, PATH_MAX,                        \
+                                      "%s/run/quota/disable", _volpath);       \
+        if ((_crawl_pid_len < 0) || (_crawl_pid_len >= PATH_MAX)) {            \
+            piddir[0] = 0;                                                     \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_TMP_PATH(abspath, path)                                   \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/tmp%s", path);                    \
+    } while (0)
+
+#define GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(abspath, volname, path)             \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_list%s", volname, path); \
+    } while (0)
 
-int
-glusterd_store_quota_config (glusterd_volinfo_t *volinfo, char *path,
-                             char *gfid_str, int opcode, char **op_errstr);
+const char *gd_quota_op_list[GF_QUOTA_OPTION_TYPE_MAX + 1] = {
+    [GF_QUOTA_OPTION_TYPE_NONE] = "none",
+    [GF_QUOTA_OPTION_TYPE_ENABLE] = "enable",
+    [GF_QUOTA_OPTION_TYPE_DISABLE] = "disable",
+    [GF_QUOTA_OPTION_TYPE_LIMIT_USAGE] = "limit-usage",
+    [GF_QUOTA_OPTION_TYPE_REMOVE] = "remove",
+    [GF_QUOTA_OPTION_TYPE_LIST] = "list",
+    [GF_QUOTA_OPTION_TYPE_VERSION] = "version",
+    [GF_QUOTA_OPTION_TYPE_ALERT_TIME] = "alert-time",
+    [GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT] = "soft-timeout",
+    [GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT] = "hard-timeout",
+    [GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT] = "default-soft-limit",
+    [GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS] = "limit-objects",
+    [GF_QUOTA_OPTION_TYPE_LIST_OBJECTS] = "list-objects",
+    [GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS] = "remove-objects",
+    [GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS] = "enable-objects",
+    [GF_QUOTA_OPTION_TYPE_UPGRADE] = "upgrade",
+    [GF_QUOTA_OPTION_TYPE_MAX] = NULL};
 
 gf_boolean_t
-glusterd_is_quota_supported (int32_t type, char **op_errstr)
+glusterd_is_quota_supported(int32_t type, char **op_errstr)
 {
-        xlator_t           *this        = NULL;
-        glusterd_conf_t    *conf        = NULL;
-        gf_boolean_t        supported   = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        if ((conf->op_version == GD_OP_VERSION_MIN) &&
-            (type > GF_QUOTA_OPTION_TYPE_VERSION))
-                goto out;
-
-        if ((conf->op_version < GD_OP_VERSION_3_7_0) &&
-            (type > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS))
-                goto out;
-
-        /* Quota Operations that change quota.conf shouldn't
-         * be allowed as the quota.conf format changes in 3.7
-         */
-        if ((conf->op_version < GD_OP_VERSION_3_7_0) &&
-            (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
-             type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
-             type == GF_QUOTA_OPTION_TYPE_REMOVE))
-                goto out;
-
-        /* Quota xattr version implemented in 3.7.6
-         * quota-version is incremented when quota is enabled
-         * Quota enable and disable performance enhancement has been done
-         * in version 3.7.12.
-         * so don't allow enabling/disabling quota in heterogeneous
-         * cluster during upgrade
-         */
-        if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
-            type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS ||
-            type == GF_QUOTA_OPTION_TYPE_DISABLE) {
-                if (conf->op_version < GD_OP_VERSION_3_7_12)
-                        goto out;
-        }
-
-        supported = _gf_true;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t supported = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if ((conf->op_version == GD_OP_VERSION_MIN) &&
+        (type > GF_QUOTA_OPTION_TYPE_VERSION))
+        goto out;
+
+    if ((conf->op_version < GD_OP_VERSION_3_7_0) &&
+        (type > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS))
+        goto out;
+
+    /* Quota Operations that change quota.conf shouldn't
+     * be allowed as the quota.conf format changes in 3.7
+     */
+    if ((conf->op_version < GD_OP_VERSION_3_7_0) &&
+        (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+         type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+         type == GF_QUOTA_OPTION_TYPE_REMOVE))
+        goto out;
+
+    /* Quota xattr version implemented in 3.7.6
+     * quota-version is incremented when quota is enabled
+     * Quota enable and disable performance enhancement has been done
+     * in version 3.7.12.
+     * so don't allow enabling/disabling quota in heterogeneous
+     * cluster during upgrade
+     */
+    if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+        type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS ||
+        type == GF_QUOTA_OPTION_TYPE_DISABLE) {
+        if (conf->op_version < GD_OP_VERSION_3_7_12)
+            goto out;
+    }
+
+    supported = _gf_true;
 
 out:
-        if (!supported && op_errstr != NULL && conf)
-                gf_asprintf (op_errstr, "Volume quota failed. The cluster is "
-                             "operating at version %d. Quota command"
-                             " %s is unavailable in this version.",
-                             conf->op_version, gd_quota_op_list[type]);
-
-        return supported;
+    if (!supported && op_errstr != NULL && conf)
+        gf_asprintf(op_errstr,
+                    "Volume quota failed. The cluster is "
+                    "operating at version %d. Quota command"
+                    " %s is unavailable in this version.",
+                    conf->op_version, gd_quota_op_list[type]);
+
+    return supported;
 }
 
 int
-__glusterd_handle_quota (rpcsvc_request_t *req)
+__glusterd_handle_quota(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                         *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_QUOTA;
-        char                           *volname = NULL;
-        int32_t                         type = 0;
-        char                            msg[2048] = {0,};
-        xlator_t                       *this = NULL;
-        glusterd_conf_t                *conf = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_QUOTA;
+    char *volname = NULL;
+    int32_t type = 0;
+    char msg[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                    "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get volume name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name, "
-                        "while handling quota command");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get type of command");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get type of cmd, "
-                        "while handling quota command");
-                goto out;
-        }
-
-        if (!glusterd_is_quota_supported (type, NULL)) {
-                snprintf (msg, sizeof (msg), "Volume quota failed. The cluster "
-                          "is operating at version %d. Quota command"
-                          " %s is unavailable in this version.",
-                          conf->op_version, gd_quota_op_list[type]);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_op_begin_synctask (req, GD_OP_QUOTA, dict);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name, "
+               "while handling quota command");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get type of command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type of cmd, "
+               "while handling quota command");
+        goto out;
+    }
+
+    if (!glusterd_is_quota_supported(type, NULL)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume quota failed. The cluster "
+                 "is operating at version %d. Quota command"
+                 " %s is unavailable in this version.",
+                 conf->op_version, gd_quota_op_list[type]);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_QUOTA, dict);
 
 out:
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, msg);
-        }
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, msg);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_quota (rpcsvc_request_t *req)
+glusterd_handle_quota(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_quota);
+    return glusterd_big_locked_handler(req, __glusterd_handle_quota);
 }
 
 int32_t
-glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo)
+glusterd_check_if_quota_trans_enabled(glusterd_volinfo_t *volinfo)
 {
-        int32_t  ret           = 0;
-        int      flag          = _gf_false;
-
-        flag = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
-        if (flag == -1) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_QUOTA_GET_STAT_FAIL,
-                        "failed to get the quota status");
-                ret = -1;
-                goto out;
-        }
-
-        if (flag == _gf_false) {
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+    int32_t ret = 0;
+    int flag = _gf_false;
+
+    flag = glusterd_volinfo_get_boolean(volinfo, VKEY_FEATURES_QUOTA);
+    if (flag == -1) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_QUOTA_GET_STAT_FAIL,
+               "failed to get the quota status");
+        ret = -1;
+        goto out;
+    }
+
+    if (flag == _gf_false) {
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-_glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv,
+_glusterd_quota_initiate_fs_crawl(glusterd_conf_t *priv,
                                   glusterd_volinfo_t *volinfo,
                                   glusterd_brickinfo_t *brick, int type,
                                   char *pid_dir)
 {
-        pid_t                      pid;
-        int32_t                    ret                 = -1;
-        int                        status              = 0;
-        char                       mountdir[PATH_MAX]  = {0,};
-        char                       logfile[PATH_MAX]   = {0,};
-        char                       brickpath[PATH_MAX] = {0,};
-        char                       vol_id[PATH_MAX]    = {0,};
-        char                       pidfile[PATH_MAX]   = {0,};
-        runner_t                   runner              = {0};
-        char                      *volfileserver       = NULL;
-        FILE                      *pidfp               = NULL;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", THIS, out);
-
-        GLUSTERD_GET_TMP_PATH (mountdir, "/");
-        ret = sys_mkdir (mountdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                        GD_MSG_MOUNT_REQ_FAIL, "failed to create temporary "
-                        "directory %s", mountdir);
-                ret = -1;
-                goto out;
+    pid_t pid;
+    int32_t ret = -1;
+    int status = 0;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char brickpath[PATH_MAX] = {
+        0,
+    };
+    char vol_id[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {0};
+    char *volfileserver = NULL;
+    FILE *pidfp = NULL;
+    int32_t len = 0;
+
+    GF_VALIDATE_OR_GOTO("glusterd", THIS, out);
+
+    GLUSTERD_GET_TMP_PATH(mountdir, "/");
+    ret = sys_mkdir(mountdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_MOUNT_REQ_FAIL,
+               "failed to create temporary "
+               "directory %s",
+               mountdir);
+        ret = -1;
+        goto out;
+    }
+
+    strcat(mountdir, "mntXXXXXX");
+    if (mkdtemp(mountdir) == NULL) {
+        gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_MOUNT_REQ_FAIL,
+               "failed to create a temporary "
+               "mount directory: %s",
+               mountdir);
+        ret = -1;
+        goto out;
+    }
+
+    GLUSTERD_REMOVE_SLASH_FROM_PATH(brick->path, brickpath);
+    len = snprintf(logfile, sizeof(logfile),
+                   DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY "/%s.log", brickpath);
+    if ((len < 0) || (len >= sizeof(vol_id))) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_strn(THIS->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0)
+        volfileserver = "localhost";
+
+    len = snprintf(vol_id, sizeof(vol_id), "client_per_brick/%s.%s.%s.%s.vol",
+                   volinfo->volname, "client", brick->hostname, brickpath);
+    if ((len < 0) || (len >= sizeof(vol_id))) {
+        ret = -1;
+        goto out;
+    }
+
+    runinit(&runner);
+
+    if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+        type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)
+        runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s", volfileserver,
+                        "--volfile-id", vol_id, "--use-readdirp=yes",
+                        "--client-pid", QUOTA_CRAWL_PID, "-l", logfile,
+                        mountdir, NULL);
+    else
+        runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s", volfileserver,
+                        "--volfile-id", vol_id, "--use-readdirp=no",
+                        "--client-pid", QUOTA_CRAWL_PID, "-l", logfile,
+                        mountdir, NULL);
+
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run_reuse(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret == -1) {
+        runner_log(&runner, "glusterd", GF_LOG_DEBUG, "command failed");
+        runner_end(&runner);
+        goto out;
+    }
+    runner_end(&runner);
+
+    if ((pid = fork()) < 0) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_FORK_FAIL,
+               "fork from parent failed");
+        gf_umount_lazy("glusterd", mountdir, 1);
+        ret = -1;
+        goto out;
+    } else if (pid == 0) {  // first child
+        /* fork one more to not hold back main process on
+         * blocking call below
+         */
+        pid = fork();
+        if (pid < 0) {
+            gf_umount_lazy("glusterd", mountdir, 1);
+            _exit(EXIT_FAILURE);
+        } else if (pid > 0) {
+            _exit(EXIT_SUCCESS);
         }
 
-        strcat (mountdir, "mntXXXXXX");
-        if (mkdtemp (mountdir) == NULL) {
-                gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                        GD_MSG_MOUNT_REQ_FAIL, "failed to create a temporary "
-                        "mount directory: %s", mountdir);
-                ret = -1;
-                goto out;
+        ret = chdir(mountdir);
+        if (ret == -1) {
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DIR_OP_FAILED,
+                   "chdir %s failed", mountdir);
+            gf_umount_lazy("glusterd", mountdir, 1);
+            exit(EXIT_FAILURE);
         }
-
-        GLUSTERD_REMOVE_SLASH_FROM_PATH (brick->path, brickpath);
-        snprintf (logfile, sizeof (logfile),
-                  DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY"/%s.log",
-                  brickpath);
-
-        if (dict_get_str (THIS->options, "transport.socket.bind-address",
-                          &volfileserver) != 0)
-                volfileserver = "localhost";
-
-        snprintf (vol_id, sizeof (vol_id), "client_per_brick/%s.%s.%s.%s.vol",
-                  volinfo->volname, "client", brick->hostname, brickpath);
-
-        runinit (&runner);
+        runinit(&runner);
 
         if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
             type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)
-                runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                                 "-s", volfileserver,
-                                 "--volfile-id", vol_id,
-                                 "--use-readdirp=yes",
-                                 "--client-pid", QUOTA_CRAWL_PID,
-                                 "-l", logfile, mountdir, NULL);
-        else
-                runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                                 "-s", volfileserver,
-                                 "--volfile-id", vol_id,
-                                 "--use-readdirp=no",
-                                 "--client-pid", QUOTA_CRAWL_PID,
-                                 "-l", logfile, mountdir, NULL);
-
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run_reuse (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret == -1) {
-                runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
-                runner_end (&runner);
-                goto out;
-        }
-        runner_end (&runner);
-
-        if ((pid = fork ()) < 0) {
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        GD_MSG_FORK_FAIL, "fork from parent failed");
-                ret = -1;
-                goto out;
-        } else if (pid == 0) {//first child
-                /* fork one more to not hold back main process on
-                 * blocking call below
-                 */
-                pid = fork ();
-                if (pid)
-                        _exit (pid > 0 ? EXIT_SUCCESS : EXIT_FAILURE);
-
-                ret = chdir (mountdir);
-                if (ret == -1) {
-                        gf_msg (THIS->name, GF_LOG_WARNING, errno,
-                                GD_MSG_DIR_OP_FAILED, "chdir %s failed",
-                                mountdir);
-                        exit (EXIT_FAILURE);
-                }
-                runinit (&runner);
-
-                if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
-                    type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)
-                        runner_add_args (&runner, "/usr/bin/find", ".", NULL);
-
-                else if (type == GF_QUOTA_OPTION_TYPE_DISABLE) {
+            runner_add_args(&runner, "/usr/bin/find", ".", "-exec",
+                            "/usr/bin/stat", "{}", "\\", ";", NULL);
 
+        else if (type == GF_QUOTA_OPTION_TYPE_DISABLE) {
 #if defined(GF_DARWIN_HOST_OS)
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", "/usr/bin/xattr", "-w",
-                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1",
-                                         "{}", "\\", ";", NULL);
+            runner_add_args(
+                &runner, "/usr/bin/find", ".", "-exec", "/usr/bin/xattr", "-w",
+                VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1", "{}", "\\", ";", NULL);
 #elif defined(__FreeBSD__)
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", "/usr/sbin/setextattr",
-                                         EXTATTR_NAMESPACE_USER,
-                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1",
-                                         "{}", "\\", ";", NULL);
+            runner_add_args(&runner, "/usr/bin/find", ".", "-exec",
+                            "/usr/sbin/setextattr", EXTATTR_NAMESPACE_USER,
+                            VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1", "{}", "\\",
+                            ";", NULL);
 #else
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", _PATH_SETFATTR, "-n",
-                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v",
-                                         "1", "{}", "\\", ";", NULL);
+            runner_add_args(&runner, "find", ".", "-exec", _PATH_SETFATTR, "-n",
+                            VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v", "1", "{}",
+                            "\\", ";", NULL);
 #endif
+        }
 
-                }
-
-                if (runner_start (&runner) == -1) {
-                        gf_umount_lazy ("glusterd", mountdir, 1);
-                        _exit (EXIT_FAILURE);
-                }
+        if (runner_start(&runner) == -1) {
+            gf_umount_lazy("glusterd", mountdir, 1);
+            _exit(EXIT_FAILURE);
+        }
 
-                snprintf (pidfile, sizeof (pidfile), "%s/%s.pid", pid_dir,
-                          brickpath);
-                pidfp = fopen (pidfile, "w");
-                if (pidfp) {
-                        fprintf (pidfp, "%d\n", runner.chpid);
-                        fflush (pidfp);
-                        fclose (pidfp);
-                }
+        len = snprintf(pidfile, sizeof(pidfile), "%s/%s.pid", pid_dir,
+                       brickpath);
+        if ((len >= 0) && (len < sizeof(pidfile))) {
+            pidfp = fopen(pidfile, "w");
+            if (pidfp != NULL) {
+                fprintf(pidfp, "%d\n", runner.chpid);
+                fflush(pidfp);
+                fclose(pidfp);
+            }
+        }
 
 #ifndef GF_LINUX_HOST_OS
-                runner_end (&runner); /* blocks in waitpid */
+        runner_end(&runner); /* blocks in waitpid */
 #endif
-                gf_umount_lazy ("glusterd", mountdir, 1);
+        gf_umount_lazy("glusterd", mountdir, 1);
 
-                _exit (EXIT_SUCCESS);
-        }
-        ret = (waitpid (pid, &status, 0) == pid &&
-               WIFEXITED (status) && WEXITSTATUS (status) == EXIT_SUCCESS) ? 0 : -1;
+        _exit(EXIT_SUCCESS);
+    }
+    ret = (waitpid(pid, &status, 0) == pid && WIFEXITED(status) &&
+           WEXITSTATUS(status) == EXIT_SUCCESS)
+              ? 0
+              : -1;
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-glusterd_stop_all_quota_crawl_service (glusterd_conf_t *priv,
-                                       glusterd_volinfo_t *volinfo, int type)
+glusterd_stop_all_quota_crawl_service(glusterd_conf_t *priv,
+                                      glusterd_volinfo_t *volinfo, int type)
 {
-        char                       pid_dir[PATH_MAX]  = {0, };
-        char                       pidfile[PATH_MAX]  = {0,};
-        struct dirent             *entry              = NULL;
-        DIR                       *dir                = NULL;
-
-        GLUSTERD_GET_QUOTA_CRAWL_PIDDIR (pid_dir, volinfo, type);
-
-        dir = sys_opendir (pid_dir);
-        if (dir == NULL)
-                return;
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        while (entry) {
-                snprintf (pidfile, sizeof (pidfile), "%s/%s",
-                          pid_dir, entry->d_name);
-
-                glusterd_service_stop_nolock ("quota_crawl", pidfile, SIGKILL,
-                                              _gf_true);
-                sys_unlink (pidfile);
-
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
-        sys_closedir (dir);
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char pid_dir[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(pid_dir, volinfo, type);
+
+    dir = sys_opendir(pid_dir);
+    if (dir == NULL)
+        return;
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        len = snprintf(pidfile, sizeof(pidfile), "%s/%s", pid_dir,
+                       entry->d_name);
+        if ((len >= 0) && (len < sizeof(pidfile))) {
+            glusterd_service_stop_nolock("quota_crawl", pidfile, SIGKILL,
+                                         _gf_true);
+            sys_unlink(pidfile);
+        }
+    }
+    sys_closedir(dir);
 }
 
 int32_t
-glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv,
-                                  glusterd_volinfo_t *volinfo, int type)
+glusterd_quota_initiate_fs_crawl(glusterd_conf_t *priv,
+                                 glusterd_volinfo_t *volinfo, int type)
 {
-        int32_t                    ret                = -1;
-        glusterd_brickinfo_t      *brick              = NULL;
-        char                       pid_dir[PATH_MAX]  = {0, };
-
-        GF_VALIDATE_OR_GOTO ("glusterd", THIS, out);
-
-        ret = glusterd_generate_client_per_brick_volfile (volinfo);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_OP_FAILED,
-                        "failed to generate client volume file");
-                goto out;
-        }
-
-        ret = mkdir_p (DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY, 0777, _gf_true);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_GLUSTERD_OP_FAILED,
-                        "failed to create dir %s: %s",
-                        DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY, strerror (errno));
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_brickinfo_t *brick = NULL;
+    char pid_dir[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", THIS, out);
+
+    ret = glusterd_generate_client_per_brick_volfile(volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_OP_FAILED,
+               "failed to generate client volume file");
+        goto out;
+    }
+
+    ret = mkdir_p(DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY, 0755, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_GLUSTERD_OP_FAILED,
+               "failed to create dir %s: %s", DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY,
+               strerror(errno));
+        goto out;
+    }
+
+    GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(pid_dir, volinfo, type);
+    ret = mkdir_p(pid_dir, 0755, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_GLUSTERD_OP_FAILED,
+               "failed to create dir %s: %s", pid_dir, strerror(errno));
+        goto out;
+    }
+
+    /* When quota enable is performed, stop alreday running enable crawl
+     * process and start fresh crawl process. let disable process continue
+     * if running to cleanup the older xattrs
+     * When quota disable is performed, stop both enable/disable crawl
+     * process and start fresh crawl process to cleanup the xattrs
+     */
+    glusterd_stop_all_quota_crawl_service(priv, volinfo,
+                                          GF_QUOTA_OPTION_TYPE_ENABLE);
+    if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
+        glusterd_stop_all_quota_crawl_service(priv, volinfo,
+                                              GF_QUOTA_OPTION_TYPE_DISABLE);
+
+    cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brick->uuid, MY_UUID))
+            continue;
+
+        ret = _glusterd_quota_initiate_fs_crawl(priv, volinfo, brick, type,
+                                                pid_dir);
 
-        GLUSTERD_GET_QUOTA_CRAWL_PIDDIR (pid_dir, volinfo, type);
-        ret = mkdir_p (pid_dir, 0777, _gf_true);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_GLUSTERD_OP_FAILED,
-                        "failed to create dir %s: %s",
-                        pid_dir, strerror (errno));
-                goto out;
-        }
-
-        /* When quota enable is performed, stop alreday running enable crawl
-         * process and start fresh crawl process. let disable process continue
-         * if running to cleanup the older xattrs
-         * When quota disable is performed, stop both enable/disable crawl
-         * process and start fresh crawl process to cleanup the xattrs
-         */
-        glusterd_stop_all_quota_crawl_service (priv, volinfo,
-                                               GF_QUOTA_OPTION_TYPE_ENABLE);
-        if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
-                glusterd_stop_all_quota_crawl_service (priv, volinfo,
-                                               GF_QUOTA_OPTION_TYPE_DISABLE);
-
-        cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brick->uuid, MY_UUID))
-                        continue;
-
-                ret = _glusterd_quota_initiate_fs_crawl (priv, volinfo, brick,
-                                                         type, pid_dir);
-
-                if (ret)
-                        goto out;
-        }
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_quota_get_default_soft_limit (glusterd_volinfo_t *volinfo,
-                                       dict_t *rsp_dict)
+glusterd_quota_get_default_soft_limit(glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict)
 {
-        int32_t            ret             = 0;
-        xlator_t          *this            = NULL;
-        glusterd_conf_t   *conf            = NULL;
-        char              *default_limit   = NULL;
-        char              *val             = NULL;
-
-        if (rsp_dict == NULL)
-                return -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = glusterd_volinfo_get (volinfo, "features.default-soft-limit",
-                                    &default_limit);
-        if (default_limit)
-                val = gf_strdup (default_limit);
-        else
-                val = gf_strdup ("80%");
-
-        ret = dict_set_dynstr (rsp_dict, "default-soft-limit", val);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set default "
-                        "soft-limit into dict");
-                goto out;
-        }
-        ret = 0;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *default_limit = NULL;
+    char *val = NULL;
+
+    if (rsp_dict == NULL)
+        return -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = glusterd_volinfo_get(volinfo, "features.default-soft-limit",
+                               &default_limit);
+    if (default_limit)
+        val = gf_strdup(default_limit);
+    else
+        val = gf_strdup("80%");
+
+    ret = dict_set_dynstr_sizen(rsp_dict, "default-soft-limit", val);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set default "
+               "soft-limit into dict");
+        goto out;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_inode_quota_enable (glusterd_volinfo_t *volinfo, char **op_errstr,
-                             gf_boolean_t *crawl)
+glusterd_inode_quota_enable(glusterd_volinfo_t *volinfo, char **op_errstr,
+                            gf_boolean_t *crawl)
 {
-        int32_t         ret     = -1;
-        xlator_t        *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, crawl, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        if (glusterd_is_volume_started (volinfo) == 0) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "to enable inode quota.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret != 0) {
-                *op_errstr = gf_strdup ("Quota is disabled. Enabling quota "
-                                        "will enable inode quota");
-                ret = -1;
-                goto out;
-        }
-
-        if (glusterd_is_volume_inode_quota_enabled (volinfo)) {
-                *op_errstr = gf_strdup ("Inode Quota is already enabled");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict,
-                                          VKEY_FEATURES_INODE_QUOTA, "on");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED,
-                        "dict set failed");
-                goto out;
-        }
-
-        *crawl = _gf_true;
-
-        ret = glusterd_store_quota_config (volinfo, NULL, NULL,
-                                           GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS,
-                                           op_errstr);
-
-        ret = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, crawl, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    if (glusterd_is_volume_started(volinfo) == 0) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "to enable inode quota.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret != 0) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled. Enabling quota "
+            "will enable inode quota");
+        ret = -1;
+        goto out;
+    }
+
+    if (glusterd_is_volume_inode_quota_enabled(volinfo)) {
+        *op_errstr = gf_strdup("Inode Quota is already enabled");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_INODE_QUOTA,
+                                     "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    *crawl = _gf_true;
+
+    ret = glusterd_store_quota_config(
+        volinfo, NULL, NULL, GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS, op_errstr);
+
+    ret = 0;
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Enabling inode quota on volume %s has "
-                             "been unsuccessful", volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Enabling inode quota on volume %s has "
+                    "been unsuccessful",
+                    volinfo->volname);
+    return ret;
 }
 
 int32_t
-glusterd_quota_enable (glusterd_volinfo_t *volinfo, char **op_errstr,
-                       gf_boolean_t *crawl)
+glusterd_quota_enable(glusterd_volinfo_t *volinfo, char **op_errstr,
+                      gf_boolean_t *crawl)
 {
-        int32_t         ret     = -1;
-        xlator_t        *this         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, crawl, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        if (glusterd_is_volume_started (volinfo) == 0) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "to enable quota.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == 0) {
-                *op_errstr = gf_strdup ("Quota is already enabled");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, VKEY_FEATURES_QUOTA,
-                                          "on");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "dict set failed");
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict,
-                                          VKEY_FEATURES_INODE_QUOTA, "on");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "dict set failed");
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict,
-                                          "features.quota-deem-statfs",
-                                          "on");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "setting quota-deem-statfs"
-                        "in volinfo failed");
-                goto out;
-        }
-
-        *crawl = _gf_true;
-
-        ret = glusterd_store_quota_config (volinfo, NULL, NULL,
-                                           GF_QUOTA_OPTION_TYPE_ENABLE,
-                                           op_errstr);
-
-        ret = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, crawl, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    if (glusterd_is_volume_started(volinfo) == 0) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "to enable quota.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == 0) {
+        *op_errstr = gf_strdup("Quota is already enabled");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_QUOTA, "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_INODE_QUOTA,
+                                     "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict,
+                                     "features.quota-deem-statfs", "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "setting quota-deem-statfs"
+               "in volinfo failed");
+        goto out;
+    }
+
+    *crawl = _gf_true;
+
+    ret = glusterd_store_quota_config(volinfo, NULL, NULL,
+                                      GF_QUOTA_OPTION_TYPE_ENABLE, op_errstr);
+
+    ret = 0;
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Enabling quota on volume %s has been "
-                             "unsuccessful", volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Enabling quota on volume %s has been "
+                    "unsuccessful",
+                    volinfo->volname);
+    return ret;
 }
 
 int32_t
-glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
-                        gf_boolean_t *crawl)
+glusterd_quota_disable(glusterd_volinfo_t *volinfo, char **op_errstr,
+                       gf_boolean_t *crawl)
 {
-        int32_t    ret            = -1;
-        int        i              =  0;
-        char      *value          = NULL;
-        xlator_t  *this           = NULL;
-        glusterd_conf_t *conf     = NULL;
-        char *quota_options[]     = {"features.soft-timeout",
-                                     "features.hard-timeout",
-                                     "features.alert-time",
-                                     "features.default-soft-limit",
-                                     "features.quota-deem-statfs",
-                                     "features.quota-timeout", NULL};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is already disabled");
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, VKEY_FEATURES_QUOTA,
-                                          "off");
+    int32_t ret = -1;
+    int i = 0;
+    char *value = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *quota_options[] = {"features.soft-timeout",
+                             "features.hard-timeout",
+                             "features.alert-time",
+                             "features.default-soft-limit",
+                             "features.quota-deem-statfs",
+                             "features.quota-timeout",
+                             NULL};
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        *op_errstr = gf_strdup("Quota is already disabled");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_QUOTA, "off");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_INODE_QUOTA,
+                                     "off");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    for (i = 0; quota_options[i]; i++) {
+        ret = glusterd_volinfo_get(volinfo, quota_options[i], &value);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED, "dict set failed");
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict,
-                                          VKEY_FEATURES_INODE_QUOTA, "off");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "dict set failed");
-                goto out;
-        }
-
-        for (i = 0; quota_options [i]; i++) {
-                ret = glusterd_volinfo_get (volinfo, quota_options[i], &value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_VOLINFO_GET_FAIL, "failed to get option"
-                                " %s", quota_options[i]);
-                } else {
-                dict_del (volinfo->dict, quota_options[i]);
-                }
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "failed to get option"
+                   " %s",
+                   quota_options[i]);
+        } else {
+            dict_del(volinfo->dict, quota_options[i]);
         }
+    }
 
-        //Remove aux mount of the volume on every node in the cluster
-        ret = glusterd_remove_auxiliary_mount (volinfo->volname);
-        if (ret)
-                goto out;
-
-        *crawl = _gf_true;
+    *crawl = _gf_true;
 
-        (void) glusterd_clean_up_quota_store (volinfo);
+    (void)glusterd_clean_up_quota_store(volinfo);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Disabling quota on volume %s has been "
-                             "unsuccessful", volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Disabling quota on volume %s has been "
+                    "unsuccessful",
+                    volinfo->volname);
+    return ret;
 }
 
 static int
-glusterd_set_quota_limit (char *volname, char *path, char *hard_limit,
-                          char *soft_limit, char *key, char **op_errstr)
+glusterd_set_quota_limit(char *volname, char *path, char *hard_limit,
+                         char *soft_limit, char *key, char **op_errstr)
 {
-        int               ret                = -1;
-        xlator_t         *this               = NULL;
-        char              abspath[PATH_MAX]  = {0,};
-        glusterd_conf_t  *priv               = NULL;
-	quota_limits_t    existing_limit     = {0,};
-	quota_limits_t    new_limit          = {0,};
-        double            soft_limit_double  = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
-        ret = gf_lstat_dir (abspath, NULL);
-        if (ret) {
-                gf_asprintf (op_errstr, "Failed to find the directory %s. "
-                             "Reason : %s", abspath, strerror (errno));
-                goto out;
-        }
-
-        if (!soft_limit) {
-                ret = sys_lgetxattr (abspath, key, (void *)&existing_limit,
-                                     sizeof (existing_limit));
-                if (ret < 0) {
-                        switch (errno) {
+    int ret = -1;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    quota_limits_t existing_limit = {
+        0,
+    };
+    quota_limits_t new_limit = {
+        0,
+    };
+    double soft_limit_double = 0;
+    int64_t local_hl = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(abspath, volname, path);
+    ret = gf_lstat_dir(abspath, NULL);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Failed to find the directory %s. "
+                    "Reason : %s",
+                    abspath, strerror(errno));
+        goto out;
+    }
+
+    if (!soft_limit) {
+        ret = sys_lgetxattr(abspath, key, (void *)&existing_limit,
+                            sizeof(existing_limit));
+        if (ret < 0) {
+            switch (errno) {
 #if defined(ENOATTR) && (ENOATTR != ENODATA)
-                        case ENODATA: /* FALLTHROUGH */
+                case ENODATA: /* FALLTHROUGH */
 #endif
-                        case ENOATTR:
-                                existing_limit.sl = -1;
-                            break;
-                        default:
-                                gf_asprintf (op_errstr, "Failed to get the "
-                                             "xattr %s from %s. Reason : %s",
-                                             key, abspath, strerror (errno));
-                                goto out;
-                        }
-                } else {
-                        existing_limit.hl = ntoh64 (existing_limit.hl);
-                        existing_limit.sl = ntoh64 (existing_limit.sl);
-                }
-                new_limit.sl = existing_limit.sl;
-
+                case ENOATTR:
+                    existing_limit.sl = -1;
+                    break;
+                default:
+                    gf_asprintf(op_errstr,
+                                "Failed to get the "
+                                "xattr %s from %s. Reason : %s",
+                                key, abspath, strerror(errno));
+                    goto out;
+            }
         } else {
-                ret = gf_string2percent (soft_limit, &soft_limit_double);
-                if (ret)
-                        goto out;
-                new_limit.sl = soft_limit_double;
+            existing_limit.hl = ntoh64(existing_limit.hl);
+            existing_limit.sl = ntoh64(existing_limit.sl);
         }
+        new_limit.sl = existing_limit.sl;
 
-        new_limit.sl = hton64 (new_limit.sl);
-
-        ret = gf_string2bytesize_int64 (hard_limit, &new_limit.hl);
+    } else {
+        ret = gf_string2percent(soft_limit, &soft_limit_double);
         if (ret)
-                goto out;
+            goto out;
+        new_limit.sl = soft_limit_double;
+    }
 
-        new_limit.hl = hton64 (new_limit.hl);
+    new_limit.sl = hton64(new_limit.sl);
 
-        ret = sys_lsetxattr (abspath, key, (char *)(void *)&new_limit,
-                             sizeof (new_limit), 0);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "setxattr of %s failed on %s."
-                             " Reason : %s", key, abspath, strerror (errno));
-                goto out;
-        }
-        ret = 0;
+    ret = gf_string2bytesize_int64(hard_limit, &local_hl);
+    if (ret)
+        goto out;
+
+    new_limit.hl = hton64(local_hl);
+
+    ret = sys_lsetxattr(abspath, key, (char *)(void *)&new_limit,
+                        sizeof(new_limit), 0);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "setxattr of %s failed on %s."
+                    " Reason : %s",
+                    key, abspath, strerror(errno));
+        goto out;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_update_quota_conf_version (glusterd_volinfo_t *volinfo)
+glusterd_update_quota_conf_version(glusterd_volinfo_t *volinfo)
 {
-        volinfo->quota_conf_version++;
-        return 0;
+    volinfo->quota_conf_version++;
+    return 0;
 }
 
 /*The function glusterd_find_gfid_match () does the following:
@@ -822,872 +908,928 @@ glusterd_update_quota_conf_version (glusterd_volinfo_t *volinfo)
  * and continue the search.
  */
 static gf_boolean_t
-glusterd_find_gfid_match_3_6 (uuid_t gfid, unsigned char *buf,
-                              size_t bytes_read, int opcode,
-                              size_t *write_byte_count)
+glusterd_find_gfid_match_3_6(uuid_t gfid, unsigned char *buf, size_t bytes_read,
+                             int opcode, size_t *write_byte_count)
 {
-        int           gfid_index  = 0;
-        int           shift_count = 0;
-        unsigned char tmp_buf[17] = {0,};
-
-        /* This function if for backward compatibility */
-
-        while (gfid_index != bytes_read) {
-                memcpy ((void *)tmp_buf, (void *)&buf[gfid_index], 16);
-                if (!gf_uuid_compare (gfid, tmp_buf)) {
-                        if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE) {
-                                shift_count = bytes_read - (gfid_index + 16);
-                                memmove ((void *)&buf[gfid_index],
-                                         (void *)&buf[gfid_index+16],
-                                         shift_count);
-                                *write_byte_count = bytes_read - 16;
-                        } else {
-                                *write_byte_count = bytes_read;
-                        }
-                        return _gf_true;
-                } else {
-                        gfid_index += 16;
-                }
-        }
-        if (gfid_index == bytes_read)
+    int gfid_index = 0;
+    int shift_count = 0;
+    unsigned char tmp_buf[17] = {
+        0,
+    };
+
+    /* This function if for backward compatibility */
+
+    while (gfid_index != bytes_read) {
+        memcpy((void *)tmp_buf, (void *)&buf[gfid_index], 16);
+        if (!gf_uuid_compare(gfid, tmp_buf)) {
+            if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE) {
+                shift_count = bytes_read - (gfid_index + 16);
+                memmove((void *)&buf[gfid_index], (void *)&buf[gfid_index + 16],
+                        shift_count);
+                *write_byte_count = bytes_read - 16;
+            } else {
                 *write_byte_count = bytes_read;
+            }
+            return _gf_true;
+        } else {
+            gfid_index += 16;
+        }
+    }
+    if (gfid_index == bytes_read)
+        *write_byte_count = bytes_read;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-glusterd_find_gfid_match (uuid_t gfid, char gfid_type, unsigned char *buf,
-                          size_t bytes_read, int opcode,
-                          size_t *write_byte_count)
+glusterd_find_gfid_match(uuid_t gfid, char gfid_type, unsigned char *buf,
+                         size_t bytes_read, int opcode,
+                         size_t *write_byte_count)
 {
-        int                 gfid_index  = 0;
-        int                 shift_count = 0;
-        unsigned char       tmp_buf[17] = {0,};
-        char                type        = 0;
-        xlator_t           *this        = NULL;
-        glusterd_conf_t    *conf        = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        if (conf->op_version < GD_OP_VERSION_3_7_0)
-                return glusterd_find_gfid_match_3_6 (gfid, buf, bytes_read,
-                                                     opcode, write_byte_count);
-
-        while (gfid_index != bytes_read) {
-                memcpy ((void *)tmp_buf, (void *)&buf[gfid_index], 16);
-                type = buf[gfid_index + 16];
-
-                if (!gf_uuid_compare (gfid, tmp_buf) && type == gfid_type) {
-                        if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE ||
-                            opcode == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
-                                shift_count = bytes_read - (gfid_index + 17);
-                                memmove ((void *)&buf[gfid_index],
-                                         (void *)&buf[gfid_index + 17],
-                                         shift_count);
-                                *write_byte_count = bytes_read - 17;
-                        } else {
-                                *write_byte_count = bytes_read;
-                        }
-                        return _gf_true;
-                } else {
-                        gfid_index += 17;
-                }
-        }
-        if (gfid_index == bytes_read)
+    int gfid_index = 0;
+    int shift_count = 0;
+    unsigned char tmp_buf[17] = {
+        0,
+    };
+    char type = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (conf->op_version < GD_OP_VERSION_3_7_0)
+        return glusterd_find_gfid_match_3_6(gfid, buf, bytes_read, opcode,
+                                            write_byte_count);
+
+    while (gfid_index != bytes_read) {
+        memcpy((void *)tmp_buf, (void *)&buf[gfid_index], 16);
+        type = buf[gfid_index + 16];
+
+        if (!gf_uuid_compare(gfid, tmp_buf) && type == gfid_type) {
+            if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE ||
+                opcode == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+                shift_count = bytes_read - (gfid_index + 17);
+                memmove((void *)&buf[gfid_index], (void *)&buf[gfid_index + 17],
+                        shift_count);
+                *write_byte_count = bytes_read - 17;
+            } else {
                 *write_byte_count = bytes_read;
+            }
+            return _gf_true;
+        } else {
+            gfid_index += 17;
+        }
+    }
+    if (gfid_index == bytes_read)
+        *write_byte_count = bytes_read;
 
 out:
 
-        return _gf_false;
+    return _gf_false;
 }
 
 /* The function glusterd_copy_to_tmp_file() reads the "remaining" bytes from
- * the source fd and writes them to destination fd, at the rate of 128K bytes
- * of read+write at a time.
+ * the source fd and writes them to destination fd, at the rate of 1000 entries
+ * a time (qconf_line_sz is the size of an entry)
  */
 
 static int
-glusterd_copy_to_tmp_file (int src_fd, int dst_fd)
+glusterd_copy_to_tmp_file(int src_fd, int dst_fd, int qconf_line_sz)
 {
-        int            ret         = 0;
-        size_t         entry_sz    = 131072;
-        ssize_t        bytes_read  = 0;
-        unsigned char  buf[131072] = {0,};
-        xlator_t      *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        while ((bytes_read = sys_read (src_fd, (void *)&buf, entry_sz)) > 0) {
-                if (bytes_read % 16 != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_QUOTA_CONF_CORRUPT, "quota.conf "
-                                "corrupted");
-                        ret = -1;
-                        goto out;
-                }
-                ret = sys_write (dst_fd, (void *) buf, bytes_read);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_QUOTA_CONF_WRITE_FAIL,
-                                "write into quota.conf failed.");
-                        goto out;
-                }
+    int ret = 0;
+    ssize_t bytes_read = 0;
+    xlator_t *this = NULL;
+    unsigned char *buf = 0;
+    int buf_sz = qconf_line_sz * 1000;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(buf_sz > 0);
+
+    buf = GF_CALLOC(buf_sz, 1, gf_common_mt_char);
+    if (!buf) {
+        ret = -1;
+        goto out;
+    }
+
+    while ((bytes_read = sys_read(src_fd, buf, buf_sz)) > 0) {
+        if (bytes_read % qconf_line_sz != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_CORRUPT,
+                   "quota.conf "
+                   "corrupted");
+            ret = -1;
+            goto out;
+        }
+        ret = sys_write(dst_fd, (void *)buf, bytes_read);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                   "write into quota.conf failed.");
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    if (buf)
+        GF_FREE(buf);
+    return ret;
 }
 
 int
-glusterd_store_quota_conf_upgrade (glusterd_volinfo_t *volinfo)
+glusterd_store_quota_conf_upgrade(glusterd_volinfo_t *volinfo)
 {
-        int                ret                   = -1;
-        int                fd                    = -1;
-        int                conf_fd               = -1;
-        unsigned char      gfid[17]              = {0,};
-        xlator_t          *this                  = NULL;
-        char               type                  = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        fd = gf_store_mkstemp (volinfo->quota_conf_shandle);
-        if (fd < 0) {
-                ret = -1;
-                goto out;
-        }
-
-        conf_fd = open (volinfo->quota_conf_shandle->path, O_RDONLY);
-        if (conf_fd == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = quota_conf_skip_header (conf_fd);
-        if (ret)
-                goto out;
-
-        ret = glusterd_quota_conf_write_header (fd);
-        if (ret)
-                goto out;
-
-        while (1) {
-                ret = quota_conf_read_gfid (conf_fd, gfid, &type, 1.1);
-                if (ret == 0)
-                        break;
-                else if (ret < 0)
-                        goto out;
-
-                ret = glusterd_quota_conf_write_gfid (fd, gfid,
+    int ret = -1;
+    int fd = -1;
+    int conf_fd = -1;
+    unsigned char gfid[17] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char type = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    fd = gf_store_mkstemp(volinfo->quota_conf_shandle);
+    if (fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    conf_fd = open(volinfo->quota_conf_shandle->path, O_RDONLY);
+    if (conf_fd == -1) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = quota_conf_skip_header(conf_fd);
+    if (ret)
+        goto out;
+
+    ret = glusterd_quota_conf_write_header(fd);
+    if (ret)
+        goto out;
+
+    while (1) {
+        ret = quota_conf_read_gfid(conf_fd, gfid, &type, 1.1);
+        if (ret == 0)
+            break;
+        else if (ret < 0)
+            goto out;
+
+        ret = glusterd_quota_conf_write_gfid(fd, gfid,
                                              GF_QUOTA_CONF_TYPE_USAGE);
-                if (ret < 0)
-                        goto out;
-        }
+        if (ret < 0)
+            goto out;
+    }
 
 out:
-        if (conf_fd != -1)
-                sys_close (conf_fd);
+    if (conf_fd != -1)
+        sys_close(conf_fd);
 
-        if (ret && (fd > 0)) {
-                gf_store_unlink_tmppath (volinfo->quota_conf_shandle);
-        } else if (!ret) {
-                ret = gf_store_rename_tmppath (volinfo->quota_conf_shandle);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Failed to rename "
-                                "quota conf file");
-                        return ret;
-                }
-
-                ret = glusterd_compute_cksum (volinfo, _gf_true);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_CKSUM_COMPUTE_FAIL, "Failed to "
-                                "compute cksum for quota conf file");
-                        return ret;
-                }
+    if (ret && (fd > 0)) {
+        gf_store_unlink_tmppath(volinfo->quota_conf_shandle);
+    } else if (!ret) {
+        ret = gf_store_rename_tmppath(volinfo->quota_conf_shandle);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Failed to rename "
+                   "quota conf file");
+            return ret;
+        }
 
-                ret = glusterd_store_save_quota_version_and_cksum (volinfo);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_QUOTA_CKSUM_VER_STORE_FAIL, "Failed to "
-                                "store quota version and cksum");
+        ret = glusterd_compute_cksum(volinfo, _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_COMPUTE_FAIL,
+                   "Failed to "
+                   "compute cksum for quota conf file");
+            return ret;
         }
 
-        return ret;
+        ret = glusterd_store_save_quota_version_and_cksum(volinfo);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_QUOTA_CKSUM_VER_STORE_FAIL,
+                   "Failed to "
+                   "store quota version and cksum");
+    }
+
+    return ret;
 }
 
 int
-glusterd_store_quota_config (glusterd_volinfo_t *volinfo, char *path,
-                             char *gfid_str, int opcode, char **op_errstr)
+glusterd_store_quota_config(glusterd_volinfo_t *volinfo, char *path,
+                            char *gfid_str, int opcode, char **op_errstr)
 {
-        int                ret                   = -1;
-        int                fd                    = -1;
-        int                conf_fd               = -1;
-        ssize_t            bytes_read            = 0;
-        size_t             bytes_to_write        = 0;
-        unsigned char      buf[131072]           = {0,};
-        uuid_t             gfid                  = {0,};
-        xlator_t          *this                  = NULL;
-        gf_boolean_t       found                 = _gf_false;
-        gf_boolean_t       modified              = _gf_false;
-        gf_boolean_t       is_file_empty         = _gf_false;
-        gf_boolean_t       is_first_read         = _gf_true;
-        glusterd_conf_t   *conf                  = NULL;
-        float              version               = 0.0f;
-        char               type                  = 0;
-        int                quota_conf_line_sz    = 16;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        glusterd_store_create_quota_conf_sh_on_absence (volinfo);
-
-        conf_fd = open (volinfo->quota_conf_shandle->path, O_RDONLY);
-        if (conf_fd == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = quota_conf_read_version (conf_fd, &version);
+    int ret = -1;
+    int fd = -1;
+    int conf_fd = -1;
+    ssize_t bytes_read = 0;
+    size_t bytes_to_write = 0;
+    uuid_t gfid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_boolean_t found = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    gf_boolean_t is_file_empty = _gf_false;
+    gf_boolean_t is_first_read = _gf_true;
+    glusterd_conf_t *conf = NULL;
+    float version = 0.0f;
+    char type = 0;
+    int quota_conf_line_sz = 16;
+    unsigned char *buf = 0;
+    int buf_sz = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    glusterd_store_create_quota_conf_sh_on_absence(volinfo);
+
+    conf_fd = open(volinfo->quota_conf_shandle->path, O_RDONLY);
+    if (conf_fd == -1) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = quota_conf_read_version(conf_fd, &version);
+    if (ret)
+        goto out;
+
+    if (version < 1.2f && conf->op_version >= GD_OP_VERSION_3_7_0) {
+        /* Upgrade quota.conf file to newer format */
+        sys_close(conf_fd);
+        conf_fd = -1;
+
+        ret = glusterd_store_quota_conf_upgrade(volinfo);
         if (ret)
-                goto out;
+            goto out;
 
-        if (version < 1.2f && conf->op_version >= GD_OP_VERSION_3_7_0) {
-                /* Upgrade quota.conf file to newer format */
-                sys_close (conf_fd);
-                ret = glusterd_store_quota_conf_upgrade(volinfo);
-                if (ret)
-                        goto out;
-
-                conf_fd = open (volinfo->quota_conf_shandle->path, O_RDONLY);
-                if (conf_fd == -1) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = quota_conf_skip_header (conf_fd);
-                if (ret)
-                        goto out;
+        if (GF_QUOTA_OPTION_TYPE_UPGRADE == opcode) {
+            /* Nothing more to be done here */
+            goto out;
         }
 
-        /* If op-ver is gt 3.7, then quota.conf will be upgraded, and 17 bytes
-         * storted in the new format. 16 bytes uuid and
-         * 1 byte type (usage/object)
-         */
-        if (conf->op_version >= GD_OP_VERSION_3_7_0)
-                quota_conf_line_sz++;
-
-        fd = gf_store_mkstemp (volinfo->quota_conf_shandle);
-        if (fd < 0) {
-                ret = -1;
-                goto out;
+        conf_fd = open(volinfo->quota_conf_shandle->path, O_RDONLY);
+        if (conf_fd == -1) {
+            ret = -1;
+            goto out;
         }
 
-        ret = glusterd_quota_conf_write_header (fd);
+        ret = quota_conf_skip_header(conf_fd);
         if (ret)
-                goto out;
-
-        /* Just create empty quota.conf file if create */
-        if (GF_QUOTA_OPTION_TYPE_ENABLE == opcode ||
-            GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS == opcode) {
-                modified = _gf_true;
-                goto out;
+            goto out;
+    } else if (GF_QUOTA_OPTION_TYPE_UPGRADE == opcode) {
+        /* No change to be done in quota_conf*/
+        goto out;
+    }
+
+    /* If op-ver is gt 3.7, then quota.conf will be upgraded, and 17 bytes
+     * storted in the new format. 16 bytes uuid and
+     * 1 byte type (usage/object)
+     */
+    if (conf->op_version >= GD_OP_VERSION_3_7_0)
+        quota_conf_line_sz++;
+
+    buf_sz = quota_conf_line_sz * 1000;
+
+    buf = GF_CALLOC(buf_sz, 1, gf_common_mt_char);
+    if (!buf) {
+        ret = -1;
+        goto out;
+    }
+
+    fd = gf_store_mkstemp(volinfo->quota_conf_shandle);
+    if (fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_quota_conf_write_header(fd);
+    if (ret)
+        goto out;
+
+    /* Just create empty quota.conf file if create */
+    if (GF_QUOTA_OPTION_TYPE_ENABLE == opcode ||
+        GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS == opcode) {
+        modified = _gf_true;
+        goto out;
+    }
+
+    /* Check if gfid_str is given for opts other than ENABLE */
+    if (!gfid_str) {
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_parse(gfid_str, gfid);
+
+    if (opcode > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS)
+        type = GF_QUOTA_CONF_TYPE_OBJECTS;
+    else
+        type = GF_QUOTA_CONF_TYPE_USAGE;
+
+    for (;;) {
+        bytes_read = sys_read(conf_fd, buf, buf_sz);
+        if (bytes_read <= 0) {
+            /*The flag @is_first_read is TRUE when the loop is
+             * entered, and is set to false if the first read
+             * reads non-zero bytes of data. The flag is used to
+             * detect if quota.conf is an empty file, but for the
+             * header. This is done to log appropriate error message
+             * when 'quota remove' is attempted when there are no
+             * limits set on the given volume.
+             */
+            if (is_first_read)
+                is_file_empty = _gf_true;
+            break;
+        }
+        if ((bytes_read % quota_conf_line_sz) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_CORRUPT,
+                   "quota.conf "
+                   "corrupted");
+            ret = -1;
+            goto out;
+        }
+        found = glusterd_find_gfid_match(gfid, type, buf, bytes_read, opcode,
+                                         &bytes_to_write);
+
+        ret = sys_write(fd, (void *)buf, bytes_to_write);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                   "write into quota.conf failed.");
+            goto out;
         }
 
-        /* Check if gfid_str is given for opts other than ENABLE */
-        if (!gfid_str) {
-                ret = -1;
+        /*If the match is found in this iteration, copy the rest of
+         * quota.conf into quota.conf.tmp and break.
+         * Else continue with the search.
+         */
+        if (found) {
+            ret = glusterd_copy_to_tmp_file(conf_fd, fd, quota_conf_line_sz);
+            if (ret)
                 goto out;
+            break;
         }
-        gf_uuid_parse (gfid_str, gfid);
-
-        if (opcode > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS)
-                type = GF_QUOTA_CONF_TYPE_OBJECTS;
-        else
-                type = GF_QUOTA_CONF_TYPE_USAGE;
-
-        for (;;) {
-                bytes_read = sys_read (conf_fd, (void *)&buf, sizeof (buf));
-                if (bytes_read <= 0) {
-                        /*The flag @is_first_read is TRUE when the loop is
-                         * entered, and is set to false if the first read
-                         * reads non-zero bytes of data. The flag is used to
-                         * detect if quota.conf is an empty file, but for the
-                         * header. This is done to log appropriate error message
-                         * when 'quota remove' is attempted when there are no
-                         * limits set on the given volume.
-                         */
-                        if (is_first_read)
-                                is_file_empty = _gf_true;
-                        break;
-                }
-                if ((bytes_read % quota_conf_line_sz) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_QUOTA_CONF_CORRUPT, "quota.conf "
-                                "corrupted");
-                        ret = -1;
-                        goto out;
-                }
-                found = glusterd_find_gfid_match (gfid, type, buf, bytes_read,
-                                                  opcode, &bytes_to_write);
-
-                ret = sys_write (fd, (void *) buf, bytes_to_write);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_QUOTA_CONF_WRITE_FAIL,
-                                "write into quota.conf failed.");
-                        goto out;
-                }
-
-                /*If the match is found in this iteration, copy the rest of
-                 * quota.conf into quota.conf.tmp and break.
-                 * Else continue with the search.
-                 */
-                if (found) {
-                        ret = glusterd_copy_to_tmp_file (conf_fd, fd);
-                        if (ret)
-                                goto out;
-                        break;
-                }
-                is_first_read = _gf_false;
-        }
+        is_first_read = _gf_false;
+    }
 
-        switch (opcode) {
+    switch (opcode) {
         case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                if (!found) {
-                        ret = glusterd_quota_conf_write_gfid (fd, gfid,
+            if (!found) {
+                ret = glusterd_quota_conf_write_gfid(fd, gfid,
                                                      GF_QUOTA_CONF_TYPE_USAGE);
-                        if (ret == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_QUOTA_CONF_WRITE_FAIL,
-                                        "write into quota.conf failed. ");
-                                goto out;
-                        }
-                        modified = _gf_true;
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                           "write into quota.conf failed. ");
+                    goto out;
                 }
-                break;
+                modified = _gf_true;
+            }
+            break;
         case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-                if (!found) {
-                        ret = glusterd_quota_conf_write_gfid (fd, gfid,
-                                                   GF_QUOTA_CONF_TYPE_OBJECTS);
-                        if (ret == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_QUOTA_CONF_WRITE_FAIL,
-                                        "write into quota.conf failed. ");
-                                goto out;
-                        }
-                        modified = _gf_true;
+            if (!found) {
+                ret = glusterd_quota_conf_write_gfid(
+                    fd, gfid, GF_QUOTA_CONF_TYPE_OBJECTS);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                           "write into quota.conf failed. ");
+                    goto out;
                 }
-                break;
+                modified = _gf_true;
+            }
+            break;
 
         case GF_QUOTA_OPTION_TYPE_REMOVE:
         case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                if (is_file_empty) {
-                        gf_asprintf (op_errstr, "Cannot remove limit on"
-                                     " %s. The quota configuration file"
-                                     " for volume %s is empty.", path,
-                                     volinfo->volname);
-                        ret = -1;
-                        goto out;
+            if (is_file_empty) {
+                gf_asprintf(op_errstr,
+                            "Cannot remove limit on"
+                            " %s. The quota configuration file"
+                            " for volume %s is empty.",
+                            path, volinfo->volname);
+                ret = -1;
+                goto out;
+            } else {
+                if (!found) {
+                    gf_asprintf(op_errstr,
+                                "Error. gfid %s"
+                                " for path %s not found in"
+                                " store",
+                                gfid_str, path);
+                    ret = -1;
+                    goto out;
                 } else {
-                        if (!found) {
-                                gf_asprintf (op_errstr, "Error. gfid %s"
-                                             " for path %s not found in"
-                                             " store", gfid_str, path);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                modified = _gf_true;
-                        }
+                    modified = _gf_true;
                 }
-                break;
+            }
+            break;
 
         default:
-                ret = 0;
-                break;
-        }
+            ret = 0;
+            break;
+    }
 
-        if (modified)
-                glusterd_update_quota_conf_version (volinfo);
+    if (modified)
+        glusterd_update_quota_conf_version(volinfo);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (conf_fd != -1) {
-                sys_close (conf_fd);
-        }
-
-        if (ret && (fd > 0)) {
-                gf_store_unlink_tmppath (volinfo->quota_conf_shandle);
-        } else if (!ret) {
-                ret = gf_store_rename_tmppath (volinfo->quota_conf_shandle);
-                if (modified) {
-                        ret = glusterd_compute_cksum (volinfo, _gf_true);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_CKSUM_COMPUTE_FAIL, "Failed to "
-                                        "compute cksum for quota conf file");
-                                return ret;
-                        }
-
-                        ret = glusterd_store_save_quota_version_and_cksum
-                                                                      (volinfo);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VERS_CKSUM_STORE_FAIL,
-                                        "Failed to "
-                                        "store quota version and cksum");
-                }
-        }
-
-        return ret;
+    if (conf_fd != -1) {
+        sys_close(conf_fd);
+    }
+
+    if (buf)
+        GF_FREE(buf);
+
+    if (ret && (fd > 0)) {
+        gf_store_unlink_tmppath(volinfo->quota_conf_shandle);
+    } else if (!ret && GF_QUOTA_OPTION_TYPE_UPGRADE != opcode) {
+        ret = gf_store_rename_tmppath(volinfo->quota_conf_shandle);
+        if (modified) {
+            ret = glusterd_compute_cksum(volinfo, _gf_true);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_COMPUTE_FAIL,
+                       "Failed to "
+                       "compute cksum for quota conf file");
+                return ret;
+            }
+
+            ret = glusterd_store_save_quota_version_and_cksum(volinfo);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_VERS_CKSUM_STORE_FAIL,
+                       "Failed to "
+                       "store quota version and cksum");
+        }
+    }
+    return ret;
 }
 
 int32_t
-glusterd_quota_limit_usage (glusterd_volinfo_t *volinfo, dict_t *dict,
-                            int opcode, char **op_errstr)
+glusterd_quota_limit_usage(glusterd_volinfo_t *volinfo, dict_t *dict,
+                           int opcode, char **op_errstr)
 {
-        int32_t          ret                = -1;
-        char            *path               = NULL;
-        char            *hard_limit         = NULL;
-        char            *soft_limit         = NULL;
-        char            *gfid_str           = NULL;
-        xlator_t        *this               = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable "
-                                        "quota");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch path");
-                goto out;
-        }
-        ret = gf_canonicalize_path (path);
-        if (ret)
-                goto out;
-
-        ret = dict_get_str (dict, "hard-limit", &hard_limit);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch hard limit");
-                goto out;
-        }
-
-        if (dict_get (dict, "soft-limit")) {
-                ret = dict_get_str (dict, "soft-limit", &soft_limit);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Unable to fetch "
-                                "soft limit");
-                        goto out;
-                }
-        }
-
-        if (is_origin_glusterd (dict)) {
-                if (opcode == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
-                        ret = glusterd_set_quota_limit (volinfo->volname, path,
-                                                        hard_limit, soft_limit,
-                                                        QUOTA_LIMIT_KEY,
-                                                        op_errstr);
-                } else {
-                        ret = glusterd_set_quota_limit (volinfo->volname, path,
-                                                        hard_limit, soft_limit,
-                                                        QUOTA_LIMIT_OBJECTS_KEY,
-                                                        op_errstr);
-                }
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_get_str (dict, "gfid", &gfid_str);
+    int32_t ret = -1;
+    char *path = NULL;
+    char *hard_limit = NULL;
+    char *soft_limit = NULL;
+    char *gfid_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled, please enable "
+            "quota");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch path");
+        goto out;
+    }
+    ret = gf_canonicalize_path(path);
+    if (ret)
+        goto out;
+
+    ret = dict_get_strn(dict, "hard-limit", SLEN("hard-limit"), &hard_limit);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch hard limit");
+        goto out;
+    }
+
+    if (dict_getn(dict, "soft-limit", SLEN("soft-limit"))) {
+        ret = dict_get_strn(dict, "soft-limit", SLEN("soft-limit"),
+                            &soft_limit);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get gfid of path "
-                        "%s", path);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch "
+                   "soft limit");
+            goto out;
         }
+    }
 
-        ret = glusterd_store_quota_config (volinfo, path, gfid_str, opcode,
+    if (is_origin_glusterd(dict)) {
+        if (opcode == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+            ret = glusterd_set_quota_limit(volinfo->volname, path, hard_limit,
+                                           soft_limit, QUOTA_LIMIT_KEY,
+                                           op_errstr);
+        } else {
+            ret = glusterd_set_quota_limit(volinfo->volname, path, hard_limit,
+                                           soft_limit, QUOTA_LIMIT_OBJECTS_KEY,
                                            op_errstr);
+        }
         if (ret)
-                goto out;
-
-        ret = 0;
+            goto out;
+    }
+
+    ret = dict_get_strn(dict, "gfid", SLEN("gfid"), &gfid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get gfid of path "
+               "%s",
+               path);
+        goto out;
+    }
+
+    ret = glusterd_store_quota_config(volinfo, path, gfid_str, opcode,
+                                      op_errstr);
+    if (ret)
+        goto out;
+
+    ret = 0;
 out:
 
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Failed to set hard limit on path %s "
-                             "for volume %s", path, volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Failed to set hard limit on path %s "
+                    "for volume %s",
+                    path, volinfo->volname);
+    return ret;
 }
 
 static int
-glusterd_remove_quota_limit (char *volname, char *path, char **op_errstr,
-                             int type)
+glusterd_remove_quota_limit(char *volname, char *path, char **op_errstr,
+                            int type)
 {
-        int               ret                = -1;
-        xlator_t         *this               = NULL;
-        char              abspath[PATH_MAX]  = {0,};
-        glusterd_conf_t  *priv               = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
-        ret = gf_lstat_dir (abspath, NULL);
+    int ret = -1;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(abspath, volname, path);
+    ret = gf_lstat_dir(abspath, NULL);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Failed to find the directory %s. "
+                    "Reason : %s",
+                    abspath, strerror(errno));
+        goto out;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_REMOVE) {
+        ret = sys_lremovexattr(abspath, QUOTA_LIMIT_KEY);
         if (ret) {
-                gf_asprintf (op_errstr, "Failed to find the directory %s. "
-                             "Reason : %s", abspath, strerror (errno));
-                goto out;
-        }
-
-        if (type == GF_QUOTA_OPTION_TYPE_REMOVE) {
-                ret = sys_lremovexattr (abspath, QUOTA_LIMIT_KEY);
-                if (ret) {
-                        gf_asprintf (op_errstr, "removexattr failed on %s. "
-                                     "Reason : %s", abspath, strerror (errno));
-                        goto out;
-                }
+            gf_asprintf(op_errstr,
+                        "removexattr failed on %s. "
+                        "Reason : %s",
+                        abspath, strerror(errno));
+            goto out;
         }
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
-                ret = sys_lremovexattr (abspath, QUOTA_LIMIT_OBJECTS_KEY);
-                if (ret) {
-                        gf_asprintf (op_errstr, "removexattr failed on %s. "
-                                     "Reason : %s", abspath, strerror (errno));
-                        goto out;
-                }
+    if (type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+        ret = sys_lremovexattr(abspath, QUOTA_LIMIT_OBJECTS_KEY);
+        if (ret) {
+            gf_asprintf(op_errstr,
+                        "removexattr failed on %s. "
+                        "Reason : %s",
+                        abspath, strerror(errno));
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_quota_remove_limits (glusterd_volinfo_t *volinfo, dict_t *dict,
-                              int opcode, char **op_errstr, int type)
+glusterd_quota_remove_limits(glusterd_volinfo_t *volinfo, dict_t *dict,
+                             int opcode, char **op_errstr, int type)
 {
-        int32_t         ret                   = -1;
-        char            *path                 = NULL;
-        char            *gfid_str             = NULL;
-        xlator_t        *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable "
-                                        "quota");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch path");
-                goto out;
-        }
-
-        ret = gf_canonicalize_path (path);
+    int32_t ret = -1;
+    char *path = NULL;
+    char *gfid_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled, please enable "
+            "quota");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch path");
+        goto out;
+    }
+
+    ret = gf_canonicalize_path(path);
+    if (ret)
+        goto out;
+
+    if (is_origin_glusterd(dict)) {
+        ret = glusterd_remove_quota_limit(volinfo->volname, path, op_errstr,
+                                          type);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (is_origin_glusterd (dict)) {
-                ret = glusterd_remove_quota_limit (volinfo->volname, path,
-                                                   op_errstr, type);
-                if (ret)
-                        goto out;
-        }
+    ret = dict_get_strn(dict, "gfid", SLEN("gfid"), &gfid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get gfid of path "
+               "%s",
+               path);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "gfid", &gfid_str);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get gfid of path "
-                        "%s", path);
-                goto out;
-        }
-
-        ret = glusterd_store_quota_config (volinfo, path, gfid_str, opcode,
-                                           op_errstr);
-        if (ret)
-                goto out;
+    ret = glusterd_store_quota_config(volinfo, path, gfid_str, opcode,
+                                      op_errstr);
+    if (ret)
+        goto out;
 
-
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_set_quota_option (glusterd_volinfo_t *volinfo, dict_t *dict,
-                           char *key, char **op_errstr)
+glusterd_set_quota_option(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                          char **op_errstr)
 {
-        int        ret    = 0;
-        char      *value  = NULL;
-        xlator_t  *this   = NULL;
-        char      *option = NULL;
+    int ret = 0;
+    char *value = NULL;
+    xlator_t *this = NULL;
+    char *option = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "Cannot set %s. Quota on volume %s is "
+                    "disabled",
+                    key, volinfo->volname);
+        return -1;
+    }
+
+    ret = dict_get_strn(dict, "value", SLEN("value"), &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Option value absent.");
+        return -1;
+    }
+
+    option = gf_strdup(value);
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to set option %s", key);
+        return -1;
+    }
+
+    return 0;
+}
 
-        this = THIS;
-        GF_ASSERT (this);
+static int
+glusterd_quotad_op(int opcode)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "Cannot set %s. Quota on volume %s is "
-                                        "disabled", key, volinfo->volname);
-                return -1;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        ret = dict_get_str (dict, "value", &value);
-        if(ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Option value absent.");
-                return -1;
-        }
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        option = gf_strdup (value);
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if(ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to set option %s",
-                        key);
-                return -1;
-        }
+    switch (opcode) {
+        case GF_QUOTA_OPTION_TYPE_ENABLE:
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
+
+            if (glusterd_all_volumes_with_quota_stopped())
+                ret = glusterd_svc_stop(&(priv->quotad_svc), SIGTERM);
+            else
+                ret = priv->quotad_svc.manager(&(priv->quotad_svc), NULL,
+                                               PROC_START);
+            break;
 
-        return 0;
+        default:
+            ret = 0;
+            break;
+    }
+    return ret;
 }
 
-static int
-glusterd_quotad_op (int opcode)
+int
+glusterd_op_quota(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int              ret  = -1;
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t ret = -1;
+    char *volname = NULL;
+    int type = -1;
+    gf_boolean_t start_crawl = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        switch (opcode) {
-                case GF_QUOTA_OPTION_TYPE_ENABLE:
-                case GF_QUOTA_OPTION_TYPE_DISABLE:
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
 
-                        if (glusterd_all_volumes_with_quota_stopped ())
-                                ret = glusterd_svc_stop (&(priv->quotad_svc),
-                                                         SIGTERM);
-                        else
-                                ret = priv->quotad_svc.manager
-                                                (&(priv->quotad_svc), NULL,
-                                                 PROC_START);
-                        break;
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
 
-                default:
-                        ret = 0;
-                        break;
-        }
-        return ret;
-}
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
 
-int
-glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
-{
-        glusterd_volinfo_t     *volinfo      = NULL;
-        int32_t                 ret          = -1;
-        char                   *volname      = NULL;
-        int                     type         = -1;
-        gf_boolean_t            start_crawl  = _gf_false;
-        glusterd_conf_t        *priv         = NULL;
-        xlator_t               *this         = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
+    if (!glusterd_is_quota_supported(type, op_errstr)) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+    switch (type) {
+        case GF_QUOTA_OPTION_TYPE_ENABLE:
+            ret = glusterd_quota_enable(volinfo, op_errstr, &start_crawl);
+            if (ret < 0)
                 goto out;
-        }
+            break;
 
-        ret = dict_get_int32 (dict, "type", &type);
+        case GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS:
+            ret = glusterd_inode_quota_enable(volinfo, op_errstr, &start_crawl);
+            if (ret < 0)
+                goto out;
+            break;
 
-        if (!glusterd_is_quota_supported (type, op_errstr)) {
-                ret = -1;
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
+            ret = glusterd_quota_disable(volinfo, op_errstr, &start_crawl);
+            if (ret < 0)
                 goto out;
-        }
 
-        switch (type) {
-                case GF_QUOTA_OPTION_TYPE_ENABLE:
-                        ret = glusterd_quota_enable (volinfo, op_errstr,
-                                                     &start_crawl);
-                        if (ret < 0)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS:
-                        ret = glusterd_inode_quota_enable (volinfo, op_errstr,
-                                                           &start_crawl);
-                        if (ret < 0)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_DISABLE:
-                        ret = glusterd_quota_disable (volinfo, op_errstr,
-                                                      &start_crawl);
-                        if (ret < 0)
-                                goto out;
-
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-                        ret = glusterd_quota_limit_usage (volinfo, dict, type,
-                                                          op_errstr);
-                        goto out;
-
-                case GF_QUOTA_OPTION_TYPE_REMOVE:
-                case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                        ret = glusterd_quota_remove_limits (volinfo, dict, type,
-                                                            op_errstr, type);
-                        goto out;
-
-                case GF_QUOTA_OPTION_TYPE_LIST:
-                case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
-                        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-                        if (ret == -1) {
-                                *op_errstr = gf_strdup ("Cannot list limits, "
-                                                        "quota is disabled");
-                                goto out;
-                        }
-                        ret = glusterd_quota_get_default_soft_limit (volinfo,
-                                                               rsp_dict);
-                        goto out;
-
-                case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                        "features.soft-timeout",
-                                                         op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                        "features.hard-timeout",
-                                                         op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                         "features.alert-time",
-                                                         op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                  "features.default-soft-limit",
-                                                  op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
+            break;
 
-                default:
-                        gf_asprintf (op_errstr, "Quota command failed. Invalid "
-                                     "opcode");
-                        ret = -1;
-                        goto out;
-        }
+        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+        case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+            ret = glusterd_quota_limit_usage(volinfo, dict, type, op_errstr);
+            goto out;
 
-        if (priv->op_version > GD_OP_VERSION_MIN) {
-                ret = glusterd_quotad_op (type);
-                if (ret)
-                        goto out;
-        }
+        case GF_QUOTA_OPTION_TYPE_REMOVE:
+        case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+            ret = glusterd_quota_remove_limits(volinfo, dict, type, op_errstr,
+                                               type);
+            goto out;
 
+        case GF_QUOTA_OPTION_TYPE_LIST:
+        case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
+            ret = glusterd_check_if_quota_trans_enabled(volinfo);
+            if (ret == -1) {
+                *op_errstr = gf_strdup(
+                    "Cannot list limits, "
+                    "quota is disabled");
+                goto out;
+            }
+            ret = glusterd_quota_get_default_soft_limit(volinfo, rsp_dict);
+            goto out;
 
-        if (GF_QUOTA_OPTION_TYPE_ENABLE == type)
-                volinfo->quota_xattr_version++;
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                if (GF_QUOTA_OPTION_TYPE_ENABLE == type)
-                        volinfo->quota_xattr_version--;
+        case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+            ret = glusterd_set_quota_option(volinfo, dict,
+                                            "features.soft-timeout", op_errstr);
+            if (ret)
                 goto out;
-        }
+            break;
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "Unable to re-create "
-                                                  "volfiles");
-                if (GF_QUOTA_OPTION_TYPE_ENABLE == type) {
-                        /* rollback volinfo */
-                        volinfo->quota_xattr_version--;
-                        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                }
+        case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+            ret = glusterd_set_quota_option(volinfo, dict,
+                                            "features.hard-timeout", op_errstr);
+            if (ret)
+                goto out;
+            break;
 
-                ret = -1;
+        case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+            ret = glusterd_set_quota_option(volinfo, dict,
+                                            "features.alert-time", op_errstr);
+            if (ret)
                 goto out;
-        }
+            break;
 
-        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                if (priv->op_version == GD_OP_VERSION_MIN)
-                        ret = priv->nfs_svc.manager (&(priv->nfs_svc), NULL, 0);
-        }
+        case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+            ret = glusterd_set_quota_option(
+                volinfo, dict, "features.default-soft-limit", op_errstr);
+            if (ret)
+                goto out;
+            break;
 
-        if (rsp_dict && start_crawl == _gf_true)
-                glusterd_quota_initiate_fs_crawl (priv, volinfo, type);
+        default:
+            gf_asprintf(op_errstr,
+                        "Quota command failed. Invalid "
+                        "opcode");
+            ret = -1;
+            goto out;
+    }
+
+    if (priv->op_version > GD_OP_VERSION_MIN) {
+        ret = glusterd_quotad_op(type);
+        if (ret)
+            goto out;
+    }
+
+    if (GF_QUOTA_OPTION_TYPE_ENABLE == type)
+        volinfo->quota_xattr_version++;
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        if (GF_QUOTA_OPTION_TYPE_ENABLE == type)
+            volinfo->quota_xattr_version--;
+        goto out;
+    }
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to re-create "
+               "volfiles");
+        if (GF_QUOTA_OPTION_TYPE_ENABLE == type) {
+            /* rollback volinfo */
+            volinfo->quota_xattr_version--;
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "Failed to store volinfo for volume %s",
+                       volinfo->volname);
+            }
+        }
+
+        ret = -1;
+        goto out;
+    }
+
+#if BUILD_GNFS
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        if (priv->op_version == GD_OP_VERSION_MIN)
+            (void)priv->nfs_svc.manager(&(priv->nfs_svc), NULL, 0);
+    }
+#endif
+
+    if (rsp_dict && start_crawl == _gf_true)
+        glusterd_quota_initiate_fs_crawl(priv, volinfo, type);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+        type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS ||
+        type == GF_QUOTA_OPTION_TYPE_REMOVE ||
+        type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+        /* During a list operation we need the aux mount to be
+         * accessible until the listing is done at the cli
+         */
+        glusterd_remove_auxiliary_mount(volinfo->volname);
+    }
+
+    return ret;
 }
 
 /*
@@ -1697,410 +1839,421 @@ out:
  * is not treated as error.
  */
 static int
-glusterd_get_gfid_from_brick (dict_t *dict, glusterd_volinfo_t *volinfo,
-                              dict_t *rsp_dict, char **op_errstr)
+glusterd_get_gfid_from_brick(dict_t *dict, glusterd_volinfo_t *volinfo,
+                             dict_t *rsp_dict, char **op_errstr)
 {
-        int                    ret                    = -1;
-        int                    count                  = 0;
-        char                  *path                   = NULL;
-        char                   backend_path[PATH_MAX] = {0,};
-        xlator_t              *this                   = NULL;
-        glusterd_conf_t       *priv                   = NULL;
-        glusterd_brickinfo_t  *brickinfo              = NULL;
-        char                   key[256]               = {0,};
-        char                  *gfid_str               = NULL;
-        uuid_t                 gfid;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "path", &path);
+    int ret = -1;
+    int count = 0;
+    char *path = NULL;
+    char backend_path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *gfid_str = NULL;
+    uuid_t gfid;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get path");
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = glusterd_resolve_brick(brickinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get path");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   FMTSTR_RESOLVE_BRICK, brickinfo->hostname, brickinfo->path);
+            goto out;
         }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_RESOLVE_BRICK_FAIL, FMTSTR_RESOLVE_BRICK,
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
-
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
 
-                if (brickinfo->vg[0])
-                        continue;
+        if (brickinfo->vg[0])
+            continue;
 
-                snprintf (backend_path, sizeof (backend_path), "%s%s",
-                          brickinfo->path, path);
+        snprintf(backend_path, sizeof(backend_path), "%s%s", brickinfo->path,
+                 path);
 
-                ret = gf_lstat_dir (backend_path, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, errno,
-                                GD_MSG_DIR_OP_FAILED, "Failed to find "
-                                "directory %s.", backend_path);
-                        ret = 0;
-                        continue;
-                }
-                ret = sys_lgetxattr (backend_path, GFID_XATTR_KEY, gfid, 16);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, errno,
-                                GD_MSG_SETXATTR_FAIL, "Failed to get "
-                                "extended attribute %s for directory %s. ",
-                                GFID_XATTR_KEY, backend_path);
-                        ret = 0;
-                        continue;
-                }
-                snprintf (key, sizeof (key), "gfid%d", count);
-
-                gfid_str = gf_strdup (uuid_utoa (gfid));
-                if (!gfid_str) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr (rsp_dict, key, gfid_str);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to place "
-                                "gfid of %s in dict", backend_path);
-                        GF_FREE (gfid_str);
-                        goto out;
-                }
-                count++;
+        ret = gf_lstat_dir(backend_path, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DIR_OP_FAILED,
+                   "Failed to find "
+                   "directory %s.",
+                   backend_path);
+            ret = 0;
+            continue;
+        }
+        ret = sys_lgetxattr(backend_path, GFID_XATTR_KEY, gfid, 16);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_GET_XATTR_FAIL,
+                    "Attribute=%s, Directory=%s", GFID_XATTR_KEY, backend_path,
+                    NULL);
+            ret = 0;
+            continue;
         }
+        keylen = snprintf(key, sizeof(key), "gfid%d", count);
 
-        ret = dict_set_int32 (rsp_dict, "count", count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set count");
-                goto out;
+        gfid_str = gf_strdup(uuid_utoa(gfid));
+        if (!gfid_str) {
+            ret = -1;
+            goto out;
         }
 
-        ret = 0;
+        ret = dict_set_dynstrn(rsp_dict, key, keylen, gfid_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to place "
+                   "gfid of %s in dict",
+                   backend_path);
+            GF_FREE(gfid_str);
+            goto out;
+        }
+        count++;
+    }
+
+    ret = dict_set_int32n(rsp_dict, "count", SLEN("count"), count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set count");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-_glusterd_validate_quota_opts (dict_t *dict, int type, char **errstr)
+_glusterd_validate_quota_opts(dict_t *dict, int type, char **errstr)
 {
-        int                     ret = -1;
-        xlator_t                *this = THIS;
-        void                    *quota_xl = NULL;
-        volume_opt_list_t       opt_list = {{0},};
-        volume_option_t         *opt = NULL;
-        char                    *key = NULL;
-        char                    *value = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (this);
-
-        ret = xlator_volopt_dynload ("features/quota", &quota_xl, &opt_list);
-        if (ret)
-                goto out;
-
-        switch (type) {
+    int ret = -1;
+    xlator_t *this = THIS;
+    void *quota_xl = NULL;
+    volume_opt_list_t opt_list = {
+        {0},
+    };
+    volume_option_t *opt = NULL;
+    char *key = NULL;
+    char *value = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(this);
+
+    ret = xlator_volopt_dynload("features/quota", &quota_xl, &opt_list);
+    if (ret)
+        goto out;
+
+    switch (type) {
         case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
         case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                key = (char *)gd_quota_op_list[type];
-                break;
+            key = (char *)gd_quota_op_list[type];
+            break;
         default:
-                ret = -1;
-                goto out;
-        }
-
-        opt = xlator_volume_option_get_list (&opt_list, key);
-        if (!opt) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_UNKNOWN_KEY, "Unknown option: %s", key);
-                goto out;
-        }
-        ret = dict_get_str (dict, "value", &value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Value not found for key %s",
-                        key);
-                goto out;
-        }
-
-        ret = xlator_option_validate (this, key, value, opt, errstr);
+            ret = -1;
+            goto out;
+    }
+
+    opt = xlator_volume_option_get_list(&opt_list, key);
+    if (!opt) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNKNOWN_KEY,
+               "Unknown option: %s", key);
+        goto out;
+    }
+    ret = dict_get_strn(dict, "value", SLEN("value"), &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Value not found for key %s", key);
+        goto out;
+    }
+
+    ret = xlator_option_validate(this, key, value, opt, errstr);
 
 out:
-        if (quota_xl) {
-                dlclose (quota_xl);
-                quota_xl = NULL;
-        }
-        return ret;
+    if (quota_xl) {
+        dlclose(quota_xl);
+        quota_xl = NULL;
+    }
+    return ret;
 }
 
 static int
-glusterd_create_quota_auxiliary_mount (xlator_t *this, char *volname)
+glusterd_create_quota_auxiliary_mount(xlator_t *this, char *volname, int type)
 {
-        int                ret                     = -1;
-        int                retry                   = 0;
-        char               mountdir[PATH_MAX]      = {0,};
-        char               pidfile_path[PATH_MAX]  = {0,};
-        char               logfile[PATH_MAX]       = {0,};
-        char               qpid[16]                = {0,};
-        char              *volfileserver           = NULL;
-        glusterd_conf_t   *priv                    = NULL;
-        struct stat        buf                     = {0,};
-
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile_path, volname);
-
-        if (gf_is_service_running (pidfile_path, NULL)) {
-                gf_msg_debug (this->name, 0, "Aux mount of volume %s is running"
-                              " already", volname);
-                ret = 0;
-                goto out;
-        }
-
-        if (glusterd_is_fuse_available () == _gf_false) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MOUNT_REQ_FAIL, "Fuse unavailable");
-                ret = -1;
-                goto out;
-        }
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
-        ret = sys_mkdir (mountdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_MOUNT_REQ_FAIL, "Failed to create auxiliary "
-                        "mount directory %s", mountdir);
-                goto out;
-        }
-        snprintf (logfile, PATH_MAX-1, "%s/quota-mount-%s.log",
-                  DEFAULT_LOG_FILE_DIRECTORY, volname);
-        snprintf(qpid, 15, "%d", GF_CLIENT_PID_QUOTA_MOUNT);
-
-        if (dict_get_str (this->options, "transport.socket.bind-address",
-                          &volfileserver) != 0)
-                volfileserver = "localhost";
-
-        synclock_unlock (&priv->big_lock);
-        ret = runcmd (SBIN_DIR"/glusterfs",
-                      "--volfile-server", volfileserver,
-                      "--volfile-id", volname,
-                      "-l", logfile,
-                      "-p", pidfile_path,
-                      "--client-pid", qpid,
-                      mountdir,
-                      NULL);
-        if (ret == 0) {
-                /* Block here till mount process is ready to accept FOPs.
-                 * Else, if glusterd acquires biglock below before
-                 * mount process is ready, then glusterd and mount process
-                 * can get into a deadlock situation.
-                 */
-                ret = sys_stat (mountdir, &buf);
-                if (ret < 0)
-                        ret = -errno;
-        } else {
-                ret = -errno;
-        }
-
-        synclock_lock (&priv->big_lock);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        GD_MSG_MOUNT_REQ_FAIL, "Failed to mount glusterfs "
-                        "client. Please check the log file %s for more details",
-                        logfile);
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char pidfile_path[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char qpid[16] = {
+        0,
+    };
+    char *volfileserver = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat buf = {
+        0,
+    };
+    FILE *file = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST ||
+        type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+        GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE(pidfile_path, volname);
+        GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, "/");
+    } else {
+        GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE(pidfile_path, volname);
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(mountdir, volname, "/");
+    }
+
+    file = fopen(pidfile_path, "r");
+    if (file) {
+        /* Previous command did not clean up pid file.
+         * remove aux mount if it exists*/
+        gf_umount_lazy(this->name, mountdir, 1);
+        fclose(file);
+    }
+
+    ret = sys_mkdir(mountdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_MOUNT_REQ_FAIL,
+               "Failed to create auxiliary "
+               "mount directory %s",
+               mountdir);
+        goto out;
+    }
+    snprintf(logfile, PATH_MAX - 1, "%s/quota-mount-%s.log", priv->logdir,
+             volname);
+    snprintf(qpid, 15, "%d", GF_CLIENT_PID_QUOTA_MOUNT);
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0)
+        volfileserver = "localhost";
+
+    synclock_unlock(&priv->big_lock);
+    ret = runcmd(SBIN_DIR "/glusterfs", "--volfile-server", volfileserver,
+                 "--volfile-id", volname, "-l", logfile, "-p", pidfile_path,
+                 "--client-pid", qpid, mountdir, NULL);
+    if (ret == 0) {
+        /* Block here till mount process is ready to accept FOPs.
+         * Else, if glusterd acquires biglock below before
+         * mount process is ready, then glusterd and mount process
+         * can get into a deadlock situation.
+         */
+        ret = sys_stat(mountdir, &buf);
+        if (ret < 0)
+            ret = -errno;
+    } else {
+        ret = -errno;
+    }
+
+    synclock_lock(&priv->big_lock);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, GD_MSG_MOUNT_REQ_FAIL,
+               "Failed to mount glusterfs "
+               "client. Please check the log file %s for more details",
+               logfile);
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_quota(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                 ret            = 0;
-        char               *volname        = NULL;
-        gf_boolean_t        exists         = _gf_false;
-        int                 type           = 0;
-        xlator_t           *this           = NULL;
-        glusterd_conf_t    *priv           = NULL;
-        glusterd_volinfo_t *volinfo        = NULL;
-        char               *hard_limit_str = NULL;
-        int64_t             hard_limit     = 0;
-        gf_boolean_t        get_gfid       = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        if (!exists) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                ret = -1;
-                goto out;
-        }
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-
-        if (!glusterd_is_volume_started (volinfo)) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "before executing quota command.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                *op_errstr = gf_strdup ("Volume quota failed, internal error, "
-                                        "unable to get type of operation");
-                goto out;
-        }
-
-        if ((!glusterd_is_volume_quota_enabled (volinfo)) &&
-            (type != GF_QUOTA_OPTION_TYPE_ENABLE)) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable "
-                                        "quota");
-                ret = -1;
-                goto out;
-        }
-
-        if (type > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS) {
-                if (!glusterd_is_volume_inode_quota_enabled (volinfo) &&
-                    type != GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS) {
-                        *op_errstr = gf_strdup ("Inode Quota is disabled, "
-                                                "please enable inode quota");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        if (!glusterd_is_quota_supported (type, op_errstr)) {
-                ret = -1;
-                goto out;
-        }
-
-        if ((GF_QUOTA_OPTION_TYPE_ENABLE != type) &&
-            (glusterd_check_if_quota_trans_enabled (volinfo) != 0)) {
-                ret = -1;
-                gf_asprintf (op_errstr, "Quota is not enabled on volume %s",
-                             volname);
-                goto out;
-        }
-
-        switch (type) {
+    int ret = 0;
+    char *volname = NULL;
+    int type = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *hard_limit_str = NULL;
+    int64_t hard_limit = 0;
+    gf_boolean_t get_gfid = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(volinfo)) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "before executing quota command.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        *op_errstr = gf_strdup(
+            "Volume quota failed, internal error, "
+            "unable to get type of operation");
+        goto out;
+    }
+
+    if ((!glusterd_is_volume_quota_enabled(volinfo)) &&
+        (type != GF_QUOTA_OPTION_TYPE_ENABLE)) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled, please enable "
+            "quota");
+        ret = -1;
+        goto out;
+    }
+
+    if (type > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS) {
+        if (!glusterd_is_volume_inode_quota_enabled(volinfo) &&
+            type != GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS) {
+            *op_errstr = gf_strdup(
+                "Inode Quota is disabled, "
+                "please enable inode quota");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!glusterd_is_quota_supported(type, op_errstr)) {
+        ret = -1;
+        goto out;
+    }
+
+    if ((GF_QUOTA_OPTION_TYPE_ENABLE != type) &&
+        (glusterd_check_if_quota_trans_enabled(volinfo) != 0)) {
+        ret = -1;
+        gf_asprintf(op_errstr, "Quota is not enabled on volume %s", volname);
+        goto out;
+    }
+
+    switch (type) {
         case GF_QUOTA_OPTION_TYPE_LIST:
         case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
         case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
         case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
         case GF_QUOTA_OPTION_TYPE_REMOVE:
         case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                /* Quota auxiliary mount is needed by CLI
-                 * for list command and need by glusterd for
-                 * setting/removing limit
-                 */
-                if (is_origin_glusterd (dict)) {
-                        ret = glusterd_create_quota_auxiliary_mount (this,
-                                                                     volname);
-                        if (ret) {
-                                *op_errstr = gf_strdup ("Failed to start aux "
-                                                        "mount");
-                                goto out;
-                        }
+            /* Quota auxiliary mount is needed by CLI
+             * for list command and need by glusterd for
+             * setting/removing limit
+             */
+            if (is_origin_glusterd(dict)) {
+                ret = glusterd_create_quota_auxiliary_mount(this, volname,
+                                                            type);
+                if (ret) {
+                    *op_errstr = gf_strdup(
+                        "Failed to start aux "
+                        "mount");
+                    goto out;
                 }
-                break;
-        }
+            }
+            break;
+    }
 
-        switch (type) {
+    switch (type) {
         case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                ret = dict_get_str (dict, "hard-limit", &hard_limit_str);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Failed to get hard-limit from dict");
-                        goto out;
-                }
-                ret = gf_string2bytesize_int64 (hard_limit_str, &hard_limit);
-                if (ret) {
-                        if (errno == ERANGE || hard_limit < 0)
-                                gf_asprintf (op_errstr, "Hard-limit "
-                                        "value out of range (0 - %"PRId64
-                                        "): %s", hard_limit_str);
-                        else
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_CONVERSION_FAILED,
-                                        "Failed to convert hard-limit "
-                                        "string to value");
-                        goto out;
-                }
-                get_gfid = _gf_true;
-                break;
+            ret = dict_get_strn(dict, "hard-limit", SLEN("hard-limit"),
+                                &hard_limit_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get hard-limit from dict");
+                goto out;
+            }
+            ret = gf_string2bytesize_int64(hard_limit_str, &hard_limit);
+            if (ret) {
+                if (errno == ERANGE || hard_limit < 0)
+                    gf_asprintf(op_errstr,
+                                "Hard-limit "
+                                "value out of range (0 - %" PRId64 "): %s",
+                                hard_limit, hard_limit_str);
+                else
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_CONVERSION_FAILED,
+                           "Failed to convert hard-limit "
+                           "string to value");
+                goto out;
+            }
+            get_gfid = _gf_true;
+            break;
         case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-                get_gfid = _gf_true;
-                break;
+            get_gfid = _gf_true;
+            break;
 
         case GF_QUOTA_OPTION_TYPE_REMOVE:
         case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                get_gfid = _gf_true;
-                break;
+            get_gfid = _gf_true;
+            break;
 
         case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
         case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                ret = _glusterd_validate_quota_opts (dict, type, op_errstr);
-                if (ret)
-                        goto out;
-                break;
+            ret = _glusterd_validate_quota_opts(dict, type, op_errstr);
+            if (ret)
+                goto out;
+            break;
 
         default:
-                break;
-        }
+            break;
+    }
 
-        if (get_gfid == _gf_true) {
-                ret = glusterd_get_gfid_from_brick (dict, volinfo, rsp_dict,
-                                                    op_errstr);
-                if (ret)
-                        goto out;
-        }
+    if (get_gfid == _gf_true) {
+        ret = glusterd_get_gfid_from_brick(dict, volinfo, rsp_dict, op_errstr);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
- out:
-        if (ret && op_errstr && *op_errstr)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_STAGE_QUOTA_FAIL, "%s", *op_errstr);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+out:
+    if (ret && op_errstr && *op_errstr)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_QUOTA_FAIL, "%s",
+               *op_errstr);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-         return ret;
+    return ret;
 }
diff --git a/xlators/features/ganesha/src/ganesha-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-quota.h
index c4976c01afc..ab2092a9c6a 100644
--- a/xlators/features/ganesha/src/ganesha-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -7,15 +7,11 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef __GANESHA_MEM_TYPES_H__
-#define __GANESHA_MEM_TYPES_H__
+#ifndef _GLUSTERD_QUOTA_
+#define _GLUSTERD_QUOTA_
 
-
-#include "mem-types.h"
-
-enum gf_ganesha_mem_types_ {
-     gf_ganesha_mt_priv_t = gf_common_mt_end + 1,
-     gf_ganesha_mt_end
-};
+int
+glusterd_store_quota_config(glusterd_volinfo_t *volinfo, char *path,
+                            char *gfid_str, int opcode, char **op_errstr);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c
index f3475a3f0ec..f26d832a06d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c
@@ -8,8 +8,8 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
@@ -20,203 +20,198 @@
 char *quotad_svc_name = "quotad";
 
 void
-glusterd_quotadsvc_build (glusterd_svc_t *svc)
+glusterd_quotadsvc_build(glusterd_svc_t *svc)
 {
-        svc->manager = glusterd_quotadsvc_manager;
-        svc->start = glusterd_quotadsvc_start;
-        svc->stop = glusterd_svc_stop;
+    svc->manager = glusterd_quotadsvc_manager;
+    svc->start = glusterd_quotadsvc_start;
+    svc->stop = glusterd_svc_stop;
 }
 
-int glusterd_quotadsvc_init (glusterd_svc_t *svc)
+int
+glusterd_quotadsvc_init(glusterd_svc_t *svc)
 {
-        int              ret                = -1;
-        char             volfile[PATH_MAX]  = {0,};
-        glusterd_conf_t *conf               = THIS->private;
+    int ret = -1;
 
-        ret = glusterd_svc_init (svc, quotad_svc_name);
-        if (ret)
-                goto out;
+    ret = glusterd_svc_init(svc, quotad_svc_name);
+    if (ret)
+        goto out;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_quotadsvc_create_volfile ()
+glusterd_quotadsvc_create_volfile()
 {
-        char             filepath[PATH_MAX] = {0,};
-        glusterd_conf_t *conf               = THIS->private;
-
-        glusterd_svc_build_volfile_path (quotad_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
-        return glusterd_create_global_volfile (build_quotad_graph,
-                                               filepath, NULL);
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *conf = THIS->private;
+
+    glusterd_svc_build_volfile_path(quotad_svc_name, conf->workdir, filepath,
+                                    sizeof(filepath));
+    return glusterd_create_global_volfile(build_quotad_graph, filepath, NULL);
 }
 
 int
-glusterd_quotadsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_quotadsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                 ret     = 0;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        if (!svc->inited) {
-                ret = glusterd_quotadsvc_init (svc);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FAILED_INIT_QUOTASVC, "Failed to init "
-                                "quotad service");
-                        goto out;
-                } else {
-                        svc->inited = _gf_true;
-                        gf_msg_debug (THIS->name, 0, "quotad service "
-                                      "initialized");
-                }
-        }
-
-        volinfo = data;
-
-        /* If all the volumes are stopped or all shd compatible volumes
-         * are stopped then stop the service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
-         * Otherwise create volfile and restart service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
-         */
-        if (glusterd_are_all_volumes_stopped () ||
-            glusterd_all_volumes_with_quota_stopped ()) {
-                if (!(volinfo && !glusterd_is_volume_quota_enabled (volinfo))) {
-                        ret = svc->stop (svc, SIGTERM);
-                }
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    if (!svc->inited) {
+        ret = glusterd_quotadsvc_init(svc);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_QUOTASVC,
+                   "Failed to init "
+                   "quotad service");
+            goto out;
         } else {
-                if (!(volinfo && !glusterd_is_volume_quota_enabled (volinfo))) {
-                        ret = glusterd_quotadsvc_create_volfile ();
-                        if (ret)
-                                goto out;
-
-                        ret = svc->stop (svc, SIGTERM);
-                        if (ret)
-                                goto out;
-
-                        ret = svc->start (svc, flags);
-                        if (ret)
-                                goto out;
-
-                        ret = glusterd_conn_connect (&(svc->conn));
-                        if (ret)
-                                goto out;
-                }
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0,
+                         "quotad service "
+                         "initialized");
         }
-out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    }
+
+    volinfo = data;
+
+    /* If all the volumes are stopped or all shd compatible volumes
+     * are stopped then stop the service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     * Otherwise create volfile and restart service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     */
+    if (glusterd_are_all_volumes_stopped() ||
+        glusterd_all_volumes_with_quota_stopped()) {
+        if (!(volinfo && !glusterd_is_volume_quota_enabled(volinfo))) {
+            ret = svc->stop(svc, SIGTERM);
+        }
+    } else {
+        if (!(volinfo && !glusterd_is_volume_quota_enabled(volinfo))) {
+            ret = glusterd_quotadsvc_create_volfile();
+            if (ret)
+                goto out;
 
-        return ret;
-}
+            ret = svc->stop(svc, SIGTERM);
+            if (ret)
+                goto out;
 
-int
-glusterd_quotadsvc_start (glusterd_svc_t *svc, int flags)
-{
-        int              i         = 0;
-        int              ret       = -1;
-        dict_t          *cmdline   = NULL;
-        char             key[16]   = {0};
-        char            *options[] = {
-                                      "*replicate*.entry-self-heal=off",
-                                      "--xlator-option",
-                                      "*replicate*.metadata-self-heal=off",
-                                      "--xlator-option",
-                                      "*replicate*.data-self-heal=off",
-                                      "--xlator-option",
-                                      NULL
-                                      };
-
-        cmdline = dict_new ();
-        if (!cmdline)
+            ret = svc->start(svc, flags);
+            if (ret)
                 goto out;
 
-        for (i = 0; options[i]; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "arg%d", i);
-                ret = dict_set_str (cmdline, key, options[i]);
-                if (ret)
-                        goto out;
+            ret = glusterd_conn_connect(&(svc->conn));
+            if (ret)
+                goto out;
         }
-
-        ret = glusterd_svc_start (svc, flags, cmdline);
-
+    }
 out:
-        if (cmdline)
-                dict_unref (cmdline);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
 
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_quotadsvc_reconfigure ()
+glusterd_quotadsvc_start(glusterd_svc_t *svc, int flags)
 {
-        int              ret             = -1;
-        xlator_t        *this            = NULL;
-        glusterd_conf_t *priv            = NULL;
-        gf_boolean_t     identical       = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        if (glusterd_all_volumes_with_quota_stopped ())
-                goto manager;
-
-        /*
-         * Check both OLD and NEW volfiles, if they are SAME by size
-         * and cksum i.e. "character-by-character". If YES, then
-         * NOTHING has been changed, just return.
-         */
-        ret = glusterd_svc_check_volfile_identical (priv->quotad_svc.name,
-                                                    build_quotad_graph,
-                                                    &identical);
+    int i = 0;
+    int ret = -1;
+    dict_t *cmdline = NULL;
+    char key[16] = {0};
+    char *options[] = {svc->name, "--process-name", NULL};
+
+    cmdline = dict_new();
+    if (!cmdline) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    for (i = 0; options[i]; i++) {
+        ret = snprintf(key, sizeof(key), "arg%d", i);
+        ret = dict_set_strn(cmdline, key, ret, options[i]);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (identical) {
-                ret = 0;
-                goto out;
-        }
+    ret = glusterd_svc_start(svc, flags, cmdline);
 
-        /*
-         * They are not identical. Find out if the topology is changed
-         * OR just the volume options. If just the options which got
-         * changed, then inform the xlator to reconfigure the options.
-         */
-        identical = _gf_false; /* RESET the FLAG */
-        ret = glusterd_svc_check_topology_identical (priv->quotad_svc.name,
-                                                     build_quotad_graph,
-                                                     &identical);
-        if (ret)
-                goto out;
+out:
+    if (cmdline)
+        dict_unref(cmdline);
 
-        /* Topology is not changed, but just the options. But write the
-         * options to quotad volfile, so that quotad will be reconfigured.
-         */
-        if (identical) {
-                ret = glusterd_quotadsvc_create_volfile ();
-                if (ret == 0) {/* Only if above PASSES */
-                        ret = glusterd_fetchspec_notify (THIS);
-                }
-                goto out;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+int
+glusterd_quotadsvc_reconfigure()
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (glusterd_all_volumes_with_quota_stopped())
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_volfile_identical(priv->quotad_svc.name,
+                                               build_quotad_graph, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->quotad_svc.name,
+                                                build_quotad_graph, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to quotad volfile, so that quotad will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_quotadsvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
         }
+        goto out;
+    }
 manager:
-        /*
-         * quotad volfile's topology has been changed. quotad server needs
-         * to be RESTARTED to ACT on the changed volfile.
-         */
-        ret = priv->quotad_svc.manager (&(priv->quotad_svc), NULL,
-                                        PROC_START_NO_WAIT);
+    /*
+     * quotad volfile's topology has been changed. quotad server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->quotad_svc.manager(&(priv->quotad_svc), NULL,
+                                   PROC_START_NO_WAIT);
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "Quotad", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h
index 91da50dc36d..e8d9bbee964 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h
@@ -14,18 +14,18 @@
 #include "glusterd-svc-mgmt.h"
 
 void
-glusterd_quotadsvc_build (glusterd_svc_t *svc);
+glusterd_quotadsvc_build(glusterd_svc_t *svc);
 
 int
-glusterd_quotadsvc_init (glusterd_svc_t *svc);
+glusterd_quotadsvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_quotadsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_quotadsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_quotadsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_quotadsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_quotadsvc_reconfigure ();
+glusterd_quotadsvc_reconfigure();
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-rcu.h b/xlators/mgmt/glusterd/src/glusterd-rcu.h
index 15beac5a745..c85f9bea8f8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rcu.h
+++ b/xlators/mgmt/glusterd/src/glusterd-rcu.h
@@ -21,7 +21,7 @@
 #include "rculist-extra.h"
 #endif
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 /* gd_rcu_head is a composite struct, composed of struct rcu_head and a this
  * pointer, which is used to pass the THIS pointer to call_rcu callbacks.
@@ -29,8 +29,8 @@
  * Use this in place of struct rcu_head when embedding into another struct
  */
 typedef struct glusterd_rcu_head_ {
-        struct rcu_head head;
-        xlator_t *this;
+    struct rcu_head head;
+    xlator_t *this;
 } gd_rcu_head;
 
 #endif /* _GLUSTERD_RCU_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index e0eee02ed52..458bf168ede 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -13,1121 +13,1410 @@
 #include <sys/resource.h>
 #include <sys/statvfs.h>
 
-#include "globals.h"
-#include "compat.h"
+#include <glusterfs/compat.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/timer.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-messages.h"
 #include "glusterd-store.h"
-#include "run.h"
+#include <glusterfs/run.h>
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 
+#define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo)                           \
+    do {                                                                       \
+        int32_t _defrag_sockfile_len;                                          \
+        char tmppath[PATH_MAX] = {                                             \
+            0,                                                                 \
+        };                                                                     \
+        _defrag_sockfile_len = snprintf(                                       \
+            tmppath, PATH_MAX,                                                 \
+            DEFAULT_VAR_RUN_DIRECTORY "/gluster-%s-%s-%s.sock", "rebalance",   \
+            volinfo->volname, uuid_utoa(MY_UUID));                             \
+        if ((_defrag_sockfile_len < 0) ||                                      \
+            (_defrag_sockfile_len >= PATH_MAX)) {                              \
+            path[0] = 0;                                                       \
+        } else {                                                               \
+            glusterd_set_socket_filepath(tmppath, path, sizeof(path));         \
+        }                                                                      \
+    } while (0)
+
 int32_t
-glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe);
+glusterd_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe);
 int
-glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
-                                size_t len, glusterd_op_t op)
+glusterd_defrag_start_validate(glusterd_volinfo_t *volinfo, char *op_errstr,
+                               size_t len, glusterd_op_t op)
 {
-        int      ret = -1;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        /* Check only if operation is not remove-brick */
-        if ((GD_OP_REMOVE_BRICK != op) &&
-            !gd_is_remove_brick_committed (volinfo)) {
-                gf_msg_debug (this->name, 0, "A remove-brick task on "
-                        "volume %s is not yet committed", volinfo->volname);
-                snprintf (op_errstr, len, "A remove-brick task on volume %s is"
-                          " not yet committed. Either commit or stop the "
-                          "remove-brick task.", volinfo->volname);
-                goto out;
-        }
-
-        if (glusterd_is_defrag_on (volinfo)) {
-                gf_msg_debug (this->name, 0,
-                        "rebalance on volume %s already started",
-                        volinfo->volname);
-                snprintf (op_errstr, len, "Rebalance on %s is already started",
-                          volinfo->volname);
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    /* Check only if operation is not remove-brick */
+    if ((GD_OP_REMOVE_BRICK != op) && !gd_is_remove_brick_committed(volinfo)) {
+        gf_msg_debug(this->name, 0,
+                     "A remove-brick task on "
+                     "volume %s is not yet committed",
+                     volinfo->volname);
+        snprintf(op_errstr, len,
+                 "A remove-brick task on volume %s is"
+                 " not yet committed. Either commit or stop the "
+                 "remove-brick task.",
+                 volinfo->volname);
+        goto out;
+    }
+
+    if (glusterd_is_defrag_on(volinfo)) {
+        gf_msg_debug(this->name, 0, "rebalance on volume %s already started",
+                     volinfo->volname);
+        snprintf(op_errstr, len, "Rebalance on %s is already started",
+                 volinfo->volname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-__glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event, void *data)
+__glusterd_defrag_notify(struct rpc_clnt *rpc, void *mydata,
+                         rpc_clnt_event_t event, void *data)
 {
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_defrag_info_t  *defrag  = NULL;
-        int                     ret      = 0;
-        char                    pidfile[PATH_MAX];
-        glusterd_conf_t        *priv    = NULL;
-        xlator_t               *this    = NULL;
-
-        this = THIS;
-        if (!this)
-                return 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    char pidfile[PATH_MAX];
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int pid = -1;
+
+    this = THIS;
+    if (!this)
+        return 0;
 
-        priv = this->private;
-        if (!priv)
-                return 0;
+    priv = this->private;
+    if (!priv)
+        return 0;
 
-        volinfo = mydata;
-        if (!volinfo)
-                return 0;
+    volinfo = mydata;
+    if (!volinfo)
+        return 0;
 
-        defrag = volinfo->rebal.defrag;
-        if (!defrag)
-                return 0;
+    defrag = volinfo->rebal.defrag;
+    if (!defrag)
+        return 0;
 
-        if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
-                volinfo->rebal.defrag = NULL;
+    if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
+        volinfo->rebal.defrag = NULL;
 
-        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+    GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                if (defrag->connected)
-                        return 0;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            if (defrag->connected)
+                return 0;
 
-                LOCK (&defrag->lock);
-                {
-                        defrag->connected = 1;
-                }
-                UNLOCK (&defrag->lock);
+            LOCK(&defrag->lock);
+            {
+                defrag->connected = 1;
+            }
+            UNLOCK(&defrag->lock);
 
-               gf_msg_debug (this->name, 0, "%s got RPC_CLNT_CONNECT",
-                        rpc->conn.name);
-               break;
+            gf_msg_debug(this->name, 0, "%s got RPC_CLNT_CONNECT",
+                         rpc->conn.name);
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                if (!defrag->connected)
-                        return 0;
+        case RPC_CLNT_DISCONNECT: {
+            if (!defrag->connected)
+                return 0;
+
+            LOCK(&defrag->lock);
+            {
+                defrag->connected = 0;
+            }
+            UNLOCK(&defrag->lock);
 
-                LOCK (&defrag->lock);
-                {
-                        defrag->connected = 0;
+            if (!gf_is_service_running(pidfile, &pid)) {
+                if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_STARTED) {
+                    volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_FAILED;
                 }
-                UNLOCK (&defrag->lock);
-
-                if (!gf_is_service_running (pidfile, NULL)) {
-                        if (volinfo->rebal.defrag_status ==
-                                                GF_DEFRAG_STATUS_STARTED) {
-                                volinfo->rebal.defrag_status =
-                                                   GF_DEFRAG_STATUS_FAILED;
-                        }
-                 }
-
-                glusterd_store_perform_node_state_store (volinfo);
-
-                glusterd_defrag_rpc_put (defrag);
-                if (defrag->cbk_fn)
-                        defrag->cbk_fn (volinfo,
-                                        volinfo->rebal.defrag_status);
-
-                GF_FREE (defrag);
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_REBALANCE_DISCONNECTED,
-                        "Rebalance process for volume %s has disconnected.",
-                        volinfo->volname);
-                break;
+            }
+
+            glusterd_store_perform_node_state_store(volinfo);
+
+            rpc_clnt_disable(defrag->rpc);
+            glusterd_defrag_rpc_put(defrag);
+            if (defrag->cbk_fn)
+                defrag->cbk_fn(volinfo, volinfo->rebal.defrag_status);
+
+            GF_FREE(defrag);
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REBALANCE_DISCONNECTED,
+                   "Rebalance process for volume %s has disconnected.",
+                   volinfo->volname);
+            break;
         }
         case RPC_CLNT_DESTROY:
-                glusterd_volinfo_unref (volinfo);
-                break;
+            glusterd_volinfo_unref(volinfo);
+            break;
         default:
-                gf_msg_trace (this->name, 0,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
-                        rpc_clnt_event_t event, void *data)
+glusterd_defrag_notify(struct rpc_clnt *rpc, void *mydata,
+                       rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify (rpc, mydata, event,
-                                           data, __glusterd_defrag_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_defrag_notify);
 }
 
 int
-glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
-                              size_t len, int cmd, defrag_cbk_fn_t cbk,
-                              glusterd_op_t op)
+glusterd_handle_defrag_start(glusterd_volinfo_t *volinfo, char *op_errstr,
+                             size_t len, int cmd, defrag_cbk_fn_t cbk,
+                             glusterd_op_t op)
 {
-        int                    ret = -1;
-        glusterd_defrag_info_t *defrag =  NULL;
-        runner_t               runner = {0,};
-        glusterd_conf_t        *priv = NULL;
-        char                   defrag_path[PATH_MAX];
-        char                   sockfile[PATH_MAX] = {0,};
-        char                   pidfile[PATH_MAX] = {0,};
-        char                   logfile[PATH_MAX] = {0,};
-        char                   volname[PATH_MAX] = {0,};
-        char                   valgrind_logfile[PATH_MAX] = {0,};
-        char                   *volfileserver = NULL;
-
-        priv    = THIS->private;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (op_errstr);
-
-
-        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
-        if (ret)
-                goto out;
-        if (!volinfo->rebal.defrag)
-                volinfo->rebal.defrag =
-                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
-                                   gf_gld_mt_defrag_info);
-        if (!volinfo->rebal.defrag)
-                goto out;
+    xlator_t *this = NULL;
+    int ret = -1;
+    glusterd_defrag_info_t *defrag = NULL;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char defrag_path[PATH_MAX];
+    char sockfile[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volname[PATH_MAX] = {
+        0,
+    };
+    char valgrind_logfile[PATH_MAX] = {
+        0,
+    };
+    char msg[1024] = {
+        0,
+    };
+    char *volfileserver = NULL;
+    char *localtime_logging = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", priv, out);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(op_errstr);
+
+    ret = glusterd_defrag_start_validate(volinfo, op_errstr, len, op);
+    if (ret)
+        goto out;
+    if (!volinfo->rebal.defrag)
+        volinfo->rebal.defrag = GF_CALLOC(1, sizeof(*volinfo->rebal.defrag),
+                                          gf_gld_mt_defrag_info);
+    if (!volinfo->rebal.defrag)
+        goto out;
+
+    defrag = volinfo->rebal.defrag;
+
+    defrag->cmd = cmd;
+
+    volinfo->rebal.defrag_cmd = cmd;
+    volinfo->rebal.op = op;
+
+    LOCK_INIT(&defrag->lock);
+
+    volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+    glusterd_volinfo_reset_defrag_stats(volinfo);
+    glusterd_store_perform_node_state_store(volinfo);
+
+    GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);
+    ret = mkdir_p(defrag_path, 0755, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create "
+               "directory %s",
+               defrag_path);
+        goto out;
+    }
+
+    GLUSTERD_GET_DEFRAG_SOCK_FILE(sockfile, volinfo);
+    GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+    snprintf(logfile, PATH_MAX, "%s/%s-%s.log", priv->logdir, volinfo->volname,
+             "rebalance");
+    runinit(&runner);
+
+    if (this->ctx->cmd_args.vgtool != _gf_none) {
+        snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s-rebalance.log",
+                 priv->logdir, volinfo->volname);
+
+        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+            runner_add_args(&runner, "valgrind", "--leak-check=full",
+                            "--trace-children=yes", "--track-origins=yes",
+                            NULL);
+        else
+            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
+
+        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
+    }
+
+    snprintf(volname, sizeof(volname), "rebalance/%s", volinfo->volname);
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+
+    runner_add_args(
+        &runner, SBIN_DIR "/glusterfs", "-s", volfileserver, "--volfile-id",
+        volname, "--xlator-option", "*dht.use-readdirp=yes", "--xlator-option",
+        "*dht.lookup-unhashed=yes", "--xlator-option",
+        "*dht.assert-no-child-down=yes", "--xlator-option",
+        "*dht.readdir-optimize=on", "--process-name", "rebalance", NULL);
+
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "*dht.rebalance-cmd=%d", cmd);
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "*dht.commit-hash=%u",
+                     volinfo->rebal.commit_hash);
+    runner_add_arg(&runner, "--socket-file");
+    runner_argprintf(&runner, "%s", sockfile);
+    runner_add_arg(&runner, "--pid-file");
+    runner_argprintf(&runner, "%s", pidfile);
+    runner_add_arg(&runner, "-l");
+    runner_argprintf(&runner, "%s", logfile);
+    if (volinfo->memory_accounting)
+        runner_add_arg(&runner, "--mem-accounting");
+    if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                      SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
+                      &localtime_logging) == 0) {
+        if (strcmp(localtime_logging, "enable") == 0)
+            runner_add_arg(&runner, "--localtime-logging");
+    }
+
+    snprintf(msg, sizeof(msg), "Starting the rebalance service for volume %s",
+             volinfo->volname);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    ret = runner_run_nowait(&runner);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "rebalance command failed");
+        goto out;
+    }
+
+    sleep(5);
+
+    ret = glusterd_rebalance_rpc_create(volinfo);
+
+    // FIXME: this cbk is passed as NULL in all occurrences. May be
+    // we never needed it.
+    if (cbk)
+        defrag->cbk_fn = cbk;
 
-        defrag = volinfo->rebal.defrag;
-
-        defrag->cmd = cmd;
-
-        volinfo->rebal.defrag_cmd = cmd;
-        volinfo->rebal.op = op;
-
-        LOCK_INIT (&defrag->lock);
-
-        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
-
-        glusterd_volinfo_reset_defrag_stats (volinfo);
-        glusterd_store_perform_node_state_store (volinfo);
-
-        GLUSTERD_GET_DEFRAG_DIR (defrag_path, volinfo, priv);
-        ret = mkdir_p (defrag_path, 0777, _gf_true);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Failed to create "
-                        "directory %s", defrag_path);
-                goto out;
-        }
-
-        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
-        GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, priv);
-        snprintf (logfile, PATH_MAX, "%s/%s-%s.log",
-                    DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname,
-                    (cmd == GF_DEFRAG_CMD_START_TIER ? "tier":"rebalance"));
-        runinit (&runner);
-
-        if (priv->valgrind) {
-                snprintf (valgrind_logfile, PATH_MAX,
-                          "%s/valgrind-%s-rebalance.log",
-                          DEFAULT_LOG_FILE_DIRECTORY,
-                          volinfo->volname);
-
-                runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", "--track-origins=yes",
-                                 NULL);
-                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
-        }
-
-        snprintf (volname, sizeof(volname), "rebalance/%s", volinfo->volname);
-
-        if (dict_get_str (THIS->options, "transport.socket.bind-address",
-                          &volfileserver) == 0) {
-               /*In the case of running multiple glusterds on a single machine,
-                *we should ensure that log file and unix socket file shouls be
-                *unique in given cluster */
-
-                GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD (sockfile, volinfo,
-                                                   priv);
-                snprintf (logfile, PATH_MAX, "%s/%s-%s-%s.log",
-                          DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname,
-                          (cmd == GF_DEFRAG_CMD_START_TIER ?
-                           "tier":"rebalance"),
-                          uuid_utoa(MY_UUID));
-
-        } else {
-                volfileserver = "localhost";
-        }
-
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", volfileserver, "--volfile-id", volname,
-                         "--xlator-option", "*dht.use-readdirp=yes",
-                         "--xlator-option", "*dht.lookup-unhashed=yes",
-                         "--xlator-option", "*dht.assert-no-child-down=yes",
-                         "--xlator-option", "*replicate*.data-self-heal=off",
-                         "--xlator-option",
-                         "*replicate*.metadata-self-heal=off",
-                         "--xlator-option", "*replicate*.entry-self-heal=off",
-                         "--xlator-option", "*dht.readdir-optimize=on",
-                         NULL);
-
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                runner_add_arg (&runner, "--xlator-option");
-                runner_argprintf (&runner,
-                                  "*tier-dht.xattr-name=trusted.tier.tier-dht");
-        }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf ( &runner, "*dht.rebalance-cmd=%d",cmd);
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "*dht.commit-hash=%u",
-                          volinfo->rebal.commit_hash);
-        runner_add_arg (&runner, "--socket-file");
-        runner_argprintf (&runner, "%s",sockfile);
-        runner_add_arg (&runner, "--pid-file");
-        runner_argprintf (&runner, "%s",pidfile);
-        runner_add_arg (&runner, "-l");
-        runner_argprintf (&runner, logfile);
-        if (volinfo->memory_accounting)
-                runner_add_arg (&runner, "--mem-accounting");
-
-        ret = runner_run_nowait (&runner);
-        if (ret) {
-                gf_msg_debug ("glusterd", 0, "rebalance command failed");
-                goto out;
-        }
+int
+glusterd_rebalance_defrag_init(glusterd_volinfo_t *volinfo, defrag_cbk_fn_t cbk)
 
-        sleep (5);
+{
+    glusterd_defrag_info_t *defrag = NULL;
+    int ret = -1;
+
+    if (!volinfo->rebal.defrag) {
+        volinfo->rebal.defrag = GF_CALLOC(1, sizeof(*volinfo->rebal.defrag),
+                                          gf_gld_mt_defrag_info);
+    } else {
+        /*
+         * if defrag variable is already initialized,
+         * we skip the initialization.
+         */
+        ret = 0;
+        goto out;
+    }
+
+    if (!volinfo->rebal.defrag)
+        goto out;
+    defrag = volinfo->rebal.defrag;
+
+    defrag->cmd = volinfo->rebal.defrag_cmd;
+    LOCK_INIT(&defrag->lock);
+    if (cbk)
+        defrag->cbk_fn = cbk;
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = glusterd_rebalance_rpc_create (volinfo, _gf_false);
+int
+glusterd_rebalance_rpc_create(glusterd_volinfo_t *volinfo)
+{
+    dict_t *options = NULL;
+    char sockfile[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    glusterd_defrag_info_t *defrag = volinfo->rebal.defrag;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    // rebalance process is not started
+    if (!defrag)
+        goto out;
+
+    options = dict_new();
+    if (!options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    GLUSTERD_GET_DEFRAG_SOCK_FILE(sockfile, volinfo);
+
+    /* Setting frame-timeout to 10mins (600seconds).
+     * Unix domain sockets ensures that the connection is reliable. The
+     * default timeout of 30mins used for unreliable network connections is
+     * too long for unix domain socket connections.
+     */
+    ret = rpc_transport_unix_options_build(options, sockfile, 600);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNIX_OP_BUILD_FAIL,
+               "Unix options build failed");
+        goto out;
+    }
+
+    glusterd_volinfo_ref(volinfo);
+    ret = glusterd_rpc_create(&defrag->rpc, options, glusterd_defrag_notify,
+                              volinfo, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
+               "Glusterd RPC creation failed");
+        goto out;
+    }
+    ret = 0;
+out:
+    if (options)
+        dict_unref(options);
+    return ret;
+}
 
-        //FIXME: this cbk is passed as NULL in all occurrences. May be
-        //we never needed it.
-        if (cbk)
-                defrag->cbk_fn = cbk;
+int
+glusterd_rebalance_cmd_validate(int cmd, char *volname,
+                                glusterd_volinfo_t **volinfo, char *op_errstr,
+                                size_t len)
+{
+    int ret = -1;
+
+    if (glusterd_volinfo_find(volname, volinfo)) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Received rebalance on invalid"
+               " volname %s",
+               volname);
+        snprintf(op_errstr, len, "Volume %s does not exist", volname);
+        goto out;
+    }
+    if ((*volinfo)->brick_count <= (*volinfo)->dist_leaf_count) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_DISTRIBUTE,
+               "Volume %s is not a "
+               "distribute type or contains only 1 brick",
+               volname);
+        snprintf(op_errstr, len,
+                 "Volume %s is not a distribute "
+                 "volume or contains only 1 brick.\n"
+                 "Not performing rebalance",
+                 volname);
+        goto out;
+    }
+
+    if ((*volinfo)->status != GLUSTERD_STATUS_STARTED) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_STOPPED,
+               "Received rebalance on stopped"
+               " volname %s",
+               volname);
+        snprintf(op_errstr, len,
+                 "Volume %s needs to "
+                 "be started to perform rebalance",
+                 volname);
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_rebalance_defrag_init (glusterd_volinfo_t *volinfo,
-                                defrag_cbk_fn_t cbk)
-
+__glusterd_handle_defrag_volume(rpcsvc_request_t *req)
 {
-        glusterd_defrag_info_t  *defrag         = NULL;
-        int                      ret            = -1;
-
-        if (!volinfo->rebal.defrag) {
-                volinfo->rebal.defrag =
-                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
-                                   gf_gld_mt_defrag_info);
-        } else {
-                /*
-                 * if defrag variable is already initialized,
-                 * we skip the initialization.
-                 */
-                ret = 0;
-                goto out;
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    glusterd_conf_t *priv = NULL;
+    int32_t op = GD_OP_NONE;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    gf_cli_defrag_type cmd = 0;
+    char msg[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
         }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          (int32_t *)&cmd);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_set_static_bin(dict, "node-uuid", MY_UUID, 16);
+    if (ret)
+        goto out;
+
+    if ((cmd == GF_DEFRAG_CMD_STATUS) || (cmd == GF_DEFRAG_CMD_STOP)) {
+        op = GD_OP_DEFRAG_BRICK_VOLUME;
+    } else
+        op = GD_OP_REBALANCE;
+
+    if (priv->op_version < GD_OP_VERSION_6_0) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than %d. Falling back "
+                     "to op-sm framework.",
+                     GD_OP_VERSION_6_0);
+        ret = glusterd_op_begin(req, op, dict, msg, sizeof(msg));
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(req, op,
+                                                                      dict);
+    }
+out:
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        ret = glusterd_op_send_cli_response(GD_OP_REBALANCE, ret, 0, req, dict,
+                                            msg);
+    }
+
+    free(cli_req.dict.dict_val);  // malloced by xdr
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        if (!volinfo->rebal.defrag)
-                goto out;
-        defrag = volinfo->rebal.defrag;
+int
+glusterd_handle_defrag_volume(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req, __glusterd_handle_defrag_volume);
+}
 
-        defrag->cmd = volinfo->rebal.defrag_cmd;
-        LOCK_INIT (&defrag->lock);
-        if (cbk)
-                defrag->cbk_fn = cbk;
-        ret = 0;
-out:
+static int
+glusterd_brick_validation(dict_t *dict, char *key, data_t *value, void *data)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = data;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_volume_brickinfo_get_by_brick(value->data, volinfo,
+                                                 &brickinfo, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+               "Incorrect brick %s for "
+               "volume %s",
+               value->data, volinfo->volname);
+        return ret;
+    }
+
+    if (!brickinfo->decommissioned) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+               "Incorrect brick %s for "
+               "volume %s",
+               value->data, volinfo->volname);
+        ret = -1;
         return ret;
+    }
 
+    return ret;
 }
 
 int
-glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
-                               gf_boolean_t reconnect)
+glusterd_set_rebalance_id_in_rsp_dict(dict_t *req_dict, dict_t *rsp_dict)
 {
-        dict_t                  *options = NULL;
-        char                     sockfile[PATH_MAX] = {0,};
-        int                      ret = -1;
-        glusterd_defrag_info_t  *defrag = volinfo->rebal.defrag;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        struct stat             buf = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        //rebalance process is not started
-        if (!defrag)
-                goto out;
-
-        //rpc obj for rebalance process already in place.
-        if (glusterd_defrag_rpc_get (defrag)) {
+    int ret = -1;
+    int32_t cmd = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(req_dict);
+
+    ret = dict_get_strn(rsp_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_int32n(rsp_dict, "rebalance-command",
+                          SLEN("rebalance-command"), &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd not found");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to validate");
+        goto out;
+    }
+
+    /* reblance id is generted in glusterd_mgmt_v3_op_stage_rebalance(), but
+     * rsp_dict is unavailable there. So copying it to rsp_dict from req_dict
+     * here. So that cli can display the rebalance id.*/
+    if ((cmd == GF_DEFRAG_CMD_START) ||
+        (cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX) ||
+        (cmd == GF_DEFRAG_CMD_START_FORCE)) {
+        if (is_origin_glusterd(rsp_dict)) {
+            ret = dict_get_strn(req_dict, GF_REBALANCE_TID_KEY,
+                                SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+            if (ret) {
+                snprintf(msg, sizeof(msg), "Missing rebalance-id");
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
                 ret = 0;
-                glusterd_defrag_rpc_put (defrag);
-                goto out;
-        }
-        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
-        /* If reconnecting check if defrag sockfile exists in the new location
-         * in /var/run/ , if it does not try the old location
-         */
-        if (reconnect) {
-                ret = sys_stat (sockfile, &buf);
-                /* TODO: Remove this once we don't need backward compatibility
-                 * with the older path
-                 */
-                if (ret && (errno == ENOENT)) {
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                GD_MSG_FILE_OP_FAILED, "Rebalance sockfile "
-                                "%s does not exist. Trying old path.",
-                                sockfile);
-                        GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD (sockfile, volinfo,
-                                                           priv);
-                        ret =sys_stat (sockfile, &buf);
-                        if (ret && (ENOENT == errno)) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_REBAL_NO_SOCK_FILE, "Rebalance "
-                                        "sockfile %s does not exist", sockfile);
-                                goto out;
-                        }
+            } else {
+                gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 rsp_dict, GF_REBALANCE_TID_KEY,
+                                                 SLEN(GF_REBALANCE_TID_KEY));
+                if (ret) {
+                    snprintf(msg, sizeof(msg),
+                             "Failed to set rebalance id for volume %s",
+                             volname);
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_DICT_SET_FAILED, "%s", msg);
                 }
+            }
         }
-
-        /* Setting frame-timeout to 10mins (600seconds).
-         * Unix domain sockets ensures that the connection is reliable. The
-         * default timeout of 30mins used for unreliable network connections is
-         * too long for unix domain socket connections.
-         */
-        ret = rpc_transport_unix_options_build (&options, sockfile, 600);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNIX_OP_BUILD_FAIL,
-                        "Unix options build failed");
-                goto out;
-        }
-
-        glusterd_volinfo_ref (volinfo);
-        ret = glusterd_rpc_create (&defrag->rpc, options,
-                                   glusterd_defrag_notify, volinfo);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
-                        "Glusterd RPC creation failed");
+    }
+
+    /* Set task-id, if available, in rsp_dict for operations other than
+     * start. This is needed when we want rebalance id in xml output
+     */
+    if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+        if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+            if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+                ret = glusterd_copy_uuid_to_dict(
+                    volinfo->rebal.rebalance_id, rsp_dict,
+                    GF_REMOVE_BRICK_TID_KEY, SLEN(GF_REMOVE_BRICK_TID_KEY));
+            else
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 rsp_dict, GF_REBALANCE_TID_KEY,
+                                                 SLEN(GF_REBALANCE_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set task-id for volume %s", volname);
                 goto out;
+            }
         }
-        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_rebalance_cmd_validate (int cmd, char *volname,
-                                 glusterd_volinfo_t **volinfo,
-                                 char *op_errstr, size_t len)
+glusterd_mgmt_v3_op_stage_rebalance(dict_t *dict, char **op_errstr)
 {
-        int ret = -1;
-
-        if (glusterd_volinfo_find(volname, volinfo)) {
-                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "Received rebalance on invalid"
-                        " volname %s", volname);
-                snprintf (op_errstr, len, "Volume %s does not exist",
-                          volname);
+    char *volname = NULL;
+    char *cmd_str = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    char *task_id_str = NULL;
+    xlator_t *this = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd not found");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to validate");
+        goto out;
+    }
+    switch (cmd) {
+        case GF_DEFRAG_CMD_START:
+        case GF_DEFRAG_CMD_START_LAYOUT_FIX:
+            /* Check if the connected clients are all of version
+             * glusterfs-3.6 and higher. This is needed to prevent some data
+             * loss issues that could occur when older clients are connected
+             * when rebalance is run. This check can be bypassed by using
+             * 'force'
+             */
+            ret = glusterd_check_client_op_version_support(
+                volname, GD_OP_VERSION_3_6_0, NULL);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume %s has one or "
+                                  "more connected clients of a version"
+                                  " lower than GlusterFS-v3.6.0. "
+                                  "Starting rebalance in this state "
+                                  "could lead to data loss.\nPlease "
+                                  "disconnect those clients before "
+                                  "attempting this command again.",
+                                  volname);
                 goto out;
-        }
-        if ((*volinfo)->brick_count <= (*volinfo)->dist_leaf_count) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_DISTRIBUTE, "Volume %s is not a "
-                "distribute type or contains only 1 brick", volname);
-                snprintf (op_errstr, len, "Volume %s is not a distribute "
-                          "volume or contains only 1 brick.\n"
-                          "Not performing rebalance", volname);
+            }
+            /* Fall through */
+        case GF_DEFRAG_CMD_START_FORCE:
+            if (is_origin_glusterd(dict)) {
+                ret = glusterd_generate_and_set_task_id(
+                    dict, GF_REBALANCE_TID_KEY, SLEN(GF_REBALANCE_TID_KEY));
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                           "Failed to generate task-id");
+                    goto out;
+                }
+            } else {
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+                if (ret) {
+                    snprintf(msg, sizeof(msg), "Missing rebalance-id");
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
+                    ret = 0;
+                }
+            }
+            ret = glusterd_defrag_start_validate(volinfo, msg, sizeof(msg),
+                                                 GD_OP_REBALANCE);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "defrag start validate "
+                             "failed for volume %s.",
+                             volinfo->volname);
                 goto out;
-        }
+            }
+            break;
+        case GF_DEFRAG_CMD_STATUS:
+        case GF_DEFRAG_CMD_STOP:
 
-        if ((*volinfo)->status != GLUSTERD_STATUS_STARTED) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_STOPPED, "Received rebalance on stopped"
-                        " volname %s", volname);
-                snprintf (op_errstr, len, "Volume %s needs to "
-                          "be started to perform rebalance", volname);
+            ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get "
+                       "command string");
+                ret = -1;
                 goto out;
-        }
-
-        ret = glusterd_disallow_op_for_tier (*volinfo, GD_OP_REBALANCE, cmd);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_REBALANCE_CMD_IN_TIER_VOL,
-                        "Received rebalance command "
-                        "on Tier volume %s", volname);
-                snprintf (op_errstr, len, "Rebalance operations are not "
-                          "supported on a tiered volume");
+            }
+            if ((strstr(cmd_str, "rebalance") != NULL) &&
+                (volinfo->rebal.op != GD_OP_REBALANCE)) {
+                snprintf(msg, sizeof(msg),
+                         "Rebalance not started "
+                         "for volume %s.",
+                         volinfo->volname);
+                ret = -1;
                 goto out;
-        }
+            }
 
-        ret = 0;
+            if (strstr(cmd_str, "remove-brick") != NULL) {
+                if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
+                    snprintf(msg, sizeof(msg),
+                             "remove-brick not "
+                             "started for volume %s.",
+                             volinfo->volname);
+                    ret = -1;
+                    goto out;
+                }
+
+                /* For remove-brick status/stop command check whether
+                 * given input brick is part of volume or not.*/
+
+                ret = dict_foreach_fnmatch(dict, "brick*",
+                                           glusterd_brick_validation, volinfo);
+                if (ret == -1) {
+                    snprintf(msg, sizeof(msg),
+                             "Incorrect brick"
+                             " for volume %s",
+                             volinfo->volname);
+                    goto out;
+                }
+            }
+            break;
+
+        default:
+            break;
+    }
 
+    ret = 0;
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
+
+    return ret;
 }
 
 int
-__glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+glusterd_mgmt_v3_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t                 ret       = -1;
-        gf_cli_req              cli_req   = {{0,}};
-        glusterd_conf_t        *priv      = NULL;
-        dict_t                 *dict      = NULL;
-        char                   *volname   = NULL;
-        gf_cli_defrag_type      cmd       = 0;
-        char                    msg[2048] = {0,};
-        xlator_t               *this      = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    char *volname = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *tmp = NULL;
+    gf_boolean_t volfile_update = _gf_false;
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    uint32_t commit_hash;
+    int32_t is_force = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not given");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "command not given");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd validate failed");
+        goto out;
+    }
+
+    switch (cmd) {
+        case GF_DEFRAG_CMD_START:
+        case GF_DEFRAG_CMD_START_LAYOUT_FIX:
+        case GF_DEFRAG_CMD_START_FORCE:
+
+            ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+            if (ret)
+                is_force = 0;
+            if (!is_force) {
+                /* Reset defrag status to 'NOT STARTED' whenever a
+                 * remove-brick/rebalance command is issued to remove
+                 * stale information from previous run.
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+                if (ret) {
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
+                } else {
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
                 }
-        }
+                if (!gd_should_i_start_rebalance(volinfo)) {
+                    /* Store the rebalance-id and rebalance command
+                     * even if the peer isn't starting a rebalance
+                     * process. On peers where a rebalance process
+                     * is started, glusterd_handle_defrag_start
+                     * performs the storing.
+                     * Storing this is needed for having
+                     * 'volume status' work correctly.
+                     */
+                    glusterd_store_perform_node_state_store(volinfo);
+                    break;
+                }
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
+                }
+                ret = glusterd_handle_defrag_start(volinfo, msg, sizeof(msg),
+                                                   cmd, NULL, GD_OP_REBALANCE);
+                break;
+            } else {
+                /* Reset defrag status to 'STARTED' so that the
+                 * pid is checked and restarted accordingly.
+                 * If the pid is not running it executes the
+                 * "NOT_STARTED" case and restarts the process
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+                volinfo->rebal.defrag_cmd = cmd;
+                volinfo->rebal.op = GD_OP_REBALANCE;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get volume name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", msg);
-                goto out;
-        }
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+                if (ret) {
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
+                } else {
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
+                }
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
+                }
+                ret = glusterd_restart_rebalance_for_volume(volinfo);
+                break;
+            }
+        case GF_DEFRAG_CMD_STOP:
+            /* Clear task-id only on explicitly stopping rebalance.
+             * Also clear the stored operation, so it doesn't cause trouble
+             * with future rebalance/remove-brick starts
+             */
+            gf_uuid_clear(volinfo->rebal.rebalance_id);
+            volinfo->rebal.op = GD_OP_NONE;
+
+            /* Fall back to the old volume file in case of decommission*/
+            cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks,
+                                         brick_list)
+            {
+                if (!brickinfo->decommissioned)
+                    continue;
+                brickinfo->decommissioned = 0;
+                volfile_update = _gf_true;
+            }
+
+            if (volfile_update == _gf_false) {
+                ret = 0;
+                break;
+            }
 
-        ret = dict_get_int32 (dict, "rebalance-command", (int32_t*)&cmd);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get command");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", msg);
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_VOLFILE_CREATE_FAIL, "failed to create volfiles");
                 goto out;
-        }
+            }
 
-        ret = dict_set_static_bin (dict, "node-uuid", MY_UUID, 16);
-        if (ret)
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "failed to store volinfo");
                 goto out;
+            }
 
-        if ((cmd == GF_DEFRAG_CMD_STATUS) ||
-            (cmd == GF_DEFRAG_CMD_STATUS_TIER) ||
-            (cmd == GF_DEFRAG_CMD_STOP_DETACH_TIER) ||
-            (cmd == GF_DEFRAG_CMD_STOP) ||
-            (cmd == GF_DEFRAG_CMD_DETACH_STATUS)) {
-                ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
-                                         dict, msg, sizeof (msg));
-        } else
-                ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict,
-                                         msg, sizeof (msg));
-
-out:
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0,
-                                                     req, dict, msg);
-
-        }
-
-        free (cli_req.dict.dict_val);//malloced by xdr
-
-        return 0;
-}
-
-int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __glusterd_handle_defrag_volume);
-}
+            ret = 0;
+            break;
 
-static int
-glusterd_brick_validation  (dict_t *dict, char *key, data_t *value,
-                            void *data)
-{
-        int32_t                   ret              = -1;
-        xlator_t                 *this             = NULL;
-        glusterd_volinfo_t       *volinfo          = data;
-        glusterd_brickinfo_t     *brickinfo        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_volume_brickinfo_get_by_brick (value->data, volinfo,
-                                                      &brickinfo,
-                                                      _gf_false);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_BRICK_NOT_FOUND,
-                        "Incorrect brick %s for "
-                        "volume %s", value->data, volinfo->volname);
-                return ret;
-        }
+        case GF_DEFRAG_CMD_STATUS:
+            break;
+        default:
+            break;
+    }
 
-        if (!brickinfo->decommissioned) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_BRICK_NOT_FOUND, "Incorrect brick %s for "
-                        "volume %s", value->data, volinfo->volname);
-                ret = -1;
-                return ret;
-        }
+out:
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
+glusterd_op_stage_rebalance(dict_t *dict, char **op_errstr)
 {
-        char                    *volname     = NULL;
-        char                    *cmd_str     = NULL;
-        int                     ret          = 0;
-        int32_t                 cmd          = 0;
-        char                    msg[2048]    = {0};
-        glusterd_volinfo_t      *volinfo     = NULL;
-        glusterd_brickinfo_t    *brickinfo   = NULL;
-        glusterd_peerinfo_t     *peerinfo    = NULL;
-        char                    *task_id_str = NULL;
-        dict_t                  *op_ctx      = NULL;
-        xlator_t                *this        = 0;
-        int32_t                 is_force     = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "volname not found");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "cmd not found");
-                goto out;
-        }
-
-        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
-                                               msg, sizeof (msg));
-        if (ret) {
-                gf_msg_debug (this->name, 0, "failed to validate");
-                goto out;
-        }
-        switch (cmd) {
-        case GF_DEFRAG_CMD_START_TIER:
-                ret = dict_get_int32 (dict, "force", &is_force);
-                if (ret)
-                        is_force = 0;
-
-                if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                        gf_asprintf (op_errstr, "volume %s is not a tier "
-                                     "volume.", volinfo->volname);
-                        ret = -1;
-                        goto out;
-                }
-                if ((!is_force) && glusterd_is_tier_daemon_running (volinfo)) {
-                        ret = gf_asprintf (op_errstr, "A Tier daemon is "
-                                        "already running on volume %s",
-                                        volname);
-                        ret = -1;
-                        goto out;
-                }
+    char *volname = NULL;
+    char *cmd_str = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    char *task_id_str = NULL;
+    dict_t *op_ctx = NULL;
+    xlator_t *this = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd not found");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to validate");
+        goto out;
+    }
+    switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
-                /* Check if the connected clients are all of version
-                 * glusterfs-3.6 and higher. This is needed to prevent some data
-                 * loss issues that could occur when older clients are connected
-                 * when rebalance is run. This check can be bypassed by using
-                 * 'force'
-                 */
-                ret = glusterd_check_client_op_version_support
-                        (volname, GD_OP_VERSION_3_6_0, NULL);
-                if (ret) {
-                        ret = gf_asprintf (op_errstr, "Volume %s has one or "
-                                           "more connected clients of a version"
-                                           " lower than GlusterFS-v3.6.0. "
-                                           "Starting rebalance in this state "
-                                           "could lead to data loss.\nPlease "
-                                           "disconnect those clients before "
-                                           "attempting this command again.",
-                                           volname);
-                        goto out;
-                }
-
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (glusterd_is_local_brick (THIS, volinfo, brickinfo)) {
-                                if (brickinfo->status != GF_BRICK_STARTED) {
-                                        gf_asprintf (op_errstr, "Received"
-                                                     " rebalance on volume with "
-                                                     " stopped brick %s",
-                                                     brickinfo->path);
-                                        ret = -1;
-                                        goto out;
-                                }
-                        } else {
-                                rcu_read_lock ();
-                                peerinfo = glusterd_peerinfo_find_by_uuid
-                                           (brickinfo->uuid);
-                                if (!peerinfo) {
-                                        gf_asprintf (op_errstr, "Host node %s "
-                                                     "of brick %s doesn't "
-                                                     "belong to cluster",
-                                                     brickinfo->hostname,
-                                                     brickinfo->path);
-                                        ret = -1;
-                                        rcu_read_unlock ();
-                                        goto out;
-                                } else if (!peerinfo->connected) {
-                                        gf_asprintf (op_errstr, "Host node %s "
-                                                     "of brick %s is down",
-                                                     brickinfo->hostname,
-                                                     brickinfo->path);
-                                        ret = -1;
-                                        rcu_read_unlock ();
-                                        goto out;
-                                }
-                                rcu_read_unlock ();
-                        }
+            /* Check if the connected clients are all of version
+             * glusterfs-3.6 and higher. This is needed to prevent some data
+             * loss issues that could occur when older clients are connected
+             * when rebalance is run. This check can be bypassed by using
+             * 'force'
+             */
+            ret = glusterd_check_client_op_version_support(
+                volname, GD_OP_VERSION_3_6_0, NULL);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume %s has one or "
+                                  "more connected clients of a version"
+                                  " lower than GlusterFS-v3.6.0. "
+                                  "Starting rebalance in this state "
+                                  "could lead to data loss.\nPlease "
+                                  "disconnect those clients before "
+                                  "attempting this command again.",
+                                  volname);
+                goto out;
+            }
+            /* Fall through */
+        case GF_DEFRAG_CMD_START_FORCE:
+            if (is_origin_glusterd(dict)) {
+                op_ctx = glusterd_op_get_ctx();
+                if (!op_ctx) {
+                    ret = -1;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+                           "Failed to get op_ctx");
+                    goto out;
                 }
 
-        case GF_DEFRAG_CMD_START_FORCE:
-                if (is_origin_glusterd (dict)) {
-                        op_ctx = glusterd_op_get_ctx ();
-                        if (!op_ctx) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_OPCTX_GET_FAIL,
-                                        "Failed to get op_ctx");
-                                goto out;
-                        }
-
-                        ret = glusterd_generate_and_set_task_id
-                                (op_ctx, GF_REBALANCE_TID_KEY);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_TASKID_GEN_FAIL,
-                                        "Failed to generate task-id");
-                                goto out;
-                        }
-                } else {
-                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                snprintf (msg, sizeof (msg),
-                                          "Missing rebalance-id");
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
-                                ret = 0;
-                        }
+                ret = glusterd_generate_and_set_task_id(
+                    op_ctx, GF_REBALANCE_TID_KEY, SLEN(GF_REBALANCE_TID_KEY));
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                           "Failed to generate task-id");
+                    goto out;
                 }
-                ret = glusterd_defrag_start_validate (volinfo, msg,
-                                                      sizeof (msg),
-                                                      GD_OP_REBALANCE);
+            } else {
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
                 if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                        "start validate failed");
-                        goto out;
+                    snprintf(msg, sizeof(msg), "Missing rebalance-id");
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
+                    ret = 0;
                 }
-                break;
-        case GF_DEFRAG_CMD_STATUS_TIER:
+            }
+            ret = glusterd_defrag_start_validate(volinfo, msg, sizeof(msg),
+                                                 GD_OP_REBALANCE);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "defrag start validate "
+                             "failed for volume %s.",
+                             volinfo->volname);
+                goto out;
+            }
+            break;
         case GF_DEFRAG_CMD_STATUS:
         case GF_DEFRAG_CMD_STOP:
 
-                ret = dict_get_str (dict, "cmd-str", &cmd_str);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to get "
-                                "command string");
-                        ret = -1;
-                        goto out;
-                }
-                if ((strstr(cmd_str, "rebalance") != NULL) &&
-                    (volinfo->rebal.op != GD_OP_REBALANCE)) {
-                        snprintf (msg, sizeof(msg), "Rebalance not started.");
-                        ret = -1;
-                        goto out;
-                }
+            ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get "
+                       "command string");
+                ret = -1;
+                goto out;
+            }
+            if ((strstr(cmd_str, "rebalance") != NULL) &&
+                (volinfo->rebal.op != GD_OP_REBALANCE)) {
+                snprintf(msg, sizeof(msg),
+                         "Rebalance not started "
+                         "for volume %s.",
+                         volinfo->volname);
+                ret = -1;
+                goto out;
+            }
 
-                if (strstr(cmd_str, "remove-brick") != NULL) {
-                        if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
-                                snprintf (msg, sizeof(msg), "remove-brick not "
-                                          "started.");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        /* For remove-brick status/stop command check whether
-                         * given input brick is part of volume or not.*/
-
-                        ret = dict_foreach_fnmatch (dict, "brick*",
-                                                    glusterd_brick_validation,
-                                                    volinfo);
-                        if (ret == -1) {
-                                snprintf (msg, sizeof (msg), "Incorrect brick"
-                                          " for volume %s", volinfo->volname);
-                                goto out;
-                        }
-                }
-                if (cmd == GF_DEFRAG_CMD_STATUS_TIER) {
-                        if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                                snprintf (msg, sizeof(msg), "volume %s is not "
-                                          "a tier volume.", volinfo->volname);
-                                ret = -1;
-                                goto out;
-                        }
+            if (strstr(cmd_str, "remove-brick") != NULL) {
+                if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
+                    snprintf(msg, sizeof(msg),
+                             "remove-brick not "
+                             "started for volume %s.",
+                             volinfo->volname);
+                    ret = -1;
+                    goto out;
                 }
 
-                break;
-
-        case GF_DEFRAG_CMD_STOP_DETACH_TIER:
-        case GF_DEFRAG_CMD_DETACH_STATUS:
-                if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                        snprintf (msg, sizeof(msg), "volume %s is not "
-                                  "a tier volume.", volinfo->volname);
-                        ret = -1;
-                        goto out;
+                /* For remove-brick status/stop command check whether
+                 * given input brick is part of volume or not.*/
+
+                ret = dict_foreach_fnmatch(dict, "brick*",
+                                           glusterd_brick_validation, volinfo);
+                if (ret == -1) {
+                    snprintf(msg, sizeof(msg),
+                             "Incorrect brick"
+                             " for volume %s",
+                             volinfo->volname);
+                    goto out;
                 }
+            }
+            break;
 
-                if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
-                        snprintf (msg, sizeof(msg), "Detach-tier "
-                                  "not started");
-                        ret = -1;
-                        goto out;
-                }
-                break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && op_errstr && msg[0])
-                *op_errstr = gf_strdup (msg);
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char                    *volname   = NULL;
-        int                     ret       = 0;
-        int32_t                 cmd       = 0;
-        char                    msg[2048] = {0};
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_conf_t         *priv      = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_brickinfo_t    *tmp      = NULL;
-        gf_boolean_t            volfile_update = _gf_false;
-        char                    *task_id_str = NULL;
-        dict_t                  *ctx = NULL;
-        xlator_t                *this = NULL;
-        uint32_t                commit_hash;
-        int32_t                 is_force    = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "volname not given");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "command not given");
+    char *volname = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *tmp = NULL;
+    gf_boolean_t volfile_update = _gf_false;
+    char *task_id_str = NULL;
+    dict_t *ctx = NULL;
+    xlator_t *this = NULL;
+    uint32_t commit_hash;
+    int32_t is_force = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not given");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "command not given");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd validate failed");
+        goto out;
+    }
+
+    /* Set task-id, if available, in op_ctx dict for operations other than
+     * start
+     */
+    if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+        if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+            ctx = glusterd_op_get_ctx();
+            if (!ctx) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+                       "Failed to get op_ctx");
+                ret = -1;
                 goto out;
-        }
-
-
-        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
-                                               msg, sizeof (msg));
-        if (ret) {
-                gf_msg_debug (this->name, 0, "cmd validate failed");
+            }
+
+            if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 ctx, GF_REMOVE_BRICK_TID_KEY,
+                                                 SLEN(GF_REMOVE_BRICK_TID_KEY));
+            else
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 ctx, GF_REBALANCE_TID_KEY,
+                                                 SLEN(GF_REBALANCE_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                       "Failed to set task-id");
                 goto out;
+            }
         }
+    }
 
-        /* Set task-id, if available, in op_ctx dict for operations other than
-         * start
-         */
-        if (cmd == GF_DEFRAG_CMD_STATUS ||
-            cmd == GF_DEFRAG_CMD_STOP ||
-            cmd == GF_DEFRAG_CMD_STATUS_TIER) {
-                if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
-                        ctx = glusterd_op_get_ctx ();
-                        if (!ctx) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_OPCTX_GET_FAIL,
-                                        "Failed to get op_ctx");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
-                                ret = glusterd_copy_uuid_to_dict
-                                        (volinfo->rebal.rebalance_id, ctx,
-                                         GF_REMOVE_BRICK_TID_KEY);
-                        else
-                                ret = glusterd_copy_uuid_to_dict
-                                        (volinfo->rebal.rebalance_id, ctx,
-                                         GF_REBALANCE_TID_KEY);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_TASKID_GEN_FAIL,
-                                        "Failed to set task-id");
-                                goto out;
-                        }
-                }
-        }
-
-        switch (cmd) {
+    switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_FORCE:
-        case GF_DEFRAG_CMD_START_TIER:
-
-
-                ret = dict_get_int32 (dict, "force", &is_force);
-                if (ret)
-                        is_force = 0;
-                if (!is_force) {
-                        /* Reset defrag status to 'NOT STARTED' whenever a
-                         * remove-brick/rebalance command is issued to remove
-                         * stale information from previous run.
-                         */
-                        volinfo->rebal.defrag_status =
-                               GF_DEFRAG_STATUS_NOT_STARTED;
-
-                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0, "Missing rebalance"
-                                              " id");
-                                ret = 0;
-                        } else {
-                                gf_uuid_parse (task_id_str,
-                                               volinfo->rebal.rebalance_id);
-                                volinfo->rebal.op = GD_OP_REBALANCE;
-                        }
-                        if (!gd_should_i_start_rebalance (volinfo)) {
-                                /* Store the rebalance-id and rebalance command
-                                 * even if the peer isn't starting a rebalance
-                                 * process. On peers where a rebalance process
-                                 * is started, glusterd_handle_defrag_start
-                                 * performs the storing.
-                                 * Storing this is needed for having
-                                 * 'volume status' work correctly.
-                                 */
-                                glusterd_store_perform_node_state_store
-                                        (volinfo);
-                                break;
-                        }
-                        if (dict_get_uint32 (dict, "commit-hash", &commit_hash)
-                                             == 0) {
-                                volinfo->rebal.commit_hash = commit_hash;
-                        }
-                        ret = glusterd_handle_defrag_start (volinfo, msg,
-                                        sizeof (msg),
-                                        cmd, NULL, GD_OP_REBALANCE);
-                        break;
-                } else {
-                        /* Reset defrag status to 'STARTED' so that the
-                         * pid is checked and restarted accordingly.
-                         * If the pid is not running it executes the
-                         * "NOT_STARTED" case and restarts the process
-                         */
-                        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
-                        volinfo->rebal.defrag_cmd = cmd;
-                        volinfo->rebal.op = GD_OP_REBALANCE;
-
-                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0, "Missing rebalance"
-                                              " id");
-                                ret = 0;
-                        } else {
-                                gf_uuid_parse (task_id_str,
-                                               volinfo->rebal.rebalance_id);
-                                volinfo->rebal.op = GD_OP_REBALANCE;
-                        }
-                        if (dict_get_uint32 (dict, "commit-hash", &commit_hash)
-                                             == 0) {
-                                volinfo->rebal.commit_hash = commit_hash;
-                        }
-                        ret = glusterd_restart_rebalance_for_volume (volinfo);
-                        break;
-                }
-        case GF_DEFRAG_CMD_STOP:
-        case GF_DEFRAG_CMD_STOP_DETACH_TIER:
-                /* Clear task-id only on explicitly stopping rebalance.
-                 * Also clear the stored operation, so it doesn't cause trouble
-                 * with future rebalance/remove-brick starts
-                 */
-                gf_uuid_clear (volinfo->rebal.rebalance_id);
-                volinfo->rebal.op = GD_OP_NONE;
-
-                /* Fall back to the old volume file in case of decommission*/
-                cds_list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
-                                              brick_list) {
-                        if (!brickinfo->decommissioned)
-                                continue;
-                        brickinfo->decommissioned = 0;
-                        volfile_update = _gf_true;
-                }
 
-                if (volfile_update == _gf_false) {
-                        ret = 0;
-                        break;
-                }
+            ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+            if (ret)
+                is_force = 0;
+            if (!is_force) {
+                /* Reset defrag status to 'NOT STARTED' whenever a
+                 * remove-brick/rebalance command is issued to remove
+                 * stale information from previous run.
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
 
-                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOLFILE_CREATE_FAIL,
-                                "failed to create volfiles");
-                        goto out;
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
+                } else {
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
+                }
+                if (!gd_should_i_start_rebalance(volinfo)) {
+                    /* Store the rebalance-id and rebalance command
+                     * even if the peer isn't starting a rebalance
+                     * process. On peers where a rebalance process
+                     * is started, glusterd_handle_defrag_start
+                     * performs the storing.
+                     * Storing this is needed for having
+                     * 'volume status' work correctly.
+                     */
+                    glusterd_store_perform_node_state_store(volinfo);
+                    break;
                 }
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
+                }
+                ret = glusterd_handle_defrag_start(volinfo, msg, sizeof(msg),
+                                                   cmd, NULL, GD_OP_REBALANCE);
+                break;
+            } else {
+                /* Reset defrag status to 'STARTED' so that the
+                 * pid is checked and restarted accordingly.
+                 * If the pid is not running it executes the
+                 * "NOT_STARTED" case and restarts the process
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+                volinfo->rebal.defrag_cmd = cmd;
+                volinfo->rebal.op = GD_OP_REBALANCE;
 
-                ret = glusterd_store_volinfo (volinfo,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOLINFO_SET_FAIL,
-                                "failed to store volinfo");
-                        goto out;
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
+                } else {
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
                 }
-
-                if (volinfo->type == GF_CLUSTER_TYPE_TIER &&
-                                cmd == GF_OP_CMD_STOP_DETACH_TIER) {
-                        glusterd_defrag_info_set (volinfo, dict,
-                                        GF_DEFRAG_CMD_START_TIER,
-                                        GF_DEFRAG_CMD_START,
-                                        GD_OP_REBALANCE);
-                        glusterd_restart_rebalance_for_volume (volinfo);
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
                 }
-
+                ret = glusterd_restart_rebalance_for_volume(volinfo);
+                break;
+            }
+        case GF_DEFRAG_CMD_STOP:
+            /* Clear task-id only on explicitly stopping rebalance.
+             * Also clear the stored operation, so it doesn't cause trouble
+             * with future rebalance/remove-brick starts
+             */
+            gf_uuid_clear(volinfo->rebal.rebalance_id);
+            volinfo->rebal.op = GD_OP_NONE;
+
+            /* Fall back to the old volume file in case of decommission*/
+            cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks,
+                                         brick_list)
+            {
+                if (!brickinfo->decommissioned)
+                    continue;
+                brickinfo->decommissioned = 0;
+                volfile_update = _gf_true;
+            }
+
+            if (volfile_update == _gf_false) {
                 ret = 0;
                 break;
+            }
+
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_VOLFILE_CREATE_FAIL, "failed to create volfiles");
+                goto out;
+            }
+
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "failed to store volinfo");
+                goto out;
+            }
+
+            ret = 0;
+            break;
 
-        case GF_DEFRAG_CMD_START_DETACH_TIER:
         case GF_DEFRAG_CMD_STATUS:
-        case GF_DEFRAG_CMD_STATUS_TIER:
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        if (ret && op_errstr && msg[0])
-                *op_errstr = gf_strdup (msg);
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_defrag_event_notify_handle (dict_t *dict)
+glusterd_defrag_event_notify_handle(dict_t *dict)
 {
-        glusterd_volinfo_t      *volinfo     = NULL;
-        char                    *volname     = NULL;
-        char                    *volname_ptr = NULL;
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get volname");
-                return ret;
-        }
-
-        volname_ptr = strstr (volname, "rebalance/");
-        if (volname_ptr) {
-                volname_ptr = strchr (volname_ptr, '/');
-                if (!volname_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volname = volname_ptr + 1;
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME,
-                        "volname received (%s) is not prefixed with rebalance.",
-                        volname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_GET_FAIL,
-                        "Failed to get volinfo for %s"
-                        , volname);
-                return ret;
-        }
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    char *volname_ptr = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volname");
+        return ret;
+    }
+
+    volname_ptr = strstr(volname, "rebalance/");
+    if (volname_ptr) {
+        volname_ptr = strchr(volname_ptr, '/');
+        volname = volname_ptr + 1;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME,
+               "volname received (%s) is not prefixed with rebalance.",
+               volname);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo for %s", volname);
+        return ret;
+    }
 
-        ret = glusterd_defrag_volume_status_update (volinfo, dict);
+    ret = glusterd_defrag_volume_status_update(volinfo, dict, 0);
 
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DEFRAG_STATUS_UPDATE_FAIL,
-                        "Failed to update status");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DEFRAG_STATUS_UPDATE_FAIL,
+               "Failed to update status");
+        gf_event(EVENT_REBALANCE_STATUS_UPDATE_FAILED, "volume=%s",
+                 volinfo->volname);
+    }
 
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 2b2120738ca..43c2f4373e0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -7,10 +7,10 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-geo-rep.h"
@@ -18,888 +18,699 @@
 #include "glusterd-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-svc-helper.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
+#include "glusterd-server-quorum.h"
 #include "glusterd-mgmt.h"
-#include "run.h"
-#include "syscall.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 
 #include <signal.h>
 
-#define GLUSTERD_GET_RB_MNTPT(path, len, volinfo)                           \
-        snprintf (path, len,                                                \
-                  DEFAULT_VAR_RUN_DIRECTORY"/%s-"RB_CLIENT_MOUNTPOINT,      \
-                  volinfo->volname);
-
-extern uuid_t global_txn_id;
-
 int
-glusterd_mgmt_v3_initiate_replace_brick_cmd_phases (rpcsvc_request_t *req,
-                                                    glusterd_op_t op,
-                                                    dict_t *dict);
+glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(rpcsvc_request_t *req,
+                                                   glusterd_op_t op,
+                                                   dict_t *dict);
 int
-__glusterd_handle_replace_brick (rpcsvc_request_t *req)
+__glusterd_handle_replace_brick(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        char                            *src_brick = NULL;
-        char                            *dst_brick = NULL;
-        int32_t                         op = 0;
-        glusterd_op_t                   cli_op = GD_OP_REPLACE_BRICK;
-        char                            *volname = NULL;
-        char                            msg[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    char *cli_op = NULL;
+    glusterd_op_t op = -1;
+    char *volname = NULL;
+    char msg[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REPLACE_BRK_REQ_RCVD,
+           "Received replace brick req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REQ_DECODE_FAIL, "Failed to decode "
-                        "request received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_REPLACE_BRK_REQ_RCVD,
-                "Received replace brick req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Could not get volume name");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", msg);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "operation", &op);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "dict_get on operation failed");
-                snprintf (msg, sizeof (msg), "Could not get operation");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Could not get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "operation", SLEN("operation"), &cli_op);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "dict_get on operation failed");
+        snprintf(msg, sizeof(msg), "Could not get operation");
+        goto out;
+    }
+
+    op = gd_cli_to_gd_op(cli_op);
+
+    if (conf->op_version < GD_OP_VERSION_3_9_0 &&
+        strcmp(cli_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        snprintf(msg, sizeof(msg),
+                 "Cannot execute command. The "
+                 "cluster is operating at version %d. reset-brick "
+                 "command %s is unavailable in this version.",
+                 conf->op_version, gd_rb_op_to_str(cli_op));
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "src-brick", SLEN("src-brick"), &src_brick);
+
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get src brick");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
+
+    if (!strcmp(cli_op, "GF_RESET_OP_COMMIT") ||
+        !strcmp(cli_op, "GF_RESET_OP_COMMIT_FORCE") ||
+        !strcmp(cli_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        ret = dict_get_strn(dict, "dst-brick", SLEN("dst-brick"), &dst_brick);
 
         if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get src brick");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", msg);
-                goto out;
+            snprintf(msg, sizeof(msg),
+                     "Failed to get"
+                     "dest brick");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   msg);
+            goto out;
         }
-        gf_msg_debug (this->name, 0,
-                "src brick=%s", src_brick);
 
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+        gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
+    }
 
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get dest brick");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s", msg);
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "dst brick=%s", dst_brick);
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_REPLACE_BRK_COMMIT_FORCE_REQ_RCVD,
-                "Received replace brick commit-force "
-                "request operation");
+    gf_msg(this->name, GF_LOG_INFO, 0,
+           (op == GD_OP_REPLACE_BRICK)
+               ? GD_MSG_REPLACE_BRK_COMMIT_FORCE_REQ_RCVD
+               : GD_MSG_RESET_BRICK_COMMIT_FORCE_REQ_RCVD,
+           "Received %s request.", gd_rb_op_to_str(cli_op));
 
-        ret = glusterd_mgmt_v3_initiate_replace_brick_cmd_phases (req,
-                                            GD_OP_REPLACE_BRICK, dict);
+    ret = glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(req, op, dict);
 
 out:
-        free (cli_req.dict.dict_val);//malloced by xdr
+    if (ret) {
+        glusterd_op_send_cli_response(op, ret, 0, req, dict, msg);
+    }
+    ret = 0;
+    free(cli_req.dict.dict_val);  // malloced by xdr
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_replace_brick (rpcsvc_request_t *req)
+glusterd_handle_reset_brick(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_replace_brick);
-}
-
-static int
-glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo)
-{
-        int32_t                 ret = -1;
-
-        if (!volinfo || !brickinfo)
-                goto out;
-
-        *brickinfo = volinfo->rep_brick.dst_brick;
-
-        ret = 0;
-
-out:
-        return ret;
+    return glusterd_big_locked_handler(req, __glusterd_handle_replace_brick);
 }
 
 int
-glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
-{
-        int                                      ret                = 0;
-        int32_t                                  port               = 0;
-        char                                    *src_brick          = NULL;
-        char                                    *dst_brick          = NULL;
-        char                                    *volname            = NULL;
-        char                                    *replace_op         = NULL;
-        glusterd_volinfo_t                      *volinfo            = NULL;
-        glusterd_brickinfo_t                    *src_brickinfo      = NULL;
-        char                                    *host               = NULL;
-        char                                    *path               = NULL;
-        char                                     msg[2048]          = {0};
-        char                                    *dup_dstbrick       = NULL;
-        glusterd_peerinfo_t                     *peerinfo           = NULL;
-        glusterd_brickinfo_t                    *dst_brickinfo      = NULL;
-        gf_boolean_t                             enabled            = _gf_false;
-        glusterd_conf_t                         *priv               = NULL;
-        char                                    *savetok            = NULL;
-        char                                     pidfile[PATH_MAX]  = {0};
-        char                                    *task_id_str        = NULL;
-        xlator_t                                *this               = NULL;
-        gf_boolean_t                             is_force           = _gf_false;
-        gsync_status_param_t                     param              = {0,};
-        char                                    *c                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get src brick");
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "src brick=%s", src_brick);
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get dest brick");
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "dst brick=%s", dst_brick);
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "operation", &replace_op);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "dict get on replace-brick operation failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "volume: %s does not exist",
-                          volname);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "volume: %s is not started",
-                          volname);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = glusterd_disallow_op_for_tier (volinfo, GD_OP_REPLACE_BRICK, -1);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Replace brick commands are not "
-                          "supported on tiered volume %s", volname);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (!glusterd_store_is_valid_brickpath (volname, dst_brick) ||
-                !glusterd_is_valid_volfpath (volname, dst_brick)) {
-                snprintf (msg, sizeof (msg), "brick path %s is too "
-                          "long.", dst_brick);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRKPATH_TOO_LONG, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-
-                ret = -1;
-                goto out;
-        }
-
-        /* If geo-rep is configured, for this volume, it should be stopped. */
-        param.volinfo = volinfo;
-        ret = glusterd_check_geo_rep_running (&param, op_errstr);
-        if (ret || param.is_active) {
-                ret = -1;
-                goto out;
-        }
-
-        if (glusterd_is_defrag_on(volinfo)) {
-                snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
-                          "progress. Please retry after completion", volname);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OIP_RETRY_LATER, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
-        if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
-                is_force = _gf_true;
-        } else {
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      _gf_false);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "brick: %s does not exist in "
-                          "volume: %s", src_brick, volname);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (dict) {
-                if (!glusterd_is_fuse_available ()) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_RB_CMD_FAIL, "Unable to open /dev/"
-                                "fuse (%s), replace-brick command failed",
-                                strerror (errno));
-                        snprintf (msg, sizeof(msg), "Fuse unavailable\n "
-                                "Replace-brick failed");
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        if (gf_is_local_addr (src_brickinfo->hostname)) {
-                gf_msg_debug (this->name, 0,
-                        "I AM THE SOURCE HOST");
-                if (src_brickinfo->port && rsp_dict) {
-                        ret = dict_set_int32 (rsp_dict, "src-brick-port",
-                                              src_brickinfo->port);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0,
-                                        "Could not set src-brick-port=%d",
-                                        src_brickinfo->port);
-                        }
-                }
-
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, src_brickinfo,
-                                            priv);
-
-        }
-
-        dup_dstbrick = gf_strdup (dst_brick);
-        if (!dup_dstbrick) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Memory allocation failed");
-                goto out;
-        }
-
-        /*
-         * IPv4 address contains '.' and ipv6 addresses contains ':'
-         * So finding the last occurance of ':' to
-         * mark the start of brick path
-         */
-        c = strrchr(dup_dstbrick, ':');
-        if (c != NULL) {
-                c[0] = '\0';
-                host = dup_dstbrick;
-                path = c++;
-        }
-
-        if (!host || !path) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BAD_FORMAT,
-                        "dst brick %s is not of form <HOSTNAME>:<export-dir>",
-                        dst_brick);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_brickinfo_new_from_brick (dst_brick, &dst_brickinfo,
-                                                 _gf_true, NULL);
-        if (ret)
-                goto out;
-
-        ret = glusterd_new_brick_validate (dst_brick, dst_brickinfo,
-                                           msg, sizeof (msg));
-        if (ret) {
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_VALIDATE_FAIL, "%s", *op_errstr);
-                goto out;
-        }
-
-        if (!strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
-
-                volinfo->rep_brick.src_brick = src_brickinfo;
-                volinfo->rep_brick.dst_brick = dst_brickinfo;
-        }
-
-        if (glusterd_rb_check_bricks (volinfo, src_brickinfo, dst_brickinfo)) {
-
-                ret = -1;
-                *op_errstr = gf_strdup ("Incorrect source or "
-                                        "destination brick");
-                if (*op_errstr)
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_BRICK_NOT_FOUND, "%s", *op_errstr);
-                goto out;
-       }
-
-        if (gf_is_local_addr (host)) {
-                ret = glusterd_validate_and_create_brickpath (dst_brickinfo,
-                                                  volinfo->volume_id,
-                                                  op_errstr, is_force);
-                if (ret)
-                        goto out;
-        }
-
-        if (!gf_is_local_addr (host)) {
-                rcu_read_lock ();
-
-                peerinfo = glusterd_peerinfo_find (NULL, host);
-                if (peerinfo == NULL) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "%s, is not a friend",
-                                  host);
-                        *op_errstr = gf_strdup (msg);
-
-                } else if (!peerinfo->connected) {
-                        snprintf (msg, sizeof (msg), "%s, is not connected at "
-                                  "the moment", host);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-
-                } else if (GD_FRIEND_STATE_BEFRIENDED !=
-                                peerinfo->state.state) {
-                        snprintf (msg, sizeof (msg), "%s, is not befriended "
-                                  "at the moment", host);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                }
-                rcu_read_unlock ();
-
-                if (ret)
-                        goto out;
-
-        } else if (priv->op_version >= GD_OP_VERSION_3_6_0) {
-                /* A bricks mount dir is required only by snapshots which were
-                 * introduced in gluster-3.6.0
-                 */
-                ret = glusterd_get_brick_mount_dir (dst_brickinfo->path,
-                                                    dst_brickinfo->hostname,
-                                                    dst_brickinfo->mount_dir);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
-                                "Failed to get brick mount_dir");
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "brick1.mount_dir",
-                                                  dst_brickinfo->mount_dir);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set brick1.mount_dir");
-                        goto out;
-                }
-
-                ret = dict_set_int32 (rsp_dict, "brick_count", 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set local_brick_count");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-
-out:
-        GF_FREE (dup_dstbrick);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-
-        return ret;
-}
-
-static int
-rb_kill_destination_brick (glusterd_volinfo_t *volinfo,
-                           glusterd_brickinfo_t *dst_brickinfo)
-{
-        glusterd_conf_t  *priv               = NULL;
-        char              pidfile[PATH_MAX]  = {0,};
-
-        priv = THIS->private;
-
-        snprintf (pidfile, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_DSTBRICK_PIDFILE);
-
-        return glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_true);
-}
-
-static int
-rb_update_dstbrick_port (glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
-                         dict_t *req_dict, char *replace_op)
+glusterd_handle_replace_brick(rpcsvc_request_t *req)
 {
-        int     ret           = 0;
-        int     dict_ret      = 0;
-        int     dst_port      = 0;
-
-        dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
-        if (!dict_ret)
-                dst_brickinfo->port = dst_port;
-
-        if (gf_is_local_addr (dst_brickinfo->hostname)) {
-                gf_msg ("glusterd", GF_LOG_INFO, 0,
-                        GD_MSG_BRK_PORT_NO_ADD_INDO,
-                        "adding dst-brick port no");
-
-                if (rsp_dict) {
-                        ret = dict_set_int32 (rsp_dict, "dst-brick-port",
-                                              dst_brickinfo->port);
-                        if (ret) {
-                                gf_msg_debug ("glusterd", 0,
-                                        "Could not set dst-brick port no in rsp dict");
-                                goto out;
-                        }
-                }
-
-                if (req_dict) {
-                        ret = dict_set_int32 (req_dict, "dst-brick-port",
-                                              dst_brickinfo->port);
-                        if (ret) {
-                                gf_msg_debug ("glusterd", 0,
-                                        "Could not set dst-brick port no");
-                                goto out;
-                        }
-                }
-        }
-out:
-        return ret;
+    return glusterd_big_locked_handler(req, __glusterd_handle_replace_brick);
 }
 
-static int
-glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
-                                   char *old_brick, char *new_brick,
-                                   dict_t *dict)
+int
+glusterd_op_stage_replace_brick(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
 {
-        char                                    *brick_mount_dir = NULL;
-        glusterd_brickinfo_t                    *old_brickinfo = NULL;
-        glusterd_brickinfo_t                    *new_brickinfo = NULL;
-        int32_t                                 ret = -1;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *conf = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = glusterd_brickinfo_new_from_brick (new_brick, &new_brickinfo,
-                                                 _gf_true, NULL);
-        if (ret)
-                goto out;
-
-        ret = glusterd_resolve_brick (new_brickinfo);
-
+    int ret = 0;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    char *volname = NULL;
+    char *op = NULL;
+    glusterd_op_t gd_op = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    char *host = NULL;
+    char msg[2048] = {0};
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    char pidfile[PATH_MAX] = {0};
+    xlator_t *this = NULL;
+    gf_boolean_t is_force = _gf_false;
+    char *dup_dstbrick = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_brick_op_prerequisites(dict, &op, &gd_op, &volname, &volinfo,
+                                          &src_brick, &src_brickinfo, pidfile,
+                                          op_errstr, rsp_dict);
+    if (ret)
+        goto out;
+
+    if (volinfo->type == GF_CLUSTER_TYPE_NONE) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_NOT_PERMITTED,
+               "replace-brick is not permitted on distribute only "
+               "volumes");
+        gf_asprintf(op_errstr,
+                    "replace-brick is not permitted on "
+                    "distribute only volumes. Please use add-brick "
+                    "and remove-brick operations instead.");
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_validate_quorum(this, gd_op, dict, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        goto out;
+    }
+
+    if (strcmp(op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    } else {
+        is_force = _gf_true;
+    }
+
+    if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s  has %" PRIu64
+                 " snapshots. "
+                 "Changing the volume configuration will not effect snapshots."
+                 "But the snapshot brick mount should be intact to "
+                 "make them function.",
+                 volname, volinfo->snap_count);
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", msg);
+        msg[0] = '\0';
+    }
+
+    glusterd_add_peers_to_auth_list(volname);
+
+    ret = glusterd_get_dst_brick_info(&dst_brick, volname, op_errstr,
+                                      &dst_brickinfo, &host, dict,
+                                      &dup_dstbrick);
+    if (ret)
+        goto out;
+
+    ret = glusterd_new_brick_validate(dst_brick, dst_brickinfo, msg,
+                                      sizeof(msg), op);
+    /* fail if brick being replaced with itself */
+    if (ret) {
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_VALIDATE_FAIL, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    volinfo->rep_brick.src_brick = src_brickinfo;
+    volinfo->rep_brick.dst_brick = dst_brickinfo;
+
+    if (glusterd_rb_check_bricks(volinfo, src_brickinfo, dst_brickinfo)) {
+        ret = -1;
+        *op_errstr = gf_strdup(
+            "Incorrect source or "
+            "destination brick");
+        if (*op_errstr)
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+                   "%s", *op_errstr);
+        goto out;
+    }
+
+    if (gf_is_local_addr(host)) {
+        ret = glusterd_validate_and_create_brickpath(
+            dst_brickinfo, volinfo->volume_id, volinfo->volname, op_errstr,
+            is_force, _gf_false);
         if (ret)
-                goto out;
-
-        ret = glusterd_volume_brickinfo_get_by_brick (old_brick,
-                                                      volinfo, &old_brickinfo,
-                                                      _gf_false);
-        if (ret)
-                goto out;
-
-        strncpy (new_brickinfo->brick_id, old_brickinfo->brick_id,
-                 sizeof (new_brickinfo->brick_id));
-
+            goto out;
+    }
+
+    if (!gf_is_local_addr(host)) {
+        RCU_READ_LOCK;
+
+        peerinfo = glusterd_peerinfo_find(NULL, host);
+        if (peerinfo == NULL) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s, is not a friend", host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not connected at "
+                     "the moment",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not befriended "
+                     "at the moment",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+        RCU_READ_UNLOCK;
+
+    } else if (priv->op_version >= GD_OP_VERSION_3_6_0) {
         /* A bricks mount dir is required only by snapshots which were
          * introduced in gluster-3.6.0
          */
-        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                ret = dict_get_str (dict, "brick1.mount_dir", &brick_mount_dir);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
-                                "brick1.mount_dir not present");
-                        goto out;
-                }
-                strncpy (new_brickinfo->mount_dir, brick_mount_dir,
-                         sizeof(new_brickinfo->mount_dir));
-        }
-
-        cds_list_add_tail (&new_brickinfo->brick_list,
-                           &old_brickinfo->brick_list);
 
-        volinfo->brick_count++;
-
-        ret = glusterd_op_perform_remove_brick (volinfo, old_brick, 1, NULL);
-        if (ret)
+        if (!(gf_uuid_compare(dst_brickinfo->uuid, MY_UUID))) {
+            ret = glusterd_get_brick_mount_dir(dst_brickinfo->path,
+                                               dst_brickinfo->hostname,
+                                               dst_brickinfo->mount_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                       "Failed to get brick mount_dir");
                 goto out;
-
-        /* if the volume is a replicate volume, do: */
-        if (glusterd_is_volume_replicate (volinfo)) {
-                if (!gf_uuid_compare (new_brickinfo->uuid, MY_UUID)) {
-                        ret = glusterd_handle_replicate_brick_ops (volinfo,
-                                        new_brickinfo, GD_OP_REPLACE_BRICK);
-                        if (ret < 0)
-                                goto out;
-                }
-        }
-
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret)
+            }
+            ret = dict_set_dynstr_with_alloc(rsp_dict, "brick1.mount_dir",
+                                             dst_brickinfo->mount_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set brick.mount_dir");
                 goto out;
+            }
+        }
 
-        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                ret = glusterd_brick_start (volinfo, new_brickinfo, _gf_false);
-                if (ret)
-                        goto out;
+        ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"), 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set local_brick_count");
+            goto out;
         }
+    }
+
+    ret = 0;
 
 out:
+    GF_FREE(dup_dstbrick);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
+glusterd_op_perform_replace_brick(glusterd_volinfo_t *volinfo, char *old_brick,
+                                  char *new_brick, dict_t *dict)
 {
-        int                                      ret           = 0;
-        dict_t                                  *ctx           = NULL;
-        char                                    *replace_op    = NULL;
-        glusterd_volinfo_t                      *volinfo       = NULL;
-        char                                    *volname       = NULL;
-        xlator_t                                *this          = NULL;
-        glusterd_conf_t                         *priv          = NULL;
-        char                                    *src_brick     = NULL;
-        char                                    *dst_brick     = NULL;
-        glusterd_brickinfo_t                    *src_brickinfo = NULL;
-        glusterd_brickinfo_t                    *dst_brickinfo = NULL;
-        char                                    *task_id_str   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
+    char *brick_mount_dir = NULL;
+    glusterd_brickinfo_t *old_brickinfo = NULL;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    struct statvfs brickstat = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = glusterd_brickinfo_new_from_brick(new_brick, &new_brickinfo, _gf_true,
+                                            NULL);
+    if (ret)
+        goto out;
+
+    ret = glusterd_resolve_brick(new_brickinfo);
+    if (ret)
+        goto out;
+
+    if (!gf_uuid_compare(new_brickinfo->uuid, MY_UUID)) {
+        ret = sys_statvfs(new_brickinfo->path, &brickstat);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get src brick");
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "src brick=%s", src_brick);
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_STATVFS_FAILED,
+                   "Failed to fetch disk utilization "
+                   "from the brick (%s:%s). Please check the health of "
+                   "the brick. Error code was %s",
+                   new_brickinfo->hostname, new_brickinfo->path,
+                   strerror(errno));
+
+            goto out;
+        }
+        new_brickinfo->statfs_fsid = brickstat.f_fsid;
+    }
+
+    ret = glusterd_volume_brickinfo_get_by_brick(old_brick, volinfo,
+                                                 &old_brickinfo, _gf_false);
+    if (ret)
+        goto out;
+
+    (void)snprintf(new_brickinfo->brick_id, sizeof(new_brickinfo->brick_id),
+                   "%s", old_brickinfo->brick_id);
+    new_brickinfo->port = old_brickinfo->port;
+
+    /* A bricks mount dir is required only by snapshots which were
+     * introduced in gluster-3.6.0
+     */
+    if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+        ret = dict_get_strn(dict, "brick1.mount_dir", SLEN("brick1.mount_dir"),
+                            &brick_mount_dir);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get dst brick");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                   "brick1.mount_dir not present");
+            goto out;
         }
+        (void)snprintf(new_brickinfo->mount_dir,
+                       sizeof(new_brickinfo->mount_dir), "%s", brick_mount_dir);
+    }
 
-        gf_msg_debug (this->name, 0, "dst brick=%s", dst_brick);
+    cds_list_add(&new_brickinfo->brick_list, &old_brickinfo->brick_list);
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
-                goto out;
-        }
+    volinfo->brick_count++;
 
-        ret = dict_get_str (dict, "operation", &replace_op);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Unable to allocate memory");
-                goto out;
-        }
+    ret = glusterd_op_perform_remove_brick(volinfo, old_brick, 1, NULL);
+    if (ret)
+        goto out;
 
-        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo,
-                                                      _gf_false);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "Unable to get src-brickinfo");
-                goto out;
-        }
-
-
-        ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RB_BRICKINFO_GET_FAIL, "Unable to get "
-                         "replace brick destination brickinfo");
+    /* if the volume is a replicate volume, do: */
+    if (glusterd_is_volume_replicate(volinfo)) {
+        if (!gf_uuid_compare(new_brickinfo->uuid, MY_UUID)) {
+            ret = glusterd_handle_replicate_brick_ops(volinfo, new_brickinfo,
+                                                      GD_OP_REPLACE_BRICK);
+            if (ret < 0)
                 goto out;
         }
+    }
 
-        ret = glusterd_resolve_brick (dst_brickinfo);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "Unable to resolve dst-brickinfo");
-                goto out;
-        }
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret)
+        goto out;
 
-        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
-                                       dict, replace_op);
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_brick_start(volinfo, new_brickinfo, _gf_false,
+                                   _gf_false);
         if (ret)
-                goto out;
-
-        if (strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
-                ret = -1;
-                goto out;
-        }
-
-        if (gf_is_local_addr (dst_brickinfo->hostname)) {
-                gf_msg_debug (this->name, 0, "I AM THE DESTINATION HOST");
-                ret = rb_kill_destination_brick (volinfo, dst_brickinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                GD_MSG_BRK_CLEANUP_FAIL,
-                                "Unable to cleanup dst brick");
-                        goto out;
-                }
-        }
-
-        ret = glusterd_svcs_stop (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NFS_SERVER_STOP_FAIL,
-                        "Unable to stop nfs server, ret: %d", ret);
-        }
-
-        ret = glusterd_op_perform_replace_brick (volinfo, src_brick,
-                                                 dst_brick, dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_BRICK_ADD_FAIL, "Unable to add dst-brick: "
-                        "%s to volume: %s", dst_brick, volinfo->volname);
-		(void) glusterd_svcs_manager (volinfo);
-                goto out;
-        }
-
-        volinfo->rebal.defrag_status = 0;
-
-        ret = glusterd_svcs_manager (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_NFS_VOL_FILE_GEN_FAIL,
-                        "Failed to generate nfs volume file");
-        }
+            goto out;
+    }
 
+out:
 
-        ret = glusterd_fetchspec_notify (THIS);
-        glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
-        volinfo->rep_brick.src_brick = NULL;
-        volinfo->rep_brick.dst_brick = NULL;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        if (!ret)
-		ret = glusterd_store_volinfo (volinfo,
-                                              GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RBOP_STATE_STORE_FAIL, "Couldn't store"
-                        " replace brick operation's state");
+int
+glusterd_op_replace_brick(dict_t *dict, dict_t *rsp_dict)
+{
+    int ret = 0;
+    char *replace_op = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "src-brick", SLEN("src-brick"), &src_brick);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get src brick");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
+
+    ret = dict_get_strn(dict, "dst-brick", SLEN("dst-brick"), &dst_brick);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get dst brick");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "operation", SLEN("operation"), &replace_op);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "dict_get on operation failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = glusterd_volume_brickinfo_get_by_brick(src_brick, volinfo,
+                                                 &src_brickinfo, _gf_false);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get src-brickinfo");
+        goto out;
+    }
+
+    ret = glusterd_get_rb_dst_brickinfo(volinfo, &dst_brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RB_BRICKINFO_GET_FAIL,
+               "Unable to get "
+               "replace brick destination brickinfo");
+        goto out;
+    }
+
+    ret = glusterd_resolve_brick(dst_brickinfo);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to resolve dst-brickinfo");
+        goto out;
+    }
+
+    ret = rb_update_dstbrick_port(dst_brickinfo, rsp_dict, dict);
+    if (ret)
+        goto out;
+
+    if (strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_svcs_stop(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTER_SERVICES_STOP_FAIL,
+               "Unable to stop gluster services, ret: %d", ret);
+    }
+
+    ret = glusterd_op_perform_replace_brick(volinfo, src_brick, dst_brick,
+                                            dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_ADD_FAIL,
+               "Unable to add dst-brick: "
+               "%s to volume: %s",
+               dst_brick, volinfo->volname);
+        (void)glusterd_svcs_manager(volinfo);
+        goto out;
+    }
+
+    volinfo->rebal.defrag_status = 0;
+
+    ret = glusterd_svcs_manager(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0,
+               GD_MSG_GLUSTER_SERVICE_START_FAIL,
+               "Failed to start one or more gluster services.");
+    }
+
+    ret = glusterd_fetchspec_notify(THIS);
+    glusterd_brickinfo_delete(volinfo->rep_brick.dst_brick);
+    volinfo->rep_brick.src_brick = NULL;
+    volinfo->rep_brick.dst_brick = NULL;
+
+    if (!ret)
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RBOP_STATE_STORE_FAIL,
+               "Couldn't store"
+               " replace brick operation's state");
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_initiate_replace_brick_cmd_phases (rpcsvc_request_t *req,
-                                                    glusterd_op_t op,
-                                                    dict_t *dict)
+glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(rpcsvc_request_t *req,
+                                                   glusterd_op_t op,
+                                                   dict_t *dict)
 {
-        int32_t                     ret              = -1;
-        int32_t                     op_ret           = -1;
-        uint32_t                    txn_generation   = 0;
-        uint32_t                    op_errno         = 0;
-        char                        *cli_errstr      = NULL;
-        char                        *op_errstr       = NULL;
-        dict_t                      *req_dict        = NULL;
-        dict_t                      *tmp_dict        = NULL;
-        uuid_t                      *originator_uuid = NULL;
-        xlator_t                    *this            = NULL;
-        glusterd_conf_t             *conf            = NULL;
-        gf_boolean_t                success          = _gf_false;
-        gf_boolean_t                is_acquired      = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        txn_generation = conf->generation;
-        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
-                                     gf_common_mt_uuid_t);
-        if (!originator_uuid) {
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+    char *op_errstr = NULL;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    uuid_t *originator_uuid = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    txn_generation = conf->generation;
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    ret = dict_set_int32n(dict, "is_synctasked", SLEN("is_synctasked"),
+                          _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag to true.");
+        goto out;
+    }
+
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        goto out;
+    }
+
+    ret = 0;
 
-        gf_uuid_copy (*originator_uuid, MY_UUID);
-        ret = dict_set_bin (dict, "originator_uuid",
-                            originator_uuid, sizeof (uuid_t));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set originator_uuid.");
-                GF_FREE (originator_uuid);
-                goto out;
-        }
-
-        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set synctasked flag to true.");
-                goto out;
-        }
-
-        tmp_dict = dict_new();
-        if (!tmp_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL, "Unable to create dict");
-                goto out;
-        }
-        dict_copy (dict, tmp_dict);
-
-        ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr,
-                                                  &op_errno, &is_acquired,
-                                                  txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_LOCKDOWN_FAIL,
-                        "mgmt_v3 lockdown failed.");
-                goto out;
-        }
-
-        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                goto out;
-        }
+out:
+    op_ret = ret;
 
-        ret = glusterd_mgmt_v3_pre_validate (op, req_dict, &op_errstr,
-                                             &op_errno, txn_generation);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL, "Pre Validation Failed");
-                goto out;
-        }
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
 
-        ret = glusterd_mgmt_v3_commit (op, dict, req_dict, &op_errstr,
-                                       &op_errno, txn_generation);
+    if (is_acquired) {
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_OP_FAIL, "Commit Op Failed");
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        op_ret = ret;
-
-        (void) glusterd_mgmt_v3_release_peer_locks (op, dict, op_ret,
-                                                    &op_errstr, is_acquired,
-                                                    txn_generation);
-
-        if (is_acquired) {
-                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MGMTV3_UNLOCK_FAIL,
-                                "Failed to release mgmt_v3 locks on "
-                                "localhost.");
-                        op_ret = ret;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on "
+                   "localhost.");
+            op_ret = ret;
         }
-        /* SEND CLI RESPONSE */
-        glusterd_op_send_cli_response (op, op_ret, op_errno, req,
-                                       dict, op_errstr);
+    }
+    /* SEND CLI RESPONSE */
+    glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict, op_errstr);
 
-        if (req_dict)
-                dict_unref (req_dict);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        if (tmp_dict)
-                dict_unref (tmp_dict);
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        if (op_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-        }
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-reset-brick.c b/xlators/mgmt/glusterd/src/glusterd-reset-brick.c
new file mode 100644
index 00000000000..e4d247a1d6c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-reset-brick.c
@@ -0,0 +1,376 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <glusterfs/common-utils.h>
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include <glusterfs/glusterfs.h>
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-geo-rep.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-svc-mgmt.h"
+#include "glusterd-svc-helper.h"
+#include "glusterd-volgen.h"
+#include "glusterd-messages.h"
+#include "glusterd-mgmt.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+
+#include <signal.h>
+
+int
+glusterd_reset_brick_prevalidate(dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+    int ret = 0;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    char *volname = NULL;
+    char *op = NULL;
+    glusterd_op_t gd_op = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    char *host = NULL;
+    char msg[2048] = {0};
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    char pidfile[PATH_MAX] = {0};
+    xlator_t *this = NULL;
+    gf_boolean_t is_force = _gf_false;
+    int32_t ignore_partition = 0;
+    pid_t pid = -1;
+    uuid_t volume_id = {
+        0,
+    };
+    char *dup_dstbrick = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_brick_op_prerequisites(dict, &op, &gd_op, &volname, &volinfo,
+                                          &src_brick, &src_brickinfo, pidfile,
+                                          op_errstr, rsp_dict);
+    if (ret)
+        goto out;
+
+    if (!strcmp(op, "GF_RESET_OP_START"))
+        goto done;
+
+    if (!strcmp(op, "GF_RESET_OP_COMMIT_FORCE"))
+        is_force = _gf_true;
+
+    ret = glusterd_get_dst_brick_info(&dst_brick, volname, op_errstr,
+                                      &dst_brickinfo, &host, dict,
+                                      &dup_dstbrick);
+    if (ret)
+        goto out;
+
+    ret = glusterd_new_brick_validate(dst_brick, dst_brickinfo, msg,
+                                      sizeof(msg), op);
+    /* if bricks are not same and reset brick was used, fail command.
+     * Only replace brick should be used to replace with new bricks
+     * to the volume.
+     */
+    if (ret == 0) {
+        if (!gf_uuid_compare(MY_UUID, dst_brickinfo->uuid)) {
+            ret = -1;
+            *op_errstr = gf_strdup(
+                "When destination brick is new,"
+                " please use"
+                " gluster volume "
+                "replace-brick <volname> "
+                "<src-brick> <dst-brick> "
+                "commit force");
+            if (*op_errstr)
+                gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                       GD_MSG_BRICK_VALIDATE_FAIL, "%s", *op_errstr);
+            goto out;
+        }
+    } else if (ret == 1) {
+        if (gf_is_service_running(pidfile, &pid)) {
+            ret = -1;
+            *op_errstr = gf_strdup(
+                "Source brick"
+                " must be stopped."
+                " Please use "
+                "gluster volume "
+                "reset-brick <volname> "
+                "<dst-brick> start.");
+            if (*op_errstr)
+                gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                       GD_MSG_BRICK_VALIDATE_FAIL, "%s", *op_errstr);
+            goto out;
+        }
+        ret = sys_lgetxattr(dst_brickinfo->path, GF_XATTR_VOL_ID_KEY, volume_id,
+                            16);
+        if (gf_uuid_compare(dst_brickinfo->uuid, src_brickinfo->uuid) ||
+            (ret >= 0 && is_force == _gf_false)) {
+            ret = -1;
+            *op_errstr = gf_strdup(
+                "Brick not available."
+                "It may be containing "
+                "or be contained "
+                "by an existing brick."
+                "Use 'force' option to "
+                "override this.");
+            if (*op_errstr)
+                gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                       GD_MSG_BRICK_VALIDATE_FAIL, "%s", *op_errstr);
+            goto out;
+        }
+        ret = 0;
+    } else {
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_VALIDATE_FAIL, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    volinfo->rep_brick.src_brick = src_brickinfo;
+    volinfo->rep_brick.dst_brick = dst_brickinfo;
+
+    ret = dict_get_int32n(dict, "ignore-partition", SLEN("ignore-partition"),
+                          &ignore_partition);
+    ret = 0;
+    if (gf_is_local_addr(host)) {
+        ret = glusterd_validate_and_create_brickpath(
+            dst_brickinfo, volinfo->volume_id, volinfo->volname, op_errstr,
+            is_force, ignore_partition);
+        if (ret)
+            goto out;
+    } else {
+        RCU_READ_LOCK;
+
+        peerinfo = glusterd_peerinfo_find(NULL, host);
+        if (peerinfo == NULL) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s, is not a friend.", host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s,"
+                     "is not connected at "
+                     "the moment.",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not befriended "
+                     "at the moment.",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+        RCU_READ_UNLOCK;
+    }
+
+    if (!(gf_uuid_compare(dst_brickinfo->uuid, MY_UUID))) {
+        ret = glusterd_get_brick_mount_dir(dst_brickinfo->path,
+                                           dst_brickinfo->hostname,
+                                           dst_brickinfo->mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                   "Failed to get brick mount_dir");
+            goto out;
+        }
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "brick1.mount_dir",
+                                         dst_brickinfo->mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set brick.mount_dir");
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"), 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set local_brick_count.");
+        goto out;
+    }
+
+done:
+    ret = 0;
+out:
+    GF_FREE(dup_dstbrick);
+    gf_msg_debug(this->name, 0, "Returning %d.", ret);
+
+    return ret;
+}
+
+int
+glusterd_op_reset_brick(dict_t *dict, dict_t *rsp_dict)
+{
+    int ret = 0;
+    char *op = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "operation", SLEN("operation"), &op);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "dict_get on operation failed");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_strn(dict, "src-brick", SLEN("src-brick"), &src_brick);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get src brick");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
+
+    ret = glusterd_volume_brickinfo_get_by_brick(src_brick, volinfo,
+                                                 &src_brickinfo, _gf_false);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get src-brickinfo");
+        goto out;
+    }
+
+    if (!strcmp(op, "GF_RESET_OP_START")) {
+        ret = glusterd_volume_stop_glusterfs(volinfo, src_brickinfo, _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_STOP_FAIL,
+                   "Unable to stop"
+                   " brick: %s:%s",
+                   src_brickinfo->hostname, src_brickinfo->path);
+        }
+
+        goto out;
+
+    } else if (!strcmp(op, "GF_RESET_OP_COMMIT") ||
+               !strcmp(op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = dict_get_strn(dict, "dst-brick", SLEN("dst-brick"), &dst_brick);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get dst brick");
+            goto out;
+        }
+
+        gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
+
+        ret = glusterd_get_rb_dst_brickinfo(volinfo, &dst_brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RB_BRICKINFO_GET_FAIL,
+                   "Unable to get "
+                   "reset brick "
+                   "destination brickinfo");
+            goto out;
+        }
+
+        ret = glusterd_resolve_brick(dst_brickinfo);
+        if (ret) {
+            gf_msg_debug(this->name, 0, "Unable to resolve dst-brickinfo");
+            goto out;
+        }
+
+        ret = rb_update_dstbrick_port(dst_brickinfo, rsp_dict, dict);
+        if (ret)
+            goto out;
+
+        if (gf_uuid_compare(dst_brickinfo->uuid, MY_UUID)) {
+            gf_msg_debug(this->name, 0, "I AM THE DESTINATION HOST");
+            ret = glusterd_volume_stop_glusterfs(volinfo, src_brickinfo,
+                                                 _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_STOP_FAIL,
+                       "Unable to stop brick: %s:%s", src_brickinfo->hostname,
+                       src_brickinfo->path);
+                goto out;
+            }
+        }
+
+        ret = glusterd_svcs_stop(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GLUSTER_SERVICES_STOP_FAIL,
+                   "Unable to stop gluster services, ret: %d", ret);
+            goto out;
+        }
+        ret = glusterd_op_perform_replace_brick(volinfo, src_brick, dst_brick,
+                                                dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_ADD_FAIL,
+                   "Unable to add dst-brick: "
+                   "%s to volume: %s",
+                   dst_brick, volinfo->volname);
+            (void)glusterd_svcs_manager(volinfo);
+            goto out;
+        }
+
+        volinfo->rebal.defrag_status = 0;
+
+        ret = glusterd_svcs_manager(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0,
+                   GD_MSG_GLUSTER_SERVICE_START_FAIL,
+                   "Failed to start one or more gluster services.");
+        }
+
+        ret = glusterd_fetchspec_notify(THIS);
+        glusterd_brickinfo_delete(volinfo->rep_brick.dst_brick);
+        volinfo->rep_brick.src_brick = NULL;
+        volinfo->rep_brick.dst_brick = NULL;
+
+        if (!ret)
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RBOP_STATE_STORE_FAIL,
+                   "Couldn't store"
+                   " reset brick operation's state.");
+        }
+    } else {
+        ret = -1;
+        goto out;
+    }
+
+out:
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index f7e8cfe248c..88662e3bbae 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -14,113 +14,107 @@
 
 #include "xdr-generic.h"
 
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "glusterd-op-sm.h"
 #include "glusterd-sm.h"
 #include "glusterd.h"
 #include "protocol-common.h"
 #include "glusterd-utils.h"
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd-messages.h"
 #include "glusterd-snapshot-utils.h"
 #include <sys/uio.h>
 
+#define SERVER_PATH_MAX (16 * 1024)
 
-#define SERVER_PATH_MAX  (16 * 1024)
-
+#define GLUSTERD_STACK_DESTROY(frame)                                          \
+    do {                                                                       \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+    } while (0)
 
 extern glusterd_op_info_t opinfo;
 extern uuid_t global_txn_id;
 
 int32_t
-glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
-                               int32_t op_errno, rpcsvc_request_t *req,
-                               void *op_ctx, char *op_errstr)
+glusterd_op_send_cli_response(glusterd_op_t op, int32_t op_ret,
+                              int32_t op_errno, rpcsvc_request_t *req,
+                              void *op_ctx, char *op_errstr)
 {
-        int32_t         ret = -1;
-        void            *cli_rsp = NULL;
-        dict_t          *ctx = NULL;
-        char            *free_ptr = NULL;
-        glusterd_conf_t *conf = NULL;
-        xdrproc_t       xdrproc = NULL;
-        char            *errstr = NULL;
-        int32_t         status = 0;
-        int32_t         count = 0;
-        gf_cli_rsp      rsp = {0,};
-        xlator_t        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        ctx = op_ctx;
-
-        switch (op) {
-        case GD_OP_DETACH_TIER:
-        case GD_OP_REMOVE_BRICK:
-        {
-                if (ctx)
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-                break;
+    int32_t ret = -1;
+    void *cli_rsp = NULL;
+    dict_t *ctx = NULL;
+    char *free_ptr = NULL;
+    glusterd_conf_t *conf = NULL;
+    xdrproc_t xdrproc = NULL;
+    char *errstr = NULL;
+    int32_t status = 0;
+    int32_t count = 0;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    ctx = op_ctx;
+
+    switch (op) {
+        case GD_OP_REMOVE_BRICK: {
+            if (ctx)
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+            break;
+        }
+        case GD_OP_RESET_VOLUME: {
+            if (op_ret && !op_errstr)
+                errstr = "Error while resetting options";
+            break;
         }
-        case GD_OP_RESET_VOLUME:
-        {
-                if (op_ret && !op_errstr)
-                        errstr = "Error while resetting options";
-                break;
-        }
-        case GD_OP_TIER_MIGRATE:
         case GD_OP_REBALANCE:
-        case GD_OP_DEFRAG_BRICK_VOLUME:
-        {
-                if (ctx) {
-                        ret = dict_get_int32 (ctx, "status", &status);
-                        if (ret) {
-                                gf_msg_trace (this->name, 0,
-                                        "failed to get status");
-                        }
+        case GD_OP_DEFRAG_BRICK_VOLUME: {
+            if (ctx) {
+                ret = dict_get_int32n(ctx, "status", SLEN("status"), &status);
+                if (ret) {
+                    gf_msg_trace(this->name, 0, "failed to get status");
                 }
-                break;
+            }
+            break;
         }
         case GD_OP_GSYNC_CREATE:
-        case GD_OP_GSYNC_SET:
-        {
-               if (ctx) {
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-                        ret = dict_set_str (ctx, "glusterd_workdir", conf->workdir);
-                        /* swallow error here, that will be re-triggered in cli */
-
-               }
-               break;
-
-        }
-        case GD_OP_PROFILE_VOLUME:
-        {
-                if (ctx && dict_get_int32 (ctx, "count", &count)) {
-                        ret = dict_set_int32 (ctx, "count", 0);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "failed to set count in dictionary");
-                        }
+        case GD_OP_GSYNC_SET: {
+            if (ctx) {
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+                ret = dict_set_strn(ctx, "glusterd_workdir",
+                                    SLEN("glusterd_workdir"), conf->workdir);
+                /* swallow error here, that will be re-triggered in cli */
+            }
+            break;
+        }
+        case GD_OP_PROFILE_VOLUME: {
+            if (ctx && dict_get_int32n(ctx, "count", SLEN("count"), &count)) {
+                ret = dict_set_int32n(ctx, "count", SLEN("count"), 0);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "failed to set count in dictionary");
                 }
-                break;
+            }
+            break;
         }
         case GD_OP_START_BRICK:
-        case GD_OP_STOP_BRICK:
-        {
-                gf_msg_debug (this->name, 0, "op '%s' not supported",
-                        gd_op_list[op]);
-                break;
+        case GD_OP_STOP_BRICK: {
+            gf_msg_debug(this->name, 0, "op '%s' not supported",
+                         gd_op_list[op]);
+            break;
         }
         case GD_OP_NONE:
-        case GD_OP_MAX:
-        {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_OP_UNSUPPORTED, "invalid operation");
-                break;
+        case GD_OP_MAX: {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_OP_UNSUPPORTED,
+                   "invalid operation");
+            break;
         }
         case GD_OP_CREATE_VOLUME:
         case GD_OP_START_VOLUME:
@@ -134,7 +128,6 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_REPLACE_BRICK:
         case GD_OP_STATUS_VOLUME:
         case GD_OP_SET_VOLUME:
-        case GD_OP_GANESHA:
         case GD_OP_LIST_VOLUME:
         case GD_OP_CLEARLOCKS_VOLUME:
         case GD_OP_HEAL_VOLUME:
@@ -143,2310 +136,2313 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_BARRIER:
         case GD_OP_BITROT:
         case GD_OP_SCRUB_STATUS:
-        {
-                /*nothing specific to be done*/
-                break;
-        }
-        case GD_OP_COPY_FILE:
-        {
-               if (ctx)
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-               break;
-        }
-        case GD_OP_SYS_EXEC:
-        {
-               if (ctx) {
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-                        ret = dict_set_str (ctx, "glusterd_workdir",
-                                            conf->workdir);
-               }
-               break;
-        }
-        }
+        case GD_OP_SCRUB_ONDEMAND:
+        case GD_OP_RESET_BRICK:
+        case GD_OP_MAX_OPVERSION:
+        case GD_OP_DETACH_NOT_STARTED:
+        case GD_OP_GANESHA:
+        case GD_OP_DETACH_TIER:
+        case GD_OP_TIER_MIGRATE:
+        case GD_OP_TIER_START_STOP:
+        case GD_OP_TIER_STATUS:
+        case GD_OP_DETACH_TIER_STATUS:
+        case GD_OP_REMOVE_TIER_BRICK:
+        case GD_OP_ADD_TIER_BRICK:
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
-
-        if (errstr)
-                rsp.op_errstr = errstr;
-        else if (op_errstr)
-                rsp.op_errstr = op_errstr;
-
-        if (!rsp.op_errstr)
-                rsp.op_errstr = "";
-
-        if (ctx) {
-                ret = dict_allocate_and_serialize (ctx, &rsp.dict.dict_val,
-                                                   &rsp.dict.dict_len);
-                if (ret < 0 )
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SERL_LENGTH_GET_FAIL, "failed to "
-                                "serialize buffer");
-                else
-                        free_ptr = rsp.dict.dict_val;
-        }
+        {
+            /*nothing specific to be done*/
+            break;
+        }
+        case GD_OP_COPY_FILE: {
+            if (ctx)
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+            break;
+        }
+        case GD_OP_SYS_EXEC: {
+            if (ctx) {
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+                ret = dict_set_strn(ctx, "glusterd_workdir",
+                                    SLEN("glusterd_workdir"), conf->workdir);
+            }
+            break;
+        }
+    }
+
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+
+    if (errstr)
+        rsp.op_errstr = errstr;
+    else if (op_errstr)
+        rsp.op_errstr = op_errstr;
+
+    if (!rsp.op_errstr)
+        rsp.op_errstr = "";
+
+    if (ctx) {
+        ret = dict_allocate_and_serialize(ctx, &rsp.dict.dict_val,
+                                          &rsp.dict.dict_len);
+        if (ret < 0)
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        else
+            free_ptr = rsp.dict.dict_val;
+    }
 
-        /* needed by 'rebalance status' */
-        if (status)
-                rsp.op_errno = status;
+    /* needed by 'rebalance status' */
+    if (status)
+        rsp.op_errno = status;
 
-        cli_rsp = &rsp;
-        xdrproc = (xdrproc_t) xdr_gf_cli_rsp;
+    cli_rsp = &rsp;
+    xdrproc = (xdrproc_t)xdr_gf_cli_rsp;
 
-        glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
-                         xdrproc, ctx);
-        ret = 0;
+    glusterd_to_cli(req, cli_rsp, NULL, 0, NULL, xdrproc, ctx);
+    ret = 0;
 
-        GF_FREE (free_ptr);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(free_ptr);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe, fop_cbk_fn_t fn)
+glusterd_big_locked_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe, fop_cbk_fn_t fn)
 {
-        glusterd_conf_t *priv = THIS->private;
-        int             ret   = -1;
+    glusterd_conf_t *priv = THIS->private;
+    int ret = -1;
 
-        synclock_lock (&priv->big_lock);
-        ret = fn (req, iov, count, myframe);
-        synclock_unlock (&priv->big_lock);
+    synclock_lock(&priv->big_lock);
+    ret = fn(req, iov, count, myframe);
+    synclock_unlock(&priv->big_lock);
 
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+__glusterd_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        gd1_mgmt_probe_rsp          rsp      = {{0},};
-        int                         ret      = 0;
-        glusterd_peerinfo_t        *peerinfo = NULL;
-        glusterd_friend_sm_event_t *event    = NULL;
-        glusterd_probe_ctx_t       *ctx      = NULL;
-        xlator_t                   *this     = NULL;
-        glusterd_conf_t            *conf     = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
+    gd1_mgmt_probe_rsp rsp = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_probe_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL, "error");
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_PROBE_REQ_RESP_RCVD,
+           "Received probe resp from uuid: %s, host: %s", uuid_utoa(rsp.uuid),
+           rsp.hostname);
+    if (rsp.op_ret != 0) {
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RES_DECODE_FAIL, "error");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
+        GF_ASSERT(ctx);
+
+        if (ctx->req) {
+            glusterd_xfer_cli_probe_resp(ctx->req, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr, ctx->hostname,
+                                         ctx->port, ctx->dict);
+        }
+
+        glusterd_destroy_probe_ctx(ctx);
+        (void)glusterd_friend_remove(rsp.uuid, rsp.hostname);
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, rsp.hostname);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peerd %s(%s)", rsp.hostname,
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    /*
+     * In the case of a fresh probe rsp.uuid and peerinfo.uuid will not
+     * match, as peerinfo->uuid will be NULL.
+     *
+     * In the case of a peer probe being done to add a new network to a
+     * peer, rsp.uuid will match an existing peerinfo.uuid. If we have this
+     * stage it means that the current address/hostname being used isn't
+     * present in the found peerinfo. If it were, we would have found out
+     * earlier in the probe process and wouldn't even reach till here. So,
+     * we need to add the new hostname to the peer.
+     *
+     * This addition should only be done for cluster op-version >=
+     * GD_OP_VERSION_3_6_0 as address lists are only supported from then on.
+     * Also, this update should only be done when an explicit CLI probe
+     * command was used to begin the probe process.
+     */
+    if ((conf->op_version >= GD_OP_VERSION_3_6_0) &&
+        (gf_uuid_compare(rsp.uuid, peerinfo->uuid) == 0)) {
+        ctx = ((call_frame_t *)myframe)->local;
+        /* Presence of ctx->req implies this probe was started by a cli
+         * probe command
+         */
+        if (ctx->req == NULL)
+            goto cont;
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_PROBE_REQ_RESP_RCVD,
-                "Received probe resp from uuid: %s, host: %s",
-                uuid_utoa (rsp.uuid), rsp.hostname);
-        if (rsp.op_ret != 0) {
-                ctx = ((call_frame_t *)myframe)->local;
-                ((call_frame_t *)myframe)->local = NULL;
-
-                GF_ASSERT (ctx);
-
-                if (ctx->req) {
-                        glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
-                                                      rsp.op_errno,
-                                                      rsp.op_errstr,
-                                                      ctx->hostname, ctx->port,
-                                                      ctx->dict);
-                }
+        gf_msg_debug(this->name, 0,
+                     "Adding address '%s' to "
+                     "existing peer %s",
+                     rsp.hostname, uuid_utoa(rsp.uuid));
 
-                glusterd_destroy_probe_ctx (ctx);
-                (void) glusterd_friend_remove (rsp.uuid, rsp.hostname);
-                ret = rsp.op_ret;
-                goto out;
+        ret = glusterd_friend_remove(NULL, rsp.hostname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_STALE_PEERINFO_REMOVE_FAIL,
+                   "Could not remove "
+                   "stale peerinfo with name %s",
+                   rsp.hostname);
+            goto reply;
         }
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, rsp.hostname);
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_PEER_NOT_FOUND, "Could not find peerd %s(%s)",
-                        rsp.hostname, uuid_utoa (rsp.uuid));
-                goto unlock;
+        ret = gd_add_address_to_peer(peerinfo, rsp.hostname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
+                   "Couldn't add hostname to peer list");
+            goto reply;
         }
 
-        /*
-         * In the case of a fresh probe rsp.uuid and peerinfo.uuid will not
-         * match, as peerinfo->uuid will be NULL.
-         *
-         * In the case of a peer probe being done to add a new network to a
-         * peer, rsp.uuid will match an existing peerinfo.uuid. If we have this
-         * stage it means that the current address/hostname being used isn't
-         * present in the found peerinfo. If it were, we would have found out
-         * earlier in the probe process and wouldn't even reach till here. So,
-         * we need to add the new hostname to the peer.
-         *
-         * This addition should only be done for cluster op-version >=
-         * GD_OP_VERSION_3_6_0 as address lists are only supported from then on.
-         * Also, this update should only be done when an explicit CLI probe
-         * command was used to begin the probe process.
-         */
-        if ((conf->op_version >= GD_OP_VERSION_3_6_0) &&
-            (gf_uuid_compare (rsp.uuid, peerinfo->uuid) == 0)) {
-                ctx = ((call_frame_t *)myframe)->local;
-                /* Presence of ctx->req implies this probe was started by a cli
-                 * probe command
-                 */
-                if (ctx->req == NULL)
-                        goto cont;
-
-                gf_msg_debug (this->name, 0, "Adding address '%s' to "
-                        "existing peer %s", rsp.hostname, uuid_utoa (rsp.uuid));
-
-                ret = glusterd_friend_remove (NULL, rsp.hostname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_STALE_PEERINFO_REMOVE_FAIL,
-                                "Could not remove "
-                                "stale peerinfo with name %s", rsp.hostname);
-                        goto reply;
-                }
-
-                ret = gd_add_address_to_peer (peerinfo, rsp.hostname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
-                                "Couldn't add hostname to peer list");
-                        goto reply;
-                }
-
-                /* Injecting EVENT_NEW_NAME to send update */
-                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_NEW_NAME,
-                                                    &event);
-                if (!ret) {
-                        event->peername = gf_strdup (peerinfo->hostname);
-                        gf_uuid_copy (event->peerid, peerinfo->uuid);
+        /* Injecting EVENT_NEW_NAME to send update */
+        ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_NEW_NAME, &event);
+        if (!ret) {
+            event->peername = gf_strdup(peerinfo->hostname);
+            gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-                        ret = glusterd_friend_sm_inject_event (event);
-                }
-                rsp.op_errno = GF_PROBE_FRIEND;
+            ret = glusterd_friend_sm_inject_event(event);
+        }
+        rsp.op_errno = GF_PROBE_FRIEND;
 
-reply:
-                ctx = ((call_frame_t *)myframe)->local;
-                ((call_frame_t *)myframe)->local = NULL;
+    reply:
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
 
-                if (!ctx) {
-                        ret = -1;
-                        goto unlock;
-                }
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
 
-                if (ctx->req) {
-                        glusterd_xfer_cli_probe_resp (ctx->req, ret,
-                                                      rsp.op_errno,
-                                                      rsp.op_errstr,
-                                                      ctx->hostname, ctx->port,
-                                                      ctx->dict);
-                }
+        if (ctx->req) {
+            glusterd_xfer_cli_probe_resp(ctx->req, ret, rsp.op_errno,
+                                         rsp.op_errstr, ctx->hostname,
+                                         ctx->port, ctx->dict);
+        }
 
-                glusterd_destroy_probe_ctx (ctx);
+        glusterd_destroy_probe_ctx(ctx);
 
-                goto unlock;
+        goto unlock;
 
-        } else if (strncasecmp (rsp.hostname, peerinfo->hostname, 1024)) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0,
-                        GD_MSG_HOST_PRESENT_ALREADY, "Host: %s  with uuid: %s "
-                        "already present in cluster with alias hostname: %s",
-                        rsp.hostname, uuid_utoa (rsp.uuid), peerinfo->hostname);
+    } else if (strncasecmp(rsp.hostname, peerinfo->hostname, 1024)) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_HOST_PRESENT_ALREADY,
+               "Host: %s  with uuid: %s "
+               "already present in cluster with alias hostname: %s",
+               rsp.hostname, uuid_utoa(rsp.uuid), peerinfo->hostname);
 
-                ctx = ((call_frame_t *)myframe)->local;
-                ((call_frame_t *)myframe)->local = NULL;
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
 
-                if (!ctx) {
-                        ret = -1;
-                        goto unlock;
-                }
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
 
-                rsp.op_errno = GF_PROBE_FRIEND;
-                if (ctx->req) {
-                        glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
-                                                      rsp.op_errno,
-                                                      rsp.op_errstr,
-                                                      ctx->hostname, ctx->port,
-                                                      ctx->dict);
-                }
+        rsp.op_errno = GF_PROBE_FRIEND;
+        if (ctx->req) {
+            glusterd_xfer_cli_probe_resp(ctx->req, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr, ctx->hostname,
+                                         ctx->port, ctx->dict);
+        }
 
-                glusterd_destroy_probe_ctx (ctx);
-                (void) glusterd_friend_remove (NULL, rsp.hostname);
-                ret = rsp.op_ret;
+        glusterd_destroy_probe_ctx(ctx);
+        (void)glusterd_friend_remove(NULL, rsp.hostname);
+        ret = rsp.op_ret;
 
-                goto unlock;
-        }
+        goto unlock;
+    }
 
 cont:
-        gf_uuid_copy (peerinfo->uuid, rsp.uuid);
-
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_INIT_FRIEND_REQ, &event);
+    gf_uuid_copy(peerinfo->uuid, rsp.uuid);
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_NEW_FRIEND_SM_EVENT_GET_FAIL,
-                         "Unable to get event");
-                goto unlock;
-        }
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_INIT_FRIEND_REQ, &event);
 
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_NEW_FRIEND_SM_EVENT_GET_FAIL,
+               "Unable to get event");
+        goto out;
+    }
 
-        event->ctx      = ((call_frame_t *)myframe)->local;
-        ((call_frame_t *)myframe)->local = NULL;
-        ret = glusterd_friend_sm_inject_event (event);
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
 
+    event->ctx = ((call_frame_t *)myframe)->local;
+    ((call_frame_t *)myframe)->local = NULL;
+    ret = glusterd_friend_sm_inject_event(event);
 
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_PROBE_REQ_RESP_RCVD, "Received resp to probe req");
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PROBE_REQ_RESP_RCVD,
+           "Received resp to probe req");
 
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
 out:
-        free (rsp.hostname);//malloced by xdr
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-
-        /* Attempt to start the state machine. Needed as no state machine could
-         * be running at time this RPC reply was received
-         */
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-
-        return ret;
+    free(rsp.hostname);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+
+    /* Attempt to start the state machine. Needed as no state machine could
+     * be running at time this RPC reply was received
+     */
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    return ret;
 }
 
 int
-glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_probe_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_probe_cbk);
 }
 
-
 int
-__glusterd_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
-                            int count, void *myframe)
+__glusterd_friend_add_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        gd1_mgmt_friend_rsp           rsp   = {{0},};
-        int                           ret   = -1;
-        glusterd_friend_sm_event_t        *event = NULL;
-        glusterd_friend_sm_event_type_t    event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        int32_t                       op_ret = -1;
-        int32_t                       op_errno = -1;
-        glusterd_probe_ctx_t          *ctx = NULL;
-        glusterd_friend_update_ctx_t  *ev_ctx = NULL;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
-        if (ret < 0) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_RES_DECODE_FAIL, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_RESPONSE_INFO,
-                "Received %s from uuid: %s, host: %s, port: %d",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid), rsp.hostname, rsp.port);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, rsp.hostname);
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_RESP_FROM_UNKNOWN_PEER,
-                        "received friend add response from"
-                        " unknown peer uuid: %s", uuid_utoa (rsp.uuid));
-                goto unlock;
-        }
-
-        if (op_ret)
-                event_type = GD_FRIEND_EVENT_RCVD_RJT;
-        else
-                event_type = GD_FRIEND_EVENT_RCVD_ACC;
-
-        ret = glusterd_friend_sm_new_event (event_type, &event);
-
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_NEW_GET_FAIL,
-                         "Unable to get event");
-                goto unlock;
-        }
-
-        ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx),
-                                gf_gld_mt_friend_update_ctx_t);
-        if (!ev_ctx) {
-                ret = -1;
-                goto unlock;
-        }
-
-        gf_uuid_copy (ev_ctx->uuid, rsp.uuid);
-        ev_ctx->hostname = gf_strdup (rsp.hostname);
-
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
-        event->ctx = ev_ctx;
-        ret = glusterd_friend_sm_inject_event (event);
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glusterd_probe_ctx_t *ctx = NULL;
+    glusterd_friend_update_ctx_t *ev_ctx = NULL;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_RES_DECODE_FAIL,
+               "error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Received %s from uuid: %s, host: %s, port: %d",
+           (op_ret) ? "RJT" : "ACC", uuid_utoa(rsp.uuid), rsp.hostname,
+           rsp.port);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, rsp.hostname);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "received friend add response from"
+               " unknown peer uuid: %s",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret)
+        event_type = GD_FRIEND_EVENT_RCVD_RJT;
+    else
+        event_type = GD_FRIEND_EVENT_RCVD_ACC;
+
+    ret = glusterd_friend_sm_new_event(event_type, &event);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get event");
+        goto unlock;
+    }
+
+    ev_ctx = GF_CALLOC(1, sizeof(*ev_ctx), gf_gld_mt_friend_update_ctx_t);
+    if (!ev_ctx) {
+        ret = -1;
+        goto unlock;
+    }
+
+    gf_uuid_copy(ev_ctx->uuid, rsp.uuid);
+    ev_ctx->hostname = gf_strdup(rsp.hostname);
+
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
+    event->ctx = ev_ctx;
+    ret = glusterd_friend_sm_inject_event(event);
 
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 out:
-        ctx = ((call_frame_t *)myframe)->local;
-        ((call_frame_t *)myframe)->local = NULL;
-
-        GF_ASSERT (ctx);
-
-        if (ctx->req)//reverse probe doesn't have req
-                ret = glusterd_xfer_cli_probe_resp (ctx->req, op_ret, op_errno,
-                                                    NULL, ctx->hostname,
-                                                    ctx->port, ctx->dict);
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-
-        if (ctx)
-                glusterd_destroy_probe_ctx (ctx);
-        free (rsp.hostname);//malloced by xdr
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    ctx = ((call_frame_t *)myframe)->local;
+    ((call_frame_t *)myframe)->local = NULL;
+
+    if (ctx && ctx->req) {
+        /*reverse probe doesn't have req*/
+        ret = glusterd_xfer_cli_probe_resp(ctx->req, op_ret, op_errno, NULL,
+                                           ctx->hostname, ctx->port, ctx->dict);
+    }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    if (ctx)
+        glusterd_destroy_probe_ctx(ctx);
+    free(rsp.hostname);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int
-glusterd_friend_add_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_friend_add_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_friend_add_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_friend_add_cbk);
 }
 
 int
-__glusterd_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
-                               int count, void *myframe)
+__glusterd_friend_remove_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        gd1_mgmt_friend_rsp             rsp   = {{0},};
-        glusterd_conf_t                 *conf = NULL;
-        int                             ret   = -1;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        int32_t                         op_ret = -1;
-        int32_t                         op_errno = -1;
-        glusterd_probe_ctx_t            *ctx = NULL;
-        gf_boolean_t                    move_sm_now = _gf_true;
-
-        conf  = THIS->private;
-        GF_ASSERT (conf);
-
-        ctx = ((call_frame_t *)myframe)->local;
-        ((call_frame_t *)myframe)->local = NULL;
-        GF_ASSERT (ctx);
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                move_sm_now = _gf_false;
-                goto inject;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
-        if (ret < 0) {
-                gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                        GD_MSG_RES_DECODE_FAIL, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto respond;
-        }
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
-
-        gf_msg ("glusterd", GF_LOG_INFO, 0,
-                GD_MSG_RESPONSE_INFO,
-                "Received %s from uuid: %s, host: %s, port: %d",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid), rsp.hostname, rsp.port);
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    glusterd_probe_ctx_t *ctx = NULL;
+    gf_boolean_t move_sm_now = _gf_true;
+
+    conf = THIS->private;
+    GF_ASSERT(conf);
+
+    ctx = ((call_frame_t *)myframe)->local;
+    ((call_frame_t *)myframe)->local = NULL;
+    if (!ctx) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get glusterd probe context");
+        goto out;
+    }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        move_sm_now = _gf_false;
+        goto inject;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_RES_DECODE_FAIL,
+               "error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto respond;
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Received %s from uuid: %s, host: %s, port: %d",
+           (op_ret) ? "RJT" : "ACC", uuid_utoa(rsp.uuid), rsp.hostname,
+           rsp.port);
 
 inject:
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, ctx->hostname);
-        if (peerinfo == NULL) {
-                //can happen as part of rpc clnt connection cleanup
-                //when the frame timeout happens after 30 minutes
-                ret = -1;
-                goto unlock;
-        }
+    RCU_READ_LOCK;
 
-        event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, ctx->hostname);
+    if (peerinfo == NULL) {
+        // can happen as part of rpc clnt connection cleanup
+        // when the frame timeout happens after 30 minutes
+        goto unlock;
+    }
 
-        ret = glusterd_friend_sm_new_event (event_type, &event);
+    event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
 
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_NEW_GET_FAIL,
-                         "Unable to get event");
-                goto unlock;
-        }
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
+    ret = glusterd_friend_sm_new_event(event_type, &event);
 
-        ret = glusterd_friend_sm_inject_event (event);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get event");
+        goto unlock;
+    }
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-        if (ret)
-                goto unlock;
+    ret = glusterd_friend_sm_inject_event(event);
 
-        /*friend_sm would be moved on CLNT_DISCONNECT, consequently
-          cleaning up peerinfo. Else, we run the risk of triggering
-          a clnt_destroy within saved_frames_unwind.
-        */
-        op_ret = 0;
+    if (ret)
+        goto unlock;
+
+    /*friend_sm would be moved on CLNT_DISCONNECT, consequently
+      cleaning up peerinfo. Else, we run the risk of triggering
+      a clnt_destroy within saved_frames_unwind.
+    */
+    op_ret = 0;
 
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
 respond:
-        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno, NULL,
-                                              ctx->hostname, ctx->dict);
-        if (!ret && move_sm_now) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-
-        if (ctx) {
-                glusterd_broadcast_friend_delete (ctx->hostname, NULL);
-                glusterd_destroy_probe_ctx (ctx);
-        }
-
-        free (rsp.hostname);//malloced by xdr
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    ret = glusterd_xfer_cli_deprobe_resp(ctx->req, op_ret, op_errno, NULL,
+                                         ctx->hostname, ctx->dict);
+    if (!ret && move_sm_now) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    glusterd_broadcast_friend_delete(ctx->hostname, NULL);
+    glusterd_destroy_probe_ctx(ctx);
+out:
+    free(rsp.hostname);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int
-glusterd_friend_remove_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_friend_remove_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_friend_remove_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_friend_remove_cbk);
 }
 
 int32_t
-__glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+__glusterd_friend_update_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        int                           ret    = -1;
-        gd1_mgmt_friend_update_rsp    rsp    = {{0}, };
-        xlator_t                      *this  = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-
-        if (-1 == req->rpc_status) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RPC_FAILURE, "RPC Error");
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RES_DECODE_FAIL, "Failed to serialize friend"
-                        " update repsonse");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    gd1_mgmt_friend_update_rsp rsp = {
+        {0},
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+
+    if (-1 == req->rpc_status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE, "RPC Error");
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to serialize friend"
+               " update response");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                GD_MSG_RESPONSE_INFO, "Received %s from uuid: %s",
-                (ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Received %s from uuid: %s", (ret) ? "RJT" : "ACC",
+           uuid_utoa(rsp.uuid));
 
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int
-glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_friend_update_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_friend_update_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_friend_update_cbk);
 }
 
 int32_t
-__glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+__glusterd_cluster_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
 {
-        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        xlator_t                      *this = NULL;
-        uuid_t                        *txn_id = NULL;
-        glusterd_conf_t               *priv = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        if (-1 == req->rpc_status) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_LOCK_RESP_FROM_PEER,  "Lock response is not "
-                        "received from one of the peer");
-                err_str = "Lock response is not received from one of the peer";
-                glusterd_set_opinfo (err_str, ENETRESET, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RES_DECODE_FAIL, "Failed to decode "
-                        "cluster lock response received from peer");
-                err_str = "Failed to decode cluster lock response received from"
-                          " peer";
-                glusterd_set_opinfo (err_str, EINVAL, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        if (op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_LOCK_FROM_UUID_REJCT,
-                        "Received lock RJT from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Received lock ACC from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_RESP_FROM_UNKNOWN_PEER,
-                        "cluster lock response received from unknown peer: %s."
-                        "Ignoring response", uuid_utoa (rsp.uuid));
-                err_str = "cluster lock response received from unknown peer";
-                goto out;
-
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
-                                              "progress. Please try again after"
-                                              " sometime.");
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    if (-1 == req->rpc_status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_LOCK_RESP_FROM_PEER,
+               "Lock response is not "
+               "received from one of the peer");
+        err_str = "Lock response is not received from one of the peer";
+        glusterd_set_opinfo(err_str, ENETRESET, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode "
+               "cluster lock response received from peer");
+        err_str =
+            "Failed to decode cluster lock response received from"
+            " peer";
+        glusterd_set_opinfo(err_str, EINVAL, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_LOCK_FROM_UUID_REJCT,
+               "Received lock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received lock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "cluster lock response received from unknown peer: %s."
+               "Ignoring response",
+               uuid_utoa(rsp.uuid));
+        err_str = "cluster lock response received from unknown peer";
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        opinfo.op_errstr = gf_strdup(
+            "Another transaction could be in "
+            "progress. Please try again after"
+            " some time.");
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
 out:
 
-        ret = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
 
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+glusterd_cluster_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_cluster_lock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_cluster_lock_cbk);
 }
 
 void
-glusterd_set_opinfo (char *errstr, int32_t op_errno, int32_t op_ret)
+glusterd_set_opinfo(char *errstr, int32_t op_errno, int32_t op_ret)
 {
-        opinfo.op_errstr = gf_strdup (errstr);
-        opinfo.op_errno = op_errno;
-        opinfo.op_ret = op_ret;
+    opinfo.op_errstr = gf_strdup(errstr);
+    opinfo.op_errno = op_errno;
+    opinfo.op_ret = op_ret;
 }
 
 static int32_t
-glusterd_mgmt_v3_lock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                    int count, void *myframe)
+glusterd_mgmt_v3_lock_peers_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                   int count, void *myframe)
 {
-        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        xlator_t                      *this = NULL;
-        call_frame_t                  *frame  = NULL;
-        uuid_t                        *txn_id = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        frame = myframe;
-        txn_id = frame->cookie;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_LOCK_RESP_FROM_PEER, "Lock response is not "
-                        "received from one of the peer");
-                err_str = "Lock response is not received from one of the peer";
-                glusterd_set_opinfo (err_str, ENETRESET, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RES_DECODE_FAIL, "Failed to decode "
-                        "mgmt_v3 lock response received from peer");
-                err_str = "Failed to decode mgmt_v3 lock response received from"
-                          " peer";
-                glusterd_set_opinfo (err_str, EINVAL, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        txn_id = &rsp.txn_id;
-
-        if (op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_LOCK_FROM_UUID_REJCT,
-                        "Received mgmt_v3 lock RJT from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Received mgmt_v3 lock ACC from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_RESP_FROM_UNKNOWN_PEER,
-                        "mgmt_v3 lock response received "
-                        "from unknown peer: %s. Ignoring response",
-                         uuid_utoa (rsp.uuid));
-                goto out;
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
-                                              "progress. Please try again after"
-                                              " sometime.");
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    uuid_t *txn_id = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_LOCK_RESP_FROM_PEER,
+               "Lock response is not "
+               "received from one of the peer");
+        err_str = "Lock response is not received from one of the peer";
+        glusterd_set_opinfo(err_str, ENETRESET, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode "
+               "mgmt_v3 lock response received from peer");
+        err_str =
+            "Failed to decode mgmt_v3 lock response received from"
+            " peer";
+        glusterd_set_opinfo(err_str, EINVAL, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    txn_id = &rsp.txn_id;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_FROM_UUID_REJCT,
+               "Received mgmt_v3 lock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received mgmt_v3 lock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "mgmt_v3 lock response received "
+               "from unknown peer: %s. Ignoring response",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        opinfo.op_errstr = gf_strdup(
+            "Another transaction could be in "
+            "progress. Please try again after"
+            " some time.");
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
 out:
 
-        ret = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-
-        GF_FREE (frame->cookie);
-        GLUSTERD_STACK_DESTROY (frame);
-        return ret;
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(frame);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_lock_peers_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+glusterd_mgmt_v3_lock_peers_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        glusterd_mgmt_v3_lock_peers_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   glusterd_mgmt_v3_lock_peers_cbk_fn);
 }
 
 static int32_t
-glusterd_mgmt_v3_unlock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                      int count, void *myframe)
+glusterd_mgmt_v3_unlock_peers_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
 {
-        gd1_mgmt_v3_unlock_rsp        rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        xlator_t                      *this = NULL;
-        call_frame_t                  *frame = NULL;
-        uuid_t                        *txn_id = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        frame = myframe;
-        txn_id = frame->cookie;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                err_str = "Unlock response not received from one of the peer.";
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "UnLock response is not received from one of the peer");
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "Failed to decode mgmt_v3 unlock response received from"
-                        "peer");
-                err_str = "Failed to decode mgmt_v3 unlock response received "
-                          "from peer";
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        txn_id = &rsp.txn_id;
-
-        if (op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MGMTV3_UNLOCK_FROM_UUID_REJCT,
-                        "Received mgmt_v3 unlock RJT from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Received mgmt_v3 unlock ACC from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "mgmt_v3 unlock response received "
-                        "from unknown peer: %s. Ignoring response",
-                        uuid_utoa (rsp.uuid));
-                goto out;
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
-                                              "progress. Please try again after"
-                                              " sometime.");
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    uuid_t *txn_id = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        err_str = "Unlock response not received from one of the peer.";
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "UnLock response is not received from one of the peer");
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "Failed to decode mgmt_v3 unlock response received from"
+               "peer");
+        err_str =
+            "Failed to decode mgmt_v3 unlock response received "
+            "from peer";
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    txn_id = &rsp.txn_id;
+
+    if (op_ret) {
+        gf_msg(
+            this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FROM_UUID_REJCT,
+            "Received mgmt_v3 unlock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received mgmt_v3 unlock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "mgmt_v3 unlock response received "
+               "from unknown peer: %s. Ignoring response",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        opinfo.op_errstr = gf_strdup(
+            "Another transaction could be in "
+            "progress. Please try again after"
+            " some time.");
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
 out:
 
-        ret = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
 
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        GF_FREE (frame->cookie);
-        GLUSTERD_STACK_DESTROY (frame);
-        return ret;
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(frame);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_unlock_peers_cbk (struct rpc_req *req, struct iovec *iov,
-                                   int count, void *myframe)
+glusterd_mgmt_v3_unlock_peers_cbk(struct rpc_req *req, struct iovec *iov,
+                                  int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        glusterd_mgmt_v3_unlock_peers_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   glusterd_mgmt_v3_unlock_peers_cbk_fn);
 }
 
 int32_t
-__glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+__glusterd_cluster_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                              void *myframe)
 {
-        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        xlator_t                      *this = NULL;
-        uuid_t                        *txn_id = NULL;
-        glusterd_conf_t               *priv = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        if (-1 == req->rpc_status) {
-                err_str = "Unlock response not received from one of the peer.";
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "UnLock response is not received from one of the peer");
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "Failed to decode unlock response received from peer");
-                err_str = "Failed to decode cluster unlock response received "
-                          "from peer";
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        if (op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UNLOCK_FROM_UUID_REJCT,
-                        "Received unlock RJT from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Received unlock ACC from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "Unlock response received from unknown peer %s",
-                        uuid_utoa (rsp.uuid));
-                goto out;
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    if (-1 == req->rpc_status) {
+        err_str = "Unlock response not received from one of the peer.";
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "UnLock response is not received from one of the peer");
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "Failed to decode unlock response received from peer");
+        err_str =
+            "Failed to decode cluster unlock response received "
+            "from peer";
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNLOCK_FROM_UUID_REJCT,
+               "Received unlock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received unlock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "Unlock response received from unknown peer %s",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
 out:
 
-        ret = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
 
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+glusterd_cluster_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_cluster_unlock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_cluster_unlock_cbk);
 }
 
 int32_t
-__glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+__glusterd_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gd1_mgmt_stage_op_rsp         rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        dict_t                        *dict   = NULL;
-        char                          err_str[2048] = {0};
-        char                          *peer_str = NULL;
-        xlator_t                      *this = NULL;
-        glusterd_conf_t               *priv = NULL;
-        uuid_t                        *txn_id = NULL;
-        call_frame_t                  *frame = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT(myframe);
-
-        frame = myframe;
-        txn_id = frame->cookie;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("error");
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    gd1_mgmt_stage_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+    char *peer_str = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uuid_t *txn_id = NULL;
+    call_frame_t *frame = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup("error");
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode stage "
+               "response received from peer");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup(
+            "Failed to decode stage response "
+            "received from peer.");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RES_DECODE_FAIL, "Failed to decode stage "
-                        "response received from peer");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("Failed to decode stage response "
-                                        "received from peer.");
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize rsp-buffer to dictionary");
-			event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-out:
-        op_ret = rsp.op_ret;
-
-        if (op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STAGE_FROM_UUID_REJCT,
-                        "Received stage RJT from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Received stage ACC from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, NULL);
-        if (peerinfo == NULL) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_RESP_FROM_UNKNOWN_PEER, "Stage response received "
-                        "from unknown peer: %s. Ignoring response.",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                if (strcmp ("", rsp.op_errstr)) {
-                        opinfo.op_errstr = gf_strdup (rsp.op_errstr);
-                } else {
-                        if (peerinfo)
-                                peer_str = peerinfo->hostname;
-                        else
-                                peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str),
-                                  OPERRSTR_STAGE_FAIL, peer_str);
-                        opinfo.op_errstr = gf_strdup (err_str);
-                }
-                if (!opinfo.op_errstr)
-                        ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize rsp-buffer to dictionary");
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
         } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
-
-        rcu_read_unlock ();
-
-
-        ret = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
-
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
+            dict->extra_stdfree = rsp.dict.dict_val;
         }
+    }
 
-        free (rsp.op_errstr); //malloced by xdr
-        if (dict) {
-                if (!dict->extra_stdfree && rsp.dict.dict_val)
-                        free (rsp.dict.dict_val); //malloced by xdr
-                dict_unref (dict);
+out:
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAGE_FROM_UUID_REJCT,
+               "Received stage RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received stage ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, NULL);
+    if (peerinfo == NULL) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Stage response received "
+               "from unknown peer: %s. Ignoring response.",
+               uuid_utoa(rsp.uuid));
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        if (strcmp("", rsp.op_errstr)) {
+            opinfo.op_errstr = gf_strdup(rsp.op_errstr);
         } else {
-                free (rsp.dict.dict_val); //malloced by xdr
-        }
-        GF_FREE (frame->cookie);
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+            if (peerinfo)
+                peer_str = peerinfo->hostname;
+            else
+                peer_str = uuid_utoa(rsp.uuid);
+            char err_str[2048];
+            snprintf(err_str, sizeof(err_str), OPERRSTR_STAGE_FAIL, peer_str);
+            opinfo.op_errstr = gf_strdup(err_str);
+        }
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
+
+    RCU_READ_UNLOCK;
+
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
+
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    free(rsp.op_errstr);  // malloced by xdr
+    if (dict) {
+        if (!dict->extra_stdfree && rsp.dict.dict_val)
+            free(rsp.dict.dict_val);  // malloced by xdr
+        dict_unref(dict);
+    } else {
+        free(rsp.dict.dict_val);  // malloced by xdr
+    }
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+glusterd_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_stage_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_stage_op_cbk);
 }
 
 int32_t
-__glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+__glusterd_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gd1_mgmt_commit_op_rsp        rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        dict_t                        *dict = NULL;
-        char                          err_str[2048] = {0};
-        char                          *peer_str = NULL;
-        xlator_t                      *this = NULL;
-        glusterd_conf_t               *priv = NULL;
-        uuid_t                        *txn_id = NULL;
-        glusterd_op_info_t            txn_op_info = {{0},};
-        call_frame_t                  *frame  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT(myframe);
-
-        frame = myframe;
-        txn_id = frame->cookie;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("error");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    gd1_mgmt_commit_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+    char *peer_str = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup("error");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode commit "
+               "response received from peer");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup(
+            "Failed to decode commit response "
+            "received from peer.");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RES_DECODE_FAIL, "Failed to decode commit "
-                        "response received from peer");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("Failed to decode commit response "
-                                        "received from peer.");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "failed to "
-                                "unserialize rsp-buffer to dictionary");
-			event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        op_ret = rsp.op_ret;
-
-        if (op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMIT_FROM_UUID_REJCT,
-                        "Received commit RJT from uuid: %s",
-                        uuid_utoa (rsp.uuid));
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize rsp-buffer to dictionary");
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
         } else {
-                gf_msg_debug (this->name, 0,
-                        "Received commit ACC from uuid: %s",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_OPINFO_GET_FAIL,
-                        "Failed to get txn_op_info "
-                        "for txn_id = %s", uuid_utoa (*txn_id));
-        }
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, NULL);
-        if (peerinfo == NULL) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_RESP_FROM_UNKNOWN_PEER, "Commit response for "
-                        "'Volume %s' received from unknown peer: %s",
-                        gd_op_list[opinfo.op], uuid_utoa (rsp.uuid));
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                if (strcmp ("", rsp.op_errstr)) {
-                        opinfo.op_errstr = gf_strdup(rsp.op_errstr);
-                } else {
-                        if (peerinfo)
-                                peer_str = peerinfo->hostname;
-                        else
-                                peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str),
-                                  OPERRSTR_COMMIT_FAIL, peer_str);
-                        opinfo.op_errstr = gf_strdup (err_str);
-                }
-                if (!opinfo.op_errstr) {
-                        ret = -1;
-                        goto unlock;
-                }
+            dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_FROM_UUID_REJCT,
+               "Received commit RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received commit ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    ret = glusterd_get_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_GET_FAIL,
+               "Failed to get txn_op_info "
+               "for txn_id = %s",
+               uuid_utoa(*txn_id));
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, NULL);
+    if (peerinfo == NULL) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Commit response for "
+               "'Volume %s' received from unknown peer: %s",
+               gd_op_list[opinfo.op], uuid_utoa(rsp.uuid));
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        if (strcmp("", rsp.op_errstr)) {
+            opinfo.op_errstr = gf_strdup(rsp.op_errstr);
         } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-                GF_ASSERT (rsp.op == txn_op_info.op);
-
-                switch (rsp.op) {
-
-                case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_profile_volume_use_rsp_dict (txn_op_info.op_ctx, dict);
-                        if (ret)
-                                goto unlock;
+            if (peerinfo)
+                peer_str = peerinfo->hostname;
+            else
+                peer_str = uuid_utoa(rsp.uuid);
+            char err_str[2048];
+            snprintf(err_str, sizeof(err_str), OPERRSTR_COMMIT_FAIL, peer_str);
+            opinfo.op_errstr = gf_strdup(err_str);
+        }
+        if (!opinfo.op_errstr) {
+            goto unlock;
+        }
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+        GF_ASSERT(rsp.op == txn_op_info.op);
+
+        switch (rsp.op) {
+            case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_use_rsp_dict(txn_op_info.op_ctx,
+                                                           dict);
+                if (ret)
+                    goto unlock;
                 break;
 
-                case GD_OP_REBALANCE:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        ret = glusterd_volume_rebalance_use_rsp_dict (txn_op_info.op_ctx, dict);
-                        if (ret)
-                                goto unlock;
+            case GD_OP_REBALANCE:
+            case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_volume_rebalance_use_rsp_dict(txn_op_info.op_ctx,
+                                                             dict);
+                if (ret)
+                    goto unlock;
                 break;
 
-                default:
+            default:
                 break;
-                }
         }
+    }
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
 out:
 
-        ret = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
-
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
-
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr); //malloced by xdr
-        GF_FREE (frame->cookie);
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
+
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    if (dict)
+        dict_unref(dict);
+    free(rsp.op_errstr);  // malloced by xdr
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+glusterd_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_commit_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_commit_op_cbk);
 }
 
 int32_t
-glusterd_rpc_probe (call_frame_t *frame, xlator_t *this,
-                    void *data)
+glusterd_rpc_probe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_probe_req      req = {{0},};
-        int                     ret = 0;
-        int                     port = 0;
-        char                    *hostname = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = GF_DEFAULT_BASE_PORT;
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-        req.hostname = gf_strdup (hostname);
-        req.port = port;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
-                                       GLUSTERD_PROBE_QUERY,
-                                       NULL, this, glusterd_probe_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_probe_req);
+    gd1_mgmt_probe_req req = {
+        {0},
+    };
+    int ret = 0;
+    int port = 0;
+    char *hostname = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+
+    if (!frame || !this || !data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=hostname", NULL);
+        goto out;
+    }
+    ret = dict_get_int32n(dict, "port", SLEN("port"), &port);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_DEBUG, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=port", NULL);
+        port = GF_DEFAULT_BASE_PORT;
+    }
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+    req.hostname = gf_strdup(hostname);
+    req.port = port;
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, peerinfo->peer, GLUSTERD_PROBE_QUERY, NULL,
+        this, glusterd_probe_cbk, (xdrproc_t)xdr_gd1_mgmt_probe_req);
 
 out:
-        GF_FREE (req.hostname);
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.hostname);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-glusterd_rpc_friend_add (call_frame_t *frame, xlator_t *this,
-                         void *data)
+glusterd_rpc_friend_add(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_friend_req         req       = {{0},};
-        int                         ret       = 0;
-        glusterd_peerinfo_t        *peerinfo  = NULL;
-        glusterd_conf_t            *priv      = NULL;
-        glusterd_friend_sm_event_t *event     = NULL;
-        dict_t                     *peer_data = NULL;
-
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        event = data;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                rcu_read_unlock ();
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
-        }
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-        req.hostname = gf_strdup (peerinfo->hostname);
-        req.port = peerinfo->port;
-
-        rcu_read_unlock ();
-
-        ret = glusterd_add_volumes_to_export_dict (&peer_data);
+    gd1_mgmt_friend_req req = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    dict_t *peer_data = NULL;
+
+    if (!frame || !this || !data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    event = data;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    req.hostname = gf_strdup(peerinfo->hostname);
+    req.port = peerinfo->port;
+
+    RCU_READ_UNLOCK;
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+
+    peer_data = dict_new();
+    if (!peer_data) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(peer_data, "hostname_in_cluster",
+                                     peerinfo->hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Unable to add hostname of the peer");
+        goto out;
+    }
+
+    if (priv->op_version >= GD_OP_VERSION_3_6_0) {
+        ret = glusterd_add_missed_snaps_to_export_dict(peer_data);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to add list of volumes "
-                        "in the peer_data dict for handshake");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Unable to add list of missed snapshots "
+                   "in the peer_data dict for handshake");
+            goto out;
         }
 
-        ret = dict_set_dynstr_with_alloc (peer_data,
-                                          "hostname_in_cluster",
-                                          peerinfo->hostname);
+        ret = glusterd_add_snapshots_to_export_dict(peer_data);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to add hostname of the peer");
-                goto out;
-        }
-
-        if (priv->op_version >= GD_OP_VERSION_3_6_0) {
-                ret = glusterd_add_missed_snaps_to_export_dict (peer_data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
-                                "Unable to add list of missed snapshots "
-                                "in the peer_data dict for handshake");
-                        goto out;
-                }
-
-                ret = glusterd_add_snapshots_to_export_dict (peer_data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_LIST_SET_FAIL,
-                                "Unable to add list of snapshots "
-                                "in the peer_data dict for handshake");
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_LIST_SET_FAIL,
+                   "Unable to add list of snapshots "
+                   "in the peer_data dict for handshake");
+            goto out;
+        }
+    }
+
+    /* Don't add any key-value in peer_data dictionary after call this function
+     */
+    ret = glusterd_add_volumes_to_export_dict(peer_data, &req.vols.vols_val,
+                                              &req.vols.vols_len);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to add list of volumes "
+               "in the peer_data dict for handshake");
+        goto out;
+    }
+
+    if (!req.vols.vols_len) {
+        ret = dict_allocate_and_serialize(peer_data, &req.vols.vols_val,
+                                          &req.vols.vols_len);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+            goto out;
         }
+    }
 
-        ret = dict_allocate_and_serialize (peer_data, &req.vols.vols_val,
-                                           &req.vols.vols_len);
-        if (ret)
-                goto out;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
-                                       GLUSTERD_FRIEND_ADD,
-                                       NULL, this, glusterd_friend_add_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_friend_req);
-
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, peerinfo->peer, GLUSTERD_FRIEND_ADD, NULL,
+        this, glusterd_friend_add_cbk, (xdrproc_t)xdr_gd1_mgmt_friend_req);
 
 out:
-        GF_FREE (req.vols.vols_val);
-        GF_FREE (req.hostname);
+    GF_FREE(req.vols.vols_val);
+    GF_FREE(req.hostname);
 
-        if (peer_data)
-                dict_unref (peer_data);
+    if (peer_data)
+        dict_unref(peer_data);
 
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_rpc_friend_remove (call_frame_t *frame, xlator_t *this,
-                            void *data)
+glusterd_rpc_friend_remove(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_friend_req             req = {{0},};
-        int                             ret = 0;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        event = data;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                rcu_read_unlock ();
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
-        }
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-        req.hostname = gf_strdup (peerinfo->hostname);
-        req.port = peerinfo->port;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
-                                       GLUSTERD_FRIEND_REMOVE, NULL,
-                                       this, glusterd_friend_remove_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_friend_req);
-
-        rcu_read_unlock ();
+    gd1_mgmt_friend_req req = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    event = data;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+    req.hostname = gf_strdup(peerinfo->hostname);
+    req.port = peerinfo->port;
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->peer,
+                                  GLUSTERD_FRIEND_REMOVE, NULL, this,
+                                  glusterd_friend_remove_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_friend_req);
+
+    RCU_READ_UNLOCK;
 out:
-        GF_FREE (req.hostname);
+    GF_FREE(req.hostname);
 
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-glusterd_rpc_friend_update (call_frame_t *frame, xlator_t *this,
-                            void *data)
+glusterd_rpc_friend_update(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_friend_update  req         = {{0},};
-        int                     ret         = 0;
-        glusterd_conf_t        *priv        = NULL;
-        dict_t                 *friends     = NULL;
-        call_frame_t           *dummy_frame = NULL;
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        friends = data;
-        if (!friends)
-                goto out;
-
-        ret = dict_get_ptr (friends, "peerinfo", VOID(&peerinfo));
-        if (ret)
-                goto out;
-        /* Don't want to send the pointer over */
-        dict_del (friends, "peerinfo");
-
-        ret = dict_allocate_and_serialize (friends, &req.friends.friends_val,
-                                           &req.friends.friends_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-
-        dummy_frame = create_frame (this, this->ctx->pool);
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->peer,
-                                       GLUSTERD_FRIEND_UPDATE, NULL,
-                                       this, glusterd_friend_update_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_friend_update);
+    gd1_mgmt_friend_update req = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    call_frame_t *dummy_frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    friends = data;
+    if (!friends)
+        goto out;
+
+    ret = dict_get_ptr(friends, "peerinfo", VOID(&peerinfo));
+    if (ret)
+        goto out;
+    /* Don't want to send the pointer over */
+    dict_deln(friends, "peerinfo", SLEN("peerinfo"));
+
+    ret = dict_allocate_and_serialize(friends, &req.friends.friends_val,
+                                      &req.friends.friends_len);
+    if (ret)
+        goto out;
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+
+    dummy_frame = create_frame(this, this->ctx->pool);
+    ret = glusterd_submit_request(peerinfo->rpc, &req, dummy_frame,
+                                  peerinfo->peer, GLUSTERD_FRIEND_UPDATE, NULL,
+                                  this, glusterd_friend_update_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_friend_update);
 
 out:
-        GF_FREE (req.friends.friends_val);
+    GF_FREE(req.friends.friends_val);
 
-        if (ret && dummy_frame)
-                STACK_DESTROY (dummy_frame->root);
+    if (ret && dummy_frame)
+        STACK_DESTROY(dummy_frame->root);
 
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_cluster_lock (call_frame_t *frame, xlator_t *this,
-                          void *data)
+glusterd_cluster_lock(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_cluster_lock_req       req = {{0},};
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        call_frame_t                    *dummy_frame = NULL;
+    gd1_mgmt_cluster_lock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    call_frame_t *dummy_frame = NULL;
 
-        if (!this)
-                goto out;
+    if (!this)
+        goto out;
 
-        peerinfo = data;
+    peerinfo = data;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        glusterd_get_uuid (&req.uuid);
+    glusterd_get_uuid(&req.uuid);
 
-        dummy_frame = create_frame (this, this->ctx->pool);
-        if (!dummy_frame)
-                goto out;
+    dummy_frame = create_frame(this, this->ctx->pool);
+    if (!dummy_frame)
+        goto out;
 
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_LOCK,
-                                       NULL,
-                                       this, glusterd_cluster_lock_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+    ret = glusterd_submit_request(peerinfo->rpc, &req, dummy_frame,
+                                  peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_LOCK,
+                                  NULL, this, glusterd_cluster_lock_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
 
-        if (ret && dummy_frame)
-                STACK_DESTROY (dummy_frame->root);
-        return ret;
+    if (ret && dummy_frame)
+        STACK_DESTROY(dummy_frame->root);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_lock_peers (call_frame_t *frame, xlator_t *this,
-                             void *data)
+glusterd_mgmt_v3_lock_peers(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_v3_lock_req             req         = {{0},};
-        int                              ret         = -1;
-        glusterd_peerinfo_t             *peerinfo    = NULL;
-        glusterd_conf_t                 *priv        = NULL;
-        dict_t                          *dict        = NULL;
-        uuid_t                          *txn_id      = NULL;
-
-        if (!this)
-                goto out;
-
-        dict = data;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL, "Failed to serialize dict "
-                        "to request buffer");
-                goto out;
-        }
-
-        /* Sending valid transaction ID to peers */
-        ret = dict_get_bin (dict, "transaction_id",
-                            (void **)&txn_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_ID_GET_FAIL,
-                       "Failed to get transaction id.");
-                goto out;
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Transaction_id = %s", uuid_utoa (*txn_id));
-                gf_uuid_copy (req.txn_id, *txn_id);
-        }
-
-        if (!frame)
-                frame = create_frame (this, this->ctx->pool);
-
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-        frame->cookie = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!frame->cookie) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (frame->cookie, req.txn_id);
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       peerinfo->mgmt_v3,
-                                       GLUSTERD_MGMT_V3_LOCK, NULL,
-                                       this, glusterd_mgmt_v3_lock_peers_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+    gd1_mgmt_v3_lock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this)
+        goto out;
+
+    dict = data;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+        gf_uuid_copy(req.txn_id, *txn_id);
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, req.txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt_v3,
+                                  GLUSTERD_MGMT_V3_LOCK, NULL, this,
+                                  glusterd_mgmt_v3_lock_peers_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    if (dict)
+        dict_unref(dict);
+    if (req.dict.dict_val)
+        GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_unlock_peers (call_frame_t *frame, xlator_t *this,
-                   void *data)
+glusterd_mgmt_v3_unlock_peers(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_v3_unlock_req           req         = {{0},};
-        int                              ret         = -1;
-        glusterd_peerinfo_t             *peerinfo    = NULL;
-        glusterd_conf_t                 *priv        = NULL;
-        dict_t                          *dict        = NULL;
-        uuid_t                          *txn_id      = NULL;
-
-        if (!this)
-                goto out;
-
-        dict = data;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "Failed to serialize dict "
-                        "to request buffer");
-                goto out;
-        }
-
-        /* Sending valid transaction ID to peers */
-        ret = dict_get_bin (dict, "transaction_id",
-                            (void **)&txn_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_ID_GET_FAIL,
-                       "Failed to get transaction id.");
-                goto out;
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Transaction_id = %s", uuid_utoa (*txn_id));
-                gf_uuid_copy (req.txn_id, *txn_id);
-        }
-
-        if (!frame)
-                frame = create_frame (this, this->ctx->pool);
-
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-        frame->cookie = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!frame->cookie) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (frame->cookie, req.txn_id);
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       peerinfo->mgmt_v3,
-                                       GLUSTERD_MGMT_V3_UNLOCK, NULL,
-                                       this, glusterd_mgmt_v3_unlock_peers_cbk,
-                                       (xdrproc_t)
-                                       xdr_gd1_mgmt_v3_unlock_req);
+    gd1_mgmt_v3_unlock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this)
+        goto out;
+
+    dict = data;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+        gf_uuid_copy(req.txn_id, *txn_id);
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, req.txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt_v3,
+                                  GLUSTERD_MGMT_V3_UNLOCK, NULL, this,
+                                  glusterd_mgmt_v3_unlock_peers_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    if (dict)
+        dict_unref(dict);
+
+    if (req.dict.dict_val)
+        GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
 int32_t
-glusterd_cluster_unlock (call_frame_t *frame, xlator_t *this,
-                            void *data)
+glusterd_cluster_unlock(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_cluster_lock_req       req = {{0},};
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        call_frame_t                    *dummy_frame = NULL;
-
-        if (!this ) {
-                ret = -1;
-                goto out;
-        }
-        peerinfo = data;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        glusterd_get_uuid (&req.uuid);
-
-        dummy_frame = create_frame (this, this->ctx->pool);
-        if (!dummy_frame)
-                goto out;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_UNLOCK,
-                                       NULL,
-                                       this, glusterd_cluster_unlock_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+    gd1_mgmt_cluster_lock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    call_frame_t *dummy_frame = NULL;
+
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+    peerinfo = data;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    glusterd_get_uuid(&req.uuid);
+
+    dummy_frame = create_frame(this, this->ctx->pool);
+    if (!dummy_frame)
+        goto out;
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, dummy_frame,
+                                  peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                  NULL, this, glusterd_cluster_unlock_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
 out:
-        gf_msg_debug (this ? this->name : "glusterd", 0, "Returning %d", ret);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
 
-        if (ret && dummy_frame)
-                STACK_DESTROY (dummy_frame->root);
+    if (ret && dummy_frame)
+        STACK_DESTROY(dummy_frame->root);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_stage_op (call_frame_t *frame, xlator_t *this,
-                      void *data)
+glusterd_stage_op(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_stage_op_req           req = {{0,},};
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        dict_t                          *dict = NULL;
-        gf_boolean_t                    is_alloc = _gf_true;
-        uuid_t                          *txn_id      = NULL;
-
-        if (!this) {
-                goto out;
-        }
-
-        dict = data;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-        req.op = glusterd_op_get_op ();
-
-        ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           &req.buf.buf_len);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "Failed to serialize dict "
-                        "to request buffer");
-                goto out;
-        }
-        /* Sending valid transaction ID to peers */
-        ret = dict_get_bin (dict, "transaction_id",
-                            (void **)&txn_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_ID_GET_FAIL,
-                       "Failed to get transaction id.");
-                goto out;
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Transaction_id = %s", uuid_utoa (*txn_id));
-        }
-
-        if (!frame)
-                frame = create_frame (this, this->ctx->pool);
-
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-        frame->cookie = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!frame->cookie) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (frame->cookie, *txn_id);
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_STAGE_OP,
-                                       NULL,
-                                       this, glusterd_stage_op_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+    gd1_mgmt_stage_op_req req = {
+        {
+            0,
+        },
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this) {
+        goto out;
+    }
+
+    dict = data;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+    req.op = glusterd_op_get_op();
+
+    ret = dict_allocate_and_serialize(dict, &req.buf.buf_val, &req.buf.buf_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, *txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt,
+                                  GLUSTERD_MGMT_STAGE_OP, NULL, this,
+                                  glusterd_stage_op_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
 
 out:
-        if ((_gf_true == is_alloc) && req.buf.buf_val)
-                GF_FREE (req.buf.buf_val);
+    if (req.buf.buf_val)
+        GF_FREE(req.buf.buf_val);
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_commit_op (call_frame_t *frame, xlator_t *this,
-                      void *data)
+glusterd_commit_op(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_commit_op_req  req         = {{0,},};
-        int                     ret         = -1;
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-        glusterd_conf_t        *priv        = NULL;
-        call_frame_t           *dummy_frame = NULL;
-        dict_t                 *dict        = NULL;
-        gf_boolean_t            is_alloc    = _gf_true;
-        uuid_t                 *txn_id      = NULL;
-
-        if (!this) {
-                goto out;
-        }
-
-        dict = data;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-        req.op = glusterd_op_get_op ();
-
-        ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           &req.buf.buf_len);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SERL_LENGTH_GET_FAIL,
-                        "Failed to serialize dict to "
-                        "request buffer");
-                goto out;
-        }
-        /* Sending valid transaction ID to peers */
-        ret = dict_get_bin (dict, "transaction_id",
-                            (void **)&txn_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TRANS_ID_GET_FAIL,
-                       "Failed to get transaction id.");
-                goto out;
-        } else {
-                gf_msg_debug (this->name, 0,
-                        "Transaction_id = %s", uuid_utoa (*txn_id));
-        }
-
-        if (!frame)
-                frame = create_frame (this, this->ctx->pool);
-
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-        frame->cookie = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!frame->cookie) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (frame->cookie, *txn_id);
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_COMMIT_OP,
-                                       NULL,
-                                       this, glusterd_commit_op_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+    gd1_mgmt_commit_op_req req = {
+        {
+            0,
+        },
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this) {
+        goto out;
+    }
+
+    dict = data;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+    req.op = glusterd_op_get_op();
+
+    ret = dict_allocate_and_serialize(dict, &req.buf.buf_val, &req.buf.buf_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, *txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt,
+                                  GLUSTERD_MGMT_COMMIT_OP, NULL, this,
+                                  glusterd_commit_op_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
 
 out:
-        if ((_gf_true == is_alloc) && req.buf.buf_val)
-                GF_FREE (req.buf.buf_val);
+    if (req.buf.buf_val)
+        GF_FREE(req.buf.buf_val);
 
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-__glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+__glusterd_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gd1_mgmt_brick_op_rsp         rsp   = {0};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        call_frame_t                  *frame = NULL;
-        glusterd_op_brick_rsp_ctx_t   *ev_ctx = NULL;
-        dict_t                        *dict = NULL;
-        int                           index = 0;
-        glusterd_req_ctx_t            *req_ctx = NULL;
-        glusterd_pending_node_t       *node = NULL;
-        xlator_t                      *this = NULL;
-        uuid_t                        *txn_id = NULL;
-        glusterd_conf_t               *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-        frame = myframe;
-        req_ctx = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("error");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret =  xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    gd1_mgmt_brick_op_rsp rsp = {0};
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    call_frame_t *frame = NULL;
+    glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+    dict_t *dict = NULL;
+    int index = 0;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    glusterd_pending_node_t *node = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+    frame = myframe;
+    req_ctx = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup("error");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode brick op "
+               "response received");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        rsp.op_errstr = strdup("Unable to decode brick op response");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    if (rsp.output.output_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.output.output_val, rsp.output.output_len,
+                               &dict);
         if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_RES_DECODE_FAIL,
-                        "Failed to decode brick op "
-                        "response received");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                rsp.op_errstr = strdup ("Unable to decode brick op response");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        if (rsp.output.output_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.output.output_val,
-                                        rsp.output.output_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL,
-                                "Failed to "
-                                "unserialize rsp-buffer to dictionary");
-			event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = rsp.output.output_val;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to "
+                   "unserialize rsp-buffer to dictionary");
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
+        } else {
+            dict->extra_stdfree = rsp.output.output_val;
         }
+    }
 
-        op_ret = rsp.op_ret;
+    op_ret = rsp.op_ret;
 
-        /* Add index to rsp_dict for GD_OP_STATUS_VOLUME */
-        if (GD_OP_STATUS_VOLUME == req_ctx->op) {
-                node = frame->cookie;
-                index = node->index;
-                ret = dict_set_int32 (dict, "index", index);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Error setting index on brick status rsp dict");
-                        rsp.op_ret = -1;
-                        event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                }
+    /* Add index to rsp_dict for GD_OP_STATUS_VOLUME */
+    if (GD_OP_STATUS_VOLUME == req_ctx->op) {
+        node = frame->cookie;
+        index = node->index;
+        ret = dict_set_int32n(dict, "index", SLEN("index"), index);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Error setting index on brick status rsp dict");
+            rsp.op_ret = -1;
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
         }
+    }
 out:
 
-        if (req_ctx && req_ctx->dict) {
-                ret = dict_get_bin (req_ctx->dict, "transaction_id",
-                                    (void **)&txn_id);
-                gf_msg_debug (this->name, 0,
-                        "transaction ID = %s", uuid_utoa (*txn_id));
-        }
+    if (req_ctx && req_ctx->dict) {
+        ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+        gf_msg_debug(this->name, -ret, "transaction ID = %s",
+                     uuid_utoa(*txn_id));
+    }
 
-        ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
-        GF_ASSERT (ev_ctx);
+    ev_ctx = GF_CALLOC(1, sizeof(*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
+    if (ev_ctx) {
         if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                ev_ctx->op_ret = op_ret;
-                ev_ctx->op_errstr = gf_strdup(rsp.op_errstr);
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            ev_ctx->op_ret = op_ret;
+            ev_ctx->op_errstr = gf_strdup(rsp.op_errstr);
         } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
+            event_type = GD_OP_EVENT_RCVD_ACC;
         }
         ev_ctx->pending_node = frame->cookie;
-        ev_ctx->rsp_dict  = dict;
+        ev_ctx->rsp_dict = dict;
         ev_ctx->commit_ctx = frame->local;
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, ev_ctx);
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
+        ret = glusterd_op_sm_inject_event(event_type, txn_id, ev_ctx);
+    }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    if (ret) {
+        if (dict) {
+            dict_unref(dict);
         }
-
-        if (ret && dict)
-                dict_unref (dict);
-        free (rsp.op_errstr); //malloced by xdr
-        GLUSTERD_STACK_DESTROY (frame);
-        return ret;
+        if (ev_ctx) {
+            GF_FREE(ev_ctx->op_errstr);
+            GF_FREE(ev_ctx);
+        }
+    }
+    free(rsp.op_errstr);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(frame);
+    return ret;
 }
 
 int32_t
-glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+glusterd_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_brick_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_brick_op_cbk);
 }
 
 int32_t
-glusterd_brick_op (call_frame_t *frame, xlator_t *this,
-                   void *data)
+glusterd_brick_op(call_frame_t *frame, xlator_t *this, void *data)
 {
-
-        gd1_mgmt_brick_op_req           *req = NULL;
-        int                             ret = 0;
-        int                             ret1 = 0;
-        glusterd_conf_t                 *priv = NULL;
-        call_frame_t                    *dummy_frame = NULL;
-        char                            *op_errstr = NULL;
-        int                             pending_bricks = 0;
-        glusterd_pending_node_t         *pending_node;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        struct rpc_clnt                 *rpc = NULL;
-        dict_t                          *op_ctx = NULL;
-        uuid_t                          *txn_id = NULL;
-
-        if (!this) {
-                ret = -1;
-                goto out;
-        }
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        txn_id = &priv->global_txn_id;
-
-        req_ctx = data;
-        GF_ASSERT (req_ctx);
-        CDS_INIT_LIST_HEAD (&opinfo.pending_bricks);
-
-        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
-        gf_msg_debug (this->name, 0, "transaction ID = %s",
-                uuid_utoa (*txn_id));
-
-        ret = glusterd_op_bricks_select (req_ctx->op, req_ctx->dict, &op_errstr,
-                                         &opinfo.pending_bricks, NULL);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_SELECT_FAIL, "Failed to select bricks "
-                        "while performing brick op during 'Volume %s'",
-                        gd_op_list[opinfo.op]);
-                opinfo.op_errstr = op_errstr;
-                goto out;
-        }
-
-        cds_list_for_each_entry (pending_node, &opinfo.pending_bricks, list) {
-                dummy_frame = create_frame (this, this->ctx->pool);
-                if (!dummy_frame)
-                        continue;
-
-                if ((pending_node->type == GD_NODE_NFS) ||
-                    (pending_node->type == GD_NODE_QUOTAD) ||
-                    (pending_node->type == GD_NODE_SNAPD) ||
-                    (pending_node->type == GD_NODE_SCRUB) ||
-                    ((pending_node->type == GD_NODE_SHD) &&
-                     (req_ctx->op == GD_OP_STATUS_VOLUME)))
-                        ret = glusterd_node_op_build_payload
-                                (req_ctx->op,
-                                 (gd1_mgmt_brick_op_req **)&req,
-                                 req_ctx->dict);
-                else {
-                        ret = glusterd_brick_op_build_payload
-                                (req_ctx->op, pending_node->node,
-                                 (gd1_mgmt_brick_op_req **)&req,
-                                 req_ctx->dict);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
-                                        "Failed to "
-                                        "build brick op payload during "
-                                        "'Volume %s'", gd_op_list[req_ctx->op]);
-                                goto out;
-                        }
-                }
-
-                dummy_frame->local = data;
-                dummy_frame->cookie = pending_node;
-
-                rpc = glusterd_pending_node_get_rpc (pending_node);
-                if (!rpc) {
-                        if (pending_node->type == GD_NODE_REBALANCE) {
-                                opinfo.brick_pending_count = 0;
-                                ret = 0;
-                                if (req) {
-                                        GF_FREE (req->input.input_val);
-                                        GF_FREE (req);
-                                        req = NULL;
-                                }
-                                GLUSTERD_STACK_DESTROY (dummy_frame);
-
-                                op_ctx = glusterd_op_get_ctx ();
-                                if (!op_ctx)
-                                        goto out;
-                                glusterd_defrag_volume_node_rsp (req_ctx->dict,
-                                                                 NULL, op_ctx);
-
-                                goto out;
-                        }
-
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_RPC_FAILURE, "Brick Op failed "
-                                "due to rpc failure.");
-                        goto out;
-                }
-
-                ret = glusterd_submit_request (rpc, req, dummy_frame,
-                                               priv->gfs_mgmt,
-                                               req->op, NULL,
-                                               this, glusterd_brick_op_cbk,
-                                               (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-                if (req) {
-                        GF_FREE (req->input.input_val);
-                        GF_FREE (req);
-                        req = NULL;
-                }
-                if (!ret)
-                        pending_bricks++;
-
-                glusterd_pending_node_put_rpc (pending_node);
-        }
-
-        gf_msg_trace (this->name, 0, "Sent brick op req for operation "
-                "'Volume %s' to %d bricks", gd_op_list[req_ctx->op],
-                pending_bricks);
-        opinfo.brick_pending_count = pending_bricks;
+    gd1_mgmt_brick_op_req *req = NULL;
+    int ret = 0;
+    int ret1 = 0;
+    glusterd_conf_t *priv = NULL;
+    call_frame_t *dummy_frame = NULL;
+    char *op_errstr = NULL;
+    int pending_bricks = 0;
+    glusterd_pending_node_t *pending_node;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *op_ctx = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    txn_id = &priv->global_txn_id;
+
+    req_ctx = data;
+    GF_ASSERT(req_ctx);
+    CDS_INIT_LIST_HEAD(&opinfo.pending_bricks);
+
+    ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_SELECT_FAIL,
+               "Could not get transaction ID from dict, global"
+               "transaction ID = %s",
+               uuid_utoa(*txn_id));
+    } else {
+        gf_msg_debug(this->name, 0, "transaction ID = %s", uuid_utoa(*txn_id));
+    }
+    ret = glusterd_op_bricks_select(req_ctx->op, req_ctx->dict, &op_errstr,
+                                    &opinfo.pending_bricks, NULL);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SELECT_FAIL,
+               "Failed to select bricks "
+               "while performing brick op during 'Volume %s'",
+               gd_op_list[opinfo.op]);
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    cds_list_for_each_entry(pending_node, &opinfo.pending_bricks, list)
+    {
+        dummy_frame = create_frame(this, this->ctx->pool);
+        if (!dummy_frame)
+            continue;
+
+        if ((pending_node->type == GD_NODE_NFS) ||
+            (pending_node->type == GD_NODE_QUOTAD) ||
+            (pending_node->type == GD_NODE_SNAPD) ||
+            (pending_node->type == GD_NODE_SCRUB) ||
+            ((pending_node->type == GD_NODE_SHD) &&
+             (req_ctx->op == GD_OP_STATUS_VOLUME))) {
+            ret = glusterd_node_op_build_payload(
+                req_ctx->op, (gd1_mgmt_brick_op_req **)&req, req_ctx->dict);
+        } else {
+            ret = glusterd_brick_op_build_payload(
+                req_ctx->op, pending_node->node, (gd1_mgmt_brick_op_req **)&req,
+                req_ctx->dict);
+        }
+        if (ret || !req) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+                   "Failed to "
+                   "build op payload during "
+                   "'Volume %s'",
+                   gd_op_list[req_ctx->op]);
+            goto out;
+        }
+
+        dummy_frame->local = data;
+        dummy_frame->cookie = pending_node;
+
+        rpc = glusterd_pending_node_get_rpc(pending_node);
+        if (!rpc) {
+            if (pending_node->type == GD_NODE_REBALANCE) {
+                opinfo.brick_pending_count = 0;
+                ret = 0;
+                GF_FREE(req->input.input_val);
+                GF_FREE(req);
+                req = NULL;
+                GLUSTERD_STACK_DESTROY(dummy_frame);
+
+                op_ctx = glusterd_op_get_ctx();
+                if (!op_ctx)
+                    goto out;
+                glusterd_defrag_volume_node_rsp(req_ctx->dict, NULL, op_ctx);
+
+                goto out;
+            }
+
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE,
+                   "Brick Op failed "
+                   "due to rpc failure.");
+            goto out;
+        }
+
+        ret = glusterd_submit_request(
+            rpc, req, dummy_frame, priv->gfs_mgmt, req->op, NULL, this,
+            glusterd_brick_op_cbk, (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        GF_FREE(req->input.input_val);
+        GF_FREE(req);
+        req = NULL;
+
+        if (!ret)
+            pending_bricks++;
+
+        glusterd_pending_node_put_rpc(pending_node);
+    }
+
+    gf_msg_trace(this->name, 0,
+                 "Sent brick op req for operation "
+                 "'Volume %s' to %d bricks",
+                 gd_op_list[req_ctx->op], pending_bricks);
+    opinfo.brick_pending_count = pending_bricks;
 
 out:
 
-        if (ret)
-                opinfo.op_ret = ret;
+    if (ret)
+        opinfo.op_ret = ret;
 
-        ret1 = glusterd_set_txn_opinfo (txn_id, &opinfo);
-        if (ret1)
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TRANS_OPINFO_SET_FAIL,
-                                "Unable to set "
-                                "transaction's opinfo");
+    ret1 = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret1)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                             txn_id, data);
-                opinfo.op_ret = ret;
-        }
+    if (ret) {
+        glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, txn_id, data);
+        opinfo.op_ret = ret;
+    }
 
-        gf_msg_debug (this ? this->name : "glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 struct rpc_clnt_procedure gd_brick_actors[GLUSTERD_BRICK_MAXVALUE] = {
-        [GLUSTERD_BRICK_NULL] = {"NULL", NULL },
-        [GLUSTERD_BRICK_OP]   = {"BRICK_OP", glusterd_brick_op},
+    [GLUSTERD_BRICK_NULL] = {"NULL", NULL},
+    [GLUSTERD_BRICK_OP] = {"BRICK_OP", glusterd_brick_op},
 };
 
 struct rpc_clnt_procedure gd_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
-        [GLUSTERD_FRIEND_NULL]   = {"NULL", NULL },
-        [GLUSTERD_PROBE_QUERY]   = {"PROBE_QUERY", glusterd_rpc_probe},
-        [GLUSTERD_FRIEND_ADD]    = {"FRIEND_ADD", glusterd_rpc_friend_add},
-        [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd_rpc_friend_remove},
-        [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_rpc_friend_update},
+    [GLUSTERD_FRIEND_NULL] = {"NULL", NULL},
+    [GLUSTERD_PROBE_QUERY] = {"PROBE_QUERY", glusterd_rpc_probe},
+    [GLUSTERD_FRIEND_ADD] = {"FRIEND_ADD", glusterd_rpc_friend_add},
+    [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd_rpc_friend_remove},
+    [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_rpc_friend_update},
 };
 
 struct rpc_clnt_procedure gd_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
-        [GLUSTERD_MGMT_NULL]           = {"NULL", NULL },
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = {"CLUSTER_LOCK", glusterd_cluster_lock},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK", glusterd_cluster_unlock},
-        [GLUSTERD_MGMT_STAGE_OP]       = {"STAGE_OP", glusterd_stage_op},
-        [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd_commit_op},
+    [GLUSTERD_MGMT_NULL] = {"NULL", NULL},
+    [GLUSTERD_MGMT_CLUSTER_LOCK] = {"CLUSTER_LOCK", glusterd_cluster_lock},
+    [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK",
+                                      glusterd_cluster_unlock},
+    [GLUSTERD_MGMT_STAGE_OP] = {"STAGE_OP", glusterd_stage_op},
+    [GLUSTERD_MGMT_COMMIT_OP] = {"COMMIT_OP", glusterd_commit_op},
 };
 
 struct rpc_clnt_procedure gd_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
-        [GLUSTERD_MGMT_V3_NULL]    = {"NULL", NULL },
-        [GLUSTERD_MGMT_V3_LOCK]    = {"MGMT_V3_LOCK", glusterd_mgmt_v3_lock_peers},
-        [GLUSTERD_MGMT_V3_UNLOCK]  = {"MGMT_V3_UNLOCK", glusterd_mgmt_v3_unlock_peers},
+    [GLUSTERD_MGMT_V3_NULL] = {"NULL", NULL},
+    [GLUSTERD_MGMT_V3_LOCK] = {"MGMT_V3_LOCK", glusterd_mgmt_v3_lock_peers},
+    [GLUSTERD_MGMT_V3_UNLOCK] = {"MGMT_V3_UNLOCK",
+                                 glusterd_mgmt_v3_unlock_peers},
 };
 
 struct rpc_clnt_program gd_mgmt_prog = {
-        .progname  = "glusterd mgmt",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_VERSION,
-        .proctable = gd_mgmt_actors,
-        .numproc   = GLUSTERD_MGMT_MAXVALUE,
+    .progname = "glusterd mgmt",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_VERSION,
+    .proctable = gd_mgmt_actors,
+    .numproc = GLUSTERD_MGMT_MAXVALUE,
 };
 
 struct rpc_clnt_program gd_brick_prog = {
-        .progname  = "brick operations",
-        .prognum   = GD_BRICK_PROGRAM,
-        .progver   = GD_BRICK_VERSION,
-        .proctable = gd_brick_actors,
-        .numproc   = GLUSTERD_BRICK_MAXVALUE,
+    .progname = "brick operations",
+    .prognum = GD_BRICK_PROGRAM,
+    .progver = GD_BRICK_VERSION,
+    .proctable = gd_brick_actors,
+    .numproc = GLUSTERD_BRICK_MAXVALUE,
 };
 
 struct rpc_clnt_program gd_peer_prog = {
-        .progname  = "Peer mgmt",
-        .prognum   = GD_FRIEND_PROGRAM,
-        .progver   = GD_FRIEND_VERSION,
-        .proctable = gd_peer_actors,
-        .numproc   = GLUSTERD_FRIEND_MAXVALUE,
+    .progname = "Peer mgmt",
+    .prognum = GD_FRIEND_PROGRAM,
+    .progver = GD_FRIEND_VERSION,
+    .proctable = gd_peer_actors,
+    .numproc = GLUSTERD_FRIEND_MAXVALUE,
 };
 
 struct rpc_clnt_program gd_mgmt_v3_prog = {
-        .progname  = "glusterd mgmt v3",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_V3_VERSION,
-        .proctable = gd_mgmt_v3_actors,
-        .numproc   = GLUSTERD_MGMT_V3_MAXVALUE,
+    .progname = "glusterd mgmt v3",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_V3_VERSION,
+    .proctable = gd_mgmt_v3_actors,
+    .numproc = GLUSTERD_MGMT_V3_MAXVALUE,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
index 3761dbadfd1..c49a0eefba5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
@@ -8,8 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
@@ -19,189 +19,189 @@
 char *scrub_svc_name = "scrub";
 
 void
-glusterd_scrubsvc_build (glusterd_svc_t *svc)
+glusterd_scrubsvc_build(glusterd_svc_t *svc)
 {
-        svc->manager = glusterd_scrubsvc_manager;
-        svc->start = glusterd_scrubsvc_start;
-        svc->stop = glusterd_scrubsvc_stop;
+    svc->manager = glusterd_scrubsvc_manager;
+    svc->start = glusterd_scrubsvc_start;
+    svc->stop = glusterd_scrubsvc_stop;
 }
 
 int
-glusterd_scrubsvc_init (glusterd_svc_t *svc)
+glusterd_scrubsvc_init(glusterd_svc_t *svc)
 {
-        return glusterd_svc_init (svc, scrub_svc_name);
+    return glusterd_svc_init(svc, scrub_svc_name);
 }
 
 static int
-glusterd_scrubsvc_create_volfile ()
+glusterd_scrubsvc_create_volfile()
 {
-        char              filepath[PATH_MAX] = {0,};
-        int               ret                = -1;
-        glusterd_conf_t   *conf              = NULL;
-        xlator_t          *this              = NULL;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        glusterd_svc_build_volfile_path (scrub_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
-
-        ret = glusterd_create_global_volfile (build_scrub_graph,
-                                              filepath, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "Failed to create volfile");
-                goto out;
-        }
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    glusterd_svc_build_volfile_path(scrub_svc_name, conf->workdir, filepath,
+                                    sizeof(filepath));
+
+    ret = glusterd_create_global_volfile(build_scrub_graph, filepath, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_scrubsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_scrubsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int          ret    = -EINVAL;
-
-        if (!svc->inited) {
-                ret = glusterd_scrubsvc_init (svc);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SCRUB_INIT_FAIL, "Failed to init "
-                                "scrub service");
-                        goto out;
-                } else {
-                        svc->inited = _gf_true;
-                        gf_msg_debug (THIS->name, 0, "scrub service "
-                                      "initialized");
-                }
-        }
+    int ret = -EINVAL;
 
-        if (glusterd_should_i_stop_bitd ()) {
-                ret = svc->stop (svc, SIGTERM);
+    if (!svc->inited) {
+        ret = glusterd_scrubsvc_init(svc);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SCRUB_INIT_FAIL,
+                   "Failed to init "
+                   "scrub service");
+            goto out;
         } else {
-                ret = glusterd_scrubsvc_create_volfile ();
-                if (ret)
-                        goto out;
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0,
+                         "scrub service "
+                         "initialized");
+        }
+    }
+
+    if (glusterd_should_i_stop_bitd()) {
+        ret = svc->stop(svc, SIGTERM);
+    } else {
+        ret = glusterd_scrubsvc_create_volfile();
+        if (ret)
+            goto out;
 
-                ret = svc->stop (svc, SIGKILL);
-                if (ret)
-                        goto out;
+        ret = svc->stop(svc, SIGKILL);
+        if (ret)
+            goto out;
 
-                ret = svc->start (svc, flags);
-                if (ret)
-                        goto out;
+        ret = svc->start(svc, flags);
+        if (ret)
+            goto out;
 
-                ret = glusterd_conn_connect (&(svc->conn));
-                if (ret)
-                        goto out;
-        }
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_scrubsvc_start (glusterd_svc_t *svc, int flags)
+glusterd_scrubsvc_start(glusterd_svc_t *svc, int flags)
 {
-        int ret = -1;
-        dict_t *cmdict = NULL;
+    int ret = -1;
+    dict_t *cmdict = NULL;
 
-        cmdict = dict_new ();
-        if (!cmdict)
-                goto error_return;
+    cmdict = dict_new();
+    if (!cmdict) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto error_return;
+    }
 
-        ret = dict_set_str (cmdict, "cmdarg0", "--global-timer-wheel");
-        if (ret)
-                goto dealloc_dict;
+    ret = dict_set_str(cmdict, "cmdarg0", "--global-timer-wheel");
+    if (ret)
+        goto dealloc_dict;
 
-        ret = glusterd_svc_start (svc, flags, cmdict);
+    ret = glusterd_svc_start(svc, flags, cmdict);
 
- dealloc_dict:
-        dict_unref (cmdict);
- error_return:
-        return ret;
+dealloc_dict:
+    dict_unref(cmdict);
+error_return:
+    return ret;
 }
 
 int
-glusterd_scrubsvc_stop (glusterd_svc_t *svc, int sig)
+glusterd_scrubsvc_stop(glusterd_svc_t *svc, int sig)
 {
-        return glusterd_svc_stop (svc, sig);
+    return glusterd_svc_stop(svc, sig);
 }
 
 int
-glusterd_scrubsvc_reconfigure ()
+glusterd_scrubsvc_reconfigure()
 {
-        int              ret             = -1;
-        xlator_t        *this            = NULL;
-        glusterd_conf_t *priv            = NULL;
-        gf_boolean_t     identical       = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        if (glusterd_should_i_stop_bitd ())
-                goto  manager;
-
-
-        /*
-         * Check both OLD and NEW volfiles, if they are SAME by size
-         * and cksum i.e. "character-by-character". If YES, then
-         * NOTHING has been changed, just return.
-         */
-        ret = glusterd_svc_check_volfile_identical (priv->scrub_svc.name,
-                                                    build_scrub_graph,
-                                                    &identical);
-        if (ret)
-                goto out;
-
-        if (identical) {
-                ret = 0;
-                goto out;
-        }
-
-        /*
-         * They are not identical. Find out if the topology is changed
-         * OR just the volume options. If just the options which got
-         * changed, then inform the xlator to reconfigure the options.
-         */
-        identical = _gf_false; /* RESET the FLAG */
-        ret = glusterd_svc_check_topology_identical (priv->scrub_svc.name,
-                                                     build_scrub_graph,
-                                                     &identical);
-        if (ret)
-                goto out;
-
-        /* Topology is not changed, but just the options. But write the
-         * options to scrub volfile, so that scrub will be reconfigured.
-         */
-        if (identical) {
-                ret = glusterd_scrubsvc_create_volfile ();
-                if (ret == 0) {/* Only if above PASSES */
-                        ret = glusterd_fetchspec_notify (THIS);
-                }
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (glusterd_should_i_stop_bitd())
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_volfile_identical(priv->scrub_svc.name,
+                                               build_scrub_graph, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->scrub_svc.name,
+                                                build_scrub_graph, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to scrub volfile, so that scrub will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_scrubsvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
         }
+        goto out;
+    }
 
 manager:
-        /*
-         * scrub volfile's topology has been changed. scrub server needs
-         * to be RESTARTED to ACT on the changed volfile.
-         */
-        ret = priv->scrub_svc.manager (&(priv->scrub_svc),
-                                       NULL,
-                                       PROC_START_NO_WAIT);
+    /*
+     * scrub volfile's topology has been changed. scrub server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->scrub_svc.manager(&(priv->scrub_svc), NULL, PROC_START_NO_WAIT);
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
-
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
index dbdcf43529c..514b1de96a0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
@@ -15,31 +15,31 @@
 
 typedef struct glusterd_scrubsvc_ glusterd_scrubsvc_t;
 
-struct glusterd_scrubsvc_{
-        glusterd_svc_t      svc;
-        gf_store_handle_t   *handle;
+struct glusterd_scrubsvc_ {
+    glusterd_svc_t svc;
+    gf_store_handle_t *handle;
 };
 
 void
-glusterd_scrubsvc_build (glusterd_svc_t *svc);
+glusterd_scrubsvc_build(glusterd_svc_t *svc);
 
 int
-glusterd_scrubsvc_init (glusterd_svc_t *svc);
+glusterd_scrubsvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_scrubsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_scrubsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_scrubsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_scrubsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_scrubsvc_stop (glusterd_svc_t *svc, int sig);
+glusterd_scrubsvc_stop(glusterd_svc_t *svc, int sig);
 
 int
-glusterd_scrubsvc_reconfigure ();
+glusterd_scrubsvc_reconfigure();
 
 void
-glusterd_scrubsvc_build_volfile_path (char *server, char *workdir,
-                                     char *volfile, size_t len);
+glusterd_scrubsvc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                     size_t len);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
index 7b3f0b79921..b0b8a2e4018 100644
--- a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
+++ b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
@@ -7,413 +7,480 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-messages.h"
 #include "glusterd-server-quorum.h"
+#include "glusterd-store.h"
 #include "glusterd-syncop.h"
 #include "glusterd-op-sm.h"
 
-#define CEILING_POS(X) (((X)-(int)(X)) > 0 ? (int)((X)+1) : (int)(X))
+#define CEILING_POS(X) (((X) - (int)(X)) > 0 ? (int)((X) + 1) : (int)(X))
 
 static gf_boolean_t
-glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
+glusterd_is_get_op(xlator_t *this, glusterd_op_t op, dict_t *dict)
 {
-        char            *key            = NULL;
-        char            *volname        = NULL;
-        int             ret             = 0;
-
-        if (op == GD_OP_STATUS_VOLUME)
+    char *key = NULL;
+    char *volname = NULL;
+    int ret = 0;
+
+    if (op == GD_OP_STATUS_VOLUME)
+        return _gf_true;
+
+    if (op == GD_OP_SET_VOLUME) {
+        /*check for set volume help*/
+        ret = dict_get_str(dict, "volname", &volname);
+        if (volname && ((strcmp(volname, "help") == 0) ||
+                        (strcmp(volname, "help-xml") == 0))) {
+            ret = dict_get_str(dict, "key1", &key);
+            if (ret < 0)
                 return _gf_true;
-
-        if (op == GD_OP_SET_VOLUME) {
-                /*check for set volume help*/
-                ret = dict_get_str (dict, "volname", &volname);
-                if (volname &&
-                    ((strcmp (volname, "help") == 0) ||
-                     (strcmp (volname, "help-xml") == 0))) {
-                        ret = dict_get_str (dict, "key1", &key);
-                        if (ret < 0)
-                                return _gf_true;
-                }
         }
-        return _gf_false;
+    }
+    return _gf_false;
 }
 
 gf_boolean_t
-glusterd_is_quorum_validation_required (xlator_t *this, glusterd_op_t op,
-                                        dict_t *dict)
+glusterd_is_quorum_validation_required(xlator_t *this, glusterd_op_t op,
+                                       dict_t *dict)
 {
-        gf_boolean_t    required        = _gf_true;
-        char            *key            = NULL;
-        char            *key_fixed      = NULL;
-        int             ret             = -1;
-
-        if (glusterd_is_get_op (this, op, dict)) {
-                required = _gf_false;
-                goto out;
-        }
-        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
-                goto out;
-        if (op == GD_OP_SET_VOLUME)
-                ret = dict_get_str (dict, "key1", &key);
-        else if (op == GD_OP_RESET_VOLUME)
-                ret = dict_get_str (dict, "key", &key);
-        if (ret)
-                goto out;
-        ret = glusterd_check_option_exists (key, &key_fixed);
-        if (ret <= 0)
-                goto out;
-        if (key_fixed)
-                key = key_fixed;
-        if (glusterd_is_quorum_option (key))
-                required = _gf_false;
+    gf_boolean_t required = _gf_true;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    int ret = -1;
+
+    if (glusterd_is_get_op(this, op, dict)) {
+        required = _gf_false;
+        goto out;
+    }
+    if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
+        goto out;
+    if (op == GD_OP_SET_VOLUME)
+        ret = dict_get_str(dict, "key1", &key);
+    else if (op == GD_OP_RESET_VOLUME)
+        ret = dict_get_str(dict, "key", &key);
+    if (ret)
+        goto out;
+    ret = glusterd_check_option_exists(key, &key_fixed);
+    if (ret <= 0)
+        goto out;
+    if (key_fixed)
+        key = key_fixed;
+    if (glusterd_is_quorum_option(key))
+        required = _gf_false;
 out:
-        GF_FREE (key_fixed);
-        return required;
+    GF_FREE(key_fixed);
+    return required;
 }
 
 int
-glusterd_validate_quorum (xlator_t *this, glusterd_op_t op,
-                             dict_t *dict, char **op_errstr)
+glusterd_validate_quorum(xlator_t *this, glusterd_op_t op, dict_t *dict,
+                         char **op_errstr)
 {
-        int                      ret     = 0;
-        char                    *volname = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    *errstr  = NULL;
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *errstr = NULL;
+
+    errstr = "Quorum not met. Volume operation not allowed.";
+    if (!glusterd_is_quorum_validation_required(this, op, dict))
+        goto out;
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        ret = 0;
+        goto out;
+    }
 
-        errstr = "Quorum not met. Volume operation not allowed.";
-        if (!glusterd_is_quorum_validation_required (this, op, dict))
-                goto out;
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
+    if (!glusterd_is_volume_in_server_quorum(volinfo)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
+    if (does_gd_meet_server_quorum(this)) {
+        ret = 0;
+        goto out;
+    }
 
-        if (does_gd_meet_server_quorum (this)) {
-                ret = 0;
-                goto out;
-        }
+    ret = -1;
+    *op_errstr = gf_strdup(errstr);
 
-        if (glusterd_is_volume_in_server_quorum (volinfo)) {
-                ret = -1;
-                *op_errstr = gf_strdup (errstr);
-                goto out;
-        }
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 gf_boolean_t
-glusterd_is_quorum_option (char *option)
+glusterd_is_quorum_option(char *option)
 {
-        gf_boolean_t    res     = _gf_false;
-        int             i       = 0;
-        static const char * const keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
-                                            GLUSTERD_QUORUM_RATIO_KEY,
-                                            NULL};
-
-        for (i = 0; keys[i]; i++) {
-                if (strcmp (option, keys[i]) == 0) {
-                        res = _gf_true;
-                        break;
-                }
+    gf_boolean_t res = _gf_false;
+    int i = 0;
+    static const char *const keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
+                                       GLUSTERD_QUORUM_RATIO_KEY, NULL};
+
+    for (i = 0; keys[i]; i++) {
+        if (strcmp(option, keys[i]) == 0) {
+            res = _gf_true;
+            break;
         }
-        return res;
+    }
+    return res;
 }
 
 gf_boolean_t
-glusterd_is_quorum_changed (dict_t *options, char *option, char *value)
+glusterd_is_quorum_changed(dict_t *options, char *option, char *value)
 {
-        int             ret             = 0;
-        gf_boolean_t    reconfigured    = _gf_false;
-        gf_boolean_t    all             = _gf_false;
-        char            *oldquorum      = NULL;
-        char            *newquorum      = NULL;
-        char            *oldratio       = NULL;
-        char            *newratio       = NULL;
-
-        if ((strcmp ("all", option) != 0) &&
-            !glusterd_is_quorum_option (option))
-                goto out;
-
-        if (strcmp ("all", option) == 0)
-                all = _gf_true;
-
-        if (all || (strcmp (GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
-                newquorum = value;
-                ret = dict_get_str (options, GLUSTERD_QUORUM_TYPE_KEY,
-                                    &oldquorum);
-        }
+    int ret = 0;
+    gf_boolean_t reconfigured = _gf_false;
+    gf_boolean_t all = _gf_false;
+    char *oldquorum = NULL;
+    char *newquorum = NULL;
+    char *oldratio = NULL;
+    char *newratio = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    if ((strcmp("all", option) != 0) && !glusterd_is_quorum_option(option))
+        goto out;
+
+    if (strcmp("all", option) == 0)
+        all = _gf_true;
+
+    if (all || (strcmp(GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
+        newquorum = value;
+        ret = dict_get_str(options, GLUSTERD_QUORUM_TYPE_KEY, &oldquorum);
+        if (ret)
+            gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
+                   "dict_get_str failed on %s", GLUSTERD_QUORUM_TYPE_KEY);
+    }
 
-        if (all || (strcmp (GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
-                newratio = value;
-                ret = dict_get_str (options, GLUSTERD_QUORUM_RATIO_KEY,
-                                    &oldratio);
-        }
+    if (all || (strcmp(GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
+        newratio = value;
+        ret = dict_get_str(options, GLUSTERD_QUORUM_RATIO_KEY, &oldratio);
+        if (ret)
+            gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
+                   "dict_get_str failed on %s", GLUSTERD_QUORUM_RATIO_KEY);
+    }
 
-        reconfigured = _gf_true;
+    reconfigured = _gf_true;
 
-        if (oldquorum && newquorum && (strcmp (oldquorum, newquorum) == 0))
-                reconfigured = _gf_false;
-        if (oldratio && newratio && (strcmp (oldratio, newratio) == 0))
-                reconfigured = _gf_false;
+    if (oldquorum && newquorum && (strcmp(oldquorum, newquorum) == 0))
+        reconfigured = _gf_false;
+    if (oldratio && newratio && (strcmp(oldratio, newratio) == 0))
+        reconfigured = _gf_false;
 
-        if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
-            (newquorum == NULL))
-                reconfigured = _gf_false;
+    if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
+        (newquorum == NULL))
+        reconfigured = _gf_false;
 out:
-        return reconfigured;
+    return reconfigured;
 }
 
 static gf_boolean_t
-_is_contributing_to_quorum (gd_quorum_contrib_t contrib)
+_is_contributing_to_quorum(gd_quorum_contrib_t contrib)
 {
-        if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
-                return _gf_true;
-        return _gf_false;
+    if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
+        return _gf_true;
+    return _gf_false;
 }
 
 gf_boolean_t
-does_quorum_meet (int active_count, int quorum_count)
+does_quorum_meet(int active_count, int quorum_count)
 {
-        return (active_count >= quorum_count);
+    return (active_count >= quorum_count);
 }
 
 int
-glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
-                                    int *quorum_count)
+glusterd_get_quorum_cluster_counts(xlator_t *this, int *active_count,
+                                   int *quorum_count)
 {
-        glusterd_peerinfo_t *peerinfo      = NULL;
-        glusterd_conf_t     *conf          = NULL;
-        int                 ret            = -1;
-        int                 inquorum_count = 0;
-        char                *val           = NULL;
-        double              quorum_percentage = 0.0;
-        gf_boolean_t        ratio          = _gf_false;
-        int                 count          = 0;
-
-        conf = this->private;
-
-        /* Start with counting self */
-        inquorum_count = 1;
-        if (active_count)
-                *active_count = 1;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                if (_is_contributing_to_quorum (peerinfo->quorum_contrib))
-                        inquorum_count = inquorum_count + 1;
-                if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
-                        *active_count = *active_count + 1;
-        }
-        rcu_read_unlock ();
-
-        ret = dict_get_str (conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
-        if (ret == 0) {
-                ratio = _gf_true;
-                ret = gf_string2percent (val, &quorum_percentage);
-                if (!ret)
-                        ratio = _gf_true;
-        }
-        if (ratio)
-                count = CEILING_POS (inquorum_count *
-                                     quorum_percentage / 100.0);
-        else
-                count = (inquorum_count * 50 / 100) + 1;
-
-        *quorum_count = count;
-        ret = 0;
-
-        return ret;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    int inquorum_count = 0;
+    char *val = NULL;
+    double quorum_percentage = 0.0;
+    gf_boolean_t ratio = _gf_false;
+    int count = 0;
+
+    conf = this->private;
+
+    /* Start with counting self */
+    inquorum_count = 1;
+    if (active_count)
+        *active_count = 1;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        if (_is_contributing_to_quorum(peerinfo->quorum_contrib))
+            inquorum_count = inquorum_count + 1;
+        if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
+            *active_count = *active_count + 1;
+    }
+    RCU_READ_UNLOCK;
+
+    ret = dict_get_str(conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
+    if (ret == 0) {
+        ret = gf_string2percent(val, &quorum_percentage);
+        if (ret == 0)
+            ratio = _gf_true;
+    }
+    if (ratio)
+        count = CEILING_POS(inquorum_count * quorum_percentage / 100.0);
+    else
+        count = (inquorum_count * 50 / 100) + 1;
+
+    *quorum_count = count;
+    ret = 0;
+
+    return ret;
 }
 
 gf_boolean_t
-glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo)
+glusterd_is_volume_in_server_quorum(glusterd_volinfo_t *volinfo)
 {
-        gf_boolean_t    res             = _gf_false;
-        char            *quorum_type    = NULL;
-        int             ret             = 0;
-
-        ret = dict_get_str (volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY,
-                            &quorum_type);
-        if (ret)
-                goto out;
-
-        if (strcmp (quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
-                res = _gf_true;
+    gf_boolean_t res = _gf_false;
+    char *quorum_type = NULL;
+    int ret = 0;
+
+    ret = dict_get_str(volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY, &quorum_type);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GLUSTERD_QUORUM_TYPE_KEY, NULL);
+        goto out;
+    }
+
+    if (strcmp(quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
+        res = _gf_true;
 out:
-        return res;
+    return res;
 }
 
 gf_boolean_t
-glusterd_is_any_volume_in_server_quorum (xlator_t *this)
+glusterd_is_any_volume_in_server_quorum(xlator_t *this)
 {
-        glusterd_conf_t     *conf     = NULL;
-        glusterd_volinfo_t  *volinfo  = NULL;
-
-        conf = this->private;
-        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                if (glusterd_is_volume_in_server_quorum (volinfo)) {
-                        return _gf_true;
-                }
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    conf = this->private;
+    list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        if (glusterd_is_volume_in_server_quorum(volinfo)) {
+            return _gf_true;
         }
-        return _gf_false;
+    }
+    return _gf_false;
 }
 
 gf_boolean_t
-does_gd_meet_server_quorum (xlator_t *this)
+does_gd_meet_server_quorum(xlator_t *this)
 {
-        int                     quorum_count    = 0;
-        int                     active_count    = 0;
-        gf_boolean_t            in              = _gf_false;
-        glusterd_conf_t         *conf           = NULL;
-        int                     ret             = -1;
-
-        conf = this->private;
-        ret = glusterd_get_quorum_cluster_counts (this, &active_count,
-                                                  &quorum_count);
-        if (ret)
-                goto out;
-
-        if (!does_quorum_meet (active_count, quorum_count)) {
-                goto out;
-        }
-
-        in = _gf_true;
+    int quorum_count = 0;
+    int active_count = 0;
+    gf_boolean_t in = _gf_false;
+    int ret = -1;
+
+    ret = glusterd_get_quorum_cluster_counts(this, &active_count,
+                                             &quorum_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL, NULL);
+        goto out;
+    }
+
+    if (!does_quorum_meet(active_count, quorum_count)) {
+        goto out;
+    }
+
+    in = _gf_true;
 out:
-        return in;
+    return in;
 }
 
 void
-glusterd_do_volume_quorum_action (xlator_t *this, glusterd_volinfo_t *volinfo,
-                                  gf_boolean_t meets_quorum)
+glusterd_do_volume_quorum_action(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                 gf_boolean_t meets_quorum)
 {
-        glusterd_brickinfo_t *brickinfo     = NULL;
-        glusterd_conf_t      *conf          = NULL;
-        gd_quorum_status_t   quorum_status  = NOT_APPLICABLE_QUORUM;
-        gf_boolean_t         follows_quorum = _gf_false;
-
-        conf = this->private;
-        if (volinfo->status != GLUSTERD_STATUS_STARTED) {
-                volinfo->quorum_status = NOT_APPLICABLE_QUORUM;
-                goto out;
-        }
-
-        follows_quorum = glusterd_is_volume_in_server_quorum (volinfo);
-        if (follows_quorum) {
-                if (meets_quorum)
-                        quorum_status = MEETS_QUORUM;
-                else
-                        quorum_status = DOESNT_MEET_QUORUM;
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gd_quorum_status_t quorum_status = NOT_APPLICABLE_QUORUM;
+    gf_boolean_t follows_quorum = _gf_false;
+    gf_boolean_t quorum_status_unchanged = _gf_false;
+
+    if (volinfo->status != GLUSTERD_STATUS_STARTED) {
+        volinfo->quorum_status = NOT_APPLICABLE_QUORUM;
+        goto out;
+    }
+
+    follows_quorum = glusterd_is_volume_in_server_quorum(volinfo);
+    if (follows_quorum) {
+        if (meets_quorum)
+            quorum_status = MEETS_QUORUM;
+        else
+            quorum_status = DOESNT_MEET_QUORUM;
+    } else {
+        quorum_status = NOT_APPLICABLE_QUORUM;
+    }
+
+    /*
+     * The following check is added to prevent spurious brick starts when
+     * events occur that affect quorum.
+     * Example:
+     * There is a cluster of 10 peers. Volume is in quorum. User
+     * takes down one brick from the volume to perform maintenance.
+     * Suddenly one of the peers go down. Cluster is still in quorum. But
+     * because of this 'peer going down' event, quorum is calculated and
+     * the bricks that are down are brought up again. In this process it
+     * also brings up the brick that is purposefully taken down.
+     */
+    if (volinfo->quorum_status == quorum_status) {
+        quorum_status_unchanged = _gf_true;
+        goto out;
+    }
+
+    if (quorum_status == MEETS_QUORUM) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0,
+               GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS,
+               "Server quorum regained for volume %s. Starting local "
+               "bricks.",
+               volinfo->volname);
+        gf_event(EVENT_QUORUM_REGAINED, "volume=%s", volinfo->volname);
+    } else if (quorum_status == DOESNT_MEET_QUORUM) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0,
+               GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
+               "Server quorum lost for volume %s. Stopping local "
+               "bricks.",
+               volinfo->volname);
+        gf_event(EVENT_QUORUM_LOST, "volume=%s", volinfo->volname);
+    }
+
+    list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+            continue;
+        if (quorum_status == DOESNT_MEET_QUORUM) {
+            ret = glusterd_brick_stop(volinfo, brickinfo, _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                       "Failed to "
+                       "stop brick %s:%s",
+                       brickinfo->hostname, brickinfo->path);
+            }
         } else {
-                quorum_status = NOT_APPLICABLE_QUORUM;
+            if (!brickinfo->start_triggered) {
+                pthread_mutex_lock(&brickinfo->restart_mutex);
+                {
+                    /* coverity[SLEEP] */
+                    ret = glusterd_brick_start(volinfo, brickinfo, _gf_false,
+                                               _gf_false);
+                }
+                pthread_mutex_unlock(&brickinfo->restart_mutex);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRICK_DISCONNECTED, "Failed to start %s:%s",
+                           brickinfo->hostname, brickinfo->path);
+                }
+            }
         }
-
-        /*
-         * The following check is added to prevent spurious brick starts when
-         * events occur that affect quorum.
-         * Example:
-         * There is a cluster of 10 peers. Volume is in quorum. User
-         * takes down one brick from the volume to perform maintenance.
-         * Suddenly one of the peers go down. Cluster is still in quorum. But
-         * because of this 'peer going down' event, quorum is calculated and
-         * the bricks that are down are brought up again. In this process it
-         * also brings up the brick that is purposefully taken down.
+    }
+    volinfo->quorum_status = quorum_status;
+    if (quorum_status == MEETS_QUORUM) {
+        /* bricks might have been restarted and so as the port change
+         * might have happened
          */
-        if (volinfo->quorum_status == quorum_status)
-                goto out;
-
-        if (quorum_status == MEETS_QUORUM) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS,
-                        "Server quorum regained for volume %s. Starting local "
-                        "bricks.", volinfo->volname);
-        } else if (quorum_status == DOESNT_MEET_QUORUM) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
-                        "Server quorum lost for volume %s. Stopping local "
-                        "bricks.", volinfo->volname);
-        }
-
-        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (!glusterd_is_local_brick (this, volinfo, brickinfo))
-                        continue;
-                if (quorum_status == DOESNT_MEET_QUORUM)
-                        glusterd_brick_stop (volinfo, brickinfo, _gf_false);
-                else
-                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+        ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                   "Failed to write volinfo for volume %s", volinfo->volname);
+            goto out;
         }
-        volinfo->quorum_status = quorum_status;
+    }
 out:
-        return;
+    if (quorum_status_unchanged) {
+        list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+                continue;
+            ret = glusterd_brick_start(volinfo, brickinfo, _gf_false, _gf_true);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "Failed to "
+                       "connect to %s:%s",
+                       brickinfo->hostname, brickinfo->path);
+            }
+        }
+    }
+    return;
 }
 
 int
-glusterd_do_quorum_action ()
+glusterd_do_quorum_action()
 {
-        xlator_t            *this          = NULL;
-        glusterd_conf_t     *conf          = NULL;
-        glusterd_volinfo_t  *volinfo       = NULL;
-        int                 ret            = 0;
-        int                 active_count   = 0;
-        int                 quorum_count   = 0;
-        gf_boolean_t        meets          = _gf_false;
-
-        this = THIS;
-        conf = this->private;
-
-        conf->pending_quorum_action = _gf_true;
-        ret = glusterd_lock (conf->uuid);
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    int active_count = 0;
+    int quorum_count = 0;
+    gf_boolean_t meets = _gf_false;
+
+    this = THIS;
+    conf = this->private;
+
+    conf->pending_quorum_action = _gf_true;
+    ret = glusterd_lock(conf->uuid);
+    if (ret)
+        goto out;
+
+    {
+        ret = glusterd_get_quorum_cluster_counts(this, &active_count,
+                                                 &quorum_count);
         if (ret)
-                goto out;
+            goto unlock;
 
+        if (does_quorum_meet(active_count, quorum_count))
+            meets = _gf_true;
+        list_for_each_entry(volinfo, &conf->volumes, vol_list)
         {
-                ret = glusterd_get_quorum_cluster_counts (this, &active_count,
-                                                          &quorum_count);
-                if (ret)
-                        goto unlock;
-
-                if (does_quorum_meet (active_count, quorum_count))
-                        meets = _gf_true;
-                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                        glusterd_do_volume_quorum_action (this, volinfo, meets);
-                }
+            glusterd_do_volume_quorum_action(this, volinfo, meets);
         }
+    }
 unlock:
-        (void)glusterd_unlock (conf->uuid);
-        conf->pending_quorum_action = _gf_false;
+    (void)glusterd_unlock(conf->uuid);
+    conf->pending_quorum_action = _gf_false;
 out:
-        return ret;
+    return ret;
 }
 
-/* ret = 1 represents quorum is met or quorum not applicable,
-   ret = 0 represents quorum is not met
-*/
+/* ret = 0 represents quorum is not met
+ * ret = 1 represents quorum is met
+ * ret = 2 represents quorum not applicable
+ */
+
 int
-check_quorum_for_brick_start (glusterd_volinfo_t *volinfo,
-                              gf_boolean_t node_quorum)
+check_quorum_for_brick_start(glusterd_volinfo_t *volinfo,
+                             gf_boolean_t node_quorum)
 {
-        gf_boolean_t        volume_quorum  =  _gf_false;
-        int                 ret            = 0;
-
-        volume_quorum = glusterd_is_volume_in_server_quorum (volinfo);
-        if (volume_quorum) {
-                if (node_quorum)
-                        ret = 1;
-        } else {
-                ret = 1;
-        }
-        return ret;
+    gf_boolean_t volume_quorum = _gf_false;
+    int ret = 0;
+
+    volume_quorum = glusterd_is_volume_in_server_quorum(volinfo);
+    if (volume_quorum) {
+        if (node_quorum)
+            ret = 1;
+    } else {
+        ret = 2;
+    }
+    return ret;
 }
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-server-quorum.h b/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
index ea6a8bd6158..e11bf1a9206 100644
--- a/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
+++ b/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
@@ -11,36 +11,36 @@
 #define _GLUSTERD_SERVER_QUORUM_H
 
 int
-glusterd_validate_quorum (xlator_t *this, glusterd_op_t op, dict_t *dict,
-                          char **op_errstr);
+glusterd_validate_quorum(xlator_t *this, glusterd_op_t op, dict_t *dict,
+                         char **op_errstr);
 
 gf_boolean_t
-glusterd_is_quorum_changed (dict_t *options, char *option, char *value);
+glusterd_is_quorum_changed(dict_t *options, char *option, char *value);
 
 int
-glusterd_do_quorum_action ();
+glusterd_do_quorum_action();
 
 int
-glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
-                                    int *quorum_count);
+glusterd_get_quorum_cluster_counts(xlator_t *this, int *active_count,
+                                   int *quorum_count);
 
 gf_boolean_t
-glusterd_is_quorum_option (char *option);
+glusterd_is_quorum_option(char *option);
 
 gf_boolean_t
-glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo);
+glusterd_is_volume_in_server_quorum(glusterd_volinfo_t *volinfo);
 
 gf_boolean_t
-glusterd_is_any_volume_in_server_quorum (xlator_t *this);
+glusterd_is_any_volume_in_server_quorum(xlator_t *this);
 
 gf_boolean_t
-does_gd_meet_server_quorum (xlator_t *this);
+does_gd_meet_server_quorum(xlator_t *this);
 
 int
-check_quorum_for_brick_start (glusterd_volinfo_t *volinfo,
-                              gf_boolean_t node_quorum);
+check_quorum_for_brick_start(glusterd_volinfo_t *volinfo,
+                             gf_boolean_t node_quorum);
 
 gf_boolean_t
-does_quorum_meet (int active_count, int quorum_count);
+does_quorum_meet(int active_count, int quorum_count);
 
 #endif /* _GLUSTERD_SERVER_QUORUM_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c
new file mode 100644
index 00000000000..5661e391a9c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c
@@ -0,0 +1,153 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-shd-svc-helper.h"
+#include "glusterd-messages.h"
+#include "glusterd-volgen.h"
+
+void
+glusterd_svc_build_shd_socket_filepath(glusterd_volinfo_t *volinfo, char *path,
+                                       int path_len)
+{
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+    glusterd_conf_t *priv = THIS->private;
+
+    if (!priv)
+        return;
+
+    GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv);
+    len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+                   uuid_utoa(MY_UUID));
+    if ((len < 0) || (len >= sizeof(sockfilepath))) {
+        sockfilepath[0] = 0;
+    }
+
+    glusterd_set_socket_filepath(sockfilepath, path, path_len);
+}
+
+void
+glusterd_svc_build_shd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                               int path_len)
+{
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    if (!priv)
+        return;
+
+    GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/%s-shd.pid", rundir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_shd_volfile_path(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    if (!priv)
+        return;
+
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/%s-shd.vol", workdir, volinfo->volname);
+}
+
+void
+glusterd_shd_svcproc_cleanup(glusterd_shdsvc_t *shd)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    glusterd_svc_t *svc = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t need_unref = _gf_false;
+    rpc_clnt_t *rpc = NULL;
+
+    conf = THIS->private;
+    if (!conf)
+        return;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, conf, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, shd, out);
+
+    svc = &shd->svc;
+    shd->attached = _gf_false;
+
+    if (svc->conn.rpc) {
+        rpc_clnt_unref(svc->conn.rpc);
+        svc->conn.rpc = NULL;
+    }
+
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        svc_proc = svc->svc_proc;
+        svc->svc_proc = NULL;
+        svc->inited = _gf_false;
+        cds_list_del_init(&svc->mux_svc);
+        glusterd_unlink_file(svc->proc.pidfile);
+
+        if (svc_proc && cds_list_empty(&svc_proc->svcs)) {
+            cds_list_del_init(&svc_proc->svc_proc_list);
+            /* We cannot free svc_proc list from here. Because
+             * if there are pending events on the rpc, it will
+             * try to access the corresponding svc_proc, so unrefing
+             * rpc request and then cleaning up the memory is carried
+             * from the notify function upon RPC_CLNT_DESTROY destroy.
+             */
+            need_unref = _gf_true;
+            rpc = svc_proc->rpc;
+            svc_proc->rpc = NULL;
+        }
+    }
+    pthread_mutex_unlock(&conf->attach_lock);
+    /*rpc unref has to be performed outside the lock*/
+    if (need_unref && rpc)
+        rpc_clnt_unref(rpc);
+out:
+    return;
+}
+
+int
+glusterd_svc_set_shd_pidfile(glusterd_volinfo_t *volinfo, dict_t *dict)
+{
+    int ret = -1;
+    glusterd_svc_t *svc = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    svc = &(volinfo->shd.svc);
+
+    ret = dict_set_dynstr_with_alloc(dict, "pidfile", svc->proc.pidfile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set pidfile %s in dict", svc->proc.pidfile);
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h
new file mode 100644
index 00000000000..1f0984ba857
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h
@@ -0,0 +1,42 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERD_SHD_SVC_HELPER_H_
+#define _GLUSTERD_SHD_SVC_HELPER_H_
+
+#include "glusterd.h"
+#include "glusterd-svc-mgmt.h"
+
+void
+glusterd_svc_build_shd_socket_filepath(glusterd_volinfo_t *volinfo, char *path,
+                                       int path_len);
+
+void
+glusterd_svc_build_shd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                               int path_len);
+
+void
+glusterd_svc_build_shd_volfile_path(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len);
+
+void
+glusterd_shd_svcproc_cleanup(glusterd_shdsvc_t *shd);
+
+int
+glusterd_recover_shd_attach_failure(glusterd_volinfo_t *volinfo,
+                                    glusterd_svc_t *svc, int flags);
+
+int
+glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_svc_set_shd_pidfile(glusterd_volinfo_t *volinfo, dict_t *dict);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
index 0e664b5c786..1c56384a14b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
@@ -8,243 +8,789 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
-#include "glusterd-svc-mgmt.h"
 #include "glusterd-shd-svc.h"
+#include "glusterd-shd-svc-helper.h"
 #include "glusterd-svc-helper.h"
+#include "glusterd-store.h"
 
+#define GD_SHD_PROCESS_NAME "--process-name"
 char *shd_svc_name = "glustershd";
 
 void
-glusterd_shdsvc_build (glusterd_svc_t *svc)
+glusterd_shdsvc_build(glusterd_svc_t *svc)
 {
-        svc->manager = glusterd_shdsvc_manager;
-        svc->start = glusterd_shdsvc_start;
-        svc->stop = glusterd_svc_stop;
+    int ret = -1;
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", shd_svc_name);
+    if (ret < 0)
+        return;
+
+    CDS_INIT_LIST_HEAD(&svc->mux_svc);
+    svc->manager = glusterd_shdsvc_manager;
+    svc->start = glusterd_shdsvc_start;
+    svc->stop = glusterd_shdsvc_stop;
+    svc->reconfigure = glusterd_shdsvc_reconfigure;
 }
 
 int
-glusterd_shdsvc_init (glusterd_svc_t *svc)
+glusterd_shdsvc_init(void *data, glusterd_conn_t *mux_conn,
+                     glusterd_svc_proc_t *mux_svc)
 {
-        return glusterd_svc_init (svc, shd_svc_name);
-}
-
-static int
-glusterd_shdsvc_create_volfile ()
-{
-        char            filepath[PATH_MAX] = {0,};
-        int             ret = -1;
-        glusterd_conf_t *conf = THIS->private;
-        dict_t          *mod_dict = NULL;
-
-        mod_dict = dict_new ();
-        if (!mod_dict)
-                goto out;
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    char sockpath[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char logdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    glusterd_svc_t *svc = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_muxsvc_conn_notify_t notify = NULL;
+    xlator_t *this = NULL;
+    char *volfileserver = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    volinfo = data;
+    GF_VALIDATE_OR_GOTO(this->name, data, out);
+    GF_VALIDATE_OR_GOTO(this->name, mux_svc, out);
+
+    svc = &(volinfo->shd.svc);
+
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", shd_svc_name);
+    if (ret < 0)
+        goto out;
+
+    notify = glusterd_muxsvc_common_rpc_notify;
+    glusterd_store_perform_node_state_store(volinfo);
+
+    GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv);
+    glusterd_svc_create_rundir(rundir);
+
+    glusterd_svc_build_logfile_path(shd_svc_name, priv->logdir, logfile,
+                                    sizeof(logfile));
+
+    /* Initialize the connection mgmt */
+    if (mux_conn && mux_svc->rpc) {
+        /* multiplexed svc */
+        svc->conn.frame_timeout = mux_conn->frame_timeout;
+        /* This will be unrefed from glusterd_shd_svcproc_cleanup*/
+        svc->conn.rpc = rpc_clnt_ref(mux_svc->rpc);
+        ret = snprintf(svc->conn.sockpath, sizeof(svc->conn.sockpath), "%s",
+                       mux_conn->sockpath);
+        if (ret < 0)
+            goto out;
+    } else {
+        ret = mkdir_p(priv->logdir, 0755, _gf_true);
+        if ((ret == -1) && (EEXIST != errno)) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+                   "Unable to create logdir %s", logdir);
+            goto out;
+        }
 
-        ret = dict_set_uint32 (mod_dict, "cluster.background-self-heal-count",
-                               0);
+        glusterd_svc_build_shd_socket_filepath(volinfo, sockpath,
+                                               sizeof(sockpath));
+        ret = glusterd_muxsvc_conn_init(&(svc->conn), mux_svc, sockpath, 600,
+                                        notify);
         if (ret)
-                goto out;
+            goto out;
+        /* This will be unrefed when the last svcs is detached from the list */
+        if (!mux_svc->rpc)
+            mux_svc->rpc = rpc_clnt_ref(svc->conn.rpc);
+    }
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_shd_pidfile(volinfo, pidfile, sizeof(pidfile));
+    glusterd_svc_build_shd_volfile_path(volinfo, volfile, PATH_MAX);
+    len = snprintf(volfileid, sizeof(volfileid), "shd/%s", volinfo->volname);
+    if ((len < 0) || (len >= sizeof(volfileid))) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+    ret = glusterd_proc_init(&(svc->proc), shd_svc_name, pidfile, logdir,
+                             logfile, volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (mod_dict, "cluster.data-self-heal", "on");
-        if (ret)
-                goto out;
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        ret = dict_set_str (mod_dict, "cluster.metadata-self-heal", "on");
-        if (ret)
-                goto out;
+int
+glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo)
+{
+    char filepath[PATH_MAX] = {
+        0,
+    };
+
+    int ret = -1;
+    dict_t *mod_dict = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    glusterd_svc_build_shd_volfile_path(volinfo, filepath, PATH_MAX);
+    if (!glusterd_is_shd_compatible_volume(volinfo)) {
+        /* If volfile exist, delete it. This case happens when we
+         * change from replica/ec to distribute.
+         */
+        (void)glusterd_unlink_file(filepath);
+        ret = 0;
+        goto out;
+    }
+    mod_dict = dict_new();
+    if (!mod_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_set_uint32(mod_dict, "cluster.background-self-heal-count", 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.background-self-heal-count", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.data-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.data-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.metadata-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.metadata-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.entry-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.entry-self-heal", NULL);
+        goto out;
+    }
+
+    ret = glusterd_shdsvc_generate_volfile(volinfo, filepath, mod_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
 
-        ret = dict_set_str (mod_dict, "cluster.entry-self-heal", "on");
-        if (ret)
-                goto out;
+out:
+    if (mod_dict)
+        dict_unref(mod_dict);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        glusterd_svc_build_volfile_path (shd_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
-        ret = glusterd_create_global_volfile (build_shd_graph, filepath,
-                                              mod_dict);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "Failed to create volfile");
-                goto out;
-        }
+    return ret;
+}
 
+gf_boolean_t
+glusterd_svcs_shd_compatible_volumes_stopped(glusterd_svc_t *svc)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_svc_t *temp_svc = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t comp = _gf_false;
+    glusterd_conf_t *conf = THIS->private;
+
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        svc_proc = svc->svc_proc;
+        if (!svc_proc)
+            goto unlock;
+        cds_list_for_each_entry(temp_svc, &svc_proc->svcs, mux_svc)
+        {
+            /* Get volinfo->shd from svc object */
+            shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+            if (!shd) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+                       "Failed to get shd object "
+                       "from shd service");
+                goto unlock;
+            }
+
+            /* Get volinfo from shd */
+            volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+            if (!volinfo) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                       "Failed to get volinfo from "
+                       "from shd");
+                goto unlock;
+            }
+            if (!glusterd_is_shd_compatible_volume(volinfo))
+                continue;
+            if (volinfo->status == GLUSTERD_STATUS_STARTED)
+                goto unlock;
+        }
+        comp = _gf_true;
+    }
+unlock:
+    pthread_mutex_unlock(&conf->attach_lock);
 out:
-        if (mod_dict)
-                dict_unref (mod_dict);
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-
-        return ret;
+    return comp;
 }
 
 int
-glusterd_shdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                 ret     = 0;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        if (!svc->inited) {
-                ret = glusterd_shdsvc_init (svc);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FAILED_INIT_SHDSVC, "Failed to init shd "
-                                "service");
-                        goto out;
-                } else {
-                        svc->inited = _gf_true;
-                        gf_msg_debug (THIS->name, 0, "shd service initialized");
-                }
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t shd_restart = _gf_false;
+
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    volinfo = data;
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
+
+    if (volinfo->is_snap_volume) {
+        /* healing of a snap volume is not supported yet*/
+        ret = 0;
+        goto out;
+    }
+
+    while (conf->restart_shd) {
+        synccond_wait(&conf->cond_restart_shd, &conf->big_lock);
+    }
+    conf->restart_shd = _gf_true;
+    shd_restart = _gf_true;
+
+    if (volinfo)
+        glusterd_volinfo_ref(volinfo);
+
+    if (!glusterd_is_shd_compatible_volume(volinfo)) {
+        ret = 0;
+        if (svc->inited) {
+            /* This means glusterd was running for this volume and now
+             * it was converted to a non-shd volume. So just stop the shd
+             */
+            ret = svc->stop(svc, SIGTERM);
         }
-
-        volinfo = data;
-
-        /* If all the volumes are stopped or all shd compatible volumes
-         * are stopped then stop the service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
-         * Otherwise create volfile and restart service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
+        goto out;
+    }
+    ret = glusterd_shdsvc_create_volfile(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_shd_svc_mux_init(volinfo, svc);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_SHDSVC,
+               "Failed to init shd service");
+        goto out;
+    }
+
+    /* If all the volumes are stopped or all shd compatible volumes
+     * are stopped then stop the service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     * Otherwise create volfile and restart service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     */
+    if (glusterd_svcs_shd_compatible_volumes_stopped(svc)) {
+        /* TODO
+         * Take a lock and detach all svc's to stop the process
+         * also reset the init flag
          */
-        if (glusterd_are_all_volumes_stopped () ||
-            glusterd_all_shd_compatible_volumes_stopped ()) {
-                if (!(volinfo &&
-                      !glusterd_is_shd_compatible_volume (volinfo))) {
-                        ret = svc->stop (svc, SIGTERM);
-                }
-        } else {
-                if (!(volinfo &&
-                      !glusterd_is_shd_compatible_volume (volinfo))) {
-                        ret = glusterd_shdsvc_create_volfile ();
-                        if (ret)
-                                goto out;
-
-                        ret = svc->stop (svc, SIGTERM);
-                        if (ret)
-                                goto out;
-
-                        ret = svc->start (svc, flags);
-                        if (ret)
-                                goto out;
-
-                        ret = glusterd_conn_connect (&(svc->conn));
-                        if (ret)
-                                goto out;
-                }
+        ret = svc->stop(svc, SIGTERM);
+    } else if (volinfo) {
+        if (volinfo->status != GLUSTERD_STATUS_STARTED) {
+            ret = svc->stop(svc, SIGTERM);
+            if (ret)
+                goto out;
         }
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            ret = svc->start(svc, flags);
+            if (ret)
+                goto out;
+        }
+    }
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-
-        return ret;
+    if (shd_restart) {
+        conf->restart_shd = _gf_false;
+        synccond_broadcast(&conf->cond_restart_shd);
+    }
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
 int
-glusterd_shdsvc_start (glusterd_svc_t *svc, int flags)
+glusterd_new_shd_svc_start(glusterd_svc_t *svc, int flags)
 {
-        int              ret                            = -1;
-        char             glusterd_uuid_option[PATH_MAX] = {0};
-        dict_t          *cmdline                        = NULL;
-
-        cmdline = dict_new ();
-        if (!cmdline)
-                goto out;
-
-        ret = snprintf (glusterd_uuid_option, sizeof (glusterd_uuid_option),
-                        "*replicate*.node-uuid=%s", uuid_utoa (MY_UUID));
-        if (ret < 0)
-                goto out;
+    int ret = -1;
+    char glusterd_uuid_option[PATH_MAX] = {0};
+    char client_pid[32] = {0};
+    dict_t *cmdline = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    cmdline = dict_new();
+    if (!cmdline)
+        goto out;
+
+    ret = snprintf(glusterd_uuid_option, sizeof(glusterd_uuid_option),
+                   "*replicate*.node-uuid=%s", uuid_utoa(MY_UUID));
+    if (ret < 0)
+        goto out;
+
+    ret = snprintf(client_pid, sizeof(client_pid), "--client-pid=%d",
+                   GF_CLIENT_PID_SELF_HEALD);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_set_str(cmdline, "arg", client_pid);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg", NULL);
+        goto out;
+    }
+
+    /* Pass cmdline arguments as key-value pair. The key is merely
+     * a carrier and is not used. Since dictionary follows LIFO the value
+     * should be put in reverse order*/
+    ret = dict_set_str(cmdline, "arg4", svc->name);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg4", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(cmdline, "arg3", GD_SHD_PROCESS_NAME);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg3", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(cmdline, "arg2", glusterd_uuid_option);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg2", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(cmdline, "arg1", "--xlator-option");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg1", NULL);
+        goto out;
+    }
+
+    ret = glusterd_svc_start(svc, flags, cmdline);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_GLUSTER_SERVICE_START_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_conn_connect(&(svc->conn));
+out:
+    if (cmdline)
+        dict_unref(cmdline);
+    return ret;
+}
 
-        /* Pass cmdline arguments as key-value pair. The key is merely
-         * a carrier and is not used. Since dictionary follows LIFO the value
-         * should be put in reverse order*/
-        ret = dict_set_str (cmdline, "arg2", glusterd_uuid_option);
-        if (ret)
-                goto out;
+int
+glusterd_recover_shd_attach_failure(glusterd_volinfo_t *volinfo,
+                                    glusterd_svc_t *svc, int flags)
+{
+    int ret = -1;
+    glusterd_svc_proc_t *mux_proc = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    conf = THIS->private;
+
+    if (!conf || !volinfo || !svc)
+        return -1;
+    glusterd_shd_svcproc_cleanup(&volinfo->shd);
+    mux_proc = glusterd_svcprocess_new();
+    if (!mux_proc) {
+        return -1;
+    }
+    ret = glusterd_shdsvc_init(volinfo, NULL, mux_proc);
+    if (ret)
+        return -1;
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        cds_list_add_tail(&mux_proc->svc_proc_list, &conf->shd_procs);
+        svc->svc_proc = mux_proc;
+        cds_list_del_init(&svc->mux_svc);
+        cds_list_add_tail(&svc->mux_svc, &mux_proc->svcs);
+    }
+    pthread_mutex_unlock(&conf->attach_lock);
+
+    ret = glusterd_new_shd_svc_start(svc, flags);
+    if (!ret) {
+        volinfo->shd.attached = _gf_true;
+    }
+    return ret;
+}
 
-        ret = dict_set_str (cmdline, "arg1", "--xlator-option");
+int
+glusterd_shdsvc_start(glusterd_svc_t *svc, int flags)
+{
+    int ret = -1;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+
+    /* Get volinfo->shd from svc object */
+    shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+    if (!shd) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+               "Failed to get shd object "
+               "from shd service");
+        return -1;
+    }
+
+    /* Get volinfo from shd */
+    volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+    if (!volinfo) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from shd");
+        return -1;
+    }
+
+    if (volinfo->status != GLUSTERD_STATUS_STARTED)
+        return -1;
+
+    glusterd_volinfo_ref(volinfo);
+
+    if (!svc->inited) {
+        ret = glusterd_shd_svc_mux_init(volinfo, svc);
         if (ret)
-                goto out;
-
-        ret = glusterd_svc_start (svc, flags, cmdline);
+            goto out;
+    }
 
+    if (shd->attached) {
+        glusterd_volinfo_ref(volinfo);
+        /* Unref will happen from glusterd_svc_attach_cbk */
+        ret = glusterd_attach_svc(svc, volinfo, flags);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to attach shd svc(volume=%s) to pid=%d",
+                   volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+            glusterd_shd_svcproc_cleanup(&volinfo->shd);
+            glusterd_volinfo_unref(volinfo);
+            goto out1;
+        }
+        goto out;
+    }
+    ret = glusterd_new_shd_svc_start(svc, flags);
+    if (!ret) {
+        shd->attached = _gf_true;
+    }
 out:
-        if (cmdline)
-                dict_unref (cmdline);
-
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-
-        return ret;
+    if (ret && volinfo)
+        glusterd_shd_svcproc_cleanup(&volinfo->shd);
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+out1:
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
-
 int
-glusterd_shdsvc_reconfigure ()
+glusterd_shdsvc_reconfigure(glusterd_volinfo_t *volinfo)
 {
-        int              ret             = -1;
-        xlator_t        *this            = NULL;
-        glusterd_conf_t *priv            = NULL;
-        gf_boolean_t     identical       = _gf_false;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        if (glusterd_all_shd_compatible_volumes_stopped ())
-                goto manager;
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_boolean_t identical = _gf_false;
+    dict_t *mod_dict = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    if (!volinfo) {
+        /* reconfigure will be called separately*/
+        ret = 0;
+        goto out;
+    }
+
+    glusterd_volinfo_ref(volinfo);
+    svc = &(volinfo->shd.svc);
+    if (glusterd_svcs_shd_compatible_volumes_stopped(svc))
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+
+    if (!glusterd_is_shd_compatible_volume(volinfo)) {
+        if (svc->inited)
+            goto manager;
+
+        /* Nothing to do if not shd compatible */
+        ret = 0;
+        goto out;
+    }
+    mod_dict = dict_new();
+    if (!mod_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_set_uint32(mod_dict, "cluster.background-self-heal-count", 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.background-self-heal-count", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.data-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.data-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.metadata-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.metadata-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(mod_dict, "graph-check", 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=graph-check", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.entry-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.entry-self-heal", NULL);
+        goto out;
+    }
+
+    ret = glusterd_volume_svc_check_volfile_identical(
+        "glustershd", mod_dict, volinfo, glusterd_shdsvc_generate_volfile,
+        &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_volume_svc_check_topology_identical(
+        "glustershd", mod_dict, volinfo, glusterd_shdsvc_generate_volfile,
+        &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to shd volfile, so that shd will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_shdsvc_create_volfile(volinfo);
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
+        }
+        goto out;
+    }
+manager:
+    /*
+     * shd volfile's topology has been changed. volfile needs
+     * to be RECONFIGURED to ACT on the changed volfile.
+     */
+    ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
 
-        /*
-         * Check both OLD and NEW volfiles, if they are SAME by size
-         * and cksum i.e. "character-by-character". If YES, then
-         * NOTHING has been changed, just return.
-         */
-        ret = glusterd_svc_check_volfile_identical (priv->shd_svc.name,
-                                                    build_shd_graph,
-                                                    &identical);
-        if (ret)
-                goto out;
+out:
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+    if (mod_dict)
+        dict_unref(mod_dict);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        if (identical) {
-                ret = 0;
+int
+glusterd_shdsvc_restart()
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    pthread_mutex_lock(&conf->volume_lock);
+    cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list)
+    {
+        glusterd_volinfo_ref(volinfo);
+        pthread_mutex_unlock(&conf->volume_lock);
+        /* Start per volume shd svc */
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            svc = &(volinfo->shd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SHD_START_FAIL,
+                       "Couldn't start shd for "
+                       "vol: %s on restart",
+                       volinfo->volname);
+                gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                glusterd_volinfo_unref(volinfo);
                 goto out;
+            }
         }
+        glusterd_volinfo_unref(volinfo);
+        pthread_mutex_lock(&conf->volume_lock);
+    }
+    pthread_mutex_unlock(&conf->volume_lock);
+out:
+    return ret;
+}
 
+int
+glusterd_shdsvc_stop(glusterd_svc_t *svc, int sig)
+{
+    int ret = -1;
+    glusterd_svc_proc_t *svc_proc = NULL;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t empty = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    int pid = -1;
+
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    svc_proc = svc->svc_proc;
+    if (!svc_proc) {
         /*
-         * They are not identical. Find out if the topology is changed
-         * OR just the volume options. If just the options which got
-         * changed, then inform the xlator to reconfigure the options.
-         */
-        identical = _gf_false; /* RESET the FLAG */
-        ret = glusterd_svc_check_topology_identical (priv->shd_svc.name,
-                                                     build_shd_graph,
-                                                     &identical);
-        if (ret)
-                goto out;
-
-        /* Topology is not changed, but just the options. But write the
-         * options to shd volfile, so that shd will be reconfigured.
+         * This can happen when stop was called on a volume that is not shd
+         * compatible.
          */
-        if (identical) {
-                ret = glusterd_shdsvc_create_volfile ();
-                if (ret == 0) {/* Only if above PASSES */
-                        ret = glusterd_fetchspec_notify (THIS);
-                }
-                goto out;
+        gf_msg_debug("glusterd", 0, "svc_proc is null, ie shd already stopped");
+        ret = 0;
+        goto out;
+    }
+
+    /* Get volinfo->shd from svc object */
+    shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+    if (!shd) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+               "Failed to get shd object "
+               "from shd service");
+        return -1;
+    }
+
+    /* Get volinfo from shd */
+    volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+    if (!volinfo) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from shd");
+        return -1;
+    }
+
+    glusterd_volinfo_ref(volinfo);
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        if (!gf_is_service_running(svc->proc.pidfile, &pid)) {
+            gf_msg_debug(THIS->name, 0, "shd isn't running");
         }
-manager:
-        /*
-         * shd volfile's topology has been changed. shd server needs
-         * to be RESTARTED to ACT on the changed volfile.
-         */
-        ret = priv->shd_svc.manager (&(priv->shd_svc), NULL,
-                                     PROC_START_NO_WAIT);
-
+        cds_list_del_init(&svc->mux_svc);
+        empty = cds_list_empty(&svc_proc->svcs);
+        if (empty) {
+            svc_proc->status = GF_SVC_STOPPING;
+            cds_list_del_init(&svc_proc->svc_proc_list);
+        }
+    }
+    pthread_mutex_unlock(&conf->attach_lock);
+    if (empty) {
+        /* Unref will happen when destroying the connection */
+        glusterd_volinfo_ref(volinfo);
+        svc_proc->data = volinfo;
+        ret = glusterd_svc_stop(svc, sig);
+        if (ret) {
+            glusterd_volinfo_unref(volinfo);
+            goto out;
+        }
+    }
+    if (!empty && pid != -1) {
+        ret = glusterd_detach_svc(svc, volinfo, sig);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                   "shd service is failed to detach volume %s from pid %d",
+                   volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+        else
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
+                   "Shd service is detached for volume %s from pid %d",
+                   volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+    }
+    svc->online = _gf_false;
+    (void)glusterd_unlink_file((char *)svc->proc.pidfile);
+    glusterd_shd_svcproc_cleanup(shd);
+    ret = 0;
+    glusterd_volinfo_unref(volinfo);
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
-
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.h b/xlators/mgmt/glusterd/src/glusterd-shd-svc.h
index 38a3fd1afd1..55b409f4b69 100644
--- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.h
@@ -12,19 +12,34 @@
 #define _GLUSTERD_SHD_SVC_H_
 
 #include "glusterd-svc-mgmt.h"
+#include "glusterd.h"
+
+typedef struct glusterd_shdsvc_ glusterd_shdsvc_t;
+struct glusterd_shdsvc_ {
+    glusterd_svc_t svc;
+    gf_boolean_t attached;
+};
 
 void
-glusterd_shdsvc_build (glusterd_svc_t *svc);
+glusterd_shdsvc_build(glusterd_svc_t *svc);
+
+int
+glusterd_shdsvc_init(void *data, glusterd_conn_t *mux_conn,
+                     glusterd_svc_proc_t *svc_proc);
 
 int
-glusterd_shdsvc_init (glusterd_svc_t *svc);
+glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_shdsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_shdsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_shdsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_shdsvc_reconfigure();
 
 int
-glusterd_shdsvc_reconfigure ();
+glusterd_shdsvc_restart();
+
+int
+glusterd_shdsvc_stop(glusterd_svc_t *svc, int sig);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index c1fb3181b90..bf2d81b644a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -13,20 +13,20 @@
 #include <sys/resource.h>
 
 #include <libgen.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "fnmatch.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "protocol-common.h"
 #include "glusterd.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "list.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/list.h>
 #include "glusterd-messages.h"
-#include "dict.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
@@ -34,823 +34,900 @@
 #include "glusterd-svc-helper.h"
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-server-quorum.h"
+#include "glusterd-gfproxyd-svc-helper.h"
 
-char local_node_hostname[PATH_MAX] = {0, };
+char local_node_hostname[PATH_MAX] = {
+    0,
+};
 
 static struct cds_list_head gd_friend_sm_queue;
 
-static  char *glusterd_friend_sm_state_names[] = {
-        "Establishing Connection",
-        "Probe Sent to Peer",
-        "Probe Received from Peer",
-        "Peer in Cluster",
-        "Accepted peer request",
-        "Sent and Received peer request",
-        "Peer Rejected",
-        "Peer detach in progress",
-        "Probe Received from peer",
-        "Connected to Peer",
-        "Peer is connected and Accepted",
-        "Invalid State"
-};
+static char *glusterd_friend_sm_state_names[] = {
+    "Establishing Connection",
+    "Probe Sent to Peer",
+    "Probe Received from Peer",
+    "Peer in Cluster",
+    "Accepted peer request",
+    "Sent and Received peer request",
+    "Peer Rejected",
+    "Peer detach in progress",
+    "Probe Received from peer",
+    "Connected to Peer",
+    "Peer is connected and Accepted",
+    "Invalid State"};
 
 static char *glusterd_friend_sm_event_names[] = {
-        "GD_FRIEND_EVENT_NONE",
-        "GD_FRIEND_EVENT_PROBE",
-        "GD_FRIEND_EVENT_INIT_FRIEND_REQ",
-        "GD_FRIEND_EVENT_RCVD_ACC",
-        "GD_FRIEND_EVENT_LOCAL_ACC",
-        "GD_FRIEND_EVENT_RCVD_RJT",
-        "GD_FRIEND_EVENT_LOCAL_RJT",
-        "GD_FRIEND_EVENT_RCVD_FRIEND_REQ",
-        "GD_FRIEND_EVENT_INIT_REMOVE_FRIEND",
-        "GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND",
-        "GD_FRIEND_EVENT_REMOVE_FRIEND",
-        "GD_FRIEND_EVENT_CONNECTED",
-        "GD_FRIEND_EVENT_NEW_NAME",
-        "GD_FRIEND_EVENT_MAX"
-};
-
-char*
-glusterd_friend_sm_state_name_get (int state)
+    "GD_FRIEND_EVENT_NONE",
+    "GD_FRIEND_EVENT_PROBE",
+    "GD_FRIEND_EVENT_INIT_FRIEND_REQ",
+    "GD_FRIEND_EVENT_RCVD_ACC",
+    "GD_FRIEND_EVENT_LOCAL_ACC",
+    "GD_FRIEND_EVENT_RCVD_RJT",
+    "GD_FRIEND_EVENT_LOCAL_RJT",
+    "GD_FRIEND_EVENT_RCVD_FRIEND_REQ",
+    "GD_FRIEND_EVENT_INIT_REMOVE_FRIEND",
+    "GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND",
+    "GD_FRIEND_EVENT_REMOVE_FRIEND",
+    "GD_FRIEND_EVENT_CONNECTED",
+    "GD_FRIEND_EVENT_NEW_NAME",
+    "GD_FRIEND_EVENT_MAX"};
+
+char *
+glusterd_friend_sm_state_name_get(int state)
 {
-        if (state < 0 || state >= GD_FRIEND_STATE_MAX)
-                return glusterd_friend_sm_state_names[GD_FRIEND_STATE_MAX];
-        return glusterd_friend_sm_state_names[state];
+    if (state < 0 || state >= GD_FRIEND_STATE_MAX)
+        return glusterd_friend_sm_state_names[GD_FRIEND_STATE_MAX];
+    return glusterd_friend_sm_state_names[state];
 }
 
-char*
-glusterd_friend_sm_event_name_get (int event)
+char *
+glusterd_friend_sm_event_name_get(int event)
 {
-        if (event < 0 || event >= GD_FRIEND_EVENT_MAX)
-                return glusterd_friend_sm_event_names[GD_FRIEND_EVENT_MAX];
-        return glusterd_friend_sm_event_names[event];
+    if (event < 0 || event >= GD_FRIEND_EVENT_MAX)
+        return glusterd_friend_sm_event_names[GD_FRIEND_EVENT_MAX];
+    return glusterd_friend_sm_event_names[event];
 }
 
 void
-glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx)
+glusterd_destroy_probe_ctx(glusterd_probe_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
+    if (!ctx)
+        return;
 
-        GF_FREE (ctx->hostname);
-        GF_FREE (ctx);
+    GF_FREE(ctx->hostname);
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx)
+glusterd_destroy_friend_req_ctx(glusterd_friend_req_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
+    if (!ctx)
+        return;
 
-        if (ctx->vols)
-                dict_unref (ctx->vols);
-        GF_FREE (ctx->hostname);
-        GF_FREE (ctx);
+    if (ctx->vols)
+        dict_unref(ctx->vols);
+    GF_FREE(ctx->hostname);
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx)
+glusterd_destroy_friend_update_ctx(glusterd_friend_update_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
-        GF_FREE (ctx->hostname);
-        GF_FREE (ctx);
+    if (!ctx)
+        return;
+    GF_FREE(ctx->hostname);
+    GF_FREE(ctx);
 }
 
 int
-glusterd_broadcast_friend_delete (char *hostname, uuid_t uuid)
+glusterd_broadcast_friend_delete(char *hostname, uuid_t uuid)
 {
-        int                             ret = 0;
-        rpc_clnt_procedure_t            *proc = NULL;
-        xlator_t                        *this = NULL;
-        glusterd_friend_update_ctx_t    ctx = {{0},};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        dict_t                          *friends = NULL;
-        char                            key[100] = {0,};
-        int32_t                         count = 0;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        ctx.hostname = hostname;
-        ctx.op = GD_FRIEND_UPDATE_DEL;
-
-        friends = dict_new ();
-        if (!friends)
-                goto out;
-
-        snprintf (key, sizeof (key), "op");
-        ret = dict_set_int32 (friends, key, ctx.op);
-        if (ret)
-                goto out;
-
-        snprintf (key, sizeof (key), "hostname");
-        ret = dict_set_str (friends, key, hostname);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (friends, "count", count);
-        if (ret)
-                goto out;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                if (!peerinfo->connected || !peerinfo->peer)
-                        continue;
-
-                /* Setting a direct reference to peerinfo in the dict is okay as
-                 * it is only going to be used within this read critical section
-                 * (in glusterd_rpc_friend_update)
-                 */
-                ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "failed to set peerinfo");
-                        goto unlock;
-                }
-
-                proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, friends);
-                }
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_update_ctx_t ctx = {
+        {0},
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t count = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    ctx.hostname = hostname;
+    ctx.op = GD_FRIEND_UPDATE_DEL;
+
+    friends = dict_new();
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "op");
+    ret = dict_set_int32n(friends, key, keylen, ctx.op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "hostname");
+    ret = dict_set_strn(friends, key, keylen, hostname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!peerinfo->connected || !peerinfo->peer)
+            continue;
+
+        /* Setting a direct reference to peerinfo in the dict is okay as
+         * it is only going to be used within this read critical section
+         * (in glusterd_rpc_friend_update)
+         */
+        ret = dict_set_static_ptr(friends, "peerinfo", peerinfo);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peerinfo");
+            goto out;
         }
-unlock:
-        rcu_read_unlock ();
 
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
+        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+        if (proc->fn) {
+            ret = proc->fn(NULL, this, friends);
+        }
+    }
 
+    RCU_READ_UNLOCK;
 out:
-        if (friends)
-                dict_unref (friends);
+    if (friends)
+        dict_unref(friends);
 
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
-
 static int
-glusterd_ac_none (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_none(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_ac_error (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_error(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                GD_MSG_AC_ERROR, "Received event %d ", event->event);
+    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_AC_ERROR, "Received event %d ",
+           event->event);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_ac_reverse_probe_begin (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_reverse_probe_begin(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_probe_ctx_t            *new_ev_ctx = NULL;
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_friend_sm_new_event
-                (GD_FRIEND_EVENT_PROBE, &new_event);
-
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_NEW_GET_FAIL,
-                        "Unable to get new new_event");
-                ret = -1;
-                goto out;
-        }
-
-        new_ev_ctx = GF_CALLOC (1, sizeof(*new_ev_ctx), gf_gld_mt_probe_ctx_t);
-
-        if (!new_ev_ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        new_ev_ctx->hostname = gf_strdup (peerinfo->hostname);
-        new_ev_ctx->port = peerinfo->port;
-        new_ev_ctx->req = NULL;
-
-        new_event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (new_event->peerid, peerinfo->uuid);
-        new_event->ctx = new_ev_ctx;
-
-        ret = glusterd_friend_sm_inject_event (new_event);
-
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_EVENT_INJECT_FAIL,
-                        "Unable to inject new_event %d, "
-                        "ret = %d", new_event->event, ret);
-        }
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_probe_ctx_t *new_ev_ctx = NULL;
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+
+    new_ev_ctx = GF_CALLOC(1, sizeof(*new_ev_ctx), gf_gld_mt_probe_ctx_t);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_PROBE, &new_event);
+
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get new new_event");
+        ret = -1;
+        goto out;
+    }
+
+    if (!new_ev_ctx) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        goto out;
+    }
+
+    new_ev_ctx->hostname = gf_strdup(peerinfo->hostname);
+    new_ev_ctx->port = peerinfo->port;
+    new_ev_ctx->req = NULL;
+
+    new_event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(new_event->peerid, peerinfo->uuid);
+    new_event->ctx = new_ev_ctx;
+
+    ret = glusterd_friend_sm_inject_event(new_event);
+
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject new_event %d, "
+               "ret = %d",
+               new_event->event, ret);
+    }
 
 out:
-        rcu_read_unlock ();
-
-        if (ret) {
-                if (new_event)
-                        GF_FREE (new_event->peername);
-                GF_FREE (new_event);
-                if (new_ev_ctx)
-                        GF_FREE (new_ev_ctx->hostname);
-                GF_FREE (new_ev_ctx);
-        }
-        gf_msg_debug ("glusterd", 0, "returning with %d", ret);
-        return ret;
+    if (ret) {
+        if (new_event)
+            GF_FREE(new_event->peername);
+        GF_FREE(new_event);
+        if (new_ev_ctx)
+            GF_FREE(new_ev_ctx->hostname);
+        GF_FREE(new_ev_ctx);
+    }
+    gf_msg_debug("glusterd", 0, "returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_ac_friend_add (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_friend_add(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        glusterd_conf_t         *conf = NULL;
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (event);
-
-        this = THIS;
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND,
-                        "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
-        }
-
-        if (!peerinfo->peer)
-                goto out;
-        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_ADD];
-        if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                frame->local = ctx;
-                ret = proc->fn (frame, this, event);
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    if (!peerinfo->peer) {
+        RCU_READ_UNLOCK;
+        goto out;
+    }
+    proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_ADD];
+    if (proc->fn) {
+        frame = create_frame(this, this->ctx->pool);
+        if (!frame) {
+            RCU_READ_UNLOCK;
+            goto out;
         }
+        frame->local = ctx;
+        ret = proc->fn(frame, this, event);
+    }
 
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
 
-        if (ret && frame)
-                STACK_DESTROY (frame->root);
-
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_ac_friend_probe (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_friend_probe(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        glusterd_conf_t         *conf = NULL;
-        xlator_t                *this = NULL;
-        glusterd_probe_ctx_t    *probe_ctx = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        dict_t                  *dict = NULL;
-
-        GF_ASSERT (ctx);
-
-        probe_ctx = ctx;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (NULL, probe_ctx->hostname);
-        if (peerinfo == NULL) {
-                //We should not reach this state ideally
-                ret = -1;
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    glusterd_probe_ctx_t *probe_ctx = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(ctx);
+
+    probe_ctx = ctx;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(NULL, probe_ctx->hostname);
+    if (peerinfo == NULL) {
+        // We should not reach this state ideally
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
+        ret = -1;
+        goto unlock;
+    }
+
+    if (!peerinfo->peer) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL,
+                NULL);
+        goto unlock;
+    }
+    proc = &peerinfo->peer->proctable[GLUSTERD_PROBE_QUERY];
+    if (proc->fn) {
+        frame = create_frame(this, this->ctx->pool);
+        if (!frame) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
+                    NULL);
+            goto unlock;
+        }
+        frame->local = ctx;
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto unlock;
+        }
+        ret = dict_set_strn(dict, "hostname", SLEN("hostname"),
+                            probe_ctx->hostname);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=hostname", NULL);
+            goto unlock;
         }
 
-        if (!peerinfo->peer)
-                goto out;
-        proc = &peerinfo->peer->proctable[GLUSTERD_PROBE_QUERY];
-        if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                frame->local = ctx;
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-                ret = dict_set_str (dict, "hostname", probe_ctx->hostname);
-                if (ret)
-                        goto out;
-
-                ret = dict_set_int32 (dict, "port", probe_ctx->port);
-                if (ret)
-                        goto out;
-
-                /* The peerinfo reference being set here is going to be used
-                 * only within this critical section, in glusterd_rpc_probe
-                 * (ie. proc->fn).
-                 */
-                ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "failed to set peerinfo");
-                        goto out;
-                }
-
-                ret = proc->fn (frame, this, dict);
-                if (ret)
-                        goto out;
+        ret = dict_set_int32n(dict, "port", SLEN("port"), probe_ctx->port);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=port", NULL);
+            goto unlock;
+        }
 
+        /* The peerinfo reference being set here is going to be used
+         * only within this critical section, in glusterd_rpc_probe
+         * (ie. proc->fn).
+         */
+        ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peerinfo");
+            goto out;
         }
 
+        ret = proc->fn(frame, this, dict);
+        if (ret)
+            goto unlock;
+    }
+unlock:
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
 
-        if (dict)
-                dict_unref (dict);
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
+    if (dict)
+        dict_unref(dict);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
 
-        if (ret && frame)
-                STACK_DESTROY (frame->root);
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_ac_send_friend_remove_req (glusterd_friend_sm_event_t *event,
-                                    void *data)
+glusterd_ac_send_friend_remove_req(glusterd_friend_sm_event_t *event,
+                                   void *data)
 {
-        int                     ret = 0;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        glusterd_conf_t         *conf = NULL;
-        xlator_t                *this = NULL;
-        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_probe_ctx_t            *ctx = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-
-        GF_ASSERT (event);
-
-        this = THIS;
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_probe_ctx_t *ctx = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+    ctx = event->ctx;
+
+    if (!peerinfo->connected) {
+        event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
+
+        ret = glusterd_friend_sm_new_event(event_type, &new_event);
+
+        if (!ret) {
+            new_event->peername = peerinfo->hostname;
+            gf_uuid_copy(new_event->peerid, peerinfo->uuid);
+            ret = glusterd_friend_sm_inject_event(new_event);
+        } else {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+                   "Unable to get event");
         }
-        ctx = event->ctx;
-
-        if (!peerinfo->connected) {
-                event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
-
-                ret = glusterd_friend_sm_new_event (event_type, &new_event);
 
-                if (!ret) {
-                        new_event->peername = peerinfo->hostname;
-                        gf_uuid_copy (new_event->peerid, peerinfo->uuid);
-                        ret = glusterd_friend_sm_inject_event (new_event);
-                } else {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_EVENT_NEW_GET_FAIL,
-                                "Unable to get event");
-                }
-
-                if (ctx) {
-                        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, ret, 0,
-                                                              NULL,
-                                                              ctx->hostname,
-                                                              ctx->dict);
-                        glusterd_broadcast_friend_delete (ctx->hostname, NULL);
-                        glusterd_destroy_probe_ctx (ctx);
-                }
-                goto out;
+        if (ctx) {
+            ret = glusterd_xfer_cli_deprobe_resp(ctx->req, ret, 0, NULL,
+                                                 ctx->hostname, ctx->dict);
+            glusterd_broadcast_friend_delete(ctx->hostname, NULL);
+            glusterd_destroy_probe_ctx(ctx);
         }
-
-        if (!peerinfo->peer)
-                goto out;
-        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_REMOVE];
-        if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                frame->local = data;
-                ret = proc->fn (frame, this, event);
+        goto unlock;
+    }
+
+    if (!peerinfo->peer) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL,
+                NULL);
+        goto unlock;
+    }
+    proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_REMOVE];
+    if (proc->fn) {
+        frame = create_frame(this, this->ctx->pool);
+        if (!frame) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
+                    NULL);
+            goto unlock;
         }
+        frame->local = data;
+        ret = proc->fn(frame, this, event);
+    }
 
+unlock:
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
 
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
 
-        if (ret && frame)
-                STACK_DESTROY (frame->root);
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
 
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-glusterd_should_update_peer (glusterd_peerinfo_t *peerinfo,
-                             glusterd_peerinfo_t *cur_peerinfo)
+glusterd_should_update_peer(glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerinfo_t *cur_peerinfo)
 {
-        gf_boolean_t is_valid = _gf_false;
-
-        if ((peerinfo == cur_peerinfo) ||
-            (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED))
-                is_valid = _gf_true;
+    if ((peerinfo == cur_peerinfo) ||
+        (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED))
+        return _gf_true;
 
-        return is_valid;
+    return _gf_false;
 }
 
 static int
-glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_send_friend_update(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                           ret               = 0;
-        glusterd_peerinfo_t          *cur_peerinfo      = NULL;
-        glusterd_peerinfo_t          *peerinfo          = NULL;
-        rpc_clnt_procedure_t         *proc              = NULL;
-        xlator_t                     *this              = NULL;
-        glusterd_friend_update_ctx_t  ev_ctx            = {{0}};
-        glusterd_conf_t              *priv              = NULL;
-        dict_t                       *friends           = NULL;
-        char                          key[100]          = {0,};
-        int32_t                       count             = 0;
-
-        GF_ASSERT (event);
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-
-        cur_peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!cur_peerinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                ret = -1;
-                goto out;
-        }
-
-        ev_ctx.op = GD_FRIEND_UPDATE_ADD;
-
-        friends = dict_new ();
-        if (!friends)
-                goto out;
-
-        snprintf (key, sizeof (key), "op");
-        ret = dict_set_int32 (friends, key, ev_ctx.op);
+    int ret = 0;
+    glusterd_peerinfo_t *cur_peerinfo = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_update_ctx_t ev_ctx = {{0}};
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t count = 0;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    keylen = snprintf(key, sizeof(key), "op");
+    friends = dict_new();
+
+    RCU_READ_LOCK;
+
+    cur_peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!cur_peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto unlock;
+    }
+
+    ev_ctx.op = GD_FRIEND_UPDATE_ADD;
+    ret = dict_set_int32n(friends, key, keylen, ev_ctx.op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto unlock;
+    }
+
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
+            continue;
+
+        count++;
+
+        snprintf(key, sizeof(key), "friend%d", count);
+        ret = gd_add_friend_to_dict(peerinfo, friends, key);
         if (ret)
-                goto out;
+            goto unlock;
+    }
 
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
-                        continue;
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto unlock;
+    }
 
-                count++;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!peerinfo->connected || !peerinfo->peer)
+            continue;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d", count);
-                ret = gd_add_friend_to_dict (peerinfo, friends, key);
-                if (ret)
-                        goto out;
-        }
+        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
+            continue;
 
-        ret = dict_set_int32 (friends, "count", count);
-        if (ret)
-                goto out;
-
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                if (!peerinfo->connected || !peerinfo->peer)
-                        continue;
-
-                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
-                        continue;
-
-                ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "failed to set peerinfo");
-                        goto out;
-                }
-
-                proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, friends);
-                }
+        ret = dict_set_static_ptr(friends, "peerinfo", peerinfo);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peerinfo");
+            goto out;
         }
 
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
+        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+        if (proc->fn) {
+            ret = proc->fn(NULL, this, friends);
+        }
+    }
 
+unlock:
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
 
-        if (friends)
-                dict_unref (friends);
+    if (friends)
+        dict_unref(friends);
 
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* ac_update_friend only sends friend update to the friend that caused this
  * event to happen
  */
 static int
-glusterd_ac_update_friend (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_update_friend(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                           ret               = 0;
-        glusterd_peerinfo_t          *cur_peerinfo      = NULL;
-        glusterd_peerinfo_t          *peerinfo          = NULL;
-        rpc_clnt_procedure_t         *proc              = NULL;
-        xlator_t                     *this              = NULL;
-        glusterd_friend_update_ctx_t  ev_ctx            = {{0}};
-        glusterd_conf_t              *priv              = NULL;
-        dict_t                       *friends           = NULL;
-        char                          key[100]          = {0,};
-        int32_t                       count             = 0;
-
-        GF_ASSERT (event);
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-
-        cur_peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!cur_peerinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                ret = -1;
-                goto out;
-        }
-
-        /* Bail out early if peer is not connected.
-         * We cannot send requests to the peer until we have established our
-         * client connection to it.
-         */
-        if (!cur_peerinfo->connected || !cur_peerinfo->peer) {
-                ret = 0;
-                goto out;
-        }
-
-        ev_ctx.op = GD_FRIEND_UPDATE_ADD;
-
-        friends = dict_new ();
-        if (!friends)
-                goto out;
-
-        snprintf (key, sizeof (key), "op");
-        ret = dict_set_int32 (friends, key, ev_ctx.op);
-        if (ret)
-                goto out;
-
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
-                        continue;
-
-                count++;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d", count);
-                ret = gd_add_friend_to_dict (peerinfo, friends, key);
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_set_int32 (friends, "count", count);
+    int ret = 0;
+    glusterd_peerinfo_t *cur_peerinfo = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_update_ctx_t ev_ctx = {{0}};
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t count = 0;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    friends = dict_new();
+    keylen = snprintf(key, sizeof(key), "op");
+
+    RCU_READ_LOCK;
+
+    cur_peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!cur_peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    /* Bail out early if peer is not connected.
+     * We cannot send requests to the peer until we have established our
+     * client connection to it.
+     */
+    if (!cur_peerinfo->connected || !cur_peerinfo->peer) {
+        ret = 0;
+        goto unlock;
+    }
+
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ev_ctx.op = GD_FRIEND_UPDATE_ADD;
+    ret = dict_set_int32n(friends, key, keylen, ev_ctx.op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto unlock;
+    }
+
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
+            continue;
+
+        count++;
+
+        snprintf(key, sizeof(key), "friend%d", count);
+        ret = gd_add_friend_to_dict(peerinfo, friends, key);
         if (ret)
-                goto out;
-
-        ret = dict_set_static_ptr (friends, "peerinfo", cur_peerinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
-                        "failed to set peerinfo");
-                goto out;
-        }
-
-        proc = &cur_peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
-        if (proc->fn)
-                ret = proc->fn (NULL, this, friends);
-
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+            goto unlock;
+    }
+
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto unlock;
+    }
+
+    ret = dict_set_static_ptr(friends, "peerinfo", cur_peerinfo);
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set peerinfo");
+        goto out;
+    }
+
+    proc = &cur_peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+    if (proc->fn)
+        ret = proc->fn(NULL, this, friends);
+
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
+unlock:
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
 
-        if (friends)
-                dict_unref (friends);
+    if (friends)
+        dict_unref(friends);
 
-        return ret;
+    return ret;
 }
 
 /* Clean up stale volumes on the peer being detached. The volumes which have
  * bricks on other peers are stale with respect to the detached peer.
  */
 static void
-glusterd_peer_detach_cleanup (glusterd_conf_t *priv)
+glusterd_peer_detach_cleanup(glusterd_conf_t *priv)
 {
-        int                     ret          = -1;
-        glusterd_volinfo_t      *volinfo     = NULL;
-        glusterd_volinfo_t      *tmp_volinfo = NULL;
-        glusterd_svc_t          *svc         = NULL;
-
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry_safe (volinfo, tmp_volinfo, &priv->volumes,
-                                      vol_list) {
-                /* The peer detach checks make sure that, at this point in the
-                 * detach process, there are only volumes contained completely
-                 * within or completely outside the detached peer.
-                 * The only stale volumes at this point are the ones
-                 * completely outside the peer and can be safely deleted.
-                 */
-                if (!glusterd_friend_contains_vol_bricks (volinfo,
-                                                          MY_UUID)) {
-                        gf_msg (THIS->name, GF_LOG_INFO, 0,
-                                GD_MSG_STALE_VOL_DELETE_INFO,
-                                "Deleting stale volume %s", volinfo->volname);
-
-                        /*Stop snapd daemon service if snapd daemon is running*/
-                        if (!volinfo->is_snap_volume) {
-                                svc = &(volinfo->snapd.svc);
-                                ret = svc->stop (svc, SIGTERM);
-                                if (ret) {
-                                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_SVC_STOP_FAIL, "Failed "
-                                                "to stop snapd daemon service");
-                                }
-                        }
-
-                        ret = glusterd_cleanup_snaps_for_volume (volinfo);
-                        if (ret) {
-                                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOL_DELETE_FAIL,
-                                        "Error deleting snapshots for volume %s",
-                                        volinfo->volname);
-                        }
-
-                        ret = glusterd_delete_volume (volinfo);
-                        if (ret) {
-                                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_STALE_VOL_REMOVE_FAIL,
-                                        "Error deleting stale volume");
-                        }
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry_safe(volinfo, tmp_volinfo, &priv->volumes, vol_list)
+    {
+        /* The peer detach checks make sure that, at this point in the
+         * detach process, there are only volumes contained completely
+         * within or completely outside the detached peer.
+         * The only stale volumes at this point are the ones
+         * completely outside the peer and can be safely deleted.
+         */
+        if (!glusterd_friend_contains_vol_bricks(volinfo, MY_UUID)) {
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_STALE_VOL_DELETE_INFO,
+                   "Deleting stale volume %s", volinfo->volname);
+
+            /*Stop snapd daemon service if snapd daemon is running*/
+            if (!volinfo->is_snap_volume) {
+                svc = &(volinfo->snapd.svc);
+                ret = svc->stop(svc, SIGTERM);
+                if (ret) {
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop snapd daemon service");
                 }
-        }
+            }
 
-        /*Reconfigure all daemon services upon peer detach*/
-        ret = glusterd_svcs_reconfigure ();
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SVC_STOP_FAIL,
-                        "Failed to reconfigure all daemon services.");
-        }
-}
-
-static int
-glusterd_ac_handle_friend_remove_req (glusterd_friend_sm_event_t *event,
-                                      void *ctx)
-{
-        int                             ret = 0;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_req_ctx_t       *ev_ctx = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        ret = glusterd_xfer_friend_remove_resp (ev_ctx->req, ev_ctx->hostname,
-                                                ev_ctx->port);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-
-                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
-                                                    &new_event);
+            if (glusterd_is_shd_compatible_volume(volinfo)) {
+                svc = &(volinfo->shd.svc);
+                ret = svc->stop(svc, SIGTERM);
                 if (ret) {
-                        rcu_read_unlock ();
-                        goto out;
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop shd daemon service");
                 }
+            }
 
-                new_event->peername = gf_strdup (peerinfo->hostname);
-                gf_uuid_copy (new_event->peerid, peerinfo->uuid);
-
-                ret = glusterd_friend_sm_inject_event (new_event);
+            if (glusterd_is_gfproxyd_enabled(volinfo)) {
+                svc = &(volinfo->gfproxyd.svc);
+                ret = svc->stop(svc, SIGTERM);
                 if (ret) {
-                        rcu_read_unlock ();
-                        goto out;
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop gfproxyd daemon service");
                 }
-
-                new_event = NULL;
+            }
+
+            ret = glusterd_cleanup_snaps_for_volume(volinfo);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_DELETE_FAIL,
+                       "Error deleting snapshots for volume %s",
+                       volinfo->volname);
+            }
+
+            ret = glusterd_delete_volume(volinfo);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0,
+                       GD_MSG_STALE_VOL_REMOVE_FAIL,
+                       "Error deleting stale volume");
+            }
         }
-        rcu_read_unlock ();
-
-        glusterd_peer_detach_cleanup (priv);
-out:
-        if (new_event)
-                GF_FREE (new_event->peername);
-        GF_FREE (new_event);
-
-        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);
-        return ret;
+    }
+
+    /*Reconfigure all daemon services upon peer detach*/
+    ret = glusterd_svcs_reconfigure(NULL);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+               "Failed to reconfigure all daemon services.");
+    }
 }
 
 static int
-glusterd_ac_friend_remove (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_handle_friend_remove_req(glusterd_friend_sm_event_t *event,
+                                     void *ctx)
 {
-        int                  ret = -1;
-        glusterd_peerinfo_t *peerinfo = NULL;
-
-        GF_ASSERT (event);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND,
-                        "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                rcu_read_unlock ();
-                goto out;
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_req_ctx_t *ev_ctx = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(ctx);
+    ev_ctx = ctx;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_xfer_friend_remove_resp(ev_ctx->req, ev_ctx->hostname,
+                                           ev_ctx->port);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_REMOVE_FRIEND,
+                                           &new_event);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            goto out;
         }
-        ret = glusterd_friend_remove_cleanup_vols (peerinfo->uuid);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_CLEANUP_FAIL,
-                        "Volumes cleanup failed");
 
-        rcu_read_unlock ();
-        /* Exiting read critical section as glusterd_peerinfo_cleanup calls
-         * synchronize_rcu before freeing the peerinfo
-         */
+        new_event->peername = gf_strdup(peerinfo->hostname);
+        gf_uuid_copy(new_event->peerid, peerinfo->uuid);
 
-        ret = glusterd_peerinfo_cleanup (peerinfo);
+        ret = glusterd_friend_sm_inject_event(new_event);
         if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_DETACH_CLEANUP_FAIL,
-                        "Cleanup returned: %d", ret);
+            RCU_READ_UNLOCK;
+            goto out;
         }
+
+        new_event = NULL;
+    }
+    RCU_READ_UNLOCK;
+
+    glusterd_peer_detach_cleanup(priv);
 out:
-        return 0;
+    if (new_event)
+        GF_FREE(new_event->peername);
+    GF_FREE(new_event);
+
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
+}
+
+static int
+glusterd_ac_friend_remove(glusterd_friend_sm_event_t *event, void *ctx)
+{
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    GF_ASSERT(event);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+    ret = glusterd_friend_remove_cleanup_vols(peerinfo->uuid);
+    RCU_READ_UNLOCK;
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_CLEANUP_FAIL,
+               "Volumes cleanup failed");
+
+    /* Exiting read critical section as glusterd_peerinfo_cleanup calls
+     * synchronize_rcu before freeing the peerinfo
+     */
+
+    ret = glusterd_peerinfo_cleanup(peerinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_DETACH_CLEANUP_FAIL,
+               "Cleanup returned: %d", ret);
+    }
+out:
+    return 0;
 }
 
 /*static int
@@ -864,637 +941,682 @@ glusterd_ac_none (void *ctx)
 }*/
 
 static int
-glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_handle_friend_add_req(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                             ret = 0;
-        uuid_t                          uuid;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_req_ctx_t       *ev_ctx = NULL;
-        glusterd_friend_update_ctx_t    *new_ev_ctx = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_conf_t                *conf       = NULL;
-        int                             status = 0;
-        int32_t                         op_ret = -1;
-        int32_t                         op_errno = 0;
-        xlator_t                       *this       = NULL;
-        char                           *hostname   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
-        gf_uuid_copy (uuid, ev_ctx->uuid);
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                ret = -1;
-                rcu_read_unlock ();
-                goto out;
+    int ret = 0;
+    uuid_t uuid;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_req_ctx_t *ev_ctx = NULL;
+    glusterd_friend_update_ctx_t *new_ev_ctx = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_conf_t *conf = NULL;
+    int status = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    xlator_t *this = NULL;
+    char *hostname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(ctx);
+    ev_ctx = ctx;
+    gf_uuid_copy(uuid, ev_ctx->uuid);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    /* TODO: How do you do an atomic copy of uuid_t */
+    /* TODO: Updating within a read-critical section is also invalid
+     *       Update properly with updater synchronization
+     */
+    gf_uuid_copy(peerinfo->uuid, ev_ctx->uuid);
+
+    RCU_READ_UNLOCK;
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Passing the peername from the event. glusterd_compare_friend_data
+     * updates volumes and will use synchronize_rcu. If we were to pass
+     * peerinfo->hostname, we would have to do it under a read critical
+     * section which would lead to a deadlock
+     */
+
+    // Build comparison logic here.
+    pthread_mutex_lock(&conf->import_volumes);
+    {
+        ret = glusterd_compare_friend_data(ev_ctx->vols, &status,
+                                           event->peername);
+        if (ret) {
+            pthread_mutex_unlock(&conf->import_volumes);
+            goto out;
         }
 
-        /* TODO: How do you do an atomic copy of uuid_t */
-        /* TODO: Updating within a read-critical section is also invalid
-         *       Update properly with updater synchronization
-         */
-        gf_uuid_copy (peerinfo->uuid, ev_ctx->uuid);
-
-        rcu_read_unlock ();
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        /* Passing the peername from the event. glusterd_compare_friend_data
-         * updates volumes and will use synchronize_rcu. If we were to pass
-         * peerinfo->hostname, we would have to do it under a read critical
-         * section which would lead to a deadlock
-         */
-
-        //Build comparison logic here.
-        ret = glusterd_compare_friend_data (ev_ctx->vols, &status,
-                                            event->peername);
-        if (ret)
-                goto out;
-
         if (GLUSTERD_VOL_COMP_RJT != status) {
-                event_type = GD_FRIEND_EVENT_LOCAL_ACC;
-                op_ret = 0;
+            event_type = GD_FRIEND_EVENT_LOCAL_ACC;
+            op_ret = 0;
         } else {
-                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
-                op_errno = GF_PROBE_VOLUME_CONFLICT;
-                op_ret = -1;
+            event_type = GD_FRIEND_EVENT_LOCAL_RJT;
+            op_errno = GF_PROBE_VOLUME_CONFLICT;
+            op_ret = -1;
         }
 
         /* Compare missed_snapshot list with the peer *
          * if volume comparison is successful */
-        if ((op_ret == 0) &&
-            (conf->op_version >= GD_OP_VERSION_3_6_0)) {
-                ret = glusterd_import_friend_missed_snap_list (ev_ctx->vols);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
-                                "Failed to import peer's "
-                                "missed_snaps_list.");
-                        event_type = GD_FRIEND_EVENT_LOCAL_RJT;
-                        op_errno = GF_PROBE_MISSED_SNAP_CONFLICT;
-                        op_ret = -1;
-                }
-
-                /* glusterd_compare_friend_snapshots and functions only require
-                 * a peers hostname and uuid. It also does updates, which
-                 * require use of synchronize_rcu. So we pass the hostname and
-                 * id from the event instead of the peerinfo object to prevent
-                 * deadlocks as above.
-                 */
-                ret = glusterd_compare_friend_snapshots (ev_ctx->vols,
-                                                         event->peername,
-                                                         event->peerid);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_COMPARE_CONFLICT,
-                                "Conflict in comparing peer's snapshots");
-                        event_type = GD_FRIEND_EVENT_LOCAL_RJT;
-                        op_errno = GF_PROBE_SNAP_CONFLICT;
-                        op_ret = -1;
-                }
-        }
-
-        ret = glusterd_friend_sm_new_event (event_type, &new_event);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out of Memory");
-        }
-
-        new_event->peername = gf_strdup (event->peername);
-        gf_uuid_copy (new_event->peerid, event->peerid);
-
-        new_ev_ctx = GF_CALLOC (1, sizeof (*new_ev_ctx),
-                                gf_gld_mt_friend_update_ctx_t);
-        if (!new_ev_ctx) {
-                ret = -1;
-                goto out;
+        if ((op_ret == 0) && (conf->op_version >= GD_OP_VERSION_3_6_0)) {
+            ret = glusterd_import_friend_missed_snap_list(ev_ctx->vols);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                       "Failed to import peer's "
+                       "missed_snaps_list.");
+                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
+                op_errno = GF_PROBE_MISSED_SNAP_CONFLICT;
+                op_ret = -1;
+            }
+
+            /* glusterd_compare_friend_snapshots and functions only require
+             * a peers hostname and uuid. It also does updates, which
+             * require use of synchronize_rcu. So we pass the hostname and
+             * id from the event instead of the peerinfo object to prevent
+             * deadlocks as above.
+             */
+            ret = glusterd_compare_friend_snapshots(
+                ev_ctx->vols, event->peername, event->peerid);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SNAP_COMPARE_CONFLICT,
+                       "Conflict in comparing peer's snapshots");
+                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
+                op_errno = GF_PROBE_SNAP_CONFLICT;
+                op_ret = -1;
+            }
         }
-
-        gf_uuid_copy (new_ev_ctx->uuid, ev_ctx->uuid);
-        new_ev_ctx->hostname = gf_strdup (ev_ctx->hostname);
-        new_ev_ctx->op = GD_FRIEND_UPDATE_ADD;
-
-        new_event->ctx = new_ev_ctx;
-
-        ret = dict_get_str (ev_ctx->vols, "hostname_in_cluster",
-                            &hostname);
-        if (ret || !hostname) {
-                gf_msg_debug (this->name, 0,
-                        "Unable to fetch local hostname from peer");
-        } else
-                strncpy (local_node_hostname, hostname,
-                         sizeof(local_node_hostname));
-
-        glusterd_friend_sm_inject_event (new_event);
-        new_event = NULL;
-
-        ret = glusterd_xfer_friend_add_resp (ev_ctx->req, ev_ctx->hostname,
-                                             event->peername, ev_ctx->port,
-                                             op_ret, op_errno);
+    }
+    pthread_mutex_unlock(&conf->import_volumes);
+    ret = glusterd_friend_sm_new_event(event_type, &new_event);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of Memory");
+        goto out;
+    }
+
+    new_event->peername = gf_strdup(event->peername);
+    gf_uuid_copy(new_event->peerid, event->peerid);
+
+    new_ev_ctx = GF_CALLOC(1, sizeof(*new_ev_ctx),
+                           gf_gld_mt_friend_update_ctx_t);
+    if (!new_ev_ctx) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(new_ev_ctx->uuid, ev_ctx->uuid);
+    new_ev_ctx->hostname = gf_strdup(ev_ctx->hostname);
+    new_ev_ctx->op = GD_FRIEND_UPDATE_ADD;
+
+    new_event->ctx = new_ev_ctx;
+
+    ret = dict_get_strn(ev_ctx->vols, "hostname_in_cluster",
+                        SLEN("hostname_in_cluster"), &hostname);
+    if (ret || !hostname) {
+        gf_msg_debug(this->name, 0, "Unable to fetch local hostname from peer");
+    } else if (snprintf(local_node_hostname, sizeof(local_node_hostname), "%s",
+                        hostname) >= sizeof(local_node_hostname)) {
+        gf_msg_debug(this->name, 0, "local_node_hostname truncated");
+        ret = -1;
+        goto out;
+    }
+
+    glusterd_friend_sm_inject_event(new_event);
+    new_event = NULL;
+
+    ret = glusterd_xfer_friend_add_resp(ev_ctx->req, ev_ctx->hostname,
+                                        event->peername, ev_ctx->port, op_ret,
+                                        op_errno);
 
 out:
-        if (new_event)
-                GF_FREE (new_event->peername);
-        GF_FREE (new_event);
+    if (new_event)
+        GF_FREE(new_event->peername);
+    GF_FREE(new_event);
 
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_friend_sm_transition_state (uuid_t peerid, char *peername,
-                                     glusterd_sm_t *state,
-                                     glusterd_friend_sm_event_type_t event_type)
+glusterd_friend_sm_transition_state(uuid_t peerid, char *peername,
+                                    glusterd_sm_t *state,
+                                    glusterd_friend_sm_event_type_t event_type)
 {
-        int ret = -1;
-        glusterd_peerinfo_t *peerinfo = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
 
-        GF_ASSERT (state);
-        GF_ASSERT (peername);
+    GF_ASSERT(state);
+    GF_ASSERT(peername);
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (peerid, peername);
-        if (!peerinfo) {
-                goto out;
-        }
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(peerid, peername);
+    if (!peerinfo) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
+        goto out;
+    }
 
-        (void) glusterd_sm_tr_log_transition_add (&peerinfo->sm_log,
-                                           peerinfo->state.state,
-                                           state[event_type].next_state,
-                                           event_type);
+    (void)glusterd_sm_tr_log_transition_add(
+        &peerinfo->sm_log, peerinfo->state.state, state[event_type].next_state,
+        event_type);
 
-        uatomic_set (&peerinfo->state.state, state[event_type].next_state);
+    uatomic_set(&peerinfo->state.state, state[event_type].next_state);
 
-        ret = 0;
+    ret = 0;
 out:
-        rcu_read_unlock ();
-        return ret;
+    RCU_READ_UNLOCK;
+    return ret;
 }
 
-
-glusterd_sm_t glusterd_state_default [] = {
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},//EV_PROBE
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_NEW_NAME
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_default[] = {
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},  // EV_PROBE
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add},   // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                          // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_MAX
 };
 
-glusterd_sm_t glusterd_state_probe_rcvd [] = {
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EV_PROBE
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_NEW_NAME
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_probe_rcvd[] = {
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EV_PROBE
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},         // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},         // EVENT_MAX
 };
 
-glusterd_sm_t glusterd_state_connected_rcvd [] = {
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EV_PROBE
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_reverse_probe_begin}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_NEW_NAME
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_connected_rcvd[] = {
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EV_PROBE
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_reverse_probe_begin},                   // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},        // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_CONNECTED_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t glusterd_state_connected_accepted [] = {
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_friend_probe}, //EV_PROBE
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_friend_add}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_NEW_NAME
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_connected_accepted[] = {
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_friend_probe},  // EV_PROBE
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_friend_add},  // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_none},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_sent [] = {
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_NEW_NAME
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_sent[] = {
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},      // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_send_friend_remove_req},          // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                   // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_rcvd [] = {
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},//EVENT_NEW_NAME
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_rcvd[] = {
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_none},                                // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},      // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},        // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_befriended [] = {
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_update_friend}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_update_friend}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_friend_add},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update},//EVENT_NEW_NAME
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_befriended[] = {
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_update_friend},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_update_friend},                   // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_friend_add},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},                // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_sent_rcvd [] = {
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_NEW_NAME
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_sent_rcvd[] = {
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},                   // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},       // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                        // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_rejected [] = {
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_probe}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_add},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},//EVENT_NEW_NAME
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_rejected[] = {
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},          // EVENT_NONE,
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_probe},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT,
+     glusterd_ac_friend_add},                        // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},    // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},    // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_add},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},        // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},        // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_accepted [] = {
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_reverse_probe_begin},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},//EVENT_NEW_NAME
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_accepted[] = {
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},              // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_ACCEPTED,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_REQ_ACCEPTED,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_reverse_probe_begin},                 // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_unfriend_sent [] = {
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_NEW_NAME
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_unfriend_sent[] = {
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_NONE,
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_none},  // EVENT_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                        // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t *glusterd_friend_state_table [] = {
-        glusterd_state_default,
-        glusterd_state_req_sent,
-        glusterd_state_req_rcvd,
-        glusterd_state_befriended,
-        glusterd_state_req_accepted,
-        glusterd_state_req_sent_rcvd,
-        glusterd_state_rejected,
-        glusterd_state_unfriend_sent,
-        glusterd_state_probe_rcvd,
-        glusterd_state_connected_rcvd,
-        glusterd_state_connected_accepted
-};
+glusterd_sm_t *glusterd_friend_state_table[] = {
+    glusterd_state_default,           glusterd_state_req_sent,
+    glusterd_state_req_rcvd,          glusterd_state_befriended,
+    glusterd_state_req_accepted,      glusterd_state_req_sent_rcvd,
+    glusterd_state_rejected,          glusterd_state_unfriend_sent,
+    glusterd_state_probe_rcvd,        glusterd_state_connected_rcvd,
+    glusterd_state_connected_accepted};
 
 int
-glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
-                              glusterd_friend_sm_event_t **new_event)
+glusterd_friend_sm_new_event(glusterd_friend_sm_event_type_t event_type,
+                             glusterd_friend_sm_event_t **new_event)
 {
-        glusterd_friend_sm_event_t      *event = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
 
-        GF_ASSERT (new_event);
-        GF_ASSERT (GD_FRIEND_EVENT_NONE <= event_type &&
-                        GD_FRIEND_EVENT_MAX > event_type);
+    GF_ASSERT(new_event);
+    GF_ASSERT(GD_FRIEND_EVENT_NONE <= event_type &&
+              GD_FRIEND_EVENT_MAX > event_type);
 
-        event = GF_CALLOC (1, sizeof (*event), gf_gld_mt_friend_sm_event_t);
+    event = GF_CALLOC(1, sizeof(*event), gf_gld_mt_friend_sm_event_t);
 
-        if (!event)
-                return -1;
+    if (!event)
+        return -1;
 
-        *new_event = event;
-        event->event = event_type;
-        CDS_INIT_LIST_HEAD (&event->list);
+    *new_event = event;
+    event->event = event_type;
+    CDS_INIT_LIST_HEAD(&event->list);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_friend_sm_inject_event (glusterd_friend_sm_event_t *event)
+glusterd_friend_sm_inject_event(glusterd_friend_sm_event_t *event)
 {
-        GF_ASSERT (event);
-        gf_msg_debug ("glusterd", 0, "Enqueue event: '%s'",
-                glusterd_friend_sm_event_name_get (event->event));
-        cds_list_add_tail (&event->list, &gd_friend_sm_queue);
+    GF_ASSERT(event);
+    gf_msg_debug("glusterd", 0, "Enqueue event: '%s'",
+                 glusterd_friend_sm_event_name_get(event->event));
+    cds_list_add_tail(&event->list, &gd_friend_sm_queue);
 
-        return 0;
+    return 0;
 }
 
 void
-glusterd_destroy_friend_event_context (glusterd_friend_sm_event_t *event)
+glusterd_destroy_friend_event_context(glusterd_friend_sm_event_t *event)
 {
-        if (!event)
-                return;
+    if (!event)
+        return;
 
-        switch (event->event) {
+    switch (event->event) {
         case GD_FRIEND_EVENT_RCVD_FRIEND_REQ:
         case GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND:
-                glusterd_destroy_friend_req_ctx (event->ctx);
-                break;
+            glusterd_destroy_friend_req_ctx(event->ctx);
+            break;
         case GD_FRIEND_EVENT_LOCAL_ACC:
         case GD_FRIEND_EVENT_LOCAL_RJT:
         case GD_FRIEND_EVENT_RCVD_ACC:
         case GD_FRIEND_EVENT_RCVD_RJT:
-                glusterd_destroy_friend_update_ctx (event->ctx);
-                break;
+            glusterd_destroy_friend_update_ctx(event->ctx);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 gf_boolean_t
-gd_does_peer_affect_quorum (glusterd_friend_sm_state_t old_state,
-                            glusterd_friend_sm_event_type_t event_type,
-                            glusterd_peerinfo_t *peerinfo)
+gd_does_peer_affect_quorum(glusterd_friend_sm_state_t old_state,
+                           glusterd_friend_sm_event_type_t event_type,
+                           glusterd_peerinfo_t *peerinfo)
 {
-        gf_boolean_t    affects = _gf_false;
-
-        //When glusterd comes up with friends in BEFRIENDED state in store,
-        //wait until compare-data happens.
-        if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
-            (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
-            (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
-                goto out;
-        if ((peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)
-            && peerinfo->connected) {
-                affects = _gf_true;
-        }
+    gf_boolean_t affects = _gf_false;
+
+    // When glusterd comes up with friends in BEFRIENDED state in store,
+    // wait until compare-data happens.
+    if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
+        (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
+        (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
+        goto out;
+    if ((peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED) &&
+        peerinfo->connected) {
+        affects = _gf_true;
+    }
 out:
-        return affects;
+    return affects;
 }
 
 int
-glusterd_friend_sm ()
+glusterd_friend_sm()
 {
-        glusterd_friend_sm_event_t      *event      = NULL;
-        glusterd_friend_sm_event_t      *tmp        = NULL;
-        int                              ret        = -1;
-        glusterd_friend_sm_ac_fn         handler    = NULL;
-        glusterd_sm_t                   *state      = NULL;
-        glusterd_peerinfo_t             *peerinfo   = NULL;
-        glusterd_friend_sm_event_type_t  event_type = 0;
-        gf_boolean_t                     is_await_conn = _gf_false;
-        gf_boolean_t                     quorum_action = _gf_false;
-        glusterd_friend_sm_state_t       old_state = GD_FRIEND_STATE_DEFAULT;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        while (!cds_list_empty (&gd_friend_sm_queue)) {
-                cds_list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue,
-                                              list) {
-
-                        cds_list_del_init (&event->list);
-                        event_type = event->event;
-
-                        rcu_read_lock ();
-
-                        peerinfo = glusterd_peerinfo_find (event->peerid,
-                                                           event->peername);
-                        if (!peerinfo) {
-                                gf_msg ("glusterd", GF_LOG_CRITICAL, 0,
-                                        GD_MSG_PEER_NOT_FOUND, "Received"
-                                        " event %s with empty peer info",
-                                glusterd_friend_sm_event_name_get (event_type));
-
-                                GF_FREE (event);
-                                rcu_read_unlock ();
-                                continue;
-                        }
-                        gf_msg_debug ("glusterd", 0, "Dequeued event of type: '%s'",
-                                glusterd_friend_sm_event_name_get (event_type));
-
-
-                        old_state = peerinfo->state.state;
-
-                        rcu_read_unlock ();
-                        /* Giving up read-critical section here as we only need
-                         * the current state to call the handler.
-                         *
-                         * We cannot continue into the handler in a read
-                         * critical section as there are handlers who do
-                         * updates, and could cause deadlocks.
-                         */
-
-                        state = glusterd_friend_state_table[old_state];
-
-                        GF_ASSERT (state);
-
-                        handler = state[event_type].handler;
-                        GF_ASSERT (handler);
-
-                        ret = handler (event, event->ctx);
-                        if (ret == GLUSTERD_CONNECTION_AWAITED) {
-                                is_await_conn = _gf_true;
-                                ret = 0;
-                        }
-
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_HANDLER_RETURNED,
-                                        "handler returned: "
-                                                "%d", ret);
-                                glusterd_destroy_friend_event_context (event);
-                                GF_FREE (event);
-                                continue;
-                        }
-
-                        if ((GD_FRIEND_EVENT_REMOVE_FRIEND == event_type) ||
-                           (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND == event_type)){
-                                glusterd_destroy_friend_event_context (event);
-                                GF_FREE (event);
-                                continue;
-                        }
-
-                        ret = glusterd_friend_sm_transition_state
-                                (event->peerid, event->peername, state,
-                                 event_type);
-
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_EVENT_STATE_TRANSITION_FAIL,
-                                        "Unable to transition"
-                                        " state from '%s' to '%s' for event '%s'",
-                        glusterd_friend_sm_state_name_get(old_state),
-                        glusterd_friend_sm_state_name_get(state[event_type].next_state),
-                                glusterd_friend_sm_event_name_get(event_type));
-                                goto out;
-                        }
-
-                        peerinfo = NULL;
-                        /* We need to obtain peerinfo reference once again as we
-                         * had exited the read critical section above.
-                         */
-                        rcu_read_lock ();
-                        peerinfo = glusterd_peerinfo_find (event->peerid,
-                                        event->peername);
-                        if (!peerinfo) {
-                                rcu_read_unlock ();
-                                /* A peer can only be deleted as a effect of
-                                 * this state machine, and two such state
-                                 * machines can never run at the same time.
-                                 * So if we cannot find the peerinfo here,
-                                 * something has gone terribly wrong.
-                                 */
-                                ret = -1;
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_PEER_NOT_FOUND,
-                                        "Cannot find peer %s(%s)",
-                                        event->peername, uuid_utoa (event->peerid));
-                                goto out;
-                        }
-                        if (gd_does_peer_affect_quorum (old_state, event_type,
-                                                        peerinfo)) {
-                                peerinfo->quorum_contrib = QUORUM_UP;
-                                if (peerinfo->quorum_action) {
-                                        peerinfo->quorum_action = _gf_false;
-                                        quorum_action = _gf_true;
-                                }
-                        }
-
-                        ret = glusterd_store_peerinfo (peerinfo);
-                        rcu_read_unlock ();
-
-                        glusterd_destroy_friend_event_context (event);
-                        GF_FREE (event);
-                        if (is_await_conn)
-                                break;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_sm_event_t *tmp = NULL;
+    int ret = -1;
+    glusterd_friend_sm_ac_fn handler = NULL;
+    glusterd_sm_t *state = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_type_t event_type = 0;
+    gf_boolean_t is_await_conn = _gf_false;
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_friend_sm_state_t old_state = GD_FRIEND_STATE_DEFAULT;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    while (!cds_list_empty(&gd_friend_sm_queue)) {
+        cds_list_for_each_entry_safe(event, tmp, &gd_friend_sm_queue, list)
+        {
+            cds_list_del_init(&event->list);
+            event_type = event->event;
+
+            RCU_READ_LOCK;
+
+            peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+            if (!peerinfo) {
+                RCU_READ_UNLOCK;
+                gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_PEER_NOT_FOUND,
+                       "Received"
+                       " event %s with empty peer info",
+                       glusterd_friend_sm_event_name_get(event_type));
+
+                GF_FREE(event);
+                continue;
+            }
+            old_state = peerinfo->state.state;
+            RCU_READ_UNLOCK;
+            gf_msg_debug("glusterd", 0, "Dequeued event of type: '%s'",
+                         glusterd_friend_sm_event_name_get(event_type));
+
+            /* Giving up read-critical section here as we only need
+             * the current state to call the handler.
+             *
+             * We cannot continue into the handler in a read
+             * critical section as there are handlers who do
+             * updates, and could cause deadlocks.
+             */
+
+            state = glusterd_friend_state_table[old_state];
+
+            GF_ASSERT(state);
+
+            handler = state[event_type].handler;
+            GF_ASSERT(handler);
+
+            ret = handler(event, event->ctx);
+            if (ret == GLUSTERD_CONNECTION_AWAITED) {
+                is_await_conn = _gf_true;
+                ret = 0;
+            }
+
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_HANDLER_RETURNED,
+                       "handler returned: "
+                       "%d",
+                       ret);
+                glusterd_destroy_friend_event_context(event);
+                GF_FREE(event);
+                continue;
+            }
+
+            if ((GD_FRIEND_EVENT_REMOVE_FRIEND == event_type) ||
+                (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND == event_type)) {
+                glusterd_destroy_friend_event_context(event);
+                GF_FREE(event);
+                continue;
+            }
+
+            ret = glusterd_friend_sm_transition_state(
+                event->peerid, event->peername, state, event_type);
+
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0,
+                       GD_MSG_EVENT_STATE_TRANSITION_FAIL,
+                       "Unable to transition"
+                       " state from '%s' to '%s' for event '%s'",
+                       glusterd_friend_sm_state_name_get(old_state),
+                       glusterd_friend_sm_state_name_get(
+                           state[event_type].next_state),
+                       glusterd_friend_sm_event_name_get(event_type));
+                goto out;
+            }
+
+            peerinfo = NULL;
+            /* We need to obtain peerinfo reference once again as we
+             * had exited the read critical section above.
+             */
+            RCU_READ_LOCK;
+            peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+            if (!peerinfo) {
+                RCU_READ_UNLOCK;
+                /* A peer can only be deleted as a effect of
+                 * this state machine, and two such state
+                 * machines can never run at the same time.
+                 * So if we cannot find the peerinfo here,
+                 * something has gone terribly wrong.
+                 */
+                ret = -1;
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+                       "Cannot find peer %s(%s)", event->peername,
+                       uuid_utoa(event->peerid));
+                goto out;
+            }
+            if (gd_does_peer_affect_quorum(old_state, event_type, peerinfo)) {
+                peerinfo->quorum_contrib = QUORUM_UP;
+                if (peerinfo->quorum_action) {
+                    peerinfo->quorum_action = _gf_false;
+                    quorum_action = _gf_true;
                 }
-                if (is_await_conn)
-                        break;
+            }
+
+            ret = glusterd_store_peerinfo(peerinfo);
+            RCU_READ_UNLOCK;
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                       "Failed to store peerinfo");
+            }
+
+            glusterd_destroy_friend_event_context(event);
+            GF_FREE(event);
+            if (is_await_conn)
+                break;
         }
+        if (is_await_conn)
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (quorum_action) {
-            /* When glusterd is restarted, it needs to wait until the 'friends' view
-             * of the volumes settle, before it starts any of the internal daemons.
-             *
-             * Every friend that was part of the cluster, would send its
-             * cluster-view, 'our' way. For every friend, who belongs to
-             * a partition which has a different cluster-view from our
-             * partition, we may update our cluster-view. For subsequent
-             * friends from that partition would agree with us, if the first
-             * friend wasn't rejected. For every first friend, whom we agreed with,
-             * we would need to start internal daemons/bricks belonging to the
-             * new volumes.
-             * glusterd_spawn_daemons calls functions that are idempotent. ie,
-             * the functions spawn process(es) only if they are not started yet.
-             *
-             * */
-                synclock_unlock (&priv->big_lock);
-                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
-                synclock_lock (&priv->big_lock);
-                glusterd_do_quorum_action ();
-        }
-        return ret;
+    if (quorum_action) {
+        /* When glusterd is restarted, it needs to wait until the 'friends' view
+         * of the volumes settle, before it starts any of the internal daemons.
+         *
+         * Every friend that was part of the cluster, would send its
+         * cluster-view, 'our' way. For every friend, who belongs to
+         * a partition which has a different cluster-view from our
+         * partition, we may update our cluster-view. For subsequent
+         * friends from that partition would agree with us, if the first
+         * friend wasn't rejected. For every first friend, whom we agreed with,
+         * we would need to start internal daemons/bricks belonging to the
+         * new volumes.
+         * glusterd_spawn_daemons calls functions that are idempotent. ie,
+         * the functions spawn process(es) only if they are not started yet.
+         *
+         * */
+        synclock_unlock(&priv->big_lock);
+        glusterd_launch_synctask(glusterd_spawn_daemons, NULL);
+        synclock_lock(&priv->big_lock);
+        glusterd_do_quorum_action();
+    }
+    return ret;
 }
 
-
 int
-glusterd_friend_sm_init ()
+glusterd_friend_sm_init()
 {
-        CDS_INIT_LIST_HEAD (&gd_friend_sm_queue);
-        return 0;
+    CDS_INIT_LIST_HEAD(&gd_friend_sm_queue);
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index 9e4fe33b558..11cbd85b3e3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -11,212 +11,206 @@
 #define _GLUSTERD_SM_H_
 
 #include <pthread.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "rpc-clnt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
-//#include "glusterd.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "rpcsvc.h"
-#include "store.h"
+#include <glusterfs/store.h>
 
 #include "glusterd-rcu.h"
 
 typedef enum gd_quorum_contribution_ {
-        QUORUM_NONE,
-        QUORUM_WAITING,
-        QUORUM_DOWN,
-        QUORUM_UP
+    QUORUM_NONE,
+    QUORUM_WAITING,
+    QUORUM_DOWN,
+    QUORUM_UP
 } gd_quorum_contrib_t;
 
 typedef enum glusterd_friend_sm_state_ {
-        GD_FRIEND_STATE_DEFAULT = 0,
-        GD_FRIEND_STATE_REQ_SENT,
-        GD_FRIEND_STATE_REQ_RCVD,
-        GD_FRIEND_STATE_BEFRIENDED,
-        GD_FRIEND_STATE_REQ_ACCEPTED,
-        GD_FRIEND_STATE_REQ_SENT_RCVD,
-        GD_FRIEND_STATE_REJECTED,
-        GD_FRIEND_STATE_UNFRIEND_SENT,
-        GD_FRIEND_STATE_PROBE_RCVD,
-        GD_FRIEND_STATE_CONNECTED_RCVD,
-        GD_FRIEND_STATE_CONNECTED_ACCEPTED,
-        GD_FRIEND_STATE_MAX
+    GD_FRIEND_STATE_DEFAULT = 0,
+    GD_FRIEND_STATE_REQ_SENT,
+    GD_FRIEND_STATE_REQ_RCVD,
+    GD_FRIEND_STATE_BEFRIENDED,
+    GD_FRIEND_STATE_REQ_ACCEPTED,
+    GD_FRIEND_STATE_REQ_SENT_RCVD,
+    GD_FRIEND_STATE_REJECTED,
+    GD_FRIEND_STATE_UNFRIEND_SENT,
+    GD_FRIEND_STATE_PROBE_RCVD,
+    GD_FRIEND_STATE_CONNECTED_RCVD,
+    GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+    GD_FRIEND_STATE_MAX
 } glusterd_friend_sm_state_t;
 
 typedef struct glusterd_peer_state_info_ {
-        glusterd_friend_sm_state_t   state;
-        struct timeval          transition_time;
-}glusterd_peer_state_info_t;
+    glusterd_friend_sm_state_t state;
+    struct timeval transition_time;
+} glusterd_peer_state_info_t;
 
 typedef struct glusterd_peer_hostname_ {
-        char                    *hostname;
-        struct cds_list_head     hostname_list;
+    char *hostname;
+    struct cds_list_head hostname_list;
 } glusterd_peer_hostname_t;
 
 typedef struct glusterd_sm_transition_ {
-        int             old_state;
-        int             event;
-        int             new_state;
-        time_t          time;
+    int old_state;
+    int event;
+    int new_state;
+    time_t time;
 } glusterd_sm_transition_t;
 
 typedef struct glusterd_sm_tr_log_ {
-        glusterd_sm_transition_t    *transitions;
-        size_t                      current;
-        size_t                      size;
-        size_t                      count;
-        char*                       (*state_name_get) (int);
-        char*                       (*event_name_get) (int);
+    glusterd_sm_transition_t *transitions;
+    size_t current;
+    size_t size;
+    size_t count;
+    char *(*state_name_get)(int);
+    char *(*event_name_get)(int);
 } glusterd_sm_tr_log_t;
 
 struct glusterd_peerinfo_ {
-        uuid_t                          uuid;
-        char                            uuid_str[50]; /* Retrieve this using
-                                                       * gd_peer_uuid_str ()
-                                                       */
-        glusterd_peer_state_info_t      state;
-        char                            *hostname;
-        struct cds_list_head            hostnames;
-        int                             port;
-        struct cds_list_head            uuid_list;
-        struct cds_list_head            op_peers_list;
-        struct rpc_clnt                 *rpc;
-        rpc_clnt_prog_t                 *mgmt;
-        rpc_clnt_prog_t                 *peer;
-        rpc_clnt_prog_t                 *mgmt_v3;
-        int                             connected;
-        gf_store_handle_t               *shandle;
-        glusterd_sm_tr_log_t            sm_log;
-        gf_boolean_t                    quorum_action;
-        gd_quorum_contrib_t             quorum_contrib;
-        gf_boolean_t                    locked;
-        gf_boolean_t                    detaching;
-        /* Members required for proper cleanup using RCU */
-        gd_rcu_head                     rcu_head;
-        pthread_mutex_t                 delete_lock;
-        uint32_t                        generation;
+    uuid_t uuid;
+    char uuid_str[50]; /* Retrieve this using
+                        * gd_peer_uuid_str ()
+                        */
+    glusterd_peer_state_info_t state;
+    char *hostname;
+    struct cds_list_head hostnames;
+    int port;
+    struct cds_list_head uuid_list;
+    struct cds_list_head op_peers_list;
+    struct rpc_clnt *rpc;
+    rpc_clnt_prog_t *mgmt;
+    rpc_clnt_prog_t *peer;
+    rpc_clnt_prog_t *mgmt_v3;
+    int connected;
+    gf_store_handle_t *shandle;
+    glusterd_sm_tr_log_t sm_log;
+    gf_boolean_t quorum_action;
+    gd_quorum_contrib_t quorum_contrib;
+    gf_boolean_t locked;
+    gf_boolean_t detaching;
+    /* Members required for proper cleanup using RCU */
+    gd_rcu_head rcu_head;
+    pthread_mutex_t delete_lock;
+    uint32_t generation;
 };
 
 typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
 
 typedef struct glusterd_local_peers_ {
-        glusterd_peerinfo_t   *peerinfo;
-        struct cds_list_head  op_peers_list;
+    glusterd_peerinfo_t *peerinfo;
+    struct cds_list_head op_peers_list;
 } glusterd_local_peers_t;
 
 typedef enum glusterd_ev_gen_mode_ {
-        GD_MODE_OFF,
-        GD_MODE_ON,
-        GD_MODE_SWITCH_ON
+    GD_MODE_OFF,
+    GD_MODE_ON,
+    GD_MODE_SWITCH_ON
 } glusterd_ev_gen_mode_t;
 
 typedef struct glusterd_peer_ctx_args_ {
-        rpcsvc_request_t        *req;
-        glusterd_ev_gen_mode_t  mode;
-        dict_t                  *dict;
+    rpcsvc_request_t *req;
+    glusterd_ev_gen_mode_t mode;
+    dict_t *dict;
 } glusterd_peerctx_args_t;
 
 typedef struct glusterd_peer_ctx_ {
-        glusterd_peerctx_args_t        args;
-        uuid_t                         peerid;
-        char                           *peername;
-        uint32_t                       peerinfo_gen;
-        char                           *errstr;
+    glusterd_peerctx_args_t args;
+    uuid_t peerid;
+    char *peername;
+    uint32_t peerinfo_gen;
+    char *errstr;
 } glusterd_peerctx_t;
 
 typedef enum glusterd_friend_sm_event_type_ {
-        GD_FRIEND_EVENT_NONE = 0,
-        GD_FRIEND_EVENT_PROBE,
-        GD_FRIEND_EVENT_INIT_FRIEND_REQ,
-        GD_FRIEND_EVENT_RCVD_ACC,
-        GD_FRIEND_EVENT_LOCAL_ACC,
-        GD_FRIEND_EVENT_RCVD_RJT,
-        GD_FRIEND_EVENT_LOCAL_RJT,
-        GD_FRIEND_EVENT_RCVD_FRIEND_REQ,
-        GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
-        GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
-        GD_FRIEND_EVENT_REMOVE_FRIEND,
-        GD_FRIEND_EVENT_CONNECTED,
-        GD_FRIEND_EVENT_NEW_NAME,
-        GD_FRIEND_EVENT_MAX
+    GD_FRIEND_EVENT_NONE = 0,
+    GD_FRIEND_EVENT_PROBE,
+    GD_FRIEND_EVENT_INIT_FRIEND_REQ,
+    GD_FRIEND_EVENT_RCVD_ACC,
+    GD_FRIEND_EVENT_LOCAL_ACC,
+    GD_FRIEND_EVENT_RCVD_RJT,
+    GD_FRIEND_EVENT_LOCAL_RJT,
+    GD_FRIEND_EVENT_RCVD_FRIEND_REQ,
+    GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
+    GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
+    GD_FRIEND_EVENT_REMOVE_FRIEND,
+    GD_FRIEND_EVENT_CONNECTED,
+    GD_FRIEND_EVENT_NEW_NAME,
+    GD_FRIEND_EVENT_MAX
 } glusterd_friend_sm_event_type_t;
 
-
 typedef enum glusterd_friend_update_op_ {
-        GD_FRIEND_UPDATE_NONE = 0,
-        GD_FRIEND_UPDATE_ADD,
-        GD_FRIEND_UPDATE_DEL,
+    GD_FRIEND_UPDATE_NONE = 0,
+    GD_FRIEND_UPDATE_ADD,
+    GD_FRIEND_UPDATE_DEL,
 } glusterd_friend_update_op_t;
 
-
 struct glusterd_friend_sm_event_ {
-        struct cds_list_head             list;
-        uuid_t                           peerid;
-        char                            *peername;
-        void                            *ctx;
-        glusterd_friend_sm_event_type_t  event;
+    struct cds_list_head list;
+    uuid_t peerid;
+    char *peername;
+    void *ctx;
+    glusterd_friend_sm_event_type_t event;
 };
 
 typedef struct glusterd_friend_sm_event_ glusterd_friend_sm_event_t;
 
-typedef int (*glusterd_friend_sm_ac_fn) (glusterd_friend_sm_event_t *, void *);
+typedef int (*glusterd_friend_sm_ac_fn)(glusterd_friend_sm_event_t *, void *);
 
 typedef struct glusterd_sm_ {
-        glusterd_friend_sm_state_t      next_state;
-        glusterd_friend_sm_ac_fn        handler;
+    glusterd_friend_sm_state_t next_state;
+    glusterd_friend_sm_ac_fn handler;
 } glusterd_sm_t;
 
 typedef struct glusterd_friend_req_ctx_ {
-        uuid_t                  uuid;
-        char                    *hostname;
-        rpcsvc_request_t        *req;
-        int                      port;
-        dict_t                  *vols;
+    uuid_t uuid;
+    char *hostname;
+    rpcsvc_request_t *req;
+    int port;
+    dict_t *vols;
 } glusterd_friend_req_ctx_t;
 
 typedef struct glusterd_friend_update_ctx_ {
-        uuid_t                  uuid;
-        char                    *hostname;
-        int                     op;
+    uuid_t uuid;
+    char *hostname;
+    int op;
 } glusterd_friend_update_ctx_t;
 
 typedef struct glusterd_probe_ctx_ {
-        char                    *hostname;
-        rpcsvc_request_t        *req;
-        int                      port;
-        dict_t                  *dict;
+    char *hostname;
+    rpcsvc_request_t *req;
+    int port;
+    dict_t *dict;
 } glusterd_probe_ctx_t;
 int
-glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
-                              glusterd_friend_sm_event_t **new_event);
+glusterd_friend_sm_new_event(glusterd_friend_sm_event_type_t event_type,
+                             glusterd_friend_sm_event_t **new_event);
 int
-glusterd_friend_sm_inject_event (glusterd_friend_sm_event_t *event);
+glusterd_friend_sm_inject_event(glusterd_friend_sm_event_t *event);
 
 int
-glusterd_friend_sm_init ();
+glusterd_friend_sm_init();
 
 int
-glusterd_friend_sm ();
+glusterd_friend_sm();
 
 void
-glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx);
+glusterd_destroy_probe_ctx(glusterd_probe_ctx_t *ctx);
 
 void
-glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx);
+glusterd_destroy_friend_req_ctx(glusterd_friend_req_ctx_t *ctx);
 
-char*
-glusterd_friend_sm_state_name_get (int state);
+char *
+glusterd_friend_sm_state_name_get(int state);
 
-char*
-glusterd_friend_sm_event_name_get (int event);
+char *
+glusterd_friend_sm_event_name_get(int event);
 
 int
-glusterd_broadcast_friend_delete (char *hostname, uuid_t uuid);
+glusterd_broadcast_friend_delete(char *hostname, uuid_t uuid);
 void
-glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx);
+glusterd_destroy_friend_update_ctx(glusterd_friend_update_ctx_t *ctx);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
index 826b4ca7463..42ef51b01b4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
@@ -13,51 +13,63 @@
 #include "glusterd-snapd-svc-helper.h"
 
 void
-glusterd_svc_build_snapd_rundir (glusterd_volinfo_t *volinfo,
-                                 char *path, int path_len)
+glusterd_svc_build_snapd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                int path_len)
 {
-        char                    workdir[PATH_MAX]      = {0,};
-        glusterd_conf_t        *priv                   = THIS->private;
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
-
-        snprintf (path, path_len, "%s/run", workdir);
+    GLUSTERD_GET_VOLUME_PID_DIR(workdir, volinfo, priv);
+    snprintf(path, path_len, "%s", workdir);
 }
 
 void
-glusterd_svc_build_snapd_socket_filepath (glusterd_volinfo_t *volinfo,
-                                          char *path, int path_len)
+glusterd_svc_build_snapd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len)
 {
-        char                    sockfilepath[PATH_MAX] = {0,};
-        char                    rundir[PATH_MAX]       = {0,};
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
 
-        glusterd_svc_build_snapd_rundir (volinfo, rundir, sizeof (rundir));
-        snprintf (sockfilepath, sizeof (sockfilepath), "%s/run-%s",
-                  rundir, uuid_utoa (MY_UUID));
+    glusterd_svc_build_snapd_rundir(volinfo, rundir, sizeof(rundir));
+    len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+                   uuid_utoa(MY_UUID));
+    if ((len < 0) || (len >= sizeof(sockfilepath))) {
+        sockfilepath[0] = 0;
+    }
 
-        glusterd_set_socket_filepath (sockfilepath, path, path_len);
+    glusterd_set_socket_filepath(sockfilepath, path, path_len);
 }
 
 void
-glusterd_svc_build_snapd_pidfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len)
+glusterd_svc_build_snapd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len)
 {
-        char            rundir[PATH_MAX]      = {0,};
+    char rundir[PATH_MAX] = {
+        0,
+    };
 
-        glusterd_svc_build_snapd_rundir (volinfo, rundir, sizeof (rundir));
+    glusterd_svc_build_snapd_rundir(volinfo, rundir, sizeof(rundir));
 
-        snprintf (path, path_len, "%s/%s-snapd.pid", rundir, volinfo->volname);
+    snprintf(path, path_len, "%s/%s-snapd.pid", rundir, volinfo->volname);
 }
 
 void
-glusterd_svc_build_snapd_volfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len)
+glusterd_svc_build_snapd_volfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len)
 {
-        char                    workdir[PATH_MAX]      = {0,};
-        glusterd_conf_t        *priv                   = THIS->private;
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
 
-        snprintf (path, path_len, "%s/%s-snapd.vol", workdir,
-                  volinfo->volname);
+    snprintf(path, path_len, "%s/%s-snapd.vol", workdir, volinfo->volname);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h
index 4c452b91658..3e23c2ce942 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h
@@ -14,19 +14,19 @@
 #include "glusterd.h"
 
 void
-glusterd_svc_build_snapd_rundir (glusterd_volinfo_t *volinfo,
-                                 char *path, int path_len);
+glusterd_svc_build_snapd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                int path_len);
 
 void
-glusterd_svc_build_snapd_socket_filepath (glusterd_volinfo_t *volinfo,
-                                          char *path, int path_len);
+glusterd_svc_build_snapd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len);
 
 void
-glusterd_svc_build_snapd_pidfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len);
+glusterd_svc_build_snapd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len);
 
 void
-glusterd_svc_build_snapd_volfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len);
+glusterd_svc_build_snapd_volfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
index 830dc1a706d..d75f249b29e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
@@ -8,8 +8,8 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
@@ -20,420 +20,459 @@
 #include "glusterd-snapd-svc.h"
 #include "glusterd-snapd-svc-helper.h"
 #include "glusterd-snapshot-utils.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
 char *snapd_svc_name = "snapd";
 
 static void
-glusterd_svc_build_snapd_logdir (char *logdir, char *volname, size_t len)
+glusterd_svc_build_snapd_logdir(char *logdir, char *volname, size_t len)
 {
-        snprintf (logdir, len, "%s/snaps/%s", DEFAULT_LOG_FILE_DIRECTORY,
-                  volname);
+    glusterd_conf_t *priv = THIS->private;
+    snprintf(logdir, len, "%s/snaps/%s", priv->logdir, volname);
 }
 
 static void
-glusterd_svc_build_snapd_logfile (char *logfile, char *logdir, size_t len)
+glusterd_svc_build_snapd_logfile(char *logfile, char *logdir, size_t len)
 {
-        snprintf (logfile, len, "%s/snapd.log", logdir);
+    snprintf(logfile, len, "%s/snapd.log", logdir);
 }
 
 void
-glusterd_snapdsvc_build (glusterd_svc_t *svc)
+glusterd_snapdsvc_build(glusterd_svc_t *svc)
 {
-        svc->manager = glusterd_snapdsvc_manager;
-        svc->start = glusterd_snapdsvc_start;
-        svc->stop = glusterd_svc_stop;
+    svc->manager = glusterd_snapdsvc_manager;
+    svc->start = glusterd_snapdsvc_start;
+    svc->stop = glusterd_svc_stop;
 }
 
 int
-glusterd_snapdsvc_init (void *data)
+glusterd_snapdsvc_init(void *data)
 {
-        int                     ret                = -1;
-        char                    rundir[PATH_MAX]   = {0,};
-        char                    sockpath[PATH_MAX] = {0,};
-        char                    pidfile[PATH_MAX]  = {0,};
-        char                    volfile[PATH_MAX]  = {0,};
-        char                    logdir[PATH_MAX]   = {0,};
-        char                    logfile[PATH_MAX]  = {0,};
-        char                    volfileid[256]     = {0};
-        glusterd_svc_t         *svc                = NULL;
-        glusterd_volinfo_t     *volinfo            = NULL;
-        glusterd_conf_t        *priv               = NULL;
-        glusterd_conn_notify_t  notify             = NULL;
-        xlator_t               *this               = NULL;
-        char                    *volfileserver     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        volinfo = data;
-
-        svc = &(volinfo->snapd.svc);
-
-        ret = snprintf (svc->name, sizeof (svc->name), "%s", snapd_svc_name);
-        if (ret < 0)
-                goto out;
-
-        notify = glusterd_snapdsvc_rpc_notify;
-
-        glusterd_svc_build_snapd_rundir (volinfo, rundir, sizeof (rundir));
-        glusterd_svc_create_rundir (rundir);
-
-        /* Initialize the connection mgmt */
-        glusterd_svc_build_snapd_socket_filepath (volinfo, sockpath,
-                                                  sizeof (sockpath));
-        ret = glusterd_conn_init (&(svc->conn), sockpath, 600, notify);
-        if (ret)
-                goto out;
-
-        /* Initialize the process mgmt */
-        glusterd_svc_build_snapd_pidfile (volinfo, pidfile, sizeof (pidfile));
-        glusterd_svc_build_snapd_volfile (volinfo, volfile, sizeof (volfile));
-        glusterd_svc_build_snapd_logdir (logdir, volinfo->volname,
-                                         sizeof (logdir));
-        ret = mkdir_p (logdir, 0755, _gf_true);
-        if ((ret == -1) && (EEXIST != errno)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Unable to create logdir %s",
-                        logdir);
-                goto out;
-        }
-        glusterd_svc_build_snapd_logfile (logfile, logdir, sizeof (logfile));
-        snprintf (volfileid, sizeof (volfileid), "snapd/%s", volinfo->volname);
-
-        if (dict_get_str (this->options, "transport.socket.bind-address",
-                          &volfileserver) != 0) {
-                volfileserver = "localhost";
-        }
-        ret = glusterd_proc_init (&(svc->proc), snapd_svc_name, pidfile, logdir,
-                                  logfile, volfile, volfileid, volfileserver);
-        if (ret)
-                goto out;
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    char sockpath[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char logdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    glusterd_svc_t *svc = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_conn_notify_t notify = NULL;
+    xlator_t *this = NULL;
+    char *volfileserver = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    volinfo = data;
+
+    svc = &(volinfo->snapd.svc);
+
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", snapd_svc_name);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    notify = glusterd_snapdsvc_rpc_notify;
+
+    glusterd_svc_build_snapd_rundir(volinfo, rundir, sizeof(rundir));
+    glusterd_svc_create_rundir(rundir);
+
+    /* Initialize the connection mgmt */
+    glusterd_svc_build_snapd_socket_filepath(volinfo, sockpath,
+                                             sizeof(sockpath));
+    ret = glusterd_conn_init(&(svc->conn), sockpath, 600, notify);
+    if (ret)
+        goto out;
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_snapd_pidfile(volinfo, pidfile, sizeof(pidfile));
+    glusterd_svc_build_snapd_volfile(volinfo, volfile, sizeof(volfile));
+    glusterd_svc_build_snapd_logdir(logdir, volinfo->volname, sizeof(logdir));
+    ret = mkdir_p(logdir, 0755, _gf_true);
+    if ((ret == -1) && (EEXIST != errno)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create logdir %s", logdir);
+        goto out;
+    }
+    glusterd_svc_build_snapd_logfile(logfile, logdir, sizeof(logfile));
+    len = snprintf(volfileid, sizeof(volfileid), "snapd/%s", volinfo->volname);
+    if ((len < 0) || (len >= sizeof(volfileid))) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_str(this->options, "transport.socket.bind-address",
+                     &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+    ret = glusterd_proc_init(&(svc->proc), snapd_svc_name, pidfile, logdir,
+                             logfile, volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_snapdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_snapdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                 ret     = 0;
-        xlator_t           *this    = THIS;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        volinfo = data;
-
-        if (!svc->inited) {
-                ret = glusterd_snapdsvc_init (volinfo);
-                if (ret) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPD_INIT_FAIL, "Failed to initialize "
-                                "snapd service for volume %s",
-                                volinfo->volname);
-                        goto out;
-                } else {
-                        svc->inited = _gf_true;
-                        gf_msg_debug (THIS->name, 0, "snapd service "
-                                      "initialized");
-                }
-        }
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_volinfo_t *volinfo = NULL;
 
-        ret = glusterd_is_snapd_enabled (volinfo);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_GET_FAIL, "Failed to read volume "
-                        "options");
-                goto out;
-        }
+    volinfo = data;
 
+    if (!svc->inited) {
+        ret = glusterd_snapdsvc_init(volinfo);
         if (ret) {
-                if (!glusterd_is_volume_started (volinfo)) {
-                        if (glusterd_proc_is_running (&svc->proc)) {
-                                ret = svc->stop (svc, SIGTERM);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_SNAPD_STOP_FAIL,
-                                                "Couldn't stop snapd for "
-                                                "volume: %s",
-                                                volinfo->volname);
-                        } else {
-                                /* Since snapd is not running set ret to 0 */
-                                ret = 0;
-                        }
-                        goto out;
-                }
-
-                ret = glusterd_snapdsvc_create_volfile (volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPD_CREATE_FAIL, "Couldn't create "
-                                "snapd volfile for volume: %s",
-                                volinfo->volname);
-                        goto out;
-                }
-
-                ret = svc->start (svc, flags);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPD_START_FAIL, "Couldn't start "
-                                "snapd for volume: %s", volinfo->volname);
-                        goto out;
-                }
-
-                glusterd_volinfo_ref (volinfo);
-                ret = glusterd_conn_connect (&(svc->conn));
-                if (ret) {
-                        glusterd_volinfo_unref (volinfo);
-                        goto out;
-                }
-
-        } else if (glusterd_proc_is_running (&svc->proc)) {
-                ret = svc->stop (svc, SIGTERM);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPD_STOP_FAIL,
-                                "Couldn't stop snapd for volume: %s",
-                                volinfo->volname);
-                        goto out;
-                }
-                volinfo->snapd.port = 0;
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_INIT_FAIL,
+                   "Failed to initialize "
+                   "snapd service for volume %s",
+                   volinfo->volname);
+            goto out;
+        } else {
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0,
+                         "snapd service "
+                         "initialized");
         }
-
-out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-
-        return ret;
-}
-
-int32_t
-glusterd_snapdsvc_start (glusterd_svc_t *svc, int flags)
-{
-        int                  ret                        = -1;
-        runner_t             runner                     = {0,};
-        glusterd_conf_t     *priv                       = NULL;
-        xlator_t            *this                       = NULL;
-        char                 valgrind_logfile[PATH_MAX] = {0};
-        int                  snapd_port                 = 0;
-        char                 msg[1024]                  = {0,};
-        char                 snapd_id[PATH_MAX]         = {0,};
-        glusterd_volinfo_t  *volinfo                    = NULL;
-        glusterd_snapdsvc_t *snapd                      = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (glusterd_proc_is_running (&svc->proc)) {
+    }
+
+    ret = glusterd_is_snapd_enabled(volinfo);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to read volume "
+               "options");
+        goto out;
+    }
+
+    if (ret) {
+        if (!glusterd_is_volume_started(volinfo)) {
+            if (glusterd_proc_is_running(&svc->proc)) {
+                ret = svc->stop(svc, SIGTERM);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                           "Couldn't stop snapd for "
+                           "volume: %s",
+                           volinfo->volname);
+            } else {
+                /* Since snapd is not running set ret to 0 */
                 ret = 0;
-                goto out;
+            }
+            goto out;
         }
 
-        /* Get volinfo->snapd from svc object */
-        snapd = cds_list_entry (svc, glusterd_snapdsvc_t, svc);
-        if (!snapd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPD_OBJ_GET_FAIL, "Failed to get snapd object "
-                        "from snapd service");
-                goto out;
-        }
-
-        /* Get volinfo from snapd */
-        volinfo = cds_list_entry (snapd, glusterd_volinfo_t, snapd);
-        if (!volinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_GET_FAIL, "Failed to get volinfo from "
-                        "from snapd");
-                goto out;
-        }
-
-        ret = sys_access (svc->proc.volfile, F_OK);
+        ret = glusterd_snapdsvc_create_volfile(volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_DEBUG, 0,
-                        GD_MSG_VOLINFO_GET_FAIL,
-                        "snapd Volfile %s is not present", svc->proc.volfile);
-                /* If glusterd is down on one of the nodes and during
-                 * that time "USS is enabled" for the first time. After some
-                 * time when the glusterd which was down comes back it tries
-                 * to look for the snapd volfile and it does not find snapd
-                 * volfile and because of this starting of snapd fails.
-                 * Therefore, if volfile is not present then create a fresh
-                 * volfile.
-                 */
-                ret = glusterd_snapdsvc_create_volfile (volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLFILE_CREATE_FAIL, "Couldn't create "
-                                "snapd volfile for volume: %s",
-                                volinfo->volname);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_CREATE_FAIL,
+                   "Couldn't create "
+                   "snapd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
-        runinit (&runner);
 
-        if (priv->valgrind) {
-                snprintf (valgrind_logfile, PATH_MAX, "%s/valgrind-snapd.log",
-                          svc->proc.logdir);
-
-                runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", "--track-origins=yes",
-                                 NULL);
-                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+        ret = svc->start(svc, flags);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                   "Couldn't start "
+                   "snapd for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
 
-        snprintf (snapd_id, sizeof (snapd_id), "snapd-%s", volinfo->volname);
-        runner_add_args (&runner, SBIN_DIR"/glusterfsd",
-                         "-s", svc->proc.volfileserver,
-                         "--volfile-id", svc->proc.volfileid,
-                         "-p", svc->proc.pidfile,
-                         "-l", svc->proc.logfile,
-                         "--brick-name", snapd_id,
-                         "-S", svc->conn.sockpath, NULL);
-
-        /* Do a pmap registry remove on the older connected port */
-        if (volinfo->snapd.port) {
-                ret = pmap_registry_remove (this, volinfo->snapd.port,
-                                            snapd_id, GF_PMAP_PORT_BRICKSERVER,
-                                            NULL);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Failed to remove pmap "
-                                  "registry for older signin");
-                        goto out;
-                }
+        glusterd_volinfo_ref(volinfo);
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret) {
+            glusterd_volinfo_unref(volinfo);
+            goto out;
         }
 
-        snapd_port = pmap_registry_alloc (THIS);
-        if (!snapd_port) {
-                snprintf (msg, sizeof (msg), "Could not allocate port "
-                          "for snapd service for volume %s",
-                          volinfo->volname);
-                runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-                ret = -1;
-                goto out;
+    } else if (glusterd_proc_is_running(&svc->proc)) {
+        ret = svc->stop(svc, SIGTERM);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                   "Couldn't stop snapd for volume: %s", volinfo->volname);
+            goto out;
         }
+        volinfo->snapd.port = 0;
+    }
 
-        volinfo->snapd.port = snapd_port;
+out:
+    if (ret) {
+        gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                 volinfo->volname, svc->name);
+    }
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        runner_add_arg (&runner, "--brick-port");
-        runner_argprintf (&runner, "%d", snapd_port);
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "%s-server.listen-port=%d",
-                         volinfo->volname, snapd_port);
-        runner_add_arg (&runner, "--no-mem-accounting");
+    return ret;
+}
 
-        snprintf (msg, sizeof (msg),
-                  "Starting the snapd service for volume %s", volinfo->volname);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
+int32_t
+glusterd_snapdsvc_start(glusterd_svc_t *svc, int flags)
+{
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char valgrind_logfile[PATH_MAX] = {0};
+    int snapd_port = 0;
+    char msg[1024] = {
+        0,
+    };
+    char snapd_id[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_snapdsvc_t *snapd = NULL;
+    char *localtime_logging = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (glusterd_proc_is_running(&svc->proc)) {
+        ret = 0;
+        goto out;
+    }
+
+    /* Get volinfo->snapd from svc object */
+    snapd = cds_list_entry(svc, glusterd_snapdsvc_t, svc);
+    if (!snapd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
+               "Failed to get snapd object "
+               "from snapd service");
+        goto out;
+    }
+
+    /* Get volinfo from snapd */
+    volinfo = cds_list_entry(snapd, glusterd_volinfo_t, snapd);
+    if (!volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from snapd");
+        goto out;
+    }
+
+    ret = sys_access(svc->proc.volfile, F_OK);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "snapd Volfile %s is not present", svc->proc.volfile);
+        /* If glusterd is down on one of the nodes and during
+         * that time "USS is enabled" for the first time. After some
+         * time when the glusterd which was down comes back it tries
+         * to look for the snapd volfile and it does not find snapd
+         * volfile and because of this starting of snapd fails.
+         * Therefore, if volfile is not present then create a fresh
+         * volfile.
+         */
+        ret = glusterd_snapdsvc_create_volfile(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Couldn't create "
+                   "snapd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+    }
+    runinit(&runner);
+
+    if (this->ctx->cmd_args.vgtool != _gf_none) {
+        len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-snapd.log",
+                       svc->proc.logdir);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
 
-        if (flags == PROC_START_NO_WAIT) {
-                ret = runner_run_nowait (&runner);
-        } else {
-                synclock_unlock (&priv->big_lock);
-                {
-                        ret = runner_run (&runner);
-                }
-                synclock_lock (&priv->big_lock);
+        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+            runner_add_args(&runner, "valgrind", "--leak-check=full",
+                            "--trace-children=yes", "--track-origins=yes",
+                            NULL);
+        else
+            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
+
+        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
+    }
+
+    snprintf(snapd_id, sizeof(snapd_id), "snapd-%s", volinfo->volname);
+    runner_add_args(&runner, SBIN_DIR "/glusterfsd", "-s",
+                    svc->proc.volfileserver, "--volfile-id",
+                    svc->proc.volfileid, "-p", svc->proc.pidfile, "-l",
+                    svc->proc.logfile, "--brick-name", snapd_id, "-S",
+                    svc->conn.sockpath, "--process-name", svc->name, NULL);
+    if (dict_get_str(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                     &localtime_logging) == 0) {
+        if (strcmp(localtime_logging, "enable") == 0)
+            runner_add_arg(&runner, "--localtime-logging");
+    }
+
+    snapd_port = pmap_assign_port(THIS, volinfo->snapd.port, snapd_id);
+    if (!snapd_port) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORTS_EXHAUSTED,
+               "All the ports in the range are exhausted, can't start "
+               "snapd for volume %s",
+               volinfo->volname);
+        ret = -1;
+        goto out;
+    }
+
+    volinfo->snapd.port = snapd_port;
+
+    runner_add_arg(&runner, "--brick-port");
+    runner_argprintf(&runner, "%d", snapd_port);
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "%s-server.listen-port=%d", volinfo->volname,
+                     snapd_port);
+    runner_add_arg(&runner, "--no-mem-accounting");
+
+    snprintf(msg, sizeof(msg), "Starting the snapd service for volume %s",
+             volinfo->volname);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    if (flags == PROC_START_NO_WAIT) {
+        ret = runner_run_nowait(&runner);
+    } else {
+        synclock_unlock(&priv->big_lock);
+        {
+            ret = runner_run(&runner);
         }
+        synclock_lock(&priv->big_lock);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapdsvc_restart ()
+glusterd_snapdsvc_restart()
 {
-        glusterd_volinfo_t      *volinfo        = NULL;
-        int                     ret             = 0;
-        xlator_t                *this           = THIS;
-        glusterd_conf_t         *conf           = NULL;
-        glusterd_svc_t          *svc            = NULL;
-
-        GF_ASSERT (this);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                /* Start per volume snapd svc */
-                if (volinfo->status == GLUSTERD_STATUS_STARTED) {
-                        svc = &(volinfo->snapd.svc);
-                        ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAPD_START_FAIL,
-                                        "Couldn't resolve snapd for "
-                                        "vol: %s on restart", volinfo->volname);
-                                goto out;
-                        }
-                }
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list)
+    {
+        /* Start per volume snapd svc */
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            svc = &(volinfo->snapd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                       "Couldn't resolve snapd for "
+                       "vol: %s on restart",
+                       volinfo->volname);
+                gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                goto out;
+            }
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapdsvc_rpc_notify (glusterd_conn_t *conn, rpc_clnt_event_t event)
+glusterd_snapdsvc_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event)
 {
-        int                  ret     = 0;
-        glusterd_svc_t      *svc     = NULL;
-        xlator_t            *this    = NULL;
-        glusterd_volinfo_t  *volinfo = NULL;
-        glusterd_snapdsvc_t *snapd   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        svc = cds_list_entry (conn, glusterd_svc_t, conn);
-        if (!svc) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SVC_GET_FAIL, "Failed to get the service");
-                return -1;
-        }
-
-        switch (event) {
+    int ret = 0;
+    glusterd_svc_t *svc = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_snapdsvc_t *snapd = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    svc = cds_list_entry(conn, glusterd_svc_t, conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        return -1;
+    }
+    snapd = cds_list_entry(svc, glusterd_snapdsvc_t, svc);
+    if (!snapd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
+               "Failed to get the "
+               "snapd object");
+        return -1;
+    }
+
+    volinfo = cds_list_entry(snapd, glusterd_volinfo_t, snapd);
+    if (!volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get the "
+               "volinfo object");
+        return -1;
+    }
+
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                gf_msg_debug (this->name, 0, "%s has connected with "
-                        "glusterd.", svc->name);
-                svc->online =  _gf_true;
-                break;
+            gf_msg_debug(this->name, 0,
+                         "%s has connected with "
+                         "glusterd.",
+                         svc->name);
+            gf_event(EVENT_SVC_CONNECTED, "volume=%s;svc_name=%s",
+                     volinfo->volname, svc->name);
+            svc->online = _gf_true;
+            break;
 
         case RPC_CLNT_DISCONNECT:
-                if (svc->online) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_NODE_DISCONNECTED, "%s has disconnected "
-                                "from glusterd.", svc->name);
-                        svc->online =  _gf_false;
-                }
-                break;
+            if (svc->online) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED,
+                       "%s has disconnected "
+                       "from glusterd.",
+                       svc->name);
+                gf_event(EVENT_SVC_DISCONNECTED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                svc->online = _gf_false;
+            }
+            break;
 
         case RPC_CLNT_DESTROY:
-                snapd = cds_list_entry (svc, glusterd_snapdsvc_t, svc);
-                if (!snapd) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPD_OBJ_GET_FAIL, "Failed to get the "
-                                "snapd object");
-                        return -1;
-                }
-
-                volinfo = cds_list_entry (snapd, glusterd_volinfo_t, snapd);
-                if (!volinfo) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL, "Failed to get the "
-                                "volinfo object");
-                        return -1;
-                }
-                glusterd_volinfo_unref (volinfo);
+            glusterd_volinfo_unref(volinfo);
+            break;
 
         default:
-                gf_msg_trace (this->name, 0,
-                        "got some other RPC event %d", event);
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h
index 40dae848f58..e15dbf54315 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h
@@ -15,28 +15,28 @@
 
 typedef struct glusterd_snapdsvc_ glusterd_snapdsvc_t;
 
-struct glusterd_snapdsvc_{
-        glusterd_svc_t     svc;
-        int                port;
-        gf_store_handle_t *handle;
+struct glusterd_snapdsvc_ {
+    glusterd_svc_t svc;
+    gf_store_handle_t *handle;
+    int port;
 };
 
 void
-glusterd_snapdsvc_build (glusterd_svc_t *svc);
+glusterd_snapdsvc_build(glusterd_svc_t *svc);
 
 int
-glusterd_snapdsvc_init (void *data);
+glusterd_snapdsvc_init(void *data);
 
 int
-glusterd_snapdsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_snapdsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_snapdsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_snapdsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_snapdsvc_restart ();
+glusterd_snapdsvc_restart();
 
 int
-glusterd_snapdsvc_rpc_notify (glusterd_conn_t *conn, rpc_clnt_event_t event);
+glusterd_snapdsvc_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
index 46c8bad1aa9..995268b796d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
@@ -16,8 +16,8 @@
 #endif
 #include <dlfcn.h>
 
-#include "dict.h"
-#include "syscall.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/syscall.h>
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-messages.h"
@@ -38,143 +38,148 @@
  */
 
 int32_t
-glusterd_snapobject_delete (glusterd_snap_t *snap)
+glusterd_snapobject_delete(glusterd_snap_t *snap)
 {
-        if (snap == NULL) {
-                gf_msg(THIS->name, GF_LOG_WARNING, 0,
-                       GD_MSG_PARAM_NULL, "snap is NULL");
-                return -1;
-        }
-
-        cds_list_del_init (&snap->snap_list);
-        cds_list_del_init (&snap->volumes);
-        if (LOCK_DESTROY(&snap->lock))
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        GD_MSG_LOCK_DESTROY_FAILED,
-                        "Failed destroying lock"
-                        "of snap %s", snap->snapname);
-
-        GF_FREE (snap->description);
-        GF_FREE (snap);
-
-        return 0;
+    if (snap == NULL) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_PARAM_NULL,
+               "snap is NULL");
+        return -1;
+    }
+
+    cds_list_del_init(&snap->snap_list);
+    cds_list_del_init(&snap->volumes);
+    if (LOCK_DESTROY(&snap->lock))
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_LOCK_DESTROY_FAILED,
+               "Failed destroying lock"
+               "of snap %s",
+               snap->snapname);
+
+    GF_FREE(snap->description);
+    GF_FREE(snap);
+
+    return 0;
 }
 
-
 /*
  * This function is to be called only from glusterd_peer_detach_cleanup()
- * as this continues to delete snaps inspite of faiure while deleting
+ * as this continues to delete snaps in spite of faiure while deleting
  * one, as we don't want to fail peer_detach in such a case.
  */
 int
-glusterd_cleanup_snaps_for_volume (glusterd_volinfo_t *volinfo)
+glusterd_cleanup_snaps_for_volume(glusterd_volinfo_t *volinfo)
 {
-        int32_t                  op_ret         = 0;
-        int32_t                  ret            = 0;
-        xlator_t                *this           = NULL;
-        glusterd_volinfo_t      *snap_vol       = NULL;
-        glusterd_volinfo_t      *dummy_snap_vol = NULL;
-        glusterd_snap_t         *snap           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        cds_list_for_each_entry_safe (snap_vol, dummy_snap_vol,
-                                      &volinfo->snap_volumes,
-                                      snapvol_list) {
-                ret = glusterd_store_delete_volume (snap_vol);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_WARNING, 0,
-                               GD_MSG_VOL_DELETE_FAIL, "Failed to remove "
-                               "volume %s from store", snap_vol->volname);
-                        op_ret = ret;
-                        continue;
-                }
+    int32_t op_ret = 0;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *dummy_snap_vol = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    cds_list_for_each_entry_safe(snap_vol, dummy_snap_vol,
+                                 &volinfo->snap_volumes, snapvol_list)
+    {
+        snap = snap_vol->snapshot;
+        ret = glusterd_store_delete_snap(snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to remove "
+                   "snap %s from store",
+                   snap->snapname);
+            op_ret = ret;
+            continue;
+        }
 
-                ret = glusterd_volinfo_delete (snap_vol);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_WARNING, 0,
-                               GD_MSG_VOL_DELETE_FAIL, "Failed to remove "
-                              "volinfo %s ", snap_vol->volname);
-                        op_ret = ret;
-                        continue;
-                }
+        ret = glusterd_snapobject_delete(snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to delete "
+                   "snap object %s",
+                   snap->snapname);
+            op_ret = ret;
+            continue;
+        }
 
-                snap = snap_vol->snapshot;
-                ret = glusterd_store_delete_snap (snap);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_WARNING, 0,
-                               GD_MSG_VOL_DELETE_FAIL, "Failed to remove "
-                               "snap %s from store", snap->snapname);
-                        op_ret = ret;
-                        continue;
-                }
+        ret = glusterd_store_delete_volume(snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to remove "
+                   "volume %s from store",
+                   snap_vol->volname);
+            op_ret = ret;
+            continue;
+        }
 
-                ret = glusterd_snapobject_delete (snap);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOL_DELETE_FAIL, "Failed to delete "
-                                "snap object %s", snap->snapname);
-                        op_ret = ret;
-                        continue;
-                }
+        ret = glusterd_volinfo_delete(snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to remove "
+                   "volinfo %s ",
+                   snap_vol->volname);
+            op_ret = ret;
+            continue;
         }
+    }
 
-        return op_ret;
+    return op_ret;
 }
 
-
-
 int
-glusterd_snap_geo_rep_restore (glusterd_volinfo_t *snap_volinfo,
-                               glusterd_volinfo_t *new_volinfo)
+glusterd_snap_geo_rep_restore(glusterd_volinfo_t *snap_volinfo,
+                              glusterd_volinfo_t *new_volinfo)
 {
-        char                    vol_tstamp_file[PATH_MAX]  = {0,};
-        char                    snap_tstamp_file[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv                      = NULL;
-        xlator_t                *this                      = NULL;
-        int                     geo_rep_indexing_on        = 0;
-        int                     ret                        = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_volinfo);
-        GF_ASSERT (new_volinfo);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Check if geo-rep indexing is enabled, if yes, we need restore
-         * back the mtime of 'marker.tstamp' file.
-         */
-        geo_rep_indexing_on = glusterd_volinfo_get_boolean (new_volinfo,
-                                                            VKEY_MARKER_XTIME);
-        if (geo_rep_indexing_on == -1) {
-                gf_msg_debug (this->name, 0, "Failed"
-                        " to check whether geo-rep-indexing enabled or not");
-                ret = 0;
-                goto out;
-        }
-
-        if (geo_rep_indexing_on == 1) {
-                GLUSTERD_GET_VOLUME_DIR (vol_tstamp_file, new_volinfo, priv);
-                strncat (vol_tstamp_file, "/marker.tstamp",
-                         PATH_MAX - strlen(vol_tstamp_file) - 1);
-                GLUSTERD_GET_VOLUME_DIR (snap_tstamp_file, snap_volinfo, priv);
-                strncat (snap_tstamp_file, "/marker.tstamp",
-                         PATH_MAX - strlen(snap_tstamp_file) - 1);
-                ret = gf_set_timestamp (snap_tstamp_file, vol_tstamp_file);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TSTAMP_SET_FAIL,
-                                "Unable to set atime and mtime of %s as of %s",
-                                vol_tstamp_file, snap_tstamp_file);
-                        goto out;
-                }
+    char vol_tstamp_file[PATH_MAX] = {
+        0,
+    };
+    char snap_tstamp_file[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int geo_rep_indexing_on = 0;
+    int ret = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_volinfo);
+    GF_ASSERT(new_volinfo);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Check if geo-rep indexing is enabled, if yes, we need restore
+     * back the mtime of 'marker.tstamp' file.
+     */
+    geo_rep_indexing_on = glusterd_volinfo_get_boolean(new_volinfo,
+                                                       VKEY_MARKER_XTIME);
+    if (geo_rep_indexing_on == -1) {
+        gf_msg_debug(this->name, 0,
+                     "Failed"
+                     " to check whether geo-rep-indexing enabled or not");
+        ret = 0;
+        goto out;
+    }
+
+    if (geo_rep_indexing_on == 1) {
+        GLUSTERD_GET_VOLUME_DIR(vol_tstamp_file, new_volinfo, priv);
+        strncat(vol_tstamp_file, "/marker.tstamp",
+                PATH_MAX - strlen(vol_tstamp_file) - 1);
+        GLUSTERD_GET_VOLUME_DIR(snap_tstamp_file, snap_volinfo, priv);
+        strncat(snap_tstamp_file, "/marker.tstamp",
+                PATH_MAX - strlen(snap_tstamp_file) - 1);
+        ret = gf_set_timestamp(snap_tstamp_file, vol_tstamp_file);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TSTAMP_SET_FAIL,
+                   "Unable to set atime and mtime of %s as of %s",
+                   vol_tstamp_file, snap_tstamp_file);
+            goto out;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will copy snap volinfo to the new
@@ -189,342 +194,330 @@ out:
  * TODO: Duplicate all members of volinfo, e.g. geo-rep sync slaves
  */
 int32_t
-glusterd_snap_volinfo_restore (dict_t *dict, dict_t *rsp_dict,
-                               glusterd_volinfo_t *new_volinfo,
-                               glusterd_volinfo_t *snap_volinfo,
-                               int32_t volcount)
+glusterd_snap_volinfo_restore(dict_t *dict, dict_t *rsp_dict,
+                              glusterd_volinfo_t *new_volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              int32_t volcount)
 {
-        char                    *value          = NULL;
-        char                    key[PATH_MAX]   = "";
-        int32_t                 brick_count     = -1;
-        int32_t                 ret             = -1;
-        xlator_t                *this           = NULL;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        glusterd_brickinfo_t    *new_brickinfo  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        GF_VALIDATE_OR_GOTO (this->name, new_volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, snap_volinfo, out);
-
-        brick_count = 0;
-        cds_list_for_each_entry (brickinfo, &snap_volinfo->bricks, brick_list) {
-                brick_count++;
-                ret = glusterd_brickinfo_new (&new_brickinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_NEW_INFO_FAIL, "Failed to create "
-                                "new brickinfo");
-                        goto out;
-                }
-
-                /* Duplicate brickinfo */
-                ret = glusterd_brickinfo_dup (brickinfo, new_brickinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_SET_INFO_FAIL, "Failed to dup "
-                                "brickinfo");
-                        goto out;
-                }
-
-                /* Fetch values if present in dict These values won't
-                 * be present in case of a missed restore. In that case
-                 * it's fine to use the local node's value
-                 */
-                snprintf (key, sizeof (key), "snap%d.brick%d.path",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->path, value,
-                                 sizeof(new_brickinfo->path));
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.snap_status",
-                          volcount, brick_count);
-                ret = dict_get_int32 (dict, key, &new_brickinfo->snap_status);
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.device_path",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->device_path, value,
-                                 sizeof(new_brickinfo->device_path));
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.fs_type",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->fstype, value,
-                                 sizeof(new_brickinfo->fstype));
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.mnt_opts",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->mnt_opts, value,
-                                 sizeof(new_brickinfo->mnt_opts));
-
-                /* If the brick is not of this peer, or snapshot is missed *
-                 * for the brick do not replace the xattr for it */
-                if ((!gf_uuid_compare (brickinfo->uuid, MY_UUID)) &&
-                    (brickinfo->snap_status != -1)) {
-                        /* We need to replace the volume id of all the bricks
-                         * to the volume id of the origin volume. new_volinfo
-                         * has the origin volume's volume id*/
-                        ret = sys_lsetxattr (new_brickinfo->path,
-                                             GF_XATTR_VOL_ID_KEY,
-                                             new_volinfo->volume_id,
-                                             sizeof (new_volinfo->volume_id),
-                                             XATTR_REPLACE);
-                        if (ret == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SETXATTR_FAIL, "Failed to "
-                                        "set extended attribute %s on %s. "
-                                        "Reason: %s, snap: %s",
-                                        GF_XATTR_VOL_ID_KEY,
-                                        new_brickinfo->path, strerror (errno),
-                                        new_volinfo->volname);
-                                goto out;
-                        }
-                }
-
-                /* If a snapshot is pending for this brick then
-                 * restore should also be pending
-                 */
-                if (brickinfo->snap_status == -1) {
-                        /* Adding missed delete to the dict */
-                        ret = glusterd_add_missed_snaps_to_dict
-                                                (rsp_dict,
-                                                 snap_volinfo,
-                                                 brickinfo,
-                                                 brick_count,
-                                                 GF_SNAP_OPTION_TYPE_RESTORE);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
-                                        "Failed to add missed snapshot info "
-                                        "for %s:%s in the rsp_dict",
-                                        brickinfo->hostname,
-                                        brickinfo->path);
-                                goto out;
-                        }
-                }
-
-                cds_list_add_tail (&new_brickinfo->brick_list,
-                                   &new_volinfo->bricks);
-                /* ownership of new_brickinfo is passed to new_volinfo */
-                new_brickinfo = NULL;
+    char *value = NULL;
+    char key[64] = "";
+    int32_t brick_count = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    GF_VALIDATE_OR_GOTO(this->name, new_volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, snap_volinfo, out);
+
+    brick_count = 0;
+    cds_list_for_each_entry(brickinfo, &snap_volinfo->bricks, brick_list)
+    {
+        brick_count++;
+        ret = glusterd_brickinfo_new(&new_brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NEW_INFO_FAIL,
+                   "Failed to create "
+                   "new brickinfo");
+            goto out;
         }
 
-        /* Regenerate all volfiles */
-        ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
+        /* Duplicate brickinfo */
+        ret = glusterd_brickinfo_dup(brickinfo, new_brickinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "Failed to regenerate volfiles");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SET_INFO_FAIL,
+                   "Failed to dup "
+                   "brickinfo");
+            goto out;
         }
 
-        /* Restore geo-rep marker.tstamp's timestamp */
-        ret = glusterd_snap_geo_rep_restore (snap_volinfo, new_volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_TSTAMP_SET_FAIL,
-                        "Geo-rep: marker.tstamp's timestamp restoration failed");
+        /* Fetch values if present in dict These values won't
+         * be present in case of a missed restore. In that case
+         * it's fine to use the local node's value
+         */
+        snprintf(key, sizeof(key), "snap%d.brick%d.path", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->path, value, sizeof(new_brickinfo->path));
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.snap_status", volcount,
+                 brick_count);
+        ret = dict_get_int32(dict, key, &new_brickinfo->snap_status);
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.device_path", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->device_path, value,
+                       sizeof(new_brickinfo->device_path));
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.fs_type", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->fstype, value,
+                       sizeof(new_brickinfo->fstype));
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.mnt_opts", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->mnt_opts, value,
+                       sizeof(new_brickinfo->mnt_opts));
+
+        /* If the brick is not of this peer, or snapshot is missed *
+         * for the brick do not replace the xattr for it */
+        if ((!gf_uuid_compare(brickinfo->uuid, MY_UUID)) &&
+            (brickinfo->snap_status != -1)) {
+            /* We need to replace the volume id of all the bricks
+             * to the volume id of the origin volume. new_volinfo
+             * has the origin volume's volume id*/
+            ret = sys_lsetxattr(new_brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                new_volinfo->volume_id,
+                                sizeof(new_volinfo->volume_id), XATTR_REPLACE);
+            if (ret == -1) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SET_XATTR_FAIL,
+                        "Attribute=%s, Path=%s, Reason=%s, Snap=%s",
+                        GF_XATTR_VOL_ID_KEY, new_brickinfo->path,
+                        strerror(errno), new_volinfo->volname, NULL);
                 goto out;
+            }
         }
 
+        /* If a snapshot is pending for this brick then
+         * restore should also be pending
+         */
+        if (brickinfo->snap_status == -1) {
+            /* Adding missed delete to the dict */
+            ret = glusterd_add_missed_snaps_to_dict(
+                rsp_dict, snap_volinfo, brickinfo, brick_count,
+                GF_SNAP_OPTION_TYPE_RESTORE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+                       "Failed to add missed snapshot info "
+                       "for %s:%s in the rsp_dict",
+                       brickinfo->hostname, brickinfo->path);
+                goto out;
+            }
+        }
+
+        cds_list_add_tail(&new_brickinfo->brick_list, &new_volinfo->bricks);
+        /* ownership of new_brickinfo is passed to new_volinfo */
+        new_brickinfo = NULL;
+    }
+
+    /* Regenerate all volfiles */
+    ret = glusterd_create_volfiles_and_notify_services(new_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to regenerate volfiles");
+        goto out;
+    }
+
+    /* Restore geo-rep marker.tstamp's timestamp */
+    ret = glusterd_snap_geo_rep_restore(snap_volinfo, new_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TSTAMP_SET_FAIL,
+               "Geo-rep: marker.tstamp's timestamp restoration failed");
+        goto out;
+    }
+
 out:
-        if (ret && (NULL != new_brickinfo)) {
-                (void) glusterd_brickinfo_delete (new_brickinfo);
-        }
+    if (ret && (NULL != new_brickinfo)) {
+        (void)glusterd_brickinfo_delete(new_brickinfo);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
-                                         glusterd_volinfo_t **volinfo)
+glusterd_snap_volinfo_find_by_volume_id(uuid_t volume_id,
+                                        glusterd_volinfo_t **volinfo)
 {
-        int32_t                  ret     = -1;
-        xlator_t                *this    = NULL;
-        glusterd_volinfo_t      *voliter = NULL;
-        glusterd_snap_t         *snap    = NULL;
-        glusterd_conf_t         *priv    = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        if (gf_uuid_is_null(volume_id)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_UUID_NULL, "Volume UUID is NULL");
-                goto out;
-        }
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                cds_list_for_each_entry (voliter, &snap->volumes, vol_list) {
-                        if (gf_uuid_compare (volume_id, voliter->volume_id))
-                                continue;
-                        *volinfo = voliter;
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        gf_msg (this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_NOT_FOUND,
-                "Snap volume not found");
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    if (gf_uuid_is_null(volume_id)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_NULL,
+               "Volume UUID is NULL");
+        goto out;
+    }
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        cds_list_for_each_entry(voliter, &snap->volumes, vol_list)
+        {
+            if (gf_uuid_compare(volume_id, voliter->volume_id))
+                continue;
+            *volinfo = voliter;
+            ret = 0;
+            goto out;
+        }
+    }
+
+    gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_NOT_FOUND,
+           "Snap volume not found");
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_snap_volinfo_find (char *snap_volname, glusterd_snap_t *snap,
-                            glusterd_volinfo_t **volinfo)
+glusterd_snap_volinfo_find(char *snap_volname, glusterd_snap_t *snap,
+                           glusterd_volinfo_t **volinfo)
 {
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
-        glusterd_volinfo_t      *snap_vol    = NULL;
-        glusterd_conf_t         *priv        = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-        GF_ASSERT (snap_volname);
-
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                if (!strcmp (snap_vol->volname, snap_volname)) {
-                        ret = 0;
-                        *volinfo = snap_vol;
-                        goto out;
-                }
-        }
-
-        gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                GD_MSG_SNAP_NOT_FOUND, "Snap volume %s not found",
-                snap_volname);
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+    GF_ASSERT(snap_volname);
+
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        if (!strcmp(snap_vol->volname, snap_volname)) {
+            ret = 0;
+            *volinfo = snap_vol;
+            goto out;
+        }
+    }
+
+    gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+           "Snap volume %s not found", snap_volname);
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
-                                      glusterd_snap_t *snap,
-                                      glusterd_volinfo_t **volinfo)
+glusterd_snap_volinfo_find_from_parent_volname(char *origin_volname,
+                                               glusterd_snap_t *snap,
+                                               glusterd_volinfo_t **volinfo)
 {
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
-        glusterd_volinfo_t      *snap_vol    = NULL;
-        glusterd_conf_t         *priv        = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-        GF_ASSERT (origin_volname);
-
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                if (!strcmp (snap_vol->parent_volname, origin_volname)) {
-                        ret = 0;
-                        *volinfo = snap_vol;
-                        goto out;
-                }
-        }
-
-        gf_msg_debug (this->name, 0, "Snap volume not found(snap: %s, "
-                "origin-volume: %s", snap->snapname, origin_volname);
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+    GF_ASSERT(origin_volname);
+
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        if (!strcmp(snap_vol->parent_volname, origin_volname)) {
+            ret = 0;
+            *volinfo = snap_vol;
+            goto out;
+        }
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Snap volume not found(snap: %s, "
+                 "origin-volume: %s",
+                 snap->snapname, origin_volname);
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Exports a bricks snapshot details only if required
  *
- * The details will be exported only if the cluster op-version is greather than
+ * The details will be exported only if the cluster op-version is greater than
  * 4, ie. snapshot is supported in the cluster
  */
 int
-gd_add_brick_snap_details_to_dict (dict_t *dict, char *prefix,
-                                   glusterd_brickinfo_t *brickinfo)
+gd_add_brick_snap_details_to_dict(dict_t *dict, char *prefix,
+                                  glusterd_brickinfo_t *brickinfo)
 {
-        int ret = -1;
-        xlator_t *this = NULL;
-        glusterd_conf_t *conf = NULL;
-        char key[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (brickinfo != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap_status", prefix);
-        ret = dict_set_int32 (dict, key, brickinfo->snap_status);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_STATUS_FAIL,
-                        "Failed to set snap_status for %s:%s",
-                        brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.device_path", prefix);
-        ret = dict_set_str (dict, key, brickinfo->device_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set snap_device for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.fs_type", prefix);
-        ret = dict_set_str (dict, key, brickinfo->fstype);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set fstype for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.mnt_opts", prefix);
-        ret = dict_set_str (dict, key, brickinfo->mnt_opts);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRK_MOUNTOPTS_FAIL,
-                        "Failed to set mnt_opts for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mount_dir", prefix);
-        ret = dict_set_str (dict, key, brickinfo->mount_dir);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to set mount_dir for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[256] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (brickinfo != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.snap_status", prefix);
+    ret = dict_set_int32(dict, key, brickinfo->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+               "Failed to set snap_status for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.device_path", prefix);
+    ret = dict_set_str(dict, key, brickinfo->device_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap_device for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.fs_type", prefix);
+    ret = dict_set_str(dict, key, brickinfo->fstype);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set fstype for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.mnt_opts", prefix);
+    ret = dict_set_str(dict, key, brickinfo->mnt_opts);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MOUNTOPTS_FAIL,
+               "Failed to set mnt_opts for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.mount_dir", prefix);
+    ret = dict_set_str(dict, key, brickinfo->mount_dir);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to set mount_dir for %s:%s", brickinfo->hostname,
+               brickinfo->path);
 
 out:
-        return ret;
+    return ret;
 }
 
 /* Exports a volumes snapshot details only if required.
@@ -533,323 +526,294 @@ out:
  * greater than 4, ie. snapshot is supported in the cluster
  */
 int
-gd_add_vol_snap_details_to_dict (dict_t *dict, char *prefix,
-                                 glusterd_volinfo_t *volinfo)
+gd_add_vol_snap_details_to_dict(dict_t *dict, char *prefix,
+                                glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
-        xlator_t *this = NULL;
-        glusterd_conf_t *conf = NULL;
-        char key[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.restored_from_snap", prefix);
-        ret = dict_set_dynstr_with_alloc
-                                  (dict, key,
-                                   uuid_utoa (volinfo->restored_from_snap));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Unable to set %s for volume"
-                        "%s", key, volinfo->volname);
-                goto out;
-        }
-
-        if (strlen (volinfo->parent_volname) > 0) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.parent_volname", prefix);
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                                  volinfo->parent_volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Unable to set %s "
-                                "for volume %s", key, volinfo->volname);
-                        goto out;
-                }
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.is_snap_volume", prefix);
-        ret = dict_set_uint32 (dict, key, volinfo->is_snap_volume);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Unable to set %s for volume"
-                        "%s", key, volinfo->volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.snap-max-hard-limit", prefix);
-        ret = dict_set_uint64 (dict, key, volinfo->snap_max_hard_limit);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[256] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.restored_from_snap", prefix);
+    ret = dict_set_dynstr_with_alloc(dict, key,
+                                     uuid_utoa(volinfo->restored_from_snap));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set %s for volume"
+               "%s",
+               key, volinfo->volname);
+        goto out;
+    }
+
+    if (strlen(volinfo->parent_volname) > 0) {
+        snprintf(key, sizeof(key), "%s.parent_volname", prefix);
+        ret = dict_set_dynstr_with_alloc(dict, key, volinfo->parent_volname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Unable to set %s for volume"
-                        "%s", key, volinfo->volname);
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set %s "
+                   "for volume %s",
+                   key, volinfo->volname);
+            goto out;
+        }
+    }
+
+    snprintf(key, sizeof(key), "%s.is_snap_volume", prefix);
+    ret = dict_set_uint32(dict, key, volinfo->is_snap_volume);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set %s for volume"
+               "%s",
+               key, volinfo->volname);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.snap-max-hard-limit", prefix);
+    ret = dict_set_uint64(dict, key, volinfo->snap_max_hard_limit);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set %s for volume"
+               "%s",
+               key, volinfo->volname);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_add_missed_snaps_to_export_dict (dict_t *peer_data)
+glusterd_add_missed_snaps_to_export_dict(dict_t *peer_data)
 {
-        char                           name_buf[PATH_MAX]   = "";
-        char                           value[PATH_MAX]      = "";
-        int32_t                        missed_snap_count    = 0;
-        int32_t                        ret                  = -1;
-        glusterd_conf_t               *priv                 = NULL;
-        glusterd_missed_snap_info     *missed_snapinfo      = NULL;
-        glusterd_snap_op_t            *snap_opinfo          = NULL;
-        xlator_t                      *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Add the missed_entries in the dict */
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        snprintf (name_buf, sizeof(name_buf),
-                                  "missed_snaps_%d", missed_snap_count);
-                        snprintf (value, sizeof(value), "%s:%s=%s:%d:%s:%d:%d",
-                                  missed_snapinfo->node_uuid,
-                                  missed_snapinfo->snap_uuid,
-                                  snap_opinfo->snap_vol_id,
-                                  snap_opinfo->brick_num,
-                                  snap_opinfo->brick_path,
-                                  snap_opinfo->op,
-                                  snap_opinfo->status);
-
-                        ret = dict_set_dynstr_with_alloc (peer_data, name_buf,
-                                                          value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to set %s",
-                                        name_buf);
-                                goto out;
-                        }
-                        missed_snap_count++;
-                }
-        }
-
-        ret = dict_set_int32 (peer_data, "missed_snap_count",
-                              missed_snap_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set missed_snap_count");
+    char name_buf[PATH_MAX] = "";
+    char value[PATH_MAX] = "";
+    int32_t missed_snap_count = 0;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Add the missed_entries in the dict */
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d",
+                     missed_snap_count);
+            snprintf(value, sizeof(value), "%s:%s=%s:%d:%s:%d:%d",
+                     missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid,
+                     snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                     snap_opinfo->brick_path, snap_opinfo->op,
+                     snap_opinfo->status);
+
+            ret = dict_set_dynstr_with_alloc(peer_data, name_buf, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set %s", name_buf);
                 goto out;
+            }
+            missed_snap_count++;
         }
+    }
+
+    ret = dict_set_int32(peer_data, "missed_snap_count", missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set missed_snap_count");
+        goto out;
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_add_snap_to_dict (glusterd_snap_t *snap, dict_t *peer_data,
-                           int32_t snap_count)
+glusterd_add_snap_to_dict(glusterd_snap_t *snap, dict_t *peer_data,
+                          int32_t snap_count)
 {
-        char                    buf[NAME_MAX]    = "";
-        char                    prefix[NAME_MAX] = "";
-        int32_t                 ret              = -1;
-        int32_t                 volcount         = 0;
-        glusterd_volinfo_t     *volinfo          = NULL;
-        glusterd_brickinfo_t   *brickinfo        = NULL;
-        gf_boolean_t            host_bricks      = _gf_false;
-        xlator_t               *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap);
-        GF_ASSERT (peer_data);
-
-        snprintf (prefix, sizeof(prefix), "snap%d", snap_count);
-
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                volcount++;
-                ret = glusterd_add_volume_to_dict (volinfo, peer_data,
-                                                   volcount, prefix);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to add snap:%s volume:%s "
-                                "to peer_data dict for handshake",
-                                snap->snapname, volinfo->volname);
-                        goto out;
-                }
-
-                if (glusterd_is_volume_quota_enabled (volinfo)) {
-
-                        ret = glusterd_vol_add_quota_conf_to_dict (volinfo,
-                                                                   peer_data,
-                                                                   volcount,
-                                                                   prefix);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to add quota conf for "
-                                        "snap:%s volume:%s to peer_data "
-                                        "dict for handshake", snap->snapname,
-                                        volinfo->volname);
-                                goto out;
-                        }
-                }
-
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                host_bricks = _gf_true;
-                                break;
-                        }
-                }
-        }
-
-        snprintf (buf, sizeof(buf), "%s.host_bricks", prefix);
-        ret = dict_set_int8 (peer_data, buf, (int8_t) host_bricks);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set host_bricks for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.volcount", prefix);
-        ret = dict_set_int32 (peer_data, buf, volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set volcount for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snapname", prefix);
-        ret = dict_set_dynstr_with_alloc (peer_data, buf, snap->snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set snapname for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_id", prefix);
-        ret = dict_set_dynstr_with_alloc (peer_data, buf,
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set snap_id for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        if (snap->description) {
-                snprintf (buf, sizeof(buf), "%s.snapid", prefix);
-                ret = dict_set_dynstr_with_alloc (peer_data, buf,
-                                                  snap->description);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set description for snap %s",
-                                snap->snapname);
-                        goto out;
-                }
-        }
-
-        snprintf (buf, sizeof(buf), "%s.time_stamp", prefix);
-        ret = dict_set_int64 (peer_data, buf, (int64_t)snap->time_stamp);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set time_stamp for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_restored", prefix);
-        ret = dict_set_int8 (peer_data, buf, snap->snap_restored);
+    char buf[64] = "";
+    char prefix[32] = "";
+    int32_t ret = -1;
+    int32_t volcount = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gf_boolean_t host_bricks = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+    GF_ASSERT(peer_data);
+
+    snprintf(prefix, sizeof(prefix), "snap%d", snap_count);
+
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        volcount++;
+        ret = glusterd_add_volume_to_dict(volinfo, peer_data, volcount, prefix);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set snap_restored for snap %s",
-                        snap->snapname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add snap:%s volume:%s "
+                   "to peer_data dict for handshake",
+                   snap->snapname, volinfo->volname);
+            goto out;
+        }
+
+        if (glusterd_is_volume_quota_enabled(volinfo)) {
+            ret = glusterd_vol_add_quota_conf_to_dict(volinfo, peer_data,
+                                                      volcount, prefix);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to add quota conf for "
+                       "snap:%s volume:%s to peer_data "
+                       "dict for handshake",
+                       snap->snapname, volinfo->volname);
                 goto out;
+            }
         }
 
-        snprintf (buf, sizeof(buf), "%s.snap_status", prefix);
-        ret = dict_set_int32 (peer_data, buf, snap->snap_status);
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                host_bricks = _gf_true;
+                break;
+            }
+        }
+    }
+
+    snprintf(buf, sizeof(buf), "%s.host_bricks", prefix);
+    ret = dict_set_int8(peer_data, buf, (int8_t)host_bricks);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set host_bricks for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.volcount", prefix);
+    ret = dict_set_int32(peer_data, buf, volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set volcount for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snapname", prefix);
+    ret = dict_set_dynstr_with_alloc(peer_data, buf, snap->snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snapname for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_id", prefix);
+    ret = dict_set_dynstr_with_alloc(peer_data, buf, uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap_id for snap %s", snap->snapname);
+        goto out;
+    }
+
+    if (snap->description) {
+        snprintf(buf, sizeof(buf), "%s.description", prefix);
+        ret = dict_set_dynstr_with_alloc(peer_data, buf, snap->description);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set snap_status for snap %s",
-                        snap->snapname);
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set description for snap %s", snap->snapname);
+            goto out;
+        }
+    }
+
+    snprintf(buf, sizeof(buf), "%s.time_stamp", prefix);
+    ret = dict_set_int64(peer_data, buf, (int64_t)snap->time_stamp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set time_stamp for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_restored", prefix);
+    ret = dict_set_int8(peer_data, buf, snap->snap_restored);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap_restored for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_status", prefix);
+    ret = dict_set_int32(peer_data, buf, snap->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap_status for snap %s", snap->snapname);
+        goto out;
+    }
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_add_snapshots_to_export_dict (dict_t *peer_data)
+glusterd_add_snapshots_to_export_dict(dict_t *peer_data)
 {
-        int32_t                 snap_count = 0;
-        int32_t                 ret        = -1;
-        glusterd_conf_t        *priv       = NULL;
-        glusterd_snap_t        *snap       = NULL;
-        xlator_t               *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (peer_data);
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                snap_count++;
-                ret = glusterd_add_snap_to_dict (snap, peer_data, snap_count);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to add snap(%s) to the "
-                                " peer_data dict for handshake",
-                                snap->snapname);
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (peer_data, "snap_count", snap_count);
+    int32_t snap_count = 0;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(peer_data);
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        snap_count++;
+        ret = glusterd_add_snap_to_dict(snap, peer_data, snap_count);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap_count");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add snap(%s) to the "
+                   " peer_data dict for handshake",
+                   snap->snapname);
+            goto out;
         }
+    }
+
+    ret = dict_set_int32(peer_data, "snap_count", snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap_count");
+        goto out;
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Imports the snapshot details of a brick if required and available
@@ -857,82 +821,80 @@ out:
  * Snapshot details will be imported only if the cluster op-version is >= 4
  */
 int
-gd_import_new_brick_snap_details (dict_t *dict, char *prefix,
-                                  glusterd_brickinfo_t *brickinfo)
+gd_import_new_brick_snap_details(dict_t *dict, char *prefix,
+                                 glusterd_brickinfo_t *brickinfo)
 {
-        int              ret         = -1;
-        xlator_t        *this        = NULL;
-        glusterd_conf_t *conf        = NULL;
-        char             key[512]    = {0,};
-        char            *snap_device = NULL;
-        char            *fs_type     = NULL;
-        char            *mnt_opts    = NULL;
-        char            *mount_dir   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (brickinfo != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap_status", prefix);
-        ret = dict_get_int32 (dict, key, &brickinfo->snap_status);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s missing in payload", key);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.device_path", prefix);
-        ret = dict_get_str (dict, key, &snap_device);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s missing in payload", key);
-                goto out;
-        }
-        strcpy (brickinfo->device_path, snap_device);
-
-        snprintf (key, sizeof (key), "%s.fs_type", prefix);
-        ret = dict_get_str (dict, key, &fs_type);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s missing in payload", key);
-                goto out;
-        }
-        strcpy (brickinfo->fstype, fs_type);
-
-        snprintf (key, sizeof (key), "%s.mnt_opts", prefix);
-        ret = dict_get_str (dict, key, &mnt_opts);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s missing in payload", key);
-                goto out;
-        }
-        strcpy (brickinfo->mnt_opts, mnt_opts);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mount_dir", prefix);
-        ret = dict_get_str (dict, key, &mount_dir);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "%s missing in payload", key);
-                goto out;
-        }
-        strncpy (brickinfo->mount_dir, mount_dir,
-                 (sizeof (brickinfo->mount_dir) - 1));
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[512] = {
+        0,
+    };
+    char *snap_device = NULL;
+    char *fs_type = NULL;
+    char *mnt_opts = NULL;
+    char *mount_dir = NULL;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (brickinfo != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.snap_status", prefix);
+    ret = dict_get_int32(dict, key, &brickinfo->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.device_path", prefix);
+    ret = dict_get_str(dict, key, &snap_device);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->device_path, snap_device,
+               sizeof(brickinfo->device_path));
+    snprintf(key, sizeof(key), "%s.fs_type", prefix);
+    ret = dict_get_str(dict, key, &fs_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->fstype, fs_type, sizeof(brickinfo->fstype));
+
+    snprintf(key, sizeof(key), "%s.mnt_opts", prefix);
+    ret = dict_get_str(dict, key, &mnt_opts);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->mnt_opts, mnt_opts, sizeof(brickinfo->mnt_opts));
+
+    snprintf(key, sizeof(key), "%s.mount_dir", prefix);
+    ret = dict_get_str(dict, key, &mount_dir);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->mount_dir, mount_dir, sizeof(brickinfo->mount_dir));
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -943,335 +905,327 @@ out:
  * supported.
  */
 int
-gd_import_volume_snap_details (dict_t *dict, glusterd_volinfo_t *volinfo,
-                               char *prefix, char *volname)
+gd_import_volume_snap_details(dict_t *dict, glusterd_volinfo_t *volinfo,
+                              char *prefix, char *volname)
 {
-        int              ret           = -1;
-        xlator_t        *this          = NULL;
-        glusterd_conf_t *conf          = NULL;
-        char             key[256]      = {0,};
-        char            *restored_snap = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volname != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.is_snap_volume", prefix);
-        ret = dict_get_uint32 (dict, key, &volinfo->is_snap_volume);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s missing in payload "
-                        "for %s", key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.restored_from_snap", prefix);
-        ret = dict_get_str (dict, key, &restored_snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s missing in payload "
-                        "for %s", key, volname);
-                goto out;
-        }
-
-        gf_uuid_parse (restored_snap, volinfo->restored_from_snap);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.snap-max-hard-limit", prefix);
-        ret = dict_get_uint64 (dict, key,
-                               &volinfo->snap_max_hard_limit);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "%s missing in payload "
-                        "for %s", key, volname);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[256] = {
+        0,
+    };
+    char *restored_snap = NULL;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volname != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.is_snap_volume", prefix);
+    uint32_t is_snap_int;
+    ret = dict_get_uint32(dict, key, &is_snap_int);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload "
+               "for %s",
+               key, volname);
+        goto out;
+    }
+    volinfo->is_snap_volume = (is_snap_int != 0);
+
+    snprintf(key, sizeof(key), "%s.restored_from_snap", prefix);
+    ret = dict_get_str(dict, key, &restored_snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload "
+               "for %s",
+               key, volname);
+        goto out;
+    }
+
+    gf_uuid_parse(restored_snap, volinfo->restored_from_snap);
+
+    snprintf(key, sizeof(key), "%s.snap-max-hard-limit", prefix);
+    ret = dict_get_uint64(dict, key, &volinfo->snap_max_hard_limit);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload "
+               "for %s",
+               key, volname);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_perform_missed_op (glusterd_snap_t *snap, int32_t op)
+glusterd_perform_missed_op(glusterd_snap_t *snap, int32_t op)
 {
-        dict_t                  *dict           = NULL;
-        int32_t                  ret            = -1;
-        glusterd_conf_t         *priv           = NULL;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        glusterd_volinfo_t      *tmp            = NULL;
-        xlator_t                *this           = NULL;
-        uuid_t                   null_uuid      = {0};
-        char                    *parent_volname = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-
-        dict = dict_new();
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL, "Unable to create dict");
-                ret = -1;
+    dict_t *dict = NULL;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    xlator_t *this = NULL;
+    uuid_t null_uuid = {0};
+    char *parent_volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        ret = -1;
+        goto out;
+    }
+
+    switch (op) {
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = glusterd_snap_remove(dict, snap, _gf_true, _gf_false,
+                                       _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to remove snap");
                 goto out;
-        }
+            }
 
-        switch (op) {
-        case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_false,
-                                            _gf_false);
+            break;
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            cds_list_for_each_entry_safe(snap_volinfo, tmp, &snap->volumes,
+                                         vol_list)
+            {
+                parent_volname = gf_strdup(snap_volinfo->parent_volname);
+                if (!parent_volname)
+                    goto out;
+
+                ret = glusterd_volinfo_find(parent_volname, &volinfo);
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL,
-                                "Failed to remove snap");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                           "Could not get volinfo of %s", parent_volname);
+                    goto out;
                 }
 
-                break;
-        case GF_SNAP_OPTION_TYPE_RESTORE:
-                cds_list_for_each_entry_safe (snap_volinfo, tmp, &snap->volumes,
-                                              vol_list) {
-                        parent_volname = gf_strdup
-                                            (snap_volinfo->parent_volname);
-                        if (!parent_volname)
-                                goto out;
-
-                        ret = glusterd_volinfo_find (parent_volname, &volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOLINFO_GET_FAIL,
-                                        "Could not get volinfo of %s",
-                                        parent_volname);
-                                goto out;
-                        }
-
-                        volinfo->version--;
-                        gf_uuid_copy (volinfo->restored_from_snap, null_uuid);
-
-                        /* gd_restore_snap_volume() uses the dict and volcount
-                         * to fetch snap brick info from other nodes, which were
-                         * collected during prevalidation. As this is an ad-hoc
-                         * op and only local node's data matter, hence sending
-                         * volcount as 0 and re-using the same dict because we
-                         * need not record any missed creates in the rsp_dict.
-                         */
-                        ret = gd_restore_snap_volume (dict, dict, volinfo,
-                                                      snap_volinfo, 0);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_RESTORE_FAIL,
-                                        "Failed to restore snap for %s",
-                                        snap->snapname);
-                                volinfo->version++;
-                                goto out;
-                        }
-
-                        /* Restore is successful therefore delete the original
-                         * volume's volinfo. If the volinfo is already restored
-                         * then we should delete the backend LVMs */
-                        if (!gf_uuid_is_null (volinfo->restored_from_snap)) {
-                                ret = glusterd_lvm_snapshot_remove (dict,
-                                                                    volinfo);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_SNAP_REMOVE_FAIL,
-                                                "Failed to remove LVM backend");
-                                        goto out;
-                                }
-                        }
-
-                        /* Detach the volinfo from priv->volumes, so that no new
-                         * command can ref it any more and then unref it.
-                         */
-                        cds_list_del_init (&volinfo->vol_list);
-                        glusterd_volinfo_unref (volinfo);
-
-                        ret = glusterd_snapshot_restore_cleanup (dict,
-                                                                 parent_volname,
-                                                                 snap);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_CLEANUP_FAIL,
-                                        "Failed to perform snapshot restore "
-                                        "cleanup for %s volume",
-                                        parent_volname);
-                                goto out;
-                        }
-
-                        GF_FREE (parent_volname);
-                        parent_volname = NULL;
+                volinfo->version--;
+                gf_uuid_copy(volinfo->restored_from_snap, null_uuid);
+
+                /* gd_restore_snap_volume() uses the dict and volcount
+                 * to fetch snap brick info from other nodes, which were
+                 * collected during prevalidation. As this is an ad-hoc
+                 * op and only local node's data matter, hence sending
+                 * volcount as 0 and re-using the same dict because we
+                 * need not record any missed creates in the rsp_dict.
+                 */
+                ret = gd_restore_snap_volume(dict, dict, volinfo, snap_volinfo,
+                                             0);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SNAP_RESTORE_FAIL,
+                           "Failed to restore snap for %s", snap->snapname);
+                    volinfo->version++;
+                    goto out;
                 }
 
-                break;
-        default:
-                /* The entry must be a create, delete, or
-                 * restore entry
+                /* Restore is successful therefore delete the original
+                 * volume's volinfo. If the volinfo is already restored
+                 * then we should delete the backend LVMs */
+                if (!gf_uuid_is_null(volinfo->restored_from_snap)) {
+                    ret = glusterd_lvm_snapshot_remove(dict, volinfo);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_SNAP_REMOVE_FAIL,
+                               "Failed to remove LVM backend");
+                        goto out;
+                    }
+                }
+
+                /* Detach the volinfo from priv->volumes, so that no new
+                 * command can ref it any more and then unref it.
                  */
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid missed snap entry");
-                ret = -1;
-                goto out;
-        }
+                cds_list_del_init(&volinfo->vol_list);
+                glusterd_volinfo_unref(volinfo);
 
-out:
-        dict_unref (dict);
-        if (parent_volname) {
-                GF_FREE (parent_volname);
+                ret = glusterd_snapshot_restore_cleanup(dict, parent_volname,
+                                                        snap);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SNAP_CLEANUP_FAIL,
+                           "Failed to perform snapshot restore "
+                           "cleanup for %s volume",
+                           parent_volname);
+                    goto out;
+                }
+
+                GF_FREE(parent_volname);
                 parent_volname = NULL;
-        }
+            }
+
+            break;
+        default:
+            /* The entry must be a create, delete, or
+             * restore entry
+             */
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid missed snap entry");
+            ret = -1;
+            goto out;
+    }
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+out:
+    dict_unref(dict);
+    if (parent_volname) {
+        GF_FREE(parent_volname);
+        parent_volname = NULL;
+    }
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Perform missed deletes and restores on this node */
 int32_t
-glusterd_perform_missed_snap_ops ()
+glusterd_perform_missed_snap_ops()
 {
-        int32_t                      ret                 = -1;
-        int32_t                      op_status           = -1;
-        glusterd_conf_t             *priv                = NULL;
-        glusterd_missed_snap_info   *missed_snapinfo     = NULL;
-        glusterd_snap_op_t          *snap_opinfo         = NULL;
-        glusterd_snap_t             *snap                = NULL;
-        uuid_t                       snap_uuid           = {0,};
-        xlator_t                    *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                /* If the pending snap_op is not for this node then continue */
-                if (strcmp (missed_snapinfo->node_uuid, uuid_utoa (MY_UUID)))
-                        continue;
-
-                /* Find the snap id */
-                gf_uuid_parse (missed_snapinfo->snap_uuid, snap_uuid);
-                snap = NULL;
-                snap = glusterd_find_snap_by_id (snap_uuid);
-                if (!snap) {
-                        /* If the snap is not found, then a delete or a
-                         * restore can't be pending on that snap_uuid.
-                         */
-                        gf_msg_debug (this->name, 0,
-                                "Not a pending delete or restore op");
-                        continue;
+    int32_t ret = -1;
+    int32_t op_status = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    uuid_t snap_uuid = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        /* If the pending snap_op is not for this node then continue */
+        if (strcmp(missed_snapinfo->node_uuid, uuid_utoa(MY_UUID)))
+            continue;
+
+        /* Find the snap id */
+        gf_uuid_parse(missed_snapinfo->snap_uuid, snap_uuid);
+        snap = NULL;
+        snap = glusterd_find_snap_by_id(snap_uuid);
+        if (!snap) {
+            /* If the snap is not found, then a delete or a
+             * restore can't be pending on that snap_uuid.
+             */
+            gf_msg_debug(this->name, 0, "Not a pending delete or restore op");
+            continue;
+        }
+
+        op_status = GD_MISSED_SNAP_PENDING;
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            /* If the snap_op is create or its status is
+             * GD_MISSED_SNAP_DONE then continue
+             */
+            if ((snap_opinfo->status == GD_MISSED_SNAP_DONE) ||
+                (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE))
+                continue;
+
+            /* Perform the actual op for the first time for
+             * this snap, and mark the snap_status as
+             * GD_MISSED_SNAP_DONE. For other entries for the same
+             * snap, just mark the entry as done.
+             */
+            if (op_status == GD_MISSED_SNAP_PENDING) {
+                ret = glusterd_perform_missed_op(snap, snap_opinfo->op);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SNAPSHOT_OP_FAILED,
+                           "Failed to perform missed snap op");
+                    goto out;
                 }
+                op_status = GD_MISSED_SNAP_DONE;
+            }
 
-                op_status = GD_MISSED_SNAP_PENDING;
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        /* If the snap_op is create or its status is
-                         * GD_MISSED_SNAP_DONE then continue
-                         */
-                        if ((snap_opinfo->status == GD_MISSED_SNAP_DONE) ||
-                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE))
-                                continue;
-
-                        /* Perform the actual op for the first time for
-                         * this snap, and mark the snap_status as
-                         * GD_MISSED_SNAP_DONE. For other entries for the same
-                         * snap, just mark the entry as done.
-                         */
-                        if (op_status == GD_MISSED_SNAP_PENDING) {
-                                ret = glusterd_perform_missed_op
-                                                             (snap,
-                                                              snap_opinfo->op);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_SNAPSHOT_OP_FAILED,
-                                                "Failed to perform missed snap op");
-                                        goto out;
-                                }
-                                op_status = GD_MISSED_SNAP_DONE;
-                        }
-
-                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
-                }
+            snap_opinfo->status = GD_MISSED_SNAP_DONE;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Import friend volumes missed_snap_list and update *
  * missed_snap_list if need be */
 int32_t
-glusterd_import_friend_missed_snap_list (dict_t *peer_data)
+glusterd_import_friend_missed_snap_list(dict_t *peer_data)
 {
-        int32_t                      missed_snap_count     = -1;
-        int32_t                      ret                   = -1;
-        glusterd_conf_t             *priv                  = NULL;
-        xlator_t                    *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Add the friends missed_snaps entries to the in-memory list */
-        ret = dict_get_int32 (peer_data, "missed_snap_count",
-                              &missed_snap_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_MISSED_SNAP_GET_FAIL,
-                        "No missed snaps");
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_add_missed_snaps_to_list (peer_data,
-                                                 missed_snap_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
-                        "Failed to add missed snaps to list");
-                goto out;
-        }
-
-        ret = glusterd_perform_missed_snap_ops ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED,
-                        "Failed to perform snap operations");
-                /* Not going to out at this point coz some *
-                 * missed ops might have been performed. We *
-                 * need to persist the current list *
-                 */
-        }
+    int32_t missed_snap_count = -1;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Add the friends missed_snaps entries to the in-memory list */
+    ret = dict_get_int32(peer_data, "missed_snap_count", &missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_GET_FAIL,
+               "No missed snaps");
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_add_missed_snaps_to_list(peer_data, missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+               "Failed to add missed snaps to list");
+        goto out;
+    }
+
+    ret = glusterd_perform_missed_snap_ops();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+               "Failed to perform snap operations");
+        /* Not going to out at this point coz some *
+         * missed ops might have been performed. We *
+         * need to persist the current list *
+         */
+    }
 
-        ret = glusterd_store_update_missed_snaps ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
-                        "Failed to update missed_snaps_list");
-                goto out;
-        }
+    ret = glusterd_store_update_missed_snaps();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+               "Failed to update missed_snaps_list");
+        goto out;
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /*
@@ -1280,59 +1234,62 @@ out:
  * boolean "conflict" will be set to false.
  */
 int
-glusterd_check_peer_has_higher_snap_version (dict_t *peer_data,
-                               char *peer_snap_name, int volcount,
-                               gf_boolean_t *conflict, char *prefix,
-                               glusterd_snap_t *snap, char *hostname)
+glusterd_check_peer_has_higher_snap_version(dict_t *peer_data,
+                                            char *peer_snap_name, int volcount,
+                                            gf_boolean_t *conflict,
+                                            char *prefix, glusterd_snap_t *snap,
+                                            char *hostname)
 {
-        glusterd_volinfo_t      *snap_volinfo = NULL;
-        char                     key[256]     = {0};
-        int                      version      = 0, i = 0;
-        int                      ret          = 0;
-        xlator_t                *this         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap);
-        GF_ASSERT (peer_data);
-
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "%s%d.version", prefix, i);
-                ret = dict_get_int32 (peer_data, key, &version);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "failed to get "
-                                "version of snap volume = %s", peer_snap_name);
-                        return -1;
-                }
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    char key[256] = {0};
+    int version = 0, i = 0;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+    GF_ASSERT(peer_data);
+
+    for (i = 1; i <= volcount; i++) {
+        snprintf(key, sizeof(key), "%s%d.version", prefix, i);
+        ret = dict_get_int32(peer_data, key, &version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get "
+                   "version of snap volume = %s",
+                   peer_snap_name);
+            return -1;
+        }
 
-               /* TODO : As of now there is only one volume in snapshot.
-                * Change this when multiple volume snapshot is introduced
-                */
-               snap_volinfo = cds_list_entry (snap->volumes.next,
-                                          glusterd_volinfo_t, vol_list);
-               if (!snap_volinfo) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL, "Failed to get snap "
-                                "volinfo %s", snap->snapname);
-                   return -1;
-               }
-
-               if (version > snap_volinfo->version) {
-                        /* Mismatch detected */
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                 GD_MSG_VOL_VERS_MISMATCH,
-                                "Version of volume %s differ. "
-                                "local version = %d, remote version = %d "
-                                "on peer %s", snap_volinfo->volname,
-                                snap_volinfo->version, version, hostname);
-                        *conflict = _gf_true;
-                        break;
-                } else {
-                        *conflict = _gf_false;
-                }
+        /* TODO : As of now there is only one volume in snapshot.
+         * Change this when multiple volume snapshot is introduced
+         */
+        snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                      vol_list);
+        if (!snap_volinfo) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to get snap "
+                   "volinfo %s",
+                   snap->snapname);
+            return -1;
+        }
+
+        if (version > snap_volinfo->version) {
+            /* Mismatch detected */
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_VERS_MISMATCH,
+                   "Version of volume %s differ. "
+                   "local version = %d, remote version = %d "
+                   "on peer %s",
+                   snap_volinfo->volname, snap_volinfo->version, version,
+                   hostname);
+            *conflict = _gf_true;
+            break;
+        } else {
+            *conflict = _gf_false;
         }
-        return 0;
+    }
+    return 0;
 }
 
 /* Check for the peer_snap_name in the list of existing snapshots.
@@ -1345,377 +1302,404 @@ glusterd_check_peer_has_higher_snap_version (dict_t *peer_data,
  * and snap to NULL.
  */
 void
-glusterd_is_peer_snap_conflicting (char *peer_snap_name, char *peer_snap_id,
-                                   gf_boolean_t *conflict,
-                                   glusterd_snap_t **snap, char *hostname)
+glusterd_is_peer_snap_conflicting(char *peer_snap_name, char *peer_snap_id,
+                                  gf_boolean_t *conflict,
+                                  glusterd_snap_t **snap, char *hostname)
 {
-        uuid_t       peer_snap_uuid = {0,};
-        xlator_t    *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_snap_name);
-        GF_ASSERT (peer_snap_id);
-        GF_ASSERT (conflict);
-        GF_ASSERT (snap);
-        GF_ASSERT (hostname);
-
-        *snap = glusterd_find_snap_by_name (peer_snap_name);
-        if (*snap) {
-                gf_uuid_parse (peer_snap_id, peer_snap_uuid);
-                if (!gf_uuid_compare (peer_snap_uuid, (*snap)->snap_id)) {
-                        /* Current node contains the same snap having
-                         * the same snapname and snap_id
-                         */
-                        gf_msg_debug (this->name, 0,
-                                "Snapshot %s from peer %s present in "
-                                "localhost", peer_snap_name, hostname);
-                        *conflict = _gf_false;
-                } else {
-                        /* Current node contains the same snap having
-                         * the same snapname but different snap_id
-                         */
-                        gf_msg_debug (this->name, 0,
-                                "Snapshot %s from peer %s conflicts with "
-                                "snapshot in localhost", peer_snap_name,
-                                hostname);
-                        *conflict = _gf_true;
-                }
+    uuid_t peer_snap_uuid = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_snap_name);
+    GF_ASSERT(peer_snap_id);
+    GF_ASSERT(conflict);
+    GF_ASSERT(snap);
+    GF_ASSERT(hostname);
+
+    *snap = glusterd_find_snap_by_name(peer_snap_name);
+    if (*snap) {
+        gf_uuid_parse(peer_snap_id, peer_snap_uuid);
+        if (!gf_uuid_compare(peer_snap_uuid, (*snap)->snap_id)) {
+            /* Current node contains the same snap having
+             * the same snapname and snap_id
+             */
+            gf_msg_debug(this->name, 0,
+                         "Snapshot %s from peer %s present in "
+                         "localhost",
+                         peer_snap_name, hostname);
+            *conflict = _gf_false;
         } else {
-                /* Peer contains snapshots missing on the current node */
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_MISSED_SNAP_PRESENT,
-                        "Snapshot %s from peer %s missing on localhost",
-                        peer_snap_name, hostname);
-                *conflict = _gf_false;
-        }
+            /* Current node contains the same snap having
+             * the same snapname but different snap_id
+             */
+            gf_msg_debug(this->name, 0,
+                         "Snapshot %s from peer %s conflicts with "
+                         "snapshot in localhost",
+                         peer_snap_name, hostname);
+            *conflict = _gf_true;
+        }
+    } else {
+        /* Peer contains snapshots missing on the current node */
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_PRESENT,
+               "Snapshot %s from peer %s missing on localhost", peer_snap_name,
+               hostname);
+        *conflict = _gf_false;
+    }
 }
 
 /* Check if the local node is hosting any bricks for the given snapshot */
 gf_boolean_t
-glusterd_are_snap_bricks_local (glusterd_snap_t *snap)
+glusterd_are_snap_bricks_local(glusterd_snap_t *snap)
 {
-        gf_boolean_t            is_local   = _gf_false;
-        glusterd_volinfo_t     *volinfo    = NULL;
-        glusterd_brickinfo_t   *brickinfo  = NULL;
-        xlator_t               *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap);
-
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                is_local = _gf_true;
-                                goto out;
-                        }
-                }
+    gf_boolean_t is_local = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                is_local = _gf_true;
+                goto out;
+            }
         }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", is_local);
-        return is_local;
+    gf_msg_trace(this->name, 0, "Returning %d", is_local);
+    return is_local;
 }
 
 /* Check if the peer has missed any snap delete
  * or restore for the given snap_id
  */
 gf_boolean_t
-glusterd_peer_has_missed_snap_delete (uuid_t peerid, char *peer_snap_id)
+glusterd_peer_has_missed_snap_delete(uuid_t peerid, char *peer_snap_id)
 {
-        char                        *peer_uuid           = NULL;
-        gf_boolean_t                 missed_delete       = _gf_false;
-        glusterd_conf_t             *priv                = NULL;
-        glusterd_missed_snap_info   *missed_snapinfo     = NULL;
-        glusterd_snap_op_t          *snap_opinfo         = NULL;
-        xlator_t                    *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (peer_snap_id);
-
-        peer_uuid = uuid_utoa (peerid);
-
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                /* Look for missed snap for the same peer, and
-                 * the same snap_id
-                 */
-                if ((!strcmp (peer_uuid, missed_snapinfo->node_uuid)) &&
-                    (!strcmp (peer_snap_id, missed_snapinfo->snap_uuid))) {
-                        /* Check if the missed snap's op is delete and the
-                         * status is pending
-                         */
-                        cds_list_for_each_entry (snap_opinfo,
-                                                 &missed_snapinfo->snap_ops,
-                                                 snap_ops_list) {
-                                if (((snap_opinfo->op ==
-                                              GF_SNAP_OPTION_TYPE_DELETE) ||
-                                     (snap_opinfo->op ==
-                                              GF_SNAP_OPTION_TYPE_RESTORE)) &&
-                                    (snap_opinfo->status ==
-                                             GD_MISSED_SNAP_PENDING)) {
-                                        missed_delete = _gf_true;
-                                        goto out;
-                                }
-                        }
+    char *peer_uuid = NULL;
+    gf_boolean_t missed_delete = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(peer_snap_id);
+
+    peer_uuid = uuid_utoa(peerid);
+
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        /* Look for missed snap for the same peer, and
+         * the same snap_id
+         */
+        if ((!strcmp(peer_uuid, missed_snapinfo->node_uuid)) &&
+            (!strcmp(peer_snap_id, missed_snapinfo->snap_uuid))) {
+            /* Check if the missed snap's op is delete and the
+             * status is pending
+             */
+            cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                    snap_ops_list)
+            {
+                if (((snap_opinfo->op == GF_SNAP_OPTION_TYPE_DELETE) ||
+                     (snap_opinfo->op == GF_SNAP_OPTION_TYPE_RESTORE)) &&
+                    (snap_opinfo->status == GD_MISSED_SNAP_PENDING)) {
+                    missed_delete = _gf_true;
+                    goto out;
                 }
+            }
         }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", missed_delete);
-        return missed_delete;
+    gf_msg_trace(this->name, 0, "Returning %d", missed_delete);
+    return missed_delete;
 }
 
-/* Genrate and store snap volfiles for imported snap object */
+/* Generate and store snap volfiles for imported snap object */
 int32_t
-glusterd_gen_snap_volfiles (glusterd_volinfo_t *snap_vol, char *peer_snap_name)
+glusterd_gen_snap_volfiles(glusterd_volinfo_t *snap_vol, char *peer_snap_name)
 {
-        int32_t                 ret              = -1;
-        xlator_t               *this             = NULL;
-        glusterd_volinfo_t     *parent_volinfo   = NULL;
-        glusterd_brickinfo_t   *brickinfo        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (peer_snap_name);
-
-        ret = glusterd_store_volinfo (snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_SET_FAIL, "Failed to store snapshot "
-                        "volinfo (%s) for snap %s", snap_vol->volname,
-                        peer_snap_name);
-                goto out;
-        }
-
-        ret = generate_brick_volfiles (snap_vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "generating the brick volfiles for the "
-                        "snap %s failed", peer_snap_name);
-                goto out;
-        }
-
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_TRUSTED);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "generating the trusted client volfiles for "
-                        "the snap %s failed", peer_snap_name);
-                goto out;
-        }
-
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_OTHER);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL,
-                        "generating the client volfiles for the "
-                        "snap %s failed", peer_snap_name);
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (snap_vol->parent_volname,
-                                     &parent_volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_GET_FAIL, "Parent volinfo "
-                        "not found for %s volume of snap %s",
-                        snap_vol->volname, peer_snap_name);
-                goto out;
-        }
-
-        glusterd_list_add_snapvol (parent_volinfo, snap_vol);
-
-        ret = glusterd_store_volinfo (snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_SET_FAIL,
-                        "Failed to store snap volinfo");
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *parent_volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(peer_snap_name);
+
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store snapshot "
+               "volinfo (%s) for snap %s",
+               snap_vol->volname, peer_snap_name);
+        goto out;
+    }
+
+    ret = generate_brick_volfiles(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the brick volfiles for the "
+               "snap %s failed",
+               peer_snap_name);
+        goto out;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the trusted client volfiles for "
+               "the snap %s failed",
+               peer_snap_name);
+        goto out;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_OTHER);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the client volfiles for the "
+               "snap %s failed",
+               peer_snap_name);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(snap_vol->parent_volname, &parent_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Parent volinfo "
+               "not found for %s volume of snap %s",
+               snap_vol->volname, peer_snap_name);
+        goto out;
+    }
+
+    glusterd_list_add_snapvol(parent_volinfo, snap_vol);
+
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store snap volinfo");
+        goto out;
+    }
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Import snapshot info from peer_data and add it to priv */
 int32_t
-glusterd_import_friend_snap (dict_t *peer_data, int32_t snap_count,
-                             char *peer_snap_name, char *peer_snap_id)
+glusterd_import_friend_snap(dict_t *peer_data, int32_t snap_count,
+                            char *peer_snap_name, char *peer_snap_id)
 {
-        char                 buf[NAME_MAX]    = "";
-        char                 prefix[NAME_MAX] = "";
-        dict_t              *dict             = NULL;
-        glusterd_snap_t     *snap             = NULL;
-        glusterd_volinfo_t  *snap_vol         = NULL;
-        glusterd_conf_t     *priv             = NULL;
-        int32_t              ret              = -1;
-        int32_t              volcount         = -1;
-        int32_t              i                = -1;
-        xlator_t            *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (peer_snap_name);
-        GF_ASSERT (peer_snap_id);
-
-        snprintf (prefix, sizeof(prefix), "snap%d", snap_count);
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Could not create "
-                        "the snap object for snap %s", peer_snap_name);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to create dict");
-                ret = -1;
-                goto out;
-        }
-
-        strncpy (snap->snapname, peer_snap_name, sizeof (snap->snapname) - 1);
-        gf_uuid_parse (peer_snap_id, snap->snap_id);
-
-        snprintf (buf, sizeof(buf), "%s.snapid", prefix);
-        ret = dict_get_str (peer_data, buf, &snap->description);
-
-        snprintf (buf, sizeof(buf), "%s.time_stamp", prefix);
-        ret = dict_get_int64 (peer_data, buf, &snap->time_stamp);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get time_stamp for snap %s",
-                        peer_snap_name);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_restored", prefix);
-        ret = dict_get_int8 (peer_data, buf, (int8_t *) &snap->snap_restored);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get snap_restored for snap %s",
-                        peer_snap_name);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_status", prefix);
-        ret = dict_get_int32 (peer_data, buf, (int32_t *) &snap->snap_status);
+    char buf[64] = "";
+    char prefix[32] = "";
+    char *description = NULL;
+    dict_t *dict = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t volcount = -1;
+    int32_t i = -1;
+    xlator_t *this = NULL;
+    int64_t time_stamp;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(peer_snap_name);
+    GF_ASSERT(peer_snap_id);
+
+    snprintf(prefix, sizeof(prefix), "snap%d", snap_count);
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not create "
+               "the snap object for snap %s",
+               peer_snap_name);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create dict");
+        ret = -1;
+        goto out;
+    }
+
+    gf_strncpy(snap->snapname, peer_snap_name, sizeof(snap->snapname));
+    gf_uuid_parse(peer_snap_id, snap->snap_id);
+
+    snprintf(buf, sizeof(buf), "%s.description", prefix);
+    ret = dict_get_str(peer_data, buf, &description);
+    if (ret == 0 && description) {
+        snap->description = gf_strdup(description);
+        if (snap->description == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "Saving the Snapshot Description Failed");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    snprintf(buf, sizeof(buf), "%s.time_stamp", prefix);
+    ret = dict_get_int64(peer_data, buf, &time_stamp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get time_stamp for snap %s", peer_snap_name);
+        goto out;
+    }
+    snap->time_stamp = (time_t)time_stamp;
+
+    snprintf(buf, sizeof(buf), "%s.snap_restored", prefix);
+    ret = dict_get_int8(peer_data, buf, (int8_t *)&snap->snap_restored);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get snap_restored for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_status", prefix);
+    ret = dict_get_int32(peer_data, buf, (int32_t *)&snap->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get snap_status for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    /* If the snap is scheduled to be decommissioned, then
+     * don't accept the snap */
+    if (snap->snap_status == GD_SNAP_STATUS_DECOMMISSION) {
+        gf_msg_debug(this->name, 0,
+                     "The snap(%s) is scheduled to be decommissioned "
+                     "Not accepting the snap.",
+                     peer_snap_name);
+        glusterd_snap_remove(dict, snap, _gf_true, _gf_true, _gf_false);
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.volcount", prefix);
+    ret = dict_get_int32(peer_data, buf, &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volcount for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    ret = glusterd_store_create_snap_dir(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPDIR_CREATE_FAIL,
+               "Failed to create snap dir");
+        goto out;
+    }
+
+    glusterd_list_add_order(&snap->snap_list, &priv->snapshots,
+                            glusterd_compare_snap_time);
+
+    for (i = 1; i <= volcount; i++) {
+        ret = glusterd_import_volinfo(peer_data, i, &snap_vol, prefix);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get snap_status for snap %s",
-                        peer_snap_name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+                   "Failed to import snap volinfo for "
+                   "snap %s",
+                   peer_snap_name);
+            goto out;
         }
 
-        /* If the snap is scheduled to be decommissioned, then
-         * don't accept the snap */
-        if (snap->snap_status == GD_SNAP_STATUS_DECOMMISSION) {
-                gf_msg_debug (this->name, 0,
-                        "The snap(%s) is scheduled to be decommissioned "
-                        "Not accepting the snap.", peer_snap_name);
-                glusterd_snap_remove (dict, snap,
-                                      _gf_true, _gf_true, _gf_false);
-                ret = 0;
-                goto out;
-        }
+        snap_vol->snapshot = snap;
 
-        snprintf (buf, sizeof(buf), "%s.volcount", prefix);
-        ret = dict_get_int32 (peer_data, buf, &volcount);
+        ret = glusterd_gen_snap_volfiles(snap_vol, peer_snap_name);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get volcount for snap %s",
-                        peer_snap_name);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Failed to generate snap vol files "
+                   "for snap %s",
+                   peer_snap_name);
+            goto out;
+        }
+        /* During handshake, after getting updates from friend mount
+         * point for activated snapshot should exist and should not
+         * for deactivated snapshot.
+         */
+        if (glusterd_is_volume_started(snap_vol)) {
+            ret = glusterd_recreate_vol_brick_mounts(this, snap_vol);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRK_MNT_RECREATE_FAIL,
+                       "Failed to recreate brick mounts"
+                       " for %s",
+                       snap->snapname);
                 goto out;
-        }
-
-        ret = glusterd_store_create_snap_dir (snap);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPDIR_CREATE_FAIL,
-                        "Failed to create snap dir");
+            }
+
+            (void)glusterd_start_bricks(snap_vol);
+            ret = glusterd_store_volinfo(snap_vol,
+                                         GLUSTERD_VOLINFO_VER_AC_NONE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                       "Failed to "
+                       "write volinfo for volume %s",
+                       snap_vol->volname);
                 goto out;
+            }
+        } else {
+            (void)glusterd_stop_bricks(snap_vol);
+            ret = glusterd_snap_unmount(this, snap_vol);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+                       "Failed to unmounts for %s", snap->snapname);
+            }
         }
 
-        glusterd_list_add_order (&snap->snap_list, &priv->snapshots,
-                                 glusterd_compare_snap_time);
-
-
-        for (i = 1; i <= volcount; i++) {
-                ret = glusterd_import_volinfo (peer_data, i,
-                                               &snap_vol, prefix);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_SET_FAIL,
-                                "Failed to import snap volinfo for "
-                                "snap %s", peer_snap_name);
-                        goto out;
-                }
-
-                snap_vol->snapshot = snap;
-
-                ret = glusterd_gen_snap_volfiles (snap_vol, peer_snap_name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLFILE_CREATE_FAIL,
-                                "Failed to generate snap vol files "
-                                "for snap %s", peer_snap_name);
-                        goto out;
-                }
-                if (glusterd_is_volume_started (snap_vol)) {
-                        (void) glusterd_start_bricks (snap_vol);
-                } else {
-                        (void) glusterd_stop_bricks(snap_vol);
-                }
-
-                ret = glusterd_import_quota_conf (peer_data, i,
-                                                  snap_vol, prefix);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_QUOTA_CONFIG_IMPORT_FAIL,
-                                "Failed to import quota conf "
-                                "for snap %s", peer_snap_name);
-                        goto out;
-                }
-
-                snap_vol = NULL;
-        }
-
-        ret = glusterd_store_snap (snap);
+        ret = glusterd_import_quota_conf(peer_data, i, snap_vol, prefix);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Could not store snap"
-                        "object %s", peer_snap_name);
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONFIG_IMPORT_FAIL,
+                   "Failed to import quota conf "
+                   "for snap %s",
+                   peer_snap_name);
+            goto out;
+        }
+
+        snap_vol = NULL;
+    }
+
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not store snap"
+               "object %s",
+               peer_snap_name);
+        goto out;
+    }
+    glusterd_fetchsnap_notify(this);
 
 out:
-        if (ret)
-                glusterd_snap_remove (dict, snap,
-                                      _gf_true, _gf_true, _gf_false);
+    if (ret)
+        glusterd_snap_remove(dict, snap, _gf_true, _gf_true, _gf_false);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* During a peer-handshake, after the volumes have synced, and the list of
@@ -1727,8 +1711,11 @@ out:
  * state, i.e either both would be hosting bricks or both would not be hosting
  * bricks, then a decision can't be taken and a peer-reject will happen.
  *
- * glusterd_compare_and_update_snap() implements the following algorithm to
- * perform the above task:
+ * glusterd_compare_snap()  & glusterd_update_snaps () implement the following
+ * algorithm to perform the above task. Please note the former function tries to
+ * iterate over the snaps one at a time and updating the relevant fields in the
+ * dictionary and then glusterd_update_snaps () go over all the snaps and update
+ * them at one go as part of a synctask.
  * Step  1: Start.
  * Step  2: Check if the peer is missing a delete or restore on the said snap.
  *          If yes, goto step 6.
@@ -1753,1936 +1740,2098 @@ out:
  *
  */
 int32_t
-glusterd_compare_and_update_snap (dict_t *peer_data, int32_t snap_count,
-                                  char *peername, uuid_t peerid)
+glusterd_compare_snap(dict_t *peer_data, int32_t snap_count, char *peername,
+                      uuid_t peerid)
 {
-        char              buf[NAME_MAX]    = "";
-        char              prefix[NAME_MAX] = "";
-        char             *peer_snap_name   = NULL;
-        char             *peer_snap_id     = NULL;
-        dict_t           *dict             = NULL;
-        glusterd_snap_t  *snap             = NULL;
-        gf_boolean_t      conflict         = _gf_false;
-        gf_boolean_t      is_local         = _gf_false;
-        gf_boolean_t      is_hosted        = _gf_false;
-        gf_boolean_t      missed_delete    = _gf_false;
-        gf_boolean_t      remove_lvm       = _gf_true;
-
-        int32_t           ret              = -1;
-        int32_t           volcount         = 0;
-        xlator_t         *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (peername);
-
-        snprintf (prefix, sizeof(prefix), "snap%d", snap_count);
-
-        /* Fetch the peer's snapname */
-        snprintf (buf, sizeof(buf), "%s.snapname", prefix);
-        ret = dict_get_str (peer_data, buf, &peer_snap_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch snapname from peer: %s",
-                        peername);
-                goto out;
-        }
-
-        /* Fetch the peer's snap_id */
-        snprintf (buf, sizeof(buf), "%s.snap_id", prefix);
-        ret = dict_get_str (peer_data, buf, &peer_snap_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch snap_id from peer: %s",
-                        peername);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.volcount", prefix);
-        ret = dict_get_int32 (peer_data, buf, &volcount);
+    char buf[64] = "";
+    char prefix[32] = "";
+    char *peer_snap_name = NULL;
+    char *peer_snap_id = NULL;
+    glusterd_snap_t *snap = NULL;
+    gf_boolean_t conflict = _gf_false;
+    gf_boolean_t is_local = _gf_false;
+    gf_boolean_t is_hosted = _gf_false;
+    gf_boolean_t missed_delete = _gf_false;
+    int32_t ret = -1;
+    int32_t volcount = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(peername);
+
+    snprintf(prefix, sizeof(prefix), "snap%d", snap_count);
+
+    ret = dict_set_uint32(peer_data, buf, 0);
+    snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 0);
+    snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+    ret = dict_set_uint32(peer_data, buf, 0);
+    snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 0);
+
+    /* Fetch the peer's snapname */
+    snprintf(buf, sizeof(buf), "%s.snapname", prefix);
+    ret = dict_get_str(peer_data, buf, &peer_snap_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snapname from peer: %s", peername);
+        goto out;
+    }
+
+    /* Fetch the peer's snap_id */
+    snprintf(buf, sizeof(buf), "%s.snap_id", prefix);
+    ret = dict_get_str(peer_data, buf, &peer_snap_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_id from peer: %s", peername);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.volcount", prefix);
+    ret = dict_get_int32(peer_data, buf, &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volcount for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    /* Check if the peer has missed a snap delete or restore
+     * resulting in stale data for the snap in question
+     */
+    missed_delete = glusterd_peer_has_missed_snap_delete(peerid, peer_snap_id);
+    if (missed_delete == _gf_true) {
+        /* Peer has missed delete on the missing/conflicting snap_id */
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_DELETE,
+               "Peer %s has missed a delete "
+               "on snap %s",
+               peername, peer_snap_name);
+        ret = 0;
+        goto out;
+    }
+
+    /* Check if there is a conflict, and if the
+     * peer data is already present
+     */
+    glusterd_is_peer_snap_conflicting(peer_snap_name, peer_snap_id, &conflict,
+                                      &snap, peername);
+    if (conflict == _gf_false) {
+        if (!snap) {
+            /* Peer has snap with the same snapname
+             * and snap_id, which local node doesn't have.
+             */
+            snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+            ret = dict_set_uint32(peer_data, buf, 1);
+            goto out;
+        }
+        /* Peer has snap with the same snapname
+         * and snap_id. Now check if peer has a
+         * snap with higher snap version than local
+         * node has.
+         */
+        ret = glusterd_check_peer_has_higher_snap_version(
+            peer_data, peer_snap_name, volcount, &conflict, prefix, snap,
+            peername);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
-                        "Unable to get volcount for snap %s",
-                        peer_snap_name);
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_VERS_MISMATCH,
+                   "Failed "
+                   "to check version of snap volume");
+            goto out;
+        }
+        if (conflict == _gf_true) {
+            /*
+             * Snap version of peer is higher than snap
+             * version of local node.
+             *
+             * Remove data in local node and accept peer data.
+             * We just need to heal snap info of local node, So
+             * When removing data from local node, make sure
+             * we are not removing backend lvm of the snap.
+             */
+            snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+            ret = dict_set_uint32(peer_data, buf, 0);
+            snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+            ret = dict_set_uint32(peer_data, buf, 1);
+            snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+            ret = dict_set_uint32(peer_data, buf, 1);
 
-        /* Check if the peer has missed a snap delete or restore
-         * resulting in stale data for the snap in question
+        } else {
+            ret = 0;
+        }
+        goto out;
+    }
+
+    /* There is a conflict. Check if the current node is
+     * hosting bricks for the conflicted snap.
+     */
+    is_local = glusterd_are_snap_bricks_local(snap);
+
+    /* Check if the peer is hosting any bricks for the
+     * conflicting snap
+     */
+    snprintf(buf, sizeof(buf), "%s.host_bricks", prefix);
+    ret = dict_get_int8(peer_data, buf, (int8_t *)&is_hosted);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch host_bricks from peer: %s "
+               "for %s",
+               peername, peer_snap_name);
+        goto out;
+    }
+
+    /* As there is a conflict at this point of time, the data of the
+     * node that hosts a brick takes precedence. If both the local
+     * node and the peer are in the same state, i.e if both of them
+     * are either hosting or not hosting the bricks, for the snap,
+     * then it's a peer reject
+     */
+    if (is_hosted == is_local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CONFLICT,
+               "Conflict in snapshot %s with peer %s", peer_snap_name,
+               peername);
+        ret = -1;
+        goto out;
+    }
+
+    if (is_hosted == _gf_false) {
+        /* If there was a conflict, and the peer is not hosting
+         * any brick, then don't accept peer data
          */
-        missed_delete = glusterd_peer_has_missed_snap_delete (peerid,
-                                                              peer_snap_id);
-        if (missed_delete == _gf_true) {
-                /* Peer has missed delete on the missing/conflicting snap_id */
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_MISSED_SNAP_DELETE,
-                        "Peer %s has missed a delete "
-                        "on snap %s", peername, peer_snap_name);
-                ret = 0;
-                goto out;
-        }
+        gf_msg_debug(this->name, 0,
+                     "Peer doesn't hosts bricks for conflicting "
+                     "snap(%s). Not accepting peer data.",
+                     peer_snap_name);
+        ret = 0;
+        goto out;
+    }
+
+    /* The peer is hosting a brick in case of conflict
+     * And local node isn't. Hence remove local node's
+     * data and accept peer data
+     */
+    gf_msg_debug(this->name, 0,
+                 "Peer hosts bricks for conflicting "
+                 "snap(%s). Removing local data. Accepting peer data.",
+                 peer_snap_name);
+    snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+    ret = dict_set_uint32(peer_data, buf, 1);
+    snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 1);
+    snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 1);
 
-        /* Check if there is a conflict, and if the
-         * peer data is already present
-         */
-        glusterd_is_peer_snap_conflicting (peer_snap_name, peer_snap_id,
-                                           &conflict, &snap, peername);
-        if (conflict == _gf_false) {
-                if (!snap) {
-                        /* Peer has snap with the same snapname
-                        * and snap_id, which local node doesn't have.
-                        */
-                        goto accept_peer_data;
-                }
-                /* Peer has snap with the same snapname
-                 * and snap_id. Now check if peer has a
-                 * snap with higher snap version than local
-                 * node has.
-                 */
-                ret = glusterd_check_peer_has_higher_snap_version (peer_data,
-                                                    peer_snap_name, volcount,
-                                                    &conflict, prefix, snap,
-                                                    peername);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOL_VERS_MISMATCH, "Failed "
-                                "to check version of snap volume");
-                        goto out;
-                }
-                if (conflict == _gf_true) {
-                        /*
-                         * Snap version of peer is higher than snap
-                         * version of local node.
-                         *
-                         * Remove data in local node and accept peer data.
-                         * We just need to heal snap info of local node, So
-                         * When removing data from local node, make sure
-                         * we are not removing backend lvm of the snap.
-                         */
-                        remove_lvm = _gf_false;
-                        goto remove_my_data;
-                } else {
-                        ret = 0;
-                        goto out;
-                }
-        }
+out:
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        /* There is a conflict. Check if the current node is
-         * hosting bricks for the conflicted snap.
-         */
-        is_local = glusterd_are_snap_bricks_local (snap);
+int32_t
+glusterd_update_snaps_synctask(void *opaque)
+{
+    int32_t ret = -1;
+    int32_t snap_count = 0;
+    int i = 1;
+    xlator_t *this = NULL;
+    dict_t *peer_data = NULL;
+    char buf[64] = "";
+    char prefix[32] = "";
+    char *peer_snap_name = NULL;
+    char *peer_snap_id = NULL;
+    char *peername = NULL;
+    gf_boolean_t remove_lvm = _gf_false;
+    gf_boolean_t remove_my_data = _gf_false;
+    gf_boolean_t accept_peer_data = _gf_false;
+    int32_t val = 0;
+    glusterd_snap_t *snap = NULL;
+    dict_t *dict = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    peer_data = (dict_t *)opaque;
+    GF_ASSERT(peer_data);
+
+    synclock_lock(&conf->big_lock);
+
+    while (conf->restart_bricks) {
+        synccond_wait(&conf->cond_restart_bricks, &conf->big_lock);
+    }
+    conf->restart_bricks = _gf_true;
+
+    ret = dict_get_int32(peer_data, "snap_count", &snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to fetch snap_count");
+        goto out;
+    }
+    ret = dict_get_str(peer_data, "peername", &peername);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to fetch peername");
+        goto out;
+    }
+
+    for (i = 1; i <= snap_count; i++) {
+        snprintf(prefix, sizeof(prefix), "snap%d", i);
 
-        /* Check if the peer is hosting any bricks for the
-         * conflicting snap
-         */
-        snprintf (buf, sizeof(buf), "%s.host_bricks", prefix);
-        ret = dict_get_int8 (peer_data, buf, (int8_t *) &is_hosted);
+        /* Fetch the peer's snapname */
+        snprintf(buf, sizeof(buf), "%s.snapname", prefix);
+        ret = dict_get_str(peer_data, buf, &peer_snap_name);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch host_bricks from peer: %s "
-                        "for %s", peername, peer_snap_name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch snapname from peer: %s", peername);
+            goto out;
         }
 
-        /* As there is a conflict at this point of time, the data of the
-         * node that hosts a brick takes precedence. If both the local
-         * node and the peer are in the same state, i.e if both of them
-         * are either hosting or not hosting the bricks, for the snap,
-         * then it's a peer reject
-         */
-        if (is_hosted == is_local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CONFLICT,
-                        "Conflict in snapshot %s with peer %s",
-                        peer_snap_name, peername);
-                ret = -1;
-                goto out;
+        /* Fetch the peer's snap_id */
+        snprintf(buf, sizeof(buf), "%s.snap_id", prefix);
+        ret = dict_get_str(peer_data, buf, &peer_snap_id);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch snap_id from peer: %s", peername);
+            goto out;
         }
 
-        if (is_hosted == _gf_false) {
-                /* If there was a conflict, and the peer is not hosting
-                 * any brick, then don't accept peer data
-                 */
-                gf_msg_debug (this->name, 0,
-                        "Peer doesn't hosts bricks for conflicting "
-                        "snap(%s). Not accepting peer data.",
-                        peer_snap_name);
-                ret = 0;
+        /* remove_my_data */
+        snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+        ret = dict_get_int32(peer_data, buf, &val);
+        if (val)
+            remove_my_data = _gf_true;
+        else
+            remove_my_data = _gf_false;
+
+        if (remove_my_data) {
+            snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+            ret = dict_get_int32(peer_data, buf, &val);
+            if (val)
+                remove_lvm = _gf_true;
+            else
+                remove_lvm = _gf_false;
+
+            dict = dict_new();
+            if (!dict) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+                       "Unable to create dict");
+                ret = -1;
                 goto out;
-        }
-
-        /* The peer is hosting a brick in case of conflict
-         * And local node isn't. Hence remove local node's
-         * data and accept peer data
-         */
-
-        gf_msg_debug (this->name, 0, "Peer hosts bricks for conflicting "
-                "snap(%s). Removing local data. Accepting peer data.",
-                peer_snap_name);
-        remove_lvm = _gf_true;
-
-remove_my_data:
-
-        dict = dict_new();
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Unable to create dict");
+            }
+            snap = glusterd_find_snap_by_name(peer_snap_name);
+            if (!snap) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_PRESENT,
+                       "Snapshot %s from peer %s missing on "
+                       "localhost",
+                       peer_snap_name, peername);
                 ret = -1;
                 goto out;
-        }
+            }
 
-        ret = glusterd_snap_remove (dict, snap, remove_lvm, _gf_false,
-                                    _gf_false);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_REMOVE_FAIL,
-                        "Failed to remove snap %s", snap->snapname);
+            ret = glusterd_snap_remove(dict, snap, remove_lvm, _gf_false,
+                                       _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to remove snap %s", snap->snapname);
                 goto out;
-        }
-
-accept_peer_data:
+            }
 
-        /* Accept Peer Data */
-        ret = glusterd_import_friend_snap (peer_data, snap_count,
-                                           peer_snap_name, peer_snap_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_IMPORT_FAIL,
-                        "Failed to import snap %s from peer %s",
-                        peer_snap_name, peername);
+            dict_unref(dict);
+            dict = NULL;
+        }
+        snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+        ret = dict_get_int32(peer_data, buf, &val);
+        if (val)
+            accept_peer_data = _gf_true;
+        else
+            accept_peer_data = _gf_false;
+
+        if (accept_peer_data) {
+            /* Accept Peer Data */
+            ret = glusterd_import_friend_snap(peer_data, i, peer_snap_name,
+                                              peer_snap_id);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_IMPORT_FAIL,
+                       "Failed to import snap %s from peer %s", peer_snap_name,
+                       peername);
                 goto out;
+            }
         }
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
-
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (peer_data)
+        dict_unref(peer_data);
+    if (dict)
+        dict_unref(dict);
+    conf->restart_bricks = _gf_false;
+    synccond_broadcast(&conf->cond_restart_bricks);
+
+    return ret;
 }
 
 /* Compare snapshots present in peer_data, with the snapshots in
  * the current node
  */
 int32_t
-glusterd_compare_friend_snapshots (dict_t *peer_data, char *peername,
-                                   uuid_t peerid)
+glusterd_compare_friend_snapshots(dict_t *peer_data, char *peername,
+                                  uuid_t peerid)
 {
-        int32_t          ret          = -1;
-        int32_t          snap_count   = 0;
-        int              i            = 1;
-        xlator_t        *this         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (peername);
-
-        ret = dict_get_int32 (peer_data, "snap_count", &snap_count);
+    int32_t ret = -1;
+    int32_t snap_count = 0;
+    int i = 1;
+    xlator_t *this = NULL;
+    dict_t *peer_data_copy = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(peername);
+
+    ret = dict_get_int32(peer_data, "snap_count", &snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to fetch snap_count");
+        goto out;
+    }
+
+    if (!snap_count)
+        goto out;
+
+    for (i = 1; i <= snap_count; i++) {
+        /* Compare one snapshot from peer_data at a time */
+        ret = glusterd_compare_snap(peer_data, i, peername, peerid);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to fetch snap_count");
-                goto out;
-        }
-
-        for (i = 1; i <= snap_count; i++) {
-                /* Compare one snapshot from peer_data at a time */
-                ret = glusterd_compare_and_update_snap (peer_data, i, peername,
-                                                        peerid);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPSHOT_OP_FAILED,
-                                "Failed to compare snapshots with peer %s",
-                                peername);
-                        goto out;
-                }
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+                   "Failed to compare snapshots with peer %s", peername);
+            goto out;
+        }
+    }
+    /* Update the snaps at one go */
+    peer_data_copy = dict_copy_with_ref(peer_data, NULL);
+    ret = dict_set_str(peer_data_copy, "peername", peername);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set peername into the dict");
+        if (peer_data_copy)
+            dict_unref(peer_data_copy);
+        goto out;
+    }
+    glusterd_launch_synctask(glusterd_update_snaps_synctask, peer_data_copy);
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_add_snapd_to_dict (glusterd_volinfo_t *volinfo,
-                            dict_t  *dict, int32_t count)
+glusterd_add_snapd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                           int32_t count)
 {
+    int ret = -1;
+    int32_t pid = -1;
+    int32_t brick_online = -1;
+    char key[64] = {0};
+    char base_key[32] = {0};
+    char pidfile[PATH_MAX] = {0};
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    snprintf(base_key, sizeof(base_key), "brick%d", count);
+    snprintf(key, sizeof(key), "%s.hostname", base_key);
+    ret = dict_set_str(dict, key, "Snapshot Daemon");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.path", base_key);
+    ret = dict_set_dynstr(dict, key, gf_strdup(uuid_utoa(MY_UUID)));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.port", base_key);
+    ret = dict_set_int32(dict, key, volinfo->snapd.port);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    glusterd_svc_build_snapd_pidfile(volinfo, pidfile, sizeof(pidfile));
+
+    brick_online = gf_is_service_running(pidfile, &pid);
+    if (brick_online == _gf_false)
+        pid = -1;
+
+    snprintf(key, sizeof(key), "%s.pid", base_key);
+    ret = dict_set_int32(dict, key, pid);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.status", base_key);
+    ret = dict_set_int32(dict, key, brick_online);
 
-        int             ret                   = -1;
-        int32_t         pid                   = -1;
-        int32_t         brick_online          = -1;
-        char            key[1024]             = {0};
-        char            base_key[1024]        = {0};
-        char            pidfile[PATH_MAX]     = {0};
-        xlator_t        *this                 = NULL;
-        glusterd_conf_t *priv                 = NULL;
-
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
+out:
+    if (ret)
+        gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        snprintf (base_key, sizeof (base_key), "brick%d", count);
-        snprintf (key, sizeof (key), "%s.hostname", base_key);
-        ret = dict_set_str (dict, key, "Snapshot Daemon");
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        snprintf (key, sizeof (key), "%s.path", base_key);
-        ret = dict_set_dynstr (dict, key, gf_strdup (uuid_utoa (MY_UUID)));
-        if (ret)
+int
+glusterd_snap_config_use_rsp_dict(dict_t *dst, dict_t *src)
+{
+    char buf[PATH_MAX] = "";
+    char *volname = NULL;
+    int ret = -1;
+    int config_command = 0;
+    uint64_t i = 0;
+    uint64_t hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t soft_limit = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    uint64_t value = 0;
+    uint64_t voldisplaycount = 0;
+
+    if (!dst || !src) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "config-command", &config_command);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get config-command type");
+        goto out;
+    }
+
+    switch (config_command) {
+        case GF_SNAP_CONFIG_DISPLAY:
+            ret = dict_get_uint64(src, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                  &hard_limit);
+            if (!ret) {
+                ret = dict_set_uint64(
+                    dst, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, hard_limit);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set snap_max_hard_limit");
+                    goto out;
+                }
+            } else {
+                /* Received dummy response from other nodes */
+                ret = 0;
                 goto out;
+            }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.port", base_key);
-        ret = dict_set_int32 (dict, key, volinfo->snapd.port);
-        if (ret)
+            ret = dict_get_uint64(src, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                                  &soft_limit);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get snap_max_soft_limit");
                 goto out;
+            }
 
-        glusterd_svc_build_snapd_pidfile (volinfo, pidfile, sizeof (pidfile));
-
-        brick_online = gf_is_service_running (pidfile, &pid);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", base_key);
-        ret = dict_set_int32 (dict, key, pid);
-        if (ret)
+            ret = dict_set_uint64(dst, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                                  soft_limit);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set snap_max_soft_limit");
                 goto out;
+            }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.status", base_key);
-        ret = dict_set_int32 (dict, key, brick_online);
-
-out:
-        if (ret)
-                gf_msg_debug (this->name, 0, "Returning %d", ret);
-
-        return ret;
-}
-
-int
-glusterd_snap_config_use_rsp_dict (dict_t *dst, dict_t *src)
-{
-        char       buf[PATH_MAX]    = "";
-        char      *volname          = NULL;
-        int        ret              = -1;
-        int        config_command   = 0;
-        uint64_t   i                = 0;
-        uint64_t   hard_limit       = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t   soft_limit       = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-        uint64_t   value            = 0;
-        uint64_t   voldisplaycount  = 0;
-
-        if (!dst || !src) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "Source or Destination "
-                        "dict is empty.");
+            ret = dict_get_uint64(src, "voldisplaycount", &voldisplaycount);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get voldisplaycount");
                 goto out;
-        }
+            }
 
-        ret = dict_get_int32 (dst, "config-command", &config_command);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "failed to get config-command type");
+            ret = dict_set_uint64(dst, "voldisplaycount", voldisplaycount);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set voldisplaycount");
                 goto out;
-        }
+            }
 
-        switch (config_command) {
-        case GF_SNAP_CONFIG_DISPLAY:
-                ret = dict_get_uint64 (src,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                       &hard_limit);
-                if (!ret) {
-                        ret = dict_set_uint64 (dst,
-                                         GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                         hard_limit);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to set snap_max_hard_limit");
-                                goto out;
-                        }
-                } else {
-                        /* Received dummy response from other nodes */
-                        ret = 0;
-                        goto out;
+            for (i = 0; i < voldisplaycount; i++) {
+                snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname", i);
+                ret = dict_get_str(src, buf, &volname);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
                 }
-
-                ret = dict_get_uint64 (src,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                                       &soft_limit);
+                ret = dict_set_str(dst, buf, volname);
                 if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get snap_max_soft_limit");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
                 }
 
-                ret = dict_set_uint64 (dst,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                                       soft_limit);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-hard-limit", i);
+                ret = dict_get_uint64(src, buf, &value);
                 if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set snap_max_soft_limit");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
                 }
-
-                ret = dict_get_uint64 (src, "voldisplaycount",
-                                       &voldisplaycount);
+                ret = dict_set_uint64(dst, buf, value);
                 if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get voldisplaycount");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
                 }
 
-                ret = dict_set_uint64 (dst, "voldisplaycount",
-                                       voldisplaycount);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-active-hard-limit", i);
+                ret = dict_get_uint64(src, buf, &value);
                 if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set voldisplaycount");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
+                }
+                ret = dict_set_uint64(dst, buf, value);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
                 }
 
-                for (i = 0; i < voldisplaycount; i++) {
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-volname", i);
-                        ret = dict_get_str (src, buf, &volname);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_str (dst, buf, volname);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-snap-max-hard-limit", i);
-                        ret = dict_get_uint64 (src, buf, &value);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_uint64 (dst, buf, value);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-active-hard-limit", i);
-                        ret = dict_get_uint64 (src, buf, &value);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_uint64 (dst, buf, value);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-snap-max-soft-limit", i);
-                        ret = dict_get_uint64 (src, buf, &value);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_uint64 (dst, buf, value);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-soft-limit", i);
+                ret = dict_get_uint64(src, buf, &value);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
                 }
+                ret = dict_set_uint64(dst, buf, value);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
+                }
+            }
 
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_merge_brick_status (dict_t *dst, dict_t *src)
+glusterd_merge_brick_status(dict_t *dst, dict_t *src)
 {
-        int64_t        volume_count             = 0;
-        int64_t        index                    = 0;
-        int64_t        j                        = 0;
-        int64_t        brick_count              = 0;
-        int64_t        brick_order              = 0;
-        char           key[PATH_MAX]            = {0, };
-        char           key_prefix[PATH_MAX]     = {0, };
-        char           snapbrckcnt[PATH_MAX]    = {0, };
-        char           snapbrckord[PATH_MAX]    = {0, };
-        char          *clonename                = NULL;
-        int            ret                      = -1;
-        int32_t        brick_online             = 0;
-        xlator_t      *this                     = NULL;
-        int32_t        snap_command             = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dst, "type", &snap_command);
+    int64_t volume_count = 0;
+    int64_t index = 0;
+    int64_t j = 0;
+    int64_t brick_count = 0;
+    int64_t brick_order = 0;
+    char key[64] = {
+        0,
+    };
+    char key_prefix[16] = {
+        0,
+    };
+    char snapbrckcnt[PATH_MAX] = {
+        0,
+    };
+    char snapbrckord[PATH_MAX] = {
+        0,
+    };
+    char *clonename = NULL;
+    int ret = -1;
+    int32_t brick_online = 0;
+    xlator_t *this = NULL;
+    int32_t snap_command = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "type", &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    if (snap_command == GF_SNAP_OPTION_TYPE_DELETE) {
+        gf_msg_debug(this->name, 0,
+                     "snapshot delete command."
+                     " Need not merge the status of the bricks");
+        ret = 0;
+        goto out;
+    }
+
+    /* Try and fetch clonename. If present set status with clonename *
+     * else do so as snap-vol */
+    ret = dict_get_str(dst, "clonename", &clonename);
+    if (ret) {
+        snprintf(key_prefix, sizeof(key_prefix), "snap-vol");
+    } else
+        snprintf(key_prefix, sizeof(key_prefix), "clone");
+
+    ret = dict_get_int64(src, "volcount", &volume_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+
+    for (index = 0; index < volume_count; index++) {
+        ret = snprintf(snapbrckcnt, sizeof(snapbrckcnt) - 1,
+                       "snap-vol%" PRId64 "_brickcount", index + 1);
+        ret = dict_get_int64(src, snapbrckcnt, &brick_count);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "unable to get the type of "
-                        "the snapshot command");
+            gf_msg_trace(this->name, 0,
+                         "No bricks for this volume in this dict (%s)",
+                         snapbrckcnt);
+            continue;
+        }
+
+        for (j = 0; j < brick_count; j++) {
+            /* Fetching data from source dict */
+            snprintf(snapbrckord, sizeof(snapbrckord) - 1,
+                     "snap-vol%" PRId64 ".brick%" PRId64 ".order", index + 1,
+                     j);
+
+            ret = dict_get_int64(src, snapbrckord, &brick_order);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get brick order (%s)", snapbrckord);
                 goto out;
-        }
-
-        if (snap_command == GF_SNAP_OPTION_TYPE_DELETE) {
-                gf_msg_debug (this->name, 0, "snapshot delete command."
-                        " Need not merge the status of the bricks");
-                ret = 0;
+            }
+
+            snprintf(key, sizeof(key), "%s%" PRId64 ".brick%" PRId64 ".status",
+                     key_prefix, index + 1, brick_order);
+            ret = dict_get_int32(src, key, &brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "failed to "
+                       "get the brick status (%s)",
+                       key);
                 goto out;
-        }
-
-        /* Try and fetch clonename. If present set status with clonename *
-         * else do so as snap-vol */
-        ret = dict_get_str (dst, "clonename", &clonename);
-        if (ret) {
-                snprintf (key_prefix, sizeof (key_prefix), "snap-vol");
-        } else
-                snprintf (key_prefix, sizeof (key_prefix), "clone");
-
-        ret = dict_get_int64 (src, "volcount", &volume_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to "
-                        "get the volume count");
+            }
+
+            ret = dict_set_int32(dst, key, brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to "
+                       "set the brick status (%s)",
+                       key);
                 goto out;
+            }
+            brick_online = 0;
         }
+    }
 
-        for (index = 0; index < volume_count; index++) {
-                ret = snprintf (snapbrckcnt, sizeof(snapbrckcnt) - 1,
-                                "snap-vol%"PRId64"_brickcount", index+1);
-                ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
-                if (ret) {
-                        gf_msg_trace (this->name, 0,
-                                "No bricks for this volume in this dict (%s)",
-                                snapbrckcnt);
-                        continue;
-                }
-
-                for (j = 0; j < brick_count; j++) {
-                        /* Fetching data from source dict */
-                        snprintf (snapbrckord, sizeof(snapbrckord) - 1,
-                                  "snap-vol%"PRId64".brick%"PRId64".order",
-                                  index+1, j);
-
-                        ret = dict_get_int64 (src, snapbrckord, &brick_order);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get brick order (%s)",
-                                        snapbrckord);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key) - 1,
-                                  "%s%"PRId64".brick%"PRId64".status",
-                                  key_prefix, index+1, brick_order);
-                        ret = dict_get_int32 (src, key, &brick_online);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "failed to "
-                                        "get the brick status (%s)", key);
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (dst, key, brick_online);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "failed to "
-                                        "set the brick status (%s)", key);
-                                goto out;
-                        }
-                        brick_online = 0;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* Aggregate missed_snap_counts from different nodes and save it *
  * in the req_dict of the originator node */
 int
-glusterd_snap_create_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_create_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        char          *buf                      = NULL;
-        char          *tmp_str                  = NULL;
-        char           name_buf[PATH_MAX]       = "";
-        int32_t        i                        = -1;
-        int32_t        ret                      = -1;
-        int32_t        src_missed_snap_count    = -1;
-        int32_t        dst_missed_snap_count    = -1;
-        xlator_t      *this                     = NULL;
-        int8_t         soft_limit_flag          = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = glusterd_merge_brick_status (dst, src);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_SET_INFO_FAIL, "failed to merge brick "
-                        "status");
-                goto out;
-        }
-
-        ret = dict_get_str (src, "snapuuid", &buf);
+    char *buf = NULL;
+    char *tmp_str = NULL;
+    char name_buf[PATH_MAX] = "";
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t src_missed_snap_count = -1;
+    int32_t dst_missed_snap_count = -1;
+    xlator_t *this = NULL;
+    int8_t soft_limit_flag = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = glusterd_merge_brick_status(dst, src);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SET_INFO_FAIL,
+               "failed to merge brick "
+               "status");
+        goto out;
+    }
+
+    ret = dict_get_str(src, "snapuuid", &buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get snap UUID");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dst, "snapuuid", buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap uuid in dict");
+        goto out;
+    }
+
+    /* set in dst dictionary soft-limit-reach only if soft-limit-reach
+     * is present src dictionary */
+    ret = dict_get_int8(src, "soft-limit-reach", &soft_limit_flag);
+    if (!ret) {
+        ret = dict_set_int8(dst, "soft-limit-reach", soft_limit_flag);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to get snap UUID");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "soft_limit_flag");
+            goto out;
         }
+    }
 
-        ret = dict_set_dynstr_with_alloc (dst, "snapuuid", buf);
+    ret = dict_get_int32(src, "missed_snap_count", &src_missed_snap_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No missed snaps");
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "missed_snap_count", &dst_missed_snap_count);
+    if (ret) {
+        /* Initialize dst_missed_count for the first time */
+        dst_missed_snap_count = 0;
+    }
+
+    for (i = 0; i < src_missed_snap_count; i++) {
+        snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d", i);
+        ret = dict_get_str(src, name_buf, &buf);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set snap uuid in dict");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %s", name_buf);
+            goto out;
         }
 
-        /* set in dst dictionary soft-limit-reach only if soft-limit-reach
-         * is present src dictionary */
-        ret = dict_get_int8 (src, "soft-limit-reach", &soft_limit_flag);
-        if (!ret) {
-                ret = dict_set_int8 (dst, "soft-limit-reach", soft_limit_flag);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set "
-                                "soft_limit_flag");
-                        goto out;
-                }
-        }
+        snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d",
+                 dst_missed_snap_count);
 
-        ret = dict_get_int32 (src, "missed_snap_count",
-                              &src_missed_snap_count);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "No missed snaps");
-                ret = 0;
-                goto out;
+        tmp_str = gf_strdup(buf);
+        if (!tmp_str) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dst, "missed_snap_count",
-                              &dst_missed_snap_count);
+        ret = dict_set_dynstr(dst, name_buf, tmp_str);
         if (ret) {
-                /* Initialize dst_missed_count for the first time */
-                dst_missed_snap_count = 0;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set %s", name_buf);
+            goto out;
         }
 
-        for (i = 0; i < src_missed_snap_count; i++) {
-                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d", i);
-                ret = dict_get_str (src, name_buf, &buf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to fetch %s", name_buf);
-                        goto out;
-                }
-
-                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
-                          dst_missed_snap_count);
-
-                tmp_str = gf_strdup (buf);
-                if (!tmp_str) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr (dst, name_buf, tmp_str);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set %s", name_buf);
-                        goto out;
-                }
+        tmp_str = NULL;
+        dst_missed_snap_count++;
+    }
 
-                tmp_str = NULL;
-                dst_missed_snap_count++;
-        }
-
-        ret = dict_set_int32 (dst, "missed_snap_count", dst_missed_snap_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set dst_missed_snap_count");
-                goto out;
-        }
+    ret = dict_set_int32(dst, "missed_snap_count", dst_missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set dst_missed_snap_count");
+        goto out;
+    }
 
 out:
-        if (ret && tmp_str)
-                GF_FREE(tmp_str);
+    if (ret && tmp_str)
+        GF_FREE(tmp_str);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_snap_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        int            ret            = -1;
-        int32_t        snap_command   = 0;
-
-        if (!dst || !src) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dst, "type", &snap_command);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int ret = -1;
+    int32_t snap_command = 0;
+
+    if (!dst || !src) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "type", &snap_command);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
         case GF_SNAP_OPTION_TYPE_DELETE:
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_snap_create_use_rsp_dict (dst, src);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_RSP_DICT_USE_FAIL,
-                                "Unable to use rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_create_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RSP_DICT_USE_FAIL,
+                       "Unable to use rsp dict");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = glusterd_snap_config_use_rsp_dict (dst, src);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                GD_MSG_RSP_DICT_USE_FAIL,
-                                "Unable to use rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_config_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RSP_DICT_USE_FAIL,
+                       "Unable to use rsp dict");
+                goto out;
+            }
+            break;
         default:
-                /* copy the response dictinary's contents to the dict to be
-                 * sent back to the cli */
-                dict_copy (src, dst);
-                break;
-        }
+            /* copy the response dictinary's contents to the dict to be
+             * sent back to the cli */
+            dict_copy(src, dst);
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_compare_snap_time (struct cds_list_head *list1,
-                            struct cds_list_head *list2)
+glusterd_compare_snap_time(struct cds_list_head *list1,
+                           struct cds_list_head *list2)
 {
-        glusterd_snap_t *snap1 = NULL;
-        glusterd_snap_t *snap2 = NULL;
-        double diff_time       = 0;
+    glusterd_snap_t *snap1 = NULL;
+    glusterd_snap_t *snap2 = NULL;
+    double diff_time = 0;
 
-        GF_ASSERT (list1);
-        GF_ASSERT (list2);
+    GF_ASSERT(list1);
+    GF_ASSERT(list2);
 
-        snap1 = cds_list_entry (list1, glusterd_snap_t, snap_list);
-        snap2 = cds_list_entry (list2, glusterd_snap_t, snap_list);
-        diff_time = difftime(snap1->time_stamp, snap2->time_stamp);
+    snap1 = cds_list_entry(list1, glusterd_snap_t, snap_list);
+    snap2 = cds_list_entry(list2, glusterd_snap_t, snap_list);
+    diff_time = difftime(snap1->time_stamp, snap2->time_stamp);
 
-        return (int)diff_time;
+    return (int)diff_time;
 }
 
 int
-glusterd_compare_snap_vol_time (struct cds_list_head *list1,
-                                struct cds_list_head *list2)
+glusterd_compare_snap_vol_time(struct cds_list_head *list1,
+                               struct cds_list_head *list2)
 {
-        glusterd_volinfo_t *snapvol1 = NULL;
-        glusterd_volinfo_t *snapvol2 = NULL;
-        double diff_time             = 0;
+    glusterd_volinfo_t *snapvol1 = NULL;
+    glusterd_volinfo_t *snapvol2 = NULL;
+    double diff_time = 0;
 
-        GF_ASSERT (list1);
-        GF_ASSERT (list2);
+    GF_ASSERT(list1);
+    GF_ASSERT(list2);
 
-        snapvol1 = cds_list_entry (list1, glusterd_volinfo_t, snapvol_list);
-        snapvol2 = cds_list_entry (list2, glusterd_volinfo_t, snapvol_list);
-        diff_time = difftime(snapvol1->snapshot->time_stamp,
-                             snapvol2->snapshot->time_stamp);
+    snapvol1 = cds_list_entry(list1, glusterd_volinfo_t, snapvol_list);
+    snapvol2 = cds_list_entry(list2, glusterd_volinfo_t, snapvol_list);
+    diff_time = difftime(snapvol1->snapshot->time_stamp,
+                         snapvol2->snapshot->time_stamp);
 
-        return (int)diff_time;
+    return (int)diff_time;
 }
 
 int32_t
-glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo)
+glusterd_missed_snapinfo_new(glusterd_missed_snap_info **missed_snapinfo)
 {
-        glusterd_missed_snap_info      *new_missed_snapinfo = NULL;
-        int32_t                         ret                 = -1;
-        xlator_t                       *this                = NULL;
+    glusterd_missed_snap_info *new_missed_snapinfo = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (missed_snapinfo);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(missed_snapinfo);
 
-        new_missed_snapinfo = GF_CALLOC (1, sizeof(*new_missed_snapinfo),
-                                         gf_gld_mt_missed_snapinfo_t);
+    new_missed_snapinfo = GF_CALLOC(1, sizeof(*new_missed_snapinfo),
+                                    gf_gld_mt_missed_snapinfo_t);
 
-        if (!new_missed_snapinfo)
-                goto out;
+    if (!new_missed_snapinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        CDS_INIT_LIST_HEAD (&new_missed_snapinfo->missed_snaps);
-        CDS_INIT_LIST_HEAD (&new_missed_snapinfo->snap_ops);
+    CDS_INIT_LIST_HEAD(&new_missed_snapinfo->missed_snaps);
+    CDS_INIT_LIST_HEAD(&new_missed_snapinfo->snap_ops);
 
-        *missed_snapinfo = new_missed_snapinfo;
+    *missed_snapinfo = new_missed_snapinfo;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op)
+glusterd_missed_snap_op_new(glusterd_snap_op_t **snap_op)
 {
-        glusterd_snap_op_t      *new_snap_op = NULL;
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
+    glusterd_snap_op_t *new_snap_op = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_op);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_op);
 
-        new_snap_op = GF_CALLOC (1, sizeof(*new_snap_op),
-                                 gf_gld_mt_missed_snapinfo_t);
+    new_snap_op = GF_CALLOC(1, sizeof(*new_snap_op),
+                            gf_gld_mt_missed_snapinfo_t);
 
-        if (!new_snap_op)
-                goto out;
+    if (!new_snap_op) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        new_snap_op->brick_num = -1;
-        new_snap_op->op = -1;
-        new_snap_op->status = -1;
-        CDS_INIT_LIST_HEAD (&new_snap_op->snap_ops_list);
+    new_snap_op->brick_num = -1;
+    new_snap_op->op = -1;
+    new_snap_op->status = -1;
+    CDS_INIT_LIST_HEAD(&new_snap_op->snap_ops_list);
 
-        *snap_op = new_snap_op;
+    *snap_op = new_snap_op;
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 gf_boolean_t
-mntopts_exists (const char *str, const char *opts)
+mntopts_exists(const char *str, const char *opts)
 {
-        char          *dup_val     = NULL;
-        char          *savetok     = NULL;
-        char          *token       = NULL;
-        gf_boolean_t   exists      = _gf_false;
+    char *dup_val = NULL;
+    char *savetok = NULL;
+    char *token = NULL;
+    gf_boolean_t exists = _gf_false;
 
-        GF_ASSERT (opts);
+    GF_ASSERT(opts);
 
-        if (!str || !strlen(str))
-                goto out;
+    if (!str || !strlen(str))
+        goto out;
 
-        dup_val = gf_strdup (str);
-        if (!dup_val)
-                goto out;
+    dup_val = gf_strdup(str);
+    if (!dup_val)
+        goto out;
 
-        token = strtok_r (dup_val, ",", &savetok);
-        while (token) {
-                if (!strcmp (token, opts)) {
-                        exists = _gf_true;
-                        goto out;
-                }
-                token = strtok_r (NULL, ",", &savetok);
+    token = strtok_r(dup_val, ",", &savetok);
+    while (token) {
+        if (!strcmp(token, opts)) {
+            exists = _gf_true;
+            goto out;
         }
+        token = strtok_r(NULL, ",", &savetok);
+    }
 
 out:
-        GF_FREE (dup_val);
-        return exists;
+    GF_FREE(dup_val);
+    return exists;
 }
 
 int32_t
-glusterd_mount_lvm_snapshot (glusterd_brickinfo_t *brickinfo,
-                             char *brick_mount_path)
+glusterd_mount_lvm_snapshot(glusterd_brickinfo_t *brickinfo,
+                            char *brick_mount_path)
 {
-        char               msg[NAME_MAX]  = "";
-        char               mnt_opts[1024] = "";
-        int32_t            ret            = -1;
-        runner_t           runner         = {0, };
-        xlator_t          *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_mount_path);
-        GF_ASSERT (brickinfo);
-
-
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "mount %s %s",
-                  brickinfo->device_path, brick_mount_path);
-
-        strcpy (mnt_opts, brickinfo->mnt_opts);
-
-        /* XFS file-system does not allow to mount file-system with duplicate
-         * UUID. File-system UUID of snapshot and its origin volume is same.
-         * Therefore to mount such a snapshot in XFS we need to pass nouuid
-         * option
-         */
-        if (!strcmp (brickinfo->fstype, "xfs") &&
-            !mntopts_exists (mnt_opts, "nouuid")) {
-                if (strlen (mnt_opts) > 0)
-                        strcat (mnt_opts, ",");
-                strcat (mnt_opts, "nouuid");
-        }
-
-
-        if (strlen (mnt_opts) > 0) {
-                runner_add_args (&runner, "mount", "-o", mnt_opts,
-                                brickinfo->device_path, brick_mount_path, NULL);
-        } else {
-                runner_add_args (&runner, "mount", brickinfo->device_path,
-                                 brick_mount_path, NULL);
-        }
-
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_MOUNT_FAIL, "mounting the snapshot "
-                        "logical device %s failed (error: %s)",
-                        brickinfo->device_path, strerror (errno));
-                goto out;
-        } else
-                gf_msg_debug (this->name, 0, "mounting the snapshot "
-                        "logical device %s successful", brickinfo->device_path);
+    char msg[NAME_MAX] = "";
+    char mnt_opts[1024] = "";
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_mount_path);
+    GF_ASSERT(brickinfo);
+
+    runinit(&runner);
+    len = snprintf(msg, sizeof(msg), "mount %s %s", brickinfo->device_path,
+                   brick_mount_path);
+    if (len < 0) {
+        strcpy(msg, "<error>");
+    }
+
+    gf_strncpy(mnt_opts, brickinfo->mnt_opts, sizeof(mnt_opts));
+
+    /* XFS file-system does not allow to mount file-system with duplicate
+     * UUID. File-system UUID of snapshot and its origin volume is same.
+     * Therefore to mount such a snapshot in XFS we need to pass nouuid
+     * option
+     */
+    if (!strcmp(brickinfo->fstype, "xfs") &&
+        !mntopts_exists(mnt_opts, "nouuid")) {
+        if (strlen(mnt_opts) > 0)
+            strcat(mnt_opts, ",");
+        strcat(mnt_opts, "nouuid");
+    }
+
+    if (strlen(mnt_opts) > 0) {
+        runner_add_args(&runner, "mount", "-o", mnt_opts,
+                        brickinfo->device_path, brick_mount_path, NULL);
+    } else {
+        runner_add_args(&runner, "mount", brickinfo->device_path,
+                        brick_mount_path, NULL);
+    }
+
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_MOUNT_FAIL,
+               "mounting the snapshot "
+               "logical device %s failed (error: %s)",
+               brickinfo->device_path, strerror(errno));
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0,
+                     "mounting the snapshot "
+                     "logical device %s successful",
+                     brickinfo->device_path);
 
 out:
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 gf_boolean_t
-glusterd_volume_quorum_calculate (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                  int down_count, gf_boolean_t first_brick_on,
-                                  int8_t snap_force, int quorum_count,
-                                  char *quorum_type, char **op_errstr,
-                                  uint32_t *op_errno)
+glusterd_volume_quorum_calculate(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                 int down_count, gf_boolean_t first_brick_on,
+                                 int8_t snap_force, int quorum_count,
+                                 char *quorum_type, char **op_errstr,
+                                 uint32_t *op_errno)
 {
-        gf_boolean_t  quorum_met        = _gf_false;
-        char          err_str[PATH_MAX] = {0, };
-        xlator_t     *this              = NULL;
-        int           up_count          = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!volinfo || !dict) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
-                goto out;
-        }
+    gf_boolean_t quorum_met = _gf_false;
+    const char err_str[] = "One or more bricks may be down.";
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!volinfo || !dict) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    /* In a n-way replication where n >= 3 we should not take a snapshot
+     * if even one brick is down, irrespective of the quorum being met.
+     * TODO: Remove this restriction once n-way replication is
+     * supported with snapshot.
+     */
+    if (down_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+        *op_errno = EG_BRCKDWN;
+    } else {
+        quorum_met = _gf_true;
+    }
+
+    /* TODO : Support for n-way relication in snapshot*/
+out:
+    return quorum_met;
+}
 
-        /* In a n-way replication where n >= 3 we should not take a snapshot
-         * if even one brick is down, irrespective of the quorum being met.
-         * TODO: Remove this restriction once n-way replication is
-         * supported with snapshot.
-         */
-        if (down_count) {
-                snprintf (err_str, sizeof (err_str), "One or more bricks may "
-                          "be down.");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_DISCONNECTED, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
+static int32_t
+glusterd_volume_quorum_check(glusterd_volinfo_t *volinfo, int64_t index,
+                             dict_t *dict, const char *key_prefix,
+                             int8_t snap_force, int quorum_count,
+                             char *quorum_type, char **op_errstr,
+                             uint32_t *op_errno)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    int64_t i = 0;
+    int64_t j = 0;
+    char key[128] = {
+        0,
+    }; /* key_prefix is passed from above, but is really quite small */
+    int keylen;
+    int down_count = 0;
+    gf_boolean_t first_brick_on = _gf_true;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t quorum_met = _gf_false;
+    int distribute_subvols = 0;
+    int32_t brick_online = 0;
+    const char err_str[] = "quorum is not met";
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!volinfo || !dict) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    if ((!glusterd_is_volume_replicate(volinfo) ||
+         volinfo->replica_count < 3) &&
+        (GF_CLUSTER_TYPE_DISPERSE != volinfo->type)) {
+        for (i = 0; i < volinfo->brick_count; i++) {
+            /* for a pure distribute volume, and replica volume
+               with replica count 2, quorum is not met if even
+               one of its subvolumes is down
+            */
+            keylen = snprintf(key, sizeof(key),
+                              "%s%" PRId64 ".brick%" PRId64 ".status",
+                              key_prefix, index, i);
+            ret = dict_get_int32n(dict, key, keylen, &brick_online);
+            if (ret || !brick_online) {
+                ret = 1;
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SERVER_QUORUM_NOT_MET, "%s", err_str);
+                *op_errstr = gf_strdup(err_str);
                 *op_errno = EG_BRCKDWN;
                 goto out;
-        } else {
-                quorum_met = _gf_true;
-                goto out;
+            }
         }
-
-        up_count = volinfo->dist_leaf_count - down_count;
-
-        if (quorum_type && !strcmp (quorum_type, "fixed")) {
-                if (up_count >= quorum_count) {
-                        quorum_met = _gf_true;
-                        goto out;
-                }
-        } else {
-                if ((GF_CLUSTER_TYPE_DISPERSE != volinfo->type) &&
-                    (volinfo->dist_leaf_count % 2 == 0)) {
-                        if ((up_count > quorum_count) ||
-                            ((up_count == quorum_count) && first_brick_on)) {
-                                quorum_met = _gf_true;
-                                goto out;
-                        }
-                } else {
-                        if (up_count >= quorum_count) {
-                                quorum_met = _gf_true;
-                                goto out;
-                        }
+        ret = 0;
+        quorum_met = _gf_true;
+    } else {
+        distribute_subvols = volinfo->brick_count / volinfo->dist_leaf_count;
+        for (j = 0; j < distribute_subvols; j++) {
+            /* by default assume quorum is not met
+               TODO: Handle distributed striped replicate volumes
+               Currently only distributed replicate volumes are
+               handled.
+            */
+            ret = 1;
+            quorum_met = _gf_false;
+            for (i = 0; i < volinfo->dist_leaf_count; i++) {
+                keylen = snprintf(
+                    key, sizeof(key), "%s%" PRId64 ".brick%" PRId64 ".status",
+                    key_prefix, index, (j * volinfo->dist_leaf_count) + i);
+                ret = dict_get_int32n(dict, key, keylen, &brick_online);
+                if (ret || !brick_online) {
+                    if (i == 0)
+                        first_brick_on = _gf_false;
+                    down_count++;
                 }
-        }
-
-        if (!quorum_met) {
-                snprintf (err_str, sizeof (err_str), "quorum is not met");
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-                *op_errno = EG_BRCKDWN;
-        }
+            }
 
-out:
-        return quorum_met;
-}
-
-int32_t
-glusterd_volume_quorum_check (glusterd_volinfo_t *volinfo, int64_t index,
-                              dict_t *dict, char *key_prefix,
-                              int8_t snap_force, int quorum_count,
-                              char *quorum_type, char **op_errstr,
-                              uint32_t *op_errno)
-{
-        int                      ret                = 0;
-        xlator_t                *this               = NULL;
-        int64_t                  i                  = 0;
-        int64_t                  j                  = 0;
-        char                     key[1024]          = {0, };
-        int                      down_count         = 0;
-        gf_boolean_t             first_brick_on     = _gf_true;
-        glusterd_conf_t         *priv               = NULL;
-        gf_boolean_t             quorum_met         = _gf_false;
-        int                      distribute_subvols = 0;
-        int32_t                  brick_online       = 0;
-        char                     err_str[PATH_MAX]  = {0, };
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!volinfo || !dict) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
+            quorum_met = glusterd_volume_quorum_calculate(
+                volinfo, dict, down_count, first_brick_on, snap_force,
+                quorum_count, quorum_type, op_errstr, op_errno);
+            /* goto out if quorum is not met */
+            if (!quorum_met) {
+                ret = -1;
                 goto out;
-        }
+            }
 
-        if ((!glusterd_is_volume_replicate (volinfo) ||
-             volinfo->replica_count < 3) &&
-            (GF_CLUSTER_TYPE_DISPERSE != volinfo->type)) {
-                for (i = 0; i < volinfo->brick_count ; i++) {
-                        /* for a pure distribute volume, and replica volume
-                           with replica count 2, quorum is not met if even
-                           one of its subvolumes is down
-                        */
-                        snprintf (key, sizeof (key),
-                                  "%s%"PRId64".brick%"PRId64".status",
-                                  key_prefix, index, i);
-                        ret = dict_get_int32 (dict, key, &brick_online);
-                        if (ret || !brick_online) {
-                                ret = 1;
-                                snprintf (err_str, sizeof (err_str), "quorum "
-                                          "is not met");
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        0, GD_MSG_SERVER_QUORUM_NOT_MET, "%s",
-                                        err_str);
-                                *op_errstr = gf_strdup (err_str);
-                                *op_errno = EG_BRCKDWN;
-                                goto out;
-                        }
-                }
-                ret = 0;
-                quorum_met = _gf_true;
-        } else {
-             distribute_subvols = volinfo->brick_count /
-                                  volinfo->dist_leaf_count;
-             for (j = 0; j < distribute_subvols; j++) {
-                        /* by default assume quorum is not met
-                           TODO: Handle distributed striped replicate volumes
-                           Currently only distributed replicate volumes are
-                           handled.
-                        */
-                        ret = 1;
-                        quorum_met = _gf_false;
-                        for (i = 0; i < volinfo->dist_leaf_count; i++) {
-                                snprintf (key, sizeof (key),
-                                          "%s%"PRId64".brick%"PRId64".status",
-                                          key_prefix, index,
-                                          (j * volinfo->dist_leaf_count) + i);
-                                ret = dict_get_int32 (dict, key, &brick_online);
-                                if (ret || !brick_online) {
-                                        if (i == 0)
-                                                first_brick_on = _gf_false;
-                                        down_count++;
-                                }
-                        }
-
-                        quorum_met = glusterd_volume_quorum_calculate (volinfo,
-                                                                       dict,
-                                                                    down_count,
-                                                                first_brick_on,
-                                                                    snap_force,
-                                                                  quorum_count,
-                                                                   quorum_type,
-                                                                   op_errstr,
-                                                                   op_errno);
-                        /* goto out if quorum is not met */
-                        if (!quorum_met) {
-                                ret = -1;
-                                goto out;
-                        }
-
-                        down_count = 0;
-                        first_brick_on = _gf_true;
-                }
+            down_count = 0;
+            first_brick_on = _gf_true;
         }
+    }
 
-        if (quorum_met) {
-                gf_msg_debug (this->name, 0, "volume %s is in quorum",
-                        volinfo->volname);
-                ret = 0;
-        }
+    if (quorum_met) {
+        gf_msg_debug(this->name, 0, "volume %s is in quorum", volinfo->volname);
+        ret = 0;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_snap_common_quorum_calculate (glusterd_volinfo_t *volinfo,
-                                       dict_t *dict, int64_t index,
-                                       char *key_prefix,
-                                       int8_t snap_force,
-                                       gf_boolean_t snap_volume,
-                                       char **op_errstr,
-                                       uint32_t *op_errno)
+static int32_t
+glusterd_snap_common_quorum_calculate(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                      int64_t index, const char *key_prefix,
+                                      int8_t snap_force,
+                                      gf_boolean_t snap_volume,
+                                      char **op_errstr, uint32_t *op_errno)
 {
-        int                 quorum_count      = 0;
-        char               *quorum_type       = NULL;
-        int32_t             tmp               = 0;
-        int32_t             ret               = -1;
-        xlator_t           *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        /* for replicate volumes with replica count equal to or
-           greater than 3, do quorum check by getting what type
-           of quorum rule has been set by getting the volume
-           option set. If getting the option fails, then assume
-           default.
-           AFR does this:
-           if quorum type is "auto":
-           - for odd numner of bricks (n), n/2 + 1
-           bricks should be present
-           - for even number of bricks n, n/2 bricks
-           should be present along with the 1st
-           subvolume
-           if quorum type is not "auto":
-           - get the quorum count from dict with the
-           help of the option "cluster.quorum-count"
-           if the option is not there in the dict,
-           then assume quorum type is auto and follow
-           the above method.
-           For non replicate volumes quorum is met only if all
-           the bricks of the volume are online
+    int quorum_count = 0;
+    char *quorum_type = NULL;
+    int32_t tmp = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+
+    /* for replicate volumes with replica count equal to or
+       greater than 3, do quorum check by getting what type
+       of quorum rule has been set by getting the volume
+       option set. If getting the option fails, then assume
+       default.
+       AFR does this:
+       if quorum type is "auto":
+       - for odd number of bricks (n), n/2 + 1
+       bricks should be present
+       - for even number of bricks n, n/2 bricks
+       should be present along with the 1st
+       subvolume
+       if quorum type is not "auto":
+       - get the quorum count from dict with the
+       help of the option "cluster.quorum-count"
+       if the option is not there in the dict,
+       then assume quorum type is auto and follow
+       the above method.
+       For non replicate volumes quorum is met only if all
+       the bricks of the volume are online
+     */
+
+    if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
+        if (volinfo->replica_count % 2 == 0)
+            quorum_count = volinfo->replica_count / 2;
+        else
+            quorum_count = volinfo->replica_count / 2 + 1;
+    } else if (GF_CLUSTER_TYPE_DISPERSE == volinfo->type) {
+        quorum_count = volinfo->disperse_count - volinfo->redundancy_count;
+    } else {
+        quorum_count = volinfo->brick_count;
+    }
+
+    ret = dict_get_str_sizen(volinfo->dict, "cluster.quorum-type",
+                             &quorum_type);
+    if (!ret && !strcmp(quorum_type, "fixed")) {
+        ret = dict_get_int32_sizen(volinfo->dict, "cluster.quorum-count", &tmp);
+        /* if quorum-type option is not found in the
+           dict assume auto quorum type. i.e n/2 + 1.
+           The same assumption is made when quorum-count
+           option cannot be obtained from the dict (even
+           if the quorum-type option is not set to auto,
+           the behavior is set to the default behavior)
          */
-
-        if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
-                if (volinfo->replica_count % 2 == 0)
-                        quorum_count = volinfo->replica_count/2;
-                else
-                        quorum_count =
-                                       volinfo->replica_count/2 + 1;
-        } else if (GF_CLUSTER_TYPE_DISPERSE == volinfo->type) {
-                quorum_count = volinfo->disperse_count -
-                               volinfo->redundancy_count;
-        } else {
-                quorum_count = volinfo->brick_count;
-        }
-
-        ret = dict_get_str (volinfo->dict, "cluster.quorum-type",
-                            &quorum_type);
-        if (!ret && !strcmp (quorum_type, "fixed")) {
-                ret = dict_get_int32 (volinfo->dict,
-                                      "cluster.quorum-count", &tmp);
-                /* if quorum-type option is not found in the
-                   dict assume auto quorum type. i.e n/2 + 1.
-                   The same assumption is made when quorum-count
-                   option cannot be obtained from the dict (even
-                   if the quorum-type option is not set to auto,
-                   the behavior is set to the default behavior)
-                 */
-                if (!ret) {
-                        /* for dispersed volumes, only allow quorums
-                           equal or larger than minimum functional
-                           value.
-                         */
-                        if ((GF_CLUSTER_TYPE_DISPERSE != volinfo->type) ||
-                            (tmp >= quorum_count)) {
-                                quorum_count = tmp;
-                        } else {
-                                gf_msg(this->name, GF_LOG_INFO, 0,
-                                       GD_MSG_QUORUM_COUNT_IGNORED,
-                                       "Ignoring small quorum-count "
-                                       "(%d) on dispersed volume", tmp);
-                                quorum_type = NULL;
-                        }
-                } else
-                        quorum_type = NULL;
-        }
-
-        ret = glusterd_volume_quorum_check (volinfo, index, dict,
-                                            key_prefix,
-                                            snap_force,
-                                            quorum_count,
-                                            quorum_type,
-                                            op_errstr,
-                                            op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOL_NOT_FOUND, "volume %s "
-                        "is not in quorum", volinfo->volname);
-                goto out;
-        }
+        if (!ret) {
+            /* for dispersed volumes, only allow quorums
+               equal or larger than minimum functional
+               value.
+             */
+            if ((GF_CLUSTER_TYPE_DISPERSE != volinfo->type) ||
+                (tmp >= quorum_count)) {
+                quorum_count = tmp;
+            } else {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_QUORUM_COUNT_IGNORED,
+                       "Ignoring small quorum-count "
+                       "(%d) on dispersed volume",
+                       tmp);
+                quorum_type = NULL;
+            }
+        } else
+            quorum_type = NULL;
+    }
+
+    ret = glusterd_volume_quorum_check(volinfo, index, dict, key_prefix,
+                                       snap_force, quorum_count, quorum_type,
+                                       op_errstr, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume %s "
+               "is not in quorum",
+               volinfo->volname);
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_snap_quorum_check_for_clone (dict_t *dict, gf_boolean_t snap_volume,
-                                      char **op_errstr, uint32_t *op_errno)
+static int32_t
+glusterd_snap_quorum_check_for_clone(dict_t *dict, gf_boolean_t snap_volume,
+                                     char **op_errstr, uint32_t *op_errno)
 {
-        int32_t             force             = 0;
-        char                err_str[PATH_MAX] = {0, };
-        char                key_prefix[PATH_MAX] = {0, };
-        char               *snapname          = NULL;
-        glusterd_snap_t    *snap              = NULL;
-        glusterd_volinfo_t *volinfo           = NULL;
-        glusterd_volinfo_t *tmp_volinfo       = NULL;
-        char               *volname           = NULL;
-        int64_t             volcount          = 0;
-        char                key[PATH_MAX]     = {0, };
-        int64_t             i                 = 0;
-        int32_t             ret               = -1;
-        xlator_t           *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "dict is NULL");
-                goto out;
-        }
-
-        if (snap_volume) {
-                ret = dict_get_str (dict, "snapname", &snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "failed to "
-                                "get snapname");
-                        goto out;
-                }
-
-                snap = glusterd_find_snap_by_name (snapname);
-                if (!snap) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_NOT_FOUND, "failed to "
-                                "get the snapshot %s", snapname);
-                        ret = -1;
-                        goto out;
-                }
+    const char err_str[] = "glusterds are not in quorum";
+    char key_prefix[16] = {
+        0,
+    };
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    char *volname = NULL;
+    int64_t volcount = 0;
+    int64_t i = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is NULL");
+        goto out;
+    }
+
+    if (snap_volume) {
+        ret = dict_get_str_sizen(dict, "snapname", &snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get snapname");
+            goto out;
         }
 
-        /* Do a quorum check of glusterds also. Because, the missed snapshot
-         * information will be saved by glusterd and if glusterds are not in
-         * quorum, then better fail the snapshot
-         */
-        if (!does_gd_meet_server_quorum (this)) {
-                snprintf (err_str, sizeof (err_str),
-                          "glusterds are not in quorum");
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-                *op_errno = EG_NODEDWN;
-                ret = -1;
-                goto out;
-        } else
-                gf_msg_debug (this->name, 0, "glusterds are in quorum");
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
+        snap = glusterd_find_snap_by_name(snapname);
+        if (!snap) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                   "failed to "
+                   "get the snapshot %s",
+                   snapname);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* Do a quorum check of glusterds also. Because, the missed snapshot
+     * information will be saved by glusterd and if glusterds are not in
+     * quorum, then better fail the snapshot
+     */
+    if (!does_gd_meet_server_quorum(this)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "%s", err_str);
+        *op_errstr = gf_strdup(err_str);
+        *op_errno = EG_NODEDWN;
+        ret = -1;
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0, "glusterds are in quorum");
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get "
+               "volcount");
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        ret = dict_get_str_sizen(dict, "clonename", &volname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to get "
-                        "volcount");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get clonename");
+            goto out;
+        }
+
+        if (snap_volume && snap) {
+            cds_list_for_each_entry(tmp_volinfo, &snap->volumes, vol_list)
+            {
+                if (!tmp_volinfo) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                           "failed to get snap volume "
+                           "for snap %s",
+                           snapname);
+                    ret = -1;
+                    goto out;
+                }
+                volinfo = tmp_volinfo;
+            }
+        } else {
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                       "failed to find the volume %s", volname);
                 goto out;
+            }
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "%s%"PRId64,
-                          snap_volume?"snap-volname":"volname", i);
-                ret = dict_get_str (dict, "clonename", &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "failed to "
-                                "get clonename");
-                        goto out;
-                }
-
-                if (snap_volume && snap) {
-                        cds_list_for_each_entry (tmp_volinfo, &snap->volumes,
-                                                 vol_list) {
-                                if (!tmp_volinfo) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_SNAP_NOT_FOUND,
-                                                "failed to get snap volume "
-                                                "for snap %s", snapname);
-                                        ret = -1;
-                                        goto out;
-                                }
-                                volinfo = tmp_volinfo;
-                        }
-                } else {
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOL_NOT_FOUND,
-                                        "failed to find the volume %s",
-                                        volname);
-                                goto out;
-                        }
-                }
-
-                snprintf (key_prefix, sizeof (key_prefix),
-                          "%s", snap_volume?"vol":"clone");
+        snprintf(key_prefix, sizeof(key_prefix), "%s",
+                 snap_volume ? "vol" : "clone");
 
-                ret = glusterd_snap_common_quorum_calculate (volinfo,
-                                                             dict, i,
-                                                             key_prefix,
-                                                             0,
-                                                             snap_volume,
-                                                             op_errstr,
-                                                             op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOL_NOT_FOUND, "volume %s "
-                                "is not in quorum", volinfo->volname);
-                        goto out;
-                }
+        ret = glusterd_snap_common_quorum_calculate(
+            volinfo, dict, i, key_prefix, 0, snap_volume, op_errstr, op_errno);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+                   "volume %s "
+                   "is not in quorum",
+                   volname);
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
-int32_t
-glusterd_snap_quorum_check_for_create (dict_t *dict, gf_boolean_t snap_volume,
-                                       char **op_errstr, uint32_t *op_errno)
+static int32_t
+glusterd_snap_quorum_check_for_create(dict_t *dict, gf_boolean_t snap_volume,
+                                      char **op_errstr, uint32_t *op_errno)
 {
-        int8_t              snap_force        = 0;
-        int32_t             force             = 0;
-        char                err_str[PATH_MAX] = {0, };
-        char                key_prefix[PATH_MAX] = {0, };
-        char               *snapname          = NULL;
-        glusterd_snap_t    *snap              = NULL;
-        glusterd_volinfo_t *volinfo           = NULL;
-        char               *volname           = NULL;
-        int64_t             volcount          = 0;
-        char                key[PATH_MAX]     = {0, };
-        int64_t             i                 = 0;
-        int32_t             ret               = -1;
-        xlator_t           *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "dict is NULL");
-                goto out;
+    int8_t snap_force = 0;
+    int32_t force = 0;
+    const char err_str[] = "glusterds are not in quorum";
+    char key_prefix[16] = {
+        0,
+    };
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    int64_t volcount = 0;
+    char key[32] = {
+        0,
+    };
+    int64_t i = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is NULL");
+        goto out;
+    }
+
+    if (snap_volume) {
+        ret = dict_get_str(dict, "snapname", &snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get snapname");
+            goto out;
         }
 
-        if (snap_volume) {
-                ret = dict_get_str (dict, "snapname", &snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "failed to "
-                                "get snapname");
-                        goto out;
-                }
-
-                snap = glusterd_find_snap_by_name (snapname);
-                if (!snap) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_NOT_FOUND, "failed to "
-                                "get the snapshot %s", snapname);
-                        ret = -1;
-                        goto out;
-                }
+        snap = glusterd_find_snap_by_name(snapname);
+        if (!snap) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                   "failed to "
+                   "get the snapshot %s",
+                   snapname);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = dict_get_int32(dict, "flags", &force);
+    if (!ret && (force & GF_CLI_FLAG_OP_FORCE))
+        snap_force = 1;
+
+    /* Do a quorum check of glusterds also. Because, the missed snapshot
+     * information will be saved by glusterd and if glusterds are not in
+     * quorum, then better fail the snapshot
+     */
+    if (!does_gd_meet_server_quorum(this)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "%s", err_str);
+        *op_errstr = gf_strdup(err_str);
+        *op_errno = EG_NODEDWN;
+        ret = -1;
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0, "glusterds are in quorum");
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get "
+               "volcount");
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        snprintf(key, sizeof(key), "%s%" PRId64,
+                 snap_volume ? "snap-volname" : "volname", i);
+        ret = dict_get_str(dict, key, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get volname");
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "flags", &force);
-        if (!ret && (force & GF_CLI_FLAG_OP_FORCE))
-                snap_force = 1;
-
-        /* Do a quorum check of glusterds also. Because, the missed snapshot
-         * information will be saved by glusterd and if glusterds are not in
-         * quorum, then better fail the snapshot
-         */
-        if (!does_gd_meet_server_quorum (this)) {
-                snprintf (err_str, sizeof (err_str),
-                          "glusterds are not in quorum");
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-                *op_errno = EG_NODEDWN;
-                ret = -1;
+        if (snap_volume) {
+            ret = glusterd_snap_volinfo_find(volname, snap, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                       "failed to get snap volume %s "
+                       "for snap %s",
+                       volname, snapname);
                 goto out;
-        } else
-                gf_msg_debug (this->name, 0, "glusterds are in quorum");
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to get "
-                        "volcount");
+            }
+        } else {
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                       "failed to find the volume %s", volname);
                 goto out;
+            }
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "%s%"PRId64,
-                          snap_volume?"snap-volname":"volname", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "failed to "
-                                "get volname");
-                        goto out;
-                }
+        snprintf(key_prefix, sizeof(key_prefix), "%s",
+                 snap_volume ? "snap-vol" : "vol");
 
-                if (snap_volume) {
-                        ret = glusterd_snap_volinfo_find (volname, snap,
-                                                          &volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_NOT_FOUND,
-                                        "failed to get snap volume %s "
-                                        "for snap %s", volname,
-                                        snapname);
-                                goto out;
-                        }
-                } else {
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOL_NOT_FOUND,
-                                        "failed to find the volume %s",
-                                        volname);
-                                goto out;
-                        }
-                }
-
-                snprintf (key_prefix, sizeof (key_prefix),
-                          "%s", snap_volume?"snap-vol":"vol");
-
-                ret = glusterd_snap_common_quorum_calculate (volinfo,
-                                                             dict, i,
-                                                             key_prefix,
-                                                             snap_force,
-                                                             snap_volume,
-                                                             op_errstr,
-                                                             op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_VOL_NOT_FOUND, "volume %s "
-                                "is not in quorum", volinfo->volname);
-                        goto out;
-                }
+        ret = glusterd_snap_common_quorum_calculate(
+            volinfo, dict, i, key_prefix, snap_force, snap_volume, op_errstr,
+            op_errno);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+                   "volume %s "
+                   "is not in quorum",
+                   volinfo->volname);
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snap_quorum_check (dict_t *dict, gf_boolean_t snap_volume,
-                            char **op_errstr, uint32_t *op_errno)
+glusterd_snap_quorum_check(dict_t *dict, gf_boolean_t snap_volume,
+                           char **op_errstr, uint32_t *op_errno)
 {
-        int32_t             ret               = -1;
-        xlator_t           *this              = NULL;
-        int32_t             snap_command      = 0;
-        char                err_str[PATH_MAX] = {0, };
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_EMPTY, "dict is NULL");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    int32_t snap_command = 0;
+    const char err_str[] = "glusterds are not in quorum";
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is NULL");
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "type", &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = glusterd_snap_quorum_check_for_create (dict, snap_volume,
-                                                             op_errstr,
-                                                             op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_QUORUM_CHECK_FAIL, "Quorum check"
-                                "failed during snapshot create command");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_quorum_check_for_create(dict, snap_volume,
+                                                        op_errstr, op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+                       "Quorum check"
+                       "failed during snapshot create command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_snap_quorum_check_for_clone (dict, !snap_volume,
-                                                            op_errstr,
-                                                            op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_QUORUM_CHECK_FAIL, "Quorum check"
-                                "failed during snapshot clone command");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_quorum_check_for_clone(dict, !snap_volume,
+                                                       op_errstr, op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+                       "Quorum check"
+                       "failed during snapshot clone command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                if (!does_gd_meet_server_quorum (this)) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str),
-                                  "glusterds are not in quorum");
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SERVER_QUORUM_NOT_MET, "%s",
-                                err_str);
-                        *op_errstr = gf_strdup (err_str);
-                        *op_errno = EG_NODEDWN;
-                        goto out;
-                }
+            if (!does_gd_meet_server_quorum(this)) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SERVER_QUORUM_NOT_MET, "%s", err_str);
+                *op_errstr = gf_strdup(err_str);
+                *op_errno = EG_NODEDWN;
+                goto out;
+            }
 
-                gf_msg_debug (this->name, 0, "glusterds are in "
-                        "quorum");
-                break;
+            gf_msg_debug(this->name, 0,
+                         "glusterds are in "
+                         "quorum");
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_umount (const char *path)
+int
+glusterd_is_path_mounted(const char *path)
 {
-        char               msg[NAME_MAX] = "";
-        int32_t            ret           = -1;
-        runner_t           runner        = {0, };
-        xlator_t          *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (path);
-
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "umount path %s", path);
-        runner_add_args (&runner, _PATH_UMOUNT, "-f", path, NULL);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_GLUSTERD_UMOUNT_FAIL, "umounting %s failed (%s)",
-                        path, strerror (errno));
-
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    FILE *mtab = NULL;
+    struct mntent *part = NULL;
+    int is_mounted = 0;
+
+    if ((mtab = setmntent("/etc/mtab", "r")) != NULL) {
+        while ((part = getmntent(mtab)) != NULL) {
+            if ((part->mnt_fsname != NULL) &&
+                (strcmp(part->mnt_dir, path)) == 0) {
+                is_mounted = 1;
+                break;
+            }
+        }
+        endmntent(mtab);
+    }
+    return is_mounted;
 }
-
+/* This function will do unmount for snaps.
+ */
 int32_t
-glusterd_copy_file (const char *source, const char *destination)
+glusterd_snap_unmount(xlator_t *this, glusterd_volinfo_t *volinfo)
 {
-        int32_t         ret             =       -1;
-        xlator_t        *this           =       NULL;
-        char            buffer[1024]    =       "";
-        int             src_fd          =       -1;
-        int             dest_fd         =       -1;
-        int             read_len        =       -1;
-        struct  stat    stbuf           =       {0,};
-        mode_t          dest_mode       =       0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (source);
-        GF_ASSERT (destination);
-
-        /* Here is stat is made to get the file permission of source file*/
-        ret = sys_lstat (source, &stbuf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED, "%s not found", source);
-                goto out;
-        }
+    char *brick_mount_path = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    int retry_count = 0;
 
-        dest_mode = stbuf.st_mode & 0777;
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo);
 
-        src_fd = open (source, O_RDONLY);
-        if (src_fd < 0) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED, "Unable to open file %s",
-                        source);
-                goto out;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        /* If the brick is not of this node, we continue */
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            continue;
+        }
+        /* If snapshot is pending, we continue */
+        if (brickinfo->snap_status == -1) {
+            continue;
         }
 
-        dest_fd = sys_creat (destination, dest_mode);
-        if (dest_fd < 0) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Unble to open a file %s", destination);
-                goto out;
+        ret = glusterd_find_brick_mount_path(brickinfo->path,
+                                             &brick_mount_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNTPATH_GET_FAIL,
+                   "Failed to find brick_mount_path for %s", brickinfo->path);
+            goto out;
+        }
+        /* unmount cannot be done when the brick process is still in
+         * the process of shutdown, so give three re-tries
+         */
+        retry_count = 0;
+        while (retry_count <= 2) {
+            retry_count++;
+            /* umount2 system call doesn't cleanup mtab entry
+             * after un-mount, using external umount command.
+             */
+            ret = glusterd_umount(brick_mount_path);
+            if (!ret)
+                break;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+                   "umount failed "
+                   "for path %s (brick: %s): %s. Retry(%d)",
+                   brick_mount_path, brickinfo->path, strerror(errno),
+                   retry_count);
+            sleep(3);
         }
+    }
 
-        do {
-                ret = sys_read (src_fd, buffer, sizeof (buffer));
-                if (ret ==  -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED, "Error reading file "
-                                "%s", source);
-                        goto out;
-                }
-                read_len = ret;
-                if (read_len == 0)
-                        break;
-
-                ret = sys_write (dest_fd, buffer, read_len);
-                if (ret != read_len) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FILE_OP_FAILED, "Error writing in "
-                                "file %s", destination);
-                        goto out;
-                }
-        } while (ret > 0);
 out:
-        if (src_fd > 0)
-                sys_close (src_fd);
+    if (brick_mount_path)
+        GF_FREE(brick_mount_path);
 
-        if (dest_fd > 0)
-                sys_close (dest_fd);
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_copy_folder (const char *source, const char *destination)
+glusterd_umount(const char *path)
 {
-        DIR             *dir_ptr                =       NULL;
-        struct dirent   *direntp                =       NULL;
-        int32_t         ret                     =       -1;
-        char            src_path[PATH_MAX]      =       "";
-        char            dest_path[PATH_MAX]     =       "";
-        xlator_t        *this                   =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (source);
-        GF_ASSERT (destination);
-
-        dir_ptr = sys_opendir (source);
-        if (!dir_ptr) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,  "Unable to open %s", source);
-                goto out;
-        }
-
-        while ((direntp = sys_readdir (dir_ptr)) != NULL) {
-                if (strcmp (direntp->d_name, ".") == 0 ||
-                    strcmp (direntp->d_name, "..") == 0)
-                        continue;
-                ret = snprintf (src_path, sizeof (src_path), "%s/%s",
-                                source, direntp->d_name);
-                if (ret < 0)
-                        goto out;
-
-                ret = snprintf (dest_path, sizeof (dest_path), "%s/%s",
-                                destination, direntp->d_name);
-                if (ret < 0)
-                        goto out;
+    char msg[NAME_MAX] = "";
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(path);
+
+    if (!glusterd_is_path_mounted(path)) {
+        return 0;
+    }
+
+    runinit(&runner);
+    snprintf(msg, sizeof(msg), "umount path %s", path);
+    runner_add_args(&runner, _PATH_UMOUNT, "-f", path, NULL);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+               "umounting %s failed (%s)", path, strerror(errno));
+
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
+}
 
-                ret = glusterd_copy_file (src_path, dest_path);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY, "Could not copy "
-                                "%s to %s", src_path, dest_path);
-                        goto out;
-                }
-        }
+int32_t
+glusterd_copy_file(const char *source, const char *destination)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char buffer[1024] = "";
+    int src_fd = -1;
+    int dest_fd = -1;
+    int read_len = -1;
+    struct stat stbuf = {
+        0,
+    };
+    mode_t dest_mode = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(source);
+    GF_ASSERT(destination);
+
+    /* Here is stat is made to get the file permission of source file*/
+    ret = sys_lstat(source, &stbuf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "%s not found", source);
+        goto out;
+    }
+
+    dest_mode = stbuf.st_mode & 0777;
+
+    src_fd = open(source, O_RDONLY);
+    if (src_fd == -1) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to open file %s", source);
+        goto out;
+    }
+
+    dest_fd = sys_creat(destination, dest_mode);
+    if (dest_fd < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+               "Unble to open a file %s", destination);
+        goto out;
+    }
+
+    do {
+        ret = sys_read(src_fd, buffer, sizeof(buffer));
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Error reading file "
+                   "%s",
+                   source);
+            goto out;
+        }
+        read_len = ret;
+        if (read_len == 0)
+            break;
+
+        ret = sys_write(dest_fd, buffer, read_len);
+        if (ret != read_len) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                   "Writing in "
+                   "file %s failed with error %s",
+                   destination, strerror(errno));
+            goto out;
+        }
+    } while (ret > 0);
 out:
-        if (dir_ptr)
-                sys_closedir (dir_ptr);
+    if (src_fd != -1)
+        sys_close(src_fd);
 
-        return ret;
+    if (dest_fd > 0)
+        sys_close(dest_fd);
+    return ret;
 }
 
 int32_t
-glusterd_get_geo_rep_session (char *slave_key, char *origin_volname,
-                              dict_t *gsync_slaves_dict, char *session,
-                              char *slave)
+glusterd_copy_folder(const char *source, const char *destination)
 {
-        int32_t         ret             =       -1;
-        char            *token          =       NULL;
-        char            *tok            =       NULL;
-        char            *temp           =       NULL;
-        char            *ip             =       NULL;
-        char            *ip_i           =       NULL;
-        char            *ip_temp        =       NULL;
-        char            *buffer         =       NULL;
-        xlator_t        *this           =       NULL;
-        char            *slave_temp     =       NULL;
-        char            *save_ptr       =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (slave_key);
-        GF_ASSERT (origin_volname);
-        GF_ASSERT (gsync_slaves_dict);
-
-        ret = dict_get_str (gsync_slaves_dict, slave_key, &buffer);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to "
-                        "get value for key %s", slave_key);
-                goto out;
-        }
-
-        temp = gf_strdup (buffer);
-        if (!temp) {
-                ret = -1;
-                goto out;
-        }
-
-        /* geo-rep session string format being parsed:
-         * "master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid"
-         */
-        token = strtok_r (temp, "/", &save_ptr);
-
-        token = strtok_r (NULL, ":", &save_ptr);
-        if (!token) {
-                ret = -1;
-                goto out;
-        }
-        token++;
-
-        ip = gf_strdup (token);
-        if (!ip) {
-                ret = -1;
-                goto out;
-        }
-        ip_i = ip;
-
-        token = strtok_r (NULL, ":", &save_ptr);
-        if (!token) {
-                ret = -1;
-                goto out;
-        }
-
-        slave_temp = gf_strdup (token);
-        if (!slave) {
-                ret = -1;
-                goto out;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    DIR *dir_ptr = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char src_path[PATH_MAX] = {
+        0,
+    };
+    char dest_path[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(source);
+    GF_ASSERT(destination);
+
+    dir_ptr = sys_opendir(source);
+    if (!dir_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Unable to open %s", source);
+        goto out;
+    }
+
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(dir_ptr, scratch);
+        if (!entry || errno != 0)
+            break;
+
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
+        ret = snprintf(src_path, sizeof(src_path), "%s/%s", source,
+                       entry->d_name);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        /* If 'ip' has 'root@slavehost', point to 'slavehost' as
-         * working directory for root users are created without
-         * 'root@' */
-        ip_temp = gf_strdup (ip);
-        tok = strtok_r (ip_temp, "@", &save_ptr);
-        if (tok && !strcmp (tok, "root"))
-                ip_i = ip + 5;
-
-        ret = snprintf (session, PATH_MAX, "%s_%s_%s",
-                        origin_volname, ip_i, slave_temp);
-        if (ret < 0) /* Negative value is an error */
-                goto out;
-
-        ret = snprintf  (slave, PATH_MAX, "%s::%s", ip, slave_temp);
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/%s", destination,
+                       entry->d_name);
         if (ret < 0) {
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = 0; /* Success */
-
+        ret = glusterd_copy_file(src_path, dest_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Could not copy "
+                   "%s to %s",
+                   src_path, dest_path);
+            goto out;
+        }
+    }
 out:
-        if (temp)
-                GF_FREE (temp);
-
-        if (ip)
-                GF_FREE (ip);
+    if (dir_ptr)
+        (void)sys_closedir(dir_ptr);
 
-        if (ip_temp)
-                GF_FREE (ip_temp);
-
-        if (slave_temp)
-                GF_FREE (slave_temp);
-
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_copy_quota_files (glusterd_volinfo_t *src_vol,
-                           glusterd_volinfo_t *dest_vol,
-                           gf_boolean_t *conf_present) {
-
-        int32_t         ret                     = -1;
-        char            src_dir[PATH_MAX]       = "";
-        char            dest_dir[PATH_MAX]      = "";
-        char            src_path[PATH_MAX]      = "";
-        char            dest_path[PATH_MAX]     = "";
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        struct  stat    stbuf                   = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (src_vol);
-        GF_ASSERT (dest_vol);
-
-        GLUSTERD_GET_VOLUME_DIR (src_dir, src_vol, priv);
-
-        GLUSTERD_GET_VOLUME_DIR (dest_dir, dest_vol, priv);
-
-        ret = snprintf (src_path, sizeof (src_path), "%s/quota.conf",
-                        src_dir);
-        if (ret < 0)
-                goto out;
+glusterd_get_geo_rep_session(char *slave_key, char *origin_volname,
+                             dict_t *gsync_slaves_dict, char *session,
+                             char *slave)
+{
+    int32_t ret = -1;
+    int32_t len = 0;
+    char *token = NULL;
+    char *tok = NULL;
+    char *temp = NULL;
+    char *ip = NULL;
+    char *ip_i = NULL;
+    char *ip_temp = NULL;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    char *slave_temp = NULL;
+    char *save_ptr = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(slave_key);
+    GF_ASSERT(origin_volname);
+    GF_ASSERT(gsync_slaves_dict);
+
+    ret = dict_get_str(gsync_slaves_dict, slave_key, &buffer);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to "
+               "get value for key %s",
+               slave_key);
+        goto out;
+    }
+
+    temp = gf_strdup(buffer);
+    if (!temp) {
+        ret = -1;
+        goto out;
+    }
+
+    /* geo-rep session string format being parsed:
+     * "master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid"
+     */
+    token = strtok_r(temp, "/", &save_ptr);
+
+    token = strtok_r(NULL, ":", &save_ptr);
+    if (!token) {
+        ret = -1;
+        goto out;
+    }
+    token++;
+
+    ip = gf_strdup(token);
+    if (!ip) {
+        ret = -1;
+        goto out;
+    }
+    ip_i = ip;
+
+    token = strtok_r(NULL, ":", &save_ptr);
+    if (!token) {
+        ret = -1;
+        goto out;
+    }
+
+    slave_temp = gf_strdup(token);
+    if (!slave) {
+        ret = -1;
+        goto out;
+    }
+
+    /* If 'ip' has 'root@slavehost', point to 'slavehost' as
+     * working directory for root users are created without
+     * 'root@' */
+    ip_temp = gf_strdup(ip);
+    tok = strtok_r(ip_temp, "@", &save_ptr);
+    len = strlen(tok);
+    tok = strtok_r(NULL, "@", &save_ptr);
+    if (tok != NULL)
+        ip_i = ip + len + 1;
+
+    ret = snprintf(session, PATH_MAX, "%s_%s_%s", origin_volname, ip_i,
+                   slave_temp);
+    if (ret < 0) /* Negative value is an error */
+        goto out;
+
+    ret = snprintf(slave, PATH_MAX, "%s::%s", ip, slave_temp);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = 0; /* Success */
 
-        /* quota.conf is not present if quota is not enabled, Hence ignoring
-         * the absence of this file
-         */
-        ret = sys_lstat (src_path, &stbuf);
-        if (ret) {
-                ret = 0;
-                gf_msg_debug (this->name, 0, "%s not found", src_path);
-                goto out;
-        }
+out:
+    if (temp)
+        GF_FREE(temp);
 
-        ret = snprintf (dest_path, sizeof (dest_path), "%s/quota.conf",
-                       dest_dir);
-        if (ret < 0)
-                goto out;
+    if (ip)
+        GF_FREE(ip);
 
-        ret = glusterd_copy_file (src_path, dest_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Failed to copy %s in %s",
-                        src_path, dest_path);
-                goto out;
-        }
+    if (ip_temp)
+        GF_FREE(ip_temp);
 
-        ret = snprintf (src_path, sizeof (src_path), "%s/quota.cksum",
-                        src_dir);
-        if (ret < 0)
-                goto out;
+    if (slave_temp)
+        GF_FREE(slave_temp);
 
-        /* if quota.conf is present, quota.cksum has to be present. *
-         * Fail snapshot operation if file is absent                *
-         */
-        ret = sys_lstat (src_path, &stbuf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_FILE_NOT_FOUND, "%s not found", src_path);
-                goto out;
-        }
-
-        ret = snprintf (dest_path, sizeof (dest_path), "%s/quota.cksum",
-                       dest_dir);
-        if (ret < 0)
-                goto out;
-
-        ret = glusterd_copy_file (src_path, dest_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Failed to copy %s in %s",
-                        src_path, dest_path);
-                goto out;
-        }
+    return ret;
+}
 
-        *conf_present = _gf_true;
+int32_t
+glusterd_copy_quota_files(glusterd_volinfo_t *src_vol,
+                          glusterd_volinfo_t *dest_vol,
+                          gf_boolean_t *conf_present)
+{
+    int32_t ret = -1;
+    char src_dir[PATH_MAX] = "";
+    char dest_dir[PATH_MAX] = "";
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(src_vol);
+    GF_ASSERT(dest_vol);
+
+    GLUSTERD_GET_VOLUME_DIR(src_dir, src_vol, priv);
+
+    GLUSTERD_GET_VOLUME_DIR(dest_dir, dest_vol, priv);
+
+    ret = snprintf(src_path, sizeof(src_path), "%s/quota.conf", src_dir);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    /* quota.conf is not present if quota is not enabled, Hence ignoring
+     * the absence of this file
+     */
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        ret = 0;
+        gf_msg_debug(this->name, 0, "%s not found", src_path);
+        goto out;
+    }
+
+    ret = snprintf(dest_path, sizeof(dest_path), "%s/quota.conf", dest_dir);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_copy_file(src_path, dest_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to copy %s in %s", src_path, dest_path);
+        goto out;
+    }
+
+    ret = snprintf(src_path, sizeof(src_path), "%s/quota.cksum", src_dir);
+    if (ret < 0)
+        goto out;
+
+    /* if quota.conf is present, quota.cksum has to be present. *
+     * Fail snapshot operation if file is absent                *
+     */
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_NOT_FOUND,
+               "%s not found", src_path);
+        goto out;
+    }
+
+    ret = snprintf(dest_path, sizeof(dest_path), "%s/quota.cksum", dest_dir);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_copy_file(src_path, dest_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to copy %s in %s", src_path, dest_path);
+        goto out;
+    }
+
+    *conf_present = _gf_true;
 out:
-        return ret;
-
+    return ret;
 }
 
 /* *
@@ -3693,357 +3842,374 @@ out:
  * volname with clonename
  */
 int
-glusterd_copy_nfs_ganesha_file (glusterd_volinfo_t *src_vol,
-                                glusterd_volinfo_t *dest_vol)
+glusterd_copy_nfs_ganesha_file(glusterd_volinfo_t *src_vol,
+                               glusterd_volinfo_t *dest_vol)
 {
+    int32_t ret = -1;
+    char snap_dir[PATH_MAX] = {
+        0,
+    };
+    char src_path[PATH_MAX] = {
+        0,
+    };
+    char dest_path[PATH_MAX] = {
+        0,
+    };
+    char buffer[BUFSIZ] = {
+        0,
+    };
+    char *find_ptr = NULL;
+    char *buff_ptr = NULL;
+    char *tmp_ptr = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    FILE *src = NULL;
+    FILE *dest = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("snapshot", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, src_vol, out);
+    GF_VALIDATE_OR_GOTO(this->name, dest_vol, out);
+
+    if (glusterd_check_ganesha_export(src_vol) == _gf_false) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not exported via "
+                     "NFS-Ganesha. Skipping copy of export conf.",
+                     src_vol->volname);
+        ret = 0;
+        goto out;
+    }
+
+    if (src_vol->is_snap_volume) {
+        GLUSTERD_GET_SNAP_DIR(snap_dir, src_vol->snapshot, priv);
+        ret = snprintf(src_path, PATH_MAX, "%s/export.%s.conf", snap_dir,
+                       src_vol->snapshot->snapname);
+    } else {
+        ret = snprintf(src_path, PATH_MAX, "%s/export.%s.conf",
+                       GANESHA_EXPORT_DIRECTORY, src_vol->volname);
+    }
+    if (ret < 0 || ret >= PATH_MAX)
+        goto out;
+
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        /*
+         * This code path is hit, only when the src_vol is being *
+         * exported via NFS-Ganesha. So if the conf file is not  *
+         * available, we fail the snapshot operation.            *
+         */
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Stat on %s failed with %s", src_path, strerror(errno));
+        goto out;
+    }
+
+    if (dest_vol->is_snap_volume) {
+        memset(snap_dir, 0, PATH_MAX);
+        GLUSTERD_GET_SNAP_DIR(snap_dir, dest_vol->snapshot, priv);
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf",
+                       snap_dir, dest_vol->snapshot->snapname);
+        if (ret < 0)
+            goto out;
 
-        int32_t         ret                     = -1;
-        char            snap_dir[PATH_MAX]      = {0,};
-        char            src_path[PATH_MAX]      = {0,};
-        char            dest_path[PATH_MAX]     = {0,};
-        char            buffer[BUFSIZ]          = {0,};
-        char            *find_ptr               = NULL;
-        char            *buff_ptr               = NULL;
-        char            *tmp_ptr                = NULL;
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        struct  stat    stbuf                   = {0,};
-        FILE            *src                    = NULL;
-        FILE            *dest                   = NULL;
-
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("snapshot", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, src_vol, out);
-        GF_VALIDATE_OR_GOTO (this->name, dest_vol, out);
-
-        if (src_vol->is_snap_volume) {
-                GLUSTERD_GET_SNAP_DIR (snap_dir, src_vol->snapshot, priv);
-                ret = snprintf (src_path, PATH_MAX, "%s/export.%s.conf",
-                                snap_dir, src_vol->snapshot->snapname);
-        } else {
-                ret = snprintf (src_path, PATH_MAX, "%s/export.%s.conf",
-                                GANESHA_EXPORT_DIRECTORY, src_vol->volname);
-        }
-        if (ret < 0 || ret >= PATH_MAX)
-                goto out;
-
-        ret = sys_lstat (src_path, &stbuf);
+        ret = glusterd_copy_file(src_path, dest_path);
         if (ret) {
-                /* *
-                * If export file is not present, volume is not exported
-                * via ganesha. So it is not necessary to copy that during
-                * snapshot.
-                */
-                if (errno == ENOENT) {
-                        ret = 0;
-                        gf_msg_debug (this->name, 0, "%s not found", src_path);
-                } else
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Stat on %s failed with %s",
-                                src_path, strerror (errno));
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Failed to copy %s in %s", src_path, dest_path);
+            goto out;
         }
 
-        if (dest_vol->is_snap_volume) {
-                memset (snap_dir, 0 , PATH_MAX);
-                GLUSTERD_GET_SNAP_DIR (snap_dir, dest_vol->snapshot, priv);
-                ret = snprintf (dest_path, sizeof (dest_path),
-                                "%s/export.%s.conf", snap_dir,
-                                dest_vol->snapshot->snapname);
-                if (ret < 0)
-                        goto out;
-
-                ret = glusterd_copy_file (src_path, dest_path);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY, "Failed to copy %s in %s",
-                                src_path, dest_path);
-                        goto out;
-                }
-
-        } else {
-                ret = snprintf (dest_path, sizeof (dest_path),
-                                "%s/export.%s.conf", GANESHA_EXPORT_DIRECTORY,
-                                dest_vol->volname);
-                if (ret < 0)
-                        goto out;
-
-                src = fopen (src_path, "r");
-                dest = fopen (dest_path, "w");
-
-                /* *
-                 * if the source volume is snapshot, the export conf file
-                 * consists of orginal volname
-                 */
-                if (src_vol->is_snap_volume)
-                        find_ptr = gf_strdup (src_vol->parent_volname);
-                else
-                        find_ptr = gf_strdup (src_vol->volname);
+    } else {
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf",
+                       GANESHA_EXPORT_DIRECTORY, dest_vol->volname);
+        if (ret < 0)
+            goto out;
 
-                if (!find_ptr)
-                        goto out;
+        src = fopen(src_path, "r");
+        dest = fopen(dest_path, "w");
 
-                /* Replacing volname with clonename */
-                while (fgets(buffer, BUFSIZ, src)) {
-                        buff_ptr = buffer;
-                        while ((tmp_ptr = strstr(buff_ptr, find_ptr))) {
-                                while (buff_ptr < tmp_ptr)
-                                        fputc((int)*buff_ptr++, dest);
-                                fputs(dest_vol->volname, dest);
-                                buff_ptr += strlen(find_ptr);
-                        }
-                        fputs(buff_ptr, dest);
-                        memset (buffer, 0, BUFSIZ);
-                }
+        if (!src || !dest) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                   "Failed to open %s", dest ? src_path : dest_path);
+            ret = -1;
+            goto out;
         }
+
+        /* *
+         * if the source volume is snapshot, the export conf file
+         * consists of orginal volname
+         */
+        if (src_vol->is_snap_volume)
+            find_ptr = gf_strdup(src_vol->parent_volname);
+        else
+            find_ptr = gf_strdup(src_vol->volname);
+
+        if (!find_ptr)
+            goto out;
+
+        /* Replacing volname with clonename */
+        while (fgets(buffer, BUFSIZ, src)) {
+            buff_ptr = buffer;
+            while ((tmp_ptr = strstr(buff_ptr, find_ptr))) {
+                while (buff_ptr < tmp_ptr)
+                    fputc((int)*buff_ptr++, dest);
+                fputs(dest_vol->volname, dest);
+                buff_ptr += strlen(find_ptr);
+            }
+            fputs(buff_ptr, dest);
+            memset(buffer, 0, BUFSIZ);
+        }
+    }
 out:
-        if (src)
-                fclose (src);
-        if (dest)
-                fclose (dest);
-        if (find_ptr)
-                GF_FREE(find_ptr);
-
-        return ret;
+    if (src)
+        fclose(src);
+    if (dest)
+        fclose(dest);
+    if (find_ptr)
+        GF_FREE(find_ptr);
+
+    return ret;
 }
 
 int32_t
-glusterd_restore_geo_rep_files (glusterd_volinfo_t *snap_vol)
+glusterd_restore_geo_rep_files(glusterd_volinfo_t *snap_vol)
 {
-        int32_t                 ret                     =       -1;
-        char                    src_path[PATH_MAX]      =       "";
-        char                    dest_path[PATH_MAX]     =       "";
-        xlator_t                *this                   =       NULL;
-        char                    *origin_volname         =       NULL;
-        glusterd_volinfo_t      *origin_vol             =       NULL;
-        int                     i                       =       0;
-        char                    key[PATH_MAX]           =       "";
-        char                    session[PATH_MAX]       =       "";
-        char                    slave[PATH_MAX]         =       "";
-        char                    snapgeo_dir[PATH_MAX]   =       "";
-        glusterd_conf_t         *priv                   =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (snap_vol);
-
-        origin_volname = gf_strdup (snap_vol->parent_volname);
-        if (!origin_volname) {
-                ret = -1;
-                goto out;
+    int32_t ret = -1;
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    char *origin_volname = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    int i = 0;
+    char key[32] = "";
+    char session[PATH_MAX] = "";
+    char slave[PATH_MAX] = "";
+    char snapgeo_dir[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(snap_vol);
+
+    origin_volname = gf_strdup(snap_vol->parent_volname);
+    if (!origin_volname) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(origin_volname, &origin_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to fetch "
+               "volinfo for volname %s",
+               origin_volname);
+        goto out;
+    }
+
+    for (i = 1; i <= snap_vol->gsync_slaves->count; i++) {
+        ret = snprintf(key, sizeof(key), "slave%d", i);
+        if (ret < 0) {
+            goto out;
         }
 
-        ret = glusterd_volinfo_find (origin_volname, &origin_vol);
+        /* "origin_vol" is used here because geo-replication saves
+         * the session in the form of master_ip_slave.
+         * As we need the master volume to be same even after
+         * restore, we are passing the origin volume name.
+         *
+         * "snap_vol->gsync_slaves" contain the slave information
+         * when the snapshot was taken, hence we have to restore all
+         * those slaves information when we do snapshot restore.
+         */
+        ret = glusterd_get_geo_rep_session(
+            key, origin_vol->volname, snap_vol->gsync_slaves, session, slave);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "Unable to fetch "
-                        "volinfo for volname %s", origin_volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GEOREP_GET_FAILED,
+                   "Failed to get geo-rep session");
+            goto out;
         }
 
-        for (i = 1 ; i <= snap_vol->gsync_slaves->count; i++) {
-                ret = snprintf (key, sizeof (key), "slave%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                /* "origin_vol" is used here because geo-replication saves
-                 * the session in the form of master_ip_slave.
-                 * As we need the master volume to be same even after
-                 * restore, we are passing the origin volume name.
-                 *
-                 * "snap_vol->gsync_slaves" contain the slave information
-                 * when the snapshot was taken, hence we have to restore all
-                 * those slaves information when we do snapshot restore.
-                 */
-                ret = glusterd_get_geo_rep_session (key, origin_vol->volname,
-                                                    snap_vol->gsync_slaves,
-                                                    session, slave);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GEOREP_GET_FAILED,
-                                "Failed to get geo-rep session");
-                        goto out;
-                }
-
-                GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot,
-                                              priv);
-                ret = snprintf (src_path, sizeof (src_path),
-                                "%s/%s", snapgeo_dir, session);
-                if (ret < 0)
-                        goto out;
+        GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot, priv);
+        ret = snprintf(src_path, sizeof(src_path), "%s/%s", snapgeo_dir,
+                       session);
+        if (ret < 0)
+            goto out;
 
-                ret = snprintf (dest_path, sizeof (dest_path),
-                                "%s/%s/%s", priv->workdir, GEOREP,
-                                session);
-                if (ret < 0)
-                        goto out;
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/%s/%s", priv->workdir,
+                       GEOREP, session);
+        if (ret < 0)
+            goto out;
 
-                ret = glusterd_copy_folder (src_path, dest_path);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DIR_OP_FAILED, "Could not copy "
-                                "%s to %s", src_path, dest_path);
-                        goto out;
-                }
+        ret = glusterd_copy_folder(src_path, dest_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DIR_OP_FAILED,
+                   "Could not copy "
+                   "%s to %s",
+                   src_path, dest_path);
+            goto out;
         }
+    }
 out:
-        if (origin_volname)
-                GF_ASSERT (origin_volname);
+    if (origin_volname)
+        GF_FREE(origin_volname);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_restore_nfs_ganesha_file (glusterd_volinfo_t *src_vol,
-                                   glusterd_snap_t *snap)
+glusterd_restore_nfs_ganesha_file(glusterd_volinfo_t *src_vol,
+                                  glusterd_snap_t *snap)
 {
-
-        int32_t         ret                     = -1;
-        char            snap_dir[PATH_MAX]      = "";
-        char            src_path[PATH_MAX]      = "";
-        char            dest_path[PATH_MAX]     = "";
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        struct  stat    stbuf                   = {0,};
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("snapshot", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, src_vol, out);
-        GF_VALIDATE_OR_GOTO (this->name, snap, out);
-
-        GLUSTERD_GET_SNAP_DIR (snap_dir, snap, priv);
-
-        ret = snprintf (src_path, sizeof (src_path), "%s/export.%s.conf",
-                       snap_dir, snap->snapname);
-        if (ret < 0)
-                goto out;
-
-        ret = sys_lstat (src_path, &stbuf);
-        if (ret) {
-                if (errno == ENOENT) {
-                        ret = 0;
-                        gf_msg_debug (this->name, 0, "%s not found", src_path);
-                } else
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Stat on %s failed with %s",
-                                src_path, strerror (errno));
-                goto out;
-        }
-
-        ret = snprintf (dest_path, sizeof (dest_path), "%s/export.%s.conf",
-                        GANESHA_EXPORT_DIRECTORY, src_vol->volname);
-        if (ret < 0)
-                goto out;
-
-        ret = glusterd_copy_file (src_path, dest_path);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Failed to copy %s in %s",
-                        src_path, dest_path);
+    int32_t ret = -1;
+    char snap_dir[PATH_MAX] = "";
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("snapshot", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, src_vol, out);
+    GF_VALIDATE_OR_GOTO(this->name, snap, out);
+
+    GLUSTERD_GET_SNAP_DIR(snap_dir, snap, priv);
+
+    ret = snprintf(src_path, sizeof(src_path), "%s/export.%s.conf", snap_dir,
+                   snap->snapname);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        if (errno == ENOENT) {
+            ret = 0;
+            gf_msg_debug(this->name, 0, "%s not found", src_path);
+        } else
+            gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+                   "Stat on %s failed with %s", src_path, strerror(errno));
+        goto out;
+    }
+
+    ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf",
+                   GANESHA_EXPORT_DIRECTORY, src_vol->volname);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_copy_file(src_path, dest_path);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to copy %s in %s", src_path, dest_path);
 
 out:
-        return ret;
-
+    return ret;
 }
+
 /* Snapd functions */
 int
-glusterd_is_snapd_enabled (glusterd_volinfo_t *volinfo)
+glusterd_is_snapd_enabled(glusterd_volinfo_t *volinfo)
 {
-        int              ret    = 0;
-        xlator_t        *this   = THIS;
-
-        ret = dict_get_str_boolean (volinfo->dict, "features.uss", -2);
-        if (ret == -2) {
-                gf_msg_debug (this->name, 0, "Key features.uss not "
-                        "present in the dict for volume %s", volinfo->volname);
-                ret = 0;
+    int ret = 0;
+    xlator_t *this = THIS;
+
+    ret = dict_get_str_boolean(volinfo->dict, "features.uss", -2);
+    if (ret == -2) {
+        gf_msg_debug(this->name, 0,
+                     "Key features.uss not "
+                     "present in the dict for volume %s",
+                     volinfo->volname);
+        ret = 0;
 
-        } else if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get 'features.uss'"
-                        " from dict for volume %s", volinfo->volname);
-        }
+    } else if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get 'features.uss'"
+               " from dict for volume %s",
+               volinfo->volname);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-glusterd_is_snap_soft_limit_reached (glusterd_volinfo_t *volinfo, dict_t *dict)
+glusterd_is_snap_soft_limit_reached(glusterd_volinfo_t *volinfo, dict_t *dict)
 {
-        int32_t         ret            = -1;
-        uint64_t        opt_max_hard   = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t        opt_max_soft   = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-        uint64_t        limit          = 0;
-        int             auto_delete    = 0;
-        uint64_t        effective_max_limit = 0;
-        xlator_t        *this          = NULL;
-        glusterd_conf_t *priv          = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (priv->opts, &opt_max_hard,
-                                            &opt_max_soft);
-
-        /* "auto-delete" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        auto_delete = dict_get_str_boolean (priv->opts,
-                                GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                _gf_false);
-
-        if (volinfo->snap_max_hard_limit < opt_max_hard)
-                effective_max_limit = volinfo->snap_max_hard_limit;
-        else
-                effective_max_limit = opt_max_hard;
-
-        limit = (opt_max_soft * effective_max_limit)/100;
-
-        if (volinfo->snap_count >= limit && auto_delete != _gf_true) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SOFT_LIMIT_REACHED, "Soft-limit "
-                        "(value = %"PRIu64") of volume %s is reached. "
-                        "Snapshot creation is not possible once effective "
-                        "hard-limit (value = %"PRIu64") is reached.",
-                        limit, volinfo->volname, effective_max_limit);
-
-                ret = dict_set_int8 (dict, "soft-limit-reach",
-                                     _gf_true);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to "
-                                "set soft limit exceed flag in "
-                                "response dictionary");
-                }
-                goto out;
+    int32_t ret = -1;
+    uint64_t opt_max_hard = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_max_soft = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    uint64_t limit = 0;
+    int auto_delete = 0;
+    uint64_t effective_max_limit = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(priv->opts, &opt_max_hard,
+                                       &opt_max_soft);
+
+    /* "auto-delete" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    auto_delete = dict_get_str_boolean(
+        priv->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, _gf_false);
+
+    if (volinfo->snap_max_hard_limit < opt_max_hard)
+        effective_max_limit = volinfo->snap_max_hard_limit;
+    else
+        effective_max_limit = opt_max_hard;
+
+    limit = (opt_max_soft * effective_max_limit) / 100;
+
+    if (volinfo->snap_count >= limit && auto_delete != _gf_true) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SOFT_LIMIT_REACHED,
+               "Soft-limit "
+               "(value = %" PRIu64
+               ") of volume %s is reached. "
+               "Snapshot creation is not possible once effective "
+               "hard-limit (value = %" PRIu64 ") is reached.",
+               limit, volinfo->volname, effective_max_limit);
+
+        ret = dict_set_int8(dict, "soft-limit-reach", _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set soft limit exceed flag in "
+                   "response dictionary");
         }
-        ret = 0;
+
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function initializes the parameter sys_hard_limit,
@@ -4053,35 +4219,72 @@ out:
  * return any values.
  */
 void
-gd_get_snap_conf_values_if_present (dict_t *dict, uint64_t *sys_hard_limit,
-                                    uint64_t *sys_soft_limit)
+gd_get_snap_conf_values_if_present(dict_t *dict, uint64_t *sys_hard_limit,
+                                   uint64_t *sys_soft_limit)
 {
-        xlator_t        *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (dict);
-
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        if (dict_get_uint64 (dict, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               sys_hard_limit)) {
-                gf_msg_debug (this->name, 0, "%s is not present in"
-                        "dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    if (dict_get_uint64(dict, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                        sys_hard_limit)) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in"
+                     "dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    /* "snap-max-soft-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    if (dict_get_uint64(dict, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                        sys_soft_limit)) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in"
+                     "dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+    }
+}
 
-        /* "snap-max-soft-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        if (dict_get_uint64 (dict, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                              sys_soft_limit)) {
-                gf_msg_debug (this->name, 0, "%s is not present in"
-                        "dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
-        }
+int
+glusterd_get_snap_status_str(glusterd_snap_t *snapinfo, char *snap_status_str)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, snapinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, snap_status_str, out);
+
+    switch (snapinfo->snap_status) {
+        case GD_SNAP_STATUS_NONE:
+            sprintf(snap_status_str, "%s", "none");
+            break;
+        case GD_SNAP_STATUS_INIT:
+            sprintf(snap_status_str, "%s", "init");
+            break;
+        case GD_SNAP_STATUS_IN_USE:
+            sprintf(snap_status_str, "%s", "in_use");
+            break;
+        case GD_SNAP_STATUS_DECOMMISSION:
+            sprintf(snap_status_str, "%s", "decommissioned");
+            break;
+        case GD_SNAP_STATUS_UNDER_RESTORE:
+            sprintf(snap_status_str, "%s", "under_restore");
+            break;
+        case GD_SNAP_STATUS_RESTORED:
+            sprintf(snap_status_str, "%s", "restored");
+            break;
+        default:
+            goto out;
+    }
+    ret = 0;
+out:
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h
index c0e7e8e218d..5762999bba7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h
@@ -10,157 +10,160 @@
 #ifndef _GLUSTERD_SNAP_UTILS_H
 #define _GLUSTERD_SNAP_UTILS_H
 
+#define GLUSTERD_GET_SNAP_DIR(path, snap, priv)                                \
+    do {                                                                       \
+        int32_t _snap_dir_len;                                                 \
+        _snap_dir_len = snprintf(path, PATH_MAX, "%s/snaps/%s", priv->workdir, \
+                                 snap->snapname);                              \
+        if ((_snap_dir_len < 0) || (_snap_dir_len >= PATH_MAX)) {              \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
 int32_t
-glusterd_snap_volinfo_find (char *volname, glusterd_snap_t *snap,
-                            glusterd_volinfo_t **volinfo);
+glusterd_snap_volinfo_find(char *volname, glusterd_snap_t *snap,
+                           glusterd_volinfo_t **volinfo);
 
 int32_t
-glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
-                                      glusterd_snap_t *snap,
-                                      glusterd_volinfo_t **volinfo);
+glusterd_snap_volinfo_find_from_parent_volname(char *origin_volname,
+                                               glusterd_snap_t *snap,
+                                               glusterd_volinfo_t **volinfo);
 
 int
-glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
-                                         glusterd_volinfo_t **volinfo);
+glusterd_snap_volinfo_find_by_volume_id(uuid_t volume_id,
+                                        glusterd_volinfo_t **volinfo);
 
 int32_t
-glusterd_add_snapd_to_dict (glusterd_volinfo_t *volinfo,
-                            dict_t  *dict, int32_t count);
+glusterd_add_snapd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                           int32_t count);
 
 int
-glusterd_compare_snap_time (struct cds_list_head *, struct cds_list_head *);
+glusterd_compare_snap_time(struct cds_list_head *, struct cds_list_head *);
 
 int
-glusterd_compare_snap_vol_time (struct cds_list_head *, struct cds_list_head *);
+glusterd_compare_snap_vol_time(struct cds_list_head *, struct cds_list_head *);
 
 int32_t
-glusterd_snap_volinfo_restore (dict_t *dict, dict_t *rsp_dict,
-                               glusterd_volinfo_t *new_volinfo,
-                               glusterd_volinfo_t *snap_volinfo,
-                               int32_t volcount);
+glusterd_snap_volinfo_restore(dict_t *dict, dict_t *rsp_dict,
+                              glusterd_volinfo_t *new_volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              int32_t volcount);
 int32_t
-glusterd_snapobject_delete (glusterd_snap_t *snap);
+glusterd_snapobject_delete(glusterd_snap_t *snap);
 
 int32_t
-glusterd_cleanup_snaps_for_volume (glusterd_volinfo_t *volinfo);
+glusterd_cleanup_snaps_for_volume(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo);
+glusterd_missed_snapinfo_new(glusterd_missed_snap_info **missed_snapinfo);
 
 int32_t
-glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op);
+glusterd_missed_snap_op_new(glusterd_snap_op_t **snap_op);
 
 int32_t
-glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict,
-                                   glusterd_volinfo_t *snap_vol,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   int32_t brick_number, int32_t op);
+glusterd_add_missed_snaps_to_dict(dict_t *rsp_dict,
+                                  glusterd_volinfo_t *snap_vol,
+                                  glusterd_brickinfo_t *brickinfo,
+                                  int32_t brick_number, int32_t op);
 
 int32_t
-glusterd_add_missed_snaps_to_export_dict (dict_t *peer_data);
+glusterd_add_missed_snaps_to_export_dict(dict_t *peer_data);
 
 int32_t
-glusterd_import_friend_missed_snap_list (dict_t *peer_data);
+glusterd_import_friend_missed_snap_list(dict_t *peer_data);
 
 int
-gd_restore_snap_volume (dict_t *dict, dict_t *rsp_dict,
-                        glusterd_volinfo_t *orig_vol,
-                        glusterd_volinfo_t *snap_vol,
-                        int32_t volcount);
+gd_restore_snap_volume(dict_t *dict, dict_t *rsp_dict,
+                       glusterd_volinfo_t *orig_vol,
+                       glusterd_volinfo_t *snap_vol, int32_t volcount);
 
 int32_t
-glusterd_mount_lvm_snapshot (glusterd_brickinfo_t *brickinfo,
-                             char *brick_mount_path);
+glusterd_mount_lvm_snapshot(glusterd_brickinfo_t *brickinfo,
+                            char *brick_mount_path);
 
 int32_t
-glusterd_umount (const char *path);
+glusterd_umount(const char *path);
 
 int32_t
-glusterd_add_snapshots_to_export_dict (dict_t *peer_data);
+glusterd_snap_unmount(xlator_t *this, glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_compare_friend_snapshots (dict_t *peer_data, char *peername,
-                                   uuid_t peerid);
+glusterd_add_snapshots_to_export_dict(dict_t *peer_data);
 
 int32_t
-glusterd_store_create_snap_dir (glusterd_snap_t *snap);
+glusterd_compare_friend_snapshots(dict_t *peer_data, char *peername,
+                                  uuid_t peerid);
 
 int32_t
-glusterd_copy_file (const char *source, const char *destination);
+glusterd_store_create_snap_dir(glusterd_snap_t *snap);
 
 int32_t
-glusterd_copy_folder (const char *source, const char *destination);
+glusterd_copy_file(const char *source, const char *destination);
 
 int32_t
-glusterd_get_geo_rep_session (char *slave_key, char *origin_volname,
-                              dict_t *gsync_slaves_dict, char *session,
-                              char *slave);
+glusterd_copy_folder(const char *source, const char *destination);
 
 int32_t
-glusterd_restore_geo_rep_files (glusterd_volinfo_t *snap_vol);
+glusterd_get_geo_rep_session(char *slave_key, char *origin_volname,
+                             dict_t *gsync_slaves_dict, char *session,
+                             char *slave);
 
-int
-glusterd_restore_nfs_ganesha_file (glusterd_volinfo_t *src_vol,
-                                   glusterd_snap_t *snap);
 int32_t
-glusterd_copy_quota_files (glusterd_volinfo_t *src_vol,
-                           glusterd_volinfo_t *dest_vol,
-                           gf_boolean_t *conf_present);
+glusterd_restore_geo_rep_files(glusterd_volinfo_t *snap_vol);
 
-int
-glusterd_copy_nfs_ganesha_file (glusterd_volinfo_t *src_vol,
-                                glusterd_volinfo_t *dest_vol);
+int32_t
+glusterd_copy_quota_files(glusterd_volinfo_t *src_vol,
+                          glusterd_volinfo_t *dest_vol,
+                          gf_boolean_t *conf_present);
 
 int
-glusterd_snap_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_snap_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 
 int
-gd_add_vol_snap_details_to_dict (dict_t *dict, char *prefix,
-                                 glusterd_volinfo_t *volinfo);
+gd_add_vol_snap_details_to_dict(dict_t *dict, char *prefix,
+                                glusterd_volinfo_t *volinfo);
 
 int
-gd_add_brick_snap_details_to_dict (dict_t *dict, char *prefix,
-                                   glusterd_brickinfo_t *brickinfo);
+gd_add_brick_snap_details_to_dict(dict_t *dict, char *prefix,
+                                  glusterd_brickinfo_t *brickinfo);
 
 int
-gd_import_new_brick_snap_details (dict_t *dict, char *prefix,
-                                  glusterd_brickinfo_t *brickinfo);
+gd_import_new_brick_snap_details(dict_t *dict, char *prefix,
+                                 glusterd_brickinfo_t *brickinfo);
 
 int
-gd_import_volume_snap_details (dict_t *dict, glusterd_volinfo_t *volinfo,
-                               char *prefix, char *volname);
+gd_import_volume_snap_details(dict_t *dict, glusterd_volinfo_t *volinfo,
+                              char *prefix, char *volname);
 
 int32_t
-glusterd_snap_quorum_check (dict_t *dict, gf_boolean_t snap_volume,
-                            char **op_errstr, uint32_t *op_errno);
+glusterd_snap_quorum_check(dict_t *dict, gf_boolean_t snap_volume,
+                           char **op_errstr, uint32_t *op_errno);
 
 int32_t
-glusterd_snap_brick_create (glusterd_volinfo_t *snap_volinfo,
-                            glusterd_brickinfo_t *brickinfo,
-                            int32_t brick_count);
+glusterd_snap_brick_create(glusterd_volinfo_t *snap_volinfo,
+                           glusterd_brickinfo_t *brickinfo, int32_t brick_count,
+                           int32_t clone);
 
 int
-glusterd_snapshot_restore_cleanup (dict_t *rsp_dict,
-                                   char *volname,
-                                   glusterd_snap_t *snap);
+glusterd_snapshot_restore_cleanup(dict_t *rsp_dict, char *volname,
+                                  glusterd_snap_t *snap);
 
 void
-glusterd_get_snapd_dir (glusterd_volinfo_t *volinfo,
-                        char *path, int path_len);
+glusterd_get_snapd_dir(glusterd_volinfo_t *volinfo, char *path, int path_len);
 
 int
-glusterd_is_snapd_enabled (glusterd_volinfo_t *volinfo);
+glusterd_is_snapd_enabled(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_check_and_set_config_limit (glusterd_conf_t *priv);
+glusterd_check_and_set_config_limit(glusterd_conf_t *priv);
 
 int32_t
-glusterd_is_snap_soft_limit_reached (glusterd_volinfo_t *volinfo,
-                                     dict_t *dict);
+glusterd_is_snap_soft_limit_reached(glusterd_volinfo_t *volinfo, dict_t *dict);
 
 void
-gd_get_snap_conf_values_if_present (dict_t *opts, uint64_t *sys_hard_limit,
-                                    uint64_t *sys_soft_limit);
+gd_get_snap_conf_values_if_present(dict_t *opts, uint64_t *sys_hard_limit,
+                                   uint64_t *sys_soft_limit);
+int
+glusterd_get_snap_status_str(glusterd_snap_t *snapinfo, char *snap_status_str);
 
 #endif
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index 7967722d798..aeaa8d15214 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -35,19 +35,18 @@
 
 #include <regex.h>
 
-#include "globals.h"
-#include "compat.h"
+#include <glusterfs/compat.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/timer.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
-#include "run.h"
+#include <glusterfs/run.h>
 #include "glusterd-volgen.h"
 #include "glusterd-mgmt.h"
 #include "glusterd-syncop.h"
@@ -56,31 +55,47 @@
 
 #include "glusterfs3.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 
-#include "lvm-defaults.h"
-
-char snap_mount_dir[PATH_MAX];
+#include <glusterfs/lvm-defaults.h>
+#include <glusterfs/events.h>
+
+#define GLUSTERD_GET_UUID_NOHYPHEN(ret_string, uuid)                           \
+    do {                                                                       \
+        char *snap_volname_ptr = ret_string;                                   \
+        char tmp_uuid[64];                                                     \
+        char *snap_volid_ptr = uuid_utoa_r(uuid, tmp_uuid);                    \
+        while (*snap_volid_ptr) {                                              \
+            if (*snap_volid_ptr == '-') {                                      \
+                snap_volid_ptr++;                                              \
+            } else {                                                           \
+                (*snap_volname_ptr++) = (*snap_volid_ptr++);                   \
+            }                                                                  \
+        }                                                                      \
+        *snap_volname_ptr = '\0';                                              \
+    } while (0)
+
+char snap_mount_dir[VALID_GLUSTERD_PATHMAX];
 struct snap_create_args_ {
-        xlator_t             *this;
-        dict_t               *dict;
-        dict_t               *rsp_dict;
-        glusterd_volinfo_t   *snap_vol;
-        glusterd_brickinfo_t *brickinfo;
-        struct syncargs      *args;
-        int32_t               volcount;
-        int32_t               brickcount;
-        int32_t               brickorder;
+    xlator_t *this;
+    dict_t *dict;
+    dict_t *rsp_dict;
+    glusterd_volinfo_t *snap_vol;
+    glusterd_brickinfo_t *brickinfo;
+    struct syncargs *args;
+    int32_t volcount;
+    int32_t brickcount;
+    int32_t brickorder;
 };
 
-/* This structure is used to store unsupported options and thier values
+/* This structure is used to store unsupported options and their values
  * for snapshotted volume.
  */
 struct gd_snap_unsupported_opt_t {
-        char                  *key;
-        char                  *value;
+    char *key;
+    char *value;
 };
 
 typedef struct snap_create_args_ snap_create_args_t;
@@ -92,499 +107,501 @@ typedef struct snap_create_args_ snap_create_args_t;
 */
 
 char *
-glusterd_build_snap_device_path (char *device, char *snapname,
-                                 int32_t brickcount)
+glusterd_build_snap_device_path(char *device, char *snapname,
+                                int32_t brickcount)
 {
-        char        snap[PATH_MAX]      = "";
-        char        msg[1024]           = "";
-        char        volgroup[PATH_MAX]  = "";
-        char       *snap_device         = NULL;
-        xlator_t   *this                = NULL;
-        runner_t    runner              = {0,};
-        char       *ptr                 = NULL;
-        int         ret                 = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        if (!device) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY,
-                        "device is NULL");
-                goto out;
-        }
-        if (!snapname) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY,
-                        "snapname is NULL");
-                goto out;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, "/sbin/lvs", "--noheadings", "-o", "vg_name",
-                         device, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        snprintf (msg, sizeof (msg), "Get volume group for device %s", device);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_VG_GET_FAIL, "Failed to get volume group "
-                        "for device %s", device);
-                runner_end (&runner);
-                goto out;
-        }
-        ptr = fgets(volgroup, sizeof(volgroup),
-                    runner_chio (&runner, STDOUT_FILENO));
-        if (!ptr || !strlen(volgroup)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_VG_GET_FAIL, "Failed to get volume group "
-                        "for snap %s", snapname);
-                runner_end (&runner);
-                ret = -1;
-                goto out;
-        }
-        runner_end (&runner);
-
-        snprintf (snap, sizeof(snap), "/dev/%s/%s_%d", gf_trim(volgroup),
-                  snapname, brickcount);
-        snap_device = gf_strdup (snap);
-        if (!snap_device) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        GD_MSG_NO_MEMORY,
-                        "Cannot copy the snapshot device name for snapname: %s",
-                        snapname);
-        }
+    char snap[PATH_MAX] = "";
+    char msg[1024] = "";
+    char volgroup[PATH_MAX] = "";
+    char *snap_device = NULL;
+    xlator_t *this = NULL;
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    if (!device) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "device is NULL");
+        goto out;
+    }
+    if (!snapname) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "snapname is NULL");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, "lvs", "--noheadings", "-o", "vg_name", device,
+                    NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    snprintf(msg, sizeof(msg), "Get volume group for device %s", device);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_VG_GET_FAIL,
+               "Failed to get volume group "
+               "for device %s",
+               device);
+        runner_end(&runner);
+        goto out;
+    }
+    ptr = fgets(volgroup, sizeof(volgroup),
+                runner_chio(&runner, STDOUT_FILENO));
+    if (!ptr || !strlen(volgroup)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_VG_GET_FAIL,
+               "Failed to get volume group "
+               "for snap %s",
+               snapname);
+        runner_end(&runner);
+        ret = -1;
+        goto out;
+    }
+    runner_end(&runner);
+
+    snprintf(snap, sizeof(snap), "/dev/%s/%s_%d", gf_trim(volgroup), snapname,
+             brickcount);
+    snap_device = gf_strdup(snap);
+    if (!snap_device) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_NO_MEMORY,
+               "Cannot copy the snapshot device name for snapname: %s",
+               snapname);
+    }
 
 out:
-        return snap_device;
+    return snap_device;
 }
 
 /* Look for disconnected peers, for missed snap creates or deletes */
 static int32_t
-glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
-                           struct cds_list_head *peers, int32_t op)
+glusterd_find_missed_snap(dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                          struct cds_list_head *peers, int32_t op)
 {
-        int32_t                   brick_count          = -1;
-        int32_t                   ret                  = -1;
-        xlator_t                 *this                 = NULL;
-        glusterd_peerinfo_t      *peerinfo             = NULL;
-        glusterd_brickinfo_t     *brickinfo            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (peers);
-        GF_ASSERT (vol);
-
-        brick_count = 0;
-        cds_list_for_each_entry (brickinfo, &vol->bricks, brick_list) {
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                        /* If the brick belongs to the same node */
-                        brick_count++;
-                        continue;
-                }
-
-                rcu_read_lock ();
-                cds_list_for_each_entry_rcu (peerinfo, peers, uuid_list) {
-                        if (gf_uuid_compare (peerinfo->uuid, brickinfo->uuid)) {
-                                /* If the brick doesnt belong to this peer */
-                                continue;
-                        }
-
-                        /* Found peer who owns the brick,    *
-                         * if peer is not connected or not   *
-                         * friend add it to missed snap list */
-                        if (!(peerinfo->connected) ||
-                           (peerinfo->state.state !=
-                             GD_FRIEND_STATE_BEFRIENDED)) {
-                                ret = glusterd_add_missed_snaps_to_dict
-                                                   (rsp_dict,
-                                                    vol, brickinfo,
-                                                    brick_count + 1,
-                                                    op);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                                                "Failed to add missed snapshot "
-                                                "info for %s:%s in the "
-                                                "rsp_dict", brickinfo->hostname,
-                                                brickinfo->path);
-                                        rcu_read_unlock ();
-                                        goto out;
-                                }
-                        }
-                }
-                rcu_read_unlock ();
-                brick_count++;
-        }
-
-        ret = 0;
+    int32_t brick_count = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(peers);
+    GF_ASSERT(vol);
+
+    brick_count = 0;
+    cds_list_for_each_entry(brickinfo, &vol->bricks, brick_list)
+    {
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            /* If the brick belongs to the same node */
+            brick_count++;
+            continue;
+        }
+
+        RCU_READ_LOCK;
+        cds_list_for_each_entry_rcu(peerinfo, peers, uuid_list)
+        {
+            if (gf_uuid_compare(peerinfo->uuid, brickinfo->uuid)) {
+                /* If the brick doesn't belong to this peer */
+                continue;
+            }
+
+            /* Found peer who owns the brick,    *
+             * if peer is not connected or not   *
+             * friend add it to missed snap list */
+            if (!(peerinfo->connected) ||
+                (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)) {
+                ret = glusterd_add_missed_snaps_to_dict(
+                    rsp_dict, vol, brickinfo, brick_count + 1, op);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                           "Failed to add missed snapshot "
+                           "info for %s:%s in the "
+                           "rsp_dict",
+                           brickinfo->hostname, brickinfo->path);
+                    goto out;
+                }
+            }
+        }
+        RCU_READ_UNLOCK;
+        brick_count++;
+    }
+
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-snap_max_limits_display_commit (dict_t *rsp_dict, char *volname,
-                                char *op_errstr, int len)
+snap_max_limits_display_commit(dict_t *rsp_dict, char *volname, char *op_errstr,
+                               int len)
 {
-        char          err_str[PATH_MAX] = "";
-        char          buf[PATH_MAX]     = "";
-        glusterd_conf_t    *conf        = NULL;
-        glusterd_volinfo_t *volinfo     = NULL;
-        int           ret               = -1;
-        uint64_t      active_hard_limit = 0;
-        uint64_t      snap_max_limit    = 0;
-        uint64_t      soft_limit_value  = -1;
-        uint64_t      count             = 0;
-        xlator_t     *this              = NULL;
-        uint64_t      opt_hard_max      = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t      opt_soft_max      = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-        char         *auto_delete       = "disable";
-        char         *snap_activate     = "disable";
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (conf->opts, &opt_hard_max,
-                                            &opt_soft_max);
-
-        if (!volname) {
-                /* For system limit */
-                cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                        if (volinfo->is_snap_volume == _gf_true)
-                                continue;
-
-                        snap_max_limit = volinfo->snap_max_hard_limit;
-                        if (snap_max_limit > opt_hard_max)
-                                active_hard_limit = opt_hard_max;
-                        else
-                                active_hard_limit = snap_max_limit;
-
-                        soft_limit_value = (opt_soft_max *
-                                            active_hard_limit) / 100;
-
-                        snprintf (buf, sizeof(buf), "volume%"PRId64"-volname",
-                                  count);
-                        ret = dict_set_str (rsp_dict, buf, volinfo->volname);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRId64"-snap-max-hard-limit", count);
-                        ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRId64"-active-hard-limit", count);
-                        ret = dict_set_uint64 (rsp_dict, buf,
-                                               active_hard_limit);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRId64"-snap-max-soft-limit", count);
-                        ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-                        count++;
-                }
-
-                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set voldisplaycount");
-                        goto out;
-                }
-        } else {
-                /*  For one volume */
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX, "Volume (%s) does not "
-                                  "exist", volname);
-                        goto out;
-                }
+    char err_str[PATH_MAX] = "";
+    char key[64] = "";
+    int keylen;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    uint64_t active_hard_limit = 0;
+    uint64_t snap_max_limit = 0;
+    uint64_t soft_limit_value = -1;
+    uint64_t count = 0;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_soft_max = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    char *auto_delete = "disable";
+    char *snap_activate = "disable";
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(conf->opts, &opt_hard_max,
+                                       &opt_soft_max);
+
+    if (!volname) {
+        /* For system limit */
+        cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+        {
+            if (volinfo->is_snap_volume == _gf_true)
+                continue;
 
-                snap_max_limit = volinfo->snap_max_hard_limit;
-                if (snap_max_limit > opt_hard_max)
-                        active_hard_limit = opt_hard_max;
-                else
-                        active_hard_limit = snap_max_limit;
+            snap_max_limit = volinfo->snap_max_hard_limit;
+            if (snap_max_limit > opt_hard_max)
+                active_hard_limit = opt_hard_max;
+            else
+                active_hard_limit = snap_max_limit;
 
-                soft_limit_value = (opt_soft_max *
-                                    active_hard_limit) / 100;
+            soft_limit_value = (opt_soft_max * active_hard_limit) / 100;
 
-                snprintf (buf, sizeof(buf), "volume%"PRId64"-volname", count);
-                ret = dict_set_str (rsp_dict, buf, volinfo->volname);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
+            keylen = snprintf(key, sizeof(key), "volume%" PRId64 "-volname",
+                              count);
+            ret = dict_set_strn(rsp_dict, key, keylen, volinfo->volname);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
+                goto out;
+            }
 
-                snprintf (buf, sizeof(buf),
-                          "volume%"PRId64"-snap-max-hard-limit", count);
-                ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
+            snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-hard-limit",
+                     count);
+            ret = dict_set_uint64(rsp_dict, key, snap_max_limit);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
+                goto out;
+            }
 
-                snprintf (buf, sizeof(buf),
-                          "volume%"PRId64"-active-hard-limit", count);
-                ret = dict_set_uint64 (rsp_dict, buf, active_hard_limit);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
+            snprintf(key, sizeof(key), "volume%" PRId64 "-active-hard-limit",
+                     count);
+            ret = dict_set_uint64(rsp_dict, key, active_hard_limit);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
+                goto out;
+            }
 
-                snprintf (buf, sizeof(buf),
-                          "volume%"PRId64"-snap-max-soft-limit", count);
-                ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
+            snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-soft-limit",
+                     count);
+            ret = dict_set_uint64(rsp_dict, key, soft_limit_value);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
-
-                count++;
-
-                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set voldisplaycount");
-                        goto out;
-                }
-
-        }
-
-        ret = dict_set_uint64 (rsp_dict,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               opt_hard_max);
-        if (ret) {
-                snprintf (err_str, PATH_MAX,
-                          "Failed to set %s in response dictionary",
-                          GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
                 goto out;
+            }
+            count++;
         }
 
-        ret = dict_set_uint64 (rsp_dict,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                               opt_soft_max);
+        ret = dict_set_uint64(rsp_dict, "voldisplaycount", count);
         if (ret) {
-                snprintf (err_str, PATH_MAX,
-                         "Failed to set %s in response dictionary",
-                         GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
-                goto out;
+            snprintf(err_str, PATH_MAX, "Failed to set voldisplaycount");
+            goto out;
         }
-
-        /* "auto-delete" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_str (conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                            &auto_delete);
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict,
-                                     GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                     auto_delete);
+    } else {
+        /*  For one volume */
+        ret = glusterd_volinfo_find(volname, &volinfo);
         if (ret) {
-                snprintf (err_str, PATH_MAX,
-                          "Failed to set %s in response dictionary",
-                          GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE);
-                goto out;
+            snprintf(err_str, PATH_MAX,
+                     "Volume (%s) does not "
+                     "exist",
+                     volname);
+            goto out;
         }
 
-        /* "snap-activate-on-create" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_str (conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                            &snap_activate);
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict,
-                                     GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+        snap_max_limit = volinfo->snap_max_hard_limit;
+        if (snap_max_limit > opt_hard_max)
+            active_hard_limit = opt_hard_max;
+        else
+            active_hard_limit = snap_max_limit;
+
+        soft_limit_value = (opt_soft_max * active_hard_limit) / 100;
+
+        keylen = snprintf(key, sizeof(key), "volume%" PRId64 "-volname", count);
+        ret = dict_set_strn(rsp_dict, key, keylen, volinfo->volname);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-hard-limit",
+                 count);
+        ret = dict_set_uint64(rsp_dict, key, snap_max_limit);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        snprintf(key, sizeof(key), "volume%" PRId64 "-active-hard-limit",
+                 count);
+        ret = dict_set_uint64(rsp_dict, key, active_hard_limit);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-soft-limit",
+                 count);
+        ret = dict_set_uint64(rsp_dict, key, soft_limit_value);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        count++;
+
+        ret = dict_set_uint64(rsp_dict, "voldisplaycount", count);
+        if (ret) {
+            snprintf(err_str, PATH_MAX, "Failed to set voldisplaycount");
+            goto out;
+        }
+    }
+
+    ret = dict_set_uint64(rsp_dict, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          opt_hard_max);
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+        goto out;
+    }
+
+    ret = dict_set_uint64(rsp_dict, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                          opt_soft_max);
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+        goto out;
+    }
+
+    /* "auto-delete" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_strn(conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                        SLEN(GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE),
+                        &auto_delete);
+
+    ret = dict_set_dynstr_with_alloc(
+        rsp_dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, auto_delete);
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE);
+        goto out;
+    }
+
+    /* "snap-activate-on-create" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_strn(conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+                        SLEN(GLUSTERD_STORE_KEY_SNAP_ACTIVATE), &snap_activate);
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
                                      snap_activate);
-        if (ret) {
-                snprintf (err_str, PATH_MAX,
-                          "Failed to set %s in response dictionary",
-                          GLUSTERD_STORE_KEY_SNAP_ACTIVATE);
-                goto out;
-        }
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_ACTIVATE);
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret) {
-                strncpy (op_errstr, err_str, len);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "%s", err_str);
-        }
-        return ret;
+    if (ret) {
+        strncpy(op_errstr, err_str, len);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+               err_str);
+    }
+    return ret;
 }
 
-
 /* Third argument of scandir(used in glusterd_copy_geo_rep_session_files)
- * is filter function. As we dont want "." and ".." files present in the
+ * is filter function. As we don't want "." and ".." files present in the
  * directory, we are excliding these 2 files.
  * "file_select" function here does the job of filtering.
  */
 int
-file_select (const struct dirent *entry)
+file_select(const struct dirent *entry)
 {
-        if (entry == NULL)
-                return (FALSE);
+    if (entry == NULL)
+        return (FALSE);
 
-        if ((strcmp(entry->d_name, ".") == 0) ||
-            (strcmp(entry->d_name, "..") == 0))
-                return (FALSE);
-        else
-                return (TRUE);
+    if ((strcmp(entry->d_name, ".") == 0) || (strcmp(entry->d_name, "..") == 0))
+        return (FALSE);
+    else
+        return (TRUE);
 }
 
 int32_t
-glusterd_copy_geo_rep_session_files (char *session,
-                                     glusterd_volinfo_t *snap_vol)
+glusterd_copy_geo_rep_session_files(char *session, glusterd_volinfo_t *snap_vol)
 {
-        int32_t         ret                             = -1;
-        char            snap_session_dir[PATH_MAX]      = "";
-        char            georep_session_dir[PATH_MAX]    = "";
-        regex_t         *reg_exp                        = NULL;
-        int             file_count                      = -1;
-        struct  dirent  **files                         = {0,};
-        xlator_t        *this                           = NULL;
-        int             i                               = 0;
-        char            src_path[PATH_MAX]              = "";
-        char            dest_path[PATH_MAX]             = "";
-        glusterd_conf_t *priv                           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (session);
-        GF_ASSERT (snap_vol);
-
-        ret = snprintf (georep_session_dir, sizeof (georep_session_dir),
-                        "%s/%s/%s", priv->workdir, GEOREP,
-                        session);
-        if (ret < 0) { /* Negative value is an error */
-                goto out;
-        }
-
-        ret = snprintf (snap_session_dir, sizeof (snap_session_dir),
-                        "%s/%s/%s/%s/%s", priv->workdir,
-                        GLUSTERD_VOL_SNAP_DIR_PREFIX,
-                        snap_vol->snapshot->snapname, GEOREP, session);
-        if (ret < 0) { /* Negative value is an error */
-                goto out;
-        }
-
-        ret = mkdir_p (snap_session_dir, 0777, _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "Creating directory %s failed", snap_session_dir);
-                goto out;
+    int32_t ret = -1;
+    char snap_session_dir[PATH_MAX] = "";
+    char georep_session_dir[PATH_MAX] = "";
+    regex_t *reg_exp = NULL;
+    int file_count = -1;
+    struct dirent **files = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int i = 0;
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(session);
+    GF_ASSERT(snap_vol);
+
+    ret = snprintf(georep_session_dir, sizeof(georep_session_dir), "%s/%s/%s",
+                   priv->workdir, GEOREP, session);
+    if (ret < 0) { /* Negative value is an error */
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = snprintf(snap_session_dir, sizeof(snap_session_dir), "%s/%s/%s/%s/%s",
+                   priv->workdir, GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                   snap_vol->snapshot->snapname, GEOREP, session);
+    if (ret < 0) { /* Negative value is an error */
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = mkdir_p(snap_session_dir, 0755, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Creating directory %s failed", snap_session_dir);
+        goto out;
+    }
+
+    /* TODO : good to have - Allocate in stack instead of heap */
+    reg_exp = GF_CALLOC(1, sizeof(regex_t), gf_common_mt_regex_t);
+    if (!reg_exp) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to allocate memory for regular expression");
+        goto out;
+    }
+
+    ret = regcomp(reg_exp, "(.*status$)|(.*conf$)\0", REG_EXTENDED);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REG_COMPILE_FAILED,
+               "Failed to compile the regular expression");
+        goto out;
+    }
+
+    /* If there are no files in a particular session then fail it*/
+    file_count = scandir(georep_session_dir, &files, file_select, alphasort);
+    if (file_count <= 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Session files not present "
+               "in %s",
+               georep_session_dir);
+        goto out;
+    }
+
+    /* Now compare the file name with regular expression to see if
+     * there is a match
+     */
+    for (i = 0; i < file_count; i++) {
+        if (regexec(reg_exp, files[i]->d_name, 0, NULL, 0))
+            continue;
+
+        ret = snprintf(src_path, sizeof(src_path), "%s/%s", georep_session_dir,
+                       files[i]->d_name);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        /* TODO : good to have - Allocate in stack instead of heap */
-        reg_exp = GF_CALLOC (1, sizeof (regex_t), gf_common_mt_regex_t);
-        if (!reg_exp) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                       GD_MSG_NO_MEMORY,
-                       "Failed to allocate memory for regular expression");
-                goto out;
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/%s", snap_session_dir,
+                       files[i]->d_name);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = regcomp (reg_exp, "(.*status$)|(.*conf$)\0", REG_EXTENDED);
+        ret = glusterd_copy_file(src_path, dest_path);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_REG_COMPILE_FAILED,
-                        "Failed to compile the regular expression");
-                goto out;
-        }
-
-        /* If there are no files in a particular session then fail it*/
-        file_count = scandir (georep_session_dir, &files, file_select,
-                              alphasort);
-        if (file_count <= 0) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOENT,
-                        GD_MSG_FILE_OP_FAILED, "Session files not present "
-                        "in %s", georep_session_dir);
-                goto out;
-        }
-
-        /* Now compare the file name with regular expression to see if
-         * there is a match
-         */
-        for (i = 0 ; i < file_count; i++) {
-                if (regexec (reg_exp, files[i]->d_name, 0, NULL, 0))
-                        continue;
-
-                ret = snprintf (src_path, sizeof (src_path), "%s/%s",
-                                georep_session_dir, files[i]->d_name);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = snprintf (dest_path , sizeof (dest_path), "%s/%s",
-                                snap_session_dir, files[i]->d_name);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = glusterd_copy_file (src_path, dest_path);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY,
-                                "Could not copy file %s of session %s",
-                                files[i]->d_name, session);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Could not copy file %s of session %s", files[i]->d_name,
+                   session);
+            goto out;
         }
+    }
 out:
-        /* files are malloc'd by scandir, free them */
-        if (file_count > 0) {
-                while (file_count--) {
-                        free(files[file_count]);
-                }
-                free(files);
+    /* files are malloc'd by scandir, free them */
+    if (file_count > 0) {
+        while (file_count--) {
+            free(files[file_count]);
         }
+        free(files);
+    }
 
-        if (reg_exp)
-                GF_FREE (reg_exp);
+    if (reg_exp)
+        GF_FREE(reg_exp);
 
-        return ret;
+    return ret;
 }
 
 /* This function will take backup of the volume store
@@ -596,165 +613,168 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_backup_vol (glusterd_volinfo_t *volinfo)
+glusterd_snapshot_backup_vol(glusterd_volinfo_t *volinfo)
 {
-        char             pathname[PATH_MAX]    = {0,};
-        int              ret                   = -1;
-        int              op_ret                = 0;
-        char             delete_path[PATH_MAX] = {0,};
-        char             trashdir[PATH_MAX]    = {0,};
-        glusterd_conf_t *priv                  = NULL;
-        xlator_t        *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
-
-        snprintf (delete_path, sizeof (delete_path),
-                  "%s/"GLUSTERD_TRASH"/vols-%s.deleted", priv->workdir,
-                  volinfo->volname);
-
-        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
-                  priv->workdir);
-
-        /* Create trash folder if it is not there */
-        ret = sys_mkdir (trashdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "Failed to create trash directory, reason : %s",
-                        strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        /* Move the origin volume volder to the backup location */
-        ret = sys_rename (pathname, delete_path);
+    char pathname[PATH_MAX] = "";
+    int ret = -1;
+    int op_ret = 0;
+    char delete_path[PATH_MAX] = "";
+    char trashdir[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(pathname, volinfo, priv);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir,
+                   volinfo->volname);
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
+                   priv->workdir);
+    if ((len < 0) || (len >= sizeof(trashdir))) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    /* Create trash folder if it is not there */
+    ret = sys_mkdir(trashdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to create trash directory, reason : %s",
+               strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    /* Move the origin volume volder to the backup location */
+    ret = sys_rename(pathname, delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Failed to rename snap "
+               "directory %s to %s",
+               pathname, delete_path);
+        goto out;
+    }
+
+    /* Re-create an empty origin volume folder so that restore can
+     * happen. */
+    ret = sys_mkdir(pathname, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to create origin "
+               "volume directory (%s), reason : %s",
+               pathname, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    /* Save the actual return value */
+    op_ret = ret;
+    if (ret) {
+        /* Revert the changes in case of failure */
+        ret = sys_rmdir(pathname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Failed to rename snap "
-                        "directory %s to %s", pathname, delete_path);
-                goto out;
+            gf_msg_debug(this->name, 0, "Failed to rmdir: %s,err: %s", pathname,
+                         strerror(errno));
         }
 
-        /* Re-create an empty origin volume folder so that restore can
-         * happen. */
-        ret = sys_mkdir (pathname, 0777);
-        if (ret && errno != EEXIST) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "Failed to create origin "
-                        "volume directory (%s), reason : %s",
-                        pathname, strerror (errno));
-                ret = -1;
-                goto out;
+        ret = sys_rename(delete_path, pathname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Failed to rename directory %s to %s", delete_path,
+                   pathname);
         }
 
-        ret = 0;
-out:
-        /* Save the actual return value */
-        op_ret = ret;
+        ret = sys_rmdir(trashdir);
         if (ret) {
-                /* Revert the changes in case of failure */
-                ret = sys_rmdir (pathname);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Failed to rmdir: %s,err: %s",
-                                pathname, strerror (errno));
-                }
-
-                ret = sys_rename (delete_path, pathname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Failed to rename directory %s to %s",
-                                delete_path, pathname);
-                }
-
-                ret = sys_rmdir (trashdir);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Failed to rmdir: %s, Reason: %s",
-                                trashdir, strerror (errno));
-                }
+            gf_msg_debug(this->name, 0, "Failed to rmdir: %s, Reason: %s",
+                         trashdir, strerror(errno));
         }
+    }
 
-        gf_msg_trace (this->name, 0, "Returning %d", op_ret);
+    gf_msg_trace(this->name, 0, "Returning %d", op_ret);
 
-        return op_ret;
+    return op_ret;
 }
 
-int32_t
-glusterd_copy_geo_rep_files (glusterd_volinfo_t *origin_vol,
-                             glusterd_volinfo_t *snap_vol, dict_t *rsp_dict)
+static int32_t
+glusterd_copy_geo_rep_files(glusterd_volinfo_t *origin_vol,
+                            glusterd_volinfo_t *snap_vol, dict_t *rsp_dict)
 {
-        int32_t         ret                     =       -1;
-        int             i                       =       0;
-        xlator_t        *this                   =       NULL;
-        char            key[PATH_MAX]           =       "";
-        char            session[PATH_MAX]       =       "";
-        char            slave[PATH_MAX]         =       "";
-        char            snapgeo_dir[PATH_MAX]   =       "";
-        glusterd_conf_t *priv                   =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (origin_vol);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (rsp_dict);
-
-        /* This condition is not satisfied if the volume
-         * is slave volume.
-         */
-        if (!origin_vol->gsync_slaves) {
-                ret = 0;
-                goto out;
-        }
+    int32_t ret = -1;
+    int i = 0;
+    xlator_t *this = NULL;
+    char key[32] = "";
+    char session[PATH_MAX] = "";
+    char slave[PATH_MAX] = "";
+    char snapgeo_dir[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(origin_vol);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(rsp_dict);
+
+    /* This condition is not satisfied if the volume
+     * is slave volume.
+     */
+    if (!origin_vol->gsync_slaves) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_SLAVE, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot, priv);
+    GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot, priv);
 
-        ret = sys_mkdir (snapgeo_dir, 0777);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "Creating directory %s failed", snapgeo_dir);
-                goto out;
-        }
+    ret = sys_mkdir(snapgeo_dir, 0755);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Creating directory %s failed", snapgeo_dir);
+        goto out;
+    }
 
-        for (i = 1 ; i <= origin_vol->gsync_slaves->count ; i++) {
-                ret = snprintf (key, sizeof (key), "slave%d", i);
-                if (ret < 0) /* Negative value is an error */
-                        goto out;
+    for (i = 1; i <= origin_vol->gsync_slaves->count; i++) {
+        ret = snprintf(key, sizeof(key), "slave%d", i);
+        if (ret < 0) /* Negative value is an error */
+            goto out;
 
-                ret = glusterd_get_geo_rep_session (key, origin_vol->volname,
-                                                    origin_vol->gsync_slaves,
-                                                    session, slave);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GEOREP_GET_FAILED,
-                                "Failed to get geo-rep session");
-                        goto out;
-                }
+        ret = glusterd_get_geo_rep_session(
+            key, origin_vol->volname, origin_vol->gsync_slaves, session, slave);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GEOREP_GET_FAILED,
+                   "Failed to get geo-rep session");
+            goto out;
+        }
 
-                ret = glusterd_copy_geo_rep_session_files (session, snap_vol);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FILE_OP_FAILED, "Failed to copy files"
-                                " related to session %s", session);
-                        goto out;
-                }
+        ret = glusterd_copy_geo_rep_session_files(session, snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                   "Failed to copy files"
+                   " related to session %s",
+                   session);
+            goto out;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will restore a snapshot volumes
@@ -765,157 +785,140 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_restore(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                     ret             = -1;
-        int32_t                 volcount        = -1;
-        char                    *snapname       = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        glusterd_volinfo_t      *tmp            = NULL;
-        glusterd_volinfo_t      *parent_volinfo = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        glusterd_conf_t         *priv           = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to get snap name");
-                goto out;
+    int ret = -1;
+    int32_t volcount = -1;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    glusterd_volinfo_t *parent_volinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get snap name");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (NULL == snap) {
+        ret = gf_asprintf(op_errstr, "Snapshot (%s) does not exist", snapname);
+        if (ret < 0) {
+            goto out;
         }
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND, "%s",
+               *op_errstr);
+        ret = -1;
+        goto out;
+    }
 
-        snap = glusterd_find_snap_by_name (snapname);
-        if (NULL == snap) {
-                ret = gf_asprintf (op_errstr, "Snapshot (%s) does not exist",
-                                   snapname);
-                if (ret < 0) {
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_NOT_FOUND,
-                        "%s", *op_errstr);
-                ret = -1;
-                goto out;
+    volcount = 0;
+    cds_list_for_each_entry_safe(snap_volinfo, tmp, &snap->volumes, vol_list)
+    {
+        volcount++;
+        ret = glusterd_volinfo_find(snap_volinfo->parent_volname,
+                                    &parent_volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "Could not get volinfo of %s", snap_volinfo->parent_volname);
+            goto out;
         }
 
-        volcount = 0;
-        cds_list_for_each_entry_safe (snap_volinfo, tmp, &snap->volumes,
-                                      vol_list) {
-                volcount++;
-                ret = glusterd_volinfo_find (snap_volinfo->parent_volname,
-                                             &parent_volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_FOUND,
-                                "Could not get volinfo of %s",
-                                snap_volinfo->parent_volname);
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                                  uuid_utoa (snap->snap_id));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set snap "
-                                "uuid in response dictionary for %s snapshot",
-                                snap->snapname);
-                        goto out;
-                }
-
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "volname",
-                                                  snap_volinfo->parent_volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set snap "
-                                "uuid in response dictionary for %s snapshot",
-                                snap->snapname);
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "volid",
-                                        uuid_utoa (parent_volinfo->volume_id));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set snap "
-                                "uuid in response dictionary for %s snapshot",
-                                snap->snapname);
-                        goto out;
-                }
-
-                if (is_origin_glusterd (dict) == _gf_true) {
-                        /* From origin glusterd check if      *
-                         * any peers with snap bricks is down */
-                        ret = glusterd_find_missed_snap
-                                               (rsp_dict, snap_volinfo,
-                                                &priv->peers,
-                                                GF_SNAP_OPTION_TYPE_RESTORE);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MISSED_SNAP_GET_FAIL,
-                                        "Failed to find missed snap restores");
-                                goto out;
-                        }
-                }
-
-                ret = gd_restore_snap_volume (dict, rsp_dict, parent_volinfo,
-                                              snap_volinfo, volcount);
-                if (ret) {
-                        /* No need to update op_errstr because it is assumed
-                         * that the called function will do that in case of
-                         * failure.
-                         */
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_RESTORE_FAIL, "Failed to restore "
-                                "snap for %s", snapname);
-                        goto out;
-                }
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                         uuid_utoa(snap->snap_id));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "uuid in response dictionary for %s snapshot",
+                   snap->snapname);
+            goto out;
+        }
 
-                /* Restore is successful therefore delete the original volume's
-                 * volinfo. If the volinfo is already restored then we should
-                 * delete the backend LVMs */
-                if (!gf_uuid_is_null (parent_volinfo->restored_from_snap)) {
-                        ret = glusterd_lvm_snapshot_remove (rsp_dict,
-                                                            parent_volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_LVM_REMOVE_FAILED,
-                                        "Failed to remove LVM backend");
-                        }
-                }
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "volname",
+                                         snap_volinfo->parent_volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "uuid in response dictionary for %s snapshot",
+                   snap->snapname);
+            goto out;
+        }
 
-                /* Detach the volinfo from priv->volumes, so that no new
-                 * command can ref it any more and then unref it.
-                 */
-                cds_list_del_init (&parent_volinfo->vol_list);
-                glusterd_volinfo_unref (parent_volinfo);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "volid",
+                                         uuid_utoa(parent_volinfo->volume_id));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "uuid in response dictionary for %s snapshot",
+                   snap->snapname);
+            goto out;
+        }
 
-                if (ret)
-                        goto out;
+        if (is_origin_glusterd(dict) == _gf_true) {
+            /* From origin glusterd check if      *
+             * any peers with snap bricks is down */
+            ret = glusterd_find_missed_snap(rsp_dict, snap_volinfo,
+                                            &priv->peers,
+                                            GF_SNAP_OPTION_TYPE_RESTORE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_GET_FAIL,
+                       "Failed to find missed snap restores");
+                goto out;
+            }
+        }
+        /* During snapshot restore, mount point for stopped snap
+         * should exist as it is required to set extended attribute.
+         */
+        ret = glusterd_recreate_vol_brick_mounts(this, snap_volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNT_RECREATE_FAIL,
+                   "Failed to recreate brick mounts for %s", snap->snapname);
+            goto out;
         }
 
-        ret = 0;
+        ret = gd_restore_snap_volume(dict, rsp_dict, parent_volinfo,
+                                     snap_volinfo, volcount);
+        if (ret) {
+            /* No need to update op_errstr because it is assumed
+             * that the called function will do that in case of
+             * failure.
+             */
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                   "Failed to restore "
+                   "snap for %s",
+                   snapname);
+            goto out;
+        }
 
-        /* TODO: Need to check if we need to delete the snap after the
-         * operation is successful or not. Also need to persist the state
-         * of restore operation in the store.
+        /* Detach the volinfo from priv->volumes, so that no new
+         * command can ref it any more and then unref it.
          */
+        cds_list_del_init(&parent_volinfo->vol_list);
+        glusterd_volinfo_unref(parent_volinfo);
+    }
+
+    ret = 0;
+
+    /* TODO: Need to check if we need to delete the snap after the
+     * operation is successful or not. Also need to persist the state
+     * of restore operation in the store.
+     */
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called before actual restore is taken place. This function
@@ -928,470 +931,466 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int32_t
-glusterd_snapshot_restore_prevalidate (dict_t *dict, char **op_errstr,
-                                       uint32_t *op_errno, dict_t *rsp_dict)
+glusterd_snapshot_restore_prevalidate(dict_t *dict, char **op_errstr,
+                                      uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int                     ret             = -1;
-        int32_t                 i               = 0;
-        int32_t                 volcount        = 0;
-        int32_t                 brick_count     = 0;
-        gf_boolean_t            snap_restored   = _gf_false;
-        char                    key[PATH_MAX]   = {0, };
-        char                    *volname        = NULL;
-        char                    *snapname       = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        xlator_t                *this           = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get "
-                        "snap name");
-                goto out;
+    int ret = -1;
+    int32_t i = 0;
+    int32_t volcount = 0;
+    int32_t brick_count = 0;
+    gf_boolean_t snap_restored = _gf_false;
+    char key[64] = "";
+    int keylen;
+    char *volname = NULL;
+    char *snapname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get "
+               "snap name");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (NULL == snap) {
+        ret = gf_asprintf(op_errstr, "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_SNAPEXST;
+        if (ret < 0) {
+            goto out;
         }
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND, "%s",
+               *op_errstr);
+        ret = -1;
+        goto out;
+    }
 
-        snap = glusterd_find_snap_by_name (snapname);
-        if (NULL == snap) {
-                ret = gf_asprintf (op_errstr, "Snapshot (%s) does not exist",
-                                snapname);
-                *op_errno = EG_SNAPEXST;
-                if (ret < 0) {
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND, "%s", *op_errstr);
-                ret = -1;
-                goto out;
-        }
+    snap_restored = snap->snap_restored;
 
-        snap_restored = snap->snap_restored;
+    if (snap_restored) {
+        ret = gf_asprintf(op_errstr,
+                          "Snapshot (%s) is already "
+                          "restored",
+                          snapname);
+        if (ret < 0) {
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               *op_errstr);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_strn(rsp_dict, "snapname", SLEN("snapname"), snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap name(%s)",
+               snapname);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "volcount", SLEN("volcount"), &volcount);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volume count");
+        goto out;
+    }
+
+    /* Snapshot restore will only work if all the volumes,
+       that are part of the snapshot, are stopped. */
+    for (i = 1; i <= volcount; ++i) {
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to "
+                   "get volume name");
+            goto out;
+        }
+
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            ret = gf_asprintf(op_errstr,
+                              "Volume (%s) "
+                              "does not exist",
+                              volname);
+            *op_errno = EG_NOVOL;
+            if (ret < 0) {
+                goto out;
+            }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+                   *op_errstr);
+            ret = -1;
+            goto out;
+        }
+
+        if (glusterd_is_volume_started(volinfo)) {
+            ret = gf_asprintf(
+                op_errstr,
+                "Volume (%s) has been "
+                "started. Volume needs to be stopped before restoring "
+                "a snapshot.",
+                volname);
+            *op_errno = EG_VOLRUN;
+            if (ret < 0) {
+                goto out;
+            }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+                   *op_errstr);
+            ret = -1;
+            goto out;
+        }
+
+        /* Take backup of the volinfo folder */
+        ret = glusterd_snapshot_backup_vol(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to backup "
+                   "volume backend files for %s volume",
+                   volinfo->volname);
+            goto out;
+        }
+    }
+
+    /* Get brickinfo for snap_volumes */
+    volcount = 0;
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        volcount++;
+        brick_count = 0;
 
-        if (snap_restored) {
-                ret = gf_asprintf (op_errstr, "Snapshot (%s) is already "
-                                  "restored", snapname);
-                if (ret < 0) {
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED, "%s", *op_errstr);
-                ret = -1;
-                goto out;
-        }
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            brick_count++;
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
 
-        ret = dict_set_str (rsp_dict, "snapname", snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "snap name(%s)", snapname);
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.path", volcount,
+                              brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->path);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
                 goto out;
-        }
-
-        ret = dict_get_int32 (dict, "volcount", &volcount);
+            }
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get volume count");
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.snap_status",
+                              volcount, brick_count);
+            ret = dict_set_int32n(rsp_dict, key, keylen,
+                                  brickinfo->snap_status);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
                 goto out;
-        }
-
-        /* Snapshot restore will only work if all the volumes,
-           that are part of the snapshot, are stopped. */
-        for (i = 1; i <= volcount; ++i) {
-                snprintf (key, sizeof (key), "volname%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to "
-                                "get volume name");
-                        goto out;
-                }
-
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        ret = gf_asprintf (op_errstr, "Volume (%s) "
-                                           "does not exist", volname);
-                        *op_errno = EG_NOVOL;
-                        if (ret < 0) {
-                                goto out;
-                        }
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_FOUND, "%s", *op_errstr);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (glusterd_is_volume_started (volinfo)) {
-                        ret = gf_asprintf (op_errstr, "Volume (%s) has been "
-                        "started. Volume needs to be stopped before restoring "
-                        "a snapshot.", volname);
-                        *op_errno = EG_VOLRUN;
-                        if (ret < 0) {
-                                goto out;
-                        }
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAPSHOT_OP_FAILED, "%s", *op_errstr);
-                        ret = -1;
-                        goto out;
-                }
+            }
 
-                /* Take backup of the volinfo folder */
-                ret = glusterd_snapshot_backup_vol (volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_OP_FAILED,
-                                "Failed to backup "
-                                "volume backend files for %s volume",
-                                volinfo->volname);
-                        goto out;
-                }
-        }
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.device_path",
+                              volcount, brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->device_path);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
 
-        /* Get brickinfo for snap_volumes */
-        volcount = 0;
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                volcount++;
-                brick_count = 0;
-
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        brick_count++;
-                        if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                                continue;
-
-                        snprintf (key, sizeof (key), "snap%d.brick%d.path",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key, brickinfo->path);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.snap_status",
-                                  volcount, brick_count);
-                        ret = dict_set_int32 (rsp_dict, key,
-                                              brickinfo->snap_status);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.device_path",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key,
-                                            brickinfo->device_path);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.fs_type",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key,
-                                            brickinfo->fstype);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.mnt_opts",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key,
-                                            brickinfo->mnt_opts);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-                }
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.fs_type",
+                              volcount, brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->fstype);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
 
-                snprintf (key, sizeof (key), "snap%d.brick_count", volcount);
-                ret = dict_set_int32 (rsp_dict, key, brick_count);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set %s", key);
-                        goto out;
-                }
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.mnt_opts",
+                              volcount, brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->mnt_opts);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
         }
 
-        ret = dict_set_int32 (rsp_dict, "volcount", volcount);
+        keylen = snprintf(key, sizeof(key), "snap%d.brick_count", volcount);
+        ret = dict_set_int32n(rsp_dict, key, keylen, brick_count);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set %s", key);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
+    }
+
+    ret = dict_set_int32n(rsp_dict, "volcount", SLEN("volcount"), volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-snap_max_hard_limits_validate (dict_t *dict, char *volname,
-                               uint64_t value, char **op_errstr)
+snap_max_hard_limits_validate(dict_t *dict, char *volname, uint64_t value,
+                              char **op_errstr)
 {
-        char                err_str[PATH_MAX] = "";
-        glusterd_conf_t    *conf              = NULL;
-        glusterd_volinfo_t *volinfo           = NULL;
-        int                 ret               = -1;
-        uint64_t            max_limit         = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        xlator_t           *this              = NULL;
-        uint64_t            opt_hard_max      = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        if (volname) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (!ret) {
-                        if (volinfo->is_snap_volume) {
-                                ret = -1;
-                                snprintf (err_str, PATH_MAX,
-                                         "%s is a snap volume. Configuring "
-                                         "snap-max-hard-limit for a snap "
-                                         "volume is prohibited.", volname);
-                                goto out;
-                        }
-                }
-        }
+    char err_str[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    uint64_t max_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
 
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to use the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_uint64 (conf->opts,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               &opt_hard_max);
-        if (ret) {
-                ret = 0;
-                gf_msg_debug (this->name, 0, "%s is not present in "
-                        "opts dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
+    this = THIS;
 
-        /* volume snap-max-hard-limit cannot exceed system snap-max-hard-limit.
-         * Hence during prevalidate following checks are made to ensure the
-         * snap-max-hard-limit set on one particular volume does not
-         * exceed snap-max-hard-limit set globally (system limit).
-         */
-        if (value && volname) {
-                max_limit = opt_hard_max;
-        }
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
 
-        if (value > max_limit) {
-                ret = -1;
-                snprintf (err_str, PATH_MAX, "Invalid snap-max-hard-limit "
-                          "%"PRIu64 ". Expected range 1 - %"PRIu64,
-                          value, max_limit);
-                goto out;
-        }
+    conf = this->private;
 
+    GF_ASSERT(conf);
+
+    if (volname) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (!ret) {
+            if (volinfo->is_snap_volume) {
+                ret = -1;
+                snprintf(err_str, PATH_MAX,
+                         "%s is a snap volume. Configuring "
+                         "snap-max-hard-limit for a snap "
+                         "volume is prohibited.",
+                         volname);
+                goto out;
+            }
+        }
+    }
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to use the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_uint64(conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          &opt_hard_max);
+    if (ret) {
         ret = 0;
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in "
+                     "opts dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    /* volume snap-max-hard-limit cannot exceed system snap-max-hard-limit.
+     * Hence during prevalidate following checks are made to ensure the
+     * snap-max-hard-limit set on one particular volume does not
+     * exceed snap-max-hard-limit set globally (system limit).
+     */
+    if (value && volname) {
+        max_limit = opt_hard_max;
+    }
+
+    if (value > max_limit) {
+        ret = -1;
+        snprintf(err_str, PATH_MAX,
+                 "Invalid snap-max-hard-limit "
+                 "%" PRIu64 ". Expected range 1 - %" PRIu64,
+                 value, max_limit);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (ret) {
-                *op_errstr = gf_strdup (err_str);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED, "%s", err_str);
-        }
-        return ret;
+    if (ret) {
+        *op_errstr = gf_strdup(err_str);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+    }
+    return ret;
 }
 
 int
-glusterd_snapshot_config_prevalidate (dict_t *dict, char **op_errstr,
-                                      uint32_t *op_errno)
+glusterd_snapshot_config_prevalidate(dict_t *dict, char **op_errstr,
+                                     uint32_t *op_errno)
 {
-        char               *volname             = NULL;
-        glusterd_volinfo_t *volinfo             = NULL;
-        xlator_t           *this                = NULL;
-        int                 ret                 = -1;
-        int                 config_command      = 0;
-        char                err_str[PATH_MAX]   = {0,};
-        glusterd_conf_t    *conf                = NULL;
-        uint64_t            hard_limit          = 0;
-        uint64_t            soft_limit          = 0;
-        gf_loglevel_t       loglevel            = GF_LOG_ERROR;
-        uint64_t            max_limit           = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        int32_t             cur_auto_delete     = 0;
-        int32_t             req_auto_delete     = 0;
-        int32_t             cur_snap_activate   = 0;
-        int32_t             req_snap_activate   = 0;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str),
-                          "failed to get config-command type");
-                goto out;
-        }
-
-        if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (volname) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume (%s) does not exist.", volname);
-                        *op_errno = EG_NOVOL;
-                        goto out;
-                }
-        }
-
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (dict, &hard_limit, &soft_limit);
-
-        if (hard_limit) {
-                /* Validations for snap-max-hard-limits */
-                ret = snap_max_hard_limits_validate (dict, volname,
-                                                     hard_limit, op_errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_HARD_LIMIT_SET_FAIL,
-                                "snap-max-hard-limit validation failed.");
-                        *op_errno = EINVAL;
-                        goto out;
-                }
-        }
-
-        if (soft_limit) {
-                max_limit = GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT;
-                if (soft_limit > max_limit) {
-                        ret = -1;
-                        snprintf (err_str, PATH_MAX, "Invalid "
-                                 "snap-max-soft-limit ""%"
-                                 PRIu64 ". Expected range 1 - %"PRIu64,
-                                 soft_limit, max_limit);
-                        *op_errno = EINVAL;
-                        goto out;
-                }
-        }
-
-        if (hard_limit || soft_limit) {
-                ret = 0;
-                goto out;
-        }
-
-        if (dict_get(dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE)) {
-                req_auto_delete = dict_get_str_boolean (dict,
-                                        GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                        _gf_false);
-                if (req_auto_delete < 0) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str), "Please enter a "
-                                  "valid boolean value for auto-delete");
-                        *op_errno = EINVAL;
-                        goto out;
-                }
-
-                /* Ignoring the error as the auto-delete is optional and
-                   might not be present in the options dictionary.*/
-                cur_auto_delete = dict_get_str_boolean (conf->opts,
-                                        GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                        _gf_false);
-
-                if (cur_auto_delete == req_auto_delete) {
-                        ret = -1;
-                        if (cur_auto_delete == _gf_true)
-                                snprintf (err_str, sizeof (err_str),
-                                          "auto-delete is already enabled");
-                        else
-                                snprintf (err_str, sizeof (err_str),
-                                          "auto-delete is already disabled");
-                        *op_errno = EINVAL;
-                        goto out;
-                }
-        } else if (dict_get(dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE)) {
-                req_snap_activate = dict_get_str_boolean (dict,
-                                        GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                        _gf_false);
-                if (req_snap_activate < 0) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str), "Please enter a "
-                                  "valid boolean value for activate-on-create");
-                        *op_errno = EINVAL;
-                        goto out;
-                }
-
-                /* Ignoring the error as the activate-on-create is optional and
-                   might not be present in the options dictionary.*/
-                cur_snap_activate = dict_get_str_boolean (conf->opts,
-                                        GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                        _gf_false);
-
-                if (cur_snap_activate == req_snap_activate) {
-                        ret = -1;
-                        if (cur_snap_activate == _gf_true)
-                                snprintf (err_str, sizeof (err_str),
-                                      "activate-on-create is already enabled");
-                        else
-                                snprintf (err_str, sizeof (err_str),
-                                      "activate-on-create is already disabled");
-                        *op_errno = EINVAL;
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Invalid option");
-                *op_errno = EINVAL;
-                goto out;
-        }
-
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int config_command = 0;
+    char err_str[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    uint64_t max_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    int32_t cur_auto_delete = 0;
+    int32_t req_auto_delete = 0;
+    int32_t cur_snap_activate = 0;
+    int32_t req_snap_activate = 0;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    ret = dict_get_int32n(dict, "config-command", SLEN("config-command"),
+                          &config_command);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "failed to get config-command type");
+        goto out;
+    }
+
+    if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (volname) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str), "Volume (%s) does not exist.",
+                     volname);
+            *op_errno = EG_NOVOL;
+            goto out;
+        }
+    }
+
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(dict, &hard_limit, &soft_limit);
+
+    if (hard_limit) {
+        /* Validations for snap-max-hard-limits */
+        ret = snap_max_hard_limits_validate(dict, volname, hard_limit,
+                                            op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HARD_LIMIT_SET_FAIL,
+                   "snap-max-hard-limit validation failed.");
+            *op_errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (soft_limit) {
+        max_limit = GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT;
+        if (soft_limit > max_limit) {
+            ret = -1;
+            snprintf(err_str, PATH_MAX,
+                     "Invalid "
+                     "snap-max-soft-limit "
+                     "%" PRIu64 ". Expected range 1 - %" PRIu64,
+                     soft_limit, max_limit);
+            *op_errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (hard_limit || soft_limit) {
         ret = 0;
+        goto out;
+    }
+
+    if (dict_getn(dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                  SLEN(GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE))) {
+        req_auto_delete = dict_get_str_boolean(
+            dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, _gf_false);
+        if (req_auto_delete < 0) {
+            ret = -1;
+            snprintf(err_str, sizeof(err_str),
+                     "Please enter a "
+                     "valid boolean value for auto-delete");
+            *op_errno = EINVAL;
+            goto out;
+        }
+
+        /* Ignoring the error as the auto-delete is optional and
+           might not be present in the options dictionary.*/
+        cur_auto_delete = dict_get_str_boolean(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, _gf_false);
+
+        if (cur_auto_delete == req_auto_delete) {
+            ret = -1;
+            if (cur_auto_delete == _gf_true)
+                snprintf(err_str, sizeof(err_str),
+                         "auto-delete is already enabled");
+            else
+                snprintf(err_str, sizeof(err_str),
+                         "auto-delete is already disabled");
+            *op_errno = EINVAL;
+            goto out;
+        }
+    } else if (dict_getn(dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+                         SLEN(GLUSTERD_STORE_KEY_SNAP_ACTIVATE))) {
+        req_snap_activate = dict_get_str_boolean(
+            dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+        if (req_snap_activate < 0) {
+            ret = -1;
+            snprintf(err_str, sizeof(err_str),
+                     "Please enter a "
+                     "valid boolean value for activate-on-create");
+            *op_errno = EINVAL;
+            goto out;
+        }
+
+        /* Ignoring the error as the activate-on-create is optional and
+           might not be present in the options dictionary.*/
+        cur_snap_activate = dict_get_str_boolean(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+
+        if (cur_snap_activate == req_snap_activate) {
+            ret = -1;
+            if (cur_snap_activate == _gf_true)
+                snprintf(err_str, sizeof(err_str),
+                         "activate-on-create is already enabled");
+            else
+                snprintf(err_str, sizeof(err_str),
+                         "activate-on-create is already disabled");
+            *op_errno = EINVAL;
+            goto out;
+        }
+    } else {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str), "Invalid option");
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        if (ret && err_str[0] != '\0') {
-                gf_msg (this->name, loglevel, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
+    if (ret && err_str[0] != '\0') {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 /* This function will be called from RPC handler routine.
@@ -1405,485 +1404,452 @@ out:
  * @return -1 on error and 0 on success
  */
 int
-glusterd_handle_snapshot_config (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_config(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, size_t len)
 {
-        int32_t         ret             = -1;
-        char            *volname        = NULL;
-        xlator_t        *this           = NULL;
-        int             config_command  = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, req, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        /* TODO : Type of lock to be taken when we are setting
-         * limits system wide
-         */
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                snprintf (err_str, len,
-                         "Failed to get config-command type");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        switch (config_command) {
+    int32_t ret = -1;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    int config_command = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    /* TODO : Type of lock to be taken when we are setting
+     * limits system wide
+     */
+    ret = dict_get_int32n(dict, "config-command", SLEN("config-command"),
+                          &config_command);
+    if (ret) {
+        snprintf(err_str, len, "Failed to get config-command type");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=config-command", NULL);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    switch (config_command) {
         case GF_SNAP_CONFIG_TYPE_SET:
-                if (!volname) {
-                        ret = dict_set_int32 (dict, "hold_vol_locks",
-                                              _gf_false);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Unable to set hold_vol_locks value "
-                                        "as _gf_false");
-                                goto out;
-                        }
-
-                }
-                ret = glusterd_mgmt_v3_initiate_all_phases (req, op, dict);
-                break;
+            if (!volname) {
+                ret = dict_set_int32n(dict, "hold_vol_locks",
+                                      SLEN("hold_vol_locks"), _gf_false);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set hold_vol_locks value "
+                           "as _gf_false");
+                    goto out;
+                }
+            }
+            ret = glusterd_mgmt_v3_initiate_all_phases(req, op, dict);
+            break;
         case GF_SNAP_CONFIG_DISPLAY:
-                /* Reading data from local node only */
-                ret = snap_max_limits_display_commit (dict, volname,
-                                                      err_str, len);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_HARD_LIMIT_SET_FAIL,
-                                "snap-max-limit "
-                                "display commit failed.");
-                        goto out;
-                }
-
-                /* If everything is successful then send the response
-                 * back to cli
-                 */
-                ret = glusterd_op_send_cli_response (op, 0, 0, req, dict,
-                                                     err_str);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_NO_CLI_RESP, "Failed to send cli "
-                                        "response");
-                        goto out;
-                }
-
-                break;
+            /* Reading data from local node only */
+            ret = snap_max_limits_display_commit(dict, volname, err_str, len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HARD_LIMIT_SET_FAIL,
+                       "snap-max-limit "
+                       "display commit failed.");
+                goto out;
+            }
+
+            /* If everything is successful then send the response
+             * back to cli
+             */
+            ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+                       "Failed to send cli "
+                       "response");
+                goto out;
+            }
+
+            break;
         default:
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_COMMAND_NOT_FOUND, "Unknown config type");
-                ret = -1;
-                break;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "Unknown config type");
+            ret = -1;
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 int
-glusterd_snap_create_clone_pre_val_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_create_clone_pre_val_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        char                  *snap_brick_dir        = NULL;
-        char                  *snap_device           = NULL;
-        char                   key[PATH_MAX]         = "";
-        char                  *value                 = "";
-        char                   snapbrckcnt[PATH_MAX] = "";
-        char                   snapbrckord[PATH_MAX] = "";
-        int                    ret                   = -1;
-        int64_t                i                     = -1;
-        int64_t                j                     = -1;
-        int64_t                volume_count          = 0;
-        int64_t                brick_count           = 0;
-        int64_t                brick_order           = 0;
-        xlator_t              *this                  = NULL;
-        int32_t                brick_online          = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dst);
-        GF_ASSERT (src);
-
-        ret = dict_get_int64 (src, "volcount", &volume_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to "
-                        "get the volume count");
-                goto out;
-        }
-
-        for (i = 0; i < volume_count; i++) {
-                memset (snapbrckcnt, '\0', sizeof(snapbrckcnt));
-                ret = snprintf (snapbrckcnt, sizeof(snapbrckcnt) - 1,
-                                "vol%"PRId64"_brickcount", i+1);
-                ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
-                if (ret) {
-                        gf_msg_trace (this->name, 0,
-                                "No bricks for this volume in this dict");
-                        continue;
-                }
-
-                for (j = 0; j < brick_count; j++) {
-                        /* Fetching data from source dict */
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brickdir%"PRId64, i+1, j);
-                        ret = dict_get_ptr (src, key,
-                                            (void **)&snap_brick_dir);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to fetch %s", key);
-                                continue;
-                        }
-
-                        /* Fetching brick order from source dict */
-                        snprintf (snapbrckord, sizeof(snapbrckord) - 1,
-                                  "vol%"PRId64".brick%"PRId64".order", i+1, j);
-                        ret = dict_get_int64 (src, snapbrckord, &brick_order);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get brick order");
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brickdir%"PRId64, i+1,
-                                  brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key,
-                                                          snap_brick_dir);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".fstype%"PRId64, i+1, j);
-                        ret = dict_get_str (src, key, &value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to fetch %s", key);
-                                continue;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".fstype%"PRId64, i+1,
-                                  brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".mnt_opts%"PRId64, i+1, j);
-                        ret = dict_get_str (src, key, &value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to fetch %s", key);
-                                continue;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".mnt_opts%"PRId64, i+1,
-                                  brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brick_snapdevice%"PRId64,
-                                  i+1, j);
-                        ret = dict_get_ptr (src, key,
-                                            (void **)&snap_device);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to fetch snap_device");
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brick_snapdevice%"PRId64,
-                                  i+1, brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key,
-                                                          snap_device);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "vol%"PRId64".brick%"PRId64".status", i+1, brick_order);
-                        ret = dict_get_int32 (src, key, &brick_online);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "failed to "
-                                        "get the brick status");
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (dst, key, brick_online);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "failed to "
-                                        "set the brick status");
-                                goto out;
-                        }
-                        brick_online = 0;
-                }
-        }
-        ret = 0;
+    char *snap_brick_dir = NULL;
+    char *snap_device = NULL;
+    char key[64] = "";
+    int keylen;
+    char *value = "";
+    char snapbrckcnt[PATH_MAX] = "";
+    char snapbrckord[PATH_MAX] = "";
+    int ret = -1;
+    int64_t i = -1;
+    int64_t j = -1;
+    int64_t volume_count = 0;
+    int64_t brick_count = 0;
+    int64_t brick_order = 0;
+    xlator_t *this = NULL;
+    int32_t brick_online = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dst);
+    GF_ASSERT(src);
+
+    ret = dict_get_int64(src, "volcount", &volume_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+
+    for (i = 0; i < volume_count; i++) {
+        ret = snprintf(snapbrckcnt, sizeof(snapbrckcnt) - 1,
+                       "vol%" PRId64 "_brickcount", i + 1);
+        ret = dict_get_int64(src, snapbrckcnt, &brick_count);
+        if (ret) {
+            gf_msg_trace(this->name, 0,
+                         "No bricks for this volume in this dict");
+            continue;
+        }
+
+        for (j = 0; j < brick_count; j++) {
+            /* Fetching data from source dict */
+            snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%" PRId64, i + 1,
+                     j);
+            ret = dict_get_ptr(src, key, (void **)&snap_brick_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch %s", key);
+                continue;
+            }
+
+            /* Fetching brick order from source dict */
+            snprintf(snapbrckord, sizeof(snapbrckord) - 1,
+                     "vol%" PRId64 ".brick%" PRId64 ".order", i + 1, j);
+            ret = dict_get_int64(src, snapbrckord, &brick_order);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get brick order");
+                goto out;
+            }
+
+            snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, snap_brick_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%" PRId64,
+                              i + 1, j);
+            ret = dict_get_strn(src, key, keylen, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch %s", key);
+                continue;
+            }
+
+            snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key),
+                              "vol%" PRId64 ".mnt_opts%" PRId64, i + 1, j);
+            ret = dict_get_strn(src, key, keylen, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch %s", key);
+                continue;
+            }
+
+            snprintf(key, sizeof(key), "vol%" PRId64 ".mnt_opts%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            snprintf(key, sizeof(key),
+                     "vol%" PRId64 ".brick_snapdevice%" PRId64, i + 1, j);
+            ret = dict_get_ptr(src, key, (void **)&snap_device);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch snap_device");
+                goto out;
+            }
+
+            snprintf(key, sizeof(key),
+                     "vol%" PRId64 ".brick_snapdevice%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, snap_device);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key),
+                              "vol%" PRId64 ".brick%" PRId64 ".status", i + 1,
+                              brick_order);
+            ret = dict_get_int32n(src, key, keylen, &brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "failed to "
+                       "get the brick status");
+                goto out;
+            }
+
+            ret = dict_set_int32n(dst, key, keylen, brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to "
+                       "set the brick status");
+                goto out;
+            }
+            brick_online = 0;
+        }
+    }
+    ret = 0;
 out:
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Aggregate brickinfo's of the snap volumes to be restored from */
 int32_t
-glusterd_snap_restore_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_restore_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        char           key[PATH_MAX]  = "";
-        char          *strvalue       = NULL;
-        int32_t        value          = -1;
-        int32_t        i              = -1;
-        int32_t        j              = -1;
-        int32_t        vol_count       = -1;
-        int32_t        brickcount    = -1;
-        int32_t        ret            = -1;
-        xlator_t      *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                      GD_MSG_INVALID_ENTRY, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
+    char key[64] = "";
+    int keylen;
+    char *strvalue = NULL;
+    int32_t value = -1;
+    int32_t i = -1;
+    int32_t j = -1;
+    int32_t vol_count = -1;
+    int32_t brickcount = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(src, "volcount", &vol_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No volumes");
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_int32 (src, "volcount", &vol_count);
+    for (i = 1; i <= vol_count; i++) {
+        keylen = snprintf(key, sizeof(key), "snap%d.brick_count", i);
+        ret = dict_get_int32n(src, key, keylen, &brickcount);
         if (ret) {
-                gf_msg_debug (this->name, 0, "No volumes");
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get %s", key);
+            goto out;
         }
 
-        for (i = 1; i <= vol_count; i++) {
-                snprintf (key, sizeof (key), "snap%d.brick_count", i);
-                ret = dict_get_int32 (src, key, &brickcount);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Failed to get %s", key);
-                        goto out;
-                }
-
-                for (j = 1; j <= brickcount; j++) {
-                        snprintf (key, sizeof (key), "snap%d.brick%d.path",
-                                  i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                /* The brickinfo will be present in
-                                 * another rsp_dict */
-                                gf_msg_debug (this->name, 0,
-                                        "%s not present", key);
-                                ret = 0;
-                                continue;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.snap_status", i, j);
-                        ret = dict_get_int32 (src, key, &value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dst, key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.device_path", i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.fs_type", i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.mnt_opts", i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-                }
-        }
+        for (j = 1; j <= brickcount; j++) {
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.path", i, j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                /* The brickinfo will be present in
+                 * another rsp_dict */
+                gf_msg_debug(this->name, 0, "%s not present", key);
+                ret = 0;
+                continue;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.snap_status", i,
+                              j);
+            ret = dict_get_int32n(src, key, keylen, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_int32n(dst, key, keylen, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.device_path", i,
+                              j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.fs_type", i, j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.mnt_opts", i,
+                              j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+        }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_pre_validate_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        int             ret             = -1;
-        int32_t         snap_command    = 0;
-        xlator_t       *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dst, "type", &snap_command);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int ret = -1;
+    int32_t snap_command = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dst, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_snap_create_clone_pre_val_use_rsp_dict (dst,
-                                                                       src);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Unable to use "
-                                "rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_create_clone_pre_val_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to use "
+                       "rsp dict");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snap_restore_use_rsp_dict (dst, src);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_RSP_DICT_USE_FAIL, "Unable to use "
-                                "rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_restore_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RSP_DICT_USE_FAIL,
+                       "Unable to use "
+                       "rsp dict");
+                goto out;
+            }
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_add_brick_status_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   char *key_prefix)
+glusterd_add_brick_status_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brickinfo,
+                                  char *key_prefix)
 {
-        char                   pidfile[PATH_MAX]         = {0, };
-        int32_t                brick_online              = 0;
-        pid_t                  pid                       = 0;
-        xlator_t              *this                      = NULL;
-        glusterd_conf_t       *conf                      = NULL;
-        int                    ret                       = -1;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (!key_prefix) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "key prefix is NULL");
-                goto out;
-        }
-
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, conf);
-
-        brick_online = gf_is_service_running (pidfile, &pid);
-
-        ret = dict_set_int32 (dict, key_prefix, brick_online);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set %s", key_prefix);
-                goto out;
-        }
-        brick_online = 0;
-
-        ret = 0;
+    char pidfile[PATH_MAX] = "";
+    int32_t brick_online = 0;
+    pid_t pid = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (!key_prefix) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "key prefix is NULL");
+        goto out;
+    }
+
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+
+    brick_online = gf_is_service_running(pidfile, &pid);
+
+    ret = dict_set_int32(dict, key_prefix, brick_online);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key_prefix);
+        goto out;
+    }
+    brick_online = 0;
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will check whether the given device
@@ -1894,1772 +1860,1719 @@ out:
  * @return              _gf_true if LV is thin else _gf_false
  */
 gf_boolean_t
-glusterd_is_thinp_brick (char *device, uint32_t *op_errno)
+glusterd_is_thinp_brick(char *device, uint32_t *op_errno)
 {
-        int             ret                     = -1;
-        char            msg [1024]              = "";
-        char            pool_name [PATH_MAX]    = "";
-        char           *ptr                     = NULL;
-        xlator_t       *this                    = NULL;
-        runner_t        runner                  = {0,};
-        gf_boolean_t    is_thin                 = _gf_false;
-
-        this = THIS;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, device, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        snprintf (msg, sizeof (msg), "Get thin pool name for device %s",
-                  device);
-
-        runinit (&runner);
-
-        runner_add_args (&runner, "/sbin/lvs", "--noheadings", "-o", "pool_lv",
-                         device, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_TPOOL_GET_FAIL, "Failed to get thin pool "
-                        "name for device %s", device);
-                runner_end (&runner);
-                goto out;
-        }
-
-        ptr = fgets(pool_name, sizeof(pool_name),
-                    runner_chio (&runner, STDOUT_FILENO));
-        if (!ptr || !strlen(pool_name)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_TPOOL_GET_FAIL, "Failed to get pool name "
-                        "for device %s", device);
-                runner_end (&runner);
-                ret = -1;
-                goto out;
-        }
-
-        runner_end (&runner);
-
-        /* Trim all the whitespaces. */
-        ptr = gf_trim (pool_name);
-
-        /* If the LV has thin pool associated with this
-         * then it is a thinly provisioned LV else it is
-         * regular LV */
-        if (0 != ptr [0]) {
-                is_thin = _gf_true;
-        }
+    int ret = -1;
+    char msg[1024] = "";
+    char pool_name[PATH_MAX] = "";
+    char *ptr = NULL;
+    xlator_t *this = NULL;
+    runner_t runner = {
+        0,
+    };
+    gf_boolean_t is_thin = _gf_false;
+
+    this = THIS;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, device, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    snprintf(msg, sizeof(msg), "Get thin pool name for device %s", device);
+
+    runinit(&runner);
+
+    runner_add_args(&runner, "lvs", "--noheadings", "-o", "pool_lv", device,
+                    NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_TPOOL_GET_FAIL,
+               "Failed to get thin pool "
+               "name for device %s",
+               device);
+        runner_end(&runner);
+        goto out;
+    }
+
+    ptr = fgets(pool_name, sizeof(pool_name),
+                runner_chio(&runner, STDOUT_FILENO));
+    if (!ptr || !strlen(pool_name)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_TPOOL_GET_FAIL,
+               "Failed to get pool name "
+               "for device %s",
+               device);
+        runner_end(&runner);
+        ret = -1;
+        goto out;
+    }
+
+    runner_end(&runner);
+
+    /* Trim all the whitespaces. */
+    ptr = gf_trim(pool_name);
+
+    /* If the LV has thin pool associated with this
+     * then it is a thinly provisioned LV else it is
+     * regular LV */
+    if (0 != ptr[0]) {
+        is_thin = _gf_true;
+    }
 
 out:
-        if (!is_thin)
-                *op_errno = EG_NOTTHINP;
+    if (!is_thin)
+        *op_errno = EG_NOTTHINP;
 
-        return is_thin;
+    return is_thin;
 }
 
 int
-glusterd_snapshot_pause_tier (xlator_t *this, glusterd_volinfo_t *volinfo)
+glusterd_snap_create_clone_common_prevalidate(
+    dict_t *rsp_dict, int flags, char *snapname, char *err_str,
+    char *snap_volname, int64_t volcount, glusterd_volinfo_t *volinfo,
+    gf_loglevel_t *loglevel, int clone, uint32_t *op_errno)
 {
-        int                     ret             = -1;
-        dict_t                 *dict            = NULL;
-        char                   *op_errstr       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-
-        if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                ret = 0;
+    char *device = NULL;
+    char *orig_device = NULL;
+    char key[128] = "";
+    int ret = -1;
+    int64_t i = 1;
+    int64_t brick_order = 0;
+    int64_t brick_count = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!snapname || !volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Failed to validate "
+               "snapname or volume information");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            brick_order++;
+            continue;
+        }
+
+        if (!glusterd_is_brick_started(brickinfo)) {
+            if (!clone && (flags & GF_CLI_FLAG_OP_FORCE)) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "brick %s:%s is not started", brickinfo->hostname,
+                       brickinfo->path);
+                brick_order++;
+                brick_count++;
+                continue;
+            }
+            if (!clone) {
+                snprintf(err_str, PATH_MAX,
+                         "One or more bricks are not running. "
+                         "Please run volume status command to see "
+                         "brick status.\n"
+                         "Please start the stopped brick "
+                         "and then issue snapshot create "
+                         "command or use [force] option in "
+                         "snapshot create to override this "
+                         "behavior.");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_BRICK_NOT_RUNNING,
+                        "Please run volume status command to see brick "
+                        "status.Please start the stopped brick and then issue "
+                        "snapshot create command or use 'force' option in "
+                        "snapshot create to override this behavior.",
+                        NULL);
+            } else {
+                snprintf(err_str, PATH_MAX,
+                         "One or more bricks are not running. "
+                         "Please run snapshot status command to see "
+                         "brick status.\n"
+                         "Please start the stopped brick "
+                         "and then issue snapshot clone "
+                         "command ");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_BRICK_NOT_RUNNING,
+                        "Please run snapshot status command to see brick "
+                        "status. Please start the stopped brick and then issue "
+                        "snapshot clone command.",
+                        NULL);
+            }
+            *op_errno = EG_BRCKDWN;
+            ret = -1;
+            goto out;
+        }
+
+        orig_device = glusterd_get_brick_mount_device(brickinfo->path);
+        if (!orig_device) {
+            len = snprintf(err_str, PATH_MAX,
+                           "getting device name for the brick "
+                           "%s:%s failed",
+                           brickinfo->hostname, brickinfo->path);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_BRK_MNTPATH_GET_FAIL,
+                    "Brick_hostname=%s, Brick_path=%s", brickinfo->hostname,
+                    brickinfo->path, NULL);
+            ret = -1;
+            goto out;
+        }
+        if (!clone) {
+            if (!glusterd_is_thinp_brick(orig_device, op_errno)) {
+                snprintf(err_str, PATH_MAX,
+                         "Snapshot is supported only for "
+                         "thin provisioned LV. Ensure that "
+                         "all bricks of %s are thinly "
+                         "provisioned LV.",
+                         volinfo->volname);
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED,
+                        "Ensure that all bricks of volume are thinly "
+                        "provisioned LV, Volume=%s",
+                        volinfo->volname, NULL);
+                ret = -1;
                 goto out;
+            }
         }
 
-        dict = dict_new ();
-        if (!dict) {
-                goto out;
+        device = glusterd_build_snap_device_path(orig_device, snap_volname,
+                                                 brick_count);
+        if (!device) {
+            snprintf(err_str, PATH_MAX,
+                     "cannot copy the snapshot device "
+                     "name (volname: %s, snapname: %s)",
+                     volinfo->volname, snapname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_SNAP_DEVICE_NAME_GET_FAIL, "Volname=%s, Snapname=%s",
+                    volinfo->volname, snapname, NULL);
+            *loglevel = GF_LOG_WARNING;
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "rebalance-command",
-                              GF_DEFRAG_CMD_PAUSE_TIER);
+        GF_FREE(orig_device);
+        orig_device = NULL;
+
+        snprintf(key, sizeof(key), "vol%" PRId64 ".brick_snapdevice%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, device);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set rebalance-command");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
 
-        ret = dict_set_str (dict, "volname", volinfo->volname);
+        ret = glusterd_update_mntopts(brickinfo->path, brickinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set volname");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MOUNTOPTS_FAIL,
+                   "Failed to "
+                   "update mount options for %s brick",
+                   brickinfo->path);
         }
 
-        ret = gd_brick_op_phase (GD_OP_DEFRAG_BRICK_VOLUME, NULL,
-                                 dict, &op_errstr);
+        snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, brickinfo->fstype);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_PAUSE_TIER_FAIL,
-                        "Failed to pause tier. Errstr=%s",
-                        op_errstr);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
 
-out:
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
-}
-
-
-int
-glusterd_snapshot_resume_tier (xlator_t *this, dict_t *snap_dict)
-{
-        int                     ret             = -1;
-        dict_t                 *dict            = NULL;
-        int64_t                 volcount        = 0;
-        char                    key[PATH_MAX]   = "";
-        char                   *volname         = NULL;
-        int                     i               = 0;
-        char                   *op_errstr       = NULL;
-        glusterd_volinfo_t     *volinfo         = NULL;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, snap_dict, out);
-
-        ret = dict_get_int64 (snap_dict, "volcount", &volcount);
+        snprintf(key, sizeof(key), "vol%" PRId64 ".mnt_opts%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, brickinfo->mnt_opts);
         if (ret) {
-                goto out;
-        }
-        if (volcount <= 0) {
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%d", i);
-                ret = dict_get_str (snap_dict, key, &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to get key %s", volname);
-                        goto out;
-                }
-
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret)
-                        goto out;
-
-                if (volinfo->type != GF_CLUSTER_TYPE_TIER)
-                        continue;
-
-                ret = dict_set_int32 (dict, "rebalance-command",
-                                      GF_DEFRAG_CMD_RESUME_TIER);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                        "Failed to set rebalance-command");
+        snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, brickinfo->mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
+        }
 
-                        goto out;
-                }
+        snprintf(key, sizeof(key) - 1, "vol%" PRId64 ".brick%" PRId64 ".order",
+                 i, brick_count);
+        ret = dict_set_int64(rsp_dict, key, brick_order);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
+        }
 
-                ret = dict_set_str (dict, "volname", volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set volname");
-                        goto out;
-                }
+        snprintf(key, sizeof(key), "vol%" PRId64 ".brick%" PRId64 ".status", i,
+                 brick_order);
 
-                ret = gd_brick_op_phase (GD_OP_DEFRAG_BRICK_VOLUME, NULL,
-                                         dict, &op_errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_RESUME_TIER_FAIL,
-                                "Failed to resume tier");
-                        goto out;
-                }
+        ret = glusterd_add_brick_status_to_dict(rsp_dict, volinfo, brickinfo,
+                                                key);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to "
+                   "add brick status to dict");
+            goto out;
         }
-
+        brick_count++;
+        brick_order++;
+        if (device) {
+            GF_FREE(device);
+            device = NULL;
+        }
+    }
+    snprintf(key, sizeof(key) - 1, "vol%" PRId64 "_brickcount", volcount);
+    ret = dict_set_int64(rsp_dict, key, brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (orig_device)
+        GF_FREE(orig_device);
 
-        return ret;
-}
+    if (device)
+        GF_FREE(device);
 
+    return ret;
+}
 
 int
-glusterd_snap_create_clone_common_prevalidate (dict_t *rsp_dict, int flags,
-                                               char *snapname, char *err_str,
-                                               char *snap_volname,
-                                               int64_t volcount,
-                                               glusterd_volinfo_t *volinfo,
-                                               gf_loglevel_t *loglevel,
-                                               int clone, uint32_t *op_errno)
+glusterd_snapshot_clone_prevalidate(dict_t *dict, char **op_errstr,
+                                    dict_t *rsp_dict, uint32_t *op_errno)
 {
-        char                  *device            = NULL;
-        char                   key[PATH_MAX]     = "";
-        int                    ret               = -1;
-        int64_t                i                 = 1;
-        int64_t                brick_order       = 0;
-        int64_t                brick_count       = 0;
-        xlator_t              *this              = NULL;
-        glusterd_conf_t       *conf              = NULL;
-        glusterd_brickinfo_t  *brickinfo         = NULL;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!snapname || !volinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Failed to validate "
-                        "snapname or volume information");
-                ret = -1;
-                goto out;
-        }
+    char *clonename = NULL;
+    char *snapname = NULL;
+    char device_name[64] = "";
+    glusterd_snap_t *snap = NULL;
+    char err_str[PATH_MAX] = "";
+    int ret = -1;
+    int64_t volcount = 1;
+    glusterd_volinfo_t *snap_vol = NULL;
+    xlator_t *this = NULL;
+    uuid_t *snap_volid = NULL;
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to "
+                 "get the clone name");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Failed to get snapname");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(clonename, &volinfo);
+    if (!ret) {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Volume with name:%s "
+                 "already exists",
+                 clonename);
+        *op_errno = EG_VOLEXST;
+        goto out;
+    }
+    /* need to find snap volinfo*/
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to find :%s "
+                 "snap",
+                 snapname);
+        goto out;
+    }
+
+    /* TODO : As of now there is only one volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_vol = list_entry(snap->volumes.next, glusterd_volinfo_t, vol_list);
+    if (!snap_vol) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get snap "
+               "volinfo %s",
+               snap->snapname);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(snap_vol)) {
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot %s is "
+                 "not activated",
+                 snap->snapname);
+        loglevel = GF_LOG_WARNING;
+        *op_errno = EG_VOLSTP;
+        goto out;
+    }
+
+    ret = dict_get_bin(dict, "vol1_volid", (void **)&snap_volid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_volid");
+        goto out;
+    }
+
+    GLUSTERD_GET_UUID_NOHYPHEN(device_name, *snap_volid);
+
+    /* Adding snap bricks mount paths to the dict */
+    ret = glusterd_snap_create_clone_common_prevalidate(
+        rsp_dict, 0, snapname, err_str, device_name, 1, snap_vol, &loglevel, 1,
+        op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Failed to pre validate");
+        goto out;
+    }
+
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                 brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                        brick_order++;
-                        continue;
-                }
-
-                if (!glusterd_is_brick_started (brickinfo)) {
-                        if (!clone && (flags & GF_CLI_FLAG_OP_FORCE)) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_BRICK_DISCONNECTED,
-                                        "brick %s:%s is not started",
-                                        brickinfo->hostname,
-                                        brickinfo->path);
-                                brick_order++;
-                                brick_count++;
-                                continue;
-                        }
-                        if (!clone) {
-                        snprintf (err_str, PATH_MAX,
-                                  "One or more bricks are not running. "
-                                  "Please run volume status command to see "
-                                  "brick status.\n"
-                                  "Please start the stopped brick "
-                                  "and then issue snapshot create "
-                                  "command or use [force] option in "
-                                  "snapshot create to override this "
-                                  "behavior.");
-                        } else {
-                        snprintf (err_str, PATH_MAX,
-                                  "One or more bricks are not running. "
-                                  "Please run snapshot status command to see "
-                                  "brick status.\n"
-                                  "Please start the stopped brick "
-                                  "and then issue snapshot clone "
-                                  "command ");
-                        }
-                        *op_errno = EG_BRCKDWN;
-                        ret = -1;
-                        goto out;
-                }
-
-
-                device = glusterd_get_brick_mount_device
-                                          (brickinfo->path);
-                if (!device) {
-                        snprintf (err_str, PATH_MAX,
-                                  "getting device name for the brick "
-                                 "%s:%s failed", brickinfo->hostname,
-                                  brickinfo->path);
-                        ret = -1;
-                        goto out;
-                }
-                if (!clone) {
-                     if (!glusterd_is_thinp_brick (device, op_errno)) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Snapshot is supported only for "
-                                          "thin provisioned LV. Ensure that "
-                                          "all bricks of %s are thinly "
-                                          "provisioned LV.", volinfo->volname);
-                                ret = -1;
-                                goto out;
-                     }
-                }
-
-                device = glusterd_build_snap_device_path (device,
-                                                  snap_volname,
-                                                  brick_count);
-                if (!device) {
-                        snprintf (err_str, PATH_MAX,
-                                  "cannot copy the snapshot device "
-                                "name (volname: %s, snapname: %s)",
-                                volinfo->volname, snapname);
-                        *loglevel = GF_LOG_WARNING;
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key),
-                        "vol%"PRId64".brick_snapdevice%"PRId64,
-                        i, brick_count);
-                ret = dict_set_dynstr (rsp_dict, key, device);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set %s", key);
-                        GF_FREE (device);
-                        goto out;
-                }
-                device = NULL;
-
-                ret = glusterd_update_mntopts (brickinfo->path,
-                                               brickinfo);
-                if (ret) {
-                         gf_msg (this->name, GF_LOG_ERROR, 0,
-                                 GD_MSG_BRK_MOUNTOPTS_FAIL, "Failed to "
-                                "update mount options for %s brick",
-                                brickinfo->path);
-                }
-
-                snprintf (key, sizeof(key), "vol%"PRId64".fstype%"
-                          PRId64, i, brick_count);
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  brickinfo->fstype);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set %s", key);
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key), "vol%"PRId64".mnt_opts%"
-                          PRId64, i, brick_count);
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  brickinfo->mnt_opts);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set %s", key);
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key), "vol%"PRId64".brickdir%"PRId64, i,
-                          brick_count);
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  brickinfo->mount_dir);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set %s", key);
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key) - 1,
-                          "vol%"PRId64".brick%"PRId64".order", i, brick_count);
-                ret = dict_set_int64 (rsp_dict, key, brick_order);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set %s", key);
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key),
-                          "vol%"PRId64".brick%"PRId64".status", i, brick_order);
-
-                ret = glusterd_add_brick_status_to_dict (rsp_dict,
-                                                         volinfo,
-                                                         brickinfo,
-                                                         key);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "failed to "
-                                "add brick status to dict");
-                        goto out;
-                }
-                brick_count++;
-                brick_order++;
-        }
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64"_brickcount", volcount);
-        ret = dict_set_int64 (rsp_dict, key, brick_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set %s",
-                        key);
-                goto out;
-        }
-        ret = 0;
 out:
-        if (device)
-               GF_FREE (device);
 
-       return ret;
+    if (ret && err_str[0] != '\0') {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAP_CLONE_PREVAL_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
 
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int
-glusterd_snapshot_clone_prevalidate (dict_t *dict, char **op_errstr,
+glusterd_snapshot_create_prevalidate(dict_t *dict, char **op_errstr,
                                      dict_t *rsp_dict, uint32_t *op_errno)
 {
-        char                  *clonename         = NULL;
-        char                  *snapname          = NULL;
-        char                   key[PATH_MAX]     = "";
-        glusterd_snap_t       *snap              = NULL;
-        char                   err_str[PATH_MAX] = "";
-        int                    ret               = -1;
-        int64_t                volcount          = 1;
-        glusterd_volinfo_t    *snap_vol          = NULL;
-        xlator_t              *this              = NULL;
-        uuid_t                *snap_volid        = NULL;
-        gf_loglevel_t          loglevel          = GF_LOG_ERROR;
-
-        this = THIS;
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        ret = dict_get_str (dict, "clonename", &clonename);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to "
-                          "get the clone name");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get snapname");
-                goto out;
+    char *volname = NULL;
+    char *snapname = NULL;
+    char key[64] = "";
+    int keylen;
+    char snap_volname[64] = "";
+    char err_str[PATH_MAX] = "";
+    int ret = -1;
+    int64_t i = 0;
+    int64_t volcount = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    uuid_t *snap_volid = NULL;
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    glusterd_conf_t *conf = NULL;
+    int64_t effective_max_limit = 0;
+    int flags = 0;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    char *description = NULL;
+
+    this = THIS;
+    GF_ASSERT(op_errstr);
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to "
+                 "get the volume count");
+        goto out;
+    }
+    if (volcount <= 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Invalid volume count %" PRId64 " supplied", volcount);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Failed to get snapname");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "description", SLEN("description"), &description);
+    if (description && !(*description)) {
+        /* description should have a non-null value */
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot cannot be "
+                 "created with empty description");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get flags");
+        goto out;
+    }
+
+    if (glusterd_find_snap_by_name(snapname)) {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot %s already "
+                 "exists",
+                 snapname);
+        *op_errno = EG_SNAPEXST;
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%" PRId64, i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str), "failed to get volume name");
+            goto out;
+        }
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str), "Volume (%s) does not exist ",
+                     volname);
+            *op_errno = EG_NOVOL;
+            goto out;
+        }
+
+        ret = -1;
+        if (!glusterd_is_volume_started(volinfo)) {
+            snprintf(err_str, sizeof(err_str),
+                     "volume %s is "
+                     "not started",
+                     volinfo->volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_VOLSTP;
+            goto out;
+        }
+
+        if (glusterd_is_defrag_on(volinfo)) {
+            snprintf(err_str, sizeof(err_str),
+                     "rebalance process is running for the "
+                     "volume %s",
+                     volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_RBALRUN;
+            goto out;
+        }
+
+        if (gd_vol_is_geo_rep_active(volinfo)) {
+            snprintf(err_str, sizeof(err_str),
+                     "geo-replication session is running for "
+                     "the volume %s. Session needs to be "
+                     "stopped before taking a snapshot.",
+                     volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_GEOREPRUN;
+            goto out;
+        }
+
+        if (volinfo->is_snap_volume == _gf_true) {
+            snprintf(err_str, sizeof(err_str), "Volume %s is a snap volume",
+                     volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_ISSNAP;
+            goto out;
         }
 
-        if (glusterd_check_volume_exists(clonename)) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Volume with name:%s "
-                          "already exists", clonename);
-                *op_errno = EG_VOLEXST;
-                goto out;
-        }
-        /* need to find snap volinfo*/
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Failed to find :%s "
-                          "snap", snapname);
-                goto out;
-        }
-
-        /* TODO : As of now there is only one volume in snapshot.
-         * Change this when multiple volume snapshot is introduced
+        /* "snap-max-hard-limit" might not be set by user explicitly,
+         * in that case it's better to consider the default value.
+         * Hence not erroring out if Key is not found.
          */
-        snap_vol = list_entry (snap->volumes.next,
-                                   glusterd_volinfo_t, vol_list);
-        if (!snap_vol) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_GET_FAIL, "Failed to get snap "
-                "volinfo %s", snap->snapname);
-                goto out;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol1_volid");
-        ret = dict_get_bin (dict, key, (void **)&snap_volid);
+        ret = dict_get_uint64(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, &opt_hard_max);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch snap_volid");
-                goto out;
+            ret = 0;
+            gf_msg_debug(this->name, 0,
+                         "%s is not present "
+                         "in opts dictionary",
+                         GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
         }
 
-        /* Adding snap bricks mount paths to the dict */
-        ret = glusterd_snap_create_clone_common_prevalidate (rsp_dict, 0,
-                                                             snapname, err_str,
-                                                             clonename, 1,
-                                                             snap_vol,
-                                                             &loglevel,
-                                                             1, op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_PRE_VALIDATION_FAIL, "Failed to pre validate");
-                goto out;
-        }
-
-        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set volcount");
-                goto out;
-        }
-
-out:
+        if (volinfo->snap_max_hard_limit < opt_hard_max)
+            effective_max_limit = volinfo->snap_max_hard_limit;
+        else
+            effective_max_limit = opt_hard_max;
 
-        if (ret && err_str[0] != '\0') {
-                gf_msg (this->name, loglevel, 0,
-                        GD_MSG_SNAP_CLONE_PREVAL_FAILED, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
+        if (volinfo->snap_count >= effective_max_limit) {
+            ret = -1;
+            snprintf(err_str, sizeof(err_str),
+                     "The number of existing snaps has reached "
+                     "the effective maximum limit of %" PRIu64
+                     ", "
+                     "for the volume (%s). Please delete few "
+                     "snapshots before taking further snapshots.",
+                     effective_max_limit, volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_HRDLMT;
+            goto out;
         }
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
-}
-
-
-int
-glusterd_snapshot_create_prevalidate (dict_t *dict, char **op_errstr,
-                                      dict_t *rsp_dict, uint32_t *op_errno)
-{
-        char                  *volname           = NULL;
-        char                  *snapname          = NULL;
-        char                   key[PATH_MAX]     = "";
-        char                   snap_volname[64]  = "";
-        char                   err_str[PATH_MAX] = "";
-        int                    ret               = -1;
-        int64_t                i                 = 0;
-        int64_t                volcount          = 0;
-        glusterd_volinfo_t    *volinfo           = NULL;
-        xlator_t              *this              = NULL;
-        uuid_t                *snap_volid        = NULL;
-        gf_loglevel_t          loglevel          = GF_LOG_ERROR;
-        glusterd_conf_t       *conf              = NULL;
-        int64_t                effective_max_limit = 0;
-        int                    flags             = 0;
-        uint64_t               opt_hard_max      = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-        GF_ASSERT (op_errstr);
-        conf = this->private;
-        GF_ASSERT (conf);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
+        snprintf(key, sizeof(key), "vol%" PRId64 "_volid", i);
+        ret = dict_get_bin(dict, key, (void **)&snap_volid);
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to "
-                          "get the volume count");
-                goto out;
-        }
-        if (volcount <= 0) {
-                snprintf (err_str, sizeof (err_str),
-                          "Invalid volume count %"PRId64" supplied", volcount);
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch snap_volid");
+            goto out;
         }
 
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get snapname");
-                goto out;
-        }
+        /* snap volume uuid is used as lvm snapshot name.
+           This will avoid restrictions on snapshot names
+           provided by user */
+        GLUSTERD_GET_UUID_NOHYPHEN(snap_volname, *snap_volid);
 
-        ret = dict_get_int32 (dict, "flags", &flags);
+        ret = glusterd_snap_create_clone_common_prevalidate(
+            rsp_dict, flags, snapname, err_str, snap_volname, i, volinfo,
+            &loglevel, 0, op_errno);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get flags");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                   "Failed to pre validate");
+            goto out;
         }
+    }
 
-        if (glusterd_find_snap_by_name (snapname)) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Snapshot %s already "
-                          "exists", snapname);
-                *op_errno = EG_SNAPEXST;
-                goto out;
-        }
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%"PRId64, i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "failed to get volume name");
-                        goto out;
-                }
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume (%s) does not exist ", volname);
-                        *op_errno = EG_NOVOL;
-                        goto out;
-                }
-
-                ret = -1;
-                if (!glusterd_is_volume_started (volinfo)) {
-                        snprintf (err_str, sizeof (err_str), "volume %s is "
-                                  "not started", volinfo->volname);
-                        loglevel = GF_LOG_WARNING;
-                        *op_errno = EG_VOLSTP;
-                        goto out;
-                }
-
-                if (glusterd_is_defrag_on (volinfo)) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "rebalance process is running for the "
-                                  "volume %s", volname);
-                        loglevel = GF_LOG_WARNING;
-                        *op_errno = EG_RBALRUN;
-                        goto out;
-                }
-
-                if (gd_vol_is_geo_rep_active (volinfo)) {
-                         snprintf (err_str, sizeof (err_str),
-                                   "geo-replication session is running for "
-                                   "the volume %s. Session needs to be "
-                                   "stopped before taking a snapshot.",
-                                   volname);
-                         loglevel = GF_LOG_WARNING;
-                         *op_errno = EG_GEOREPRUN;
-                         goto out;
-                }
-
-                if (volinfo->is_snap_volume == _gf_true) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume %s is a snap volume", volname);
-                        loglevel = GF_LOG_WARNING;
-                         *op_errno = EG_ISSNAP;
-                        goto out;
-                }
-
-                /* "snap-max-hard-limit" might not be set by user explicitly,
-                 * in that case it's better to consider the default value.
-                 * Hence not erroring out if Key is not found.
-                 */
-                ret = dict_get_uint64 (conf->opts,
-                                      GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                      &opt_hard_max);
-                if (ret) {
-                        ret = 0;
-                        gf_msg_debug (this->name, 0, "%s is not present "
-                                "in opts dictionary",
-                                GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-                }
-
-                if (volinfo->snap_max_hard_limit < opt_hard_max)
-                        effective_max_limit = volinfo->snap_max_hard_limit;
-                else
-                        effective_max_limit = opt_hard_max;
-
-                if (volinfo->snap_count >= effective_max_limit) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str),
-                                  "The number of existing snaps has reached "
-                                  "the effective maximum limit of %"PRIu64", "
-                                  "for the volume (%s). Please delete few "
-                                  "snapshots before taking further snapshots.",
-                                  effective_max_limit, volname);
-                        loglevel = GF_LOG_WARNING;
-                        *op_errno = EG_HRDLMT;
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key) - 1, "vol%"PRId64"_volid", i);
-                ret = dict_get_bin (dict, key, (void **)&snap_volid);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to fetch snap_volid");
-                        goto out;
-                }
-
-                /* snap volume uuid is used as lvm snapshot name.
-                   This will avoid restrictions on snapshot names
-                   provided by user */
-                GLUSTERD_GET_UUID_NOHYPHEN (snap_volname, *snap_volid);
-
-                ret = glusterd_snap_create_clone_common_prevalidate (rsp_dict,
-                                                                        flags,
-                                                                     snapname,
-                                                                      err_str,
-                                                                 snap_volname,
-                                                                            i,
-                                                                      volinfo,
-                                                                    &loglevel,
-                                                                 0, op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_PRE_VALIDATION_FAIL,
-                                "Failed to pre validate");
-                        goto out;
-                }
-
-                ret = glusterd_snapshot_pause_tier (this, volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_PAUSE_TIER_FAIL,
-                                "Failed to pause tier in snap prevalidate.");
-                        goto out;
-                }
-
-        }
-
-        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set volcount");
-                goto out;
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret && err_str[0] != '\0') {
-                gf_msg (this->name, loglevel, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
-
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && err_str[0] != '\0') {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-glusterd_snap_t*
+glusterd_snap_t *
 glusterd_new_snap_object()
 {
-        glusterd_snap_t        *snap    = NULL;
+    glusterd_snap_t *snap = NULL;
 
-        snap = GF_CALLOC (1, sizeof (*snap), gf_gld_mt_snap_t);
+    snap = GF_CALLOC(1, sizeof(*snap), gf_gld_mt_snap_t);
 
-        if (snap) {
-                if (LOCK_INIT (&snap->lock)) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_LOCK_INIT_FAILED, "Failed initiating"
-                                " snap lock");
-                        GF_FREE (snap);
-                        return NULL;
-                }
-
-                CDS_INIT_LIST_HEAD (&snap->snap_list);
-                CDS_INIT_LIST_HEAD (&snap->volumes);
-                snap->snapname[0] = 0;
-                snap->snap_status = GD_SNAP_STATUS_INIT;
+    if (snap) {
+        if (LOCK_INIT(&snap->lock)) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_LOCK_INIT_FAILED,
+                   "Failed initiating"
+                   " snap lock");
+            GF_FREE(snap);
+            return NULL;
         }
 
-        return snap;
+        CDS_INIT_LIST_HEAD(&snap->snap_list);
+        CDS_INIT_LIST_HEAD(&snap->volumes);
+        snap->snapname[0] = 0;
+        snap->snap_status = GD_SNAP_STATUS_INIT;
+    }
 
+    return snap;
 };
 
 /* Function glusterd_list_add_snapvol adds the volinfo object (snapshot volume)
    to the snapshot object list and to the parent volume list */
 int32_t
-glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
-                           glusterd_volinfo_t *snap_vol)
+glusterd_list_add_snapvol(glusterd_volinfo_t *origin_vol,
+                          glusterd_volinfo_t *snap_vol)
 {
-        int ret               = -1;
-        glusterd_snap_t *snap = NULL;
+    int ret = -1;
+    glusterd_snap_t *snap = NULL;
 
-        GF_VALIDATE_OR_GOTO ("glusterd", origin_vol, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", snap_vol, out);
+    GF_VALIDATE_OR_GOTO("glusterd", origin_vol, out);
+    GF_VALIDATE_OR_GOTO("glusterd", snap_vol, out);
 
-        snap = snap_vol->snapshot;
-        GF_ASSERT (snap);
+    snap = snap_vol->snapshot;
+    GF_ASSERT(snap);
 
-        cds_list_add_tail (&snap_vol->vol_list, &snap->volumes);
-        LOCK (&origin_vol->lock);
-        {
-                glusterd_list_add_order (&snap_vol->snapvol_list,
-                                         &origin_vol->snap_volumes,
-                                         glusterd_compare_snap_vol_time);
+    cds_list_add_tail(&snap_vol->vol_list, &snap->volumes);
+    LOCK(&origin_vol->lock);
+    {
+        glusterd_list_add_order(&snap_vol->snapvol_list,
+                                &origin_vol->snap_volumes,
+                                glusterd_compare_snap_vol_time);
 
-                origin_vol->snap_count++;
-        }
-        UNLOCK (&origin_vol->lock);
+        origin_vol->snap_count++;
+    }
+    UNLOCK(&origin_vol->lock);
 
-        gf_msg_debug (THIS->name, 0, "Snapshot %s added to the list",
-                snap->snapname);
-        ret = 0;
- out:
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Snapshot %s added to the list",
+                 snap->snapname);
+    ret = 0;
+out:
+    return ret;
 }
 
-glusterd_snap_t*
-glusterd_find_snap_by_name (char *snapname)
+glusterd_snap_t *
+glusterd_find_snap_by_name(char *snapname)
 {
-        glusterd_snap_t *snap  = NULL;
-        glusterd_conf_t *priv  = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snapname);
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                if (!strcmp (snap->snapname, snapname)) {
-                        gf_msg_debug (THIS->name, 0, "Found "
-                                "snap %s (%s)", snap->snapname,
-                                uuid_utoa (snap->snap_id));
-                        goto out;
-                }
-        }
-        snap = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snapname);
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        if (!strcmp(snap->snapname, snapname)) {
+            gf_msg_debug(THIS->name, 0,
+                         "Found "
+                         "snap %s (%s)",
+                         snap->snapname, uuid_utoa(snap->snap_id));
+            goto out;
+        }
+    }
+    snap = NULL;
 out:
-        return snap;
+    return snap;
 }
 
-glusterd_snap_t*
-glusterd_find_snap_by_id (uuid_t snap_id)
+glusterd_snap_t *
+glusterd_find_snap_by_id(uuid_t snap_id)
 {
-        glusterd_snap_t *snap  = NULL;
-        glusterd_conf_t *priv  = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        if (gf_uuid_is_null(snap_id))
-                goto out;
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                if (!gf_uuid_compare (snap->snap_id, snap_id)) {
-                        gf_msg_debug (THIS->name, 0, "Found "
-                                "snap %s (%s)", snap->snapname,
-                                uuid_utoa (snap->snap_id));
-                        goto out;
-                }
-        }
-        snap = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    if (gf_uuid_is_null(snap_id))
+        goto out;
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        if (!gf_uuid_compare(snap->snap_id, snap_id)) {
+            gf_msg_debug(THIS->name, 0,
+                         "Found "
+                         "snap %s (%s)",
+                         snap->snapname, uuid_utoa(snap->snap_id));
+            goto out;
+        }
+    }
+    snap = NULL;
 out:
-        return snap;
+    return snap;
 }
 
 int
-glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol,
-                                 glusterd_brickinfo_t *brickinfo,
-                                 const char *mount_pt, const char *snap_device)
+glusterd_do_lvm_snapshot_remove(glusterd_volinfo_t *snap_vol,
+                                glusterd_brickinfo_t *brickinfo,
+                                const char *mount_pt, const char *snap_device)
 {
-        int                     ret               = -1;
-        xlator_t               *this              = NULL;
-        glusterd_conf_t        *priv              = NULL;
-        runner_t                runner            = {0,};
-        char                    msg[1024]         = {0, };
-        char                    pidfile[PATH_MAX] = {0, };
-        pid_t                   pid               = -1;
-        int                     retry_count       = 0;
-        char                   *mnt_pt            = NULL;
-        struct mntent          *entry             = NULL;
-        gf_boolean_t            unmount           = _gf_true;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!brickinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "brickinfo NULL");
-                goto out;
-        }
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (mount_pt);
-        GF_ASSERT (snap_device);
-
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv);
-        if (gf_is_service_running (pidfile, &pid)) {
-                ret = kill (pid, SIGKILL);
-                if (ret && errno != ESRCH) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_PID_KILL_FAIL, "Unable to kill pid "
-                                "%d reason : %s", pid, strerror(errno));
-                        goto out;
-                }
-        }
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    char msg[1024] = "";
+    char pidfile[PATH_MAX] = "";
+    pid_t pid = -1;
+    int retry_count = 0;
+    char *mnt_pt = NULL;
+    gf_boolean_t unmount = _gf_true;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!brickinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "brickinfo NULL");
+        goto out;
+    }
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(mount_pt);
+    GF_ASSERT(snap_device);
+
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, snap_vol, brickinfo, priv);
+    if (gf_is_service_running(pidfile, &pid)) {
+        (void)send_attach_req(this, brickinfo->rpc, brickinfo->path, NULL, NULL,
+                              GLUSTERD_BRICK_TERMINATE);
+        brickinfo->status = GF_BRICK_STOPPED;
+    }
+
+    /* Check if the brick is mounted and then try unmounting the brick */
+    ret = glusterd_get_brick_root(brickinfo->path, &mnt_pt);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_PATH_UNMOUNTED,
+               "Getting the root "
+               "of the brick for volume %s (snap %s) failed. "
+               "Removing lv (%s).",
+               snap_vol->volname, snap_vol->snapshot->snapname, snap_device);
+        /* The brick path is already unmounted. Remove the lv only *
+         * Need not fail the operation */
+        ret = 0;
+        unmount = _gf_false;
+    }
+
+    if ((unmount == _gf_true) && (strcmp(mnt_pt, mount_pt))) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_PATH_UNMOUNTED,
+               "Lvm is not mounted for brick %s:%s. "
+               "Removing lv (%s).",
+               brickinfo->hostname, brickinfo->path, snap_device);
+        /* The brick path is already unmounted. Remove the lv only *
+         * Need not fail the operation */
+        unmount = _gf_false;
+    }
+
+    /* umount cannot be done when the brick process is still in the process
+       of shutdown, so give three re-tries */
+    while ((unmount == _gf_true) && (retry_count < 3)) {
+        retry_count++;
+        /*umount2 system call doesn't cleanup mtab entry after un-mount.
+          So use external umount command*/
+        ret = glusterd_umount(mount_pt);
+        if (!ret)
+            break;
+
+        gf_msg_debug(this->name, 0,
+                     "umount failed for "
+                     "path %s (brick: %s): %s. Retry(%d)",
+                     mount_pt, brickinfo->path, strerror(errno), retry_count);
 
-        /* Check if the brick is mounted and then try unmounting the brick */
-        ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_BRICK_PATH_UNMOUNTED, "Getting the root "
-                        "of the brick for volume %s (snap %s) failed. "
-                        "Removing lv (%s).", snap_vol->volname,
-                         snap_vol->snapshot->snapname, snap_device);
-                /* The brick path is already unmounted. Remove the lv only *
-                 * Need not fail the operation */
-                ret = 0;
-                unmount = _gf_false;
-        }
+        /*
+         * This used to be one second, but that wasn't long enough
+         * to get past the spurious EPERM errors that prevent some
+         * tests (especially bug-1162462.t) from passing reliably.
+         *
+         * TBD: figure out where that garbage is coming from
+         */
+        sleep(3);
+    }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNOUNT_FAILED,
+               "umount failed for "
+               "path %s (brick: %s): %s.",
+               mount_pt, brickinfo->path, strerror(errno));
+        /*
+         * This is cheating, but necessary until we figure out how to
+         * shut down a brick within a still-living brick daemon so that
+         * random translators aren't keeping the mountpoint alive.
+         *
+         * TBD: figure out a real solution
+         */
+        ret = 0;
+        goto out;
+    }
+
+    runinit(&runner);
+    len = snprintf(msg, sizeof(msg),
+                   "remove snapshot of the brick %s:%s, "
+                   "device: %s",
+                   brickinfo->hostname, brickinfo->path, snap_device);
+    if (len < 0) {
+        strcpy(msg, "<error>");
+    }
+    runner_add_args(&runner, LVM_REMOVE, "-f", snap_device, NULL);
+    runner_log(&runner, "", GF_LOG_DEBUG, msg);
+
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "removing snapshot of the "
+               "brick (%s:%s) of device %s failed",
+               brickinfo->hostname, brickinfo->path, snap_device);
+        goto out;
+    }
 
-        if ((unmount == _gf_true) && (strcmp (mnt_pt, mount_pt))) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_BRICK_PATH_UNMOUNTED,
-                        "Lvm is not mounted for brick %s:%s. "
-                        "Removing lv (%s).", brickinfo->hostname,
-                        brickinfo->path, snap_device);
-                /* The brick path is already unmounted. Remove the lv only *
-                 * Need not fail the operation */
-                unmount = _gf_false;
-        }
+out:
+    if (mnt_pt)
+        GF_FREE(mnt_pt);
 
-        /* umount cannot be done when the brick process is still in the process
-           of shutdown, so give three re-tries */
-        while ((unmount == _gf_true) && (retry_count < 3)) {
-                retry_count++;
-                /*umount2 system call doesn't cleanup mtab entry after un-mount.
-                  So use external umount command*/
-                ret = glusterd_umount(mount_pt);
-                if (!ret)
-                        break;
+    return ret;
+}
 
-                gf_msg_debug (this->name, 0, "umount failed for "
-                        "path %s (brick: %s): %s. Retry(%d)", mount_pt,
-                        brickinfo->path, strerror (errno), retry_count);
+int32_t
+glusterd_lvm_snapshot_remove(dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
+{
+    int32_t brick_count = -1;
+    int32_t ret = -1;
+    int32_t err = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    char brick_dir[PATH_MAX] = "";
+    char snap_path[PATH_MAX] = "";
+    char *tmp = NULL;
+    char *brick_mount_path = NULL;
+    gf_boolean_t is_brick_dir_present = _gf_false;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_vol);
+
+    if ((snap_vol->is_snap_volume == _gf_false) &&
+        (gf_uuid_is_null(snap_vol->restored_from_snap))) {
+        gf_msg_debug(this->name, 0,
+                     "Not a snap volume, or a restored snap volume.");
+        ret = 0;
+        goto out;
+    }
 
-                sleep (1);
+    brick_count = -1;
+    cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+    {
+        brick_count++;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            gf_msg_debug(this->name, 0, "%s:%s belongs to a different node",
+                         brickinfo->hostname, brickinfo->path);
+            continue;
         }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UNOUNT_FAILED, "umount failed for "
-                        "path %s (brick: %s): %s.", mount_pt,
-                        brickinfo->path, strerror (errno));
-                goto out;
-        }
-
-        runinit (&runner);
-        snprintf (msg, sizeof(msg), "remove snapshot of the brick %s:%s, "
-                  "device: %s", brickinfo->hostname, brickinfo->path,
-                  snap_device);
-        runner_add_args (&runner, LVM_REMOVE, "-f", snap_device, NULL);
-        runner_log (&runner, "", GF_LOG_DEBUG, msg);
 
-        ret = runner_run (&runner);
+        /* Fetch the brick mount path from the brickinfo->path */
+        ret = glusterd_find_brick_mount_path(brickinfo->path,
+                                             &brick_mount_path);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_REMOVE_FAIL, "removing snapshot of the "
-                        "brick (%s:%s) of device %s failed",
-                        brickinfo->hostname, brickinfo->path, snap_device);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+                   "Failed to find brick_mount_path for %s", brickinfo->path);
+            ret = 0;
+            continue;
         }
 
-out:
-        return ret;
-}
-
-int32_t
-glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
-{
-        int32_t               brick_count          = -1;
-        int32_t               ret                  = -1;
-        int32_t               err                  = 0;
-        glusterd_brickinfo_t *brickinfo            = NULL;
-        xlator_t             *this                 = NULL;
-        char                  buff[PATH_MAX]       = "";
-        char                  brick_dir[PATH_MAX]  = "";
-        char                 *tmp                  = NULL;
-        char                 *brick_mount_path     = NULL;
-        gf_boolean_t          is_brick_dir_present = _gf_false;
-        struct stat           stbuf                = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_vol);
-
-        if ((snap_vol->is_snap_volume == _gf_false) &&
-            (gf_uuid_is_null (snap_vol->restored_from_snap))) {
-                gf_msg_debug (this->name, 0,
-                        "Not a snap volume, or a restored snap volume.");
+        /* As deactivated snapshot have no active mount point we
+         * check only for activated snapshot.
+         */
+        if (snap_vol->status == GLUSTERD_STATUS_STARTED) {
+            ret = sys_lstat(brick_mount_path, &stbuf);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Brick %s:%s already deleted.",
+                             brickinfo->hostname, brickinfo->path);
                 ret = 0;
-                goto out;
+                continue;
+            }
         }
 
-        brick_count = -1;
-        cds_list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
-                brick_count++;
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                        gf_msg_debug (this->name, 0,
-                                "%s:%s belongs to a different node",
-                                brickinfo->hostname, brickinfo->path);
-                        continue;
-                }
-
-                /* Fetch the brick mount path from the brickinfo->path */
-                ret = glusterd_find_brick_mount_path (brickinfo->path,
-                                                      &brick_mount_path);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_GET_INFO_FAIL,
-                                "Failed to find brick_mount_path for %s",
-                                brickinfo->path);
-                        ret = 0;
-                        continue;
-                }
-
-                ret = sys_lstat (brick_mount_path, &stbuf);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "Brick %s:%s already deleted.",
-                                brickinfo->hostname, brickinfo->path);
-                        ret = 0;
-                        continue;
-                }
-
-                if (brickinfo->snap_status == -1) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_SNAPSHOT_PENDING,
-                                "snapshot was pending. lvm not present "
-                                "for brick %s:%s of the snap %s.",
-                                brickinfo->hostname, brickinfo->path,
-                                snap_vol->snapshot->snapname);
-
-                        if (rsp_dict &&
-                            (snap_vol->is_snap_volume == _gf_true)) {
-                                /* Adding missed delete to the dict */
-                                ret = glusterd_add_missed_snaps_to_dict
-                                                   (rsp_dict,
-                                                    snap_vol,
-                                                    brickinfo,
-                                                    brick_count + 1,
-                                                    GF_SNAP_OPTION_TYPE_DELETE);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                                                "Failed to add missed snapshot "
-                                                "info for %s:%s in the "
-                                                "rsp_dict", brickinfo->hostname,
-                                                brickinfo->path);
-                                        goto out;
-                                }
-                        }
-
-                        continue;
-                }
-
-                /* Check if the brick has a LV associated with it */
-                if (strlen(brickinfo->device_path) == 0) {
-                        gf_msg_debug (this->name, 0,
-                                "Brick (%s:%s) does not have a LV "
-                                "associated with it. Removing the brick path",
-                                brickinfo->hostname, brickinfo->path);
-                        goto remove_brick_path;
-                }
-
-                /* Verify if the device path exists or not */
-                ret = sys_stat (brickinfo->device_path, &stbuf);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                "LV (%s) for brick (%s:%s) not present. "
-                                "Removing the brick path",
-                                brickinfo->device_path,
-                                brickinfo->hostname, brickinfo->path);
-                        /* Making ret = 0 as absence of device path should *
-                         * not fail the remove operation */
-                        ret = 0;
-                        goto remove_brick_path;
-                }
+        if (brickinfo->snap_status == -1) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SNAPSHOT_PENDING,
+                   "snapshot was pending. lvm not present "
+                   "for brick %s:%s of the snap %s.",
+                   brickinfo->hostname, brickinfo->path,
+                   snap_vol->snapshot->snapname);
 
-                ret = glusterd_do_lvm_snapshot_remove (snap_vol, brickinfo,
-                                                       brick_mount_path,
-                                                       brickinfo->device_path);
+            if (rsp_dict && (snap_vol->is_snap_volume == _gf_true)) {
+                /* Adding missed delete to the dict */
+                ret = glusterd_add_missed_snaps_to_dict(
+                    rsp_dict, snap_vol, brickinfo, brick_count + 1,
+                    GF_SNAP_OPTION_TYPE_DELETE);
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL, "Failed to "
-                                "remove the snapshot %s (%s)",
-                                brickinfo->path, brickinfo->device_path);
-                        err = -1; /* We need to record this failure */
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                           "Failed to add missed snapshot "
+                           "info for %s:%s in the "
+                           "rsp_dict",
+                           brickinfo->hostname, brickinfo->path);
+                    goto out;
                 }
+            }
 
-remove_brick_path:
-                /* After removing the brick dir fetch the parent path
-                 * i.e /var/run/gluster/snaps/<snap-vol-id>/
-                 */
-                if (is_brick_dir_present == _gf_false) {
-                        /* Need to fetch brick_dir to be removed from
-                         * brickinfo->path, as in a restored volume,
-                         * snap_vol won't have the non-hyphenated snap_vol_id
-                         */
-                        tmp = strstr (brick_mount_path, "brick");
-                        if (!tmp) {
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_INVALID_ENTRY,
-                                        "Invalid brick %s", brickinfo->path);
-                                GF_FREE (brick_mount_path);
-                                brick_mount_path = NULL;
-                                continue;
-                        }
-
-                        strncpy (brick_dir, brick_mount_path,
-                                 (size_t) (tmp - brick_mount_path));
-
-                        /* Peers not hosting bricks will have _gf_false */
-                        is_brick_dir_present = _gf_true;
-                }
-
-                GF_FREE (brick_mount_path);
-                brick_mount_path = NULL;
+            continue;
         }
 
-        if (is_brick_dir_present == _gf_true) {
-                ret = recursive_rmdir (brick_dir);
-                if (ret) {
-                        if (errno == ENOTEMPTY) {
-                                /* Will occur when multiple glusterds
-                                 * are running in the same node
-                                 */
-                                gf_msg (this->name, GF_LOG_WARNING, errno,
-                                        GD_MSG_DIR_OP_FAILED,
-                                        "Failed to rmdir: %s, err: %s. "
-                                        "More than one glusterd running "
-                                        "on this node.",
-                                        brick_dir, strerror (errno));
-                                ret = 0;
-                                goto out;
-                        } else
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_DIR_OP_FAILED,
-                                        "Failed to rmdir: %s, err: %s",
-                                        brick_dir, strerror (errno));
-                                goto out;
-                }
+        /* Check if the brick has a LV associated with it */
+        if (strlen(brickinfo->device_path) == 0) {
+            gf_msg_debug(this->name, 0,
+                         "Brick (%s:%s) does not have a LV "
+                         "associated with it. Removing the brick path",
+                         brickinfo->hostname, brickinfo->path);
+            goto remove_brick_path;
         }
 
-        ret = 0;
-out:
-        if (err) {
-                ret = err;
+        /* Verify if the device path exists or not */
+        ret = sys_stat(brickinfo->device_path, &stbuf);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "LV (%s) for brick (%s:%s) not present. "
+                         "Removing the brick path",
+                         brickinfo->device_path, brickinfo->hostname,
+                         brickinfo->path);
+            /* Making ret = 0 as absence of device path should *
+             * not fail the remove operation */
+            ret = 0;
+            goto remove_brick_path;
         }
-        GF_FREE (brick_mount_path);
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
-}
 
+        ret = glusterd_do_lvm_snapshot_remove(
+            snap_vol, brickinfo, brick_mount_path, brickinfo->device_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to "
+                   "remove the snapshot %s (%s)",
+                   brickinfo->path, brickinfo->device_path);
+            err = -1; /* We need to record this failure */
+        }
 
-int32_t
-glusterd_snap_volume_remove (dict_t *rsp_dict,
-                             glusterd_volinfo_t *snap_vol,
-                             gf_boolean_t remove_lvm,
-                             gf_boolean_t force)
-{
-        int                   ret         = -1;
-        int                   save_ret    =  0;
-        glusterd_brickinfo_t *brickinfo   = NULL;
-        glusterd_volinfo_t   *origin_vol  = NULL;
-        xlator_t             *this        = NULL;
+    remove_brick_path:
+        /* After removing the brick dir fetch the parent path
+         * i.e /var/run/gluster/snaps/<snap-vol-id>/
+         */
+        if (is_brick_dir_present == _gf_false) {
+            /* Need to fetch brick_dir to be removed from
+             * brickinfo->path, as in a restored volume,
+             * snap_vol won't have the non-hyphenated snap_vol_id
+             */
+            tmp = strstr(brick_mount_path, "brick");
+            if (!tmp) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid brick %s", brickinfo->path);
+                GF_FREE(brick_mount_path);
+                brick_mount_path = NULL;
+                continue;
+            }
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap_vol);
+            strncpy(brick_dir, brick_mount_path,
+                    (size_t)(tmp - brick_mount_path));
 
-        if (!snap_vol) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "snap_vol in NULL");
-                ret = -1;
-                goto out;
+            /* Peers not hosting bricks will have _gf_false */
+            is_brick_dir_present = _gf_true;
         }
 
-        cds_list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
+        GF_FREE(brick_mount_path);
+        brick_mount_path = NULL;
+    }
 
-                ret = glusterd_brick_stop (snap_vol, brickinfo, _gf_false);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_WARNING, 0,
-                               GD_MSG_BRICK_STOP_FAIL, "Failed to stop "
-                               "brick for volume %s", snap_vol->volname);
-                        save_ret = ret;
-
-                        /* Don't clean up the snap on error when
-                           force flag is disabled */
-                        if (!force)
-                                goto out;
-                }
+    if (is_brick_dir_present == _gf_true) {
+        ret = recursive_rmdir(brick_dir);
+        if (ret) {
+            if (errno == ENOTEMPTY) {
+                /* Will occur when multiple glusterds
+                 * are running in the same node
+                 */
+                gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_DIR_OP_FAILED,
+                       "Failed to rmdir: %s, err: %s. "
+                       "More than one glusterd running "
+                       "on this node.",
+                       brick_dir, strerror(errno));
+                ret = 0;
+                goto out;
+            } else
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                       "Failed to rmdir: %s, err: %s", brick_dir,
+                       strerror(errno));
+            goto out;
         }
 
-        /* Only remove the backend lvm when required */
-        if (remove_lvm) {
-                ret = glusterd_lvm_snapshot_remove (rsp_dict, snap_vol);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_WARNING, 0,
-                               GD_MSG_SNAP_REMOVE_FAIL, "Failed to remove "
-                               "lvm snapshot volume %s", snap_vol->volname);
-                        save_ret = ret;
-                        if (!force)
-                                goto out;
-                }
+        /* After removing brick_dir, fetch and remove snap path
+         * i.e. /var/run/gluster/snaps/<snap-name>.
+         */
+        if (!snap_vol->snapshot) {
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "snapshot not"
+                   "present in snap_vol");
+            ret = -1;
+            goto out;
         }
 
-        ret = glusterd_store_delete_volume (snap_vol);
+        snprintf(snap_path, sizeof(snap_path), "%s/%s", snap_mount_dir,
+                 snap_vol->snapshot->snapname);
+        ret = recursive_rmdir(snap_path);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOL_DELETE_FAIL, "Failed to remove volume %s "
-                       "from store", snap_vol->volname);
-                save_ret = ret;
-                if (!force)
-                        goto out;
-        }
-
-        if (!cds_list_empty (&snap_vol->snapvol_list)) {
-                ret = glusterd_volinfo_find (snap_vol->parent_volname,
-                                             &origin_vol);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_FOUND, "Failed to get "
-                                "parent volinfo %s  for volume  %s",
-                                snap_vol->parent_volname, snap_vol->volname);
-                        save_ret = ret;
-                        if (!force)
-                                goto out;
-                }
-                origin_vol->snap_count--;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Failed to remove "
+                   "%s directory : error : %s",
+                   snap_path, strerror(errno));
+            goto out;
         }
+    }
 
-        glusterd_volinfo_unref (snap_vol);
-
-        if (save_ret)
-                ret = save_ret;
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "returning %d", ret);
-        return ret;
+    if (err) {
+        ret = err;
+    }
+    GF_FREE(brick_mount_path);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_snap_remove (dict_t *rsp_dict,
-                      glusterd_snap_t *snap,
-                      gf_boolean_t remove_lvm,
-                      gf_boolean_t force,
-                      gf_boolean_t is_clone)
+glusterd_snap_volume_remove(dict_t *rsp_dict, glusterd_volinfo_t *snap_vol,
+                            gf_boolean_t remove_lvm, gf_boolean_t force)
 {
-        int                 ret       = -1;
-        int                 save_ret  =  0;
-        glusterd_volinfo_t *snap_vol  = NULL;
-        glusterd_volinfo_t *tmp       = NULL;
-        xlator_t           *this      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap);
-
-        if (!snap) {
-                gf_msg(this->name, GF_LOG_WARNING, EINVAL,
-                       GD_MSG_INVALID_ENTRY, "snap is NULL");
-                ret = -1;
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
-                ret = glusterd_snap_volume_remove (rsp_dict, snap_vol,
-                                                   remove_lvm, force);
-                if (ret && !force) {
-                        /* Don't clean up the snap on error when
-                           force flag is disabled */
-                        gf_msg(this->name, GF_LOG_WARNING, 0,
-                               GD_MSG_SNAP_REMOVE_FAIL, "Failed to remove "
-                               "volinfo %s for snap %s", snap_vol->volname,
-                               snap->snapname);
-                        save_ret = ret;
-                        goto out;
-                }
-        }
-
-        /* A clone does not persist snap info in /var/lib/glusterd/snaps/ *
-         * and hence there is no snap info to be deleted from there       *
-         */
-        if (!is_clone) {
-                ret = glusterd_store_delete_snap (snap);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_WARNING, 0,
-                               GD_MSG_SNAP_REMOVE_FAIL,
-                               "Failed to remove snap %s from store",
-                               snap->snapname);
-                        save_ret = ret;
-                        if (!force)
-                                goto out;
-                }
-        }
-
-        ret = glusterd_snapobject_delete (snap);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_REMOVE_FAIL, "Failed to delete "
-                        "snap object %s", snap->snapname);
+    int ret = -1;
+    int save_ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap_vol);
+
+    if (!snap_vol) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+               "snap_vol in NULL");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        ret = glusterd_brick_stop(snap_vol, brickinfo, _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_STOP_FAIL,
+                   "Failed to stop "
+                   "brick for volume %s",
+                   snap_vol->volname);
+            save_ret = ret;
+
+            /* Don't clean up the snap on error when
+               force flag is disabled */
+            if (!force)
+                goto out;
+        }
+    }
+
+    /* Only remove the backend lvm when required */
+    if (remove_lvm) {
+        ret = glusterd_lvm_snapshot_remove(rsp_dict, snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to remove "
+                   "lvm snapshot volume %s",
+                   snap_vol->volname);
+            save_ret = ret;
+            if (!force)
+                goto out;
+        }
+    }
+
+    ret = glusterd_store_delete_volume(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+               "Failed to remove volume %s "
+               "from store",
+               snap_vol->volname);
+        save_ret = ret;
+        if (!force)
+            goto out;
+    }
+
+    if (!cds_list_empty(&snap_vol->snapvol_list)) {
+        ret = glusterd_volinfo_find(snap_vol->parent_volname, &origin_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "Failed to get "
+                   "parent volinfo %s  for volume  %s",
+                   snap_vol->parent_volname, snap_vol->volname);
+            save_ret = ret;
+            if (!force)
+                goto out;
+        }
+        origin_vol->snap_count--;
+    }
+
+    glusterd_volinfo_unref(snap_vol);
+
+    if (save_ret)
+        ret = save_ret;
+out:
+    gf_msg_trace(this->name, 0, "returning %d", ret);
+    return ret;
+}
 
-        if (save_ret)
-                ret = save_ret;
+int32_t
+glusterd_snap_remove(dict_t *rsp_dict, glusterd_snap_t *snap,
+                     gf_boolean_t remove_lvm, gf_boolean_t force,
+                     gf_boolean_t is_clone)
+{
+    int ret = -1;
+    int save_ret = 0;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap);
+
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+               "snap is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap_vol, tmp, &snap->volumes, vol_list)
+    {
+        ret = glusterd_snap_volume_remove(rsp_dict, snap_vol, remove_lvm,
+                                          force);
+        if (ret && !force) {
+            /* Don't clean up the snap on error when
+               force flag is disabled */
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to remove "
+                   "volinfo %s for snap %s",
+                   snap_vol->volname, snap->snapname);
+            save_ret = ret;
+            goto out;
+        }
+    }
+
+    /* A clone does not persist snap info in /var/lib/glusterd/snaps/ *
+     * and hence there is no snap info to be deleted from there       *
+     */
+    if (!is_clone) {
+        ret = glusterd_store_delete_snap(snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to remove snap %s from store", snap->snapname);
+            save_ret = ret;
+            if (!force)
+                goto out;
+        }
+    }
+
+    ret = glusterd_snapobject_delete(snap);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to delete "
+               "snap object %s",
+               snap->snapname);
+
+    if (save_ret)
+        ret = save_ret;
 out:
-        gf_msg_trace (THIS->name, 0, "returning %d", ret);
-        return ret;
+    gf_msg_trace(THIS->name, 0, "returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_snapshot_get_snapvol_detail (dict_t *dict,
-                                      glusterd_volinfo_t *snap_vol,
-                                      char *keyprefix, int detail)
+glusterd_snapshot_get_snapvol_detail(dict_t *dict, glusterd_volinfo_t *snap_vol,
+                                     const char *keyprefix, const int detail)
 {
-        int                 ret           = -1;
-        int                 snap_limit    = 0;
-        char                key[PATH_MAX] = {0,};
-        char               *value         = NULL;
-        glusterd_volinfo_t *origin_vol    = NULL;
-        glusterd_conf_t    *conf          = NULL;
-        xlator_t           *this          = NULL;
-        uint64_t           opt_hard_max   = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (keyprefix);
-
-        /* Volume Name */
-        value = gf_strdup (snap_vol->volname);
-        if (!value)
-                goto out;
-
-        snprintf (key, sizeof (key), "%s.volname", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "volume name in dictionary: %s", key);
-                goto out;
-        }
-
-        /* Volume ID */
-        value = gf_strdup (uuid_utoa (snap_vol->volume_id));
-        if (NULL == value) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.vol-id", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_MEMORY, "Failed to set "
-                        "volume id in dictionary: %s", key);
-                goto out;
-        }
-        value = NULL;
-
-        /* volume status */
-        snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
-        switch (snap_vol->status) {
+    int ret = -1;
+    int snap_limit = 0;
+    char key[64] = ""; /* keyprefix is quite small, up to 32 byts */
+    int keylen;
+    char *value = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(keyprefix);
+
+    /* Volume Name */
+    value = gf_strdup(snap_vol->volname);
+    if (!value)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.volname", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "volume name in dictionary: %s",
+               key);
+        goto out;
+    }
+
+    /* Volume ID */
+    value = gf_strdup(uuid_utoa(snap_vol->volume_id));
+    if (NULL == value) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.vol-id", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY,
+               "Failed to set "
+               "volume id in dictionary: %s",
+               key);
+        goto out;
+    }
+    value = NULL;
+
+    /* volume status */
+    keylen = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    switch (snap_vol->status) {
         case GLUSTERD_STATUS_STARTED:
-                ret = dict_set_str (dict, key, "Started");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Started", SLEN("Started"));
+            break;
         case GLUSTERD_STATUS_STOPPED:
-                ret = dict_set_str (dict, key, "Stopped");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Stopped", SLEN("Stopped"));
+            break;
         case GD_SNAP_STATUS_NONE:
-                ret = dict_set_str (dict, key, "None");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "None", SLEN("None"));
+            break;
         default:
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid volume status");
-                ret = -1;
-                goto out;
-        }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set volume status"
-                        " in dictionary: %s", key);
-                goto out;
-        }
-
-
-        ret = glusterd_volinfo_find (snap_vol->parent_volname, &origin_vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "failed to get the parent "
-                        "volinfo for the volume %s", snap_vol->volname);
-                goto out;
-        }
-
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_uint64 (conf->opts,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               &opt_hard_max);
-        if (ret) {
-                ret = 0;
-                gf_msg_debug (this->name, 0, "%s is not present in "
-                        "opts dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
-
-        if (opt_hard_max < origin_vol->snap_max_hard_limit) {
-                snap_limit = opt_hard_max;
-                gf_msg_debug (this->name, 0, "system snap-max-hard-limit is"
-                       " lesser than volume snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        } else {
-                snap_limit = origin_vol->snap_max_hard_limit;
-                gf_msg_debug (this->name, 0, "volume snap-max-hard-limit is"
-                       " lesser than system snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        }
-
-        snprintf (key, sizeof (key), "%s.snaps-available", keyprefix);
-        if (snap_limit > origin_vol->snap_count)
-                ret = dict_set_int32 (dict, key,
-                        snap_limit - origin_vol->snap_count);
-        else
-                ret = dict_set_int32 (dict, key, 0);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set available snaps");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snapcount", keyprefix);
-        ret = dict_set_int32 (dict, key, origin_vol->snap_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Could not save snapcount");
-                goto out;
-        }
-
-        if (!detail)
-                goto out;
-
-        /* Parent volume name */
-        value = gf_strdup (snap_vol->parent_volname);
-        if (!value)
-                goto out;
-
-        snprintf (key, sizeof (key), "%s.origin-volname", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set parent "
-                        "volume name in dictionary: %s", key);
-                goto out;
-        }
-        value = NULL;
-
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid volume status");
+            ret = -1;
+            goto out;
+    }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volume status"
+               " in dictionary: %s",
+               key);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(snap_vol->parent_volname, &origin_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "failed to get the parent "
+               "volinfo for the volume %s",
+               snap_vol->volname);
+        goto out;
+    }
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_uint64(conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          &opt_hard_max);
+    if (ret) {
         ret = 0;
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in "
+                     "opts dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    if (opt_hard_max < origin_vol->snap_max_hard_limit) {
+        snap_limit = opt_hard_max;
+        gf_msg_debug(this->name, 0,
+                     "system snap-max-hard-limit is"
+                     " lesser than volume snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    } else {
+        snap_limit = origin_vol->snap_max_hard_limit;
+        gf_msg_debug(this->name, 0,
+                     "volume snap-max-hard-limit is"
+                     " lesser than system snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snaps-available", keyprefix);
+    if (snap_limit > origin_vol->snap_count)
+        ret = dict_set_int32n(dict, key, keylen,
+                              snap_limit - origin_vol->snap_count);
+    else
+        ret = dict_set_int32(dict, key, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set available snaps");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snapcount", keyprefix);
+    ret = dict_set_int32n(dict, key, keylen, origin_vol->snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save snapcount");
+        goto out;
+    }
+
+    if (!detail)
+        goto out;
+
+    /* Parent volume name */
+    value = gf_strdup(snap_vol->parent_volname);
+    if (!value)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.origin-volname", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set parent "
+               "volume name in dictionary: %s",
+               key);
+        goto out;
+    }
+    value = NULL;
+
+    ret = 0;
 out:
-        if (value)
-                GF_FREE (value);
+    if (value)
+        GF_FREE(value);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_snapshot_get_snap_detail (dict_t *dict, glusterd_snap_t *snap,
-                                char *keyprefix, glusterd_volinfo_t *volinfo)
+glusterd_snapshot_get_snap_detail(dict_t *dict, glusterd_snap_t *snap,
+                                  const char *keyprefix,
+                                  glusterd_volinfo_t *volinfo)
 {
-        int                 ret           = -1;
-        int                 volcount      = 0;
-        char                key[PATH_MAX] = {0,};
-        char                timestr[64]   = {0,};
-        char               *value         = NULL;
-        glusterd_volinfo_t *snap_vol      = NULL;
-        glusterd_volinfo_t *tmp_vol       = NULL;
-        xlator_t           *this          = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (snap);
-        GF_ASSERT (keyprefix);
-
-        /* Snap Name */
-        value = gf_strdup (snap->snapname);
-        if (!value)
-                goto out;
-
-        snprintf (key, sizeof (key), "%s.snapname", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "snap name in dictionary");
-                goto out;
-        }
-
-        /* Snap ID */
-        value = gf_strdup (uuid_utoa (snap->snap_id));
-        if (NULL == value) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap-id", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "snap id in dictionary");
-                goto out;
-        }
-        value = NULL;
-
-        gf_time_fmt (timestr, sizeof timestr, snap->time_stamp,
-                     gf_timefmt_FT);
-        value = gf_strdup (timestr);
-
+    int ret = -1;
+    int volcount = 0;
+    char key[32] = ""; /* keyprefix is quite small, up to 16 bytes */
+    int keylen;
+    char timestr[GF_TIMESTR_SIZE] = "";
+    char *value = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(snap);
+    GF_ASSERT(keyprefix);
+
+    /* Snap Name */
+    value = gf_strdup(snap->snapname);
+    if (!value)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap name in dictionary");
+        goto out;
+    }
+
+    /* Snap ID */
+    value = gf_strdup(uuid_utoa(snap->snap_id));
+    if (NULL == value) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snap-id", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap id in dictionary");
+        goto out;
+    }
+    value = NULL;
+
+    gf_time_fmt(timestr, sizeof timestr, snap->time_stamp, gf_timefmt_FT);
+    value = gf_strdup(timestr);
+
+    if (NULL == value) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snap-time", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap time stamp in dictionary");
+        goto out;
+    }
+    value = NULL;
+
+    /* If snap description is provided then add that into dictionary */
+    if (NULL != snap->description) {
+        value = gf_strdup(snap->description);
         if (NULL == value) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
 
-        snprintf (key, sizeof (key), "%s.snap-time", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
+        keylen = snprintf(key, sizeof(key), "%s.snap-desc", keyprefix);
+        ret = dict_set_dynstrn(dict, key, keylen, value);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set "
-                        "snap time stamp in dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap description in dictionary");
+            goto out;
         }
         value = NULL;
+    }
 
-        /* If snap description is provided then add that into dictionary */
-        if (NULL != snap->description) {
-                value = gf_strdup (snap->description);
-                if (NULL == value) {
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key), "%s.snap-desc", keyprefix);
-                ret = dict_set_dynstr (dict, key, value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set "
-                                        "snap description in dictionary");
-                        goto out;
-                }
-                value = NULL;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap-status", keyprefix);
-        switch (snap->snap_status) {
+    keylen = snprintf(key, sizeof(key), "%s.snap-status", keyprefix);
+    switch (snap->snap_status) {
         case GD_SNAP_STATUS_INIT:
-                ret = dict_set_str (dict, key, "Init");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Init", SLEN("Init"));
+            break;
         case GD_SNAP_STATUS_IN_USE:
-                ret = dict_set_str (dict, key, "In-use");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "In-use", SLEN("In-use"));
+            break;
         case GD_SNAP_STATUS_DECOMMISSION:
-                ret = dict_set_str (dict, key, "Decommisioned");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Decommisioned",
+                                 SLEN("Decommisioned"));
+            break;
         case GD_SNAP_STATUS_RESTORED:
-                ret = dict_set_str (dict, key, "Restored");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Restored",
+                                 SLEN("Restored"));
+            break;
         case GD_SNAP_STATUS_NONE:
-                ret = dict_set_str (dict, key, "None");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "None", SLEN("None"));
+            break;
         default:
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid snap status");
-                ret = -1;
-                goto out;
-        }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap status "
-                        "in dictionary");
-                goto out;
-        }
-
-        if (volinfo) {
-                volcount = 1;
-                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
-                ret = glusterd_snapshot_get_snapvol_detail (dict,
-                                                        volinfo, key, 0);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_DICT_GET_FAILED, "Failed to "
-                                "get volume detail %s for snap %s",
-                                snap_vol->volname, snap->snapname);
-                        goto out;
-                }
-                goto done;
-        }
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &snap->volumes,
-                                      vol_list) {
-                volcount++;
-                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
-                ret = glusterd_snapshot_get_snapvol_detail (dict,
-                                                        snap_vol, key, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to "
-                                "get volume detail %s for snap %s",
-                                snap_vol->volname, snap->snapname);
-                        goto out;
-                }
-        }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid snap status");
+            ret = -1;
+            goto out;
+    }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap status "
+               "in dictionary");
+        goto out;
+    }
+
+    if (volinfo) {
+        volcount = 1;
+        snprintf(key, sizeof(key), "%s.vol%d", keyprefix, volcount);
+        ret = glusterd_snapshot_get_snapvol_detail(dict, volinfo, key, 0);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_DICT_GET_FAILED,
+                   "Failed to "
+                   "get volume detail %s for snap %s",
+                   snap_vol->volname, snap->snapname);
+            goto out;
+        }
+        goto done;
+    }
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &snap->volumes, vol_list)
+    {
+        volcount++;
+        snprintf(key, sizeof(key), "%s.vol%d", keyprefix, volcount);
+        ret = glusterd_snapshot_get_snapvol_detail(dict, snap_vol, key, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to "
+                   "get volume detail %s for snap %s",
+                   snap_vol->volname, snap->snapname);
+            goto out;
+        }
+    }
 
 done:
-        snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
-        ret = dict_set_int32 (dict, key, volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set %s",
-                        key);
-                goto out;
-        }
-
-        ret = 0;
+    keylen = snprintf(key, sizeof(key), "%s.vol-count", keyprefix);
+    ret = dict_set_int32n(dict, key, keylen, volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (value)
-                GF_FREE (value);
+    if (value)
+        GF_FREE(value);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_snapshot_get_all_snap_info (dict_t *dict)
+glusterd_snapshot_get_all_snap_info(dict_t *dict)
 {
-        int                 ret           = -1;
-        int                 snapcount     = 0;
-        char                key[PATH_MAX] = {0,};
-        glusterd_snap_t    *snap          = NULL;
-        glusterd_snap_t    *tmp_snap      = NULL;
-        glusterd_conf_t    *priv          = NULL;
-        xlator_t           *this          = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* General parameter validation */
-        GF_ASSERT (dict);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                snapcount++;
-                snprintf (key, sizeof (key), "snap%d", snapcount);
-                ret = glusterd_snapshot_get_snap_detail (dict, snap, key, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to get "
-                                "snapdetail for snap %s", snap->snapname);
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snapcount");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    int snapcount = 0;
+    char key[16] = "";
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* General parameter validation */
+    GF_ASSERT(dict);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        snapcount++;
+        snprintf(key, sizeof(key), "snap%d", snapcount);
+        ret = glusterd_snapshot_get_snap_detail(dict, snap, key, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "snapdetail for snap %s",
+                   snap->snapname);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_get_info_by_volume (dict_t *dict, char *volname,
-                                      char *err_str, size_t len)
+glusterd_snapshot_get_info_by_volume(dict_t *dict, char *volname, char *err_str,
+                                     size_t len)
 {
-        int                  ret           = -1;
-        int                  snapcount     = 0;
-        int                  snap_limit    = 0;
-        char                *value         = NULL;
-        char                 key[PATH_MAX] = "";
-        glusterd_volinfo_t  *volinfo       = NULL;
-        glusterd_volinfo_t  *snap_vol      = NULL;
-        glusterd_volinfo_t  *tmp_vol       = NULL;
-        glusterd_conf_t     *conf          = NULL;
-        xlator_t            *this          = NULL;
-        uint64_t            opt_hard_max   = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volname);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (err_str, len, "Volume (%s) does not exist", volname);
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "%s", err_str);
-                goto out;
-        }
-
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_uint64 (conf->opts,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               &opt_hard_max);
-        if (ret) {
-                ret = 0;
-                gf_msg_debug (this->name, 0, "%s is not present in "
-                        "opts dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
-
-        if (opt_hard_max < volinfo->snap_max_hard_limit) {
-                snap_limit = opt_hard_max;
-                gf_msg_debug (this->name, 0, "system snap-max-hard-limit is"
-                       " lesser than volume snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        } else {
-                snap_limit = volinfo->snap_max_hard_limit;
-                gf_msg_debug (this->name, 0, "volume snap-max-hard-limit is"
-                       " lesser than system snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        }
-
-        if (snap_limit > volinfo->snap_count)
-                ret = dict_set_int32 (dict, "snaps-available",
-                        snap_limit - volinfo->snap_count);
-        else
-                ret = dict_set_int32 (dict, "snaps-available", 0);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set available snaps");
-                goto out;
-        }
-
-        /* Origin volume name */
-        value = gf_strdup (volinfo->volname);
-        if (!value)
-                goto out;
-
-        ret = dict_set_dynstr (dict, "origin-volname", value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set parent "
-                        "volume name in dictionary: %s", key);
-                goto out;
-        }
-        value = NULL;
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
-                                      snapvol_list) {
-                snapcount++;
-                snprintf (key, sizeof (key), "snap%d", snapcount);
-                ret = glusterd_snapshot_get_snap_detail (dict,
-                                                         snap_vol->snapshot,
-                                                         key, snap_vol);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to get "
-                                "snapdetail for snap %s",
-                                snap_vol->snapshot->snapname);
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snapcount");
-                goto out;
-        }
-
+    int ret = -1;
+    int snapcount = 0;
+    int snap_limit = 0;
+    char *value = NULL;
+    char key[16] = "";
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volname);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, len, "Volume (%s) does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+               err_str);
+        goto out;
+    }
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_uint64(conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          &opt_hard_max);
+    if (ret) {
         ret = 0;
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in "
+                     "opts dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    if (opt_hard_max < volinfo->snap_max_hard_limit) {
+        snap_limit = opt_hard_max;
+        gf_msg_debug(this->name, 0,
+                     "system snap-max-hard-limit is"
+                     " lesser than volume snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    } else {
+        snap_limit = volinfo->snap_max_hard_limit;
+        gf_msg_debug(this->name, 0,
+                     "volume snap-max-hard-limit is"
+                     " lesser than system snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    }
+
+    if (snap_limit > volinfo->snap_count)
+        ret = dict_set_int32n(dict, "snaps-available", SLEN("snaps-available"),
+                              snap_limit - volinfo->snap_count);
+    else
+        ret = dict_set_int32n(dict, "snaps-available", SLEN("snaps-available"),
+                              0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set available snaps");
+        goto out;
+    }
+
+    /* Origin volume name */
+    value = gf_strdup(volinfo->volname);
+    if (!value)
+        goto out;
+
+    ret = dict_set_dynstrn(dict, "origin-volname", SLEN("origin-volname"),
+                           value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set parent "
+               "volume name in dictionary: %s",
+               value);
+        goto out;
+    }
+    value = NULL;
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        snapcount++;
+        snprintf(key, sizeof(key), "snap%d", snapcount);
+        ret = glusterd_snapshot_get_snap_detail(dict, snap_vol->snapshot, key,
+                                                snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "snapdetail for snap %s",
+                   snap_vol->snapshot->snapname);
+            goto out;
+        }
+    }
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (value)
-                GF_FREE (value);
+    if (value)
+        GF_FREE(value);
 
-        return ret;
+    return ret;
 }
 
 /* This function will be called from RPC handler routine.
@@ -3673,294 +3586,280 @@ out:
  * @return -1 on error and 0 on success
  */
 int
-glusterd_handle_snapshot_info (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_info(rpcsvc_request_t *req, glusterd_op_t op,
+                              dict_t *dict, char *err_str, size_t len)
 {
-        int              ret             =       -1;
-        int8_t          snap_driven      =        1;
-        char            *volname         =      NULL;
-        char            *snapname        =      NULL;
-        glusterd_snap_t *snap            =      NULL;
-        xlator_t        *this            =      NULL;
-        int32_t         cmd              =      GF_SNAP_INFO_TYPE_ALL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, req, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-
-        ret = dict_get_int32 (dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get type "
-                        "of snapshot info");
-                goto out;
-        }
-
-        switch (cmd) {
-                case GF_SNAP_INFO_TYPE_ALL:
-                {
-                        ret = glusterd_snapshot_get_all_snap_info (dict);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get info of all snaps");
-                                goto out;
-                        }
-                        break;
-                }
-
-                case GF_SNAP_INFO_TYPE_SNAP:
-                {
-                        ret = dict_get_str (dict, "snapname", &snapname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get snap name");
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (dict, "snapcount", 1);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set snapcount");
-                                goto out;
-                        }
-
-                        snap = glusterd_find_snap_by_name (snapname);
-                        if (!snap) {
-                                snprintf (err_str, len,
-                                          "Snapshot (%s) does not exist",
-                                          snapname);
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_SNAP_NOT_FOUND,
-                                        "%s", err_str);
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = glusterd_snapshot_get_snap_detail (dict, snap,
-                                                                "snap1", NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_NOT_FOUND,
-                                        "Failed to get snap detail of snap "
-                                        "%s", snap->snapname);
-                                goto out;
-                        }
-                        break;
-                }
-
-                case GF_SNAP_INFO_TYPE_VOL:
-                {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOL_NOT_FOUND,
-                                        "Failed to get volname");
-                                goto out;
-                        }
-                        ret = glusterd_snapshot_get_info_by_volume (dict,
-                                                        volname, err_str, len);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_VOL_NOT_FOUND,
-                                        "Failed to get volume info of volume "
-                                        "%s", volname);
-                                goto out;
-                        }
-                        snap_driven = 0;
-                        break;
-                }
-        }
-
-        ret = dict_set_int8 (dict, "snap-driven", snap_driven);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap-driven");
+    int ret = -1;
+    int8_t snap_driven = 1;
+    char *volname = NULL;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    int32_t cmd = GF_SNAP_INFO_TYPE_ALL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get type "
+               "of snapshot info");
+        goto out;
+    }
+
+    switch (cmd) {
+        case GF_SNAP_INFO_TYPE_ALL: {
+            ret = glusterd_snapshot_get_all_snap_info(dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get info of all snaps");
+                goto out;
+            }
+            break;
+        }
+
+        case GF_SNAP_INFO_TYPE_SNAP: {
+            ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get snap name");
+                goto out;
+            }
+
+            ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), 1);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set snapcount");
+                goto out;
+            }
+
+            snap = glusterd_find_snap_by_name(snapname);
+            if (!snap) {
+                snprintf(err_str, len, "Snapshot (%s) does not exist",
+                         snapname);
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+                       "%s", err_str);
+                ret = -1;
                 goto out;
-        }
-
-        /* If everything is successful then send the response back to cli.
-         * In case of failure the caller of this function will take care
-           of the response */
-        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_CLI_RESP, "Failed to send cli "
-                        "response");
+            }
+            ret = glusterd_snapshot_get_snap_detail(dict, snap, "snap1", NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                       "Failed to get snap detail of snap "
+                       "%s",
+                       snap->snapname);
                 goto out;
-        }
-
-        ret = 0;
+            }
+            break;
+        }
+
+        case GF_SNAP_INFO_TYPE_VOL: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                       "Failed to get volname");
+                goto out;
+            }
+            ret = glusterd_snapshot_get_info_by_volume(dict, volname, err_str,
+                                                       len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Failed to get volume info of volume "
+                       "%s",
+                       volname);
+                goto out;
+            }
+            snap_driven = 0;
+            break;
+        }
+    }
+
+    ret = dict_set_int8(dict, "snap-driven", snap_driven);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap-driven");
+        goto out;
+    }
+
+    /* If everything is successful then send the response back to cli.
+     * In case of failure the caller of this function will take care
+       of the response */
+    ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+               "Failed to send cli "
+               "response");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function sets all the snapshot names in the dictionary */
 int
-glusterd_snapshot_get_all_snapnames (dict_t *dict)
+glusterd_snapshot_get_all_snapnames(dict_t *dict)
 {
-        int              ret             = -1;
-        int              snapcount       = 0;
-        char            *snapname        = NULL;
-        char             key[PATH_MAX]   = {0,};
-        glusterd_snap_t *snap            = NULL;
-        glusterd_snap_t *tmp_snap        = NULL;
-        glusterd_conf_t *priv            = NULL;
-        xlator_t        *this            = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                snapcount++;
-                snapname = gf_strdup (snap->snapname);
-                if (!snapname) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY, "strdup failed");
-                        ret = -1;
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "snapname%d", snapcount);
-                ret = dict_set_dynstr (dict, key, snapname);
-                if (ret) {
-                        GF_FREE (snapname);
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set %s",
-                                key);
-                        goto out;
-                }
+    int ret = -1;
+    int snapcount = 0;
+    char *snapname = NULL;
+    char key[64] = "";
+    int keylen;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        snapcount++;
+        snapname = gf_strdup(snap->snapname);
+        if (!snapname) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "strdup failed");
+            ret = -1;
+            goto out;
         }
-
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
+        keylen = snprintf(key, sizeof(key), "snapname%d", snapcount);
+        ret = dict_set_dynstrn(dict, key, keylen, snapname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snapcount");
-                goto out;
+            GF_FREE(snapname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function sets all the snapshot names
    under a given volume in the dictionary */
 int
-glusterd_snapshot_get_vol_snapnames (dict_t *dict, glusterd_volinfo_t *volinfo)
+glusterd_snapshot_get_vol_snapnames(dict_t *dict, glusterd_volinfo_t *volinfo)
 {
-        int                 ret             = -1;
-        int                 snapcount       = 0;
-        char               *snapname        = NULL;
-        char                key[PATH_MAX]   = {0,};
-        glusterd_volinfo_t *snap_vol        = NULL;
-        glusterd_volinfo_t *tmp_vol         = NULL;
-        xlator_t           *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol,
-                                  &volinfo->snap_volumes, snapvol_list) {
-                snapcount++;
-                snprintf (key, sizeof (key), "snapname%d", snapcount);
-
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                                  snap_vol->snapshot->snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to "
-                                "set %s", key);
-                        GF_FREE (snapname);
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snapcount");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    int snapcount = 0;
+    char *snapname = NULL;
+    char key[32] = "";
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        snapcount++;
+        snprintf(key, sizeof(key), "snapname%d", snapcount);
+
+        ret = dict_set_dynstr_with_alloc(dict, key,
+                                         snap_vol->snapshot->snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set %s",
+                   key);
+            GF_FREE(snapname);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_snapshot_list (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len,
-                               uint32_t *op_errno)
+glusterd_handle_snapshot_list(rpcsvc_request_t *req, glusterd_op_t op,
+                              dict_t *dict, char *err_str, size_t len,
+                              uint32_t *op_errno)
 {
-        int                 ret             = -1;
-        char               *volname         = NULL;
-        glusterd_volinfo_t *volinfo         = NULL;
-        xlator_t           *this            = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        GF_VALIDATE_OR_GOTO (this->name, req, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
 
-        /* Ignore error for getting volname as it is optional */
-        ret = dict_get_str (dict, "volname", &volname);
+    /* Ignore error for getting volname as it is optional */
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-        if (NULL == volname) {
-                ret = glusterd_snapshot_get_all_snapnames (dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_SNAP_LIST_GET_FAIL,
-                                "Failed to get snapshot list");
-                        goto out;
-                }
-        } else {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, len,
-                                 "Volume (%s) does not exist", volname);
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_FOUND,
-                                "%s", err_str);
-                        *op_errno = EG_NOVOL;
-                        goto out;
-                }
-
-                ret = glusterd_snapshot_get_vol_snapnames (dict, volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_SNAP_LIST_GET_FAIL,
-                                "Failed to get snapshot list for volume %s",
-                                volname);
-                        goto out;
-                }
+    if (NULL == volname) {
+        ret = glusterd_snapshot_get_all_snapnames(dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_LIST_GET_FAIL,
+                   "Failed to get snapshot list");
+            goto out;
+        }
+    } else {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, len, "Volume (%s) does not exist", volname);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+                   err_str);
+            *op_errno = EG_NOVOL;
+            goto out;
         }
 
-        /* If everything is successful then send the response back to cli.
-        In case of failure the caller of this function will take of response.*/
-        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        ret = glusterd_snapshot_get_vol_snapnames(dict, volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_NO_CLI_RESP, "Failed to send cli "
-                                "response");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_LIST_GET_FAIL,
+                   "Failed to get snapshot list for volume %s", volname);
+            goto out;
         }
+    }
 
-        ret = 0;
+    /* If everything is successful then send the response back to cli.
+    In case of failure the caller of this function will take of response.*/
+    ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+               "Failed to send cli "
+               "response");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This is a snapshot create handler function. This function will be
@@ -3976,185 +3875,185 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_create (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_create(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, size_t len)
 {
-        int           ret                                  = -1;
-        char         *volname                              = NULL;
-        char         *snapname                             = NULL;
-        int64_t       volcount                             = 0;
-        xlator_t     *this                                 = NULL;
-        char          key[PATH_MAX]                        = "";
-        char         *username                             = NULL;
-        char         *password                             = NULL;
-        uuid_t       *uuid_ptr                             = NULL;
-        uuid_t        tmp_uuid                             = {0};
-        int           i                                    = 0;
-        gf_boolean_t  timestamp                            = _gf_false;
-        char          snap_volname[GD_VOLUME_NAME_MAX]     = {0, };
-        char          new_snapname[GLUSTERD_MAX_SNAP_NAME] = {0, };
-        char          gmt_snaptime[GLUSTERD_MAX_SNAP_NAME] = {0, };
-        time_t        snap_time;
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to "
-                        "get the volume count");
-                goto out;
-        }
-        if (volcount <= 0) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid volume count %"PRId64
-                        " supplied", volcount);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to get the snapname");
-                goto out;
-        }
-
-        timestamp = dict_get_str_boolean (dict, "no-timestamp", _gf_false);
-        if (timestamp == -1) {
-                 gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                         "no-timestamp flag ");
-                 goto out;
-        }
-
-        ret = dict_set_int64 (dict, "snap-time", (int64_t)time(&snap_time));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Unable to set snap-time");
-                goto out;
-        }
-
-        if (!timestamp) {
-                strftime (gmt_snaptime, sizeof (gmt_snaptime),
-                          "_GMT-%Y.%m.%d-%H.%M.%S", gmtime(&snap_time));
-                snprintf (new_snapname, sizeof (new_snapname), "%s%s",
-                          snapname, gmt_snaptime);
-                ret = dict_set_dynstr_with_alloc (dict, "snapname",
-                                                  new_snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Unable to update "
-                                "snap-name");
-                        goto out;
-                }
-                snapname = new_snapname;
-        }
-
-        if (strlen(snapname) >= GLUSTERD_MAX_SNAP_NAME) {
-                snprintf (err_str, len, "snapname cannot exceed 255 "
-                          "characters");
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+    int ret = -1;
+    char *volname = NULL;
+    char *snapname = NULL;
+    int64_t volcount = 0;
+    xlator_t *this = NULL;
+    char key[64] = "";
+    int keylen;
+    char *username = NULL;
+    char *password = NULL;
+    uuid_t *uuid_ptr = NULL;
+    uuid_t tmp_uuid = {0};
+    int i = 0;
+    int timestamp = 0;
+    char snap_volname[GD_VOLUME_NAME_MAX] = "";
+    char new_snapname[GLUSTERD_MAX_SNAP_NAME] = "";
+    char gmt_snaptime[GLUSTERD_MAX_SNAP_NAME] = "";
+    time_t snap_time;
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+    if (volcount <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid volume count %" PRId64 " supplied", volcount);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the snapname");
+        goto out;
+    }
+
+    timestamp = dict_get_str_boolean(dict, "no-timestamp", _gf_false);
+    if (timestamp == -1) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to get "
+               "no-timestamp flag ");
+        goto out;
+    }
+
+    snap_time = gf_time();
+    ret = dict_set_int64(dict, "snap-time", (int64_t)snap_time);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap-time");
+        goto out;
+    }
+
+    if (!timestamp) {
+        strftime(gmt_snaptime, sizeof(gmt_snaptime), "_GMT-%Y.%m.%d-%H.%M.%S",
+                 gmtime(&snap_time));
+        snprintf(new_snapname, sizeof(new_snapname), "%s%s", snapname,
+                 gmt_snaptime);
+        ret = dict_set_dynstr_with_alloc(dict, "snapname", new_snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to update "
+                   "snap-name");
+            goto out;
+        }
+        snapname = new_snapname;
+    }
+
+    if (strlen(snapname) >= GLUSTERD_MAX_SNAP_NAME) {
+        snprintf(err_str, len,
+                 "snapname cannot exceed 255 "
+                 "characters");
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!uuid_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out Of Memory");
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_generate(*uuid_ptr);
+    ret = dict_set_bin(dict, "snap-id", uuid_ptr, sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap-id");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+    uuid_ptr = NULL;
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get volume name");
+            goto out;
+        }
+
+        /* generate internal username and password  for the snap*/
+        gf_uuid_generate(tmp_uuid);
+        username = gf_strdup(uuid_utoa(tmp_uuid));
+        keylen = snprintf(key, sizeof(key), "volume%d_username", i);
+        ret = dict_set_dynstrn(dict, key, keylen, username);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "username for volume %s",
+                   volname);
+            GF_FREE(username);
+            goto out;
+        }
+
+        gf_uuid_generate(tmp_uuid);
+        password = gf_strdup(uuid_utoa(tmp_uuid));
+        keylen = snprintf(key, sizeof(key), "volume%d_password", i);
+        ret = dict_set_dynstrn(dict, key, keylen, password);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "password for volume %s",
+                   volname);
+            GF_FREE(password);
+            goto out;
+        }
+
+        uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
         if (!uuid_ptr) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out Of Memory");
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Out Of Memory");
+            ret = -1;
+            goto out;
         }
 
-        gf_uuid_generate (*uuid_ptr);
-        ret = dict_set_bin (dict, "snap-id", uuid_ptr, sizeof(uuid_t));
+        snprintf(key, sizeof(key), "vol%d_volid", i);
+        gf_uuid_generate(*uuid_ptr);
+        ret = dict_set_bin(dict, key, uuid_ptr, sizeof(uuid_t));
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Unable to set snap-id");
-                GF_FREE (uuid_ptr);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set snap_volid");
+            GF_FREE(uuid_ptr);
+            goto out;
         }
-        uuid_ptr = NULL;
-
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Failed to get volume name");
-                        goto out;
-                }
-
-                /* generate internal username and password  for the snap*/
-                gf_uuid_generate (tmp_uuid);
-                username = gf_strdup (uuid_utoa (tmp_uuid));
-                snprintf (key, sizeof(key), "volume%d_username", i);
-                ret = dict_set_dynstr (dict, key, username);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set snap "
-                                "username for volume %s", volname);
-                        GF_FREE (username);
-                        goto out;
-                }
-
-                gf_uuid_generate (tmp_uuid);
-                password = gf_strdup (uuid_utoa (tmp_uuid));
-                snprintf (key, sizeof(key), "volume%d_password", i);
-                ret = dict_set_dynstr (dict, key, password);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set snap "
-                                "password for volume %s", volname);
-                        GF_FREE (password);
-                        goto out;
-                }
-
-                uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-                if (!uuid_ptr) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY, "Out Of Memory");
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key) - 1, "vol%d_volid", i);
-                gf_uuid_generate (*uuid_ptr);
-                ret = dict_set_bin (dict, key, uuid_ptr, sizeof(uuid_t));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set snap_volid");
-                        GF_FREE (uuid_ptr);
-                        goto out;
-                }
-                GLUSTERD_GET_UUID_NOHYPHEN (snap_volname, *uuid_ptr);
-                snprintf (key, sizeof (key), "snap-volname%d", i);
-                ret = dict_set_dynstr_with_alloc (dict, key, snap_volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Unable to set snap volname");
-                        GF_FREE (uuid_ptr);
-                        goto out;
-                }
-        }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        GLUSTERD_GET_UUID_NOHYPHEN(snap_volname, *uuid_ptr);
+        snprintf(key, sizeof(key), "snap-volname%d", i);
+        ret = dict_set_dynstr_with_alloc(dict, key, snap_volname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_INIT_FAIL,
-                                "Failed to initiate snap "
-                                "phases");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set snap volname");
+            GF_FREE(uuid_ptr);
+            goto out;
         }
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate snap "
+               "phases");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This is a snapshot status handler function. This function will be
@@ -4173,31 +4072,30 @@ out:
  *
  */
 int
-glusterd_handle_snapshot_status (rpcsvc_request_t *req, glusterd_op_t op,
-                                 dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_status(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, size_t len)
 {
-        int                     ret             =       -1;
-        xlator_t                *this           =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
+    this = THIS;
+    GF_ASSERT(this);
 
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
 
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_INIT_FAIL, "Failed to initiate "
-                        "snap phases");
-                goto out;
-        }
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate "
+               "snap phases");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This is a snapshot clone handler function. This function will be
@@ -4213,147 +4111,143 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_clone (rpcsvc_request_t *req, glusterd_op_t op,
+glusterd_handle_snapshot_clone(rpcsvc_request_t *req, glusterd_op_t op,
                                dict_t *dict, char *err_str, size_t len)
 {
-        int           ret                              = -1;
-        char         *clonename                        = NULL;
-        char         *snapname                         = NULL;
-        int64_t       volcount                         = 0;
-        xlator_t     *this                             = NULL;
-        char          key[PATH_MAX]                    = "";
-        char         *username                         = NULL;
-        char         *password                         = NULL;
-        char         *volname                          = NULL;
-        uuid_t       *uuid_ptr                         = NULL;
-        uuid_t        tmp_uuid                         = {0};
-        int           i                                = 0;
-        char          snap_volname[GD_VOLUME_NAME_MAX] = {0, };
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_str (dict, "clonename", &clonename);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to "
-                        "get the clone name");
-                goto out;
-        }
-        /*We need to take a volume lock on clone name*/
-        volname = gf_strdup (clonename);
-        snprintf (key, sizeof(key), "volname1");
-        ret = dict_set_dynstr (dict, key, volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set clone "
-                        "name for volume locking");
-                GF_FREE (volname);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to get the snapname");
-                goto out;
-        }
-
-        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!uuid_ptr) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out Of Memory");
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_generate (*uuid_ptr);
-        ret = dict_set_bin (dict, "clone-id", uuid_ptr, sizeof(uuid_t));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Unable to set clone-id");
-                GF_FREE (uuid_ptr);
-                goto out;
-        }
-        uuid_ptr = NULL;
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to get snapname name");
-                goto out;
-        }
-
-        gf_uuid_generate (tmp_uuid);
-        username = gf_strdup (uuid_utoa (tmp_uuid));
-        snprintf (key, sizeof(key), "volume1_username");
-        ret = dict_set_dynstr (dict, key, username);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set clone "
-                        "username for volume %s", clonename);
-                GF_FREE (username);
-                goto out;
-        }
-
-        gf_uuid_generate (tmp_uuid);
-        password = gf_strdup (uuid_utoa (tmp_uuid));
-        snprintf (key, sizeof(key), "volume1_password");
-        ret = dict_set_dynstr (dict, key, password);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set clone "
-                        "password for volume %s", clonename);
-                GF_FREE (password);
-                goto out;
-        }
-
-        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!uuid_ptr) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out Of Memory");
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol1_volid");
-        gf_uuid_generate (*uuid_ptr);
-        ret = dict_set_bin (dict, key, uuid_ptr, sizeof(uuid_t));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set clone_volid");
-                GF_FREE (uuid_ptr);
-                goto out;
-        }
-        snprintf (key, sizeof (key), "clone-volname%d", i);
-        ret = dict_set_dynstr_with_alloc (dict, key, snap_volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set snap volname");
-                GF_FREE (uuid_ptr);
-                goto out;
-        }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_INIT_FAIL, "Failed to initiate "
-                        "snap phases");
-        }
+    int ret = -1;
+    char *clonename = NULL;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    char key[64] = "";
+    int keylen;
+    char *username = NULL;
+    char *password = NULL;
+    char *volname = NULL;
+    uuid_t *uuid_ptr = NULL;
+    uuid_t tmp_uuid = {0};
+    int i = 0;
+    char snap_volname[GD_VOLUME_NAME_MAX] = "";
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the clone name");
+        goto out;
+    }
+    /*We need to take a volume lock on clone name*/
+    volname = gf_strdup(clonename);
+    keylen = snprintf(key, sizeof(key), "volname1");
+    ret = dict_set_dynstrn(dict, key, keylen, volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set clone "
+               "name for volume locking");
+        GF_FREE(volname);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the snapname");
+        goto out;
+    }
+
+    uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!uuid_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out Of Memory");
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_generate(*uuid_ptr);
+    ret = dict_set_bin(dict, "clone-id", uuid_ptr, sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set clone-id");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+    uuid_ptr = NULL;
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get snapname name");
+        goto out;
+    }
+
+    gf_uuid_generate(tmp_uuid);
+    username = gf_strdup(uuid_utoa(tmp_uuid));
+    keylen = snprintf(key, sizeof(key), "volume1_username");
+    ret = dict_set_dynstrn(dict, key, keylen, username);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set clone "
+               "username for volume %s",
+               clonename);
+        GF_FREE(username);
+        goto out;
+    }
+
+    gf_uuid_generate(tmp_uuid);
+    password = gf_strdup(uuid_utoa(tmp_uuid));
+    keylen = snprintf(key, sizeof(key), "volume1_password");
+    ret = dict_set_dynstrn(dict, key, keylen, password);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set clone "
+               "password for volume %s",
+               clonename);
+        GF_FREE(password);
+        goto out;
+    }
+
+    uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!uuid_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out Of Memory");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "vol1_volid");
+    gf_uuid_generate(*uuid_ptr);
+    ret = dict_set_bin(dict, key, uuid_ptr, sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set clone_volid");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+    snprintf(key, sizeof(key), "clone-volname%d", i);
+    ret = dict_set_dynstr_with_alloc(dict, key, snap_volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap volname");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate "
+               "snap phases");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 /* This is a snapshot restore handler function. This function will be
  * executed in the originator node. This function is responsible for
  * calling mgmt_v3 framework to do the actual restore on all the bricks
@@ -4367,282 +4261,284 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_restore (rpcsvc_request_t *req, glusterd_op_t op,
-                                  dict_t *dict, char *err_str,
-                                  uint32_t *op_errno, size_t len)
+glusterd_handle_snapshot_restore(rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str,
+                                 uint32_t *op_errno, size_t len)
 {
-        int                     ret             = -1;
-        char                    *snapname       = NULL;
-        char                    *buf            = NULL;
-        glusterd_conf_t         *conf           = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        int32_t                 i               =    0;
-        char                    key[PATH_MAX]   =    "";
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-
-        GF_ASSERT (conf);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to "
-                        "get snapname");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                snprintf (err_str, len, "Snapshot (%s) does not exist",
-                          snapname);
-                *op_errno = EG_NOSNAP;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        list_for_each_entry (snap_volinfo, &snap->volumes, vol_list) {
-                i++;
-                snprintf (key, sizeof (key), "volname%d", i);
-                buf = gf_strdup (snap_volinfo->parent_volname);
-                if (!buf) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, key, buf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Could not set "
-                                "parent volume name %s in the dict",
-                                snap_volinfo->parent_volname);
-                        GF_FREE (buf);
-                        goto out;
-                }
-                buf = NULL;
-        }
-
-        ret = dict_set_int32 (dict, "volcount", i);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Could not save volume count");
-                goto out;
-        }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_INIT_FAIL,
-                        "Failed to initiate snap phases");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    char *snapname = NULL;
+    char *buf = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    int32_t i = 0;
+    char key[64] = "";
+    int keylen;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    GF_ASSERT(conf);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to "
+               "get snapname");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        snprintf(err_str, len, "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_NOSNAP;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    list_for_each_entry(snap_volinfo, &snap->volumes, vol_list)
+    {
+        i++;
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        buf = gf_strdup(snap_volinfo->parent_volname);
+        if (!buf) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstrn(dict, key, keylen, buf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not set "
+                   "parent volume name %s in the dict",
+                   snap_volinfo->parent_volname);
+            GF_FREE(buf);
+            goto out;
+        }
+        buf = NULL;
+    }
+
+    ret = dict_set_int32n(dict, "volcount", SLEN("volcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save volume count");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate snap phases");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-glusterd_snap_t*
-glusterd_create_snap_object (dict_t *dict, dict_t *rsp_dict)
+glusterd_snap_t *
+glusterd_create_snap_object(dict_t *dict, dict_t *rsp_dict)
 {
-        char              *snapname    = NULL;
-        uuid_t            *snap_id     = NULL;
-        char              *description = NULL;
-        glusterd_snap_t   *snap        = NULL;
-        xlator_t          *this        = NULL;
-        glusterd_conf_t   *priv        = NULL;
-        int                ret         = -1;
-        int64_t            time_stamp  = 0;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        /* Fetch snapname, description, id and time from dict */
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch snapname");
-                goto out;
-        }
-
-        /* Ignore ret value for description*/
-        ret = dict_get_str (dict, "description", &description);
-
-        ret = dict_get_bin (dict, "snap-id", (void **)&snap_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch snap_id");
-                goto out;
-        }
-
-        ret = dict_get_int64 (dict, "snap-time", &time_stamp);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch snap-time");
-                goto out;
-        }
-        if (time_stamp <= 0) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid time-stamp: %"PRId64,
-                        time_stamp);
-                goto out;
-        }
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                if (!strcmp (snap->snapname, snapname) ||
-                    !gf_uuid_compare (snap->snap_id, *snap_id)) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_CREATION_FAIL,
-                                "Found duplicate snap %s (%s)",
-                                snap->snapname, uuid_utoa (snap->snap_id));
-                        ret = -1;
-                        break;
-                }
-        }
-        if (ret) {
-                snap = NULL;
-                goto out;
-        }
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Could not create "
-                        "the snap object for snap %s", snapname);
-                goto out;
-        }
-
-        strcpy (snap->snapname, snapname);
-        gf_uuid_copy (snap->snap_id, *snap_id);
-        snap->time_stamp = (time_t)time_stamp;
-        /* Set the status as GD_SNAP_STATUS_INIT and once the backend snapshot
-           is taken and snap is really ready to use, set the status to
-           GD_SNAP_STATUS_IN_USE. This helps in identifying the incomplete
-           snapshots and cleaning them up.
-        */
-        snap->snap_status = GD_SNAP_STATUS_INIT;
-        if (description) {
-                snap->description = gf_strdup (description);
-                if (snap->description == NULL) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_CREATION_FAIL,
-                                "Saving the Snapshot Description Failed");
-                        ret = -1;
-                        goto out;
-                 }
-        }
-
-        ret = glusterd_store_snap (snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Could not store snap"
-                        "object %s", snap->snapname);
-                goto out;
-        }
-
-        glusterd_list_add_order (&snap->snap_list, &priv->snapshots,
-                                 glusterd_compare_snap_time);
-
-        gf_msg_trace (this->name, 0, "Snapshot %s added to the list",
-                snap->snapname);
-
-        ret = 0;
+    char *snapname = NULL;
+    uuid_t *snap_id = NULL;
+    char *description = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int64_t time_stamp = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    /* Fetch snapname, description, id and time from dict */
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snapname");
+        goto out;
+    }
+
+    /* Ignore ret value for description*/
+    ret = dict_get_strn(dict, "description", SLEN("description"), &description);
+
+    ret = dict_get_bin(dict, "snap-id", (void **)&snap_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_id");
+        goto out;
+    }
+
+    ret = dict_get_int64(dict, "snap-time", &time_stamp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap-time");
+        goto out;
+    }
+    if (time_stamp <= 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid time-stamp: %" PRId64, time_stamp);
+        goto out;
+    }
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        if (!strcmp(snap->snapname, snapname) ||
+            !gf_uuid_compare(snap->snap_id, *snap_id)) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "Found duplicate snap %s (%s)", snap->snapname,
+                   uuid_utoa(snap->snap_id));
+            ret = -1;
+            break;
+        }
+    }
+    if (ret) {
+        snap = NULL;
+        goto out;
+    }
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not create "
+               "the snap object for snap %s",
+               snapname);
+        goto out;
+    }
+
+    gf_strncpy(snap->snapname, snapname, sizeof(snap->snapname));
+    gf_uuid_copy(snap->snap_id, *snap_id);
+    snap->time_stamp = (time_t)time_stamp;
+    /* Set the status as GD_SNAP_STATUS_INIT and once the backend snapshot
+       is taken and snap is really ready to use, set the status to
+       GD_SNAP_STATUS_IN_USE. This helps in identifying the incomplete
+       snapshots and cleaning them up.
+    */
+    snap->snap_status = GD_SNAP_STATUS_INIT;
+    if (description) {
+        snap->description = gf_strdup(description);
+        if (snap->description == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "Saving the Snapshot Description Failed");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not store snap"
+               "object %s",
+               snap->snapname);
+        goto out;
+    }
+
+    glusterd_list_add_order(&snap->snap_list, &priv->snapshots,
+                            glusterd_compare_snap_time);
+
+    gf_msg_trace(this->name, 0, "Snapshot %s added to the list",
+                 snap->snapname);
+
+    ret = 0;
 
 out:
-        if (ret) {
-                if (snap)
-                        glusterd_snap_remove (rsp_dict, snap,
-                                              _gf_true, _gf_true,
-                                              _gf_false);
-                snap = NULL;
-        }
+    if (ret) {
+        if (snap)
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_false);
+        snap = NULL;
+    }
 
-        return snap;
+    return snap;
 }
 
 /* Added missed_snap_entry to rsp_dict */
 int32_t
-glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict,
-                                   glusterd_volinfo_t *snap_vol,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   int32_t brick_number, int32_t op)
+glusterd_add_missed_snaps_to_dict(dict_t *rsp_dict,
+                                  glusterd_volinfo_t *snap_vol,
+                                  glusterd_brickinfo_t *brickinfo,
+                                  int32_t brick_number, int32_t op)
 {
-        char                   *snap_uuid                       = NULL;
-        char                    missed_snap_entry[PATH_MAX]     = "";
-        char                    name_buf[PATH_MAX]              = "";
-        int32_t                 missed_snap_count               = -1;
-        int32_t                 ret                             = -1;
-        xlator_t               *this                            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (brickinfo);
-
-        snap_uuid = gf_strdup (uuid_utoa (snap_vol->snapshot->snap_id));
-        if (!snap_uuid) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (missed_snap_entry, sizeof(missed_snap_entry),
-                  "%s:%s=%s:%d:%s:%d:%d", uuid_utoa(brickinfo->uuid),
-                  snap_uuid, snap_vol->volname, brick_number, brickinfo->path,
-                  op, GD_MISSED_SNAP_PENDING);
-
-        /* Fetch the missed_snap_count from the dict */
-        ret = dict_get_int32 (rsp_dict, "missed_snap_count",
-                              &missed_snap_count);
-        if (ret) {
-                /* Initialize the missed_snap_count for the first time */
-                missed_snap_count = 0;
-        }
-
-        /* Setting the missed_snap_entry in the rsp_dict */
-        snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
-                  missed_snap_count);
-        ret = dict_set_dynstr_with_alloc (rsp_dict, name_buf,
-                                          missed_snap_entry);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set missed_snap_entry (%s) "
-                        "in the rsp_dict.", missed_snap_entry);
-                goto out;
-        }
-        missed_snap_count++;
-
-        /* Setting the new missed_snap_count in the dict */
-        ret = dict_set_int32 (rsp_dict, "missed_snap_count",
-                              missed_snap_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Failed to set missed_snap_count for %s "
-                        "in the rsp_dict.", missed_snap_entry);
-                goto out;
-        }
+    char *snap_uuid = NULL;
+    char missed_snap_entry[PATH_MAX] = "";
+    char name_buf[PATH_MAX] = "";
+    int32_t missed_snap_count = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(brickinfo);
+
+    snap_uuid = gf_strdup(uuid_utoa(snap_vol->snapshot->snap_id));
+    if (!snap_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    len = snprintf(missed_snap_entry, sizeof(missed_snap_entry),
+                   "%s:%s=%s:%d:%s:%d:%d", uuid_utoa(brickinfo->uuid),
+                   snap_uuid, snap_vol->volname, brick_number, brickinfo->path,
+                   op, GD_MISSED_SNAP_PENDING);
+    if ((len < 0) || (len >= sizeof(missed_snap_entry))) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    /* Fetch the missed_snap_count from the dict */
+    ret = dict_get_int32n(rsp_dict, "missed_snap_count",
+                          SLEN("missed_snap_count"), &missed_snap_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=missed_snap_count", NULL);
+        /* Initialize the missed_snap_count for the first time */
+        missed_snap_count = 0;
+    }
+
+    /* Setting the missed_snap_entry in the rsp_dict */
+    snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d", missed_snap_count);
+    ret = dict_set_dynstr_with_alloc(rsp_dict, name_buf, missed_snap_entry);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set missed_snap_entry (%s) "
+               "in the rsp_dict.",
+               missed_snap_entry);
+        goto out;
+    }
+    missed_snap_count++;
+
+    /* Setting the new missed_snap_count in the dict */
+    ret = dict_set_int32n(rsp_dict, "missed_snap_count",
+                          SLEN("missed_snap_count"), missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set missed_snap_count for %s "
+               "in the rsp_dict.",
+               missed_snap_entry);
+        goto out;
+    }
 
 out:
-        if (snap_uuid)
-                GF_FREE (snap_uuid);
+    if (snap_uuid)
+        GF_FREE(snap_uuid);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function actually calls the command (or the API) for taking the
@@ -4651,353 +4547,384 @@ out:
    for glusterd
 */
 int32_t
-glusterd_take_lvm_snapshot (glusterd_brickinfo_t *brickinfo,
-                            char *origin_brick_path)
+glusterd_take_lvm_snapshot(glusterd_brickinfo_t *brickinfo,
+                           char *origin_brick_path)
 {
-        char             msg[NAME_MAX]    = "";
-        char             buf[PATH_MAX]    = "";
-        char            *ptr              = NULL;
-        char            *origin_device    = NULL;
-        int              ret              = -1;
-        gf_boolean_t     match            = _gf_false;
-        runner_t         runner           = {0,};
-        xlator_t        *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (origin_brick_path);
-
-        origin_device = glusterd_get_brick_mount_device (origin_brick_path);
-        if (!origin_device) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_GET_INFO_FAIL, "getting device name for "
-                        "the brick %s failed", origin_brick_path);
-                goto out;
-        }
-
-        /* Figuring out if setactivationskip flag is supported or not */
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "running lvcreate help");
-        runner_add_args (&runner, LVM_CREATE, "--help", NULL);
-        runner_log (&runner, "", GF_LOG_DEBUG, msg);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_LVCREATE_FAIL,
-                        "Failed to run lvcreate help");
-                runner_end (&runner);
-                goto out;
-        }
-
-        /* Looking for setactivationskip in lvcreate --help */
-        do {
-                ptr = fgets(buf, sizeof(buf),
-                            runner_chio (&runner, STDOUT_FILENO));
-                if (ptr) {
-                        if (strstr(buf, "setactivationskip")) {
-                                match = _gf_true;
-                                break;
-                        }
-                }
-        } while (ptr != NULL);
-        runner_end (&runner);
-
-        /* Taking the actual snapshot */
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "taking snapshot of the brick %s",
-                  origin_brick_path);
-        if (match == _gf_true)
-                runner_add_args (&runner, LVM_CREATE, "-s", origin_device,
-                                 "--setactivationskip", "n", "--name",
-                                 brickinfo->device_path, NULL);
-        else
-                runner_add_args (&runner, LVM_CREATE, "-s", origin_device,
-                                 "--name", brickinfo->device_path, NULL);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "taking snapshot of the "
-                        "brick (%s) of device %s failed",
-                        origin_brick_path, origin_device);
-        }
+    char msg[NAME_MAX] = "";
+    char buf[PATH_MAX] = "";
+    char *ptr = NULL;
+    char *origin_device = NULL;
+    int ret = -1;
+    gf_boolean_t match = _gf_false;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(origin_brick_path);
+
+    origin_device = glusterd_get_brick_mount_device(origin_brick_path);
+    if (!origin_device) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "getting device name for "
+               "the brick %s failed",
+               origin_brick_path);
+        goto out;
+    }
+
+    /* Figuring out if setactivationskip flag is supported or not */
+    runinit(&runner);
+    snprintf(msg, sizeof(msg), "running lvcreate help");
+    runner_add_args(&runner, LVM_CREATE, "--help", NULL);
+    runner_log(&runner, "", GF_LOG_DEBUG, msg);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_LVCREATE_FAIL,
+               "Failed to run lvcreate help");
+        runner_end(&runner);
+        goto out;
+    }
+
+    /* Looking for setactivationskip in lvcreate --help */
+    do {
+        ptr = fgets(buf, sizeof(buf), runner_chio(&runner, STDOUT_FILENO));
+        if (ptr) {
+            if (strstr(buf, "setactivationskip")) {
+                match = _gf_true;
+                break;
+            }
+        }
+    } while (ptr != NULL);
+    runner_end(&runner);
+
+    /* Taking the actual snapshot */
+    runinit(&runner);
+    snprintf(msg, sizeof(msg), "taking snapshot of the brick %s",
+             origin_brick_path);
+    if (match == _gf_true)
+        runner_add_args(&runner, LVM_CREATE, "-s", origin_device,
+                        "--setactivationskip", "n", "--name",
+                        brickinfo->device_path, NULL);
+    else
+        runner_add_args(&runner, LVM_CREATE, "-s", origin_device, "--name",
+                        brickinfo->device_path, NULL);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "taking snapshot of the "
+               "brick (%s) of device %s failed",
+               origin_brick_path, origin_device);
+    }
 
 out:
-        return ret;
+    if (origin_device)
+        GF_FREE(origin_device);
+
+    return ret;
 }
 
 int32_t
-glusterd_snap_brick_create (glusterd_volinfo_t *snap_volinfo,
-                            glusterd_brickinfo_t *brickinfo,
-                            int32_t brick_count)
+glusterd_snap_brick_create(glusterd_volinfo_t *snap_volinfo,
+                           glusterd_brickinfo_t *brickinfo, int32_t brick_count,
+                           int32_t clone)
 {
-        int32_t          ret                             = -1;
-        xlator_t        *this                            = NULL;
-        glusterd_conf_t *priv                            = NULL;
-        char             snap_brick_mount_path[PATH_MAX] = "";
-        struct stat      statbuf                         = {0, };
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (snap_volinfo);
-        GF_ASSERT (brickinfo);
-
-        snprintf (snap_brick_mount_path, sizeof (snap_brick_mount_path),
-                  "%s/%s/brick%d",  snap_mount_dir, snap_volinfo->volname,
-                  brick_count + 1);
-
-        ret = mkdir_p (snap_brick_mount_path, 0777, _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "creating the brick directory"
-                        " %s for the snapshot %s(device: %s) failed",
-                        snap_brick_mount_path, snap_volinfo->volname,
-                        brickinfo->device_path);
-                goto out;
-        }
-        /* mount the snap logical device on the directory inside
-           /run/gluster/snaps/<snapname>/@snap_brick_mount_path
-           Way to mount the snap brick via mount api is this.
-           ret = mount (device, snap_brick_mount_path, entry->mnt_type,
-                        MS_MGC_VAL, "nouuid");
-           But for now, mounting using runner apis.
-        */
-        ret = glusterd_mount_lvm_snapshot (brickinfo, snap_brick_mount_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_LVM_MOUNT_FAILED,
-                        "Failed to mount lvm snapshot.");
-                goto out;
-        }
-
-        ret = sys_stat (brickinfo->path, &statbuf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        GD_MSG_FILE_OP_FAILED,
-                        "stat of the brick %s"
-                        "(brick mount: %s) failed (%s)", brickinfo->path,
-                        snap_brick_mount_path, strerror (errno));
-                goto out;
-        }
-        ret = sys_lsetxattr (brickinfo->path,
-                             GF_XATTR_VOL_ID_KEY,
-                             snap_volinfo->volume_id, 16,
-                             XATTR_REPLACE);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_SETXATTR_FAIL, "Failed to set "
-                        "extended attribute %s on %s. Reason: "
-                        "%s, snap: %s", GF_XATTR_VOL_ID_KEY,
-                        brickinfo->path, strerror (errno),
-                        snap_volinfo->volname);
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char snap_brick_mount_path[PATH_MAX] = "";
+    char clone_uuid[64] = "";
+    struct stat statbuf = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+
+    GF_ASSERT(snap_volinfo);
+    GF_ASSERT(brickinfo);
+
+    if (clone) {
+        GLUSTERD_GET_UUID_NOHYPHEN(clone_uuid, snap_volinfo->volume_id);
+        len = snprintf(snap_brick_mount_path, sizeof(snap_brick_mount_path),
+                       "%s/%s/brick%d", snap_mount_dir, clone_uuid,
+                       brick_count + 1);
+    } else {
+        len = snprintf(snap_brick_mount_path, sizeof(snap_brick_mount_path),
+                       "%s/%s/brick%d", snap_mount_dir, snap_volinfo->volname,
+                       brick_count + 1);
+    }
+    if ((len < 0) || (len >= sizeof(snap_brick_mount_path))) {
+        goto out;
+    }
+
+    ret = mkdir_p(snap_brick_mount_path, 0755, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "creating the brick directory"
+               " %s for the snapshot %s(device: %s) failed",
+               snap_brick_mount_path, snap_volinfo->volname,
+               brickinfo->device_path);
+        goto out;
+    }
+    /* mount the snap logical device on the directory inside
+       /run/gluster/snaps/<snapname>/@snap_brick_mount_path
+       Way to mount the snap brick via mount api is this.
+       ret = mount (device, snap_brick_mount_path, entry->mnt_type,
+                    MS_MGC_VAL, "nouuid");
+       But for now, mounting using runner apis.
+    */
+    ret = glusterd_mount_lvm_snapshot(brickinfo, snap_brick_mount_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_LVM_MOUNT_FAILED,
+               "Failed to mount lvm snapshot.");
+        goto out;
+    }
+
+    ret = sys_stat(brickinfo->path, &statbuf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "stat of the brick %s"
+               "(brick mount: %s) failed (%s)",
+               brickinfo->path, snap_brick_mount_path, strerror(errno));
+        goto out;
+    }
+    ret = sys_lsetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                        snap_volinfo->volume_id, 16, XATTR_REPLACE);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL,
+               "Failed to set "
+               "extended attribute %s on %s. Reason: "
+               "%s, snap: %s",
+               GF_XATTR_VOL_ID_KEY, brickinfo->path, strerror(errno),
+               snap_volinfo->volname);
+        goto out;
+    }
 
 out:
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_UMOUNTING_SNAP_BRICK, "unmounting the snap brick"
-                        " mount %s", snap_brick_mount_path);
-                /*umount2 system call doesn't cleanup mtab entry after un-mount.
-                  So use external umount command*/
-                glusterd_umount (snap_brick_mount_path);
-        }
-
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UMOUNTING_SNAP_BRICK,
+               "unmounting the snap brick"
+               " mount %s",
+               snap_brick_mount_path);
+        /*umount2 system call doesn't cleanup mtab entry after un-mount.
+          So use external umount command*/
+        glusterd_umount(snap_brick_mount_path);
+    }
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int32_t
-glusterd_add_brick_to_snap_volume (dict_t *dict, dict_t *rsp_dict,
-                                    glusterd_volinfo_t  *snap_vol,
-                                    glusterd_brickinfo_t *original_brickinfo,
-                                    int64_t volcount, int32_t brick_count,
-                                    int clone)
+glusterd_add_brick_to_snap_volume(dict_t *dict, dict_t *rsp_dict,
+                                  glusterd_volinfo_t *snap_vol,
+                                  glusterd_brickinfo_t *original_brickinfo,
+                                  int64_t volcount, int32_t brick_count,
+                                  int clone)
 {
-        char                    key[PATH_MAX]                   = "";
-        char                   *value                           = NULL;
-        char                   *snap_brick_dir                  = NULL;
-        char                    snap_brick_path[PATH_MAX]       = "";
-        char                   *snap_device                     = NULL;
-        glusterd_brickinfo_t   *snap_brickinfo                  = NULL;
-        gf_boolean_t            add_missed_snap                 = _gf_false;
-        int32_t                 ret                             = -1;
-        xlator_t               *this                            = NULL;
-        char                    abspath[PATH_MAX]               = {0};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (original_brickinfo);
-
-        snprintf (key, sizeof(key), "vol%"PRId64".origin_brickpath%d",
-                  volcount, brick_count);
-        ret = dict_set_dynstr_with_alloc (dict, key, original_brickinfo->path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set %s", key);
-                goto out;
-        }
-
-        ret = glusterd_brickinfo_new (&snap_brickinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_NEW_INFO_FAIL,
-                        "initializing the brick for the snap "
-                        "volume failed (snapname: %s)",
-                        snap_vol->snapshot->snapname);
-                goto out;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64".fstype%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &value);
-        if (!ret) {
-                /* Update the fstype in original brickinfo as well */
-                strcpy (original_brickinfo->fstype, value);
-                strncpy (snap_brickinfo->fstype, value,
-                         (sizeof (snap_brickinfo->fstype) - 1));
-        } else {
-                if (is_origin_glusterd (dict) == _gf_true)
-                        add_missed_snap = _gf_true;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64".mnt_opts%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &value);
-        if (!ret) {
-                /* Update the mnt_opts in original brickinfo as well */
-                strcpy (original_brickinfo->mnt_opts, value);
-                strcpy (snap_brickinfo->mnt_opts, value);
-        } else {
-                if (is_origin_glusterd (dict) == _gf_true)
-                        add_missed_snap = _gf_true;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64".brickdir%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &snap_brick_dir);
-        if (ret) {
-                /* Using original brickinfo here because it will be a
-                 * pending snapshot and storing the original brickinfo
-                 * will help in mapping while recreating the missed snapshot
-                 */
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_NOT_FOUND, "Unable to fetch "
-                        "snap mount path(%s). Adding to missed_snap_list", key);
-                snap_brickinfo->snap_status = -1;
-
-                snap_brick_dir = original_brickinfo->mount_dir;
-
-                /* In origiator node add snaps missed
-                 * from different nodes to the dict
-                 */
-                if (is_origin_glusterd (dict) == _gf_true)
-                        add_missed_snap = _gf_true;
-        }
-
-        if ((snap_brickinfo->snap_status != -1) &&
-            (!gf_uuid_compare (original_brickinfo->uuid, MY_UUID)) &&
-            (!glusterd_is_brick_started (original_brickinfo))) {
-                /* In case if the brick goes down after prevalidate. */
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_BRICK_DISCONNECTED, "brick %s:%s is not"
-                        " started (snap: %s)",
-                        original_brickinfo->hostname,
-                        original_brickinfo->path,
-                        snap_vol->snapshot->snapname);
-
-                snap_brickinfo->snap_status = -1;
-                add_missed_snap = _gf_true;
-        }
-
-        if (add_missed_snap) {
-                ret = glusterd_add_missed_snaps_to_dict (rsp_dict,
-                                            snap_vol,
-                                            original_brickinfo,
-                                            brick_count + 1,
-                                            GF_SNAP_OPTION_TYPE_CREATE);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
-                                "Failed to add missed"
-                                " snapshot info for %s:%s in the rsp_dict",
-                                original_brickinfo->hostname,
-                                original_brickinfo->path);
-                        goto out;
-                }
-        }
-
-        /* Create brick-path in the format /var/run/gluster/snaps/ *
-         * <snap-uuid>/<original-brick#>/snap-brick-dir *
+    char key[64] = "";
+    int keylen;
+    char *value = NULL;
+    char *snap_brick_dir = NULL;
+    char snap_brick_path[PATH_MAX] = "";
+    char clone_uuid[64] = "";
+    char *snap_device = NULL;
+    glusterd_brickinfo_t *snap_brickinfo = NULL;
+    gf_boolean_t add_missed_snap = _gf_false;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = "";
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(original_brickinfo);
+
+    snprintf(key, sizeof(key), "vol%" PRId64 ".origin_brickpath%d", volcount,
+             brick_count);
+    ret = dict_set_dynstr_with_alloc(dict, key, original_brickinfo->path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
+
+    ret = glusterd_brickinfo_new(&snap_brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NEW_INFO_FAIL,
+               "initializing the brick for the snap "
+               "volume failed (snapname: %s)",
+               snap_vol->snapshot->snapname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &value);
+    if (!ret) {
+        /* Update the fstype in original brickinfo as well */
+        gf_strncpy(original_brickinfo->fstype, value,
+                   sizeof(original_brickinfo->fstype));
+        gf_strncpy(snap_brickinfo->fstype, value,
+                   sizeof(snap_brickinfo->fstype));
+    } else {
+        if (is_origin_glusterd(dict) == _gf_true)
+            add_missed_snap = _gf_true;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".mnt_opts%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &value);
+    if (!ret) {
+        /* Update the mnt_opts in original brickinfo as well */
+        gf_strncpy(original_brickinfo->mnt_opts, value,
+                   sizeof(original_brickinfo->mnt_opts));
+        gf_strncpy(snap_brickinfo->mnt_opts, value,
+                   sizeof(snap_brickinfo->mnt_opts));
+    } else {
+        if (is_origin_glusterd(dict) == _gf_true)
+            add_missed_snap = _gf_true;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &snap_brick_dir);
+    if (ret) {
+        /* Using original brickinfo here because it will be a
+         * pending snapshot and storing the original brickinfo
+         * will help in mapping while recreating the missed snapshot
          */
-        snprintf (snap_brick_path, sizeof(snap_brick_path),
-                  "%s/%s/brick%d%s", snap_mount_dir,
-                  snap_vol->volname, brick_count+1,
-                  snap_brick_dir);
-
-        snprintf (key, sizeof(key), "vol%"PRId64".brick_snapdevice%d",
-                  volcount, brick_count);
-        ret = dict_get_str (dict, key, &snap_device);
-        if (ret) {
-                /* If the device name is empty, so will be the brick path
-                 * Hence the missed snap has already been added above
-                 */
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_NOT_FOUND, "Unable to fetch "
-                        "snap device (%s). Leaving empty", key);
-        } else
-                strcpy (snap_brickinfo->device_path, snap_device);
-
-        ret = gf_canonicalize_path (snap_brick_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CANONICALIZE_FAIL,
-                        "Failed to canonicalize path");
-                goto out;
-        }
-
-        strcpy (snap_brickinfo->hostname, original_brickinfo->hostname);
-        strcpy (snap_brickinfo->path, snap_brick_path);
-
-        if (!realpath (snap_brick_path, abspath)) {
-                /* ENOENT indicates that brick path has not been created which
-                 * is a valid scenario */
-                if (errno != ENOENT) {
-                        gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                                GD_MSG_BRICKINFO_CREATE_FAIL, "realpath () "
-                                "failed for brick %s. The underlying filesystem"
-                                " may be in bad state", snap_brick_path);
-                        ret = -1;
-                        goto out;
-                }
-        }
-        strncpy (snap_brickinfo->real_path, abspath, strlen(abspath));
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Unable to fetch "
+               "snap mount path(%s). Adding to missed_snap_list",
+               key);
+        snap_brickinfo->snap_status = -1;
 
-        strcpy (snap_brickinfo->mount_dir, original_brickinfo->mount_dir);
-        gf_uuid_copy (snap_brickinfo->uuid, original_brickinfo->uuid);
-        /* AFR changelog names are based on brick_id and hence the snap
-         * volume's bricks must retain the same ID */
-        cds_list_add_tail (&snap_brickinfo->brick_list, &snap_vol->bricks);
+        snap_brick_dir = original_brickinfo->mount_dir;
 
-        if (clone) {
-                GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (snap_brickinfo, snap_vol,
-                                                      brick_count);
-        } else
-                strcpy (snap_brickinfo->brick_id, original_brickinfo->brick_id);
+        /* In origiator node add snaps missed
+         * from different nodes to the dict
+         */
+        if (is_origin_glusterd(dict) == _gf_true)
+            add_missed_snap = _gf_true;
+    }
+
+    if ((snap_brickinfo->snap_status != -1) &&
+        (!gf_uuid_compare(original_brickinfo->uuid, MY_UUID)) &&
+        (!glusterd_is_brick_started(original_brickinfo))) {
+        /* In case if the brick goes down after prevalidate. */
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_DISCONNECTED,
+               "brick %s:%s is not"
+               " started (snap: %s)",
+               original_brickinfo->hostname, original_brickinfo->path,
+               snap_vol->snapshot->snapname);
+
+        snap_brickinfo->snap_status = -1;
+        add_missed_snap = _gf_true;
+    }
+
+    if (add_missed_snap) {
+        ret = glusterd_add_missed_snaps_to_dict(
+            rsp_dict, snap_vol, original_brickinfo, brick_count + 1,
+            GF_SNAP_OPTION_TYPE_CREATE);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+                   "Failed to add missed"
+                   " snapshot info for %s:%s in the rsp_dict",
+                   original_brickinfo->hostname, original_brickinfo->path);
+            goto out;
+        }
+    }
+
+    /* Create brick-path in the format /var/run/gluster/snaps/ *
+     * <snap-uuid>/<original-brick#>/snap-brick-dir *
+     */
+    if (clone) {
+        GLUSTERD_GET_UUID_NOHYPHEN(clone_uuid, snap_vol->volume_id);
+        len = snprintf(snap_brick_path, sizeof(snap_brick_path),
+                       "%s/%s/brick%d%s", snap_mount_dir, clone_uuid,
+                       brick_count + 1, snap_brick_dir);
+    } else {
+        len = snprintf(snap_brick_path, sizeof(snap_brick_path),
+                       "%s/%s/brick%d%s", snap_mount_dir, snap_vol->volname,
+                       brick_count + 1, snap_brick_dir);
+    }
+    if ((len < 0) || (len >= sizeof(snap_brick_path))) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".brick_snapdevice%d",
+                      volcount, brick_count);
+    ret = dict_get_strn(dict, key, keylen, &snap_device);
+    if (ret) {
+        /* If the device name is empty, so will be the brick path
+         * Hence the missed snap has already been added above
+         */
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Unable to fetch "
+               "snap device (%s). Leaving empty",
+               key);
+    } else
+        gf_strncpy(snap_brickinfo->device_path, snap_device,
+                   sizeof(snap_brickinfo->device_path));
+
+    ret = gf_canonicalize_path(snap_brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CANONICALIZE_FAIL,
+               "Failed to canonicalize path");
+        goto out;
+    }
+
+    gf_strncpy(snap_brickinfo->hostname, original_brickinfo->hostname,
+               sizeof(snap_brickinfo->hostname));
+    gf_strncpy(snap_brickinfo->path, snap_brick_path,
+               sizeof(snap_brickinfo->path));
+
+    if (!realpath(snap_brick_path, abspath)) {
+        /* ENOENT indicates that brick path has not been created which
+         * is a valid scenario */
+        if (errno != ENOENT) {
+            gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                   GD_MSG_BRICKINFO_CREATE_FAIL,
+                   "realpath () "
+                   "failed for brick %s. The underlying filesystem"
+                   " may be in bad state",
+                   snap_brick_path);
+            ret = -1;
+            goto out;
+        }
+    }
+    gf_strncpy(snap_brickinfo->real_path, abspath,
+               sizeof(snap_brickinfo->real_path));
+
+    gf_strncpy(snap_brickinfo->mount_dir, original_brickinfo->mount_dir,
+               sizeof(snap_brickinfo->mount_dir));
+    gf_uuid_copy(snap_brickinfo->uuid, original_brickinfo->uuid);
+    /* AFR changelog names are based on brick_id and hence the snap
+     * volume's bricks must retain the same ID */
+    cds_list_add_tail(&snap_brickinfo->brick_list, &snap_vol->bricks);
+
+    if (clone) {
+        GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(snap_brickinfo, snap_vol,
+                                             brick_count);
+    } else
+        gf_strncpy(snap_brickinfo->brick_id, original_brickinfo->brick_id,
+                   sizeof(snap_brickinfo->brick_id));
 
 out:
-        if (ret && snap_brickinfo)
-                GF_FREE (snap_brickinfo);
+    if (ret && snap_brickinfo)
+        GF_FREE(snap_brickinfo);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function will update the file-system label of the
@@ -5008,473 +4935,536 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_update_fs_label (glusterd_brickinfo_t *brickinfo)
+glusterd_update_fs_label(glusterd_brickinfo_t *brickinfo)
 {
-        int32_t         ret                     = -1;
-        char            msg [PATH_MAX]          = "";
-        char            label [NAME_MAX]        = "";
-        uuid_t          uuid                    = {0,};
-        runner_t        runner                  = {0,};
-        xlator_t       *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickinfo);
-
-        /* Generate a new UUID */
-        gf_uuid_generate (uuid);
-
-        GLUSTERD_GET_UUID_NOHYPHEN (label, uuid);
-
-        runinit (&runner);
-
-        /* Call the file-system specific tools to update the file-system
-         * label. Currently we are only supporting xfs and ext2/ext3/ext4
-         * file-system.
-         */
-        if (0 == strcmp (brickinfo->fstype, "xfs")) {
-                /* XFS label is of size 12. Therefore we should truncate the
-                 * label to 12 bytes*/
-                label [12] = '\0';
-                snprintf (msg, sizeof (msg), "Changing filesystem label of "
-                          "%s brick to %s", brickinfo->path, label);
-                /* Run the run xfs_admin tool to change the label
-                 * of the file-system */
-                runner_add_args (&runner, "xfs_admin", "-L", label,
-                                 brickinfo->device_path, NULL);
-        } else if (0 == strcmp (brickinfo->fstype, "ext4") ||
-                   0 == strcmp (brickinfo->fstype, "ext3") ||
-                   0 == strcmp (brickinfo->fstype, "ext2")) {
-                /* Ext2/Ext3/Ext4 label is of size 16. Therefore we should
-                 * truncate the label to 16 bytes*/
-                label [16] = '\0';
-                snprintf (msg, sizeof (msg), "Changing filesystem label of "
-                          "%s brick to %s", brickinfo->path, label);
-                /* For ext2/ext3/ext4 run tune2fs to change the
-                 * file-system label */
-                runner_add_args (&runner, "tune2fs", "-L", label,
-                                 brickinfo->device_path, NULL);
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, EOPNOTSUPP,
-                        GD_MSG_OP_UNSUPPORTED, "Changing file-system "
-                        "label of %s file-system is not supported as of now",
-                        brickinfo->fstype);
-                runner_end (&runner);
-                ret = -1;
-                goto out;
-        }
-
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_FS_LABEL_UPDATE_FAIL, "Failed to change "
-                        "filesystem label of %s brick to %s",
-                        brickinfo->path, label);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char msg[PATH_MAX] = "";
+    char label[NAME_MAX] = "";
+    uuid_t uuid = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickinfo);
+
+    /* Generate a new UUID */
+    gf_uuid_generate(uuid);
+
+    GLUSTERD_GET_UUID_NOHYPHEN(label, uuid);
+
+    runinit(&runner);
+
+    /* Call the file-system specific tools to update the file-system
+     * label. Currently we are only supporting xfs and ext2/ext3/ext4
+     * file-system.
+     */
+    if (0 == strcmp(brickinfo->fstype, "xfs")) {
+        /* XFS label is of size 12. Therefore we should truncate the
+         * label to 12 bytes*/
+        label[12] = '\0';
+        len = snprintf(msg, sizeof(msg),
+                       "Changing filesystem label "
+                       "of %s brick to %s",
+                       brickinfo->path, label);
+        if (len < 0) {
+            strcpy(msg, "<error>");
+        }
+        /* Run the run xfs_admin tool to change the label
+         * of the file-system */
+        runner_add_args(&runner, "xfs_admin", "-L", label,
+                        brickinfo->device_path, NULL);
+    } else if (0 == strcmp(brickinfo->fstype, "ext4") ||
+               0 == strcmp(brickinfo->fstype, "ext3") ||
+               0 == strcmp(brickinfo->fstype, "ext2")) {
+        /* Ext2/Ext3/Ext4 label is of size 16. Therefore we should
+         * truncate the label to 16 bytes*/
+        label[16] = '\0';
+        len = snprintf(msg, sizeof(msg),
+                       "Changing filesystem label "
+                       "of %s brick to %s",
+                       brickinfo->path, label);
+        if (len < 0) {
+            strcpy(msg, "<error>");
+        }
+        /* For ext2/ext3/ext4 run tune2fs to change the
+         * file-system label */
+        runner_add_args(&runner, "tune2fs", "-L", label, brickinfo->device_path,
+                        NULL);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, EOPNOTSUPP, GD_MSG_OP_UNSUPPORTED,
+               "Changing file-system "
+               "label of %s file-system is not supported as of now",
+               brickinfo->fstype);
+        runner_end(&runner);
+        ret = -1;
+        goto out;
+    }
+
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FS_LABEL_UPDATE_FAIL,
+               "Failed to change "
+               "filesystem label of %s brick to %s",
+               brickinfo->path, label);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int32_t
-glusterd_take_brick_snapshot (dict_t *dict, glusterd_volinfo_t *snap_vol,
-                              glusterd_brickinfo_t *brickinfo,
-                              int32_t volcount, int32_t brick_count)
+glusterd_take_brick_snapshot(dict_t *dict, glusterd_volinfo_t *snap_vol,
+                             glusterd_brickinfo_t *brickinfo, int32_t volcount,
+                             int32_t brick_count, int32_t clone)
 {
-        char                   *origin_brick_path   = NULL;
-        char                    key[PATH_MAX]       = "";
-        int32_t                 ret                 = -1;
-        xlator_t               *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (brickinfo);
-
-        if (strlen(brickinfo->device_path) == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Device path is empty "
-                        "brick %s:%s", brickinfo->hostname, brickinfo->path);
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%d.origin_brickpath%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &origin_brick_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch "
-                        "brick path (%s)", key);
-                goto out;
-        }
-
-        ret = glusterd_take_lvm_snapshot (brickinfo, origin_brick_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Failed to take snapshot of "
-                        "brick %s:%s", brickinfo->hostname, origin_brick_path);
-                goto out;
-        }
-
-        /* After the snapshot both the origin brick (LVM brick) and
-         * the snapshot brick will have the same file-system label. This
-         * will cause lot of problems at mount time. Therefore we must
-         * generate a new label for the snapshot brick
+    char *origin_brick_path = NULL;
+    char key[64] = "";
+    int keylen;
+    int32_t ret = -1;
+    gf_boolean_t snap_activate = _gf_false;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(priv);
+
+    if (strlen(brickinfo->device_path) == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Device path is empty "
+               "brick %s:%s",
+               brickinfo->hostname, brickinfo->path);
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%d.origin_brickpath%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &origin_brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch "
+               "brick path (%s)",
+               key);
+        goto out;
+    }
+
+    ret = glusterd_take_lvm_snapshot(brickinfo, origin_brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to take snapshot of "
+               "brick %s:%s",
+               brickinfo->hostname, origin_brick_path);
+        goto out;
+    }
+
+    /* After the snapshot both the origin brick (LVM brick) and
+     * the snapshot brick will have the same file-system label. This
+     * will cause lot of problems at mount time. Therefore we must
+     * generate a new label for the snapshot brick
+     */
+    ret = glusterd_update_fs_label(brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FS_LABEL_UPDATE_FAIL,
+               "Failed to update "
+               "file-system label for %s brick",
+               brickinfo->path);
+        /* Failing to update label should not cause snapshot failure.
+         * Currently label is updated only for XFS and ext2/ext3/ext4
+         * file-system.
          */
-        ret = glusterd_update_fs_label (brickinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_FS_LABEL_UPDATE_FAIL, "Failed to update "
-                        "file-system label for %s brick", brickinfo->path);
-                /* Failing to update label should not cause snapshot failure.
-                 * Currently label is updated only for XFS and ext2/ext3/ext4
-                 * file-system.
-                 */
-        }
+    }
 
-        /* create the complete brick here */
-        ret = glusterd_snap_brick_create (snap_vol, brickinfo, brick_count);
+    /* create the complete brick here in case of clone and
+     * activate-on-create configuration.
+     */
+    snap_activate = dict_get_str_boolean(
+        priv->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+    if (clone || snap_activate) {
+        ret = glusterd_snap_brick_create(snap_vol, brickinfo, brick_count,
+                                         clone);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_CREATION_FAIL, "not able to"
-                        " create the brick for the snap %s"
-                        ", volume %s", snap_vol->snapshot->snapname,
-                        snap_vol->volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_CREATION_FAIL,
+                   "not able to "
+                   "create the brick for the snap %s, volume %s",
+                   snap_vol->snapshot->snapname, snap_vol->volname);
+            goto out;
         }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_snap_clear_unsupported_opt (glusterd_volinfo_t *volinfo,
-                              struct gd_snap_unsupported_opt_t *unsupported_opt)
+glusterd_snap_clear_unsupported_opt(
+    glusterd_volinfo_t *volinfo,
+    struct gd_snap_unsupported_opt_t *unsupported_opt)
 {
-        int      ret          = -1;
-        int      i            = 0;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
-
-        for (i = 0; unsupported_opt[i].key; i++) {
-                glusterd_volinfo_get (volinfo, unsupported_opt[i].key,
-                                      &unsupported_opt[i].value);
-
-                if (unsupported_opt[i].value) {
-                        unsupported_opt[i].value = gf_strdup (
-                                                      unsupported_opt[i].value);
-                        if (!unsupported_opt[i].value) {
-                                ret = -1;
-                                goto out;
-                        }
-                        dict_del (volinfo->dict, unsupported_opt[i].key);
-                }
-        }
+    int ret = -1;
+    int i = 0;
 
-        ret = 0;
-out:
-        if (ret) {
-                for (i = 0; unsupported_opt[i].key; i++) {
-                        if (unsupported_opt[i].value) {
-                                /* Freeing the memory */
-                                GF_FREE (unsupported_opt[i].value);
-                                unsupported_opt[i].value = NULL;
-                        }
-                }
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
+
+    for (i = 0; unsupported_opt[i].key; i++) {
+        glusterd_volinfo_get(volinfo, unsupported_opt[i].key,
+                             &unsupported_opt[i].value);
+
+        if (unsupported_opt[i].value) {
+            unsupported_opt[i].value = gf_strdup(unsupported_opt[i].value);
+            if (!unsupported_opt[i].value) {
+                ret = -1;
+                goto out;
+            }
+            dict_del(volinfo->dict, unsupported_opt[i].key);
         }
+    }
 
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
 
 static int
-glusterd_snap_set_unsupported_opt (glusterd_volinfo_t *volinfo,
-                              struct gd_snap_unsupported_opt_t *unsupported_opt)
+glusterd_snap_set_unsupported_opt(
+    glusterd_volinfo_t *volinfo,
+    struct gd_snap_unsupported_opt_t *unsupported_opt)
 {
-        int       ret       = -1;
-        int       i         = 0;
+    int ret = -1;
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("glusterd", volinfo, out);
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
 
-        for (i = 0; unsupported_opt[i].key; i++) {
-                if (!unsupported_opt[i].value)
-                        continue;
+    for (i = 0; unsupported_opt[i].key; i++) {
+        if (!unsupported_opt[i].value)
+            continue;
 
-                ret = dict_set_dynstr (volinfo->dict, unsupported_opt[i].key,
-                                       unsupported_opt[i].value);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, errno,
-                                GD_MSG_DICT_SET_FAILED, "dict set failed");
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
+        ret = dict_set_dynstr(volinfo->dict, unsupported_opt[i].key,
+                              unsupported_opt[i].value);
         if (ret) {
-                for (; unsupported_opt[i].key; i++) {
-                        if (unsupported_opt[i].value) {
-                                /* Freeing the memory */
-                                GF_FREE (unsupported_opt[i].value);
-                                unsupported_opt[i].value = NULL;
-                        }
-                }
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "dict set failed");
+            goto out;
         }
+        unsupported_opt[i].value = NULL;
+    }
 
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
 
 glusterd_volinfo_t *
-glusterd_do_snap_vol (glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
-                      dict_t *dict, dict_t *rsp_dict, int64_t volcount,
-                      int clone)
+glusterd_do_snap_vol(glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
+                     dict_t *dict, dict_t *rsp_dict, int64_t volcount,
+                     int clone)
 {
-        char                         key[PATH_MAX]           = "";
-        char                        *username                = NULL;
-        char                        *password                = NULL;
-        glusterd_brickinfo_t        *brickinfo               = NULL;
-        glusterd_conf_t             *priv                    = NULL;
-        glusterd_volinfo_t          *snap_vol                = NULL;
-        uuid_t                      *snap_volid              = NULL;
-        int32_t                      ret                     = -1;
-        int32_t                      brick_count             = 0;
-        xlator_t                    *this                    = NULL;
-        int64_t                      brick_order             = 0;
-        char                        *clonename               = NULL;
-        gf_boolean_t                 conf_present            = _gf_false;
-
-       struct  gd_snap_unsupported_opt_t  unsupported_opt[]  = {
-                                         {.key   = VKEY_FEATURES_QUOTA,
-                                          .value = NULL},
-                                         {.key   = VKEY_FEATURES_INODE_QUOTA,
-                                          .value = NULL},
-                                         {.key   = "feature.deem-statfs",
-                                          .value = NULL},
-                                         {.key   = "features.quota-deem-statfs",
-                                          .value = NULL},
-                                         {.key   = NULL,
-                                          .value = NULL}
-                                                         };
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-        GF_ASSERT (origin_vol);
-        GF_ASSERT (rsp_dict);
-
-        /* fetch username, password and vol_id from dict*/
-        snprintf (key, sizeof(key), "volume%"PRId64"_username", volcount);
-        ret = dict_get_str (dict, key, &username);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get %s for "
-                        "snap %s", key, snap->snapname);
-                goto out;
-        }
-        snprintf (key, sizeof(key), "volume%"PRId64"_password", volcount);
-        ret = dict_get_str (dict, key, &password);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get %s for "
-                        "snap %s", key, snap->snapname);
-                goto out;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64"_volid", volcount);
-        ret = dict_get_bin (dict, key, (void **)&snap_volid);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to fetch snap_volid");
-                goto out;
-        }
-
-        /* We are not setting the username and password here as
-         * we need to set the user name and password passed in
-         * the dictionary
-         */
-        ret = glusterd_volinfo_dup (origin_vol, &snap_vol, _gf_false);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_OP_FAILED, "Failed to duplicate volinfo "
-                        "for the snapshot %s", snap->snapname);
-                goto out;
+    char key[64] = "";
+    int keylen;
+    char *username = NULL;
+    char *password = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    uuid_t *snap_volid = NULL;
+    int32_t ret = -1;
+    int32_t brick_count = 0;
+    xlator_t *this = NULL;
+    char *clonename = NULL;
+    gf_boolean_t conf_present = _gf_false;
+    int i = 0;
+
+    struct gd_snap_unsupported_opt_t unsupported_opt[] = {
+        {.key = VKEY_FEATURES_QUOTA, .value = NULL},
+        {.key = VKEY_FEATURES_INODE_QUOTA, .value = NULL},
+        {.key = "feature.deem-statfs", .value = NULL},
+        {.key = "features.quota-deem-statfs", .value = NULL},
+        {.key = NULL, .value = NULL}};
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+    GF_ASSERT(origin_vol);
+    GF_ASSERT(rsp_dict);
+
+    /* fetch username, password and vol_id from dict*/
+    keylen = snprintf(key, sizeof(key), "volume%" PRId64 "_username", volcount);
+    ret = dict_get_strn(dict, key, keylen, &username);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get %s for "
+               "snap %s",
+               key, snap->snapname);
+        goto out;
+    }
+    keylen = snprintf(key, sizeof(key), "volume%" PRId64 "_password", volcount);
+    ret = dict_get_strn(dict, key, keylen, &password);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get %s for "
+               "snap %s",
+               key, snap->snapname);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "vol%" PRId64 "_volid", volcount);
+    ret = dict_get_bin(dict, key, (void **)&snap_volid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_volid");
+        goto out;
+    }
+
+    /* We are not setting the username and password here as
+     * we need to set the user name and password passed in
+     * the dictionary
+     */
+    ret = glusterd_volinfo_dup(origin_vol, &snap_vol, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to duplicate volinfo "
+               "for the snapshot %s",
+               snap->snapname);
+        goto out;
+    }
+
+    /* uuid is used as lvm snapshot name.
+       This will avoid restrictions on snapshot names provided by user */
+    gf_uuid_copy(snap_vol->volume_id, *snap_volid);
+    snap_vol->is_snap_volume = _gf_true;
+    snap_vol->snapshot = snap;
+
+    if (clone) {
+        snap_vol->is_snap_volume = _gf_false;
+        ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get %s "
+                   "for snap %s",
+                   key, snap->snapname);
+            goto out;
+        }
+        cds_list_add_tail(&snap_vol->vol_list, &snap->volumes);
+        gf_strncpy(snap_vol->volname, clonename, sizeof(snap_vol->volname));
+        gf_uuid_copy(snap_vol->restored_from_snap,
+                     origin_vol->snapshot->snap_id);
+
+    } else {
+        GLUSTERD_GET_UUID_NOHYPHEN(snap_vol->volname, *snap_volid);
+        gf_strncpy(snap_vol->parent_volname, origin_vol->volname,
+                   sizeof(snap_vol->parent_volname));
+        ret = glusterd_list_add_snapvol(origin_vol, snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_LIST_SET_FAIL,
+                   "could not add the "
+                   "snap volume %s to the list",
+                   snap_vol->volname);
+            goto out;
         }
-
-        /* uuid is used as lvm snapshot name.
-           This will avoid restrictions on snapshot names provided by user */
-        gf_uuid_copy (snap_vol->volume_id, *snap_volid);
-        snap_vol->is_snap_volume = _gf_true;
-        snap_vol->snapshot = snap;
-
-        if (clone) {
-                snap_vol->is_snap_volume = _gf_false;
-                ret = dict_get_str (dict, "clonename", &clonename);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to get %s "
-                                "for snap %s", key, snap->snapname);
-                        goto out;
-                }
-                cds_list_add_tail (&snap_vol->vol_list, &snap->volumes);
-                strcpy(snap_vol->volname, clonename);
-                gf_uuid_copy (snap_vol->restored_from_snap,
-                              origin_vol->snapshot->snap_id);
-
-        } else {
-                GLUSTERD_GET_UUID_NOHYPHEN (snap_vol->volname, *snap_volid);
-                strcpy (snap_vol->parent_volname, origin_vol->volname);
-                ret = glusterd_list_add_snapvol (origin_vol, snap_vol);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_LIST_SET_FAIL, "could not add the "
-                                "snap volume %s to the list",
-                                snap_vol->volname);
-                        goto out;
-                }
         /* TODO : Sync before taking a snapshot */
         /* Copy the status and config files of geo-replication before
          * taking a snapshot. During restore operation these files needs
          * to be copied back in /var/lib/glusterd/georeplication/
          */
-                ret = glusterd_copy_geo_rep_files (origin_vol, snap_vol,
-                                                   rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_OP_FAILED, "Failed to copy "
-                                "geo-rep config and status files for volume %s",
-                                origin_vol->volname);
-                        goto out;
-                }
-
+        ret = glusterd_copy_geo_rep_files(origin_vol, snap_vol, rsp_dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to copy "
+                   "geo-rep config and status files for volume %s",
+                   origin_vol->volname);
+            goto out;
+        }
+    }
+
+    glusterd_auth_set_username(snap_vol, username);
+    glusterd_auth_set_password(snap_vol, password);
+
+    /* Adding snap brickinfos to the snap volinfo */
+    brick_count = 0;
+    cds_list_for_each_entry(brickinfo, &origin_vol->bricks, brick_list)
+    {
+        ret = glusterd_add_brick_to_snap_volume(
+            dict, rsp_dict, snap_vol, brickinfo, volcount, brick_count, clone);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL,
+                   "Failed to add the snap brick for "
+                   "%s:%s to the snap volume",
+                   brickinfo->hostname, brickinfo->path);
+            goto out;
+        }
+        brick_count++;
+    }
+
+    /* During snapshot creation if I/O is in progress,
+     * then barrier value is enabled. Hence during snapshot create
+     * and in-turn snapshot restore the barrier value is set to enable.
+     * Because of this further I/O on the mount point fails.
+     * Hence remove the barrier key from newly created snap volinfo
+     * before storing and generating the brick volfiles. Also update
+     * the snap vol's version after removing the barrier key.
+     */
+    dict_deln(snap_vol->dict, "features.barrier", SLEN("features.barrier"));
+    gd_update_volume_op_versions(snap_vol);
+
+    /* *
+     * Create the export file from the node where ganesha.enable "on"
+     * is executed
+     * */
+    if (glusterd_is_ganesha_cluster() &&
+        glusterd_check_ganesha_export(snap_vol)) {
+        if (is_origin_glusterd(dict)) {
+            ret = manage_export_config(clonename, "on", NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                       "Failed to create"
+                       "export file for NFS-Ganesha\n");
+                goto out;
+            }
+        }
+
+        ret = dict_set_dynstr_with_alloc(snap_vol->dict,
+                                         "features.cache-invalidation", "on");
+        ret = gd_ganesha_send_dbus(clonename, "on");
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                   "Dynamic export addition/deletion failed."
+                   " Please see log file for details. Clone name = %s",
+                   clonename);
+            goto out;
+        }
+    }
+    if (!glusterd_is_ganesha_cluster() &&
+        glusterd_check_ganesha_export(snap_vol)) {
+        /* This happens when a snapshot was created when Ganesha was
+         * enabled globally. Then Ganesha disabled from the cluster.
+         * In such cases, we will have the volume level option set
+         * on dict, So we have to disable it as it doesn't make sense
+         * to keep the option.
+         */
 
-        }
+        ret = dict_set_dynstr(snap_vol->dict, "ganesha.enable", "off");
+        if (ret)
+            goto out;
+    }
+
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store snapshot "
+               "volinfo (%s) for snap %s",
+               snap_vol->volname, snap->snapname);
+        goto out;
+    }
+
+    ret = glusterd_copy_quota_files(origin_vol, snap_vol, &conf_present);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_VOL_CONFIG_FAIL,
+               "Failed to copy quota "
+               "config and cksum for volume %s",
+               origin_vol->volname);
+        goto out;
+    }
+
+    if (snap_vol->is_snap_volume) {
+        ret = glusterd_snap_clear_unsupported_opt(snap_vol, unsupported_opt);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to clear quota "
+                   "option for the snap %s (volume: %s)",
+                   snap->snapname, origin_vol->volname);
+            goto out;
+        }
+    }
+
+    ret = generate_brick_volfiles(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the brick "
+               "volfiles for the snap %s (volume: %s) failed",
+               snap->snapname, origin_vol->volname);
+        goto reset_option;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the trusted "
+               "client volfiles for the snap %s (volume: %s) failed",
+               snap->snapname, origin_vol->volname);
+        goto reset_option;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_OTHER);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the client "
+               "volfiles for the snap %s (volume: %s) failed",
+               snap->snapname, origin_vol->volname);
+        goto reset_option;
+    }
 
-        ret = glusterd_copy_nfs_ganesha_file (origin_vol, snap_vol);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_OP_FAILED, "Failed to copy export "
-                        "file for volume %s", origin_vol->volname);
-                goto out;
-        }
-        glusterd_auth_set_username (snap_vol, username);
-        glusterd_auth_set_password (snap_vol, password);
+reset_option:
+    if (snap_vol->is_snap_volume) {
+        if (glusterd_snap_set_unsupported_opt(snap_vol, unsupported_opt)) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to reset quota "
+                   "option for the snap %s (volume: %s)",
+                   snap->snapname, origin_vol->volname);
+        }
+    }
+out:
+    if (ret) {
+        for (i = 0; unsupported_opt[i].key; i++)
+            GF_FREE(unsupported_opt[i].value);
 
-        /* Adding snap brickinfos to the snap volinfo */
-        brick_count = 0;
-        cds_list_for_each_entry (brickinfo, &origin_vol->bricks, brick_list) {
-                ret = glusterd_add_brick_to_snap_volume (dict, rsp_dict,
-                                                         snap_vol, brickinfo,
-                                                         volcount, brick_count,
-                                                         clone);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_ADD_FAIL,
-                                "Failed to add the snap brick for "
-                                "%s:%s to the snap volume",
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
+        if (snap_vol) {
+            if (glusterd_is_ganesha_cluster() &&
+                glusterd_check_ganesha_export(snap_vol)) {
+                if (is_origin_glusterd(dict)) {
+                    ret = manage_export_config(clonename, "on", NULL);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                               "Failed to create"
+                               "export file for NFS-Ganesha\n");
+                    }
                 }
-                brick_count++;
-        }
-
-
-        /* During snapshot creation if I/O is in progress,
-         * then barrier value is enabled. Hence during snapshot create
-         * and in-turn snapshot restore the barrier value is set to enable.
-         * Because of this further I/O on the mount point fails.
-         * Hence remove the barrier key from newly created snap volinfo
-         * before storing and generating the brick volfiles. Also update
-         * the snap vol's version after removing the barrier key.
-         */
-        dict_del (snap_vol->dict, "features.barrier");
-        gd_update_volume_op_versions (snap_vol);
-
-        ret = glusterd_store_volinfo (snap_vol,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_SET_FAIL, "Failed to store snapshot "
-                        "volinfo (%s) for snap %s", snap_vol->volname,
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = glusterd_copy_quota_files (origin_vol, snap_vol, &conf_present);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_VOL_CONFIG_FAIL, "Failed to copy quota "
-                        "config and cksum for volume %s", origin_vol->volname);
-                goto out;
-        }
 
-        if (snap_vol->is_snap_volume) {
-                ret = glusterd_snap_clear_unsupported_opt (snap_vol,
-                                                           unsupported_opt);
+                ret = gd_ganesha_send_dbus(clonename, "off");
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_OP_FAILED, "Failed to clear quota "
-                                "option for the snap %s (volume: %s)",
-                                snap->snapname, origin_vol->volname);
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                           "Dynamic export addition/deletion failed."
+                           " Please see log file for details. Clone name = %s",
+                           clonename);
                 }
-        }
-
-        ret = generate_brick_volfiles (snap_vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "generating the brick "
-                        "volfiles for the snap %s (volume: %s) failed",
-                        snap->snapname, origin_vol->volname);
-                goto reset_option;
-        }
-
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_TRUSTED);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "generating the trusted "
-                        "client volfiles for the snap %s (volume: %s) failed",
-                        snap->snapname, origin_vol->volname);
-                goto reset_option;
-        }
-
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_OTHER);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "generating the client "
-                        "volfiles for the snap %s (volume: %s) failed",
-                        snap->snapname, origin_vol->volname);
-                goto reset_option;
-        }
+            }
 
-reset_option:
-        if (snap_vol->is_snap_volume) {
-                if (glusterd_snap_set_unsupported_opt (snap_vol,
-                                                       unsupported_opt)) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_OP_FAILED, "Failed to reset quota "
-                                "option for the snap %s (volume: %s)",
-                                snap->snapname, origin_vol->volname);
-                }
-        }
-out:
-        if (ret) {
-                if (snap_vol)
-                        glusterd_snap_volume_remove (rsp_dict, snap_vol,
-                                                     _gf_true, _gf_true);
-                snap_vol = NULL;
+            glusterd_snap_volume_remove(rsp_dict, snap_vol, _gf_true, _gf_true);
         }
+        snap_vol = NULL;
+    }
 
-        return snap_vol;
+    return snap_vol;
 }
 
 /*This is the prevalidate function for both activate and deactive of snap
@@ -5482,298 +5472,294 @@ out:
  * For Deactivate operation pass is_op_activate as _gf_false
  * */
 int
-glusterd_snapshot_activate_deactivate_prevalidate (dict_t *dict,
-                                                   char **op_errstr,
-                                                   uint32_t *op_errno,
-                                                   dict_t *rsp_dict,
-                                                   gf_boolean_t is_op_activate)
+glusterd_snapshot_activate_deactivate_prevalidate(dict_t *dict,
+                                                  char **op_errstr,
+                                                  uint32_t *op_errno,
+                                                  dict_t *rsp_dict,
+                                                  gf_boolean_t is_op_activate)
 {
-        int32_t                 ret                   = -1;
-        char                    *snapname             = NULL;
-        xlator_t                *this                 = NULL;
-        glusterd_snap_t         *snap                 = NULL;
-        glusterd_volinfo_t      *snap_volinfo         = NULL;
-        char                    err_str[PATH_MAX]     = "";
-        gf_loglevel_t           loglevel              = GF_LOG_ERROR;
-        glusterd_volume_status  volume_status         = GLUSTERD_STATUS_STOPPED;
-        int                     flags                 = 0;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!dict || !op_errstr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                snprintf (err_str, sizeof (err_str), "Snapshot (%s) does not "
-                          "exist.", snapname);
-                *op_errno = EG_NOSNAP;
-                ret = -1;
-                goto out;
-        }
-
-        /*If its activation of snap then fetch the flags*/
+    int32_t ret = -1;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    char err_str[PATH_MAX] = "";
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    glusterd_volume_status volume_status = GLUSTERD_STATUS_STOPPED;
+    int flags = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot (%s) does not "
+                 "exist.",
+                 snapname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                "Snapname=%s", snapname, NULL);
+        *op_errno = EG_NOSNAP;
+        ret = -1;
+        goto out;
+    }
+
+    /*If its activation of snap then fetch the flags*/
+    if (is_op_activate) {
+        ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get flags");
+            goto out;
+        }
+    }
+
+    /* TODO : As of now there is only volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+    if (!snap_volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOLINFO_GET_FAIL,
+               "Unable to fetch snap_volinfo");
+        ret = -1;
+        goto out;
+    }
+
+    /*TODO: When multiple snapvolume are involved a cumulative
+     * logic is required to tell whether is snapshot is
+     * started/partially started/stopped*/
+    if (is_op_activate) {
+        volume_status = GLUSTERD_STATUS_STARTED;
+    }
+
+    if (snap_volinfo->status == volume_status) {
         if (is_op_activate) {
-                ret = dict_get_int32 (dict, "flags", &flags);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to get flags");
-                        goto out;
-                }
-        }
-
-        /* TODO : As of now there is only volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-        if (!snap_volinfo) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOLINFO_GET_FAIL,
-                        "Unable to fetch snap_volinfo");
+            /* if flag is to GF_CLI_FLAG_OP_FORCE
+             * try to start the snap volume, even
+             * if the volume_status is GLUSTERD_STATUS_STARTED.
+             * By doing so we try to bring
+             * back the brick processes that are down*/
+            if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                snprintf(err_str, sizeof(err_str),
+                         "Snapshot %s is already activated.", snapname);
+                *op_errno = EINVAL;
                 ret = -1;
-                goto out;
-        }
-
-        /*TODO: When multiple snapvolume are involved a cummulative
-         * logic is required to tell whether is snapshot is
-         * started/partially started/stopped*/
-        if (is_op_activate) {
-                volume_status = GLUSTERD_STATUS_STARTED;
-        }
-
-        if (snap_volinfo->status == volume_status) {
-                if (is_op_activate) {
-                        /* if flag is to GF_CLI_FLAG_OP_FORCE
-                         * try to start the snap volume, even
-                         * if the volume_status is GLUSTERD_STATUS_STARTED.
-                         * By doing so we try to bring
-                         * back the brick processes that are down*/
-                        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                                snprintf (err_str, sizeof (err_str),
-                                          "Snapshot %s is already activated.",
-                                          snapname);
-                                *op_errno = EINVAL;
-                                ret = -1;
-                        }
-                } else {
-                        snprintf (err_str, sizeof (err_str),
-                               "Snapshot %s is already deactivated.", snapname);
-                        *op_errno = EINVAL;
-                        ret = -1;
-                }
-                goto out;
-        }
-        ret = 0;
+            }
+        } else {
+            snprintf(err_str, sizeof(err_str),
+                     "Snapshot %s is already deactivated.", snapname);
+            *op_errno = EINVAL;
+            ret = -1;
+        }
+        goto out;
+    }
+    ret = 0;
 out:
 
-        if (ret && err_str[0] != '\0') {
-                gf_msg (this->name, loglevel, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
+    if (ret && err_str[0] != '\0' && op_errstr) {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_handle_snapshot_delete_vol (dict_t *dict, char *err_str,
-                                     uint32_t *op_errno, int len)
+glusterd_handle_snapshot_delete_vol(dict_t *dict, char *err_str,
+                                    uint32_t *op_errno, int len)
 {
-        int32_t                 ret             = -1;
-        int32_t                 i               = 0;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        glusterd_volinfo_t      *temp_volinfo   = NULL;
-        char                    key[PATH_MAX]   =  "";
-        xlator_t                *this           = NULL;
-        char                    *volname        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get "
-                        "volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (err_str, len, "Volume (%s) does not exist", volname);
-                *op_errno = EG_NOVOL;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "Failed to get volinfo of "
-                        "volume %s", volname);
-                goto out;
-        }
-
-        ret = glusterd_snapshot_get_vol_snapnames (dict, volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_LIST_GET_FAIL,
-                        "Failed to get snapshot list for volume %s", volname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get "
+               "volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, len, "Volume (%s) does not exist", volname);
+        *op_errno = EG_NOVOL;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Failed to get volinfo of "
+               "volume %s",
+               volname);
+        goto out;
+    }
+
+    ret = glusterd_snapshot_get_vol_snapnames(dict, volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_LIST_GET_FAIL,
+               "Failed to get snapshot list for volume %s", volname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_handle_snapshot_delete_all (dict_t *dict)
+static int32_t
+glusterd_handle_snapshot_delete_all(dict_t *dict)
 {
-        int32_t         ret             = -1;
-        int32_t         i               =  0;
-        char            key[PATH_MAX]   =  "";
-        glusterd_conf_t *priv           = NULL;
-        glusterd_snap_t *snap           = NULL;
-        glusterd_snap_t *tmp_snap       = NULL;
-        xlator_t        *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                /* indexing from 1 to n, to keep it uniform with other code
-                 * paths
-                 */
-                i++;
-                ret = snprintf (key, sizeof (key), "snapname%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (dict, key, snap->snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Could not save "
-                                "snap name");
-                        goto out;
-                }
+    int32_t ret = -1;
+    int32_t i = 0;
+    char key[32] = "";
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        /* indexing from 1 to n, to keep it uniform with other code
+         * paths
+         */
+        i++;
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
+        if (ret < 0) {
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "snapcount", i);
+        ret = dict_set_dynstr_with_alloc(dict, key, snap->snapname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                               GD_MSG_DICT_SET_FAILED,
-                               "Could not save snapcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap name");
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save snapcount");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_handle_snapshot_delete_type_snap (rpcsvc_request_t *req,
-                                           glusterd_op_t op,
-                                           dict_t *dict, char *err_str,
-                                           uint32_t *op_errno, size_t len)
+glusterd_handle_snapshot_delete_type_snap(rpcsvc_request_t *req,
+                                          glusterd_op_t op, dict_t *dict,
+                                          char *err_str, uint32_t *op_errno,
+                                          size_t len)
 {
-        int32_t                 ret             = -1;
-        int64_t                  volcount       = 0;
-        char                    *snapname       = NULL;
-        char                    *volname        = NULL;
-        char                     key[PATH_MAX]  = "";
-        glusterd_snap_t         *snap           = NULL;
-        glusterd_volinfo_t      *snap_vol       = NULL;
-        glusterd_volinfo_t      *tmp            = NULL;
-        xlator_t                *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Failed to get snapname");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                snprintf (err_str, len, "Snapshot (%s) does not exist",
-                          snapname);
-                *op_errno = EG_NOSNAP;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        /* Set volnames in the dict to get mgmt_v3 lock */
-        cds_list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
-                volcount++;
-                volname = gf_strdup (snap_vol->parent_volname);
-                if (!volname) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                GD_MSG_NO_MEMORY, "strdup failed");
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key), "volname%"PRId64, volcount);
-                ret = dict_set_dynstr (dict, key, volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set "
-                                "volume name in dictionary");
-                        GF_FREE (volname);
-                        goto out;
-                }
-                volname = NULL;
-        }
-        ret = dict_set_int64 (dict, "volcount", volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to set volcount");
-                goto out;
-        }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_INIT_FAIL, "Failed to initiate snap "
-                        "phases");
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    int64_t volcount = 0;
+    char *snapname = NULL;
+    char *volname = NULL;
+    char key[64] = "";
+    int keylen;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get snapname");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        snprintf(err_str, len, "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_NOSNAP;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    /* Set volnames in the dict to get mgmt_v3 lock */
+    cds_list_for_each_entry_safe(snap_vol, tmp, &snap->volumes, vol_list)
+    {
+        volcount++;
+        volname = gf_strdup(snap_vol->parent_volname);
+        if (!volname) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "strdup failed");
+            goto out;
+        }
+
+        keylen = snprintf(key, sizeof(key), "volname%" PRId64, volcount);
+        ret = dict_set_dynstrn(dict, key, keylen, volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "volume name in dictionary");
+            GF_FREE(volname);
+            goto out;
+        }
+        volname = NULL;
+    }
+    ret = dict_set_int64(dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate snap "
+               "phases");
+        goto out;
+    }
+
+    ret = 0;
 
-out :
-        return ret;
+out:
+    return ret;
 }
 
 /* This is a snapshot remove handler function. This function will be
@@ -5789,2844 +5775,2949 @@ out :
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_delete (rpcsvc_request_t *req, glusterd_op_t op,
-                                 dict_t *dict, char *err_str,
-                                 uint32_t *op_errno, size_t len)
+glusterd_handle_snapshot_delete(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, uint32_t *op_errno,
+                                size_t len)
 {
-        int                      ret            = -1;
-        xlator_t                *this           = NULL;
-        int32_t                  delete_cmd     = -1;
+    int ret = -1;
+    xlator_t *this = NULL;
+    int32_t delete_cmd = -1;
 
-        this = THIS;
+    this = THIS;
 
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
 
-        ret = dict_get_int32 (dict, "sub-cmd", &delete_cmd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMAND_NOT_FOUND, "Failed to get sub-cmd");
-                goto out;
-        }
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &delete_cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "Failed to get sub-cmd");
+        goto out;
+    }
 
-        switch (delete_cmd) {
+    switch (delete_cmd) {
         case GF_SNAP_DELETE_TYPE_SNAP:
         case GF_SNAP_DELETE_TYPE_ITER:
-                ret = glusterd_handle_snapshot_delete_type_snap (req, op, dict,
-                                                                 err_str,
-                                                                 op_errno, len);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL, "Failed to handle "
-                                "snapshot delete for type SNAP");
-                        goto out;
-                }
-                break;
+            ret = glusterd_handle_snapshot_delete_type_snap(
+                req, op, dict, err_str, op_errno, len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to handle "
+                       "snapshot delete for type SNAP");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_DELETE_TYPE_ALL:
-                ret = glusterd_handle_snapshot_delete_all (dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL, "Failed to handle "
-                                "snapshot delete for type ALL");
-                        goto out;
-                }
-                break;
+            ret = glusterd_handle_snapshot_delete_all(dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to handle "
+                       "snapshot delete for type ALL");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_DELETE_TYPE_VOL:
-                ret = glusterd_handle_snapshot_delete_vol (dict, err_str,
-                                                           op_errno, len);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL, "Failed to handle "
-                                "snapshot delete for type VOL");
-                        goto out;
-                }
-                break;
+            ret = glusterd_handle_snapshot_delete_vol(dict, err_str, op_errno,
+                                                      len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to handle "
+                       "snapshot delete for type VOL");
+                goto out;
+            }
+            break;
 
         default:
-                *op_errno = EINVAL;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Wrong snapshot delete type");
-                break;
-        }
-
-        if ( ret == 0 && (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
-                          delete_cmd == GF_SNAP_DELETE_TYPE_VOL)) {
-                ret = glusterd_op_send_cli_response (op, 0, 0, req, dict,
-                                                     err_str);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_NO_CLI_RESP, "Failed to send cli "
-                                "response");
-                        goto out;
-                }
-        }
-        ret = 0;
+            *op_errno = EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Wrong snapshot delete type");
+            break;
+    }
+
+    if (ret == 0 && (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
+                     delete_cmd == GF_SNAP_DELETE_TYPE_VOL)) {
+        ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+                   "Failed to send cli "
+                   "response");
+            goto out;
+        }
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_remove_prevalidate (dict_t *dict, char **op_errstr,
-                                      uint32_t *op_errno, dict_t *rsp_dict)
+glusterd_snapshot_remove_prevalidate(dict_t *dict, char **op_errstr,
+                                     uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int32_t             ret         = -1;
-        char               *snapname    = NULL;
-        xlator_t           *this        = NULL;
-        glusterd_snap_t    *snap        = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!dict || !op_errstr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND,
-                        "Snapshot (%s) does not exist", snapname);
-                *op_errno = EG_NOSNAP;
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_NOSNAP;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_status_prevalidate (dict_t *dict, char **op_errstr,
-                                      uint32_t *op_errno, dict_t *rsp_dict)
+glusterd_snapshot_status_prevalidate(dict_t *dict, char **op_errstr,
+                                     uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int                     ret             =       -1;
-        char                    *snapname       =       NULL;
-        glusterd_conf_t         *conf           =       NULL;
-        xlator_t                *this           =       NULL;
-        int32_t                 cmd             =       -1;
-        glusterd_volinfo_t      *volinfo        =       NULL;
-        char                    *volname        =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-        GF_ASSERT (op_errstr);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "Input dict is NULL");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Could not fetch status cmd");
-                goto out;
-        }
-
-        switch (cmd) {
-                case GF_SNAP_STATUS_TYPE_ALL:
-                {
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_SNAP:
-                {
-                        ret = dict_get_str (dict, "snapname", &snapname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Could not fetch snapname");
-                                goto out;
-                        }
-
-                        if (!glusterd_find_snap_by_name (snapname)) {
-                                ret = gf_asprintf (op_errstr, "Snapshot (%s) "
-                                                  "does not exist", snapname);
-                                *op_errno = EG_NOSNAP;
-                                if (ret < 0) {
-                                        goto out;
-                                }
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_SNAP_NOT_FOUND,
-                                        "Snapshot (%s) does not exist",
-                                        snapname);
-                                goto out;
-                        }
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_VOL:
-                {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                       GD_MSG_DICT_GET_FAILED,
-                                        "Could not fetch volname");
-                                goto out;
-                        }
-
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret) {
-                                ret = gf_asprintf (op_errstr, "Volume (%s) "
-                                                  "does not exist", volname);
-                                *op_errno = EG_NOVOL;
-                                if (ret < 0) {
-                                        goto out;
-                                }
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_VOL_NOT_FOUND, "Volume "
-                                        "%s not present", volname);
-                                goto out;
-                        }
-                        break;
-
+    int ret = -1;
+    char *snapname = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int32_t cmd = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "Input dict is NULL");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Could not fetch status cmd");
+        goto out;
+    }
+
+    switch (cmd) {
+        case GF_SNAP_STATUS_TYPE_ALL: {
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_ITER:
+        case GF_SNAP_STATUS_TYPE_SNAP: {
+            ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Could not fetch snapname");
+                goto out;
+            }
+
+            if (!glusterd_find_snap_by_name(snapname)) {
+                ret = gf_asprintf(op_errstr,
+                                  "Snapshot (%s) "
+                                  "does not exist",
+                                  snapname);
+                *op_errno = EG_NOSNAP;
+                if (ret < 0) {
+                    goto out;
                 }
-                default:
-                {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_COMMAND_NOT_FOUND, "Invalid command");
-                        *op_errno = EINVAL;
-                        break;
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+                       "Snapshot (%s) does not exist", snapname);
+                goto out;
+            }
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_VOL: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Could not fetch volname");
+                goto out;
+            }
+
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume (%s) "
+                                  "does not exist",
+                                  volname);
+                *op_errno = EG_NOVOL;
+                if (ret < 0) {
+                    goto out;
                 }
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Volume "
+                       "%s not present",
+                       volname);
+                goto out;
+            }
+            break;
         }
-        ret = 0;
+        default: {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "Invalid command");
+            *op_errno = EINVAL;
+            break;
+        }
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_activate_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_activate_commit(dict_t *dict, char **op_errstr,
+                                  dict_t *rsp_dict)
 {
-        int32_t                   ret                  = -1;
-        char                     *snapname             = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-        glusterd_volinfo_t       *snap_volinfo         = NULL;
-        xlator_t                 *this                 = NULL;
-        int                      flags                 = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        if (!dict || !op_errstr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get flags");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND,
-                        "Snapshot (%s) does not exist", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        /* TODO : As of now there is only volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-        if (!snap_volinfo) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL,
-                                "Unable to fetch snap_volinfo");
-                        ret = -1;
-                        goto out;
-        }
-
-        ret = glusterd_start_volume (snap_volinfo, flags, _gf_true);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_ACTIVATE_FAIL,
-                        "Failed to activate snap volume %s of the snap %s",
-                        snap_volinfo->volname, snap->snapname);
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    int flags = 0;
+    int brick_count = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get flags");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO : As of now there is only volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+    if (!snap_volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Unable to fetch snap_volinfo");
+        ret = -1;
+        goto out;
+    }
+
+    /* create the complete brick here */
+    cds_list_for_each_entry(brickinfo, &snap_volinfo->bricks, brick_list)
+    {
+        brick_count++;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+        ret = glusterd_snap_brick_create(snap_volinfo, brickinfo, brick_count,
+                                         _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_CREATION_FAIL,
+                   "not able to "
+                   "create the brick for the snap %s, volume %s",
+                   snap_volinfo->snapshot->snapname, snap_volinfo->volname);
+            goto out;
+        }
+    }
+
+    ret = glusterd_start_volume(snap_volinfo, flags, _gf_true);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+               "Failed to activate snap volume %s of the snap %s",
+               snap_volinfo->volname, snap->snapname);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_deactivate_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_deactivate_commit(dict_t *dict, char **op_errstr,
+                                    dict_t *rsp_dict)
 {
-        int32_t                   ret                  = -1;
-        char                     *snapname             = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-        glusterd_volinfo_t       *snap_volinfo         = NULL;
-        xlator_t                 *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        if (!dict || !op_errstr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND,
-                        "Snapshot (%s) does not exist", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        /* TODO : As of now there is only volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-        if (!snap_volinfo) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL,
-                                "Unable to fetch snap_volinfo");
-                        ret = -1;
-                        goto out;
-        }
-
-        ret = glusterd_stop_volume (snap_volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_DEACTIVATE_FAIL, "Failed to deactivate"
-                        "snap %s", snapname);
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    xlator_t *this = NULL;
+    char snap_path[PATH_MAX] = "";
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO : As of now there is only volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+    if (!snap_volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Unable to fetch snap_volinfo");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_stop_volume(snap_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_DEACTIVATE_FAIL,
+               "Failed to deactivate"
+               "snap %s",
+               snapname);
+        goto out;
+    }
+
+    ret = glusterd_snap_unmount(this, snap_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+               "Failed to unmounts for %s", snap->snapname);
+    }
+
+    /*Remove /var/run/gluster/snaps/<snap-name> entry for deactivated snaps.
+     * This entry will be created again during snap activate.
+     */
+    snprintf(snap_path, sizeof(snap_path), "%s/%s", snap_mount_dir, snapname);
+    ret = recursive_rmdir(snap_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "%s directory : error : %s",
+               snap_path, strerror(errno));
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_remove_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_remove_commit(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
 {
-        int32_t                   ret                  = -1;
-        char                     *snapname             = NULL;
-        char                     *dup_snapname         = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-        glusterd_conf_t          *priv                 = NULL;
-        glusterd_volinfo_t       *snap_volinfo         = NULL;
-        xlator_t                 *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!dict || !op_errstr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND,
-                        "Snapshot (%s) does not exist", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap uuid in "
-                        "response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        /* Save the snap status as GD_SNAP_STATUS_DECOMMISSION so
-         * that if the node goes down the snap would be removed
+    int32_t ret = -1;
+    char *snapname = NULL;
+    char *dup_snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap uuid in "
+               "response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    /* Save the snap status as GD_SNAP_STATUS_DECOMMISSION so
+     * that if the node goes down the snap would be removed
+     */
+    snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+               "Failed to "
+               "store snap object %s",
+               snap->snapname);
+        goto out;
+    } else
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_SUCCESS,
+               "Successfully marked "
+               "snap %s for decommission.",
+               snap->snapname);
+
+    if (is_origin_glusterd(dict) == _gf_true) {
+        /* TODO : As of now there is only volume in snapshot.
+         * Change this when multiple volume snapshot is introduced
          */
-        snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
-        ret = glusterd_store_snap (snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_OBJECT_STORE_FAIL, "Failed to "
-                        "store snap object %s", snap->snapname);
-                goto out;
-        } else
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_OP_SUCCESS, "Successfully marked "
-                        "snap %s for decommission.", snap->snapname);
-
-        if (is_origin_glusterd (dict) == _gf_true) {
-                /* TODO : As of now there is only volume in snapshot.
-                 * Change this when multiple volume snapshot is introduced
-                 */
-                snap_volinfo = cds_list_entry (snap->volumes.next,
-                                               glusterd_volinfo_t,
-                                               vol_list);
-                if (!snap_volinfo) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL,
-                                "Unable to fetch snap_volinfo");
-                        ret = -1;
-                        goto out;
-                }
-
-                /* From origin glusterd check if      *
-                 * any peers with snap bricks is down */
-                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
-                                                 &priv->peers,
-                                                 GF_SNAP_OPTION_TYPE_DELETE);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_GET_FAIL,
-                                "Failed to find missed snap deletes");
-                        goto out;
-                }
-        }
-
-        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_false,
-                                    _gf_false);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_REMOVE_FAIL, "Failed to remove snap %s",
-                        snapname);
-                goto out;
-        }
-
-        dup_snapname = gf_strdup (snapname);
-        if (!dup_snapname) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Strdup failed");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (rsp_dict, "snapname", dup_snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set the snapname");
-                GF_FREE (dup_snapname);
-                goto out;
-        }
-
-        ret = 0;
+        snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                      vol_list);
+        if (!snap_volinfo) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Unable to fetch snap_volinfo");
+            ret = -1;
+            goto out;
+        }
+
+        /* From origin glusterd check if      *
+         * any peers with snap bricks is down */
+        ret = glusterd_find_missed_snap(rsp_dict, snap_volinfo, &priv->peers,
+                                        GF_SNAP_OPTION_TYPE_DELETE);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_GET_FAIL,
+                   "Failed to find missed snap deletes");
+            goto out;
+        }
+    }
+
+    ret = glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_false, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to remove snap %s", snapname);
+        goto out;
+    }
+
+    dup_snapname = gf_strdup(snapname);
+    if (!dup_snapname) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Strdup failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(rsp_dict, "snapname", dup_snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set the snapname");
+        GF_FREE(dup_snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_do_snap_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_do_snap_cleanup(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t                  ret                   = -1;
-        char                     *name                 = NULL;
-        char                     *volname              = NULL;
-        xlator_t                 *this                 = NULL;
-        glusterd_conf_t          *conf                 = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (!dict || !op_errstr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_INVALID_ENTRY, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to get"
-                        " volume name");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "getting the snap "
-                        "name failed (volume: %s)", volname);
-                goto out;
-        }
-
-        /*
-          If the snapname is not found that means the failure happened at
-          staging, or in commit, before the snap object is created, in which
-          case there is nothing to cleanup. So set ret to 0.
-        */
-        snap = glusterd_find_snap_by_name (name);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_INFO, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND, "Snapshot (%s) does not exist",
-                        name);
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_true,
-                                    _gf_false);
-        if (ret) {
-                /* Ignore failure as this is a cleanup of half cooked
-                   snapshot */
-                gf_msg_debug (this->name, 0, "removing the snap %s failed",
-                        name);
-                ret = 0;
-        }
+    int32_t ret = -1;
+    char *name = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    /* As of now snapshot of multiple volumes are not supported */
+    ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get"
+               " volume name");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "getting the snap "
+               "name failed (volume: %s)",
+               volname);
+        goto out;
+    }
+
+    /*
+      If the snapname is not found that means the failure happened at
+      staging, or in commit, before the snap object is created, in which
+      case there is nothing to cleanup. So set ret to 0.
+    */
+    snap = glusterd_find_snap_by_name(name);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_INFO, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", name);
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_false);
+    if (ret) {
+        /* Ignore failure as this is a cleanup of half cooked
+           snapshot */
+        gf_msg_debug(this->name, 0, "removing the snap %s failed", name);
+        ret = 0;
+    }
 
-        name = NULL;
+    name = NULL;
 
-        ret = 0;
+    ret = 0;
 
 out:
 
-        return ret;
+    return ret;
 }
 
 /* In case of a successful, delete or create operation, during post_validate *
  * look for missed snap operations and update the missed snap lists */
 int32_t
-glusterd_snapshot_update_snaps_post_validate (dict_t *dict, char **op_errstr,
-                                              dict_t *rsp_dict)
+glusterd_snapshot_update_snaps_post_validate(dict_t *dict, char **op_errstr,
+                                             dict_t *rsp_dict)
 {
-        int32_t                  ret                    = -1;
-        int32_t                  missed_snap_count      = -1;
-        xlator_t                *this                   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_int32 (dict, "missed_snap_count",
-                              &missed_snap_count);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "No missed snaps");
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_add_missed_snaps_to_list (dict, missed_snap_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
-                        "Failed to add missed snaps to list");
-                goto out;
-        }
-
-        ret = glusterd_store_update_missed_snaps ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
-                        "Failed to update missed_snaps_list");
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t missed_snap_count = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_int32n(dict, "missed_snap_count", SLEN("missed_snap_count"),
+                          &missed_snap_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No missed snaps");
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_add_missed_snaps_to_list(dict, missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+               "Failed to add missed snaps to list");
+        goto out;
+    }
+
+    ret = glusterd_store_update_missed_snaps();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+               "Failed to update missed_snaps_list");
+        goto out;
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_take_brick_snapshot_task (void *opaque)
+glusterd_take_brick_snapshot_task(void *opaque)
 {
-        int ret                             = 0;
-        snap_create_args_t  *snap_args      = NULL;
-        char                *clonename      = NULL;
-        char                 key[PATH_MAX]  = "";
-
-        GF_ASSERT (opaque);
-
-        snap_args = (snap_create_args_t*) opaque;
-        THIS = snap_args->this;
-
-        /* Try and fetch clonename. If present set status with clonename *
-         * else do so as snap-vol */
-        ret = dict_get_str (snap_args->dict, "clonename", &clonename);
-        if (ret) {
-                snprintf (key, sizeof (key), "snap-vol%d.brick%d.status",
+    int ret = 0;
+    int32_t clone = 0;
+    snap_create_args_t *snap_args = NULL;
+    char *clonename = NULL;
+    char key[64] = "";
+    int keylen;
+
+    GF_ASSERT(opaque);
+
+    snap_args = (snap_create_args_t *)opaque;
+    THIS = snap_args->this;
+
+    /* Try and fetch clonename. If present set status with clonename *
+     * else do so as snap-vol */
+    ret = dict_get_strn(snap_args->dict, "clonename", SLEN("clonename"),
+                        &clonename);
+    if (ret) {
+        keylen = snprintf(key, sizeof(key), "snap-vol%d.brick%d.status",
                           snap_args->volcount, snap_args->brickorder);
-        } else
-                snprintf (key, sizeof (key), "clone%d.brick%d.status",
+    } else {
+        keylen = snprintf(key, sizeof(key), "clone%d.brick%d.status",
                           snap_args->volcount, snap_args->brickorder);
-
-        ret = glusterd_take_brick_snapshot (snap_args->dict,
-                                            snap_args->snap_vol,
-                                            snap_args->brickinfo,
-                                            snap_args->volcount,
-                                            snap_args->brickorder);
-
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Failed to "
-                        "take backend snapshot for brick "
-                        "%s:%s volume(%s)", snap_args->brickinfo->hostname,
-                        snap_args->brickinfo->path,
-                        snap_args->snap_vol->volname);
-        }
-
-        if (dict_set_int32 (snap_args->rsp_dict, key, (ret)?0:1)) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "failed to "
-                        "add %s to dict", key);
-                ret = -1;
-                goto out;
-        }
+        clone = 1;
+    }
+
+    ret = glusterd_take_brick_snapshot(
+        snap_args->dict, snap_args->snap_vol, snap_args->brickinfo,
+        snap_args->volcount, snap_args->brickorder, clone);
+
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to "
+               "take backend snapshot for brick "
+               "%s:%s volume(%s)",
+               snap_args->brickinfo->hostname, snap_args->brickinfo->path,
+               snap_args->snap_vol->volname);
+    }
+
+    if (dict_set_int32n(snap_args->rsp_dict, key, keylen, (ret) ? 0 : 1)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to "
+               "add %s to dict",
+               key);
+        ret = -1;
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_take_brick_snapshot_cbk (int ret, call_frame_t *frame, void *opaque)
+glusterd_take_brick_snapshot_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        snap_create_args_t  *snap_args = NULL;
-        struct syncargs     *args      = NULL;
+    snap_create_args_t *snap_args = NULL;
+    struct syncargs *args = NULL;
 
-        GF_ASSERT (opaque);
+    GF_ASSERT(opaque);
 
-        snap_args = (snap_create_args_t*) opaque;
-        args  = snap_args->args;
+    snap_args = (snap_create_args_t *)opaque;
+    args = snap_args->args;
 
-        if (ret)
-                args->op_ret = ret;
+    if (ret)
+        args->op_ret = ret;
 
-        GF_FREE (opaque);
-        synctask_barrier_wake(args);
-        return 0;
+    GF_FREE(opaque);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-glusterd_schedule_brick_snapshot (dict_t *dict, dict_t *rsp_dict,
-                                  glusterd_snap_t *snap)
+glusterd_schedule_brick_snapshot(dict_t *dict, dict_t *rsp_dict,
+                                 glusterd_snap_t *snap)
 {
-        int                     ret             = -1;
-        int32_t                 volcount        = 0;
-        int32_t                 brickcount      = 0;
-        int32_t                 brickorder      = 0;
-        int32_t                 taskcount       = 0;
-        char                    key[PATH_MAX]   = "";
-        xlator_t               *this            = NULL;
-        glusterd_volinfo_t     *snap_vol        = NULL;
-        glusterd_brickinfo_t   *brickinfo       = NULL;
-        struct syncargs         args            = {0};
-        snap_create_args_t     *snap_args       = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-        GF_ASSERT(snap);
-
-        synctask_barrier_init ((&args));
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                volcount++;
-                brickcount = 0;
-                brickorder = 0;
-                cds_list_for_each_entry (brickinfo, &snap_vol->bricks,
-                                                            brick_list) {
-                        snprintf (key, sizeof(key) - 1,
-                                  "snap-vol%d.brick%d.order", volcount,
-                                  brickcount);
-                        ret = dict_set_int32 (rsp_dict, key, brickorder);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        if ((gf_uuid_compare (brickinfo->uuid, MY_UUID)) ||
-                            (brickinfo->snap_status == -1)) {
-                                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                        brickcount++;
-                                        snprintf (key, sizeof (key),
-                                                  "snap-vol%d.brick%d.status",
-                                                  volcount, brickorder);
-                                        ret = dict_set_int32 (rsp_dict, key, 0);
-                                        if (ret) {
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        GD_MSG_DICT_SET_FAILED,
-                                                        "failed to add %s to "
-                                                        "dict", key);
-                                                goto out;
-                                        }
-                                }
-                                brickorder++;
-                                continue;
-                        }
-
-                        snap_args = GF_CALLOC (1, sizeof (*snap_args),
-                                               gf_gld_mt_snap_create_args_t);
-                        if (!snap_args) {
-                                ret = -1;
-                                goto out;
-                        }
-
-
-                        snap_args->this = this;
-                        snap_args->dict = dict;
-                        snap_args->rsp_dict = rsp_dict;
-                        snap_args->snap_vol = snap_vol;
-                        snap_args->brickinfo = brickinfo;
-                        snap_args->volcount = volcount;
-                        snap_args->brickcount = brickcount;
-                        snap_args->brickorder = brickorder;
-                        snap_args->args = &args;
-
-                        ret = synctask_new (this->ctx->env,
-                                            glusterd_take_brick_snapshot_task,
-                                            glusterd_take_brick_snapshot_cbk,
-                                            NULL, snap_args);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_CREATION_FAIL, "Failed to "
-                                        "spawn task for snapshot create");
-                                GF_FREE (snap_args);
-                                goto out;
-                        }
-                        taskcount++;
-                        brickcount++;
-                        brickorder++;
-                }
-
-                snprintf (key, sizeof (key), "snap-vol%d_brickcount", volcount);
-                ret = dict_set_int64 (rsp_dict, key, brickcount);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "failed to "
-                                "add %s to dict", key);
+    int ret = -1;
+    int32_t volcount = 0;
+    int32_t brickcount = 0;
+    int32_t brickorder = 0;
+    int32_t taskcount = 0;
+    char key[64] = "";
+    int keylen;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    struct syncargs args = {0};
+    snap_create_args_t *snap_args = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(snap);
+
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        volcount++;
+        brickcount = 0;
+        brickorder = 0;
+        cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+        {
+            keylen = snprintf(key, sizeof(key), "snap-vol%d.brick%d.order",
+                              volcount, brickcount);
+            ret = dict_set_int32n(rsp_dict, key, keylen, brickorder);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            if ((gf_uuid_compare(brickinfo->uuid, MY_UUID)) ||
+                (brickinfo->snap_status == -1)) {
+                if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                    brickcount++;
+                    keylen = snprintf(key, sizeof(key),
+                                      "snap-vol%d.brick%d.status", volcount,
+                                      brickorder);
+                    ret = dict_set_int32n(rsp_dict, key, keylen, 0);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_DICT_SET_FAILED,
+                               "failed to add %s to "
+                               "dict",
+                               key);
                         goto out;
+                    }
                 }
-        }
-        synctask_barrier_wait ((&args), taskcount);
-        taskcount = 0;
-
-        if (args.op_ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Failed to create snapshot");
+                brickorder++;
+                continue;
+            }
 
-        ret = args.op_ret;
+            snap_args = GF_CALLOC(1, sizeof(*snap_args),
+                                  gf_gld_mt_snap_create_args_t);
+            if (!snap_args) {
+                ret = -1;
+                goto out;
+            }
+
+            snap_args->this = this;
+            snap_args->dict = dict;
+            snap_args->rsp_dict = rsp_dict;
+            snap_args->snap_vol = snap_vol;
+            snap_args->brickinfo = brickinfo;
+            snap_args->volcount = volcount;
+            snap_args->brickcount = brickcount;
+            snap_args->brickorder = brickorder;
+            snap_args->args = &args;
+
+            ret = synctask_new(
+                this->ctx->env, glusterd_take_brick_snapshot_task,
+                glusterd_take_brick_snapshot_cbk, NULL, snap_args);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Failed to "
+                       "spawn task for snapshot create");
+                GF_FREE(snap_args);
+                goto out;
+            }
+            taskcount++;
+            brickcount++;
+            brickorder++;
+        }
+
+        snprintf(key, sizeof(key), "snap-vol%d_brickcount", volcount);
+        ret = dict_set_int64(rsp_dict, key, brickcount);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to "
+                   "add %s to dict",
+                   key);
+            goto out;
+        }
+    }
+    synctask_barrier_wait((&args), taskcount);
+    taskcount = 0;
+
+    if (args.op_ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to create snapshot");
+
+    ret = args.op_ret;
 out:
-        if (ret && taskcount)
-                synctask_barrier_wait ((&args), taskcount);
+    if (ret && taskcount)
+        synctask_barrier_wait((&args), taskcount);
 
-        return ret;
+    return ret;
 }
 
-glusterd_snap_t*
-glusterd_create_snap_object_for_clone (dict_t *dict, dict_t *rsp_dict)
+glusterd_snap_t *
+glusterd_create_snap_object_for_clone(dict_t *dict, dict_t *rsp_dict)
 {
-        char              *snapname    = NULL;
-        uuid_t            *snap_id     = NULL;
-        glusterd_snap_t   *snap        = NULL;
-        xlator_t          *this        = NULL;
-        glusterd_conf_t   *priv        = NULL;
-        int                ret         = -1;
-        int64_t            time_stamp  = 0;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        /* Fetch snapname, description, id and time from dict */
-        ret = dict_get_str (dict, "clonename", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch clonename");
-                goto out;
-        }
-
-        ret = dict_get_bin (dict, "clone-id", (void **)&snap_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch clone_id");
-                goto out;
-        }
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_OBJ_NEW_FAIL, "Could not create "
-                        "the snap object for snap %s", snapname);
-                goto out;
-        }
-
-        strcpy (snap->snapname, snapname);
-        gf_uuid_copy (snap->snap_id, *snap_id);
-
-        ret = 0;
+    char *snapname = NULL;
+    uuid_t *snap_id = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    /* Fetch snapname, description, id and time from dict */
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch clonename");
+        goto out;
+    }
+
+    ret = dict_get_bin(dict, "clone-id", (void **)&snap_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch clone_id");
+        goto out;
+    }
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJ_NEW_FAIL,
+               "Could not create "
+               "the snap object for snap %s",
+               snapname);
+        goto out;
+    }
+
+    gf_strncpy(snap->snapname, snapname, sizeof(snap->snapname));
+    gf_uuid_copy(snap->snap_id, *snap_id);
+
+    ret = 0;
 
 out:
-        if (ret) {
-                if (snap)
-                        glusterd_snap_remove (rsp_dict, snap,
-                                              _gf_true, _gf_true, _gf_true);
-                snap = NULL;
-        }
+    if (ret) {
+        snap = NULL;
+    }
 
-        return snap;
+    return snap;
 }
 
-
 int32_t
-glusterd_snapshot_clone_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_clone_commit(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                     ret                     = -1;
-        int64_t                 i                       = 0;
-        int64_t                 volcount                = 0;
-        int32_t                 snap_activate           = 0;
-        char                    *snapname               = NULL;
-        char                    *volname                = NULL;
-        char                    *tmp_name               = NULL;
-        char                    key[PATH_MAX]           = "";
-        xlator_t                *this                   = NULL;
-        glusterd_snap_t         *snap_parent            = NULL;
-        glusterd_snap_t         *snap                   = NULL;
-        glusterd_volinfo_t      *origin_vol             = NULL;
-        glusterd_volinfo_t      *snap_vol               = NULL;
-        glusterd_brickinfo_t    *brickinfo              = NULL;
-        glusterd_conf_t         *priv                   = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-        GF_ASSERT(op_errstr);
-        GF_ASSERT(rsp_dict);
-        priv = this->private;
-        GF_ASSERT(priv);
-
-
-        ret = dict_get_str (dict, "clonename", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch clonename");
-                goto out;
-        }
-        tmp_name = gf_strdup (snapname);
-        if (!tmp_name) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (rsp_dict, "clonename", tmp_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set clonename in rsp_dict");
-                GF_FREE (tmp_name);
-                goto out;
-        }
-        tmp_name = NULL;
-
-
-       ret = dict_get_str (dict, "snapname", &volname);
-       if (ret) {
-               gf_msg (this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_DICT_GET_FAILED,
-                      "failed to get snap name");
-               goto out;
-       }
-       snap_parent = glusterd_find_snap_by_name (volname);
-       /* TODO : As of now there is only one volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-       origin_vol = cds_list_entry (snap_parent->volumes.next,
-                                  glusterd_volinfo_t, vol_list);
-       if (!origin_vol) {
-               gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                       GD_MSG_VOLINFO_GET_FAIL, "Failed to get snap "
-                       "volinfo %s", snap_parent->snapname);
-               goto out;
-       }
-       snap = glusterd_create_snap_object_for_clone (dict, rsp_dict);
-       if (!snap) {
-               gf_msg (this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_SNAP_OBJ_NEW_FAIL, "creating the"
-                       "snap object %s failed", snapname);
-               ret = -1;
-               goto out;
-       }
-
-       snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
-                                       rsp_dict, 1, 1);
-       if (!snap_vol) {
-               ret = -1;
-               gf_msg (this->name, GF_LOG_WARNING, 0,
-                       GD_MSG_SNAP_CREATION_FAIL, "taking the "
-                       "snapshot of the volume %s failed", volname);
-               goto out;
-       }
-       volcount = 1;
-       ret = dict_set_int64 (rsp_dict, "volcount", volcount);
-       if (ret) {
-               gf_msg (this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_DICT_SET_FAILED, "Failed to set volcount");
-               goto out;
-       }
-
-       ret = glusterd_schedule_brick_snapshot (dict, rsp_dict, snap);
-       if (ret) {
-               gf_msg (this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_SNAP_BACKEND_MAKE_FAIL, "Failed to take backend "
-                       "snapshot %s", snap->snapname);
-               goto out;
-       }
-
-       cds_list_del_init (&snap_vol->vol_list);
-       ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                         uuid_utoa (snap->snap_id));
-       if (ret) {
-               gf_msg (this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_DICT_SET_FAILED, "Failed to set snap "
-                       "uuid in response dictionary for %s snapshot",
-                       snap->snapname);
-               goto out;
-       }
-
-       glusterd_list_add_order (&snap_vol->vol_list, &priv->volumes,
-                       glusterd_compare_volume_name);
-
-       ret = 0;
-
+    int ret = -1;
+    int64_t volcount = 0;
+    char *snapname = NULL;
+    char *volname = NULL;
+    char *tmp_name = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap_parent = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch clonename");
+        goto out;
+    }
+    tmp_name = gf_strdup(snapname);
+    if (!tmp_name) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(rsp_dict, "clonename", tmp_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set clonename in rsp_dict");
+        GF_FREE(tmp_name);
+        goto out;
+    }
+    tmp_name = NULL;
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get snap name");
+        goto out;
+    }
+
+    snap_parent = glusterd_find_snap_by_name(volname);
+    if (!snap_parent) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Failed to "
+               "fetch snap %s",
+               volname);
+        goto out;
+    }
+
+    /* TODO : As of now there is only one volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    origin_vol = cds_list_entry(snap_parent->volumes.next, glusterd_volinfo_t,
+                                vol_list);
+    if (!origin_vol) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get snap "
+               "volinfo %s",
+               snap_parent->snapname);
+        goto out;
+    }
+
+    snap = glusterd_create_snap_object_for_clone(dict, rsp_dict);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJ_NEW_FAIL,
+               "creating the"
+               "snap object %s failed",
+               snapname);
+        ret = -1;
+        goto out;
+    }
+
+    snap_vol = glusterd_do_snap_vol(origin_vol, snap, dict, rsp_dict, 1, 1);
+    if (!snap_vol) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "taking the "
+               "snapshot of the volume %s failed",
+               volname);
+        goto out;
+    }
+
+    volcount = 1;
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
+
+    ret = glusterd_schedule_brick_snapshot(dict, rsp_dict, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_BACKEND_MAKE_FAIL,
+               "Failed to take backend "
+               "snapshot %s",
+               snap->snapname);
+        goto out;
+    }
+
+    cds_list_del_init(&snap_vol->vol_list);
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap_vol->volume_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    glusterd_list_add_order(&snap_vol->vol_list, &priv->volumes,
+                            glusterd_compare_volume_name);
+
+    ret = 0;
 
 out:
-       if (ret) {
-               if (snap)
-                       glusterd_snap_remove (rsp_dict, snap,
-                                             _gf_true, _gf_true,
-                                             _gf_true);
-               snap = NULL;
-       }
-
-       gf_msg_trace (this->name, 0, "Returning %d", ret);
-       return ret;
-}
+    if (ret) {
+        if (snap)
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_true);
+        snap = NULL;
+    }
 
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
 int32_t
-glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
-                                 uint32_t *op_errno, dict_t *rsp_dict)
+glusterd_snapshot_create_commit(dict_t *dict, char **op_errstr,
+                                uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int                     ret                     = -1;
-        int64_t                 i                       = 0;
-        int64_t                 volcount                = 0;
-        int32_t                 snap_activate           = 0;
-        char                    *snapname               = NULL;
-        char                    *volname                = NULL;
-        char                    *tmp_name               = NULL;
-        char                    key[PATH_MAX]           = "";
-        xlator_t                *this                   = NULL;
-        glusterd_snap_t         *snap                   = NULL;
-        glusterd_volinfo_t      *origin_vol             = NULL;
-        glusterd_volinfo_t      *snap_vol               = NULL;
-        glusterd_brickinfo_t    *brickinfo              = NULL;
-        glusterd_conf_t         *priv                   = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-        GF_ASSERT(op_errstr);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-        GF_ASSERT(rsp_dict);
-        priv = this->private;
-        GF_ASSERT(priv);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to "
-                        "get the volume count");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch snapname");
-                goto out;
-        }
-        tmp_name = gf_strdup (snapname);
-        if (!tmp_name) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        GD_MSG_NO_MEMORY, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (rsp_dict, "snapname", tmp_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Unable to set snapname in rsp_dict");
-                GF_FREE (tmp_name);
-                goto out;
-        }
-        tmp_name = NULL;
-
-        snap = glusterd_create_snap_object (dict, rsp_dict);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "creating the"
-                        "snap object %s failed", snapname);
-                ret = -1;
+    int ret = -1;
+    int64_t i = 0;
+    int64_t volcount = 0;
+    int32_t snap_activate = 0;
+    int32_t flags = 0;
+    char *snapname = NULL;
+    char *volname = NULL;
+    char *tmp_name = NULL;
+    char key[64] = "";
+    int keylen;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snapname");
+        goto out;
+    }
+    tmp_name = gf_strdup(snapname);
+    if (!tmp_name) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(rsp_dict, "snapname", tmp_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snapname in rsp_dict");
+        GF_FREE(tmp_name);
+        goto out;
+    }
+    tmp_name = NULL;
+
+    snap = glusterd_create_snap_object(dict, rsp_dict);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "creating the"
+               "snap object %s failed",
+               snapname);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%" PRId64, i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get volume name");
+            goto out;
+        }
+
+        ret = glusterd_volinfo_find(volname, &origin_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "failed to get the volinfo for "
+                   "the volume %s",
+                   volname);
+            goto out;
+        }
+
+        if (is_origin_glusterd(dict)) {
+            ret = glusterd_is_snap_soft_limit_reached(origin_vol, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+                       "Failed to "
+                       "check soft limit exceeded or not, "
+                       "for volume %s ",
+                       origin_vol->volname);
+                goto out;
+            }
+        }
+
+        snap_vol = glusterd_do_snap_vol(origin_vol, snap, dict, rsp_dict, i, 0);
+        if (!snap_vol) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "taking the "
+                   "snapshot of the volume %s failed",
+                   volname);
+            goto out;
+        }
+    }
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
+
+    ret = glusterd_schedule_brick_snapshot(dict, rsp_dict, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to take backend "
+               "snapshot %s",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    snap_activate = dict_get_str_boolean(
+        priv->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+    if (!snap_activate) {
+        cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+        {
+            snap_vol->status = GLUSTERD_STATUS_STOPPED;
+            ret = glusterd_store_volinfo(snap_vol,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "Failed to store snap volinfo %s", snap_vol->volname);
                 goto out;
+            }
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%"PRId64, i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "failed to get volume name");
-                        goto out;
-                }
+        goto out;
+    }
 
-                ret = glusterd_volinfo_find (volname, &origin_vol);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_FOUND,
-                                "failed to get the volinfo for "
-                                "the volume %s", volname);
-                        goto out;
-                }
+    /* Activate created bricks in case of activate-on-create config. */
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get flags");
+        goto out;
+    }
 
-                if (is_origin_glusterd (dict)) {
-                        ret = glusterd_is_snap_soft_limit_reached (origin_vol,
-                                                                   rsp_dict);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAPSHOT_OP_FAILED, "Failed to "
-                                        "check soft limit exceeded or not, "
-                                        "for volume %s ", origin_vol->volname);
-                                goto out;
-                        }
-                }
-
-                snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
-                                                 rsp_dict, i, 0);
-                if (!snap_vol) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CREATION_FAIL, "taking the "
-                                "snapshot of the volume %s failed", volname);
-                        goto out;
-                }
-        }
-        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set volcount");
-                goto out;
-        }
-
-        ret = glusterd_schedule_brick_snapshot (dict, rsp_dict, snap);
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        ret = glusterd_start_volume(snap_vol, flags, _gf_true);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Failed to take backend "
-                        "snapshot %s", snap->snapname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                   "Failed to activate snap volume %s of the "
+                   "snap %s",
+                   snap_vol->volname, snap->snapname);
+            goto out;
         }
+    }
 
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        snap_activate = dict_get_str_boolean (priv->opts,
-                                              GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                              _gf_false);
-        if (!snap_activate) {
-                cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                        snap_vol->status = GLUSTERD_STATUS_STOPPED;
-                        ret = glusterd_store_volinfo (snap_vol,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOLINFO_SET_FAIL,
-                                        "Failed to store snap volinfo %s",
-                                        snap_vol->volname);
-                                goto out;
-                        }
-                }
-
-                goto out;
-        }
-
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &snap_vol->bricks,
-                                         brick_list) {
-                        ret = glusterd_brick_start (snap_vol, brickinfo,
-                                                    _gf_false);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_BRICK_DISCONNECTED, "starting "
-                                        "the brick %s:%s for the snap %s "
-                                        "(volume: %s) failed",
-                                        brickinfo->hostname, brickinfo->path,
-                                        snap_vol->snapshot->snapname,
-                                        snap_vol->volname);
-                                goto out;
-                        }
-                }
-
-                snap_vol->status = GLUSTERD_STATUS_STARTED;
-                ret = glusterd_store_volinfo (snap_vol,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_SET_FAIL, "Failed to store "
-                                 "snap volinfo %s", snap_vol->volname);
-                        goto out;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret) {
-                if (snap)
-                        glusterd_snap_remove (rsp_dict, snap,
-                                              _gf_true, _gf_true,
-                                              _gf_false);
-                snap = NULL;
-        }
+    if (ret) {
+        if (snap)
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_false);
+        snap = NULL;
+    }
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-snap_max_hard_limit_set_commit (dict_t *dict, uint64_t value,
-                                char *volname, char **op_errstr)
+snap_max_hard_limit_set_commit(dict_t *dict, uint64_t value, char *volname,
+                               char **op_errstr)
 {
-        char                err_str[PATH_MAX]    = "";
-        glusterd_conf_t    *conf                 = NULL;
-        glusterd_volinfo_t *volinfo              = NULL;
-        int                 ret                  = -1;
-        xlator_t           *this                 = NULL;
-        char               *next_version         = NULL;
-
-        this = THIS;
+    char err_str[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    char *next_version = NULL;
 
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
+    this = THIS;
 
-        conf = this->private;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
 
-        GF_ASSERT (conf);
+    conf = this->private;
 
-        /* TODO: Initiate auto deletion when there is a limit change */
-        if (!volname) {
-                /* For system limit */
-                ret = dict_set_uint64 (conf->opts,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                       value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to store "
-                                "%s in the options",
-                                GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-                        goto out;
-                }
+    GF_ASSERT(conf);
 
+    /* TODO: Initiate auto deletion when there is a limit change */
+    if (!volname) {
+        /* For system limit */
+        ret = dict_set_uint64(conf->opts,
+                              GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to store "
+                   "%s in the options",
+                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+            goto out;
+        }
 
-                ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                                &next_version);
-                if (ret)
-                        goto out;
+        ret = glusterd_get_next_global_opt_version_str(conf->opts,
+                                                       &next_version);
+        if (ret)
+            goto out;
 
-                ret = dict_set_str (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                                    next_version);
-                if (ret)
-                        goto out;
+        ret = dict_set_strn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                            SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+        if (ret)
+            goto out;
 
-                ret = glusterd_store_options (this, conf->opts);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_STORE_FAIL, "Failed to store "
-                                "options");
-                        goto out;
-                }
-        } else {
-               /*  For one volume */
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX, "Failed to get the"
-                                   " volinfo for volume %s", volname);
-                        goto out;
-                }
+        ret = glusterd_store_options(this, conf->opts);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_FAIL,
+                   "Failed to store "
+                   "options");
+            goto out;
+        }
+    } else {
+        /*  For one volume */
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, PATH_MAX,
+                     "Failed to get the"
+                     " volinfo for volume %s",
+                     volname);
+            goto out;
+        }
 
-                volinfo->snap_max_hard_limit = value;
+        volinfo->snap_max_hard_limit = value;
 
-                ret = glusterd_store_volinfo (volinfo,
-                                        GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX, "Failed to store "
-                                 "snap-max-hard-limit for volume %s", volname);
-                        goto out;
-                }
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+            snprintf(err_str, PATH_MAX,
+                     "Failed to store "
+                     "snap-max-hard-limit for volume %s",
+                     volname);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret) {
-                *op_errstr = gf_strdup (err_str);
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPSHOT_OP_FAILED, "%s", err_str);
-        }
-        return ret;
+    if (ret) {
+        *op_errstr = gf_strdup(err_str);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+    }
+    return ret;
 }
 
 int
-glusterd_snapshot_config_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_config_commit(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
 {
-        char               *volname              = NULL;
-        xlator_t           *this                 = NULL;
-        int                 ret                  = -1;
-        glusterd_conf_t    *conf                 = NULL;
-        int                 config_command       = 0;
-        uint64_t            hard_limit           = 0;
-        uint64_t            soft_limit           = 0;
-        char               *next_version         = NULL;
-        char               *auto_delete          = NULL;
-        char               *snap_activate        = NULL;
-        gf_boolean_t        system_conf          = _gf_false;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    int config_command = 0;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *next_version = NULL;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+    gf_boolean_t system_conf = _gf_false;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    ret = dict_get_int32n(dict, "config-command", SLEN("config-command"),
+                          &config_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "failed to get config-command type");
+        goto out;
+    }
+    if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
+        ret = 0;
+        goto out;
+    }
 
-        conf = this->private;
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-        GF_ASSERT (conf);
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(dict, &hard_limit, &soft_limit);
 
-        ret = dict_get_int32 (dict, "config-command", &config_command);
+    if (hard_limit) {
+        /* Commit ops for snap-max-hard-limit */
+        ret = snap_max_hard_limit_set_commit(dict, hard_limit, volname,
+                                             op_errstr);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMAND_NOT_FOUND,
-                        "failed to get config-command type");
-                goto out;
-        }
-        if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HARD_LIMIT_SET_FAIL,
+                   "snap-max-hard-limit set commit failed.");
+            goto out;
         }
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
-
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (dict, &hard_limit,
-                                            &soft_limit);
-
-        if (hard_limit) {
-                /* Commit ops for snap-max-hard-limit */
-                ret = snap_max_hard_limit_set_commit (dict, hard_limit, volname,
-                                                      op_errstr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_HARD_LIMIT_SET_FAIL,
-                                "snap-max-hard-limit set commit failed.");
-                        goto out;
-                }
+    if (soft_limit) {
+        /* For system limit */
+        system_conf = _gf_true;
+        ret = dict_set_uint64(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT, soft_limit);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to save %s in the dictionary",
+                   GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+            goto out;
         }
+    }
 
-        if (soft_limit) {
-                /* For system limit */
-                system_conf = _gf_true;
-                ret = dict_set_uint64 (conf->opts,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                                       soft_limit);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Failed to save %s in the dictionary",
-                                GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
-                        goto out;
-                }
-        }
+    if (hard_limit || soft_limit) {
+        ret = 0;
+        goto done;
+    }
+
+    if (!dict_get_strn(dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                       SLEN(GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE),
+                       &auto_delete)) {
+        system_conf = _gf_true;
+        ret = dict_set_dynstr_with_alloc(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, auto_delete);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not "
+                   "save auto-delete value in conf->opts");
+            goto out;
+        }
+    } else if (!dict_get_strn(dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+                              SLEN(GLUSTERD_STORE_KEY_SNAP_ACTIVATE),
+                              &snap_activate)) {
+        system_conf = _gf_true;
+        ret = dict_set_dynstr_with_alloc(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, snap_activate);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap-activate-on-create value in conf->opts");
+            goto out;
+        }
+    } else {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid option");
+        goto out;
+    }
 
-        if (hard_limit || soft_limit) {
-                ret = 0;
-                goto done;
+done:
+    if (system_conf) {
+        ret = glusterd_get_next_global_opt_version_str(conf->opts,
+                                                       &next_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GLOBAL_OP_VERSION_GET_FAIL,
+                   "Failed to get next global opt-version");
+            goto out;
         }
 
-        if (!dict_get_str(dict,
-                          GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                          &auto_delete)) {
-                system_conf = _gf_true;
-                ret = dict_set_dynstr_with_alloc (conf->opts,
-                                GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                auto_delete);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Could not "
-                                "save auto-delete value in conf->opts");
-                        goto out;
-                }
-        } else if (!dict_get_str(dict,
-                                 GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                 &snap_activate)) {
-                system_conf = _gf_true;
-                ret = dict_set_dynstr_with_alloc (conf->opts,
-                                GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                snap_activate);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Could not save "
-                                "snap-activate-on-create value in conf->opts");
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "Invalid option");
-                goto out;
+        ret = dict_set_strn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                            SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GLOBAL_OP_VERSION_SET_FAIL,
+                   "Failed to set next global opt-version");
+            goto out;
         }
 
-done:
-        if (system_conf) {
-                ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                                &next_version);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GLOBAL_OP_VERSION_GET_FAIL,
-                                "Failed to get next global opt-version");
-                                goto out;
-                }
-
-                ret = dict_set_str (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                                    next_version);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GLOBAL_OP_VERSION_SET_FAIL,
-                                "Failed to set next global opt-version");
-                        goto out;
-                }
-
-                ret = glusterd_store_options (this, conf->opts);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_STORE_FAIL,
-                                "Failed to store options");
-                        goto out;
-                }
+        ret = glusterd_store_options(this, conf->opts);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_FAIL,
+                   "Failed to store options");
+            goto out;
         }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int
-glusterd_get_brick_lvm_details (dict_t *rsp_dict,
+static int
+glusterd_get_brick_lvm_details(dict_t *rsp_dict,
                                glusterd_brickinfo_t *brickinfo, char *volname,
-                                char *device, char *key_prefix)
+                               char *device, const char *key_prefix)
 {
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char msg[PATH_MAX] = "";
+    char buf[PATH_MAX] = "";
+    char *ptr = NULL;
+    char *token = NULL;
+    char key[160] = ""; /* key_prefix is 128 bytes at most */
+    char *value = NULL;
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(volname);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    device = glusterd_get_brick_mount_device(brickinfo->path);
+    if (!device) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "Getting device name for "
+               "the brick %s:%s failed",
+               brickinfo->hostname, brickinfo->path);
+        goto out;
+    }
+    runinit(&runner);
+    snprintf(msg, sizeof(msg),
+             "running lvs command, "
+             "for getting snap status");
+    /* Using lvs command fetch the Volume Group name,
+     * Percentage of data filled and Logical Volume size
+     *
+     * "-o" argument is used to get the desired information,
+     * example : "lvs /dev/VolGroup/thin_vol -o vgname,lv_size",
+     * will get us Volume Group name and Logical Volume size.
+     *
+     * Here separator used is ":",
+     * for the above given command with separator ":",
+     * The output will be "vgname:lvsize"
+     */
+    runner_add_args(&runner, LVS, device, "--noheading", "-o",
+                    "vg_name,data_percent,lv_size", "--separator", ":", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    runner_log(&runner, "", GF_LOG_DEBUG, msg);
+    ret = runner_start(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_LVS_FAIL,
+               "Could not perform lvs action");
+        goto end;
+    }
+    do {
+        ptr = fgets(buf, sizeof(buf), runner_chio(&runner, STDOUT_FILENO));
+
+        if (ptr == NULL)
+            break;
+        token = strtok(buf, ":");
+        if (token != NULL) {
+            while (token[0] == ' ')
+                token++;
+            value = gf_strdup(token);
+            if (!value) {
+                ret = -1;
+                goto end;
+            }
+            ret = snprintf(key, sizeof(key), "%s.vgname", key_prefix);
+            if (ret < 0) {
+                goto end;
+            }
 
-        int                     ret             =       -1;
-        glusterd_conf_t         *priv           =       NULL;
-        runner_t                runner          =       {0,};
-        xlator_t                *this           =       NULL;
-        char                    msg[PATH_MAX]   =       "";
-        char                    buf[PATH_MAX]   =       "";
-        char                    *ptr            =       NULL;
-        char                    *token          =       NULL;
-        char                    key[PATH_MAX]   =       "";
-        char                    *value          =       NULL;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (volname);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        device = glusterd_get_brick_mount_device (brickinfo->path);
-        if (!device) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
-                        "Getting device name for "
-                        "the brick %s:%s failed", brickinfo->hostname,
-                         brickinfo->path);
-                goto out;
-        }
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "running lvs command, "
-                  "for getting snap status");
-        /* Using lvs command fetch the Volume Group name,
-         * Percentage of data filled and Logical Volume size
-         *
-         * "-o" argument is used to get the desired information,
-         * example : "lvs /dev/VolGroup/thin_vol -o vgname,lv_size",
-         * will get us Volume Group name and Logical Volume size.
-         *
-         * Here separator used is ":",
-         * for the above given command with separator ":",
-         * The output will be "vgname:lvsize"
-         */
-        runner_add_args (&runner, LVS, device, "--noheading", "-o",
-                         "vg_name,data_percent,lv_size",
-                         "--separator", ":", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        runner_log (&runner, "", GF_LOG_DEBUG, msg);
-        ret = runner_start (&runner);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_LVS_FAIL,
-                        "Could not perform lvs action");
+            ret = dict_set_dynstr(rsp_dict, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save vgname ");
                 goto end;
+            }
         }
-        do {
-                ptr = fgets (buf, sizeof (buf),
-                             runner_chio (&runner, STDOUT_FILENO));
-
-                if (ptr == NULL)
-                        break;
-                token = strtok (buf, ":");
-                if (token != NULL) {
-                        while (token && token[0] == ' ')
-                                token++;
-                        if (!token) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                        GD_MSG_INVALID_ENTRY,
-                                        "Invalid vg entry");
-                                goto end;
-                        }
-                        value = gf_strdup (token);
-                        if (!value) {
-                                ret = -1;
-                                goto end;
-                        }
-                        ret = snprintf (key, sizeof (key), "%s.vgname",
-                                        key_prefix);
-                        if (ret < 0) {
-                                goto end;
-                        }
-
-                        ret = dict_set_dynstr (rsp_dict, key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Could not save vgname ");
-                                goto end;
-                        }
-                }
 
-                token = strtok (NULL, ":");
-                if (token != NULL) {
-                        value = gf_strdup (token);
-                        if (!value) {
-                                ret = -1;
-                                goto end;
-                        }
-                        ret = snprintf (key, sizeof (key), "%s.data",
-                                        key_prefix);
-                        if (ret < 0) {
-                                goto end;
-                        }
-
-                        ret = dict_set_dynstr (rsp_dict, key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Could not save data percent ");
-                                goto end;
-                        }
-                }
-                token = strtok (NULL, ":");
-                if (token != NULL) {
-                        value = gf_strdup (token);
-                        if (!value) {
-                                ret = -1;
-                                goto end;
-                        }
-                        ret = snprintf (key, sizeof (key), "%s.lvsize",
-                                        key_prefix);
-                        if (ret < 0) {
-                                goto end;
-                        }
-
-                        ret = dict_set_dynstr (rsp_dict, key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Could not save meta data percent ");
-                                goto end;
-                        }
-                }
+        token = strtok(NULL, ":");
+        if (token != NULL) {
+            value = gf_strdup(token);
+            if (!value) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                        "token=%s", token, NULL);
+                ret = -1;
+                goto end;
+            }
+            ret = snprintf(key, sizeof(key), "%s.data", key_prefix);
+            if (ret < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                goto end;
+            }
 
-        } while (ptr != NULL);
+            ret = dict_set_dynstr(rsp_dict, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save data percent ");
+                goto end;
+            }
+        }
+        token = strtok(NULL, ":");
+        if (token != NULL) {
+            value = gf_strdup(token);
+            if (!value) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                        "token=%s", token, NULL);
+                ret = -1;
+                goto end;
+            }
+            ret = snprintf(key, sizeof(key), "%s.lvsize", key_prefix);
+            if (ret < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                goto end;
+            }
 
-        ret = 0;
+            ret = dict_set_dynstr(rsp_dict, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save meta data percent ");
+                goto end;
+            }
+        }
+
+    } while (ptr != NULL);
+
+    ret = 0;
 
 end:
-        runner_end (&runner);
+    runner_end(&runner);
 
 out:
-        if (ret && value) {
-                GF_FREE (value);
-        }
+    if (ret && value) {
+        GF_FREE(value);
+    }
 
-        return ret;
+    if (device)
+        GF_FREE(device);
+
+    return ret;
 }
 
-int
-glusterd_get_single_brick_status (char **op_errstr, dict_t *rsp_dict,
-                                 char *keyprefix, int index,
+static int
+glusterd_get_single_brick_status(char **op_errstr, dict_t *rsp_dict,
+                                 const char *keyprefix, int index,
                                  glusterd_volinfo_t *snap_volinfo,
                                  glusterd_brickinfo_t *brickinfo)
 {
-        int             ret                     = -1;
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        char            key[PATH_MAX]           = "";
-        char            *device                 = NULL;
-        char            *value                  = NULL;
-        char            brick_path[PATH_MAX]    = "";
-        char            pidfile[PATH_MAX]       = "";
-        pid_t           pid                     = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (snap_volinfo);
-        GF_ASSERT (brickinfo);
-
-        ret = snprintf (key, sizeof (key), "%s.brick%d.path", keyprefix,
-                        index);
-        if (ret < 0) {
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char key[128] = ""; /* keyprefix is not longer than 64 bytes */
+    int keylen;
+    char *device = NULL;
+    char *value = NULL;
+    char brick_path[PATH_MAX] = "";
+    char pidfile[PATH_MAX] = "";
+    pid_t pid = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(snap_volinfo);
+    GF_ASSERT(brickinfo);
+
+    keylen = snprintf(key, sizeof(key), "%s.brick%d.path", keyprefix, index);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(brick_path, sizeof(brick_path), "%s:%s", brickinfo->hostname,
+                   brickinfo->path);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    value = gf_strdup(brick_path);
+    if (!value) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "brick_path=%s", brick_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstrn(rsp_dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store "
+               "brick_path %s",
+               brickinfo->path);
+        goto out;
+    }
+
+    if (brickinfo->snap_status == -1) {
+        /* Setting vgname as "Pending Snapshot" */
+        value = gf_strdup("Pending Snapshot");
+        if (!value) {
+            ret = -1;
+            goto out;
         }
 
-        ret = snprintf (brick_path, sizeof (brick_path),
-                        "%s:%s", brickinfo->hostname, brickinfo->path);
-        if (ret < 0) {
-                goto out;
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.vgname", keyprefix,
+                          index);
+        ret = dict_set_dynstrn(rsp_dict, key, keylen, value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save vgname ");
+            goto out;
         }
 
-        value = gf_strdup (brick_path);
+        ret = 0;
+        goto out;
+    }
+    value = NULL;
+
+    keylen = snprintf(key, sizeof(key), "%s.brick%d.status", keyprefix, index);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    if (brickinfo->status == GF_BRICK_STOPPED) {
+        value = gf_strdup("No");
         if (!value) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
-
-        ret = dict_set_dynstr (rsp_dict, key, value);
+        ret = dict_set_strn(rsp_dict, key, keylen, value);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Unable to store "
-                        "brick_path %s", brickinfo->path);
-                goto out;
-        }
-
-        if (brickinfo->snap_status == -1) {
-                /* Setting vgname as "Pending Snapshot" */
-                value = gf_strdup ("Pending Snapshot");
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key), "%s.brick%d.vgname",
-                          keyprefix, index);
-                ret = dict_set_dynstr (rsp_dict, key, value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Could not save vgname ");
-                        goto out;
-                }
-
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save brick status");
+            goto out;
         }
         value = NULL;
-
-        ret = snprintf (key, sizeof (key), "%s.brick%d.status",
-                        keyprefix, index);
-        if (ret < 0) {
-                goto out;
-        }
-
-        if (brickinfo->status == GF_BRICK_STOPPED) {
-                value = gf_strdup ("No");
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (rsp_dict, key, value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Could not save brick status");
-                        goto out;
-                }
-                value = NULL;
-        } else {
-                value = gf_strdup ("Yes");
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (rsp_dict, key, value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Could not save brick status");
-                        goto out;
-                }
-                value = NULL;
-
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_volinfo,
-                                            brickinfo, priv);
-                ret = gf_is_service_running (pidfile, &pid);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
-                                keyprefix, index);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_int32 (rsp_dict, key, pid);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Could not save pid %d", pid);
-                        goto out;
-                }
-        }
-
-        ret = snprintf (key, sizeof (key), "%s.brick%d",
-                        keyprefix, index);
-        if (ret < 0) {
-                goto out;
+    } else {
+        value = gf_strdup("Yes");
+        if (!value) {
+            ret = -1;
+            goto out;
         }
-
-        ret = glusterd_get_brick_lvm_details (rsp_dict, brickinfo,
-                                              snap_volinfo->volname,
-                                              device, key);
+        ret = dict_set_strn(rsp_dict, key, keylen, value);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_BRICK_GET_INFO_FAIL, "Failed to get "
-                        "brick LVM details");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save brick status");
+            goto out;
         }
-out:
-        if (ret && value) {
-                GF_FREE (value);
-        }
-
-        return ret;
-}
-
-int
-glusterd_get_single_snap_status (char **op_errstr, dict_t *rsp_dict,
-                                 char *keyprefix, glusterd_snap_t *snap)
-{
-        int                      ret                 =       -1;
-        xlator_t                *this                =       NULL;
-        char                     key[PATH_MAX]       =       "";
-        char                     brickkey[PATH_MAX]  =       "";
-        glusterd_volinfo_t      *snap_volinfo        =       NULL;
-        glusterd_volinfo_t      *tmp_volinfo         =       NULL;
-        glusterd_brickinfo_t    *brickinfo           =       NULL;
-        int                      volcount            =       0;
-        int                      brickcount          =       0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (snap);
-
-        cds_list_for_each_entry_safe (snap_volinfo, tmp_volinfo, &snap->volumes,
-                                      vol_list) {
-                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix,
-                                volcount);
-                if (ret < 0) {
-                        goto out;
-                }
-                cds_list_for_each_entry (brickinfo, &snap_volinfo->bricks,
-                                         brick_list) {
-                        if (!glusterd_is_local_brick (this, snap_volinfo,
-                            brickinfo)) {
-                                brickcount++;
-                                continue;
-                        }
-
-                        ret = glusterd_get_single_brick_status (op_errstr,
-                                                  rsp_dict, key, brickcount,
-                                                  snap_volinfo, brickinfo);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_STATUS_FAIL, "Getting "
-                                        "single snap status failed");
-                                goto out;
-                        }
-                        brickcount++;
-                }
-                ret = snprintf (brickkey, sizeof (brickkey), "%s.brickcount",
-                                key);
-                if (ret < 0) {
-                        goto out;
-                }
+        value = NULL;
 
-                ret = dict_set_int32 (rsp_dict, brickkey, brickcount);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Could not save brick count");
-                        goto out;
-                }
-                volcount++;
-        }
+        GLUSTERD_GET_BRICK_PIDFILE(pidfile, snap_volinfo, brickinfo, priv);
 
-        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
-        if (ret < 0) {
-                goto out;
+        if (gf_is_service_running(pidfile, &pid)) {
+            keylen = snprintf(key, sizeof(key), "%s.brick%d.pid", keyprefix,
+                              index);
+            if (keylen < 0) {
+                ret = -1;
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                goto out;
+            }
+
+            ret = dict_set_int32n(rsp_dict, key, keylen, pid);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save pid %d", pid);
+                goto out;
+            }
+        }
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.brick%d", keyprefix, index);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+    /* While getting snap status we should show relevant information
+     * for deactivated snaps.
+     */
+    if (snap_volinfo->status == GLUSTERD_STATUS_STOPPED) {
+        /* Setting vgname as "Deactivated Snapshot" */
+        value = gf_strdup("N/A (Deactivated Snapshot)");
+        if (!value) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_set_int32 (rsp_dict, key, volcount);
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.vgname", keyprefix,
+                          index);
+        ret = dict_set_dynstrn(rsp_dict, key, keylen, value);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Could not save volcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save vgname ");
+            goto out;
         }
 
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_get_brick_lvm_details(rsp_dict, brickinfo,
+                                         snap_volinfo->volname, device, key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "Failed to get "
+               "brick LVM details");
+        goto out;
+    }
 out:
+    if (ret && value) {
+        GF_FREE(value);
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-glusterd_get_each_snap_object_status (char **op_errstr, dict_t *rsp_dict,
-                                      glusterd_snap_t *snap, char *keyprefix)
+static int
+glusterd_get_single_snap_status(char **op_errstr, dict_t *rsp_dict,
+                                const char *keyprefix, glusterd_snap_t *snap)
 {
-        int                     ret             =       -1;
-        char                    key[PATH_MAX]   =       "";
-        char                    *temp           =       NULL;
-        xlator_t                *this           =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap);
-        GF_ASSERT (keyprefix);
-
-        /* TODO : Get all the snap volume info present in snap object,
-         * as of now, There will be only one snapvolinfo per snap object
-         */
-        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
-        if (ret < 0) {
-                goto out;
-        }
-
-        temp = gf_strdup (snap->snapname);
-        if (temp == NULL) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_dynstr (rsp_dict, key, temp);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Could not save "
-                        "snap name");
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *this = NULL;
+    char key[64] = ""; /* keyprefix is "status.snap0" */
+    int keylen;
+    char brickkey[PATH_MAX] = "";
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int volcount = 0;
+    int brickcount = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(snap);
+
+    cds_list_for_each_entry_safe(snap_volinfo, tmp_volinfo, &snap->volumes,
+                                 vol_list)
+    {
+        keylen = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, volcount);
+        if (keylen < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        cds_list_for_each_entry(brickinfo, &snap_volinfo->bricks, brick_list)
+        {
+            if (!glusterd_is_local_brick(this, snap_volinfo, brickinfo)) {
+                brickcount++;
+                continue;
+            }
 
-        temp = NULL;
+            ret = glusterd_get_single_brick_status(
+                op_errstr, rsp_dict, key, brickcount, snap_volinfo, brickinfo);
 
-        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
-        if (ret < 0) {
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Getting "
+                       "single snap status failed");
                 goto out;
+            }
+            brickcount++;
         }
-
-        temp = gf_strdup (uuid_utoa (snap->snap_id));
-        if (temp == NULL) {
-                ret = -1;
-                goto out;
+        keylen = snprintf(brickkey, sizeof(brickkey), "%s.brickcount", key);
+        if (keylen < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = dict_set_dynstr (rsp_dict, key, temp);
+        ret = dict_set_int32n(rsp_dict, brickkey, keylen, brickcount);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Could not save "
-                        "snap UUID");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save brick count");
+            goto out;
         }
+        volcount++;
+    }
 
-        temp = NULL;
+    keylen = snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_get_single_snap_status (op_errstr, rsp_dict, keyprefix,
-                                               snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_STATUS_FAIL,
-                        "Could not get single snap status");
-                goto out;
-        }
-
-        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
-        if (ret < 0) {
-                goto out;
-        }
+    ret = dict_set_int32n(rsp_dict, key, keylen, volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save volcount");
+        goto out;
+    }
 
-        ret = dict_set_int32 (rsp_dict, key, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Could not save volcount");
-                goto out;
-        }
 out:
-        if (ret && temp)
-                GF_FREE (temp);
 
-        return ret;
+    return ret;
 }
 
-int
-glusterd_get_snap_status_of_volume (char **op_errstr, dict_t *rsp_dict,
-                                    char *volname, char *keyprefix) {
-        int                     ret              =       -1;
-        glusterd_volinfo_t      *snap_volinfo    =       NULL;
-        glusterd_volinfo_t      *temp_volinfo    =       NULL;
-        glusterd_volinfo_t      *volinfo         =       NULL;
-        char                    key[PATH_MAX]    =        "";
-        xlator_t                *this            =       NULL;
-        glusterd_conf_t         *priv            =       NULL;
-        int                     i                =        0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (volname);
-        GF_ASSERT (keyprefix);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND, "Failed to get volinfo of "
-                        "volume %s", volname);
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap_volinfo, temp_volinfo,
-                                      &volinfo->snap_volumes, snapvol_list) {
-                ret = snprintf (key, sizeof (key),
-                                "status.snap%d.snapname", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                    snap_volinfo->snapshot->snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Could not save "
-                                "snap name");
-                        goto out;
-                }
-
-                i++;
-        }
-
-        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to save snapcount");
-                ret = -1;
-                goto out;
-        }
+static int
+glusterd_get_each_snap_object_status(char **op_errstr, dict_t *rsp_dict,
+                                     glusterd_snap_t *snap,
+                                     const char *keyprefix)
+{
+    int ret = -1;
+    char key[32] = ""; /* keyprefix is "status.snap0" */
+    int keylen;
+    char *temp = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap);
+    GF_ASSERT(keyprefix);
+
+    /* TODO : Get all the snap volume info present in snap object,
+     * as of now, There will be only one snapvolinfo per snap object
+     */
+    keylen = snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    temp = gf_strdup(snap->snapname);
+    if (temp == NULL) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_dynstrn(rsp_dict, key, keylen, temp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save "
+               "snap name");
+        goto out;
+    }
+
+    temp = NULL;
+
+    keylen = snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    temp = gf_strdup(uuid_utoa(snap->snap_id));
+    if (temp == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstrn(rsp_dict, key, keylen, temp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save "
+               "snap UUID");
+        goto out;
+    }
+
+    temp = NULL;
+
+    ret = glusterd_get_single_snap_status(op_errstr, rsp_dict, keyprefix, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+               "Could not get single snap status");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+    if (keylen < 0) {
+        ret = keylen;
+        goto out;
+    }
+
+    ret = dict_set_int32n(rsp_dict, key, keylen, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save volcount");
+        goto out;
+    }
 out:
-        return ret;
+    if (ret && temp)
+        GF_FREE(temp);
+
+    return ret;
 }
 
 int
-glusterd_get_all_snapshot_status (dict_t *dict, char **op_errstr,
-                              dict_t *rsp_dict)
+glusterd_get_snap_status_of_volume(char **op_errstr, dict_t *rsp_dict,
+                                   char *volname, char *keyprefix)
 {
-        int32_t                 i               =       0;
-        int                     ret             =       -1;
-        char                    key[PATH_MAX]   =       "";
-        glusterd_conf_t         *priv           =       NULL;
-        glusterd_snap_t         *snap           =       NULL;
-        glusterd_snap_t         *tmp_snap       =       NULL;
-        xlator_t                *this           =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                ret = snprintf (key, sizeof (key),
-                                "status.snap%d.snapname", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  snap->snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Could not save "
-                                "snap name");
-                        goto out;
-                }
-
-                i++;
+    int ret = -1;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *temp_volinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char key[64] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int i = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(volname);
+    GF_ASSERT(keyprefix);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Failed to get volinfo of "
+               "volume %s",
+               volname);
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap_volinfo, temp_volinfo,
+                                 &volinfo->snap_volumes, snapvol_list)
+    {
+        ret = snprintf(key, sizeof(key), "status.snap%d.snapname", i);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key,
+                                         snap_volinfo->snapshot->snapname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Could not save snapcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap name");
+            goto out;
         }
 
-        ret = 0;
-out :
-        return ret;
-}
+        i++;
+    }
 
+    ret = dict_set_int32n(rsp_dict, "status.snapcount",
+                          SLEN("status.snapcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to save snapcount");
+        ret = -1;
+        goto out;
+    }
+out:
+    return ret;
+}
 
 int
-glusterd_snapshot_status_commit (dict_t *dict, char **op_errstr,
+glusterd_get_all_snapshot_status(dict_t *dict, char **op_errstr,
                                  dict_t *rsp_dict)
 {
-        xlator_t        *this             =       NULL;
-        int             ret               =       -1;
-        glusterd_conf_t *conf             =       NULL;
-        char            *get_buffer       =       NULL;
-        int32_t         cmd               =       -1;
-        char            *snapname         =       NULL;
-        glusterd_snap_t *snap             =       NULL;
-        char            *volname          =       NULL;
+    int32_t i = 0;
+    int ret = -1;
+    char key[64] = "";
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        ret = snprintf(key, sizeof(key), "status.snap%d.snapname", i);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
+        }
 
-        this = THIS;
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, snap->snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap name");
+            goto out;
+        }
 
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
+        i++;
+    }
 
-        conf = this->private;
+    ret = dict_set_int32n(rsp_dict, "status.snapcount",
+                          SLEN("status.snapcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save snapcount");
+        goto out;
+    }
 
-        GF_ASSERT (conf);
-        ret = dict_get_int32 (dict, "sub-cmd", &cmd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Failed to get status cmd type");
-                goto out;
-        }
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = dict_set_int32 (rsp_dict, "sub-cmd", cmd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED,
-                        "Could not save status cmd in rsp dictionary");
-                goto out;
-        }
-        switch (cmd) {
-                case GF_SNAP_STATUS_TYPE_ALL:
-                {
-                        ret = glusterd_get_all_snapshot_status (dict, op_errstr,
-                                                           rsp_dict);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_STATUS_FAIL, "Unable to "
-                                        "get snapshot status");
-                                goto out;
-                        }
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_SNAP:
-                {
-
-                        ret = dict_get_str (dict, "snapname", &snapname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "Unable to "
-                                        "get snap name");
-                                goto out;
-                        }
-
-                        snap = glusterd_find_snap_by_name (snapname);
-                        if (!snap) {
-                                ret = gf_asprintf (op_errstr, "Snapshot (%s) "
-                                                  "does not exist", snapname);
-                                if (ret < 0) {
-                                        goto out;
-                                }
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_VOLINFO_GET_FAIL, "Unable to "
-                                        "get snap volinfo");
-                                goto out;
-                        }
-                        ret = glusterd_get_each_snap_object_status (op_errstr,
-                                          rsp_dict, snap, "status.snap0");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_STATUS_FAIL, "Unable to "
-                                        "get status of snap %s", get_buffer);
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (rsp_dict, "status.snapcount", 1);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "Unable to "
-                                        "set snapcount to 1");
-                                goto out;
-                        }
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_VOL:
-                {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "Unable to"
-                                        " get volume name");
-                                goto out;
-                        }
-
-                        ret = glusterd_get_snap_status_of_volume (op_errstr,
-                                           rsp_dict, volname, "status.vol0");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_STATUS_FAIL, "Function :"
-                                        " glusterd_get_snap_status_of_volume "
-                                        "failed");
-                                goto out;
-                        }
+int
+glusterd_snapshot_status_commit(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
+{
+    xlator_t *this = NULL;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    int32_t cmd = -1;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get status cmd type");
+        goto out;
+    }
+
+    ret = dict_set_int32n(rsp_dict, "sub-cmd", SLEN("sub-cmd"), cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save status cmd in rsp dictionary");
+        goto out;
+    }
+    switch (cmd) {
+        case GF_SNAP_STATUS_TYPE_ALL: {
+            ret = glusterd_get_all_snapshot_status(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Unable to "
+                       "get snapshot status");
+                goto out;
+            }
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_ITER:
+        case GF_SNAP_STATUS_TYPE_SNAP: {
+            ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to "
+                       "get snap name");
+                goto out;
+            }
+
+            snap = glusterd_find_snap_by_name(snapname);
+            if (!snap) {
+                ret = gf_asprintf(op_errstr,
+                                  "Snapshot (%s) "
+                                  "does not exist",
+                                  snapname);
+                if (ret < 0) {
+                    goto out;
                 }
-        }
-        ret = 0;
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                       "Unable to "
+                       "get snap volinfo");
+                goto out;
+            }
+            ret = glusterd_get_each_snap_object_status(op_errstr, rsp_dict,
+                                                       snap, "status.snap0");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Unable to "
+                       "get status of snap");
+                goto out;
+            }
+
+            ret = dict_set_int32n(rsp_dict, "status.snapcount",
+                                  SLEN("status.snapcount"), 1);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to "
+                       "set snapcount to 1");
+                goto out;
+            }
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_VOL: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to"
+                       " get volume name");
+                goto out;
+            }
+
+            ret = glusterd_get_snap_status_of_volume(op_errstr, rsp_dict,
+                                                     volname, "status.vol0");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Function :"
+                       " glusterd_get_snap_status_of_volume "
+                       "failed");
+                goto out;
+            }
+        }
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_handle_snap_limit (dict_t *dict, dict_t *rsp_dict)
+glusterd_handle_snap_limit(dict_t *dict, dict_t *rsp_dict)
 {
-        int32_t             ret                 = -1;
-        xlator_t           *this                = NULL;
-        glusterd_conf_t    *priv                = NULL;
-        uint64_t            effective_max_limit = 0;
-        int64_t             volcount            = 0;
-        int                 i                   = 0;
-        char               *volname             = NULL;
-        char                key[PATH_MAX]       = {0, };
-        glusterd_volinfo_t *volinfo             = NULL;
-        uint64_t            limit               = 0;
-        int64_t             count               = 0;
-        glusterd_snap_t    *snap                = NULL;
-        glusterd_volinfo_t *tmp_volinfo         = NULL;
-        glusterd_volinfo_t *other_volinfo       = NULL;
-        uint64_t            opt_max_hard        = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t            opt_max_soft        = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "failed to get the volcount");
-                goto out;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uint64_t effective_max_limit = 0;
+    int64_t volcount = 0;
+    int i = 0;
+    char *volname = NULL;
+    char key[64] = "";
+    int keylen;
+    char msg[PATH_MAX] = "";
+    glusterd_volinfo_t *volinfo = NULL;
+    uint64_t limit = 0;
+    int64_t count = 0;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    uint64_t opt_max_hard = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_max_soft = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the volcount");
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get the "
+                   "volname");
+            goto out;
+        }
+
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "volinfo for %s "
+                   "not found",
+                   volname);
+            goto out;
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "failed to get the "
-                                "volname");
-                        goto out;
-                }
+        /* config values snap-max-hard-limit and snap-max-soft-limit are
+         * optional and hence we are not erroring out if values are not
+         * present
+         */
+        gd_get_snap_conf_values_if_present(priv->opts, &opt_max_hard,
+                                           &opt_max_soft);
 
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_VOL_NOT_FOUND, "volinfo for %s "
-                                "not found", volname);
-                        goto out;
-                }
+        /* The minimum of the 2 limits i.e system wide limit and
+           volume wide limit will be considered
+        */
+        if (volinfo->snap_max_hard_limit < opt_max_hard)
+            effective_max_limit = volinfo->snap_max_hard_limit;
+        else
+            effective_max_limit = opt_max_hard;
 
-                /* config values snap-max-hard-limit and snap-max-soft-limit are
-                 * optional and hence we are not erroring out if values are not
-                 * present
-                 */
-                gd_get_snap_conf_values_if_present (priv->opts, &opt_max_hard,
-                                                    &opt_max_soft);
+        limit = (opt_max_soft * effective_max_limit) / 100;
 
-                /* The minimum of the 2 limits i.e system wide limit and
-                   volume wide limit will be considered
-                */
-                if (volinfo->snap_max_hard_limit < opt_max_hard)
-                        effective_max_limit = volinfo->snap_max_hard_limit;
-                else
-                        effective_max_limit = opt_max_hard;
+        count = volinfo->snap_count - limit;
+        if (count <= 0)
+            goto out;
 
-                limit = (opt_max_soft * effective_max_limit)/100;
+        tmp_volinfo = cds_list_entry(volinfo->snap_volumes.next,
+                                     glusterd_volinfo_t, snapvol_list);
+        snap = tmp_volinfo->snapshot;
+        GF_ASSERT(snap);
 
-                count = volinfo->snap_count - limit;
-                if (count <= 0)
-                        goto out;
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SOFT_LIMIT_REACHED,
+               "Soft-limit "
+               "(value = %" PRIu64
+               ") of volume %s is reached. "
+               "Deleting snapshot %s.",
+               limit, volinfo->volname, snap->snapname);
 
-                tmp_volinfo = cds_list_entry (volinfo->snap_volumes.next,
-                                              glusterd_volinfo_t, snapvol_list);
-                snap = tmp_volinfo->snapshot;
-                GF_ASSERT (snap);
-
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SOFT_LIMIT_REACHED, "Soft-limit "
-                        "(value = %"PRIu64") of volume %s is reached. "
-                        "Deleting snapshot %s.", limit, volinfo->volname,
-                        snap->snapname);
-
-                LOCK (&snap->lock);
-                {
-                        snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
-                        ret = glusterd_store_snap (snap);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_OBJECT_STORE_FAIL, "could "
-                                        "not store snap object %s",
-                                        snap->snapname);
-                                goto unlock;
-                        }
-
-                        ret = glusterd_snap_remove (rsp_dict, snap,
-                                                    _gf_true, _gf_true,
-                                                    _gf_false);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_SNAP_REMOVE_FAIL,
-                                        "failed to remove snap %s",
-                                        snap->snapname);
-                }
-        unlock: UNLOCK (&snap->lock);
+        snprintf(msg, sizeof(msg),
+                 "snapshot_name=%s;"
+                 "snapshot_uuid=%s",
+                 snap->snapname, uuid_utoa(snap->snap_id));
+
+        LOCK(&snap->lock);
+        {
+            snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
+            ret = glusterd_store_snap(snap);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SNAP_OBJECT_STORE_FAIL,
+                       "could "
+                       "not store snap object %s",
+                       snap->snapname);
+                goto unlock;
+            }
+
+            ret = glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true,
+                                       _gf_false);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "failed to remove snap %s", snap->snapname);
         }
+    unlock:
+        UNLOCK(&snap->lock);
+        if (is_origin_glusterd(dict) == _gf_true) {
+            if (ret)
+                gf_event(EVENT_SNAPSHOT_DELETE_FAILED, "%s", msg);
+            else
+                gf_event(EVENT_SNAPSHOT_DELETED, "%s", msg);
+        }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_clone_postvalidate (dict_t *dict, int32_t op_ret,
-                                       char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_clone_postvalidate(dict_t *dict, int32_t op_ret,
+                                     char **op_errstr, dict_t *rsp_dict)
 {
-        xlator_t              *this           = NULL;
-        glusterd_conf_t       *priv           = NULL;
-        int                    ret            = -1;
-        int32_t                cleanup        = 0;
-        glusterd_snap_t       *snap           = NULL;
-        glusterd_volinfo_t    *snap_vol       = NULL;
-        char                  *clonename      = NULL;
-        char                  *auto_delete    = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "clonename", &clonename);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch "
-                        "clonename");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (clonename, &snap_vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND, "unable to find clone "
-                        "%s volinfo", clonename);
-                goto out;
-        }
-
-        if (snap_vol)
-                snap = snap_vol->snapshot;
-
-        /* Fetch snap object from snap_vol and delete it all in case of *
-         * a failure, or else, just delete the snap object as it is not *
-         * needed in case of a clone                                    *
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int32_t cleanup = 0;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    char *clonename = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch "
+               "clonename");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(clonename, &snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "unable to find clone "
+               "%s volinfo",
+               clonename);
+        goto out;
+    }
+
+    if (snap_vol)
+        snap = snap_vol->snapshot;
+    else {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot volume is null");
+        goto out;
+    }
+
+    /* Fetch snap object from snap_vol and delete it all in case of *
+     * a failure, or else, just delete the snap object as it is not *
+     * needed in case of a clone                                    *
+     */
+    if (op_ret) {
+        ret = dict_get_int32n(dict, "cleanup", SLEN("cleanup"), &cleanup);
+        if (!ret && cleanup && snap) {
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_true);
+        }
+        /* Irrespective of status of cleanup its better
+         * to return from this function. As the functions
+         * following this block is not required to be
+         * executed in case of failure scenario.
          */
-        if (op_ret) {
-                ret = dict_get_int32 (dict, "cleanup", &cleanup);
-                if (!ret && cleanup && snap) {
-                        glusterd_snap_remove (rsp_dict, snap,
-                                              _gf_true, _gf_true,
-                                              _gf_true);
-                }
-                /* Irrespective of status of cleanup its better
-                 * to return from this function. As the functions
-                 * following this block is not required to be
-                 * executed in case of failure scenario.
-                 */
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_snapobject_delete (snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_REMOVE_FAIL, "Failed to delete "
-                        "snap object %s", snap->snapname);
-                goto out;
-        }
-        snap_vol->snapshot = NULL;
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_snapobject_delete(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to delete "
+               "snap object %s",
+               snap->snapname);
+        goto out;
+    }
+    snap_vol->snapshot = NULL;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-glusterd_snapshot_create_postvalidate (dict_t *dict, int32_t op_ret,
-                                       char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_create_postvalidate(dict_t *dict, int32_t op_ret,
+                                      char **op_errstr, dict_t *rsp_dict)
 {
-        xlator_t        *this           = NULL;
-        glusterd_conf_t *priv           = NULL;
-        int              ret            = -1;
-        int32_t          cleanup        = 0;
-        glusterd_snap_t *snap           = NULL;
-        char            *snapname       = NULL;
-        char            *auto_delete    = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (op_ret) {
-                ret = dict_get_int32 (dict, "cleanup", &cleanup);
-                if (!ret && cleanup) {
-                        ret = glusterd_do_snap_cleanup (dict, op_errstr,
-                                                        rsp_dict);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_SNAP_CLEANUP_FAIL, "cleanup "
-                                        "operation failed");
-                                goto out;
-                        }
-                }
-                /* Irrespective of status of cleanup its better
-                 * to return from this function. As the functions
-                 * following this block is not required to be
-                 * executed in case of failure scenario.
-                 */
-                ret = 0;
-                goto out;
-        }
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int32_t cleanup = 0;
+    glusterd_snap_t *snap = NULL;
+    char *snapname = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_max_soft = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    int64_t effective_max_limit = 0;
+    int64_t soft_limit = 0;
+    int32_t snap_activate = _gf_false;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (op_ret) {
+        ret = dict_get_int32n(dict, "cleanup", SLEN("cleanup"), &cleanup);
+        if (!ret && cleanup) {
+            ret = glusterd_do_snap_cleanup(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CLEANUP_FAIL,
+                       "cleanup "
+                       "operation failed");
+                goto out;
+            }
+        }
+        /* Irrespective of status of cleanup its better
+         * to return from this function. As the functions
+         * following this block is not required to be
+         * executed in case of failure scenario.
+         */
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch "
+               "snapname");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "unable to find snap "
+               "%s",
+               snapname);
+        goto out;
+    }
+
+    snap->snap_status = GD_SNAP_STATUS_IN_USE;
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+               "Could not store snap"
+               "object %s",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = glusterd_snapshot_update_snaps_post_validate(dict, op_errstr,
+                                                       rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to "
+               "create snapshot");
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "Unable to fetch "
-                        "snapname");
-                goto out;
-        }
+    /*
+     * If activate_on_create was enabled, and we have reached this  *
+     * section of the code, that means, that after successfully     *
+     * creating the snapshot, we have also successfully started the *
+     * snapshot bricks on all nodes. So from originator node we can *
+     * send EVENT_SNAPSHOT_ACTIVATED event.                         *
+     *                                                              *
+     * Also check, if hard limit and soft limit is reached in case  *
+     * of successfully creating the snapshot, and generate the event *
+     */
+    if (is_origin_glusterd(dict) == _gf_true) {
+        snap_activate = dict_get_str_boolean(
+            priv->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
 
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_NOT_FOUND, "unable to find snap "
-                        "%s", snapname);
-                goto out;
+        if (snap_activate == _gf_true) {
+            gf_event(EVENT_SNAPSHOT_ACTIVATED,
+                     "snapshot_name=%s;"
+                     "snapshot_uuid=%s",
+                     snap->snapname, uuid_utoa(snap->snap_id));
         }
 
-        snap->snap_status = GD_SNAP_STATUS_IN_USE;
-        ret = glusterd_store_snap (snap);
+        ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_OBJECT_STORE_FAIL, "Could not store snap"
-                        "object %s", snap->snapname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get volname.");
+            goto out;
         }
 
-        ret = glusterd_snapshot_update_snaps_post_validate (dict,
-                                                            op_errstr,
-                                                            rsp_dict);
+        ret = glusterd_volinfo_find(volname, &volinfo);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CREATION_FAIL, "Failed to "
-                        "create snapshot");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                   "Failed to get volinfo.");
+            goto out;
         }
 
-        /* "auto-delete" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
+        /* config values snap-max-hard-limit and snap-max-soft-limit are
+         * optional and hence we are not erroring out if values are not
+         * present
          */
-        ret = dict_get_str_boolean (priv->opts,
-                                    GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                    _gf_false);
-        if ( _gf_true == ret ) {
-                //ignore the errors of autodelete
-                ret = glusterd_handle_snap_limit (dict, rsp_dict);
-        }
+        gd_get_snap_conf_values_if_present(priv->opts, &opt_hard_max,
+                                           &opt_max_soft);
 
-        ret = glusterd_snapshot_resume_tier (this, dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_RESUME_TIER_FAIL,
-                        "Failed to resume tier in snapshot postvalidate.");
-        }
+        if (volinfo->snap_max_hard_limit < opt_hard_max)
+            effective_max_limit = volinfo->snap_max_hard_limit;
+        else
+            effective_max_limit = opt_hard_max;
+
+        /*
+         * Check for hard limit. If it is reached after taking *
+         * this snapshot, then generate event for the same. If *
+         * it is not reached, then check for the soft limit,   *
+         * and generate event accordingly.                     *
+         */
+        if (volinfo->snap_count >= effective_max_limit) {
+            gf_event(EVENT_SNAPSHOT_HARD_LIMIT_REACHED,
+                     "volume_name=%s;volume_id=%s", volname,
+                     uuid_utoa(volinfo->volume_id));
+        } else {
+            soft_limit = (opt_max_soft * effective_max_limit) / 100;
+            if (volinfo->snap_count >= soft_limit) {
+                gf_event(EVENT_SNAPSHOT_SOFT_LIMIT_REACHED,
+                         "volume_name=%s;volume_id=%s", volname,
+                         uuid_utoa(volinfo->volume_id));
+            }
+        }
+    }
+
+    /* "auto-delete" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                               _gf_false);
+    if (_gf_true == ret) {
+        ret = glusterd_handle_snap_limit(dict, rsp_dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "failed to remove snap");
+            /* ignore the errors of autodelete */
+            ret = 0;
+        }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot (dict_t *dict, char **op_errstr,
-                   uint32_t *op_errno, dict_t *rsp_dict)
+glusterd_snapshot(dict_t *dict, char **op_errstr, uint32_t *op_errno,
+                  dict_t *rsp_dict)
 {
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t snap_command = 0;
+    char *snap_name = NULL;
+    char temp[PATH_MAX] = "";
+    int ret = -1;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+            ret = glusterd_snapshot_create_commit(dict, op_errstr, op_errno,
+                                                  rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Failed to "
+                       "create snapshot");
+                goto out;
+            }
+            break;
 
-        xlator_t        *this           = NULL;
-        glusterd_conf_t *priv           = NULL;
-        int32_t          snap_command   = 0;
-        char            *snap_name      = NULL;
-        char             temp[PATH_MAX] = "";
-        int              ret            = -1;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMAND_NOT_FOUND, "unable to get the type of "
-                        "the snapshot command");
+        case (GF_SNAP_OPTION_TYPE_CLONE):
+            ret = glusterd_snapshot_clone_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CLONE_FAILED,
+                       "Failed to "
+                       "clone snapshot");
                 goto out;
-        }
+            }
+            break;
 
-        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            ret = glusterd_snapshot_config_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CONFIG_FAIL,
+                       "snapshot config failed");
+                goto out;
+            }
+            break;
 
-        case (GF_SNAP_OPTION_TYPE_CREATE):
-                ret = glusterd_snapshot_create_commit (dict, op_errstr,
-                                                       op_errno, rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_CREATION_FAIL, "Failed to "
-                                "create snapshot");
-                        goto out;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = glusterd_snapshot_remove_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to "
+                       "delete snapshot");
+                if (*op_errstr) {
+                    /* If error string is already set
+                     * then goto out */
+                    goto out;
                 }
-                break;
 
-       case (GF_SNAP_OPTION_TYPE_CLONE):
-                ret = glusterd_snapshot_clone_commit (dict, op_errstr,
-                                                       rsp_dict);
+                ret = dict_get_strn(dict, "snapname", SLEN("snapname"),
+                                    &snap_name);
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_CLONE_FAILED, "Failed to "
-                                "clone snapshot");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Failed to get snapname");
+                    snap_name = "NA";
                 }
-                break;
 
-        case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = glusterd_snapshot_config_commit (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_CONFIG_FAIL,
-                                "snapshot config failed");
-                        goto out;
-                }
-                break;
+                snprintf(temp, sizeof(temp),
+                         "Snapshot %s might "
+                         "not be in an usable state.",
+                         snap_name);
 
-        case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_snapshot_remove_commit (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL, "Failed to "
-                                "delete snapshot");
-                        if (*op_errstr) {
-                                /* If error string is already set
-                                 * then goto out */
-                                goto out;
-                        }
-
-                        ret = dict_get_str (dict, "snapname", &snap_name);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Failed to get snapname");
-                                snap_name = "NA";
-                        }
-
-                        snprintf (temp, sizeof (temp), "Snapshot %s might "
-                                  "not be in an usable state.", snap_name);
-
-                        *op_errstr = gf_strdup (temp);
-                        ret = -1;
-                        goto out;
-                }
-                break;
+                *op_errstr = gf_strdup(temp);
+                ret = -1;
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snapshot_restore (dict, op_errstr,
-                                                 rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_RESTORE_FAIL, "Failed to "
-                                "restore snapshot");
-                        goto out;
-                }
+            ret = glusterd_snapshot_restore(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Failed to "
+                       "restore snapshot");
+                goto out;
+            }
 
-                break;
-        case  GF_SNAP_OPTION_TYPE_ACTIVATE:
-                ret = glusterd_snapshot_activate_commit (dict, op_errstr,
-                                                 rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_ACTIVATE_FAIL, "Failed to "
-                                "activate snapshot");
-                        goto out;
-                }
+            break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            ret = glusterd_snapshot_activate_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                       "Failed to "
+                       "activate snapshot");
+                goto out;
+            }
 
-                break;
+            break;
 
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = glusterd_snapshot_deactivate_commit (dict, op_errstr,
-                                                 rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_DEACTIVATE_FAIL, "Failed to "
-                                "deactivate snapshot");
-                        goto out;
-                }
+            ret = glusterd_snapshot_deactivate_commit(dict, op_errstr,
+                                                      rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_DEACTIVATE_FAIL,
+                       "Failed to "
+                       "deactivate snapshot");
+                goto out;
+            }
 
-                break;
+            break;
 
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = glusterd_snapshot_status_commit (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_STATUS_FAIL, "Failed to "
-                                "show snapshot status");
-                        goto out;
-                }
-                break;
-
+            ret = glusterd_snapshot_status_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Failed to "
+                       "show snapshot status");
+                goto out;
+            }
+            break;
 
         default:
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "invalid snap command");
-                goto out;
-                break;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "invalid snap command");
+            goto out;
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_brickop(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int            ret       = -1;
-        int64_t        vol_count = 0;
-        int64_t        count     = 1;
-        char           key[1024] = {0,};
-        char           *volname  = NULL;
-        int32_t        snap_command = 0;
-        xlator_t       *this     = NULL;
-        char           *op_type  = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMAND_NOT_FOUND, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int ret = -1;
+    int64_t vol_count = 0;
+    int64_t count = 1;
+    char key[64] = "";
+    int keylen;
+    char *volname = NULL;
+    int32_t snap_command = 0;
+    xlator_t *this = NULL;
+    char *op_type = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
 
-                /* op_type with tell us whether its pre-commit operation
-                 * or post-commit
+            /* op_type with tell us whether its pre-commit operation
+             * or post-commit
+             */
+            ret = dict_get_strn(dict, "operation-type", SLEN("operation-type"),
+                                &op_type);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to fetch "
+                       "operation type");
+                goto out;
+            }
+
+            if (strcmp(op_type, "pre") == 0) {
+                /* BRICK OP PHASE for enabling barrier, Enable barrier
+                 * if its a pre-commit operation
                  */
-                ret = dict_get_str (dict, "operation-type", &op_type);
+                ret = glusterd_set_barrier_value(dict, "enable");
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to fetch "
-                                "operation type");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to "
+                           "set barrier value as enable in dict");
+                    goto out;
                 }
+            } else if (strcmp(op_type, "post") == 0) {
+                /* BRICK OP PHASE for disabling barrier, Disable barrier
+                 * if its a post-commit operation
+                 */
+                ret = glusterd_set_barrier_value(dict, "disable");
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Failed to "
+                           "set barrier value as disable in "
+                           "dict");
+                    goto out;
+                }
+            } else {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid op_type");
+                goto out;
+            }
 
-                if (strcmp (op_type, "pre") == 0) {
-                        /* BRICK OP PHASE for enabling barrier, Enable barrier
-                         * if its a pre-commit operation
-                         */
-                        ret = glusterd_set_barrier_value (dict, "enable");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "Failed to "
-                                        "set barrier value as enable in dict");
-                                goto out;
-                        }
-                } else if (strcmp (op_type, "post") == 0) {
-                        /* BRICK OP PHASE for disabling barrier, Disable barrier
-                         * if its a post-commit operation
-                         */
-                        ret = glusterd_set_barrier_value (dict, "disable");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED, "Failed to "
-                                        "set barrier value as disable in "
-                                        "dict");
-                                goto out;
-                        }
-                } else {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_ENTRY, "Invalid op_type");
-                        goto out;
+            ret = dict_get_int64(dict, "volcount", &vol_count);
+            if (ret)
+                goto out;
+            while (count <= vol_count) {
+                keylen = snprintf(key, sizeof(key), "volname%" PRId64, count);
+                ret = dict_get_strn(dict, key, keylen, &volname);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get volname");
+                    goto out;
                 }
+                ret = dict_set_strn(dict, "volname", SLEN("volname"), volname);
+                if (ret)
+                    goto out;
 
-                ret = dict_get_int64 (dict, "volcount", &vol_count);
+                ret = gd_brick_op_phase(GD_OP_SNAP, NULL, dict, op_errstr);
                 if (ret)
-                        goto out;
-                while (count <= vol_count) {
-                        snprintf (key, 1024, "volname%"PRId64, count);
-                        ret = dict_get_str (dict, key, &volname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_GET_FAILED,
-                                        "Unable to get volname");
-                                goto out;
-                        }
-                        ret = dict_set_str (dict, "volname", volname);
-                        if (ret)
-                                goto out;
-
-                        ret = gd_brick_op_phase (GD_OP_SNAP, NULL, dict,
-                                        op_errstr);
-                        if (ret)
-                                goto out;
-                        volname = NULL;
-                        count++;
-                }
+                    goto out;
+                volname = NULL;
+                count++;
+            }
 
-                dict_del (dict, "volname");
-                ret = 0;
-                break;
+            dict_deln(dict, "volname", SLEN("volname"));
+            ret = 0;
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
-                               dict_t *rsp_dict, uint32_t *op_errno)
+glusterd_snapshot_prevalidate(dict_t *dict, char **op_errstr, dict_t *rsp_dict,
+                              uint32_t *op_errno)
 {
-        int                snap_command          = 0;
-        xlator_t           *this                 = NULL;
-        int                ret                   = -1;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMAND_NOT_FOUND, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int snap_command = 0;
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case (GF_SNAP_OPTION_TYPE_CREATE):
-                ret = glusterd_snapshot_create_prevalidate (dict, op_errstr,
-                                                            rsp_dict, op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CREATION_FAIL, "Snapshot create "
-                                "pre-validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_create_prevalidate(dict, op_errstr,
+                                                       rsp_dict, op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Snapshot create "
+                       "pre-validation failed");
+                goto out;
+            }
+            break;
 
         case (GF_SNAP_OPTION_TYPE_CLONE):
-                ret = glusterd_snapshot_clone_prevalidate (dict, op_errstr,
-                                                           rsp_dict, op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CLONE_PREVAL_FAILED,
-                                "Snapshot clone "
-                                "pre-validation failed");
-                        goto out;
-                }
-                break;
-
+            ret = glusterd_snapshot_clone_prevalidate(dict, op_errstr, rsp_dict,
+                                                      op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_CLONE_PREVAL_FAILED,
+                       "Snapshot clone "
+                       "pre-validation failed");
+                goto out;
+            }
+            break;
 
         case (GF_SNAP_OPTION_TYPE_CONFIG):
-                ret = glusterd_snapshot_config_prevalidate (dict, op_errstr,
-                                                            op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CONFIG_FAIL, "Snapshot config "
-                                "pre-validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_config_prevalidate(dict, op_errstr,
+                                                       op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CONFIG_FAIL,
+                       "Snapshot config "
+                       "pre-validation failed");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snapshot_restore_prevalidate (dict, op_errstr,
-                                                             op_errno,
-                                                             rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_RESTORE_FAIL, "Snapshot restore "
-                                        "validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_restore_prevalidate(dict, op_errstr,
+                                                        op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Snapshot restore "
+                       "validation failed");
+                goto out;
+            }
+            break;
 
-        case  GF_SNAP_OPTION_TYPE_ACTIVATE:
-                ret = glusterd_snapshot_activate_deactivate_prevalidate (dict,
-                                                                    op_errstr,
-                                                                     op_errno,
-                                                                     rsp_dict,
-                                                                    _gf_true);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_ACTIVATE_FAIL, "Snapshot activate "
-                                        "validation failed");
-                        goto out;
-                }
-                break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            ret = glusterd_snapshot_activate_deactivate_prevalidate(
+                dict, op_errstr, op_errno, rsp_dict, _gf_true);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                       "Snapshot activate "
+                       "validation failed");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = glusterd_snapshot_activate_deactivate_prevalidate (dict,
-                                                                    op_errstr,
-                                                                     op_errno,
-                                                                     rsp_dict,
-                                                                   _gf_false);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_DEACTIVATE_FAIL,
-                        "Snapshot deactivate validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_activate_deactivate_prevalidate(
+                dict, op_errstr, op_errno, rsp_dict, _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_DEACTIVATE_FAIL,
+                       "Snapshot deactivate validation failed");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_snapshot_remove_prevalidate (dict, op_errstr,
-                                                            op_errno, rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL, "Snapshot remove "
-                                "validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_remove_prevalidate(dict, op_errstr,
+                                                       op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Snapshot remove "
+                       "validation failed");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = glusterd_snapshot_status_prevalidate (dict, op_errstr,
-                                                            op_errno, rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_STATUS_FAIL, "Snapshot status "
-                                "validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_status_prevalidate(dict, op_errstr,
+                                                       op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Snapshot status "
+                       "validation failed");
+                goto out;
+            }
+            break;
 
         default:
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        GD_MSG_COMMAND_NOT_FOUND, "invalid snap command");
-                *op_errno = EINVAL;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "invalid snap command");
+            *op_errno = EINVAL;
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called to remove the trashpath, in cases
@@ -8639,52 +8730,62 @@ out:
  * @return 0 on success or -1 on failure
  */
 int
-glusterd_remove_trashpath (char *volname)
+glusterd_remove_trashpath(char *volname)
 {
-        int                     ret                     = -1;
-        char                    delete_path[PATH_MAX]   = {0,};
-        xlator_t               *this                    = NULL;
-        glusterd_conf_t        *priv                    = NULL;
-        struct stat             stbuf                   = {0, };
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (volname);
-
-        snprintf (delete_path, sizeof (delete_path),
-                  "%s/"GLUSTERD_TRASH"/vols-%s.deleted", priv->workdir,
-                  volname);
-
-        ret = lstat (delete_path, &stbuf);
-        if (ret) {
-                /* If the trash dir does not exist, return *
-                 * without failure                         *
-                 */
-                if (errno == ENOENT) {
-                        ret = 0;
-                        goto out;
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DIR_OP_FAILED, "Failed to lstat "
-                                "backup dir (%s)", delete_path);
-                        goto out;
-                }
-        }
-
-        /* Delete the backup copy of volume folder */
-        ret = recursive_rmdir (delete_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "Failed to remove "
-                        "backup dir (%s)", delete_path);
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    char delete_path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(volname);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir,
+                   volname);
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        goto out;
+    }
+
+    ret = sys_lstat(delete_path, &stbuf);
+    if (ret) {
+        /* If the trash dir does not exist, return *
+         * without failure                         *
+         */
+        if (errno == ENOENT) {
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Failed to lstat "
+                   "backup dir (%s)",
+                   delete_path);
+            goto out;
+        }
+    }
+
+    /* Delete the backup copy of volume folder */
+    ret = recursive_rmdir(delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "backup dir (%s)",
+               delete_path);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called if snapshot restore operation
@@ -8698,44 +8799,41 @@ out:
  * @return 0 on success or -1 on failure
  */
 int
-glusterd_snapshot_restore_cleanup (dict_t *rsp_dict,
-                                   char *volname,
-                                   glusterd_snap_t *snap)
+glusterd_snapshot_restore_cleanup(dict_t *rsp_dict, char *volname,
+                                  glusterd_snap_t *snap)
 {
-        int                     ret                     = -1;
-        xlator_t               *this                    = NULL;
-        glusterd_conf_t        *priv                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (volname);
-        GF_ASSERT (snap);
-
-        /* Now delete the snap entry. */
-        ret = glusterd_snap_remove (rsp_dict, snap, _gf_false, _gf_true,
-                                    _gf_false);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_REMOVE_FAIL, "Failed to delete "
-                        "snap %s", snap->snapname);
-                goto out;
-        }
-
-        /* Delete the backup copy of volume folder */
-        ret = glusterd_remove_trashpath(volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "Failed to remove "
-                        "backup dir");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(volname);
+    GF_ASSERT(snap);
+
+    /* Now delete the snap entry. */
+    ret = glusterd_snap_remove(rsp_dict, snap, _gf_false, _gf_true, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to delete "
+               "snap %s",
+               snap->snapname);
+        goto out;
+    }
+
+    /* Delete the backup copy of volume folder */
+    ret = glusterd_remove_trashpath(volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "backup dir");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called when the snapshot restore operation failed
@@ -8746,73 +8844,108 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_revert_partial_restored_vol (glusterd_volinfo_t *volinfo)
+glusterd_snapshot_revert_partial_restored_vol(glusterd_volinfo_t *volinfo)
 {
-        int                     ret                     = 0;
-        char                    pathname [PATH_MAX]     = {0,};
-        char                    trash_path[PATH_MAX]    = {0,};
-        glusterd_volinfo_t     *reverted_vol            = NULL;
-        glusterd_volinfo_t     *snap_vol                = NULL;
-        glusterd_volinfo_t     *tmp_vol                 = NULL;
-        glusterd_conf_t        *priv                    = NULL;
-        xlator_t               *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
-
-        snprintf (trash_path, sizeof (trash_path),
-                  "%s/"GLUSTERD_TRASH"/vols-%s.deleted", priv->workdir,
-                  volinfo->volname);
-
-        /* Since snapshot restore failed we cannot rely on the volume
-         * data stored under vols folder. Therefore delete the origin
-         * volume's backend folder.*/
-        ret = recursive_rmdir (pathname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "Failed to remove "
-                        "%s directory", pathname);
-                goto out;
-        }
-
-        /* Now move the backup copy of the vols to its original
-         * location.*/
-        ret = sys_rename (trash_path, pathname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "Failed to rename folder "
-                        "from %s to %s", trash_path, pathname);
-                goto out;
-        }
-
-        /* Retrieve the volume from the store */
-        reverted_vol = glusterd_store_retrieve_volume (volinfo->volname, NULL);
-        if (NULL == reverted_vol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_OP_FAILED, "Failed to load restored "
-                        "%s volume", volinfo->volname);
-                goto out;
-        }
-
-        /* Retrieve the snap_volumes list from the older volinfo */
-        reverted_vol->snap_count = volinfo->snap_count;
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
-                                      snapvol_list) {
-                cds_list_add_tail (&snap_vol->snapvol_list,
-                                   &reverted_vol->snap_volumes);
-        }
-
-        /* Since we retrieved the volinfo from store now we don't
-         * want the older volinfo. Therefore delete the older volinfo */
-        glusterd_volinfo_unref (volinfo);
-        ret = 0;
+    int ret = 0;
+    char pathname[PATH_MAX] = "";
+    char trash_path[PATH_MAX] = "";
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *reverted_vol = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(pathname, volinfo, priv);
+
+    len = snprintf(trash_path, sizeof(trash_path),
+                   "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir,
+                   volinfo->volname);
+    if ((len < 0) || (len >= sizeof(trash_path))) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* Since snapshot restore failed we cannot rely on the volume
+     * data stored under vols folder. Therefore delete the origin
+     * volume's backend folder.*/
+    ret = recursive_rmdir(pathname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "%s directory",
+               pathname);
+        goto out;
+    }
+
+    /* Now move the backup copy of the vols to its original
+     * location.*/
+    ret = sys_rename(trash_path, pathname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to rename folder "
+               "from %s to %s",
+               trash_path, pathname);
+        goto out;
+    }
+
+    /* Retrieve the volume from the store */
+    reverted_vol = glusterd_store_retrieve_volume(volinfo->volname, NULL);
+    if (NULL == reverted_vol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to load restored "
+               "%s volume",
+               volinfo->volname);
+        goto out;
+    }
+
+    /* Retrieve the snap_volumes list from the older volinfo */
+    reverted_vol->snap_count = volinfo->snap_count;
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        cds_list_add_tail(&snap_vol->snapvol_list, &reverted_vol->snap_volumes);
+
+        cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+        {
+            /*
+             * If the brick is not of this peer, or snapshot is    *
+             * missed for the brick don't restore the xattr for it *
+             */
+            if ((!gf_uuid_compare(brickinfo->uuid, MY_UUID)) &&
+                (brickinfo->snap_status != -1)) {
+                /*
+                 * We need to restore volume id of all snap *
+                 * bricks to volume id of the snap volume.  *
+                 */
+                ret = sys_lsetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                    snap_vol->volume_id,
+                                    sizeof(snap_vol->volume_id), XATTR_REPLACE);
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SET_XATTR_FAIL,
+                            "Attribute=%s, Path=%s, Reason=%s, Snap=%s",
+                            GF_XATTR_VOL_ID_KEY, brickinfo->path,
+                            strerror(errno), snap_vol->volname, NULL);
+                    goto out;
+                }
+            }
+        }
+    }
+
+    /* Since we retrieved the volinfo from store now we don't
+     * want the older volinfo. Therefore delete the older volinfo */
+    glusterd_volinfo_unref(volinfo);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called when glusterd is started and we need
@@ -8823,45 +8956,46 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_revert_restore_from_snap (glusterd_snap_t *snap)
+glusterd_snapshot_revert_restore_from_snap(glusterd_snap_t *snap)
 {
-        int                     ret                     = -1;
-        char                    volname [PATH_MAX]      = {0,};
-        glusterd_volinfo_t     *snap_volinfo            = NULL;
-        glusterd_volinfo_t     *volinfo                 = NULL;
-        xlator_t               *this                    = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (snap);
-
-        /* TODO : As of now there is only one volume in snapshot.
-         * Change this when multiple volume snapshot is introduced
-         */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-
-        strcpy (volname, snap_volinfo->parent_volname);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                       GD_MSG_VOL_NOT_FOUND, "Could not get volinfo of "
-                        "%s", snap_volinfo->parent_volname);
-                goto out;
-        }
-
-        ret = glusterd_snapshot_revert_partial_restored_vol (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_RESTORE_REVERT_FAIL,
-                        "Failed to revert snapshot "
-                        "restore operation for %s volume", volname);
-                goto out;
-        }
+    int ret = -1;
+    char volname[PATH_MAX] = "";
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+
+    /* TODO : As of now there is only one volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+
+    gf_strncpy(volname, snap_volinfo->parent_volname, sizeof(volname));
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Could not get volinfo of "
+               "%s",
+               snap_volinfo->parent_volname);
+        goto out;
+    }
+
+    ret = glusterd_snapshot_revert_partial_restored_vol(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_REVERT_FAIL,
+               "Failed to revert snapshot "
+               "restore operation for %s volume",
+               volname);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called from post-validation. Based on the op_ret
@@ -8876,236 +9010,239 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_restore_postop (dict_t *dict, int32_t op_ret,
-                                  char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_restore_postop(dict_t *dict, int32_t op_ret, char **op_errstr,
+                                 dict_t *rsp_dict)
 {
-        int                     ret             = -1;
-        char                   *name            = NULL;
-        char                   *volname         = NULL;
-        int                     cleanup         = 0;
-        glusterd_snap_t        *snap            = NULL;
-        glusterd_volinfo_t     *volinfo         = NULL;
-        xlator_t               *this            = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_str (dict, "snapname", &name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "getting the snap "
-                        "name failed (volume: %s)", name);
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (name);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_NOT_FOUND,
-                        "Snapshot (%s) does not exist", name);
-                ret = -1;
-                goto out;
-        }
-
-        /* TODO: fix this when multiple volume support will come */
-        ret = dict_get_str (dict, "volname1", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED,
-                        "failed to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_NOT_FOUND,
-                        "Volume (%s) does not exist ", volname);
-                goto out;
+    int ret = -1;
+    char *name = NULL;
+    char *volname = NULL;
+    int cleanup = 0;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "getting the snap "
+               "name failed (volume: %s)",
+               name);
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(name);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", name);
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO: fix this when multiple volume support will come */
+    ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume (%s) does not exist ", volname);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "getting the snap "
+               "name failed (volume: %s)",
+               volinfo->volname);
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(name);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "snap %s is not found", name);
+        ret = -1;
+        goto out;
+    }
+
+    /* On success perform the cleanup operation */
+    if (0 == op_ret) {
+        ret = glusterd_snapshot_restore_cleanup(rsp_dict, volname, snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CLEANUP_FAIL,
+                   "Failed to perform "
+                   "snapshot restore cleanup for %s volume",
+                   volname);
+            goto out;
+        }
+    } else { /* On failure revert snapshot restore */
+        ret = dict_get_int32n(dict, "cleanup", SLEN("cleanup"), &cleanup);
+        /* Perform cleanup only when required */
+        if (ret || (0 == cleanup)) {
+            /* Delete the backup copy of volume folder */
+            ret = glusterd_remove_trashpath(volinfo->volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                       "Failed to remove backup dir");
+                goto out;
+            }
+            ret = 0;
+            goto out;
+        }
+
+        ret = glusterd_snapshot_revert_partial_restored_vol(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_REVERT_FAIL,
+                   "Failed to revert "
+                   "restore operation for %s volume",
+                   volname);
+            goto out;
         }
 
-        ret = dict_get_str (dict, "snapname", &name);
+        snap->snap_status = GD_SNAP_STATUS_IN_USE;
+        /* We need to save this in disk */
+        ret = glusterd_store_snap(snap);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_GET_FAILED, "getting the snap "
-                        "name failed (volume: %s)", volinfo->volname);
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (name);
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_SNAP_NOT_FOUND, "snap %s is not found", name);
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+                   "Could not store snap object for %s snap", snap->snapname);
+            goto out;
         }
 
-        /* On success perform the cleanup operation */
-        if (0 == op_ret) {
-                ret = glusterd_snapshot_restore_cleanup (rsp_dict,
-                                                         volname,
-                                                         snap);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_CLEANUP_FAIL, "Failed to perform "
-                                "snapshot restore cleanup for %s volume",
-                                volname);
-                        goto out;
-                }
-        } else { /* On failure revert snapshot restore */
-                ret = dict_get_int32 (dict, "cleanup", &cleanup);
-                /* Perform cleanup only when required */
-                if (ret || (0 == cleanup)) {
-                        /* Delete the backup copy of volume folder */
-                        ret = glusterd_remove_trashpath(volinfo->volname);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_DIR_OP_FAILED,
-                                        "Failed to remove backup dir");
-                                goto out;
-                        }
-                        ret = 0;
-                        goto out;
-                }
-
-                ret = glusterd_snapshot_revert_partial_restored_vol (volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_RESTORE_REVERT_FAIL,
-                                "Failed to revert "
-                                "restore operation for %s volume", volname);
-                        goto out;
-                }
-
-                snap->snap_status = GD_SNAP_STATUS_IN_USE;
-                /* We need to save this in disk */
-                ret = glusterd_store_snap (snap);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_OBJECT_STORE_FAIL,
-                                "Could not store snap object for %s snap",
-                                snap->snapname);
-                        goto out;
-                }
+        /* After restore fails, we have to remove mount point for
+         * deactivated snaps which was created at start of restore op.
+         */
+        if (volinfo->status == GLUSTERD_STATUS_STOPPED) {
+            ret = glusterd_snap_unmount(this, volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+                       "Failed to unmounts for %s", snap->snapname);
+            }
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
-                                dict_t *rsp_dict)
+glusterd_snapshot_postvalidate(dict_t *dict, int32_t op_ret, char **op_errstr,
+                               dict_t *rsp_dict)
 {
-        int                snap_command          = 0;
-        xlator_t           *this                 = NULL;
-        int                ret                   = -1;
+    int snap_command = 0;
+    xlator_t *this = NULL;
+    int ret = -1;
 
-        this = THIS;
+    this = THIS;
 
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
 
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                       GD_MSG_COMMAND_NOT_FOUND, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
 
-        switch (snap_command) {
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = glusterd_snapshot_create_postvalidate (dict, op_ret,
-                                                             op_errstr,
-                                                             rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CREATION_FAIL, "Snapshot create "
-                                "post-validation failed");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
-         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_snapshot_clone_postvalidate (dict, op_ret,
-                                                             op_errstr,
-                                                             rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CLONE_POSTVAL_FAILED,
-                                "Snapshot create "
-                                "post-validation failed");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
+            ret = glusterd_snapshot_create_postvalidate(dict, op_ret, op_errstr,
+                                                        rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Snapshot create "
+                       "post-validation failed");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            ret = glusterd_snapshot_clone_postvalidate(dict, op_ret, op_errstr,
+                                                       rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_CLONE_POSTVAL_FAILED,
+                       "Snapshot create "
+                       "post-validation failed");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                if (op_ret) {
-                        gf_msg_debug (this->name, 0,
-                                "op_ret = %d. Not performing delete "
-                                "post_validate", op_ret);
-                        ret = 0;
-                        goto out;
-                }
-                ret = glusterd_snapshot_update_snaps_post_validate (dict,
-                                                                    op_errstr,
-                                                                    rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_LIST_STORE_FAIL, "Failed to "
-                                "update missed snaps list");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
+            if (op_ret) {
+                gf_msg_debug(this->name, 0,
+                             "op_ret = %d. Not performing delete "
+                             "post_validate",
+                             op_ret);
+                ret = 0;
+                goto out;
+            }
+            ret = glusterd_snapshot_update_snaps_post_validate(dict, op_errstr,
+                                                               rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                       "Failed to "
+                       "update missed snaps list");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snapshot_update_snaps_post_validate (dict,
-                                                                    op_errstr,
-                                                                    rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_RESTORE_FAIL, "Failed to "
-                                "update missed snaps list");
-                        goto out;
-                }
+            ret = glusterd_snapshot_update_snaps_post_validate(dict, op_errstr,
+                                                               rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Failed to "
+                       "update missed snaps list");
+                goto out;
+            }
 
-                ret = glusterd_snapshot_restore_postop (dict, op_ret,
-                                                        op_errstr, rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_RESTORE_FAIL, "Failed to "
-                                "perform snapshot restore post-op");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
+            ret = glusterd_snapshot_restore_postop(dict, op_ret, op_errstr,
+                                                   rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Failed to "
+                       "perform snapshot restore post-op");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_ACTIVATE:
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                glusterd_fetchsnap_notify (this);
-                break;
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_STATUS:
         case GF_SNAP_OPTION_TYPE_CONFIG:
         case GF_SNAP_OPTION_TYPE_INFO:
         case GF_SNAP_OPTION_TYPE_LIST:
-                 /*Nothing to be done. But want to
-                 * avoid the default case warning*/
-                ret = 0;
-                break;
+            /*Nothing to be done. But want to
+             * avoid the default case warning*/
+            ret = 0;
+            break;
         default:
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_COMMAND_NOT_FOUND, "invalid snap command");
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_COMMAND_NOT_FOUND,
+                   "invalid snap command");
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -9113,622 +9250,588 @@ out:
 */
 
 static gf_boolean_t
-glusterd_is_lvm_cmd_available (char *lvm_cmd)
+glusterd_is_lvm_cmd_available(char *lvm_cmd)
 {
-        int32_t     ret  = 0;
-        struct stat buf  = {0,};
-
-        if (!lvm_cmd)
-                return _gf_false;
-
-        ret = sys_stat (lvm_cmd, &buf);
-        if (ret != 0) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED,
-                        "stat fails on %s, exiting. (errno = %d (%s))",
-                        lvm_cmd, errno, strerror(errno));
-                return _gf_false;
-        }
-
-        if ((!ret) && (!S_ISREG(buf.st_mode))) {
-                gf_msg (THIS->name, GF_LOG_CRITICAL, EINVAL,
-                        GD_MSG_COMMAND_NOT_FOUND,
-                        "Provided command %s is not a regular file,"
-                        "exiting", lvm_cmd);
-                return _gf_false;
-        }
-
-        if ((!ret) && (!(buf.st_mode & S_IXUSR))) {
-                gf_msg (THIS->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_NO_EXEC_PERMS,
-                        "Provided command %s has no exec permissions,"
-                        "exiting", lvm_cmd);
-                return _gf_false;
-        }
-
-        return _gf_true;
+    int32_t ret = 0;
+    struct stat buf = {
+        0,
+    };
+
+    if (!lvm_cmd)
+        return _gf_false;
+
+    ret = sys_stat(lvm_cmd, &buf);
+    if (ret != 0) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "stat fails on %s, exiting. (errno = %d (%s))", lvm_cmd, errno,
+               strerror(errno));
+        return _gf_false;
+    }
+
+    if ((!ret) && (!S_ISREG(buf.st_mode))) {
+        gf_msg(THIS->name, GF_LOG_CRITICAL, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+               "Provided command %s is not a regular file,"
+               "exiting",
+               lvm_cmd);
+        return _gf_false;
+    }
+
+    if ((!ret) && (!(buf.st_mode & S_IXUSR))) {
+        gf_msg(THIS->name, GF_LOG_CRITICAL, 0, GD_MSG_NO_EXEC_PERMS,
+               "Provided command %s has no exec permissions,"
+               "exiting",
+               lvm_cmd);
+        return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 int
-glusterd_handle_snapshot_fn (rpcsvc_request_t *req)
+glusterd_handle_snapshot_fn(rpcsvc_request_t *req)
 {
-        int32_t               ret            = 0;
-        dict_t               *dict           = NULL;
-        gf_cli_req            cli_req        = {{0},};
-        glusterd_op_t         cli_op         = GD_OP_SNAP;
-        int                   type           = 0;
-        glusterd_conf_t      *conf           = NULL;
-        char                 *host_uuid      = NULL;
-        char                  err_str[2048]  = {0,};
-        xlator_t             *this           = NULL;
-        uint32_t              op_errno       = 0;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len > 0) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_UNSERIALIZE_FAIL, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-
-                dict->extra_stdfree = cli_req.dict.dict_val;
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret) {
-                        GF_FREE (host_uuid);
-                        goto out;
-                }
-
-
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY, "request dict length is %d",
-                        cli_req.dict.dict_len);
-                goto out;
-        }
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                snprintf (err_str, sizeof (err_str), "Cluster operating version"
-                          " is lesser than the supported version "
-                          "for a snapshot");
-                op_errno = EG_OPNOTSUP;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_UNSUPPORTED_VERSION,
-                        "%s (%d < %d)", err_str,
-                        conf->op_version, GD_OP_VERSION_3_6_0);
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_SNAP;
+    int type = 0;
+    glusterd_conf_t *conf = NULL;
+    char *host_uuid = NULL;
+    char err_str[2048] = "";
+    xlator_t *this = NULL;
+    uint32_t op_errno = 0;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len > 0) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
 
-        ret = dict_get_int32 (dict, "type", &type);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Command type not found");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMAND_NOT_FOUND, "%s", err_str);
-                goto out;
-        }
-
-        if (!glusterd_is_lvm_cmd_available (LVM_CREATE)) {
-                snprintf (err_str, sizeof (err_str), "LVM commands not found,"
-                          " snapshot functionality is disabled");
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_COMMAND_NOT_FOUND, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        switch (type) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+
+        dict->extra_stdfree = cli_req.dict.dict_val;
+
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstrn(dict, "host-uuid", SLEN("host-uuid"), host_uuid);
+        if (ret) {
+            GF_FREE(host_uuid);
+            goto out;
+        }
+
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "request dict length is %d", cli_req.dict.dict_len);
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Cluster operating version"
+                 " is lesser than the supported version "
+                 "for a snapshot");
+        op_errno = EG_OPNOTSUP;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "%s (%d < %d)", err_str, conf->op_version, GD_OP_VERSION_3_6_0);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str), "Command type not found");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (!glusterd_is_lvm_cmd_available(LVM_CREATE)) {
+        snprintf(err_str, sizeof(err_str),
+                 "LVM commands not found,"
+                 " snapshot functionality is disabled");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    switch (type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = glusterd_handle_snapshot_create (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CREATION_FAIL,
-                                "Snapshot create failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_create(req, cli_op, dict, err_str,
+                                                  sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Snapshot create failed: %s", err_str);
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_handle_snapshot_clone (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CLONE_FAILED, "Snapshot clone "
-                                "failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_clone(req, cli_op, dict, err_str,
+                                                 sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CLONE_FAILED,
+                       "Snapshot clone "
+                       "failed: %s",
+                       err_str);
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_handle_snapshot_restore (req, cli_op, dict,
-                                                        err_str, &op_errno,
-                                                        sizeof (err_str));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_RESTORE_FAIL,
-                                "Snapshot restore failed: %s", err_str);
-                }
-
-                break;
+            ret = glusterd_handle_snapshot_restore(req, cli_op, dict, err_str,
+                                                   &op_errno, sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Snapshot restore failed: %s", err_str);
+            }
+
+            break;
         case GF_SNAP_OPTION_TYPE_INFO:
-                ret = glusterd_handle_snapshot_info (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_INFO_FAIL,
-                                "Snapshot info failed");
-                }
-                break;
+            ret = glusterd_handle_snapshot_info(req, cli_op, dict, err_str,
+                                                sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_INFO_FAIL,
+                       "Snapshot info failed");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_LIST:
-                ret = glusterd_handle_snapshot_list (req, cli_op, dict,
-                                                     err_str, sizeof (err_str),
-                                                     &op_errno);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_LIST_GET_FAIL,
-                                "Snapshot list failed");
-                }
-                break;
+            ret = glusterd_handle_snapshot_list(req, cli_op, dict, err_str,
+                                                sizeof(err_str), &op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_LIST_GET_FAIL,
+                       "Snapshot list failed");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = glusterd_handle_snapshot_config (req, cli_op, dict,
-                                                 err_str, sizeof (err_str));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_CONFIG_FAIL,
-                                "snapshot config failed");
-                }
-                break;
+            ret = glusterd_handle_snapshot_config(req, cli_op, dict, err_str,
+                                                  sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CONFIG_FAIL,
+                       "snapshot config failed");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_handle_snapshot_delete (req, cli_op, dict,
-                                                       err_str, &op_errno,
-                                                       sizeof (err_str));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_REMOVE_FAIL,
-                                "Snapshot delete failed: %s", err_str);
-                }
-                break;
-        case  GF_SNAP_OPTION_TYPE_ACTIVATE:
-                ret = glusterd_mgmt_v3_initiate_snap_phases (req, cli_op,
-                                                             dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_ACTIVATE_FAIL,
-                                "Snapshot activate failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_delete(req, cli_op, dict, err_str,
+                                                  &op_errno, sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Snapshot delete failed: %s", err_str);
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            ret = glusterd_mgmt_v3_initiate_snap_phases(req, cli_op, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                       "Snapshot activate failed: %s", err_str);
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = glusterd_mgmt_v3_initiate_snap_phases (req, cli_op,
-                                                             dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_DEACTIVATE_FAIL,
-                                "Snapshot deactivate failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_mgmt_v3_initiate_snap_phases(req, cli_op, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_DEACTIVATE_FAIL,
+                       "Snapshot deactivate failed: %s", err_str);
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = glusterd_handle_snapshot_status (req, cli_op, dict,
-                                                       err_str,
-                                                       sizeof (err_str));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                GD_MSG_SNAP_STATUS_FAIL,
-                                "Snapshot status failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_status(req, cli_op, dict, err_str,
+                                                  sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Snapshot status failed: %s", err_str);
+            }
+            break;
         default:
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_COMMAND_NOT_FOUND, "Unkown snapshot request "
-                        "type (%d)", type);
-                ret = -1; /* Failure */
-        }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "Unknown snapshot request "
+                   "type (%d)",
+                   type);
+            ret = -1; /* Failure */
+    }
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
 
-                if (ret && (op_errno == 0))
-                        op_errno = EG_INTRNL;
+        if (ret && (op_errno == 0))
+            op_errno = EG_INTRNL;
 
-                ret = glusterd_op_send_cli_response (cli_op, ret, op_errno, req,
-                                                     dict, err_str);
-        }
+        ret = glusterd_op_send_cli_response(cli_op, ret, op_errno, req, dict,
+                                            err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_snapshot (rpcsvc_request_t *req)
+glusterd_handle_snapshot(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, glusterd_handle_snapshot_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_snapshot_fn);
 }
 
 static void
-glusterd_free_snap_op (glusterd_snap_op_t *snap_op)
+glusterd_free_snap_op(glusterd_snap_op_t *snap_op)
 {
-        if (snap_op) {
-                if (snap_op->brick_path)
-                        GF_FREE (snap_op->brick_path);
+    if (snap_op) {
+        if (snap_op->brick_path)
+            GF_FREE(snap_op->brick_path);
 
-                GF_FREE (snap_op);
-        }
+        GF_FREE(snap_op);
+    }
 }
 
 static void
-glusterd_free_missed_snapinfo (glusterd_missed_snap_info *missed_snapinfo)
+glusterd_free_missed_snapinfo(glusterd_missed_snap_info *missed_snapinfo)
 {
-        glusterd_snap_op_t *snap_opinfo = NULL;
-        glusterd_snap_op_t *tmp         = NULL;
-
-        if (missed_snapinfo) {
-                cds_list_for_each_entry_safe (snap_opinfo, tmp,
-                                              &missed_snapinfo->snap_ops,
-                                              snap_ops_list) {
-                        glusterd_free_snap_op (snap_opinfo);
-                        snap_opinfo = NULL;
-                }
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    glusterd_snap_op_t *tmp = NULL;
 
-                if (missed_snapinfo->node_uuid)
-                        GF_FREE (missed_snapinfo->node_uuid);
+    if (missed_snapinfo) {
+        cds_list_for_each_entry_safe(snap_opinfo, tmp,
+                                     &missed_snapinfo->snap_ops, snap_ops_list)
+        {
+            glusterd_free_snap_op(snap_opinfo);
+            snap_opinfo = NULL;
+        }
 
-                if (missed_snapinfo->snap_uuid)
-                        GF_FREE (missed_snapinfo->snap_uuid);
+        if (missed_snapinfo->node_uuid)
+            GF_FREE(missed_snapinfo->node_uuid);
 
-                GF_FREE (missed_snapinfo);
-        }
+        if (missed_snapinfo->snap_uuid)
+            GF_FREE(missed_snapinfo->snap_uuid);
+
+        GF_FREE(missed_snapinfo);
+    }
 }
 
 /* Look for duplicates and accordingly update the list */
 int32_t
-glusterd_update_missed_snap_entry (glusterd_missed_snap_info *missed_snapinfo,
-                                   glusterd_snap_op_t *missed_snap_op)
+glusterd_update_missed_snap_entry(glusterd_missed_snap_info *missed_snapinfo,
+                                  glusterd_snap_op_t *missed_snap_op)
 {
-        int32_t                        ret                         = -1;
-        glusterd_snap_op_t            *snap_opinfo                 = NULL;
-        gf_boolean_t                   match                       = _gf_false;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(missed_snapinfo);
-        GF_ASSERT(missed_snap_op);
-
-        cds_list_for_each_entry (snap_opinfo, &missed_snapinfo->snap_ops,
-                                 snap_ops_list) {
-                /* If the entry is not for the same snap_vol_id
-                 * then continue
-                 */
-                if (strcmp (snap_opinfo->snap_vol_id,
-                            missed_snap_op->snap_vol_id))
-                        continue;
-
-                if ((!strcmp (snap_opinfo->brick_path,
-                              missed_snap_op->brick_path)) &&
-                    (snap_opinfo->op == missed_snap_op->op)) {
-                        /* If two entries have conflicting status
-                         * GD_MISSED_SNAP_DONE takes precedence
-                         */
-                        if ((snap_opinfo->status == GD_MISSED_SNAP_PENDING) &&
-                            (missed_snap_op->status == GD_MISSED_SNAP_DONE)) {
-                                snap_opinfo->status = GD_MISSED_SNAP_DONE;
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        GD_MSG_MISSED_SNAP_STATUS_DONE,
-                                        "Updating missed snap status "
-                                        "for %s:%s=%s:%d:%s:%d as DONE",
-                                        missed_snapinfo->node_uuid,
-                                        missed_snapinfo->snap_uuid,
-                                        snap_opinfo->snap_vol_id,
-                                        snap_opinfo->brick_num,
-                                        snap_opinfo->brick_path,
-                                        snap_opinfo->op);
-                                ret = 0;
-                                glusterd_free_snap_op (missed_snap_op);
-                                goto out;
-                        }
-                        match = _gf_true;
-                        break;
-                } else if ((snap_opinfo->brick_num ==
-                            missed_snap_op->brick_num) &&
-                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
-                            ((missed_snap_op->op ==
-                              GF_SNAP_OPTION_TYPE_DELETE) ||
-                             (missed_snap_op->op ==
-                              GF_SNAP_OPTION_TYPE_RESTORE))) {
-                        /* Optimizing create and delete entries for the same
-                         * brick and same node
-                         */
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_MISSED_SNAP_STATUS_DONE,
-                                "Updating missed snap status "
-                                "for %s:%s=%s:%d:%s:%d as DONE",
-                                missed_snapinfo->node_uuid,
-                                missed_snapinfo->snap_uuid,
-                                snap_opinfo->snap_vol_id,
-                                snap_opinfo->brick_num,
-                                snap_opinfo->brick_path,
-                                snap_opinfo->op);
-                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
-                        ret = 0;
-                        glusterd_free_snap_op (missed_snap_op);
-                        goto out;
-                }
-        }
-
-        if (match == _gf_true) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_DUP_ENTRY,
-                        "Duplicate entry. Not updating");
-                glusterd_free_snap_op (missed_snap_op);
-        } else {
-                cds_list_add_tail (&missed_snap_op->snap_ops_list,
-                                   &missed_snapinfo->snap_ops);
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    gf_boolean_t match = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(missed_snapinfo);
+    GF_ASSERT(missed_snap_op);
+
+    cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                            snap_ops_list)
+    {
+        /* If the entry is not for the same snap_vol_id
+         * then continue
+         */
+        if (strcmp(snap_opinfo->snap_vol_id, missed_snap_op->snap_vol_id))
+            continue;
+
+        if ((!strcmp(snap_opinfo->brick_path, missed_snap_op->brick_path)) &&
+            (snap_opinfo->op == missed_snap_op->op)) {
+            /* If two entries have conflicting status
+             * GD_MISSED_SNAP_DONE takes precedence
+             */
+            if ((snap_opinfo->status == GD_MISSED_SNAP_PENDING) &&
+                (missed_snap_op->status == GD_MISSED_SNAP_DONE)) {
+                snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       GD_MSG_MISSED_SNAP_STATUS_DONE,
+                       "Updating missed snap status "
+                       "for %s:%s=%s:%d:%s:%d as DONE",
+                       missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid,
+                       snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                       snap_opinfo->brick_path, snap_opinfo->op);
+                ret = 0;
+                glusterd_free_snap_op(missed_snap_op);
+                goto out;
+            }
+            match = _gf_true;
+            break;
+        } else if ((snap_opinfo->brick_num == missed_snap_op->brick_num) &&
+                   (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
+                   ((missed_snap_op->op == GF_SNAP_OPTION_TYPE_DELETE) ||
+                    (missed_snap_op->op == GF_SNAP_OPTION_TYPE_RESTORE))) {
+            /* Optimizing create and delete entries for the same
+             * brick and same node
+             */
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_STATUS_DONE,
+                   "Updating missed snap status "
+                   "for %s:%s=%s:%d:%s:%d as DONE",
+                   missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid,
+                   snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                   snap_opinfo->brick_path, snap_opinfo->op);
+            snap_opinfo->status = GD_MISSED_SNAP_DONE;
+            ret = 0;
+            glusterd_free_snap_op(missed_snap_op);
+            goto out;
+        }
+    }
+
+    if (match == _gf_true) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DUP_ENTRY,
+               "Duplicate entry. Not updating");
+        glusterd_free_snap_op(missed_snap_op);
+    } else {
+        cds_list_add_tail(&missed_snap_op->snap_ops_list,
+                          &missed_snapinfo->snap_ops);
+    }
+
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Add new missed snap entry to the missed_snaps list. */
 int32_t
-glusterd_add_new_entry_to_list (char *missed_info, char *snap_vol_id,
-                                int32_t brick_num, char *brick_path,
-                                int32_t snap_op, int32_t snap_status)
+glusterd_add_new_entry_to_list(char *missed_info, char *snap_vol_id,
+                               int32_t brick_num, char *brick_path,
+                               int32_t snap_op, int32_t snap_status)
 {
-        char                          *buf                         = NULL;
-        char                          *save_ptr                    = NULL;
-        char                           node_snap_info[PATH_MAX]    = "";
-        int32_t                        ret                         = -1;
-        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
-        glusterd_snap_op_t            *missed_snap_op              = NULL;
-        glusterd_conf_t               *priv                        = NULL;
-        gf_boolean_t                   match                       = _gf_false;
-        gf_boolean_t                   free_missed_snap_info       = _gf_false;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(missed_info);
-        GF_ASSERT(snap_vol_id);
-        GF_ASSERT(brick_path);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Create the snap_op object consisting of the *
-         * snap id and the op */
-        ret = glusterd_missed_snap_op_new (&missed_snap_op);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                        "Failed to create new missed snap object.");
-                ret = -1;
-                goto out;
-        }
-
-        missed_snap_op->snap_vol_id = gf_strdup(snap_vol_id);
-        if (!missed_snap_op->snap_vol_id) {
-                ret = -1;
-                goto out;
-        }
-        missed_snap_op->brick_path = gf_strdup(brick_path);
-        if (!missed_snap_op->brick_path) {
-                ret = -1;
-                goto out;
-        }
-        missed_snap_op->brick_num = brick_num;
-        missed_snap_op->op = snap_op;
-        missed_snap_op->status = snap_status;
-
-        /* Look for other entries for the same node and same snap */
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                snprintf (node_snap_info, sizeof(node_snap_info),
-                          "%s:%s", missed_snapinfo->node_uuid,
-                          missed_snapinfo->snap_uuid);
-                if (!strcmp (node_snap_info, missed_info)) {
-                        /* Found missed snapshot info for *
-                         * the same node and same snap */
-                        match = _gf_true;
-                        break;
-                }
-        }
-
-        if (match == _gf_false) {
-                /* First snap op missed for the brick */
-                ret = glusterd_missed_snapinfo_new (&missed_snapinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                                "Failed to create missed snapinfo");
-                        goto out;
-                }
-                free_missed_snap_info = _gf_true;
-                buf = strtok_r (missed_info, ":", &save_ptr);
-                if (!buf) {
-                        ret = -1;
-                        goto out;
-                }
-                missed_snapinfo->node_uuid = gf_strdup(buf);
-                if (!missed_snapinfo->node_uuid) {
-                        ret = -1;
-                        goto out;
-                }
-
-                buf = strtok_r (NULL, ":", &save_ptr);
-                if (!buf) {
-                        ret = -1;
-                        goto out;
-                }
-                missed_snapinfo->snap_uuid = gf_strdup(buf);
-                if (!missed_snapinfo->snap_uuid) {
-                        ret = -1;
-                        goto out;
-                }
-
-                cds_list_add_tail (&missed_snap_op->snap_ops_list,
-                                   &missed_snapinfo->snap_ops);
-                cds_list_add_tail (&missed_snapinfo->missed_snaps,
-                                   &priv->missed_snaps_list);
+    char *buf = NULL;
+    char *save_ptr = NULL;
+    char node_snap_info[PATH_MAX] = "";
+    int32_t ret = -1;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *missed_snap_op = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t match = _gf_false;
+    gf_boolean_t free_missed_snap_info = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(missed_info);
+    GF_ASSERT(snap_vol_id);
+    GF_ASSERT(brick_path);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Create the snap_op object consisting of the *
+     * snap id and the op */
+    ret = glusterd_missed_snap_op_new(&missed_snap_op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+               "Failed to create new missed snap object.");
+        ret = -1;
+        goto out;
+    }
+
+    missed_snap_op->snap_vol_id = gf_strdup(snap_vol_id);
+    if (!missed_snap_op->snap_vol_id) {
+        ret = -1;
+        goto out;
+    }
+    missed_snap_op->brick_path = gf_strdup(brick_path);
+    if (!missed_snap_op->brick_path) {
+        ret = -1;
+        goto out;
+    }
+    missed_snap_op->brick_num = brick_num;
+    missed_snap_op->op = snap_op;
+    missed_snap_op->status = snap_status;
+
+    /* Look for other entries for the same node and same snap */
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        snprintf(node_snap_info, sizeof(node_snap_info), "%s:%s",
+                 missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid);
+        if (!strcmp(node_snap_info, missed_info)) {
+            /* Found missed snapshot info for *
+             * the same node and same snap */
+            match = _gf_true;
+            break;
+        }
+    }
+
+    if (match == _gf_false) {
+        /* First snap op missed for the brick */
+        ret = glusterd_missed_snapinfo_new(&missed_snapinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                   "Failed to create missed snapinfo");
+            goto out;
+        }
+        free_missed_snap_info = _gf_true;
+        buf = strtok_r(missed_info, ":", &save_ptr);
+        if (!buf) {
+            ret = -1;
+            goto out;
+        }
+        missed_snapinfo->node_uuid = gf_strdup(buf);
+        if (!missed_snapinfo->node_uuid) {
+            ret = -1;
+            goto out;
+        }
+
+        buf = strtok_r(NULL, ":", &save_ptr);
+        if (!buf) {
+            ret = -1;
+            goto out;
+        }
+        missed_snapinfo->snap_uuid = gf_strdup(buf);
+        if (!missed_snapinfo->snap_uuid) {
+            ret = -1;
+            goto out;
+        }
+
+        cds_list_add_tail(&missed_snap_op->snap_ops_list,
+                          &missed_snapinfo->snap_ops);
+        cds_list_add_tail(&missed_snapinfo->missed_snaps,
+                          &priv->missed_snaps_list);
 
-                ret = 0;
-                goto out;
-        } else {
-                ret = glusterd_update_missed_snap_entry (missed_snapinfo,
-                                                         missed_snap_op);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                                "Failed to update existing missed snap entry.");
-                        goto out;
-                }
+        ret = 0;
+        goto out;
+    } else {
+        ret = glusterd_update_missed_snap_entry(missed_snapinfo,
+                                                missed_snap_op);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                   "Failed to update existing missed snap entry.");
+            goto out;
         }
+    }
 
 out:
-        if (ret) {
-                glusterd_free_snap_op (missed_snap_op);
+    if (ret) {
+        glusterd_free_snap_op(missed_snap_op);
 
-                if (missed_snapinfo &&
-                    (free_missed_snap_info == _gf_true))
-                        glusterd_free_missed_snapinfo (missed_snapinfo);
-        }
+        if (missed_snapinfo && (free_missed_snap_info == _gf_true))
+            glusterd_free_missed_snapinfo(missed_snapinfo);
+    }
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Add  missing snap entries to the in-memory conf->missed_snap_list */
 int32_t
-glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count)
+glusterd_add_missed_snaps_to_list(dict_t *dict, int32_t missed_snap_count)
 {
-        char                          *buf                         = NULL;
-        char                          *tmp                         = NULL;
-        char                          *save_ptr                    = NULL;
-        char                          *nodeid                      = NULL;
-        char                          *snap_uuid                   = NULL;
-        char                          *snap_vol_id                 = NULL;
-        char                          *brick_path                  = NULL;
-        char                           missed_info[PATH_MAX]       = "";
-        char                           name_buf[PATH_MAX]          = "";
-        int32_t                        i                           = -1;
-        int32_t                        ret                         = -1;
-        int32_t                        brick_num                   = -1;
-        int32_t                        snap_op                     = -1;
-        int32_t                        snap_status                 = -1;
-        glusterd_conf_t               *priv                        = NULL;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* We can update the missed_snaps_list without acquiring *
-         * any additional locks as big lock will be held.        */
-        for (i = 0; i < missed_snap_count; i++) {
-                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
-                          i);
-                ret = dict_get_str (dict, name_buf, &buf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED,
-                                "Unable to fetch %s", name_buf);
-                        goto out;
-                }
-
-                gf_msg_debug (this->name, 0, "missed_snap_entry = %s",
-                        buf);
-
-                /* Need to make a duplicate string coz the same dictionary *
-                 * is resent to the non-originator nodes */
-                tmp = gf_strdup (buf);
-                if (!tmp) {
-                        ret = -1;
-                        goto out;
-                }
-
-                /* Fetch the node-id, snap-id, brick_num,
-                 * brick_path, snap_op and snap status
-                 */
-                nodeid = strtok_r (tmp, ":", &save_ptr);
-                snap_uuid = strtok_r (NULL, "=", &save_ptr);
-                snap_vol_id = strtok_r (NULL, ":", &save_ptr);
-                brick_num = atoi(strtok_r (NULL, ":", &save_ptr));
-                brick_path = strtok_r (NULL, ":", &save_ptr);
-                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
-                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
-
-                if (!nodeid || !snap_uuid || !brick_path ||
-                    !snap_vol_id || brick_num < 1 || snap_op < 1 ||
-                    snap_status < 1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_INVALID_MISSED_SNAP_ENTRY,
-                                "Invalid missed_snap_entry");
-                        ret = -1;
-                        goto out;
-                }
+    char *buf = NULL;
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char *nodeid = NULL;
+    char *snap_uuid = NULL;
+    char *snap_vol_id = NULL;
+    char *brick_path = NULL;
+    char missed_info[PATH_MAX] = "";
+    char key[64] = "";
+    int keylen;
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t brick_num = -1;
+    int32_t snap_op = -1;
+    int32_t snap_status = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* We can update the missed_snaps_list without acquiring *
+     * any additional locks as big lock will be held.        */
+    for (i = 0; i < missed_snap_count; i++) {
+        keylen = snprintf(key, sizeof(key), "missed_snaps_%d", i);
+        ret = dict_get_strn(dict, key, keylen, &buf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %s", key);
+            goto out;
+        }
+
+        gf_msg_debug(this->name, 0, "missed_snap_entry = %s", buf);
+
+        /* Need to make a duplicate string coz the same dictionary *
+         * is resent to the non-originator nodes */
+        tmp = gf_strdup(buf);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+
+        /* Fetch the node-id, snap-id, brick_num,
+         * brick_path, snap_op and snap status
+         */
+        nodeid = strtok_r(tmp, ":", &save_ptr);
+        snap_uuid = strtok_r(NULL, "=", &save_ptr);
+        snap_vol_id = strtok_r(NULL, ":", &save_ptr);
+        brick_num = atoi(strtok_r(NULL, ":", &save_ptr));
+        brick_path = strtok_r(NULL, ":", &save_ptr);
+        snap_op = atoi(strtok_r(NULL, ":", &save_ptr));
+        snap_status = atoi(strtok_r(NULL, ":", &save_ptr));
 
-                snprintf (missed_info, sizeof(missed_info), "%s:%s",
-                          nodeid, snap_uuid);
+        if (!nodeid || !snap_uuid || !brick_path || !snap_vol_id ||
+            brick_num < 1 || snap_op < 1 || snap_status < 1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_INVALID_MISSED_SNAP_ENTRY,
+                   "Invalid missed_snap_entry");
+            ret = -1;
+            goto out;
+        }
 
-                ret = glusterd_add_new_entry_to_list (missed_info,
-                                                      snap_vol_id,
-                                                      brick_num,
-                                                      brick_path,
-                                                      snap_op,
-                                                      snap_status);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
-                                "Failed to store missed snaps_list");
-                        goto out;
-                }
+        snprintf(missed_info, sizeof(missed_info), "%s:%s", nodeid, snap_uuid);
 
-                GF_FREE (tmp);
-                tmp = NULL;
+        ret = glusterd_add_new_entry_to_list(missed_info, snap_vol_id,
+                                             brick_num, brick_path, snap_op,
+                                             snap_status);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Failed to store missed snaps_list");
+            goto out;
         }
 
-        ret = 0;
+        GF_FREE(tmp);
+        tmp = NULL;
+    }
+
+    ret = 0;
 out:
-        if (tmp)
-                GF_FREE (tmp);
+    if (tmp)
+        GF_FREE(tmp);
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function will restore origin volume to it's snap.
@@ -9743,263 +9846,242 @@ out:
  * @return 0 on success and negative value on error
  */
 int
-gd_restore_snap_volume (dict_t *dict, dict_t *rsp_dict,
-                        glusterd_volinfo_t *orig_vol,
-                        glusterd_volinfo_t *snap_vol,
-                        int32_t volcount)
+gd_restore_snap_volume(dict_t *dict, dict_t *rsp_dict,
+                       glusterd_volinfo_t *orig_vol,
+                       glusterd_volinfo_t *snap_vol, int32_t volcount)
 {
-        int                     ret             = -1;
-        glusterd_volinfo_t      *new_volinfo    = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_conf_t         *conf           = NULL;
-        glusterd_volinfo_t      *temp_volinfo   = NULL;
-        glusterd_volinfo_t      *voliter        = NULL;
-        gf_boolean_t             conf_present   = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_VALIDATE_OR_GOTO (this->name, orig_vol, out);
-        GF_VALIDATE_OR_GOTO (this->name, snap_vol, out);
-        snap = snap_vol->snapshot;
-        GF_VALIDATE_OR_GOTO (this->name, snap, out);
-
-        /* Set the status to under restore so that if the
-         * the node goes down during restore and comes back
-         * the state of the volume can be reverted correctly
-         */
-        snap->snap_status = GD_SNAP_STATUS_UNDER_RESTORE;
-
-        /* We need to save this in disk so that if node goes
-         * down the status is in updated state.
-         */
-        ret = glusterd_store_snap (snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Could not store snap "
-                        "object for %s snap of %s volume", snap_vol->volname,
-                        snap_vol->parent_volname);
-                goto out;
-        }
-
-        /* Snap volume must be stoped before performing the
-         * restore operation.
-         */
-        ret = glusterd_stop_volume (snap_vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_STOP_FAILED,
-                        "Failed to stop "
-                        "snap volume");
-                goto out;
-        }
-
-        /* Create a new volinfo for the restored volume */
-        ret = glusterd_volinfo_dup (snap_vol, &new_volinfo, _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_OP_FAILED, "Failed to create volinfo");
-                goto out;
-        }
-
-        /* Following entries need to be derived from origin volume. */
-        strcpy (new_volinfo->volname, orig_vol->volname);
-        gf_uuid_copy (new_volinfo->volume_id, orig_vol->volume_id);
-        new_volinfo->snap_count = orig_vol->snap_count;
-        gf_uuid_copy (new_volinfo->restored_from_snap,
-                   snap_vol->snapshot->snap_id);
-
-        /* Use the same version as the original version */
-        new_volinfo->version = orig_vol->version;
-
-        /* Copy the snap vol info to the new_volinfo.*/
-        ret = glusterd_snap_volinfo_restore (dict, rsp_dict, new_volinfo,
-                                             snap_vol, volcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_RESTORE_FAIL, "Failed to restore snap");
-                goto out;
-        }
-
-        /* In case a new node is added to the peer, after a snapshot was
-         * taken, the geo-rep files are not synced to that node. This
-         * leads to the failure of snapshot restore. Hence, ignoring the
-         * missing geo-rep files in the new node, and proceeding with
-         * snapshot restore. Once the restore is successful, the missing
-         * geo-rep files can be generated with "gluster volume geo-rep
-         * <master-vol> <slave-vol> create push-pem force"
-         */
-        ret = glusterd_restore_geo_rep_files (snap_vol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_RESTORE_FAIL,
-                        "Failed to restore "
-                        "geo-rep files for snap %s",
-                        snap_vol->snapshot->snapname);
-        }
-
-        ret = glusterd_restore_nfs_ganesha_file (orig_vol, snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_SNAP_RESTORE_FAIL,
-                        "Failed to restore "
-                        "nfs-ganesha export file for snap %s",
-                        snap_vol->snapshot->snapname);
-                goto out;
-        }
-
-        /* Need not save cksum, as we will copy cksum file in *
-         * this function                                           *
-         */
-        ret = glusterd_copy_quota_files (snap_vol, orig_vol, &conf_present);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_RESTORE_FAIL, "Failed to restore "
-                        "quota files for snap %s",
-                        snap_vol->snapshot->snapname);
-                goto out;
-        }
-
-        /* New volinfo always shows the status as created. Therefore
-         * set the status to the original volume's status. */
-        glusterd_set_volume_status (new_volinfo, orig_vol->status);
-
-        cds_list_add_tail (&new_volinfo->vol_list, &conf->volumes);
-
-        ret = glusterd_store_volinfo (new_volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_OP_FAILED, "Failed to store volinfo");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    glusterd_volinfo_t *new_volinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *temp_volinfo = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    gf_boolean_t conf_present = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_VALIDATE_OR_GOTO(this->name, orig_vol, out);
+    GF_VALIDATE_OR_GOTO(this->name, snap_vol, out);
+    snap = snap_vol->snapshot;
+    GF_VALIDATE_OR_GOTO(this->name, snap, out);
+
+    /* Set the status to under restore so that if the
+     * the node goes down during restore and comes back
+     * the state of the volume can be reverted correctly
+     */
+    snap->snap_status = GD_SNAP_STATUS_UNDER_RESTORE;
+
+    /* We need to save this in disk so that if node goes
+     * down the status is in updated state.
+     */
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+               "Could not store snap "
+               "object for %s snap of %s volume",
+               snap_vol->volname, snap_vol->parent_volname);
+        goto out;
+    }
+
+    /* Snap volume must be stopped before performing the
+     * restore operation.
+     */
+    ret = glusterd_stop_volume(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_STOP_FAILED,
+               "Failed to stop "
+               "snap volume");
+        goto out;
+    }
+
+    /* Create a new volinfo for the restored volume */
+    ret = glusterd_volinfo_dup(snap_vol, &new_volinfo, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to create volinfo");
+        goto out;
+    }
+
+    /* Following entries need to be derived from origin volume. */
+    gf_strncpy(new_volinfo->volname, orig_vol->volname,
+               sizeof(new_volinfo->volname));
+    gf_uuid_copy(new_volinfo->volume_id, orig_vol->volume_id);
+    new_volinfo->snap_count = orig_vol->snap_count;
+    gf_uuid_copy(new_volinfo->restored_from_snap, snap_vol->snapshot->snap_id);
+
+    /* Use the same version as the original version */
+    new_volinfo->version = orig_vol->version;
+
+    /* Copy the snap vol info to the new_volinfo.*/
+    ret = glusterd_snap_volinfo_restore(dict, rsp_dict, new_volinfo, snap_vol,
+                                        volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+               "Failed to restore snap");
+        goto out;
+    }
+
+    /* In case a new node is added to the peer, after a snapshot was
+     * taken, the geo-rep files are not synced to that node. This
+     * leads to the failure of snapshot restore. Hence, ignoring the
+     * missing geo-rep files in the new node, and proceeding with
+     * snapshot restore. Once the restore is successful, the missing
+     * geo-rep files can be generated with "gluster volume geo-rep
+     * <master-vol> <slave-vol> create push-pem force"
+     */
+    ret = glusterd_restore_geo_rep_files(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+               "Failed to restore "
+               "geo-rep files for snap %s",
+               snap_vol->snapshot->snapname);
+    }
+
+    /* Need not save cksum, as we will copy cksum file in *
+     * this function                                           *
+     */
+    ret = glusterd_copy_quota_files(snap_vol, orig_vol, &conf_present);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+               "Failed to restore "
+               "quota files for snap %s",
+               snap_vol->snapshot->snapname);
+        goto out;
+    }
+
+    /* New volinfo always shows the status as created. Therefore
+     * set the status to the original volume's status. */
+    glusterd_set_volume_status(new_volinfo, orig_vol->status);
+
+    cds_list_add_tail(&new_volinfo->vol_list, &conf->volumes);
+
+    ret = glusterd_store_volinfo(new_volinfo,
+                                 GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to store volinfo");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (ret) {
-                /* In case of any failure we should free new_volinfo. Doing
-                 * this will also remove the entry we added in conf->volumes
-                 * if it was added there.
-                 */
-                if (new_volinfo)
-                        (void)glusterd_volinfo_delete (new_volinfo);
-        } else {
-                cds_list_for_each_entry_safe (voliter, temp_volinfo,
-                                              &orig_vol->snap_volumes,
-                                              snapvol_list) {
-                        cds_list_add_tail (&voliter->snapvol_list,
-                                           &new_volinfo->snap_volumes);
-                }
+    if (ret) {
+        /* In case of any failure we should free new_volinfo. Doing
+         * this will also remove the entry we added in conf->volumes
+         * if it was added there.
+         */
+        if (new_volinfo)
+            (void)glusterd_volinfo_delete(new_volinfo);
+    } else {
+        cds_list_for_each_entry_safe(voliter, temp_volinfo,
+                                     &orig_vol->snap_volumes, snapvol_list)
+        {
+            cds_list_add_tail(&voliter->snapvol_list,
+                              &new_volinfo->snap_volumes);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
 int
-glusterd_snapshot_get_volnames_uuids (dict_t *dict,
-                                      char *volname,
-                                      gf_getsnap_name_uuid_rsp  *snap_info_rsp)
+glusterd_snapshot_get_volnames_uuids(dict_t *dict, char *volname,
+                                     gf_getsnap_name_uuid_rsp *snap_info_rsp)
 {
-        int                 ret             = -1;
-        int                 snapcount       = 0;
-        char                key[PATH_MAX]   = {0,};
-        glusterd_volinfo_t *snap_vol        = NULL;
-        glusterd_volinfo_t *volinfo         = NULL;
-        glusterd_volinfo_t *tmp_vol         = NULL;
-        xlator_t           *this            = NULL;
-        int                 op_errno        = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volname);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, dict, out,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, volname, out,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, snap_info_rsp, out,
-                                        op_errno, EINVAL);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND,
-                        "Failed to get volinfo of volume %s",
-                        volname);
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
-                                      snapvol_list) {
-
-                if (GLUSTERD_STATUS_STARTED != snap_vol->status)
-                        continue;
-
-                snapcount++;
-
-                /* Set Snap Name */
-                snprintf (key, sizeof (key), "snapname.%d", snapcount);
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                           snap_vol->snapshot->snapname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set "
-                                "snap name in dictionary");
-                        goto out;
-                }
-
-                /* Set Snap ID */
-                snprintf (key, sizeof (key), "snap-id.%d", snapcount);
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                     uuid_utoa(snap_vol->snapshot->snap_id));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set "
-                                "snap id in dictionary");
-                        goto out;
-                }
-
-                /* Snap Volname which is used to activate the snap vol */
-                snprintf (key, sizeof (key), "snap-volname.%d", snapcount);
-                ret = dict_set_dynstr_with_alloc (dict, key, snap_vol->volname);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED, "Failed to set "
-                                "snap id in dictionary");
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "snap-count", snapcount);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_SET_FAILED, "Failed to set snapcount");
-                op_errno = -ret;
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &snap_info_rsp->dict.dict_val,
-                                           &snap_info_rsp->dict.dict_len);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    int snapcount = 0;
+    char key[32] = "";
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    xlator_t *this = NULL;
+    int op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volname);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, dict, out, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, volname, out, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, snap_info_rsp, out, op_errno,
+                                   EINVAL);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Failed to get volinfo of volume %s", volname);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        if (GLUSTERD_STATUS_STARTED != snap_vol->status)
+            continue;
+
+        snapcount++;
+
+        /* Set Snap Name */
+        snprintf(key, sizeof(key), "snapname.%d", snapcount);
+        ret = dict_set_dynstr_with_alloc(dict, key,
+                                         snap_vol->snapshot->snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap name in dictionary");
+            goto out;
+        }
+
+        /* Set Snap ID */
+        snprintf(key, sizeof(key), "snap-id.%d", snapcount);
+        ret = dict_set_dynstr_with_alloc(
+            dict, key, uuid_utoa(snap_vol->snapshot->snap_id));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap id in dictionary");
+            goto out;
+        }
+
+        /* Snap Volname which is used to activate the snap vol */
+        snprintf(key, sizeof(key), "snap-volname.%d", snapcount);
+        ret = dict_set_dynstr_with_alloc(dict, key, snap_vol->volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap id in dictionary");
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(dict, "snap-count", SLEN("snap-count"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        op_errno = -ret;
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &snap_info_rsp->dict.dict_val,
+                                      &snap_info_rsp->dict.dict_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        snap_info_rsp->op_ret = ret;
-        snap_info_rsp->op_errno = op_errno;
-        snap_info_rsp->op_errstr = "";
+    snap_info_rsp->op_ret = ret;
+    snap_info_rsp->op_errno = op_errno;
+    snap_info_rsp->op_errstr = "";
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-statedump.c b/xlators/mgmt/glusterd/src/glusterd-statedump.c
index a01a6b8bfed..225d10cc546 100644
--- a/xlators/mgmt/glusterd/src/glusterd-statedump.c
+++ b/xlators/mgmt/glusterd/src/glusterd-statedump.c
@@ -8,240 +8,236 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "glusterd.h"
 #include "glusterd-shd-svc.h"
 #include "glusterd-quotad-svc.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-locks.h"
 #include "glusterd-messages.h"
 
 static void
-glusterd_dump_peer (glusterd_peerinfo_t *peerinfo, char *input_key, int index,
-                    gf_boolean_t xpeers)
+glusterd_dump_peer(glusterd_peerinfo_t *peerinfo, char *input_key, int index,
+                   gf_boolean_t xpeers)
 {
-        char   subkey[50]               = {0,};
-        char   key[GF_DUMP_MAX_BUF_LEN] = {0,};
+    char subkey[GF_DUMP_MAX_BUF_LEN + 11] = "";
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
 
-        strncpy (key, input_key, (GF_DUMP_MAX_BUF_LEN - 1));
+    strncpy(key, input_key, sizeof(key) - 1);
 
-        snprintf (subkey, sizeof (subkey), "%s%d", key, index);
+    snprintf(subkey, sizeof(subkey), "%s%d", key, index);
 
-        gf_proc_dump_build_key (key, subkey, "uuid");
-        gf_proc_dump_write (key, "%s",
-                            uuid_utoa (peerinfo->uuid));
+    gf_proc_dump_build_key(key, subkey, "uuid");
+    gf_proc_dump_write(key, "%s", uuid_utoa(peerinfo->uuid));
 
-        gf_proc_dump_build_key (key, subkey, "hostname");
-        gf_proc_dump_write (key, "%s", peerinfo->hostname);
+    gf_proc_dump_build_key(key, subkey, "hostname");
+    gf_proc_dump_write(key, "%s", peerinfo->hostname);
 
-        gf_proc_dump_build_key (key, subkey, "port");
-        gf_proc_dump_write (key, "%d", peerinfo->port);
+    gf_proc_dump_build_key(key, subkey, "port");
+    gf_proc_dump_write(key, "%d", peerinfo->port);
 
-        gf_proc_dump_build_key (key, subkey, "state");
-        gf_proc_dump_write (key, "%d", peerinfo->state.state);
+    gf_proc_dump_build_key(key, subkey, "state");
+    gf_proc_dump_write(key, "%d", peerinfo->state.state);
 
-        gf_proc_dump_build_key (key, subkey, "quorum-action");
-        gf_proc_dump_write (key, "%d", peerinfo->quorum_action);
+    gf_proc_dump_build_key(key, subkey, "quorum-action");
+    gf_proc_dump_write(key, "%d", peerinfo->quorum_action);
 
-        gf_proc_dump_build_key (key, subkey, "quorum-contrib");
-        gf_proc_dump_write (key, "%d",
-                            peerinfo->quorum_contrib);
+    gf_proc_dump_build_key(key, subkey, "quorum-contrib");
+    gf_proc_dump_write(key, "%d", peerinfo->quorum_contrib);
 
-        gf_proc_dump_build_key (key, subkey, "detaching");
-        gf_proc_dump_write (key, "%d", peerinfo->detaching);
-
-        gf_proc_dump_build_key (key, subkey, "locked");
-        gf_proc_dump_write (key, "%d", peerinfo->locked);
+    gf_proc_dump_build_key(key, subkey, "detaching");
+    gf_proc_dump_write(key, "%d", peerinfo->detaching);
 
+    gf_proc_dump_build_key(key, subkey, "locked");
+    gf_proc_dump_write(key, "%d", peerinfo->locked);
 }
 
-
 static void
-glusterd_dump_peer_rpcstat (glusterd_peerinfo_t *peerinfo, char *input_key,
-                            int index)
+glusterd_dump_peer_rpcstat(glusterd_peerinfo_t *peerinfo, char *input_key,
+                           int index)
 {
-        rpc_clnt_connection_t *conn                                = NULL;
-        int                    ret                                 = -1;
-        rpc_clnt_t            *rpc                                 = NULL;
-        char                   rpcsvc_peername[RPCSVC_PEER_STRLEN] = {0,};
-        char                   subkey[50]                          = {0,};
-        char                   key[GF_DUMP_MAX_BUF_LEN]            = {0,};
-
-        strncpy (key, input_key, (GF_DUMP_MAX_BUF_LEN - 1));
-
-        /* Dump the rpc connection statistics */
-        rpc = peerinfo->rpc;
-        if (rpc) {
-                conn = &rpc->conn;
-                snprintf (subkey, sizeof (subkey), "%s%d", key, index);
-                ret = rpcsvc_transport_peername (conn->trans,
-                                                 (char *)&rpcsvc_peername,
-                                                 sizeof (rpcsvc_peername));
-                if (!ret) {
-                        gf_proc_dump_build_key (key, subkey, "rpc.peername");
-                        gf_proc_dump_write (key, "%s", rpcsvc_peername);
-                }
-                gf_proc_dump_build_key (key, subkey, "rpc.connected");
-                gf_proc_dump_write (key, "%d", conn->connected);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.total-bytes-read");
-                gf_proc_dump_write (key, "%"PRIu64,
-                                    conn->trans->total_bytes_read);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.total-bytes-written");
-                gf_proc_dump_write (key, "%"PRIu64,
-                                    conn->trans->total_bytes_write);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.ping_msgs_sent");
-                gf_proc_dump_write (key, "%"PRIu64, conn->pingcnt);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.msgs_sent");
-                gf_proc_dump_write (key, "%"PRIu64, conn->msgcnt);
+    rpc_clnt_connection_t *conn = NULL;
+    int ret = -1;
+    rpc_clnt_t *rpc = NULL;
+    char rpcsvc_peername[RPCSVC_PEER_STRLEN] = "";
+    char subkey[GF_DUMP_MAX_BUF_LEN + 11] = "";
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+
+    strncpy(key, input_key, sizeof(key) - 1);
+
+    /* Dump the rpc connection statistics */
+    rpc = peerinfo->rpc;
+    if (rpc) {
+        conn = &rpc->conn;
+        snprintf(subkey, sizeof(subkey), "%s%d", key, index);
+        ret = rpcsvc_transport_peername(conn->trans, (char *)&rpcsvc_peername,
+                                        sizeof(rpcsvc_peername));
+        if (!ret) {
+            gf_proc_dump_build_key(key, subkey, "rpc.peername");
+            gf_proc_dump_write(key, "%s", rpcsvc_peername);
         }
+        gf_proc_dump_build_key(key, subkey, "rpc.connected");
+        gf_proc_dump_write(key, "%d", conn->connected);
 
-}
+        gf_proc_dump_build_key(key, subkey, "rpc.total-bytes-read");
+        gf_proc_dump_write(key, "%" PRIu64, conn->trans->total_bytes_read);
+
+        gf_proc_dump_build_key(key, subkey, "rpc.total-bytes-written");
+        gf_proc_dump_write(key, "%" PRIu64, conn->trans->total_bytes_write);
 
+        gf_proc_dump_build_key(key, subkey, "rpc.ping_msgs_sent");
+        gf_proc_dump_write(key, "%" PRIu64, conn->pingcnt);
+
+        gf_proc_dump_build_key(key, subkey, "rpc.msgs_sent");
+        gf_proc_dump_write(key, "%" PRIu64, conn->msgcnt);
+    }
+}
 
 static void
-glusterd_dump_client_details (glusterd_conf_t *conf)
+glusterd_dump_client_details(glusterd_conf_t *conf)
 {
-        rpc_transport_t         *xprt                     = NULL;
-        char                     key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        char                     subkey[50]               = {0,};
-        int                      index                    = 1;
-
-        pthread_mutex_lock (&conf->xprt_lock);
+    rpc_transport_t *xprt = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+    char subkey[50] = "";
+    int index = 1;
+
+    pthread_mutex_lock(&conf->xprt_lock);
+    {
+        list_for_each_entry(xprt, &conf->xprt_list, list)
         {
-                list_for_each_entry (xprt, &conf->xprt_list, list) {
-                        snprintf (subkey, sizeof (subkey), "glusterd.client%d",
-                                  index);
-
-                        gf_proc_dump_build_key (key, subkey, "identifier");
-                        gf_proc_dump_write (key, "%s",
-                                            xprt->peerinfo.identifier);
-
-                        gf_proc_dump_build_key (key, subkey, "volname");
-                        gf_proc_dump_write (key, "%s",
-                                            xprt->peerinfo.volname);
-
-                        gf_proc_dump_build_key (key, subkey, "max-op-version");
-                        gf_proc_dump_write (key, "%u",
-                                            xprt->peerinfo.max_op_version);
-
-                        gf_proc_dump_build_key (key, subkey, "min-op-version");
-                        gf_proc_dump_write (key, "%u",
-                                            xprt->peerinfo.min_op_version);
-                        index++;
-                }
+            snprintf(subkey, sizeof(subkey), "glusterd.client%d", index);
+
+            gf_proc_dump_build_key(key, subkey, "identifier");
+            gf_proc_dump_write(key, "%s", xprt->peerinfo.identifier);
+
+            gf_proc_dump_build_key(key, subkey, "volname");
+            gf_proc_dump_write(key, "%s", xprt->peerinfo.volname);
+
+            gf_proc_dump_build_key(key, subkey, "max-op-version");
+            gf_proc_dump_write(key, "%u", xprt->peerinfo.max_op_version);
+
+            gf_proc_dump_build_key(key, subkey, "min-op-version");
+            gf_proc_dump_write(key, "%u", xprt->peerinfo.min_op_version);
+            index++;
         }
-        pthread_mutex_unlock (&conf->xprt_lock);
+    }
+    pthread_mutex_unlock(&conf->xprt_lock);
 }
 
-
 /* The following function is just for dumping mgmt_v3_lock dictionary, any other
  * dict passed to this API will not work */
 
 static void
-glusterd_dict_mgmt_v3_lock_statedump (dict_t *dict)
+glusterd_dict_mgmt_v3_lock_statedump(dict_t *dict)
 {
-        int          ret                       = 0;
-        int          dumplen                   = 0;
-        data_pair_t *trav                      = NULL;
-        char         key[GF_DUMP_MAX_BUF_LEN]  = {0,};
-        char         dump[64*1024]             = {0,};
-
-        if (!dict) {
-                gf_msg_callingfn ("glusterd", GF_LOG_WARNING, EINVAL,
-                                  GD_MSG_DICT_EMPTY,
-                                  "dict NULL");
-                goto out;
-        }
-        for (trav = dict->members_list; trav; trav = trav->next) {
-                if (strstr (trav->key, "debug.last-success-bt") != NULL) {
-                        ret = snprintf (&dump[dumplen], sizeof(dump) - dumplen,
-                                        "\n\t%s:%s", trav->key,
-                                        trav->value->data);
-                } else {
-                        ret = snprintf (&dump[dumplen], sizeof(dump) - dumplen,
-                                        "\n\t%s:%s", trav->key,
-                                        uuid_utoa (((glusterd_mgmt_v3_lock_obj *)
-                                         (trav->value->data))->lock_owner));
-                }
-                if ((ret == -1) || !ret)
-                        return;
-                dumplen += ret;
+    int ret = 0;
+    int dumplen = 0;
+    data_pair_t *trav = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+    char dump[64 * 1024] = "";
+
+    if (!dict) {
+        gf_msg_callingfn("glusterd", GF_LOG_WARNING, EINVAL, GD_MSG_DICT_EMPTY,
+                         "dict NULL");
+        goto out;
+    }
+    for (trav = dict->members_list; trav; trav = trav->next) {
+        if (strstr(trav->key, "debug.last-success-bt") != NULL) {
+            ret = snprintf(&dump[dumplen], sizeof(dump) - dumplen, "\n\t%s:%s",
+                           trav->key, trav->value->data);
+        } else {
+            ret = snprintf(
+                &dump[dumplen], sizeof(dump) - dumplen, "\n\t%s:%s", trav->key,
+                uuid_utoa(((glusterd_mgmt_v3_lock_obj *)(trav->value->data))
+                              ->lock_owner));
         }
+        if ((ret == -1) || !ret)
+            return;
+        dumplen += ret;
+    }
 
-        if (dumplen) {
-                gf_proc_dump_build_key (key, "glusterd", "mgmt_v3_lock");
-                gf_proc_dump_write (key, "%s", dump);
-        }
+    if (dumplen) {
+        gf_proc_dump_build_key(key, "glusterd", "mgmt_v3_lock");
+        gf_proc_dump_write(key, "%s", dump);
+    }
 
 out:
-        return;
+    return;
 }
 
-
 int
-glusterd_dump_priv (xlator_t *this)
+glusterd_dump_priv(xlator_t *this)
 {
-        int                   index                    = 1;
-        glusterd_conf_t      *priv                     = NULL;
-        char                  key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        glusterd_peerinfo_t  *peerinfo                 = NULL;
-        glusterd_volinfo_t   *volinfo                  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+    glusterd_conf_t *priv = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+    int port = 0;
+    struct pmap_registry *pmap = NULL;
 
-        priv = this->private;
-        if (!priv)
-                return 0;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        gf_proc_dump_build_key (key, "xlator.glusterd", "priv");
-        gf_proc_dump_add_section (key);
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                gf_proc_dump_build_key (key, "glusterd", "my-uuid");
-                gf_proc_dump_write (key, "%s", uuid_utoa (priv->uuid));
-
-                gf_proc_dump_build_key (key, "glusterd", "working-directory");
-                gf_proc_dump_write (key, "%s", priv->workdir);
-
-                gf_proc_dump_build_key (key, "glusterd", "max-op-version");
-                gf_proc_dump_write (key, "%d", GD_OP_VERSION_MAX);
-
-                gf_proc_dump_build_key (key, "glusterd", "min-op-version");
-                gf_proc_dump_write (key, "%d", GD_OP_VERSION_MIN);
-
-                gf_proc_dump_build_key (key, "glusterd", "current-op-version");
-                gf_proc_dump_write (key, "%d", priv->op_version);
-
-                gf_proc_dump_build_key (key, "glusterd", "ping-timeout");
-                gf_proc_dump_write (key, "%d", priv->ping_timeout);
-
-                gf_proc_dump_build_key (key, "glusterd", "shd.online");
-                gf_proc_dump_write (key, "%d", priv->shd_svc.online);
-
-                gf_proc_dump_build_key (key, "glusterd", "nfs.online");
-                gf_proc_dump_write (key, "%d", priv->nfs_svc.online);
-
-                gf_proc_dump_build_key (key, "glusterd", "quotad.online");
-                gf_proc_dump_write (key, "%d", priv->quotad_svc.online);
-
-                gf_proc_dump_build_key (key, "glusterd", "bitd.online");
-                gf_proc_dump_write (key, "%d", priv->bitd_svc.online);
-
-                gf_proc_dump_build_key (key, "glusterd", "scrub.online");
-                gf_proc_dump_write (key, "%d", priv->scrub_svc.online);
+    priv = this->private;
+    if (!priv)
+        return 0;
 
-                GLUSTERD_DUMP_PEERS (&priv->peers, uuid_list, _gf_false);
-                glusterd_dump_client_details (priv);
-                glusterd_dict_mgmt_v3_lock_statedump(priv->mgmt_v3_lock);
-                dict_dump_to_statedump (priv->opts, "options", "glusterd");
+    gf_proc_dump_build_key(key, "xlator.glusterd", "priv");
+    gf_proc_dump_add_section("%s", key);
+
+    pthread_mutex_lock(&priv->mutex);
+    {
+        gf_proc_dump_build_key(key, "glusterd", "my-uuid");
+        gf_proc_dump_write(key, "%s", uuid_utoa(priv->uuid));
+
+        gf_proc_dump_build_key(key, "glusterd", "working-directory");
+        gf_proc_dump_write(key, "%s", priv->workdir);
+
+        gf_proc_dump_build_key(key, "glusterd", "max-op-version");
+        gf_proc_dump_write(key, "%d", GD_OP_VERSION_MAX);
+
+        gf_proc_dump_build_key(key, "glusterd", "min-op-version");
+        gf_proc_dump_write(key, "%d", GD_OP_VERSION_MIN);
+
+        gf_proc_dump_build_key(key, "glusterd", "current-op-version");
+        gf_proc_dump_write(key, "%d", priv->op_version);
+
+        gf_proc_dump_build_key(key, "glusterd", "ping-timeout");
+        gf_proc_dump_write(key, "%d", priv->ping_timeout);
+#ifdef BUILD_GNFS
+        gf_proc_dump_build_key(key, "glusterd", "nfs.online");
+        gf_proc_dump_write(key, "%d", priv->nfs_svc.online);
+#endif
+        gf_proc_dump_build_key(key, "glusterd", "quotad.online");
+        gf_proc_dump_write(key, "%d", priv->quotad_svc.online);
+
+        gf_proc_dump_build_key(key, "glusterd", "bitd.online");
+        gf_proc_dump_write(key, "%d", priv->bitd_svc.online);
+
+        gf_proc_dump_build_key(key, "glusterd", "scrub.online");
+        gf_proc_dump_write(key, "%d", priv->scrub_svc.online);
+
+        /* Dump peer details */
+        GLUSTERD_DUMP_PEERS(&priv->peers, uuid_list, _gf_false);
+
+        /* Dump pmap data structure from base port to last alloc */
+        pmap = priv->pmap;
+        if (pmap) {
+            for (port = pmap->base_port; port <= pmap->last_alloc; port++) {
+                gf_proc_dump_build_key(key, "glusterd", "pmap_port");
+                gf_proc_dump_write(key, "%d", port);
+                gf_proc_dump_build_key(key, "glusterd", "pmap[%d].type", port);
+                gf_proc_dump_write(key, "%d", pmap->ports[port].type);
+                gf_proc_dump_build_key(key, "glusterd", "pmap[%d].brickname",
+                                       port);
+                gf_proc_dump_write(key, "%s", pmap->ports[port].brickname);
+            }
         }
-        pthread_mutex_unlock (&priv->mutex);
+        /* Dump client details */
+        glusterd_dump_client_details(priv);
+
+        /* Dump mgmt_v3_lock from the dictionary if any */
+        glusterd_dict_mgmt_v3_lock_statedump(priv->mgmt_v3_lock);
+        dict_dump_to_statedump(priv->opts, "options", "glusterd");
+    }
+    pthread_mutex_unlock(&priv->mutex);
 
 out:
-        return 0;
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-statedump.h b/xlators/mgmt/glusterd/src/glusterd-statedump.h
index 3ac8659f293..b5ef1f48e82 100644
--- a/xlators/mgmt/glusterd/src/glusterd-statedump.h
+++ b/xlators/mgmt/glusterd/src/glusterd-statedump.h
@@ -11,8 +11,8 @@
 #ifndef _GLUSTERD_STATEDUMP_H_
 #define _GLUSTERD_STATEDUMP_H_
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 int
-glusterd_dump_priv (xlator_t *this);
+glusterd_dump_priv(xlator_t *this);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index aa2be17bd9a..d94dceb10b7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -11,34 +11,32 @@
 #include "glusterd-op-sm.h"
 #include <inttypes.h>
 
-
-#include "globals.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "dict.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
-#include "syscall.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-hooks.h"
-#include "store.h"
+#include <glusterfs/store.h>
 #include "glusterd-store.h"
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-messages.h"
 
 #include "rpc-clnt.h"
-#include "common-utils.h"
-#include "quota-common-utils.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/quota-common-utils.h>
 
 #include <sys/resource.h>
 #include <inttypes.h>
@@ -50,239 +48,269 @@
 #include "mntent_compat.h"
 #endif
 
+#define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv)                            \
+    do {                                                                       \
+        int32_t _brick_len;                                                    \
+        if (volinfo->is_snap_volume) {                                         \
+            _brick_len = snprintf(path, PATH_MAX, "%s/snaps/%s/%s/%s",         \
+                                  priv->workdir, volinfo->snapshot->snapname,  \
+                                  volinfo->volname, GLUSTERD_BRICK_INFO_DIR);  \
+        } else {                                                               \
+            _brick_len = snprintf(path, PATH_MAX, "%s/%s/%s/%s",               \
+                                  priv->workdir, GLUSTERD_VOLUME_DIR_PREFIX,   \
+                                  volinfo->volname, GLUSTERD_BRICK_INFO_DIR);  \
+        }                                                                      \
+        if ((_brick_len < 0) || (_brick_len >= PATH_MAX)) {                    \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
 void
-glusterd_replace_slash_with_hyphen (char *str)
+glusterd_replace_slash_with_hyphen(char *str)
 {
-        char                    *ptr = NULL;
+    char *ptr = NULL;
 
-        ptr = strchr (str, '/');
+    ptr = strchr(str, '/');
 
-        while (ptr) {
-                *ptr = '-';
-                ptr = strchr (str, '/');
-        }
+    while (ptr) {
+        *ptr = '-';
+        ptr = strchr(ptr, '/');
+    }
 }
 
 int32_t
-glusterd_store_create_brick_dir (glusterd_volinfo_t *volinfo)
+glusterd_store_create_brick_dir(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret = -1;
-        char                    brickdirpath[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv = NULL;
+    int32_t ret = -1;
+    char brickdirpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
-        ret = gf_store_mkdir (brickdirpath);
+    GLUSTERD_GET_BRICK_DIR(brickdirpath, volinfo, priv);
+    ret = gf_store_mkdir(brickdirpath);
 
-        return ret;
+    return ret;
 }
 
 static void
-glusterd_store_key_vol_brick_set (glusterd_brickinfo_t *brickinfo,
-                                char *key_vol_brick, size_t len)
+glusterd_store_key_vol_brick_set(glusterd_brickinfo_t *brickinfo,
+                                 char *key_vol_brick, size_t len)
 {
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (key_vol_brick);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(key_vol_brick);
+    GF_ASSERT(len >= PATH_MAX);
 
-        snprintf (key_vol_brick, len, "%s", brickinfo->path);
-        glusterd_replace_slash_with_hyphen (key_vol_brick);
+    snprintf(key_vol_brick, len, "%s", brickinfo->path);
+    glusterd_replace_slash_with_hyphen(key_vol_brick);
 }
 
 static void
-glusterd_store_brickinfofname_set (glusterd_brickinfo_t *brickinfo,
-                                char *brickfname, size_t len)
+glusterd_store_brickinfofname_set(glusterd_brickinfo_t *brickinfo,
+                                  char *brickfname, size_t len)
 {
-        char                    key_vol_brick[PATH_MAX] = {0};
+    char key_vol_brick[PATH_MAX] = {0};
 
-        GF_ASSERT (brickfname);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(brickfname);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(len >= PATH_MAX);
 
-        glusterd_store_key_vol_brick_set (brickinfo, key_vol_brick,
-                                        sizeof (key_vol_brick));
-        snprintf (brickfname, len, "%s:%s", brickinfo->hostname, key_vol_brick);
+    glusterd_store_key_vol_brick_set(brickinfo, key_vol_brick,
+                                     sizeof(key_vol_brick));
+    snprintf(brickfname, len, "%s:%s", brickinfo->hostname, key_vol_brick);
 }
 
 static void
-glusterd_store_brickinfopath_set (glusterd_volinfo_t *volinfo,
-                              glusterd_brickinfo_t *brickinfo,
-                              char *brickpath, size_t len)
+glusterd_store_brickinfopath_set(glusterd_volinfo_t *volinfo,
+                                 glusterd_brickinfo_t *brickinfo,
+                                 char *brickpath, size_t len)
 {
-        char                    brickfname[PATH_MAX] = {0};
-        char                    brickdirpath[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv = NULL;
-
-        GF_ASSERT (brickpath);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (len >= PATH_MAX);
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
-        glusterd_store_brickinfofname_set (brickinfo, brickfname,
-                                        sizeof (brickfname));
-        snprintf (brickpath, len, "%s/%s", brickdirpath, brickfname);
+    char brickfname[PATH_MAX] = {0};
+    char brickdirpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(brickpath);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(len >= PATH_MAX);
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_BRICK_DIR(brickdirpath, volinfo, priv);
+    glusterd_store_brickinfofname_set(brickinfo, brickfname,
+                                      sizeof(brickfname));
+    snprintf(brickpath, len, "%s/%s", brickdirpath, brickfname);
 }
 
 static void
-glusterd_store_snapd_path_set (glusterd_volinfo_t *volinfo,
-                               char *snapd_path, size_t len)
+glusterd_store_snapd_path_set(glusterd_volinfo_t *volinfo, char *snapd_path,
+                              size_t len)
 {
-        char                    volpath[PATH_MAX] = {0, };
-        glusterd_conf_t         *priv = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(len >= PATH_MAX);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
 
-        snprintf (snapd_path, len, "%s/snapd.info", volpath);
+    snprintf(snapd_path, len, "%s/snapd.info", volpath);
 }
 
 gf_boolean_t
-glusterd_store_is_valid_brickpath (char *volname, char *brick)
+glusterd_store_is_valid_brickpath(char *volname, char *brick)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        int32_t                 ret = 0;
-        size_t                  volname_len = strlen (volname);
-        xlator_t                *this = NULL;
-        int                     bpath_len = 0;
-        const char              delim[2] = "/";
-        char                    *sub_dir = NULL;
-        char                    *saveptr = NULL;
-        char                    *brickpath_ptr = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo, _gf_false,
-                                                 NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_BRICK_CREATION_FAIL, "Failed to create brick "
-                        "info for brick %s", brick);
-                ret = 0;
-                goto out;
-        }
-        ret = glusterd_volinfo_new (&volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_VOLFILE_CREATE_FAIL, "Failed to create volinfo");
-                ret = 0;
-                goto out;
-        }
-        if (volname_len >= sizeof (volinfo->volname)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        GD_MSG_NAME_TOO_LONG, "volume name too long");
-                ret = 0;
-                goto out;
-        }
-        memcpy (volinfo->volname, volname, volname_len+1);
-
-        /* Check whether brickpath is less than PATH_MAX */
-        ret = 1;
-        bpath_len = strlen (brickinfo->path);
-
-        if (brickinfo->path[bpath_len - 1] != '/') {
-                if (strlen (brickinfo->path) >= PATH_MAX) {
-                        ret = 0;
-                        goto out;
-                }
-        } else {
-                /* Path has a trailing "/" which should not be considered in
-                 * length check validation
-                 */
-                if (strlen (brickinfo->path) >= PATH_MAX + 1) {
-                        ret = 0;
-                        goto out;
-                }
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t ret = 0;
+    size_t volname_len = strlen(volname);
+    xlator_t *this = NULL;
+    int bpath_len = 0;
+    const char delim[2] = "/";
+    char *sub_dir = NULL;
+    char *saveptr = NULL;
+    char *brickpath_ptr = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_false, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_CREATION_FAIL,
+               "Failed to create brick "
+               "info for brick %s",
+               brick);
+        ret = 0;
+        goto out;
+    }
+    ret = glusterd_volinfo_new(&volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volinfo");
+        ret = 0;
+        goto out;
+    }
+    if (volname_len >= sizeof(volinfo->volname)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NAME_TOO_LONG,
+               "volume name too long");
+        ret = 0;
+        goto out;
+    }
+    memcpy(volinfo->volname, volname, volname_len + 1);
+
+    /* Check whether brickpath is less than PATH_MAX */
+    ret = 1;
+    bpath_len = strlen(brickinfo->path);
+
+    if (brickinfo->path[bpath_len - 1] != '/') {
+        if (bpath_len >= PATH_MAX) {
+            ret = 0;
+            goto out;
+        }
+    } else {
+        /* Path has a trailing "/" which should not be considered in
+         * length check validation
+         */
+        if (bpath_len >= PATH_MAX + 1) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        /* The following validation checks whether each sub directories in the
-         * brick path meets the POSIX max length validation
-         */
+    /* The following validation checks whether each sub directories in the
+     * brick path meets the POSIX max length validation
+     */
 
-        brickpath_ptr = brickinfo->path;
-        sub_dir = strtok_r (brickpath_ptr, delim, &saveptr);
+    brickpath_ptr = brickinfo->path;
+    sub_dir = strtok_r(brickpath_ptr, delim, &saveptr);
 
-        while (sub_dir != NULL) {
-                if (strlen(sub_dir) >= _POSIX_PATH_MAX) {
-                        ret = 0;
-                        goto out;
-                }
-                sub_dir = strtok_r (NULL, delim, &saveptr);
+    while (sub_dir != NULL) {
+        if (strlen(sub_dir) >= _POSIX_PATH_MAX) {
+            ret = 0;
+            goto out;
         }
+        sub_dir = strtok_r(NULL, delim, &saveptr);
+    }
 
 out:
-        if (brickinfo)
-                glusterd_brickinfo_delete (brickinfo);
-        if (volinfo)
-                glusterd_volinfo_unref (volinfo);
+    if (brickinfo)
+        glusterd_brickinfo_delete(brickinfo);
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_volinfo_brick_fname_write (int vol_fd,
+glusterd_store_volinfo_brick_fname_write(int vol_fd,
                                          glusterd_brickinfo_t *brickinfo,
-                                         int32_t brick_count)
+                                         int32_t brick_count,
+                                         int is_thin_arbiter)
 {
-        char                    key[PATH_MAX] = {0,};
-        char                    brickfname[PATH_MAX] = {0,};
-        int32_t                 ret = -1;
-
-        snprintf (key, sizeof (key), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
-                  brick_count);
-        glusterd_store_brickinfofname_set (brickinfo, brickfname,
-                                        sizeof (brickfname));
-        ret = gf_store_save_value (vol_fd, key, brickfname);
-        if (ret)
-                goto out;
-
-out:
-        return ret;
+    char key[64] = {
+        0,
+    };
+    char brickfname[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = -1;
+
+    if (!is_thin_arbiter) {
+        snprintf(key, sizeof(key), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
+                 brick_count);
+    } else {
+        snprintf(key, sizeof(key), "%s-%d", GLUSTERD_STORE_KEY_VOL_TA_BRICK,
+                 brick_count);
+    }
+    glusterd_store_brickinfofname_set(brickinfo, brickfname,
+                                      sizeof(brickfname));
+    ret = gf_store_save_value(vol_fd, key, brickfname);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_brick_shandle_on_absence (glusterd_volinfo_t *volinfo,
-                                                glusterd_brickinfo_t *brickinfo)
+glusterd_store_create_brick_shandle_on_absence(glusterd_volinfo_t *volinfo,
+                                               glusterd_brickinfo_t *brickinfo)
 {
-        char                    brickpath[PATH_MAX] = {0,};
-        int32_t                 ret = 0;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-
-        glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
-                                          sizeof (brickpath));
-        ret = gf_store_handle_create_on_absence (&brickinfo->shandle,
-                                                 brickpath);
-        return ret;
+    char brickpath[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+
+    glusterd_store_brickinfopath_set(volinfo, brickinfo, brickpath,
+                                     sizeof(brickpath));
+    ret = gf_store_handle_create_on_absence(&brickinfo->shandle, brickpath);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_snapd_shandle_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_snapd_shandle_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char                    snapd_path[PATH_MAX] = {0,};
-        int32_t                 ret = 0;
+    char snapd_path[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_snapd_path_set (volinfo, snapd_path,
-                                       sizeof (snapd_path));
-        ret = gf_store_handle_create_on_absence (&volinfo->snapd.handle,
-                                                 snapd_path);
-        return ret;
+    glusterd_store_snapd_path_set(volinfo, snapd_path, sizeof(snapd_path));
+    ret = gf_store_handle_create_on_absence(&volinfo->snapd.handle, snapd_path);
+    return ret;
 }
 
 /* Store the bricks snapshot details only if required
@@ -290,485 +318,415 @@ glusterd_store_create_snapd_shandle_on_absence (glusterd_volinfo_t *volinfo)
  * The snapshot details will be stored only if the cluster op-version is
  * greater than or equal to 4
  */
-int
-gd_store_brick_snap_details_write (int fd, glusterd_brickinfo_t *brickinfo)
+static int
+gd_store_brick_snap_details_write(int fd, glusterd_brickinfo_t *brickinfo)
 {
-        int ret = -1;
-        xlator_t *this = NULL;
-        glusterd_conf_t *conf = NULL;
-        char value[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (fd > 0), out);
-        GF_VALIDATE_OR_GOTO (this->name, (brickinfo != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char value[5 * PATH_MAX];
+    uint total_len = 0;
 
-        if (strlen(brickinfo->device_path) > 0) {
-                snprintf (value, sizeof(value), "%s", brickinfo->device_path);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH, value);
-                if (ret)
-                        goto out;
-        }
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
 
-        if (strlen(brickinfo->mount_dir) > 0) {
-                memset (value, 0, sizeof (value));
-                snprintf (value, sizeof(value), "%s", brickinfo->mount_dir);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR, value);
-                if (ret)
-                        goto out;
-        }
-
-        if (strlen (brickinfo->fstype) > 0) {
-                snprintf (value, sizeof (value), "%s", brickinfo->fstype);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_FSTYPE, value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FS_LABEL_UPDATE_FAIL, "Failed to save "
-                                "brick fs type of brick %s", brickinfo->path);
-                        goto out;
-                }
-        }
-
-        if (strlen (brickinfo->mnt_opts) > 0) {
-                snprintf (value, sizeof (value), "%s", brickinfo->mnt_opts);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_MNTOPTS, value);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRK_MOUNTOPTS_FAIL, "Failed to save "
-                                "brick mnt opts of brick %s", brickinfo->path);
-                        goto out;
-                }
-        }
-
-        memset (value, 0, sizeof (value));
-        snprintf (value, sizeof(value), "%d", brickinfo->snap_status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
-                                   value);
+    GF_VALIDATE_OR_GOTO(this->name, (fd > 0), out);
+    GF_VALIDATE_OR_GOTO(this->name, (brickinfo != NULL), out);
 
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    if (brickinfo->device_path[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                       brickinfo->device_path);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    if (brickinfo->mount_dir[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
+                       brickinfo->mount_dir);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    if (brickinfo->fstype[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_FSTYPE, brickinfo->fstype);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    if (brickinfo->mnt_opts[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_MNTOPTS, brickinfo->mnt_opts);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%d\n",
+                   GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                   brickinfo->snap_status);
+    if (ret < 0 || ret >= sizeof(value) - total_len) {
+        ret = -1;
+        goto err;
+    }
+    total_len += ret;
+
+    ret = snprintf(value + total_len, sizeof(value) - total_len,
+                   "%s=%" PRIu64 "\n", GLUSTERD_STORE_KEY_BRICK_FSID,
+                   brickinfo->statfs_fsid);
+    if (ret < 0 || ret >= sizeof(value) - total_len) {
+        ret = -1;
+        goto err;
+    }
+
+    ret = gf_store_save_items(fd, value);
+err:
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FS_LABEL_UPDATE_FAIL,
+               "Failed to save "
+               "snap detils of brick %s",
+               brickinfo->path);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
+static int32_t
+glusterd_store_brickinfo_write(int fd, glusterd_brickinfo_t *brickinfo)
 {
-        char                    value[256] = {0,};
-        int32_t                 ret = 0;
-
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (fd > 0);
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
-                                   brickinfo->hostname);
-        if (ret)
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PATH,
-                                   brickinfo->path);
-        if (ret)
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
-                                   brickinfo->path);
-        if (ret)
-                goto out;
-
-        snprintf (value, sizeof(value), "%d", brickinfo->port);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT, value);
-
-        snprintf (value, sizeof(value), "%d", brickinfo->rdma_port);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
-                                   value);
-
-        snprintf (value, sizeof(value), "%d", brickinfo->decommissioned);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
-                                   value);
-        if (ret)
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_ID,
-                                   brickinfo->brick_id);
-        if (ret)
-                goto out;
-
-        ret = gd_store_brick_snap_details_write (fd, brickinfo);
-        if (ret)
-                goto out;
-
-        if (!brickinfo->vg[0])
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
-                                         brickinfo->vg);
+    char value[5 * PATH_MAX];
+    int32_t ret = -1;
+
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(fd > 0);
+
+    ret = snprintf(value, sizeof(value),
+                   "%s=%s\n%s=%s\n%s=%s\n%s=%s\n%s=%d\n%s=%d\n%s=%d\n%s=%s\n",
+                   GLUSTERD_STORE_KEY_BRICK_UUID, uuid_utoa(brickinfo->uuid),
+                   GLUSTERD_STORE_KEY_BRICK_HOSTNAME, brickinfo->hostname,
+                   GLUSTERD_STORE_KEY_BRICK_PATH, brickinfo->path,
+                   GLUSTERD_STORE_KEY_BRICK_REAL_PATH, brickinfo->path,
+                   GLUSTERD_STORE_KEY_BRICK_PORT, brickinfo->port,
+                   GLUSTERD_STORE_KEY_BRICK_RDMA_PORT, brickinfo->rdma_port,
+                   GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                   brickinfo->decommissioned, GLUSTERD_STORE_KEY_BRICK_ID,
+                   brickinfo->brick_id);
+
+    if (ret < 0 || ret >= sizeof(value)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(fd, value);
+    if (ret)
+        goto out;
+
+    ret = gd_store_brick_snap_details_write(fd, brickinfo);
+    if (ret)
+        goto out;
+
+    if (!brickinfo->vg[0])
+        goto out;
+
+    ret = gf_store_save_value(fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                              brickinfo->vg);
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_snapd_write (int fd, glusterd_volinfo_t *volinfo)
+glusterd_store_snapd_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        char                    value[256] = {0,};
-        int32_t                 ret        = 0;
-        xlator_t               *this       = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (fd > 0);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        snprintf (value, sizeof(value), "%d", volinfo->snapd.port);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAPD_PORT, value);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPD_PORT_STORE_FAIL,
-                        "failed to store the snapd "
-                        "port of volume %s", volinfo->volname);
-
-
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    char value[64] = {
+        0,
+    };
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(fd > 0);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    snprintf(value, sizeof(value), "%d", volinfo->snapd.port);
+    ret = gf_store_save_value(fd, GLUSTERD_STORE_KEY_SNAPD_PORT, value);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_PORT_STORE_FAIL,
+               "failed to store the snapd "
+               "port of volume %s",
+               volinfo->volname);
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int32_t
-glusterd_store_perform_brick_store (glusterd_brickinfo_t *brickinfo)
+static int32_t
+glusterd_store_perform_brick_store(glusterd_brickinfo_t *brickinfo)
 {
-        int                         fd = -1;
-        int32_t                     ret = -1;
-        GF_ASSERT (brickinfo);
-
-        fd = gf_store_mkstemp (brickinfo->shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_store_brickinfo_write (fd, brickinfo);
-        if (ret)
-                goto out;
+    int fd = -1;
+    int32_t ret = -1;
+    GF_ASSERT(brickinfo);
+
+    fd = gf_store_mkstemp(brickinfo->shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_store_brickinfo_write(fd, brickinfo);
+    if (ret)
+        goto out;
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (brickinfo->shandle);
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0)) {
+        gf_store_unlink_tmppath(brickinfo->shandle);
+    }
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_perform_snapd_store (glusterd_volinfo_t *volinfo)
+glusterd_store_perform_snapd_store(glusterd_volinfo_t *volinfo)
 {
-        int                         fd  = -1;
-        int32_t                     ret = -1;
-        xlator_t                  *this = NULL;
-
-        GF_ASSERT (volinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        fd = gf_store_mkstemp (volinfo->snapd.handle);
-        if (fd <= 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED, "failed to create the "
-                        "temporary file for the snapd store handle of volume "
-                        "%s", volinfo->volname);
-                goto out;
-        }
-
-        ret = glusterd_store_snapd_write (fd, volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPD_PORT_STORE_FAIL,
-                        "failed to write snapd port "
-                        "info to store handle (volume: %s", volinfo->volname);
-                goto out;
-        }
-
-        ret = gf_store_rename_tmppath (volinfo->snapd.handle);
+    int fd = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    fd = gf_store_mkstemp(volinfo->snapd.handle);
+    if (fd <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "failed to create the "
+               "temporary file for the snapd store handle of volume "
+               "%s",
+               volinfo->volname);
+        goto out;
+    }
+
+    ret = glusterd_store_snapd_write(fd, volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_PORT_STORE_FAIL,
+               "failed to write snapd port "
+               "info to store handle (volume: %s",
+               volinfo->volname);
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(volinfo->snapd.handle);
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (volinfo->snapd.handle);
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(volinfo->snapd.handle);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int32_t
-glusterd_store_brickinfo (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *brickinfo, int32_t brick_count,
-                          int vol_fd)
+static int32_t
+glusterd_store_brickinfo(glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo, int32_t brick_count,
+                         int vol_fd, int is_thin_arbiter)
 {
-        int32_t                 ret = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
 
-        ret = glusterd_store_volinfo_brick_fname_write (vol_fd, brickinfo,
-                                                       brick_count);
-        if (ret)
-                goto out;
+    ret = glusterd_store_volinfo_brick_fname_write(
+        vol_fd, brickinfo, brick_count, is_thin_arbiter);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_create_brick_dir (volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_create_brick_shandle_on_absence (volinfo,
-                                                              brickinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_brick_shandle_on_absence(volinfo, brickinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_perform_brick_store (brickinfo);
+    ret = glusterd_store_perform_brick_store(brickinfo);
 out:
-        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_snapd_info (glusterd_volinfo_t *volinfo)
+glusterd_store_snapd_info(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret  = -1;
-        xlator_t               *this = NULL;
-
-        GF_ASSERT (volinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_store_create_snapd_shandle_on_absence (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_HANDLE_CREATE_FAIL,
-                        "failed to create store "
-                        "handle for snapd (volume: %s)", volinfo->volname);
-                goto out;
-        }
-
-        ret = glusterd_store_perform_snapd_store (volinfo);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPD_INFO_STORE_FAIL,
-                        "failed to store snapd info "
-                        "of the volume %s", volinfo->volname);
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_store_create_snapd_shandle_on_absence(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_CREATE_FAIL,
+               "failed to create store "
+               "handle for snapd (volume: %s)",
+               volinfo->volname);
+        goto out;
+    }
+
+    ret = glusterd_store_perform_snapd_store(volinfo);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_INFO_STORE_FAIL,
+               "failed to store snapd info "
+               "of the volume %s",
+               volinfo->volname);
 
 out:
-        if (ret)
-                gf_store_unlink_tmppath (volinfo->snapd.handle);
+    if (ret)
+        gf_store_unlink_tmppath(volinfo->snapd.handle);
 
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo, char *delete_path)
+glusterd_store_delete_brick(glusterd_brickinfo_t *brickinfo, char *delete_path)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        char                    brickpath[PATH_MAX] = {0,};
-        char                    *ptr = NULL;
-        char                    *tmppath = NULL;
-        xlator_t                *this = NULL;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    char brickpath[PATH_MAX] = {
+        0,
+    };
+    char *ptr = NULL;
+    char *tmppath = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickinfo);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickinfo);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        tmppath = gf_strdup (brickinfo->path);
+    tmppath = gf_strdup(brickinfo->path);
 
-        ptr = strchr (tmppath, '/');
+    ptr = strchr(tmppath, '/');
 
-        while (ptr) {
-                *ptr = '-';
-                ptr = strchr (tmppath, '/');
-        }
-
-        snprintf (brickpath, sizeof (brickpath),
-                  "%s/"GLUSTERD_BRICK_INFO_DIR"/%s:%s", delete_path,
-                  brickinfo->hostname, tmppath);
+    while (ptr) {
+        *ptr = '-';
+        ptr = strchr(tmppath, '/');
+    }
 
-        GF_FREE (tmppath);
+    snprintf(brickpath, sizeof(brickpath),
+             "%s/" GLUSTERD_BRICK_INFO_DIR "/%s:%s", delete_path,
+             brickinfo->hostname, tmppath);
 
-        ret = sys_unlink (brickpath);
+    GF_FREE(tmppath);
 
-        if ((ret < 0) && (errno != ENOENT)) {
-                gf_msg_debug (this->name, 0, "Unlink failed on %s",
-                              brickpath);
-                ret = -1;
-                goto out;
-        } else {
-                ret = 0;
-        }
+    ret = sys_unlink(brickpath);
 
-out:
-        if (brickinfo->shandle) {
-                gf_store_handle_destroy (brickinfo->shandle);
-                brickinfo->shandle = NULL;
-        }
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo, char *delete_path)
-{
-        int32_t                 ret = 0;
-        glusterd_brickinfo_t    *tmp = NULL;
-        glusterd_conf_t         *priv = NULL;
-        char                    brickdir [PATH_MAX] = {0,};
-        DIR                     *dir = NULL;
-        struct dirent           *entry = NULL;
-        char                    path[PATH_MAX] = {0,};
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-
-        cds_list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-                ret = glusterd_store_delete_brick (tmp, delete_path);
-                if (ret)
-                        goto out;
-        }
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        snprintf (brickdir, sizeof (brickdir), "%s/%s", delete_path,
-                  GLUSTERD_BRICK_INFO_DIR);
-
-        dir = sys_opendir (brickdir);
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-
-        while (entry) {
-                snprintf (path, sizeof (path), "%s/%s",
-                          brickdir, entry->d_name);
-                ret = sys_unlink (path);
-                if (ret && errno != ENOENT) {
-                        gf_msg_debug (this->name, 0, "Unable to unlink %s",
-                                      path);
-                }
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
-
-        sys_closedir (dir);
-
-        ret = sys_rmdir (brickdir);
+    if ((ret < 0) && (errno != ENOENT)) {
+        gf_msg_debug(this->name, 0, "Unlink failed on %s", brickpath);
+        ret = -1;
+        goto out;
+    } else {
+        ret = 0;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-        return ret;
+    if (brickinfo->shandle) {
+        gf_store_handle_destroy(brickinfo->shandle);
+        brickinfo->shandle = NULL;
+    }
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-_storeslaves (dict_t *this, char *key, data_t *value, void *data)
+_storeopts(dict_t *dict_value, char *key, data_t *value, void *data)
 {
-        int32_t                      ret = 0;
-        gf_store_handle_t           *shandle = NULL;
-        xlator_t                    *xl = NULL;
-
-        xl = THIS;
-        GF_ASSERT (xl);
-
-        shandle = (gf_store_handle_t*)data;
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->fd > 0);
-        GF_ASSERT (shandle->path);
-        GF_ASSERT (key);
-        GF_ASSERT (value && value->data);
-
-        if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return -1;
-
-        if (!key)
-                return -1;
-        if (!value || !value->data)
-                return -1;
-
-        gf_msg_debug (xl->name, 0, "Storing in volinfo:key= %s, val=%s",
-                key, value->data);
-
-        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
-        if (ret) {
-                gf_msg (xl->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_HANDLE_WRITE_FAIL,
-                        "Unable to write into store"
-                        " handle for path: %s", shandle->path);
-                return -1;
-        }
-        return 0;
-}
-
-
-int _storeopts (dict_t *this, char *key, data_t *value, void *data)
-{
-        int32_t                      ret = 0;
-        int32_t                      exists = 0;
-        gf_store_handle_t           *shandle = NULL;
-        xlator_t                    *xl = NULL;
-
-        xl = THIS;
-        GF_ASSERT (xl);
-
-        shandle = (gf_store_handle_t*)data;
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->fd > 0);
-        GF_ASSERT (shandle->path);
-        GF_ASSERT (key);
-        GF_ASSERT (value && value->data);
-
-        if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return -1;
+    int32_t ret = 0;
+    int32_t exists = 0;
+    int32_t option_len = 0;
+    gf_store_handle_t *shandle = NULL;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+    xlator_t *this = NULL;
 
-        if (!key)
-                return -1;
-        if (!value || !value->data)
-                return -1;
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (is_key_glusterd_hooks_friendly (key)) {
-                exists = 1;
+    dict_data = (glusterd_volinfo_data_store_t *)data;
+    shandle = dict_data->shandle;
 
-        } else {
-                exists = glusterd_check_option_exists (key, NULL);
-        }
+    GF_ASSERT(shandle);
+    GF_ASSERT(shandle->fd > 0);
+    GF_ASSERT(key);
+    GF_ASSERT(value);
+    GF_ASSERT(value->data);
 
-        if (1 == exists) {
-                gf_msg_debug (xl->name, 0, "Storing in volinfo:key= %s, "
-                        "val=%s", key, value->data);
+    if (dict_data->key_check == 1) {
+        if (is_key_glusterd_hooks_friendly(key)) {
+            exists = 1;
 
         } else {
-                gf_msg_debug (xl->name, 0, "Discarding:key= %s, val=%s",
-                        key, value->data);
-                return 0;
-        }
-
-        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
-        if (ret) {
-                gf_msg (xl->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_HANDLE_WRITE_FAIL,
-                        "Unable to write into store"
-                        " handle for path: %s", shandle->path);
-                return -1;
-        }
+            exists = glusterd_check_option_exists(key, NULL);
+        }
+    }
+    if (exists == 1 || dict_data->key_check == 0) {
+        gf_msg_debug(this->name, 0,
+                     "Storing in buffer for volinfo:key= %s, "
+                     "val=%s",
+                     key, value->data);
+    } else {
+        gf_msg_debug(this->name, 0, "Discarding:key= %s, val=%s", key,
+                     value->data);
         return 0;
+    }
+
+    /*
+     * The option_len considers the length of the key value
+     * pair and along with that '=' and '\n', but as value->len
+     * already considers a NULL at the end of the data, adding
+     * just 1.
+     */
+    option_len = strlen(key) + value->len + 1;
+
+    if ((VOLINFO_BUFFER_SIZE - dict_data->buffer_len - 1) < option_len) {
+        ret = gf_store_save_items(shandle->fd, dict_data->buffer);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
+            return -1;
+        }
+        dict_data->buffer_len = 0;
+        dict_data->buffer[0] = '\0';
+    }
+    ret = snprintf(dict_data->buffer + dict_data->buffer_len, option_len + 1,
+                   "%s=%s\n", key, value->data);
+    if (ret < 0 || ret > option_len + 1) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COPY_FAIL, NULL);
+        return -1;
+    }
+
+    dict_data->buffer_len += ret;
+
+    return 0;
 }
 
 /* Store the volumes snapshot details only if required
@@ -776,3416 +734,3963 @@ int _storeopts (dict_t *this, char *key, data_t *value, void *data)
  * The snapshot details will be stored only if the cluster op-version is
  * greater than or equal to 4
  */
-int
-glusterd_volume_write_snap_details (int fd, glusterd_volinfo_t *volinfo)
+static int
+glusterd_volume_write_snap_details(int fd, glusterd_volinfo_t *volinfo)
 {
-        int              ret           = -1;
-        xlator_t        *this          = NULL;
-        glusterd_conf_t *conf          = NULL;
-        char             buf[PATH_MAX] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (fd > 0), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volinfo != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%s", volinfo->parent_volname);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PARENT_VOLNAME, buf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_FAIL, "Failed to store "
-                        GLUSTERD_STORE_KEY_PARENT_VOLNAME);
-                goto out;
-        }
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
-                                   uuid_utoa (volinfo->restored_from_snap));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_HANDLE_WRITE_FAIL,
-                        "Unable to write restored_from_snap");
-                goto out;
-        }
-
-        memset (buf, 0, sizeof (buf));
-        snprintf (buf, sizeof (buf), "%"PRIu64, volinfo->snap_max_hard_limit);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                   buf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HARD_LIMIT_SET_FAIL,
-                        "Unable to write snap-max-hard-limit");
-                goto out;
-        }
-
-        ret = glusterd_store_snapd_info (volinfo);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPD_INFO_STORE_FAIL, "snapd info store failed "
-                        "volume: %s", volinfo->volname);
-
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char buf[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (fd > 0), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volinfo != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "%s=%s\n%s=%s\n%s=%" PRIu64 "\n",
+                   GLUSTERD_STORE_KEY_PARENT_VOLNAME, volinfo->parent_volname,
+                   GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
+                   uuid_utoa(volinfo->restored_from_snap),
+                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                   volinfo->snap_max_hard_limit);
+    if (ret < 0 || ret >= sizeof(buf)) {
+        ret = -1;
+        goto err;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret) {
+        goto err;
+    }
+    ret = glusterd_store_snapd_info(volinfo);
+err:
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_WRITE_FAIL,
+               "Failed to write snap details"
+               " for volume %s",
+               volinfo->volname);
 out:
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPINFO_WRITE_FAIL,
-                        "Failed to write snap details"
-                        " for volume %s", volinfo->volname);
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_volume_write_tier_details (int fd, glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_volume_exclude_options_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        int32_t      ret            = -1;
-        char         buf[PATH_MAX]  = "";
+    char *str = NULL;
+    char buf[PATH_MAX];
+    uint total_len = 0;
+    int32_t ret = -1;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(this);
+    GF_ASSERT(fd > 0);
+    GF_ASSERT(volinfo);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%d\n%s=%d\n%s=%d\n%s=%d\n%s=%d\n%s=%d\n",
+                   GLUSTERD_STORE_KEY_VOL_TYPE, volinfo->type,
+                   GLUSTERD_STORE_KEY_VOL_COUNT, volinfo->brick_count,
+                   GLUSTERD_STORE_KEY_VOL_STATUS, volinfo->status,
+                   GLUSTERD_STORE_KEY_VOL_SUB_COUNT, volinfo->sub_count,
+                   GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, volinfo->stripe_count,
+                   GLUSTERD_STORE_KEY_VOL_REPLICA_CNT, volinfo->replica_count);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    if ((conf->op_version >= GD_OP_VERSION_3_7_6) && volinfo->arbiter_count) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                       GLUSTERD_STORE_KEY_VOL_ARBITER_CNT,
+                       volinfo->arbiter_count);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+        ret = snprintf(
+            buf + total_len, sizeof(buf) - total_len, "%s=%d\n%s=%d\n",
+            GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT, volinfo->disperse_count,
+            GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT, volinfo->redundancy_count);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%d\n%s=%d\n%s=%s\n", GLUSTERD_STORE_KEY_VOL_VERSION,
+                   volinfo->version, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
+                   volinfo->transport_type, GLUSTERD_STORE_KEY_VOL_ID,
+                   uuid_utoa(volinfo->volume_id));
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    str = glusterd_auth_get_username(volinfo);
+    if (str) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_USERNAME, str);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    str = glusterd_auth_get_password(volinfo);
+    if (str) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_PASSWORD, str);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n%s=%d\n",
+                   GLUSTERD_STORE_KEY_VOL_OP_VERSION, volinfo->op_version,
+                   GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                   volinfo->client_op_version);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    if (conf->op_version >= GD_OP_VERSION_3_7_6) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                       GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
+                       volinfo->quota_xattr_version);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+    if (conf->op_version >= GD_OP_VERSION_3_10_0) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=0\n",
+                       GF_TIER_ENABLED);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    if ((conf->op_version >= GD_OP_VERSION_7_0) &&
+        volinfo->thin_arbiter_count) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                       GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT,
+                       volinfo->thin_arbiter_count);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret)
+        goto out;
+
+    ret = glusterd_volume_write_snap_details(fd, volinfo);
 
-        if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.cold_brick_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_COLD_COUNT, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d",
-                  volinfo->tier_info.cold_replica_count);
-        ret = gf_store_save_value (fd,
-                                   GLUSTERD_STORE_KEY_COLD_REPLICA_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.cold_disperse_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_COLD_DISPERSE_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d",
-                  volinfo->tier_info.cold_redundancy_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_COLD_REDUNDANCY_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.hot_brick_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_HOT_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.hot_replica_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.hot_type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_HOT_TYPE, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.cold_type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_COLD_TYPE, buf);
-        if (ret)
-                goto out;
-
- out:
-        return ret;
+out:
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_VALS_WRITE_FAIL,
+               "Unable to write volume "
+               "values for %s",
+               volinfo->volname);
+    return ret;
 }
 
-int32_t
-glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
+static void
+glusterd_store_voldirpath_set(glusterd_volinfo_t *volinfo, char *voldirpath)
 {
-        char        *str            = NULL;
-        char         buf[PATH_MAX]  = "";
-        int32_t      ret            = -1;
-        xlator_t    *this           = NULL;
-        glusterd_conf_t  *conf      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (volinfo);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->brick_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_COUNT, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STATUS, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->sub_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->stripe_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->replica_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        if ((conf->op_version >= GD_OP_VERSION_3_7_6) &&
-            volinfo->arbiter_count) {
-                snprintf (buf, sizeof (buf), "%d", volinfo->arbiter_count);
-                ret = gf_store_save_value (fd,
-                                           GLUSTERD_STORE_KEY_VOL_ARBITER_CNT,
-                                           buf);
-                if (ret)
-                        goto out;
-        }
-
-        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                snprintf (buf, sizeof (buf), "%d", volinfo->disperse_count);
-                ret = gf_store_save_value (fd,
-                                           GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT,
-                                           buf);
-                if (ret)
-                        goto out;
-
-                snprintf (buf, sizeof (buf), "%d", volinfo->redundancy_count);
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
-                                           buf);
-                if (ret)
-                        goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->version);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_VERSION, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->transport_type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TRANSPORT, buf);
-        if (ret)
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
-                                   uuid_utoa (volinfo->volume_id));
-        if (ret)
-                goto out;
-
-        str = glusterd_auth_get_username (volinfo);
-        if (str) {
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_USERNAME,
-                                           str);
-                if (ret)
-                goto out;
-        }
-
-        str = glusterd_auth_get_password (volinfo);
-        if (str) {
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PASSWORD,
-                                           str);
-                if (ret)
-                        goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->op_version);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_OP_VERSION, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->client_op_version);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
-                                   buf);
-        if (ret)
-                goto out;
-        if (volinfo->caps) {
-                snprintf (buf, sizeof (buf), "%d", volinfo->caps);
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CAPS,
-                                           buf);
-                if (ret)
-                        goto out;
-        }
-
-        if (conf->op_version >= GD_OP_VERSION_3_7_6) {
-                snprintf (buf, sizeof (buf), "%d",
-                          volinfo->quota_xattr_version);
-                ret = gf_store_save_value (fd,
-                                           GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
-                                           buf);
-                if (ret)
-                        goto out;
-        }
-
-        ret = glusterd_volume_write_tier_details (fd, volinfo);
+    glusterd_conf_t *priv = NULL;
 
-        ret = glusterd_volume_write_snap_details (fd, volinfo);
+    GF_ASSERT(volinfo);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-out:
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOL_VALS_WRITE_FAIL, "Unable to write volume "
-                        "values for %s", volinfo->volname);
-        return ret;
+    GLUSTERD_GET_VOLUME_DIR(voldirpath, volinfo, priv);
 }
 
 static void
-glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath,
-                               size_t len)
+glusterd_store_piddirpath_set(glusterd_volinfo_t *volinfo, char *piddirpath)
 {
-        glusterd_conf_t         *priv = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    GF_ASSERT(volinfo);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv);
+    GLUSTERD_GET_VOLUME_PID_DIR(piddirpath, volinfo, priv);
 }
 
 static int32_t
-glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
+glusterd_store_create_volume_dirs(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret = -1;
-        char                    voldirpath[PATH_MAX] = {0,};
+    int32_t ret = -1;
+    char dirpath[PATH_MAX] = {
+        0,
+    };
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        ret = gf_store_mkdir (voldirpath);
+    glusterd_store_voldirpath_set(volinfo, dirpath);
+    ret = gf_store_mkdir(dirpath);
+    if (ret)
+        goto out;
 
-        gf_msg_debug (THIS->name, 0, "Returning with %d", ret);
-        return ret;
+    glusterd_store_piddirpath_set(volinfo, dirpath);
+    ret = gf_store_mkdir(dirpath);
+    if (ret)
+        goto out;
+
+out:
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_snap_dir (glusterd_snap_t *snap)
+glusterd_store_create_snap_dir(glusterd_snap_t *snap)
 {
-        int32_t            ret                   = -1;
-        char               snapdirpath[PATH_MAX] = {0,};
-        glusterd_conf_t   *priv                  = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-
-        GLUSTERD_GET_SNAP_DIR (snapdirpath, snap, priv);
-
-        ret = mkdir_p (snapdirpath, 0755, _gf_true);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Failed to create snaps dir "
-                        "%s", snapdirpath);
-        }
-        return ret;
+    int32_t ret = -1;
+    char snapdirpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+
+    GLUSTERD_GET_SNAP_DIR(snapdirpath, snap, priv);
+
+    ret = mkdir_p(snapdirpath, 0755, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create snaps dir "
+               "%s",
+               snapdirpath);
+    }
+    return ret;
 }
 
-int32_t
-glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_store_volinfo_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        int32_t                         ret = -1;
-        gf_store_handle_t              *shandle = NULL;
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volinfo->shandle);
-
-        shandle = volinfo->shandle;
-        ret = glusterd_volume_exclude_options_write (fd, volinfo);
-        if (ret)
-                goto out;
-
-        shandle->fd = fd;
-        dict_foreach (volinfo->dict, _storeopts, shandle);
+    int32_t ret = -1;
+    gf_store_handle_t *shandle = NULL;
+    GF_ASSERT(fd > 0);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volinfo->shandle);
+    xlator_t *this = NULL;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    shandle = volinfo->shandle;
+
+    dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
+                          gf_gld_mt_volinfo_dict_data_t);
+    if (dict_data == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+        return -1;
+    }
+
+    ret = glusterd_volume_exclude_options_write(fd, volinfo);
+    if (ret) {
+        goto out;
+    }
+
+    dict_data->shandle = shandle;
+    dict_data->key_check = 1;
+
+    shandle->fd = fd;
+    dict_foreach(volinfo->dict, _storeopts, (void *)dict_data);
+
+    dict_data->key_check = 0;
+    dict_foreach(volinfo->gsync_slaves, _storeopts, (void *)dict_data);
+
+    if (dict_data->buffer_len > 0) {
+        ret = gf_store_save_items(fd, dict_data->buffer);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
+            goto out;
+        }
+    }
 
-        dict_foreach (volinfo->gsync_slaves, _storeslaves, shandle);
-        shandle->fd = 0;
+    shandle->fd = 0;
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(dict_data);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int32_t
-glusterd_store_snapinfo_write (glusterd_snap_t *snap)
+static int32_t
+glusterd_store_snapinfo_write(glusterd_snap_t *snap)
 {
-        int32_t  ret            = -1;
-        int      fd             = 0;
-        char     buf[PATH_MAX]  = "";
-
-        GF_ASSERT (snap);
-
-        fd = gf_store_mkstemp (snap->shandle);
-        if (fd <= 0)
-               goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_ID,
-                                   uuid_utoa (snap->snap_id));
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", snap->snap_status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_STATUS, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", snap->snap_restored);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_RESTORED, buf);
-        if (ret)
-                goto out;
-
-        if (snap->description) {
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_DESC,
-                                           snap->description);
-                if (ret)
-                        goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%ld", snap->time_stamp);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, buf);
+    int32_t ret = -1;
+    int fd = 0;
+    char buf[PATH_MAX];
+    uint total_len = 0;
+
+    GF_ASSERT(snap);
+
+    fd = gf_store_mkstemp(snap->shandle);
+    if (fd <= 0)
+        goto out;
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%s\n%s=%d\n%s=%d\n", GLUSTERD_STORE_KEY_SNAP_ID,
+                   uuid_utoa(snap->snap_id), GLUSTERD_STORE_KEY_SNAP_STATUS,
+                   snap->snap_status, GLUSTERD_STORE_KEY_SNAP_RESTORED,
+                   snap->snap_restored);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    if (snap->description) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_SNAP_DESC, snap->description);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%ld\n",
+                   GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, snap->time_stamp);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    ret = gf_store_save_items(fd, buf);
 
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static void
-glusterd_store_volfpath_set (glusterd_volinfo_t *volinfo, char *volfpath,
-                             size_t len)
+glusterd_store_volfpath_set(glusterd_volinfo_t *volinfo, char *volfpath,
+                            size_t len)
 {
-        char    voldirpath[PATH_MAX] = {0,};
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volfpath);
-        GF_ASSERT (len <= PATH_MAX);
-
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        snprintf (volfpath, len, "%s/%s", voldirpath, GLUSTERD_VOLUME_INFO_FILE);
+    char voldirpath[PATH_MAX] = {
+        0,
+    };
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volfpath);
+    GF_ASSERT(len <= PATH_MAX);
+
+    glusterd_store_voldirpath_set(volinfo, voldirpath);
+    snprintf(volfpath, len, "%s/%s", voldirpath, GLUSTERD_VOLUME_INFO_FILE);
 }
 
 static void
-glusterd_store_node_state_path_set (glusterd_volinfo_t *volinfo,
-                                    char *node_statepath, size_t len)
+glusterd_store_node_state_path_set(glusterd_volinfo_t *volinfo,
+                                   char *node_statepath, size_t len)
 {
-        char    voldirpath[PATH_MAX] = {0,};
-        GF_ASSERT (volinfo);
-        GF_ASSERT (node_statepath);
-        GF_ASSERT (len <= PATH_MAX);
-
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        snprintf (node_statepath, len, "%s/%s", voldirpath,
-                  GLUSTERD_NODE_STATE_FILE);
+    char voldirpath[PATH_MAX] = {
+        0,
+    };
+    GF_ASSERT(volinfo);
+    GF_ASSERT(node_statepath);
+    GF_ASSERT(len <= PATH_MAX);
+
+    glusterd_store_voldirpath_set(volinfo, voldirpath);
+    snprintf(node_statepath, len, "%s/%s", voldirpath,
+             GLUSTERD_NODE_STATE_FILE);
 }
 
 static void
-glusterd_store_quota_conf_path_set (glusterd_volinfo_t *volinfo,
-                                    char *quota_conf_path, size_t len)
+glusterd_store_quota_conf_path_set(glusterd_volinfo_t *volinfo,
+                                   char *quota_conf_path, size_t len)
 {
-        char    voldirpath[PATH_MAX] = {0,};
-        GF_ASSERT (volinfo);
-        GF_ASSERT (quota_conf_path);
-        GF_ASSERT (len <= PATH_MAX);
-
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        snprintf (quota_conf_path, len, "%s/%s", voldirpath,
-                  GLUSTERD_VOLUME_QUOTA_CONFIG);
+    char voldirpath[PATH_MAX] = {
+        0,
+    };
+    GF_ASSERT(volinfo);
+    GF_ASSERT(quota_conf_path);
+    GF_ASSERT(len <= PATH_MAX);
+
+    glusterd_store_voldirpath_set(volinfo, voldirpath);
+    snprintf(quota_conf_path, len, "%s/%s", voldirpath,
+             GLUSTERD_VOLUME_QUOTA_CONFIG);
 }
 
 static void
-glusterd_store_missed_snaps_list_path_set (char *missed_snaps_list,
-                                           size_t len)
+glusterd_store_missed_snaps_list_path_set(char *missed_snaps_list, size_t len)
 {
-        glusterd_conf_t *priv = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (missed_snaps_list);
-        GF_ASSERT (len <= PATH_MAX);
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(missed_snaps_list);
+    GF_ASSERT(len <= PATH_MAX);
 
-        snprintf (missed_snaps_list, len, "%s/snaps/"
-                  GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
+    snprintf(missed_snaps_list, len,
+             "%s/snaps/" GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
 }
 
 static void
-glusterd_store_snapfpath_set (glusterd_snap_t *snap, char *snap_fpath,
-                              size_t len)
+glusterd_store_snapfpath_set(glusterd_snap_t *snap, char *snap_fpath,
+                             size_t len)
 {
-        glusterd_conf_t *priv = NULL;
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-        GF_ASSERT (snap_fpath);
-        GF_ASSERT (len <= PATH_MAX);
-
-        snprintf (snap_fpath, len, "%s/snaps/%s/%s", priv->workdir,
-                  snap->snapname, GLUSTERD_SNAP_INFO_FILE);
+    glusterd_conf_t *priv = NULL;
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+    GF_ASSERT(snap_fpath);
+    GF_ASSERT(len <= PATH_MAX);
+
+    snprintf(snap_fpath, len, "%s/snaps/%s/%s", priv->workdir, snap->snapname,
+             GLUSTERD_SNAP_INFO_FILE);
 }
 
 int32_t
-glusterd_store_create_vol_shandle_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_vol_shandle_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char            volfpath[PATH_MAX] = {0};
-        int32_t         ret = 0;
+    char volfpath[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_volfpath_set (volinfo, volfpath, sizeof (volfpath));
-        ret = gf_store_handle_create_on_absence (&volinfo->shandle, volfpath);
-        return ret;
+    glusterd_store_volfpath_set(volinfo, volfpath, sizeof(volfpath));
+    ret = gf_store_handle_create_on_absence(&volinfo->shandle, volfpath);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_nodestate_sh_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_nodestate_sh_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char            node_state_path[PATH_MAX] = {0};
-        int32_t         ret                   = 0;
+    char node_state_path[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_node_state_path_set (volinfo, node_state_path,
-                                            sizeof (node_state_path));
-        ret =
-          gf_store_handle_create_on_absence (&volinfo->node_state_shandle,
-                                             node_state_path);
+    glusterd_store_node_state_path_set(volinfo, node_state_path,
+                                       sizeof(node_state_path));
+    ret = gf_store_handle_create_on_absence(&volinfo->node_state_shandle,
+                                            node_state_path);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_create_quota_conf_sh_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_quota_conf_sh_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char            quota_conf_path[PATH_MAX] = {0};
-        int32_t         ret                       = 0;
+    char quota_conf_path[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_quota_conf_path_set (volinfo, quota_conf_path,
-                                            sizeof (quota_conf_path));
-        ret =
-          gf_store_handle_create_on_absence (&volinfo->quota_conf_shandle,
-                                             quota_conf_path);
+    glusterd_store_quota_conf_path_set(volinfo, quota_conf_path,
+                                       sizeof(quota_conf_path));
+    ret = gf_store_handle_create_on_absence(&volinfo->quota_conf_shandle,
+                                            quota_conf_path);
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-glusterd_store_create_missed_snaps_list_shandle_on_absence ()
+glusterd_store_create_missed_snaps_list_shandle_on_absence()
 {
-        char               missed_snaps_list[PATH_MAX] = "";
-        int32_t            ret                         = -1;
-        glusterd_conf_t   *priv                        = NULL;
-        xlator_t          *this                        = NULL;
+    char missed_snaps_list[PATH_MAX] = "";
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        glusterd_store_missed_snaps_list_path_set (missed_snaps_list,
-                                                   sizeof(missed_snaps_list));
+    glusterd_store_missed_snaps_list_path_set(missed_snaps_list,
+                                              sizeof(missed_snaps_list));
 
-        ret = gf_store_handle_create_on_absence
-                                            (&priv->missed_snaps_list_shandle,
-                                             missed_snaps_list);
-        return ret;
+    ret = gf_store_handle_create_on_absence(&priv->missed_snaps_list_shandle,
+                                            missed_snaps_list);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_snap_shandle_on_absence (glusterd_snap_t *snap)
+glusterd_store_create_snap_shandle_on_absence(glusterd_snap_t *snap)
 {
-        char            snapfpath[PATH_MAX] = {0};
-        int32_t         ret = 0;
+    char snapfpath[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (snap);
+    GF_ASSERT(snap);
 
-        glusterd_store_snapfpath_set (snap, snapfpath, sizeof (snapfpath));
-        ret = gf_store_handle_create_on_absence (&snap->shandle, snapfpath);
-        return ret;
+    glusterd_store_snapfpath_set(snap, snapfpath, sizeof(snapfpath));
+    ret = gf_store_handle_create_on_absence(&snap->shandle, snapfpath);
+    return ret;
 }
 
-int32_t
-glusterd_store_brickinfos (glusterd_volinfo_t *volinfo, int vol_fd)
+static int32_t
+glusterd_store_brickinfos(glusterd_volinfo_t *volinfo, int vol_fd)
 {
-        int32_t                 ret = 0;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int32_t                 brick_count = 0;
-
-        GF_ASSERT (volinfo);
+    int32_t ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    int32_t brick_count = 0;
+    int32_t ta_brick_count = 0;
+
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = glusterd_store_brickinfo(volinfo, brickinfo, brick_count, vol_fd,
+                                       0);
+        if (ret)
+            goto out;
+        brick_count++;
+    }
+    if (volinfo->thin_arbiter_count == 1) {
+        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
+                                        glusterd_brickinfo_t, brick_list);
+        ret = glusterd_store_brickinfo(volinfo, ta_brickinfo, ta_brick_count,
+                                       vol_fd, 1);
+        if (ret)
+            goto out;
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_store_brickinfo (volinfo, brickinfo,
-                                brick_count, vol_fd);
-                if (ret)
-                        goto out;
-                brick_count++;
-        }
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
-}
-
-int
-_gd_store_rebalance_dict (dict_t *dict, char *key, data_t *value, void *data)
-{
-        int             ret = -1;
-        int             fd = 0;
-
-        fd = *(int *)data;
-
-        ret = gf_store_save_value (fd, key, value->data);
-
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
+glusterd_store_node_state_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        int     ret             = -1;
-        char    buf[PATH_MAX]   = {0, };
-
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (volinfo);
-
-        if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.defrag_cmd);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.defrag_status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
-                                   buf);
-        if (ret)
-                goto out;
+    int ret = -1;
+    char buf[PATH_MAX];
+    char uuid[UUID_SIZE + 1];
+    uint total_len = 0;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+    gf_store_handle_t shandle;
+    xlator_t *this = NULL;
 
+    this = THIS;
+    GF_ASSERT(this);
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.op);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_DEFRAG_OP, buf);
-        if (ret)
-                goto out;
-
-        gf_uuid_unparse (volinfo->rebal.rebalance_id, buf);
-        ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
-        if (ret)
-                goto out;
+    GF_ASSERT(fd > 0);
+    GF_ASSERT(volinfo);
 
-        if (volinfo->rebal.dict) {
-                dict_foreach (volinfo->rebal.dict, _gd_store_rebalance_dict,
-                              &fd);
-        }
+    if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
+        ret = 0;
+        goto out;
+    }
+
+    gf_uuid_unparse(volinfo->rebal.rebalance_id, uuid);
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%d\n%s=%d\n%s=%d\n%s=%s\n",
+                   GLUSTERD_STORE_KEY_VOL_DEFRAG, volinfo->rebal.defrag_cmd,
+                   GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
+                   volinfo->rebal.defrag_status, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                   volinfo->rebal.op, GF_REBALANCE_TID_KEY, uuid);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    ret = snprintf(
+        buf + total_len, sizeof(buf) - total_len,
+        "%s=%" PRIu64 "\n%s=%" PRIu64 "\n%s=%" PRIu64 "\n%s=%" PRIu64
+        "\n%s=%" PRIu64 "\n%s=%lf\n",
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES, volinfo->rebal.rebalance_files,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE, volinfo->rebal.rebalance_data,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED, volinfo->rebal.lookedup_files,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES,
+        volinfo->rebal.rebalance_failures,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED, volinfo->rebal.skipped_files,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME, volinfo->rebal.rebalance_time);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret) {
+        goto out;
+    }
+
+    if (volinfo->rebal.dict) {
+        dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
+                              gf_gld_mt_volinfo_dict_data_t);
+        if (dict_data == NULL) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+            return -1;
+        }
+        dict_data->shandle = &shandle;
+        shandle.fd = fd;
+        dict_foreach(volinfo->rebal.dict, _storeopts, (void *)dict_data);
+        if (dict_data->buffer_len > 0) {
+            ret = gf_store_save_items(fd, dict_data->buffer);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                        NULL);
+                goto out;
+                ;
+            }
+        }
+    }
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(dict_data);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo)
+glusterd_store_perform_node_state_store(glusterd_volinfo_t *volinfo)
 {
-        int                         fd = -1;
-        int32_t                     ret = -1;
-        GF_ASSERT (volinfo);
+    int fd = -1;
+    int32_t ret = -1;
+    GF_ASSERT(volinfo);
 
-        fd = gf_store_mkstemp (volinfo->node_state_shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
+    fd = gf_store_mkstemp(volinfo->node_state_shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_node_state_write (fd, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_node_state_write(fd, volinfo);
+    if (ret)
+        goto out;
 
-        ret = gf_store_rename_tmppath (volinfo->node_state_shandle);
-        if (ret)
-                goto out;
+    ret = gf_store_rename_tmppath(volinfo->node_state_shandle);
+    if (ret)
+        goto out;
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (volinfo->node_state_shandle);
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(volinfo->node_state_shandle);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int32_t
-glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_store_perform_volume_store(glusterd_volinfo_t *volinfo)
 {
-        int                         fd = -1;
-        int32_t                     ret = -1;
-        GF_ASSERT (volinfo);
+    int fd = -1;
+    int32_t ret = -1;
+    GF_ASSERT(volinfo);
 
-        fd = gf_store_mkstemp (volinfo->shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
+    fd = gf_store_mkstemp(volinfo->shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_volinfo_write (fd, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_volinfo_write(fd, volinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_brickinfos (volinfo, fd);
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_brick_dir(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_brickinfos(volinfo, fd);
+    if (ret)
+        goto out;
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (volinfo->shandle);
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(volinfo->shandle);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 void
-glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
-                                         glusterd_volinfo_ver_ac_t ac)
+glusterd_perform_volinfo_version_action(glusterd_volinfo_t *volinfo,
+                                        glusterd_volinfo_ver_ac_t ac)
 {
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        switch (ac) {
+    switch (ac) {
         case GLUSTERD_VOLINFO_VER_AC_NONE:
-        break;
+            break;
         case GLUSTERD_VOLINFO_VER_AC_INCREMENT:
-                volinfo->version++;
-        break;
+            volinfo->version++;
+            break;
         case GLUSTERD_VOLINFO_VER_AC_DECREMENT:
-                volinfo->version--;
-        break;
-        }
+            volinfo->version--;
+            break;
+    }
 }
 
 void
-glusterd_store_bricks_cleanup_tmp (glusterd_volinfo_t *volinfo)
+glusterd_store_bricks_cleanup_tmp(glusterd_volinfo_t *volinfo)
 {
-        glusterd_brickinfo_t    *brickinfo           = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                gf_store_unlink_tmppath (brickinfo->shandle);
-        }
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        gf_store_unlink_tmppath(brickinfo->shandle);
+    }
 }
 
 void
-glusterd_store_volume_cleanup_tmp (glusterd_volinfo_t *volinfo)
+glusterd_store_volume_cleanup_tmp(glusterd_volinfo_t *volinfo)
 {
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_bricks_cleanup_tmp (volinfo);
+    glusterd_store_bricks_cleanup_tmp(volinfo);
 
-        gf_store_unlink_tmppath (volinfo->shandle);
+    gf_store_unlink_tmppath(volinfo->shandle);
 
-        gf_store_unlink_tmppath (volinfo->node_state_shandle);
+    gf_store_unlink_tmppath(volinfo->node_state_shandle);
 
-        gf_store_unlink_tmppath (volinfo->snapd.handle);
+    gf_store_unlink_tmppath(volinfo->snapd.handle);
 }
 
 int32_t
-glusterd_store_brickinfos_atomic_update (glusterd_volinfo_t *volinfo)
+glusterd_store_brickinfos_atomic_update(glusterd_volinfo_t *volinfo)
 {
-        int                      ret            = -1;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = gf_store_rename_tmppath(brickinfo->shandle);
+        if (ret)
+            goto out;
+    }
+
+    if (volinfo->thin_arbiter_count == 1) {
+        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
+                                        glusterd_brickinfo_t, brick_list);
+        ret = gf_store_rename_tmppath(ta_brickinfo->shandle);
+        if (ret)
+            goto out;
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = gf_store_rename_tmppath (brickinfo->shandle);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_volinfo_atomic_update (glusterd_volinfo_t *volinfo)
+glusterd_store_volinfo_atomic_update(glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
-        GF_ASSERT (volinfo);
+    int ret = -1;
+    GF_ASSERT(volinfo);
 
-        ret = gf_store_rename_tmppath (volinfo->shandle);
-        if (ret)
-                goto out;
+    ret = gf_store_rename_tmppath(volinfo->shandle);
+    if (ret)
+        goto out;
 
 out:
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED, "Couldn't rename "
-                        "temporary file(s)");
-        return ret;
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Couldn't rename "
+               "temporary file(s)");
+    return ret;
 }
 
 int32_t
-glusterd_store_volume_atomic_update (glusterd_volinfo_t *volinfo)
+glusterd_store_volume_atomic_update(glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
-        GF_ASSERT (volinfo);
+    int ret = -1;
+    GF_ASSERT(volinfo);
 
-        ret = glusterd_store_brickinfos_atomic_update (volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_brickinfos_atomic_update(volinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_volinfo_atomic_update (volinfo);
+    ret = glusterd_store_volinfo_atomic_update(volinfo);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_snap_atomic_update (glusterd_snap_t *snap)
+glusterd_store_snap_atomic_update(glusterd_snap_t *snap)
 {
-        int ret = -1;
-        GF_ASSERT (snap);
+    int ret = -1;
+    GF_ASSERT(snap);
 
-        ret = gf_store_rename_tmppath (snap->shandle);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED, "Couldn't rename "
-                        "temporary file(s)");
+    ret = gf_store_rename_tmppath(snap->shandle);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Couldn't rename "
+               "temporary file(s)");
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_snap (glusterd_snap_t *snap)
+glusterd_store_snap(glusterd_snap_t *snap)
 {
-        int32_t                 ret = -1;
-
-        GF_ASSERT (snap);
-
-        ret = glusterd_store_create_snap_dir (snap);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPDIR_CREATE_FAIL,
-                        "Failed to create snap dir");
-                goto out;
-        }
-
-        ret = glusterd_store_create_snap_shandle_on_absence (snap);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                       GD_MSG_SNAPINFO_CREATE_FAIL,
-                       "Failed to create snap info "
-                        "file");
-                goto out;
-        }
-
-        ret = glusterd_store_snapinfo_write (snap);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPINFO_WRITE_FAIL,
-                        "Failed to write snap info");
-                goto out;
-        }
-
-        ret = glusterd_store_snap_atomic_update (snap);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_AUTOMIC_UPDATE_FAIL,
-                        "Failed to do automic update");
-                goto out;
-        }
+    int32_t ret = -1;
+
+    GF_ASSERT(snap);
+
+    ret = glusterd_store_create_snap_dir(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPDIR_CREATE_FAIL,
+               "Failed to create snap dir");
+        goto out;
+    }
+
+    ret = glusterd_store_create_snap_shandle_on_absence(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_CREATE_FAIL,
+               "Failed to create snap info "
+               "file");
+        goto out;
+    }
+
+    ret = glusterd_store_snapinfo_write(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_WRITE_FAIL,
+               "Failed to write snap info");
+        goto out;
+    }
+
+    ret = glusterd_store_snap_atomic_update(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_AUTOMIC_UPDATE_FAIL,
+               "Failed to do automic update");
+        goto out;
+    }
 
 out:
-        if (ret && snap->shandle)
-                gf_store_unlink_tmppath (snap->shandle);
+    if (ret && snap->shandle)
+        gf_store_unlink_tmppath(snap->shandle);
 
-        gf_msg_trace (THIS->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac)
+glusterd_store_volinfo(glusterd_volinfo_t *volinfo,
+                       glusterd_volinfo_ver_ac_t ac)
 {
-        int32_t                 ret = -1;
-
-        GF_ASSERT (volinfo);
-
-        glusterd_perform_volinfo_version_action (volinfo, ac);
-        ret = glusterd_store_create_volume_dir (volinfo);
+    int32_t ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    GF_ASSERT(volinfo);
+
+    pthread_mutex_lock(&ctx->cleanup_lock);
+    pthread_mutex_lock(&volinfo->store_volinfo_lock);
+    {
+        glusterd_perform_volinfo_version_action(volinfo, ac);
+
+        ret = glusterd_store_create_volume_dirs(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_create_vol_shandle_on_absence (volinfo);
+        ret = glusterd_store_create_vol_shandle_on_absence(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_create_nodestate_sh_on_absence (volinfo);
+        ret = glusterd_store_create_nodestate_sh_on_absence(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_perform_volume_store (volinfo);
+        ret = glusterd_store_perform_volume_store(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_volume_atomic_update (volinfo);
+        ret = glusterd_store_volume_atomic_update(volinfo);
         if (ret) {
-                glusterd_perform_volinfo_version_action (volinfo,
-                                                         GLUSTERD_VOLINFO_VER_AC_DECREMENT);
-                goto out;
+            glusterd_perform_volinfo_version_action(
+                volinfo, GLUSTERD_VOLINFO_VER_AC_DECREMENT);
+            goto unlock;
         }
 
-        ret = glusterd_store_perform_node_state_store (volinfo);
+        ret = glusterd_store_perform_node_state_store(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
         /* checksum should be computed at the end */
-        ret = glusterd_compute_cksum (volinfo, _gf_false);
+        ret = glusterd_compute_cksum(volinfo, _gf_false);
         if (ret)
-                goto out;
+            goto unlock;
+    }
+unlock:
+    pthread_mutex_unlock(&volinfo->store_volinfo_lock);
+    pthread_mutex_unlock(&ctx->cleanup_lock);
 
-out:
-        if (ret)
-                glusterd_store_volume_cleanup_tmp (volinfo);
+    if (ret)
+        glusterd_store_volume_cleanup_tmp(volinfo);
 
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
+glusterd_store_delete_volume(glusterd_volinfo_t *volinfo)
 {
-        char             pathname[PATH_MAX]    = {0,};
-        int32_t          ret                   = 0;
-        glusterd_conf_t *priv                  = NULL;
-        char             delete_path[PATH_MAX] = {0,};
-        char             trashdir[PATH_MAX]    = {0,};
-        xlator_t        *this                  = NULL;
-        gf_boolean_t     rename_fail           = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
-
-        snprintf (delete_path, sizeof (delete_path),
-        "%s/"GLUSTERD_TRASH"/%s.deleted", priv->workdir,
-        uuid_utoa (volinfo->volume_id));
-
-        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
-                  priv->workdir);
-
-        ret = sys_mkdir (trashdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Failed to create trash "
-                        "directory");
-                ret = -1;
-                goto out;
-        }
-
-        ret = sys_rename (pathname, delete_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "Failed to rename volume "
-                        "directory for volume %s", volinfo->volname);
-                rename_fail = _gf_true;
-                goto out;
-        }
-
-        ret = recursive_rmdir (trashdir);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to rmdir: %s",
-                                 trashdir);
-        }
+    char pathname[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    char delete_path[PATH_MAX] = {
+        0,
+    };
+    char trashdir[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_boolean_t rename_fail = _gf_false;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_VOLUME_DIR(pathname, volinfo, priv);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/%s.deleted", priv->workdir,
+                   uuid_utoa(volinfo->volume_id));
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        goto out;
+    }
+
+    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
+                   priv->workdir);
+    if ((len < 0) || (len >= sizeof(trashdir))) {
+        goto out;
+    }
+
+    ret = sys_mkdir(trashdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create trash "
+               "directory");
+        goto out;
+    }
+
+    ret = sys_rename(pathname, delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to rename volume "
+               "directory for volume %s",
+               volinfo->volname);
+        rename_fail = _gf_true;
+        goto out;
+    }
+
+    ret = recursive_rmdir(trashdir);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", trashdir);
+    }
 
 out:
-        if (volinfo->shandle) {
-                gf_store_handle_destroy (volinfo->shandle);
-                volinfo->shandle = NULL;
-        }
-        ret = (rename_fail == _gf_true) ? -1: 0;
-
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (volinfo->shandle) {
+        gf_store_handle_destroy(volinfo->shandle);
+        volinfo->shandle = NULL;
+    }
+    ret = (rename_fail == _gf_true) ? -1 : 0;
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /*TODO: cleanup the duplicate code and implement a generic function for
  * deleting snap/volume depending on the parameter flag */
 int32_t
-glusterd_store_delete_snap (glusterd_snap_t *snap)
+glusterd_store_delete_snap(glusterd_snap_t *snap)
 {
-        char             pathname[PATH_MAX]    = {0,};
-        int32_t          ret                   = 0;
-        glusterd_conf_t *priv                  = NULL;
-        DIR             *dir                   = NULL;
-        struct dirent   *entry                 = NULL;
-        char             path[PATH_MAX]        = {0,};
-        char             delete_path[PATH_MAX] = {0,};
-        char             trashdir[PATH_MAX]    = {0,};
-        struct stat      st                    = {0, };
-        xlator_t        *this                  = NULL;
-        gf_boolean_t     rename_fail           = _gf_false;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (snap);
-        GLUSTERD_GET_SNAP_DIR (pathname, snap, priv);
-
-        snprintf (delete_path, sizeof (delete_path),
-        "%s/"GLUSTERD_TRASH"/snap-%s.deleted", priv->workdir,
-        uuid_utoa (snap->snap_id));
-
-        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
-                  priv->workdir);
-
-        ret = sys_mkdir (trashdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Failed to create trash "
-                        "directory");
-                ret = -1;
-                goto out;
-        }
-
-        ret = sys_rename (pathname, delete_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "Failed to rename snap "
-                        "directory %s to %s", pathname, delete_path);
-                rename_fail = _gf_true;
-                goto out;
-        }
-
-        dir = sys_opendir (delete_path);
-        if (!dir) {
-                gf_msg_debug (this->name, 0, "Failed to open directory %s.",
-                                 delete_path);
-                ret = 0;
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        while (entry) {
-                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
-                ret = sys_stat (path, &st);
-                if (ret == -1) {
-                        gf_msg_debug (this->name, 0, "Failed to stat "
-                                "entry %s", path);
-                        goto stat_failed;
-                }
-
-                if (S_ISDIR (st.st_mode))
-                        ret = sys_rmdir (path);
-                else
-                        ret = sys_unlink (path);
-
-                if (ret) {
-                        gf_msg_debug (this->name, 0, " Failed to remove "
-                                "%s", path);
-                }
-
-                gf_msg_debug (this->name, 0, "%s %s",
-                                ret ? "Failed to remove":"Removed",
-                                entry->d_name);
-stat_failed:
-                memset (path, 0, sizeof(path));
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
-
-        ret = sys_closedir (dir);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to close dir %s.",
-                                delete_path);
-        }
+    char pathname[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char delete_path[PATH_MAX] = {
+        0,
+    };
+    char trashdir[PATH_MAX] = {
+        0,
+    };
+    struct stat st = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_boolean_t rename_fail = _gf_false;
+    int32_t len = 0;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(snap);
+    GLUSTERD_GET_SNAP_DIR(pathname, snap, priv);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/snap-%s.deleted", priv->workdir,
+                   uuid_utoa(snap->snap_id));
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        goto out;
+    }
+
+    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
+                   priv->workdir);
+    if ((len < 0) || (len >= sizeof(trashdir))) {
+        goto out;
+    }
+
+    ret = sys_mkdir(trashdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create trash "
+               "directory");
+        goto out;
+    }
+
+    ret = sys_rename(pathname, delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to rename snap "
+               "directory %s to %s",
+               pathname, delete_path);
+        rename_fail = _gf_true;
+        goto out;
+    }
+
+    dir = sys_opendir(delete_path);
+    if (!dir) {
+        gf_msg_debug(this->name, 0, "Failed to open directory %s.",
+                     delete_path);
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        len = snprintf(path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto stat_failed;
+        }
+        ret = sys_stat(path, &st);
+        if (ret == -1) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to stat "
+                         "entry %s",
+                         path);
+            goto stat_failed;
+        }
+
+        if (S_ISDIR(st.st_mode))
+            ret = sys_rmdir(path);
+        else
+            ret = sys_unlink(path);
 
-        ret = sys_rmdir (delete_path);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to rmdir: %s",
-                        delete_path);
-        }
-        ret = sys_rmdir (trashdir);
         if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to rmdir: %s",
-                                 trashdir);
-        }
+            gf_msg_debug(this->name, 0,
+                         " Failed to remove "
+                         "%s",
+                         path);
+        }
+
+        gf_msg_debug(this->name, 0, "%s %s",
+                     ret ? "Failed to remove" : "Removed", entry->d_name);
+    stat_failed:
+        memset(path, 0, sizeof(path));
+    }
+
+    ret = sys_closedir(dir);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to close dir %s.", delete_path);
+    }
+
+    ret = sys_rmdir(delete_path);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", delete_path);
+    }
+    ret = sys_rmdir(trashdir);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", trashdir);
+    }
 
 out:
-        if (snap->shandle) {
-                gf_store_handle_destroy (snap->shandle);
-                snap->shandle = NULL;
-        }
-        ret = (rename_fail == _gf_true) ? -1: 0;
-
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    if (snap->shandle) {
+        gf_store_handle_destroy(snap->shandle);
+        snap->shandle = NULL;
+    }
+    ret = (rename_fail == _gf_true) ? -1 : 0;
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_store_global_info (xlator_t *this)
+glusterd_store_global_info(xlator_t *this)
 {
-        int                     ret                     = -1;
-        glusterd_conf_t         *conf                   = NULL;
-        char                    op_version_str[15]      = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        gf_store_handle_t       *handle                 = NULL;
-        char                    *uuid_str               = NULL;
-
-        conf = this->private;
-
-        uuid_str = gf_strdup (uuid_utoa (MY_UUID));
-        if (!uuid_str)
-                goto out;
-
-        if (!conf->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", conf->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_new (path, &handle);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_STORE_HANDLE_GET_FAIL,
-                                "Unable to get store handle");
-                        goto out;
-                }
-
-                conf->handle = handle;
-        } else
-                handle = conf->handle;
-
-        /* These options need to be available for all users */
-        ret = sys_chmod (handle->path, 0644);
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    char buf[PATH_MAX];
+    uint total_len = 0;
+    gf_store_handle_t *handle = NULL;
+    char *uuid_str = NULL;
+
+    conf = this->private;
+
+    uuid_str = gf_strdup(uuid_utoa(MY_UUID));
+    if (!uuid_str)
+        goto out;
+
+    if (!conf->handle) {
+        ret = snprintf(buf, sizeof(buf), "%s/%s", conf->workdir,
+                       GLUSTERD_INFO_FILE);
+        if ((ret < 0) || (ret >= sizeof(buf))) {
+            ret = -1;
+            goto out;
+        }
+        ret = gf_store_handle_new(buf, &handle);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED, "chmod error for %s",
-                        GLUSTERD_INFO_FILE);
-                goto out;
-        }
-
-        handle->fd = gf_store_mkstemp (handle);
-        if (handle->fd <= 0) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = gf_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY,
-                                   uuid_str);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_UUID_SET_FAIL,
-                        "Storing uuid failed ret = %d", ret);
-                goto out;
-        }
-
-        snprintf (op_version_str, 15, "%d", conf->op_version);
-        ret = gf_store_save_value (handle->fd, GD_OP_VERSION_KEY,
-                                   op_version_str);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_OP_VERS_STORE_FAIL,
-                        "Storing op-version failed ret = %d", ret);
-                goto out;
-        }
-
-        ret = gf_store_rename_tmppath (handle);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
+                   "Unable to get store handle");
+            goto out;
+        }
+
+        conf->handle = handle;
+    } else
+        handle = conf->handle;
+
+    /* These options need to be available for all users */
+    ret = sys_chmod(handle->path, 0644);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "chmod error for %s", GLUSTERD_INFO_FILE);
+        goto out;
+    }
+
+    handle->fd = gf_store_mkstemp(handle);
+    if (handle->fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "%s=%s\n", GLUSTERD_STORE_UUID_KEY,
+                   uuid_str);
+    if (ret < 0 || ret >= sizeof(buf)) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                   GD_OP_VERSION_KEY, conf->op_version);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(handle->fd, buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_OP_VERS_STORE_FAIL,
+               "Storing glusterd global-info failed ret = %d", ret);
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(handle);
 out:
-        if (handle) {
-                if (ret && (handle->fd > 0))
-                        gf_store_unlink_tmppath (handle);
+    if (handle) {
+        if (ret && (handle->fd >= 0))
+            gf_store_unlink_tmppath(handle);
+    }
 
-                if (handle->fd > 0) {
-                        handle->fd = 0;
-                }
-        }
-
-        if (uuid_str)
-                GF_FREE (uuid_str);
+    if (uuid_str)
+        GF_FREE(uuid_str);
 
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL,
-                        "Failed to store glusterd global-info");
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL,
+               "Failed to store glusterd global-info");
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_retrieve_op_version (xlator_t *this, int *op_version)
+glusterd_store_max_op_version(xlator_t *this)
 {
-        char                    *op_version_str = NULL;
-        glusterd_conf_t         *priv           = NULL;
-        int                     ret             = -1;
-        int                     tmp_version     = 0;
-        char                    *tmp            = NULL;
-        char                    path[PATH_MAX]  = {0,};
-        gf_store_handle_t       *handle         = NULL;
-
-        priv = this->private;
-
-        if (!priv->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_retrieve (path, &handle);
-
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "Unable to get store "
-                                "handle!");
-                        goto out;
-                }
-
-                priv->handle = handle;
-        }
-
-        ret = gf_store_retrieve_value (priv->handle, GD_OP_VERSION_KEY,
-                                       &op_version_str);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "No previous op_version present");
-                goto out;
-        }
-
-        tmp_version = strtol (op_version_str, &tmp, 10);
-        if ((tmp_version <= 0) || (tmp && strlen (tmp) > 1)) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        GD_MSG_UNSUPPORTED_VERSION, "invalid version number");
-                goto out;
-        }
-
-        *op_version = tmp_version;
-
-        ret = 0;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    char op_version_str[15] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    conf = this->private;
+
+    len = snprintf(path, PATH_MAX, "%s/%s", conf->workdir,
+                   GLUSTERD_UPGRADE_FILE);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+    ret = gf_store_handle_new(path, &handle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
+               "Unable to get store handle");
+        goto out;
+    }
+
+    /* These options need to be available for all users */
+    ret = sys_chmod(handle->path, 0644);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "chmod error for %s", GLUSTERD_UPGRADE_FILE);
+        goto out;
+    }
+
+    handle->fd = gf_store_mkstemp(handle);
+    if (handle->fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(op_version_str, sizeof(op_version_str), "%d", GD_OP_VERSION_MAX);
+    ret = gf_store_save_value(handle->fd, GD_MAX_OP_VERSION_KEY,
+                              op_version_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_STORE_FAIL,
+               "Storing op-version failed ret = %d", ret);
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(handle);
 out:
-        if (op_version_str)
-                GF_FREE (op_version_str);
-
-        return ret;
+    if (handle) {
+        if (ret && (handle->fd >= 0))
+            gf_store_unlink_tmppath(handle);
+    }
+
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL,
+               "Failed to store max op-version");
+    if (handle)
+        gf_store_handle_destroy(handle);
+    return ret;
 }
 
 int
-glusterd_retrieve_sys_snap_max_limit (xlator_t *this, uint64_t *limit,
-                                      char *key)
+glusterd_retrieve_max_op_version(xlator_t *this, int *op_version)
 {
-        char                    *limit_str      = NULL;
-        glusterd_conf_t         *priv           = NULL;
-        int                     ret             = -1;
-        uint64_t                tmp_limit       = 0;
-        char                    *tmp            = NULL;
-        char                    path[PATH_MAX]  = {0,};
-        gf_store_handle_t       *handle         = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-        GF_ASSERT (limit);
-        GF_ASSERT (key);
-
-        if (!priv->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_retrieve (path, &handle);
-
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "Unable to get store "
-                                "handle!");
-                        goto out;
-                }
+    char *op_version_str = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int tmp_version = 0;
+    char *tmp = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                   GLUSTERD_UPGRADE_FILE);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+    ret = gf_store_handle_retrieve(path, &handle);
+
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Unable to get store "
+                     "handle!");
+        goto out;
+    }
+
+    ret = gf_store_retrieve_value(handle, GD_MAX_OP_VERSION_KEY,
+                                  &op_version_str);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No previous op_version present");
+        goto out;
+    }
+
+    tmp_version = strtol(op_version_str, &tmp, 10);
+    if ((tmp_version <= 0) || (tmp && strlen(tmp) > 1)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
+               "invalid version number");
+        goto out;
+    }
+
+    *op_version = tmp_version;
+
+    ret = 0;
+out:
+    if (op_version_str)
+        GF_FREE(op_version_str);
+    if (handle)
+        gf_store_handle_destroy(handle);
+    return ret;
+}
 
-                priv->handle = handle;
-        }
+int
+glusterd_retrieve_op_version(xlator_t *this, int *op_version)
+{
+    char *op_version_str = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int tmp_version = 0;
+    char *tmp = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    if (!priv->handle) {
+        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                       GLUSTERD_INFO_FILE);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
+        ret = gf_store_handle_retrieve(path, &handle);
 
-        ret = gf_store_retrieve_value (priv->handle,
-                                       key,
-                                       &limit_str);
         if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "No previous %s present", key);
-                goto out;
+            gf_msg_debug(this->name, 0,
+                         "Unable to get store "
+                         "handle!");
+            goto out;
         }
 
-        tmp_limit = strtoul (limit_str, &tmp, 10);
-        if ((tmp_limit <= 0) || (tmp && strlen (tmp) > 1)) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        GD_MSG_UNSUPPORTED_VERSION, "invalid version number");
-                goto out;
-        }
+        priv->handle = handle;
+    }
 
-        *limit = tmp_limit;
+    ret = gf_store_retrieve_value(priv->handle, GD_OP_VERSION_KEY,
+                                  &op_version_str);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No previous op_version present");
+        goto out;
+    }
 
-        ret = 0;
+    tmp_version = strtol(op_version_str, &tmp, 10);
+    if ((tmp_version <= 0) || (tmp && strlen(tmp) > 1)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
+               "invalid version number");
+        goto out;
+    }
+
+    *op_version = tmp_version;
+
+    ret = 0;
 out:
-        if (limit_str)
-                GF_FREE (limit_str);
+    if (op_version_str)
+        GF_FREE(op_version_str);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_restore_op_version (xlator_t *this)
+glusterd_restore_op_version(xlator_t *this)
 {
-        glusterd_conf_t *conf           = NULL;
-        int              ret            = 0;
-        int              op_version     = 0;
-
-        conf = this->private;
-
-        ret = glusterd_retrieve_op_version (this, &op_version);
-        if (!ret) {
-                if ((op_version < GD_OP_VERSION_MIN) ||
-                    (op_version > GD_OP_VERSION_MAX)) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_UNSUPPORTED_VERSION,
-                                "wrong op-version (%d) retrieved", op_version);
-                        ret = -1;
-                        goto out;
-                }
-                conf->op_version = op_version;
-                gf_msg ("glusterd", GF_LOG_INFO, 0,
-                        GD_MSG_OP_VERS_INFO,
-                        "retrieved op-version: %d", conf->op_version);
-                goto out;
-        }
-
-        /* op-version can be missing from the store file in 2 cases,
-         * 1. This is a new install of glusterfs
-         * 2. This is an upgrade of glusterfs from a version without op-version
-         *    to a version with op-version (eg. 3.3 -> 3.4)
-         *
-         * Detection of a new install or an upgrade from an older install can be
-         * done by checking for the presence of the its peer-id in the store
-         * file.  If peer-id is present, the installation is an upgrade else, it
-         * is a new install.
-         *
-         * For case 1, set op-version to GD_OP_VERSION_MAX.
-         * For case 2, set op-version to GD_OP_VERSION_MIN.
-         */
-        ret = glusterd_retrieve_uuid();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_OP_VERS_SET_INFO,
-                        "Detected new install. Setting"
-                        " op-version to maximum : %d", GD_OP_VERSION_MAX);
-                conf->op_version = GD_OP_VERSION_MAX;
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_OP_VERS_SET_INFO,
-                        "Upgrade detected. Setting"
-                        " op-version to minimum : %d", GD_OP_VERSION_MIN);
-                conf->op_version = GD_OP_VERSION_MIN;
-        }
-        ret = 0;
+    glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    int op_version = 0;
+
+    conf = this->private;
+
+    ret = glusterd_retrieve_op_version(this, &op_version);
+    if (!ret) {
+        if ((op_version < GD_OP_VERSION_MIN) ||
+            (op_version > GD_OP_VERSION_MAX)) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
+                   "wrong op-version (%d) retrieved", op_version);
+            ret = -1;
+            goto out;
+        }
+        conf->op_version = op_version;
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_OP_VERS_INFO,
+               "retrieved op-version: %d", conf->op_version);
+        goto out;
+    }
+
+    /* op-version can be missing from the store file in 2 cases,
+     * 1. This is a new install of glusterfs
+     * 2. This is an upgrade of glusterfs from a version without op-version
+     *    to a version with op-version (eg. 3.3 -> 3.4)
+     *
+     * Detection of a new install or an upgrade from an older install can be
+     * done by checking for the presence of the its peer-id in the store
+     * file.  If peer-id is present, the installation is an upgrade else, it
+     * is a new install.
+     *
+     * For case 1, set op-version to GD_OP_VERSION_MAX.
+     * For case 2, set op-version to GD_OP_VERSION_MIN.
+     */
+    ret = glusterd_retrieve_uuid();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_VERS_SET_INFO,
+               "Detected new install. Setting"
+               " op-version to maximum : %d",
+               GD_OP_VERSION_MAX);
+        conf->op_version = GD_OP_VERSION_MAX;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_VERS_SET_INFO,
+               "Upgrade detected. Setting"
+               " op-version to minimum : %d",
+               GD_OP_VERSION_MIN);
+        conf->op_version = GD_OP_VERSION_MIN;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_retrieve_uuid ()
+glusterd_retrieve_uuid()
 {
-        char            *uuid_str = NULL;
-        int32_t         ret = -1;
-        gf_store_handle_t *handle = NULL;
-        glusterd_conf_t *priv = NULL;
-        xlator_t        *this = NULL;
-        char            path[PATH_MAX] = {0,};
-
-        this = THIS;
-        priv = this->private;
-
-        if (!priv->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_retrieve (path, &handle);
-
-                if (ret) {
-                        gf_msg_debug (this->name, 0, "Unable to get store"
-                                "handle!");
-                        goto out;
-                }
-
-                priv->handle = handle;
-        }
-
-        ret = gf_store_retrieve_value (priv->handle, GLUSTERD_STORE_UUID_KEY,
-                                       &uuid_str);
+    char *uuid_str = NULL;
+    int32_t ret = -1;
+    gf_store_handle_t *handle = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    if (!priv->handle) {
+        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                       GLUSTERD_INFO_FILE);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
+        ret = gf_store_handle_retrieve(path, &handle);
 
         if (ret) {
-                gf_msg_debug (this->name, 0, "No previous uuid is present");
-                goto out;
-        }
-
-        gf_uuid_parse (uuid_str, priv->uuid);
+            gf_msg_debug(this->name, 0,
+                         "Unable to get store"
+                         "handle!");
+            goto out;
+        }
+
+        priv->handle = handle;
+    }
+    pthread_mutex_lock(&priv->mutex);
+    {
+        ret = gf_store_retrieve_value(priv->handle, GLUSTERD_STORE_UUID_KEY,
+                                      &uuid_str);
+    }
+    pthread_mutex_unlock(&priv->mutex);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No previous uuid is present");
+        goto out;
+    }
+
+    gf_uuid_parse(uuid_str, priv->uuid);
 
 out:
-        GF_FREE (uuid_str);
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(uuid_str);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_store_retrieve_snapd (glusterd_volinfo_t *volinfo)
+glusterd_store_retrieve_snapd(glusterd_volinfo_t *volinfo)
 {
-        int                     ret                     = -1;
-        int                     exists                  = 0;
-        char                    *key                    = NULL;
-        char                    *value                  = NULL;
-        char                    volpath[PATH_MAX]       = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_conf_t         *conf                   = NULL;
-        gf_store_iter_t         *iter                   = NULL;
-        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = THIS->private;
-        GF_ASSERT (volinfo);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        /*
-         * This is needed for upgrade situations. Say a volume is created with
-         * older version of glusterfs and upgraded to a glusterfs version equal
-         * to or greater than GD_OP_VERSION_3_6_0. The older glusterd would not
-         * have created the snapd.info file related to snapshot daemon for user
-         * serviceable snapshots. So as part of upgrade when the new glusterd
-         * starts, as part of restore (restoring the volume to be precise), it
-         * tries to snapd related info from snapd.info file. But since there was
-         * no such file till now, the restore operation fails. Thus, to prevent
-         * it from happening check whether user serviceable snapshots features
-         * is enabled before restoring snapd. If its disbaled, then simply
-         * exit by returning success (without even checking for the snapd.info).
-         */
-
-        if (!dict_get_str_boolean (volinfo->dict, "features.uss", _gf_false)) {
-                ret = 0;
-                goto out;
-        }
-
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
-
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_VOLUME_SNAPD_INFO_FILE);
-
-        ret = gf_store_handle_retrieve (path, &volinfo->snapd.handle);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HANDLE_NULL, "volinfo handle is NULL");
-                goto out;
-        }
-
-        ret = gf_store_iter_new (volinfo->snapd.handle, &iter);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_GET_FAIL, "Failed to get new store "
-                        "iter");
-                goto out;
-        }
-
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_GET_FAIL, "Failed to get next store "
-                        "iter");
-                goto out;
-        }
-
-        while (!ret) {
-                if (!strncmp (key, GLUSTERD_STORE_KEY_SNAPD_PORT,
-                              strlen (GLUSTERD_STORE_KEY_SNAPD_PORT))) {
-                        volinfo->snapd.port = atoi (value);
-                }
-
-                ret = gf_store_iter_get_next (iter, &key, &value,
-                                              &op_errno);
-        }
-
-        if (op_errno != GD_STORE_EOF)
-                goto out;
-
-        ret = gf_store_iter_destroy (iter);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_DESTROY_FAIL,
-                        "Failed to destroy store "
-                        "iter");
-                goto out;
-        }
-
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_iter_t *iter = NULL;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = THIS->private;
+    GF_ASSERT(volinfo);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * This is needed for upgrade situations. Say a volume is created with
+     * older version of glusterfs and upgraded to a glusterfs version equal
+     * to or greater than GD_OP_VERSION_3_6_0. The older glusterd would not
+     * have created the snapd.info file related to snapshot daemon for user
+     * serviceable snapshots. So as part of upgrade when the new glusterd
+     * starts, as part of restore (restoring the volume to be precise), it
+     * tries to snapd related info from snapd.info file. But since there was
+     * no such file till now, the restore operation fails. Thus, to prevent
+     * it from happening check whether user serviceable snapshots features
+     * is enabled before restoring snapd. If its disabled, then simply
+     * exit by returning success (without even checking for the snapd.info).
+     */
+
+    if (!dict_get_str_boolean(volinfo->dict, "features.uss", _gf_false)) {
         ret = 0;
+        goto out;
+    }
+
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
+
+    len = snprintf(path, sizeof(path), "%s/%s", volpath,
+                   GLUSTERD_VOLUME_SNAPD_INFO_FILE);
+    if ((len < 0) || (len >= sizeof(path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &volinfo->snapd.handle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
+               "volinfo handle is NULL");
+        goto out;
+    }
+
+    ret = gf_store_iter_new(volinfo->snapd.handle, &iter);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get new store "
+               "iter");
+        goto out;
+    }
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get next store "
+               "iter");
+        goto out;
+    }
+
+    while (!ret) {
+        if (!strncmp(key, GLUSTERD_STORE_KEY_SNAPD_PORT,
+                     SLEN(GLUSTERD_STORE_KEY_SNAPD_PORT))) {
+            volinfo->snapd.port = atoi(value);
+        }
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+
+    ret = 0;
 
 out:
-        return ret;
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
+
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
+glusterd_store_retrieve_bricks(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret = 0;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        gf_store_iter_t         *iter = NULL;
-        char                    *key = NULL;
-        char                    *value = NULL;
-        char                    brickdir[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv = NULL;
-        int32_t                 brick_count = 0;
-        char                    tmpkey[4096] = {0,};
-        gf_store_iter_t         *tmpiter = NULL;
-        char                    *tmpvalue = NULL;
-        char                     abspath[PATH_MAX] = {0};
-        struct pmap_registry    *pmap = NULL;
-        xlator_t                *this = NULL;
-        int                      brickid = 0;
-        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volinfo->volname);
-
-        this = THIS;
-        priv = this->private;
-
-        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv);
-
-        ret = gf_store_iter_new (volinfo->shandle, &tmpiter);
+    int32_t ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char brickdir[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    int32_t brick_count = 0;
+    int32_t ta_brick_count = 0;
+    char tmpkey[32] = {
+        0,
+    };
+    gf_store_iter_t *tmpiter = NULL;
+    char *tmpvalue = NULL;
+    char abspath[PATH_MAX] = {0};
+    struct pmap_registry *pmap = NULL;
+    xlator_t *this = NULL;
+    int brickid = 0;
+    /* ta_brick_id initialization with 2 since ta-brick id starts with
+     * volname-ta-2
+     */
+    int ta_brick_id = 2;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volinfo->volname);
+
+    this = THIS;
+    priv = this->private;
+
+    GLUSTERD_GET_BRICK_DIR(brickdir, volinfo, priv);
+
+    ret = gf_store_iter_new(volinfo->shandle, &tmpiter);
+
+    if (ret)
+        goto out;
+
+    while (brick_count < volinfo->brick_count) {
+        ret = glusterd_brickinfo_new(&brickinfo);
 
         if (ret)
-                goto out;
-
-        while (brick_count < volinfo->brick_count) {
-                ret = glusterd_brickinfo_new (&brickinfo);
-
-                if (ret)
-                        goto out;
-                snprintf (tmpkey, sizeof (tmpkey), "%s-%d",
-                          GLUSTERD_STORE_KEY_VOL_BRICK,brick_count);
-                ret = gf_store_iter_get_matching (tmpiter, tmpkey, &tmpvalue);
-                snprintf (path, sizeof (path), "%s/%s", brickdir, tmpvalue);
-
-                GF_FREE (tmpvalue);
-
-                tmpvalue = NULL;
+            goto out;
+        snprintf(tmpkey, sizeof(tmpkey), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
+                 brick_count);
+        ret = gf_store_iter_get_matching(tmpiter, tmpkey, &tmpvalue);
+        len = snprintf(path, sizeof(path), "%s/%s", brickdir, tmpvalue);
+        GF_FREE(tmpvalue);
+        tmpvalue = NULL;
+        if ((len < 0) || (len >= sizeof(path))) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = gf_store_handle_retrieve (path, &brickinfo->shandle);
+        ret = gf_store_handle_retrieve(path, &brickinfo->shandle);
 
-                if (ret)
-                        goto out;
+        if (ret)
+            goto out;
 
-                ret = gf_store_iter_new (brickinfo->shandle, &iter);
+        ret = gf_store_iter_new(brickinfo->shandle, &iter);
 
-                if (ret)
-                        goto out;
+        if (ret)
+            goto out;
 
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-                if (ret) {
-                        gf_msg ("glusterd", GF_LOG_ERROR, op_errno,
-                                GD_MSG_STORE_ITER_GET_FAIL, "Unable to iterate "
-                                "the store for brick: %s", path);
-                        goto out;
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                   GD_MSG_STORE_ITER_GET_FAIL,
+                   "Unable to iterate "
+                   "the store for brick: %s",
+                   path);
+            goto out;
+        }
+        while (!ret) {
+            if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                         SLEN(GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
+                if (snprintf(brickinfo->hostname, sizeof(brickinfo->hostname),
+                             "%s", value) >= sizeof(brickinfo->hostname)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brick hostname truncated: %s", brickinfo->hostname);
+                    goto out;
                 }
-                while (!ret) {
-                        if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
-                                      strlen (GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
-                                strncpy (brickinfo->hostname, value, 1024);
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PATH,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_PATH))) {
-                                strncpy (brickinfo->path, value,
-                                         sizeof (brickinfo->path));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_REAL_PATH))) {
-                                strncpy (brickinfo->real_path, value,
-                                         sizeof (brickinfo->real_path));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PORT,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
-                                gf_string2int (value, &brickinfo->port);
-
-                                if (brickinfo->port < priv->base_port) {
-                                        /* This is required to adhere to the
-                                           IANA standards */
-                                        brickinfo->port = 0;
-                                } else {
-                                        /* This is required to have proper ports
-                                           assigned to bricks after restart */
-                                        pmap = pmap_registry_get (THIS);
-                                        if (pmap->last_alloc <= brickinfo->port)
-                                                pmap->last_alloc =
-                                                        brickinfo->port + 1;
-                                }
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
-                                gf_string2int (value, &brickinfo->rdma_port);
-
-                                if (brickinfo->rdma_port < priv->base_port) {
-                                        /* This is required to adhere to the
-                                           IANA standards */
-                                        brickinfo->rdma_port = 0;
-                                } else {
-                                        /* This is required to have proper ports
-                                           assigned to bricks after restart */
-                                        pmap = pmap_registry_get (THIS);
-                                        if (pmap->last_alloc <=
-                                            brickinfo->rdma_port)
-                                                pmap->last_alloc =
-                                                        brickinfo->rdma_port +1;
-                                }
-
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
-                                gf_string2int (value, &brickinfo->decommissioned);
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH))) {
-                                strncpy (brickinfo->device_path, value,
-                                         sizeof (brickinfo->device_path));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR))) {
-                                strncpy (brickinfo->mount_dir, value,
-                                         sizeof (brickinfo->mount_dir));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
-                                gf_string2int (value, &brickinfo->snap_status);
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_FSTYPE,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_FSTYPE))) {
-                                strncpy (brickinfo->fstype, value,
-                                         sizeof (brickinfo->fstype));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_MNTOPTS,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_MNTOPTS))) {
-                                strncpy (brickinfo->mnt_opts, value,
-                                         sizeof (brickinfo->mnt_opts));
-                        } else if (!strncmp (key,
-                                    GLUSTERD_STORE_KEY_BRICK_VGNAME,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
-                                strncpy (brickinfo->vg, value,
-                                         sizeof (brickinfo->vg));
-                        } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
-                                strncpy (brickinfo->brick_id, value,
-                                         sizeof (brickinfo->brick_id));
-                        } else {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_UNKNOWN_KEY, "Unknown key: %s",
-                                        key);
-                        }
-
-                        GF_FREE (key);
-                        GF_FREE (value);
-                        key = NULL;
-                        value = NULL;
-
-                        ret = gf_store_iter_get_next (iter, &key, &value,
-                                                      &op_errno);
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_PATH))) {
+                if (snprintf(brickinfo->path, sizeof(brickinfo->path), "%s",
+                             value) >= sizeof(brickinfo->path)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brick path truncated: %s", brickinfo->path);
+                    goto out;
                 }
-
-                if (op_errno != GD_STORE_EOF) {
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                GD_MSG_PARSE_BRICKINFO_FAIL,
-                                "Error parsing brickinfo: "
-                                "op_errno=%d", op_errno);
-                        goto out;
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_REAL_PATH))) {
+                if (snprintf(brickinfo->real_path, sizeof(brickinfo->real_path),
+                             "%s", value) >= sizeof(brickinfo->real_path)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "real_path truncated: %s", brickinfo->real_path);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PORT,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_PORT))) {
+                ret = gf_string2int(value, &brickinfo->port);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
                 }
-                ret = gf_store_iter_destroy (iter);
-
-                if (ret)
-                        goto out;
 
-                if (brickinfo->brick_id[0] == '\0') {
-                        /* This is an old volume upgraded to op_version 4 */
-                       GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
-                                                             brickid++);
+                if (brickinfo->port < priv->base_port) {
+                    /* This is required to adhere to the
+                       IANA standards */
+                    brickinfo->port = 0;
+                } else {
+                    /* This is required to have proper ports
+                       assigned to bricks after restart */
+                    pmap = pmap_registry_get(THIS);
+                    if (pmap->last_alloc <= brickinfo->port)
+                        pmap->last_alloc = brickinfo->port + 1;
                 }
-                /* Populate brickinfo->real_path for normal volumes, for
-                 * snapshot or snapshot restored volume this would be done post
-                 * creating the brick mounts
-                 */
-                if (brickinfo->real_path[0] == '\0' && !volinfo->is_snap_volume
-                    && gf_uuid_is_null (volinfo->restored_from_snap)) {
-                        /* By now if the brick is a local brick then it will be
-                         * able to resolve which is the only thing we want now
-                         * for checking  whether the brickinfo->uuid matches
-                         * with MY_UUID for realpath check. Hence do not handle
-                         * error
-                         */
-                        (void)glusterd_resolve_brick (brickinfo);
-                        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
-                                if (!realpath (brickinfo->path, abspath)) {
-                                        gf_msg (this->name, GF_LOG_CRITICAL,
-                                                errno,
-                                                GD_MSG_BRICKINFO_CREATE_FAIL,
-                                                "realpath() failed for brick %s"
-                                                ". The underlying file system "
-                                                "may be in bad state",
-                                                brickinfo->path);
-                                        ret = -1;
-                                        goto out;
-                                }
-                                strncpy (brickinfo->real_path, abspath,
-                                         strlen(abspath));
-                        }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+                ret = gf_string2int(value, &brickinfo->rdma_port);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
                 }
-                cds_list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
-                brick_count++;
-        }
 
-        assign_brick_groups (volinfo);
-        ret = gf_store_iter_destroy (tmpiter);
-        if (ret)
-                goto out;
-out:
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
-
-        return ret;
-}
-
-int32_t
-glusterd_store_retrieve_node_state (glusterd_volinfo_t *volinfo)
-{
-        int32_t              ret               = -1;
-        gf_store_iter_t     *iter              = NULL;
-        char                *key               = NULL;
-        char                *value             = NULL;
-        char                *dup_value         = NULL;
-        char                 volpath[PATH_MAX] = {0,};
-        glusterd_conf_t     *priv              = NULL;
-        char                 path[PATH_MAX]    = {0,};
-        gf_store_op_errno_t  op_errno          = GD_STORE_SUCCESS;
-        dict_t              *tmp_dict          = NULL;
-        xlator_t            *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_NODE_STATE_FILE);
-
-        ret = gf_store_handle_retrieve (path, &volinfo->node_state_shandle);
-        if (ret)
-                goto out;
-
-        ret = gf_store_iter_new (volinfo->node_state_shandle, &iter);
+                if (brickinfo->rdma_port < priv->base_port) {
+                    /* This is required to adhere to the
+                       IANA standards */
+                    brickinfo->rdma_port = 0;
+                } else {
+                    /* This is required to have proper ports
+                       assigned to bricks after restart */
+                    pmap = pmap_registry_get(THIS);
+                    if (pmap->last_alloc <= brickinfo->rdma_port)
+                        pmap->last_alloc = brickinfo->rdma_port + 1;
+                }
 
-        if (ret)
-                goto out;
+            } else if (!strncmp(
+                           key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                           SLEN(GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
+                ret = gf_string2int(value, &brickinfo->decommissioned);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
+                }
 
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret)
-                goto out;
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH))) {
+                if (snprintf(brickinfo->device_path,
+                             sizeof(brickinfo->device_path), "%s",
+                             value) >= sizeof(brickinfo->device_path)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "device_path truncated: %s", brickinfo->device_path);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR))) {
+                if (snprintf(brickinfo->mount_dir, sizeof(brickinfo->mount_dir),
+                             "%s", value) >= sizeof(brickinfo->mount_dir)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "mount_dir truncated: %s", brickinfo->mount_dir);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
+                ret = gf_string2int(value, &brickinfo->snap_status);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
+                }
 
-        while (ret == 0) {
-                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
-                              strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
-                        volinfo->rebal.defrag_cmd = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
-                           strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS))) {
-                                volinfo->rebal.defrag_status = atoi (value);
-                } else if (!strncmp (key, GF_REBALANCE_TID_KEY,
-                                     strlen (GF_REBALANCE_TID_KEY))) {
-                        gf_uuid_parse (value, volinfo->rebal.rebalance_id);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
-                                        strlen (GLUSTERD_STORE_KEY_DEFRAG_OP))) {
-                        volinfo->rebal.op = atoi (value);
-                } else {
-                        if (!tmp_dict) {
-                                tmp_dict = dict_new ();
-                                if (!tmp_dict) {
-                                        ret = -1;
-                                        goto out;
-                                }
-                        }
-                        dup_value = gf_strdup (value);
-                        if (!dup_value) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        GD_MSG_NO_MEMORY,
-                                        "Failed to strdup value string");
-                                goto out;
-                        }
-                        ret = dict_set_str (tmp_dict, key, dup_value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED,
-                                        "Error setting data in rebal "
-                                        "dict.");
-                                goto out;
-                        }
-                        dup_value = NULL;
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSTYPE,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_FSTYPE))) {
+                if (snprintf(brickinfo->fstype, sizeof(brickinfo->fstype), "%s",
+                             value) >= sizeof(brickinfo->fstype)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL, "fstype truncated: %s",
+                           brickinfo->fstype);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_MNTOPTS,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_MNTOPTS))) {
+                if (snprintf(brickinfo->mnt_opts, sizeof(brickinfo->mnt_opts),
+                             "%s", value) >= sizeof(brickinfo->mnt_opts)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "mnt_opts truncated: %s", brickinfo->mnt_opts);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
+                if (snprintf(brickinfo->vg, sizeof(brickinfo->vg), "%s",
+                             value) >= sizeof(brickinfo->vg)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brickinfo->vg truncated: %s", brickinfo->vg);
+                    goto out;
+                }
+            } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
+                if (snprintf(brickinfo->brick_id, sizeof(brickinfo->brick_id),
+                             "%s", value) >= sizeof(brickinfo->brick_id)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brick_id truncated: %s", brickinfo->brick_id);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSID,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_FSID))) {
+                ret = gf_string2uint64(value, &brickinfo->statfs_fsid);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                           "%s "
+                           "is not a valid uint64_t value",
+                           value);
                 }
 
-                GF_FREE (key);
-                GF_FREE (value);
-                key = NULL;
-                value = NULL;
+            } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_UUID)) {
+                gf_uuid_parse(value, brickinfo->uuid);
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+                       "Unknown key: %s", key);
+            }
 
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
+            GF_FREE(key);
+            GF_FREE(value);
+            key = NULL;
+            value = NULL;
+
+            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
         }
-        if (tmp_dict)
-                volinfo->rebal.dict = dict_ref (tmp_dict);
 
         if (op_errno != GD_STORE_EOF) {
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   GD_MSG_PARSE_BRICKINFO_FAIL,
+                   "Error parsing brickinfo: "
+                   "op_errno=%d",
+                   op_errno);
+            goto out;
         }
 
-        ret = gf_store_iter_destroy (iter);
-
-        if (ret)
+        if (brickinfo->brick_id[0] == '\0') {
+            /* This is an old volume upgraded to op_version 4 */
+            GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid++);
+        }
+        /* Populate brickinfo->real_path for normal volumes, for
+         * snapshot or snapshot restored volume this would be done post
+         * creating the brick mounts
+         */
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+        if (brickinfo->real_path[0] == '\0' && !volinfo->is_snap_volume &&
+            gf_uuid_is_null(volinfo->restored_from_snap)) {
+            /* By now if the brick is a local brick then it will be
+             * able to resolve which is the only thing we want now
+             * for checking  whether the brickinfo->uuid matches
+             * with MY_UUID for realpath check. Hence do not handle
+             * error
+             */
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                if (!realpath(brickinfo->path, abspath)) {
+                    gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                           GD_MSG_BRICKINFO_CREATE_FAIL,
+                           "realpath() failed for brick %s"
+                           ". The underlying file system "
+                           "may be in bad state",
+                           brickinfo->path);
+                    ret = -1;
+                    goto out;
+                }
+                if (strlen(abspath) >= sizeof(brickinfo->real_path)) {
+                    ret = -1;
+                    goto out;
+                }
+                (void)strncpy(brickinfo->real_path, abspath,
+                              sizeof(brickinfo->real_path));
+            }
+        }
+
+        /* Handle upgrade case of shared_brick_count 'fsid' */
+        /* Ideally statfs_fsid should never be 0 if done right */
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID) &&
+            brickinfo->statfs_fsid == 0) {
+            struct statvfs brickstat = {
+                0,
+            };
+            ret = sys_statvfs(brickinfo->path, &brickstat);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, errno,
+                       GD_MSG_BRICKINFO_CREATE_FAIL,
+                       "failed to get statfs() call on brick %s",
+                       brickinfo->path);
+                /* No need for treating it as an error, lets continue
+                   with just a message */
+            } else {
+                brickinfo->statfs_fsid = brickstat.f_fsid;
+            }
+        }
+
+        cds_list_add_tail(&brickinfo->brick_list, &volinfo->bricks);
+        brick_count++;
+    }
+
+    if (gf_store_iter_destroy(&tmpiter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_iter_new(volinfo->shandle, &tmpiter);
+
+    if (ret)
+        goto out;
+
+    if (volinfo->thin_arbiter_count == 1) {
+        snprintf(tmpkey, sizeof(tmpkey), "%s-%d",
+                 GLUSTERD_STORE_KEY_VOL_TA_BRICK, 0);
+        while (ta_brick_count < volinfo->subvol_count) {
+            ret = glusterd_brickinfo_new(&ta_brickinfo);
+            if (ret)
+                goto out;
+
+            ret = gf_store_iter_get_matching(tmpiter, tmpkey, &tmpvalue);
+
+            len = snprintf(path, sizeof(path), "%s/%s", brickdir, tmpvalue);
+            GF_FREE(tmpvalue);
+            tmpvalue = NULL;
+            if ((len < 0) || (len >= sizeof(path))) {
+                ret = -1;
                 goto out;
+            }
 
-out:
-        if (dup_value)
-                GF_FREE (dup_value);
-        if (ret && volinfo->rebal.dict)
-                dict_unref (volinfo->rebal.dict);
-        if (tmp_dict)
-                dict_unref (tmp_dict);
-
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
+            ret = gf_store_handle_retrieve(path, &ta_brickinfo->shandle);
 
-        return ret;
-}
+            if (ret)
+                goto out;
 
+            ret = gf_store_iter_new(ta_brickinfo->shandle, &iter);
 
-int
-glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
-{
-        int                     ret                     = -1;
-        int                     exists                  = 0;
-        char                    *key                    = NULL;
-        char                    *value                  = NULL;
-        char                    volpath[PATH_MAX]       = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_conf_t         *conf                   = NULL;
-        gf_store_iter_t         *iter                   = NULL;
-        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = THIS->private;
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
-
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_VOLUME_INFO_FILE);
-
-        ret = gf_store_handle_retrieve (path, &volinfo->shandle);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HANDLE_NULL, "volinfo handle is NULL");
+            if (ret)
                 goto out;
-        }
 
-        ret = gf_store_iter_new (volinfo->shandle, &iter);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_GET_FAIL, "Failed to get new store "
-                        "iter");
+            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                       GD_MSG_STORE_ITER_GET_FAIL,
+                       "Unable to iterate "
+                       "the store for brick: %s",
+                       path);
                 goto out;
-        }
+            }
 
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_GET_FAIL, "Failed to get next store "
-                        "iter");
-                goto out;
-        }
+            while (!ret) {
+                if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                             SLEN(GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
+                    if (snprintf(ta_brickinfo->hostname,
+                                 sizeof(ta_brickinfo->hostname), "%s",
+                                 value) >= sizeof(ta_brickinfo->hostname)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "brick hostname truncated: %s",
+                               ta_brickinfo->hostname);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_PATH))) {
+                    if (snprintf(ta_brickinfo->path, sizeof(ta_brickinfo->path),
+                                 "%s", value) >= sizeof(ta_brickinfo->path)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "brick path truncated: %s", ta_brickinfo->path);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_REAL_PATH))) {
+                    if (snprintf(ta_brickinfo->real_path,
+                                 sizeof(ta_brickinfo->real_path), "%s",
+                                 value) >= sizeof(ta_brickinfo->real_path)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "real_path truncated: %s",
+                               ta_brickinfo->real_path);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PORT,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_PORT))) {
+                    ret = gf_string2int(value, &ta_brickinfo->port);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
+
+                    if (ta_brickinfo->port < priv->base_port) {
+                        /* This is required to adhere to the
+                        IANA standards */
+                        ta_brickinfo->port = 0;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+                    ret = gf_string2int(value, &ta_brickinfo->rdma_port);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
+
+                    if (ta_brickinfo->rdma_port < priv->base_port) {
+                        /* This is required to adhere to the
+                        IANA standards */
+                        ta_brickinfo->rdma_port = 0;
+                    }
+                } else if (!strncmp(
+                               key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                               SLEN(GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
+                    ret = gf_string2int(value, &ta_brickinfo->decommissioned);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
+
+                } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
+                    if (snprintf(ta_brickinfo->brick_id,
+                                 sizeof(ta_brickinfo->brick_id), "%s",
+                                 value) >= sizeof(ta_brickinfo->brick_id)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "brick_id truncated: %s",
+                               ta_brickinfo->brick_id);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSID,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_FSID))) {
+                    ret = gf_string2uint64(value, &ta_brickinfo->statfs_fsid);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_INVALID_ENTRY,
+                               "%s "
+                               "is not a valid uint64_t value",
+                               value);
+                    }
+                } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_UUID)) {
+                    gf_uuid_parse(value, brickinfo->uuid);
+                } else if (!strncmp(
+                               key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                               SLEN(GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
+                    ret = gf_string2int(value, &ta_brickinfo->snap_status);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
 
-        while (!ret) {
-                gf_msg_debug (this->name, 0, "key = %s value = %s", key, value);
-                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TYPE,
-                              strlen (GLUSTERD_STORE_KEY_VOL_TYPE))) {
-                        volinfo->type = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_COUNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_COUNT))) {
-                        volinfo->brick_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_STATUS,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_STATUS))) {
-                        volinfo->status = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_VERSION,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_VERSION))) {
-                        volinfo->version = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_PORT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_PORT))) {
-                        volinfo->port = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_SUB_COUNT))) {
-                        volinfo->sub_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_STRIPE_CNT))) {
-                        volinfo->stripe_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_REPLICA_CNT))) {
-                        volinfo->replica_count = atoi (value);
-                } else if (!strcmp (key, GLUSTERD_STORE_KEY_VOL_ARBITER_CNT)) {
-                        volinfo->arbiter_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT))) {
-                        volinfo->disperse_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT))) {
-                        volinfo->redundancy_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_TRANSPORT))) {
-                        volinfo->transport_type = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_ID,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_ID))) {
-                        ret = gf_uuid_parse (value, volinfo->volume_id);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_UUID_PARSE_FAIL,
-                                        "failed to parse uuid");
-
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_USERNAME,
-                                     strlen (GLUSTERD_STORE_KEY_USERNAME))) {
-
-                        glusterd_auth_set_username (volinfo, value);
-
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PASSWORD,
-                                     strlen (GLUSTERD_STORE_KEY_PASSWORD))) {
-
-                        glusterd_auth_set_password (volinfo, value);
-
-                } else if (strstr (key, "slave")) {
-                        ret = dict_set_dynstr (volinfo->gsync_slaves, key,
-                                                gf_strdup (value));
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_DICT_SET_FAILED, "Error in "
-                                        "dict_set_str");
-                                goto out;
-                        }
-                        gf_msg_debug (this->name, 0, "Parsed as "GEOREP" "
-                                " slave:key=%s,value:%s", key, value);
-
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
-                                strlen (GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
-                        volinfo->op_version = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
-                                strlen (GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
-                        volinfo->client_op_version = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CAPS,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_CAPS))) {
-                        volinfo->caps = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                strlen (GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
-                        volinfo->snap_max_hard_limit = (uint64_t) atoll (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
-                              strlen (GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP))) {
-                        ret = gf_uuid_parse (value, volinfo->restored_from_snap);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_UUID_PARSE_FAIL,
-                                        "failed to parse restored snap's uuid");
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
-                                strlen (GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
-                        strncpy (volinfo->parent_volname, value,
-                                 sizeof(volinfo->parent_volname) - 1);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_brick_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_REPLICA_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_replica_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_DISPERSE_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_disperse_count = atoi (value);
-                } else if (!strncmp (key,
-                                     GLUSTERD_STORE_KEY_COLD_REDUNDANCY_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_redundancy_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.hot_brick_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.hot_replica_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_TYPE,
-                                     strlen (key))) {
-                        volinfo->tier_info.hot_type = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_TYPE,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_type = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
-                            strlen (GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION))) {
-                        volinfo->quota_xattr_version = atoi (value);
                 } else {
-
-                        if (is_key_glusterd_hooks_friendly (key)) {
-                                exists = 1;
-
-                        } else  {
-                                exists = glusterd_check_option_exists (key,
-                                                                       NULL);
-                        }
-
-                        switch (exists) {
-                        case -1:
-                                ret = -1;
-                                goto out;
-
-                        case 0:
-                                 /*Ignore GLUSTERD_STORE_KEY_VOL_BRICK since
-                                  glusterd_store_retrieve_bricks gets it later*/
-                                if (!strstr (key, GLUSTERD_STORE_KEY_VOL_BRICK))
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                GD_MSG_UNKNOWN_KEY,
-                                                "Unknown key: %s", key);
-                                break;
-
-                        case 1:
-                                /*The following strcmp check is to ensure that
-                                 * glusterd does not restore the quota limits
-                                 * into volinfo->dict post upgradation from 3.3
-                                 * to 3.4 as the same limits will now be stored
-                                 * in xattrs on the respective directories.
-                                 */
-                                if (!strcmp (key, "features.limit-usage"))
-                                        break;
-                                ret = dict_set_str(volinfo->dict, key,
-                                                   gf_strdup (value));
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_DICT_SET_FAILED,
-                                                "Error in "
-                                                "dict_set_str");
-                                        goto out;
-                                }
-                                gf_msg_debug (this->name, 0, "Parsed as Volume-"
-                                        "set:key=%s,value:%s", key, value);
-                                break;
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+                           "Unknown key: %s", key);
                 }
 
-                GF_FREE (key);
-                GF_FREE (value);
+                GF_FREE(key);
+                GF_FREE(value);
                 key = NULL;
                 value = NULL;
+                ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+            }
 
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
-
-        /* backward compatibility */
-        {
-
-                switch (volinfo->type) {
-
-                        case GF_CLUSTER_TYPE_NONE:
-                                volinfo->stripe_count  = 1;
-                                volinfo->replica_count = 1;
-                        break;
+            GLUSTERD_ASSIGN_BRICKID_TO_TA_BRICKINFO(ta_brickinfo, volinfo,
+                                                    ta_brick_id);
+            ta_brick_id += 3;
 
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                volinfo->stripe_count  = volinfo->sub_count;
-                                volinfo->replica_count = 1;
-                        break;
-
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                volinfo->stripe_count  = 1;
-                                volinfo->replica_count = volinfo->sub_count;
-                        break;
-
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                                /* Introduced in 3.3 */
-                                GF_ASSERT (volinfo->stripe_count > 0);
-                                GF_ASSERT (volinfo->replica_count > 0);
-                        break;
-
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                GF_ASSERT (volinfo->disperse_count > 0);
-                                GF_ASSERT (volinfo->redundancy_count > 0);
-                        break;
-
-                        case GF_CLUSTER_TYPE_TIER:
-                        if (volinfo->tier_info.cold_type ==
-                                        GF_CLUSTER_TYPE_DISPERSE)
-                                volinfo->tier_info.cold_dist_leaf_count
-                                        = volinfo->disperse_count;
-                        else
-                                volinfo->tier_info.cold_dist_leaf_count
-                                        = glusterd_calc_dist_leaf_count (
-                                                        volinfo->tier_info.
-                                                        cold_replica_count,
-                                                        1);
-
-                        break;
-
-                        default:
-                                GF_ASSERT (0);
-                        break;
-                }
-
-                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
-
-                volinfo->subvol_count = (volinfo->brick_count /
-                                         volinfo->dist_leaf_count);
-
-                /* Only calculate volume op-versions if they are not found */
-                if (!volinfo->op_version && !volinfo->client_op_version)
-                        gd_update_volume_op_versions (volinfo);
+            cds_list_add_tail(&ta_brickinfo->brick_list, &volinfo->ta_bricks);
+            ta_brick_count++;
         }
+    }
 
-        if (op_errno != GD_STORE_EOF)
-                goto out;
+    assign_brick_groups(volinfo);
+    ret = 0;
 
-        ret = gf_store_iter_destroy (iter);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_DESTROY_FAIL,
-                        "Failed to destroy store "
-                        "iter");
-                goto out;
-        }
-
-        ret = 0;
 out:
-        return ret;
-}
+    if (gf_store_iter_destroy(&tmpiter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
 
-glusterd_volinfo_t*
-glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap)
-{
-        int32_t                    ret                  = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
-        glusterd_volinfo_t        *origin_volinfo       = NULL;
-        glusterd_conf_t           *priv                 = NULL;
-        xlator_t                  *this                 = NULL;
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
 
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volname);
+    return ret;
+}
 
-        ret = glusterd_volinfo_new (&volinfo);
-        if (ret)
+int32_t
+glusterd_store_retrieve_node_state(glusterd_volinfo_t *volinfo)
+{
+    int32_t ret = -1;
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *dup_value = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    dict_t *tmp_dict = NULL;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+    len = snprintf(path, sizeof(path), "%s/%s", volpath,
+                   GLUSTERD_NODE_STATE_FILE);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &volinfo->node_state_shandle);
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_new(volinfo->node_state_shandle, &iter);
+
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+
+    if (ret)
+        goto out;
+
+    while (ret == 0) {
+        if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
+                     SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
+            volinfo->rebal.defrag_cmd = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS))) {
+            volinfo->rebal.defrag_status = atoi(value);
+        } else if (!strncmp(key, GF_REBALANCE_TID_KEY,
+                            SLEN(GF_REBALANCE_TID_KEY))) {
+            gf_uuid_parse(value, volinfo->rebal.rebalance_id);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                            SLEN(GLUSTERD_STORE_KEY_DEFRAG_OP))) {
+            volinfo->rebal.op = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_files);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_data);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.lookedup_files);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_failures);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.skipped_files);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME))) {
+            volinfo->rebal.rebalance_time = atoi(value);
+        } else {
+            if (!tmp_dict) {
+                tmp_dict = dict_new();
+                if (!tmp_dict) {
+                    ret = -1;
+                    goto out;
+                }
+            }
+            dup_value = gf_strdup(value);
+            if (!dup_value) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Failed to strdup value string");
                 goto out;
-
-        strncpy (volinfo->volname, volname, GD_VOLUME_NAME_MAX);
-        volinfo->snapshot = snap;
-        if (snap)
-                volinfo->is_snap_volume = _gf_true;
-
-        ret = glusterd_store_update_volinfo (volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLINFO_UPDATE_FAIL, "Failed to update volinfo "
-                        "for %s volume", volname);
+            }
+            ret = dict_set_str(tmp_dict, key, dup_value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Error setting data in rebal "
+                       "dict.");
                 goto out;
+            }
+            dup_value = NULL;
         }
 
-        ret = glusterd_store_retrieve_bricks (volinfo);
-        if (ret)
-                goto out;
+        GF_FREE(key);
+        GF_FREE(value);
+        key = NULL;
+        value = NULL;
 
-        ret = glusterd_store_retrieve_snapd (volinfo);
-        if (ret)
-                goto out;
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+    if (tmp_dict) {
+        volinfo->rebal.dict = dict_ref(tmp_dict);
+    }
 
-        ret = glusterd_compute_cksum (volinfo, _gf_false);
-        if (ret)
-                goto out;
+    if (op_errno != GD_STORE_EOF) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_retrieve_quota_version (volinfo);
-        if (ret)
-                goto out;
+    ret = 0;
 
-        ret = glusterd_store_create_quota_conf_sh_on_absence (volinfo);
-        if (ret)
-                goto out;
+out:
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
+
+    if (dup_value)
+        GF_FREE(dup_value);
+    if (ret) {
+        if (volinfo->rebal.dict)
+            dict_unref(volinfo->rebal.dict);
+    }
+    if (tmp_dict)
+        dict_unref(tmp_dict);
+
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+
+    return ret;
+}
 
-        ret = glusterd_compute_cksum (volinfo, _gf_true);
-        if (ret)
-                goto out;
+int
+glusterd_store_update_volinfo(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    int exists = 0;
+    char *key = NULL;
+    char *value = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_iter_t *iter = NULL;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = THIS->private;
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
+
+    len = snprintf(path, sizeof(path), "%s/%s", volpath,
+                   GLUSTERD_VOLUME_INFO_FILE);
+    if ((len < 0) || (len >= sizeof(path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &volinfo->shandle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
+               "volinfo handle is NULL");
+        goto out;
+    }
+
+    ret = gf_store_iter_new(volinfo->shandle, &iter);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get new store "
+               "iter");
+        goto out;
+    }
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get next store "
+               "iter");
+        goto out;
+    }
+
+    while (!ret) {
+        gf_msg_debug(this->name, 0, "key = %s value = %s", key, value);
+        if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_TYPE,
+                     SLEN(GLUSTERD_STORE_KEY_VOL_TYPE))) {
+            volinfo->type = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_COUNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_COUNT))) {
+            volinfo->brick_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_STATUS,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_STATUS))) {
+            volinfo->status = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_VERSION))) {
+            volinfo->version = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_PORT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_PORT))) {
+            volinfo->port = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_SUB_COUNT))) {
+            volinfo->sub_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_STRIPE_CNT))) {
+            volinfo->stripe_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_REPLICA_CNT))) {
+            volinfo->replica_count = atoi(value);
+        } else if (!strcmp(key, GLUSTERD_STORE_KEY_VOL_ARBITER_CNT)) {
+            volinfo->arbiter_count = atoi(value);
+        } else if (!strcmp(key, GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT)) {
+            volinfo->thin_arbiter_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT))) {
+            volinfo->disperse_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT))) {
+            volinfo->redundancy_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_TRANSPORT))) {
+            volinfo->transport_type = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_ID,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_ID))) {
+            ret = gf_uuid_parse(value, volinfo->volume_id);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
+                       "failed to parse uuid");
+
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_USERNAME,
+                            SLEN(GLUSTERD_STORE_KEY_USERNAME))) {
+            glusterd_auth_set_username(volinfo, value);
+
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_PASSWORD,
+                            SLEN(GLUSTERD_STORE_KEY_PASSWORD))) {
+            glusterd_auth_set_password(volinfo, value);
+
+        } else if (strstr(key, "slave")) {
+            ret = dict_set_dynstr(volinfo->gsync_slaves, key, gf_strdup(value));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Error in "
+                       "dict_set_str");
+                goto out;
+            }
+            gf_msg_debug(this->name, 0,
+                         "Parsed as " GEOREP
+                         " "
+                         " slave:key=%s,value:%s",
+                         key, value);
+
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
+            volinfo->op_version = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
+            volinfo->client_op_version = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
+            volinfo->snap_max_hard_limit = (uint64_t)atoll(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP))) {
+            ret = gf_uuid_parse(value, volinfo->restored_from_snap);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
+                       "failed to parse restored snap's uuid");
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
+                            SLEN(GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
+            if (snprintf(volinfo->parent_volname,
+                         sizeof(volinfo->parent_volname), "%s",
+                         value) >= sizeof(volinfo->parent_volname)) {
+                gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                       GD_MSG_PARSE_BRICKINFO_FAIL,
+                       "parent_volname truncated: %s", volinfo->parent_volname);
+                goto out;
+            }
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION))) {
+            volinfo->quota_xattr_version = atoi(value);
+        } else {
+            if (is_key_glusterd_hooks_friendly(key)) {
+                exists = 1;
 
-        ret = glusterd_store_save_quota_version_and_cksum (volinfo);
-        if (ret)
-                goto out;
+            } else {
+                exists = glusterd_check_option_exists(key, NULL);
+            }
+
+            switch (exists) {
+                case -1:
+                    ret = -1;
+                    goto out;
+
+                case 0:
+                    /*Ignore GLUSTERD_STORE_KEY_VOL_BRICK since
+                     glusterd_store_retrieve_bricks gets it later.
+                     also, ignore tier-enabled key as we deprecated
+                     tier xlator*/
+                    if (!strstr(key, GLUSTERD_STORE_KEY_VOL_BRICK) ||
+                        !strstr(key, GF_TIER_ENABLED))
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               GD_MSG_UNKNOWN_KEY, "Unknown key: %s", key);
+                    break;
+
+                case 1:
+                    /*The following strcmp check is to ensure that
+                     * glusterd does not restore the quota limits
+                     * into volinfo->dict post upgradation from 3.3
+                     * to 3.4 as the same limits will now be stored
+                     * in xattrs on the respective directories.
+                     */
+                    if (!strcmp(key, "features.limit-usage"))
+                        break;
+                    ret = dict_set_str(volinfo->dict, key, gf_strdup(value));
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_DICT_SET_FAILED,
+                               "Error in "
+                               "dict_set_str");
+                        goto out;
+                    }
+                    gf_msg_debug(this->name, 0,
+                                 "Parsed as Volume-"
+                                 "set:key=%s,value:%s",
+                                 key, value);
+                    break;
+            }
+        }
+
+        GF_FREE(key);
+        GF_FREE(value);
+        key = NULL;
+        value = NULL;
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+
+    /* backward compatibility */
+    {
+        switch (volinfo->type) {
+            case GF_CLUSTER_TYPE_NONE:
+                volinfo->stripe_count = 1;
+                volinfo->replica_count = 1;
+                break;
+
+            case GF_CLUSTER_TYPE_REPLICATE:
+                volinfo->stripe_count = 1;
+                volinfo->replica_count = volinfo->sub_count;
+                break;
+
+            case GF_CLUSTER_TYPE_DISPERSE:
+                GF_ASSERT(volinfo->disperse_count > 0);
+                GF_ASSERT(volinfo->redundancy_count > 0);
+                break;
+
+            case GF_CLUSTER_TYPE_STRIPE:
+            case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                gf_msg(this->name, GF_LOG_CRITICAL, ENOTSUP,
+                       GD_MSG_VOLINFO_STORE_FAIL,
+                       "The volume type is no more supported. Please refer to "
+                       "glusterfs-6.0 release-notes for how to migrate from "
+                       "this volume type");
+                break;
+
+            default:
+                GF_ASSERT(0);
+                break;
+        }
+
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
+
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
+
+        /* Only calculate volume op-versions if they are not found */
+        if (!volinfo->op_version && !volinfo->client_op_version)
+            gd_update_volume_op_versions(volinfo);
+    }
+
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+
+    ret = 0;
 
+out:
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
 
-        if (!snap) {
-                glusterd_list_add_order (&volinfo->vol_list, &priv->volumes,
-                                         glusterd_compare_volume_name);
+    return ret;
+}
 
-        } else {
-                ret = glusterd_volinfo_find (volinfo->parent_volname,
-                                             &origin_volinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOLINFO_GET_FAIL, "Parent volinfo "
-                                "not found for %s volume", volname);
-                        goto out;
-                }
-                glusterd_list_add_snapvol (origin_volinfo, volinfo);
+glusterd_volinfo_t *
+glusterd_store_retrieve_volume(char *volname, glusterd_snap_t *snap)
+{
+    int32_t ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *origin_volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volname);
+
+    ret = glusterd_volinfo_new(&volinfo);
+    if (ret)
+        goto out;
+
+    if (snprintf(volinfo->volname, NAME_MAX + 1, "%s", volname) >= NAME_MAX + 1)
+        goto out;
+    volinfo->snapshot = snap;
+    if (snap)
+        volinfo->is_snap_volume = _gf_true;
+
+    ret = glusterd_store_update_volinfo(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_UPDATE_FAIL,
+               "Failed to update volinfo "
+               "for %s volume",
+               volname);
+        goto out;
+    }
+
+    ret = glusterd_store_retrieve_bricks(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_retrieve_snapd(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_compute_cksum(volinfo, _gf_false);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_retrieve_quota_version(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_create_quota_conf_sh_on_absence(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_compute_cksum(volinfo, _gf_true);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_save_quota_version_and_cksum(volinfo);
+    if (ret)
+        goto out;
+
+    if (!snap) {
+        glusterd_list_add_order(&volinfo->vol_list, &priv->volumes,
+                                glusterd_compare_volume_name);
+
+    } else {
+        ret = glusterd_volinfo_find(volinfo->parent_volname, &origin_volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Parent volinfo "
+                   "not found for %s volume",
+                   volname);
+            goto out;
         }
+        glusterd_list_add_snapvol(origin_volinfo, volinfo);
+    }
 
 out:
-        if (ret) {
-                if (volinfo)
-                        glusterd_volinfo_unref (volinfo);
-                volinfo = NULL;
-        }
+    if (ret) {
+        if (volinfo)
+            glusterd_volinfo_unref(volinfo);
+        volinfo = NULL;
+    }
 
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
 
-        return volinfo;
+    return volinfo;
 }
 
 static void
-glusterd_store_set_options_path (glusterd_conf_t *conf, char *path, size_t len)
+glusterd_store_set_options_path(glusterd_conf_t *conf, char *path, size_t len)
 {
-        snprintf (path, len, "%s/options", conf->workdir);
-}
-
-int
-_store_global_opts (dict_t *this, char *key, data_t *value, void *data)
-{
-        gf_store_handle_t *shandle = data;
-
-        gf_store_save_value (shandle->fd, key, (char*)value->data);
-        return 0;
+    snprintf(path, len, "%s/options", conf->workdir);
 }
 
 int32_t
-glusterd_store_options (xlator_t *this, dict_t *opts)
+glusterd_store_options(xlator_t *this, dict_t *opts)
 {
-        gf_store_handle_t               *shandle = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        char                            path[PATH_MAX] = {0};
-        int                             fd = -1;
-        int32_t                         ret = -1;
-
-        conf = this->private;
-        glusterd_store_set_options_path (conf, path, sizeof (path));
-
-        ret = gf_store_handle_new (path, &shandle);
-        if (ret)
-                goto out;
-
-        fd = gf_store_mkstemp (shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
+    gf_store_handle_t *shandle = NULL;
+    glusterd_conf_t *conf = NULL;
+    char path[PATH_MAX] = {0};
+    int fd = -1;
+    int32_t ret = -1;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+
+    conf = this->private;
+    glusterd_store_set_options_path(conf, path, sizeof(path));
+
+    ret = gf_store_handle_new(path, &shandle);
+    if (ret) {
+        goto out;
+    }
+
+    fd = gf_store_mkstemp(shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
+
+    dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
+                          gf_gld_mt_volinfo_dict_data_t);
+    if (dict_data == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+        return -1;
+    }
+    dict_data->shandle = shandle;
+    shandle->fd = fd;
+    dict_foreach(opts, _storeopts, (void *)dict_data);
+    if (dict_data->buffer_len > 0) {
+        ret = gf_store_save_items(fd, dict_data->buffer);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
+            goto out;
         }
+    }
 
-        shandle->fd = fd;
-        dict_foreach (opts, _store_global_opts, shandle);
-        shandle->fd = 0;
-        ret = gf_store_rename_tmppath (shandle);
-        if (ret)
-                goto out;
+    ret = gf_store_rename_tmppath(shandle);
 out:
-        if ((ret < 0) && (fd > 0))
-                gf_store_unlink_tmppath (shandle);
-        gf_store_handle_destroy (shandle);
-        return ret;
+    shandle->fd = 0;
+    GF_FREE(dict_data);
+    if ((ret < 0) && (fd > 0)) {
+        gf_store_unlink_tmppath(shandle);
+    }
+    gf_store_handle_destroy(shandle);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_options (xlator_t *this)
+glusterd_store_retrieve_options(xlator_t *this)
 {
-        char                            path[PATH_MAX] = {0};
-        glusterd_conf_t                 *conf = NULL;
-        gf_store_handle_t               *shandle = NULL;
-        gf_store_iter_t                 *iter = NULL;
-        char                            *key = NULL;
-        char                            *value = NULL;
-        gf_store_op_errno_t             op_errno = 0;
-        int                             ret = -1;
-
-        conf = this->private;
-        glusterd_store_set_options_path (conf, path, sizeof (path));
-
-        ret = gf_store_handle_retrieve (path, &shandle);
-        if (ret)
-                goto out;
-
-        ret = gf_store_iter_new (shandle, &iter);
-        if (ret)
-                goto out;
-
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        while (!ret) {
-                ret = dict_set_dynstr (conf->opts, key, value);
-                if (ret) {
-                        GF_FREE (key);
-                        GF_FREE (value);
-                        goto out;
-                }
-                GF_FREE (key);
-                key = NULL;
-                value = NULL;
-
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
-        if (op_errno != GD_STORE_EOF)
-                goto out;
-        ret = 0;
+    char path[PATH_MAX] = {0};
+    glusterd_conf_t *conf = NULL;
+    gf_store_handle_t *shandle = NULL;
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    gf_store_op_errno_t op_errno = 0;
+    int ret = -1;
+
+    conf = this->private;
+    glusterd_store_set_options_path(conf, path, sizeof(path));
+
+    ret = gf_store_handle_retrieve(path, &shandle);
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_new(shandle, &iter);
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    while (!ret) {
+        ret = dict_set_dynstr(conf->opts, key, value);
+        if (ret) {
+            GF_FREE(key);
+            GF_FREE(value);
+            goto out;
+        }
+        GF_FREE(key);
+        key = NULL;
+        value = NULL;
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+    ret = 0;
 out:
-        gf_store_iter_destroy (iter);
-        gf_store_handle_destroy (shandle);
-        return ret;
+    (void)gf_store_iter_destroy(&iter);
+    gf_store_handle_destroy(shandle);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_volumes (xlator_t  *this, glusterd_snap_t *snap)
+glusterd_store_retrieve_volumes(xlator_t *this, glusterd_snap_t *snap)
 {
-        int32_t                ret              = -1;
-        char                   path[PATH_MAX]   = {0,};
-        glusterd_conf_t       *priv             = NULL;
-        DIR                   *dir              = NULL;
-        struct dirent         *entry            = NULL;
-        glusterd_volinfo_t    *volinfo          = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        if (snap)
-                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,
-                          snap->snapname);
-        else
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_VOLUME_DIR_PREFIX);
-
-        dir = sys_opendir (path);
-
-        if (!dir) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED, "Unable to open dir %s", path);
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-
-        while (entry) {
-                if (snap && ((!strcmp (entry->d_name, "geo-replication")) ||
-                             (!strcmp (entry->d_name, "info"))))
-                        goto next;
-
-                volinfo = glusterd_store_retrieve_volume (entry->d_name, snap);
-                if (!volinfo) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_VOL_RESTORE_FAIL, "Unable to restore "
-                                "volume: %s", entry->d_name);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_store_retrieve_node_state (volinfo);
-                if (ret) {
-                        /* Backward compatibility */
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_NEW_NODE_STATE_CREATION,
-                                "Creating a new node_state "
-                                "for volume: %s.", entry->d_name);
-                        glusterd_store_create_nodestate_sh_on_absence (volinfo);
-                        ret = glusterd_store_perform_node_state_store (volinfo);
-
-                }
-next:
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+    int32_t ret = -1;
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    struct stat st = {
+        0,
+    };
+    char entry_path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    if (snap)
+        len = snprintf(path, PATH_MAX, "%s/snaps/%s", priv->workdir,
+                       snap->snapname);
+    else
+        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                       GLUSTERD_VOLUME_DIR_PREFIX);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+
+    dir = sys_opendir(path);
+
+    if (!dir) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Unable to open dir %s", path);
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (snap && ((!strcmp(entry->d_name, "geo-replication")) ||
+                     (!strcmp(entry->d_name, "info"))))
+            continue;
+
+        len = snprintf(entry_path, PATH_MAX, "%s/%s", path, entry->d_name);
+        if ((len < 0) || (len >= PATH_MAX))
+            continue;
+
+        ret = sys_lstat(entry_path, &st);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Failed to stat entry %s : %s", path, strerror(errno));
+            continue;
+        }
+
+        if (!S_ISDIR(st.st_mode)) {
+            gf_msg_debug(this->name, 0, "%s is not a valid volume",
+                         entry->d_name);
+            continue;
+        }
+
+        volinfo = glusterd_store_retrieve_volume(entry->d_name, snap);
+        if (!volinfo) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_RESTORE_FAIL,
+                   "Unable to restore "
+                   "volume: %s",
+                   entry->d_name);
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_store_retrieve_node_state(volinfo);
+        if (ret) {
+            /* Backward compatibility */
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NEW_NODE_STATE_CREATION,
+                   "Creating a new node_state "
+                   "for volume: %s.",
+                   entry->d_name);
+            glusterd_store_create_nodestate_sh_on_absence(volinfo);
+            glusterd_store_perform_node_state_store(volinfo);
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (dir)
-                sys_closedir (dir);
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+    if (dir)
+        sys_closedir(dir);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 /* Figure out the brick mount path, from the brick path */
 int32_t
-glusterd_find_brick_mount_path (char *brick_path, char **brick_mount_path)
+glusterd_find_brick_mount_path(char *brick_path, char **brick_mount_path)
 {
-        char                    *ptr                 = NULL;
-        char                    *save_ptr            = NULL;
-        int32_t                  ret                 = -1;
-        xlator_t                *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_path);
-        GF_ASSERT (brick_mount_path);
-
-        *brick_mount_path = gf_strdup (brick_path);
-        if (!*brick_mount_path) {
-                ret = -1;
-                goto out;
-        }
-
-        /* Finding the pointer to the end of
-         * /var/run/gluster/snaps/<snap-uuid>
+    char *ptr = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_path);
+    GF_ASSERT(brick_mount_path);
+
+    *brick_mount_path = gf_strdup(brick_path);
+    if (!*brick_mount_path) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Finding the pointer to the end of
+     * /var/run/gluster/snaps/<snap-uuid>
+     */
+    ptr = strstr(*brick_mount_path, "brick");
+    if (!ptr) {
+        /* Snapshot bricks must have brick num as part
+         * of the brickpath
          */
-        ptr = strstr (*brick_mount_path, "brick");
-        if (!ptr) {
-                /* Snapshot bricks must have brick num as part
-                 * of the brickpath
-                 */
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_INVALID_ENTRY,
-                        "Invalid brick path(%s)", brick_path);
-                ret = -1;
-                goto out;
-        }
-
-        /* Moving the pointer to the end of
-         * /var/run/gluster/snaps/<snap-uuid>/<brick_num>
-         * and assigning '\0' to it.
-         */
-        while ((*ptr != '\0') && (*ptr != '/'))
-                ptr++;
-
-        if (*ptr == '/') {
-                *ptr = '\0';
-        }
-
-        ret = 0;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid brick path(%s)", brick_path);
+        ret = -1;
+        goto out;
+    }
+
+    /* Moving the pointer to the end of
+     * /var/run/gluster/snaps/<snap-uuid>/<brick_num>
+     * and assigning '\0' to it.
+     */
+    while ((*ptr != '\0') && (*ptr != '/'))
+        ptr++;
+
+    if (*ptr == '/') {
+        *ptr = '\0';
+    }
+
+    ret = 0;
 out:
-        if (ret && *brick_mount_path) {
-                GF_FREE (*brick_mount_path);
-                *brick_mount_path = NULL;
-        }
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    if (ret && *brick_mount_path) {
+        GF_FREE(*brick_mount_path);
+        *brick_mount_path = NULL;
+    }
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* Check if brick_mount_path is already mounted. If not, mount the device_path
  * at the brick_mount_path
  */
 int32_t
-glusterd_mount_brick_paths (char *brick_mount_path,
-                            glusterd_brickinfo_t *brickinfo)
+glusterd_mount_brick_paths(char *brick_mount_path,
+                           glusterd_brickinfo_t *brickinfo)
 {
-        int32_t                  ret                 = -1;
-        runner_t                 runner              = {0, };
-        char                     buff [PATH_MAX]     = {0, };
-        struct mntent            save_entry          = {0, };
-        struct mntent           *entry               = NULL;
-        xlator_t                *this                = NULL;
-        glusterd_conf_t         *priv                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_mount_path);
-        GF_ASSERT (brickinfo);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Check if the brick_mount_path is already mounted */
-        entry = glusterd_get_mnt_entry_info (brick_mount_path, buff,
-                                             sizeof (buff), &save_entry);
-        if (entry) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_ALREADY_MOUNTED,
-                        "brick_mount_path (%s) already mounted.",
-                        brick_mount_path);
-                ret = 0;
-                goto out;
-        }
-
-        /* TODO RHEL 6.5 has the logical volumes inactive by default
-         * on reboot. Hence activating the logical vol. Check behaviour
-         * on other systems
-         */
-        /* Activate the snapshot */
-        runinit (&runner);
-        runner_add_args (&runner, "lvchange", "-ay", brickinfo->device_path,
-                         NULL);
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_SNAP_ACTIVATE_FAIL,
-                        "Failed to activate %s.",
-                        brickinfo->device_path);
-                goto out;
-        } else
-                gf_msg_debug (this->name, 0,
-                        "Activating %s successful", brickinfo->device_path);
-
-        /* Mount the snapshot */
-        ret = glusterd_mount_lvm_snapshot (brickinfo, brick_mount_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_MOUNT_FAIL,
-                        "Failed to mount lvm snapshot.");
-                goto out;
-        }
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    char buff[PATH_MAX] = {
+        0,
+    };
+    struct mntent save_entry = {
+        0,
+    };
+    struct mntent *entry = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_mount_path);
+    GF_ASSERT(brickinfo);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Check if the brick_mount_path is already mounted */
+    entry = glusterd_get_mnt_entry_info(brick_mount_path, buff, sizeof(buff),
+                                        &save_entry);
+    if (entry) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_ALREADY_MOUNTED,
+               "brick_mount_path (%s) already mounted.", brick_mount_path);
+        ret = 0;
+        goto out;
+    }
+
+    /* TODO RHEL 6.5 has the logical volumes inactive by default
+     * on reboot. Hence activating the logical vol. Check behaviour
+     * on other systems
+     */
+    /* Activate the snapshot */
+    runinit(&runner);
+    runner_add_args(&runner, "lvchange", "-ay", brickinfo->device_path, NULL);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAP_ACTIVATE_FAIL,
+               "Failed to activate %s.", brickinfo->device_path);
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0, "Activating %s successful",
+                     brickinfo->device_path);
+
+    /* Mount the snapshot */
+    ret = glusterd_mount_lvm_snapshot(brickinfo, brick_mount_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_MOUNT_FAIL,
+               "Failed to mount lvm snapshot.");
+        goto out;
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
-static int32_t
-glusterd_recreate_vol_brick_mounts (xlator_t  *this,
-                                    glusterd_volinfo_t *volinfo)
+int32_t
+glusterd_recreate_vol_brick_mounts(xlator_t *this, glusterd_volinfo_t *volinfo)
 {
-        char                    *brick_mount_path    = NULL;
-        glusterd_brickinfo_t    *brickinfo           = NULL;
-        int32_t                  ret                 = -1;
-        struct stat              st_buf              = {0, };
-        char                     abspath[PATH_MAX]   = {0};
-
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo);
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                /* If the brick is not of this node, or its
-                 * snapshot is pending, or the brick is not
-                 * a snapshotted brick, we continue
-                */
-                if ((gf_uuid_compare (brickinfo->uuid, MY_UUID)) ||
-                    (brickinfo->snap_status == -1) ||
-                    (strlen(brickinfo->device_path) == 0))
-                        continue;
-
-                /* Fetch the brick mount path from the brickinfo->path */
-                ret = glusterd_find_brick_mount_path (brickinfo->path,
-                                                      &brick_mount_path);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRK_MNTPATH_GET_FAIL,
-                                "Failed to find brick_mount_path for %s",
-                                brickinfo->path);
-                        goto out;
-                }
+    char *brick_mount_path = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    struct stat st_buf = {
+        0,
+    };
+    char abspath[PATH_MAX] = {0};
+
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        /* If the brick is not of this node, or its
+         * snapshot is pending, or the brick is not
+         * a snapshotted brick, we continue
+         */
+        if ((gf_uuid_compare(brickinfo->uuid, MY_UUID)) ||
+            (brickinfo->snap_status == -1) ||
+            (strlen(brickinfo->device_path) == 0))
+            continue;
+
+        /* Fetch the brick mount path from the brickinfo->path */
+        ret = glusterd_find_brick_mount_path(brickinfo->path,
+                                             &brick_mount_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNTPATH_GET_FAIL,
+                   "Failed to find brick_mount_path for %s", brickinfo->path);
+            goto out;
+        }
 
-                /* Check if the brickinfo path is present.
-                 * If not create the brick_mount_path */
-                ret = sys_lstat (brickinfo->path, &st_buf);
+        /* Check if the brickinfo path is present.
+         * If not create the brick_mount_path */
+        ret = sys_lstat(brickinfo->path, &st_buf);
+        if (ret) {
+            if (errno == ENOENT) {
+                ret = mkdir_p(brick_mount_path, 0755, _gf_true);
                 if (ret) {
-                        if (errno == ENOENT) {
-                                ret = mkdir_p (brick_mount_path, 0777,
-                                               _gf_true);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                                GD_MSG_CREATE_DIR_FAILED,
-                                                "Failed to create %s. ",
-                                                 brick_mount_path);
-                                        goto out;
-                                }
-                        } else {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        GD_MSG_FILE_OP_FAILED,
-                                        "Brick Path(%s) not valid. ",
-                                         brickinfo->path);
-                                goto out;
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_CREATE_DIR_FAILED, "Failed to create %s. ",
+                           brick_mount_path);
+                    goto out;
                 }
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                       "Brick Path(%s) not valid. ", brickinfo->path);
+                goto out;
+            }
+        }
 
-                /* Check if brick_mount_path is already mounted.
-                 * If not, mount the device_path at the brick_mount_path */
-                ret = glusterd_mount_brick_paths (brick_mount_path, brickinfo);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRK_MNTPATH_MOUNT_FAIL,
-                                "Failed to mount brick_mount_path");
+        /* Check if brick_mount_path is already mounted.
+         * If not, mount the device_path at the brick_mount_path */
+        ret = glusterd_mount_brick_paths(brick_mount_path, brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNTPATH_MOUNT_FAIL,
+                   "Failed to mount brick_mount_path");
+        }
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            if (brickinfo->real_path[0] == '\0') {
+                if (!realpath(brickinfo->path, abspath)) {
+                    gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                           GD_MSG_BRICKINFO_CREATE_FAIL,
+                           "realpath() failed for brick %s"
+                           ". The underlying file system "
+                           "may be in bad state",
+                           brickinfo->path);
+                    ret = -1;
+                    goto out;
                 }
-                if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
-                        if (brickinfo->real_path[0] == '\0') {
-                                if (!realpath (brickinfo->path, abspath)) {
-                                        gf_msg (this->name, GF_LOG_CRITICAL,
-                                                errno,
-                                                GD_MSG_BRICKINFO_CREATE_FAIL,
-                                                "realpath() failed for brick %s"
-                                                ". The underlying file system "
-                                                "may be in bad state",
-                                                brickinfo->path);
-                                        ret = -1;
-                                        goto out;
-                                }
-                                strncpy (brickinfo->real_path, abspath,
-                                         strlen(abspath));
-                        }
+                if (strlen(abspath) >= sizeof(brickinfo->real_path)) {
+                    ret = -1;
+                    goto out;
                 }
+                (void)strncpy(brickinfo->real_path, abspath,
+                              sizeof(brickinfo->real_path));
+            }
+        }
 
-                if (brick_mount_path) {
-                        GF_FREE (brick_mount_path);
-                        brick_mount_path = NULL;
-                }
+        if (brick_mount_path) {
+            GF_FREE(brick_mount_path);
+            brick_mount_path = NULL;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && brick_mount_path)
-                GF_FREE (brick_mount_path);
+    if (ret && brick_mount_path)
+        GF_FREE(brick_mount_path);
 
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_resolve_snap_bricks (xlator_t  *this, glusterd_snap_t *snap)
+glusterd_resolve_snap_bricks(xlator_t *this, glusterd_snap_t *snap)
 {
-        int32_t                 ret = -1;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, snap, out);
-
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        ret = glusterd_resolve_brick (brickinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_RESOLVE_BRICK_FAIL,
-                                        "resolve brick failed in restore");
-                                goto out;
-                        }
-                }
+    int32_t ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, snap, out);
+
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            ret = glusterd_resolve_brick(brickinfo);
+            if (ret) {
+                gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
+                         "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
+                         volinfo->volname, brickinfo->path);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                       "resolve brick failed in restore");
+                goto out;
+            }
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_store_update_snap (glusterd_snap_t *snap)
+glusterd_store_update_snap(glusterd_snap_t *snap)
 {
-        int                     ret = -1;
-        char                    *key                    = NULL;
-        char                    *value                  = NULL;
-        char                    snappath[PATH_MAX]      = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_conf_t         *conf                   = NULL;
-        gf_store_iter_t         *iter                   = NULL;
-        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (snap);
-
-        GLUSTERD_GET_SNAP_DIR (snappath, snap, conf);
-
-        snprintf (path, sizeof (path), "%s/%s", snappath,
-                  GLUSTERD_SNAP_INFO_FILE);
-
-        ret = gf_store_handle_retrieve (path, &snap->shandle);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_HANDLE_NULL, "snap handle is NULL");
-                goto out;
-        }
-
-        ret = gf_store_iter_new (snap->shandle, &iter);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_GET_FAIL, "Failed to get new store "
-                        "iter");
-                goto out;
-        }
-
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_GET_FAIL, "Failed to get next store "
-                        "iter");
-                goto out;
-        }
-
-        while (!ret) {
-                gf_msg_debug (this->name, 0, "key = %s value = %s",
-                        key, value);
-
-                if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_ID,
-                                     strlen (GLUSTERD_STORE_KEY_SNAP_ID))) {
-                        ret = gf_uuid_parse (value, snap->snap_id);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_UUID_PARSE_FAIL,
-                                        "Failed to parse uuid");
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
-                                   strlen (GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
-                        snap->snap_restored = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_STATUS,
-                                     strlen (GLUSTERD_STORE_KEY_SNAP_STATUS))) {
-                        snap->snap_status = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_DESC,
-                                     strlen (GLUSTERD_STORE_KEY_SNAP_DESC))) {
-                        snap->description = gf_strdup (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
-                                  strlen (GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
-                        snap->time_stamp = atoi (value);
-                }
-
-                GF_FREE (key);
-                GF_FREE (value);
-                key = NULL;
-                value = NULL;
-
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
-
-        if (op_errno != GD_STORE_EOF)
-                goto out;
-
-        ret = gf_store_iter_destroy (iter);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_ITER_DESTROY_FAIL,
-                        "Failed to destroy store "
-                        "iter");
-        }
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char snappath[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_iter_t *iter = NULL;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(snap);
+
+    GLUSTERD_GET_SNAP_DIR(snappath, snap, conf);
+
+    len = snprintf(path, sizeof(path), "%s/%s", snappath,
+                   GLUSTERD_SNAP_INFO_FILE);
+    if ((len < 0) || (len >= sizeof(path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &snap->shandle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
+               "snap handle is NULL");
+        goto out;
+    }
+
+    ret = gf_store_iter_new(snap->shandle, &iter);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get new store "
+               "iter");
+        goto out;
+    }
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get next store "
+               "iter");
+        goto out;
+    }
+
+    while (!ret) {
+        gf_msg_debug(this->name, 0, "key = %s value = %s", key, value);
+
+        if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_ID,
+                     SLEN(GLUSTERD_STORE_KEY_SNAP_ID))) {
+            ret = gf_uuid_parse(value, snap->snap_id);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
+                       "Failed to parse uuid");
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
+            snap->snap_restored = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_STATUS,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_STATUS))) {
+            snap->snap_status = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_DESC,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_DESC))) {
+            snap->description = gf_strdup(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
+            snap->time_stamp = atoi(value);
+        }
+
+        GF_FREE(key);
+        GF_FREE(value);
+        key = NULL;
+        value = NULL;
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+
+    ret = 0;
 
 out:
-        return ret;
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
+
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_snap (char *snapname)
+glusterd_store_retrieve_snap(char *snapname)
 {
-        int32_t                ret      = -1;
-        glusterd_snap_t       *snap     = NULL;
-        glusterd_conf_t       *priv     = NULL;
-        xlator_t              *this     = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snapname);
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_OBJECT_STORE_FAIL, "Failed to create "
-                        " snap object");
-                goto out;
-        }
-
-        strncpy (snap->snapname, snapname, strlen(snapname));
-        ret = glusterd_store_update_snap (snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAPSHOT_UPDATE_FAIL,
-                        "Failed to update snapshot "
-                        "for %s snap", snapname);
-                goto out;
-        }
-
-        ret = glusterd_store_retrieve_volumes (this, snap);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_VOL_RETRIEVE_FAIL, "Failed to retrieve "
-                        "snap volumes for snap %s", snapname);
-                goto out;
-        }
-
-        /* TODO: list_add_order can do 'N-square' comparisons and
-           is not efficient. Find a better solution to store the snap
-           in order */
-        glusterd_list_add_order (&snap->snap_list, &priv->snapshots,
-                                 glusterd_compare_snap_time);
+    int32_t ret = -1;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snapname);
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+               "Failed to create "
+               " snap object");
+        goto out;
+    }
+
+    if (snprintf(snap->snapname, sizeof(snap->snapname), "%s", snapname) >=
+        sizeof(snap->snapname))
+        goto out;
+    ret = glusterd_store_update_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_UPDATE_FAIL,
+               "Failed to update snapshot "
+               "for %s snap",
+               snapname);
+        goto out;
+    }
+
+    ret = glusterd_store_retrieve_volumes(this, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_VOL_RETRIEVE_FAIL,
+               "Failed to retrieve "
+               "snap volumes for snap %s",
+               snapname);
+        goto out;
+    }
+
+    /* TODO: list_add_order can do 'N-square' comparisons and
+       is not efficient. Find a better solution to store the snap
+       in order */
+    glusterd_list_add_order(&snap->snap_list, &priv->snapshots,
+                            glusterd_compare_snap_time);
 
 out:
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* Read the missed_snap_list and update the in-memory structs */
 int32_t
-glusterd_store_retrieve_missed_snaps_list (xlator_t  *this)
+glusterd_store_retrieve_missed_snaps_list(xlator_t *this)
 {
-        char                   buf[PATH_MAX]    = "";
-        char                   path[PATH_MAX]   = "";
-        char                  *snap_vol_id      = NULL;
-        char                  *missed_node_info = NULL;
-        char                  *brick_path       = NULL;
-        char                  *value            = NULL;
-        char                  *save_ptr         = NULL;
-        FILE                  *fp               = NULL;
-        int32_t                brick_num        = -1;
-        int32_t                snap_op          = -1;
-        int32_t                snap_status      = -1;
-        int32_t                ret              = -1;
-        glusterd_conf_t       *priv             = NULL;
-        gf_store_op_errno_t    store_errno      = GD_STORE_SUCCESS;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Get the path of the missed_snap_list */
-        glusterd_store_missed_snaps_list_path_set (path, sizeof(path));
-
-        fp = fopen (path, "r");
-        if (!fp) {
-                /* If errno is ENOENT then there are no missed snaps yet */
-                if (errno != ENOENT) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_FILE_OP_FAILED,
-                                "Failed to open %s. ",
-                                path);
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_MISSED_SNAP_LIST_EMPTY,
-                                "No missed snaps list.");
-                        ret = 0;
-                }
-                goto out;
+    char path[PATH_MAX] = "";
+    char *snap_vol_id = NULL;
+    char *missed_node_info = NULL;
+    char *brick_path = NULL;
+    char *value = NULL;
+    char *save_ptr = NULL;
+    FILE *fp = NULL;
+    int32_t brick_num = -1;
+    int32_t snap_op = -1;
+    int32_t snap_status = -1;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    gf_store_op_errno_t store_errno = GD_STORE_SUCCESS;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Get the path of the missed_snap_list */
+    glusterd_store_missed_snaps_list_path_set(path, sizeof(path));
+
+    fp = fopen(path, "r");
+    if (!fp) {
+        /* If errno is ENOENT then there are no missed snaps yet */
+        if (errno != ENOENT) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Failed to open %s. ", path);
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_LIST_EMPTY,
+                   "No missed snaps list.");
+            ret = 0;
         }
+        goto out;
+    }
 
-        do {
-                ret = gf_store_read_and_tokenize (fp, buf, sizeof (buf),
-                                                  &missed_node_info, &value,
-                                                  &store_errno);
-                if (ret) {
-                        if (store_errno == GD_STORE_EOF) {
-                                gf_msg_debug (this->name,
-                                        0,
-                                        "EOF for missed_snap_list");
-                                ret = 0;
-                                break;
-                        }
-                        gf_msg (this->name, GF_LOG_ERROR, store_errno,
-                                GD_MSG_MISSED_SNAP_GET_FAIL,
-                                "Failed to fetch data from "
-                                "missed_snaps_list.");
-                        goto out;
-                }
-
-                /* Fetch the brick_num, brick_path, snap_op and snap status */
-                snap_vol_id = strtok_r (value, ":", &save_ptr);
-                brick_num = atoi(strtok_r (NULL, ":", &save_ptr));
-                brick_path = strtok_r (NULL, ":", &save_ptr);
-                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
-                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
-
-                if (!missed_node_info || !brick_path || !snap_vol_id ||
-                    brick_num < 1 || snap_op < 1 ||
-                    snap_status < 1) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                GD_MSG_INVALID_MISSED_SNAP_ENTRY,
-                                "Invalid missed_snap_entry");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_add_new_entry_to_list (missed_node_info,
-                                                      snap_vol_id,
-                                                      brick_num,
-                                                      brick_path,
-                                                      snap_op,
-                                                      snap_status);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
-                                "Failed to store missed snaps_list");
-                        goto out;
-                }
+    do {
+        ret = gf_store_read_and_tokenize(fp, &missed_node_info, &value,
+                                         &store_errno);
+        if (ret) {
+            if (store_errno == GD_STORE_EOF) {
+                gf_msg_debug(this->name, 0, "EOF for missed_snap_list");
+                ret = 0;
+                break;
+            }
+            gf_msg(this->name, GF_LOG_ERROR, store_errno,
+                   GD_MSG_MISSED_SNAP_GET_FAIL,
+                   "Failed to fetch data from "
+                   "missed_snaps_list.");
+            goto out;
+        }
+
+        /* Fetch the brick_num, brick_path, snap_op and snap status */
+        snap_vol_id = strtok_r(value, ":", &save_ptr);
+        brick_num = atoi(strtok_r(NULL, ":", &save_ptr));
+        brick_path = strtok_r(NULL, ":", &save_ptr);
+        snap_op = atoi(strtok_r(NULL, ":", &save_ptr));
+        snap_status = atoi(strtok_r(NULL, ":", &save_ptr));
+
+        if (!missed_node_info || !brick_path || !snap_vol_id || brick_num < 1 ||
+            snap_op < 1 || snap_status < 1) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                   GD_MSG_INVALID_MISSED_SNAP_ENTRY,
+                   "Invalid missed_snap_entry");
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_add_new_entry_to_list(missed_node_info, snap_vol_id,
+                                             brick_num, brick_path, snap_op,
+                                             snap_status);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Failed to store missed snaps_list");
+            goto out;
+        }
 
-        } while (store_errno == GD_STORE_SUCCESS);
+    } while (store_errno == GD_STORE_SUCCESS);
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    if (fp)
+        fclose(fp);
+
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_snaps (xlator_t  *this)
+glusterd_store_retrieve_snaps(xlator_t *this)
 {
-        int32_t                ret              = 0;
-        char                   path[PATH_MAX]   = {0,};
-        glusterd_conf_t       *priv             = NULL;
-        DIR                   *dir              = NULL;
-        struct dirent         *entry            = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        snprintf (path, PATH_MAX, "%s/snaps", priv->workdir);
-
-        dir = sys_opendir (path);
-
-        if (!dir) {
-                /* If snaps dir doesn't exists ignore the error for
-                   backward compatibility */
-                if (errno != ENOENT) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_DIR_OP_FAILED, "Unable to open dir %s",
-                                path);
-                }
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-
-        while (entry) {
-                if (strcmp (entry->d_name, GLUSTERD_MISSED_SNAPS_LIST_FILE)) {
-                        ret = glusterd_store_retrieve_snap (entry->d_name);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_RESTORE_FAIL,
-                                        "Unable to restore snapshot: %s",
-                                        entry->d_name);
-                                goto out;
-                        }
-                }
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
-
-        /* Retrieve missed_snaps_list */
-        ret = glusterd_store_retrieve_missed_snaps_list (this);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                        "Failed to retrieve missed_snaps_list");
-                goto out;
-        }
+    int32_t ret = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    int32_t len = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    len = snprintf(path, PATH_MAX, "%s/snaps", priv->workdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    dir = sys_opendir(path);
+
+    if (!dir) {
+        /* If snaps dir doesn't exists ignore the error for
+           backward compatibility */
+        if (errno != ENOENT) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Unable to open dir %s", path);
+        }
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (strcmp(entry->d_name, GLUSTERD_MISSED_SNAPS_LIST_FILE)) {
+            ret = glusterd_store_retrieve_snap(entry->d_name);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Unable to restore snapshot: %s", entry->d_name);
+                goto out;
+            }
+        }
+    }
+
+    /* Retrieve missed_snaps_list */
+    ret = glusterd_store_retrieve_missed_snaps_list(this);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to retrieve missed_snaps_list");
+        goto out;
+    }
 
 out:
-        if (dir)
-                sys_closedir (dir);
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+    if (dir)
+        sys_closedir(dir);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 /* Writes all the contents of conf->missed_snap_list */
 int32_t
-glusterd_store_write_missed_snapinfo (int32_t fd)
+glusterd_store_write_missed_snapinfo(int32_t fd)
 {
-        char                           key[PATH_MAX]               = "";
-        char                           value[PATH_MAX]             = "";
-        int32_t                        ret                         = -1;
-        glusterd_conf_t               *priv                        = NULL;
-        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
-        glusterd_snap_op_t            *snap_opinfo                 = NULL;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Write the missed_snap_entry */
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        snprintf (key, sizeof(key), "%s:%s",
-                                  missed_snapinfo->node_uuid,
-                                  missed_snapinfo->snap_uuid);
-                        snprintf (value, sizeof(value), "%s:%d:%s:%d:%d",
-                                  snap_opinfo->snap_vol_id,
-                                  snap_opinfo->brick_num,
-                                  snap_opinfo->brick_path,
-                                  snap_opinfo->op, snap_opinfo->status);
-                        ret = gf_store_save_value (fd, key, value);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
-                                        "Failed to write missed snapinfo");
-                                goto out;
-                        }
-                }
-        }
-
-        ret = 0;
+    char key[(UUID_SIZE * 2) + 2];
+    char value[PATH_MAX];
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Write the missed_snap_entry */
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            snprintf(key, sizeof(key), "%s:%s", missed_snapinfo->node_uuid,
+                     missed_snapinfo->snap_uuid);
+            snprintf(value, sizeof(value), "%s:%d:%s:%d:%d",
+                     snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                     snap_opinfo->brick_path, snap_opinfo->op,
+                     snap_opinfo->status);
+            ret = gf_store_save_value(fd, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+                       "Failed to write missed snapinfo");
+                goto out;
+            }
+        }
+    }
+
+    ret = 0;
 out:
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Adds the missed snap entries to the in-memory conf->missed_snap_list *
  * and writes them to disk */
 int32_t
-glusterd_store_update_missed_snaps ()
+glusterd_store_update_missed_snaps()
 {
-        int32_t                        fd                          = -1;
-        int32_t                        ret                         = -1;
-        glusterd_conf_t               *priv                        = NULL;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = glusterd_store_create_missed_snaps_list_shandle_on_absence ();
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MISSED_SNAP_LIST_STORE_HANDLE_GET_FAIL,
-                        "Unable to obtain "
-                        "missed_snaps_list store handle.");
-                goto out;
-        }
-
-        fd = gf_store_mkstemp (priv->missed_snaps_list_shandle);
-        if (fd <= 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Failed to create tmp file");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_store_write_missed_snapinfo (fd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_MISSED_SNAP_CREATE_FAIL,
-                        "Failed to write missed snaps to disk");
-                goto out;
-        }
-
-        ret = gf_store_rename_tmppath (priv->missed_snaps_list_shandle);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_FILE_OP_FAILED,
-                        "Failed to rename the tmp file");
-                goto out;
-        }
+    int32_t fd = -1;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_store_create_missed_snaps_list_shandle_on_absence();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_MISSED_SNAP_LIST_STORE_HANDLE_GET_FAIL,
+               "Unable to obtain "
+               "missed_snaps_list store handle.");
+        goto out;
+    }
+
+    fd = gf_store_mkstemp(priv->missed_snaps_list_shandle);
+    if (fd <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Failed to create tmp file");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_write_missed_snapinfo(fd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+               "Failed to write missed snaps to disk");
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(priv->missed_snaps_list_shandle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Failed to rename the tmp file");
+        goto out;
+    }
 out:
-        if (ret && (fd > 0)) {
-                ret = gf_store_unlink_tmppath (priv->missed_snaps_list_shandle);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_TMP_FILE_UNLINK_FAIL,
-                                "Failed to unlink the tmp file");
-                }
-                ret = -1;
+    if (ret && (fd > 0)) {
+        ret = gf_store_unlink_tmppath(priv->missed_snaps_list_shandle);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TMP_FILE_UNLINK_FAIL,
+                   "Failed to unlink the tmp file");
         }
+        ret = -1;
+    }
 
-        gf_msg_trace (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
+glusterd_store_delete_peerinfo(glusterd_peerinfo_t *peerinfo)
 {
-        int32_t                         ret = -1;
-        glusterd_conf_t                 *priv = NULL;
-        xlator_t                        *this = NULL;
-        char                            peerdir[PATH_MAX] = {0,};
-        char                            filepath[PATH_MAX] = {0,};
-        char                            hostname_path[PATH_MAX] = {0,};
-
-
-        if (!peerinfo) {
-                ret = 0;
-                goto out;
-        }
-
-        this = THIS;
-        priv = this->private;
-
-        snprintf (peerdir, PATH_MAX, "%s/peers", priv->workdir);
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char peerdir[PATH_MAX] = {
+        0,
+    };
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    char hostname_path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    if (!peerinfo) {
+        ret = 0;
+        goto out;
+    }
 
+    this = THIS;
+    priv = this->private;
 
-        if (gf_uuid_is_null (peerinfo->uuid)) {
+    len = snprintf(peerdir, PATH_MAX, "%s/peers", priv->workdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
 
-                if (peerinfo->hostname) {
-                        snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
-                                  peerinfo->hostname);
-                } else {
-                       ret = 0;
-                       goto out;
-                }
+    if (gf_uuid_is_null(peerinfo->uuid)) {
+        if (peerinfo->hostname) {
+            len = snprintf(filepath, PATH_MAX, "%s/%s", peerdir,
+                           peerinfo->hostname);
+            if ((len < 0) || (len >= PATH_MAX)) {
+                goto out;
+            }
         } else {
+            ret = 0;
+            goto out;
+        }
+    } else {
+        len = snprintf(filepath, PATH_MAX, "%s/%s", peerdir,
+                       uuid_utoa(peerinfo->uuid));
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
+        len = snprintf(hostname_path, PATH_MAX, "%s/%s", peerdir,
+                       peerinfo->hostname);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
 
-                snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
-                          uuid_utoa (peerinfo->uuid));
-                snprintf (hostname_path, PATH_MAX, "%s/%s",
-                          peerdir, peerinfo->hostname);
-
-                ret = sys_unlink (hostname_path);
+        ret = sys_unlink(hostname_path);
 
-                if (!ret)
-                        goto out;
-        }
+        if (!ret)
+            goto out;
+    }
 
-        ret = sys_unlink (filepath);
-        if (ret && (errno == ENOENT))
-                ret = 0;
+    ret = sys_unlink(filepath);
+    if (ret && (errno == ENOENT))
+        ret = 0;
 
 out:
-        if (peerinfo && peerinfo->shandle) {
-                        gf_store_handle_destroy (peerinfo->shandle);
-                        peerinfo->shandle = NULL;
-                }
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+    if (peerinfo && peerinfo->shandle) {
+        gf_store_handle_destroy(peerinfo->shandle);
+        peerinfo->shandle = NULL;
+    }
+    gf_msg_debug((this ? this->name : "glusterd"), 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 void
-glusterd_store_peerinfo_dirpath_set (char *path, size_t len)
+glusterd_store_peerinfo_dirpath_set(char *path, size_t len)
 {
-        glusterd_conf_t                 *priv = NULL;
-        GF_ASSERT (path);
-        GF_ASSERT (len >= PATH_MAX);
+    glusterd_conf_t *priv = NULL;
+    GF_ASSERT(path);
+    GF_ASSERT(len >= PATH_MAX);
 
-        priv = THIS->private;
-        snprintf (path, len, "%s/peers", priv->workdir);
+    priv = THIS->private;
+    snprintf(path, len, "%s/peers", priv->workdir);
 }
 
 int32_t
-glusterd_store_create_peer_dir ()
+glusterd_store_create_peer_dir()
 {
-        int32_t                             ret = 0;
-        char                            path[PATH_MAX];
+    int32_t ret = 0;
+    char path[PATH_MAX];
 
-        glusterd_store_peerinfo_dirpath_set (path, sizeof (path));
-        ret = gf_store_mkdir (path);
+    glusterd_store_peerinfo_dirpath_set(path, sizeof(path));
+    ret = gf_store_mkdir(path);
 
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 static void
-glusterd_store_uuid_peerpath_set (glusterd_peerinfo_t *peerinfo, char *peerfpath,
-                             size_t len)
+glusterd_store_uuid_peerpath_set(glusterd_peerinfo_t *peerinfo, char *peerfpath,
+                                 size_t len)
 {
-        char                    peerdir[PATH_MAX];
-        char                    str[50] = {0};
+    char peerdir[PATH_MAX];
+    char str[50] = {0};
 
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (peerfpath);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(peerfpath);
+    GF_ASSERT(len >= PATH_MAX);
 
-        glusterd_store_peerinfo_dirpath_set (peerdir, sizeof (peerdir));
-        gf_uuid_unparse (peerinfo->uuid, str);
-        snprintf (peerfpath, len, "%s/%s", peerdir, str);
+    glusterd_store_peerinfo_dirpath_set(peerdir, sizeof(peerdir));
+    gf_uuid_unparse(peerinfo->uuid, str);
+    snprintf(peerfpath, len, "%s/%s", peerdir, str);
 }
 
 static void
-glusterd_store_hostname_peerpath_set (glusterd_peerinfo_t *peerinfo,
-                                       char *peerfpath, size_t len)
+glusterd_store_hostname_peerpath_set(glusterd_peerinfo_t *peerinfo,
+                                     char *peerfpath, size_t len)
 {
-        char                    peerdir[PATH_MAX];
+    char peerdir[PATH_MAX];
 
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (peerfpath);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(peerfpath);
+    GF_ASSERT(len >= PATH_MAX);
 
-        glusterd_store_peerinfo_dirpath_set (peerdir, sizeof (peerdir));
-        snprintf (peerfpath, len, "%s/%s", peerdir, peerinfo->hostname);
+    glusterd_store_peerinfo_dirpath_set(peerdir, sizeof(peerdir));
+    snprintf(peerfpath, len, "%s/%s", peerdir, peerinfo->hostname);
 }
 
 int32_t
-glusterd_store_peerinfo_hostname_shandle_create (glusterd_peerinfo_t *peerinfo)
+glusterd_store_peerinfo_hostname_shandle_create(glusterd_peerinfo_t *peerinfo)
 {
-        char                    peerfpath[PATH_MAX];
-        int32_t                 ret = -1;
-
-        glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
-                                              sizeof (peerfpath));
-        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
-                                                 peerfpath);
-        return ret;
+    char peerfpath[PATH_MAX];
+    int32_t ret = -1;
+
+    glusterd_store_hostname_peerpath_set(peerinfo, peerfpath,
+                                         sizeof(peerfpath));
+    ret = gf_store_handle_create_on_absence(&peerinfo->shandle, peerfpath);
+    return ret;
 }
 
 int32_t
-glusterd_store_peerinfo_uuid_shandle_create (glusterd_peerinfo_t *peerinfo)
+glusterd_store_peerinfo_uuid_shandle_create(glusterd_peerinfo_t *peerinfo)
 {
-        char                    peerfpath[PATH_MAX];
-        int32_t                 ret = -1;
-
-        glusterd_store_uuid_peerpath_set (peerinfo, peerfpath,
-                                          sizeof (peerfpath));
-        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
-                                                 peerfpath);
-        return ret;
+    char peerfpath[PATH_MAX];
+    int32_t ret = -1;
+
+    glusterd_store_uuid_peerpath_set(peerinfo, peerfpath, sizeof(peerfpath));
+    ret = gf_store_handle_create_on_absence(&peerinfo->shandle, peerfpath);
+    return ret;
 }
 
 int32_t
-glusterd_peerinfo_hostname_shandle_check_destroy (glusterd_peerinfo_t *peerinfo)
+glusterd_peerinfo_hostname_shandle_check_destroy(glusterd_peerinfo_t *peerinfo)
 {
-        char                    peerfpath[PATH_MAX];
-        int32_t                 ret = -1;
-        struct  stat            stbuf = {0,};
-
-        glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
-                                              sizeof (peerfpath));
-        ret = sys_stat (peerfpath, &stbuf);
-        if (!ret) {
-                if (peerinfo->shandle)
-                        gf_store_handle_destroy (peerinfo->shandle);
-                peerinfo->shandle = NULL;
-                ret = sys_unlink (peerfpath);
-        }
-        return ret;
+    char peerfpath[PATH_MAX];
+    int32_t ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+
+    glusterd_store_hostname_peerpath_set(peerinfo, peerfpath,
+                                         sizeof(peerfpath));
+    ret = sys_stat(peerfpath, &stbuf);
+    if (!ret) {
+        if (peerinfo->shandle)
+            gf_store_handle_destroy(peerinfo->shandle);
+        peerinfo->shandle = NULL;
+        ret = sys_unlink(peerfpath);
+    }
+    return ret;
 }
 
 int32_t
-glusterd_store_create_peer_shandle (glusterd_peerinfo_t *peerinfo)
+glusterd_store_create_peer_shandle(glusterd_peerinfo_t *peerinfo)
 {
-        int32_t                 ret = 0;
+    int32_t ret = 0;
 
-        GF_ASSERT (peerinfo);
+    GF_ASSERT(peerinfo);
 
-        if (gf_uuid_is_null (peerinfo->uuid)) {
-                ret = glusterd_store_peerinfo_hostname_shandle_create (peerinfo);
-        } else {
-                ret = glusterd_peerinfo_hostname_shandle_check_destroy (peerinfo);
-                ret = glusterd_store_peerinfo_uuid_shandle_create (peerinfo);
-        }
-        return ret;
+    if (gf_uuid_is_null(peerinfo->uuid)) {
+        ret = glusterd_store_peerinfo_hostname_shandle_create(peerinfo);
+    } else {
+        ret = glusterd_peerinfo_hostname_shandle_check_destroy(peerinfo);
+        ret = glusterd_store_peerinfo_uuid_shandle_create(peerinfo);
+    }
+    return ret;
 }
 
-int32_t
-glusterd_store_peer_write (int fd, glusterd_peerinfo_t *peerinfo)
+static int32_t
+glusterd_store_peer_write(int fd, glusterd_peerinfo_t *peerinfo)
 {
-        char                      buf[50] = {0};
-        int32_t                   ret = 0;
-        int32_t                   i = 1;
-        glusterd_peer_hostname_t *hostname = NULL;
-        char                     *key = NULL;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_UUID,
-                                   uuid_utoa (peerinfo->uuid));
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", peerinfo->state.state);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_STATE, buf);
-        if (ret)
-                goto out;
-
-        cds_list_for_each_entry (hostname, &peerinfo->hostnames,
-                                 hostname_list) {
-                ret = gf_asprintf (&key, GLUSTERD_STORE_KEY_PEER_HOSTNAME"%d",
-                                   i);
-                if (ret < 0)
-                        goto out;
-                ret = gf_store_save_value (fd, key, hostname->hostname);
-                if (ret)
-                        goto out;
-                GF_FREE (key);
-                key = NULL;
-                i++;
-        }
-
+    char buf[PATH_MAX];
+    uint total_len = 0;
+    int32_t ret = 0;
+    int32_t i = 1;
+    glusterd_peer_hostname_t *hostname = NULL;
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n%s=%d\n",
+                   GLUSTERD_STORE_KEY_PEER_UUID, uuid_utoa(peerinfo->uuid),
+                   GLUSTERD_STORE_KEY_PEER_STATE, peerinfo->state.state);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    cds_list_for_each_entry(hostname, &peerinfo->hostnames, hostname_list)
+    {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                       GLUSTERD_STORE_KEY_PEER_HOSTNAME "%d=%s\n", i,
+                       hostname->hostname);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+        i++;
+    }
+
+    ret = gf_store_save_items(fd, buf);
 out:
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_perform_peer_store (glusterd_peerinfo_t *peerinfo)
+glusterd_store_perform_peer_store(glusterd_peerinfo_t *peerinfo)
 {
-        int                                 fd = -1;
-        int32_t                             ret = -1;
+    int fd = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (peerinfo);
+    GF_ASSERT(peerinfo);
 
-        fd = gf_store_mkstemp (peerinfo->shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
+    fd = gf_store_mkstemp(peerinfo->shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_peer_write (fd, peerinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_peer_write(fd, peerinfo);
+    if (ret)
+        goto out;
 
-        ret = gf_store_rename_tmppath (peerinfo->shandle);
+    ret = gf_store_rename_tmppath(peerinfo->shandle);
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (peerinfo->shandle);
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(peerinfo->shandle);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_peerinfo (glusterd_peerinfo_t *peerinfo)
+glusterd_store_peerinfo(glusterd_peerinfo_t *peerinfo)
 {
-        int32_t                         ret = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (peerinfo);
+    GF_ASSERT(peerinfo);
 
-        ret = glusterd_store_create_peer_dir ();
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_peer_dir();
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_create_peer_shandle (peerinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_peer_shandle(peerinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_perform_peer_store (peerinfo);
+    ret = glusterd_store_perform_peer_store(peerinfo);
 out:
-        gf_msg_debug ("glusterd", 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_peers (xlator_t *this)
+glusterd_store_retrieve_peers(xlator_t *this)
 {
-        int32_t                   ret                = 0;
-        glusterd_conf_t          *priv               = NULL;
-        DIR                      *dir                = NULL;
-        struct dirent            *entry              = NULL;
-        char                      path[PATH_MAX]     = {0,};
-        glusterd_peerinfo_t      *peerinfo           = NULL;
-        gf_store_handle_t        *shandle            = NULL;
-        char                      filepath[PATH_MAX] = {0,};
-        gf_store_iter_t          *iter               = NULL;
-        char                     *key                = NULL;
-        char                     *value              = NULL;
-        glusterd_peerctx_args_t   args               = {0};
-        gf_store_op_errno_t       op_errno           = GD_STORE_SUCCESS;
-        glusterd_peer_hostname_t *address            = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                  GLUSTERD_PEER_DIR_PREFIX);
-
-        dir = sys_opendir (path);
-
-        if (!dir) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "Unable to open dir %s", path);
-                ret = -1;
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gf_store_handle_t *shandle = NULL;
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    glusterd_peerctx_args_t args = {0};
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    glusterd_peer_hostname_t *address = NULL;
+    uuid_t tmp_uuid;
+    gf_boolean_t is_ok;
+    int32_t len;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                   GLUSTERD_PEER_DIR_PREFIX);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    dir = sys_opendir(path);
+
+    if (!dir) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Unable to open dir %s", path);
+        ret = -1;
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (gf_uuid_parse(entry->d_name, tmp_uuid) != 0) {
+            gf_log(this->name, GF_LOG_WARNING, "skipping non-peer file %s",
+                   entry->d_name);
+            continue;
+        }
+        is_ok = _gf_false;
+        len = snprintf(filepath, PATH_MAX, "%s/%s", path, entry->d_name);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto next;
+        }
+        ret = gf_store_handle_retrieve(filepath, &shandle);
+        if (ret)
+            goto next;
 
-        while (entry) {
-                snprintf (filepath, PATH_MAX, "%s/%s", path, entry->d_name);
-                ret = gf_store_handle_retrieve (filepath, &shandle);
-                if (ret)
-                        goto out;
+        ret = gf_store_iter_new(shandle, &iter);
+        if (ret)
+            goto next;
 
-                ret = gf_store_iter_new (shandle, &iter);
-                if (ret)
-                        goto out;
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+        if (ret) {
+            goto next;
+        }
 
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-                if (ret)
-                        goto out;
+        /* Create an empty peerinfo object before reading in the
+         * details
+         */
+        peerinfo = glusterd_peerinfo_new(GD_FRIEND_STATE_DEFAULT, NULL, NULL,
+                                         0);
+        if (peerinfo == NULL) {
+            ret = -1;
+            goto next;
+        }
 
-                /* Create an empty peerinfo object before reading in the
-                 * details
-                 */
-                peerinfo = glusterd_peerinfo_new (GD_FRIEND_STATE_DEFAULT, NULL,
-                                                  NULL, 0);
-                if (peerinfo == NULL) {
-                        ret = -1;
-                        goto out;
+        while (!ret) {
+            if (!strncmp(GLUSTERD_STORE_KEY_PEER_UUID, key,
+                         SLEN(GLUSTERD_STORE_KEY_PEER_UUID))) {
+                if (value)
+                    gf_uuid_parse(value, peerinfo->uuid);
+            } else if (!strncmp(GLUSTERD_STORE_KEY_PEER_STATE, key,
+                                SLEN(GLUSTERD_STORE_KEY_PEER_STATE))) {
+                peerinfo->state.state = atoi(value);
+            } else if (!strncmp(GLUSTERD_STORE_KEY_PEER_HOSTNAME, key,
+                                SLEN(GLUSTERD_STORE_KEY_PEER_HOSTNAME))) {
+                ret = gd_add_address_to_peer(peerinfo, value);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
+                           "Could not add address to peer");
                 }
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+                       "Unknown key: %s", key);
+            }
 
-                while (!ret) {
-
-                        if (!strncmp (GLUSTERD_STORE_KEY_PEER_UUID, key,
-                                      strlen (GLUSTERD_STORE_KEY_PEER_UUID))) {
-                                if (value)
-                                        gf_uuid_parse (value, peerinfo->uuid);
-                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_STATE,
-                                    key,
-                                    strlen (GLUSTERD_STORE_KEY_PEER_STATE))) {
-                                peerinfo->state.state = atoi (value);
-                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_HOSTNAME,
-                                   key,
-                                   strlen (GLUSTERD_STORE_KEY_PEER_HOSTNAME))) {
-                                ret = gd_add_address_to_peer (peerinfo, value);
-                        } else {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_UNKNOWN_KEY, "Unknown key: %s",
-                                        key);
-                        }
-
-                        GF_FREE (key);
-                        GF_FREE (value);
-                        key = NULL;
-                        value = NULL;
-
-                        ret = gf_store_iter_get_next (iter, &key, &value,
-                                                      &op_errno);
-                }
-                if (op_errno != GD_STORE_EOF) {
-                        goto out;
-                }
+            GF_FREE(key);
+            GF_FREE(value);
+            key = NULL;
+            value = NULL;
 
-                (void) gf_store_iter_destroy (iter);
+            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+        }
+        if (op_errno != GD_STORE_EOF) {
+            goto next;
+        }
 
-                /* Set first hostname from peerinfo->hostnames to
-                 * peerinfo->hostname
-                 */
-                address = cds_list_entry (peerinfo->hostnames.next,
-                                          glusterd_peer_hostname_t,
-                                          hostname_list);
-                if (!address) {
-                        ret = -1;
-                        goto out;
-                }
-                peerinfo->hostname = gf_strdup (address->hostname);
+        if (gf_uuid_is_null(peerinfo->uuid)) {
+            gf_log("", GF_LOG_ERROR,
+                   "Null UUID while attempting to read peer from '%s'",
+                   filepath);
+            goto next;
+        }
 
-                ret = glusterd_friend_add_from_peerinfo (peerinfo, 1, NULL);
-                if (ret)
-                        goto out;
+        /* Set first hostname from peerinfo->hostnames to
+         * peerinfo->hostname
+         */
+        address = cds_list_entry(peerinfo->hostnames.next,
+                                 glusterd_peer_hostname_t, hostname_list);
+        peerinfo->hostname = gf_strdup(address->hostname);
 
-                peerinfo->shandle = shandle;
-                peerinfo = NULL;
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
+        ret = glusterd_friend_add_from_peerinfo(peerinfo, 1, NULL);
+        if (ret)
+            goto next;
 
-        args.mode = GD_MODE_ON;
+        peerinfo->shandle = shandle;
+        is_ok = _gf_true;
 
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                ret = glusterd_friend_rpc_create (this, peerinfo, &args);
-                if (ret)
-                        break;
+    next:
+        (void)gf_store_iter_destroy(&iter);
+
+        if (!is_ok) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "skipping malformed peer file %s", entry->d_name);
+            if (peerinfo) {
+                glusterd_peerinfo_cleanup(peerinfo);
+            }
         }
-        rcu_read_unlock ();
         peerinfo = NULL;
+    }
+
+    args.mode = GD_MODE_ON;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        ret = glusterd_friend_rpc_create(this, peerinfo, &args);
+        if (ret)
+            break;
+    }
+    RCU_READ_UNLOCK;
+    peerinfo = NULL;
 
 out:
-        if (peerinfo)
-                glusterd_peerinfo_cleanup (peerinfo);
 
-        if (dir)
-                sys_closedir (dir);
-        gf_msg_debug (this->name, 0, "Returning with %d", ret);
+    if (dir)
+        sys_closedir(dir);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 /* Bricks for snap volumes are hosted at /var/run/gluster/snaps
@@ -4194,51 +4699,58 @@ out:
  * paths need to be recreated and re-mounted
  */
 int32_t
-glusterd_recreate_all_snap_brick_mounts (xlator_t  *this)
+glusterd_recreate_all_snap_brick_mounts(xlator_t *this)
 {
-        int32_t                  ret       = 0;
-        glusterd_conf_t         *priv      = NULL;
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_snap_t         *snap      = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Recreate bricks of volumes restored from snaps */
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                /* If the volume is not a restored volume then continue */
-                if (gf_uuid_is_null (volinfo->restored_from_snap))
-                        continue;
-
-                ret = glusterd_recreate_vol_brick_mounts (this, volinfo);
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Recreate bricks of volumes restored from snaps */
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        /* If the volume is not a restored volume then continue */
+        if (gf_uuid_is_null(volinfo->restored_from_snap))
+            continue;
+
+        ret = glusterd_recreate_vol_brick_mounts(this, volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNT_RECREATE_FAIL,
+                   "Failed to recreate brick mounts "
+                   "for %s",
+                   volinfo->volname);
+            goto out;
+        }
+    }
+
+    /* Recreate bricks of snapshot volumes
+     * We are not creating brick mounts for stopped snaps.
+     */
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+        {
+            if (volinfo->status != GLUSTERD_STATUS_STOPPED) {
+                ret = glusterd_recreate_vol_brick_mounts(this, volinfo);
                 if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRK_MNT_RECREATE_FAIL,
-                                "Failed to recreate brick mounts "
-                                "for %s", volinfo->volname);
-                        goto out;
-                }
-        }
-
-        /* Recreate bricks of snapshot volumes */
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                        ret = glusterd_recreate_vol_brick_mounts (this,
-                                                                  volinfo);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_BRK_MNT_RECREATE_FAIL,
-                                        "Failed to recreate brick mounts "
-                                        "for %s", snap->snapname);
-                                goto out;
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRK_MNT_RECREATE_FAIL,
+                           "Failed to recreate brick "
+                           "mounts for %s",
+                           snap->snapname);
+                    goto out;
                 }
+            }
         }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* When the snapshot command from cli is received, the on disk and
@@ -4255,345 +4767,359 @@ out:
  * taken.
  */
 int32_t
-glusterd_snap_cleanup (xlator_t  *this)
+glusterd_snap_cleanup(xlator_t *this)
 {
-        dict_t               *dict = NULL;
-        int32_t               ret  = 0;
-        glusterd_conf_t      *priv = NULL;
-        glusterd_snap_t      *snap = NULL;
-        glusterd_snap_t      *tmp_snap = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        dict = dict_new();
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_DICT_CREATE_FAIL,
-                        "Failed to create dict");
-                ret = -1;
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                if (snap->snap_status == GD_SNAP_STATUS_RESTORED) {
-                        ret = glusterd_snapshot_revert_restore_from_snap (snap);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        GD_MSG_SNAP_RESTORE_REVERT_FAIL,
-                                        "Failed to "
-                                        "revert partially restored snapshot "
-                                        "(%s)", snap->snapname);
-                                goto out;
-                        }
-                } else if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
-                        ret = glusterd_snap_remove (dict, snap,
-                                                    _gf_true, _gf_true,
-                                                    _gf_false);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_SNAP_REMOVE_FAIL,
-                                        "Failed to remove the snapshot %s",
-                                        snap->snapname);
-                                goto out;
-                        }
-                }
-        }
+    dict_t *dict = NULL;
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create dict");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        if (snap->snap_status == GD_SNAP_STATUS_RESTORED) {
+            ret = glusterd_snapshot_revert_restore_from_snap(snap);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_RESTORE_REVERT_FAIL,
+                       "Failed to "
+                       "revert partially restored snapshot "
+                       "(%s)",
+                       snap->snapname);
+                goto out;
+            }
+        } else if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
+            ret = glusterd_snap_remove(dict, snap, _gf_true, _gf_true,
+                                       _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to remove the snapshot %s", snap->snapname);
+                goto out;
+            }
+        }
+    }
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_resolve_all_bricks (xlator_t  *this)
+glusterd_resolve_all_bricks(xlator_t *this)
 {
-        int32_t                 ret        = 0;
-        glusterd_conf_t         *priv      = NULL;
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_snap_t         *snap      = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        /* Resolve bricks of volumes */
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        ret = glusterd_resolve_brick (brickinfo);
-                        if (ret) {
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_RESOLVE_BRICK_FAIL,
-                                        "resolve brick failed in restore");
-                                goto out;
-                        }
-                }
-        }
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_snap_t *snap = NULL;
 
-        /* Resolve bricks of snapshot volumes */
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                ret = glusterd_resolve_snap_bricks (this, snap);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_SNAP_RESOLVE_BRICK_FAIL,
-                                "resolving the snap bricks"
-                                " failed for snap: %s",
-                                snap->snapname);
-                        goto out;
-                }
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    /* Resolve bricks of volumes */
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            ret = glusterd_resolve_brick(brickinfo);
+            if (ret) {
+                gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
+                         "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
+                         volinfo->volname, brickinfo->path);
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                       "Failed to resolve brick %s with host %s of volume %s"
+                       " in restore",
+                       brickinfo->path, brickinfo->hostname, volinfo->volname);
+                goto out;
+            }
+        }
+    }
+
+    /* Resolve bricks of snapshot volumes */
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        ret = glusterd_resolve_snap_bricks(this, snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESOLVE_BRICK_FAIL,
+                   "resolving the snap bricks"
+                   " failed for snap: %s",
+                   snap->snapname);
+            goto out;
         }
+    }
 
 out:
-        gf_msg_trace (this->name, 0, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_restore ()
+glusterd_restore()
 {
-        int32_t         ret = -1;
-        xlator_t        *this = NULL;
-
-        this = THIS;
-
-        ret = glusterd_store_retrieve_volumes (this, NULL);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_retrieve_peers (this);
-        if (ret)
-                goto out;
-
-        /* While retrieving snapshots, if the snapshot status
-           is not GD_SNAP_STATUS_IN_USE, then the snapshot is
-           cleaned up. To do that, the snap volume has to be
-           stopped by stopping snapshot volume's bricks. And for
-           that the snapshot bricks should be resolved. But without
-           retrieving the peers, resolving bricks will fail. So
-           do retrieving of snapshots after retrieving peers.
-        */
-        ret = glusterd_store_retrieve_snaps (this);
-        if (ret)
-                goto out;
-
-        ret = glusterd_resolve_all_bricks (this);
-        if (ret)
-                goto out;
-
-        ret = glusterd_snap_cleanup (this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_CLEANUP_FAIL, "Failed to perform "
-                        "a cleanup of the snapshots");
-                goto out;
-        }
-
-        ret = glusterd_recreate_all_snap_brick_mounts (this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SNAP_BRK_MNT_RECREATE_FAIL, "Failed to recreate "
-                        "all snap brick mounts");
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    ret = glusterd_options_init(this);
+    if (ret < 0)
+        goto out;
+
+    ret = glusterd_store_retrieve_volumes(this, NULL);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_retrieve_peers(this);
+    if (ret)
+        goto out;
+
+    /* While retrieving snapshots, if the snapshot status
+       is not GD_SNAP_STATUS_IN_USE, then the snapshot is
+       cleaned up. To do that, the snap volume has to be
+       stopped by stopping snapshot volume's bricks. And for
+       that the snapshot bricks should be resolved. But without
+       retrieving the peers, resolving bricks will fail. So
+       do retrieving of snapshots after retrieving peers.
+    */
+    ret = glusterd_store_retrieve_snaps(this);
+    if (ret)
+        goto out;
+
+    ret = glusterd_resolve_all_bricks(this);
+    if (ret)
+        goto out;
+
+    ret = glusterd_snap_cleanup(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CLEANUP_FAIL,
+               "Failed to perform "
+               "a cleanup of the snapshots");
+        goto out;
+    }
+
+    ret = glusterd_recreate_all_snap_brick_mounts(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_BRK_MNT_RECREATE_FAIL,
+               "Failed to recreate "
+               "all snap brick mounts");
+        goto out;
+    }
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_store_retrieve_quota_version (glusterd_volinfo_t *volinfo)
+glusterd_store_retrieve_quota_version(glusterd_volinfo_t *volinfo)
 {
-        int                 ret                  = -1;
-        uint32_t            version              = 0;
-        char                cksum_path[PATH_MAX] = {0,};
-        char                path[PATH_MAX]       = {0,};
-        char               *version_str          = NULL;
-        char               *tmp                  = NULL;
-        xlator_t           *this                 = NULL;
-        glusterd_conf_t    *conf                 = NULL;
-        gf_store_handle_t  *handle               = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
-        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
-                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
-
-        ret = gf_store_handle_new (cksum_path, &handle);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_STORE_HANDLE_GET_FAIL,
-                        "Unable to get store handle "
-                        "for %s", cksum_path);
-                goto out;
-        }
-
-        ret = gf_store_retrieve_value (handle, "version", &version_str);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Version absent");
-                ret = 0;
-                goto out;
-        }
-
-        version = strtoul (version_str, &tmp, 10);
-	if ((errno == ERANGE) || (errno == EINVAL)) {
-                gf_msg_debug (this->name, 0, "Invalid version number");
-                goto out;
-        }
-        volinfo->quota_conf_version = version;
+    int ret = -1;
+    uint32_t version = 0;
+    char cksum_path[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *version_str = NULL;
+    char *tmp = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, conf);
+    len = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
+                   GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+    if ((len < 0) || (len >= sizeof(cksum_path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_new(cksum_path, &handle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
+               "Unable to get store handle "
+               "for %s",
+               cksum_path);
+        goto out;
+    }
+
+    ret = gf_store_retrieve_value(handle, "version", &version_str);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Version absent");
         ret = 0;
+        goto out;
+    }
+
+    version = strtoul(version_str, &tmp, 10);
+    if ((errno == ERANGE) || (errno == EINVAL)) {
+        gf_msg_debug(this->name, 0, "Invalid version number");
+        goto out;
+    }
+    volinfo->quota_conf_version = version;
+    ret = 0;
 
 out:
-        if (version_str)
-                GF_FREE (version_str);
-        gf_store_handle_destroy (handle);
-        return ret;
+    if (version_str)
+        GF_FREE(version_str);
+    gf_store_handle_destroy(handle);
+    return ret;
 }
 
 int
-glusterd_store_save_quota_version_and_cksum (glusterd_volinfo_t *volinfo)
+glusterd_store_save_quota_version_and_cksum(glusterd_volinfo_t *volinfo)
 {
-        gf_store_handle_t               *shandle = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        xlator_t                        *this = NULL;
-        char                            path[PATH_MAX] = {0};
-        char                            cksum_path[PATH_MAX] = {0,};
-        char                            buf[256] = {0};
-        int                             fd = -1;
-        int32_t                         ret = -1;
-
-        this = THIS;
-        conf = this->private;
-
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
-        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
-                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
-
-        ret = gf_store_handle_new (cksum_path, &shandle);
-        if (ret)
-                goto out;
-
-        fd = gf_store_mkstemp (shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (buf, sizeof (buf)-1, "%u", volinfo->quota_conf_cksum);
-        ret = gf_store_save_value (fd, "cksum", buf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CKSUM_STORE_FAIL, "Failed to store cksum");
-                goto out;
-        }
-
-        memset (buf, 0, sizeof (buf));
-        snprintf (buf, sizeof (buf)-1, "%u", volinfo->quota_conf_version);
-        ret = gf_store_save_value (fd, "version", buf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VERS_STORE_FAIL, "Failed to store version");
-                goto out;
-        }
-
-        ret = gf_store_rename_tmppath (shandle);
-        if (ret)
-                goto out;
+    gf_store_handle_t *shandle = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    char path[PATH_MAX] = {0};
+    char cksum_path[PATH_MAX + 32] = {
+        0,
+    };
+    char buf[64] = {0};
+    int fd = -1;
+    int32_t ret = -1;
+    int32_t len = 0;
+
+    this = THIS;
+    conf = this->private;
+
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, conf);
+    len = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
+                   GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+    if ((len < 0) || (len >= sizeof(cksum_path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_new(cksum_path, &shandle);
+    if (ret)
+        goto out;
+
+    fd = gf_store_mkstemp(shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "cksum=%u\nversion=%u\n",
+                   volinfo->quota_conf_cksum, volinfo->quota_conf_version);
+    if (ret < 0 || ret >= sizeof(buf)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_STORE_FAIL,
+               "Failed to store quota cksum and version");
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(shandle);
+    if (ret)
+        goto out;
 
 out:
-        if ((ret < 0) && (fd > 0))
-                gf_store_unlink_tmppath (shandle);
-        gf_store_handle_destroy (shandle);
-        return ret;
+    if ((ret < 0) && (fd > 0))
+        gf_store_unlink_tmppath(shandle);
+    gf_store_handle_destroy(shandle);
+    return ret;
 }
 
 int32_t
-glusterd_quota_conf_write_header (int fd)
+glusterd_quota_conf_write_header(int fd)
 {
-        int                 header_len    = 0;
-        int                 ret           = -1;
-        xlator_t           *this          = NULL;
-        glusterd_conf_t    *conf          = NULL;
+    int header_len = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("quota", this, out);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("quota", this, out);
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
+    if (conf->op_version < GD_OP_VERSION_3_7_0) {
+        header_len = SLEN(QUOTA_CONF_HEADER_1_1);
+        ret = gf_nwrite(fd, QUOTA_CONF_HEADER_1_1, header_len);
+    } else {
+        header_len = SLEN(QUOTA_CONF_HEADER);
+        ret = gf_nwrite(fd, QUOTA_CONF_HEADER, header_len);
+    }
 
-        if (conf->op_version < GD_OP_VERSION_3_7_0) {
-                header_len = strlen (QUOTA_CONF_HEADER_1_1);
-                ret = gf_nwrite (fd, QUOTA_CONF_HEADER_1_1, header_len);
-        } else {
-                header_len = strlen (QUOTA_CONF_HEADER);
-                ret = gf_nwrite (fd, QUOTA_CONF_HEADER, header_len);
-        }
+    if (ret != header_len) {
+        ret = -1;
+        goto out;
+    }
 
-        if (ret != header_len) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret < 0)
-                gf_msg_callingfn ("quota", GF_LOG_ERROR, 0,
-                                  GD_MSG_QUOTA_CONF_WRITE_FAIL,
-                                  "failed to write "
-                                  "header to a quota conf");
+    if (ret < 0)
+        gf_msg_callingfn("quota", GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                         "failed to write "
+                         "header to a quota conf");
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_quota_conf_write_gfid (int fd, void *buf, char type)
+glusterd_quota_conf_write_gfid(int fd, void *buf, char type)
 {
-        int                 ret        = -1;
-        xlator_t           *this       = NULL;
-        glusterd_conf_t    *conf       = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("quota", this, out);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("quota", this, out);
 
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        ret = gf_nwrite (fd, buf, 16);
-        if (ret != 16) {
-                ret = -1;
-                goto out;
-        }
+    ret = gf_nwrite(fd, buf, 16);
+    if (ret != 16) {
+        ret = -1;
+        goto out;
+    }
 
-        if (conf->op_version >= GD_OP_VERSION_3_7_0) {
-                ret = gf_nwrite (fd, &type, 1);
-                if (ret != 1) {
-                        ret = -1;
-                        goto out;
-                }
+    if (conf->op_version >= GD_OP_VERSION_3_7_0) {
+        ret = gf_nwrite(fd, &type, 1);
+        if (ret != 1) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret < 0)
-                gf_msg_callingfn ("quota", GF_LOG_ERROR, 0,
-                                  GD_MSG_QUOTA_CONF_WRITE_FAIL,
-                                  "failed to write "
-                                  "gfid %s to a quota conf", uuid_utoa (buf));
+    if (ret < 0)
+        gf_msg_callingfn("quota", GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                         "failed to write "
+                         "gfid %s to a quota conf",
+                         uuid_utoa(buf));
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index ec293b64e82..83f4df0783e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -11,178 +11,206 @@
 #define _GLUSTERD_HA_H_
 
 #include <pthread.h>
-#include "compat-uuid.h"
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "run.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
+#include <glusterfs/compat-uuid.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/run.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "glusterd.h"
 #include "rpcsvc.h"
 
-typedef enum glusterd_store_ver_ac_{
-        GLUSTERD_VOLINFO_VER_AC_NONE = 0,
-        GLUSTERD_VOLINFO_VER_AC_INCREMENT = 1,
-        GLUSTERD_VOLINFO_VER_AC_DECREMENT = 2,
+typedef enum glusterd_store_ver_ac_ {
+    GLUSTERD_VOLINFO_VER_AC_NONE = 0,
+    GLUSTERD_VOLINFO_VER_AC_INCREMENT = 1,
+    GLUSTERD_VOLINFO_VER_AC_DECREMENT = 2,
 } glusterd_volinfo_ver_ac_t;
 
-
-#define GLUSTERD_STORE_UUID_KEY                 "UUID"
-
-#define GLUSTERD_STORE_KEY_VOL_TYPE             "type"
-#define GLUSTERD_STORE_KEY_VOL_COUNT            "count"
-#define GLUSTERD_STORE_KEY_VOL_STATUS           "status"
-#define GLUSTERD_STORE_KEY_VOL_PORT             "port"
-#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT        "sub_count"
-#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT       "stripe_count"
-#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT      "replica_count"
-#define GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT     "disperse_count"
-#define GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT   "redundancy_count"
-#define GLUSTERD_STORE_KEY_VOL_ARBITER_CNT      "arbiter_count"
-#define GLUSTERD_STORE_KEY_VOL_BRICK            "brick"
-#define GLUSTERD_STORE_KEY_VOL_VERSION          "version"
-#define GLUSTERD_STORE_KEY_VOL_TRANSPORT        "transport-type"
-#define GLUSTERD_STORE_KEY_VOL_ID               "volume-id"
-#define GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP    "restored_from_snap"
-#define GLUSTERD_STORE_KEY_RB_STATUS            "rb_status"
-#define GLUSTERD_STORE_KEY_RB_SRC_BRICK         "rb_src"
-#define GLUSTERD_STORE_KEY_RB_DST_BRICK         "rb_dst"
-#define GLUSTERD_STORE_KEY_RB_DST_PORT          "rb_port"
-#define GLUSTERD_STORE_KEY_VOL_DEFRAG           "rebalance_status"
-#define GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS    "status"
-#define GLUSTERD_STORE_KEY_DEFRAG_OP            "rebalance_op"
-#define GLUSTERD_STORE_KEY_USERNAME             "username"
-#define GLUSTERD_STORE_KEY_PASSWORD             "password"
-#define GLUSTERD_STORE_KEY_PARENT_VOLNAME       "parent_volname"
-#define GLUSTERD_STORE_KEY_VOL_OP_VERSION       "op-version"
+#define UUID_SIZE 36
+#define VOLINFO_BUFFER_SIZE 4093
+#define GLUSTERD_STORE_UUID_KEY "UUID"
+
+#define GLUSTERD_STORE_KEY_VOL_TYPE "type"
+#define GLUSTERD_STORE_KEY_VOL_COUNT "count"
+#define GLUSTERD_STORE_KEY_VOL_STATUS "status"
+#define GLUSTERD_STORE_KEY_VOL_PORT "port"
+#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT "sub_count"
+#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT "stripe_count"
+#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT "replica_count"
+#define GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT "disperse_count"
+#define GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT "redundancy_count"
+#define GLUSTERD_STORE_KEY_VOL_ARBITER_CNT "arbiter_count"
+#define GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT "thin_arbiter_count"
+#define GLUSTERD_STORE_KEY_VOL_BRICK "brick"
+#define GLUSTERD_STORE_KEY_VOL_TA_BRICK "ta-brick"
+#define GLUSTERD_STORE_KEY_VOL_VERSION "version"
+#define GLUSTERD_STORE_KEY_VOL_TRANSPORT "transport-type"
+#define GLUSTERD_STORE_KEY_VOL_ID "volume-id"
+#define GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP "restored_from_snap"
+#define GLUSTERD_STORE_KEY_RB_STATUS "rb_status"
+#define GLUSTERD_STORE_KEY_RB_SRC_BRICK "rb_src"
+#define GLUSTERD_STORE_KEY_RB_DST_BRICK "rb_dst"
+#define GLUSTERD_STORE_KEY_RB_DST_PORT "rb_port"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG "rebalance_status"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS "status"
+#define GLUSTERD_STORE_KEY_DEFRAG_OP "rebalance_op"
+#define GLUSTERD_STORE_KEY_USERNAME "username"
+#define GLUSTERD_STORE_KEY_PASSWORD "password"
+#define GLUSTERD_STORE_KEY_PARENT_VOLNAME "parent_volname"
+#define GLUSTERD_STORE_KEY_VOL_OP_VERSION "op-version"
 #define GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION "client-op-version"
-#define GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION    "quota-version"
-
-#define GLUSTERD_STORE_KEY_COLD_TYPE            "cold_type"
-#define GLUSTERD_STORE_KEY_COLD_COUNT           "cold_count"
-#define GLUSTERD_STORE_KEY_COLD_REPLICA_COUNT   "cold_replica_count"
-#define GLUSTERD_STORE_KEY_COLD_DISPERSE_COUNT  "cold_disperse_count"
-#define GLUSTERD_STORE_KEY_COLD_REDUNDANCY_COUNT  "cold_redundancy_count"
-#define GLUSTERD_STORE_KEY_HOT_TYPE             "hot_type"
-#define GLUSTERD_STORE_KEY_HOT_COUNT            "hot_count"
-#define GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT    "hot_replica_count"
-
-#define GLUSTERD_STORE_KEY_SNAP_NAME            "name"
-#define GLUSTERD_STORE_KEY_SNAP_ID              "snap-id"
-#define GLUSTERD_STORE_KEY_SNAP_DESC            "desc"
-#define GLUSTERD_STORE_KEY_SNAP_TIMESTAMP       "time-stamp"
-#define GLUSTERD_STORE_KEY_SNAP_STATUS          "status"
-#define GLUSTERD_STORE_KEY_SNAP_RESTORED        "snap-restored"
-#define GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT  "snap-max-hard-limit"
-#define GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE     "auto-delete"
-#define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT  "snap-max-soft-limit"
-#define GLUSTERD_STORE_KEY_SNAPD_PORT           "snapd-port"
-#define GLUSTERD_STORE_KEY_SNAP_ACTIVATE        "snap-activate-on-create"
-#define GLUSTERD_STORE_KEY_GANESHA_GLOBAL       "nfs-ganesha"
-
-#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME       "hostname"
-#define GLUSTERD_STORE_KEY_BRICK_PATH           "path"
-#define GLUSTERD_STORE_KEY_BRICK_REAL_PATH      "real_path"
-#define GLUSTERD_STORE_KEY_BRICK_PORT           "listen-port"
-#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT      "rdma.listen-port"
+#define GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION "quota-version"
+
+#define GLUSTERD_STORE_KEY_SNAP_NAME "name"
+#define GLUSTERD_STORE_KEY_SNAP_ID "snap-id"
+#define GLUSTERD_STORE_KEY_SNAP_DESC "desc"
+#define GLUSTERD_STORE_KEY_SNAP_TIMESTAMP "time-stamp"
+#define GLUSTERD_STORE_KEY_SNAP_STATUS "status"
+#define GLUSTERD_STORE_KEY_SNAP_RESTORED "snap-restored"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT "snap-max-hard-limit"
+#define GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE "auto-delete"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT "snap-max-soft-limit"
+#define GLUSTERD_STORE_KEY_SNAPD_PORT "snapd-port"
+#define GLUSTERD_STORE_KEY_SNAP_ACTIVATE "snap-activate-on-create"
+
+#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
+#define GLUSTERD_STORE_KEY_BRICK_PATH "path"
+#define GLUSTERD_STORE_KEY_BRICK_REAL_PATH "real_path"
+#define GLUSTERD_STORE_KEY_BRICK_PORT "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
 #define GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED "decommissioned"
-#define GLUSTERD_STORE_KEY_BRICK_VGNAME         "vg"
-#define GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH    "device_path"
-#define GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR      "mount_dir"
-#define GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS    "snap-status"
-#define GLUSTERD_STORE_KEY_BRICK_FSTYPE         "fs-type"
-#define GLUSTERD_STORE_KEY_BRICK_MNTOPTS        "mnt-opts"
-#define GLUSTERD_STORE_KEY_BRICK_ID             "brick-id"
-
-#define GLUSTERD_STORE_KEY_PEER_UUID            "uuid"
-#define GLUSTERD_STORE_KEY_PEER_HOSTNAME        "hostname"
-#define GLUSTERD_STORE_KEY_PEER_STATE           "state"
+#define GLUSTERD_STORE_KEY_BRICK_VGNAME "vg"
+#define GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH "device_path"
+#define GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR "mount_dir"
+#define GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS "snap-status"
+#define GLUSTERD_STORE_KEY_BRICK_FSTYPE "fs-type"
+#define GLUSTERD_STORE_KEY_BRICK_MNTOPTS "mnt-opts"
+#define GLUSTERD_STORE_KEY_BRICK_ID "brick-id"
+#define GLUSTERD_STORE_KEY_BRICK_FSID "brick-fsid"
+#define GLUSTERD_STORE_KEY_BRICK_UUID "uuid"
+
+#define GLUSTERD_STORE_KEY_PEER_UUID "uuid"
+#define GLUSTERD_STORE_KEY_PEER_HOSTNAME "hostname"
+#define GLUSTERD_STORE_KEY_PEER_STATE "state"
+#define GLUSTERD_STORE_KEY_VOL_CAPS "caps" /* left just for backward compat */
+
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES "rebalanced-files"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE "size"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED "scanned"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES "failures"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED "skipped"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME "run-time"
+
+#define GLUSTERD_STORE_KEY_VOL_MIGRATED_FILES "migrated-files"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATED_SIZE "migration-size"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATIONS_SCANNED "migration-scanned"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATIONS_FAILURES "migration-failures"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATIONS_SKIPPED "migration-skipped"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATION_RUN_TIME "migration-run-time"
+
+#define GLUSTERD_STORE_KEY_GANESHA_GLOBAL "nfs-ganesha"
 
-#define GLUSTERD_STORE_KEY_VOL_CAPS             "caps"
+/*
+ * The structure is responsible for handling the parameter for writes into
+ * the buffer before it is finally written to the file. The writes will be
+ * of the form of key-value pairs.
+ */
+struct glusterd_volinfo_data_store_ {
+    gf_store_handle_t *shandle; /*Contains fd and path of the file */
+    int16_t buffer_len;
+    char key_check; /* flag to check if key is to be validated before write*/
+    char buffer[VOLINFO_BUFFER_SIZE];
+};
+typedef struct glusterd_volinfo_data_store_ glusterd_volinfo_data_store_t;
 
 int32_t
-glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac);
+glusterd_store_volinfo(glusterd_volinfo_t *volinfo,
+                       glusterd_volinfo_ver_ac_t ac);
 
 int32_t
-glusterd_store_delete_volume (glusterd_volinfo_t *volinfo);
+glusterd_store_delete_volume(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_delete_snap (glusterd_snap_t *snap);
+glusterd_store_delete_snap(glusterd_snap_t *snap);
 
 int32_t
-glusterd_retrieve_uuid ();
+glusterd_retrieve_uuid();
 
 int32_t
-glusterd_store_peerinfo (glusterd_peerinfo_t *peerinfo);
+glusterd_store_peerinfo(glusterd_peerinfo_t *peerinfo);
 
 int32_t
-glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo);
+glusterd_store_delete_peerinfo(glusterd_peerinfo_t *peerinfo);
 
 int32_t
-glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo,
-                             char *delete_path);
+glusterd_store_delete_brick(glusterd_brickinfo_t *brickinfo, char *delete_path);
 
 int32_t
-glusterd_restore ();
+glusterd_restore();
 
 void
-glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
-                                         glusterd_volinfo_ver_ac_t ac);
+glusterd_perform_volinfo_version_action(glusterd_volinfo_t *volinfo,
+                                        glusterd_volinfo_ver_ac_t ac);
 gf_boolean_t
-glusterd_store_is_valid_brickpath (char *volname, char *brick);
+glusterd_store_is_valid_brickpath(char *volname, char *brick);
 
 int32_t
-glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo);
+glusterd_store_perform_node_state_store(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_retrieve_op_version (xlator_t *this, int *op_version);
+glusterd_retrieve_op_version(xlator_t *this, int *op_version);
 
 int
-glusterd_store_global_info (xlator_t *this);
+glusterd_retrieve_max_op_version(xlator_t *this, int *op_version);
+
+int
+glusterd_store_max_op_version(xlator_t *this);
+
+int
+glusterd_store_global_info(xlator_t *this);
 
 int32_t
-glusterd_store_retrieve_options (xlator_t *this);
+glusterd_store_retrieve_options(xlator_t *this);
 
 int32_t
-glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo);
+glusterd_store_retrieve_bricks(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_options (xlator_t *this, dict_t *opts);
+glusterd_store_options(xlator_t *this, dict_t *opts);
 
 void
-glusterd_replace_slash_with_hyphen (char *str);
-
-int32_t
-glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo);
+glusterd_replace_slash_with_hyphen(char *str);
 
 int32_t
-glusterd_store_create_quota_conf_sh_on_absence (glusterd_volinfo_t *volinfo);
+glusterd_store_create_quota_conf_sh_on_absence(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_store_retrieve_quota_version (glusterd_volinfo_t *volinfo);
+glusterd_store_retrieve_quota_version(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_store_save_quota_version_and_cksum (glusterd_volinfo_t *volinfo);
+glusterd_store_save_quota_version_and_cksum(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_snap (glusterd_snap_t *snap);
+glusterd_store_snap(glusterd_snap_t *snap);
 
 int32_t
-glusterd_store_update_missed_snaps ();
+glusterd_store_update_missed_snaps();
 
-glusterd_volinfo_t*
-glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap);
+glusterd_volinfo_t *
+glusterd_store_retrieve_volume(char *volname, glusterd_snap_t *snap);
 
 int
-glusterd_restore_op_version (xlator_t *this);
+glusterd_restore_op_version(xlator_t *this);
+
+int32_t
+glusterd_quota_conf_write_header(int fd);
 
 int32_t
-glusterd_quota_conf_write_header (int fd);
+glusterd_quota_conf_write_gfid(int fd, void *buf, char type);
 
 int32_t
-glusterd_quota_conf_write_gfid (int fd, void *buf, char type);
+glusterd_recreate_vol_brick_mounts(xlator_t *this, glusterd_volinfo_t *volinfo);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
index 44ee6d08d68..ca845903c4f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
@@ -7,245 +7,1041 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
+#include <signal.h>
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glusterd-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-shd-svc.h"
 #include "glusterd-quotad-svc.h"
+#ifdef BUILD_GNFS
 #include "glusterd-nfs-svc.h"
+#endif
 #include "glusterd-bitd-svc.h"
+#include "glusterd-shd-svc-helper.h"
 #include "glusterd-scrub-svc.h"
 #include "glusterd-svc-helper.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
+#include "glusterd-snapshot-utils.h"
 
 int
-glusterd_svcs_reconfigure ()
+glusterd_svcs_reconfigure(glusterd_volinfo_t *volinfo)
 {
-        int              ret  = 0;
-        xlator_t        *this = THIS;
-        glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    char *svc_name = NULL;
 
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        conf = this->private;
-        GF_ASSERT (conf);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        ret = glusterd_nfssvc_reconfigure ();
+#ifdef BUILD_GNFS
+    svc_name = "nfs";
+    ret = glusterd_nfssvc_reconfigure();
+    if (ret)
+        goto out;
+#endif
+    svc_name = "self-heald";
+    if (volinfo) {
+        ret = glusterd_shdsvc_reconfigure(volinfo);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        ret = glusterd_shdsvc_reconfigure ();
-        if (ret)
-                goto out;
+    if (conf->op_version == GD_OP_VERSION_MIN)
+        goto out;
 
-        if (conf->op_version == GD_OP_VERSION_MIN)
-                goto out;
+    svc_name = "quotad";
+    ret = glusterd_quotadsvc_reconfigure();
+    if (ret)
+        goto out;
 
-        ret = glusterd_quotadsvc_reconfigure ();
-        if (ret)
-                goto out;
+    svc_name = "bitd";
+    ret = glusterd_bitdsvc_reconfigure();
+    if (ret)
+        goto out;
 
-        ret = glusterd_bitdsvc_reconfigure ();
-        if (ret)
-                goto out;
-
-        ret = glusterd_scrubsvc_reconfigure ();
-        if (ret)
-                goto out;
+    svc_name = "scrubber";
+    ret = glusterd_scrubsvc_reconfigure();
 out:
-        return ret;
+    if (ret && svc_name)
+        gf_event(EVENT_SVC_RECONFIGURE_FAILED, "svc_name=%s", svc_name);
+    return ret;
 }
 
 int
-glusterd_svcs_stop ()
+glusterd_svcs_stop(glusterd_volinfo_t *volinfo)
 {
-        int              ret  = 0;
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        ret = glusterd_svc_stop (&(priv->nfs_svc), SIGKILL);
-        if (ret)
-                goto out;
+#ifdef BUILD_GNFS
+    ret = priv->nfs_svc.stop(&(priv->nfs_svc), SIGKILL);
+    if (ret)
+        goto out;
+#endif
+    ret = priv->quotad_svc.stop(&(priv->quotad_svc), SIGTERM);
+    if (ret)
+        goto out;
 
-        ret = glusterd_svc_stop (&(priv->shd_svc), SIGTERM);
+    if (volinfo) {
+        ret = volinfo->shd.svc.stop(&(volinfo->shd.svc), SIGTERM);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        ret = glusterd_svc_stop (&(priv->quotad_svc), SIGTERM);
-        if (ret)
-                goto out;
+    ret = priv->bitd_svc.stop(&(priv->bitd_svc), SIGTERM);
+    if (ret)
+        goto out;
 
-        ret = glusterd_svc_stop (&(priv->bitd_svc), SIGTERM);
-        if (ret)
-                goto out;
-
-        ret = glusterd_svc_stop (&(priv->scrub_svc), SIGTERM);
+    ret = priv->scrub_svc.stop(&(priv->scrub_svc), SIGTERM);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_svcs_manager (glusterd_volinfo_t *volinfo)
+glusterd_svcs_manager(glusterd_volinfo_t *volinfo)
 {
-        int              ret  = 0;
-        xlator_t        *this = THIS;
-        glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
 
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        conf = this->private;
-        GF_ASSERT (conf);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        if (volinfo && volinfo->is_snap_volume)
-                return 0;
+    if (volinfo && volinfo->is_snap_volume)
+        return 0;
 
-        ret = conf->nfs_svc.manager (&(conf->nfs_svc), NULL,
-                                     PROC_START_NO_WAIT);
-        if (ret)
-                goto out;
+#if BUILD_GNFS
+    ret = conf->nfs_svc.manager(&(conf->nfs_svc), NULL, PROC_START_NO_WAIT);
+    if (ret)
+        goto out;
+#endif
+    if (conf->op_version == GD_OP_VERSION_MIN)
+        goto out;
 
-        ret = conf->shd_svc.manager (&(conf->shd_svc), volinfo,
-                                     PROC_START_NO_WAIT);
-        if (ret == -EINVAL)
-                ret = 0;
-        if (ret)
-                goto out;
+    ret = conf->quotad_svc.manager(&(conf->quotad_svc), volinfo,
+                                   PROC_START_NO_WAIT);
+    if (ret == -EINVAL)
+        ret = 0;
+    if (ret)
+        goto out;
 
-        if (conf->op_version == GD_OP_VERSION_MIN)
-                goto out;
+    ret = conf->bitd_svc.manager(&(conf->bitd_svc), NULL, PROC_START_NO_WAIT);
+    if (ret == -EINVAL)
+        ret = 0;
+    if (ret)
+        goto out;
 
-        ret = conf->quotad_svc.manager (&(conf->quotad_svc), volinfo,
-                                        PROC_START_NO_WAIT);
+    if (volinfo) {
+        ret = volinfo->shd.svc.manager(&(volinfo->shd.svc), volinfo,
+                                       PROC_START_NO_WAIT);
         if (ret == -EINVAL)
-                ret = 0;
+            ret = 0;
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        ret = conf->bitd_svc.manager (&(conf->bitd_svc), NULL,
-                                      PROC_START_NO_WAIT);
-        if (ret == -EINVAL)
-                ret = 0;
-        if (ret)
-                goto out;
+    ret = conf->scrub_svc.manager(&(conf->scrub_svc), NULL, PROC_START_NO_WAIT);
+    if (ret == -EINVAL)
+        ret = 0;
+out:
+    return ret;
+}
 
-        ret = conf->scrub_svc.manager (&(conf->scrub_svc), NULL,
-                                       PROC_START_NO_WAIT);
-        if (ret == -EINVAL)
-                ret = 0;
+int
+glusterd_svc_check_volfile_identical(char *svc_name,
+                                     glusterd_graph_builder_t builder,
+                                     gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int need_unlink = 0;
+    int tmp_fd = -1;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(identical);
+    conf = this->private;
+
+    glusterd_svc_build_volfile_path(svc_name, conf->workdir, orgvol,
+                                    sizeof(orgvol));
 
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(tmpvol);
+    if (tmp_fd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    need_unlink = 1;
+
+    ret = glusterd_create_global_volfile(builder, tmpvol, NULL);
+    if (ret)
+        goto out;
+
+    ret = glusterd_check_files_identical(orgvol, tmpvol, identical);
 out:
-        return ret;
-}
+    if (need_unlink)
+        sys_unlink(tmpvol);
 
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    if (tmp_fd >= 0)
+        sys_close(tmp_fd);
+
+    return ret;
+}
 
 int
-glusterd_svc_check_volfile_identical (char *svc_name,
+glusterd_svc_check_topology_identical(char *svc_name,
                                       glusterd_graph_builder_t builder,
                                       gf_boolean_t *identical)
 {
-        char            orgvol[PATH_MAX]        = {0,};
-        char            tmpvol[PATH_MAX]        = {0,};
-        glusterd_conf_t *conf                   = NULL;
-        xlator_t        *this                   = NULL;
-        int             ret                     = -1;
-        int             need_unlink             = 0;
-        int             tmp_fd                  = -1;
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = THIS;
+    int ret = -1;
+    int tmpclean = 0;
+    int tmpfd = -1;
+
+    if ((!identical) || (!this) || (!this->private)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* Fetch the original volfile */
+    glusterd_svc_build_volfile_path(svc_name, conf->workdir, orgvol,
+                                    sizeof(orgvol));
+
+    /* Create the temporary volfile */
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmpfd = mkstemp(tmpvol);
+    if (tmpfd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+    ret = glusterd_create_global_volfile(builder, tmpvol, NULL);
+    if (ret)
+        goto out;
+
+    /* Compare the topology of volfiles */
+    ret = glusterd_check_topology_identical(orgvol, tmpvol, identical);
+out:
+    if (tmpfd >= 0)
+        sys_close(tmpfd);
+    if (tmpclean)
+        sys_unlink(tmpvol);
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+    return ret;
+}
+
+int
+glusterd_volume_svc_check_volfile_identical(
+    char *svc_name, dict_t *mode_dict, glusterd_volinfo_t *volinfo,
+    glusterd_vol_graph_builder_t builder, gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int need_unlink = 0;
+    int tmp_fd = -1;
+
+    this = THIS;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, identical, out);
+
+    /* This builds volfile for volume level dameons */
+    glusterd_volume_svc_build_volfile_path(svc_name, volinfo, orgvol,
+                                           sizeof(orgvol));
+
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(tmpvol);
+    if (tmp_fd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    need_unlink = 1;
+
+    ret = builder(volinfo, tmpvol, mode_dict);
+    if (ret)
+        goto out;
+
+    ret = glusterd_check_files_identical(orgvol, tmpvol, identical);
+out:
+    if (need_unlink)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    if (tmp_fd >= 0)
+        sys_close(tmp_fd);
+
+    return ret;
+}
+
+int
+glusterd_volume_svc_check_topology_identical(
+    char *svc_name, dict_t *mode_dict, glusterd_volinfo_t *volinfo,
+    glusterd_vol_graph_builder_t builder, gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = THIS;
+    int ret = -1;
+    int tmpclean = 0;
+    int tmpfd = -1;
+
+    if ((!identical) || (!this) || (!this->private)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* This builds volfile for volume level dameons */
+    glusterd_volume_svc_build_volfile_path(svc_name, volinfo, orgvol,
+                                           sizeof(orgvol));
+    /* Create the temporary volfile */
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmpfd = mkstemp(tmpvol);
+    if (tmpfd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+    ret = builder(volinfo, tmpvol, mode_dict);
+    if (ret)
+        goto out;
+
+    /* Compare the topology of volfiles */
+    ret = glusterd_check_topology_identical(orgvol, tmpvol, identical);
+out:
+    if (tmpfd >= 0)
+        sys_close(tmpfd);
+    if (tmpclean)
+        sys_unlink(tmpvol);
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+    return ret;
+}
+
+gf_boolean_t
+glusterd_is_svcproc_attachable(glusterd_svc_proc_t *svc_proc)
+{
+    int pid = -1;
+    glusterd_svc_t *parent_svc = NULL;
+
+    if (!svc_proc)
+        return _gf_false;
 
-        this = THIS;
+    if (svc_proc->status == GF_SVC_STARTING)
+        return _gf_true;
 
-        GF_ASSERT (this);
-        GF_ASSERT (identical);
-        conf = this->private;
+    if (svc_proc->status == GF_SVC_STARTED ||
+        svc_proc->status == GF_SVC_DISCONNECTED) {
+        parent_svc = cds_list_entry(svc_proc->svcs.next, glusterd_svc_t,
+                                    mux_svc);
+        if (parent_svc && gf_is_service_running(parent_svc->proc.pidfile, &pid))
+            return _gf_true;
+    }
 
-        glusterd_svc_build_volfile_path (svc_name, conf->workdir,
-                                         orgvol, sizeof (orgvol));
+    if (svc_proc->status == GF_SVC_DIED || svc_proc->status == GF_SVC_STOPPING)
+        return _gf_false;
 
-        snprintf (tmpvol, sizeof (tmpvol), "/tmp/g%s-XXXXXX", svc_name);
+    return _gf_false;
+}
+
+void *
+__gf_find_compatible_svc(gd_node_type daemon)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    struct cds_list_head *svc_procs = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        tmp_fd = mkstemp (tmpvol);
-        if (tmp_fd < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        GD_MSG_FILE_OP_FAILED, "Unable to create temp file"
-                        " %s:(%s)", tmpvol, strerror (errno));
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+
+    switch (daemon) {
+        case GD_NODE_SHD: {
+            svc_procs = &conf->shd_procs;
+            if (!svc_procs)
                 goto out;
+        } break;
+        default:
+            /* Add support for other client daemons here */
+            goto out;
+    }
+
+    cds_list_for_each_entry(svc_proc, svc_procs, svc_proc_list)
+    {
+        if (glusterd_is_svcproc_attachable(svc_proc))
+            return (void *)svc_proc;
+        /*
+         * Logic to select one process goes here. Currently there is only one
+         * shd_proc. So selecting the first one;
+         */
+    }
+out:
+    return NULL;
+}
+
+glusterd_svc_proc_t *
+glusterd_svcprocess_new()
+{
+    glusterd_svc_proc_t *new_svcprocess = NULL;
+
+    new_svcprocess = GF_CALLOC(1, sizeof(*new_svcprocess),
+                               gf_gld_mt_glusterd_svc_proc_t);
+
+    if (!new_svcprocess)
+        return NULL;
+
+    CDS_INIT_LIST_HEAD(&new_svcprocess->svc_proc_list);
+    CDS_INIT_LIST_HEAD(&new_svcprocess->svcs);
+    new_svcprocess->notify = glusterd_muxsvc_common_rpc_notify;
+    new_svcprocess->status = GF_SVC_STARTING;
+    return new_svcprocess;
+}
+
+int
+glusterd_shd_svc_mux_init(glusterd_volinfo_t *volinfo, glusterd_svc_t *svc)
+{
+    int ret = -1;
+    glusterd_svc_proc_t *mux_proc = NULL;
+    glusterd_conn_t *mux_conn = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *parent_svc = NULL;
+    int pid = -1;
+    gf_boolean_t stop_daemon = _gf_false;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        if (svc->inited && !glusterd_proc_is_running(&(svc->proc))) {
+            /* This is the case when shd process was abnormally killed */
+            pthread_mutex_unlock(&conf->attach_lock);
+            glusterd_shd_svcproc_cleanup(&volinfo->shd);
+            pthread_mutex_lock(&conf->attach_lock);
         }
 
-        need_unlink = 1;
+        if (!svc->inited) {
+            glusterd_svc_build_shd_pidfile(volinfo, pidfile, sizeof(pidfile));
+            ret = snprintf(svc->proc.name, sizeof(svc->proc.name), "%s",
+                           "glustershd");
+            if (ret < 0)
+                goto unlock;
 
-        ret = glusterd_create_global_volfile (builder, tmpvol, NULL);
-        if (ret)
+            ret = snprintf(svc->proc.pidfile, sizeof(svc->proc.pidfile), "%s",
+                           pidfile);
+            if (ret < 0)
+                goto unlock;
+
+            if (gf_is_service_running(pidfile, &pid)) {
+                /* Just connect is required, but we don't know what happens
+                 * during the disconnect. So better to reattach.
+                 */
+                mux_proc = __gf_find_compatible_svc_from_pid(GD_NODE_SHD, pid);
+            }
+
+            if (!mux_proc) {
+                if (pid != -1 && sys_access(pidfile, R_OK) == 0) {
+                    /* stale pid file, stop and unlink it. This has to be
+                     * done outside the attach_lock.
+                     */
+                    stop_daemon = _gf_true;
+                }
+                mux_proc = __gf_find_compatible_svc(GD_NODE_SHD);
+            }
+            if (mux_proc) {
+                /* Take first entry from the process */
+                parent_svc = cds_list_entry(mux_proc->svcs.next, glusterd_svc_t,
+                                            mux_svc);
+                mux_conn = &parent_svc->conn;
+                if (volinfo)
+                    volinfo->shd.attached = _gf_true;
+            } else {
+                mux_proc = glusterd_svcprocess_new();
+                if (!mux_proc) {
+                    ret = -1;
+                    goto unlock;
+                }
+                cds_list_add_tail(&mux_proc->svc_proc_list, &conf->shd_procs);
+            }
+            svc->svc_proc = mux_proc;
+            cds_list_del_init(&svc->mux_svc);
+            cds_list_add_tail(&svc->mux_svc, &mux_proc->svcs);
+            ret = glusterd_shdsvc_init(volinfo, mux_conn, mux_proc);
+            if (ret) {
+                pthread_mutex_unlock(&conf->attach_lock);
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_SHDSVC,
+                       "Failed to init shd "
+                       "service");
                 goto out;
+            }
+            gf_msg_debug(THIS->name, 0, "shd service initialized");
+            svc->inited = _gf_true;
+        }
+        ret = 0;
+    }
+unlock:
+    pthread_mutex_unlock(&conf->attach_lock);
+out:
+    if (stop_daemon) {
+        glusterd_proc_stop(&svc->proc, SIGTERM, PROC_STOP_FORCE);
+        glusterd_unlink_file(pidfile);
+    }
+    return ret;
+}
+
+void *
+__gf_find_compatible_svc_from_pid(gd_node_type daemon, pid_t pid)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    struct cds_list_head *svc_procs = NULL;
+    glusterd_svc_t *svc = NULL;
+    pid_t mux_pid = -1;
+    glusterd_conf_t *conf = NULL;
+
+    conf = THIS->private;
+    if (!conf)
+        return NULL;
+
+    switch (daemon) {
+        case GD_NODE_SHD: {
+            svc_procs = &conf->shd_procs;
+            if (!svc_procs)
+                return NULL;
+        } break;
+        default:
+            /* Add support for other client daemons here */
+            return NULL;
+    }
+
+    cds_list_for_each_entry(svc_proc, svc_procs, svc_proc_list)
+    {
+        cds_list_for_each_entry(svc, &svc_proc->svcs, mux_svc)
+        {
+            if (gf_is_service_running(svc->proc.pidfile, &mux_pid)) {
+                if (mux_pid == pid &&
+                    glusterd_is_svcproc_attachable(svc_proc)) {
+                    /*TODO
+                     * inefficient loop, but at the moment, there is only
+                     * one shd.
+                     */
+                    return svc_proc;
+                }
+            }
+        }
+    }
+    return NULL;
+}
+
+static int32_t
+my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame)
+{
+    call_frame_t *frame = v_frame;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", frame, out);
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
+
+    STACK_DESTROY(frame->root);
+out:
+    return 0;
+}
+
+static int32_t
+glusterd_svc_attach_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *v_frame)
+{
+    call_frame_t *frame = v_frame;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_svc_t *svc = frame->cookie;
+    glusterd_conf_t *conf = NULL;
+    int *flag = (int *)frame->local;
+    xlator_t *this = THIS;
+    int ret = -1;
+    gf_getspec_rsp rsp = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", frame, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (!strcmp(svc->name, "glustershd")) {
+        /* Get volinfo->shd from svc object */
+        shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+        if (!shd) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+                   "Failed to get shd object "
+                   "from shd service");
+            goto out;
+        }
 
-        ret = glusterd_check_files_identical (orgvol, tmpvol, identical);
+        /* Get volinfo from shd */
+        volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+        if (!volinfo) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to get volinfo from "
+                   "from shd");
+            goto out;
+        }
+    }
+
+    if (!iov) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "iov is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
 
+    if (rsp.op_ret == 0) {
+        svc->online = _gf_true;
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_ATTACH_FAIL,
+               "svc %s of volume %s attached successfully to pid %d", svc->name,
+               volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL,
+               "svc %s of volume %s failed to attach to pid %d", svc->name,
+               volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+        if (!strcmp(svc->name, "glustershd")) {
+            glusterd_shd_svcproc_cleanup(&volinfo->shd);
+        }
+    }
 out:
-        if (need_unlink)
-                sys_unlink (tmpvol);
+    if (flag) {
+        GF_FREE(flag);
+    }
 
-        if (tmp_fd >= 0)
-                sys_close (tmp_fd);
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
 
-        return ret;
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
+extern size_t
+build_volfile_path(char *volume_id, char *path, size_t path_len,
+                   char *trusted_str, dict_t *dict);
+
 int
-glusterd_svc_check_topology_identical (char *svc_name,
-                                       glusterd_graph_builder_t builder,
-                                       gf_boolean_t *identical)
-{
-        char            orgvol[PATH_MAX]        = {0,};
-        char            tmpvol[PATH_MAX]        = {0,};
-        glusterd_conf_t *conf                   = NULL;
-        xlator_t        *this                   = THIS;
-        int             ret                     = -1;
-        int             tmpclean                = 0;
-        int             tmpfd                   = -1;
-
-        if ((!identical) || (!this) || (!this->private))
-                goto out;
+__glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flags,
+                                  struct rpc_clnt *rpc, char *volfile_id,
+                                  int op)
+{
+    int ret = -1;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec iov = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        '\0',
+    };
+    struct stat stbuf = {
+        0,
+    };
+    int32_t spec_fd = -1;
+    size_t file_len = -1;
+    char *volfile_content = NULL;
+    ssize_t req_size = 0;
+    call_frame_t *frame = NULL;
+    gd1_mgmt_brick_op_req brick_req;
+    dict_t *dict = NULL;
+    void *req = &brick_req;
+    void *errlbl = &&err;
+    struct rpc_clnt_connection *conn;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = THIS->private;
+    extern struct rpc_clnt_program gd_brick_prog;
+    fop_cbk_fn_t cbkfn = my_callback;
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    if (!rpc) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PARAM_NULL,
+               "called with null rpc");
+        return -1;
+    }
 
-        /* Fetch the original volfile */
-        glusterd_svc_build_volfile_path (svc_name, conf->workdir,
-                                         orgvol, sizeof (orgvol));
+    conn = &rpc->conn;
+    if (!conn->connected || conn->disconnected) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONNECT_RETURNED,
+               "not connected yet");
+        return -1;
+    }
 
-        /* Create the temporary volfile */
-        snprintf (tmpvol, sizeof (tmpvol), "/tmp/g%s-XXXXXX", svc_name);
-        tmpfd = mkstemp (tmpvol);
-        if (tmpfd < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        GD_MSG_FILE_OP_FAILED, "Unable to create temp file"
-                        " %s:(%s)", tmpvol, strerror (errno));
-                goto out;
+    brick_req.op = op;
+    brick_req.name = volfile_id;
+    brick_req.input.input_val = NULL;
+    brick_req.input.input_len = 0;
+    brick_req.dict.dict_val = NULL;
+    brick_req.dict.dict_len = 0;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
+                NULL);
+        goto *errlbl;
+    }
+
+    if (op == GLUSTERD_SVC_ATTACH) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -ENOMEM;
+            goto *errlbl;
         }
 
-        tmpclean = 1; /* SET the flag to unlink() tmpfile */
+        (void)build_volfile_path(volfile_id, path, sizeof(path), NULL, dict);
 
-        ret = glusterd_create_global_volfile (builder,
-                                              tmpvol, NULL);
-        if (ret)
+        ret = sys_stat(path, &stbuf);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "Unable to stat %s (%s)", path, strerror(errno));
+            ret = -EINVAL;
+            goto *errlbl;
+        }
+
+        file_len = stbuf.st_size;
+        volfile_content = GF_MALLOC(file_len + 1, gf_common_mt_char);
+        if (!volfile_content) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+            ret = -ENOMEM;
+            goto *errlbl;
+        }
+        spec_fd = open(path, O_RDONLY);
+        if (spec_fd < 0) {
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "failed to read volfile %s", path);
+            ret = -EIO;
+            goto *errlbl;
+        }
+        ret = sys_read(spec_fd, volfile_content, file_len);
+        if (ret == file_len) {
+            brick_req.input.input_val = volfile_content;
+            brick_req.input.input_len = file_len;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "read failed on path %s. File size=%" GF_PRI_SIZET
+                   "read size=%d",
+                   path, file_len, ret);
+            ret = -EIO;
+            goto *errlbl;
+        }
+        if (dict->count > 0) {
+            ret = dict_allocate_and_serialize(dict, &brick_req.dict.dict_val,
+                                              &brick_req.dict.dict_len);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+                goto *errlbl;
+            }
+        }
+
+        frame->cookie = svc;
+        frame->local = GF_CALLOC(1, sizeof(int), gf_gld_mt_int);
+        *((int *)frame->local) = flags;
+        cbkfn = glusterd_svc_attach_cbk;
+    }
+
+    req_size = xdr_sizeof((xdrproc_t)xdr_gd1_mgmt_brick_op_req, req);
+    iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size);
+    if (!iobuf) {
+        goto *errlbl;
+    }
+    errlbl = &&maybe_free_iobuf;
+
+    iov.iov_base = iobuf->ptr;
+    iov.iov_len = iobuf_pagesize(iobuf);
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto *errlbl;
+    }
+    errlbl = &&free_iobref;
+
+    iobref_add(iobref, iobuf);
+    /*
+     * Drop our reference to the iobuf.  The iobref should already have
+     * one after iobref_add, so when we unref that we'll free the iobuf as
+     * well.  This allows us to pass just the iobref as frame->local.
+     */
+    iobuf_unref(iobuf);
+    /* Set the pointer to null so we don't free it on a later error. */
+    iobuf = NULL;
+
+    /* Create the xdr payload */
+    ret = xdr_serialize_generic(iov, req, (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret == -1) {
+        goto *errlbl;
+    }
+    iov.iov_len = ret;
+
+    /* Send the msg */
+    GF_ATOMIC_INC(conf->blockers);
+    ret = rpc_clnt_submit(rpc, &gd_brick_prog, op, cbkfn, &iov, 1, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(volfile_content);
+    if (spec_fd >= 0)
+        sys_close(spec_fd);
+    return ret;
+
+free_iobref:
+    iobref_unref(iobref);
+maybe_free_iobuf:
+    if (iobuf) {
+        iobuf_unref(iobuf);
+    }
+err:
+    if (dict)
+        dict_unref(dict);
+    if (brick_req.dict.dict_val)
+        GF_FREE(brick_req.dict.dict_val);
+
+    GF_FREE(volfile_content);
+    if (spec_fd >= 0)
+        sys_close(spec_fd);
+    if (frame)
+        STACK_DESTROY(frame->root);
+    return -1;
+}
+
+int
+glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int flags)
+{
+    glusterd_conf_t *conf = THIS->private;
+    int ret = -1;
+    int tries;
+    rpc_clnt_t *rpc = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_ATTACH_INFO,
+           "adding svc %s (volume=%s) to existing "
+           "process with pid %d",
+           svc->name, volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+
+    rpc = rpc_clnt_ref(svc->conn.rpc);
+    for (tries = 15; tries > 0; --tries) {
+        /* There might be a case that the volume for which we're attempting to
+         * attach a shd svc might become stale and in the process of deletion.
+         * Given that the volinfo object is being already passed here before
+         * that sequence of operation has happened we might be operating on a
+         * stale volume. At every sync task switch we should check for existance
+         * of the volume now
+         */
+        if (!glusterd_volume_exists(volinfo->volname)) {
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "Volume %s "
+                   " is marked as stale, not attempting further shd svc attach "
+                   "attempts",
+                   volinfo->volname);
+            ret = 0;
+            goto out;
+        }
+        if (rpc) {
+            pthread_mutex_lock(&conf->attach_lock);
+            {
+                ret = __glusterd_send_svc_configure_req(
+                    svc, flags, rpc, svc->proc.volfileid, GLUSTERD_SVC_ATTACH);
+            }
+            pthread_mutex_unlock(&conf->attach_lock);
+            if (!ret) {
+                volinfo->shd.attached = _gf_true;
                 goto out;
+            }
+        }
+        /*
+         * It might not actually be safe to manipulate the lock
+         * like this, but if we don't then the connection can
+         * never actually complete and retries are useless.
+         * Unfortunately, all of the alternatives (e.g. doing
+         * all of this in a separate thread) are much more
+         * complicated and risky.
+         * TBD: see if there's a better way
+         */
+        synclock_unlock(&conf->big_lock);
+        synctask_sleep(1);
+        synclock_lock(&conf->big_lock);
+    }
+    ret = -1;
+    gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SVC_ATTACH_FAIL,
+           "attach failed for %s(volume=%s)", svc->name, volinfo->volname);
+out:
+    if (rpc)
+        rpc_clnt_unref(rpc);
+    return ret;
+}
 
-        /* Compare the topology of volfiles */
-        ret = glusterd_check_topology_identical (orgvol, tmpvol,
-                                                 identical);
+int
+glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig)
+{
+    glusterd_conf_t *conf = THIS->private;
+    int ret = -1;
+    int tries;
+    rpc_clnt_t *rpc = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, conf, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, svc, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+
+    gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DETACH_INFO,
+           "removing svc %s (volume=%s) from existing "
+           "process with pid %d",
+           svc->name, volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+
+    rpc = rpc_clnt_ref(svc->conn.rpc);
+    for (tries = 15; tries > 0; --tries) {
+        if (rpc) {
+            /*For detach there is no flags, and we are not using sig.*/
+            pthread_mutex_lock(&conf->attach_lock);
+            {
+                ret = __glusterd_send_svc_configure_req(svc, 0, svc->conn.rpc,
+                                                        svc->proc.volfileid,
+                                                        GLUSTERD_SVC_DETACH);
+            }
+            pthread_mutex_unlock(&conf->attach_lock);
+            if (!ret) {
+                goto out;
+            }
+        }
+        /*
+         * It might not actually be safe to manipulate the lock
+         * like this, but if we don't then the connection can
+         * never actually complete and retries are useless.
+         * Unfortunately, all of the alternatives (e.g. doing
+         * all of this in a separate thread) are much more
+         * complicated and risky.
+         * TBD: see if there's a better way
+         */
+        synclock_unlock(&conf->big_lock);
+        synctask_sleep(1);
+        synclock_lock(&conf->big_lock);
+    }
+    ret = -1;
+    gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SVC_DETACH_FAIL,
+           "detach failed for %s(volume=%s)", svc->name, volinfo->volname);
 out:
-        if (tmpfd >= 0)
-                sys_close (tmpfd);
-        if (tmpclean)
-                sys_unlink (tmpvol);
-        return ret;
+    if (rpc)
+        rpc_clnt_unref(rpc);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-svc-helper.h
index b5aafefc1b5..12717dc58ac 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.h
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.h
@@ -16,21 +16,57 @@
 #include "glusterd-volgen.h"
 
 int
-glusterd_svcs_reconfigure ();
+glusterd_svcs_reconfigure(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_svcs_stop ();
+glusterd_svcs_stop(glusterd_volinfo_t *vol);
 
 int
-glusterd_svcs_manager (glusterd_volinfo_t *volinfo);
+glusterd_svcs_manager(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_svc_check_volfile_identical (char *svc_name,
+glusterd_svc_check_volfile_identical(char *svc_name,
+                                     glusterd_graph_builder_t builder,
+                                     gf_boolean_t *identical);
+int
+glusterd_svc_check_topology_identical(char *svc_name,
                                       glusterd_graph_builder_t builder,
                                       gf_boolean_t *identical);
 int
-glusterd_svc_check_topology_identical (char *svc_name,
-                                       glusterd_graph_builder_t builder,
-                                       gf_boolean_t *identical);
+glusterd_volume_svc_check_volfile_identical(char *svc_name, dict_t *mode_dict,
+                                            glusterd_volinfo_t *volinfo,
+                                            glusterd_vol_graph_builder_t,
+                                            gf_boolean_t *identical);
+int
+glusterd_volume_svc_check_topology_identical(char *svc_name, dict_t *mode_dict,
+                                             glusterd_volinfo_t *volinfo,
+                                             glusterd_vol_graph_builder_t,
+                                             gf_boolean_t *identical);
+void
+glusterd_volume_svc_build_volfile_path(char *server, glusterd_volinfo_t *vol,
+                                       char *volfile, size_t len);
+void *
+__gf_find_compatible_svc(gd_node_type daemon);
+
+glusterd_svc_proc_t *
+glusterd_svcprocess_new();
+
+int
+glusterd_shd_svc_mux_init(glusterd_volinfo_t *volinfo, glusterd_svc_t *svc);
+
+void *
+__gf_find_compatible_svc_from_pid(gd_node_type daemon, pid_t pid);
+
+int
+glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo,
+                    int flags);
+
+int
+glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig);
+
+int
+__glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flag,
+                                  struct rpc_clnt *rpc, char *volfile_id,
+                                  int op);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
index 454c2a453b2..18b3fb13630 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
@@ -8,331 +8,529 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glusterd-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-proc-mgmt.h"
 #include "glusterd-conn-mgmt.h"
 #include "glusterd-messages.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
+#include "glusterd-shd-svc-helper.h"
 
 int
-glusterd_svc_create_rundir (char *rundir)
+glusterd_svc_create_rundir(char *rundir)
 {
-        int ret      = -1;
-
-        ret = mkdir_p (rundir, 0777, _gf_true);
-        if ((ret == -1) && (EEXIST != errno)) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "Unable to create rundir %s",
-                        rundir);
-        }
-        return ret;
+    int ret = -1;
+
+    ret = mkdir_p(rundir, 0755, _gf_true);
+    if ((ret == -1) && (EEXIST != errno)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create rundir %s", rundir);
+    }
+    return ret;
 }
 
-static void
-glusterd_svc_build_logfile_path (char *server, char *logdir, char *logfile,
-                                 size_t len)
+void
+glusterd_svc_build_logfile_path(char *server, char *logdir, char *logfile,
+                                size_t len)
 {
-        snprintf (logfile, len, "%s/%s.log", logdir, server);
+    snprintf(logfile, len, "%s/%s.log", logdir, server);
 }
 
-static void
-glusterd_svc_build_volfileid_path (char *server, char *volfileid, size_t len)
+void
+glusterd_svc_build_volfileid_path(char *server, char *volfileid, size_t len)
 {
-        snprintf (volfileid, len, "gluster/%s", server);
+    snprintf(volfileid, len, "gluster/%s", server);
 }
 
 static int
-glusterd_svc_init_common (glusterd_svc_t *svc,
-                          char *svc_name, char *workdir,
-                          char *rundir, char *logdir,
-                          glusterd_conn_notify_t notify)
+glusterd_svc_init_common(glusterd_svc_t *svc, char *svc_name, char *workdir,
+                         char *rundir, char *logdir,
+                         glusterd_conn_notify_t notify)
 {
-        int                     ret                        = -1;
-        glusterd_conf_t        *priv                       = NULL;
-        xlator_t               *this                       = NULL;
-        char                    pidfile[PATH_MAX]          = {0,};
-        char                    logfile[PATH_MAX]          = {0,};
-        char                    volfile[PATH_MAX]          = {0,};
-        char                    sockfpath[PATH_MAX]        = {0,};
-        char                    volfileid[256]             = {0};
-        char                    *volfileserver             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = snprintf (svc->name, sizeof (svc->name), "%s", svc_name);
-        if (ret < 0)
-                goto out;
-
-        if (!notify)
-                notify = glusterd_svc_common_rpc_notify;
-
-        glusterd_svc_create_rundir (rundir);
-
-        /* Initialize the connection mgmt */
-        glusterd_conn_build_socket_filepath (rundir, MY_UUID,
-                                             sockfpath, sizeof (sockfpath));
-
-        ret = glusterd_conn_init (&(svc->conn), sockfpath, 600, notify);
-        if (ret)
-                goto out;
-
-        /* Initialize the process mgmt */
-        glusterd_svc_build_pidfile_path (svc_name, workdir, pidfile,
-                                         sizeof(pidfile));
-        glusterd_svc_build_volfile_path (svc_name, workdir, volfile,
-                                         sizeof (volfile));
-
-        glusterd_svc_build_logfile_path (svc_name, logdir, logfile,
-                                         sizeof (logfile));
-        glusterd_svc_build_volfileid_path (svc_name, volfileid,
-                                           sizeof(volfileid));
-
-        if (dict_get_str (this->options, "transport.socket.bind-address",
-                          &volfileserver) != 0) {
-                volfileserver = "localhost";
-        }
-
-        ret = glusterd_proc_init (&(svc->proc), svc_name, pidfile, logdir,
-                                  logfile, volfile, volfileid, volfileserver);
-        if (ret)
-                goto out;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char sockfpath[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    char *volfileserver = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", svc_name);
+    if (ret < 0)
+        goto out;
+
+    if (!notify)
+        notify = glusterd_svc_common_rpc_notify;
+
+    glusterd_svc_create_rundir(rundir);
+
+    /* Initialize the connection mgmt */
+    glusterd_conn_build_socket_filepath(rundir, MY_UUID, sockfpath,
+                                        sizeof(sockfpath));
+
+    ret = glusterd_conn_init(&(svc->conn), sockfpath, 600, notify);
+    if (ret)
+        goto out;
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_pidfile_path(svc_name, priv->rundir, pidfile,
+                                    sizeof(pidfile));
+
+    glusterd_svc_build_volfile_path(svc_name, workdir, volfile,
+                                    sizeof(volfile));
+
+    glusterd_svc_build_logfile_path(svc_name, logdir, logfile, sizeof(logfile));
+    glusterd_svc_build_volfileid_path(svc_name, volfileid, sizeof(volfileid));
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+
+    ret = glusterd_proc_init(&(svc->proc), svc_name, pidfile, logdir, logfile,
+                             volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
 
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 static int
-svc_add_args (dict_t *cmdline, char *arg, data_t *value, void *data)
+svc_add_args(dict_t *cmdline, char *arg, data_t *value, void *data)
 {
-        runner_t *runner = data;
-        runner_add_arg (runner, value->data);
-        return 0;
+    runner_t *runner = data;
+    runner_add_arg(runner, value->data);
+    return 0;
 }
 
-int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name)
+int
+glusterd_svc_init(glusterd_svc_t *svc, char *svc_name)
 {
-        int              ret               = -1;
-        char             rundir[PATH_MAX]  = {0,};
-        glusterd_conf_t *priv              = NULL;
-        xlator_t        *this              = NULL;
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        glusterd_svc_build_rundir (svc_name, priv->workdir, rundir,
-                                   sizeof (rundir));
-        ret = glusterd_svc_init_common (svc, svc_name, priv->workdir, rundir,
-                                        DEFAULT_LOG_FILE_DIRECTORY, NULL);
+    glusterd_svc_build_rundir(svc_name, priv->rundir, rundir, sizeof(rundir));
+    ret = glusterd_svc_init_common(svc, svc_name, priv->workdir, rundir,
+                                   priv->logdir, NULL);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_svc_start (glusterd_svc_t *svc, int flags, dict_t *cmdline)
+glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline)
 {
-        int                  ret                        = -1;
-        runner_t             runner                     = {0,};
-        glusterd_conf_t     *priv                       = NULL;
-        xlator_t            *this                       = NULL;
-        char                 valgrind_logfile[PATH_MAX] = {0};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (glusterd_proc_is_running (&(svc->proc))) {
-                ret = 0;
-                goto out;
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char valgrind_logfile[PATH_MAX] = {0};
+    char *localtime_logging = NULL;
+    char *log_level = NULL;
+    char daemon_log_level[30] = {0};
+    char msg[1024] = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", priv, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+
+    pthread_mutex_lock(&priv->attach_lock);
+    {
+        if (glusterd_proc_is_running(&(svc->proc))) {
+            ret = 0;
+            goto unlock;
         }
 
-        ret = sys_access (svc->proc.volfile, F_OK);
+        ret = sys_access(svc->proc.volfile, F_OK);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VOLFILE_NOT_FOUND, "Volfile %s is not present",
-                        svc->proc.volfile);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_NOT_FOUND,
+                   "Volfile %s is not present", svc->proc.volfile);
+            goto unlock;
         }
 
-        runinit (&runner);
+        runinit(&runner);
+
+        if (this->ctx->cmd_args.vgtool != _gf_none) {
+            len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s.log",
+                           svc->proc.logdir, svc->name);
+            if ((len < 0) || (len >= PATH_MAX)) {
+                ret = -1;
+                goto unlock;
+            }
 
-        if (priv->valgrind) {
-                snprintf (valgrind_logfile, PATH_MAX, "%s/valgrind-%s.log",
-                          svc->proc.logfile, svc->name);
+            if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+                runner_add_args(&runner, "valgrind", "--leak-check=full",
+                                "--trace-children=yes", "--track-origins=yes",
+                                NULL);
+            else
+                runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
 
-                runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", "--track-origins=yes",
-                                 NULL);
-                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+            runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
         }
 
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", svc->proc.volfileserver,
-                         "--volfile-id", svc->proc.volfileid,
-                         "-p", svc->proc.pidfile,
-                         "-l", svc->proc.logfile,
-                         "-S", svc->conn.sockpath,
-                         NULL);
+        runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s",
+                        svc->proc.volfileserver, "--volfile-id",
+                        svc->proc.volfileid, "-p", svc->proc.pidfile, "-l",
+                        svc->proc.logfile, "-S", svc->conn.sockpath, NULL);
+
+        if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                          SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
+                          &localtime_logging) == 0) {
+            if (strcmp(localtime_logging, "enable") == 0)
+                runner_add_arg(&runner, "--localtime-logging");
+        }
+        if (dict_get_strn(priv->opts, GLUSTERD_DAEMON_LOG_LEVEL_KEY,
+                          SLEN(GLUSTERD_DAEMON_LOG_LEVEL_KEY),
+                          &log_level) == 0) {
+            snprintf(daemon_log_level, 30, "--log-level=%s", log_level);
+            runner_add_arg(&runner, daemon_log_level);
+        }
+
+        if (this->ctx->cmd_args.global_threading) {
+            runner_add_arg(&runner, "--global-threading");
+        }
 
         if (cmdline)
-                dict_foreach (cmdline, svc_add_args, (void *) &runner);
+            dict_foreach(cmdline, svc_add_args, (void *)&runner);
 
-        gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_SVC_START_SUCCESS,
-                "Starting %s service", svc->name);
+        snprintf(msg, sizeof(msg), "Starting %s service", svc->name);
+        runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
 
         if (flags == PROC_START_NO_WAIT) {
-                ret = runner_run_nowait (&runner);
+            ret = runner_run_nowait(&runner);
         } else {
-                synclock_unlock (&priv->big_lock);
-                {
-                        ret = runner_run (&runner);
-                }
-                synclock_lock (&priv->big_lock);
+            synclock_unlock(&priv->big_lock);
+            {
+                ret = runner_run(&runner);
+            }
+            synclock_lock(&priv->big_lock);
         }
-
+    }
+unlock:
+    pthread_mutex_unlock(&priv->attach_lock);
 out:
-        gf_msg_debug (this->name, 0, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
-int glusterd_svc_stop (glusterd_svc_t *svc, int sig)
+int
+glusterd_svc_stop(glusterd_svc_t *svc, int sig)
 {
-        int ret = -1;
-
-        ret = glusterd_proc_stop (&(svc->proc), sig, PROC_STOP_FORCE);
-        if (ret)
-                goto out;
-        glusterd_conn_disconnect (&(svc->conn));
-
-        if (ret == 0) {
-                svc->online =  _gf_false;
-                (void) glusterd_unlink_file ((char *)svc->conn.sockpath);
-        }
-        gf_msg (THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
-                "%s service is stopped", svc->name);
+    int ret = -1;
+
+    ret = glusterd_proc_stop(&(svc->proc), sig, PROC_STOP_FORCE);
+    if (ret)
+        goto out;
+    glusterd_conn_disconnect(&(svc->conn));
+
+    if (ret == 0) {
+        svc->online = _gf_false;
+        (void)glusterd_unlink_file((char *)svc->conn.sockpath);
+    }
+    gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
+           "%s service is stopped", svc->name);
 out:
-        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 void
-glusterd_svc_build_pidfile_path (char *server, char *workdir, char *path,
-                                 size_t len)
+glusterd_svc_build_pidfile_path(char *server, char *workdir, char *path,
+                                size_t len)
 {
-        char    dir[PATH_MAX] = {0};
+    char dir[PATH_MAX] = {0};
 
-        GF_ASSERT (len == PATH_MAX);
+    GF_ASSERT(len == PATH_MAX);
 
-        glusterd_svc_build_rundir (server, workdir, dir, sizeof (dir));
-        snprintf (path, len, "%s/%s.pid", dir, server);
+    glusterd_svc_build_rundir(server, workdir, dir, sizeof(dir));
+    snprintf(path, len, "%s/%s.pid", dir, server);
 }
 
 void
-glusterd_svc_build_volfile_path (char *server, char *workdir, char *volfile,
-                                 size_t len)
+glusterd_svc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                size_t len)
 {
-        char  dir[PATH_MAX] = {0,};
+    char dir[PATH_MAX] = {
+        0,
+    };
 
-        GF_ASSERT (len == PATH_MAX);
+    GF_ASSERT(len == PATH_MAX);
 
-        glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir));
+    glusterd_svc_build_svcdir(server, workdir, dir, sizeof(dir));
 
-        if (!strcmp(server, "quotad")) /*quotad has different volfile name*/
-                snprintf (volfile, len, "%s/%s.vol", dir, server);
-       else
-                snprintf (volfile, len, "%s/%s-server.vol", dir, server);
+    if (!strcmp(server, "quotad"))
+        /*quotad has different volfile name*/
+        snprintf(volfile, len, "%s/%s.vol", dir, server);
+    else
+        snprintf(volfile, len, "%s/%s-server.vol", dir, server);
 }
 
 void
-glusterd_svc_build_svcdir (char *server, char *workdir, char *path, size_t len)
+glusterd_svc_build_svcdir(char *server, char *workdir, char *path, size_t len)
 {
-        GF_ASSERT (len == PATH_MAX);
+    GF_ASSERT(len == PATH_MAX);
 
-        snprintf (path, len, "%s/%s", workdir, server);
+    snprintf(path, len, "%s/%s", workdir, server);
 }
 
 void
-glusterd_svc_build_rundir (char *server, char *workdir, char *path, size_t len)
+glusterd_svc_build_rundir(char *server, char *workdir, char *path, size_t len)
 {
-        char    dir[PATH_MAX] = {0};
+    char dir[PATH_MAX] = {0};
 
-        GF_ASSERT (len == PATH_MAX);
+    GF_ASSERT(len == PATH_MAX);
 
-        glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir));
-        snprintf (path, len, "%s/run", dir);
+    glusterd_svc_build_svcdir(server, workdir, dir, sizeof(dir));
+    snprintf(path, len, "%s", dir);
 }
 
 int
-glusterd_svc_reconfigure (int (*create_volfile) ())
+glusterd_svc_reconfigure(int (*create_volfile)())
 {
-        int ret = -1;
+    int ret = -1;
 
-        ret = create_volfile ();
-        if (ret)
-                goto out;
+    ret = create_volfile();
+    if (ret)
+        goto out;
 
-        ret = glusterd_fetchspec_notify (THIS);
+    ret = glusterd_fetchspec_notify(THIS);
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_svc_common_rpc_notify (glusterd_conn_t *conn,
-                                rpc_clnt_event_t event)
+glusterd_svc_common_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event)
 {
-        int                  ret  = 0;
-        glusterd_svc_t      *svc  = NULL;
-        xlator_t            *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        /* Get the parent onject i.e. svc using list_entry macro */
-        svc = cds_list_entry (conn, glusterd_svc_t, conn);
-        if (!svc) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_SVC_GET_FAIL, "Failed to get the service");
-                return -1;
-        }
+    int ret = 0;
+    glusterd_svc_t *svc = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    /* Get the parent onject i.e. svc using list_entry macro */
+    svc = cds_list_entry(conn, glusterd_svc_t, conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        return -1;
+    }
+
+    switch (event) {
+        case RPC_CLNT_CONNECT:
+            gf_msg_debug(this->name, 0,
+                         "%s has connected with "
+                         "glusterd.",
+                         svc->name);
+            gf_event(EVENT_SVC_CONNECTED, "svc_name=%s", svc->name);
+            svc->online = _gf_true;
+            break;
+
+        case RPC_CLNT_DISCONNECT:
+            if (svc->online) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED,
+                       "%s has disconnected "
+                       "from glusterd.",
+                       svc->name);
+                gf_event(EVENT_SVC_DISCONNECTED, "svc_name=%s", svc->name);
+                svc->online = _gf_false;
+            }
+            break;
+
+        default:
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
+
+    return ret;
+}
+
+void
+glusterd_volume_svc_build_volfile_path(char *server, glusterd_volinfo_t *vol,
+                                       char *volfile, size_t len)
+{
+    GF_ASSERT(len == PATH_MAX);
+
+    if (!strcmp(server, "glustershd")) {
+        glusterd_svc_build_shd_volfile_path(vol, volfile, len);
+    }
+}
 
-        switch (event) {
+int
+glusterd_muxsvc_common_rpc_notify(glusterd_svc_proc_t *mux_proc,
+                                  rpc_clnt_event_t event)
+{
+    int ret = 0;
+    glusterd_svc_t *svc = NULL;
+    glusterd_svc_t *tmp = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t need_logging = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!mux_proc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the svc proc data");
+        return -1;
+    }
+
+    /* Currently this function was used for shd svc, if this function is
+     * using for another svc, change ths glustershd reference. We can get
+     * the svc name from any of the attached svc's
+     */
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                gf_msg_debug (this->name, 0, "%s has connected with "
-                        "glusterd.", svc->name);
-                svc->online =  _gf_true;
-                break;
+            gf_msg_debug(this->name, 0,
+                         "glustershd has connected with glusterd.");
+            gf_event(EVENT_SVC_CONNECTED, "svc_name=glustershd");
+            cds_list_for_each_entry_safe(svc, tmp, &mux_proc->svcs, mux_svc)
+            {
+                if (svc->online)
+                    continue;
+                svc->online = _gf_true;
+            }
+            if (mux_proc->status != GF_SVC_STARTED)
+                mux_proc->status = GF_SVC_STARTED;
+
+            break;
 
         case RPC_CLNT_DISCONNECT:
+            cds_list_for_each_entry_safe(svc, tmp, &mux_proc->svcs, mux_svc)
+            {
                 if (svc->online) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_NODE_DISCONNECTED, "%s has disconnected "
-                                "from glusterd.", svc->name);
-                        svc->online =  _gf_false;
+                    if (!need_logging)
+                        need_logging = _gf_true;
+                    svc->online = _gf_false;
+                }
+            }
+            if (mux_proc->status != GF_SVC_DIED) {
+                svc = cds_list_entry(mux_proc->svcs.next, glusterd_svc_t,
+                                     mux_svc);
+                if (svc && !glusterd_proc_is_running(&svc->proc)) {
+                    mux_proc->status = GF_SVC_DIED;
+                } else {
+                    mux_proc->status = GF_SVC_DISCONNECTED;
                 }
-                break;
+            }
+
+            if (need_logging) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED,
+                       "glustershd has disconnected from glusterd.");
+                gf_event(EVENT_SVC_DISCONNECTED, "svc_name=glustershd");
+            }
+            break;
 
         default:
-                gf_msg_trace (this->name, 0,
-                        "got some other RPC event %d", event);
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
 
-        return ret;
+    return ret;
+}
+
+int
+glusterd_muxsvc_conn_init(glusterd_conn_t *conn, glusterd_svc_proc_t *mux_proc,
+                          char *sockpath, int frame_timeout,
+                          glusterd_muxsvc_conn_notify_t notify)
+{
+    int ret = -1;
+    dict_t *options = NULL;
+    struct rpc_clnt *rpc = NULL;
+    xlator_t *this = THIS;
+    glusterd_svc_t *svc = NULL;
+
+    options = dict_new();
+    if (!this || !options)
+        goto out;
+
+    svc = cds_list_entry(conn, glusterd_svc_t, conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        goto out;
+    }
+
+    ret = rpc_transport_unix_options_build(options, sockpath, frame_timeout);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32n(options, "transport.socket.ignore-enoent",
+                          SLEN("transport.socket.ignore-enoent"), 1);
+    if (ret)
+        goto out;
+
+    /* @options is free'd by rpc_transport when destroyed */
+    rpc = rpc_clnt_new(options, this, (char *)svc->name, 16);
+    if (!rpc) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, glusterd_muxsvc_conn_common_notify,
+                                   mux_proc);
+    if (ret)
+        goto out;
+
+    ret = snprintf(conn->sockpath, sizeof(conn->sockpath), "%s", sockpath);
+    if (ret < 0)
+        goto out;
+    else
+        ret = 0;
+
+    conn->frame_timeout = frame_timeout;
+    conn->rpc = rpc;
+    mux_proc->notify = notify;
+out:
+    if (options)
+        dict_unref(options);
+    if (ret) {
+        if (rpc) {
+            rpc_clnt_unref(rpc);
+            rpc = NULL;
+        }
+    }
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h
index fe7a19385cd..5daee993833 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h
@@ -13,62 +13,100 @@
 
 #include "glusterd-proc-mgmt.h"
 #include "glusterd-conn-mgmt.h"
+#include "glusterd-rcu.h"
 
 struct glusterd_svc_;
-typedef struct glusterd_svc_ glusterd_svc_t;
-
-typedef void (*glusterd_svc_build_t) (glusterd_svc_t *svc);
 
-typedef int (*glusterd_svc_manager_t) (glusterd_svc_t *svc,
-                                       void *data, int flags);
-typedef int (*glusterd_svc_start_t) (glusterd_svc_t *svc, int flags);
-typedef int (*glusterd_svc_stop_t) (glusterd_svc_t *svc, int sig);
+typedef struct glusterd_svc_ glusterd_svc_t;
+typedef struct glusterd_svc_proc_ glusterd_svc_proc_t;
+
+typedef void (*glusterd_svc_build_t)(glusterd_svc_t *svc);
+
+typedef int (*glusterd_svc_manager_t)(glusterd_svc_t *svc, void *data,
+                                      int flags);
+typedef int (*glusterd_svc_start_t)(glusterd_svc_t *svc, int flags);
+typedef int (*glusterd_svc_stop_t)(glusterd_svc_t *svc, int sig);
+typedef int (*glusterd_svc_reconfigure_t)(void *data);
+
+typedef int (*glusterd_muxsvc_conn_notify_t)(glusterd_svc_proc_t *mux_proc,
+                                             rpc_clnt_event_t event);
+
+typedef enum gf_svc_status {
+    GF_SVC_STARTING,
+    GF_SVC_STARTED,
+    GF_SVC_STOPPING,
+    GF_SVC_DISCONNECTED,
+    GF_SVC_DIED,
+} gf_svc_status_t;
+
+struct glusterd_svc_proc_ {
+    struct cds_list_head svc_proc_list;
+    struct cds_list_head svcs;
+    glusterd_muxsvc_conn_notify_t notify;
+    rpc_clnt_t *rpc;
+    void *data;
+    gf_svc_status_t status;
+};
 
 struct glusterd_svc_ {
-        char                      name[PATH_MAX];
-        glusterd_conn_t           conn;
-        glusterd_proc_t           proc;
-        glusterd_svc_build_t      build;
-        glusterd_svc_manager_t    manager;
-        glusterd_svc_start_t      start;
-        glusterd_svc_stop_t       stop;
-        gf_boolean_t              online;
-        gf_boolean_t              inited;
+    glusterd_conn_t conn;
+    glusterd_svc_manager_t manager;
+    glusterd_svc_start_t start;
+    glusterd_svc_stop_t stop;
+    glusterd_svc_reconfigure_t reconfigure;
+    glusterd_svc_proc_t *svc_proc;
+    struct cds_list_head mux_svc;
+    glusterd_proc_t proc;
+    char name[NAME_MAX];
+    gf_boolean_t online;
+    gf_boolean_t inited;
 };
 
 int
-glusterd_svc_create_rundir (char *rundir);
+glusterd_svc_create_rundir(char *rundir);
 
 int
-glusterd_svc_init (glusterd_svc_t *svc, char *svc_name);
+glusterd_svc_init(glusterd_svc_t *svc, char *svc_name);
 
 int
-glusterd_svc_start (glusterd_svc_t *svc, int flags, dict_t *cmdline);
+glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline);
 
 int
-glusterd_svc_stop (glusterd_svc_t *svc, int sig);
+glusterd_svc_stop(glusterd_svc_t *svc, int sig);
+
+void
+glusterd_svc_build_pidfile_path(char *server, char *workdir, char *path,
+                                size_t len);
 
 void
-glusterd_svc_build_pidfile_path (char *server, char *workdir,
-                                 char *path, size_t len);
+glusterd_svc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                size_t len);
 
 void
-glusterd_svc_build_volfile_path (char *server, char *workdir,
-                                 char *volfile, size_t len);
+glusterd_svc_build_logfile_path(char *server, char *logdir, char *logfile,
+                                size_t len);
 
 void
-glusterd_svc_build_svcdir (char *server, char *workdir,
-                           char *path, size_t len);
+glusterd_svc_build_svcdir(char *server, char *workdir, char *path, size_t len);
 
 void
-glusterd_svc_build_rundir (char *server, char *workdir,
-                           char *path, size_t len);
+glusterd_svc_build_rundir(char *server, char *workdir, char *path, size_t len);
 
 int
-glusterd_svc_reconfigure (int (*create_volfile) ());
+glusterd_svc_reconfigure(int (*create_volfile)());
 
 int
-glusterd_svc_common_rpc_notify (glusterd_conn_t *conn,
-                                rpc_clnt_event_t event);
+glusterd_svc_common_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event);
 
+int
+glusterd_muxsvc_common_rpc_notify(glusterd_svc_proc_t *conn,
+                                  rpc_clnt_event_t event);
+
+int
+glusterd_proc_get_pid(glusterd_proc_t *proc);
+
+int
+glusterd_muxsvc_conn_init(glusterd_conn_t *conn, glusterd_svc_proc_t *mux_proc,
+                          char *sockpath, int frame_timeout,
+                          glusterd_muxsvc_conn_notify_t notify);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index 7c5721f25d0..b73d37ad08e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -27,197 +27,189 @@
 extern glusterd_op_info_t opinfo;
 
 void
-gd_synctask_barrier_wait (struct syncargs *args, int count)
+gd_synctask_barrier_wait(struct syncargs *args, int count)
 {
-        glusterd_conf_t *conf = THIS->private;
+    glusterd_conf_t *conf = THIS->private;
 
-        synclock_unlock (&conf->big_lock);
-        synctask_barrier_wait (args, count);
-        synclock_lock (&conf->big_lock);
+    synclock_unlock(&conf->big_lock);
+    synctask_barrier_wait(args, count);
+    synclock_lock(&conf->big_lock);
 
-	syncbarrier_destroy (&args->barrier);
+    syncbarrier_destroy(&args->barrier);
 }
 
 static void
-gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
-                   char *op_errstr, int op_code, uuid_t peerid, u_char *uuid)
+gd_collate_errors(struct syncargs *args, int op_ret, int op_errno,
+                  char *op_errstr, int op_code, uuid_t peerid, u_char *uuid)
 {
-        char     err_str[PATH_MAX] = "Please check log file for details.";
-        char     op_err[PATH_MAX] = "";
-        int      len               = -1;
-        char    *peer_str      = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-
-        if (op_ret) {
-                args->op_ret = op_ret;
-                args->op_errno = op_errno;
-
-                rcu_read_lock ();
-                peerinfo = glusterd_peerinfo_find (peerid, NULL);
-                if (peerinfo)
-                        peer_str = gf_strdup (peerinfo->hostname);
-                else
-                        peer_str = gf_strdup (uuid_utoa (uuid));
-                rcu_read_unlock ();
-
-                if (op_errstr && strcmp (op_errstr, "")) {
-                        len = snprintf (err_str, sizeof(err_str) - 1,
-                                        "Error: %s", op_errstr);
-                        err_str[len] = '\0';
-                }
+    char err_str[PATH_MAX] = "Please check log file for details.";
+    char op_err[PATH_MAX] = "";
+    int len = -1;
+    char *peer_str = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    if (op_ret) {
+        args->op_ret = op_ret;
+        args->op_errno = op_errno;
+
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find(peerid, NULL);
+        if (peerinfo)
+            peer_str = gf_strdup(peerinfo->hostname);
+        else
+            peer_str = gf_strdup(uuid_utoa(uuid));
+        RCU_READ_UNLOCK;
 
-                switch (op_code){
-                        case GLUSTERD_MGMT_CLUSTER_LOCK :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Locking failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                        case GLUSTERD_MGMT_CLUSTER_UNLOCK :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Unlocking failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                        case GLUSTERD_MGMT_STAGE_OP :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Staging failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                        case GLUSTERD_MGMT_COMMIT_OP :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Commit failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                }
-                op_err[len] = '\0';
-
-                if (args->errstr) {
-                        len = snprintf (err_str, sizeof(err_str) - 1,
-                                        "%s\n%s", args->errstr,
-                                        op_err);
-                        GF_FREE (args->errstr);
-                        args->errstr = NULL;
-                } else
-                        len = snprintf (err_str, sizeof(err_str) - 1,
-                                        "%s", op_err);
-                err_str[len] = '\0';
-
-                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                        GD_MSG_MGMT_OP_FAIL, "%s", op_err);
-                args->errstr = gf_strdup (err_str);
+        if (op_errstr && strcmp(op_errstr, "")) {
+            len = snprintf(err_str, sizeof(err_str) - 1, "Error: %s",
+                           op_errstr);
+            err_str[len] = '\0';
         }
 
-        GF_FREE (peer_str);
+        switch (op_code) {
+            case GLUSTERD_MGMT_CLUSTER_LOCK: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Locking failed on %s. %s", peer_str, err_str);
+                break;
+            }
+            case GLUSTERD_MGMT_CLUSTER_UNLOCK: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Unlocking failed on %s. %s", peer_str, err_str);
+                break;
+            }
+            case GLUSTERD_MGMT_STAGE_OP: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Staging failed on %s. %s", peer_str, err_str);
+                break;
+            }
+            case GLUSTERD_MGMT_COMMIT_OP: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Commit failed on %s. %s", peer_str, err_str);
+                break;
+            }
+        }
 
-        return;
+        if (len > 0)
+            op_err[len] = '\0';
+
+        if (args->errstr) {
+            len = snprintf(err_str, sizeof(err_str) - 1, "%s\n%s", args->errstr,
+                           op_err);
+            GF_FREE(args->errstr);
+            args->errstr = NULL;
+        } else
+            len = snprintf(err_str, sizeof(err_str) - 1, "%s", op_err);
+        err_str[len] = '\0';
+
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MGMT_OP_FAIL, "%s", op_err);
+        args->errstr = gf_strdup(err_str);
+    }
+
+    GF_FREE(peer_str);
+
+    return;
 }
 
 void
-gd_syncargs_init (struct syncargs *args, dict_t *op_ctx)
+gd_syncargs_init(struct syncargs *args, dict_t *op_ctx)
 {
-        args->dict = op_ctx;
-        pthread_mutex_init (&args->lock_dict, NULL);
+    args->dict = op_ctx;
+    pthread_mutex_init(&args->lock_dict, NULL);
 }
 
 static void
-gd_stage_op_req_free (gd1_mgmt_stage_op_req *req)
+gd_stage_op_req_free(gd1_mgmt_stage_op_req *req)
 {
-        if (!req)
-                return;
+    if (!req)
+        return;
 
-        GF_FREE (req->buf.buf_val);
-        GF_FREE (req);
+    GF_FREE(req->buf.buf_val);
+    GF_FREE(req);
 }
 
 static void
-gd_commit_op_req_free (gd1_mgmt_commit_op_req *req)
+gd_commit_op_req_free(gd1_mgmt_commit_op_req *req)
 {
-        if (!req)
-                return;
+    if (!req)
+        return;
 
-        GF_FREE (req->buf.buf_val);
-        GF_FREE (req);
+    GF_FREE(req->buf.buf_val);
+    GF_FREE(req);
 }
 
 static void
-gd_brick_op_req_free (gd1_mgmt_brick_op_req *req)
+gd_brick_op_req_free(gd1_mgmt_brick_op_req *req)
 {
-        if (!req)
-                return;
+    if (!req)
+        return;
 
-        if (strcmp (req->name, "") != 0)
-                GF_FREE (req->name);
-        GF_FREE (req->input.input_val);
-        GF_FREE (req);
+    if (req->dict.dict_val)
+        GF_FREE(req->dict.dict_val);
+    GF_FREE(req->input.input_val);
+    GF_FREE(req);
 }
 
 int
-gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
-                          void *cookie, rpc_clnt_prog_t *prog, int procnum,
-                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+gd_syncop_submit_request(struct rpc_clnt *rpc, void *req, void *local,
+                         void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int            ret      = -1;
-        struct iobuf  *iobuf    = NULL;
-        struct iobref *iobref   = NULL;
-        int            count    = 0;
-        struct iovec   iov      = {0, };
-        ssize_t        req_size = 0;
-        call_frame_t  *frame    = NULL;
-
-        GF_ASSERT (rpc);
-        if (!req)
-                goto out;
+    int ret = -1;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t req_size = 0;
+    call_frame_t *frame = NULL;
 
-        req_size = xdr_sizeof (xdrproc, req);
-        iobuf = iobuf_get2 (rpc->ctx->iobuf_pool, req_size);
-        if (!iobuf)
-                goto out;
+    GF_ASSERT(rpc);
+    if (!req)
+        goto out;
 
-        iobref = iobref_new ();
-        if (!iobref)
-                goto out;
+    req_size = xdr_sizeof(xdrproc, req);
+    iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size);
+    if (!iobuf)
+        goto out;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    iobref = iobref_new();
+    if (!iobref)
+        goto out;
 
-        iobref_add (iobref, iobuf);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        iov.iov_base = iobuf->ptr;
-        iov.iov_len  = iobuf_pagesize (iobuf);
+    iobref_add(iobref, iobuf);
 
-        /* Create the xdr payload */
-        ret = xdr_serialize_generic (iov, req, xdrproc);
-        if (ret == -1)
-                goto out;
+    iov.iov_base = iobuf->ptr;
+    iov.iov_len = iobuf_pagesize(iobuf);
+
+    /* Create the xdr payload */
+    ret = xdr_serialize_generic(iov, req, xdrproc);
+    if (ret == -1)
+        goto out;
 
-        iov.iov_len = ret;
-        count = 1;
+    iov.iov_len = ret;
+    count = 1;
 
-        frame->local = local;
-        frame->cookie = cookie;
+    frame->local = local;
+    frame->cookie = cookie;
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
-                               &iov, count, NULL, 0, iobref,
-                               frame, NULL, 0, NULL, 0, NULL);
+    /* Send the msg */
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
-        /* TODO: do we need to start ping also? */
+    /* TODO: do we need to start ping also? */
 
 out:
-        iobref_unref (iobref);
-        iobuf_unref (iobuf);
+    iobref_unref(iobref);
+    iobuf_unref(iobuf);
 
-        if (ret && frame)
-                STACK_DESTROY (frame->root);
-        return ret;
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
+    return ret;
 }
 
 /* Defined in glusterd-rpc-ops.c */
@@ -226,1753 +218,1826 @@ extern struct rpc_clnt_program gd_brick_prog;
 extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
 int
-glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+glusterd_syncop_aggr_rsp_dict(glusterd_op_t op, dict_t *aggr, dict_t *rsp)
 {
-        int        ret  = 0;
-        xlator_t  *this = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_CREATE_VOLUME:
         case GD_OP_ADD_BRICK:
         case GD_OP_START_VOLUME:
-                ret = glusterd_aggr_brick_mount_dirs (aggr, rsp);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL, "Failed to "
-                                "aggregate brick mount dirs");
-                        goto out;
-                }
-        break;
+            ret = glusterd_aggr_brick_mount_dirs(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL,
+                       "Failed to "
+                       "aggregate brick mount dirs");
+                goto out;
+            }
+            break;
 
         case GD_OP_REPLACE_BRICK:
-                ret = glusterd_rb_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+        case GD_OP_RESET_BRICK:
+            ret = glusterd_rb_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_SYNC_VOLUME:
-                ret = glusterd_sync_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_sync_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_GSYNC_CREATE:
-        break;
+            break;
 
         case GD_OP_GSYNC_SET:
-                ret = glusterd_gsync_use_rsp_dict (aggr, rsp, NULL);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_gsync_use_rsp_dict(aggr, rsp, NULL);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_STATUS_VOLUME:
-                ret = glusterd_volume_status_copy_to_op_ctx_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
-
+            ret = glusterd_volume_status_copy_to_op_ctx_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_HEAL_VOLUME:
-                ret = glusterd_volume_heal_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
+            ret = glusterd_volume_heal_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
 
-        break;
+            break;
 
         case GD_OP_CLEARLOCKS_VOLUME:
-                ret = glusterd_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_QUOTA:
-                ret = glusterd_volume_quota_copy_to_op_ctx_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-                break;
+            ret = glusterd_volume_quota_copy_to_op_ctx_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_SYS_EXEC:
-                ret = glusterd_sys_exec_output_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_sys_exec_output_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_SNAP:
-                ret = glusterd_snap_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_snap_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_SCRUB_STATUS:
-                ret = glusterd_volume_bitrot_scrub_use_rsp_dict (aggr, rsp);
-        break;
-        default:
-        break;
-        }
-out:
-        return ret;
-}
+            ret = glusterd_volume_bitrot_scrub_use_rsp_dict(aggr, rsp);
+            break;
 
-int32_t
-gd_syncop_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                               int count, void *myframe)
-{
-        int                         ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_lock_rsp        rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int                         op_ret        = -1;
-        int                         op_errno      = -1;
-        xlator_t                   *this          = NULL;
-        uuid_t                     *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT(req);
-        GF_ASSERT(myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
+        case GD_OP_SCRUB_ONDEMAND:
+            break;
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
+        case GD_OP_MAX_OPVERSION:
+            ret = glusterd_max_opversion_use_rsp_dict(aggr, rsp);
+            break;
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
-        if (ret < 0)
-                goto out;
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_profile_volume_use_rsp_dict(aggr, rsp);
+            break;
 
-        gf_uuid_copy (args->uuid, rsp.uuid);
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_volume_rebalance_use_rsp_dict(aggr, rsp);
+            break;
 
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+        default:
+            break;
+    }
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                   GLUSTERD_MGMT_V3_LOCK, *peerid, rsp.uuid);
+    return ret;
+}
 
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+int32_t
+gd_syncop_mgmt_v3_lock_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                              void *myframe)
+{
+    int ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
+out:
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_LOCK, *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
+gd_syncop_mgmt_v3_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_syncop_mgmt_v3_lock_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_syncop_mgmt_v3_lock_cbk_fn);
 }
 
 int
-gd_syncop_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
-                        glusterd_peerinfo_t *peerinfo,
-                        struct syncargs *args, uuid_t my_uuid,
-                        uuid_t recv_uuid, uuid_t txn_id)
+gd_syncop_mgmt_v3_lock(glusterd_op_t op, dict_t *op_ctx,
+                       glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                       uuid_t my_uuid, uuid_t recv_uuid, uuid_t txn_id)
 {
-        int                      ret  = -1;
-        gd1_mgmt_v3_lock_req     req  = {{0},};
-        uuid_t                  *peerid = NULL;
-
-        GF_ASSERT(op_ctx);
-        GF_ASSERT(peerinfo);
-        GF_ASSERT(args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        gf_uuid_copy (req.txn_id, txn_id);
-        req.op = op;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_LOCK,
-                                        gd_syncop_mgmt_v3_lock_cbk,
-                                        (xdrproc_t)
-                                        xdr_gd1_mgmt_v3_lock_req);
+    int ret = -1;
+    gd1_mgmt_v3_lock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    gf_uuid_copy(req.txn_id, txn_id);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_LOCK,
+                                   gd_syncop_mgmt_v3_lock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-gd_syncop_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+gd_syncop_mgmt_v3_unlock_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        int                          ret           = -1;
-        struct syncargs             *args          = NULL;
-        glusterd_peerinfo_t         *peerinfo      = NULL;
-        gd1_mgmt_v3_unlock_rsp       rsp           = {{0},};
-        call_frame_t                *frame         = NULL;
-        int                          op_ret        = -1;
-        int                          op_errno      = -1;
-        xlator_t                    *this          = NULL;
-        uuid_t                      *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT(req);
-        GF_ASSERT(myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    int ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                   GLUSTERD_MGMT_V3_UNLOCK, *peerid, rsp.uuid);
-
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_UNLOCK, *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+gd_syncop_mgmt_v3_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_syncop_mgmt_v3_unlock_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_syncop_mgmt_v3_unlock_cbk_fn);
 }
 
 int
-gd_syncop_mgmt_v3_unlock (dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
-                          struct syncargs *args, uuid_t my_uuid,
-                          uuid_t recv_uuid, uuid_t txn_id)
+gd_syncop_mgmt_v3_unlock(dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                         struct syncargs *args, uuid_t my_uuid,
+                         uuid_t recv_uuid, uuid_t txn_id)
 {
-        int                          ret  = -1;
-        gd1_mgmt_v3_unlock_req       req  = {{0},};
-        uuid_t                      *peerid = NULL;
-
-        GF_ASSERT(op_ctx);
-        GF_ASSERT(peerinfo);
-        GF_ASSERT(args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        gf_uuid_copy (req.txn_id, txn_id);
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_UNLOCK,
-                                        gd_syncop_mgmt_v3_unlock_cbk,
-                                        (xdrproc_t)
-                                        xdr_gd1_mgmt_v3_unlock_req);
+    int ret = -1;
+    gd1_mgmt_v3_unlock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    gf_uuid_copy(req.txn_id, txn_id);
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_UNLOCK,
+                                   gd_syncop_mgmt_v3_unlock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_msg_debug ("glusterd", 0, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-_gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+_gd_syncop_mgmt_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int                         ret             = -1;
-        struct syncargs             *args           = NULL;
-        glusterd_peerinfo_t         *peerinfo       = NULL;
-        gd1_mgmt_cluster_lock_rsp   rsp             = {{0},};
-        call_frame_t                *frame          = NULL;
-        int                         op_ret          = -1;
-        int                         op_errno        = -1;
-        xlator_t                    *this           = NULL;
-        uuid_t                      *peerid         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (*peerid, NULL);
-        if (peerinfo) {
-                /* Set peer as locked, so we unlock only the locked peers */
-                if (rsp.op_ret == 0)
-                        peerinfo->locked = _gf_true;
-        } else {
-                rsp.op_ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_PEER_NOT_FOUND,
-                        "Could not find peer with "
-                        "ID %s", uuid_utoa (*peerid));
-        }
-        rcu_read_unlock ();
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    int ret = -1;
+    struct syncargs *args = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(*peerid, NULL);
+    if (peerinfo) {
+        /* Set peer as locked, so we unlock only the locked peers */
+        if (rsp.op_ret == 0)
+            peerinfo->locked = _gf_true;
+        RCU_READ_UNLOCK;
+    } else {
+        RCU_READ_UNLOCK;
+        rsp.op_ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer with "
+               "ID %s",
+               uuid_utoa(*peerid));
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 out:
-        gd_collate_errors (args, op_ret, op_errno, NULL,
-                           GLUSTERD_MGMT_CLUSTER_LOCK, *peerid, rsp.uuid);
-
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, NULL, GLUSTERD_MGMT_CLUSTER_LOCK,
+                      *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void *myframe)
+gd_syncop_mgmt_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_mgmt_lock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_mgmt_lock_cbk);
 }
 
 int
-gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                     uuid_t my_uuid, uuid_t recv_uuid)
+gd_syncop_mgmt_lock(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                    uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int                       ret = -1;
-        gd1_mgmt_cluster_lock_req req  = {{0},};
-        uuid_t                    *peerid = NULL;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_prog,
-                                        GLUSTERD_MGMT_CLUSTER_LOCK,
-                                        gd_syncop_mgmt_lock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+    int ret = -1;
+    gd1_mgmt_cluster_lock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_prog, GLUSTERD_MGMT_CLUSTER_LOCK,
+                                   gd_syncop_mgmt_lock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-_gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
+_gd_syncop_mgmt_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        int                          ret           = -1;
-        struct syncargs             *args          = NULL;
-        glusterd_peerinfo_t         *peerinfo      = NULL;
-        gd1_mgmt_cluster_unlock_rsp  rsp           = {{0},};
-        call_frame_t                *frame         = NULL;
-        int                          op_ret        = -1;
-        int                          op_errno      = -1;
-        xlator_t                    *this          = NULL;
-        uuid_t                      *peerid        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args  = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (*peerid, NULL);
-        if (peerinfo) {
-                peerinfo->locked = _gf_false;
-        } else {
-                rsp.op_ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_PEER_NOT_FOUND, "Could not find peer with "
-                        "ID %s", uuid_utoa (*peerid));
-        }
-        rcu_read_unlock ();
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    int ret = -1;
+    struct syncargs *args = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gd1_mgmt_cluster_unlock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(*peerid, NULL);
+    if (peerinfo) {
+        peerinfo->locked = _gf_false;
+        RCU_READ_UNLOCK;
+    } else {
+        RCU_READ_UNLOCK;
+        rsp.op_ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer with "
+               "ID %s",
+               uuid_utoa(*peerid));
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 out:
-        gd_collate_errors (args, op_ret, op_errno, NULL,
-                           GLUSTERD_MGMT_CLUSTER_UNLOCK, *peerid, rsp.uuid);
-
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, NULL,
+                      GLUSTERD_MGMT_CLUSTER_UNLOCK, *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
+gd_syncop_mgmt_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_mgmt_unlock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_mgmt_unlock_cbk);
 }
 
-
 int
-gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                       uuid_t my_uuid, uuid_t recv_uuid)
+gd_syncop_mgmt_unlock(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                      uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int                         ret     = -1;
-        gd1_mgmt_cluster_unlock_req req     = {{0},};
-        uuid_t                      *peerid = NULL;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerid,
-                                        &gd_mgmt_prog,
-                                        GLUSTERD_MGMT_CLUSTER_UNLOCK,
-                                        gd_syncop_mgmt_unlock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
+    int ret = -1;
+    gd1_mgmt_cluster_unlock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_prog, GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                   gd_syncop_mgmt_unlock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-_gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+_gd_syncop_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        int                         ret           = -1;
-        gd1_mgmt_stage_op_rsp       rsp           = {{0},};
-        struct syncargs             *args         = NULL;
-        xlator_t                    *this         = NULL;
-        dict_t                      *rsp_dict     = NULL;
-        call_frame_t                *frame        = NULL;
-        int                         op_ret        = -1;
-        int                         op_errno      = -1;
-        uuid_t                      *peerid       = NULL;
-
-        this  = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args  = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        GF_FREE (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_RESP_FROM_UNKNOWN_PEER, "Staging response "
-                        "for 'Volume %s' received from unknown "
-                        "peer: %s", gd_op_list[rsp.op],
-                        uuid_utoa (rsp.uuid));
-                goto out;
+    int ret = -1;
+    gd1_mgmt_stage_op_rsp rsp = {
+        {0},
+    };
+    struct syncargs *args = NULL;
+    xlator_t *this = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            GF_FREE(rsp.dict.dict_val);
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
         }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        if (rsp.op == GD_OP_REPLACE_BRICK || rsp.op == GD_OP_QUOTA ||
-            rsp.op == GD_OP_CREATE_VOLUME || rsp.op == GD_OP_ADD_BRICK ||
-            rsp.op == GD_OP_START_VOLUME) {
-                pthread_mutex_lock (&args->lock_dict);
-                {
-                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
-                                                             rsp_dict);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_RESP_AGGR_FAIL, "%s",
-                                        "Failed to aggregate response from "
-                                        " node/brick");
-                }
-                pthread_mutex_unlock (&args->lock_dict);
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Staging response "
+               "for 'Volume %s' received from unknown "
+               "peer: %s",
+               gd_op_list[rsp.op], uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    if (rsp.op == GD_OP_REPLACE_BRICK || rsp.op == GD_OP_QUOTA ||
+        rsp.op == GD_OP_CREATE_VOLUME || rsp.op == GD_OP_ADD_BRICK ||
+        rsp.op == GD_OP_START_VOLUME) {
+        pthread_mutex_lock(&args->lock_dict);
+        {
+            ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                       "Failed to aggregate response from "
+                       " node/brick");
         }
+        pthread_mutex_unlock(&args->lock_dict);
+    }
 
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
 out:
-        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                           GLUSTERD_MGMT_STAGE_OP, *peerid, rsp.uuid);
-
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                      GLUSTERD_MGMT_STAGE_OP, *peerid, rsp.uuid);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+gd_syncop_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_stage_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_stage_op_cbk);
 }
 
-
 int
-gd_syncop_mgmt_stage_op (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                         uuid_t my_uuid, uuid_t recv_uuid, int op,
-                         dict_t *dict_out, dict_t *op_ctx)
+gd_syncop_mgmt_stage_op(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                        uuid_t my_uuid, uuid_t recv_uuid, int op,
+                        dict_t *dict_out, dict_t *op_ctx)
 {
-        gd1_mgmt_stage_op_req *req  = NULL;
-        int                   ret   = -1;
-        uuid_t                *peerid = NULL;
-
-        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_stage_req_t);
-        if (!req)
-                goto out;
-
-        gf_uuid_copy (req->uuid, my_uuid);
-        req->op = op;
-
-        ret = dict_allocate_and_serialize (dict_out,
-                                           &req->buf.buf_val, &req->buf.buf_len);
-        if (ret)
-                goto out;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, req, args, peerid,
-                                        &gd_mgmt_prog, GLUSTERD_MGMT_STAGE_OP,
-                                        gd_syncop_stage_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_stage_op_req);
+    gd1_mgmt_stage_op_req *req = NULL;
+    int ret = -1;
+    uuid_t *peerid = NULL;
+
+    req = GF_CALLOC(1, sizeof(*req), gf_gld_mt_mop_stage_req_t);
+    if (!req) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req->uuid, my_uuid);
+    req->op = op;
+
+    ret = dict_allocate_and_serialize(dict_out, &req->buf.buf_val,
+                                      &req->buf.buf_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, req, args, peerid, &gd_mgmt_prog, GLUSTERD_MGMT_STAGE_OP,
+        gd_syncop_stage_op_cbk, (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
 out:
-        gd_stage_op_req_free (req);
-        return ret;
-
+    gd_stage_op_req_free(req);
+    return ret;
 }
 
 int32_t
-_gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+_gd_syncop_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        struct syncargs        *args  = NULL;
-        gd1_mgmt_brick_op_rsp  rsp   = {0,};
-        int                    ret   = -1;
-        call_frame_t           *frame = NULL;
-        xlator_t               *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args = frame->local;
-        frame->local = NULL;
-
-        /* initialize */
-        args->op_ret   = -1;
-        args->op_errno = EINVAL;
-
-        if (-1 == req->rpc_status) {
-                args->op_errno = ENOTCONN;
-                goto out;
+    struct syncargs *args = NULL;
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    frame->local = NULL;
+
+    /* initialize */
+    args->op_ret = -1;
+    args->op_errno = EINVAL;
+
+    if (-1 == req->rpc_status) {
+        args->op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, args->op_errno,
+                                   EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.output.output_len) {
+        args->dict = dict_new();
+        if (!args->dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -1;
+            args->op_errno = ENOMEM;
+            goto out;
         }
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, args->op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.output.output_len) {
-                args->dict  = dict_new ();
-                if (!args->dict) {
-                        ret = -1;
-                        args->op_errno = ENOMEM;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.output.output_val,
-                                        rsp.output.output_len,
-                                        &args->dict);
-                if (ret < 0)
-                        goto out;
+        ret = dict_unserialize(rsp.output.output_val, rsp.output.output_len,
+                               &args->dict);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_UNSERIALIZE_FAIL, NULL);
+            goto out;
         }
+    }
 
-        args->op_ret = rsp.op_ret;
-        args->op_errno = rsp.op_errno;
-        args->errstr = gf_strdup (rsp.op_errstr);
+    args->op_ret = rsp.op_ret;
+    args->op_errno = rsp.op_errno;
+    args->errstr = gf_strdup(rsp.op_errstr);
 
 out:
-        if ((rsp.op_errstr) && (strcmp (rsp.op_errstr, "") != 0))
-                free (rsp.op_errstr);
-        free (rsp.output.output_val);
-
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        __wake (args);
-
-        return 0;
+    if ((rsp.op_errstr) && (strcmp(rsp.op_errstr, "") != 0))
+        free(rsp.op_errstr);
+    free(rsp.output.output_val);
+
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    __wake(args);
+
+    return 0;
 }
 
 int32_t
-gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+gd_syncop_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_brick_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_brick_op_cbk);
 }
 
 int
-gd_syncop_mgmt_brick_op (struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
-                         int op, dict_t *dict_out, dict_t *op_ctx,
-                         char **errstr)
+gd_syncop_mgmt_brick_op(struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
+                        int op, dict_t *dict_out, dict_t *op_ctx, char **errstr)
 {
-        struct syncargs        args = {0, };
-        gd1_mgmt_brick_op_req  *req  = NULL;
-        int                    ret  = 0;
-        xlator_t               *this = NULL;
-
-        this = THIS;
-        args.op_ret = -1;
-        args.op_errno = ENOTCONN;
-
-        if ((pnode->type == GD_NODE_NFS) ||
-            (pnode->type == GD_NODE_QUOTAD) || (pnode->type == GD_NODE_SCRUB) ||
-            ((pnode->type == GD_NODE_SHD) && (op == GD_OP_STATUS_VOLUME))) {
-                ret = glusterd_node_op_build_payload (op, &req, dict_out);
-
-        } else {
-                ret = glusterd_brick_op_build_payload (op, pnode->node, &req,
-                                                       dict_out);
+    struct syncargs args = {
+        0,
+    };
+    gd1_mgmt_brick_op_req *req = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    args.op_ret = -1;
+    args.op_errno = ENOTCONN;
+
+    if ((pnode->type == GD_NODE_NFS) || (pnode->type == GD_NODE_QUOTAD) ||
+        (pnode->type == GD_NODE_SCRUB) ||
+        ((pnode->type == GD_NODE_SHD) && (op == GD_OP_STATUS_VOLUME))) {
+        ret = glusterd_node_op_build_payload(op, &req, dict_out);
+
+    } else {
+        ret = glusterd_brick_op_build_payload(op, pnode->node, &req, dict_out);
+    }
+
+    if (ret)
+        goto out;
+
+    GD_SYNCOP(rpc, (&args), NULL, gd_syncop_brick_op_cbk, req, &gd_brick_prog,
+              req->op, xdr_gd1_mgmt_brick_op_req);
+
+    if (args.errstr) {
+        if ((strlen(args.errstr) > 0) && errstr)
+            *errstr = args.errstr;
+        else
+            GF_FREE(args.errstr);
+    }
 
+    if (GD_OP_STATUS_VOLUME == op) {
+        ret = dict_set_int32(args.dict, "index", pnode->index);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Error setting index on brick status"
+                   " rsp dict");
+            args.op_ret = -1;
+            goto out;
         }
-
-        if (ret)
-                goto out;
-
-        GD_SYNCOP (rpc, (&args), NULL, gd_syncop_brick_op_cbk, req,
-                   &gd_brick_prog, req->op, xdr_gd1_mgmt_brick_op_req);
-
-        if (args.errstr) {
-                if ((strlen(args.errstr) > 0) && errstr)
-                        *errstr = args.errstr;
-                else
-                        GF_FREE (args.errstr);
+    }
+
+    if (req->op == GLUSTERD_BRICK_TERMINATE) {
+        if (args.op_ret && (args.op_errno == ENOTCONN)) {
+            /*
+             * This is actually OK.  It happens when the target
+             * brick process exits and we saw the closed connection
+             * before we read the response.  If we didn't read the
+             * response quickly enough that's kind of our own
+             * fault, and the fact that the process exited means
+             * that our goal of terminating the brick was achieved.
+             */
+            args.op_ret = 0;
         }
+    }
 
-        if (GD_OP_STATUS_VOLUME == op) {
-                ret = dict_set_int32 (args.dict, "index", pnode->index);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_SET_FAILED,
-                                "Error setting index on brick status"
-                                " rsp dict");
-                        args.op_ret = -1;
-                        goto out;
-                }
-        }
-        if (args.op_ret == 0)
-                glusterd_handle_node_rsp (dict_out, pnode->node, op,
-                                          args.dict, op_ctx, errstr,
-                                          pnode->type);
+    if (args.op_ret == 0)
+        glusterd_handle_node_rsp(dict_out, pnode->node, op, args.dict, op_ctx,
+                                 errstr, pnode->type);
 
 out:
-        errno = args.op_errno;
-        if (args.dict)
-                dict_unref (args.dict);
-        gd_brick_op_req_free (req);
-        return args.op_ret;
-
+    errno = args.op_errno;
+    if (args.dict)
+        dict_unref(args.dict);
+    if (args.op_ret && errstr && (*errstr == NULL)) {
+        if (op == GD_OP_HEAL_VOLUME) {
+            gf_asprintf(errstr,
+                        "Glusterd Syncop Mgmt brick op '%s' failed."
+                        " Please check glustershd log file for details.",
+                        gd_op_list[op]);
+        } else {
+            gf_asprintf(errstr,
+                        "Glusterd Syncop Mgmt brick op '%s' failed."
+                        " Please check brick log file for details.",
+                        gd_op_list[op]);
+        }
+    }
+    gd_brick_op_req_free(req);
+    return args.op_ret;
 }
 
 int32_t
-_gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+_gd_syncop_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int                     ret               = -1;
-        gd1_mgmt_commit_op_rsp  rsp               = {{0},};
-        struct syncargs        *args              = NULL;
-        xlator_t               *this              = NULL;
-        dict_t                 *rsp_dict          = NULL;
-        call_frame_t           *frame             = NULL;
-        int                     op_ret            = -1;
-        int                     op_errno          = -1;
-        int                     type              = GF_QUOTA_OPTION_TYPE_NONE;
-        uuid_t                 *peerid            = NULL;
-
-        this  = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args  = frame->local;
-        peerid = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    int ret = -1;
+    gd1_mgmt_commit_op_rsp rsp = {
+        {0},
+    };
+    struct syncargs *args = NULL;
+    xlator_t *this = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    int type = GF_QUOTA_OPTION_TYPE_NONE;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    if (ret < 0) {
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret < 0) {
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        GF_FREE (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
+            GF_FREE(rsp.dict.dict_val);
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
         }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == 0);
-        rcu_read_unlock ();
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == 0);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Commit response "
+               "for 'Volume %s' received from unknown "
+               "peer: %s",
+               gd_op_list[rsp.op], uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    if (rsp.op == GD_OP_QUOTA) {
+        ret = dict_get_int32(args->dict, "type", &type);
         if (ret) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_RESP_FROM_UNKNOWN_PEER, "Commit response "
-                        "for 'Volume %s' received from unknown "
-                        "peer: %s", gd_op_list[rsp.op],
-                        uuid_utoa (rsp.uuid));
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "opcode");
+            goto out;
         }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        if (rsp.op == GD_OP_QUOTA) {
-                ret = dict_get_int32 (args->dict, "type", &type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to get "
-                                "opcode");
-                        goto out;
-                }
+    }
+
+    if ((rsp.op != GD_OP_QUOTA) || (type == GF_QUOTA_OPTION_TYPE_LIST)) {
+        pthread_mutex_lock(&args->lock_dict);
+        {
+            ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                       "Failed to aggregate response from "
+                       " node/brick");
         }
+        pthread_mutex_unlock(&args->lock_dict);
+    }
 
-        if ((rsp.op != GD_OP_QUOTA) || (type == GF_QUOTA_OPTION_TYPE_LIST)) {
-                pthread_mutex_lock (&args->lock_dict);
-                {
-                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
-                                                             rsp_dict);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_RESP_AGGR_FAIL, "%s",
-                                        "Failed to aggregate response from "
-                                        " node/brick");
-                }
-                pthread_mutex_unlock (&args->lock_dict);
-        }
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
 out:
-        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                           GLUSTERD_MGMT_COMMIT_OP, *peerid, rsp.uuid);
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-        GF_FREE (peerid);
-        /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
-         * the caller function.
-         */
-        if (req->rpc_status != -1)
-                STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                      GLUSTERD_MGMT_COMMIT_OP, *peerid, rsp.uuid);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+
+    return 0;
 }
 
 int32_t
-gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+gd_syncop_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_commit_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_commit_op_cbk);
 }
 
-
 int
-gd_syncop_mgmt_commit_op (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                          uuid_t my_uuid, uuid_t recv_uuid,
-                          int op, dict_t *dict_out, dict_t *op_ctx)
+gd_syncop_mgmt_commit_op(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                         uuid_t my_uuid, uuid_t recv_uuid, int op,
+                         dict_t *dict_out, dict_t *op_ctx)
 {
-        gd1_mgmt_commit_op_req *req  = NULL;
-        int                    ret  = -1;
-        uuid_t                 *peerid = NULL;
-
-        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_commit_req_t);
-        if (!req)
-                goto out;
-
-        gf_uuid_copy (req->uuid, my_uuid);
-        req->op = op;
-
-        ret = dict_allocate_and_serialize (dict_out,
-                                           &req->buf.buf_val, &req->buf.buf_len);
-        if (ret)
-                goto out;
-
-        GD_ALLOC_COPY_UUID (peerid, peerinfo->uuid, ret);
-        if (ret)
-                goto out;
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, req, args, peerid,
-                                        &gd_mgmt_prog, GLUSTERD_MGMT_COMMIT_OP,
-                                        gd_syncop_commit_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_commit_op_req);
+    gd1_mgmt_commit_op_req *req = NULL;
+    int ret = -1;
+    uuid_t *peerid = NULL;
+
+    req = GF_CALLOC(1, sizeof(*req), gf_gld_mt_mop_commit_req_t);
+    if (!req) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req->uuid, my_uuid);
+    req->op = op;
+
+    ret = dict_allocate_and_serialize(dict_out, &req->buf.buf_val,
+                                      &req->buf.buf_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, req, args, peerid,
+                                   &gd_mgmt_prog, GLUSTERD_MGMT_COMMIT_OP,
+                                   gd_syncop_commit_op_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
 out:
-        gd_commit_op_req_free (req);
-        return ret;
+    gd_commit_op_req_free(req);
+    return ret;
 }
 
-
 int
-gd_lock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, dict_t *op_ctx,
-                  char **op_errstr, uuid_t txn_id,
-                  glusterd_op_info_t *txn_opinfo, gf_boolean_t cluster_lock)
+gd_lock_op_phase(glusterd_conf_t *conf, glusterd_op_t op, dict_t *op_ctx,
+                 char **op_errstr, uuid_t txn_id,
+                 glusterd_op_info_t *txn_opinfo, gf_boolean_t cluster_lock)
 {
-        int                     ret         = -1;
-        int                     peer_cnt    = 0;
-        uuid_t                  peer_uuid   = {0};
-        xlator_t               *this        = NULL;
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-        struct syncargs         args        = {0};
-
-        this = THIS;
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_opinfo->txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-
-                if (cluster_lock) {
-                        /* Reset lock status */
-                        peerinfo->locked = _gf_false;
-                        gd_syncop_mgmt_lock (peerinfo, &args,
-                                             MY_UUID, peer_uuid);
-                } else
-                        gd_syncop_mgmt_v3_lock (op, op_ctx, peerinfo, &args,
-                                                MY_UUID, peer_uuid, txn_id);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
-
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
+    int ret = -1;
+    int peer_cnt = 0;
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_opinfo->txn_generation)
+            continue;
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
 
-        if (args.op_ret) {
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-                else {
-                        ret = gf_asprintf (op_errstr, "Another transaction "
-                                           "could be in progress. Please try "
-                                           "again after sometime.");
-                        if (ret == -1)
-                                *op_errstr = NULL;
+        if (cluster_lock) {
+            /* Reset lock status */
+            peerinfo->locked = _gf_false;
+            gd_syncop_mgmt_lock(peerinfo, &args, MY_UUID, peer_uuid);
+        } else
+            gd_syncop_mgmt_v3_lock(op, op_ctx, peerinfo, &args, MY_UUID,
+                                   peer_uuid, txn_id);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_PEER_LOCK_FAIL,
-                                "Failed to acquire lock");
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-                }
+    if (args.op_ret) {
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+        else {
+            ret = gf_asprintf(op_errstr,
+                              "Another transaction "
+                              "could be in progress. Please try "
+                              "again after some time.");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_LOCK_FAIL,
+                   "Failed to acquire lock");
         }
+    }
 
-        ret = args.op_ret;
+    ret = args.op_ret;
 
-        gf_msg_debug (this->name, 0, "Sent lock op req for 'Volume %s' "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent lock op req for 'Volume %s' "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int
-gd_stage_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
-                   char **op_errstr, glusterd_op_info_t *txn_opinfo)
+gd_stage_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                  char **op_errstr, glusterd_op_info_t *txn_opinfo)
 {
-        int                     ret             = -1;
-        int                     peer_cnt        = 0;
-        dict_t                 *rsp_dict        = NULL;
-        char                   *hostname        = NULL;
-        xlator_t               *this            = NULL;
-        glusterd_conf_t        *conf            = NULL;
-        glusterd_peerinfo_t    *peerinfo        = NULL;
-        uuid_t                  tmp_uuid        = {0};
-        char                   *errstr          = NULL;
-        struct syncargs         args            = {0};
-        dict_t                 *aggr_dict       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict)
-                goto out;
-
-        if ((op == GD_OP_CREATE_VOLUME) || (op == GD_OP_ADD_BRICK) ||
-            (op == GD_OP_START_VOLUME))
-                aggr_dict = req_dict;
-        else
-                aggr_dict = op_ctx;
-
-        ret = glusterd_validate_quorum (this, op, req_dict, op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET,
-                        "Server quorum not met. Rejecting operation.");
-                goto out;
-        }
-
-        ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
+    int ret = -1;
+    int peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    char *hostname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    uuid_t tmp_uuid = {0};
+    char *errstr = NULL;
+    struct syncargs args = {0};
+    dict_t *aggr_dict = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    if ((op == GD_OP_CREATE_VOLUME) || (op == GD_OP_ADD_BRICK) ||
+        (op == GD_OP_START_VOLUME))
+        aggr_dict = req_dict;
+    else
+        aggr_dict = op_ctx;
+
+    ret = glusterd_validate_quorum(this, op, req_dict, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        goto out;
+    }
+
+    ret = glusterd_op_stage_validate(op, req_dict, op_errstr, rsp_dict);
+    if (ret) {
+        hostname = "localhost";
+        goto stage_done;
+    }
+
+    if ((op == GD_OP_REPLACE_BRICK || op == GD_OP_QUOTA ||
+         op == GD_OP_CREATE_VOLUME || op == GD_OP_ADD_BRICK ||
+         op == GD_OP_START_VOLUME)) {
+        ret = glusterd_syncop_aggr_rsp_dict(op, aggr_dict, rsp_dict);
         if (ret) {
-                hostname = "localhost";
-                goto stage_done;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                   "Failed to aggregate response from node/brick");
+            goto out;
         }
-
-        if ((op == GD_OP_REPLACE_BRICK || op == GD_OP_QUOTA ||
-             op == GD_OP_CREATE_VOLUME || op == GD_OP_ADD_BRICK ||
-             op == GD_OP_START_VOLUME)) {
-                ret = glusterd_syncop_aggr_rsp_dict (op, aggr_dict, rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_RESP_AGGR_FAIL, "%s",
-                                "Failed to aggregate response from node/brick");
-                        goto out;
-                }
-        }
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
+    }
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
 
 stage_done:
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_VALIDATE_FAILED, LOGSTR_STAGE_FAIL,
-                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
-                        (*op_errstr) ? *op_errstr : " ");
-                if (*op_errstr == NULL)
-                        gf_asprintf (op_errstr, OPERRSTR_STAGE_FAIL, hostname);
-                goto out;
-        }
-
-        gd_syncargs_init (&args, aggr_dict);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                /* Only send requests to peers who were available before the
-                 * transaction started
-                 */
-                if (peerinfo->generation > txn_opinfo->txn_generation)
-                        continue;
-
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                ret = gd_syncop_mgmt_stage_op (peerinfo, &args,
-                                               MY_UUID, tmp_uuid,
-                                               op, req_dict, op_ctx);
-                peer_cnt++;
-        }
-        rcu_read_unlock ();
-
-        if (0 == peer_cnt) {
-                ret = 0;
-                goto out;
-        }
-
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               LOGSTR_STAGE_FAIL, gd_op_list[op], hostname,
+               (*op_errstr) ? ":" : " ", (*op_errstr) ? *op_errstr : " ");
+        if (*op_errstr == NULL)
+            gf_asprintf(op_errstr, OPERRSTR_STAGE_FAIL, hostname);
+        goto out;
+    }
+
+    gd_syncargs_init(&args, aggr_dict);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_opinfo->txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        (void)gd_syncop_mgmt_stage_op(peerinfo, &args, MY_UUID, tmp_uuid, op,
+                                      req_dict, op_ctx);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        gf_msg_debug (this->name, 0, "Sent stage op req for 'Volume %s' "
-                "to %d peers", gd_op_list[op], peer_cnt);
+    gf_msg_debug(this->name, 0,
+                 "Sent stage op req for 'Volume %s' "
+                 "to %d peers",
+                 gd_op_list[op], peer_cnt);
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (args.errstr)
-                 *op_errstr = gf_strdup (args.errstr);
-        else if (dict_get_str (aggr_dict, "errstr", &errstr) == 0)
-                *op_errstr = gf_strdup (errstr);
+    if (args.errstr)
+        *op_errstr = gf_strdup(args.errstr);
+    else if (dict_get_str(aggr_dict, "errstr", &errstr) == 0)
+        *op_errstr = gf_strdup(errstr);
 
-        ret = args.op_ret;
+    ret = args.op_ret;
 
 out:
-        if ((ret == 0) && (op == GD_OP_QUOTA)) {
-                ret = glusterd_validate_and_set_gfid (op_ctx, req_dict,
-                                                      op_errstr);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_GFID_VALIDATE_SET_FAIL,
-                                "Failed to validate and set gfid");
-        }
+    if ((ret == 0) && (op == GD_OP_QUOTA)) {
+        ret = glusterd_validate_and_set_gfid(op_ctx, req_dict, op_errstr);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GFID_VALIDATE_SET_FAIL,
+                   "Failed to validate and set gfid");
+    }
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
 }
 
 int
-gd_commit_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
-                    char **op_errstr, glusterd_op_info_t *txn_opinfo)
+gd_commit_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr, glusterd_op_info_t *txn_opinfo)
 {
-        dict_t                 *rsp_dict      = NULL;
-        int                     peer_cnt      = -1;
-        int                     ret           = -1;
-        char                   *hostname      = NULL;
-        glusterd_peerinfo_t    *peerinfo      = NULL;
-        xlator_t               *this          = NULL;
-        glusterd_conf_t        *conf          = NULL;
-        uuid_t                  tmp_uuid      = {0};
-        char                   *errstr        = NULL;
-        struct syncargs         args          = {0};
-        int                     type          = GF_QUOTA_OPTION_TYPE_NONE;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_op_commit_perform (op, req_dict, op_errstr, rsp_dict);
+    dict_t *rsp_dict = NULL;
+    int peer_cnt = -1;
+    int ret = -1;
+    char *hostname = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    uuid_t tmp_uuid = {0};
+    char *errstr = NULL;
+    struct syncargs args = {0};
+    int type = GF_QUOTA_OPTION_TYPE_NONE;
+    uint32_t cmd = 0;
+    gf_boolean_t origin_glusterd = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_op_commit_perform(op, req_dict, op_errstr, rsp_dict);
+    if (ret) {
+        hostname = "localhost";
+        goto commit_done;
+    }
+
+    if (op == GD_OP_QUOTA) {
+        ret = dict_get_int32(op_ctx, "type", &type);
         if (ret) {
-                hostname = "localhost";
-                goto commit_done;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "opcode");
+            goto out;
         }
+    }
 
-        if (op == GD_OP_QUOTA) {
-                ret = dict_get_int32 (op_ctx, "type", &type);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_DICT_GET_FAILED, "Failed to get "
-                                "opcode");
-                        goto out;
-                }
-        }
-
-        if (((op == GD_OP_QUOTA) && ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
-             (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS))) ||
-            ((op != GD_OP_SYNC_VOLUME) && (op != GD_OP_QUOTA))) {
-
-                ret =  glusterd_syncop_aggr_rsp_dict (op, op_ctx,
-                                                      rsp_dict);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_RESP_AGGR_FAIL, "%s",
-                                "Failed to aggregate "
-                                "response from node/brick");
-                        goto out;
-                }
+    if (((op == GD_OP_QUOTA) &&
+         ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+          (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS))) ||
+        ((op != GD_OP_SYNC_VOLUME) && (op != GD_OP_QUOTA))) {
+        ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                   "Failed to aggregate "
+                   "response from node/brick");
+            goto out;
         }
+    }
 
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
+    dict_unref(rsp_dict);